drivers/gpu/drm/i915/display/intel_display.c

   1 /*
   2  * Copyright © 2006-2007 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  21  * DEALINGS IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *      Eric Anholt <eric@anholt.net>
  25  */
  26
  27 #include <acpi/video.h>
  28 #include <linux/i2c.h>
  29 #include <linux/input.h>
  30 #include <linux/intel-iommu.h>
  31 #include <linux/kernel.h>
  32 #include <linux/module.h>
  33 #include <linux/dma-resv.h>
  34 #include <linux/slab.h>
  35
  36 #include <drm/drm_atomic.h>
  37 #include <drm/drm_atomic_helper.h>
  38 #include <drm/drm_atomic_uapi.h>
  39 #include <drm/drm_damage_helper.h>
  40 #include <drm/drm_dp_helper.h>
  41 #include <drm/drm_edid.h>
  42 #include <drm/drm_fourcc.h>
  43 #include <drm/drm_plane_helper.h>
  44 #include <drm/drm_probe_helper.h>
  45 #include <drm/drm_rect.h>
  46
  47 #include "display/intel_audio.h"
  48 #include "display/intel_crt.h"
  49 #include "display/intel_ddi.h"
  50 #include "display/intel_display_debugfs.h"
  51 #include "display/intel_dp.h"
  52 #include "display/intel_dp_mst.h"
  53 #include "display/intel_dpll.h"
  54 #include "display/intel_dpll_mgr.h"
  55 #include "display/intel_drrs.h"
  56 #include "display/intel_dsi.h"
  57 #include "display/intel_dvo.h"
  58 #include "display/intel_fb.h"
  59 #include "display/intel_gmbus.h"
  60 #include "display/intel_hdmi.h"
  61 #include "display/intel_lvds.h"
  62 #include "display/intel_sdvo.h"
  63 #include "display/intel_snps_phy.h"
  64 #include "display/intel_tv.h"
  65 #include "display/intel_vdsc.h"
  66 #include "display/intel_vrr.h"
  67
  68 #include "gem/i915_gem_lmem.h"
  69 #include "gem/i915_gem_object.h"
  70
  71 #include "gt/intel_rps.h"
  72 #include "gt/gen8_ppgtt.h"
  73
  74 #include "g4x_dp.h"
  75 #include "g4x_hdmi.h"
  76 #include "i915_drv.h"
  77 #include "intel_acpi.h"
  78 #include "intel_atomic.h"
  79 #include "intel_atomic_plane.h"
  80 #include "intel_bw.h"
  81 #include "intel_cdclk.h"
  82 #include "intel_color.h"
  83 #include "intel_crtc.h"
  84 #include "intel_de.h"
  85 #include "intel_display_types.h"
  86 #include "intel_dmc.h"
  87 #include "intel_dp_link_training.h"
  88 #include "intel_dpt.h"
  89 #include "intel_fbc.h"
  90 #include "intel_fdi.h"
  91 #include "intel_fbdev.h"
  92 #include "intel_fifo_underrun.h"
  93 #include "intel_frontbuffer.h"
  94 #include "intel_hdcp.h"
  95 #include "intel_hotplug.h"
  96 #include "intel_overlay.h"
  97 #include "intel_panel.h"
  98 #include "intel_pipe_crc.h"
  99 #include "intel_pm.h"
 100 #include "intel_pps.h"
 101 #include "intel_psr.h"
 102 #include "intel_quirks.h"
 103 #include "intel_sideband.h"
 104 #include "intel_sprite.h"
 105 #include "intel_tc.h"
 106 #include "intel_vga.h"
 107 #include "i9xx_plane.h"
 108 #include "skl_scaler.h"
 109 #include "skl_universal_plane.h"
 110
 111 static void i9xx_crtc_clock_get(struct intel_crtc *crtc,
 112                                 struct intel_crtc_state *pipe_config);
 113 static void ilk_pch_clock_get(struct intel_crtc *crtc,
 114                               struct intel_crtc_state *pipe_config);
 115
 116 static void intel_set_transcoder_timings(const struct intel_crtc_state *crtc_state);
 117 static void intel_set_pipe_src_size(const struct intel_crtc_state *crtc_state);
 118 static void intel_cpu_transcoder_set_m_n(const struct intel_crtc_state *crtc_state,
 119                                          const struct intel_link_m_n *m_n,
 120                                          const struct intel_link_m_n *m2_n2);
 121 static void i9xx_set_pipeconf(const struct intel_crtc_state *crtc_state);
 122 static void ilk_set_pipeconf(const struct intel_crtc_state *crtc_state);
 123 static void hsw_set_transconf(const struct intel_crtc_state *crtc_state);
 124 static void bdw_set_pipemisc(const struct intel_crtc_state *crtc_state);
 125 static void ilk_pfit_enable(const struct intel_crtc_state *crtc_state);
 126 static void intel_modeset_setup_hw_state(struct drm_device *dev,
 127                                          struct drm_modeset_acquire_ctx *ctx);
 128
 129 /* returns HPLL frequency in kHz */
 130 int vlv_get_hpll_vco(struct drm_i915_private *dev_priv)
 131 {
 132         int hpll_freq, vco_freq[] = { 800, 1600, 2000, 2400 };
 133
 134         /* Obtain SKU information */
 135         hpll_freq = vlv_cck_read(dev_priv, CCK_FUSE_REG) &
 136                 CCK_FUSE_HPLL_FREQ_MASK;
 137
 138         return vco_freq[hpll_freq] * 1000;
 139 }
 140
 141 int vlv_get_cck_clock(struct drm_i915_private *dev_priv,
 142                       const char *name, u32 reg, int ref_freq)
 143 {
 144         u32 val;
 145         int divider;
 146
 147         val = vlv_cck_read(dev_priv, reg);
 148         divider = val & CCK_FREQUENCY_VALUES;
 149
 150         drm_WARN(&dev_priv->drm, (val & CCK_FREQUENCY_STATUS) !=
 151                  (divider << CCK_FREQUENCY_STATUS_SHIFT),
 152                  "%s change in progress\n", name);
 153
 154         return DIV_ROUND_CLOSEST(ref_freq << 1, divider + 1);
 155 }
 156
 157 int vlv_get_cck_clock_hpll(struct drm_i915_private *dev_priv,
 158                            const char *name, u32 reg)
 159 {
 160         int hpll;
 161
 162         vlv_cck_get(dev_priv);
 163
 164         if (dev_priv->hpll_freq == 0)
 165                 dev_priv->hpll_freq = vlv_get_hpll_vco(dev_priv);
 166
 167         hpll = vlv_get_cck_clock(dev_priv, name, reg, dev_priv->hpll_freq);
 168
 169         vlv_cck_put(dev_priv);
 170
 171         return hpll;
 172 }
 173
 174 static void intel_update_czclk(struct drm_i915_private *dev_priv)
 175 {
 176         if (!(IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)))
 177                 return;
 178
 179         dev_priv->czclk_freq = vlv_get_cck_clock_hpll(dev_priv, "czclk",
 180                                                       CCK_CZ_CLOCK_CONTROL);
 181
 182         drm_dbg(&dev_priv->drm, "CZ clock rate: %d kHz\n",
 183                 dev_priv->czclk_freq);
 184 }
 185
 186 static bool is_hdr_mode(const struct intel_crtc_state *crtc_state)
 187 {
 188         return (crtc_state->active_planes &
 189                 ~(icl_hdr_plane_mask() | BIT(PLANE_CURSOR))) == 0;
 190 }
 191
 192 /* WA Display #0827: Gen9:all */
 193 static void
 194 skl_wa_827(struct drm_i915_private *dev_priv, enum pipe pipe, bool enable)
 195 {
 196         if (enable)
 197                 intel_de_write(dev_priv, CLKGATE_DIS_PSL(pipe),
 198                                intel_de_read(dev_priv, CLKGATE_DIS_PSL(pipe)) | DUPS1_GATING_DIS | DUPS2_GATING_DIS);
 199         else
 200                 intel_de_write(dev_priv, CLKGATE_DIS_PSL(pipe),
 201                                intel_de_read(dev_priv, CLKGATE_DIS_PSL(pipe)) & ~(DUPS1_GATING_DIS | DUPS2_GATING_DIS));
 202 }
 203
 204 /* Wa_2006604312:icl,ehl */
 205 static void
 206 icl_wa_scalerclkgating(struct drm_i915_private *dev_priv, enum pipe pipe,
 207                        bool enable)
 208 {
 209         if (enable)
 210                 intel_de_write(dev_priv, CLKGATE_DIS_PSL(pipe),
 211                                intel_de_read(dev_priv, CLKGATE_DIS_PSL(pipe)) | DPFR_GATING_DIS);
 212         else
 213                 intel_de_write(dev_priv, CLKGATE_DIS_PSL(pipe),
 214                                intel_de_read(dev_priv, CLKGATE_DIS_PSL(pipe)) & ~DPFR_GATING_DIS);
 215 }
 216
 217 static bool
 218 is_trans_port_sync_slave(const struct intel_crtc_state *crtc_state)
 219 {
 220         return crtc_state->master_transcoder != INVALID_TRANSCODER;
 221 }
 222
 223 static bool
 224 is_trans_port_sync_master(const struct intel_crtc_state *crtc_state)
 225 {
 226         return crtc_state->sync_mode_slaves_mask != 0;
 227 }
 228
 229 bool
 230 is_trans_port_sync_mode(const struct intel_crtc_state *crtc_state)
 231 {
 232         return is_trans_port_sync_master(crtc_state) ||
 233                 is_trans_port_sync_slave(crtc_state);
 234 }
 235
 236 static bool pipe_scanline_is_moving(struct drm_i915_private *dev_priv,
 237                                     enum pipe pipe)
 238 {
 239         i915_reg_t reg = PIPEDSL(pipe);
 240         u32 line1, line2;
 241         u32 line_mask;
 242
 243         if (DISPLAY_VER(dev_priv) == 2)
 244                 line_mask = DSL_LINEMASK_GEN2;
 245         else
 246                 line_mask = DSL_LINEMASK_GEN3;
 247
 248         line1 = intel_de_read(dev_priv, reg) & line_mask;
 249         msleep(5);
 250         line2 = intel_de_read(dev_priv, reg) & line_mask;
 251
 252         return line1 != line2;
 253 }
 254
 255 static void wait_for_pipe_scanline_moving(struct intel_crtc *crtc, bool state)
 256 {
 257         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 258         enum pipe pipe = crtc->pipe;
 259
 260         /* Wait for the display line to settle/start moving */
 261         if (wait_for(pipe_scanline_is_moving(dev_priv, pipe) == state, 100))
 262                 drm_err(&dev_priv->drm,
 263                         "pipe %c scanline %s wait timed out\n",
 264                         pipe_name(pipe), onoff(state));
 265 }
 266
 267 static void intel_wait_for_pipe_scanline_stopped(struct intel_crtc *crtc)
 268 {
 269         wait_for_pipe_scanline_moving(crtc, false);
 270 }
 271
 272 static void intel_wait_for_pipe_scanline_moving(struct intel_crtc *crtc)
 273 {
 274         wait_for_pipe_scanline_moving(crtc, true);
 275 }
 276
 277 static void
 278 intel_wait_for_pipe_off(const struct intel_crtc_state *old_crtc_state)
 279 {
 280         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->uapi.crtc);
 281         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 282
 283         if (DISPLAY_VER(dev_priv) >= 4) {
 284                 enum transcoder cpu_transcoder = old_crtc_state->cpu_transcoder;
 285                 i915_reg_t reg = PIPECONF(cpu_transcoder);
 286
 287                 /* Wait for the Pipe State to go off */
 288                 if (intel_de_wait_for_clear(dev_priv, reg,
 289                                             I965_PIPECONF_ACTIVE, 100))
 290                         drm_WARN(&dev_priv->drm, 1,
 291                                  "pipe_off wait timed out\n");
 292         } else {
 293                 intel_wait_for_pipe_scanline_stopped(crtc);
 294         }
 295 }
 296
 297 /* Only for pre-ILK configs */
 298 void assert_pll(struct drm_i915_private *dev_priv,
 299                 enum pipe pipe, bool state)
 300 {
 301         u32 val;
 302         bool cur_state;
 303
 304         val = intel_de_read(dev_priv, DPLL(pipe));
 305         cur_state = !!(val & DPLL_VCO_ENABLE);
 306         I915_STATE_WARN(cur_state != state,
 307              "PLL state assertion failure (expected %s, current %s)\n",
 308                         onoff(state), onoff(cur_state));
 309 }
 310
 311 /* XXX: the dsi pll is shared between MIPI DSI ports */
 312 void assert_dsi_pll(struct drm_i915_private *dev_priv, bool state)
 313 {
 314         u32 val;
 315         bool cur_state;
 316
 317         vlv_cck_get(dev_priv);
 318         val = vlv_cck_read(dev_priv, CCK_REG_DSI_PLL_CONTROL);
 319         vlv_cck_put(dev_priv);
 320
 321         cur_state = val & DSI_PLL_VCO_EN;
 322         I915_STATE_WARN(cur_state != state,
 323              "DSI PLL state assertion failure (expected %s, current %s)\n",
 324                         onoff(state), onoff(cur_state));
 325 }
 326
 327 static void assert_fdi_tx(struct drm_i915_private *dev_priv,
 328                           enum pipe pipe, bool state)
 329 {
 330         bool cur_state;
 331
 332         if (HAS_DDI(dev_priv)) {
 333                 /*
 334                  * DDI does not have a specific FDI_TX register.
 335                  *
 336                  * FDI is never fed from EDP transcoder
 337                  * so pipe->transcoder cast is fine here.
 338                  */
 339                 enum transcoder cpu_transcoder = (enum transcoder)pipe;
 340                 u32 val = intel_de_read(dev_priv,
 341                                         TRANS_DDI_FUNC_CTL(cpu_transcoder));
 342                 cur_state = !!(val & TRANS_DDI_FUNC_ENABLE);
 343         } else {
 344                 u32 val = intel_de_read(dev_priv, FDI_TX_CTL(pipe));
 345                 cur_state = !!(val & FDI_TX_ENABLE);
 346         }
 347         I915_STATE_WARN(cur_state != state,
 348              "FDI TX state assertion failure (expected %s, current %s)\n",
 349                         onoff(state), onoff(cur_state));
 350 }
 351 #define assert_fdi_tx_enabled(d, p) assert_fdi_tx(d, p, true)
 352 #define assert_fdi_tx_disabled(d, p) assert_fdi_tx(d, p, false)
 353
 354 static void assert_fdi_rx(struct drm_i915_private *dev_priv,
 355                           enum pipe pipe, bool state)
 356 {
 357         u32 val;
 358         bool cur_state;
 359
 360         val = intel_de_read(dev_priv, FDI_RX_CTL(pipe));
 361         cur_state = !!(val & FDI_RX_ENABLE);
 362         I915_STATE_WARN(cur_state != state,
 363              "FDI RX state assertion failure (expected %s, current %s)\n",
 364                         onoff(state), onoff(cur_state));
 365 }
 366 #define assert_fdi_rx_enabled(d, p) assert_fdi_rx(d, p, true)
 367 #define assert_fdi_rx_disabled(d, p) assert_fdi_rx(d, p, false)
 368
 369 static void assert_fdi_tx_pll_enabled(struct drm_i915_private *dev_priv,
 370                                       enum pipe pipe)
 371 {
 372         u32 val;
 373
 374         /* ILK FDI PLL is always enabled */
 375         if (IS_IRONLAKE(dev_priv))
 376                 return;
 377
 378         /* On Haswell, DDI ports are responsible for the FDI PLL setup */
 379         if (HAS_DDI(dev_priv))
 380                 return;
 381
 382         val = intel_de_read(dev_priv, FDI_TX_CTL(pipe));
 383         I915_STATE_WARN(!(val & FDI_TX_PLL_ENABLE), "FDI TX PLL assertion failure, should be active but is disabled\n");
 384 }
 385
 386 void assert_fdi_rx_pll(struct drm_i915_private *dev_priv,
 387                        enum pipe pipe, bool state)
 388 {
 389         u32 val;
 390         bool cur_state;
 391
 392         val = intel_de_read(dev_priv, FDI_RX_CTL(pipe));
 393         cur_state = !!(val & FDI_RX_PLL_ENABLE);
 394         I915_STATE_WARN(cur_state != state,
 395              "FDI RX PLL assertion failure (expected %s, current %s)\n",
 396                         onoff(state), onoff(cur_state));
 397 }
 398
 399 void assert_panel_unlocked(struct drm_i915_private *dev_priv, enum pipe pipe)
 400 {
 401         i915_reg_t pp_reg;
 402         u32 val;
 403         enum pipe panel_pipe = INVALID_PIPE;
 404         bool locked = true;
 405
 406         if (drm_WARN_ON(&dev_priv->drm, HAS_DDI(dev_priv)))
 407                 return;
 408
 409         if (HAS_PCH_SPLIT(dev_priv)) {
 410                 u32 port_sel;
 411
 412                 pp_reg = PP_CONTROL(0);
 413                 port_sel = intel_de_read(dev_priv, PP_ON_DELAYS(0)) & PANEL_PORT_SELECT_MASK;
 414
 415                 switch (port_sel) {
 416                 case PANEL_PORT_SELECT_LVDS:
 417                         intel_lvds_port_enabled(dev_priv, PCH_LVDS, &panel_pipe);
 418                         break;
 419                 case PANEL_PORT_SELECT_DPA:
 420                         g4x_dp_port_enabled(dev_priv, DP_A, PORT_A, &panel_pipe);
 421                         break;
 422                 case PANEL_PORT_SELECT_DPC:
 423                         g4x_dp_port_enabled(dev_priv, PCH_DP_C, PORT_C, &panel_pipe);
 424                         break;
 425                 case PANEL_PORT_SELECT_DPD:
 426                         g4x_dp_port_enabled(dev_priv, PCH_DP_D, PORT_D, &panel_pipe);
 427                         break;
 428                 default:
 429                         MISSING_CASE(port_sel);
 430                         break;
 431                 }
 432         } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
 433                 /* presumably write lock depends on pipe, not port select */
 434                 pp_reg = PP_CONTROL(pipe);
 435                 panel_pipe = pipe;
 436         } else {
 437                 u32 port_sel;
 438
 439                 pp_reg = PP_CONTROL(0);
 440                 port_sel = intel_de_read(dev_priv, PP_ON_DELAYS(0)) & PANEL_PORT_SELECT_MASK;
 441
 442                 drm_WARN_ON(&dev_priv->drm,
 443                             port_sel != PANEL_PORT_SELECT_LVDS);
 444                 intel_lvds_port_enabled(dev_priv, LVDS, &panel_pipe);
 445         }
 446
 447         val = intel_de_read(dev_priv, pp_reg);
 448         if (!(val & PANEL_POWER_ON) ||
 449             ((val & PANEL_UNLOCK_MASK) == PANEL_UNLOCK_REGS))
 450                 locked = false;
 451
 452         I915_STATE_WARN(panel_pipe == pipe && locked,
 453              "panel assertion failure, pipe %c regs locked\n",
 454              pipe_name(pipe));
 455 }
 456
 457 void assert_transcoder(struct drm_i915_private *dev_priv,
 458                        enum transcoder cpu_transcoder, bool state)
 459 {
 460         bool cur_state;
 461         enum intel_display_power_domain power_domain;
 462         intel_wakeref_t wakeref;
 463
 464         /* we keep both pipes enabled on 830 */
 465         if (IS_I830(dev_priv))
 466                 state = true;
 467
 468         power_domain = POWER_DOMAIN_TRANSCODER(cpu_transcoder);
 469         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
 470         if (wakeref) {
 471                 u32 val = intel_de_read(dev_priv, PIPECONF(cpu_transcoder));
 472                 cur_state = !!(val & PIPECONF_ENABLE);
 473
 474                 intel_display_power_put(dev_priv, power_domain, wakeref);
 475         } else {
 476                 cur_state = false;
 477         }
 478
 479         I915_STATE_WARN(cur_state != state,
 480                         "transcoder %s assertion failure (expected %s, current %s)\n",
 481                         transcoder_name(cpu_transcoder),
 482                         onoff(state), onoff(cur_state));
 483 }
 484
 485 static void assert_plane(struct intel_plane *plane, bool state)
 486 {
 487         enum pipe pipe;
 488         bool cur_state;
 489
 490         cur_state = plane->get_hw_state(plane, &pipe);
 491
 492         I915_STATE_WARN(cur_state != state,
 493                         "%s assertion failure (expected %s, current %s)\n",
 494                         plane->base.name, onoff(state), onoff(cur_state));
 495 }
 496
 497 #define assert_plane_enabled(p) assert_plane(p, true)
 498 #define assert_plane_disabled(p) assert_plane(p, false)
 499
 500 static void assert_planes_disabled(struct intel_crtc *crtc)
 501 {
 502         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 503         struct intel_plane *plane;
 504
 505         for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane)
 506                 assert_plane_disabled(plane);
 507 }
 508
 509 void assert_pch_transcoder_disabled(struct drm_i915_private *dev_priv,
 510                                     enum pipe pipe)
 511 {
 512         u32 val;
 513         bool enabled;
 514
 515         val = intel_de_read(dev_priv, PCH_TRANSCONF(pipe));
 516         enabled = !!(val & TRANS_ENABLE);
 517         I915_STATE_WARN(enabled,
 518              "transcoder assertion failed, should be off on pipe %c but is still active\n",
 519              pipe_name(pipe));
 520 }
 521
 522 static void assert_pch_dp_disabled(struct drm_i915_private *dev_priv,
 523                                    enum pipe pipe, enum port port,
 524                                    i915_reg_t dp_reg)
 525 {
 526         enum pipe port_pipe;
 527         bool state;
 528
 529         state = g4x_dp_port_enabled(dev_priv, dp_reg, port, &port_pipe);
 530
 531         I915_STATE_WARN(state && port_pipe == pipe,
 532                         "PCH DP %c enabled on transcoder %c, should be disabled\n",
 533                         port_name(port), pipe_name(pipe));
 534
 535         I915_STATE_WARN(HAS_PCH_IBX(dev_priv) && !state && port_pipe == PIPE_B,
 536                         "IBX PCH DP %c still using transcoder B\n",
 537                         port_name(port));
 538 }
 539
 540 static void assert_pch_hdmi_disabled(struct drm_i915_private *dev_priv,
 541                                      enum pipe pipe, enum port port,
 542                                      i915_reg_t hdmi_reg)
 543 {
 544         enum pipe port_pipe;
 545         bool state;
 546
 547         state = intel_sdvo_port_enabled(dev_priv, hdmi_reg, &port_pipe);
 548
 549         I915_STATE_WARN(state && port_pipe == pipe,
 550                         "PCH HDMI %c enabled on transcoder %c, should be disabled\n",
 551                         port_name(port), pipe_name(pipe));
 552
 553         I915_STATE_WARN(HAS_PCH_IBX(dev_priv) && !state && port_pipe == PIPE_B,
 554                         "IBX PCH HDMI %c still using transcoder B\n",
 555                         port_name(port));
 556 }
 557
 558 static void assert_pch_ports_disabled(struct drm_i915_private *dev_priv,
 559                                       enum pipe pipe)
 560 {
 561         enum pipe port_pipe;
 562
 563         assert_pch_dp_disabled(dev_priv, pipe, PORT_B, PCH_DP_B);
 564         assert_pch_dp_disabled(dev_priv, pipe, PORT_C, PCH_DP_C);
 565         assert_pch_dp_disabled(dev_priv, pipe, PORT_D, PCH_DP_D);
 566
 567         I915_STATE_WARN(intel_crt_port_enabled(dev_priv, PCH_ADPA, &port_pipe) &&
 568                         port_pipe == pipe,
 569                         "PCH VGA enabled on transcoder %c, should be disabled\n",
 570                         pipe_name(pipe));
 571
 572         I915_STATE_WARN(intel_lvds_port_enabled(dev_priv, PCH_LVDS, &port_pipe) &&
 573                         port_pipe == pipe,
 574                         "PCH LVDS enabled on transcoder %c, should be disabled\n",
 575                         pipe_name(pipe));
 576
 577         /* PCH SDVOB multiplex with HDMIB */
 578         assert_pch_hdmi_disabled(dev_priv, pipe, PORT_B, PCH_HDMIB);
 579         assert_pch_hdmi_disabled(dev_priv, pipe, PORT_C, PCH_HDMIC);
 580         assert_pch_hdmi_disabled(dev_priv, pipe, PORT_D, PCH_HDMID);
 581 }
 582
 583 void vlv_wait_port_ready(struct drm_i915_private *dev_priv,
 584                          struct intel_digital_port *dig_port,
 585                          unsigned int expected_mask)
 586 {
 587         u32 port_mask;
 588         i915_reg_t dpll_reg;
 589
 590         switch (dig_port->base.port) {
 591         case PORT_B:
 592                 port_mask = DPLL_PORTB_READY_MASK;
 593                 dpll_reg = DPLL(0);
 594                 break;
 595         case PORT_C:
 596                 port_mask = DPLL_PORTC_READY_MASK;
 597                 dpll_reg = DPLL(0);
 598                 expected_mask <<= 4;
 599                 break;
 600         case PORT_D:
 601                 port_mask = DPLL_PORTD_READY_MASK;
 602                 dpll_reg = DPIO_PHY_STATUS;
 603                 break;
 604         default:
 605                 BUG();
 606         }
 607
 608         if (intel_de_wait_for_register(dev_priv, dpll_reg,
 609                                        port_mask, expected_mask, 1000))
 610                 drm_WARN(&dev_priv->drm, 1,
 611                          "timed out waiting for [ENCODER:%d:%s] port ready: got 0x%x, expected 0x%x\n",
 612                          dig_port->base.base.base.id, dig_port->base.base.name,
 613                          intel_de_read(dev_priv, dpll_reg) & port_mask,
 614                          expected_mask);
 615 }
 616
 617 static void ilk_enable_pch_transcoder(const struct intel_crtc_state *crtc_state)
 618 {
 619         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 620         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 621         enum pipe pipe = crtc->pipe;
 622         i915_reg_t reg;
 623         u32 val, pipeconf_val;
 624
 625         /* Make sure PCH DPLL is enabled */
 626         assert_shared_dpll_enabled(dev_priv, crtc_state->shared_dpll);
 627
 628         /* FDI must be feeding us bits for PCH ports */
 629         assert_fdi_tx_enabled(dev_priv, pipe);
 630         assert_fdi_rx_enabled(dev_priv, pipe);
 631
 632         if (HAS_PCH_CPT(dev_priv)) {
 633                 reg = TRANS_CHICKEN2(pipe);
 634                 val = intel_de_read(dev_priv, reg);
 635                 /*
 636                  * Workaround: Set the timing override bit
 637                  * before enabling the pch transcoder.
 638                  */
 639                 val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
 640                 /* Configure frame start delay to match the CPU */
 641                 val &= ~TRANS_CHICKEN2_FRAME_START_DELAY_MASK;
 642                 val |= TRANS_CHICKEN2_FRAME_START_DELAY(dev_priv->framestart_delay - 1);
 643                 intel_de_write(dev_priv, reg, val);
 644         }
 645
 646         reg = PCH_TRANSCONF(pipe);
 647         val = intel_de_read(dev_priv, reg);
 648         pipeconf_val = intel_de_read(dev_priv, PIPECONF(pipe));
 649
 650         if (HAS_PCH_IBX(dev_priv)) {
 651                 /* Configure frame start delay to match the CPU */
 652                 val &= ~TRANS_FRAME_START_DELAY_MASK;
 653                 val |= TRANS_FRAME_START_DELAY(dev_priv->framestart_delay - 1);
 654
 655                 /*
 656                  * Make the BPC in transcoder be consistent with
 657                  * that in pipeconf reg. For HDMI we must use 8bpc
 658                  * here for both 8bpc and 12bpc.
 659                  */
 660                 val &= ~PIPECONF_BPC_MASK;
 661                 if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI))
 662                         val |= PIPECONF_8BPC;
 663                 else
 664                         val |= pipeconf_val & PIPECONF_BPC_MASK;
 665         }
 666
 667         val &= ~TRANS_INTERLACE_MASK;
 668         if ((pipeconf_val & PIPECONF_INTERLACE_MASK) == PIPECONF_INTERLACED_ILK) {
 669                 if (HAS_PCH_IBX(dev_priv) &&
 670                     intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO))
 671                         val |= TRANS_LEGACY_INTERLACED_ILK;
 672                 else
 673                         val |= TRANS_INTERLACED;
 674         } else {
 675                 val |= TRANS_PROGRESSIVE;
 676         }
 677
 678         intel_de_write(dev_priv, reg, val | TRANS_ENABLE);
 679         if (intel_de_wait_for_set(dev_priv, reg, TRANS_STATE_ENABLE, 100))
 680                 drm_err(&dev_priv->drm, "failed to enable transcoder %c\n",
 681                         pipe_name(pipe));
 682 }
 683
 684 static void lpt_enable_pch_transcoder(struct drm_i915_private *dev_priv,
 685                                       enum transcoder cpu_transcoder)
 686 {
 687         u32 val, pipeconf_val;
 688
 689         /* FDI must be feeding us bits for PCH ports */
 690         assert_fdi_tx_enabled(dev_priv, (enum pipe) cpu_transcoder);
 691         assert_fdi_rx_enabled(dev_priv, PIPE_A);
 692
 693         val = intel_de_read(dev_priv, TRANS_CHICKEN2(PIPE_A));
 694         /* Workaround: set timing override bit. */
 695         val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
 696         /* Configure frame start delay to match the CPU */
 697         val &= ~TRANS_CHICKEN2_FRAME_START_DELAY_MASK;
 698         val |= TRANS_CHICKEN2_FRAME_START_DELAY(dev_priv->framestart_delay - 1);
 699         intel_de_write(dev_priv, TRANS_CHICKEN2(PIPE_A), val);
 700
 701         val = TRANS_ENABLE;
 702         pipeconf_val = intel_de_read(dev_priv, PIPECONF(cpu_transcoder));
 703
 704         if ((pipeconf_val & PIPECONF_INTERLACE_MASK_HSW) ==
 705             PIPECONF_INTERLACED_ILK)
 706                 val |= TRANS_INTERLACED;
 707         else
 708                 val |= TRANS_PROGRESSIVE;
 709
 710         intel_de_write(dev_priv, LPT_TRANSCONF, val);
 711         if (intel_de_wait_for_set(dev_priv, LPT_TRANSCONF,
 712                                   TRANS_STATE_ENABLE, 100))
 713                 drm_err(&dev_priv->drm, "Failed to enable PCH transcoder\n");
 714 }
 715
 716 static void ilk_disable_pch_transcoder(struct drm_i915_private *dev_priv,
 717                                        enum pipe pipe)
 718 {
 719         i915_reg_t reg;
 720         u32 val;
 721
 722         /* FDI relies on the transcoder */
 723         assert_fdi_tx_disabled(dev_priv, pipe);
 724         assert_fdi_rx_disabled(dev_priv, pipe);
 725
 726         /* Ports must be off as well */
 727         assert_pch_ports_disabled(dev_priv, pipe);
 728
 729         reg = PCH_TRANSCONF(pipe);
 730         val = intel_de_read(dev_priv, reg);
 731         val &= ~TRANS_ENABLE;
 732         intel_de_write(dev_priv, reg, val);
 733         /* wait for PCH transcoder off, transcoder state */
 734         if (intel_de_wait_for_clear(dev_priv, reg, TRANS_STATE_ENABLE, 50))
 735                 drm_err(&dev_priv->drm, "failed to disable transcoder %c\n",
 736                         pipe_name(pipe));
 737
 738         if (HAS_PCH_CPT(dev_priv)) {
 739                 /* Workaround: Clear the timing override chicken bit again. */
 740                 reg = TRANS_CHICKEN2(pipe);
 741                 val = intel_de_read(dev_priv, reg);
 742                 val &= ~TRANS_CHICKEN2_TIMING_OVERRIDE;
 743                 intel_de_write(dev_priv, reg, val);
 744         }
 745 }
 746
 747 void lpt_disable_pch_transcoder(struct drm_i915_private *dev_priv)
 748 {
 749         u32 val;
 750
 751         val = intel_de_read(dev_priv, LPT_TRANSCONF);
 752         val &= ~TRANS_ENABLE;
 753         intel_de_write(dev_priv, LPT_TRANSCONF, val);
 754         /* wait for PCH transcoder off, transcoder state */
 755         if (intel_de_wait_for_clear(dev_priv, LPT_TRANSCONF,
 756                                     TRANS_STATE_ENABLE, 50))
 757                 drm_err(&dev_priv->drm, "Failed to disable PCH transcoder\n");
 758
 759         /* Workaround: clear timing override bit. */
 760         val = intel_de_read(dev_priv, TRANS_CHICKEN2(PIPE_A));
 761         val &= ~TRANS_CHICKEN2_TIMING_OVERRIDE;
 762         intel_de_write(dev_priv, TRANS_CHICKEN2(PIPE_A), val);
 763 }
 764
 765 enum pipe intel_crtc_pch_transcoder(struct intel_crtc *crtc)
 766 {
 767         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 768
 769         if (HAS_PCH_LPT(dev_priv))
 770                 return PIPE_A;
 771         else
 772                 return crtc->pipe;
 773 }
 774
 775 void intel_enable_transcoder(const struct intel_crtc_state *new_crtc_state)
 776 {
 777         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
 778         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 779         enum transcoder cpu_transcoder = new_crtc_state->cpu_transcoder;
 780         enum pipe pipe = crtc->pipe;
 781         i915_reg_t reg;
 782         u32 val;
 783
 784         drm_dbg_kms(&dev_priv->drm, "enabling pipe %c\n", pipe_name(pipe));
 785
 786         assert_planes_disabled(crtc);
 787
 788         /*
 789          * A pipe without a PLL won't actually be able to drive bits from
 790          * a plane.  On ILK+ the pipe PLLs are integrated, so we don't
 791          * need the check.
 792          */
 793         if (HAS_GMCH(dev_priv)) {
 794                 if (intel_crtc_has_type(new_crtc_state, INTEL_OUTPUT_DSI))
 795                         assert_dsi_pll_enabled(dev_priv);
 796                 else
 797                         assert_pll_enabled(dev_priv, pipe);
 798         } else {
 799                 if (new_crtc_state->has_pch_encoder) {
 800                         /* if driving the PCH, we need FDI enabled */
 801                         assert_fdi_rx_pll_enabled(dev_priv,
 802                                                   intel_crtc_pch_transcoder(crtc));
 803                         assert_fdi_tx_pll_enabled(dev_priv,
 804                                                   (enum pipe) cpu_transcoder);
 805                 }
 806                 /* FIXME: assert CPU port conditions for SNB+ */
 807         }
 808
 809         /* Wa_22012358565:adl-p */
 810         if (DISPLAY_VER(dev_priv) == 13)
 811                 intel_de_rmw(dev_priv, PIPE_ARB_CTL(pipe),
 812                              0, PIPE_ARB_USE_PROG_SLOTS);
 813
 814         reg = PIPECONF(cpu_transcoder);
 815         val = intel_de_read(dev_priv, reg);
 816         if (val & PIPECONF_ENABLE) {
 817                 /* we keep both pipes enabled on 830 */
 818                 drm_WARN_ON(&dev_priv->drm, !IS_I830(dev_priv));
 819                 return;
 820         }
 821
 822         intel_de_write(dev_priv, reg, val | PIPECONF_ENABLE);
 823         intel_de_posting_read(dev_priv, reg);
 824
 825         /*
 826          * Until the pipe starts PIPEDSL reads will return a stale value,
 827          * which causes an apparent vblank timestamp jump when PIPEDSL
 828          * resets to its proper value. That also messes up the frame count
 829          * when it's derived from the timestamps. So let's wait for the
 830          * pipe to start properly before we call drm_crtc_vblank_on()
 831          */
 832         if (intel_crtc_max_vblank_count(new_crtc_state) == 0)
 833                 intel_wait_for_pipe_scanline_moving(crtc);
 834 }
 835
 836 void intel_disable_transcoder(const struct intel_crtc_state *old_crtc_state)
 837 {
 838         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->uapi.crtc);
 839         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 840         enum transcoder cpu_transcoder = old_crtc_state->cpu_transcoder;
 841         enum pipe pipe = crtc->pipe;
 842         i915_reg_t reg;
 843         u32 val;
 844
 845         drm_dbg_kms(&dev_priv->drm, "disabling pipe %c\n", pipe_name(pipe));
 846
 847         /*
 848          * Make sure planes won't keep trying to pump pixels to us,
 849          * or we might hang the display.
 850          */
 851         assert_planes_disabled(crtc);
 852
 853         reg = PIPECONF(cpu_transcoder);
 854         val = intel_de_read(dev_priv, reg);
 855         if ((val & PIPECONF_ENABLE) == 0)
 856                 return;
 857
 858         /*
 859          * Double wide has implications for planes
 860          * so best keep it disabled when not needed.
 861          */
 862         if (old_crtc_state->double_wide)
 863                 val &= ~PIPECONF_DOUBLE_WIDE;
 864
 865         /* Don't disable pipe or pipe PLLs if needed */
 866         if (!IS_I830(dev_priv))
 867                 val &= ~PIPECONF_ENABLE;
 868
 869         if (DISPLAY_VER(dev_priv) >= 12)
 870                 intel_de_rmw(dev_priv, CHICKEN_TRANS(cpu_transcoder),
 871                              FECSTALL_DIS_DPTSTREAM_DPTTG, 0);
 872
 873         intel_de_write(dev_priv, reg, val);
 874         if ((val & PIPECONF_ENABLE) == 0)
 875                 intel_wait_for_pipe_off(old_crtc_state);
 876 }
 877
 878 bool
 879 intel_format_info_is_yuv_semiplanar(const struct drm_format_info *info,
 880                                     u64 modifier)
 881 {
 882         return info->is_yuv &&
 883                info->num_planes == (is_ccs_modifier(modifier) ? 4 : 2);
 884 }
 885
 886 unsigned int intel_rotation_info_size(const struct intel_rotation_info *rot_info)
 887 {
 888         unsigned int size = 0;
 889         int i;
 890
 891         for (i = 0 ; i < ARRAY_SIZE(rot_info->plane); i++)
 892                 size += rot_info->plane[i].dst_stride * rot_info->plane[i].width;
 893
 894         return size;
 895 }
 896
 897 unsigned int intel_remapped_info_size(const struct intel_remapped_info *rem_info)
 898 {
 899         unsigned int size = 0;
 900         int i;
 901
 902         for (i = 0 ; i < ARRAY_SIZE(rem_info->plane); i++) {
 903                 if (rem_info->plane_alignment)
 904                         size = ALIGN(size, rem_info->plane_alignment);
 905                 size += rem_info->plane[i].dst_stride * rem_info->plane[i].height;
 906         }
 907
 908         return size;
 909 }
 910
 911 static bool intel_plane_uses_fence(const struct intel_plane_state *plane_state)
 912 {
 913         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
 914         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
 915
 916         return DISPLAY_VER(dev_priv) < 4 ||
 917                 (plane->has_fbc &&
 918                  plane_state->view.gtt.type == I915_GGTT_VIEW_NORMAL);
 919 }
 920
 921 static struct i915_vma *
 922 intel_pin_fb_obj_dpt(struct drm_framebuffer *fb,
 923                      const struct i915_ggtt_view *view,
 924                      bool uses_fence,
 925                      unsigned long *out_flags,
 926                      struct i915_address_space *vm)
 927 {
 928         struct drm_device *dev = fb->dev;
 929         struct drm_i915_private *dev_priv = to_i915(dev);
 930         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
 931         struct i915_vma *vma;
 932         u32 alignment;
 933         int ret;
 934
 935         if (WARN_ON(!i915_gem_object_is_framebuffer(obj)))
 936                 return ERR_PTR(-EINVAL);
 937
 938         alignment = 4096 * 512;
 939
 940         atomic_inc(&dev_priv->gpu_error.pending_fb_pin);
 941
 942         ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE);
 943         if (ret) {
 944                 vma = ERR_PTR(ret);
 945                 goto err;
 946         }
 947
 948         vma = i915_vma_instance(obj, vm, view);
 949         if (IS_ERR(vma))
 950                 goto err;
 951
 952         if (i915_vma_misplaced(vma, 0, alignment, 0)) {
 953                 ret = i915_vma_unbind(vma);
 954                 if (ret) {
 955                         vma = ERR_PTR(ret);
 956                         goto err;
 957                 }
 958         }
 959
 960         ret = i915_vma_pin(vma, 0, alignment, PIN_GLOBAL);
 961         if (ret) {
 962                 vma = ERR_PTR(ret);
 963                 goto err;
 964         }
 965
 966         vma->display_alignment = max_t(u64, vma->display_alignment, alignment);
 967
 968         i915_gem_object_flush_if_display(obj);
 969
 970         i915_vma_get(vma);
 971 err:
 972         atomic_dec(&dev_priv->gpu_error.pending_fb_pin);
 973
 974         return vma;
 975 }
 976
 977 struct i915_vma *
 978 intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb,
 979                            bool phys_cursor,
 980                            const struct i915_ggtt_view *view,
 981                            bool uses_fence,
 982                            unsigned long *out_flags)
 983 {
 984         struct drm_device *dev = fb->dev;
 985         struct drm_i915_private *dev_priv = to_i915(dev);
 986         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
 987         intel_wakeref_t wakeref;
 988         struct i915_gem_ww_ctx ww;
 989         struct i915_vma *vma;
 990         unsigned int pinctl;
 991         u32 alignment;
 992         int ret;
 993
 994         if (drm_WARN_ON(dev, !i915_gem_object_is_framebuffer(obj)))
 995                 return ERR_PTR(-EINVAL);
 996
 997         if (phys_cursor)
 998                 alignment = intel_cursor_alignment(dev_priv);
 999         else
1000                 alignment = intel_surf_alignment(fb, 0);
1001         if (drm_WARN_ON(dev, alignment && !is_power_of_2(alignment)))
1002                 return ERR_PTR(-EINVAL);
1003
1004         /* Note that the w/a also requires 64 PTE of padding following the
1005          * bo. We currently fill all unused PTE with the shadow page and so
1006          * we should always have valid PTE following the scanout preventing
1007          * the VT-d warning.
1008          */
1009         if (intel_scanout_needs_vtd_wa(dev_priv) && alignment < 256 * 1024)
1010                 alignment = 256 * 1024;
1011
1012         /*
1013          * Global gtt pte registers are special registers which actually forward
1014          * writes to a chunk of system memory. Which means that there is no risk
1015          * that the register values disappear as soon as we call
1016          * intel_runtime_pm_put(), so it is correct to wrap only the
1017          * pin/unpin/fence and not more.
1018          */
1019         wakeref = intel_runtime_pm_get(&dev_priv->runtime_pm);
1020
1021         atomic_inc(&dev_priv->gpu_error.pending_fb_pin);
1022
1023         /*
1024          * Valleyview is definitely limited to scanning out the first
1025          * 512MiB. Lets presume this behaviour was inherited from the
1026          * g4x display engine and that all earlier gen are similarly
1027          * limited. Testing suggests that it is a little more
1028          * complicated than this. For example, Cherryview appears quite
1029          * happy to scanout from anywhere within its global aperture.
1030          */
1031         pinctl = 0;
1032         if (HAS_GMCH(dev_priv))
1033                 pinctl |= PIN_MAPPABLE;
1034
1035         i915_gem_ww_ctx_init(&ww, true);
1036 retry:
1037         ret = i915_gem_object_lock(obj, &ww);
1038         if (!ret && phys_cursor)
1039                 ret = i915_gem_object_attach_phys(obj, alignment);
1040         else if (!ret && HAS_LMEM(dev_priv))
1041                 ret = i915_gem_object_migrate(obj, &ww, INTEL_REGION_LMEM);
1042         /* TODO: Do we need to sync when migration becomes async? */
1043         if (!ret)
1044                 ret = i915_gem_object_pin_pages(obj);
1045         if (ret)
1046                 goto err;
1047
1048         if (!ret) {
1049                 vma = i915_gem_object_pin_to_display_plane(obj, &ww, alignment,
1050                                                            view, pinctl);
1051                 if (IS_ERR(vma)) {
1052                         ret = PTR_ERR(vma);
1053                         goto err_unpin;
1054                 }
1055         }
1056
1057         if (uses_fence && i915_vma_is_map_and_fenceable(vma)) {
1058                 /*
1059                  * Install a fence for tiled scan-out. Pre-i965 always needs a
1060                  * fence, whereas 965+ only requires a fence if using
1061                  * framebuffer compression.  For simplicity, we always, when
1062                  * possible, install a fence as the cost is not that onerous.
1063                  *
1064                  * If we fail to fence the tiled scanout, then either the
1065                  * modeset will reject the change (which is highly unlikely as
1066                  * the affected systems, all but one, do not have unmappable
1067                  * space) or we will not be able to enable full powersaving
1068                  * techniques (also likely not to apply due to various limits
1069                  * FBC and the like impose on the size of the buffer, which
1070                  * presumably we violated anyway with this unmappable buffer).
1071                  * Anyway, it is presumably better to stumble onwards with
1072                  * something and try to run the system in a "less than optimal"
1073                  * mode that matches the user configuration.
1074                  */
1075                 ret = i915_vma_pin_fence(vma);
1076                 if (ret != 0 && DISPLAY_VER(dev_priv) < 4) {
1077                         i915_vma_unpin(vma);
1078                         goto err_unpin;
1079                 }
1080                 ret = 0;
1081
1082                 if (vma->fence)
1083                         *out_flags |= PLANE_HAS_FENCE;
1084         }
1085
1086         i915_vma_get(vma);
1087
1088 err_unpin:
1089         i915_gem_object_unpin_pages(obj);
1090 err:
1091         if (ret == -EDEADLK) {
1092                 ret = i915_gem_ww_ctx_backoff(&ww);
1093                 if (!ret)
1094                         goto retry;
1095         }
1096         i915_gem_ww_ctx_fini(&ww);
1097         if (ret)
1098                 vma = ERR_PTR(ret);
1099
1100         atomic_dec(&dev_priv->gpu_error.pending_fb_pin);
1101         intel_runtime_pm_put(&dev_priv->runtime_pm, wakeref);
1102         return vma;
1103 }
1104
1105 void intel_unpin_fb_vma(struct i915_vma *vma, unsigned long flags)
1106 {
1107         if (flags & PLANE_HAS_FENCE)
1108                 i915_vma_unpin_fence(vma);
1109         i915_vma_unpin(vma);
1110         i915_vma_put(vma);
1111 }
1112
1113 /*
1114  * Convert the x/y offsets into a linear offset.
1115  * Only valid with 0/180 degree rotation, which is fine since linear
1116  * offset is only used with linear buffers on pre-hsw and tiled buffers
1117  * with gen2/3, and 90/270 degree rotations isn't supported on any of them.
1118  */
1119 u32 intel_fb_xy_to_linear(int x, int y,
1120                           const struct intel_plane_state *state,
1121                           int color_plane)
1122 {
1123         const struct drm_framebuffer *fb = state->hw.fb;
1124         unsigned int cpp = fb->format->cpp[color_plane];
1125         unsigned int pitch = state->view.color_plane[color_plane].stride;
1126
1127         return y * pitch + x * cpp;
1128 }
1129
1130 /*
1131  * Add the x/y offsets derived from fb->offsets[] to the user
1132  * specified plane src x/y offsets. The resulting x/y offsets
1133  * specify the start of scanout from the beginning of the gtt mapping.
1134  */
1135 void intel_add_fb_offsets(int *x, int *y,
1136                           const struct intel_plane_state *state,
1137                           int color_plane)
1138
1139 {
1140         *x += state->view.color_plane[color_plane].x;
1141         *y += state->view.color_plane[color_plane].y;
1142 }
1143
1144 /*
1145  * From the Sky Lake PRM:
1146  * "The Color Control Surface (CCS) contains the compression status of
1147  *  the cache-line pairs. The compression state of the cache-line pair
1148  *  is specified by 2 bits in the CCS. Each CCS cache-line represents
1149  *  an area on the main surface of 16 x16 sets of 128 byte Y-tiled
1150  *  cache-line-pairs. CCS is always Y tiled."
1151  *
1152  * Since cache line pairs refers to horizontally adjacent cache lines,
1153  * each cache line in the CCS corresponds to an area of 32x16 cache
1154  * lines on the main surface. Since each pixel is 4 bytes, this gives
1155  * us a ratio of one byte in the CCS for each 8x16 pixels in the
1156  * main surface.
1157  */
1158 static const struct drm_format_info skl_ccs_formats[] = {
1159         { .format = DRM_FORMAT_XRGB8888, .depth = 24, .num_planes = 2,
1160           .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, },
1161         { .format = DRM_FORMAT_XBGR8888, .depth = 24, .num_planes = 2,
1162           .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, },
1163         { .format = DRM_FORMAT_ARGB8888, .depth = 32, .num_planes = 2,
1164           .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, .has_alpha = true, },
1165         { .format = DRM_FORMAT_ABGR8888, .depth = 32, .num_planes = 2,
1166           .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, .has_alpha = true, },
1167 };
1168
1169 /*
1170  * Gen-12 compression uses 4 bits of CCS data for each cache line pair in the
1171  * main surface. And each 64B CCS cache line represents an area of 4x1 Y-tiles
1172  * in the main surface. With 4 byte pixels and each Y-tile having dimensions of
1173  * 32x32 pixels, the ratio turns out to 1B in the CCS for every 2x32 pixels in
1174  * the main surface.
1175  */
1176 static const struct drm_format_info gen12_ccs_formats[] = {
1177         { .format = DRM_FORMAT_XRGB8888, .depth = 24, .num_planes = 2,
1178           .char_per_block = { 4, 1 }, .block_w = { 1, 2 }, .block_h = { 1, 1 },
1179           .hsub = 1, .vsub = 1, },
1180         { .format = DRM_FORMAT_XBGR8888, .depth = 24, .num_planes = 2,
1181           .char_per_block = { 4, 1 }, .block_w = { 1, 2 }, .block_h = { 1, 1 },
1182           .hsub = 1, .vsub = 1, },
1183         { .format = DRM_FORMAT_ARGB8888, .depth = 32, .num_planes = 2,
1184           .char_per_block = { 4, 1 }, .block_w = { 1, 2 }, .block_h = { 1, 1 },
1185           .hsub = 1, .vsub = 1, .has_alpha = true },
1186         { .format = DRM_FORMAT_ABGR8888, .depth = 32, .num_planes = 2,
1187           .char_per_block = { 4, 1 }, .block_w = { 1, 2 }, .block_h = { 1, 1 },
1188           .hsub = 1, .vsub = 1, .has_alpha = true },
1189         { .format = DRM_FORMAT_YUYV, .num_planes = 2,
1190           .char_per_block = { 2, 1 }, .block_w = { 1, 2 }, .block_h = { 1, 1 },
1191           .hsub = 2, .vsub = 1, .is_yuv = true },
1192         { .format = DRM_FORMAT_YVYU, .num_planes = 2,
1193           .char_per_block = { 2, 1 }, .block_w = { 1, 2 }, .block_h = { 1, 1 },
1194           .hsub = 2, .vsub = 1, .is_yuv = true },
1195         { .format = DRM_FORMAT_UYVY, .num_planes = 2,
1196           .char_per_block = { 2, 1 }, .block_w = { 1, 2 }, .block_h = { 1, 1 },
1197           .hsub = 2, .vsub = 1, .is_yuv = true },
1198         { .format = DRM_FORMAT_VYUY, .num_planes = 2,
1199           .char_per_block = { 2, 1 }, .block_w = { 1, 2 }, .block_h = { 1, 1 },
1200           .hsub = 2, .vsub = 1, .is_yuv = true },
1201         { .format = DRM_FORMAT_XYUV8888, .num_planes = 2,
1202           .char_per_block = { 4, 1 }, .block_w = { 1, 2 }, .block_h = { 1, 1 },
1203           .hsub = 1, .vsub = 1, .is_yuv = true },
1204         { .format = DRM_FORMAT_NV12, .num_planes = 4,
1205           .char_per_block = { 1, 2, 1, 1 }, .block_w = { 1, 1, 4, 4 }, .block_h = { 1, 1, 1, 1 },
1206           .hsub = 2, .vsub = 2, .is_yuv = true },
1207         { .format = DRM_FORMAT_P010, .num_planes = 4,
1208           .char_per_block = { 2, 4, 1, 1 }, .block_w = { 1, 1, 2, 2 }, .block_h = { 1, 1, 1, 1 },
1209           .hsub = 2, .vsub = 2, .is_yuv = true },
1210         { .format = DRM_FORMAT_P012, .num_planes = 4,
1211           .char_per_block = { 2, 4, 1, 1 }, .block_w = { 1, 1, 2, 2 }, .block_h = { 1, 1, 1, 1 },
1212           .hsub = 2, .vsub = 2, .is_yuv = true },
1213         { .format = DRM_FORMAT_P016, .num_planes = 4,
1214           .char_per_block = { 2, 4, 1, 1 }, .block_w = { 1, 1, 2, 2 }, .block_h = { 1, 1, 1, 1 },
1215           .hsub = 2, .vsub = 2, .is_yuv = true },
1216 };
1217
1218 /*
1219  * Same as gen12_ccs_formats[] above, but with additional surface used
1220  * to pass Clear Color information in plane 2 with 64 bits of data.
1221  */
1222 static const struct drm_format_info gen12_ccs_cc_formats[] = {
1223         { .format = DRM_FORMAT_XRGB8888, .depth = 24, .num_planes = 3,
1224           .char_per_block = { 4, 1, 0 }, .block_w = { 1, 2, 2 }, .block_h = { 1, 1, 1 },
1225           .hsub = 1, .vsub = 1, },
1226         { .format = DRM_FORMAT_XBGR8888, .depth = 24, .num_planes = 3,
1227           .char_per_block = { 4, 1, 0 }, .block_w = { 1, 2, 2 }, .block_h = { 1, 1, 1 },
1228           .hsub = 1, .vsub = 1, },
1229         { .format = DRM_FORMAT_ARGB8888, .depth = 32, .num_planes = 3,
1230           .char_per_block = { 4, 1, 0 }, .block_w = { 1, 2, 2 }, .block_h = { 1, 1, 1 },
1231           .hsub = 1, .vsub = 1, .has_alpha = true },
1232         { .format = DRM_FORMAT_ABGR8888, .depth = 32, .num_planes = 3,
1233           .char_per_block = { 4, 1, 0 }, .block_w = { 1, 2, 2 }, .block_h = { 1, 1, 1 },
1234           .hsub = 1, .vsub = 1, .has_alpha = true },
1235 };
1236
1237 static const struct drm_format_info *
1238 lookup_format_info(const struct drm_format_info formats[],
1239                    int num_formats, u32 format)
1240 {
1241         int i;
1242
1243         for (i = 0; i < num_formats; i++) {
1244                 if (formats[i].format == format)
1245                         return &formats[i];
1246         }
1247
1248         return NULL;
1249 }
1250
1251 static const struct drm_format_info *
1252 intel_get_format_info(const struct drm_mode_fb_cmd2 *cmd)
1253 {
1254         switch (cmd->modifier[0]) {
1255         case I915_FORMAT_MOD_Y_TILED_CCS:
1256         case I915_FORMAT_MOD_Yf_TILED_CCS:
1257                 return lookup_format_info(skl_ccs_formats,
1258                                           ARRAY_SIZE(skl_ccs_formats),
1259                                           cmd->pixel_format);
1260         case I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS:
1261         case I915_FORMAT_MOD_Y_TILED_GEN12_MC_CCS:
1262                 return lookup_format_info(gen12_ccs_formats,
1263                                           ARRAY_SIZE(gen12_ccs_formats),
1264                                           cmd->pixel_format);
1265         case I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS_CC:
1266                 return lookup_format_info(gen12_ccs_cc_formats,
1267                                           ARRAY_SIZE(gen12_ccs_cc_formats),
1268                                           cmd->pixel_format);
1269         default:
1270                 return NULL;
1271         }
1272 }
1273
1274 u32 intel_plane_fb_max_stride(struct drm_i915_private *dev_priv,
1275                               u32 pixel_format, u64 modifier)
1276 {
1277         struct intel_crtc *crtc;
1278         struct intel_plane *plane;
1279
1280         if (!HAS_DISPLAY(dev_priv))
1281                 return 0;
1282
1283         /*
1284          * We assume the primary plane for pipe A has
1285          * the highest stride limits of them all,
1286          * if in case pipe A is disabled, use the first pipe from pipe_mask.
1287          */
1288         crtc = intel_get_first_crtc(dev_priv);
1289         if (!crtc)
1290                 return 0;
1291
1292         plane = to_intel_plane(crtc->base.primary);
1293
1294         return plane->max_stride(plane, pixel_format, modifier,
1295                                  DRM_MODE_ROTATE_0);
1296 }
1297
1298 static struct i915_vma *
1299 initial_plane_vma(struct drm_i915_private *i915,
1300                   struct intel_initial_plane_config *plane_config)
1301 {
1302         struct drm_i915_gem_object *obj;
1303         struct i915_vma *vma;
1304         u32 base, size;
1305
1306         if (plane_config->size == 0)
1307                 return NULL;
1308
1309         base = round_down(plane_config->base,
1310                           I915_GTT_MIN_ALIGNMENT);
1311         size = round_up(plane_config->base + plane_config->size,
1312                         I915_GTT_MIN_ALIGNMENT);
1313         size -= base;
1314
1315         /*
1316          * If the FB is too big, just don't use it since fbdev is not very
1317          * important and we should probably use that space with FBC or other
1318          * features.
1319          */
1320         if (IS_ENABLED(CONFIG_FRAMEBUFFER_CONSOLE) &&
1321             size * 2 > i915->stolen_usable_size)
1322                 return NULL;
1323
1324         obj = i915_gem_object_create_stolen_for_preallocated(i915, base, size);
1325         if (IS_ERR(obj))
1326                 return NULL;
1327
1328         /*
1329          * Mark it WT ahead of time to avoid changing the
1330          * cache_level during fbdev initialization. The
1331          * unbind there would get stuck waiting for rcu.
1332          */
1333         i915_gem_object_set_cache_coherency(obj, HAS_WT(i915) ?
1334                                             I915_CACHE_WT : I915_CACHE_NONE);
1335
1336         switch (plane_config->tiling) {
1337         case I915_TILING_NONE:
1338                 break;
1339         case I915_TILING_X:
1340         case I915_TILING_Y:
1341                 obj->tiling_and_stride =
1342                         plane_config->fb->base.pitches[0] |
1343                         plane_config->tiling;
1344                 break;
1345         default:
1346                 MISSING_CASE(plane_config->tiling);
1347                 goto err_obj;
1348         }
1349
1350         vma = i915_vma_instance(obj, &i915->ggtt.vm, NULL);
1351         if (IS_ERR(vma))
1352                 goto err_obj;
1353
1354         if (i915_ggtt_pin(vma, NULL, 0, PIN_MAPPABLE | PIN_OFFSET_FIXED | base))
1355                 goto err_obj;
1356
1357         if (i915_gem_object_is_tiled(obj) &&
1358             !i915_vma_is_map_and_fenceable(vma))
1359                 goto err_obj;
1360
1361         return vma;
1362
1363 err_obj:
1364         i915_gem_object_put(obj);
1365         return NULL;
1366 }
1367
1368 static bool
1369 intel_alloc_initial_plane_obj(struct intel_crtc *crtc,
1370                               struct intel_initial_plane_config *plane_config)
1371 {
1372         struct drm_device *dev = crtc->base.dev;
1373         struct drm_i915_private *dev_priv = to_i915(dev);
1374         struct drm_mode_fb_cmd2 mode_cmd = { 0 };
1375         struct drm_framebuffer *fb = &plane_config->fb->base;
1376         struct i915_vma *vma;
1377
1378         switch (fb->modifier) {
1379         case DRM_FORMAT_MOD_LINEAR:
1380         case I915_FORMAT_MOD_X_TILED:
1381         case I915_FORMAT_MOD_Y_TILED:
1382                 break;
1383         default:
1384                 drm_dbg(&dev_priv->drm,
1385                         "Unsupported modifier for initial FB: 0x%llx\n",
1386                         fb->modifier);
1387                 return false;
1388         }
1389
1390         vma = initial_plane_vma(dev_priv, plane_config);
1391         if (!vma)
1392                 return false;
1393
1394         mode_cmd.pixel_format = fb->format->format;
1395         mode_cmd.width = fb->width;
1396         mode_cmd.height = fb->height;
1397         mode_cmd.pitches[0] = fb->pitches[0];
1398         mode_cmd.modifier[0] = fb->modifier;
1399         mode_cmd.flags = DRM_MODE_FB_MODIFIERS;
1400
1401         if (intel_framebuffer_init(to_intel_framebuffer(fb),
1402                                    vma->obj, &mode_cmd)) {
1403                 drm_dbg_kms(&dev_priv->drm, "intel fb init failed\n");
1404                 goto err_vma;
1405         }
1406
1407         plane_config->vma = vma;
1408         return true;
1409
1410 err_vma:
1411         i915_vma_put(vma);
1412         return false;
1413 }
1414
1415 static void
1416 intel_set_plane_visible(struct intel_crtc_state *crtc_state,
1417                         struct intel_plane_state *plane_state,
1418                         bool visible)
1419 {
1420         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1421
1422         plane_state->uapi.visible = visible;
1423
1424         if (visible)
1425                 crtc_state->uapi.plane_mask |= drm_plane_mask(&plane->base);
1426         else
1427                 crtc_state->uapi.plane_mask &= ~drm_plane_mask(&plane->base);
1428 }
1429
1430 static void fixup_plane_bitmasks(struct intel_crtc_state *crtc_state)
1431 {
1432         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1433         struct drm_plane *plane;
1434
1435         /*
1436          * Active_planes aliases if multiple "primary" or cursor planes
1437          * have been used on the same (or wrong) pipe. plane_mask uses
1438          * unique ids, hence we can use that to reconstruct active_planes.
1439          */
1440         crtc_state->enabled_planes = 0;
1441         crtc_state->active_planes = 0;
1442
1443         drm_for_each_plane_mask(plane, &dev_priv->drm,
1444                                 crtc_state->uapi.plane_mask) {
1445                 crtc_state->enabled_planes |= BIT(to_intel_plane(plane)->id);
1446                 crtc_state->active_planes |= BIT(to_intel_plane(plane)->id);
1447         }
1448 }
1449
1450 static void intel_plane_disable_noatomic(struct intel_crtc *crtc,
1451                                          struct intel_plane *plane)
1452 {
1453         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1454         struct intel_crtc_state *crtc_state =
1455                 to_intel_crtc_state(crtc->base.state);
1456         struct intel_plane_state *plane_state =
1457                 to_intel_plane_state(plane->base.state);
1458
1459         drm_dbg_kms(&dev_priv->drm,
1460                     "Disabling [PLANE:%d:%s] on [CRTC:%d:%s]\n",
1461                     plane->base.base.id, plane->base.name,
1462                     crtc->base.base.id, crtc->base.name);
1463
1464         intel_set_plane_visible(crtc_state, plane_state, false);
1465         fixup_plane_bitmasks(crtc_state);
1466         crtc_state->data_rate[plane->id] = 0;
1467         crtc_state->min_cdclk[plane->id] = 0;
1468
1469         if (plane->id == PLANE_PRIMARY)
1470                 hsw_disable_ips(crtc_state);
1471
1472         /*
1473          * Vblank time updates from the shadow to live plane control register
1474          * are blocked if the memory self-refresh mode is active at that
1475          * moment. So to make sure the plane gets truly disabled, disable
1476          * first the self-refresh mode. The self-refresh enable bit in turn
1477          * will be checked/applied by the HW only at the next frame start
1478          * event which is after the vblank start event, so we need to have a
1479          * wait-for-vblank between disabling the plane and the pipe.
1480          */
1481         if (HAS_GMCH(dev_priv) &&
1482             intel_set_memory_cxsr(dev_priv, false))
1483                 intel_wait_for_vblank(dev_priv, crtc->pipe);
1484
1485         /*
1486          * Gen2 reports pipe underruns whenever all planes are disabled.
1487          * So disable underrun reporting before all the planes get disabled.
1488          */
1489         if (DISPLAY_VER(dev_priv) == 2 && !crtc_state->active_planes)
1490                 intel_set_cpu_fifo_underrun_reporting(dev_priv, crtc->pipe, false);
1491
1492         intel_disable_plane(plane, crtc_state);
1493         intel_wait_for_vblank(dev_priv, crtc->pipe);
1494 }
1495
1496 static bool
1497 intel_reuse_initial_plane_obj(struct drm_i915_private *i915,
1498                               const struct intel_initial_plane_config *plane_config,
1499                               struct drm_framebuffer **fb,
1500                               struct i915_vma **vma)
1501 {
1502         struct intel_crtc *crtc;
1503
1504         for_each_intel_crtc(&i915->drm, crtc) {
1505                 struct intel_crtc_state *crtc_state =
1506                         to_intel_crtc_state(crtc->base.state);
1507                 struct intel_plane *plane =
1508                         to_intel_plane(crtc->base.primary);
1509                 struct intel_plane_state *plane_state =
1510                         to_intel_plane_state(plane->base.state);
1511
1512                 if (!crtc_state->uapi.active)
1513                         continue;
1514
1515                 if (!plane_state->ggtt_vma)
1516                         continue;
1517
1518                 if (intel_plane_ggtt_offset(plane_state) == plane_config->base) {
1519                         *fb = plane_state->hw.fb;
1520                         *vma = plane_state->ggtt_vma;
1521                         return true;
1522                 }
1523         }
1524
1525         return false;
1526 }
1527
1528 static void
1529 intel_find_initial_plane_obj(struct intel_crtc *crtc,
1530                              struct intel_initial_plane_config *plane_config)
1531 {
1532         struct drm_device *dev = crtc->base.dev;
1533         struct drm_i915_private *dev_priv = to_i915(dev);
1534         struct intel_crtc_state *crtc_state =
1535                 to_intel_crtc_state(crtc->base.state);
1536         struct intel_plane *plane =
1537                 to_intel_plane(crtc->base.primary);
1538         struct intel_plane_state *plane_state =
1539                 to_intel_plane_state(plane->base.state);
1540         struct drm_framebuffer *fb;
1541         struct i915_vma *vma;
1542
1543         /*
1544          * TODO:
1545          *   Disable planes if get_initial_plane_config() failed.
1546          *   Make sure things work if the surface base is not page aligned.
1547          */
1548         if (!plane_config->fb)
1549                 return;
1550
1551         if (intel_alloc_initial_plane_obj(crtc, plane_config)) {
1552                 fb = &plane_config->fb->base;
1553                 vma = plane_config->vma;
1554                 goto valid_fb;
1555         }
1556
1557         /*
1558          * Failed to alloc the obj, check to see if we should share
1559          * an fb with another CRTC instead
1560          */
1561         if (intel_reuse_initial_plane_obj(dev_priv, plane_config, &fb, &vma))
1562                 goto valid_fb;
1563
1564         /*
1565          * We've failed to reconstruct the BIOS FB.  Current display state
1566          * indicates that the primary plane is visible, but has a NULL FB,
1567          * which will lead to problems later if we don't fix it up.  The
1568          * simplest solution is to just disable the primary plane now and
1569          * pretend the BIOS never had it enabled.
1570          */
1571         intel_plane_disable_noatomic(crtc, plane);
1572         if (crtc_state->bigjoiner) {
1573                 struct intel_crtc *slave =
1574                         crtc_state->bigjoiner_linked_crtc;
1575                 intel_plane_disable_noatomic(slave, to_intel_plane(slave->base.primary));
1576         }
1577
1578         return;
1579
1580 valid_fb:
1581         plane_state->uapi.rotation = plane_config->rotation;
1582         intel_fb_fill_view(to_intel_framebuffer(fb),
1583                            plane_state->uapi.rotation, &plane_state->view);
1584
1585         __i915_vma_pin(vma);
1586         plane_state->ggtt_vma = i915_vma_get(vma);
1587         if (intel_plane_uses_fence(plane_state) &&
1588             i915_vma_pin_fence(vma) == 0 && vma->fence)
1589                 plane_state->flags |= PLANE_HAS_FENCE;
1590
1591         plane_state->uapi.src_x = 0;
1592         plane_state->uapi.src_y = 0;
1593         plane_state->uapi.src_w = fb->width << 16;
1594         plane_state->uapi.src_h = fb->height << 16;
1595
1596         plane_state->uapi.crtc_x = 0;
1597         plane_state->uapi.crtc_y = 0;
1598         plane_state->uapi.crtc_w = fb->width;
1599         plane_state->uapi.crtc_h = fb->height;
1600
1601         if (plane_config->tiling)
1602                 dev_priv->preserve_bios_swizzle = true;
1603
1604         plane_state->uapi.fb = fb;
1605         drm_framebuffer_get(fb);
1606
1607         plane_state->uapi.crtc = &crtc->base;
1608         intel_plane_copy_uapi_to_hw_state(plane_state, plane_state, crtc);
1609
1610         intel_frontbuffer_flush(to_intel_frontbuffer(fb), ORIGIN_DIRTYFB);
1611
1612         atomic_or(plane->frontbuffer_bit, &to_intel_frontbuffer(fb)->bits);
1613 }
1614
1615 unsigned int
1616 intel_plane_fence_y_offset(const struct intel_plane_state *plane_state)
1617 {
1618         int x = 0, y = 0;
1619
1620         intel_plane_adjust_aligned_offset(&x, &y, plane_state, 0,
1621                                           plane_state->view.color_plane[0].offset, 0);
1622
1623         return y;
1624 }
1625
1626 static int
1627 __intel_display_resume(struct drm_device *dev,
1628                        struct drm_atomic_state *state,
1629                        struct drm_modeset_acquire_ctx *ctx)
1630 {
1631         struct drm_crtc_state *crtc_state;
1632         struct drm_crtc *crtc;
1633         int i, ret;
1634
1635         intel_modeset_setup_hw_state(dev, ctx);
1636         intel_vga_redisable(to_i915(dev));
1637
1638         if (!state)
1639                 return 0;
1640
1641         /*
1642          * We've duplicated the state, pointers to the old state are invalid.
1643          *
1644          * Don't attempt to use the old state until we commit the duplicated state.
1645          */
1646         for_each_new_crtc_in_state(state, crtc, crtc_state, i) {
1647                 /*
1648                  * Force recalculation even if we restore
1649                  * current state. With fast modeset this may not result
1650                  * in a modeset when the state is compatible.
1651                  */
1652                 crtc_state->mode_changed = true;
1653         }
1654
1655         /* ignore any reset values/BIOS leftovers in the WM registers */
1656         if (!HAS_GMCH(to_i915(dev)))
1657                 to_intel_atomic_state(state)->skip_intermediate_wm = true;
1658
1659         ret = drm_atomic_helper_commit_duplicated_state(state, ctx);
1660
1661         drm_WARN_ON(dev, ret == -EDEADLK);
1662         return ret;
1663 }
1664
1665 static bool gpu_reset_clobbers_display(struct drm_i915_private *dev_priv)
1666 {
1667         return (INTEL_INFO(dev_priv)->gpu_reset_clobbers_display &&
1668                 intel_has_gpu_reset(&dev_priv->gt));
1669 }
1670
1671 void intel_display_prepare_reset(struct drm_i915_private *dev_priv)
1672 {
1673         struct drm_device *dev = &dev_priv->drm;
1674         struct drm_modeset_acquire_ctx *ctx = &dev_priv->reset_ctx;
1675         struct drm_atomic_state *state;
1676         int ret;
1677
1678         if (!HAS_DISPLAY(dev_priv))
1679                 return;
1680
1681         /* reset doesn't touch the display */
1682         if (!dev_priv->params.force_reset_modeset_test &&
1683             !gpu_reset_clobbers_display(dev_priv))
1684                 return;
1685
1686         /* We have a modeset vs reset deadlock, defensively unbreak it. */
1687         set_bit(I915_RESET_MODESET, &dev_priv->gt.reset.flags);
1688         smp_mb__after_atomic();
1689         wake_up_bit(&dev_priv->gt.reset.flags, I915_RESET_MODESET);
1690
1691         if (atomic_read(&dev_priv->gpu_error.pending_fb_pin)) {
1692                 drm_dbg_kms(&dev_priv->drm,
1693                             "Modeset potentially stuck, unbreaking through wedging\n");
1694                 intel_gt_set_wedged(&dev_priv->gt);
1695         }
1696
1697         /*
1698          * Need mode_config.mutex so that we don't
1699          * trample ongoing ->detect() and whatnot.
1700          */
1701         mutex_lock(&dev->mode_config.mutex);
1702         drm_modeset_acquire_init(ctx, 0);
1703         while (1) {
1704                 ret = drm_modeset_lock_all_ctx(dev, ctx);
1705                 if (ret != -EDEADLK)
1706                         break;
1707
1708                 drm_modeset_backoff(ctx);
1709         }
1710         /*
1711          * Disabling the crtcs gracefully seems nicer. Also the
1712          * g33 docs say we should at least disable all the planes.
1713          */
1714         state = drm_atomic_helper_duplicate_state(dev, ctx);
1715         if (IS_ERR(state)) {
1716                 ret = PTR_ERR(state);
1717                 drm_err(&dev_priv->drm, "Duplicating state failed with %i\n",
1718                         ret);
1719                 return;
1720         }
1721
1722         ret = drm_atomic_helper_disable_all(dev, ctx);
1723         if (ret) {
1724                 drm_err(&dev_priv->drm, "Suspending crtc's failed with %i\n",
1725                         ret);
1726                 drm_atomic_state_put(state);
1727                 return;
1728         }
1729
1730         dev_priv->modeset_restore_state = state;
1731         state->acquire_ctx = ctx;
1732 }
1733
1734 void intel_display_finish_reset(struct drm_i915_private *dev_priv)
1735 {
1736         struct drm_device *dev = &dev_priv->drm;
1737         struct drm_modeset_acquire_ctx *ctx = &dev_priv->reset_ctx;
1738         struct drm_atomic_state *state;
1739         int ret;
1740
1741         if (!HAS_DISPLAY(dev_priv))
1742                 return;
1743
1744         /* reset doesn't touch the display */
1745         if (!test_bit(I915_RESET_MODESET, &dev_priv->gt.reset.flags))
1746                 return;
1747
1748         state = fetch_and_zero(&dev_priv->modeset_restore_state);
1749         if (!state)
1750                 goto unlock;
1751
1752         /* reset doesn't touch the display */
1753         if (!gpu_reset_clobbers_display(dev_priv)) {
1754                 /* for testing only restore the display */
1755                 ret = __intel_display_resume(dev, state, ctx);
1756                 if (ret)
1757                         drm_err(&dev_priv->drm,
1758                                 "Restoring old state failed with %i\n", ret);
1759         } else {
1760                 /*
1761                  * The display has been reset as well,
1762                  * so need a full re-initialization.
1763                  */
1764                 intel_pps_unlock_regs_wa(dev_priv);
1765                 intel_modeset_init_hw(dev_priv);
1766                 intel_init_clock_gating(dev_priv);
1767                 intel_hpd_init(dev_priv);
1768
1769                 ret = __intel_display_resume(dev, state, ctx);
1770                 if (ret)
1771                         drm_err(&dev_priv->drm,
1772                                 "Restoring old state failed with %i\n", ret);
1773
1774                 intel_hpd_poll_disable(dev_priv);
1775         }
1776
1777         drm_atomic_state_put(state);
1778 unlock:
1779         drm_modeset_drop_locks(ctx);
1780         drm_modeset_acquire_fini(ctx);
1781         mutex_unlock(&dev->mode_config.mutex);
1782
1783         clear_bit_unlock(I915_RESET_MODESET, &dev_priv->gt.reset.flags);
1784 }
1785
1786 static bool underrun_recovery_supported(const struct intel_crtc_state *crtc_state)
1787 {
1788         if (crtc_state->pch_pfit.enabled &&
1789             (crtc_state->pipe_src_w > drm_rect_width(&crtc_state->pch_pfit.dst) ||
1790              crtc_state->pipe_src_h > drm_rect_height(&crtc_state->pch_pfit.dst) ||
1791              crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR420))
1792                 return false;
1793
1794         if (crtc_state->dsc.compression_enable)
1795                 return false;
1796
1797         if (crtc_state->has_psr2)
1798                 return false;
1799
1800         if (crtc_state->splitter.enable)
1801                 return false;
1802
1803         return true;
1804 }
1805
1806 static void icl_set_pipe_chicken(const struct intel_crtc_state *crtc_state)
1807 {
1808         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1809         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1810         enum pipe pipe = crtc->pipe;
1811         u32 tmp;
1812
1813         tmp = intel_de_read(dev_priv, PIPE_CHICKEN(pipe));
1814
1815         /*
1816          * Display WA #1153: icl
1817          * enable hardware to bypass the alpha math
1818          * and rounding for per-pixel values 00 and 0xff
1819          */
1820         tmp |= PER_PIXEL_ALPHA_BYPASS_EN;
1821         /*
1822          * Display WA # 1605353570: icl
1823          * Set the pixel rounding bit to 1 for allowing
1824          * passthrough of Frame buffer pixels unmodified
1825          * across pipe
1826          */
1827         tmp |= PIXEL_ROUNDING_TRUNC_FB_PASSTHRU;
1828
1829         if (IS_DG2(dev_priv)) {
1830                 /*
1831                  * Underrun recovery must always be disabled on DG2.  However
1832                  * the chicken bit meaning is inverted compared to other
1833                  * platforms.
1834                  */
1835                 tmp &= ~UNDERRUN_RECOVERY_ENABLE_DG2;
1836         } else if (DISPLAY_VER(dev_priv) >= 13) {
1837                 if (underrun_recovery_supported(crtc_state))
1838                         tmp &= ~UNDERRUN_RECOVERY_DISABLE_ADLP;
1839                 else
1840                         tmp |= UNDERRUN_RECOVERY_DISABLE_ADLP;
1841         }
1842
1843         intel_de_write(dev_priv, PIPE_CHICKEN(pipe), tmp);
1844 }
1845
1846 bool intel_has_pending_fb_unpin(struct drm_i915_private *dev_priv)
1847 {
1848         struct drm_crtc *crtc;
1849         bool cleanup_done;
1850
1851         drm_for_each_crtc(crtc, &dev_priv->drm) {
1852                 struct drm_crtc_commit *commit;
1853                 spin_lock(&crtc->commit_lock);
1854                 commit = list_first_entry_or_null(&crtc->commit_list,
1855                                                   struct drm_crtc_commit, commit_entry);
1856                 cleanup_done = commit ?
1857                         try_wait_for_completion(&commit->cleanup_done) : true;
1858                 spin_unlock(&crtc->commit_lock);
1859
1860                 if (cleanup_done)
1861                         continue;
1862
1863                 drm_crtc_wait_one_vblank(crtc);
1864
1865                 return true;
1866         }
1867
1868         return false;
1869 }
1870
1871 void lpt_disable_iclkip(struct drm_i915_private *dev_priv)
1872 {
1873         u32 temp;
1874
1875         intel_de_write(dev_priv, PIXCLK_GATE, PIXCLK_GATE_GATE);
1876
1877         mutex_lock(&dev_priv->sb_lock);
1878
1879         temp = intel_sbi_read(dev_priv, SBI_SSCCTL6, SBI_ICLK);
1880         temp |= SBI_SSCCTL_DISABLE;
1881         intel_sbi_write(dev_priv, SBI_SSCCTL6, temp, SBI_ICLK);
1882
1883         mutex_unlock(&dev_priv->sb_lock);
1884 }
1885
1886 /* Program iCLKIP clock to the desired frequency */
1887 static void lpt_program_iclkip(const struct intel_crtc_state *crtc_state)
1888 {
1889         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1890         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1891         int clock = crtc_state->hw.adjusted_mode.crtc_clock;
1892         u32 divsel, phaseinc, auxdiv, phasedir = 0;
1893         u32 temp;
1894
1895         lpt_disable_iclkip(dev_priv);
1896
1897         /* The iCLK virtual clock root frequency is in MHz,
1898          * but the adjusted_mode->crtc_clock in in KHz. To get the
1899          * divisors, it is necessary to divide one by another, so we
1900          * convert the virtual clock precision to KHz here for higher
1901          * precision.
1902          */
1903         for (auxdiv = 0; auxdiv < 2; auxdiv++) {
1904                 u32 iclk_virtual_root_freq = 172800 * 1000;
1905                 u32 iclk_pi_range = 64;
1906                 u32 desired_divisor;
1907
1908                 desired_divisor = DIV_ROUND_CLOSEST(iclk_virtual_root_freq,
1909                                                     clock << auxdiv);
1910                 divsel = (desired_divisor / iclk_pi_range) - 2;
1911                 phaseinc = desired_divisor % iclk_pi_range;
1912
1913                 /*
1914                  * Near 20MHz is a corner case which is
1915                  * out of range for the 7-bit divisor
1916                  */
1917                 if (divsel <= 0x7f)
1918                         break;
1919         }
1920
1921         /* This should not happen with any sane values */
1922         drm_WARN_ON(&dev_priv->drm, SBI_SSCDIVINTPHASE_DIVSEL(divsel) &
1923                     ~SBI_SSCDIVINTPHASE_DIVSEL_MASK);
1924         drm_WARN_ON(&dev_priv->drm, SBI_SSCDIVINTPHASE_DIR(phasedir) &
1925                     ~SBI_SSCDIVINTPHASE_INCVAL_MASK);
1926
1927         drm_dbg_kms(&dev_priv->drm,
1928                     "iCLKIP clock: found settings for %dKHz refresh rate: auxdiv=%x, divsel=%x, phasedir=%x, phaseinc=%x\n",
1929                     clock, auxdiv, divsel, phasedir, phaseinc);
1930
1931         mutex_lock(&dev_priv->sb_lock);
1932
1933         /* Program SSCDIVINTPHASE6 */
1934         temp = intel_sbi_read(dev_priv, SBI_SSCDIVINTPHASE6, SBI_ICLK);
1935         temp &= ~SBI_SSCDIVINTPHASE_DIVSEL_MASK;
1936         temp |= SBI_SSCDIVINTPHASE_DIVSEL(divsel);
1937         temp &= ~SBI_SSCDIVINTPHASE_INCVAL_MASK;
1938         temp |= SBI_SSCDIVINTPHASE_INCVAL(phaseinc);
1939         temp |= SBI_SSCDIVINTPHASE_DIR(phasedir);
1940         temp |= SBI_SSCDIVINTPHASE_PROPAGATE;
1941         intel_sbi_write(dev_priv, SBI_SSCDIVINTPHASE6, temp, SBI_ICLK);
1942
1943         /* Program SSCAUXDIV */
1944         temp = intel_sbi_read(dev_priv, SBI_SSCAUXDIV6, SBI_ICLK);
1945         temp &= ~SBI_SSCAUXDIV_FINALDIV2SEL(1);
1946         temp |= SBI_SSCAUXDIV_FINALDIV2SEL(auxdiv);
1947         intel_sbi_write(dev_priv, SBI_SSCAUXDIV6, temp, SBI_ICLK);
1948
1949         /* Enable modulator and associated divider */
1950         temp = intel_sbi_read(dev_priv, SBI_SSCCTL6, SBI_ICLK);
1951         temp &= ~SBI_SSCCTL_DISABLE;
1952         intel_sbi_write(dev_priv, SBI_SSCCTL6, temp, SBI_ICLK);
1953
1954         mutex_unlock(&dev_priv->sb_lock);
1955
1956         /* Wait for initialization time */
1957         udelay(24);
1958
1959         intel_de_write(dev_priv, PIXCLK_GATE, PIXCLK_GATE_UNGATE);
1960 }
1961
1962 int lpt_get_iclkip(struct drm_i915_private *dev_priv)
1963 {
1964         u32 divsel, phaseinc, auxdiv;
1965         u32 iclk_virtual_root_freq = 172800 * 1000;
1966         u32 iclk_pi_range = 64;
1967         u32 desired_divisor;
1968         u32 temp;
1969
1970         if ((intel_de_read(dev_priv, PIXCLK_GATE) & PIXCLK_GATE_UNGATE) == 0)
1971                 return 0;
1972
1973         mutex_lock(&dev_priv->sb_lock);
1974
1975         temp = intel_sbi_read(dev_priv, SBI_SSCCTL6, SBI_ICLK);
1976         if (temp & SBI_SSCCTL_DISABLE) {
1977                 mutex_unlock(&dev_priv->sb_lock);
1978                 return 0;
1979         }
1980
1981         temp = intel_sbi_read(dev_priv, SBI_SSCDIVINTPHASE6, SBI_ICLK);
1982         divsel = (temp & SBI_SSCDIVINTPHASE_DIVSEL_MASK) >>
1983                 SBI_SSCDIVINTPHASE_DIVSEL_SHIFT;
1984         phaseinc = (temp & SBI_SSCDIVINTPHASE_INCVAL_MASK) >>
1985                 SBI_SSCDIVINTPHASE_INCVAL_SHIFT;
1986
1987         temp = intel_sbi_read(dev_priv, SBI_SSCAUXDIV6, SBI_ICLK);
1988         auxdiv = (temp & SBI_SSCAUXDIV_FINALDIV2SEL_MASK) >>
1989                 SBI_SSCAUXDIV_FINALDIV2SEL_SHIFT;
1990
1991         mutex_unlock(&dev_priv->sb_lock);
1992
1993         desired_divisor = (divsel + 2) * iclk_pi_range + phaseinc;
1994
1995         return DIV_ROUND_CLOSEST(iclk_virtual_root_freq,
1996                                  desired_divisor << auxdiv);
1997 }
1998
1999 static void ilk_pch_transcoder_set_timings(const struct intel_crtc_state *crtc_state,
2000                                            enum pipe pch_transcoder)
2001 {
2002         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2003         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2004         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
2005
2006         intel_de_write(dev_priv, PCH_TRANS_HTOTAL(pch_transcoder),
2007                        intel_de_read(dev_priv, HTOTAL(cpu_transcoder)));
2008         intel_de_write(dev_priv, PCH_TRANS_HBLANK(pch_transcoder),
2009                        intel_de_read(dev_priv, HBLANK(cpu_transcoder)));
2010         intel_de_write(dev_priv, PCH_TRANS_HSYNC(pch_transcoder),
2011                        intel_de_read(dev_priv, HSYNC(cpu_transcoder)));
2012
2013         intel_de_write(dev_priv, PCH_TRANS_VTOTAL(pch_transcoder),
2014                        intel_de_read(dev_priv, VTOTAL(cpu_transcoder)));
2015         intel_de_write(dev_priv, PCH_TRANS_VBLANK(pch_transcoder),
2016                        intel_de_read(dev_priv, VBLANK(cpu_transcoder)));
2017         intel_de_write(dev_priv, PCH_TRANS_VSYNC(pch_transcoder),
2018                        intel_de_read(dev_priv, VSYNC(cpu_transcoder)));
2019         intel_de_write(dev_priv, PCH_TRANS_VSYNCSHIFT(pch_transcoder),
2020                        intel_de_read(dev_priv, VSYNCSHIFT(cpu_transcoder)));
2021 }
2022
2023 /*
2024  * Finds the encoder associated with the given CRTC. This can only be
2025  * used when we know that the CRTC isn't feeding multiple encoders!
2026  */
2027 struct intel_encoder *
2028 intel_get_crtc_new_encoder(const struct intel_atomic_state *state,
2029                            const struct intel_crtc_state *crtc_state)
2030 {
2031         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2032         const struct drm_connector_state *connector_state;
2033         const struct drm_connector *connector;
2034         struct intel_encoder *encoder = NULL;
2035         int num_encoders = 0;
2036         int i;
2037
2038         for_each_new_connector_in_state(&state->base, connector, connector_state, i) {
2039                 if (connector_state->crtc != &crtc->base)
2040                         continue;
2041
2042                 encoder = to_intel_encoder(connector_state->best_encoder);
2043                 num_encoders++;
2044         }
2045
2046         drm_WARN(encoder->base.dev, num_encoders != 1,
2047                  "%d encoders for pipe %c\n",
2048                  num_encoders, pipe_name(crtc->pipe));
2049
2050         return encoder;
2051 }
2052
2053 /*
2054  * Enable PCH resources required for PCH ports:
2055  *   - PCH PLLs
2056  *   - FDI training & RX/TX
2057  *   - update transcoder timings
2058  *   - DP transcoding bits
2059  *   - transcoder
2060  */
2061 static void ilk_pch_enable(const struct intel_atomic_state *state,
2062                            const struct intel_crtc_state *crtc_state)
2063 {
2064         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2065         struct drm_device *dev = crtc->base.dev;
2066         struct drm_i915_private *dev_priv = to_i915(dev);
2067         enum pipe pipe = crtc->pipe;
2068         u32 temp;
2069
2070         assert_pch_transcoder_disabled(dev_priv, pipe);
2071
2072         /* For PCH output, training FDI link */
2073         dev_priv->display.fdi_link_train(crtc, crtc_state);
2074
2075         /* We need to program the right clock selection before writing the pixel
2076          * mutliplier into the DPLL. */
2077         if (HAS_PCH_CPT(dev_priv)) {
2078                 u32 sel;
2079
2080                 temp = intel_de_read(dev_priv, PCH_DPLL_SEL);
2081                 temp |= TRANS_DPLL_ENABLE(pipe);
2082                 sel = TRANS_DPLLB_SEL(pipe);
2083                 if (crtc_state->shared_dpll ==
2084                     intel_get_shared_dpll_by_id(dev_priv, DPLL_ID_PCH_PLL_B))
2085                         temp |= sel;
2086                 else
2087                         temp &= ~sel;
2088                 intel_de_write(dev_priv, PCH_DPLL_SEL, temp);
2089         }
2090
2091         /* XXX: pch pll's can be enabled any time before we enable the PCH
2092          * transcoder, and we actually should do this to not upset any PCH
2093          * transcoder that already use the clock when we share it.
2094          *
2095          * Note that enable_shared_dpll tries to do the right thing, but
2096          * get_shared_dpll unconditionally resets the pll - we need that to have
2097          * the right LVDS enable sequence. */
2098         intel_enable_shared_dpll(crtc_state);
2099
2100         /* set transcoder timing, panel must allow it */
2101         assert_panel_unlocked(dev_priv, pipe);
2102         ilk_pch_transcoder_set_timings(crtc_state, pipe);
2103
2104         intel_fdi_normal_train(crtc);
2105
2106         /* For PCH DP, enable TRANS_DP_CTL */
2107         if (HAS_PCH_CPT(dev_priv) &&
2108             intel_crtc_has_dp_encoder(crtc_state)) {
2109                 const struct drm_display_mode *adjusted_mode =
2110                         &crtc_state->hw.adjusted_mode;
2111                 u32 bpc = (intel_de_read(dev_priv, PIPECONF(pipe)) & PIPECONF_BPC_MASK) >> 5;
2112                 i915_reg_t reg = TRANS_DP_CTL(pipe);
2113                 enum port port;
2114
2115                 temp = intel_de_read(dev_priv, reg);
2116                 temp &= ~(TRANS_DP_PORT_SEL_MASK |
2117                           TRANS_DP_SYNC_MASK |
2118                           TRANS_DP_BPC_MASK);
2119                 temp |= TRANS_DP_OUTPUT_ENABLE;
2120                 temp |= bpc << 9; /* same format but at 11:9 */
2121
2122                 if (adjusted_mode->flags & DRM_MODE_FLAG_PHSYNC)
2123                         temp |= TRANS_DP_HSYNC_ACTIVE_HIGH;
2124                 if (adjusted_mode->flags & DRM_MODE_FLAG_PVSYNC)
2125                         temp |= TRANS_DP_VSYNC_ACTIVE_HIGH;
2126
2127                 port = intel_get_crtc_new_encoder(state, crtc_state)->port;
2128                 drm_WARN_ON(dev, port < PORT_B || port > PORT_D);
2129                 temp |= TRANS_DP_PORT_SEL(port);
2130
2131                 intel_de_write(dev_priv, reg, temp);
2132         }
2133
2134         ilk_enable_pch_transcoder(crtc_state);
2135 }
2136
2137 void lpt_pch_enable(const struct intel_crtc_state *crtc_state)
2138 {
2139         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2140         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2141         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
2142
2143         assert_pch_transcoder_disabled(dev_priv, PIPE_A);
2144
2145         lpt_program_iclkip(crtc_state);
2146
2147         /* Set transcoder timing. */
2148         ilk_pch_transcoder_set_timings(crtc_state, PIPE_A);
2149
2150         lpt_enable_pch_transcoder(dev_priv, cpu_transcoder);
2151 }
2152
2153 static void cpt_verify_modeset(struct drm_i915_private *dev_priv,
2154                                enum pipe pipe)
2155 {
2156         i915_reg_t dslreg = PIPEDSL(pipe);
2157         u32 temp;
2158
2159         temp = intel_de_read(dev_priv, dslreg);
2160         udelay(500);
2161         if (wait_for(intel_de_read(dev_priv, dslreg) != temp, 5)) {
2162                 if (wait_for(intel_de_read(dev_priv, dslreg) != temp, 5))
2163                         drm_err(&dev_priv->drm,
2164                                 "mode set failed: pipe %c stuck\n",
2165                                 pipe_name(pipe));
2166         }
2167 }
2168
2169 static void ilk_pfit_enable(const struct intel_crtc_state *crtc_state)
2170 {
2171         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2172         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2173         const struct drm_rect *dst = &crtc_state->pch_pfit.dst;
2174         enum pipe pipe = crtc->pipe;
2175         int width = drm_rect_width(dst);
2176         int height = drm_rect_height(dst);
2177         int x = dst->x1;
2178         int y = dst->y1;
2179
2180         if (!crtc_state->pch_pfit.enabled)
2181                 return;
2182
2183         /* Force use of hard-coded filter coefficients
2184          * as some pre-programmed values are broken,
2185          * e.g. x201.
2186          */
2187         if (IS_IVYBRIDGE(dev_priv) || IS_HASWELL(dev_priv))
2188                 intel_de_write(dev_priv, PF_CTL(pipe), PF_ENABLE |
2189                                PF_FILTER_MED_3x3 | PF_PIPE_SEL_IVB(pipe));
2190         else
2191                 intel_de_write(dev_priv, PF_CTL(pipe), PF_ENABLE |
2192                                PF_FILTER_MED_3x3);
2193         intel_de_write(dev_priv, PF_WIN_POS(pipe), x << 16 | y);
2194         intel_de_write(dev_priv, PF_WIN_SZ(pipe), width << 16 | height);
2195 }
2196
2197 void hsw_enable_ips(const struct intel_crtc_state *crtc_state)
2198 {
2199         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2200         struct drm_device *dev = crtc->base.dev;
2201         struct drm_i915_private *dev_priv = to_i915(dev);
2202
2203         if (!crtc_state->ips_enabled)
2204                 return;
2205
2206         /*
2207          * We can only enable IPS after we enable a plane and wait for a vblank
2208          * This function is called from post_plane_update, which is run after
2209          * a vblank wait.
2210          */
2211         drm_WARN_ON(dev, !(crtc_state->active_planes & ~BIT(PLANE_CURSOR)));
2212
2213         if (IS_BROADWELL(dev_priv)) {
2214                 drm_WARN_ON(dev, sandybridge_pcode_write(dev_priv, DISPLAY_IPS_CONTROL,
2215                                                          IPS_ENABLE | IPS_PCODE_CONTROL));
2216                 /* Quoting Art Runyan: "its not safe to expect any particular
2217                  * value in IPS_CTL bit 31 after enabling IPS through the
2218                  * mailbox." Moreover, the mailbox may return a bogus state,
2219                  * so we need to just enable it and continue on.
2220                  */
2221         } else {
2222                 intel_de_write(dev_priv, IPS_CTL, IPS_ENABLE);
2223                 /* The bit only becomes 1 in the next vblank, so this wait here
2224                  * is essentially intel_wait_for_vblank. If we don't have this
2225                  * and don't wait for vblanks until the end of crtc_enable, then
2226                  * the HW state readout code will complain that the expected
2227                  * IPS_CTL value is not the one we read. */
2228                 if (intel_de_wait_for_set(dev_priv, IPS_CTL, IPS_ENABLE, 50))
2229                         drm_err(&dev_priv->drm,
2230                                 "Timed out waiting for IPS enable\n");
2231         }
2232 }
2233
2234 void hsw_disable_ips(const struct intel_crtc_state *crtc_state)
2235 {
2236         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2237         struct drm_device *dev = crtc->base.dev;
2238         struct drm_i915_private *dev_priv = to_i915(dev);
2239
2240         if (!crtc_state->ips_enabled)
2241                 return;
2242
2243         if (IS_BROADWELL(dev_priv)) {
2244                 drm_WARN_ON(dev,
2245                             sandybridge_pcode_write(dev_priv, DISPLAY_IPS_CONTROL, 0));
2246                 /*
2247                  * Wait for PCODE to finish disabling IPS. The BSpec specified
2248                  * 42ms timeout value leads to occasional timeouts so use 100ms
2249                  * instead.
2250                  */
2251                 if (intel_de_wait_for_clear(dev_priv, IPS_CTL, IPS_ENABLE, 100))
2252                         drm_err(&dev_priv->drm,
2253                                 "Timed out waiting for IPS disable\n");
2254         } else {
2255                 intel_de_write(dev_priv, IPS_CTL, 0);
2256                 intel_de_posting_read(dev_priv, IPS_CTL);
2257         }
2258
2259         /* We need to wait for a vblank before we can disable the plane. */
2260         intel_wait_for_vblank(dev_priv, crtc->pipe);
2261 }
2262
2263 static void intel_crtc_dpms_overlay_disable(struct intel_crtc *crtc)
2264 {
2265         if (crtc->overlay)
2266                 (void) intel_overlay_switch_off(crtc->overlay);
2267
2268         /* Let userspace switch the overlay on again. In most cases userspace
2269          * has to recompute where to put it anyway.
2270          */
2271 }
2272
2273 static bool hsw_pre_update_disable_ips(const struct intel_crtc_state *old_crtc_state,
2274                                        const struct intel_crtc_state *new_crtc_state)
2275 {
2276         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
2277         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2278
2279         if (!old_crtc_state->ips_enabled)
2280                 return false;
2281
2282         if (intel_crtc_needs_modeset(new_crtc_state))
2283                 return true;
2284
2285         /*
2286          * Workaround : Do not read or write the pipe palette/gamma data while
2287          * GAMMA_MODE is configured for split gamma and IPS_CTL has IPS enabled.
2288          *
2289          * Disable IPS before we program the LUT.
2290          */
2291         if (IS_HASWELL(dev_priv) &&
2292             (new_crtc_state->uapi.color_mgmt_changed ||
2293              new_crtc_state->update_pipe) &&
2294             new_crtc_state->gamma_mode == GAMMA_MODE_MODE_SPLIT)
2295                 return true;
2296
2297         return !new_crtc_state->ips_enabled;
2298 }
2299
2300 static bool hsw_post_update_enable_ips(const struct intel_crtc_state *old_crtc_state,
2301                                        const struct intel_crtc_state *new_crtc_state)
2302 {
2303         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
2304         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2305
2306         if (!new_crtc_state->ips_enabled)
2307                 return false;
2308
2309         if (intel_crtc_needs_modeset(new_crtc_state))
2310                 return true;
2311
2312         /*
2313          * Workaround : Do not read or write the pipe palette/gamma data while
2314          * GAMMA_MODE is configured for split gamma and IPS_CTL has IPS enabled.
2315          *
2316          * Re-enable IPS after the LUT has been programmed.
2317          */
2318         if (IS_HASWELL(dev_priv) &&
2319             (new_crtc_state->uapi.color_mgmt_changed ||
2320              new_crtc_state->update_pipe) &&
2321             new_crtc_state->gamma_mode == GAMMA_MODE_MODE_SPLIT)
2322                 return true;
2323
2324         /*
2325          * We can't read out IPS on broadwell, assume the worst and
2326          * forcibly enable IPS on the first fastset.
2327          */
2328         if (new_crtc_state->update_pipe && old_crtc_state->inherited)
2329                 return true;
2330
2331         return !old_crtc_state->ips_enabled;
2332 }
2333
2334 static bool needs_nv12_wa(const struct intel_crtc_state *crtc_state)
2335 {
2336         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
2337
2338         if (!crtc_state->nv12_planes)
2339                 return false;
2340
2341         /* WA Display #0827: Gen9:all */
2342         if (DISPLAY_VER(dev_priv) == 9)
2343                 return true;
2344
2345         return false;
2346 }
2347
2348 static bool needs_scalerclk_wa(const struct intel_crtc_state *crtc_state)
2349 {
2350         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
2351
2352         /* Wa_2006604312:icl,ehl */
2353         if (crtc_state->scaler_state.scaler_users > 0 && DISPLAY_VER(dev_priv) == 11)
2354                 return true;
2355
2356         return false;
2357 }
2358
2359 static bool planes_enabling(const struct intel_crtc_state *old_crtc_state,
2360                             const struct intel_crtc_state *new_crtc_state)
2361 {
2362         return (!old_crtc_state->active_planes || intel_crtc_needs_modeset(new_crtc_state)) &&
2363                 new_crtc_state->active_planes;
2364 }
2365
2366 static bool planes_disabling(const struct intel_crtc_state *old_crtc_state,
2367                              const struct intel_crtc_state *new_crtc_state)
2368 {
2369         return old_crtc_state->active_planes &&
2370                 (!new_crtc_state->active_planes || intel_crtc_needs_modeset(new_crtc_state));
2371 }
2372
2373 static void intel_post_plane_update(struct intel_atomic_state *state,
2374                                     struct intel_crtc *crtc)
2375 {
2376         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2377         const struct intel_crtc_state *old_crtc_state =
2378                 intel_atomic_get_old_crtc_state(state, crtc);
2379         const struct intel_crtc_state *new_crtc_state =
2380                 intel_atomic_get_new_crtc_state(state, crtc);
2381         enum pipe pipe = crtc->pipe;
2382
2383         intel_frontbuffer_flip(dev_priv, new_crtc_state->fb_bits);
2384
2385         if (new_crtc_state->update_wm_post && new_crtc_state->hw.active)
2386                 intel_update_watermarks(crtc);
2387
2388         if (hsw_post_update_enable_ips(old_crtc_state, new_crtc_state))
2389                 hsw_enable_ips(new_crtc_state);
2390
2391         intel_fbc_post_update(state, crtc);
2392         intel_drrs_page_flip(state, crtc);
2393
2394         if (needs_nv12_wa(old_crtc_state) &&
2395             !needs_nv12_wa(new_crtc_state))
2396                 skl_wa_827(dev_priv, pipe, false);
2397
2398         if (needs_scalerclk_wa(old_crtc_state) &&
2399             !needs_scalerclk_wa(new_crtc_state))
2400                 icl_wa_scalerclkgating(dev_priv, pipe, false);
2401 }
2402
2403 static void intel_crtc_enable_flip_done(struct intel_atomic_state *state,
2404                                         struct intel_crtc *crtc)
2405 {
2406         const struct intel_crtc_state *crtc_state =
2407                 intel_atomic_get_new_crtc_state(state, crtc);
2408         u8 update_planes = crtc_state->update_planes;
2409         const struct intel_plane_state *plane_state;
2410         struct intel_plane *plane;
2411         int i;
2412
2413         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
2414                 if (plane->enable_flip_done &&
2415                     plane->pipe == crtc->pipe &&
2416                     update_planes & BIT(plane->id))
2417                         plane->enable_flip_done(plane);
2418         }
2419 }
2420
2421 static void intel_crtc_disable_flip_done(struct intel_atomic_state *state,
2422                                          struct intel_crtc *crtc)
2423 {
2424         const struct intel_crtc_state *crtc_state =
2425                 intel_atomic_get_new_crtc_state(state, crtc);
2426         u8 update_planes = crtc_state->update_planes;
2427         const struct intel_plane_state *plane_state;
2428         struct intel_plane *plane;
2429         int i;
2430
2431         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
2432                 if (plane->disable_flip_done &&
2433                     plane->pipe == crtc->pipe &&
2434                     update_planes & BIT(plane->id))
2435                         plane->disable_flip_done(plane);
2436         }
2437 }
2438
2439 static void intel_crtc_async_flip_disable_wa(struct intel_atomic_state *state,
2440                                              struct intel_crtc *crtc)
2441 {
2442         struct drm_i915_private *i915 = to_i915(state->base.dev);
2443         const struct intel_crtc_state *old_crtc_state =
2444                 intel_atomic_get_old_crtc_state(state, crtc);
2445         const struct intel_crtc_state *new_crtc_state =
2446                 intel_atomic_get_new_crtc_state(state, crtc);
2447         u8 update_planes = new_crtc_state->update_planes;
2448         const struct intel_plane_state *old_plane_state;
2449         struct intel_plane *plane;
2450         bool need_vbl_wait = false;
2451         int i;
2452
2453         for_each_old_intel_plane_in_state(state, plane, old_plane_state, i) {
2454                 if (plane->need_async_flip_disable_wa &&
2455                     plane->pipe == crtc->pipe &&
2456                     update_planes & BIT(plane->id)) {
2457                         /*
2458                          * Apart from the async flip bit we want to
2459                          * preserve the old state for the plane.
2460                          */
2461                         plane->async_flip(plane, old_crtc_state,
2462                                           old_plane_state, false);
2463                         need_vbl_wait = true;
2464                 }
2465         }
2466
2467         if (need_vbl_wait)
2468                 intel_wait_for_vblank(i915, crtc->pipe);
2469 }
2470
2471 static void intel_pre_plane_update(struct intel_atomic_state *state,
2472                                    struct intel_crtc *crtc)
2473 {
2474         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2475         const struct intel_crtc_state *old_crtc_state =
2476                 intel_atomic_get_old_crtc_state(state, crtc);
2477         const struct intel_crtc_state *new_crtc_state =
2478                 intel_atomic_get_new_crtc_state(state, crtc);
2479         enum pipe pipe = crtc->pipe;
2480
2481         if (hsw_pre_update_disable_ips(old_crtc_state, new_crtc_state))
2482                 hsw_disable_ips(old_crtc_state);
2483
2484         if (intel_fbc_pre_update(state, crtc))
2485                 intel_wait_for_vblank(dev_priv, pipe);
2486
2487         /* Display WA 827 */
2488         if (!needs_nv12_wa(old_crtc_state) &&
2489             needs_nv12_wa(new_crtc_state))
2490                 skl_wa_827(dev_priv, pipe, true);
2491
2492         /* Wa_2006604312:icl,ehl */
2493         if (!needs_scalerclk_wa(old_crtc_state) &&
2494             needs_scalerclk_wa(new_crtc_state))
2495                 icl_wa_scalerclkgating(dev_priv, pipe, true);
2496
2497         /*
2498          * Vblank time updates from the shadow to live plane control register
2499          * are blocked if the memory self-refresh mode is active at that
2500          * moment. So to make sure the plane gets truly disabled, disable
2501          * first the self-refresh mode. The self-refresh enable bit in turn
2502          * will be checked/applied by the HW only at the next frame start
2503          * event which is after the vblank start event, so we need to have a
2504          * wait-for-vblank between disabling the plane and the pipe.
2505          */
2506         if (HAS_GMCH(dev_priv) && old_crtc_state->hw.active &&
2507             new_crtc_state->disable_cxsr && intel_set_memory_cxsr(dev_priv, false))
2508                 intel_wait_for_vblank(dev_priv, pipe);
2509
2510         /*
2511          * IVB workaround: must disable low power watermarks for at least
2512          * one frame before enabling scaling.  LP watermarks can be re-enabled
2513          * when scaling is disabled.
2514          *
2515          * WaCxSRDisabledForSpriteScaling:ivb
2516          */
2517         if (old_crtc_state->hw.active &&
2518             new_crtc_state->disable_lp_wm && ilk_disable_lp_wm(dev_priv))
2519                 intel_wait_for_vblank(dev_priv, pipe);
2520
2521         /*
2522          * If we're doing a modeset we don't need to do any
2523          * pre-vblank watermark programming here.
2524          */
2525         if (!intel_crtc_needs_modeset(new_crtc_state)) {
2526                 /*
2527                  * For platforms that support atomic watermarks, program the
2528                  * 'intermediate' watermarks immediately.  On pre-gen9 platforms, these
2529                  * will be the intermediate values that are safe for both pre- and
2530                  * post- vblank; when vblank happens, the 'active' values will be set
2531                  * to the final 'target' values and we'll do this again to get the
2532                  * optimal watermarks.  For gen9+ platforms, the values we program here
2533                  * will be the final target values which will get automatically latched
2534                  * at vblank time; no further programming will be necessary.
2535                  *
2536                  * If a platform hasn't been transitioned to atomic watermarks yet,
2537                  * we'll continue to update watermarks the old way, if flags tell
2538                  * us to.
2539                  */
2540                 if (dev_priv->display.initial_watermarks)
2541                         dev_priv->display.initial_watermarks(state, crtc);
2542                 else if (new_crtc_state->update_wm_pre)
2543                         intel_update_watermarks(crtc);
2544         }
2545
2546         /*
2547          * Gen2 reports pipe underruns whenever all planes are disabled.
2548          * So disable underrun reporting before all the planes get disabled.
2549          *
2550          * We do this after .initial_watermarks() so that we have a
2551          * chance of catching underruns with the intermediate watermarks
2552          * vs. the old plane configuration.
2553          */
2554         if (DISPLAY_VER(dev_priv) == 2 && planes_disabling(old_crtc_state, new_crtc_state))
2555                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
2556
2557         /*
2558          * WA for platforms where async address update enable bit
2559          * is double buffered and only latched at start of vblank.
2560          */
2561         if (old_crtc_state->uapi.async_flip && !new_crtc_state->uapi.async_flip)
2562                 intel_crtc_async_flip_disable_wa(state, crtc);
2563 }
2564
2565 static void intel_crtc_disable_planes(struct intel_atomic_state *state,
2566                                       struct intel_crtc *crtc)
2567 {
2568         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2569         const struct intel_crtc_state *new_crtc_state =
2570                 intel_atomic_get_new_crtc_state(state, crtc);
2571         unsigned int update_mask = new_crtc_state->update_planes;
2572         const struct intel_plane_state *old_plane_state;
2573         struct intel_plane *plane;
2574         unsigned fb_bits = 0;
2575         int i;
2576
2577         intel_crtc_dpms_overlay_disable(crtc);
2578
2579         for_each_old_intel_plane_in_state(state, plane, old_plane_state, i) {
2580                 if (crtc->pipe != plane->pipe ||
2581                     !(update_mask & BIT(plane->id)))
2582                         continue;
2583
2584                 intel_disable_plane(plane, new_crtc_state);
2585
2586                 if (old_plane_state->uapi.visible)
2587                         fb_bits |= plane->frontbuffer_bit;
2588         }
2589
2590         intel_frontbuffer_flip(dev_priv, fb_bits);
2591 }
2592
2593 /*
2594  * intel_connector_primary_encoder - get the primary encoder for a connector
2595  * @connector: connector for which to return the encoder
2596  *
2597  * Returns the primary encoder for a connector. There is a 1:1 mapping from
2598  * all connectors to their encoder, except for DP-MST connectors which have
2599  * both a virtual and a primary encoder. These DP-MST primary encoders can be
2600  * pointed to by as many DP-MST connectors as there are pipes.
2601  */
2602 static struct intel_encoder *
2603 intel_connector_primary_encoder(struct intel_connector *connector)
2604 {
2605         struct intel_encoder *encoder;
2606
2607         if (connector->mst_port)
2608                 return &dp_to_dig_port(connector->mst_port)->base;
2609
2610         encoder = intel_attached_encoder(connector);
2611         drm_WARN_ON(connector->base.dev, !encoder);
2612
2613         return encoder;
2614 }
2615
2616 static void intel_encoders_update_prepare(struct intel_atomic_state *state)
2617 {
2618         struct drm_connector_state *new_conn_state;
2619         struct drm_connector *connector;
2620         int i;
2621
2622         for_each_new_connector_in_state(&state->base, connector, new_conn_state,
2623                                         i) {
2624                 struct intel_connector *intel_connector;
2625                 struct intel_encoder *encoder;
2626                 struct intel_crtc *crtc;
2627
2628                 if (!intel_connector_needs_modeset(state, connector))
2629                         continue;
2630
2631                 intel_connector = to_intel_connector(connector);
2632                 encoder = intel_connector_primary_encoder(intel_connector);
2633                 if (!encoder->update_prepare)
2634                         continue;
2635
2636                 crtc = new_conn_state->crtc ?
2637                         to_intel_crtc(new_conn_state->crtc) : NULL;
2638                 encoder->update_prepare(state, encoder, crtc);
2639         }
2640 }
2641
2642 static void intel_encoders_update_complete(struct intel_atomic_state *state)
2643 {
2644         struct drm_connector_state *new_conn_state;
2645         struct drm_connector *connector;
2646         int i;
2647
2648         for_each_new_connector_in_state(&state->base, connector, new_conn_state,
2649                                         i) {
2650                 struct intel_connector *intel_connector;
2651                 struct intel_encoder *encoder;
2652                 struct intel_crtc *crtc;
2653
2654                 if (!intel_connector_needs_modeset(state, connector))
2655                         continue;
2656
2657                 intel_connector = to_intel_connector(connector);
2658                 encoder = intel_connector_primary_encoder(intel_connector);
2659                 if (!encoder->update_complete)
2660                         continue;
2661
2662                 crtc = new_conn_state->crtc ?
2663                         to_intel_crtc(new_conn_state->crtc) : NULL;
2664                 encoder->update_complete(state, encoder, crtc);
2665         }
2666 }
2667
2668 static void intel_encoders_pre_pll_enable(struct intel_atomic_state *state,
2669                                           struct intel_crtc *crtc)
2670 {
2671         const struct intel_crtc_state *crtc_state =
2672                 intel_atomic_get_new_crtc_state(state, crtc);
2673         const struct drm_connector_state *conn_state;
2674         struct drm_connector *conn;
2675         int i;
2676
2677         for_each_new_connector_in_state(&state->base, conn, conn_state, i) {
2678                 struct intel_encoder *encoder =
2679                         to_intel_encoder(conn_state->best_encoder);
2680
2681                 if (conn_state->crtc != &crtc->base)
2682                         continue;
2683
2684                 if (encoder->pre_pll_enable)
2685                         encoder->pre_pll_enable(state, encoder,
2686                                                 crtc_state, conn_state);
2687         }
2688 }
2689
2690 static void intel_encoders_pre_enable(struct intel_atomic_state *state,
2691                                       struct intel_crtc *crtc)
2692 {
2693         const struct intel_crtc_state *crtc_state =
2694                 intel_atomic_get_new_crtc_state(state, crtc);
2695         const struct drm_connector_state *conn_state;
2696         struct drm_connector *conn;
2697         int i;
2698
2699         for_each_new_connector_in_state(&state->base, conn, conn_state, i) {
2700                 struct intel_encoder *encoder =
2701                         to_intel_encoder(conn_state->best_encoder);
2702
2703                 if (conn_state->crtc != &crtc->base)
2704                         continue;
2705
2706                 if (encoder->pre_enable)
2707                         encoder->pre_enable(state, encoder,
2708                                             crtc_state, conn_state);
2709         }
2710 }
2711
2712 static void intel_encoders_enable(struct intel_atomic_state *state,
2713                                   struct intel_crtc *crtc)
2714 {
2715         const struct intel_crtc_state *crtc_state =
2716                 intel_atomic_get_new_crtc_state(state, crtc);
2717         const struct drm_connector_state *conn_state;
2718         struct drm_connector *conn;
2719         int i;
2720
2721         for_each_new_connector_in_state(&state->base, conn, conn_state, i) {
2722                 struct intel_encoder *encoder =
2723                         to_intel_encoder(conn_state->best_encoder);
2724
2725                 if (conn_state->crtc != &crtc->base)
2726                         continue;
2727
2728                 if (encoder->enable)
2729                         encoder->enable(state, encoder,
2730                                         crtc_state, conn_state);
2731                 intel_opregion_notify_encoder(encoder, true);
2732         }
2733 }
2734
2735 static void intel_encoders_pre_disable(struct intel_atomic_state *state,
2736                                        struct intel_crtc *crtc)
2737 {
2738         const struct intel_crtc_state *old_crtc_state =
2739                 intel_atomic_get_old_crtc_state(state, crtc);
2740         const struct drm_connector_state *old_conn_state;
2741         struct drm_connector *conn;
2742         int i;
2743
2744         for_each_old_connector_in_state(&state->base, conn, old_conn_state, i) {
2745                 struct intel_encoder *encoder =
2746                         to_intel_encoder(old_conn_state->best_encoder);
2747
2748                 if (old_conn_state->crtc != &crtc->base)
2749                         continue;
2750
2751                 if (encoder->pre_disable)
2752                         encoder->pre_disable(state, encoder, old_crtc_state,
2753                                              old_conn_state);
2754         }
2755 }
2756
2757 static void intel_encoders_disable(struct intel_atomic_state *state,
2758                                    struct intel_crtc *crtc)
2759 {
2760         const struct intel_crtc_state *old_crtc_state =
2761                 intel_atomic_get_old_crtc_state(state, crtc);
2762         const struct drm_connector_state *old_conn_state;
2763         struct drm_connector *conn;
2764         int i;
2765
2766         for_each_old_connector_in_state(&state->base, conn, old_conn_state, i) {
2767                 struct intel_encoder *encoder =
2768                         to_intel_encoder(old_conn_state->best_encoder);
2769
2770                 if (old_conn_state->crtc != &crtc->base)
2771                         continue;
2772
2773                 intel_opregion_notify_encoder(encoder, false);
2774                 if (encoder->disable)
2775                         encoder->disable(state, encoder,
2776                                          old_crtc_state, old_conn_state);
2777         }
2778 }
2779
2780 static void intel_encoders_post_disable(struct intel_atomic_state *state,
2781                                         struct intel_crtc *crtc)
2782 {
2783         const struct intel_crtc_state *old_crtc_state =
2784                 intel_atomic_get_old_crtc_state(state, crtc);
2785         const struct drm_connector_state *old_conn_state;
2786         struct drm_connector *conn;
2787         int i;
2788
2789         for_each_old_connector_in_state(&state->base, conn, old_conn_state, i) {
2790                 struct intel_encoder *encoder =
2791                         to_intel_encoder(old_conn_state->best_encoder);
2792
2793                 if (old_conn_state->crtc != &crtc->base)
2794                         continue;
2795
2796                 if (encoder->post_disable)
2797                         encoder->post_disable(state, encoder,
2798                                               old_crtc_state, old_conn_state);
2799         }
2800 }
2801
2802 static void intel_encoders_post_pll_disable(struct intel_atomic_state *state,
2803                                             struct intel_crtc *crtc)
2804 {
2805         const struct intel_crtc_state *old_crtc_state =
2806                 intel_atomic_get_old_crtc_state(state, crtc);
2807         const struct drm_connector_state *old_conn_state;
2808         struct drm_connector *conn;
2809         int i;
2810
2811         for_each_old_connector_in_state(&state->base, conn, old_conn_state, i) {
2812                 struct intel_encoder *encoder =
2813                         to_intel_encoder(old_conn_state->best_encoder);
2814
2815                 if (old_conn_state->crtc != &crtc->base)
2816                         continue;
2817
2818                 if (encoder->post_pll_disable)
2819                         encoder->post_pll_disable(state, encoder,
2820                                                   old_crtc_state, old_conn_state);
2821         }
2822 }
2823
2824 static void intel_encoders_update_pipe(struct intel_atomic_state *state,
2825                                        struct intel_crtc *crtc)
2826 {
2827         const struct intel_crtc_state *crtc_state =
2828                 intel_atomic_get_new_crtc_state(state, crtc);
2829         const struct drm_connector_state *conn_state;
2830         struct drm_connector *conn;
2831         int i;
2832
2833         for_each_new_connector_in_state(&state->base, conn, conn_state, i) {
2834                 struct intel_encoder *encoder =
2835                         to_intel_encoder(conn_state->best_encoder);
2836
2837                 if (conn_state->crtc != &crtc->base)
2838                         continue;
2839
2840                 if (encoder->update_pipe)
2841                         encoder->update_pipe(state, encoder,
2842                                              crtc_state, conn_state);
2843         }
2844 }
2845
2846 static void intel_disable_primary_plane(const struct intel_crtc_state *crtc_state)
2847 {
2848         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2849         struct intel_plane *plane = to_intel_plane(crtc->base.primary);
2850
2851         plane->disable_plane(plane, crtc_state);
2852 }
2853
2854 static void ilk_crtc_enable(struct intel_atomic_state *state,
2855                             struct intel_crtc *crtc)
2856 {
2857         const struct intel_crtc_state *new_crtc_state =
2858                 intel_atomic_get_new_crtc_state(state, crtc);
2859         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2860         enum pipe pipe = crtc->pipe;
2861
2862         if (drm_WARN_ON(&dev_priv->drm, crtc->active))
2863                 return;
2864
2865         /*
2866          * Sometimes spurious CPU pipe underruns happen during FDI
2867          * training, at least with VGA+HDMI cloning. Suppress them.
2868          *
2869          * On ILK we get an occasional spurious CPU pipe underruns
2870          * between eDP port A enable and vdd enable. Also PCH port
2871          * enable seems to result in the occasional CPU pipe underrun.
2872          *
2873          * Spurious PCH underruns also occur during PCH enabling.
2874          */
2875         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
2876         intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, false);
2877
2878         if (intel_crtc_has_dp_encoder(new_crtc_state))
2879                 intel_dp_set_m_n(new_crtc_state, M1_N1);
2880
2881         intel_set_transcoder_timings(new_crtc_state);
2882         intel_set_pipe_src_size(new_crtc_state);
2883
2884         if (new_crtc_state->has_pch_encoder)
2885                 intel_cpu_transcoder_set_m_n(new_crtc_state,
2886                                              &new_crtc_state->fdi_m_n, NULL);
2887
2888         ilk_set_pipeconf(new_crtc_state);
2889
2890         crtc->active = true;
2891
2892         intel_encoders_pre_enable(state, crtc);
2893
2894         if (new_crtc_state->has_pch_encoder) {
2895                 /* Note: FDI PLL enabling _must_ be done before we enable the
2896                  * cpu pipes, hence this is separate from all the other fdi/pch
2897                  * enabling. */
2898                 ilk_fdi_pll_enable(new_crtc_state);
2899         } else {
2900                 assert_fdi_tx_disabled(dev_priv, pipe);
2901                 assert_fdi_rx_disabled(dev_priv, pipe);
2902         }
2903
2904         ilk_pfit_enable(new_crtc_state);
2905
2906         /*
2907          * On ILK+ LUT must be loaded before the pipe is running but with
2908          * clocks enabled
2909          */
2910         intel_color_load_luts(new_crtc_state);
2911         intel_color_commit(new_crtc_state);
2912         /* update DSPCNTR to configure gamma for pipe bottom color */
2913         intel_disable_primary_plane(new_crtc_state);
2914
2915         if (dev_priv->display.initial_watermarks)
2916                 dev_priv->display.initial_watermarks(state, crtc);
2917         intel_enable_transcoder(new_crtc_state);
2918
2919         if (new_crtc_state->has_pch_encoder)
2920                 ilk_pch_enable(state, new_crtc_state);
2921
2922         intel_crtc_vblank_on(new_crtc_state);
2923
2924         intel_encoders_enable(state, crtc);
2925
2926         if (HAS_PCH_CPT(dev_priv))
2927                 cpt_verify_modeset(dev_priv, pipe);
2928
2929         /*
2930          * Must wait for vblank to avoid spurious PCH FIFO underruns.
2931          * And a second vblank wait is needed at least on ILK with
2932          * some interlaced HDMI modes. Let's do the double wait always
2933          * in case there are more corner cases we don't know about.
2934          */
2935         if (new_crtc_state->has_pch_encoder) {
2936                 intel_wait_for_vblank(dev_priv, pipe);
2937                 intel_wait_for_vblank(dev_priv, pipe);
2938         }
2939         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
2940         intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, true);
2941 }
2942
2943 /* IPS only exists on ULT machines and is tied to pipe A. */
2944 static bool hsw_crtc_supports_ips(struct intel_crtc *crtc)
2945 {
2946         return HAS_IPS(to_i915(crtc->base.dev)) && crtc->pipe == PIPE_A;
2947 }
2948
2949 static void glk_pipe_scaler_clock_gating_wa(struct drm_i915_private *dev_priv,
2950                                             enum pipe pipe, bool apply)
2951 {
2952         u32 val = intel_de_read(dev_priv, CLKGATE_DIS_PSL(pipe));
2953         u32 mask = DPF_GATING_DIS | DPF_RAM_GATING_DIS | DPFR_GATING_DIS;
2954
2955         if (apply)
2956                 val |= mask;
2957         else
2958                 val &= ~mask;
2959
2960         intel_de_write(dev_priv, CLKGATE_DIS_PSL(pipe), val);
2961 }
2962
2963 static void icl_pipe_mbus_enable(struct intel_crtc *crtc, bool joined_mbus)
2964 {
2965         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2966         enum pipe pipe = crtc->pipe;
2967         u32 val;
2968
2969         /* Wa_22010947358:adl-p */
2970         if (IS_ALDERLAKE_P(dev_priv))
2971                 val = joined_mbus ? MBUS_DBOX_A_CREDIT(6) : MBUS_DBOX_A_CREDIT(4);
2972         else
2973                 val = MBUS_DBOX_A_CREDIT(2);
2974
2975         if (DISPLAY_VER(dev_priv) >= 12) {
2976                 val |= MBUS_DBOX_BW_CREDIT(2);
2977                 val |= MBUS_DBOX_B_CREDIT(12);
2978         } else {
2979                 val |= MBUS_DBOX_BW_CREDIT(1);
2980                 val |= MBUS_DBOX_B_CREDIT(8);
2981         }
2982
2983         intel_de_write(dev_priv, PIPE_MBUS_DBOX_CTL(pipe), val);
2984 }
2985
2986 static void hsw_set_linetime_wm(const struct intel_crtc_state *crtc_state)
2987 {
2988         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2989         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2990
2991         intel_de_write(dev_priv, WM_LINETIME(crtc->pipe),
2992                        HSW_LINETIME(crtc_state->linetime) |
2993                        HSW_IPS_LINETIME(crtc_state->ips_linetime));
2994 }
2995
2996 static void hsw_set_frame_start_delay(const struct intel_crtc_state *crtc_state)
2997 {
2998         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2999         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3000         i915_reg_t reg = CHICKEN_TRANS(crtc_state->cpu_transcoder);
3001         u32 val;
3002
3003         val = intel_de_read(dev_priv, reg);
3004         val &= ~HSW_FRAME_START_DELAY_MASK;
3005         val |= HSW_FRAME_START_DELAY(dev_priv->framestart_delay - 1);
3006         intel_de_write(dev_priv, reg, val);
3007 }
3008
3009 static void icl_ddi_bigjoiner_pre_enable(struct intel_atomic_state *state,
3010                                          const struct intel_crtc_state *crtc_state)
3011 {
3012         struct intel_crtc *master = to_intel_crtc(crtc_state->uapi.crtc);
3013         struct drm_i915_private *dev_priv = to_i915(master->base.dev);
3014         struct intel_crtc_state *master_crtc_state;
3015         struct drm_connector_state *conn_state;
3016         struct drm_connector *conn;
3017         struct intel_encoder *encoder = NULL;
3018         int i;
3019
3020         if (crtc_state->bigjoiner_slave)
3021                 master = crtc_state->bigjoiner_linked_crtc;
3022
3023         master_crtc_state = intel_atomic_get_new_crtc_state(state, master);
3024
3025         for_each_new_connector_in_state(&state->base, conn, conn_state, i) {
3026                 if (conn_state->crtc != &master->base)
3027                         continue;
3028
3029                 encoder = to_intel_encoder(conn_state->best_encoder);
3030                 break;
3031         }
3032
3033         if (!crtc_state->bigjoiner_slave) {
3034                 /* need to enable VDSC, which we skipped in pre-enable */
3035                 intel_dsc_enable(encoder, crtc_state);
3036         } else {
3037                 /*
3038                  * Enable sequence steps 1-7 on bigjoiner master
3039                  */
3040                 intel_encoders_pre_pll_enable(state, master);
3041                 if (master_crtc_state->shared_dpll)
3042                         intel_enable_shared_dpll(master_crtc_state);
3043                 intel_encoders_pre_enable(state, master);
3044
3045                 /* and DSC on slave */
3046                 intel_dsc_enable(NULL, crtc_state);
3047         }
3048
3049         if (DISPLAY_VER(dev_priv) >= 13)
3050                 intel_uncompressed_joiner_enable(crtc_state);
3051 }
3052
3053 static void hsw_crtc_enable(struct intel_atomic_state *state,
3054                             struct intel_crtc *crtc)
3055 {
3056         const struct intel_crtc_state *new_crtc_state =
3057                 intel_atomic_get_new_crtc_state(state, crtc);
3058         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3059         enum pipe pipe = crtc->pipe, hsw_workaround_pipe;
3060         enum transcoder cpu_transcoder = new_crtc_state->cpu_transcoder;
3061         bool psl_clkgate_wa;
3062
3063         if (drm_WARN_ON(&dev_priv->drm, crtc->active))
3064                 return;
3065
3066         if (!new_crtc_state->bigjoiner) {
3067                 intel_encoders_pre_pll_enable(state, crtc);
3068
3069                 if (new_crtc_state->shared_dpll)
3070                         intel_enable_shared_dpll(new_crtc_state);
3071
3072                 intel_encoders_pre_enable(state, crtc);
3073         } else {
3074                 icl_ddi_bigjoiner_pre_enable(state, new_crtc_state);
3075         }
3076
3077         intel_set_pipe_src_size(new_crtc_state);
3078         if (DISPLAY_VER(dev_priv) >= 9 || IS_BROADWELL(dev_priv))
3079                 bdw_set_pipemisc(new_crtc_state);
3080
3081         if (!new_crtc_state->bigjoiner_slave && !transcoder_is_dsi(cpu_transcoder)) {
3082                 intel_set_transcoder_timings(new_crtc_state);
3083
3084                 if (cpu_transcoder != TRANSCODER_EDP)
3085                         intel_de_write(dev_priv, PIPE_MULT(cpu_transcoder),
3086                                        new_crtc_state->pixel_multiplier - 1);
3087
3088                 if (new_crtc_state->has_pch_encoder)
3089                         intel_cpu_transcoder_set_m_n(new_crtc_state,
3090                                                      &new_crtc_state->fdi_m_n, NULL);
3091
3092                 hsw_set_frame_start_delay(new_crtc_state);
3093         }
3094
3095         if (!transcoder_is_dsi(cpu_transcoder))
3096                 hsw_set_transconf(new_crtc_state);
3097
3098         crtc->active = true;
3099
3100         /* Display WA #1180: WaDisableScalarClockGating: glk */
3101         psl_clkgate_wa = DISPLAY_VER(dev_priv) == 10 &&
3102                 new_crtc_state->pch_pfit.enabled;
3103         if (psl_clkgate_wa)
3104                 glk_pipe_scaler_clock_gating_wa(dev_priv, pipe, true);
3105
3106         if (DISPLAY_VER(dev_priv) >= 9)
3107                 skl_pfit_enable(new_crtc_state);
3108         else
3109                 ilk_pfit_enable(new_crtc_state);
3110
3111         /*
3112          * On ILK+ LUT must be loaded before the pipe is running but with
3113          * clocks enabled
3114          */
3115         intel_color_load_luts(new_crtc_state);
3116         intel_color_commit(new_crtc_state);
3117         /* update DSPCNTR to configure gamma/csc for pipe bottom color */
3118         if (DISPLAY_VER(dev_priv) < 9)
3119                 intel_disable_primary_plane(new_crtc_state);
3120
3121         hsw_set_linetime_wm(new_crtc_state);
3122
3123         if (DISPLAY_VER(dev_priv) >= 11)
3124                 icl_set_pipe_chicken(new_crtc_state);
3125
3126         if (dev_priv->display.initial_watermarks)
3127                 dev_priv->display.initial_watermarks(state, crtc);
3128
3129         if (DISPLAY_VER(dev_priv) >= 11) {
3130                 const struct intel_dbuf_state *dbuf_state =
3131                                 intel_atomic_get_new_dbuf_state(state);
3132
3133                 icl_pipe_mbus_enable(crtc, dbuf_state->joined_mbus);
3134         }
3135
3136         if (new_crtc_state->bigjoiner_slave)
3137                 intel_crtc_vblank_on(new_crtc_state);
3138
3139         intel_encoders_enable(state, crtc);
3140
3141         if (psl_clkgate_wa) {
3142                 intel_wait_for_vblank(dev_priv, pipe);
3143                 glk_pipe_scaler_clock_gating_wa(dev_priv, pipe, false);
3144         }
3145
3146         /* If we change the relative order between pipe/planes enabling, we need
3147          * to change the workaround. */
3148         hsw_workaround_pipe = new_crtc_state->hsw_workaround_pipe;
3149         if (IS_HASWELL(dev_priv) && hsw_workaround_pipe != INVALID_PIPE) {
3150                 intel_wait_for_vblank(dev_priv, hsw_workaround_pipe);
3151                 intel_wait_for_vblank(dev_priv, hsw_workaround_pipe);
3152         }
3153 }
3154
3155 void ilk_pfit_disable(const struct intel_crtc_state *old_crtc_state)
3156 {
3157         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->uapi.crtc);
3158         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3159         enum pipe pipe = crtc->pipe;
3160
3161         /* To avoid upsetting the power well on haswell only disable the pfit if
3162          * it's in use. The hw state code will make sure we get this right. */
3163         if (!old_crtc_state->pch_pfit.enabled)
3164                 return;
3165
3166         intel_de_write(dev_priv, PF_CTL(pipe), 0);
3167         intel_de_write(dev_priv, PF_WIN_POS(pipe), 0);
3168         intel_de_write(dev_priv, PF_WIN_SZ(pipe), 0);
3169 }
3170
3171 static void ilk_crtc_disable(struct intel_atomic_state *state,
3172                              struct intel_crtc *crtc)
3173 {
3174         const struct intel_crtc_state *old_crtc_state =
3175                 intel_atomic_get_old_crtc_state(state, crtc);
3176         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3177         enum pipe pipe = crtc->pipe;
3178
3179         /*
3180          * Sometimes spurious CPU pipe underruns happen when the
3181          * pipe is already disabled, but FDI RX/TX is still enabled.
3182          * Happens at least with VGA+HDMI cloning. Suppress them.
3183          */
3184         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
3185         intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, false);
3186
3187         intel_encoders_disable(state, crtc);
3188
3189         intel_crtc_vblank_off(old_crtc_state);
3190
3191         intel_disable_transcoder(old_crtc_state);
3192
3193         ilk_pfit_disable(old_crtc_state);
3194
3195         if (old_crtc_state->has_pch_encoder)
3196                 ilk_fdi_disable(crtc);
3197
3198         intel_encoders_post_disable(state, crtc);
3199
3200         if (old_crtc_state->has_pch_encoder) {
3201                 ilk_disable_pch_transcoder(dev_priv, pipe);
3202
3203                 if (HAS_PCH_CPT(dev_priv)) {
3204                         i915_reg_t reg;
3205                         u32 temp;
3206
3207                         /* disable TRANS_DP_CTL */
3208                         reg = TRANS_DP_CTL(pipe);
3209                         temp = intel_de_read(dev_priv, reg);
3210                         temp &= ~(TRANS_DP_OUTPUT_ENABLE |
3211                                   TRANS_DP_PORT_SEL_MASK);
3212                         temp |= TRANS_DP_PORT_SEL_NONE;
3213                         intel_de_write(dev_priv, reg, temp);
3214
3215                         /* disable DPLL_SEL */
3216                         temp = intel_de_read(dev_priv, PCH_DPLL_SEL);
3217                         temp &= ~(TRANS_DPLL_ENABLE(pipe) | TRANS_DPLLB_SEL(pipe));
3218                         intel_de_write(dev_priv, PCH_DPLL_SEL, temp);
3219                 }
3220
3221                 ilk_fdi_pll_disable(crtc);
3222         }
3223
3224         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
3225         intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, true);
3226 }
3227
3228 static void hsw_crtc_disable(struct intel_atomic_state *state,
3229                              struct intel_crtc *crtc)
3230 {
3231         /*
3232          * FIXME collapse everything to one hook.
3233          * Need care with mst->ddi interactions.
3234          */
3235         intel_encoders_disable(state, crtc);
3236         intel_encoders_post_disable(state, crtc);
3237 }
3238
3239 static void i9xx_pfit_enable(const struct intel_crtc_state *crtc_state)
3240 {
3241         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3242         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3243
3244         if (!crtc_state->gmch_pfit.control)
3245                 return;
3246
3247         /*
3248          * The panel fitter should only be adjusted whilst the pipe is disabled,
3249          * according to register description and PRM.
3250          */
3251         drm_WARN_ON(&dev_priv->drm,
3252                     intel_de_read(dev_priv, PFIT_CONTROL) & PFIT_ENABLE);
3253         assert_transcoder_disabled(dev_priv, crtc_state->cpu_transcoder);
3254
3255         intel_de_write(dev_priv, PFIT_PGM_RATIOS,
3256                        crtc_state->gmch_pfit.pgm_ratios);
3257         intel_de_write(dev_priv, PFIT_CONTROL, crtc_state->gmch_pfit.control);
3258
3259         /* Border color in case we don't scale up to the full screen. Black by
3260          * default, change to something else for debugging. */
3261         intel_de_write(dev_priv, BCLRPAT(crtc->pipe), 0);
3262 }
3263
3264 bool intel_phy_is_combo(struct drm_i915_private *dev_priv, enum phy phy)
3265 {
3266         if (phy == PHY_NONE)
3267                 return false;
3268         else if (IS_DG2(dev_priv))
3269                 /*
3270                  * DG2 outputs labelled as "combo PHY" in the bspec use
3271                  * SNPS PHYs with completely different programming,
3272                  * hence we always return false here.
3273                  */
3274                 return false;
3275         else if (IS_ALDERLAKE_S(dev_priv))
3276                 return phy <= PHY_E;
3277         else if (IS_DG1(dev_priv) || IS_ROCKETLAKE(dev_priv))
3278                 return phy <= PHY_D;
3279         else if (IS_JSL_EHL(dev_priv))
3280                 return phy <= PHY_C;
3281         else if (DISPLAY_VER(dev_priv) >= 11)
3282                 return phy <= PHY_B;
3283         else
3284                 return false;
3285 }
3286
3287 bool intel_phy_is_tc(struct drm_i915_private *dev_priv, enum phy phy)
3288 {
3289         if (IS_DG2(dev_priv))
3290                 /* DG2's "TC1" output uses a SNPS PHY */
3291                 return false;
3292         else if (IS_ALDERLAKE_P(dev_priv))
3293                 return phy >= PHY_F && phy <= PHY_I;
3294         else if (IS_TIGERLAKE(dev_priv))
3295                 return phy >= PHY_D && phy <= PHY_I;
3296         else if (IS_ICELAKE(dev_priv))
3297                 return phy >= PHY_C && phy <= PHY_F;
3298         else
3299                 return false;
3300 }
3301
3302 bool intel_phy_is_snps(struct drm_i915_private *dev_priv, enum phy phy)
3303 {
3304         if (phy == PHY_NONE)
3305                 return false;
3306         else if (IS_DG2(dev_priv))
3307                 /*
3308                  * All four "combo" ports and the TC1 port (PHY E) use
3309                  * Synopsis PHYs.
3310                  */
3311                 return phy <= PHY_E;
3312
3313         return false;
3314 }
3315
3316 enum phy intel_port_to_phy(struct drm_i915_private *i915, enum port port)
3317 {
3318         if (DISPLAY_VER(i915) >= 13 && port >= PORT_D_XELPD)
3319                 return PHY_D + port - PORT_D_XELPD;
3320         else if (DISPLAY_VER(i915) >= 13 && port >= PORT_TC1)
3321                 return PHY_F + port - PORT_TC1;
3322         else if (IS_ALDERLAKE_S(i915) && port >= PORT_TC1)
3323                 return PHY_B + port - PORT_TC1;
3324         else if ((IS_DG1(i915) || IS_ROCKETLAKE(i915)) && port >= PORT_TC1)
3325                 return PHY_C + port - PORT_TC1;
3326         else if (IS_JSL_EHL(i915) && port == PORT_D)
3327                 return PHY_A;
3328
3329         return PHY_A + port - PORT_A;
3330 }
3331
3332 enum tc_port intel_port_to_tc(struct drm_i915_private *dev_priv, enum port port)
3333 {
3334         if (!intel_phy_is_tc(dev_priv, intel_port_to_phy(dev_priv, port)))
3335                 return TC_PORT_NONE;
3336
3337         if (DISPLAY_VER(dev_priv) >= 12)
3338                 return TC_PORT_1 + port - PORT_TC1;
3339         else
3340                 return TC_PORT_1 + port - PORT_C;
3341 }
3342
3343 enum intel_display_power_domain intel_port_to_power_domain(enum port port)
3344 {
3345         switch (port) {
3346         case PORT_A:
3347                 return POWER_DOMAIN_PORT_DDI_A_LANES;
3348         case PORT_B:
3349                 return POWER_DOMAIN_PORT_DDI_B_LANES;
3350         case PORT_C:
3351                 return POWER_DOMAIN_PORT_DDI_C_LANES;
3352         case PORT_D:
3353                 return POWER_DOMAIN_PORT_DDI_D_LANES;
3354         case PORT_E:
3355                 return POWER_DOMAIN_PORT_DDI_E_LANES;
3356         case PORT_F:
3357                 return POWER_DOMAIN_PORT_DDI_F_LANES;
3358         case PORT_G:
3359                 return POWER_DOMAIN_PORT_DDI_G_LANES;
3360         case PORT_H:
3361                 return POWER_DOMAIN_PORT_DDI_H_LANES;
3362         case PORT_I:
3363                 return POWER_DOMAIN_PORT_DDI_I_LANES;
3364         default:
3365                 MISSING_CASE(port);
3366                 return POWER_DOMAIN_PORT_OTHER;
3367         }
3368 }
3369
3370 enum intel_display_power_domain
3371 intel_aux_power_domain(struct intel_digital_port *dig_port)
3372 {
3373         struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev);
3374         enum phy phy = intel_port_to_phy(dev_priv, dig_port->base.port);
3375
3376         if (intel_phy_is_tc(dev_priv, phy) &&
3377             dig_port->tc_mode == TC_PORT_TBT_ALT) {
3378                 switch (dig_port->aux_ch) {
3379                 case AUX_CH_C:
3380                         return POWER_DOMAIN_AUX_C_TBT;
3381                 case AUX_CH_D:
3382                         return POWER_DOMAIN_AUX_D_TBT;
3383                 case AUX_CH_E:
3384                         return POWER_DOMAIN_AUX_E_TBT;
3385                 case AUX_CH_F:
3386                         return POWER_DOMAIN_AUX_F_TBT;
3387                 case AUX_CH_G:
3388                         return POWER_DOMAIN_AUX_G_TBT;
3389                 case AUX_CH_H:
3390                         return POWER_DOMAIN_AUX_H_TBT;
3391                 case AUX_CH_I:
3392                         return POWER_DOMAIN_AUX_I_TBT;
3393                 default:
3394                         MISSING_CASE(dig_port->aux_ch);
3395                         return POWER_DOMAIN_AUX_C_TBT;
3396                 }
3397         }
3398
3399         return intel_legacy_aux_to_power_domain(dig_port->aux_ch);
3400 }
3401
3402 /*
3403  * Converts aux_ch to power_domain without caring about TBT ports for that use
3404  * intel_aux_power_domain()
3405  */
3406 enum intel_display_power_domain
3407 intel_legacy_aux_to_power_domain(enum aux_ch aux_ch)
3408 {
3409         switch (aux_ch) {
3410         case AUX_CH_A:
3411                 return POWER_DOMAIN_AUX_A;
3412         case AUX_CH_B:
3413                 return POWER_DOMAIN_AUX_B;
3414         case AUX_CH_C:
3415                 return POWER_DOMAIN_AUX_C;
3416         case AUX_CH_D:
3417                 return POWER_DOMAIN_AUX_D;
3418         case AUX_CH_E:
3419                 return POWER_DOMAIN_AUX_E;
3420         case AUX_CH_F:
3421                 return POWER_DOMAIN_AUX_F;
3422         case AUX_CH_G:
3423                 return POWER_DOMAIN_AUX_G;
3424         case AUX_CH_H:
3425                 return POWER_DOMAIN_AUX_H;
3426         case AUX_CH_I:
3427                 return POWER_DOMAIN_AUX_I;
3428         default:
3429                 MISSING_CASE(aux_ch);
3430                 return POWER_DOMAIN_AUX_A;
3431         }
3432 }
3433
3434 static u64 get_crtc_power_domains(struct intel_crtc_state *crtc_state)
3435 {
3436         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3437         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3438         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
3439         struct drm_encoder *encoder;
3440         enum pipe pipe = crtc->pipe;
3441         u64 mask;
3442
3443         if (!crtc_state->hw.active)
3444                 return 0;
3445
3446         mask = BIT_ULL(POWER_DOMAIN_PIPE(pipe));
3447         mask |= BIT_ULL(POWER_DOMAIN_TRANSCODER(cpu_transcoder));
3448         if (crtc_state->pch_pfit.enabled ||
3449             crtc_state->pch_pfit.force_thru)
3450                 mask |= BIT_ULL(POWER_DOMAIN_PIPE_PANEL_FITTER(pipe));
3451
3452         drm_for_each_encoder_mask(encoder, &dev_priv->drm,
3453                                   crtc_state->uapi.encoder_mask) {
3454                 struct intel_encoder *intel_encoder = to_intel_encoder(encoder);
3455
3456                 mask |= BIT_ULL(intel_encoder->power_domain);
3457         }
3458
3459         if (HAS_DDI(dev_priv) && crtc_state->has_audio)
3460                 mask |= BIT_ULL(POWER_DOMAIN_AUDIO_MMIO);
3461
3462         if (crtc_state->shared_dpll)
3463                 mask |= BIT_ULL(POWER_DOMAIN_DISPLAY_CORE);
3464
3465         if (crtc_state->dsc.compression_enable)
3466                 mask |= BIT_ULL(intel_dsc_power_domain(crtc, cpu_transcoder));
3467
3468         return mask;
3469 }
3470
3471 static u64
3472 modeset_get_crtc_power_domains(struct intel_crtc_state *crtc_state)
3473 {
3474         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3475         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3476         enum intel_display_power_domain domain;
3477         u64 domains, new_domains, old_domains;
3478
3479         domains = get_crtc_power_domains(crtc_state);
3480
3481         new_domains = domains & ~crtc->enabled_power_domains.mask;
3482         old_domains = crtc->enabled_power_domains.mask & ~domains;
3483
3484         for_each_power_domain(domain, new_domains)
3485                 intel_display_power_get_in_set(dev_priv,
3486                                                &crtc->enabled_power_domains,
3487                                                domain);
3488
3489         return old_domains;
3490 }
3491
3492 static void modeset_put_crtc_power_domains(struct intel_crtc *crtc,
3493                                            u64 domains)
3494 {
3495         intel_display_power_put_mask_in_set(to_i915(crtc->base.dev),
3496                                             &crtc->enabled_power_domains,
3497                                             domains);
3498 }
3499
3500 static void valleyview_crtc_enable(struct intel_atomic_state *state,
3501                                    struct intel_crtc *crtc)
3502 {
3503         const struct intel_crtc_state *new_crtc_state =
3504                 intel_atomic_get_new_crtc_state(state, crtc);
3505         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3506         enum pipe pipe = crtc->pipe;
3507
3508         if (drm_WARN_ON(&dev_priv->drm, crtc->active))
3509                 return;
3510
3511         if (intel_crtc_has_dp_encoder(new_crtc_state))
3512                 intel_dp_set_m_n(new_crtc_state, M1_N1);
3513
3514         intel_set_transcoder_timings(new_crtc_state);
3515         intel_set_pipe_src_size(new_crtc_state);
3516
3517         if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_B) {
3518                 intel_de_write(dev_priv, CHV_BLEND(pipe), CHV_BLEND_LEGACY);
3519                 intel_de_write(dev_priv, CHV_CANVAS(pipe), 0);
3520         }
3521
3522         i9xx_set_pipeconf(new_crtc_state);
3523
3524         crtc->active = true;
3525
3526         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
3527
3528         intel_encoders_pre_pll_enable(state, crtc);
3529
3530         if (IS_CHERRYVIEW(dev_priv))
3531                 chv_enable_pll(new_crtc_state);
3532         else
3533                 vlv_enable_pll(new_crtc_state);
3534
3535         intel_encoders_pre_enable(state, crtc);
3536
3537         i9xx_pfit_enable(new_crtc_state);
3538
3539         intel_color_load_luts(new_crtc_state);
3540         intel_color_commit(new_crtc_state);
3541         /* update DSPCNTR to configure gamma for pipe bottom color */
3542         intel_disable_primary_plane(new_crtc_state);
3543
3544         dev_priv->display.initial_watermarks(state, crtc);
3545         intel_enable_transcoder(new_crtc_state);
3546
3547         intel_crtc_vblank_on(new_crtc_state);
3548
3549         intel_encoders_enable(state, crtc);
3550 }
3551
3552 static void i9xx_crtc_enable(struct intel_atomic_state *state,
3553                              struct intel_crtc *crtc)
3554 {
3555         const struct intel_crtc_state *new_crtc_state =
3556                 intel_atomic_get_new_crtc_state(state, crtc);
3557         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3558         enum pipe pipe = crtc->pipe;
3559
3560         if (drm_WARN_ON(&dev_priv->drm, crtc->active))
3561                 return;
3562
3563         if (intel_crtc_has_dp_encoder(new_crtc_state))
3564                 intel_dp_set_m_n(new_crtc_state, M1_N1);
3565
3566         intel_set_transcoder_timings(new_crtc_state);
3567         intel_set_pipe_src_size(new_crtc_state);
3568
3569         i9xx_set_pipeconf(new_crtc_state);
3570
3571         crtc->active = true;
3572
3573         if (DISPLAY_VER(dev_priv) != 2)
3574                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
3575
3576         intel_encoders_pre_enable(state, crtc);
3577
3578         i9xx_enable_pll(new_crtc_state);
3579
3580         i9xx_pfit_enable(new_crtc_state);
3581
3582         intel_color_load_luts(new_crtc_state);
3583         intel_color_commit(new_crtc_state);
3584         /* update DSPCNTR to configure gamma for pipe bottom color */
3585         intel_disable_primary_plane(new_crtc_state);
3586
3587         if (dev_priv->display.initial_watermarks)
3588                 dev_priv->display.initial_watermarks(state, crtc);
3589         else
3590                 intel_update_watermarks(crtc);
3591         intel_enable_transcoder(new_crtc_state);
3592
3593         intel_crtc_vblank_on(new_crtc_state);
3594
3595         intel_encoders_enable(state, crtc);
3596
3597         /* prevents spurious underruns */
3598         if (DISPLAY_VER(dev_priv) == 2)
3599                 intel_wait_for_vblank(dev_priv, pipe);
3600 }
3601
3602 static void i9xx_pfit_disable(const struct intel_crtc_state *old_crtc_state)
3603 {
3604         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->uapi.crtc);
3605         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3606
3607         if (!old_crtc_state->gmch_pfit.control)
3608                 return;
3609
3610         assert_transcoder_disabled(dev_priv, old_crtc_state->cpu_transcoder);
3611
3612         drm_dbg_kms(&dev_priv->drm, "disabling pfit, current: 0x%08x\n",
3613                     intel_de_read(dev_priv, PFIT_CONTROL));
3614         intel_de_write(dev_priv, PFIT_CONTROL, 0);
3615 }
3616
3617 static void i9xx_crtc_disable(struct intel_atomic_state *state,
3618                               struct intel_crtc *crtc)
3619 {
3620         struct intel_crtc_state *old_crtc_state =
3621                 intel_atomic_get_old_crtc_state(state, crtc);
3622         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3623         enum pipe pipe = crtc->pipe;
3624
3625         /*
3626          * On gen2 planes are double buffered but the pipe isn't, so we must
3627          * wait for planes to fully turn off before disabling the pipe.
3628          */
3629         if (DISPLAY_VER(dev_priv) == 2)
3630                 intel_wait_for_vblank(dev_priv, pipe);
3631
3632         intel_encoders_disable(state, crtc);
3633
3634         intel_crtc_vblank_off(old_crtc_state);
3635
3636         intel_disable_transcoder(old_crtc_state);
3637
3638         i9xx_pfit_disable(old_crtc_state);
3639
3640         intel_encoders_post_disable(state, crtc);
3641
3642         if (!intel_crtc_has_type(old_crtc_state, INTEL_OUTPUT_DSI)) {
3643                 if (IS_CHERRYVIEW(dev_priv))
3644                         chv_disable_pll(dev_priv, pipe);
3645                 else if (IS_VALLEYVIEW(dev_priv))
3646                         vlv_disable_pll(dev_priv, pipe);
3647                 else
3648                         i9xx_disable_pll(old_crtc_state);
3649         }
3650
3651         intel_encoders_post_pll_disable(state, crtc);
3652
3653         if (DISPLAY_VER(dev_priv) != 2)
3654                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
3655
3656         if (!dev_priv->display.initial_watermarks)
3657                 intel_update_watermarks(crtc);
3658
3659         /* clock the pipe down to 640x480@60 to potentially save power */
3660         if (IS_I830(dev_priv))
3661                 i830_enable_pipe(dev_priv, pipe);
3662 }
3663
3664 static void intel_crtc_disable_noatomic(struct intel_crtc *crtc,
3665                                         struct drm_modeset_acquire_ctx *ctx)
3666 {
3667         struct intel_encoder *encoder;
3668         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3669         struct intel_bw_state *bw_state =
3670                 to_intel_bw_state(dev_priv->bw_obj.state);
3671         struct intel_cdclk_state *cdclk_state =
3672                 to_intel_cdclk_state(dev_priv->cdclk.obj.state);
3673         struct intel_dbuf_state *dbuf_state =
3674                 to_intel_dbuf_state(dev_priv->dbuf.obj.state);
3675         struct intel_crtc_state *crtc_state =
3676                 to_intel_crtc_state(crtc->base.state);
3677         struct intel_plane *plane;
3678         struct drm_atomic_state *state;
3679         struct intel_crtc_state *temp_crtc_state;
3680         enum pipe pipe = crtc->pipe;
3681         int ret;
3682
3683         if (!crtc_state->hw.active)
3684                 return;
3685
3686         for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
3687                 const struct intel_plane_state *plane_state =
3688                         to_intel_plane_state(plane->base.state);
3689
3690                 if (plane_state->uapi.visible)
3691                         intel_plane_disable_noatomic(crtc, plane);
3692         }
3693
3694         state = drm_atomic_state_alloc(&dev_priv->drm);
3695         if (!state) {
3696                 drm_dbg_kms(&dev_priv->drm,
3697                             "failed to disable [CRTC:%d:%s], out of memory",
3698                             crtc->base.base.id, crtc->base.name);
3699                 return;
3700         }
3701
3702         state->acquire_ctx = ctx;
3703
3704         /* Everything's already locked, -EDEADLK can't happen. */
3705         temp_crtc_state = intel_atomic_get_crtc_state(state, crtc);
3706         ret = drm_atomic_add_affected_connectors(state, &crtc->base);
3707
3708         drm_WARN_ON(&dev_priv->drm, IS_ERR(temp_crtc_state) || ret);
3709
3710         dev_priv->display.crtc_disable(to_intel_atomic_state(state), crtc);
3711
3712         drm_atomic_state_put(state);
3713
3714         drm_dbg_kms(&dev_priv->drm,
3715                     "[CRTC:%d:%s] hw state adjusted, was enabled, now disabled\n",
3716                     crtc->base.base.id, crtc->base.name);
3717
3718         crtc->active = false;
3719         crtc->base.enabled = false;
3720
3721         drm_WARN_ON(&dev_priv->drm,
3722                     drm_atomic_set_mode_for_crtc(&crtc_state->uapi, NULL) < 0);
3723         crtc_state->uapi.active = false;
3724         crtc_state->uapi.connector_mask = 0;
3725         crtc_state->uapi.encoder_mask = 0;
3726         intel_crtc_free_hw_state(crtc_state);
3727         memset(&crtc_state->hw, 0, sizeof(crtc_state->hw));
3728
3729         for_each_encoder_on_crtc(&dev_priv->drm, &crtc->base, encoder)
3730                 encoder->base.crtc = NULL;
3731
3732         intel_fbc_disable(crtc);
3733         intel_update_watermarks(crtc);
3734         intel_disable_shared_dpll(crtc_state);
3735
3736         intel_display_power_put_all_in_set(dev_priv, &crtc->enabled_power_domains);
3737
3738         cdclk_state->min_cdclk[pipe] = 0;
3739         cdclk_state->min_voltage_level[pipe] = 0;
3740         cdclk_state->active_pipes &= ~BIT(pipe);
3741
3742         dbuf_state->active_pipes &= ~BIT(pipe);
3743
3744         bw_state->data_rate[pipe] = 0;
3745         bw_state->num_active_planes[pipe] = 0;
3746 }
3747
3748 /*
3749  * turn all crtc's off, but do not adjust state
3750  * This has to be paired with a call to intel_modeset_setup_hw_state.
3751  */
3752 int intel_display_suspend(struct drm_device *dev)
3753 {
3754         struct drm_i915_private *dev_priv = to_i915(dev);
3755         struct drm_atomic_state *state;
3756         int ret;
3757
3758         if (!HAS_DISPLAY(dev_priv))
3759                 return 0;
3760
3761         state = drm_atomic_helper_suspend(dev);
3762         ret = PTR_ERR_OR_ZERO(state);
3763         if (ret)
3764                 drm_err(&dev_priv->drm, "Suspending crtc's failed with %i\n",
3765                         ret);
3766         else
3767                 dev_priv->modeset_restore_state = state;
3768         return ret;
3769 }
3770
3771 void intel_encoder_destroy(struct drm_encoder *encoder)
3772 {
3773         struct intel_encoder *intel_encoder = to_intel_encoder(encoder);
3774
3775         drm_encoder_cleanup(encoder);
3776         kfree(intel_encoder);
3777 }
3778
3779 /* Cross check the actual hw state with our own modeset state tracking (and it's
3780  * internal consistency). */
3781 static void intel_connector_verify_state(struct intel_crtc_state *crtc_state,
3782                                          struct drm_connector_state *conn_state)
3783 {
3784         struct intel_connector *connector = to_intel_connector(conn_state->connector);
3785         struct drm_i915_private *i915 = to_i915(connector->base.dev);
3786
3787         drm_dbg_kms(&i915->drm, "[CONNECTOR:%d:%s]\n",
3788                     connector->base.base.id, connector->base.name);
3789
3790         if (connector->get_hw_state(connector)) {
3791                 struct intel_encoder *encoder = intel_attached_encoder(connector);
3792
3793                 I915_STATE_WARN(!crtc_state,
3794                          "connector enabled without attached crtc\n");
3795
3796                 if (!crtc_state)
3797                         return;
3798
3799                 I915_STATE_WARN(!crtc_state->hw.active,
3800                                 "connector is active, but attached crtc isn't\n");
3801
3802                 if (!encoder || encoder->type == INTEL_OUTPUT_DP_MST)
3803                         return;
3804
3805                 I915_STATE_WARN(conn_state->best_encoder != &encoder->base,
3806                         "atomic encoder doesn't match attached encoder\n");
3807
3808                 I915_STATE_WARN(conn_state->crtc != encoder->base.crtc,
3809                         "attached encoder crtc differs from connector crtc\n");
3810         } else {
3811                 I915_STATE_WARN(crtc_state && crtc_state->hw.active,
3812                                 "attached crtc is active, but connector isn't\n");
3813                 I915_STATE_WARN(!crtc_state && conn_state->best_encoder,
3814                         "best encoder set without crtc!\n");
3815         }
3816 }
3817
3818 bool hsw_crtc_state_ips_capable(const struct intel_crtc_state *crtc_state)
3819 {
3820         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3821         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3822
3823         /* IPS only exists on ULT machines and is tied to pipe A. */
3824         if (!hsw_crtc_supports_ips(crtc))
3825                 return false;
3826
3827         if (!dev_priv->params.enable_ips)
3828                 return false;
3829
3830         if (crtc_state->pipe_bpp > 24)
3831                 return false;
3832
3833         /*
3834          * We compare against max which means we must take
3835          * the increased cdclk requirement into account when
3836          * calculating the new cdclk.
3837          *
3838          * Should measure whether using a lower cdclk w/o IPS
3839          */
3840         if (IS_BROADWELL(dev_priv) &&
3841             crtc_state->pixel_rate > dev_priv->max_cdclk_freq * 95 / 100)
3842                 return false;
3843
3844         return true;
3845 }
3846
3847 static int hsw_compute_ips_config(struct intel_crtc_state *crtc_state)
3848 {
3849         struct drm_i915_private *dev_priv =
3850                 to_i915(crtc_state->uapi.crtc->dev);
3851         struct intel_atomic_state *state =
3852                 to_intel_atomic_state(crtc_state->uapi.state);
3853
3854         crtc_state->ips_enabled = false;
3855
3856         if (!hsw_crtc_state_ips_capable(crtc_state))
3857                 return 0;
3858
3859         /*
3860          * When IPS gets enabled, the pipe CRC changes. Since IPS gets
3861          * enabled and disabled dynamically based on package C states,
3862          * user space can't make reliable use of the CRCs, so let's just
3863          * completely disable it.
3864          */
3865         if (crtc_state->crc_enabled)
3866                 return 0;
3867
3868         /* IPS should be fine as long as at least one plane is enabled. */
3869         if (!(crtc_state->active_planes & ~BIT(PLANE_CURSOR)))
3870                 return 0;
3871
3872         if (IS_BROADWELL(dev_priv)) {
3873                 const struct intel_cdclk_state *cdclk_state;
3874
3875                 cdclk_state = intel_atomic_get_cdclk_state(state);
3876                 if (IS_ERR(cdclk_state))
3877                         return PTR_ERR(cdclk_state);
3878
3879                 /* pixel rate mustn't exceed 95% of cdclk with IPS on BDW */
3880                 if (crtc_state->pixel_rate > cdclk_state->logical.cdclk * 95 / 100)
3881                         return 0;
3882         }
3883
3884         crtc_state->ips_enabled = true;
3885
3886         return 0;
3887 }
3888
3889 static bool intel_crtc_supports_double_wide(const struct intel_crtc *crtc)
3890 {
3891         const struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3892
3893         /* GDG double wide on either pipe, otherwise pipe A only */
3894         return DISPLAY_VER(dev_priv) < 4 &&
3895                 (crtc->pipe == PIPE_A || IS_I915G(dev_priv));
3896 }
3897
3898 static u32 ilk_pipe_pixel_rate(const struct intel_crtc_state *crtc_state)
3899 {
3900         u32 pixel_rate = crtc_state->hw.pipe_mode.crtc_clock;
3901         struct drm_rect src;
3902
3903         /*
3904          * We only use IF-ID interlacing. If we ever use
3905          * PF-ID we'll need to adjust the pixel_rate here.
3906          */
3907
3908         if (!crtc_state->pch_pfit.enabled)
3909                 return pixel_rate;
3910
3911         drm_rect_init(&src, 0, 0,
3912                       crtc_state->pipe_src_w << 16,
3913                       crtc_state->pipe_src_h << 16);
3914
3915         return intel_adjusted_rate(&src, &crtc_state->pch_pfit.dst,
3916                                    pixel_rate);
3917 }
3918
3919 static void intel_mode_from_crtc_timings(struct drm_display_mode *mode,
3920                                          const struct drm_display_mode *timings)
3921 {
3922         mode->hdisplay = timings->crtc_hdisplay;
3923         mode->htotal = timings->crtc_htotal;
3924         mode->hsync_start = timings->crtc_hsync_start;
3925         mode->hsync_end = timings->crtc_hsync_end;
3926
3927         mode->vdisplay = timings->crtc_vdisplay;
3928         mode->vtotal = timings->crtc_vtotal;
3929         mode->vsync_start = timings->crtc_vsync_start;
3930         mode->vsync_end = timings->crtc_vsync_end;
3931
3932         mode->flags = timings->flags;
3933         mode->type = DRM_MODE_TYPE_DRIVER;
3934
3935         mode->clock = timings->crtc_clock;
3936
3937         drm_mode_set_name(mode);
3938 }
3939
3940 static void intel_crtc_compute_pixel_rate(struct intel_crtc_state *crtc_state)
3941 {
3942         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
3943
3944         if (HAS_GMCH(dev_priv))
3945                 /* FIXME calculate proper pipe pixel rate for GMCH pfit */
3946                 crtc_state->pixel_rate =
3947                         crtc_state->hw.pipe_mode.crtc_clock;
3948         else
3949                 crtc_state->pixel_rate =
3950                         ilk_pipe_pixel_rate(crtc_state);
3951 }
3952
3953 static void intel_crtc_readout_derived_state(struct intel_crtc_state *crtc_state)
3954 {
3955         struct drm_display_mode *mode = &crtc_state->hw.mode;
3956         struct drm_display_mode *pipe_mode = &crtc_state->hw.pipe_mode;
3957         struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode;
3958
3959         drm_mode_copy(pipe_mode, adjusted_mode);
3960
3961         if (crtc_state->bigjoiner) {
3962                 /*
3963                  * transcoder is programmed to the full mode,
3964                  * but pipe timings are half of the transcoder mode
3965                  */
3966                 pipe_mode->crtc_hdisplay /= 2;
3967                 pipe_mode->crtc_hblank_start /= 2;
3968                 pipe_mode->crtc_hblank_end /= 2;
3969                 pipe_mode->crtc_hsync_start /= 2;
3970                 pipe_mode->crtc_hsync_end /= 2;
3971                 pipe_mode->crtc_htotal /= 2;
3972                 pipe_mode->crtc_clock /= 2;
3973         }
3974
3975         if (crtc_state->splitter.enable) {
3976                 int n = crtc_state->splitter.link_count;
3977                 int overlap = crtc_state->splitter.pixel_overlap;
3978
3979                 /*
3980                  * eDP MSO uses segment timings from EDID for transcoder
3981                  * timings, but full mode for everything else.
3982                  *
3983                  * h_full = (h_segment - pixel_overlap) * link_count
3984                  */
3985                 pipe_mode->crtc_hdisplay = (pipe_mode->crtc_hdisplay - overlap) * n;
3986                 pipe_mode->crtc_hblank_start = (pipe_mode->crtc_hblank_start - overlap) * n;
3987                 pipe_mode->crtc_hblank_end = (pipe_mode->crtc_hblank_end - overlap) * n;
3988                 pipe_mode->crtc_hsync_start = (pipe_mode->crtc_hsync_start - overlap) * n;
3989                 pipe_mode->crtc_hsync_end = (pipe_mode->crtc_hsync_end - overlap) * n;
3990                 pipe_mode->crtc_htotal = (pipe_mode->crtc_htotal - overlap) * n;
3991                 pipe_mode->crtc_clock *= n;
3992
3993                 intel_mode_from_crtc_timings(pipe_mode, pipe_mode);
3994                 intel_mode_from_crtc_timings(adjusted_mode, pipe_mode);
3995         } else {
3996                 intel_mode_from_crtc_timings(pipe_mode, pipe_mode);
3997                 intel_mode_from_crtc_timings(adjusted_mode, adjusted_mode);
3998         }
3999
4000         intel_crtc_compute_pixel_rate(crtc_state);
4001
4002         drm_mode_copy(mode, adjusted_mode);
4003         mode->hdisplay = crtc_state->pipe_src_w << crtc_state->bigjoiner;
4004         mode->vdisplay = crtc_state->pipe_src_h;
4005 }
4006
4007 static void intel_encoder_get_config(struct intel_encoder *encoder,
4008                                      struct intel_crtc_state *crtc_state)
4009 {
4010         encoder->get_config(encoder, crtc_state);
4011
4012         intel_crtc_readout_derived_state(crtc_state);
4013 }
4014
4015 static int intel_crtc_compute_config(struct intel_crtc *crtc,
4016                                      struct intel_crtc_state *pipe_config)
4017 {
4018         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4019         struct drm_display_mode *pipe_mode = &pipe_config->hw.pipe_mode;
4020         int clock_limit = dev_priv->max_dotclk_freq;
4021
4022         drm_mode_copy(pipe_mode, &pipe_config->hw.adjusted_mode);
4023
4024         /* Adjust pipe_mode for bigjoiner, with half the horizontal mode */
4025         if (pipe_config->bigjoiner) {
4026                 pipe_mode->crtc_clock /= 2;
4027                 pipe_mode->crtc_hdisplay /= 2;
4028                 pipe_mode->crtc_hblank_start /= 2;
4029                 pipe_mode->crtc_hblank_end /= 2;
4030                 pipe_mode->crtc_hsync_start /= 2;
4031                 pipe_mode->crtc_hsync_end /= 2;
4032                 pipe_mode->crtc_htotal /= 2;
4033                 pipe_config->pipe_src_w /= 2;
4034         }
4035
4036         if (pipe_config->splitter.enable) {
4037                 int n = pipe_config->splitter.link_count;
4038                 int overlap = pipe_config->splitter.pixel_overlap;
4039
4040                 pipe_mode->crtc_hdisplay = (pipe_mode->crtc_hdisplay - overlap) * n;
4041                 pipe_mode->crtc_hblank_start = (pipe_mode->crtc_hblank_start - overlap) * n;
4042                 pipe_mode->crtc_hblank_end = (pipe_mode->crtc_hblank_end - overlap) * n;
4043                 pipe_mode->crtc_hsync_start = (pipe_mode->crtc_hsync_start - overlap) * n;
4044                 pipe_mode->crtc_hsync_end = (pipe_mode->crtc_hsync_end - overlap) * n;
4045                 pipe_mode->crtc_htotal = (pipe_mode->crtc_htotal - overlap) * n;
4046                 pipe_mode->crtc_clock *= n;
4047         }
4048
4049         intel_mode_from_crtc_timings(pipe_mode, pipe_mode);
4050
4051         if (DISPLAY_VER(dev_priv) < 4) {
4052                 clock_limit = dev_priv->max_cdclk_freq * 9 / 10;
4053
4054                 /*
4055                  * Enable double wide mode when the dot clock
4056                  * is > 90% of the (display) core speed.
4057                  */
4058                 if (intel_crtc_supports_double_wide(crtc) &&
4059                     pipe_mode->crtc_clock > clock_limit) {
4060                         clock_limit = dev_priv->max_dotclk_freq;
4061                         pipe_config->double_wide = true;
4062                 }
4063         }
4064
4065         if (pipe_mode->crtc_clock > clock_limit) {
4066                 drm_dbg_kms(&dev_priv->drm,
4067                             "requested pixel clock (%d kHz) too high (max: %d kHz, double wide: %s)\n",
4068                             pipe_mode->crtc_clock, clock_limit,
4069                             yesno(pipe_config->double_wide));
4070                 return -EINVAL;
4071         }
4072
4073         /*
4074          * Pipe horizontal size must be even in:
4075          * - DVO ganged mode
4076          * - LVDS dual channel mode
4077          * - Double wide pipe
4078          */
4079         if (pipe_config->pipe_src_w & 1) {
4080                 if (pipe_config->double_wide) {
4081                         drm_dbg_kms(&dev_priv->drm,
4082                                     "Odd pipe source width not supported with double wide pipe\n");
4083                         return -EINVAL;
4084                 }
4085
4086                 if (intel_crtc_has_type(pipe_config, INTEL_OUTPUT_LVDS) &&
4087                     intel_is_dual_link_lvds(dev_priv)) {
4088                         drm_dbg_kms(&dev_priv->drm,
4089                                     "Odd pipe source width not supported with dual link LVDS\n");
4090                         return -EINVAL;
4091                 }
4092         }
4093
4094         /* Cantiga+ cannot handle modes with a hsync front porch of 0.
4095          * WaPruneModeWithIncorrectHsyncOffset:ctg,elk,ilk,snb,ivb,vlv,hsw.
4096          */
4097         if ((DISPLAY_VER(dev_priv) > 4 || IS_G4X(dev_priv)) &&
4098             pipe_mode->crtc_hsync_start == pipe_mode->crtc_hdisplay)
4099                 return -EINVAL;
4100
4101         intel_crtc_compute_pixel_rate(pipe_config);
4102
4103         if (pipe_config->has_pch_encoder)
4104                 return ilk_fdi_compute_config(crtc, pipe_config);
4105
4106         return 0;
4107 }
4108
4109 static void
4110 intel_reduce_m_n_ratio(u32 *num, u32 *den)
4111 {
4112         while (*num > DATA_LINK_M_N_MASK ||
4113                *den > DATA_LINK_M_N_MASK) {
4114                 *num >>= 1;
4115                 *den >>= 1;
4116         }
4117 }
4118
4119 static void compute_m_n(unsigned int m, unsigned int n,
4120                         u32 *ret_m, u32 *ret_n,
4121                         bool constant_n)
4122 {
4123         /*
4124          * Several DP dongles in particular seem to be fussy about
4125          * too large link M/N values. Give N value as 0x8000 that
4126          * should be acceptable by specific devices. 0x8000 is the
4127          * specified fixed N value for asynchronous clock mode,
4128          * which the devices expect also in synchronous clock mode.
4129          */
4130         if (constant_n)
4131                 *ret_n = DP_LINK_CONSTANT_N_VALUE;
4132         else
4133                 *ret_n = min_t(unsigned int, roundup_pow_of_two(n), DATA_LINK_N_MAX);
4134
4135         *ret_m = div_u64(mul_u32_u32(m, *ret_n), n);
4136         intel_reduce_m_n_ratio(ret_m, ret_n);
4137 }
4138
4139 void
4140 intel_link_compute_m_n(u16 bits_per_pixel, int nlanes,
4141                        int pixel_clock, int link_clock,
4142                        struct intel_link_m_n *m_n,
4143                        bool constant_n, bool fec_enable)
4144 {
4145         u32 data_clock = bits_per_pixel * pixel_clock;
4146
4147         if (fec_enable)
4148                 data_clock = intel_dp_mode_to_fec_clock(data_clock);
4149
4150         m_n->tu = 64;
4151         compute_m_n(data_clock,
4152                     link_clock * nlanes * 8,
4153                     &m_n->gmch_m, &m_n->gmch_n,
4154                     constant_n);
4155
4156         compute_m_n(pixel_clock, link_clock,
4157                     &m_n->link_m, &m_n->link_n,
4158                     constant_n);
4159 }
4160
4161 static void intel_panel_sanitize_ssc(struct drm_i915_private *dev_priv)
4162 {
4163         /*
4164          * There may be no VBT; and if the BIOS enabled SSC we can
4165          * just keep using it to avoid unnecessary flicker.  Whereas if the
4166          * BIOS isn't using it, don't assume it will work even if the VBT
4167          * indicates as much.
4168          */
4169         if (HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv)) {
4170                 bool bios_lvds_use_ssc = intel_de_read(dev_priv,
4171                                                        PCH_DREF_CONTROL) &
4172                         DREF_SSC1_ENABLE;
4173
4174                 if (dev_priv->vbt.lvds_use_ssc != bios_lvds_use_ssc) {
4175                         drm_dbg_kms(&dev_priv->drm,
4176                                     "SSC %s by BIOS, overriding VBT which says %s\n",
4177                                     enableddisabled(bios_lvds_use_ssc),
4178                                     enableddisabled(dev_priv->vbt.lvds_use_ssc));
4179                         dev_priv->vbt.lvds_use_ssc = bios_lvds_use_ssc;
4180                 }
4181         }
4182 }
4183
4184 static void intel_pch_transcoder_set_m_n(const struct intel_crtc_state *crtc_state,
4185                                          const struct intel_link_m_n *m_n)
4186 {
4187         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
4188         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4189         enum pipe pipe = crtc->pipe;
4190
4191         intel_de_write(dev_priv, PCH_TRANS_DATA_M1(pipe),
4192                        TU_SIZE(m_n->tu) | m_n->gmch_m);
4193         intel_de_write(dev_priv, PCH_TRANS_DATA_N1(pipe), m_n->gmch_n);
4194         intel_de_write(dev_priv, PCH_TRANS_LINK_M1(pipe), m_n->link_m);
4195         intel_de_write(dev_priv, PCH_TRANS_LINK_N1(pipe), m_n->link_n);
4196 }
4197
4198 static bool transcoder_has_m2_n2(struct drm_i915_private *dev_priv,
4199                                  enum transcoder transcoder)
4200 {
4201         if (IS_HASWELL(dev_priv))
4202                 return transcoder == TRANSCODER_EDP;
4203
4204         /*
4205          * Strictly speaking some registers are available before
4206          * gen7, but we only support DRRS on gen7+
4207          */
4208         return DISPLAY_VER(dev_priv) == 7 || IS_CHERRYVIEW(dev_priv);
4209 }
4210
4211 static void intel_cpu_transcoder_set_m_n(const struct intel_crtc_state *crtc_state,
4212                                          const struct intel_link_m_n *m_n,
4213                                          const struct intel_link_m_n *m2_n2)
4214 {
4215         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
4216         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4217         enum pipe pipe = crtc->pipe;
4218         enum transcoder transcoder = crtc_state->cpu_transcoder;
4219
4220         if (DISPLAY_VER(dev_priv) >= 5) {
4221                 intel_de_write(dev_priv, PIPE_DATA_M1(transcoder),
4222                                TU_SIZE(m_n->tu) | m_n->gmch_m);
4223                 intel_de_write(dev_priv, PIPE_DATA_N1(transcoder),
4224                                m_n->gmch_n);
4225                 intel_de_write(dev_priv, PIPE_LINK_M1(transcoder),
4226                                m_n->link_m);
4227                 intel_de_write(dev_priv, PIPE_LINK_N1(transcoder),
4228                                m_n->link_n);
4229                 /*
4230                  *  M2_N2 registers are set only if DRRS is supported
4231                  * (to make sure the registers are not unnecessarily accessed).
4232                  */
4233                 if (m2_n2 && crtc_state->has_drrs &&
4234                     transcoder_has_m2_n2(dev_priv, transcoder)) {
4235                         intel_de_write(dev_priv, PIPE_DATA_M2(transcoder),
4236                                        TU_SIZE(m2_n2->tu) | m2_n2->gmch_m);
4237                         intel_de_write(dev_priv, PIPE_DATA_N2(transcoder),
4238                                        m2_n2->gmch_n);
4239                         intel_de_write(dev_priv, PIPE_LINK_M2(transcoder),
4240                                        m2_n2->link_m);
4241                         intel_de_write(dev_priv, PIPE_LINK_N2(transcoder),
4242                                        m2_n2->link_n);
4243                 }
4244         } else {
4245                 intel_de_write(dev_priv, PIPE_DATA_M_G4X(pipe),
4246                                TU_SIZE(m_n->tu) | m_n->gmch_m);
4247                 intel_de_write(dev_priv, PIPE_DATA_N_G4X(pipe), m_n->gmch_n);
4248                 intel_de_write(dev_priv, PIPE_LINK_M_G4X(pipe), m_n->link_m);
4249                 intel_de_write(dev_priv, PIPE_LINK_N_G4X(pipe), m_n->link_n);
4250         }
4251 }
4252
4253 void intel_dp_set_m_n(const struct intel_crtc_state *crtc_state, enum link_m_n_set m_n)
4254 {
4255         const struct intel_link_m_n *dp_m_n, *dp_m2_n2 = NULL;
4256         struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
4257
4258         if (m_n == M1_N1) {
4259                 dp_m_n = &crtc_state->dp_m_n;
4260                 dp_m2_n2 = &crtc_state->dp_m2_n2;
4261         } else if (m_n == M2_N2) {
4262
4263                 /*
4264                  * M2_N2 registers are not supported. Hence m2_n2 divider value
4265                  * needs to be programmed into M1_N1.
4266                  */
4267                 dp_m_n = &crtc_state->dp_m2_n2;
4268         } else {
4269                 drm_err(&i915->drm, "Unsupported divider value\n");
4270                 return;
4271         }
4272
4273         if (crtc_state->has_pch_encoder)
4274                 intel_pch_transcoder_set_m_n(crtc_state, &crtc_state->dp_m_n);
4275         else
4276                 intel_cpu_transcoder_set_m_n(crtc_state, dp_m_n, dp_m2_n2);
4277 }
4278
4279 static void intel_set_transcoder_timings(const struct intel_crtc_state *crtc_state)
4280 {
4281         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
4282         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4283         enum pipe pipe = crtc->pipe;
4284         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
4285         const struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode;
4286         u32 crtc_vtotal, crtc_vblank_end;
4287         int vsyncshift = 0;
4288
4289         /* We need to be careful not to changed the adjusted mode, for otherwise
4290          * the hw state checker will get angry at the mismatch. */
4291         crtc_vtotal = adjusted_mode->crtc_vtotal;
4292         crtc_vblank_end = adjusted_mode->crtc_vblank_end;
4293
4294         if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) {
4295                 /* the chip adds 2 halflines automatically */
4296                 crtc_vtotal -= 1;
4297                 crtc_vblank_end -= 1;
4298
4299                 if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO))
4300                         vsyncshift = (adjusted_mode->crtc_htotal - 1) / 2;
4301                 else
4302                         vsyncshift = adjusted_mode->crtc_hsync_start -
4303                                 adjusted_mode->crtc_htotal / 2;
4304                 if (vsyncshift < 0)
4305                         vsyncshift += adjusted_mode->crtc_htotal;
4306         }
4307
4308         if (DISPLAY_VER(dev_priv) > 3)
4309                 intel_de_write(dev_priv, VSYNCSHIFT(cpu_transcoder),
4310                                vsyncshift);
4311
4312         intel_de_write(dev_priv, HTOTAL(cpu_transcoder),
4313                        (adjusted_mode->crtc_hdisplay - 1) | ((adjusted_mode->crtc_htotal - 1) << 16));
4314         intel_de_write(dev_priv, HBLANK(cpu_transcoder),
4315                        (adjusted_mode->crtc_hblank_start - 1) | ((adjusted_mode->crtc_hblank_end - 1) << 16));
4316         intel_de_write(dev_priv, HSYNC(cpu_transcoder),
4317                        (adjusted_mode->crtc_hsync_start - 1) | ((adjusted_mode->crtc_hsync_end - 1) << 16));
4318
4319         intel_de_write(dev_priv, VTOTAL(cpu_transcoder),
4320                        (adjusted_mode->crtc_vdisplay - 1) | ((crtc_vtotal - 1) << 16));
4321         intel_de_write(dev_priv, VBLANK(cpu_transcoder),
4322                        (adjusted_mode->crtc_vblank_start - 1) | ((crtc_vblank_end - 1) << 16));
4323         intel_de_write(dev_priv, VSYNC(cpu_transcoder),
4324                        (adjusted_mode->crtc_vsync_start - 1) | ((adjusted_mode->crtc_vsync_end - 1) << 16));
4325
4326         /* Workaround: when the EDP input selection is B, the VTOTAL_B must be
4327          * programmed with the VTOTAL_EDP value. Same for VTOTAL_C. This is
4328          * documented on the DDI_FUNC_CTL register description, EDP Input Select
4329          * bits. */
4330         if (IS_HASWELL(dev_priv) && cpu_transcoder == TRANSCODER_EDP &&
4331             (pipe == PIPE_B || pipe == PIPE_C))
4332                 intel_de_write(dev_priv, VTOTAL(pipe),
4333                                intel_de_read(dev_priv, VTOTAL(cpu_transcoder)));
4334
4335 }
4336
4337 static void intel_set_pipe_src_size(const struct intel_crtc_state *crtc_state)
4338 {
4339         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
4340         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4341         enum pipe pipe = crtc->pipe;
4342
4343         /* pipesrc controls the size that is scaled from, which should
4344          * always be the user's requested size.
4345          */
4346         intel_de_write(dev_priv, PIPESRC(pipe),
4347                        ((crtc_state->pipe_src_w - 1) << 16) | (crtc_state->pipe_src_h - 1));
4348 }
4349
4350 static bool intel_pipe_is_interlaced(const struct intel_crtc_state *crtc_state)
4351 {
4352         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
4353         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
4354
4355         if (DISPLAY_VER(dev_priv) == 2)
4356                 return false;
4357
4358         if (DISPLAY_VER(dev_priv) >= 9 ||
4359             IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
4360                 return intel_de_read(dev_priv, PIPECONF(cpu_transcoder)) & PIPECONF_INTERLACE_MASK_HSW;
4361         else
4362                 return intel_de_read(dev_priv, PIPECONF(cpu_transcoder)) & PIPECONF_INTERLACE_MASK;
4363 }
4364
4365 static void intel_get_transcoder_timings(struct intel_crtc *crtc,
4366                                          struct intel_crtc_state *pipe_config)
4367 {
4368         struct drm_device *dev = crtc->base.dev;
4369         struct drm_i915_private *dev_priv = to_i915(dev);
4370         enum transcoder cpu_transcoder = pipe_config->cpu_transcoder;
4371         u32 tmp;
4372
4373         tmp = intel_de_read(dev_priv, HTOTAL(cpu_transcoder));
4374         pipe_config->hw.adjusted_mode.crtc_hdisplay = (tmp & 0xffff) + 1;
4375         pipe_config->hw.adjusted_mode.crtc_htotal = ((tmp >> 16) & 0xffff) + 1;
4376
4377         if (!transcoder_is_dsi(cpu_transcoder)) {
4378                 tmp = intel_de_read(dev_priv, HBLANK(cpu_transcoder));
4379                 pipe_config->hw.adjusted_mode.crtc_hblank_start =
4380                                                         (tmp & 0xffff) + 1;
4381                 pipe_config->hw.adjusted_mode.crtc_hblank_end =
4382                                                 ((tmp >> 16) & 0xffff) + 1;
4383         }
4384         tmp = intel_de_read(dev_priv, HSYNC(cpu_transcoder));
4385         pipe_config->hw.adjusted_mode.crtc_hsync_start = (tmp & 0xffff) + 1;
4386         pipe_config->hw.adjusted_mode.crtc_hsync_end = ((tmp >> 16) & 0xffff) + 1;
4387
4388         tmp = intel_de_read(dev_priv, VTOTAL(cpu_transcoder));
4389         pipe_config->hw.adjusted_mode.crtc_vdisplay = (tmp & 0xffff) + 1;
4390         pipe_config->hw.adjusted_mode.crtc_vtotal = ((tmp >> 16) & 0xffff) + 1;
4391
4392         if (!transcoder_is_dsi(cpu_transcoder)) {
4393                 tmp = intel_de_read(dev_priv, VBLANK(cpu_transcoder));
4394                 pipe_config->hw.adjusted_mode.crtc_vblank_start =
4395                                                         (tmp & 0xffff) + 1;
4396                 pipe_config->hw.adjusted_mode.crtc_vblank_end =
4397                                                 ((tmp >> 16) & 0xffff) + 1;
4398         }
4399         tmp = intel_de_read(dev_priv, VSYNC(cpu_transcoder));
4400         pipe_config->hw.adjusted_mode.crtc_vsync_start = (tmp & 0xffff) + 1;
4401         pipe_config->hw.adjusted_mode.crtc_vsync_end = ((tmp >> 16) & 0xffff) + 1;
4402
4403         if (intel_pipe_is_interlaced(pipe_config)) {
4404                 pipe_config->hw.adjusted_mode.flags |= DRM_MODE_FLAG_INTERLACE;
4405                 pipe_config->hw.adjusted_mode.crtc_vtotal += 1;
4406                 pipe_config->hw.adjusted_mode.crtc_vblank_end += 1;
4407         }
4408 }
4409
4410 static void intel_get_pipe_src_size(struct intel_crtc *crtc,
4411                                     struct intel_crtc_state *pipe_config)
4412 {
4413         struct drm_device *dev = crtc->base.dev;
4414         struct drm_i915_private *dev_priv = to_i915(dev);
4415         u32 tmp;
4416
4417         tmp = intel_de_read(dev_priv, PIPESRC(crtc->pipe));
4418         pipe_config->pipe_src_h = (tmp & 0xffff) + 1;
4419         pipe_config->pipe_src_w = ((tmp >> 16) & 0xffff) + 1;
4420 }
4421
4422 static void i9xx_set_pipeconf(const struct intel_crtc_state *crtc_state)
4423 {
4424         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
4425         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4426         u32 pipeconf;
4427
4428         pipeconf = 0;
4429
4430         /* we keep both pipes enabled on 830 */
4431         if (IS_I830(dev_priv))
4432                 pipeconf |= intel_de_read(dev_priv, PIPECONF(crtc->pipe)) & PIPECONF_ENABLE;
4433
4434         if (crtc_state->double_wide)
4435                 pipeconf |= PIPECONF_DOUBLE_WIDE;
4436
4437         /* only g4x and later have fancy bpc/dither controls */
4438         if (IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
4439             IS_CHERRYVIEW(dev_priv)) {
4440                 /* Bspec claims that we can't use dithering for 30bpp pipes. */
4441                 if (crtc_state->dither && crtc_state->pipe_bpp != 30)
4442                         pipeconf |= PIPECONF_DITHER_EN |
4443                                     PIPECONF_DITHER_TYPE_SP;
4444
4445                 switch (crtc_state->pipe_bpp) {
4446                 case 18:
4447                         pipeconf |= PIPECONF_6BPC;
4448                         break;
4449                 case 24:
4450                         pipeconf |= PIPECONF_8BPC;
4451                         break;
4452                 case 30:
4453                         pipeconf |= PIPECONF_10BPC;
4454                         break;
4455                 default:
4456                         /* Case prevented by intel_choose_pipe_bpp_dither. */
4457                         BUG();
4458                 }
4459         }
4460
4461         if (crtc_state->hw.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE) {
4462                 if (DISPLAY_VER(dev_priv) < 4 ||
4463                     intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO))
4464                         pipeconf |= PIPECONF_INTERLACE_W_FIELD_INDICATION;
4465                 else
4466                         pipeconf |= PIPECONF_INTERLACE_W_SYNC_SHIFT;
4467         } else {
4468                 pipeconf |= PIPECONF_PROGRESSIVE;
4469         }
4470
4471         if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
4472              crtc_state->limited_color_range)
4473                 pipeconf |= PIPECONF_COLOR_RANGE_SELECT;
4474
4475         pipeconf |= PIPECONF_GAMMA_MODE(crtc_state->gamma_mode);
4476
4477         pipeconf |= PIPECONF_FRAME_START_DELAY(dev_priv->framestart_delay - 1);
4478
4479         intel_de_write(dev_priv, PIPECONF(crtc->pipe), pipeconf);
4480         intel_de_posting_read(dev_priv, PIPECONF(crtc->pipe));
4481 }
4482
4483 static bool i9xx_has_pfit(struct drm_i915_private *dev_priv)
4484 {
4485         if (IS_I830(dev_priv))
4486                 return false;
4487
4488         return DISPLAY_VER(dev_priv) >= 4 ||
4489                 IS_PINEVIEW(dev_priv) || IS_MOBILE(dev_priv);
4490 }
4491
4492 static void i9xx_get_pfit_config(struct intel_crtc_state *crtc_state)
4493 {
4494         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
4495         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4496         u32 tmp;
4497
4498         if (!i9xx_has_pfit(dev_priv))
4499                 return;
4500
4501         tmp = intel_de_read(dev_priv, PFIT_CONTROL);
4502         if (!(tmp & PFIT_ENABLE))
4503                 return;
4504
4505         /* Check whether the pfit is attached to our pipe. */
4506         if (DISPLAY_VER(dev_priv) < 4) {
4507                 if (crtc->pipe != PIPE_B)
4508                         return;
4509         } else {
4510                 if ((tmp & PFIT_PIPE_MASK) != (crtc->pipe << PFIT_PIPE_SHIFT))
4511                         return;
4512         }
4513
4514         crtc_state->gmch_pfit.control = tmp;
4515         crtc_state->gmch_pfit.pgm_ratios =
4516                 intel_de_read(dev_priv, PFIT_PGM_RATIOS);
4517 }
4518
4519 static void vlv_crtc_clock_get(struct intel_crtc *crtc,
4520                                struct intel_crtc_state *pipe_config)
4521 {
4522         struct drm_device *dev = crtc->base.dev;
4523         struct drm_i915_private *dev_priv = to_i915(dev);
4524         enum pipe pipe = crtc->pipe;
4525         struct dpll clock;
4526         u32 mdiv;
4527         int refclk = 100000;
4528
4529         /* In case of DSI, DPLL will not be used */
4530         if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0)
4531                 return;
4532
4533         vlv_dpio_get(dev_priv);
4534         mdiv = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW3(pipe));
4535         vlv_dpio_put(dev_priv);
4536
4537         clock.m1 = (mdiv >> DPIO_M1DIV_SHIFT) & 7;
4538         clock.m2 = mdiv & DPIO_M2DIV_MASK;
4539         clock.n = (mdiv >> DPIO_N_SHIFT) & 0xf;
4540         clock.p1 = (mdiv >> DPIO_P1_SHIFT) & 7;
4541         clock.p2 = (mdiv >> DPIO_P2_SHIFT) & 0x1f;
4542
4543         pipe_config->port_clock = vlv_calc_dpll_params(refclk, &clock);
4544 }
4545
4546 static void chv_crtc_clock_get(struct intel_crtc *crtc,
4547                                struct intel_crtc_state *pipe_config)
4548 {
4549         struct drm_device *dev = crtc->base.dev;
4550         struct drm_i915_private *dev_priv = to_i915(dev);
4551         enum pipe pipe = crtc->pipe;
4552         enum dpio_channel port = vlv_pipe_to_channel(pipe);
4553         struct dpll clock;
4554         u32 cmn_dw13, pll_dw0, pll_dw1, pll_dw2, pll_dw3;
4555         int refclk = 100000;
4556
4557         /* In case of DSI, DPLL will not be used */
4558         if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0)
4559                 return;
4560
4561         vlv_dpio_get(dev_priv);
4562         cmn_dw13 = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW13(port));
4563         pll_dw0 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW0(port));
4564         pll_dw1 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW1(port));
4565         pll_dw2 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW2(port));
4566         pll_dw3 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW3(port));
4567         vlv_dpio_put(dev_priv);
4568
4569         clock.m1 = (pll_dw1 & 0x7) == DPIO_CHV_M1_DIV_BY_2 ? 2 : 0;
4570         clock.m2 = (pll_dw0 & 0xff) << 22;
4571         if (pll_dw3 & DPIO_CHV_FRAC_DIV_EN)
4572                 clock.m2 |= pll_dw2 & 0x3fffff;
4573         clock.n = (pll_dw1 >> DPIO_CHV_N_DIV_SHIFT) & 0xf;
4574         clock.p1 = (cmn_dw13 >> DPIO_CHV_P1_DIV_SHIFT) & 0x7;
4575         clock.p2 = (cmn_dw13 >> DPIO_CHV_P2_DIV_SHIFT) & 0x1f;
4576
4577         pipe_config->port_clock = chv_calc_dpll_params(refclk, &clock);
4578 }
4579
4580 static enum intel_output_format
4581 bdw_get_pipemisc_output_format(struct intel_crtc *crtc)
4582 {
4583         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4584         u32 tmp;
4585
4586         tmp = intel_de_read(dev_priv, PIPEMISC(crtc->pipe));
4587
4588         if (tmp & PIPEMISC_YUV420_ENABLE) {
4589                 /* We support 4:2:0 in full blend mode only */
4590                 drm_WARN_ON(&dev_priv->drm,
4591                             (tmp & PIPEMISC_YUV420_MODE_FULL_BLEND) == 0);
4592
4593                 return INTEL_OUTPUT_FORMAT_YCBCR420;
4594         } else if (tmp & PIPEMISC_OUTPUT_COLORSPACE_YUV) {
4595                 return INTEL_OUTPUT_FORMAT_YCBCR444;
4596         } else {
4597                 return INTEL_OUTPUT_FORMAT_RGB;
4598         }
4599 }
4600
4601 static void i9xx_get_pipe_color_config(struct intel_crtc_state *crtc_state)
4602 {
4603         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
4604         struct intel_plane *plane = to_intel_plane(crtc->base.primary);
4605         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4606         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
4607         u32 tmp;
4608
4609         tmp = intel_de_read(dev_priv, DSPCNTR(i9xx_plane));
4610
4611         if (tmp & DISPPLANE_GAMMA_ENABLE)
4612                 crtc_state->gamma_enable = true;
4613
4614         if (!HAS_GMCH(dev_priv) &&
4615             tmp & DISPPLANE_PIPE_CSC_ENABLE)
4616                 crtc_state->csc_enable = true;
4617 }
4618
4619 static bool i9xx_get_pipe_config(struct intel_crtc *crtc,
4620                                  struct intel_crtc_state *pipe_config)
4621 {
4622         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4623         enum intel_display_power_domain power_domain;
4624         intel_wakeref_t wakeref;
4625         u32 tmp;
4626         bool ret;
4627
4628         power_domain = POWER_DOMAIN_PIPE(crtc->pipe);
4629         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
4630         if (!wakeref)
4631                 return false;
4632
4633         pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB;
4634         pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe;
4635         pipe_config->shared_dpll = NULL;
4636
4637         ret = false;
4638
4639         tmp = intel_de_read(dev_priv, PIPECONF(crtc->pipe));
4640         if (!(tmp & PIPECONF_ENABLE))
4641                 goto out;
4642
4643         if (IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
4644             IS_CHERRYVIEW(dev_priv)) {
4645                 switch (tmp & PIPECONF_BPC_MASK) {
4646                 case PIPECONF_6BPC:
4647                         pipe_config->pipe_bpp = 18;
4648                         break;
4649                 case PIPECONF_8BPC:
4650                         pipe_config->pipe_bpp = 24;
4651                         break;
4652                 case PIPECONF_10BPC:
4653                         pipe_config->pipe_bpp = 30;
4654                         break;
4655                 default:
4656                         break;
4657                 }
4658         }
4659
4660         if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
4661             (tmp & PIPECONF_COLOR_RANGE_SELECT))
4662                 pipe_config->limited_color_range = true;
4663
4664         pipe_config->gamma_mode = (tmp & PIPECONF_GAMMA_MODE_MASK_I9XX) >>
4665                 PIPECONF_GAMMA_MODE_SHIFT;
4666
4667         if (IS_CHERRYVIEW(dev_priv))
4668                 pipe_config->cgm_mode = intel_de_read(dev_priv,
4669                                                       CGM_PIPE_MODE(crtc->pipe));
4670
4671         i9xx_get_pipe_color_config(pipe_config);
4672         intel_color_get_config(pipe_config);
4673
4674         if (DISPLAY_VER(dev_priv) < 4)
4675                 pipe_config->double_wide = tmp & PIPECONF_DOUBLE_WIDE;
4676
4677         intel_get_transcoder_timings(crtc, pipe_config);
4678         intel_get_pipe_src_size(crtc, pipe_config);
4679
4680         i9xx_get_pfit_config(pipe_config);
4681
4682         if (DISPLAY_VER(dev_priv) >= 4) {
4683                 /* No way to read it out on pipes B and C */
4684                 if (IS_CHERRYVIEW(dev_priv) && crtc->pipe != PIPE_A)
4685                         tmp = dev_priv->chv_dpll_md[crtc->pipe];
4686                 else
4687                         tmp = intel_de_read(dev_priv, DPLL_MD(crtc->pipe));
4688                 pipe_config->pixel_multiplier =
4689                         ((tmp & DPLL_MD_UDI_MULTIPLIER_MASK)
4690                          >> DPLL_MD_UDI_MULTIPLIER_SHIFT) + 1;
4691                 pipe_config->dpll_hw_state.dpll_md = tmp;
4692         } else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv) ||
4693                    IS_G33(dev_priv) || IS_PINEVIEW(dev_priv)) {
4694                 tmp = intel_de_read(dev_priv, DPLL(crtc->pipe));
4695                 pipe_config->pixel_multiplier =
4696                         ((tmp & SDVO_MULTIPLIER_MASK)
4697                          >> SDVO_MULTIPLIER_SHIFT_HIRES) + 1;
4698         } else {
4699                 /* Note that on i915G/GM the pixel multiplier is in the sdvo
4700                  * port and will be fixed up in the encoder->get_config
4701                  * function. */
4702                 pipe_config->pixel_multiplier = 1;
4703         }
4704         pipe_config->dpll_hw_state.dpll = intel_de_read(dev_priv,
4705                                                         DPLL(crtc->pipe));
4706         if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv)) {
4707                 pipe_config->dpll_hw_state.fp0 = intel_de_read(dev_priv,
4708                                                                FP0(crtc->pipe));
4709                 pipe_config->dpll_hw_state.fp1 = intel_de_read(dev_priv,
4710                                                                FP1(crtc->pipe));
4711         } else {
4712                 /* Mask out read-only status bits. */
4713                 pipe_config->dpll_hw_state.dpll &= ~(DPLL_LOCK_VLV |
4714                                                      DPLL_PORTC_READY_MASK |
4715                                                      DPLL_PORTB_READY_MASK);
4716         }
4717
4718         if (IS_CHERRYVIEW(dev_priv))
4719                 chv_crtc_clock_get(crtc, pipe_config);
4720         else if (IS_VALLEYVIEW(dev_priv))
4721                 vlv_crtc_clock_get(crtc, pipe_config);
4722         else
4723                 i9xx_crtc_clock_get(crtc, pipe_config);
4724
4725         /*
4726          * Normally the dotclock is filled in by the encoder .get_config()
4727          * but in case the pipe is enabled w/o any ports we need a sane
4728          * default.
4729          */
4730         pipe_config->hw.adjusted_mode.crtc_clock =
4731                 pipe_config->port_clock / pipe_config->pixel_multiplier;
4732
4733         ret = true;
4734
4735 out:
4736         intel_display_power_put(dev_priv, power_domain, wakeref);
4737
4738         return ret;
4739 }
4740
4741 static void ilk_init_pch_refclk(struct drm_i915_private *dev_priv)
4742 {
4743         struct intel_encoder *encoder;
4744         int i;
4745         u32 val, final;
4746         bool has_lvds = false;
4747         bool has_cpu_edp = false;
4748         bool has_panel = false;
4749         bool has_ck505 = false;
4750         bool can_ssc = false;
4751         bool using_ssc_source = false;
4752
4753         /* We need to take the global config into account */
4754         for_each_intel_encoder(&dev_priv->drm, encoder) {
4755                 switch (encoder->type) {
4756                 case INTEL_OUTPUT_LVDS:
4757                         has_panel = true;
4758                         has_lvds = true;
4759                         break;
4760                 case INTEL_OUTPUT_EDP:
4761                         has_panel = true;
4762                         if (encoder->port == PORT_A)
4763                                 has_cpu_edp = true;
4764                         break;
4765                 default:
4766                         break;
4767                 }
4768         }
4769
4770         if (HAS_PCH_IBX(dev_priv)) {
4771                 has_ck505 = dev_priv->vbt.display_clock_mode;
4772                 can_ssc = has_ck505;
4773         } else {
4774                 has_ck505 = false;
4775                 can_ssc = true;
4776         }
4777
4778         /* Check if any DPLLs are using the SSC source */
4779         for (i = 0; i < dev_priv->dpll.num_shared_dpll; i++) {
4780                 u32 temp = intel_de_read(dev_priv, PCH_DPLL(i));
4781
4782                 if (!(temp & DPLL_VCO_ENABLE))
4783                         continue;
4784
4785                 if ((temp & PLL_REF_INPUT_MASK) ==
4786                     PLLB_REF_INPUT_SPREADSPECTRUMIN) {
4787                         using_ssc_source = true;
4788                         break;
4789                 }
4790         }
4791
4792         drm_dbg_kms(&dev_priv->drm,
4793                     "has_panel %d has_lvds %d has_ck505 %d using_ssc_source %d\n",
4794                     has_panel, has_lvds, has_ck505, using_ssc_source);
4795
4796         /* Ironlake: try to setup display ref clock before DPLL
4797          * enabling. This is only under driver's control after
4798          * PCH B stepping, previous chipset stepping should be
4799          * ignoring this setting.
4800          */
4801         val = intel_de_read(dev_priv, PCH_DREF_CONTROL);
4802
4803         /* As we must carefully and slowly disable/enable each source in turn,
4804          * compute the final state we want first and check if we need to
4805          * make any changes at all.
4806          */
4807         final = val;
4808         final &= ~DREF_NONSPREAD_SOURCE_MASK;
4809         if (has_ck505)
4810                 final |= DREF_NONSPREAD_CK505_ENABLE;
4811         else
4812                 final |= DREF_NONSPREAD_SOURCE_ENABLE;
4813
4814         final &= ~DREF_SSC_SOURCE_MASK;
4815         final &= ~DREF_CPU_SOURCE_OUTPUT_MASK;
4816         final &= ~DREF_SSC1_ENABLE;
4817
4818         if (has_panel) {
4819                 final |= DREF_SSC_SOURCE_ENABLE;
4820
4821                 if (intel_panel_use_ssc(dev_priv) && can_ssc)
4822                         final |= DREF_SSC1_ENABLE;
4823
4824                 if (has_cpu_edp) {
4825                         if (intel_panel_use_ssc(dev_priv) && can_ssc)
4826                                 final |= DREF_CPU_SOURCE_OUTPUT_DOWNSPREAD;
4827                         else
4828                                 final |= DREF_CPU_SOURCE_OUTPUT_NONSPREAD;
4829                 } else
4830                         final |= DREF_CPU_SOURCE_OUTPUT_DISABLE;
4831         } else if (using_ssc_source) {
4832                 final |= DREF_SSC_SOURCE_ENABLE;
4833                 final |= DREF_SSC1_ENABLE;
4834         }
4835
4836         if (final == val)
4837                 return;
4838
4839         /* Always enable nonspread source */
4840         val &= ~DREF_NONSPREAD_SOURCE_MASK;
4841
4842         if (has_ck505)
4843                 val |= DREF_NONSPREAD_CK505_ENABLE;
4844         else
4845                 val |= DREF_NONSPREAD_SOURCE_ENABLE;
4846
4847         if (has_panel) {
4848                 val &= ~DREF_SSC_SOURCE_MASK;
4849                 val |= DREF_SSC_SOURCE_ENABLE;
4850
4851                 /* SSC must be turned on before enabling the CPU output  */
4852                 if (intel_panel_use_ssc(dev_priv) && can_ssc) {
4853                         drm_dbg_kms(&dev_priv->drm, "Using SSC on panel\n");
4854                         val |= DREF_SSC1_ENABLE;
4855                 } else
4856                         val &= ~DREF_SSC1_ENABLE;
4857
4858                 /* Get SSC going before enabling the outputs */
4859                 intel_de_write(dev_priv, PCH_DREF_CONTROL, val);
4860                 intel_de_posting_read(dev_priv, PCH_DREF_CONTROL);
4861                 udelay(200);
4862
4863                 val &= ~DREF_CPU_SOURCE_OUTPUT_MASK;
4864
4865                 /* Enable CPU source on CPU attached eDP */
4866                 if (has_cpu_edp) {
4867                         if (intel_panel_use_ssc(dev_priv) && can_ssc) {
4868                                 drm_dbg_kms(&dev_priv->drm,
4869                                             "Using SSC on eDP\n");
4870                                 val |= DREF_CPU_SOURCE_OUTPUT_DOWNSPREAD;
4871                         } else
4872                                 val |= DREF_CPU_SOURCE_OUTPUT_NONSPREAD;
4873                 } else
4874                         val |= DREF_CPU_SOURCE_OUTPUT_DISABLE;
4875
4876                 intel_de_write(dev_priv, PCH_DREF_CONTROL, val);
4877                 intel_de_posting_read(dev_priv, PCH_DREF_CONTROL);
4878                 udelay(200);
4879         } else {
4880                 drm_dbg_kms(&dev_priv->drm, "Disabling CPU source output\n");
4881
4882                 val &= ~DREF_CPU_SOURCE_OUTPUT_MASK;
4883
4884                 /* Turn off CPU output */
4885                 val |= DREF_CPU_SOURCE_OUTPUT_DISABLE;
4886
4887                 intel_de_write(dev_priv, PCH_DREF_CONTROL, val);
4888                 intel_de_posting_read(dev_priv, PCH_DREF_CONTROL);
4889                 udelay(200);
4890
4891                 if (!using_ssc_source) {
4892                         drm_dbg_kms(&dev_priv->drm, "Disabling SSC source\n");
4893
4894                         /* Turn off the SSC source */
4895                         val &= ~DREF_SSC_SOURCE_MASK;
4896                         val |= DREF_SSC_SOURCE_DISABLE;
4897
4898                         /* Turn off SSC1 */
4899                         val &= ~DREF_SSC1_ENABLE;
4900
4901                         intel_de_write(dev_priv, PCH_DREF_CONTROL, val);
4902                         intel_de_posting_read(dev_priv, PCH_DREF_CONTROL);
4903                         udelay(200);
4904                 }
4905         }
4906
4907         BUG_ON(val != final);
4908 }
4909
4910 /* Implements 3 different sequences from BSpec chapter "Display iCLK
4911  * Programming" based on the parameters passed:
4912  * - Sequence to enable CLKOUT_DP
4913  * - Sequence to enable CLKOUT_DP without spread
4914  * - Sequence to enable CLKOUT_DP for FDI usage and configure PCH FDI I/O
4915  */
4916 static void lpt_enable_clkout_dp(struct drm_i915_private *dev_priv,
4917                                  bool with_spread, bool with_fdi)
4918 {
4919         u32 reg, tmp;
4920
4921         if (drm_WARN(&dev_priv->drm, with_fdi && !with_spread,
4922                      "FDI requires downspread\n"))
4923                 with_spread = true;
4924         if (drm_WARN(&dev_priv->drm, HAS_PCH_LPT_LP(dev_priv) &&
4925                      with_fdi, "LP PCH doesn't have FDI\n"))
4926                 with_fdi = false;
4927
4928         mutex_lock(&dev_priv->sb_lock);
4929
4930         tmp = intel_sbi_read(dev_priv, SBI_SSCCTL, SBI_ICLK);
4931         tmp &= ~SBI_SSCCTL_DISABLE;
4932         tmp |= SBI_SSCCTL_PATHALT;
4933         intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
4934
4935         udelay(24);
4936
4937         if (with_spread) {
4938                 tmp = intel_sbi_read(dev_priv, SBI_SSCCTL, SBI_ICLK);
4939                 tmp &= ~SBI_SSCCTL_PATHALT;
4940                 intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
4941
4942                 if (with_fdi)
4943                         lpt_fdi_program_mphy(dev_priv);
4944         }
4945
4946         reg = HAS_PCH_LPT_LP(dev_priv) ? SBI_GEN0 : SBI_DBUFF0;
4947         tmp = intel_sbi_read(dev_priv, reg, SBI_ICLK);
4948         tmp |= SBI_GEN0_CFG_BUFFENABLE_DISABLE;
4949         intel_sbi_write(dev_priv, reg, tmp, SBI_ICLK);
4950
4951         mutex_unlock(&dev_priv->sb_lock);
4952 }
4953
4954 /* Sequence to disable CLKOUT_DP */
4955 void lpt_disable_clkout_dp(struct drm_i915_private *dev_priv)
4956 {
4957         u32 reg, tmp;
4958
4959         mutex_lock(&dev_priv->sb_lock);
4960
4961         reg = HAS_PCH_LPT_LP(dev_priv) ? SBI_GEN0 : SBI_DBUFF0;
4962         tmp = intel_sbi_read(dev_priv, reg, SBI_ICLK);
4963         tmp &= ~SBI_GEN0_CFG_BUFFENABLE_DISABLE;
4964         intel_sbi_write(dev_priv, reg, tmp, SBI_ICLK);
4965
4966         tmp = intel_sbi_read(dev_priv, SBI_SSCCTL, SBI_ICLK);
4967         if (!(tmp & SBI_SSCCTL_DISABLE)) {
4968                 if (!(tmp & SBI_SSCCTL_PATHALT)) {
4969                         tmp |= SBI_SSCCTL_PATHALT;
4970                         intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
4971                         udelay(32);
4972                 }
4973                 tmp |= SBI_SSCCTL_DISABLE;
4974                 intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
4975         }
4976
4977         mutex_unlock(&dev_priv->sb_lock);
4978 }
4979
4980 #define BEND_IDX(steps) ((50 + (steps)) / 5)
4981
4982 static const u16 sscdivintphase[] = {
4983         [BEND_IDX( 50)] = 0x3B23,
4984         [BEND_IDX( 45)] = 0x3B23,
4985         [BEND_IDX( 40)] = 0x3C23,
4986         [BEND_IDX( 35)] = 0x3C23,
4987         [BEND_IDX( 30)] = 0x3D23,
4988         [BEND_IDX( 25)] = 0x3D23,
4989         [BEND_IDX( 20)] = 0x3E23,
4990         [BEND_IDX( 15)] = 0x3E23,
4991         [BEND_IDX( 10)] = 0x3F23,
4992         [BEND_IDX(  5)] = 0x3F23,
4993         [BEND_IDX(  0)] = 0x0025,
4994         [BEND_IDX( -5)] = 0x0025,
4995         [BEND_IDX(-10)] = 0x0125,
4996         [BEND_IDX(-15)] = 0x0125,
4997         [BEND_IDX(-20)] = 0x0225,
4998         [BEND_IDX(-25)] = 0x0225,
4999         [BEND_IDX(-30)] = 0x0325,
5000         [BEND_IDX(-35)] = 0x0325,
5001         [BEND_IDX(-40)] = 0x0425,
5002         [BEND_IDX(-45)] = 0x0425,
5003         [BEND_IDX(-50)] = 0x0525,
5004 };
5005
5006 /*
5007  * Bend CLKOUT_DP
5008  * steps -50 to 50 inclusive, in steps of 5
5009  * < 0 slow down the clock, > 0 speed up the clock, 0 == no bend (135MHz)
5010  * change in clock period = -(steps / 10) * 5.787 ps
5011  */
5012 static void lpt_bend_clkout_dp(struct drm_i915_private *dev_priv, int steps)
5013 {
5014         u32 tmp;
5015         int idx = BEND_IDX(steps);
5016
5017         if (drm_WARN_ON(&dev_priv->drm, steps % 5 != 0))
5018                 return;
5019
5020         if (drm_WARN_ON(&dev_priv->drm, idx >= ARRAY_SIZE(sscdivintphase)))
5021                 return;
5022
5023         mutex_lock(&dev_priv->sb_lock);
5024
5025         if (steps % 10 != 0)
5026                 tmp = 0xAAAAAAAB;
5027         else
5028                 tmp = 0x00000000;
5029         intel_sbi_write(dev_priv, SBI_SSCDITHPHASE, tmp, SBI_ICLK);
5030
5031         tmp = intel_sbi_read(dev_priv, SBI_SSCDIVINTPHASE, SBI_ICLK);
5032         tmp &= 0xffff0000;
5033         tmp |= sscdivintphase[idx];
5034         intel_sbi_write(dev_priv, SBI_SSCDIVINTPHASE, tmp, SBI_ICLK);
5035
5036         mutex_unlock(&dev_priv->sb_lock);
5037 }
5038
5039 #undef BEND_IDX
5040
5041 static bool spll_uses_pch_ssc(struct drm_i915_private *dev_priv)
5042 {
5043         u32 fuse_strap = intel_de_read(dev_priv, FUSE_STRAP);
5044         u32 ctl = intel_de_read(dev_priv, SPLL_CTL);
5045
5046         if ((ctl & SPLL_PLL_ENABLE) == 0)
5047                 return false;
5048
5049         if ((ctl & SPLL_REF_MASK) == SPLL_REF_MUXED_SSC &&
5050             (fuse_strap & HSW_CPU_SSC_ENABLE) == 0)
5051                 return true;
5052
5053         if (IS_BROADWELL(dev_priv) &&
5054             (ctl & SPLL_REF_MASK) == SPLL_REF_PCH_SSC_BDW)
5055                 return true;
5056
5057         return false;
5058 }
5059
5060 static bool wrpll_uses_pch_ssc(struct drm_i915_private *dev_priv,
5061                                enum intel_dpll_id id)
5062 {
5063         u32 fuse_strap = intel_de_read(dev_priv, FUSE_STRAP);
5064         u32 ctl = intel_de_read(dev_priv, WRPLL_CTL(id));
5065
5066         if ((ctl & WRPLL_PLL_ENABLE) == 0)
5067                 return false;
5068
5069         if ((ctl & WRPLL_REF_MASK) == WRPLL_REF_PCH_SSC)
5070                 return true;
5071
5072         if ((IS_BROADWELL(dev_priv) || IS_HSW_ULT(dev_priv)) &&
5073             (ctl & WRPLL_REF_MASK) == WRPLL_REF_MUXED_SSC_BDW &&
5074             (fuse_strap & HSW_CPU_SSC_ENABLE) == 0)
5075                 return true;
5076
5077         return false;
5078 }
5079
5080 static void lpt_init_pch_refclk(struct drm_i915_private *dev_priv)
5081 {
5082         struct intel_encoder *encoder;
5083         bool has_fdi = false;
5084
5085         for_each_intel_encoder(&dev_priv->drm, encoder) {
5086                 switch (encoder->type) {
5087                 case INTEL_OUTPUT_ANALOG:
5088                         has_fdi = true;
5089                         break;
5090                 default:
5091                         break;
5092                 }
5093         }
5094
5095         /*
5096          * The BIOS may have decided to use the PCH SSC
5097          * reference so we must not disable it until the
5098          * relevant PLLs have stopped relying on it. We'll
5099          * just leave the PCH SSC reference enabled in case
5100          * any active PLL is using it. It will get disabled
5101          * after runtime suspend if we don't have FDI.
5102          *
5103          * TODO: Move the whole reference clock handling
5104          * to the modeset sequence proper so that we can
5105          * actually enable/disable/reconfigure these things
5106          * safely. To do that we need to introduce a real
5107          * clock hierarchy. That would also allow us to do
5108          * clock bending finally.
5109          */
5110         dev_priv->pch_ssc_use = 0;
5111
5112         if (spll_uses_pch_ssc(dev_priv)) {
5113                 drm_dbg_kms(&dev_priv->drm, "SPLL using PCH SSC\n");
5114                 dev_priv->pch_ssc_use |= BIT(DPLL_ID_SPLL);
5115         }
5116
5117         if (wrpll_uses_pch_ssc(dev_priv, DPLL_ID_WRPLL1)) {
5118                 drm_dbg_kms(&dev_priv->drm, "WRPLL1 using PCH SSC\n");
5119                 dev_priv->pch_ssc_use |= BIT(DPLL_ID_WRPLL1);
5120         }
5121
5122         if (wrpll_uses_pch_ssc(dev_priv, DPLL_ID_WRPLL2)) {
5123                 drm_dbg_kms(&dev_priv->drm, "WRPLL2 using PCH SSC\n");
5124                 dev_priv->pch_ssc_use |= BIT(DPLL_ID_WRPLL2);
5125         }
5126
5127         if (dev_priv->pch_ssc_use)
5128                 return;
5129
5130         if (has_fdi) {
5131                 lpt_bend_clkout_dp(dev_priv, 0);
5132                 lpt_enable_clkout_dp(dev_priv, true, true);
5133         } else {
5134                 lpt_disable_clkout_dp(dev_priv);
5135         }
5136 }
5137
5138 /*
5139  * Initialize reference clocks when the driver loads
5140  */
5141 void intel_init_pch_refclk(struct drm_i915_private *dev_priv)
5142 {
5143         if (HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv))
5144                 ilk_init_pch_refclk(dev_priv);
5145         else if (HAS_PCH_LPT(dev_priv))
5146                 lpt_init_pch_refclk(dev_priv);
5147 }
5148
5149 static void ilk_set_pipeconf(const struct intel_crtc_state *crtc_state)
5150 {
5151         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5152         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5153         enum pipe pipe = crtc->pipe;
5154         u32 val;
5155
5156         val = 0;
5157
5158         switch (crtc_state->pipe_bpp) {
5159         case 18:
5160                 val |= PIPECONF_6BPC;
5161                 break;
5162         case 24:
5163                 val |= PIPECONF_8BPC;
5164                 break;
5165         case 30:
5166                 val |= PIPECONF_10BPC;
5167                 break;
5168         case 36:
5169                 val |= PIPECONF_12BPC;
5170                 break;
5171         default:
5172                 /* Case prevented by intel_choose_pipe_bpp_dither. */
5173                 BUG();
5174         }
5175
5176         if (crtc_state->dither)
5177                 val |= (PIPECONF_DITHER_EN | PIPECONF_DITHER_TYPE_SP);
5178
5179         if (crtc_state->hw.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE)
5180                 val |= PIPECONF_INTERLACED_ILK;
5181         else
5182                 val |= PIPECONF_PROGRESSIVE;
5183
5184         /*
5185          * This would end up with an odd purple hue over
5186          * the entire display. Make sure we don't do it.
5187          */
5188         drm_WARN_ON(&dev_priv->drm, crtc_state->limited_color_range &&
5189                     crtc_state->output_format != INTEL_OUTPUT_FORMAT_RGB);
5190
5191         if (crtc_state->limited_color_range &&
5192             !intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO))
5193                 val |= PIPECONF_COLOR_RANGE_SELECT;
5194
5195         if (crtc_state->output_format != INTEL_OUTPUT_FORMAT_RGB)
5196                 val |= PIPECONF_OUTPUT_COLORSPACE_YUV709;
5197
5198         val |= PIPECONF_GAMMA_MODE(crtc_state->gamma_mode);
5199
5200         val |= PIPECONF_FRAME_START_DELAY(dev_priv->framestart_delay - 1);
5201
5202         intel_de_write(dev_priv, PIPECONF(pipe), val);
5203         intel_de_posting_read(dev_priv, PIPECONF(pipe));
5204 }
5205
5206 static void hsw_set_transconf(const struct intel_crtc_state *crtc_state)
5207 {
5208         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5209         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5210         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
5211         u32 val = 0;
5212
5213         if (IS_HASWELL(dev_priv) && crtc_state->dither)
5214                 val |= (PIPECONF_DITHER_EN | PIPECONF_DITHER_TYPE_SP);
5215
5216         if (crtc_state->hw.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE)
5217                 val |= PIPECONF_INTERLACED_ILK;
5218         else
5219                 val |= PIPECONF_PROGRESSIVE;
5220
5221         if (IS_HASWELL(dev_priv) &&
5222             crtc_state->output_format != INTEL_OUTPUT_FORMAT_RGB)
5223                 val |= PIPECONF_OUTPUT_COLORSPACE_YUV_HSW;
5224
5225         intel_de_write(dev_priv, PIPECONF(cpu_transcoder), val);
5226         intel_de_posting_read(dev_priv, PIPECONF(cpu_transcoder));
5227 }
5228
5229 static void bdw_set_pipemisc(const struct intel_crtc_state *crtc_state)
5230 {
5231         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5232         const struct intel_crtc_scaler_state *scaler_state =
5233                 &crtc_state->scaler_state;
5234
5235         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5236         u32 val = 0;
5237         int i;
5238
5239         switch (crtc_state->pipe_bpp) {
5240         case 18:
5241                 val |= PIPEMISC_6_BPC;
5242                 break;
5243         case 24:
5244                 val |= PIPEMISC_8_BPC;
5245                 break;
5246         case 30:
5247                 val |= PIPEMISC_10_BPC;
5248                 break;
5249         case 36:
5250                 /* Port output 12BPC defined for ADLP+ */
5251                 if (DISPLAY_VER(dev_priv) > 12)
5252                         val |= PIPEMISC_12_BPC_ADLP;
5253                 break;
5254         default:
5255                 MISSING_CASE(crtc_state->pipe_bpp);
5256                 break;
5257         }
5258
5259         if (crtc_state->dither)
5260                 val |= PIPEMISC_DITHER_ENABLE | PIPEMISC_DITHER_TYPE_SP;
5261
5262         if (crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR420 ||
5263             crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR444)
5264                 val |= PIPEMISC_OUTPUT_COLORSPACE_YUV;
5265
5266         if (crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR420)
5267                 val |= PIPEMISC_YUV420_ENABLE |
5268                         PIPEMISC_YUV420_MODE_FULL_BLEND;
5269
5270         if (DISPLAY_VER(dev_priv) >= 11 && is_hdr_mode(crtc_state))
5271                 val |= PIPEMISC_HDR_MODE_PRECISION;
5272
5273         if (DISPLAY_VER(dev_priv) >= 12)
5274                 val |= PIPEMISC_PIXEL_ROUNDING_TRUNC;
5275
5276         if (IS_ALDERLAKE_P(dev_priv)) {
5277                 bool scaler_in_use = false;
5278
5279                 for (i = 0; i < crtc->num_scalers; i++) {
5280                         if (!scaler_state->scalers[i].in_use)
5281                                 continue;
5282
5283                         scaler_in_use = true;
5284                         break;
5285                 }
5286
5287                 intel_de_rmw(dev_priv, PIPE_MISC2(crtc->pipe),
5288                              PIPE_MISC2_UNDERRUN_BUBBLE_COUNTER_MASK,
5289                              scaler_in_use ? PIPE_MISC2_BUBBLE_COUNTER_SCALER_EN :
5290                              PIPE_MISC2_BUBBLE_COUNTER_SCALER_DIS);
5291         }
5292
5293         intel_de_write(dev_priv, PIPEMISC(crtc->pipe), val);
5294 }
5295
5296 int bdw_get_pipemisc_bpp(struct intel_crtc *crtc)
5297 {
5298         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5299         u32 tmp;
5300
5301         tmp = intel_de_read(dev_priv, PIPEMISC(crtc->pipe));
5302
5303         switch (tmp & PIPEMISC_BPC_MASK) {
5304         case PIPEMISC_6_BPC:
5305                 return 18;
5306         case PIPEMISC_8_BPC:
5307                 return 24;
5308         case PIPEMISC_10_BPC:
5309                 return 30;
5310         /*
5311          * PORT OUTPUT 12 BPC defined for ADLP+.
5312          *
5313          * TODO:
5314          * For previous platforms with DSI interface, bits 5:7
5315          * are used for storing pipe_bpp irrespective of dithering.
5316          * Since the value of 12 BPC is not defined for these bits
5317          * on older platforms, need to find a workaround for 12 BPC
5318          * MIPI DSI HW readout.
5319          */
5320         case PIPEMISC_12_BPC_ADLP:
5321                 if (DISPLAY_VER(dev_priv) > 12)
5322                         return 36;
5323                 fallthrough;
5324         default:
5325                 MISSING_CASE(tmp);
5326                 return 0;
5327         }
5328 }
5329
5330 int ilk_get_lanes_required(int target_clock, int link_bw, int bpp)
5331 {
5332         /*
5333          * Account for spread spectrum to avoid
5334          * oversubscribing the link. Max center spread
5335          * is 2.5%; use 5% for safety's sake.
5336          */
5337         u32 bps = target_clock * bpp * 21 / 20;
5338         return DIV_ROUND_UP(bps, link_bw * 8);
5339 }
5340
5341 static void intel_pch_transcoder_get_m_n(struct intel_crtc *crtc,
5342                                          struct intel_link_m_n *m_n)
5343 {
5344         struct drm_device *dev = crtc->base.dev;
5345         struct drm_i915_private *dev_priv = to_i915(dev);
5346         enum pipe pipe = crtc->pipe;
5347
5348         m_n->link_m = intel_de_read(dev_priv, PCH_TRANS_LINK_M1(pipe));
5349         m_n->link_n = intel_de_read(dev_priv, PCH_TRANS_LINK_N1(pipe));
5350         m_n->gmch_m = intel_de_read(dev_priv, PCH_TRANS_DATA_M1(pipe))
5351                 & ~TU_SIZE_MASK;
5352         m_n->gmch_n = intel_de_read(dev_priv, PCH_TRANS_DATA_N1(pipe));
5353         m_n->tu = ((intel_de_read(dev_priv, PCH_TRANS_DATA_M1(pipe))
5354                     & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
5355 }
5356
5357 static void intel_cpu_transcoder_get_m_n(struct intel_crtc *crtc,
5358                                          enum transcoder transcoder,
5359                                          struct intel_link_m_n *m_n,
5360                                          struct intel_link_m_n *m2_n2)
5361 {
5362         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5363         enum pipe pipe = crtc->pipe;
5364
5365         if (DISPLAY_VER(dev_priv) >= 5) {
5366                 m_n->link_m = intel_de_read(dev_priv,
5367                                             PIPE_LINK_M1(transcoder));
5368                 m_n->link_n = intel_de_read(dev_priv,
5369                                             PIPE_LINK_N1(transcoder));
5370                 m_n->gmch_m = intel_de_read(dev_priv,
5371                                             PIPE_DATA_M1(transcoder))
5372                         & ~TU_SIZE_MASK;
5373                 m_n->gmch_n = intel_de_read(dev_priv,
5374                                             PIPE_DATA_N1(transcoder));
5375                 m_n->tu = ((intel_de_read(dev_priv, PIPE_DATA_M1(transcoder))
5376                             & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
5377
5378                 if (m2_n2 && transcoder_has_m2_n2(dev_priv, transcoder)) {
5379                         m2_n2->link_m = intel_de_read(dev_priv,
5380                                                       PIPE_LINK_M2(transcoder));
5381                         m2_n2->link_n = intel_de_read(dev_priv,
5382                                                              PIPE_LINK_N2(transcoder));
5383                         m2_n2->gmch_m = intel_de_read(dev_priv,
5384                                                              PIPE_DATA_M2(transcoder))
5385                                         & ~TU_SIZE_MASK;
5386                         m2_n2->gmch_n = intel_de_read(dev_priv,
5387                                                              PIPE_DATA_N2(transcoder));
5388                         m2_n2->tu = ((intel_de_read(dev_priv, PIPE_DATA_M2(transcoder))
5389                                         & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
5390                 }
5391         } else {
5392                 m_n->link_m = intel_de_read(dev_priv, PIPE_LINK_M_G4X(pipe));
5393                 m_n->link_n = intel_de_read(dev_priv, PIPE_LINK_N_G4X(pipe));
5394                 m_n->gmch_m = intel_de_read(dev_priv, PIPE_DATA_M_G4X(pipe))
5395                         & ~TU_SIZE_MASK;
5396                 m_n->gmch_n = intel_de_read(dev_priv, PIPE_DATA_N_G4X(pipe));
5397                 m_n->tu = ((intel_de_read(dev_priv, PIPE_DATA_M_G4X(pipe))
5398                             & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
5399         }
5400 }
5401
5402 void intel_dp_get_m_n(struct intel_crtc *crtc,
5403                       struct intel_crtc_state *pipe_config)
5404 {
5405         if (pipe_config->has_pch_encoder)
5406                 intel_pch_transcoder_get_m_n(crtc, &pipe_config->dp_m_n);
5407         else
5408                 intel_cpu_transcoder_get_m_n(crtc, pipe_config->cpu_transcoder,
5409                                              &pipe_config->dp_m_n,
5410                                              &pipe_config->dp_m2_n2);
5411 }
5412
5413 static void ilk_get_fdi_m_n_config(struct intel_crtc *crtc,
5414                                    struct intel_crtc_state *pipe_config)
5415 {
5416         intel_cpu_transcoder_get_m_n(crtc, pipe_config->cpu_transcoder,
5417                                      &pipe_config->fdi_m_n, NULL);
5418 }
5419
5420 static void ilk_get_pfit_pos_size(struct intel_crtc_state *crtc_state,
5421                                   u32 pos, u32 size)
5422 {
5423         drm_rect_init(&crtc_state->pch_pfit.dst,
5424                       pos >> 16, pos & 0xffff,
5425                       size >> 16, size & 0xffff);
5426 }
5427
5428 static void skl_get_pfit_config(struct intel_crtc_state *crtc_state)
5429 {
5430         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5431         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5432         struct intel_crtc_scaler_state *scaler_state = &crtc_state->scaler_state;
5433         int id = -1;
5434         int i;
5435
5436         /* find scaler attached to this pipe */
5437         for (i = 0; i < crtc->num_scalers; i++) {
5438                 u32 ctl, pos, size;
5439
5440                 ctl = intel_de_read(dev_priv, SKL_PS_CTRL(crtc->pipe, i));
5441                 if ((ctl & (PS_SCALER_EN | PS_PLANE_SEL_MASK)) != PS_SCALER_EN)
5442                         continue;
5443
5444                 id = i;
5445                 crtc_state->pch_pfit.enabled = true;
5446
5447                 pos = intel_de_read(dev_priv, SKL_PS_WIN_POS(crtc->pipe, i));
5448                 size = intel_de_read(dev_priv, SKL_PS_WIN_SZ(crtc->pipe, i));
5449
5450                 ilk_get_pfit_pos_size(crtc_state, pos, size);
5451
5452                 scaler_state->scalers[i].in_use = true;
5453                 break;
5454         }
5455
5456         scaler_state->scaler_id = id;
5457         if (id >= 0)
5458                 scaler_state->scaler_users |= (1 << SKL_CRTC_INDEX);
5459         else
5460                 scaler_state->scaler_users &= ~(1 << SKL_CRTC_INDEX);
5461 }
5462
5463 static void ilk_get_pfit_config(struct intel_crtc_state *crtc_state)
5464 {
5465         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5466         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5467         u32 ctl, pos, size;
5468
5469         ctl = intel_de_read(dev_priv, PF_CTL(crtc->pipe));
5470         if ((ctl & PF_ENABLE) == 0)
5471                 return;
5472
5473         crtc_state->pch_pfit.enabled = true;
5474
5475         pos = intel_de_read(dev_priv, PF_WIN_POS(crtc->pipe));
5476         size = intel_de_read(dev_priv, PF_WIN_SZ(crtc->pipe));
5477
5478         ilk_get_pfit_pos_size(crtc_state, pos, size);
5479
5480         /*
5481          * We currently do not free assignements of panel fitters on
5482          * ivb/hsw (since we don't use the higher upscaling modes which
5483          * differentiates them) so just WARN about this case for now.
5484          */
5485         drm_WARN_ON(&dev_priv->drm, DISPLAY_VER(dev_priv) == 7 &&
5486                     (ctl & PF_PIPE_SEL_MASK_IVB) != PF_PIPE_SEL_IVB(crtc->pipe));
5487 }
5488
5489 static bool ilk_get_pipe_config(struct intel_crtc *crtc,
5490                                 struct intel_crtc_state *pipe_config)
5491 {
5492         struct drm_device *dev = crtc->base.dev;
5493         struct drm_i915_private *dev_priv = to_i915(dev);
5494         enum intel_display_power_domain power_domain;
5495         intel_wakeref_t wakeref;
5496         u32 tmp;
5497         bool ret;
5498
5499         power_domain = POWER_DOMAIN_PIPE(crtc->pipe);
5500         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
5501         if (!wakeref)
5502                 return false;
5503
5504         pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe;
5505         pipe_config->shared_dpll = NULL;
5506
5507         ret = false;
5508         tmp = intel_de_read(dev_priv, PIPECONF(crtc->pipe));
5509         if (!(tmp & PIPECONF_ENABLE))
5510                 goto out;
5511
5512         switch (tmp & PIPECONF_BPC_MASK) {
5513         case PIPECONF_6BPC:
5514                 pipe_config->pipe_bpp = 18;
5515                 break;
5516         case PIPECONF_8BPC:
5517                 pipe_config->pipe_bpp = 24;
5518                 break;
5519         case PIPECONF_10BPC:
5520                 pipe_config->pipe_bpp = 30;
5521                 break;
5522         case PIPECONF_12BPC:
5523                 pipe_config->pipe_bpp = 36;
5524                 break;
5525         default:
5526                 break;
5527         }
5528
5529         if (tmp & PIPECONF_COLOR_RANGE_SELECT)
5530                 pipe_config->limited_color_range = true;
5531
5532         switch (tmp & PIPECONF_OUTPUT_COLORSPACE_MASK) {
5533         case PIPECONF_OUTPUT_COLORSPACE_YUV601:
5534         case PIPECONF_OUTPUT_COLORSPACE_YUV709:
5535                 pipe_config->output_format = INTEL_OUTPUT_FORMAT_YCBCR444;
5536                 break;
5537         default:
5538                 pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB;
5539                 break;
5540         }
5541
5542         pipe_config->gamma_mode = (tmp & PIPECONF_GAMMA_MODE_MASK_ILK) >>
5543                 PIPECONF_GAMMA_MODE_SHIFT;
5544
5545         pipe_config->csc_mode = intel_de_read(dev_priv,
5546                                               PIPE_CSC_MODE(crtc->pipe));
5547
5548         i9xx_get_pipe_color_config(pipe_config);
5549         intel_color_get_config(pipe_config);
5550
5551         if (intel_de_read(dev_priv, PCH_TRANSCONF(crtc->pipe)) & TRANS_ENABLE) {
5552                 struct intel_shared_dpll *pll;
5553                 enum intel_dpll_id pll_id;
5554                 bool pll_active;
5555
5556                 pipe_config->has_pch_encoder = true;
5557
5558                 tmp = intel_de_read(dev_priv, FDI_RX_CTL(crtc->pipe));
5559                 pipe_config->fdi_lanes = ((FDI_DP_PORT_WIDTH_MASK & tmp) >>
5560                                           FDI_DP_PORT_WIDTH_SHIFT) + 1;
5561
5562                 ilk_get_fdi_m_n_config(crtc, pipe_config);
5563
5564                 if (HAS_PCH_IBX(dev_priv)) {
5565                         /*
5566                          * The pipe->pch transcoder and pch transcoder->pll
5567                          * mapping is fixed.
5568                          */
5569                         pll_id = (enum intel_dpll_id) crtc->pipe;
5570                 } else {
5571                         tmp = intel_de_read(dev_priv, PCH_DPLL_SEL);
5572                         if (tmp & TRANS_DPLLB_SEL(crtc->pipe))
5573                                 pll_id = DPLL_ID_PCH_PLL_B;
5574                         else
5575                                 pll_id= DPLL_ID_PCH_PLL_A;
5576                 }
5577
5578                 pipe_config->shared_dpll =
5579                         intel_get_shared_dpll_by_id(dev_priv, pll_id);
5580                 pll = pipe_config->shared_dpll;
5581
5582                 pll_active = intel_dpll_get_hw_state(dev_priv, pll,
5583                                                      &pipe_config->dpll_hw_state);
5584                 drm_WARN_ON(dev, !pll_active);
5585
5586                 tmp = pipe_config->dpll_hw_state.dpll;
5587                 pipe_config->pixel_multiplier =
5588                         ((tmp & PLL_REF_SDVO_HDMI_MULTIPLIER_MASK)
5589                          >> PLL_REF_SDVO_HDMI_MULTIPLIER_SHIFT) + 1;
5590
5591                 ilk_pch_clock_get(crtc, pipe_config);
5592         } else {
5593                 pipe_config->pixel_multiplier = 1;
5594         }
5595
5596         intel_get_transcoder_timings(crtc, pipe_config);
5597         intel_get_pipe_src_size(crtc, pipe_config);
5598
5599         ilk_get_pfit_config(pipe_config);
5600
5601         ret = true;
5602
5603 out:
5604         intel_display_power_put(dev_priv, power_domain, wakeref);
5605
5606         return ret;
5607 }
5608
5609 static u8 hsw_panel_transcoders(struct drm_i915_private *i915)
5610 {
5611         u8 panel_transcoder_mask = BIT(TRANSCODER_EDP);
5612
5613         if (DISPLAY_VER(i915) >= 11)
5614                 panel_transcoder_mask |= BIT(TRANSCODER_DSI_0) | BIT(TRANSCODER_DSI_1);
5615
5616         return panel_transcoder_mask;
5617 }
5618
5619 static bool hsw_get_transcoder_state(struct intel_crtc *crtc,
5620                                      struct intel_crtc_state *pipe_config,
5621                                      struct intel_display_power_domain_set *power_domain_set)
5622 {
5623         struct drm_device *dev = crtc->base.dev;
5624         struct drm_i915_private *dev_priv = to_i915(dev);
5625         u8 panel_transcoder_mask = hsw_panel_transcoders(dev_priv);
5626         unsigned long enabled_panel_transcoders = 0;
5627         enum transcoder panel_transcoder;
5628         u32 tmp;
5629
5630         /*
5631          * The pipe->transcoder mapping is fixed with the exception of the eDP
5632          * and DSI transcoders handled below.
5633          */
5634         pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe;
5635
5636         /*
5637          * XXX: Do intel_display_power_get_if_enabled before reading this (for
5638          * consistency and less surprising code; it's in always on power).
5639          */
5640         for_each_cpu_transcoder_masked(dev_priv, panel_transcoder,
5641                                        panel_transcoder_mask) {
5642                 bool force_thru = false;
5643                 enum pipe trans_pipe;
5644
5645                 tmp = intel_de_read(dev_priv,
5646                                     TRANS_DDI_FUNC_CTL(panel_transcoder));
5647                 if (!(tmp & TRANS_DDI_FUNC_ENABLE))
5648                         continue;
5649
5650                 /*
5651                  * Log all enabled ones, only use the first one.
5652                  *
5653                  * FIXME: This won't work for two separate DSI displays.
5654                  */
5655                 enabled_panel_transcoders |= BIT(panel_transcoder);
5656                 if (enabled_panel_transcoders != BIT(panel_transcoder))
5657                         continue;
5658
5659                 switch (tmp & TRANS_DDI_EDP_INPUT_MASK) {
5660                 default:
5661                         drm_WARN(dev, 1,
5662                                  "unknown pipe linked to transcoder %s\n",
5663                                  transcoder_name(panel_transcoder));
5664                         fallthrough;
5665                 case TRANS_DDI_EDP_INPUT_A_ONOFF:
5666                         force_thru = true;
5667                         fallthrough;
5668                 case TRANS_DDI_EDP_INPUT_A_ON:
5669                         trans_pipe = PIPE_A;
5670                         break;
5671                 case TRANS_DDI_EDP_INPUT_B_ONOFF:
5672                         trans_pipe = PIPE_B;
5673                         break;
5674                 case TRANS_DDI_EDP_INPUT_C_ONOFF:
5675                         trans_pipe = PIPE_C;
5676                         break;
5677                 case TRANS_DDI_EDP_INPUT_D_ONOFF:
5678                         trans_pipe = PIPE_D;
5679                         break;
5680                 }
5681
5682                 if (trans_pipe == crtc->pipe) {
5683                         pipe_config->cpu_transcoder = panel_transcoder;
5684                         pipe_config->pch_pfit.force_thru = force_thru;
5685                 }
5686         }
5687
5688         /*
5689          * Valid combos: none, eDP, DSI0, DSI1, DSI0+DSI1
5690          */
5691         drm_WARN_ON(dev, (enabled_panel_transcoders & BIT(TRANSCODER_EDP)) &&
5692                     enabled_panel_transcoders != BIT(TRANSCODER_EDP));
5693
5694         if (!intel_display_power_get_in_set_if_enabled(dev_priv, power_domain_set,
5695                                                        POWER_DOMAIN_TRANSCODER(pipe_config->cpu_transcoder)))
5696                 return false;
5697
5698         tmp = intel_de_read(dev_priv, PIPECONF(pipe_config->cpu_transcoder));
5699
5700         return tmp & PIPECONF_ENABLE;
5701 }
5702
5703 static bool bxt_get_dsi_transcoder_state(struct intel_crtc *crtc,
5704                                          struct intel_crtc_state *pipe_config,
5705                                          struct intel_display_power_domain_set *power_domain_set)
5706 {
5707         struct drm_device *dev = crtc->base.dev;
5708         struct drm_i915_private *dev_priv = to_i915(dev);
5709         enum transcoder cpu_transcoder;
5710         enum port port;
5711         u32 tmp;
5712
5713         for_each_port_masked(port, BIT(PORT_A) | BIT(PORT_C)) {
5714                 if (port == PORT_A)
5715                         cpu_transcoder = TRANSCODER_DSI_A;
5716                 else
5717                         cpu_transcoder = TRANSCODER_DSI_C;
5718
5719                 if (!intel_display_power_get_in_set_if_enabled(dev_priv, power_domain_set,
5720                                                                POWER_DOMAIN_TRANSCODER(cpu_transcoder)))
5721                         continue;
5722
5723                 /*
5724                  * The PLL needs to be enabled with a valid divider
5725                  * configuration, otherwise accessing DSI registers will hang
5726                  * the machine. See BSpec North Display Engine
5727                  * registers/MIPI[BXT]. We can break out here early, since we
5728                  * need the same DSI PLL to be enabled for both DSI ports.
5729                  */
5730                 if (!bxt_dsi_pll_is_enabled(dev_priv))
5731                         break;
5732
5733                 /* XXX: this works for video mode only */
5734                 tmp = intel_de_read(dev_priv, BXT_MIPI_PORT_CTRL(port));
5735                 if (!(tmp & DPI_ENABLE))
5736                         continue;
5737
5738                 tmp = intel_de_read(dev_priv, MIPI_CTRL(port));
5739                 if ((tmp & BXT_PIPE_SELECT_MASK) != BXT_PIPE_SELECT(crtc->pipe))
5740                         continue;
5741
5742                 pipe_config->cpu_transcoder = cpu_transcoder;
5743                 break;
5744         }
5745
5746         return transcoder_is_dsi(pipe_config->cpu_transcoder);
5747 }
5748
5749 static void hsw_get_ddi_port_state(struct intel_crtc *crtc,
5750                                    struct intel_crtc_state *pipe_config)
5751 {
5752         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5753         enum transcoder cpu_transcoder = pipe_config->cpu_transcoder;
5754         enum port port;
5755         u32 tmp;
5756
5757         if (transcoder_is_dsi(cpu_transcoder)) {
5758                 port = (cpu_transcoder == TRANSCODER_DSI_A) ?
5759                                                 PORT_A : PORT_B;
5760         } else {
5761                 tmp = intel_de_read(dev_priv,
5762                                     TRANS_DDI_FUNC_CTL(cpu_transcoder));
5763                 if (!(tmp & TRANS_DDI_FUNC_ENABLE))
5764                         return;
5765                 if (DISPLAY_VER(dev_priv) >= 12)
5766                         port = TGL_TRANS_DDI_FUNC_CTL_VAL_TO_PORT(tmp);
5767                 else
5768                         port = TRANS_DDI_FUNC_CTL_VAL_TO_PORT(tmp);
5769         }
5770
5771         /*
5772          * Haswell has only FDI/PCH transcoder A. It is which is connected to
5773          * DDI E. So just check whether this pipe is wired to DDI E and whether
5774          * the PCH transcoder is on.
5775          */
5776         if (DISPLAY_VER(dev_priv) < 9 &&
5777             (port == PORT_E) && intel_de_read(dev_priv, LPT_TRANSCONF) & TRANS_ENABLE) {
5778                 pipe_config->has_pch_encoder = true;
5779
5780                 tmp = intel_de_read(dev_priv, FDI_RX_CTL(PIPE_A));
5781                 pipe_config->fdi_lanes = ((FDI_DP_PORT_WIDTH_MASK & tmp) >>
5782                                           FDI_DP_PORT_WIDTH_SHIFT) + 1;
5783
5784                 ilk_get_fdi_m_n_config(crtc, pipe_config);
5785         }
5786 }
5787
5788 static bool hsw_get_pipe_config(struct intel_crtc *crtc,
5789                                 struct intel_crtc_state *pipe_config)
5790 {
5791         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5792         struct intel_display_power_domain_set power_domain_set = { };
5793         bool active;
5794         u32 tmp;
5795
5796         if (!intel_display_power_get_in_set_if_enabled(dev_priv, &power_domain_set,
5797                                                        POWER_DOMAIN_PIPE(crtc->pipe)))
5798                 return false;
5799
5800         pipe_config->shared_dpll = NULL;
5801
5802         active = hsw_get_transcoder_state(crtc, pipe_config, &power_domain_set);
5803
5804         if ((IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) &&
5805             bxt_get_dsi_transcoder_state(crtc, pipe_config, &power_domain_set)) {
5806                 drm_WARN_ON(&dev_priv->drm, active);
5807                 active = true;
5808         }
5809
5810         intel_dsc_get_config(pipe_config);
5811         if (DISPLAY_VER(dev_priv) >= 13 && !pipe_config->dsc.compression_enable)
5812                 intel_uncompressed_joiner_get_config(pipe_config);
5813
5814         if (!active) {
5815                 /* bigjoiner slave doesn't enable transcoder */
5816                 if (!pipe_config->bigjoiner_slave)
5817                         goto out;
5818
5819                 active = true;
5820                 pipe_config->pixel_multiplier = 1;
5821
5822                 /* we cannot read out most state, so don't bother.. */
5823                 pipe_config->quirks |= PIPE_CONFIG_QUIRK_BIGJOINER_SLAVE;
5824         } else if (!transcoder_is_dsi(pipe_config->cpu_transcoder) ||
5825             DISPLAY_VER(dev_priv) >= 11) {
5826                 hsw_get_ddi_port_state(crtc, pipe_config);
5827                 intel_get_transcoder_timings(crtc, pipe_config);
5828         }
5829
5830         if (HAS_VRR(dev_priv) && !transcoder_is_dsi(pipe_config->cpu_transcoder))
5831                 intel_vrr_get_config(crtc, pipe_config);
5832
5833         intel_get_pipe_src_size(crtc, pipe_config);
5834
5835         if (IS_HASWELL(dev_priv)) {
5836                 u32 tmp = intel_de_read(dev_priv,
5837                                         PIPECONF(pipe_config->cpu_transcoder));
5838
5839                 if (tmp & PIPECONF_OUTPUT_COLORSPACE_YUV_HSW)
5840                         pipe_config->output_format = INTEL_OUTPUT_FORMAT_YCBCR444;
5841                 else
5842                         pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB;
5843         } else {
5844                 pipe_config->output_format =
5845                         bdw_get_pipemisc_output_format(crtc);
5846         }
5847
5848         pipe_config->gamma_mode = intel_de_read(dev_priv,
5849                                                 GAMMA_MODE(crtc->pipe));
5850
5851         pipe_config->csc_mode = intel_de_read(dev_priv,
5852                                               PIPE_CSC_MODE(crtc->pipe));
5853
5854         if (DISPLAY_VER(dev_priv) >= 9) {
5855                 tmp = intel_de_read(dev_priv, SKL_BOTTOM_COLOR(crtc->pipe));
5856
5857                 if (tmp & SKL_BOTTOM_COLOR_GAMMA_ENABLE)
5858                         pipe_config->gamma_enable = true;
5859
5860                 if (tmp & SKL_BOTTOM_COLOR_CSC_ENABLE)
5861                         pipe_config->csc_enable = true;
5862         } else {
5863                 i9xx_get_pipe_color_config(pipe_config);
5864         }
5865
5866         intel_color_get_config(pipe_config);
5867
5868         tmp = intel_de_read(dev_priv, WM_LINETIME(crtc->pipe));
5869         pipe_config->linetime = REG_FIELD_GET(HSW_LINETIME_MASK, tmp);
5870         if (IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
5871                 pipe_config->ips_linetime =
5872                         REG_FIELD_GET(HSW_IPS_LINETIME_MASK, tmp);
5873
5874         if (intel_display_power_get_in_set_if_enabled(dev_priv, &power_domain_set,
5875                                                       POWER_DOMAIN_PIPE_PANEL_FITTER(crtc->pipe))) {
5876                 if (DISPLAY_VER(dev_priv) >= 9)
5877                         skl_get_pfit_config(pipe_config);
5878                 else
5879                         ilk_get_pfit_config(pipe_config);
5880         }
5881
5882         if (hsw_crtc_supports_ips(crtc)) {
5883                 if (IS_HASWELL(dev_priv))
5884                         pipe_config->ips_enabled = intel_de_read(dev_priv,
5885                                                                  IPS_CTL) & IPS_ENABLE;
5886                 else {
5887                         /*
5888                          * We cannot readout IPS state on broadwell, set to
5889                          * true so we can set it to a defined state on first
5890                          * commit.
5891                          */
5892                         pipe_config->ips_enabled = true;
5893                 }
5894         }
5895
5896         if (pipe_config->bigjoiner_slave) {
5897                 /* Cannot be read out as a slave, set to 0. */
5898                 pipe_config->pixel_multiplier = 0;
5899         } else if (pipe_config->cpu_transcoder != TRANSCODER_EDP &&
5900             !transcoder_is_dsi(pipe_config->cpu_transcoder)) {
5901                 pipe_config->pixel_multiplier =
5902                         intel_de_read(dev_priv,
5903                                       PIPE_MULT(pipe_config->cpu_transcoder)) + 1;
5904         } else {
5905                 pipe_config->pixel_multiplier = 1;
5906         }
5907
5908 out:
5909         intel_display_power_put_all_in_set(dev_priv, &power_domain_set);
5910
5911         return active;
5912 }
5913
5914 static bool intel_crtc_get_pipe_config(struct intel_crtc_state *crtc_state)
5915 {
5916         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5917         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
5918
5919         if (!i915->display.get_pipe_config(crtc, crtc_state))
5920                 return false;
5921
5922         crtc_state->hw.active = true;
5923
5924         intel_crtc_readout_derived_state(crtc_state);
5925
5926         return true;
5927 }
5928
5929 /* VESA 640x480x72Hz mode to set on the pipe */
5930 static const struct drm_display_mode load_detect_mode = {
5931         DRM_MODE("640x480", DRM_MODE_TYPE_DEFAULT, 31500, 640, 664,
5932                  704, 832, 0, 480, 489, 491, 520, 0, DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
5933 };
5934
5935 static int intel_modeset_disable_planes(struct drm_atomic_state *state,
5936                                         struct drm_crtc *crtc)
5937 {
5938         struct drm_plane *plane;
5939         struct drm_plane_state *plane_state;
5940         int ret, i;
5941
5942         ret = drm_atomic_add_affected_planes(state, crtc);
5943         if (ret)
5944                 return ret;
5945
5946         for_each_new_plane_in_state(state, plane, plane_state, i) {
5947                 if (plane_state->crtc != crtc)
5948                         continue;
5949
5950                 ret = drm_atomic_set_crtc_for_plane(plane_state, NULL);
5951                 if (ret)
5952                         return ret;
5953
5954                 drm_atomic_set_fb_for_plane(plane_state, NULL);
5955         }
5956
5957         return 0;
5958 }
5959
5960 int intel_get_load_detect_pipe(struct drm_connector *connector,
5961                                struct intel_load_detect_pipe *old,
5962                                struct drm_modeset_acquire_ctx *ctx)
5963 {
5964         struct intel_encoder *encoder =
5965                 intel_attached_encoder(to_intel_connector(connector));
5966         struct intel_crtc *possible_crtc;
5967         struct intel_crtc *crtc = NULL;
5968         struct drm_device *dev = encoder->base.dev;
5969         struct drm_i915_private *dev_priv = to_i915(dev);
5970         struct drm_mode_config *config = &dev->mode_config;
5971         struct drm_atomic_state *state = NULL, *restore_state = NULL;
5972         struct drm_connector_state *connector_state;
5973         struct intel_crtc_state *crtc_state;
5974         int ret;
5975
5976         drm_dbg_kms(&dev_priv->drm, "[CONNECTOR:%d:%s], [ENCODER:%d:%s]\n",
5977                     connector->base.id, connector->name,
5978                     encoder->base.base.id, encoder->base.name);
5979
5980         old->restore_state = NULL;
5981
5982         drm_WARN_ON(dev, !drm_modeset_is_locked(&config->connection_mutex));
5983
5984         /*
5985          * Algorithm gets a little messy:
5986          *
5987          *   - if the connector already has an assigned crtc, use it (but make
5988          *     sure it's on first)
5989          *
5990          *   - try to find the first unused crtc that can drive this connector,
5991          *     and use that if we find one
5992          */
5993
5994         /* See if we already have a CRTC for this connector */
5995         if (connector->state->crtc) {
5996                 crtc = to_intel_crtc(connector->state->crtc);
5997
5998                 ret = drm_modeset_lock(&crtc->base.mutex, ctx);
5999                 if (ret)
6000                         goto fail;
6001
6002                 /* Make sure the crtc and connector are running */
6003                 goto found;
6004         }
6005
6006         /* Find an unused one (if possible) */
6007         for_each_intel_crtc(dev, possible_crtc) {
6008                 if (!(encoder->base.possible_crtcs &
6009                       drm_crtc_mask(&possible_crtc->base)))
6010                         continue;
6011
6012                 ret = drm_modeset_lock(&possible_crtc->base.mutex, ctx);
6013                 if (ret)
6014                         goto fail;
6015
6016                 if (possible_crtc->base.state->enable) {
6017                         drm_modeset_unlock(&possible_crtc->base.mutex);
6018                         continue;
6019                 }
6020
6021                 crtc = possible_crtc;
6022                 break;
6023         }
6024
6025         /*
6026          * If we didn't find an unused CRTC, don't use any.
6027          */
6028         if (!crtc) {
6029                 drm_dbg_kms(&dev_priv->drm,
6030                             "no pipe available for load-detect\n");
6031                 ret = -ENODEV;
6032                 goto fail;
6033         }
6034
6035 found:
6036         state = drm_atomic_state_alloc(dev);
6037         restore_state = drm_atomic_state_alloc(dev);
6038         if (!state || !restore_state) {
6039                 ret = -ENOMEM;
6040                 goto fail;
6041         }
6042
6043         state->acquire_ctx = ctx;
6044         restore_state->acquire_ctx = ctx;
6045
6046         connector_state = drm_atomic_get_connector_state(state, connector);
6047         if (IS_ERR(connector_state)) {
6048                 ret = PTR_ERR(connector_state);
6049                 goto fail;
6050         }
6051
6052         ret = drm_atomic_set_crtc_for_connector(connector_state, &crtc->base);
6053         if (ret)
6054                 goto fail;
6055
6056         crtc_state = intel_atomic_get_crtc_state(state, crtc);
6057         if (IS_ERR(crtc_state)) {
6058                 ret = PTR_ERR(crtc_state);
6059                 goto fail;
6060         }
6061
6062         crtc_state->uapi.active = true;
6063
6064         ret = drm_atomic_set_mode_for_crtc(&crtc_state->uapi,
6065                                            &load_detect_mode);
6066         if (ret)
6067                 goto fail;
6068
6069         ret = intel_modeset_disable_planes(state, &crtc->base);
6070         if (ret)
6071                 goto fail;
6072
6073         ret = PTR_ERR_OR_ZERO(drm_atomic_get_connector_state(restore_state, connector));
6074         if (!ret)
6075                 ret = PTR_ERR_OR_ZERO(drm_atomic_get_crtc_state(restore_state, &crtc->base));
6076         if (!ret)
6077                 ret = drm_atomic_add_affected_planes(restore_state, &crtc->base);
6078         if (ret) {
6079                 drm_dbg_kms(&dev_priv->drm,
6080                             "Failed to create a copy of old state to restore: %i\n",
6081                             ret);
6082                 goto fail;
6083         }
6084
6085         ret = drm_atomic_commit(state);
6086         if (ret) {
6087                 drm_dbg_kms(&dev_priv->drm,
6088                             "failed to set mode on load-detect pipe\n");
6089                 goto fail;
6090         }
6091
6092         old->restore_state = restore_state;
6093         drm_atomic_state_put(state);
6094
6095         /* let the connector get through one full cycle before testing */
6096         intel_wait_for_vblank(dev_priv, crtc->pipe);
6097         return true;
6098
6099 fail:
6100         if (state) {
6101                 drm_atomic_state_put(state);
6102                 state = NULL;
6103         }
6104         if (restore_state) {
6105                 drm_atomic_state_put(restore_state);
6106                 restore_state = NULL;
6107         }
6108
6109         if (ret == -EDEADLK)
6110                 return ret;
6111
6112         return false;
6113 }
6114
6115 void intel_release_load_detect_pipe(struct drm_connector *connector,
6116                                     struct intel_load_detect_pipe *old,
6117                                     struct drm_modeset_acquire_ctx *ctx)
6118 {
6119         struct intel_encoder *intel_encoder =
6120                 intel_attached_encoder(to_intel_connector(connector));
6121         struct drm_i915_private *i915 = to_i915(intel_encoder->base.dev);
6122         struct drm_encoder *encoder = &intel_encoder->base;
6123         struct drm_atomic_state *state = old->restore_state;
6124         int ret;
6125
6126         drm_dbg_kms(&i915->drm, "[CONNECTOR:%d:%s], [ENCODER:%d:%s]\n",
6127                     connector->base.id, connector->name,
6128                     encoder->base.id, encoder->name);
6129
6130         if (!state)
6131                 return;
6132
6133         ret = drm_atomic_helper_commit_duplicated_state(state, ctx);
6134         if (ret)
6135                 drm_dbg_kms(&i915->drm,
6136                             "Couldn't release load detect pipe: %i\n", ret);
6137         drm_atomic_state_put(state);
6138 }
6139
6140 static int i9xx_pll_refclk(struct drm_device *dev,
6141                            const struct intel_crtc_state *pipe_config)
6142 {
6143         struct drm_i915_private *dev_priv = to_i915(dev);
6144         u32 dpll = pipe_config->dpll_hw_state.dpll;
6145
6146         if ((dpll & PLL_REF_INPUT_MASK) == PLLB_REF_INPUT_SPREADSPECTRUMIN)
6147                 return dev_priv->vbt.lvds_ssc_freq;
6148         else if (HAS_PCH_SPLIT(dev_priv))
6149                 return 120000;
6150         else if (DISPLAY_VER(dev_priv) != 2)
6151                 return 96000;
6152         else
6153                 return 48000;
6154 }
6155
6156 /* Returns the clock of the currently programmed mode of the given pipe. */
6157 static void i9xx_crtc_clock_get(struct intel_crtc *crtc,
6158                                 struct intel_crtc_state *pipe_config)
6159 {
6160         struct drm_device *dev = crtc->base.dev;
6161         struct drm_i915_private *dev_priv = to_i915(dev);
6162         u32 dpll = pipe_config->dpll_hw_state.dpll;
6163         u32 fp;
6164         struct dpll clock;
6165         int port_clock;
6166         int refclk = i9xx_pll_refclk(dev, pipe_config);
6167
6168         if ((dpll & DISPLAY_RATE_SELECT_FPA1) == 0)
6169                 fp = pipe_config->dpll_hw_state.fp0;
6170         else
6171                 fp = pipe_config->dpll_hw_state.fp1;
6172
6173         clock.m1 = (fp & FP_M1_DIV_MASK) >> FP_M1_DIV_SHIFT;
6174         if (IS_PINEVIEW(dev_priv)) {
6175                 clock.n = ffs((fp & FP_N_PINEVIEW_DIV_MASK) >> FP_N_DIV_SHIFT) - 1;
6176                 clock.m2 = (fp & FP_M2_PINEVIEW_DIV_MASK) >> FP_M2_DIV_SHIFT;
6177         } else {
6178                 clock.n = (fp & FP_N_DIV_MASK) >> FP_N_DIV_SHIFT;
6179                 clock.m2 = (fp & FP_M2_DIV_MASK) >> FP_M2_DIV_SHIFT;
6180         }
6181
6182         if (DISPLAY_VER(dev_priv) != 2) {
6183                 if (IS_PINEVIEW(dev_priv))
6184                         clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK_PINEVIEW) >>
6185                                 DPLL_FPA01_P1_POST_DIV_SHIFT_PINEVIEW);
6186                 else
6187                         clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK) >>
6188                                DPLL_FPA01_P1_POST_DIV_SHIFT);
6189
6190                 switch (dpll & DPLL_MODE_MASK) {
6191                 case DPLLB_MODE_DAC_SERIAL:
6192                         clock.p2 = dpll & DPLL_DAC_SERIAL_P2_CLOCK_DIV_5 ?
6193                                 5 : 10;
6194                         break;
6195                 case DPLLB_MODE_LVDS:
6196                         clock.p2 = dpll & DPLLB_LVDS_P2_CLOCK_DIV_7 ?
6197                                 7 : 14;
6198                         break;
6199                 default:
6200                         drm_dbg_kms(&dev_priv->drm,
6201                                     "Unknown DPLL mode %08x in programmed "
6202                                     "mode\n", (int)(dpll & DPLL_MODE_MASK));
6203                         return;
6204                 }
6205
6206                 if (IS_PINEVIEW(dev_priv))
6207                         port_clock = pnv_calc_dpll_params(refclk, &clock);
6208                 else
6209                         port_clock = i9xx_calc_dpll_params(refclk, &clock);
6210         } else {
6211                 enum pipe lvds_pipe;
6212
6213                 if (IS_I85X(dev_priv) &&
6214                     intel_lvds_port_enabled(dev_priv, LVDS, &lvds_pipe) &&
6215                     lvds_pipe == crtc->pipe) {
6216                         u32 lvds = intel_de_read(dev_priv, LVDS);
6217
6218                         clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK_I830_LVDS) >>
6219                                        DPLL_FPA01_P1_POST_DIV_SHIFT);
6220
6221                         if (lvds & LVDS_CLKB_POWER_UP)
6222                                 clock.p2 = 7;
6223                         else
6224                                 clock.p2 = 14;
6225                 } else {
6226                         if (dpll & PLL_P1_DIVIDE_BY_TWO)
6227                                 clock.p1 = 2;
6228                         else {
6229                                 clock.p1 = ((dpll & DPLL_FPA01_P1_POST_DIV_MASK_I830) >>
6230                                             DPLL_FPA01_P1_POST_DIV_SHIFT) + 2;
6231                         }
6232                         if (dpll & PLL_P2_DIVIDE_BY_4)
6233                                 clock.p2 = 4;
6234                         else
6235                                 clock.p2 = 2;
6236                 }
6237
6238                 port_clock = i9xx_calc_dpll_params(refclk, &clock);
6239         }
6240
6241         /*
6242          * This value includes pixel_multiplier. We will use
6243          * port_clock to compute adjusted_mode.crtc_clock in the
6244          * encoder's get_config() function.
6245          */
6246         pipe_config->port_clock = port_clock;
6247 }
6248
6249 int intel_dotclock_calculate(int link_freq,
6250                              const struct intel_link_m_n *m_n)
6251 {
6252         /*
6253          * The calculation for the data clock is:
6254          * pixel_clock = ((m/n)*(link_clock * nr_lanes))/bpp
6255          * But we want to avoid losing precison if possible, so:
6256          * pixel_clock = ((m * link_clock * nr_lanes)/(n*bpp))
6257          *
6258          * and the link clock is simpler:
6259          * link_clock = (m * link_clock) / n
6260          */
6261
6262         if (!m_n->link_n)
6263                 return 0;
6264
6265         return div_u64(mul_u32_u32(m_n->link_m, link_freq), m_n->link_n);
6266 }
6267
6268 static void ilk_pch_clock_get(struct intel_crtc *crtc,
6269                               struct intel_crtc_state *pipe_config)
6270 {
6271         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6272
6273         /* read out port_clock from the DPLL */
6274         i9xx_crtc_clock_get(crtc, pipe_config);
6275
6276         /*
6277          * In case there is an active pipe without active ports,
6278          * we may need some idea for the dotclock anyway.
6279          * Calculate one based on the FDI configuration.
6280          */
6281         pipe_config->hw.adjusted_mode.crtc_clock =
6282                 intel_dotclock_calculate(intel_fdi_link_freq(dev_priv, pipe_config),
6283                                          &pipe_config->fdi_m_n);
6284 }
6285
6286 /* Returns the currently programmed mode of the given encoder. */
6287 struct drm_display_mode *
6288 intel_encoder_current_mode(struct intel_encoder *encoder)
6289 {
6290         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
6291         struct intel_crtc_state *crtc_state;
6292         struct drm_display_mode *mode;
6293         struct intel_crtc *crtc;
6294         enum pipe pipe;
6295
6296         if (!encoder->get_hw_state(encoder, &pipe))
6297                 return NULL;
6298
6299         crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
6300
6301         mode = kzalloc(sizeof(*mode), GFP_KERNEL);
6302         if (!mode)
6303                 return NULL;
6304
6305         crtc_state = intel_crtc_state_alloc(crtc);
6306         if (!crtc_state) {
6307                 kfree(mode);
6308                 return NULL;
6309         }
6310
6311         if (!intel_crtc_get_pipe_config(crtc_state)) {
6312                 kfree(crtc_state);
6313                 kfree(mode);
6314                 return NULL;
6315         }
6316
6317         intel_encoder_get_config(encoder, crtc_state);
6318
6319         intel_mode_from_crtc_timings(mode, &crtc_state->hw.adjusted_mode);
6320
6321         kfree(crtc_state);
6322
6323         return mode;
6324 }
6325
6326 /**
6327  * intel_wm_need_update - Check whether watermarks need updating
6328  * @cur: current plane state
6329  * @new: new plane state
6330  *
6331  * Check current plane state versus the new one to determine whether
6332  * watermarks need to be recalculated.
6333  *
6334  * Returns true or false.
6335  */
6336 static bool intel_wm_need_update(const struct intel_plane_state *cur,
6337                                  struct intel_plane_state *new)
6338 {
6339         /* Update watermarks on tiling or size changes. */
6340         if (new->uapi.visible != cur->uapi.visible)
6341                 return true;
6342
6343         if (!cur->hw.fb || !new->hw.fb)
6344                 return false;
6345
6346         if (cur->hw.fb->modifier != new->hw.fb->modifier ||
6347             cur->hw.rotation != new->hw.rotation ||
6348             drm_rect_width(&new->uapi.src) != drm_rect_width(&cur->uapi.src) ||
6349             drm_rect_height(&new->uapi.src) != drm_rect_height(&cur->uapi.src) ||
6350             drm_rect_width(&new->uapi.dst) != drm_rect_width(&cur->uapi.dst) ||
6351             drm_rect_height(&new->uapi.dst) != drm_rect_height(&cur->uapi.dst))
6352                 return true;
6353
6354         return false;
6355 }
6356
6357 static bool needs_scaling(const struct intel_plane_state *state)
6358 {
6359         int src_w = drm_rect_width(&state->uapi.src) >> 16;
6360         int src_h = drm_rect_height(&state->uapi.src) >> 16;
6361         int dst_w = drm_rect_width(&state->uapi.dst);
6362         int dst_h = drm_rect_height(&state->uapi.dst);
6363
6364         return (src_w != dst_w || src_h != dst_h);
6365 }
6366
6367 int intel_plane_atomic_calc_changes(const struct intel_crtc_state *old_crtc_state,
6368                                     struct intel_crtc_state *new_crtc_state,
6369                                     const struct intel_plane_state *old_plane_state,
6370                                     struct intel_plane_state *new_plane_state)
6371 {
6372         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
6373         struct intel_plane *plane = to_intel_plane(new_plane_state->uapi.plane);
6374         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6375         bool mode_changed = intel_crtc_needs_modeset(new_crtc_state);
6376         bool was_crtc_enabled = old_crtc_state->hw.active;
6377         bool is_crtc_enabled = new_crtc_state->hw.active;
6378         bool turn_off, turn_on, visible, was_visible;
6379         int ret;
6380
6381         if (DISPLAY_VER(dev_priv) >= 9 && plane->id != PLANE_CURSOR) {
6382                 ret = skl_update_scaler_plane(new_crtc_state, new_plane_state);
6383                 if (ret)
6384                         return ret;
6385         }
6386
6387         was_visible = old_plane_state->uapi.visible;
6388         visible = new_plane_state->uapi.visible;
6389
6390         if (!was_crtc_enabled && drm_WARN_ON(&dev_priv->drm, was_visible))
6391                 was_visible = false;
6392
6393         /*
6394          * Visibility is calculated as if the crtc was on, but
6395          * after scaler setup everything depends on it being off
6396          * when the crtc isn't active.
6397          *
6398          * FIXME this is wrong for watermarks. Watermarks should also
6399          * be computed as if the pipe would be active. Perhaps move
6400          * per-plane wm computation to the .check_plane() hook, and
6401          * only combine the results from all planes in the current place?
6402          */
6403         if (!is_crtc_enabled) {
6404                 intel_plane_set_invisible(new_crtc_state, new_plane_state);
6405                 visible = false;
6406         }
6407
6408         if (!was_visible && !visible)
6409                 return 0;
6410
6411         turn_off = was_visible && (!visible || mode_changed);
6412         turn_on = visible && (!was_visible || mode_changed);
6413
6414         drm_dbg_atomic(&dev_priv->drm,
6415                        "[CRTC:%d:%s] with [PLANE:%d:%s] visible %i -> %i, off %i, on %i, ms %i\n",
6416                        crtc->base.base.id, crtc->base.name,
6417                        plane->base.base.id, plane->base.name,
6418                        was_visible, visible,
6419                        turn_off, turn_on, mode_changed);
6420
6421         if (turn_on) {
6422                 if (DISPLAY_VER(dev_priv) < 5 && !IS_G4X(dev_priv))
6423                         new_crtc_state->update_wm_pre = true;
6424
6425                 /* must disable cxsr around plane enable/disable */
6426                 if (plane->id != PLANE_CURSOR)
6427                         new_crtc_state->disable_cxsr = true;
6428         } else if (turn_off) {
6429                 if (DISPLAY_VER(dev_priv) < 5 && !IS_G4X(dev_priv))
6430                         new_crtc_state->update_wm_post = true;
6431
6432                 /* must disable cxsr around plane enable/disable */
6433                 if (plane->id != PLANE_CURSOR)
6434                         new_crtc_state->disable_cxsr = true;
6435         } else if (intel_wm_need_update(old_plane_state, new_plane_state)) {
6436                 if (DISPLAY_VER(dev_priv) < 5 && !IS_G4X(dev_priv)) {
6437                         /* FIXME bollocks */
6438                         new_crtc_state->update_wm_pre = true;
6439                         new_crtc_state->update_wm_post = true;
6440                 }
6441         }
6442
6443         if (visible || was_visible)
6444                 new_crtc_state->fb_bits |= plane->frontbuffer_bit;
6445
6446         /*
6447          * ILK/SNB DVSACNTR/Sprite Enable
6448          * IVB SPR_CTL/Sprite Enable
6449          * "When in Self Refresh Big FIFO mode, a write to enable the
6450          *  plane will be internally buffered and delayed while Big FIFO
6451          *  mode is exiting."
6452          *
6453          * Which means that enabling the sprite can take an extra frame
6454          * when we start in big FIFO mode (LP1+). Thus we need to drop
6455          * down to LP0 and wait for vblank in order to make sure the
6456          * sprite gets enabled on the next vblank after the register write.
6457          * Doing otherwise would risk enabling the sprite one frame after
6458          * we've already signalled flip completion. We can resume LP1+
6459          * once the sprite has been enabled.
6460          *
6461          *
6462          * WaCxSRDisabledForSpriteScaling:ivb
6463          * IVB SPR_SCALE/Scaling Enable
6464          * "Low Power watermarks must be disabled for at least one
6465          *  frame before enabling sprite scaling, and kept disabled
6466          *  until sprite scaling is disabled."
6467          *
6468          * ILK/SNB DVSASCALE/Scaling Enable
6469          * "When in Self Refresh Big FIFO mode, scaling enable will be
6470          *  masked off while Big FIFO mode is exiting."
6471          *
6472          * Despite the w/a only being listed for IVB we assume that
6473          * the ILK/SNB note has similar ramifications, hence we apply
6474          * the w/a on all three platforms.
6475          *
6476          * With experimental results seems this is needed also for primary
6477          * plane, not only sprite plane.
6478          */
6479         if (plane->id != PLANE_CURSOR &&
6480             (IS_IRONLAKE(dev_priv) || IS_SANDYBRIDGE(dev_priv) ||
6481              IS_IVYBRIDGE(dev_priv)) &&
6482             (turn_on || (!needs_scaling(old_plane_state) &&
6483                          needs_scaling(new_plane_state))))
6484                 new_crtc_state->disable_lp_wm = true;
6485
6486         return 0;
6487 }
6488
6489 static bool encoders_cloneable(const struct intel_encoder *a,
6490                                const struct intel_encoder *b)
6491 {
6492         /* masks could be asymmetric, so check both ways */
6493         return a == b || (a->cloneable & (1 << b->type) &&
6494                           b->cloneable & (1 << a->type));
6495 }
6496
6497 static bool check_single_encoder_cloning(struct intel_atomic_state *state,
6498                                          struct intel_crtc *crtc,
6499                                          struct intel_encoder *encoder)
6500 {
6501         struct intel_encoder *source_encoder;
6502         struct drm_connector *connector;
6503         struct drm_connector_state *connector_state;
6504         int i;
6505
6506         for_each_new_connector_in_state(&state->base, connector, connector_state, i) {
6507                 if (connector_state->crtc != &crtc->base)
6508                         continue;
6509
6510                 source_encoder =
6511                         to_intel_encoder(connector_state->best_encoder);
6512                 if (!encoders_cloneable(encoder, source_encoder))
6513                         return false;
6514         }
6515
6516         return true;
6517 }
6518
6519 static int icl_add_linked_planes(struct intel_atomic_state *state)
6520 {
6521         struct intel_plane *plane, *linked;
6522         struct intel_plane_state *plane_state, *linked_plane_state;
6523         int i;
6524
6525         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
6526                 linked = plane_state->planar_linked_plane;
6527
6528                 if (!linked)
6529                         continue;
6530
6531                 linked_plane_state = intel_atomic_get_plane_state(state, linked);
6532                 if (IS_ERR(linked_plane_state))
6533                         return PTR_ERR(linked_plane_state);
6534
6535                 drm_WARN_ON(state->base.dev,
6536                             linked_plane_state->planar_linked_plane != plane);
6537                 drm_WARN_ON(state->base.dev,
6538                             linked_plane_state->planar_slave == plane_state->planar_slave);
6539         }
6540
6541         return 0;
6542 }
6543
6544 static int icl_check_nv12_planes(struct intel_crtc_state *crtc_state)
6545 {
6546         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
6547         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6548         struct intel_atomic_state *state = to_intel_atomic_state(crtc_state->uapi.state);
6549         struct intel_plane *plane, *linked;
6550         struct intel_plane_state *plane_state;
6551         int i;
6552
6553         if (DISPLAY_VER(dev_priv) < 11)
6554                 return 0;
6555
6556         /*
6557          * Destroy all old plane links and make the slave plane invisible
6558          * in the crtc_state->active_planes mask.
6559          */
6560         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
6561                 if (plane->pipe != crtc->pipe || !plane_state->planar_linked_plane)
6562                         continue;
6563
6564                 plane_state->planar_linked_plane = NULL;
6565                 if (plane_state->planar_slave && !plane_state->uapi.visible) {
6566                         crtc_state->enabled_planes &= ~BIT(plane->id);
6567                         crtc_state->active_planes &= ~BIT(plane->id);
6568                         crtc_state->update_planes |= BIT(plane->id);
6569                 }
6570
6571                 plane_state->planar_slave = false;
6572         }
6573
6574         if (!crtc_state->nv12_planes)
6575                 return 0;
6576
6577         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
6578                 struct intel_plane_state *linked_state = NULL;
6579
6580                 if (plane->pipe != crtc->pipe ||
6581                     !(crtc_state->nv12_planes & BIT(plane->id)))
6582                         continue;
6583
6584                 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, linked) {
6585                         if (!icl_is_nv12_y_plane(dev_priv, linked->id))
6586                                 continue;
6587
6588                         if (crtc_state->active_planes & BIT(linked->id))
6589                                 continue;
6590
6591                         linked_state = intel_atomic_get_plane_state(state, linked);
6592                         if (IS_ERR(linked_state))
6593                                 return PTR_ERR(linked_state);
6594
6595                         break;
6596                 }
6597
6598                 if (!linked_state) {
6599                         drm_dbg_kms(&dev_priv->drm,
6600                                     "Need %d free Y planes for planar YUV\n",
6601                                     hweight8(crtc_state->nv12_planes));
6602
6603                         return -EINVAL;
6604                 }
6605
6606                 plane_state->planar_linked_plane = linked;
6607
6608                 linked_state->planar_slave = true;
6609                 linked_state->planar_linked_plane = plane;
6610                 crtc_state->enabled_planes |= BIT(linked->id);
6611                 crtc_state->active_planes |= BIT(linked->id);
6612                 crtc_state->update_planes |= BIT(linked->id);
6613                 drm_dbg_kms(&dev_priv->drm, "Using %s as Y plane for %s\n",
6614                             linked->base.name, plane->base.name);
6615
6616                 /* Copy parameters to slave plane */
6617                 linked_state->ctl = plane_state->ctl | PLANE_CTL_YUV420_Y_PLANE;
6618                 linked_state->color_ctl = plane_state->color_ctl;
6619                 linked_state->view = plane_state->view;
6620
6621                 intel_plane_copy_hw_state(linked_state, plane_state);
6622                 linked_state->uapi.src = plane_state->uapi.src;
6623                 linked_state->uapi.dst = plane_state->uapi.dst;
6624
6625                 if (icl_is_hdr_plane(dev_priv, plane->id)) {
6626                         if (linked->id == PLANE_SPRITE5)
6627                                 plane_state->cus_ctl |= PLANE_CUS_PLANE_7;
6628                         else if (linked->id == PLANE_SPRITE4)
6629                                 plane_state->cus_ctl |= PLANE_CUS_PLANE_6;
6630                         else if (linked->id == PLANE_SPRITE3)
6631                                 plane_state->cus_ctl |= PLANE_CUS_PLANE_5_RKL;
6632                         else if (linked->id == PLANE_SPRITE2)
6633                                 plane_state->cus_ctl |= PLANE_CUS_PLANE_4_RKL;
6634                         else
6635                                 MISSING_CASE(linked->id);
6636                 }
6637         }
6638
6639         return 0;
6640 }
6641
6642 static bool c8_planes_changed(const struct intel_crtc_state *new_crtc_state)
6643 {
6644         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
6645         struct intel_atomic_state *state =
6646                 to_intel_atomic_state(new_crtc_state->uapi.state);
6647         const struct intel_crtc_state *old_crtc_state =
6648                 intel_atomic_get_old_crtc_state(state, crtc);
6649
6650         return !old_crtc_state->c8_planes != !new_crtc_state->c8_planes;
6651 }
6652
6653 static u16 hsw_linetime_wm(const struct intel_crtc_state *crtc_state)
6654 {
6655         const struct drm_display_mode *pipe_mode =
6656                 &crtc_state->hw.pipe_mode;
6657         int linetime_wm;
6658
6659         if (!crtc_state->hw.enable)
6660                 return 0;
6661
6662         linetime_wm = DIV_ROUND_CLOSEST(pipe_mode->crtc_htotal * 1000 * 8,
6663                                         pipe_mode->crtc_clock);
6664
6665         return min(linetime_wm, 0x1ff);
6666 }
6667
6668 static u16 hsw_ips_linetime_wm(const struct intel_crtc_state *crtc_state,
6669                                const struct intel_cdclk_state *cdclk_state)
6670 {
6671         const struct drm_display_mode *pipe_mode =
6672                 &crtc_state->hw.pipe_mode;
6673         int linetime_wm;
6674
6675         if (!crtc_state->hw.enable)
6676                 return 0;
6677
6678         linetime_wm = DIV_ROUND_CLOSEST(pipe_mode->crtc_htotal * 1000 * 8,
6679                                         cdclk_state->logical.cdclk);
6680
6681         return min(linetime_wm, 0x1ff);
6682 }
6683
6684 static u16 skl_linetime_wm(const struct intel_crtc_state *crtc_state)
6685 {
6686         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
6687         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6688         const struct drm_display_mode *pipe_mode =
6689                 &crtc_state->hw.pipe_mode;
6690         int linetime_wm;
6691
6692         if (!crtc_state->hw.enable)
6693                 return 0;
6694
6695         linetime_wm = DIV_ROUND_UP(pipe_mode->crtc_htotal * 1000 * 8,
6696                                    crtc_state->pixel_rate);
6697
6698         /* Display WA #1135: BXT:ALL GLK:ALL */
6699         if ((IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) &&
6700             dev_priv->ipc_enabled)
6701                 linetime_wm /= 2;
6702
6703         return min(linetime_wm, 0x1ff);
6704 }
6705
6706 static int hsw_compute_linetime_wm(struct intel_atomic_state *state,
6707                                    struct intel_crtc *crtc)
6708 {
6709         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6710         struct intel_crtc_state *crtc_state =
6711                 intel_atomic_get_new_crtc_state(state, crtc);
6712         const struct intel_cdclk_state *cdclk_state;
6713
6714         if (DISPLAY_VER(dev_priv) >= 9)
6715                 crtc_state->linetime = skl_linetime_wm(crtc_state);
6716         else
6717                 crtc_state->linetime = hsw_linetime_wm(crtc_state);
6718
6719         if (!hsw_crtc_supports_ips(crtc))
6720                 return 0;
6721
6722         cdclk_state = intel_atomic_get_cdclk_state(state);
6723         if (IS_ERR(cdclk_state))
6724                 return PTR_ERR(cdclk_state);
6725
6726         crtc_state->ips_linetime = hsw_ips_linetime_wm(crtc_state,
6727                                                        cdclk_state);
6728
6729         return 0;
6730 }
6731
6732 static int intel_crtc_atomic_check(struct intel_atomic_state *state,
6733                                    struct intel_crtc *crtc)
6734 {
6735         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6736         struct intel_crtc_state *crtc_state =
6737                 intel_atomic_get_new_crtc_state(state, crtc);
6738         bool mode_changed = intel_crtc_needs_modeset(crtc_state);
6739         int ret;
6740
6741         if (DISPLAY_VER(dev_priv) < 5 && !IS_G4X(dev_priv) &&
6742             mode_changed && !crtc_state->hw.active)
6743                 crtc_state->update_wm_post = true;
6744
6745         if (mode_changed && crtc_state->hw.enable &&
6746             dev_priv->display.crtc_compute_clock &&
6747             !crtc_state->bigjoiner_slave &&
6748             !drm_WARN_ON(&dev_priv->drm, crtc_state->shared_dpll)) {
6749                 ret = dev_priv->display.crtc_compute_clock(crtc_state);
6750                 if (ret)
6751                         return ret;
6752         }
6753
6754         /*
6755          * May need to update pipe gamma enable bits
6756          * when C8 planes are getting enabled/disabled.
6757          */
6758         if (c8_planes_changed(crtc_state))
6759                 crtc_state->uapi.color_mgmt_changed = true;
6760
6761         if (mode_changed || crtc_state->update_pipe ||
6762             crtc_state->uapi.color_mgmt_changed) {
6763                 ret = intel_color_check(crtc_state);
6764                 if (ret)
6765                         return ret;
6766         }
6767
6768         if (dev_priv->display.compute_pipe_wm) {
6769                 ret = dev_priv->display.compute_pipe_wm(state, crtc);
6770                 if (ret) {
6771                         drm_dbg_kms(&dev_priv->drm,
6772                                     "Target pipe watermarks are invalid\n");
6773                         return ret;
6774                 }
6775
6776         }
6777
6778         if (dev_priv->display.compute_intermediate_wm) {
6779                 if (drm_WARN_ON(&dev_priv->drm,
6780                                 !dev_priv->display.compute_pipe_wm))
6781                         return 0;
6782
6783                 /*
6784                  * Calculate 'intermediate' watermarks that satisfy both the
6785                  * old state and the new state.  We can program these
6786                  * immediately.
6787                  */
6788                 ret = dev_priv->display.compute_intermediate_wm(state, crtc);
6789                 if (ret) {
6790                         drm_dbg_kms(&dev_priv->drm,
6791                                     "No valid intermediate pipe watermarks are possible\n");
6792                         return ret;
6793                 }
6794         }
6795
6796         if (DISPLAY_VER(dev_priv) >= 9) {
6797                 if (mode_changed || crtc_state->update_pipe) {
6798                         ret = skl_update_scaler_crtc(crtc_state);
6799                         if (ret)
6800                                 return ret;
6801                 }
6802
6803                 ret = intel_atomic_setup_scalers(dev_priv, crtc, crtc_state);
6804                 if (ret)
6805                         return ret;
6806         }
6807
6808         if (HAS_IPS(dev_priv)) {
6809                 ret = hsw_compute_ips_config(crtc_state);
6810                 if (ret)
6811                         return ret;
6812         }
6813
6814         if (DISPLAY_VER(dev_priv) >= 9 ||
6815             IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv)) {
6816                 ret = hsw_compute_linetime_wm(state, crtc);
6817                 if (ret)
6818                         return ret;
6819
6820         }
6821
6822         ret = intel_psr2_sel_fetch_update(state, crtc);
6823         if (ret)
6824                 return ret;
6825
6826         return 0;
6827 }
6828
6829 static void intel_modeset_update_connector_atomic_state(struct drm_device *dev)
6830 {
6831         struct intel_connector *connector;
6832         struct drm_connector_list_iter conn_iter;
6833
6834         drm_connector_list_iter_begin(dev, &conn_iter);
6835         for_each_intel_connector_iter(connector, &conn_iter) {
6836                 struct drm_connector_state *conn_state = connector->base.state;
6837                 struct intel_encoder *encoder =
6838                         to_intel_encoder(connector->base.encoder);
6839
6840                 if (conn_state->crtc)
6841                         drm_connector_put(&connector->base);
6842
6843                 if (encoder) {
6844                         struct intel_crtc *crtc =
6845                                 to_intel_crtc(encoder->base.crtc);
6846                         const struct intel_crtc_state *crtc_state =
6847                                 to_intel_crtc_state(crtc->base.state);
6848
6849                         conn_state->best_encoder = &encoder->base;
6850                         conn_state->crtc = &crtc->base;
6851                         conn_state->max_bpc = (crtc_state->pipe_bpp ?: 24) / 3;
6852
6853                         drm_connector_get(&connector->base);
6854                 } else {
6855                         conn_state->best_encoder = NULL;
6856                         conn_state->crtc = NULL;
6857                 }
6858         }
6859         drm_connector_list_iter_end(&conn_iter);
6860 }
6861
6862 static int
6863 compute_sink_pipe_bpp(const struct drm_connector_state *conn_state,
6864                       struct intel_crtc_state *pipe_config)
6865 {
6866         struct drm_connector *connector = conn_state->connector;
6867         struct drm_i915_private *i915 = to_i915(pipe_config->uapi.crtc->dev);
6868         const struct drm_display_info *info = &connector->display_info;
6869         int bpp;
6870
6871         switch (conn_state->max_bpc) {
6872         case 6 ... 7:
6873                 bpp = 6 * 3;
6874                 break;
6875         case 8 ... 9:
6876                 bpp = 8 * 3;
6877                 break;
6878         case 10 ... 11:
6879                 bpp = 10 * 3;
6880                 break;
6881         case 12 ... 16:
6882                 bpp = 12 * 3;
6883                 break;
6884         default:
6885                 MISSING_CASE(conn_state->max_bpc);
6886                 return -EINVAL;
6887         }
6888
6889         if (bpp < pipe_config->pipe_bpp) {
6890                 drm_dbg_kms(&i915->drm,
6891                             "[CONNECTOR:%d:%s] Limiting display bpp to %d instead of "
6892                             "EDID bpp %d, requested bpp %d, max platform bpp %d\n",
6893                             connector->base.id, connector->name,
6894                             bpp, 3 * info->bpc,
6895                             3 * conn_state->max_requested_bpc,
6896                             pipe_config->pipe_bpp);
6897
6898                 pipe_config->pipe_bpp = bpp;
6899         }
6900
6901         return 0;
6902 }
6903
6904 static int
6905 compute_baseline_pipe_bpp(struct intel_crtc *crtc,
6906                           struct intel_crtc_state *pipe_config)
6907 {
6908         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6909         struct drm_atomic_state *state = pipe_config->uapi.state;
6910         struct drm_connector *connector;
6911         struct drm_connector_state *connector_state;
6912         int bpp, i;
6913
6914         if ((IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
6915             IS_CHERRYVIEW(dev_priv)))
6916                 bpp = 10*3;
6917         else if (DISPLAY_VER(dev_priv) >= 5)
6918                 bpp = 12*3;
6919         else
6920                 bpp = 8*3;
6921
6922         pipe_config->pipe_bpp = bpp;
6923
6924         /* Clamp display bpp to connector max bpp */
6925         for_each_new_connector_in_state(state, connector, connector_state, i) {
6926                 int ret;
6927
6928                 if (connector_state->crtc != &crtc->base)
6929                         continue;
6930
6931                 ret = compute_sink_pipe_bpp(connector_state, pipe_config);
6932                 if (ret)
6933                         return ret;
6934         }
6935
6936         return 0;
6937 }
6938
6939 static void intel_dump_crtc_timings(struct drm_i915_private *i915,
6940                                     const struct drm_display_mode *mode)
6941 {
6942         drm_dbg_kms(&i915->drm, "crtc timings: %d %d %d %d %d %d %d %d %d, "
6943                     "type: 0x%x flags: 0x%x\n",
6944                     mode->crtc_clock,
6945                     mode->crtc_hdisplay, mode->crtc_hsync_start,
6946                     mode->crtc_hsync_end, mode->crtc_htotal,
6947                     mode->crtc_vdisplay, mode->crtc_vsync_start,
6948                     mode->crtc_vsync_end, mode->crtc_vtotal,
6949                     mode->type, mode->flags);
6950 }
6951
6952 static void
6953 intel_dump_m_n_config(const struct intel_crtc_state *pipe_config,
6954                       const char *id, unsigned int lane_count,
6955                       const struct intel_link_m_n *m_n)
6956 {
6957         struct drm_i915_private *i915 = to_i915(pipe_config->uapi.crtc->dev);
6958
6959         drm_dbg_kms(&i915->drm,
6960                     "%s: lanes: %i; gmch_m: %u, gmch_n: %u, link_m: %u, link_n: %u, tu: %u\n",
6961                     id, lane_count,
6962                     m_n->gmch_m, m_n->gmch_n,
6963                     m_n->link_m, m_n->link_n, m_n->tu);
6964 }
6965
6966 static void
6967 intel_dump_infoframe(struct drm_i915_private *dev_priv,
6968                      const union hdmi_infoframe *frame)
6969 {
6970         if (!drm_debug_enabled(DRM_UT_KMS))
6971                 return;
6972
6973         hdmi_infoframe_log(KERN_DEBUG, dev_priv->drm.dev, frame);
6974 }
6975
6976 static void
6977 intel_dump_dp_vsc_sdp(struct drm_i915_private *dev_priv,
6978                       const struct drm_dp_vsc_sdp *vsc)
6979 {
6980         if (!drm_debug_enabled(DRM_UT_KMS))
6981                 return;
6982
6983         drm_dp_vsc_sdp_log(KERN_DEBUG, dev_priv->drm.dev, vsc);
6984 }
6985
6986 #define OUTPUT_TYPE(x) [INTEL_OUTPUT_ ## x] = #x
6987
6988 static const char * const output_type_str[] = {
6989         OUTPUT_TYPE(UNUSED),
6990         OUTPUT_TYPE(ANALOG),
6991         OUTPUT_TYPE(DVO),
6992         OUTPUT_TYPE(SDVO),
6993         OUTPUT_TYPE(LVDS),
6994         OUTPUT_TYPE(TVOUT),
6995         OUTPUT_TYPE(HDMI),
6996         OUTPUT_TYPE(DP),
6997         OUTPUT_TYPE(EDP),
6998         OUTPUT_TYPE(DSI),
6999         OUTPUT_TYPE(DDI),
7000         OUTPUT_TYPE(DP_MST),
7001 };
7002
7003 #undef OUTPUT_TYPE
7004
7005 static void snprintf_output_types(char *buf, size_t len,
7006                                   unsigned int output_types)
7007 {
7008         char *str = buf;
7009         int i;
7010
7011         str[0] = '\0';
7012
7013         for (i = 0; i < ARRAY_SIZE(output_type_str); i++) {
7014                 int r;
7015
7016                 if ((output_types & BIT(i)) == 0)
7017                         continue;
7018
7019                 r = snprintf(str, len, "%s%s",
7020                              str != buf ? "," : "", output_type_str[i]);
7021                 if (r >= len)
7022                         break;
7023                 str += r;
7024                 len -= r;
7025
7026                 output_types &= ~BIT(i);
7027         }
7028
7029         WARN_ON_ONCE(output_types != 0);
7030 }
7031
7032 static const char * const output_format_str[] = {
7033         [INTEL_OUTPUT_FORMAT_RGB] = "RGB",
7034         [INTEL_OUTPUT_FORMAT_YCBCR420] = "YCBCR4:2:0",
7035         [INTEL_OUTPUT_FORMAT_YCBCR444] = "YCBCR4:4:4",
7036 };
7037
7038 static const char *output_formats(enum intel_output_format format)
7039 {
7040         if (format >= ARRAY_SIZE(output_format_str))
7041                 return "invalid";
7042         return output_format_str[format];
7043 }
7044
7045 static void intel_dump_plane_state(const struct intel_plane_state *plane_state)
7046 {
7047         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
7048         struct drm_i915_private *i915 = to_i915(plane->base.dev);
7049         const struct drm_framebuffer *fb = plane_state->hw.fb;
7050
7051         if (!fb) {
7052                 drm_dbg_kms(&i915->drm,
7053                             "[PLANE:%d:%s] fb: [NOFB], visible: %s\n",
7054                             plane->base.base.id, plane->base.name,
7055                             yesno(plane_state->uapi.visible));
7056                 return;
7057         }
7058
7059         drm_dbg_kms(&i915->drm,
7060                     "[PLANE:%d:%s] fb: [FB:%d] %ux%u format = %p4cc modifier = 0x%llx, visible: %s\n",
7061                     plane->base.base.id, plane->base.name,
7062                     fb->base.id, fb->width, fb->height, &fb->format->format,
7063                     fb->modifier, yesno(plane_state->uapi.visible));
7064         drm_dbg_kms(&i915->drm, "\trotation: 0x%x, scaler: %d\n",
7065                     plane_state->hw.rotation, plane_state->scaler_id);
7066         if (plane_state->uapi.visible)
7067                 drm_dbg_kms(&i915->drm,
7068                             "\tsrc: " DRM_RECT_FP_FMT " dst: " DRM_RECT_FMT "\n",
7069                             DRM_RECT_FP_ARG(&plane_state->uapi.src),
7070                             DRM_RECT_ARG(&plane_state->uapi.dst));
7071 }
7072
7073 static void intel_dump_pipe_config(const struct intel_crtc_state *pipe_config,
7074                                    struct intel_atomic_state *state,
7075                                    const char *context)
7076 {
7077         struct intel_crtc *crtc = to_intel_crtc(pipe_config->uapi.crtc);
7078         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7079         const struct intel_plane_state *plane_state;
7080         struct intel_plane *plane;
7081         char buf[64];
7082         int i;
7083
7084         drm_dbg_kms(&dev_priv->drm, "[CRTC:%d:%s] enable: %s %s\n",
7085                     crtc->base.base.id, crtc->base.name,
7086                     yesno(pipe_config->hw.enable), context);
7087
7088         if (!pipe_config->hw.enable)
7089                 goto dump_planes;
7090
7091         snprintf_output_types(buf, sizeof(buf), pipe_config->output_types);
7092         drm_dbg_kms(&dev_priv->drm,
7093                     "active: %s, output_types: %s (0x%x), output format: %s\n",
7094                     yesno(pipe_config->hw.active),
7095                     buf, pipe_config->output_types,
7096                     output_formats(pipe_config->output_format));
7097
7098         drm_dbg_kms(&dev_priv->drm,
7099                     "cpu_transcoder: %s, pipe bpp: %i, dithering: %i\n",
7100                     transcoder_name(pipe_config->cpu_transcoder),
7101                     pipe_config->pipe_bpp, pipe_config->dither);
7102
7103         drm_dbg_kms(&dev_priv->drm, "MST master transcoder: %s\n",
7104                     transcoder_name(pipe_config->mst_master_transcoder));
7105
7106         drm_dbg_kms(&dev_priv->drm,
7107                     "port sync: master transcoder: %s, slave transcoder bitmask = 0x%x\n",
7108                     transcoder_name(pipe_config->master_transcoder),
7109                     pipe_config->sync_mode_slaves_mask);
7110
7111         drm_dbg_kms(&dev_priv->drm, "bigjoiner: %s\n",
7112                     pipe_config->bigjoiner_slave ? "slave" :
7113                     pipe_config->bigjoiner ? "master" : "no");
7114
7115         drm_dbg_kms(&dev_priv->drm, "splitter: %s, link count %d, overlap %d\n",
7116                     enableddisabled(pipe_config->splitter.enable),
7117                     pipe_config->splitter.link_count,
7118                     pipe_config->splitter.pixel_overlap);
7119
7120         if (pipe_config->has_pch_encoder)
7121                 intel_dump_m_n_config(pipe_config, "fdi",
7122                                       pipe_config->fdi_lanes,
7123                                       &pipe_config->fdi_m_n);
7124
7125         if (intel_crtc_has_dp_encoder(pipe_config)) {
7126                 intel_dump_m_n_config(pipe_config, "dp m_n",
7127                                 pipe_config->lane_count, &pipe_config->dp_m_n);
7128                 if (pipe_config->has_drrs)
7129                         intel_dump_m_n_config(pipe_config, "dp m2_n2",
7130                                               pipe_config->lane_count,
7131                                               &pipe_config->dp_m2_n2);
7132         }
7133
7134         drm_dbg_kms(&dev_priv->drm,
7135                     "audio: %i, infoframes: %i, infoframes enabled: 0x%x\n",
7136                     pipe_config->has_audio, pipe_config->has_infoframe,
7137                     pipe_config->infoframes.enable);
7138
7139         if (pipe_config->infoframes.enable &
7140             intel_hdmi_infoframe_enable(HDMI_PACKET_TYPE_GENERAL_CONTROL))
7141                 drm_dbg_kms(&dev_priv->drm, "GCP: 0x%x\n",
7142                             pipe_config->infoframes.gcp);
7143         if (pipe_config->infoframes.enable &
7144             intel_hdmi_infoframe_enable(HDMI_INFOFRAME_TYPE_AVI))
7145                 intel_dump_infoframe(dev_priv, &pipe_config->infoframes.avi);
7146         if (pipe_config->infoframes.enable &
7147             intel_hdmi_infoframe_enable(HDMI_INFOFRAME_TYPE_SPD))
7148                 intel_dump_infoframe(dev_priv, &pipe_config->infoframes.spd);
7149         if (pipe_config->infoframes.enable &
7150             intel_hdmi_infoframe_enable(HDMI_INFOFRAME_TYPE_VENDOR))
7151                 intel_dump_infoframe(dev_priv, &pipe_config->infoframes.hdmi);
7152         if (pipe_config->infoframes.enable &
7153             intel_hdmi_infoframe_enable(HDMI_INFOFRAME_TYPE_DRM))
7154                 intel_dump_infoframe(dev_priv, &pipe_config->infoframes.drm);
7155         if (pipe_config->infoframes.enable &
7156             intel_hdmi_infoframe_enable(HDMI_PACKET_TYPE_GAMUT_METADATA))
7157                 intel_dump_infoframe(dev_priv, &pipe_config->infoframes.drm);
7158         if (pipe_config->infoframes.enable &
7159             intel_hdmi_infoframe_enable(DP_SDP_VSC))
7160                 intel_dump_dp_vsc_sdp(dev_priv, &pipe_config->infoframes.vsc);
7161
7162         drm_dbg_kms(&dev_priv->drm, "vrr: %s, vmin: %d, vmax: %d, pipeline full: %d, guardband: %d flipline: %d, vmin vblank: %d, vmax vblank: %d\n",
7163                     yesno(pipe_config->vrr.enable),
7164                     pipe_config->vrr.vmin, pipe_config->vrr.vmax,
7165                     pipe_config->vrr.pipeline_full, pipe_config->vrr.guardband,
7166                     pipe_config->vrr.flipline,
7167                     intel_vrr_vmin_vblank_start(pipe_config),
7168                     intel_vrr_vmax_vblank_start(pipe_config));
7169
7170         drm_dbg_kms(&dev_priv->drm, "requested mode:\n");
7171         drm_mode_debug_printmodeline(&pipe_config->hw.mode);
7172         drm_dbg_kms(&dev_priv->drm, "adjusted mode:\n");
7173         drm_mode_debug_printmodeline(&pipe_config->hw.adjusted_mode);
7174         intel_dump_crtc_timings(dev_priv, &pipe_config->hw.adjusted_mode);
7175         drm_dbg_kms(&dev_priv->drm, "pipe mode:\n");
7176         drm_mode_debug_printmodeline(&pipe_config->hw.pipe_mode);
7177         intel_dump_crtc_timings(dev_priv, &pipe_config->hw.pipe_mode);
7178         drm_dbg_kms(&dev_priv->drm,
7179                     "port clock: %d, pipe src size: %dx%d, pixel rate %d\n",
7180                     pipe_config->port_clock,
7181                     pipe_config->pipe_src_w, pipe_config->pipe_src_h,
7182                     pipe_config->pixel_rate);
7183
7184         drm_dbg_kms(&dev_priv->drm, "linetime: %d, ips linetime: %d\n",
7185                     pipe_config->linetime, pipe_config->ips_linetime);
7186
7187         if (DISPLAY_VER(dev_priv) >= 9)
7188                 drm_dbg_kms(&dev_priv->drm,
7189                             "num_scalers: %d, scaler_users: 0x%x, scaler_id: %d\n",
7190                             crtc->num_scalers,
7191                             pipe_config->scaler_state.scaler_users,
7192                             pipe_config->scaler_state.scaler_id);
7193
7194         if (HAS_GMCH(dev_priv))
7195                 drm_dbg_kms(&dev_priv->drm,
7196                             "gmch pfit: control: 0x%08x, ratios: 0x%08x, lvds border: 0x%08x\n",
7197                             pipe_config->gmch_pfit.control,
7198                             pipe_config->gmch_pfit.pgm_ratios,
7199                             pipe_config->gmch_pfit.lvds_border_bits);
7200         else
7201                 drm_dbg_kms(&dev_priv->drm,
7202                             "pch pfit: " DRM_RECT_FMT ", %s, force thru: %s\n",
7203                             DRM_RECT_ARG(&pipe_config->pch_pfit.dst),
7204                             enableddisabled(pipe_config->pch_pfit.enabled),
7205                             yesno(pipe_config->pch_pfit.force_thru));
7206
7207         drm_dbg_kms(&dev_priv->drm, "ips: %i, double wide: %i\n",
7208                     pipe_config->ips_enabled, pipe_config->double_wide);
7209
7210         intel_dpll_dump_hw_state(dev_priv, &pipe_config->dpll_hw_state);
7211
7212         if (IS_CHERRYVIEW(dev_priv))
7213                 drm_dbg_kms(&dev_priv->drm,
7214                             "cgm_mode: 0x%x gamma_mode: 0x%x gamma_enable: %d csc_enable: %d\n",
7215                             pipe_config->cgm_mode, pipe_config->gamma_mode,
7216                             pipe_config->gamma_enable, pipe_config->csc_enable);
7217         else
7218                 drm_dbg_kms(&dev_priv->drm,
7219                             "csc_mode: 0x%x gamma_mode: 0x%x gamma_enable: %d csc_enable: %d\n",
7220                             pipe_config->csc_mode, pipe_config->gamma_mode,
7221                             pipe_config->gamma_enable, pipe_config->csc_enable);
7222
7223         drm_dbg_kms(&dev_priv->drm, "degamma lut: %d entries, gamma lut: %d entries\n",
7224                     pipe_config->hw.degamma_lut ?
7225                     drm_color_lut_size(pipe_config->hw.degamma_lut) : 0,
7226                     pipe_config->hw.gamma_lut ?
7227                     drm_color_lut_size(pipe_config->hw.gamma_lut) : 0);
7228
7229 dump_planes:
7230         if (!state)
7231                 return;
7232
7233         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
7234                 if (plane->pipe == crtc->pipe)
7235                         intel_dump_plane_state(plane_state);
7236         }
7237 }
7238
7239 static bool check_digital_port_conflicts(struct intel_atomic_state *state)
7240 {
7241         struct drm_device *dev = state->base.dev;
7242         struct drm_connector *connector;
7243         struct drm_connector_list_iter conn_iter;
7244         unsigned int used_ports = 0;
7245         unsigned int used_mst_ports = 0;
7246         bool ret = true;
7247
7248         /*
7249          * We're going to peek into connector->state,
7250          * hence connection_mutex must be held.
7251          */
7252         drm_modeset_lock_assert_held(&dev->mode_config.connection_mutex);
7253
7254         /*
7255          * Walk the connector list instead of the encoder
7256          * list to detect the problem on ddi platforms
7257          * where there's just one encoder per digital port.
7258          */
7259         drm_connector_list_iter_begin(dev, &conn_iter);
7260         drm_for_each_connector_iter(connector, &conn_iter) {
7261                 struct drm_connector_state *connector_state;
7262                 struct intel_encoder *encoder;
7263
7264                 connector_state =
7265                         drm_atomic_get_new_connector_state(&state->base,
7266                                                            connector);
7267                 if (!connector_state)
7268                         connector_state = connector->state;
7269
7270                 if (!connector_state->best_encoder)
7271                         continue;
7272
7273                 encoder = to_intel_encoder(connector_state->best_encoder);
7274
7275                 drm_WARN_ON(dev, !connector_state->crtc);
7276
7277                 switch (encoder->type) {
7278                 case INTEL_OUTPUT_DDI:
7279                         if (drm_WARN_ON(dev, !HAS_DDI(to_i915(dev))))
7280                                 break;
7281                         fallthrough;
7282                 case INTEL_OUTPUT_DP:
7283                 case INTEL_OUTPUT_HDMI:
7284                 case INTEL_OUTPUT_EDP:
7285                         /* the same port mustn't appear more than once */
7286                         if (used_ports & BIT(encoder->port))
7287                                 ret = false;
7288
7289                         used_ports |= BIT(encoder->port);
7290                         break;
7291                 case INTEL_OUTPUT_DP_MST:
7292                         used_mst_ports |=
7293                                 1 << encoder->port;
7294                         break;
7295                 default:
7296                         break;
7297                 }
7298         }
7299         drm_connector_list_iter_end(&conn_iter);
7300
7301         /* can't mix MST and SST/HDMI on the same port */
7302         if (used_ports & used_mst_ports)
7303                 return false;
7304
7305         return ret;
7306 }
7307
7308 static void
7309 intel_crtc_copy_uapi_to_hw_state_nomodeset(struct intel_atomic_state *state,
7310                                            struct intel_crtc_state *crtc_state)
7311 {
7312         const struct intel_crtc_state *from_crtc_state = crtc_state;
7313
7314         if (crtc_state->bigjoiner_slave) {
7315                 from_crtc_state = intel_atomic_get_new_crtc_state(state,
7316                                                                   crtc_state->bigjoiner_linked_crtc);
7317
7318                 /* No need to copy state if the master state is unchanged */
7319                 if (!from_crtc_state)
7320                         return;
7321         }
7322
7323         intel_crtc_copy_color_blobs(crtc_state, from_crtc_state);
7324 }
7325
7326 static void
7327 intel_crtc_copy_uapi_to_hw_state(struct intel_atomic_state *state,
7328                                  struct intel_crtc_state *crtc_state)
7329 {
7330         crtc_state->hw.enable = crtc_state->uapi.enable;
7331         crtc_state->hw.active = crtc_state->uapi.active;
7332         crtc_state->hw.mode = crtc_state->uapi.mode;
7333         crtc_state->hw.adjusted_mode = crtc_state->uapi.adjusted_mode;
7334         crtc_state->hw.scaling_filter = crtc_state->uapi.scaling_filter;
7335
7336         intel_crtc_copy_uapi_to_hw_state_nomodeset(state, crtc_state);
7337 }
7338
7339 static void intel_crtc_copy_hw_to_uapi_state(struct intel_crtc_state *crtc_state)
7340 {
7341         if (crtc_state->bigjoiner_slave)
7342                 return;
7343
7344         crtc_state->uapi.enable = crtc_state->hw.enable;
7345         crtc_state->uapi.active = crtc_state->hw.active;
7346         drm_WARN_ON(crtc_state->uapi.crtc->dev,
7347                     drm_atomic_set_mode_for_crtc(&crtc_state->uapi, &crtc_state->hw.mode) < 0);
7348
7349         crtc_state->uapi.adjusted_mode = crtc_state->hw.adjusted_mode;
7350         crtc_state->uapi.scaling_filter = crtc_state->hw.scaling_filter;
7351
7352         /* copy color blobs to uapi */
7353         drm_property_replace_blob(&crtc_state->uapi.degamma_lut,
7354                                   crtc_state->hw.degamma_lut);
7355         drm_property_replace_blob(&crtc_state->uapi.gamma_lut,
7356                                   crtc_state->hw.gamma_lut);
7357         drm_property_replace_blob(&crtc_state->uapi.ctm,
7358                                   crtc_state->hw.ctm);
7359 }
7360
7361 static int
7362 copy_bigjoiner_crtc_state(struct intel_crtc_state *crtc_state,
7363                           const struct intel_crtc_state *from_crtc_state)
7364 {
7365         struct intel_crtc_state *saved_state;
7366         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
7367
7368         saved_state = kmemdup(from_crtc_state, sizeof(*saved_state), GFP_KERNEL);
7369         if (!saved_state)
7370                 return -ENOMEM;
7371
7372         saved_state->uapi = crtc_state->uapi;
7373         saved_state->scaler_state = crtc_state->scaler_state;
7374         saved_state->shared_dpll = crtc_state->shared_dpll;
7375         saved_state->dpll_hw_state = crtc_state->dpll_hw_state;
7376         saved_state->crc_enabled = crtc_state->crc_enabled;
7377
7378         intel_crtc_free_hw_state(crtc_state);
7379         memcpy(crtc_state, saved_state, sizeof(*crtc_state));
7380         kfree(saved_state);
7381
7382         /* Re-init hw state */
7383         memset(&crtc_state->hw, 0, sizeof(saved_state->hw));
7384         crtc_state->hw.enable = from_crtc_state->hw.enable;
7385         crtc_state->hw.active = from_crtc_state->hw.active;
7386         crtc_state->hw.pipe_mode = from_crtc_state->hw.pipe_mode;
7387         crtc_state->hw.adjusted_mode = from_crtc_state->hw.adjusted_mode;
7388
7389         /* Some fixups */
7390         crtc_state->uapi.mode_changed = from_crtc_state->uapi.mode_changed;
7391         crtc_state->uapi.connectors_changed = from_crtc_state->uapi.connectors_changed;
7392         crtc_state->uapi.active_changed = from_crtc_state->uapi.active_changed;
7393         crtc_state->nv12_planes = crtc_state->c8_planes = crtc_state->update_planes = 0;
7394         crtc_state->bigjoiner_linked_crtc = to_intel_crtc(from_crtc_state->uapi.crtc);
7395         crtc_state->bigjoiner_slave = true;
7396         crtc_state->cpu_transcoder = (enum transcoder)crtc->pipe;
7397         crtc_state->has_audio = false;
7398
7399         return 0;
7400 }
7401
7402 static int
7403 intel_crtc_prepare_cleared_state(struct intel_atomic_state *state,
7404                                  struct intel_crtc_state *crtc_state)
7405 {
7406         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
7407         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7408         struct intel_crtc_state *saved_state;
7409
7410         saved_state = intel_crtc_state_alloc(crtc);
7411         if (!saved_state)
7412                 return -ENOMEM;
7413
7414         /* free the old crtc_state->hw members */
7415         intel_crtc_free_hw_state(crtc_state);
7416
7417         /* FIXME: before the switch to atomic started, a new pipe_config was
7418          * kzalloc'd. Code that depends on any field being zero should be
7419          * fixed, so that the crtc_state can be safely duplicated. For now,
7420          * only fields that are know to not cause problems are preserved. */
7421
7422         saved_state->uapi = crtc_state->uapi;
7423         saved_state->scaler_state = crtc_state->scaler_state;
7424         saved_state->shared_dpll = crtc_state->shared_dpll;
7425         saved_state->dpll_hw_state = crtc_state->dpll_hw_state;
7426         memcpy(saved_state->icl_port_dplls, crtc_state->icl_port_dplls,
7427                sizeof(saved_state->icl_port_dplls));
7428         saved_state->crc_enabled = crtc_state->crc_enabled;
7429         if (IS_G4X(dev_priv) ||
7430             IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
7431                 saved_state->wm = crtc_state->wm;
7432
7433         memcpy(crtc_state, saved_state, sizeof(*crtc_state));
7434         kfree(saved_state);
7435
7436         intel_crtc_copy_uapi_to_hw_state(state, crtc_state);
7437
7438         return 0;
7439 }
7440
7441 static int
7442 intel_modeset_pipe_config(struct intel_atomic_state *state,
7443                           struct intel_crtc_state *pipe_config)
7444 {
7445         struct drm_crtc *crtc = pipe_config->uapi.crtc;
7446         struct drm_i915_private *i915 = to_i915(pipe_config->uapi.crtc->dev);
7447         struct drm_connector *connector;
7448         struct drm_connector_state *connector_state;
7449         int base_bpp, ret, i;
7450         bool retry = true;
7451
7452         pipe_config->cpu_transcoder =
7453                 (enum transcoder) to_intel_crtc(crtc)->pipe;
7454
7455         /*
7456          * Sanitize sync polarity flags based on requested ones. If neither
7457          * positive or negative polarity is requested, treat this as meaning
7458          * negative polarity.
7459          */
7460         if (!(pipe_config->hw.adjusted_mode.flags &
7461               (DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_NHSYNC)))
7462                 pipe_config->hw.adjusted_mode.flags |= DRM_MODE_FLAG_NHSYNC;
7463
7464         if (!(pipe_config->hw.adjusted_mode.flags &
7465               (DRM_MODE_FLAG_PVSYNC | DRM_MODE_FLAG_NVSYNC)))
7466                 pipe_config->hw.adjusted_mode.flags |= DRM_MODE_FLAG_NVSYNC;
7467
7468         ret = compute_baseline_pipe_bpp(to_intel_crtc(crtc),
7469                                         pipe_config);
7470         if (ret)
7471                 return ret;
7472
7473         base_bpp = pipe_config->pipe_bpp;
7474
7475         /*
7476          * Determine the real pipe dimensions. Note that stereo modes can
7477          * increase the actual pipe size due to the frame doubling and
7478          * insertion of additional space for blanks between the frame. This
7479          * is stored in the crtc timings. We use the requested mode to do this
7480          * computation to clearly distinguish it from the adjusted mode, which
7481          * can be changed by the connectors in the below retry loop.
7482          */
7483         drm_mode_get_hv_timing(&pipe_config->hw.mode,
7484                                &pipe_config->pipe_src_w,
7485                                &pipe_config->pipe_src_h);
7486
7487         for_each_new_connector_in_state(&state->base, connector, connector_state, i) {
7488                 struct intel_encoder *encoder =
7489                         to_intel_encoder(connector_state->best_encoder);
7490
7491                 if (connector_state->crtc != crtc)
7492                         continue;
7493
7494                 if (!check_single_encoder_cloning(state, to_intel_crtc(crtc), encoder)) {
7495                         drm_dbg_kms(&i915->drm,
7496                                     "rejecting invalid cloning configuration\n");
7497                         return -EINVAL;
7498                 }
7499
7500                 /*
7501                  * Determine output_types before calling the .compute_config()
7502                  * hooks so that the hooks can use this information safely.
7503                  */
7504                 if (encoder->compute_output_type)
7505                         pipe_config->output_types |=
7506                                 BIT(encoder->compute_output_type(encoder, pipe_config,
7507                                                                  connector_state));
7508                 else
7509                         pipe_config->output_types |= BIT(encoder->type);
7510         }
7511
7512 encoder_retry:
7513         /* Ensure the port clock defaults are reset when retrying. */
7514         pipe_config->port_clock = 0;
7515         pipe_config->pixel_multiplier = 1;
7516
7517         /* Fill in default crtc timings, allow encoders to overwrite them. */
7518         drm_mode_set_crtcinfo(&pipe_config->hw.adjusted_mode,
7519                               CRTC_STEREO_DOUBLE);
7520
7521         /* Pass our mode to the connectors and the CRTC to give them a chance to
7522          * adjust it according to limitations or connector properties, and also
7523          * a chance to reject the mode entirely.
7524          */
7525         for_each_new_connector_in_state(&state->base, connector, connector_state, i) {
7526                 struct intel_encoder *encoder =
7527                         to_intel_encoder(connector_state->best_encoder);
7528
7529                 if (connector_state->crtc != crtc)
7530                         continue;
7531
7532                 ret = encoder->compute_config(encoder, pipe_config,
7533                                               connector_state);
7534                 if (ret < 0) {
7535                         if (ret != -EDEADLK)
7536                                 drm_dbg_kms(&i915->drm,
7537                                             "Encoder config failure: %d\n",
7538                                             ret);
7539                         return ret;
7540                 }
7541         }
7542
7543         /* Set default port clock if not overwritten by the encoder. Needs to be
7544          * done afterwards in case the encoder adjusts the mode. */
7545         if (!pipe_config->port_clock)
7546                 pipe_config->port_clock = pipe_config->hw.adjusted_mode.crtc_clock
7547                         * pipe_config->pixel_multiplier;
7548
7549         ret = intel_crtc_compute_config(to_intel_crtc(crtc), pipe_config);
7550         if (ret == -EDEADLK)
7551                 return ret;
7552         if (ret < 0) {
7553                 drm_dbg_kms(&i915->drm, "CRTC fixup failed\n");
7554                 return ret;
7555         }
7556
7557         if (ret == I915_DISPLAY_CONFIG_RETRY) {
7558                 if (drm_WARN(&i915->drm, !retry,
7559                              "loop in pipe configuration computation\n"))
7560                         return -EINVAL;
7561
7562                 drm_dbg_kms(&i915->drm, "CRTC bw constrained, retrying\n");
7563                 retry = false;
7564                 goto encoder_retry;
7565         }
7566
7567         /* Dithering seems to not pass-through bits correctly when it should, so
7568          * only enable it on 6bpc panels and when its not a compliance
7569          * test requesting 6bpc video pattern.
7570          */
7571         pipe_config->dither = (pipe_config->pipe_bpp == 6*3) &&
7572                 !pipe_config->dither_force_disable;
7573         drm_dbg_kms(&i915->drm,
7574                     "hw max bpp: %i, pipe bpp: %i, dithering: %i\n",
7575                     base_bpp, pipe_config->pipe_bpp, pipe_config->dither);
7576
7577         return 0;
7578 }
7579
7580 static int
7581 intel_modeset_pipe_config_late(struct intel_crtc_state *crtc_state)
7582 {
7583         struct intel_atomic_state *state =
7584                 to_intel_atomic_state(crtc_state->uapi.state);
7585         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
7586         struct drm_connector_state *conn_state;
7587         struct drm_connector *connector;
7588         int i;
7589
7590         for_each_new_connector_in_state(&state->base, connector,
7591                                         conn_state, i) {
7592                 struct intel_encoder *encoder =
7593                         to_intel_encoder(conn_state->best_encoder);
7594                 int ret;
7595
7596                 if (conn_state->crtc != &crtc->base ||
7597                     !encoder->compute_config_late)
7598                         continue;
7599
7600                 ret = encoder->compute_config_late(encoder, crtc_state,
7601                                                    conn_state);
7602                 if (ret)
7603                         return ret;
7604         }
7605
7606         return 0;
7607 }
7608
7609 bool intel_fuzzy_clock_check(int clock1, int clock2)
7610 {
7611         int diff;
7612
7613         if (clock1 == clock2)
7614                 return true;
7615
7616         if (!clock1 || !clock2)
7617                 return false;
7618
7619         diff = abs(clock1 - clock2);
7620
7621         if (((((diff + clock1 + clock2) * 100)) / (clock1 + clock2)) < 105)
7622                 return true;
7623
7624         return false;
7625 }
7626
7627 static bool
7628 intel_compare_m_n(unsigned int m, unsigned int n,
7629                   unsigned int m2, unsigned int n2,
7630                   bool exact)
7631 {
7632         if (m == m2 && n == n2)
7633                 return true;
7634
7635         if (exact || !m || !n || !m2 || !n2)
7636                 return false;
7637
7638         BUILD_BUG_ON(DATA_LINK_M_N_MASK > INT_MAX);
7639
7640         if (n > n2) {
7641                 while (n > n2) {
7642                         m2 <<= 1;
7643                         n2 <<= 1;
7644                 }
7645         } else if (n < n2) {
7646                 while (n < n2) {
7647                         m <<= 1;
7648                         n <<= 1;
7649                 }
7650         }
7651
7652         if (n != n2)
7653                 return false;
7654
7655         return intel_fuzzy_clock_check(m, m2);
7656 }
7657
7658 static bool
7659 intel_compare_link_m_n(const struct intel_link_m_n *m_n,
7660                        const struct intel_link_m_n *m2_n2,
7661                        bool exact)
7662 {
7663         return m_n->tu == m2_n2->tu &&
7664                 intel_compare_m_n(m_n->gmch_m, m_n->gmch_n,
7665                                   m2_n2->gmch_m, m2_n2->gmch_n, exact) &&
7666                 intel_compare_m_n(m_n->link_m, m_n->link_n,
7667                                   m2_n2->link_m, m2_n2->link_n, exact);
7668 }
7669
7670 static bool
7671 intel_compare_infoframe(const union hdmi_infoframe *a,
7672                         const union hdmi_infoframe *b)
7673 {
7674         return memcmp(a, b, sizeof(*a)) == 0;
7675 }
7676
7677 static bool
7678 intel_compare_dp_vsc_sdp(const struct drm_dp_vsc_sdp *a,
7679                          const struct drm_dp_vsc_sdp *b)
7680 {
7681         return memcmp(a, b, sizeof(*a)) == 0;
7682 }
7683
7684 static void
7685 pipe_config_infoframe_mismatch(struct drm_i915_private *dev_priv,
7686                                bool fastset, const char *name,
7687                                const union hdmi_infoframe *a,
7688                                const union hdmi_infoframe *b)
7689 {
7690         if (fastset) {
7691                 if (!drm_debug_enabled(DRM_UT_KMS))
7692                         return;
7693
7694                 drm_dbg_kms(&dev_priv->drm,
7695                             "fastset mismatch in %s infoframe\n", name);
7696                 drm_dbg_kms(&dev_priv->drm, "expected:\n");
7697                 hdmi_infoframe_log(KERN_DEBUG, dev_priv->drm.dev, a);
7698                 drm_dbg_kms(&dev_priv->drm, "found:\n");
7699                 hdmi_infoframe_log(KERN_DEBUG, dev_priv->drm.dev, b);
7700         } else {
7701                 drm_err(&dev_priv->drm, "mismatch in %s infoframe\n", name);
7702                 drm_err(&dev_priv->drm, "expected:\n");
7703                 hdmi_infoframe_log(KERN_ERR, dev_priv->drm.dev, a);
7704                 drm_err(&dev_priv->drm, "found:\n");
7705                 hdmi_infoframe_log(KERN_ERR, dev_priv->drm.dev, b);
7706         }
7707 }
7708
7709 static void
7710 pipe_config_dp_vsc_sdp_mismatch(struct drm_i915_private *dev_priv,
7711                                 bool fastset, const char *name,
7712                                 const struct drm_dp_vsc_sdp *a,
7713                                 const struct drm_dp_vsc_sdp *b)
7714 {
7715         if (fastset) {
7716                 if (!drm_debug_enabled(DRM_UT_KMS))
7717                         return;
7718
7719                 drm_dbg_kms(&dev_priv->drm,
7720                             "fastset mismatch in %s dp sdp\n", name);
7721                 drm_dbg_kms(&dev_priv->drm, "expected:\n");
7722                 drm_dp_vsc_sdp_log(KERN_DEBUG, dev_priv->drm.dev, a);
7723                 drm_dbg_kms(&dev_priv->drm, "found:\n");
7724                 drm_dp_vsc_sdp_log(KERN_DEBUG, dev_priv->drm.dev, b);
7725         } else {
7726                 drm_err(&dev_priv->drm, "mismatch in %s dp sdp\n", name);
7727                 drm_err(&dev_priv->drm, "expected:\n");
7728                 drm_dp_vsc_sdp_log(KERN_ERR, dev_priv->drm.dev, a);
7729                 drm_err(&dev_priv->drm, "found:\n");
7730                 drm_dp_vsc_sdp_log(KERN_ERR, dev_priv->drm.dev, b);
7731         }
7732 }
7733
7734 static void __printf(4, 5)
7735 pipe_config_mismatch(bool fastset, const struct intel_crtc *crtc,
7736                      const char *name, const char *format, ...)
7737 {
7738         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
7739         struct va_format vaf;
7740         va_list args;
7741
7742         va_start(args, format);
7743         vaf.fmt = format;
7744         vaf.va = &args;
7745
7746         if (fastset)
7747                 drm_dbg_kms(&i915->drm,
7748                             "[CRTC:%d:%s] fastset mismatch in %s %pV\n",
7749                             crtc->base.base.id, crtc->base.name, name, &vaf);
7750         else
7751                 drm_err(&i915->drm, "[CRTC:%d:%s] mismatch in %s %pV\n",
7752                         crtc->base.base.id, crtc->base.name, name, &vaf);
7753
7754         va_end(args);
7755 }
7756
7757 static bool fastboot_enabled(struct drm_i915_private *dev_priv)
7758 {
7759         if (dev_priv->params.fastboot != -1)
7760                 return dev_priv->params.fastboot;
7761
7762         /* Enable fastboot by default on Skylake and newer */
7763         if (DISPLAY_VER(dev_priv) >= 9)
7764                 return true;
7765
7766         /* Enable fastboot by default on VLV and CHV */
7767         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
7768                 return true;
7769
7770         /* Disabled by default on all others */
7771         return false;
7772 }
7773
7774 static bool
7775 intel_pipe_config_compare(const struct intel_crtc_state *current_config,
7776                           const struct intel_crtc_state *pipe_config,
7777                           bool fastset)
7778 {
7779         struct drm_i915_private *dev_priv = to_i915(current_config->uapi.crtc->dev);
7780         struct intel_crtc *crtc = to_intel_crtc(pipe_config->uapi.crtc);
7781         bool ret = true;
7782         u32 bp_gamma = 0;
7783         bool fixup_inherited = fastset &&
7784                 current_config->inherited && !pipe_config->inherited;
7785
7786         if (fixup_inherited && !fastboot_enabled(dev_priv)) {
7787                 drm_dbg_kms(&dev_priv->drm,
7788                             "initial modeset and fastboot not set\n");
7789                 ret = false;
7790         }
7791
7792 #define PIPE_CONF_CHECK_X(name) do { \
7793         if (current_config->name != pipe_config->name) { \
7794                 pipe_config_mismatch(fastset, crtc, __stringify(name), \
7795                                      "(expected 0x%08x, found 0x%08x)", \
7796                                      current_config->name, \
7797                                      pipe_config->name); \
7798                 ret = false; \
7799         } \
7800 } while (0)
7801
7802 #define PIPE_CONF_CHECK_X_WITH_MASK(name, mask) do { \
7803         if ((current_config->name & (mask)) != (pipe_config->name & (mask))) { \
7804                 pipe_config_mismatch(fastset, crtc, __stringify(name), \
7805                                      "(expected 0x%08x, found 0x%08x)", \
7806                                      current_config->name & (mask), \
7807                                      pipe_config->name & (mask)); \
7808                 ret = false; \
7809         } \
7810 } while (0)
7811
7812 #define PIPE_CONF_CHECK_I(name) do { \
7813         if (current_config->name != pipe_config->name) { \
7814                 pipe_config_mismatch(fastset, crtc, __stringify(name), \
7815                                      "(expected %i, found %i)", \
7816                                      current_config->name, \
7817                                      pipe_config->name); \
7818                 ret = false; \
7819         } \
7820 } while (0)
7821
7822 #define PIPE_CONF_CHECK_BOOL(name) do { \
7823         if (current_config->name != pipe_config->name) { \
7824                 pipe_config_mismatch(fastset, crtc,  __stringify(name), \
7825                                      "(expected %s, found %s)", \
7826                                      yesno(current_config->name), \
7827                                      yesno(pipe_config->name)); \
7828                 ret = false; \
7829         } \
7830 } while (0)
7831
7832 /*
7833  * Checks state where we only read out the enabling, but not the entire
7834  * state itself (like full infoframes or ELD for audio). These states
7835  * require a full modeset on bootup to fix up.
7836  */
7837 #define PIPE_CONF_CHECK_BOOL_INCOMPLETE(name) do { \
7838         if (!fixup_inherited || (!current_config->name && !pipe_config->name)) { \
7839                 PIPE_CONF_CHECK_BOOL(name); \
7840         } else { \
7841                 pipe_config_mismatch(fastset, crtc, __stringify(name), \
7842                                      "unable to verify whether state matches exactly, forcing modeset (expected %s, found %s)", \
7843                                      yesno(current_config->name), \
7844                                      yesno(pipe_config->name)); \
7845                 ret = false; \
7846         } \
7847 } while (0)
7848
7849 #define PIPE_CONF_CHECK_P(name) do { \
7850         if (current_config->name != pipe_config->name) { \
7851                 pipe_config_mismatch(fastset, crtc, __stringify(name), \
7852                                      "(expected %p, found %p)", \
7853                                      current_config->name, \
7854                                      pipe_config->name); \
7855                 ret = false; \
7856         } \
7857 } while (0)
7858
7859 #define PIPE_CONF_CHECK_M_N(name) do { \
7860         if (!intel_compare_link_m_n(&current_config->name, \
7861                                     &pipe_config->name,\
7862                                     !fastset)) { \
7863                 pipe_config_mismatch(fastset, crtc, __stringify(name), \
7864                                      "(expected tu %i gmch %i/%i link %i/%i, " \
7865                                      "found tu %i, gmch %i/%i link %i/%i)", \
7866                                      current_config->name.tu, \
7867                                      current_config->name.gmch_m, \
7868                                      current_config->name.gmch_n, \
7869                                      current_config->name.link_m, \
7870                                      current_config->name.link_n, \
7871                                      pipe_config->name.tu, \
7872                                      pipe_config->name.gmch_m, \
7873                                      pipe_config->name.gmch_n, \
7874                                      pipe_config->name.link_m, \
7875                                      pipe_config->name.link_n); \
7876                 ret = false; \
7877         } \
7878 } while (0)
7879
7880 /* This is required for BDW+ where there is only one set of registers for
7881  * switching between high and low RR.
7882  * This macro can be used whenever a comparison has to be made between one
7883  * hw state and multiple sw state variables.
7884  */
7885 #define PIPE_CONF_CHECK_M_N_ALT(name, alt_name) do { \
7886         if (!intel_compare_link_m_n(&current_config->name, \
7887                                     &pipe_config->name, !fastset) && \
7888             !intel_compare_link_m_n(&current_config->alt_name, \
7889                                     &pipe_config->name, !fastset)) { \
7890                 pipe_config_mismatch(fastset, crtc, __stringify(name), \
7891                                      "(expected tu %i gmch %i/%i link %i/%i, " \
7892                                      "or tu %i gmch %i/%i link %i/%i, " \
7893                                      "found tu %i, gmch %i/%i link %i/%i)", \
7894                                      current_config->name.tu, \
7895                                      current_config->name.gmch_m, \
7896                                      current_config->name.gmch_n, \
7897                                      current_config->name.link_m, \
7898                                      current_config->name.link_n, \
7899                                      current_config->alt_name.tu, \
7900                                      current_config->alt_name.gmch_m, \
7901                                      current_config->alt_name.gmch_n, \
7902                                      current_config->alt_name.link_m, \
7903                                      current_config->alt_name.link_n, \
7904                                      pipe_config->name.tu, \
7905                                      pipe_config->name.gmch_m, \
7906                                      pipe_config->name.gmch_n, \
7907                                      pipe_config->name.link_m, \
7908                                      pipe_config->name.link_n); \
7909                 ret = false; \
7910         } \
7911 } while (0)
7912
7913 #define PIPE_CONF_CHECK_FLAGS(name, mask) do { \
7914         if ((current_config->name ^ pipe_config->name) & (mask)) { \
7915                 pipe_config_mismatch(fastset, crtc, __stringify(name), \
7916                                      "(%x) (expected %i, found %i)", \
7917                                      (mask), \
7918                                      current_config->name & (mask), \
7919                                      pipe_config->name & (mask)); \
7920                 ret = false; \
7921         } \
7922 } while (0)
7923
7924 #define PIPE_CONF_CHECK_CLOCK_FUZZY(name) do { \
7925         if (!intel_fuzzy_clock_check(current_config->name, pipe_config->name)) { \
7926                 pipe_config_mismatch(fastset, crtc, __stringify(name), \
7927                                      "(expected %i, found %i)", \
7928                                      current_config->name, \
7929                                      pipe_config->name); \
7930                 ret = false; \
7931         } \
7932 } while (0)
7933
7934 #define PIPE_CONF_CHECK_INFOFRAME(name) do { \
7935         if (!intel_compare_infoframe(&current_config->infoframes.name, \
7936                                      &pipe_config->infoframes.name)) { \
7937                 pipe_config_infoframe_mismatch(dev_priv, fastset, __stringify(name), \
7938                                                &current_config->infoframes.name, \
7939                                                &pipe_config->infoframes.name); \
7940                 ret = false; \
7941         } \
7942 } while (0)
7943
7944 #define PIPE_CONF_CHECK_DP_VSC_SDP(name) do { \
7945         if (!current_config->has_psr && !pipe_config->has_psr && \
7946             !intel_compare_dp_vsc_sdp(&current_config->infoframes.name, \
7947                                       &pipe_config->infoframes.name)) { \
7948                 pipe_config_dp_vsc_sdp_mismatch(dev_priv, fastset, __stringify(name), \
7949                                                 &current_config->infoframes.name, \
7950                                                 &pipe_config->infoframes.name); \
7951                 ret = false; \
7952         } \
7953 } while (0)
7954
7955 #define PIPE_CONF_CHECK_COLOR_LUT(name1, name2, bit_precision) do { \
7956         if (current_config->name1 != pipe_config->name1) { \
7957                 pipe_config_mismatch(fastset, crtc, __stringify(name1), \
7958                                 "(expected %i, found %i, won't compare lut values)", \
7959                                 current_config->name1, \
7960                                 pipe_config->name1); \
7961                 ret = false;\
7962         } else { \
7963                 if (!intel_color_lut_equal(current_config->name2, \
7964                                         pipe_config->name2, pipe_config->name1, \
7965                                         bit_precision)) { \
7966                         pipe_config_mismatch(fastset, crtc, __stringify(name2), \
7967                                         "hw_state doesn't match sw_state"); \
7968                         ret = false; \
7969                 } \
7970         } \
7971 } while (0)
7972
7973 #define PIPE_CONF_QUIRK(quirk) \
7974         ((current_config->quirks | pipe_config->quirks) & (quirk))
7975
7976         PIPE_CONF_CHECK_I(cpu_transcoder);
7977
7978         PIPE_CONF_CHECK_BOOL(has_pch_encoder);
7979         PIPE_CONF_CHECK_I(fdi_lanes);
7980         PIPE_CONF_CHECK_M_N(fdi_m_n);
7981
7982         PIPE_CONF_CHECK_I(lane_count);
7983         PIPE_CONF_CHECK_X(lane_lat_optim_mask);
7984
7985         if (DISPLAY_VER(dev_priv) < 8) {
7986                 PIPE_CONF_CHECK_M_N(dp_m_n);
7987
7988                 if (current_config->has_drrs)
7989                         PIPE_CONF_CHECK_M_N(dp_m2_n2);
7990         } else
7991                 PIPE_CONF_CHECK_M_N_ALT(dp_m_n, dp_m2_n2);
7992
7993         PIPE_CONF_CHECK_X(output_types);
7994
7995         /* FIXME do the readout properly and get rid of this quirk */
7996         if (!PIPE_CONF_QUIRK(PIPE_CONFIG_QUIRK_BIGJOINER_SLAVE)) {
7997                 PIPE_CONF_CHECK_I(hw.pipe_mode.crtc_hdisplay);
7998                 PIPE_CONF_CHECK_I(hw.pipe_mode.crtc_htotal);
7999                 PIPE_CONF_CHECK_I(hw.pipe_mode.crtc_hblank_start);
8000                 PIPE_CONF_CHECK_I(hw.pipe_mode.crtc_hblank_end);
8001                 PIPE_CONF_CHECK_I(hw.pipe_mode.crtc_hsync_start);
8002                 PIPE_CONF_CHECK_I(hw.pipe_mode.crtc_hsync_end);
8003
8004                 PIPE_CONF_CHECK_I(hw.pipe_mode.crtc_vdisplay);
8005                 PIPE_CONF_CHECK_I(hw.pipe_mode.crtc_vtotal);
8006                 PIPE_CONF_CHECK_I(hw.pipe_mode.crtc_vblank_start);
8007                 PIPE_CONF_CHECK_I(hw.pipe_mode.crtc_vblank_end);
8008                 PIPE_CONF_CHECK_I(hw.pipe_mode.crtc_vsync_start);
8009                 PIPE_CONF_CHECK_I(hw.pipe_mode.crtc_vsync_end);
8010
8011                 PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_hdisplay);
8012                 PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_htotal);
8013                 PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_hblank_start);
8014                 PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_hblank_end);
8015                 PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_hsync_start);
8016                 PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_hsync_end);
8017
8018                 PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_vdisplay);
8019                 PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_vtotal);
8020                 PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_vblank_start);
8021                 PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_vblank_end);
8022                 PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_vsync_start);
8023                 PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_vsync_end);
8024
8025                 PIPE_CONF_CHECK_I(pixel_multiplier);
8026
8027                 PIPE_CONF_CHECK_FLAGS(hw.adjusted_mode.flags,
8028                                       DRM_MODE_FLAG_INTERLACE);
8029
8030                 if (!PIPE_CONF_QUIRK(PIPE_CONFIG_QUIRK_MODE_SYNC_FLAGS)) {
8031                         PIPE_CONF_CHECK_FLAGS(hw.adjusted_mode.flags,
8032                                               DRM_MODE_FLAG_PHSYNC);
8033                         PIPE_CONF_CHECK_FLAGS(hw.adjusted_mode.flags,
8034                                               DRM_MODE_FLAG_NHSYNC);
8035                         PIPE_CONF_CHECK_FLAGS(hw.adjusted_mode.flags,
8036                                               DRM_MODE_FLAG_PVSYNC);
8037                         PIPE_CONF_CHECK_FLAGS(hw.adjusted_mode.flags,
8038                                               DRM_MODE_FLAG_NVSYNC);
8039                 }
8040         }
8041
8042         PIPE_CONF_CHECK_I(output_format);
8043         PIPE_CONF_CHECK_BOOL(has_hdmi_sink);
8044         if ((DISPLAY_VER(dev_priv) < 8 && !IS_HASWELL(dev_priv)) ||
8045             IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
8046                 PIPE_CONF_CHECK_BOOL(limited_color_range);
8047
8048         PIPE_CONF_CHECK_BOOL(hdmi_scrambling);
8049         PIPE_CONF_CHECK_BOOL(hdmi_high_tmds_clock_ratio);
8050         PIPE_CONF_CHECK_BOOL(has_infoframe);
8051         /* FIXME do the readout properly and get rid of this quirk */
8052         if (!PIPE_CONF_QUIRK(PIPE_CONFIG_QUIRK_BIGJOINER_SLAVE))
8053                 PIPE_CONF_CHECK_BOOL(fec_enable);
8054
8055         PIPE_CONF_CHECK_BOOL_INCOMPLETE(has_audio);
8056
8057         PIPE_CONF_CHECK_X(gmch_pfit.control);
8058         /* pfit ratios are autocomputed by the hw on gen4+ */
8059         if (DISPLAY_VER(dev_priv) < 4)
8060                 PIPE_CONF_CHECK_X(gmch_pfit.pgm_ratios);
8061         PIPE_CONF_CHECK_X(gmch_pfit.lvds_border_bits);
8062
8063         /*
8064          * Changing the EDP transcoder input mux
8065          * (A_ONOFF vs. A_ON) requires a full modeset.
8066          */
8067         PIPE_CONF_CHECK_BOOL(pch_pfit.force_thru);
8068
8069         if (!fastset) {
8070                 PIPE_CONF_CHECK_I(pipe_src_w);
8071                 PIPE_CONF_CHECK_I(pipe_src_h);
8072
8073                 PIPE_CONF_CHECK_BOOL(pch_pfit.enabled);
8074                 if (current_config->pch_pfit.enabled) {
8075                         PIPE_CONF_CHECK_I(pch_pfit.dst.x1);
8076                         PIPE_CONF_CHECK_I(pch_pfit.dst.y1);
8077                         PIPE_CONF_CHECK_I(pch_pfit.dst.x2);
8078                         PIPE_CONF_CHECK_I(pch_pfit.dst.y2);
8079                 }
8080
8081                 PIPE_CONF_CHECK_I(scaler_state.scaler_id);
8082                 /* FIXME do the readout properly and get rid of this quirk */
8083                 if (!PIPE_CONF_QUIRK(PIPE_CONFIG_QUIRK_BIGJOINER_SLAVE))
8084                         PIPE_CONF_CHECK_CLOCK_FUZZY(pixel_rate);
8085
8086                 PIPE_CONF_CHECK_X(gamma_mode);
8087                 if (IS_CHERRYVIEW(dev_priv))
8088                         PIPE_CONF_CHECK_X(cgm_mode);
8089                 else
8090                         PIPE_CONF_CHECK_X(csc_mode);
8091                 PIPE_CONF_CHECK_BOOL(gamma_enable);
8092                 PIPE_CONF_CHECK_BOOL(csc_enable);
8093
8094                 PIPE_CONF_CHECK_I(linetime);
8095                 PIPE_CONF_CHECK_I(ips_linetime);
8096
8097                 bp_gamma = intel_color_get_gamma_bit_precision(pipe_config);
8098                 if (bp_gamma)
8099                         PIPE_CONF_CHECK_COLOR_LUT(gamma_mode, hw.gamma_lut, bp_gamma);
8100
8101                 if (current_config->active_planes) {
8102                         PIPE_CONF_CHECK_BOOL(has_psr);
8103                         PIPE_CONF_CHECK_BOOL(has_psr2);
8104                         PIPE_CONF_CHECK_BOOL(enable_psr2_sel_fetch);
8105                         PIPE_CONF_CHECK_I(dc3co_exitline);
8106                 }
8107         }
8108
8109         PIPE_CONF_CHECK_BOOL(double_wide);
8110
8111         if (dev_priv->dpll.mgr)
8112                 PIPE_CONF_CHECK_P(shared_dpll);
8113
8114         /* FIXME do the readout properly and get rid of this quirk */
8115         if (dev_priv->dpll.mgr && !PIPE_CONF_QUIRK(PIPE_CONFIG_QUIRK_BIGJOINER_SLAVE)) {
8116                 PIPE_CONF_CHECK_X(dpll_hw_state.dpll);
8117                 PIPE_CONF_CHECK_X(dpll_hw_state.dpll_md);
8118                 PIPE_CONF_CHECK_X(dpll_hw_state.fp0);
8119                 PIPE_CONF_CHECK_X(dpll_hw_state.fp1);
8120                 PIPE_CONF_CHECK_X(dpll_hw_state.wrpll);
8121                 PIPE_CONF_CHECK_X(dpll_hw_state.spll);
8122                 PIPE_CONF_CHECK_X(dpll_hw_state.ctrl1);
8123                 PIPE_CONF_CHECK_X(dpll_hw_state.cfgcr1);
8124                 PIPE_CONF_CHECK_X(dpll_hw_state.cfgcr2);
8125                 PIPE_CONF_CHECK_X(dpll_hw_state.cfgcr0);
8126                 PIPE_CONF_CHECK_X(dpll_hw_state.ebb0);
8127                 PIPE_CONF_CHECK_X(dpll_hw_state.ebb4);
8128                 PIPE_CONF_CHECK_X(dpll_hw_state.pll0);
8129                 PIPE_CONF_CHECK_X(dpll_hw_state.pll1);
8130                 PIPE_CONF_CHECK_X(dpll_hw_state.pll2);
8131                 PIPE_CONF_CHECK_X(dpll_hw_state.pll3);
8132                 PIPE_CONF_CHECK_X(dpll_hw_state.pll6);
8133                 PIPE_CONF_CHECK_X(dpll_hw_state.pll8);
8134                 PIPE_CONF_CHECK_X(dpll_hw_state.pll9);
8135                 PIPE_CONF_CHECK_X(dpll_hw_state.pll10);
8136                 PIPE_CONF_CHECK_X(dpll_hw_state.pcsdw12);
8137                 PIPE_CONF_CHECK_X(dpll_hw_state.mg_refclkin_ctl);
8138                 PIPE_CONF_CHECK_X(dpll_hw_state.mg_clktop2_coreclkctl1);
8139                 PIPE_CONF_CHECK_X(dpll_hw_state.mg_clktop2_hsclkctl);
8140                 PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_div0);
8141                 PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_div1);
8142                 PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_lf);
8143                 PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_frac_lock);
8144                 PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_ssc);
8145                 PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_bias);
8146                 PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_tdc_coldst_bias);
8147         }
8148
8149         if (!PIPE_CONF_QUIRK(PIPE_CONFIG_QUIRK_BIGJOINER_SLAVE)) {
8150                 PIPE_CONF_CHECK_X(dsi_pll.ctrl);
8151                 PIPE_CONF_CHECK_X(dsi_pll.div);
8152
8153                 if (IS_G4X(dev_priv) || DISPLAY_VER(dev_priv) >= 5)
8154                         PIPE_CONF_CHECK_I(pipe_bpp);
8155
8156                 PIPE_CONF_CHECK_CLOCK_FUZZY(hw.pipe_mode.crtc_clock);
8157                 PIPE_CONF_CHECK_CLOCK_FUZZY(hw.adjusted_mode.crtc_clock);
8158                 PIPE_CONF_CHECK_CLOCK_FUZZY(port_clock);
8159
8160                 PIPE_CONF_CHECK_I(min_voltage_level);
8161         }
8162
8163         if (current_config->has_psr || pipe_config->has_psr)
8164                 PIPE_CONF_CHECK_X_WITH_MASK(infoframes.enable,
8165                                             ~intel_hdmi_infoframe_enable(DP_SDP_VSC));
8166         else
8167                 PIPE_CONF_CHECK_X(infoframes.enable);
8168
8169         PIPE_CONF_CHECK_X(infoframes.gcp);
8170         PIPE_CONF_CHECK_INFOFRAME(avi);
8171         PIPE_CONF_CHECK_INFOFRAME(spd);
8172         PIPE_CONF_CHECK_INFOFRAME(hdmi);
8173         PIPE_CONF_CHECK_INFOFRAME(drm);
8174         PIPE_CONF_CHECK_DP_VSC_SDP(vsc);
8175
8176         PIPE_CONF_CHECK_X(sync_mode_slaves_mask);
8177         PIPE_CONF_CHECK_I(master_transcoder);
8178         PIPE_CONF_CHECK_BOOL(bigjoiner);
8179         PIPE_CONF_CHECK_BOOL(bigjoiner_slave);
8180         PIPE_CONF_CHECK_P(bigjoiner_linked_crtc);
8181
8182         PIPE_CONF_CHECK_I(dsc.compression_enable);
8183         PIPE_CONF_CHECK_I(dsc.dsc_split);
8184         PIPE_CONF_CHECK_I(dsc.compressed_bpp);
8185
8186         PIPE_CONF_CHECK_BOOL(splitter.enable);
8187         PIPE_CONF_CHECK_I(splitter.link_count);
8188         PIPE_CONF_CHECK_I(splitter.pixel_overlap);
8189
8190         PIPE_CONF_CHECK_I(mst_master_transcoder);
8191
8192         PIPE_CONF_CHECK_BOOL(vrr.enable);
8193         PIPE_CONF_CHECK_I(vrr.vmin);
8194         PIPE_CONF_CHECK_I(vrr.vmax);
8195         PIPE_CONF_CHECK_I(vrr.flipline);
8196         PIPE_CONF_CHECK_I(vrr.pipeline_full);
8197         PIPE_CONF_CHECK_I(vrr.guardband);
8198
8199 #undef PIPE_CONF_CHECK_X
8200 #undef PIPE_CONF_CHECK_I
8201 #undef PIPE_CONF_CHECK_BOOL
8202 #undef PIPE_CONF_CHECK_BOOL_INCOMPLETE
8203 #undef PIPE_CONF_CHECK_P
8204 #undef PIPE_CONF_CHECK_FLAGS
8205 #undef PIPE_CONF_CHECK_CLOCK_FUZZY
8206 #undef PIPE_CONF_CHECK_COLOR_LUT
8207 #undef PIPE_CONF_QUIRK
8208
8209         return ret;
8210 }
8211
8212 static void intel_pipe_config_sanity_check(struct drm_i915_private *dev_priv,
8213                                            const struct intel_crtc_state *pipe_config)
8214 {
8215         if (pipe_config->has_pch_encoder) {
8216                 int fdi_dotclock = intel_dotclock_calculate(intel_fdi_link_freq(dev_priv, pipe_config),
8217                                                             &pipe_config->fdi_m_n);
8218                 int dotclock = pipe_config->hw.adjusted_mode.crtc_clock;
8219
8220                 /*
8221                  * FDI already provided one idea for the dotclock.
8222                  * Yell if the encoder disagrees.
8223                  */
8224                 drm_WARN(&dev_priv->drm,
8225                          !intel_fuzzy_clock_check(fdi_dotclock, dotclock),
8226                          "FDI dotclock and encoder dotclock mismatch, fdi: %i, encoder: %i\n",
8227                          fdi_dotclock, dotclock);
8228         }
8229 }
8230
8231 static void verify_wm_state(struct intel_crtc *crtc,
8232                             struct intel_crtc_state *new_crtc_state)
8233 {
8234         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8235         struct skl_hw_state {
8236                 struct skl_ddb_entry ddb_y[I915_MAX_PLANES];
8237                 struct skl_ddb_entry ddb_uv[I915_MAX_PLANES];
8238                 struct skl_pipe_wm wm;
8239         } *hw;
8240         const struct skl_pipe_wm *sw_wm = &new_crtc_state->wm.skl.optimal;
8241         int level, max_level = ilk_wm_max_level(dev_priv);
8242         struct intel_plane *plane;
8243         u8 hw_enabled_slices;
8244
8245         if (DISPLAY_VER(dev_priv) < 9 || !new_crtc_state->hw.active)
8246                 return;
8247
8248         hw = kzalloc(sizeof(*hw), GFP_KERNEL);
8249         if (!hw)
8250                 return;
8251
8252         skl_pipe_wm_get_hw_state(crtc, &hw->wm);
8253
8254         skl_pipe_ddb_get_hw_state(crtc, hw->ddb_y, hw->ddb_uv);
8255
8256         hw_enabled_slices = intel_enabled_dbuf_slices_mask(dev_priv);
8257
8258         if (DISPLAY_VER(dev_priv) >= 11 &&
8259             hw_enabled_slices != dev_priv->dbuf.enabled_slices)
8260                 drm_err(&dev_priv->drm,
8261                         "mismatch in DBUF Slices (expected 0x%x, got 0x%x)\n",
8262                         dev_priv->dbuf.enabled_slices,
8263                         hw_enabled_slices);
8264
8265         for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
8266                 const struct skl_ddb_entry *hw_ddb_entry, *sw_ddb_entry;
8267                 const struct skl_wm_level *hw_wm_level, *sw_wm_level;
8268
8269                 /* Watermarks */
8270                 for (level = 0; level <= max_level; level++) {
8271                         hw_wm_level = &hw->wm.planes[plane->id].wm[level];
8272                         sw_wm_level = skl_plane_wm_level(sw_wm, plane->id, level);
8273
8274                         if (skl_wm_level_equals(hw_wm_level, sw_wm_level))
8275                                 continue;
8276
8277                         drm_err(&dev_priv->drm,
8278                                 "[PLANE:%d:%s] mismatch in WM%d (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
8279                                 plane->base.base.id, plane->base.name, level,
8280                                 sw_wm_level->enable,
8281                                 sw_wm_level->blocks,
8282                                 sw_wm_level->lines,
8283                                 hw_wm_level->enable,
8284                                 hw_wm_level->blocks,
8285                                 hw_wm_level->lines);
8286                 }
8287
8288                 hw_wm_level = &hw->wm.planes[plane->id].trans_wm;
8289                 sw_wm_level = skl_plane_trans_wm(sw_wm, plane->id);
8290
8291                 if (!skl_wm_level_equals(hw_wm_level, sw_wm_level)) {
8292                         drm_err(&dev_priv->drm,
8293                                 "[PLANE:%d:%s] mismatch in trans WM (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
8294                                 plane->base.base.id, plane->base.name,
8295                                 sw_wm_level->enable,
8296                                 sw_wm_level->blocks,
8297                                 sw_wm_level->lines,
8298                                 hw_wm_level->enable,
8299                                 hw_wm_level->blocks,
8300                                 hw_wm_level->lines);
8301                 }
8302
8303                 hw_wm_level = &hw->wm.planes[plane->id].sagv.wm0;
8304                 sw_wm_level = &sw_wm->planes[plane->id].sagv.wm0;
8305
8306                 if (HAS_HW_SAGV_WM(dev_priv) &&
8307                     !skl_wm_level_equals(hw_wm_level, sw_wm_level)) {
8308                         drm_err(&dev_priv->drm,
8309                                 "[PLANE:%d:%s] mismatch in SAGV WM (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
8310                                 plane->base.base.id, plane->base.name,
8311                                 sw_wm_level->enable,
8312                                 sw_wm_level->blocks,
8313                                 sw_wm_level->lines,
8314                                 hw_wm_level->enable,
8315                                 hw_wm_level->blocks,
8316                                 hw_wm_level->lines);
8317                 }
8318
8319                 hw_wm_level = &hw->wm.planes[plane->id].sagv.trans_wm;
8320                 sw_wm_level = &sw_wm->planes[plane->id].sagv.trans_wm;
8321
8322                 if (HAS_HW_SAGV_WM(dev_priv) &&
8323                     !skl_wm_level_equals(hw_wm_level, sw_wm_level)) {
8324                         drm_err(&dev_priv->drm,
8325                                 "[PLANE:%d:%s] mismatch in SAGV trans WM (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
8326                                 plane->base.base.id, plane->base.name,
8327                                 sw_wm_level->enable,
8328                                 sw_wm_level->blocks,
8329                                 sw_wm_level->lines,
8330                                 hw_wm_level->enable,
8331                                 hw_wm_level->blocks,
8332                                 hw_wm_level->lines);
8333                 }
8334
8335                 /* DDB */
8336                 hw_ddb_entry = &hw->ddb_y[plane->id];
8337                 sw_ddb_entry = &new_crtc_state->wm.skl.plane_ddb_y[plane->id];
8338
8339                 if (!skl_ddb_entry_equal(hw_ddb_entry, sw_ddb_entry)) {
8340                         drm_err(&dev_priv->drm,
8341                                 "[PLANE:%d:%s] mismatch in DDB (expected (%u,%u), found (%u,%u))\n",
8342                                 plane->base.base.id, plane->base.name,
8343                                 sw_ddb_entry->start, sw_ddb_entry->end,
8344                                 hw_ddb_entry->start, hw_ddb_entry->end);
8345                 }
8346         }
8347
8348         kfree(hw);
8349 }
8350
8351 static void
8352 verify_connector_state(struct intel_atomic_state *state,
8353                        struct intel_crtc *crtc)
8354 {
8355         struct drm_connector *connector;
8356         struct drm_connector_state *new_conn_state;
8357         int i;
8358
8359         for_each_new_connector_in_state(&state->base, connector, new_conn_state, i) {
8360                 struct drm_encoder *encoder = connector->encoder;
8361                 struct intel_crtc_state *crtc_state = NULL;
8362
8363                 if (new_conn_state->crtc != &crtc->base)
8364                         continue;
8365
8366                 if (crtc)
8367                         crtc_state = intel_atomic_get_new_crtc_state(state, crtc);
8368
8369                 intel_connector_verify_state(crtc_state, new_conn_state);
8370
8371                 I915_STATE_WARN(new_conn_state->best_encoder != encoder,
8372                      "connector's atomic encoder doesn't match legacy encoder\n");
8373         }
8374 }
8375
8376 static void
8377 verify_encoder_state(struct drm_i915_private *dev_priv, struct intel_atomic_state *state)
8378 {
8379         struct intel_encoder *encoder;
8380         struct drm_connector *connector;
8381         struct drm_connector_state *old_conn_state, *new_conn_state;
8382         int i;
8383
8384         for_each_intel_encoder(&dev_priv->drm, encoder) {
8385                 bool enabled = false, found = false;
8386                 enum pipe pipe;
8387
8388                 drm_dbg_kms(&dev_priv->drm, "[ENCODER:%d:%s]\n",
8389                             encoder->base.base.id,
8390                             encoder->base.name);
8391
8392                 for_each_oldnew_connector_in_state(&state->base, connector, old_conn_state,
8393                                                    new_conn_state, i) {
8394                         if (old_conn_state->best_encoder == &encoder->base)
8395                                 found = true;
8396
8397                         if (new_conn_state->best_encoder != &encoder->base)
8398                                 continue;
8399                         found = enabled = true;
8400
8401                         I915_STATE_WARN(new_conn_state->crtc !=
8402                                         encoder->base.crtc,
8403                              "connector's crtc doesn't match encoder crtc\n");
8404                 }
8405
8406                 if (!found)
8407                         continue;
8408
8409                 I915_STATE_WARN(!!encoder->base.crtc != enabled,
8410                      "encoder's enabled state mismatch "
8411                      "(expected %i, found %i)\n",
8412                      !!encoder->base.crtc, enabled);
8413
8414                 if (!encoder->base.crtc) {
8415                         bool active;
8416
8417                         active = encoder->get_hw_state(encoder, &pipe);
8418                         I915_STATE_WARN(active,
8419                              "encoder detached but still enabled on pipe %c.\n",
8420                              pipe_name(pipe));
8421                 }
8422         }
8423 }
8424
8425 static void
8426 verify_crtc_state(struct intel_crtc *crtc,
8427                   struct intel_crtc_state *old_crtc_state,
8428                   struct intel_crtc_state *new_crtc_state)
8429 {
8430         struct drm_device *dev = crtc->base.dev;
8431         struct drm_i915_private *dev_priv = to_i915(dev);
8432         struct intel_encoder *encoder;
8433         struct intel_crtc_state *pipe_config = old_crtc_state;
8434         struct drm_atomic_state *state = old_crtc_state->uapi.state;
8435         struct intel_crtc *master = crtc;
8436
8437         __drm_atomic_helper_crtc_destroy_state(&old_crtc_state->uapi);
8438         intel_crtc_free_hw_state(old_crtc_state);
8439         intel_crtc_state_reset(old_crtc_state, crtc);
8440         old_crtc_state->uapi.state = state;
8441
8442         drm_dbg_kms(&dev_priv->drm, "[CRTC:%d:%s]\n", crtc->base.base.id,
8443                     crtc->base.name);
8444
8445         pipe_config->hw.enable = new_crtc_state->hw.enable;
8446
8447         intel_crtc_get_pipe_config(pipe_config);
8448
8449         /* we keep both pipes enabled on 830 */
8450         if (IS_I830(dev_priv) && pipe_config->hw.active)
8451                 pipe_config->hw.active = new_crtc_state->hw.active;
8452
8453         I915_STATE_WARN(new_crtc_state->hw.active != pipe_config->hw.active,
8454                         "crtc active state doesn't match with hw state "
8455                         "(expected %i, found %i)\n",
8456                         new_crtc_state->hw.active, pipe_config->hw.active);
8457
8458         I915_STATE_WARN(crtc->active != new_crtc_state->hw.active,
8459                         "transitional active state does not match atomic hw state "
8460                         "(expected %i, found %i)\n",
8461                         new_crtc_state->hw.active, crtc->active);
8462
8463         if (new_crtc_state->bigjoiner_slave)
8464                 master = new_crtc_state->bigjoiner_linked_crtc;
8465
8466         for_each_encoder_on_crtc(dev, &master->base, encoder) {
8467                 enum pipe pipe;
8468                 bool active;
8469
8470                 active = encoder->get_hw_state(encoder, &pipe);
8471                 I915_STATE_WARN(active != new_crtc_state->hw.active,
8472                                 "[ENCODER:%i] active %i with crtc active %i\n",
8473                                 encoder->base.base.id, active,
8474                                 new_crtc_state->hw.active);
8475
8476                 I915_STATE_WARN(active && master->pipe != pipe,
8477                                 "Encoder connected to wrong pipe %c\n",
8478                                 pipe_name(pipe));
8479
8480                 if (active)
8481                         intel_encoder_get_config(encoder, pipe_config);
8482         }
8483
8484         if (!new_crtc_state->hw.active)
8485                 return;
8486
8487         if (new_crtc_state->bigjoiner_slave)
8488                 /* No PLLs set for slave */
8489                 pipe_config->shared_dpll = NULL;
8490
8491         intel_pipe_config_sanity_check(dev_priv, pipe_config);
8492
8493         if (!intel_pipe_config_compare(new_crtc_state,
8494                                        pipe_config, false)) {
8495                 I915_STATE_WARN(1, "pipe state doesn't match!\n");
8496                 intel_dump_pipe_config(pipe_config, NULL, "[hw state]");
8497                 intel_dump_pipe_config(new_crtc_state, NULL, "[sw state]");
8498         }
8499 }
8500
8501 static void
8502 intel_verify_planes(struct intel_atomic_state *state)
8503 {
8504         struct intel_plane *plane;
8505         const struct intel_plane_state *plane_state;
8506         int i;
8507
8508         for_each_new_intel_plane_in_state(state, plane,
8509                                           plane_state, i)
8510                 assert_plane(plane, plane_state->planar_slave ||
8511                              plane_state->uapi.visible);
8512 }
8513
8514 static void
8515 verify_single_dpll_state(struct drm_i915_private *dev_priv,
8516                          struct intel_shared_dpll *pll,
8517                          struct intel_crtc *crtc,
8518                          struct intel_crtc_state *new_crtc_state)
8519 {
8520         struct intel_dpll_hw_state dpll_hw_state;
8521         u8 pipe_mask;
8522         bool active;
8523
8524         memset(&dpll_hw_state, 0, sizeof(dpll_hw_state));
8525
8526         drm_dbg_kms(&dev_priv->drm, "%s\n", pll->info->name);
8527
8528         active = intel_dpll_get_hw_state(dev_priv, pll, &dpll_hw_state);
8529
8530         if (!(pll->info->flags & INTEL_DPLL_ALWAYS_ON)) {
8531                 I915_STATE_WARN(!pll->on && pll->active_mask,
8532                      "pll in active use but not on in sw tracking\n");
8533                 I915_STATE_WARN(pll->on && !pll->active_mask,
8534                      "pll is on but not used by any active pipe\n");
8535                 I915_STATE_WARN(pll->on != active,
8536                      "pll on state mismatch (expected %i, found %i)\n",
8537                      pll->on, active);
8538         }
8539
8540         if (!crtc) {
8541                 I915_STATE_WARN(pll->active_mask & ~pll->state.pipe_mask,
8542                                 "more active pll users than references: 0x%x vs 0x%x\n",
8543                                 pll->active_mask, pll->state.pipe_mask);
8544
8545                 return;
8546         }
8547
8548         pipe_mask = BIT(crtc->pipe);
8549
8550         if (new_crtc_state->hw.active)
8551                 I915_STATE_WARN(!(pll->active_mask & pipe_mask),
8552                                 "pll active mismatch (expected pipe %c in active mask 0x%x)\n",
8553                                 pipe_name(crtc->pipe), pll->active_mask);
8554         else
8555                 I915_STATE_WARN(pll->active_mask & pipe_mask,
8556                                 "pll active mismatch (didn't expect pipe %c in active mask 0x%x)\n",
8557                                 pipe_name(crtc->pipe), pll->active_mask);
8558
8559         I915_STATE_WARN(!(pll->state.pipe_mask & pipe_mask),
8560                         "pll enabled crtcs mismatch (expected 0x%x in 0x%x)\n",
8561                         pipe_mask, pll->state.pipe_mask);
8562
8563         I915_STATE_WARN(pll->on && memcmp(&pll->state.hw_state,
8564                                           &dpll_hw_state,
8565                                           sizeof(dpll_hw_state)),
8566                         "pll hw state mismatch\n");
8567 }
8568
8569 static void
8570 verify_shared_dpll_state(struct intel_crtc *crtc,
8571                          struct intel_crtc_state *old_crtc_state,
8572                          struct intel_crtc_state *new_crtc_state)
8573 {
8574         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8575
8576         if (new_crtc_state->shared_dpll)
8577                 verify_single_dpll_state(dev_priv, new_crtc_state->shared_dpll, crtc, new_crtc_state);
8578
8579         if (old_crtc_state->shared_dpll &&
8580             old_crtc_state->shared_dpll != new_crtc_state->shared_dpll) {
8581                 u8 pipe_mask = BIT(crtc->pipe);
8582                 struct intel_shared_dpll *pll = old_crtc_state->shared_dpll;
8583
8584                 I915_STATE_WARN(pll->active_mask & pipe_mask,
8585                                 "pll active mismatch (didn't expect pipe %c in active mask (0x%x))\n",
8586                                 pipe_name(crtc->pipe), pll->active_mask);
8587                 I915_STATE_WARN(pll->state.pipe_mask & pipe_mask,
8588                                 "pll enabled crtcs mismatch (found %x in enabled mask (0x%x))\n",
8589                                 pipe_name(crtc->pipe), pll->state.pipe_mask);
8590         }
8591 }
8592
8593 static void
8594 verify_mpllb_state(struct intel_atomic_state *state,
8595                    struct intel_crtc_state *new_crtc_state)
8596 {
8597         struct drm_i915_private *i915 = to_i915(state->base.dev);
8598         struct intel_mpllb_state mpllb_hw_state = { 0 };
8599         struct intel_mpllb_state *mpllb_sw_state = &new_crtc_state->mpllb_state;
8600         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
8601         struct intel_encoder *encoder;
8602
8603         if (!IS_DG2(i915))
8604                 return;
8605
8606         if (!new_crtc_state->hw.active)
8607                 return;
8608
8609         if (new_crtc_state->bigjoiner_slave)
8610                 return;
8611
8612         encoder = intel_get_crtc_new_encoder(state, new_crtc_state);
8613         intel_mpllb_readout_hw_state(encoder, &mpllb_hw_state);
8614
8615 #define MPLLB_CHECK(name) do { \
8616         if (mpllb_sw_state->name != mpllb_hw_state.name) { \
8617                 pipe_config_mismatch(false, crtc, "MPLLB:" __stringify(name), \
8618                                      "(expected 0x%08x, found 0x%08x)", \
8619                                      mpllb_sw_state->name, \
8620                                      mpllb_hw_state.name); \
8621         } \
8622 } while (0)
8623
8624         MPLLB_CHECK(mpllb_cp);
8625         MPLLB_CHECK(mpllb_div);
8626         MPLLB_CHECK(mpllb_div2);
8627         MPLLB_CHECK(mpllb_fracn1);
8628         MPLLB_CHECK(mpllb_fracn2);
8629         MPLLB_CHECK(mpllb_sscen);
8630         MPLLB_CHECK(mpllb_sscstep);
8631
8632         /*
8633          * ref_control is handled by the hardware/firemware and never
8634          * programmed by the software, but the proper values are supplied
8635          * in the bspec for verification purposes.
8636          */
8637         MPLLB_CHECK(ref_control);
8638
8639 #undef MPLLB_CHECK
8640 }
8641
8642 static void
8643 intel_modeset_verify_crtc(struct intel_crtc *crtc,
8644                           struct intel_atomic_state *state,
8645                           struct intel_crtc_state *old_crtc_state,
8646                           struct intel_crtc_state *new_crtc_state)
8647 {
8648         if (!intel_crtc_needs_modeset(new_crtc_state) && !new_crtc_state->update_pipe)
8649                 return;
8650
8651         verify_wm_state(crtc, new_crtc_state);
8652         verify_connector_state(state, crtc);
8653         verify_crtc_state(crtc, old_crtc_state, new_crtc_state);
8654         verify_shared_dpll_state(crtc, old_crtc_state, new_crtc_state);
8655         verify_mpllb_state(state, new_crtc_state);
8656 }
8657
8658 static void
8659 verify_disabled_dpll_state(struct drm_i915_private *dev_priv)
8660 {
8661         int i;
8662
8663         for (i = 0; i < dev_priv->dpll.num_shared_dpll; i++)
8664                 verify_single_dpll_state(dev_priv,
8665                                          &dev_priv->dpll.shared_dplls[i],
8666                                          NULL, NULL);
8667 }
8668
8669 static void
8670 intel_modeset_verify_disabled(struct drm_i915_private *dev_priv,
8671                               struct intel_atomic_state *state)
8672 {
8673         verify_encoder_state(dev_priv, state);
8674         verify_connector_state(state, NULL);
8675         verify_disabled_dpll_state(dev_priv);
8676 }
8677
8678 int intel_modeset_all_pipes(struct intel_atomic_state *state)
8679 {
8680         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
8681         struct intel_crtc *crtc;
8682
8683         /*
8684          * Add all pipes to the state, and force
8685          * a modeset on all the active ones.
8686          */
8687         for_each_intel_crtc(&dev_priv->drm, crtc) {
8688                 struct intel_crtc_state *crtc_state;
8689                 int ret;
8690
8691                 crtc_state = intel_atomic_get_crtc_state(&state->base, crtc);
8692                 if (IS_ERR(crtc_state))
8693                         return PTR_ERR(crtc_state);
8694
8695                 if (!crtc_state->hw.active ||
8696                     drm_atomic_crtc_needs_modeset(&crtc_state->uapi))
8697                         continue;
8698
8699                 crtc_state->uapi.mode_changed = true;
8700
8701                 ret = drm_atomic_add_affected_connectors(&state->base,
8702                                                          &crtc->base);
8703                 if (ret)
8704                         return ret;
8705
8706                 ret = intel_atomic_add_affected_planes(state, crtc);
8707                 if (ret)
8708                         return ret;
8709
8710                 crtc_state->update_planes |= crtc_state->active_planes;
8711         }
8712
8713         return 0;
8714 }
8715
8716 static void
8717 intel_crtc_update_active_timings(const struct intel_crtc_state *crtc_state)
8718 {
8719         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
8720         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8721         struct drm_display_mode adjusted_mode =
8722                 crtc_state->hw.adjusted_mode;
8723
8724         if (crtc_state->vrr.enable) {
8725                 adjusted_mode.crtc_vtotal = crtc_state->vrr.vmax;
8726                 adjusted_mode.crtc_vblank_end = crtc_state->vrr.vmax;
8727                 adjusted_mode.crtc_vblank_start = intel_vrr_vmin_vblank_start(crtc_state);
8728                 crtc->vmax_vblank_start = intel_vrr_vmax_vblank_start(crtc_state);
8729         }
8730
8731         drm_calc_timestamping_constants(&crtc->base, &adjusted_mode);
8732
8733         crtc->mode_flags = crtc_state->mode_flags;
8734
8735         /*
8736          * The scanline counter increments at the leading edge of hsync.
8737          *
8738          * On most platforms it starts counting from vtotal-1 on the
8739          * first active line. That means the scanline counter value is
8740          * always one less than what we would expect. Ie. just after
8741          * start of vblank, which also occurs at start of hsync (on the
8742          * last active line), the scanline counter will read vblank_start-1.
8743          *
8744          * On gen2 the scanline counter starts counting from 1 instead
8745          * of vtotal-1, so we have to subtract one (or rather add vtotal-1
8746          * to keep the value positive), instead of adding one.
8747          *
8748          * On HSW+ the behaviour of the scanline counter depends on the output
8749          * type. For DP ports it behaves like most other platforms, but on HDMI
8750          * there's an extra 1 line difference. So we need to add two instead of
8751          * one to the value.
8752          *
8753          * On VLV/CHV DSI the scanline counter would appear to increment
8754          * approx. 1/3 of a scanline before start of vblank. Unfortunately
8755          * that means we can't tell whether we're in vblank or not while
8756          * we're on that particular line. We must still set scanline_offset
8757          * to 1 so that the vblank timestamps come out correct when we query
8758          * the scanline counter from within the vblank interrupt handler.
8759          * However if queried just before the start of vblank we'll get an
8760          * answer that's slightly in the future.
8761          */
8762         if (DISPLAY_VER(dev_priv) == 2) {
8763                 int vtotal;
8764
8765                 vtotal = adjusted_mode.crtc_vtotal;
8766                 if (adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE)
8767                         vtotal /= 2;
8768
8769                 crtc->scanline_offset = vtotal - 1;
8770         } else if (HAS_DDI(dev_priv) &&
8771                    intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) {
8772                 crtc->scanline_offset = 2;
8773         } else {
8774                 crtc->scanline_offset = 1;
8775         }
8776 }
8777
8778 static void intel_modeset_clear_plls(struct intel_atomic_state *state)
8779 {
8780         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
8781         struct intel_crtc_state *new_crtc_state;
8782         struct intel_crtc *crtc;
8783         int i;
8784
8785         if (!dev_priv->display.crtc_compute_clock)
8786                 return;
8787
8788         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
8789                 if (!intel_crtc_needs_modeset(new_crtc_state))
8790                         continue;
8791
8792                 intel_release_shared_dplls(state, crtc);
8793         }
8794 }
8795
8796 /*
8797  * This implements the workaround described in the "notes" section of the mode
8798  * set sequence documentation. When going from no pipes or single pipe to
8799  * multiple pipes, and planes are enabled after the pipe, we need to wait at
8800  * least 2 vblanks on the first pipe before enabling planes on the second pipe.
8801  */
8802 static int hsw_mode_set_planes_workaround(struct intel_atomic_state *state)
8803 {
8804         struct intel_crtc_state *crtc_state;
8805         struct intel_crtc *crtc;
8806         struct intel_crtc_state *first_crtc_state = NULL;
8807         struct intel_crtc_state *other_crtc_state = NULL;
8808         enum pipe first_pipe = INVALID_PIPE, enabled_pipe = INVALID_PIPE;
8809         int i;
8810
8811         /* look at all crtc's that are going to be enabled in during modeset */
8812         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
8813                 if (!crtc_state->hw.active ||
8814                     !intel_crtc_needs_modeset(crtc_state))
8815                         continue;
8816
8817                 if (first_crtc_state) {
8818                         other_crtc_state = crtc_state;
8819                         break;
8820                 } else {
8821                         first_crtc_state = crtc_state;
8822                         first_pipe = crtc->pipe;
8823                 }
8824         }
8825
8826         /* No workaround needed? */
8827         if (!first_crtc_state)
8828                 return 0;
8829
8830         /* w/a possibly needed, check how many crtc's are already enabled. */
8831         for_each_intel_crtc(state->base.dev, crtc) {
8832                 crtc_state = intel_atomic_get_crtc_state(&state->base, crtc);
8833                 if (IS_ERR(crtc_state))
8834                         return PTR_ERR(crtc_state);
8835
8836                 crtc_state->hsw_workaround_pipe = INVALID_PIPE;
8837
8838                 if (!crtc_state->hw.active ||
8839                     intel_crtc_needs_modeset(crtc_state))
8840                         continue;
8841
8842                 /* 2 or more enabled crtcs means no need for w/a */
8843                 if (enabled_pipe != INVALID_PIPE)
8844                         return 0;
8845
8846                 enabled_pipe = crtc->pipe;
8847         }
8848
8849         if (enabled_pipe != INVALID_PIPE)
8850                 first_crtc_state->hsw_workaround_pipe = enabled_pipe;
8851         else if (other_crtc_state)
8852                 other_crtc_state->hsw_workaround_pipe = first_pipe;
8853
8854         return 0;
8855 }
8856
8857 u8 intel_calc_active_pipes(struct intel_atomic_state *state,
8858                            u8 active_pipes)
8859 {
8860         const struct intel_crtc_state *crtc_state;
8861         struct intel_crtc *crtc;
8862         int i;
8863
8864         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
8865                 if (crtc_state->hw.active)
8866                         active_pipes |= BIT(crtc->pipe);
8867                 else
8868                         active_pipes &= ~BIT(crtc->pipe);
8869         }
8870
8871         return active_pipes;
8872 }
8873
8874 static int intel_modeset_checks(struct intel_atomic_state *state)
8875 {
8876         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
8877
8878         state->modeset = true;
8879
8880         if (IS_HASWELL(dev_priv))
8881                 return hsw_mode_set_planes_workaround(state);
8882
8883         return 0;
8884 }
8885
8886 /*
8887  * Handle calculation of various watermark data at the end of the atomic check
8888  * phase.  The code here should be run after the per-crtc and per-plane 'check'
8889  * handlers to ensure that all derived state has been updated.
8890  */
8891 static int calc_watermark_data(struct intel_atomic_state *state)
8892 {
8893         struct drm_device *dev = state->base.dev;
8894         struct drm_i915_private *dev_priv = to_i915(dev);
8895
8896         /* Is there platform-specific watermark information to calculate? */
8897         if (dev_priv->display.compute_global_watermarks)
8898                 return dev_priv->display.compute_global_watermarks(state);
8899
8900         return 0;
8901 }
8902
8903 static void intel_crtc_check_fastset(const struct intel_crtc_state *old_crtc_state,
8904                                      struct intel_crtc_state *new_crtc_state)
8905 {
8906         if (!intel_pipe_config_compare(old_crtc_state, new_crtc_state, true))
8907                 return;
8908
8909         new_crtc_state->uapi.mode_changed = false;
8910         new_crtc_state->update_pipe = true;
8911 }
8912
8913 static void intel_crtc_copy_fastset(const struct intel_crtc_state *old_crtc_state,
8914                                     struct intel_crtc_state *new_crtc_state)
8915 {
8916         /*
8917          * If we're not doing the full modeset we want to
8918          * keep the current M/N values as they may be
8919          * sufficiently different to the computed values
8920          * to cause problems.
8921          *
8922          * FIXME: should really copy more fuzzy state here
8923          */
8924         new_crtc_state->fdi_m_n = old_crtc_state->fdi_m_n;
8925         new_crtc_state->dp_m_n = old_crtc_state->dp_m_n;
8926         new_crtc_state->dp_m2_n2 = old_crtc_state->dp_m2_n2;
8927         new_crtc_state->has_drrs = old_crtc_state->has_drrs;
8928 }
8929
8930 static int intel_crtc_add_planes_to_state(struct intel_atomic_state *state,
8931                                           struct intel_crtc *crtc,
8932                                           u8 plane_ids_mask)
8933 {
8934         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
8935         struct intel_plane *plane;
8936
8937         for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
8938                 struct intel_plane_state *plane_state;
8939
8940                 if ((plane_ids_mask & BIT(plane->id)) == 0)
8941                         continue;
8942
8943                 plane_state = intel_atomic_get_plane_state(state, plane);
8944                 if (IS_ERR(plane_state))
8945                         return PTR_ERR(plane_state);
8946         }
8947
8948         return 0;
8949 }
8950
8951 int intel_atomic_add_affected_planes(struct intel_atomic_state *state,
8952                                      struct intel_crtc *crtc)
8953 {
8954         const struct intel_crtc_state *old_crtc_state =
8955                 intel_atomic_get_old_crtc_state(state, crtc);
8956         const struct intel_crtc_state *new_crtc_state =
8957                 intel_atomic_get_new_crtc_state(state, crtc);
8958
8959         return intel_crtc_add_planes_to_state(state, crtc,
8960                                               old_crtc_state->enabled_planes |
8961                                               new_crtc_state->enabled_planes);
8962 }
8963
8964 static bool active_planes_affects_min_cdclk(struct drm_i915_private *dev_priv)
8965 {
8966         /* See {hsw,vlv,ivb}_plane_ratio() */
8967         return IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv) ||
8968                 IS_CHERRYVIEW(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
8969                 IS_IVYBRIDGE(dev_priv);
8970 }
8971
8972 static int intel_crtc_add_bigjoiner_planes(struct intel_atomic_state *state,
8973                                            struct intel_crtc *crtc,
8974                                            struct intel_crtc *other)
8975 {
8976         const struct intel_plane_state *plane_state;
8977         struct intel_plane *plane;
8978         u8 plane_ids = 0;
8979         int i;
8980
8981         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
8982                 if (plane->pipe == crtc->pipe)
8983                         plane_ids |= BIT(plane->id);
8984         }
8985
8986         return intel_crtc_add_planes_to_state(state, other, plane_ids);
8987 }
8988
8989 static int intel_bigjoiner_add_affected_planes(struct intel_atomic_state *state)
8990 {
8991         const struct intel_crtc_state *crtc_state;
8992         struct intel_crtc *crtc;
8993         int i;
8994
8995         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
8996                 int ret;
8997
8998                 if (!crtc_state->bigjoiner)
8999                         continue;
9000
9001                 ret = intel_crtc_add_bigjoiner_planes(state, crtc,
9002                                                       crtc_state->bigjoiner_linked_crtc);
9003                 if (ret)
9004                         return ret;
9005         }
9006
9007         return 0;
9008 }
9009
9010 static int intel_atomic_check_planes(struct intel_atomic_state *state)
9011 {
9012         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
9013         struct intel_crtc_state *old_crtc_state, *new_crtc_state;
9014         struct intel_plane_state *plane_state;
9015         struct intel_plane *plane;
9016         struct intel_crtc *crtc;
9017         int i, ret;
9018
9019         ret = icl_add_linked_planes(state);
9020         if (ret)
9021                 return ret;
9022
9023         ret = intel_bigjoiner_add_affected_planes(state);
9024         if (ret)
9025                 return ret;
9026
9027         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
9028                 ret = intel_plane_atomic_check(state, plane);
9029                 if (ret) {
9030                         drm_dbg_atomic(&dev_priv->drm,
9031                                        "[PLANE:%d:%s] atomic driver check failed\n",
9032                                        plane->base.base.id, plane->base.name);
9033                         return ret;
9034                 }
9035         }
9036
9037         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
9038                                             new_crtc_state, i) {
9039                 u8 old_active_planes, new_active_planes;
9040
9041                 ret = icl_check_nv12_planes(new_crtc_state);
9042                 if (ret)
9043                         return ret;
9044
9045                 /*
9046                  * On some platforms the number of active planes affects
9047                  * the planes' minimum cdclk calculation. Add such planes
9048                  * to the state before we compute the minimum cdclk.
9049                  */
9050                 if (!active_planes_affects_min_cdclk(dev_priv))
9051                         continue;
9052
9053                 old_active_planes = old_crtc_state->active_planes & ~BIT(PLANE_CURSOR);
9054                 new_active_planes = new_crtc_state->active_planes & ~BIT(PLANE_CURSOR);
9055
9056                 if (hweight8(old_active_planes) == hweight8(new_active_planes))
9057                         continue;
9058
9059                 ret = intel_crtc_add_planes_to_state(state, crtc, new_active_planes);
9060                 if (ret)
9061                         return ret;
9062         }
9063
9064         return 0;
9065 }
9066
9067 static int intel_atomic_check_cdclk(struct intel_atomic_state *state,
9068                                     bool *need_cdclk_calc)
9069 {
9070         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
9071         const struct intel_cdclk_state *old_cdclk_state;
9072         const struct intel_cdclk_state *new_cdclk_state;
9073         struct intel_plane_state *plane_state;
9074         struct intel_bw_state *new_bw_state;
9075         struct intel_plane *plane;
9076         int min_cdclk = 0;
9077         enum pipe pipe;
9078         int ret;
9079         int i;
9080         /*
9081          * active_planes bitmask has been updated, and potentially
9082          * affected planes are part of the state. We can now
9083          * compute the minimum cdclk for each plane.
9084          */
9085         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
9086                 ret = intel_plane_calc_min_cdclk(state, plane, need_cdclk_calc);
9087                 if (ret)
9088                         return ret;
9089         }
9090
9091         old_cdclk_state = intel_atomic_get_old_cdclk_state(state);
9092         new_cdclk_state = intel_atomic_get_new_cdclk_state(state);
9093
9094         if (new_cdclk_state &&
9095             old_cdclk_state->force_min_cdclk != new_cdclk_state->force_min_cdclk)
9096                 *need_cdclk_calc = true;
9097
9098         ret = dev_priv->display.bw_calc_min_cdclk(state);
9099         if (ret)
9100                 return ret;
9101
9102         new_bw_state = intel_atomic_get_new_bw_state(state);
9103
9104         if (!new_cdclk_state || !new_bw_state)
9105                 return 0;
9106
9107         for_each_pipe(dev_priv, pipe) {
9108                 min_cdclk = max(new_cdclk_state->min_cdclk[pipe], min_cdclk);
9109
9110                 /*
9111                  * Currently do this change only if we need to increase
9112                  */
9113                 if (new_bw_state->min_cdclk > min_cdclk)
9114                         *need_cdclk_calc = true;
9115         }
9116
9117         return 0;
9118 }
9119
9120 static int intel_atomic_check_crtcs(struct intel_atomic_state *state)
9121 {
9122         struct intel_crtc_state *crtc_state;
9123         struct intel_crtc *crtc;
9124         int i;
9125
9126         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
9127                 struct drm_i915_private *i915 = to_i915(crtc->base.dev);
9128                 int ret;
9129
9130                 ret = intel_crtc_atomic_check(state, crtc);
9131                 if (ret) {
9132                         drm_dbg_atomic(&i915->drm,
9133                                        "[CRTC:%d:%s] atomic driver check failed\n",
9134                                        crtc->base.base.id, crtc->base.name);
9135                         return ret;
9136                 }
9137         }
9138
9139         return 0;
9140 }
9141
9142 static bool intel_cpu_transcoders_need_modeset(struct intel_atomic_state *state,
9143                                                u8 transcoders)
9144 {
9145         const struct intel_crtc_state *new_crtc_state;
9146         struct intel_crtc *crtc;
9147         int i;
9148
9149         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
9150                 if (new_crtc_state->hw.enable &&
9151                     transcoders & BIT(new_crtc_state->cpu_transcoder) &&
9152                     intel_crtc_needs_modeset(new_crtc_state))
9153                         return true;
9154         }
9155
9156         return false;
9157 }
9158
9159 static int intel_atomic_check_bigjoiner(struct intel_atomic_state *state,
9160                                         struct intel_crtc *crtc,
9161                                         struct intel_crtc_state *old_crtc_state,
9162                                         struct intel_crtc_state *new_crtc_state)
9163 {
9164         struct intel_crtc_state *slave_crtc_state, *master_crtc_state;
9165         struct intel_crtc *slave, *master;
9166
9167         /* slave being enabled, is master is still claiming this crtc? */
9168         if (old_crtc_state->bigjoiner_slave) {
9169                 slave = crtc;
9170                 master = old_crtc_state->bigjoiner_linked_crtc;
9171                 master_crtc_state = intel_atomic_get_new_crtc_state(state, master);
9172                 if (!master_crtc_state || !intel_crtc_needs_modeset(master_crtc_state))
9173                         goto claimed;
9174         }
9175
9176         if (!new_crtc_state->bigjoiner)
9177                 return 0;
9178
9179         slave = intel_dsc_get_bigjoiner_secondary(crtc);
9180         if (!slave) {
9181                 DRM_DEBUG_KMS("[CRTC:%d:%s] Big joiner configuration requires "
9182                               "CRTC + 1 to be used, doesn't exist\n",
9183                               crtc->base.base.id, crtc->base.name);
9184                 return -EINVAL;
9185         }
9186
9187         new_crtc_state->bigjoiner_linked_crtc = slave;
9188         slave_crtc_state = intel_atomic_get_crtc_state(&state->base, slave);
9189         master = crtc;
9190         if (IS_ERR(slave_crtc_state))
9191                 return PTR_ERR(slave_crtc_state);
9192
9193         /* master being enabled, slave was already configured? */
9194         if (slave_crtc_state->uapi.enable)
9195                 goto claimed;
9196
9197         DRM_DEBUG_KMS("[CRTC:%d:%s] Used as slave for big joiner\n",
9198                       slave->base.base.id, slave->base.name);
9199
9200         return copy_bigjoiner_crtc_state(slave_crtc_state, new_crtc_state);
9201
9202 claimed:
9203         DRM_DEBUG_KMS("[CRTC:%d:%s] Slave is enabled as normal CRTC, but "
9204                       "[CRTC:%d:%s] claiming this CRTC for bigjoiner.\n",
9205                       slave->base.base.id, slave->base.name,
9206                       master->base.base.id, master->base.name);
9207         return -EINVAL;
9208 }
9209
9210 static void kill_bigjoiner_slave(struct intel_atomic_state *state,
9211                                  struct intel_crtc_state *master_crtc_state)
9212 {
9213         struct intel_crtc_state *slave_crtc_state =
9214                 intel_atomic_get_new_crtc_state(state, master_crtc_state->bigjoiner_linked_crtc);
9215
9216         slave_crtc_state->bigjoiner = master_crtc_state->bigjoiner = false;
9217         slave_crtc_state->bigjoiner_slave = master_crtc_state->bigjoiner_slave = false;
9218         slave_crtc_state->bigjoiner_linked_crtc = master_crtc_state->bigjoiner_linked_crtc = NULL;
9219         intel_crtc_copy_uapi_to_hw_state(state, slave_crtc_state);
9220 }
9221
9222 /**
9223  * DOC: asynchronous flip implementation
9224  *
9225  * Asynchronous page flip is the implementation for the DRM_MODE_PAGE_FLIP_ASYNC
9226  * flag. Currently async flip is only supported via the drmModePageFlip IOCTL.
9227  * Correspondingly, support is currently added for primary plane only.
9228  *
9229  * Async flip can only change the plane surface address, so anything else
9230  * changing is rejected from the intel_atomic_check_async() function.
9231  * Once this check is cleared, flip done interrupt is enabled using
9232  * the intel_crtc_enable_flip_done() function.
9233  *
9234  * As soon as the surface address register is written, flip done interrupt is
9235  * generated and the requested events are sent to the usersapce in the interrupt
9236  * handler itself. The timestamp and sequence sent during the flip done event
9237  * correspond to the last vblank and have no relation to the actual time when
9238  * the flip done event was sent.
9239  */
9240 static int intel_atomic_check_async(struct intel_atomic_state *state)
9241 {
9242         struct drm_i915_private *i915 = to_i915(state->base.dev);
9243         const struct intel_crtc_state *old_crtc_state, *new_crtc_state;
9244         const struct intel_plane_state *new_plane_state, *old_plane_state;
9245         struct intel_crtc *crtc;
9246         struct intel_plane *plane;
9247         int i;
9248
9249         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
9250                                             new_crtc_state, i) {
9251                 if (intel_crtc_needs_modeset(new_crtc_state)) {
9252                         drm_dbg_kms(&i915->drm, "Modeset Required. Async flip not supported\n");
9253                         return -EINVAL;
9254                 }
9255
9256                 if (!new_crtc_state->hw.active) {
9257                         drm_dbg_kms(&i915->drm, "CRTC inactive\n");
9258                         return -EINVAL;
9259                 }
9260                 if (old_crtc_state->active_planes != new_crtc_state->active_planes) {
9261                         drm_dbg_kms(&i915->drm,
9262                                     "Active planes cannot be changed during async flip\n");
9263                         return -EINVAL;
9264                 }
9265         }
9266
9267         for_each_oldnew_intel_plane_in_state(state, plane, old_plane_state,
9268                                              new_plane_state, i) {
9269                 /*
9270                  * TODO: Async flip is only supported through the page flip IOCTL
9271                  * as of now. So support currently added for primary plane only.
9272                  * Support for other planes on platforms on which supports
9273                  * this(vlv/chv and icl+) should be added when async flip is
9274                  * enabled in the atomic IOCTL path.
9275                  */
9276                 if (!plane->async_flip)
9277                         return -EINVAL;
9278
9279                 /*
9280                  * FIXME: This check is kept generic for all platforms.
9281                  * Need to verify this for all gen9 platforms to enable
9282                  * this selectively if required.
9283                  */
9284                 switch (new_plane_state->hw.fb->modifier) {
9285                 case I915_FORMAT_MOD_X_TILED:
9286                 case I915_FORMAT_MOD_Y_TILED:
9287                 case I915_FORMAT_MOD_Yf_TILED:
9288                         break;
9289                 default:
9290                         drm_dbg_kms(&i915->drm,
9291                                     "Linear memory/CCS does not support async flips\n");
9292                         return -EINVAL;
9293                 }
9294
9295                 if (old_plane_state->view.color_plane[0].stride !=
9296                     new_plane_state->view.color_plane[0].stride) {
9297                         drm_dbg_kms(&i915->drm, "Stride cannot be changed in async flip\n");
9298                         return -EINVAL;
9299                 }
9300
9301                 if (old_plane_state->hw.fb->modifier !=
9302                     new_plane_state->hw.fb->modifier) {
9303                         drm_dbg_kms(&i915->drm,
9304                                     "Framebuffer modifiers cannot be changed in async flip\n");
9305                         return -EINVAL;
9306                 }
9307
9308                 if (old_plane_state->hw.fb->format !=
9309                     new_plane_state->hw.fb->format) {
9310                         drm_dbg_kms(&i915->drm,
9311                                     "Framebuffer format cannot be changed in async flip\n");
9312                         return -EINVAL;
9313                 }
9314
9315                 if (old_plane_state->hw.rotation !=
9316                     new_plane_state->hw.rotation) {
9317                         drm_dbg_kms(&i915->drm, "Rotation cannot be changed in async flip\n");
9318                         return -EINVAL;
9319                 }
9320
9321                 if (!drm_rect_equals(&old_plane_state->uapi.src, &new_plane_state->uapi.src) ||
9322                     !drm_rect_equals(&old_plane_state->uapi.dst, &new_plane_state->uapi.dst)) {
9323                         drm_dbg_kms(&i915->drm,
9324                                     "Plane size/co-ordinates cannot be changed in async flip\n");
9325                         return -EINVAL;
9326                 }
9327
9328                 if (old_plane_state->hw.alpha != new_plane_state->hw.alpha) {
9329                         drm_dbg_kms(&i915->drm, "Alpha value cannot be changed in async flip\n");
9330                         return -EINVAL;
9331                 }
9332
9333                 if (old_plane_state->hw.pixel_blend_mode !=
9334                     new_plane_state->hw.pixel_blend_mode) {
9335                         drm_dbg_kms(&i915->drm,
9336                                     "Pixel blend mode cannot be changed in async flip\n");
9337                         return -EINVAL;
9338                 }
9339
9340                 if (old_plane_state->hw.color_encoding != new_plane_state->hw.color_encoding) {
9341                         drm_dbg_kms(&i915->drm,
9342                                     "Color encoding cannot be changed in async flip\n");
9343                         return -EINVAL;
9344                 }
9345
9346                 if (old_plane_state->hw.color_range != new_plane_state->hw.color_range) {
9347                         drm_dbg_kms(&i915->drm, "Color range cannot be changed in async flip\n");
9348                         return -EINVAL;
9349                 }
9350         }
9351
9352         return 0;
9353 }
9354
9355 static int intel_bigjoiner_add_affected_crtcs(struct intel_atomic_state *state)
9356 {
9357         struct intel_crtc_state *crtc_state;
9358         struct intel_crtc *crtc;
9359         int i;
9360
9361         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
9362                 struct intel_crtc_state *linked_crtc_state;
9363                 struct intel_crtc *linked_crtc;
9364                 int ret;
9365
9366                 if (!crtc_state->bigjoiner)
9367                         continue;
9368
9369                 linked_crtc = crtc_state->bigjoiner_linked_crtc;
9370                 linked_crtc_state = intel_atomic_get_crtc_state(&state->base, linked_crtc);
9371                 if (IS_ERR(linked_crtc_state))
9372                         return PTR_ERR(linked_crtc_state);
9373
9374                 if (!intel_crtc_needs_modeset(crtc_state))
9375                         continue;
9376
9377                 linked_crtc_state->uapi.mode_changed = true;
9378
9379                 ret = drm_atomic_add_affected_connectors(&state->base,
9380                                                          &linked_crtc->base);
9381                 if (ret)
9382                         return ret;
9383
9384                 ret = intel_atomic_add_affected_planes(state, linked_crtc);
9385                 if (ret)
9386                         return ret;
9387         }
9388
9389         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
9390                 /* Kill old bigjoiner link, we may re-establish afterwards */
9391                 if (intel_crtc_needs_modeset(crtc_state) &&
9392                     crtc_state->bigjoiner && !crtc_state->bigjoiner_slave)
9393                         kill_bigjoiner_slave(state, crtc_state);
9394         }
9395
9396         return 0;
9397 }
9398
9399 /**
9400  * intel_atomic_check - validate state object
9401  * @dev: drm device
9402  * @_state: state to validate
9403  */
9404 static int intel_atomic_check(struct drm_device *dev,
9405                               struct drm_atomic_state *_state)
9406 {
9407         struct drm_i915_private *dev_priv = to_i915(dev);
9408         struct intel_atomic_state *state = to_intel_atomic_state(_state);
9409         struct intel_crtc_state *old_crtc_state, *new_crtc_state;
9410         struct intel_crtc *crtc;
9411         int ret, i;
9412         bool any_ms = false;
9413
9414         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
9415                                             new_crtc_state, i) {
9416                 if (new_crtc_state->inherited != old_crtc_state->inherited)
9417                         new_crtc_state->uapi.mode_changed = true;
9418         }
9419
9420         intel_vrr_check_modeset(state);
9421
9422         ret = drm_atomic_helper_check_modeset(dev, &state->base);
9423         if (ret)
9424                 goto fail;
9425
9426         ret = intel_bigjoiner_add_affected_crtcs(state);
9427         if (ret)
9428                 goto fail;
9429
9430         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
9431                                             new_crtc_state, i) {
9432                 if (!intel_crtc_needs_modeset(new_crtc_state)) {
9433                         /* Light copy */
9434                         intel_crtc_copy_uapi_to_hw_state_nomodeset(state, new_crtc_state);
9435
9436                         continue;
9437                 }
9438
9439                 if (!new_crtc_state->uapi.enable) {
9440                         if (!new_crtc_state->bigjoiner_slave) {
9441                                 intel_crtc_copy_uapi_to_hw_state(state, new_crtc_state);
9442                                 any_ms = true;
9443                         }
9444                         continue;
9445                 }
9446
9447                 ret = intel_crtc_prepare_cleared_state(state, new_crtc_state);
9448                 if (ret)
9449                         goto fail;
9450
9451                 ret = intel_modeset_pipe_config(state, new_crtc_state);
9452                 if (ret)
9453                         goto fail;
9454
9455                 ret = intel_atomic_check_bigjoiner(state, crtc, old_crtc_state,
9456                                                    new_crtc_state);
9457                 if (ret)
9458                         goto fail;
9459         }
9460
9461         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
9462                                             new_crtc_state, i) {
9463                 if (!intel_crtc_needs_modeset(new_crtc_state))
9464                         continue;
9465
9466                 ret = intel_modeset_pipe_config_late(new_crtc_state);
9467                 if (ret)
9468                         goto fail;
9469
9470                 intel_crtc_check_fastset(old_crtc_state, new_crtc_state);
9471         }
9472
9473         /**
9474          * Check if fastset is allowed by external dependencies like other
9475          * pipes and transcoders.
9476          *
9477          * Right now it only forces a fullmodeset when the MST master
9478          * transcoder did not changed but the pipe of the master transcoder
9479          * needs a fullmodeset so all slaves also needs to do a fullmodeset or
9480          * in case of port synced crtcs, if one of the synced crtcs
9481          * needs a full modeset, all other synced crtcs should be
9482          * forced a full modeset.
9483          */
9484         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
9485                 if (!new_crtc_state->hw.enable || intel_crtc_needs_modeset(new_crtc_state))
9486                         continue;
9487
9488                 if (intel_dp_mst_is_slave_trans(new_crtc_state)) {
9489                         enum transcoder master = new_crtc_state->mst_master_transcoder;
9490
9491                         if (intel_cpu_transcoders_need_modeset(state, BIT(master))) {
9492                                 new_crtc_state->uapi.mode_changed = true;
9493                                 new_crtc_state->update_pipe = false;
9494                         }
9495                 }
9496
9497                 if (is_trans_port_sync_mode(new_crtc_state)) {
9498                         u8 trans = new_crtc_state->sync_mode_slaves_mask;
9499
9500                         if (new_crtc_state->master_transcoder != INVALID_TRANSCODER)
9501                                 trans |= BIT(new_crtc_state->master_transcoder);
9502
9503                         if (intel_cpu_transcoders_need_modeset(state, trans)) {
9504                                 new_crtc_state->uapi.mode_changed = true;
9505                                 new_crtc_state->update_pipe = false;
9506                         }
9507                 }
9508
9509                 if (new_crtc_state->bigjoiner) {
9510                         struct intel_crtc_state *linked_crtc_state =
9511                                 intel_atomic_get_new_crtc_state(state, new_crtc_state->bigjoiner_linked_crtc);
9512
9513                         if (intel_crtc_needs_modeset(linked_crtc_state)) {
9514                                 new_crtc_state->uapi.mode_changed = true;
9515                                 new_crtc_state->update_pipe = false;
9516                         }
9517                 }
9518         }
9519
9520         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
9521                                             new_crtc_state, i) {
9522                 if (intel_crtc_needs_modeset(new_crtc_state)) {
9523                         any_ms = true;
9524                         continue;
9525                 }
9526
9527                 if (!new_crtc_state->update_pipe)
9528                         continue;
9529
9530                 intel_crtc_copy_fastset(old_crtc_state, new_crtc_state);
9531         }
9532
9533         if (any_ms && !check_digital_port_conflicts(state)) {
9534                 drm_dbg_kms(&dev_priv->drm,
9535                             "rejecting conflicting digital port configuration\n");
9536                 ret = -EINVAL;
9537                 goto fail;
9538         }
9539
9540         ret = drm_dp_mst_atomic_check(&state->base);
9541         if (ret)
9542                 goto fail;
9543
9544         ret = intel_atomic_check_planes(state);
9545         if (ret)
9546                 goto fail;
9547
9548         intel_fbc_choose_crtc(dev_priv, state);
9549         ret = calc_watermark_data(state);
9550         if (ret)
9551                 goto fail;
9552
9553         ret = intel_bw_atomic_check(state);
9554         if (ret)
9555                 goto fail;
9556
9557         ret = intel_atomic_check_cdclk(state, &any_ms);
9558         if (ret)
9559                 goto fail;
9560
9561         if (intel_any_crtc_needs_modeset(state))
9562                 any_ms = true;
9563
9564         if (any_ms) {
9565                 ret = intel_modeset_checks(state);
9566                 if (ret)
9567                         goto fail;
9568
9569                 ret = intel_modeset_calc_cdclk(state);
9570                 if (ret)
9571                         return ret;
9572
9573                 intel_modeset_clear_plls(state);
9574         }
9575
9576         ret = intel_atomic_check_crtcs(state);
9577         if (ret)
9578                 goto fail;
9579
9580         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
9581                                             new_crtc_state, i) {
9582                 if (new_crtc_state->uapi.async_flip) {
9583                         ret = intel_atomic_check_async(state);
9584                         if (ret)
9585                                 goto fail;
9586                 }
9587
9588                 if (!intel_crtc_needs_modeset(new_crtc_state) &&
9589                     !new_crtc_state->update_pipe)
9590                         continue;
9591
9592                 intel_dump_pipe_config(new_crtc_state, state,
9593                                        intel_crtc_needs_modeset(new_crtc_state) ?
9594                                        "[modeset]" : "[fastset]");
9595         }
9596
9597         return 0;
9598
9599  fail:
9600         if (ret == -EDEADLK)
9601                 return ret;
9602
9603         /*
9604          * FIXME would probably be nice to know which crtc specifically
9605          * caused the failure, in cases where we can pinpoint it.
9606          */
9607         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
9608                                             new_crtc_state, i)
9609                 intel_dump_pipe_config(new_crtc_state, state, "[failed]");
9610
9611         return ret;
9612 }
9613
9614 static int intel_atomic_prepare_commit(struct intel_atomic_state *state)
9615 {
9616         struct intel_crtc_state *crtc_state;
9617         struct intel_crtc *crtc;
9618         int i, ret;
9619
9620         ret = drm_atomic_helper_prepare_planes(state->base.dev, &state->base);
9621         if (ret < 0)
9622                 return ret;
9623
9624         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
9625                 bool mode_changed = intel_crtc_needs_modeset(crtc_state);
9626
9627                 if (mode_changed || crtc_state->update_pipe ||
9628                     crtc_state->uapi.color_mgmt_changed) {
9629                         intel_dsb_prepare(crtc_state);
9630                 }
9631         }
9632
9633         return 0;
9634 }
9635
9636 void intel_crtc_arm_fifo_underrun(struct intel_crtc *crtc,
9637                                   struct intel_crtc_state *crtc_state)
9638 {
9639         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9640
9641         if (DISPLAY_VER(dev_priv) != 2 || crtc_state->active_planes)
9642                 intel_set_cpu_fifo_underrun_reporting(dev_priv, crtc->pipe, true);
9643
9644         if (crtc_state->has_pch_encoder) {
9645                 enum pipe pch_transcoder =
9646                         intel_crtc_pch_transcoder(crtc);
9647
9648                 intel_set_pch_fifo_underrun_reporting(dev_priv, pch_transcoder, true);
9649         }
9650 }
9651
9652 static void intel_pipe_fastset(const struct intel_crtc_state *old_crtc_state,
9653                                const struct intel_crtc_state *new_crtc_state)
9654 {
9655         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
9656         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9657
9658         /*
9659          * Update pipe size and adjust fitter if needed: the reason for this is
9660          * that in compute_mode_changes we check the native mode (not the pfit
9661          * mode) to see if we can flip rather than do a full mode set. In the
9662          * fastboot case, we'll flip, but if we don't update the pipesrc and
9663          * pfit state, we'll end up with a big fb scanned out into the wrong
9664          * sized surface.
9665          */
9666         intel_set_pipe_src_size(new_crtc_state);
9667
9668         /* on skylake this is done by detaching scalers */
9669         if (DISPLAY_VER(dev_priv) >= 9) {
9670                 if (new_crtc_state->pch_pfit.enabled)
9671                         skl_pfit_enable(new_crtc_state);
9672         } else if (HAS_PCH_SPLIT(dev_priv)) {
9673                 if (new_crtc_state->pch_pfit.enabled)
9674                         ilk_pfit_enable(new_crtc_state);
9675                 else if (old_crtc_state->pch_pfit.enabled)
9676                         ilk_pfit_disable(old_crtc_state);
9677         }
9678
9679         /*
9680          * The register is supposedly single buffered so perhaps
9681          * not 100% correct to do this here. But SKL+ calculate
9682          * this based on the adjust pixel rate so pfit changes do
9683          * affect it and so it must be updated for fastsets.
9684          * HSW/BDW only really need this here for fastboot, after
9685          * that the value should not change without a full modeset.
9686          */
9687         if (DISPLAY_VER(dev_priv) >= 9 ||
9688             IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
9689                 hsw_set_linetime_wm(new_crtc_state);
9690
9691         if (DISPLAY_VER(dev_priv) >= 11)
9692                 icl_set_pipe_chicken(new_crtc_state);
9693 }
9694
9695 static void commit_pipe_pre_planes(struct intel_atomic_state *state,
9696                                    struct intel_crtc *crtc)
9697 {
9698         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
9699         const struct intel_crtc_state *old_crtc_state =
9700                 intel_atomic_get_old_crtc_state(state, crtc);
9701         const struct intel_crtc_state *new_crtc_state =
9702                 intel_atomic_get_new_crtc_state(state, crtc);
9703         bool modeset = intel_crtc_needs_modeset(new_crtc_state);
9704
9705         /*
9706          * During modesets pipe configuration was programmed as the
9707          * CRTC was enabled.
9708          */
9709         if (!modeset) {
9710                 if (new_crtc_state->uapi.color_mgmt_changed ||
9711                     new_crtc_state->update_pipe)
9712                         intel_color_commit(new_crtc_state);
9713
9714                 if (DISPLAY_VER(dev_priv) >= 9 || IS_BROADWELL(dev_priv))
9715                         bdw_set_pipemisc(new_crtc_state);
9716
9717                 if (new_crtc_state->update_pipe)
9718                         intel_pipe_fastset(old_crtc_state, new_crtc_state);
9719         }
9720
9721         intel_psr2_program_trans_man_trk_ctl(new_crtc_state);
9722
9723         if (dev_priv->display.atomic_update_watermarks)
9724                 dev_priv->display.atomic_update_watermarks(state, crtc);
9725 }
9726
9727 static void commit_pipe_post_planes(struct intel_atomic_state *state,
9728                                     struct intel_crtc *crtc)
9729 {
9730         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
9731         const struct intel_crtc_state *new_crtc_state =
9732                 intel_atomic_get_new_crtc_state(state, crtc);
9733
9734         /*
9735          * Disable the scaler(s) after the plane(s) so that we don't
9736          * get a catastrophic underrun even if the two operations
9737          * end up happening in two different frames.
9738          */
9739         if (DISPLAY_VER(dev_priv) >= 9 &&
9740             !intel_crtc_needs_modeset(new_crtc_state))
9741                 skl_detach_scalers(new_crtc_state);
9742 }
9743
9744 static void intel_enable_crtc(struct intel_atomic_state *state,
9745                               struct intel_crtc *crtc)
9746 {
9747         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
9748         const struct intel_crtc_state *new_crtc_state =
9749                 intel_atomic_get_new_crtc_state(state, crtc);
9750
9751         if (!intel_crtc_needs_modeset(new_crtc_state))
9752                 return;
9753
9754         intel_crtc_update_active_timings(new_crtc_state);
9755
9756         dev_priv->display.crtc_enable(state, crtc);
9757
9758         if (new_crtc_state->bigjoiner_slave)
9759                 return;
9760
9761         /* vblanks work again, re-enable pipe CRC. */
9762         intel_crtc_enable_pipe_crc(crtc);
9763 }
9764
9765 static void intel_update_crtc(struct intel_atomic_state *state,
9766                               struct intel_crtc *crtc)
9767 {
9768         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
9769         const struct intel_crtc_state *old_crtc_state =
9770                 intel_atomic_get_old_crtc_state(state, crtc);
9771         struct intel_crtc_state *new_crtc_state =
9772                 intel_atomic_get_new_crtc_state(state, crtc);
9773         bool modeset = intel_crtc_needs_modeset(new_crtc_state);
9774
9775         if (!modeset) {
9776                 if (new_crtc_state->preload_luts &&
9777                     (new_crtc_state->uapi.color_mgmt_changed ||
9778                      new_crtc_state->update_pipe))
9779                         intel_color_load_luts(new_crtc_state);
9780
9781                 intel_pre_plane_update(state, crtc);
9782
9783                 if (new_crtc_state->update_pipe)
9784                         intel_encoders_update_pipe(state, crtc);
9785         }
9786
9787         intel_fbc_update(state, crtc);
9788
9789         /* Perform vblank evasion around commit operation */
9790         intel_pipe_update_start(new_crtc_state);
9791
9792         commit_pipe_pre_planes(state, crtc);
9793
9794         if (DISPLAY_VER(dev_priv) >= 9)
9795                 skl_update_planes_on_crtc(state, crtc);
9796         else
9797                 i9xx_update_planes_on_crtc(state, crtc);
9798
9799         commit_pipe_post_planes(state, crtc);
9800
9801         intel_pipe_update_end(new_crtc_state);
9802
9803         /*
9804          * We usually enable FIFO underrun interrupts as part of the
9805          * CRTC enable sequence during modesets.  But when we inherit a
9806          * valid pipe configuration from the BIOS we need to take care
9807          * of enabling them on the CRTC's first fastset.
9808          */
9809         if (new_crtc_state->update_pipe && !modeset &&
9810             old_crtc_state->inherited)
9811                 intel_crtc_arm_fifo_underrun(crtc, new_crtc_state);
9812 }
9813
9814 static void intel_old_crtc_state_disables(struct intel_atomic_state *state,
9815                                           struct intel_crtc_state *old_crtc_state,
9816                                           struct intel_crtc_state *new_crtc_state,
9817                                           struct intel_crtc *crtc)
9818 {
9819         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
9820
9821         drm_WARN_ON(&dev_priv->drm, old_crtc_state->bigjoiner_slave);
9822
9823         intel_encoders_pre_disable(state, crtc);
9824
9825         intel_crtc_disable_planes(state, crtc);
9826
9827         /*
9828          * We still need special handling for disabling bigjoiner master
9829          * and slaves since for slave we do not have encoder or plls
9830          * so we dont need to disable those.
9831          */
9832         if (old_crtc_state->bigjoiner) {
9833                 intel_crtc_disable_planes(state,
9834                                           old_crtc_state->bigjoiner_linked_crtc);
9835                 old_crtc_state->bigjoiner_linked_crtc->active = false;
9836         }
9837
9838         /*
9839          * We need to disable pipe CRC before disabling the pipe,
9840          * or we race against vblank off.
9841          */
9842         intel_crtc_disable_pipe_crc(crtc);
9843
9844         dev_priv->display.crtc_disable(state, crtc);
9845         crtc->active = false;
9846         intel_fbc_disable(crtc);
9847         intel_disable_shared_dpll(old_crtc_state);
9848
9849         /* FIXME unify this for all platforms */
9850         if (!new_crtc_state->hw.active &&
9851             !HAS_GMCH(dev_priv) &&
9852             dev_priv->display.initial_watermarks)
9853                 dev_priv->display.initial_watermarks(state, crtc);
9854 }
9855
9856 static void intel_commit_modeset_disables(struct intel_atomic_state *state)
9857 {
9858         struct intel_crtc_state *new_crtc_state, *old_crtc_state;
9859         struct intel_crtc *crtc;
9860         u32 handled = 0;
9861         int i;
9862
9863         /* Only disable port sync and MST slaves */
9864         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
9865                                             new_crtc_state, i) {
9866                 if (!intel_crtc_needs_modeset(new_crtc_state) || old_crtc_state->bigjoiner)
9867                         continue;
9868
9869                 if (!old_crtc_state->hw.active)
9870                         continue;
9871
9872                 /* In case of Transcoder port Sync master slave CRTCs can be
9873                  * assigned in any order and we need to make sure that
9874                  * slave CRTCs are disabled first and then master CRTC since
9875                  * Slave vblanks are masked till Master Vblanks.
9876                  */
9877                 if (!is_trans_port_sync_slave(old_crtc_state) &&
9878                     !intel_dp_mst_is_slave_trans(old_crtc_state))
9879                         continue;
9880
9881                 intel_pre_plane_update(state, crtc);
9882                 intel_old_crtc_state_disables(state, old_crtc_state,
9883                                               new_crtc_state, crtc);
9884                 handled |= BIT(crtc->pipe);
9885         }
9886
9887         /* Disable everything else left on */
9888         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
9889                                             new_crtc_state, i) {
9890                 if (!intel_crtc_needs_modeset(new_crtc_state) ||
9891                     (handled & BIT(crtc->pipe)) ||
9892                     old_crtc_state->bigjoiner_slave)
9893                         continue;
9894
9895                 intel_pre_plane_update(state, crtc);
9896                 if (old_crtc_state->bigjoiner) {
9897                         struct intel_crtc *slave =
9898                                 old_crtc_state->bigjoiner_linked_crtc;
9899
9900                         intel_pre_plane_update(state, slave);
9901                 }
9902
9903                 if (old_crtc_state->hw.active)
9904                         intel_old_crtc_state_disables(state, old_crtc_state,
9905                                                       new_crtc_state, crtc);
9906         }
9907 }
9908
9909 static void intel_commit_modeset_enables(struct intel_atomic_state *state)
9910 {
9911         struct intel_crtc_state *new_crtc_state;
9912         struct intel_crtc *crtc;
9913         int i;
9914
9915         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
9916                 if (!new_crtc_state->hw.active)
9917                         continue;
9918
9919                 intel_enable_crtc(state, crtc);
9920                 intel_update_crtc(state, crtc);
9921         }
9922 }
9923
9924 static void skl_commit_modeset_enables(struct intel_atomic_state *state)
9925 {
9926         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
9927         struct intel_crtc *crtc;
9928         struct intel_crtc_state *old_crtc_state, *new_crtc_state;
9929         struct skl_ddb_entry entries[I915_MAX_PIPES] = {};
9930         u8 update_pipes = 0, modeset_pipes = 0;
9931         int i;
9932
9933         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
9934                 enum pipe pipe = crtc->pipe;
9935
9936                 if (!new_crtc_state->hw.active)
9937                         continue;
9938
9939                 /* ignore allocations for crtc's that have been turned off. */
9940                 if (!intel_crtc_needs_modeset(new_crtc_state)) {
9941                         entries[pipe] = old_crtc_state->wm.skl.ddb;
9942                         update_pipes |= BIT(pipe);
9943                 } else {
9944                         modeset_pipes |= BIT(pipe);
9945                 }
9946         }
9947
9948         /*
9949          * Whenever the number of active pipes changes, we need to make sure we
9950          * update the pipes in the right order so that their ddb allocations
9951          * never overlap with each other between CRTC updates. Otherwise we'll
9952          * cause pipe underruns and other bad stuff.
9953          *
9954          * So first lets enable all pipes that do not need a fullmodeset as
9955          * those don't have any external dependency.
9956          */
9957         while (update_pipes) {
9958                 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
9959                                                     new_crtc_state, i) {
9960                         enum pipe pipe = crtc->pipe;
9961
9962                         if ((update_pipes & BIT(pipe)) == 0)
9963                                 continue;
9964
9965                         if (skl_ddb_allocation_overlaps(&new_crtc_state->wm.skl.ddb,
9966                                                         entries, I915_MAX_PIPES, pipe))
9967                                 continue;
9968
9969                         entries[pipe] = new_crtc_state->wm.skl.ddb;
9970                         update_pipes &= ~BIT(pipe);
9971
9972                         intel_update_crtc(state, crtc);
9973
9974                         /*
9975                          * If this is an already active pipe, it's DDB changed,
9976                          * and this isn't the last pipe that needs updating
9977                          * then we need to wait for a vblank to pass for the
9978                          * new ddb allocation to take effect.
9979                          */
9980                         if (!skl_ddb_entry_equal(&new_crtc_state->wm.skl.ddb,
9981                                                  &old_crtc_state->wm.skl.ddb) &&
9982                             (update_pipes | modeset_pipes))
9983                                 intel_wait_for_vblank(dev_priv, pipe);
9984                 }
9985         }
9986
9987         update_pipes = modeset_pipes;
9988
9989         /*
9990          * Enable all pipes that needs a modeset and do not depends on other
9991          * pipes
9992          */
9993         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
9994                 enum pipe pipe = crtc->pipe;
9995
9996                 if ((modeset_pipes & BIT(pipe)) == 0)
9997                         continue;
9998
9999                 if (intel_dp_mst_is_slave_trans(new_crtc_state) ||
10000                     is_trans_port_sync_master(new_crtc_state) ||
10001                     (new_crtc_state->bigjoiner && !new_crtc_state->bigjoiner_slave))
10002                         continue;
10003
10004                 modeset_pipes &= ~BIT(pipe);
10005
10006                 intel_enable_crtc(state, crtc);
10007         }
10008
10009         /*
10010          * Then we enable all remaining pipes that depend on other
10011          * pipes: MST slaves and port sync masters, big joiner master
10012          */
10013         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
10014                 enum pipe pipe = crtc->pipe;
10015
10016                 if ((modeset_pipes & BIT(pipe)) == 0)
10017                         continue;
10018
10019                 modeset_pipes &= ~BIT(pipe);
10020
10021                 intel_enable_crtc(state, crtc);
10022         }
10023
10024         /*
10025          * Finally we do the plane updates/etc. for all pipes that got enabled.
10026          */
10027         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
10028                 enum pipe pipe = crtc->pipe;
10029
10030                 if ((update_pipes & BIT(pipe)) == 0)
10031                         continue;
10032
10033                 drm_WARN_ON(&dev_priv->drm, skl_ddb_allocation_overlaps(&new_crtc_state->wm.skl.ddb,
10034                                                                         entries, I915_MAX_PIPES, pipe));
10035
10036                 entries[pipe] = new_crtc_state->wm.skl.ddb;
10037                 update_pipes &= ~BIT(pipe);
10038
10039                 intel_update_crtc(state, crtc);
10040         }
10041
10042         drm_WARN_ON(&dev_priv->drm, modeset_pipes);
10043         drm_WARN_ON(&dev_priv->drm, update_pipes);
10044 }
10045
10046 static void intel_atomic_helper_free_state(struct drm_i915_private *dev_priv)
10047 {
10048         struct intel_atomic_state *state, *next;
10049         struct llist_node *freed;
10050
10051         freed = llist_del_all(&dev_priv->atomic_helper.free_list);
10052         llist_for_each_entry_safe(state, next, freed, freed)
10053                 drm_atomic_state_put(&state->base);
10054 }
10055
10056 static void intel_atomic_helper_free_state_worker(struct work_struct *work)
10057 {
10058         struct drm_i915_private *dev_priv =
10059                 container_of(work, typeof(*dev_priv), atomic_helper.free_work);
10060
10061         intel_atomic_helper_free_state(dev_priv);
10062 }
10063
10064 static void intel_atomic_commit_fence_wait(struct intel_atomic_state *intel_state)
10065 {
10066         struct wait_queue_entry wait_fence, wait_reset;
10067         struct drm_i915_private *dev_priv = to_i915(intel_state->base.dev);
10068
10069         init_wait_entry(&wait_fence, 0);
10070         init_wait_entry(&wait_reset, 0);
10071         for (;;) {
10072                 prepare_to_wait(&intel_state->commit_ready.wait,
10073                                 &wait_fence, TASK_UNINTERRUPTIBLE);
10074                 prepare_to_wait(bit_waitqueue(&dev_priv->gt.reset.flags,
10075                                               I915_RESET_MODESET),
10076                                 &wait_reset, TASK_UNINTERRUPTIBLE);
10077
10078
10079                 if (i915_sw_fence_done(&intel_state->commit_ready) ||
10080                     test_bit(I915_RESET_MODESET, &dev_priv->gt.reset.flags))
10081                         break;
10082
10083                 schedule();
10084         }
10085         finish_wait(&intel_state->commit_ready.wait, &wait_fence);
10086         finish_wait(bit_waitqueue(&dev_priv->gt.reset.flags,
10087                                   I915_RESET_MODESET),
10088                     &wait_reset);
10089 }
10090
10091 static void intel_cleanup_dsbs(struct intel_atomic_state *state)
10092 {
10093         struct intel_crtc_state *old_crtc_state, *new_crtc_state;
10094         struct intel_crtc *crtc;
10095         int i;
10096
10097         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
10098                                             new_crtc_state, i)
10099                 intel_dsb_cleanup(old_crtc_state);
10100 }
10101
10102 static void intel_atomic_cleanup_work(struct work_struct *work)
10103 {
10104         struct intel_atomic_state *state =
10105                 container_of(work, struct intel_atomic_state, base.commit_work);
10106         struct drm_i915_private *i915 = to_i915(state->base.dev);
10107
10108         intel_cleanup_dsbs(state);
10109         drm_atomic_helper_cleanup_planes(&i915->drm, &state->base);
10110         drm_atomic_helper_commit_cleanup_done(&state->base);
10111         drm_atomic_state_put(&state->base);
10112
10113         intel_atomic_helper_free_state(i915);
10114 }
10115
10116 static void intel_atomic_prepare_plane_clear_colors(struct intel_atomic_state *state)
10117 {
10118         struct drm_i915_private *i915 = to_i915(state->base.dev);
10119         struct intel_plane *plane;
10120         struct intel_plane_state *plane_state;
10121         int i;
10122
10123         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
10124                 struct drm_framebuffer *fb = plane_state->hw.fb;
10125                 int ret;
10126
10127                 if (!fb ||
10128                     fb->modifier != I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS_CC)
10129                         continue;
10130
10131                 /*
10132                  * The layout of the fast clear color value expected by HW
10133                  * (the DRM ABI requiring this value to be located in fb at offset 0 of plane#2):
10134                  * - 4 x 4 bytes per-channel value
10135                  *   (in surface type specific float/int format provided by the fb user)
10136                  * - 8 bytes native color value used by the display
10137                  *   (converted/written by GPU during a fast clear operation using the
10138                  *    above per-channel values)
10139                  *
10140                  * The commit's FB prepare hook already ensured that FB obj is pinned and the
10141                  * caller made sure that the object is synced wrt. the related color clear value
10142                  * GPU write on it.
10143                  */
10144                 ret = i915_gem_object_read_from_page(intel_fb_obj(fb),
10145                                                      fb->offsets[2] + 16,
10146                                                      &plane_state->ccval,
10147                                                      sizeof(plane_state->ccval));
10148                 /* The above could only fail if the FB obj has an unexpected backing store type. */
10149                 drm_WARN_ON(&i915->drm, ret);
10150         }
10151 }
10152
10153 static void intel_atomic_commit_tail(struct intel_atomic_state *state)
10154 {
10155         struct drm_device *dev = state->base.dev;
10156         struct drm_i915_private *dev_priv = to_i915(dev);
10157         struct intel_crtc_state *new_crtc_state, *old_crtc_state;
10158         struct intel_crtc *crtc;
10159         u64 put_domains[I915_MAX_PIPES] = {};
10160         intel_wakeref_t wakeref = 0;
10161         int i;
10162
10163         intel_atomic_commit_fence_wait(state);
10164
10165         drm_atomic_helper_wait_for_dependencies(&state->base);
10166
10167         if (state->modeset)
10168                 wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_MODESET);
10169
10170         intel_atomic_prepare_plane_clear_colors(state);
10171
10172         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
10173                                             new_crtc_state, i) {
10174                 if (intel_crtc_needs_modeset(new_crtc_state) ||
10175                     new_crtc_state->update_pipe) {
10176
10177                         put_domains[crtc->pipe] =
10178                                 modeset_get_crtc_power_domains(new_crtc_state);
10179                 }
10180         }
10181
10182         intel_commit_modeset_disables(state);
10183
10184         /* FIXME: Eventually get rid of our crtc->config pointer */
10185         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i)
10186                 crtc->config = new_crtc_state;
10187
10188         if (state->modeset) {
10189                 drm_atomic_helper_update_legacy_modeset_state(dev, &state->base);
10190
10191                 intel_set_cdclk_pre_plane_update(state);
10192
10193                 intel_modeset_verify_disabled(dev_priv, state);
10194         }
10195
10196         intel_sagv_pre_plane_update(state);
10197
10198         /* Complete the events for pipes that have now been disabled */
10199         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
10200                 bool modeset = intel_crtc_needs_modeset(new_crtc_state);
10201
10202                 /* Complete events for now disable pipes here. */
10203                 if (modeset && !new_crtc_state->hw.active && new_crtc_state->uapi.event) {
10204                         spin_lock_irq(&dev->event_lock);
10205                         drm_crtc_send_vblank_event(&crtc->base,
10206                                                    new_crtc_state->uapi.event);
10207                         spin_unlock_irq(&dev->event_lock);
10208
10209                         new_crtc_state->uapi.event = NULL;
10210                 }
10211         }
10212
10213         if (state->modeset)
10214                 intel_encoders_update_prepare(state);
10215
10216         intel_dbuf_pre_plane_update(state);
10217         intel_psr_pre_plane_update(state);
10218
10219         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
10220                 if (new_crtc_state->uapi.async_flip)
10221                         intel_crtc_enable_flip_done(state, crtc);
10222         }
10223
10224         /* Now enable the clocks, plane, pipe, and connectors that we set up. */
10225         dev_priv->display.commit_modeset_enables(state);
10226
10227         if (state->modeset) {
10228                 intel_encoders_update_complete(state);
10229
10230                 intel_set_cdclk_post_plane_update(state);
10231         }
10232
10233         /* FIXME: We should call drm_atomic_helper_commit_hw_done() here
10234          * already, but still need the state for the delayed optimization. To
10235          * fix this:
10236          * - wrap the optimization/post_plane_update stuff into a per-crtc work.
10237          * - schedule that vblank worker _before_ calling hw_done
10238          * - at the start of commit_tail, cancel it _synchrously
10239          * - switch over to the vblank wait helper in the core after that since
10240          *   we don't need out special handling any more.
10241          */
10242         drm_atomic_helper_wait_for_flip_done(dev, &state->base);
10243
10244         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
10245                 if (new_crtc_state->uapi.async_flip)
10246                         intel_crtc_disable_flip_done(state, crtc);
10247
10248                 if (new_crtc_state->hw.active &&
10249                     !intel_crtc_needs_modeset(new_crtc_state) &&
10250                     !new_crtc_state->preload_luts &&
10251                     (new_crtc_state->uapi.color_mgmt_changed ||
10252                      new_crtc_state->update_pipe))
10253                         intel_color_load_luts(new_crtc_state);
10254         }
10255
10256         /*
10257          * Now that the vblank has passed, we can go ahead and program the
10258          * optimal watermarks on platforms that need two-step watermark
10259          * programming.
10260          *
10261          * TODO: Move this (and other cleanup) to an async worker eventually.
10262          */
10263         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
10264                                             new_crtc_state, i) {
10265                 /*
10266                  * Gen2 reports pipe underruns whenever all planes are disabled.
10267                  * So re-enable underrun reporting after some planes get enabled.
10268                  *
10269                  * We do this before .optimize_watermarks() so that we have a
10270                  * chance of catching underruns with the intermediate watermarks
10271                  * vs. the new plane configuration.
10272                  */
10273                 if (DISPLAY_VER(dev_priv) == 2 && planes_enabling(old_crtc_state, new_crtc_state))
10274                         intel_set_cpu_fifo_underrun_reporting(dev_priv, crtc->pipe, true);
10275
10276                 if (dev_priv->display.optimize_watermarks)
10277                         dev_priv->display.optimize_watermarks(state, crtc);
10278         }
10279
10280         intel_dbuf_post_plane_update(state);
10281         intel_psr_post_plane_update(state);
10282
10283         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
10284                 intel_post_plane_update(state, crtc);
10285
10286                 modeset_put_crtc_power_domains(crtc, put_domains[crtc->pipe]);
10287
10288                 intel_modeset_verify_crtc(crtc, state, old_crtc_state, new_crtc_state);
10289
10290                 /*
10291                  * DSB cleanup is done in cleanup_work aligning with framebuffer
10292                  * cleanup. So copy and reset the dsb structure to sync with
10293                  * commit_done and later do dsb cleanup in cleanup_work.
10294                  */
10295                 old_crtc_state->dsb = fetch_and_zero(&new_crtc_state->dsb);
10296         }
10297
10298         /* Underruns don't always raise interrupts, so check manually */
10299         intel_check_cpu_fifo_underruns(dev_priv);
10300         intel_check_pch_fifo_underruns(dev_priv);
10301
10302         if (state->modeset)
10303                 intel_verify_planes(state);
10304
10305         intel_sagv_post_plane_update(state);
10306
10307         drm_atomic_helper_commit_hw_done(&state->base);
10308
10309         if (state->modeset) {
10310                 /* As one of the primary mmio accessors, KMS has a high
10311                  * likelihood of triggering bugs in unclaimed access. After we
10312                  * finish modesetting, see if an error has been flagged, and if
10313                  * so enable debugging for the next modeset - and hope we catch
10314                  * the culprit.
10315                  */
10316                 intel_uncore_arm_unclaimed_mmio_detection(&dev_priv->uncore);
10317                 intel_display_power_put(dev_priv, POWER_DOMAIN_MODESET, wakeref);
10318         }
10319         intel_runtime_pm_put(&dev_priv->runtime_pm, state->wakeref);
10320
10321         /*
10322          * Defer the cleanup of the old state to a separate worker to not
10323          * impede the current task (userspace for blocking modesets) that
10324          * are executed inline. For out-of-line asynchronous modesets/flips,
10325          * deferring to a new worker seems overkill, but we would place a
10326          * schedule point (cond_resched()) here anyway to keep latencies
10327          * down.
10328          */
10329         INIT_WORK(&state->base.commit_work, intel_atomic_cleanup_work);
10330         queue_work(system_highpri_wq, &state->base.commit_work);
10331 }
10332
10333 static void intel_atomic_commit_work(struct work_struct *work)
10334 {
10335         struct intel_atomic_state *state =
10336                 container_of(work, struct intel_atomic_state, base.commit_work);
10337
10338         intel_atomic_commit_tail(state);
10339 }
10340
10341 static int __i915_sw_fence_call
10342 intel_atomic_commit_ready(struct i915_sw_fence *fence,
10343                           enum i915_sw_fence_notify notify)
10344 {
10345         struct intel_atomic_state *state =
10346                 container_of(fence, struct intel_atomic_state, commit_ready);
10347
10348         switch (notify) {
10349         case FENCE_COMPLETE:
10350                 /* we do blocking waits in the worker, nothing to do here */
10351                 break;
10352         case FENCE_FREE:
10353                 {
10354                         struct intel_atomic_helper *helper =
10355                                 &to_i915(state->base.dev)->atomic_helper;
10356
10357                         if (llist_add(&state->freed, &helper->free_list))
10358                                 schedule_work(&helper->free_work);
10359                         break;
10360                 }
10361         }
10362
10363         return NOTIFY_DONE;
10364 }
10365
10366 static void intel_atomic_track_fbs(struct intel_atomic_state *state)
10367 {
10368         struct intel_plane_state *old_plane_state, *new_plane_state;
10369         struct intel_plane *plane;
10370         int i;
10371
10372         for_each_oldnew_intel_plane_in_state(state, plane, old_plane_state,
10373                                              new_plane_state, i)
10374                 intel_frontbuffer_track(to_intel_frontbuffer(old_plane_state->hw.fb),
10375                                         to_intel_frontbuffer(new_plane_state->hw.fb),
10376                                         plane->frontbuffer_bit);
10377 }
10378
10379 static int intel_atomic_commit(struct drm_device *dev,
10380                                struct drm_atomic_state *_state,
10381                                bool nonblock)
10382 {
10383         struct intel_atomic_state *state = to_intel_atomic_state(_state);
10384         struct drm_i915_private *dev_priv = to_i915(dev);
10385         int ret = 0;
10386
10387         state->wakeref = intel_runtime_pm_get(&dev_priv->runtime_pm);
10388
10389         drm_atomic_state_get(&state->base);
10390         i915_sw_fence_init(&state->commit_ready,
10391                            intel_atomic_commit_ready);
10392
10393         /*
10394          * The intel_legacy_cursor_update() fast path takes care
10395          * of avoiding the vblank waits for simple cursor
10396          * movement and flips. For cursor on/off and size changes,
10397          * we want to perform the vblank waits so that watermark
10398          * updates happen during the correct frames. Gen9+ have
10399          * double buffered watermarks and so shouldn't need this.
10400          *
10401          * Unset state->legacy_cursor_update before the call to
10402          * drm_atomic_helper_setup_commit() because otherwise
10403          * drm_atomic_helper_wait_for_flip_done() is a noop and
10404          * we get FIFO underruns because we didn't wait
10405          * for vblank.
10406          *
10407          * FIXME doing watermarks and fb cleanup from a vblank worker
10408          * (assuming we had any) would solve these problems.
10409          */
10410         if (DISPLAY_VER(dev_priv) < 9 && state->base.legacy_cursor_update) {
10411                 struct intel_crtc_state *new_crtc_state;
10412                 struct intel_crtc *crtc;
10413                 int i;
10414
10415                 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i)
10416                         if (new_crtc_state->wm.need_postvbl_update ||
10417                             new_crtc_state->update_wm_post)
10418                                 state->base.legacy_cursor_update = false;
10419         }
10420
10421         ret = intel_atomic_prepare_commit(state);
10422         if (ret) {
10423                 drm_dbg_atomic(&dev_priv->drm,
10424                                "Preparing state failed with %i\n", ret);
10425                 i915_sw_fence_commit(&state->commit_ready);
10426                 intel_runtime_pm_put(&dev_priv->runtime_pm, state->wakeref);
10427                 return ret;
10428         }
10429
10430         ret = drm_atomic_helper_setup_commit(&state->base, nonblock);
10431         if (!ret)
10432                 ret = drm_atomic_helper_swap_state(&state->base, true);
10433         if (!ret)
10434                 intel_atomic_swap_global_state(state);
10435
10436         if (ret) {
10437                 struct intel_crtc_state *new_crtc_state;
10438                 struct intel_crtc *crtc;
10439                 int i;
10440
10441                 i915_sw_fence_commit(&state->commit_ready);
10442
10443                 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i)
10444                         intel_dsb_cleanup(new_crtc_state);
10445
10446                 drm_atomic_helper_cleanup_planes(dev, &state->base);
10447                 intel_runtime_pm_put(&dev_priv->runtime_pm, state->wakeref);
10448                 return ret;
10449         }
10450         intel_shared_dpll_swap_state(state);
10451         intel_atomic_track_fbs(state);
10452
10453         drm_atomic_state_get(&state->base);
10454         INIT_WORK(&state->base.commit_work, intel_atomic_commit_work);
10455
10456         i915_sw_fence_commit(&state->commit_ready);
10457         if (nonblock && state->modeset) {
10458                 queue_work(dev_priv->modeset_wq, &state->base.commit_work);
10459         } else if (nonblock) {
10460                 queue_work(dev_priv->flip_wq, &state->base.commit_work);
10461         } else {
10462                 if (state->modeset)
10463                         flush_workqueue(dev_priv->modeset_wq);
10464                 intel_atomic_commit_tail(state);
10465         }
10466
10467         return 0;
10468 }
10469
10470 struct wait_rps_boost {
10471         struct wait_queue_entry wait;
10472
10473         struct drm_crtc *crtc;
10474         struct i915_request *request;
10475 };
10476
10477 static int do_rps_boost(struct wait_queue_entry *_wait,
10478                         unsigned mode, int sync, void *key)
10479 {
10480         struct wait_rps_boost *wait = container_of(_wait, typeof(*wait), wait);
10481         struct i915_request *rq = wait->request;
10482
10483         /*
10484          * If we missed the vblank, but the request is already running it
10485          * is reasonable to assume that it will complete before the next
10486          * vblank without our intervention, so leave RPS alone.
10487          */
10488         if (!i915_request_started(rq))
10489                 intel_rps_boost(rq);
10490         i915_request_put(rq);
10491
10492         drm_crtc_vblank_put(wait->crtc);
10493
10494         list_del(&wait->wait.entry);
10495         kfree(wait);
10496         return 1;
10497 }
10498
10499 static void add_rps_boost_after_vblank(struct drm_crtc *crtc,
10500                                        struct dma_fence *fence)
10501 {
10502         struct wait_rps_boost *wait;
10503
10504         if (!dma_fence_is_i915(fence))
10505                 return;
10506
10507         if (DISPLAY_VER(to_i915(crtc->dev)) < 6)
10508                 return;
10509
10510         if (drm_crtc_vblank_get(crtc))
10511                 return;
10512
10513         wait = kmalloc(sizeof(*wait), GFP_KERNEL);
10514         if (!wait) {
10515                 drm_crtc_vblank_put(crtc);
10516                 return;
10517         }
10518
10519         wait->request = to_request(dma_fence_get(fence));
10520         wait->crtc = crtc;
10521
10522         wait->wait.func = do_rps_boost;
10523         wait->wait.flags = 0;
10524
10525         add_wait_queue(drm_crtc_vblank_waitqueue(crtc), &wait->wait);
10526 }
10527
10528 int intel_plane_pin_fb(struct intel_plane_state *plane_state)
10529 {
10530         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
10531         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
10532         struct drm_framebuffer *fb = plane_state->hw.fb;
10533         struct i915_vma *vma;
10534         bool phys_cursor =
10535                 plane->id == PLANE_CURSOR &&
10536                 INTEL_INFO(dev_priv)->display.cursor_needs_physical;
10537
10538         if (!intel_fb_uses_dpt(fb)) {
10539                 vma = intel_pin_and_fence_fb_obj(fb, phys_cursor,
10540                                                  &plane_state->view.gtt,
10541                                                  intel_plane_uses_fence(plane_state),
10542                                                  &plane_state->flags);
10543                 if (IS_ERR(vma))
10544                         return PTR_ERR(vma);
10545
10546                 plane_state->ggtt_vma = vma;
10547         } else {
10548                 struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
10549
10550                 vma = intel_dpt_pin(intel_fb->dpt_vm);
10551                 if (IS_ERR(vma))
10552                         return PTR_ERR(vma);
10553
10554                 plane_state->ggtt_vma = vma;
10555
10556                 vma = intel_pin_fb_obj_dpt(fb, &plane_state->view.gtt, false,
10557                                            &plane_state->flags, intel_fb->dpt_vm);
10558                 if (IS_ERR(vma)) {
10559                         intel_dpt_unpin(intel_fb->dpt_vm);
10560                         plane_state->ggtt_vma = NULL;
10561                         return PTR_ERR(vma);
10562                 }
10563
10564                 plane_state->dpt_vma = vma;
10565
10566                 WARN_ON(plane_state->ggtt_vma == plane_state->dpt_vma);
10567         }
10568
10569         return 0;
10570 }
10571
10572 void intel_plane_unpin_fb(struct intel_plane_state *old_plane_state)
10573 {
10574         struct drm_framebuffer *fb = old_plane_state->hw.fb;
10575         struct i915_vma *vma;
10576
10577         if (!intel_fb_uses_dpt(fb)) {
10578                 vma = fetch_and_zero(&old_plane_state->ggtt_vma);
10579                 if (vma)
10580                         intel_unpin_fb_vma(vma, old_plane_state->flags);
10581         } else {
10582                 struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
10583
10584                 vma = fetch_and_zero(&old_plane_state->dpt_vma);
10585                 if (vma)
10586                         intel_unpin_fb_vma(vma, old_plane_state->flags);
10587
10588                 vma = fetch_and_zero(&old_plane_state->ggtt_vma);
10589                 if (vma)
10590                         intel_dpt_unpin(intel_fb->dpt_vm);
10591         }
10592 }
10593
10594 /**
10595  * intel_prepare_plane_fb - Prepare fb for usage on plane
10596  * @_plane: drm plane to prepare for
10597  * @_new_plane_state: the plane state being prepared
10598  *
10599  * Prepares a framebuffer for usage on a display plane.  Generally this
10600  * involves pinning the underlying object and updating the frontbuffer tracking
10601  * bits.  Some older platforms need special physical address handling for
10602  * cursor planes.
10603  *
10604  * Returns 0 on success, negative error code on failure.
10605  */
10606 int
10607 intel_prepare_plane_fb(struct drm_plane *_plane,
10608                        struct drm_plane_state *_new_plane_state)
10609 {
10610         struct i915_sched_attr attr = { .priority = I915_PRIORITY_DISPLAY };
10611         struct intel_plane *plane = to_intel_plane(_plane);
10612         struct intel_plane_state *new_plane_state =
10613                 to_intel_plane_state(_new_plane_state);
10614         struct intel_atomic_state *state =
10615                 to_intel_atomic_state(new_plane_state->uapi.state);
10616         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
10617         const struct intel_plane_state *old_plane_state =
10618                 intel_atomic_get_old_plane_state(state, plane);
10619         struct drm_i915_gem_object *obj = intel_fb_obj(new_plane_state->hw.fb);
10620         struct drm_i915_gem_object *old_obj = intel_fb_obj(old_plane_state->hw.fb);
10621         int ret;
10622
10623         if (old_obj) {
10624                 const struct intel_crtc_state *crtc_state =
10625                         intel_atomic_get_new_crtc_state(state,
10626                                                         to_intel_crtc(old_plane_state->hw.crtc));
10627
10628                 /* Big Hammer, we also need to ensure that any pending
10629                  * MI_WAIT_FOR_EVENT inside a user batch buffer on the
10630                  * current scanout is retired before unpinning the old
10631                  * framebuffer. Note that we rely on userspace rendering
10632                  * into the buffer attached to the pipe they are waiting
10633                  * on. If not, userspace generates a GPU hang with IPEHR
10634                  * point to the MI_WAIT_FOR_EVENT.
10635                  *
10636                  * This should only fail upon a hung GPU, in which case we
10637                  * can safely continue.
10638                  */
10639                 if (intel_crtc_needs_modeset(crtc_state)) {
10640                         ret = i915_sw_fence_await_reservation(&state->commit_ready,
10641                                                               old_obj->base.resv, NULL,
10642                                                               false, 0,
10643                                                               GFP_KERNEL);
10644                         if (ret < 0)
10645                                 return ret;
10646                 }
10647         }
10648
10649         if (new_plane_state->uapi.fence) { /* explicit fencing */
10650                 i915_gem_fence_wait_priority(new_plane_state->uapi.fence,
10651                                              &attr);
10652                 ret = i915_sw_fence_await_dma_fence(&state->commit_ready,
10653                                                     new_plane_state->uapi.fence,
10654                                                     i915_fence_timeout(dev_priv),
10655                                                     GFP_KERNEL);
10656                 if (ret < 0)
10657                         return ret;
10658         }
10659
10660         if (!obj)
10661                 return 0;
10662
10663
10664         ret = intel_plane_pin_fb(new_plane_state);
10665         if (ret)
10666                 return ret;
10667
10668         i915_gem_object_wait_priority(obj, 0, &attr);
10669         i915_gem_object_flush_frontbuffer(obj, ORIGIN_DIRTYFB);
10670
10671         if (!new_plane_state->uapi.fence) { /* implicit fencing */
10672                 struct dma_fence *fence;
10673
10674                 ret = i915_sw_fence_await_reservation(&state->commit_ready,
10675                                                       obj->base.resv, NULL,
10676                                                       false,
10677                                                       i915_fence_timeout(dev_priv),
10678                                                       GFP_KERNEL);
10679                 if (ret < 0)
10680                         goto unpin_fb;
10681
10682                 fence = dma_resv_get_excl_unlocked(obj->base.resv);
10683                 if (fence) {
10684                         add_rps_boost_after_vblank(new_plane_state->hw.crtc,
10685                                                    fence);
10686                         dma_fence_put(fence);
10687                 }
10688         } else {
10689                 add_rps_boost_after_vblank(new_plane_state->hw.crtc,
10690                                            new_plane_state->uapi.fence);
10691         }
10692
10693         /*
10694          * We declare pageflips to be interactive and so merit a small bias
10695          * towards upclocking to deliver the frame on time. By only changing
10696          * the RPS thresholds to sample more regularly and aim for higher
10697          * clocks we can hopefully deliver low power workloads (like kodi)
10698          * that are not quite steady state without resorting to forcing
10699          * maximum clocks following a vblank miss (see do_rps_boost()).
10700          */
10701         if (!state->rps_interactive) {
10702                 intel_rps_mark_interactive(&dev_priv->gt.rps, true);
10703                 state->rps_interactive = true;
10704         }
10705
10706         return 0;
10707
10708 unpin_fb:
10709         intel_plane_unpin_fb(new_plane_state);
10710
10711         return ret;
10712 }
10713
10714 /**
10715  * intel_cleanup_plane_fb - Cleans up an fb after plane use
10716  * @plane: drm plane to clean up for
10717  * @_old_plane_state: the state from the previous modeset
10718  *
10719  * Cleans up a framebuffer that has just been removed from a plane.
10720  */
10721 void
10722 intel_cleanup_plane_fb(struct drm_plane *plane,
10723                        struct drm_plane_state *_old_plane_state)
10724 {
10725         struct intel_plane_state *old_plane_state =
10726                 to_intel_plane_state(_old_plane_state);
10727         struct intel_atomic_state *state =
10728                 to_intel_atomic_state(old_plane_state->uapi.state);
10729         struct drm_i915_private *dev_priv = to_i915(plane->dev);
10730         struct drm_i915_gem_object *obj = intel_fb_obj(old_plane_state->hw.fb);
10731
10732         if (!obj)
10733                 return;
10734
10735         if (state->rps_interactive) {
10736                 intel_rps_mark_interactive(&dev_priv->gt.rps, false);
10737                 state->rps_interactive = false;
10738         }
10739
10740         /* Should only be called after a successful intel_prepare_plane_fb()! */
10741         intel_plane_unpin_fb(old_plane_state);
10742 }
10743
10744 /**
10745  * intel_plane_destroy - destroy a plane
10746  * @plane: plane to destroy
10747  *
10748  * Common destruction function for all types of planes (primary, cursor,
10749  * sprite).
10750  */
10751 void intel_plane_destroy(struct drm_plane *plane)
10752 {
10753         drm_plane_cleanup(plane);
10754         kfree(to_intel_plane(plane));
10755 }
10756
10757 static void intel_plane_possible_crtcs_init(struct drm_i915_private *dev_priv)
10758 {
10759         struct intel_plane *plane;
10760
10761         for_each_intel_plane(&dev_priv->drm, plane) {
10762                 struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv,
10763                                                                   plane->pipe);
10764
10765                 plane->base.possible_crtcs = drm_crtc_mask(&crtc->base);
10766         }
10767 }
10768
10769
10770 int intel_get_pipe_from_crtc_id_ioctl(struct drm_device *dev, void *data,
10771                                       struct drm_file *file)
10772 {
10773         struct drm_i915_get_pipe_from_crtc_id *pipe_from_crtc_id = data;
10774         struct drm_crtc *drmmode_crtc;
10775         struct intel_crtc *crtc;
10776
10777         drmmode_crtc = drm_crtc_find(dev, file, pipe_from_crtc_id->crtc_id);
10778         if (!drmmode_crtc)
10779                 return -ENOENT;
10780
10781         crtc = to_intel_crtc(drmmode_crtc);
10782         pipe_from_crtc_id->pipe = crtc->pipe;
10783
10784         return 0;
10785 }
10786
10787 static u32 intel_encoder_possible_clones(struct intel_encoder *encoder)
10788 {
10789         struct drm_device *dev = encoder->base.dev;
10790         struct intel_encoder *source_encoder;
10791         u32 possible_clones = 0;
10792
10793         for_each_intel_encoder(dev, source_encoder) {
10794                 if (encoders_cloneable(encoder, source_encoder))
10795                         possible_clones |= drm_encoder_mask(&source_encoder->base);
10796         }
10797
10798         return possible_clones;
10799 }
10800
10801 static u32 intel_encoder_possible_crtcs(struct intel_encoder *encoder)
10802 {
10803         struct drm_device *dev = encoder->base.dev;
10804         struct intel_crtc *crtc;
10805         u32 possible_crtcs = 0;
10806
10807         for_each_intel_crtc(dev, crtc) {
10808                 if (encoder->pipe_mask & BIT(crtc->pipe))
10809                         possible_crtcs |= drm_crtc_mask(&crtc->base);
10810         }
10811
10812         return possible_crtcs;
10813 }
10814
10815 static bool ilk_has_edp_a(struct drm_i915_private *dev_priv)
10816 {
10817         if (!IS_MOBILE(dev_priv))
10818                 return false;
10819
10820         if ((intel_de_read(dev_priv, DP_A) & DP_DETECTED) == 0)
10821                 return false;
10822
10823         if (IS_IRONLAKE(dev_priv) && (intel_de_read(dev_priv, FUSE_STRAP) & ILK_eDP_A_DISABLE))
10824                 return false;
10825
10826         return true;
10827 }
10828
10829 static bool intel_ddi_crt_present(struct drm_i915_private *dev_priv)
10830 {
10831         if (DISPLAY_VER(dev_priv) >= 9)
10832                 return false;
10833
10834         if (IS_HSW_ULT(dev_priv) || IS_BDW_ULT(dev_priv))
10835                 return false;
10836
10837         if (HAS_PCH_LPT_H(dev_priv) &&
10838             intel_de_read(dev_priv, SFUSE_STRAP) & SFUSE_STRAP_CRT_DISABLED)
10839                 return false;
10840
10841         /* DDI E can't be used if DDI A requires 4 lanes */
10842         if (intel_de_read(dev_priv, DDI_BUF_CTL(PORT_A)) & DDI_A_4_LANES)
10843                 return false;
10844
10845         if (!dev_priv->vbt.int_crt_support)
10846                 return false;
10847
10848         return true;
10849 }
10850
10851 static void intel_setup_outputs(struct drm_i915_private *dev_priv)
10852 {
10853         struct intel_encoder *encoder;
10854         bool dpd_is_edp = false;
10855
10856         intel_pps_unlock_regs_wa(dev_priv);
10857
10858         if (!HAS_DISPLAY(dev_priv))
10859                 return;
10860
10861         if (IS_DG2(dev_priv)) {
10862                 intel_ddi_init(dev_priv, PORT_A);
10863                 intel_ddi_init(dev_priv, PORT_B);
10864                 intel_ddi_init(dev_priv, PORT_C);
10865                 intel_ddi_init(dev_priv, PORT_D_XELPD);
10866         } else if (IS_ALDERLAKE_P(dev_priv)) {
10867                 intel_ddi_init(dev_priv, PORT_A);
10868                 intel_ddi_init(dev_priv, PORT_B);
10869                 intel_ddi_init(dev_priv, PORT_TC1);
10870                 intel_ddi_init(dev_priv, PORT_TC2);
10871                 intel_ddi_init(dev_priv, PORT_TC3);
10872                 intel_ddi_init(dev_priv, PORT_TC4);
10873                 icl_dsi_init(dev_priv);
10874         } else if (IS_ALDERLAKE_S(dev_priv)) {
10875                 intel_ddi_init(dev_priv, PORT_A);
10876                 intel_ddi_init(dev_priv, PORT_TC1);
10877                 intel_ddi_init(dev_priv, PORT_TC2);
10878                 intel_ddi_init(dev_priv, PORT_TC3);
10879                 intel_ddi_init(dev_priv, PORT_TC4);
10880         } else if (IS_DG1(dev_priv) || IS_ROCKETLAKE(dev_priv)) {
10881                 intel_ddi_init(dev_priv, PORT_A);
10882                 intel_ddi_init(dev_priv, PORT_B);
10883                 intel_ddi_init(dev_priv, PORT_TC1);
10884                 intel_ddi_init(dev_priv, PORT_TC2);
10885         } else if (DISPLAY_VER(dev_priv) >= 12) {
10886                 intel_ddi_init(dev_priv, PORT_A);
10887                 intel_ddi_init(dev_priv, PORT_B);
10888                 intel_ddi_init(dev_priv, PORT_TC1);
10889                 intel_ddi_init(dev_priv, PORT_TC2);
10890                 intel_ddi_init(dev_priv, PORT_TC3);
10891                 intel_ddi_init(dev_priv, PORT_TC4);
10892                 intel_ddi_init(dev_priv, PORT_TC5);
10893                 intel_ddi_init(dev_priv, PORT_TC6);
10894                 icl_dsi_init(dev_priv);
10895         } else if (IS_JSL_EHL(dev_priv)) {
10896                 intel_ddi_init(dev_priv, PORT_A);
10897                 intel_ddi_init(dev_priv, PORT_B);
10898                 intel_ddi_init(dev_priv, PORT_C);
10899                 intel_ddi_init(dev_priv, PORT_D);
10900                 icl_dsi_init(dev_priv);
10901         } else if (DISPLAY_VER(dev_priv) == 11) {
10902                 intel_ddi_init(dev_priv, PORT_A);
10903                 intel_ddi_init(dev_priv, PORT_B);
10904                 intel_ddi_init(dev_priv, PORT_C);
10905                 intel_ddi_init(dev_priv, PORT_D);
10906                 intel_ddi_init(dev_priv, PORT_E);
10907                 intel_ddi_init(dev_priv, PORT_F);
10908                 icl_dsi_init(dev_priv);
10909         } else if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) {
10910                 intel_ddi_init(dev_priv, PORT_A);
10911                 intel_ddi_init(dev_priv, PORT_B);
10912                 intel_ddi_init(dev_priv, PORT_C);
10913                 vlv_dsi_init(dev_priv);
10914         } else if (DISPLAY_VER(dev_priv) >= 9) {
10915                 intel_ddi_init(dev_priv, PORT_A);
10916                 intel_ddi_init(dev_priv, PORT_B);
10917                 intel_ddi_init(dev_priv, PORT_C);
10918                 intel_ddi_init(dev_priv, PORT_D);
10919                 intel_ddi_init(dev_priv, PORT_E);
10920         } else if (HAS_DDI(dev_priv)) {
10921                 u32 found;
10922
10923                 if (intel_ddi_crt_present(dev_priv))
10924                         intel_crt_init(dev_priv);
10925
10926                 /* Haswell uses DDI functions to detect digital outputs. */
10927                 found = intel_de_read(dev_priv, DDI_BUF_CTL(PORT_A)) & DDI_INIT_DISPLAY_DETECTED;
10928                 if (found)
10929                         intel_ddi_init(dev_priv, PORT_A);
10930
10931                 found = intel_de_read(dev_priv, SFUSE_STRAP);
10932                 if (found & SFUSE_STRAP_DDIB_DETECTED)
10933                         intel_ddi_init(dev_priv, PORT_B);
10934                 if (found & SFUSE_STRAP_DDIC_DETECTED)
10935                         intel_ddi_init(dev_priv, PORT_C);
10936                 if (found & SFUSE_STRAP_DDID_DETECTED)
10937                         intel_ddi_init(dev_priv, PORT_D);
10938                 if (found & SFUSE_STRAP_DDIF_DETECTED)
10939                         intel_ddi_init(dev_priv, PORT_F);
10940         } else if (HAS_PCH_SPLIT(dev_priv)) {
10941                 int found;
10942
10943                 /*
10944                  * intel_edp_init_connector() depends on this completing first,
10945                  * to prevent the registration of both eDP and LVDS and the
10946                  * incorrect sharing of the PPS.
10947                  */
10948                 intel_lvds_init(dev_priv);
10949                 intel_crt_init(dev_priv);
10950
10951                 dpd_is_edp = intel_dp_is_port_edp(dev_priv, PORT_D);
10952
10953                 if (ilk_has_edp_a(dev_priv))
10954                         g4x_dp_init(dev_priv, DP_A, PORT_A);
10955
10956                 if (intel_de_read(dev_priv, PCH_HDMIB) & SDVO_DETECTED) {
10957                         /* PCH SDVOB multiplex with HDMIB */
10958                         found = intel_sdvo_init(dev_priv, PCH_SDVOB, PORT_B);
10959                         if (!found)
10960                                 g4x_hdmi_init(dev_priv, PCH_HDMIB, PORT_B);
10961                         if (!found && (intel_de_read(dev_priv, PCH_DP_B) & DP_DETECTED))
10962                                 g4x_dp_init(dev_priv, PCH_DP_B, PORT_B);
10963                 }
10964
10965                 if (intel_de_read(dev_priv, PCH_HDMIC) & SDVO_DETECTED)
10966                         g4x_hdmi_init(dev_priv, PCH_HDMIC, PORT_C);
10967
10968                 if (!dpd_is_edp && intel_de_read(dev_priv, PCH_HDMID) & SDVO_DETECTED)
10969                         g4x_hdmi_init(dev_priv, PCH_HDMID, PORT_D);
10970
10971                 if (intel_de_read(dev_priv, PCH_DP_C) & DP_DETECTED)
10972                         g4x_dp_init(dev_priv, PCH_DP_C, PORT_C);
10973
10974                 if (intel_de_read(dev_priv, PCH_DP_D) & DP_DETECTED)
10975                         g4x_dp_init(dev_priv, PCH_DP_D, PORT_D);
10976         } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
10977                 bool has_edp, has_port;
10978
10979                 if (IS_VALLEYVIEW(dev_priv) && dev_priv->vbt.int_crt_support)
10980                         intel_crt_init(dev_priv);
10981
10982                 /*
10983                  * The DP_DETECTED bit is the latched state of the DDC
10984                  * SDA pin at boot. However since eDP doesn't require DDC
10985                  * (no way to plug in a DP->HDMI dongle) the DDC pins for
10986                  * eDP ports may have been muxed to an alternate function.
10987                  * Thus we can't rely on the DP_DETECTED bit alone to detect
10988                  * eDP ports. Consult the VBT as well as DP_DETECTED to
10989                  * detect eDP ports.
10990                  *
10991                  * Sadly the straps seem to be missing sometimes even for HDMI
10992                  * ports (eg. on Voyo V3 - CHT x7-Z8700), so check both strap
10993                  * and VBT for the presence of the port. Additionally we can't
10994                  * trust the port type the VBT declares as we've seen at least
10995                  * HDMI ports that the VBT claim are DP or eDP.
10996                  */
10997                 has_edp = intel_dp_is_port_edp(dev_priv, PORT_B);
10998                 has_port = intel_bios_is_port_present(dev_priv, PORT_B);
10999                 if (intel_de_read(dev_priv, VLV_DP_B) & DP_DETECTED || has_port)
11000                         has_edp &= g4x_dp_init(dev_priv, VLV_DP_B, PORT_B);
11001                 if ((intel_de_read(dev_priv, VLV_HDMIB) & SDVO_DETECTED || has_port) && !has_edp)
11002                         g4x_hdmi_init(dev_priv, VLV_HDMIB, PORT_B);
11003
11004                 has_edp = intel_dp_is_port_edp(dev_priv, PORT_C);
11005                 has_port = intel_bios_is_port_present(dev_priv, PORT_C);
11006                 if (intel_de_read(dev_priv, VLV_DP_C) & DP_DETECTED || has_port)
11007                         has_edp &= g4x_dp_init(dev_priv, VLV_DP_C, PORT_C);
11008                 if ((intel_de_read(dev_priv, VLV_HDMIC) & SDVO_DETECTED || has_port) && !has_edp)
11009                         g4x_hdmi_init(dev_priv, VLV_HDMIC, PORT_C);
11010
11011                 if (IS_CHERRYVIEW(dev_priv)) {
11012                         /*
11013                          * eDP not supported on port D,
11014                          * so no need to worry about it
11015                          */
11016                         has_port = intel_bios_is_port_present(dev_priv, PORT_D);
11017                         if (intel_de_read(dev_priv, CHV_DP_D) & DP_DETECTED || has_port)
11018                                 g4x_dp_init(dev_priv, CHV_DP_D, PORT_D);
11019                         if (intel_de_read(dev_priv, CHV_HDMID) & SDVO_DETECTED || has_port)
11020                                 g4x_hdmi_init(dev_priv, CHV_HDMID, PORT_D);
11021                 }
11022
11023                 vlv_dsi_init(dev_priv);
11024         } else if (IS_PINEVIEW(dev_priv)) {
11025                 intel_lvds_init(dev_priv);
11026                 intel_crt_init(dev_priv);
11027         } else if (IS_DISPLAY_VER(dev_priv, 3, 4)) {
11028                 bool found = false;
11029
11030                 if (IS_MOBILE(dev_priv))
11031                         intel_lvds_init(dev_priv);
11032
11033                 intel_crt_init(dev_priv);
11034
11035                 if (intel_de_read(dev_priv, GEN3_SDVOB) & SDVO_DETECTED) {
11036                         drm_dbg_kms(&dev_priv->drm, "probing SDVOB\n");
11037                         found = intel_sdvo_init(dev_priv, GEN3_SDVOB, PORT_B);
11038                         if (!found && IS_G4X(dev_priv)) {
11039                                 drm_dbg_kms(&dev_priv->drm,
11040                                             "probing HDMI on SDVOB\n");
11041                                 g4x_hdmi_init(dev_priv, GEN4_HDMIB, PORT_B);
11042                         }
11043
11044                         if (!found && IS_G4X(dev_priv))
11045                                 g4x_dp_init(dev_priv, DP_B, PORT_B);
11046                 }
11047
11048                 /* Before G4X SDVOC doesn't have its own detect register */
11049
11050                 if (intel_de_read(dev_priv, GEN3_SDVOB) & SDVO_DETECTED) {
11051                         drm_dbg_kms(&dev_priv->drm, "probing SDVOC\n");
11052                         found = intel_sdvo_init(dev_priv, GEN3_SDVOC, PORT_C);
11053                 }
11054
11055                 if (!found && (intel_de_read(dev_priv, GEN3_SDVOC) & SDVO_DETECTED)) {
11056
11057                         if (IS_G4X(dev_priv)) {
11058                                 drm_dbg_kms(&dev_priv->drm,
11059                                             "probing HDMI on SDVOC\n");
11060                                 g4x_hdmi_init(dev_priv, GEN4_HDMIC, PORT_C);
11061                         }
11062                         if (IS_G4X(dev_priv))
11063                                 g4x_dp_init(dev_priv, DP_C, PORT_C);
11064                 }
11065
11066                 if (IS_G4X(dev_priv) && (intel_de_read(dev_priv, DP_D) & DP_DETECTED))
11067                         g4x_dp_init(dev_priv, DP_D, PORT_D);
11068
11069                 if (SUPPORTS_TV(dev_priv))
11070                         intel_tv_init(dev_priv);
11071         } else if (DISPLAY_VER(dev_priv) == 2) {
11072                 if (IS_I85X(dev_priv))
11073                         intel_lvds_init(dev_priv);
11074
11075                 intel_crt_init(dev_priv);
11076                 intel_dvo_init(dev_priv);
11077         }
11078
11079         for_each_intel_encoder(&dev_priv->drm, encoder) {
11080                 encoder->base.possible_crtcs =
11081                         intel_encoder_possible_crtcs(encoder);
11082                 encoder->base.possible_clones =
11083                         intel_encoder_possible_clones(encoder);
11084         }
11085
11086         intel_init_pch_refclk(dev_priv);
11087
11088         drm_helper_move_panel_connectors_to_head(&dev_priv->drm);
11089 }
11090
11091 static enum drm_mode_status
11092 intel_mode_valid(struct drm_device *dev,
11093                  const struct drm_display_mode *mode)
11094 {
11095         struct drm_i915_private *dev_priv = to_i915(dev);
11096         int hdisplay_max, htotal_max;
11097         int vdisplay_max, vtotal_max;
11098
11099         /*
11100          * Can't reject DBLSCAN here because Xorg ddxen can add piles
11101          * of DBLSCAN modes to the output's mode list when they detect
11102          * the scaling mode property on the connector. And they don't
11103          * ask the kernel to validate those modes in any way until
11104          * modeset time at which point the client gets a protocol error.
11105          * So in order to not upset those clients we silently ignore the
11106          * DBLSCAN flag on such connectors. For other connectors we will
11107          * reject modes with the DBLSCAN flag in encoder->compute_config().
11108          * And we always reject DBLSCAN modes in connector->mode_valid()
11109          * as we never want such modes on the connector's mode list.
11110          */
11111
11112         if (mode->vscan > 1)
11113                 return MODE_NO_VSCAN;
11114
11115         if (mode->flags & DRM_MODE_FLAG_HSKEW)
11116                 return MODE_H_ILLEGAL;
11117
11118         if (mode->flags & (DRM_MODE_FLAG_CSYNC |
11119                            DRM_MODE_FLAG_NCSYNC |
11120                            DRM_MODE_FLAG_PCSYNC))
11121                 return MODE_HSYNC;
11122
11123         if (mode->flags & (DRM_MODE_FLAG_BCAST |
11124                            DRM_MODE_FLAG_PIXMUX |
11125                            DRM_MODE_FLAG_CLKDIV2))
11126                 return MODE_BAD;
11127
11128         /* Transcoder timing limits */
11129         if (DISPLAY_VER(dev_priv) >= 11) {
11130                 hdisplay_max = 16384;
11131                 vdisplay_max = 8192;
11132                 htotal_max = 16384;
11133                 vtotal_max = 8192;
11134         } else if (DISPLAY_VER(dev_priv) >= 9 ||
11135                    IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv)) {
11136                 hdisplay_max = 8192; /* FDI max 4096 handled elsewhere */
11137                 vdisplay_max = 4096;
11138                 htotal_max = 8192;
11139                 vtotal_max = 8192;
11140         } else if (DISPLAY_VER(dev_priv) >= 3) {
11141                 hdisplay_max = 4096;
11142                 vdisplay_max = 4096;
11143                 htotal_max = 8192;
11144                 vtotal_max = 8192;
11145         } else {
11146                 hdisplay_max = 2048;
11147                 vdisplay_max = 2048;
11148                 htotal_max = 4096;
11149                 vtotal_max = 4096;
11150         }
11151
11152         if (mode->hdisplay > hdisplay_max ||
11153             mode->hsync_start > htotal_max ||
11154             mode->hsync_end > htotal_max ||
11155             mode->htotal > htotal_max)
11156                 return MODE_H_ILLEGAL;
11157
11158         if (mode->vdisplay > vdisplay_max ||
11159             mode->vsync_start > vtotal_max ||
11160             mode->vsync_end > vtotal_max ||
11161             mode->vtotal > vtotal_max)
11162                 return MODE_V_ILLEGAL;
11163
11164         if (DISPLAY_VER(dev_priv) >= 5) {
11165                 if (mode->hdisplay < 64 ||
11166                     mode->htotal - mode->hdisplay < 32)
11167                         return MODE_H_ILLEGAL;
11168
11169                 if (mode->vtotal - mode->vdisplay < 5)
11170                         return MODE_V_ILLEGAL;
11171         } else {
11172                 if (mode->htotal - mode->hdisplay < 32)
11173                         return MODE_H_ILLEGAL;
11174
11175                 if (mode->vtotal - mode->vdisplay < 3)
11176                         return MODE_V_ILLEGAL;
11177         }
11178
11179         return MODE_OK;
11180 }
11181
11182 enum drm_mode_status
11183 intel_mode_valid_max_plane_size(struct drm_i915_private *dev_priv,
11184                                 const struct drm_display_mode *mode,
11185                                 bool bigjoiner)
11186 {
11187         int plane_width_max, plane_height_max;
11188
11189         /*
11190          * intel_mode_valid() should be
11191          * sufficient on older platforms.
11192          */
11193         if (DISPLAY_VER(dev_priv) < 9)
11194                 return MODE_OK;
11195
11196         /*
11197          * Most people will probably want a fullscreen
11198          * plane so let's not advertize modes that are
11199          * too big for that.
11200          */
11201         if (DISPLAY_VER(dev_priv) >= 11) {
11202                 plane_width_max = 5120 << bigjoiner;
11203                 plane_height_max = 4320;
11204         } else {
11205                 plane_width_max = 5120;
11206                 plane_height_max = 4096;
11207         }
11208
11209         if (mode->hdisplay > plane_width_max)
11210                 return MODE_H_ILLEGAL;
11211
11212         if (mode->vdisplay > plane_height_max)
11213                 return MODE_V_ILLEGAL;
11214
11215         return MODE_OK;
11216 }
11217
11218 static const struct drm_mode_config_funcs intel_mode_funcs = {
11219         .fb_create = intel_user_framebuffer_create,
11220         .get_format_info = intel_get_format_info,
11221         .output_poll_changed = intel_fbdev_output_poll_changed,
11222         .mode_valid = intel_mode_valid,
11223         .atomic_check = intel_atomic_check,
11224         .atomic_commit = intel_atomic_commit,
11225         .atomic_state_alloc = intel_atomic_state_alloc,
11226         .atomic_state_clear = intel_atomic_state_clear,
11227         .atomic_state_free = intel_atomic_state_free,
11228 };
11229
11230 /**
11231  * intel_init_display_hooks - initialize the display modesetting hooks
11232  * @dev_priv: device private
11233  */
11234 void intel_init_display_hooks(struct drm_i915_private *dev_priv)
11235 {
11236         if (!HAS_DISPLAY(dev_priv))
11237                 return;
11238
11239         intel_init_cdclk_hooks(dev_priv);
11240         intel_init_audio_hooks(dev_priv);
11241
11242         intel_dpll_init_clock_hook(dev_priv);
11243
11244         if (DISPLAY_VER(dev_priv) >= 9) {
11245                 dev_priv->display.get_pipe_config = hsw_get_pipe_config;
11246                 dev_priv->display.crtc_enable = hsw_crtc_enable;
11247                 dev_priv->display.crtc_disable = hsw_crtc_disable;
11248         } else if (HAS_DDI(dev_priv)) {
11249                 dev_priv->display.get_pipe_config = hsw_get_pipe_config;
11250                 dev_priv->display.crtc_enable = hsw_crtc_enable;
11251                 dev_priv->display.crtc_disable = hsw_crtc_disable;
11252         } else if (HAS_PCH_SPLIT(dev_priv)) {
11253                 dev_priv->display.get_pipe_config = ilk_get_pipe_config;
11254                 dev_priv->display.crtc_enable = ilk_crtc_enable;
11255                 dev_priv->display.crtc_disable = ilk_crtc_disable;
11256         } else if (IS_CHERRYVIEW(dev_priv) ||
11257                    IS_VALLEYVIEW(dev_priv)) {
11258                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
11259                 dev_priv->display.crtc_enable = valleyview_crtc_enable;
11260                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
11261         } else {
11262                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
11263                 dev_priv->display.crtc_enable = i9xx_crtc_enable;
11264                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
11265         }
11266
11267         intel_fdi_init_hook(dev_priv);
11268
11269         if (DISPLAY_VER(dev_priv) >= 9) {
11270                 dev_priv->display.commit_modeset_enables = skl_commit_modeset_enables;
11271                 dev_priv->display.get_initial_plane_config = skl_get_initial_plane_config;
11272         } else {
11273                 dev_priv->display.commit_modeset_enables = intel_commit_modeset_enables;
11274                 dev_priv->display.get_initial_plane_config = i9xx_get_initial_plane_config;
11275         }
11276
11277 }
11278
11279 void intel_modeset_init_hw(struct drm_i915_private *i915)
11280 {
11281         struct intel_cdclk_state *cdclk_state;
11282
11283         if (!HAS_DISPLAY(i915))
11284                 return;
11285
11286         cdclk_state = to_intel_cdclk_state(i915->cdclk.obj.state);
11287
11288         intel_update_cdclk(i915);
11289         intel_dump_cdclk_config(&i915->cdclk.hw, "Current CDCLK");
11290         cdclk_state->logical = cdclk_state->actual = i915->cdclk.hw;
11291 }
11292
11293 static int sanitize_watermarks_add_affected(struct drm_atomic_state *state)
11294 {
11295         struct drm_plane *plane;
11296         struct intel_crtc *crtc;
11297
11298         for_each_intel_crtc(state->dev, crtc) {
11299                 struct intel_crtc_state *crtc_state;
11300
11301                 crtc_state = intel_atomic_get_crtc_state(state, crtc);
11302                 if (IS_ERR(crtc_state))
11303                         return PTR_ERR(crtc_state);
11304
11305                 if (crtc_state->hw.active) {
11306                         /*
11307                          * Preserve the inherited flag to avoid
11308                          * taking the full modeset path.
11309                          */
11310                         crtc_state->inherited = true;
11311                 }
11312         }
11313
11314         drm_for_each_plane(plane, state->dev) {
11315                 struct drm_plane_state *plane_state;
11316
11317                 plane_state = drm_atomic_get_plane_state(state, plane);
11318                 if (IS_ERR(plane_state))
11319                         return PTR_ERR(plane_state);
11320         }
11321
11322         return 0;
11323 }
11324
11325 /*
11326  * Calculate what we think the watermarks should be for the state we've read
11327  * out of the hardware and then immediately program those watermarks so that
11328  * we ensure the hardware settings match our internal state.
11329  *
11330  * We can calculate what we think WM's should be by creating a duplicate of the
11331  * current state (which was constructed during hardware readout) and running it
11332  * through the atomic check code to calculate new watermark values in the
11333  * state object.
11334  */
11335 static void sanitize_watermarks(struct drm_i915_private *dev_priv)
11336 {
11337         struct drm_atomic_state *state;
11338         struct intel_atomic_state *intel_state;
11339         struct intel_crtc *crtc;
11340         struct intel_crtc_state *crtc_state;
11341         struct drm_modeset_acquire_ctx ctx;
11342         int ret;
11343         int i;
11344
11345         /* Only supported on platforms that use atomic watermark design */
11346         if (!dev_priv->display.optimize_watermarks)
11347                 return;
11348
11349         state = drm_atomic_state_alloc(&dev_priv->drm);
11350         if (drm_WARN_ON(&dev_priv->drm, !state))
11351                 return;
11352
11353         intel_state = to_intel_atomic_state(state);
11354
11355         drm_modeset_acquire_init(&ctx, 0);
11356
11357 retry:
11358         state->acquire_ctx = &ctx;
11359
11360         /*
11361          * Hardware readout is the only time we don't want to calculate
11362          * intermediate watermarks (since we don't trust the current
11363          * watermarks).
11364          */
11365         if (!HAS_GMCH(dev_priv))
11366                 intel_state->skip_intermediate_wm = true;
11367
11368         ret = sanitize_watermarks_add_affected(state);
11369         if (ret)
11370                 goto fail;
11371
11372         ret = intel_atomic_check(&dev_priv->drm, state);
11373         if (ret)
11374                 goto fail;
11375
11376         /* Write calculated watermark values back */
11377         for_each_new_intel_crtc_in_state(intel_state, crtc, crtc_state, i) {
11378                 crtc_state->wm.need_postvbl_update = true;
11379                 dev_priv->display.optimize_watermarks(intel_state, crtc);
11380
11381                 to_intel_crtc_state(crtc->base.state)->wm = crtc_state->wm;
11382         }
11383
11384 fail:
11385         if (ret == -EDEADLK) {
11386                 drm_atomic_state_clear(state);
11387                 drm_modeset_backoff(&ctx);
11388                 goto retry;
11389         }
11390
11391         /*
11392          * If we fail here, it means that the hardware appears to be
11393          * programmed in a way that shouldn't be possible, given our
11394          * understanding of watermark requirements.  This might mean a
11395          * mistake in the hardware readout code or a mistake in the
11396          * watermark calculations for a given platform.  Raise a WARN
11397          * so that this is noticeable.
11398          *
11399          * If this actually happens, we'll have to just leave the
11400          * BIOS-programmed watermarks untouched and hope for the best.
11401          */
11402         drm_WARN(&dev_priv->drm, ret,
11403                  "Could not determine valid watermarks for inherited state\n");
11404
11405         drm_atomic_state_put(state);
11406
11407         drm_modeset_drop_locks(&ctx);
11408         drm_modeset_acquire_fini(&ctx);
11409 }
11410
11411 static int intel_initial_commit(struct drm_device *dev)
11412 {
11413         struct drm_atomic_state *state = NULL;
11414         struct drm_modeset_acquire_ctx ctx;
11415         struct intel_crtc *crtc;
11416         int ret = 0;
11417
11418         state = drm_atomic_state_alloc(dev);
11419         if (!state)
11420                 return -ENOMEM;
11421
11422         drm_modeset_acquire_init(&ctx, 0);
11423
11424 retry:
11425         state->acquire_ctx = &ctx;
11426
11427         for_each_intel_crtc(dev, crtc) {
11428                 struct intel_crtc_state *crtc_state =
11429                         intel_atomic_get_crtc_state(state, crtc);
11430
11431                 if (IS_ERR(crtc_state)) {
11432                         ret = PTR_ERR(crtc_state);
11433                         goto out;
11434                 }
11435
11436                 if (crtc_state->hw.active) {
11437                         struct intel_encoder *encoder;
11438
11439                         /*
11440                          * We've not yet detected sink capabilities
11441                          * (audio,infoframes,etc.) and thus we don't want to
11442                          * force a full state recomputation yet. We want that to
11443                          * happen only for the first real commit from userspace.
11444                          * So preserve the inherited flag for the time being.
11445                          */
11446                         crtc_state->inherited = true;
11447
11448                         ret = drm_atomic_add_affected_planes(state, &crtc->base);
11449                         if (ret)
11450                                 goto out;
11451
11452                         /*
11453                          * FIXME hack to force a LUT update to avoid the
11454                          * plane update forcing the pipe gamma on without
11455                          * having a proper LUT loaded. Remove once we
11456                          * have readout for pipe gamma enable.
11457                          */
11458                         crtc_state->uapi.color_mgmt_changed = true;
11459
11460                         for_each_intel_encoder_mask(dev, encoder,
11461                                                     crtc_state->uapi.encoder_mask) {
11462                                 if (encoder->initial_fastset_check &&
11463                                     !encoder->initial_fastset_check(encoder, crtc_state)) {
11464                                         ret = drm_atomic_add_affected_connectors(state,
11465                                                                                  &crtc->base);
11466                                         if (ret)
11467                                                 goto out;
11468                                 }
11469                         }
11470                 }
11471         }
11472
11473         ret = drm_atomic_commit(state);
11474
11475 out:
11476         if (ret == -EDEADLK) {
11477                 drm_atomic_state_clear(state);
11478                 drm_modeset_backoff(&ctx);
11479                 goto retry;
11480         }
11481
11482         drm_atomic_state_put(state);
11483
11484         drm_modeset_drop_locks(&ctx);
11485         drm_modeset_acquire_fini(&ctx);
11486
11487         return ret;
11488 }
11489
11490 static void intel_mode_config_init(struct drm_i915_private *i915)
11491 {
11492         struct drm_mode_config *mode_config = &i915->drm.mode_config;
11493
11494         drm_mode_config_init(&i915->drm);
11495         INIT_LIST_HEAD(&i915->global_obj_list);
11496
11497         mode_config->min_width = 0;
11498         mode_config->min_height = 0;
11499
11500         mode_config->preferred_depth = 24;
11501         mode_config->prefer_shadow = 1;
11502
11503         mode_config->funcs = &intel_mode_funcs;
11504
11505         mode_config->async_page_flip = HAS_ASYNC_FLIPS(i915);
11506
11507         /*
11508          * Maximum framebuffer dimensions, chosen to match
11509          * the maximum render engine surface size on gen4+.
11510          */
11511         if (DISPLAY_VER(i915) >= 7) {
11512                 mode_config->max_width = 16384;
11513                 mode_config->max_height = 16384;
11514         } else if (DISPLAY_VER(i915) >= 4) {
11515                 mode_config->max_width = 8192;
11516                 mode_config->max_height = 8192;
11517         } else if (DISPLAY_VER(i915) == 3) {
11518                 mode_config->max_width = 4096;
11519                 mode_config->max_height = 4096;
11520         } else {
11521                 mode_config->max_width = 2048;
11522                 mode_config->max_height = 2048;
11523         }
11524
11525         if (IS_I845G(i915) || IS_I865G(i915)) {
11526                 mode_config->cursor_width = IS_I845G(i915) ? 64 : 512;
11527                 mode_config->cursor_height = 1023;
11528         } else if (IS_I830(i915) || IS_I85X(i915) ||
11529                    IS_I915G(i915) || IS_I915GM(i915)) {
11530                 mode_config->cursor_width = 64;
11531                 mode_config->cursor_height = 64;
11532         } else {
11533                 mode_config->cursor_width = 256;
11534                 mode_config->cursor_height = 256;
11535         }
11536 }
11537
11538 static void intel_mode_config_cleanup(struct drm_i915_private *i915)
11539 {
11540         intel_atomic_global_obj_cleanup(i915);
11541         drm_mode_config_cleanup(&i915->drm);
11542 }
11543
11544 static void plane_config_fini(struct intel_initial_plane_config *plane_config)
11545 {
11546         if (plane_config->fb) {
11547                 struct drm_framebuffer *fb = &plane_config->fb->base;
11548
11549                 /* We may only have the stub and not a full framebuffer */
11550                 if (drm_framebuffer_read_refcount(fb))
11551                         drm_framebuffer_put(fb);
11552                 else
11553                         kfree(fb);
11554         }
11555
11556         if (plane_config->vma)
11557                 i915_vma_put(plane_config->vma);
11558 }
11559
11560 /* part #1: call before irq install */
11561 int intel_modeset_init_noirq(struct drm_i915_private *i915)
11562 {
11563         int ret;
11564
11565         if (i915_inject_probe_failure(i915))
11566                 return -ENODEV;
11567
11568         if (HAS_DISPLAY(i915)) {
11569                 ret = drm_vblank_init(&i915->drm,
11570                                       INTEL_NUM_PIPES(i915));
11571                 if (ret)
11572                         return ret;
11573         }
11574
11575         intel_bios_init(i915);
11576
11577         ret = intel_vga_register(i915);
11578         if (ret)
11579                 goto cleanup_bios;
11580
11581         /* FIXME: completely on the wrong abstraction layer */
11582         intel_power_domains_init_hw(i915, false);
11583
11584         if (!HAS_DISPLAY(i915))
11585                 return 0;
11586
11587         intel_dmc_ucode_init(i915);
11588
11589         i915->modeset_wq = alloc_ordered_workqueue("i915_modeset", 0);
11590         i915->flip_wq = alloc_workqueue("i915_flip", WQ_HIGHPRI |
11591                                         WQ_UNBOUND, WQ_UNBOUND_MAX_ACTIVE);
11592
11593         i915->framestart_delay = 1; /* 1-4 */
11594
11595         i915->window2_delay = 0; /* No DSB so no window2 delay */
11596
11597         intel_mode_config_init(i915);
11598
11599         ret = intel_cdclk_init(i915);
11600         if (ret)
11601                 goto cleanup_vga_client_pw_domain_dmc;
11602
11603         ret = intel_dbuf_init(i915);
11604         if (ret)
11605                 goto cleanup_vga_client_pw_domain_dmc;
11606
11607         ret = intel_bw_init(i915);
11608         if (ret)
11609                 goto cleanup_vga_client_pw_domain_dmc;
11610
11611         init_llist_head(&i915->atomic_helper.free_list);
11612         INIT_WORK(&i915->atomic_helper.free_work,
11613                   intel_atomic_helper_free_state_worker);
11614
11615         intel_init_quirks(i915);
11616
11617         intel_fbc_init(i915);
11618
11619         return 0;
11620
11621 cleanup_vga_client_pw_domain_dmc:
11622         intel_dmc_ucode_fini(i915);
11623         intel_power_domains_driver_remove(i915);
11624         intel_vga_unregister(i915);
11625 cleanup_bios:
11626         intel_bios_driver_remove(i915);
11627
11628         return ret;
11629 }
11630
11631 /* part #2: call after irq install, but before gem init */
11632 int intel_modeset_init_nogem(struct drm_i915_private *i915)
11633 {
11634         struct drm_device *dev = &i915->drm;
11635         enum pipe pipe;
11636         struct intel_crtc *crtc;
11637         int ret;
11638
11639         if (!HAS_DISPLAY(i915))
11640                 return 0;
11641
11642         intel_init_pm(i915);
11643
11644         intel_panel_sanitize_ssc(i915);
11645
11646         intel_pps_setup(i915);
11647
11648         intel_gmbus_setup(i915);
11649
11650         drm_dbg_kms(&i915->drm, "%d display pipe%s available.\n",
11651                     INTEL_NUM_PIPES(i915),
11652                     INTEL_NUM_PIPES(i915) > 1 ? "s" : "");
11653
11654         for_each_pipe(i915, pipe) {
11655                 ret = intel_crtc_init(i915, pipe);
11656                 if (ret) {
11657                         intel_mode_config_cleanup(i915);
11658                         return ret;
11659                 }
11660         }
11661
11662         intel_plane_possible_crtcs_init(i915);
11663         intel_shared_dpll_init(dev);
11664         intel_fdi_pll_freq_update(i915);
11665
11666         intel_update_czclk(i915);
11667         intel_modeset_init_hw(i915);
11668         intel_dpll_update_ref_clks(i915);
11669
11670         intel_hdcp_component_init(i915);
11671
11672         if (i915->max_cdclk_freq == 0)
11673                 intel_update_max_cdclk(i915);
11674
11675         /*
11676          * If the platform has HTI, we need to find out whether it has reserved
11677          * any display resources before we create our display outputs.
11678          */
11679         if (INTEL_INFO(i915)->display.has_hti)
11680                 i915->hti_state = intel_de_read(i915, HDPORT_STATE);
11681
11682         /* Just disable it once at startup */
11683         intel_vga_disable(i915);
11684         intel_setup_outputs(i915);
11685
11686         drm_modeset_lock_all(dev);
11687         intel_modeset_setup_hw_state(dev, dev->mode_config.acquire_ctx);
11688         intel_acpi_assign_connector_fwnodes(i915);
11689         drm_modeset_unlock_all(dev);
11690
11691         for_each_intel_crtc(dev, crtc) {
11692                 struct intel_initial_plane_config plane_config = {};
11693
11694                 if (!to_intel_crtc_state(crtc->base.state)->uapi.active)
11695                         continue;
11696
11697                 /*
11698                  * Note that reserving the BIOS fb up front prevents us
11699                  * from stuffing other stolen allocations like the ring
11700                  * on top.  This prevents some ugliness at boot time, and
11701                  * can even allow for smooth boot transitions if the BIOS
11702                  * fb is large enough for the active pipe configuration.
11703                  */
11704                 i915->display.get_initial_plane_config(crtc, &plane_config);
11705
11706                 /*
11707                  * If the fb is shared between multiple heads, we'll
11708                  * just get the first one.
11709                  */
11710                 intel_find_initial_plane_obj(crtc, &plane_config);
11711
11712                 plane_config_fini(&plane_config);
11713         }
11714
11715         /*
11716          * Make sure hardware watermarks really match the state we read out.
11717          * Note that we need to do this after reconstructing the BIOS fb's
11718          * since the watermark calculation done here will use pstate->fb.
11719          */
11720         if (!HAS_GMCH(i915))
11721                 sanitize_watermarks(i915);
11722
11723         return 0;
11724 }
11725
11726 /* part #3: call after gem init */
11727 int intel_modeset_init(struct drm_i915_private *i915)
11728 {
11729         int ret;
11730
11731         if (!HAS_DISPLAY(i915))
11732                 return 0;
11733
11734         /*
11735          * Force all active planes to recompute their states. So that on
11736          * mode_setcrtc after probe, all the intel_plane_state variables
11737          * are already calculated and there is no assert_plane warnings
11738          * during bootup.
11739          */
11740         ret = intel_initial_commit(&i915->drm);
11741         if (ret)
11742                 drm_dbg_kms(&i915->drm, "Initial modeset failed, %d\n", ret);
11743
11744         intel_overlay_setup(i915);
11745
11746         ret = intel_fbdev_init(&i915->drm);
11747         if (ret)
11748                 return ret;
11749
11750         /* Only enable hotplug handling once the fbdev is fully set up. */
11751         intel_hpd_init(i915);
11752         intel_hpd_poll_disable(i915);
11753
11754         intel_init_ipc(i915);
11755
11756         return 0;
11757 }
11758
11759 void i830_enable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe)
11760 {
11761         struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
11762         /* 640x480@60Hz, ~25175 kHz */
11763         struct dpll clock = {
11764                 .m1 = 18,
11765                 .m2 = 7,
11766                 .p1 = 13,
11767                 .p2 = 4,
11768                 .n = 2,
11769         };
11770         u32 dpll, fp;
11771         int i;
11772
11773         drm_WARN_ON(&dev_priv->drm,
11774                     i9xx_calc_dpll_params(48000, &clock) != 25154);
11775
11776         drm_dbg_kms(&dev_priv->drm,
11777                     "enabling pipe %c due to force quirk (vco=%d dot=%d)\n",
11778                     pipe_name(pipe), clock.vco, clock.dot);
11779
11780         fp = i9xx_dpll_compute_fp(&clock);
11781         dpll = DPLL_DVO_2X_MODE |
11782                 DPLL_VGA_MODE_DIS |
11783                 ((clock.p1 - 2) << DPLL_FPA01_P1_POST_DIV_SHIFT) |
11784                 PLL_P2_DIVIDE_BY_4 |
11785                 PLL_REF_INPUT_DREFCLK |
11786                 DPLL_VCO_ENABLE;
11787
11788         intel_de_write(dev_priv, FP0(pipe), fp);
11789         intel_de_write(dev_priv, FP1(pipe), fp);
11790
11791         intel_de_write(dev_priv, HTOTAL(pipe), (640 - 1) | ((800 - 1) << 16));
11792         intel_de_write(dev_priv, HBLANK(pipe), (640 - 1) | ((800 - 1) << 16));
11793         intel_de_write(dev_priv, HSYNC(pipe), (656 - 1) | ((752 - 1) << 16));
11794         intel_de_write(dev_priv, VTOTAL(pipe), (480 - 1) | ((525 - 1) << 16));
11795         intel_de_write(dev_priv, VBLANK(pipe), (480 - 1) | ((525 - 1) << 16));
11796         intel_de_write(dev_priv, VSYNC(pipe), (490 - 1) | ((492 - 1) << 16));
11797         intel_de_write(dev_priv, PIPESRC(pipe), ((640 - 1) << 16) | (480 - 1));
11798
11799         /*
11800          * Apparently we need to have VGA mode enabled prior to changing
11801          * the P1/P2 dividers. Otherwise the DPLL will keep using the old
11802          * dividers, even though the register value does change.
11803          */
11804         intel_de_write(dev_priv, DPLL(pipe), dpll & ~DPLL_VGA_MODE_DIS);
11805         intel_de_write(dev_priv, DPLL(pipe), dpll);
11806
11807         /* Wait for the clocks to stabilize. */
11808         intel_de_posting_read(dev_priv, DPLL(pipe));
11809         udelay(150);
11810
11811         /* The pixel multiplier can only be updated once the
11812          * DPLL is enabled and the clocks are stable.
11813          *
11814          * So write it again.
11815          */
11816         intel_de_write(dev_priv, DPLL(pipe), dpll);
11817
11818         /* We do this three times for luck */
11819         for (i = 0; i < 3 ; i++) {
11820                 intel_de_write(dev_priv, DPLL(pipe), dpll);
11821                 intel_de_posting_read(dev_priv, DPLL(pipe));
11822                 udelay(150); /* wait for warmup */
11823         }
11824
11825         intel_de_write(dev_priv, PIPECONF(pipe),
11826                        PIPECONF_ENABLE | PIPECONF_PROGRESSIVE);
11827         intel_de_posting_read(dev_priv, PIPECONF(pipe));
11828
11829         intel_wait_for_pipe_scanline_moving(crtc);
11830 }
11831
11832 void i830_disable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe)
11833 {
11834         struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
11835
11836         drm_dbg_kms(&dev_priv->drm, "disabling pipe %c due to force quirk\n",
11837                     pipe_name(pipe));
11838
11839         drm_WARN_ON(&dev_priv->drm,
11840                     intel_de_read(dev_priv, DSPCNTR(PLANE_A)) &
11841                     DISPLAY_PLANE_ENABLE);
11842         drm_WARN_ON(&dev_priv->drm,
11843                     intel_de_read(dev_priv, DSPCNTR(PLANE_B)) &
11844                     DISPLAY_PLANE_ENABLE);
11845         drm_WARN_ON(&dev_priv->drm,
11846                     intel_de_read(dev_priv, DSPCNTR(PLANE_C)) &
11847                     DISPLAY_PLANE_ENABLE);
11848         drm_WARN_ON(&dev_priv->drm,
11849                     intel_de_read(dev_priv, CURCNTR(PIPE_A)) & MCURSOR_MODE);
11850         drm_WARN_ON(&dev_priv->drm,
11851                     intel_de_read(dev_priv, CURCNTR(PIPE_B)) & MCURSOR_MODE);
11852
11853         intel_de_write(dev_priv, PIPECONF(pipe), 0);
11854         intel_de_posting_read(dev_priv, PIPECONF(pipe));
11855
11856         intel_wait_for_pipe_scanline_stopped(crtc);
11857
11858         intel_de_write(dev_priv, DPLL(pipe), DPLL_VGA_MODE_DIS);
11859         intel_de_posting_read(dev_priv, DPLL(pipe));
11860 }
11861
11862 static void
11863 intel_sanitize_plane_mapping(struct drm_i915_private *dev_priv)
11864 {
11865         struct intel_crtc *crtc;
11866
11867         if (DISPLAY_VER(dev_priv) >= 4)
11868                 return;
11869
11870         for_each_intel_crtc(&dev_priv->drm, crtc) {
11871                 struct intel_plane *plane =
11872                         to_intel_plane(crtc->base.primary);
11873                 struct intel_crtc *plane_crtc;
11874                 enum pipe pipe;
11875
11876                 if (!plane->get_hw_state(plane, &pipe))
11877                         continue;
11878
11879                 if (pipe == crtc->pipe)
11880                         continue;
11881
11882                 drm_dbg_kms(&dev_priv->drm,
11883                             "[PLANE:%d:%s] attached to the wrong pipe, disabling plane\n",
11884                             plane->base.base.id, plane->base.name);
11885
11886                 plane_crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
11887                 intel_plane_disable_noatomic(plane_crtc, plane);
11888         }
11889 }
11890
11891 static bool intel_crtc_has_encoders(struct intel_crtc *crtc)
11892 {
11893         struct drm_device *dev = crtc->base.dev;
11894         struct intel_encoder *encoder;
11895
11896         for_each_encoder_on_crtc(dev, &crtc->base, encoder)
11897                 return true;
11898
11899         return false;
11900 }
11901
11902 static struct intel_connector *intel_encoder_find_connector(struct intel_encoder *encoder)
11903 {
11904         struct drm_device *dev = encoder->base.dev;
11905         struct intel_connector *connector;
11906
11907         for_each_connector_on_encoder(dev, &encoder->base, connector)
11908                 return connector;
11909
11910         return NULL;
11911 }
11912
11913 static bool has_pch_trancoder(struct drm_i915_private *dev_priv,
11914                               enum pipe pch_transcoder)
11915 {
11916         return HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv) ||
11917                 (HAS_PCH_LPT_H(dev_priv) && pch_transcoder == PIPE_A);
11918 }
11919
11920 static void intel_sanitize_frame_start_delay(const struct intel_crtc_state *crtc_state)
11921 {
11922         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
11923         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
11924         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
11925
11926         if (DISPLAY_VER(dev_priv) >= 9 ||
11927             IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv)) {
11928                 i915_reg_t reg = CHICKEN_TRANS(cpu_transcoder);
11929                 u32 val;
11930
11931                 if (transcoder_is_dsi(cpu_transcoder))
11932                         return;
11933
11934                 val = intel_de_read(dev_priv, reg);
11935                 val &= ~HSW_FRAME_START_DELAY_MASK;
11936                 val |= HSW_FRAME_START_DELAY(dev_priv->framestart_delay - 1);
11937                 intel_de_write(dev_priv, reg, val);
11938         } else {
11939                 i915_reg_t reg = PIPECONF(cpu_transcoder);
11940                 u32 val;
11941
11942                 val = intel_de_read(dev_priv, reg);
11943                 val &= ~PIPECONF_FRAME_START_DELAY_MASK;
11944                 val |= PIPECONF_FRAME_START_DELAY(dev_priv->framestart_delay - 1);
11945                 intel_de_write(dev_priv, reg, val);
11946         }
11947
11948         if (!crtc_state->has_pch_encoder)
11949                 return;
11950
11951         if (HAS_PCH_IBX(dev_priv)) {
11952                 i915_reg_t reg = PCH_TRANSCONF(crtc->pipe);
11953                 u32 val;
11954
11955                 val = intel_de_read(dev_priv, reg);
11956                 val &= ~TRANS_FRAME_START_DELAY_MASK;
11957                 val |= TRANS_FRAME_START_DELAY(dev_priv->framestart_delay - 1);
11958                 intel_de_write(dev_priv, reg, val);
11959         } else {
11960                 enum pipe pch_transcoder = intel_crtc_pch_transcoder(crtc);
11961                 i915_reg_t reg = TRANS_CHICKEN2(pch_transcoder);
11962                 u32 val;
11963
11964                 val = intel_de_read(dev_priv, reg);
11965                 val &= ~TRANS_CHICKEN2_FRAME_START_DELAY_MASK;
11966                 val |= TRANS_CHICKEN2_FRAME_START_DELAY(dev_priv->framestart_delay - 1);
11967                 intel_de_write(dev_priv, reg, val);
11968         }
11969 }
11970
11971 static void intel_sanitize_crtc(struct intel_crtc *crtc,
11972                                 struct drm_modeset_acquire_ctx *ctx)
11973 {
11974         struct drm_device *dev = crtc->base.dev;
11975         struct drm_i915_private *dev_priv = to_i915(dev);
11976         struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state);
11977
11978         if (crtc_state->hw.active) {
11979                 struct intel_plane *plane;
11980
11981                 /* Clear any frame start delays used for debugging left by the BIOS */
11982                 intel_sanitize_frame_start_delay(crtc_state);
11983
11984                 /* Disable everything but the primary plane */
11985                 for_each_intel_plane_on_crtc(dev, crtc, plane) {
11986                         const struct intel_plane_state *plane_state =
11987                                 to_intel_plane_state(plane->base.state);
11988
11989                         if (plane_state->uapi.visible &&
11990                             plane->base.type != DRM_PLANE_TYPE_PRIMARY)
11991                                 intel_plane_disable_noatomic(crtc, plane);
11992                 }
11993
11994                 /*
11995                  * Disable any background color set by the BIOS, but enable the
11996                  * gamma and CSC to match how we program our planes.
11997                  */
11998                 if (DISPLAY_VER(dev_priv) >= 9)
11999                         intel_de_write(dev_priv, SKL_BOTTOM_COLOR(crtc->pipe),
12000                                        SKL_BOTTOM_COLOR_GAMMA_ENABLE | SKL_BOTTOM_COLOR_CSC_ENABLE);
12001         }
12002
12003         /* Adjust the state of the output pipe according to whether we
12004          * have active connectors/encoders. */
12005         if (crtc_state->hw.active && !intel_crtc_has_encoders(crtc) &&
12006             !crtc_state->bigjoiner_slave)
12007                 intel_crtc_disable_noatomic(crtc, ctx);
12008
12009         if (crtc_state->hw.active || HAS_GMCH(dev_priv)) {
12010                 /*
12011                  * We start out with underrun reporting disabled to avoid races.
12012                  * For correct bookkeeping mark this on active crtcs.
12013                  *
12014                  * Also on gmch platforms we dont have any hardware bits to
12015                  * disable the underrun reporting. Which means we need to start
12016                  * out with underrun reporting disabled also on inactive pipes,
12017                  * since otherwise we'll complain about the garbage we read when
12018                  * e.g. coming up after runtime pm.
12019                  *
12020                  * No protection against concurrent access is required - at
12021                  * worst a fifo underrun happens which also sets this to false.
12022                  */
12023                 crtc->cpu_fifo_underrun_disabled = true;
12024                 /*
12025                  * We track the PCH trancoder underrun reporting state
12026                  * within the crtc. With crtc for pipe A housing the underrun
12027                  * reporting state for PCH transcoder A, crtc for pipe B housing
12028                  * it for PCH transcoder B, etc. LPT-H has only PCH transcoder A,
12029                  * and marking underrun reporting as disabled for the non-existing
12030                  * PCH transcoders B and C would prevent enabling the south
12031                  * error interrupt (see cpt_can_enable_serr_int()).
12032                  */
12033                 if (has_pch_trancoder(dev_priv, crtc->pipe))
12034                         crtc->pch_fifo_underrun_disabled = true;
12035         }
12036 }
12037
12038 static bool has_bogus_dpll_config(const struct intel_crtc_state *crtc_state)
12039 {
12040         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
12041
12042         /*
12043          * Some SNB BIOSen (eg. ASUS K53SV) are known to misprogram
12044          * the hardware when a high res displays plugged in. DPLL P
12045          * divider is zero, and the pipe timings are bonkers. We'll
12046          * try to disable everything in that case.
12047          *
12048          * FIXME would be nice to be able to sanitize this state
12049          * without several WARNs, but for now let's take the easy
12050          * road.
12051          */
12052         return IS_SANDYBRIDGE(dev_priv) &&
12053                 crtc_state->hw.active &&
12054                 crtc_state->shared_dpll &&
12055                 crtc_state->port_clock == 0;
12056 }
12057
12058 static void intel_sanitize_encoder(struct intel_encoder *encoder)
12059 {
12060         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
12061         struct intel_connector *connector;
12062         struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
12063         struct intel_crtc_state *crtc_state = crtc ?
12064                 to_intel_crtc_state(crtc->base.state) : NULL;
12065
12066         /* We need to check both for a crtc link (meaning that the
12067          * encoder is active and trying to read from a pipe) and the
12068          * pipe itself being active. */
12069         bool has_active_crtc = crtc_state &&
12070                 crtc_state->hw.active;
12071
12072         if (crtc_state && has_bogus_dpll_config(crtc_state)) {
12073                 drm_dbg_kms(&dev_priv->drm,
12074                             "BIOS has misprogrammed the hardware. Disabling pipe %c\n",
12075                             pipe_name(crtc->pipe));
12076                 has_active_crtc = false;
12077         }
12078
12079         connector = intel_encoder_find_connector(encoder);
12080         if (connector && !has_active_crtc) {
12081                 drm_dbg_kms(&dev_priv->drm,
12082                             "[ENCODER:%d:%s] has active connectors but no active pipe!\n",
12083                             encoder->base.base.id,
12084                             encoder->base.name);
12085
12086                 /* Connector is active, but has no active pipe. This is
12087                  * fallout from our resume register restoring. Disable
12088                  * the encoder manually again. */
12089                 if (crtc_state) {
12090                         struct drm_encoder *best_encoder;
12091
12092                         drm_dbg_kms(&dev_priv->drm,
12093                                     "[ENCODER:%d:%s] manually disabled\n",
12094                                     encoder->base.base.id,
12095                                     encoder->base.name);
12096
12097                         /* avoid oopsing in case the hooks consult best_encoder */
12098                         best_encoder = connector->base.state->best_encoder;
12099                         connector->base.state->best_encoder = &encoder->base;
12100
12101                         /* FIXME NULL atomic state passed! */
12102                         if (encoder->disable)
12103                                 encoder->disable(NULL, encoder, crtc_state,
12104                                                  connector->base.state);
12105                         if (encoder->post_disable)
12106                                 encoder->post_disable(NULL, encoder, crtc_state,
12107                                                       connector->base.state);
12108
12109                         connector->base.state->best_encoder = best_encoder;
12110                 }
12111                 encoder->base.crtc = NULL;
12112
12113                 /* Inconsistent output/port/pipe state happens presumably due to
12114                  * a bug in one of the get_hw_state functions. Or someplace else
12115                  * in our code, like the register restore mess on resume. Clamp
12116                  * things to off as a safer default. */
12117
12118                 connector->base.dpms = DRM_MODE_DPMS_OFF;
12119                 connector->base.encoder = NULL;
12120         }
12121
12122         /* notify opregion of the sanitized encoder state */
12123         intel_opregion_notify_encoder(encoder, connector && has_active_crtc);
12124
12125         if (HAS_DDI(dev_priv))
12126                 intel_ddi_sanitize_encoder_pll_mapping(encoder);
12127 }
12128
12129 /* FIXME read out full plane state for all planes */
12130 static void readout_plane_state(struct drm_i915_private *dev_priv)
12131 {
12132         struct intel_plane *plane;
12133         struct intel_crtc *crtc;
12134
12135         for_each_intel_plane(&dev_priv->drm, plane) {
12136                 struct intel_plane_state *plane_state =
12137                         to_intel_plane_state(plane->base.state);
12138                 struct intel_crtc_state *crtc_state;
12139                 enum pipe pipe = PIPE_A;
12140                 bool visible;
12141
12142                 visible = plane->get_hw_state(plane, &pipe);
12143
12144                 crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
12145                 crtc_state = to_intel_crtc_state(crtc->base.state);
12146
12147                 intel_set_plane_visible(crtc_state, plane_state, visible);
12148
12149                 drm_dbg_kms(&dev_priv->drm,
12150                             "[PLANE:%d:%s] hw state readout: %s, pipe %c\n",
12151                             plane->base.base.id, plane->base.name,
12152                             enableddisabled(visible), pipe_name(pipe));
12153         }
12154
12155         for_each_intel_crtc(&dev_priv->drm, crtc) {
12156                 struct intel_crtc_state *crtc_state =
12157                         to_intel_crtc_state(crtc->base.state);
12158
12159                 fixup_plane_bitmasks(crtc_state);
12160         }
12161 }
12162
12163 static void intel_modeset_readout_hw_state(struct drm_device *dev)
12164 {
12165         struct drm_i915_private *dev_priv = to_i915(dev);
12166         struct intel_cdclk_state *cdclk_state =
12167                 to_intel_cdclk_state(dev_priv->cdclk.obj.state);
12168         struct intel_dbuf_state *dbuf_state =
12169                 to_intel_dbuf_state(dev_priv->dbuf.obj.state);
12170         enum pipe pipe;
12171         struct intel_crtc *crtc;
12172         struct intel_encoder *encoder;
12173         struct intel_connector *connector;
12174         struct drm_connector_list_iter conn_iter;
12175         u8 active_pipes = 0;
12176
12177         for_each_intel_crtc(dev, crtc) {
12178                 struct intel_crtc_state *crtc_state =
12179                         to_intel_crtc_state(crtc->base.state);
12180
12181                 __drm_atomic_helper_crtc_destroy_state(&crtc_state->uapi);
12182                 intel_crtc_free_hw_state(crtc_state);
12183                 intel_crtc_state_reset(crtc_state, crtc);
12184
12185                 intel_crtc_get_pipe_config(crtc_state);
12186
12187                 crtc_state->hw.enable = crtc_state->hw.active;
12188
12189                 crtc->base.enabled = crtc_state->hw.enable;
12190                 crtc->active = crtc_state->hw.active;
12191
12192                 if (crtc_state->hw.active)
12193                         active_pipes |= BIT(crtc->pipe);
12194
12195                 drm_dbg_kms(&dev_priv->drm,
12196                             "[CRTC:%d:%s] hw state readout: %s\n",
12197                             crtc->base.base.id, crtc->base.name,
12198                             enableddisabled(crtc_state->hw.active));
12199         }
12200
12201         cdclk_state->active_pipes = dbuf_state->active_pipes = active_pipes;
12202
12203         readout_plane_state(dev_priv);
12204
12205         for_each_intel_encoder(dev, encoder) {
12206                 pipe = 0;
12207
12208                 if (encoder->get_hw_state(encoder, &pipe)) {
12209                         struct intel_crtc_state *crtc_state;
12210
12211                         crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
12212                         crtc_state = to_intel_crtc_state(crtc->base.state);
12213
12214                         encoder->base.crtc = &crtc->base;
12215                         intel_encoder_get_config(encoder, crtc_state);
12216                         if (encoder->sync_state)
12217                                 encoder->sync_state(encoder, crtc_state);
12218
12219                         /* read out to slave crtc as well for bigjoiner */
12220                         if (crtc_state->bigjoiner) {
12221                                 /* encoder should read be linked to bigjoiner master */
12222                                 WARN_ON(crtc_state->bigjoiner_slave);
12223
12224                                 crtc = crtc_state->bigjoiner_linked_crtc;
12225                                 crtc_state = to_intel_crtc_state(crtc->base.state);
12226                                 intel_encoder_get_config(encoder, crtc_state);
12227                         }
12228                 } else {
12229                         encoder->base.crtc = NULL;
12230                 }
12231
12232                 drm_dbg_kms(&dev_priv->drm,
12233                             "[ENCODER:%d:%s] hw state readout: %s, pipe %c\n",
12234                             encoder->base.base.id, encoder->base.name,
12235                             enableddisabled(encoder->base.crtc),
12236                             pipe_name(pipe));
12237         }
12238
12239         intel_dpll_readout_hw_state(dev_priv);
12240
12241         drm_connector_list_iter_begin(dev, &conn_iter);
12242         for_each_intel_connector_iter(connector, &conn_iter) {
12243                 if (connector->get_hw_state(connector)) {
12244                         struct intel_crtc_state *crtc_state;
12245                         struct intel_crtc *crtc;
12246
12247                         connector->base.dpms = DRM_MODE_DPMS_ON;
12248
12249                         encoder = intel_attached_encoder(connector);
12250                         connector->base.encoder = &encoder->base;
12251
12252                         crtc = to_intel_crtc(encoder->base.crtc);
12253                         crtc_state = crtc ? to_intel_crtc_state(crtc->base.state) : NULL;
12254
12255                         if (crtc_state && crtc_state->hw.active) {
12256                                 /*
12257                                  * This has to be done during hardware readout
12258                                  * because anything calling .crtc_disable may
12259                                  * rely on the connector_mask being accurate.
12260                                  */
12261                                 crtc_state->uapi.connector_mask |=
12262                                         drm_connector_mask(&connector->base);
12263                                 crtc_state->uapi.encoder_mask |=
12264                                         drm_encoder_mask(&encoder->base);
12265                         }
12266                 } else {
12267                         connector->base.dpms = DRM_MODE_DPMS_OFF;
12268                         connector->base.encoder = NULL;
12269                 }
12270                 drm_dbg_kms(&dev_priv->drm,
12271                             "[CONNECTOR:%d:%s] hw state readout: %s\n",
12272                             connector->base.base.id, connector->base.name,
12273                             enableddisabled(connector->base.encoder));
12274         }
12275         drm_connector_list_iter_end(&conn_iter);
12276
12277         for_each_intel_crtc(dev, crtc) {
12278                 struct intel_bw_state *bw_state =
12279                         to_intel_bw_state(dev_priv->bw_obj.state);
12280                 struct intel_crtc_state *crtc_state =
12281                         to_intel_crtc_state(crtc->base.state);
12282                 struct intel_plane *plane;
12283                 int min_cdclk = 0;
12284
12285                 if (crtc_state->bigjoiner_slave)
12286                         continue;
12287
12288                 if (crtc_state->hw.active) {
12289                         /*
12290                          * The initial mode needs to be set in order to keep
12291                          * the atomic core happy. It wants a valid mode if the
12292                          * crtc's enabled, so we do the above call.
12293                          *
12294                          * But we don't set all the derived state fully, hence
12295                          * set a flag to indicate that a full recalculation is
12296                          * needed on the next commit.
12297                          */
12298                         crtc_state->inherited = true;
12299
12300                         intel_crtc_update_active_timings(crtc_state);
12301
12302                         intel_crtc_copy_hw_to_uapi_state(crtc_state);
12303                 }
12304
12305                 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
12306                         const struct intel_plane_state *plane_state =
12307                                 to_intel_plane_state(plane->base.state);
12308
12309                         /*
12310                          * FIXME don't have the fb yet, so can't
12311                          * use intel_plane_data_rate() :(
12312                          */
12313                         if (plane_state->uapi.visible)
12314                                 crtc_state->data_rate[plane->id] =
12315                                         4 * crtc_state->pixel_rate;
12316                         /*
12317                          * FIXME don't have the fb yet, so can't
12318                          * use plane->min_cdclk() :(
12319                          */
12320                         if (plane_state->uapi.visible && plane->min_cdclk) {
12321                                 if (crtc_state->double_wide || DISPLAY_VER(dev_priv) >= 10)
12322                                         crtc_state->min_cdclk[plane->id] =
12323                                                 DIV_ROUND_UP(crtc_state->pixel_rate, 2);
12324                                 else
12325                                         crtc_state->min_cdclk[plane->id] =
12326                                                 crtc_state->pixel_rate;
12327                         }
12328                         drm_dbg_kms(&dev_priv->drm,
12329                                     "[PLANE:%d:%s] min_cdclk %d kHz\n",
12330                                     plane->base.base.id, plane->base.name,
12331                                     crtc_state->min_cdclk[plane->id]);
12332                 }
12333
12334                 if (crtc_state->hw.active) {
12335                         min_cdclk = intel_crtc_compute_min_cdclk(crtc_state);
12336                         if (drm_WARN_ON(dev, min_cdclk < 0))
12337                                 min_cdclk = 0;
12338                 }
12339
12340                 cdclk_state->min_cdclk[crtc->pipe] = min_cdclk;
12341                 cdclk_state->min_voltage_level[crtc->pipe] =
12342                         crtc_state->min_voltage_level;
12343
12344                 intel_bw_crtc_update(bw_state, crtc_state);
12345
12346                 intel_pipe_config_sanity_check(dev_priv, crtc_state);
12347
12348                 /* discard our incomplete slave state, copy it from master */
12349                 if (crtc_state->bigjoiner && crtc_state->hw.active) {
12350                         struct intel_crtc *slave = crtc_state->bigjoiner_linked_crtc;
12351                         struct intel_crtc_state *slave_crtc_state =
12352                                 to_intel_crtc_state(slave->base.state);
12353
12354                         copy_bigjoiner_crtc_state(slave_crtc_state, crtc_state);
12355                         slave->base.mode = crtc->base.mode;
12356
12357                         cdclk_state->min_cdclk[slave->pipe] = min_cdclk;
12358                         cdclk_state->min_voltage_level[slave->pipe] =
12359                                 crtc_state->min_voltage_level;
12360
12361                         for_each_intel_plane_on_crtc(&dev_priv->drm, slave, plane) {
12362                                 const struct intel_plane_state *plane_state =
12363                                         to_intel_plane_state(plane->base.state);
12364
12365                                 /*
12366                                  * FIXME don't have the fb yet, so can't
12367                                  * use intel_plane_data_rate() :(
12368                                  */
12369                                 if (plane_state->uapi.visible)
12370                                         crtc_state->data_rate[plane->id] =
12371                                                 4 * crtc_state->pixel_rate;
12372                                 else
12373                                         crtc_state->data_rate[plane->id] = 0;
12374                         }
12375
12376                         intel_bw_crtc_update(bw_state, slave_crtc_state);
12377                         drm_calc_timestamping_constants(&slave->base,
12378                                                         &slave_crtc_state->hw.adjusted_mode);
12379                 }
12380         }
12381 }
12382
12383 static void
12384 get_encoder_power_domains(struct drm_i915_private *dev_priv)
12385 {
12386         struct intel_encoder *encoder;
12387
12388         for_each_intel_encoder(&dev_priv->drm, encoder) {
12389                 struct intel_crtc_state *crtc_state;
12390
12391                 if (!encoder->get_power_domains)
12392                         continue;
12393
12394                 /*
12395                  * MST-primary and inactive encoders don't have a crtc state
12396                  * and neither of these require any power domain references.
12397                  */
12398                 if (!encoder->base.crtc)
12399                         continue;
12400
12401                 crtc_state = to_intel_crtc_state(encoder->base.crtc->state);
12402                 encoder->get_power_domains(encoder, crtc_state);
12403         }
12404 }
12405
12406 static void intel_early_display_was(struct drm_i915_private *dev_priv)
12407 {
12408         /*
12409          * Display WA #1185 WaDisableDARBFClkGating:glk,icl,ehl,tgl
12410          * Also known as Wa_14010480278.
12411          */
12412         if (IS_DISPLAY_VER(dev_priv, 10, 12))
12413                 intel_de_write(dev_priv, GEN9_CLKGATE_DIS_0,
12414                                intel_de_read(dev_priv, GEN9_CLKGATE_DIS_0) | DARBF_GATING_DIS);
12415
12416         if (IS_HASWELL(dev_priv)) {
12417                 /*
12418                  * WaRsPkgCStateDisplayPMReq:hsw
12419                  * System hang if this isn't done before disabling all planes!
12420                  */
12421                 intel_de_write(dev_priv, CHICKEN_PAR1_1,
12422                                intel_de_read(dev_priv, CHICKEN_PAR1_1) | FORCE_ARB_IDLE_PLANES);
12423         }
12424
12425         if (IS_KABYLAKE(dev_priv) || IS_COFFEELAKE(dev_priv) || IS_COMETLAKE(dev_priv)) {
12426                 /* Display WA #1142:kbl,cfl,cml */
12427                 intel_de_rmw(dev_priv, CHICKEN_PAR1_1,
12428                              KBL_ARB_FILL_SPARE_22, KBL_ARB_FILL_SPARE_22);
12429                 intel_de_rmw(dev_priv, CHICKEN_MISC_2,
12430                              KBL_ARB_FILL_SPARE_13 | KBL_ARB_FILL_SPARE_14,
12431                              KBL_ARB_FILL_SPARE_14);
12432         }
12433 }
12434
12435 static void ibx_sanitize_pch_hdmi_port(struct drm_i915_private *dev_priv,
12436                                        enum port port, i915_reg_t hdmi_reg)
12437 {
12438         u32 val = intel_de_read(dev_priv, hdmi_reg);
12439
12440         if (val & SDVO_ENABLE ||
12441             (val & SDVO_PIPE_SEL_MASK) == SDVO_PIPE_SEL(PIPE_A))
12442                 return;
12443
12444         drm_dbg_kms(&dev_priv->drm,
12445                     "Sanitizing transcoder select for HDMI %c\n",
12446                     port_name(port));
12447
12448         val &= ~SDVO_PIPE_SEL_MASK;
12449         val |= SDVO_PIPE_SEL(PIPE_A);
12450
12451         intel_de_write(dev_priv, hdmi_reg, val);
12452 }
12453
12454 static void ibx_sanitize_pch_dp_port(struct drm_i915_private *dev_priv,
12455                                      enum port port, i915_reg_t dp_reg)
12456 {
12457         u32 val = intel_de_read(dev_priv, dp_reg);
12458
12459         if (val & DP_PORT_EN ||
12460             (val & DP_PIPE_SEL_MASK) == DP_PIPE_SEL(PIPE_A))
12461                 return;
12462
12463         drm_dbg_kms(&dev_priv->drm,
12464                     "Sanitizing transcoder select for DP %c\n",
12465                     port_name(port));
12466
12467         val &= ~DP_PIPE_SEL_MASK;
12468         val |= DP_PIPE_SEL(PIPE_A);
12469
12470         intel_de_write(dev_priv, dp_reg, val);
12471 }
12472
12473 static void ibx_sanitize_pch_ports(struct drm_i915_private *dev_priv)
12474 {
12475         /*
12476          * The BIOS may select transcoder B on some of the PCH
12477          * ports even it doesn't enable the port. This would trip
12478          * assert_pch_dp_disabled() and assert_pch_hdmi_disabled().
12479          * Sanitize the transcoder select bits to prevent that. We
12480          * assume that the BIOS never actually enabled the port,
12481          * because if it did we'd actually have to toggle the port
12482          * on and back off to make the transcoder A select stick
12483          * (see. intel_dp_link_down(), intel_disable_hdmi(),
12484          * intel_disable_sdvo()).
12485          */
12486         ibx_sanitize_pch_dp_port(dev_priv, PORT_B, PCH_DP_B);
12487         ibx_sanitize_pch_dp_port(dev_priv, PORT_C, PCH_DP_C);
12488         ibx_sanitize_pch_dp_port(dev_priv, PORT_D, PCH_DP_D);
12489
12490         /* PCH SDVOB multiplex with HDMIB */
12491         ibx_sanitize_pch_hdmi_port(dev_priv, PORT_B, PCH_HDMIB);
12492         ibx_sanitize_pch_hdmi_port(dev_priv, PORT_C, PCH_HDMIC);
12493         ibx_sanitize_pch_hdmi_port(dev_priv, PORT_D, PCH_HDMID);
12494 }
12495
12496 /* Scan out the current hw modeset state,
12497  * and sanitizes it to the current state
12498  */
12499 static void
12500 intel_modeset_setup_hw_state(struct drm_device *dev,
12501                              struct drm_modeset_acquire_ctx *ctx)
12502 {
12503         struct drm_i915_private *dev_priv = to_i915(dev);
12504         struct intel_encoder *encoder;
12505         struct intel_crtc *crtc;
12506         intel_wakeref_t wakeref;
12507
12508         wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_INIT);
12509
12510         intel_early_display_was(dev_priv);
12511         intel_modeset_readout_hw_state(dev);
12512
12513         /* HW state is read out, now we need to sanitize this mess. */
12514
12515         /* Sanitize the TypeC port mode upfront, encoders depend on this */
12516         for_each_intel_encoder(dev, encoder) {
12517                 enum phy phy = intel_port_to_phy(dev_priv, encoder->port);
12518
12519                 /* We need to sanitize only the MST primary port. */
12520                 if (encoder->type != INTEL_OUTPUT_DP_MST &&
12521                     intel_phy_is_tc(dev_priv, phy))
12522                         intel_tc_port_sanitize(enc_to_dig_port(encoder));
12523         }
12524
12525         get_encoder_power_domains(dev_priv);
12526
12527         if (HAS_PCH_IBX(dev_priv))
12528                 ibx_sanitize_pch_ports(dev_priv);
12529
12530         /*
12531          * intel_sanitize_plane_mapping() may need to do vblank
12532          * waits, so we need vblank interrupts restored beforehand.
12533          */
12534         for_each_intel_crtc(&dev_priv->drm, crtc) {
12535                 struct intel_crtc_state *crtc_state =
12536                         to_intel_crtc_state(crtc->base.state);
12537
12538                 drm_crtc_vblank_reset(&crtc->base);
12539
12540                 if (crtc_state->hw.active)
12541                         intel_crtc_vblank_on(crtc_state);
12542         }
12543
12544         intel_sanitize_plane_mapping(dev_priv);
12545
12546         for_each_intel_encoder(dev, encoder)
12547                 intel_sanitize_encoder(encoder);
12548
12549         for_each_intel_crtc(&dev_priv->drm, crtc) {
12550                 struct intel_crtc_state *crtc_state =
12551                         to_intel_crtc_state(crtc->base.state);
12552
12553                 intel_sanitize_crtc(crtc, ctx);
12554                 intel_dump_pipe_config(crtc_state, NULL, "[setup_hw_state]");
12555         }
12556
12557         intel_modeset_update_connector_atomic_state(dev);
12558
12559         intel_dpll_sanitize_state(dev_priv);
12560
12561         if (IS_G4X(dev_priv)) {
12562                 g4x_wm_get_hw_state(dev_priv);
12563                 g4x_wm_sanitize(dev_priv);
12564         } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
12565                 vlv_wm_get_hw_state(dev_priv);
12566                 vlv_wm_sanitize(dev_priv);
12567         } else if (DISPLAY_VER(dev_priv) >= 9) {
12568                 skl_wm_get_hw_state(dev_priv);
12569         } else if (HAS_PCH_SPLIT(dev_priv)) {
12570                 ilk_wm_get_hw_state(dev_priv);
12571         }
12572
12573         for_each_intel_crtc(dev, crtc) {
12574                 struct intel_crtc_state *crtc_state =
12575                         to_intel_crtc_state(crtc->base.state);
12576                 u64 put_domains;
12577
12578                 put_domains = modeset_get_crtc_power_domains(crtc_state);
12579                 if (drm_WARN_ON(dev, put_domains))
12580                         modeset_put_crtc_power_domains(crtc, put_domains);
12581         }
12582
12583         intel_display_power_put(dev_priv, POWER_DOMAIN_INIT, wakeref);
12584 }
12585
12586 void intel_display_resume(struct drm_device *dev)
12587 {
12588         struct drm_i915_private *dev_priv = to_i915(dev);
12589         struct drm_atomic_state *state = dev_priv->modeset_restore_state;
12590         struct drm_modeset_acquire_ctx ctx;
12591         int ret;
12592
12593         if (!HAS_DISPLAY(dev_priv))
12594                 return;
12595
12596         dev_priv->modeset_restore_state = NULL;
12597         if (state)
12598                 state->acquire_ctx = &ctx;
12599
12600         drm_modeset_acquire_init(&ctx, 0);
12601
12602         while (1) {
12603                 ret = drm_modeset_lock_all_ctx(dev, &ctx);
12604                 if (ret != -EDEADLK)
12605                         break;
12606
12607                 drm_modeset_backoff(&ctx);
12608         }
12609
12610         if (!ret)
12611                 ret = __intel_display_resume(dev, state, &ctx);
12612
12613         intel_enable_ipc(dev_priv);
12614         drm_modeset_drop_locks(&ctx);
12615         drm_modeset_acquire_fini(&ctx);
12616
12617         if (ret)
12618                 drm_err(&dev_priv->drm,
12619                         "Restoring old state failed with %i\n", ret);
12620         if (state)
12621                 drm_atomic_state_put(state);
12622 }
12623
12624 static void intel_hpd_poll_fini(struct drm_i915_private *i915)
12625 {
12626         struct intel_connector *connector;
12627         struct drm_connector_list_iter conn_iter;
12628
12629         /* Kill all the work that may have been queued by hpd. */
12630         drm_connector_list_iter_begin(&i915->drm, &conn_iter);
12631         for_each_intel_connector_iter(connector, &conn_iter) {
12632                 if (connector->modeset_retry_work.func)
12633                         cancel_work_sync(&connector->modeset_retry_work);
12634                 if (connector->hdcp.shim) {
12635                         cancel_delayed_work_sync(&connector->hdcp.check_work);
12636                         cancel_work_sync(&connector->hdcp.prop_work);
12637                 }
12638         }
12639         drm_connector_list_iter_end(&conn_iter);
12640 }
12641
12642 /* part #1: call before irq uninstall */
12643 void intel_modeset_driver_remove(struct drm_i915_private *i915)
12644 {
12645         if (!HAS_DISPLAY(i915))
12646                 return;
12647
12648         flush_workqueue(i915->flip_wq);
12649         flush_workqueue(i915->modeset_wq);
12650
12651         flush_work(&i915->atomic_helper.free_work);
12652         drm_WARN_ON(&i915->drm, !llist_empty(&i915->atomic_helper.free_list));
12653 }
12654
12655 /* part #2: call after irq uninstall */
12656 void intel_modeset_driver_remove_noirq(struct drm_i915_private *i915)
12657 {
12658         if (!HAS_DISPLAY(i915))
12659                 return;
12660
12661         /*
12662          * Due to the hpd irq storm handling the hotplug work can re-arm the
12663          * poll handlers. Hence disable polling after hpd handling is shut down.
12664          */
12665         intel_hpd_poll_fini(i915);
12666
12667         /*
12668          * MST topology needs to be suspended so we don't have any calls to
12669          * fbdev after it's finalized. MST will be destroyed later as part of
12670          * drm_mode_config_cleanup()
12671          */
12672         intel_dp_mst_suspend(i915);
12673
12674         /* poll work can call into fbdev, hence clean that up afterwards */
12675         intel_fbdev_fini(i915);
12676
12677         intel_unregister_dsm_handler();
12678
12679         intel_fbc_global_disable(i915);
12680
12681         /* flush any delayed tasks or pending work */
12682         flush_scheduled_work();
12683
12684         intel_hdcp_component_fini(i915);
12685
12686         intel_mode_config_cleanup(i915);
12687
12688         intel_overlay_cleanup(i915);
12689
12690         intel_gmbus_teardown(i915);
12691
12692         destroy_workqueue(i915->flip_wq);
12693         destroy_workqueue(i915->modeset_wq);
12694
12695         intel_fbc_cleanup_cfb(i915);
12696 }
12697
12698 /* part #3: call after gem init */
12699 void intel_modeset_driver_remove_nogem(struct drm_i915_private *i915)
12700 {
12701         intel_dmc_ucode_fini(i915);
12702
12703         intel_power_domains_driver_remove(i915);
12704
12705         intel_vga_unregister(i915);
12706
12707         intel_bios_driver_remove(i915);
12708 }
12709
12710 void intel_display_driver_register(struct drm_i915_private *i915)
12711 {
12712         if (!HAS_DISPLAY(i915))
12713                 return;
12714
12715         intel_display_debugfs_register(i915);
12716
12717         /* Must be done after probing outputs */
12718         intel_opregion_register(i915);
12719         acpi_video_register();
12720
12721         intel_audio_init(i915);
12722
12723         /*
12724          * Some ports require correctly set-up hpd registers for
12725          * detection to work properly (leading to ghost connected
12726          * connector status), e.g. VGA on gm45.  Hence we can only set
12727          * up the initial fbdev config after hpd irqs are fully
12728          * enabled. We do it last so that the async config cannot run
12729          * before the connectors are registered.
12730          */
12731         intel_fbdev_initial_config_async(&i915->drm);
12732
12733         /*
12734          * We need to coordinate the hotplugs with the asynchronous
12735          * fbdev configuration, for which we use the
12736          * fbdev->async_cookie.
12737          */
12738         drm_kms_helper_poll_init(&i915->drm);
12739 }
12740
12741 void intel_display_driver_unregister(struct drm_i915_private *i915)
12742 {
12743         if (!HAS_DISPLAY(i915))
12744                 return;
12745
12746         intel_fbdev_unregister(i915);
12747         intel_audio_deinit(i915);
12748
12749         /*
12750          * After flushing the fbdev (incl. a late async config which
12751          * will have delayed queuing of a hotplug event), then flush
12752          * the hotplug events.
12753          */
12754         drm_kms_helper_poll_fini(&i915->drm);
12755         drm_atomic_helper_shutdown(&i915->drm);
12756
12757         acpi_video_unregister();
12758         intel_opregion_unregister(i915);
12759 }