drivers/gpu/drm/i915/display/intel_display.c

   1 /*
   2  * Copyright © 2006-2007 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  21  * DEALINGS IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *      Eric Anholt <eric@anholt.net>
  25  */
  26
  27 #include <linux/i2c.h>
  28 #include <linux/input.h>
  29 #include <linux/intel-iommu.h>
  30 #include <linux/kernel.h>
  31 #include <linux/module.h>
  32 #include <linux/dma-resv.h>
  33 #include <linux/slab.h>
  34
  35 #include <drm/drm_atomic.h>
  36 #include <drm/drm_atomic_helper.h>
  37 #include <drm/drm_atomic_uapi.h>
  38 #include <drm/drm_damage_helper.h>
  39 #include <drm/drm_dp_helper.h>
  40 #include <drm/drm_edid.h>
  41 #include <drm/drm_fourcc.h>
  42 #include <drm/drm_plane_helper.h>
  43 #include <drm/drm_probe_helper.h>
  44 #include <drm/drm_rect.h>
  45
  46 #include "display/intel_crt.h"
  47 #include "display/intel_ddi.h"
  48 #include "display/intel_display_debugfs.h"
  49 #include "display/intel_dp.h"
  50 #include "display/intel_dp_mst.h"
  51 #include "display/intel_dpll.h"
  52 #include "display/intel_dpll_mgr.h"
  53 #include "display/intel_dsi.h"
  54 #include "display/intel_dvo.h"
  55 #include "display/intel_gmbus.h"
  56 #include "display/intel_hdmi.h"
  57 #include "display/intel_lvds.h"
  58 #include "display/intel_sdvo.h"
  59 #include "display/intel_tv.h"
  60 #include "display/intel_vdsc.h"
  61 #include "display/intel_vrr.h"
  62
  63 #include "gem/i915_gem_object.h"
  64
  65 #include "gt/intel_rps.h"
  66
  67 #include "i915_drv.h"
  68 #include "i915_trace.h"
  69 #include "intel_acpi.h"
  70 #include "intel_atomic.h"
  71 #include "intel_atomic_plane.h"
  72 #include "intel_bw.h"
  73 #include "intel_cdclk.h"
  74 #include "intel_color.h"
  75 #include "intel_crtc.h"
  76 #include "intel_csr.h"
  77 #include "intel_display_types.h"
  78 #include "intel_dp_link_training.h"
  79 #include "intel_fbc.h"
  80 #include "intel_fdi.h"
  81 #include "intel_fbdev.h"
  82 #include "intel_fifo_underrun.h"
  83 #include "intel_frontbuffer.h"
  84 #include "intel_hdcp.h"
  85 #include "intel_hotplug.h"
  86 #include "intel_overlay.h"
  87 #include "intel_pipe_crc.h"
  88 #include "intel_pm.h"
  89 #include "intel_pps.h"
  90 #include "intel_psr.h"
  91 #include "intel_quirks.h"
  92 #include "intel_sideband.h"
  93 #include "intel_sprite.h"
  94 #include "intel_tc.h"
  95 #include "intel_vga.h"
  96 #include "i9xx_plane.h"
  97
  98 static void i9xx_crtc_clock_get(struct intel_crtc *crtc,
  99                                 struct intel_crtc_state *pipe_config);
 100 static void ilk_pch_clock_get(struct intel_crtc *crtc,
 101                               struct intel_crtc_state *pipe_config);
 102
 103 static int intel_framebuffer_init(struct intel_framebuffer *ifb,
 104                                   struct drm_i915_gem_object *obj,
 105                                   struct drm_mode_fb_cmd2 *mode_cmd);
 106 static void intel_set_transcoder_timings(const struct intel_crtc_state *crtc_state);
 107 static void intel_set_pipe_src_size(const struct intel_crtc_state *crtc_state);
 108 static void intel_cpu_transcoder_set_m_n(const struct intel_crtc_state *crtc_state,
 109                                          const struct intel_link_m_n *m_n,
 110                                          const struct intel_link_m_n *m2_n2);
 111 static void i9xx_set_pipeconf(const struct intel_crtc_state *crtc_state);
 112 static void ilk_set_pipeconf(const struct intel_crtc_state *crtc_state);
 113 static void hsw_set_pipeconf(const struct intel_crtc_state *crtc_state);
 114 static void bdw_set_pipemisc(const struct intel_crtc_state *crtc_state);
 115 static void vlv_prepare_pll(struct intel_crtc *crtc,
 116                             const struct intel_crtc_state *pipe_config);
 117 static void chv_prepare_pll(struct intel_crtc *crtc,
 118                             const struct intel_crtc_state *pipe_config);
 119 static void skl_pfit_enable(const struct intel_crtc_state *crtc_state);
 120 static void ilk_pfit_enable(const struct intel_crtc_state *crtc_state);
 121 static void intel_modeset_setup_hw_state(struct drm_device *dev,
 122                                          struct drm_modeset_acquire_ctx *ctx);
 123
 124 /* returns HPLL frequency in kHz */
 125 int vlv_get_hpll_vco(struct drm_i915_private *dev_priv)
 126 {
 127         int hpll_freq, vco_freq[] = { 800, 1600, 2000, 2400 };
 128
 129         /* Obtain SKU information */
 130         hpll_freq = vlv_cck_read(dev_priv, CCK_FUSE_REG) &
 131                 CCK_FUSE_HPLL_FREQ_MASK;
 132
 133         return vco_freq[hpll_freq] * 1000;
 134 }
 135
 136 int vlv_get_cck_clock(struct drm_i915_private *dev_priv,
 137                       const char *name, u32 reg, int ref_freq)
 138 {
 139         u32 val;
 140         int divider;
 141
 142         val = vlv_cck_read(dev_priv, reg);
 143         divider = val & CCK_FREQUENCY_VALUES;
 144
 145         drm_WARN(&dev_priv->drm, (val & CCK_FREQUENCY_STATUS) !=
 146                  (divider << CCK_FREQUENCY_STATUS_SHIFT),
 147                  "%s change in progress\n", name);
 148
 149         return DIV_ROUND_CLOSEST(ref_freq << 1, divider + 1);
 150 }
 151
 152 int vlv_get_cck_clock_hpll(struct drm_i915_private *dev_priv,
 153                            const char *name, u32 reg)
 154 {
 155         int hpll;
 156
 157         vlv_cck_get(dev_priv);
 158
 159         if (dev_priv->hpll_freq == 0)
 160                 dev_priv->hpll_freq = vlv_get_hpll_vco(dev_priv);
 161
 162         hpll = vlv_get_cck_clock(dev_priv, name, reg, dev_priv->hpll_freq);
 163
 164         vlv_cck_put(dev_priv);
 165
 166         return hpll;
 167 }
 168
 169 static void intel_update_czclk(struct drm_i915_private *dev_priv)
 170 {
 171         if (!(IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)))
 172                 return;
 173
 174         dev_priv->czclk_freq = vlv_get_cck_clock_hpll(dev_priv, "czclk",
 175                                                       CCK_CZ_CLOCK_CONTROL);
 176
 177         drm_dbg(&dev_priv->drm, "CZ clock rate: %d kHz\n",
 178                 dev_priv->czclk_freq);
 179 }
 180
 181 /* WA Display #0827: Gen9:all */
 182 static void
 183 skl_wa_827(struct drm_i915_private *dev_priv, enum pipe pipe, bool enable)
 184 {
 185         if (enable)
 186                 intel_de_write(dev_priv, CLKGATE_DIS_PSL(pipe),
 187                                intel_de_read(dev_priv, CLKGATE_DIS_PSL(pipe)) | DUPS1_GATING_DIS | DUPS2_GATING_DIS);
 188         else
 189                 intel_de_write(dev_priv, CLKGATE_DIS_PSL(pipe),
 190                                intel_de_read(dev_priv, CLKGATE_DIS_PSL(pipe)) & ~(DUPS1_GATING_DIS | DUPS2_GATING_DIS));
 191 }
 192
 193 /* Wa_2006604312:icl,ehl */
 194 static void
 195 icl_wa_scalerclkgating(struct drm_i915_private *dev_priv, enum pipe pipe,
 196                        bool enable)
 197 {
 198         if (enable)
 199                 intel_de_write(dev_priv, CLKGATE_DIS_PSL(pipe),
 200                                intel_de_read(dev_priv, CLKGATE_DIS_PSL(pipe)) | DPFR_GATING_DIS);
 201         else
 202                 intel_de_write(dev_priv, CLKGATE_DIS_PSL(pipe),
 203                                intel_de_read(dev_priv, CLKGATE_DIS_PSL(pipe)) & ~DPFR_GATING_DIS);
 204 }
 205
 206 static bool
 207 is_trans_port_sync_slave(const struct intel_crtc_state *crtc_state)
 208 {
 209         return crtc_state->master_transcoder != INVALID_TRANSCODER;
 210 }
 211
 212 static bool
 213 is_trans_port_sync_master(const struct intel_crtc_state *crtc_state)
 214 {
 215         return crtc_state->sync_mode_slaves_mask != 0;
 216 }
 217
 218 bool
 219 is_trans_port_sync_mode(const struct intel_crtc_state *crtc_state)
 220 {
 221         return is_trans_port_sync_master(crtc_state) ||
 222                 is_trans_port_sync_slave(crtc_state);
 223 }
 224
 225 static bool pipe_scanline_is_moving(struct drm_i915_private *dev_priv,
 226                                     enum pipe pipe)
 227 {
 228         i915_reg_t reg = PIPEDSL(pipe);
 229         u32 line1, line2;
 230         u32 line_mask;
 231
 232         if (IS_GEN(dev_priv, 2))
 233                 line_mask = DSL_LINEMASK_GEN2;
 234         else
 235                 line_mask = DSL_LINEMASK_GEN3;
 236
 237         line1 = intel_de_read(dev_priv, reg) & line_mask;
 238         msleep(5);
 239         line2 = intel_de_read(dev_priv, reg) & line_mask;
 240
 241         return line1 != line2;
 242 }
 243
 244 static void wait_for_pipe_scanline_moving(struct intel_crtc *crtc, bool state)
 245 {
 246         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 247         enum pipe pipe = crtc->pipe;
 248
 249         /* Wait for the display line to settle/start moving */
 250         if (wait_for(pipe_scanline_is_moving(dev_priv, pipe) == state, 100))
 251                 drm_err(&dev_priv->drm,
 252                         "pipe %c scanline %s wait timed out\n",
 253                         pipe_name(pipe), onoff(state));
 254 }
 255
 256 static void intel_wait_for_pipe_scanline_stopped(struct intel_crtc *crtc)
 257 {
 258         wait_for_pipe_scanline_moving(crtc, false);
 259 }
 260
 261 static void intel_wait_for_pipe_scanline_moving(struct intel_crtc *crtc)
 262 {
 263         wait_for_pipe_scanline_moving(crtc, true);
 264 }
 265
 266 static void
 267 intel_wait_for_pipe_off(const struct intel_crtc_state *old_crtc_state)
 268 {
 269         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->uapi.crtc);
 270         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 271
 272         if (INTEL_GEN(dev_priv) >= 4) {
 273                 enum transcoder cpu_transcoder = old_crtc_state->cpu_transcoder;
 274                 i915_reg_t reg = PIPECONF(cpu_transcoder);
 275
 276                 /* Wait for the Pipe State to go off */
 277                 if (intel_de_wait_for_clear(dev_priv, reg,
 278                                             I965_PIPECONF_ACTIVE, 100))
 279                         drm_WARN(&dev_priv->drm, 1,
 280                                  "pipe_off wait timed out\n");
 281         } else {
 282                 intel_wait_for_pipe_scanline_stopped(crtc);
 283         }
 284 }
 285
 286 /* Only for pre-ILK configs */
 287 void assert_pll(struct drm_i915_private *dev_priv,
 288                 enum pipe pipe, bool state)
 289 {
 290         u32 val;
 291         bool cur_state;
 292
 293         val = intel_de_read(dev_priv, DPLL(pipe));
 294         cur_state = !!(val & DPLL_VCO_ENABLE);
 295         I915_STATE_WARN(cur_state != state,
 296              "PLL state assertion failure (expected %s, current %s)\n",
 297                         onoff(state), onoff(cur_state));
 298 }
 299
 300 /* XXX: the dsi pll is shared between MIPI DSI ports */
 301 void assert_dsi_pll(struct drm_i915_private *dev_priv, bool state)
 302 {
 303         u32 val;
 304         bool cur_state;
 305
 306         vlv_cck_get(dev_priv);
 307         val = vlv_cck_read(dev_priv, CCK_REG_DSI_PLL_CONTROL);
 308         vlv_cck_put(dev_priv);
 309
 310         cur_state = val & DSI_PLL_VCO_EN;
 311         I915_STATE_WARN(cur_state != state,
 312              "DSI PLL state assertion failure (expected %s, current %s)\n",
 313                         onoff(state), onoff(cur_state));
 314 }
 315
 316 static void assert_fdi_tx(struct drm_i915_private *dev_priv,
 317                           enum pipe pipe, bool state)
 318 {
 319         bool cur_state;
 320
 321         if (HAS_DDI(dev_priv)) {
 322                 /*
 323                  * DDI does not have a specific FDI_TX register.
 324                  *
 325                  * FDI is never fed from EDP transcoder
 326                  * so pipe->transcoder cast is fine here.
 327                  */
 328                 enum transcoder cpu_transcoder = (enum transcoder)pipe;
 329                 u32 val = intel_de_read(dev_priv,
 330                                         TRANS_DDI_FUNC_CTL(cpu_transcoder));
 331                 cur_state = !!(val & TRANS_DDI_FUNC_ENABLE);
 332         } else {
 333                 u32 val = intel_de_read(dev_priv, FDI_TX_CTL(pipe));
 334                 cur_state = !!(val & FDI_TX_ENABLE);
 335         }
 336         I915_STATE_WARN(cur_state != state,
 337              "FDI TX state assertion failure (expected %s, current %s)\n",
 338                         onoff(state), onoff(cur_state));
 339 }
 340 #define assert_fdi_tx_enabled(d, p) assert_fdi_tx(d, p, true)
 341 #define assert_fdi_tx_disabled(d, p) assert_fdi_tx(d, p, false)
 342
 343 static void assert_fdi_rx(struct drm_i915_private *dev_priv,
 344                           enum pipe pipe, bool state)
 345 {
 346         u32 val;
 347         bool cur_state;
 348
 349         val = intel_de_read(dev_priv, FDI_RX_CTL(pipe));
 350         cur_state = !!(val & FDI_RX_ENABLE);
 351         I915_STATE_WARN(cur_state != state,
 352              "FDI RX state assertion failure (expected %s, current %s)\n",
 353                         onoff(state), onoff(cur_state));
 354 }
 355 #define assert_fdi_rx_enabled(d, p) assert_fdi_rx(d, p, true)
 356 #define assert_fdi_rx_disabled(d, p) assert_fdi_rx(d, p, false)
 357
 358 static void assert_fdi_tx_pll_enabled(struct drm_i915_private *dev_priv,
 359                                       enum pipe pipe)
 360 {
 361         u32 val;
 362
 363         /* ILK FDI PLL is always enabled */
 364         if (IS_GEN(dev_priv, 5))
 365                 return;
 366
 367         /* On Haswell, DDI ports are responsible for the FDI PLL setup */
 368         if (HAS_DDI(dev_priv))
 369                 return;
 370
 371         val = intel_de_read(dev_priv, FDI_TX_CTL(pipe));
 372         I915_STATE_WARN(!(val & FDI_TX_PLL_ENABLE), "FDI TX PLL assertion failure, should be active but is disabled\n");
 373 }
 374
 375 void assert_fdi_rx_pll(struct drm_i915_private *dev_priv,
 376                        enum pipe pipe, bool state)
 377 {
 378         u32 val;
 379         bool cur_state;
 380
 381         val = intel_de_read(dev_priv, FDI_RX_CTL(pipe));
 382         cur_state = !!(val & FDI_RX_PLL_ENABLE);
 383         I915_STATE_WARN(cur_state != state,
 384              "FDI RX PLL assertion failure (expected %s, current %s)\n",
 385                         onoff(state), onoff(cur_state));
 386 }
 387
 388 void assert_panel_unlocked(struct drm_i915_private *dev_priv, enum pipe pipe)
 389 {
 390         i915_reg_t pp_reg;
 391         u32 val;
 392         enum pipe panel_pipe = INVALID_PIPE;
 393         bool locked = true;
 394
 395         if (drm_WARN_ON(&dev_priv->drm, HAS_DDI(dev_priv)))
 396                 return;
 397
 398         if (HAS_PCH_SPLIT(dev_priv)) {
 399                 u32 port_sel;
 400
 401                 pp_reg = PP_CONTROL(0);
 402                 port_sel = intel_de_read(dev_priv, PP_ON_DELAYS(0)) & PANEL_PORT_SELECT_MASK;
 403
 404                 switch (port_sel) {
 405                 case PANEL_PORT_SELECT_LVDS:
 406                         intel_lvds_port_enabled(dev_priv, PCH_LVDS, &panel_pipe);
 407                         break;
 408                 case PANEL_PORT_SELECT_DPA:
 409                         intel_dp_port_enabled(dev_priv, DP_A, PORT_A, &panel_pipe);
 410                         break;
 411                 case PANEL_PORT_SELECT_DPC:
 412                         intel_dp_port_enabled(dev_priv, PCH_DP_C, PORT_C, &panel_pipe);
 413                         break;
 414                 case PANEL_PORT_SELECT_DPD:
 415                         intel_dp_port_enabled(dev_priv, PCH_DP_D, PORT_D, &panel_pipe);
 416                         break;
 417                 default:
 418                         MISSING_CASE(port_sel);
 419                         break;
 420                 }
 421         } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
 422                 /* presumably write lock depends on pipe, not port select */
 423                 pp_reg = PP_CONTROL(pipe);
 424                 panel_pipe = pipe;
 425         } else {
 426                 u32 port_sel;
 427
 428                 pp_reg = PP_CONTROL(0);
 429                 port_sel = intel_de_read(dev_priv, PP_ON_DELAYS(0)) & PANEL_PORT_SELECT_MASK;
 430
 431                 drm_WARN_ON(&dev_priv->drm,
 432                             port_sel != PANEL_PORT_SELECT_LVDS);
 433                 intel_lvds_port_enabled(dev_priv, LVDS, &panel_pipe);
 434         }
 435
 436         val = intel_de_read(dev_priv, pp_reg);
 437         if (!(val & PANEL_POWER_ON) ||
 438             ((val & PANEL_UNLOCK_MASK) == PANEL_UNLOCK_REGS))
 439                 locked = false;
 440
 441         I915_STATE_WARN(panel_pipe == pipe && locked,
 442              "panel assertion failure, pipe %c regs locked\n",
 443              pipe_name(pipe));
 444 }
 445
 446 void assert_pipe(struct drm_i915_private *dev_priv,
 447                  enum transcoder cpu_transcoder, bool state)
 448 {
 449         bool cur_state;
 450         enum intel_display_power_domain power_domain;
 451         intel_wakeref_t wakeref;
 452
 453         /* we keep both pipes enabled on 830 */
 454         if (IS_I830(dev_priv))
 455                 state = true;
 456
 457         power_domain = POWER_DOMAIN_TRANSCODER(cpu_transcoder);
 458         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
 459         if (wakeref) {
 460                 u32 val = intel_de_read(dev_priv, PIPECONF(cpu_transcoder));
 461                 cur_state = !!(val & PIPECONF_ENABLE);
 462
 463                 intel_display_power_put(dev_priv, power_domain, wakeref);
 464         } else {
 465                 cur_state = false;
 466         }
 467
 468         I915_STATE_WARN(cur_state != state,
 469                         "transcoder %s assertion failure (expected %s, current %s)\n",
 470                         transcoder_name(cpu_transcoder),
 471                         onoff(state), onoff(cur_state));
 472 }
 473
 474 static void assert_plane(struct intel_plane *plane, bool state)
 475 {
 476         enum pipe pipe;
 477         bool cur_state;
 478
 479         cur_state = plane->get_hw_state(plane, &pipe);
 480
 481         I915_STATE_WARN(cur_state != state,
 482                         "%s assertion failure (expected %s, current %s)\n",
 483                         plane->base.name, onoff(state), onoff(cur_state));
 484 }
 485
 486 #define assert_plane_enabled(p) assert_plane(p, true)
 487 #define assert_plane_disabled(p) assert_plane(p, false)
 488
 489 static void assert_planes_disabled(struct intel_crtc *crtc)
 490 {
 491         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 492         struct intel_plane *plane;
 493
 494         for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane)
 495                 assert_plane_disabled(plane);
 496 }
 497
 498 void assert_pch_transcoder_disabled(struct drm_i915_private *dev_priv,
 499                                     enum pipe pipe)
 500 {
 501         u32 val;
 502         bool enabled;
 503
 504         val = intel_de_read(dev_priv, PCH_TRANSCONF(pipe));
 505         enabled = !!(val & TRANS_ENABLE);
 506         I915_STATE_WARN(enabled,
 507              "transcoder assertion failed, should be off on pipe %c but is still active\n",
 508              pipe_name(pipe));
 509 }
 510
 511 static void assert_pch_dp_disabled(struct drm_i915_private *dev_priv,
 512                                    enum pipe pipe, enum port port,
 513                                    i915_reg_t dp_reg)
 514 {
 515         enum pipe port_pipe;
 516         bool state;
 517
 518         state = intel_dp_port_enabled(dev_priv, dp_reg, port, &port_pipe);
 519
 520         I915_STATE_WARN(state && port_pipe == pipe,
 521                         "PCH DP %c enabled on transcoder %c, should be disabled\n",
 522                         port_name(port), pipe_name(pipe));
 523
 524         I915_STATE_WARN(HAS_PCH_IBX(dev_priv) && !state && port_pipe == PIPE_B,
 525                         "IBX PCH DP %c still using transcoder B\n",
 526                         port_name(port));
 527 }
 528
 529 static void assert_pch_hdmi_disabled(struct drm_i915_private *dev_priv,
 530                                      enum pipe pipe, enum port port,
 531                                      i915_reg_t hdmi_reg)
 532 {
 533         enum pipe port_pipe;
 534         bool state;
 535
 536         state = intel_sdvo_port_enabled(dev_priv, hdmi_reg, &port_pipe);
 537
 538         I915_STATE_WARN(state && port_pipe == pipe,
 539                         "PCH HDMI %c enabled on transcoder %c, should be disabled\n",
 540                         port_name(port), pipe_name(pipe));
 541
 542         I915_STATE_WARN(HAS_PCH_IBX(dev_priv) && !state && port_pipe == PIPE_B,
 543                         "IBX PCH HDMI %c still using transcoder B\n",
 544                         port_name(port));
 545 }
 546
 547 static void assert_pch_ports_disabled(struct drm_i915_private *dev_priv,
 548                                       enum pipe pipe)
 549 {
 550         enum pipe port_pipe;
 551
 552         assert_pch_dp_disabled(dev_priv, pipe, PORT_B, PCH_DP_B);
 553         assert_pch_dp_disabled(dev_priv, pipe, PORT_C, PCH_DP_C);
 554         assert_pch_dp_disabled(dev_priv, pipe, PORT_D, PCH_DP_D);
 555
 556         I915_STATE_WARN(intel_crt_port_enabled(dev_priv, PCH_ADPA, &port_pipe) &&
 557                         port_pipe == pipe,
 558                         "PCH VGA enabled on transcoder %c, should be disabled\n",
 559                         pipe_name(pipe));
 560
 561         I915_STATE_WARN(intel_lvds_port_enabled(dev_priv, PCH_LVDS, &port_pipe) &&
 562                         port_pipe == pipe,
 563                         "PCH LVDS enabled on transcoder %c, should be disabled\n",
 564                         pipe_name(pipe));
 565
 566         /* PCH SDVOB multiplex with HDMIB */
 567         assert_pch_hdmi_disabled(dev_priv, pipe, PORT_B, PCH_HDMIB);
 568         assert_pch_hdmi_disabled(dev_priv, pipe, PORT_C, PCH_HDMIC);
 569         assert_pch_hdmi_disabled(dev_priv, pipe, PORT_D, PCH_HDMID);
 570 }
 571
 572 static void _vlv_enable_pll(struct intel_crtc *crtc,
 573                             const struct intel_crtc_state *pipe_config)
 574 {
 575         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 576         enum pipe pipe = crtc->pipe;
 577
 578         intel_de_write(dev_priv, DPLL(pipe), pipe_config->dpll_hw_state.dpll);
 579         intel_de_posting_read(dev_priv, DPLL(pipe));
 580         udelay(150);
 581
 582         if (intel_de_wait_for_set(dev_priv, DPLL(pipe), DPLL_LOCK_VLV, 1))
 583                 drm_err(&dev_priv->drm, "DPLL %d failed to lock\n", pipe);
 584 }
 585
 586 static void vlv_enable_pll(struct intel_crtc *crtc,
 587                            const struct intel_crtc_state *pipe_config)
 588 {
 589         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 590         enum pipe pipe = crtc->pipe;
 591
 592         assert_pipe_disabled(dev_priv, pipe_config->cpu_transcoder);
 593
 594         /* PLL is protected by panel, make sure we can write it */
 595         assert_panel_unlocked(dev_priv, pipe);
 596
 597         if (pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE)
 598                 _vlv_enable_pll(crtc, pipe_config);
 599
 600         intel_de_write(dev_priv, DPLL_MD(pipe),
 601                        pipe_config->dpll_hw_state.dpll_md);
 602         intel_de_posting_read(dev_priv, DPLL_MD(pipe));
 603 }
 604
 605
 606 static void _chv_enable_pll(struct intel_crtc *crtc,
 607                             const struct intel_crtc_state *pipe_config)
 608 {
 609         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 610         enum pipe pipe = crtc->pipe;
 611         enum dpio_channel port = vlv_pipe_to_channel(pipe);
 612         u32 tmp;
 613
 614         vlv_dpio_get(dev_priv);
 615
 616         /* Enable back the 10bit clock to display controller */
 617         tmp = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW14(port));
 618         tmp |= DPIO_DCLKP_EN;
 619         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW14(port), tmp);
 620
 621         vlv_dpio_put(dev_priv);
 622
 623         /*
 624          * Need to wait > 100ns between dclkp clock enable bit and PLL enable.
 625          */
 626         udelay(1);
 627
 628         /* Enable PLL */
 629         intel_de_write(dev_priv, DPLL(pipe), pipe_config->dpll_hw_state.dpll);
 630
 631         /* Check PLL is locked */
 632         if (intel_de_wait_for_set(dev_priv, DPLL(pipe), DPLL_LOCK_VLV, 1))
 633                 drm_err(&dev_priv->drm, "PLL %d failed to lock\n", pipe);
 634 }
 635
 636 static void chv_enable_pll(struct intel_crtc *crtc,
 637                            const struct intel_crtc_state *pipe_config)
 638 {
 639         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 640         enum pipe pipe = crtc->pipe;
 641
 642         assert_pipe_disabled(dev_priv, pipe_config->cpu_transcoder);
 643
 644         /* PLL is protected by panel, make sure we can write it */
 645         assert_panel_unlocked(dev_priv, pipe);
 646
 647         if (pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE)
 648                 _chv_enable_pll(crtc, pipe_config);
 649
 650         if (pipe != PIPE_A) {
 651                 /*
 652                  * WaPixelRepeatModeFixForC0:chv
 653                  *
 654                  * DPLLCMD is AWOL. Use chicken bits to propagate
 655                  * the value from DPLLBMD to either pipe B or C.
 656                  */
 657                 intel_de_write(dev_priv, CBR4_VLV, CBR_DPLLBMD_PIPE(pipe));
 658                 intel_de_write(dev_priv, DPLL_MD(PIPE_B),
 659                                pipe_config->dpll_hw_state.dpll_md);
 660                 intel_de_write(dev_priv, CBR4_VLV, 0);
 661                 dev_priv->chv_dpll_md[pipe] = pipe_config->dpll_hw_state.dpll_md;
 662
 663                 /*
 664                  * DPLLB VGA mode also seems to cause problems.
 665                  * We should always have it disabled.
 666                  */
 667                 drm_WARN_ON(&dev_priv->drm,
 668                             (intel_de_read(dev_priv, DPLL(PIPE_B)) &
 669                              DPLL_VGA_MODE_DIS) == 0);
 670         } else {
 671                 intel_de_write(dev_priv, DPLL_MD(pipe),
 672                                pipe_config->dpll_hw_state.dpll_md);
 673                 intel_de_posting_read(dev_priv, DPLL_MD(pipe));
 674         }
 675 }
 676
 677 static bool i9xx_has_pps(struct drm_i915_private *dev_priv)
 678 {
 679         if (IS_I830(dev_priv))
 680                 return false;
 681
 682         return IS_PINEVIEW(dev_priv) || IS_MOBILE(dev_priv);
 683 }
 684
 685 static void i9xx_enable_pll(struct intel_crtc *crtc,
 686                             const struct intel_crtc_state *crtc_state)
 687 {
 688         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 689         i915_reg_t reg = DPLL(crtc->pipe);
 690         u32 dpll = crtc_state->dpll_hw_state.dpll;
 691         int i;
 692
 693         assert_pipe_disabled(dev_priv, crtc_state->cpu_transcoder);
 694
 695         /* PLL is protected by panel, make sure we can write it */
 696         if (i9xx_has_pps(dev_priv))
 697                 assert_panel_unlocked(dev_priv, crtc->pipe);
 698
 699         /*
 700          * Apparently we need to have VGA mode enabled prior to changing
 701          * the P1/P2 dividers. Otherwise the DPLL will keep using the old
 702          * dividers, even though the register value does change.
 703          */
 704         intel_de_write(dev_priv, reg, dpll & ~DPLL_VGA_MODE_DIS);
 705         intel_de_write(dev_priv, reg, dpll);
 706
 707         /* Wait for the clocks to stabilize. */
 708         intel_de_posting_read(dev_priv, reg);
 709         udelay(150);
 710
 711         if (INTEL_GEN(dev_priv) >= 4) {
 712                 intel_de_write(dev_priv, DPLL_MD(crtc->pipe),
 713                                crtc_state->dpll_hw_state.dpll_md);
 714         } else {
 715                 /* The pixel multiplier can only be updated once the
 716                  * DPLL is enabled and the clocks are stable.
 717                  *
 718                  * So write it again.
 719                  */
 720                 intel_de_write(dev_priv, reg, dpll);
 721         }
 722
 723         /* We do this three times for luck */
 724         for (i = 0; i < 3; i++) {
 725                 intel_de_write(dev_priv, reg, dpll);
 726                 intel_de_posting_read(dev_priv, reg);
 727                 udelay(150); /* wait for warmup */
 728         }
 729 }
 730
 731 static void i9xx_disable_pll(const struct intel_crtc_state *crtc_state)
 732 {
 733         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 734         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 735         enum pipe pipe = crtc->pipe;
 736
 737         /* Don't disable pipe or pipe PLLs if needed */
 738         if (IS_I830(dev_priv))
 739                 return;
 740
 741         /* Make sure the pipe isn't still relying on us */
 742         assert_pipe_disabled(dev_priv, crtc_state->cpu_transcoder);
 743
 744         intel_de_write(dev_priv, DPLL(pipe), DPLL_VGA_MODE_DIS);
 745         intel_de_posting_read(dev_priv, DPLL(pipe));
 746 }
 747
 748 static void vlv_disable_pll(struct drm_i915_private *dev_priv, enum pipe pipe)
 749 {
 750         u32 val;
 751
 752         /* Make sure the pipe isn't still relying on us */
 753         assert_pipe_disabled(dev_priv, (enum transcoder)pipe);
 754
 755         val = DPLL_INTEGRATED_REF_CLK_VLV |
 756                 DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
 757         if (pipe != PIPE_A)
 758                 val |= DPLL_INTEGRATED_CRI_CLK_VLV;
 759
 760         intel_de_write(dev_priv, DPLL(pipe), val);
 761         intel_de_posting_read(dev_priv, DPLL(pipe));
 762 }
 763
 764 static void chv_disable_pll(struct drm_i915_private *dev_priv, enum pipe pipe)
 765 {
 766         enum dpio_channel port = vlv_pipe_to_channel(pipe);
 767         u32 val;
 768
 769         /* Make sure the pipe isn't still relying on us */
 770         assert_pipe_disabled(dev_priv, (enum transcoder)pipe);
 771
 772         val = DPLL_SSC_REF_CLK_CHV |
 773                 DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
 774         if (pipe != PIPE_A)
 775                 val |= DPLL_INTEGRATED_CRI_CLK_VLV;
 776
 777         intel_de_write(dev_priv, DPLL(pipe), val);
 778         intel_de_posting_read(dev_priv, DPLL(pipe));
 779
 780         vlv_dpio_get(dev_priv);
 781
 782         /* Disable 10bit clock to display controller */
 783         val = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW14(port));
 784         val &= ~DPIO_DCLKP_EN;
 785         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW14(port), val);
 786
 787         vlv_dpio_put(dev_priv);
 788 }
 789
 790 void vlv_wait_port_ready(struct drm_i915_private *dev_priv,
 791                          struct intel_digital_port *dig_port,
 792                          unsigned int expected_mask)
 793 {
 794         u32 port_mask;
 795         i915_reg_t dpll_reg;
 796
 797         switch (dig_port->base.port) {
 798         case PORT_B:
 799                 port_mask = DPLL_PORTB_READY_MASK;
 800                 dpll_reg = DPLL(0);
 801                 break;
 802         case PORT_C:
 803                 port_mask = DPLL_PORTC_READY_MASK;
 804                 dpll_reg = DPLL(0);
 805                 expected_mask <<= 4;
 806                 break;
 807         case PORT_D:
 808                 port_mask = DPLL_PORTD_READY_MASK;
 809                 dpll_reg = DPIO_PHY_STATUS;
 810                 break;
 811         default:
 812                 BUG();
 813         }
 814
 815         if (intel_de_wait_for_register(dev_priv, dpll_reg,
 816                                        port_mask, expected_mask, 1000))
 817                 drm_WARN(&dev_priv->drm, 1,
 818                          "timed out waiting for [ENCODER:%d:%s] port ready: got 0x%x, expected 0x%x\n",
 819                          dig_port->base.base.base.id, dig_port->base.base.name,
 820                          intel_de_read(dev_priv, dpll_reg) & port_mask,
 821                          expected_mask);
 822 }
 823
 824 static void ilk_enable_pch_transcoder(const struct intel_crtc_state *crtc_state)
 825 {
 826         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 827         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 828         enum pipe pipe = crtc->pipe;
 829         i915_reg_t reg;
 830         u32 val, pipeconf_val;
 831
 832         /* Make sure PCH DPLL is enabled */
 833         assert_shared_dpll_enabled(dev_priv, crtc_state->shared_dpll);
 834
 835         /* FDI must be feeding us bits for PCH ports */
 836         assert_fdi_tx_enabled(dev_priv, pipe);
 837         assert_fdi_rx_enabled(dev_priv, pipe);
 838
 839         if (HAS_PCH_CPT(dev_priv)) {
 840                 reg = TRANS_CHICKEN2(pipe);
 841                 val = intel_de_read(dev_priv, reg);
 842                 /*
 843                  * Workaround: Set the timing override bit
 844                  * before enabling the pch transcoder.
 845                  */
 846                 val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
 847                 /* Configure frame start delay to match the CPU */
 848                 val &= ~TRANS_CHICKEN2_FRAME_START_DELAY_MASK;
 849                 val |= TRANS_CHICKEN2_FRAME_START_DELAY(dev_priv->framestart_delay - 1);
 850                 intel_de_write(dev_priv, reg, val);
 851         }
 852
 853         reg = PCH_TRANSCONF(pipe);
 854         val = intel_de_read(dev_priv, reg);
 855         pipeconf_val = intel_de_read(dev_priv, PIPECONF(pipe));
 856
 857         if (HAS_PCH_IBX(dev_priv)) {
 858                 /* Configure frame start delay to match the CPU */
 859                 val &= ~TRANS_FRAME_START_DELAY_MASK;
 860                 val |= TRANS_FRAME_START_DELAY(dev_priv->framestart_delay - 1);
 861
 862                 /*
 863                  * Make the BPC in transcoder be consistent with
 864                  * that in pipeconf reg. For HDMI we must use 8bpc
 865                  * here for both 8bpc and 12bpc.
 866                  */
 867                 val &= ~PIPECONF_BPC_MASK;
 868                 if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI))
 869                         val |= PIPECONF_8BPC;
 870                 else
 871                         val |= pipeconf_val & PIPECONF_BPC_MASK;
 872         }
 873
 874         val &= ~TRANS_INTERLACE_MASK;
 875         if ((pipeconf_val & PIPECONF_INTERLACE_MASK) == PIPECONF_INTERLACED_ILK) {
 876                 if (HAS_PCH_IBX(dev_priv) &&
 877                     intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO))
 878                         val |= TRANS_LEGACY_INTERLACED_ILK;
 879                 else
 880                         val |= TRANS_INTERLACED;
 881         } else {
 882                 val |= TRANS_PROGRESSIVE;
 883         }
 884
 885         intel_de_write(dev_priv, reg, val | TRANS_ENABLE);
 886         if (intel_de_wait_for_set(dev_priv, reg, TRANS_STATE_ENABLE, 100))
 887                 drm_err(&dev_priv->drm, "failed to enable transcoder %c\n",
 888                         pipe_name(pipe));
 889 }
 890
 891 static void lpt_enable_pch_transcoder(struct drm_i915_private *dev_priv,
 892                                       enum transcoder cpu_transcoder)
 893 {
 894         u32 val, pipeconf_val;
 895
 896         /* FDI must be feeding us bits for PCH ports */
 897         assert_fdi_tx_enabled(dev_priv, (enum pipe) cpu_transcoder);
 898         assert_fdi_rx_enabled(dev_priv, PIPE_A);
 899
 900         val = intel_de_read(dev_priv, TRANS_CHICKEN2(PIPE_A));
 901         /* Workaround: set timing override bit. */
 902         val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
 903         /* Configure frame start delay to match the CPU */
 904         val &= ~TRANS_CHICKEN2_FRAME_START_DELAY_MASK;
 905         val |= TRANS_CHICKEN2_FRAME_START_DELAY(dev_priv->framestart_delay - 1);
 906         intel_de_write(dev_priv, TRANS_CHICKEN2(PIPE_A), val);
 907
 908         val = TRANS_ENABLE;
 909         pipeconf_val = intel_de_read(dev_priv, PIPECONF(cpu_transcoder));
 910
 911         if ((pipeconf_val & PIPECONF_INTERLACE_MASK_HSW) ==
 912             PIPECONF_INTERLACED_ILK)
 913                 val |= TRANS_INTERLACED;
 914         else
 915                 val |= TRANS_PROGRESSIVE;
 916
 917         intel_de_write(dev_priv, LPT_TRANSCONF, val);
 918         if (intel_de_wait_for_set(dev_priv, LPT_TRANSCONF,
 919                                   TRANS_STATE_ENABLE, 100))
 920                 drm_err(&dev_priv->drm, "Failed to enable PCH transcoder\n");
 921 }
 922
 923 static void ilk_disable_pch_transcoder(struct drm_i915_private *dev_priv,
 924                                        enum pipe pipe)
 925 {
 926         i915_reg_t reg;
 927         u32 val;
 928
 929         /* FDI relies on the transcoder */
 930         assert_fdi_tx_disabled(dev_priv, pipe);
 931         assert_fdi_rx_disabled(dev_priv, pipe);
 932
 933         /* Ports must be off as well */
 934         assert_pch_ports_disabled(dev_priv, pipe);
 935
 936         reg = PCH_TRANSCONF(pipe);
 937         val = intel_de_read(dev_priv, reg);
 938         val &= ~TRANS_ENABLE;
 939         intel_de_write(dev_priv, reg, val);
 940         /* wait for PCH transcoder off, transcoder state */
 941         if (intel_de_wait_for_clear(dev_priv, reg, TRANS_STATE_ENABLE, 50))
 942                 drm_err(&dev_priv->drm, "failed to disable transcoder %c\n",
 943                         pipe_name(pipe));
 944
 945         if (HAS_PCH_CPT(dev_priv)) {
 946                 /* Workaround: Clear the timing override chicken bit again. */
 947                 reg = TRANS_CHICKEN2(pipe);
 948                 val = intel_de_read(dev_priv, reg);
 949                 val &= ~TRANS_CHICKEN2_TIMING_OVERRIDE;
 950                 intel_de_write(dev_priv, reg, val);
 951         }
 952 }
 953
 954 void lpt_disable_pch_transcoder(struct drm_i915_private *dev_priv)
 955 {
 956         u32 val;
 957
 958         val = intel_de_read(dev_priv, LPT_TRANSCONF);
 959         val &= ~TRANS_ENABLE;
 960         intel_de_write(dev_priv, LPT_TRANSCONF, val);
 961         /* wait for PCH transcoder off, transcoder state */
 962         if (intel_de_wait_for_clear(dev_priv, LPT_TRANSCONF,
 963                                     TRANS_STATE_ENABLE, 50))
 964                 drm_err(&dev_priv->drm, "Failed to disable PCH transcoder\n");
 965
 966         /* Workaround: clear timing override bit. */
 967         val = intel_de_read(dev_priv, TRANS_CHICKEN2(PIPE_A));
 968         val &= ~TRANS_CHICKEN2_TIMING_OVERRIDE;
 969         intel_de_write(dev_priv, TRANS_CHICKEN2(PIPE_A), val);
 970 }
 971
 972 enum pipe intel_crtc_pch_transcoder(struct intel_crtc *crtc)
 973 {
 974         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 975
 976         if (HAS_PCH_LPT(dev_priv))
 977                 return PIPE_A;
 978         else
 979                 return crtc->pipe;
 980 }
 981
 982 void intel_enable_pipe(const struct intel_crtc_state *new_crtc_state)
 983 {
 984         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
 985         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 986         enum transcoder cpu_transcoder = new_crtc_state->cpu_transcoder;
 987         enum pipe pipe = crtc->pipe;
 988         i915_reg_t reg;
 989         u32 val;
 990
 991         drm_dbg_kms(&dev_priv->drm, "enabling pipe %c\n", pipe_name(pipe));
 992
 993         assert_planes_disabled(crtc);
 994
 995         /*
 996          * A pipe without a PLL won't actually be able to drive bits from
 997          * a plane.  On ILK+ the pipe PLLs are integrated, so we don't
 998          * need the check.
 999          */
1000         if (HAS_GMCH(dev_priv)) {
1001                 if (intel_crtc_has_type(new_crtc_state, INTEL_OUTPUT_DSI))
1002                         assert_dsi_pll_enabled(dev_priv);
1003                 else
1004                         assert_pll_enabled(dev_priv, pipe);
1005         } else {
1006                 if (new_crtc_state->has_pch_encoder) {
1007                         /* if driving the PCH, we need FDI enabled */
1008                         assert_fdi_rx_pll_enabled(dev_priv,
1009                                                   intel_crtc_pch_transcoder(crtc));
1010                         assert_fdi_tx_pll_enabled(dev_priv,
1011                                                   (enum pipe) cpu_transcoder);
1012                 }
1013                 /* FIXME: assert CPU port conditions for SNB+ */
1014         }
1015
1016         trace_intel_pipe_enable(crtc);
1017
1018         reg = PIPECONF(cpu_transcoder);
1019         val = intel_de_read(dev_priv, reg);
1020         if (val & PIPECONF_ENABLE) {
1021                 /* we keep both pipes enabled on 830 */
1022                 drm_WARN_ON(&dev_priv->drm, !IS_I830(dev_priv));
1023                 return;
1024         }
1025
1026         intel_de_write(dev_priv, reg, val | PIPECONF_ENABLE);
1027         intel_de_posting_read(dev_priv, reg);
1028
1029         /*
1030          * Until the pipe starts PIPEDSL reads will return a stale value,
1031          * which causes an apparent vblank timestamp jump when PIPEDSL
1032          * resets to its proper value. That also messes up the frame count
1033          * when it's derived from the timestamps. So let's wait for the
1034          * pipe to start properly before we call drm_crtc_vblank_on()
1035          */
1036         if (intel_crtc_max_vblank_count(new_crtc_state) == 0)
1037                 intel_wait_for_pipe_scanline_moving(crtc);
1038 }
1039
1040 void intel_disable_pipe(const struct intel_crtc_state *old_crtc_state)
1041 {
1042         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->uapi.crtc);
1043         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1044         enum transcoder cpu_transcoder = old_crtc_state->cpu_transcoder;
1045         enum pipe pipe = crtc->pipe;
1046         i915_reg_t reg;
1047         u32 val;
1048
1049         drm_dbg_kms(&dev_priv->drm, "disabling pipe %c\n", pipe_name(pipe));
1050
1051         /*
1052          * Make sure planes won't keep trying to pump pixels to us,
1053          * or we might hang the display.
1054          */
1055         assert_planes_disabled(crtc);
1056
1057         trace_intel_pipe_disable(crtc);
1058
1059         reg = PIPECONF(cpu_transcoder);
1060         val = intel_de_read(dev_priv, reg);
1061         if ((val & PIPECONF_ENABLE) == 0)
1062                 return;
1063
1064         /*
1065          * Double wide has implications for planes
1066          * so best keep it disabled when not needed.
1067          */
1068         if (old_crtc_state->double_wide)
1069                 val &= ~PIPECONF_DOUBLE_WIDE;
1070
1071         /* Don't disable pipe or pipe PLLs if needed */
1072         if (!IS_I830(dev_priv))
1073                 val &= ~PIPECONF_ENABLE;
1074
1075         intel_de_write(dev_priv, reg, val);
1076         if ((val & PIPECONF_ENABLE) == 0)
1077                 intel_wait_for_pipe_off(old_crtc_state);
1078 }
1079
1080 static unsigned int intel_tile_size(const struct drm_i915_private *dev_priv)
1081 {
1082         return IS_GEN(dev_priv, 2) ? 2048 : 4096;
1083 }
1084
1085 static bool is_ccs_plane(const struct drm_framebuffer *fb, int plane)
1086 {
1087         if (!is_ccs_modifier(fb->modifier))
1088                 return false;
1089
1090         return plane >= fb->format->num_planes / 2;
1091 }
1092
1093 static bool is_gen12_ccs_modifier(u64 modifier)
1094 {
1095         return modifier == I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS ||
1096                modifier == I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS_CC ||
1097                modifier == I915_FORMAT_MOD_Y_TILED_GEN12_MC_CCS;
1098 }
1099
1100 static bool is_gen12_ccs_plane(const struct drm_framebuffer *fb, int plane)
1101 {
1102         return is_gen12_ccs_modifier(fb->modifier) && is_ccs_plane(fb, plane);
1103 }
1104
1105 static bool is_gen12_ccs_cc_plane(const struct drm_framebuffer *fb, int plane)
1106 {
1107         return fb->modifier == I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS_CC &&
1108                plane == 2;
1109 }
1110
1111 static bool is_aux_plane(const struct drm_framebuffer *fb, int plane)
1112 {
1113         if (is_ccs_modifier(fb->modifier))
1114                 return is_ccs_plane(fb, plane);
1115
1116         return plane == 1;
1117 }
1118
1119 static int main_to_ccs_plane(const struct drm_framebuffer *fb, int main_plane)
1120 {
1121         drm_WARN_ON(fb->dev, !is_ccs_modifier(fb->modifier) ||
1122                     (main_plane && main_plane >= fb->format->num_planes / 2));
1123
1124         return fb->format->num_planes / 2 + main_plane;
1125 }
1126
1127 static int ccs_to_main_plane(const struct drm_framebuffer *fb, int ccs_plane)
1128 {
1129         drm_WARN_ON(fb->dev, !is_ccs_modifier(fb->modifier) ||
1130                     ccs_plane < fb->format->num_planes / 2);
1131
1132         if (is_gen12_ccs_cc_plane(fb, ccs_plane))
1133                 return 0;
1134
1135         return ccs_plane - fb->format->num_planes / 2;
1136 }
1137
1138 int intel_main_to_aux_plane(const struct drm_framebuffer *fb, int main_plane)
1139 {
1140         struct drm_i915_private *i915 = to_i915(fb->dev);
1141
1142         if (is_ccs_modifier(fb->modifier))
1143                 return main_to_ccs_plane(fb, main_plane);
1144         else if (INTEL_GEN(i915) < 11 &&
1145                  intel_format_info_is_yuv_semiplanar(fb->format, fb->modifier))
1146                 return 1;
1147         else
1148                 return 0;
1149 }
1150
1151 bool
1152 intel_format_info_is_yuv_semiplanar(const struct drm_format_info *info,
1153                                     u64 modifier)
1154 {
1155         return info->is_yuv &&
1156                info->num_planes == (is_ccs_modifier(modifier) ? 4 : 2);
1157 }
1158
1159 static bool is_semiplanar_uv_plane(const struct drm_framebuffer *fb,
1160                                    int color_plane)
1161 {
1162         return intel_format_info_is_yuv_semiplanar(fb->format, fb->modifier) &&
1163                color_plane == 1;
1164 }
1165
1166 static unsigned int
1167 intel_tile_width_bytes(const struct drm_framebuffer *fb, int color_plane)
1168 {
1169         struct drm_i915_private *dev_priv = to_i915(fb->dev);
1170         unsigned int cpp = fb->format->cpp[color_plane];
1171
1172         switch (fb->modifier) {
1173         case DRM_FORMAT_MOD_LINEAR:
1174                 return intel_tile_size(dev_priv);
1175         case I915_FORMAT_MOD_X_TILED:
1176                 if (IS_GEN(dev_priv, 2))
1177                         return 128;
1178                 else
1179                         return 512;
1180         case I915_FORMAT_MOD_Y_TILED_CCS:
1181                 if (is_ccs_plane(fb, color_plane))
1182                         return 128;
1183                 fallthrough;
1184         case I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS:
1185         case I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS_CC:
1186         case I915_FORMAT_MOD_Y_TILED_GEN12_MC_CCS:
1187                 if (is_ccs_plane(fb, color_plane))
1188                         return 64;
1189                 fallthrough;
1190         case I915_FORMAT_MOD_Y_TILED:
1191                 if (IS_GEN(dev_priv, 2) || HAS_128_BYTE_Y_TILING(dev_priv))
1192                         return 128;
1193                 else
1194                         return 512;
1195         case I915_FORMAT_MOD_Yf_TILED_CCS:
1196                 if (is_ccs_plane(fb, color_plane))
1197                         return 128;
1198                 fallthrough;
1199         case I915_FORMAT_MOD_Yf_TILED:
1200                 switch (cpp) {
1201                 case 1:
1202                         return 64;
1203                 case 2:
1204                 case 4:
1205                         return 128;
1206                 case 8:
1207                 case 16:
1208                         return 256;
1209                 default:
1210                         MISSING_CASE(cpp);
1211                         return cpp;
1212                 }
1213                 break;
1214         default:
1215                 MISSING_CASE(fb->modifier);
1216                 return cpp;
1217         }
1218 }
1219
1220 static unsigned int
1221 intel_tile_height(const struct drm_framebuffer *fb, int color_plane)
1222 {
1223         if (is_gen12_ccs_plane(fb, color_plane))
1224                 return 1;
1225
1226         return intel_tile_size(to_i915(fb->dev)) /
1227                 intel_tile_width_bytes(fb, color_plane);
1228 }
1229
1230 /* Return the tile dimensions in pixel units */
1231 static void intel_tile_dims(const struct drm_framebuffer *fb, int color_plane,
1232                             unsigned int *tile_width,
1233                             unsigned int *tile_height)
1234 {
1235         unsigned int tile_width_bytes = intel_tile_width_bytes(fb, color_plane);
1236         unsigned int cpp = fb->format->cpp[color_plane];
1237
1238         *tile_width = tile_width_bytes / cpp;
1239         *tile_height = intel_tile_height(fb, color_plane);
1240 }
1241
1242 static unsigned int intel_tile_row_size(const struct drm_framebuffer *fb,
1243                                         int color_plane)
1244 {
1245         unsigned int tile_width, tile_height;
1246
1247         intel_tile_dims(fb, color_plane, &tile_width, &tile_height);
1248
1249         return fb->pitches[color_plane] * tile_height;
1250 }
1251
1252 unsigned int
1253 intel_fb_align_height(const struct drm_framebuffer *fb,
1254                       int color_plane, unsigned int height)
1255 {
1256         unsigned int tile_height = intel_tile_height(fb, color_plane);
1257
1258         return ALIGN(height, tile_height);
1259 }
1260
1261 unsigned int intel_rotation_info_size(const struct intel_rotation_info *rot_info)
1262 {
1263         unsigned int size = 0;
1264         int i;
1265
1266         for (i = 0 ; i < ARRAY_SIZE(rot_info->plane); i++)
1267                 size += rot_info->plane[i].width * rot_info->plane[i].height;
1268
1269         return size;
1270 }
1271
1272 unsigned int intel_remapped_info_size(const struct intel_remapped_info *rem_info)
1273 {
1274         unsigned int size = 0;
1275         int i;
1276
1277         for (i = 0 ; i < ARRAY_SIZE(rem_info->plane); i++)
1278                 size += rem_info->plane[i].width * rem_info->plane[i].height;
1279
1280         return size;
1281 }
1282
1283 static void
1284 intel_fill_fb_ggtt_view(struct i915_ggtt_view *view,
1285                         const struct drm_framebuffer *fb,
1286                         unsigned int rotation)
1287 {
1288         view->type = I915_GGTT_VIEW_NORMAL;
1289         if (drm_rotation_90_or_270(rotation)) {
1290                 view->type = I915_GGTT_VIEW_ROTATED;
1291                 view->rotated = to_intel_framebuffer(fb)->rot_info;
1292         }
1293 }
1294
1295 static unsigned int intel_cursor_alignment(const struct drm_i915_private *dev_priv)
1296 {
1297         if (IS_I830(dev_priv))
1298                 return 16 * 1024;
1299         else if (IS_I85X(dev_priv))
1300                 return 256;
1301         else if (IS_I845G(dev_priv) || IS_I865G(dev_priv))
1302                 return 32;
1303         else
1304                 return 4 * 1024;
1305 }
1306
1307 static unsigned int intel_linear_alignment(const struct drm_i915_private *dev_priv)
1308 {
1309         if (INTEL_GEN(dev_priv) >= 9)
1310                 return 256 * 1024;
1311         else if (IS_I965G(dev_priv) || IS_I965GM(dev_priv) ||
1312                  IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
1313                 return 128 * 1024;
1314         else if (INTEL_GEN(dev_priv) >= 4)
1315                 return 4 * 1024;
1316         else
1317                 return 0;
1318 }
1319
1320 static bool has_async_flips(struct drm_i915_private *i915)
1321 {
1322         return INTEL_GEN(i915) >= 5;
1323 }
1324
1325 static unsigned int intel_surf_alignment(const struct drm_framebuffer *fb,
1326                                          int color_plane)
1327 {
1328         struct drm_i915_private *dev_priv = to_i915(fb->dev);
1329
1330         /* AUX_DIST needs only 4K alignment */
1331         if ((INTEL_GEN(dev_priv) < 12 && is_aux_plane(fb, color_plane)) ||
1332             is_ccs_plane(fb, color_plane))
1333                 return 4096;
1334
1335         switch (fb->modifier) {
1336         case DRM_FORMAT_MOD_LINEAR:
1337                 return intel_linear_alignment(dev_priv);
1338         case I915_FORMAT_MOD_X_TILED:
1339                 if (has_async_flips(dev_priv))
1340                         return 256 * 1024;
1341                 return 0;
1342         case I915_FORMAT_MOD_Y_TILED_GEN12_MC_CCS:
1343                 if (is_semiplanar_uv_plane(fb, color_plane))
1344                         return intel_tile_row_size(fb, color_plane);
1345                 fallthrough;
1346         case I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS:
1347         case I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS_CC:
1348                 return 16 * 1024;
1349         case I915_FORMAT_MOD_Y_TILED_CCS:
1350         case I915_FORMAT_MOD_Yf_TILED_CCS:
1351         case I915_FORMAT_MOD_Y_TILED:
1352                 if (INTEL_GEN(dev_priv) >= 12 &&
1353                     is_semiplanar_uv_plane(fb, color_plane))
1354                         return intel_tile_row_size(fb, color_plane);
1355                 fallthrough;
1356         case I915_FORMAT_MOD_Yf_TILED:
1357                 return 1 * 1024 * 1024;
1358         default:
1359                 MISSING_CASE(fb->modifier);
1360                 return 0;
1361         }
1362 }
1363
1364 static bool intel_plane_uses_fence(const struct intel_plane_state *plane_state)
1365 {
1366         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1367         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1368
1369         return INTEL_GEN(dev_priv) < 4 ||
1370                 (plane->has_fbc &&
1371                  plane_state->view.type == I915_GGTT_VIEW_NORMAL);
1372 }
1373
1374 struct i915_vma *
1375 intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb,
1376                            const struct i915_ggtt_view *view,
1377                            bool uses_fence,
1378                            unsigned long *out_flags)
1379 {
1380         struct drm_device *dev = fb->dev;
1381         struct drm_i915_private *dev_priv = to_i915(dev);
1382         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
1383         intel_wakeref_t wakeref;
1384         struct i915_vma *vma;
1385         unsigned int pinctl;
1386         u32 alignment;
1387
1388         if (drm_WARN_ON(dev, !i915_gem_object_is_framebuffer(obj)))
1389                 return ERR_PTR(-EINVAL);
1390
1391         alignment = intel_surf_alignment(fb, 0);
1392         if (drm_WARN_ON(dev, alignment && !is_power_of_2(alignment)))
1393                 return ERR_PTR(-EINVAL);
1394
1395         /* Note that the w/a also requires 64 PTE of padding following the
1396          * bo. We currently fill all unused PTE with the shadow page and so
1397          * we should always have valid PTE following the scanout preventing
1398          * the VT-d warning.
1399          */
1400         if (intel_scanout_needs_vtd_wa(dev_priv) && alignment < 256 * 1024)
1401                 alignment = 256 * 1024;
1402
1403         /*
1404          * Global gtt pte registers are special registers which actually forward
1405          * writes to a chunk of system memory. Which means that there is no risk
1406          * that the register values disappear as soon as we call
1407          * intel_runtime_pm_put(), so it is correct to wrap only the
1408          * pin/unpin/fence and not more.
1409          */
1410         wakeref = intel_runtime_pm_get(&dev_priv->runtime_pm);
1411
1412         atomic_inc(&dev_priv->gpu_error.pending_fb_pin);
1413
1414         /*
1415          * Valleyview is definitely limited to scanning out the first
1416          * 512MiB. Lets presume this behaviour was inherited from the
1417          * g4x display engine and that all earlier gen are similarly
1418          * limited. Testing suggests that it is a little more
1419          * complicated than this. For example, Cherryview appears quite
1420          * happy to scanout from anywhere within its global aperture.
1421          */
1422         pinctl = 0;
1423         if (HAS_GMCH(dev_priv))
1424                 pinctl |= PIN_MAPPABLE;
1425
1426         vma = i915_gem_object_pin_to_display_plane(obj,
1427                                                    alignment, view, pinctl);
1428         if (IS_ERR(vma))
1429                 goto err;
1430
1431         if (uses_fence && i915_vma_is_map_and_fenceable(vma)) {
1432                 int ret;
1433
1434                 /*
1435                  * Install a fence for tiled scan-out. Pre-i965 always needs a
1436                  * fence, whereas 965+ only requires a fence if using
1437                  * framebuffer compression.  For simplicity, we always, when
1438                  * possible, install a fence as the cost is not that onerous.
1439                  *
1440                  * If we fail to fence the tiled scanout, then either the
1441                  * modeset will reject the change (which is highly unlikely as
1442                  * the affected systems, all but one, do not have unmappable
1443                  * space) or we will not be able to enable full powersaving
1444                  * techniques (also likely not to apply due to various limits
1445                  * FBC and the like impose on the size of the buffer, which
1446                  * presumably we violated anyway with this unmappable buffer).
1447                  * Anyway, it is presumably better to stumble onwards with
1448                  * something and try to run the system in a "less than optimal"
1449                  * mode that matches the user configuration.
1450                  */
1451                 ret = i915_vma_pin_fence(vma);
1452                 if (ret != 0 && INTEL_GEN(dev_priv) < 4) {
1453                         i915_gem_object_unpin_from_display_plane(vma);
1454                         vma = ERR_PTR(ret);
1455                         goto err;
1456                 }
1457
1458                 if (ret == 0 && vma->fence)
1459                         *out_flags |= PLANE_HAS_FENCE;
1460         }
1461
1462         i915_vma_get(vma);
1463 err:
1464         atomic_dec(&dev_priv->gpu_error.pending_fb_pin);
1465         intel_runtime_pm_put(&dev_priv->runtime_pm, wakeref);
1466         return vma;
1467 }
1468
1469 void intel_unpin_fb_vma(struct i915_vma *vma, unsigned long flags)
1470 {
1471         i915_gem_object_lock(vma->obj, NULL);
1472         if (flags & PLANE_HAS_FENCE)
1473                 i915_vma_unpin_fence(vma);
1474         i915_gem_object_unpin_from_display_plane(vma);
1475         i915_gem_object_unlock(vma->obj);
1476
1477         i915_vma_put(vma);
1478 }
1479
1480 static int intel_fb_pitch(const struct drm_framebuffer *fb, int color_plane,
1481                           unsigned int rotation)
1482 {
1483         if (drm_rotation_90_or_270(rotation))
1484                 return to_intel_framebuffer(fb)->rotated[color_plane].pitch;
1485         else
1486                 return fb->pitches[color_plane];
1487 }
1488
1489 /*
1490  * Convert the x/y offsets into a linear offset.
1491  * Only valid with 0/180 degree rotation, which is fine since linear
1492  * offset is only used with linear buffers on pre-hsw and tiled buffers
1493  * with gen2/3, and 90/270 degree rotations isn't supported on any of them.
1494  */
1495 u32 intel_fb_xy_to_linear(int x, int y,
1496                           const struct intel_plane_state *state,
1497                           int color_plane)
1498 {
1499         const struct drm_framebuffer *fb = state->hw.fb;
1500         unsigned int cpp = fb->format->cpp[color_plane];
1501         unsigned int pitch = state->color_plane[color_plane].stride;
1502
1503         return y * pitch + x * cpp;
1504 }
1505
1506 /*
1507  * Add the x/y offsets derived from fb->offsets[] to the user
1508  * specified plane src x/y offsets. The resulting x/y offsets
1509  * specify the start of scanout from the beginning of the gtt mapping.
1510  */
1511 void intel_add_fb_offsets(int *x, int *y,
1512                           const struct intel_plane_state *state,
1513                           int color_plane)
1514
1515 {
1516         *x += state->color_plane[color_plane].x;
1517         *y += state->color_plane[color_plane].y;
1518 }
1519
1520 static u32 intel_adjust_tile_offset(int *x, int *y,
1521                                     unsigned int tile_width,
1522                                     unsigned int tile_height,
1523                                     unsigned int tile_size,
1524                                     unsigned int pitch_tiles,
1525                                     u32 old_offset,
1526                                     u32 new_offset)
1527 {
1528         unsigned int pitch_pixels = pitch_tiles * tile_width;
1529         unsigned int tiles;
1530
1531         WARN_ON(old_offset & (tile_size - 1));
1532         WARN_ON(new_offset & (tile_size - 1));
1533         WARN_ON(new_offset > old_offset);
1534
1535         tiles = (old_offset - new_offset) / tile_size;
1536
1537         *y += tiles / pitch_tiles * tile_height;
1538         *x += tiles % pitch_tiles * tile_width;
1539
1540         /* minimize x in case it got needlessly big */
1541         *y += *x / pitch_pixels * tile_height;
1542         *x %= pitch_pixels;
1543
1544         return new_offset;
1545 }
1546
1547 static bool is_surface_linear(const struct drm_framebuffer *fb, int color_plane)
1548 {
1549         return fb->modifier == DRM_FORMAT_MOD_LINEAR ||
1550                is_gen12_ccs_plane(fb, color_plane);
1551 }
1552
1553 static u32 intel_adjust_aligned_offset(int *x, int *y,
1554                                        const struct drm_framebuffer *fb,
1555                                        int color_plane,
1556                                        unsigned int rotation,
1557                                        unsigned int pitch,
1558                                        u32 old_offset, u32 new_offset)
1559 {
1560         struct drm_i915_private *dev_priv = to_i915(fb->dev);
1561         unsigned int cpp = fb->format->cpp[color_plane];
1562
1563         drm_WARN_ON(&dev_priv->drm, new_offset > old_offset);
1564
1565         if (!is_surface_linear(fb, color_plane)) {
1566                 unsigned int tile_size, tile_width, tile_height;
1567                 unsigned int pitch_tiles;
1568
1569                 tile_size = intel_tile_size(dev_priv);
1570                 intel_tile_dims(fb, color_plane, &tile_width, &tile_height);
1571
1572                 if (drm_rotation_90_or_270(rotation)) {
1573                         pitch_tiles = pitch / tile_height;
1574                         swap(tile_width, tile_height);
1575                 } else {
1576                         pitch_tiles = pitch / (tile_width * cpp);
1577                 }
1578
1579                 intel_adjust_tile_offset(x, y, tile_width, tile_height,
1580                                          tile_size, pitch_tiles,
1581                                          old_offset, new_offset);
1582         } else {
1583                 old_offset += *y * pitch + *x * cpp;
1584
1585                 *y = (old_offset - new_offset) / pitch;
1586                 *x = ((old_offset - new_offset) - *y * pitch) / cpp;
1587         }
1588
1589         return new_offset;
1590 }
1591
1592 /*
1593  * Adjust the tile offset by moving the difference into
1594  * the x/y offsets.
1595  */
1596 static u32 intel_plane_adjust_aligned_offset(int *x, int *y,
1597                                              const struct intel_plane_state *state,
1598                                              int color_plane,
1599                                              u32 old_offset, u32 new_offset)
1600 {
1601         return intel_adjust_aligned_offset(x, y, state->hw.fb, color_plane,
1602                                            state->hw.rotation,
1603                                            state->color_plane[color_plane].stride,
1604                                            old_offset, new_offset);
1605 }
1606
1607 /*
1608  * Computes the aligned offset to the base tile and adjusts
1609  * x, y. bytes per pixel is assumed to be a power-of-two.
1610  *
1611  * In the 90/270 rotated case, x and y are assumed
1612  * to be already rotated to match the rotated GTT view, and
1613  * pitch is the tile_height aligned framebuffer height.
1614  *
1615  * This function is used when computing the derived information
1616  * under intel_framebuffer, so using any of that information
1617  * here is not allowed. Anything under drm_framebuffer can be
1618  * used. This is why the user has to pass in the pitch since it
1619  * is specified in the rotated orientation.
1620  */
1621 static u32 intel_compute_aligned_offset(struct drm_i915_private *dev_priv,
1622                                         int *x, int *y,
1623                                         const struct drm_framebuffer *fb,
1624                                         int color_plane,
1625                                         unsigned int pitch,
1626                                         unsigned int rotation,
1627                                         u32 alignment)
1628 {
1629         unsigned int cpp = fb->format->cpp[color_plane];
1630         u32 offset, offset_aligned;
1631
1632         if (!is_surface_linear(fb, color_plane)) {
1633                 unsigned int tile_size, tile_width, tile_height;
1634                 unsigned int tile_rows, tiles, pitch_tiles;
1635
1636                 tile_size = intel_tile_size(dev_priv);
1637                 intel_tile_dims(fb, color_plane, &tile_width, &tile_height);
1638
1639                 if (drm_rotation_90_or_270(rotation)) {
1640                         pitch_tiles = pitch / tile_height;
1641                         swap(tile_width, tile_height);
1642                 } else {
1643                         pitch_tiles = pitch / (tile_width * cpp);
1644                 }
1645
1646                 tile_rows = *y / tile_height;
1647                 *y %= tile_height;
1648
1649                 tiles = *x / tile_width;
1650                 *x %= tile_width;
1651
1652                 offset = (tile_rows * pitch_tiles + tiles) * tile_size;
1653
1654                 offset_aligned = offset;
1655                 if (alignment)
1656                         offset_aligned = rounddown(offset_aligned, alignment);
1657
1658                 intel_adjust_tile_offset(x, y, tile_width, tile_height,
1659                                          tile_size, pitch_tiles,
1660                                          offset, offset_aligned);
1661         } else {
1662                 offset = *y * pitch + *x * cpp;
1663                 offset_aligned = offset;
1664                 if (alignment) {
1665                         offset_aligned = rounddown(offset_aligned, alignment);
1666                         *y = (offset % alignment) / pitch;
1667                         *x = ((offset % alignment) - *y * pitch) / cpp;
1668                 } else {
1669                         *y = *x = 0;
1670                 }
1671         }
1672
1673         return offset_aligned;
1674 }
1675
1676 u32 intel_plane_compute_aligned_offset(int *x, int *y,
1677                                        const struct intel_plane_state *state,
1678                                        int color_plane)
1679 {
1680         struct intel_plane *intel_plane = to_intel_plane(state->uapi.plane);
1681         struct drm_i915_private *dev_priv = to_i915(intel_plane->base.dev);
1682         const struct drm_framebuffer *fb = state->hw.fb;
1683         unsigned int rotation = state->hw.rotation;
1684         int pitch = state->color_plane[color_plane].stride;
1685         u32 alignment;
1686
1687         if (intel_plane->id == PLANE_CURSOR)
1688                 alignment = intel_cursor_alignment(dev_priv);
1689         else
1690                 alignment = intel_surf_alignment(fb, color_plane);
1691
1692         return intel_compute_aligned_offset(dev_priv, x, y, fb, color_plane,
1693                                             pitch, rotation, alignment);
1694 }
1695
1696 /* Convert the fb->offset[] into x/y offsets */
1697 static int intel_fb_offset_to_xy(int *x, int *y,
1698                                  const struct drm_framebuffer *fb,
1699                                  int color_plane)
1700 {
1701         struct drm_i915_private *dev_priv = to_i915(fb->dev);
1702         unsigned int height;
1703         u32 alignment;
1704
1705         if (INTEL_GEN(dev_priv) >= 12 &&
1706             is_semiplanar_uv_plane(fb, color_plane))
1707                 alignment = intel_tile_row_size(fb, color_plane);
1708         else if (fb->modifier != DRM_FORMAT_MOD_LINEAR)
1709                 alignment = intel_tile_size(dev_priv);
1710         else
1711                 alignment = 0;
1712
1713         if (alignment != 0 && fb->offsets[color_plane] % alignment) {
1714                 drm_dbg_kms(&dev_priv->drm,
1715                             "Misaligned offset 0x%08x for color plane %d\n",
1716                             fb->offsets[color_plane], color_plane);
1717                 return -EINVAL;
1718         }
1719
1720         height = drm_framebuffer_plane_height(fb->height, fb, color_plane);
1721         height = ALIGN(height, intel_tile_height(fb, color_plane));
1722
1723         /* Catch potential overflows early */
1724         if (add_overflows_t(u32, mul_u32_u32(height, fb->pitches[color_plane]),
1725                             fb->offsets[color_plane])) {
1726                 drm_dbg_kms(&dev_priv->drm,
1727                             "Bad offset 0x%08x or pitch %d for color plane %d\n",
1728                             fb->offsets[color_plane], fb->pitches[color_plane],
1729                             color_plane);
1730                 return -ERANGE;
1731         }
1732
1733         *x = 0;
1734         *y = 0;
1735
1736         intel_adjust_aligned_offset(x, y,
1737                                     fb, color_plane, DRM_MODE_ROTATE_0,
1738                                     fb->pitches[color_plane],
1739                                     fb->offsets[color_plane], 0);
1740
1741         return 0;
1742 }
1743
1744 static unsigned int intel_fb_modifier_to_tiling(u64 fb_modifier)
1745 {
1746         switch (fb_modifier) {
1747         case I915_FORMAT_MOD_X_TILED:
1748                 return I915_TILING_X;
1749         case I915_FORMAT_MOD_Y_TILED:
1750         case I915_FORMAT_MOD_Y_TILED_CCS:
1751         case I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS:
1752         case I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS_CC:
1753         case I915_FORMAT_MOD_Y_TILED_GEN12_MC_CCS:
1754                 return I915_TILING_Y;
1755         default:
1756                 return I915_TILING_NONE;
1757         }
1758 }
1759
1760 /*
1761  * From the Sky Lake PRM:
1762  * "The Color Control Surface (CCS) contains the compression status of
1763  *  the cache-line pairs. The compression state of the cache-line pair
1764  *  is specified by 2 bits in the CCS. Each CCS cache-line represents
1765  *  an area on the main surface of 16 x16 sets of 128 byte Y-tiled
1766  *  cache-line-pairs. CCS is always Y tiled."
1767  *
1768  * Since cache line pairs refers to horizontally adjacent cache lines,
1769  * each cache line in the CCS corresponds to an area of 32x16 cache
1770  * lines on the main surface. Since each pixel is 4 bytes, this gives
1771  * us a ratio of one byte in the CCS for each 8x16 pixels in the
1772  * main surface.
1773  */
1774 static const struct drm_format_info skl_ccs_formats[] = {
1775         { .format = DRM_FORMAT_XRGB8888, .depth = 24, .num_planes = 2,
1776           .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, },
1777         { .format = DRM_FORMAT_XBGR8888, .depth = 24, .num_planes = 2,
1778           .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, },
1779         { .format = DRM_FORMAT_ARGB8888, .depth = 32, .num_planes = 2,
1780           .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, .has_alpha = true, },
1781         { .format = DRM_FORMAT_ABGR8888, .depth = 32, .num_planes = 2,
1782           .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, .has_alpha = true, },
1783 };
1784
1785 /*
1786  * Gen-12 compression uses 4 bits of CCS data for each cache line pair in the
1787  * main surface. And each 64B CCS cache line represents an area of 4x1 Y-tiles
1788  * in the main surface. With 4 byte pixels and each Y-tile having dimensions of
1789  * 32x32 pixels, the ratio turns out to 1B in the CCS for every 2x32 pixels in
1790  * the main surface.
1791  */
1792 static const struct drm_format_info gen12_ccs_formats[] = {
1793         { .format = DRM_FORMAT_XRGB8888, .depth = 24, .num_planes = 2,
1794           .char_per_block = { 4, 1 }, .block_w = { 1, 2 }, .block_h = { 1, 1 },
1795           .hsub = 1, .vsub = 1, },
1796         { .format = DRM_FORMAT_XBGR8888, .depth = 24, .num_planes = 2,
1797           .char_per_block = { 4, 1 }, .block_w = { 1, 2 }, .block_h = { 1, 1 },
1798           .hsub = 1, .vsub = 1, },
1799         { .format = DRM_FORMAT_ARGB8888, .depth = 32, .num_planes = 2,
1800           .char_per_block = { 4, 1 }, .block_w = { 1, 2 }, .block_h = { 1, 1 },
1801           .hsub = 1, .vsub = 1, .has_alpha = true },
1802         { .format = DRM_FORMAT_ABGR8888, .depth = 32, .num_planes = 2,
1803           .char_per_block = { 4, 1 }, .block_w = { 1, 2 }, .block_h = { 1, 1 },
1804           .hsub = 1, .vsub = 1, .has_alpha = true },
1805         { .format = DRM_FORMAT_YUYV, .num_planes = 2,
1806           .char_per_block = { 2, 1 }, .block_w = { 1, 2 }, .block_h = { 1, 1 },
1807           .hsub = 2, .vsub = 1, .is_yuv = true },
1808         { .format = DRM_FORMAT_YVYU, .num_planes = 2,
1809           .char_per_block = { 2, 1 }, .block_w = { 1, 2 }, .block_h = { 1, 1 },
1810           .hsub = 2, .vsub = 1, .is_yuv = true },
1811         { .format = DRM_FORMAT_UYVY, .num_planes = 2,
1812           .char_per_block = { 2, 1 }, .block_w = { 1, 2 }, .block_h = { 1, 1 },
1813           .hsub = 2, .vsub = 1, .is_yuv = true },
1814         { .format = DRM_FORMAT_VYUY, .num_planes = 2,
1815           .char_per_block = { 2, 1 }, .block_w = { 1, 2 }, .block_h = { 1, 1 },
1816           .hsub = 2, .vsub = 1, .is_yuv = true },
1817         { .format = DRM_FORMAT_NV12, .num_planes = 4,
1818           .char_per_block = { 1, 2, 1, 1 }, .block_w = { 1, 1, 4, 4 }, .block_h = { 1, 1, 1, 1 },
1819           .hsub = 2, .vsub = 2, .is_yuv = true },
1820         { .format = DRM_FORMAT_P010, .num_planes = 4,
1821           .char_per_block = { 2, 4, 1, 1 }, .block_w = { 1, 1, 2, 2 }, .block_h = { 1, 1, 1, 1 },
1822           .hsub = 2, .vsub = 2, .is_yuv = true },
1823         { .format = DRM_FORMAT_P012, .num_planes = 4,
1824           .char_per_block = { 2, 4, 1, 1 }, .block_w = { 1, 1, 2, 2 }, .block_h = { 1, 1, 1, 1 },
1825           .hsub = 2, .vsub = 2, .is_yuv = true },
1826         { .format = DRM_FORMAT_P016, .num_planes = 4,
1827           .char_per_block = { 2, 4, 1, 1 }, .block_w = { 1, 1, 2, 2 }, .block_h = { 1, 1, 1, 1 },
1828           .hsub = 2, .vsub = 2, .is_yuv = true },
1829 };
1830
1831 /*
1832  * Same as gen12_ccs_formats[] above, but with additional surface used
1833  * to pass Clear Color information in plane 2 with 64 bits of data.
1834  */
1835 static const struct drm_format_info gen12_ccs_cc_formats[] = {
1836         { .format = DRM_FORMAT_XRGB8888, .depth = 24, .num_planes = 3,
1837           .char_per_block = { 4, 1, 0 }, .block_w = { 1, 2, 2 }, .block_h = { 1, 1, 1 },
1838           .hsub = 1, .vsub = 1, },
1839         { .format = DRM_FORMAT_XBGR8888, .depth = 24, .num_planes = 3,
1840           .char_per_block = { 4, 1, 0 }, .block_w = { 1, 2, 2 }, .block_h = { 1, 1, 1 },
1841           .hsub = 1, .vsub = 1, },
1842         { .format = DRM_FORMAT_ARGB8888, .depth = 32, .num_planes = 3,
1843           .char_per_block = { 4, 1, 0 }, .block_w = { 1, 2, 2 }, .block_h = { 1, 1, 1 },
1844           .hsub = 1, .vsub = 1, .has_alpha = true },
1845         { .format = DRM_FORMAT_ABGR8888, .depth = 32, .num_planes = 3,
1846           .char_per_block = { 4, 1, 0 }, .block_w = { 1, 2, 2 }, .block_h = { 1, 1, 1 },
1847           .hsub = 1, .vsub = 1, .has_alpha = true },
1848 };
1849
1850 static const struct drm_format_info *
1851 lookup_format_info(const struct drm_format_info formats[],
1852                    int num_formats, u32 format)
1853 {
1854         int i;
1855
1856         for (i = 0; i < num_formats; i++) {
1857                 if (formats[i].format == format)
1858                         return &formats[i];
1859         }
1860
1861         return NULL;
1862 }
1863
1864 static const struct drm_format_info *
1865 intel_get_format_info(const struct drm_mode_fb_cmd2 *cmd)
1866 {
1867         switch (cmd->modifier[0]) {
1868         case I915_FORMAT_MOD_Y_TILED_CCS:
1869         case I915_FORMAT_MOD_Yf_TILED_CCS:
1870                 return lookup_format_info(skl_ccs_formats,
1871                                           ARRAY_SIZE(skl_ccs_formats),
1872                                           cmd->pixel_format);
1873         case I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS:
1874         case I915_FORMAT_MOD_Y_TILED_GEN12_MC_CCS:
1875                 return lookup_format_info(gen12_ccs_formats,
1876                                           ARRAY_SIZE(gen12_ccs_formats),
1877                                           cmd->pixel_format);
1878         case I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS_CC:
1879                 return lookup_format_info(gen12_ccs_cc_formats,
1880                                           ARRAY_SIZE(gen12_ccs_cc_formats),
1881                                           cmd->pixel_format);
1882         default:
1883                 return NULL;
1884         }
1885 }
1886
1887 bool is_ccs_modifier(u64 modifier)
1888 {
1889         return modifier == I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS ||
1890                modifier == I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS_CC ||
1891                modifier == I915_FORMAT_MOD_Y_TILED_GEN12_MC_CCS ||
1892                modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
1893                modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
1894 }
1895
1896 static int gen12_ccs_aux_stride(struct drm_framebuffer *fb, int ccs_plane)
1897 {
1898         return DIV_ROUND_UP(fb->pitches[ccs_to_main_plane(fb, ccs_plane)],
1899                             512) * 64;
1900 }
1901
1902 u32 intel_plane_fb_max_stride(struct drm_i915_private *dev_priv,
1903                               u32 pixel_format, u64 modifier)
1904 {
1905         struct intel_crtc *crtc;
1906         struct intel_plane *plane;
1907
1908         /*
1909          * We assume the primary plane for pipe A has
1910          * the highest stride limits of them all,
1911          * if in case pipe A is disabled, use the first pipe from pipe_mask.
1912          */
1913         crtc = intel_get_first_crtc(dev_priv);
1914         if (!crtc)
1915                 return 0;
1916
1917         plane = to_intel_plane(crtc->base.primary);
1918
1919         return plane->max_stride(plane, pixel_format, modifier,
1920                                  DRM_MODE_ROTATE_0);
1921 }
1922
1923 static
1924 u32 intel_fb_max_stride(struct drm_i915_private *dev_priv,
1925                         u32 pixel_format, u64 modifier)
1926 {
1927         /*
1928          * Arbitrary limit for gen4+ chosen to match the
1929          * render engine max stride.
1930          *
1931          * The new CCS hash mode makes remapping impossible
1932          */
1933         if (!is_ccs_modifier(modifier)) {
1934                 if (INTEL_GEN(dev_priv) >= 7)
1935                         return 256*1024;
1936                 else if (INTEL_GEN(dev_priv) >= 4)
1937                         return 128*1024;
1938         }
1939
1940         return intel_plane_fb_max_stride(dev_priv, pixel_format, modifier);
1941 }
1942
1943 static u32
1944 intel_fb_stride_alignment(const struct drm_framebuffer *fb, int color_plane)
1945 {
1946         struct drm_i915_private *dev_priv = to_i915(fb->dev);
1947         u32 tile_width;
1948
1949         if (is_surface_linear(fb, color_plane)) {
1950                 u32 max_stride = intel_plane_fb_max_stride(dev_priv,
1951                                                            fb->format->format,
1952                                                            fb->modifier);
1953
1954                 /*
1955                  * To make remapping with linear generally feasible
1956                  * we need the stride to be page aligned.
1957                  */
1958                 if (fb->pitches[color_plane] > max_stride &&
1959                     !is_ccs_modifier(fb->modifier))
1960                         return intel_tile_size(dev_priv);
1961                 else
1962                         return 64;
1963         }
1964
1965         tile_width = intel_tile_width_bytes(fb, color_plane);
1966         if (is_ccs_modifier(fb->modifier)) {
1967                 /*
1968                  * Display WA #0531: skl,bxt,kbl,glk
1969                  *
1970                  * Render decompression and plane width > 3840
1971                  * combined with horizontal panning requires the
1972                  * plane stride to be a multiple of 4. We'll just
1973                  * require the entire fb to accommodate that to avoid
1974                  * potential runtime errors at plane configuration time.
1975                  */
1976                 if (IS_GEN(dev_priv, 9) && color_plane == 0 && fb->width > 3840)
1977                         tile_width *= 4;
1978                 /*
1979                  * The main surface pitch must be padded to a multiple of four
1980                  * tile widths.
1981                  */
1982                 else if (INTEL_GEN(dev_priv) >= 12)
1983                         tile_width *= 4;
1984         }
1985         return tile_width;
1986 }
1987
1988 bool intel_plane_can_remap(const struct intel_plane_state *plane_state)
1989 {
1990         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1991         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1992         const struct drm_framebuffer *fb = plane_state->hw.fb;
1993         int i;
1994
1995         /* We don't want to deal with remapping with cursors */
1996         if (plane->id == PLANE_CURSOR)
1997                 return false;
1998
1999         /*
2000          * The display engine limits already match/exceed the
2001          * render engine limits, so not much point in remapping.
2002          * Would also need to deal with the fence POT alignment
2003          * and gen2 2KiB GTT tile size.
2004          */
2005         if (INTEL_GEN(dev_priv) < 4)
2006                 return false;
2007
2008         /*
2009          * The new CCS hash mode isn't compatible with remapping as
2010          * the virtual address of the pages affects the compressed data.
2011          */
2012         if (is_ccs_modifier(fb->modifier))
2013                 return false;
2014
2015         /* Linear needs a page aligned stride for remapping */
2016         if (fb->modifier == DRM_FORMAT_MOD_LINEAR) {
2017                 unsigned int alignment = intel_tile_size(dev_priv) - 1;
2018
2019                 for (i = 0; i < fb->format->num_planes; i++) {
2020                         if (fb->pitches[i] & alignment)
2021                                 return false;
2022                 }
2023         }
2024
2025         return true;
2026 }
2027
2028 static bool intel_plane_needs_remap(const struct intel_plane_state *plane_state)
2029 {
2030         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
2031         const struct drm_framebuffer *fb = plane_state->hw.fb;
2032         unsigned int rotation = plane_state->hw.rotation;
2033         u32 stride, max_stride;
2034
2035         /*
2036          * No remapping for invisible planes since we don't have
2037          * an actual source viewport to remap.
2038          */
2039         if (!plane_state->uapi.visible)
2040                 return false;
2041
2042         if (!intel_plane_can_remap(plane_state))
2043                 return false;
2044
2045         /*
2046          * FIXME: aux plane limits on gen9+ are
2047          * unclear in Bspec, for now no checking.
2048          */
2049         stride = intel_fb_pitch(fb, 0, rotation);
2050         max_stride = plane->max_stride(plane, fb->format->format,
2051                                        fb->modifier, rotation);
2052
2053         return stride > max_stride;
2054 }
2055
2056 static void
2057 intel_fb_plane_get_subsampling(int *hsub, int *vsub,
2058                                const struct drm_framebuffer *fb,
2059                                int color_plane)
2060 {
2061         int main_plane;
2062
2063         if (color_plane == 0) {
2064                 *hsub = 1;
2065                 *vsub = 1;
2066
2067                 return;
2068         }
2069
2070         /*
2071          * TODO: Deduct the subsampling from the char block for all CCS
2072          * formats and planes.
2073          */
2074         if (!is_gen12_ccs_plane(fb, color_plane)) {
2075                 *hsub = fb->format->hsub;
2076                 *vsub = fb->format->vsub;
2077
2078                 return;
2079         }
2080
2081         main_plane = ccs_to_main_plane(fb, color_plane);
2082         *hsub = drm_format_info_block_width(fb->format, color_plane) /
2083                 drm_format_info_block_width(fb->format, main_plane);
2084
2085         /*
2086          * The min stride check in the core framebuffer_check() function
2087          * assumes that format->hsub applies to every plane except for the
2088          * first plane. That's incorrect for the CCS AUX plane of the first
2089          * plane, but for the above check to pass we must define the block
2090          * width with that subsampling applied to it. Adjust the width here
2091          * accordingly, so we can calculate the actual subsampling factor.
2092          */
2093         if (main_plane == 0)
2094                 *hsub *= fb->format->hsub;
2095
2096         *vsub = 32;
2097 }
2098 static int
2099 intel_fb_check_ccs_xy(struct drm_framebuffer *fb, int ccs_plane, int x, int y)
2100 {
2101         struct drm_i915_private *i915 = to_i915(fb->dev);
2102         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
2103         int main_plane;
2104         int hsub, vsub;
2105         int tile_width, tile_height;
2106         int ccs_x, ccs_y;
2107         int main_x, main_y;
2108
2109         if (!is_ccs_plane(fb, ccs_plane) || is_gen12_ccs_cc_plane(fb, ccs_plane))
2110                 return 0;
2111
2112         intel_tile_dims(fb, ccs_plane, &tile_width, &tile_height);
2113         intel_fb_plane_get_subsampling(&hsub, &vsub, fb, ccs_plane);
2114
2115         tile_width *= hsub;
2116         tile_height *= vsub;
2117
2118         ccs_x = (x * hsub) % tile_width;
2119         ccs_y = (y * vsub) % tile_height;
2120
2121         main_plane = ccs_to_main_plane(fb, ccs_plane);
2122         main_x = intel_fb->normal[main_plane].x % tile_width;
2123         main_y = intel_fb->normal[main_plane].y % tile_height;
2124
2125         /*
2126          * CCS doesn't have its own x/y offset register, so the intra CCS tile
2127          * x/y offsets must match between CCS and the main surface.
2128          */
2129         if (main_x != ccs_x || main_y != ccs_y) {
2130                 drm_dbg_kms(&i915->drm,
2131                               "Bad CCS x/y (main %d,%d ccs %d,%d) full (main %d,%d ccs %d,%d)\n",
2132                               main_x, main_y,
2133                               ccs_x, ccs_y,
2134                               intel_fb->normal[main_plane].x,
2135                               intel_fb->normal[main_plane].y,
2136                               x, y);
2137                 return -EINVAL;
2138         }
2139
2140         return 0;
2141 }
2142
2143 static void
2144 intel_fb_plane_dims(int *w, int *h, struct drm_framebuffer *fb, int color_plane)
2145 {
2146         int main_plane = is_ccs_plane(fb, color_plane) ?
2147                          ccs_to_main_plane(fb, color_plane) : 0;
2148         int main_hsub, main_vsub;
2149         int hsub, vsub;
2150
2151         intel_fb_plane_get_subsampling(&main_hsub, &main_vsub, fb, main_plane);
2152         intel_fb_plane_get_subsampling(&hsub, &vsub, fb, color_plane);
2153         *w = fb->width / main_hsub / hsub;
2154         *h = fb->height / main_vsub / vsub;
2155 }
2156
2157 /*
2158  * Setup the rotated view for an FB plane and return the size the GTT mapping
2159  * requires for this view.
2160  */
2161 static u32
2162 setup_fb_rotation(int plane, const struct intel_remapped_plane_info *plane_info,
2163                   u32 gtt_offset_rotated, int x, int y,
2164                   unsigned int width, unsigned int height,
2165                   unsigned int tile_size,
2166                   unsigned int tile_width, unsigned int tile_height,
2167                   struct drm_framebuffer *fb)
2168 {
2169         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
2170         struct intel_rotation_info *rot_info = &intel_fb->rot_info;
2171         unsigned int pitch_tiles;
2172         struct drm_rect r;
2173
2174         /* Y or Yf modifiers required for 90/270 rotation */
2175         if (fb->modifier != I915_FORMAT_MOD_Y_TILED &&
2176             fb->modifier != I915_FORMAT_MOD_Yf_TILED)
2177                 return 0;
2178
2179         if (drm_WARN_ON(fb->dev, plane >= ARRAY_SIZE(rot_info->plane)))
2180                 return 0;
2181
2182         rot_info->plane[plane] = *plane_info;
2183
2184         intel_fb->rotated[plane].pitch = plane_info->height * tile_height;
2185
2186         /* rotate the x/y offsets to match the GTT view */
2187         drm_rect_init(&r, x, y, width, height);
2188         drm_rect_rotate(&r,
2189                         plane_info->width * tile_width,
2190                         plane_info->height * tile_height,
2191                         DRM_MODE_ROTATE_270);
2192         x = r.x1;
2193         y = r.y1;
2194
2195         /* rotate the tile dimensions to match the GTT view */
2196         pitch_tiles = intel_fb->rotated[plane].pitch / tile_height;
2197         swap(tile_width, tile_height);
2198
2199         /*
2200          * We only keep the x/y offsets, so push all of the
2201          * gtt offset into the x/y offsets.
2202          */
2203         intel_adjust_tile_offset(&x, &y,
2204                                  tile_width, tile_height,
2205                                  tile_size, pitch_tiles,
2206                                  gtt_offset_rotated * tile_size, 0);
2207
2208         /*
2209          * First pixel of the framebuffer from
2210          * the start of the rotated gtt mapping.
2211          */
2212         intel_fb->rotated[plane].x = x;
2213         intel_fb->rotated[plane].y = y;
2214
2215         return plane_info->width * plane_info->height;
2216 }
2217
2218 static int
2219 intel_fill_fb_info(struct drm_i915_private *dev_priv,
2220                    struct drm_framebuffer *fb)
2221 {
2222         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
2223         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
2224         u32 gtt_offset_rotated = 0;
2225         unsigned int max_size = 0;
2226         int i, num_planes = fb->format->num_planes;
2227         unsigned int tile_size = intel_tile_size(dev_priv);
2228
2229         for (i = 0; i < num_planes; i++) {
2230                 unsigned int width, height;
2231                 unsigned int cpp, size;
2232                 u32 offset;
2233                 int x, y;
2234                 int ret;
2235
2236                 /*
2237                  * Plane 2 of Render Compression with Clear Color fb modifier
2238                  * is consumed by the driver and not passed to DE. Skip the
2239                  * arithmetic related to alignment and offset calculation.
2240                  */
2241                 if (is_gen12_ccs_cc_plane(fb, i)) {
2242                         if (IS_ALIGNED(fb->offsets[i], PAGE_SIZE))
2243                                 continue;
2244                         else
2245                                 return -EINVAL;
2246                 }
2247
2248                 cpp = fb->format->cpp[i];
2249                 intel_fb_plane_dims(&width, &height, fb, i);
2250
2251                 ret = intel_fb_offset_to_xy(&x, &y, fb, i);
2252                 if (ret) {
2253                         drm_dbg_kms(&dev_priv->drm,
2254                                     "bad fb plane %d offset: 0x%x\n",
2255                                     i, fb->offsets[i]);
2256                         return ret;
2257                 }
2258
2259                 ret = intel_fb_check_ccs_xy(fb, i, x, y);
2260                 if (ret)
2261                         return ret;
2262
2263                 /*
2264                  * The fence (if used) is aligned to the start of the object
2265                  * so having the framebuffer wrap around across the edge of the
2266                  * fenced region doesn't really work. We have no API to configure
2267                  * the fence start offset within the object (nor could we probably
2268                  * on gen2/3). So it's just easier if we just require that the
2269                  * fb layout agrees with the fence layout. We already check that the
2270                  * fb stride matches the fence stride elsewhere.
2271                  */
2272                 if (i == 0 && i915_gem_object_is_tiled(obj) &&
2273                     (x + width) * cpp > fb->pitches[i]) {
2274                         drm_dbg_kms(&dev_priv->drm,
2275                                     "bad fb plane %d offset: 0x%x\n",
2276                                      i, fb->offsets[i]);
2277                         return -EINVAL;
2278                 }
2279
2280                 /*
2281                  * First pixel of the framebuffer from
2282                  * the start of the normal gtt mapping.
2283                  */
2284                 intel_fb->normal[i].x = x;
2285                 intel_fb->normal[i].y = y;
2286
2287                 offset = intel_compute_aligned_offset(dev_priv, &x, &y, fb, i,
2288                                                       fb->pitches[i],
2289                                                       DRM_MODE_ROTATE_0,
2290                                                       tile_size);
2291                 offset /= tile_size;
2292
2293                 if (!is_surface_linear(fb, i)) {
2294                         struct intel_remapped_plane_info plane_info;
2295                         unsigned int tile_width, tile_height;
2296
2297                         intel_tile_dims(fb, i, &tile_width, &tile_height);
2298
2299                         plane_info.offset = offset;
2300                         plane_info.stride = DIV_ROUND_UP(fb->pitches[i],
2301                                                          tile_width * cpp);
2302                         plane_info.width = DIV_ROUND_UP(x + width, tile_width);
2303                         plane_info.height = DIV_ROUND_UP(y + height,
2304                                                          tile_height);
2305
2306                         /* how many tiles does this plane need */
2307                         size = plane_info.stride * plane_info.height;
2308                         /*
2309                          * If the plane isn't horizontally tile aligned,
2310                          * we need one more tile.
2311                          */
2312                         if (x != 0)
2313                                 size++;
2314
2315                         gtt_offset_rotated +=
2316                                 setup_fb_rotation(i, &plane_info,
2317                                                   gtt_offset_rotated,
2318                                                   x, y, width, height,
2319                                                   tile_size,
2320                                                   tile_width, tile_height,
2321                                                   fb);
2322                 } else {
2323                         size = DIV_ROUND_UP((y + height) * fb->pitches[i] +
2324                                             x * cpp, tile_size);
2325                 }
2326
2327                 /* how many tiles in total needed in the bo */
2328                 max_size = max(max_size, offset + size);
2329         }
2330
2331         if (mul_u32_u32(max_size, tile_size) > obj->base.size) {
2332                 drm_dbg_kms(&dev_priv->drm,
2333                             "fb too big for bo (need %llu bytes, have %zu bytes)\n",
2334                             mul_u32_u32(max_size, tile_size), obj->base.size);
2335                 return -EINVAL;
2336         }
2337
2338         return 0;
2339 }
2340
2341 static void
2342 intel_plane_remap_gtt(struct intel_plane_state *plane_state)
2343 {
2344         struct drm_i915_private *dev_priv =
2345                 to_i915(plane_state->uapi.plane->dev);
2346         struct drm_framebuffer *fb = plane_state->hw.fb;
2347         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
2348         struct intel_rotation_info *info = &plane_state->view.rotated;
2349         unsigned int rotation = plane_state->hw.rotation;
2350         int i, num_planes = fb->format->num_planes;
2351         unsigned int tile_size = intel_tile_size(dev_priv);
2352         unsigned int src_x, src_y;
2353         unsigned int src_w, src_h;
2354         u32 gtt_offset = 0;
2355
2356         memset(&plane_state->view, 0, sizeof(plane_state->view));
2357         plane_state->view.type = drm_rotation_90_or_270(rotation) ?
2358                 I915_GGTT_VIEW_ROTATED : I915_GGTT_VIEW_REMAPPED;
2359
2360         src_x = plane_state->uapi.src.x1 >> 16;
2361         src_y = plane_state->uapi.src.y1 >> 16;
2362         src_w = drm_rect_width(&plane_state->uapi.src) >> 16;
2363         src_h = drm_rect_height(&plane_state->uapi.src) >> 16;
2364
2365         drm_WARN_ON(&dev_priv->drm, is_ccs_modifier(fb->modifier));
2366
2367         /* Make src coordinates relative to the viewport */
2368         drm_rect_translate(&plane_state->uapi.src,
2369                            -(src_x << 16), -(src_y << 16));
2370
2371         /* Rotate src coordinates to match rotated GTT view */
2372         if (drm_rotation_90_or_270(rotation))
2373                 drm_rect_rotate(&plane_state->uapi.src,
2374                                 src_w << 16, src_h << 16,
2375                                 DRM_MODE_ROTATE_270);
2376
2377         for (i = 0; i < num_planes; i++) {
2378                 unsigned int hsub = i ? fb->format->hsub : 1;
2379                 unsigned int vsub = i ? fb->format->vsub : 1;
2380                 unsigned int cpp = fb->format->cpp[i];
2381                 unsigned int tile_width, tile_height;
2382                 unsigned int width, height;
2383                 unsigned int pitch_tiles;
2384                 unsigned int x, y;
2385                 u32 offset;
2386
2387                 intel_tile_dims(fb, i, &tile_width, &tile_height);
2388
2389                 x = src_x / hsub;
2390                 y = src_y / vsub;
2391                 width = src_w / hsub;
2392                 height = src_h / vsub;
2393
2394                 /*
2395                  * First pixel of the src viewport from the
2396                  * start of the normal gtt mapping.
2397                  */
2398                 x += intel_fb->normal[i].x;
2399                 y += intel_fb->normal[i].y;
2400
2401                 offset = intel_compute_aligned_offset(dev_priv, &x, &y,
2402                                                       fb, i, fb->pitches[i],
2403                                                       DRM_MODE_ROTATE_0, tile_size);
2404                 offset /= tile_size;
2405
2406                 drm_WARN_ON(&dev_priv->drm, i >= ARRAY_SIZE(info->plane));
2407                 info->plane[i].offset = offset;
2408                 info->plane[i].stride = DIV_ROUND_UP(fb->pitches[i],
2409                                                      tile_width * cpp);
2410                 info->plane[i].width = DIV_ROUND_UP(x + width, tile_width);
2411                 info->plane[i].height = DIV_ROUND_UP(y + height, tile_height);
2412
2413                 if (drm_rotation_90_or_270(rotation)) {
2414                         struct drm_rect r;
2415
2416                         /* rotate the x/y offsets to match the GTT view */
2417                         drm_rect_init(&r, x, y, width, height);
2418                         drm_rect_rotate(&r,
2419                                         info->plane[i].width * tile_width,
2420                                         info->plane[i].height * tile_height,
2421                                         DRM_MODE_ROTATE_270);
2422                         x = r.x1;
2423                         y = r.y1;
2424
2425                         pitch_tiles = info->plane[i].height;
2426                         plane_state->color_plane[i].stride = pitch_tiles * tile_height;
2427
2428                         /* rotate the tile dimensions to match the GTT view */
2429                         swap(tile_width, tile_height);
2430                 } else {
2431                         pitch_tiles = info->plane[i].width;
2432                         plane_state->color_plane[i].stride = pitch_tiles * tile_width * cpp;
2433                 }
2434
2435                 /*
2436                  * We only keep the x/y offsets, so push all of the
2437                  * gtt offset into the x/y offsets.
2438                  */
2439                 intel_adjust_tile_offset(&x, &y,
2440                                          tile_width, tile_height,
2441                                          tile_size, pitch_tiles,
2442                                          gtt_offset * tile_size, 0);
2443
2444                 gtt_offset += info->plane[i].width * info->plane[i].height;
2445
2446                 plane_state->color_plane[i].offset = 0;
2447                 plane_state->color_plane[i].x = x;
2448                 plane_state->color_plane[i].y = y;
2449         }
2450 }
2451
2452 int
2453 intel_plane_compute_gtt(struct intel_plane_state *plane_state)
2454 {
2455         const struct intel_framebuffer *fb =
2456                 to_intel_framebuffer(plane_state->hw.fb);
2457         unsigned int rotation = plane_state->hw.rotation;
2458         int i, num_planes;
2459
2460         if (!fb)
2461                 return 0;
2462
2463         num_planes = fb->base.format->num_planes;
2464
2465         if (intel_plane_needs_remap(plane_state)) {
2466                 intel_plane_remap_gtt(plane_state);
2467
2468                 /*
2469                  * Sometimes even remapping can't overcome
2470                  * the stride limitations :( Can happen with
2471                  * big plane sizes and suitably misaligned
2472                  * offsets.
2473                  */
2474                 return intel_plane_check_stride(plane_state);
2475         }
2476
2477         intel_fill_fb_ggtt_view(&plane_state->view, &fb->base, rotation);
2478
2479         for (i = 0; i < num_planes; i++) {
2480                 plane_state->color_plane[i].stride = intel_fb_pitch(&fb->base, i, rotation);
2481                 plane_state->color_plane[i].offset = 0;
2482
2483                 if (drm_rotation_90_or_270(rotation)) {
2484                         plane_state->color_plane[i].x = fb->rotated[i].x;
2485                         plane_state->color_plane[i].y = fb->rotated[i].y;
2486                 } else {
2487                         plane_state->color_plane[i].x = fb->normal[i].x;
2488                         plane_state->color_plane[i].y = fb->normal[i].y;
2489                 }
2490         }
2491
2492         /* Rotate src coordinates to match rotated GTT view */
2493         if (drm_rotation_90_or_270(rotation))
2494                 drm_rect_rotate(&plane_state->uapi.src,
2495                                 fb->base.width << 16, fb->base.height << 16,
2496                                 DRM_MODE_ROTATE_270);
2497
2498         return intel_plane_check_stride(plane_state);
2499 }
2500
2501 static int i9xx_format_to_fourcc(int format)
2502 {
2503         switch (format) {
2504         case DISPPLANE_8BPP:
2505                 return DRM_FORMAT_C8;
2506         case DISPPLANE_BGRA555:
2507                 return DRM_FORMAT_ARGB1555;
2508         case DISPPLANE_BGRX555:
2509                 return DRM_FORMAT_XRGB1555;
2510         case DISPPLANE_BGRX565:
2511                 return DRM_FORMAT_RGB565;
2512         default:
2513         case DISPPLANE_BGRX888:
2514                 return DRM_FORMAT_XRGB8888;
2515         case DISPPLANE_RGBX888:
2516                 return DRM_FORMAT_XBGR8888;
2517         case DISPPLANE_BGRA888:
2518                 return DRM_FORMAT_ARGB8888;
2519         case DISPPLANE_RGBA888:
2520                 return DRM_FORMAT_ABGR8888;
2521         case DISPPLANE_BGRX101010:
2522                 return DRM_FORMAT_XRGB2101010;
2523         case DISPPLANE_RGBX101010:
2524                 return DRM_FORMAT_XBGR2101010;
2525         case DISPPLANE_BGRA101010:
2526                 return DRM_FORMAT_ARGB2101010;
2527         case DISPPLANE_RGBA101010:
2528                 return DRM_FORMAT_ABGR2101010;
2529         case DISPPLANE_RGBX161616:
2530                 return DRM_FORMAT_XBGR16161616F;
2531         }
2532 }
2533
2534 int skl_format_to_fourcc(int format, bool rgb_order, bool alpha)
2535 {
2536         switch (format) {
2537         case PLANE_CTL_FORMAT_RGB_565:
2538                 return DRM_FORMAT_RGB565;
2539         case PLANE_CTL_FORMAT_NV12:
2540                 return DRM_FORMAT_NV12;
2541         case PLANE_CTL_FORMAT_XYUV:
2542                 return DRM_FORMAT_XYUV8888;
2543         case PLANE_CTL_FORMAT_P010:
2544                 return DRM_FORMAT_P010;
2545         case PLANE_CTL_FORMAT_P012:
2546                 return DRM_FORMAT_P012;
2547         case PLANE_CTL_FORMAT_P016:
2548                 return DRM_FORMAT_P016;
2549         case PLANE_CTL_FORMAT_Y210:
2550                 return DRM_FORMAT_Y210;
2551         case PLANE_CTL_FORMAT_Y212:
2552                 return DRM_FORMAT_Y212;
2553         case PLANE_CTL_FORMAT_Y216:
2554                 return DRM_FORMAT_Y216;
2555         case PLANE_CTL_FORMAT_Y410:
2556                 return DRM_FORMAT_XVYU2101010;
2557         case PLANE_CTL_FORMAT_Y412:
2558                 return DRM_FORMAT_XVYU12_16161616;
2559         case PLANE_CTL_FORMAT_Y416:
2560                 return DRM_FORMAT_XVYU16161616;
2561         default:
2562         case PLANE_CTL_FORMAT_XRGB_8888:
2563                 if (rgb_order) {
2564                         if (alpha)
2565                                 return DRM_FORMAT_ABGR8888;
2566                         else
2567                                 return DRM_FORMAT_XBGR8888;
2568                 } else {
2569                         if (alpha)
2570                                 return DRM_FORMAT_ARGB8888;
2571                         else
2572                                 return DRM_FORMAT_XRGB8888;
2573                 }
2574         case PLANE_CTL_FORMAT_XRGB_2101010:
2575                 if (rgb_order) {
2576                         if (alpha)
2577                                 return DRM_FORMAT_ABGR2101010;
2578                         else
2579                                 return DRM_FORMAT_XBGR2101010;
2580                 } else {
2581                         if (alpha)
2582                                 return DRM_FORMAT_ARGB2101010;
2583                         else
2584                                 return DRM_FORMAT_XRGB2101010;
2585                 }
2586         case PLANE_CTL_FORMAT_XRGB_16161616F:
2587                 if (rgb_order) {
2588                         if (alpha)
2589                                 return DRM_FORMAT_ABGR16161616F;
2590                         else
2591                                 return DRM_FORMAT_XBGR16161616F;
2592                 } else {
2593                         if (alpha)
2594                                 return DRM_FORMAT_ARGB16161616F;
2595                         else
2596                                 return DRM_FORMAT_XRGB16161616F;
2597                 }
2598         }
2599 }
2600
2601 static struct i915_vma *
2602 initial_plane_vma(struct drm_i915_private *i915,
2603                   struct intel_initial_plane_config *plane_config)
2604 {
2605         struct drm_i915_gem_object *obj;
2606         struct i915_vma *vma;
2607         u32 base, size;
2608
2609         if (plane_config->size == 0)
2610                 return NULL;
2611
2612         base = round_down(plane_config->base,
2613                           I915_GTT_MIN_ALIGNMENT);
2614         size = round_up(plane_config->base + plane_config->size,
2615                         I915_GTT_MIN_ALIGNMENT);
2616         size -= base;
2617
2618         /*
2619          * If the FB is too big, just don't use it since fbdev is not very
2620          * important and we should probably use that space with FBC or other
2621          * features.
2622          */
2623         if (size * 2 > i915->stolen_usable_size)
2624                 return NULL;
2625
2626         obj = i915_gem_object_create_stolen_for_preallocated(i915, base, size);
2627         if (IS_ERR(obj))
2628                 return NULL;
2629
2630         /*
2631          * Mark it WT ahead of time to avoid changing the
2632          * cache_level during fbdev initialization. The
2633          * unbind there would get stuck waiting for rcu.
2634          */
2635         i915_gem_object_set_cache_coherency(obj, HAS_WT(i915) ?
2636                                             I915_CACHE_WT : I915_CACHE_NONE);
2637
2638         switch (plane_config->tiling) {
2639         case I915_TILING_NONE:
2640                 break;
2641         case I915_TILING_X:
2642         case I915_TILING_Y:
2643                 obj->tiling_and_stride =
2644                         plane_config->fb->base.pitches[0] |
2645                         plane_config->tiling;
2646                 break;
2647         default:
2648                 MISSING_CASE(plane_config->tiling);
2649                 goto err_obj;
2650         }
2651
2652         vma = i915_vma_instance(obj, &i915->ggtt.vm, NULL);
2653         if (IS_ERR(vma))
2654                 goto err_obj;
2655
2656         if (i915_ggtt_pin(vma, NULL, 0, PIN_MAPPABLE | PIN_OFFSET_FIXED | base))
2657                 goto err_obj;
2658
2659         if (i915_gem_object_is_tiled(obj) &&
2660             !i915_vma_is_map_and_fenceable(vma))
2661                 goto err_obj;
2662
2663         return vma;
2664
2665 err_obj:
2666         i915_gem_object_put(obj);
2667         return NULL;
2668 }
2669
2670 static bool
2671 intel_alloc_initial_plane_obj(struct intel_crtc *crtc,
2672                               struct intel_initial_plane_config *plane_config)
2673 {
2674         struct drm_device *dev = crtc->base.dev;
2675         struct drm_i915_private *dev_priv = to_i915(dev);
2676         struct drm_mode_fb_cmd2 mode_cmd = { 0 };
2677         struct drm_framebuffer *fb = &plane_config->fb->base;
2678         struct i915_vma *vma;
2679
2680         switch (fb->modifier) {
2681         case DRM_FORMAT_MOD_LINEAR:
2682         case I915_FORMAT_MOD_X_TILED:
2683         case I915_FORMAT_MOD_Y_TILED:
2684                 break;
2685         default:
2686                 drm_dbg(&dev_priv->drm,
2687                         "Unsupported modifier for initial FB: 0x%llx\n",
2688                         fb->modifier);
2689                 return false;
2690         }
2691
2692         vma = initial_plane_vma(dev_priv, plane_config);
2693         if (!vma)
2694                 return false;
2695
2696         mode_cmd.pixel_format = fb->format->format;
2697         mode_cmd.width = fb->width;
2698         mode_cmd.height = fb->height;
2699         mode_cmd.pitches[0] = fb->pitches[0];
2700         mode_cmd.modifier[0] = fb->modifier;
2701         mode_cmd.flags = DRM_MODE_FB_MODIFIERS;
2702
2703         if (intel_framebuffer_init(to_intel_framebuffer(fb),
2704                                    vma->obj, &mode_cmd)) {
2705                 drm_dbg_kms(&dev_priv->drm, "intel fb init failed\n");
2706                 goto err_vma;
2707         }
2708
2709         plane_config->vma = vma;
2710         return true;
2711
2712 err_vma:
2713         i915_vma_put(vma);
2714         return false;
2715 }
2716
2717 static void
2718 intel_set_plane_visible(struct intel_crtc_state *crtc_state,
2719                         struct intel_plane_state *plane_state,
2720                         bool visible)
2721 {
2722         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
2723
2724         plane_state->uapi.visible = visible;
2725
2726         if (visible)
2727                 crtc_state->uapi.plane_mask |= drm_plane_mask(&plane->base);
2728         else
2729                 crtc_state->uapi.plane_mask &= ~drm_plane_mask(&plane->base);
2730 }
2731
2732 static void fixup_plane_bitmasks(struct intel_crtc_state *crtc_state)
2733 {
2734         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
2735         struct drm_plane *plane;
2736
2737         /*
2738          * Active_planes aliases if multiple "primary" or cursor planes
2739          * have been used on the same (or wrong) pipe. plane_mask uses
2740          * unique ids, hence we can use that to reconstruct active_planes.
2741          */
2742         crtc_state->enabled_planes = 0;
2743         crtc_state->active_planes = 0;
2744
2745         drm_for_each_plane_mask(plane, &dev_priv->drm,
2746                                 crtc_state->uapi.plane_mask) {
2747                 crtc_state->enabled_planes |= BIT(to_intel_plane(plane)->id);
2748                 crtc_state->active_planes |= BIT(to_intel_plane(plane)->id);
2749         }
2750 }
2751
2752 static void intel_plane_disable_noatomic(struct intel_crtc *crtc,
2753                                          struct intel_plane *plane)
2754 {
2755         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2756         struct intel_crtc_state *crtc_state =
2757                 to_intel_crtc_state(crtc->base.state);
2758         struct intel_plane_state *plane_state =
2759                 to_intel_plane_state(plane->base.state);
2760
2761         drm_dbg_kms(&dev_priv->drm,
2762                     "Disabling [PLANE:%d:%s] on [CRTC:%d:%s]\n",
2763                     plane->base.base.id, plane->base.name,
2764                     crtc->base.base.id, crtc->base.name);
2765
2766         intel_set_plane_visible(crtc_state, plane_state, false);
2767         fixup_plane_bitmasks(crtc_state);
2768         crtc_state->data_rate[plane->id] = 0;
2769         crtc_state->min_cdclk[plane->id] = 0;
2770
2771         if (plane->id == PLANE_PRIMARY)
2772                 hsw_disable_ips(crtc_state);
2773
2774         /*
2775          * Vblank time updates from the shadow to live plane control register
2776          * are blocked if the memory self-refresh mode is active at that
2777          * moment. So to make sure the plane gets truly disabled, disable
2778          * first the self-refresh mode. The self-refresh enable bit in turn
2779          * will be checked/applied by the HW only at the next frame start
2780          * event which is after the vblank start event, so we need to have a
2781          * wait-for-vblank between disabling the plane and the pipe.
2782          */
2783         if (HAS_GMCH(dev_priv) &&
2784             intel_set_memory_cxsr(dev_priv, false))
2785                 intel_wait_for_vblank(dev_priv, crtc->pipe);
2786
2787         /*
2788          * Gen2 reports pipe underruns whenever all planes are disabled.
2789          * So disable underrun reporting before all the planes get disabled.
2790          */
2791         if (IS_GEN(dev_priv, 2) && !crtc_state->active_planes)
2792                 intel_set_cpu_fifo_underrun_reporting(dev_priv, crtc->pipe, false);
2793
2794         intel_disable_plane(plane, crtc_state);
2795 }
2796
2797 static void
2798 intel_find_initial_plane_obj(struct intel_crtc *intel_crtc,
2799                              struct intel_initial_plane_config *plane_config)
2800 {
2801         struct drm_device *dev = intel_crtc->base.dev;
2802         struct drm_i915_private *dev_priv = to_i915(dev);
2803         struct drm_crtc *c;
2804         struct drm_plane *primary = intel_crtc->base.primary;
2805         struct drm_plane_state *plane_state = primary->state;
2806         struct intel_plane *intel_plane = to_intel_plane(primary);
2807         struct intel_plane_state *intel_state =
2808                 to_intel_plane_state(plane_state);
2809         struct intel_crtc_state *crtc_state =
2810                 to_intel_crtc_state(intel_crtc->base.state);
2811         struct drm_framebuffer *fb;
2812         struct i915_vma *vma;
2813
2814         if (!plane_config->fb)
2815                 return;
2816
2817         if (intel_alloc_initial_plane_obj(intel_crtc, plane_config)) {
2818                 fb = &plane_config->fb->base;
2819                 vma = plane_config->vma;
2820                 goto valid_fb;
2821         }
2822
2823         /*
2824          * Failed to alloc the obj, check to see if we should share
2825          * an fb with another CRTC instead
2826          */
2827         for_each_crtc(dev, c) {
2828                 struct intel_plane_state *state;
2829
2830                 if (c == &intel_crtc->base)
2831                         continue;
2832
2833                 if (!to_intel_crtc_state(c->state)->uapi.active)
2834                         continue;
2835
2836                 state = to_intel_plane_state(c->primary->state);
2837                 if (!state->vma)
2838                         continue;
2839
2840                 if (intel_plane_ggtt_offset(state) == plane_config->base) {
2841                         fb = state->hw.fb;
2842                         vma = state->vma;
2843                         goto valid_fb;
2844                 }
2845         }
2846
2847         /*
2848          * We've failed to reconstruct the BIOS FB.  Current display state
2849          * indicates that the primary plane is visible, but has a NULL FB,
2850          * which will lead to problems later if we don't fix it up.  The
2851          * simplest solution is to just disable the primary plane now and
2852          * pretend the BIOS never had it enabled.
2853          */
2854         intel_plane_disable_noatomic(intel_crtc, intel_plane);
2855         if (crtc_state->bigjoiner) {
2856                 struct intel_crtc *slave =
2857                         crtc_state->bigjoiner_linked_crtc;
2858                 intel_plane_disable_noatomic(slave, to_intel_plane(slave->base.primary));
2859         }
2860
2861         return;
2862
2863 valid_fb:
2864         intel_state->hw.rotation = plane_config->rotation;
2865         intel_fill_fb_ggtt_view(&intel_state->view, fb,
2866                                 intel_state->hw.rotation);
2867         intel_state->color_plane[0].stride =
2868                 intel_fb_pitch(fb, 0, intel_state->hw.rotation);
2869
2870         __i915_vma_pin(vma);
2871         intel_state->vma = i915_vma_get(vma);
2872         if (intel_plane_uses_fence(intel_state) && i915_vma_pin_fence(vma) == 0)
2873                 if (vma->fence)
2874                         intel_state->flags |= PLANE_HAS_FENCE;
2875
2876         plane_state->src_x = 0;
2877         plane_state->src_y = 0;
2878         plane_state->src_w = fb->width << 16;
2879         plane_state->src_h = fb->height << 16;
2880
2881         plane_state->crtc_x = 0;
2882         plane_state->crtc_y = 0;
2883         plane_state->crtc_w = fb->width;
2884         plane_state->crtc_h = fb->height;
2885
2886         intel_state->uapi.src = drm_plane_state_src(plane_state);
2887         intel_state->uapi.dst = drm_plane_state_dest(plane_state);
2888
2889         if (plane_config->tiling)
2890                 dev_priv->preserve_bios_swizzle = true;
2891
2892         plane_state->fb = fb;
2893         drm_framebuffer_get(fb);
2894
2895         plane_state->crtc = &intel_crtc->base;
2896         intel_plane_copy_uapi_to_hw_state(intel_state, intel_state,
2897                                           intel_crtc);
2898
2899         intel_frontbuffer_flush(to_intel_frontbuffer(fb), ORIGIN_DIRTYFB);
2900
2901         atomic_or(to_intel_plane(primary)->frontbuffer_bit,
2902                   &to_intel_frontbuffer(fb)->bits);
2903 }
2904
2905
2906 static bool
2907 skl_check_main_ccs_coordinates(struct intel_plane_state *plane_state,
2908                                int main_x, int main_y, u32 main_offset,
2909                                int ccs_plane)
2910 {
2911         const struct drm_framebuffer *fb = plane_state->hw.fb;
2912         int aux_x = plane_state->color_plane[ccs_plane].x;
2913         int aux_y = plane_state->color_plane[ccs_plane].y;
2914         u32 aux_offset = plane_state->color_plane[ccs_plane].offset;
2915         u32 alignment = intel_surf_alignment(fb, ccs_plane);
2916         int hsub;
2917         int vsub;
2918
2919         intel_fb_plane_get_subsampling(&hsub, &vsub, fb, ccs_plane);
2920         while (aux_offset >= main_offset && aux_y <= main_y) {
2921                 int x, y;
2922
2923                 if (aux_x == main_x && aux_y == main_y)
2924                         break;
2925
2926                 if (aux_offset == 0)
2927                         break;
2928
2929                 x = aux_x / hsub;
2930                 y = aux_y / vsub;
2931                 aux_offset = intel_plane_adjust_aligned_offset(&x, &y,
2932                                                                plane_state,
2933                                                                ccs_plane,
2934                                                                aux_offset,
2935                                                                aux_offset -
2936                                                                 alignment);
2937                 aux_x = x * hsub + aux_x % hsub;
2938                 aux_y = y * vsub + aux_y % vsub;
2939         }
2940
2941         if (aux_x != main_x || aux_y != main_y)
2942                 return false;
2943
2944         plane_state->color_plane[ccs_plane].offset = aux_offset;
2945         plane_state->color_plane[ccs_plane].x = aux_x;
2946         plane_state->color_plane[ccs_plane].y = aux_y;
2947
2948         return true;
2949 }
2950
2951 unsigned int
2952 intel_plane_fence_y_offset(const struct intel_plane_state *plane_state)
2953 {
2954         int x = 0, y = 0;
2955
2956         intel_plane_adjust_aligned_offset(&x, &y, plane_state, 0,
2957                                           plane_state->color_plane[0].offset, 0);
2958
2959         return y;
2960 }
2961
2962 static int intel_plane_min_width(struct intel_plane *plane,
2963                                  const struct drm_framebuffer *fb,
2964                                  int color_plane,
2965                                  unsigned int rotation)
2966 {
2967         if (plane->min_width)
2968                 return plane->min_width(fb, color_plane, rotation);
2969         else
2970                 return 1;
2971 }
2972
2973 static int intel_plane_max_width(struct intel_plane *plane,
2974                                  const struct drm_framebuffer *fb,
2975                                  int color_plane,
2976                                  unsigned int rotation)
2977 {
2978         if (plane->max_width)
2979                 return plane->max_width(fb, color_plane, rotation);
2980         else
2981                 return INT_MAX;
2982 }
2983
2984 static int intel_plane_max_height(struct intel_plane *plane,
2985                                   const struct drm_framebuffer *fb,
2986                                   int color_plane,
2987                                   unsigned int rotation)
2988 {
2989         if (plane->max_height)
2990                 return plane->max_height(fb, color_plane, rotation);
2991         else
2992                 return INT_MAX;
2993 }
2994
2995 int skl_calc_main_surface_offset(const struct intel_plane_state *plane_state,
2996                                  int *x, int *y, u32 *offset)
2997 {
2998         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
2999         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
3000         const struct drm_framebuffer *fb = plane_state->hw.fb;
3001         const int aux_plane = intel_main_to_aux_plane(fb, 0);
3002         const u32 aux_offset = plane_state->color_plane[aux_plane].offset;
3003         const u32 alignment = intel_surf_alignment(fb, 0);
3004         const int w = drm_rect_width(&plane_state->uapi.src) >> 16;
3005
3006         intel_add_fb_offsets(x, y, plane_state, 0);
3007         *offset = intel_plane_compute_aligned_offset(x, y, plane_state, 0);
3008         if (drm_WARN_ON(&dev_priv->drm, alignment && !is_power_of_2(alignment)))
3009                 return -EINVAL;
3010
3011         /*
3012          * AUX surface offset is specified as the distance from the
3013          * main surface offset, and it must be non-negative. Make
3014          * sure that is what we will get.
3015          */
3016         if (aux_plane && *offset > aux_offset)
3017                 *offset = intel_plane_adjust_aligned_offset(x, y, plane_state, 0,
3018                                                             *offset,
3019                                                             aux_offset & ~(alignment - 1));
3020
3021         /*
3022          * When using an X-tiled surface, the plane blows up
3023          * if the x offset + width exceed the stride.
3024          *
3025          * TODO: linear and Y-tiled seem fine, Yf untested,
3026          */
3027         if (fb->modifier == I915_FORMAT_MOD_X_TILED) {
3028                 int cpp = fb->format->cpp[0];
3029
3030                 while ((*x + w) * cpp > plane_state->color_plane[0].stride) {
3031                         if (*offset == 0) {
3032                                 drm_dbg_kms(&dev_priv->drm,
3033                                             "Unable to find suitable display surface offset due to X-tiling\n");
3034                                 return -EINVAL;
3035                         }
3036
3037                         *offset = intel_plane_adjust_aligned_offset(x, y, plane_state, 0,
3038                                                                     *offset,
3039                                                                     *offset - alignment);
3040                 }
3041         }
3042
3043         return 0;
3044 }
3045
3046 static int skl_check_main_surface(struct intel_plane_state *plane_state)
3047 {
3048         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
3049         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
3050         const struct drm_framebuffer *fb = plane_state->hw.fb;
3051         const unsigned int rotation = plane_state->hw.rotation;
3052         int x = plane_state->uapi.src.x1 >> 16;
3053         int y = plane_state->uapi.src.y1 >> 16;
3054         const int w = drm_rect_width(&plane_state->uapi.src) >> 16;
3055         const int h = drm_rect_height(&plane_state->uapi.src) >> 16;
3056         const int min_width = intel_plane_min_width(plane, fb, 0, rotation);
3057         const int max_width = intel_plane_max_width(plane, fb, 0, rotation);
3058         const int max_height = intel_plane_max_height(plane, fb, 0, rotation);
3059         const int aux_plane = intel_main_to_aux_plane(fb, 0);
3060         const u32 alignment = intel_surf_alignment(fb, 0);
3061         u32 offset;
3062         int ret;
3063
3064         if (w > max_width || w < min_width || h > max_height) {
3065                 drm_dbg_kms(&dev_priv->drm,
3066                             "requested Y/RGB source size %dx%d outside limits (min: %dx1 max: %dx%d)\n",
3067                             w, h, min_width, max_width, max_height);
3068                 return -EINVAL;
3069         }
3070
3071         ret = skl_calc_main_surface_offset(plane_state, &x, &y, &offset);
3072         if (ret)
3073                 return ret;
3074
3075         /*
3076          * CCS AUX surface doesn't have its own x/y offsets, we must make sure
3077          * they match with the main surface x/y offsets.
3078          */
3079         if (is_ccs_modifier(fb->modifier)) {
3080                 while (!skl_check_main_ccs_coordinates(plane_state, x, y,
3081                                                        offset, aux_plane)) {
3082                         if (offset == 0)
3083                                 break;
3084
3085                         offset = intel_plane_adjust_aligned_offset(&x, &y, plane_state, 0,
3086                                                                    offset, offset - alignment);
3087                 }
3088
3089                 if (x != plane_state->color_plane[aux_plane].x ||
3090                     y != plane_state->color_plane[aux_plane].y) {
3091                         drm_dbg_kms(&dev_priv->drm,
3092                                     "Unable to find suitable display surface offset due to CCS\n");
3093                         return -EINVAL;
3094                 }
3095         }
3096
3097         drm_WARN_ON(&dev_priv->drm, x > 8191 || y > 8191);
3098
3099         plane_state->color_plane[0].offset = offset;
3100         plane_state->color_plane[0].x = x;
3101         plane_state->color_plane[0].y = y;
3102
3103         /*
3104          * Put the final coordinates back so that the src
3105          * coordinate checks will see the right values.
3106          */
3107         drm_rect_translate_to(&plane_state->uapi.src,
3108                               x << 16, y << 16);
3109
3110         return 0;
3111 }
3112
3113 static int skl_check_nv12_aux_surface(struct intel_plane_state *plane_state)
3114 {
3115         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
3116         struct drm_i915_private *i915 = to_i915(plane->base.dev);
3117         const struct drm_framebuffer *fb = plane_state->hw.fb;
3118         unsigned int rotation = plane_state->hw.rotation;
3119         int uv_plane = 1;
3120         int max_width = intel_plane_max_width(plane, fb, uv_plane, rotation);
3121         int max_height = intel_plane_max_height(plane, fb, uv_plane, rotation);
3122         int x = plane_state->uapi.src.x1 >> 17;
3123         int y = plane_state->uapi.src.y1 >> 17;
3124         int w = drm_rect_width(&plane_state->uapi.src) >> 17;
3125         int h = drm_rect_height(&plane_state->uapi.src) >> 17;
3126         u32 offset;
3127
3128         /* FIXME not quite sure how/if these apply to the chroma plane */
3129         if (w > max_width || h > max_height) {
3130                 drm_dbg_kms(&i915->drm,
3131                             "CbCr source size %dx%d too big (limit %dx%d)\n",
3132                             w, h, max_width, max_height);
3133                 return -EINVAL;
3134         }
3135
3136         intel_add_fb_offsets(&x, &y, plane_state, uv_plane);
3137         offset = intel_plane_compute_aligned_offset(&x, &y,
3138                                                     plane_state, uv_plane);
3139
3140         if (is_ccs_modifier(fb->modifier)) {
3141                 int ccs_plane = main_to_ccs_plane(fb, uv_plane);
3142                 u32 aux_offset = plane_state->color_plane[ccs_plane].offset;
3143                 u32 alignment = intel_surf_alignment(fb, uv_plane);
3144
3145                 if (offset > aux_offset)
3146                         offset = intel_plane_adjust_aligned_offset(&x, &y,
3147                                                                    plane_state,
3148                                                                    uv_plane,
3149                                                                    offset,
3150                                                                    aux_offset & ~(alignment - 1));
3151
3152                 while (!skl_check_main_ccs_coordinates(plane_state, x, y,
3153                                                        offset, ccs_plane)) {
3154                         if (offset == 0)
3155                                 break;
3156
3157                         offset = intel_plane_adjust_aligned_offset(&x, &y,
3158                                                                    plane_state,
3159                                                                    uv_plane,
3160                                                                    offset, offset - alignment);
3161                 }
3162
3163                 if (x != plane_state->color_plane[ccs_plane].x ||
3164                     y != plane_state->color_plane[ccs_plane].y) {
3165                         drm_dbg_kms(&i915->drm,
3166                                     "Unable to find suitable display surface offset due to CCS\n");
3167                         return -EINVAL;
3168                 }
3169         }
3170
3171         drm_WARN_ON(&i915->drm, x > 8191 || y > 8191);
3172
3173         plane_state->color_plane[uv_plane].offset = offset;
3174         plane_state->color_plane[uv_plane].x = x;
3175         plane_state->color_plane[uv_plane].y = y;
3176
3177         return 0;
3178 }
3179
3180 static int skl_check_ccs_aux_surface(struct intel_plane_state *plane_state)
3181 {
3182         const struct drm_framebuffer *fb = plane_state->hw.fb;
3183         int src_x = plane_state->uapi.src.x1 >> 16;
3184         int src_y = plane_state->uapi.src.y1 >> 16;
3185         u32 offset;
3186         int ccs_plane;
3187
3188         for (ccs_plane = 0; ccs_plane < fb->format->num_planes; ccs_plane++) {
3189                 int main_hsub, main_vsub;
3190                 int hsub, vsub;
3191                 int x, y;
3192
3193                 if (!is_ccs_plane(fb, ccs_plane) ||
3194                     is_gen12_ccs_cc_plane(fb, ccs_plane))
3195                         continue;
3196
3197                 intel_fb_plane_get_subsampling(&main_hsub, &main_vsub, fb,
3198                                                ccs_to_main_plane(fb, ccs_plane));
3199                 intel_fb_plane_get_subsampling(&hsub, &vsub, fb, ccs_plane);
3200
3201                 hsub *= main_hsub;
3202                 vsub *= main_vsub;
3203                 x = src_x / hsub;
3204                 y = src_y / vsub;
3205
3206                 intel_add_fb_offsets(&x, &y, plane_state, ccs_plane);
3207
3208                 offset = intel_plane_compute_aligned_offset(&x, &y,
3209                                                             plane_state,
3210                                                             ccs_plane);
3211
3212                 plane_state->color_plane[ccs_plane].offset = offset;
3213                 plane_state->color_plane[ccs_plane].x = (x * hsub +
3214                                                          src_x % hsub) /
3215                                                         main_hsub;
3216                 plane_state->color_plane[ccs_plane].y = (y * vsub +
3217                                                          src_y % vsub) /
3218                                                         main_vsub;
3219         }
3220
3221         return 0;
3222 }
3223
3224 int skl_check_plane_surface(struct intel_plane_state *plane_state)
3225 {
3226         const struct drm_framebuffer *fb = plane_state->hw.fb;
3227         int ret, i;
3228
3229         ret = intel_plane_compute_gtt(plane_state);
3230         if (ret)
3231                 return ret;
3232
3233         if (!plane_state->uapi.visible)
3234                 return 0;
3235
3236         /*
3237          * Handle the AUX surface first since the main surface setup depends on
3238          * it.
3239          */
3240         if (is_ccs_modifier(fb->modifier)) {
3241                 ret = skl_check_ccs_aux_surface(plane_state);
3242                 if (ret)
3243                         return ret;
3244         }
3245
3246         if (intel_format_info_is_yuv_semiplanar(fb->format,
3247                                                 fb->modifier)) {
3248                 ret = skl_check_nv12_aux_surface(plane_state);
3249                 if (ret)
3250                         return ret;
3251         }
3252
3253         for (i = fb->format->num_planes; i < ARRAY_SIZE(plane_state->color_plane); i++) {
3254                 plane_state->color_plane[i].offset = 0;
3255                 plane_state->color_plane[i].x = 0;
3256                 plane_state->color_plane[i].y = 0;
3257         }
3258
3259         ret = skl_check_main_surface(plane_state);
3260         if (ret)
3261                 return ret;
3262
3263         return 0;
3264 }
3265
3266 static void skl_detach_scaler(struct intel_crtc *intel_crtc, int id)
3267 {
3268         struct drm_device *dev = intel_crtc->base.dev;
3269         struct drm_i915_private *dev_priv = to_i915(dev);
3270         unsigned long irqflags;
3271
3272         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
3273
3274         intel_de_write_fw(dev_priv, SKL_PS_CTRL(intel_crtc->pipe, id), 0);
3275         intel_de_write_fw(dev_priv, SKL_PS_WIN_POS(intel_crtc->pipe, id), 0);
3276         intel_de_write_fw(dev_priv, SKL_PS_WIN_SZ(intel_crtc->pipe, id), 0);
3277
3278         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
3279 }
3280
3281 /*
3282  * This function detaches (aka. unbinds) unused scalers in hardware
3283  */
3284 static void skl_detach_scalers(const struct intel_crtc_state *crtc_state)
3285 {
3286         struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->uapi.crtc);
3287         const struct intel_crtc_scaler_state *scaler_state =
3288                 &crtc_state->scaler_state;
3289         int i;
3290
3291         /* loop through and disable scalers that aren't in use */
3292         for (i = 0; i < intel_crtc->num_scalers; i++) {
3293                 if (!scaler_state->scalers[i].in_use)
3294                         skl_detach_scaler(intel_crtc, i);
3295         }
3296 }
3297
3298 static unsigned int skl_plane_stride_mult(const struct drm_framebuffer *fb,
3299                                           int color_plane, unsigned int rotation)
3300 {
3301         /*
3302          * The stride is either expressed as a multiple of 64 bytes chunks for
3303          * linear buffers or in number of tiles for tiled buffers.
3304          */
3305         if (is_surface_linear(fb, color_plane))
3306                 return 64;
3307         else if (drm_rotation_90_or_270(rotation))
3308                 return intel_tile_height(fb, color_plane);
3309         else
3310                 return intel_tile_width_bytes(fb, color_plane);
3311 }
3312
3313 u32 skl_plane_stride(const struct intel_plane_state *plane_state,
3314                      int color_plane)
3315 {
3316         const struct drm_framebuffer *fb = plane_state->hw.fb;
3317         unsigned int rotation = plane_state->hw.rotation;
3318         u32 stride = plane_state->color_plane[color_plane].stride;
3319
3320         if (color_plane >= fb->format->num_planes)
3321                 return 0;
3322
3323         return stride / skl_plane_stride_mult(fb, color_plane, rotation);
3324 }
3325
3326 static u32 skl_plane_ctl_format(u32 pixel_format)
3327 {
3328         switch (pixel_format) {
3329         case DRM_FORMAT_C8:
3330                 return PLANE_CTL_FORMAT_INDEXED;
3331         case DRM_FORMAT_RGB565:
3332                 return PLANE_CTL_FORMAT_RGB_565;
3333         case DRM_FORMAT_XBGR8888:
3334         case DRM_FORMAT_ABGR8888:
3335                 return PLANE_CTL_FORMAT_XRGB_8888 | PLANE_CTL_ORDER_RGBX;
3336         case DRM_FORMAT_XRGB8888:
3337         case DRM_FORMAT_ARGB8888:
3338                 return PLANE_CTL_FORMAT_XRGB_8888;
3339         case DRM_FORMAT_XBGR2101010:
3340         case DRM_FORMAT_ABGR2101010:
3341                 return PLANE_CTL_FORMAT_XRGB_2101010 | PLANE_CTL_ORDER_RGBX;
3342         case DRM_FORMAT_XRGB2101010:
3343         case DRM_FORMAT_ARGB2101010:
3344                 return PLANE_CTL_FORMAT_XRGB_2101010;
3345         case DRM_FORMAT_XBGR16161616F:
3346         case DRM_FORMAT_ABGR16161616F:
3347                 return PLANE_CTL_FORMAT_XRGB_16161616F | PLANE_CTL_ORDER_RGBX;
3348         case DRM_FORMAT_XRGB16161616F:
3349         case DRM_FORMAT_ARGB16161616F:
3350                 return PLANE_CTL_FORMAT_XRGB_16161616F;
3351         case DRM_FORMAT_XYUV8888:
3352                 return PLANE_CTL_FORMAT_XYUV;
3353         case DRM_FORMAT_YUYV:
3354                 return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_YUYV;
3355         case DRM_FORMAT_YVYU:
3356                 return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_YVYU;
3357         case DRM_FORMAT_UYVY:
3358                 return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_UYVY;
3359         case DRM_FORMAT_VYUY:
3360                 return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_VYUY;
3361         case DRM_FORMAT_NV12:
3362                 return PLANE_CTL_FORMAT_NV12;
3363         case DRM_FORMAT_P010:
3364                 return PLANE_CTL_FORMAT_P010;
3365         case DRM_FORMAT_P012:
3366                 return PLANE_CTL_FORMAT_P012;
3367         case DRM_FORMAT_P016:
3368                 return PLANE_CTL_FORMAT_P016;
3369         case DRM_FORMAT_Y210:
3370                 return PLANE_CTL_FORMAT_Y210;
3371         case DRM_FORMAT_Y212:
3372                 return PLANE_CTL_FORMAT_Y212;
3373         case DRM_FORMAT_Y216:
3374                 return PLANE_CTL_FORMAT_Y216;
3375         case DRM_FORMAT_XVYU2101010:
3376                 return PLANE_CTL_FORMAT_Y410;
3377         case DRM_FORMAT_XVYU12_16161616:
3378                 return PLANE_CTL_FORMAT_Y412;
3379         case DRM_FORMAT_XVYU16161616:
3380                 return PLANE_CTL_FORMAT_Y416;
3381         default:
3382                 MISSING_CASE(pixel_format);
3383         }
3384
3385         return 0;
3386 }
3387
3388 static u32 skl_plane_ctl_alpha(const struct intel_plane_state *plane_state)
3389 {
3390         if (!plane_state->hw.fb->format->has_alpha)
3391                 return PLANE_CTL_ALPHA_DISABLE;
3392
3393         switch (plane_state->hw.pixel_blend_mode) {
3394         case DRM_MODE_BLEND_PIXEL_NONE:
3395                 return PLANE_CTL_ALPHA_DISABLE;
3396         case DRM_MODE_BLEND_PREMULTI:
3397                 return PLANE_CTL_ALPHA_SW_PREMULTIPLY;
3398         case DRM_MODE_BLEND_COVERAGE:
3399                 return PLANE_CTL_ALPHA_HW_PREMULTIPLY;
3400         default:
3401                 MISSING_CASE(plane_state->hw.pixel_blend_mode);
3402                 return PLANE_CTL_ALPHA_DISABLE;
3403         }
3404 }
3405
3406 static u32 glk_plane_color_ctl_alpha(const struct intel_plane_state *plane_state)
3407 {
3408         if (!plane_state->hw.fb->format->has_alpha)
3409                 return PLANE_COLOR_ALPHA_DISABLE;
3410
3411         switch (plane_state->hw.pixel_blend_mode) {
3412         case DRM_MODE_BLEND_PIXEL_NONE:
3413                 return PLANE_COLOR_ALPHA_DISABLE;
3414         case DRM_MODE_BLEND_PREMULTI:
3415                 return PLANE_COLOR_ALPHA_SW_PREMULTIPLY;
3416         case DRM_MODE_BLEND_COVERAGE:
3417                 return PLANE_COLOR_ALPHA_HW_PREMULTIPLY;
3418         default:
3419                 MISSING_CASE(plane_state->hw.pixel_blend_mode);
3420                 return PLANE_COLOR_ALPHA_DISABLE;
3421         }
3422 }
3423
3424 static u32 skl_plane_ctl_tiling(u64 fb_modifier)
3425 {
3426         switch (fb_modifier) {
3427         case DRM_FORMAT_MOD_LINEAR:
3428                 break;
3429         case I915_FORMAT_MOD_X_TILED:
3430                 return PLANE_CTL_TILED_X;
3431         case I915_FORMAT_MOD_Y_TILED:
3432                 return PLANE_CTL_TILED_Y;
3433         case I915_FORMAT_MOD_Y_TILED_CCS:
3434         case I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS_CC:
3435                 return PLANE_CTL_TILED_Y | PLANE_CTL_RENDER_DECOMPRESSION_ENABLE;
3436         case I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS:
3437                 return PLANE_CTL_TILED_Y |
3438                        PLANE_CTL_RENDER_DECOMPRESSION_ENABLE |
3439                        PLANE_CTL_CLEAR_COLOR_DISABLE;
3440         case I915_FORMAT_MOD_Y_TILED_GEN12_MC_CCS:
3441                 return PLANE_CTL_TILED_Y | PLANE_CTL_MEDIA_DECOMPRESSION_ENABLE;
3442         case I915_FORMAT_MOD_Yf_TILED:
3443                 return PLANE_CTL_TILED_YF;
3444         case I915_FORMAT_MOD_Yf_TILED_CCS:
3445                 return PLANE_CTL_TILED_YF | PLANE_CTL_RENDER_DECOMPRESSION_ENABLE;
3446         default:
3447                 MISSING_CASE(fb_modifier);
3448         }
3449
3450         return 0;
3451 }
3452
3453 static u32 skl_plane_ctl_rotate(unsigned int rotate)
3454 {
3455         switch (rotate) {
3456         case DRM_MODE_ROTATE_0:
3457                 break;
3458         /*
3459          * DRM_MODE_ROTATE_ is counter clockwise to stay compatible with Xrandr
3460          * while i915 HW rotation is clockwise, thats why this swapping.
3461          */
3462         case DRM_MODE_ROTATE_90:
3463                 return PLANE_CTL_ROTATE_270;
3464         case DRM_MODE_ROTATE_180:
3465                 return PLANE_CTL_ROTATE_180;
3466         case DRM_MODE_ROTATE_270:
3467                 return PLANE_CTL_ROTATE_90;
3468         default:
3469                 MISSING_CASE(rotate);
3470         }
3471
3472         return 0;
3473 }
3474
3475 static u32 cnl_plane_ctl_flip(unsigned int reflect)
3476 {
3477         switch (reflect) {
3478         case 0:
3479                 break;
3480         case DRM_MODE_REFLECT_X:
3481                 return PLANE_CTL_FLIP_HORIZONTAL;
3482         case DRM_MODE_REFLECT_Y:
3483         default:
3484                 MISSING_CASE(reflect);
3485         }
3486
3487         return 0;
3488 }
3489
3490 u32 skl_plane_ctl_crtc(const struct intel_crtc_state *crtc_state)
3491 {
3492         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
3493         u32 plane_ctl = 0;
3494
3495         if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
3496                 return plane_ctl;
3497
3498         if (crtc_state->gamma_enable)
3499                 plane_ctl |= PLANE_CTL_PIPE_GAMMA_ENABLE;
3500
3501         if (crtc_state->csc_enable)
3502                 plane_ctl |= PLANE_CTL_PIPE_CSC_ENABLE;
3503
3504         return plane_ctl;
3505 }
3506
3507 u32 skl_plane_ctl(const struct intel_crtc_state *crtc_state,
3508                   const struct intel_plane_state *plane_state)
3509 {
3510         struct drm_i915_private *dev_priv =
3511                 to_i915(plane_state->uapi.plane->dev);
3512         const struct drm_framebuffer *fb = plane_state->hw.fb;
3513         unsigned int rotation = plane_state->hw.rotation;
3514         const struct drm_intel_sprite_colorkey *key = &plane_state->ckey;
3515         u32 plane_ctl;
3516
3517         plane_ctl = PLANE_CTL_ENABLE;
3518
3519         if (INTEL_GEN(dev_priv) < 10 && !IS_GEMINILAKE(dev_priv)) {
3520                 plane_ctl |= skl_plane_ctl_alpha(plane_state);
3521                 plane_ctl |= PLANE_CTL_PLANE_GAMMA_DISABLE;
3522
3523                 if (plane_state->hw.color_encoding == DRM_COLOR_YCBCR_BT709)
3524                         plane_ctl |= PLANE_CTL_YUV_TO_RGB_CSC_FORMAT_BT709;
3525
3526                 if (plane_state->hw.color_range == DRM_COLOR_YCBCR_FULL_RANGE)
3527                         plane_ctl |= PLANE_CTL_YUV_RANGE_CORRECTION_DISABLE;
3528         }
3529
3530         plane_ctl |= skl_plane_ctl_format(fb->format->format);
3531         plane_ctl |= skl_plane_ctl_tiling(fb->modifier);
3532         plane_ctl |= skl_plane_ctl_rotate(rotation & DRM_MODE_ROTATE_MASK);
3533
3534         if (INTEL_GEN(dev_priv) >= 10)
3535                 plane_ctl |= cnl_plane_ctl_flip(rotation &
3536                                                 DRM_MODE_REFLECT_MASK);
3537
3538         if (key->flags & I915_SET_COLORKEY_DESTINATION)
3539                 plane_ctl |= PLANE_CTL_KEY_ENABLE_DESTINATION;
3540         else if (key->flags & I915_SET_COLORKEY_SOURCE)
3541                 plane_ctl |= PLANE_CTL_KEY_ENABLE_SOURCE;
3542
3543         return plane_ctl;
3544 }
3545
3546 u32 glk_plane_color_ctl_crtc(const struct intel_crtc_state *crtc_state)
3547 {
3548         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
3549         u32 plane_color_ctl = 0;
3550
3551         if (INTEL_GEN(dev_priv) >= 11)
3552                 return plane_color_ctl;
3553
3554         if (crtc_state->gamma_enable)
3555                 plane_color_ctl |= PLANE_COLOR_PIPE_GAMMA_ENABLE;
3556
3557         if (crtc_state->csc_enable)
3558                 plane_color_ctl |= PLANE_COLOR_PIPE_CSC_ENABLE;
3559
3560         return plane_color_ctl;
3561 }
3562
3563 u32 glk_plane_color_ctl(const struct intel_crtc_state *crtc_state,
3564                         const struct intel_plane_state *plane_state)
3565 {
3566         struct drm_i915_private *dev_priv =
3567                 to_i915(plane_state->uapi.plane->dev);
3568         const struct drm_framebuffer *fb = plane_state->hw.fb;
3569         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
3570         u32 plane_color_ctl = 0;
3571
3572         plane_color_ctl |= PLANE_COLOR_PLANE_GAMMA_DISABLE;
3573         plane_color_ctl |= glk_plane_color_ctl_alpha(plane_state);
3574
3575         if (fb->format->is_yuv && !icl_is_hdr_plane(dev_priv, plane->id)) {
3576                 switch (plane_state->hw.color_encoding) {
3577                 case DRM_COLOR_YCBCR_BT709:
3578                         plane_color_ctl |= PLANE_COLOR_CSC_MODE_YUV709_TO_RGB709;
3579                         break;
3580                 case DRM_COLOR_YCBCR_BT2020:
3581                         plane_color_ctl |=
3582                                 PLANE_COLOR_CSC_MODE_YUV2020_TO_RGB2020;
3583                         break;
3584                 default:
3585                         plane_color_ctl |=
3586                                 PLANE_COLOR_CSC_MODE_YUV601_TO_RGB601;
3587                 }
3588                 if (plane_state->hw.color_range == DRM_COLOR_YCBCR_FULL_RANGE)
3589                         plane_color_ctl |= PLANE_COLOR_YUV_RANGE_CORRECTION_DISABLE;
3590         } else if (fb->format->is_yuv) {
3591                 plane_color_ctl |= PLANE_COLOR_INPUT_CSC_ENABLE;
3592                 if (plane_state->hw.color_range == DRM_COLOR_YCBCR_FULL_RANGE)
3593                         plane_color_ctl |= PLANE_COLOR_YUV_RANGE_CORRECTION_DISABLE;
3594         }
3595
3596         return plane_color_ctl;
3597 }
3598
3599 static int
3600 __intel_display_resume(struct drm_device *dev,
3601                        struct drm_atomic_state *state,
3602                        struct drm_modeset_acquire_ctx *ctx)
3603 {
3604         struct drm_crtc_state *crtc_state;
3605         struct drm_crtc *crtc;
3606         int i, ret;
3607
3608         intel_modeset_setup_hw_state(dev, ctx);
3609         intel_vga_redisable(to_i915(dev));
3610
3611         if (!state)
3612                 return 0;
3613
3614         /*
3615          * We've duplicated the state, pointers to the old state are invalid.
3616          *
3617          * Don't attempt to use the old state until we commit the duplicated state.
3618          */
3619         for_each_new_crtc_in_state(state, crtc, crtc_state, i) {
3620                 /*
3621                  * Force recalculation even if we restore
3622                  * current state. With fast modeset this may not result
3623                  * in a modeset when the state is compatible.
3624                  */
3625                 crtc_state->mode_changed = true;
3626         }
3627
3628         /* ignore any reset values/BIOS leftovers in the WM registers */
3629         if (!HAS_GMCH(to_i915(dev)))
3630                 to_intel_atomic_state(state)->skip_intermediate_wm = true;
3631
3632         ret = drm_atomic_helper_commit_duplicated_state(state, ctx);
3633
3634         drm_WARN_ON(dev, ret == -EDEADLK);
3635         return ret;
3636 }
3637
3638 static bool gpu_reset_clobbers_display(struct drm_i915_private *dev_priv)
3639 {
3640         return (INTEL_INFO(dev_priv)->gpu_reset_clobbers_display &&
3641                 intel_has_gpu_reset(&dev_priv->gt));
3642 }
3643
3644 void intel_display_prepare_reset(struct drm_i915_private *dev_priv)
3645 {
3646         struct drm_device *dev = &dev_priv->drm;
3647         struct drm_modeset_acquire_ctx *ctx = &dev_priv->reset_ctx;
3648         struct drm_atomic_state *state;
3649         int ret;
3650
3651         if (!HAS_DISPLAY(dev_priv))
3652                 return;
3653
3654         /* reset doesn't touch the display */
3655         if (!dev_priv->params.force_reset_modeset_test &&
3656             !gpu_reset_clobbers_display(dev_priv))
3657                 return;
3658
3659         /* We have a modeset vs reset deadlock, defensively unbreak it. */
3660         set_bit(I915_RESET_MODESET, &dev_priv->gt.reset.flags);
3661         smp_mb__after_atomic();
3662         wake_up_bit(&dev_priv->gt.reset.flags, I915_RESET_MODESET);
3663
3664         if (atomic_read(&dev_priv->gpu_error.pending_fb_pin)) {
3665                 drm_dbg_kms(&dev_priv->drm,
3666                             "Modeset potentially stuck, unbreaking through wedging\n");
3667                 intel_gt_set_wedged(&dev_priv->gt);
3668         }
3669
3670         /*
3671          * Need mode_config.mutex so that we don't
3672          * trample ongoing ->detect() and whatnot.
3673          */
3674         mutex_lock(&dev->mode_config.mutex);
3675         drm_modeset_acquire_init(ctx, 0);
3676         while (1) {
3677                 ret = drm_modeset_lock_all_ctx(dev, ctx);
3678                 if (ret != -EDEADLK)
3679                         break;
3680
3681                 drm_modeset_backoff(ctx);
3682         }
3683         /*
3684          * Disabling the crtcs gracefully seems nicer. Also the
3685          * g33 docs say we should at least disable all the planes.
3686          */
3687         state = drm_atomic_helper_duplicate_state(dev, ctx);
3688         if (IS_ERR(state)) {
3689                 ret = PTR_ERR(state);
3690                 drm_err(&dev_priv->drm, "Duplicating state failed with %i\n",
3691                         ret);
3692                 return;
3693         }
3694
3695         ret = drm_atomic_helper_disable_all(dev, ctx);
3696         if (ret) {
3697                 drm_err(&dev_priv->drm, "Suspending crtc's failed with %i\n",
3698                         ret);
3699                 drm_atomic_state_put(state);
3700                 return;
3701         }
3702
3703         dev_priv->modeset_restore_state = state;
3704         state->acquire_ctx = ctx;
3705 }
3706
3707 void intel_display_finish_reset(struct drm_i915_private *dev_priv)
3708 {
3709         struct drm_device *dev = &dev_priv->drm;
3710         struct drm_modeset_acquire_ctx *ctx = &dev_priv->reset_ctx;
3711         struct drm_atomic_state *state;
3712         int ret;
3713
3714         if (!HAS_DISPLAY(dev_priv))
3715                 return;
3716
3717         /* reset doesn't touch the display */
3718         if (!test_bit(I915_RESET_MODESET, &dev_priv->gt.reset.flags))
3719                 return;
3720
3721         state = fetch_and_zero(&dev_priv->modeset_restore_state);
3722         if (!state)
3723                 goto unlock;
3724
3725         /* reset doesn't touch the display */
3726         if (!gpu_reset_clobbers_display(dev_priv)) {
3727                 /* for testing only restore the display */
3728                 ret = __intel_display_resume(dev, state, ctx);
3729                 if (ret)
3730                         drm_err(&dev_priv->drm,
3731                                 "Restoring old state failed with %i\n", ret);
3732         } else {
3733                 /*
3734                  * The display has been reset as well,
3735                  * so need a full re-initialization.
3736                  */
3737                 intel_pps_unlock_regs_wa(dev_priv);
3738                 intel_modeset_init_hw(dev_priv);
3739                 intel_init_clock_gating(dev_priv);
3740                 intel_hpd_init(dev_priv);
3741
3742                 ret = __intel_display_resume(dev, state, ctx);
3743                 if (ret)
3744                         drm_err(&dev_priv->drm,
3745                                 "Restoring old state failed with %i\n", ret);
3746
3747                 intel_hpd_poll_disable(dev_priv);
3748         }
3749
3750         drm_atomic_state_put(state);
3751 unlock:
3752         drm_modeset_drop_locks(ctx);
3753         drm_modeset_acquire_fini(ctx);
3754         mutex_unlock(&dev->mode_config.mutex);
3755
3756         clear_bit_unlock(I915_RESET_MODESET, &dev_priv->gt.reset.flags);
3757 }
3758
3759 static void icl_set_pipe_chicken(struct intel_crtc *crtc)
3760 {
3761         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3762         enum pipe pipe = crtc->pipe;
3763         u32 tmp;
3764
3765         tmp = intel_de_read(dev_priv, PIPE_CHICKEN(pipe));
3766
3767         /*
3768          * Display WA #1153: icl
3769          * enable hardware to bypass the alpha math
3770          * and rounding for per-pixel values 00 and 0xff
3771          */
3772         tmp |= PER_PIXEL_ALPHA_BYPASS_EN;
3773         /*
3774          * Display WA # 1605353570: icl
3775          * Set the pixel rounding bit to 1 for allowing
3776          * passthrough of Frame buffer pixels unmodified
3777          * across pipe
3778          */
3779         tmp |= PIXEL_ROUNDING_TRUNC_FB_PASSTHRU;
3780         intel_de_write(dev_priv, PIPE_CHICKEN(pipe), tmp);
3781 }
3782
3783 bool intel_has_pending_fb_unpin(struct drm_i915_private *dev_priv)
3784 {
3785         struct drm_crtc *crtc;
3786         bool cleanup_done;
3787
3788         drm_for_each_crtc(crtc, &dev_priv->drm) {
3789                 struct drm_crtc_commit *commit;
3790                 spin_lock(&crtc->commit_lock);
3791                 commit = list_first_entry_or_null(&crtc->commit_list,
3792                                                   struct drm_crtc_commit, commit_entry);
3793                 cleanup_done = commit ?
3794                         try_wait_for_completion(&commit->cleanup_done) : true;
3795                 spin_unlock(&crtc->commit_lock);
3796
3797                 if (cleanup_done)
3798                         continue;
3799
3800                 drm_crtc_wait_one_vblank(crtc);
3801
3802                 return true;
3803         }
3804
3805         return false;
3806 }
3807
3808 void lpt_disable_iclkip(struct drm_i915_private *dev_priv)
3809 {
3810         u32 temp;
3811
3812         intel_de_write(dev_priv, PIXCLK_GATE, PIXCLK_GATE_GATE);
3813
3814         mutex_lock(&dev_priv->sb_lock);
3815
3816         temp = intel_sbi_read(dev_priv, SBI_SSCCTL6, SBI_ICLK);
3817         temp |= SBI_SSCCTL_DISABLE;
3818         intel_sbi_write(dev_priv, SBI_SSCCTL6, temp, SBI_ICLK);
3819
3820         mutex_unlock(&dev_priv->sb_lock);
3821 }
3822
3823 /* Program iCLKIP clock to the desired frequency */
3824 static void lpt_program_iclkip(const struct intel_crtc_state *crtc_state)
3825 {
3826         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3827         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3828         int clock = crtc_state->hw.adjusted_mode.crtc_clock;
3829         u32 divsel, phaseinc, auxdiv, phasedir = 0;
3830         u32 temp;
3831
3832         lpt_disable_iclkip(dev_priv);
3833
3834         /* The iCLK virtual clock root frequency is in MHz,
3835          * but the adjusted_mode->crtc_clock in in KHz. To get the
3836          * divisors, it is necessary to divide one by another, so we
3837          * convert the virtual clock precision to KHz here for higher
3838          * precision.
3839          */
3840         for (auxdiv = 0; auxdiv < 2; auxdiv++) {
3841                 u32 iclk_virtual_root_freq = 172800 * 1000;
3842                 u32 iclk_pi_range = 64;
3843                 u32 desired_divisor;
3844
3845                 desired_divisor = DIV_ROUND_CLOSEST(iclk_virtual_root_freq,
3846                                                     clock << auxdiv);
3847                 divsel = (desired_divisor / iclk_pi_range) - 2;
3848                 phaseinc = desired_divisor % iclk_pi_range;
3849
3850                 /*
3851                  * Near 20MHz is a corner case which is
3852                  * out of range for the 7-bit divisor
3853                  */
3854                 if (divsel <= 0x7f)
3855                         break;
3856         }
3857
3858         /* This should not happen with any sane values */
3859         drm_WARN_ON(&dev_priv->drm, SBI_SSCDIVINTPHASE_DIVSEL(divsel) &
3860                     ~SBI_SSCDIVINTPHASE_DIVSEL_MASK);
3861         drm_WARN_ON(&dev_priv->drm, SBI_SSCDIVINTPHASE_DIR(phasedir) &
3862                     ~SBI_SSCDIVINTPHASE_INCVAL_MASK);
3863
3864         drm_dbg_kms(&dev_priv->drm,
3865                     "iCLKIP clock: found settings for %dKHz refresh rate: auxdiv=%x, divsel=%x, phasedir=%x, phaseinc=%x\n",
3866                     clock, auxdiv, divsel, phasedir, phaseinc);
3867
3868         mutex_lock(&dev_priv->sb_lock);
3869
3870         /* Program SSCDIVINTPHASE6 */
3871         temp = intel_sbi_read(dev_priv, SBI_SSCDIVINTPHASE6, SBI_ICLK);
3872         temp &= ~SBI_SSCDIVINTPHASE_DIVSEL_MASK;
3873         temp |= SBI_SSCDIVINTPHASE_DIVSEL(divsel);
3874         temp &= ~SBI_SSCDIVINTPHASE_INCVAL_MASK;
3875         temp |= SBI_SSCDIVINTPHASE_INCVAL(phaseinc);
3876         temp |= SBI_SSCDIVINTPHASE_DIR(phasedir);
3877         temp |= SBI_SSCDIVINTPHASE_PROPAGATE;
3878         intel_sbi_write(dev_priv, SBI_SSCDIVINTPHASE6, temp, SBI_ICLK);
3879
3880         /* Program SSCAUXDIV */
3881         temp = intel_sbi_read(dev_priv, SBI_SSCAUXDIV6, SBI_ICLK);
3882         temp &= ~SBI_SSCAUXDIV_FINALDIV2SEL(1);
3883         temp |= SBI_SSCAUXDIV_FINALDIV2SEL(auxdiv);
3884         intel_sbi_write(dev_priv, SBI_SSCAUXDIV6, temp, SBI_ICLK);
3885
3886         /* Enable modulator and associated divider */
3887         temp = intel_sbi_read(dev_priv, SBI_SSCCTL6, SBI_ICLK);
3888         temp &= ~SBI_SSCCTL_DISABLE;
3889         intel_sbi_write(dev_priv, SBI_SSCCTL6, temp, SBI_ICLK);
3890
3891         mutex_unlock(&dev_priv->sb_lock);
3892
3893         /* Wait for initialization time */
3894         udelay(24);
3895
3896         intel_de_write(dev_priv, PIXCLK_GATE, PIXCLK_GATE_UNGATE);
3897 }
3898
3899 int lpt_get_iclkip(struct drm_i915_private *dev_priv)
3900 {
3901         u32 divsel, phaseinc, auxdiv;
3902         u32 iclk_virtual_root_freq = 172800 * 1000;
3903         u32 iclk_pi_range = 64;
3904         u32 desired_divisor;
3905         u32 temp;
3906
3907         if ((intel_de_read(dev_priv, PIXCLK_GATE) & PIXCLK_GATE_UNGATE) == 0)
3908                 return 0;
3909
3910         mutex_lock(&dev_priv->sb_lock);
3911
3912         temp = intel_sbi_read(dev_priv, SBI_SSCCTL6, SBI_ICLK);
3913         if (temp & SBI_SSCCTL_DISABLE) {
3914                 mutex_unlock(&dev_priv->sb_lock);
3915                 return 0;
3916         }
3917
3918         temp = intel_sbi_read(dev_priv, SBI_SSCDIVINTPHASE6, SBI_ICLK);
3919         divsel = (temp & SBI_SSCDIVINTPHASE_DIVSEL_MASK) >>
3920                 SBI_SSCDIVINTPHASE_DIVSEL_SHIFT;
3921         phaseinc = (temp & SBI_SSCDIVINTPHASE_INCVAL_MASK) >>
3922                 SBI_SSCDIVINTPHASE_INCVAL_SHIFT;
3923
3924         temp = intel_sbi_read(dev_priv, SBI_SSCAUXDIV6, SBI_ICLK);
3925         auxdiv = (temp & SBI_SSCAUXDIV_FINALDIV2SEL_MASK) >>
3926                 SBI_SSCAUXDIV_FINALDIV2SEL_SHIFT;
3927
3928         mutex_unlock(&dev_priv->sb_lock);
3929
3930         desired_divisor = (divsel + 2) * iclk_pi_range + phaseinc;
3931
3932         return DIV_ROUND_CLOSEST(iclk_virtual_root_freq,
3933                                  desired_divisor << auxdiv);
3934 }
3935
3936 static void ilk_pch_transcoder_set_timings(const struct intel_crtc_state *crtc_state,
3937                                            enum pipe pch_transcoder)
3938 {
3939         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3940         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3941         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
3942
3943         intel_de_write(dev_priv, PCH_TRANS_HTOTAL(pch_transcoder),
3944                        intel_de_read(dev_priv, HTOTAL(cpu_transcoder)));
3945         intel_de_write(dev_priv, PCH_TRANS_HBLANK(pch_transcoder),
3946                        intel_de_read(dev_priv, HBLANK(cpu_transcoder)));
3947         intel_de_write(dev_priv, PCH_TRANS_HSYNC(pch_transcoder),
3948                        intel_de_read(dev_priv, HSYNC(cpu_transcoder)));
3949
3950         intel_de_write(dev_priv, PCH_TRANS_VTOTAL(pch_transcoder),
3951                        intel_de_read(dev_priv, VTOTAL(cpu_transcoder)));
3952         intel_de_write(dev_priv, PCH_TRANS_VBLANK(pch_transcoder),
3953                        intel_de_read(dev_priv, VBLANK(cpu_transcoder)));
3954         intel_de_write(dev_priv, PCH_TRANS_VSYNC(pch_transcoder),
3955                        intel_de_read(dev_priv, VSYNC(cpu_transcoder)));
3956         intel_de_write(dev_priv, PCH_TRANS_VSYNCSHIFT(pch_transcoder),
3957                        intel_de_read(dev_priv, VSYNCSHIFT(cpu_transcoder)));
3958 }
3959
3960 static void cpt_set_fdi_bc_bifurcation(struct drm_i915_private *dev_priv, bool enable)
3961 {
3962         u32 temp;
3963
3964         temp = intel_de_read(dev_priv, SOUTH_CHICKEN1);
3965         if (!!(temp & FDI_BC_BIFURCATION_SELECT) == enable)
3966                 return;
3967
3968         drm_WARN_ON(&dev_priv->drm,
3969                     intel_de_read(dev_priv, FDI_RX_CTL(PIPE_B)) &
3970                     FDI_RX_ENABLE);
3971         drm_WARN_ON(&dev_priv->drm,
3972                     intel_de_read(dev_priv, FDI_RX_CTL(PIPE_C)) &
3973                     FDI_RX_ENABLE);
3974
3975         temp &= ~FDI_BC_BIFURCATION_SELECT;
3976         if (enable)
3977                 temp |= FDI_BC_BIFURCATION_SELECT;
3978
3979         drm_dbg_kms(&dev_priv->drm, "%sabling fdi C rx\n",
3980                     enable ? "en" : "dis");
3981         intel_de_write(dev_priv, SOUTH_CHICKEN1, temp);
3982         intel_de_posting_read(dev_priv, SOUTH_CHICKEN1);
3983 }
3984
3985 static void ivb_update_fdi_bc_bifurcation(const struct intel_crtc_state *crtc_state)
3986 {
3987         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3988         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3989
3990         switch (crtc->pipe) {
3991         case PIPE_A:
3992                 break;
3993         case PIPE_B:
3994                 if (crtc_state->fdi_lanes > 2)
3995                         cpt_set_fdi_bc_bifurcation(dev_priv, false);
3996                 else
3997                         cpt_set_fdi_bc_bifurcation(dev_priv, true);
3998
3999                 break;
4000         case PIPE_C:
4001                 cpt_set_fdi_bc_bifurcation(dev_priv, true);
4002
4003                 break;
4004         default:
4005                 BUG();
4006         }
4007 }
4008
4009 /*
4010  * Finds the encoder associated with the given CRTC. This can only be
4011  * used when we know that the CRTC isn't feeding multiple encoders!
4012  */
4013 struct intel_encoder *
4014 intel_get_crtc_new_encoder(const struct intel_atomic_state *state,
4015                            const struct intel_crtc_state *crtc_state)
4016 {
4017         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
4018         const struct drm_connector_state *connector_state;
4019         const struct drm_connector *connector;
4020         struct intel_encoder *encoder = NULL;
4021         int num_encoders = 0;
4022         int i;
4023
4024         for_each_new_connector_in_state(&state->base, connector, connector_state, i) {
4025                 if (connector_state->crtc != &crtc->base)
4026                         continue;
4027
4028                 encoder = to_intel_encoder(connector_state->best_encoder);
4029                 num_encoders++;
4030         }
4031
4032         drm_WARN(encoder->base.dev, num_encoders != 1,
4033                  "%d encoders for pipe %c\n",
4034                  num_encoders, pipe_name(crtc->pipe));
4035
4036         return encoder;
4037 }
4038
4039 /*
4040  * Enable PCH resources required for PCH ports:
4041  *   - PCH PLLs
4042  *   - FDI training & RX/TX
4043  *   - update transcoder timings
4044  *   - DP transcoding bits
4045  *   - transcoder
4046  */
4047 static void ilk_pch_enable(const struct intel_atomic_state *state,
4048                            const struct intel_crtc_state *crtc_state)
4049 {
4050         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
4051         struct drm_device *dev = crtc->base.dev;
4052         struct drm_i915_private *dev_priv = to_i915(dev);
4053         enum pipe pipe = crtc->pipe;
4054         u32 temp;
4055
4056         assert_pch_transcoder_disabled(dev_priv, pipe);
4057
4058         if (IS_IVYBRIDGE(dev_priv))
4059                 ivb_update_fdi_bc_bifurcation(crtc_state);
4060
4061         /* Write the TU size bits before fdi link training, so that error
4062          * detection works. */
4063         intel_de_write(dev_priv, FDI_RX_TUSIZE1(pipe),
4064                        intel_de_read(dev_priv, PIPE_DATA_M1(pipe)) & TU_SIZE_MASK);
4065
4066         /* For PCH output, training FDI link */
4067         dev_priv->display.fdi_link_train(crtc, crtc_state);
4068
4069         /* We need to program the right clock selection before writing the pixel
4070          * mutliplier into the DPLL. */
4071         if (HAS_PCH_CPT(dev_priv)) {
4072                 u32 sel;
4073
4074                 temp = intel_de_read(dev_priv, PCH_DPLL_SEL);
4075                 temp |= TRANS_DPLL_ENABLE(pipe);
4076                 sel = TRANS_DPLLB_SEL(pipe);
4077                 if (crtc_state->shared_dpll ==
4078                     intel_get_shared_dpll_by_id(dev_priv, DPLL_ID_PCH_PLL_B))
4079                         temp |= sel;
4080                 else
4081                         temp &= ~sel;
4082                 intel_de_write(dev_priv, PCH_DPLL_SEL, temp);
4083         }
4084
4085         /* XXX: pch pll's can be enabled any time before we enable the PCH
4086          * transcoder, and we actually should do this to not upset any PCH
4087          * transcoder that already use the clock when we share it.
4088          *
4089          * Note that enable_shared_dpll tries to do the right thing, but
4090          * get_shared_dpll unconditionally resets the pll - we need that to have
4091          * the right LVDS enable sequence. */
4092         intel_enable_shared_dpll(crtc_state);
4093
4094         /* set transcoder timing, panel must allow it */
4095         assert_panel_unlocked(dev_priv, pipe);
4096         ilk_pch_transcoder_set_timings(crtc_state, pipe);
4097
4098         intel_fdi_normal_train(crtc);
4099
4100         /* For PCH DP, enable TRANS_DP_CTL */
4101         if (HAS_PCH_CPT(dev_priv) &&
4102             intel_crtc_has_dp_encoder(crtc_state)) {
4103                 const struct drm_display_mode *adjusted_mode =
4104                         &crtc_state->hw.adjusted_mode;
4105                 u32 bpc = (intel_de_read(dev_priv, PIPECONF(pipe)) & PIPECONF_BPC_MASK) >> 5;
4106                 i915_reg_t reg = TRANS_DP_CTL(pipe);
4107                 enum port port;
4108
4109                 temp = intel_de_read(dev_priv, reg);
4110                 temp &= ~(TRANS_DP_PORT_SEL_MASK |
4111                           TRANS_DP_SYNC_MASK |
4112                           TRANS_DP_BPC_MASK);
4113                 temp |= TRANS_DP_OUTPUT_ENABLE;
4114                 temp |= bpc << 9; /* same format but at 11:9 */
4115
4116                 if (adjusted_mode->flags & DRM_MODE_FLAG_PHSYNC)
4117                         temp |= TRANS_DP_HSYNC_ACTIVE_HIGH;
4118                 if (adjusted_mode->flags & DRM_MODE_FLAG_PVSYNC)
4119                         temp |= TRANS_DP_VSYNC_ACTIVE_HIGH;
4120
4121                 port = intel_get_crtc_new_encoder(state, crtc_state)->port;
4122                 drm_WARN_ON(dev, port < PORT_B || port > PORT_D);
4123                 temp |= TRANS_DP_PORT_SEL(port);
4124
4125                 intel_de_write(dev_priv, reg, temp);
4126         }
4127
4128         ilk_enable_pch_transcoder(crtc_state);
4129 }
4130
4131 void lpt_pch_enable(const struct intel_crtc_state *crtc_state)
4132 {
4133         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
4134         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4135         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
4136
4137         assert_pch_transcoder_disabled(dev_priv, PIPE_A);
4138
4139         lpt_program_iclkip(crtc_state);
4140
4141         /* Set transcoder timing. */
4142         ilk_pch_transcoder_set_timings(crtc_state, PIPE_A);
4143
4144         lpt_enable_pch_transcoder(dev_priv, cpu_transcoder);
4145 }
4146
4147 static void cpt_verify_modeset(struct drm_i915_private *dev_priv,
4148                                enum pipe pipe)
4149 {
4150         i915_reg_t dslreg = PIPEDSL(pipe);
4151         u32 temp;
4152
4153         temp = intel_de_read(dev_priv, dslreg);
4154         udelay(500);
4155         if (wait_for(intel_de_read(dev_priv, dslreg) != temp, 5)) {
4156                 if (wait_for(intel_de_read(dev_priv, dslreg) != temp, 5))
4157                         drm_err(&dev_priv->drm,
4158                                 "mode set failed: pipe %c stuck\n",
4159                                 pipe_name(pipe));
4160         }
4161 }
4162
4163 /*
4164  * The hardware phase 0.0 refers to the center of the pixel.
4165  * We want to start from the top/left edge which is phase
4166  * -0.5. That matches how the hardware calculates the scaling
4167  * factors (from top-left of the first pixel to bottom-right
4168  * of the last pixel, as opposed to the pixel centers).
4169  *
4170  * For 4:2:0 subsampled chroma planes we obviously have to
4171  * adjust that so that the chroma sample position lands in
4172  * the right spot.
4173  *
4174  * Note that for packed YCbCr 4:2:2 formats there is no way to
4175  * control chroma siting. The hardware simply replicates the
4176  * chroma samples for both of the luma samples, and thus we don't
4177  * actually get the expected MPEG2 chroma siting convention :(
4178  * The same behaviour is observed on pre-SKL platforms as well.
4179  *
4180  * Theory behind the formula (note that we ignore sub-pixel
4181  * source coordinates):
4182  * s = source sample position
4183  * d = destination sample position
4184  *
4185  * Downscaling 4:1:
4186  * -0.5
4187  * | 0.0
4188  * | |     1.5 (initial phase)
4189  * | |     |
4190  * v v     v
4191  * | s | s | s | s |
4192  * |       d       |
4193  *
4194  * Upscaling 1:4:
4195  * -0.5
4196  * | -0.375 (initial phase)
4197  * | |     0.0
4198  * | |     |
4199  * v v     v
4200  * |       s       |
4201  * | d | d | d | d |
4202  */
4203 u16 skl_scaler_calc_phase(int sub, int scale, bool chroma_cosited)
4204 {
4205         int phase = -0x8000;
4206         u16 trip = 0;
4207
4208         if (chroma_cosited)
4209                 phase += (sub - 1) * 0x8000 / sub;
4210
4211         phase += scale / (2 * sub);
4212
4213         /*
4214          * Hardware initial phase limited to [-0.5:1.5].
4215          * Since the max hardware scale factor is 3.0, we
4216          * should never actually excdeed 1.0 here.
4217          */
4218         WARN_ON(phase < -0x8000 || phase > 0x18000);
4219
4220         if (phase < 0)
4221                 phase = 0x10000 + phase;
4222         else
4223                 trip = PS_PHASE_TRIP;
4224
4225         return ((phase >> 2) & PS_PHASE_MASK) | trip;
4226 }
4227
4228 #define SKL_MIN_SRC_W 8
4229 #define SKL_MAX_SRC_W 4096
4230 #define SKL_MIN_SRC_H 8
4231 #define SKL_MAX_SRC_H 4096
4232 #define SKL_MIN_DST_W 8
4233 #define SKL_MAX_DST_W 4096
4234 #define SKL_MIN_DST_H 8
4235 #define SKL_MAX_DST_H 4096
4236 #define ICL_MAX_SRC_W 5120
4237 #define ICL_MAX_SRC_H 4096
4238 #define ICL_MAX_DST_W 5120
4239 #define ICL_MAX_DST_H 4096
4240 #define SKL_MIN_YUV_420_SRC_W 16
4241 #define SKL_MIN_YUV_420_SRC_H 16
4242
4243 static int
4244 skl_update_scaler(struct intel_crtc_state *crtc_state, bool force_detach,
4245                   unsigned int scaler_user, int *scaler_id,
4246                   int src_w, int src_h, int dst_w, int dst_h,
4247                   const struct drm_format_info *format,
4248                   u64 modifier, bool need_scaler)
4249 {
4250         struct intel_crtc_scaler_state *scaler_state =
4251                 &crtc_state->scaler_state;
4252         struct intel_crtc *intel_crtc =
4253                 to_intel_crtc(crtc_state->uapi.crtc);
4254         struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
4255         const struct drm_display_mode *adjusted_mode =
4256                 &crtc_state->hw.adjusted_mode;
4257
4258         /*
4259          * Src coordinates are already rotated by 270 degrees for
4260          * the 90/270 degree plane rotation cases (to match the
4261          * GTT mapping), hence no need to account for rotation here.
4262          */
4263         if (src_w != dst_w || src_h != dst_h)
4264                 need_scaler = true;
4265
4266         /*
4267          * Scaling/fitting not supported in IF-ID mode in GEN9+
4268          * TODO: Interlace fetch mode doesn't support YUV420 planar formats.
4269          * Once NV12 is enabled, handle it here while allocating scaler
4270          * for NV12.
4271          */
4272         if (INTEL_GEN(dev_priv) >= 9 && crtc_state->hw.enable &&
4273             need_scaler && adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) {
4274                 drm_dbg_kms(&dev_priv->drm,
4275                             "Pipe/Plane scaling not supported with IF-ID mode\n");
4276                 return -EINVAL;
4277         }
4278
4279         /*
4280          * if plane is being disabled or scaler is no more required or force detach
4281          *  - free scaler binded to this plane/crtc
4282          *  - in order to do this, update crtc->scaler_usage
4283          *
4284          * Here scaler state in crtc_state is set free so that
4285          * scaler can be assigned to other user. Actual register
4286          * update to free the scaler is done in plane/panel-fit programming.
4287          * For this purpose crtc/plane_state->scaler_id isn't reset here.
4288          */
4289         if (force_detach || !need_scaler) {
4290                 if (*scaler_id >= 0) {
4291                         scaler_state->scaler_users &= ~(1 << scaler_user);
4292                         scaler_state->scalers[*scaler_id].in_use = 0;
4293
4294                         drm_dbg_kms(&dev_priv->drm,
4295                                     "scaler_user index %u.%u: "
4296                                     "Staged freeing scaler id %d scaler_users = 0x%x\n",
4297                                     intel_crtc->pipe, scaler_user, *scaler_id,
4298                                     scaler_state->scaler_users);
4299                         *scaler_id = -1;
4300                 }
4301                 return 0;
4302         }
4303
4304         if (format && intel_format_info_is_yuv_semiplanar(format, modifier) &&
4305             (src_h < SKL_MIN_YUV_420_SRC_H || src_w < SKL_MIN_YUV_420_SRC_W)) {
4306                 drm_dbg_kms(&dev_priv->drm,
4307                             "Planar YUV: src dimensions not met\n");
4308                 return -EINVAL;
4309         }
4310
4311         /* range checks */
4312         if (src_w < SKL_MIN_SRC_W || src_h < SKL_MIN_SRC_H ||
4313             dst_w < SKL_MIN_DST_W || dst_h < SKL_MIN_DST_H ||
4314             (INTEL_GEN(dev_priv) >= 11 &&
4315              (src_w > ICL_MAX_SRC_W || src_h > ICL_MAX_SRC_H ||
4316               dst_w > ICL_MAX_DST_W || dst_h > ICL_MAX_DST_H)) ||
4317             (INTEL_GEN(dev_priv) < 11 &&
4318              (src_w > SKL_MAX_SRC_W || src_h > SKL_MAX_SRC_H ||
4319               dst_w > SKL_MAX_DST_W || dst_h > SKL_MAX_DST_H))) {
4320                 drm_dbg_kms(&dev_priv->drm,
4321                             "scaler_user index %u.%u: src %ux%u dst %ux%u "
4322                             "size is out of scaler range\n",
4323                             intel_crtc->pipe, scaler_user, src_w, src_h,
4324                             dst_w, dst_h);
4325                 return -EINVAL;
4326         }
4327
4328         /* mark this plane as a scaler user in crtc_state */
4329         scaler_state->scaler_users |= (1 << scaler_user);
4330         drm_dbg_kms(&dev_priv->drm, "scaler_user index %u.%u: "
4331                     "staged scaling request for %ux%u->%ux%u scaler_users = 0x%x\n",
4332                     intel_crtc->pipe, scaler_user, src_w, src_h, dst_w, dst_h,
4333                     scaler_state->scaler_users);
4334
4335         return 0;
4336 }
4337
4338 static int skl_update_scaler_crtc(struct intel_crtc_state *crtc_state)
4339 {
4340         const struct drm_display_mode *pipe_mode = &crtc_state->hw.pipe_mode;
4341         int width, height;
4342
4343         if (crtc_state->pch_pfit.enabled) {
4344                 width = drm_rect_width(&crtc_state->pch_pfit.dst);
4345                 height = drm_rect_height(&crtc_state->pch_pfit.dst);
4346         } else {
4347                 width = pipe_mode->crtc_hdisplay;
4348                 height = pipe_mode->crtc_vdisplay;
4349         }
4350         return skl_update_scaler(crtc_state, !crtc_state->hw.active,
4351                                  SKL_CRTC_INDEX,
4352                                  &crtc_state->scaler_state.scaler_id,
4353                                  crtc_state->pipe_src_w, crtc_state->pipe_src_h,
4354                                  width, height, NULL, 0,
4355                                  crtc_state->pch_pfit.enabled);
4356 }
4357
4358 /**
4359  * skl_update_scaler_plane - Stages update to scaler state for a given plane.
4360  * @crtc_state: crtc's scaler state
4361  * @plane_state: atomic plane state to update
4362  *
4363  * Return
4364  *     0 - scaler_usage updated successfully
4365  *    error - requested scaling cannot be supported or other error condition
4366  */
4367 static int skl_update_scaler_plane(struct intel_crtc_state *crtc_state,
4368                                    struct intel_plane_state *plane_state)
4369 {
4370         struct intel_plane *intel_plane =
4371                 to_intel_plane(plane_state->uapi.plane);
4372         struct drm_i915_private *dev_priv = to_i915(intel_plane->base.dev);
4373         struct drm_framebuffer *fb = plane_state->hw.fb;
4374         int ret;
4375         bool force_detach = !fb || !plane_state->uapi.visible;
4376         bool need_scaler = false;
4377
4378         /* Pre-gen11 and SDR planes always need a scaler for planar formats. */
4379         if (!icl_is_hdr_plane(dev_priv, intel_plane->id) &&
4380             fb && intel_format_info_is_yuv_semiplanar(fb->format, fb->modifier))
4381                 need_scaler = true;
4382
4383         ret = skl_update_scaler(crtc_state, force_detach,
4384                                 drm_plane_index(&intel_plane->base),
4385                                 &plane_state->scaler_id,
4386                                 drm_rect_width(&plane_state->uapi.src) >> 16,
4387                                 drm_rect_height(&plane_state->uapi.src) >> 16,
4388                                 drm_rect_width(&plane_state->uapi.dst),
4389                                 drm_rect_height(&plane_state->uapi.dst),
4390                                 fb ? fb->format : NULL,
4391                                 fb ? fb->modifier : 0,
4392                                 need_scaler);
4393
4394         if (ret || plane_state->scaler_id < 0)
4395                 return ret;
4396
4397         /* check colorkey */
4398         if (plane_state->ckey.flags) {
4399                 drm_dbg_kms(&dev_priv->drm,
4400                             "[PLANE:%d:%s] scaling with color key not allowed",
4401                             intel_plane->base.base.id,
4402                             intel_plane->base.name);
4403                 return -EINVAL;
4404         }
4405
4406         /* Check src format */
4407         switch (fb->format->format) {
4408         case DRM_FORMAT_RGB565:
4409         case DRM_FORMAT_XBGR8888:
4410         case DRM_FORMAT_XRGB8888:
4411         case DRM_FORMAT_ABGR8888:
4412         case DRM_FORMAT_ARGB8888:
4413         case DRM_FORMAT_XRGB2101010:
4414         case DRM_FORMAT_XBGR2101010:
4415         case DRM_FORMAT_ARGB2101010:
4416         case DRM_FORMAT_ABGR2101010:
4417         case DRM_FORMAT_YUYV:
4418         case DRM_FORMAT_YVYU:
4419         case DRM_FORMAT_UYVY:
4420         case DRM_FORMAT_VYUY:
4421         case DRM_FORMAT_NV12:
4422         case DRM_FORMAT_XYUV8888:
4423         case DRM_FORMAT_P010:
4424         case DRM_FORMAT_P012:
4425         case DRM_FORMAT_P016:
4426         case DRM_FORMAT_Y210:
4427         case DRM_FORMAT_Y212:
4428         case DRM_FORMAT_Y216:
4429         case DRM_FORMAT_XVYU2101010:
4430         case DRM_FORMAT_XVYU12_16161616:
4431         case DRM_FORMAT_XVYU16161616:
4432                 break;
4433         case DRM_FORMAT_XBGR16161616F:
4434         case DRM_FORMAT_ABGR16161616F:
4435         case DRM_FORMAT_XRGB16161616F:
4436         case DRM_FORMAT_ARGB16161616F:
4437                 if (INTEL_GEN(dev_priv) >= 11)
4438                         break;
4439                 fallthrough;
4440         default:
4441                 drm_dbg_kms(&dev_priv->drm,
4442                             "[PLANE:%d:%s] FB:%d unsupported scaling format 0x%x\n",
4443                             intel_plane->base.base.id, intel_plane->base.name,
4444                             fb->base.id, fb->format->format);
4445                 return -EINVAL;
4446         }
4447
4448         return 0;
4449 }
4450
4451 void skl_scaler_disable(const struct intel_crtc_state *old_crtc_state)
4452 {
4453         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->uapi.crtc);
4454         int i;
4455
4456         for (i = 0; i < crtc->num_scalers; i++)
4457                 skl_detach_scaler(crtc, i);
4458 }
4459
4460 static int cnl_coef_tap(int i)
4461 {
4462         return i % 7;
4463 }
4464
4465 static u16 cnl_nearest_filter_coef(int t)
4466 {
4467         return t == 3 ? 0x0800 : 0x3000;
4468 }
4469
4470 /*
4471  *  Theory behind setting nearest-neighbor integer scaling:
4472  *
4473  *  17 phase of 7 taps requires 119 coefficients in 60 dwords per set.
4474  *  The letter represents the filter tap (D is the center tap) and the number
4475  *  represents the coefficient set for a phase (0-16).
4476  *
4477  *         +------------+------------------------+------------------------+
4478  *         |Index value | Data value coeffient 1 | Data value coeffient 2 |
4479  *         +------------+------------------------+------------------------+
4480  *         |   00h      |          B0            |          A0            |
4481  *         +------------+------------------------+------------------------+
4482  *         |   01h      |          D0            |          C0            |
4483  *         +------------+------------------------+------------------------+
4484  *         |   02h      |          F0            |          E0            |
4485  *         +------------+------------------------+------------------------+
4486  *         |   03h      |          A1            |          G0            |
4487  *         +------------+------------------------+------------------------+
4488  *         |   04h      |          C1            |          B1            |
4489  *         +------------+------------------------+------------------------+
4490  *         |   ...      |          ...           |          ...           |
4491  *         +------------+------------------------+------------------------+
4492  *         |   38h      |          B16           |          A16           |
4493  *         +------------+------------------------+------------------------+
4494  *         |   39h      |          D16           |          C16           |
4495  *         +------------+------------------------+------------------------+
4496  *         |   3Ah      |          F16           |          C16           |
4497  *         +------------+------------------------+------------------------+
4498  *         |   3Bh      |        Reserved        |          G16           |
4499  *         +------------+------------------------+------------------------+
4500  *
4501  *  To enable nearest-neighbor scaling:  program scaler coefficents with
4502  *  the center tap (Dxx) values set to 1 and all other values set to 0 as per
4503  *  SCALER_COEFFICIENT_FORMAT
4504  *
4505  */
4506
4507 static void cnl_program_nearest_filter_coefs(struct drm_i915_private *dev_priv,
4508                                              enum pipe pipe, int id, int set)
4509 {
4510         int i;
4511
4512         intel_de_write_fw(dev_priv, CNL_PS_COEF_INDEX_SET(pipe, id, set),
4513                           PS_COEE_INDEX_AUTO_INC);
4514
4515         for (i = 0; i < 17 * 7; i += 2) {
4516                 u32 tmp;
4517                 int t;
4518
4519                 t = cnl_coef_tap(i);
4520                 tmp = cnl_nearest_filter_coef(t);
4521
4522                 t = cnl_coef_tap(i + 1);
4523                 tmp |= cnl_nearest_filter_coef(t) << 16;
4524
4525                 intel_de_write_fw(dev_priv, CNL_PS_COEF_DATA_SET(pipe, id, set),
4526                                   tmp);
4527         }
4528
4529         intel_de_write_fw(dev_priv, CNL_PS_COEF_INDEX_SET(pipe, id, set), 0);
4530 }
4531
4532 u32 skl_scaler_get_filter_select(enum drm_scaling_filter filter, int set)
4533 {
4534         if (filter == DRM_SCALING_FILTER_NEAREST_NEIGHBOR) {
4535                 return (PS_FILTER_PROGRAMMED |
4536                         PS_Y_VERT_FILTER_SELECT(set) |
4537                         PS_Y_HORZ_FILTER_SELECT(set) |
4538                         PS_UV_VERT_FILTER_SELECT(set) |
4539                         PS_UV_HORZ_FILTER_SELECT(set));
4540         }
4541
4542         return PS_FILTER_MEDIUM;
4543 }
4544
4545 void skl_scaler_setup_filter(struct drm_i915_private *dev_priv, enum pipe pipe,
4546                              int id, int set, enum drm_scaling_filter filter)
4547 {
4548         switch (filter) {
4549         case DRM_SCALING_FILTER_DEFAULT:
4550                 break;
4551         case DRM_SCALING_FILTER_NEAREST_NEIGHBOR:
4552                 cnl_program_nearest_filter_coefs(dev_priv, pipe, id, set);
4553                 break;
4554         default:
4555                 MISSING_CASE(filter);
4556         }
4557 }
4558
4559 static void skl_pfit_enable(const struct intel_crtc_state *crtc_state)
4560 {
4561         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
4562         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4563         const struct intel_crtc_scaler_state *scaler_state =
4564                 &crtc_state->scaler_state;
4565         struct drm_rect src = {
4566                 .x2 = crtc_state->pipe_src_w << 16,
4567                 .y2 = crtc_state->pipe_src_h << 16,
4568         };
4569         const struct drm_rect *dst = &crtc_state->pch_pfit.dst;
4570         u16 uv_rgb_hphase, uv_rgb_vphase;
4571         enum pipe pipe = crtc->pipe;
4572         int width = drm_rect_width(dst);
4573         int height = drm_rect_height(dst);
4574         int x = dst->x1;
4575         int y = dst->y1;
4576         int hscale, vscale;
4577         unsigned long irqflags;
4578         int id;
4579         u32 ps_ctrl;
4580
4581         if (!crtc_state->pch_pfit.enabled)
4582                 return;
4583
4584         if (drm_WARN_ON(&dev_priv->drm,
4585                         crtc_state->scaler_state.scaler_id < 0))
4586                 return;
4587
4588         hscale = drm_rect_calc_hscale(&src, dst, 0, INT_MAX);
4589         vscale = drm_rect_calc_vscale(&src, dst, 0, INT_MAX);
4590
4591         uv_rgb_hphase = skl_scaler_calc_phase(1, hscale, false);
4592         uv_rgb_vphase = skl_scaler_calc_phase(1, vscale, false);
4593
4594         id = scaler_state->scaler_id;
4595
4596         ps_ctrl = skl_scaler_get_filter_select(crtc_state->hw.scaling_filter, 0);
4597         ps_ctrl |=  PS_SCALER_EN | scaler_state->scalers[id].mode;
4598
4599         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
4600
4601         skl_scaler_setup_filter(dev_priv, pipe, id, 0,
4602                                 crtc_state->hw.scaling_filter);
4603
4604         intel_de_write_fw(dev_priv, SKL_PS_CTRL(pipe, id), ps_ctrl);
4605
4606         intel_de_write_fw(dev_priv, SKL_PS_VPHASE(pipe, id),
4607                           PS_Y_PHASE(0) | PS_UV_RGB_PHASE(uv_rgb_vphase));
4608         intel_de_write_fw(dev_priv, SKL_PS_HPHASE(pipe, id),
4609                           PS_Y_PHASE(0) | PS_UV_RGB_PHASE(uv_rgb_hphase));
4610         intel_de_write_fw(dev_priv, SKL_PS_WIN_POS(pipe, id),
4611                           x << 16 | y);
4612         intel_de_write_fw(dev_priv, SKL_PS_WIN_SZ(pipe, id),
4613                           width << 16 | height);
4614
4615         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
4616 }
4617
4618 static void ilk_pfit_enable(const struct intel_crtc_state *crtc_state)
4619 {
4620         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
4621         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4622         const struct drm_rect *dst = &crtc_state->pch_pfit.dst;
4623         enum pipe pipe = crtc->pipe;
4624         int width = drm_rect_width(dst);
4625         int height = drm_rect_height(dst);
4626         int x = dst->x1;
4627         int y = dst->y1;
4628
4629         if (!crtc_state->pch_pfit.enabled)
4630                 return;
4631
4632         /* Force use of hard-coded filter coefficients
4633          * as some pre-programmed values are broken,
4634          * e.g. x201.
4635          */
4636         if (IS_IVYBRIDGE(dev_priv) || IS_HASWELL(dev_priv))
4637                 intel_de_write(dev_priv, PF_CTL(pipe), PF_ENABLE |
4638                                PF_FILTER_MED_3x3 | PF_PIPE_SEL_IVB(pipe));
4639         else
4640                 intel_de_write(dev_priv, PF_CTL(pipe), PF_ENABLE |
4641                                PF_FILTER_MED_3x3);
4642         intel_de_write(dev_priv, PF_WIN_POS(pipe), x << 16 | y);
4643         intel_de_write(dev_priv, PF_WIN_SZ(pipe), width << 16 | height);
4644 }
4645
4646 void hsw_enable_ips(const struct intel_crtc_state *crtc_state)
4647 {
4648         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
4649         struct drm_device *dev = crtc->base.dev;
4650         struct drm_i915_private *dev_priv = to_i915(dev);
4651
4652         if (!crtc_state->ips_enabled)
4653                 return;
4654
4655         /*
4656          * We can only enable IPS after we enable a plane and wait for a vblank
4657          * This function is called from post_plane_update, which is run after
4658          * a vblank wait.
4659          */
4660         drm_WARN_ON(dev, !(crtc_state->active_planes & ~BIT(PLANE_CURSOR)));
4661
4662         if (IS_BROADWELL(dev_priv)) {
4663                 drm_WARN_ON(dev, sandybridge_pcode_write(dev_priv, DISPLAY_IPS_CONTROL,
4664                                                          IPS_ENABLE | IPS_PCODE_CONTROL));
4665                 /* Quoting Art Runyan: "its not safe to expect any particular
4666                  * value in IPS_CTL bit 31 after enabling IPS through the
4667                  * mailbox." Moreover, the mailbox may return a bogus state,
4668                  * so we need to just enable it and continue on.
4669                  */
4670         } else {
4671                 intel_de_write(dev_priv, IPS_CTL, IPS_ENABLE);
4672                 /* The bit only becomes 1 in the next vblank, so this wait here
4673                  * is essentially intel_wait_for_vblank. If we don't have this
4674                  * and don't wait for vblanks until the end of crtc_enable, then
4675                  * the HW state readout code will complain that the expected
4676                  * IPS_CTL value is not the one we read. */
4677                 if (intel_de_wait_for_set(dev_priv, IPS_CTL, IPS_ENABLE, 50))
4678                         drm_err(&dev_priv->drm,
4679                                 "Timed out waiting for IPS enable\n");
4680         }
4681 }
4682
4683 void hsw_disable_ips(const struct intel_crtc_state *crtc_state)
4684 {
4685         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
4686         struct drm_device *dev = crtc->base.dev;
4687         struct drm_i915_private *dev_priv = to_i915(dev);
4688
4689         if (!crtc_state->ips_enabled)
4690                 return;
4691
4692         if (IS_BROADWELL(dev_priv)) {
4693                 drm_WARN_ON(dev,
4694                             sandybridge_pcode_write(dev_priv, DISPLAY_IPS_CONTROL, 0));
4695                 /*
4696                  * Wait for PCODE to finish disabling IPS. The BSpec specified
4697                  * 42ms timeout value leads to occasional timeouts so use 100ms
4698                  * instead.
4699                  */
4700                 if (intel_de_wait_for_clear(dev_priv, IPS_CTL, IPS_ENABLE, 100))
4701                         drm_err(&dev_priv->drm,
4702                                 "Timed out waiting for IPS disable\n");
4703         } else {
4704                 intel_de_write(dev_priv, IPS_CTL, 0);
4705                 intel_de_posting_read(dev_priv, IPS_CTL);
4706         }
4707
4708         /* We need to wait for a vblank before we can disable the plane. */
4709         intel_wait_for_vblank(dev_priv, crtc->pipe);
4710 }
4711
4712 static void intel_crtc_dpms_overlay_disable(struct intel_crtc *intel_crtc)
4713 {
4714         if (intel_crtc->overlay)
4715                 (void) intel_overlay_switch_off(intel_crtc->overlay);
4716
4717         /* Let userspace switch the overlay on again. In most cases userspace
4718          * has to recompute where to put it anyway.
4719          */
4720 }
4721
4722 static bool hsw_pre_update_disable_ips(const struct intel_crtc_state *old_crtc_state,
4723                                        const struct intel_crtc_state *new_crtc_state)
4724 {
4725         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
4726         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4727
4728         if (!old_crtc_state->ips_enabled)
4729                 return false;
4730
4731         if (intel_crtc_needs_modeset(new_crtc_state))
4732                 return true;
4733
4734         /*
4735          * Workaround : Do not read or write the pipe palette/gamma data while
4736          * GAMMA_MODE is configured for split gamma and IPS_CTL has IPS enabled.
4737          *
4738          * Disable IPS before we program the LUT.
4739          */
4740         if (IS_HASWELL(dev_priv) &&
4741             (new_crtc_state->uapi.color_mgmt_changed ||
4742              new_crtc_state->update_pipe) &&
4743             new_crtc_state->gamma_mode == GAMMA_MODE_MODE_SPLIT)
4744                 return true;
4745
4746         return !new_crtc_state->ips_enabled;
4747 }
4748
4749 static bool hsw_post_update_enable_ips(const struct intel_crtc_state *old_crtc_state,
4750                                        const struct intel_crtc_state *new_crtc_state)
4751 {
4752         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
4753         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4754
4755         if (!new_crtc_state->ips_enabled)
4756                 return false;
4757
4758         if (intel_crtc_needs_modeset(new_crtc_state))
4759                 return true;
4760
4761         /*
4762          * Workaround : Do not read or write the pipe palette/gamma data while
4763          * GAMMA_MODE is configured for split gamma and IPS_CTL has IPS enabled.
4764          *
4765          * Re-enable IPS after the LUT has been programmed.
4766          */
4767         if (IS_HASWELL(dev_priv) &&
4768             (new_crtc_state->uapi.color_mgmt_changed ||
4769              new_crtc_state->update_pipe) &&
4770             new_crtc_state->gamma_mode == GAMMA_MODE_MODE_SPLIT)
4771                 return true;
4772
4773         /*
4774          * We can't read out IPS on broadwell, assume the worst and
4775          * forcibly enable IPS on the first fastset.
4776          */
4777         if (new_crtc_state->update_pipe && old_crtc_state->inherited)
4778                 return true;
4779
4780         return !old_crtc_state->ips_enabled;
4781 }
4782
4783 static bool needs_nv12_wa(const struct intel_crtc_state *crtc_state)
4784 {
4785         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
4786
4787         if (!crtc_state->nv12_planes)
4788                 return false;
4789
4790         /* WA Display #0827: Gen9:all */
4791         if (IS_GEN(dev_priv, 9) && !IS_GEMINILAKE(dev_priv))
4792                 return true;
4793
4794         return false;
4795 }
4796
4797 static bool needs_scalerclk_wa(const struct intel_crtc_state *crtc_state)
4798 {
4799         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
4800
4801         /* Wa_2006604312:icl,ehl */
4802         if (crtc_state->scaler_state.scaler_users > 0 && IS_GEN(dev_priv, 11))
4803                 return true;
4804
4805         return false;
4806 }
4807
4808 static bool planes_enabling(const struct intel_crtc_state *old_crtc_state,
4809                             const struct intel_crtc_state *new_crtc_state)
4810 {
4811         return (!old_crtc_state->active_planes || intel_crtc_needs_modeset(new_crtc_state)) &&
4812                 new_crtc_state->active_planes;
4813 }
4814
4815 static bool planes_disabling(const struct intel_crtc_state *old_crtc_state,
4816                              const struct intel_crtc_state *new_crtc_state)
4817 {
4818         return old_crtc_state->active_planes &&
4819                 (!new_crtc_state->active_planes || intel_crtc_needs_modeset(new_crtc_state));
4820 }
4821
4822 static void intel_post_plane_update(struct intel_atomic_state *state,
4823                                     struct intel_crtc *crtc)
4824 {
4825         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
4826         const struct intel_crtc_state *old_crtc_state =
4827                 intel_atomic_get_old_crtc_state(state, crtc);
4828         const struct intel_crtc_state *new_crtc_state =
4829                 intel_atomic_get_new_crtc_state(state, crtc);
4830         enum pipe pipe = crtc->pipe;
4831
4832         intel_frontbuffer_flip(dev_priv, new_crtc_state->fb_bits);
4833
4834         if (new_crtc_state->update_wm_post && new_crtc_state->hw.active)
4835                 intel_update_watermarks(crtc);
4836
4837         if (hsw_post_update_enable_ips(old_crtc_state, new_crtc_state))
4838                 hsw_enable_ips(new_crtc_state);
4839
4840         intel_fbc_post_update(state, crtc);
4841
4842         if (needs_nv12_wa(old_crtc_state) &&
4843             !needs_nv12_wa(new_crtc_state))
4844                 skl_wa_827(dev_priv, pipe, false);
4845
4846         if (needs_scalerclk_wa(old_crtc_state) &&
4847             !needs_scalerclk_wa(new_crtc_state))
4848                 icl_wa_scalerclkgating(dev_priv, pipe, false);
4849 }
4850
4851 static void intel_crtc_enable_flip_done(struct intel_atomic_state *state,
4852                                         struct intel_crtc *crtc)
4853 {
4854         const struct intel_crtc_state *crtc_state =
4855                 intel_atomic_get_new_crtc_state(state, crtc);
4856         u8 update_planes = crtc_state->update_planes;
4857         const struct intel_plane_state *plane_state;
4858         struct intel_plane *plane;
4859         int i;
4860
4861         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
4862                 if (plane->enable_flip_done &&
4863                     plane->pipe == crtc->pipe &&
4864                     update_planes & BIT(plane->id))
4865                         plane->enable_flip_done(plane);
4866         }
4867 }
4868
4869 static void intel_crtc_disable_flip_done(struct intel_atomic_state *state,
4870                                          struct intel_crtc *crtc)
4871 {
4872         const struct intel_crtc_state *crtc_state =
4873                 intel_atomic_get_new_crtc_state(state, crtc);
4874         u8 update_planes = crtc_state->update_planes;
4875         const struct intel_plane_state *plane_state;
4876         struct intel_plane *plane;
4877         int i;
4878
4879         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
4880                 if (plane->disable_flip_done &&
4881                     plane->pipe == crtc->pipe &&
4882                     update_planes & BIT(plane->id))
4883                         plane->disable_flip_done(plane);
4884         }
4885 }
4886
4887 static void intel_crtc_async_flip_disable_wa(struct intel_atomic_state *state,
4888                                              struct intel_crtc *crtc)
4889 {
4890         struct drm_i915_private *i915 = to_i915(state->base.dev);
4891         const struct intel_crtc_state *old_crtc_state =
4892                 intel_atomic_get_old_crtc_state(state, crtc);
4893         const struct intel_crtc_state *new_crtc_state =
4894                 intel_atomic_get_new_crtc_state(state, crtc);
4895         u8 update_planes = new_crtc_state->update_planes;
4896         const struct intel_plane_state *old_plane_state;
4897         struct intel_plane *plane;
4898         bool need_vbl_wait = false;
4899         int i;
4900
4901         for_each_old_intel_plane_in_state(state, plane, old_plane_state, i) {
4902                 if (plane->need_async_flip_disable_wa &&
4903                     plane->pipe == crtc->pipe &&
4904                     update_planes & BIT(plane->id)) {
4905                         /*
4906                          * Apart from the async flip bit we want to
4907                          * preserve the old state for the plane.
4908                          */
4909                         plane->async_flip(plane, old_crtc_state,
4910                                           old_plane_state, false);
4911                         need_vbl_wait = true;
4912                 }
4913         }
4914
4915         if (need_vbl_wait)
4916                 intel_wait_for_vblank(i915, crtc->pipe);
4917 }
4918
4919 static void intel_pre_plane_update(struct intel_atomic_state *state,
4920                                    struct intel_crtc *crtc)
4921 {
4922         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
4923         const struct intel_crtc_state *old_crtc_state =
4924                 intel_atomic_get_old_crtc_state(state, crtc);
4925         const struct intel_crtc_state *new_crtc_state =
4926                 intel_atomic_get_new_crtc_state(state, crtc);
4927         enum pipe pipe = crtc->pipe;
4928
4929         if (hsw_pre_update_disable_ips(old_crtc_state, new_crtc_state))
4930                 hsw_disable_ips(old_crtc_state);
4931
4932         if (intel_fbc_pre_update(state, crtc))
4933                 intel_wait_for_vblank(dev_priv, pipe);
4934
4935         /* Display WA 827 */
4936         if (!needs_nv12_wa(old_crtc_state) &&
4937             needs_nv12_wa(new_crtc_state))
4938                 skl_wa_827(dev_priv, pipe, true);
4939
4940         /* Wa_2006604312:icl,ehl */
4941         if (!needs_scalerclk_wa(old_crtc_state) &&
4942             needs_scalerclk_wa(new_crtc_state))
4943                 icl_wa_scalerclkgating(dev_priv, pipe, true);
4944
4945         /*
4946          * Vblank time updates from the shadow to live plane control register
4947          * are blocked if the memory self-refresh mode is active at that
4948          * moment. So to make sure the plane gets truly disabled, disable
4949          * first the self-refresh mode. The self-refresh enable bit in turn
4950          * will be checked/applied by the HW only at the next frame start
4951          * event which is after the vblank start event, so we need to have a
4952          * wait-for-vblank between disabling the plane and the pipe.
4953          */
4954         if (HAS_GMCH(dev_priv) && old_crtc_state->hw.active &&
4955             new_crtc_state->disable_cxsr && intel_set_memory_cxsr(dev_priv, false))
4956                 intel_wait_for_vblank(dev_priv, pipe);
4957
4958         /*
4959          * IVB workaround: must disable low power watermarks for at least
4960          * one frame before enabling scaling.  LP watermarks can be re-enabled
4961          * when scaling is disabled.
4962          *
4963          * WaCxSRDisabledForSpriteScaling:ivb
4964          */
4965         if (old_crtc_state->hw.active &&
4966             new_crtc_state->disable_lp_wm && ilk_disable_lp_wm(dev_priv))
4967                 intel_wait_for_vblank(dev_priv, pipe);
4968
4969         /*
4970          * If we're doing a modeset we don't need to do any
4971          * pre-vblank watermark programming here.
4972          */
4973         if (!intel_crtc_needs_modeset(new_crtc_state)) {
4974                 /*
4975                  * For platforms that support atomic watermarks, program the
4976                  * 'intermediate' watermarks immediately.  On pre-gen9 platforms, these
4977                  * will be the intermediate values that are safe for both pre- and
4978                  * post- vblank; when vblank happens, the 'active' values will be set
4979                  * to the final 'target' values and we'll do this again to get the
4980                  * optimal watermarks.  For gen9+ platforms, the values we program here
4981                  * will be the final target values which will get automatically latched
4982                  * at vblank time; no further programming will be necessary.
4983                  *
4984                  * If a platform hasn't been transitioned to atomic watermarks yet,
4985                  * we'll continue to update watermarks the old way, if flags tell
4986                  * us to.
4987                  */
4988                 if (dev_priv->display.initial_watermarks)
4989                         dev_priv->display.initial_watermarks(state, crtc);
4990                 else if (new_crtc_state->update_wm_pre)
4991                         intel_update_watermarks(crtc);
4992         }
4993
4994         /*
4995          * Gen2 reports pipe underruns whenever all planes are disabled.
4996          * So disable underrun reporting before all the planes get disabled.
4997          *
4998          * We do this after .initial_watermarks() so that we have a
4999          * chance of catching underruns with the intermediate watermarks
5000          * vs. the old plane configuration.
5001          */
5002         if (IS_GEN(dev_priv, 2) && planes_disabling(old_crtc_state, new_crtc_state))
5003                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
5004
5005         /*
5006          * WA for platforms where async address update enable bit
5007          * is double buffered and only latched at start of vblank.
5008          */
5009         if (old_crtc_state->uapi.async_flip && !new_crtc_state->uapi.async_flip)
5010                 intel_crtc_async_flip_disable_wa(state, crtc);
5011 }
5012
5013 static void intel_crtc_disable_planes(struct intel_atomic_state *state,
5014                                       struct intel_crtc *crtc)
5015 {
5016         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5017         const struct intel_crtc_state *new_crtc_state =
5018                 intel_atomic_get_new_crtc_state(state, crtc);
5019         unsigned int update_mask = new_crtc_state->update_planes;
5020         const struct intel_plane_state *old_plane_state;
5021         struct intel_plane *plane;
5022         unsigned fb_bits = 0;
5023         int i;
5024
5025         intel_crtc_dpms_overlay_disable(crtc);
5026
5027         for_each_old_intel_plane_in_state(state, plane, old_plane_state, i) {
5028                 if (crtc->pipe != plane->pipe ||
5029                     !(update_mask & BIT(plane->id)))
5030                         continue;
5031
5032                 intel_disable_plane(plane, new_crtc_state);
5033
5034                 if (old_plane_state->uapi.visible)
5035                         fb_bits |= plane->frontbuffer_bit;
5036         }
5037
5038         intel_frontbuffer_flip(dev_priv, fb_bits);
5039 }
5040
5041 /*
5042  * intel_connector_primary_encoder - get the primary encoder for a connector
5043  * @connector: connector for which to return the encoder
5044  *
5045  * Returns the primary encoder for a connector. There is a 1:1 mapping from
5046  * all connectors to their encoder, except for DP-MST connectors which have
5047  * both a virtual and a primary encoder. These DP-MST primary encoders can be
5048  * pointed to by as many DP-MST connectors as there are pipes.
5049  */
5050 static struct intel_encoder *
5051 intel_connector_primary_encoder(struct intel_connector *connector)
5052 {
5053         struct intel_encoder *encoder;
5054
5055         if (connector->mst_port)
5056                 return &dp_to_dig_port(connector->mst_port)->base;
5057
5058         encoder = intel_attached_encoder(connector);
5059         drm_WARN_ON(connector->base.dev, !encoder);
5060
5061         return encoder;
5062 }
5063
5064 static void intel_encoders_update_prepare(struct intel_atomic_state *state)
5065 {
5066         struct drm_connector_state *new_conn_state;
5067         struct drm_connector *connector;
5068         int i;
5069
5070         for_each_new_connector_in_state(&state->base, connector, new_conn_state,
5071                                         i) {
5072                 struct intel_connector *intel_connector;
5073                 struct intel_encoder *encoder;
5074                 struct intel_crtc *crtc;
5075
5076                 if (!intel_connector_needs_modeset(state, connector))
5077                         continue;
5078
5079                 intel_connector = to_intel_connector(connector);
5080                 encoder = intel_connector_primary_encoder(intel_connector);
5081                 if (!encoder->update_prepare)
5082                         continue;
5083
5084                 crtc = new_conn_state->crtc ?
5085                         to_intel_crtc(new_conn_state->crtc) : NULL;
5086                 encoder->update_prepare(state, encoder, crtc);
5087         }
5088 }
5089
5090 static void intel_encoders_update_complete(struct intel_atomic_state *state)
5091 {
5092         struct drm_connector_state *new_conn_state;
5093         struct drm_connector *connector;
5094         int i;
5095
5096         for_each_new_connector_in_state(&state->base, connector, new_conn_state,
5097                                         i) {
5098                 struct intel_connector *intel_connector;
5099                 struct intel_encoder *encoder;
5100                 struct intel_crtc *crtc;
5101
5102                 if (!intel_connector_needs_modeset(state, connector))
5103                         continue;
5104
5105                 intel_connector = to_intel_connector(connector);
5106                 encoder = intel_connector_primary_encoder(intel_connector);
5107                 if (!encoder->update_complete)
5108                         continue;
5109
5110                 crtc = new_conn_state->crtc ?
5111                         to_intel_crtc(new_conn_state->crtc) : NULL;
5112                 encoder->update_complete(state, encoder, crtc);
5113         }
5114 }
5115
5116 static void intel_encoders_pre_pll_enable(struct intel_atomic_state *state,
5117                                           struct intel_crtc *crtc)
5118 {
5119         const struct intel_crtc_state *crtc_state =
5120                 intel_atomic_get_new_crtc_state(state, crtc);
5121         const struct drm_connector_state *conn_state;
5122         struct drm_connector *conn;
5123         int i;
5124
5125         for_each_new_connector_in_state(&state->base, conn, conn_state, i) {
5126                 struct intel_encoder *encoder =
5127                         to_intel_encoder(conn_state->best_encoder);
5128
5129                 if (conn_state->crtc != &crtc->base)
5130                         continue;
5131
5132                 if (encoder->pre_pll_enable)
5133                         encoder->pre_pll_enable(state, encoder,
5134                                                 crtc_state, conn_state);
5135         }
5136 }
5137
5138 static void intel_encoders_pre_enable(struct intel_atomic_state *state,
5139                                       struct intel_crtc *crtc)
5140 {
5141         const struct intel_crtc_state *crtc_state =
5142                 intel_atomic_get_new_crtc_state(state, crtc);
5143         const struct drm_connector_state *conn_state;
5144         struct drm_connector *conn;
5145         int i;
5146
5147         for_each_new_connector_in_state(&state->base, conn, conn_state, i) {
5148                 struct intel_encoder *encoder =
5149                         to_intel_encoder(conn_state->best_encoder);
5150
5151                 if (conn_state->crtc != &crtc->base)
5152                         continue;
5153
5154                 if (encoder->pre_enable)
5155                         encoder->pre_enable(state, encoder,
5156                                             crtc_state, conn_state);
5157         }
5158 }
5159
5160 static void intel_encoders_enable(struct intel_atomic_state *state,
5161                                   struct intel_crtc *crtc)
5162 {
5163         const struct intel_crtc_state *crtc_state =
5164                 intel_atomic_get_new_crtc_state(state, crtc);
5165         const struct drm_connector_state *conn_state;
5166         struct drm_connector *conn;
5167         int i;
5168
5169         for_each_new_connector_in_state(&state->base, conn, conn_state, i) {
5170                 struct intel_encoder *encoder =
5171                         to_intel_encoder(conn_state->best_encoder);
5172
5173                 if (conn_state->crtc != &crtc->base)
5174                         continue;
5175
5176                 if (encoder->enable)
5177                         encoder->enable(state, encoder,
5178                                         crtc_state, conn_state);
5179                 intel_opregion_notify_encoder(encoder, true);
5180         }
5181 }
5182
5183 static void intel_encoders_disable(struct intel_atomic_state *state,
5184                                    struct intel_crtc *crtc)
5185 {
5186         const struct intel_crtc_state *old_crtc_state =
5187                 intel_atomic_get_old_crtc_state(state, crtc);
5188         const struct drm_connector_state *old_conn_state;
5189         struct drm_connector *conn;
5190         int i;
5191
5192         for_each_old_connector_in_state(&state->base, conn, old_conn_state, i) {
5193                 struct intel_encoder *encoder =
5194                         to_intel_encoder(old_conn_state->best_encoder);
5195
5196                 if (old_conn_state->crtc != &crtc->base)
5197                         continue;
5198
5199                 intel_opregion_notify_encoder(encoder, false);
5200                 if (encoder->disable)
5201                         encoder->disable(state, encoder,
5202                                          old_crtc_state, old_conn_state);
5203         }
5204 }
5205
5206 static void intel_encoders_post_disable(struct intel_atomic_state *state,
5207                                         struct intel_crtc *crtc)
5208 {
5209         const struct intel_crtc_state *old_crtc_state =
5210                 intel_atomic_get_old_crtc_state(state, crtc);
5211         const struct drm_connector_state *old_conn_state;
5212         struct drm_connector *conn;
5213         int i;
5214
5215         for_each_old_connector_in_state(&state->base, conn, old_conn_state, i) {
5216                 struct intel_encoder *encoder =
5217                         to_intel_encoder(old_conn_state->best_encoder);
5218
5219                 if (old_conn_state->crtc != &crtc->base)
5220                         continue;
5221
5222                 if (encoder->post_disable)
5223                         encoder->post_disable(state, encoder,
5224                                               old_crtc_state, old_conn_state);
5225         }
5226 }
5227
5228 static void intel_encoders_post_pll_disable(struct intel_atomic_state *state,
5229                                             struct intel_crtc *crtc)
5230 {
5231         const struct intel_crtc_state *old_crtc_state =
5232                 intel_atomic_get_old_crtc_state(state, crtc);
5233         const struct drm_connector_state *old_conn_state;
5234         struct drm_connector *conn;
5235         int i;
5236
5237         for_each_old_connector_in_state(&state->base, conn, old_conn_state, i) {
5238                 struct intel_encoder *encoder =
5239                         to_intel_encoder(old_conn_state->best_encoder);
5240
5241                 if (old_conn_state->crtc != &crtc->base)
5242                         continue;
5243
5244                 if (encoder->post_pll_disable)
5245                         encoder->post_pll_disable(state, encoder,
5246                                                   old_crtc_state, old_conn_state);
5247         }
5248 }
5249
5250 static void intel_encoders_update_pipe(struct intel_atomic_state *state,
5251                                        struct intel_crtc *crtc)
5252 {
5253         const struct intel_crtc_state *crtc_state =
5254                 intel_atomic_get_new_crtc_state(state, crtc);
5255         const struct drm_connector_state *conn_state;
5256         struct drm_connector *conn;
5257         int i;
5258
5259         for_each_new_connector_in_state(&state->base, conn, conn_state, i) {
5260                 struct intel_encoder *encoder =
5261                         to_intel_encoder(conn_state->best_encoder);
5262
5263                 if (conn_state->crtc != &crtc->base)
5264                         continue;
5265
5266                 if (encoder->update_pipe)
5267                         encoder->update_pipe(state, encoder,
5268                                              crtc_state, conn_state);
5269         }
5270 }
5271
5272 static void intel_disable_primary_plane(const struct intel_crtc_state *crtc_state)
5273 {
5274         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5275         struct intel_plane *plane = to_intel_plane(crtc->base.primary);
5276
5277         plane->disable_plane(plane, crtc_state);
5278 }
5279
5280 static void ilk_crtc_enable(struct intel_atomic_state *state,
5281                             struct intel_crtc *crtc)
5282 {
5283         const struct intel_crtc_state *new_crtc_state =
5284                 intel_atomic_get_new_crtc_state(state, crtc);
5285         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5286         enum pipe pipe = crtc->pipe;
5287
5288         if (drm_WARN_ON(&dev_priv->drm, crtc->active))
5289                 return;
5290
5291         /*
5292          * Sometimes spurious CPU pipe underruns happen during FDI
5293          * training, at least with VGA+HDMI cloning. Suppress them.
5294          *
5295          * On ILK we get an occasional spurious CPU pipe underruns
5296          * between eDP port A enable and vdd enable. Also PCH port
5297          * enable seems to result in the occasional CPU pipe underrun.
5298          *
5299          * Spurious PCH underruns also occur during PCH enabling.
5300          */
5301         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
5302         intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, false);
5303
5304         if (new_crtc_state->has_pch_encoder)
5305                 intel_prepare_shared_dpll(new_crtc_state);
5306
5307         if (intel_crtc_has_dp_encoder(new_crtc_state))
5308                 intel_dp_set_m_n(new_crtc_state, M1_N1);
5309
5310         intel_set_transcoder_timings(new_crtc_state);
5311         intel_set_pipe_src_size(new_crtc_state);
5312
5313         if (new_crtc_state->has_pch_encoder)
5314                 intel_cpu_transcoder_set_m_n(new_crtc_state,
5315                                              &new_crtc_state->fdi_m_n, NULL);
5316
5317         ilk_set_pipeconf(new_crtc_state);
5318
5319         crtc->active = true;
5320
5321         intel_encoders_pre_enable(state, crtc);
5322
5323         if (new_crtc_state->has_pch_encoder) {
5324                 /* Note: FDI PLL enabling _must_ be done before we enable the
5325                  * cpu pipes, hence this is separate from all the other fdi/pch
5326                  * enabling. */
5327                 ilk_fdi_pll_enable(new_crtc_state);
5328         } else {
5329                 assert_fdi_tx_disabled(dev_priv, pipe);
5330                 assert_fdi_rx_disabled(dev_priv, pipe);
5331         }
5332
5333         ilk_pfit_enable(new_crtc_state);
5334
5335         /*
5336          * On ILK+ LUT must be loaded before the pipe is running but with
5337          * clocks enabled
5338          */
5339         intel_color_load_luts(new_crtc_state);
5340         intel_color_commit(new_crtc_state);
5341         /* update DSPCNTR to configure gamma for pipe bottom color */
5342         intel_disable_primary_plane(new_crtc_state);
5343
5344         if (dev_priv->display.initial_watermarks)
5345                 dev_priv->display.initial_watermarks(state, crtc);
5346         intel_enable_pipe(new_crtc_state);
5347
5348         if (new_crtc_state->has_pch_encoder)
5349                 ilk_pch_enable(state, new_crtc_state);
5350
5351         intel_crtc_vblank_on(new_crtc_state);
5352
5353         intel_encoders_enable(state, crtc);
5354
5355         if (HAS_PCH_CPT(dev_priv))
5356                 cpt_verify_modeset(dev_priv, pipe);
5357
5358         /*
5359          * Must wait for vblank to avoid spurious PCH FIFO underruns.
5360          * And a second vblank wait is needed at least on ILK with
5361          * some interlaced HDMI modes. Let's do the double wait always
5362          * in case there are more corner cases we don't know about.
5363          */
5364         if (new_crtc_state->has_pch_encoder) {
5365                 intel_wait_for_vblank(dev_priv, pipe);
5366                 intel_wait_for_vblank(dev_priv, pipe);
5367         }
5368         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
5369         intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, true);
5370 }
5371
5372 /* IPS only exists on ULT machines and is tied to pipe A. */
5373 static bool hsw_crtc_supports_ips(struct intel_crtc *crtc)
5374 {
5375         return HAS_IPS(to_i915(crtc->base.dev)) && crtc->pipe == PIPE_A;
5376 }
5377
5378 static void glk_pipe_scaler_clock_gating_wa(struct drm_i915_private *dev_priv,
5379                                             enum pipe pipe, bool apply)
5380 {
5381         u32 val = intel_de_read(dev_priv, CLKGATE_DIS_PSL(pipe));
5382         u32 mask = DPF_GATING_DIS | DPF_RAM_GATING_DIS | DPFR_GATING_DIS;
5383
5384         if (apply)
5385                 val |= mask;
5386         else
5387                 val &= ~mask;
5388
5389         intel_de_write(dev_priv, CLKGATE_DIS_PSL(pipe), val);
5390 }
5391
5392 static void icl_pipe_mbus_enable(struct intel_crtc *crtc)
5393 {
5394         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5395         enum pipe pipe = crtc->pipe;
5396         u32 val;
5397
5398         val = MBUS_DBOX_A_CREDIT(2);
5399
5400         if (INTEL_GEN(dev_priv) >= 12) {
5401                 val |= MBUS_DBOX_BW_CREDIT(2);
5402                 val |= MBUS_DBOX_B_CREDIT(12);
5403         } else {
5404                 val |= MBUS_DBOX_BW_CREDIT(1);
5405                 val |= MBUS_DBOX_B_CREDIT(8);
5406         }
5407
5408         intel_de_write(dev_priv, PIPE_MBUS_DBOX_CTL(pipe), val);
5409 }
5410
5411 static void hsw_set_linetime_wm(const struct intel_crtc_state *crtc_state)
5412 {
5413         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5414         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5415
5416         intel_de_write(dev_priv, WM_LINETIME(crtc->pipe),
5417                        HSW_LINETIME(crtc_state->linetime) |
5418                        HSW_IPS_LINETIME(crtc_state->ips_linetime));
5419 }
5420
5421 static void hsw_set_frame_start_delay(const struct intel_crtc_state *crtc_state)
5422 {
5423         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5424         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5425         i915_reg_t reg = CHICKEN_TRANS(crtc_state->cpu_transcoder);
5426         u32 val;
5427
5428         val = intel_de_read(dev_priv, reg);
5429         val &= ~HSW_FRAME_START_DELAY_MASK;
5430         val |= HSW_FRAME_START_DELAY(dev_priv->framestart_delay - 1);
5431         intel_de_write(dev_priv, reg, val);
5432 }
5433
5434 static void icl_ddi_bigjoiner_pre_enable(struct intel_atomic_state *state,
5435                                          const struct intel_crtc_state *crtc_state)
5436 {
5437         struct intel_crtc *master = to_intel_crtc(crtc_state->uapi.crtc);
5438         struct intel_crtc_state *master_crtc_state;
5439         struct drm_connector_state *conn_state;
5440         struct drm_connector *conn;
5441         struct intel_encoder *encoder = NULL;
5442         int i;
5443
5444         if (crtc_state->bigjoiner_slave)
5445                 master = crtc_state->bigjoiner_linked_crtc;
5446
5447         master_crtc_state = intel_atomic_get_new_crtc_state(state, master);
5448
5449         for_each_new_connector_in_state(&state->base, conn, conn_state, i) {
5450                 if (conn_state->crtc != &master->base)
5451                         continue;
5452
5453                 encoder = to_intel_encoder(conn_state->best_encoder);
5454                 break;
5455         }
5456
5457         if (!crtc_state->bigjoiner_slave) {
5458                 /* need to enable VDSC, which we skipped in pre-enable */
5459                 intel_dsc_enable(encoder, crtc_state);
5460         } else {
5461                 /*
5462                  * Enable sequence steps 1-7 on bigjoiner master
5463                  */
5464                 intel_encoders_pre_pll_enable(state, master);
5465                 intel_enable_shared_dpll(master_crtc_state);
5466                 intel_encoders_pre_enable(state, master);
5467
5468                 /* and DSC on slave */
5469                 intel_dsc_enable(NULL, crtc_state);
5470         }
5471 }
5472
5473 static void hsw_crtc_enable(struct intel_atomic_state *state,
5474                             struct intel_crtc *crtc)
5475 {
5476         const struct intel_crtc_state *new_crtc_state =
5477                 intel_atomic_get_new_crtc_state(state, crtc);
5478         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5479         enum pipe pipe = crtc->pipe, hsw_workaround_pipe;
5480         enum transcoder cpu_transcoder = new_crtc_state->cpu_transcoder;
5481         bool psl_clkgate_wa;
5482
5483         if (drm_WARN_ON(&dev_priv->drm, crtc->active))
5484                 return;
5485
5486         if (!new_crtc_state->bigjoiner) {
5487                 intel_encoders_pre_pll_enable(state, crtc);
5488
5489                 if (new_crtc_state->shared_dpll)
5490                         intel_enable_shared_dpll(new_crtc_state);
5491
5492                 intel_encoders_pre_enable(state, crtc);
5493         } else {
5494                 icl_ddi_bigjoiner_pre_enable(state, new_crtc_state);
5495         }
5496
5497         intel_set_pipe_src_size(new_crtc_state);
5498         if (INTEL_GEN(dev_priv) >= 9 || IS_BROADWELL(dev_priv))
5499                 bdw_set_pipemisc(new_crtc_state);
5500
5501         if (!new_crtc_state->bigjoiner_slave && !transcoder_is_dsi(cpu_transcoder)) {
5502                 intel_set_transcoder_timings(new_crtc_state);
5503
5504                 if (cpu_transcoder != TRANSCODER_EDP)
5505                         intel_de_write(dev_priv, PIPE_MULT(cpu_transcoder),
5506                                        new_crtc_state->pixel_multiplier - 1);
5507
5508                 if (new_crtc_state->has_pch_encoder)
5509                         intel_cpu_transcoder_set_m_n(new_crtc_state,
5510                                                      &new_crtc_state->fdi_m_n, NULL);
5511
5512                 hsw_set_frame_start_delay(new_crtc_state);
5513         }
5514
5515         if (!transcoder_is_dsi(cpu_transcoder))
5516                 hsw_set_pipeconf(new_crtc_state);
5517
5518         crtc->active = true;
5519
5520         /* Display WA #1180: WaDisableScalarClockGating: glk, cnl */
5521         psl_clkgate_wa = (IS_GEMINILAKE(dev_priv) || IS_CANNONLAKE(dev_priv)) &&
5522                 new_crtc_state->pch_pfit.enabled;
5523         if (psl_clkgate_wa)
5524                 glk_pipe_scaler_clock_gating_wa(dev_priv, pipe, true);
5525
5526         if (INTEL_GEN(dev_priv) >= 9)
5527                 skl_pfit_enable(new_crtc_state);
5528         else
5529                 ilk_pfit_enable(new_crtc_state);
5530
5531         /*
5532          * On ILK+ LUT must be loaded before the pipe is running but with
5533          * clocks enabled
5534          */
5535         intel_color_load_luts(new_crtc_state);
5536         intel_color_commit(new_crtc_state);
5537         /* update DSPCNTR to configure gamma/csc for pipe bottom color */
5538         if (INTEL_GEN(dev_priv) < 9)
5539                 intel_disable_primary_plane(new_crtc_state);
5540
5541         hsw_set_linetime_wm(new_crtc_state);
5542
5543         if (INTEL_GEN(dev_priv) >= 11)
5544                 icl_set_pipe_chicken(crtc);
5545
5546         if (dev_priv->display.initial_watermarks)
5547                 dev_priv->display.initial_watermarks(state, crtc);
5548
5549         if (INTEL_GEN(dev_priv) >= 11)
5550                 icl_pipe_mbus_enable(crtc);
5551
5552         if (new_crtc_state->bigjoiner_slave) {
5553                 trace_intel_pipe_enable(crtc);
5554                 intel_crtc_vblank_on(new_crtc_state);
5555         }
5556
5557         intel_encoders_enable(state, crtc);
5558
5559         if (psl_clkgate_wa) {
5560                 intel_wait_for_vblank(dev_priv, pipe);
5561                 glk_pipe_scaler_clock_gating_wa(dev_priv, pipe, false);
5562         }
5563
5564         /* If we change the relative order between pipe/planes enabling, we need
5565          * to change the workaround. */
5566         hsw_workaround_pipe = new_crtc_state->hsw_workaround_pipe;
5567         if (IS_HASWELL(dev_priv) && hsw_workaround_pipe != INVALID_PIPE) {
5568                 intel_wait_for_vblank(dev_priv, hsw_workaround_pipe);
5569                 intel_wait_for_vblank(dev_priv, hsw_workaround_pipe);
5570         }
5571 }
5572
5573 void ilk_pfit_disable(const struct intel_crtc_state *old_crtc_state)
5574 {
5575         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->uapi.crtc);
5576         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5577         enum pipe pipe = crtc->pipe;
5578
5579         /* To avoid upsetting the power well on haswell only disable the pfit if
5580          * it's in use. The hw state code will make sure we get this right. */
5581         if (!old_crtc_state->pch_pfit.enabled)
5582                 return;
5583
5584         intel_de_write(dev_priv, PF_CTL(pipe), 0);
5585         intel_de_write(dev_priv, PF_WIN_POS(pipe), 0);
5586         intel_de_write(dev_priv, PF_WIN_SZ(pipe), 0);
5587 }
5588
5589 static void ilk_crtc_disable(struct intel_atomic_state *state,
5590                              struct intel_crtc *crtc)
5591 {
5592         const struct intel_crtc_state *old_crtc_state =
5593                 intel_atomic_get_old_crtc_state(state, crtc);
5594         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5595         enum pipe pipe = crtc->pipe;
5596
5597         /*
5598          * Sometimes spurious CPU pipe underruns happen when the
5599          * pipe is already disabled, but FDI RX/TX is still enabled.
5600          * Happens at least with VGA+HDMI cloning. Suppress them.
5601          */
5602         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
5603         intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, false);
5604
5605         intel_encoders_disable(state, crtc);
5606
5607         intel_crtc_vblank_off(old_crtc_state);
5608
5609         intel_disable_pipe(old_crtc_state);
5610
5611         ilk_pfit_disable(old_crtc_state);
5612
5613         if (old_crtc_state->has_pch_encoder)
5614                 ilk_fdi_disable(crtc);
5615
5616         intel_encoders_post_disable(state, crtc);
5617
5618         if (old_crtc_state->has_pch_encoder) {
5619                 ilk_disable_pch_transcoder(dev_priv, pipe);
5620
5621                 if (HAS_PCH_CPT(dev_priv)) {
5622                         i915_reg_t reg;
5623                         u32 temp;
5624
5625                         /* disable TRANS_DP_CTL */
5626                         reg = TRANS_DP_CTL(pipe);
5627                         temp = intel_de_read(dev_priv, reg);
5628                         temp &= ~(TRANS_DP_OUTPUT_ENABLE |
5629                                   TRANS_DP_PORT_SEL_MASK);
5630                         temp |= TRANS_DP_PORT_SEL_NONE;
5631                         intel_de_write(dev_priv, reg, temp);
5632
5633                         /* disable DPLL_SEL */
5634                         temp = intel_de_read(dev_priv, PCH_DPLL_SEL);
5635                         temp &= ~(TRANS_DPLL_ENABLE(pipe) | TRANS_DPLLB_SEL(pipe));
5636                         intel_de_write(dev_priv, PCH_DPLL_SEL, temp);
5637                 }
5638
5639                 ilk_fdi_pll_disable(crtc);
5640         }
5641
5642         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
5643         intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, true);
5644 }
5645
5646 static void hsw_crtc_disable(struct intel_atomic_state *state,
5647                              struct intel_crtc *crtc)
5648 {
5649         /*
5650          * FIXME collapse everything to one hook.
5651          * Need care with mst->ddi interactions.
5652          */
5653         intel_encoders_disable(state, crtc);
5654         intel_encoders_post_disable(state, crtc);
5655 }
5656
5657 static void i9xx_pfit_enable(const struct intel_crtc_state *crtc_state)
5658 {
5659         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5660         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5661
5662         if (!crtc_state->gmch_pfit.control)
5663                 return;
5664
5665         /*
5666          * The panel fitter should only be adjusted whilst the pipe is disabled,
5667          * according to register description and PRM.
5668          */
5669         drm_WARN_ON(&dev_priv->drm,
5670                     intel_de_read(dev_priv, PFIT_CONTROL) & PFIT_ENABLE);
5671         assert_pipe_disabled(dev_priv, crtc_state->cpu_transcoder);
5672
5673         intel_de_write(dev_priv, PFIT_PGM_RATIOS,
5674                        crtc_state->gmch_pfit.pgm_ratios);
5675         intel_de_write(dev_priv, PFIT_CONTROL, crtc_state->gmch_pfit.control);
5676
5677         /* Border color in case we don't scale up to the full screen. Black by
5678          * default, change to something else for debugging. */
5679         intel_de_write(dev_priv, BCLRPAT(crtc->pipe), 0);
5680 }
5681
5682 bool intel_phy_is_combo(struct drm_i915_private *dev_priv, enum phy phy)
5683 {
5684         if (phy == PHY_NONE)
5685                 return false;
5686         else if (IS_ALDERLAKE_S(dev_priv))
5687                 return phy <= PHY_E;
5688         else if (IS_DG1(dev_priv) || IS_ROCKETLAKE(dev_priv))
5689                 return phy <= PHY_D;
5690         else if (IS_JSL_EHL(dev_priv))
5691                 return phy <= PHY_C;
5692         else if (INTEL_GEN(dev_priv) >= 11)
5693                 return phy <= PHY_B;
5694         else
5695                 return false;
5696 }
5697
5698 bool intel_phy_is_tc(struct drm_i915_private *dev_priv, enum phy phy)
5699 {
5700         if (IS_TIGERLAKE(dev_priv))
5701                 return phy >= PHY_D && phy <= PHY_I;
5702         else if (IS_ICELAKE(dev_priv))
5703                 return phy >= PHY_C && phy <= PHY_F;
5704         else
5705                 return false;
5706 }
5707
5708 enum phy intel_port_to_phy(struct drm_i915_private *i915, enum port port)
5709 {
5710         if (IS_ALDERLAKE_S(i915) && port >= PORT_TC1)
5711                 return PHY_B + port - PORT_TC1;
5712         else if ((IS_DG1(i915) || IS_ROCKETLAKE(i915)) && port >= PORT_TC1)
5713                 return PHY_C + port - PORT_TC1;
5714         else if (IS_JSL_EHL(i915) && port == PORT_D)
5715                 return PHY_A;
5716
5717         return PHY_A + port - PORT_A;
5718 }
5719
5720 enum tc_port intel_port_to_tc(struct drm_i915_private *dev_priv, enum port port)
5721 {
5722         if (!intel_phy_is_tc(dev_priv, intel_port_to_phy(dev_priv, port)))
5723                 return TC_PORT_NONE;
5724
5725         if (INTEL_GEN(dev_priv) >= 12)
5726                 return TC_PORT_1 + port - PORT_TC1;
5727         else
5728                 return TC_PORT_1 + port - PORT_C;
5729 }
5730
5731 enum intel_display_power_domain intel_port_to_power_domain(enum port port)
5732 {
5733         switch (port) {
5734         case PORT_A:
5735                 return POWER_DOMAIN_PORT_DDI_A_LANES;
5736         case PORT_B:
5737                 return POWER_DOMAIN_PORT_DDI_B_LANES;
5738         case PORT_C:
5739                 return POWER_DOMAIN_PORT_DDI_C_LANES;
5740         case PORT_D:
5741                 return POWER_DOMAIN_PORT_DDI_D_LANES;
5742         case PORT_E:
5743                 return POWER_DOMAIN_PORT_DDI_E_LANES;
5744         case PORT_F:
5745                 return POWER_DOMAIN_PORT_DDI_F_LANES;
5746         case PORT_G:
5747                 return POWER_DOMAIN_PORT_DDI_G_LANES;
5748         case PORT_H:
5749                 return POWER_DOMAIN_PORT_DDI_H_LANES;
5750         case PORT_I:
5751                 return POWER_DOMAIN_PORT_DDI_I_LANES;
5752         default:
5753                 MISSING_CASE(port);
5754                 return POWER_DOMAIN_PORT_OTHER;
5755         }
5756 }
5757
5758 enum intel_display_power_domain
5759 intel_aux_power_domain(struct intel_digital_port *dig_port)
5760 {
5761         struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev);
5762         enum phy phy = intel_port_to_phy(dev_priv, dig_port->base.port);
5763
5764         if (intel_phy_is_tc(dev_priv, phy) &&
5765             dig_port->tc_mode == TC_PORT_TBT_ALT) {
5766                 switch (dig_port->aux_ch) {
5767                 case AUX_CH_C:
5768                         return POWER_DOMAIN_AUX_C_TBT;
5769                 case AUX_CH_D:
5770                         return POWER_DOMAIN_AUX_D_TBT;
5771                 case AUX_CH_E:
5772                         return POWER_DOMAIN_AUX_E_TBT;
5773                 case AUX_CH_F:
5774                         return POWER_DOMAIN_AUX_F_TBT;
5775                 case AUX_CH_G:
5776                         return POWER_DOMAIN_AUX_G_TBT;
5777                 case AUX_CH_H:
5778                         return POWER_DOMAIN_AUX_H_TBT;
5779                 case AUX_CH_I:
5780                         return POWER_DOMAIN_AUX_I_TBT;
5781                 default:
5782                         MISSING_CASE(dig_port->aux_ch);
5783                         return POWER_DOMAIN_AUX_C_TBT;
5784                 }
5785         }
5786
5787         return intel_legacy_aux_to_power_domain(dig_port->aux_ch);
5788 }
5789
5790 /*
5791  * Converts aux_ch to power_domain without caring about TBT ports for that use
5792  * intel_aux_power_domain()
5793  */
5794 enum intel_display_power_domain
5795 intel_legacy_aux_to_power_domain(enum aux_ch aux_ch)
5796 {
5797         switch (aux_ch) {
5798         case AUX_CH_A:
5799                 return POWER_DOMAIN_AUX_A;
5800         case AUX_CH_B:
5801                 return POWER_DOMAIN_AUX_B;
5802         case AUX_CH_C:
5803                 return POWER_DOMAIN_AUX_C;
5804         case AUX_CH_D:
5805                 return POWER_DOMAIN_AUX_D;
5806         case AUX_CH_E:
5807                 return POWER_DOMAIN_AUX_E;
5808         case AUX_CH_F:
5809                 return POWER_DOMAIN_AUX_F;
5810         case AUX_CH_G:
5811                 return POWER_DOMAIN_AUX_G;
5812         case AUX_CH_H:
5813                 return POWER_DOMAIN_AUX_H;
5814         case AUX_CH_I:
5815                 return POWER_DOMAIN_AUX_I;
5816         default:
5817                 MISSING_CASE(aux_ch);
5818                 return POWER_DOMAIN_AUX_A;
5819         }
5820 }
5821
5822 static u64 get_crtc_power_domains(struct intel_crtc_state *crtc_state)
5823 {
5824         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5825         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5826         struct drm_encoder *encoder;
5827         enum pipe pipe = crtc->pipe;
5828         u64 mask;
5829         enum transcoder transcoder = crtc_state->cpu_transcoder;
5830
5831         if (!crtc_state->hw.active)
5832                 return 0;
5833
5834         mask = BIT_ULL(POWER_DOMAIN_PIPE(pipe));
5835         mask |= BIT_ULL(POWER_DOMAIN_TRANSCODER(transcoder));
5836         if (crtc_state->pch_pfit.enabled ||
5837             crtc_state->pch_pfit.force_thru)
5838                 mask |= BIT_ULL(POWER_DOMAIN_PIPE_PANEL_FITTER(pipe));
5839
5840         drm_for_each_encoder_mask(encoder, &dev_priv->drm,
5841                                   crtc_state->uapi.encoder_mask) {
5842                 struct intel_encoder *intel_encoder = to_intel_encoder(encoder);
5843
5844                 mask |= BIT_ULL(intel_encoder->power_domain);
5845         }
5846
5847         if (HAS_DDI(dev_priv) && crtc_state->has_audio)
5848                 mask |= BIT_ULL(POWER_DOMAIN_AUDIO);
5849
5850         if (crtc_state->shared_dpll)
5851                 mask |= BIT_ULL(POWER_DOMAIN_DISPLAY_CORE);
5852
5853         if (crtc_state->dsc.compression_enable)
5854                 mask |= BIT_ULL(intel_dsc_power_domain(crtc_state));
5855
5856         return mask;
5857 }
5858
5859 static u64
5860 modeset_get_crtc_power_domains(struct intel_crtc_state *crtc_state)
5861 {
5862         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5863         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5864         enum intel_display_power_domain domain;
5865         u64 domains, new_domains, old_domains;
5866
5867         domains = get_crtc_power_domains(crtc_state);
5868
5869         new_domains = domains & ~crtc->enabled_power_domains.mask;
5870         old_domains = crtc->enabled_power_domains.mask & ~domains;
5871
5872         for_each_power_domain(domain, new_domains)
5873                 intel_display_power_get_in_set(dev_priv,
5874                                                &crtc->enabled_power_domains,
5875                                                domain);
5876
5877         return old_domains;
5878 }
5879
5880 static void modeset_put_crtc_power_domains(struct intel_crtc *crtc,
5881                                            u64 domains)
5882 {
5883         intel_display_power_put_mask_in_set(to_i915(crtc->base.dev),
5884                                             &crtc->enabled_power_domains,
5885                                             domains);
5886 }
5887
5888 static void valleyview_crtc_enable(struct intel_atomic_state *state,
5889                                    struct intel_crtc *crtc)
5890 {
5891         const struct intel_crtc_state *new_crtc_state =
5892                 intel_atomic_get_new_crtc_state(state, crtc);
5893         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5894         enum pipe pipe = crtc->pipe;
5895
5896         if (drm_WARN_ON(&dev_priv->drm, crtc->active))
5897                 return;
5898
5899         if (intel_crtc_has_dp_encoder(new_crtc_state))
5900                 intel_dp_set_m_n(new_crtc_state, M1_N1);
5901
5902         intel_set_transcoder_timings(new_crtc_state);
5903         intel_set_pipe_src_size(new_crtc_state);
5904
5905         if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_B) {
5906                 intel_de_write(dev_priv, CHV_BLEND(pipe), CHV_BLEND_LEGACY);
5907                 intel_de_write(dev_priv, CHV_CANVAS(pipe), 0);
5908         }
5909
5910         i9xx_set_pipeconf(new_crtc_state);
5911
5912         crtc->active = true;
5913
5914         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
5915
5916         intel_encoders_pre_pll_enable(state, crtc);
5917
5918         if (IS_CHERRYVIEW(dev_priv)) {
5919                 chv_prepare_pll(crtc, new_crtc_state);
5920                 chv_enable_pll(crtc, new_crtc_state);
5921         } else {
5922                 vlv_prepare_pll(crtc, new_crtc_state);
5923                 vlv_enable_pll(crtc, new_crtc_state);
5924         }
5925
5926         intel_encoders_pre_enable(state, crtc);
5927
5928         i9xx_pfit_enable(new_crtc_state);
5929
5930         intel_color_load_luts(new_crtc_state);
5931         intel_color_commit(new_crtc_state);
5932         /* update DSPCNTR to configure gamma for pipe bottom color */
5933         intel_disable_primary_plane(new_crtc_state);
5934
5935         dev_priv->display.initial_watermarks(state, crtc);
5936         intel_enable_pipe(new_crtc_state);
5937
5938         intel_crtc_vblank_on(new_crtc_state);
5939
5940         intel_encoders_enable(state, crtc);
5941 }
5942
5943 static void i9xx_set_pll_dividers(const struct intel_crtc_state *crtc_state)
5944 {
5945         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5946         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5947
5948         intel_de_write(dev_priv, FP0(crtc->pipe),
5949                        crtc_state->dpll_hw_state.fp0);
5950         intel_de_write(dev_priv, FP1(crtc->pipe),
5951                        crtc_state->dpll_hw_state.fp1);
5952 }
5953
5954 static void i9xx_crtc_enable(struct intel_atomic_state *state,
5955                              struct intel_crtc *crtc)
5956 {
5957         const struct intel_crtc_state *new_crtc_state =
5958                 intel_atomic_get_new_crtc_state(state, crtc);
5959         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5960         enum pipe pipe = crtc->pipe;
5961
5962         if (drm_WARN_ON(&dev_priv->drm, crtc->active))
5963                 return;
5964
5965         i9xx_set_pll_dividers(new_crtc_state);
5966
5967         if (intel_crtc_has_dp_encoder(new_crtc_state))
5968                 intel_dp_set_m_n(new_crtc_state, M1_N1);
5969
5970         intel_set_transcoder_timings(new_crtc_state);
5971         intel_set_pipe_src_size(new_crtc_state);
5972
5973         i9xx_set_pipeconf(new_crtc_state);
5974
5975         crtc->active = true;
5976
5977         if (!IS_GEN(dev_priv, 2))
5978                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
5979
5980         intel_encoders_pre_enable(state, crtc);
5981
5982         i9xx_enable_pll(crtc, new_crtc_state);
5983
5984         i9xx_pfit_enable(new_crtc_state);
5985
5986         intel_color_load_luts(new_crtc_state);
5987         intel_color_commit(new_crtc_state);
5988         /* update DSPCNTR to configure gamma for pipe bottom color */
5989         intel_disable_primary_plane(new_crtc_state);
5990
5991         if (dev_priv->display.initial_watermarks)
5992                 dev_priv->display.initial_watermarks(state, crtc);
5993         else
5994                 intel_update_watermarks(crtc);
5995         intel_enable_pipe(new_crtc_state);
5996
5997         intel_crtc_vblank_on(new_crtc_state);
5998
5999         intel_encoders_enable(state, crtc);
6000
6001         /* prevents spurious underruns */
6002         if (IS_GEN(dev_priv, 2))
6003                 intel_wait_for_vblank(dev_priv, pipe);
6004 }
6005
6006 static void i9xx_pfit_disable(const struct intel_crtc_state *old_crtc_state)
6007 {
6008         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->uapi.crtc);
6009         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6010
6011         if (!old_crtc_state->gmch_pfit.control)
6012                 return;
6013
6014         assert_pipe_disabled(dev_priv, old_crtc_state->cpu_transcoder);
6015
6016         drm_dbg_kms(&dev_priv->drm, "disabling pfit, current: 0x%08x\n",
6017                     intel_de_read(dev_priv, PFIT_CONTROL));
6018         intel_de_write(dev_priv, PFIT_CONTROL, 0);
6019 }
6020
6021 static void i9xx_crtc_disable(struct intel_atomic_state *state,
6022                               struct intel_crtc *crtc)
6023 {
6024         struct intel_crtc_state *old_crtc_state =
6025                 intel_atomic_get_old_crtc_state(state, crtc);
6026         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6027         enum pipe pipe = crtc->pipe;
6028
6029         /*
6030          * On gen2 planes are double buffered but the pipe isn't, so we must
6031          * wait for planes to fully turn off before disabling the pipe.
6032          */
6033         if (IS_GEN(dev_priv, 2))
6034                 intel_wait_for_vblank(dev_priv, pipe);
6035
6036         intel_encoders_disable(state, crtc);
6037
6038         intel_crtc_vblank_off(old_crtc_state);
6039
6040         intel_disable_pipe(old_crtc_state);
6041
6042         i9xx_pfit_disable(old_crtc_state);
6043
6044         intel_encoders_post_disable(state, crtc);
6045
6046         if (!intel_crtc_has_type(old_crtc_state, INTEL_OUTPUT_DSI)) {
6047                 if (IS_CHERRYVIEW(dev_priv))
6048                         chv_disable_pll(dev_priv, pipe);
6049                 else if (IS_VALLEYVIEW(dev_priv))
6050                         vlv_disable_pll(dev_priv, pipe);
6051                 else
6052                         i9xx_disable_pll(old_crtc_state);
6053         }
6054
6055         intel_encoders_post_pll_disable(state, crtc);
6056
6057         if (!IS_GEN(dev_priv, 2))
6058                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
6059
6060         if (!dev_priv->display.initial_watermarks)
6061                 intel_update_watermarks(crtc);
6062
6063         /* clock the pipe down to 640x480@60 to potentially save power */
6064         if (IS_I830(dev_priv))
6065                 i830_enable_pipe(dev_priv, pipe);
6066 }
6067
6068 static void intel_crtc_disable_noatomic(struct intel_crtc *crtc,
6069                                         struct drm_modeset_acquire_ctx *ctx)
6070 {
6071         struct intel_encoder *encoder;
6072         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6073         struct intel_bw_state *bw_state =
6074                 to_intel_bw_state(dev_priv->bw_obj.state);
6075         struct intel_cdclk_state *cdclk_state =
6076                 to_intel_cdclk_state(dev_priv->cdclk.obj.state);
6077         struct intel_dbuf_state *dbuf_state =
6078                 to_intel_dbuf_state(dev_priv->dbuf.obj.state);
6079         struct intel_crtc_state *crtc_state =
6080                 to_intel_crtc_state(crtc->base.state);
6081         struct intel_plane *plane;
6082         struct drm_atomic_state *state;
6083         struct intel_crtc_state *temp_crtc_state;
6084         enum pipe pipe = crtc->pipe;
6085         int ret;
6086
6087         if (!crtc_state->hw.active)
6088                 return;
6089
6090         for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
6091                 const struct intel_plane_state *plane_state =
6092                         to_intel_plane_state(plane->base.state);
6093
6094                 if (plane_state->uapi.visible)
6095                         intel_plane_disable_noatomic(crtc, plane);
6096         }
6097
6098         state = drm_atomic_state_alloc(&dev_priv->drm);
6099         if (!state) {
6100                 drm_dbg_kms(&dev_priv->drm,
6101                             "failed to disable [CRTC:%d:%s], out of memory",
6102                             crtc->base.base.id, crtc->base.name);
6103                 return;
6104         }
6105
6106         state->acquire_ctx = ctx;
6107
6108         /* Everything's already locked, -EDEADLK can't happen. */
6109         temp_crtc_state = intel_atomic_get_crtc_state(state, crtc);
6110         ret = drm_atomic_add_affected_connectors(state, &crtc->base);
6111
6112         drm_WARN_ON(&dev_priv->drm, IS_ERR(temp_crtc_state) || ret);
6113
6114         dev_priv->display.crtc_disable(to_intel_atomic_state(state), crtc);
6115
6116         drm_atomic_state_put(state);
6117
6118         drm_dbg_kms(&dev_priv->drm,
6119                     "[CRTC:%d:%s] hw state adjusted, was enabled, now disabled\n",
6120                     crtc->base.base.id, crtc->base.name);
6121
6122         crtc->active = false;
6123         crtc->base.enabled = false;
6124
6125         drm_WARN_ON(&dev_priv->drm,
6126                     drm_atomic_set_mode_for_crtc(&crtc_state->uapi, NULL) < 0);
6127         crtc_state->uapi.active = false;
6128         crtc_state->uapi.connector_mask = 0;
6129         crtc_state->uapi.encoder_mask = 0;
6130         intel_crtc_free_hw_state(crtc_state);
6131         memset(&crtc_state->hw, 0, sizeof(crtc_state->hw));
6132
6133         for_each_encoder_on_crtc(&dev_priv->drm, &crtc->base, encoder)
6134                 encoder->base.crtc = NULL;
6135
6136         intel_fbc_disable(crtc);
6137         intel_update_watermarks(crtc);
6138         intel_disable_shared_dpll(crtc_state);
6139
6140         intel_display_power_put_all_in_set(dev_priv, &crtc->enabled_power_domains);
6141
6142         dev_priv->active_pipes &= ~BIT(pipe);
6143         cdclk_state->min_cdclk[pipe] = 0;
6144         cdclk_state->min_voltage_level[pipe] = 0;
6145         cdclk_state->active_pipes &= ~BIT(pipe);
6146
6147         dbuf_state->active_pipes &= ~BIT(pipe);
6148
6149         bw_state->data_rate[pipe] = 0;
6150         bw_state->num_active_planes[pipe] = 0;
6151 }
6152
6153 /*
6154  * turn all crtc's off, but do not adjust state
6155  * This has to be paired with a call to intel_modeset_setup_hw_state.
6156  */
6157 int intel_display_suspend(struct drm_device *dev)
6158 {
6159         struct drm_i915_private *dev_priv = to_i915(dev);
6160         struct drm_atomic_state *state;
6161         int ret;
6162
6163         state = drm_atomic_helper_suspend(dev);
6164         ret = PTR_ERR_OR_ZERO(state);
6165         if (ret)
6166                 drm_err(&dev_priv->drm, "Suspending crtc's failed with %i\n",
6167                         ret);
6168         else
6169                 dev_priv->modeset_restore_state = state;
6170         return ret;
6171 }
6172
6173 void intel_encoder_destroy(struct drm_encoder *encoder)
6174 {
6175         struct intel_encoder *intel_encoder = to_intel_encoder(encoder);
6176
6177         drm_encoder_cleanup(encoder);
6178         kfree(intel_encoder);
6179 }
6180
6181 /* Cross check the actual hw state with our own modeset state tracking (and it's
6182  * internal consistency). */
6183 static void intel_connector_verify_state(struct intel_crtc_state *crtc_state,
6184                                          struct drm_connector_state *conn_state)
6185 {
6186         struct intel_connector *connector = to_intel_connector(conn_state->connector);
6187         struct drm_i915_private *i915 = to_i915(connector->base.dev);
6188
6189         drm_dbg_kms(&i915->drm, "[CONNECTOR:%d:%s]\n",
6190                     connector->base.base.id, connector->base.name);
6191
6192         if (connector->get_hw_state(connector)) {
6193                 struct intel_encoder *encoder = intel_attached_encoder(connector);
6194
6195                 I915_STATE_WARN(!crtc_state,
6196                          "connector enabled without attached crtc\n");
6197
6198                 if (!crtc_state)
6199                         return;
6200
6201                 I915_STATE_WARN(!crtc_state->hw.active,
6202                                 "connector is active, but attached crtc isn't\n");
6203
6204                 if (!encoder || encoder->type == INTEL_OUTPUT_DP_MST)
6205                         return;
6206
6207                 I915_STATE_WARN(conn_state->best_encoder != &encoder->base,
6208                         "atomic encoder doesn't match attached encoder\n");
6209
6210                 I915_STATE_WARN(conn_state->crtc != encoder->base.crtc,
6211                         "attached encoder crtc differs from connector crtc\n");
6212         } else {
6213                 I915_STATE_WARN(crtc_state && crtc_state->hw.active,
6214                                 "attached crtc is active, but connector isn't\n");
6215                 I915_STATE_WARN(!crtc_state && conn_state->best_encoder,
6216                         "best encoder set without crtc!\n");
6217         }
6218 }
6219
6220 bool hsw_crtc_state_ips_capable(const struct intel_crtc_state *crtc_state)
6221 {
6222         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
6223         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6224
6225         /* IPS only exists on ULT machines and is tied to pipe A. */
6226         if (!hsw_crtc_supports_ips(crtc))
6227                 return false;
6228
6229         if (!dev_priv->params.enable_ips)
6230                 return false;
6231
6232         if (crtc_state->pipe_bpp > 24)
6233                 return false;
6234
6235         /*
6236          * We compare against max which means we must take
6237          * the increased cdclk requirement into account when
6238          * calculating the new cdclk.
6239          *
6240          * Should measure whether using a lower cdclk w/o IPS
6241          */
6242         if (IS_BROADWELL(dev_priv) &&
6243             crtc_state->pixel_rate > dev_priv->max_cdclk_freq * 95 / 100)
6244                 return false;
6245
6246         return true;
6247 }
6248
6249 static int hsw_compute_ips_config(struct intel_crtc_state *crtc_state)
6250 {
6251         struct drm_i915_private *dev_priv =
6252                 to_i915(crtc_state->uapi.crtc->dev);
6253         struct intel_atomic_state *state =
6254                 to_intel_atomic_state(crtc_state->uapi.state);
6255
6256         crtc_state->ips_enabled = false;
6257
6258         if (!hsw_crtc_state_ips_capable(crtc_state))
6259                 return 0;
6260
6261         /*
6262          * When IPS gets enabled, the pipe CRC changes. Since IPS gets
6263          * enabled and disabled dynamically based on package C states,
6264          * user space can't make reliable use of the CRCs, so let's just
6265          * completely disable it.
6266          */
6267         if (crtc_state->crc_enabled)
6268                 return 0;
6269
6270         /* IPS should be fine as long as at least one plane is enabled. */
6271         if (!(crtc_state->active_planes & ~BIT(PLANE_CURSOR)))
6272                 return 0;
6273
6274         if (IS_BROADWELL(dev_priv)) {
6275                 const struct intel_cdclk_state *cdclk_state;
6276
6277                 cdclk_state = intel_atomic_get_cdclk_state(state);
6278                 if (IS_ERR(cdclk_state))
6279                         return PTR_ERR(cdclk_state);
6280
6281                 /* pixel rate mustn't exceed 95% of cdclk with IPS on BDW */
6282                 if (crtc_state->pixel_rate > cdclk_state->logical.cdclk * 95 / 100)
6283                         return 0;
6284         }
6285
6286         crtc_state->ips_enabled = true;
6287
6288         return 0;
6289 }
6290
6291 static bool intel_crtc_supports_double_wide(const struct intel_crtc *crtc)
6292 {
6293         const struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6294
6295         /* GDG double wide on either pipe, otherwise pipe A only */
6296         return INTEL_GEN(dev_priv) < 4 &&
6297                 (crtc->pipe == PIPE_A || IS_I915G(dev_priv));
6298 }
6299
6300 static u32 ilk_pipe_pixel_rate(const struct intel_crtc_state *crtc_state)
6301 {
6302         u32 pixel_rate = crtc_state->hw.pipe_mode.crtc_clock;
6303         unsigned int pipe_w, pipe_h, pfit_w, pfit_h;
6304
6305         /*
6306          * We only use IF-ID interlacing. If we ever use
6307          * PF-ID we'll need to adjust the pixel_rate here.
6308          */
6309
6310         if (!crtc_state->pch_pfit.enabled)
6311                 return pixel_rate;
6312
6313         pipe_w = crtc_state->pipe_src_w;
6314         pipe_h = crtc_state->pipe_src_h;
6315
6316         pfit_w = drm_rect_width(&crtc_state->pch_pfit.dst);
6317         pfit_h = drm_rect_height(&crtc_state->pch_pfit.dst);
6318
6319         if (pipe_w < pfit_w)
6320                 pipe_w = pfit_w;
6321         if (pipe_h < pfit_h)
6322                 pipe_h = pfit_h;
6323
6324         if (drm_WARN_ON(crtc_state->uapi.crtc->dev,
6325                         !pfit_w || !pfit_h))
6326                 return pixel_rate;
6327
6328         return div_u64(mul_u32_u32(pixel_rate, pipe_w * pipe_h),
6329                        pfit_w * pfit_h);
6330 }
6331
6332 static void intel_mode_from_crtc_timings(struct drm_display_mode *mode,
6333                                          const struct drm_display_mode *timings)
6334 {
6335         mode->hdisplay = timings->crtc_hdisplay;
6336         mode->htotal = timings->crtc_htotal;
6337         mode->hsync_start = timings->crtc_hsync_start;
6338         mode->hsync_end = timings->crtc_hsync_end;
6339
6340         mode->vdisplay = timings->crtc_vdisplay;
6341         mode->vtotal = timings->crtc_vtotal;
6342         mode->vsync_start = timings->crtc_vsync_start;
6343         mode->vsync_end = timings->crtc_vsync_end;
6344
6345         mode->flags = timings->flags;
6346         mode->type = DRM_MODE_TYPE_DRIVER;
6347
6348         mode->clock = timings->crtc_clock;
6349
6350         drm_mode_set_name(mode);
6351 }
6352
6353 static void intel_crtc_compute_pixel_rate(struct intel_crtc_state *crtc_state)
6354 {
6355         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
6356
6357         if (HAS_GMCH(dev_priv))
6358                 /* FIXME calculate proper pipe pixel rate for GMCH pfit */
6359                 crtc_state->pixel_rate =
6360                         crtc_state->hw.pipe_mode.crtc_clock;
6361         else
6362                 crtc_state->pixel_rate =
6363                         ilk_pipe_pixel_rate(crtc_state);
6364 }
6365
6366 static void intel_crtc_readout_derived_state(struct intel_crtc_state *crtc_state)
6367 {
6368         struct drm_display_mode *mode = &crtc_state->hw.mode;
6369         struct drm_display_mode *pipe_mode = &crtc_state->hw.pipe_mode;
6370         struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode;
6371
6372         drm_mode_copy(pipe_mode, adjusted_mode);
6373
6374         if (crtc_state->bigjoiner) {
6375                 /*
6376                  * transcoder is programmed to the full mode,
6377                  * but pipe timings are half of the transcoder mode
6378                  */
6379                 pipe_mode->crtc_hdisplay /= 2;
6380                 pipe_mode->crtc_hblank_start /= 2;
6381                 pipe_mode->crtc_hblank_end /= 2;
6382                 pipe_mode->crtc_hsync_start /= 2;
6383                 pipe_mode->crtc_hsync_end /= 2;
6384                 pipe_mode->crtc_htotal /= 2;
6385                 pipe_mode->crtc_clock /= 2;
6386         }
6387
6388         intel_mode_from_crtc_timings(pipe_mode, pipe_mode);
6389         intel_mode_from_crtc_timings(adjusted_mode, adjusted_mode);
6390
6391         intel_crtc_compute_pixel_rate(crtc_state);
6392
6393         drm_mode_copy(mode, adjusted_mode);
6394         mode->hdisplay = crtc_state->pipe_src_w << crtc_state->bigjoiner;
6395         mode->vdisplay = crtc_state->pipe_src_h;
6396 }
6397
6398 static void intel_encoder_get_config(struct intel_encoder *encoder,
6399                                      struct intel_crtc_state *crtc_state)
6400 {
6401         encoder->get_config(encoder, crtc_state);
6402
6403         intel_crtc_readout_derived_state(crtc_state);
6404 }
6405
6406 static int intel_crtc_compute_config(struct intel_crtc *crtc,
6407                                      struct intel_crtc_state *pipe_config)
6408 {
6409         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6410         struct drm_display_mode *pipe_mode = &pipe_config->hw.pipe_mode;
6411         int clock_limit = dev_priv->max_dotclk_freq;
6412
6413         drm_mode_copy(pipe_mode, &pipe_config->hw.adjusted_mode);
6414
6415         /* Adjust pipe_mode for bigjoiner, with half the horizontal mode */
6416         if (pipe_config->bigjoiner) {
6417                 pipe_mode->crtc_clock /= 2;
6418                 pipe_mode->crtc_hdisplay /= 2;
6419                 pipe_mode->crtc_hblank_start /= 2;
6420                 pipe_mode->crtc_hblank_end /= 2;
6421                 pipe_mode->crtc_hsync_start /= 2;
6422                 pipe_mode->crtc_hsync_end /= 2;
6423                 pipe_mode->crtc_htotal /= 2;
6424                 pipe_config->pipe_src_w /= 2;
6425         }
6426
6427         intel_mode_from_crtc_timings(pipe_mode, pipe_mode);
6428
6429         if (INTEL_GEN(dev_priv) < 4) {
6430                 clock_limit = dev_priv->max_cdclk_freq * 9 / 10;
6431
6432                 /*
6433                  * Enable double wide mode when the dot clock
6434                  * is > 90% of the (display) core speed.
6435                  */
6436                 if (intel_crtc_supports_double_wide(crtc) &&
6437                     pipe_mode->crtc_clock > clock_limit) {
6438                         clock_limit = dev_priv->max_dotclk_freq;
6439                         pipe_config->double_wide = true;
6440                 }
6441         }
6442
6443         if (pipe_mode->crtc_clock > clock_limit) {
6444                 drm_dbg_kms(&dev_priv->drm,
6445                             "requested pixel clock (%d kHz) too high (max: %d kHz, double wide: %s)\n",
6446                             pipe_mode->crtc_clock, clock_limit,
6447                             yesno(pipe_config->double_wide));
6448                 return -EINVAL;
6449         }
6450
6451         /*
6452          * Pipe horizontal size must be even in:
6453          * - DVO ganged mode
6454          * - LVDS dual channel mode
6455          * - Double wide pipe
6456          */
6457         if (pipe_config->pipe_src_w & 1) {
6458                 if (pipe_config->double_wide) {
6459                         drm_dbg_kms(&dev_priv->drm,
6460                                     "Odd pipe source width not supported with double wide pipe\n");
6461                         return -EINVAL;
6462                 }
6463
6464                 if (intel_crtc_has_type(pipe_config, INTEL_OUTPUT_LVDS) &&
6465                     intel_is_dual_link_lvds(dev_priv)) {
6466                         drm_dbg_kms(&dev_priv->drm,
6467                                     "Odd pipe source width not supported with dual link LVDS\n");
6468                         return -EINVAL;
6469                 }
6470         }
6471
6472         /* Cantiga+ cannot handle modes with a hsync front porch of 0.
6473          * WaPruneModeWithIncorrectHsyncOffset:ctg,elk,ilk,snb,ivb,vlv,hsw.
6474          */
6475         if ((INTEL_GEN(dev_priv) > 4 || IS_G4X(dev_priv)) &&
6476             pipe_mode->crtc_hsync_start == pipe_mode->crtc_hdisplay)
6477                 return -EINVAL;
6478
6479         intel_crtc_compute_pixel_rate(pipe_config);
6480
6481         if (pipe_config->has_pch_encoder)
6482                 return ilk_fdi_compute_config(crtc, pipe_config);
6483
6484         return 0;
6485 }
6486
6487 static void
6488 intel_reduce_m_n_ratio(u32 *num, u32 *den)
6489 {
6490         while (*num > DATA_LINK_M_N_MASK ||
6491                *den > DATA_LINK_M_N_MASK) {
6492                 *num >>= 1;
6493                 *den >>= 1;
6494         }
6495 }
6496
6497 static void compute_m_n(unsigned int m, unsigned int n,
6498                         u32 *ret_m, u32 *ret_n,
6499                         bool constant_n)
6500 {
6501         /*
6502          * Several DP dongles in particular seem to be fussy about
6503          * too large link M/N values. Give N value as 0x8000 that
6504          * should be acceptable by specific devices. 0x8000 is the
6505          * specified fixed N value for asynchronous clock mode,
6506          * which the devices expect also in synchronous clock mode.
6507          */
6508         if (constant_n)
6509                 *ret_n = DP_LINK_CONSTANT_N_VALUE;
6510         else
6511                 *ret_n = min_t(unsigned int, roundup_pow_of_two(n), DATA_LINK_N_MAX);
6512
6513         *ret_m = div_u64(mul_u32_u32(m, *ret_n), n);
6514         intel_reduce_m_n_ratio(ret_m, ret_n);
6515 }
6516
6517 void
6518 intel_link_compute_m_n(u16 bits_per_pixel, int nlanes,
6519                        int pixel_clock, int link_clock,
6520                        struct intel_link_m_n *m_n,
6521                        bool constant_n, bool fec_enable)
6522 {
6523         u32 data_clock = bits_per_pixel * pixel_clock;
6524
6525         if (fec_enable)
6526                 data_clock = intel_dp_mode_to_fec_clock(data_clock);
6527
6528         m_n->tu = 64;
6529         compute_m_n(data_clock,
6530                     link_clock * nlanes * 8,
6531                     &m_n->gmch_m, &m_n->gmch_n,
6532                     constant_n);
6533
6534         compute_m_n(pixel_clock, link_clock,
6535                     &m_n->link_m, &m_n->link_n,
6536                     constant_n);
6537 }
6538
6539 static void intel_panel_sanitize_ssc(struct drm_i915_private *dev_priv)
6540 {
6541         /*
6542          * There may be no VBT; and if the BIOS enabled SSC we can
6543          * just keep using it to avoid unnecessary flicker.  Whereas if the
6544          * BIOS isn't using it, don't assume it will work even if the VBT
6545          * indicates as much.
6546          */
6547         if (HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv)) {
6548                 bool bios_lvds_use_ssc = intel_de_read(dev_priv,
6549                                                        PCH_DREF_CONTROL) &
6550                         DREF_SSC1_ENABLE;
6551
6552                 if (dev_priv->vbt.lvds_use_ssc != bios_lvds_use_ssc) {
6553                         drm_dbg_kms(&dev_priv->drm,
6554                                     "SSC %s by BIOS, overriding VBT which says %s\n",
6555                                     enableddisabled(bios_lvds_use_ssc),
6556                                     enableddisabled(dev_priv->vbt.lvds_use_ssc));
6557                         dev_priv->vbt.lvds_use_ssc = bios_lvds_use_ssc;
6558                 }
6559         }
6560 }
6561
6562 static void vlv_pllb_recal_opamp(struct drm_i915_private *dev_priv, enum pipe
6563                 pipe)
6564 {
6565         u32 reg_val;
6566
6567         /*
6568          * PLLB opamp always calibrates to max value of 0x3f, force enable it
6569          * and set it to a reasonable value instead.
6570          */
6571         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW9(1));
6572         reg_val &= 0xffffff00;
6573         reg_val |= 0x00000030;
6574         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW9(1), reg_val);
6575
6576         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_REF_DW13);
6577         reg_val &= 0x00ffffff;
6578         reg_val |= 0x8c000000;
6579         vlv_dpio_write(dev_priv, pipe, VLV_REF_DW13, reg_val);
6580
6581         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW9(1));
6582         reg_val &= 0xffffff00;
6583         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW9(1), reg_val);
6584
6585         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_REF_DW13);
6586         reg_val &= 0x00ffffff;
6587         reg_val |= 0xb0000000;
6588         vlv_dpio_write(dev_priv, pipe, VLV_REF_DW13, reg_val);
6589 }
6590
6591 static void intel_pch_transcoder_set_m_n(const struct intel_crtc_state *crtc_state,
6592                                          const struct intel_link_m_n *m_n)
6593 {
6594         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
6595         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6596         enum pipe pipe = crtc->pipe;
6597
6598         intel_de_write(dev_priv, PCH_TRANS_DATA_M1(pipe),
6599                        TU_SIZE(m_n->tu) | m_n->gmch_m);
6600         intel_de_write(dev_priv, PCH_TRANS_DATA_N1(pipe), m_n->gmch_n);
6601         intel_de_write(dev_priv, PCH_TRANS_LINK_M1(pipe), m_n->link_m);
6602         intel_de_write(dev_priv, PCH_TRANS_LINK_N1(pipe), m_n->link_n);
6603 }
6604
6605 static bool transcoder_has_m2_n2(struct drm_i915_private *dev_priv,
6606                                  enum transcoder transcoder)
6607 {
6608         if (IS_HASWELL(dev_priv))
6609                 return transcoder == TRANSCODER_EDP;
6610
6611         /*
6612          * Strictly speaking some registers are available before
6613          * gen7, but we only support DRRS on gen7+
6614          */
6615         return IS_GEN(dev_priv, 7) || IS_CHERRYVIEW(dev_priv);
6616 }
6617
6618 static void intel_cpu_transcoder_set_m_n(const struct intel_crtc_state *crtc_state,
6619                                          const struct intel_link_m_n *m_n,
6620                                          const struct intel_link_m_n *m2_n2)
6621 {
6622         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
6623         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6624         enum pipe pipe = crtc->pipe;
6625         enum transcoder transcoder = crtc_state->cpu_transcoder;
6626
6627         if (INTEL_GEN(dev_priv) >= 5) {
6628                 intel_de_write(dev_priv, PIPE_DATA_M1(transcoder),
6629                                TU_SIZE(m_n->tu) | m_n->gmch_m);
6630                 intel_de_write(dev_priv, PIPE_DATA_N1(transcoder),
6631                                m_n->gmch_n);
6632                 intel_de_write(dev_priv, PIPE_LINK_M1(transcoder),
6633                                m_n->link_m);
6634                 intel_de_write(dev_priv, PIPE_LINK_N1(transcoder),
6635                                m_n->link_n);
6636                 /*
6637                  *  M2_N2 registers are set only if DRRS is supported
6638                  * (to make sure the registers are not unnecessarily accessed).
6639                  */
6640                 if (m2_n2 && crtc_state->has_drrs &&
6641                     transcoder_has_m2_n2(dev_priv, transcoder)) {
6642                         intel_de_write(dev_priv, PIPE_DATA_M2(transcoder),
6643                                        TU_SIZE(m2_n2->tu) | m2_n2->gmch_m);
6644                         intel_de_write(dev_priv, PIPE_DATA_N2(transcoder),
6645                                        m2_n2->gmch_n);
6646                         intel_de_write(dev_priv, PIPE_LINK_M2(transcoder),
6647                                        m2_n2->link_m);
6648                         intel_de_write(dev_priv, PIPE_LINK_N2(transcoder),
6649                                        m2_n2->link_n);
6650                 }
6651         } else {
6652                 intel_de_write(dev_priv, PIPE_DATA_M_G4X(pipe),
6653                                TU_SIZE(m_n->tu) | m_n->gmch_m);
6654                 intel_de_write(dev_priv, PIPE_DATA_N_G4X(pipe), m_n->gmch_n);
6655                 intel_de_write(dev_priv, PIPE_LINK_M_G4X(pipe), m_n->link_m);
6656                 intel_de_write(dev_priv, PIPE_LINK_N_G4X(pipe), m_n->link_n);
6657         }
6658 }
6659
6660 void intel_dp_set_m_n(const struct intel_crtc_state *crtc_state, enum link_m_n_set m_n)
6661 {
6662         const struct intel_link_m_n *dp_m_n, *dp_m2_n2 = NULL;
6663         struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
6664
6665         if (m_n == M1_N1) {
6666                 dp_m_n = &crtc_state->dp_m_n;
6667                 dp_m2_n2 = &crtc_state->dp_m2_n2;
6668         } else if (m_n == M2_N2) {
6669
6670                 /*
6671                  * M2_N2 registers are not supported. Hence m2_n2 divider value
6672                  * needs to be programmed into M1_N1.
6673                  */
6674                 dp_m_n = &crtc_state->dp_m2_n2;
6675         } else {
6676                 drm_err(&i915->drm, "Unsupported divider value\n");
6677                 return;
6678         }
6679
6680         if (crtc_state->has_pch_encoder)
6681                 intel_pch_transcoder_set_m_n(crtc_state, &crtc_state->dp_m_n);
6682         else
6683                 intel_cpu_transcoder_set_m_n(crtc_state, dp_m_n, dp_m2_n2);
6684 }
6685
6686 static void vlv_prepare_pll(struct intel_crtc *crtc,
6687                             const struct intel_crtc_state *pipe_config)
6688 {
6689         struct drm_device *dev = crtc->base.dev;
6690         struct drm_i915_private *dev_priv = to_i915(dev);
6691         enum pipe pipe = crtc->pipe;
6692         u32 mdiv;
6693         u32 bestn, bestm1, bestm2, bestp1, bestp2;
6694         u32 coreclk, reg_val;
6695
6696         /* Enable Refclk */
6697         intel_de_write(dev_priv, DPLL(pipe),
6698                        pipe_config->dpll_hw_state.dpll & ~(DPLL_VCO_ENABLE | DPLL_EXT_BUFFER_ENABLE_VLV));
6699
6700         /* No need to actually set up the DPLL with DSI */
6701         if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0)
6702                 return;
6703
6704         vlv_dpio_get(dev_priv);
6705
6706         bestn = pipe_config->dpll.n;
6707         bestm1 = pipe_config->dpll.m1;
6708         bestm2 = pipe_config->dpll.m2;
6709         bestp1 = pipe_config->dpll.p1;
6710         bestp2 = pipe_config->dpll.p2;
6711
6712         /* See eDP HDMI DPIO driver vbios notes doc */
6713
6714         /* PLL B needs special handling */
6715         if (pipe == PIPE_B)
6716                 vlv_pllb_recal_opamp(dev_priv, pipe);
6717
6718         /* Set up Tx target for periodic Rcomp update */
6719         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW9_BCAST, 0x0100000f);
6720
6721         /* Disable target IRef on PLL */
6722         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW8(pipe));
6723         reg_val &= 0x00ffffff;
6724         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW8(pipe), reg_val);
6725
6726         /* Disable fast lock */
6727         vlv_dpio_write(dev_priv, pipe, VLV_CMN_DW0, 0x610);
6728
6729         /* Set idtafcrecal before PLL is enabled */
6730         mdiv = ((bestm1 << DPIO_M1DIV_SHIFT) | (bestm2 & DPIO_M2DIV_MASK));
6731         mdiv |= ((bestp1 << DPIO_P1_SHIFT) | (bestp2 << DPIO_P2_SHIFT));
6732         mdiv |= ((bestn << DPIO_N_SHIFT));
6733         mdiv |= (1 << DPIO_K_SHIFT);
6734
6735         /*
6736          * Post divider depends on pixel clock rate, DAC vs digital (and LVDS,
6737          * but we don't support that).
6738          * Note: don't use the DAC post divider as it seems unstable.
6739          */
6740         mdiv |= (DPIO_POST_DIV_HDMIDP << DPIO_POST_DIV_SHIFT);
6741         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW3(pipe), mdiv);
6742
6743         mdiv |= DPIO_ENABLE_CALIBRATION;
6744         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW3(pipe), mdiv);
6745
6746         /* Set HBR and RBR LPF coefficients */
6747         if (pipe_config->port_clock == 162000 ||
6748             intel_crtc_has_type(pipe_config, INTEL_OUTPUT_ANALOG) ||
6749             intel_crtc_has_type(pipe_config, INTEL_OUTPUT_HDMI))
6750                 vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW10(pipe),
6751                                  0x009f0003);
6752         else
6753                 vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW10(pipe),
6754                                  0x00d0000f);
6755
6756         if (intel_crtc_has_dp_encoder(pipe_config)) {
6757                 /* Use SSC source */
6758                 if (pipe == PIPE_A)
6759                         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
6760                                          0x0df40000);
6761                 else
6762                         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
6763                                          0x0df70000);
6764         } else { /* HDMI or VGA */
6765                 /* Use bend source */
6766                 if (pipe == PIPE_A)
6767                         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
6768                                          0x0df70000);
6769                 else
6770                         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
6771                                          0x0df40000);
6772         }
6773
6774         coreclk = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW7(pipe));
6775         coreclk = (coreclk & 0x0000ff00) | 0x01c00000;
6776         if (intel_crtc_has_dp_encoder(pipe_config))
6777                 coreclk |= 0x01000000;
6778         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW7(pipe), coreclk);
6779
6780         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW11(pipe), 0x87871000);
6781
6782         vlv_dpio_put(dev_priv);
6783 }
6784
6785 static void chv_prepare_pll(struct intel_crtc *crtc,
6786                             const struct intel_crtc_state *pipe_config)
6787 {
6788         struct drm_device *dev = crtc->base.dev;
6789         struct drm_i915_private *dev_priv = to_i915(dev);
6790         enum pipe pipe = crtc->pipe;
6791         enum dpio_channel port = vlv_pipe_to_channel(pipe);
6792         u32 loopfilter, tribuf_calcntr;
6793         u32 bestn, bestm1, bestm2, bestp1, bestp2, bestm2_frac;
6794         u32 dpio_val;
6795         int vco;
6796
6797         /* Enable Refclk and SSC */
6798         intel_de_write(dev_priv, DPLL(pipe),
6799                        pipe_config->dpll_hw_state.dpll & ~DPLL_VCO_ENABLE);
6800
6801         /* No need to actually set up the DPLL with DSI */
6802         if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0)
6803                 return;
6804
6805         bestn = pipe_config->dpll.n;
6806         bestm2_frac = pipe_config->dpll.m2 & 0x3fffff;
6807         bestm1 = pipe_config->dpll.m1;
6808         bestm2 = pipe_config->dpll.m2 >> 22;
6809         bestp1 = pipe_config->dpll.p1;
6810         bestp2 = pipe_config->dpll.p2;
6811         vco = pipe_config->dpll.vco;
6812         dpio_val = 0;
6813         loopfilter = 0;
6814
6815         vlv_dpio_get(dev_priv);
6816
6817         /* p1 and p2 divider */
6818         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW13(port),
6819                         5 << DPIO_CHV_S1_DIV_SHIFT |
6820                         bestp1 << DPIO_CHV_P1_DIV_SHIFT |
6821                         bestp2 << DPIO_CHV_P2_DIV_SHIFT |
6822                         1 << DPIO_CHV_K_DIV_SHIFT);
6823
6824         /* Feedback post-divider - m2 */
6825         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW0(port), bestm2);
6826
6827         /* Feedback refclk divider - n and m1 */
6828         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW1(port),
6829                         DPIO_CHV_M1_DIV_BY_2 |
6830                         1 << DPIO_CHV_N_DIV_SHIFT);
6831
6832         /* M2 fraction division */
6833         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW2(port), bestm2_frac);
6834
6835         /* M2 fraction division enable */
6836         dpio_val = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW3(port));
6837         dpio_val &= ~(DPIO_CHV_FEEDFWD_GAIN_MASK | DPIO_CHV_FRAC_DIV_EN);
6838         dpio_val |= (2 << DPIO_CHV_FEEDFWD_GAIN_SHIFT);
6839         if (bestm2_frac)
6840                 dpio_val |= DPIO_CHV_FRAC_DIV_EN;
6841         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW3(port), dpio_val);
6842
6843         /* Program digital lock detect threshold */
6844         dpio_val = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW9(port));
6845         dpio_val &= ~(DPIO_CHV_INT_LOCK_THRESHOLD_MASK |
6846                                         DPIO_CHV_INT_LOCK_THRESHOLD_SEL_COARSE);
6847         dpio_val |= (0x5 << DPIO_CHV_INT_LOCK_THRESHOLD_SHIFT);
6848         if (!bestm2_frac)
6849                 dpio_val |= DPIO_CHV_INT_LOCK_THRESHOLD_SEL_COARSE;
6850         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW9(port), dpio_val);
6851
6852         /* Loop filter */
6853         if (vco == 5400000) {
6854                 loopfilter |= (0x3 << DPIO_CHV_PROP_COEFF_SHIFT);
6855                 loopfilter |= (0x8 << DPIO_CHV_INT_COEFF_SHIFT);
6856                 loopfilter |= (0x1 << DPIO_CHV_GAIN_CTRL_SHIFT);
6857                 tribuf_calcntr = 0x9;
6858         } else if (vco <= 6200000) {
6859                 loopfilter |= (0x5 << DPIO_CHV_PROP_COEFF_SHIFT);
6860                 loopfilter |= (0xB << DPIO_CHV_INT_COEFF_SHIFT);
6861                 loopfilter |= (0x3 << DPIO_CHV_GAIN_CTRL_SHIFT);
6862                 tribuf_calcntr = 0x9;
6863         } else if (vco <= 6480000) {
6864                 loopfilter |= (0x4 << DPIO_CHV_PROP_COEFF_SHIFT);
6865                 loopfilter |= (0x9 << DPIO_CHV_INT_COEFF_SHIFT);
6866                 loopfilter |= (0x3 << DPIO_CHV_GAIN_CTRL_SHIFT);
6867                 tribuf_calcntr = 0x8;
6868         } else {
6869                 /* Not supported. Apply the same limits as in the max case */
6870                 loopfilter |= (0x4 << DPIO_CHV_PROP_COEFF_SHIFT);
6871                 loopfilter |= (0x9 << DPIO_CHV_INT_COEFF_SHIFT);
6872                 loopfilter |= (0x3 << DPIO_CHV_GAIN_CTRL_SHIFT);
6873                 tribuf_calcntr = 0;
6874         }
6875         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW6(port), loopfilter);
6876
6877         dpio_val = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW8(port));
6878         dpio_val &= ~DPIO_CHV_TDC_TARGET_CNT_MASK;
6879         dpio_val |= (tribuf_calcntr << DPIO_CHV_TDC_TARGET_CNT_SHIFT);
6880         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW8(port), dpio_val);
6881
6882         /* AFC Recal */
6883         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW14(port),
6884                         vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW14(port)) |
6885                         DPIO_AFC_RECAL);
6886
6887         vlv_dpio_put(dev_priv);
6888 }
6889
6890 /**
6891  * vlv_force_pll_on - forcibly enable just the PLL
6892  * @dev_priv: i915 private structure
6893  * @pipe: pipe PLL to enable
6894  * @dpll: PLL configuration
6895  *
6896  * Enable the PLL for @pipe using the supplied @dpll config. To be used
6897  * in cases where we need the PLL enabled even when @pipe is not going to
6898  * be enabled.
6899  */
6900 int vlv_force_pll_on(struct drm_i915_private *dev_priv, enum pipe pipe,
6901                      const struct dpll *dpll)
6902 {
6903         struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
6904         struct intel_crtc_state *pipe_config;
6905
6906         pipe_config = intel_crtc_state_alloc(crtc);
6907         if (!pipe_config)
6908                 return -ENOMEM;
6909
6910         pipe_config->cpu_transcoder = (enum transcoder)pipe;
6911         pipe_config->pixel_multiplier = 1;
6912         pipe_config->dpll = *dpll;
6913
6914         if (IS_CHERRYVIEW(dev_priv)) {
6915                 chv_compute_dpll(crtc, pipe_config);
6916                 chv_prepare_pll(crtc, pipe_config);
6917                 chv_enable_pll(crtc, pipe_config);
6918         } else {
6919                 vlv_compute_dpll(crtc, pipe_config);
6920                 vlv_prepare_pll(crtc, pipe_config);
6921                 vlv_enable_pll(crtc, pipe_config);
6922         }
6923
6924         kfree(pipe_config);
6925
6926         return 0;
6927 }
6928
6929 /**
6930  * vlv_force_pll_off - forcibly disable just the PLL
6931  * @dev_priv: i915 private structure
6932  * @pipe: pipe PLL to disable
6933  *
6934  * Disable the PLL for @pipe. To be used in cases where we need
6935  * the PLL enabled even when @pipe is not going to be enabled.
6936  */
6937 void vlv_force_pll_off(struct drm_i915_private *dev_priv, enum pipe pipe)
6938 {
6939         if (IS_CHERRYVIEW(dev_priv))
6940                 chv_disable_pll(dev_priv, pipe);
6941         else
6942                 vlv_disable_pll(dev_priv, pipe);
6943 }
6944
6945
6946
6947 static void intel_set_transcoder_timings(const struct intel_crtc_state *crtc_state)
6948 {
6949         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
6950         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6951         enum pipe pipe = crtc->pipe;
6952         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
6953         const struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode;
6954         u32 crtc_vtotal, crtc_vblank_end;
6955         int vsyncshift = 0;
6956
6957         /* We need to be careful not to changed the adjusted mode, for otherwise
6958          * the hw state checker will get angry at the mismatch. */
6959         crtc_vtotal = adjusted_mode->crtc_vtotal;
6960         crtc_vblank_end = adjusted_mode->crtc_vblank_end;
6961
6962         if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) {
6963                 /* the chip adds 2 halflines automatically */
6964                 crtc_vtotal -= 1;
6965                 crtc_vblank_end -= 1;
6966
6967                 if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO))
6968                         vsyncshift = (adjusted_mode->crtc_htotal - 1) / 2;
6969                 else
6970                         vsyncshift = adjusted_mode->crtc_hsync_start -
6971                                 adjusted_mode->crtc_htotal / 2;
6972                 if (vsyncshift < 0)
6973                         vsyncshift += adjusted_mode->crtc_htotal;
6974         }
6975
6976         if (INTEL_GEN(dev_priv) > 3)
6977                 intel_de_write(dev_priv, VSYNCSHIFT(cpu_transcoder),
6978                                vsyncshift);
6979
6980         intel_de_write(dev_priv, HTOTAL(cpu_transcoder),
6981                        (adjusted_mode->crtc_hdisplay - 1) | ((adjusted_mode->crtc_htotal - 1) << 16));
6982         intel_de_write(dev_priv, HBLANK(cpu_transcoder),
6983                        (adjusted_mode->crtc_hblank_start - 1) | ((adjusted_mode->crtc_hblank_end - 1) << 16));
6984         intel_de_write(dev_priv, HSYNC(cpu_transcoder),
6985                        (adjusted_mode->crtc_hsync_start - 1) | ((adjusted_mode->crtc_hsync_end - 1) << 16));
6986
6987         intel_de_write(dev_priv, VTOTAL(cpu_transcoder),
6988                        (adjusted_mode->crtc_vdisplay - 1) | ((crtc_vtotal - 1) << 16));
6989         intel_de_write(dev_priv, VBLANK(cpu_transcoder),
6990                        (adjusted_mode->crtc_vblank_start - 1) | ((crtc_vblank_end - 1) << 16));
6991         intel_de_write(dev_priv, VSYNC(cpu_transcoder),
6992                        (adjusted_mode->crtc_vsync_start - 1) | ((adjusted_mode->crtc_vsync_end - 1) << 16));
6993
6994         /* Workaround: when the EDP input selection is B, the VTOTAL_B must be
6995          * programmed with the VTOTAL_EDP value. Same for VTOTAL_C. This is
6996          * documented on the DDI_FUNC_CTL register description, EDP Input Select
6997          * bits. */
6998         if (IS_HASWELL(dev_priv) && cpu_transcoder == TRANSCODER_EDP &&
6999             (pipe == PIPE_B || pipe == PIPE_C))
7000                 intel_de_write(dev_priv, VTOTAL(pipe),
7001                                intel_de_read(dev_priv, VTOTAL(cpu_transcoder)));
7002
7003 }
7004
7005 static void intel_set_pipe_src_size(const struct intel_crtc_state *crtc_state)
7006 {
7007         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
7008         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7009         enum pipe pipe = crtc->pipe;
7010
7011         /* pipesrc controls the size that is scaled from, which should
7012          * always be the user's requested size.
7013          */
7014         intel_de_write(dev_priv, PIPESRC(pipe),
7015                        ((crtc_state->pipe_src_w - 1) << 16) | (crtc_state->pipe_src_h - 1));
7016 }
7017
7018 static bool intel_pipe_is_interlaced(const struct intel_crtc_state *crtc_state)
7019 {
7020         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
7021         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
7022
7023         if (IS_GEN(dev_priv, 2))
7024                 return false;
7025
7026         if (INTEL_GEN(dev_priv) >= 9 ||
7027             IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
7028                 return intel_de_read(dev_priv, PIPECONF(cpu_transcoder)) & PIPECONF_INTERLACE_MASK_HSW;
7029         else
7030                 return intel_de_read(dev_priv, PIPECONF(cpu_transcoder)) & PIPECONF_INTERLACE_MASK;
7031 }
7032
7033 static void intel_get_transcoder_timings(struct intel_crtc *crtc,
7034                                          struct intel_crtc_state *pipe_config)
7035 {
7036         struct drm_device *dev = crtc->base.dev;
7037         struct drm_i915_private *dev_priv = to_i915(dev);
7038         enum transcoder cpu_transcoder = pipe_config->cpu_transcoder;
7039         u32 tmp;
7040
7041         tmp = intel_de_read(dev_priv, HTOTAL(cpu_transcoder));
7042         pipe_config->hw.adjusted_mode.crtc_hdisplay = (tmp & 0xffff) + 1;
7043         pipe_config->hw.adjusted_mode.crtc_htotal = ((tmp >> 16) & 0xffff) + 1;
7044
7045         if (!transcoder_is_dsi(cpu_transcoder)) {
7046                 tmp = intel_de_read(dev_priv, HBLANK(cpu_transcoder));
7047                 pipe_config->hw.adjusted_mode.crtc_hblank_start =
7048                                                         (tmp & 0xffff) + 1;
7049                 pipe_config->hw.adjusted_mode.crtc_hblank_end =
7050                                                 ((tmp >> 16) & 0xffff) + 1;
7051         }
7052         tmp = intel_de_read(dev_priv, HSYNC(cpu_transcoder));
7053         pipe_config->hw.adjusted_mode.crtc_hsync_start = (tmp & 0xffff) + 1;
7054         pipe_config->hw.adjusted_mode.crtc_hsync_end = ((tmp >> 16) & 0xffff) + 1;
7055
7056         tmp = intel_de_read(dev_priv, VTOTAL(cpu_transcoder));
7057         pipe_config->hw.adjusted_mode.crtc_vdisplay = (tmp & 0xffff) + 1;
7058         pipe_config->hw.adjusted_mode.crtc_vtotal = ((tmp >> 16) & 0xffff) + 1;
7059
7060         if (!transcoder_is_dsi(cpu_transcoder)) {
7061                 tmp = intel_de_read(dev_priv, VBLANK(cpu_transcoder));
7062                 pipe_config->hw.adjusted_mode.crtc_vblank_start =
7063                                                         (tmp & 0xffff) + 1;
7064                 pipe_config->hw.adjusted_mode.crtc_vblank_end =
7065                                                 ((tmp >> 16) & 0xffff) + 1;
7066         }
7067         tmp = intel_de_read(dev_priv, VSYNC(cpu_transcoder));
7068         pipe_config->hw.adjusted_mode.crtc_vsync_start = (tmp & 0xffff) + 1;
7069         pipe_config->hw.adjusted_mode.crtc_vsync_end = ((tmp >> 16) & 0xffff) + 1;
7070
7071         if (intel_pipe_is_interlaced(pipe_config)) {
7072                 pipe_config->hw.adjusted_mode.flags |= DRM_MODE_FLAG_INTERLACE;
7073                 pipe_config->hw.adjusted_mode.crtc_vtotal += 1;
7074                 pipe_config->hw.adjusted_mode.crtc_vblank_end += 1;
7075         }
7076 }
7077
7078 static void intel_get_pipe_src_size(struct intel_crtc *crtc,
7079                                     struct intel_crtc_state *pipe_config)
7080 {
7081         struct drm_device *dev = crtc->base.dev;
7082         struct drm_i915_private *dev_priv = to_i915(dev);
7083         u32 tmp;
7084
7085         tmp = intel_de_read(dev_priv, PIPESRC(crtc->pipe));
7086         pipe_config->pipe_src_h = (tmp & 0xffff) + 1;
7087         pipe_config->pipe_src_w = ((tmp >> 16) & 0xffff) + 1;
7088 }
7089
7090 static void i9xx_set_pipeconf(const struct intel_crtc_state *crtc_state)
7091 {
7092         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
7093         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7094         u32 pipeconf;
7095
7096         pipeconf = 0;
7097
7098         /* we keep both pipes enabled on 830 */
7099         if (IS_I830(dev_priv))
7100                 pipeconf |= intel_de_read(dev_priv, PIPECONF(crtc->pipe)) & PIPECONF_ENABLE;
7101
7102         if (crtc_state->double_wide)
7103                 pipeconf |= PIPECONF_DOUBLE_WIDE;
7104
7105         /* only g4x and later have fancy bpc/dither controls */
7106         if (IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
7107             IS_CHERRYVIEW(dev_priv)) {
7108                 /* Bspec claims that we can't use dithering for 30bpp pipes. */
7109                 if (crtc_state->dither && crtc_state->pipe_bpp != 30)
7110                         pipeconf |= PIPECONF_DITHER_EN |
7111                                     PIPECONF_DITHER_TYPE_SP;
7112
7113                 switch (crtc_state->pipe_bpp) {
7114                 case 18:
7115                         pipeconf |= PIPECONF_6BPC;
7116                         break;
7117                 case 24:
7118                         pipeconf |= PIPECONF_8BPC;
7119                         break;
7120                 case 30:
7121                         pipeconf |= PIPECONF_10BPC;
7122                         break;
7123                 default:
7124                         /* Case prevented by intel_choose_pipe_bpp_dither. */
7125                         BUG();
7126                 }
7127         }
7128
7129         if (crtc_state->hw.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE) {
7130                 if (INTEL_GEN(dev_priv) < 4 ||
7131                     intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO))
7132                         pipeconf |= PIPECONF_INTERLACE_W_FIELD_INDICATION;
7133                 else
7134                         pipeconf |= PIPECONF_INTERLACE_W_SYNC_SHIFT;
7135         } else {
7136                 pipeconf |= PIPECONF_PROGRESSIVE;
7137         }
7138
7139         if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
7140              crtc_state->limited_color_range)
7141                 pipeconf |= PIPECONF_COLOR_RANGE_SELECT;
7142
7143         pipeconf |= PIPECONF_GAMMA_MODE(crtc_state->gamma_mode);
7144
7145         pipeconf |= PIPECONF_FRAME_START_DELAY(dev_priv->framestart_delay - 1);
7146
7147         intel_de_write(dev_priv, PIPECONF(crtc->pipe), pipeconf);
7148         intel_de_posting_read(dev_priv, PIPECONF(crtc->pipe));
7149 }
7150
7151 static bool i9xx_has_pfit(struct drm_i915_private *dev_priv)
7152 {
7153         if (IS_I830(dev_priv))
7154                 return false;
7155
7156         return INTEL_GEN(dev_priv) >= 4 ||
7157                 IS_PINEVIEW(dev_priv) || IS_MOBILE(dev_priv);
7158 }
7159
7160 static void i9xx_get_pfit_config(struct intel_crtc_state *crtc_state)
7161 {
7162         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
7163         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7164         u32 tmp;
7165
7166         if (!i9xx_has_pfit(dev_priv))
7167                 return;
7168
7169         tmp = intel_de_read(dev_priv, PFIT_CONTROL);
7170         if (!(tmp & PFIT_ENABLE))
7171                 return;
7172
7173         /* Check whether the pfit is attached to our pipe. */
7174         if (INTEL_GEN(dev_priv) < 4) {
7175                 if (crtc->pipe != PIPE_B)
7176                         return;
7177         } else {
7178                 if ((tmp & PFIT_PIPE_MASK) != (crtc->pipe << PFIT_PIPE_SHIFT))
7179                         return;
7180         }
7181
7182         crtc_state->gmch_pfit.control = tmp;
7183         crtc_state->gmch_pfit.pgm_ratios =
7184                 intel_de_read(dev_priv, PFIT_PGM_RATIOS);
7185 }
7186
7187 static void vlv_crtc_clock_get(struct intel_crtc *crtc,
7188                                struct intel_crtc_state *pipe_config)
7189 {
7190         struct drm_device *dev = crtc->base.dev;
7191         struct drm_i915_private *dev_priv = to_i915(dev);
7192         enum pipe pipe = crtc->pipe;
7193         struct dpll clock;
7194         u32 mdiv;
7195         int refclk = 100000;
7196
7197         /* In case of DSI, DPLL will not be used */
7198         if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0)
7199                 return;
7200
7201         vlv_dpio_get(dev_priv);
7202         mdiv = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW3(pipe));
7203         vlv_dpio_put(dev_priv);
7204
7205         clock.m1 = (mdiv >> DPIO_M1DIV_SHIFT) & 7;
7206         clock.m2 = mdiv & DPIO_M2DIV_MASK;
7207         clock.n = (mdiv >> DPIO_N_SHIFT) & 0xf;
7208         clock.p1 = (mdiv >> DPIO_P1_SHIFT) & 7;
7209         clock.p2 = (mdiv >> DPIO_P2_SHIFT) & 0x1f;
7210
7211         pipe_config->port_clock = vlv_calc_dpll_params(refclk, &clock);
7212 }
7213
7214 static void
7215 i9xx_get_initial_plane_config(struct intel_crtc *crtc,
7216                               struct intel_initial_plane_config *plane_config)
7217 {
7218         struct drm_device *dev = crtc->base.dev;
7219         struct drm_i915_private *dev_priv = to_i915(dev);
7220         struct intel_plane *plane = to_intel_plane(crtc->base.primary);
7221         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
7222         enum pipe pipe;
7223         u32 val, base, offset;
7224         int fourcc, pixel_format;
7225         unsigned int aligned_height;
7226         struct drm_framebuffer *fb;
7227         struct intel_framebuffer *intel_fb;
7228
7229         if (!plane->get_hw_state(plane, &pipe))
7230                 return;
7231
7232         drm_WARN_ON(dev, pipe != crtc->pipe);
7233
7234         intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL);
7235         if (!intel_fb) {
7236                 drm_dbg_kms(&dev_priv->drm, "failed to alloc fb\n");
7237                 return;
7238         }
7239
7240         fb = &intel_fb->base;
7241
7242         fb->dev = dev;
7243
7244         val = intel_de_read(dev_priv, DSPCNTR(i9xx_plane));
7245
7246         if (INTEL_GEN(dev_priv) >= 4) {
7247                 if (val & DISPPLANE_TILED) {
7248                         plane_config->tiling = I915_TILING_X;
7249                         fb->modifier = I915_FORMAT_MOD_X_TILED;
7250                 }
7251
7252                 if (val & DISPPLANE_ROTATE_180)
7253                         plane_config->rotation = DRM_MODE_ROTATE_180;
7254         }
7255
7256         if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_B &&
7257             val & DISPPLANE_MIRROR)
7258                 plane_config->rotation |= DRM_MODE_REFLECT_X;
7259
7260         pixel_format = val & DISPPLANE_PIXFORMAT_MASK;
7261         fourcc = i9xx_format_to_fourcc(pixel_format);
7262         fb->format = drm_format_info(fourcc);
7263
7264         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
7265                 offset = intel_de_read(dev_priv, DSPOFFSET(i9xx_plane));
7266                 base = intel_de_read(dev_priv, DSPSURF(i9xx_plane)) & 0xfffff000;
7267         } else if (INTEL_GEN(dev_priv) >= 4) {
7268                 if (plane_config->tiling)
7269                         offset = intel_de_read(dev_priv,
7270                                                DSPTILEOFF(i9xx_plane));
7271                 else
7272                         offset = intel_de_read(dev_priv,
7273                                                DSPLINOFF(i9xx_plane));
7274                 base = intel_de_read(dev_priv, DSPSURF(i9xx_plane)) & 0xfffff000;
7275         } else {
7276                 base = intel_de_read(dev_priv, DSPADDR(i9xx_plane));
7277         }
7278         plane_config->base = base;
7279
7280         val = intel_de_read(dev_priv, PIPESRC(pipe));
7281         fb->width = ((val >> 16) & 0xfff) + 1;
7282         fb->height = ((val >> 0) & 0xfff) + 1;
7283
7284         val = intel_de_read(dev_priv, DSPSTRIDE(i9xx_plane));
7285         fb->pitches[0] = val & 0xffffffc0;
7286
7287         aligned_height = intel_fb_align_height(fb, 0, fb->height);
7288
7289         plane_config->size = fb->pitches[0] * aligned_height;
7290
7291         drm_dbg_kms(&dev_priv->drm,
7292                     "%s/%s with fb: size=%dx%d@%d, offset=%x, pitch %d, size 0x%x\n",
7293                     crtc->base.name, plane->base.name, fb->width, fb->height,
7294                     fb->format->cpp[0] * 8, base, fb->pitches[0],
7295                     plane_config->size);
7296
7297         plane_config->fb = intel_fb;
7298 }
7299
7300 static void chv_crtc_clock_get(struct intel_crtc *crtc,
7301                                struct intel_crtc_state *pipe_config)
7302 {
7303         struct drm_device *dev = crtc->base.dev;
7304         struct drm_i915_private *dev_priv = to_i915(dev);
7305         enum pipe pipe = crtc->pipe;
7306         enum dpio_channel port = vlv_pipe_to_channel(pipe);
7307         struct dpll clock;
7308         u32 cmn_dw13, pll_dw0, pll_dw1, pll_dw2, pll_dw3;
7309         int refclk = 100000;
7310
7311         /* In case of DSI, DPLL will not be used */
7312         if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0)
7313                 return;
7314
7315         vlv_dpio_get(dev_priv);
7316         cmn_dw13 = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW13(port));
7317         pll_dw0 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW0(port));
7318         pll_dw1 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW1(port));
7319         pll_dw2 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW2(port));
7320         pll_dw3 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW3(port));
7321         vlv_dpio_put(dev_priv);
7322
7323         clock.m1 = (pll_dw1 & 0x7) == DPIO_CHV_M1_DIV_BY_2 ? 2 : 0;
7324         clock.m2 = (pll_dw0 & 0xff) << 22;
7325         if (pll_dw3 & DPIO_CHV_FRAC_DIV_EN)
7326                 clock.m2 |= pll_dw2 & 0x3fffff;
7327         clock.n = (pll_dw1 >> DPIO_CHV_N_DIV_SHIFT) & 0xf;
7328         clock.p1 = (cmn_dw13 >> DPIO_CHV_P1_DIV_SHIFT) & 0x7;
7329         clock.p2 = (cmn_dw13 >> DPIO_CHV_P2_DIV_SHIFT) & 0x1f;
7330
7331         pipe_config->port_clock = chv_calc_dpll_params(refclk, &clock);
7332 }
7333
7334 static enum intel_output_format
7335 bdw_get_pipemisc_output_format(struct intel_crtc *crtc)
7336 {
7337         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7338         u32 tmp;
7339
7340         tmp = intel_de_read(dev_priv, PIPEMISC(crtc->pipe));
7341
7342         if (tmp & PIPEMISC_YUV420_ENABLE) {
7343                 /* We support 4:2:0 in full blend mode only */
7344                 drm_WARN_ON(&dev_priv->drm,
7345                             (tmp & PIPEMISC_YUV420_MODE_FULL_BLEND) == 0);
7346
7347                 return INTEL_OUTPUT_FORMAT_YCBCR420;
7348         } else if (tmp & PIPEMISC_OUTPUT_COLORSPACE_YUV) {
7349                 return INTEL_OUTPUT_FORMAT_YCBCR444;
7350         } else {
7351                 return INTEL_OUTPUT_FORMAT_RGB;
7352         }
7353 }
7354
7355 static void i9xx_get_pipe_color_config(struct intel_crtc_state *crtc_state)
7356 {
7357         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
7358         struct intel_plane *plane = to_intel_plane(crtc->base.primary);
7359         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7360         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
7361         u32 tmp;
7362
7363         tmp = intel_de_read(dev_priv, DSPCNTR(i9xx_plane));
7364
7365         if (tmp & DISPPLANE_GAMMA_ENABLE)
7366                 crtc_state->gamma_enable = true;
7367
7368         if (!HAS_GMCH(dev_priv) &&
7369             tmp & DISPPLANE_PIPE_CSC_ENABLE)
7370                 crtc_state->csc_enable = true;
7371 }
7372
7373 static bool i9xx_get_pipe_config(struct intel_crtc *crtc,
7374                                  struct intel_crtc_state *pipe_config)
7375 {
7376         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7377         enum intel_display_power_domain power_domain;
7378         intel_wakeref_t wakeref;
7379         u32 tmp;
7380         bool ret;
7381
7382         power_domain = POWER_DOMAIN_PIPE(crtc->pipe);
7383         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
7384         if (!wakeref)
7385                 return false;
7386
7387         pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB;
7388         pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe;
7389         pipe_config->shared_dpll = NULL;
7390
7391         ret = false;
7392
7393         tmp = intel_de_read(dev_priv, PIPECONF(crtc->pipe));
7394         if (!(tmp & PIPECONF_ENABLE))
7395                 goto out;
7396
7397         if (IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
7398             IS_CHERRYVIEW(dev_priv)) {
7399                 switch (tmp & PIPECONF_BPC_MASK) {
7400                 case PIPECONF_6BPC:
7401                         pipe_config->pipe_bpp = 18;
7402                         break;
7403                 case PIPECONF_8BPC:
7404                         pipe_config->pipe_bpp = 24;
7405                         break;
7406                 case PIPECONF_10BPC:
7407                         pipe_config->pipe_bpp = 30;
7408                         break;
7409                 default:
7410                         break;
7411                 }
7412         }
7413
7414         if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
7415             (tmp & PIPECONF_COLOR_RANGE_SELECT))
7416                 pipe_config->limited_color_range = true;
7417
7418         pipe_config->gamma_mode = (tmp & PIPECONF_GAMMA_MODE_MASK_I9XX) >>
7419                 PIPECONF_GAMMA_MODE_SHIFT;
7420
7421         if (IS_CHERRYVIEW(dev_priv))
7422                 pipe_config->cgm_mode = intel_de_read(dev_priv,
7423                                                       CGM_PIPE_MODE(crtc->pipe));
7424
7425         i9xx_get_pipe_color_config(pipe_config);
7426         intel_color_get_config(pipe_config);
7427
7428         if (INTEL_GEN(dev_priv) < 4)
7429                 pipe_config->double_wide = tmp & PIPECONF_DOUBLE_WIDE;
7430
7431         intel_get_transcoder_timings(crtc, pipe_config);
7432         intel_get_pipe_src_size(crtc, pipe_config);
7433
7434         i9xx_get_pfit_config(pipe_config);
7435
7436         if (INTEL_GEN(dev_priv) >= 4) {
7437                 /* No way to read it out on pipes B and C */
7438                 if (IS_CHERRYVIEW(dev_priv) && crtc->pipe != PIPE_A)
7439                         tmp = dev_priv->chv_dpll_md[crtc->pipe];
7440                 else
7441                         tmp = intel_de_read(dev_priv, DPLL_MD(crtc->pipe));
7442                 pipe_config->pixel_multiplier =
7443                         ((tmp & DPLL_MD_UDI_MULTIPLIER_MASK)
7444                          >> DPLL_MD_UDI_MULTIPLIER_SHIFT) + 1;
7445                 pipe_config->dpll_hw_state.dpll_md = tmp;
7446         } else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv) ||
7447                    IS_G33(dev_priv) || IS_PINEVIEW(dev_priv)) {
7448                 tmp = intel_de_read(dev_priv, DPLL(crtc->pipe));
7449                 pipe_config->pixel_multiplier =
7450                         ((tmp & SDVO_MULTIPLIER_MASK)
7451                          >> SDVO_MULTIPLIER_SHIFT_HIRES) + 1;
7452         } else {
7453                 /* Note that on i915G/GM the pixel multiplier is in the sdvo
7454                  * port and will be fixed up in the encoder->get_config
7455                  * function. */
7456                 pipe_config->pixel_multiplier = 1;
7457         }
7458         pipe_config->dpll_hw_state.dpll = intel_de_read(dev_priv,
7459                                                         DPLL(crtc->pipe));
7460         if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv)) {
7461                 pipe_config->dpll_hw_state.fp0 = intel_de_read(dev_priv,
7462                                                                FP0(crtc->pipe));
7463                 pipe_config->dpll_hw_state.fp1 = intel_de_read(dev_priv,
7464                                                                FP1(crtc->pipe));
7465         } else {
7466                 /* Mask out read-only status bits. */
7467                 pipe_config->dpll_hw_state.dpll &= ~(DPLL_LOCK_VLV |
7468                                                      DPLL_PORTC_READY_MASK |
7469                                                      DPLL_PORTB_READY_MASK);
7470         }
7471
7472         if (IS_CHERRYVIEW(dev_priv))
7473                 chv_crtc_clock_get(crtc, pipe_config);
7474         else if (IS_VALLEYVIEW(dev_priv))
7475                 vlv_crtc_clock_get(crtc, pipe_config);
7476         else
7477                 i9xx_crtc_clock_get(crtc, pipe_config);
7478
7479         /*
7480          * Normally the dotclock is filled in by the encoder .get_config()
7481          * but in case the pipe is enabled w/o any ports we need a sane
7482          * default.
7483          */
7484         pipe_config->hw.adjusted_mode.crtc_clock =
7485                 pipe_config->port_clock / pipe_config->pixel_multiplier;
7486
7487         ret = true;
7488
7489 out:
7490         intel_display_power_put(dev_priv, power_domain, wakeref);
7491
7492         return ret;
7493 }
7494
7495 static void ilk_init_pch_refclk(struct drm_i915_private *dev_priv)
7496 {
7497         struct intel_encoder *encoder;
7498         int i;
7499         u32 val, final;
7500         bool has_lvds = false;
7501         bool has_cpu_edp = false;
7502         bool has_panel = false;
7503         bool has_ck505 = false;
7504         bool can_ssc = false;
7505         bool using_ssc_source = false;
7506
7507         /* We need to take the global config into account */
7508         for_each_intel_encoder(&dev_priv->drm, encoder) {
7509                 switch (encoder->type) {
7510                 case INTEL_OUTPUT_LVDS:
7511                         has_panel = true;
7512                         has_lvds = true;
7513                         break;
7514                 case INTEL_OUTPUT_EDP:
7515                         has_panel = true;
7516                         if (encoder->port == PORT_A)
7517                                 has_cpu_edp = true;
7518                         break;
7519                 default:
7520                         break;
7521                 }
7522         }
7523
7524         if (HAS_PCH_IBX(dev_priv)) {
7525                 has_ck505 = dev_priv->vbt.display_clock_mode;
7526                 can_ssc = has_ck505;
7527         } else {
7528                 has_ck505 = false;
7529                 can_ssc = true;
7530         }
7531
7532         /* Check if any DPLLs are using the SSC source */
7533         for (i = 0; i < dev_priv->dpll.num_shared_dpll; i++) {
7534                 u32 temp = intel_de_read(dev_priv, PCH_DPLL(i));
7535
7536                 if (!(temp & DPLL_VCO_ENABLE))
7537                         continue;
7538
7539                 if ((temp & PLL_REF_INPUT_MASK) ==
7540                     PLLB_REF_INPUT_SPREADSPECTRUMIN) {
7541                         using_ssc_source = true;
7542                         break;
7543                 }
7544         }
7545
7546         drm_dbg_kms(&dev_priv->drm,
7547                     "has_panel %d has_lvds %d has_ck505 %d using_ssc_source %d\n",
7548                     has_panel, has_lvds, has_ck505, using_ssc_source);
7549
7550         /* Ironlake: try to setup display ref clock before DPLL
7551          * enabling. This is only under driver's control after
7552          * PCH B stepping, previous chipset stepping should be
7553          * ignoring this setting.
7554          */
7555         val = intel_de_read(dev_priv, PCH_DREF_CONTROL);
7556
7557         /* As we must carefully and slowly disable/enable each source in turn,
7558          * compute the final state we want first and check if we need to
7559          * make any changes at all.
7560          */
7561         final = val;
7562         final &= ~DREF_NONSPREAD_SOURCE_MASK;
7563         if (has_ck505)
7564                 final |= DREF_NONSPREAD_CK505_ENABLE;
7565         else
7566                 final |= DREF_NONSPREAD_SOURCE_ENABLE;
7567
7568         final &= ~DREF_SSC_SOURCE_MASK;
7569         final &= ~DREF_CPU_SOURCE_OUTPUT_MASK;
7570         final &= ~DREF_SSC1_ENABLE;
7571
7572         if (has_panel) {
7573                 final |= DREF_SSC_SOURCE_ENABLE;
7574
7575                 if (intel_panel_use_ssc(dev_priv) && can_ssc)
7576                         final |= DREF_SSC1_ENABLE;
7577
7578                 if (has_cpu_edp) {
7579                         if (intel_panel_use_ssc(dev_priv) && can_ssc)
7580                                 final |= DREF_CPU_SOURCE_OUTPUT_DOWNSPREAD;
7581                         else
7582                                 final |= DREF_CPU_SOURCE_OUTPUT_NONSPREAD;
7583                 } else
7584                         final |= DREF_CPU_SOURCE_OUTPUT_DISABLE;
7585         } else if (using_ssc_source) {
7586                 final |= DREF_SSC_SOURCE_ENABLE;
7587                 final |= DREF_SSC1_ENABLE;
7588         }
7589
7590         if (final == val)
7591                 return;
7592
7593         /* Always enable nonspread source */
7594         val &= ~DREF_NONSPREAD_SOURCE_MASK;
7595
7596         if (has_ck505)
7597                 val |= DREF_NONSPREAD_CK505_ENABLE;
7598         else
7599                 val |= DREF_NONSPREAD_SOURCE_ENABLE;
7600
7601         if (has_panel) {
7602                 val &= ~DREF_SSC_SOURCE_MASK;
7603                 val |= DREF_SSC_SOURCE_ENABLE;
7604
7605                 /* SSC must be turned on before enabling the CPU output  */
7606                 if (intel_panel_use_ssc(dev_priv) && can_ssc) {
7607                         drm_dbg_kms(&dev_priv->drm, "Using SSC on panel\n");
7608                         val |= DREF_SSC1_ENABLE;
7609                 } else
7610                         val &= ~DREF_SSC1_ENABLE;
7611
7612                 /* Get SSC going before enabling the outputs */
7613                 intel_de_write(dev_priv, PCH_DREF_CONTROL, val);
7614                 intel_de_posting_read(dev_priv, PCH_DREF_CONTROL);
7615                 udelay(200);
7616
7617                 val &= ~DREF_CPU_SOURCE_OUTPUT_MASK;
7618
7619                 /* Enable CPU source on CPU attached eDP */
7620                 if (has_cpu_edp) {
7621                         if (intel_panel_use_ssc(dev_priv) && can_ssc) {
7622                                 drm_dbg_kms(&dev_priv->drm,
7623                                             "Using SSC on eDP\n");
7624                                 val |= DREF_CPU_SOURCE_OUTPUT_DOWNSPREAD;
7625                         } else
7626                                 val |= DREF_CPU_SOURCE_OUTPUT_NONSPREAD;
7627                 } else
7628                         val |= DREF_CPU_SOURCE_OUTPUT_DISABLE;
7629
7630                 intel_de_write(dev_priv, PCH_DREF_CONTROL, val);
7631                 intel_de_posting_read(dev_priv, PCH_DREF_CONTROL);
7632                 udelay(200);
7633         } else {
7634                 drm_dbg_kms(&dev_priv->drm, "Disabling CPU source output\n");
7635
7636                 val &= ~DREF_CPU_SOURCE_OUTPUT_MASK;
7637
7638                 /* Turn off CPU output */
7639                 val |= DREF_CPU_SOURCE_OUTPUT_DISABLE;
7640
7641                 intel_de_write(dev_priv, PCH_DREF_CONTROL, val);
7642                 intel_de_posting_read(dev_priv, PCH_DREF_CONTROL);
7643                 udelay(200);
7644
7645                 if (!using_ssc_source) {
7646                         drm_dbg_kms(&dev_priv->drm, "Disabling SSC source\n");
7647
7648                         /* Turn off the SSC source */
7649                         val &= ~DREF_SSC_SOURCE_MASK;
7650                         val |= DREF_SSC_SOURCE_DISABLE;
7651
7652                         /* Turn off SSC1 */
7653                         val &= ~DREF_SSC1_ENABLE;
7654
7655                         intel_de_write(dev_priv, PCH_DREF_CONTROL, val);
7656                         intel_de_posting_read(dev_priv, PCH_DREF_CONTROL);
7657                         udelay(200);
7658                 }
7659         }
7660
7661         BUG_ON(val != final);
7662 }
7663
7664 static void lpt_reset_fdi_mphy(struct drm_i915_private *dev_priv)
7665 {
7666         u32 tmp;
7667
7668         tmp = intel_de_read(dev_priv, SOUTH_CHICKEN2);
7669         tmp |= FDI_MPHY_IOSFSB_RESET_CTL;
7670         intel_de_write(dev_priv, SOUTH_CHICKEN2, tmp);
7671
7672         if (wait_for_us(intel_de_read(dev_priv, SOUTH_CHICKEN2) &
7673                         FDI_MPHY_IOSFSB_RESET_STATUS, 100))
7674                 drm_err(&dev_priv->drm, "FDI mPHY reset assert timeout\n");
7675
7676         tmp = intel_de_read(dev_priv, SOUTH_CHICKEN2);
7677         tmp &= ~FDI_MPHY_IOSFSB_RESET_CTL;
7678         intel_de_write(dev_priv, SOUTH_CHICKEN2, tmp);
7679
7680         if (wait_for_us((intel_de_read(dev_priv, SOUTH_CHICKEN2) &
7681                          FDI_MPHY_IOSFSB_RESET_STATUS) == 0, 100))
7682                 drm_err(&dev_priv->drm, "FDI mPHY reset de-assert timeout\n");
7683 }
7684
7685 /* WaMPhyProgramming:hsw */
7686 static void lpt_program_fdi_mphy(struct drm_i915_private *dev_priv)
7687 {
7688         u32 tmp;
7689
7690         tmp = intel_sbi_read(dev_priv, 0x8008, SBI_MPHY);
7691         tmp &= ~(0xFF << 24);
7692         tmp |= (0x12 << 24);
7693         intel_sbi_write(dev_priv, 0x8008, tmp, SBI_MPHY);
7694
7695         tmp = intel_sbi_read(dev_priv, 0x2008, SBI_MPHY);
7696         tmp |= (1 << 11);
7697         intel_sbi_write(dev_priv, 0x2008, tmp, SBI_MPHY);
7698
7699         tmp = intel_sbi_read(dev_priv, 0x2108, SBI_MPHY);
7700         tmp |= (1 << 11);
7701         intel_sbi_write(dev_priv, 0x2108, tmp, SBI_MPHY);
7702
7703         tmp = intel_sbi_read(dev_priv, 0x206C, SBI_MPHY);
7704         tmp |= (1 << 24) | (1 << 21) | (1 << 18);
7705         intel_sbi_write(dev_priv, 0x206C, tmp, SBI_MPHY);
7706
7707         tmp = intel_sbi_read(dev_priv, 0x216C, SBI_MPHY);
7708         tmp |= (1 << 24) | (1 << 21) | (1 << 18);
7709         intel_sbi_write(dev_priv, 0x216C, tmp, SBI_MPHY);
7710
7711         tmp = intel_sbi_read(dev_priv, 0x2080, SBI_MPHY);
7712         tmp &= ~(7 << 13);
7713         tmp |= (5 << 13);
7714         intel_sbi_write(dev_priv, 0x2080, tmp, SBI_MPHY);
7715
7716         tmp = intel_sbi_read(dev_priv, 0x2180, SBI_MPHY);
7717         tmp &= ~(7 << 13);
7718         tmp |= (5 << 13);
7719         intel_sbi_write(dev_priv, 0x2180, tmp, SBI_MPHY);
7720
7721         tmp = intel_sbi_read(dev_priv, 0x208C, SBI_MPHY);
7722         tmp &= ~0xFF;
7723         tmp |= 0x1C;
7724         intel_sbi_write(dev_priv, 0x208C, tmp, SBI_MPHY);
7725
7726         tmp = intel_sbi_read(dev_priv, 0x218C, SBI_MPHY);
7727         tmp &= ~0xFF;
7728         tmp |= 0x1C;
7729         intel_sbi_write(dev_priv, 0x218C, tmp, SBI_MPHY);
7730
7731         tmp = intel_sbi_read(dev_priv, 0x2098, SBI_MPHY);
7732         tmp &= ~(0xFF << 16);
7733         tmp |= (0x1C << 16);
7734         intel_sbi_write(dev_priv, 0x2098, tmp, SBI_MPHY);
7735
7736         tmp = intel_sbi_read(dev_priv, 0x2198, SBI_MPHY);
7737         tmp &= ~(0xFF << 16);
7738         tmp |= (0x1C << 16);
7739         intel_sbi_write(dev_priv, 0x2198, tmp, SBI_MPHY);
7740
7741         tmp = intel_sbi_read(dev_priv, 0x20C4, SBI_MPHY);
7742         tmp |= (1 << 27);
7743         intel_sbi_write(dev_priv, 0x20C4, tmp, SBI_MPHY);
7744
7745         tmp = intel_sbi_read(dev_priv, 0x21C4, SBI_MPHY);
7746         tmp |= (1 << 27);
7747         intel_sbi_write(dev_priv, 0x21C4, tmp, SBI_MPHY);
7748
7749         tmp = intel_sbi_read(dev_priv, 0x20EC, SBI_MPHY);
7750         tmp &= ~(0xF << 28);
7751         tmp |= (4 << 28);
7752         intel_sbi_write(dev_priv, 0x20EC, tmp, SBI_MPHY);
7753
7754         tmp = intel_sbi_read(dev_priv, 0x21EC, SBI_MPHY);
7755         tmp &= ~(0xF << 28);
7756         tmp |= (4 << 28);
7757         intel_sbi_write(dev_priv, 0x21EC, tmp, SBI_MPHY);
7758 }
7759
7760 /* Implements 3 different sequences from BSpec chapter "Display iCLK
7761  * Programming" based on the parameters passed:
7762  * - Sequence to enable CLKOUT_DP
7763  * - Sequence to enable CLKOUT_DP without spread
7764  * - Sequence to enable CLKOUT_DP for FDI usage and configure PCH FDI I/O
7765  */
7766 static void lpt_enable_clkout_dp(struct drm_i915_private *dev_priv,
7767                                  bool with_spread, bool with_fdi)
7768 {
7769         u32 reg, tmp;
7770
7771         if (drm_WARN(&dev_priv->drm, with_fdi && !with_spread,
7772                      "FDI requires downspread\n"))
7773                 with_spread = true;
7774         if (drm_WARN(&dev_priv->drm, HAS_PCH_LPT_LP(dev_priv) &&
7775                      with_fdi, "LP PCH doesn't have FDI\n"))
7776                 with_fdi = false;
7777
7778         mutex_lock(&dev_priv->sb_lock);
7779
7780         tmp = intel_sbi_read(dev_priv, SBI_SSCCTL, SBI_ICLK);
7781         tmp &= ~SBI_SSCCTL_DISABLE;
7782         tmp |= SBI_SSCCTL_PATHALT;
7783         intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
7784
7785         udelay(24);
7786
7787         if (with_spread) {
7788                 tmp = intel_sbi_read(dev_priv, SBI_SSCCTL, SBI_ICLK);
7789                 tmp &= ~SBI_SSCCTL_PATHALT;
7790                 intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
7791
7792                 if (with_fdi) {
7793                         lpt_reset_fdi_mphy(dev_priv);
7794                         lpt_program_fdi_mphy(dev_priv);
7795                 }
7796         }
7797
7798         reg = HAS_PCH_LPT_LP(dev_priv) ? SBI_GEN0 : SBI_DBUFF0;
7799         tmp = intel_sbi_read(dev_priv, reg, SBI_ICLK);
7800         tmp |= SBI_GEN0_CFG_BUFFENABLE_DISABLE;
7801         intel_sbi_write(dev_priv, reg, tmp, SBI_ICLK);
7802
7803         mutex_unlock(&dev_priv->sb_lock);
7804 }
7805
7806 /* Sequence to disable CLKOUT_DP */
7807 void lpt_disable_clkout_dp(struct drm_i915_private *dev_priv)
7808 {
7809         u32 reg, tmp;
7810
7811         mutex_lock(&dev_priv->sb_lock);
7812
7813         reg = HAS_PCH_LPT_LP(dev_priv) ? SBI_GEN0 : SBI_DBUFF0;
7814         tmp = intel_sbi_read(dev_priv, reg, SBI_ICLK);
7815         tmp &= ~SBI_GEN0_CFG_BUFFENABLE_DISABLE;
7816         intel_sbi_write(dev_priv, reg, tmp, SBI_ICLK);
7817
7818         tmp = intel_sbi_read(dev_priv, SBI_SSCCTL, SBI_ICLK);
7819         if (!(tmp & SBI_SSCCTL_DISABLE)) {
7820                 if (!(tmp & SBI_SSCCTL_PATHALT)) {
7821                         tmp |= SBI_SSCCTL_PATHALT;
7822                         intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
7823                         udelay(32);
7824                 }
7825                 tmp |= SBI_SSCCTL_DISABLE;
7826                 intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
7827         }
7828
7829         mutex_unlock(&dev_priv->sb_lock);
7830 }
7831
7832 #define BEND_IDX(steps) ((50 + (steps)) / 5)
7833
7834 static const u16 sscdivintphase[] = {
7835         [BEND_IDX( 50)] = 0x3B23,
7836         [BEND_IDX( 45)] = 0x3B23,
7837         [BEND_IDX( 40)] = 0x3C23,
7838         [BEND_IDX( 35)] = 0x3C23,
7839         [BEND_IDX( 30)] = 0x3D23,
7840         [BEND_IDX( 25)] = 0x3D23,
7841         [BEND_IDX( 20)] = 0x3E23,
7842         [BEND_IDX( 15)] = 0x3E23,
7843         [BEND_IDX( 10)] = 0x3F23,
7844         [BEND_IDX(  5)] = 0x3F23,
7845         [BEND_IDX(  0)] = 0x0025,
7846         [BEND_IDX( -5)] = 0x0025,
7847         [BEND_IDX(-10)] = 0x0125,
7848         [BEND_IDX(-15)] = 0x0125,
7849         [BEND_IDX(-20)] = 0x0225,
7850         [BEND_IDX(-25)] = 0x0225,
7851         [BEND_IDX(-30)] = 0x0325,
7852         [BEND_IDX(-35)] = 0x0325,
7853         [BEND_IDX(-40)] = 0x0425,
7854         [BEND_IDX(-45)] = 0x0425,
7855         [BEND_IDX(-50)] = 0x0525,
7856 };
7857
7858 /*
7859  * Bend CLKOUT_DP
7860  * steps -50 to 50 inclusive, in steps of 5
7861  * < 0 slow down the clock, > 0 speed up the clock, 0 == no bend (135MHz)
7862  * change in clock period = -(steps / 10) * 5.787 ps
7863  */
7864 static void lpt_bend_clkout_dp(struct drm_i915_private *dev_priv, int steps)
7865 {
7866         u32 tmp;
7867         int idx = BEND_IDX(steps);
7868
7869         if (drm_WARN_ON(&dev_priv->drm, steps % 5 != 0))
7870                 return;
7871
7872         if (drm_WARN_ON(&dev_priv->drm, idx >= ARRAY_SIZE(sscdivintphase)))
7873                 return;
7874
7875         mutex_lock(&dev_priv->sb_lock);
7876
7877         if (steps % 10 != 0)
7878                 tmp = 0xAAAAAAAB;
7879         else
7880                 tmp = 0x00000000;
7881         intel_sbi_write(dev_priv, SBI_SSCDITHPHASE, tmp, SBI_ICLK);
7882
7883         tmp = intel_sbi_read(dev_priv, SBI_SSCDIVINTPHASE, SBI_ICLK);
7884         tmp &= 0xffff0000;
7885         tmp |= sscdivintphase[idx];
7886         intel_sbi_write(dev_priv, SBI_SSCDIVINTPHASE, tmp, SBI_ICLK);
7887
7888         mutex_unlock(&dev_priv->sb_lock);
7889 }
7890
7891 #undef BEND_IDX
7892
7893 static bool spll_uses_pch_ssc(struct drm_i915_private *dev_priv)
7894 {
7895         u32 fuse_strap = intel_de_read(dev_priv, FUSE_STRAP);
7896         u32 ctl = intel_de_read(dev_priv, SPLL_CTL);
7897
7898         if ((ctl & SPLL_PLL_ENABLE) == 0)
7899                 return false;
7900
7901         if ((ctl & SPLL_REF_MASK) == SPLL_REF_MUXED_SSC &&
7902             (fuse_strap & HSW_CPU_SSC_ENABLE) == 0)
7903                 return true;
7904
7905         if (IS_BROADWELL(dev_priv) &&
7906             (ctl & SPLL_REF_MASK) == SPLL_REF_PCH_SSC_BDW)
7907                 return true;
7908
7909         return false;
7910 }
7911
7912 static bool wrpll_uses_pch_ssc(struct drm_i915_private *dev_priv,
7913                                enum intel_dpll_id id)
7914 {
7915         u32 fuse_strap = intel_de_read(dev_priv, FUSE_STRAP);
7916         u32 ctl = intel_de_read(dev_priv, WRPLL_CTL(id));
7917
7918         if ((ctl & WRPLL_PLL_ENABLE) == 0)
7919                 return false;
7920
7921         if ((ctl & WRPLL_REF_MASK) == WRPLL_REF_PCH_SSC)
7922                 return true;
7923
7924         if ((IS_BROADWELL(dev_priv) || IS_HSW_ULT(dev_priv)) &&
7925             (ctl & WRPLL_REF_MASK) == WRPLL_REF_MUXED_SSC_BDW &&
7926             (fuse_strap & HSW_CPU_SSC_ENABLE) == 0)
7927                 return true;
7928
7929         return false;
7930 }
7931
7932 static void lpt_init_pch_refclk(struct drm_i915_private *dev_priv)
7933 {
7934         struct intel_encoder *encoder;
7935         bool has_fdi = false;
7936
7937         for_each_intel_encoder(&dev_priv->drm, encoder) {
7938                 switch (encoder->type) {
7939                 case INTEL_OUTPUT_ANALOG:
7940                         has_fdi = true;
7941                         break;
7942                 default:
7943                         break;
7944                 }
7945         }
7946
7947         /*
7948          * The BIOS may have decided to use the PCH SSC
7949          * reference so we must not disable it until the
7950          * relevant PLLs have stopped relying on it. We'll
7951          * just leave the PCH SSC reference enabled in case
7952          * any active PLL is using it. It will get disabled
7953          * after runtime suspend if we don't have FDI.
7954          *
7955          * TODO: Move the whole reference clock handling
7956          * to the modeset sequence proper so that we can
7957          * actually enable/disable/reconfigure these things
7958          * safely. To do that we need to introduce a real
7959          * clock hierarchy. That would also allow us to do
7960          * clock bending finally.
7961          */
7962         dev_priv->pch_ssc_use = 0;
7963
7964         if (spll_uses_pch_ssc(dev_priv)) {
7965                 drm_dbg_kms(&dev_priv->drm, "SPLL using PCH SSC\n");
7966                 dev_priv->pch_ssc_use |= BIT(DPLL_ID_SPLL);
7967         }
7968
7969         if (wrpll_uses_pch_ssc(dev_priv, DPLL_ID_WRPLL1)) {
7970                 drm_dbg_kms(&dev_priv->drm, "WRPLL1 using PCH SSC\n");
7971                 dev_priv->pch_ssc_use |= BIT(DPLL_ID_WRPLL1);
7972         }
7973
7974         if (wrpll_uses_pch_ssc(dev_priv, DPLL_ID_WRPLL2)) {
7975                 drm_dbg_kms(&dev_priv->drm, "WRPLL2 using PCH SSC\n");
7976                 dev_priv->pch_ssc_use |= BIT(DPLL_ID_WRPLL2);
7977         }
7978
7979         if (dev_priv->pch_ssc_use)
7980                 return;
7981
7982         if (has_fdi) {
7983                 lpt_bend_clkout_dp(dev_priv, 0);
7984                 lpt_enable_clkout_dp(dev_priv, true, true);
7985         } else {
7986                 lpt_disable_clkout_dp(dev_priv);
7987         }
7988 }
7989
7990 /*
7991  * Initialize reference clocks when the driver loads
7992  */
7993 void intel_init_pch_refclk(struct drm_i915_private *dev_priv)
7994 {
7995         if (HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv))
7996                 ilk_init_pch_refclk(dev_priv);
7997         else if (HAS_PCH_LPT(dev_priv))
7998                 lpt_init_pch_refclk(dev_priv);
7999 }
8000
8001 static void ilk_set_pipeconf(const struct intel_crtc_state *crtc_state)
8002 {
8003         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
8004         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8005         enum pipe pipe = crtc->pipe;
8006         u32 val;
8007
8008         val = 0;
8009
8010         switch (crtc_state->pipe_bpp) {
8011         case 18:
8012                 val |= PIPECONF_6BPC;
8013                 break;
8014         case 24:
8015                 val |= PIPECONF_8BPC;
8016                 break;
8017         case 30:
8018                 val |= PIPECONF_10BPC;
8019                 break;
8020         case 36:
8021                 val |= PIPECONF_12BPC;
8022                 break;
8023         default:
8024                 /* Case prevented by intel_choose_pipe_bpp_dither. */
8025                 BUG();
8026         }
8027
8028         if (crtc_state->dither)
8029                 val |= (PIPECONF_DITHER_EN | PIPECONF_DITHER_TYPE_SP);
8030
8031         if (crtc_state->hw.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE)
8032                 val |= PIPECONF_INTERLACED_ILK;
8033         else
8034                 val |= PIPECONF_PROGRESSIVE;
8035
8036         /*
8037          * This would end up with an odd purple hue over
8038          * the entire display. Make sure we don't do it.
8039          */
8040         drm_WARN_ON(&dev_priv->drm, crtc_state->limited_color_range &&
8041                     crtc_state->output_format != INTEL_OUTPUT_FORMAT_RGB);
8042
8043         if (crtc_state->limited_color_range &&
8044             !intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO))
8045                 val |= PIPECONF_COLOR_RANGE_SELECT;
8046
8047         if (crtc_state->output_format != INTEL_OUTPUT_FORMAT_RGB)
8048                 val |= PIPECONF_OUTPUT_COLORSPACE_YUV709;
8049
8050         val |= PIPECONF_GAMMA_MODE(crtc_state->gamma_mode);
8051
8052         val |= PIPECONF_FRAME_START_DELAY(dev_priv->framestart_delay - 1);
8053
8054         intel_de_write(dev_priv, PIPECONF(pipe), val);
8055         intel_de_posting_read(dev_priv, PIPECONF(pipe));
8056 }
8057
8058 static void hsw_set_pipeconf(const struct intel_crtc_state *crtc_state)
8059 {
8060         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
8061         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8062         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
8063         u32 val = 0;
8064
8065         if (IS_HASWELL(dev_priv) && crtc_state->dither)
8066                 val |= (PIPECONF_DITHER_EN | PIPECONF_DITHER_TYPE_SP);
8067
8068         if (crtc_state->hw.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE)
8069                 val |= PIPECONF_INTERLACED_ILK;
8070         else
8071                 val |= PIPECONF_PROGRESSIVE;
8072
8073         if (IS_HASWELL(dev_priv) &&
8074             crtc_state->output_format != INTEL_OUTPUT_FORMAT_RGB)
8075                 val |= PIPECONF_OUTPUT_COLORSPACE_YUV_HSW;
8076
8077         intel_de_write(dev_priv, PIPECONF(cpu_transcoder), val);
8078         intel_de_posting_read(dev_priv, PIPECONF(cpu_transcoder));
8079 }
8080
8081 static void bdw_set_pipemisc(const struct intel_crtc_state *crtc_state)
8082 {
8083         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
8084         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8085         u32 val = 0;
8086
8087         switch (crtc_state->pipe_bpp) {
8088         case 18:
8089                 val |= PIPEMISC_DITHER_6_BPC;
8090                 break;
8091         case 24:
8092                 val |= PIPEMISC_DITHER_8_BPC;
8093                 break;
8094         case 30:
8095                 val |= PIPEMISC_DITHER_10_BPC;
8096                 break;
8097         case 36:
8098                 val |= PIPEMISC_DITHER_12_BPC;
8099                 break;
8100         default:
8101                 MISSING_CASE(crtc_state->pipe_bpp);
8102                 break;
8103         }
8104
8105         if (crtc_state->dither)
8106                 val |= PIPEMISC_DITHER_ENABLE | PIPEMISC_DITHER_TYPE_SP;
8107
8108         if (crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR420 ||
8109             crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR444)
8110                 val |= PIPEMISC_OUTPUT_COLORSPACE_YUV;
8111
8112         if (crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR420)
8113                 val |= PIPEMISC_YUV420_ENABLE |
8114                         PIPEMISC_YUV420_MODE_FULL_BLEND;
8115
8116         if (INTEL_GEN(dev_priv) >= 11 &&
8117             (crtc_state->active_planes & ~(icl_hdr_plane_mask() |
8118                                            BIT(PLANE_CURSOR))) == 0)
8119                 val |= PIPEMISC_HDR_MODE_PRECISION;
8120
8121         if (INTEL_GEN(dev_priv) >= 12)
8122                 val |= PIPEMISC_PIXEL_ROUNDING_TRUNC;
8123
8124         intel_de_write(dev_priv, PIPEMISC(crtc->pipe), val);
8125 }
8126
8127 int bdw_get_pipemisc_bpp(struct intel_crtc *crtc)
8128 {
8129         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8130         u32 tmp;
8131
8132         tmp = intel_de_read(dev_priv, PIPEMISC(crtc->pipe));
8133
8134         switch (tmp & PIPEMISC_DITHER_BPC_MASK) {
8135         case PIPEMISC_DITHER_6_BPC:
8136                 return 18;
8137         case PIPEMISC_DITHER_8_BPC:
8138                 return 24;
8139         case PIPEMISC_DITHER_10_BPC:
8140                 return 30;
8141         case PIPEMISC_DITHER_12_BPC:
8142                 return 36;
8143         default:
8144                 MISSING_CASE(tmp);
8145                 return 0;
8146         }
8147 }
8148
8149 int ilk_get_lanes_required(int target_clock, int link_bw, int bpp)
8150 {
8151         /*
8152          * Account for spread spectrum to avoid
8153          * oversubscribing the link. Max center spread
8154          * is 2.5%; use 5% for safety's sake.
8155          */
8156         u32 bps = target_clock * bpp * 21 / 20;
8157         return DIV_ROUND_UP(bps, link_bw * 8);
8158 }
8159
8160 static void intel_pch_transcoder_get_m_n(struct intel_crtc *crtc,
8161                                          struct intel_link_m_n *m_n)
8162 {
8163         struct drm_device *dev = crtc->base.dev;
8164         struct drm_i915_private *dev_priv = to_i915(dev);
8165         enum pipe pipe = crtc->pipe;
8166
8167         m_n->link_m = intel_de_read(dev_priv, PCH_TRANS_LINK_M1(pipe));
8168         m_n->link_n = intel_de_read(dev_priv, PCH_TRANS_LINK_N1(pipe));
8169         m_n->gmch_m = intel_de_read(dev_priv, PCH_TRANS_DATA_M1(pipe))
8170                 & ~TU_SIZE_MASK;
8171         m_n->gmch_n = intel_de_read(dev_priv, PCH_TRANS_DATA_N1(pipe));
8172         m_n->tu = ((intel_de_read(dev_priv, PCH_TRANS_DATA_M1(pipe))
8173                     & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
8174 }
8175
8176 static void intel_cpu_transcoder_get_m_n(struct intel_crtc *crtc,
8177                                          enum transcoder transcoder,
8178                                          struct intel_link_m_n *m_n,
8179                                          struct intel_link_m_n *m2_n2)
8180 {
8181         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8182         enum pipe pipe = crtc->pipe;
8183
8184         if (INTEL_GEN(dev_priv) >= 5) {
8185                 m_n->link_m = intel_de_read(dev_priv,
8186                                             PIPE_LINK_M1(transcoder));
8187                 m_n->link_n = intel_de_read(dev_priv,
8188                                             PIPE_LINK_N1(transcoder));
8189                 m_n->gmch_m = intel_de_read(dev_priv,
8190                                             PIPE_DATA_M1(transcoder))
8191                         & ~TU_SIZE_MASK;
8192                 m_n->gmch_n = intel_de_read(dev_priv,
8193                                             PIPE_DATA_N1(transcoder));
8194                 m_n->tu = ((intel_de_read(dev_priv, PIPE_DATA_M1(transcoder))
8195                             & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
8196
8197                 if (m2_n2 && transcoder_has_m2_n2(dev_priv, transcoder)) {
8198                         m2_n2->link_m = intel_de_read(dev_priv,
8199                                                       PIPE_LINK_M2(transcoder));
8200                         m2_n2->link_n = intel_de_read(dev_priv,
8201                                                              PIPE_LINK_N2(transcoder));
8202                         m2_n2->gmch_m = intel_de_read(dev_priv,
8203                                                              PIPE_DATA_M2(transcoder))
8204                                         & ~TU_SIZE_MASK;
8205                         m2_n2->gmch_n = intel_de_read(dev_priv,
8206                                                              PIPE_DATA_N2(transcoder));
8207                         m2_n2->tu = ((intel_de_read(dev_priv, PIPE_DATA_M2(transcoder))
8208                                         & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
8209                 }
8210         } else {
8211                 m_n->link_m = intel_de_read(dev_priv, PIPE_LINK_M_G4X(pipe));
8212                 m_n->link_n = intel_de_read(dev_priv, PIPE_LINK_N_G4X(pipe));
8213                 m_n->gmch_m = intel_de_read(dev_priv, PIPE_DATA_M_G4X(pipe))
8214                         & ~TU_SIZE_MASK;
8215                 m_n->gmch_n = intel_de_read(dev_priv, PIPE_DATA_N_G4X(pipe));
8216                 m_n->tu = ((intel_de_read(dev_priv, PIPE_DATA_M_G4X(pipe))
8217                             & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
8218         }
8219 }
8220
8221 void intel_dp_get_m_n(struct intel_crtc *crtc,
8222                       struct intel_crtc_state *pipe_config)
8223 {
8224         if (pipe_config->has_pch_encoder)
8225                 intel_pch_transcoder_get_m_n(crtc, &pipe_config->dp_m_n);
8226         else
8227                 intel_cpu_transcoder_get_m_n(crtc, pipe_config->cpu_transcoder,
8228                                              &pipe_config->dp_m_n,
8229                                              &pipe_config->dp_m2_n2);
8230 }
8231
8232 static void ilk_get_fdi_m_n_config(struct intel_crtc *crtc,
8233                                    struct intel_crtc_state *pipe_config)
8234 {
8235         intel_cpu_transcoder_get_m_n(crtc, pipe_config->cpu_transcoder,
8236                                      &pipe_config->fdi_m_n, NULL);
8237 }
8238
8239 static void ilk_get_pfit_pos_size(struct intel_crtc_state *crtc_state,
8240                                   u32 pos, u32 size)
8241 {
8242         drm_rect_init(&crtc_state->pch_pfit.dst,
8243                       pos >> 16, pos & 0xffff,
8244                       size >> 16, size & 0xffff);
8245 }
8246
8247 static void skl_get_pfit_config(struct intel_crtc_state *crtc_state)
8248 {
8249         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
8250         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8251         struct intel_crtc_scaler_state *scaler_state = &crtc_state->scaler_state;
8252         int id = -1;
8253         int i;
8254
8255         /* find scaler attached to this pipe */
8256         for (i = 0; i < crtc->num_scalers; i++) {
8257                 u32 ctl, pos, size;
8258
8259                 ctl = intel_de_read(dev_priv, SKL_PS_CTRL(crtc->pipe, i));
8260                 if ((ctl & (PS_SCALER_EN | PS_PLANE_SEL_MASK)) != PS_SCALER_EN)
8261                         continue;
8262
8263                 id = i;
8264                 crtc_state->pch_pfit.enabled = true;
8265
8266                 pos = intel_de_read(dev_priv, SKL_PS_WIN_POS(crtc->pipe, i));
8267                 size = intel_de_read(dev_priv, SKL_PS_WIN_SZ(crtc->pipe, i));
8268
8269                 ilk_get_pfit_pos_size(crtc_state, pos, size);
8270
8271                 scaler_state->scalers[i].in_use = true;
8272                 break;
8273         }
8274
8275         scaler_state->scaler_id = id;
8276         if (id >= 0)
8277                 scaler_state->scaler_users |= (1 << SKL_CRTC_INDEX);
8278         else
8279                 scaler_state->scaler_users &= ~(1 << SKL_CRTC_INDEX);
8280 }
8281
8282 static void
8283 skl_get_initial_plane_config(struct intel_crtc *crtc,
8284                              struct intel_initial_plane_config *plane_config)
8285 {
8286         struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state);
8287         struct drm_device *dev = crtc->base.dev;
8288         struct drm_i915_private *dev_priv = to_i915(dev);
8289         struct intel_plane *plane = to_intel_plane(crtc->base.primary);
8290         enum plane_id plane_id = plane->id;
8291         enum pipe pipe;
8292         u32 val, base, offset, stride_mult, tiling, alpha;
8293         int fourcc, pixel_format;
8294         unsigned int aligned_height;
8295         struct drm_framebuffer *fb;
8296         struct intel_framebuffer *intel_fb;
8297
8298         if (!plane->get_hw_state(plane, &pipe))
8299                 return;
8300
8301         drm_WARN_ON(dev, pipe != crtc->pipe);
8302
8303         if (crtc_state->bigjoiner) {
8304                 drm_dbg_kms(&dev_priv->drm,
8305                             "Unsupported bigjoiner configuration for initial FB\n");
8306                 return;
8307         }
8308
8309         intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL);
8310         if (!intel_fb) {
8311                 drm_dbg_kms(&dev_priv->drm, "failed to alloc fb\n");
8312                 return;
8313         }
8314
8315         fb = &intel_fb->base;
8316
8317         fb->dev = dev;
8318
8319         val = intel_de_read(dev_priv, PLANE_CTL(pipe, plane_id));
8320
8321         if (INTEL_GEN(dev_priv) >= 11)
8322                 pixel_format = val & ICL_PLANE_CTL_FORMAT_MASK;
8323         else
8324                 pixel_format = val & PLANE_CTL_FORMAT_MASK;
8325
8326         if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv)) {
8327                 alpha = intel_de_read(dev_priv,
8328                                       PLANE_COLOR_CTL(pipe, plane_id));
8329                 alpha &= PLANE_COLOR_ALPHA_MASK;
8330         } else {
8331                 alpha = val & PLANE_CTL_ALPHA_MASK;
8332         }
8333
8334         fourcc = skl_format_to_fourcc(pixel_format,
8335                                       val & PLANE_CTL_ORDER_RGBX, alpha);
8336         fb->format = drm_format_info(fourcc);
8337
8338         tiling = val & PLANE_CTL_TILED_MASK;
8339         switch (tiling) {
8340         case PLANE_CTL_TILED_LINEAR:
8341                 fb->modifier = DRM_FORMAT_MOD_LINEAR;
8342                 break;
8343         case PLANE_CTL_TILED_X:
8344                 plane_config->tiling = I915_TILING_X;
8345                 fb->modifier = I915_FORMAT_MOD_X_TILED;
8346                 break;
8347         case PLANE_CTL_TILED_Y:
8348                 plane_config->tiling = I915_TILING_Y;
8349                 if (val & PLANE_CTL_RENDER_DECOMPRESSION_ENABLE)
8350                         fb->modifier = INTEL_GEN(dev_priv) >= 12 ?
8351                                 I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS :
8352                                 I915_FORMAT_MOD_Y_TILED_CCS;
8353                 else if (val & PLANE_CTL_MEDIA_DECOMPRESSION_ENABLE)
8354                         fb->modifier = I915_FORMAT_MOD_Y_TILED_GEN12_MC_CCS;
8355                 else
8356                         fb->modifier = I915_FORMAT_MOD_Y_TILED;
8357                 break;
8358         case PLANE_CTL_TILED_YF:
8359                 if (val & PLANE_CTL_RENDER_DECOMPRESSION_ENABLE)
8360                         fb->modifier = I915_FORMAT_MOD_Yf_TILED_CCS;
8361                 else
8362                         fb->modifier = I915_FORMAT_MOD_Yf_TILED;
8363                 break;
8364         default:
8365                 MISSING_CASE(tiling);
8366                 goto error;
8367         }
8368
8369         /*
8370          * DRM_MODE_ROTATE_ is counter clockwise to stay compatible with Xrandr
8371          * while i915 HW rotation is clockwise, thats why this swapping.
8372          */
8373         switch (val & PLANE_CTL_ROTATE_MASK) {
8374         case PLANE_CTL_ROTATE_0:
8375                 plane_config->rotation = DRM_MODE_ROTATE_0;
8376                 break;
8377         case PLANE_CTL_ROTATE_90:
8378                 plane_config->rotation = DRM_MODE_ROTATE_270;
8379                 break;
8380         case PLANE_CTL_ROTATE_180:
8381                 plane_config->rotation = DRM_MODE_ROTATE_180;
8382                 break;
8383         case PLANE_CTL_ROTATE_270:
8384                 plane_config->rotation = DRM_MODE_ROTATE_90;
8385                 break;
8386         }
8387
8388         if (INTEL_GEN(dev_priv) >= 10 &&
8389             val & PLANE_CTL_FLIP_HORIZONTAL)
8390                 plane_config->rotation |= DRM_MODE_REFLECT_X;
8391
8392         /* 90/270 degree rotation would require extra work */
8393         if (drm_rotation_90_or_270(plane_config->rotation))
8394                 goto error;
8395
8396         base = intel_de_read(dev_priv, PLANE_SURF(pipe, plane_id)) & 0xfffff000;
8397         plane_config->base = base;
8398
8399         offset = intel_de_read(dev_priv, PLANE_OFFSET(pipe, plane_id));
8400
8401         val = intel_de_read(dev_priv, PLANE_SIZE(pipe, plane_id));
8402         fb->height = ((val >> 16) & 0xffff) + 1;
8403         fb->width = ((val >> 0) & 0xffff) + 1;
8404
8405         val = intel_de_read(dev_priv, PLANE_STRIDE(pipe, plane_id));
8406         stride_mult = skl_plane_stride_mult(fb, 0, DRM_MODE_ROTATE_0);
8407         fb->pitches[0] = (val & 0x3ff) * stride_mult;
8408
8409         aligned_height = intel_fb_align_height(fb, 0, fb->height);
8410
8411         plane_config->size = fb->pitches[0] * aligned_height;
8412
8413         drm_dbg_kms(&dev_priv->drm,
8414                     "%s/%s with fb: size=%dx%d@%d, offset=%x, pitch %d, size 0x%x\n",
8415                     crtc->base.name, plane->base.name, fb->width, fb->height,
8416                     fb->format->cpp[0] * 8, base, fb->pitches[0],
8417                     plane_config->size);
8418
8419         plane_config->fb = intel_fb;
8420         return;
8421
8422 error:
8423         kfree(intel_fb);
8424 }
8425
8426 static void ilk_get_pfit_config(struct intel_crtc_state *crtc_state)
8427 {
8428         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
8429         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8430         u32 ctl, pos, size;
8431
8432         ctl = intel_de_read(dev_priv, PF_CTL(crtc->pipe));
8433         if ((ctl & PF_ENABLE) == 0)
8434                 return;
8435
8436         crtc_state->pch_pfit.enabled = true;
8437
8438         pos = intel_de_read(dev_priv, PF_WIN_POS(crtc->pipe));
8439         size = intel_de_read(dev_priv, PF_WIN_SZ(crtc->pipe));
8440
8441         ilk_get_pfit_pos_size(crtc_state, pos, size);
8442
8443         /*
8444          * We currently do not free assignements of panel fitters on
8445          * ivb/hsw (since we don't use the higher upscaling modes which
8446          * differentiates them) so just WARN about this case for now.
8447          */
8448         drm_WARN_ON(&dev_priv->drm, IS_GEN(dev_priv, 7) &&
8449                     (ctl & PF_PIPE_SEL_MASK_IVB) != PF_PIPE_SEL_IVB(crtc->pipe));
8450 }
8451
8452 static bool ilk_get_pipe_config(struct intel_crtc *crtc,
8453                                 struct intel_crtc_state *pipe_config)
8454 {
8455         struct drm_device *dev = crtc->base.dev;
8456         struct drm_i915_private *dev_priv = to_i915(dev);
8457         enum intel_display_power_domain power_domain;
8458         intel_wakeref_t wakeref;
8459         u32 tmp;
8460         bool ret;
8461
8462         power_domain = POWER_DOMAIN_PIPE(crtc->pipe);
8463         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
8464         if (!wakeref)
8465                 return false;
8466
8467         pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe;
8468         pipe_config->shared_dpll = NULL;
8469
8470         ret = false;
8471         tmp = intel_de_read(dev_priv, PIPECONF(crtc->pipe));
8472         if (!(tmp & PIPECONF_ENABLE))
8473                 goto out;
8474
8475         switch (tmp & PIPECONF_BPC_MASK) {
8476         case PIPECONF_6BPC:
8477                 pipe_config->pipe_bpp = 18;
8478                 break;
8479         case PIPECONF_8BPC:
8480                 pipe_config->pipe_bpp = 24;
8481                 break;
8482         case PIPECONF_10BPC:
8483                 pipe_config->pipe_bpp = 30;
8484                 break;
8485         case PIPECONF_12BPC:
8486                 pipe_config->pipe_bpp = 36;
8487                 break;
8488         default:
8489                 break;
8490         }
8491
8492         if (tmp & PIPECONF_COLOR_RANGE_SELECT)
8493                 pipe_config->limited_color_range = true;
8494
8495         switch (tmp & PIPECONF_OUTPUT_COLORSPACE_MASK) {
8496         case PIPECONF_OUTPUT_COLORSPACE_YUV601:
8497         case PIPECONF_OUTPUT_COLORSPACE_YUV709:
8498                 pipe_config->output_format = INTEL_OUTPUT_FORMAT_YCBCR444;
8499                 break;
8500         default:
8501                 pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB;
8502                 break;
8503         }
8504
8505         pipe_config->gamma_mode = (tmp & PIPECONF_GAMMA_MODE_MASK_ILK) >>
8506                 PIPECONF_GAMMA_MODE_SHIFT;
8507
8508         pipe_config->csc_mode = intel_de_read(dev_priv,
8509                                               PIPE_CSC_MODE(crtc->pipe));
8510
8511         i9xx_get_pipe_color_config(pipe_config);
8512         intel_color_get_config(pipe_config);
8513
8514         if (intel_de_read(dev_priv, PCH_TRANSCONF(crtc->pipe)) & TRANS_ENABLE) {
8515                 struct intel_shared_dpll *pll;
8516                 enum intel_dpll_id pll_id;
8517                 bool pll_active;
8518
8519                 pipe_config->has_pch_encoder = true;
8520
8521                 tmp = intel_de_read(dev_priv, FDI_RX_CTL(crtc->pipe));
8522                 pipe_config->fdi_lanes = ((FDI_DP_PORT_WIDTH_MASK & tmp) >>
8523                                           FDI_DP_PORT_WIDTH_SHIFT) + 1;
8524
8525                 ilk_get_fdi_m_n_config(crtc, pipe_config);
8526
8527                 if (HAS_PCH_IBX(dev_priv)) {
8528                         /*
8529                          * The pipe->pch transcoder and pch transcoder->pll
8530                          * mapping is fixed.
8531                          */
8532                         pll_id = (enum intel_dpll_id) crtc->pipe;
8533                 } else {
8534                         tmp = intel_de_read(dev_priv, PCH_DPLL_SEL);
8535                         if (tmp & TRANS_DPLLB_SEL(crtc->pipe))
8536                                 pll_id = DPLL_ID_PCH_PLL_B;
8537                         else
8538                                 pll_id= DPLL_ID_PCH_PLL_A;
8539                 }
8540
8541                 pipe_config->shared_dpll =
8542                         intel_get_shared_dpll_by_id(dev_priv, pll_id);
8543                 pll = pipe_config->shared_dpll;
8544
8545                 pll_active = intel_dpll_get_hw_state(dev_priv, pll,
8546                                                      &pipe_config->dpll_hw_state);
8547                 drm_WARN_ON(dev, !pll_active);
8548
8549                 tmp = pipe_config->dpll_hw_state.dpll;
8550                 pipe_config->pixel_multiplier =
8551                         ((tmp & PLL_REF_SDVO_HDMI_MULTIPLIER_MASK)
8552                          >> PLL_REF_SDVO_HDMI_MULTIPLIER_SHIFT) + 1;
8553
8554                 ilk_pch_clock_get(crtc, pipe_config);
8555         } else {
8556                 pipe_config->pixel_multiplier = 1;
8557         }
8558
8559         intel_get_transcoder_timings(crtc, pipe_config);
8560         intel_get_pipe_src_size(crtc, pipe_config);
8561
8562         ilk_get_pfit_config(pipe_config);
8563
8564         ret = true;
8565
8566 out:
8567         intel_display_power_put(dev_priv, power_domain, wakeref);
8568
8569         return ret;
8570 }
8571
8572 static void dg1_get_ddi_pll(struct drm_i915_private *dev_priv, enum port port,
8573                             struct intel_crtc_state *pipe_config)
8574 {
8575         enum icl_port_dpll_id port_dpll_id = ICL_PORT_DPLL_DEFAULT;
8576         enum phy phy = intel_port_to_phy(dev_priv, port);
8577         struct icl_port_dpll *port_dpll;
8578         struct intel_shared_dpll *pll;
8579         enum intel_dpll_id id;
8580         bool pll_active;
8581         u32 clk_sel;
8582
8583         clk_sel = intel_de_read(dev_priv, DG1_DPCLKA_CFGCR0(phy)) & DG1_DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(phy);
8584         id = DG1_DPCLKA_CFGCR0_DDI_CLK_SEL_DPLL_MAP(clk_sel, phy);
8585
8586         if (WARN_ON(id > DPLL_ID_DG1_DPLL3))
8587                 return;
8588
8589         pll = intel_get_shared_dpll_by_id(dev_priv, id);
8590         port_dpll = &pipe_config->icl_port_dplls[port_dpll_id];
8591
8592         port_dpll->pll = pll;
8593         pll_active = intel_dpll_get_hw_state(dev_priv, pll,
8594                                              &port_dpll->hw_state);
8595         drm_WARN_ON(&dev_priv->drm, !pll_active);
8596
8597         icl_set_active_port_dpll(pipe_config, port_dpll_id);
8598 }
8599
8600 static void icl_get_ddi_pll(struct drm_i915_private *dev_priv, enum port port,
8601                             struct intel_crtc_state *pipe_config)
8602 {
8603         enum phy phy = intel_port_to_phy(dev_priv, port);
8604         enum icl_port_dpll_id port_dpll_id;
8605         struct icl_port_dpll *port_dpll;
8606         struct intel_shared_dpll *pll;
8607         enum intel_dpll_id id;
8608         bool pll_active;
8609         i915_reg_t reg;
8610         u32 temp;
8611
8612         if (intel_phy_is_combo(dev_priv, phy)) {
8613                 u32 mask, shift;
8614
8615                 if (IS_ALDERLAKE_S(dev_priv)) {
8616                         reg = ADLS_DPCLKA_CFGCR(phy);
8617                         mask = ADLS_DPCLKA_CFGCR_DDI_CLK_SEL_MASK(phy);
8618                         shift = ADLS_DPCLKA_CFGCR_DDI_SHIFT(phy);
8619                 } else if (IS_ROCKETLAKE(dev_priv)) {
8620                         reg = ICL_DPCLKA_CFGCR0;
8621                         mask = RKL_DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(phy);
8622                         shift = RKL_DPCLKA_CFGCR0_DDI_CLK_SEL_SHIFT(phy);
8623                 } else {
8624                         reg = ICL_DPCLKA_CFGCR0;
8625                         mask = ICL_DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(phy);
8626                         shift = ICL_DPCLKA_CFGCR0_DDI_CLK_SEL_SHIFT(phy);
8627                 }
8628
8629                 temp = intel_de_read(dev_priv, reg) & mask;
8630                 id = temp >> shift;
8631                 port_dpll_id = ICL_PORT_DPLL_DEFAULT;
8632         } else if (intel_phy_is_tc(dev_priv, phy)) {
8633                 u32 clk_sel = intel_de_read(dev_priv, DDI_CLK_SEL(port)) & DDI_CLK_SEL_MASK;
8634
8635                 if (clk_sel == DDI_CLK_SEL_MG) {
8636                         id = icl_tc_port_to_pll_id(intel_port_to_tc(dev_priv,
8637                                                                     port));
8638                         port_dpll_id = ICL_PORT_DPLL_MG_PHY;
8639                 } else {
8640                         drm_WARN_ON(&dev_priv->drm,
8641                                     clk_sel < DDI_CLK_SEL_TBT_162);
8642                         id = DPLL_ID_ICL_TBTPLL;
8643                         port_dpll_id = ICL_PORT_DPLL_DEFAULT;
8644                 }
8645         } else {
8646                 drm_WARN(&dev_priv->drm, 1, "Invalid port %x\n", port);
8647                 return;
8648         }
8649
8650         pll = intel_get_shared_dpll_by_id(dev_priv, id);
8651         port_dpll = &pipe_config->icl_port_dplls[port_dpll_id];
8652
8653         port_dpll->pll = pll;
8654         pll_active = intel_dpll_get_hw_state(dev_priv, pll,
8655                                              &port_dpll->hw_state);
8656         drm_WARN_ON(&dev_priv->drm, !pll_active);
8657
8658         icl_set_active_port_dpll(pipe_config, port_dpll_id);
8659 }
8660
8661 static void cnl_get_ddi_pll(struct drm_i915_private *dev_priv, enum port port,
8662                             struct intel_crtc_state *pipe_config)
8663 {
8664         struct intel_shared_dpll *pll;
8665         enum intel_dpll_id id;
8666         bool pll_active;
8667         u32 temp;
8668
8669         temp = intel_de_read(dev_priv, DPCLKA_CFGCR0) & DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(port);
8670         id = temp >> DPCLKA_CFGCR0_DDI_CLK_SEL_SHIFT(port);
8671
8672         if (drm_WARN_ON(&dev_priv->drm, id < SKL_DPLL0 || id > SKL_DPLL2))
8673                 return;
8674
8675         pll = intel_get_shared_dpll_by_id(dev_priv, id);
8676
8677         pipe_config->shared_dpll = pll;
8678         pll_active = intel_dpll_get_hw_state(dev_priv, pll,
8679                                              &pipe_config->dpll_hw_state);
8680         drm_WARN_ON(&dev_priv->drm, !pll_active);
8681 }
8682
8683 static void bxt_get_ddi_pll(struct drm_i915_private *dev_priv,
8684                                 enum port port,
8685                                 struct intel_crtc_state *pipe_config)
8686 {
8687         struct intel_shared_dpll *pll;
8688         enum intel_dpll_id id;
8689         bool pll_active;
8690
8691         switch (port) {
8692         case PORT_A:
8693                 id = DPLL_ID_SKL_DPLL0;
8694                 break;
8695         case PORT_B:
8696                 id = DPLL_ID_SKL_DPLL1;
8697                 break;
8698         case PORT_C:
8699                 id = DPLL_ID_SKL_DPLL2;
8700                 break;
8701         default:
8702                 drm_err(&dev_priv->drm, "Incorrect port type\n");
8703                 return;
8704         }
8705
8706         pll = intel_get_shared_dpll_by_id(dev_priv, id);
8707
8708         pipe_config->shared_dpll = pll;
8709         pll_active = intel_dpll_get_hw_state(dev_priv, pll,
8710                                              &pipe_config->dpll_hw_state);
8711         drm_WARN_ON(&dev_priv->drm, !pll_active);
8712 }
8713
8714 static void skl_get_ddi_pll(struct drm_i915_private *dev_priv, enum port port,
8715                             struct intel_crtc_state *pipe_config)
8716 {
8717         struct intel_shared_dpll *pll;
8718         enum intel_dpll_id id;
8719         bool pll_active;
8720         u32 temp;
8721
8722         temp = intel_de_read(dev_priv, DPLL_CTRL2) & DPLL_CTRL2_DDI_CLK_SEL_MASK(port);
8723         id = temp >> (port * 3 + 1);
8724
8725         if (drm_WARN_ON(&dev_priv->drm, id < SKL_DPLL0 || id > SKL_DPLL3))
8726                 return;
8727
8728         pll = intel_get_shared_dpll_by_id(dev_priv, id);
8729
8730         pipe_config->shared_dpll = pll;
8731         pll_active = intel_dpll_get_hw_state(dev_priv, pll,
8732                                              &pipe_config->dpll_hw_state);
8733         drm_WARN_ON(&dev_priv->drm, !pll_active);
8734 }
8735
8736 static void hsw_get_ddi_pll(struct drm_i915_private *dev_priv, enum port port,
8737                             struct intel_crtc_state *pipe_config)
8738 {
8739         struct intel_shared_dpll *pll;
8740         enum intel_dpll_id id;
8741         u32 ddi_pll_sel = intel_de_read(dev_priv, PORT_CLK_SEL(port));
8742         bool pll_active;
8743
8744         switch (ddi_pll_sel) {
8745         case PORT_CLK_SEL_WRPLL1:
8746                 id = DPLL_ID_WRPLL1;
8747                 break;
8748         case PORT_CLK_SEL_WRPLL2:
8749                 id = DPLL_ID_WRPLL2;
8750                 break;
8751         case PORT_CLK_SEL_SPLL:
8752                 id = DPLL_ID_SPLL;
8753                 break;
8754         case PORT_CLK_SEL_LCPLL_810:
8755                 id = DPLL_ID_LCPLL_810;
8756                 break;
8757         case PORT_CLK_SEL_LCPLL_1350:
8758                 id = DPLL_ID_LCPLL_1350;
8759                 break;
8760         case PORT_CLK_SEL_LCPLL_2700:
8761                 id = DPLL_ID_LCPLL_2700;
8762                 break;
8763         default:
8764                 MISSING_CASE(ddi_pll_sel);
8765                 fallthrough;
8766         case PORT_CLK_SEL_NONE:
8767                 return;
8768         }
8769
8770         pll = intel_get_shared_dpll_by_id(dev_priv, id);
8771
8772         pipe_config->shared_dpll = pll;
8773         pll_active = intel_dpll_get_hw_state(dev_priv, pll,
8774                                              &pipe_config->dpll_hw_state);
8775         drm_WARN_ON(&dev_priv->drm, !pll_active);
8776 }
8777
8778 static bool hsw_get_transcoder_state(struct intel_crtc *crtc,
8779                                      struct intel_crtc_state *pipe_config,
8780                                      struct intel_display_power_domain_set *power_domain_set)
8781 {
8782         struct drm_device *dev = crtc->base.dev;
8783         struct drm_i915_private *dev_priv = to_i915(dev);
8784         unsigned long panel_transcoder_mask = BIT(TRANSCODER_EDP);
8785         unsigned long enabled_panel_transcoders = 0;
8786         enum transcoder panel_transcoder;
8787         u32 tmp;
8788
8789         if (INTEL_GEN(dev_priv) >= 11)
8790                 panel_transcoder_mask |=
8791                         BIT(TRANSCODER_DSI_0) | BIT(TRANSCODER_DSI_1);
8792
8793         /*
8794          * The pipe->transcoder mapping is fixed with the exception of the eDP
8795          * and DSI transcoders handled below.
8796          */
8797         pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe;
8798
8799         /*
8800          * XXX: Do intel_display_power_get_if_enabled before reading this (for
8801          * consistency and less surprising code; it's in always on power).
8802          */
8803         for_each_cpu_transcoder_masked(dev_priv, panel_transcoder,
8804                                        panel_transcoder_mask) {
8805                 bool force_thru = false;
8806                 enum pipe trans_pipe;
8807
8808                 tmp = intel_de_read(dev_priv,
8809                                     TRANS_DDI_FUNC_CTL(panel_transcoder));
8810                 if (!(tmp & TRANS_DDI_FUNC_ENABLE))
8811                         continue;
8812
8813                 /*
8814                  * Log all enabled ones, only use the first one.
8815                  *
8816                  * FIXME: This won't work for two separate DSI displays.
8817                  */
8818                 enabled_panel_transcoders |= BIT(panel_transcoder);
8819                 if (enabled_panel_transcoders != BIT(panel_transcoder))
8820                         continue;
8821
8822                 switch (tmp & TRANS_DDI_EDP_INPUT_MASK) {
8823                 default:
8824                         drm_WARN(dev, 1,
8825                                  "unknown pipe linked to transcoder %s\n",
8826                                  transcoder_name(panel_transcoder));
8827                         fallthrough;
8828                 case TRANS_DDI_EDP_INPUT_A_ONOFF:
8829                         force_thru = true;
8830                         fallthrough;
8831                 case TRANS_DDI_EDP_INPUT_A_ON:
8832                         trans_pipe = PIPE_A;
8833                         break;
8834                 case TRANS_DDI_EDP_INPUT_B_ONOFF:
8835                         trans_pipe = PIPE_B;
8836                         break;
8837                 case TRANS_DDI_EDP_INPUT_C_ONOFF:
8838                         trans_pipe = PIPE_C;
8839                         break;
8840                 case TRANS_DDI_EDP_INPUT_D_ONOFF:
8841                         trans_pipe = PIPE_D;
8842                         break;
8843                 }
8844
8845                 if (trans_pipe == crtc->pipe) {
8846                         pipe_config->cpu_transcoder = panel_transcoder;
8847                         pipe_config->pch_pfit.force_thru = force_thru;
8848                 }
8849         }
8850
8851         /*
8852          * Valid combos: none, eDP, DSI0, DSI1, DSI0+DSI1
8853          */
8854         drm_WARN_ON(dev, (enabled_panel_transcoders & BIT(TRANSCODER_EDP)) &&
8855                     enabled_panel_transcoders != BIT(TRANSCODER_EDP));
8856
8857         if (!intel_display_power_get_in_set_if_enabled(dev_priv, power_domain_set,
8858                                                        POWER_DOMAIN_TRANSCODER(pipe_config->cpu_transcoder)))
8859                 return false;
8860
8861         tmp = intel_de_read(dev_priv, PIPECONF(pipe_config->cpu_transcoder));
8862
8863         return tmp & PIPECONF_ENABLE;
8864 }
8865
8866 static bool bxt_get_dsi_transcoder_state(struct intel_crtc *crtc,
8867                                          struct intel_crtc_state *pipe_config,
8868                                          struct intel_display_power_domain_set *power_domain_set)
8869 {
8870         struct drm_device *dev = crtc->base.dev;
8871         struct drm_i915_private *dev_priv = to_i915(dev);
8872         enum transcoder cpu_transcoder;
8873         enum port port;
8874         u32 tmp;
8875
8876         for_each_port_masked(port, BIT(PORT_A) | BIT(PORT_C)) {
8877                 if (port == PORT_A)
8878                         cpu_transcoder = TRANSCODER_DSI_A;
8879                 else
8880                         cpu_transcoder = TRANSCODER_DSI_C;
8881
8882                 if (!intel_display_power_get_in_set_if_enabled(dev_priv, power_domain_set,
8883                                                                POWER_DOMAIN_TRANSCODER(cpu_transcoder)))
8884                         continue;
8885
8886                 /*
8887                  * The PLL needs to be enabled with a valid divider
8888                  * configuration, otherwise accessing DSI registers will hang
8889                  * the machine. See BSpec North Display Engine
8890                  * registers/MIPI[BXT]. We can break out here early, since we
8891                  * need the same DSI PLL to be enabled for both DSI ports.
8892                  */
8893                 if (!bxt_dsi_pll_is_enabled(dev_priv))
8894                         break;
8895
8896                 /* XXX: this works for video mode only */
8897                 tmp = intel_de_read(dev_priv, BXT_MIPI_PORT_CTRL(port));
8898                 if (!(tmp & DPI_ENABLE))
8899                         continue;
8900
8901                 tmp = intel_de_read(dev_priv, MIPI_CTRL(port));
8902                 if ((tmp & BXT_PIPE_SELECT_MASK) != BXT_PIPE_SELECT(crtc->pipe))
8903                         continue;
8904
8905                 pipe_config->cpu_transcoder = cpu_transcoder;
8906                 break;
8907         }
8908
8909         return transcoder_is_dsi(pipe_config->cpu_transcoder);
8910 }
8911
8912 static void hsw_get_ddi_port_state(struct intel_crtc *crtc,
8913                                    struct intel_crtc_state *pipe_config)
8914 {
8915         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8916         enum transcoder cpu_transcoder = pipe_config->cpu_transcoder;
8917         enum port port;
8918         u32 tmp;
8919
8920         if (transcoder_is_dsi(cpu_transcoder)) {
8921                 port = (cpu_transcoder == TRANSCODER_DSI_A) ?
8922                                                 PORT_A : PORT_B;
8923         } else {
8924                 tmp = intel_de_read(dev_priv,
8925                                     TRANS_DDI_FUNC_CTL(cpu_transcoder));
8926                 if (!(tmp & TRANS_DDI_FUNC_ENABLE))
8927                         return;
8928                 if (INTEL_GEN(dev_priv) >= 12)
8929                         port = TGL_TRANS_DDI_FUNC_CTL_VAL_TO_PORT(tmp);
8930                 else
8931                         port = TRANS_DDI_FUNC_CTL_VAL_TO_PORT(tmp);
8932         }
8933
8934         if (IS_DG1(dev_priv))
8935                 dg1_get_ddi_pll(dev_priv, port, pipe_config);
8936         else if (INTEL_GEN(dev_priv) >= 11)
8937                 icl_get_ddi_pll(dev_priv, port, pipe_config);
8938         else if (IS_CANNONLAKE(dev_priv))
8939                 cnl_get_ddi_pll(dev_priv, port, pipe_config);
8940         else if (IS_GEN9_LP(dev_priv))
8941                 bxt_get_ddi_pll(dev_priv, port, pipe_config);
8942         else if (IS_GEN9_BC(dev_priv))
8943                 skl_get_ddi_pll(dev_priv, port, pipe_config);
8944         else
8945                 hsw_get_ddi_pll(dev_priv, port, pipe_config);
8946
8947         /*
8948          * Haswell has only FDI/PCH transcoder A. It is which is connected to
8949          * DDI E. So just check whether this pipe is wired to DDI E and whether
8950          * the PCH transcoder is on.
8951          */
8952         if (INTEL_GEN(dev_priv) < 9 &&
8953             (port == PORT_E) && intel_de_read(dev_priv, LPT_TRANSCONF) & TRANS_ENABLE) {
8954                 pipe_config->has_pch_encoder = true;
8955
8956                 tmp = intel_de_read(dev_priv, FDI_RX_CTL(PIPE_A));
8957                 pipe_config->fdi_lanes = ((FDI_DP_PORT_WIDTH_MASK & tmp) >>
8958                                           FDI_DP_PORT_WIDTH_SHIFT) + 1;
8959
8960                 ilk_get_fdi_m_n_config(crtc, pipe_config);
8961         }
8962 }
8963
8964 static bool hsw_get_pipe_config(struct intel_crtc *crtc,
8965                                 struct intel_crtc_state *pipe_config)
8966 {
8967         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8968         struct intel_display_power_domain_set power_domain_set = { };
8969         bool active;
8970         u32 tmp;
8971
8972         if (!intel_display_power_get_in_set_if_enabled(dev_priv, &power_domain_set,
8973                                                        POWER_DOMAIN_PIPE(crtc->pipe)))
8974                 return false;
8975
8976         pipe_config->shared_dpll = NULL;
8977
8978         active = hsw_get_transcoder_state(crtc, pipe_config, &power_domain_set);
8979
8980         if (IS_GEN9_LP(dev_priv) &&
8981             bxt_get_dsi_transcoder_state(crtc, pipe_config, &power_domain_set)) {
8982                 drm_WARN_ON(&dev_priv->drm, active);
8983                 active = true;
8984         }
8985
8986         intel_dsc_get_config(pipe_config);
8987
8988         if (!active) {
8989                 /* bigjoiner slave doesn't enable transcoder */
8990                 if (!pipe_config->bigjoiner_slave)
8991                         goto out;
8992
8993                 active = true;
8994                 pipe_config->pixel_multiplier = 1;
8995
8996                 /* we cannot read out most state, so don't bother.. */
8997                 pipe_config->quirks |= PIPE_CONFIG_QUIRK_BIGJOINER_SLAVE;
8998         } else if (!transcoder_is_dsi(pipe_config->cpu_transcoder) ||
8999             INTEL_GEN(dev_priv) >= 11) {
9000                 hsw_get_ddi_port_state(crtc, pipe_config);
9001                 intel_get_transcoder_timings(crtc, pipe_config);
9002         }
9003
9004         if (HAS_VRR(dev_priv) && !transcoder_is_dsi(pipe_config->cpu_transcoder))
9005                 intel_vrr_get_config(crtc, pipe_config);
9006
9007         intel_get_pipe_src_size(crtc, pipe_config);
9008
9009         if (IS_HASWELL(dev_priv)) {
9010                 u32 tmp = intel_de_read(dev_priv,
9011                                         PIPECONF(pipe_config->cpu_transcoder));
9012
9013                 if (tmp & PIPECONF_OUTPUT_COLORSPACE_YUV_HSW)
9014                         pipe_config->output_format = INTEL_OUTPUT_FORMAT_YCBCR444;
9015                 else
9016                         pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB;
9017         } else {
9018                 pipe_config->output_format =
9019                         bdw_get_pipemisc_output_format(crtc);
9020         }
9021
9022         pipe_config->gamma_mode = intel_de_read(dev_priv,
9023                                                 GAMMA_MODE(crtc->pipe));
9024
9025         pipe_config->csc_mode = intel_de_read(dev_priv,
9026                                               PIPE_CSC_MODE(crtc->pipe));
9027
9028         if (INTEL_GEN(dev_priv) >= 9) {
9029                 tmp = intel_de_read(dev_priv, SKL_BOTTOM_COLOR(crtc->pipe));
9030
9031                 if (tmp & SKL_BOTTOM_COLOR_GAMMA_ENABLE)
9032                         pipe_config->gamma_enable = true;
9033
9034                 if (tmp & SKL_BOTTOM_COLOR_CSC_ENABLE)
9035                         pipe_config->csc_enable = true;
9036         } else {
9037                 i9xx_get_pipe_color_config(pipe_config);
9038         }
9039
9040         intel_color_get_config(pipe_config);
9041
9042         tmp = intel_de_read(dev_priv, WM_LINETIME(crtc->pipe));
9043         pipe_config->linetime = REG_FIELD_GET(HSW_LINETIME_MASK, tmp);
9044         if (IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
9045                 pipe_config->ips_linetime =
9046                         REG_FIELD_GET(HSW_IPS_LINETIME_MASK, tmp);
9047
9048         if (intel_display_power_get_in_set_if_enabled(dev_priv, &power_domain_set,
9049                                                       POWER_DOMAIN_PIPE_PANEL_FITTER(crtc->pipe))) {
9050                 if (INTEL_GEN(dev_priv) >= 9)
9051                         skl_get_pfit_config(pipe_config);
9052                 else
9053                         ilk_get_pfit_config(pipe_config);
9054         }
9055
9056         if (hsw_crtc_supports_ips(crtc)) {
9057                 if (IS_HASWELL(dev_priv))
9058                         pipe_config->ips_enabled = intel_de_read(dev_priv,
9059                                                                  IPS_CTL) & IPS_ENABLE;
9060                 else {
9061                         /*
9062                          * We cannot readout IPS state on broadwell, set to
9063                          * true so we can set it to a defined state on first
9064                          * commit.
9065                          */
9066                         pipe_config->ips_enabled = true;
9067                 }
9068         }
9069
9070         if (pipe_config->bigjoiner_slave) {
9071                 /* Cannot be read out as a slave, set to 0. */
9072                 pipe_config->pixel_multiplier = 0;
9073         } else if (pipe_config->cpu_transcoder != TRANSCODER_EDP &&
9074             !transcoder_is_dsi(pipe_config->cpu_transcoder)) {
9075                 pipe_config->pixel_multiplier =
9076                         intel_de_read(dev_priv,
9077                                       PIPE_MULT(pipe_config->cpu_transcoder)) + 1;
9078         } else {
9079                 pipe_config->pixel_multiplier = 1;
9080         }
9081
9082 out:
9083         intel_display_power_put_all_in_set(dev_priv, &power_domain_set);
9084
9085         return active;
9086 }
9087
9088 static bool intel_crtc_get_pipe_config(struct intel_crtc_state *crtc_state)
9089 {
9090         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
9091         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
9092
9093         if (!i915->display.get_pipe_config(crtc, crtc_state))
9094                 return false;
9095
9096         crtc_state->hw.active = true;
9097
9098         intel_crtc_readout_derived_state(crtc_state);
9099
9100         return true;
9101 }
9102
9103 /* VESA 640x480x72Hz mode to set on the pipe */
9104 static const struct drm_display_mode load_detect_mode = {
9105         DRM_MODE("640x480", DRM_MODE_TYPE_DEFAULT, 31500, 640, 664,
9106                  704, 832, 0, 480, 489, 491, 520, 0, DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
9107 };
9108
9109 struct drm_framebuffer *
9110 intel_framebuffer_create(struct drm_i915_gem_object *obj,
9111                          struct drm_mode_fb_cmd2 *mode_cmd)
9112 {
9113         struct intel_framebuffer *intel_fb;
9114         int ret;
9115
9116         intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL);
9117         if (!intel_fb)
9118                 return ERR_PTR(-ENOMEM);
9119
9120         ret = intel_framebuffer_init(intel_fb, obj, mode_cmd);
9121         if (ret)
9122                 goto err;
9123
9124         return &intel_fb->base;
9125
9126 err:
9127         kfree(intel_fb);
9128         return ERR_PTR(ret);
9129 }
9130
9131 static int intel_modeset_disable_planes(struct drm_atomic_state *state,
9132                                         struct drm_crtc *crtc)
9133 {
9134         struct drm_plane *plane;
9135         struct drm_plane_state *plane_state;
9136         int ret, i;
9137
9138         ret = drm_atomic_add_affected_planes(state, crtc);
9139         if (ret)
9140                 return ret;
9141
9142         for_each_new_plane_in_state(state, plane, plane_state, i) {
9143                 if (plane_state->crtc != crtc)
9144                         continue;
9145
9146                 ret = drm_atomic_set_crtc_for_plane(plane_state, NULL);
9147                 if (ret)
9148                         return ret;
9149
9150                 drm_atomic_set_fb_for_plane(plane_state, NULL);
9151         }
9152
9153         return 0;
9154 }
9155
9156 int intel_get_load_detect_pipe(struct drm_connector *connector,
9157                                struct intel_load_detect_pipe *old,
9158                                struct drm_modeset_acquire_ctx *ctx)
9159 {
9160         struct intel_crtc *intel_crtc;
9161         struct intel_encoder *intel_encoder =
9162                 intel_attached_encoder(to_intel_connector(connector));
9163         struct drm_crtc *possible_crtc;
9164         struct drm_encoder *encoder = &intel_encoder->base;
9165         struct drm_crtc *crtc = NULL;
9166         struct drm_device *dev = encoder->dev;
9167         struct drm_i915_private *dev_priv = to_i915(dev);
9168         struct drm_mode_config *config = &dev->mode_config;
9169         struct drm_atomic_state *state = NULL, *restore_state = NULL;
9170         struct drm_connector_state *connector_state;
9171         struct intel_crtc_state *crtc_state;
9172         int ret, i = -1;
9173
9174         drm_dbg_kms(&dev_priv->drm, "[CONNECTOR:%d:%s], [ENCODER:%d:%s]\n",
9175                     connector->base.id, connector->name,
9176                     encoder->base.id, encoder->name);
9177
9178         old->restore_state = NULL;
9179
9180         drm_WARN_ON(dev, !drm_modeset_is_locked(&config->connection_mutex));
9181
9182         /*
9183          * Algorithm gets a little messy:
9184          *
9185          *   - if the connector already has an assigned crtc, use it (but make
9186          *     sure it's on first)
9187          *
9188          *   - try to find the first unused crtc that can drive this connector,
9189          *     and use that if we find one
9190          */
9191
9192         /* See if we already have a CRTC for this connector */
9193         if (connector->state->crtc) {
9194                 crtc = connector->state->crtc;
9195
9196                 ret = drm_modeset_lock(&crtc->mutex, ctx);
9197                 if (ret)
9198                         goto fail;
9199
9200                 /* Make sure the crtc and connector are running */
9201                 goto found;
9202         }
9203
9204         /* Find an unused one (if possible) */
9205         for_each_crtc(dev, possible_crtc) {
9206                 i++;
9207                 if (!(encoder->possible_crtcs & (1 << i)))
9208                         continue;
9209
9210                 ret = drm_modeset_lock(&possible_crtc->mutex, ctx);
9211                 if (ret)
9212                         goto fail;
9213
9214                 if (possible_crtc->state->enable) {
9215                         drm_modeset_unlock(&possible_crtc->mutex);
9216                         continue;
9217                 }
9218
9219                 crtc = possible_crtc;
9220                 break;
9221         }
9222
9223         /*
9224          * If we didn't find an unused CRTC, don't use any.
9225          */
9226         if (!crtc) {
9227                 drm_dbg_kms(&dev_priv->drm,
9228                             "no pipe available for load-detect\n");
9229                 ret = -ENODEV;
9230                 goto fail;
9231         }
9232
9233 found:
9234         intel_crtc = to_intel_crtc(crtc);
9235
9236         state = drm_atomic_state_alloc(dev);
9237         restore_state = drm_atomic_state_alloc(dev);
9238         if (!state || !restore_state) {
9239                 ret = -ENOMEM;
9240                 goto fail;
9241         }
9242
9243         state->acquire_ctx = ctx;
9244         restore_state->acquire_ctx = ctx;
9245
9246         connector_state = drm_atomic_get_connector_state(state, connector);
9247         if (IS_ERR(connector_state)) {
9248                 ret = PTR_ERR(connector_state);
9249                 goto fail;
9250         }
9251
9252         ret = drm_atomic_set_crtc_for_connector(connector_state, crtc);
9253         if (ret)
9254                 goto fail;
9255
9256         crtc_state = intel_atomic_get_crtc_state(state, intel_crtc);
9257         if (IS_ERR(crtc_state)) {
9258                 ret = PTR_ERR(crtc_state);
9259                 goto fail;
9260         }
9261
9262         crtc_state->uapi.active = true;
9263
9264         ret = drm_atomic_set_mode_for_crtc(&crtc_state->uapi,
9265                                            &load_detect_mode);
9266         if (ret)
9267                 goto fail;
9268
9269         ret = intel_modeset_disable_planes(state, crtc);
9270         if (ret)
9271                 goto fail;
9272
9273         ret = PTR_ERR_OR_ZERO(drm_atomic_get_connector_state(restore_state, connector));
9274         if (!ret)
9275                 ret = PTR_ERR_OR_ZERO(drm_atomic_get_crtc_state(restore_state, crtc));
9276         if (!ret)
9277                 ret = drm_atomic_add_affected_planes(restore_state, crtc);
9278         if (ret) {
9279                 drm_dbg_kms(&dev_priv->drm,
9280                             "Failed to create a copy of old state to restore: %i\n",
9281                             ret);
9282                 goto fail;
9283         }
9284
9285         ret = drm_atomic_commit(state);
9286         if (ret) {
9287                 drm_dbg_kms(&dev_priv->drm,
9288                             "failed to set mode on load-detect pipe\n");
9289                 goto fail;
9290         }
9291
9292         old->restore_state = restore_state;
9293         drm_atomic_state_put(state);
9294
9295         /* let the connector get through one full cycle before testing */
9296         intel_wait_for_vblank(dev_priv, intel_crtc->pipe);
9297         return true;
9298
9299 fail:
9300         if (state) {
9301                 drm_atomic_state_put(state);
9302                 state = NULL;
9303         }
9304         if (restore_state) {
9305                 drm_atomic_state_put(restore_state);
9306                 restore_state = NULL;
9307         }
9308
9309         if (ret == -EDEADLK)
9310                 return ret;
9311
9312         return false;
9313 }
9314
9315 void intel_release_load_detect_pipe(struct drm_connector *connector,
9316                                     struct intel_load_detect_pipe *old,
9317                                     struct drm_modeset_acquire_ctx *ctx)
9318 {
9319         struct intel_encoder *intel_encoder =
9320                 intel_attached_encoder(to_intel_connector(connector));
9321         struct drm_i915_private *i915 = to_i915(intel_encoder->base.dev);
9322         struct drm_encoder *encoder = &intel_encoder->base;
9323         struct drm_atomic_state *state = old->restore_state;
9324         int ret;
9325
9326         drm_dbg_kms(&i915->drm, "[CONNECTOR:%d:%s], [ENCODER:%d:%s]\n",
9327                     connector->base.id, connector->name,
9328                     encoder->base.id, encoder->name);
9329
9330         if (!state)
9331                 return;
9332
9333         ret = drm_atomic_helper_commit_duplicated_state(state, ctx);
9334         if (ret)
9335                 drm_dbg_kms(&i915->drm,
9336                             "Couldn't release load detect pipe: %i\n", ret);
9337         drm_atomic_state_put(state);
9338 }
9339
9340 static int i9xx_pll_refclk(struct drm_device *dev,
9341                            const struct intel_crtc_state *pipe_config)
9342 {
9343         struct drm_i915_private *dev_priv = to_i915(dev);
9344         u32 dpll = pipe_config->dpll_hw_state.dpll;
9345
9346         if ((dpll & PLL_REF_INPUT_MASK) == PLLB_REF_INPUT_SPREADSPECTRUMIN)
9347                 return dev_priv->vbt.lvds_ssc_freq;
9348         else if (HAS_PCH_SPLIT(dev_priv))
9349                 return 120000;
9350         else if (!IS_GEN(dev_priv, 2))
9351                 return 96000;
9352         else
9353                 return 48000;
9354 }
9355
9356 /* Returns the clock of the currently programmed mode of the given pipe. */
9357 static void i9xx_crtc_clock_get(struct intel_crtc *crtc,
9358                                 struct intel_crtc_state *pipe_config)
9359 {
9360         struct drm_device *dev = crtc->base.dev;
9361         struct drm_i915_private *dev_priv = to_i915(dev);
9362         enum pipe pipe = crtc->pipe;
9363         u32 dpll = pipe_config->dpll_hw_state.dpll;
9364         u32 fp;
9365         struct dpll clock;
9366         int port_clock;
9367         int refclk = i9xx_pll_refclk(dev, pipe_config);
9368
9369         if ((dpll & DISPLAY_RATE_SELECT_FPA1) == 0)
9370                 fp = pipe_config->dpll_hw_state.fp0;
9371         else
9372                 fp = pipe_config->dpll_hw_state.fp1;
9373
9374         clock.m1 = (fp & FP_M1_DIV_MASK) >> FP_M1_DIV_SHIFT;
9375         if (IS_PINEVIEW(dev_priv)) {
9376                 clock.n = ffs((fp & FP_N_PINEVIEW_DIV_MASK) >> FP_N_DIV_SHIFT) - 1;
9377                 clock.m2 = (fp & FP_M2_PINEVIEW_DIV_MASK) >> FP_M2_DIV_SHIFT;
9378         } else {
9379                 clock.n = (fp & FP_N_DIV_MASK) >> FP_N_DIV_SHIFT;
9380                 clock.m2 = (fp & FP_M2_DIV_MASK) >> FP_M2_DIV_SHIFT;
9381         }
9382
9383         if (!IS_GEN(dev_priv, 2)) {
9384                 if (IS_PINEVIEW(dev_priv))
9385                         clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK_PINEVIEW) >>
9386                                 DPLL_FPA01_P1_POST_DIV_SHIFT_PINEVIEW);
9387                 else
9388                         clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK) >>
9389                                DPLL_FPA01_P1_POST_DIV_SHIFT);
9390
9391                 switch (dpll & DPLL_MODE_MASK) {
9392                 case DPLLB_MODE_DAC_SERIAL:
9393                         clock.p2 = dpll & DPLL_DAC_SERIAL_P2_CLOCK_DIV_5 ?
9394                                 5 : 10;
9395                         break;
9396                 case DPLLB_MODE_LVDS:
9397                         clock.p2 = dpll & DPLLB_LVDS_P2_CLOCK_DIV_7 ?
9398                                 7 : 14;
9399                         break;
9400                 default:
9401                         drm_dbg_kms(&dev_priv->drm,
9402                                     "Unknown DPLL mode %08x in programmed "
9403                                     "mode\n", (int)(dpll & DPLL_MODE_MASK));
9404                         return;
9405                 }
9406
9407                 if (IS_PINEVIEW(dev_priv))
9408                         port_clock = pnv_calc_dpll_params(refclk, &clock);
9409                 else
9410                         port_clock = i9xx_calc_dpll_params(refclk, &clock);
9411         } else {
9412                 u32 lvds = IS_I830(dev_priv) ? 0 : intel_de_read(dev_priv,
9413                                                                  LVDS);
9414                 bool is_lvds = (pipe == 1) && (lvds & LVDS_PORT_EN);
9415
9416                 if (is_lvds) {
9417                         clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK_I830_LVDS) >>
9418                                        DPLL_FPA01_P1_POST_DIV_SHIFT);
9419
9420                         if (lvds & LVDS_CLKB_POWER_UP)
9421                                 clock.p2 = 7;
9422                         else
9423                                 clock.p2 = 14;
9424                 } else {
9425                         if (dpll & PLL_P1_DIVIDE_BY_TWO)
9426                                 clock.p1 = 2;
9427                         else {
9428                                 clock.p1 = ((dpll & DPLL_FPA01_P1_POST_DIV_MASK_I830) >>
9429                                             DPLL_FPA01_P1_POST_DIV_SHIFT) + 2;
9430                         }
9431                         if (dpll & PLL_P2_DIVIDE_BY_4)
9432                                 clock.p2 = 4;
9433                         else
9434                                 clock.p2 = 2;
9435                 }
9436
9437                 port_clock = i9xx_calc_dpll_params(refclk, &clock);
9438         }
9439
9440         /*
9441          * This value includes pixel_multiplier. We will use
9442          * port_clock to compute adjusted_mode.crtc_clock in the
9443          * encoder's get_config() function.
9444          */
9445         pipe_config->port_clock = port_clock;
9446 }
9447
9448 int intel_dotclock_calculate(int link_freq,
9449                              const struct intel_link_m_n *m_n)
9450 {
9451         /*
9452          * The calculation for the data clock is:
9453          * pixel_clock = ((m/n)*(link_clock * nr_lanes))/bpp
9454          * But we want to avoid losing precison if possible, so:
9455          * pixel_clock = ((m * link_clock * nr_lanes)/(n*bpp))
9456          *
9457          * and the link clock is simpler:
9458          * link_clock = (m * link_clock) / n
9459          */
9460
9461         if (!m_n->link_n)
9462                 return 0;
9463
9464         return div_u64(mul_u32_u32(m_n->link_m, link_freq), m_n->link_n);
9465 }
9466
9467 static void ilk_pch_clock_get(struct intel_crtc *crtc,
9468                               struct intel_crtc_state *pipe_config)
9469 {
9470         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9471
9472         /* read out port_clock from the DPLL */
9473         i9xx_crtc_clock_get(crtc, pipe_config);
9474
9475         /*
9476          * In case there is an active pipe without active ports,
9477          * we may need some idea for the dotclock anyway.
9478          * Calculate one based on the FDI configuration.
9479          */
9480         pipe_config->hw.adjusted_mode.crtc_clock =
9481                 intel_dotclock_calculate(intel_fdi_link_freq(dev_priv, pipe_config),
9482                                          &pipe_config->fdi_m_n);
9483 }
9484
9485 /* Returns the currently programmed mode of the given encoder. */
9486 struct drm_display_mode *
9487 intel_encoder_current_mode(struct intel_encoder *encoder)
9488 {
9489         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
9490         struct intel_crtc_state *crtc_state;
9491         struct drm_display_mode *mode;
9492         struct intel_crtc *crtc;
9493         enum pipe pipe;
9494
9495         if (!encoder->get_hw_state(encoder, &pipe))
9496                 return NULL;
9497
9498         crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
9499
9500         mode = kzalloc(sizeof(*mode), GFP_KERNEL);
9501         if (!mode)
9502                 return NULL;
9503
9504         crtc_state = intel_crtc_state_alloc(crtc);
9505         if (!crtc_state) {
9506                 kfree(mode);
9507                 return NULL;
9508         }
9509
9510         if (!intel_crtc_get_pipe_config(crtc_state)) {
9511                 kfree(crtc_state);
9512                 kfree(mode);
9513                 return NULL;
9514         }
9515
9516         intel_encoder_get_config(encoder, crtc_state);
9517
9518         intel_mode_from_crtc_timings(mode, &crtc_state->hw.adjusted_mode);
9519
9520         kfree(crtc_state);
9521
9522         return mode;
9523 }
9524
9525 /**
9526  * intel_wm_need_update - Check whether watermarks need updating
9527  * @cur: current plane state
9528  * @new: new plane state
9529  *
9530  * Check current plane state versus the new one to determine whether
9531  * watermarks need to be recalculated.
9532  *
9533  * Returns true or false.
9534  */
9535 static bool intel_wm_need_update(const struct intel_plane_state *cur,
9536                                  struct intel_plane_state *new)
9537 {
9538         /* Update watermarks on tiling or size changes. */
9539         if (new->uapi.visible != cur->uapi.visible)
9540                 return true;
9541
9542         if (!cur->hw.fb || !new->hw.fb)
9543                 return false;
9544
9545         if (cur->hw.fb->modifier != new->hw.fb->modifier ||
9546             cur->hw.rotation != new->hw.rotation ||
9547             drm_rect_width(&new->uapi.src) != drm_rect_width(&cur->uapi.src) ||
9548             drm_rect_height(&new->uapi.src) != drm_rect_height(&cur->uapi.src) ||
9549             drm_rect_width(&new->uapi.dst) != drm_rect_width(&cur->uapi.dst) ||
9550             drm_rect_height(&new->uapi.dst) != drm_rect_height(&cur->uapi.dst))
9551                 return true;
9552
9553         return false;
9554 }
9555
9556 static bool needs_scaling(const struct intel_plane_state *state)
9557 {
9558         int src_w = drm_rect_width(&state->uapi.src) >> 16;
9559         int src_h = drm_rect_height(&state->uapi.src) >> 16;
9560         int dst_w = drm_rect_width(&state->uapi.dst);
9561         int dst_h = drm_rect_height(&state->uapi.dst);
9562
9563         return (src_w != dst_w || src_h != dst_h);
9564 }
9565
9566 int intel_plane_atomic_calc_changes(const struct intel_crtc_state *old_crtc_state,
9567                                     struct intel_crtc_state *crtc_state,
9568                                     const struct intel_plane_state *old_plane_state,
9569                                     struct intel_plane_state *plane_state)
9570 {
9571         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
9572         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
9573         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9574         bool mode_changed = intel_crtc_needs_modeset(crtc_state);
9575         bool was_crtc_enabled = old_crtc_state->hw.active;
9576         bool is_crtc_enabled = crtc_state->hw.active;
9577         bool turn_off, turn_on, visible, was_visible;
9578         int ret;
9579
9580         if (INTEL_GEN(dev_priv) >= 9 && plane->id != PLANE_CURSOR) {
9581                 ret = skl_update_scaler_plane(crtc_state, plane_state);
9582                 if (ret)
9583                         return ret;
9584         }
9585
9586         was_visible = old_plane_state->uapi.visible;
9587         visible = plane_state->uapi.visible;
9588
9589         if (!was_crtc_enabled && drm_WARN_ON(&dev_priv->drm, was_visible))
9590                 was_visible = false;
9591
9592         /*
9593          * Visibility is calculated as if the crtc was on, but
9594          * after scaler setup everything depends on it being off
9595          * when the crtc isn't active.
9596          *
9597          * FIXME this is wrong for watermarks. Watermarks should also
9598          * be computed as if the pipe would be active. Perhaps move
9599          * per-plane wm computation to the .check_plane() hook, and
9600          * only combine the results from all planes in the current place?
9601          */
9602         if (!is_crtc_enabled) {
9603                 intel_plane_set_invisible(crtc_state, plane_state);
9604                 visible = false;
9605         }
9606
9607         if (!was_visible && !visible)
9608                 return 0;
9609
9610         turn_off = was_visible && (!visible || mode_changed);
9611         turn_on = visible && (!was_visible || mode_changed);
9612
9613         drm_dbg_atomic(&dev_priv->drm,
9614                        "[CRTC:%d:%s] with [PLANE:%d:%s] visible %i -> %i, off %i, on %i, ms %i\n",
9615                        crtc->base.base.id, crtc->base.name,
9616                        plane->base.base.id, plane->base.name,
9617                        was_visible, visible,
9618                        turn_off, turn_on, mode_changed);
9619
9620         if (turn_on) {
9621                 if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv))
9622                         crtc_state->update_wm_pre = true;
9623
9624                 /* must disable cxsr around plane enable/disable */
9625                 if (plane->id != PLANE_CURSOR)
9626                         crtc_state->disable_cxsr = true;
9627         } else if (turn_off) {
9628                 if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv))
9629                         crtc_state->update_wm_post = true;
9630
9631                 /* must disable cxsr around plane enable/disable */
9632                 if (plane->id != PLANE_CURSOR)
9633                         crtc_state->disable_cxsr = true;
9634         } else if (intel_wm_need_update(old_plane_state, plane_state)) {
9635                 if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv)) {
9636                         /* FIXME bollocks */
9637                         crtc_state->update_wm_pre = true;
9638                         crtc_state->update_wm_post = true;
9639                 }
9640         }
9641
9642         if (visible || was_visible)
9643                 crtc_state->fb_bits |= plane->frontbuffer_bit;
9644
9645         /*
9646          * ILK/SNB DVSACNTR/Sprite Enable
9647          * IVB SPR_CTL/Sprite Enable
9648          * "When in Self Refresh Big FIFO mode, a write to enable the
9649          *  plane will be internally buffered and delayed while Big FIFO
9650          *  mode is exiting."
9651          *
9652          * Which means that enabling the sprite can take an extra frame
9653          * when we start in big FIFO mode (LP1+). Thus we need to drop
9654          * down to LP0 and wait for vblank in order to make sure the
9655          * sprite gets enabled on the next vblank after the register write.
9656          * Doing otherwise would risk enabling the sprite one frame after
9657          * we've already signalled flip completion. We can resume LP1+
9658          * once the sprite has been enabled.
9659          *
9660          *
9661          * WaCxSRDisabledForSpriteScaling:ivb
9662          * IVB SPR_SCALE/Scaling Enable
9663          * "Low Power watermarks must be disabled for at least one
9664          *  frame before enabling sprite scaling, and kept disabled
9665          *  until sprite scaling is disabled."
9666          *
9667          * ILK/SNB DVSASCALE/Scaling Enable
9668          * "When in Self Refresh Big FIFO mode, scaling enable will be
9669          *  masked off while Big FIFO mode is exiting."
9670          *
9671          * Despite the w/a only being listed for IVB we assume that
9672          * the ILK/SNB note has similar ramifications, hence we apply
9673          * the w/a on all three platforms.
9674          *
9675          * With experimental results seems this is needed also for primary
9676          * plane, not only sprite plane.
9677          */
9678         if (plane->id != PLANE_CURSOR &&
9679             (IS_GEN_RANGE(dev_priv, 5, 6) ||
9680              IS_IVYBRIDGE(dev_priv)) &&
9681             (turn_on || (!needs_scaling(old_plane_state) &&
9682                          needs_scaling(plane_state))))
9683                 crtc_state->disable_lp_wm = true;
9684
9685         return 0;
9686 }
9687
9688 static bool encoders_cloneable(const struct intel_encoder *a,
9689                                const struct intel_encoder *b)
9690 {
9691         /* masks could be asymmetric, so check both ways */
9692         return a == b || (a->cloneable & (1 << b->type) &&
9693                           b->cloneable & (1 << a->type));
9694 }
9695
9696 static bool check_single_encoder_cloning(struct intel_atomic_state *state,
9697                                          struct intel_crtc *crtc,
9698                                          struct intel_encoder *encoder)
9699 {
9700         struct intel_encoder *source_encoder;
9701         struct drm_connector *connector;
9702         struct drm_connector_state *connector_state;
9703         int i;
9704
9705         for_each_new_connector_in_state(&state->base, connector, connector_state, i) {
9706                 if (connector_state->crtc != &crtc->base)
9707                         continue;
9708
9709                 source_encoder =
9710                         to_intel_encoder(connector_state->best_encoder);
9711                 if (!encoders_cloneable(encoder, source_encoder))
9712                         return false;
9713         }
9714
9715         return true;
9716 }
9717
9718 static int icl_add_linked_planes(struct intel_atomic_state *state)
9719 {
9720         struct intel_plane *plane, *linked;
9721         struct intel_plane_state *plane_state, *linked_plane_state;
9722         int i;
9723
9724         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
9725                 linked = plane_state->planar_linked_plane;
9726
9727                 if (!linked)
9728                         continue;
9729
9730                 linked_plane_state = intel_atomic_get_plane_state(state, linked);
9731                 if (IS_ERR(linked_plane_state))
9732                         return PTR_ERR(linked_plane_state);
9733
9734                 drm_WARN_ON(state->base.dev,
9735                             linked_plane_state->planar_linked_plane != plane);
9736                 drm_WARN_ON(state->base.dev,
9737                             linked_plane_state->planar_slave == plane_state->planar_slave);
9738         }
9739
9740         return 0;
9741 }
9742
9743 static int icl_check_nv12_planes(struct intel_crtc_state *crtc_state)
9744 {
9745         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
9746         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9747         struct intel_atomic_state *state = to_intel_atomic_state(crtc_state->uapi.state);
9748         struct intel_plane *plane, *linked;
9749         struct intel_plane_state *plane_state;
9750         int i;
9751
9752         if (INTEL_GEN(dev_priv) < 11)
9753                 return 0;
9754
9755         /*
9756          * Destroy all old plane links and make the slave plane invisible
9757          * in the crtc_state->active_planes mask.
9758          */
9759         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
9760                 if (plane->pipe != crtc->pipe || !plane_state->planar_linked_plane)
9761                         continue;
9762
9763                 plane_state->planar_linked_plane = NULL;
9764                 if (plane_state->planar_slave && !plane_state->uapi.visible) {
9765                         crtc_state->enabled_planes &= ~BIT(plane->id);
9766                         crtc_state->active_planes &= ~BIT(plane->id);
9767                         crtc_state->update_planes |= BIT(plane->id);
9768                 }
9769
9770                 plane_state->planar_slave = false;
9771         }
9772
9773         if (!crtc_state->nv12_planes)
9774                 return 0;
9775
9776         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
9777                 struct intel_plane_state *linked_state = NULL;
9778
9779                 if (plane->pipe != crtc->pipe ||
9780                     !(crtc_state->nv12_planes & BIT(plane->id)))
9781                         continue;
9782
9783                 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, linked) {
9784                         if (!icl_is_nv12_y_plane(dev_priv, linked->id))
9785                                 continue;
9786
9787                         if (crtc_state->active_planes & BIT(linked->id))
9788                                 continue;
9789
9790                         linked_state = intel_atomic_get_plane_state(state, linked);
9791                         if (IS_ERR(linked_state))
9792                                 return PTR_ERR(linked_state);
9793
9794                         break;
9795                 }
9796
9797                 if (!linked_state) {
9798                         drm_dbg_kms(&dev_priv->drm,
9799                                     "Need %d free Y planes for planar YUV\n",
9800                                     hweight8(crtc_state->nv12_planes));
9801
9802                         return -EINVAL;
9803                 }
9804
9805                 plane_state->planar_linked_plane = linked;
9806
9807                 linked_state->planar_slave = true;
9808                 linked_state->planar_linked_plane = plane;
9809                 crtc_state->enabled_planes |= BIT(linked->id);
9810                 crtc_state->active_planes |= BIT(linked->id);
9811                 crtc_state->update_planes |= BIT(linked->id);
9812                 drm_dbg_kms(&dev_priv->drm, "Using %s as Y plane for %s\n",
9813                             linked->base.name, plane->base.name);
9814
9815                 /* Copy parameters to slave plane */
9816                 linked_state->ctl = plane_state->ctl | PLANE_CTL_YUV420_Y_PLANE;
9817                 linked_state->color_ctl = plane_state->color_ctl;
9818                 linked_state->view = plane_state->view;
9819                 memcpy(linked_state->color_plane, plane_state->color_plane,
9820                        sizeof(linked_state->color_plane));
9821
9822                 intel_plane_copy_hw_state(linked_state, plane_state);
9823                 linked_state->uapi.src = plane_state->uapi.src;
9824                 linked_state->uapi.dst = plane_state->uapi.dst;
9825
9826                 if (icl_is_hdr_plane(dev_priv, plane->id)) {
9827                         if (linked->id == PLANE_SPRITE5)
9828                                 plane_state->cus_ctl |= PLANE_CUS_PLANE_7;
9829                         else if (linked->id == PLANE_SPRITE4)
9830                                 plane_state->cus_ctl |= PLANE_CUS_PLANE_6;
9831                         else if (linked->id == PLANE_SPRITE3)
9832                                 plane_state->cus_ctl |= PLANE_CUS_PLANE_5_RKL;
9833                         else if (linked->id == PLANE_SPRITE2)
9834                                 plane_state->cus_ctl |= PLANE_CUS_PLANE_4_RKL;
9835                         else
9836                                 MISSING_CASE(linked->id);
9837                 }
9838         }
9839
9840         return 0;
9841 }
9842
9843 static bool c8_planes_changed(const struct intel_crtc_state *new_crtc_state)
9844 {
9845         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
9846         struct intel_atomic_state *state =
9847                 to_intel_atomic_state(new_crtc_state->uapi.state);
9848         const struct intel_crtc_state *old_crtc_state =
9849                 intel_atomic_get_old_crtc_state(state, crtc);
9850
9851         return !old_crtc_state->c8_planes != !new_crtc_state->c8_planes;
9852 }
9853
9854 static u16 hsw_linetime_wm(const struct intel_crtc_state *crtc_state)
9855 {
9856         const struct drm_display_mode *pipe_mode =
9857                 &crtc_state->hw.pipe_mode;
9858         int linetime_wm;
9859
9860         if (!crtc_state->hw.enable)
9861                 return 0;
9862
9863         linetime_wm = DIV_ROUND_CLOSEST(pipe_mode->crtc_htotal * 1000 * 8,
9864                                         pipe_mode->crtc_clock);
9865
9866         return min(linetime_wm, 0x1ff);
9867 }
9868
9869 static u16 hsw_ips_linetime_wm(const struct intel_crtc_state *crtc_state,
9870                                const struct intel_cdclk_state *cdclk_state)
9871 {
9872         const struct drm_display_mode *pipe_mode =
9873                 &crtc_state->hw.pipe_mode;
9874         int linetime_wm;
9875
9876         if (!crtc_state->hw.enable)
9877                 return 0;
9878
9879         linetime_wm = DIV_ROUND_CLOSEST(pipe_mode->crtc_htotal * 1000 * 8,
9880                                         cdclk_state->logical.cdclk);
9881
9882         return min(linetime_wm, 0x1ff);
9883 }
9884
9885 static u16 skl_linetime_wm(const struct intel_crtc_state *crtc_state)
9886 {
9887         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
9888         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9889         const struct drm_display_mode *pipe_mode =
9890                 &crtc_state->hw.pipe_mode;
9891         int linetime_wm;
9892
9893         if (!crtc_state->hw.enable)
9894                 return 0;
9895
9896         linetime_wm = DIV_ROUND_UP(pipe_mode->crtc_htotal * 1000 * 8,
9897                                    crtc_state->pixel_rate);
9898
9899         /* Display WA #1135: BXT:ALL GLK:ALL */
9900         if (IS_GEN9_LP(dev_priv) && dev_priv->ipc_enabled)
9901                 linetime_wm /= 2;
9902
9903         return min(linetime_wm, 0x1ff);
9904 }
9905
9906 static int hsw_compute_linetime_wm(struct intel_atomic_state *state,
9907                                    struct intel_crtc *crtc)
9908 {
9909         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9910         struct intel_crtc_state *crtc_state =
9911                 intel_atomic_get_new_crtc_state(state, crtc);
9912         const struct intel_cdclk_state *cdclk_state;
9913
9914         if (INTEL_GEN(dev_priv) >= 9)
9915                 crtc_state->linetime = skl_linetime_wm(crtc_state);
9916         else
9917                 crtc_state->linetime = hsw_linetime_wm(crtc_state);
9918
9919         if (!hsw_crtc_supports_ips(crtc))
9920                 return 0;
9921
9922         cdclk_state = intel_atomic_get_cdclk_state(state);
9923         if (IS_ERR(cdclk_state))
9924                 return PTR_ERR(cdclk_state);
9925
9926         crtc_state->ips_linetime = hsw_ips_linetime_wm(crtc_state,
9927                                                        cdclk_state);
9928
9929         return 0;
9930 }
9931
9932 static int intel_crtc_atomic_check(struct intel_atomic_state *state,
9933                                    struct intel_crtc *crtc)
9934 {
9935         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9936         struct intel_crtc_state *crtc_state =
9937                 intel_atomic_get_new_crtc_state(state, crtc);
9938         bool mode_changed = intel_crtc_needs_modeset(crtc_state);
9939         int ret;
9940
9941         if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv) &&
9942             mode_changed && !crtc_state->hw.active)
9943                 crtc_state->update_wm_post = true;
9944
9945         if (mode_changed && crtc_state->hw.enable &&
9946             dev_priv->display.crtc_compute_clock &&
9947             !crtc_state->bigjoiner_slave &&
9948             !drm_WARN_ON(&dev_priv->drm, crtc_state->shared_dpll)) {
9949                 ret = dev_priv->display.crtc_compute_clock(crtc, crtc_state);
9950                 if (ret)
9951                         return ret;
9952         }
9953
9954         /*
9955          * May need to update pipe gamma enable bits
9956          * when C8 planes are getting enabled/disabled.
9957          */
9958         if (c8_planes_changed(crtc_state))
9959                 crtc_state->uapi.color_mgmt_changed = true;
9960
9961         if (mode_changed || crtc_state->update_pipe ||
9962             crtc_state->uapi.color_mgmt_changed) {
9963                 ret = intel_color_check(crtc_state);
9964                 if (ret)
9965                         return ret;
9966         }
9967
9968         if (dev_priv->display.compute_pipe_wm) {
9969                 ret = dev_priv->display.compute_pipe_wm(crtc_state);
9970                 if (ret) {
9971                         drm_dbg_kms(&dev_priv->drm,
9972                                     "Target pipe watermarks are invalid\n");
9973                         return ret;
9974                 }
9975         }
9976
9977         if (dev_priv->display.compute_intermediate_wm) {
9978                 if (drm_WARN_ON(&dev_priv->drm,
9979                                 !dev_priv->display.compute_pipe_wm))
9980                         return 0;
9981
9982                 /*
9983                  * Calculate 'intermediate' watermarks that satisfy both the
9984                  * old state and the new state.  We can program these
9985                  * immediately.
9986                  */
9987                 ret = dev_priv->display.compute_intermediate_wm(crtc_state);
9988                 if (ret) {
9989                         drm_dbg_kms(&dev_priv->drm,
9990                                     "No valid intermediate pipe watermarks are possible\n");
9991                         return ret;
9992                 }
9993         }
9994
9995         if (INTEL_GEN(dev_priv) >= 9) {
9996                 if (mode_changed || crtc_state->update_pipe) {
9997                         ret = skl_update_scaler_crtc(crtc_state);
9998                         if (ret)
9999                                 return ret;
10000                 }
10001
10002                 ret = intel_atomic_setup_scalers(dev_priv, crtc, crtc_state);
10003                 if (ret)
10004                         return ret;
10005         }
10006
10007         if (HAS_IPS(dev_priv)) {
10008                 ret = hsw_compute_ips_config(crtc_state);
10009                 if (ret)
10010                         return ret;
10011         }
10012
10013         if (INTEL_GEN(dev_priv) >= 9 ||
10014             IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv)) {
10015                 ret = hsw_compute_linetime_wm(state, crtc);
10016                 if (ret)
10017                         return ret;
10018
10019         }
10020
10021         if (!mode_changed) {
10022                 ret = intel_psr2_sel_fetch_update(state, crtc);
10023                 if (ret)
10024                         return ret;
10025         }
10026
10027         return 0;
10028 }
10029
10030 static void intel_modeset_update_connector_atomic_state(struct drm_device *dev)
10031 {
10032         struct intel_connector *connector;
10033         struct drm_connector_list_iter conn_iter;
10034
10035         drm_connector_list_iter_begin(dev, &conn_iter);
10036         for_each_intel_connector_iter(connector, &conn_iter) {
10037                 if (connector->base.state->crtc)
10038                         drm_connector_put(&connector->base);
10039
10040                 if (connector->base.encoder) {
10041                         connector->base.state->best_encoder =
10042                                 connector->base.encoder;
10043                         connector->base.state->crtc =
10044                                 connector->base.encoder->crtc;
10045
10046                         drm_connector_get(&connector->base);
10047                 } else {
10048                         connector->base.state->best_encoder = NULL;
10049                         connector->base.state->crtc = NULL;
10050                 }
10051         }
10052         drm_connector_list_iter_end(&conn_iter);
10053 }
10054
10055 static int
10056 compute_sink_pipe_bpp(const struct drm_connector_state *conn_state,
10057                       struct intel_crtc_state *pipe_config)
10058 {
10059         struct drm_connector *connector = conn_state->connector;
10060         struct drm_i915_private *i915 = to_i915(pipe_config->uapi.crtc->dev);
10061         const struct drm_display_info *info = &connector->display_info;
10062         int bpp;
10063
10064         switch (conn_state->max_bpc) {
10065         case 6 ... 7:
10066                 bpp = 6 * 3;
10067                 break;
10068         case 8 ... 9:
10069                 bpp = 8 * 3;
10070                 break;
10071         case 10 ... 11:
10072                 bpp = 10 * 3;
10073                 break;
10074         case 12 ... 16:
10075                 bpp = 12 * 3;
10076                 break;
10077         default:
10078                 MISSING_CASE(conn_state->max_bpc);
10079                 return -EINVAL;
10080         }
10081
10082         if (bpp < pipe_config->pipe_bpp) {
10083                 drm_dbg_kms(&i915->drm,
10084                             "[CONNECTOR:%d:%s] Limiting display bpp to %d instead of "
10085                             "EDID bpp %d, requested bpp %d, max platform bpp %d\n",
10086                             connector->base.id, connector->name,
10087                             bpp, 3 * info->bpc,
10088                             3 * conn_state->max_requested_bpc,
10089                             pipe_config->pipe_bpp);
10090
10091                 pipe_config->pipe_bpp = bpp;
10092         }
10093
10094         return 0;
10095 }
10096
10097 static int
10098 compute_baseline_pipe_bpp(struct intel_crtc *crtc,
10099                           struct intel_crtc_state *pipe_config)
10100 {
10101         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
10102         struct drm_atomic_state *state = pipe_config->uapi.state;
10103         struct drm_connector *connector;
10104         struct drm_connector_state *connector_state;
10105         int bpp, i;
10106
10107         if ((IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
10108             IS_CHERRYVIEW(dev_priv)))
10109                 bpp = 10*3;
10110         else if (INTEL_GEN(dev_priv) >= 5)
10111                 bpp = 12*3;
10112         else
10113                 bpp = 8*3;
10114
10115         pipe_config->pipe_bpp = bpp;
10116
10117         /* Clamp display bpp to connector max bpp */
10118         for_each_new_connector_in_state(state, connector, connector_state, i) {
10119                 int ret;
10120
10121                 if (connector_state->crtc != &crtc->base)
10122                         continue;
10123
10124                 ret = compute_sink_pipe_bpp(connector_state, pipe_config);
10125                 if (ret)
10126                         return ret;
10127         }
10128
10129         return 0;
10130 }
10131
10132 static void intel_dump_crtc_timings(struct drm_i915_private *i915,
10133                                     const struct drm_display_mode *mode)
10134 {
10135         drm_dbg_kms(&i915->drm, "crtc timings: %d %d %d %d %d %d %d %d %d, "
10136                     "type: 0x%x flags: 0x%x\n",
10137                     mode->crtc_clock,
10138                     mode->crtc_hdisplay, mode->crtc_hsync_start,
10139                     mode->crtc_hsync_end, mode->crtc_htotal,
10140                     mode->crtc_vdisplay, mode->crtc_vsync_start,
10141                     mode->crtc_vsync_end, mode->crtc_vtotal,
10142                     mode->type, mode->flags);
10143 }
10144
10145 static void
10146 intel_dump_m_n_config(const struct intel_crtc_state *pipe_config,
10147                       const char *id, unsigned int lane_count,
10148                       const struct intel_link_m_n *m_n)
10149 {
10150         struct drm_i915_private *i915 = to_i915(pipe_config->uapi.crtc->dev);
10151
10152         drm_dbg_kms(&i915->drm,
10153                     "%s: lanes: %i; gmch_m: %u, gmch_n: %u, link_m: %u, link_n: %u, tu: %u\n",
10154                     id, lane_count,
10155                     m_n->gmch_m, m_n->gmch_n,
10156                     m_n->link_m, m_n->link_n, m_n->tu);
10157 }
10158
10159 static void
10160 intel_dump_infoframe(struct drm_i915_private *dev_priv,
10161                      const union hdmi_infoframe *frame)
10162 {
10163         if (!drm_debug_enabled(DRM_UT_KMS))
10164                 return;
10165
10166         hdmi_infoframe_log(KERN_DEBUG, dev_priv->drm.dev, frame);
10167 }
10168
10169 static void
10170 intel_dump_dp_vsc_sdp(struct drm_i915_private *dev_priv,
10171                       const struct drm_dp_vsc_sdp *vsc)
10172 {
10173         if (!drm_debug_enabled(DRM_UT_KMS))
10174                 return;
10175
10176         drm_dp_vsc_sdp_log(KERN_DEBUG, dev_priv->drm.dev, vsc);
10177 }
10178
10179 #define OUTPUT_TYPE(x) [INTEL_OUTPUT_ ## x] = #x
10180
10181 static const char * const output_type_str[] = {
10182         OUTPUT_TYPE(UNUSED),
10183         OUTPUT_TYPE(ANALOG),
10184         OUTPUT_TYPE(DVO),
10185         OUTPUT_TYPE(SDVO),
10186         OUTPUT_TYPE(LVDS),
10187         OUTPUT_TYPE(TVOUT),
10188         OUTPUT_TYPE(HDMI),
10189         OUTPUT_TYPE(DP),
10190         OUTPUT_TYPE(EDP),
10191         OUTPUT_TYPE(DSI),
10192         OUTPUT_TYPE(DDI),
10193         OUTPUT_TYPE(DP_MST),
10194 };
10195
10196 #undef OUTPUT_TYPE
10197
10198 static void snprintf_output_types(char *buf, size_t len,
10199                                   unsigned int output_types)
10200 {
10201         char *str = buf;
10202         int i;
10203
10204         str[0] = '\0';
10205
10206         for (i = 0; i < ARRAY_SIZE(output_type_str); i++) {
10207                 int r;
10208
10209                 if ((output_types & BIT(i)) == 0)
10210                         continue;
10211
10212                 r = snprintf(str, len, "%s%s",
10213                              str != buf ? "," : "", output_type_str[i]);
10214                 if (r >= len)
10215                         break;
10216                 str += r;
10217                 len -= r;
10218
10219                 output_types &= ~BIT(i);
10220         }
10221
10222         WARN_ON_ONCE(output_types != 0);
10223 }
10224
10225 static const char * const output_format_str[] = {
10226         [INTEL_OUTPUT_FORMAT_INVALID] = "Invalid",
10227         [INTEL_OUTPUT_FORMAT_RGB] = "RGB",
10228         [INTEL_OUTPUT_FORMAT_YCBCR420] = "YCBCR4:2:0",
10229         [INTEL_OUTPUT_FORMAT_YCBCR444] = "YCBCR4:4:4",
10230 };
10231
10232 static const char *output_formats(enum intel_output_format format)
10233 {
10234         if (format >= ARRAY_SIZE(output_format_str))
10235                 format = INTEL_OUTPUT_FORMAT_INVALID;
10236         return output_format_str[format];
10237 }
10238
10239 static void intel_dump_plane_state(const struct intel_plane_state *plane_state)
10240 {
10241         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
10242         struct drm_i915_private *i915 = to_i915(plane->base.dev);
10243         const struct drm_framebuffer *fb = plane_state->hw.fb;
10244         struct drm_format_name_buf format_name;
10245
10246         if (!fb) {
10247                 drm_dbg_kms(&i915->drm,
10248                             "[PLANE:%d:%s] fb: [NOFB], visible: %s\n",
10249                             plane->base.base.id, plane->base.name,
10250                             yesno(plane_state->uapi.visible));
10251                 return;
10252         }
10253
10254         drm_dbg_kms(&i915->drm,
10255                     "[PLANE:%d:%s] fb: [FB:%d] %ux%u format = %s modifier = 0x%llx, visible: %s\n",
10256                     plane->base.base.id, plane->base.name,
10257                     fb->base.id, fb->width, fb->height,
10258                     drm_get_format_name(fb->format->format, &format_name),
10259                     fb->modifier, yesno(plane_state->uapi.visible));
10260         drm_dbg_kms(&i915->drm, "\trotation: 0x%x, scaler: %d\n",
10261                     plane_state->hw.rotation, plane_state->scaler_id);
10262         if (plane_state->uapi.visible)
10263                 drm_dbg_kms(&i915->drm,
10264                             "\tsrc: " DRM_RECT_FP_FMT " dst: " DRM_RECT_FMT "\n",
10265                             DRM_RECT_FP_ARG(&plane_state->uapi.src),
10266                             DRM_RECT_ARG(&plane_state->uapi.dst));
10267 }
10268
10269 static void intel_dump_pipe_config(const struct intel_crtc_state *pipe_config,
10270                                    struct intel_atomic_state *state,
10271                                    const char *context)
10272 {
10273         struct intel_crtc *crtc = to_intel_crtc(pipe_config->uapi.crtc);
10274         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
10275         const struct intel_plane_state *plane_state;
10276         struct intel_plane *plane;
10277         char buf[64];
10278         int i;
10279
10280         drm_dbg_kms(&dev_priv->drm, "[CRTC:%d:%s] enable: %s %s\n",
10281                     crtc->base.base.id, crtc->base.name,
10282                     yesno(pipe_config->hw.enable), context);
10283
10284         if (!pipe_config->hw.enable)
10285                 goto dump_planes;
10286
10287         snprintf_output_types(buf, sizeof(buf), pipe_config->output_types);
10288         drm_dbg_kms(&dev_priv->drm,
10289                     "active: %s, output_types: %s (0x%x), output format: %s\n",
10290                     yesno(pipe_config->hw.active),
10291                     buf, pipe_config->output_types,
10292                     output_formats(pipe_config->output_format));
10293
10294         drm_dbg_kms(&dev_priv->drm,
10295                     "cpu_transcoder: %s, pipe bpp: %i, dithering: %i\n",
10296                     transcoder_name(pipe_config->cpu_transcoder),
10297                     pipe_config->pipe_bpp, pipe_config->dither);
10298
10299         drm_dbg_kms(&dev_priv->drm, "MST master transcoder: %s\n",
10300                     transcoder_name(pipe_config->mst_master_transcoder));
10301
10302         drm_dbg_kms(&dev_priv->drm,
10303                     "port sync: master transcoder: %s, slave transcoder bitmask = 0x%x\n",
10304                     transcoder_name(pipe_config->master_transcoder),
10305                     pipe_config->sync_mode_slaves_mask);
10306
10307         drm_dbg_kms(&dev_priv->drm, "bigjoiner: %s\n",
10308                     pipe_config->bigjoiner_slave ? "slave" :
10309                     pipe_config->bigjoiner ? "master" : "no");
10310
10311         if (pipe_config->has_pch_encoder)
10312                 intel_dump_m_n_config(pipe_config, "fdi",
10313                                       pipe_config->fdi_lanes,
10314                                       &pipe_config->fdi_m_n);
10315
10316         if (intel_crtc_has_dp_encoder(pipe_config)) {
10317                 intel_dump_m_n_config(pipe_config, "dp m_n",
10318                                 pipe_config->lane_count, &pipe_config->dp_m_n);
10319                 if (pipe_config->has_drrs)
10320                         intel_dump_m_n_config(pipe_config, "dp m2_n2",
10321                                               pipe_config->lane_count,
10322                                               &pipe_config->dp_m2_n2);
10323         }
10324
10325         drm_dbg_kms(&dev_priv->drm,
10326                     "audio: %i, infoframes: %i, infoframes enabled: 0x%x\n",
10327                     pipe_config->has_audio, pipe_config->has_infoframe,
10328                     pipe_config->infoframes.enable);
10329
10330         if (pipe_config->infoframes.enable &
10331             intel_hdmi_infoframe_enable(HDMI_PACKET_TYPE_GENERAL_CONTROL))
10332                 drm_dbg_kms(&dev_priv->drm, "GCP: 0x%x\n",
10333                             pipe_config->infoframes.gcp);
10334         if (pipe_config->infoframes.enable &
10335             intel_hdmi_infoframe_enable(HDMI_INFOFRAME_TYPE_AVI))
10336                 intel_dump_infoframe(dev_priv, &pipe_config->infoframes.avi);
10337         if (pipe_config->infoframes.enable &
10338             intel_hdmi_infoframe_enable(HDMI_INFOFRAME_TYPE_SPD))
10339                 intel_dump_infoframe(dev_priv, &pipe_config->infoframes.spd);
10340         if (pipe_config->infoframes.enable &
10341             intel_hdmi_infoframe_enable(HDMI_INFOFRAME_TYPE_VENDOR))
10342                 intel_dump_infoframe(dev_priv, &pipe_config->infoframes.hdmi);
10343         if (pipe_config->infoframes.enable &
10344             intel_hdmi_infoframe_enable(HDMI_INFOFRAME_TYPE_DRM))
10345                 intel_dump_infoframe(dev_priv, &pipe_config->infoframes.drm);
10346         if (pipe_config->infoframes.enable &
10347             intel_hdmi_infoframe_enable(HDMI_PACKET_TYPE_GAMUT_METADATA))
10348                 intel_dump_infoframe(dev_priv, &pipe_config->infoframes.drm);
10349         if (pipe_config->infoframes.enable &
10350             intel_hdmi_infoframe_enable(DP_SDP_VSC))
10351                 intel_dump_dp_vsc_sdp(dev_priv, &pipe_config->infoframes.vsc);
10352
10353         drm_dbg_kms(&dev_priv->drm, "vrr: %s, vmin: %d, vmax: %d, pipeline full: %d, flipline: %d, vmin vblank: %d, vmax vblank: %d\n",
10354                     yesno(pipe_config->vrr.enable),
10355                     pipe_config->vrr.vmin, pipe_config->vrr.vmax,
10356                     pipe_config->vrr.pipeline_full, pipe_config->vrr.flipline,
10357                     intel_vrr_vmin_vblank_start(pipe_config),
10358                     intel_vrr_vmax_vblank_start(pipe_config));
10359
10360         drm_dbg_kms(&dev_priv->drm, "requested mode:\n");
10361         drm_mode_debug_printmodeline(&pipe_config->hw.mode);
10362         drm_dbg_kms(&dev_priv->drm, "adjusted mode:\n");
10363         drm_mode_debug_printmodeline(&pipe_config->hw.adjusted_mode);
10364         intel_dump_crtc_timings(dev_priv, &pipe_config->hw.adjusted_mode);
10365         drm_dbg_kms(&dev_priv->drm, "pipe mode:\n");
10366         drm_mode_debug_printmodeline(&pipe_config->hw.pipe_mode);
10367         intel_dump_crtc_timings(dev_priv, &pipe_config->hw.pipe_mode);
10368         drm_dbg_kms(&dev_priv->drm,
10369                     "port clock: %d, pipe src size: %dx%d, pixel rate %d\n",
10370                     pipe_config->port_clock,
10371                     pipe_config->pipe_src_w, pipe_config->pipe_src_h,
10372                     pipe_config->pixel_rate);
10373
10374         drm_dbg_kms(&dev_priv->drm, "linetime: %d, ips linetime: %d\n",
10375                     pipe_config->linetime, pipe_config->ips_linetime);
10376
10377         if (INTEL_GEN(dev_priv) >= 9)
10378                 drm_dbg_kms(&dev_priv->drm,
10379                             "num_scalers: %d, scaler_users: 0x%x, scaler_id: %d\n",
10380                             crtc->num_scalers,
10381                             pipe_config->scaler_state.scaler_users,
10382                             pipe_config->scaler_state.scaler_id);
10383
10384         if (HAS_GMCH(dev_priv))
10385                 drm_dbg_kms(&dev_priv->drm,
10386                             "gmch pfit: control: 0x%08x, ratios: 0x%08x, lvds border: 0x%08x\n",
10387                             pipe_config->gmch_pfit.control,
10388                             pipe_config->gmch_pfit.pgm_ratios,
10389                             pipe_config->gmch_pfit.lvds_border_bits);
10390         else
10391                 drm_dbg_kms(&dev_priv->drm,
10392                             "pch pfit: " DRM_RECT_FMT ", %s, force thru: %s\n",
10393                             DRM_RECT_ARG(&pipe_config->pch_pfit.dst),
10394                             enableddisabled(pipe_config->pch_pfit.enabled),
10395                             yesno(pipe_config->pch_pfit.force_thru));
10396
10397         drm_dbg_kms(&dev_priv->drm, "ips: %i, double wide: %i\n",
10398                     pipe_config->ips_enabled, pipe_config->double_wide);
10399
10400         intel_dpll_dump_hw_state(dev_priv, &pipe_config->dpll_hw_state);
10401
10402         if (IS_CHERRYVIEW(dev_priv))
10403                 drm_dbg_kms(&dev_priv->drm,
10404                             "cgm_mode: 0x%x gamma_mode: 0x%x gamma_enable: %d csc_enable: %d\n",
10405                             pipe_config->cgm_mode, pipe_config->gamma_mode,
10406                             pipe_config->gamma_enable, pipe_config->csc_enable);
10407         else
10408                 drm_dbg_kms(&dev_priv->drm,
10409                             "csc_mode: 0x%x gamma_mode: 0x%x gamma_enable: %d csc_enable: %d\n",
10410                             pipe_config->csc_mode, pipe_config->gamma_mode,
10411                             pipe_config->gamma_enable, pipe_config->csc_enable);
10412
10413         drm_dbg_kms(&dev_priv->drm, "degamma lut: %d entries, gamma lut: %d entries\n",
10414                     pipe_config->hw.degamma_lut ?
10415                     drm_color_lut_size(pipe_config->hw.degamma_lut) : 0,
10416                     pipe_config->hw.gamma_lut ?
10417                     drm_color_lut_size(pipe_config->hw.gamma_lut) : 0);
10418
10419 dump_planes:
10420         if (!state)
10421                 return;
10422
10423         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
10424                 if (plane->pipe == crtc->pipe)
10425                         intel_dump_plane_state(plane_state);
10426         }
10427 }
10428
10429 static bool check_digital_port_conflicts(struct intel_atomic_state *state)
10430 {
10431         struct drm_device *dev = state->base.dev;
10432         struct drm_connector *connector;
10433         struct drm_connector_list_iter conn_iter;
10434         unsigned int used_ports = 0;
10435         unsigned int used_mst_ports = 0;
10436         bool ret = true;
10437
10438         /*
10439          * We're going to peek into connector->state,
10440          * hence connection_mutex must be held.
10441          */
10442         drm_modeset_lock_assert_held(&dev->mode_config.connection_mutex);
10443
10444         /*
10445          * Walk the connector list instead of the encoder
10446          * list to detect the problem on ddi platforms
10447          * where there's just one encoder per digital port.
10448          */
10449         drm_connector_list_iter_begin(dev, &conn_iter);
10450         drm_for_each_connector_iter(connector, &conn_iter) {
10451                 struct drm_connector_state *connector_state;
10452                 struct intel_encoder *encoder;
10453
10454                 connector_state =
10455                         drm_atomic_get_new_connector_state(&state->base,
10456                                                            connector);
10457                 if (!connector_state)
10458                         connector_state = connector->state;
10459
10460                 if (!connector_state->best_encoder)
10461                         continue;
10462
10463                 encoder = to_intel_encoder(connector_state->best_encoder);
10464
10465                 drm_WARN_ON(dev, !connector_state->crtc);
10466
10467                 switch (encoder->type) {
10468                 case INTEL_OUTPUT_DDI:
10469                         if (drm_WARN_ON(dev, !HAS_DDI(to_i915(dev))))
10470                                 break;
10471                         fallthrough;
10472                 case INTEL_OUTPUT_DP:
10473                 case INTEL_OUTPUT_HDMI:
10474                 case INTEL_OUTPUT_EDP:
10475                         /* the same port mustn't appear more than once */
10476                         if (used_ports & BIT(encoder->port))
10477                                 ret = false;
10478
10479                         used_ports |= BIT(encoder->port);
10480                         break;
10481                 case INTEL_OUTPUT_DP_MST:
10482                         used_mst_ports |=
10483                                 1 << encoder->port;
10484                         break;
10485                 default:
10486                         break;
10487                 }
10488         }
10489         drm_connector_list_iter_end(&conn_iter);
10490
10491         /* can't mix MST and SST/HDMI on the same port */
10492         if (used_ports & used_mst_ports)
10493                 return false;
10494
10495         return ret;
10496 }
10497
10498 static void
10499 intel_crtc_copy_uapi_to_hw_state_nomodeset(struct intel_atomic_state *state,
10500                                            struct intel_crtc_state *crtc_state)
10501 {
10502         const struct intel_crtc_state *from_crtc_state = crtc_state;
10503
10504         if (crtc_state->bigjoiner_slave) {
10505                 from_crtc_state = intel_atomic_get_new_crtc_state(state,
10506                                                                   crtc_state->bigjoiner_linked_crtc);
10507
10508                 /* No need to copy state if the master state is unchanged */
10509                 if (!from_crtc_state)
10510                         return;
10511         }
10512
10513         intel_crtc_copy_color_blobs(crtc_state, from_crtc_state);
10514 }
10515
10516 static void
10517 intel_crtc_copy_uapi_to_hw_state(struct intel_atomic_state *state,
10518                                  struct intel_crtc_state *crtc_state)
10519 {
10520         crtc_state->hw.enable = crtc_state->uapi.enable;
10521         crtc_state->hw.active = crtc_state->uapi.active;
10522         crtc_state->hw.mode = crtc_state->uapi.mode;
10523         crtc_state->hw.adjusted_mode = crtc_state->uapi.adjusted_mode;
10524         crtc_state->hw.scaling_filter = crtc_state->uapi.scaling_filter;
10525
10526         intel_crtc_copy_uapi_to_hw_state_nomodeset(state, crtc_state);
10527 }
10528
10529 static void intel_crtc_copy_hw_to_uapi_state(struct intel_crtc_state *crtc_state)
10530 {
10531         if (crtc_state->bigjoiner_slave)
10532                 return;
10533
10534         crtc_state->uapi.enable = crtc_state->hw.enable;
10535         crtc_state->uapi.active = crtc_state->hw.active;
10536         drm_WARN_ON(crtc_state->uapi.crtc->dev,
10537                     drm_atomic_set_mode_for_crtc(&crtc_state->uapi, &crtc_state->hw.mode) < 0);
10538
10539         crtc_state->uapi.adjusted_mode = crtc_state->hw.adjusted_mode;
10540         crtc_state->uapi.scaling_filter = crtc_state->hw.scaling_filter;
10541
10542         /* copy color blobs to uapi */
10543         drm_property_replace_blob(&crtc_state->uapi.degamma_lut,
10544                                   crtc_state->hw.degamma_lut);
10545         drm_property_replace_blob(&crtc_state->uapi.gamma_lut,
10546                                   crtc_state->hw.gamma_lut);
10547         drm_property_replace_blob(&crtc_state->uapi.ctm,
10548                                   crtc_state->hw.ctm);
10549 }
10550
10551 static int
10552 copy_bigjoiner_crtc_state(struct intel_crtc_state *crtc_state,
10553                           const struct intel_crtc_state *from_crtc_state)
10554 {
10555         struct intel_crtc_state *saved_state;
10556         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
10557
10558         saved_state = kmemdup(from_crtc_state, sizeof(*saved_state), GFP_KERNEL);
10559         if (!saved_state)
10560                 return -ENOMEM;
10561
10562         saved_state->uapi = crtc_state->uapi;
10563         saved_state->scaler_state = crtc_state->scaler_state;
10564         saved_state->shared_dpll = crtc_state->shared_dpll;
10565         saved_state->dpll_hw_state = crtc_state->dpll_hw_state;
10566         saved_state->crc_enabled = crtc_state->crc_enabled;
10567
10568         intel_crtc_free_hw_state(crtc_state);
10569         memcpy(crtc_state, saved_state, sizeof(*crtc_state));
10570         kfree(saved_state);
10571
10572         /* Re-init hw state */
10573         memset(&crtc_state->hw, 0, sizeof(saved_state->hw));
10574         crtc_state->hw.enable = from_crtc_state->hw.enable;
10575         crtc_state->hw.active = from_crtc_state->hw.active;
10576         crtc_state->hw.pipe_mode = from_crtc_state->hw.pipe_mode;
10577         crtc_state->hw.adjusted_mode = from_crtc_state->hw.adjusted_mode;
10578
10579         /* Some fixups */
10580         crtc_state->uapi.mode_changed = from_crtc_state->uapi.mode_changed;
10581         crtc_state->uapi.connectors_changed = from_crtc_state->uapi.connectors_changed;
10582         crtc_state->uapi.active_changed = from_crtc_state->uapi.active_changed;
10583         crtc_state->nv12_planes = crtc_state->c8_planes = crtc_state->update_planes = 0;
10584         crtc_state->bigjoiner_linked_crtc = to_intel_crtc(from_crtc_state->uapi.crtc);
10585         crtc_state->bigjoiner_slave = true;
10586         crtc_state->cpu_transcoder = (enum transcoder)crtc->pipe;
10587         crtc_state->has_audio = false;
10588
10589         return 0;
10590 }
10591
10592 static int
10593 intel_crtc_prepare_cleared_state(struct intel_atomic_state *state,
10594                                  struct intel_crtc_state *crtc_state)
10595 {
10596         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
10597         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
10598         struct intel_crtc_state *saved_state;
10599
10600         saved_state = intel_crtc_state_alloc(crtc);
10601         if (!saved_state)
10602                 return -ENOMEM;
10603
10604         /* free the old crtc_state->hw members */
10605         intel_crtc_free_hw_state(crtc_state);
10606
10607         /* FIXME: before the switch to atomic started, a new pipe_config was
10608          * kzalloc'd. Code that depends on any field being zero should be
10609          * fixed, so that the crtc_state can be safely duplicated. For now,
10610          * only fields that are know to not cause problems are preserved. */
10611
10612         saved_state->uapi = crtc_state->uapi;
10613         saved_state->scaler_state = crtc_state->scaler_state;
10614         saved_state->shared_dpll = crtc_state->shared_dpll;
10615         saved_state->dpll_hw_state = crtc_state->dpll_hw_state;
10616         memcpy(saved_state->icl_port_dplls, crtc_state->icl_port_dplls,
10617                sizeof(saved_state->icl_port_dplls));
10618         saved_state->crc_enabled = crtc_state->crc_enabled;
10619         if (IS_G4X(dev_priv) ||
10620             IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
10621                 saved_state->wm = crtc_state->wm;
10622
10623         memcpy(crtc_state, saved_state, sizeof(*crtc_state));
10624         kfree(saved_state);
10625
10626         intel_crtc_copy_uapi_to_hw_state(state, crtc_state);
10627
10628         return 0;
10629 }
10630
10631 static int
10632 intel_modeset_pipe_config(struct intel_atomic_state *state,
10633                           struct intel_crtc_state *pipe_config)
10634 {
10635         struct drm_crtc *crtc = pipe_config->uapi.crtc;
10636         struct drm_i915_private *i915 = to_i915(pipe_config->uapi.crtc->dev);
10637         struct drm_connector *connector;
10638         struct drm_connector_state *connector_state;
10639         int base_bpp, ret, i;
10640         bool retry = true;
10641
10642         pipe_config->cpu_transcoder =
10643                 (enum transcoder) to_intel_crtc(crtc)->pipe;
10644
10645         /*
10646          * Sanitize sync polarity flags based on requested ones. If neither
10647          * positive or negative polarity is requested, treat this as meaning
10648          * negative polarity.
10649          */
10650         if (!(pipe_config->hw.adjusted_mode.flags &
10651               (DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_NHSYNC)))
10652                 pipe_config->hw.adjusted_mode.flags |= DRM_MODE_FLAG_NHSYNC;
10653
10654         if (!(pipe_config->hw.adjusted_mode.flags &
10655               (DRM_MODE_FLAG_PVSYNC | DRM_MODE_FLAG_NVSYNC)))
10656                 pipe_config->hw.adjusted_mode.flags |= DRM_MODE_FLAG_NVSYNC;
10657
10658         ret = compute_baseline_pipe_bpp(to_intel_crtc(crtc),
10659                                         pipe_config);
10660         if (ret)
10661                 return ret;
10662
10663         base_bpp = pipe_config->pipe_bpp;
10664
10665         /*
10666          * Determine the real pipe dimensions. Note that stereo modes can
10667          * increase the actual pipe size due to the frame doubling and
10668          * insertion of additional space for blanks between the frame. This
10669          * is stored in the crtc timings. We use the requested mode to do this
10670          * computation to clearly distinguish it from the adjusted mode, which
10671          * can be changed by the connectors in the below retry loop.
10672          */
10673         drm_mode_get_hv_timing(&pipe_config->hw.mode,
10674                                &pipe_config->pipe_src_w,
10675                                &pipe_config->pipe_src_h);
10676
10677         for_each_new_connector_in_state(&state->base, connector, connector_state, i) {
10678                 struct intel_encoder *encoder =
10679                         to_intel_encoder(connector_state->best_encoder);
10680
10681                 if (connector_state->crtc != crtc)
10682                         continue;
10683
10684                 if (!check_single_encoder_cloning(state, to_intel_crtc(crtc), encoder)) {
10685                         drm_dbg_kms(&i915->drm,
10686                                     "rejecting invalid cloning configuration\n");
10687                         return -EINVAL;
10688                 }
10689
10690                 /*
10691                  * Determine output_types before calling the .compute_config()
10692                  * hooks so that the hooks can use this information safely.
10693                  */
10694                 if (encoder->compute_output_type)
10695                         pipe_config->output_types |=
10696                                 BIT(encoder->compute_output_type(encoder, pipe_config,
10697                                                                  connector_state));
10698                 else
10699                         pipe_config->output_types |= BIT(encoder->type);
10700         }
10701
10702 encoder_retry:
10703         /* Ensure the port clock defaults are reset when retrying. */
10704         pipe_config->port_clock = 0;
10705         pipe_config->pixel_multiplier = 1;
10706
10707         /* Fill in default crtc timings, allow encoders to overwrite them. */
10708         drm_mode_set_crtcinfo(&pipe_config->hw.adjusted_mode,
10709                               CRTC_STEREO_DOUBLE);
10710
10711         /* Pass our mode to the connectors and the CRTC to give them a chance to
10712          * adjust it according to limitations or connector properties, and also
10713          * a chance to reject the mode entirely.
10714          */
10715         for_each_new_connector_in_state(&state->base, connector, connector_state, i) {
10716                 struct intel_encoder *encoder =
10717                         to_intel_encoder(connector_state->best_encoder);
10718
10719                 if (connector_state->crtc != crtc)
10720                         continue;
10721
10722                 ret = encoder->compute_config(encoder, pipe_config,
10723                                               connector_state);
10724                 if (ret < 0) {
10725                         if (ret != -EDEADLK)
10726                                 drm_dbg_kms(&i915->drm,
10727                                             "Encoder config failure: %d\n",
10728                                             ret);
10729                         return ret;
10730                 }
10731         }
10732
10733         /* Set default port clock if not overwritten by the encoder. Needs to be
10734          * done afterwards in case the encoder adjusts the mode. */
10735         if (!pipe_config->port_clock)
10736                 pipe_config->port_clock = pipe_config->hw.adjusted_mode.crtc_clock
10737                         * pipe_config->pixel_multiplier;
10738
10739         ret = intel_crtc_compute_config(to_intel_crtc(crtc), pipe_config);
10740         if (ret == -EDEADLK)
10741                 return ret;
10742         if (ret < 0) {
10743                 drm_dbg_kms(&i915->drm, "CRTC fixup failed\n");
10744                 return ret;
10745         }
10746
10747         if (ret == I915_DISPLAY_CONFIG_RETRY) {
10748                 if (drm_WARN(&i915->drm, !retry,
10749                              "loop in pipe configuration computation\n"))
10750                         return -EINVAL;
10751
10752                 drm_dbg_kms(&i915->drm, "CRTC bw constrained, retrying\n");
10753                 retry = false;
10754                 goto encoder_retry;
10755         }
10756
10757         /* Dithering seems to not pass-through bits correctly when it should, so
10758          * only enable it on 6bpc panels and when its not a compliance
10759          * test requesting 6bpc video pattern.
10760          */
10761         pipe_config->dither = (pipe_config->pipe_bpp == 6*3) &&
10762                 !pipe_config->dither_force_disable;
10763         drm_dbg_kms(&i915->drm,
10764                     "hw max bpp: %i, pipe bpp: %i, dithering: %i\n",
10765                     base_bpp, pipe_config->pipe_bpp, pipe_config->dither);
10766
10767         return 0;
10768 }
10769
10770 static int
10771 intel_modeset_pipe_config_late(struct intel_crtc_state *crtc_state)
10772 {
10773         struct intel_atomic_state *state =
10774                 to_intel_atomic_state(crtc_state->uapi.state);
10775         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
10776         struct drm_connector_state *conn_state;
10777         struct drm_connector *connector;
10778         int i;
10779
10780         for_each_new_connector_in_state(&state->base, connector,
10781                                         conn_state, i) {
10782                 struct intel_encoder *encoder =
10783                         to_intel_encoder(conn_state->best_encoder);
10784                 int ret;
10785
10786                 if (conn_state->crtc != &crtc->base ||
10787                     !encoder->compute_config_late)
10788                         continue;
10789
10790                 ret = encoder->compute_config_late(encoder, crtc_state,
10791                                                    conn_state);
10792                 if (ret)
10793                         return ret;
10794         }
10795
10796         return 0;
10797 }
10798
10799 bool intel_fuzzy_clock_check(int clock1, int clock2)
10800 {
10801         int diff;
10802
10803         if (clock1 == clock2)
10804                 return true;
10805
10806         if (!clock1 || !clock2)
10807                 return false;
10808
10809         diff = abs(clock1 - clock2);
10810
10811         if (((((diff + clock1 + clock2) * 100)) / (clock1 + clock2)) < 105)
10812                 return true;
10813
10814         return false;
10815 }
10816
10817 static bool
10818 intel_compare_m_n(unsigned int m, unsigned int n,
10819                   unsigned int m2, unsigned int n2,
10820                   bool exact)
10821 {
10822         if (m == m2 && n == n2)
10823                 return true;
10824
10825         if (exact || !m || !n || !m2 || !n2)
10826                 return false;
10827
10828         BUILD_BUG_ON(DATA_LINK_M_N_MASK > INT_MAX);
10829
10830         if (n > n2) {
10831                 while (n > n2) {
10832                         m2 <<= 1;
10833                         n2 <<= 1;
10834                 }
10835         } else if (n < n2) {
10836                 while (n < n2) {
10837                         m <<= 1;
10838                         n <<= 1;
10839                 }
10840         }
10841
10842         if (n != n2)
10843                 return false;
10844
10845         return intel_fuzzy_clock_check(m, m2);
10846 }
10847
10848 static bool
10849 intel_compare_link_m_n(const struct intel_link_m_n *m_n,
10850                        const struct intel_link_m_n *m2_n2,
10851                        bool exact)
10852 {
10853         return m_n->tu == m2_n2->tu &&
10854                 intel_compare_m_n(m_n->gmch_m, m_n->gmch_n,
10855                                   m2_n2->gmch_m, m2_n2->gmch_n, exact) &&
10856                 intel_compare_m_n(m_n->link_m, m_n->link_n,
10857                                   m2_n2->link_m, m2_n2->link_n, exact);
10858 }
10859
10860 static bool
10861 intel_compare_infoframe(const union hdmi_infoframe *a,
10862                         const union hdmi_infoframe *b)
10863 {
10864         return memcmp(a, b, sizeof(*a)) == 0;
10865 }
10866
10867 static bool
10868 intel_compare_dp_vsc_sdp(const struct drm_dp_vsc_sdp *a,
10869                          const struct drm_dp_vsc_sdp *b)
10870 {
10871         return memcmp(a, b, sizeof(*a)) == 0;
10872 }
10873
10874 static void
10875 pipe_config_infoframe_mismatch(struct drm_i915_private *dev_priv,
10876                                bool fastset, const char *name,
10877                                const union hdmi_infoframe *a,
10878                                const union hdmi_infoframe *b)
10879 {
10880         if (fastset) {
10881                 if (!drm_debug_enabled(DRM_UT_KMS))
10882                         return;
10883
10884                 drm_dbg_kms(&dev_priv->drm,
10885                             "fastset mismatch in %s infoframe\n", name);
10886                 drm_dbg_kms(&dev_priv->drm, "expected:\n");
10887                 hdmi_infoframe_log(KERN_DEBUG, dev_priv->drm.dev, a);
10888                 drm_dbg_kms(&dev_priv->drm, "found:\n");
10889                 hdmi_infoframe_log(KERN_DEBUG, dev_priv->drm.dev, b);
10890         } else {
10891                 drm_err(&dev_priv->drm, "mismatch in %s infoframe\n", name);
10892                 drm_err(&dev_priv->drm, "expected:\n");
10893                 hdmi_infoframe_log(KERN_ERR, dev_priv->drm.dev, a);
10894                 drm_err(&dev_priv->drm, "found:\n");
10895                 hdmi_infoframe_log(KERN_ERR, dev_priv->drm.dev, b);
10896         }
10897 }
10898
10899 static void
10900 pipe_config_dp_vsc_sdp_mismatch(struct drm_i915_private *dev_priv,
10901                                 bool fastset, const char *name,
10902                                 const struct drm_dp_vsc_sdp *a,
10903                                 const struct drm_dp_vsc_sdp *b)
10904 {
10905         if (fastset) {
10906                 if (!drm_debug_enabled(DRM_UT_KMS))
10907                         return;
10908
10909                 drm_dbg_kms(&dev_priv->drm,
10910                             "fastset mismatch in %s dp sdp\n", name);
10911                 drm_dbg_kms(&dev_priv->drm, "expected:\n");
10912                 drm_dp_vsc_sdp_log(KERN_DEBUG, dev_priv->drm.dev, a);
10913                 drm_dbg_kms(&dev_priv->drm, "found:\n");
10914                 drm_dp_vsc_sdp_log(KERN_DEBUG, dev_priv->drm.dev, b);
10915         } else {
10916                 drm_err(&dev_priv->drm, "mismatch in %s dp sdp\n", name);
10917                 drm_err(&dev_priv->drm, "expected:\n");
10918                 drm_dp_vsc_sdp_log(KERN_ERR, dev_priv->drm.dev, a);
10919                 drm_err(&dev_priv->drm, "found:\n");
10920                 drm_dp_vsc_sdp_log(KERN_ERR, dev_priv->drm.dev, b);
10921         }
10922 }
10923
10924 static void __printf(4, 5)
10925 pipe_config_mismatch(bool fastset, const struct intel_crtc *crtc,
10926                      const char *name, const char *format, ...)
10927 {
10928         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
10929         struct va_format vaf;
10930         va_list args;
10931
10932         va_start(args, format);
10933         vaf.fmt = format;
10934         vaf.va = &args;
10935
10936         if (fastset)
10937                 drm_dbg_kms(&i915->drm,
10938                             "[CRTC:%d:%s] fastset mismatch in %s %pV\n",
10939                             crtc->base.base.id, crtc->base.name, name, &vaf);
10940         else
10941                 drm_err(&i915->drm, "[CRTC:%d:%s] mismatch in %s %pV\n",
10942                         crtc->base.base.id, crtc->base.name, name, &vaf);
10943
10944         va_end(args);
10945 }
10946
10947 static bool fastboot_enabled(struct drm_i915_private *dev_priv)
10948 {
10949         if (dev_priv->params.fastboot != -1)
10950                 return dev_priv->params.fastboot;
10951
10952         /* Enable fastboot by default on Skylake and newer */
10953         if (INTEL_GEN(dev_priv) >= 9)
10954                 return true;
10955
10956         /* Enable fastboot by default on VLV and CHV */
10957         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
10958                 return true;
10959
10960         /* Disabled by default on all others */
10961         return false;
10962 }
10963
10964 static bool
10965 intel_pipe_config_compare(const struct intel_crtc_state *current_config,
10966                           const struct intel_crtc_state *pipe_config,
10967                           bool fastset)
10968 {
10969         struct drm_i915_private *dev_priv = to_i915(current_config->uapi.crtc->dev);
10970         struct intel_crtc *crtc = to_intel_crtc(pipe_config->uapi.crtc);
10971         bool ret = true;
10972         u32 bp_gamma = 0;
10973         bool fixup_inherited = fastset &&
10974                 current_config->inherited && !pipe_config->inherited;
10975
10976         if (fixup_inherited && !fastboot_enabled(dev_priv)) {
10977                 drm_dbg_kms(&dev_priv->drm,
10978                             "initial modeset and fastboot not set\n");
10979                 ret = false;
10980         }
10981
10982 #define PIPE_CONF_CHECK_X(name) do { \
10983         if (current_config->name != pipe_config->name) { \
10984                 pipe_config_mismatch(fastset, crtc, __stringify(name), \
10985                                      "(expected 0x%08x, found 0x%08x)", \
10986                                      current_config->name, \
10987                                      pipe_config->name); \
10988                 ret = false; \
10989         } \
10990 } while (0)
10991
10992 #define PIPE_CONF_CHECK_I(name) do { \
10993         if (current_config->name != pipe_config->name) { \
10994                 pipe_config_mismatch(fastset, crtc, __stringify(name), \
10995                                      "(expected %i, found %i)", \
10996                                      current_config->name, \
10997                                      pipe_config->name); \
10998                 ret = false; \
10999         } \
11000 } while (0)
11001
11002 #define PIPE_CONF_CHECK_BOOL(name) do { \
11003         if (current_config->name != pipe_config->name) { \
11004                 pipe_config_mismatch(fastset, crtc,  __stringify(name), \
11005                                      "(expected %s, found %s)", \
11006                                      yesno(current_config->name), \
11007                                      yesno(pipe_config->name)); \
11008                 ret = false; \
11009         } \
11010 } while (0)
11011
11012 /*
11013  * Checks state where we only read out the enabling, but not the entire
11014  * state itself (like full infoframes or ELD for audio). These states
11015  * require a full modeset on bootup to fix up.
11016  */
11017 #define PIPE_CONF_CHECK_BOOL_INCOMPLETE(name) do { \
11018         if (!fixup_inherited || (!current_config->name && !pipe_config->name)) { \
11019                 PIPE_CONF_CHECK_BOOL(name); \
11020         } else { \
11021                 pipe_config_mismatch(fastset, crtc, __stringify(name), \
11022                                      "unable to verify whether state matches exactly, forcing modeset (expected %s, found %s)", \
11023                                      yesno(current_config->name), \
11024                                      yesno(pipe_config->name)); \
11025                 ret = false; \
11026         } \
11027 } while (0)
11028
11029 #define PIPE_CONF_CHECK_P(name) do { \
11030         if (current_config->name != pipe_config->name) { \
11031                 pipe_config_mismatch(fastset, crtc, __stringify(name), \
11032                                      "(expected %p, found %p)", \
11033                                      current_config->name, \
11034                                      pipe_config->name); \
11035                 ret = false; \
11036         } \
11037 } while (0)
11038
11039 #define PIPE_CONF_CHECK_M_N(name) do { \
11040         if (!intel_compare_link_m_n(&current_config->name, \
11041                                     &pipe_config->name,\
11042                                     !fastset)) { \
11043                 pipe_config_mismatch(fastset, crtc, __stringify(name), \
11044                                      "(expected tu %i gmch %i/%i link %i/%i, " \
11045                                      "found tu %i, gmch %i/%i link %i/%i)", \
11046                                      current_config->name.tu, \
11047                                      current_config->name.gmch_m, \
11048                                      current_config->name.gmch_n, \
11049                                      current_config->name.link_m, \
11050                                      current_config->name.link_n, \
11051                                      pipe_config->name.tu, \
11052                                      pipe_config->name.gmch_m, \
11053                                      pipe_config->name.gmch_n, \
11054                                      pipe_config->name.link_m, \
11055                                      pipe_config->name.link_n); \
11056                 ret = false; \
11057         } \
11058 } while (0)
11059
11060 /* This is required for BDW+ where there is only one set of registers for
11061  * switching between high and low RR.
11062  * This macro can be used whenever a comparison has to be made between one
11063  * hw state and multiple sw state variables.
11064  */
11065 #define PIPE_CONF_CHECK_M_N_ALT(name, alt_name) do { \
11066         if (!intel_compare_link_m_n(&current_config->name, \
11067                                     &pipe_config->name, !fastset) && \
11068             !intel_compare_link_m_n(&current_config->alt_name, \
11069                                     &pipe_config->name, !fastset)) { \
11070                 pipe_config_mismatch(fastset, crtc, __stringify(name), \
11071                                      "(expected tu %i gmch %i/%i link %i/%i, " \
11072                                      "or tu %i gmch %i/%i link %i/%i, " \
11073                                      "found tu %i, gmch %i/%i link %i/%i)", \
11074                                      current_config->name.tu, \
11075                                      current_config->name.gmch_m, \
11076                                      current_config->name.gmch_n, \
11077                                      current_config->name.link_m, \
11078                                      current_config->name.link_n, \
11079                                      current_config->alt_name.tu, \
11080                                      current_config->alt_name.gmch_m, \
11081                                      current_config->alt_name.gmch_n, \
11082                                      current_config->alt_name.link_m, \
11083                                      current_config->alt_name.link_n, \
11084                                      pipe_config->name.tu, \
11085                                      pipe_config->name.gmch_m, \
11086                                      pipe_config->name.gmch_n, \
11087                                      pipe_config->name.link_m, \
11088                                      pipe_config->name.link_n); \
11089                 ret = false; \
11090         } \
11091 } while (0)
11092
11093 #define PIPE_CONF_CHECK_FLAGS(name, mask) do { \
11094         if ((current_config->name ^ pipe_config->name) & (mask)) { \
11095                 pipe_config_mismatch(fastset, crtc, __stringify(name), \
11096                                      "(%x) (expected %i, found %i)", \
11097                                      (mask), \
11098                                      current_config->name & (mask), \
11099                                      pipe_config->name & (mask)); \
11100                 ret = false; \
11101         } \
11102 } while (0)
11103
11104 #define PIPE_CONF_CHECK_CLOCK_FUZZY(name) do { \
11105         if (!intel_fuzzy_clock_check(current_config->name, pipe_config->name)) { \
11106                 pipe_config_mismatch(fastset, crtc, __stringify(name), \
11107                                      "(expected %i, found %i)", \
11108                                      current_config->name, \
11109                                      pipe_config->name); \
11110                 ret = false; \
11111         } \
11112 } while (0)
11113
11114 #define PIPE_CONF_CHECK_INFOFRAME(name) do { \
11115         if (!intel_compare_infoframe(&current_config->infoframes.name, \
11116                                      &pipe_config->infoframes.name)) { \
11117                 pipe_config_infoframe_mismatch(dev_priv, fastset, __stringify(name), \
11118                                                &current_config->infoframes.name, \
11119                                                &pipe_config->infoframes.name); \
11120                 ret = false; \
11121         } \
11122 } while (0)
11123
11124 #define PIPE_CONF_CHECK_DP_VSC_SDP(name) do { \
11125         if (!current_config->has_psr && !pipe_config->has_psr && \
11126             !intel_compare_dp_vsc_sdp(&current_config->infoframes.name, \
11127                                       &pipe_config->infoframes.name)) { \
11128                 pipe_config_dp_vsc_sdp_mismatch(dev_priv, fastset, __stringify(name), \
11129                                                 &current_config->infoframes.name, \
11130                                                 &pipe_config->infoframes.name); \
11131                 ret = false; \
11132         } \
11133 } while (0)
11134
11135 #define PIPE_CONF_CHECK_COLOR_LUT(name1, name2, bit_precision) do { \
11136         if (current_config->name1 != pipe_config->name1) { \
11137                 pipe_config_mismatch(fastset, crtc, __stringify(name1), \
11138                                 "(expected %i, found %i, won't compare lut values)", \
11139                                 current_config->name1, \
11140                                 pipe_config->name1); \
11141                 ret = false;\
11142         } else { \
11143                 if (!intel_color_lut_equal(current_config->name2, \
11144                                         pipe_config->name2, pipe_config->name1, \
11145                                         bit_precision)) { \
11146                         pipe_config_mismatch(fastset, crtc, __stringify(name2), \
11147                                         "hw_state doesn't match sw_state"); \
11148                         ret = false; \
11149                 } \
11150         } \
11151 } while (0)
11152
11153 #define PIPE_CONF_QUIRK(quirk) \
11154         ((current_config->quirks | pipe_config->quirks) & (quirk))
11155
11156         PIPE_CONF_CHECK_I(cpu_transcoder);
11157
11158         PIPE_CONF_CHECK_BOOL(has_pch_encoder);
11159         PIPE_CONF_CHECK_I(fdi_lanes);
11160         PIPE_CONF_CHECK_M_N(fdi_m_n);
11161
11162         PIPE_CONF_CHECK_I(lane_count);
11163         PIPE_CONF_CHECK_X(lane_lat_optim_mask);
11164
11165         if (INTEL_GEN(dev_priv) < 8) {
11166                 PIPE_CONF_CHECK_M_N(dp_m_n);
11167
11168                 if (current_config->has_drrs)
11169                         PIPE_CONF_CHECK_M_N(dp_m2_n2);
11170         } else
11171                 PIPE_CONF_CHECK_M_N_ALT(dp_m_n, dp_m2_n2);
11172
11173         PIPE_CONF_CHECK_X(output_types);
11174
11175         /* FIXME do the readout properly and get rid of this quirk */
11176         if (!PIPE_CONF_QUIRK(PIPE_CONFIG_QUIRK_BIGJOINER_SLAVE)) {
11177                 PIPE_CONF_CHECK_I(hw.pipe_mode.crtc_hdisplay);
11178                 PIPE_CONF_CHECK_I(hw.pipe_mode.crtc_htotal);
11179                 PIPE_CONF_CHECK_I(hw.pipe_mode.crtc_hblank_start);
11180                 PIPE_CONF_CHECK_I(hw.pipe_mode.crtc_hblank_end);
11181                 PIPE_CONF_CHECK_I(hw.pipe_mode.crtc_hsync_start);
11182                 PIPE_CONF_CHECK_I(hw.pipe_mode.crtc_hsync_end);
11183
11184                 PIPE_CONF_CHECK_I(hw.pipe_mode.crtc_vdisplay);
11185                 PIPE_CONF_CHECK_I(hw.pipe_mode.crtc_vtotal);
11186                 PIPE_CONF_CHECK_I(hw.pipe_mode.crtc_vblank_start);
11187                 PIPE_CONF_CHECK_I(hw.pipe_mode.crtc_vblank_end);
11188                 PIPE_CONF_CHECK_I(hw.pipe_mode.crtc_vsync_start);
11189                 PIPE_CONF_CHECK_I(hw.pipe_mode.crtc_vsync_end);
11190
11191                 PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_hdisplay);
11192                 PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_htotal);
11193                 PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_hblank_start);
11194                 PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_hblank_end);
11195                 PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_hsync_start);
11196                 PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_hsync_end);
11197
11198                 PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_vdisplay);
11199                 PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_vtotal);
11200                 PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_vblank_start);
11201                 PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_vblank_end);
11202                 PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_vsync_start);
11203                 PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_vsync_end);
11204
11205                 PIPE_CONF_CHECK_I(pixel_multiplier);
11206
11207                 PIPE_CONF_CHECK_FLAGS(hw.adjusted_mode.flags,
11208                                       DRM_MODE_FLAG_INTERLACE);
11209
11210                 if (!PIPE_CONF_QUIRK(PIPE_CONFIG_QUIRK_MODE_SYNC_FLAGS)) {
11211                         PIPE_CONF_CHECK_FLAGS(hw.adjusted_mode.flags,
11212                                               DRM_MODE_FLAG_PHSYNC);
11213                         PIPE_CONF_CHECK_FLAGS(hw.adjusted_mode.flags,
11214                                               DRM_MODE_FLAG_NHSYNC);
11215                         PIPE_CONF_CHECK_FLAGS(hw.adjusted_mode.flags,
11216                                               DRM_MODE_FLAG_PVSYNC);
11217                         PIPE_CONF_CHECK_FLAGS(hw.adjusted_mode.flags,
11218                                               DRM_MODE_FLAG_NVSYNC);
11219                 }
11220         }
11221
11222         PIPE_CONF_CHECK_I(output_format);
11223         PIPE_CONF_CHECK_BOOL(has_hdmi_sink);
11224         if ((INTEL_GEN(dev_priv) < 8 && !IS_HASWELL(dev_priv)) ||
11225             IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
11226                 PIPE_CONF_CHECK_BOOL(limited_color_range);
11227
11228         PIPE_CONF_CHECK_BOOL(hdmi_scrambling);
11229         PIPE_CONF_CHECK_BOOL(hdmi_high_tmds_clock_ratio);
11230         PIPE_CONF_CHECK_BOOL(has_infoframe);
11231         /* FIXME do the readout properly and get rid of this quirk */
11232         if (!PIPE_CONF_QUIRK(PIPE_CONFIG_QUIRK_BIGJOINER_SLAVE))
11233                 PIPE_CONF_CHECK_BOOL(fec_enable);
11234
11235         PIPE_CONF_CHECK_BOOL_INCOMPLETE(has_audio);
11236
11237         PIPE_CONF_CHECK_X(gmch_pfit.control);
11238         /* pfit ratios are autocomputed by the hw on gen4+ */
11239         if (INTEL_GEN(dev_priv) < 4)
11240                 PIPE_CONF_CHECK_X(gmch_pfit.pgm_ratios);
11241         PIPE_CONF_CHECK_X(gmch_pfit.lvds_border_bits);
11242
11243         /*
11244          * Changing the EDP transcoder input mux
11245          * (A_ONOFF vs. A_ON) requires a full modeset.
11246          */
11247         PIPE_CONF_CHECK_BOOL(pch_pfit.force_thru);
11248
11249         if (!fastset) {
11250                 PIPE_CONF_CHECK_I(pipe_src_w);
11251                 PIPE_CONF_CHECK_I(pipe_src_h);
11252
11253                 PIPE_CONF_CHECK_BOOL(pch_pfit.enabled);
11254                 if (current_config->pch_pfit.enabled) {
11255                         PIPE_CONF_CHECK_I(pch_pfit.dst.x1);
11256                         PIPE_CONF_CHECK_I(pch_pfit.dst.y1);
11257                         PIPE_CONF_CHECK_I(pch_pfit.dst.x2);
11258                         PIPE_CONF_CHECK_I(pch_pfit.dst.y2);
11259                 }
11260
11261                 PIPE_CONF_CHECK_I(scaler_state.scaler_id);
11262                 /* FIXME do the readout properly and get rid of this quirk */
11263                 if (!PIPE_CONF_QUIRK(PIPE_CONFIG_QUIRK_BIGJOINER_SLAVE))
11264                         PIPE_CONF_CHECK_CLOCK_FUZZY(pixel_rate);
11265
11266                 PIPE_CONF_CHECK_X(gamma_mode);
11267                 if (IS_CHERRYVIEW(dev_priv))
11268                         PIPE_CONF_CHECK_X(cgm_mode);
11269                 else
11270                         PIPE_CONF_CHECK_X(csc_mode);
11271                 PIPE_CONF_CHECK_BOOL(gamma_enable);
11272                 PIPE_CONF_CHECK_BOOL(csc_enable);
11273
11274                 PIPE_CONF_CHECK_I(linetime);
11275                 PIPE_CONF_CHECK_I(ips_linetime);
11276
11277                 bp_gamma = intel_color_get_gamma_bit_precision(pipe_config);
11278                 if (bp_gamma)
11279                         PIPE_CONF_CHECK_COLOR_LUT(gamma_mode, hw.gamma_lut, bp_gamma);
11280         }
11281
11282         PIPE_CONF_CHECK_BOOL(double_wide);
11283
11284         PIPE_CONF_CHECK_P(shared_dpll);
11285
11286         /* FIXME do the readout properly and get rid of this quirk */
11287         if (!PIPE_CONF_QUIRK(PIPE_CONFIG_QUIRK_BIGJOINER_SLAVE)) {
11288                 PIPE_CONF_CHECK_X(dpll_hw_state.dpll);
11289                 PIPE_CONF_CHECK_X(dpll_hw_state.dpll_md);
11290                 PIPE_CONF_CHECK_X(dpll_hw_state.fp0);
11291                 PIPE_CONF_CHECK_X(dpll_hw_state.fp1);
11292                 PIPE_CONF_CHECK_X(dpll_hw_state.wrpll);
11293                 PIPE_CONF_CHECK_X(dpll_hw_state.spll);
11294                 PIPE_CONF_CHECK_X(dpll_hw_state.ctrl1);
11295                 PIPE_CONF_CHECK_X(dpll_hw_state.cfgcr1);
11296                 PIPE_CONF_CHECK_X(dpll_hw_state.cfgcr2);
11297                 PIPE_CONF_CHECK_X(dpll_hw_state.cfgcr0);
11298                 PIPE_CONF_CHECK_X(dpll_hw_state.ebb0);
11299                 PIPE_CONF_CHECK_X(dpll_hw_state.ebb4);
11300                 PIPE_CONF_CHECK_X(dpll_hw_state.pll0);
11301                 PIPE_CONF_CHECK_X(dpll_hw_state.pll1);
11302                 PIPE_CONF_CHECK_X(dpll_hw_state.pll2);
11303                 PIPE_CONF_CHECK_X(dpll_hw_state.pll3);
11304                 PIPE_CONF_CHECK_X(dpll_hw_state.pll6);
11305                 PIPE_CONF_CHECK_X(dpll_hw_state.pll8);
11306                 PIPE_CONF_CHECK_X(dpll_hw_state.pll9);
11307                 PIPE_CONF_CHECK_X(dpll_hw_state.pll10);
11308                 PIPE_CONF_CHECK_X(dpll_hw_state.pcsdw12);
11309                 PIPE_CONF_CHECK_X(dpll_hw_state.mg_refclkin_ctl);
11310                 PIPE_CONF_CHECK_X(dpll_hw_state.mg_clktop2_coreclkctl1);
11311                 PIPE_CONF_CHECK_X(dpll_hw_state.mg_clktop2_hsclkctl);
11312                 PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_div0);
11313                 PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_div1);
11314                 PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_lf);
11315                 PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_frac_lock);
11316                 PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_ssc);
11317                 PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_bias);
11318                 PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_tdc_coldst_bias);
11319
11320                 PIPE_CONF_CHECK_X(dsi_pll.ctrl);
11321                 PIPE_CONF_CHECK_X(dsi_pll.div);
11322
11323                 if (IS_G4X(dev_priv) || INTEL_GEN(dev_priv) >= 5)
11324                         PIPE_CONF_CHECK_I(pipe_bpp);
11325
11326                 PIPE_CONF_CHECK_CLOCK_FUZZY(hw.pipe_mode.crtc_clock);
11327                 PIPE_CONF_CHECK_CLOCK_FUZZY(hw.adjusted_mode.crtc_clock);
11328                 PIPE_CONF_CHECK_CLOCK_FUZZY(port_clock);
11329
11330                 PIPE_CONF_CHECK_I(min_voltage_level);
11331         }
11332
11333         PIPE_CONF_CHECK_X(infoframes.enable);
11334         PIPE_CONF_CHECK_X(infoframes.gcp);
11335         PIPE_CONF_CHECK_INFOFRAME(avi);
11336         PIPE_CONF_CHECK_INFOFRAME(spd);
11337         PIPE_CONF_CHECK_INFOFRAME(hdmi);
11338         PIPE_CONF_CHECK_INFOFRAME(drm);
11339         PIPE_CONF_CHECK_DP_VSC_SDP(vsc);
11340
11341         PIPE_CONF_CHECK_X(sync_mode_slaves_mask);
11342         PIPE_CONF_CHECK_I(master_transcoder);
11343         PIPE_CONF_CHECK_BOOL(bigjoiner);
11344         PIPE_CONF_CHECK_BOOL(bigjoiner_slave);
11345         PIPE_CONF_CHECK_P(bigjoiner_linked_crtc);
11346
11347         PIPE_CONF_CHECK_I(dsc.compression_enable);
11348         PIPE_CONF_CHECK_I(dsc.dsc_split);
11349         PIPE_CONF_CHECK_I(dsc.compressed_bpp);
11350
11351         PIPE_CONF_CHECK_I(mst_master_transcoder);
11352
11353         PIPE_CONF_CHECK_BOOL(vrr.enable);
11354         PIPE_CONF_CHECK_I(vrr.vmin);
11355         PIPE_CONF_CHECK_I(vrr.vmax);
11356         PIPE_CONF_CHECK_I(vrr.flipline);
11357         PIPE_CONF_CHECK_I(vrr.pipeline_full);
11358
11359 #undef PIPE_CONF_CHECK_X
11360 #undef PIPE_CONF_CHECK_I
11361 #undef PIPE_CONF_CHECK_BOOL
11362 #undef PIPE_CONF_CHECK_BOOL_INCOMPLETE
11363 #undef PIPE_CONF_CHECK_P
11364 #undef PIPE_CONF_CHECK_FLAGS
11365 #undef PIPE_CONF_CHECK_CLOCK_FUZZY
11366 #undef PIPE_CONF_CHECK_COLOR_LUT
11367 #undef PIPE_CONF_QUIRK
11368
11369         return ret;
11370 }
11371
11372 static void intel_pipe_config_sanity_check(struct drm_i915_private *dev_priv,
11373                                            const struct intel_crtc_state *pipe_config)
11374 {
11375         if (pipe_config->has_pch_encoder) {
11376                 int fdi_dotclock = intel_dotclock_calculate(intel_fdi_link_freq(dev_priv, pipe_config),
11377                                                             &pipe_config->fdi_m_n);
11378                 int dotclock = pipe_config->hw.adjusted_mode.crtc_clock;
11379
11380                 /*
11381                  * FDI already provided one idea for the dotclock.
11382                  * Yell if the encoder disagrees.
11383                  */
11384                 drm_WARN(&dev_priv->drm,
11385                          !intel_fuzzy_clock_check(fdi_dotclock, dotclock),
11386                          "FDI dotclock and encoder dotclock mismatch, fdi: %i, encoder: %i\n",
11387                          fdi_dotclock, dotclock);
11388         }
11389 }
11390
11391 static void verify_wm_state(struct intel_crtc *crtc,
11392                             struct intel_crtc_state *new_crtc_state)
11393 {
11394         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
11395         struct skl_hw_state {
11396                 struct skl_ddb_entry ddb_y[I915_MAX_PLANES];
11397                 struct skl_ddb_entry ddb_uv[I915_MAX_PLANES];
11398                 struct skl_pipe_wm wm;
11399         } *hw;
11400         struct skl_pipe_wm *sw_wm;
11401         struct skl_ddb_entry *hw_ddb_entry, *sw_ddb_entry;
11402         u8 hw_enabled_slices;
11403         const enum pipe pipe = crtc->pipe;
11404         int plane, level, max_level = ilk_wm_max_level(dev_priv);
11405
11406         if (INTEL_GEN(dev_priv) < 9 || !new_crtc_state->hw.active)
11407                 return;
11408
11409         hw = kzalloc(sizeof(*hw), GFP_KERNEL);
11410         if (!hw)
11411                 return;
11412
11413         skl_pipe_wm_get_hw_state(crtc, &hw->wm);
11414         sw_wm = &new_crtc_state->wm.skl.optimal;
11415
11416         skl_pipe_ddb_get_hw_state(crtc, hw->ddb_y, hw->ddb_uv);
11417
11418         hw_enabled_slices = intel_enabled_dbuf_slices_mask(dev_priv);
11419
11420         if (INTEL_GEN(dev_priv) >= 11 &&
11421             hw_enabled_slices != dev_priv->dbuf.enabled_slices)
11422                 drm_err(&dev_priv->drm,
11423                         "mismatch in DBUF Slices (expected 0x%x, got 0x%x)\n",
11424                         dev_priv->dbuf.enabled_slices,
11425                         hw_enabled_slices);
11426
11427         /* planes */
11428         for_each_universal_plane(dev_priv, pipe, plane) {
11429                 struct skl_plane_wm *hw_plane_wm, *sw_plane_wm;
11430
11431                 hw_plane_wm = &hw->wm.planes[plane];
11432                 sw_plane_wm = &sw_wm->planes[plane];
11433
11434                 /* Watermarks */
11435                 for (level = 0; level <= max_level; level++) {
11436                         if (skl_wm_level_equals(&hw_plane_wm->wm[level],
11437                                                 &sw_plane_wm->wm[level]) ||
11438                             (level == 0 && skl_wm_level_equals(&hw_plane_wm->wm[level],
11439                                                                &sw_plane_wm->sagv_wm0)))
11440                                 continue;
11441
11442                         drm_err(&dev_priv->drm,
11443                                 "mismatch in WM pipe %c plane %d level %d (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
11444                                 pipe_name(pipe), plane + 1, level,
11445                                 sw_plane_wm->wm[level].plane_en,
11446                                 sw_plane_wm->wm[level].plane_res_b,
11447                                 sw_plane_wm->wm[level].plane_res_l,
11448                                 hw_plane_wm->wm[level].plane_en,
11449                                 hw_plane_wm->wm[level].plane_res_b,
11450                                 hw_plane_wm->wm[level].plane_res_l);
11451                 }
11452
11453                 if (!skl_wm_level_equals(&hw_plane_wm->trans_wm,
11454                                          &sw_plane_wm->trans_wm)) {
11455                         drm_err(&dev_priv->drm,
11456                                 "mismatch in trans WM pipe %c plane %d (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
11457                                 pipe_name(pipe), plane + 1,
11458                                 sw_plane_wm->trans_wm.plane_en,
11459                                 sw_plane_wm->trans_wm.plane_res_b,
11460                                 sw_plane_wm->trans_wm.plane_res_l,
11461                                 hw_plane_wm->trans_wm.plane_en,
11462                                 hw_plane_wm->trans_wm.plane_res_b,
11463                                 hw_plane_wm->trans_wm.plane_res_l);
11464                 }
11465
11466                 /* DDB */
11467                 hw_ddb_entry = &hw->ddb_y[plane];
11468                 sw_ddb_entry = &new_crtc_state->wm.skl.plane_ddb_y[plane];
11469
11470                 if (!skl_ddb_entry_equal(hw_ddb_entry, sw_ddb_entry)) {
11471                         drm_err(&dev_priv->drm,
11472                                 "mismatch in DDB state pipe %c plane %d (expected (%u,%u), found (%u,%u))\n",
11473                                 pipe_name(pipe), plane + 1,
11474                                 sw_ddb_entry->start, sw_ddb_entry->end,
11475                                 hw_ddb_entry->start, hw_ddb_entry->end);
11476                 }
11477         }
11478
11479         /*
11480          * cursor
11481          * If the cursor plane isn't active, we may not have updated it's ddb
11482          * allocation. In that case since the ddb allocation will be updated
11483          * once the plane becomes visible, we can skip this check
11484          */
11485         if (1) {
11486                 struct skl_plane_wm *hw_plane_wm, *sw_plane_wm;
11487
11488                 hw_plane_wm = &hw->wm.planes[PLANE_CURSOR];
11489                 sw_plane_wm = &sw_wm->planes[PLANE_CURSOR];
11490
11491                 /* Watermarks */
11492                 for (level = 0; level <= max_level; level++) {
11493                         if (skl_wm_level_equals(&hw_plane_wm->wm[level],
11494                                                 &sw_plane_wm->wm[level]) ||
11495                             (level == 0 && skl_wm_level_equals(&hw_plane_wm->wm[level],
11496                                                                &sw_plane_wm->sagv_wm0)))
11497                                 continue;
11498
11499                         drm_err(&dev_priv->drm,
11500                                 "mismatch in WM pipe %c cursor level %d (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
11501                                 pipe_name(pipe), level,
11502                                 sw_plane_wm->wm[level].plane_en,
11503                                 sw_plane_wm->wm[level].plane_res_b,
11504                                 sw_plane_wm->wm[level].plane_res_l,
11505                                 hw_plane_wm->wm[level].plane_en,
11506                                 hw_plane_wm->wm[level].plane_res_b,
11507                                 hw_plane_wm->wm[level].plane_res_l);
11508                 }
11509
11510                 if (!skl_wm_level_equals(&hw_plane_wm->trans_wm,
11511                                          &sw_plane_wm->trans_wm)) {
11512                         drm_err(&dev_priv->drm,
11513                                 "mismatch in trans WM pipe %c cursor (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
11514                                 pipe_name(pipe),
11515                                 sw_plane_wm->trans_wm.plane_en,
11516                                 sw_plane_wm->trans_wm.plane_res_b,
11517                                 sw_plane_wm->trans_wm.plane_res_l,
11518                                 hw_plane_wm->trans_wm.plane_en,
11519                                 hw_plane_wm->trans_wm.plane_res_b,
11520                                 hw_plane_wm->trans_wm.plane_res_l);
11521                 }
11522
11523                 /* DDB */
11524                 hw_ddb_entry = &hw->ddb_y[PLANE_CURSOR];
11525                 sw_ddb_entry = &new_crtc_state->wm.skl.plane_ddb_y[PLANE_CURSOR];
11526
11527                 if (!skl_ddb_entry_equal(hw_ddb_entry, sw_ddb_entry)) {
11528                         drm_err(&dev_priv->drm,
11529                                 "mismatch in DDB state pipe %c cursor (expected (%u,%u), found (%u,%u))\n",
11530                                 pipe_name(pipe),
11531                                 sw_ddb_entry->start, sw_ddb_entry->end,
11532                                 hw_ddb_entry->start, hw_ddb_entry->end);
11533                 }
11534         }
11535
11536         kfree(hw);
11537 }
11538
11539 static void
11540 verify_connector_state(struct intel_atomic_state *state,
11541                        struct intel_crtc *crtc)
11542 {
11543         struct drm_connector *connector;
11544         struct drm_connector_state *new_conn_state;
11545         int i;
11546
11547         for_each_new_connector_in_state(&state->base, connector, new_conn_state, i) {
11548                 struct drm_encoder *encoder = connector->encoder;
11549                 struct intel_crtc_state *crtc_state = NULL;
11550
11551                 if (new_conn_state->crtc != &crtc->base)
11552                         continue;
11553
11554                 if (crtc)
11555                         crtc_state = intel_atomic_get_new_crtc_state(state, crtc);
11556
11557                 intel_connector_verify_state(crtc_state, new_conn_state);
11558
11559                 I915_STATE_WARN(new_conn_state->best_encoder != encoder,
11560                      "connector's atomic encoder doesn't match legacy encoder\n");
11561         }
11562 }
11563
11564 static void
11565 verify_encoder_state(struct drm_i915_private *dev_priv, struct intel_atomic_state *state)
11566 {
11567         struct intel_encoder *encoder;
11568         struct drm_connector *connector;
11569         struct drm_connector_state *old_conn_state, *new_conn_state;
11570         int i;
11571
11572         for_each_intel_encoder(&dev_priv->drm, encoder) {
11573                 bool enabled = false, found = false;
11574                 enum pipe pipe;
11575
11576                 drm_dbg_kms(&dev_priv->drm, "[ENCODER:%d:%s]\n",
11577                             encoder->base.base.id,
11578                             encoder->base.name);
11579
11580                 for_each_oldnew_connector_in_state(&state->base, connector, old_conn_state,
11581                                                    new_conn_state, i) {
11582                         if (old_conn_state->best_encoder == &encoder->base)
11583                                 found = true;
11584
11585                         if (new_conn_state->best_encoder != &encoder->base)
11586                                 continue;
11587                         found = enabled = true;
11588
11589                         I915_STATE_WARN(new_conn_state->crtc !=
11590                                         encoder->base.crtc,
11591                              "connector's crtc doesn't match encoder crtc\n");
11592                 }
11593
11594                 if (!found)
11595                         continue;
11596
11597                 I915_STATE_WARN(!!encoder->base.crtc != enabled,
11598                      "encoder's enabled state mismatch "
11599                      "(expected %i, found %i)\n",
11600                      !!encoder->base.crtc, enabled);
11601
11602                 if (!encoder->base.crtc) {
11603                         bool active;
11604
11605                         active = encoder->get_hw_state(encoder, &pipe);
11606                         I915_STATE_WARN(active,
11607                              "encoder detached but still enabled on pipe %c.\n",
11608                              pipe_name(pipe));
11609                 }
11610         }
11611 }
11612
11613 static void
11614 verify_crtc_state(struct intel_crtc *crtc,
11615                   struct intel_crtc_state *old_crtc_state,
11616                   struct intel_crtc_state *new_crtc_state)
11617 {
11618         struct drm_device *dev = crtc->base.dev;
11619         struct drm_i915_private *dev_priv = to_i915(dev);
11620         struct intel_encoder *encoder;
11621         struct intel_crtc_state *pipe_config = old_crtc_state;
11622         struct drm_atomic_state *state = old_crtc_state->uapi.state;
11623         struct intel_crtc *master = crtc;
11624
11625         __drm_atomic_helper_crtc_destroy_state(&old_crtc_state->uapi);
11626         intel_crtc_free_hw_state(old_crtc_state);
11627         intel_crtc_state_reset(old_crtc_state, crtc);
11628         old_crtc_state->uapi.state = state;
11629
11630         drm_dbg_kms(&dev_priv->drm, "[CRTC:%d:%s]\n", crtc->base.base.id,
11631                     crtc->base.name);
11632
11633         pipe_config->hw.enable = new_crtc_state->hw.enable;
11634
11635         intel_crtc_get_pipe_config(pipe_config);
11636
11637         /* we keep both pipes enabled on 830 */
11638         if (IS_I830(dev_priv) && pipe_config->hw.active)
11639                 pipe_config->hw.active = new_crtc_state->hw.active;
11640
11641         I915_STATE_WARN(new_crtc_state->hw.active != pipe_config->hw.active,
11642                         "crtc active state doesn't match with hw state "
11643                         "(expected %i, found %i)\n",
11644                         new_crtc_state->hw.active, pipe_config->hw.active);
11645
11646         I915_STATE_WARN(crtc->active != new_crtc_state->hw.active,
11647                         "transitional active state does not match atomic hw state "
11648                         "(expected %i, found %i)\n",
11649                         new_crtc_state->hw.active, crtc->active);
11650
11651         if (new_crtc_state->bigjoiner_slave)
11652                 master = new_crtc_state->bigjoiner_linked_crtc;
11653
11654         for_each_encoder_on_crtc(dev, &master->base, encoder) {
11655                 enum pipe pipe;
11656                 bool active;
11657
11658                 active = encoder->get_hw_state(encoder, &pipe);
11659                 I915_STATE_WARN(active != new_crtc_state->hw.active,
11660                                 "[ENCODER:%i] active %i with crtc active %i\n",
11661                                 encoder->base.base.id, active,
11662                                 new_crtc_state->hw.active);
11663
11664                 I915_STATE_WARN(active && master->pipe != pipe,
11665                                 "Encoder connected to wrong pipe %c\n",
11666                                 pipe_name(pipe));
11667
11668                 if (active)
11669                         intel_encoder_get_config(encoder, pipe_config);
11670         }
11671
11672         if (!new_crtc_state->hw.active)
11673                 return;
11674
11675         intel_pipe_config_sanity_check(dev_priv, pipe_config);
11676
11677         if (!intel_pipe_config_compare(new_crtc_state,
11678                                        pipe_config, false)) {
11679                 I915_STATE_WARN(1, "pipe state doesn't match!\n");
11680                 intel_dump_pipe_config(pipe_config, NULL, "[hw state]");
11681                 intel_dump_pipe_config(new_crtc_state, NULL, "[sw state]");
11682         }
11683 }
11684
11685 static void
11686 intel_verify_planes(struct intel_atomic_state *state)
11687 {
11688         struct intel_plane *plane;
11689         const struct intel_plane_state *plane_state;
11690         int i;
11691
11692         for_each_new_intel_plane_in_state(state, plane,
11693                                           plane_state, i)
11694                 assert_plane(plane, plane_state->planar_slave ||
11695                              plane_state->uapi.visible);
11696 }
11697
11698 static void
11699 verify_single_dpll_state(struct drm_i915_private *dev_priv,
11700                          struct intel_shared_dpll *pll,
11701                          struct intel_crtc *crtc,
11702                          struct intel_crtc_state *new_crtc_state)
11703 {
11704         struct intel_dpll_hw_state dpll_hw_state;
11705         unsigned int crtc_mask;
11706         bool active;
11707
11708         memset(&dpll_hw_state, 0, sizeof(dpll_hw_state));
11709
11710         drm_dbg_kms(&dev_priv->drm, "%s\n", pll->info->name);
11711
11712         active = intel_dpll_get_hw_state(dev_priv, pll, &dpll_hw_state);
11713
11714         if (!(pll->info->flags & INTEL_DPLL_ALWAYS_ON)) {
11715                 I915_STATE_WARN(!pll->on && pll->active_mask,
11716                      "pll in active use but not on in sw tracking\n");
11717                 I915_STATE_WARN(pll->on && !pll->active_mask,
11718                      "pll is on but not used by any active crtc\n");
11719                 I915_STATE_WARN(pll->on != active,
11720                      "pll on state mismatch (expected %i, found %i)\n",
11721                      pll->on, active);
11722         }
11723
11724         if (!crtc) {
11725                 I915_STATE_WARN(pll->active_mask & ~pll->state.crtc_mask,
11726                                 "more active pll users than references: %x vs %x\n",
11727                                 pll->active_mask, pll->state.crtc_mask);
11728
11729                 return;
11730         }
11731
11732         crtc_mask = drm_crtc_mask(&crtc->base);
11733
11734         if (new_crtc_state->hw.active)
11735                 I915_STATE_WARN(!(pll->active_mask & crtc_mask),
11736                                 "pll active mismatch (expected pipe %c in active mask 0x%02x)\n",
11737                                 pipe_name(crtc->pipe), pll->active_mask);
11738         else
11739                 I915_STATE_WARN(pll->active_mask & crtc_mask,
11740                                 "pll active mismatch (didn't expect pipe %c in active mask 0x%02x)\n",
11741                                 pipe_name(crtc->pipe), pll->active_mask);
11742
11743         I915_STATE_WARN(!(pll->state.crtc_mask & crtc_mask),
11744                         "pll enabled crtcs mismatch (expected 0x%x in 0x%02x)\n",
11745                         crtc_mask, pll->state.crtc_mask);
11746
11747         I915_STATE_WARN(pll->on && memcmp(&pll->state.hw_state,
11748                                           &dpll_hw_state,
11749                                           sizeof(dpll_hw_state)),
11750                         "pll hw state mismatch\n");
11751 }
11752
11753 static void
11754 verify_shared_dpll_state(struct intel_crtc *crtc,
11755                          struct intel_crtc_state *old_crtc_state,
11756                          struct intel_crtc_state *new_crtc_state)
11757 {
11758         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
11759
11760         if (new_crtc_state->shared_dpll)
11761                 verify_single_dpll_state(dev_priv, new_crtc_state->shared_dpll, crtc, new_crtc_state);
11762
11763         if (old_crtc_state->shared_dpll &&
11764             old_crtc_state->shared_dpll != new_crtc_state->shared_dpll) {
11765                 unsigned int crtc_mask = drm_crtc_mask(&crtc->base);
11766                 struct intel_shared_dpll *pll = old_crtc_state->shared_dpll;
11767
11768                 I915_STATE_WARN(pll->active_mask & crtc_mask,
11769                                 "pll active mismatch (didn't expect pipe %c in active mask)\n",
11770                                 pipe_name(crtc->pipe));
11771                 I915_STATE_WARN(pll->state.crtc_mask & crtc_mask,
11772                                 "pll enabled crtcs mismatch (found %x in enabled mask)\n",
11773                                 pipe_name(crtc->pipe));
11774         }
11775 }
11776
11777 static void
11778 intel_modeset_verify_crtc(struct intel_crtc *crtc,
11779                           struct intel_atomic_state *state,
11780                           struct intel_crtc_state *old_crtc_state,
11781                           struct intel_crtc_state *new_crtc_state)
11782 {
11783         if (!intel_crtc_needs_modeset(new_crtc_state) && !new_crtc_state->update_pipe)
11784                 return;
11785
11786         verify_wm_state(crtc, new_crtc_state);
11787         verify_connector_state(state, crtc);
11788         verify_crtc_state(crtc, old_crtc_state, new_crtc_state);
11789         verify_shared_dpll_state(crtc, old_crtc_state, new_crtc_state);
11790 }
11791
11792 static void
11793 verify_disabled_dpll_state(struct drm_i915_private *dev_priv)
11794 {
11795         int i;
11796
11797         for (i = 0; i < dev_priv->dpll.num_shared_dpll; i++)
11798                 verify_single_dpll_state(dev_priv,
11799                                          &dev_priv->dpll.shared_dplls[i],
11800                                          NULL, NULL);
11801 }
11802
11803 static void
11804 intel_modeset_verify_disabled(struct drm_i915_private *dev_priv,
11805                               struct intel_atomic_state *state)
11806 {
11807         verify_encoder_state(dev_priv, state);
11808         verify_connector_state(state, NULL);
11809         verify_disabled_dpll_state(dev_priv);
11810 }
11811
11812 static void
11813 intel_crtc_update_active_timings(const struct intel_crtc_state *crtc_state)
11814 {
11815         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
11816         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
11817         struct drm_display_mode adjusted_mode =
11818                 crtc_state->hw.adjusted_mode;
11819
11820         if (crtc_state->vrr.enable) {
11821                 adjusted_mode.crtc_vtotal = crtc_state->vrr.vmax;
11822                 adjusted_mode.crtc_vblank_end = crtc_state->vrr.vmax;
11823                 adjusted_mode.crtc_vblank_start = intel_vrr_vmin_vblank_start(crtc_state);
11824                 crtc->vmax_vblank_start = intel_vrr_vmax_vblank_start(crtc_state);
11825         }
11826
11827         drm_calc_timestamping_constants(&crtc->base, &adjusted_mode);
11828
11829         crtc->mode_flags = crtc_state->mode_flags;
11830
11831         /*
11832          * The scanline counter increments at the leading edge of hsync.
11833          *
11834          * On most platforms it starts counting from vtotal-1 on the
11835          * first active line. That means the scanline counter value is
11836          * always one less than what we would expect. Ie. just after
11837          * start of vblank, which also occurs at start of hsync (on the
11838          * last active line), the scanline counter will read vblank_start-1.
11839          *
11840          * On gen2 the scanline counter starts counting from 1 instead
11841          * of vtotal-1, so we have to subtract one (or rather add vtotal-1
11842          * to keep the value positive), instead of adding one.
11843          *
11844          * On HSW+ the behaviour of the scanline counter depends on the output
11845          * type. For DP ports it behaves like most other platforms, but on HDMI
11846          * there's an extra 1 line difference. So we need to add two instead of
11847          * one to the value.
11848          *
11849          * On VLV/CHV DSI the scanline counter would appear to increment
11850          * approx. 1/3 of a scanline before start of vblank. Unfortunately
11851          * that means we can't tell whether we're in vblank or not while
11852          * we're on that particular line. We must still set scanline_offset
11853          * to 1 so that the vblank timestamps come out correct when we query
11854          * the scanline counter from within the vblank interrupt handler.
11855          * However if queried just before the start of vblank we'll get an
11856          * answer that's slightly in the future.
11857          */
11858         if (IS_GEN(dev_priv, 2)) {
11859                 int vtotal;
11860
11861                 vtotal = adjusted_mode.crtc_vtotal;
11862                 if (adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE)
11863                         vtotal /= 2;
11864
11865                 crtc->scanline_offset = vtotal - 1;
11866         } else if (HAS_DDI(dev_priv) &&
11867                    intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) {
11868                 crtc->scanline_offset = 2;
11869         } else {
11870                 crtc->scanline_offset = 1;
11871         }
11872 }
11873
11874 static void intel_modeset_clear_plls(struct intel_atomic_state *state)
11875 {
11876         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
11877         struct intel_crtc_state *new_crtc_state;
11878         struct intel_crtc *crtc;
11879         int i;
11880
11881         if (!dev_priv->display.crtc_compute_clock)
11882                 return;
11883
11884         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
11885                 if (!intel_crtc_needs_modeset(new_crtc_state))
11886                         continue;
11887
11888                 intel_release_shared_dplls(state, crtc);
11889         }
11890 }
11891
11892 /*
11893  * This implements the workaround described in the "notes" section of the mode
11894  * set sequence documentation. When going from no pipes or single pipe to
11895  * multiple pipes, and planes are enabled after the pipe, we need to wait at
11896  * least 2 vblanks on the first pipe before enabling planes on the second pipe.
11897  */
11898 static int hsw_mode_set_planes_workaround(struct intel_atomic_state *state)
11899 {
11900         struct intel_crtc_state *crtc_state;
11901         struct intel_crtc *crtc;
11902         struct intel_crtc_state *first_crtc_state = NULL;
11903         struct intel_crtc_state *other_crtc_state = NULL;
11904         enum pipe first_pipe = INVALID_PIPE, enabled_pipe = INVALID_PIPE;
11905         int i;
11906
11907         /* look at all crtc's that are going to be enabled in during modeset */
11908         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
11909                 if (!crtc_state->hw.active ||
11910                     !intel_crtc_needs_modeset(crtc_state))
11911                         continue;
11912
11913                 if (first_crtc_state) {
11914                         other_crtc_state = crtc_state;
11915                         break;
11916                 } else {
11917                         first_crtc_state = crtc_state;
11918                         first_pipe = crtc->pipe;
11919                 }
11920         }
11921
11922         /* No workaround needed? */
11923         if (!first_crtc_state)
11924                 return 0;
11925
11926         /* w/a possibly needed, check how many crtc's are already enabled. */
11927         for_each_intel_crtc(state->base.dev, crtc) {
11928                 crtc_state = intel_atomic_get_crtc_state(&state->base, crtc);
11929                 if (IS_ERR(crtc_state))
11930                         return PTR_ERR(crtc_state);
11931
11932                 crtc_state->hsw_workaround_pipe = INVALID_PIPE;
11933
11934                 if (!crtc_state->hw.active ||
11935                     intel_crtc_needs_modeset(crtc_state))
11936                         continue;
11937
11938                 /* 2 or more enabled crtcs means no need for w/a */
11939                 if (enabled_pipe != INVALID_PIPE)
11940                         return 0;
11941
11942                 enabled_pipe = crtc->pipe;
11943         }
11944
11945         if (enabled_pipe != INVALID_PIPE)
11946                 first_crtc_state->hsw_workaround_pipe = enabled_pipe;
11947         else if (other_crtc_state)
11948                 other_crtc_state->hsw_workaround_pipe = first_pipe;
11949
11950         return 0;
11951 }
11952
11953 u8 intel_calc_active_pipes(struct intel_atomic_state *state,
11954                            u8 active_pipes)
11955 {
11956         const struct intel_crtc_state *crtc_state;
11957         struct intel_crtc *crtc;
11958         int i;
11959
11960         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
11961                 if (crtc_state->hw.active)
11962                         active_pipes |= BIT(crtc->pipe);
11963                 else
11964                         active_pipes &= ~BIT(crtc->pipe);
11965         }
11966
11967         return active_pipes;
11968 }
11969
11970 static int intel_modeset_checks(struct intel_atomic_state *state)
11971 {
11972         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
11973
11974         state->modeset = true;
11975
11976         if (IS_HASWELL(dev_priv))
11977                 return hsw_mode_set_planes_workaround(state);
11978
11979         return 0;
11980 }
11981
11982 /*
11983  * Handle calculation of various watermark data at the end of the atomic check
11984  * phase.  The code here should be run after the per-crtc and per-plane 'check'
11985  * handlers to ensure that all derived state has been updated.
11986  */
11987 static int calc_watermark_data(struct intel_atomic_state *state)
11988 {
11989         struct drm_device *dev = state->base.dev;
11990         struct drm_i915_private *dev_priv = to_i915(dev);
11991
11992         /* Is there platform-specific watermark information to calculate? */
11993         if (dev_priv->display.compute_global_watermarks)
11994                 return dev_priv->display.compute_global_watermarks(state);
11995
11996         return 0;
11997 }
11998
11999 static void intel_crtc_check_fastset(const struct intel_crtc_state *old_crtc_state,
12000                                      struct intel_crtc_state *new_crtc_state)
12001 {
12002         if (!intel_pipe_config_compare(old_crtc_state, new_crtc_state, true))
12003                 return;
12004
12005         new_crtc_state->uapi.mode_changed = false;
12006         new_crtc_state->update_pipe = true;
12007 }
12008
12009 static void intel_crtc_copy_fastset(const struct intel_crtc_state *old_crtc_state,
12010                                     struct intel_crtc_state *new_crtc_state)
12011 {
12012         /*
12013          * If we're not doing the full modeset we want to
12014          * keep the current M/N values as they may be
12015          * sufficiently different to the computed values
12016          * to cause problems.
12017          *
12018          * FIXME: should really copy more fuzzy state here
12019          */
12020         new_crtc_state->fdi_m_n = old_crtc_state->fdi_m_n;
12021         new_crtc_state->dp_m_n = old_crtc_state->dp_m_n;
12022         new_crtc_state->dp_m2_n2 = old_crtc_state->dp_m2_n2;
12023         new_crtc_state->has_drrs = old_crtc_state->has_drrs;
12024 }
12025
12026 static int intel_crtc_add_planes_to_state(struct intel_atomic_state *state,
12027                                           struct intel_crtc *crtc,
12028                                           u8 plane_ids_mask)
12029 {
12030         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
12031         struct intel_plane *plane;
12032
12033         for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
12034                 struct intel_plane_state *plane_state;
12035
12036                 if ((plane_ids_mask & BIT(plane->id)) == 0)
12037                         continue;
12038
12039                 plane_state = intel_atomic_get_plane_state(state, plane);
12040                 if (IS_ERR(plane_state))
12041                         return PTR_ERR(plane_state);
12042         }
12043
12044         return 0;
12045 }
12046
12047 int intel_atomic_add_affected_planes(struct intel_atomic_state *state,
12048                                      struct intel_crtc *crtc)
12049 {
12050         const struct intel_crtc_state *old_crtc_state =
12051                 intel_atomic_get_old_crtc_state(state, crtc);
12052         const struct intel_crtc_state *new_crtc_state =
12053                 intel_atomic_get_new_crtc_state(state, crtc);
12054
12055         return intel_crtc_add_planes_to_state(state, crtc,
12056                                               old_crtc_state->enabled_planes |
12057                                               new_crtc_state->enabled_planes);
12058 }
12059
12060 static bool active_planes_affects_min_cdclk(struct drm_i915_private *dev_priv)
12061 {
12062         /* See {hsw,vlv,ivb}_plane_ratio() */
12063         return IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv) ||
12064                 IS_CHERRYVIEW(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
12065                 IS_IVYBRIDGE(dev_priv) || (INTEL_GEN(dev_priv) >= 11);
12066 }
12067
12068 static int intel_crtc_add_bigjoiner_planes(struct intel_atomic_state *state,
12069                                            struct intel_crtc *crtc,
12070                                            struct intel_crtc *other)
12071 {
12072         const struct intel_plane_state *plane_state;
12073         struct intel_plane *plane;
12074         u8 plane_ids = 0;
12075         int i;
12076
12077         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
12078                 if (plane->pipe == crtc->pipe)
12079                         plane_ids |= BIT(plane->id);
12080         }
12081
12082         return intel_crtc_add_planes_to_state(state, other, plane_ids);
12083 }
12084
12085 static int intel_bigjoiner_add_affected_planes(struct intel_atomic_state *state)
12086 {
12087         const struct intel_crtc_state *crtc_state;
12088         struct intel_crtc *crtc;
12089         int i;
12090
12091         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
12092                 int ret;
12093
12094                 if (!crtc_state->bigjoiner)
12095                         continue;
12096
12097                 ret = intel_crtc_add_bigjoiner_planes(state, crtc,
12098                                                       crtc_state->bigjoiner_linked_crtc);
12099                 if (ret)
12100                         return ret;
12101         }
12102
12103         return 0;
12104 }
12105
12106 static int intel_atomic_check_planes(struct intel_atomic_state *state)
12107 {
12108         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
12109         struct intel_crtc_state *old_crtc_state, *new_crtc_state;
12110         struct intel_plane_state *plane_state;
12111         struct intel_plane *plane;
12112         struct intel_crtc *crtc;
12113         int i, ret;
12114
12115         ret = icl_add_linked_planes(state);
12116         if (ret)
12117                 return ret;
12118
12119         ret = intel_bigjoiner_add_affected_planes(state);
12120         if (ret)
12121                 return ret;
12122
12123         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
12124                 ret = intel_plane_atomic_check(state, plane);
12125                 if (ret) {
12126                         drm_dbg_atomic(&dev_priv->drm,
12127                                        "[PLANE:%d:%s] atomic driver check failed\n",
12128                                        plane->base.base.id, plane->base.name);
12129                         return ret;
12130                 }
12131         }
12132
12133         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
12134                                             new_crtc_state, i) {
12135                 u8 old_active_planes, new_active_planes;
12136
12137                 ret = icl_check_nv12_planes(new_crtc_state);
12138                 if (ret)
12139                         return ret;
12140
12141                 /*
12142                  * On some platforms the number of active planes affects
12143                  * the planes' minimum cdclk calculation. Add such planes
12144                  * to the state before we compute the minimum cdclk.
12145                  */
12146                 if (!active_planes_affects_min_cdclk(dev_priv))
12147                         continue;
12148
12149                 old_active_planes = old_crtc_state->active_planes & ~BIT(PLANE_CURSOR);
12150                 new_active_planes = new_crtc_state->active_planes & ~BIT(PLANE_CURSOR);
12151
12152                 /*
12153                  * Not only the number of planes, but if the plane configuration had
12154                  * changed might already mean we need to recompute min CDCLK,
12155                  * because different planes might consume different amount of Dbuf bandwidth
12156                  * according to formula: Bw per plane = Pixel rate * bpp * pipe/plane scale factor
12157                  */
12158                 if (old_active_planes == new_active_planes)
12159                         continue;
12160
12161                 ret = intel_crtc_add_planes_to_state(state, crtc, new_active_planes);
12162                 if (ret)
12163                         return ret;
12164         }
12165
12166         return 0;
12167 }
12168
12169 static int intel_atomic_check_cdclk(struct intel_atomic_state *state,
12170                                     bool *need_cdclk_calc)
12171 {
12172         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
12173         const struct intel_cdclk_state *old_cdclk_state;
12174         const struct intel_cdclk_state *new_cdclk_state;
12175         struct intel_plane_state *plane_state;
12176         struct intel_bw_state *new_bw_state;
12177         struct intel_plane *plane;
12178         int min_cdclk = 0;
12179         enum pipe pipe;
12180         int ret;
12181         int i;
12182         /*
12183          * active_planes bitmask has been updated, and potentially
12184          * affected planes are part of the state. We can now
12185          * compute the minimum cdclk for each plane.
12186          */
12187         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
12188                 ret = intel_plane_calc_min_cdclk(state, plane, need_cdclk_calc);
12189                 if (ret)
12190                         return ret;
12191         }
12192
12193         old_cdclk_state = intel_atomic_get_old_cdclk_state(state);
12194         new_cdclk_state = intel_atomic_get_new_cdclk_state(state);
12195
12196         if (new_cdclk_state &&
12197             old_cdclk_state->force_min_cdclk != new_cdclk_state->force_min_cdclk)
12198                 *need_cdclk_calc = true;
12199
12200         ret = dev_priv->display.bw_calc_min_cdclk(state);
12201         if (ret)
12202                 return ret;
12203
12204         new_bw_state = intel_atomic_get_new_bw_state(state);
12205
12206         if (!new_cdclk_state || !new_bw_state)
12207                 return 0;
12208
12209         for_each_pipe(dev_priv, pipe) {
12210                 min_cdclk = max(new_cdclk_state->min_cdclk[pipe], min_cdclk);
12211
12212                 /*
12213                  * Currently do this change only if we need to increase
12214                  */
12215                 if (new_bw_state->min_cdclk > min_cdclk)
12216                         *need_cdclk_calc = true;
12217         }
12218
12219         return 0;
12220 }
12221
12222 static int intel_atomic_check_crtcs(struct intel_atomic_state *state)
12223 {
12224         struct intel_crtc_state *crtc_state;
12225         struct intel_crtc *crtc;
12226         int i;
12227
12228         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
12229                 struct drm_i915_private *i915 = to_i915(crtc->base.dev);
12230                 int ret;
12231
12232                 ret = intel_crtc_atomic_check(state, crtc);
12233                 if (ret) {
12234                         drm_dbg_atomic(&i915->drm,
12235                                        "[CRTC:%d:%s] atomic driver check failed\n",
12236                                        crtc->base.base.id, crtc->base.name);
12237                         return ret;
12238                 }
12239         }
12240
12241         return 0;
12242 }
12243
12244 static bool intel_cpu_transcoders_need_modeset(struct intel_atomic_state *state,
12245                                                u8 transcoders)
12246 {
12247         const struct intel_crtc_state *new_crtc_state;
12248         struct intel_crtc *crtc;
12249         int i;
12250
12251         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
12252                 if (new_crtc_state->hw.enable &&
12253                     transcoders & BIT(new_crtc_state->cpu_transcoder) &&
12254                     intel_crtc_needs_modeset(new_crtc_state))
12255                         return true;
12256         }
12257
12258         return false;
12259 }
12260
12261 static int intel_atomic_check_bigjoiner(struct intel_atomic_state *state,
12262                                         struct intel_crtc *crtc,
12263                                         struct intel_crtc_state *old_crtc_state,
12264                                         struct intel_crtc_state *new_crtc_state)
12265 {
12266         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
12267         struct intel_crtc_state *slave_crtc_state, *master_crtc_state;
12268         struct intel_crtc *slave, *master;
12269
12270         /* slave being enabled, is master is still claiming this crtc? */
12271         if (old_crtc_state->bigjoiner_slave) {
12272                 slave = crtc;
12273                 master = old_crtc_state->bigjoiner_linked_crtc;
12274                 master_crtc_state = intel_atomic_get_new_crtc_state(state, master);
12275                 if (!master_crtc_state || !intel_crtc_needs_modeset(master_crtc_state))
12276                         goto claimed;
12277         }
12278
12279         if (!new_crtc_state->bigjoiner)
12280                 return 0;
12281
12282         if (1 + crtc->pipe >= INTEL_NUM_PIPES(dev_priv)) {
12283                 DRM_DEBUG_KMS("[CRTC:%d:%s] Big joiner configuration requires "
12284                               "CRTC + 1 to be used, doesn't exist\n",
12285                               crtc->base.base.id, crtc->base.name);
12286                 return -EINVAL;
12287         }
12288
12289         slave = new_crtc_state->bigjoiner_linked_crtc =
12290                 intel_get_crtc_for_pipe(dev_priv, crtc->pipe + 1);
12291         slave_crtc_state = intel_atomic_get_crtc_state(&state->base, slave);
12292         master = crtc;
12293         if (IS_ERR(slave_crtc_state))
12294                 return PTR_ERR(slave_crtc_state);
12295
12296         /* master being enabled, slave was already configured? */
12297         if (slave_crtc_state->uapi.enable)
12298                 goto claimed;
12299
12300         DRM_DEBUG_KMS("[CRTC:%d:%s] Used as slave for big joiner\n",
12301                       slave->base.base.id, slave->base.name);
12302
12303         return copy_bigjoiner_crtc_state(slave_crtc_state, new_crtc_state);
12304
12305 claimed:
12306         DRM_DEBUG_KMS("[CRTC:%d:%s] Slave is enabled as normal CRTC, but "
12307                       "[CRTC:%d:%s] claiming this CRTC for bigjoiner.\n",
12308                       slave->base.base.id, slave->base.name,
12309                       master->base.base.id, master->base.name);
12310         return -EINVAL;
12311 }
12312
12313 static void kill_bigjoiner_slave(struct intel_atomic_state *state,
12314                                  struct intel_crtc_state *master_crtc_state)
12315 {
12316         struct intel_crtc_state *slave_crtc_state =
12317                 intel_atomic_get_new_crtc_state(state, master_crtc_state->bigjoiner_linked_crtc);
12318
12319         slave_crtc_state->bigjoiner = master_crtc_state->bigjoiner = false;
12320         slave_crtc_state->bigjoiner_slave = master_crtc_state->bigjoiner_slave = false;
12321         slave_crtc_state->bigjoiner_linked_crtc = master_crtc_state->bigjoiner_linked_crtc = NULL;
12322         intel_crtc_copy_uapi_to_hw_state(state, slave_crtc_state);
12323 }
12324
12325 /**
12326  * DOC: asynchronous flip implementation
12327  *
12328  * Asynchronous page flip is the implementation for the DRM_MODE_PAGE_FLIP_ASYNC
12329  * flag. Currently async flip is only supported via the drmModePageFlip IOCTL.
12330  * Correspondingly, support is currently added for primary plane only.
12331  *
12332  * Async flip can only change the plane surface address, so anything else
12333  * changing is rejected from the intel_atomic_check_async() function.
12334  * Once this check is cleared, flip done interrupt is enabled using
12335  * the intel_crtc_enable_flip_done() function.
12336  *
12337  * As soon as the surface address register is written, flip done interrupt is
12338  * generated and the requested events are sent to the usersapce in the interrupt
12339  * handler itself. The timestamp and sequence sent during the flip done event
12340  * correspond to the last vblank and have no relation to the actual time when
12341  * the flip done event was sent.
12342  */
12343 static int intel_atomic_check_async(struct intel_atomic_state *state)
12344 {
12345         struct drm_i915_private *i915 = to_i915(state->base.dev);
12346         const struct intel_crtc_state *old_crtc_state, *new_crtc_state;
12347         const struct intel_plane_state *new_plane_state, *old_plane_state;
12348         struct intel_crtc *crtc;
12349         struct intel_plane *plane;
12350         int i;
12351
12352         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
12353                                             new_crtc_state, i) {
12354                 if (intel_crtc_needs_modeset(new_crtc_state)) {
12355                         drm_dbg_kms(&i915->drm, "Modeset Required. Async flip not supported\n");
12356                         return -EINVAL;
12357                 }
12358
12359                 if (!new_crtc_state->hw.active) {
12360                         drm_dbg_kms(&i915->drm, "CRTC inactive\n");
12361                         return -EINVAL;
12362                 }
12363                 if (old_crtc_state->active_planes != new_crtc_state->active_planes) {
12364                         drm_dbg_kms(&i915->drm,
12365                                     "Active planes cannot be changed during async flip\n");
12366                         return -EINVAL;
12367                 }
12368         }
12369
12370         for_each_oldnew_intel_plane_in_state(state, plane, old_plane_state,
12371                                              new_plane_state, i) {
12372                 /*
12373                  * TODO: Async flip is only supported through the page flip IOCTL
12374                  * as of now. So support currently added for primary plane only.
12375                  * Support for other planes on platforms on which supports
12376                  * this(vlv/chv and icl+) should be added when async flip is
12377                  * enabled in the atomic IOCTL path.
12378                  */
12379                 if (!plane->async_flip)
12380                         return -EINVAL;
12381
12382                 /*
12383                  * FIXME: This check is kept generic for all platforms.
12384                  * Need to verify this for all gen9 and gen10 platforms to enable
12385                  * this selectively if required.
12386                  */
12387                 switch (new_plane_state->hw.fb->modifier) {
12388                 case I915_FORMAT_MOD_X_TILED:
12389                 case I915_FORMAT_MOD_Y_TILED:
12390                 case I915_FORMAT_MOD_Yf_TILED:
12391                         break;
12392                 default:
12393                         drm_dbg_kms(&i915->drm,
12394                                     "Linear memory/CCS does not support async flips\n");
12395                         return -EINVAL;
12396                 }
12397
12398                 if (old_plane_state->color_plane[0].stride !=
12399                     new_plane_state->color_plane[0].stride) {
12400                         drm_dbg_kms(&i915->drm, "Stride cannot be changed in async flip\n");
12401                         return -EINVAL;
12402                 }
12403
12404                 if (old_plane_state->hw.fb->modifier !=
12405                     new_plane_state->hw.fb->modifier) {
12406                         drm_dbg_kms(&i915->drm,
12407                                     "Framebuffer modifiers cannot be changed in async flip\n");
12408                         return -EINVAL;
12409                 }
12410
12411                 if (old_plane_state->hw.fb->format !=
12412                     new_plane_state->hw.fb->format) {
12413                         drm_dbg_kms(&i915->drm,
12414                                     "Framebuffer format cannot be changed in async flip\n");
12415                         return -EINVAL;
12416                 }
12417
12418                 if (old_plane_state->hw.rotation !=
12419                     new_plane_state->hw.rotation) {
12420                         drm_dbg_kms(&i915->drm, "Rotation cannot be changed in async flip\n");
12421                         return -EINVAL;
12422                 }
12423
12424                 if (!drm_rect_equals(&old_plane_state->uapi.src, &new_plane_state->uapi.src) ||
12425                     !drm_rect_equals(&old_plane_state->uapi.dst, &new_plane_state->uapi.dst)) {
12426                         drm_dbg_kms(&i915->drm,
12427                                     "Plane size/co-ordinates cannot be changed in async flip\n");
12428                         return -EINVAL;
12429                 }
12430
12431                 if (old_plane_state->hw.alpha != new_plane_state->hw.alpha) {
12432                         drm_dbg_kms(&i915->drm, "Alpha value cannot be changed in async flip\n");
12433                         return -EINVAL;
12434                 }
12435
12436                 if (old_plane_state->hw.pixel_blend_mode !=
12437                     new_plane_state->hw.pixel_blend_mode) {
12438                         drm_dbg_kms(&i915->drm,
12439                                     "Pixel blend mode cannot be changed in async flip\n");
12440                         return -EINVAL;
12441                 }
12442
12443                 if (old_plane_state->hw.color_encoding != new_plane_state->hw.color_encoding) {
12444                         drm_dbg_kms(&i915->drm,
12445                                     "Color encoding cannot be changed in async flip\n");
12446                         return -EINVAL;
12447                 }
12448
12449                 if (old_plane_state->hw.color_range != new_plane_state->hw.color_range) {
12450                         drm_dbg_kms(&i915->drm, "Color range cannot be changed in async flip\n");
12451                         return -EINVAL;
12452                 }
12453         }
12454
12455         return 0;
12456 }
12457
12458 static int intel_bigjoiner_add_affected_crtcs(struct intel_atomic_state *state)
12459 {
12460         struct intel_crtc_state *crtc_state;
12461         struct intel_crtc *crtc;
12462         int i;
12463
12464         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
12465                 struct intel_crtc_state *linked_crtc_state;
12466                 struct intel_crtc *linked_crtc;
12467                 int ret;
12468
12469                 if (!crtc_state->bigjoiner)
12470                         continue;
12471
12472                 linked_crtc = crtc_state->bigjoiner_linked_crtc;
12473                 linked_crtc_state = intel_atomic_get_crtc_state(&state->base, linked_crtc);
12474                 if (IS_ERR(linked_crtc_state))
12475                         return PTR_ERR(linked_crtc_state);
12476
12477                 if (!intel_crtc_needs_modeset(crtc_state))
12478                         continue;
12479
12480                 linked_crtc_state->uapi.mode_changed = true;
12481
12482                 ret = drm_atomic_add_affected_connectors(&state->base,
12483                                                          &linked_crtc->base);
12484                 if (ret)
12485                         return ret;
12486
12487                 ret = intel_atomic_add_affected_planes(state, linked_crtc);
12488                 if (ret)
12489                         return ret;
12490         }
12491
12492         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
12493                 /* Kill old bigjoiner link, we may re-establish afterwards */
12494                 if (intel_crtc_needs_modeset(crtc_state) &&
12495                     crtc_state->bigjoiner && !crtc_state->bigjoiner_slave)
12496                         kill_bigjoiner_slave(state, crtc_state);
12497         }
12498
12499         return 0;
12500 }
12501
12502 /**
12503  * intel_atomic_check - validate state object
12504  * @dev: drm device
12505  * @_state: state to validate
12506  */
12507 static int intel_atomic_check(struct drm_device *dev,
12508                               struct drm_atomic_state *_state)
12509 {
12510         struct drm_i915_private *dev_priv = to_i915(dev);
12511         struct intel_atomic_state *state = to_intel_atomic_state(_state);
12512         struct intel_crtc_state *old_crtc_state, *new_crtc_state;
12513         struct intel_crtc *crtc;
12514         int ret, i;
12515         bool any_ms = false;
12516
12517         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
12518                                             new_crtc_state, i) {
12519                 if (new_crtc_state->inherited != old_crtc_state->inherited)
12520                         new_crtc_state->uapi.mode_changed = true;
12521         }
12522
12523         intel_vrr_check_modeset(state);
12524
12525         ret = drm_atomic_helper_check_modeset(dev, &state->base);
12526         if (ret)
12527                 goto fail;
12528
12529         ret = intel_bigjoiner_add_affected_crtcs(state);
12530         if (ret)
12531                 goto fail;
12532
12533         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
12534                                             new_crtc_state, i) {
12535                 if (!intel_crtc_needs_modeset(new_crtc_state)) {
12536                         /* Light copy */
12537                         intel_crtc_copy_uapi_to_hw_state_nomodeset(state, new_crtc_state);
12538
12539                         continue;
12540                 }
12541
12542                 if (!new_crtc_state->uapi.enable) {
12543                         if (!new_crtc_state->bigjoiner_slave) {
12544                                 intel_crtc_copy_uapi_to_hw_state(state, new_crtc_state);
12545                                 any_ms = true;
12546                         }
12547                         continue;
12548                 }
12549
12550                 ret = intel_crtc_prepare_cleared_state(state, new_crtc_state);
12551                 if (ret)
12552                         goto fail;
12553
12554                 ret = intel_modeset_pipe_config(state, new_crtc_state);
12555                 if (ret)
12556                         goto fail;
12557
12558                 ret = intel_atomic_check_bigjoiner(state, crtc, old_crtc_state,
12559                                                    new_crtc_state);
12560                 if (ret)
12561                         goto fail;
12562         }
12563
12564         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
12565                                             new_crtc_state, i) {
12566                 if (!intel_crtc_needs_modeset(new_crtc_state))
12567                         continue;
12568
12569                 ret = intel_modeset_pipe_config_late(new_crtc_state);
12570                 if (ret)
12571                         goto fail;
12572
12573                 intel_crtc_check_fastset(old_crtc_state, new_crtc_state);
12574         }
12575
12576         /**
12577          * Check if fastset is allowed by external dependencies like other
12578          * pipes and transcoders.
12579          *
12580          * Right now it only forces a fullmodeset when the MST master
12581          * transcoder did not changed but the pipe of the master transcoder
12582          * needs a fullmodeset so all slaves also needs to do a fullmodeset or
12583          * in case of port synced crtcs, if one of the synced crtcs
12584          * needs a full modeset, all other synced crtcs should be
12585          * forced a full modeset.
12586          */
12587         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
12588                 if (!new_crtc_state->hw.enable || intel_crtc_needs_modeset(new_crtc_state))
12589                         continue;
12590
12591                 if (intel_dp_mst_is_slave_trans(new_crtc_state)) {
12592                         enum transcoder master = new_crtc_state->mst_master_transcoder;
12593
12594                         if (intel_cpu_transcoders_need_modeset(state, BIT(master))) {
12595                                 new_crtc_state->uapi.mode_changed = true;
12596                                 new_crtc_state->update_pipe = false;
12597                         }
12598                 }
12599
12600                 if (is_trans_port_sync_mode(new_crtc_state)) {
12601                         u8 trans = new_crtc_state->sync_mode_slaves_mask;
12602
12603                         if (new_crtc_state->master_transcoder != INVALID_TRANSCODER)
12604                                 trans |= BIT(new_crtc_state->master_transcoder);
12605
12606                         if (intel_cpu_transcoders_need_modeset(state, trans)) {
12607                                 new_crtc_state->uapi.mode_changed = true;
12608                                 new_crtc_state->update_pipe = false;
12609                         }
12610                 }
12611
12612                 if (new_crtc_state->bigjoiner) {
12613                         struct intel_crtc_state *linked_crtc_state =
12614                                 intel_atomic_get_new_crtc_state(state, new_crtc_state->bigjoiner_linked_crtc);
12615
12616                         if (intel_crtc_needs_modeset(linked_crtc_state)) {
12617                                 new_crtc_state->uapi.mode_changed = true;
12618                                 new_crtc_state->update_pipe = false;
12619                         }
12620                 }
12621         }
12622
12623         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
12624                                             new_crtc_state, i) {
12625                 if (intel_crtc_needs_modeset(new_crtc_state)) {
12626                         any_ms = true;
12627                         continue;
12628                 }
12629
12630                 if (!new_crtc_state->update_pipe)
12631                         continue;
12632
12633                 intel_crtc_copy_fastset(old_crtc_state, new_crtc_state);
12634         }
12635
12636         if (any_ms && !check_digital_port_conflicts(state)) {
12637                 drm_dbg_kms(&dev_priv->drm,
12638                             "rejecting conflicting digital port configuration\n");
12639                 ret = -EINVAL;
12640                 goto fail;
12641         }
12642
12643         ret = drm_dp_mst_atomic_check(&state->base);
12644         if (ret)
12645                 goto fail;
12646
12647         ret = intel_atomic_check_planes(state);
12648         if (ret)
12649                 goto fail;
12650
12651         intel_fbc_choose_crtc(dev_priv, state);
12652         ret = calc_watermark_data(state);
12653         if (ret)
12654                 goto fail;
12655
12656         ret = intel_bw_atomic_check(state);
12657         if (ret)
12658                 goto fail;
12659
12660         ret = intel_atomic_check_cdclk(state, &any_ms);
12661         if (ret)
12662                 goto fail;
12663
12664         if (any_ms) {
12665                 ret = intel_modeset_checks(state);
12666                 if (ret)
12667                         goto fail;
12668
12669                 ret = intel_modeset_calc_cdclk(state);
12670                 if (ret)
12671                         return ret;
12672
12673                 intel_modeset_clear_plls(state);
12674         }
12675
12676         ret = intel_atomic_check_crtcs(state);
12677         if (ret)
12678                 goto fail;
12679
12680         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
12681                                             new_crtc_state, i) {
12682                 if (new_crtc_state->uapi.async_flip) {
12683                         ret = intel_atomic_check_async(state);
12684                         if (ret)
12685                                 goto fail;
12686                 }
12687
12688                 if (!intel_crtc_needs_modeset(new_crtc_state) &&
12689                     !new_crtc_state->update_pipe)
12690                         continue;
12691
12692                 intel_dump_pipe_config(new_crtc_state, state,
12693                                        intel_crtc_needs_modeset(new_crtc_state) ?
12694                                        "[modeset]" : "[fastset]");
12695         }
12696
12697         return 0;
12698
12699  fail:
12700         if (ret == -EDEADLK)
12701                 return ret;
12702
12703         /*
12704          * FIXME would probably be nice to know which crtc specifically
12705          * caused the failure, in cases where we can pinpoint it.
12706          */
12707         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
12708                                             new_crtc_state, i)
12709                 intel_dump_pipe_config(new_crtc_state, state, "[failed]");
12710
12711         return ret;
12712 }
12713
12714 static int intel_atomic_prepare_commit(struct intel_atomic_state *state)
12715 {
12716         struct intel_crtc_state *crtc_state;
12717         struct intel_crtc *crtc;
12718         int i, ret;
12719
12720         ret = drm_atomic_helper_prepare_planes(state->base.dev, &state->base);
12721         if (ret < 0)
12722                 return ret;
12723
12724         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
12725                 bool mode_changed = intel_crtc_needs_modeset(crtc_state);
12726
12727                 if (mode_changed || crtc_state->update_pipe ||
12728                     crtc_state->uapi.color_mgmt_changed) {
12729                         intel_dsb_prepare(crtc_state);
12730                 }
12731         }
12732
12733         return 0;
12734 }
12735
12736 void intel_crtc_arm_fifo_underrun(struct intel_crtc *crtc,
12737                                   struct intel_crtc_state *crtc_state)
12738 {
12739         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
12740
12741         if (!IS_GEN(dev_priv, 2) || crtc_state->active_planes)
12742                 intel_set_cpu_fifo_underrun_reporting(dev_priv, crtc->pipe, true);
12743
12744         if (crtc_state->has_pch_encoder) {
12745                 enum pipe pch_transcoder =
12746                         intel_crtc_pch_transcoder(crtc);
12747
12748                 intel_set_pch_fifo_underrun_reporting(dev_priv, pch_transcoder, true);
12749         }
12750 }
12751
12752 static void intel_pipe_fastset(const struct intel_crtc_state *old_crtc_state,
12753                                const struct intel_crtc_state *new_crtc_state)
12754 {
12755         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
12756         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
12757
12758         /*
12759          * Update pipe size and adjust fitter if needed: the reason for this is
12760          * that in compute_mode_changes we check the native mode (not the pfit
12761          * mode) to see if we can flip rather than do a full mode set. In the
12762          * fastboot case, we'll flip, but if we don't update the pipesrc and
12763          * pfit state, we'll end up with a big fb scanned out into the wrong
12764          * sized surface.
12765          */
12766         intel_set_pipe_src_size(new_crtc_state);
12767
12768         /* on skylake this is done by detaching scalers */
12769         if (INTEL_GEN(dev_priv) >= 9) {
12770                 skl_detach_scalers(new_crtc_state);
12771
12772                 if (new_crtc_state->pch_pfit.enabled)
12773                         skl_pfit_enable(new_crtc_state);
12774         } else if (HAS_PCH_SPLIT(dev_priv)) {
12775                 if (new_crtc_state->pch_pfit.enabled)
12776                         ilk_pfit_enable(new_crtc_state);
12777                 else if (old_crtc_state->pch_pfit.enabled)
12778                         ilk_pfit_disable(old_crtc_state);
12779         }
12780
12781         /*
12782          * The register is supposedly single buffered so perhaps
12783          * not 100% correct to do this here. But SKL+ calculate
12784          * this based on the adjust pixel rate so pfit changes do
12785          * affect it and so it must be updated for fastsets.
12786          * HSW/BDW only really need this here for fastboot, after
12787          * that the value should not change without a full modeset.
12788          */
12789         if (INTEL_GEN(dev_priv) >= 9 ||
12790             IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
12791                 hsw_set_linetime_wm(new_crtc_state);
12792
12793         if (INTEL_GEN(dev_priv) >= 11)
12794                 icl_set_pipe_chicken(crtc);
12795 }
12796
12797 static void commit_pipe_config(struct intel_atomic_state *state,
12798                                struct intel_crtc *crtc)
12799 {
12800         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
12801         const struct intel_crtc_state *old_crtc_state =
12802                 intel_atomic_get_old_crtc_state(state, crtc);
12803         const struct intel_crtc_state *new_crtc_state =
12804                 intel_atomic_get_new_crtc_state(state, crtc);
12805         bool modeset = intel_crtc_needs_modeset(new_crtc_state);
12806
12807         /*
12808          * During modesets pipe configuration was programmed as the
12809          * CRTC was enabled.
12810          */
12811         if (!modeset) {
12812                 if (new_crtc_state->uapi.color_mgmt_changed ||
12813                     new_crtc_state->update_pipe)
12814                         intel_color_commit(new_crtc_state);
12815
12816                 if (INTEL_GEN(dev_priv) >= 9)
12817                         skl_detach_scalers(new_crtc_state);
12818
12819                 if (INTEL_GEN(dev_priv) >= 9 || IS_BROADWELL(dev_priv))
12820                         bdw_set_pipemisc(new_crtc_state);
12821
12822                 if (new_crtc_state->update_pipe)
12823                         intel_pipe_fastset(old_crtc_state, new_crtc_state);
12824
12825                 intel_psr2_program_trans_man_trk_ctl(new_crtc_state);
12826         }
12827
12828         if (dev_priv->display.atomic_update_watermarks)
12829                 dev_priv->display.atomic_update_watermarks(state, crtc);
12830 }
12831
12832 static void intel_enable_crtc(struct intel_atomic_state *state,
12833                               struct intel_crtc *crtc)
12834 {
12835         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
12836         const struct intel_crtc_state *new_crtc_state =
12837                 intel_atomic_get_new_crtc_state(state, crtc);
12838
12839         if (!intel_crtc_needs_modeset(new_crtc_state))
12840                 return;
12841
12842         intel_crtc_update_active_timings(new_crtc_state);
12843
12844         dev_priv->display.crtc_enable(state, crtc);
12845
12846         if (new_crtc_state->bigjoiner_slave)
12847                 return;
12848
12849         /* vblanks work again, re-enable pipe CRC. */
12850         intel_crtc_enable_pipe_crc(crtc);
12851 }
12852
12853 static void intel_update_crtc(struct intel_atomic_state *state,
12854                               struct intel_crtc *crtc)
12855 {
12856         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
12857         const struct intel_crtc_state *old_crtc_state =
12858                 intel_atomic_get_old_crtc_state(state, crtc);
12859         struct intel_crtc_state *new_crtc_state =
12860                 intel_atomic_get_new_crtc_state(state, crtc);
12861         bool modeset = intel_crtc_needs_modeset(new_crtc_state);
12862
12863         if (!modeset) {
12864                 if (new_crtc_state->preload_luts &&
12865                     (new_crtc_state->uapi.color_mgmt_changed ||
12866                      new_crtc_state->update_pipe))
12867                         intel_color_load_luts(new_crtc_state);
12868
12869                 intel_pre_plane_update(state, crtc);
12870
12871                 if (new_crtc_state->update_pipe)
12872                         intel_encoders_update_pipe(state, crtc);
12873         }
12874
12875         if (new_crtc_state->update_pipe && !new_crtc_state->enable_fbc)
12876                 intel_fbc_disable(crtc);
12877         else
12878                 intel_fbc_enable(state, crtc);
12879
12880         /* Perform vblank evasion around commit operation */
12881         intel_pipe_update_start(new_crtc_state);
12882
12883         commit_pipe_config(state, crtc);
12884
12885         if (INTEL_GEN(dev_priv) >= 9)
12886                 skl_update_planes_on_crtc(state, crtc);
12887         else
12888                 i9xx_update_planes_on_crtc(state, crtc);
12889
12890         intel_pipe_update_end(new_crtc_state);
12891
12892         /*
12893          * We usually enable FIFO underrun interrupts as part of the
12894          * CRTC enable sequence during modesets.  But when we inherit a
12895          * valid pipe configuration from the BIOS we need to take care
12896          * of enabling them on the CRTC's first fastset.
12897          */
12898         if (new_crtc_state->update_pipe && !modeset &&
12899             old_crtc_state->inherited)
12900                 intel_crtc_arm_fifo_underrun(crtc, new_crtc_state);
12901 }
12902
12903 static void intel_old_crtc_state_disables(struct intel_atomic_state *state,
12904                                           struct intel_crtc_state *old_crtc_state,
12905                                           struct intel_crtc_state *new_crtc_state,
12906                                           struct intel_crtc *crtc)
12907 {
12908         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
12909
12910         drm_WARN_ON(&dev_priv->drm, old_crtc_state->bigjoiner_slave);
12911
12912         intel_crtc_disable_planes(state, crtc);
12913
12914         /*
12915          * We still need special handling for disabling bigjoiner master
12916          * and slaves since for slave we do not have encoder or plls
12917          * so we dont need to disable those.
12918          */
12919         if (old_crtc_state->bigjoiner) {
12920                 intel_crtc_disable_planes(state,
12921                                           old_crtc_state->bigjoiner_linked_crtc);
12922                 old_crtc_state->bigjoiner_linked_crtc->active = false;
12923         }
12924
12925         /*
12926          * We need to disable pipe CRC before disabling the pipe,
12927          * or we race against vblank off.
12928          */
12929         intel_crtc_disable_pipe_crc(crtc);
12930
12931         dev_priv->display.crtc_disable(state, crtc);
12932         crtc->active = false;
12933         intel_fbc_disable(crtc);
12934         intel_disable_shared_dpll(old_crtc_state);
12935
12936         /* FIXME unify this for all platforms */
12937         if (!new_crtc_state->hw.active &&
12938             !HAS_GMCH(dev_priv) &&
12939             dev_priv->display.initial_watermarks)
12940                 dev_priv->display.initial_watermarks(state, crtc);
12941 }
12942
12943 static void intel_commit_modeset_disables(struct intel_atomic_state *state)
12944 {
12945         struct intel_crtc_state *new_crtc_state, *old_crtc_state;
12946         struct intel_crtc *crtc;
12947         u32 handled = 0;
12948         int i;
12949
12950         /* Only disable port sync and MST slaves */
12951         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
12952                                             new_crtc_state, i) {
12953                 if (!intel_crtc_needs_modeset(new_crtc_state) || old_crtc_state->bigjoiner)
12954                         continue;
12955
12956                 if (!old_crtc_state->hw.active)
12957                         continue;
12958
12959                 /* In case of Transcoder port Sync master slave CRTCs can be
12960                  * assigned in any order and we need to make sure that
12961                  * slave CRTCs are disabled first and then master CRTC since
12962                  * Slave vblanks are masked till Master Vblanks.
12963                  */
12964                 if (!is_trans_port_sync_slave(old_crtc_state) &&
12965                     !intel_dp_mst_is_slave_trans(old_crtc_state))
12966                         continue;
12967
12968                 intel_pre_plane_update(state, crtc);
12969                 intel_old_crtc_state_disables(state, old_crtc_state,
12970                                               new_crtc_state, crtc);
12971                 handled |= BIT(crtc->pipe);
12972         }
12973
12974         /* Disable everything else left on */
12975         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
12976                                             new_crtc_state, i) {
12977                 if (!intel_crtc_needs_modeset(new_crtc_state) ||
12978                     (handled & BIT(crtc->pipe)) ||
12979                     old_crtc_state->bigjoiner_slave)
12980                         continue;
12981
12982                 intel_pre_plane_update(state, crtc);
12983                 if (old_crtc_state->bigjoiner) {
12984                         struct intel_crtc *slave =
12985                                 old_crtc_state->bigjoiner_linked_crtc;
12986
12987                         intel_pre_plane_update(state, slave);
12988                 }
12989
12990                 if (old_crtc_state->hw.active)
12991                         intel_old_crtc_state_disables(state, old_crtc_state,
12992                                                       new_crtc_state, crtc);
12993         }
12994 }
12995
12996 static void intel_commit_modeset_enables(struct intel_atomic_state *state)
12997 {
12998         struct intel_crtc_state *new_crtc_state;
12999         struct intel_crtc *crtc;
13000         int i;
13001
13002         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
13003                 if (!new_crtc_state->hw.active)
13004                         continue;
13005
13006                 intel_enable_crtc(state, crtc);
13007                 intel_update_crtc(state, crtc);
13008         }
13009 }
13010
13011 static void skl_commit_modeset_enables(struct intel_atomic_state *state)
13012 {
13013         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
13014         struct intel_crtc *crtc;
13015         struct intel_crtc_state *old_crtc_state, *new_crtc_state;
13016         struct skl_ddb_entry entries[I915_MAX_PIPES] = {};
13017         u8 update_pipes = 0, modeset_pipes = 0;
13018         int i;
13019
13020         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
13021                 enum pipe pipe = crtc->pipe;
13022
13023                 if (!new_crtc_state->hw.active)
13024                         continue;
13025
13026                 /* ignore allocations for crtc's that have been turned off. */
13027                 if (!intel_crtc_needs_modeset(new_crtc_state)) {
13028                         entries[pipe] = old_crtc_state->wm.skl.ddb;
13029                         update_pipes |= BIT(pipe);
13030                 } else {
13031                         modeset_pipes |= BIT(pipe);
13032                 }
13033         }
13034
13035         /*
13036          * Whenever the number of active pipes changes, we need to make sure we
13037          * update the pipes in the right order so that their ddb allocations
13038          * never overlap with each other between CRTC updates. Otherwise we'll
13039          * cause pipe underruns and other bad stuff.
13040          *
13041          * So first lets enable all pipes that do not need a fullmodeset as
13042          * those don't have any external dependency.
13043          */
13044         while (update_pipes) {
13045                 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
13046                                                     new_crtc_state, i) {
13047                         enum pipe pipe = crtc->pipe;
13048
13049                         if ((update_pipes & BIT(pipe)) == 0)
13050                                 continue;
13051
13052                         if (skl_ddb_allocation_overlaps(&new_crtc_state->wm.skl.ddb,
13053                                                         entries, I915_MAX_PIPES, pipe))
13054                                 continue;
13055
13056                         entries[pipe] = new_crtc_state->wm.skl.ddb;
13057                         update_pipes &= ~BIT(pipe);
13058
13059                         intel_update_crtc(state, crtc);
13060
13061                         /*
13062                          * If this is an already active pipe, it's DDB changed,
13063                          * and this isn't the last pipe that needs updating
13064                          * then we need to wait for a vblank to pass for the
13065                          * new ddb allocation to take effect.
13066                          */
13067                         if (!skl_ddb_entry_equal(&new_crtc_state->wm.skl.ddb,
13068                                                  &old_crtc_state->wm.skl.ddb) &&
13069                             (update_pipes | modeset_pipes))
13070                                 intel_wait_for_vblank(dev_priv, pipe);
13071                 }
13072         }
13073
13074         update_pipes = modeset_pipes;
13075
13076         /*
13077          * Enable all pipes that needs a modeset and do not depends on other
13078          * pipes
13079          */
13080         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
13081                 enum pipe pipe = crtc->pipe;
13082
13083                 if ((modeset_pipes & BIT(pipe)) == 0)
13084                         continue;
13085
13086                 if (intel_dp_mst_is_slave_trans(new_crtc_state) ||
13087                     is_trans_port_sync_master(new_crtc_state) ||
13088                     (new_crtc_state->bigjoiner && !new_crtc_state->bigjoiner_slave))
13089                         continue;
13090
13091                 modeset_pipes &= ~BIT(pipe);
13092
13093                 intel_enable_crtc(state, crtc);
13094         }
13095
13096         /*
13097          * Then we enable all remaining pipes that depend on other
13098          * pipes: MST slaves and port sync masters, big joiner master
13099          */
13100         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
13101                 enum pipe pipe = crtc->pipe;
13102
13103                 if ((modeset_pipes & BIT(pipe)) == 0)
13104                         continue;
13105
13106                 modeset_pipes &= ~BIT(pipe);
13107
13108                 intel_enable_crtc(state, crtc);
13109         }
13110
13111         /*
13112          * Finally we do the plane updates/etc. for all pipes that got enabled.
13113          */
13114         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
13115                 enum pipe pipe = crtc->pipe;
13116
13117                 if ((update_pipes & BIT(pipe)) == 0)
13118                         continue;
13119
13120                 drm_WARN_ON(&dev_priv->drm, skl_ddb_allocation_overlaps(&new_crtc_state->wm.skl.ddb,
13121                                                                         entries, I915_MAX_PIPES, pipe));
13122
13123                 entries[pipe] = new_crtc_state->wm.skl.ddb;
13124                 update_pipes &= ~BIT(pipe);
13125
13126                 intel_update_crtc(state, crtc);
13127         }
13128
13129         drm_WARN_ON(&dev_priv->drm, modeset_pipes);
13130         drm_WARN_ON(&dev_priv->drm, update_pipes);
13131 }
13132
13133 static void intel_atomic_helper_free_state(struct drm_i915_private *dev_priv)
13134 {
13135         struct intel_atomic_state *state, *next;
13136         struct llist_node *freed;
13137
13138         freed = llist_del_all(&dev_priv->atomic_helper.free_list);
13139         llist_for_each_entry_safe(state, next, freed, freed)
13140                 drm_atomic_state_put(&state->base);
13141 }
13142
13143 static void intel_atomic_helper_free_state_worker(struct work_struct *work)
13144 {
13145         struct drm_i915_private *dev_priv =
13146                 container_of(work, typeof(*dev_priv), atomic_helper.free_work);
13147
13148         intel_atomic_helper_free_state(dev_priv);
13149 }
13150
13151 static void intel_atomic_commit_fence_wait(struct intel_atomic_state *intel_state)
13152 {
13153         struct wait_queue_entry wait_fence, wait_reset;
13154         struct drm_i915_private *dev_priv = to_i915(intel_state->base.dev);
13155
13156         init_wait_entry(&wait_fence, 0);
13157         init_wait_entry(&wait_reset, 0);
13158         for (;;) {
13159                 prepare_to_wait(&intel_state->commit_ready.wait,
13160                                 &wait_fence, TASK_UNINTERRUPTIBLE);
13161                 prepare_to_wait(bit_waitqueue(&dev_priv->gt.reset.flags,
13162                                               I915_RESET_MODESET),
13163                                 &wait_reset, TASK_UNINTERRUPTIBLE);
13164
13165
13166                 if (i915_sw_fence_done(&intel_state->commit_ready) ||
13167                     test_bit(I915_RESET_MODESET, &dev_priv->gt.reset.flags))
13168                         break;
13169
13170                 schedule();
13171         }
13172         finish_wait(&intel_state->commit_ready.wait, &wait_fence);
13173         finish_wait(bit_waitqueue(&dev_priv->gt.reset.flags,
13174                                   I915_RESET_MODESET),
13175                     &wait_reset);
13176 }
13177
13178 static void intel_cleanup_dsbs(struct intel_atomic_state *state)
13179 {
13180         struct intel_crtc_state *old_crtc_state, *new_crtc_state;
13181         struct intel_crtc *crtc;
13182         int i;
13183
13184         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
13185                                             new_crtc_state, i)
13186                 intel_dsb_cleanup(old_crtc_state);
13187 }
13188
13189 static void intel_atomic_cleanup_work(struct work_struct *work)
13190 {
13191         struct intel_atomic_state *state =
13192                 container_of(work, struct intel_atomic_state, base.commit_work);
13193         struct drm_i915_private *i915 = to_i915(state->base.dev);
13194
13195         intel_cleanup_dsbs(state);
13196         drm_atomic_helper_cleanup_planes(&i915->drm, &state->base);
13197         drm_atomic_helper_commit_cleanup_done(&state->base);
13198         drm_atomic_state_put(&state->base);
13199
13200         intel_atomic_helper_free_state(i915);
13201 }
13202
13203 static void intel_atomic_prepare_plane_clear_colors(struct intel_atomic_state *state)
13204 {
13205         struct drm_i915_private *i915 = to_i915(state->base.dev);
13206         struct intel_plane *plane;
13207         struct intel_plane_state *plane_state;
13208         int i;
13209
13210         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
13211                 struct drm_framebuffer *fb = plane_state->hw.fb;
13212                 int ret;
13213
13214                 if (!fb ||
13215                     fb->modifier != I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS_CC)
13216                         continue;
13217
13218                 /*
13219                  * The layout of the fast clear color value expected by HW
13220                  * (the DRM ABI requiring this value to be located in fb at offset 0 of plane#2):
13221                  * - 4 x 4 bytes per-channel value
13222                  *   (in surface type specific float/int format provided by the fb user)
13223                  * - 8 bytes native color value used by the display
13224                  *   (converted/written by GPU during a fast clear operation using the
13225                  *    above per-channel values)
13226                  *
13227                  * The commit's FB prepare hook already ensured that FB obj is pinned and the
13228                  * caller made sure that the object is synced wrt. the related color clear value
13229                  * GPU write on it.
13230                  */
13231                 ret = i915_gem_object_read_from_page(intel_fb_obj(fb),
13232                                                      fb->offsets[2] + 16,
13233                                                      &plane_state->ccval,
13234                                                      sizeof(plane_state->ccval));
13235                 /* The above could only fail if the FB obj has an unexpected backing store type. */
13236                 drm_WARN_ON(&i915->drm, ret);
13237         }
13238 }
13239
13240 static void intel_atomic_commit_tail(struct intel_atomic_state *state)
13241 {
13242         struct drm_device *dev = state->base.dev;
13243         struct drm_i915_private *dev_priv = to_i915(dev);
13244         struct intel_crtc_state *new_crtc_state, *old_crtc_state;
13245         struct intel_crtc *crtc;
13246         u64 put_domains[I915_MAX_PIPES] = {};
13247         intel_wakeref_t wakeref = 0;
13248         int i;
13249
13250         intel_atomic_commit_fence_wait(state);
13251
13252         drm_atomic_helper_wait_for_dependencies(&state->base);
13253
13254         if (state->modeset)
13255                 wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_MODESET);
13256
13257         intel_atomic_prepare_plane_clear_colors(state);
13258
13259         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
13260                                             new_crtc_state, i) {
13261                 if (intel_crtc_needs_modeset(new_crtc_state) ||
13262                     new_crtc_state->update_pipe) {
13263
13264                         put_domains[crtc->pipe] =
13265                                 modeset_get_crtc_power_domains(new_crtc_state);
13266                 }
13267         }
13268
13269         intel_commit_modeset_disables(state);
13270
13271         /* FIXME: Eventually get rid of our crtc->config pointer */
13272         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i)
13273                 crtc->config = new_crtc_state;
13274
13275         if (state->modeset) {
13276                 drm_atomic_helper_update_legacy_modeset_state(dev, &state->base);
13277
13278                 intel_set_cdclk_pre_plane_update(state);
13279
13280                 intel_modeset_verify_disabled(dev_priv, state);
13281         }
13282
13283         intel_sagv_pre_plane_update(state);
13284
13285         /* Complete the events for pipes that have now been disabled */
13286         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
13287                 bool modeset = intel_crtc_needs_modeset(new_crtc_state);
13288
13289                 /* Complete events for now disable pipes here. */
13290                 if (modeset && !new_crtc_state->hw.active && new_crtc_state->uapi.event) {
13291                         spin_lock_irq(&dev->event_lock);
13292                         drm_crtc_send_vblank_event(&crtc->base,
13293                                                    new_crtc_state->uapi.event);
13294                         spin_unlock_irq(&dev->event_lock);
13295
13296                         new_crtc_state->uapi.event = NULL;
13297                 }
13298         }
13299
13300         if (state->modeset)
13301                 intel_encoders_update_prepare(state);
13302
13303         intel_dbuf_pre_plane_update(state);
13304
13305         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
13306                 if (new_crtc_state->uapi.async_flip)
13307                         intel_crtc_enable_flip_done(state, crtc);
13308         }
13309
13310         /* Now enable the clocks, plane, pipe, and connectors that we set up. */
13311         dev_priv->display.commit_modeset_enables(state);
13312
13313         if (state->modeset) {
13314                 intel_encoders_update_complete(state);
13315
13316                 intel_set_cdclk_post_plane_update(state);
13317         }
13318
13319         /* FIXME: We should call drm_atomic_helper_commit_hw_done() here
13320          * already, but still need the state for the delayed optimization. To
13321          * fix this:
13322          * - wrap the optimization/post_plane_update stuff into a per-crtc work.
13323          * - schedule that vblank worker _before_ calling hw_done
13324          * - at the start of commit_tail, cancel it _synchrously
13325          * - switch over to the vblank wait helper in the core after that since
13326          *   we don't need out special handling any more.
13327          */
13328         drm_atomic_helper_wait_for_flip_done(dev, &state->base);
13329
13330         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
13331                 if (new_crtc_state->uapi.async_flip)
13332                         intel_crtc_disable_flip_done(state, crtc);
13333
13334                 if (new_crtc_state->hw.active &&
13335                     !intel_crtc_needs_modeset(new_crtc_state) &&
13336                     !new_crtc_state->preload_luts &&
13337                     (new_crtc_state->uapi.color_mgmt_changed ||
13338                      new_crtc_state->update_pipe))
13339                         intel_color_load_luts(new_crtc_state);
13340         }
13341
13342         /*
13343          * Now that the vblank has passed, we can go ahead and program the
13344          * optimal watermarks on platforms that need two-step watermark
13345          * programming.
13346          *
13347          * TODO: Move this (and other cleanup) to an async worker eventually.
13348          */
13349         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
13350                                             new_crtc_state, i) {
13351                 /*
13352                  * Gen2 reports pipe underruns whenever all planes are disabled.
13353                  * So re-enable underrun reporting after some planes get enabled.
13354                  *
13355                  * We do this before .optimize_watermarks() so that we have a
13356                  * chance of catching underruns with the intermediate watermarks
13357                  * vs. the new plane configuration.
13358                  */
13359                 if (IS_GEN(dev_priv, 2) && planes_enabling(old_crtc_state, new_crtc_state))
13360                         intel_set_cpu_fifo_underrun_reporting(dev_priv, crtc->pipe, true);
13361
13362                 if (dev_priv->display.optimize_watermarks)
13363                         dev_priv->display.optimize_watermarks(state, crtc);
13364         }
13365
13366         intel_dbuf_post_plane_update(state);
13367
13368         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
13369                 intel_post_plane_update(state, crtc);
13370
13371                 modeset_put_crtc_power_domains(crtc, put_domains[crtc->pipe]);
13372
13373                 intel_modeset_verify_crtc(crtc, state, old_crtc_state, new_crtc_state);
13374
13375                 /*
13376                  * DSB cleanup is done in cleanup_work aligning with framebuffer
13377                  * cleanup. So copy and reset the dsb structure to sync with
13378                  * commit_done and later do dsb cleanup in cleanup_work.
13379                  */
13380                 old_crtc_state->dsb = fetch_and_zero(&new_crtc_state->dsb);
13381         }
13382
13383         /* Underruns don't always raise interrupts, so check manually */
13384         intel_check_cpu_fifo_underruns(dev_priv);
13385         intel_check_pch_fifo_underruns(dev_priv);
13386
13387         if (state->modeset)
13388                 intel_verify_planes(state);
13389
13390         intel_sagv_post_plane_update(state);
13391
13392         drm_atomic_helper_commit_hw_done(&state->base);
13393
13394         if (state->modeset) {
13395                 /* As one of the primary mmio accessors, KMS has a high
13396                  * likelihood of triggering bugs in unclaimed access. After we
13397                  * finish modesetting, see if an error has been flagged, and if
13398                  * so enable debugging for the next modeset - and hope we catch
13399                  * the culprit.
13400                  */
13401                 intel_uncore_arm_unclaimed_mmio_detection(&dev_priv->uncore);
13402                 intel_display_power_put(dev_priv, POWER_DOMAIN_MODESET, wakeref);
13403         }
13404         intel_runtime_pm_put(&dev_priv->runtime_pm, state->wakeref);
13405
13406         /*
13407          * Defer the cleanup of the old state to a separate worker to not
13408          * impede the current task (userspace for blocking modesets) that
13409          * are executed inline. For out-of-line asynchronous modesets/flips,
13410          * deferring to a new worker seems overkill, but we would place a
13411          * schedule point (cond_resched()) here anyway to keep latencies
13412          * down.
13413          */
13414         INIT_WORK(&state->base.commit_work, intel_atomic_cleanup_work);
13415         queue_work(system_highpri_wq, &state->base.commit_work);
13416 }
13417
13418 static void intel_atomic_commit_work(struct work_struct *work)
13419 {
13420         struct intel_atomic_state *state =
13421                 container_of(work, struct intel_atomic_state, base.commit_work);
13422
13423         intel_atomic_commit_tail(state);
13424 }
13425
13426 static int __i915_sw_fence_call
13427 intel_atomic_commit_ready(struct i915_sw_fence *fence,
13428                           enum i915_sw_fence_notify notify)
13429 {
13430         struct intel_atomic_state *state =
13431                 container_of(fence, struct intel_atomic_state, commit_ready);
13432
13433         switch (notify) {
13434         case FENCE_COMPLETE:
13435                 /* we do blocking waits in the worker, nothing to do here */
13436                 break;
13437         case FENCE_FREE:
13438                 {
13439                         struct intel_atomic_helper *helper =
13440                                 &to_i915(state->base.dev)->atomic_helper;
13441
13442                         if (llist_add(&state->freed, &helper->free_list))
13443                                 schedule_work(&helper->free_work);
13444                         break;
13445                 }
13446         }
13447
13448         return NOTIFY_DONE;
13449 }
13450
13451 static void intel_atomic_track_fbs(struct intel_atomic_state *state)
13452 {
13453         struct intel_plane_state *old_plane_state, *new_plane_state;
13454         struct intel_plane *plane;
13455         int i;
13456
13457         for_each_oldnew_intel_plane_in_state(state, plane, old_plane_state,
13458                                              new_plane_state, i)
13459                 intel_frontbuffer_track(to_intel_frontbuffer(old_plane_state->hw.fb),
13460                                         to_intel_frontbuffer(new_plane_state->hw.fb),
13461                                         plane->frontbuffer_bit);
13462 }
13463
13464 static int intel_atomic_commit(struct drm_device *dev,
13465                                struct drm_atomic_state *_state,
13466                                bool nonblock)
13467 {
13468         struct intel_atomic_state *state = to_intel_atomic_state(_state);
13469         struct drm_i915_private *dev_priv = to_i915(dev);
13470         int ret = 0;
13471
13472         state->wakeref = intel_runtime_pm_get(&dev_priv->runtime_pm);
13473
13474         drm_atomic_state_get(&state->base);
13475         i915_sw_fence_init(&state->commit_ready,
13476                            intel_atomic_commit_ready);
13477
13478         /*
13479          * The intel_legacy_cursor_update() fast path takes care
13480          * of avoiding the vblank waits for simple cursor
13481          * movement and flips. For cursor on/off and size changes,
13482          * we want to perform the vblank waits so that watermark
13483          * updates happen during the correct frames. Gen9+ have
13484          * double buffered watermarks and so shouldn't need this.
13485          *
13486          * Unset state->legacy_cursor_update before the call to
13487          * drm_atomic_helper_setup_commit() because otherwise
13488          * drm_atomic_helper_wait_for_flip_done() is a noop and
13489          * we get FIFO underruns because we didn't wait
13490          * for vblank.
13491          *
13492          * FIXME doing watermarks and fb cleanup from a vblank worker
13493          * (assuming we had any) would solve these problems.
13494          */
13495         if (INTEL_GEN(dev_priv) < 9 && state->base.legacy_cursor_update) {
13496                 struct intel_crtc_state *new_crtc_state;
13497                 struct intel_crtc *crtc;
13498                 int i;
13499
13500                 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i)
13501                         if (new_crtc_state->wm.need_postvbl_update ||
13502                             new_crtc_state->update_wm_post)
13503                                 state->base.legacy_cursor_update = false;
13504         }
13505
13506         ret = intel_atomic_prepare_commit(state);
13507         if (ret) {
13508                 drm_dbg_atomic(&dev_priv->drm,
13509                                "Preparing state failed with %i\n", ret);
13510                 i915_sw_fence_commit(&state->commit_ready);
13511                 intel_runtime_pm_put(&dev_priv->runtime_pm, state->wakeref);
13512                 return ret;
13513         }
13514
13515         ret = drm_atomic_helper_setup_commit(&state->base, nonblock);
13516         if (!ret)
13517                 ret = drm_atomic_helper_swap_state(&state->base, true);
13518         if (!ret)
13519                 intel_atomic_swap_global_state(state);
13520
13521         if (ret) {
13522                 struct intel_crtc_state *new_crtc_state;
13523                 struct intel_crtc *crtc;
13524                 int i;
13525
13526                 i915_sw_fence_commit(&state->commit_ready);
13527
13528                 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i)
13529                         intel_dsb_cleanup(new_crtc_state);
13530
13531                 drm_atomic_helper_cleanup_planes(dev, &state->base);
13532                 intel_runtime_pm_put(&dev_priv->runtime_pm, state->wakeref);
13533                 return ret;
13534         }
13535         intel_shared_dpll_swap_state(state);
13536         intel_atomic_track_fbs(state);
13537
13538         drm_atomic_state_get(&state->base);
13539         INIT_WORK(&state->base.commit_work, intel_atomic_commit_work);
13540
13541         i915_sw_fence_commit(&state->commit_ready);
13542         if (nonblock && state->modeset) {
13543                 queue_work(dev_priv->modeset_wq, &state->base.commit_work);
13544         } else if (nonblock) {
13545                 queue_work(dev_priv->flip_wq, &state->base.commit_work);
13546         } else {
13547                 if (state->modeset)
13548                         flush_workqueue(dev_priv->modeset_wq);
13549                 intel_atomic_commit_tail(state);
13550         }
13551
13552         return 0;
13553 }
13554
13555 struct wait_rps_boost {
13556         struct wait_queue_entry wait;
13557
13558         struct drm_crtc *crtc;
13559         struct i915_request *request;
13560 };
13561
13562 static int do_rps_boost(struct wait_queue_entry *_wait,
13563                         unsigned mode, int sync, void *key)
13564 {
13565         struct wait_rps_boost *wait = container_of(_wait, typeof(*wait), wait);
13566         struct i915_request *rq = wait->request;
13567
13568         /*
13569          * If we missed the vblank, but the request is already running it
13570          * is reasonable to assume that it will complete before the next
13571          * vblank without our intervention, so leave RPS alone.
13572          */
13573         if (!i915_request_started(rq))
13574                 intel_rps_boost(rq);
13575         i915_request_put(rq);
13576
13577         drm_crtc_vblank_put(wait->crtc);
13578
13579         list_del(&wait->wait.entry);
13580         kfree(wait);
13581         return 1;
13582 }
13583
13584 static void add_rps_boost_after_vblank(struct drm_crtc *crtc,
13585                                        struct dma_fence *fence)
13586 {
13587         struct wait_rps_boost *wait;
13588
13589         if (!dma_fence_is_i915(fence))
13590                 return;
13591
13592         if (INTEL_GEN(to_i915(crtc->dev)) < 6)
13593                 return;
13594
13595         if (drm_crtc_vblank_get(crtc))
13596                 return;
13597
13598         wait = kmalloc(sizeof(*wait), GFP_KERNEL);
13599         if (!wait) {
13600                 drm_crtc_vblank_put(crtc);
13601                 return;
13602         }
13603
13604         wait->request = to_request(dma_fence_get(fence));
13605         wait->crtc = crtc;
13606
13607         wait->wait.func = do_rps_boost;
13608         wait->wait.flags = 0;
13609
13610         add_wait_queue(drm_crtc_vblank_waitqueue(crtc), &wait->wait);
13611 }
13612
13613 int intel_plane_pin_fb(struct intel_plane_state *plane_state)
13614 {
13615         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
13616         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
13617         struct drm_framebuffer *fb = plane_state->hw.fb;
13618         struct i915_vma *vma;
13619
13620         if (plane->id == PLANE_CURSOR &&
13621             INTEL_INFO(dev_priv)->display.cursor_needs_physical) {
13622                 struct drm_i915_gem_object *obj = intel_fb_obj(fb);
13623                 const int align = intel_cursor_alignment(dev_priv);
13624                 int err;
13625
13626                 err = i915_gem_object_attach_phys(obj, align);
13627                 if (err)
13628                         return err;
13629         }
13630
13631         vma = intel_pin_and_fence_fb_obj(fb,
13632                                          &plane_state->view,
13633                                          intel_plane_uses_fence(plane_state),
13634                                          &plane_state->flags);
13635         if (IS_ERR(vma))
13636                 return PTR_ERR(vma);
13637
13638         plane_state->vma = vma;
13639
13640         return 0;
13641 }
13642
13643 void intel_plane_unpin_fb(struct intel_plane_state *old_plane_state)
13644 {
13645         struct i915_vma *vma;
13646
13647         vma = fetch_and_zero(&old_plane_state->vma);
13648         if (vma)
13649                 intel_unpin_fb_vma(vma, old_plane_state->flags);
13650 }
13651
13652 static void fb_obj_bump_render_priority(struct drm_i915_gem_object *obj)
13653 {
13654         struct i915_sched_attr attr = {
13655                 .priority = I915_USER_PRIORITY(I915_PRIORITY_DISPLAY),
13656         };
13657
13658         i915_gem_object_wait_priority(obj, 0, &attr);
13659 }
13660
13661 /**
13662  * intel_prepare_plane_fb - Prepare fb for usage on plane
13663  * @_plane: drm plane to prepare for
13664  * @_new_plane_state: the plane state being prepared
13665  *
13666  * Prepares a framebuffer for usage on a display plane.  Generally this
13667  * involves pinning the underlying object and updating the frontbuffer tracking
13668  * bits.  Some older platforms need special physical address handling for
13669  * cursor planes.
13670  *
13671  * Returns 0 on success, negative error code on failure.
13672  */
13673 int
13674 intel_prepare_plane_fb(struct drm_plane *_plane,
13675                        struct drm_plane_state *_new_plane_state)
13676 {
13677         struct intel_plane *plane = to_intel_plane(_plane);
13678         struct intel_plane_state *new_plane_state =
13679                 to_intel_plane_state(_new_plane_state);
13680         struct intel_atomic_state *state =
13681                 to_intel_atomic_state(new_plane_state->uapi.state);
13682         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
13683         const struct intel_plane_state *old_plane_state =
13684                 intel_atomic_get_old_plane_state(state, plane);
13685         struct drm_i915_gem_object *obj = intel_fb_obj(new_plane_state->hw.fb);
13686         struct drm_i915_gem_object *old_obj = intel_fb_obj(old_plane_state->hw.fb);
13687         int ret;
13688
13689         if (old_obj) {
13690                 const struct intel_crtc_state *crtc_state =
13691                         intel_atomic_get_new_crtc_state(state,
13692                                                         to_intel_crtc(old_plane_state->hw.crtc));
13693
13694                 /* Big Hammer, we also need to ensure that any pending
13695                  * MI_WAIT_FOR_EVENT inside a user batch buffer on the
13696                  * current scanout is retired before unpinning the old
13697                  * framebuffer. Note that we rely on userspace rendering
13698                  * into the buffer attached to the pipe they are waiting
13699                  * on. If not, userspace generates a GPU hang with IPEHR
13700                  * point to the MI_WAIT_FOR_EVENT.
13701                  *
13702                  * This should only fail upon a hung GPU, in which case we
13703                  * can safely continue.
13704                  */
13705                 if (intel_crtc_needs_modeset(crtc_state)) {
13706                         ret = i915_sw_fence_await_reservation(&state->commit_ready,
13707                                                               old_obj->base.resv, NULL,
13708                                                               false, 0,
13709                                                               GFP_KERNEL);
13710                         if (ret < 0)
13711                                 return ret;
13712                 }
13713         }
13714
13715         if (new_plane_state->uapi.fence) { /* explicit fencing */
13716                 ret = i915_sw_fence_await_dma_fence(&state->commit_ready,
13717                                                     new_plane_state->uapi.fence,
13718                                                     i915_fence_timeout(dev_priv),
13719                                                     GFP_KERNEL);
13720                 if (ret < 0)
13721                         return ret;
13722         }
13723
13724         if (!obj)
13725                 return 0;
13726
13727         ret = i915_gem_object_pin_pages(obj);
13728         if (ret)
13729                 return ret;
13730
13731         ret = intel_plane_pin_fb(new_plane_state);
13732
13733         i915_gem_object_unpin_pages(obj);
13734         if (ret)
13735                 return ret;
13736
13737         fb_obj_bump_render_priority(obj);
13738         i915_gem_object_flush_frontbuffer(obj, ORIGIN_DIRTYFB);
13739
13740         if (!new_plane_state->uapi.fence) { /* implicit fencing */
13741                 struct dma_fence *fence;
13742
13743                 ret = i915_sw_fence_await_reservation(&state->commit_ready,
13744                                                       obj->base.resv, NULL,
13745                                                       false,
13746                                                       i915_fence_timeout(dev_priv),
13747                                                       GFP_KERNEL);
13748                 if (ret < 0)
13749                         goto unpin_fb;
13750
13751                 fence = dma_resv_get_excl_rcu(obj->base.resv);
13752                 if (fence) {
13753                         add_rps_boost_after_vblank(new_plane_state->hw.crtc,
13754                                                    fence);
13755                         dma_fence_put(fence);
13756                 }
13757         } else {
13758                 add_rps_boost_after_vblank(new_plane_state->hw.crtc,
13759                                            new_plane_state->uapi.fence);
13760         }
13761
13762         /*
13763          * We declare pageflips to be interactive and so merit a small bias
13764          * towards upclocking to deliver the frame on time. By only changing
13765          * the RPS thresholds to sample more regularly and aim for higher
13766          * clocks we can hopefully deliver low power workloads (like kodi)
13767          * that are not quite steady state without resorting to forcing
13768          * maximum clocks following a vblank miss (see do_rps_boost()).
13769          */
13770         if (!state->rps_interactive) {
13771                 intel_rps_mark_interactive(&dev_priv->gt.rps, true);
13772                 state->rps_interactive = true;
13773         }
13774
13775         return 0;
13776
13777 unpin_fb:
13778         intel_plane_unpin_fb(new_plane_state);
13779
13780         return ret;
13781 }
13782
13783 /**
13784  * intel_cleanup_plane_fb - Cleans up an fb after plane use
13785  * @plane: drm plane to clean up for
13786  * @_old_plane_state: the state from the previous modeset
13787  *
13788  * Cleans up a framebuffer that has just been removed from a plane.
13789  */
13790 void
13791 intel_cleanup_plane_fb(struct drm_plane *plane,
13792                        struct drm_plane_state *_old_plane_state)
13793 {
13794         struct intel_plane_state *old_plane_state =
13795                 to_intel_plane_state(_old_plane_state);
13796         struct intel_atomic_state *state =
13797                 to_intel_atomic_state(old_plane_state->uapi.state);
13798         struct drm_i915_private *dev_priv = to_i915(plane->dev);
13799         struct drm_i915_gem_object *obj = intel_fb_obj(old_plane_state->hw.fb);
13800
13801         if (!obj)
13802                 return;
13803
13804         if (state->rps_interactive) {
13805                 intel_rps_mark_interactive(&dev_priv->gt.rps, false);
13806                 state->rps_interactive = false;
13807         }
13808
13809         /* Should only be called after a successful intel_prepare_plane_fb()! */
13810         intel_plane_unpin_fb(old_plane_state);
13811 }
13812
13813 /**
13814  * intel_plane_destroy - destroy a plane
13815  * @plane: plane to destroy
13816  *
13817  * Common destruction function for all types of planes (primary, cursor,
13818  * sprite).
13819  */
13820 void intel_plane_destroy(struct drm_plane *plane)
13821 {
13822         drm_plane_cleanup(plane);
13823         kfree(to_intel_plane(plane));
13824 }
13825
13826 static void intel_plane_possible_crtcs_init(struct drm_i915_private *dev_priv)
13827 {
13828         struct intel_plane *plane;
13829
13830         for_each_intel_plane(&dev_priv->drm, plane) {
13831                 struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv,
13832                                                                   plane->pipe);
13833
13834                 plane->base.possible_crtcs = drm_crtc_mask(&crtc->base);
13835         }
13836 }
13837
13838
13839 int intel_get_pipe_from_crtc_id_ioctl(struct drm_device *dev, void *data,
13840                                       struct drm_file *file)
13841 {
13842         struct drm_i915_get_pipe_from_crtc_id *pipe_from_crtc_id = data;
13843         struct drm_crtc *drmmode_crtc;
13844         struct intel_crtc *crtc;
13845
13846         drmmode_crtc = drm_crtc_find(dev, file, pipe_from_crtc_id->crtc_id);
13847         if (!drmmode_crtc)
13848                 return -ENOENT;
13849
13850         crtc = to_intel_crtc(drmmode_crtc);
13851         pipe_from_crtc_id->pipe = crtc->pipe;
13852
13853         return 0;
13854 }
13855
13856 static u32 intel_encoder_possible_clones(struct intel_encoder *encoder)
13857 {
13858         struct drm_device *dev = encoder->base.dev;
13859         struct intel_encoder *source_encoder;
13860         u32 possible_clones = 0;
13861
13862         for_each_intel_encoder(dev, source_encoder) {
13863                 if (encoders_cloneable(encoder, source_encoder))
13864                         possible_clones |= drm_encoder_mask(&source_encoder->base);
13865         }
13866
13867         return possible_clones;
13868 }
13869
13870 static u32 intel_encoder_possible_crtcs(struct intel_encoder *encoder)
13871 {
13872         struct drm_device *dev = encoder->base.dev;
13873         struct intel_crtc *crtc;
13874         u32 possible_crtcs = 0;
13875
13876         for_each_intel_crtc(dev, crtc) {
13877                 if (encoder->pipe_mask & BIT(crtc->pipe))
13878                         possible_crtcs |= drm_crtc_mask(&crtc->base);
13879         }
13880
13881         return possible_crtcs;
13882 }
13883
13884 static bool ilk_has_edp_a(struct drm_i915_private *dev_priv)
13885 {
13886         if (!IS_MOBILE(dev_priv))
13887                 return false;
13888
13889         if ((intel_de_read(dev_priv, DP_A) & DP_DETECTED) == 0)
13890                 return false;
13891
13892         if (IS_GEN(dev_priv, 5) && (intel_de_read(dev_priv, FUSE_STRAP) & ILK_eDP_A_DISABLE))
13893                 return false;
13894
13895         return true;
13896 }
13897
13898 static bool intel_ddi_crt_present(struct drm_i915_private *dev_priv)
13899 {
13900         if (INTEL_GEN(dev_priv) >= 9)
13901                 return false;
13902
13903         if (IS_HSW_ULT(dev_priv) || IS_BDW_ULT(dev_priv))
13904                 return false;
13905
13906         if (HAS_PCH_LPT_H(dev_priv) &&
13907             intel_de_read(dev_priv, SFUSE_STRAP) & SFUSE_STRAP_CRT_DISABLED)
13908                 return false;
13909
13910         /* DDI E can't be used if DDI A requires 4 lanes */
13911         if (intel_de_read(dev_priv, DDI_BUF_CTL(PORT_A)) & DDI_A_4_LANES)
13912                 return false;
13913
13914         if (!dev_priv->vbt.int_crt_support)
13915                 return false;
13916
13917         return true;
13918 }
13919
13920 static void intel_setup_outputs(struct drm_i915_private *dev_priv)
13921 {
13922         struct intel_encoder *encoder;
13923         bool dpd_is_edp = false;
13924
13925         intel_pps_unlock_regs_wa(dev_priv);
13926
13927         if (!HAS_DISPLAY(dev_priv))
13928                 return;
13929
13930         if (IS_ALDERLAKE_S(dev_priv)) {
13931                 intel_ddi_init(dev_priv, PORT_A);
13932                 intel_ddi_init(dev_priv, PORT_TC1);
13933                 intel_ddi_init(dev_priv, PORT_TC2);
13934                 intel_ddi_init(dev_priv, PORT_TC3);
13935                 intel_ddi_init(dev_priv, PORT_TC4);
13936         } else if (IS_DG1(dev_priv) || IS_ROCKETLAKE(dev_priv)) {
13937                 intel_ddi_init(dev_priv, PORT_A);
13938                 intel_ddi_init(dev_priv, PORT_B);
13939                 intel_ddi_init(dev_priv, PORT_TC1);
13940                 intel_ddi_init(dev_priv, PORT_TC2);
13941         } else if (INTEL_GEN(dev_priv) >= 12) {
13942                 intel_ddi_init(dev_priv, PORT_A);
13943                 intel_ddi_init(dev_priv, PORT_B);
13944                 intel_ddi_init(dev_priv, PORT_TC1);
13945                 intel_ddi_init(dev_priv, PORT_TC2);
13946                 intel_ddi_init(dev_priv, PORT_TC3);
13947                 intel_ddi_init(dev_priv, PORT_TC4);
13948                 intel_ddi_init(dev_priv, PORT_TC5);
13949                 intel_ddi_init(dev_priv, PORT_TC6);
13950                 icl_dsi_init(dev_priv);
13951         } else if (IS_JSL_EHL(dev_priv)) {
13952                 intel_ddi_init(dev_priv, PORT_A);
13953                 intel_ddi_init(dev_priv, PORT_B);
13954                 intel_ddi_init(dev_priv, PORT_C);
13955                 intel_ddi_init(dev_priv, PORT_D);
13956                 icl_dsi_init(dev_priv);
13957         } else if (IS_GEN(dev_priv, 11)) {
13958                 intel_ddi_init(dev_priv, PORT_A);
13959                 intel_ddi_init(dev_priv, PORT_B);
13960                 intel_ddi_init(dev_priv, PORT_C);
13961                 intel_ddi_init(dev_priv, PORT_D);
13962                 intel_ddi_init(dev_priv, PORT_E);
13963                 /*
13964                  * On some ICL SKUs port F is not present. No strap bits for
13965                  * this, so rely on VBT.
13966                  * Work around broken VBTs on SKUs known to have no port F.
13967                  */
13968                 if (IS_ICL_WITH_PORT_F(dev_priv) &&
13969                     intel_bios_is_port_present(dev_priv, PORT_F))
13970                         intel_ddi_init(dev_priv, PORT_F);
13971
13972                 icl_dsi_init(dev_priv);
13973         } else if (IS_GEN9_LP(dev_priv)) {
13974                 /*
13975                  * FIXME: Broxton doesn't support port detection via the
13976                  * DDI_BUF_CTL_A or SFUSE_STRAP registers, find another way to
13977                  * detect the ports.
13978                  */
13979                 intel_ddi_init(dev_priv, PORT_A);
13980                 intel_ddi_init(dev_priv, PORT_B);
13981                 intel_ddi_init(dev_priv, PORT_C);
13982
13983                 vlv_dsi_init(dev_priv);
13984         } else if (HAS_DDI(dev_priv)) {
13985                 int found;
13986
13987                 if (intel_ddi_crt_present(dev_priv))
13988                         intel_crt_init(dev_priv);
13989
13990                 /*
13991                  * Haswell uses DDI functions to detect digital outputs.
13992                  * On SKL pre-D0 the strap isn't connected, so we assume
13993                  * it's there.
13994                  */
13995                 found = intel_de_read(dev_priv, DDI_BUF_CTL(PORT_A)) & DDI_INIT_DISPLAY_DETECTED;
13996                 /* WaIgnoreDDIAStrap: skl */
13997                 if (found || IS_GEN9_BC(dev_priv))
13998                         intel_ddi_init(dev_priv, PORT_A);
13999
14000                 /* DDI B, C, D, and F detection is indicated by the SFUSE_STRAP
14001                  * register */
14002                 found = intel_de_read(dev_priv, SFUSE_STRAP);
14003
14004                 if (found & SFUSE_STRAP_DDIB_DETECTED)
14005                         intel_ddi_init(dev_priv, PORT_B);
14006                 if (found & SFUSE_STRAP_DDIC_DETECTED)
14007                         intel_ddi_init(dev_priv, PORT_C);
14008                 if (found & SFUSE_STRAP_DDID_DETECTED)
14009                         intel_ddi_init(dev_priv, PORT_D);
14010                 if (found & SFUSE_STRAP_DDIF_DETECTED)
14011                         intel_ddi_init(dev_priv, PORT_F);
14012                 /*
14013                  * On SKL we don't have a way to detect DDI-E so we rely on VBT.
14014                  */
14015                 if (IS_GEN9_BC(dev_priv) &&
14016                     intel_bios_is_port_present(dev_priv, PORT_E))
14017                         intel_ddi_init(dev_priv, PORT_E);
14018
14019         } else if (HAS_PCH_SPLIT(dev_priv)) {
14020                 int found;
14021
14022                 /*
14023                  * intel_edp_init_connector() depends on this completing first,
14024                  * to prevent the registration of both eDP and LVDS and the
14025                  * incorrect sharing of the PPS.
14026                  */
14027                 intel_lvds_init(dev_priv);
14028                 intel_crt_init(dev_priv);
14029
14030                 dpd_is_edp = intel_dp_is_port_edp(dev_priv, PORT_D);
14031
14032                 if (ilk_has_edp_a(dev_priv))
14033                         intel_dp_init(dev_priv, DP_A, PORT_A);
14034
14035                 if (intel_de_read(dev_priv, PCH_HDMIB) & SDVO_DETECTED) {
14036                         /* PCH SDVOB multiplex with HDMIB */
14037                         found = intel_sdvo_init(dev_priv, PCH_SDVOB, PORT_B);
14038                         if (!found)
14039                                 intel_hdmi_init(dev_priv, PCH_HDMIB, PORT_B);
14040                         if (!found && (intel_de_read(dev_priv, PCH_DP_B) & DP_DETECTED))
14041                                 intel_dp_init(dev_priv, PCH_DP_B, PORT_B);
14042                 }
14043
14044                 if (intel_de_read(dev_priv, PCH_HDMIC) & SDVO_DETECTED)
14045                         intel_hdmi_init(dev_priv, PCH_HDMIC, PORT_C);
14046
14047                 if (!dpd_is_edp && intel_de_read(dev_priv, PCH_HDMID) & SDVO_DETECTED)
14048                         intel_hdmi_init(dev_priv, PCH_HDMID, PORT_D);
14049
14050                 if (intel_de_read(dev_priv, PCH_DP_C) & DP_DETECTED)
14051                         intel_dp_init(dev_priv, PCH_DP_C, PORT_C);
14052
14053                 if (intel_de_read(dev_priv, PCH_DP_D) & DP_DETECTED)
14054                         intel_dp_init(dev_priv, PCH_DP_D, PORT_D);
14055         } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
14056                 bool has_edp, has_port;
14057
14058                 if (IS_VALLEYVIEW(dev_priv) && dev_priv->vbt.int_crt_support)
14059                         intel_crt_init(dev_priv);
14060
14061                 /*
14062                  * The DP_DETECTED bit is the latched state of the DDC
14063                  * SDA pin at boot. However since eDP doesn't require DDC
14064                  * (no way to plug in a DP->HDMI dongle) the DDC pins for
14065                  * eDP ports may have been muxed to an alternate function.
14066                  * Thus we can't rely on the DP_DETECTED bit alone to detect
14067                  * eDP ports. Consult the VBT as well as DP_DETECTED to
14068                  * detect eDP ports.
14069                  *
14070                  * Sadly the straps seem to be missing sometimes even for HDMI
14071                  * ports (eg. on Voyo V3 - CHT x7-Z8700), so check both strap
14072                  * and VBT for the presence of the port. Additionally we can't
14073                  * trust the port type the VBT declares as we've seen at least
14074                  * HDMI ports that the VBT claim are DP or eDP.
14075                  */
14076                 has_edp = intel_dp_is_port_edp(dev_priv, PORT_B);
14077                 has_port = intel_bios_is_port_present(dev_priv, PORT_B);
14078                 if (intel_de_read(dev_priv, VLV_DP_B) & DP_DETECTED || has_port)
14079                         has_edp &= intel_dp_init(dev_priv, VLV_DP_B, PORT_B);
14080                 if ((intel_de_read(dev_priv, VLV_HDMIB) & SDVO_DETECTED || has_port) && !has_edp)
14081                         intel_hdmi_init(dev_priv, VLV_HDMIB, PORT_B);
14082
14083                 has_edp = intel_dp_is_port_edp(dev_priv, PORT_C);
14084                 has_port = intel_bios_is_port_present(dev_priv, PORT_C);
14085                 if (intel_de_read(dev_priv, VLV_DP_C) & DP_DETECTED || has_port)
14086                         has_edp &= intel_dp_init(dev_priv, VLV_DP_C, PORT_C);
14087                 if ((intel_de_read(dev_priv, VLV_HDMIC) & SDVO_DETECTED || has_port) && !has_edp)
14088                         intel_hdmi_init(dev_priv, VLV_HDMIC, PORT_C);
14089
14090                 if (IS_CHERRYVIEW(dev_priv)) {
14091                         /*
14092                          * eDP not supported on port D,
14093                          * so no need to worry about it
14094                          */
14095                         has_port = intel_bios_is_port_present(dev_priv, PORT_D);
14096                         if (intel_de_read(dev_priv, CHV_DP_D) & DP_DETECTED || has_port)
14097                                 intel_dp_init(dev_priv, CHV_DP_D, PORT_D);
14098                         if (intel_de_read(dev_priv, CHV_HDMID) & SDVO_DETECTED || has_port)
14099                                 intel_hdmi_init(dev_priv, CHV_HDMID, PORT_D);
14100                 }
14101
14102                 vlv_dsi_init(dev_priv);
14103         } else if (IS_PINEVIEW(dev_priv)) {
14104                 intel_lvds_init(dev_priv);
14105                 intel_crt_init(dev_priv);
14106         } else if (IS_GEN_RANGE(dev_priv, 3, 4)) {
14107                 bool found = false;
14108
14109                 if (IS_MOBILE(dev_priv))
14110                         intel_lvds_init(dev_priv);
14111
14112                 intel_crt_init(dev_priv);
14113
14114                 if (intel_de_read(dev_priv, GEN3_SDVOB) & SDVO_DETECTED) {
14115                         drm_dbg_kms(&dev_priv->drm, "probing SDVOB\n");
14116                         found = intel_sdvo_init(dev_priv, GEN3_SDVOB, PORT_B);
14117                         if (!found && IS_G4X(dev_priv)) {
14118                                 drm_dbg_kms(&dev_priv->drm,
14119                                             "probing HDMI on SDVOB\n");
14120                                 intel_hdmi_init(dev_priv, GEN4_HDMIB, PORT_B);
14121                         }
14122
14123                         if (!found && IS_G4X(dev_priv))
14124                                 intel_dp_init(dev_priv, DP_B, PORT_B);
14125                 }
14126
14127                 /* Before G4X SDVOC doesn't have its own detect register */
14128
14129                 if (intel_de_read(dev_priv, GEN3_SDVOB) & SDVO_DETECTED) {
14130                         drm_dbg_kms(&dev_priv->drm, "probing SDVOC\n");
14131                         found = intel_sdvo_init(dev_priv, GEN3_SDVOC, PORT_C);
14132                 }
14133
14134                 if (!found && (intel_de_read(dev_priv, GEN3_SDVOC) & SDVO_DETECTED)) {
14135
14136                         if (IS_G4X(dev_priv)) {
14137                                 drm_dbg_kms(&dev_priv->drm,
14138                                             "probing HDMI on SDVOC\n");
14139                                 intel_hdmi_init(dev_priv, GEN4_HDMIC, PORT_C);
14140                         }
14141                         if (IS_G4X(dev_priv))
14142                                 intel_dp_init(dev_priv, DP_C, PORT_C);
14143                 }
14144
14145                 if (IS_G4X(dev_priv) && (intel_de_read(dev_priv, DP_D) & DP_DETECTED))
14146                         intel_dp_init(dev_priv, DP_D, PORT_D);
14147
14148                 if (SUPPORTS_TV(dev_priv))
14149                         intel_tv_init(dev_priv);
14150         } else if (IS_GEN(dev_priv, 2)) {
14151                 if (IS_I85X(dev_priv))
14152                         intel_lvds_init(dev_priv);
14153
14154                 intel_crt_init(dev_priv);
14155                 intel_dvo_init(dev_priv);
14156         }
14157
14158         for_each_intel_encoder(&dev_priv->drm, encoder) {
14159                 encoder->base.possible_crtcs =
14160                         intel_encoder_possible_crtcs(encoder);
14161                 encoder->base.possible_clones =
14162                         intel_encoder_possible_clones(encoder);
14163         }
14164
14165         intel_init_pch_refclk(dev_priv);
14166
14167         drm_helper_move_panel_connectors_to_head(&dev_priv->drm);
14168 }
14169
14170 static void intel_user_framebuffer_destroy(struct drm_framebuffer *fb)
14171 {
14172         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
14173
14174         drm_framebuffer_cleanup(fb);
14175         intel_frontbuffer_put(intel_fb->frontbuffer);
14176
14177         kfree(intel_fb);
14178 }
14179
14180 static int intel_user_framebuffer_create_handle(struct drm_framebuffer *fb,
14181                                                 struct drm_file *file,
14182                                                 unsigned int *handle)
14183 {
14184         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
14185         struct drm_i915_private *i915 = to_i915(obj->base.dev);
14186
14187         if (obj->userptr.mm) {
14188                 drm_dbg(&i915->drm,
14189                         "attempting to use a userptr for a framebuffer, denied\n");
14190                 return -EINVAL;
14191         }
14192
14193         return drm_gem_handle_create(file, &obj->base, handle);
14194 }
14195
14196 static int intel_user_framebuffer_dirty(struct drm_framebuffer *fb,
14197                                         struct drm_file *file,
14198                                         unsigned flags, unsigned color,
14199                                         struct drm_clip_rect *clips,
14200                                         unsigned num_clips)
14201 {
14202         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
14203
14204         i915_gem_object_flush_if_display(obj);
14205         intel_frontbuffer_flush(to_intel_frontbuffer(fb), ORIGIN_DIRTYFB);
14206
14207         return 0;
14208 }
14209
14210 static const struct drm_framebuffer_funcs intel_fb_funcs = {
14211         .destroy = intel_user_framebuffer_destroy,
14212         .create_handle = intel_user_framebuffer_create_handle,
14213         .dirty = intel_user_framebuffer_dirty,
14214 };
14215
14216 static int intel_framebuffer_init(struct intel_framebuffer *intel_fb,
14217                                   struct drm_i915_gem_object *obj,
14218                                   struct drm_mode_fb_cmd2 *mode_cmd)
14219 {
14220         struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
14221         struct drm_framebuffer *fb = &intel_fb->base;
14222         u32 max_stride;
14223         unsigned int tiling, stride;
14224         int ret = -EINVAL;
14225         int i;
14226
14227         intel_fb->frontbuffer = intel_frontbuffer_get(obj);
14228         if (!intel_fb->frontbuffer)
14229                 return -ENOMEM;
14230
14231         i915_gem_object_lock(obj, NULL);
14232         tiling = i915_gem_object_get_tiling(obj);
14233         stride = i915_gem_object_get_stride(obj);
14234         i915_gem_object_unlock(obj);
14235
14236         if (mode_cmd->flags & DRM_MODE_FB_MODIFIERS) {
14237                 /*
14238                  * If there's a fence, enforce that
14239                  * the fb modifier and tiling mode match.
14240                  */
14241                 if (tiling != I915_TILING_NONE &&
14242                     tiling != intel_fb_modifier_to_tiling(mode_cmd->modifier[0])) {
14243                         drm_dbg_kms(&dev_priv->drm,
14244                                     "tiling_mode doesn't match fb modifier\n");
14245                         goto err;
14246                 }
14247         } else {
14248                 if (tiling == I915_TILING_X) {
14249                         mode_cmd->modifier[0] = I915_FORMAT_MOD_X_TILED;
14250                 } else if (tiling == I915_TILING_Y) {
14251                         drm_dbg_kms(&dev_priv->drm,
14252                                     "No Y tiling for legacy addfb\n");
14253                         goto err;
14254                 }
14255         }
14256
14257         if (!drm_any_plane_has_format(&dev_priv->drm,
14258                                       mode_cmd->pixel_format,
14259                                       mode_cmd->modifier[0])) {
14260                 struct drm_format_name_buf format_name;
14261
14262                 drm_dbg_kms(&dev_priv->drm,
14263                             "unsupported pixel format %s / modifier 0x%llx\n",
14264                             drm_get_format_name(mode_cmd->pixel_format,
14265                                                 &format_name),
14266                             mode_cmd->modifier[0]);
14267                 goto err;
14268         }
14269
14270         /*
14271          * gen2/3 display engine uses the fence if present,
14272          * so the tiling mode must match the fb modifier exactly.
14273          */
14274         if (INTEL_GEN(dev_priv) < 4 &&
14275             tiling != intel_fb_modifier_to_tiling(mode_cmd->modifier[0])) {
14276                 drm_dbg_kms(&dev_priv->drm,
14277                             "tiling_mode must match fb modifier exactly on gen2/3\n");
14278                 goto err;
14279         }
14280
14281         max_stride = intel_fb_max_stride(dev_priv, mode_cmd->pixel_format,
14282                                          mode_cmd->modifier[0]);
14283         if (mode_cmd->pitches[0] > max_stride) {
14284                 drm_dbg_kms(&dev_priv->drm,
14285                             "%s pitch (%u) must be at most %d\n",
14286                             mode_cmd->modifier[0] != DRM_FORMAT_MOD_LINEAR ?
14287                             "tiled" : "linear",
14288                             mode_cmd->pitches[0], max_stride);
14289                 goto err;
14290         }
14291
14292         /*
14293          * If there's a fence, enforce that
14294          * the fb pitch and fence stride match.
14295          */
14296         if (tiling != I915_TILING_NONE && mode_cmd->pitches[0] != stride) {
14297                 drm_dbg_kms(&dev_priv->drm,
14298                             "pitch (%d) must match tiling stride (%d)\n",
14299                             mode_cmd->pitches[0], stride);
14300                 goto err;
14301         }
14302
14303         /* FIXME need to adjust LINOFF/TILEOFF accordingly. */
14304         if (mode_cmd->offsets[0] != 0) {
14305                 drm_dbg_kms(&dev_priv->drm,
14306                             "plane 0 offset (0x%08x) must be 0\n",
14307                             mode_cmd->offsets[0]);
14308                 goto err;
14309         }
14310
14311         drm_helper_mode_fill_fb_struct(&dev_priv->drm, fb, mode_cmd);
14312
14313         for (i = 0; i < fb->format->num_planes; i++) {
14314                 u32 stride_alignment;
14315
14316                 if (mode_cmd->handles[i] != mode_cmd->handles[0]) {
14317                         drm_dbg_kms(&dev_priv->drm, "bad plane %d handle\n",
14318                                     i);
14319                         goto err;
14320                 }
14321
14322                 stride_alignment = intel_fb_stride_alignment(fb, i);
14323                 if (fb->pitches[i] & (stride_alignment - 1)) {
14324                         drm_dbg_kms(&dev_priv->drm,
14325                                     "plane %d pitch (%d) must be at least %u byte aligned\n",
14326                                     i, fb->pitches[i], stride_alignment);
14327                         goto err;
14328                 }
14329
14330                 if (is_gen12_ccs_plane(fb, i) && !is_gen12_ccs_cc_plane(fb, i)) {
14331                         int ccs_aux_stride = gen12_ccs_aux_stride(fb, i);
14332
14333                         if (fb->pitches[i] != ccs_aux_stride) {
14334                                 drm_dbg_kms(&dev_priv->drm,
14335                                             "ccs aux plane %d pitch (%d) must be %d\n",
14336                                             i,
14337                                             fb->pitches[i], ccs_aux_stride);
14338                                 goto err;
14339                         }
14340                 }
14341
14342                 fb->obj[i] = &obj->base;
14343         }
14344
14345         ret = intel_fill_fb_info(dev_priv, fb);
14346         if (ret)
14347                 goto err;
14348
14349         ret = drm_framebuffer_init(&dev_priv->drm, fb, &intel_fb_funcs);
14350         if (ret) {
14351                 drm_err(&dev_priv->drm, "framebuffer init failed %d\n", ret);
14352                 goto err;
14353         }
14354
14355         return 0;
14356
14357 err:
14358         intel_frontbuffer_put(intel_fb->frontbuffer);
14359         return ret;
14360 }
14361
14362 static struct drm_framebuffer *
14363 intel_user_framebuffer_create(struct drm_device *dev,
14364                               struct drm_file *filp,
14365                               const struct drm_mode_fb_cmd2 *user_mode_cmd)
14366 {
14367         struct drm_framebuffer *fb;
14368         struct drm_i915_gem_object *obj;
14369         struct drm_mode_fb_cmd2 mode_cmd = *user_mode_cmd;
14370
14371         obj = i915_gem_object_lookup(filp, mode_cmd.handles[0]);
14372         if (!obj)
14373                 return ERR_PTR(-ENOENT);
14374
14375         fb = intel_framebuffer_create(obj, &mode_cmd);
14376         i915_gem_object_put(obj);
14377
14378         return fb;
14379 }
14380
14381 static enum drm_mode_status
14382 intel_mode_valid(struct drm_device *dev,
14383                  const struct drm_display_mode *mode)
14384 {
14385         struct drm_i915_private *dev_priv = to_i915(dev);
14386         int hdisplay_max, htotal_max;
14387         int vdisplay_max, vtotal_max;
14388
14389         /*
14390          * Can't reject DBLSCAN here because Xorg ddxen can add piles
14391          * of DBLSCAN modes to the output's mode list when they detect
14392          * the scaling mode property on the connector. And they don't
14393          * ask the kernel to validate those modes in any way until
14394          * modeset time at which point the client gets a protocol error.
14395          * So in order to not upset those clients we silently ignore the
14396          * DBLSCAN flag on such connectors. For other connectors we will
14397          * reject modes with the DBLSCAN flag in encoder->compute_config().
14398          * And we always reject DBLSCAN modes in connector->mode_valid()
14399          * as we never want such modes on the connector's mode list.
14400          */
14401
14402         if (mode->vscan > 1)
14403                 return MODE_NO_VSCAN;
14404
14405         if (mode->flags & DRM_MODE_FLAG_HSKEW)
14406                 return MODE_H_ILLEGAL;
14407
14408         if (mode->flags & (DRM_MODE_FLAG_CSYNC |
14409                            DRM_MODE_FLAG_NCSYNC |
14410                            DRM_MODE_FLAG_PCSYNC))
14411                 return MODE_HSYNC;
14412
14413         if (mode->flags & (DRM_MODE_FLAG_BCAST |
14414                            DRM_MODE_FLAG_PIXMUX |
14415                            DRM_MODE_FLAG_CLKDIV2))
14416                 return MODE_BAD;
14417
14418         /* Transcoder timing limits */
14419         if (INTEL_GEN(dev_priv) >= 11) {
14420                 hdisplay_max = 16384;
14421                 vdisplay_max = 8192;
14422                 htotal_max = 16384;
14423                 vtotal_max = 8192;
14424         } else if (INTEL_GEN(dev_priv) >= 9 ||
14425                    IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv)) {
14426                 hdisplay_max = 8192; /* FDI max 4096 handled elsewhere */
14427                 vdisplay_max = 4096;
14428                 htotal_max = 8192;
14429                 vtotal_max = 8192;
14430         } else if (INTEL_GEN(dev_priv) >= 3) {
14431                 hdisplay_max = 4096;
14432                 vdisplay_max = 4096;
14433                 htotal_max = 8192;
14434                 vtotal_max = 8192;
14435         } else {
14436                 hdisplay_max = 2048;
14437                 vdisplay_max = 2048;
14438                 htotal_max = 4096;
14439                 vtotal_max = 4096;
14440         }
14441
14442         if (mode->hdisplay > hdisplay_max ||
14443             mode->hsync_start > htotal_max ||
14444             mode->hsync_end > htotal_max ||
14445             mode->htotal > htotal_max)
14446                 return MODE_H_ILLEGAL;
14447
14448         if (mode->vdisplay > vdisplay_max ||
14449             mode->vsync_start > vtotal_max ||
14450             mode->vsync_end > vtotal_max ||
14451             mode->vtotal > vtotal_max)
14452                 return MODE_V_ILLEGAL;
14453
14454         if (INTEL_GEN(dev_priv) >= 5) {
14455                 if (mode->hdisplay < 64 ||
14456                     mode->htotal - mode->hdisplay < 32)
14457                         return MODE_H_ILLEGAL;
14458
14459                 if (mode->vtotal - mode->vdisplay < 5)
14460                         return MODE_V_ILLEGAL;
14461         } else {
14462                 if (mode->htotal - mode->hdisplay < 32)
14463                         return MODE_H_ILLEGAL;
14464
14465                 if (mode->vtotal - mode->vdisplay < 3)
14466                         return MODE_V_ILLEGAL;
14467         }
14468
14469         return MODE_OK;
14470 }
14471
14472 enum drm_mode_status
14473 intel_mode_valid_max_plane_size(struct drm_i915_private *dev_priv,
14474                                 const struct drm_display_mode *mode,
14475                                 bool bigjoiner)
14476 {
14477         int plane_width_max, plane_height_max;
14478
14479         /*
14480          * intel_mode_valid() should be
14481          * sufficient on older platforms.
14482          */
14483         if (INTEL_GEN(dev_priv) < 9)
14484                 return MODE_OK;
14485
14486         /*
14487          * Most people will probably want a fullscreen
14488          * plane so let's not advertize modes that are
14489          * too big for that.
14490          */
14491         if (INTEL_GEN(dev_priv) >= 11) {
14492                 plane_width_max = 5120 << bigjoiner;
14493                 plane_height_max = 4320;
14494         } else {
14495                 plane_width_max = 5120;
14496                 plane_height_max = 4096;
14497         }
14498
14499         if (mode->hdisplay > plane_width_max)
14500                 return MODE_H_ILLEGAL;
14501
14502         if (mode->vdisplay > plane_height_max)
14503                 return MODE_V_ILLEGAL;
14504
14505         return MODE_OK;
14506 }
14507
14508 static const struct drm_mode_config_funcs intel_mode_funcs = {
14509         .fb_create = intel_user_framebuffer_create,
14510         .get_format_info = intel_get_format_info,
14511         .output_poll_changed = intel_fbdev_output_poll_changed,
14512         .mode_valid = intel_mode_valid,
14513         .atomic_check = intel_atomic_check,
14514         .atomic_commit = intel_atomic_commit,
14515         .atomic_state_alloc = intel_atomic_state_alloc,
14516         .atomic_state_clear = intel_atomic_state_clear,
14517         .atomic_state_free = intel_atomic_state_free,
14518 };
14519
14520 /**
14521  * intel_init_display_hooks - initialize the display modesetting hooks
14522  * @dev_priv: device private
14523  */
14524 void intel_init_display_hooks(struct drm_i915_private *dev_priv)
14525 {
14526         intel_init_cdclk_hooks(dev_priv);
14527
14528         intel_dpll_init_clock_hook(dev_priv);
14529
14530         if (INTEL_GEN(dev_priv) >= 9) {
14531                 dev_priv->display.get_pipe_config = hsw_get_pipe_config;
14532                 dev_priv->display.crtc_enable = hsw_crtc_enable;
14533                 dev_priv->display.crtc_disable = hsw_crtc_disable;
14534         } else if (HAS_DDI(dev_priv)) {
14535                 dev_priv->display.get_pipe_config = hsw_get_pipe_config;
14536                 dev_priv->display.crtc_enable = hsw_crtc_enable;
14537                 dev_priv->display.crtc_disable = hsw_crtc_disable;
14538         } else if (HAS_PCH_SPLIT(dev_priv)) {
14539                 dev_priv->display.get_pipe_config = ilk_get_pipe_config;
14540                 dev_priv->display.crtc_enable = ilk_crtc_enable;
14541                 dev_priv->display.crtc_disable = ilk_crtc_disable;
14542         } else if (IS_CHERRYVIEW(dev_priv) ||
14543                    IS_VALLEYVIEW(dev_priv)) {
14544                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
14545                 dev_priv->display.crtc_enable = valleyview_crtc_enable;
14546                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
14547         } else {
14548                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
14549                 dev_priv->display.crtc_enable = i9xx_crtc_enable;
14550                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
14551         }
14552
14553         intel_fdi_init_hook(dev_priv);
14554
14555         if (INTEL_GEN(dev_priv) >= 9) {
14556                 dev_priv->display.commit_modeset_enables = skl_commit_modeset_enables;
14557                 dev_priv->display.get_initial_plane_config = skl_get_initial_plane_config;
14558         } else {
14559                 dev_priv->display.commit_modeset_enables = intel_commit_modeset_enables;
14560                 dev_priv->display.get_initial_plane_config = i9xx_get_initial_plane_config;
14561         }
14562
14563 }
14564
14565 void intel_modeset_init_hw(struct drm_i915_private *i915)
14566 {
14567         struct intel_cdclk_state *cdclk_state =
14568                 to_intel_cdclk_state(i915->cdclk.obj.state);
14569
14570         intel_update_cdclk(i915);
14571         intel_dump_cdclk_config(&i915->cdclk.hw, "Current CDCLK");
14572         cdclk_state->logical = cdclk_state->actual = i915->cdclk.hw;
14573 }
14574
14575 static int sanitize_watermarks_add_affected(struct drm_atomic_state *state)
14576 {
14577         struct drm_plane *plane;
14578         struct intel_crtc *crtc;
14579
14580         for_each_intel_crtc(state->dev, crtc) {
14581                 struct intel_crtc_state *crtc_state;
14582
14583                 crtc_state = intel_atomic_get_crtc_state(state, crtc);
14584                 if (IS_ERR(crtc_state))
14585                         return PTR_ERR(crtc_state);
14586
14587                 if (crtc_state->hw.active) {
14588                         /*
14589                          * Preserve the inherited flag to avoid
14590                          * taking the full modeset path.
14591                          */
14592                         crtc_state->inherited = true;
14593                 }
14594         }
14595
14596         drm_for_each_plane(plane, state->dev) {
14597                 struct drm_plane_state *plane_state;
14598
14599                 plane_state = drm_atomic_get_plane_state(state, plane);
14600                 if (IS_ERR(plane_state))
14601                         return PTR_ERR(plane_state);
14602         }
14603
14604         return 0;
14605 }
14606
14607 /*
14608  * Calculate what we think the watermarks should be for the state we've read
14609  * out of the hardware and then immediately program those watermarks so that
14610  * we ensure the hardware settings match our internal state.
14611  *
14612  * We can calculate what we think WM's should be by creating a duplicate of the
14613  * current state (which was constructed during hardware readout) and running it
14614  * through the atomic check code to calculate new watermark values in the
14615  * state object.
14616  */
14617 static void sanitize_watermarks(struct drm_i915_private *dev_priv)
14618 {
14619         struct drm_atomic_state *state;
14620         struct intel_atomic_state *intel_state;
14621         struct intel_crtc *crtc;
14622         struct intel_crtc_state *crtc_state;
14623         struct drm_modeset_acquire_ctx ctx;
14624         int ret;
14625         int i;
14626
14627         /* Only supported on platforms that use atomic watermark design */
14628         if (!dev_priv->display.optimize_watermarks)
14629                 return;
14630
14631         state = drm_atomic_state_alloc(&dev_priv->drm);
14632         if (drm_WARN_ON(&dev_priv->drm, !state))
14633                 return;
14634
14635         intel_state = to_intel_atomic_state(state);
14636
14637         drm_modeset_acquire_init(&ctx, 0);
14638
14639 retry:
14640         state->acquire_ctx = &ctx;
14641
14642         /*
14643          * Hardware readout is the only time we don't want to calculate
14644          * intermediate watermarks (since we don't trust the current
14645          * watermarks).
14646          */
14647         if (!HAS_GMCH(dev_priv))
14648                 intel_state->skip_intermediate_wm = true;
14649
14650         ret = sanitize_watermarks_add_affected(state);
14651         if (ret)
14652                 goto fail;
14653
14654         ret = intel_atomic_check(&dev_priv->drm, state);
14655         if (ret)
14656                 goto fail;
14657
14658         /* Write calculated watermark values back */
14659         for_each_new_intel_crtc_in_state(intel_state, crtc, crtc_state, i) {
14660                 crtc_state->wm.need_postvbl_update = true;
14661                 dev_priv->display.optimize_watermarks(intel_state, crtc);
14662
14663                 to_intel_crtc_state(crtc->base.state)->wm = crtc_state->wm;
14664         }
14665
14666 fail:
14667         if (ret == -EDEADLK) {
14668                 drm_atomic_state_clear(state);
14669                 drm_modeset_backoff(&ctx);
14670                 goto retry;
14671         }
14672
14673         /*
14674          * If we fail here, it means that the hardware appears to be
14675          * programmed in a way that shouldn't be possible, given our
14676          * understanding of watermark requirements.  This might mean a
14677          * mistake in the hardware readout code or a mistake in the
14678          * watermark calculations for a given platform.  Raise a WARN
14679          * so that this is noticeable.
14680          *
14681          * If this actually happens, we'll have to just leave the
14682          * BIOS-programmed watermarks untouched and hope for the best.
14683          */
14684         drm_WARN(&dev_priv->drm, ret,
14685                  "Could not determine valid watermarks for inherited state\n");
14686
14687         drm_atomic_state_put(state);
14688
14689         drm_modeset_drop_locks(&ctx);
14690         drm_modeset_acquire_fini(&ctx);
14691 }
14692
14693 static void intel_update_fdi_pll_freq(struct drm_i915_private *dev_priv)
14694 {
14695         if (IS_GEN(dev_priv, 5)) {
14696                 u32 fdi_pll_clk =
14697                         intel_de_read(dev_priv, FDI_PLL_BIOS_0) & FDI_PLL_FB_CLOCK_MASK;
14698
14699                 dev_priv->fdi_pll_freq = (fdi_pll_clk + 2) * 10000;
14700         } else if (IS_GEN(dev_priv, 6) || IS_IVYBRIDGE(dev_priv)) {
14701                 dev_priv->fdi_pll_freq = 270000;
14702         } else {
14703                 return;
14704         }
14705
14706         drm_dbg(&dev_priv->drm, "FDI PLL freq=%d\n", dev_priv->fdi_pll_freq);
14707 }
14708
14709 static int intel_initial_commit(struct drm_device *dev)
14710 {
14711         struct drm_atomic_state *state = NULL;
14712         struct drm_modeset_acquire_ctx ctx;
14713         struct intel_crtc *crtc;
14714         int ret = 0;
14715
14716         state = drm_atomic_state_alloc(dev);
14717         if (!state)
14718                 return -ENOMEM;
14719
14720         drm_modeset_acquire_init(&ctx, 0);
14721
14722 retry:
14723         state->acquire_ctx = &ctx;
14724
14725         for_each_intel_crtc(dev, crtc) {
14726                 struct intel_crtc_state *crtc_state =
14727                         intel_atomic_get_crtc_state(state, crtc);
14728
14729                 if (IS_ERR(crtc_state)) {
14730                         ret = PTR_ERR(crtc_state);
14731                         goto out;
14732                 }
14733
14734                 if (crtc_state->hw.active) {
14735                         struct intel_encoder *encoder;
14736
14737                         /*
14738                          * We've not yet detected sink capabilities
14739                          * (audio,infoframes,etc.) and thus we don't want to
14740                          * force a full state recomputation yet. We want that to
14741                          * happen only for the first real commit from userspace.
14742                          * So preserve the inherited flag for the time being.
14743                          */
14744                         crtc_state->inherited = true;
14745
14746                         ret = drm_atomic_add_affected_planes(state, &crtc->base);
14747                         if (ret)
14748                                 goto out;
14749
14750                         /*
14751                          * FIXME hack to force a LUT update to avoid the
14752                          * plane update forcing the pipe gamma on without
14753                          * having a proper LUT loaded. Remove once we
14754                          * have readout for pipe gamma enable.
14755                          */
14756                         crtc_state->uapi.color_mgmt_changed = true;
14757
14758                         for_each_intel_encoder_mask(dev, encoder,
14759                                                     crtc_state->uapi.encoder_mask) {
14760                                 if (encoder->initial_fastset_check &&
14761                                     !encoder->initial_fastset_check(encoder, crtc_state)) {
14762                                         ret = drm_atomic_add_affected_connectors(state,
14763                                                                                  &crtc->base);
14764                                         if (ret)
14765                                                 goto out;
14766                                 }
14767                         }
14768                 }
14769         }
14770
14771         ret = drm_atomic_commit(state);
14772
14773 out:
14774         if (ret == -EDEADLK) {
14775                 drm_atomic_state_clear(state);
14776                 drm_modeset_backoff(&ctx);
14777                 goto retry;
14778         }
14779
14780         drm_atomic_state_put(state);
14781
14782         drm_modeset_drop_locks(&ctx);
14783         drm_modeset_acquire_fini(&ctx);
14784
14785         return ret;
14786 }
14787
14788 static void intel_mode_config_init(struct drm_i915_private *i915)
14789 {
14790         struct drm_mode_config *mode_config = &i915->drm.mode_config;
14791
14792         drm_mode_config_init(&i915->drm);
14793         INIT_LIST_HEAD(&i915->global_obj_list);
14794
14795         mode_config->min_width = 0;
14796         mode_config->min_height = 0;
14797
14798         mode_config->preferred_depth = 24;
14799         mode_config->prefer_shadow = 1;
14800
14801         mode_config->allow_fb_modifiers = true;
14802
14803         mode_config->funcs = &intel_mode_funcs;
14804
14805         mode_config->async_page_flip = has_async_flips(i915);
14806
14807         /*
14808          * Maximum framebuffer dimensions, chosen to match
14809          * the maximum render engine surface size on gen4+.
14810          */
14811         if (INTEL_GEN(i915) >= 7) {
14812                 mode_config->max_width = 16384;
14813                 mode_config->max_height = 16384;
14814         } else if (INTEL_GEN(i915) >= 4) {
14815                 mode_config->max_width = 8192;
14816                 mode_config->max_height = 8192;
14817         } else if (IS_GEN(i915, 3)) {
14818                 mode_config->max_width = 4096;
14819                 mode_config->max_height = 4096;
14820         } else {
14821                 mode_config->max_width = 2048;
14822                 mode_config->max_height = 2048;
14823         }
14824
14825         if (IS_I845G(i915) || IS_I865G(i915)) {
14826                 mode_config->cursor_width = IS_I845G(i915) ? 64 : 512;
14827                 mode_config->cursor_height = 1023;
14828         } else if (IS_I830(i915) || IS_I85X(i915) ||
14829                    IS_I915G(i915) || IS_I915GM(i915)) {
14830                 mode_config->cursor_width = 64;
14831                 mode_config->cursor_height = 64;
14832         } else {
14833                 mode_config->cursor_width = 256;
14834                 mode_config->cursor_height = 256;
14835         }
14836 }
14837
14838 static void intel_mode_config_cleanup(struct drm_i915_private *i915)
14839 {
14840         intel_atomic_global_obj_cleanup(i915);
14841         drm_mode_config_cleanup(&i915->drm);
14842 }
14843
14844 static void plane_config_fini(struct intel_initial_plane_config *plane_config)
14845 {
14846         if (plane_config->fb) {
14847                 struct drm_framebuffer *fb = &plane_config->fb->base;
14848
14849                 /* We may only have the stub and not a full framebuffer */
14850                 if (drm_framebuffer_read_refcount(fb))
14851                         drm_framebuffer_put(fb);
14852                 else
14853                         kfree(fb);
14854         }
14855
14856         if (plane_config->vma)
14857                 i915_vma_put(plane_config->vma);
14858 }
14859
14860 /* part #1: call before irq install */
14861 int intel_modeset_init_noirq(struct drm_i915_private *i915)
14862 {
14863         int ret;
14864
14865         if (i915_inject_probe_failure(i915))
14866                 return -ENODEV;
14867
14868         if (HAS_DISPLAY(i915)) {
14869                 ret = drm_vblank_init(&i915->drm,
14870                                       INTEL_NUM_PIPES(i915));
14871                 if (ret)
14872                         return ret;
14873         }
14874
14875         intel_bios_init(i915);
14876
14877         ret = intel_vga_register(i915);
14878         if (ret)
14879                 goto cleanup_bios;
14880
14881         /* FIXME: completely on the wrong abstraction layer */
14882         intel_power_domains_init_hw(i915, false);
14883
14884         intel_csr_ucode_init(i915);
14885
14886         i915->modeset_wq = alloc_ordered_workqueue("i915_modeset", 0);
14887         i915->flip_wq = alloc_workqueue("i915_flip", WQ_HIGHPRI |
14888                                         WQ_UNBOUND, WQ_UNBOUND_MAX_ACTIVE);
14889
14890         i915->framestart_delay = 1; /* 1-4 */
14891
14892         intel_mode_config_init(i915);
14893
14894         ret = intel_cdclk_init(i915);
14895         if (ret)
14896                 goto cleanup_vga_client_pw_domain_csr;
14897
14898         ret = intel_dbuf_init(i915);
14899         if (ret)
14900                 goto cleanup_vga_client_pw_domain_csr;
14901
14902         ret = intel_bw_init(i915);
14903         if (ret)
14904                 goto cleanup_vga_client_pw_domain_csr;
14905
14906         init_llist_head(&i915->atomic_helper.free_list);
14907         INIT_WORK(&i915->atomic_helper.free_work,
14908                   intel_atomic_helper_free_state_worker);
14909
14910         intel_init_quirks(i915);
14911
14912         intel_fbc_init(i915);
14913
14914         return 0;
14915
14916 cleanup_vga_client_pw_domain_csr:
14917         intel_csr_ucode_fini(i915);
14918         intel_power_domains_driver_remove(i915);
14919         intel_vga_unregister(i915);
14920 cleanup_bios:
14921         intel_bios_driver_remove(i915);
14922
14923         return ret;
14924 }
14925
14926 /* part #2: call after irq install, but before gem init */
14927 int intel_modeset_init_nogem(struct drm_i915_private *i915)
14928 {
14929         struct drm_device *dev = &i915->drm;
14930         enum pipe pipe;
14931         struct intel_crtc *crtc;
14932         int ret;
14933
14934         intel_init_pm(i915);
14935
14936         intel_panel_sanitize_ssc(i915);
14937
14938         intel_pps_setup(i915);
14939
14940         intel_gmbus_setup(i915);
14941
14942         drm_dbg_kms(&i915->drm, "%d display pipe%s available.\n",
14943                     INTEL_NUM_PIPES(i915),
14944                     INTEL_NUM_PIPES(i915) > 1 ? "s" : "");
14945
14946         if (HAS_DISPLAY(i915)) {
14947                 for_each_pipe(i915, pipe) {
14948                         ret = intel_crtc_init(i915, pipe);
14949                         if (ret) {
14950                                 intel_mode_config_cleanup(i915);
14951                                 return ret;
14952                         }
14953                 }
14954         }
14955
14956         intel_plane_possible_crtcs_init(i915);
14957         intel_shared_dpll_init(dev);
14958         intel_update_fdi_pll_freq(i915);
14959
14960         intel_update_czclk(i915);
14961         intel_modeset_init_hw(i915);
14962
14963         intel_hdcp_component_init(i915);
14964
14965         if (i915->max_cdclk_freq == 0)
14966                 intel_update_max_cdclk(i915);
14967
14968         /*
14969          * If the platform has HTI, we need to find out whether it has reserved
14970          * any display resources before we create our display outputs.
14971          */
14972         if (INTEL_INFO(i915)->display.has_hti)
14973                 i915->hti_state = intel_de_read(i915, HDPORT_STATE);
14974
14975         /* Just disable it once at startup */
14976         intel_vga_disable(i915);
14977         intel_setup_outputs(i915);
14978
14979         drm_modeset_lock_all(dev);
14980         intel_modeset_setup_hw_state(dev, dev->mode_config.acquire_ctx);
14981         drm_modeset_unlock_all(dev);
14982
14983         for_each_intel_crtc(dev, crtc) {
14984                 struct intel_initial_plane_config plane_config = {};
14985
14986                 if (!to_intel_crtc_state(crtc->base.state)->uapi.active)
14987                         continue;
14988
14989                 /*
14990                  * Note that reserving the BIOS fb up front prevents us
14991                  * from stuffing other stolen allocations like the ring
14992                  * on top.  This prevents some ugliness at boot time, and
14993                  * can even allow for smooth boot transitions if the BIOS
14994                  * fb is large enough for the active pipe configuration.
14995                  */
14996                 i915->display.get_initial_plane_config(crtc, &plane_config);
14997
14998                 /*
14999                  * If the fb is shared between multiple heads, we'll
15000                  * just get the first one.
15001                  */
15002                 intel_find_initial_plane_obj(crtc, &plane_config);
15003
15004                 plane_config_fini(&plane_config);
15005         }
15006
15007         /*
15008          * Make sure hardware watermarks really match the state we read out.
15009          * Note that we need to do this after reconstructing the BIOS fb's
15010          * since the watermark calculation done here will use pstate->fb.
15011          */
15012         if (!HAS_GMCH(i915))
15013                 sanitize_watermarks(i915);
15014
15015         return 0;
15016 }
15017
15018 /* part #3: call after gem init */
15019 int intel_modeset_init(struct drm_i915_private *i915)
15020 {
15021         int ret;
15022
15023         if (!HAS_DISPLAY(i915))
15024                 return 0;
15025
15026         /*
15027          * Force all active planes to recompute their states. So that on
15028          * mode_setcrtc after probe, all the intel_plane_state variables
15029          * are already calculated and there is no assert_plane warnings
15030          * during bootup.
15031          */
15032         ret = intel_initial_commit(&i915->drm);
15033         if (ret)
15034                 drm_dbg_kms(&i915->drm, "Initial modeset failed, %d\n", ret);
15035
15036         intel_overlay_setup(i915);
15037
15038         ret = intel_fbdev_init(&i915->drm);
15039         if (ret)
15040                 return ret;
15041
15042         /* Only enable hotplug handling once the fbdev is fully set up. */
15043         intel_hpd_init(i915);
15044         intel_hpd_poll_disable(i915);
15045
15046         intel_init_ipc(i915);
15047
15048         return 0;
15049 }
15050
15051 void i830_enable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe)
15052 {
15053         struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
15054         /* 640x480@60Hz, ~25175 kHz */
15055         struct dpll clock = {
15056                 .m1 = 18,
15057                 .m2 = 7,
15058                 .p1 = 13,
15059                 .p2 = 4,
15060                 .n = 2,
15061         };
15062         u32 dpll, fp;
15063         int i;
15064
15065         drm_WARN_ON(&dev_priv->drm,
15066                     i9xx_calc_dpll_params(48000, &clock) != 25154);
15067
15068         drm_dbg_kms(&dev_priv->drm,
15069                     "enabling pipe %c due to force quirk (vco=%d dot=%d)\n",
15070                     pipe_name(pipe), clock.vco, clock.dot);
15071
15072         fp = i9xx_dpll_compute_fp(&clock);
15073         dpll = DPLL_DVO_2X_MODE |
15074                 DPLL_VGA_MODE_DIS |
15075                 ((clock.p1 - 2) << DPLL_FPA01_P1_POST_DIV_SHIFT) |
15076                 PLL_P2_DIVIDE_BY_4 |
15077                 PLL_REF_INPUT_DREFCLK |
15078                 DPLL_VCO_ENABLE;
15079
15080         intel_de_write(dev_priv, FP0(pipe), fp);
15081         intel_de_write(dev_priv, FP1(pipe), fp);
15082
15083         intel_de_write(dev_priv, HTOTAL(pipe), (640 - 1) | ((800 - 1) << 16));
15084         intel_de_write(dev_priv, HBLANK(pipe), (640 - 1) | ((800 - 1) << 16));
15085         intel_de_write(dev_priv, HSYNC(pipe), (656 - 1) | ((752 - 1) << 16));
15086         intel_de_write(dev_priv, VTOTAL(pipe), (480 - 1) | ((525 - 1) << 16));
15087         intel_de_write(dev_priv, VBLANK(pipe), (480 - 1) | ((525 - 1) << 16));
15088         intel_de_write(dev_priv, VSYNC(pipe), (490 - 1) | ((492 - 1) << 16));
15089         intel_de_write(dev_priv, PIPESRC(pipe), ((640 - 1) << 16) | (480 - 1));
15090
15091         /*
15092          * Apparently we need to have VGA mode enabled prior to changing
15093          * the P1/P2 dividers. Otherwise the DPLL will keep using the old
15094          * dividers, even though the register value does change.
15095          */
15096         intel_de_write(dev_priv, DPLL(pipe), dpll & ~DPLL_VGA_MODE_DIS);
15097         intel_de_write(dev_priv, DPLL(pipe), dpll);
15098
15099         /* Wait for the clocks to stabilize. */
15100         intel_de_posting_read(dev_priv, DPLL(pipe));
15101         udelay(150);
15102
15103         /* The pixel multiplier can only be updated once the
15104          * DPLL is enabled and the clocks are stable.
15105          *
15106          * So write it again.
15107          */
15108         intel_de_write(dev_priv, DPLL(pipe), dpll);
15109
15110         /* We do this three times for luck */
15111         for (i = 0; i < 3 ; i++) {
15112                 intel_de_write(dev_priv, DPLL(pipe), dpll);
15113                 intel_de_posting_read(dev_priv, DPLL(pipe));
15114                 udelay(150); /* wait for warmup */
15115         }
15116
15117         intel_de_write(dev_priv, PIPECONF(pipe),
15118                        PIPECONF_ENABLE | PIPECONF_PROGRESSIVE);
15119         intel_de_posting_read(dev_priv, PIPECONF(pipe));
15120
15121         intel_wait_for_pipe_scanline_moving(crtc);
15122 }
15123
15124 void i830_disable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe)
15125 {
15126         struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
15127
15128         drm_dbg_kms(&dev_priv->drm, "disabling pipe %c due to force quirk\n",
15129                     pipe_name(pipe));
15130
15131         drm_WARN_ON(&dev_priv->drm,
15132                     intel_de_read(dev_priv, DSPCNTR(PLANE_A)) &
15133                     DISPLAY_PLANE_ENABLE);
15134         drm_WARN_ON(&dev_priv->drm,
15135                     intel_de_read(dev_priv, DSPCNTR(PLANE_B)) &
15136                     DISPLAY_PLANE_ENABLE);
15137         drm_WARN_ON(&dev_priv->drm,
15138                     intel_de_read(dev_priv, DSPCNTR(PLANE_C)) &
15139                     DISPLAY_PLANE_ENABLE);
15140         drm_WARN_ON(&dev_priv->drm,
15141                     intel_de_read(dev_priv, CURCNTR(PIPE_A)) & MCURSOR_MODE);
15142         drm_WARN_ON(&dev_priv->drm,
15143                     intel_de_read(dev_priv, CURCNTR(PIPE_B)) & MCURSOR_MODE);
15144
15145         intel_de_write(dev_priv, PIPECONF(pipe), 0);
15146         intel_de_posting_read(dev_priv, PIPECONF(pipe));
15147
15148         intel_wait_for_pipe_scanline_stopped(crtc);
15149
15150         intel_de_write(dev_priv, DPLL(pipe), DPLL_VGA_MODE_DIS);
15151         intel_de_posting_read(dev_priv, DPLL(pipe));
15152 }
15153
15154 static void
15155 intel_sanitize_plane_mapping(struct drm_i915_private *dev_priv)
15156 {
15157         struct intel_crtc *crtc;
15158
15159         if (INTEL_GEN(dev_priv) >= 4)
15160                 return;
15161
15162         for_each_intel_crtc(&dev_priv->drm, crtc) {
15163                 struct intel_plane *plane =
15164                         to_intel_plane(crtc->base.primary);
15165                 struct intel_crtc *plane_crtc;
15166                 enum pipe pipe;
15167
15168                 if (!plane->get_hw_state(plane, &pipe))
15169                         continue;
15170
15171                 if (pipe == crtc->pipe)
15172                         continue;
15173
15174                 drm_dbg_kms(&dev_priv->drm,
15175                             "[PLANE:%d:%s] attached to the wrong pipe, disabling plane\n",
15176                             plane->base.base.id, plane->base.name);
15177
15178                 plane_crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
15179                 intel_plane_disable_noatomic(plane_crtc, plane);
15180         }
15181 }
15182
15183 static bool intel_crtc_has_encoders(struct intel_crtc *crtc)
15184 {
15185         struct drm_device *dev = crtc->base.dev;
15186         struct intel_encoder *encoder;
15187
15188         for_each_encoder_on_crtc(dev, &crtc->base, encoder)
15189                 return true;
15190
15191         return false;
15192 }
15193
15194 static struct intel_connector *intel_encoder_find_connector(struct intel_encoder *encoder)
15195 {
15196         struct drm_device *dev = encoder->base.dev;
15197         struct intel_connector *connector;
15198
15199         for_each_connector_on_encoder(dev, &encoder->base, connector)
15200                 return connector;
15201
15202         return NULL;
15203 }
15204
15205 static bool has_pch_trancoder(struct drm_i915_private *dev_priv,
15206                               enum pipe pch_transcoder)
15207 {
15208         return HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv) ||
15209                 (HAS_PCH_LPT_H(dev_priv) && pch_transcoder == PIPE_A);
15210 }
15211
15212 static void intel_sanitize_frame_start_delay(const struct intel_crtc_state *crtc_state)
15213 {
15214         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
15215         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
15216         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
15217
15218         if (INTEL_GEN(dev_priv) >= 9 ||
15219             IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv)) {
15220                 i915_reg_t reg = CHICKEN_TRANS(cpu_transcoder);
15221                 u32 val;
15222
15223                 if (transcoder_is_dsi(cpu_transcoder))
15224                         return;
15225
15226                 val = intel_de_read(dev_priv, reg);
15227                 val &= ~HSW_FRAME_START_DELAY_MASK;
15228                 val |= HSW_FRAME_START_DELAY(dev_priv->framestart_delay - 1);
15229                 intel_de_write(dev_priv, reg, val);
15230         } else {
15231                 i915_reg_t reg = PIPECONF(cpu_transcoder);
15232                 u32 val;
15233
15234                 val = intel_de_read(dev_priv, reg);
15235                 val &= ~PIPECONF_FRAME_START_DELAY_MASK;
15236                 val |= PIPECONF_FRAME_START_DELAY(dev_priv->framestart_delay - 1);
15237                 intel_de_write(dev_priv, reg, val);
15238         }
15239
15240         if (!crtc_state->has_pch_encoder)
15241                 return;
15242
15243         if (HAS_PCH_IBX(dev_priv)) {
15244                 i915_reg_t reg = PCH_TRANSCONF(crtc->pipe);
15245                 u32 val;
15246
15247                 val = intel_de_read(dev_priv, reg);
15248                 val &= ~TRANS_FRAME_START_DELAY_MASK;
15249                 val |= TRANS_FRAME_START_DELAY(dev_priv->framestart_delay - 1);
15250                 intel_de_write(dev_priv, reg, val);
15251         } else {
15252                 enum pipe pch_transcoder = intel_crtc_pch_transcoder(crtc);
15253                 i915_reg_t reg = TRANS_CHICKEN2(pch_transcoder);
15254                 u32 val;
15255
15256                 val = intel_de_read(dev_priv, reg);
15257                 val &= ~TRANS_CHICKEN2_FRAME_START_DELAY_MASK;
15258                 val |= TRANS_CHICKEN2_FRAME_START_DELAY(dev_priv->framestart_delay - 1);
15259                 intel_de_write(dev_priv, reg, val);
15260         }
15261 }
15262
15263 static void intel_sanitize_crtc(struct intel_crtc *crtc,
15264                                 struct drm_modeset_acquire_ctx *ctx)
15265 {
15266         struct drm_device *dev = crtc->base.dev;
15267         struct drm_i915_private *dev_priv = to_i915(dev);
15268         struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state);
15269
15270         if (crtc_state->hw.active) {
15271                 struct intel_plane *plane;
15272
15273                 /* Clear any frame start delays used for debugging left by the BIOS */
15274                 intel_sanitize_frame_start_delay(crtc_state);
15275
15276                 /* Disable everything but the primary plane */
15277                 for_each_intel_plane_on_crtc(dev, crtc, plane) {
15278                         const struct intel_plane_state *plane_state =
15279                                 to_intel_plane_state(plane->base.state);
15280
15281                         if (plane_state->uapi.visible &&
15282                             plane->base.type != DRM_PLANE_TYPE_PRIMARY)
15283                                 intel_plane_disable_noatomic(crtc, plane);
15284                 }
15285
15286                 /*
15287                  * Disable any background color set by the BIOS, but enable the
15288                  * gamma and CSC to match how we program our planes.
15289                  */
15290                 if (INTEL_GEN(dev_priv) >= 9)
15291                         intel_de_write(dev_priv, SKL_BOTTOM_COLOR(crtc->pipe),
15292                                        SKL_BOTTOM_COLOR_GAMMA_ENABLE | SKL_BOTTOM_COLOR_CSC_ENABLE);
15293         }
15294
15295         /* Adjust the state of the output pipe according to whether we
15296          * have active connectors/encoders. */
15297         if (crtc_state->hw.active && !intel_crtc_has_encoders(crtc) &&
15298             !crtc_state->bigjoiner_slave)
15299                 intel_crtc_disable_noatomic(crtc, ctx);
15300
15301         if (crtc_state->hw.active || HAS_GMCH(dev_priv)) {
15302                 /*
15303                  * We start out with underrun reporting disabled to avoid races.
15304                  * For correct bookkeeping mark this on active crtcs.
15305                  *
15306                  * Also on gmch platforms we dont have any hardware bits to
15307                  * disable the underrun reporting. Which means we need to start
15308                  * out with underrun reporting disabled also on inactive pipes,
15309                  * since otherwise we'll complain about the garbage we read when
15310                  * e.g. coming up after runtime pm.
15311                  *
15312                  * No protection against concurrent access is required - at
15313                  * worst a fifo underrun happens which also sets this to false.
15314                  */
15315                 crtc->cpu_fifo_underrun_disabled = true;
15316                 /*
15317                  * We track the PCH trancoder underrun reporting state
15318                  * within the crtc. With crtc for pipe A housing the underrun
15319                  * reporting state for PCH transcoder A, crtc for pipe B housing
15320                  * it for PCH transcoder B, etc. LPT-H has only PCH transcoder A,
15321                  * and marking underrun reporting as disabled for the non-existing
15322                  * PCH transcoders B and C would prevent enabling the south
15323                  * error interrupt (see cpt_can_enable_serr_int()).
15324                  */
15325                 if (has_pch_trancoder(dev_priv, crtc->pipe))
15326                         crtc->pch_fifo_underrun_disabled = true;
15327         }
15328 }
15329
15330 static bool has_bogus_dpll_config(const struct intel_crtc_state *crtc_state)
15331 {
15332         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
15333
15334         /*
15335          * Some SNB BIOSen (eg. ASUS K53SV) are known to misprogram
15336          * the hardware when a high res displays plugged in. DPLL P
15337          * divider is zero, and the pipe timings are bonkers. We'll
15338          * try to disable everything in that case.
15339          *
15340          * FIXME would be nice to be able to sanitize this state
15341          * without several WARNs, but for now let's take the easy
15342          * road.
15343          */
15344         return IS_GEN(dev_priv, 6) &&
15345                 crtc_state->hw.active &&
15346                 crtc_state->shared_dpll &&
15347                 crtc_state->port_clock == 0;
15348 }
15349
15350 static void intel_sanitize_encoder(struct intel_encoder *encoder)
15351 {
15352         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
15353         struct intel_connector *connector;
15354         struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
15355         struct intel_crtc_state *crtc_state = crtc ?
15356                 to_intel_crtc_state(crtc->base.state) : NULL;
15357
15358         /* We need to check both for a crtc link (meaning that the
15359          * encoder is active and trying to read from a pipe) and the
15360          * pipe itself being active. */
15361         bool has_active_crtc = crtc_state &&
15362                 crtc_state->hw.active;
15363
15364         if (crtc_state && has_bogus_dpll_config(crtc_state)) {
15365                 drm_dbg_kms(&dev_priv->drm,
15366                             "BIOS has misprogrammed the hardware. Disabling pipe %c\n",
15367                             pipe_name(crtc->pipe));
15368                 has_active_crtc = false;
15369         }
15370
15371         connector = intel_encoder_find_connector(encoder);
15372         if (connector && !has_active_crtc) {
15373                 drm_dbg_kms(&dev_priv->drm,
15374                             "[ENCODER:%d:%s] has active connectors but no active pipe!\n",
15375                             encoder->base.base.id,
15376                             encoder->base.name);
15377
15378                 /* Connector is active, but has no active pipe. This is
15379                  * fallout from our resume register restoring. Disable
15380                  * the encoder manually again. */
15381                 if (crtc_state) {
15382                         struct drm_encoder *best_encoder;
15383
15384                         drm_dbg_kms(&dev_priv->drm,
15385                                     "[ENCODER:%d:%s] manually disabled\n",
15386                                     encoder->base.base.id,
15387                                     encoder->base.name);
15388
15389                         /* avoid oopsing in case the hooks consult best_encoder */
15390                         best_encoder = connector->base.state->best_encoder;
15391                         connector->base.state->best_encoder = &encoder->base;
15392
15393                         /* FIXME NULL atomic state passed! */
15394                         if (encoder->disable)
15395                                 encoder->disable(NULL, encoder, crtc_state,
15396                                                  connector->base.state);
15397                         if (encoder->post_disable)
15398                                 encoder->post_disable(NULL, encoder, crtc_state,
15399                                                       connector->base.state);
15400
15401                         connector->base.state->best_encoder = best_encoder;
15402                 }
15403                 encoder->base.crtc = NULL;
15404
15405                 /* Inconsistent output/port/pipe state happens presumably due to
15406                  * a bug in one of the get_hw_state functions. Or someplace else
15407                  * in our code, like the register restore mess on resume. Clamp
15408                  * things to off as a safer default. */
15409
15410                 connector->base.dpms = DRM_MODE_DPMS_OFF;
15411                 connector->base.encoder = NULL;
15412         }
15413
15414         /* notify opregion of the sanitized encoder state */
15415         intel_opregion_notify_encoder(encoder, connector && has_active_crtc);
15416
15417         if (INTEL_GEN(dev_priv) >= 11)
15418                 icl_sanitize_encoder_pll_mapping(encoder);
15419 }
15420
15421 /* FIXME read out full plane state for all planes */
15422 static void readout_plane_state(struct drm_i915_private *dev_priv)
15423 {
15424         struct intel_plane *plane;
15425         struct intel_crtc *crtc;
15426
15427         for_each_intel_plane(&dev_priv->drm, plane) {
15428                 struct intel_plane_state *plane_state =
15429                         to_intel_plane_state(plane->base.state);
15430                 struct intel_crtc_state *crtc_state;
15431                 enum pipe pipe = PIPE_A;
15432                 bool visible;
15433
15434                 visible = plane->get_hw_state(plane, &pipe);
15435
15436                 crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
15437                 crtc_state = to_intel_crtc_state(crtc->base.state);
15438
15439                 intel_set_plane_visible(crtc_state, plane_state, visible);
15440
15441                 drm_dbg_kms(&dev_priv->drm,
15442                             "[PLANE:%d:%s] hw state readout: %s, pipe %c\n",
15443                             plane->base.base.id, plane->base.name,
15444                             enableddisabled(visible), pipe_name(pipe));
15445         }
15446
15447         for_each_intel_crtc(&dev_priv->drm, crtc) {
15448                 struct intel_crtc_state *crtc_state =
15449                         to_intel_crtc_state(crtc->base.state);
15450
15451                 fixup_plane_bitmasks(crtc_state);
15452         }
15453 }
15454
15455 static void intel_modeset_readout_hw_state(struct drm_device *dev)
15456 {
15457         struct drm_i915_private *dev_priv = to_i915(dev);
15458         struct intel_cdclk_state *cdclk_state =
15459                 to_intel_cdclk_state(dev_priv->cdclk.obj.state);
15460         struct intel_dbuf_state *dbuf_state =
15461                 to_intel_dbuf_state(dev_priv->dbuf.obj.state);
15462         enum pipe pipe;
15463         struct intel_crtc *crtc;
15464         struct intel_encoder *encoder;
15465         struct intel_connector *connector;
15466         struct drm_connector_list_iter conn_iter;
15467         u8 active_pipes = 0;
15468
15469         for_each_intel_crtc(dev, crtc) {
15470                 struct intel_crtc_state *crtc_state =
15471                         to_intel_crtc_state(crtc->base.state);
15472
15473                 __drm_atomic_helper_crtc_destroy_state(&crtc_state->uapi);
15474                 intel_crtc_free_hw_state(crtc_state);
15475                 intel_crtc_state_reset(crtc_state, crtc);
15476
15477                 intel_crtc_get_pipe_config(crtc_state);
15478
15479                 crtc_state->hw.enable = crtc_state->hw.active;
15480
15481                 crtc->base.enabled = crtc_state->hw.enable;
15482                 crtc->active = crtc_state->hw.active;
15483
15484                 if (crtc_state->hw.active)
15485                         active_pipes |= BIT(crtc->pipe);
15486
15487                 drm_dbg_kms(&dev_priv->drm,
15488                             "[CRTC:%d:%s] hw state readout: %s\n",
15489                             crtc->base.base.id, crtc->base.name,
15490                             enableddisabled(crtc_state->hw.active));
15491         }
15492
15493         dev_priv->active_pipes = cdclk_state->active_pipes =
15494                 dbuf_state->active_pipes = active_pipes;
15495
15496         readout_plane_state(dev_priv);
15497
15498         intel_dpll_readout_hw_state(dev_priv);
15499
15500         for_each_intel_encoder(dev, encoder) {
15501                 pipe = 0;
15502
15503                 if (encoder->get_hw_state(encoder, &pipe)) {
15504                         struct intel_crtc_state *crtc_state;
15505
15506                         crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
15507                         crtc_state = to_intel_crtc_state(crtc->base.state);
15508
15509                         encoder->base.crtc = &crtc->base;
15510                         intel_encoder_get_config(encoder, crtc_state);
15511                         if (encoder->sync_state)
15512                                 encoder->sync_state(encoder, crtc_state);
15513
15514                         /* read out to slave crtc as well for bigjoiner */
15515                         if (crtc_state->bigjoiner) {
15516                                 /* encoder should read be linked to bigjoiner master */
15517                                 WARN_ON(crtc_state->bigjoiner_slave);
15518
15519                                 crtc = crtc_state->bigjoiner_linked_crtc;
15520                                 crtc_state = to_intel_crtc_state(crtc->base.state);
15521                                 intel_encoder_get_config(encoder, crtc_state);
15522                         }
15523                 } else {
15524                         encoder->base.crtc = NULL;
15525                 }
15526
15527                 drm_dbg_kms(&dev_priv->drm,
15528                             "[ENCODER:%d:%s] hw state readout: %s, pipe %c\n",
15529                             encoder->base.base.id, encoder->base.name,
15530                             enableddisabled(encoder->base.crtc),
15531                             pipe_name(pipe));
15532         }
15533
15534         drm_connector_list_iter_begin(dev, &conn_iter);
15535         for_each_intel_connector_iter(connector, &conn_iter) {
15536                 if (connector->get_hw_state(connector)) {
15537                         struct intel_crtc_state *crtc_state;
15538                         struct intel_crtc *crtc;
15539
15540                         connector->base.dpms = DRM_MODE_DPMS_ON;
15541
15542                         encoder = intel_attached_encoder(connector);
15543                         connector->base.encoder = &encoder->base;
15544
15545                         crtc = to_intel_crtc(encoder->base.crtc);
15546                         crtc_state = crtc ? to_intel_crtc_state(crtc->base.state) : NULL;
15547
15548                         if (crtc_state && crtc_state->hw.active) {
15549                                 /*
15550                                  * This has to be done during hardware readout
15551                                  * because anything calling .crtc_disable may
15552                                  * rely on the connector_mask being accurate.
15553                                  */
15554                                 crtc_state->uapi.connector_mask |=
15555                                         drm_connector_mask(&connector->base);
15556                                 crtc_state->uapi.encoder_mask |=
15557                                         drm_encoder_mask(&encoder->base);
15558                         }
15559                 } else {
15560                         connector->base.dpms = DRM_MODE_DPMS_OFF;
15561                         connector->base.encoder = NULL;
15562                 }
15563                 drm_dbg_kms(&dev_priv->drm,
15564                             "[CONNECTOR:%d:%s] hw state readout: %s\n",
15565                             connector->base.base.id, connector->base.name,
15566                             enableddisabled(connector->base.encoder));
15567         }
15568         drm_connector_list_iter_end(&conn_iter);
15569
15570         for_each_intel_crtc(dev, crtc) {
15571                 struct intel_bw_state *bw_state =
15572                         to_intel_bw_state(dev_priv->bw_obj.state);
15573                 struct intel_crtc_state *crtc_state =
15574                         to_intel_crtc_state(crtc->base.state);
15575                 struct intel_plane *plane;
15576                 int min_cdclk = 0;
15577
15578                 if (crtc_state->bigjoiner_slave)
15579                         continue;
15580
15581                 if (crtc_state->hw.active) {
15582                         /*
15583                          * The initial mode needs to be set in order to keep
15584                          * the atomic core happy. It wants a valid mode if the
15585                          * crtc's enabled, so we do the above call.
15586                          *
15587                          * But we don't set all the derived state fully, hence
15588                          * set a flag to indicate that a full recalculation is
15589                          * needed on the next commit.
15590                          */
15591                         crtc_state->inherited = true;
15592
15593                         intel_crtc_update_active_timings(crtc_state);
15594
15595                         intel_crtc_copy_hw_to_uapi_state(crtc_state);
15596                 }
15597
15598                 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
15599                         const struct intel_plane_state *plane_state =
15600                                 to_intel_plane_state(plane->base.state);
15601
15602                         /*
15603                          * FIXME don't have the fb yet, so can't
15604                          * use intel_plane_data_rate() :(
15605                          */
15606                         if (plane_state->uapi.visible)
15607                                 crtc_state->data_rate[plane->id] =
15608                                         4 * crtc_state->pixel_rate;
15609                         /*
15610                          * FIXME don't have the fb yet, so can't
15611                          * use plane->min_cdclk() :(
15612                          */
15613                         if (plane_state->uapi.visible && plane->min_cdclk) {
15614                                 if (crtc_state->double_wide ||
15615                                     INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
15616                                         crtc_state->min_cdclk[plane->id] =
15617                                                 DIV_ROUND_UP(crtc_state->pixel_rate, 2);
15618                                 else
15619                                         crtc_state->min_cdclk[plane->id] =
15620                                                 crtc_state->pixel_rate;
15621                         }
15622                         drm_dbg_kms(&dev_priv->drm,
15623                                     "[PLANE:%d:%s] min_cdclk %d kHz\n",
15624                                     plane->base.base.id, plane->base.name,
15625                                     crtc_state->min_cdclk[plane->id]);
15626                 }
15627
15628                 if (crtc_state->hw.active) {
15629                         min_cdclk = intel_crtc_compute_min_cdclk(crtc_state);
15630                         if (drm_WARN_ON(dev, min_cdclk < 0))
15631                                 min_cdclk = 0;
15632                 }
15633
15634                 cdclk_state->min_cdclk[crtc->pipe] = min_cdclk;
15635                 cdclk_state->min_voltage_level[crtc->pipe] =
15636                         crtc_state->min_voltage_level;
15637
15638                 intel_bw_crtc_update(bw_state, crtc_state);
15639
15640                 intel_pipe_config_sanity_check(dev_priv, crtc_state);
15641
15642                 /* discard our incomplete slave state, copy it from master */
15643                 if (crtc_state->bigjoiner && crtc_state->hw.active) {
15644                         struct intel_crtc *slave = crtc_state->bigjoiner_linked_crtc;
15645                         struct intel_crtc_state *slave_crtc_state =
15646                                 to_intel_crtc_state(slave->base.state);
15647
15648                         copy_bigjoiner_crtc_state(slave_crtc_state, crtc_state);
15649                         slave->base.mode = crtc->base.mode;
15650
15651                         cdclk_state->min_cdclk[slave->pipe] = min_cdclk;
15652                         cdclk_state->min_voltage_level[slave->pipe] =
15653                                 crtc_state->min_voltage_level;
15654
15655                         for_each_intel_plane_on_crtc(&dev_priv->drm, slave, plane) {
15656                                 const struct intel_plane_state *plane_state =
15657                                         to_intel_plane_state(plane->base.state);
15658
15659                                 /*
15660                                  * FIXME don't have the fb yet, so can't
15661                                  * use intel_plane_data_rate() :(
15662                                  */
15663                                 if (plane_state->uapi.visible)
15664                                         crtc_state->data_rate[plane->id] =
15665                                                 4 * crtc_state->pixel_rate;
15666                                 else
15667                                         crtc_state->data_rate[plane->id] = 0;
15668                         }
15669
15670                         intel_bw_crtc_update(bw_state, slave_crtc_state);
15671                         drm_calc_timestamping_constants(&slave->base,
15672                                                         &slave_crtc_state->hw.adjusted_mode);
15673                 }
15674         }
15675 }
15676
15677 static void
15678 get_encoder_power_domains(struct drm_i915_private *dev_priv)
15679 {
15680         struct intel_encoder *encoder;
15681
15682         for_each_intel_encoder(&dev_priv->drm, encoder) {
15683                 struct intel_crtc_state *crtc_state;
15684
15685                 if (!encoder->get_power_domains)
15686                         continue;
15687
15688                 /*
15689                  * MST-primary and inactive encoders don't have a crtc state
15690                  * and neither of these require any power domain references.
15691                  */
15692                 if (!encoder->base.crtc)
15693                         continue;
15694
15695                 crtc_state = to_intel_crtc_state(encoder->base.crtc->state);
15696                 encoder->get_power_domains(encoder, crtc_state);
15697         }
15698 }
15699
15700 static void intel_early_display_was(struct drm_i915_private *dev_priv)
15701 {
15702         /*
15703          * Display WA #1185 WaDisableDARBFClkGating:cnl,glk,icl,ehl,tgl
15704          * Also known as Wa_14010480278.
15705          */
15706         if (IS_GEN_RANGE(dev_priv, 10, 12) || IS_GEMINILAKE(dev_priv))
15707                 intel_de_write(dev_priv, GEN9_CLKGATE_DIS_0,
15708                                intel_de_read(dev_priv, GEN9_CLKGATE_DIS_0) | DARBF_GATING_DIS);
15709
15710         if (IS_HASWELL(dev_priv)) {
15711                 /*
15712                  * WaRsPkgCStateDisplayPMReq:hsw
15713                  * System hang if this isn't done before disabling all planes!
15714                  */
15715                 intel_de_write(dev_priv, CHICKEN_PAR1_1,
15716                                intel_de_read(dev_priv, CHICKEN_PAR1_1) | FORCE_ARB_IDLE_PLANES);
15717         }
15718
15719         if (IS_KABYLAKE(dev_priv) || IS_COFFEELAKE(dev_priv) || IS_COMETLAKE(dev_priv)) {
15720                 /* Display WA #1142:kbl,cfl,cml */
15721                 intel_de_rmw(dev_priv, CHICKEN_PAR1_1,
15722                              KBL_ARB_FILL_SPARE_22, KBL_ARB_FILL_SPARE_22);
15723                 intel_de_rmw(dev_priv, CHICKEN_MISC_2,
15724                              KBL_ARB_FILL_SPARE_13 | KBL_ARB_FILL_SPARE_14,
15725                              KBL_ARB_FILL_SPARE_14);
15726         }
15727 }
15728
15729 static void ibx_sanitize_pch_hdmi_port(struct drm_i915_private *dev_priv,
15730                                        enum port port, i915_reg_t hdmi_reg)
15731 {
15732         u32 val = intel_de_read(dev_priv, hdmi_reg);
15733
15734         if (val & SDVO_ENABLE ||
15735             (val & SDVO_PIPE_SEL_MASK) == SDVO_PIPE_SEL(PIPE_A))
15736                 return;
15737
15738         drm_dbg_kms(&dev_priv->drm,
15739                     "Sanitizing transcoder select for HDMI %c\n",
15740                     port_name(port));
15741
15742         val &= ~SDVO_PIPE_SEL_MASK;
15743         val |= SDVO_PIPE_SEL(PIPE_A);
15744
15745         intel_de_write(dev_priv, hdmi_reg, val);
15746 }
15747
15748 static void ibx_sanitize_pch_dp_port(struct drm_i915_private *dev_priv,
15749                                      enum port port, i915_reg_t dp_reg)
15750 {
15751         u32 val = intel_de_read(dev_priv, dp_reg);
15752
15753         if (val & DP_PORT_EN ||
15754             (val & DP_PIPE_SEL_MASK) == DP_PIPE_SEL(PIPE_A))
15755                 return;
15756
15757         drm_dbg_kms(&dev_priv->drm,
15758                     "Sanitizing transcoder select for DP %c\n",
15759                     port_name(port));
15760
15761         val &= ~DP_PIPE_SEL_MASK;
15762         val |= DP_PIPE_SEL(PIPE_A);
15763
15764         intel_de_write(dev_priv, dp_reg, val);
15765 }
15766
15767 static void ibx_sanitize_pch_ports(struct drm_i915_private *dev_priv)
15768 {
15769         /*
15770          * The BIOS may select transcoder B on some of the PCH
15771          * ports even it doesn't enable the port. This would trip
15772          * assert_pch_dp_disabled() and assert_pch_hdmi_disabled().
15773          * Sanitize the transcoder select bits to prevent that. We
15774          * assume that the BIOS never actually enabled the port,
15775          * because if it did we'd actually have to toggle the port
15776          * on and back off to make the transcoder A select stick
15777          * (see. intel_dp_link_down(), intel_disable_hdmi(),
15778          * intel_disable_sdvo()).
15779          */
15780         ibx_sanitize_pch_dp_port(dev_priv, PORT_B, PCH_DP_B);
15781         ibx_sanitize_pch_dp_port(dev_priv, PORT_C, PCH_DP_C);
15782         ibx_sanitize_pch_dp_port(dev_priv, PORT_D, PCH_DP_D);
15783
15784         /* PCH SDVOB multiplex with HDMIB */
15785         ibx_sanitize_pch_hdmi_port(dev_priv, PORT_B, PCH_HDMIB);
15786         ibx_sanitize_pch_hdmi_port(dev_priv, PORT_C, PCH_HDMIC);
15787         ibx_sanitize_pch_hdmi_port(dev_priv, PORT_D, PCH_HDMID);
15788 }
15789
15790 /* Scan out the current hw modeset state,
15791  * and sanitizes it to the current state
15792  */
15793 static void
15794 intel_modeset_setup_hw_state(struct drm_device *dev,
15795                              struct drm_modeset_acquire_ctx *ctx)
15796 {
15797         struct drm_i915_private *dev_priv = to_i915(dev);
15798         struct intel_encoder *encoder;
15799         struct intel_crtc *crtc;
15800         intel_wakeref_t wakeref;
15801
15802         wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_INIT);
15803
15804         intel_early_display_was(dev_priv);
15805         intel_modeset_readout_hw_state(dev);
15806
15807         /* HW state is read out, now we need to sanitize this mess. */
15808
15809         /* Sanitize the TypeC port mode upfront, encoders depend on this */
15810         for_each_intel_encoder(dev, encoder) {
15811                 enum phy phy = intel_port_to_phy(dev_priv, encoder->port);
15812
15813                 /* We need to sanitize only the MST primary port. */
15814                 if (encoder->type != INTEL_OUTPUT_DP_MST &&
15815                     intel_phy_is_tc(dev_priv, phy))
15816                         intel_tc_port_sanitize(enc_to_dig_port(encoder));
15817         }
15818
15819         get_encoder_power_domains(dev_priv);
15820
15821         if (HAS_PCH_IBX(dev_priv))
15822                 ibx_sanitize_pch_ports(dev_priv);
15823
15824         /*
15825          * intel_sanitize_plane_mapping() may need to do vblank
15826          * waits, so we need vblank interrupts restored beforehand.
15827          */
15828         for_each_intel_crtc(&dev_priv->drm, crtc) {
15829                 struct intel_crtc_state *crtc_state =
15830                         to_intel_crtc_state(crtc->base.state);
15831
15832                 drm_crtc_vblank_reset(&crtc->base);
15833
15834                 if (crtc_state->hw.active)
15835                         intel_crtc_vblank_on(crtc_state);
15836         }
15837
15838         intel_sanitize_plane_mapping(dev_priv);
15839
15840         for_each_intel_encoder(dev, encoder)
15841                 intel_sanitize_encoder(encoder);
15842
15843         for_each_intel_crtc(&dev_priv->drm, crtc) {
15844                 struct intel_crtc_state *crtc_state =
15845                         to_intel_crtc_state(crtc->base.state);
15846
15847                 intel_sanitize_crtc(crtc, ctx);
15848                 intel_dump_pipe_config(crtc_state, NULL, "[setup_hw_state]");
15849         }
15850
15851         intel_modeset_update_connector_atomic_state(dev);
15852
15853         intel_dpll_sanitize_state(dev_priv);
15854
15855         if (IS_G4X(dev_priv)) {
15856                 g4x_wm_get_hw_state(dev_priv);
15857                 g4x_wm_sanitize(dev_priv);
15858         } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
15859                 vlv_wm_get_hw_state(dev_priv);
15860                 vlv_wm_sanitize(dev_priv);
15861         } else if (INTEL_GEN(dev_priv) >= 9) {
15862                 skl_wm_get_hw_state(dev_priv);
15863         } else if (HAS_PCH_SPLIT(dev_priv)) {
15864                 ilk_wm_get_hw_state(dev_priv);
15865         }
15866
15867         for_each_intel_crtc(dev, crtc) {
15868                 struct intel_crtc_state *crtc_state =
15869                         to_intel_crtc_state(crtc->base.state);
15870                 u64 put_domains;
15871
15872                 put_domains = modeset_get_crtc_power_domains(crtc_state);
15873                 if (drm_WARN_ON(dev, put_domains))
15874                         modeset_put_crtc_power_domains(crtc, put_domains);
15875         }
15876
15877         intel_display_power_put(dev_priv, POWER_DOMAIN_INIT, wakeref);
15878 }
15879
15880 void intel_display_resume(struct drm_device *dev)
15881 {
15882         struct drm_i915_private *dev_priv = to_i915(dev);
15883         struct drm_atomic_state *state = dev_priv->modeset_restore_state;
15884         struct drm_modeset_acquire_ctx ctx;
15885         int ret;
15886
15887         dev_priv->modeset_restore_state = NULL;
15888         if (state)
15889                 state->acquire_ctx = &ctx;
15890
15891         drm_modeset_acquire_init(&ctx, 0);
15892
15893         while (1) {
15894                 ret = drm_modeset_lock_all_ctx(dev, &ctx);
15895                 if (ret != -EDEADLK)
15896                         break;
15897
15898                 drm_modeset_backoff(&ctx);
15899         }
15900
15901         if (!ret)
15902                 ret = __intel_display_resume(dev, state, &ctx);
15903
15904         intel_enable_ipc(dev_priv);
15905         drm_modeset_drop_locks(&ctx);
15906         drm_modeset_acquire_fini(&ctx);
15907
15908         if (ret)
15909                 drm_err(&dev_priv->drm,
15910                         "Restoring old state failed with %i\n", ret);
15911         if (state)
15912                 drm_atomic_state_put(state);
15913 }
15914
15915 static void intel_hpd_poll_fini(struct drm_i915_private *i915)
15916 {
15917         struct intel_connector *connector;
15918         struct drm_connector_list_iter conn_iter;
15919
15920         /* Kill all the work that may have been queued by hpd. */
15921         drm_connector_list_iter_begin(&i915->drm, &conn_iter);
15922         for_each_intel_connector_iter(connector, &conn_iter) {
15923                 if (connector->modeset_retry_work.func)
15924                         cancel_work_sync(&connector->modeset_retry_work);
15925                 if (connector->hdcp.shim) {
15926                         cancel_delayed_work_sync(&connector->hdcp.check_work);
15927                         cancel_work_sync(&connector->hdcp.prop_work);
15928                 }
15929         }
15930         drm_connector_list_iter_end(&conn_iter);
15931 }
15932
15933 /* part #1: call before irq uninstall */
15934 void intel_modeset_driver_remove(struct drm_i915_private *i915)
15935 {
15936         flush_workqueue(i915->flip_wq);
15937         flush_workqueue(i915->modeset_wq);
15938
15939         flush_work(&i915->atomic_helper.free_work);
15940         drm_WARN_ON(&i915->drm, !llist_empty(&i915->atomic_helper.free_list));
15941 }
15942
15943 /* part #2: call after irq uninstall */
15944 void intel_modeset_driver_remove_noirq(struct drm_i915_private *i915)
15945 {
15946         /*
15947          * Due to the hpd irq storm handling the hotplug work can re-arm the
15948          * poll handlers. Hence disable polling after hpd handling is shut down.
15949          */
15950         intel_hpd_poll_fini(i915);
15951
15952         /*
15953          * MST topology needs to be suspended so we don't have any calls to
15954          * fbdev after it's finalized. MST will be destroyed later as part of
15955          * drm_mode_config_cleanup()
15956          */
15957         intel_dp_mst_suspend(i915);
15958
15959         /* poll work can call into fbdev, hence clean that up afterwards */
15960         intel_fbdev_fini(i915);
15961
15962         intel_unregister_dsm_handler();
15963
15964         intel_fbc_global_disable(i915);
15965
15966         /* flush any delayed tasks or pending work */
15967         flush_scheduled_work();
15968
15969         intel_hdcp_component_fini(i915);
15970
15971         intel_mode_config_cleanup(i915);
15972
15973         intel_overlay_cleanup(i915);
15974
15975         intel_gmbus_teardown(i915);
15976
15977         destroy_workqueue(i915->flip_wq);
15978         destroy_workqueue(i915->modeset_wq);
15979
15980         intel_fbc_cleanup_cfb(i915);
15981 }
15982
15983 /* part #3: call after gem init */
15984 void intel_modeset_driver_remove_nogem(struct drm_i915_private *i915)
15985 {
15986         intel_csr_ucode_fini(i915);
15987
15988         intel_power_domains_driver_remove(i915);
15989
15990         intel_vga_unregister(i915);
15991
15992         intel_bios_driver_remove(i915);
15993 }
15994
15995 #if IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR)
15996
15997 struct intel_display_error_state {
15998
15999         u32 power_well_driver;
16000
16001         struct intel_cursor_error_state {
16002                 u32 control;
16003                 u32 position;
16004                 u32 base;
16005                 u32 size;
16006         } cursor[I915_MAX_PIPES];
16007
16008         struct intel_pipe_error_state {
16009                 bool power_domain_on;
16010                 u32 source;
16011                 u32 stat;
16012         } pipe[I915_MAX_PIPES];
16013
16014         struct intel_plane_error_state {
16015                 u32 control;
16016                 u32 stride;
16017                 u32 size;
16018                 u32 pos;
16019                 u32 addr;
16020                 u32 surface;
16021                 u32 tile_offset;
16022         } plane[I915_MAX_PIPES];
16023
16024         struct intel_transcoder_error_state {
16025                 bool available;
16026                 bool power_domain_on;
16027                 enum transcoder cpu_transcoder;
16028
16029                 u32 conf;
16030
16031                 u32 htotal;
16032                 u32 hblank;
16033                 u32 hsync;
16034                 u32 vtotal;
16035                 u32 vblank;
16036                 u32 vsync;
16037         } transcoder[5];
16038 };
16039
16040 struct intel_display_error_state *
16041 intel_display_capture_error_state(struct drm_i915_private *dev_priv)
16042 {
16043         struct intel_display_error_state *error;
16044         int transcoders[] = {
16045                 TRANSCODER_A,
16046                 TRANSCODER_B,
16047                 TRANSCODER_C,
16048                 TRANSCODER_D,
16049                 TRANSCODER_EDP,
16050         };
16051         int i;
16052
16053         BUILD_BUG_ON(ARRAY_SIZE(transcoders) != ARRAY_SIZE(error->transcoder));
16054
16055         if (!HAS_DISPLAY(dev_priv))
16056                 return NULL;
16057
16058         error = kzalloc(sizeof(*error), GFP_ATOMIC);
16059         if (error == NULL)
16060                 return NULL;
16061
16062         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
16063                 error->power_well_driver = intel_de_read(dev_priv,
16064                                                          HSW_PWR_WELL_CTL2);
16065
16066         for_each_pipe(dev_priv, i) {
16067                 error->pipe[i].power_domain_on =
16068                         __intel_display_power_is_enabled(dev_priv,
16069                                                          POWER_DOMAIN_PIPE(i));
16070                 if (!error->pipe[i].power_domain_on)
16071                         continue;
16072
16073                 error->cursor[i].control = intel_de_read(dev_priv, CURCNTR(i));
16074                 error->cursor[i].position = intel_de_read(dev_priv, CURPOS(i));
16075                 error->cursor[i].base = intel_de_read(dev_priv, CURBASE(i));
16076
16077                 error->plane[i].control = intel_de_read(dev_priv, DSPCNTR(i));
16078                 error->plane[i].stride = intel_de_read(dev_priv, DSPSTRIDE(i));
16079                 if (INTEL_GEN(dev_priv) <= 3) {
16080                         error->plane[i].size = intel_de_read(dev_priv,
16081                                                              DSPSIZE(i));
16082                         error->plane[i].pos = intel_de_read(dev_priv,
16083                                                             DSPPOS(i));
16084                 }
16085                 if (INTEL_GEN(dev_priv) <= 7 && !IS_HASWELL(dev_priv))
16086                         error->plane[i].addr = intel_de_read(dev_priv,
16087                                                              DSPADDR(i));
16088                 if (INTEL_GEN(dev_priv) >= 4) {
16089                         error->plane[i].surface = intel_de_read(dev_priv,
16090                                                                 DSPSURF(i));
16091                         error->plane[i].tile_offset = intel_de_read(dev_priv,
16092                                                                     DSPTILEOFF(i));
16093                 }
16094
16095                 error->pipe[i].source = intel_de_read(dev_priv, PIPESRC(i));
16096
16097                 if (HAS_GMCH(dev_priv))
16098                         error->pipe[i].stat = intel_de_read(dev_priv,
16099                                                             PIPESTAT(i));
16100         }
16101
16102         for (i = 0; i < ARRAY_SIZE(error->transcoder); i++) {
16103                 enum transcoder cpu_transcoder = transcoders[i];
16104
16105                 if (!HAS_TRANSCODER(dev_priv, cpu_transcoder))
16106                         continue;
16107
16108                 error->transcoder[i].available = true;
16109                 error->transcoder[i].power_domain_on =
16110                         __intel_display_power_is_enabled(dev_priv,
16111                                 POWER_DOMAIN_TRANSCODER(cpu_transcoder));
16112                 if (!error->transcoder[i].power_domain_on)
16113                         continue;
16114
16115                 error->transcoder[i].cpu_transcoder = cpu_transcoder;
16116
16117                 error->transcoder[i].conf = intel_de_read(dev_priv,
16118                                                           PIPECONF(cpu_transcoder));
16119                 error->transcoder[i].htotal = intel_de_read(dev_priv,
16120                                                             HTOTAL(cpu_transcoder));
16121                 error->transcoder[i].hblank = intel_de_read(dev_priv,
16122                                                             HBLANK(cpu_transcoder));
16123                 error->transcoder[i].hsync = intel_de_read(dev_priv,
16124                                                            HSYNC(cpu_transcoder));
16125                 error->transcoder[i].vtotal = intel_de_read(dev_priv,
16126                                                             VTOTAL(cpu_transcoder));
16127                 error->transcoder[i].vblank = intel_de_read(dev_priv,
16128                                                             VBLANK(cpu_transcoder));
16129                 error->transcoder[i].vsync = intel_de_read(dev_priv,
16130                                                            VSYNC(cpu_transcoder));
16131         }
16132
16133         return error;
16134 }
16135
16136 #define err_printf(e, ...) i915_error_printf(e, __VA_ARGS__)
16137
16138 void
16139 intel_display_print_error_state(struct drm_i915_error_state_buf *m,
16140                                 struct intel_display_error_state *error)
16141 {
16142         struct drm_i915_private *dev_priv = m->i915;
16143         int i;
16144
16145         if (!error)
16146                 return;
16147
16148         err_printf(m, "Num Pipes: %d\n", INTEL_NUM_PIPES(dev_priv));
16149         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
16150                 err_printf(m, "PWR_WELL_CTL2: %08x\n",
16151                            error->power_well_driver);
16152         for_each_pipe(dev_priv, i) {
16153                 err_printf(m, "Pipe [%d]:\n", i);
16154                 err_printf(m, "  Power: %s\n",
16155                            onoff(error->pipe[i].power_domain_on));
16156                 err_printf(m, "  SRC: %08x\n", error->pipe[i].source);
16157                 err_printf(m, "  STAT: %08x\n", error->pipe[i].stat);
16158
16159                 err_printf(m, "Plane [%d]:\n", i);
16160                 err_printf(m, "  CNTR: %08x\n", error->plane[i].control);
16161                 err_printf(m, "  STRIDE: %08x\n", error->plane[i].stride);
16162                 if (INTEL_GEN(dev_priv) <= 3) {
16163                         err_printf(m, "  SIZE: %08x\n", error->plane[i].size);
16164                         err_printf(m, "  POS: %08x\n", error->plane[i].pos);
16165                 }
16166                 if (INTEL_GEN(dev_priv) <= 7 && !IS_HASWELL(dev_priv))
16167                         err_printf(m, "  ADDR: %08x\n", error->plane[i].addr);
16168                 if (INTEL_GEN(dev_priv) >= 4) {
16169                         err_printf(m, "  SURF: %08x\n", error->plane[i].surface);
16170                         err_printf(m, "  TILEOFF: %08x\n", error->plane[i].tile_offset);
16171                 }
16172
16173                 err_printf(m, "Cursor [%d]:\n", i);
16174                 err_printf(m, "  CNTR: %08x\n", error->cursor[i].control);
16175                 err_printf(m, "  POS: %08x\n", error->cursor[i].position);
16176                 err_printf(m, "  BASE: %08x\n", error->cursor[i].base);
16177         }
16178
16179         for (i = 0; i < ARRAY_SIZE(error->transcoder); i++) {
16180                 if (!error->transcoder[i].available)
16181                         continue;
16182
16183                 err_printf(m, "CPU transcoder: %s\n",
16184                            transcoder_name(error->transcoder[i].cpu_transcoder));
16185                 err_printf(m, "  Power: %s\n",
16186                            onoff(error->transcoder[i].power_domain_on));
16187                 err_printf(m, "  CONF: %08x\n", error->transcoder[i].conf);
16188                 err_printf(m, "  HTOTAL: %08x\n", error->transcoder[i].htotal);
16189                 err_printf(m, "  HBLANK: %08x\n", error->transcoder[i].hblank);
16190                 err_printf(m, "  HSYNC: %08x\n", error->transcoder[i].hsync);
16191                 err_printf(m, "  VTOTAL: %08x\n", error->transcoder[i].vtotal);
16192                 err_printf(m, "  VBLANK: %08x\n", error->transcoder[i].vblank);
16193                 err_printf(m, "  VSYNC: %08x\n", error->transcoder[i].vsync);
16194         }
16195 }
16196
16197 #endif