drivers/gpu/drm/i915/intel_display.c

   1 /*
   2  * Copyright © 2006-2007 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  21  * DEALINGS IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *      Eric Anholt <eric@anholt.net>
  25  */
  26
  27 #include <linux/module.h>
  28 #include <linux/input.h>
  29 #include <linux/i2c.h>
  30 #include <linux/kernel.h>
  31 #include <linux/slab.h>
  32 #include <linux/vgaarb.h>
  33 #include <drm/drm_edid.h>
  34 #include <drm/drmP.h>
  35 #include "intel_drv.h"
  36 #include "intel_frontbuffer.h"
  37 #include <drm/i915_drm.h>
  38 #include "i915_drv.h"
  39 #include "i915_gem_clflush.h"
  40 #include "intel_dsi.h"
  41 #include "i915_trace.h"
  42 #include <drm/drm_atomic.h>
  43 #include <drm/drm_atomic_helper.h>
  44 #include <drm/drm_dp_helper.h>
  45 #include <drm/drm_crtc_helper.h>
  46 #include <drm/drm_plane_helper.h>
  47 #include <drm/drm_rect.h>
  48 #include <drm/drm_atomic_uapi.h>
  49 #include <linux/intel-iommu.h>
  50 #include <linux/reservation.h>
  51
  52 /* Primary plane formats for gen <= 3 */
  53 static const uint32_t i8xx_primary_formats[] = {
  54         DRM_FORMAT_C8,
  55         DRM_FORMAT_RGB565,
  56         DRM_FORMAT_XRGB1555,
  57         DRM_FORMAT_XRGB8888,
  58 };
  59
  60 /* Primary plane formats for gen >= 4 */
  61 static const uint32_t i965_primary_formats[] = {
  62         DRM_FORMAT_C8,
  63         DRM_FORMAT_RGB565,
  64         DRM_FORMAT_XRGB8888,
  65         DRM_FORMAT_XBGR8888,
  66         DRM_FORMAT_XRGB2101010,
  67         DRM_FORMAT_XBGR2101010,
  68 };
  69
  70 static const uint64_t i9xx_format_modifiers[] = {
  71         I915_FORMAT_MOD_X_TILED,
  72         DRM_FORMAT_MOD_LINEAR,
  73         DRM_FORMAT_MOD_INVALID
  74 };
  75
  76 /* Cursor formats */
  77 static const uint32_t intel_cursor_formats[] = {
  78         DRM_FORMAT_ARGB8888,
  79 };
  80
  81 static const uint64_t cursor_format_modifiers[] = {
  82         DRM_FORMAT_MOD_LINEAR,
  83         DRM_FORMAT_MOD_INVALID
  84 };
  85
  86 static void i9xx_crtc_clock_get(struct intel_crtc *crtc,
  87                                 struct intel_crtc_state *pipe_config);
  88 static void ironlake_pch_clock_get(struct intel_crtc *crtc,
  89                                    struct intel_crtc_state *pipe_config);
  90
  91 static int intel_framebuffer_init(struct intel_framebuffer *ifb,
  92                                   struct drm_i915_gem_object *obj,
  93                                   struct drm_mode_fb_cmd2 *mode_cmd);
  94 static void intel_set_pipe_timings(const struct intel_crtc_state *crtc_state);
  95 static void intel_set_pipe_src_size(const struct intel_crtc_state *crtc_state);
  96 static void intel_cpu_transcoder_set_m_n(const struct intel_crtc_state *crtc_state,
  97                                          const struct intel_link_m_n *m_n,
  98                                          const struct intel_link_m_n *m2_n2);
  99 static void i9xx_set_pipeconf(const struct intel_crtc_state *crtc_state);
 100 static void ironlake_set_pipeconf(const struct intel_crtc_state *crtc_state);
 101 static void haswell_set_pipeconf(const struct intel_crtc_state *crtc_state);
 102 static void haswell_set_pipemisc(const struct intel_crtc_state *crtc_state);
 103 static void vlv_prepare_pll(struct intel_crtc *crtc,
 104                             const struct intel_crtc_state *pipe_config);
 105 static void chv_prepare_pll(struct intel_crtc *crtc,
 106                             const struct intel_crtc_state *pipe_config);
 107 static void intel_begin_crtc_commit(struct drm_crtc *, struct drm_crtc_state *);
 108 static void intel_finish_crtc_commit(struct drm_crtc *, struct drm_crtc_state *);
 109 static void intel_crtc_init_scalers(struct intel_crtc *crtc,
 110                                     struct intel_crtc_state *crtc_state);
 111 static void skylake_pfit_enable(const struct intel_crtc_state *crtc_state);
 112 static void ironlake_pfit_disable(const struct intel_crtc_state *old_crtc_state);
 113 static void ironlake_pfit_enable(const struct intel_crtc_state *crtc_state);
 114 static void intel_modeset_setup_hw_state(struct drm_device *dev,
 115                                          struct drm_modeset_acquire_ctx *ctx);
 116 static void intel_pre_disable_primary_noatomic(struct drm_crtc *crtc);
 117
 118 struct intel_limit {
 119         struct {
 120                 int min, max;
 121         } dot, vco, n, m, m1, m2, p, p1;
 122
 123         struct {
 124                 int dot_limit;
 125                 int p2_slow, p2_fast;
 126         } p2;
 127 };
 128
 129 /* returns HPLL frequency in kHz */
 130 int vlv_get_hpll_vco(struct drm_i915_private *dev_priv)
 131 {
 132         int hpll_freq, vco_freq[] = { 800, 1600, 2000, 2400 };
 133
 134         /* Obtain SKU information */
 135         mutex_lock(&dev_priv->sb_lock);
 136         hpll_freq = vlv_cck_read(dev_priv, CCK_FUSE_REG) &
 137                 CCK_FUSE_HPLL_FREQ_MASK;
 138         mutex_unlock(&dev_priv->sb_lock);
 139
 140         return vco_freq[hpll_freq] * 1000;
 141 }
 142
 143 int vlv_get_cck_clock(struct drm_i915_private *dev_priv,
 144                       const char *name, u32 reg, int ref_freq)
 145 {
 146         u32 val;
 147         int divider;
 148
 149         mutex_lock(&dev_priv->sb_lock);
 150         val = vlv_cck_read(dev_priv, reg);
 151         mutex_unlock(&dev_priv->sb_lock);
 152
 153         divider = val & CCK_FREQUENCY_VALUES;
 154
 155         WARN((val & CCK_FREQUENCY_STATUS) !=
 156              (divider << CCK_FREQUENCY_STATUS_SHIFT),
 157              "%s change in progress\n", name);
 158
 159         return DIV_ROUND_CLOSEST(ref_freq << 1, divider + 1);
 160 }
 161
 162 int vlv_get_cck_clock_hpll(struct drm_i915_private *dev_priv,
 163                            const char *name, u32 reg)
 164 {
 165         if (dev_priv->hpll_freq == 0)
 166                 dev_priv->hpll_freq = vlv_get_hpll_vco(dev_priv);
 167
 168         return vlv_get_cck_clock(dev_priv, name, reg,
 169                                  dev_priv->hpll_freq);
 170 }
 171
 172 static void intel_update_czclk(struct drm_i915_private *dev_priv)
 173 {
 174         if (!(IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)))
 175                 return;
 176
 177         dev_priv->czclk_freq = vlv_get_cck_clock_hpll(dev_priv, "czclk",
 178                                                       CCK_CZ_CLOCK_CONTROL);
 179
 180         DRM_DEBUG_DRIVER("CZ clock rate: %d kHz\n", dev_priv->czclk_freq);
 181 }
 182
 183 static inline u32 /* units of 100MHz */
 184 intel_fdi_link_freq(struct drm_i915_private *dev_priv,
 185                     const struct intel_crtc_state *pipe_config)
 186 {
 187         if (HAS_DDI(dev_priv))
 188                 return pipe_config->port_clock; /* SPLL */
 189         else
 190                 return dev_priv->fdi_pll_freq;
 191 }
 192
 193 static const struct intel_limit intel_limits_i8xx_dac = {
 194         .dot = { .min = 25000, .max = 350000 },
 195         .vco = { .min = 908000, .max = 1512000 },
 196         .n = { .min = 2, .max = 16 },
 197         .m = { .min = 96, .max = 140 },
 198         .m1 = { .min = 18, .max = 26 },
 199         .m2 = { .min = 6, .max = 16 },
 200         .p = { .min = 4, .max = 128 },
 201         .p1 = { .min = 2, .max = 33 },
 202         .p2 = { .dot_limit = 165000,
 203                 .p2_slow = 4, .p2_fast = 2 },
 204 };
 205
 206 static const struct intel_limit intel_limits_i8xx_dvo = {
 207         .dot = { .min = 25000, .max = 350000 },
 208         .vco = { .min = 908000, .max = 1512000 },
 209         .n = { .min = 2, .max = 16 },
 210         .m = { .min = 96, .max = 140 },
 211         .m1 = { .min = 18, .max = 26 },
 212         .m2 = { .min = 6, .max = 16 },
 213         .p = { .min = 4, .max = 128 },
 214         .p1 = { .min = 2, .max = 33 },
 215         .p2 = { .dot_limit = 165000,
 216                 .p2_slow = 4, .p2_fast = 4 },
 217 };
 218
 219 static const struct intel_limit intel_limits_i8xx_lvds = {
 220         .dot = { .min = 25000, .max = 350000 },
 221         .vco = { .min = 908000, .max = 1512000 },
 222         .n = { .min = 2, .max = 16 },
 223         .m = { .min = 96, .max = 140 },
 224         .m1 = { .min = 18, .max = 26 },
 225         .m2 = { .min = 6, .max = 16 },
 226         .p = { .min = 4, .max = 128 },
 227         .p1 = { .min = 1, .max = 6 },
 228         .p2 = { .dot_limit = 165000,
 229                 .p2_slow = 14, .p2_fast = 7 },
 230 };
 231
 232 static const struct intel_limit intel_limits_i9xx_sdvo = {
 233         .dot = { .min = 20000, .max = 400000 },
 234         .vco = { .min = 1400000, .max = 2800000 },
 235         .n = { .min = 1, .max = 6 },
 236         .m = { .min = 70, .max = 120 },
 237         .m1 = { .min = 8, .max = 18 },
 238         .m2 = { .min = 3, .max = 7 },
 239         .p = { .min = 5, .max = 80 },
 240         .p1 = { .min = 1, .max = 8 },
 241         .p2 = { .dot_limit = 200000,
 242                 .p2_slow = 10, .p2_fast = 5 },
 243 };
 244
 245 static const struct intel_limit intel_limits_i9xx_lvds = {
 246         .dot = { .min = 20000, .max = 400000 },
 247         .vco = { .min = 1400000, .max = 2800000 },
 248         .n = { .min = 1, .max = 6 },
 249         .m = { .min = 70, .max = 120 },
 250         .m1 = { .min = 8, .max = 18 },
 251         .m2 = { .min = 3, .max = 7 },
 252         .p = { .min = 7, .max = 98 },
 253         .p1 = { .min = 1, .max = 8 },
 254         .p2 = { .dot_limit = 112000,
 255                 .p2_slow = 14, .p2_fast = 7 },
 256 };
 257
 258
 259 static const struct intel_limit intel_limits_g4x_sdvo = {
 260         .dot = { .min = 25000, .max = 270000 },
 261         .vco = { .min = 1750000, .max = 3500000},
 262         .n = { .min = 1, .max = 4 },
 263         .m = { .min = 104, .max = 138 },
 264         .m1 = { .min = 17, .max = 23 },
 265         .m2 = { .min = 5, .max = 11 },
 266         .p = { .min = 10, .max = 30 },
 267         .p1 = { .min = 1, .max = 3},
 268         .p2 = { .dot_limit = 270000,
 269                 .p2_slow = 10,
 270                 .p2_fast = 10
 271         },
 272 };
 273
 274 static const struct intel_limit intel_limits_g4x_hdmi = {
 275         .dot = { .min = 22000, .max = 400000 },
 276         .vco = { .min = 1750000, .max = 3500000},
 277         .n = { .min = 1, .max = 4 },
 278         .m = { .min = 104, .max = 138 },
 279         .m1 = { .min = 16, .max = 23 },
 280         .m2 = { .min = 5, .max = 11 },
 281         .p = { .min = 5, .max = 80 },
 282         .p1 = { .min = 1, .max = 8},
 283         .p2 = { .dot_limit = 165000,
 284                 .p2_slow = 10, .p2_fast = 5 },
 285 };
 286
 287 static const struct intel_limit intel_limits_g4x_single_channel_lvds = {
 288         .dot = { .min = 20000, .max = 115000 },
 289         .vco = { .min = 1750000, .max = 3500000 },
 290         .n = { .min = 1, .max = 3 },
 291         .m = { .min = 104, .max = 138 },
 292         .m1 = { .min = 17, .max = 23 },
 293         .m2 = { .min = 5, .max = 11 },
 294         .p = { .min = 28, .max = 112 },
 295         .p1 = { .min = 2, .max = 8 },
 296         .p2 = { .dot_limit = 0,
 297                 .p2_slow = 14, .p2_fast = 14
 298         },
 299 };
 300
 301 static const struct intel_limit intel_limits_g4x_dual_channel_lvds = {
 302         .dot = { .min = 80000, .max = 224000 },
 303         .vco = { .min = 1750000, .max = 3500000 },
 304         .n = { .min = 1, .max = 3 },
 305         .m = { .min = 104, .max = 138 },
 306         .m1 = { .min = 17, .max = 23 },
 307         .m2 = { .min = 5, .max = 11 },
 308         .p = { .min = 14, .max = 42 },
 309         .p1 = { .min = 2, .max = 6 },
 310         .p2 = { .dot_limit = 0,
 311                 .p2_slow = 7, .p2_fast = 7
 312         },
 313 };
 314
 315 static const struct intel_limit intel_limits_pineview_sdvo = {
 316         .dot = { .min = 20000, .max = 400000},
 317         .vco = { .min = 1700000, .max = 3500000 },
 318         /* Pineview's Ncounter is a ring counter */
 319         .n = { .min = 3, .max = 6 },
 320         .m = { .min = 2, .max = 256 },
 321         /* Pineview only has one combined m divider, which we treat as m2. */
 322         .m1 = { .min = 0, .max = 0 },
 323         .m2 = { .min = 0, .max = 254 },
 324         .p = { .min = 5, .max = 80 },
 325         .p1 = { .min = 1, .max = 8 },
 326         .p2 = { .dot_limit = 200000,
 327                 .p2_slow = 10, .p2_fast = 5 },
 328 };
 329
 330 static const struct intel_limit intel_limits_pineview_lvds = {
 331         .dot = { .min = 20000, .max = 400000 },
 332         .vco = { .min = 1700000, .max = 3500000 },
 333         .n = { .min = 3, .max = 6 },
 334         .m = { .min = 2, .max = 256 },
 335         .m1 = { .min = 0, .max = 0 },
 336         .m2 = { .min = 0, .max = 254 },
 337         .p = { .min = 7, .max = 112 },
 338         .p1 = { .min = 1, .max = 8 },
 339         .p2 = { .dot_limit = 112000,
 340                 .p2_slow = 14, .p2_fast = 14 },
 341 };
 342
 343 /* Ironlake / Sandybridge
 344  *
 345  * We calculate clock using (register_value + 2) for N/M1/M2, so here
 346  * the range value for them is (actual_value - 2).
 347  */
 348 static const struct intel_limit intel_limits_ironlake_dac = {
 349         .dot = { .min = 25000, .max = 350000 },
 350         .vco = { .min = 1760000, .max = 3510000 },
 351         .n = { .min = 1, .max = 5 },
 352         .m = { .min = 79, .max = 127 },
 353         .m1 = { .min = 12, .max = 22 },
 354         .m2 = { .min = 5, .max = 9 },
 355         .p = { .min = 5, .max = 80 },
 356         .p1 = { .min = 1, .max = 8 },
 357         .p2 = { .dot_limit = 225000,
 358                 .p2_slow = 10, .p2_fast = 5 },
 359 };
 360
 361 static const struct intel_limit intel_limits_ironlake_single_lvds = {
 362         .dot = { .min = 25000, .max = 350000 },
 363         .vco = { .min = 1760000, .max = 3510000 },
 364         .n = { .min = 1, .max = 3 },
 365         .m = { .min = 79, .max = 118 },
 366         .m1 = { .min = 12, .max = 22 },
 367         .m2 = { .min = 5, .max = 9 },
 368         .p = { .min = 28, .max = 112 },
 369         .p1 = { .min = 2, .max = 8 },
 370         .p2 = { .dot_limit = 225000,
 371                 .p2_slow = 14, .p2_fast = 14 },
 372 };
 373
 374 static const struct intel_limit intel_limits_ironlake_dual_lvds = {
 375         .dot = { .min = 25000, .max = 350000 },
 376         .vco = { .min = 1760000, .max = 3510000 },
 377         .n = { .min = 1, .max = 3 },
 378         .m = { .min = 79, .max = 127 },
 379         .m1 = { .min = 12, .max = 22 },
 380         .m2 = { .min = 5, .max = 9 },
 381         .p = { .min = 14, .max = 56 },
 382         .p1 = { .min = 2, .max = 8 },
 383         .p2 = { .dot_limit = 225000,
 384                 .p2_slow = 7, .p2_fast = 7 },
 385 };
 386
 387 /* LVDS 100mhz refclk limits. */
 388 static const struct intel_limit intel_limits_ironlake_single_lvds_100m = {
 389         .dot = { .min = 25000, .max = 350000 },
 390         .vco = { .min = 1760000, .max = 3510000 },
 391         .n = { .min = 1, .max = 2 },
 392         .m = { .min = 79, .max = 126 },
 393         .m1 = { .min = 12, .max = 22 },
 394         .m2 = { .min = 5, .max = 9 },
 395         .p = { .min = 28, .max = 112 },
 396         .p1 = { .min = 2, .max = 8 },
 397         .p2 = { .dot_limit = 225000,
 398                 .p2_slow = 14, .p2_fast = 14 },
 399 };
 400
 401 static const struct intel_limit intel_limits_ironlake_dual_lvds_100m = {
 402         .dot = { .min = 25000, .max = 350000 },
 403         .vco = { .min = 1760000, .max = 3510000 },
 404         .n = { .min = 1, .max = 3 },
 405         .m = { .min = 79, .max = 126 },
 406         .m1 = { .min = 12, .max = 22 },
 407         .m2 = { .min = 5, .max = 9 },
 408         .p = { .min = 14, .max = 42 },
 409         .p1 = { .min = 2, .max = 6 },
 410         .p2 = { .dot_limit = 225000,
 411                 .p2_slow = 7, .p2_fast = 7 },
 412 };
 413
 414 static const struct intel_limit intel_limits_vlv = {
 415          /*
 416           * These are the data rate limits (measured in fast clocks)
 417           * since those are the strictest limits we have. The fast
 418           * clock and actual rate limits are more relaxed, so checking
 419           * them would make no difference.
 420           */
 421         .dot = { .min = 25000 * 5, .max = 270000 * 5 },
 422         .vco = { .min = 4000000, .max = 6000000 },
 423         .n = { .min = 1, .max = 7 },
 424         .m1 = { .min = 2, .max = 3 },
 425         .m2 = { .min = 11, .max = 156 },
 426         .p1 = { .min = 2, .max = 3 },
 427         .p2 = { .p2_slow = 2, .p2_fast = 20 }, /* slow=min, fast=max */
 428 };
 429
 430 static const struct intel_limit intel_limits_chv = {
 431         /*
 432          * These are the data rate limits (measured in fast clocks)
 433          * since those are the strictest limits we have.  The fast
 434          * clock and actual rate limits are more relaxed, so checking
 435          * them would make no difference.
 436          */
 437         .dot = { .min = 25000 * 5, .max = 540000 * 5},
 438         .vco = { .min = 4800000, .max = 6480000 },
 439         .n = { .min = 1, .max = 1 },
 440         .m1 = { .min = 2, .max = 2 },
 441         .m2 = { .min = 24 << 22, .max = 175 << 22 },
 442         .p1 = { .min = 2, .max = 4 },
 443         .p2 = { .p2_slow = 1, .p2_fast = 14 },
 444 };
 445
 446 static const struct intel_limit intel_limits_bxt = {
 447         /* FIXME: find real dot limits */
 448         .dot = { .min = 0, .max = INT_MAX },
 449         .vco = { .min = 4800000, .max = 6700000 },
 450         .n = { .min = 1, .max = 1 },
 451         .m1 = { .min = 2, .max = 2 },
 452         /* FIXME: find real m2 limits */
 453         .m2 = { .min = 2 << 22, .max = 255 << 22 },
 454         .p1 = { .min = 2, .max = 4 },
 455         .p2 = { .p2_slow = 1, .p2_fast = 20 },
 456 };
 457
 458 static void
 459 skl_wa_clkgate(struct drm_i915_private *dev_priv, int pipe, bool enable)
 460 {
 461         if (enable)
 462                 I915_WRITE(CLKGATE_DIS_PSL(pipe),
 463                            DUPS1_GATING_DIS | DUPS2_GATING_DIS);
 464         else
 465                 I915_WRITE(CLKGATE_DIS_PSL(pipe),
 466                            I915_READ(CLKGATE_DIS_PSL(pipe)) &
 467                            ~(DUPS1_GATING_DIS | DUPS2_GATING_DIS));
 468 }
 469
 470 static bool
 471 needs_modeset(const struct drm_crtc_state *state)
 472 {
 473         return drm_atomic_crtc_needs_modeset(state);
 474 }
 475
 476 /*
 477  * Platform specific helpers to calculate the port PLL loopback- (clock.m),
 478  * and post-divider (clock.p) values, pre- (clock.vco) and post-divided fast
 479  * (clock.dot) clock rates. This fast dot clock is fed to the port's IO logic.
 480  * The helpers' return value is the rate of the clock that is fed to the
 481  * display engine's pipe which can be the above fast dot clock rate or a
 482  * divided-down version of it.
 483  */
 484 /* m1 is reserved as 0 in Pineview, n is a ring counter */
 485 static int pnv_calc_dpll_params(int refclk, struct dpll *clock)
 486 {
 487         clock->m = clock->m2 + 2;
 488         clock->p = clock->p1 * clock->p2;
 489         if (WARN_ON(clock->n == 0 || clock->p == 0))
 490                 return 0;
 491         clock->vco = DIV_ROUND_CLOSEST(refclk * clock->m, clock->n);
 492         clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p);
 493
 494         return clock->dot;
 495 }
 496
 497 static uint32_t i9xx_dpll_compute_m(struct dpll *dpll)
 498 {
 499         return 5 * (dpll->m1 + 2) + (dpll->m2 + 2);
 500 }
 501
 502 static int i9xx_calc_dpll_params(int refclk, struct dpll *clock)
 503 {
 504         clock->m = i9xx_dpll_compute_m(clock);
 505         clock->p = clock->p1 * clock->p2;
 506         if (WARN_ON(clock->n + 2 == 0 || clock->p == 0))
 507                 return 0;
 508         clock->vco = DIV_ROUND_CLOSEST(refclk * clock->m, clock->n + 2);
 509         clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p);
 510
 511         return clock->dot;
 512 }
 513
 514 static int vlv_calc_dpll_params(int refclk, struct dpll *clock)
 515 {
 516         clock->m = clock->m1 * clock->m2;
 517         clock->p = clock->p1 * clock->p2;
 518         if (WARN_ON(clock->n == 0 || clock->p == 0))
 519                 return 0;
 520         clock->vco = DIV_ROUND_CLOSEST(refclk * clock->m, clock->n);
 521         clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p);
 522
 523         return clock->dot / 5;
 524 }
 525
 526 int chv_calc_dpll_params(int refclk, struct dpll *clock)
 527 {
 528         clock->m = clock->m1 * clock->m2;
 529         clock->p = clock->p1 * clock->p2;
 530         if (WARN_ON(clock->n == 0 || clock->p == 0))
 531                 return 0;
 532         clock->vco = DIV_ROUND_CLOSEST_ULL((uint64_t)refclk * clock->m,
 533                         clock->n << 22);
 534         clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p);
 535
 536         return clock->dot / 5;
 537 }
 538
 539 #define INTELPllInvalid(s)   do { /* DRM_DEBUG(s); */ return false; } while (0)
 540
 541 /*
 542  * Returns whether the given set of divisors are valid for a given refclk with
 543  * the given connectors.
 544  */
 545 static bool intel_PLL_is_valid(struct drm_i915_private *dev_priv,
 546                                const struct intel_limit *limit,
 547                                const struct dpll *clock)
 548 {
 549         if (clock->n   < limit->n.min   || limit->n.max   < clock->n)
 550                 INTELPllInvalid("n out of range\n");
 551         if (clock->p1  < limit->p1.min  || limit->p1.max  < clock->p1)
 552                 INTELPllInvalid("p1 out of range\n");
 553         if (clock->m2  < limit->m2.min  || limit->m2.max  < clock->m2)
 554                 INTELPllInvalid("m2 out of range\n");
 555         if (clock->m1  < limit->m1.min  || limit->m1.max  < clock->m1)
 556                 INTELPllInvalid("m1 out of range\n");
 557
 558         if (!IS_PINEVIEW(dev_priv) && !IS_VALLEYVIEW(dev_priv) &&
 559             !IS_CHERRYVIEW(dev_priv) && !IS_GEN9_LP(dev_priv))
 560                 if (clock->m1 <= clock->m2)
 561                         INTELPllInvalid("m1 <= m2\n");
 562
 563         if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv) &&
 564             !IS_GEN9_LP(dev_priv)) {
 565                 if (clock->p < limit->p.min || limit->p.max < clock->p)
 566                         INTELPllInvalid("p out of range\n");
 567                 if (clock->m < limit->m.min || limit->m.max < clock->m)
 568                         INTELPllInvalid("m out of range\n");
 569         }
 570
 571         if (clock->vco < limit->vco.min || limit->vco.max < clock->vco)
 572                 INTELPllInvalid("vco out of range\n");
 573         /* XXX: We may need to be checking "Dot clock" depending on the multiplier,
 574          * connector, etc., rather than just a single range.
 575          */
 576         if (clock->dot < limit->dot.min || limit->dot.max < clock->dot)
 577                 INTELPllInvalid("dot out of range\n");
 578
 579         return true;
 580 }
 581
 582 static int
 583 i9xx_select_p2_div(const struct intel_limit *limit,
 584                    const struct intel_crtc_state *crtc_state,
 585                    int target)
 586 {
 587         struct drm_device *dev = crtc_state->base.crtc->dev;
 588
 589         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
 590                 /*
 591                  * For LVDS just rely on its current settings for dual-channel.
 592                  * We haven't figured out how to reliably set up different
 593                  * single/dual channel state, if we even can.
 594                  */
 595                 if (intel_is_dual_link_lvds(dev))
 596                         return limit->p2.p2_fast;
 597                 else
 598                         return limit->p2.p2_slow;
 599         } else {
 600                 if (target < limit->p2.dot_limit)
 601                         return limit->p2.p2_slow;
 602                 else
 603                         return limit->p2.p2_fast;
 604         }
 605 }
 606
 607 /*
 608  * Returns a set of divisors for the desired target clock with the given
 609  * refclk, or FALSE.  The returned values represent the clock equation:
 610  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
 611  *
 612  * Target and reference clocks are specified in kHz.
 613  *
 614  * If match_clock is provided, then best_clock P divider must match the P
 615  * divider from @match_clock used for LVDS downclocking.
 616  */
 617 static bool
 618 i9xx_find_best_dpll(const struct intel_limit *limit,
 619                     struct intel_crtc_state *crtc_state,
 620                     int target, int refclk, struct dpll *match_clock,
 621                     struct dpll *best_clock)
 622 {
 623         struct drm_device *dev = crtc_state->base.crtc->dev;
 624         struct dpll clock;
 625         int err = target;
 626
 627         memset(best_clock, 0, sizeof(*best_clock));
 628
 629         clock.p2 = i9xx_select_p2_div(limit, crtc_state, target);
 630
 631         for (clock.m1 = limit->m1.min; clock.m1 <= limit->m1.max;
 632              clock.m1++) {
 633                 for (clock.m2 = limit->m2.min;
 634                      clock.m2 <= limit->m2.max; clock.m2++) {
 635                         if (clock.m2 >= clock.m1)
 636                                 break;
 637                         for (clock.n = limit->n.min;
 638                              clock.n <= limit->n.max; clock.n++) {
 639                                 for (clock.p1 = limit->p1.min;
 640                                         clock.p1 <= limit->p1.max; clock.p1++) {
 641                                         int this_err;
 642
 643                                         i9xx_calc_dpll_params(refclk, &clock);
 644                                         if (!intel_PLL_is_valid(to_i915(dev),
 645                                                                 limit,
 646                                                                 &clock))
 647                                                 continue;
 648                                         if (match_clock &&
 649                                             clock.p != match_clock->p)
 650                                                 continue;
 651
 652                                         this_err = abs(clock.dot - target);
 653                                         if (this_err < err) {
 654                                                 *best_clock = clock;
 655                                                 err = this_err;
 656                                         }
 657                                 }
 658                         }
 659                 }
 660         }
 661
 662         return (err != target);
 663 }
 664
 665 /*
 666  * Returns a set of divisors for the desired target clock with the given
 667  * refclk, or FALSE.  The returned values represent the clock equation:
 668  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
 669  *
 670  * Target and reference clocks are specified in kHz.
 671  *
 672  * If match_clock is provided, then best_clock P divider must match the P
 673  * divider from @match_clock used for LVDS downclocking.
 674  */
 675 static bool
 676 pnv_find_best_dpll(const struct intel_limit *limit,
 677                    struct intel_crtc_state *crtc_state,
 678                    int target, int refclk, struct dpll *match_clock,
 679                    struct dpll *best_clock)
 680 {
 681         struct drm_device *dev = crtc_state->base.crtc->dev;
 682         struct dpll clock;
 683         int err = target;
 684
 685         memset(best_clock, 0, sizeof(*best_clock));
 686
 687         clock.p2 = i9xx_select_p2_div(limit, crtc_state, target);
 688
 689         for (clock.m1 = limit->m1.min; clock.m1 <= limit->m1.max;
 690              clock.m1++) {
 691                 for (clock.m2 = limit->m2.min;
 692                      clock.m2 <= limit->m2.max; clock.m2++) {
 693                         for (clock.n = limit->n.min;
 694                              clock.n <= limit->n.max; clock.n++) {
 695                                 for (clock.p1 = limit->p1.min;
 696                                         clock.p1 <= limit->p1.max; clock.p1++) {
 697                                         int this_err;
 698
 699                                         pnv_calc_dpll_params(refclk, &clock);
 700                                         if (!intel_PLL_is_valid(to_i915(dev),
 701                                                                 limit,
 702                                                                 &clock))
 703                                                 continue;
 704                                         if (match_clock &&
 705                                             clock.p != match_clock->p)
 706                                                 continue;
 707
 708                                         this_err = abs(clock.dot - target);
 709                                         if (this_err < err) {
 710                                                 *best_clock = clock;
 711                                                 err = this_err;
 712                                         }
 713                                 }
 714                         }
 715                 }
 716         }
 717
 718         return (err != target);
 719 }
 720
 721 /*
 722  * Returns a set of divisors for the desired target clock with the given
 723  * refclk, or FALSE.  The returned values represent the clock equation:
 724  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
 725  *
 726  * Target and reference clocks are specified in kHz.
 727  *
 728  * If match_clock is provided, then best_clock P divider must match the P
 729  * divider from @match_clock used for LVDS downclocking.
 730  */
 731 static bool
 732 g4x_find_best_dpll(const struct intel_limit *limit,
 733                    struct intel_crtc_state *crtc_state,
 734                    int target, int refclk, struct dpll *match_clock,
 735                    struct dpll *best_clock)
 736 {
 737         struct drm_device *dev = crtc_state->base.crtc->dev;
 738         struct dpll clock;
 739         int max_n;
 740         bool found = false;
 741         /* approximately equals target * 0.00585 */
 742         int err_most = (target >> 8) + (target >> 9);
 743
 744         memset(best_clock, 0, sizeof(*best_clock));
 745
 746         clock.p2 = i9xx_select_p2_div(limit, crtc_state, target);
 747
 748         max_n = limit->n.max;
 749         /* based on hardware requirement, prefer smaller n to precision */
 750         for (clock.n = limit->n.min; clock.n <= max_n; clock.n++) {
 751                 /* based on hardware requirement, prefere larger m1,m2 */
 752                 for (clock.m1 = limit->m1.max;
 753                      clock.m1 >= limit->m1.min; clock.m1--) {
 754                         for (clock.m2 = limit->m2.max;
 755                              clock.m2 >= limit->m2.min; clock.m2--) {
 756                                 for (clock.p1 = limit->p1.max;
 757                                      clock.p1 >= limit->p1.min; clock.p1--) {
 758                                         int this_err;
 759
 760                                         i9xx_calc_dpll_params(refclk, &clock);
 761                                         if (!intel_PLL_is_valid(to_i915(dev),
 762                                                                 limit,
 763                                                                 &clock))
 764                                                 continue;
 765
 766                                         this_err = abs(clock.dot - target);
 767                                         if (this_err < err_most) {
 768                                                 *best_clock = clock;
 769                                                 err_most = this_err;
 770                                                 max_n = clock.n;
 771                                                 found = true;
 772                                         }
 773                                 }
 774                         }
 775                 }
 776         }
 777         return found;
 778 }
 779
 780 /*
 781  * Check if the calculated PLL configuration is more optimal compared to the
 782  * best configuration and error found so far. Return the calculated error.
 783  */
 784 static bool vlv_PLL_is_optimal(struct drm_device *dev, int target_freq,
 785                                const struct dpll *calculated_clock,
 786                                const struct dpll *best_clock,
 787                                unsigned int best_error_ppm,
 788                                unsigned int *error_ppm)
 789 {
 790         /*
 791          * For CHV ignore the error and consider only the P value.
 792          * Prefer a bigger P value based on HW requirements.
 793          */
 794         if (IS_CHERRYVIEW(to_i915(dev))) {
 795                 *error_ppm = 0;
 796
 797                 return calculated_clock->p > best_clock->p;
 798         }
 799
 800         if (WARN_ON_ONCE(!target_freq))
 801                 return false;
 802
 803         *error_ppm = div_u64(1000000ULL *
 804                                 abs(target_freq - calculated_clock->dot),
 805                              target_freq);
 806         /*
 807          * Prefer a better P value over a better (smaller) error if the error
 808          * is small. Ensure this preference for future configurations too by
 809          * setting the error to 0.
 810          */
 811         if (*error_ppm < 100 && calculated_clock->p > best_clock->p) {
 812                 *error_ppm = 0;
 813
 814                 return true;
 815         }
 816
 817         return *error_ppm + 10 < best_error_ppm;
 818 }
 819
 820 /*
 821  * Returns a set of divisors for the desired target clock with the given
 822  * refclk, or FALSE.  The returned values represent the clock equation:
 823  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
 824  */
 825 static bool
 826 vlv_find_best_dpll(const struct intel_limit *limit,
 827                    struct intel_crtc_state *crtc_state,
 828                    int target, int refclk, struct dpll *match_clock,
 829                    struct dpll *best_clock)
 830 {
 831         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
 832         struct drm_device *dev = crtc->base.dev;
 833         struct dpll clock;
 834         unsigned int bestppm = 1000000;
 835         /* min update 19.2 MHz */
 836         int max_n = min(limit->n.max, refclk / 19200);
 837         bool found = false;
 838
 839         target *= 5; /* fast clock */
 840
 841         memset(best_clock, 0, sizeof(*best_clock));
 842
 843         /* based on hardware requirement, prefer smaller n to precision */
 844         for (clock.n = limit->n.min; clock.n <= max_n; clock.n++) {
 845                 for (clock.p1 = limit->p1.max; clock.p1 >= limit->p1.min; clock.p1--) {
 846                         for (clock.p2 = limit->p2.p2_fast; clock.p2 >= limit->p2.p2_slow;
 847                              clock.p2 -= clock.p2 > 10 ? 2 : 1) {
 848                                 clock.p = clock.p1 * clock.p2;
 849                                 /* based on hardware requirement, prefer bigger m1,m2 values */
 850                                 for (clock.m1 = limit->m1.min; clock.m1 <= limit->m1.max; clock.m1++) {
 851                                         unsigned int ppm;
 852
 853                                         clock.m2 = DIV_ROUND_CLOSEST(target * clock.p * clock.n,
 854                                                                      refclk * clock.m1);
 855
 856                                         vlv_calc_dpll_params(refclk, &clock);
 857
 858                                         if (!intel_PLL_is_valid(to_i915(dev),
 859                                                                 limit,
 860                                                                 &clock))
 861                                                 continue;
 862
 863                                         if (!vlv_PLL_is_optimal(dev, target,
 864                                                                 &clock,
 865                                                                 best_clock,
 866                                                                 bestppm, &ppm))
 867                                                 continue;
 868
 869                                         *best_clock = clock;
 870                                         bestppm = ppm;
 871                                         found = true;
 872                                 }
 873                         }
 874                 }
 875         }
 876
 877         return found;
 878 }
 879
 880 /*
 881  * Returns a set of divisors for the desired target clock with the given
 882  * refclk, or FALSE.  The returned values represent the clock equation:
 883  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
 884  */
 885 static bool
 886 chv_find_best_dpll(const struct intel_limit *limit,
 887                    struct intel_crtc_state *crtc_state,
 888                    int target, int refclk, struct dpll *match_clock,
 889                    struct dpll *best_clock)
 890 {
 891         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
 892         struct drm_device *dev = crtc->base.dev;
 893         unsigned int best_error_ppm;
 894         struct dpll clock;
 895         uint64_t m2;
 896         int found = false;
 897
 898         memset(best_clock, 0, sizeof(*best_clock));
 899         best_error_ppm = 1000000;
 900
 901         /*
 902          * Based on hardware doc, the n always set to 1, and m1 always
 903          * set to 2.  If requires to support 200Mhz refclk, we need to
 904          * revisit this because n may not 1 anymore.
 905          */
 906         clock.n = 1, clock.m1 = 2;
 907         target *= 5;    /* fast clock */
 908
 909         for (clock.p1 = limit->p1.max; clock.p1 >= limit->p1.min; clock.p1--) {
 910                 for (clock.p2 = limit->p2.p2_fast;
 911                                 clock.p2 >= limit->p2.p2_slow;
 912                                 clock.p2 -= clock.p2 > 10 ? 2 : 1) {
 913                         unsigned int error_ppm;
 914
 915                         clock.p = clock.p1 * clock.p2;
 916
 917                         m2 = DIV_ROUND_CLOSEST_ULL(((uint64_t)target * clock.p *
 918                                         clock.n) << 22, refclk * clock.m1);
 919
 920                         if (m2 > INT_MAX/clock.m1)
 921                                 continue;
 922
 923                         clock.m2 = m2;
 924
 925                         chv_calc_dpll_params(refclk, &clock);
 926
 927                         if (!intel_PLL_is_valid(to_i915(dev), limit, &clock))
 928                                 continue;
 929
 930                         if (!vlv_PLL_is_optimal(dev, target, &clock, best_clock,
 931                                                 best_error_ppm, &error_ppm))
 932                                 continue;
 933
 934                         *best_clock = clock;
 935                         best_error_ppm = error_ppm;
 936                         found = true;
 937                 }
 938         }
 939
 940         return found;
 941 }
 942
 943 bool bxt_find_best_dpll(struct intel_crtc_state *crtc_state, int target_clock,
 944                         struct dpll *best_clock)
 945 {
 946         int refclk = 100000;
 947         const struct intel_limit *limit = &intel_limits_bxt;
 948
 949         return chv_find_best_dpll(limit, crtc_state,
 950                                   target_clock, refclk, NULL, best_clock);
 951 }
 952
 953 bool intel_crtc_active(struct intel_crtc *crtc)
 954 {
 955         /* Be paranoid as we can arrive here with only partial
 956          * state retrieved from the hardware during setup.
 957          *
 958          * We can ditch the adjusted_mode.crtc_clock check as soon
 959          * as Haswell has gained clock readout/fastboot support.
 960          *
 961          * We can ditch the crtc->primary->state->fb check as soon as we can
 962          * properly reconstruct framebuffers.
 963          *
 964          * FIXME: The intel_crtc->active here should be switched to
 965          * crtc->state->active once we have proper CRTC states wired up
 966          * for atomic.
 967          */
 968         return crtc->active && crtc->base.primary->state->fb &&
 969                 crtc->config->base.adjusted_mode.crtc_clock;
 970 }
 971
 972 enum transcoder intel_pipe_to_cpu_transcoder(struct drm_i915_private *dev_priv,
 973                                              enum pipe pipe)
 974 {
 975         struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
 976
 977         return crtc->config->cpu_transcoder;
 978 }
 979
 980 static bool pipe_scanline_is_moving(struct drm_i915_private *dev_priv,
 981                                     enum pipe pipe)
 982 {
 983         i915_reg_t reg = PIPEDSL(pipe);
 984         u32 line1, line2;
 985         u32 line_mask;
 986
 987         if (IS_GEN2(dev_priv))
 988                 line_mask = DSL_LINEMASK_GEN2;
 989         else
 990                 line_mask = DSL_LINEMASK_GEN3;
 991
 992         line1 = I915_READ(reg) & line_mask;
 993         msleep(5);
 994         line2 = I915_READ(reg) & line_mask;
 995
 996         return line1 != line2;
 997 }
 998
 999 static void wait_for_pipe_scanline_moving(struct intel_crtc *crtc, bool state)
1000 {
1001         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1002         enum pipe pipe = crtc->pipe;
1003
1004         /* Wait for the display line to settle/start moving */
1005         if (wait_for(pipe_scanline_is_moving(dev_priv, pipe) == state, 100))
1006                 DRM_ERROR("pipe %c scanline %s wait timed out\n",
1007                           pipe_name(pipe), onoff(state));
1008 }
1009
1010 static void intel_wait_for_pipe_scanline_stopped(struct intel_crtc *crtc)
1011 {
1012         wait_for_pipe_scanline_moving(crtc, false);
1013 }
1014
1015 static void intel_wait_for_pipe_scanline_moving(struct intel_crtc *crtc)
1016 {
1017         wait_for_pipe_scanline_moving(crtc, true);
1018 }
1019
1020 static void
1021 intel_wait_for_pipe_off(const struct intel_crtc_state *old_crtc_state)
1022 {
1023         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc);
1024         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1025
1026         if (INTEL_GEN(dev_priv) >= 4) {
1027                 enum transcoder cpu_transcoder = old_crtc_state->cpu_transcoder;
1028                 i915_reg_t reg = PIPECONF(cpu_transcoder);
1029
1030                 /* Wait for the Pipe State to go off */
1031                 if (intel_wait_for_register(dev_priv,
1032                                             reg, I965_PIPECONF_ACTIVE, 0,
1033                                             100))
1034                         WARN(1, "pipe_off wait timed out\n");
1035         } else {
1036                 intel_wait_for_pipe_scanline_stopped(crtc);
1037         }
1038 }
1039
1040 /* Only for pre-ILK configs */
1041 void assert_pll(struct drm_i915_private *dev_priv,
1042                 enum pipe pipe, bool state)
1043 {
1044         u32 val;
1045         bool cur_state;
1046
1047         val = I915_READ(DPLL(pipe));
1048         cur_state = !!(val & DPLL_VCO_ENABLE);
1049         I915_STATE_WARN(cur_state != state,
1050              "PLL state assertion failure (expected %s, current %s)\n",
1051                         onoff(state), onoff(cur_state));
1052 }
1053
1054 /* XXX: the dsi pll is shared between MIPI DSI ports */
1055 void assert_dsi_pll(struct drm_i915_private *dev_priv, bool state)
1056 {
1057         u32 val;
1058         bool cur_state;
1059
1060         mutex_lock(&dev_priv->sb_lock);
1061         val = vlv_cck_read(dev_priv, CCK_REG_DSI_PLL_CONTROL);
1062         mutex_unlock(&dev_priv->sb_lock);
1063
1064         cur_state = val & DSI_PLL_VCO_EN;
1065         I915_STATE_WARN(cur_state != state,
1066              "DSI PLL state assertion failure (expected %s, current %s)\n",
1067                         onoff(state), onoff(cur_state));
1068 }
1069
1070 static void assert_fdi_tx(struct drm_i915_private *dev_priv,
1071                           enum pipe pipe, bool state)
1072 {
1073         bool cur_state;
1074         enum transcoder cpu_transcoder = intel_pipe_to_cpu_transcoder(dev_priv,
1075                                                                       pipe);
1076
1077         if (HAS_DDI(dev_priv)) {
1078                 /* DDI does not have a specific FDI_TX register */
1079                 u32 val = I915_READ(TRANS_DDI_FUNC_CTL(cpu_transcoder));
1080                 cur_state = !!(val & TRANS_DDI_FUNC_ENABLE);
1081         } else {
1082                 u32 val = I915_READ(FDI_TX_CTL(pipe));
1083                 cur_state = !!(val & FDI_TX_ENABLE);
1084         }
1085         I915_STATE_WARN(cur_state != state,
1086              "FDI TX state assertion failure (expected %s, current %s)\n",
1087                         onoff(state), onoff(cur_state));
1088 }
1089 #define assert_fdi_tx_enabled(d, p) assert_fdi_tx(d, p, true)
1090 #define assert_fdi_tx_disabled(d, p) assert_fdi_tx(d, p, false)
1091
1092 static void assert_fdi_rx(struct drm_i915_private *dev_priv,
1093                           enum pipe pipe, bool state)
1094 {
1095         u32 val;
1096         bool cur_state;
1097
1098         val = I915_READ(FDI_RX_CTL(pipe));
1099         cur_state = !!(val & FDI_RX_ENABLE);
1100         I915_STATE_WARN(cur_state != state,
1101              "FDI RX state assertion failure (expected %s, current %s)\n",
1102                         onoff(state), onoff(cur_state));
1103 }
1104 #define assert_fdi_rx_enabled(d, p) assert_fdi_rx(d, p, true)
1105 #define assert_fdi_rx_disabled(d, p) assert_fdi_rx(d, p, false)
1106
1107 static void assert_fdi_tx_pll_enabled(struct drm_i915_private *dev_priv,
1108                                       enum pipe pipe)
1109 {
1110         u32 val;
1111
1112         /* ILK FDI PLL is always enabled */
1113         if (IS_GEN5(dev_priv))
1114                 return;
1115
1116         /* On Haswell, DDI ports are responsible for the FDI PLL setup */
1117         if (HAS_DDI(dev_priv))
1118                 return;
1119
1120         val = I915_READ(FDI_TX_CTL(pipe));
1121         I915_STATE_WARN(!(val & FDI_TX_PLL_ENABLE), "FDI TX PLL assertion failure, should be active but is disabled\n");
1122 }
1123
1124 void assert_fdi_rx_pll(struct drm_i915_private *dev_priv,
1125                        enum pipe pipe, bool state)
1126 {
1127         u32 val;
1128         bool cur_state;
1129
1130         val = I915_READ(FDI_RX_CTL(pipe));
1131         cur_state = !!(val & FDI_RX_PLL_ENABLE);
1132         I915_STATE_WARN(cur_state != state,
1133              "FDI RX PLL assertion failure (expected %s, current %s)\n",
1134                         onoff(state), onoff(cur_state));
1135 }
1136
1137 void assert_panel_unlocked(struct drm_i915_private *dev_priv, enum pipe pipe)
1138 {
1139         i915_reg_t pp_reg;
1140         u32 val;
1141         enum pipe panel_pipe = INVALID_PIPE;
1142         bool locked = true;
1143
1144         if (WARN_ON(HAS_DDI(dev_priv)))
1145                 return;
1146
1147         if (HAS_PCH_SPLIT(dev_priv)) {
1148                 u32 port_sel;
1149
1150                 pp_reg = PP_CONTROL(0);
1151                 port_sel = I915_READ(PP_ON_DELAYS(0)) & PANEL_PORT_SELECT_MASK;
1152
1153                 switch (port_sel) {
1154                 case PANEL_PORT_SELECT_LVDS:
1155                         intel_lvds_port_enabled(dev_priv, PCH_LVDS, &panel_pipe);
1156                         break;
1157                 case PANEL_PORT_SELECT_DPA:
1158                         intel_dp_port_enabled(dev_priv, DP_A, PORT_A, &panel_pipe);
1159                         break;
1160                 case PANEL_PORT_SELECT_DPC:
1161                         intel_dp_port_enabled(dev_priv, PCH_DP_C, PORT_C, &panel_pipe);
1162                         break;
1163                 case PANEL_PORT_SELECT_DPD:
1164                         intel_dp_port_enabled(dev_priv, PCH_DP_D, PORT_D, &panel_pipe);
1165                         break;
1166                 default:
1167                         MISSING_CASE(port_sel);
1168                         break;
1169                 }
1170         } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
1171                 /* presumably write lock depends on pipe, not port select */
1172                 pp_reg = PP_CONTROL(pipe);
1173                 panel_pipe = pipe;
1174         } else {
1175                 u32 port_sel;
1176
1177                 pp_reg = PP_CONTROL(0);
1178                 port_sel = I915_READ(PP_ON_DELAYS(0)) & PANEL_PORT_SELECT_MASK;
1179
1180                 WARN_ON(port_sel != PANEL_PORT_SELECT_LVDS);
1181                 intel_lvds_port_enabled(dev_priv, LVDS, &panel_pipe);
1182         }
1183
1184         val = I915_READ(pp_reg);
1185         if (!(val & PANEL_POWER_ON) ||
1186             ((val & PANEL_UNLOCK_MASK) == PANEL_UNLOCK_REGS))
1187                 locked = false;
1188
1189         I915_STATE_WARN(panel_pipe == pipe && locked,
1190              "panel assertion failure, pipe %c regs locked\n",
1191              pipe_name(pipe));
1192 }
1193
1194 void assert_pipe(struct drm_i915_private *dev_priv,
1195                  enum pipe pipe, bool state)
1196 {
1197         bool cur_state;
1198         enum transcoder cpu_transcoder = intel_pipe_to_cpu_transcoder(dev_priv,
1199                                                                       pipe);
1200         enum intel_display_power_domain power_domain;
1201
1202         /* we keep both pipes enabled on 830 */
1203         if (IS_I830(dev_priv))
1204                 state = true;
1205
1206         power_domain = POWER_DOMAIN_TRANSCODER(cpu_transcoder);
1207         if (intel_display_power_get_if_enabled(dev_priv, power_domain)) {
1208                 u32 val = I915_READ(PIPECONF(cpu_transcoder));
1209                 cur_state = !!(val & PIPECONF_ENABLE);
1210
1211                 intel_display_power_put(dev_priv, power_domain);
1212         } else {
1213                 cur_state = false;
1214         }
1215
1216         I915_STATE_WARN(cur_state != state,
1217              "pipe %c assertion failure (expected %s, current %s)\n",
1218                         pipe_name(pipe), onoff(state), onoff(cur_state));
1219 }
1220
1221 static void assert_plane(struct intel_plane *plane, bool state)
1222 {
1223         enum pipe pipe;
1224         bool cur_state;
1225
1226         cur_state = plane->get_hw_state(plane, &pipe);
1227
1228         I915_STATE_WARN(cur_state != state,
1229                         "%s assertion failure (expected %s, current %s)\n",
1230                         plane->base.name, onoff(state), onoff(cur_state));
1231 }
1232
1233 #define assert_plane_enabled(p) assert_plane(p, true)
1234 #define assert_plane_disabled(p) assert_plane(p, false)
1235
1236 static void assert_planes_disabled(struct intel_crtc *crtc)
1237 {
1238         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1239         struct intel_plane *plane;
1240
1241         for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane)
1242                 assert_plane_disabled(plane);
1243 }
1244
1245 static void assert_vblank_disabled(struct drm_crtc *crtc)
1246 {
1247         if (I915_STATE_WARN_ON(drm_crtc_vblank_get(crtc) == 0))
1248                 drm_crtc_vblank_put(crtc);
1249 }
1250
1251 void assert_pch_transcoder_disabled(struct drm_i915_private *dev_priv,
1252                                     enum pipe pipe)
1253 {
1254         u32 val;
1255         bool enabled;
1256
1257         val = I915_READ(PCH_TRANSCONF(pipe));
1258         enabled = !!(val & TRANS_ENABLE);
1259         I915_STATE_WARN(enabled,
1260              "transcoder assertion failed, should be off on pipe %c but is still active\n",
1261              pipe_name(pipe));
1262 }
1263
1264 static void assert_pch_dp_disabled(struct drm_i915_private *dev_priv,
1265                                    enum pipe pipe, enum port port,
1266                                    i915_reg_t dp_reg)
1267 {
1268         enum pipe port_pipe;
1269         bool state;
1270
1271         state = intel_dp_port_enabled(dev_priv, dp_reg, port, &port_pipe);
1272
1273         I915_STATE_WARN(state && port_pipe == pipe,
1274                         "PCH DP %c enabled on transcoder %c, should be disabled\n",
1275                         port_name(port), pipe_name(pipe));
1276
1277         I915_STATE_WARN(HAS_PCH_IBX(dev_priv) && !state && port_pipe == PIPE_B,
1278                         "IBX PCH DP %c still using transcoder B\n",
1279                         port_name(port));
1280 }
1281
1282 static void assert_pch_hdmi_disabled(struct drm_i915_private *dev_priv,
1283                                      enum pipe pipe, enum port port,
1284                                      i915_reg_t hdmi_reg)
1285 {
1286         enum pipe port_pipe;
1287         bool state;
1288
1289         state = intel_sdvo_port_enabled(dev_priv, hdmi_reg, &port_pipe);
1290
1291         I915_STATE_WARN(state && port_pipe == pipe,
1292                         "PCH HDMI %c enabled on transcoder %c, should be disabled\n",
1293                         port_name(port), pipe_name(pipe));
1294
1295         I915_STATE_WARN(HAS_PCH_IBX(dev_priv) && !state && port_pipe == PIPE_B,
1296                         "IBX PCH HDMI %c still using transcoder B\n",
1297                         port_name(port));
1298 }
1299
1300 static void assert_pch_ports_disabled(struct drm_i915_private *dev_priv,
1301                                       enum pipe pipe)
1302 {
1303         enum pipe port_pipe;
1304
1305         assert_pch_dp_disabled(dev_priv, pipe, PORT_B, PCH_DP_B);
1306         assert_pch_dp_disabled(dev_priv, pipe, PORT_C, PCH_DP_C);
1307         assert_pch_dp_disabled(dev_priv, pipe, PORT_D, PCH_DP_D);
1308
1309         I915_STATE_WARN(intel_crt_port_enabled(dev_priv, PCH_ADPA, &port_pipe) &&
1310                         port_pipe == pipe,
1311                         "PCH VGA enabled on transcoder %c, should be disabled\n",
1312                         pipe_name(pipe));
1313
1314         I915_STATE_WARN(intel_lvds_port_enabled(dev_priv, PCH_LVDS, &port_pipe) &&
1315                         port_pipe == pipe,
1316                         "PCH LVDS enabled on transcoder %c, should be disabled\n",
1317                         pipe_name(pipe));
1318
1319         /* PCH SDVOB multiplex with HDMIB */
1320         assert_pch_hdmi_disabled(dev_priv, pipe, PORT_B, PCH_HDMIB);
1321         assert_pch_hdmi_disabled(dev_priv, pipe, PORT_C, PCH_HDMIC);
1322         assert_pch_hdmi_disabled(dev_priv, pipe, PORT_D, PCH_HDMID);
1323 }
1324
1325 static void _vlv_enable_pll(struct intel_crtc *crtc,
1326                             const struct intel_crtc_state *pipe_config)
1327 {
1328         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1329         enum pipe pipe = crtc->pipe;
1330
1331         I915_WRITE(DPLL(pipe), pipe_config->dpll_hw_state.dpll);
1332         POSTING_READ(DPLL(pipe));
1333         udelay(150);
1334
1335         if (intel_wait_for_register(dev_priv,
1336                                     DPLL(pipe),
1337                                     DPLL_LOCK_VLV,
1338                                     DPLL_LOCK_VLV,
1339                                     1))
1340                 DRM_ERROR("DPLL %d failed to lock\n", pipe);
1341 }
1342
1343 static void vlv_enable_pll(struct intel_crtc *crtc,
1344                            const struct intel_crtc_state *pipe_config)
1345 {
1346         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1347         enum pipe pipe = crtc->pipe;
1348
1349         assert_pipe_disabled(dev_priv, pipe);
1350
1351         /* PLL is protected by panel, make sure we can write it */
1352         assert_panel_unlocked(dev_priv, pipe);
1353
1354         if (pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE)
1355                 _vlv_enable_pll(crtc, pipe_config);
1356
1357         I915_WRITE(DPLL_MD(pipe), pipe_config->dpll_hw_state.dpll_md);
1358         POSTING_READ(DPLL_MD(pipe));
1359 }
1360
1361
1362 static void _chv_enable_pll(struct intel_crtc *crtc,
1363                             const struct intel_crtc_state *pipe_config)
1364 {
1365         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1366         enum pipe pipe = crtc->pipe;
1367         enum dpio_channel port = vlv_pipe_to_channel(pipe);
1368         u32 tmp;
1369
1370         mutex_lock(&dev_priv->sb_lock);
1371
1372         /* Enable back the 10bit clock to display controller */
1373         tmp = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW14(port));
1374         tmp |= DPIO_DCLKP_EN;
1375         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW14(port), tmp);
1376
1377         mutex_unlock(&dev_priv->sb_lock);
1378
1379         /*
1380          * Need to wait > 100ns between dclkp clock enable bit and PLL enable.
1381          */
1382         udelay(1);
1383
1384         /* Enable PLL */
1385         I915_WRITE(DPLL(pipe), pipe_config->dpll_hw_state.dpll);
1386
1387         /* Check PLL is locked */
1388         if (intel_wait_for_register(dev_priv,
1389                                     DPLL(pipe), DPLL_LOCK_VLV, DPLL_LOCK_VLV,
1390                                     1))
1391                 DRM_ERROR("PLL %d failed to lock\n", pipe);
1392 }
1393
1394 static void chv_enable_pll(struct intel_crtc *crtc,
1395                            const struct intel_crtc_state *pipe_config)
1396 {
1397         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1398         enum pipe pipe = crtc->pipe;
1399
1400         assert_pipe_disabled(dev_priv, pipe);
1401
1402         /* PLL is protected by panel, make sure we can write it */
1403         assert_panel_unlocked(dev_priv, pipe);
1404
1405         if (pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE)
1406                 _chv_enable_pll(crtc, pipe_config);
1407
1408         if (pipe != PIPE_A) {
1409                 /*
1410                  * WaPixelRepeatModeFixForC0:chv
1411                  *
1412                  * DPLLCMD is AWOL. Use chicken bits to propagate
1413                  * the value from DPLLBMD to either pipe B or C.
1414                  */
1415                 I915_WRITE(CBR4_VLV, CBR_DPLLBMD_PIPE(pipe));
1416                 I915_WRITE(DPLL_MD(PIPE_B), pipe_config->dpll_hw_state.dpll_md);
1417                 I915_WRITE(CBR4_VLV, 0);
1418                 dev_priv->chv_dpll_md[pipe] = pipe_config->dpll_hw_state.dpll_md;
1419
1420                 /*
1421                  * DPLLB VGA mode also seems to cause problems.
1422                  * We should always have it disabled.
1423                  */
1424                 WARN_ON((I915_READ(DPLL(PIPE_B)) & DPLL_VGA_MODE_DIS) == 0);
1425         } else {
1426                 I915_WRITE(DPLL_MD(pipe), pipe_config->dpll_hw_state.dpll_md);
1427                 POSTING_READ(DPLL_MD(pipe));
1428         }
1429 }
1430
1431 static int intel_num_dvo_pipes(struct drm_i915_private *dev_priv)
1432 {
1433         struct intel_crtc *crtc;
1434         int count = 0;
1435
1436         for_each_intel_crtc(&dev_priv->drm, crtc) {
1437                 count += crtc->base.state->active &&
1438                         intel_crtc_has_type(crtc->config, INTEL_OUTPUT_DVO);
1439         }
1440
1441         return count;
1442 }
1443
1444 static void i9xx_enable_pll(struct intel_crtc *crtc,
1445                             const struct intel_crtc_state *crtc_state)
1446 {
1447         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1448         i915_reg_t reg = DPLL(crtc->pipe);
1449         u32 dpll = crtc_state->dpll_hw_state.dpll;
1450         int i;
1451
1452         assert_pipe_disabled(dev_priv, crtc->pipe);
1453
1454         /* PLL is protected by panel, make sure we can write it */
1455         if (IS_MOBILE(dev_priv) && !IS_I830(dev_priv))
1456                 assert_panel_unlocked(dev_priv, crtc->pipe);
1457
1458         /* Enable DVO 2x clock on both PLLs if necessary */
1459         if (IS_I830(dev_priv) && intel_num_dvo_pipes(dev_priv) > 0) {
1460                 /*
1461                  * It appears to be important that we don't enable this
1462                  * for the current pipe before otherwise configuring the
1463                  * PLL. No idea how this should be handled if multiple
1464                  * DVO outputs are enabled simultaneosly.
1465                  */
1466                 dpll |= DPLL_DVO_2X_MODE;
1467                 I915_WRITE(DPLL(!crtc->pipe),
1468                            I915_READ(DPLL(!crtc->pipe)) | DPLL_DVO_2X_MODE);
1469         }
1470
1471         /*
1472          * Apparently we need to have VGA mode enabled prior to changing
1473          * the P1/P2 dividers. Otherwise the DPLL will keep using the old
1474          * dividers, even though the register value does change.
1475          */
1476         I915_WRITE(reg, 0);
1477
1478         I915_WRITE(reg, dpll);
1479
1480         /* Wait for the clocks to stabilize. */
1481         POSTING_READ(reg);
1482         udelay(150);
1483
1484         if (INTEL_GEN(dev_priv) >= 4) {
1485                 I915_WRITE(DPLL_MD(crtc->pipe),
1486                            crtc_state->dpll_hw_state.dpll_md);
1487         } else {
1488                 /* The pixel multiplier can only be updated once the
1489                  * DPLL is enabled and the clocks are stable.
1490                  *
1491                  * So write it again.
1492                  */
1493                 I915_WRITE(reg, dpll);
1494         }
1495
1496         /* We do this three times for luck */
1497         for (i = 0; i < 3; i++) {
1498                 I915_WRITE(reg, dpll);
1499                 POSTING_READ(reg);
1500                 udelay(150); /* wait for warmup */
1501         }
1502 }
1503
1504 static void i9xx_disable_pll(const struct intel_crtc_state *crtc_state)
1505 {
1506         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
1507         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1508         enum pipe pipe = crtc->pipe;
1509
1510         /* Disable DVO 2x clock on both PLLs if necessary */
1511         if (IS_I830(dev_priv) &&
1512             intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DVO) &&
1513             !intel_num_dvo_pipes(dev_priv)) {
1514                 I915_WRITE(DPLL(PIPE_B),
1515                            I915_READ(DPLL(PIPE_B)) & ~DPLL_DVO_2X_MODE);
1516                 I915_WRITE(DPLL(PIPE_A),
1517                            I915_READ(DPLL(PIPE_A)) & ~DPLL_DVO_2X_MODE);
1518         }
1519
1520         /* Don't disable pipe or pipe PLLs if needed */
1521         if (IS_I830(dev_priv))
1522                 return;
1523
1524         /* Make sure the pipe isn't still relying on us */
1525         assert_pipe_disabled(dev_priv, pipe);
1526
1527         I915_WRITE(DPLL(pipe), DPLL_VGA_MODE_DIS);
1528         POSTING_READ(DPLL(pipe));
1529 }
1530
1531 static void vlv_disable_pll(struct drm_i915_private *dev_priv, enum pipe pipe)
1532 {
1533         u32 val;
1534
1535         /* Make sure the pipe isn't still relying on us */
1536         assert_pipe_disabled(dev_priv, pipe);
1537
1538         val = DPLL_INTEGRATED_REF_CLK_VLV |
1539                 DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
1540         if (pipe != PIPE_A)
1541                 val |= DPLL_INTEGRATED_CRI_CLK_VLV;
1542
1543         I915_WRITE(DPLL(pipe), val);
1544         POSTING_READ(DPLL(pipe));
1545 }
1546
1547 static void chv_disable_pll(struct drm_i915_private *dev_priv, enum pipe pipe)
1548 {
1549         enum dpio_channel port = vlv_pipe_to_channel(pipe);
1550         u32 val;
1551
1552         /* Make sure the pipe isn't still relying on us */
1553         assert_pipe_disabled(dev_priv, pipe);
1554
1555         val = DPLL_SSC_REF_CLK_CHV |
1556                 DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
1557         if (pipe != PIPE_A)
1558                 val |= DPLL_INTEGRATED_CRI_CLK_VLV;
1559
1560         I915_WRITE(DPLL(pipe), val);
1561         POSTING_READ(DPLL(pipe));
1562
1563         mutex_lock(&dev_priv->sb_lock);
1564
1565         /* Disable 10bit clock to display controller */
1566         val = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW14(port));
1567         val &= ~DPIO_DCLKP_EN;
1568         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW14(port), val);
1569
1570         mutex_unlock(&dev_priv->sb_lock);
1571 }
1572
1573 void vlv_wait_port_ready(struct drm_i915_private *dev_priv,
1574                          struct intel_digital_port *dport,
1575                          unsigned int expected_mask)
1576 {
1577         u32 port_mask;
1578         i915_reg_t dpll_reg;
1579
1580         switch (dport->base.port) {
1581         case PORT_B:
1582                 port_mask = DPLL_PORTB_READY_MASK;
1583                 dpll_reg = DPLL(0);
1584                 break;
1585         case PORT_C:
1586                 port_mask = DPLL_PORTC_READY_MASK;
1587                 dpll_reg = DPLL(0);
1588                 expected_mask <<= 4;
1589                 break;
1590         case PORT_D:
1591                 port_mask = DPLL_PORTD_READY_MASK;
1592                 dpll_reg = DPIO_PHY_STATUS;
1593                 break;
1594         default:
1595                 BUG();
1596         }
1597
1598         if (intel_wait_for_register(dev_priv,
1599                                     dpll_reg, port_mask, expected_mask,
1600                                     1000))
1601                 WARN(1, "timed out waiting for port %c ready: got 0x%x, expected 0x%x\n",
1602                      port_name(dport->base.port),
1603                      I915_READ(dpll_reg) & port_mask, expected_mask);
1604 }
1605
1606 static void ironlake_enable_pch_transcoder(const struct intel_crtc_state *crtc_state)
1607 {
1608         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
1609         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1610         enum pipe pipe = crtc->pipe;
1611         i915_reg_t reg;
1612         uint32_t val, pipeconf_val;
1613
1614         /* Make sure PCH DPLL is enabled */
1615         assert_shared_dpll_enabled(dev_priv, crtc_state->shared_dpll);
1616
1617         /* FDI must be feeding us bits for PCH ports */
1618         assert_fdi_tx_enabled(dev_priv, pipe);
1619         assert_fdi_rx_enabled(dev_priv, pipe);
1620
1621         if (HAS_PCH_CPT(dev_priv)) {
1622                 /* Workaround: Set the timing override bit before enabling the
1623                  * pch transcoder. */
1624                 reg = TRANS_CHICKEN2(pipe);
1625                 val = I915_READ(reg);
1626                 val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
1627                 I915_WRITE(reg, val);
1628         }
1629
1630         reg = PCH_TRANSCONF(pipe);
1631         val = I915_READ(reg);
1632         pipeconf_val = I915_READ(PIPECONF(pipe));
1633
1634         if (HAS_PCH_IBX(dev_priv)) {
1635                 /*
1636                  * Make the BPC in transcoder be consistent with
1637                  * that in pipeconf reg. For HDMI we must use 8bpc
1638                  * here for both 8bpc and 12bpc.
1639                  */
1640                 val &= ~PIPECONF_BPC_MASK;
1641                 if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI))
1642                         val |= PIPECONF_8BPC;
1643                 else
1644                         val |= pipeconf_val & PIPECONF_BPC_MASK;
1645         }
1646
1647         val &= ~TRANS_INTERLACE_MASK;
1648         if ((pipeconf_val & PIPECONF_INTERLACE_MASK) == PIPECONF_INTERLACED_ILK)
1649                 if (HAS_PCH_IBX(dev_priv) &&
1650                     intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO))
1651                         val |= TRANS_LEGACY_INTERLACED_ILK;
1652                 else
1653                         val |= TRANS_INTERLACED;
1654         else
1655                 val |= TRANS_PROGRESSIVE;
1656
1657         I915_WRITE(reg, val | TRANS_ENABLE);
1658         if (intel_wait_for_register(dev_priv,
1659                                     reg, TRANS_STATE_ENABLE, TRANS_STATE_ENABLE,
1660                                     100))
1661                 DRM_ERROR("failed to enable transcoder %c\n", pipe_name(pipe));
1662 }
1663
1664 static void lpt_enable_pch_transcoder(struct drm_i915_private *dev_priv,
1665                                       enum transcoder cpu_transcoder)
1666 {
1667         u32 val, pipeconf_val;
1668
1669         /* FDI must be feeding us bits for PCH ports */
1670         assert_fdi_tx_enabled(dev_priv, (enum pipe) cpu_transcoder);
1671         assert_fdi_rx_enabled(dev_priv, PIPE_A);
1672
1673         /* Workaround: set timing override bit. */
1674         val = I915_READ(TRANS_CHICKEN2(PIPE_A));
1675         val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
1676         I915_WRITE(TRANS_CHICKEN2(PIPE_A), val);
1677
1678         val = TRANS_ENABLE;
1679         pipeconf_val = I915_READ(PIPECONF(cpu_transcoder));
1680
1681         if ((pipeconf_val & PIPECONF_INTERLACE_MASK_HSW) ==
1682             PIPECONF_INTERLACED_ILK)
1683                 val |= TRANS_INTERLACED;
1684         else
1685                 val |= TRANS_PROGRESSIVE;
1686
1687         I915_WRITE(LPT_TRANSCONF, val);
1688         if (intel_wait_for_register(dev_priv,
1689                                     LPT_TRANSCONF,
1690                                     TRANS_STATE_ENABLE,
1691                                     TRANS_STATE_ENABLE,
1692                                     100))
1693                 DRM_ERROR("Failed to enable PCH transcoder\n");
1694 }
1695
1696 static void ironlake_disable_pch_transcoder(struct drm_i915_private *dev_priv,
1697                                             enum pipe pipe)
1698 {
1699         i915_reg_t reg;
1700         uint32_t val;
1701
1702         /* FDI relies on the transcoder */
1703         assert_fdi_tx_disabled(dev_priv, pipe);
1704         assert_fdi_rx_disabled(dev_priv, pipe);
1705
1706         /* Ports must be off as well */
1707         assert_pch_ports_disabled(dev_priv, pipe);
1708
1709         reg = PCH_TRANSCONF(pipe);
1710         val = I915_READ(reg);
1711         val &= ~TRANS_ENABLE;
1712         I915_WRITE(reg, val);
1713         /* wait for PCH transcoder off, transcoder state */
1714         if (intel_wait_for_register(dev_priv,
1715                                     reg, TRANS_STATE_ENABLE, 0,
1716                                     50))
1717                 DRM_ERROR("failed to disable transcoder %c\n", pipe_name(pipe));
1718
1719         if (HAS_PCH_CPT(dev_priv)) {
1720                 /* Workaround: Clear the timing override chicken bit again. */
1721                 reg = TRANS_CHICKEN2(pipe);
1722                 val = I915_READ(reg);
1723                 val &= ~TRANS_CHICKEN2_TIMING_OVERRIDE;
1724                 I915_WRITE(reg, val);
1725         }
1726 }
1727
1728 void lpt_disable_pch_transcoder(struct drm_i915_private *dev_priv)
1729 {
1730         u32 val;
1731
1732         val = I915_READ(LPT_TRANSCONF);
1733         val &= ~TRANS_ENABLE;
1734         I915_WRITE(LPT_TRANSCONF, val);
1735         /* wait for PCH transcoder off, transcoder state */
1736         if (intel_wait_for_register(dev_priv,
1737                                     LPT_TRANSCONF, TRANS_STATE_ENABLE, 0,
1738                                     50))
1739                 DRM_ERROR("Failed to disable PCH transcoder\n");
1740
1741         /* Workaround: clear timing override bit. */
1742         val = I915_READ(TRANS_CHICKEN2(PIPE_A));
1743         val &= ~TRANS_CHICKEN2_TIMING_OVERRIDE;
1744         I915_WRITE(TRANS_CHICKEN2(PIPE_A), val);
1745 }
1746
1747 enum pipe intel_crtc_pch_transcoder(struct intel_crtc *crtc)
1748 {
1749         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1750
1751         if (HAS_PCH_LPT(dev_priv))
1752                 return PIPE_A;
1753         else
1754                 return crtc->pipe;
1755 }
1756
1757 static void intel_enable_pipe(const struct intel_crtc_state *new_crtc_state)
1758 {
1759         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->base.crtc);
1760         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1761         enum transcoder cpu_transcoder = new_crtc_state->cpu_transcoder;
1762         enum pipe pipe = crtc->pipe;
1763         i915_reg_t reg;
1764         u32 val;
1765
1766         DRM_DEBUG_KMS("enabling pipe %c\n", pipe_name(pipe));
1767
1768         assert_planes_disabled(crtc);
1769
1770         /*
1771          * A pipe without a PLL won't actually be able to drive bits from
1772          * a plane.  On ILK+ the pipe PLLs are integrated, so we don't
1773          * need the check.
1774          */
1775         if (HAS_GMCH_DISPLAY(dev_priv)) {
1776                 if (intel_crtc_has_type(new_crtc_state, INTEL_OUTPUT_DSI))
1777                         assert_dsi_pll_enabled(dev_priv);
1778                 else
1779                         assert_pll_enabled(dev_priv, pipe);
1780         } else {
1781                 if (new_crtc_state->has_pch_encoder) {
1782                         /* if driving the PCH, we need FDI enabled */
1783                         assert_fdi_rx_pll_enabled(dev_priv,
1784                                                   intel_crtc_pch_transcoder(crtc));
1785                         assert_fdi_tx_pll_enabled(dev_priv,
1786                                                   (enum pipe) cpu_transcoder);
1787                 }
1788                 /* FIXME: assert CPU port conditions for SNB+ */
1789         }
1790
1791         reg = PIPECONF(cpu_transcoder);
1792         val = I915_READ(reg);
1793         if (val & PIPECONF_ENABLE) {
1794                 /* we keep both pipes enabled on 830 */
1795                 WARN_ON(!IS_I830(dev_priv));
1796                 return;
1797         }
1798
1799         I915_WRITE(reg, val | PIPECONF_ENABLE);
1800         POSTING_READ(reg);
1801
1802         /*
1803          * Until the pipe starts PIPEDSL reads will return a stale value,
1804          * which causes an apparent vblank timestamp jump when PIPEDSL
1805          * resets to its proper value. That also messes up the frame count
1806          * when it's derived from the timestamps. So let's wait for the
1807          * pipe to start properly before we call drm_crtc_vblank_on()
1808          */
1809         if (dev_priv->drm.max_vblank_count == 0)
1810                 intel_wait_for_pipe_scanline_moving(crtc);
1811 }
1812
1813 static void intel_disable_pipe(const struct intel_crtc_state *old_crtc_state)
1814 {
1815         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc);
1816         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1817         enum transcoder cpu_transcoder = old_crtc_state->cpu_transcoder;
1818         enum pipe pipe = crtc->pipe;
1819         i915_reg_t reg;
1820         u32 val;
1821
1822         DRM_DEBUG_KMS("disabling pipe %c\n", pipe_name(pipe));
1823
1824         /*
1825          * Make sure planes won't keep trying to pump pixels to us,
1826          * or we might hang the display.
1827          */
1828         assert_planes_disabled(crtc);
1829
1830         reg = PIPECONF(cpu_transcoder);
1831         val = I915_READ(reg);
1832         if ((val & PIPECONF_ENABLE) == 0)
1833                 return;
1834
1835         /*
1836          * Double wide has implications for planes
1837          * so best keep it disabled when not needed.
1838          */
1839         if (old_crtc_state->double_wide)
1840                 val &= ~PIPECONF_DOUBLE_WIDE;
1841
1842         /* Don't disable pipe or pipe PLLs if needed */
1843         if (!IS_I830(dev_priv))
1844                 val &= ~PIPECONF_ENABLE;
1845
1846         I915_WRITE(reg, val);
1847         if ((val & PIPECONF_ENABLE) == 0)
1848                 intel_wait_for_pipe_off(old_crtc_state);
1849 }
1850
1851 static unsigned int intel_tile_size(const struct drm_i915_private *dev_priv)
1852 {
1853         return IS_GEN2(dev_priv) ? 2048 : 4096;
1854 }
1855
1856 static unsigned int
1857 intel_tile_width_bytes(const struct drm_framebuffer *fb, int color_plane)
1858 {
1859         struct drm_i915_private *dev_priv = to_i915(fb->dev);
1860         unsigned int cpp = fb->format->cpp[color_plane];
1861
1862         switch (fb->modifier) {
1863         case DRM_FORMAT_MOD_LINEAR:
1864                 return cpp;
1865         case I915_FORMAT_MOD_X_TILED:
1866                 if (IS_GEN2(dev_priv))
1867                         return 128;
1868                 else
1869                         return 512;
1870         case I915_FORMAT_MOD_Y_TILED_CCS:
1871                 if (color_plane == 1)
1872                         return 128;
1873                 /* fall through */
1874         case I915_FORMAT_MOD_Y_TILED:
1875                 if (IS_GEN2(dev_priv) || HAS_128_BYTE_Y_TILING(dev_priv))
1876                         return 128;
1877                 else
1878                         return 512;
1879         case I915_FORMAT_MOD_Yf_TILED_CCS:
1880                 if (color_plane == 1)
1881                         return 128;
1882                 /* fall through */
1883         case I915_FORMAT_MOD_Yf_TILED:
1884                 switch (cpp) {
1885                 case 1:
1886                         return 64;
1887                 case 2:
1888                 case 4:
1889                         return 128;
1890                 case 8:
1891                 case 16:
1892                         return 256;
1893                 default:
1894                         MISSING_CASE(cpp);
1895                         return cpp;
1896                 }
1897                 break;
1898         default:
1899                 MISSING_CASE(fb->modifier);
1900                 return cpp;
1901         }
1902 }
1903
1904 static unsigned int
1905 intel_tile_height(const struct drm_framebuffer *fb, int color_plane)
1906 {
1907         if (fb->modifier == DRM_FORMAT_MOD_LINEAR)
1908                 return 1;
1909         else
1910                 return intel_tile_size(to_i915(fb->dev)) /
1911                         intel_tile_width_bytes(fb, color_plane);
1912 }
1913
1914 /* Return the tile dimensions in pixel units */
1915 static void intel_tile_dims(const struct drm_framebuffer *fb, int color_plane,
1916                             unsigned int *tile_width,
1917                             unsigned int *tile_height)
1918 {
1919         unsigned int tile_width_bytes = intel_tile_width_bytes(fb, color_plane);
1920         unsigned int cpp = fb->format->cpp[color_plane];
1921
1922         *tile_width = tile_width_bytes / cpp;
1923         *tile_height = intel_tile_size(to_i915(fb->dev)) / tile_width_bytes;
1924 }
1925
1926 unsigned int
1927 intel_fb_align_height(const struct drm_framebuffer *fb,
1928                       int color_plane, unsigned int height)
1929 {
1930         unsigned int tile_height = intel_tile_height(fb, color_plane);
1931
1932         return ALIGN(height, tile_height);
1933 }
1934
1935 unsigned int intel_rotation_info_size(const struct intel_rotation_info *rot_info)
1936 {
1937         unsigned int size = 0;
1938         int i;
1939
1940         for (i = 0 ; i < ARRAY_SIZE(rot_info->plane); i++)
1941                 size += rot_info->plane[i].width * rot_info->plane[i].height;
1942
1943         return size;
1944 }
1945
1946 static void
1947 intel_fill_fb_ggtt_view(struct i915_ggtt_view *view,
1948                         const struct drm_framebuffer *fb,
1949                         unsigned int rotation)
1950 {
1951         view->type = I915_GGTT_VIEW_NORMAL;
1952         if (drm_rotation_90_or_270(rotation)) {
1953                 view->type = I915_GGTT_VIEW_ROTATED;
1954                 view->rotated = to_intel_framebuffer(fb)->rot_info;
1955         }
1956 }
1957
1958 static unsigned int intel_cursor_alignment(const struct drm_i915_private *dev_priv)
1959 {
1960         if (IS_I830(dev_priv))
1961                 return 16 * 1024;
1962         else if (IS_I85X(dev_priv))
1963                 return 256;
1964         else if (IS_I845G(dev_priv) || IS_I865G(dev_priv))
1965                 return 32;
1966         else
1967                 return 4 * 1024;
1968 }
1969
1970 static unsigned int intel_linear_alignment(const struct drm_i915_private *dev_priv)
1971 {
1972         if (INTEL_GEN(dev_priv) >= 9)
1973                 return 256 * 1024;
1974         else if (IS_I965G(dev_priv) || IS_I965GM(dev_priv) ||
1975                  IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
1976                 return 128 * 1024;
1977         else if (INTEL_GEN(dev_priv) >= 4)
1978                 return 4 * 1024;
1979         else
1980                 return 0;
1981 }
1982
1983 static unsigned int intel_surf_alignment(const struct drm_framebuffer *fb,
1984                                          int color_plane)
1985 {
1986         struct drm_i915_private *dev_priv = to_i915(fb->dev);
1987
1988         /* AUX_DIST needs only 4K alignment */
1989         if (color_plane == 1)
1990                 return 4096;
1991
1992         switch (fb->modifier) {
1993         case DRM_FORMAT_MOD_LINEAR:
1994                 return intel_linear_alignment(dev_priv);
1995         case I915_FORMAT_MOD_X_TILED:
1996                 if (INTEL_GEN(dev_priv) >= 9)
1997                         return 256 * 1024;
1998                 return 0;
1999         case I915_FORMAT_MOD_Y_TILED_CCS:
2000         case I915_FORMAT_MOD_Yf_TILED_CCS:
2001         case I915_FORMAT_MOD_Y_TILED:
2002         case I915_FORMAT_MOD_Yf_TILED:
2003                 return 1 * 1024 * 1024;
2004         default:
2005                 MISSING_CASE(fb->modifier);
2006                 return 0;
2007         }
2008 }
2009
2010 static bool intel_plane_uses_fence(const struct intel_plane_state *plane_state)
2011 {
2012         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
2013         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
2014
2015         return INTEL_GEN(dev_priv) < 4 || plane->has_fbc;
2016 }
2017
2018 struct i915_vma *
2019 intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb,
2020                            const struct i915_ggtt_view *view,
2021                            bool uses_fence,
2022                            unsigned long *out_flags)
2023 {
2024         struct drm_device *dev = fb->dev;
2025         struct drm_i915_private *dev_priv = to_i915(dev);
2026         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
2027         struct i915_vma *vma;
2028         unsigned int pinctl;
2029         u32 alignment;
2030
2031         WARN_ON(!mutex_is_locked(&dev->struct_mutex));
2032
2033         alignment = intel_surf_alignment(fb, 0);
2034
2035         /* Note that the w/a also requires 64 PTE of padding following the
2036          * bo. We currently fill all unused PTE with the shadow page and so
2037          * we should always have valid PTE following the scanout preventing
2038          * the VT-d warning.
2039          */
2040         if (intel_scanout_needs_vtd_wa(dev_priv) && alignment < 256 * 1024)
2041                 alignment = 256 * 1024;
2042
2043         /*
2044          * Global gtt pte registers are special registers which actually forward
2045          * writes to a chunk of system memory. Which means that there is no risk
2046          * that the register values disappear as soon as we call
2047          * intel_runtime_pm_put(), so it is correct to wrap only the
2048          * pin/unpin/fence and not more.
2049          */
2050         intel_runtime_pm_get(dev_priv);
2051
2052         atomic_inc(&dev_priv->gpu_error.pending_fb_pin);
2053
2054         pinctl = 0;
2055
2056         /* Valleyview is definitely limited to scanning out the first
2057          * 512MiB. Lets presume this behaviour was inherited from the
2058          * g4x display engine and that all earlier gen are similarly
2059          * limited. Testing suggests that it is a little more
2060          * complicated than this. For example, Cherryview appears quite
2061          * happy to scanout from anywhere within its global aperture.
2062          */
2063         if (HAS_GMCH_DISPLAY(dev_priv))
2064                 pinctl |= PIN_MAPPABLE;
2065
2066         vma = i915_gem_object_pin_to_display_plane(obj,
2067                                                    alignment, view, pinctl);
2068         if (IS_ERR(vma))
2069                 goto err;
2070
2071         if (uses_fence && i915_vma_is_map_and_fenceable(vma)) {
2072                 int ret;
2073
2074                 /* Install a fence for tiled scan-out. Pre-i965 always needs a
2075                  * fence, whereas 965+ only requires a fence if using
2076                  * framebuffer compression.  For simplicity, we always, when
2077                  * possible, install a fence as the cost is not that onerous.
2078                  *
2079                  * If we fail to fence the tiled scanout, then either the
2080                  * modeset will reject the change (which is highly unlikely as
2081                  * the affected systems, all but one, do not have unmappable
2082                  * space) or we will not be able to enable full powersaving
2083                  * techniques (also likely not to apply due to various limits
2084                  * FBC and the like impose on the size of the buffer, which
2085                  * presumably we violated anyway with this unmappable buffer).
2086                  * Anyway, it is presumably better to stumble onwards with
2087                  * something and try to run the system in a "less than optimal"
2088                  * mode that matches the user configuration.
2089                  */
2090                 ret = i915_vma_pin_fence(vma);
2091                 if (ret != 0 && INTEL_GEN(dev_priv) < 4) {
2092                         i915_gem_object_unpin_from_display_plane(vma);
2093                         vma = ERR_PTR(ret);
2094                         goto err;
2095                 }
2096
2097                 if (ret == 0 && vma->fence)
2098                         *out_flags |= PLANE_HAS_FENCE;
2099         }
2100
2101         i915_vma_get(vma);
2102 err:
2103         atomic_dec(&dev_priv->gpu_error.pending_fb_pin);
2104
2105         intel_runtime_pm_put(dev_priv);
2106         return vma;
2107 }
2108
2109 void intel_unpin_fb_vma(struct i915_vma *vma, unsigned long flags)
2110 {
2111         lockdep_assert_held(&vma->vm->i915->drm.struct_mutex);
2112
2113         if (flags & PLANE_HAS_FENCE)
2114                 i915_vma_unpin_fence(vma);
2115         i915_gem_object_unpin_from_display_plane(vma);
2116         i915_vma_put(vma);
2117 }
2118
2119 static int intel_fb_pitch(const struct drm_framebuffer *fb, int color_plane,
2120                           unsigned int rotation)
2121 {
2122         if (drm_rotation_90_or_270(rotation))
2123                 return to_intel_framebuffer(fb)->rotated[color_plane].pitch;
2124         else
2125                 return fb->pitches[color_plane];
2126 }
2127
2128 /*
2129  * Convert the x/y offsets into a linear offset.
2130  * Only valid with 0/180 degree rotation, which is fine since linear
2131  * offset is only used with linear buffers on pre-hsw and tiled buffers
2132  * with gen2/3, and 90/270 degree rotations isn't supported on any of them.
2133  */
2134 u32 intel_fb_xy_to_linear(int x, int y,
2135                           const struct intel_plane_state *state,
2136                           int color_plane)
2137 {
2138         const struct drm_framebuffer *fb = state->base.fb;
2139         unsigned int cpp = fb->format->cpp[color_plane];
2140         unsigned int pitch = state->color_plane[color_plane].stride;
2141
2142         return y * pitch + x * cpp;
2143 }
2144
2145 /*
2146  * Add the x/y offsets derived from fb->offsets[] to the user
2147  * specified plane src x/y offsets. The resulting x/y offsets
2148  * specify the start of scanout from the beginning of the gtt mapping.
2149  */
2150 void intel_add_fb_offsets(int *x, int *y,
2151                           const struct intel_plane_state *state,
2152                           int color_plane)
2153
2154 {
2155         const struct intel_framebuffer *intel_fb = to_intel_framebuffer(state->base.fb);
2156         unsigned int rotation = state->base.rotation;
2157
2158         if (drm_rotation_90_or_270(rotation)) {
2159                 *x += intel_fb->rotated[color_plane].x;
2160                 *y += intel_fb->rotated[color_plane].y;
2161         } else {
2162                 *x += intel_fb->normal[color_plane].x;
2163                 *y += intel_fb->normal[color_plane].y;
2164         }
2165 }
2166
2167 static u32 intel_adjust_tile_offset(int *x, int *y,
2168                                     unsigned int tile_width,
2169                                     unsigned int tile_height,
2170                                     unsigned int tile_size,
2171                                     unsigned int pitch_tiles,
2172                                     u32 old_offset,
2173                                     u32 new_offset)
2174 {
2175         unsigned int pitch_pixels = pitch_tiles * tile_width;
2176         unsigned int tiles;
2177
2178         WARN_ON(old_offset & (tile_size - 1));
2179         WARN_ON(new_offset & (tile_size - 1));
2180         WARN_ON(new_offset > old_offset);
2181
2182         tiles = (old_offset - new_offset) / tile_size;
2183
2184         *y += tiles / pitch_tiles * tile_height;
2185         *x += tiles % pitch_tiles * tile_width;
2186
2187         /* minimize x in case it got needlessly big */
2188         *y += *x / pitch_pixels * tile_height;
2189         *x %= pitch_pixels;
2190
2191         return new_offset;
2192 }
2193
2194 static bool is_surface_linear(u64 modifier, int color_plane)
2195 {
2196         return modifier == DRM_FORMAT_MOD_LINEAR;
2197 }
2198
2199 static u32 intel_adjust_aligned_offset(int *x, int *y,
2200                                        const struct drm_framebuffer *fb,
2201                                        int color_plane,
2202                                        unsigned int rotation,
2203                                        unsigned int pitch,
2204                                        u32 old_offset, u32 new_offset)
2205 {
2206         struct drm_i915_private *dev_priv = to_i915(fb->dev);
2207         unsigned int cpp = fb->format->cpp[color_plane];
2208
2209         WARN_ON(new_offset > old_offset);
2210
2211         if (!is_surface_linear(fb->modifier, color_plane)) {
2212                 unsigned int tile_size, tile_width, tile_height;
2213                 unsigned int pitch_tiles;
2214
2215                 tile_size = intel_tile_size(dev_priv);
2216                 intel_tile_dims(fb, color_plane, &tile_width, &tile_height);
2217
2218                 if (drm_rotation_90_or_270(rotation)) {
2219                         pitch_tiles = pitch / tile_height;
2220                         swap(tile_width, tile_height);
2221                 } else {
2222                         pitch_tiles = pitch / (tile_width * cpp);
2223                 }
2224
2225                 intel_adjust_tile_offset(x, y, tile_width, tile_height,
2226                                          tile_size, pitch_tiles,
2227                                          old_offset, new_offset);
2228         } else {
2229                 old_offset += *y * pitch + *x * cpp;
2230
2231                 *y = (old_offset - new_offset) / pitch;
2232                 *x = ((old_offset - new_offset) - *y * pitch) / cpp;
2233         }
2234
2235         return new_offset;
2236 }
2237
2238 /*
2239  * Adjust the tile offset by moving the difference into
2240  * the x/y offsets.
2241  */
2242 static u32 intel_plane_adjust_aligned_offset(int *x, int *y,
2243                                              const struct intel_plane_state *state,
2244                                              int color_plane,
2245                                              u32 old_offset, u32 new_offset)
2246 {
2247         return intel_adjust_aligned_offset(x, y, state->base.fb, color_plane,
2248                                            state->base.rotation,
2249                                            state->color_plane[color_plane].stride,
2250                                            old_offset, new_offset);
2251 }
2252
2253 /*
2254  * Computes the aligned offset to the base tile and adjusts
2255  * x, y. bytes per pixel is assumed to be a power-of-two.
2256  *
2257  * In the 90/270 rotated case, x and y are assumed
2258  * to be already rotated to match the rotated GTT view, and
2259  * pitch is the tile_height aligned framebuffer height.
2260  *
2261  * This function is used when computing the derived information
2262  * under intel_framebuffer, so using any of that information
2263  * here is not allowed. Anything under drm_framebuffer can be
2264  * used. This is why the user has to pass in the pitch since it
2265  * is specified in the rotated orientation.
2266  */
2267 static u32 intel_compute_aligned_offset(struct drm_i915_private *dev_priv,
2268                                         int *x, int *y,
2269                                         const struct drm_framebuffer *fb,
2270                                         int color_plane,
2271                                         unsigned int pitch,
2272                                         unsigned int rotation,
2273                                         u32 alignment)
2274 {
2275         unsigned int cpp = fb->format->cpp[color_plane];
2276         u32 offset, offset_aligned;
2277
2278         if (alignment)
2279                 alignment--;
2280
2281         if (!is_surface_linear(fb->modifier, color_plane)) {
2282                 unsigned int tile_size, tile_width, tile_height;
2283                 unsigned int tile_rows, tiles, pitch_tiles;
2284
2285                 tile_size = intel_tile_size(dev_priv);
2286                 intel_tile_dims(fb, color_plane, &tile_width, &tile_height);
2287
2288                 if (drm_rotation_90_or_270(rotation)) {
2289                         pitch_tiles = pitch / tile_height;
2290                         swap(tile_width, tile_height);
2291                 } else {
2292                         pitch_tiles = pitch / (tile_width * cpp);
2293                 }
2294
2295                 tile_rows = *y / tile_height;
2296                 *y %= tile_height;
2297
2298                 tiles = *x / tile_width;
2299                 *x %= tile_width;
2300
2301                 offset = (tile_rows * pitch_tiles + tiles) * tile_size;
2302                 offset_aligned = offset & ~alignment;
2303
2304                 intel_adjust_tile_offset(x, y, tile_width, tile_height,
2305                                          tile_size, pitch_tiles,
2306                                          offset, offset_aligned);
2307         } else {
2308                 offset = *y * pitch + *x * cpp;
2309                 offset_aligned = offset & ~alignment;
2310
2311                 *y = (offset & alignment) / pitch;
2312                 *x = ((offset & alignment) - *y * pitch) / cpp;
2313         }
2314
2315         return offset_aligned;
2316 }
2317
2318 static u32 intel_plane_compute_aligned_offset(int *x, int *y,
2319                                               const struct intel_plane_state *state,
2320                                               int color_plane)
2321 {
2322         struct intel_plane *intel_plane = to_intel_plane(state->base.plane);
2323         struct drm_i915_private *dev_priv = to_i915(intel_plane->base.dev);
2324         const struct drm_framebuffer *fb = state->base.fb;
2325         unsigned int rotation = state->base.rotation;
2326         int pitch = state->color_plane[color_plane].stride;
2327         u32 alignment;
2328
2329         if (intel_plane->id == PLANE_CURSOR)
2330                 alignment = intel_cursor_alignment(dev_priv);
2331         else
2332                 alignment = intel_surf_alignment(fb, color_plane);
2333
2334         return intel_compute_aligned_offset(dev_priv, x, y, fb, color_plane,
2335                                             pitch, rotation, alignment);
2336 }
2337
2338 /* Convert the fb->offset[] into x/y offsets */
2339 static int intel_fb_offset_to_xy(int *x, int *y,
2340                                  const struct drm_framebuffer *fb,
2341                                  int color_plane)
2342 {
2343         struct drm_i915_private *dev_priv = to_i915(fb->dev);
2344         unsigned int height;
2345
2346         if (fb->modifier != DRM_FORMAT_MOD_LINEAR &&
2347             fb->offsets[color_plane] % intel_tile_size(dev_priv)) {
2348                 DRM_DEBUG_KMS("Misaligned offset 0x%08x for color plane %d\n",
2349                               fb->offsets[color_plane], color_plane);
2350                 return -EINVAL;
2351         }
2352
2353         height = drm_framebuffer_plane_height(fb->height, fb, color_plane);
2354         height = ALIGN(height, intel_tile_height(fb, color_plane));
2355
2356         /* Catch potential overflows early */
2357         if (add_overflows_t(u32, mul_u32_u32(height, fb->pitches[color_plane]),
2358                             fb->offsets[color_plane])) {
2359                 DRM_DEBUG_KMS("Bad offset 0x%08x or pitch %d for color plane %d\n",
2360                               fb->offsets[color_plane], fb->pitches[color_plane],
2361                               color_plane);
2362                 return -ERANGE;
2363         }
2364
2365         *x = 0;
2366         *y = 0;
2367
2368         intel_adjust_aligned_offset(x, y,
2369                                     fb, color_plane, DRM_MODE_ROTATE_0,
2370                                     fb->pitches[color_plane],
2371                                     fb->offsets[color_plane], 0);
2372
2373         return 0;
2374 }
2375
2376 static unsigned int intel_fb_modifier_to_tiling(uint64_t fb_modifier)
2377 {
2378         switch (fb_modifier) {
2379         case I915_FORMAT_MOD_X_TILED:
2380                 return I915_TILING_X;
2381         case I915_FORMAT_MOD_Y_TILED:
2382         case I915_FORMAT_MOD_Y_TILED_CCS:
2383                 return I915_TILING_Y;
2384         default:
2385                 return I915_TILING_NONE;
2386         }
2387 }
2388
2389 /*
2390  * From the Sky Lake PRM:
2391  * "The Color Control Surface (CCS) contains the compression status of
2392  *  the cache-line pairs. The compression state of the cache-line pair
2393  *  is specified by 2 bits in the CCS. Each CCS cache-line represents
2394  *  an area on the main surface of 16 x16 sets of 128 byte Y-tiled
2395  *  cache-line-pairs. CCS is always Y tiled."
2396  *
2397  * Since cache line pairs refers to horizontally adjacent cache lines,
2398  * each cache line in the CCS corresponds to an area of 32x16 cache
2399  * lines on the main surface. Since each pixel is 4 bytes, this gives
2400  * us a ratio of one byte in the CCS for each 8x16 pixels in the
2401  * main surface.
2402  */
2403 static const struct drm_format_info ccs_formats[] = {
2404         { .format = DRM_FORMAT_XRGB8888, .depth = 24, .num_planes = 2, .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, },
2405         { .format = DRM_FORMAT_XBGR8888, .depth = 24, .num_planes = 2, .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, },
2406         { .format = DRM_FORMAT_ARGB8888, .depth = 32, .num_planes = 2, .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, },
2407         { .format = DRM_FORMAT_ABGR8888, .depth = 32, .num_planes = 2, .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, },
2408 };
2409
2410 static const struct drm_format_info *
2411 lookup_format_info(const struct drm_format_info formats[],
2412                    int num_formats, u32 format)
2413 {
2414         int i;
2415
2416         for (i = 0; i < num_formats; i++) {
2417                 if (formats[i].format == format)
2418                         return &formats[i];
2419         }
2420
2421         return NULL;
2422 }
2423
2424 static const struct drm_format_info *
2425 intel_get_format_info(const struct drm_mode_fb_cmd2 *cmd)
2426 {
2427         switch (cmd->modifier[0]) {
2428         case I915_FORMAT_MOD_Y_TILED_CCS:
2429         case I915_FORMAT_MOD_Yf_TILED_CCS:
2430                 return lookup_format_info(ccs_formats,
2431                                           ARRAY_SIZE(ccs_formats),
2432                                           cmd->pixel_format);
2433         default:
2434                 return NULL;
2435         }
2436 }
2437
2438 bool is_ccs_modifier(u64 modifier)
2439 {
2440         return modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
2441                modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
2442 }
2443
2444 static int
2445 intel_fill_fb_info(struct drm_i915_private *dev_priv,
2446                    struct drm_framebuffer *fb)
2447 {
2448         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
2449         struct intel_rotation_info *rot_info = &intel_fb->rot_info;
2450         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
2451         u32 gtt_offset_rotated = 0;
2452         unsigned int max_size = 0;
2453         int i, num_planes = fb->format->num_planes;
2454         unsigned int tile_size = intel_tile_size(dev_priv);
2455
2456         for (i = 0; i < num_planes; i++) {
2457                 unsigned int width, height;
2458                 unsigned int cpp, size;
2459                 u32 offset;
2460                 int x, y;
2461                 int ret;
2462
2463                 cpp = fb->format->cpp[i];
2464                 width = drm_framebuffer_plane_width(fb->width, fb, i);
2465                 height = drm_framebuffer_plane_height(fb->height, fb, i);
2466
2467                 ret = intel_fb_offset_to_xy(&x, &y, fb, i);
2468                 if (ret) {
2469                         DRM_DEBUG_KMS("bad fb plane %d offset: 0x%x\n",
2470                                       i, fb->offsets[i]);
2471                         return ret;
2472                 }
2473
2474                 if (is_ccs_modifier(fb->modifier) && i == 1) {
2475                         int hsub = fb->format->hsub;
2476                         int vsub = fb->format->vsub;
2477                         int tile_width, tile_height;
2478                         int main_x, main_y;
2479                         int ccs_x, ccs_y;
2480
2481                         intel_tile_dims(fb, i, &tile_width, &tile_height);
2482                         tile_width *= hsub;
2483                         tile_height *= vsub;
2484
2485                         ccs_x = (x * hsub) % tile_width;
2486                         ccs_y = (y * vsub) % tile_height;
2487                         main_x = intel_fb->normal[0].x % tile_width;
2488                         main_y = intel_fb->normal[0].y % tile_height;
2489
2490                         /*
2491                          * CCS doesn't have its own x/y offset register, so the intra CCS tile
2492                          * x/y offsets must match between CCS and the main surface.
2493                          */
2494                         if (main_x != ccs_x || main_y != ccs_y) {
2495                                 DRM_DEBUG_KMS("Bad CCS x/y (main %d,%d ccs %d,%d) full (main %d,%d ccs %d,%d)\n",
2496                                               main_x, main_y,
2497                                               ccs_x, ccs_y,
2498                                               intel_fb->normal[0].x,
2499                                               intel_fb->normal[0].y,
2500                                               x, y);
2501                                 return -EINVAL;
2502                         }
2503                 }
2504
2505                 /*
2506                  * The fence (if used) is aligned to the start of the object
2507                  * so having the framebuffer wrap around across the edge of the
2508                  * fenced region doesn't really work. We have no API to configure
2509                  * the fence start offset within the object (nor could we probably
2510                  * on gen2/3). So it's just easier if we just require that the
2511                  * fb layout agrees with the fence layout. We already check that the
2512                  * fb stride matches the fence stride elsewhere.
2513                  */
2514                 if (i == 0 && i915_gem_object_is_tiled(obj) &&
2515                     (x + width) * cpp > fb->pitches[i]) {
2516                         DRM_DEBUG_KMS("bad fb plane %d offset: 0x%x\n",
2517                                       i, fb->offsets[i]);
2518                         return -EINVAL;
2519                 }
2520
2521                 /*
2522                  * First pixel of the framebuffer from
2523                  * the start of the normal gtt mapping.
2524                  */
2525                 intel_fb->normal[i].x = x;
2526                 intel_fb->normal[i].y = y;
2527
2528                 offset = intel_compute_aligned_offset(dev_priv, &x, &y, fb, i,
2529                                                       fb->pitches[i],
2530                                                       DRM_MODE_ROTATE_0,
2531                                                       tile_size);
2532                 offset /= tile_size;
2533
2534                 if (!is_surface_linear(fb->modifier, i)) {
2535                         unsigned int tile_width, tile_height;
2536                         unsigned int pitch_tiles;
2537                         struct drm_rect r;
2538
2539                         intel_tile_dims(fb, i, &tile_width, &tile_height);
2540
2541                         rot_info->plane[i].offset = offset;
2542                         rot_info->plane[i].stride = DIV_ROUND_UP(fb->pitches[i], tile_width * cpp);
2543                         rot_info->plane[i].width = DIV_ROUND_UP(x + width, tile_width);
2544                         rot_info->plane[i].height = DIV_ROUND_UP(y + height, tile_height);
2545
2546                         intel_fb->rotated[i].pitch =
2547                                 rot_info->plane[i].height * tile_height;
2548
2549                         /* how many tiles does this plane need */
2550                         size = rot_info->plane[i].stride * rot_info->plane[i].height;
2551                         /*
2552                          * If the plane isn't horizontally tile aligned,
2553                          * we need one more tile.
2554                          */
2555                         if (x != 0)
2556                                 size++;
2557
2558                         /* rotate the x/y offsets to match the GTT view */
2559                         r.x1 = x;
2560                         r.y1 = y;
2561                         r.x2 = x + width;
2562                         r.y2 = y + height;
2563                         drm_rect_rotate(&r,
2564                                         rot_info->plane[i].width * tile_width,
2565                                         rot_info->plane[i].height * tile_height,
2566                                         DRM_MODE_ROTATE_270);
2567                         x = r.x1;
2568                         y = r.y1;
2569
2570                         /* rotate the tile dimensions to match the GTT view */
2571                         pitch_tiles = intel_fb->rotated[i].pitch / tile_height;
2572                         swap(tile_width, tile_height);
2573
2574                         /*
2575                          * We only keep the x/y offsets, so push all of the
2576                          * gtt offset into the x/y offsets.
2577                          */
2578                         intel_adjust_tile_offset(&x, &y,
2579                                                  tile_width, tile_height,
2580                                                  tile_size, pitch_tiles,
2581                                                  gtt_offset_rotated * tile_size, 0);
2582
2583                         gtt_offset_rotated += rot_info->plane[i].width * rot_info->plane[i].height;
2584
2585                         /*
2586                          * First pixel of the framebuffer from
2587                          * the start of the rotated gtt mapping.
2588                          */
2589                         intel_fb->rotated[i].x = x;
2590                         intel_fb->rotated[i].y = y;
2591                 } else {
2592                         size = DIV_ROUND_UP((y + height) * fb->pitches[i] +
2593                                             x * cpp, tile_size);
2594                 }
2595
2596                 /* how many tiles in total needed in the bo */
2597                 max_size = max(max_size, offset + size);
2598         }
2599
2600         if (mul_u32_u32(max_size, tile_size) > obj->base.size) {
2601                 DRM_DEBUG_KMS("fb too big for bo (need %llu bytes, have %zu bytes)\n",
2602                               mul_u32_u32(max_size, tile_size), obj->base.size);
2603                 return -EINVAL;
2604         }
2605
2606         return 0;
2607 }
2608
2609 static int i9xx_format_to_fourcc(int format)
2610 {
2611         switch (format) {
2612         case DISPPLANE_8BPP:
2613                 return DRM_FORMAT_C8;
2614         case DISPPLANE_BGRX555:
2615                 return DRM_FORMAT_XRGB1555;
2616         case DISPPLANE_BGRX565:
2617                 return DRM_FORMAT_RGB565;
2618         default:
2619         case DISPPLANE_BGRX888:
2620                 return DRM_FORMAT_XRGB8888;
2621         case DISPPLANE_RGBX888:
2622                 return DRM_FORMAT_XBGR8888;
2623         case DISPPLANE_BGRX101010:
2624                 return DRM_FORMAT_XRGB2101010;
2625         case DISPPLANE_RGBX101010:
2626                 return DRM_FORMAT_XBGR2101010;
2627         }
2628 }
2629
2630 int skl_format_to_fourcc(int format, bool rgb_order, bool alpha)
2631 {
2632         switch (format) {
2633         case PLANE_CTL_FORMAT_RGB_565:
2634                 return DRM_FORMAT_RGB565;
2635         case PLANE_CTL_FORMAT_NV12:
2636                 return DRM_FORMAT_NV12;
2637         default:
2638         case PLANE_CTL_FORMAT_XRGB_8888:
2639                 if (rgb_order) {
2640                         if (alpha)
2641                                 return DRM_FORMAT_ABGR8888;
2642                         else
2643                                 return DRM_FORMAT_XBGR8888;
2644                 } else {
2645                         if (alpha)
2646                                 return DRM_FORMAT_ARGB8888;
2647                         else
2648                                 return DRM_FORMAT_XRGB8888;
2649                 }
2650         case PLANE_CTL_FORMAT_XRGB_2101010:
2651                 if (rgb_order)
2652                         return DRM_FORMAT_XBGR2101010;
2653                 else
2654                         return DRM_FORMAT_XRGB2101010;
2655         }
2656 }
2657
2658 static bool
2659 intel_alloc_initial_plane_obj(struct intel_crtc *crtc,
2660                               struct intel_initial_plane_config *plane_config)
2661 {
2662         struct drm_device *dev = crtc->base.dev;
2663         struct drm_i915_private *dev_priv = to_i915(dev);
2664         struct drm_i915_gem_object *obj = NULL;
2665         struct drm_mode_fb_cmd2 mode_cmd = { 0 };
2666         struct drm_framebuffer *fb = &plane_config->fb->base;
2667         u32 base_aligned = round_down(plane_config->base, PAGE_SIZE);
2668         u32 size_aligned = round_up(plane_config->base + plane_config->size,
2669                                     PAGE_SIZE);
2670
2671         size_aligned -= base_aligned;
2672
2673         if (plane_config->size == 0)
2674                 return false;
2675
2676         /* If the FB is too big, just don't use it since fbdev is not very
2677          * important and we should probably use that space with FBC or other
2678          * features. */
2679         if (size_aligned * 2 > dev_priv->stolen_usable_size)
2680                 return false;
2681
2682         switch (fb->modifier) {
2683         case DRM_FORMAT_MOD_LINEAR:
2684         case I915_FORMAT_MOD_X_TILED:
2685         case I915_FORMAT_MOD_Y_TILED:
2686                 break;
2687         default:
2688                 DRM_DEBUG_DRIVER("Unsupported modifier for initial FB: 0x%llx\n",
2689                                  fb->modifier);
2690                 return false;
2691         }
2692
2693         mutex_lock(&dev->struct_mutex);
2694         obj = i915_gem_object_create_stolen_for_preallocated(dev_priv,
2695                                                              base_aligned,
2696                                                              base_aligned,
2697                                                              size_aligned);
2698         mutex_unlock(&dev->struct_mutex);
2699         if (!obj)
2700                 return false;
2701
2702         switch (plane_config->tiling) {
2703         case I915_TILING_NONE:
2704                 break;
2705         case I915_TILING_X:
2706         case I915_TILING_Y:
2707                 obj->tiling_and_stride = fb->pitches[0] | plane_config->tiling;
2708                 break;
2709         default:
2710                 MISSING_CASE(plane_config->tiling);
2711                 return false;
2712         }
2713
2714         mode_cmd.pixel_format = fb->format->format;
2715         mode_cmd.width = fb->width;
2716         mode_cmd.height = fb->height;
2717         mode_cmd.pitches[0] = fb->pitches[0];
2718         mode_cmd.modifier[0] = fb->modifier;
2719         mode_cmd.flags = DRM_MODE_FB_MODIFIERS;
2720
2721         if (intel_framebuffer_init(to_intel_framebuffer(fb), obj, &mode_cmd)) {
2722                 DRM_DEBUG_KMS("intel fb init failed\n");
2723                 goto out_unref_obj;
2724         }
2725
2726
2727         DRM_DEBUG_KMS("initial plane fb obj %p\n", obj);
2728         return true;
2729
2730 out_unref_obj:
2731         i915_gem_object_put(obj);
2732         return false;
2733 }
2734
2735 static void
2736 intel_set_plane_visible(struct intel_crtc_state *crtc_state,
2737                         struct intel_plane_state *plane_state,
2738                         bool visible)
2739 {
2740         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
2741
2742         plane_state->base.visible = visible;
2743
2744         if (visible)
2745                 crtc_state->base.plane_mask |= drm_plane_mask(&plane->base);
2746         else
2747                 crtc_state->base.plane_mask &= ~drm_plane_mask(&plane->base);
2748 }
2749
2750 static void fixup_active_planes(struct intel_crtc_state *crtc_state)
2751 {
2752         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
2753         struct drm_plane *plane;
2754
2755         /*
2756          * Active_planes aliases if multiple "primary" or cursor planes
2757          * have been used on the same (or wrong) pipe. plane_mask uses
2758          * unique ids, hence we can use that to reconstruct active_planes.
2759          */
2760         crtc_state->active_planes = 0;
2761
2762         drm_for_each_plane_mask(plane, &dev_priv->drm,
2763                                 crtc_state->base.plane_mask)
2764                 crtc_state->active_planes |= BIT(to_intel_plane(plane)->id);
2765 }
2766
2767 static void intel_plane_disable_noatomic(struct intel_crtc *crtc,
2768                                          struct intel_plane *plane)
2769 {
2770         struct intel_crtc_state *crtc_state =
2771                 to_intel_crtc_state(crtc->base.state);
2772         struct intel_plane_state *plane_state =
2773                 to_intel_plane_state(plane->base.state);
2774
2775         DRM_DEBUG_KMS("Disabling [PLANE:%d:%s] on [CRTC:%d:%s]\n",
2776                       plane->base.base.id, plane->base.name,
2777                       crtc->base.base.id, crtc->base.name);
2778
2779         intel_set_plane_visible(crtc_state, plane_state, false);
2780         fixup_active_planes(crtc_state);
2781
2782         if (plane->id == PLANE_PRIMARY)
2783                 intel_pre_disable_primary_noatomic(&crtc->base);
2784
2785         trace_intel_disable_plane(&plane->base, crtc);
2786         plane->disable_plane(plane, crtc_state);
2787 }
2788
2789 static void
2790 intel_find_initial_plane_obj(struct intel_crtc *intel_crtc,
2791                              struct intel_initial_plane_config *plane_config)
2792 {
2793         struct drm_device *dev = intel_crtc->base.dev;
2794         struct drm_i915_private *dev_priv = to_i915(dev);
2795         struct drm_crtc *c;
2796         struct drm_i915_gem_object *obj;
2797         struct drm_plane *primary = intel_crtc->base.primary;
2798         struct drm_plane_state *plane_state = primary->state;
2799         struct intel_plane *intel_plane = to_intel_plane(primary);
2800         struct intel_plane_state *intel_state =
2801                 to_intel_plane_state(plane_state);
2802         struct drm_framebuffer *fb;
2803
2804         if (!plane_config->fb)
2805                 return;
2806
2807         if (intel_alloc_initial_plane_obj(intel_crtc, plane_config)) {
2808                 fb = &plane_config->fb->base;
2809                 goto valid_fb;
2810         }
2811
2812         kfree(plane_config->fb);
2813
2814         /*
2815          * Failed to alloc the obj, check to see if we should share
2816          * an fb with another CRTC instead
2817          */
2818         for_each_crtc(dev, c) {
2819                 struct intel_plane_state *state;
2820
2821                 if (c == &intel_crtc->base)
2822                         continue;
2823
2824                 if (!to_intel_crtc(c)->active)
2825                         continue;
2826
2827                 state = to_intel_plane_state(c->primary->state);
2828                 if (!state->vma)
2829                         continue;
2830
2831                 if (intel_plane_ggtt_offset(state) == plane_config->base) {
2832                         fb = state->base.fb;
2833                         drm_framebuffer_get(fb);
2834                         goto valid_fb;
2835                 }
2836         }
2837
2838         /*
2839          * We've failed to reconstruct the BIOS FB.  Current display state
2840          * indicates that the primary plane is visible, but has a NULL FB,
2841          * which will lead to problems later if we don't fix it up.  The
2842          * simplest solution is to just disable the primary plane now and
2843          * pretend the BIOS never had it enabled.
2844          */
2845         intel_plane_disable_noatomic(intel_crtc, intel_plane);
2846
2847         return;
2848
2849 valid_fb:
2850         intel_state->base.rotation = plane_config->rotation;
2851         intel_fill_fb_ggtt_view(&intel_state->view, fb,
2852                                 intel_state->base.rotation);
2853         intel_state->color_plane[0].stride =
2854                 intel_fb_pitch(fb, 0, intel_state->base.rotation);
2855
2856         mutex_lock(&dev->struct_mutex);
2857         intel_state->vma =
2858                 intel_pin_and_fence_fb_obj(fb,
2859                                            &intel_state->view,
2860                                            intel_plane_uses_fence(intel_state),
2861                                            &intel_state->flags);
2862         mutex_unlock(&dev->struct_mutex);
2863         if (IS_ERR(intel_state->vma)) {
2864                 DRM_ERROR("failed to pin boot fb on pipe %d: %li\n",
2865                           intel_crtc->pipe, PTR_ERR(intel_state->vma));
2866
2867                 intel_state->vma = NULL;
2868                 drm_framebuffer_put(fb);
2869                 return;
2870         }
2871
2872         obj = intel_fb_obj(fb);
2873         intel_fb_obj_flush(obj, ORIGIN_DIRTYFB);
2874
2875         plane_state->src_x = 0;
2876         plane_state->src_y = 0;
2877         plane_state->src_w = fb->width << 16;
2878         plane_state->src_h = fb->height << 16;
2879
2880         plane_state->crtc_x = 0;
2881         plane_state->crtc_y = 0;
2882         plane_state->crtc_w = fb->width;
2883         plane_state->crtc_h = fb->height;
2884
2885         intel_state->base.src = drm_plane_state_src(plane_state);
2886         intel_state->base.dst = drm_plane_state_dest(plane_state);
2887
2888         if (i915_gem_object_is_tiled(obj))
2889                 dev_priv->preserve_bios_swizzle = true;
2890
2891         plane_state->fb = fb;
2892         plane_state->crtc = &intel_crtc->base;
2893
2894         atomic_or(to_intel_plane(primary)->frontbuffer_bit,
2895                   &obj->frontbuffer_bits);
2896 }
2897
2898 static int skl_max_plane_width(const struct drm_framebuffer *fb,
2899                                int color_plane,
2900                                unsigned int rotation)
2901 {
2902         int cpp = fb->format->cpp[color_plane];
2903
2904         switch (fb->modifier) {
2905         case DRM_FORMAT_MOD_LINEAR:
2906         case I915_FORMAT_MOD_X_TILED:
2907                 switch (cpp) {
2908                 case 8:
2909                         return 4096;
2910                 case 4:
2911                 case 2:
2912                 case 1:
2913                         return 8192;
2914                 default:
2915                         MISSING_CASE(cpp);
2916                         break;
2917                 }
2918                 break;
2919         case I915_FORMAT_MOD_Y_TILED_CCS:
2920         case I915_FORMAT_MOD_Yf_TILED_CCS:
2921                 /* FIXME AUX plane? */
2922         case I915_FORMAT_MOD_Y_TILED:
2923         case I915_FORMAT_MOD_Yf_TILED:
2924                 switch (cpp) {
2925                 case 8:
2926                         return 2048;
2927                 case 4:
2928                         return 4096;
2929                 case 2:
2930                 case 1:
2931                         return 8192;
2932                 default:
2933                         MISSING_CASE(cpp);
2934                         break;
2935                 }
2936                 break;
2937         default:
2938                 MISSING_CASE(fb->modifier);
2939         }
2940
2941         return 2048;
2942 }
2943
2944 static bool skl_check_main_ccs_coordinates(struct intel_plane_state *plane_state,
2945                                            int main_x, int main_y, u32 main_offset)
2946 {
2947         const struct drm_framebuffer *fb = plane_state->base.fb;
2948         int hsub = fb->format->hsub;
2949         int vsub = fb->format->vsub;
2950         int aux_x = plane_state->color_plane[1].x;
2951         int aux_y = plane_state->color_plane[1].y;
2952         u32 aux_offset = plane_state->color_plane[1].offset;
2953         u32 alignment = intel_surf_alignment(fb, 1);
2954
2955         while (aux_offset >= main_offset && aux_y <= main_y) {
2956                 int x, y;
2957
2958                 if (aux_x == main_x && aux_y == main_y)
2959                         break;
2960
2961                 if (aux_offset == 0)
2962                         break;
2963
2964                 x = aux_x / hsub;
2965                 y = aux_y / vsub;
2966                 aux_offset = intel_plane_adjust_aligned_offset(&x, &y, plane_state, 1,
2967                                                                aux_offset, aux_offset - alignment);
2968                 aux_x = x * hsub + aux_x % hsub;
2969                 aux_y = y * vsub + aux_y % vsub;
2970         }
2971
2972         if (aux_x != main_x || aux_y != main_y)
2973                 return false;
2974
2975         plane_state->color_plane[1].offset = aux_offset;
2976         plane_state->color_plane[1].x = aux_x;
2977         plane_state->color_plane[1].y = aux_y;
2978
2979         return true;
2980 }
2981
2982 static int skl_check_main_surface(struct intel_plane_state *plane_state)
2983 {
2984         const struct drm_framebuffer *fb = plane_state->base.fb;
2985         unsigned int rotation = plane_state->base.rotation;
2986         int x = plane_state->base.src.x1 >> 16;
2987         int y = plane_state->base.src.y1 >> 16;
2988         int w = drm_rect_width(&plane_state->base.src) >> 16;
2989         int h = drm_rect_height(&plane_state->base.src) >> 16;
2990         int max_width = skl_max_plane_width(fb, 0, rotation);
2991         int max_height = 4096;
2992         u32 alignment, offset, aux_offset = plane_state->color_plane[1].offset;
2993
2994         if (w > max_width || h > max_height) {
2995                 DRM_DEBUG_KMS("requested Y/RGB source size %dx%d too big (limit %dx%d)\n",
2996                               w, h, max_width, max_height);
2997                 return -EINVAL;
2998         }
2999
3000         intel_add_fb_offsets(&x, &y, plane_state, 0);
3001         offset = intel_plane_compute_aligned_offset(&x, &y, plane_state, 0);
3002         alignment = intel_surf_alignment(fb, 0);
3003
3004         /*
3005          * AUX surface offset is specified as the distance from the
3006          * main surface offset, and it must be non-negative. Make
3007          * sure that is what we will get.
3008          */
3009         if (offset > aux_offset)
3010                 offset = intel_plane_adjust_aligned_offset(&x, &y, plane_state, 0,
3011                                                            offset, aux_offset & ~(alignment - 1));
3012
3013         /*
3014          * When using an X-tiled surface, the plane blows up
3015          * if the x offset + width exceed the stride.
3016          *
3017          * TODO: linear and Y-tiled seem fine, Yf untested,
3018          */
3019         if (fb->modifier == I915_FORMAT_MOD_X_TILED) {
3020                 int cpp = fb->format->cpp[0];
3021
3022                 while ((x + w) * cpp > plane_state->color_plane[0].stride) {
3023                         if (offset == 0) {
3024                                 DRM_DEBUG_KMS("Unable to find suitable display surface offset due to X-tiling\n");
3025                                 return -EINVAL;
3026                         }
3027
3028                         offset = intel_plane_adjust_aligned_offset(&x, &y, plane_state, 0,
3029                                                                    offset, offset - alignment);
3030                 }
3031         }
3032
3033         /*
3034          * CCS AUX surface doesn't have its own x/y offsets, we must make sure
3035          * they match with the main surface x/y offsets.
3036          */
3037         if (is_ccs_modifier(fb->modifier)) {
3038                 while (!skl_check_main_ccs_coordinates(plane_state, x, y, offset)) {
3039                         if (offset == 0)
3040                                 break;
3041
3042                         offset = intel_plane_adjust_aligned_offset(&x, &y, plane_state, 0,
3043                                                                    offset, offset - alignment);
3044                 }
3045
3046                 if (x != plane_state->color_plane[1].x || y != plane_state->color_plane[1].y) {
3047                         DRM_DEBUG_KMS("Unable to find suitable display surface offset due to CCS\n");
3048                         return -EINVAL;
3049                 }
3050         }
3051
3052         plane_state->color_plane[0].offset = offset;
3053         plane_state->color_plane[0].x = x;
3054         plane_state->color_plane[0].y = y;
3055
3056         return 0;
3057 }
3058
3059 static int skl_check_nv12_aux_surface(struct intel_plane_state *plane_state)
3060 {
3061         const struct drm_framebuffer *fb = plane_state->base.fb;
3062         unsigned int rotation = plane_state->base.rotation;
3063         int max_width = skl_max_plane_width(fb, 1, rotation);
3064         int max_height = 4096;
3065         int x = plane_state->base.src.x1 >> 17;
3066         int y = plane_state->base.src.y1 >> 17;
3067         int w = drm_rect_width(&plane_state->base.src) >> 17;
3068         int h = drm_rect_height(&plane_state->base.src) >> 17;
3069         u32 offset;
3070
3071         intel_add_fb_offsets(&x, &y, plane_state, 1);
3072         offset = intel_plane_compute_aligned_offset(&x, &y, plane_state, 1);
3073
3074         /* FIXME not quite sure how/if these apply to the chroma plane */
3075         if (w > max_width || h > max_height) {
3076                 DRM_DEBUG_KMS("CbCr source size %dx%d too big (limit %dx%d)\n",
3077                               w, h, max_width, max_height);
3078                 return -EINVAL;
3079         }
3080
3081         plane_state->color_plane[1].offset = offset;
3082         plane_state->color_plane[1].x = x;
3083         plane_state->color_plane[1].y = y;
3084
3085         return 0;
3086 }
3087
3088 static int skl_check_ccs_aux_surface(struct intel_plane_state *plane_state)
3089 {
3090         const struct drm_framebuffer *fb = plane_state->base.fb;
3091         int src_x = plane_state->base.src.x1 >> 16;
3092         int src_y = plane_state->base.src.y1 >> 16;
3093         int hsub = fb->format->hsub;
3094         int vsub = fb->format->vsub;
3095         int x = src_x / hsub;
3096         int y = src_y / vsub;
3097         u32 offset;
3098
3099         intel_add_fb_offsets(&x, &y, plane_state, 1);
3100         offset = intel_plane_compute_aligned_offset(&x, &y, plane_state, 1);
3101
3102         plane_state->color_plane[1].offset = offset;
3103         plane_state->color_plane[1].x = x * hsub + src_x % hsub;
3104         plane_state->color_plane[1].y = y * vsub + src_y % vsub;
3105
3106         return 0;
3107 }
3108
3109 int skl_check_plane_surface(struct intel_plane_state *plane_state)
3110 {
3111         const struct drm_framebuffer *fb = plane_state->base.fb;
3112         unsigned int rotation = plane_state->base.rotation;
3113         int ret;
3114
3115         intel_fill_fb_ggtt_view(&plane_state->view, fb, rotation);
3116         plane_state->color_plane[0].stride = intel_fb_pitch(fb, 0, rotation);
3117         plane_state->color_plane[1].stride = intel_fb_pitch(fb, 1, rotation);
3118
3119         ret = intel_plane_check_stride(plane_state);
3120         if (ret)
3121                 return ret;
3122
3123         if (!plane_state->base.visible)
3124                 return 0;
3125
3126         /* Rotate src coordinates to match rotated GTT view */
3127         if (drm_rotation_90_or_270(rotation))
3128                 drm_rect_rotate(&plane_state->base.src,
3129                                 fb->width << 16, fb->height << 16,
3130                                 DRM_MODE_ROTATE_270);
3131
3132         /*
3133          * Handle the AUX surface first since
3134          * the main surface setup depends on it.
3135          */
3136         if (fb->format->format == DRM_FORMAT_NV12) {
3137                 ret = skl_check_nv12_aux_surface(plane_state);
3138                 if (ret)
3139                         return ret;
3140         } else if (is_ccs_modifier(fb->modifier)) {
3141                 ret = skl_check_ccs_aux_surface(plane_state);
3142                 if (ret)
3143                         return ret;
3144         } else {
3145                 plane_state->color_plane[1].offset = ~0xfff;
3146                 plane_state->color_plane[1].x = 0;
3147                 plane_state->color_plane[1].y = 0;
3148         }
3149
3150         ret = skl_check_main_surface(plane_state);
3151         if (ret)
3152                 return ret;
3153
3154         return 0;
3155 }
3156
3157 unsigned int
3158 i9xx_plane_max_stride(struct intel_plane *plane,
3159                       u32 pixel_format, u64 modifier,
3160                       unsigned int rotation)
3161 {
3162         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
3163
3164         if (!HAS_GMCH_DISPLAY(dev_priv)) {
3165                 return 32*1024;
3166         } else if (INTEL_GEN(dev_priv) >= 4) {
3167                 if (modifier == I915_FORMAT_MOD_X_TILED)
3168                         return 16*1024;
3169                 else
3170                         return 32*1024;
3171         } else if (INTEL_GEN(dev_priv) >= 3) {
3172                 if (modifier == I915_FORMAT_MOD_X_TILED)
3173                         return 8*1024;
3174                 else
3175                         return 16*1024;
3176         } else {
3177                 if (plane->i9xx_plane == PLANE_C)
3178                         return 4*1024;
3179                 else
3180                         return 8*1024;
3181         }
3182 }
3183
3184 static u32 i9xx_plane_ctl(const struct intel_crtc_state *crtc_state,
3185                           const struct intel_plane_state *plane_state)
3186 {
3187         struct drm_i915_private *dev_priv =
3188                 to_i915(plane_state->base.plane->dev);
3189         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
3190         const struct drm_framebuffer *fb = plane_state->base.fb;
3191         unsigned int rotation = plane_state->base.rotation;
3192         u32 dspcntr;
3193
3194         dspcntr = DISPLAY_PLANE_ENABLE | DISPPLANE_GAMMA_ENABLE;
3195
3196         if (IS_G4X(dev_priv) || IS_GEN5(dev_priv) ||
3197             IS_GEN6(dev_priv) || IS_IVYBRIDGE(dev_priv))
3198                 dspcntr |= DISPPLANE_TRICKLE_FEED_DISABLE;
3199
3200         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3201                 dspcntr |= DISPPLANE_PIPE_CSC_ENABLE;
3202
3203         if (INTEL_GEN(dev_priv) < 5)
3204                 dspcntr |= DISPPLANE_SEL_PIPE(crtc->pipe);
3205
3206         switch (fb->format->format) {
3207         case DRM_FORMAT_C8:
3208                 dspcntr |= DISPPLANE_8BPP;
3209                 break;
3210         case DRM_FORMAT_XRGB1555:
3211                 dspcntr |= DISPPLANE_BGRX555;
3212                 break;
3213         case DRM_FORMAT_RGB565:
3214                 dspcntr |= DISPPLANE_BGRX565;
3215                 break;
3216         case DRM_FORMAT_XRGB8888:
3217                 dspcntr |= DISPPLANE_BGRX888;
3218                 break;
3219         case DRM_FORMAT_XBGR8888:
3220                 dspcntr |= DISPPLANE_RGBX888;
3221                 break;
3222         case DRM_FORMAT_XRGB2101010:
3223                 dspcntr |= DISPPLANE_BGRX101010;
3224                 break;
3225         case DRM_FORMAT_XBGR2101010:
3226                 dspcntr |= DISPPLANE_RGBX101010;
3227                 break;
3228         default:
3229                 MISSING_CASE(fb->format->format);
3230                 return 0;
3231         }
3232
3233         if (INTEL_GEN(dev_priv) >= 4 &&
3234             fb->modifier == I915_FORMAT_MOD_X_TILED)
3235                 dspcntr |= DISPPLANE_TILED;
3236
3237         if (rotation & DRM_MODE_ROTATE_180)
3238                 dspcntr |= DISPPLANE_ROTATE_180;
3239
3240         if (rotation & DRM_MODE_REFLECT_X)
3241                 dspcntr |= DISPPLANE_MIRROR;
3242
3243         return dspcntr;
3244 }
3245
3246 int i9xx_check_plane_surface(struct intel_plane_state *plane_state)
3247 {
3248         struct drm_i915_private *dev_priv =
3249                 to_i915(plane_state->base.plane->dev);
3250         const struct drm_framebuffer *fb = plane_state->base.fb;
3251         unsigned int rotation = plane_state->base.rotation;
3252         int src_x = plane_state->base.src.x1 >> 16;
3253         int src_y = plane_state->base.src.y1 >> 16;
3254         u32 offset;
3255         int ret;
3256
3257         intel_fill_fb_ggtt_view(&plane_state->view, fb, rotation);
3258         plane_state->color_plane[0].stride = intel_fb_pitch(fb, 0, rotation);
3259
3260         ret = intel_plane_check_stride(plane_state);
3261         if (ret)
3262                 return ret;
3263
3264         intel_add_fb_offsets(&src_x, &src_y, plane_state, 0);
3265
3266         if (INTEL_GEN(dev_priv) >= 4)
3267                 offset = intel_plane_compute_aligned_offset(&src_x, &src_y,
3268                                                             plane_state, 0);
3269         else
3270                 offset = 0;
3271
3272         /* HSW/BDW do this automagically in hardware */
3273         if (!IS_HASWELL(dev_priv) && !IS_BROADWELL(dev_priv)) {
3274                 int src_w = drm_rect_width(&plane_state->base.src) >> 16;
3275                 int src_h = drm_rect_height(&plane_state->base.src) >> 16;
3276
3277                 if (rotation & DRM_MODE_ROTATE_180) {
3278                         src_x += src_w - 1;
3279                         src_y += src_h - 1;
3280                 } else if (rotation & DRM_MODE_REFLECT_X) {
3281                         src_x += src_w - 1;
3282                 }
3283         }
3284
3285         plane_state->color_plane[0].offset = offset;
3286         plane_state->color_plane[0].x = src_x;
3287         plane_state->color_plane[0].y = src_y;
3288
3289         return 0;
3290 }
3291
3292 static int
3293 i9xx_plane_check(struct intel_crtc_state *crtc_state,
3294                  struct intel_plane_state *plane_state)
3295 {
3296         int ret;
3297
3298         ret = chv_plane_check_rotation(plane_state);
3299         if (ret)
3300                 return ret;
3301
3302         ret = drm_atomic_helper_check_plane_state(&plane_state->base,
3303                                                   &crtc_state->base,
3304                                                   DRM_PLANE_HELPER_NO_SCALING,
3305                                                   DRM_PLANE_HELPER_NO_SCALING,
3306                                                   false, true);
3307         if (ret)
3308                 return ret;
3309
3310         if (!plane_state->base.visible)
3311                 return 0;
3312
3313         ret = intel_plane_check_src_coordinates(plane_state);
3314         if (ret)
3315                 return ret;
3316
3317         ret = i9xx_check_plane_surface(plane_state);
3318         if (ret)
3319                 return ret;
3320
3321         plane_state->ctl = i9xx_plane_ctl(crtc_state, plane_state);
3322
3323         return 0;
3324 }
3325
3326 static void i9xx_update_plane(struct intel_plane *plane,
3327                               const struct intel_crtc_state *crtc_state,
3328                               const struct intel_plane_state *plane_state)
3329 {
3330         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
3331         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
3332         u32 linear_offset;
3333         u32 dspcntr = plane_state->ctl;
3334         int x = plane_state->color_plane[0].x;
3335         int y = plane_state->color_plane[0].y;
3336         unsigned long irqflags;
3337         u32 dspaddr_offset;
3338
3339         linear_offset = intel_fb_xy_to_linear(x, y, plane_state, 0);
3340
3341         if (INTEL_GEN(dev_priv) >= 4)
3342                 dspaddr_offset = plane_state->color_plane[0].offset;
3343         else
3344                 dspaddr_offset = linear_offset;
3345
3346         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
3347
3348         I915_WRITE_FW(DSPSTRIDE(i9xx_plane), plane_state->color_plane[0].stride);
3349
3350         if (INTEL_GEN(dev_priv) < 4) {
3351                 /* pipesrc and dspsize control the size that is scaled from,
3352                  * which should always be the user's requested size.
3353                  */
3354                 I915_WRITE_FW(DSPPOS(i9xx_plane), 0);
3355                 I915_WRITE_FW(DSPSIZE(i9xx_plane),
3356                               ((crtc_state->pipe_src_h - 1) << 16) |
3357                               (crtc_state->pipe_src_w - 1));
3358         } else if (IS_CHERRYVIEW(dev_priv) && i9xx_plane == PLANE_B) {
3359                 I915_WRITE_FW(PRIMPOS(i9xx_plane), 0);
3360                 I915_WRITE_FW(PRIMSIZE(i9xx_plane),
3361                               ((crtc_state->pipe_src_h - 1) << 16) |
3362                               (crtc_state->pipe_src_w - 1));
3363                 I915_WRITE_FW(PRIMCNSTALPHA(i9xx_plane), 0);
3364         }
3365
3366         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
3367                 I915_WRITE_FW(DSPOFFSET(i9xx_plane), (y << 16) | x);
3368         } else if (INTEL_GEN(dev_priv) >= 4) {
3369                 I915_WRITE_FW(DSPLINOFF(i9xx_plane), linear_offset);
3370                 I915_WRITE_FW(DSPTILEOFF(i9xx_plane), (y << 16) | x);
3371         }
3372
3373         /*
3374          * The control register self-arms if the plane was previously
3375          * disabled. Try to make the plane enable atomic by writing
3376          * the control register just before the surface register.
3377          */
3378         I915_WRITE_FW(DSPCNTR(i9xx_plane), dspcntr);
3379         if (INTEL_GEN(dev_priv) >= 4)
3380                 I915_WRITE_FW(DSPSURF(i9xx_plane),
3381                               intel_plane_ggtt_offset(plane_state) +
3382                               dspaddr_offset);
3383         else
3384                 I915_WRITE_FW(DSPADDR(i9xx_plane),
3385                               intel_plane_ggtt_offset(plane_state) +
3386                               dspaddr_offset);
3387
3388         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
3389 }
3390
3391 static void i9xx_disable_plane(struct intel_plane *plane,
3392                                const struct intel_crtc_state *crtc_state)
3393 {
3394         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
3395         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
3396         unsigned long irqflags;
3397
3398         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
3399
3400         I915_WRITE_FW(DSPCNTR(i9xx_plane), 0);
3401         if (INTEL_GEN(dev_priv) >= 4)
3402                 I915_WRITE_FW(DSPSURF(i9xx_plane), 0);
3403         else
3404                 I915_WRITE_FW(DSPADDR(i9xx_plane), 0);
3405
3406         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
3407 }
3408
3409 static bool i9xx_plane_get_hw_state(struct intel_plane *plane,
3410                                     enum pipe *pipe)
3411 {
3412         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
3413         enum intel_display_power_domain power_domain;
3414         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
3415         bool ret;
3416         u32 val;
3417
3418         /*
3419          * Not 100% correct for planes that can move between pipes,
3420          * but that's only the case for gen2-4 which don't have any
3421          * display power wells.
3422          */
3423         power_domain = POWER_DOMAIN_PIPE(plane->pipe);
3424         if (!intel_display_power_get_if_enabled(dev_priv, power_domain))
3425                 return false;
3426
3427         val = I915_READ(DSPCNTR(i9xx_plane));
3428
3429         ret = val & DISPLAY_PLANE_ENABLE;
3430
3431         if (INTEL_GEN(dev_priv) >= 5)
3432                 *pipe = plane->pipe;
3433         else
3434                 *pipe = (val & DISPPLANE_SEL_PIPE_MASK) >>
3435                         DISPPLANE_SEL_PIPE_SHIFT;
3436
3437         intel_display_power_put(dev_priv, power_domain);
3438
3439         return ret;
3440 }
3441
3442 static u32
3443 intel_fb_stride_alignment(const struct drm_framebuffer *fb, int color_plane)
3444 {
3445         if (fb->modifier == DRM_FORMAT_MOD_LINEAR)
3446                 return 64;
3447         else
3448                 return intel_tile_width_bytes(fb, color_plane);
3449 }
3450
3451 static void skl_detach_scaler(struct intel_crtc *intel_crtc, int id)
3452 {
3453         struct drm_device *dev = intel_crtc->base.dev;
3454         struct drm_i915_private *dev_priv = to_i915(dev);
3455
3456         I915_WRITE(SKL_PS_CTRL(intel_crtc->pipe, id), 0);
3457         I915_WRITE(SKL_PS_WIN_POS(intel_crtc->pipe, id), 0);
3458         I915_WRITE(SKL_PS_WIN_SZ(intel_crtc->pipe, id), 0);
3459 }
3460
3461 /*
3462  * This function detaches (aka. unbinds) unused scalers in hardware
3463  */
3464 static void skl_detach_scalers(const struct intel_crtc_state *crtc_state)
3465 {
3466         struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc);
3467         const struct intel_crtc_scaler_state *scaler_state =
3468                 &crtc_state->scaler_state;
3469         int i;
3470
3471         /* loop through and disable scalers that aren't in use */
3472         for (i = 0; i < intel_crtc->num_scalers; i++) {
3473                 if (!scaler_state->scalers[i].in_use)
3474                         skl_detach_scaler(intel_crtc, i);
3475         }
3476 }
3477
3478 static unsigned int skl_plane_stride_mult(const struct drm_framebuffer *fb,
3479                                           int color_plane, unsigned int rotation)
3480 {
3481         /*
3482          * The stride is either expressed as a multiple of 64 bytes chunks for
3483          * linear buffers or in number of tiles for tiled buffers.
3484          */
3485         if (fb->modifier == DRM_FORMAT_MOD_LINEAR)
3486                 return 64;
3487         else if (drm_rotation_90_or_270(rotation))
3488                 return intel_tile_height(fb, color_plane);
3489         else
3490                 return intel_tile_width_bytes(fb, color_plane);
3491 }
3492
3493 u32 skl_plane_stride(const struct intel_plane_state *plane_state,
3494                      int color_plane)
3495 {
3496         const struct drm_framebuffer *fb = plane_state->base.fb;
3497         unsigned int rotation = plane_state->base.rotation;
3498         u32 stride = plane_state->color_plane[color_plane].stride;
3499
3500         if (color_plane >= fb->format->num_planes)
3501                 return 0;
3502
3503         return stride / skl_plane_stride_mult(fb, color_plane, rotation);
3504 }
3505
3506 static u32 skl_plane_ctl_format(uint32_t pixel_format)
3507 {
3508         switch (pixel_format) {
3509         case DRM_FORMAT_C8:
3510                 return PLANE_CTL_FORMAT_INDEXED;
3511         case DRM_FORMAT_RGB565:
3512                 return PLANE_CTL_FORMAT_RGB_565;
3513         case DRM_FORMAT_XBGR8888:
3514         case DRM_FORMAT_ABGR8888:
3515                 return PLANE_CTL_FORMAT_XRGB_8888 | PLANE_CTL_ORDER_RGBX;
3516         case DRM_FORMAT_XRGB8888:
3517         case DRM_FORMAT_ARGB8888:
3518                 return PLANE_CTL_FORMAT_XRGB_8888;
3519         case DRM_FORMAT_XRGB2101010:
3520                 return PLANE_CTL_FORMAT_XRGB_2101010;
3521         case DRM_FORMAT_XBGR2101010:
3522                 return PLANE_CTL_ORDER_RGBX | PLANE_CTL_FORMAT_XRGB_2101010;
3523         case DRM_FORMAT_YUYV:
3524                 return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_YUYV;
3525         case DRM_FORMAT_YVYU:
3526                 return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_YVYU;
3527         case DRM_FORMAT_UYVY:
3528                 return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_UYVY;
3529         case DRM_FORMAT_VYUY:
3530                 return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_VYUY;
3531         case DRM_FORMAT_NV12:
3532                 return PLANE_CTL_FORMAT_NV12;
3533         default:
3534                 MISSING_CASE(pixel_format);
3535         }
3536
3537         return 0;
3538 }
3539
3540 static u32 skl_plane_ctl_alpha(const struct intel_plane_state *plane_state)
3541 {
3542         if (!plane_state->base.fb->format->has_alpha)
3543                 return PLANE_CTL_ALPHA_DISABLE;
3544
3545         switch (plane_state->base.pixel_blend_mode) {
3546         case DRM_MODE_BLEND_PIXEL_NONE:
3547                 return PLANE_CTL_ALPHA_DISABLE;
3548         case DRM_MODE_BLEND_PREMULTI:
3549                 return PLANE_CTL_ALPHA_SW_PREMULTIPLY;
3550         case DRM_MODE_BLEND_COVERAGE:
3551                 return PLANE_CTL_ALPHA_HW_PREMULTIPLY;
3552         default:
3553                 MISSING_CASE(plane_state->base.pixel_blend_mode);
3554                 return PLANE_CTL_ALPHA_DISABLE;
3555         }
3556 }
3557
3558 static u32 glk_plane_color_ctl_alpha(const struct intel_plane_state *plane_state)
3559 {
3560         if (!plane_state->base.fb->format->has_alpha)
3561                 return PLANE_COLOR_ALPHA_DISABLE;
3562
3563         switch (plane_state->base.pixel_blend_mode) {
3564         case DRM_MODE_BLEND_PIXEL_NONE:
3565                 return PLANE_COLOR_ALPHA_DISABLE;
3566         case DRM_MODE_BLEND_PREMULTI:
3567                 return PLANE_COLOR_ALPHA_SW_PREMULTIPLY;
3568         case DRM_MODE_BLEND_COVERAGE:
3569                 return PLANE_COLOR_ALPHA_HW_PREMULTIPLY;
3570         default:
3571                 MISSING_CASE(plane_state->base.pixel_blend_mode);
3572                 return PLANE_COLOR_ALPHA_DISABLE;
3573         }
3574 }
3575
3576 static u32 skl_plane_ctl_tiling(uint64_t fb_modifier)
3577 {
3578         switch (fb_modifier) {
3579         case DRM_FORMAT_MOD_LINEAR:
3580                 break;
3581         case I915_FORMAT_MOD_X_TILED:
3582                 return PLANE_CTL_TILED_X;
3583         case I915_FORMAT_MOD_Y_TILED:
3584                 return PLANE_CTL_TILED_Y;
3585         case I915_FORMAT_MOD_Y_TILED_CCS:
3586                 return PLANE_CTL_TILED_Y | PLANE_CTL_RENDER_DECOMPRESSION_ENABLE;
3587         case I915_FORMAT_MOD_Yf_TILED:
3588                 return PLANE_CTL_TILED_YF;
3589         case I915_FORMAT_MOD_Yf_TILED_CCS:
3590                 return PLANE_CTL_TILED_YF | PLANE_CTL_RENDER_DECOMPRESSION_ENABLE;
3591         default:
3592                 MISSING_CASE(fb_modifier);
3593         }
3594
3595         return 0;
3596 }
3597
3598 static u32 skl_plane_ctl_rotate(unsigned int rotate)
3599 {
3600         switch (rotate) {
3601         case DRM_MODE_ROTATE_0:
3602                 break;
3603         /*
3604          * DRM_MODE_ROTATE_ is counter clockwise to stay compatible with Xrandr
3605          * while i915 HW rotation is clockwise, thats why this swapping.
3606          */
3607         case DRM_MODE_ROTATE_90:
3608                 return PLANE_CTL_ROTATE_270;
3609         case DRM_MODE_ROTATE_180:
3610                 return PLANE_CTL_ROTATE_180;
3611         case DRM_MODE_ROTATE_270:
3612                 return PLANE_CTL_ROTATE_90;
3613         default:
3614                 MISSING_CASE(rotate);
3615         }
3616
3617         return 0;
3618 }
3619
3620 static u32 cnl_plane_ctl_flip(unsigned int reflect)
3621 {
3622         switch (reflect) {
3623         case 0:
3624                 break;
3625         case DRM_MODE_REFLECT_X:
3626                 return PLANE_CTL_FLIP_HORIZONTAL;
3627         case DRM_MODE_REFLECT_Y:
3628         default:
3629                 MISSING_CASE(reflect);
3630         }
3631
3632         return 0;
3633 }
3634
3635 u32 skl_plane_ctl(const struct intel_crtc_state *crtc_state,
3636                   const struct intel_plane_state *plane_state)
3637 {
3638         struct drm_i915_private *dev_priv =
3639                 to_i915(plane_state->base.plane->dev);
3640         const struct drm_framebuffer *fb = plane_state->base.fb;
3641         unsigned int rotation = plane_state->base.rotation;
3642         const struct drm_intel_sprite_colorkey *key = &plane_state->ckey;
3643         u32 plane_ctl;
3644
3645         plane_ctl = PLANE_CTL_ENABLE;
3646
3647         if (INTEL_GEN(dev_priv) < 10 && !IS_GEMINILAKE(dev_priv)) {
3648                 plane_ctl |= skl_plane_ctl_alpha(plane_state);
3649                 plane_ctl |=
3650                         PLANE_CTL_PIPE_GAMMA_ENABLE |
3651                         PLANE_CTL_PIPE_CSC_ENABLE |
3652                         PLANE_CTL_PLANE_GAMMA_DISABLE;
3653
3654                 if (plane_state->base.color_encoding == DRM_COLOR_YCBCR_BT709)
3655                         plane_ctl |= PLANE_CTL_YUV_TO_RGB_CSC_FORMAT_BT709;
3656
3657                 if (plane_state->base.color_range == DRM_COLOR_YCBCR_FULL_RANGE)
3658                         plane_ctl |= PLANE_CTL_YUV_RANGE_CORRECTION_DISABLE;
3659         }
3660
3661         plane_ctl |= skl_plane_ctl_format(fb->format->format);
3662         plane_ctl |= skl_plane_ctl_tiling(fb->modifier);
3663         plane_ctl |= skl_plane_ctl_rotate(rotation & DRM_MODE_ROTATE_MASK);
3664
3665         if (INTEL_GEN(dev_priv) >= 10)
3666                 plane_ctl |= cnl_plane_ctl_flip(rotation &
3667                                                 DRM_MODE_REFLECT_MASK);
3668
3669         if (key->flags & I915_SET_COLORKEY_DESTINATION)
3670                 plane_ctl |= PLANE_CTL_KEY_ENABLE_DESTINATION;
3671         else if (key->flags & I915_SET_COLORKEY_SOURCE)
3672                 plane_ctl |= PLANE_CTL_KEY_ENABLE_SOURCE;
3673
3674         return plane_ctl;
3675 }
3676
3677 u32 glk_plane_color_ctl(const struct intel_crtc_state *crtc_state,
3678                         const struct intel_plane_state *plane_state)
3679 {
3680         struct drm_i915_private *dev_priv =
3681                 to_i915(plane_state->base.plane->dev);
3682         const struct drm_framebuffer *fb = plane_state->base.fb;
3683         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
3684         u32 plane_color_ctl = 0;
3685
3686         if (INTEL_GEN(dev_priv) < 11) {
3687                 plane_color_ctl |= PLANE_COLOR_PIPE_GAMMA_ENABLE;
3688                 plane_color_ctl |= PLANE_COLOR_PIPE_CSC_ENABLE;
3689         }
3690         plane_color_ctl |= PLANE_COLOR_PLANE_GAMMA_DISABLE;
3691         plane_color_ctl |= glk_plane_color_ctl_alpha(plane_state);
3692
3693         if (fb->format->is_yuv && !icl_is_hdr_plane(plane)) {
3694                 if (plane_state->base.color_encoding == DRM_COLOR_YCBCR_BT709)
3695                         plane_color_ctl |= PLANE_COLOR_CSC_MODE_YUV709_TO_RGB709;
3696                 else
3697                         plane_color_ctl |= PLANE_COLOR_CSC_MODE_YUV601_TO_RGB709;
3698
3699                 if (plane_state->base.color_range == DRM_COLOR_YCBCR_FULL_RANGE)
3700                         plane_color_ctl |= PLANE_COLOR_YUV_RANGE_CORRECTION_DISABLE;
3701         } else if (fb->format->is_yuv) {
3702                 plane_color_ctl |= PLANE_COLOR_INPUT_CSC_ENABLE;
3703         }
3704
3705         return plane_color_ctl;
3706 }
3707
3708 static int
3709 __intel_display_resume(struct drm_device *dev,
3710                        struct drm_atomic_state *state,
3711                        struct drm_modeset_acquire_ctx *ctx)
3712 {
3713         struct drm_crtc_state *crtc_state;
3714         struct drm_crtc *crtc;
3715         int i, ret;
3716
3717         intel_modeset_setup_hw_state(dev, ctx);
3718         i915_redisable_vga(to_i915(dev));
3719
3720         if (!state)
3721                 return 0;
3722
3723         /*
3724          * We've duplicated the state, pointers to the old state are invalid.
3725          *
3726          * Don't attempt to use the old state until we commit the duplicated state.
3727          */
3728         for_each_new_crtc_in_state(state, crtc, crtc_state, i) {
3729                 /*
3730                  * Force recalculation even if we restore
3731                  * current state. With fast modeset this may not result
3732                  * in a modeset when the state is compatible.
3733                  */
3734                 crtc_state->mode_changed = true;
3735         }
3736
3737         /* ignore any reset values/BIOS leftovers in the WM registers */
3738         if (!HAS_GMCH_DISPLAY(to_i915(dev)))
3739                 to_intel_atomic_state(state)->skip_intermediate_wm = true;
3740
3741         ret = drm_atomic_helper_commit_duplicated_state(state, ctx);
3742
3743         WARN_ON(ret == -EDEADLK);
3744         return ret;
3745 }
3746
3747 static bool gpu_reset_clobbers_display(struct drm_i915_private *dev_priv)
3748 {
3749         return intel_has_gpu_reset(dev_priv) &&
3750                 INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv);
3751 }
3752
3753 void intel_prepare_reset(struct drm_i915_private *dev_priv)
3754 {
3755         struct drm_device *dev = &dev_priv->drm;
3756         struct drm_modeset_acquire_ctx *ctx = &dev_priv->reset_ctx;
3757         struct drm_atomic_state *state;
3758         int ret;
3759
3760         /* reset doesn't touch the display */
3761         if (!i915_modparams.force_reset_modeset_test &&
3762             !gpu_reset_clobbers_display(dev_priv))
3763                 return;
3764
3765         /* We have a modeset vs reset deadlock, defensively unbreak it. */
3766         set_bit(I915_RESET_MODESET, &dev_priv->gpu_error.flags);
3767         wake_up_all(&dev_priv->gpu_error.wait_queue);
3768
3769         if (atomic_read(&dev_priv->gpu_error.pending_fb_pin)) {
3770                 DRM_DEBUG_KMS("Modeset potentially stuck, unbreaking through wedging\n");
3771                 i915_gem_set_wedged(dev_priv);
3772         }
3773
3774         /*
3775          * Need mode_config.mutex so that we don't
3776          * trample ongoing ->detect() and whatnot.
3777          */
3778         mutex_lock(&dev->mode_config.mutex);
3779         drm_modeset_acquire_init(ctx, 0);
3780         while (1) {
3781                 ret = drm_modeset_lock_all_ctx(dev, ctx);
3782                 if (ret != -EDEADLK)
3783                         break;
3784
3785                 drm_modeset_backoff(ctx);
3786         }
3787         /*
3788          * Disabling the crtcs gracefully seems nicer. Also the
3789          * g33 docs say we should at least disable all the planes.
3790          */
3791         state = drm_atomic_helper_duplicate_state(dev, ctx);
3792         if (IS_ERR(state)) {
3793                 ret = PTR_ERR(state);
3794                 DRM_ERROR("Duplicating state failed with %i\n", ret);
3795                 return;
3796         }
3797
3798         ret = drm_atomic_helper_disable_all(dev, ctx);
3799         if (ret) {
3800                 DRM_ERROR("Suspending crtc's failed with %i\n", ret);
3801                 drm_atomic_state_put(state);
3802                 return;
3803         }
3804
3805         dev_priv->modeset_restore_state = state;
3806         state->acquire_ctx = ctx;
3807 }
3808
3809 void intel_finish_reset(struct drm_i915_private *dev_priv)
3810 {
3811         struct drm_device *dev = &dev_priv->drm;
3812         struct drm_modeset_acquire_ctx *ctx = &dev_priv->reset_ctx;
3813         struct drm_atomic_state *state;
3814         int ret;
3815
3816         /* reset doesn't touch the display */
3817         if (!test_bit(I915_RESET_MODESET, &dev_priv->gpu_error.flags))
3818                 return;
3819
3820         state = fetch_and_zero(&dev_priv->modeset_restore_state);
3821         if (!state)
3822                 goto unlock;
3823
3824         /* reset doesn't touch the display */
3825         if (!gpu_reset_clobbers_display(dev_priv)) {
3826                 /* for testing only restore the display */
3827                 ret = __intel_display_resume(dev, state, ctx);
3828                 if (ret)
3829                         DRM_ERROR("Restoring old state failed with %i\n", ret);
3830         } else {
3831                 /*
3832                  * The display has been reset as well,
3833                  * so need a full re-initialization.
3834                  */
3835                 intel_runtime_pm_disable_interrupts(dev_priv);
3836                 intel_runtime_pm_enable_interrupts(dev_priv);
3837
3838                 intel_pps_unlock_regs_wa(dev_priv);
3839                 intel_modeset_init_hw(dev);
3840                 intel_init_clock_gating(dev_priv);
3841
3842                 spin_lock_irq(&dev_priv->irq_lock);
3843                 if (dev_priv->display.hpd_irq_setup)
3844                         dev_priv->display.hpd_irq_setup(dev_priv);
3845                 spin_unlock_irq(&dev_priv->irq_lock);
3846
3847                 ret = __intel_display_resume(dev, state, ctx);
3848                 if (ret)
3849                         DRM_ERROR("Restoring old state failed with %i\n", ret);
3850
3851                 intel_hpd_init(dev_priv);
3852         }
3853
3854         drm_atomic_state_put(state);
3855 unlock:
3856         drm_modeset_drop_locks(ctx);
3857         drm_modeset_acquire_fini(ctx);
3858         mutex_unlock(&dev->mode_config.mutex);
3859
3860         clear_bit(I915_RESET_MODESET, &dev_priv->gpu_error.flags);
3861 }
3862
3863 static void intel_update_pipe_config(const struct intel_crtc_state *old_crtc_state,
3864                                      const struct intel_crtc_state *new_crtc_state)
3865 {
3866         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->base.crtc);
3867         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3868
3869         /* drm_atomic_helper_update_legacy_modeset_state might not be called. */
3870         crtc->base.mode = new_crtc_state->base.mode;
3871
3872         /*
3873          * Update pipe size and adjust fitter if needed: the reason for this is
3874          * that in compute_mode_changes we check the native mode (not the pfit
3875          * mode) to see if we can flip rather than do a full mode set. In the
3876          * fastboot case, we'll flip, but if we don't update the pipesrc and
3877          * pfit state, we'll end up with a big fb scanned out into the wrong
3878          * sized surface.
3879          */
3880
3881         I915_WRITE(PIPESRC(crtc->pipe),
3882                    ((new_crtc_state->pipe_src_w - 1) << 16) |
3883                    (new_crtc_state->pipe_src_h - 1));
3884
3885         /* on skylake this is done by detaching scalers */
3886         if (INTEL_GEN(dev_priv) >= 9) {
3887                 skl_detach_scalers(new_crtc_state);
3888
3889                 if (new_crtc_state->pch_pfit.enabled)
3890                         skylake_pfit_enable(new_crtc_state);
3891         } else if (HAS_PCH_SPLIT(dev_priv)) {
3892                 if (new_crtc_state->pch_pfit.enabled)
3893                         ironlake_pfit_enable(new_crtc_state);
3894                 else if (old_crtc_state->pch_pfit.enabled)
3895                         ironlake_pfit_disable(old_crtc_state);
3896         }
3897 }
3898
3899 static void intel_fdi_normal_train(struct intel_crtc *crtc)
3900 {
3901         struct drm_device *dev = crtc->base.dev;
3902         struct drm_i915_private *dev_priv = to_i915(dev);
3903         int pipe = crtc->pipe;
3904         i915_reg_t reg;
3905         u32 temp;
3906
3907         /* enable normal train */
3908         reg = FDI_TX_CTL(pipe);
3909         temp = I915_READ(reg);
3910         if (IS_IVYBRIDGE(dev_priv)) {
3911                 temp &= ~FDI_LINK_TRAIN_NONE_IVB;
3912                 temp |= FDI_LINK_TRAIN_NONE_IVB | FDI_TX_ENHANCE_FRAME_ENABLE;
3913         } else {
3914                 temp &= ~FDI_LINK_TRAIN_NONE;
3915                 temp |= FDI_LINK_TRAIN_NONE | FDI_TX_ENHANCE_FRAME_ENABLE;
3916         }
3917         I915_WRITE(reg, temp);
3918
3919         reg = FDI_RX_CTL(pipe);
3920         temp = I915_READ(reg);
3921         if (HAS_PCH_CPT(dev_priv)) {
3922                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
3923                 temp |= FDI_LINK_TRAIN_NORMAL_CPT;
3924         } else {
3925                 temp &= ~FDI_LINK_TRAIN_NONE;
3926                 temp |= FDI_LINK_TRAIN_NONE;
3927         }
3928         I915_WRITE(reg, temp | FDI_RX_ENHANCE_FRAME_ENABLE);
3929
3930         /* wait one idle pattern time */
3931         POSTING_READ(reg);
3932         udelay(1000);
3933
3934         /* IVB wants error correction enabled */
3935         if (IS_IVYBRIDGE(dev_priv))
3936                 I915_WRITE(reg, I915_READ(reg) | FDI_FS_ERRC_ENABLE |
3937                            FDI_FE_ERRC_ENABLE);
3938 }
3939
3940 /* The FDI link training functions for ILK/Ibexpeak. */
3941 static void ironlake_fdi_link_train(struct intel_crtc *crtc,
3942                                     const struct intel_crtc_state *crtc_state)
3943 {
3944         struct drm_device *dev = crtc->base.dev;
3945         struct drm_i915_private *dev_priv = to_i915(dev);
3946         int pipe = crtc->pipe;
3947         i915_reg_t reg;
3948         u32 temp, tries;
3949
3950         /* FDI needs bits from pipe first */
3951         assert_pipe_enabled(dev_priv, pipe);
3952
3953         /* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit
3954            for train result */
3955         reg = FDI_RX_IMR(pipe);
3956         temp = I915_READ(reg);
3957         temp &= ~FDI_RX_SYMBOL_LOCK;
3958         temp &= ~FDI_RX_BIT_LOCK;
3959         I915_WRITE(reg, temp);
3960         I915_READ(reg);
3961         udelay(150);
3962
3963         /* enable CPU FDI TX and PCH FDI RX */
3964         reg = FDI_TX_CTL(pipe);
3965         temp = I915_READ(reg);
3966         temp &= ~FDI_DP_PORT_WIDTH_MASK;
3967         temp |= FDI_DP_PORT_WIDTH(crtc_state->fdi_lanes);
3968         temp &= ~FDI_LINK_TRAIN_NONE;
3969         temp |= FDI_LINK_TRAIN_PATTERN_1;
3970         I915_WRITE(reg, temp | FDI_TX_ENABLE);
3971
3972         reg = FDI_RX_CTL(pipe);
3973         temp = I915_READ(reg);
3974         temp &= ~FDI_LINK_TRAIN_NONE;
3975         temp |= FDI_LINK_TRAIN_PATTERN_1;
3976         I915_WRITE(reg, temp | FDI_RX_ENABLE);
3977
3978         POSTING_READ(reg);
3979         udelay(150);
3980
3981         /* Ironlake workaround, enable clock pointer after FDI enable*/
3982         I915_WRITE(FDI_RX_CHICKEN(pipe), FDI_RX_PHASE_SYNC_POINTER_OVR);
3983         I915_WRITE(FDI_RX_CHICKEN(pipe), FDI_RX_PHASE_SYNC_POINTER_OVR |
3984                    FDI_RX_PHASE_SYNC_POINTER_EN);
3985
3986         reg = FDI_RX_IIR(pipe);
3987         for (tries = 0; tries < 5; tries++) {
3988                 temp = I915_READ(reg);
3989                 DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
3990
3991                 if ((temp & FDI_RX_BIT_LOCK)) {
3992                         DRM_DEBUG_KMS("FDI train 1 done.\n");
3993                         I915_WRITE(reg, temp | FDI_RX_BIT_LOCK);
3994                         break;
3995                 }
3996         }
3997         if (tries == 5)
3998                 DRM_ERROR("FDI train 1 fail!\n");
3999
4000         /* Train 2 */
4001         reg = FDI_TX_CTL(pipe);
4002         temp = I915_READ(reg);
4003         temp &= ~FDI_LINK_TRAIN_NONE;
4004         temp |= FDI_LINK_TRAIN_PATTERN_2;
4005         I915_WRITE(reg, temp);
4006
4007         reg = FDI_RX_CTL(pipe);
4008         temp = I915_READ(reg);
4009         temp &= ~FDI_LINK_TRAIN_NONE;
4010         temp |= FDI_LINK_TRAIN_PATTERN_2;
4011         I915_WRITE(reg, temp);
4012
4013         POSTING_READ(reg);
4014         udelay(150);
4015
4016         reg = FDI_RX_IIR(pipe);
4017         for (tries = 0; tries < 5; tries++) {
4018                 temp = I915_READ(reg);
4019                 DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
4020
4021                 if (temp & FDI_RX_SYMBOL_LOCK) {
4022                         I915_WRITE(reg, temp | FDI_RX_SYMBOL_LOCK);
4023                         DRM_DEBUG_KMS("FDI train 2 done.\n");
4024                         break;
4025                 }
4026         }
4027         if (tries == 5)
4028                 DRM_ERROR("FDI train 2 fail!\n");
4029
4030         DRM_DEBUG_KMS("FDI train done\n");
4031
4032 }
4033
4034 static const int snb_b_fdi_train_param[] = {
4035         FDI_LINK_TRAIN_400MV_0DB_SNB_B,
4036         FDI_LINK_TRAIN_400MV_6DB_SNB_B,
4037         FDI_LINK_TRAIN_600MV_3_5DB_SNB_B,
4038         FDI_LINK_TRAIN_800MV_0DB_SNB_B,
4039 };
4040
4041 /* The FDI link training functions for SNB/Cougarpoint. */
4042 static void gen6_fdi_link_train(struct intel_crtc *crtc,
4043                                 const struct intel_crtc_state *crtc_state)
4044 {
4045         struct drm_device *dev = crtc->base.dev;
4046         struct drm_i915_private *dev_priv = to_i915(dev);
4047         int pipe = crtc->pipe;
4048         i915_reg_t reg;
4049         u32 temp, i, retry;
4050
4051         /* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit
4052            for train result */
4053         reg = FDI_RX_IMR(pipe);
4054         temp = I915_READ(reg);
4055         temp &= ~FDI_RX_SYMBOL_LOCK;
4056         temp &= ~FDI_RX_BIT_LOCK;
4057         I915_WRITE(reg, temp);
4058
4059         POSTING_READ(reg);
4060         udelay(150);
4061
4062         /* enable CPU FDI TX and PCH FDI RX */
4063         reg = FDI_TX_CTL(pipe);
4064         temp = I915_READ(reg);
4065         temp &= ~FDI_DP_PORT_WIDTH_MASK;
4066         temp |= FDI_DP_PORT_WIDTH(crtc_state->fdi_lanes);
4067         temp &= ~FDI_LINK_TRAIN_NONE;
4068         temp |= FDI_LINK_TRAIN_PATTERN_1;
4069         temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
4070         /* SNB-B */
4071         temp |= FDI_LINK_TRAIN_400MV_0DB_SNB_B;
4072         I915_WRITE(reg, temp | FDI_TX_ENABLE);
4073
4074         I915_WRITE(FDI_RX_MISC(pipe),
4075                    FDI_RX_TP1_TO_TP2_48 | FDI_RX_FDI_DELAY_90);
4076
4077         reg = FDI_RX_CTL(pipe);
4078         temp = I915_READ(reg);
4079         if (HAS_PCH_CPT(dev_priv)) {
4080                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
4081                 temp |= FDI_LINK_TRAIN_PATTERN_1_CPT;
4082         } else {
4083                 temp &= ~FDI_LINK_TRAIN_NONE;
4084                 temp |= FDI_LINK_TRAIN_PATTERN_1;
4085         }
4086         I915_WRITE(reg, temp | FDI_RX_ENABLE);
4087
4088         POSTING_READ(reg);
4089         udelay(150);
4090
4091         for (i = 0; i < 4; i++) {
4092                 reg = FDI_TX_CTL(pipe);
4093                 temp = I915_READ(reg);
4094                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
4095                 temp |= snb_b_fdi_train_param[i];
4096                 I915_WRITE(reg, temp);
4097
4098                 POSTING_READ(reg);
4099                 udelay(500);
4100
4101                 for (retry = 0; retry < 5; retry++) {
4102                         reg = FDI_RX_IIR(pipe);
4103                         temp = I915_READ(reg);
4104                         DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
4105                         if (temp & FDI_RX_BIT_LOCK) {
4106                                 I915_WRITE(reg, temp | FDI_RX_BIT_LOCK);
4107                                 DRM_DEBUG_KMS("FDI train 1 done.\n");
4108                                 break;
4109                         }
4110                         udelay(50);
4111                 }
4112                 if (retry < 5)
4113                         break;
4114         }
4115         if (i == 4)
4116                 DRM_ERROR("FDI train 1 fail!\n");
4117
4118         /* Train 2 */
4119         reg = FDI_TX_CTL(pipe);
4120         temp = I915_READ(reg);
4121         temp &= ~FDI_LINK_TRAIN_NONE;
4122         temp |= FDI_LINK_TRAIN_PATTERN_2;
4123         if (IS_GEN6(dev_priv)) {
4124                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
4125                 /* SNB-B */
4126                 temp |= FDI_LINK_TRAIN_400MV_0DB_SNB_B;
4127         }
4128         I915_WRITE(reg, temp);
4129
4130         reg = FDI_RX_CTL(pipe);
4131         temp = I915_READ(reg);
4132         if (HAS_PCH_CPT(dev_priv)) {
4133                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
4134                 temp |= FDI_LINK_TRAIN_PATTERN_2_CPT;
4135         } else {
4136                 temp &= ~FDI_LINK_TRAIN_NONE;
4137                 temp |= FDI_LINK_TRAIN_PATTERN_2;
4138         }
4139         I915_WRITE(reg, temp);
4140
4141         POSTING_READ(reg);
4142         udelay(150);
4143
4144         for (i = 0; i < 4; i++) {
4145                 reg = FDI_TX_CTL(pipe);
4146                 temp = I915_READ(reg);
4147                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
4148                 temp |= snb_b_fdi_train_param[i];
4149                 I915_WRITE(reg, temp);
4150
4151                 POSTING_READ(reg);
4152                 udelay(500);
4153
4154                 for (retry = 0; retry < 5; retry++) {
4155                         reg = FDI_RX_IIR(pipe);
4156                         temp = I915_READ(reg);
4157                         DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
4158                         if (temp & FDI_RX_SYMBOL_LOCK) {
4159                                 I915_WRITE(reg, temp | FDI_RX_SYMBOL_LOCK);
4160                                 DRM_DEBUG_KMS("FDI train 2 done.\n");
4161                                 break;
4162                         }
4163                         udelay(50);
4164                 }
4165                 if (retry < 5)
4166                         break;
4167         }
4168         if (i == 4)
4169                 DRM_ERROR("FDI train 2 fail!\n");
4170
4171         DRM_DEBUG_KMS("FDI train done.\n");
4172 }
4173
4174 /* Manual link training for Ivy Bridge A0 parts */
4175 static void ivb_manual_fdi_link_train(struct intel_crtc *crtc,
4176                                       const struct intel_crtc_state *crtc_state)
4177 {
4178         struct drm_device *dev = crtc->base.dev;
4179         struct drm_i915_private *dev_priv = to_i915(dev);
4180         int pipe = crtc->pipe;
4181         i915_reg_t reg;
4182         u32 temp, i, j;
4183
4184         /* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit
4185            for train result */
4186         reg = FDI_RX_IMR(pipe);
4187         temp = I915_READ(reg);
4188         temp &= ~FDI_RX_SYMBOL_LOCK;
4189         temp &= ~FDI_RX_BIT_LOCK;
4190         I915_WRITE(reg, temp);
4191
4192         POSTING_READ(reg);
4193         udelay(150);
4194
4195         DRM_DEBUG_KMS("FDI_RX_IIR before link train 0x%x\n",
4196                       I915_READ(FDI_RX_IIR(pipe)));
4197
4198         /* Try each vswing and preemphasis setting twice before moving on */
4199         for (j = 0; j < ARRAY_SIZE(snb_b_fdi_train_param) * 2; j++) {
4200                 /* disable first in case we need to retry */
4201                 reg = FDI_TX_CTL(pipe);
4202                 temp = I915_READ(reg);
4203                 temp &= ~(FDI_LINK_TRAIN_AUTO | FDI_LINK_TRAIN_NONE_IVB);
4204                 temp &= ~FDI_TX_ENABLE;
4205                 I915_WRITE(reg, temp);
4206
4207                 reg = FDI_RX_CTL(pipe);
4208                 temp = I915_READ(reg);
4209                 temp &= ~FDI_LINK_TRAIN_AUTO;
4210                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
4211                 temp &= ~FDI_RX_ENABLE;
4212                 I915_WRITE(reg, temp);
4213
4214                 /* enable CPU FDI TX and PCH FDI RX */
4215                 reg = FDI_TX_CTL(pipe);
4216                 temp = I915_READ(reg);
4217                 temp &= ~FDI_DP_PORT_WIDTH_MASK;
4218                 temp |= FDI_DP_PORT_WIDTH(crtc_state->fdi_lanes);
4219                 temp |= FDI_LINK_TRAIN_PATTERN_1_IVB;
4220                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
4221                 temp |= snb_b_fdi_train_param[j/2];
4222                 temp |= FDI_COMPOSITE_SYNC;
4223                 I915_WRITE(reg, temp | FDI_TX_ENABLE);
4224
4225                 I915_WRITE(FDI_RX_MISC(pipe),
4226                            FDI_RX_TP1_TO_TP2_48 | FDI_RX_FDI_DELAY_90);
4227
4228                 reg = FDI_RX_CTL(pipe);
4229                 temp = I915_READ(reg);
4230                 temp |= FDI_LINK_TRAIN_PATTERN_1_CPT;
4231                 temp |= FDI_COMPOSITE_SYNC;
4232                 I915_WRITE(reg, temp | FDI_RX_ENABLE);
4233
4234                 POSTING_READ(reg);
4235                 udelay(1); /* should be 0.5us */
4236
4237                 for (i = 0; i < 4; i++) {
4238                         reg = FDI_RX_IIR(pipe);
4239                         temp = I915_READ(reg);
4240                         DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
4241
4242                         if (temp & FDI_RX_BIT_LOCK ||
4243                             (I915_READ(reg) & FDI_RX_BIT_LOCK)) {
4244                                 I915_WRITE(reg, temp | FDI_RX_BIT_LOCK);
4245                                 DRM_DEBUG_KMS("FDI train 1 done, level %i.\n",
4246                                               i);
4247                                 break;
4248                         }
4249                         udelay(1); /* should be 0.5us */
4250                 }
4251                 if (i == 4) {
4252                         DRM_DEBUG_KMS("FDI train 1 fail on vswing %d\n", j / 2);
4253                         continue;
4254                 }
4255
4256                 /* Train 2 */
4257                 reg = FDI_TX_CTL(pipe);
4258                 temp = I915_READ(reg);
4259                 temp &= ~FDI_LINK_TRAIN_NONE_IVB;
4260                 temp |= FDI_LINK_TRAIN_PATTERN_2_IVB;
4261                 I915_WRITE(reg, temp);
4262
4263                 reg = FDI_RX_CTL(pipe);
4264                 temp = I915_READ(reg);
4265                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
4266                 temp |= FDI_LINK_TRAIN_PATTERN_2_CPT;
4267                 I915_WRITE(reg, temp);
4268
4269                 POSTING_READ(reg);
4270                 udelay(2); /* should be 1.5us */
4271
4272                 for (i = 0; i < 4; i++) {
4273                         reg = FDI_RX_IIR(pipe);
4274                         temp = I915_READ(reg);
4275                         DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
4276
4277                         if (temp & FDI_RX_SYMBOL_LOCK ||
4278                             (I915_READ(reg) & FDI_RX_SYMBOL_LOCK)) {
4279                                 I915_WRITE(reg, temp | FDI_RX_SYMBOL_LOCK);
4280                                 DRM_DEBUG_KMS("FDI train 2 done, level %i.\n",
4281                                               i);
4282                                 goto train_done;
4283                         }
4284                         udelay(2); /* should be 1.5us */
4285                 }
4286                 if (i == 4)
4287                         DRM_DEBUG_KMS("FDI train 2 fail on vswing %d\n", j / 2);
4288         }
4289
4290 train_done:
4291         DRM_DEBUG_KMS("FDI train done.\n");
4292 }
4293
4294 static void ironlake_fdi_pll_enable(const struct intel_crtc_state *crtc_state)
4295 {
4296         struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc);
4297         struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
4298         int pipe = intel_crtc->pipe;
4299         i915_reg_t reg;
4300         u32 temp;
4301
4302         /* enable PCH FDI RX PLL, wait warmup plus DMI latency */
4303         reg = FDI_RX_CTL(pipe);
4304         temp = I915_READ(reg);
4305         temp &= ~(FDI_DP_PORT_WIDTH_MASK | (0x7 << 16));
4306         temp |= FDI_DP_PORT_WIDTH(crtc_state->fdi_lanes);
4307         temp |= (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) << 11;
4308         I915_WRITE(reg, temp | FDI_RX_PLL_ENABLE);
4309
4310         POSTING_READ(reg);
4311         udelay(200);
4312
4313         /* Switch from Rawclk to PCDclk */
4314         temp = I915_READ(reg);
4315         I915_WRITE(reg, temp | FDI_PCDCLK);
4316
4317         POSTING_READ(reg);
4318         udelay(200);
4319
4320         /* Enable CPU FDI TX PLL, always on for Ironlake */
4321         reg = FDI_TX_CTL(pipe);
4322         temp = I915_READ(reg);
4323         if ((temp & FDI_TX_PLL_ENABLE) == 0) {
4324                 I915_WRITE(reg, temp | FDI_TX_PLL_ENABLE);
4325
4326                 POSTING_READ(reg);
4327                 udelay(100);
4328         }
4329 }
4330
4331 static void ironlake_fdi_pll_disable(struct intel_crtc *intel_crtc)
4332 {
4333         struct drm_device *dev = intel_crtc->base.dev;
4334         struct drm_i915_private *dev_priv = to_i915(dev);
4335         int pipe = intel_crtc->pipe;
4336         i915_reg_t reg;
4337         u32 temp;
4338
4339         /* Switch from PCDclk to Rawclk */
4340         reg = FDI_RX_CTL(pipe);
4341         temp = I915_READ(reg);
4342         I915_WRITE(reg, temp & ~FDI_PCDCLK);
4343
4344         /* Disable CPU FDI TX PLL */
4345         reg = FDI_TX_CTL(pipe);
4346         temp = I915_READ(reg);
4347         I915_WRITE(reg, temp & ~FDI_TX_PLL_ENABLE);
4348
4349         POSTING_READ(reg);
4350         udelay(100);
4351
4352         reg = FDI_RX_CTL(pipe);
4353         temp = I915_READ(reg);
4354         I915_WRITE(reg, temp & ~FDI_RX_PLL_ENABLE);
4355
4356         /* Wait for the clocks to turn off. */
4357         POSTING_READ(reg);
4358         udelay(100);
4359 }
4360
4361 static void ironlake_fdi_disable(struct drm_crtc *crtc)
4362 {
4363         struct drm_device *dev = crtc->dev;
4364         struct drm_i915_private *dev_priv = to_i915(dev);
4365         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
4366         int pipe = intel_crtc->pipe;
4367         i915_reg_t reg;
4368         u32 temp;
4369
4370         /* disable CPU FDI tx and PCH FDI rx */
4371         reg = FDI_TX_CTL(pipe);
4372         temp = I915_READ(reg);
4373         I915_WRITE(reg, temp & ~FDI_TX_ENABLE);
4374         POSTING_READ(reg);
4375
4376         reg = FDI_RX_CTL(pipe);
4377         temp = I915_READ(reg);
4378         temp &= ~(0x7 << 16);
4379         temp |= (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) << 11;
4380         I915_WRITE(reg, temp & ~FDI_RX_ENABLE);
4381
4382         POSTING_READ(reg);
4383         udelay(100);
4384
4385         /* Ironlake workaround, disable clock pointer after downing FDI */
4386         if (HAS_PCH_IBX(dev_priv))
4387                 I915_WRITE(FDI_RX_CHICKEN(pipe), FDI_RX_PHASE_SYNC_POINTER_OVR);
4388
4389         /* still set train pattern 1 */
4390         reg = FDI_TX_CTL(pipe);
4391         temp = I915_READ(reg);
4392         temp &= ~FDI_LINK_TRAIN_NONE;
4393         temp |= FDI_LINK_TRAIN_PATTERN_1;
4394         I915_WRITE(reg, temp);
4395
4396         reg = FDI_RX_CTL(pipe);
4397         temp = I915_READ(reg);
4398         if (HAS_PCH_CPT(dev_priv)) {
4399                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
4400                 temp |= FDI_LINK_TRAIN_PATTERN_1_CPT;
4401         } else {
4402                 temp &= ~FDI_LINK_TRAIN_NONE;
4403                 temp |= FDI_LINK_TRAIN_PATTERN_1;
4404         }
4405         /* BPC in FDI rx is consistent with that in PIPECONF */
4406         temp &= ~(0x07 << 16);
4407         temp |= (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) << 11;
4408         I915_WRITE(reg, temp);
4409
4410         POSTING_READ(reg);
4411         udelay(100);
4412 }
4413
4414 bool intel_has_pending_fb_unpin(struct drm_i915_private *dev_priv)
4415 {
4416         struct drm_crtc *crtc;
4417         bool cleanup_done;
4418
4419         drm_for_each_crtc(crtc, &dev_priv->drm) {
4420                 struct drm_crtc_commit *commit;
4421                 spin_lock(&crtc->commit_lock);
4422                 commit = list_first_entry_or_null(&crtc->commit_list,
4423                                                   struct drm_crtc_commit, commit_entry);
4424                 cleanup_done = commit ?
4425                         try_wait_for_completion(&commit->cleanup_done) : true;
4426                 spin_unlock(&crtc->commit_lock);
4427
4428                 if (cleanup_done)
4429                         continue;
4430
4431                 drm_crtc_wait_one_vblank(crtc);
4432
4433                 return true;
4434         }
4435
4436         return false;
4437 }
4438
4439 void lpt_disable_iclkip(struct drm_i915_private *dev_priv)
4440 {
4441         u32 temp;
4442
4443         I915_WRITE(PIXCLK_GATE, PIXCLK_GATE_GATE);
4444
4445         mutex_lock(&dev_priv->sb_lock);
4446
4447         temp = intel_sbi_read(dev_priv, SBI_SSCCTL6, SBI_ICLK);
4448         temp |= SBI_SSCCTL_DISABLE;
4449         intel_sbi_write(dev_priv, SBI_SSCCTL6, temp, SBI_ICLK);
4450
4451         mutex_unlock(&dev_priv->sb_lock);
4452 }
4453
4454 /* Program iCLKIP clock to the desired frequency */
4455 static void lpt_program_iclkip(const struct intel_crtc_state *crtc_state)
4456 {
4457         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
4458         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4459         int clock = crtc_state->base.adjusted_mode.crtc_clock;
4460         u32 divsel, phaseinc, auxdiv, phasedir = 0;
4461         u32 temp;
4462
4463         lpt_disable_iclkip(dev_priv);
4464
4465         /* The iCLK virtual clock root frequency is in MHz,
4466          * but the adjusted_mode->crtc_clock in in KHz. To get the
4467          * divisors, it is necessary to divide one by another, so we
4468          * convert the virtual clock precision to KHz here for higher
4469          * precision.
4470          */
4471         for (auxdiv = 0; auxdiv < 2; auxdiv++) {
4472                 u32 iclk_virtual_root_freq = 172800 * 1000;
4473                 u32 iclk_pi_range = 64;
4474                 u32 desired_divisor;
4475
4476                 desired_divisor = DIV_ROUND_CLOSEST(iclk_virtual_root_freq,
4477                                                     clock << auxdiv);
4478                 divsel = (desired_divisor / iclk_pi_range) - 2;
4479                 phaseinc = desired_divisor % iclk_pi_range;
4480
4481                 /*
4482                  * Near 20MHz is a corner case which is
4483                  * out of range for the 7-bit divisor
4484                  */
4485                 if (divsel <= 0x7f)
4486                         break;
4487         }
4488
4489         /* This should not happen with any sane values */
4490         WARN_ON(SBI_SSCDIVINTPHASE_DIVSEL(divsel) &
4491                 ~SBI_SSCDIVINTPHASE_DIVSEL_MASK);
4492         WARN_ON(SBI_SSCDIVINTPHASE_DIR(phasedir) &
4493                 ~SBI_SSCDIVINTPHASE_INCVAL_MASK);
4494
4495         DRM_DEBUG_KMS("iCLKIP clock: found settings for %dKHz refresh rate: auxdiv=%x, divsel=%x, phasedir=%x, phaseinc=%x\n",
4496                         clock,
4497                         auxdiv,
4498                         divsel,
4499                         phasedir,
4500                         phaseinc);
4501
4502         mutex_lock(&dev_priv->sb_lock);
4503
4504         /* Program SSCDIVINTPHASE6 */
4505         temp = intel_sbi_read(dev_priv, SBI_SSCDIVINTPHASE6, SBI_ICLK);
4506         temp &= ~SBI_SSCDIVINTPHASE_DIVSEL_MASK;
4507         temp |= SBI_SSCDIVINTPHASE_DIVSEL(divsel);
4508         temp &= ~SBI_SSCDIVINTPHASE_INCVAL_MASK;
4509         temp |= SBI_SSCDIVINTPHASE_INCVAL(phaseinc);
4510         temp |= SBI_SSCDIVINTPHASE_DIR(phasedir);
4511         temp |= SBI_SSCDIVINTPHASE_PROPAGATE;
4512         intel_sbi_write(dev_priv, SBI_SSCDIVINTPHASE6, temp, SBI_ICLK);
4513
4514         /* Program SSCAUXDIV */
4515         temp = intel_sbi_read(dev_priv, SBI_SSCAUXDIV6, SBI_ICLK);
4516         temp &= ~SBI_SSCAUXDIV_FINALDIV2SEL(1);
4517         temp |= SBI_SSCAUXDIV_FINALDIV2SEL(auxdiv);
4518         intel_sbi_write(dev_priv, SBI_SSCAUXDIV6, temp, SBI_ICLK);
4519
4520         /* Enable modulator and associated divider */
4521         temp = intel_sbi_read(dev_priv, SBI_SSCCTL6, SBI_ICLK);
4522         temp &= ~SBI_SSCCTL_DISABLE;
4523         intel_sbi_write(dev_priv, SBI_SSCCTL6, temp, SBI_ICLK);
4524
4525         mutex_unlock(&dev_priv->sb_lock);
4526
4527         /* Wait for initialization time */
4528         udelay(24);
4529
4530         I915_WRITE(PIXCLK_GATE, PIXCLK_GATE_UNGATE);
4531 }
4532
4533 int lpt_get_iclkip(struct drm_i915_private *dev_priv)
4534 {
4535         u32 divsel, phaseinc, auxdiv;
4536         u32 iclk_virtual_root_freq = 172800 * 1000;
4537         u32 iclk_pi_range = 64;
4538         u32 desired_divisor;
4539         u32 temp;
4540
4541         if ((I915_READ(PIXCLK_GATE) & PIXCLK_GATE_UNGATE) == 0)
4542                 return 0;
4543
4544         mutex_lock(&dev_priv->sb_lock);
4545
4546         temp = intel_sbi_read(dev_priv, SBI_SSCCTL6, SBI_ICLK);
4547         if (temp & SBI_SSCCTL_DISABLE) {
4548                 mutex_unlock(&dev_priv->sb_lock);
4549                 return 0;
4550         }
4551
4552         temp = intel_sbi_read(dev_priv, SBI_SSCDIVINTPHASE6, SBI_ICLK);
4553         divsel = (temp & SBI_SSCDIVINTPHASE_DIVSEL_MASK) >>
4554                 SBI_SSCDIVINTPHASE_DIVSEL_SHIFT;
4555         phaseinc = (temp & SBI_SSCDIVINTPHASE_INCVAL_MASK) >>
4556                 SBI_SSCDIVINTPHASE_INCVAL_SHIFT;
4557
4558         temp = intel_sbi_read(dev_priv, SBI_SSCAUXDIV6, SBI_ICLK);
4559         auxdiv = (temp & SBI_SSCAUXDIV_FINALDIV2SEL_MASK) >>
4560                 SBI_SSCAUXDIV_FINALDIV2SEL_SHIFT;
4561
4562         mutex_unlock(&dev_priv->sb_lock);
4563
4564         desired_divisor = (divsel + 2) * iclk_pi_range + phaseinc;
4565
4566         return DIV_ROUND_CLOSEST(iclk_virtual_root_freq,
4567                                  desired_divisor << auxdiv);
4568 }
4569
4570 static void ironlake_pch_transcoder_set_timings(const struct intel_crtc_state *crtc_state,
4571                                                 enum pipe pch_transcoder)
4572 {
4573         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
4574         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4575         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
4576
4577         I915_WRITE(PCH_TRANS_HTOTAL(pch_transcoder),
4578                    I915_READ(HTOTAL(cpu_transcoder)));
4579         I915_WRITE(PCH_TRANS_HBLANK(pch_transcoder),
4580                    I915_READ(HBLANK(cpu_transcoder)));
4581         I915_WRITE(PCH_TRANS_HSYNC(pch_transcoder),
4582                    I915_READ(HSYNC(cpu_transcoder)));
4583
4584         I915_WRITE(PCH_TRANS_VTOTAL(pch_transcoder),
4585                    I915_READ(VTOTAL(cpu_transcoder)));
4586         I915_WRITE(PCH_TRANS_VBLANK(pch_transcoder),
4587                    I915_READ(VBLANK(cpu_transcoder)));
4588         I915_WRITE(PCH_TRANS_VSYNC(pch_transcoder),
4589                    I915_READ(VSYNC(cpu_transcoder)));
4590         I915_WRITE(PCH_TRANS_VSYNCSHIFT(pch_transcoder),
4591                    I915_READ(VSYNCSHIFT(cpu_transcoder)));
4592 }
4593
4594 static void cpt_set_fdi_bc_bifurcation(struct drm_i915_private *dev_priv, bool enable)
4595 {
4596         uint32_t temp;
4597
4598         temp = I915_READ(SOUTH_CHICKEN1);
4599         if (!!(temp & FDI_BC_BIFURCATION_SELECT) == enable)
4600                 return;
4601
4602         WARN_ON(I915_READ(FDI_RX_CTL(PIPE_B)) & FDI_RX_ENABLE);
4603         WARN_ON(I915_READ(FDI_RX_CTL(PIPE_C)) & FDI_RX_ENABLE);
4604
4605         temp &= ~FDI_BC_BIFURCATION_SELECT;
4606         if (enable)
4607                 temp |= FDI_BC_BIFURCATION_SELECT;
4608
4609         DRM_DEBUG_KMS("%sabling fdi C rx\n", enable ? "en" : "dis");
4610         I915_WRITE(SOUTH_CHICKEN1, temp);
4611         POSTING_READ(SOUTH_CHICKEN1);
4612 }
4613
4614 static void ivybridge_update_fdi_bc_bifurcation(const struct intel_crtc_state *crtc_state)
4615 {
4616         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
4617         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4618
4619         switch (crtc->pipe) {
4620         case PIPE_A:
4621                 break;
4622         case PIPE_B:
4623                 if (crtc_state->fdi_lanes > 2)
4624                         cpt_set_fdi_bc_bifurcation(dev_priv, false);
4625                 else
4626                         cpt_set_fdi_bc_bifurcation(dev_priv, true);
4627
4628                 break;
4629         case PIPE_C:
4630                 cpt_set_fdi_bc_bifurcation(dev_priv, true);
4631
4632                 break;
4633         default:
4634                 BUG();
4635         }
4636 }
4637
4638 /*
4639  * Finds the encoder associated with the given CRTC. This can only be
4640  * used when we know that the CRTC isn't feeding multiple encoders!
4641  */
4642 static struct intel_encoder *
4643 intel_get_crtc_new_encoder(const struct intel_atomic_state *state,
4644                            const struct intel_crtc_state *crtc_state)
4645 {
4646         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
4647         const struct drm_connector_state *connector_state;
4648         const struct drm_connector *connector;
4649         struct intel_encoder *encoder = NULL;
4650         int num_encoders = 0;
4651         int i;
4652
4653         for_each_new_connector_in_state(&state->base, connector, connector_state, i) {
4654                 if (connector_state->crtc != &crtc->base)
4655                         continue;
4656
4657                 encoder = to_intel_encoder(connector_state->best_encoder);
4658                 num_encoders++;
4659         }
4660
4661         WARN(num_encoders != 1, "%d encoders for pipe %c\n",
4662              num_encoders, pipe_name(crtc->pipe));
4663
4664         return encoder;
4665 }
4666
4667 /*
4668  * Enable PCH resources required for PCH ports:
4669  *   - PCH PLLs
4670  *   - FDI training & RX/TX
4671  *   - update transcoder timings
4672  *   - DP transcoding bits
4673  *   - transcoder
4674  */
4675 static void ironlake_pch_enable(const struct intel_atomic_state *state,
4676                                 const struct intel_crtc_state *crtc_state)
4677 {
4678         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
4679         struct drm_device *dev = crtc->base.dev;
4680         struct drm_i915_private *dev_priv = to_i915(dev);
4681         int pipe = crtc->pipe;
4682         u32 temp;
4683
4684         assert_pch_transcoder_disabled(dev_priv, pipe);
4685
4686         if (IS_IVYBRIDGE(dev_priv))
4687                 ivybridge_update_fdi_bc_bifurcation(crtc_state);
4688
4689         /* Write the TU size bits before fdi link training, so that error
4690          * detection works. */
4691         I915_WRITE(FDI_RX_TUSIZE1(pipe),
4692                    I915_READ(PIPE_DATA_M1(pipe)) & TU_SIZE_MASK);
4693
4694         /* For PCH output, training FDI link */
4695         dev_priv->display.fdi_link_train(crtc, crtc_state);
4696
4697         /* We need to program the right clock selection before writing the pixel
4698          * mutliplier into the DPLL. */
4699         if (HAS_PCH_CPT(dev_priv)) {
4700                 u32 sel;
4701
4702                 temp = I915_READ(PCH_DPLL_SEL);
4703                 temp |= TRANS_DPLL_ENABLE(pipe);
4704                 sel = TRANS_DPLLB_SEL(pipe);
4705                 if (crtc_state->shared_dpll ==
4706                     intel_get_shared_dpll_by_id(dev_priv, DPLL_ID_PCH_PLL_B))
4707                         temp |= sel;
4708                 else
4709                         temp &= ~sel;
4710                 I915_WRITE(PCH_DPLL_SEL, temp);
4711         }
4712
4713         /* XXX: pch pll's can be enabled any time before we enable the PCH
4714          * transcoder, and we actually should do this to not upset any PCH
4715          * transcoder that already use the clock when we share it.
4716          *
4717          * Note that enable_shared_dpll tries to do the right thing, but
4718          * get_shared_dpll unconditionally resets the pll - we need that to have
4719          * the right LVDS enable sequence. */
4720         intel_enable_shared_dpll(crtc_state);
4721
4722         /* set transcoder timing, panel must allow it */
4723         assert_panel_unlocked(dev_priv, pipe);
4724         ironlake_pch_transcoder_set_timings(crtc_state, pipe);
4725
4726         intel_fdi_normal_train(crtc);
4727
4728         /* For PCH DP, enable TRANS_DP_CTL */
4729         if (HAS_PCH_CPT(dev_priv) &&
4730             intel_crtc_has_dp_encoder(crtc_state)) {
4731                 const struct drm_display_mode *adjusted_mode =
4732                         &crtc_state->base.adjusted_mode;
4733                 u32 bpc = (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) >> 5;
4734                 i915_reg_t reg = TRANS_DP_CTL(pipe);
4735                 enum port port;
4736
4737                 temp = I915_READ(reg);
4738                 temp &= ~(TRANS_DP_PORT_SEL_MASK |
4739                           TRANS_DP_SYNC_MASK |
4740                           TRANS_DP_BPC_MASK);
4741                 temp |= TRANS_DP_OUTPUT_ENABLE;
4742                 temp |= bpc << 9; /* same format but at 11:9 */
4743
4744                 if (adjusted_mode->flags & DRM_MODE_FLAG_PHSYNC)
4745                         temp |= TRANS_DP_HSYNC_ACTIVE_HIGH;
4746                 if (adjusted_mode->flags & DRM_MODE_FLAG_PVSYNC)
4747                         temp |= TRANS_DP_VSYNC_ACTIVE_HIGH;
4748
4749                 port = intel_get_crtc_new_encoder(state, crtc_state)->port;
4750                 WARN_ON(port < PORT_B || port > PORT_D);
4751                 temp |= TRANS_DP_PORT_SEL(port);
4752
4753                 I915_WRITE(reg, temp);
4754         }
4755
4756         ironlake_enable_pch_transcoder(crtc_state);
4757 }
4758
4759 static void lpt_pch_enable(const struct intel_atomic_state *state,
4760                            const struct intel_crtc_state *crtc_state)
4761 {
4762         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
4763         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4764         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
4765
4766         assert_pch_transcoder_disabled(dev_priv, PIPE_A);
4767
4768         lpt_program_iclkip(crtc_state);
4769
4770         /* Set transcoder timing. */
4771         ironlake_pch_transcoder_set_timings(crtc_state, PIPE_A);
4772
4773         lpt_enable_pch_transcoder(dev_priv, cpu_transcoder);
4774 }
4775
4776 static void cpt_verify_modeset(struct drm_device *dev, int pipe)
4777 {
4778         struct drm_i915_private *dev_priv = to_i915(dev);
4779         i915_reg_t dslreg = PIPEDSL(pipe);
4780         u32 temp;
4781
4782         temp = I915_READ(dslreg);
4783         udelay(500);
4784         if (wait_for(I915_READ(dslreg) != temp, 5)) {
4785                 if (wait_for(I915_READ(dslreg) != temp, 5))
4786                         DRM_ERROR("mode set failed: pipe %c stuck\n", pipe_name(pipe));
4787         }
4788 }
4789
4790 /*
4791  * The hardware phase 0.0 refers to the center of the pixel.
4792  * We want to start from the top/left edge which is phase
4793  * -0.5. That matches how the hardware calculates the scaling
4794  * factors (from top-left of the first pixel to bottom-right
4795  * of the last pixel, as opposed to the pixel centers).
4796  *
4797  * For 4:2:0 subsampled chroma planes we obviously have to
4798  * adjust that so that the chroma sample position lands in
4799  * the right spot.
4800  *
4801  * Note that for packed YCbCr 4:2:2 formats there is no way to
4802  * control chroma siting. The hardware simply replicates the
4803  * chroma samples for both of the luma samples, and thus we don't
4804  * actually get the expected MPEG2 chroma siting convention :(
4805  * The same behaviour is observed on pre-SKL platforms as well.
4806  *
4807  * Theory behind the formula (note that we ignore sub-pixel
4808  * source coordinates):
4809  * s = source sample position
4810  * d = destination sample position
4811  *
4812  * Downscaling 4:1:
4813  * -0.5
4814  * | 0.0
4815  * | |     1.5 (initial phase)
4816  * | |     |
4817  * v v     v
4818  * | s | s | s | s |
4819  * |       d       |
4820  *
4821  * Upscaling 1:4:
4822  * -0.5
4823  * | -0.375 (initial phase)
4824  * | |     0.0
4825  * | |     |
4826  * v v     v
4827  * |       s       |
4828  * | d | d | d | d |
4829  */
4830 u16 skl_scaler_calc_phase(int sub, int scale, bool chroma_cosited)
4831 {
4832         int phase = -0x8000;
4833         u16 trip = 0;
4834
4835         if (chroma_cosited)
4836                 phase += (sub - 1) * 0x8000 / sub;
4837
4838         phase += scale / (2 * sub);
4839
4840         /*
4841          * Hardware initial phase limited to [-0.5:1.5].
4842          * Since the max hardware scale factor is 3.0, we
4843          * should never actually excdeed 1.0 here.
4844          */
4845         WARN_ON(phase < -0x8000 || phase > 0x18000);
4846
4847         if (phase < 0)
4848                 phase = 0x10000 + phase;
4849         else
4850                 trip = PS_PHASE_TRIP;
4851
4852         return ((phase >> 2) & PS_PHASE_MASK) | trip;
4853 }
4854
4855 static int
4856 skl_update_scaler(struct intel_crtc_state *crtc_state, bool force_detach,
4857                   unsigned int scaler_user, int *scaler_id,
4858                   int src_w, int src_h, int dst_w, int dst_h,
4859                   const struct drm_format_info *format, bool need_scaler)
4860 {
4861         struct intel_crtc_scaler_state *scaler_state =
4862                 &crtc_state->scaler_state;
4863         struct intel_crtc *intel_crtc =
4864                 to_intel_crtc(crtc_state->base.crtc);
4865         struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
4866         const struct drm_display_mode *adjusted_mode =
4867                 &crtc_state->base.adjusted_mode;
4868
4869         /*
4870          * Src coordinates are already rotated by 270 degrees for
4871          * the 90/270 degree plane rotation cases (to match the
4872          * GTT mapping), hence no need to account for rotation here.
4873          */
4874         if (src_w != dst_w || src_h != dst_h)
4875                 need_scaler = true;
4876
4877         /*
4878          * Scaling/fitting not supported in IF-ID mode in GEN9+
4879          * TODO: Interlace fetch mode doesn't support YUV420 planar formats.
4880          * Once NV12 is enabled, handle it here while allocating scaler
4881          * for NV12.
4882          */
4883         if (INTEL_GEN(dev_priv) >= 9 && crtc_state->base.enable &&
4884             need_scaler && adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) {
4885                 DRM_DEBUG_KMS("Pipe/Plane scaling not supported with IF-ID mode\n");
4886                 return -EINVAL;
4887         }
4888
4889         /*
4890          * if plane is being disabled or scaler is no more required or force detach
4891          *  - free scaler binded to this plane/crtc
4892          *  - in order to do this, update crtc->scaler_usage
4893          *
4894          * Here scaler state in crtc_state is set free so that
4895          * scaler can be assigned to other user. Actual register
4896          * update to free the scaler is done in plane/panel-fit programming.
4897          * For this purpose crtc/plane_state->scaler_id isn't reset here.
4898          */
4899         if (force_detach || !need_scaler) {
4900                 if (*scaler_id >= 0) {
4901                         scaler_state->scaler_users &= ~(1 << scaler_user);
4902                         scaler_state->scalers[*scaler_id].in_use = 0;
4903
4904                         DRM_DEBUG_KMS("scaler_user index %u.%u: "
4905                                 "Staged freeing scaler id %d scaler_users = 0x%x\n",
4906                                 intel_crtc->pipe, scaler_user, *scaler_id,
4907                                 scaler_state->scaler_users);
4908                         *scaler_id = -1;
4909                 }
4910                 return 0;
4911         }
4912
4913         if (format && format->format == DRM_FORMAT_NV12 &&
4914             (src_h < SKL_MIN_YUV_420_SRC_H || src_w < SKL_MIN_YUV_420_SRC_W)) {
4915                 DRM_DEBUG_KMS("NV12: src dimensions not met\n");
4916                 return -EINVAL;
4917         }
4918
4919         /* range checks */
4920         if (src_w < SKL_MIN_SRC_W || src_h < SKL_MIN_SRC_H ||
4921             dst_w < SKL_MIN_DST_W || dst_h < SKL_MIN_DST_H ||
4922             (IS_GEN11(dev_priv) &&
4923              (src_w > ICL_MAX_SRC_W || src_h > ICL_MAX_SRC_H ||
4924               dst_w > ICL_MAX_DST_W || dst_h > ICL_MAX_DST_H)) ||
4925             (!IS_GEN11(dev_priv) &&
4926              (src_w > SKL_MAX_SRC_W || src_h > SKL_MAX_SRC_H ||
4927               dst_w > SKL_MAX_DST_W || dst_h > SKL_MAX_DST_H))) {
4928                 DRM_DEBUG_KMS("scaler_user index %u.%u: src %ux%u dst %ux%u "
4929                         "size is out of scaler range\n",
4930                         intel_crtc->pipe, scaler_user, src_w, src_h, dst_w, dst_h);
4931                 return -EINVAL;
4932         }
4933
4934         /* mark this plane as a scaler user in crtc_state */
4935         scaler_state->scaler_users |= (1 << scaler_user);
4936         DRM_DEBUG_KMS("scaler_user index %u.%u: "
4937                 "staged scaling request for %ux%u->%ux%u scaler_users = 0x%x\n",
4938                 intel_crtc->pipe, scaler_user, src_w, src_h, dst_w, dst_h,
4939                 scaler_state->scaler_users);
4940
4941         return 0;
4942 }
4943
4944 /**
4945  * skl_update_scaler_crtc - Stages update to scaler state for a given crtc.
4946  *
4947  * @state: crtc's scaler state
4948  *
4949  * Return
4950  *     0 - scaler_usage updated successfully
4951  *    error - requested scaling cannot be supported or other error condition
4952  */
4953 int skl_update_scaler_crtc(struct intel_crtc_state *state)
4954 {
4955         const struct drm_display_mode *adjusted_mode = &state->base.adjusted_mode;
4956         bool need_scaler = false;
4957
4958         if (state->output_format == INTEL_OUTPUT_FORMAT_YCBCR420)
4959                 need_scaler = true;
4960
4961         return skl_update_scaler(state, !state->base.active, SKL_CRTC_INDEX,
4962                                  &state->scaler_state.scaler_id,
4963                                  state->pipe_src_w, state->pipe_src_h,
4964                                  adjusted_mode->crtc_hdisplay,
4965                                  adjusted_mode->crtc_vdisplay, NULL, need_scaler);
4966 }
4967
4968 /**
4969  * skl_update_scaler_plane - Stages update to scaler state for a given plane.
4970  * @crtc_state: crtc's scaler state
4971  * @plane_state: atomic plane state to update
4972  *
4973  * Return
4974  *     0 - scaler_usage updated successfully
4975  *    error - requested scaling cannot be supported or other error condition
4976  */
4977 static int skl_update_scaler_plane(struct intel_crtc_state *crtc_state,
4978                                    struct intel_plane_state *plane_state)
4979 {
4980         struct intel_plane *intel_plane =
4981                 to_intel_plane(plane_state->base.plane);
4982         struct drm_framebuffer *fb = plane_state->base.fb;
4983         int ret;
4984         bool force_detach = !fb || !plane_state->base.visible;
4985         bool need_scaler = false;
4986
4987         /* Pre-gen11 and SDR planes always need a scaler for planar formats. */
4988         if (!icl_is_hdr_plane(intel_plane) &&
4989             fb && fb->format->format == DRM_FORMAT_NV12)
4990                 need_scaler = true;
4991
4992         ret = skl_update_scaler(crtc_state, force_detach,
4993                                 drm_plane_index(&intel_plane->base),
4994                                 &plane_state->scaler_id,
4995                                 drm_rect_width(&plane_state->base.src) >> 16,
4996                                 drm_rect_height(&plane_state->base.src) >> 16,
4997                                 drm_rect_width(&plane_state->base.dst),
4998                                 drm_rect_height(&plane_state->base.dst),
4999                                 fb ? fb->format : NULL, need_scaler);
5000
5001         if (ret || plane_state->scaler_id < 0)
5002                 return ret;
5003
5004         /* check colorkey */
5005         if (plane_state->ckey.flags) {
5006                 DRM_DEBUG_KMS("[PLANE:%d:%s] scaling with color key not allowed",
5007                               intel_plane->base.base.id,
5008                               intel_plane->base.name);
5009                 return -EINVAL;
5010         }
5011
5012         /* Check src format */
5013         switch (fb->format->format) {
5014         case DRM_FORMAT_RGB565:
5015         case DRM_FORMAT_XBGR8888:
5016         case DRM_FORMAT_XRGB8888:
5017         case DRM_FORMAT_ABGR8888:
5018         case DRM_FORMAT_ARGB8888:
5019         case DRM_FORMAT_XRGB2101010:
5020         case DRM_FORMAT_XBGR2101010:
5021         case DRM_FORMAT_YUYV:
5022         case DRM_FORMAT_YVYU:
5023         case DRM_FORMAT_UYVY:
5024         case DRM_FORMAT_VYUY:
5025         case DRM_FORMAT_NV12:
5026                 break;
5027         default:
5028                 DRM_DEBUG_KMS("[PLANE:%d:%s] FB:%d unsupported scaling format 0x%x\n",
5029                               intel_plane->base.base.id, intel_plane->base.name,
5030                               fb->base.id, fb->format->format);
5031                 return -EINVAL;
5032         }
5033
5034         return 0;
5035 }
5036
5037 static void skylake_scaler_disable(struct intel_crtc *crtc)
5038 {
5039         int i;
5040
5041         for (i = 0; i < crtc->num_scalers; i++)
5042                 skl_detach_scaler(crtc, i);
5043 }
5044
5045 static void skylake_pfit_enable(const struct intel_crtc_state *crtc_state)
5046 {
5047         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
5048         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5049         enum pipe pipe = crtc->pipe;
5050         const struct intel_crtc_scaler_state *scaler_state =
5051                 &crtc_state->scaler_state;
5052
5053         if (crtc_state->pch_pfit.enabled) {
5054                 u16 uv_rgb_hphase, uv_rgb_vphase;
5055                 int pfit_w, pfit_h, hscale, vscale;
5056                 int id;
5057
5058                 if (WARN_ON(crtc_state->scaler_state.scaler_id < 0))
5059                         return;
5060
5061                 pfit_w = (crtc_state->pch_pfit.size >> 16) & 0xFFFF;
5062                 pfit_h = crtc_state->pch_pfit.size & 0xFFFF;
5063
5064                 hscale = (crtc_state->pipe_src_w << 16) / pfit_w;
5065                 vscale = (crtc_state->pipe_src_h << 16) / pfit_h;
5066
5067                 uv_rgb_hphase = skl_scaler_calc_phase(1, hscale, false);
5068                 uv_rgb_vphase = skl_scaler_calc_phase(1, vscale, false);
5069
5070                 id = scaler_state->scaler_id;
5071                 I915_WRITE(SKL_PS_CTRL(pipe, id), PS_SCALER_EN |
5072                         PS_FILTER_MEDIUM | scaler_state->scalers[id].mode);
5073                 I915_WRITE_FW(SKL_PS_VPHASE(pipe, id),
5074                               PS_Y_PHASE(0) | PS_UV_RGB_PHASE(uv_rgb_vphase));
5075                 I915_WRITE_FW(SKL_PS_HPHASE(pipe, id),
5076                               PS_Y_PHASE(0) | PS_UV_RGB_PHASE(uv_rgb_hphase));
5077                 I915_WRITE(SKL_PS_WIN_POS(pipe, id), crtc_state->pch_pfit.pos);
5078                 I915_WRITE(SKL_PS_WIN_SZ(pipe, id), crtc_state->pch_pfit.size);
5079         }
5080 }
5081
5082 static void ironlake_pfit_enable(const struct intel_crtc_state *crtc_state)
5083 {
5084         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
5085         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5086         int pipe = crtc->pipe;
5087
5088         if (crtc_state->pch_pfit.enabled) {
5089                 /* Force use of hard-coded filter coefficients
5090                  * as some pre-programmed values are broken,
5091                  * e.g. x201.
5092                  */
5093                 if (IS_IVYBRIDGE(dev_priv) || IS_HASWELL(dev_priv))
5094                         I915_WRITE(PF_CTL(pipe), PF_ENABLE | PF_FILTER_MED_3x3 |
5095                                                  PF_PIPE_SEL_IVB(pipe));
5096                 else
5097                         I915_WRITE(PF_CTL(pipe), PF_ENABLE | PF_FILTER_MED_3x3);
5098                 I915_WRITE(PF_WIN_POS(pipe), crtc_state->pch_pfit.pos);
5099                 I915_WRITE(PF_WIN_SZ(pipe), crtc_state->pch_pfit.size);
5100         }
5101 }
5102
5103 void hsw_enable_ips(const struct intel_crtc_state *crtc_state)
5104 {
5105         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
5106         struct drm_device *dev = crtc->base.dev;
5107         struct drm_i915_private *dev_priv = to_i915(dev);
5108
5109         if (!crtc_state->ips_enabled)
5110                 return;
5111
5112         /*
5113          * We can only enable IPS after we enable a plane and wait for a vblank
5114          * This function is called from post_plane_update, which is run after
5115          * a vblank wait.
5116          */
5117         WARN_ON(!(crtc_state->active_planes & ~BIT(PLANE_CURSOR)));
5118
5119         if (IS_BROADWELL(dev_priv)) {
5120                 mutex_lock(&dev_priv->pcu_lock);
5121                 WARN_ON(sandybridge_pcode_write(dev_priv, DISPLAY_IPS_CONTROL,
5122                                                 IPS_ENABLE | IPS_PCODE_CONTROL));
5123                 mutex_unlock(&dev_priv->pcu_lock);
5124                 /* Quoting Art Runyan: "its not safe to expect any particular
5125                  * value in IPS_CTL bit 31 after enabling IPS through the
5126                  * mailbox." Moreover, the mailbox may return a bogus state,
5127                  * so we need to just enable it and continue on.
5128                  */
5129         } else {
5130                 I915_WRITE(IPS_CTL, IPS_ENABLE);
5131                 /* The bit only becomes 1 in the next vblank, so this wait here
5132                  * is essentially intel_wait_for_vblank. If we don't have this
5133                  * and don't wait for vblanks until the end of crtc_enable, then
5134                  * the HW state readout code will complain that the expected
5135                  * IPS_CTL value is not the one we read. */
5136                 if (intel_wait_for_register(dev_priv,
5137                                             IPS_CTL, IPS_ENABLE, IPS_ENABLE,
5138                                             50))
5139                         DRM_ERROR("Timed out waiting for IPS enable\n");
5140         }
5141 }
5142
5143 void hsw_disable_ips(const struct intel_crtc_state *crtc_state)
5144 {
5145         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
5146         struct drm_device *dev = crtc->base.dev;
5147         struct drm_i915_private *dev_priv = to_i915(dev);
5148
5149         if (!crtc_state->ips_enabled)
5150                 return;
5151
5152         if (IS_BROADWELL(dev_priv)) {
5153                 mutex_lock(&dev_priv->pcu_lock);
5154                 WARN_ON(sandybridge_pcode_write(dev_priv, DISPLAY_IPS_CONTROL, 0));
5155                 mutex_unlock(&dev_priv->pcu_lock);
5156                 /*
5157                  * Wait for PCODE to finish disabling IPS. The BSpec specified
5158                  * 42ms timeout value leads to occasional timeouts so use 100ms
5159                  * instead.
5160                  */
5161                 if (intel_wait_for_register(dev_priv,
5162                                             IPS_CTL, IPS_ENABLE, 0,
5163                                             100))
5164                         DRM_ERROR("Timed out waiting for IPS disable\n");
5165         } else {
5166                 I915_WRITE(IPS_CTL, 0);
5167                 POSTING_READ(IPS_CTL);
5168         }
5169
5170         /* We need to wait for a vblank before we can disable the plane. */
5171         intel_wait_for_vblank(dev_priv, crtc->pipe);
5172 }
5173
5174 static void intel_crtc_dpms_overlay_disable(struct intel_crtc *intel_crtc)
5175 {
5176         if (intel_crtc->overlay) {
5177                 struct drm_device *dev = intel_crtc->base.dev;
5178
5179                 mutex_lock(&dev->struct_mutex);
5180                 (void) intel_overlay_switch_off(intel_crtc->overlay);
5181                 mutex_unlock(&dev->struct_mutex);
5182         }
5183
5184         /* Let userspace switch the overlay on again. In most cases userspace
5185          * has to recompute where to put it anyway.
5186          */
5187 }
5188
5189 /**
5190  * intel_post_enable_primary - Perform operations after enabling primary plane
5191  * @crtc: the CRTC whose primary plane was just enabled
5192  * @new_crtc_state: the enabling state
5193  *
5194  * Performs potentially sleeping operations that must be done after the primary
5195  * plane is enabled, such as updating FBC and IPS.  Note that this may be
5196  * called due to an explicit primary plane update, or due to an implicit
5197  * re-enable that is caused when a sprite plane is updated to no longer
5198  * completely hide the primary plane.
5199  */
5200 static void
5201 intel_post_enable_primary(struct drm_crtc *crtc,
5202                           const struct intel_crtc_state *new_crtc_state)
5203 {
5204         struct drm_device *dev = crtc->dev;
5205         struct drm_i915_private *dev_priv = to_i915(dev);
5206         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5207         int pipe = intel_crtc->pipe;
5208
5209         /*
5210          * Gen2 reports pipe underruns whenever all planes are disabled.
5211          * So don't enable underrun reporting before at least some planes
5212          * are enabled.
5213          * FIXME: Need to fix the logic to work when we turn off all planes
5214          * but leave the pipe running.
5215          */
5216         if (IS_GEN2(dev_priv))
5217                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
5218
5219         /* Underruns don't always raise interrupts, so check manually. */
5220         intel_check_cpu_fifo_underruns(dev_priv);
5221         intel_check_pch_fifo_underruns(dev_priv);
5222 }
5223
5224 /* FIXME get rid of this and use pre_plane_update */
5225 static void
5226 intel_pre_disable_primary_noatomic(struct drm_crtc *crtc)
5227 {
5228         struct drm_device *dev = crtc->dev;
5229         struct drm_i915_private *dev_priv = to_i915(dev);
5230         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5231         int pipe = intel_crtc->pipe;
5232
5233         /*
5234          * Gen2 reports pipe underruns whenever all planes are disabled.
5235          * So disable underrun reporting before all the planes get disabled.
5236          */
5237         if (IS_GEN2(dev_priv))
5238                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
5239
5240         hsw_disable_ips(to_intel_crtc_state(crtc->state));
5241
5242         /*
5243          * Vblank time updates from the shadow to live plane control register
5244          * are blocked if the memory self-refresh mode is active at that
5245          * moment. So to make sure the plane gets truly disabled, disable
5246          * first the self-refresh mode. The self-refresh enable bit in turn
5247          * will be checked/applied by the HW only at the next frame start
5248          * event which is after the vblank start event, so we need to have a
5249          * wait-for-vblank between disabling the plane and the pipe.
5250          */
5251         if (HAS_GMCH_DISPLAY(dev_priv) &&
5252             intel_set_memory_cxsr(dev_priv, false))
5253                 intel_wait_for_vblank(dev_priv, pipe);
5254 }
5255
5256 static bool hsw_pre_update_disable_ips(const struct intel_crtc_state *old_crtc_state,
5257                                        const struct intel_crtc_state *new_crtc_state)
5258 {
5259         if (!old_crtc_state->ips_enabled)
5260                 return false;
5261
5262         if (needs_modeset(&new_crtc_state->base))
5263                 return true;
5264
5265         return !new_crtc_state->ips_enabled;
5266 }
5267
5268 static bool hsw_post_update_enable_ips(const struct intel_crtc_state *old_crtc_state,
5269                                        const struct intel_crtc_state *new_crtc_state)
5270 {
5271         if (!new_crtc_state->ips_enabled)
5272                 return false;
5273
5274         if (needs_modeset(&new_crtc_state->base))
5275                 return true;
5276
5277         /*
5278          * We can't read out IPS on broadwell, assume the worst and
5279          * forcibly enable IPS on the first fastset.
5280          */
5281         if (new_crtc_state->update_pipe &&
5282             old_crtc_state->base.adjusted_mode.private_flags & I915_MODE_FLAG_INHERITED)
5283                 return true;
5284
5285         return !old_crtc_state->ips_enabled;
5286 }
5287
5288 static bool needs_nv12_wa(struct drm_i915_private *dev_priv,
5289                           const struct intel_crtc_state *crtc_state)
5290 {
5291         if (!crtc_state->nv12_planes)
5292                 return false;
5293
5294         /* WA Display #0827: Gen9:all */
5295         if (IS_GEN9(dev_priv) && !IS_GEMINILAKE(dev_priv))
5296                 return true;
5297
5298         return false;
5299 }
5300
5301 static void intel_post_plane_update(struct intel_crtc_state *old_crtc_state)
5302 {
5303         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc);
5304         struct drm_device *dev = crtc->base.dev;
5305         struct drm_i915_private *dev_priv = to_i915(dev);
5306         struct drm_atomic_state *old_state = old_crtc_state->base.state;
5307         struct intel_crtc_state *pipe_config =
5308                 intel_atomic_get_new_crtc_state(to_intel_atomic_state(old_state),
5309                                                 crtc);
5310         struct drm_plane *primary = crtc->base.primary;
5311         struct drm_plane_state *old_primary_state =
5312                 drm_atomic_get_old_plane_state(old_state, primary);
5313
5314         intel_frontbuffer_flip(to_i915(crtc->base.dev), pipe_config->fb_bits);
5315
5316         if (pipe_config->update_wm_post && pipe_config->base.active)
5317                 intel_update_watermarks(crtc);
5318
5319         if (hsw_post_update_enable_ips(old_crtc_state, pipe_config))
5320                 hsw_enable_ips(pipe_config);
5321
5322         if (old_primary_state) {
5323                 struct drm_plane_state *new_primary_state =
5324                         drm_atomic_get_new_plane_state(old_state, primary);
5325
5326                 intel_fbc_post_update(crtc);
5327
5328                 if (new_primary_state->visible &&
5329                     (needs_modeset(&pipe_config->base) ||
5330                      !old_primary_state->visible))
5331                         intel_post_enable_primary(&crtc->base, pipe_config);
5332         }
5333
5334         /* Display WA 827 */
5335         if (needs_nv12_wa(dev_priv, old_crtc_state) &&
5336             !needs_nv12_wa(dev_priv, pipe_config)) {
5337                 skl_wa_clkgate(dev_priv, crtc->pipe, false);
5338         }
5339 }
5340
5341 static void intel_pre_plane_update(struct intel_crtc_state *old_crtc_state,
5342                                    struct intel_crtc_state *pipe_config)
5343 {
5344         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc);
5345         struct drm_device *dev = crtc->base.dev;
5346         struct drm_i915_private *dev_priv = to_i915(dev);
5347         struct drm_atomic_state *old_state = old_crtc_state->base.state;
5348         struct drm_plane *primary = crtc->base.primary;
5349         struct drm_plane_state *old_primary_state =
5350                 drm_atomic_get_old_plane_state(old_state, primary);
5351         bool modeset = needs_modeset(&pipe_config->base);
5352         struct intel_atomic_state *old_intel_state =
5353                 to_intel_atomic_state(old_state);
5354
5355         if (hsw_pre_update_disable_ips(old_crtc_state, pipe_config))
5356                 hsw_disable_ips(old_crtc_state);
5357
5358         if (old_primary_state) {
5359                 struct intel_plane_state *new_primary_state =
5360                         intel_atomic_get_new_plane_state(old_intel_state,
5361                                                          to_intel_plane(primary));
5362
5363                 intel_fbc_pre_update(crtc, pipe_config, new_primary_state);
5364                 /*
5365                  * Gen2 reports pipe underruns whenever all planes are disabled.
5366                  * So disable underrun reporting before all the planes get disabled.
5367                  */
5368                 if (IS_GEN2(dev_priv) && old_primary_state->visible &&
5369                     (modeset || !new_primary_state->base.visible))
5370                         intel_set_cpu_fifo_underrun_reporting(dev_priv, crtc->pipe, false);
5371         }
5372
5373         /* Display WA 827 */
5374         if (!needs_nv12_wa(dev_priv, old_crtc_state) &&
5375             needs_nv12_wa(dev_priv, pipe_config)) {
5376                 skl_wa_clkgate(dev_priv, crtc->pipe, true);
5377         }
5378
5379         /*
5380          * Vblank time updates from the shadow to live plane control register
5381          * are blocked if the memory self-refresh mode is active at that
5382          * moment. So to make sure the plane gets truly disabled, disable
5383          * first the self-refresh mode. The self-refresh enable bit in turn
5384          * will be checked/applied by the HW only at the next frame start
5385          * event which is after the vblank start event, so we need to have a
5386          * wait-for-vblank between disabling the plane and the pipe.
5387          */
5388         if (HAS_GMCH_DISPLAY(dev_priv) && old_crtc_state->base.active &&
5389             pipe_config->disable_cxsr && intel_set_memory_cxsr(dev_priv, false))
5390                 intel_wait_for_vblank(dev_priv, crtc->pipe);
5391
5392         /*
5393          * IVB workaround: must disable low power watermarks for at least
5394          * one frame before enabling scaling.  LP watermarks can be re-enabled
5395          * when scaling is disabled.
5396          *
5397          * WaCxSRDisabledForSpriteScaling:ivb
5398          */
5399         if (pipe_config->disable_lp_wm && ilk_disable_lp_wm(dev) &&
5400             old_crtc_state->base.active)
5401                 intel_wait_for_vblank(dev_priv, crtc->pipe);
5402
5403         /*
5404          * If we're doing a modeset, we're done.  No need to do any pre-vblank
5405          * watermark programming here.
5406          */
5407         if (needs_modeset(&pipe_config->base))
5408                 return;
5409
5410         /*
5411          * For platforms that support atomic watermarks, program the
5412          * 'intermediate' watermarks immediately.  On pre-gen9 platforms, these
5413          * will be the intermediate values that are safe for both pre- and
5414          * post- vblank; when vblank happens, the 'active' values will be set
5415          * to the final 'target' values and we'll do this again to get the
5416          * optimal watermarks.  For gen9+ platforms, the values we program here
5417          * will be the final target values which will get automatically latched
5418          * at vblank time; no further programming will be necessary.
5419          *
5420          * If a platform hasn't been transitioned to atomic watermarks yet,
5421          * we'll continue to update watermarks the old way, if flags tell
5422          * us to.
5423          */
5424         if (dev_priv->display.initial_watermarks != NULL)
5425                 dev_priv->display.initial_watermarks(old_intel_state,
5426                                                      pipe_config);
5427         else if (pipe_config->update_wm_pre)
5428                 intel_update_watermarks(crtc);
5429 }
5430
5431 static void intel_crtc_disable_planes(struct intel_atomic_state *state,
5432                                       struct intel_crtc *crtc)
5433 {
5434         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5435         const struct intel_crtc_state *new_crtc_state =
5436                 intel_atomic_get_new_crtc_state(state, crtc);
5437         unsigned int update_mask = new_crtc_state->update_planes;
5438         const struct intel_plane_state *old_plane_state;
5439         struct intel_plane *plane;
5440         unsigned fb_bits = 0;
5441         int i;
5442
5443         intel_crtc_dpms_overlay_disable(crtc);
5444
5445         for_each_old_intel_plane_in_state(state, plane, old_plane_state, i) {
5446                 if (crtc->pipe != plane->pipe ||
5447                     !(update_mask & BIT(plane->id)))
5448                         continue;
5449
5450                 plane->disable_plane(plane, new_crtc_state);
5451
5452                 if (old_plane_state->base.visible)
5453                         fb_bits |= plane->frontbuffer_bit;
5454         }
5455
5456         intel_frontbuffer_flip(dev_priv, fb_bits);
5457 }
5458
5459 static void intel_encoders_pre_pll_enable(struct drm_crtc *crtc,
5460                                           struct intel_crtc_state *crtc_state,
5461                                           struct drm_atomic_state *old_state)
5462 {
5463         struct drm_connector_state *conn_state;
5464         struct drm_connector *conn;
5465         int i;
5466
5467         for_each_new_connector_in_state(old_state, conn, conn_state, i) {
5468                 struct intel_encoder *encoder =
5469                         to_intel_encoder(conn_state->best_encoder);
5470
5471                 if (conn_state->crtc != crtc)
5472                         continue;
5473
5474                 if (encoder->pre_pll_enable)
5475                         encoder->pre_pll_enable(encoder, crtc_state, conn_state);
5476         }
5477 }
5478
5479 static void intel_encoders_pre_enable(struct drm_crtc *crtc,
5480                                       struct intel_crtc_state *crtc_state,
5481                                       struct drm_atomic_state *old_state)
5482 {
5483         struct drm_connector_state *conn_state;
5484         struct drm_connector *conn;
5485         int i;
5486
5487         for_each_new_connector_in_state(old_state, conn, conn_state, i) {
5488                 struct intel_encoder *encoder =
5489                         to_intel_encoder(conn_state->best_encoder);
5490
5491                 if (conn_state->crtc != crtc)
5492                         continue;
5493
5494                 if (encoder->pre_enable)
5495                         encoder->pre_enable(encoder, crtc_state, conn_state);
5496         }
5497 }
5498
5499 static void intel_encoders_enable(struct drm_crtc *crtc,
5500                                   struct intel_crtc_state *crtc_state,
5501                                   struct drm_atomic_state *old_state)
5502 {
5503         struct drm_connector_state *conn_state;
5504         struct drm_connector *conn;
5505         int i;
5506
5507         for_each_new_connector_in_state(old_state, conn, conn_state, i) {
5508                 struct intel_encoder *encoder =
5509                         to_intel_encoder(conn_state->best_encoder);
5510
5511                 if (conn_state->crtc != crtc)
5512                         continue;
5513
5514                 if (encoder->enable)
5515                         encoder->enable(encoder, crtc_state, conn_state);
5516                 intel_opregion_notify_encoder(encoder, true);
5517         }
5518 }
5519
5520 static void intel_encoders_disable(struct drm_crtc *crtc,
5521                                    struct intel_crtc_state *old_crtc_state,
5522                                    struct drm_atomic_state *old_state)
5523 {
5524         struct drm_connector_state *old_conn_state;
5525         struct drm_connector *conn;
5526         int i;
5527
5528         for_each_old_connector_in_state(old_state, conn, old_conn_state, i) {
5529                 struct intel_encoder *encoder =
5530                         to_intel_encoder(old_conn_state->best_encoder);
5531
5532                 if (old_conn_state->crtc != crtc)
5533                         continue;
5534
5535                 intel_opregion_notify_encoder(encoder, false);
5536                 if (encoder->disable)
5537                         encoder->disable(encoder, old_crtc_state, old_conn_state);
5538         }
5539 }
5540
5541 static void intel_encoders_post_disable(struct drm_crtc *crtc,
5542                                         struct intel_crtc_state *old_crtc_state,
5543                                         struct drm_atomic_state *old_state)
5544 {
5545         struct drm_connector_state *old_conn_state;
5546         struct drm_connector *conn;
5547         int i;
5548
5549         for_each_old_connector_in_state(old_state, conn, old_conn_state, i) {
5550                 struct intel_encoder *encoder =
5551                         to_intel_encoder(old_conn_state->best_encoder);
5552
5553                 if (old_conn_state->crtc != crtc)
5554                         continue;
5555
5556                 if (encoder->post_disable)
5557                         encoder->post_disable(encoder, old_crtc_state, old_conn_state);
5558         }
5559 }
5560
5561 static void intel_encoders_post_pll_disable(struct drm_crtc *crtc,
5562                                             struct intel_crtc_state *old_crtc_state,
5563                                             struct drm_atomic_state *old_state)
5564 {
5565         struct drm_connector_state *old_conn_state;
5566         struct drm_connector *conn;
5567         int i;
5568
5569         for_each_old_connector_in_state(old_state, conn, old_conn_state, i) {
5570                 struct intel_encoder *encoder =
5571                         to_intel_encoder(old_conn_state->best_encoder);
5572
5573                 if (old_conn_state->crtc != crtc)
5574                         continue;
5575
5576                 if (encoder->post_pll_disable)
5577                         encoder->post_pll_disable(encoder, old_crtc_state, old_conn_state);
5578         }
5579 }
5580
5581 static void ironlake_crtc_enable(struct intel_crtc_state *pipe_config,
5582                                  struct drm_atomic_state *old_state)
5583 {
5584         struct drm_crtc *crtc = pipe_config->base.crtc;
5585         struct drm_device *dev = crtc->dev;
5586         struct drm_i915_private *dev_priv = to_i915(dev);
5587         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5588         int pipe = intel_crtc->pipe;
5589         struct intel_atomic_state *old_intel_state =
5590                 to_intel_atomic_state(old_state);
5591
5592         if (WARN_ON(intel_crtc->active))
5593                 return;
5594
5595         /*
5596          * Sometimes spurious CPU pipe underruns happen during FDI
5597          * training, at least with VGA+HDMI cloning. Suppress them.
5598          *
5599          * On ILK we get an occasional spurious CPU pipe underruns
5600          * between eDP port A enable and vdd enable. Also PCH port
5601          * enable seems to result in the occasional CPU pipe underrun.
5602          *
5603          * Spurious PCH underruns also occur during PCH enabling.
5604          */
5605         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
5606         intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, false);
5607
5608         if (pipe_config->has_pch_encoder)
5609                 intel_prepare_shared_dpll(pipe_config);
5610
5611         if (intel_crtc_has_dp_encoder(pipe_config))
5612                 intel_dp_set_m_n(pipe_config, M1_N1);
5613
5614         intel_set_pipe_timings(pipe_config);
5615         intel_set_pipe_src_size(pipe_config);
5616
5617         if (pipe_config->has_pch_encoder) {
5618                 intel_cpu_transcoder_set_m_n(pipe_config,
5619                                              &pipe_config->fdi_m_n, NULL);
5620         }
5621
5622         ironlake_set_pipeconf(pipe_config);
5623
5624         intel_crtc->active = true;
5625
5626         intel_encoders_pre_enable(crtc, pipe_config, old_state);
5627
5628         if (pipe_config->has_pch_encoder) {
5629                 /* Note: FDI PLL enabling _must_ be done before we enable the
5630                  * cpu pipes, hence this is separate from all the other fdi/pch
5631                  * enabling. */
5632                 ironlake_fdi_pll_enable(pipe_config);
5633         } else {
5634                 assert_fdi_tx_disabled(dev_priv, pipe);
5635                 assert_fdi_rx_disabled(dev_priv, pipe);
5636         }
5637
5638         ironlake_pfit_enable(pipe_config);
5639
5640         /*
5641          * On ILK+ LUT must be loaded before the pipe is running but with
5642          * clocks enabled
5643          */
5644         intel_color_load_luts(&pipe_config->base);
5645
5646         if (dev_priv->display.initial_watermarks != NULL)
5647                 dev_priv->display.initial_watermarks(old_intel_state, pipe_config);
5648         intel_enable_pipe(pipe_config);
5649
5650         if (pipe_config->has_pch_encoder)
5651                 ironlake_pch_enable(old_intel_state, pipe_config);
5652
5653         assert_vblank_disabled(crtc);
5654         drm_crtc_vblank_on(crtc);
5655
5656         intel_encoders_enable(crtc, pipe_config, old_state);
5657
5658         if (HAS_PCH_CPT(dev_priv))
5659                 cpt_verify_modeset(dev, intel_crtc->pipe);
5660
5661         /*
5662          * Must wait for vblank to avoid spurious PCH FIFO underruns.
5663          * And a second vblank wait is needed at least on ILK with
5664          * some interlaced HDMI modes. Let's do the double wait always
5665          * in case there are more corner cases we don't know about.
5666          */
5667         if (pipe_config->has_pch_encoder) {
5668                 intel_wait_for_vblank(dev_priv, pipe);
5669                 intel_wait_for_vblank(dev_priv, pipe);
5670         }
5671         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
5672         intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, true);
5673 }
5674
5675 /* IPS only exists on ULT machines and is tied to pipe A. */
5676 static bool hsw_crtc_supports_ips(struct intel_crtc *crtc)
5677 {
5678         return HAS_IPS(to_i915(crtc->base.dev)) && crtc->pipe == PIPE_A;
5679 }
5680
5681 static void glk_pipe_scaler_clock_gating_wa(struct drm_i915_private *dev_priv,
5682                                             enum pipe pipe, bool apply)
5683 {
5684         u32 val = I915_READ(CLKGATE_DIS_PSL(pipe));
5685         u32 mask = DPF_GATING_DIS | DPF_RAM_GATING_DIS | DPFR_GATING_DIS;
5686
5687         if (apply)
5688                 val |= mask;
5689         else
5690                 val &= ~mask;
5691
5692         I915_WRITE(CLKGATE_DIS_PSL(pipe), val);
5693 }
5694
5695 static void icl_pipe_mbus_enable(struct intel_crtc *crtc)
5696 {
5697         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5698         enum pipe pipe = crtc->pipe;
5699         uint32_t val;
5700
5701         val = MBUS_DBOX_A_CREDIT(2);
5702         val |= MBUS_DBOX_BW_CREDIT(1);
5703         val |= MBUS_DBOX_B_CREDIT(8);
5704
5705         I915_WRITE(PIPE_MBUS_DBOX_CTL(pipe), val);
5706 }
5707
5708 static void haswell_crtc_enable(struct intel_crtc_state *pipe_config,
5709                                 struct drm_atomic_state *old_state)
5710 {
5711         struct drm_crtc *crtc = pipe_config->base.crtc;
5712         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
5713         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5714         int pipe = intel_crtc->pipe, hsw_workaround_pipe;
5715         enum transcoder cpu_transcoder = pipe_config->cpu_transcoder;
5716         struct intel_atomic_state *old_intel_state =
5717                 to_intel_atomic_state(old_state);
5718         bool psl_clkgate_wa;
5719         u32 pipe_chicken;
5720
5721         if (WARN_ON(intel_crtc->active))
5722                 return;
5723
5724         intel_encoders_pre_pll_enable(crtc, pipe_config, old_state);
5725
5726         if (pipe_config->shared_dpll)
5727                 intel_enable_shared_dpll(pipe_config);
5728
5729         intel_encoders_pre_enable(crtc, pipe_config, old_state);
5730
5731         if (intel_crtc_has_dp_encoder(pipe_config))
5732                 intel_dp_set_m_n(pipe_config, M1_N1);
5733
5734         if (!transcoder_is_dsi(cpu_transcoder))
5735                 intel_set_pipe_timings(pipe_config);
5736
5737         intel_set_pipe_src_size(pipe_config);
5738
5739         if (cpu_transcoder != TRANSCODER_EDP &&
5740             !transcoder_is_dsi(cpu_transcoder)) {
5741                 I915_WRITE(PIPE_MULT(cpu_transcoder),
5742                            pipe_config->pixel_multiplier - 1);
5743         }
5744
5745         if (pipe_config->has_pch_encoder) {
5746                 intel_cpu_transcoder_set_m_n(pipe_config,
5747                                              &pipe_config->fdi_m_n, NULL);
5748         }
5749
5750         if (!transcoder_is_dsi(cpu_transcoder))
5751                 haswell_set_pipeconf(pipe_config);
5752
5753         haswell_set_pipemisc(pipe_config);
5754
5755         intel_color_set_csc(&pipe_config->base);
5756
5757         intel_crtc->active = true;
5758
5759         /* Display WA #1180: WaDisableScalarClockGating: glk, cnl */
5760         psl_clkgate_wa = (IS_GEMINILAKE(dev_priv) || IS_CANNONLAKE(dev_priv)) &&
5761                          pipe_config->pch_pfit.enabled;
5762         if (psl_clkgate_wa)
5763                 glk_pipe_scaler_clock_gating_wa(dev_priv, pipe, true);
5764
5765         if (INTEL_GEN(dev_priv) >= 9)
5766                 skylake_pfit_enable(pipe_config);
5767         else
5768                 ironlake_pfit_enable(pipe_config);
5769
5770         /*
5771          * On ILK+ LUT must be loaded before the pipe is running but with
5772          * clocks enabled
5773          */
5774         intel_color_load_luts(&pipe_config->base);
5775
5776         /*
5777          * Display WA #1153: enable hardware to bypass the alpha math
5778          * and rounding for per-pixel values 00 and 0xff
5779          */
5780         if (INTEL_GEN(dev_priv) >= 11) {
5781                 pipe_chicken = I915_READ(PIPE_CHICKEN(pipe));
5782                 if (!(pipe_chicken & PER_PIXEL_ALPHA_BYPASS_EN))
5783                         I915_WRITE_FW(PIPE_CHICKEN(pipe),
5784                                       pipe_chicken | PER_PIXEL_ALPHA_BYPASS_EN);
5785         }
5786
5787         intel_ddi_set_pipe_settings(pipe_config);
5788         if (!transcoder_is_dsi(cpu_transcoder))
5789                 intel_ddi_enable_transcoder_func(pipe_config);
5790
5791         if (dev_priv->display.initial_watermarks != NULL)
5792                 dev_priv->display.initial_watermarks(old_intel_state, pipe_config);
5793
5794         if (INTEL_GEN(dev_priv) >= 11)
5795                 icl_pipe_mbus_enable(intel_crtc);
5796
5797         /* XXX: Do the pipe assertions at the right place for BXT DSI. */
5798         if (!transcoder_is_dsi(cpu_transcoder))
5799                 intel_enable_pipe(pipe_config);
5800
5801         if (pipe_config->has_pch_encoder)
5802                 lpt_pch_enable(old_intel_state, pipe_config);
5803
5804         if (intel_crtc_has_type(pipe_config, INTEL_OUTPUT_DP_MST))
5805                 intel_ddi_set_vc_payload_alloc(pipe_config, true);
5806
5807         assert_vblank_disabled(crtc);
5808         drm_crtc_vblank_on(crtc);
5809
5810         intel_encoders_enable(crtc, pipe_config, old_state);
5811
5812         if (psl_clkgate_wa) {
5813                 intel_wait_for_vblank(dev_priv, pipe);
5814                 glk_pipe_scaler_clock_gating_wa(dev_priv, pipe, false);
5815         }
5816
5817         /* If we change the relative order between pipe/planes enabling, we need
5818          * to change the workaround. */
5819         hsw_workaround_pipe = pipe_config->hsw_workaround_pipe;
5820         if (IS_HASWELL(dev_priv) && hsw_workaround_pipe != INVALID_PIPE) {
5821                 intel_wait_for_vblank(dev_priv, hsw_workaround_pipe);
5822                 intel_wait_for_vblank(dev_priv, hsw_workaround_pipe);
5823         }
5824 }
5825
5826 static void ironlake_pfit_disable(const struct intel_crtc_state *old_crtc_state)
5827 {
5828         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc);
5829         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5830         enum pipe pipe = crtc->pipe;
5831
5832         /* To avoid upsetting the power well on haswell only disable the pfit if
5833          * it's in use. The hw state code will make sure we get this right. */
5834         if (old_crtc_state->pch_pfit.enabled) {
5835                 I915_WRITE(PF_CTL(pipe), 0);
5836                 I915_WRITE(PF_WIN_POS(pipe), 0);
5837                 I915_WRITE(PF_WIN_SZ(pipe), 0);
5838         }
5839 }
5840
5841 static void ironlake_crtc_disable(struct intel_crtc_state *old_crtc_state,
5842                                   struct drm_atomic_state *old_state)
5843 {
5844         struct drm_crtc *crtc = old_crtc_state->base.crtc;
5845         struct drm_device *dev = crtc->dev;
5846         struct drm_i915_private *dev_priv = to_i915(dev);
5847         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5848         int pipe = intel_crtc->pipe;
5849
5850         /*
5851          * Sometimes spurious CPU pipe underruns happen when the
5852          * pipe is already disabled, but FDI RX/TX is still enabled.
5853          * Happens at least with VGA+HDMI cloning. Suppress them.
5854          */
5855         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
5856         intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, false);
5857
5858         intel_encoders_disable(crtc, old_crtc_state, old_state);
5859
5860         drm_crtc_vblank_off(crtc);
5861         assert_vblank_disabled(crtc);
5862
5863         intel_disable_pipe(old_crtc_state);
5864
5865         ironlake_pfit_disable(old_crtc_state);
5866
5867         if (old_crtc_state->has_pch_encoder)
5868                 ironlake_fdi_disable(crtc);
5869
5870         intel_encoders_post_disable(crtc, old_crtc_state, old_state);
5871
5872         if (old_crtc_state->has_pch_encoder) {
5873                 ironlake_disable_pch_transcoder(dev_priv, pipe);
5874
5875                 if (HAS_PCH_CPT(dev_priv)) {
5876                         i915_reg_t reg;
5877                         u32 temp;
5878
5879                         /* disable TRANS_DP_CTL */
5880                         reg = TRANS_DP_CTL(pipe);
5881                         temp = I915_READ(reg);
5882                         temp &= ~(TRANS_DP_OUTPUT_ENABLE |
5883                                   TRANS_DP_PORT_SEL_MASK);
5884                         temp |= TRANS_DP_PORT_SEL_NONE;
5885                         I915_WRITE(reg, temp);
5886
5887                         /* disable DPLL_SEL */
5888                         temp = I915_READ(PCH_DPLL_SEL);
5889                         temp &= ~(TRANS_DPLL_ENABLE(pipe) | TRANS_DPLLB_SEL(pipe));
5890                         I915_WRITE(PCH_DPLL_SEL, temp);
5891                 }
5892
5893                 ironlake_fdi_pll_disable(intel_crtc);
5894         }
5895
5896         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
5897         intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, true);
5898 }
5899
5900 static void haswell_crtc_disable(struct intel_crtc_state *old_crtc_state,
5901                                  struct drm_atomic_state *old_state)
5902 {
5903         struct drm_crtc *crtc = old_crtc_state->base.crtc;
5904         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
5905         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5906         enum transcoder cpu_transcoder = old_crtc_state->cpu_transcoder;
5907
5908         intel_encoders_disable(crtc, old_crtc_state, old_state);
5909
5910         drm_crtc_vblank_off(crtc);
5911         assert_vblank_disabled(crtc);
5912
5913         /* XXX: Do the pipe assertions at the right place for BXT DSI. */
5914         if (!transcoder_is_dsi(cpu_transcoder))
5915                 intel_disable_pipe(old_crtc_state);
5916
5917         if (intel_crtc_has_type(old_crtc_state, INTEL_OUTPUT_DP_MST))
5918                 intel_ddi_set_vc_payload_alloc(old_crtc_state, false);
5919
5920         if (!transcoder_is_dsi(cpu_transcoder))
5921                 intel_ddi_disable_transcoder_func(old_crtc_state);
5922
5923         intel_dsc_disable(old_crtc_state);
5924
5925         if (INTEL_GEN(dev_priv) >= 9)
5926                 skylake_scaler_disable(intel_crtc);
5927         else
5928                 ironlake_pfit_disable(old_crtc_state);
5929
5930         intel_encoders_post_disable(crtc, old_crtc_state, old_state);
5931
5932         intel_encoders_post_pll_disable(crtc, old_crtc_state, old_state);
5933 }
5934
5935 static void i9xx_pfit_enable(const struct intel_crtc_state *crtc_state)
5936 {
5937         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
5938         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5939
5940         if (!crtc_state->gmch_pfit.control)
5941                 return;
5942
5943         /*
5944          * The panel fitter should only be adjusted whilst the pipe is disabled,
5945          * according to register description and PRM.
5946          */
5947         WARN_ON(I915_READ(PFIT_CONTROL) & PFIT_ENABLE);
5948         assert_pipe_disabled(dev_priv, crtc->pipe);
5949
5950         I915_WRITE(PFIT_PGM_RATIOS, crtc_state->gmch_pfit.pgm_ratios);
5951         I915_WRITE(PFIT_CONTROL, crtc_state->gmch_pfit.control);
5952
5953         /* Border color in case we don't scale up to the full screen. Black by
5954          * default, change to something else for debugging. */
5955         I915_WRITE(BCLRPAT(crtc->pipe), 0);
5956 }
5957
5958 bool intel_port_is_combophy(struct drm_i915_private *dev_priv, enum port port)
5959 {
5960         if (port == PORT_NONE)
5961                 return false;
5962
5963         if (IS_ICELAKE(dev_priv))
5964                 return port <= PORT_B;
5965
5966         return false;
5967 }
5968
5969 bool intel_port_is_tc(struct drm_i915_private *dev_priv, enum port port)
5970 {
5971         if (IS_ICELAKE(dev_priv))
5972                 return port >= PORT_C && port <= PORT_F;
5973
5974         return false;
5975 }
5976
5977 enum tc_port intel_port_to_tc(struct drm_i915_private *dev_priv, enum port port)
5978 {
5979         if (!intel_port_is_tc(dev_priv, port))
5980                 return PORT_TC_NONE;
5981
5982         return port - PORT_C;
5983 }
5984
5985 enum intel_display_power_domain intel_port_to_power_domain(enum port port)
5986 {
5987         switch (port) {
5988         case PORT_A:
5989                 return POWER_DOMAIN_PORT_DDI_A_LANES;
5990         case PORT_B:
5991                 return POWER_DOMAIN_PORT_DDI_B_LANES;
5992         case PORT_C:
5993                 return POWER_DOMAIN_PORT_DDI_C_LANES;
5994         case PORT_D:
5995                 return POWER_DOMAIN_PORT_DDI_D_LANES;
5996         case PORT_E:
5997                 return POWER_DOMAIN_PORT_DDI_E_LANES;
5998         case PORT_F:
5999                 return POWER_DOMAIN_PORT_DDI_F_LANES;
6000         default:
6001                 MISSING_CASE(port);
6002                 return POWER_DOMAIN_PORT_OTHER;
6003         }
6004 }
6005
6006 enum intel_display_power_domain
6007 intel_aux_power_domain(struct intel_digital_port *dig_port)
6008 {
6009         switch (dig_port->aux_ch) {
6010         case AUX_CH_A:
6011                 return POWER_DOMAIN_AUX_A;
6012         case AUX_CH_B:
6013                 return POWER_DOMAIN_AUX_B;
6014         case AUX_CH_C:
6015                 return POWER_DOMAIN_AUX_C;
6016         case AUX_CH_D:
6017                 return POWER_DOMAIN_AUX_D;
6018         case AUX_CH_E:
6019                 return POWER_DOMAIN_AUX_E;
6020         case AUX_CH_F:
6021                 return POWER_DOMAIN_AUX_F;
6022         default:
6023                 MISSING_CASE(dig_port->aux_ch);
6024                 return POWER_DOMAIN_AUX_A;
6025         }
6026 }
6027
6028 static u64 get_crtc_power_domains(struct drm_crtc *crtc,
6029                                   struct intel_crtc_state *crtc_state)
6030 {
6031         struct drm_device *dev = crtc->dev;
6032         struct drm_i915_private *dev_priv = to_i915(dev);
6033         struct drm_encoder *encoder;
6034         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6035         enum pipe pipe = intel_crtc->pipe;
6036         u64 mask;
6037         enum transcoder transcoder = crtc_state->cpu_transcoder;
6038
6039         if (!crtc_state->base.active)
6040                 return 0;
6041
6042         mask = BIT_ULL(POWER_DOMAIN_PIPE(pipe));
6043         mask |= BIT_ULL(POWER_DOMAIN_TRANSCODER(transcoder));
6044         if (crtc_state->pch_pfit.enabled ||
6045             crtc_state->pch_pfit.force_thru)
6046                 mask |= BIT_ULL(POWER_DOMAIN_PIPE_PANEL_FITTER(pipe));
6047
6048         drm_for_each_encoder_mask(encoder, dev, crtc_state->base.encoder_mask) {
6049                 struct intel_encoder *intel_encoder = to_intel_encoder(encoder);
6050
6051                 mask |= BIT_ULL(intel_encoder->power_domain);
6052         }
6053
6054         if (HAS_DDI(dev_priv) && crtc_state->has_audio)
6055                 mask |= BIT_ULL(POWER_DOMAIN_AUDIO);
6056
6057         if (crtc_state->shared_dpll)
6058                 mask |= BIT_ULL(POWER_DOMAIN_PLLS);
6059
6060         return mask;
6061 }
6062
6063 static u64
6064 modeset_get_crtc_power_domains(struct drm_crtc *crtc,
6065                                struct intel_crtc_state *crtc_state)
6066 {
6067         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
6068         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6069         enum intel_display_power_domain domain;
6070         u64 domains, new_domains, old_domains;
6071
6072         old_domains = intel_crtc->enabled_power_domains;
6073         intel_crtc->enabled_power_domains = new_domains =
6074                 get_crtc_power_domains(crtc, crtc_state);
6075
6076         domains = new_domains & ~old_domains;
6077
6078         for_each_power_domain(domain, domains)
6079                 intel_display_power_get(dev_priv, domain);
6080
6081         return old_domains & ~new_domains;
6082 }
6083
6084 static void modeset_put_power_domains(struct drm_i915_private *dev_priv,
6085                                       u64 domains)
6086 {
6087         enum intel_display_power_domain domain;
6088
6089         for_each_power_domain(domain, domains)
6090                 intel_display_power_put(dev_priv, domain);
6091 }
6092
6093 static void valleyview_crtc_enable(struct intel_crtc_state *pipe_config,
6094                                    struct drm_atomic_state *old_state)
6095 {
6096         struct intel_atomic_state *old_intel_state =
6097                 to_intel_atomic_state(old_state);
6098         struct drm_crtc *crtc = pipe_config->base.crtc;
6099         struct drm_device *dev = crtc->dev;
6100         struct drm_i915_private *dev_priv = to_i915(dev);
6101         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6102         int pipe = intel_crtc->pipe;
6103
6104         if (WARN_ON(intel_crtc->active))
6105                 return;
6106
6107         if (intel_crtc_has_dp_encoder(pipe_config))
6108                 intel_dp_set_m_n(pipe_config, M1_N1);
6109
6110         intel_set_pipe_timings(pipe_config);
6111         intel_set_pipe_src_size(pipe_config);
6112
6113         if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_B) {
6114                 I915_WRITE(CHV_BLEND(pipe), CHV_BLEND_LEGACY);
6115                 I915_WRITE(CHV_CANVAS(pipe), 0);
6116         }
6117
6118         i9xx_set_pipeconf(pipe_config);
6119
6120         intel_color_set_csc(&pipe_config->base);
6121
6122         intel_crtc->active = true;
6123
6124         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
6125
6126         intel_encoders_pre_pll_enable(crtc, pipe_config, old_state);
6127
6128         if (IS_CHERRYVIEW(dev_priv)) {
6129                 chv_prepare_pll(intel_crtc, pipe_config);
6130                 chv_enable_pll(intel_crtc, pipe_config);
6131         } else {
6132                 vlv_prepare_pll(intel_crtc, pipe_config);
6133                 vlv_enable_pll(intel_crtc, pipe_config);
6134         }
6135
6136         intel_encoders_pre_enable(crtc, pipe_config, old_state);
6137
6138         i9xx_pfit_enable(pipe_config);
6139
6140         intel_color_load_luts(&pipe_config->base);
6141
6142         dev_priv->display.initial_watermarks(old_intel_state,
6143                                              pipe_config);
6144         intel_enable_pipe(pipe_config);
6145
6146         assert_vblank_disabled(crtc);
6147         drm_crtc_vblank_on(crtc);
6148
6149         intel_encoders_enable(crtc, pipe_config, old_state);
6150 }
6151
6152 static void i9xx_set_pll_dividers(const struct intel_crtc_state *crtc_state)
6153 {
6154         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
6155         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6156
6157         I915_WRITE(FP0(crtc->pipe), crtc_state->dpll_hw_state.fp0);
6158         I915_WRITE(FP1(crtc->pipe), crtc_state->dpll_hw_state.fp1);
6159 }
6160
6161 static void i9xx_crtc_enable(struct intel_crtc_state *pipe_config,
6162                              struct drm_atomic_state *old_state)
6163 {
6164         struct intel_atomic_state *old_intel_state =
6165                 to_intel_atomic_state(old_state);
6166         struct drm_crtc *crtc = pipe_config->base.crtc;
6167         struct drm_device *dev = crtc->dev;
6168         struct drm_i915_private *dev_priv = to_i915(dev);
6169         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6170         enum pipe pipe = intel_crtc->pipe;
6171
6172         if (WARN_ON(intel_crtc->active))
6173                 return;
6174
6175         i9xx_set_pll_dividers(pipe_config);
6176
6177         if (intel_crtc_has_dp_encoder(pipe_config))
6178                 intel_dp_set_m_n(pipe_config, M1_N1);
6179
6180         intel_set_pipe_timings(pipe_config);
6181         intel_set_pipe_src_size(pipe_config);
6182
6183         i9xx_set_pipeconf(pipe_config);
6184
6185         intel_crtc->active = true;
6186
6187         if (!IS_GEN2(dev_priv))
6188                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
6189
6190         intel_encoders_pre_enable(crtc, pipe_config, old_state);
6191
6192         i9xx_enable_pll(intel_crtc, pipe_config);
6193
6194         i9xx_pfit_enable(pipe_config);
6195
6196         intel_color_load_luts(&pipe_config->base);
6197
6198         if (dev_priv->display.initial_watermarks != NULL)
6199                 dev_priv->display.initial_watermarks(old_intel_state,
6200                                                      pipe_config);
6201         else
6202                 intel_update_watermarks(intel_crtc);
6203         intel_enable_pipe(pipe_config);
6204
6205         assert_vblank_disabled(crtc);
6206         drm_crtc_vblank_on(crtc);
6207
6208         intel_encoders_enable(crtc, pipe_config, old_state);
6209 }
6210
6211 static void i9xx_pfit_disable(const struct intel_crtc_state *old_crtc_state)
6212 {
6213         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc);
6214         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6215
6216         if (!old_crtc_state->gmch_pfit.control)
6217                 return;
6218
6219         assert_pipe_disabled(dev_priv, crtc->pipe);
6220
6221         DRM_DEBUG_KMS("disabling pfit, current: 0x%08x\n",
6222                       I915_READ(PFIT_CONTROL));
6223         I915_WRITE(PFIT_CONTROL, 0);
6224 }
6225
6226 static void i9xx_crtc_disable(struct intel_crtc_state *old_crtc_state,
6227                               struct drm_atomic_state *old_state)
6228 {
6229         struct drm_crtc *crtc = old_crtc_state->base.crtc;
6230         struct drm_device *dev = crtc->dev;
6231         struct drm_i915_private *dev_priv = to_i915(dev);
6232         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6233         int pipe = intel_crtc->pipe;
6234
6235         /*
6236          * On gen2 planes are double buffered but the pipe isn't, so we must
6237          * wait for planes to fully turn off before disabling the pipe.
6238          */
6239         if (IS_GEN2(dev_priv))
6240                 intel_wait_for_vblank(dev_priv, pipe);
6241
6242         intel_encoders_disable(crtc, old_crtc_state, old_state);
6243
6244         drm_crtc_vblank_off(crtc);
6245         assert_vblank_disabled(crtc);
6246
6247         intel_disable_pipe(old_crtc_state);
6248
6249         i9xx_pfit_disable(old_crtc_state);
6250
6251         intel_encoders_post_disable(crtc, old_crtc_state, old_state);
6252
6253         if (!intel_crtc_has_type(old_crtc_state, INTEL_OUTPUT_DSI)) {
6254                 if (IS_CHERRYVIEW(dev_priv))
6255                         chv_disable_pll(dev_priv, pipe);
6256                 else if (IS_VALLEYVIEW(dev_priv))
6257                         vlv_disable_pll(dev_priv, pipe);
6258                 else
6259                         i9xx_disable_pll(old_crtc_state);
6260         }
6261
6262         intel_encoders_post_pll_disable(crtc, old_crtc_state, old_state);
6263
6264         if (!IS_GEN2(dev_priv))
6265                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
6266
6267         if (!dev_priv->display.initial_watermarks)
6268                 intel_update_watermarks(intel_crtc);
6269
6270         /* clock the pipe down to 640x480@60 to potentially save power */
6271         if (IS_I830(dev_priv))
6272                 i830_enable_pipe(dev_priv, pipe);
6273 }
6274
6275 static void intel_crtc_disable_noatomic(struct drm_crtc *crtc,
6276                                         struct drm_modeset_acquire_ctx *ctx)
6277 {
6278         struct intel_encoder *encoder;
6279         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6280         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
6281         enum intel_display_power_domain domain;
6282         struct intel_plane *plane;
6283         u64 domains;
6284         struct drm_atomic_state *state;
6285         struct intel_crtc_state *crtc_state;
6286         int ret;
6287
6288         if (!intel_crtc->active)
6289                 return;
6290
6291         for_each_intel_plane_on_crtc(&dev_priv->drm, intel_crtc, plane) {
6292                 const struct intel_plane_state *plane_state =
6293                         to_intel_plane_state(plane->base.state);
6294
6295                 if (plane_state->base.visible)
6296                         intel_plane_disable_noatomic(intel_crtc, plane);
6297         }
6298
6299         state = drm_atomic_state_alloc(crtc->dev);
6300         if (!state) {
6301                 DRM_DEBUG_KMS("failed to disable [CRTC:%d:%s], out of memory",
6302                               crtc->base.id, crtc->name);
6303                 return;
6304         }
6305
6306         state->acquire_ctx = ctx;
6307
6308         /* Everything's already locked, -EDEADLK can't happen. */
6309         crtc_state = intel_atomic_get_crtc_state(state, intel_crtc);
6310         ret = drm_atomic_add_affected_connectors(state, crtc);
6311
6312         WARN_ON(IS_ERR(crtc_state) || ret);
6313
6314         dev_priv->display.crtc_disable(crtc_state, state);
6315
6316         drm_atomic_state_put(state);
6317
6318         DRM_DEBUG_KMS("[CRTC:%d:%s] hw state adjusted, was enabled, now disabled\n",
6319                       crtc->base.id, crtc->name);
6320
6321         WARN_ON(drm_atomic_set_mode_for_crtc(crtc->state, NULL) < 0);
6322         crtc->state->active = false;
6323         intel_crtc->active = false;
6324         crtc->enabled = false;
6325         crtc->state->connector_mask = 0;
6326         crtc->state->encoder_mask = 0;
6327
6328         for_each_encoder_on_crtc(crtc->dev, crtc, encoder)
6329                 encoder->base.crtc = NULL;
6330
6331         intel_fbc_disable(intel_crtc);
6332         intel_update_watermarks(intel_crtc);
6333         intel_disable_shared_dpll(to_intel_crtc_state(crtc->state));
6334
6335         domains = intel_crtc->enabled_power_domains;
6336         for_each_power_domain(domain, domains)
6337                 intel_display_power_put(dev_priv, domain);
6338         intel_crtc->enabled_power_domains = 0;
6339
6340         dev_priv->active_crtcs &= ~(1 << intel_crtc->pipe);
6341         dev_priv->min_cdclk[intel_crtc->pipe] = 0;
6342         dev_priv->min_voltage_level[intel_crtc->pipe] = 0;
6343 }
6344
6345 /*
6346  * turn all crtc's off, but do not adjust state
6347  * This has to be paired with a call to intel_modeset_setup_hw_state.
6348  */
6349 int intel_display_suspend(struct drm_device *dev)
6350 {
6351         struct drm_i915_private *dev_priv = to_i915(dev);
6352         struct drm_atomic_state *state;
6353         int ret;
6354
6355         state = drm_atomic_helper_suspend(dev);
6356         ret = PTR_ERR_OR_ZERO(state);
6357         if (ret)
6358                 DRM_ERROR("Suspending crtc's failed with %i\n", ret);
6359         else
6360                 dev_priv->modeset_restore_state = state;
6361         return ret;
6362 }
6363
6364 void intel_encoder_destroy(struct drm_encoder *encoder)
6365 {
6366         struct intel_encoder *intel_encoder = to_intel_encoder(encoder);
6367
6368         drm_encoder_cleanup(encoder);
6369         kfree(intel_encoder);
6370 }
6371
6372 /* Cross check the actual hw state with our own modeset state tracking (and it's
6373  * internal consistency). */
6374 static void intel_connector_verify_state(struct drm_crtc_state *crtc_state,
6375                                          struct drm_connector_state *conn_state)
6376 {
6377         struct intel_connector *connector = to_intel_connector(conn_state->connector);
6378
6379         DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n",
6380                       connector->base.base.id,
6381                       connector->base.name);
6382
6383         if (connector->get_hw_state(connector)) {
6384                 struct intel_encoder *encoder = connector->encoder;
6385
6386                 I915_STATE_WARN(!crtc_state,
6387                          "connector enabled without attached crtc\n");
6388
6389                 if (!crtc_state)
6390                         return;
6391
6392                 I915_STATE_WARN(!crtc_state->active,
6393                       "connector is active, but attached crtc isn't\n");
6394
6395                 if (!encoder || encoder->type == INTEL_OUTPUT_DP_MST)
6396                         return;
6397
6398                 I915_STATE_WARN(conn_state->best_encoder != &encoder->base,
6399                         "atomic encoder doesn't match attached encoder\n");
6400
6401                 I915_STATE_WARN(conn_state->crtc != encoder->base.crtc,
6402                         "attached encoder crtc differs from connector crtc\n");
6403         } else {
6404                 I915_STATE_WARN(crtc_state && crtc_state->active,
6405                         "attached crtc is active, but connector isn't\n");
6406                 I915_STATE_WARN(!crtc_state && conn_state->best_encoder,
6407                         "best encoder set without crtc!\n");
6408         }
6409 }
6410
6411 static int pipe_required_fdi_lanes(struct intel_crtc_state *crtc_state)
6412 {
6413         if (crtc_state->base.enable && crtc_state->has_pch_encoder)
6414                 return crtc_state->fdi_lanes;
6415
6416         return 0;
6417 }
6418
6419 static int ironlake_check_fdi_lanes(struct drm_device *dev, enum pipe pipe,
6420                                      struct intel_crtc_state *pipe_config)
6421 {
6422         struct drm_i915_private *dev_priv = to_i915(dev);
6423         struct drm_atomic_state *state = pipe_config->base.state;
6424         struct intel_crtc *other_crtc;
6425         struct intel_crtc_state *other_crtc_state;
6426
6427         DRM_DEBUG_KMS("checking fdi config on pipe %c, lanes %i\n",
6428                       pipe_name(pipe), pipe_config->fdi_lanes);
6429         if (pipe_config->fdi_lanes > 4) {
6430                 DRM_DEBUG_KMS("invalid fdi lane config on pipe %c: %i lanes\n",
6431                               pipe_name(pipe), pipe_config->fdi_lanes);
6432                 return -EINVAL;
6433         }
6434
6435         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
6436                 if (pipe_config->fdi_lanes > 2) {
6437                         DRM_DEBUG_KMS("only 2 lanes on haswell, required: %i lanes\n",
6438                                       pipe_config->fdi_lanes);
6439                         return -EINVAL;
6440                 } else {
6441                         return 0;
6442                 }
6443         }
6444
6445         if (INTEL_INFO(dev_priv)->num_pipes == 2)
6446                 return 0;
6447
6448         /* Ivybridge 3 pipe is really complicated */
6449         switch (pipe) {
6450         case PIPE_A:
6451                 return 0;
6452         case PIPE_B:
6453                 if (pipe_config->fdi_lanes <= 2)
6454                         return 0;
6455
6456                 other_crtc = intel_get_crtc_for_pipe(dev_priv, PIPE_C);
6457                 other_crtc_state =
6458                         intel_atomic_get_crtc_state(state, other_crtc);
6459                 if (IS_ERR(other_crtc_state))
6460                         return PTR_ERR(other_crtc_state);
6461
6462                 if (pipe_required_fdi_lanes(other_crtc_state) > 0) {
6463                         DRM_DEBUG_KMS("invalid shared fdi lane config on pipe %c: %i lanes\n",
6464                                       pipe_name(pipe), pipe_config->fdi_lanes);
6465                         return -EINVAL;
6466                 }
6467                 return 0;
6468         case PIPE_C:
6469                 if (pipe_config->fdi_lanes > 2) {
6470                         DRM_DEBUG_KMS("only 2 lanes on pipe %c: required %i lanes\n",
6471                                       pipe_name(pipe), pipe_config->fdi_lanes);
6472                         return -EINVAL;
6473                 }
6474
6475                 other_crtc = intel_get_crtc_for_pipe(dev_priv, PIPE_B);
6476                 other_crtc_state =
6477                         intel_atomic_get_crtc_state(state, other_crtc);
6478                 if (IS_ERR(other_crtc_state))
6479                         return PTR_ERR(other_crtc_state);
6480
6481                 if (pipe_required_fdi_lanes(other_crtc_state) > 2) {
6482                         DRM_DEBUG_KMS("fdi link B uses too many lanes to enable link C\n");
6483                         return -EINVAL;
6484                 }
6485                 return 0;
6486         default:
6487                 BUG();
6488         }
6489 }
6490
6491 #define RETRY 1
6492 static int ironlake_fdi_compute_config(struct intel_crtc *intel_crtc,
6493                                        struct intel_crtc_state *pipe_config)
6494 {
6495         struct drm_device *dev = intel_crtc->base.dev;
6496         const struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode;
6497         int lane, link_bw, fdi_dotclock, ret;
6498         bool needs_recompute = false;
6499
6500 retry:
6501         /* FDI is a binary signal running at ~2.7GHz, encoding
6502          * each output octet as 10 bits. The actual frequency
6503          * is stored as a divider into a 100MHz clock, and the
6504          * mode pixel clock is stored in units of 1KHz.
6505          * Hence the bw of each lane in terms of the mode signal
6506          * is:
6507          */
6508         link_bw = intel_fdi_link_freq(to_i915(dev), pipe_config);
6509
6510         fdi_dotclock = adjusted_mode->crtc_clock;
6511
6512         lane = ironlake_get_lanes_required(fdi_dotclock, link_bw,
6513                                            pipe_config->pipe_bpp);
6514
6515         pipe_config->fdi_lanes = lane;
6516
6517         intel_link_compute_m_n(pipe_config->pipe_bpp, lane, fdi_dotclock,
6518                                link_bw, &pipe_config->fdi_m_n, false);
6519
6520         ret = ironlake_check_fdi_lanes(dev, intel_crtc->pipe, pipe_config);
6521         if (ret == -EDEADLK)
6522                 return ret;
6523
6524         if (ret == -EINVAL && pipe_config->pipe_bpp > 6*3) {
6525                 pipe_config->pipe_bpp -= 2*3;
6526                 DRM_DEBUG_KMS("fdi link bw constraint, reducing pipe bpp to %i\n",
6527                               pipe_config->pipe_bpp);
6528                 needs_recompute = true;
6529                 pipe_config->bw_constrained = true;
6530
6531                 goto retry;
6532         }
6533
6534         if (needs_recompute)
6535                 return RETRY;
6536
6537         return ret;
6538 }
6539
6540 bool hsw_crtc_state_ips_capable(const struct intel_crtc_state *crtc_state)
6541 {
6542         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
6543         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6544
6545         /* IPS only exists on ULT machines and is tied to pipe A. */
6546         if (!hsw_crtc_supports_ips(crtc))
6547                 return false;
6548
6549         if (!i915_modparams.enable_ips)
6550                 return false;
6551
6552         if (crtc_state->pipe_bpp > 24)
6553                 return false;
6554
6555         /*
6556          * We compare against max which means we must take
6557          * the increased cdclk requirement into account when
6558          * calculating the new cdclk.
6559          *
6560          * Should measure whether using a lower cdclk w/o IPS
6561          */
6562         if (IS_BROADWELL(dev_priv) &&
6563             crtc_state->pixel_rate > dev_priv->max_cdclk_freq * 95 / 100)
6564                 return false;
6565
6566         return true;
6567 }
6568
6569 static bool hsw_compute_ips_config(struct intel_crtc_state *crtc_state)
6570 {
6571         struct drm_i915_private *dev_priv =
6572                 to_i915(crtc_state->base.crtc->dev);
6573         struct intel_atomic_state *intel_state =
6574                 to_intel_atomic_state(crtc_state->base.state);
6575
6576         if (!hsw_crtc_state_ips_capable(crtc_state))
6577                 return false;
6578
6579         if (crtc_state->ips_force_disable)
6580                 return false;
6581
6582         /* IPS should be fine as long as at least one plane is enabled. */
6583         if (!(crtc_state->active_planes & ~BIT(PLANE_CURSOR)))
6584                 return false;
6585
6586         /* pixel rate mustn't exceed 95% of cdclk with IPS on BDW */
6587         if (IS_BROADWELL(dev_priv) &&
6588             crtc_state->pixel_rate > intel_state->cdclk.logical.cdclk * 95 / 100)
6589                 return false;
6590
6591         return true;
6592 }
6593
6594 static bool intel_crtc_supports_double_wide(const struct intel_crtc *crtc)
6595 {
6596         const struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6597
6598         /* GDG double wide on either pipe, otherwise pipe A only */
6599         return INTEL_GEN(dev_priv) < 4 &&
6600                 (crtc->pipe == PIPE_A || IS_I915G(dev_priv));
6601 }
6602
6603 static uint32_t ilk_pipe_pixel_rate(const struct intel_crtc_state *pipe_config)
6604 {
6605         uint32_t pixel_rate;
6606
6607         pixel_rate = pipe_config->base.adjusted_mode.crtc_clock;
6608
6609         /*
6610          * We only use IF-ID interlacing. If we ever use
6611          * PF-ID we'll need to adjust the pixel_rate here.
6612          */
6613
6614         if (pipe_config->pch_pfit.enabled) {
6615                 uint64_t pipe_w, pipe_h, pfit_w, pfit_h;
6616                 uint32_t pfit_size = pipe_config->pch_pfit.size;
6617
6618                 pipe_w = pipe_config->pipe_src_w;
6619                 pipe_h = pipe_config->pipe_src_h;
6620
6621                 pfit_w = (pfit_size >> 16) & 0xFFFF;
6622                 pfit_h = pfit_size & 0xFFFF;
6623                 if (pipe_w < pfit_w)
6624                         pipe_w = pfit_w;
6625                 if (pipe_h < pfit_h)
6626                         pipe_h = pfit_h;
6627
6628                 if (WARN_ON(!pfit_w || !pfit_h))
6629                         return pixel_rate;
6630
6631                 pixel_rate = div_u64((uint64_t) pixel_rate * pipe_w * pipe_h,
6632                                      pfit_w * pfit_h);
6633         }
6634
6635         return pixel_rate;
6636 }
6637
6638 static void intel_crtc_compute_pixel_rate(struct intel_crtc_state *crtc_state)
6639 {
6640         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
6641
6642         if (HAS_GMCH_DISPLAY(dev_priv))
6643                 /* FIXME calculate proper pipe pixel rate for GMCH pfit */
6644                 crtc_state->pixel_rate =
6645                         crtc_state->base.adjusted_mode.crtc_clock;
6646         else
6647                 crtc_state->pixel_rate =
6648                         ilk_pipe_pixel_rate(crtc_state);
6649 }
6650
6651 static int intel_crtc_compute_config(struct intel_crtc *crtc,
6652                                      struct intel_crtc_state *pipe_config)
6653 {
6654         struct drm_device *dev = crtc->base.dev;
6655         struct drm_i915_private *dev_priv = to_i915(dev);
6656         const struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode;
6657         int clock_limit = dev_priv->max_dotclk_freq;
6658
6659         if (INTEL_GEN(dev_priv) < 4) {
6660                 clock_limit = dev_priv->max_cdclk_freq * 9 / 10;
6661
6662                 /*
6663                  * Enable double wide mode when the dot clock
6664                  * is > 90% of the (display) core speed.
6665                  */
6666                 if (intel_crtc_supports_double_wide(crtc) &&
6667                     adjusted_mode->crtc_clock > clock_limit) {
6668                         clock_limit = dev_priv->max_dotclk_freq;
6669                         pipe_config->double_wide = true;
6670                 }
6671         }
6672
6673         if (adjusted_mode->crtc_clock > clock_limit) {
6674                 DRM_DEBUG_KMS("requested pixel clock (%d kHz) too high (max: %d kHz, double wide: %s)\n",
6675                               adjusted_mode->crtc_clock, clock_limit,
6676                               yesno(pipe_config->double_wide));
6677                 return -EINVAL;
6678         }
6679
6680         if ((pipe_config->output_format == INTEL_OUTPUT_FORMAT_YCBCR420 ||
6681              pipe_config->output_format == INTEL_OUTPUT_FORMAT_YCBCR444) &&
6682              pipe_config->base.ctm) {
6683                 /*
6684                  * There is only one pipe CSC unit per pipe, and we need that
6685                  * for output conversion from RGB->YCBCR. So if CTM is already
6686                  * applied we can't support YCBCR420 output.
6687                  */
6688                 DRM_DEBUG_KMS("YCBCR420 and CTM together are not possible\n");
6689                 return -EINVAL;
6690         }
6691
6692         /*
6693          * Pipe horizontal size must be even in:
6694          * - DVO ganged mode
6695          * - LVDS dual channel mode
6696          * - Double wide pipe
6697          */
6698         if (pipe_config->pipe_src_w & 1) {
6699                 if (pipe_config->double_wide) {
6700                         DRM_DEBUG_KMS("Odd pipe source width not supported with double wide pipe\n");
6701                         return -EINVAL;
6702                 }
6703
6704                 if (intel_crtc_has_type(pipe_config, INTEL_OUTPUT_LVDS) &&
6705                     intel_is_dual_link_lvds(dev)) {
6706                         DRM_DEBUG_KMS("Odd pipe source width not supported with dual link LVDS\n");
6707                         return -EINVAL;
6708                 }
6709         }
6710
6711         /* Cantiga+ cannot handle modes with a hsync front porch of 0.
6712          * WaPruneModeWithIncorrectHsyncOffset:ctg,elk,ilk,snb,ivb,vlv,hsw.
6713          */
6714         if ((INTEL_GEN(dev_priv) > 4 || IS_G4X(dev_priv)) &&
6715                 adjusted_mode->crtc_hsync_start == adjusted_mode->crtc_hdisplay)
6716                 return -EINVAL;
6717
6718         intel_crtc_compute_pixel_rate(pipe_config);
6719
6720         if (pipe_config->has_pch_encoder)
6721                 return ironlake_fdi_compute_config(crtc, pipe_config);
6722
6723         return 0;
6724 }
6725
6726 static void
6727 intel_reduce_m_n_ratio(uint32_t *num, uint32_t *den)
6728 {
6729         while (*num > DATA_LINK_M_N_MASK ||
6730                *den > DATA_LINK_M_N_MASK) {
6731                 *num >>= 1;
6732                 *den >>= 1;
6733         }
6734 }
6735
6736 static void compute_m_n(unsigned int m, unsigned int n,
6737                         uint32_t *ret_m, uint32_t *ret_n,
6738                         bool constant_n)
6739 {
6740         /*
6741          * Several DP dongles in particular seem to be fussy about
6742          * too large link M/N values. Give N value as 0x8000 that
6743          * should be acceptable by specific devices. 0x8000 is the
6744          * specified fixed N value for asynchronous clock mode,
6745          * which the devices expect also in synchronous clock mode.
6746          */
6747         if (constant_n)
6748                 *ret_n = 0x8000;
6749         else
6750                 *ret_n = min_t(unsigned int, roundup_pow_of_two(n), DATA_LINK_N_MAX);
6751
6752         *ret_m = div_u64((uint64_t) m * *ret_n, n);
6753         intel_reduce_m_n_ratio(ret_m, ret_n);
6754 }
6755
6756 void
6757 intel_link_compute_m_n(u16 bits_per_pixel, int nlanes,
6758                        int pixel_clock, int link_clock,
6759                        struct intel_link_m_n *m_n,
6760                        bool constant_n)
6761 {
6762         m_n->tu = 64;
6763
6764         compute_m_n(bits_per_pixel * pixel_clock,
6765                     link_clock * nlanes * 8,
6766                     &m_n->gmch_m, &m_n->gmch_n,
6767                     constant_n);
6768
6769         compute_m_n(pixel_clock, link_clock,
6770                     &m_n->link_m, &m_n->link_n,
6771                     constant_n);
6772 }
6773
6774 static inline bool intel_panel_use_ssc(struct drm_i915_private *dev_priv)
6775 {
6776         if (i915_modparams.panel_use_ssc >= 0)
6777                 return i915_modparams.panel_use_ssc != 0;
6778         return dev_priv->vbt.lvds_use_ssc
6779                 && !(dev_priv->quirks & QUIRK_LVDS_SSC_DISABLE);
6780 }
6781
6782 static uint32_t pnv_dpll_compute_fp(struct dpll *dpll)
6783 {
6784         return (1 << dpll->n) << 16 | dpll->m2;
6785 }
6786
6787 static uint32_t i9xx_dpll_compute_fp(struct dpll *dpll)
6788 {
6789         return dpll->n << 16 | dpll->m1 << 8 | dpll->m2;
6790 }
6791
6792 static void i9xx_update_pll_dividers(struct intel_crtc *crtc,
6793                                      struct intel_crtc_state *crtc_state,
6794                                      struct dpll *reduced_clock)
6795 {
6796         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6797         u32 fp, fp2 = 0;
6798
6799         if (IS_PINEVIEW(dev_priv)) {
6800                 fp = pnv_dpll_compute_fp(&crtc_state->dpll);
6801                 if (reduced_clock)
6802                         fp2 = pnv_dpll_compute_fp(reduced_clock);
6803         } else {
6804                 fp = i9xx_dpll_compute_fp(&crtc_state->dpll);
6805                 if (reduced_clock)
6806                         fp2 = i9xx_dpll_compute_fp(reduced_clock);
6807         }
6808
6809         crtc_state->dpll_hw_state.fp0 = fp;
6810
6811         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS) &&
6812             reduced_clock) {
6813                 crtc_state->dpll_hw_state.fp1 = fp2;
6814         } else {
6815                 crtc_state->dpll_hw_state.fp1 = fp;
6816         }
6817 }
6818
6819 static void vlv_pllb_recal_opamp(struct drm_i915_private *dev_priv, enum pipe
6820                 pipe)
6821 {
6822         u32 reg_val;
6823
6824         /*
6825          * PLLB opamp always calibrates to max value of 0x3f, force enable it
6826          * and set it to a reasonable value instead.
6827          */
6828         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW9(1));
6829         reg_val &= 0xffffff00;
6830         reg_val |= 0x00000030;
6831         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW9(1), reg_val);
6832
6833         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_REF_DW13);
6834         reg_val &= 0x00ffffff;
6835         reg_val |= 0x8c000000;
6836         vlv_dpio_write(dev_priv, pipe, VLV_REF_DW13, reg_val);
6837
6838         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW9(1));
6839         reg_val &= 0xffffff00;
6840         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW9(1), reg_val);
6841
6842         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_REF_DW13);
6843         reg_val &= 0x00ffffff;
6844         reg_val |= 0xb0000000;
6845         vlv_dpio_write(dev_priv, pipe, VLV_REF_DW13, reg_val);
6846 }
6847
6848 static void intel_pch_transcoder_set_m_n(const struct intel_crtc_state *crtc_state,
6849                                          const struct intel_link_m_n *m_n)
6850 {
6851         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
6852         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6853         enum pipe pipe = crtc->pipe;
6854
6855         I915_WRITE(PCH_TRANS_DATA_M1(pipe), TU_SIZE(m_n->tu) | m_n->gmch_m);
6856         I915_WRITE(PCH_TRANS_DATA_N1(pipe), m_n->gmch_n);
6857         I915_WRITE(PCH_TRANS_LINK_M1(pipe), m_n->link_m);
6858         I915_WRITE(PCH_TRANS_LINK_N1(pipe), m_n->link_n);
6859 }
6860
6861 static bool transcoder_has_m2_n2(struct drm_i915_private *dev_priv,
6862                                  enum transcoder transcoder)
6863 {
6864         if (IS_HASWELL(dev_priv))
6865                 return transcoder == TRANSCODER_EDP;
6866
6867         /*
6868          * Strictly speaking some registers are available before
6869          * gen7, but we only support DRRS on gen7+
6870          */
6871         return IS_GEN7(dev_priv) || IS_CHERRYVIEW(dev_priv);
6872 }
6873
6874 static void intel_cpu_transcoder_set_m_n(const struct intel_crtc_state *crtc_state,
6875                                          const struct intel_link_m_n *m_n,
6876                                          const struct intel_link_m_n *m2_n2)
6877 {
6878         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
6879         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6880         enum pipe pipe = crtc->pipe;
6881         enum transcoder transcoder = crtc_state->cpu_transcoder;
6882
6883         if (INTEL_GEN(dev_priv) >= 5) {
6884                 I915_WRITE(PIPE_DATA_M1(transcoder), TU_SIZE(m_n->tu) | m_n->gmch_m);
6885                 I915_WRITE(PIPE_DATA_N1(transcoder), m_n->gmch_n);
6886                 I915_WRITE(PIPE_LINK_M1(transcoder), m_n->link_m);
6887                 I915_WRITE(PIPE_LINK_N1(transcoder), m_n->link_n);
6888                 /*
6889                  *  M2_N2 registers are set only if DRRS is supported
6890                  * (to make sure the registers are not unnecessarily accessed).
6891                  */
6892                 if (m2_n2 && crtc_state->has_drrs &&
6893                     transcoder_has_m2_n2(dev_priv, transcoder)) {
6894                         I915_WRITE(PIPE_DATA_M2(transcoder),
6895                                         TU_SIZE(m2_n2->tu) | m2_n2->gmch_m);
6896                         I915_WRITE(PIPE_DATA_N2(transcoder), m2_n2->gmch_n);
6897                         I915_WRITE(PIPE_LINK_M2(transcoder), m2_n2->link_m);
6898                         I915_WRITE(PIPE_LINK_N2(transcoder), m2_n2->link_n);
6899                 }
6900         } else {
6901                 I915_WRITE(PIPE_DATA_M_G4X(pipe), TU_SIZE(m_n->tu) | m_n->gmch_m);
6902                 I915_WRITE(PIPE_DATA_N_G4X(pipe), m_n->gmch_n);
6903                 I915_WRITE(PIPE_LINK_M_G4X(pipe), m_n->link_m);
6904                 I915_WRITE(PIPE_LINK_N_G4X(pipe), m_n->link_n);
6905         }
6906 }
6907
6908 void intel_dp_set_m_n(const struct intel_crtc_state *crtc_state, enum link_m_n_set m_n)
6909 {
6910         const struct intel_link_m_n *dp_m_n, *dp_m2_n2 = NULL;
6911
6912         if (m_n == M1_N1) {
6913                 dp_m_n = &crtc_state->dp_m_n;
6914                 dp_m2_n2 = &crtc_state->dp_m2_n2;
6915         } else if (m_n == M2_N2) {
6916
6917                 /*
6918                  * M2_N2 registers are not supported. Hence m2_n2 divider value
6919                  * needs to be programmed into M1_N1.
6920                  */
6921                 dp_m_n = &crtc_state->dp_m2_n2;
6922         } else {
6923                 DRM_ERROR("Unsupported divider value\n");
6924                 return;
6925         }
6926
6927         if (crtc_state->has_pch_encoder)
6928                 intel_pch_transcoder_set_m_n(crtc_state, &crtc_state->dp_m_n);
6929         else
6930                 intel_cpu_transcoder_set_m_n(crtc_state, dp_m_n, dp_m2_n2);
6931 }
6932
6933 static void vlv_compute_dpll(struct intel_crtc *crtc,
6934                              struct intel_crtc_state *pipe_config)
6935 {
6936         pipe_config->dpll_hw_state.dpll = DPLL_INTEGRATED_REF_CLK_VLV |
6937                 DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
6938         if (crtc->pipe != PIPE_A)
6939                 pipe_config->dpll_hw_state.dpll |= DPLL_INTEGRATED_CRI_CLK_VLV;
6940
6941         /* DPLL not used with DSI, but still need the rest set up */
6942         if (!intel_crtc_has_type(pipe_config, INTEL_OUTPUT_DSI))
6943                 pipe_config->dpll_hw_state.dpll |= DPLL_VCO_ENABLE |
6944                         DPLL_EXT_BUFFER_ENABLE_VLV;
6945
6946         pipe_config->dpll_hw_state.dpll_md =
6947                 (pipe_config->pixel_multiplier - 1) << DPLL_MD_UDI_MULTIPLIER_SHIFT;
6948 }
6949
6950 static void chv_compute_dpll(struct intel_crtc *crtc,
6951                              struct intel_crtc_state *pipe_config)
6952 {
6953         pipe_config->dpll_hw_state.dpll = DPLL_SSC_REF_CLK_CHV |
6954                 DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
6955         if (crtc->pipe != PIPE_A)
6956                 pipe_config->dpll_hw_state.dpll |= DPLL_INTEGRATED_CRI_CLK_VLV;
6957
6958         /* DPLL not used with DSI, but still need the rest set up */
6959         if (!intel_crtc_has_type(pipe_config, INTEL_OUTPUT_DSI))
6960                 pipe_config->dpll_hw_state.dpll |= DPLL_VCO_ENABLE;
6961
6962         pipe_config->dpll_hw_state.dpll_md =
6963                 (pipe_config->pixel_multiplier - 1) << DPLL_MD_UDI_MULTIPLIER_SHIFT;
6964 }
6965
6966 static void vlv_prepare_pll(struct intel_crtc *crtc,
6967                             const struct intel_crtc_state *pipe_config)
6968 {
6969         struct drm_device *dev = crtc->base.dev;
6970         struct drm_i915_private *dev_priv = to_i915(dev);
6971         enum pipe pipe = crtc->pipe;
6972         u32 mdiv;
6973         u32 bestn, bestm1, bestm2, bestp1, bestp2;
6974         u32 coreclk, reg_val;
6975
6976         /* Enable Refclk */
6977         I915_WRITE(DPLL(pipe),
6978                    pipe_config->dpll_hw_state.dpll &
6979                    ~(DPLL_VCO_ENABLE | DPLL_EXT_BUFFER_ENABLE_VLV));
6980
6981         /* No need to actually set up the DPLL with DSI */
6982         if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0)
6983                 return;
6984
6985         mutex_lock(&dev_priv->sb_lock);
6986
6987         bestn = pipe_config->dpll.n;
6988         bestm1 = pipe_config->dpll.m1;
6989         bestm2 = pipe_config->dpll.m2;
6990         bestp1 = pipe_config->dpll.p1;
6991         bestp2 = pipe_config->dpll.p2;
6992
6993         /* See eDP HDMI DPIO driver vbios notes doc */
6994
6995         /* PLL B needs special handling */
6996         if (pipe == PIPE_B)
6997                 vlv_pllb_recal_opamp(dev_priv, pipe);
6998
6999         /* Set up Tx target for periodic Rcomp update */
7000         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW9_BCAST, 0x0100000f);
7001
7002         /* Disable target IRef on PLL */
7003         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW8(pipe));
7004         reg_val &= 0x00ffffff;
7005         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW8(pipe), reg_val);
7006
7007         /* Disable fast lock */
7008         vlv_dpio_write(dev_priv, pipe, VLV_CMN_DW0, 0x610);
7009
7010         /* Set idtafcrecal before PLL is enabled */
7011         mdiv = ((bestm1 << DPIO_M1DIV_SHIFT) | (bestm2 & DPIO_M2DIV_MASK));
7012         mdiv |= ((bestp1 << DPIO_P1_SHIFT) | (bestp2 << DPIO_P2_SHIFT));
7013         mdiv |= ((bestn << DPIO_N_SHIFT));
7014         mdiv |= (1 << DPIO_K_SHIFT);
7015
7016         /*
7017          * Post divider depends on pixel clock rate, DAC vs digital (and LVDS,
7018          * but we don't support that).
7019          * Note: don't use the DAC post divider as it seems unstable.
7020          */
7021         mdiv |= (DPIO_POST_DIV_HDMIDP << DPIO_POST_DIV_SHIFT);
7022         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW3(pipe), mdiv);
7023
7024         mdiv |= DPIO_ENABLE_CALIBRATION;
7025         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW3(pipe), mdiv);
7026
7027         /* Set HBR and RBR LPF coefficients */
7028         if (pipe_config->port_clock == 162000 ||
7029             intel_crtc_has_type(pipe_config, INTEL_OUTPUT_ANALOG) ||
7030             intel_crtc_has_type(pipe_config, INTEL_OUTPUT_HDMI))
7031                 vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW10(pipe),
7032                                  0x009f0003);
7033         else
7034                 vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW10(pipe),
7035                                  0x00d0000f);
7036
7037         if (intel_crtc_has_dp_encoder(pipe_config)) {
7038                 /* Use SSC source */
7039                 if (pipe == PIPE_A)
7040                         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
7041                                          0x0df40000);
7042                 else
7043                         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
7044                                          0x0df70000);
7045         } else { /* HDMI or VGA */
7046                 /* Use bend source */
7047                 if (pipe == PIPE_A)
7048                         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
7049                                          0x0df70000);
7050                 else
7051                         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
7052                                          0x0df40000);
7053         }
7054
7055         coreclk = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW7(pipe));
7056         coreclk = (coreclk & 0x0000ff00) | 0x01c00000;
7057         if (intel_crtc_has_dp_encoder(pipe_config))
7058                 coreclk |= 0x01000000;
7059         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW7(pipe), coreclk);
7060
7061         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW11(pipe), 0x87871000);
7062         mutex_unlock(&dev_priv->sb_lock);
7063 }
7064
7065 static void chv_prepare_pll(struct intel_crtc *crtc,
7066                             const struct intel_crtc_state *pipe_config)
7067 {
7068         struct drm_device *dev = crtc->base.dev;
7069         struct drm_i915_private *dev_priv = to_i915(dev);
7070         enum pipe pipe = crtc->pipe;
7071         enum dpio_channel port = vlv_pipe_to_channel(pipe);
7072         u32 loopfilter, tribuf_calcntr;
7073         u32 bestn, bestm1, bestm2, bestp1, bestp2, bestm2_frac;
7074         u32 dpio_val;
7075         int vco;
7076
7077         /* Enable Refclk and SSC */
7078         I915_WRITE(DPLL(pipe),
7079                    pipe_config->dpll_hw_state.dpll & ~DPLL_VCO_ENABLE);
7080
7081         /* No need to actually set up the DPLL with DSI */
7082         if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0)
7083                 return;
7084
7085         bestn = pipe_config->dpll.n;
7086         bestm2_frac = pipe_config->dpll.m2 & 0x3fffff;
7087         bestm1 = pipe_config->dpll.m1;
7088         bestm2 = pipe_config->dpll.m2 >> 22;
7089         bestp1 = pipe_config->dpll.p1;
7090         bestp2 = pipe_config->dpll.p2;
7091         vco = pipe_config->dpll.vco;
7092         dpio_val = 0;
7093         loopfilter = 0;
7094
7095         mutex_lock(&dev_priv->sb_lock);
7096
7097         /* p1 and p2 divider */
7098         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW13(port),
7099                         5 << DPIO_CHV_S1_DIV_SHIFT |
7100                         bestp1 << DPIO_CHV_P1_DIV_SHIFT |
7101                         bestp2 << DPIO_CHV_P2_DIV_SHIFT |
7102                         1 << DPIO_CHV_K_DIV_SHIFT);
7103
7104         /* Feedback post-divider - m2 */
7105         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW0(port), bestm2);
7106
7107         /* Feedback refclk divider - n and m1 */
7108         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW1(port),
7109                         DPIO_CHV_M1_DIV_BY_2 |
7110                         1 << DPIO_CHV_N_DIV_SHIFT);
7111
7112         /* M2 fraction division */
7113         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW2(port), bestm2_frac);
7114
7115         /* M2 fraction division enable */
7116         dpio_val = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW3(port));
7117         dpio_val &= ~(DPIO_CHV_FEEDFWD_GAIN_MASK | DPIO_CHV_FRAC_DIV_EN);
7118         dpio_val |= (2 << DPIO_CHV_FEEDFWD_GAIN_SHIFT);
7119         if (bestm2_frac)
7120                 dpio_val |= DPIO_CHV_FRAC_DIV_EN;
7121         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW3(port), dpio_val);
7122
7123         /* Program digital lock detect threshold */
7124         dpio_val = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW9(port));
7125         dpio_val &= ~(DPIO_CHV_INT_LOCK_THRESHOLD_MASK |
7126                                         DPIO_CHV_INT_LOCK_THRESHOLD_SEL_COARSE);
7127         dpio_val |= (0x5 << DPIO_CHV_INT_LOCK_THRESHOLD_SHIFT);
7128         if (!bestm2_frac)
7129                 dpio_val |= DPIO_CHV_INT_LOCK_THRESHOLD_SEL_COARSE;
7130         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW9(port), dpio_val);
7131
7132         /* Loop filter */
7133         if (vco == 5400000) {
7134                 loopfilter |= (0x3 << DPIO_CHV_PROP_COEFF_SHIFT);
7135                 loopfilter |= (0x8 << DPIO_CHV_INT_COEFF_SHIFT);
7136                 loopfilter |= (0x1 << DPIO_CHV_GAIN_CTRL_SHIFT);
7137                 tribuf_calcntr = 0x9;
7138         } else if (vco <= 6200000) {
7139                 loopfilter |= (0x5 << DPIO_CHV_PROP_COEFF_SHIFT);
7140                 loopfilter |= (0xB << DPIO_CHV_INT_COEFF_SHIFT);
7141                 loopfilter |= (0x3 << DPIO_CHV_GAIN_CTRL_SHIFT);
7142                 tribuf_calcntr = 0x9;
7143         } else if (vco <= 6480000) {
7144                 loopfilter |= (0x4 << DPIO_CHV_PROP_COEFF_SHIFT);
7145                 loopfilter |= (0x9 << DPIO_CHV_INT_COEFF_SHIFT);
7146                 loopfilter |= (0x3 << DPIO_CHV_GAIN_CTRL_SHIFT);
7147                 tribuf_calcntr = 0x8;
7148         } else {
7149                 /* Not supported. Apply the same limits as in the max case */
7150                 loopfilter |= (0x4 << DPIO_CHV_PROP_COEFF_SHIFT);
7151                 loopfilter |= (0x9 << DPIO_CHV_INT_COEFF_SHIFT);
7152                 loopfilter |= (0x3 << DPIO_CHV_GAIN_CTRL_SHIFT);
7153                 tribuf_calcntr = 0;
7154         }
7155         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW6(port), loopfilter);
7156
7157         dpio_val = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW8(port));
7158         dpio_val &= ~DPIO_CHV_TDC_TARGET_CNT_MASK;
7159         dpio_val |= (tribuf_calcntr << DPIO_CHV_TDC_TARGET_CNT_SHIFT);
7160         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW8(port), dpio_val);
7161
7162         /* AFC Recal */
7163         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW14(port),
7164                         vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW14(port)) |
7165                         DPIO_AFC_RECAL);
7166
7167         mutex_unlock(&dev_priv->sb_lock);
7168 }
7169
7170 /**
7171  * vlv_force_pll_on - forcibly enable just the PLL
7172  * @dev_priv: i915 private structure
7173  * @pipe: pipe PLL to enable
7174  * @dpll: PLL configuration
7175  *
7176  * Enable the PLL for @pipe using the supplied @dpll config. To be used
7177  * in cases where we need the PLL enabled even when @pipe is not going to
7178  * be enabled.
7179  */
7180 int vlv_force_pll_on(struct drm_i915_private *dev_priv, enum pipe pipe,
7181                      const struct dpll *dpll)
7182 {
7183         struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
7184         struct intel_crtc_state *pipe_config;
7185
7186         pipe_config = kzalloc(sizeof(*pipe_config), GFP_KERNEL);
7187         if (!pipe_config)
7188                 return -ENOMEM;
7189
7190         pipe_config->base.crtc = &crtc->base;
7191         pipe_config->pixel_multiplier = 1;
7192         pipe_config->dpll = *dpll;
7193
7194         if (IS_CHERRYVIEW(dev_priv)) {
7195                 chv_compute_dpll(crtc, pipe_config);
7196                 chv_prepare_pll(crtc, pipe_config);
7197                 chv_enable_pll(crtc, pipe_config);
7198         } else {
7199                 vlv_compute_dpll(crtc, pipe_config);
7200                 vlv_prepare_pll(crtc, pipe_config);
7201                 vlv_enable_pll(crtc, pipe_config);
7202         }
7203
7204         kfree(pipe_config);
7205
7206         return 0;
7207 }
7208
7209 /**
7210  * vlv_force_pll_off - forcibly disable just the PLL
7211  * @dev_priv: i915 private structure
7212  * @pipe: pipe PLL to disable
7213  *
7214  * Disable the PLL for @pipe. To be used in cases where we need
7215  * the PLL enabled even when @pipe is not going to be enabled.
7216  */
7217 void vlv_force_pll_off(struct drm_i915_private *dev_priv, enum pipe pipe)
7218 {
7219         if (IS_CHERRYVIEW(dev_priv))
7220                 chv_disable_pll(dev_priv, pipe);
7221         else
7222                 vlv_disable_pll(dev_priv, pipe);
7223 }
7224
7225 static void i9xx_compute_dpll(struct intel_crtc *crtc,
7226                               struct intel_crtc_state *crtc_state,
7227                               struct dpll *reduced_clock)
7228 {
7229         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7230         u32 dpll;
7231         struct dpll *clock = &crtc_state->dpll;
7232
7233         i9xx_update_pll_dividers(crtc, crtc_state, reduced_clock);
7234
7235         dpll = DPLL_VGA_MODE_DIS;
7236
7237         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS))
7238                 dpll |= DPLLB_MODE_LVDS;
7239         else
7240                 dpll |= DPLLB_MODE_DAC_SERIAL;
7241
7242         if (IS_I945G(dev_priv) || IS_I945GM(dev_priv) ||
7243             IS_G33(dev_priv) || IS_PINEVIEW(dev_priv)) {
7244                 dpll |= (crtc_state->pixel_multiplier - 1)
7245                         << SDVO_MULTIPLIER_SHIFT_HIRES;
7246         }
7247
7248         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO) ||
7249             intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI))
7250                 dpll |= DPLL_SDVO_HIGH_SPEED;
7251
7252         if (intel_crtc_has_dp_encoder(crtc_state))
7253                 dpll |= DPLL_SDVO_HIGH_SPEED;
7254
7255         /* compute bitmask from p1 value */
7256         if (IS_PINEVIEW(dev_priv))
7257                 dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT_PINEVIEW;
7258         else {
7259                 dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
7260                 if (IS_G4X(dev_priv) && reduced_clock)
7261                         dpll |= (1 << (reduced_clock->p1 - 1)) << DPLL_FPA1_P1_POST_DIV_SHIFT;
7262         }
7263         switch (clock->p2) {
7264         case 5:
7265                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_5;
7266                 break;
7267         case 7:
7268                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_7;
7269                 break;
7270         case 10:
7271                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_10;
7272                 break;
7273         case 14:
7274                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_14;
7275                 break;
7276         }
7277         if (INTEL_GEN(dev_priv) >= 4)
7278                 dpll |= (6 << PLL_LOAD_PULSE_PHASE_SHIFT);
7279
7280         if (crtc_state->sdvo_tv_clock)
7281                 dpll |= PLL_REF_INPUT_TVCLKINBC;
7282         else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS) &&
7283                  intel_panel_use_ssc(dev_priv))
7284                 dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN;
7285         else
7286                 dpll |= PLL_REF_INPUT_DREFCLK;
7287
7288         dpll |= DPLL_VCO_ENABLE;
7289         crtc_state->dpll_hw_state.dpll = dpll;
7290
7291         if (INTEL_GEN(dev_priv) >= 4) {
7292                 u32 dpll_md = (crtc_state->pixel_multiplier - 1)
7293                         << DPLL_MD_UDI_MULTIPLIER_SHIFT;
7294                 crtc_state->dpll_hw_state.dpll_md = dpll_md;
7295         }
7296 }
7297
7298 static void i8xx_compute_dpll(struct intel_crtc *crtc,
7299                               struct intel_crtc_state *crtc_state,
7300                               struct dpll *reduced_clock)
7301 {
7302         struct drm_device *dev = crtc->base.dev;
7303         struct drm_i915_private *dev_priv = to_i915(dev);
7304         u32 dpll;
7305         struct dpll *clock = &crtc_state->dpll;
7306
7307         i9xx_update_pll_dividers(crtc, crtc_state, reduced_clock);
7308
7309         dpll = DPLL_VGA_MODE_DIS;
7310
7311         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
7312                 dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
7313         } else {
7314                 if (clock->p1 == 2)
7315                         dpll |= PLL_P1_DIVIDE_BY_TWO;
7316                 else
7317                         dpll |= (clock->p1 - 2) << DPLL_FPA01_P1_POST_DIV_SHIFT;
7318                 if (clock->p2 == 4)
7319                         dpll |= PLL_P2_DIVIDE_BY_4;
7320         }
7321
7322         if (!IS_I830(dev_priv) &&
7323             intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DVO))
7324                 dpll |= DPLL_DVO_2X_MODE;
7325
7326         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS) &&
7327             intel_panel_use_ssc(dev_priv))
7328                 dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN;
7329         else
7330                 dpll |= PLL_REF_INPUT_DREFCLK;
7331
7332         dpll |= DPLL_VCO_ENABLE;
7333         crtc_state->dpll_hw_state.dpll = dpll;
7334 }
7335
7336 static void intel_set_pipe_timings(const struct intel_crtc_state *crtc_state)
7337 {
7338         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
7339         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7340         enum pipe pipe = crtc->pipe;
7341         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
7342         const struct drm_display_mode *adjusted_mode = &crtc_state->base.adjusted_mode;
7343         uint32_t crtc_vtotal, crtc_vblank_end;
7344         int vsyncshift = 0;
7345
7346         /* We need to be careful not to changed the adjusted mode, for otherwise
7347          * the hw state checker will get angry at the mismatch. */
7348         crtc_vtotal = adjusted_mode->crtc_vtotal;
7349         crtc_vblank_end = adjusted_mode->crtc_vblank_end;
7350
7351         if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) {
7352                 /* the chip adds 2 halflines automatically */
7353                 crtc_vtotal -= 1;
7354                 crtc_vblank_end -= 1;
7355
7356                 if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO))
7357                         vsyncshift = (adjusted_mode->crtc_htotal - 1) / 2;
7358                 else
7359                         vsyncshift = adjusted_mode->crtc_hsync_start -
7360                                 adjusted_mode->crtc_htotal / 2;
7361                 if (vsyncshift < 0)
7362                         vsyncshift += adjusted_mode->crtc_htotal;
7363         }
7364
7365         if (INTEL_GEN(dev_priv) > 3)
7366                 I915_WRITE(VSYNCSHIFT(cpu_transcoder), vsyncshift);
7367
7368         I915_WRITE(HTOTAL(cpu_transcoder),
7369                    (adjusted_mode->crtc_hdisplay - 1) |
7370                    ((adjusted_mode->crtc_htotal - 1) << 16));
7371         I915_WRITE(HBLANK(cpu_transcoder),
7372                    (adjusted_mode->crtc_hblank_start - 1) |
7373                    ((adjusted_mode->crtc_hblank_end - 1) << 16));
7374         I915_WRITE(HSYNC(cpu_transcoder),
7375                    (adjusted_mode->crtc_hsync_start - 1) |
7376                    ((adjusted_mode->crtc_hsync_end - 1) << 16));
7377
7378         I915_WRITE(VTOTAL(cpu_transcoder),
7379                    (adjusted_mode->crtc_vdisplay - 1) |
7380                    ((crtc_vtotal - 1) << 16));
7381         I915_WRITE(VBLANK(cpu_transcoder),
7382                    (adjusted_mode->crtc_vblank_start - 1) |
7383                    ((crtc_vblank_end - 1) << 16));
7384         I915_WRITE(VSYNC(cpu_transcoder),
7385                    (adjusted_mode->crtc_vsync_start - 1) |
7386                    ((adjusted_mode->crtc_vsync_end - 1) << 16));
7387
7388         /* Workaround: when the EDP input selection is B, the VTOTAL_B must be
7389          * programmed with the VTOTAL_EDP value. Same for VTOTAL_C. This is
7390          * documented on the DDI_FUNC_CTL register description, EDP Input Select
7391          * bits. */
7392         if (IS_HASWELL(dev_priv) && cpu_transcoder == TRANSCODER_EDP &&
7393             (pipe == PIPE_B || pipe == PIPE_C))
7394                 I915_WRITE(VTOTAL(pipe), I915_READ(VTOTAL(cpu_transcoder)));
7395
7396 }
7397
7398 static void intel_set_pipe_src_size(const struct intel_crtc_state *crtc_state)
7399 {
7400         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
7401         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7402         enum pipe pipe = crtc->pipe;
7403
7404         /* pipesrc controls the size that is scaled from, which should
7405          * always be the user's requested size.
7406          */
7407         I915_WRITE(PIPESRC(pipe),
7408                    ((crtc_state->pipe_src_w - 1) << 16) |
7409                    (crtc_state->pipe_src_h - 1));
7410 }
7411
7412 static void intel_get_pipe_timings(struct intel_crtc *crtc,
7413                                    struct intel_crtc_state *pipe_config)
7414 {
7415         struct drm_device *dev = crtc->base.dev;
7416         struct drm_i915_private *dev_priv = to_i915(dev);
7417         enum transcoder cpu_transcoder = pipe_config->cpu_transcoder;
7418         uint32_t tmp;
7419
7420         tmp = I915_READ(HTOTAL(cpu_transcoder));
7421         pipe_config->base.adjusted_mode.crtc_hdisplay = (tmp & 0xffff) + 1;
7422         pipe_config->base.adjusted_mode.crtc_htotal = ((tmp >> 16) & 0xffff) + 1;
7423         tmp = I915_READ(HBLANK(cpu_transcoder));
7424         pipe_config->base.adjusted_mode.crtc_hblank_start = (tmp & 0xffff) + 1;
7425         pipe_config->base.adjusted_mode.crtc_hblank_end = ((tmp >> 16) & 0xffff) + 1;
7426         tmp = I915_READ(HSYNC(cpu_transcoder));
7427         pipe_config->base.adjusted_mode.crtc_hsync_start = (tmp & 0xffff) + 1;
7428         pipe_config->base.adjusted_mode.crtc_hsync_end = ((tmp >> 16) & 0xffff) + 1;
7429
7430         tmp = I915_READ(VTOTAL(cpu_transcoder));
7431         pipe_config->base.adjusted_mode.crtc_vdisplay = (tmp & 0xffff) + 1;
7432         pipe_config->base.adjusted_mode.crtc_vtotal = ((tmp >> 16) & 0xffff) + 1;
7433         tmp = I915_READ(VBLANK(cpu_transcoder));
7434         pipe_config->base.adjusted_mode.crtc_vblank_start = (tmp & 0xffff) + 1;
7435         pipe_config->base.adjusted_mode.crtc_vblank_end = ((tmp >> 16) & 0xffff) + 1;
7436         tmp = I915_READ(VSYNC(cpu_transcoder));
7437         pipe_config->base.adjusted_mode.crtc_vsync_start = (tmp & 0xffff) + 1;
7438         pipe_config->base.adjusted_mode.crtc_vsync_end = ((tmp >> 16) & 0xffff) + 1;
7439
7440         if (I915_READ(PIPECONF(cpu_transcoder)) & PIPECONF_INTERLACE_MASK) {
7441                 pipe_config->base.adjusted_mode.flags |= DRM_MODE_FLAG_INTERLACE;
7442                 pipe_config->base.adjusted_mode.crtc_vtotal += 1;
7443                 pipe_config->base.adjusted_mode.crtc_vblank_end += 1;
7444         }
7445 }
7446
7447 static void intel_get_pipe_src_size(struct intel_crtc *crtc,
7448                                     struct intel_crtc_state *pipe_config)
7449 {
7450         struct drm_device *dev = crtc->base.dev;
7451         struct drm_i915_private *dev_priv = to_i915(dev);
7452         u32 tmp;
7453
7454         tmp = I915_READ(PIPESRC(crtc->pipe));
7455         pipe_config->pipe_src_h = (tmp & 0xffff) + 1;
7456         pipe_config->pipe_src_w = ((tmp >> 16) & 0xffff) + 1;
7457
7458         pipe_config->base.mode.vdisplay = pipe_config->pipe_src_h;
7459         pipe_config->base.mode.hdisplay = pipe_config->pipe_src_w;
7460 }
7461
7462 void intel_mode_from_pipe_config(struct drm_display_mode *mode,
7463                                  struct intel_crtc_state *pipe_config)
7464 {
7465         mode->hdisplay = pipe_config->base.adjusted_mode.crtc_hdisplay;
7466         mode->htotal = pipe_config->base.adjusted_mode.crtc_htotal;
7467         mode->hsync_start = pipe_config->base.adjusted_mode.crtc_hsync_start;
7468         mode->hsync_end = pipe_config->base.adjusted_mode.crtc_hsync_end;
7469
7470         mode->vdisplay = pipe_config->base.adjusted_mode.crtc_vdisplay;
7471         mode->vtotal = pipe_config->base.adjusted_mode.crtc_vtotal;
7472         mode->vsync_start = pipe_config->base.adjusted_mode.crtc_vsync_start;
7473         mode->vsync_end = pipe_config->base.adjusted_mode.crtc_vsync_end;
7474
7475         mode->flags = pipe_config->base.adjusted_mode.flags;
7476         mode->type = DRM_MODE_TYPE_DRIVER;
7477
7478         mode->clock = pipe_config->base.adjusted_mode.crtc_clock;
7479
7480         mode->hsync = drm_mode_hsync(mode);
7481         mode->vrefresh = drm_mode_vrefresh(mode);
7482         drm_mode_set_name(mode);
7483 }
7484
7485 static void i9xx_set_pipeconf(const struct intel_crtc_state *crtc_state)
7486 {
7487         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
7488         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7489         uint32_t pipeconf;
7490
7491         pipeconf = 0;
7492
7493         /* we keep both pipes enabled on 830 */
7494         if (IS_I830(dev_priv))
7495                 pipeconf |= I915_READ(PIPECONF(crtc->pipe)) & PIPECONF_ENABLE;
7496
7497         if (crtc_state->double_wide)
7498                 pipeconf |= PIPECONF_DOUBLE_WIDE;
7499
7500         /* only g4x and later have fancy bpc/dither controls */
7501         if (IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
7502             IS_CHERRYVIEW(dev_priv)) {
7503                 /* Bspec claims that we can't use dithering for 30bpp pipes. */
7504                 if (crtc_state->dither && crtc_state->pipe_bpp != 30)
7505                         pipeconf |= PIPECONF_DITHER_EN |
7506                                     PIPECONF_DITHER_TYPE_SP;
7507
7508                 switch (crtc_state->pipe_bpp) {
7509                 case 18:
7510                         pipeconf |= PIPECONF_6BPC;
7511                         break;
7512                 case 24:
7513                         pipeconf |= PIPECONF_8BPC;
7514                         break;
7515                 case 30:
7516                         pipeconf |= PIPECONF_10BPC;
7517                         break;
7518                 default:
7519                         /* Case prevented by intel_choose_pipe_bpp_dither. */
7520                         BUG();
7521                 }
7522         }
7523
7524         if (crtc_state->base.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE) {
7525                 if (INTEL_GEN(dev_priv) < 4 ||
7526                     intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO))
7527                         pipeconf |= PIPECONF_INTERLACE_W_FIELD_INDICATION;
7528                 else
7529                         pipeconf |= PIPECONF_INTERLACE_W_SYNC_SHIFT;
7530         } else
7531                 pipeconf |= PIPECONF_PROGRESSIVE;
7532
7533         if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
7534              crtc_state->limited_color_range)
7535                 pipeconf |= PIPECONF_COLOR_RANGE_SELECT;
7536
7537         I915_WRITE(PIPECONF(crtc->pipe), pipeconf);
7538         POSTING_READ(PIPECONF(crtc->pipe));
7539 }
7540
7541 static int i8xx_crtc_compute_clock(struct intel_crtc *crtc,
7542                                    struct intel_crtc_state *crtc_state)
7543 {
7544         struct drm_device *dev = crtc->base.dev;
7545         struct drm_i915_private *dev_priv = to_i915(dev);
7546         const struct intel_limit *limit;
7547         int refclk = 48000;
7548
7549         memset(&crtc_state->dpll_hw_state, 0,
7550                sizeof(crtc_state->dpll_hw_state));
7551
7552         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
7553                 if (intel_panel_use_ssc(dev_priv)) {
7554                         refclk = dev_priv->vbt.lvds_ssc_freq;
7555                         DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n", refclk);
7556                 }
7557
7558                 limit = &intel_limits_i8xx_lvds;
7559         } else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DVO)) {
7560                 limit = &intel_limits_i8xx_dvo;
7561         } else {
7562                 limit = &intel_limits_i8xx_dac;
7563         }
7564
7565         if (!crtc_state->clock_set &&
7566             !i9xx_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
7567                                  refclk, NULL, &crtc_state->dpll)) {
7568                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
7569                 return -EINVAL;
7570         }
7571
7572         i8xx_compute_dpll(crtc, crtc_state, NULL);
7573
7574         return 0;
7575 }
7576
7577 static int g4x_crtc_compute_clock(struct intel_crtc *crtc,
7578                                   struct intel_crtc_state *crtc_state)
7579 {
7580         struct drm_device *dev = crtc->base.dev;
7581         struct drm_i915_private *dev_priv = to_i915(dev);
7582         const struct intel_limit *limit;
7583         int refclk = 96000;
7584
7585         memset(&crtc_state->dpll_hw_state, 0,
7586                sizeof(crtc_state->dpll_hw_state));
7587
7588         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
7589                 if (intel_panel_use_ssc(dev_priv)) {
7590                         refclk = dev_priv->vbt.lvds_ssc_freq;
7591                         DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n", refclk);
7592                 }
7593
7594                 if (intel_is_dual_link_lvds(dev))
7595                         limit = &intel_limits_g4x_dual_channel_lvds;
7596                 else
7597                         limit = &intel_limits_g4x_single_channel_lvds;
7598         } else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI) ||
7599                    intel_crtc_has_type(crtc_state, INTEL_OUTPUT_ANALOG)) {
7600                 limit = &intel_limits_g4x_hdmi;
7601         } else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO)) {
7602                 limit = &intel_limits_g4x_sdvo;
7603         } else {
7604                 /* The option is for other outputs */
7605                 limit = &intel_limits_i9xx_sdvo;
7606         }
7607
7608         if (!crtc_state->clock_set &&
7609             !g4x_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
7610                                 refclk, NULL, &crtc_state->dpll)) {
7611                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
7612                 return -EINVAL;
7613         }
7614
7615         i9xx_compute_dpll(crtc, crtc_state, NULL);
7616
7617         return 0;
7618 }
7619
7620 static int pnv_crtc_compute_clock(struct intel_crtc *crtc,
7621                                   struct intel_crtc_state *crtc_state)
7622 {
7623         struct drm_device *dev = crtc->base.dev;
7624         struct drm_i915_private *dev_priv = to_i915(dev);
7625         const struct intel_limit *limit;
7626         int refclk = 96000;
7627
7628         memset(&crtc_state->dpll_hw_state, 0,
7629                sizeof(crtc_state->dpll_hw_state));
7630
7631         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
7632                 if (intel_panel_use_ssc(dev_priv)) {
7633                         refclk = dev_priv->vbt.lvds_ssc_freq;
7634                         DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n", refclk);
7635                 }
7636
7637                 limit = &intel_limits_pineview_lvds;
7638         } else {
7639                 limit = &intel_limits_pineview_sdvo;
7640         }
7641
7642         if (!crtc_state->clock_set &&
7643             !pnv_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
7644                                 refclk, NULL, &crtc_state->dpll)) {
7645                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
7646                 return -EINVAL;
7647         }
7648
7649         i9xx_compute_dpll(crtc, crtc_state, NULL);
7650
7651         return 0;
7652 }
7653
7654 static int i9xx_crtc_compute_clock(struct intel_crtc *crtc,
7655                                    struct intel_crtc_state *crtc_state)
7656 {
7657         struct drm_device *dev = crtc->base.dev;
7658         struct drm_i915_private *dev_priv = to_i915(dev);
7659         const struct intel_limit *limit;
7660         int refclk = 96000;
7661
7662         memset(&crtc_state->dpll_hw_state, 0,
7663                sizeof(crtc_state->dpll_hw_state));
7664
7665         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
7666                 if (intel_panel_use_ssc(dev_priv)) {
7667                         refclk = dev_priv->vbt.lvds_ssc_freq;
7668                         DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n", refclk);
7669                 }
7670
7671                 limit = &intel_limits_i9xx_lvds;
7672         } else {
7673                 limit = &intel_limits_i9xx_sdvo;
7674         }
7675
7676         if (!crtc_state->clock_set &&
7677             !i9xx_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
7678                                  refclk, NULL, &crtc_state->dpll)) {
7679                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
7680                 return -EINVAL;
7681         }
7682
7683         i9xx_compute_dpll(crtc, crtc_state, NULL);
7684
7685         return 0;
7686 }
7687
7688 static int chv_crtc_compute_clock(struct intel_crtc *crtc,
7689                                   struct intel_crtc_state *crtc_state)
7690 {
7691         int refclk = 100000;
7692         const struct intel_limit *limit = &intel_limits_chv;
7693
7694         memset(&crtc_state->dpll_hw_state, 0,
7695                sizeof(crtc_state->dpll_hw_state));
7696
7697         if (!crtc_state->clock_set &&
7698             !chv_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
7699                                 refclk, NULL, &crtc_state->dpll)) {
7700                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
7701                 return -EINVAL;
7702         }
7703
7704         chv_compute_dpll(crtc, crtc_state);
7705
7706         return 0;
7707 }
7708
7709 static int vlv_crtc_compute_clock(struct intel_crtc *crtc,
7710                                   struct intel_crtc_state *crtc_state)
7711 {
7712         int refclk = 100000;
7713         const struct intel_limit *limit = &intel_limits_vlv;
7714
7715         memset(&crtc_state->dpll_hw_state, 0,
7716                sizeof(crtc_state->dpll_hw_state));
7717
7718         if (!crtc_state->clock_set &&
7719             !vlv_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
7720                                 refclk, NULL, &crtc_state->dpll)) {
7721                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
7722                 return -EINVAL;
7723         }
7724
7725         vlv_compute_dpll(crtc, crtc_state);
7726
7727         return 0;
7728 }
7729
7730 static void i9xx_get_pfit_config(struct intel_crtc *crtc,
7731                                  struct intel_crtc_state *pipe_config)
7732 {
7733         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7734         uint32_t tmp;
7735
7736         if (INTEL_GEN(dev_priv) <= 3 &&
7737             (IS_I830(dev_priv) || !IS_MOBILE(dev_priv)))
7738                 return;
7739
7740         tmp = I915_READ(PFIT_CONTROL);
7741         if (!(tmp & PFIT_ENABLE))
7742                 return;
7743
7744         /* Check whether the pfit is attached to our pipe. */
7745         if (INTEL_GEN(dev_priv) < 4) {
7746                 if (crtc->pipe != PIPE_B)
7747                         return;
7748         } else {
7749                 if ((tmp & PFIT_PIPE_MASK) != (crtc->pipe << PFIT_PIPE_SHIFT))
7750                         return;
7751         }
7752
7753         pipe_config->gmch_pfit.control = tmp;
7754         pipe_config->gmch_pfit.pgm_ratios = I915_READ(PFIT_PGM_RATIOS);
7755 }
7756
7757 static void vlv_crtc_clock_get(struct intel_crtc *crtc,
7758                                struct intel_crtc_state *pipe_config)
7759 {
7760         struct drm_device *dev = crtc->base.dev;
7761         struct drm_i915_private *dev_priv = to_i915(dev);
7762         int pipe = pipe_config->cpu_transcoder;
7763         struct dpll clock;
7764         u32 mdiv;
7765         int refclk = 100000;
7766
7767         /* In case of DSI, DPLL will not be used */
7768         if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0)
7769                 return;
7770
7771         mutex_lock(&dev_priv->sb_lock);
7772         mdiv = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW3(pipe));
7773         mutex_unlock(&dev_priv->sb_lock);
7774
7775         clock.m1 = (mdiv >> DPIO_M1DIV_SHIFT) & 7;
7776         clock.m2 = mdiv & DPIO_M2DIV_MASK;
7777         clock.n = (mdiv >> DPIO_N_SHIFT) & 0xf;
7778         clock.p1 = (mdiv >> DPIO_P1_SHIFT) & 7;
7779         clock.p2 = (mdiv >> DPIO_P2_SHIFT) & 0x1f;
7780
7781         pipe_config->port_clock = vlv_calc_dpll_params(refclk, &clock);
7782 }
7783
7784 static void
7785 i9xx_get_initial_plane_config(struct intel_crtc *crtc,
7786                               struct intel_initial_plane_config *plane_config)
7787 {
7788         struct drm_device *dev = crtc->base.dev;
7789         struct drm_i915_private *dev_priv = to_i915(dev);
7790         struct intel_plane *plane = to_intel_plane(crtc->base.primary);
7791         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
7792         enum pipe pipe;
7793         u32 val, base, offset;
7794         int fourcc, pixel_format;
7795         unsigned int aligned_height;
7796         struct drm_framebuffer *fb;
7797         struct intel_framebuffer *intel_fb;
7798
7799         if (!plane->get_hw_state(plane, &pipe))
7800                 return;
7801
7802         WARN_ON(pipe != crtc->pipe);
7803
7804         intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL);
7805         if (!intel_fb) {
7806                 DRM_DEBUG_KMS("failed to alloc fb\n");
7807                 return;
7808         }
7809
7810         fb = &intel_fb->base;
7811
7812         fb->dev = dev;
7813
7814         val = I915_READ(DSPCNTR(i9xx_plane));
7815
7816         if (INTEL_GEN(dev_priv) >= 4) {
7817                 if (val & DISPPLANE_TILED) {
7818                         plane_config->tiling = I915_TILING_X;
7819                         fb->modifier = I915_FORMAT_MOD_X_TILED;
7820                 }
7821
7822                 if (val & DISPPLANE_ROTATE_180)
7823                         plane_config->rotation = DRM_MODE_ROTATE_180;
7824         }
7825
7826         if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_B &&
7827             val & DISPPLANE_MIRROR)
7828                 plane_config->rotation |= DRM_MODE_REFLECT_X;
7829
7830         pixel_format = val & DISPPLANE_PIXFORMAT_MASK;
7831         fourcc = i9xx_format_to_fourcc(pixel_format);
7832         fb->format = drm_format_info(fourcc);
7833
7834         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
7835                 offset = I915_READ(DSPOFFSET(i9xx_plane));
7836                 base = I915_READ(DSPSURF(i9xx_plane)) & 0xfffff000;
7837         } else if (INTEL_GEN(dev_priv) >= 4) {
7838                 if (plane_config->tiling)
7839                         offset = I915_READ(DSPTILEOFF(i9xx_plane));
7840                 else
7841                         offset = I915_READ(DSPLINOFF(i9xx_plane));
7842                 base = I915_READ(DSPSURF(i9xx_plane)) & 0xfffff000;
7843         } else {
7844                 base = I915_READ(DSPADDR(i9xx_plane));
7845         }
7846         plane_config->base = base;
7847
7848         val = I915_READ(PIPESRC(pipe));
7849         fb->width = ((val >> 16) & 0xfff) + 1;
7850         fb->height = ((val >> 0) & 0xfff) + 1;
7851
7852         val = I915_READ(DSPSTRIDE(i9xx_plane));
7853         fb->pitches[0] = val & 0xffffffc0;
7854
7855         aligned_height = intel_fb_align_height(fb, 0, fb->height);
7856
7857         plane_config->size = fb->pitches[0] * aligned_height;
7858
7859         DRM_DEBUG_KMS("%s/%s with fb: size=%dx%d@%d, offset=%x, pitch %d, size 0x%x\n",
7860                       crtc->base.name, plane->base.name, fb->width, fb->height,
7861                       fb->format->cpp[0] * 8, base, fb->pitches[0],
7862                       plane_config->size);
7863
7864         plane_config->fb = intel_fb;
7865 }
7866
7867 static void chv_crtc_clock_get(struct intel_crtc *crtc,
7868                                struct intel_crtc_state *pipe_config)
7869 {
7870         struct drm_device *dev = crtc->base.dev;
7871         struct drm_i915_private *dev_priv = to_i915(dev);
7872         int pipe = pipe_config->cpu_transcoder;
7873         enum dpio_channel port = vlv_pipe_to_channel(pipe);
7874         struct dpll clock;
7875         u32 cmn_dw13, pll_dw0, pll_dw1, pll_dw2, pll_dw3;
7876         int refclk = 100000;
7877
7878         /* In case of DSI, DPLL will not be used */
7879         if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0)
7880                 return;
7881
7882         mutex_lock(&dev_priv->sb_lock);
7883         cmn_dw13 = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW13(port));
7884         pll_dw0 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW0(port));
7885         pll_dw1 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW1(port));
7886         pll_dw2 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW2(port));
7887         pll_dw3 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW3(port));
7888         mutex_unlock(&dev_priv->sb_lock);
7889
7890         clock.m1 = (pll_dw1 & 0x7) == DPIO_CHV_M1_DIV_BY_2 ? 2 : 0;
7891         clock.m2 = (pll_dw0 & 0xff) << 22;
7892         if (pll_dw3 & DPIO_CHV_FRAC_DIV_EN)
7893                 clock.m2 |= pll_dw2 & 0x3fffff;
7894         clock.n = (pll_dw1 >> DPIO_CHV_N_DIV_SHIFT) & 0xf;
7895         clock.p1 = (cmn_dw13 >> DPIO_CHV_P1_DIV_SHIFT) & 0x7;
7896         clock.p2 = (cmn_dw13 >> DPIO_CHV_P2_DIV_SHIFT) & 0x1f;
7897
7898         pipe_config->port_clock = chv_calc_dpll_params(refclk, &clock);
7899 }
7900
7901 static void intel_get_crtc_ycbcr_config(struct intel_crtc *crtc,
7902                                         struct intel_crtc_state *pipe_config)
7903 {
7904         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7905         enum intel_output_format output = INTEL_OUTPUT_FORMAT_RGB;
7906
7907         pipe_config->lspcon_downsampling = false;
7908
7909         if (IS_BROADWELL(dev_priv) || INTEL_GEN(dev_priv) >= 9) {
7910                 u32 tmp = I915_READ(PIPEMISC(crtc->pipe));
7911
7912                 if (tmp & PIPEMISC_OUTPUT_COLORSPACE_YUV) {
7913                         bool ycbcr420_enabled = tmp & PIPEMISC_YUV420_ENABLE;
7914                         bool blend = tmp & PIPEMISC_YUV420_MODE_FULL_BLEND;
7915
7916                         if (ycbcr420_enabled) {
7917                                 /* We support 4:2:0 in full blend mode only */
7918                                 if (!blend)
7919                                         output = INTEL_OUTPUT_FORMAT_INVALID;
7920                                 else if (!(IS_GEMINILAKE(dev_priv) ||
7921                                            INTEL_GEN(dev_priv) >= 10))
7922                                         output = INTEL_OUTPUT_FORMAT_INVALID;
7923                                 else
7924                                         output = INTEL_OUTPUT_FORMAT_YCBCR420;
7925                         } else {
7926                                 /*
7927                                  * Currently there is no interface defined to
7928                                  * check user preference between RGB/YCBCR444
7929                                  * or YCBCR420. So the only possible case for
7930                                  * YCBCR444 usage is driving YCBCR420 output
7931                                  * with LSPCON, when pipe is configured for
7932                                  * YCBCR444 output and LSPCON takes care of
7933                                  * downsampling it.
7934                                  */
7935                                 pipe_config->lspcon_downsampling = true;
7936                                 output = INTEL_OUTPUT_FORMAT_YCBCR444;
7937                         }
7938                 }
7939         }
7940
7941         pipe_config->output_format = output;
7942 }
7943
7944 static bool i9xx_get_pipe_config(struct intel_crtc *crtc,
7945                                  struct intel_crtc_state *pipe_config)
7946 {
7947         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7948         enum intel_display_power_domain power_domain;
7949         uint32_t tmp;
7950         bool ret;
7951
7952         power_domain = POWER_DOMAIN_PIPE(crtc->pipe);
7953         if (!intel_display_power_get_if_enabled(dev_priv, power_domain))
7954                 return false;
7955
7956         pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB;
7957         pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe;
7958         pipe_config->shared_dpll = NULL;
7959
7960         ret = false;
7961
7962         tmp = I915_READ(PIPECONF(crtc->pipe));
7963         if (!(tmp & PIPECONF_ENABLE))
7964                 goto out;
7965
7966         if (IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
7967             IS_CHERRYVIEW(dev_priv)) {
7968                 switch (tmp & PIPECONF_BPC_MASK) {
7969                 case PIPECONF_6BPC:
7970                         pipe_config->pipe_bpp = 18;
7971                         break;
7972                 case PIPECONF_8BPC:
7973                         pipe_config->pipe_bpp = 24;
7974                         break;
7975                 case PIPECONF_10BPC:
7976                         pipe_config->pipe_bpp = 30;
7977                         break;
7978                 default:
7979                         break;
7980                 }
7981         }
7982
7983         if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
7984             (tmp & PIPECONF_COLOR_RANGE_SELECT))
7985                 pipe_config->limited_color_range = true;
7986
7987         if (INTEL_GEN(dev_priv) < 4)
7988                 pipe_config->double_wide = tmp & PIPECONF_DOUBLE_WIDE;
7989
7990         intel_get_pipe_timings(crtc, pipe_config);
7991         intel_get_pipe_src_size(crtc, pipe_config);
7992
7993         i9xx_get_pfit_config(crtc, pipe_config);
7994
7995         if (INTEL_GEN(dev_priv) >= 4) {
7996                 /* No way to read it out on pipes B and C */
7997                 if (IS_CHERRYVIEW(dev_priv) && crtc->pipe != PIPE_A)
7998                         tmp = dev_priv->chv_dpll_md[crtc->pipe];
7999                 else
8000                         tmp = I915_READ(DPLL_MD(crtc->pipe));
8001                 pipe_config->pixel_multiplier =
8002                         ((tmp & DPLL_MD_UDI_MULTIPLIER_MASK)
8003                          >> DPLL_MD_UDI_MULTIPLIER_SHIFT) + 1;
8004                 pipe_config->dpll_hw_state.dpll_md = tmp;
8005         } else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv) ||
8006                    IS_G33(dev_priv) || IS_PINEVIEW(dev_priv)) {
8007                 tmp = I915_READ(DPLL(crtc->pipe));
8008                 pipe_config->pixel_multiplier =
8009                         ((tmp & SDVO_MULTIPLIER_MASK)
8010                          >> SDVO_MULTIPLIER_SHIFT_HIRES) + 1;
8011         } else {
8012                 /* Note that on i915G/GM the pixel multiplier is in the sdvo
8013                  * port and will be fixed up in the encoder->get_config
8014                  * function. */
8015                 pipe_config->pixel_multiplier = 1;
8016         }
8017         pipe_config->dpll_hw_state.dpll = I915_READ(DPLL(crtc->pipe));
8018         if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv)) {
8019                 /*
8020                  * DPLL_DVO_2X_MODE must be enabled for both DPLLs
8021                  * on 830. Filter it out here so that we don't
8022                  * report errors due to that.
8023                  */
8024                 if (IS_I830(dev_priv))
8025                         pipe_config->dpll_hw_state.dpll &= ~DPLL_DVO_2X_MODE;
8026
8027                 pipe_config->dpll_hw_state.fp0 = I915_READ(FP0(crtc->pipe));
8028                 pipe_config->dpll_hw_state.fp1 = I915_READ(FP1(crtc->pipe));
8029         } else {
8030                 /* Mask out read-only status bits. */
8031                 pipe_config->dpll_hw_state.dpll &= ~(DPLL_LOCK_VLV |
8032                                                      DPLL_PORTC_READY_MASK |
8033                                                      DPLL_PORTB_READY_MASK);
8034         }
8035
8036         if (IS_CHERRYVIEW(dev_priv))
8037                 chv_crtc_clock_get(crtc, pipe_config);
8038         else if (IS_VALLEYVIEW(dev_priv))
8039                 vlv_crtc_clock_get(crtc, pipe_config);
8040         else
8041                 i9xx_crtc_clock_get(crtc, pipe_config);
8042
8043         /*
8044          * Normally the dotclock is filled in by the encoder .get_config()
8045          * but in case the pipe is enabled w/o any ports we need a sane
8046          * default.
8047          */
8048         pipe_config->base.adjusted_mode.crtc_clock =
8049                 pipe_config->port_clock / pipe_config->pixel_multiplier;
8050
8051         ret = true;
8052
8053 out:
8054         intel_display_power_put(dev_priv, power_domain);
8055
8056         return ret;
8057 }
8058
8059 static void ironlake_init_pch_refclk(struct drm_i915_private *dev_priv)
8060 {
8061         struct intel_encoder *encoder;
8062         int i;
8063         u32 val, final;
8064         bool has_lvds = false;
8065         bool has_cpu_edp = false;
8066         bool has_panel = false;
8067         bool has_ck505 = false;
8068         bool can_ssc = false;
8069         bool using_ssc_source = false;
8070
8071         /* We need to take the global config into account */
8072         for_each_intel_encoder(&dev_priv->drm, encoder) {
8073                 switch (encoder->type) {
8074                 case INTEL_OUTPUT_LVDS:
8075                         has_panel = true;
8076                         has_lvds = true;
8077                         break;
8078                 case INTEL_OUTPUT_EDP:
8079                         has_panel = true;
8080                         if (encoder->port == PORT_A)
8081                                 has_cpu_edp = true;
8082                         break;
8083                 default:
8084                         break;
8085                 }
8086         }
8087
8088         if (HAS_PCH_IBX(dev_priv)) {
8089                 has_ck505 = dev_priv->vbt.display_clock_mode;
8090                 can_ssc = has_ck505;
8091         } else {
8092                 has_ck505 = false;
8093                 can_ssc = true;
8094         }
8095
8096         /* Check if any DPLLs are using the SSC source */
8097         for (i = 0; i < dev_priv->num_shared_dpll; i++) {
8098                 u32 temp = I915_READ(PCH_DPLL(i));
8099
8100                 if (!(temp & DPLL_VCO_ENABLE))
8101                         continue;
8102
8103                 if ((temp & PLL_REF_INPUT_MASK) ==
8104                     PLLB_REF_INPUT_SPREADSPECTRUMIN) {
8105                         using_ssc_source = true;
8106                         break;
8107                 }
8108         }
8109
8110         DRM_DEBUG_KMS("has_panel %d has_lvds %d has_ck505 %d using_ssc_source %d\n",
8111                       has_panel, has_lvds, has_ck505, using_ssc_source);
8112
8113         /* Ironlake: try to setup display ref clock before DPLL
8114          * enabling. This is only under driver's control after
8115          * PCH B stepping, previous chipset stepping should be
8116          * ignoring this setting.
8117          */
8118         val = I915_READ(PCH_DREF_CONTROL);
8119
8120         /* As we must carefully and slowly disable/enable each source in turn,
8121          * compute the final state we want first and check if we need to
8122          * make any changes at all.
8123          */
8124         final = val;
8125         final &= ~DREF_NONSPREAD_SOURCE_MASK;
8126         if (has_ck505)
8127                 final |= DREF_NONSPREAD_CK505_ENABLE;
8128         else
8129                 final |= DREF_NONSPREAD_SOURCE_ENABLE;
8130
8131         final &= ~DREF_SSC_SOURCE_MASK;
8132         final &= ~DREF_CPU_SOURCE_OUTPUT_MASK;
8133         final &= ~DREF_SSC1_ENABLE;
8134
8135         if (has_panel) {
8136                 final |= DREF_SSC_SOURCE_ENABLE;
8137
8138                 if (intel_panel_use_ssc(dev_priv) && can_ssc)
8139                         final |= DREF_SSC1_ENABLE;
8140
8141                 if (has_cpu_edp) {
8142                         if (intel_panel_use_ssc(dev_priv) && can_ssc)
8143                                 final |= DREF_CPU_SOURCE_OUTPUT_DOWNSPREAD;
8144                         else
8145                                 final |= DREF_CPU_SOURCE_OUTPUT_NONSPREAD;
8146                 } else
8147                         final |= DREF_CPU_SOURCE_OUTPUT_DISABLE;
8148         } else if (using_ssc_source) {
8149                 final |= DREF_SSC_SOURCE_ENABLE;
8150                 final |= DREF_SSC1_ENABLE;
8151         }
8152
8153         if (final == val)
8154                 return;
8155
8156         /* Always enable nonspread source */
8157         val &= ~DREF_NONSPREAD_SOURCE_MASK;
8158
8159         if (has_ck505)
8160                 val |= DREF_NONSPREAD_CK505_ENABLE;
8161         else
8162                 val |= DREF_NONSPREAD_SOURCE_ENABLE;
8163
8164         if (has_panel) {
8165                 val &= ~DREF_SSC_SOURCE_MASK;
8166                 val |= DREF_SSC_SOURCE_ENABLE;
8167
8168                 /* SSC must be turned on before enabling the CPU output  */
8169                 if (intel_panel_use_ssc(dev_priv) && can_ssc) {
8170                         DRM_DEBUG_KMS("Using SSC on panel\n");
8171                         val |= DREF_SSC1_ENABLE;
8172                 } else
8173                         val &= ~DREF_SSC1_ENABLE;
8174
8175                 /* Get SSC going before enabling the outputs */
8176                 I915_WRITE(PCH_DREF_CONTROL, val);
8177                 POSTING_READ(PCH_DREF_CONTROL);
8178                 udelay(200);
8179
8180                 val &= ~DREF_CPU_SOURCE_OUTPUT_MASK;
8181
8182                 /* Enable CPU source on CPU attached eDP */
8183                 if (has_cpu_edp) {
8184                         if (intel_panel_use_ssc(dev_priv) && can_ssc) {
8185                                 DRM_DEBUG_KMS("Using SSC on eDP\n");
8186                                 val |= DREF_CPU_SOURCE_OUTPUT_DOWNSPREAD;
8187                         } else
8188                                 val |= DREF_CPU_SOURCE_OUTPUT_NONSPREAD;
8189                 } else
8190                         val |= DREF_CPU_SOURCE_OUTPUT_DISABLE;
8191
8192                 I915_WRITE(PCH_DREF_CONTROL, val);
8193                 POSTING_READ(PCH_DREF_CONTROL);
8194                 udelay(200);
8195         } else {
8196                 DRM_DEBUG_KMS("Disabling CPU source output\n");
8197
8198                 val &= ~DREF_CPU_SOURCE_OUTPUT_MASK;
8199
8200                 /* Turn off CPU output */
8201                 val |= DREF_CPU_SOURCE_OUTPUT_DISABLE;
8202
8203                 I915_WRITE(PCH_DREF_CONTROL, val);
8204                 POSTING_READ(PCH_DREF_CONTROL);
8205                 udelay(200);
8206
8207                 if (!using_ssc_source) {
8208                         DRM_DEBUG_KMS("Disabling SSC source\n");
8209
8210                         /* Turn off the SSC source */
8211                         val &= ~DREF_SSC_SOURCE_MASK;
8212                         val |= DREF_SSC_SOURCE_DISABLE;
8213
8214                         /* Turn off SSC1 */
8215                         val &= ~DREF_SSC1_ENABLE;
8216
8217                         I915_WRITE(PCH_DREF_CONTROL, val);
8218                         POSTING_READ(PCH_DREF_CONTROL);
8219                         udelay(200);
8220                 }
8221         }
8222
8223         BUG_ON(val != final);
8224 }
8225
8226 static void lpt_reset_fdi_mphy(struct drm_i915_private *dev_priv)
8227 {
8228         uint32_t tmp;
8229
8230         tmp = I915_READ(SOUTH_CHICKEN2);
8231         tmp |= FDI_MPHY_IOSFSB_RESET_CTL;
8232         I915_WRITE(SOUTH_CHICKEN2, tmp);
8233
8234         if (wait_for_us(I915_READ(SOUTH_CHICKEN2) &
8235                         FDI_MPHY_IOSFSB_RESET_STATUS, 100))
8236                 DRM_ERROR("FDI mPHY reset assert timeout\n");
8237
8238         tmp = I915_READ(SOUTH_CHICKEN2);
8239         tmp &= ~FDI_MPHY_IOSFSB_RESET_CTL;
8240         I915_WRITE(SOUTH_CHICKEN2, tmp);
8241
8242         if (wait_for_us((I915_READ(SOUTH_CHICKEN2) &
8243                          FDI_MPHY_IOSFSB_RESET_STATUS) == 0, 100))
8244                 DRM_ERROR("FDI mPHY reset de-assert timeout\n");
8245 }
8246
8247 /* WaMPhyProgramming:hsw */
8248 static void lpt_program_fdi_mphy(struct drm_i915_private *dev_priv)
8249 {
8250         uint32_t tmp;
8251
8252         tmp = intel_sbi_read(dev_priv, 0x8008, SBI_MPHY);
8253         tmp &= ~(0xFF << 24);
8254         tmp |= (0x12 << 24);
8255         intel_sbi_write(dev_priv, 0x8008, tmp, SBI_MPHY);
8256
8257         tmp = intel_sbi_read(dev_priv, 0x2008, SBI_MPHY);
8258         tmp |= (1 << 11);
8259         intel_sbi_write(dev_priv, 0x2008, tmp, SBI_MPHY);
8260
8261         tmp = intel_sbi_read(dev_priv, 0x2108, SBI_MPHY);
8262         tmp |= (1 << 11);
8263         intel_sbi_write(dev_priv, 0x2108, tmp, SBI_MPHY);
8264
8265         tmp = intel_sbi_read(dev_priv, 0x206C, SBI_MPHY);
8266         tmp |= (1 << 24) | (1 << 21) | (1 << 18);
8267         intel_sbi_write(dev_priv, 0x206C, tmp, SBI_MPHY);
8268
8269         tmp = intel_sbi_read(dev_priv, 0x216C, SBI_MPHY);
8270         tmp |= (1 << 24) | (1 << 21) | (1 << 18);
8271         intel_sbi_write(dev_priv, 0x216C, tmp, SBI_MPHY);
8272
8273         tmp = intel_sbi_read(dev_priv, 0x2080, SBI_MPHY);
8274         tmp &= ~(7 << 13);
8275         tmp |= (5 << 13);
8276         intel_sbi_write(dev_priv, 0x2080, tmp, SBI_MPHY);
8277
8278         tmp = intel_sbi_read(dev_priv, 0x2180, SBI_MPHY);
8279         tmp &= ~(7 << 13);
8280         tmp |= (5 << 13);
8281         intel_sbi_write(dev_priv, 0x2180, tmp, SBI_MPHY);
8282
8283         tmp = intel_sbi_read(dev_priv, 0x208C, SBI_MPHY);
8284         tmp &= ~0xFF;
8285         tmp |= 0x1C;
8286         intel_sbi_write(dev_priv, 0x208C, tmp, SBI_MPHY);
8287
8288         tmp = intel_sbi_read(dev_priv, 0x218C, SBI_MPHY);
8289         tmp &= ~0xFF;
8290         tmp |= 0x1C;
8291         intel_sbi_write(dev_priv, 0x218C, tmp, SBI_MPHY);
8292
8293         tmp = intel_sbi_read(dev_priv, 0x2098, SBI_MPHY);
8294         tmp &= ~(0xFF << 16);
8295         tmp |= (0x1C << 16);
8296         intel_sbi_write(dev_priv, 0x2098, tmp, SBI_MPHY);
8297
8298         tmp = intel_sbi_read(dev_priv, 0x2198, SBI_MPHY);
8299         tmp &= ~(0xFF << 16);
8300         tmp |= (0x1C << 16);
8301         intel_sbi_write(dev_priv, 0x2198, tmp, SBI_MPHY);
8302
8303         tmp = intel_sbi_read(dev_priv, 0x20C4, SBI_MPHY);
8304         tmp |= (1 << 27);
8305         intel_sbi_write(dev_priv, 0x20C4, tmp, SBI_MPHY);
8306
8307         tmp = intel_sbi_read(dev_priv, 0x21C4, SBI_MPHY);
8308         tmp |= (1 << 27);
8309         intel_sbi_write(dev_priv, 0x21C4, tmp, SBI_MPHY);
8310
8311         tmp = intel_sbi_read(dev_priv, 0x20EC, SBI_MPHY);
8312         tmp &= ~(0xF << 28);
8313         tmp |= (4 << 28);
8314         intel_sbi_write(dev_priv, 0x20EC, tmp, SBI_MPHY);
8315
8316         tmp = intel_sbi_read(dev_priv, 0x21EC, SBI_MPHY);
8317         tmp &= ~(0xF << 28);
8318         tmp |= (4 << 28);
8319         intel_sbi_write(dev_priv, 0x21EC, tmp, SBI_MPHY);
8320 }
8321
8322 /* Implements 3 different sequences from BSpec chapter "Display iCLK
8323  * Programming" based on the parameters passed:
8324  * - Sequence to enable CLKOUT_DP
8325  * - Sequence to enable CLKOUT_DP without spread
8326  * - Sequence to enable CLKOUT_DP for FDI usage and configure PCH FDI I/O
8327  */
8328 static void lpt_enable_clkout_dp(struct drm_i915_private *dev_priv,
8329                                  bool with_spread, bool with_fdi)
8330 {
8331         uint32_t reg, tmp;
8332
8333         if (WARN(with_fdi && !with_spread, "FDI requires downspread\n"))
8334                 with_spread = true;
8335         if (WARN(HAS_PCH_LPT_LP(dev_priv) &&
8336             with_fdi, "LP PCH doesn't have FDI\n"))
8337                 with_fdi = false;
8338
8339         mutex_lock(&dev_priv->sb_lock);
8340
8341         tmp = intel_sbi_read(dev_priv, SBI_SSCCTL, SBI_ICLK);
8342         tmp &= ~SBI_SSCCTL_DISABLE;
8343         tmp |= SBI_SSCCTL_PATHALT;
8344         intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
8345
8346         udelay(24);
8347
8348         if (with_spread) {
8349                 tmp = intel_sbi_read(dev_priv, SBI_SSCCTL, SBI_ICLK);
8350                 tmp &= ~SBI_SSCCTL_PATHALT;
8351                 intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
8352
8353                 if (with_fdi) {
8354                         lpt_reset_fdi_mphy(dev_priv);
8355                         lpt_program_fdi_mphy(dev_priv);
8356                 }
8357         }
8358
8359         reg = HAS_PCH_LPT_LP(dev_priv) ? SBI_GEN0 : SBI_DBUFF0;
8360         tmp = intel_sbi_read(dev_priv, reg, SBI_ICLK);
8361         tmp |= SBI_GEN0_CFG_BUFFENABLE_DISABLE;
8362         intel_sbi_write(dev_priv, reg, tmp, SBI_ICLK);
8363
8364         mutex_unlock(&dev_priv->sb_lock);
8365 }
8366
8367 /* Sequence to disable CLKOUT_DP */
8368 static void lpt_disable_clkout_dp(struct drm_i915_private *dev_priv)
8369 {
8370         uint32_t reg, tmp;
8371
8372         mutex_lock(&dev_priv->sb_lock);
8373
8374         reg = HAS_PCH_LPT_LP(dev_priv) ? SBI_GEN0 : SBI_DBUFF0;
8375         tmp = intel_sbi_read(dev_priv, reg, SBI_ICLK);
8376         tmp &= ~SBI_GEN0_CFG_BUFFENABLE_DISABLE;
8377         intel_sbi_write(dev_priv, reg, tmp, SBI_ICLK);
8378
8379         tmp = intel_sbi_read(dev_priv, SBI_SSCCTL, SBI_ICLK);
8380         if (!(tmp & SBI_SSCCTL_DISABLE)) {
8381                 if (!(tmp & SBI_SSCCTL_PATHALT)) {
8382                         tmp |= SBI_SSCCTL_PATHALT;
8383                         intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
8384                         udelay(32);
8385                 }
8386                 tmp |= SBI_SSCCTL_DISABLE;
8387                 intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
8388         }
8389
8390         mutex_unlock(&dev_priv->sb_lock);
8391 }
8392
8393 #define BEND_IDX(steps) ((50 + (steps)) / 5)
8394
8395 static const uint16_t sscdivintphase[] = {
8396         [BEND_IDX( 50)] = 0x3B23,
8397         [BEND_IDX( 45)] = 0x3B23,
8398         [BEND_IDX( 40)] = 0x3C23,
8399         [BEND_IDX( 35)] = 0x3C23,
8400         [BEND_IDX( 30)] = 0x3D23,
8401         [BEND_IDX( 25)] = 0x3D23,
8402         [BEND_IDX( 20)] = 0x3E23,
8403         [BEND_IDX( 15)] = 0x3E23,
8404         [BEND_IDX( 10)] = 0x3F23,
8405         [BEND_IDX(  5)] = 0x3F23,
8406         [BEND_IDX(  0)] = 0x0025,
8407         [BEND_IDX( -5)] = 0x0025,
8408         [BEND_IDX(-10)] = 0x0125,
8409         [BEND_IDX(-15)] = 0x0125,
8410         [BEND_IDX(-20)] = 0x0225,
8411         [BEND_IDX(-25)] = 0x0225,
8412         [BEND_IDX(-30)] = 0x0325,
8413         [BEND_IDX(-35)] = 0x0325,
8414         [BEND_IDX(-40)] = 0x0425,
8415         [BEND_IDX(-45)] = 0x0425,
8416         [BEND_IDX(-50)] = 0x0525,
8417 };
8418
8419 /*
8420  * Bend CLKOUT_DP
8421  * steps -50 to 50 inclusive, in steps of 5
8422  * < 0 slow down the clock, > 0 speed up the clock, 0 == no bend (135MHz)
8423  * change in clock period = -(steps / 10) * 5.787 ps
8424  */
8425 static void lpt_bend_clkout_dp(struct drm_i915_private *dev_priv, int steps)
8426 {
8427         uint32_t tmp;
8428         int idx = BEND_IDX(steps);
8429
8430         if (WARN_ON(steps % 5 != 0))
8431                 return;
8432
8433         if (WARN_ON(idx >= ARRAY_SIZE(sscdivintphase)))
8434                 return;
8435
8436         mutex_lock(&dev_priv->sb_lock);
8437
8438         if (steps % 10 != 0)
8439                 tmp = 0xAAAAAAAB;
8440         else
8441                 tmp = 0x00000000;
8442         intel_sbi_write(dev_priv, SBI_SSCDITHPHASE, tmp, SBI_ICLK);
8443
8444         tmp = intel_sbi_read(dev_priv, SBI_SSCDIVINTPHASE, SBI_ICLK);
8445         tmp &= 0xffff0000;
8446         tmp |= sscdivintphase[idx];
8447         intel_sbi_write(dev_priv, SBI_SSCDIVINTPHASE, tmp, SBI_ICLK);
8448
8449         mutex_unlock(&dev_priv->sb_lock);
8450 }
8451
8452 #undef BEND_IDX
8453
8454 static void lpt_init_pch_refclk(struct drm_i915_private *dev_priv)
8455 {
8456         struct intel_encoder *encoder;
8457         bool has_vga = false;
8458
8459         for_each_intel_encoder(&dev_priv->drm, encoder) {
8460                 switch (encoder->type) {
8461                 case INTEL_OUTPUT_ANALOG:
8462                         has_vga = true;
8463                         break;
8464                 default:
8465                         break;
8466                 }
8467         }
8468
8469         if (has_vga) {
8470                 lpt_bend_clkout_dp(dev_priv, 0);
8471                 lpt_enable_clkout_dp(dev_priv, true, true);
8472         } else {
8473                 lpt_disable_clkout_dp(dev_priv);
8474         }
8475 }
8476
8477 /*
8478  * Initialize reference clocks when the driver loads
8479  */
8480 void intel_init_pch_refclk(struct drm_i915_private *dev_priv)
8481 {
8482         if (HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv))
8483                 ironlake_init_pch_refclk(dev_priv);
8484         else if (HAS_PCH_LPT(dev_priv))
8485                 lpt_init_pch_refclk(dev_priv);
8486 }
8487
8488 static void ironlake_set_pipeconf(const struct intel_crtc_state *crtc_state)
8489 {
8490         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
8491         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8492         enum pipe pipe = crtc->pipe;
8493         uint32_t val;
8494
8495         val = 0;
8496
8497         switch (crtc_state->pipe_bpp) {
8498         case 18:
8499                 val |= PIPECONF_6BPC;
8500                 break;
8501         case 24:
8502                 val |= PIPECONF_8BPC;
8503                 break;
8504         case 30:
8505                 val |= PIPECONF_10BPC;
8506                 break;
8507         case 36:
8508                 val |= PIPECONF_12BPC;
8509                 break;
8510         default:
8511                 /* Case prevented by intel_choose_pipe_bpp_dither. */
8512                 BUG();
8513         }
8514
8515         if (crtc_state->dither)
8516                 val |= (PIPECONF_DITHER_EN | PIPECONF_DITHER_TYPE_SP);
8517
8518         if (crtc_state->base.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE)
8519                 val |= PIPECONF_INTERLACED_ILK;
8520         else
8521                 val |= PIPECONF_PROGRESSIVE;
8522
8523         if (crtc_state->limited_color_range)
8524                 val |= PIPECONF_COLOR_RANGE_SELECT;
8525
8526         I915_WRITE(PIPECONF(pipe), val);
8527         POSTING_READ(PIPECONF(pipe));
8528 }
8529
8530 static void haswell_set_pipeconf(const struct intel_crtc_state *crtc_state)
8531 {
8532         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
8533         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8534         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
8535         u32 val = 0;
8536
8537         if (IS_HASWELL(dev_priv) && crtc_state->dither)
8538                 val |= (PIPECONF_DITHER_EN | PIPECONF_DITHER_TYPE_SP);
8539
8540         if (crtc_state->base.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE)
8541                 val |= PIPECONF_INTERLACED_ILK;
8542         else
8543                 val |= PIPECONF_PROGRESSIVE;
8544
8545         I915_WRITE(PIPECONF(cpu_transcoder), val);
8546         POSTING_READ(PIPECONF(cpu_transcoder));
8547 }
8548
8549 static void haswell_set_pipemisc(const struct intel_crtc_state *crtc_state)
8550 {
8551         struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc);
8552         struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
8553
8554         if (IS_BROADWELL(dev_priv) || INTEL_GEN(dev_priv) >= 9) {
8555                 u32 val = 0;
8556
8557                 switch (crtc_state->pipe_bpp) {
8558                 case 18:
8559                         val |= PIPEMISC_DITHER_6_BPC;
8560                         break;
8561                 case 24:
8562                         val |= PIPEMISC_DITHER_8_BPC;
8563                         break;
8564                 case 30:
8565                         val |= PIPEMISC_DITHER_10_BPC;
8566                         break;
8567                 case 36:
8568                         val |= PIPEMISC_DITHER_12_BPC;
8569                         break;
8570                 default:
8571                         /* Case prevented by pipe_config_set_bpp. */
8572                         BUG();
8573                 }
8574
8575                 if (crtc_state->dither)
8576                         val |= PIPEMISC_DITHER_ENABLE | PIPEMISC_DITHER_TYPE_SP;
8577
8578                 if (crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR420 ||
8579                     crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR444)
8580                         val |= PIPEMISC_OUTPUT_COLORSPACE_YUV;
8581
8582                 if (crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR420)
8583                         val |= PIPEMISC_YUV420_ENABLE |
8584                                 PIPEMISC_YUV420_MODE_FULL_BLEND;
8585
8586                 I915_WRITE(PIPEMISC(intel_crtc->pipe), val);
8587         }
8588 }
8589
8590 int ironlake_get_lanes_required(int target_clock, int link_bw, int bpp)
8591 {
8592         /*
8593          * Account for spread spectrum to avoid
8594          * oversubscribing the link. Max center spread
8595          * is 2.5%; use 5% for safety's sake.
8596          */
8597         u32 bps = target_clock * bpp * 21 / 20;
8598         return DIV_ROUND_UP(bps, link_bw * 8);
8599 }
8600
8601 static bool ironlake_needs_fb_cb_tune(struct dpll *dpll, int factor)
8602 {
8603         return i9xx_dpll_compute_m(dpll) < factor * dpll->n;
8604 }
8605
8606 static void ironlake_compute_dpll(struct intel_crtc *intel_crtc,
8607                                   struct intel_crtc_state *crtc_state,
8608                                   struct dpll *reduced_clock)
8609 {
8610         struct drm_crtc *crtc = &intel_crtc->base;
8611         struct drm_device *dev = crtc->dev;
8612         struct drm_i915_private *dev_priv = to_i915(dev);
8613         u32 dpll, fp, fp2;
8614         int factor;
8615
8616         /* Enable autotuning of the PLL clock (if permissible) */
8617         factor = 21;
8618         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
8619                 if ((intel_panel_use_ssc(dev_priv) &&
8620                      dev_priv->vbt.lvds_ssc_freq == 100000) ||
8621                     (HAS_PCH_IBX(dev_priv) && intel_is_dual_link_lvds(dev)))
8622                         factor = 25;
8623         } else if (crtc_state->sdvo_tv_clock)
8624                 factor = 20;
8625
8626         fp = i9xx_dpll_compute_fp(&crtc_state->dpll);
8627
8628         if (ironlake_needs_fb_cb_tune(&crtc_state->dpll, factor))
8629                 fp |= FP_CB_TUNE;
8630
8631         if (reduced_clock) {
8632                 fp2 = i9xx_dpll_compute_fp(reduced_clock);
8633
8634                 if (reduced_clock->m < factor * reduced_clock->n)
8635                         fp2 |= FP_CB_TUNE;
8636         } else {
8637                 fp2 = fp;
8638         }
8639
8640         dpll = 0;
8641
8642         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS))
8643                 dpll |= DPLLB_MODE_LVDS;
8644         else
8645                 dpll |= DPLLB_MODE_DAC_SERIAL;
8646
8647         dpll |= (crtc_state->pixel_multiplier - 1)
8648                 << PLL_REF_SDVO_HDMI_MULTIPLIER_SHIFT;
8649
8650         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO) ||
8651             intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI))
8652                 dpll |= DPLL_SDVO_HIGH_SPEED;
8653
8654         if (intel_crtc_has_dp_encoder(crtc_state))
8655                 dpll |= DPLL_SDVO_HIGH_SPEED;
8656
8657         /*
8658          * The high speed IO clock is only really required for
8659          * SDVO/HDMI/DP, but we also enable it for CRT to make it
8660          * possible to share the DPLL between CRT and HDMI. Enabling
8661          * the clock needlessly does no real harm, except use up a
8662          * bit of power potentially.
8663          *
8664          * We'll limit this to IVB with 3 pipes, since it has only two
8665          * DPLLs and so DPLL sharing is the only way to get three pipes
8666          * driving PCH ports at the same time. On SNB we could do this,
8667          * and potentially avoid enabling the second DPLL, but it's not
8668          * clear if it''s a win or loss power wise. No point in doing
8669          * this on ILK at all since it has a fixed DPLL<->pipe mapping.
8670          */
8671         if (INTEL_INFO(dev_priv)->num_pipes == 3 &&
8672             intel_crtc_has_type(crtc_state, INTEL_OUTPUT_ANALOG))
8673                 dpll |= DPLL_SDVO_HIGH_SPEED;
8674
8675         /* compute bitmask from p1 value */
8676         dpll |= (1 << (crtc_state->dpll.p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
8677         /* also FPA1 */
8678         dpll |= (1 << (crtc_state->dpll.p1 - 1)) << DPLL_FPA1_P1_POST_DIV_SHIFT;
8679
8680         switch (crtc_state->dpll.p2) {
8681         case 5:
8682                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_5;
8683                 break;
8684         case 7:
8685                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_7;
8686                 break;
8687         case 10:
8688                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_10;
8689                 break;
8690         case 14:
8691                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_14;
8692                 break;
8693         }
8694
8695         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS) &&
8696             intel_panel_use_ssc(dev_priv))
8697                 dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN;
8698         else
8699                 dpll |= PLL_REF_INPUT_DREFCLK;
8700
8701         dpll |= DPLL_VCO_ENABLE;
8702
8703         crtc_state->dpll_hw_state.dpll = dpll;
8704         crtc_state->dpll_hw_state.fp0 = fp;
8705         crtc_state->dpll_hw_state.fp1 = fp2;
8706 }
8707
8708 static int ironlake_crtc_compute_clock(struct intel_crtc *crtc,
8709                                        struct intel_crtc_state *crtc_state)
8710 {
8711         struct drm_device *dev = crtc->base.dev;
8712         struct drm_i915_private *dev_priv = to_i915(dev);
8713         const struct intel_limit *limit;
8714         int refclk = 120000;
8715
8716         memset(&crtc_state->dpll_hw_state, 0,
8717                sizeof(crtc_state->dpll_hw_state));
8718
8719         /* CPU eDP is the only output that doesn't need a PCH PLL of its own. */
8720         if (!crtc_state->has_pch_encoder)
8721                 return 0;
8722
8723         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
8724                 if (intel_panel_use_ssc(dev_priv)) {
8725                         DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n",
8726                                       dev_priv->vbt.lvds_ssc_freq);
8727                         refclk = dev_priv->vbt.lvds_ssc_freq;
8728                 }
8729
8730                 if (intel_is_dual_link_lvds(dev)) {
8731                         if (refclk == 100000)
8732                                 limit = &intel_limits_ironlake_dual_lvds_100m;
8733                         else
8734                                 limit = &intel_limits_ironlake_dual_lvds;
8735                 } else {
8736                         if (refclk == 100000)
8737                                 limit = &intel_limits_ironlake_single_lvds_100m;
8738                         else
8739                                 limit = &intel_limits_ironlake_single_lvds;
8740                 }
8741         } else {
8742                 limit = &intel_limits_ironlake_dac;
8743         }
8744
8745         if (!crtc_state->clock_set &&
8746             !g4x_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
8747                                 refclk, NULL, &crtc_state->dpll)) {
8748                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
8749                 return -EINVAL;
8750         }
8751
8752         ironlake_compute_dpll(crtc, crtc_state, NULL);
8753
8754         if (!intel_get_shared_dpll(crtc, crtc_state, NULL)) {
8755                 DRM_DEBUG_KMS("failed to find PLL for pipe %c\n",
8756                               pipe_name(crtc->pipe));
8757                 return -EINVAL;
8758         }
8759
8760         return 0;
8761 }
8762
8763 static void intel_pch_transcoder_get_m_n(struct intel_crtc *crtc,
8764                                          struct intel_link_m_n *m_n)
8765 {
8766         struct drm_device *dev = crtc->base.dev;
8767         struct drm_i915_private *dev_priv = to_i915(dev);
8768         enum pipe pipe = crtc->pipe;
8769
8770         m_n->link_m = I915_READ(PCH_TRANS_LINK_M1(pipe));
8771         m_n->link_n = I915_READ(PCH_TRANS_LINK_N1(pipe));
8772         m_n->gmch_m = I915_READ(PCH_TRANS_DATA_M1(pipe))
8773                 & ~TU_SIZE_MASK;
8774         m_n->gmch_n = I915_READ(PCH_TRANS_DATA_N1(pipe));
8775         m_n->tu = ((I915_READ(PCH_TRANS_DATA_M1(pipe))
8776                     & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
8777 }
8778
8779 static void intel_cpu_transcoder_get_m_n(struct intel_crtc *crtc,
8780                                          enum transcoder transcoder,
8781                                          struct intel_link_m_n *m_n,
8782                                          struct intel_link_m_n *m2_n2)
8783 {
8784         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8785         enum pipe pipe = crtc->pipe;
8786
8787         if (INTEL_GEN(dev_priv) >= 5) {
8788                 m_n->link_m = I915_READ(PIPE_LINK_M1(transcoder));
8789                 m_n->link_n = I915_READ(PIPE_LINK_N1(transcoder));
8790                 m_n->gmch_m = I915_READ(PIPE_DATA_M1(transcoder))
8791                         & ~TU_SIZE_MASK;
8792                 m_n->gmch_n = I915_READ(PIPE_DATA_N1(transcoder));
8793                 m_n->tu = ((I915_READ(PIPE_DATA_M1(transcoder))
8794                             & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
8795
8796                 if (m2_n2 && transcoder_has_m2_n2(dev_priv, transcoder)) {
8797                         m2_n2->link_m = I915_READ(PIPE_LINK_M2(transcoder));
8798                         m2_n2->link_n = I915_READ(PIPE_LINK_N2(transcoder));
8799                         m2_n2->gmch_m = I915_READ(PIPE_DATA_M2(transcoder))
8800                                         & ~TU_SIZE_MASK;
8801                         m2_n2->gmch_n = I915_READ(PIPE_DATA_N2(transcoder));
8802                         m2_n2->tu = ((I915_READ(PIPE_DATA_M2(transcoder))
8803                                         & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
8804                 }
8805         } else {
8806                 m_n->link_m = I915_READ(PIPE_LINK_M_G4X(pipe));
8807                 m_n->link_n = I915_READ(PIPE_LINK_N_G4X(pipe));
8808                 m_n->gmch_m = I915_READ(PIPE_DATA_M_G4X(pipe))
8809                         & ~TU_SIZE_MASK;
8810                 m_n->gmch_n = I915_READ(PIPE_DATA_N_G4X(pipe));
8811                 m_n->tu = ((I915_READ(PIPE_DATA_M_G4X(pipe))
8812                             & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
8813         }
8814 }
8815
8816 void intel_dp_get_m_n(struct intel_crtc *crtc,
8817                       struct intel_crtc_state *pipe_config)
8818 {
8819         if (pipe_config->has_pch_encoder)
8820                 intel_pch_transcoder_get_m_n(crtc, &pipe_config->dp_m_n);
8821         else
8822                 intel_cpu_transcoder_get_m_n(crtc, pipe_config->cpu_transcoder,
8823                                              &pipe_config->dp_m_n,
8824                                              &pipe_config->dp_m2_n2);
8825 }
8826
8827 static void ironlake_get_fdi_m_n_config(struct intel_crtc *crtc,
8828                                         struct intel_crtc_state *pipe_config)
8829 {
8830         intel_cpu_transcoder_get_m_n(crtc, pipe_config->cpu_transcoder,
8831                                      &pipe_config->fdi_m_n, NULL);
8832 }
8833
8834 static void skylake_get_pfit_config(struct intel_crtc *crtc,
8835                                     struct intel_crtc_state *pipe_config)
8836 {
8837         struct drm_device *dev = crtc->base.dev;
8838         struct drm_i915_private *dev_priv = to_i915(dev);
8839         struct intel_crtc_scaler_state *scaler_state = &pipe_config->scaler_state;
8840         uint32_t ps_ctrl = 0;
8841         int id = -1;
8842         int i;
8843
8844         /* find scaler attached to this pipe */
8845         for (i = 0; i < crtc->num_scalers; i++) {
8846                 ps_ctrl = I915_READ(SKL_PS_CTRL(crtc->pipe, i));
8847                 if (ps_ctrl & PS_SCALER_EN && !(ps_ctrl & PS_PLANE_SEL_MASK)) {
8848                         id = i;
8849                         pipe_config->pch_pfit.enabled = true;
8850                         pipe_config->pch_pfit.pos = I915_READ(SKL_PS_WIN_POS(crtc->pipe, i));
8851                         pipe_config->pch_pfit.size = I915_READ(SKL_PS_WIN_SZ(crtc->pipe, i));
8852                         break;
8853                 }
8854         }
8855
8856         scaler_state->scaler_id = id;
8857         if (id >= 0) {
8858                 scaler_state->scaler_users |= (1 << SKL_CRTC_INDEX);
8859         } else {
8860                 scaler_state->scaler_users &= ~(1 << SKL_CRTC_INDEX);
8861         }
8862 }
8863
8864 static void
8865 skylake_get_initial_plane_config(struct intel_crtc *crtc,
8866                                  struct intel_initial_plane_config *plane_config)
8867 {
8868         struct drm_device *dev = crtc->base.dev;
8869         struct drm_i915_private *dev_priv = to_i915(dev);
8870         struct intel_plane *plane = to_intel_plane(crtc->base.primary);
8871         enum plane_id plane_id = plane->id;
8872         enum pipe pipe;
8873         u32 val, base, offset, stride_mult, tiling, alpha;
8874         int fourcc, pixel_format;
8875         unsigned int aligned_height;
8876         struct drm_framebuffer *fb;
8877         struct intel_framebuffer *intel_fb;
8878
8879         if (!plane->get_hw_state(plane, &pipe))
8880                 return;
8881
8882         WARN_ON(pipe != crtc->pipe);
8883
8884         intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL);
8885         if (!intel_fb) {
8886                 DRM_DEBUG_KMS("failed to alloc fb\n");
8887                 return;
8888         }
8889
8890         fb = &intel_fb->base;
8891
8892         fb->dev = dev;
8893
8894         val = I915_READ(PLANE_CTL(pipe, plane_id));
8895
8896         if (INTEL_GEN(dev_priv) >= 11)
8897                 pixel_format = val & ICL_PLANE_CTL_FORMAT_MASK;
8898         else
8899                 pixel_format = val & PLANE_CTL_FORMAT_MASK;
8900
8901         if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv)) {
8902                 alpha = I915_READ(PLANE_COLOR_CTL(pipe, plane_id));
8903                 alpha &= PLANE_COLOR_ALPHA_MASK;
8904         } else {
8905                 alpha = val & PLANE_CTL_ALPHA_MASK;
8906         }
8907
8908         fourcc = skl_format_to_fourcc(pixel_format,
8909                                       val & PLANE_CTL_ORDER_RGBX, alpha);
8910         fb->format = drm_format_info(fourcc);
8911
8912         tiling = val & PLANE_CTL_TILED_MASK;
8913         switch (tiling) {
8914         case PLANE_CTL_TILED_LINEAR:
8915                 fb->modifier = DRM_FORMAT_MOD_LINEAR;
8916                 break;
8917         case PLANE_CTL_TILED_X:
8918                 plane_config->tiling = I915_TILING_X;
8919                 fb->modifier = I915_FORMAT_MOD_X_TILED;
8920                 break;
8921         case PLANE_CTL_TILED_Y:
8922                 plane_config->tiling = I915_TILING_Y;
8923                 if (val & PLANE_CTL_RENDER_DECOMPRESSION_ENABLE)
8924                         fb->modifier = I915_FORMAT_MOD_Y_TILED_CCS;
8925                 else
8926                         fb->modifier = I915_FORMAT_MOD_Y_TILED;
8927                 break;
8928         case PLANE_CTL_TILED_YF:
8929                 if (val & PLANE_CTL_RENDER_DECOMPRESSION_ENABLE)
8930                         fb->modifier = I915_FORMAT_MOD_Yf_TILED_CCS;
8931                 else
8932                         fb->modifier = I915_FORMAT_MOD_Yf_TILED;
8933                 break;
8934         default:
8935                 MISSING_CASE(tiling);
8936                 goto error;
8937         }
8938
8939         /*
8940          * DRM_MODE_ROTATE_ is counter clockwise to stay compatible with Xrandr
8941          * while i915 HW rotation is clockwise, thats why this swapping.
8942          */
8943         switch (val & PLANE_CTL_ROTATE_MASK) {
8944         case PLANE_CTL_ROTATE_0:
8945                 plane_config->rotation = DRM_MODE_ROTATE_0;
8946                 break;
8947         case PLANE_CTL_ROTATE_90:
8948                 plane_config->rotation = DRM_MODE_ROTATE_270;
8949                 break;
8950         case PLANE_CTL_ROTATE_180:
8951                 plane_config->rotation = DRM_MODE_ROTATE_180;
8952                 break;
8953         case PLANE_CTL_ROTATE_270:
8954                 plane_config->rotation = DRM_MODE_ROTATE_90;
8955                 break;
8956         }
8957
8958         if (INTEL_GEN(dev_priv) >= 10 &&
8959             val & PLANE_CTL_FLIP_HORIZONTAL)
8960                 plane_config->rotation |= DRM_MODE_REFLECT_X;
8961
8962         base = I915_READ(PLANE_SURF(pipe, plane_id)) & 0xfffff000;
8963         plane_config->base = base;
8964
8965         offset = I915_READ(PLANE_OFFSET(pipe, plane_id));
8966
8967         val = I915_READ(PLANE_SIZE(pipe, plane_id));
8968         fb->height = ((val >> 16) & 0xfff) + 1;
8969         fb->width = ((val >> 0) & 0x1fff) + 1;
8970
8971         val = I915_READ(PLANE_STRIDE(pipe, plane_id));
8972         stride_mult = skl_plane_stride_mult(fb, 0, DRM_MODE_ROTATE_0);
8973         fb->pitches[0] = (val & 0x3ff) * stride_mult;
8974
8975         aligned_height = intel_fb_align_height(fb, 0, fb->height);
8976
8977         plane_config->size = fb->pitches[0] * aligned_height;
8978
8979         DRM_DEBUG_KMS("%s/%s with fb: size=%dx%d@%d, offset=%x, pitch %d, size 0x%x\n",
8980                       crtc->base.name, plane->base.name, fb->width, fb->height,
8981                       fb->format->cpp[0] * 8, base, fb->pitches[0],
8982                       plane_config->size);
8983
8984         plane_config->fb = intel_fb;
8985         return;
8986
8987 error:
8988         kfree(intel_fb);
8989 }
8990
8991 static void ironlake_get_pfit_config(struct intel_crtc *crtc,
8992                                      struct intel_crtc_state *pipe_config)
8993 {
8994         struct drm_device *dev = crtc->base.dev;
8995         struct drm_i915_private *dev_priv = to_i915(dev);
8996         uint32_t tmp;
8997
8998         tmp = I915_READ(PF_CTL(crtc->pipe));
8999
9000         if (tmp & PF_ENABLE) {
9001                 pipe_config->pch_pfit.enabled = true;
9002                 pipe_config->pch_pfit.pos = I915_READ(PF_WIN_POS(crtc->pipe));
9003                 pipe_config->pch_pfit.size = I915_READ(PF_WIN_SZ(crtc->pipe));
9004
9005                 /* We currently do not free assignements of panel fitters on
9006                  * ivb/hsw (since we don't use the higher upscaling modes which
9007                  * differentiates them) so just WARN about this case for now. */
9008                 if (IS_GEN7(dev_priv)) {
9009                         WARN_ON((tmp & PF_PIPE_SEL_MASK_IVB) !=
9010                                 PF_PIPE_SEL_IVB(crtc->pipe));
9011                 }
9012         }
9013 }
9014
9015 static bool ironlake_get_pipe_config(struct intel_crtc *crtc,
9016                                      struct intel_crtc_state *pipe_config)
9017 {
9018         struct drm_device *dev = crtc->base.dev;
9019         struct drm_i915_private *dev_priv = to_i915(dev);
9020         enum intel_display_power_domain power_domain;
9021         uint32_t tmp;
9022         bool ret;
9023
9024         power_domain = POWER_DOMAIN_PIPE(crtc->pipe);
9025         if (!intel_display_power_get_if_enabled(dev_priv, power_domain))
9026                 return false;
9027
9028         pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB;
9029         pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe;
9030         pipe_config->shared_dpll = NULL;
9031
9032         ret = false;
9033         tmp = I915_READ(PIPECONF(crtc->pipe));
9034         if (!(tmp & PIPECONF_ENABLE))
9035                 goto out;
9036
9037         switch (tmp & PIPECONF_BPC_MASK) {
9038         case PIPECONF_6BPC:
9039                 pipe_config->pipe_bpp = 18;
9040                 break;
9041         case PIPECONF_8BPC:
9042                 pipe_config->pipe_bpp = 24;
9043                 break;
9044         case PIPECONF_10BPC:
9045                 pipe_config->pipe_bpp = 30;
9046                 break;
9047         case PIPECONF_12BPC:
9048                 pipe_config->pipe_bpp = 36;
9049                 break;
9050         default:
9051                 break;
9052         }
9053
9054         if (tmp & PIPECONF_COLOR_RANGE_SELECT)
9055                 pipe_config->limited_color_range = true;
9056
9057         if (I915_READ(PCH_TRANSCONF(crtc->pipe)) & TRANS_ENABLE) {
9058                 struct intel_shared_dpll *pll;
9059                 enum intel_dpll_id pll_id;
9060
9061                 pipe_config->has_pch_encoder = true;
9062
9063                 tmp = I915_READ(FDI_RX_CTL(crtc->pipe));
9064                 pipe_config->fdi_lanes = ((FDI_DP_PORT_WIDTH_MASK & tmp) >>
9065                                           FDI_DP_PORT_WIDTH_SHIFT) + 1;
9066
9067                 ironlake_get_fdi_m_n_config(crtc, pipe_config);
9068
9069                 if (HAS_PCH_IBX(dev_priv)) {
9070                         /*
9071                          * The pipe->pch transcoder and pch transcoder->pll
9072                          * mapping is fixed.
9073                          */
9074                         pll_id = (enum intel_dpll_id) crtc->pipe;
9075                 } else {
9076                         tmp = I915_READ(PCH_DPLL_SEL);
9077                         if (tmp & TRANS_DPLLB_SEL(crtc->pipe))
9078                                 pll_id = DPLL_ID_PCH_PLL_B;
9079                         else
9080                                 pll_id= DPLL_ID_PCH_PLL_A;
9081                 }
9082
9083                 pipe_config->shared_dpll =
9084                         intel_get_shared_dpll_by_id(dev_priv, pll_id);
9085                 pll = pipe_config->shared_dpll;
9086
9087                 WARN_ON(!pll->info->funcs->get_hw_state(dev_priv, pll,
9088                                                 &pipe_config->dpll_hw_state));
9089
9090                 tmp = pipe_config->dpll_hw_state.dpll;
9091                 pipe_config->pixel_multiplier =
9092                         ((tmp & PLL_REF_SDVO_HDMI_MULTIPLIER_MASK)
9093                          >> PLL_REF_SDVO_HDMI_MULTIPLIER_SHIFT) + 1;
9094
9095                 ironlake_pch_clock_get(crtc, pipe_config);
9096         } else {
9097                 pipe_config->pixel_multiplier = 1;
9098         }
9099
9100         intel_get_pipe_timings(crtc, pipe_config);
9101         intel_get_pipe_src_size(crtc, pipe_config);
9102
9103         ironlake_get_pfit_config(crtc, pipe_config);
9104
9105         ret = true;
9106
9107 out:
9108         intel_display_power_put(dev_priv, power_domain);
9109
9110         return ret;
9111 }
9112
9113 static void assert_can_disable_lcpll(struct drm_i915_private *dev_priv)
9114 {
9115         struct drm_device *dev = &dev_priv->drm;
9116         struct intel_crtc *crtc;
9117
9118         for_each_intel_crtc(dev, crtc)
9119                 I915_STATE_WARN(crtc->active, "CRTC for pipe %c enabled\n",
9120                      pipe_name(crtc->pipe));
9121
9122         I915_STATE_WARN(I915_READ(HSW_PWR_WELL_CTL2),
9123                         "Display power well on\n");
9124         I915_STATE_WARN(I915_READ(SPLL_CTL) & SPLL_PLL_ENABLE, "SPLL enabled\n");
9125         I915_STATE_WARN(I915_READ(WRPLL_CTL(0)) & WRPLL_PLL_ENABLE, "WRPLL1 enabled\n");
9126         I915_STATE_WARN(I915_READ(WRPLL_CTL(1)) & WRPLL_PLL_ENABLE, "WRPLL2 enabled\n");
9127         I915_STATE_WARN(I915_READ(PP_STATUS(0)) & PP_ON, "Panel power on\n");
9128         I915_STATE_WARN(I915_READ(BLC_PWM_CPU_CTL2) & BLM_PWM_ENABLE,
9129              "CPU PWM1 enabled\n");
9130         if (IS_HASWELL(dev_priv))
9131                 I915_STATE_WARN(I915_READ(HSW_BLC_PWM2_CTL) & BLM_PWM_ENABLE,
9132                      "CPU PWM2 enabled\n");
9133         I915_STATE_WARN(I915_READ(BLC_PWM_PCH_CTL1) & BLM_PCH_PWM_ENABLE,
9134              "PCH PWM1 enabled\n");
9135         I915_STATE_WARN(I915_READ(UTIL_PIN_CTL) & UTIL_PIN_ENABLE,
9136              "Utility pin enabled\n");
9137         I915_STATE_WARN(I915_READ(PCH_GTC_CTL) & PCH_GTC_ENABLE, "PCH GTC enabled\n");
9138
9139         /*
9140          * In theory we can still leave IRQs enabled, as long as only the HPD
9141          * interrupts remain enabled. We used to check for that, but since it's
9142          * gen-specific and since we only disable LCPLL after we fully disable
9143          * the interrupts, the check below should be enough.
9144          */
9145         I915_STATE_WARN(intel_irqs_enabled(dev_priv), "IRQs enabled\n");
9146 }
9147
9148 static uint32_t hsw_read_dcomp(struct drm_i915_private *dev_priv)
9149 {
9150         if (IS_HASWELL(dev_priv))
9151                 return I915_READ(D_COMP_HSW);
9152         else
9153                 return I915_READ(D_COMP_BDW);
9154 }
9155
9156 static void hsw_write_dcomp(struct drm_i915_private *dev_priv, uint32_t val)
9157 {
9158         if (IS_HASWELL(dev_priv)) {
9159                 mutex_lock(&dev_priv->pcu_lock);
9160                 if (sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_D_COMP,
9161                                             val))
9162                         DRM_DEBUG_KMS("Failed to write to D_COMP\n");
9163                 mutex_unlock(&dev_priv->pcu_lock);
9164         } else {
9165                 I915_WRITE(D_COMP_BDW, val);
9166                 POSTING_READ(D_COMP_BDW);
9167         }
9168 }
9169
9170 /*
9171  * This function implements pieces of two sequences from BSpec:
9172  * - Sequence for display software to disable LCPLL
9173  * - Sequence for display software to allow package C8+
9174  * The steps implemented here are just the steps that actually touch the LCPLL
9175  * register. Callers should take care of disabling all the display engine
9176  * functions, doing the mode unset, fixing interrupts, etc.
9177  */
9178 static void hsw_disable_lcpll(struct drm_i915_private *dev_priv,
9179                               bool switch_to_fclk, bool allow_power_down)
9180 {
9181         uint32_t val;
9182
9183         assert_can_disable_lcpll(dev_priv);
9184
9185         val = I915_READ(LCPLL_CTL);
9186
9187         if (switch_to_fclk) {
9188                 val |= LCPLL_CD_SOURCE_FCLK;
9189                 I915_WRITE(LCPLL_CTL, val);
9190
9191                 if (wait_for_us(I915_READ(LCPLL_CTL) &
9192                                 LCPLL_CD_SOURCE_FCLK_DONE, 1))
9193                         DRM_ERROR("Switching to FCLK failed\n");
9194
9195                 val = I915_READ(LCPLL_CTL);
9196         }
9197
9198         val |= LCPLL_PLL_DISABLE;
9199         I915_WRITE(LCPLL_CTL, val);
9200         POSTING_READ(LCPLL_CTL);
9201
9202         if (intel_wait_for_register(dev_priv, LCPLL_CTL, LCPLL_PLL_LOCK, 0, 1))
9203                 DRM_ERROR("LCPLL still locked\n");
9204
9205         val = hsw_read_dcomp(dev_priv);
9206         val |= D_COMP_COMP_DISABLE;
9207         hsw_write_dcomp(dev_priv, val);
9208         ndelay(100);
9209
9210         if (wait_for((hsw_read_dcomp(dev_priv) & D_COMP_RCOMP_IN_PROGRESS) == 0,
9211                      1))
9212                 DRM_ERROR("D_COMP RCOMP still in progress\n");
9213
9214         if (allow_power_down) {
9215                 val = I915_READ(LCPLL_CTL);
9216                 val |= LCPLL_POWER_DOWN_ALLOW;
9217                 I915_WRITE(LCPLL_CTL, val);
9218                 POSTING_READ(LCPLL_CTL);
9219         }
9220 }
9221
9222 /*
9223  * Fully restores LCPLL, disallowing power down and switching back to LCPLL
9224  * source.
9225  */
9226 static void hsw_restore_lcpll(struct drm_i915_private *dev_priv)
9227 {
9228         uint32_t val;
9229
9230         val = I915_READ(LCPLL_CTL);
9231
9232         if ((val & (LCPLL_PLL_LOCK | LCPLL_PLL_DISABLE | LCPLL_CD_SOURCE_FCLK |
9233                     LCPLL_POWER_DOWN_ALLOW)) == LCPLL_PLL_LOCK)
9234                 return;
9235
9236         /*
9237          * Make sure we're not on PC8 state before disabling PC8, otherwise
9238          * we'll hang the machine. To prevent PC8 state, just enable force_wake.
9239          */
9240         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
9241
9242         if (val & LCPLL_POWER_DOWN_ALLOW) {
9243                 val &= ~LCPLL_POWER_DOWN_ALLOW;
9244                 I915_WRITE(LCPLL_CTL, val);
9245                 POSTING_READ(LCPLL_CTL);
9246         }
9247
9248         val = hsw_read_dcomp(dev_priv);
9249         val |= D_COMP_COMP_FORCE;
9250         val &= ~D_COMP_COMP_DISABLE;
9251         hsw_write_dcomp(dev_priv, val);
9252
9253         val = I915_READ(LCPLL_CTL);
9254         val &= ~LCPLL_PLL_DISABLE;
9255         I915_WRITE(LCPLL_CTL, val);
9256
9257         if (intel_wait_for_register(dev_priv,
9258                                     LCPLL_CTL, LCPLL_PLL_LOCK, LCPLL_PLL_LOCK,
9259                                     5))
9260                 DRM_ERROR("LCPLL not locked yet\n");
9261
9262         if (val & LCPLL_CD_SOURCE_FCLK) {
9263                 val = I915_READ(LCPLL_CTL);
9264                 val &= ~LCPLL_CD_SOURCE_FCLK;
9265                 I915_WRITE(LCPLL_CTL, val);
9266
9267                 if (wait_for_us((I915_READ(LCPLL_CTL) &
9268                                  LCPLL_CD_SOURCE_FCLK_DONE) == 0, 1))
9269                         DRM_ERROR("Switching back to LCPLL failed\n");
9270         }
9271
9272         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
9273
9274         intel_update_cdclk(dev_priv);
9275         intel_dump_cdclk_state(&dev_priv->cdclk.hw, "Current CDCLK");
9276 }
9277
9278 /*
9279  * Package states C8 and deeper are really deep PC states that can only be
9280  * reached when all the devices on the system allow it, so even if the graphics
9281  * device allows PC8+, it doesn't mean the system will actually get to these
9282  * states. Our driver only allows PC8+ when going into runtime PM.
9283  *
9284  * The requirements for PC8+ are that all the outputs are disabled, the power
9285  * well is disabled and most interrupts are disabled, and these are also
9286  * requirements for runtime PM. When these conditions are met, we manually do
9287  * the other conditions: disable the interrupts, clocks and switch LCPLL refclk
9288  * to Fclk. If we're in PC8+ and we get an non-hotplug interrupt, we can hard
9289  * hang the machine.
9290  *
9291  * When we really reach PC8 or deeper states (not just when we allow it) we lose
9292  * the state of some registers, so when we come back from PC8+ we need to
9293  * restore this state. We don't get into PC8+ if we're not in RC6, so we don't
9294  * need to take care of the registers kept by RC6. Notice that this happens even
9295  * if we don't put the device in PCI D3 state (which is what currently happens
9296  * because of the runtime PM support).
9297  *
9298  * For more, read "Display Sequences for Package C8" on the hardware
9299  * documentation.
9300  */
9301 void hsw_enable_pc8(struct drm_i915_private *dev_priv)
9302 {
9303         uint32_t val;
9304
9305         DRM_DEBUG_KMS("Enabling package C8+\n");
9306
9307         if (HAS_PCH_LPT_LP(dev_priv)) {
9308                 val = I915_READ(SOUTH_DSPCLK_GATE_D);
9309                 val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
9310                 I915_WRITE(SOUTH_DSPCLK_GATE_D, val);
9311         }
9312
9313         lpt_disable_clkout_dp(dev_priv);
9314         hsw_disable_lcpll(dev_priv, true, true);
9315 }
9316
9317 void hsw_disable_pc8(struct drm_i915_private *dev_priv)
9318 {
9319         uint32_t val;
9320
9321         DRM_DEBUG_KMS("Disabling package C8+\n");
9322
9323         hsw_restore_lcpll(dev_priv);
9324         lpt_init_pch_refclk(dev_priv);
9325
9326         if (HAS_PCH_LPT_LP(dev_priv)) {
9327                 val = I915_READ(SOUTH_DSPCLK_GATE_D);
9328                 val |= PCH_LP_PARTITION_LEVEL_DISABLE;
9329                 I915_WRITE(SOUTH_DSPCLK_GATE_D, val);
9330         }
9331 }
9332
9333 static int haswell_crtc_compute_clock(struct intel_crtc *crtc,
9334                                       struct intel_crtc_state *crtc_state)
9335 {
9336         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9337         struct intel_atomic_state *state =
9338                 to_intel_atomic_state(crtc_state->base.state);
9339
9340         if (!intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DSI) ||
9341             IS_ICELAKE(dev_priv)) {
9342                 struct intel_encoder *encoder =
9343                         intel_get_crtc_new_encoder(state, crtc_state);
9344
9345                 if (!intel_get_shared_dpll(crtc, crtc_state, encoder)) {
9346                         DRM_DEBUG_KMS("failed to find PLL for pipe %c\n",
9347                                       pipe_name(crtc->pipe));
9348                         return -EINVAL;
9349                 }
9350         }
9351
9352         return 0;
9353 }
9354
9355 static void cannonlake_get_ddi_pll(struct drm_i915_private *dev_priv,
9356                                    enum port port,
9357                                    struct intel_crtc_state *pipe_config)
9358 {
9359         enum intel_dpll_id id;
9360         u32 temp;
9361
9362         temp = I915_READ(DPCLKA_CFGCR0) & DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(port);
9363         id = temp >> DPCLKA_CFGCR0_DDI_CLK_SEL_SHIFT(port);
9364
9365         if (WARN_ON(id < SKL_DPLL0 || id > SKL_DPLL2))
9366                 return;
9367
9368         pipe_config->shared_dpll = intel_get_shared_dpll_by_id(dev_priv, id);
9369 }
9370
9371 static void icelake_get_ddi_pll(struct drm_i915_private *dev_priv,
9372                                 enum port port,
9373                                 struct intel_crtc_state *pipe_config)
9374 {
9375         enum intel_dpll_id id;
9376         u32 temp;
9377
9378         /* TODO: TBT pll not implemented. */
9379         if (intel_port_is_combophy(dev_priv, port)) {
9380                 temp = I915_READ(DPCLKA_CFGCR0_ICL) &
9381                        DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(port);
9382                 id = temp >> DPCLKA_CFGCR0_DDI_CLK_SEL_SHIFT(port);
9383
9384                 if (WARN_ON(!intel_dpll_is_combophy(id)))
9385                         return;
9386         } else if (intel_port_is_tc(dev_priv, port)) {
9387                 id = icl_port_to_mg_pll_id(port);
9388         } else {
9389                 WARN(1, "Invalid port %x\n", port);
9390                 return;
9391         }
9392
9393         pipe_config->shared_dpll = intel_get_shared_dpll_by_id(dev_priv, id);
9394 }
9395
9396 static void bxt_get_ddi_pll(struct drm_i915_private *dev_priv,
9397                                 enum port port,
9398                                 struct intel_crtc_state *pipe_config)
9399 {
9400         enum intel_dpll_id id;
9401
9402         switch (port) {
9403         case PORT_A:
9404                 id = DPLL_ID_SKL_DPLL0;
9405                 break;
9406         case PORT_B:
9407                 id = DPLL_ID_SKL_DPLL1;
9408                 break;
9409         case PORT_C:
9410                 id = DPLL_ID_SKL_DPLL2;
9411                 break;
9412         default:
9413                 DRM_ERROR("Incorrect port type\n");
9414                 return;
9415         }
9416
9417         pipe_config->shared_dpll = intel_get_shared_dpll_by_id(dev_priv, id);
9418 }
9419
9420 static void skylake_get_ddi_pll(struct drm_i915_private *dev_priv,
9421                                 enum port port,
9422                                 struct intel_crtc_state *pipe_config)
9423 {
9424         enum intel_dpll_id id;
9425         u32 temp;
9426
9427         temp = I915_READ(DPLL_CTRL2) & DPLL_CTRL2_DDI_CLK_SEL_MASK(port);
9428         id = temp >> (port * 3 + 1);
9429
9430         if (WARN_ON(id < SKL_DPLL0 || id > SKL_DPLL3))
9431                 return;
9432
9433         pipe_config->shared_dpll = intel_get_shared_dpll_by_id(dev_priv, id);
9434 }
9435
9436 static void haswell_get_ddi_pll(struct drm_i915_private *dev_priv,
9437                                 enum port port,
9438                                 struct intel_crtc_state *pipe_config)
9439 {
9440         enum intel_dpll_id id;
9441         uint32_t ddi_pll_sel = I915_READ(PORT_CLK_SEL(port));
9442
9443         switch (ddi_pll_sel) {
9444         case PORT_CLK_SEL_WRPLL1:
9445                 id = DPLL_ID_WRPLL1;
9446                 break;
9447         case PORT_CLK_SEL_WRPLL2:
9448                 id = DPLL_ID_WRPLL2;
9449                 break;
9450         case PORT_CLK_SEL_SPLL:
9451                 id = DPLL_ID_SPLL;
9452                 break;
9453         case PORT_CLK_SEL_LCPLL_810:
9454                 id = DPLL_ID_LCPLL_810;
9455                 break;
9456         case PORT_CLK_SEL_LCPLL_1350:
9457                 id = DPLL_ID_LCPLL_1350;
9458                 break;
9459         case PORT_CLK_SEL_LCPLL_2700:
9460                 id = DPLL_ID_LCPLL_2700;
9461                 break;
9462         default:
9463                 MISSING_CASE(ddi_pll_sel);
9464                 /* fall through */
9465         case PORT_CLK_SEL_NONE:
9466                 return;
9467         }
9468
9469         pipe_config->shared_dpll = intel_get_shared_dpll_by_id(dev_priv, id);
9470 }
9471
9472 static bool hsw_get_transcoder_state(struct intel_crtc *crtc,
9473                                      struct intel_crtc_state *pipe_config,
9474                                      u64 *power_domain_mask)
9475 {
9476         struct drm_device *dev = crtc->base.dev;
9477         struct drm_i915_private *dev_priv = to_i915(dev);
9478         enum intel_display_power_domain power_domain;
9479         unsigned long panel_transcoder_mask = BIT(TRANSCODER_EDP);
9480         unsigned long enabled_panel_transcoders = 0;
9481         enum transcoder panel_transcoder;
9482         u32 tmp;
9483
9484         if (IS_ICELAKE(dev_priv))
9485                 panel_transcoder_mask |=
9486                         BIT(TRANSCODER_DSI_0) | BIT(TRANSCODER_DSI_1);
9487
9488         /*
9489          * The pipe->transcoder mapping is fixed with the exception of the eDP
9490          * and DSI transcoders handled below.
9491          */
9492         pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe;
9493
9494         /*
9495          * XXX: Do intel_display_power_get_if_enabled before reading this (for
9496          * consistency and less surprising code; it's in always on power).
9497          */
9498         for_each_set_bit(panel_transcoder, &panel_transcoder_mask, 32) {
9499                 enum pipe trans_pipe;
9500
9501                 tmp = I915_READ(TRANS_DDI_FUNC_CTL(panel_transcoder));
9502                 if (!(tmp & TRANS_DDI_FUNC_ENABLE))
9503                         continue;
9504
9505                 /*
9506                  * Log all enabled ones, only use the first one.
9507                  *
9508                  * FIXME: This won't work for two separate DSI displays.
9509                  */
9510                 enabled_panel_transcoders |= BIT(panel_transcoder);
9511                 if (enabled_panel_transcoders != BIT(panel_transcoder))
9512                         continue;
9513
9514                 switch (tmp & TRANS_DDI_EDP_INPUT_MASK) {
9515                 default:
9516                         WARN(1, "unknown pipe linked to transcoder %s\n",
9517                              transcoder_name(panel_transcoder));
9518                         /* fall through */
9519                 case TRANS_DDI_EDP_INPUT_A_ONOFF:
9520                 case TRANS_DDI_EDP_INPUT_A_ON:
9521                         trans_pipe = PIPE_A;
9522                         break;
9523                 case TRANS_DDI_EDP_INPUT_B_ONOFF:
9524                         trans_pipe = PIPE_B;
9525                         break;
9526                 case TRANS_DDI_EDP_INPUT_C_ONOFF:
9527                         trans_pipe = PIPE_C;
9528                         break;
9529                 }
9530
9531                 if (trans_pipe == crtc->pipe)
9532                         pipe_config->cpu_transcoder = panel_transcoder;
9533         }
9534
9535         /*
9536          * Valid combos: none, eDP, DSI0, DSI1, DSI0+DSI1
9537          */
9538         WARN_ON((enabled_panel_transcoders & BIT(TRANSCODER_EDP)) &&
9539                 enabled_panel_transcoders != BIT(TRANSCODER_EDP));
9540
9541         power_domain = POWER_DOMAIN_TRANSCODER(pipe_config->cpu_transcoder);
9542         if (!intel_display_power_get_if_enabled(dev_priv, power_domain))
9543                 return false;
9544         *power_domain_mask |= BIT_ULL(power_domain);
9545
9546         tmp = I915_READ(PIPECONF(pipe_config->cpu_transcoder));
9547
9548         return tmp & PIPECONF_ENABLE;
9549 }
9550
9551 static bool bxt_get_dsi_transcoder_state(struct intel_crtc *crtc,
9552                                          struct intel_crtc_state *pipe_config,
9553                                          u64 *power_domain_mask)
9554 {
9555         struct drm_device *dev = crtc->base.dev;
9556         struct drm_i915_private *dev_priv = to_i915(dev);
9557         enum intel_display_power_domain power_domain;
9558         enum port port;
9559         enum transcoder cpu_transcoder;
9560         u32 tmp;
9561
9562         for_each_port_masked(port, BIT(PORT_A) | BIT(PORT_C)) {
9563                 if (port == PORT_A)
9564                         cpu_transcoder = TRANSCODER_DSI_A;
9565                 else
9566                         cpu_transcoder = TRANSCODER_DSI_C;
9567
9568                 power_domain = POWER_DOMAIN_TRANSCODER(cpu_transcoder);
9569                 if (!intel_display_power_get_if_enabled(dev_priv, power_domain))
9570                         continue;
9571                 *power_domain_mask |= BIT_ULL(power_domain);
9572
9573                 /*
9574                  * The PLL needs to be enabled with a valid divider
9575                  * configuration, otherwise accessing DSI registers will hang
9576                  * the machine. See BSpec North Display Engine
9577                  * registers/MIPI[BXT]. We can break out here early, since we
9578                  * need the same DSI PLL to be enabled for both DSI ports.
9579                  */
9580                 if (!bxt_dsi_pll_is_enabled(dev_priv))
9581                         break;
9582
9583                 /* XXX: this works for video mode only */
9584                 tmp = I915_READ(BXT_MIPI_PORT_CTRL(port));
9585                 if (!(tmp & DPI_ENABLE))
9586                         continue;
9587
9588                 tmp = I915_READ(MIPI_CTRL(port));
9589                 if ((tmp & BXT_PIPE_SELECT_MASK) != BXT_PIPE_SELECT(crtc->pipe))
9590                         continue;
9591
9592                 pipe_config->cpu_transcoder = cpu_transcoder;
9593                 break;
9594         }
9595
9596         return transcoder_is_dsi(pipe_config->cpu_transcoder);
9597 }
9598
9599 static void haswell_get_ddi_port_state(struct intel_crtc *crtc,
9600                                        struct intel_crtc_state *pipe_config)
9601 {
9602         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9603         struct intel_shared_dpll *pll;
9604         enum port port;
9605         uint32_t tmp;
9606
9607         tmp = I915_READ(TRANS_DDI_FUNC_CTL(pipe_config->cpu_transcoder));
9608
9609         port = (tmp & TRANS_DDI_PORT_MASK) >> TRANS_DDI_PORT_SHIFT;
9610
9611         if (IS_ICELAKE(dev_priv))
9612                 icelake_get_ddi_pll(dev_priv, port, pipe_config);
9613         else if (IS_CANNONLAKE(dev_priv))
9614                 cannonlake_get_ddi_pll(dev_priv, port, pipe_config);
9615         else if (IS_GEN9_BC(dev_priv))
9616                 skylake_get_ddi_pll(dev_priv, port, pipe_config);
9617         else if (IS_GEN9_LP(dev_priv))
9618                 bxt_get_ddi_pll(dev_priv, port, pipe_config);
9619         else
9620                 haswell_get_ddi_pll(dev_priv, port, pipe_config);
9621
9622         pll = pipe_config->shared_dpll;
9623         if (pll) {
9624                 WARN_ON(!pll->info->funcs->get_hw_state(dev_priv, pll,
9625                                                 &pipe_config->dpll_hw_state));
9626         }
9627
9628         /*
9629          * Haswell has only FDI/PCH transcoder A. It is which is connected to
9630          * DDI E. So just check whether this pipe is wired to DDI E and whether
9631          * the PCH transcoder is on.
9632          */
9633         if (INTEL_GEN(dev_priv) < 9 &&
9634             (port == PORT_E) && I915_READ(LPT_TRANSCONF) & TRANS_ENABLE) {
9635                 pipe_config->has_pch_encoder = true;
9636
9637                 tmp = I915_READ(FDI_RX_CTL(PIPE_A));
9638                 pipe_config->fdi_lanes = ((FDI_DP_PORT_WIDTH_MASK & tmp) >>
9639                                           FDI_DP_PORT_WIDTH_SHIFT) + 1;
9640
9641                 ironlake_get_fdi_m_n_config(crtc, pipe_config);
9642         }
9643 }
9644
9645 static bool haswell_get_pipe_config(struct intel_crtc *crtc,
9646                                     struct intel_crtc_state *pipe_config)
9647 {
9648         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9649         enum intel_display_power_domain power_domain;
9650         u64 power_domain_mask;
9651         bool active;
9652
9653         intel_crtc_init_scalers(crtc, pipe_config);
9654
9655         power_domain = POWER_DOMAIN_PIPE(crtc->pipe);
9656         if (!intel_display_power_get_if_enabled(dev_priv, power_domain))
9657                 return false;
9658         power_domain_mask = BIT_ULL(power_domain);
9659
9660         pipe_config->shared_dpll = NULL;
9661
9662         active = hsw_get_transcoder_state(crtc, pipe_config, &power_domain_mask);
9663
9664         if (IS_GEN9_LP(dev_priv) &&
9665             bxt_get_dsi_transcoder_state(crtc, pipe_config, &power_domain_mask)) {
9666                 WARN_ON(active);
9667                 active = true;
9668         }
9669
9670         if (!active)
9671                 goto out;
9672
9673         if (!transcoder_is_dsi(pipe_config->cpu_transcoder) ||
9674             IS_ICELAKE(dev_priv)) {
9675                 haswell_get_ddi_port_state(crtc, pipe_config);
9676                 intel_get_pipe_timings(crtc, pipe_config);
9677         }
9678
9679         intel_get_pipe_src_size(crtc, pipe_config);
9680         intel_get_crtc_ycbcr_config(crtc, pipe_config);
9681
9682         pipe_config->gamma_mode =
9683                 I915_READ(GAMMA_MODE(crtc->pipe)) & GAMMA_MODE_MODE_MASK;
9684
9685         power_domain = POWER_DOMAIN_PIPE_PANEL_FITTER(crtc->pipe);
9686         if (intel_display_power_get_if_enabled(dev_priv, power_domain)) {
9687                 power_domain_mask |= BIT_ULL(power_domain);
9688                 if (INTEL_GEN(dev_priv) >= 9)
9689                         skylake_get_pfit_config(crtc, pipe_config);
9690                 else
9691                         ironlake_get_pfit_config(crtc, pipe_config);
9692         }
9693
9694         if (hsw_crtc_supports_ips(crtc)) {
9695                 if (IS_HASWELL(dev_priv))
9696                         pipe_config->ips_enabled = I915_READ(IPS_CTL) & IPS_ENABLE;
9697                 else {
9698                         /*
9699                          * We cannot readout IPS state on broadwell, set to
9700                          * true so we can set it to a defined state on first
9701                          * commit.
9702                          */
9703                         pipe_config->ips_enabled = true;
9704                 }
9705         }
9706
9707         if (pipe_config->cpu_transcoder != TRANSCODER_EDP &&
9708             !transcoder_is_dsi(pipe_config->cpu_transcoder)) {
9709                 pipe_config->pixel_multiplier =
9710                         I915_READ(PIPE_MULT(pipe_config->cpu_transcoder)) + 1;
9711         } else {
9712                 pipe_config->pixel_multiplier = 1;
9713         }
9714
9715 out:
9716         for_each_power_domain(power_domain, power_domain_mask)
9717                 intel_display_power_put(dev_priv, power_domain);
9718
9719         return active;
9720 }
9721
9722 static u32 intel_cursor_base(const struct intel_plane_state *plane_state)
9723 {
9724         struct drm_i915_private *dev_priv =
9725                 to_i915(plane_state->base.plane->dev);
9726         const struct drm_framebuffer *fb = plane_state->base.fb;
9727         const struct drm_i915_gem_object *obj = intel_fb_obj(fb);
9728         u32 base;
9729
9730         if (INTEL_INFO(dev_priv)->display.cursor_needs_physical)
9731                 base = obj->phys_handle->busaddr;
9732         else
9733                 base = intel_plane_ggtt_offset(plane_state);
9734
9735         base += plane_state->color_plane[0].offset;
9736
9737         /* ILK+ do this automagically */
9738         if (HAS_GMCH_DISPLAY(dev_priv) &&
9739             plane_state->base.rotation & DRM_MODE_ROTATE_180)
9740                 base += (plane_state->base.crtc_h *
9741                          plane_state->base.crtc_w - 1) * fb->format->cpp[0];
9742
9743         return base;
9744 }
9745
9746 static u32 intel_cursor_position(const struct intel_plane_state *plane_state)
9747 {
9748         int x = plane_state->base.crtc_x;
9749         int y = plane_state->base.crtc_y;
9750         u32 pos = 0;
9751
9752         if (x < 0) {
9753                 pos |= CURSOR_POS_SIGN << CURSOR_X_SHIFT;
9754                 x = -x;
9755         }
9756         pos |= x << CURSOR_X_SHIFT;
9757
9758         if (y < 0) {
9759                 pos |= CURSOR_POS_SIGN << CURSOR_Y_SHIFT;
9760                 y = -y;
9761         }
9762         pos |= y << CURSOR_Y_SHIFT;
9763
9764         return pos;
9765 }
9766
9767 static bool intel_cursor_size_ok(const struct intel_plane_state *plane_state)
9768 {
9769         const struct drm_mode_config *config =
9770                 &plane_state->base.plane->dev->mode_config;
9771         int width = plane_state->base.crtc_w;
9772         int height = plane_state->base.crtc_h;
9773
9774         return width > 0 && width <= config->cursor_width &&
9775                 height > 0 && height <= config->cursor_height;
9776 }
9777
9778 static int intel_cursor_check_surface(struct intel_plane_state *plane_state)
9779 {
9780         const struct drm_framebuffer *fb = plane_state->base.fb;
9781         unsigned int rotation = plane_state->base.rotation;
9782         int src_x, src_y;
9783         u32 offset;
9784         int ret;
9785
9786         intel_fill_fb_ggtt_view(&plane_state->view, fb, rotation);
9787         plane_state->color_plane[0].stride = intel_fb_pitch(fb, 0, rotation);
9788
9789         ret = intel_plane_check_stride(plane_state);
9790         if (ret)
9791                 return ret;
9792
9793         src_x = plane_state->base.src_x >> 16;
9794         src_y = plane_state->base.src_y >> 16;
9795
9796         intel_add_fb_offsets(&src_x, &src_y, plane_state, 0);
9797         offset = intel_plane_compute_aligned_offset(&src_x, &src_y,
9798                                                     plane_state, 0);
9799
9800         if (src_x != 0 || src_y != 0) {
9801                 DRM_DEBUG_KMS("Arbitrary cursor panning not supported\n");
9802                 return -EINVAL;
9803         }
9804
9805         plane_state->color_plane[0].offset = offset;
9806
9807         return 0;
9808 }
9809
9810 static int intel_check_cursor(struct intel_crtc_state *crtc_state,
9811                               struct intel_plane_state *plane_state)
9812 {
9813         const struct drm_framebuffer *fb = plane_state->base.fb;
9814         int ret;
9815
9816         if (fb && fb->modifier != DRM_FORMAT_MOD_LINEAR) {
9817                 DRM_DEBUG_KMS("cursor cannot be tiled\n");
9818                 return -EINVAL;
9819         }
9820
9821         ret = drm_atomic_helper_check_plane_state(&plane_state->base,
9822                                                   &crtc_state->base,
9823                                                   DRM_PLANE_HELPER_NO_SCALING,
9824                                                   DRM_PLANE_HELPER_NO_SCALING,
9825                                                   true, true);
9826         if (ret)
9827                 return ret;
9828
9829         if (!plane_state->base.visible)
9830                 return 0;
9831
9832         ret = intel_plane_check_src_coordinates(plane_state);
9833         if (ret)
9834                 return ret;
9835
9836         ret = intel_cursor_check_surface(plane_state);
9837         if (ret)
9838                 return ret;
9839
9840         return 0;
9841 }
9842
9843 static unsigned int
9844 i845_cursor_max_stride(struct intel_plane *plane,
9845                        u32 pixel_format, u64 modifier,
9846                        unsigned int rotation)
9847 {
9848         return 2048;
9849 }
9850
9851 static u32 i845_cursor_ctl(const struct intel_crtc_state *crtc_state,
9852                            const struct intel_plane_state *plane_state)
9853 {
9854         return CURSOR_ENABLE |
9855                 CURSOR_GAMMA_ENABLE |
9856                 CURSOR_FORMAT_ARGB |
9857                 CURSOR_STRIDE(plane_state->color_plane[0].stride);
9858 }
9859
9860 static bool i845_cursor_size_ok(const struct intel_plane_state *plane_state)
9861 {
9862         int width = plane_state->base.crtc_w;
9863
9864         /*
9865          * 845g/865g are only limited by the width of their cursors,
9866          * the height is arbitrary up to the precision of the register.
9867          */
9868         return intel_cursor_size_ok(plane_state) && IS_ALIGNED(width, 64);
9869 }
9870
9871 static int i845_check_cursor(struct intel_crtc_state *crtc_state,
9872                              struct intel_plane_state *plane_state)
9873 {
9874         const struct drm_framebuffer *fb = plane_state->base.fb;
9875         int ret;
9876
9877         ret = intel_check_cursor(crtc_state, plane_state);
9878         if (ret)
9879                 return ret;
9880
9881         /* if we want to turn off the cursor ignore width and height */
9882         if (!fb)
9883                 return 0;
9884
9885         /* Check for which cursor types we support */
9886         if (!i845_cursor_size_ok(plane_state)) {
9887                 DRM_DEBUG("Cursor dimension %dx%d not supported\n",
9888                           plane_state->base.crtc_w,
9889                           plane_state->base.crtc_h);
9890                 return -EINVAL;
9891         }
9892
9893         WARN_ON(plane_state->base.visible &&
9894                 plane_state->color_plane[0].stride != fb->pitches[0]);
9895
9896         switch (fb->pitches[0]) {
9897         case 256:
9898         case 512:
9899         case 1024:
9900         case 2048:
9901                 break;
9902         default:
9903                 DRM_DEBUG_KMS("Invalid cursor stride (%u)\n",
9904                               fb->pitches[0]);
9905                 return -EINVAL;
9906         }
9907
9908         plane_state->ctl = i845_cursor_ctl(crtc_state, plane_state);
9909
9910         return 0;
9911 }
9912
9913 static void i845_update_cursor(struct intel_plane *plane,
9914                                const struct intel_crtc_state *crtc_state,
9915                                const struct intel_plane_state *plane_state)
9916 {
9917         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
9918         u32 cntl = 0, base = 0, pos = 0, size = 0;
9919         unsigned long irqflags;
9920
9921         if (plane_state && plane_state->base.visible) {
9922                 unsigned int width = plane_state->base.crtc_w;
9923                 unsigned int height = plane_state->base.crtc_h;
9924
9925                 cntl = plane_state->ctl;
9926                 size = (height << 12) | width;
9927
9928                 base = intel_cursor_base(plane_state);
9929                 pos = intel_cursor_position(plane_state);
9930         }
9931
9932         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
9933
9934         /* On these chipsets we can only modify the base/size/stride
9935          * whilst the cursor is disabled.
9936          */
9937         if (plane->cursor.base != base ||
9938             plane->cursor.size != size ||
9939             plane->cursor.cntl != cntl) {
9940                 I915_WRITE_FW(CURCNTR(PIPE_A), 0);
9941                 I915_WRITE_FW(CURBASE(PIPE_A), base);
9942                 I915_WRITE_FW(CURSIZE, size);
9943                 I915_WRITE_FW(CURPOS(PIPE_A), pos);
9944                 I915_WRITE_FW(CURCNTR(PIPE_A), cntl);
9945
9946                 plane->cursor.base = base;
9947                 plane->cursor.size = size;
9948                 plane->cursor.cntl = cntl;
9949         } else {
9950                 I915_WRITE_FW(CURPOS(PIPE_A), pos);
9951         }
9952
9953         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
9954 }
9955
9956 static void i845_disable_cursor(struct intel_plane *plane,
9957                                 const struct intel_crtc_state *crtc_state)
9958 {
9959         i845_update_cursor(plane, crtc_state, NULL);
9960 }
9961
9962 static bool i845_cursor_get_hw_state(struct intel_plane *plane,
9963                                      enum pipe *pipe)
9964 {
9965         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
9966         enum intel_display_power_domain power_domain;
9967         bool ret;
9968
9969         power_domain = POWER_DOMAIN_PIPE(PIPE_A);
9970         if (!intel_display_power_get_if_enabled(dev_priv, power_domain))
9971                 return false;
9972
9973         ret = I915_READ(CURCNTR(PIPE_A)) & CURSOR_ENABLE;
9974
9975         *pipe = PIPE_A;
9976
9977         intel_display_power_put(dev_priv, power_domain);
9978
9979         return ret;
9980 }
9981
9982 static unsigned int
9983 i9xx_cursor_max_stride(struct intel_plane *plane,
9984                        u32 pixel_format, u64 modifier,
9985                        unsigned int rotation)
9986 {
9987         return plane->base.dev->mode_config.cursor_width * 4;
9988 }
9989
9990 static u32 i9xx_cursor_ctl(const struct intel_crtc_state *crtc_state,
9991                            const struct intel_plane_state *plane_state)
9992 {
9993         struct drm_i915_private *dev_priv =
9994                 to_i915(plane_state->base.plane->dev);
9995         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
9996         u32 cntl = 0;
9997
9998         if (IS_GEN6(dev_priv) || IS_IVYBRIDGE(dev_priv))
9999                 cntl |= MCURSOR_TRICKLE_FEED_DISABLE;
10000
10001         if (INTEL_GEN(dev_priv) <= 10) {
10002                 cntl |= MCURSOR_GAMMA_ENABLE;
10003
10004                 if (HAS_DDI(dev_priv))
10005                         cntl |= MCURSOR_PIPE_CSC_ENABLE;
10006         }
10007
10008         if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv))
10009                 cntl |= MCURSOR_PIPE_SELECT(crtc->pipe);
10010
10011         switch (plane_state->base.crtc_w) {
10012         case 64:
10013                 cntl |= MCURSOR_MODE_64_ARGB_AX;
10014                 break;
10015         case 128:
10016                 cntl |= MCURSOR_MODE_128_ARGB_AX;
10017                 break;
10018         case 256:
10019                 cntl |= MCURSOR_MODE_256_ARGB_AX;
10020                 break;
10021         default:
10022                 MISSING_CASE(plane_state->base.crtc_w);
10023                 return 0;
10024         }
10025
10026         if (plane_state->base.rotation & DRM_MODE_ROTATE_180)
10027                 cntl |= MCURSOR_ROTATE_180;
10028
10029         return cntl;
10030 }
10031
10032 static bool i9xx_cursor_size_ok(const struct intel_plane_state *plane_state)
10033 {
10034         struct drm_i915_private *dev_priv =
10035                 to_i915(plane_state->base.plane->dev);
10036         int width = plane_state->base.crtc_w;
10037         int height = plane_state->base.crtc_h;
10038
10039         if (!intel_cursor_size_ok(plane_state))
10040                 return false;
10041
10042         /* Cursor width is limited to a few power-of-two sizes */
10043         switch (width) {
10044         case 256:
10045         case 128:
10046         case 64:
10047                 break;
10048         default:
10049                 return false;
10050         }
10051
10052         /*
10053          * IVB+ have CUR_FBC_CTL which allows an arbitrary cursor
10054          * height from 8 lines up to the cursor width, when the
10055          * cursor is not rotated. Everything else requires square
10056          * cursors.
10057          */
10058         if (HAS_CUR_FBC(dev_priv) &&
10059             plane_state->base.rotation & DRM_MODE_ROTATE_0) {
10060                 if (height < 8 || height > width)
10061                         return false;
10062         } else {
10063                 if (height != width)
10064                         return false;
10065         }
10066
10067         return true;
10068 }
10069
10070 static int i9xx_check_cursor(struct intel_crtc_state *crtc_state,
10071                              struct intel_plane_state *plane_state)
10072 {
10073         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
10074         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
10075         const struct drm_framebuffer *fb = plane_state->base.fb;
10076         enum pipe pipe = plane->pipe;
10077         int ret;
10078
10079         ret = intel_check_cursor(crtc_state, plane_state);
10080         if (ret)
10081                 return ret;
10082
10083         /* if we want to turn off the cursor ignore width and height */
10084         if (!fb)
10085                 return 0;
10086
10087         /* Check for which cursor types we support */
10088         if (!i9xx_cursor_size_ok(plane_state)) {
10089                 DRM_DEBUG("Cursor dimension %dx%d not supported\n",
10090                           plane_state->base.crtc_w,
10091                           plane_state->base.crtc_h);
10092                 return -EINVAL;
10093         }
10094
10095         WARN_ON(plane_state->base.visible &&
10096                 plane_state->color_plane[0].stride != fb->pitches[0]);
10097
10098         if (fb->pitches[0] != plane_state->base.crtc_w * fb->format->cpp[0]) {
10099                 DRM_DEBUG_KMS("Invalid cursor stride (%u) (cursor width %d)\n",
10100                               fb->pitches[0], plane_state->base.crtc_w);
10101                 return -EINVAL;
10102         }
10103
10104         /*
10105          * There's something wrong with the cursor on CHV pipe C.
10106          * If it straddles the left edge of the screen then
10107          * moving it away from the edge or disabling it often
10108          * results in a pipe underrun, and often that can lead to
10109          * dead pipe (constant underrun reported, and it scans
10110          * out just a solid color). To recover from that, the
10111          * display power well must be turned off and on again.
10112          * Refuse the put the cursor into that compromised position.
10113          */
10114         if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_C &&
10115             plane_state->base.visible && plane_state->base.crtc_x < 0) {
10116                 DRM_DEBUG_KMS("CHV cursor C not allowed to straddle the left screen edge\n");
10117                 return -EINVAL;
10118         }
10119
10120         plane_state->ctl = i9xx_cursor_ctl(crtc_state, plane_state);
10121
10122         return 0;
10123 }
10124
10125 static void i9xx_update_cursor(struct intel_plane *plane,
10126                                const struct intel_crtc_state *crtc_state,
10127                                const struct intel_plane_state *plane_state)
10128 {
10129         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
10130         enum pipe pipe = plane->pipe;
10131         u32 cntl = 0, base = 0, pos = 0, fbc_ctl = 0;
10132         unsigned long irqflags;
10133
10134         if (plane_state && plane_state->base.visible) {
10135                 cntl = plane_state->ctl;
10136
10137                 if (plane_state->base.crtc_h != plane_state->base.crtc_w)
10138                         fbc_ctl = CUR_FBC_CTL_EN | (plane_state->base.crtc_h - 1);
10139
10140                 base = intel_cursor_base(plane_state);
10141                 pos = intel_cursor_position(plane_state);
10142         }
10143
10144         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
10145
10146         /*
10147          * On some platforms writing CURCNTR first will also
10148          * cause CURPOS to be armed by the CURBASE write.
10149          * Without the CURCNTR write the CURPOS write would
10150          * arm itself. Thus we always update CURCNTR before
10151          * CURPOS.
10152          *
10153          * On other platforms CURPOS always requires the
10154          * CURBASE write to arm the update. Additonally
10155          * a write to any of the cursor register will cancel
10156          * an already armed cursor update. Thus leaving out
10157          * the CURBASE write after CURPOS could lead to a
10158          * cursor that doesn't appear to move, or even change
10159          * shape. Thus we always write CURBASE.
10160          *
10161          * The other registers are armed by by the CURBASE write
10162          * except when the plane is getting enabled at which time
10163          * the CURCNTR write arms the update.
10164          */
10165
10166         if (INTEL_GEN(dev_priv) >= 9)
10167                 skl_write_cursor_wm(plane, crtc_state);
10168
10169         if (plane->cursor.base != base ||
10170             plane->cursor.size != fbc_ctl ||
10171             plane->cursor.cntl != cntl) {
10172                 if (HAS_CUR_FBC(dev_priv))
10173                         I915_WRITE_FW(CUR_FBC_CTL(pipe), fbc_ctl);
10174                 I915_WRITE_FW(CURCNTR(pipe), cntl);
10175                 I915_WRITE_FW(CURPOS(pipe), pos);
10176                 I915_WRITE_FW(CURBASE(pipe), base);
10177
10178                 plane->cursor.base = base;
10179                 plane->cursor.size = fbc_ctl;
10180                 plane->cursor.cntl = cntl;
10181         } else {
10182                 I915_WRITE_FW(CURPOS(pipe), pos);
10183                 I915_WRITE_FW(CURBASE(pipe), base);
10184         }
10185
10186         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
10187 }
10188
10189 static void i9xx_disable_cursor(struct intel_plane *plane,
10190                                 const struct intel_crtc_state *crtc_state)
10191 {
10192         i9xx_update_cursor(plane, crtc_state, NULL);
10193 }
10194
10195 static bool i9xx_cursor_get_hw_state(struct intel_plane *plane,
10196                                      enum pipe *pipe)
10197 {
10198         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
10199         enum intel_display_power_domain power_domain;
10200         bool ret;
10201         u32 val;
10202
10203         /*
10204          * Not 100% correct for planes that can move between pipes,
10205          * but that's only the case for gen2-3 which don't have any
10206          * display power wells.
10207          */
10208         power_domain = POWER_DOMAIN_PIPE(plane->pipe);
10209         if (!intel_display_power_get_if_enabled(dev_priv, power_domain))
10210                 return false;
10211
10212         val = I915_READ(CURCNTR(plane->pipe));
10213
10214         ret = val & MCURSOR_MODE;
10215
10216         if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv))
10217                 *pipe = plane->pipe;
10218         else
10219                 *pipe = (val & MCURSOR_PIPE_SELECT_MASK) >>
10220                         MCURSOR_PIPE_SELECT_SHIFT;
10221
10222         intel_display_power_put(dev_priv, power_domain);
10223
10224         return ret;
10225 }
10226
10227 /* VESA 640x480x72Hz mode to set on the pipe */
10228 static const struct drm_display_mode load_detect_mode = {
10229         DRM_MODE("640x480", DRM_MODE_TYPE_DEFAULT, 31500, 640, 664,
10230                  704, 832, 0, 480, 489, 491, 520, 0, DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
10231 };
10232
10233 struct drm_framebuffer *
10234 intel_framebuffer_create(struct drm_i915_gem_object *obj,
10235                          struct drm_mode_fb_cmd2 *mode_cmd)
10236 {
10237         struct intel_framebuffer *intel_fb;
10238         int ret;
10239
10240         intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL);
10241         if (!intel_fb)
10242                 return ERR_PTR(-ENOMEM);
10243
10244         ret = intel_framebuffer_init(intel_fb, obj, mode_cmd);
10245         if (ret)
10246                 goto err;
10247
10248         return &intel_fb->base;
10249
10250 err:
10251         kfree(intel_fb);
10252         return ERR_PTR(ret);
10253 }
10254
10255 static int intel_modeset_disable_planes(struct drm_atomic_state *state,
10256                                         struct drm_crtc *crtc)
10257 {
10258         struct drm_plane *plane;
10259         struct drm_plane_state *plane_state;
10260         int ret, i;
10261
10262         ret = drm_atomic_add_affected_planes(state, crtc);
10263         if (ret)
10264                 return ret;
10265
10266         for_each_new_plane_in_state(state, plane, plane_state, i) {
10267                 if (plane_state->crtc != crtc)
10268                         continue;
10269
10270                 ret = drm_atomic_set_crtc_for_plane(plane_state, NULL);
10271                 if (ret)
10272                         return ret;
10273
10274                 drm_atomic_set_fb_for_plane(plane_state, NULL);
10275         }
10276
10277         return 0;
10278 }
10279
10280 int intel_get_load_detect_pipe(struct drm_connector *connector,
10281                                const struct drm_display_mode *mode,
10282                                struct intel_load_detect_pipe *old,
10283                                struct drm_modeset_acquire_ctx *ctx)
10284 {
10285         struct intel_crtc *intel_crtc;
10286         struct intel_encoder *intel_encoder =
10287                 intel_attached_encoder(connector);
10288         struct drm_crtc *possible_crtc;
10289         struct drm_encoder *encoder = &intel_encoder->base;
10290         struct drm_crtc *crtc = NULL;
10291         struct drm_device *dev = encoder->dev;
10292         struct drm_i915_private *dev_priv = to_i915(dev);
10293         struct drm_mode_config *config = &dev->mode_config;
10294         struct drm_atomic_state *state = NULL, *restore_state = NULL;
10295         struct drm_connector_state *connector_state;
10296         struct intel_crtc_state *crtc_state;
10297         int ret, i = -1;
10298
10299         DRM_DEBUG_KMS("[CONNECTOR:%d:%s], [ENCODER:%d:%s]\n",
10300                       connector->base.id, connector->name,
10301                       encoder->base.id, encoder->name);
10302
10303         old->restore_state = NULL;
10304
10305         WARN_ON(!drm_modeset_is_locked(&config->connection_mutex));
10306
10307         /*
10308          * Algorithm gets a little messy:
10309          *
10310          *   - if the connector already has an assigned crtc, use it (but make
10311          *     sure it's on first)
10312          *
10313          *   - try to find the first unused crtc that can drive this connector,
10314          *     and use that if we find one
10315          */
10316
10317         /* See if we already have a CRTC for this connector */
10318         if (connector->state->crtc) {
10319                 crtc = connector->state->crtc;
10320
10321                 ret = drm_modeset_lock(&crtc->mutex, ctx);
10322                 if (ret)
10323                         goto fail;
10324
10325                 /* Make sure the crtc and connector are running */
10326                 goto found;
10327         }
10328
10329         /* Find an unused one (if possible) */
10330         for_each_crtc(dev, possible_crtc) {
10331                 i++;
10332                 if (!(encoder->possible_crtcs & (1 << i)))
10333                         continue;
10334
10335                 ret = drm_modeset_lock(&possible_crtc->mutex, ctx);
10336                 if (ret)
10337                         goto fail;
10338
10339                 if (possible_crtc->state->enable) {
10340                         drm_modeset_unlock(&possible_crtc->mutex);
10341                         continue;
10342                 }
10343
10344                 crtc = possible_crtc;
10345                 break;
10346         }
10347
10348         /*
10349          * If we didn't find an unused CRTC, don't use any.
10350          */
10351         if (!crtc) {
10352                 DRM_DEBUG_KMS("no pipe available for load-detect\n");
10353                 ret = -ENODEV;
10354                 goto fail;
10355         }
10356
10357 found:
10358         intel_crtc = to_intel_crtc(crtc);
10359
10360         state = drm_atomic_state_alloc(dev);
10361         restore_state = drm_atomic_state_alloc(dev);
10362         if (!state || !restore_state) {
10363                 ret = -ENOMEM;
10364                 goto fail;
10365         }
10366
10367         state->acquire_ctx = ctx;
10368         restore_state->acquire_ctx = ctx;
10369
10370         connector_state = drm_atomic_get_connector_state(state, connector);
10371         if (IS_ERR(connector_state)) {
10372                 ret = PTR_ERR(connector_state);
10373                 goto fail;
10374         }
10375
10376         ret = drm_atomic_set_crtc_for_connector(connector_state, crtc);
10377         if (ret)
10378                 goto fail;
10379
10380         crtc_state = intel_atomic_get_crtc_state(state, intel_crtc);
10381         if (IS_ERR(crtc_state)) {
10382                 ret = PTR_ERR(crtc_state);
10383                 goto fail;
10384         }
10385
10386         crtc_state->base.active = crtc_state->base.enable = true;
10387
10388         if (!mode)
10389                 mode = &load_detect_mode;
10390
10391         ret = drm_atomic_set_mode_for_crtc(&crtc_state->base, mode);
10392         if (ret)
10393                 goto fail;
10394
10395         ret = intel_modeset_disable_planes(state, crtc);
10396         if (ret)
10397                 goto fail;
10398
10399         ret = PTR_ERR_OR_ZERO(drm_atomic_get_connector_state(restore_state, connector));
10400         if (!ret)
10401                 ret = PTR_ERR_OR_ZERO(drm_atomic_get_crtc_state(restore_state, crtc));
10402         if (!ret)
10403                 ret = drm_atomic_add_affected_planes(restore_state, crtc);
10404         if (ret) {
10405                 DRM_DEBUG_KMS("Failed to create a copy of old state to restore: %i\n", ret);
10406                 goto fail;
10407         }
10408
10409         ret = drm_atomic_commit(state);
10410         if (ret) {
10411                 DRM_DEBUG_KMS("failed to set mode on load-detect pipe\n");
10412                 goto fail;
10413         }
10414
10415         old->restore_state = restore_state;
10416         drm_atomic_state_put(state);
10417
10418         /* let the connector get through one full cycle before testing */
10419         intel_wait_for_vblank(dev_priv, intel_crtc->pipe);
10420         return true;
10421
10422 fail:
10423         if (state) {
10424                 drm_atomic_state_put(state);
10425                 state = NULL;
10426         }
10427         if (restore_state) {
10428                 drm_atomic_state_put(restore_state);
10429                 restore_state = NULL;
10430         }
10431
10432         if (ret == -EDEADLK)
10433                 return ret;
10434
10435         return false;
10436 }
10437
10438 void intel_release_load_detect_pipe(struct drm_connector *connector,
10439                                     struct intel_load_detect_pipe *old,
10440                                     struct drm_modeset_acquire_ctx *ctx)
10441 {
10442         struct intel_encoder *intel_encoder =
10443                 intel_attached_encoder(connector);
10444         struct drm_encoder *encoder = &intel_encoder->base;
10445         struct drm_atomic_state *state = old->restore_state;
10446         int ret;
10447
10448         DRM_DEBUG_KMS("[CONNECTOR:%d:%s], [ENCODER:%d:%s]\n",
10449                       connector->base.id, connector->name,
10450                       encoder->base.id, encoder->name);
10451
10452         if (!state)
10453                 return;
10454
10455         ret = drm_atomic_helper_commit_duplicated_state(state, ctx);
10456         if (ret)
10457                 DRM_DEBUG_KMS("Couldn't release load detect pipe: %i\n", ret);
10458         drm_atomic_state_put(state);
10459 }
10460
10461 static int i9xx_pll_refclk(struct drm_device *dev,
10462                            const struct intel_crtc_state *pipe_config)
10463 {
10464         struct drm_i915_private *dev_priv = to_i915(dev);
10465         u32 dpll = pipe_config->dpll_hw_state.dpll;
10466
10467         if ((dpll & PLL_REF_INPUT_MASK) == PLLB_REF_INPUT_SPREADSPECTRUMIN)
10468                 return dev_priv->vbt.lvds_ssc_freq;
10469         else if (HAS_PCH_SPLIT(dev_priv))
10470                 return 120000;
10471         else if (!IS_GEN2(dev_priv))
10472                 return 96000;
10473         else
10474                 return 48000;
10475 }
10476
10477 /* Returns the clock of the currently programmed mode of the given pipe. */
10478 static void i9xx_crtc_clock_get(struct intel_crtc *crtc,
10479                                 struct intel_crtc_state *pipe_config)
10480 {
10481         struct drm_device *dev = crtc->base.dev;
10482         struct drm_i915_private *dev_priv = to_i915(dev);
10483         int pipe = pipe_config->cpu_transcoder;
10484         u32 dpll = pipe_config->dpll_hw_state.dpll;
10485         u32 fp;
10486         struct dpll clock;
10487         int port_clock;
10488         int refclk = i9xx_pll_refclk(dev, pipe_config);
10489
10490         if ((dpll & DISPLAY_RATE_SELECT_FPA1) == 0)
10491                 fp = pipe_config->dpll_hw_state.fp0;
10492         else
10493                 fp = pipe_config->dpll_hw_state.fp1;
10494
10495         clock.m1 = (fp & FP_M1_DIV_MASK) >> FP_M1_DIV_SHIFT;
10496         if (IS_PINEVIEW(dev_priv)) {
10497                 clock.n = ffs((fp & FP_N_PINEVIEW_DIV_MASK) >> FP_N_DIV_SHIFT) - 1;
10498                 clock.m2 = (fp & FP_M2_PINEVIEW_DIV_MASK) >> FP_M2_DIV_SHIFT;
10499         } else {
10500                 clock.n = (fp & FP_N_DIV_MASK) >> FP_N_DIV_SHIFT;
10501                 clock.m2 = (fp & FP_M2_DIV_MASK) >> FP_M2_DIV_SHIFT;
10502         }
10503
10504         if (!IS_GEN2(dev_priv)) {
10505                 if (IS_PINEVIEW(dev_priv))
10506                         clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK_PINEVIEW) >>
10507                                 DPLL_FPA01_P1_POST_DIV_SHIFT_PINEVIEW);
10508                 else
10509                         clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK) >>
10510                                DPLL_FPA01_P1_POST_DIV_SHIFT);
10511
10512                 switch (dpll & DPLL_MODE_MASK) {
10513                 case DPLLB_MODE_DAC_SERIAL:
10514                         clock.p2 = dpll & DPLL_DAC_SERIAL_P2_CLOCK_DIV_5 ?
10515                                 5 : 10;
10516                         break;
10517                 case DPLLB_MODE_LVDS:
10518                         clock.p2 = dpll & DPLLB_LVDS_P2_CLOCK_DIV_7 ?
10519                                 7 : 14;
10520                         break;
10521                 default:
10522                         DRM_DEBUG_KMS("Unknown DPLL mode %08x in programmed "
10523                                   "mode\n", (int)(dpll & DPLL_MODE_MASK));
10524                         return;
10525                 }
10526
10527                 if (IS_PINEVIEW(dev_priv))
10528                         port_clock = pnv_calc_dpll_params(refclk, &clock);
10529                 else
10530                         port_clock = i9xx_calc_dpll_params(refclk, &clock);
10531         } else {
10532                 u32 lvds = IS_I830(dev_priv) ? 0 : I915_READ(LVDS);
10533                 bool is_lvds = (pipe == 1) && (lvds & LVDS_PORT_EN);
10534
10535                 if (is_lvds) {
10536                         clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK_I830_LVDS) >>
10537                                        DPLL_FPA01_P1_POST_DIV_SHIFT);
10538
10539                         if (lvds & LVDS_CLKB_POWER_UP)
10540                                 clock.p2 = 7;
10541                         else
10542                                 clock.p2 = 14;
10543                 } else {
10544                         if (dpll & PLL_P1_DIVIDE_BY_TWO)
10545                                 clock.p1 = 2;
10546                         else {
10547                                 clock.p1 = ((dpll & DPLL_FPA01_P1_POST_DIV_MASK_I830) >>
10548                                             DPLL_FPA01_P1_POST_DIV_SHIFT) + 2;
10549                         }
10550                         if (dpll & PLL_P2_DIVIDE_BY_4)
10551                                 clock.p2 = 4;
10552                         else
10553                                 clock.p2 = 2;
10554                 }
10555
10556                 port_clock = i9xx_calc_dpll_params(refclk, &clock);
10557         }
10558
10559         /*
10560          * This value includes pixel_multiplier. We will use
10561          * port_clock to compute adjusted_mode.crtc_clock in the
10562          * encoder's get_config() function.
10563          */
10564         pipe_config->port_clock = port_clock;
10565 }
10566
10567 int intel_dotclock_calculate(int link_freq,
10568                              const struct intel_link_m_n *m_n)
10569 {
10570         /*
10571          * The calculation for the data clock is:
10572          * pixel_clock = ((m/n)*(link_clock * nr_lanes))/bpp
10573          * But we want to avoid losing precison if possible, so:
10574          * pixel_clock = ((m * link_clock * nr_lanes)/(n*bpp))
10575          *
10576          * and the link clock is simpler:
10577          * link_clock = (m * link_clock) / n
10578          */
10579
10580         if (!m_n->link_n)
10581                 return 0;
10582
10583         return div_u64(mul_u32_u32(m_n->link_m, link_freq), m_n->link_n);
10584 }
10585
10586 static void ironlake_pch_clock_get(struct intel_crtc *crtc,
10587                                    struct intel_crtc_state *pipe_config)
10588 {
10589         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
10590
10591         /* read out port_clock from the DPLL */
10592         i9xx_crtc_clock_get(crtc, pipe_config);
10593
10594         /*
10595          * In case there is an active pipe without active ports,
10596          * we may need some idea for the dotclock anyway.
10597          * Calculate one based on the FDI configuration.
10598          */
10599         pipe_config->base.adjusted_mode.crtc_clock =
10600                 intel_dotclock_calculate(intel_fdi_link_freq(dev_priv, pipe_config),
10601                                          &pipe_config->fdi_m_n);
10602 }
10603
10604 /* Returns the currently programmed mode of the given encoder. */
10605 struct drm_display_mode *
10606 intel_encoder_current_mode(struct intel_encoder *encoder)
10607 {
10608         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
10609         struct intel_crtc_state *crtc_state;
10610         struct drm_display_mode *mode;
10611         struct intel_crtc *crtc;
10612         enum pipe pipe;
10613
10614         if (!encoder->get_hw_state(encoder, &pipe))
10615                 return NULL;
10616
10617         crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
10618
10619         mode = kzalloc(sizeof(*mode), GFP_KERNEL);
10620         if (!mode)
10621                 return NULL;
10622
10623         crtc_state = kzalloc(sizeof(*crtc_state), GFP_KERNEL);
10624         if (!crtc_state) {
10625                 kfree(mode);
10626                 return NULL;
10627         }
10628
10629         crtc_state->base.crtc = &crtc->base;
10630
10631         if (!dev_priv->display.get_pipe_config(crtc, crtc_state)) {
10632                 kfree(crtc_state);
10633                 kfree(mode);
10634                 return NULL;
10635         }
10636
10637         encoder->get_config(encoder, crtc_state);
10638
10639         intel_mode_from_pipe_config(mode, crtc_state);
10640
10641         kfree(crtc_state);
10642
10643         return mode;
10644 }
10645
10646 static void intel_crtc_destroy(struct drm_crtc *crtc)
10647 {
10648         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
10649
10650         drm_crtc_cleanup(crtc);
10651         kfree(intel_crtc);
10652 }
10653
10654 /**
10655  * intel_wm_need_update - Check whether watermarks need updating
10656  * @plane: drm plane
10657  * @state: new plane state
10658  *
10659  * Check current plane state versus the new one to determine whether
10660  * watermarks need to be recalculated.
10661  *
10662  * Returns true or false.
10663  */
10664 static bool intel_wm_need_update(struct drm_plane *plane,
10665                                  struct drm_plane_state *state)
10666 {
10667         struct intel_plane_state *new = to_intel_plane_state(state);
10668         struct intel_plane_state *cur = to_intel_plane_state(plane->state);
10669
10670         /* Update watermarks on tiling or size changes. */
10671         if (new->base.visible != cur->base.visible)
10672                 return true;
10673
10674         if (!cur->base.fb || !new->base.fb)
10675                 return false;
10676
10677         if (cur->base.fb->modifier != new->base.fb->modifier ||
10678             cur->base.rotation != new->base.rotation ||
10679             drm_rect_width(&new->base.src) != drm_rect_width(&cur->base.src) ||
10680             drm_rect_height(&new->base.src) != drm_rect_height(&cur->base.src) ||
10681             drm_rect_width(&new->base.dst) != drm_rect_width(&cur->base.dst) ||
10682             drm_rect_height(&new->base.dst) != drm_rect_height(&cur->base.dst))
10683                 return true;
10684
10685         return false;
10686 }
10687
10688 static bool needs_scaling(const struct intel_plane_state *state)
10689 {
10690         int src_w = drm_rect_width(&state->base.src) >> 16;
10691         int src_h = drm_rect_height(&state->base.src) >> 16;
10692         int dst_w = drm_rect_width(&state->base.dst);
10693         int dst_h = drm_rect_height(&state->base.dst);
10694
10695         return (src_w != dst_w || src_h != dst_h);
10696 }
10697
10698 int intel_plane_atomic_calc_changes(const struct intel_crtc_state *old_crtc_state,
10699                                     struct drm_crtc_state *crtc_state,
10700                                     const struct intel_plane_state *old_plane_state,
10701                                     struct drm_plane_state *plane_state)
10702 {
10703         struct intel_crtc_state *pipe_config = to_intel_crtc_state(crtc_state);
10704         struct drm_crtc *crtc = crtc_state->crtc;
10705         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
10706         struct intel_plane *plane = to_intel_plane(plane_state->plane);
10707         struct drm_device *dev = crtc->dev;
10708         struct drm_i915_private *dev_priv = to_i915(dev);
10709         bool mode_changed = needs_modeset(crtc_state);
10710         bool was_crtc_enabled = old_crtc_state->base.active;
10711         bool is_crtc_enabled = crtc_state->active;
10712         bool turn_off, turn_on, visible, was_visible;
10713         struct drm_framebuffer *fb = plane_state->fb;
10714         int ret;
10715
10716         if (INTEL_GEN(dev_priv) >= 9 && plane->id != PLANE_CURSOR) {
10717                 ret = skl_update_scaler_plane(
10718                         to_intel_crtc_state(crtc_state),
10719                         to_intel_plane_state(plane_state));
10720                 if (ret)
10721                         return ret;
10722         }
10723
10724         was_visible = old_plane_state->base.visible;
10725         visible = plane_state->visible;
10726
10727         if (!was_crtc_enabled && WARN_ON(was_visible))
10728                 was_visible = false;
10729
10730         /*
10731          * Visibility is calculated as if the crtc was on, but
10732          * after scaler setup everything depends on it being off
10733          * when the crtc isn't active.
10734          *
10735          * FIXME this is wrong for watermarks. Watermarks should also
10736          * be computed as if the pipe would be active. Perhaps move
10737          * per-plane wm computation to the .check_plane() hook, and
10738          * only combine the results from all planes in the current place?
10739          */
10740         if (!is_crtc_enabled) {
10741                 plane_state->visible = visible = false;
10742                 to_intel_crtc_state(crtc_state)->active_planes &= ~BIT(plane->id);
10743         }
10744
10745         if (!was_visible && !visible)
10746                 return 0;
10747
10748         if (fb != old_plane_state->base.fb)
10749                 pipe_config->fb_changed = true;
10750
10751         turn_off = was_visible && (!visible || mode_changed);
10752         turn_on = visible && (!was_visible || mode_changed);
10753
10754         DRM_DEBUG_ATOMIC("[CRTC:%d:%s] has [PLANE:%d:%s] with fb %i\n",
10755                          intel_crtc->base.base.id, intel_crtc->base.name,
10756                          plane->base.base.id, plane->base.name,
10757                          fb ? fb->base.id : -1);
10758
10759         DRM_DEBUG_ATOMIC("[PLANE:%d:%s] visible %i -> %i, off %i, on %i, ms %i\n",
10760                          plane->base.base.id, plane->base.name,
10761                          was_visible, visible,
10762                          turn_off, turn_on, mode_changed);
10763
10764         if (turn_on) {
10765                 if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv))
10766                         pipe_config->update_wm_pre = true;
10767
10768                 /* must disable cxsr around plane enable/disable */
10769                 if (plane->id != PLANE_CURSOR)
10770                         pipe_config->disable_cxsr = true;
10771         } else if (turn_off) {
10772                 if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv))
10773                         pipe_config->update_wm_post = true;
10774
10775                 /* must disable cxsr around plane enable/disable */
10776                 if (plane->id != PLANE_CURSOR)
10777                         pipe_config->disable_cxsr = true;
10778         } else if (intel_wm_need_update(&plane->base, plane_state)) {
10779                 if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv)) {
10780                         /* FIXME bollocks */
10781                         pipe_config->update_wm_pre = true;
10782                         pipe_config->update_wm_post = true;
10783                 }
10784         }
10785
10786         if (visible || was_visible)
10787                 pipe_config->fb_bits |= plane->frontbuffer_bit;
10788
10789         /*
10790          * ILK/SNB DVSACNTR/Sprite Enable
10791          * IVB SPR_CTL/Sprite Enable
10792          * "When in Self Refresh Big FIFO mode, a write to enable the
10793          *  plane will be internally buffered and delayed while Big FIFO
10794          *  mode is exiting."
10795          *
10796          * Which means that enabling the sprite can take an extra frame
10797          * when we start in big FIFO mode (LP1+). Thus we need to drop
10798          * down to LP0 and wait for vblank in order to make sure the
10799          * sprite gets enabled on the next vblank after the register write.
10800          * Doing otherwise would risk enabling the sprite one frame after
10801          * we've already signalled flip completion. We can resume LP1+
10802          * once the sprite has been enabled.
10803          *
10804          *
10805          * WaCxSRDisabledForSpriteScaling:ivb
10806          * IVB SPR_SCALE/Scaling Enable
10807          * "Low Power watermarks must be disabled for at least one
10808          *  frame before enabling sprite scaling, and kept disabled
10809          *  until sprite scaling is disabled."
10810          *
10811          * ILK/SNB DVSASCALE/Scaling Enable
10812          * "When in Self Refresh Big FIFO mode, scaling enable will be
10813          *  masked off while Big FIFO mode is exiting."
10814          *
10815          * Despite the w/a only being listed for IVB we assume that
10816          * the ILK/SNB note has similar ramifications, hence we apply
10817          * the w/a on all three platforms.
10818          */
10819         if (plane->id == PLANE_SPRITE0 &&
10820             (IS_GEN5(dev_priv) || IS_GEN6(dev_priv) ||
10821              IS_IVYBRIDGE(dev_priv)) &&
10822             (turn_on || (!needs_scaling(old_plane_state) &&
10823                          needs_scaling(to_intel_plane_state(plane_state)))))
10824                 pipe_config->disable_lp_wm = true;
10825
10826         return 0;
10827 }
10828
10829 static bool encoders_cloneable(const struct intel_encoder *a,
10830                                const struct intel_encoder *b)
10831 {
10832         /* masks could be asymmetric, so check both ways */
10833         return a == b || (a->cloneable & (1 << b->type) &&
10834                           b->cloneable & (1 << a->type));
10835 }
10836
10837 static bool check_single_encoder_cloning(struct drm_atomic_state *state,
10838                                          struct intel_crtc *crtc,
10839                                          struct intel_encoder *encoder)
10840 {
10841         struct intel_encoder *source_encoder;
10842         struct drm_connector *connector;
10843         struct drm_connector_state *connector_state;
10844         int i;
10845
10846         for_each_new_connector_in_state(state, connector, connector_state, i) {
10847                 if (connector_state->crtc != &crtc->base)
10848                         continue;
10849
10850                 source_encoder =
10851                         to_intel_encoder(connector_state->best_encoder);
10852                 if (!encoders_cloneable(encoder, source_encoder))
10853                         return false;
10854         }
10855
10856         return true;
10857 }
10858
10859 static int icl_add_linked_planes(struct intel_atomic_state *state)
10860 {
10861         struct intel_plane *plane, *linked;
10862         struct intel_plane_state *plane_state, *linked_plane_state;
10863         int i;
10864
10865         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
10866                 linked = plane_state->linked_plane;
10867
10868                 if (!linked)
10869                         continue;
10870
10871                 linked_plane_state = intel_atomic_get_plane_state(state, linked);
10872                 if (IS_ERR(linked_plane_state))
10873                         return PTR_ERR(linked_plane_state);
10874
10875                 WARN_ON(linked_plane_state->linked_plane != plane);
10876                 WARN_ON(linked_plane_state->slave == plane_state->slave);
10877         }
10878
10879         return 0;
10880 }
10881
10882 static int icl_check_nv12_planes(struct intel_crtc_state *crtc_state)
10883 {
10884         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
10885         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
10886         struct intel_atomic_state *state = to_intel_atomic_state(crtc_state->base.state);
10887         struct intel_plane *plane, *linked;
10888         struct intel_plane_state *plane_state;
10889         int i;
10890
10891         if (INTEL_GEN(dev_priv) < 11)
10892                 return 0;
10893
10894         /*
10895          * Destroy all old plane links and make the slave plane invisible
10896          * in the crtc_state->active_planes mask.
10897          */
10898         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
10899                 if (plane->pipe != crtc->pipe || !plane_state->linked_plane)
10900                         continue;
10901
10902                 plane_state->linked_plane = NULL;
10903                 if (plane_state->slave && !plane_state->base.visible) {
10904                         crtc_state->active_planes &= ~BIT(plane->id);
10905                         crtc_state->update_planes |= BIT(plane->id);
10906                 }
10907
10908                 plane_state->slave = false;
10909         }
10910
10911         if (!crtc_state->nv12_planes)
10912                 return 0;
10913
10914         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
10915                 struct intel_plane_state *linked_state = NULL;
10916
10917                 if (plane->pipe != crtc->pipe ||
10918                     !(crtc_state->nv12_planes & BIT(plane->id)))
10919                         continue;
10920
10921                 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, linked) {
10922                         if (!icl_is_nv12_y_plane(linked->id))
10923                                 continue;
10924
10925                         if (crtc_state->active_planes & BIT(linked->id))
10926                                 continue;
10927
10928                         linked_state = intel_atomic_get_plane_state(state, linked);
10929                         if (IS_ERR(linked_state))
10930                                 return PTR_ERR(linked_state);
10931
10932                         break;
10933                 }
10934
10935                 if (!linked_state) {
10936                         DRM_DEBUG_KMS("Need %d free Y planes for NV12\n",
10937                                       hweight8(crtc_state->nv12_planes));
10938
10939                         return -EINVAL;
10940                 }
10941
10942                 plane_state->linked_plane = linked;
10943
10944                 linked_state->slave = true;
10945                 linked_state->linked_plane = plane;
10946                 crtc_state->active_planes |= BIT(linked->id);
10947                 crtc_state->update_planes |= BIT(linked->id);
10948                 DRM_DEBUG_KMS("Using %s as Y plane for %s\n", linked->base.name, plane->base.name);
10949         }
10950
10951         return 0;
10952 }
10953
10954 static int intel_crtc_atomic_check(struct drm_crtc *crtc,
10955                                    struct drm_crtc_state *crtc_state)
10956 {
10957         struct drm_device *dev = crtc->dev;
10958         struct drm_i915_private *dev_priv = to_i915(dev);
10959         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
10960         struct intel_crtc_state *pipe_config =
10961                 to_intel_crtc_state(crtc_state);
10962         int ret;
10963         bool mode_changed = needs_modeset(crtc_state);
10964
10965         if (mode_changed && !crtc_state->active)
10966                 pipe_config->update_wm_post = true;
10967
10968         if (mode_changed && crtc_state->enable &&
10969             dev_priv->display.crtc_compute_clock &&
10970             !WARN_ON(pipe_config->shared_dpll)) {
10971                 ret = dev_priv->display.crtc_compute_clock(intel_crtc,
10972                                                            pipe_config);
10973                 if (ret)
10974                         return ret;
10975         }
10976
10977         if (crtc_state->color_mgmt_changed) {
10978                 ret = intel_color_check(crtc, crtc_state);
10979                 if (ret)
10980                         return ret;
10981
10982                 /*
10983                  * Changing color management on Intel hardware is
10984                  * handled as part of planes update.
10985                  */
10986                 crtc_state->planes_changed = true;
10987         }
10988
10989         ret = 0;
10990         if (dev_priv->display.compute_pipe_wm) {
10991                 ret = dev_priv->display.compute_pipe_wm(pipe_config);
10992                 if (ret) {
10993                         DRM_DEBUG_KMS("Target pipe watermarks are invalid\n");
10994                         return ret;
10995                 }
10996         }
10997
10998         if (dev_priv->display.compute_intermediate_wm) {
10999                 if (WARN_ON(!dev_priv->display.compute_pipe_wm))
11000                         return 0;
11001
11002                 /*
11003                  * Calculate 'intermediate' watermarks that satisfy both the
11004                  * old state and the new state.  We can program these
11005                  * immediately.
11006                  */
11007                 ret = dev_priv->display.compute_intermediate_wm(dev,
11008                                                                 intel_crtc,
11009                                                                 pipe_config);
11010                 if (ret) {
11011                         DRM_DEBUG_KMS("No valid intermediate pipe watermarks are possible\n");
11012                         return ret;
11013                 }
11014         }
11015
11016         if (INTEL_GEN(dev_priv) >= 9) {
11017                 if (mode_changed)
11018                         ret = skl_update_scaler_crtc(pipe_config);
11019
11020                 if (!ret)
11021                         ret = icl_check_nv12_planes(pipe_config);
11022                 if (!ret)
11023                         ret = skl_check_pipe_max_pixel_rate(intel_crtc,
11024                                                             pipe_config);
11025                 if (!ret)
11026                         ret = intel_atomic_setup_scalers(dev_priv, intel_crtc,
11027                                                          pipe_config);
11028         }
11029
11030         if (HAS_IPS(dev_priv))
11031                 pipe_config->ips_enabled = hsw_compute_ips_config(pipe_config);
11032
11033         return ret;
11034 }
11035
11036 static const struct drm_crtc_helper_funcs intel_helper_funcs = {
11037         .atomic_check = intel_crtc_atomic_check,
11038 };
11039
11040 static void intel_modeset_update_connector_atomic_state(struct drm_device *dev)
11041 {
11042         struct intel_connector *connector;
11043         struct drm_connector_list_iter conn_iter;
11044
11045         drm_connector_list_iter_begin(dev, &conn_iter);
11046         for_each_intel_connector_iter(connector, &conn_iter) {
11047                 if (connector->base.state->crtc)
11048                         drm_connector_put(&connector->base);
11049
11050                 if (connector->base.encoder) {
11051                         connector->base.state->best_encoder =
11052                                 connector->base.encoder;
11053                         connector->base.state->crtc =
11054                                 connector->base.encoder->crtc;
11055
11056                         drm_connector_get(&connector->base);
11057                 } else {
11058                         connector->base.state->best_encoder = NULL;
11059                         connector->base.state->crtc = NULL;
11060                 }
11061         }
11062         drm_connector_list_iter_end(&conn_iter);
11063 }
11064
11065 static int
11066 compute_sink_pipe_bpp(const struct drm_connector_state *conn_state,
11067                       struct intel_crtc_state *pipe_config)
11068 {
11069         struct drm_connector *connector = conn_state->connector;
11070         const struct drm_display_info *info = &connector->display_info;
11071         int bpp;
11072
11073         switch (conn_state->max_bpc) {
11074         case 6 ... 7:
11075                 bpp = 6 * 3;
11076                 break;
11077         case 8 ... 9:
11078                 bpp = 8 * 3;
11079                 break;
11080         case 10 ... 11:
11081                 bpp = 10 * 3;
11082                 break;
11083         case 12:
11084                 bpp = 12 * 3;
11085                 break;
11086         default:
11087                 return -EINVAL;
11088         }
11089
11090         if (bpp < pipe_config->pipe_bpp) {
11091                 DRM_DEBUG_KMS("[CONNECTOR:%d:%s] Limiting display bpp to %d instead of "
11092                               "EDID bpp %d, requested bpp %d, max platform bpp %d\n",
11093                               connector->base.id, connector->name,
11094                               bpp, 3 * info->bpc, 3 * conn_state->max_requested_bpc,
11095                               pipe_config->pipe_bpp);
11096
11097                 pipe_config->pipe_bpp = bpp;
11098         }
11099
11100         return 0;
11101 }
11102
11103 static int
11104 compute_baseline_pipe_bpp(struct intel_crtc *crtc,
11105                           struct intel_crtc_state *pipe_config)
11106 {
11107         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
11108         struct drm_atomic_state *state = pipe_config->base.state;
11109         struct drm_connector *connector;
11110         struct drm_connector_state *connector_state;
11111         int bpp, i;
11112
11113         if ((IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
11114             IS_CHERRYVIEW(dev_priv)))
11115                 bpp = 10*3;
11116         else if (INTEL_GEN(dev_priv) >= 5)
11117                 bpp = 12*3;
11118         else
11119                 bpp = 8*3;
11120
11121         pipe_config->pipe_bpp = bpp;
11122
11123         /* Clamp display bpp to connector max bpp */
11124         for_each_new_connector_in_state(state, connector, connector_state, i) {
11125                 int ret;
11126
11127                 if (connector_state->crtc != &crtc->base)
11128                         continue;
11129
11130                 ret = compute_sink_pipe_bpp(connector_state, pipe_config);
11131                 if (ret)
11132                         return ret;
11133         }
11134
11135         return 0;
11136 }
11137
11138 static void intel_dump_crtc_timings(const struct drm_display_mode *mode)
11139 {
11140         DRM_DEBUG_KMS("crtc timings: %d %d %d %d %d %d %d %d %d, "
11141                         "type: 0x%x flags: 0x%x\n",
11142                 mode->crtc_clock,
11143                 mode->crtc_hdisplay, mode->crtc_hsync_start,
11144                 mode->crtc_hsync_end, mode->crtc_htotal,
11145                 mode->crtc_vdisplay, mode->crtc_vsync_start,
11146                 mode->crtc_vsync_end, mode->crtc_vtotal, mode->type, mode->flags);
11147 }
11148
11149 static inline void
11150 intel_dump_m_n_config(struct intel_crtc_state *pipe_config, char *id,
11151                       unsigned int lane_count, struct intel_link_m_n *m_n)
11152 {
11153         DRM_DEBUG_KMS("%s: lanes: %i; gmch_m: %u, gmch_n: %u, link_m: %u, link_n: %u, tu: %u\n",
11154                       id, lane_count,
11155                       m_n->gmch_m, m_n->gmch_n,
11156                       m_n->link_m, m_n->link_n, m_n->tu);
11157 }
11158
11159 #define OUTPUT_TYPE(x) [INTEL_OUTPUT_ ## x] = #x
11160
11161 static const char * const output_type_str[] = {
11162         OUTPUT_TYPE(UNUSED),
11163         OUTPUT_TYPE(ANALOG),
11164         OUTPUT_TYPE(DVO),
11165         OUTPUT_TYPE(SDVO),
11166         OUTPUT_TYPE(LVDS),
11167         OUTPUT_TYPE(TVOUT),
11168         OUTPUT_TYPE(HDMI),
11169         OUTPUT_TYPE(DP),
11170         OUTPUT_TYPE(EDP),
11171         OUTPUT_TYPE(DSI),
11172         OUTPUT_TYPE(DDI),
11173         OUTPUT_TYPE(DP_MST),
11174 };
11175
11176 #undef OUTPUT_TYPE
11177
11178 static void snprintf_output_types(char *buf, size_t len,
11179                                   unsigned int output_types)
11180 {
11181         char *str = buf;
11182         int i;
11183
11184         str[0] = '\0';
11185
11186         for (i = 0; i < ARRAY_SIZE(output_type_str); i++) {
11187                 int r;
11188
11189                 if ((output_types & BIT(i)) == 0)
11190                         continue;
11191
11192                 r = snprintf(str, len, "%s%s",
11193                              str != buf ? "," : "", output_type_str[i]);
11194                 if (r >= len)
11195                         break;
11196                 str += r;
11197                 len -= r;
11198
11199                 output_types &= ~BIT(i);
11200         }
11201
11202         WARN_ON_ONCE(output_types != 0);
11203 }
11204
11205 static const char * const output_format_str[] = {
11206         [INTEL_OUTPUT_FORMAT_INVALID] = "Invalid",
11207         [INTEL_OUTPUT_FORMAT_RGB] = "RGB",
11208         [INTEL_OUTPUT_FORMAT_YCBCR420] = "YCBCR4:2:0",
11209         [INTEL_OUTPUT_FORMAT_YCBCR444] = "YCBCR4:4:4",
11210 };
11211
11212 static const char *output_formats(enum intel_output_format format)
11213 {
11214         if (format >= ARRAY_SIZE(output_format_str))
11215                 format = INTEL_OUTPUT_FORMAT_INVALID;
11216         return output_format_str[format];
11217 }
11218
11219 static void intel_dump_pipe_config(struct intel_crtc *crtc,
11220                                    struct intel_crtc_state *pipe_config,
11221                                    const char *context)
11222 {
11223         struct drm_device *dev = crtc->base.dev;
11224         struct drm_i915_private *dev_priv = to_i915(dev);
11225         struct drm_plane *plane;
11226         struct intel_plane *intel_plane;
11227         struct intel_plane_state *state;
11228         struct drm_framebuffer *fb;
11229         char buf[64];
11230
11231         DRM_DEBUG_KMS("[CRTC:%d:%s]%s\n",
11232                       crtc->base.base.id, crtc->base.name, context);
11233
11234         snprintf_output_types(buf, sizeof(buf), pipe_config->output_types);
11235         DRM_DEBUG_KMS("output_types: %s (0x%x)\n",
11236                       buf, pipe_config->output_types);
11237
11238         DRM_DEBUG_KMS("output format: %s\n",
11239                       output_formats(pipe_config->output_format));
11240
11241         DRM_DEBUG_KMS("cpu_transcoder: %s, pipe bpp: %i, dithering: %i\n",
11242                       transcoder_name(pipe_config->cpu_transcoder),
11243                       pipe_config->pipe_bpp, pipe_config->dither);
11244
11245         if (pipe_config->has_pch_encoder)
11246                 intel_dump_m_n_config(pipe_config, "fdi",
11247                                       pipe_config->fdi_lanes,
11248                                       &pipe_config->fdi_m_n);
11249
11250         if (intel_crtc_has_dp_encoder(pipe_config)) {
11251                 intel_dump_m_n_config(pipe_config, "dp m_n",
11252                                 pipe_config->lane_count, &pipe_config->dp_m_n);
11253                 if (pipe_config->has_drrs)
11254                         intel_dump_m_n_config(pipe_config, "dp m2_n2",
11255                                               pipe_config->lane_count,
11256                                               &pipe_config->dp_m2_n2);
11257         }
11258
11259         DRM_DEBUG_KMS("audio: %i, infoframes: %i\n",
11260                       pipe_config->has_audio, pipe_config->has_infoframe);
11261
11262         DRM_DEBUG_KMS("requested mode:\n");
11263         drm_mode_debug_printmodeline(&pipe_config->base.mode);
11264         DRM_DEBUG_KMS("adjusted mode:\n");
11265         drm_mode_debug_printmodeline(&pipe_config->base.adjusted_mode);
11266         intel_dump_crtc_timings(&pipe_config->base.adjusted_mode);
11267         DRM_DEBUG_KMS("port clock: %d, pipe src size: %dx%d, pixel rate %d\n",
11268                       pipe_config->port_clock,
11269                       pipe_config->pipe_src_w, pipe_config->pipe_src_h,
11270                       pipe_config->pixel_rate);
11271
11272         if (INTEL_GEN(dev_priv) >= 9)
11273                 DRM_DEBUG_KMS("num_scalers: %d, scaler_users: 0x%x, scaler_id: %d\n",
11274                               crtc->num_scalers,
11275                               pipe_config->scaler_state.scaler_users,
11276                               pipe_config->scaler_state.scaler_id);
11277
11278         if (HAS_GMCH_DISPLAY(dev_priv))
11279                 DRM_DEBUG_KMS("gmch pfit: control: 0x%08x, ratios: 0x%08x, lvds border: 0x%08x\n",
11280                               pipe_config->gmch_pfit.control,
11281                               pipe_config->gmch_pfit.pgm_ratios,
11282                               pipe_config->gmch_pfit.lvds_border_bits);
11283         else
11284                 DRM_DEBUG_KMS("pch pfit: pos: 0x%08x, size: 0x%08x, %s\n",
11285                               pipe_config->pch_pfit.pos,
11286                               pipe_config->pch_pfit.size,
11287                               enableddisabled(pipe_config->pch_pfit.enabled));
11288
11289         DRM_DEBUG_KMS("ips: %i, double wide: %i\n",
11290                       pipe_config->ips_enabled, pipe_config->double_wide);
11291
11292         intel_dpll_dump_hw_state(dev_priv, &pipe_config->dpll_hw_state);
11293
11294         DRM_DEBUG_KMS("planes on this crtc\n");
11295         list_for_each_entry(plane, &dev->mode_config.plane_list, head) {
11296                 struct drm_format_name_buf format_name;
11297                 intel_plane = to_intel_plane(plane);
11298                 if (intel_plane->pipe != crtc->pipe)
11299                         continue;
11300
11301                 state = to_intel_plane_state(plane->state);
11302                 fb = state->base.fb;
11303                 if (!fb) {
11304                         DRM_DEBUG_KMS("[PLANE:%d:%s] disabled, scaler_id = %d\n",
11305                                       plane->base.id, plane->name, state->scaler_id);
11306                         continue;
11307                 }
11308
11309                 DRM_DEBUG_KMS("[PLANE:%d:%s] FB:%d, fb = %ux%u format = %s\n",
11310                               plane->base.id, plane->name,
11311                               fb->base.id, fb->width, fb->height,
11312                               drm_get_format_name(fb->format->format, &format_name));
11313                 if (INTEL_GEN(dev_priv) >= 9)
11314                         DRM_DEBUG_KMS("\tscaler:%d src %dx%d+%d+%d dst %dx%d+%d+%d\n",
11315                                       state->scaler_id,
11316                                       state->base.src.x1 >> 16,
11317                                       state->base.src.y1 >> 16,
11318                                       drm_rect_width(&state->base.src) >> 16,
11319                                       drm_rect_height(&state->base.src) >> 16,
11320                                       state->base.dst.x1, state->base.dst.y1,
11321                                       drm_rect_width(&state->base.dst),
11322                                       drm_rect_height(&state->base.dst));
11323         }
11324 }
11325
11326 static bool check_digital_port_conflicts(struct drm_atomic_state *state)
11327 {
11328         struct drm_device *dev = state->dev;
11329         struct drm_connector *connector;
11330         struct drm_connector_list_iter conn_iter;
11331         unsigned int used_ports = 0;
11332         unsigned int used_mst_ports = 0;
11333         bool ret = true;
11334
11335         /*
11336          * Walk the connector list instead of the encoder
11337          * list to detect the problem on ddi platforms
11338          * where there's just one encoder per digital port.
11339          */
11340         drm_connector_list_iter_begin(dev, &conn_iter);
11341         drm_for_each_connector_iter(connector, &conn_iter) {
11342                 struct drm_connector_state *connector_state;
11343                 struct intel_encoder *encoder;
11344
11345                 connector_state = drm_atomic_get_new_connector_state(state, connector);
11346                 if (!connector_state)
11347                         connector_state = connector->state;
11348
11349                 if (!connector_state->best_encoder)
11350                         continue;
11351
11352                 encoder = to_intel_encoder(connector_state->best_encoder);
11353
11354                 WARN_ON(!connector_state->crtc);
11355
11356                 switch (encoder->type) {
11357                         unsigned int port_mask;
11358                 case INTEL_OUTPUT_DDI:
11359                         if (WARN_ON(!HAS_DDI(to_i915(dev))))
11360                                 break;
11361                         /* else: fall through */
11362                 case INTEL_OUTPUT_DP:
11363                 case INTEL_OUTPUT_HDMI:
11364                 case INTEL_OUTPUT_EDP:
11365                         port_mask = 1 << encoder->port;
11366
11367                         /* the same port mustn't appear more than once */
11368                         if (used_ports & port_mask)
11369                                 ret = false;
11370
11371                         used_ports |= port_mask;
11372                         break;
11373                 case INTEL_OUTPUT_DP_MST:
11374                         used_mst_ports |=
11375                                 1 << encoder->port;
11376                         break;
11377                 default:
11378                         break;
11379                 }
11380         }
11381         drm_connector_list_iter_end(&conn_iter);
11382
11383         /* can't mix MST and SST/HDMI on the same port */
11384         if (used_ports & used_mst_ports)
11385                 return false;
11386
11387         return ret;
11388 }
11389
11390 static void
11391 clear_intel_crtc_state(struct intel_crtc_state *crtc_state)
11392 {
11393         struct drm_i915_private *dev_priv =
11394                 to_i915(crtc_state->base.crtc->dev);
11395         struct intel_crtc_scaler_state scaler_state;
11396         struct intel_dpll_hw_state dpll_hw_state;
11397         struct intel_shared_dpll *shared_dpll;
11398         struct intel_crtc_wm_state wm_state;
11399         bool force_thru, ips_force_disable;
11400
11401         /* FIXME: before the switch to atomic started, a new pipe_config was
11402          * kzalloc'd. Code that depends on any field being zero should be
11403          * fixed, so that the crtc_state can be safely duplicated. For now,
11404          * only fields that are know to not cause problems are preserved. */
11405
11406         scaler_state = crtc_state->scaler_state;
11407         shared_dpll = crtc_state->shared_dpll;
11408         dpll_hw_state = crtc_state->dpll_hw_state;
11409         force_thru = crtc_state->pch_pfit.force_thru;
11410         ips_force_disable = crtc_state->ips_force_disable;
11411         if (IS_G4X(dev_priv) ||
11412             IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
11413                 wm_state = crtc_state->wm;
11414
11415         /* Keep base drm_crtc_state intact, only clear our extended struct */
11416         BUILD_BUG_ON(offsetof(struct intel_crtc_state, base));
11417         memset(&crtc_state->base + 1, 0,
11418                sizeof(*crtc_state) - sizeof(crtc_state->base));
11419
11420         crtc_state->scaler_state = scaler_state;
11421         crtc_state->shared_dpll = shared_dpll;
11422         crtc_state->dpll_hw_state = dpll_hw_state;
11423         crtc_state->pch_pfit.force_thru = force_thru;
11424         crtc_state->ips_force_disable = ips_force_disable;
11425         if (IS_G4X(dev_priv) ||
11426             IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
11427                 crtc_state->wm = wm_state;
11428 }
11429
11430 static int
11431 intel_modeset_pipe_config(struct drm_crtc *crtc,
11432                           struct intel_crtc_state *pipe_config)
11433 {
11434         struct drm_atomic_state *state = pipe_config->base.state;
11435         struct intel_encoder *encoder;
11436         struct drm_connector *connector;
11437         struct drm_connector_state *connector_state;
11438         int base_bpp, ret;
11439         int i;
11440         bool retry = true;
11441
11442         clear_intel_crtc_state(pipe_config);
11443
11444         pipe_config->cpu_transcoder =
11445                 (enum transcoder) to_intel_crtc(crtc)->pipe;
11446
11447         /*
11448          * Sanitize sync polarity flags based on requested ones. If neither
11449          * positive or negative polarity is requested, treat this as meaning
11450          * negative polarity.
11451          */
11452         if (!(pipe_config->base.adjusted_mode.flags &
11453               (DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_NHSYNC)))
11454                 pipe_config->base.adjusted_mode.flags |= DRM_MODE_FLAG_NHSYNC;
11455
11456         if (!(pipe_config->base.adjusted_mode.flags &
11457               (DRM_MODE_FLAG_PVSYNC | DRM_MODE_FLAG_NVSYNC)))
11458                 pipe_config->base.adjusted_mode.flags |= DRM_MODE_FLAG_NVSYNC;
11459
11460         ret = compute_baseline_pipe_bpp(to_intel_crtc(crtc),
11461                                         pipe_config);
11462         if (ret)
11463                 return ret;
11464
11465         base_bpp = pipe_config->pipe_bpp;
11466
11467         /*
11468          * Determine the real pipe dimensions. Note that stereo modes can
11469          * increase the actual pipe size due to the frame doubling and
11470          * insertion of additional space for blanks between the frame. This
11471          * is stored in the crtc timings. We use the requested mode to do this
11472          * computation to clearly distinguish it from the adjusted mode, which
11473          * can be changed by the connectors in the below retry loop.
11474          */
11475         drm_mode_get_hv_timing(&pipe_config->base.mode,
11476                                &pipe_config->pipe_src_w,
11477                                &pipe_config->pipe_src_h);
11478
11479         for_each_new_connector_in_state(state, connector, connector_state, i) {
11480                 if (connector_state->crtc != crtc)
11481                         continue;
11482
11483                 encoder = to_intel_encoder(connector_state->best_encoder);
11484
11485                 if (!check_single_encoder_cloning(state, to_intel_crtc(crtc), encoder)) {
11486                         DRM_DEBUG_KMS("rejecting invalid cloning configuration\n");
11487                         return -EINVAL;
11488                 }
11489
11490                 /*
11491                  * Determine output_types before calling the .compute_config()
11492                  * hooks so that the hooks can use this information safely.
11493                  */
11494                 if (encoder->compute_output_type)
11495                         pipe_config->output_types |=
11496                                 BIT(encoder->compute_output_type(encoder, pipe_config,
11497                                                                  connector_state));
11498                 else
11499                         pipe_config->output_types |= BIT(encoder->type);
11500         }
11501
11502 encoder_retry:
11503         /* Ensure the port clock defaults are reset when retrying. */
11504         pipe_config->port_clock = 0;
11505         pipe_config->pixel_multiplier = 1;
11506
11507         /* Fill in default crtc timings, allow encoders to overwrite them. */
11508         drm_mode_set_crtcinfo(&pipe_config->base.adjusted_mode,
11509                               CRTC_STEREO_DOUBLE);
11510
11511         /* Pass our mode to the connectors and the CRTC to give them a chance to
11512          * adjust it according to limitations or connector properties, and also
11513          * a chance to reject the mode entirely.
11514          */
11515         for_each_new_connector_in_state(state, connector, connector_state, i) {
11516                 if (connector_state->crtc != crtc)
11517                         continue;
11518
11519                 encoder = to_intel_encoder(connector_state->best_encoder);
11520
11521                 if (!(encoder->compute_config(encoder, pipe_config, connector_state))) {
11522                         DRM_DEBUG_KMS("Encoder config failure\n");
11523                         return -EINVAL;
11524                 }
11525         }
11526
11527         /* Set default port clock if not overwritten by the encoder. Needs to be
11528          * done afterwards in case the encoder adjusts the mode. */
11529         if (!pipe_config->port_clock)
11530                 pipe_config->port_clock = pipe_config->base.adjusted_mode.crtc_clock
11531                         * pipe_config->pixel_multiplier;
11532
11533         ret = intel_crtc_compute_config(to_intel_crtc(crtc), pipe_config);
11534         if (ret == -EDEADLK)
11535                 return ret;
11536         if (ret < 0) {
11537                 DRM_DEBUG_KMS("CRTC fixup failed\n");
11538                 return ret;
11539         }
11540
11541         if (ret == RETRY) {
11542                 if (WARN(!retry, "loop in pipe configuration computation\n"))
11543                         return -EINVAL;
11544
11545                 DRM_DEBUG_KMS("CRTC bw constrained, retrying\n");
11546                 retry = false;
11547                 goto encoder_retry;
11548         }
11549
11550         /* Dithering seems to not pass-through bits correctly when it should, so
11551          * only enable it on 6bpc panels and when its not a compliance
11552          * test requesting 6bpc video pattern.
11553          */
11554         pipe_config->dither = (pipe_config->pipe_bpp == 6*3) &&
11555                 !pipe_config->dither_force_disable;
11556         DRM_DEBUG_KMS("hw max bpp: %i, pipe bpp: %i, dithering: %i\n",
11557                       base_bpp, pipe_config->pipe_bpp, pipe_config->dither);
11558
11559         return 0;
11560 }
11561
11562 static bool intel_fuzzy_clock_check(int clock1, int clock2)
11563 {
11564         int diff;
11565
11566         if (clock1 == clock2)
11567                 return true;
11568
11569         if (!clock1 || !clock2)
11570                 return false;
11571
11572         diff = abs(clock1 - clock2);
11573
11574         if (((((diff + clock1 + clock2) * 100)) / (clock1 + clock2)) < 105)
11575                 return true;
11576
11577         return false;
11578 }
11579
11580 static bool
11581 intel_compare_m_n(unsigned int m, unsigned int n,
11582                   unsigned int m2, unsigned int n2,
11583                   bool exact)
11584 {
11585         if (m == m2 && n == n2)
11586                 return true;
11587
11588         if (exact || !m || !n || !m2 || !n2)
11589                 return false;
11590
11591         BUILD_BUG_ON(DATA_LINK_M_N_MASK > INT_MAX);
11592
11593         if (n > n2) {
11594                 while (n > n2) {
11595                         m2 <<= 1;
11596                         n2 <<= 1;
11597                 }
11598         } else if (n < n2) {
11599                 while (n < n2) {
11600                         m <<= 1;
11601                         n <<= 1;
11602                 }
11603         }
11604
11605         if (n != n2)
11606                 return false;
11607
11608         return intel_fuzzy_clock_check(m, m2);
11609 }
11610
11611 static bool
11612 intel_compare_link_m_n(const struct intel_link_m_n *m_n,
11613                        struct intel_link_m_n *m2_n2,
11614                        bool adjust)
11615 {
11616         if (m_n->tu == m2_n2->tu &&
11617             intel_compare_m_n(m_n->gmch_m, m_n->gmch_n,
11618                               m2_n2->gmch_m, m2_n2->gmch_n, !adjust) &&
11619             intel_compare_m_n(m_n->link_m, m_n->link_n,
11620                               m2_n2->link_m, m2_n2->link_n, !adjust)) {
11621                 if (adjust)
11622                         *m2_n2 = *m_n;
11623
11624                 return true;
11625         }
11626
11627         return false;
11628 }
11629
11630 static void __printf(3, 4)
11631 pipe_config_err(bool adjust, const char *name, const char *format, ...)
11632 {
11633         struct va_format vaf;
11634         va_list args;
11635
11636         va_start(args, format);
11637         vaf.fmt = format;
11638         vaf.va = &args;
11639
11640         if (adjust)
11641                 drm_dbg(DRM_UT_KMS, "mismatch in %s %pV", name, &vaf);
11642         else
11643                 drm_err("mismatch in %s %pV", name, &vaf);
11644
11645         va_end(args);
11646 }
11647
11648 static bool
11649 intel_pipe_config_compare(struct drm_i915_private *dev_priv,
11650                           struct intel_crtc_state *current_config,
11651                           struct intel_crtc_state *pipe_config,
11652                           bool adjust)
11653 {
11654         bool ret = true;
11655         bool fixup_inherited = adjust &&
11656                 (current_config->base.mode.private_flags & I915_MODE_FLAG_INHERITED) &&
11657                 !(pipe_config->base.mode.private_flags & I915_MODE_FLAG_INHERITED);
11658
11659 #define PIPE_CONF_CHECK_X(name) do { \
11660         if (current_config->name != pipe_config->name) { \
11661                 pipe_config_err(adjust, __stringify(name), \
11662                           "(expected 0x%08x, found 0x%08x)\n", \
11663                           current_config->name, \
11664                           pipe_config->name); \
11665                 ret = false; \
11666         } \
11667 } while (0)
11668
11669 #define PIPE_CONF_CHECK_I(name) do { \
11670         if (current_config->name != pipe_config->name) { \
11671                 pipe_config_err(adjust, __stringify(name), \
11672                           "(expected %i, found %i)\n", \
11673                           current_config->name, \
11674                           pipe_config->name); \
11675                 ret = false; \
11676         } \
11677 } while (0)
11678
11679 #define PIPE_CONF_CHECK_BOOL(name) do { \
11680         if (current_config->name != pipe_config->name) { \
11681                 pipe_config_err(adjust, __stringify(name), \
11682                           "(expected %s, found %s)\n", \
11683                           yesno(current_config->name), \
11684                           yesno(pipe_config->name)); \
11685                 ret = false; \
11686         } \
11687 } while (0)
11688
11689 /*
11690  * Checks state where we only read out the enabling, but not the entire
11691  * state itself (like full infoframes or ELD for audio). These states
11692  * require a full modeset on bootup to fix up.
11693  */
11694 #define PIPE_CONF_CHECK_BOOL_INCOMPLETE(name) do { \
11695         if (!fixup_inherited || (!current_config->name && !pipe_config->name)) { \
11696                 PIPE_CONF_CHECK_BOOL(name); \
11697         } else { \
11698                 pipe_config_err(adjust, __stringify(name), \
11699                           "unable to verify whether state matches exactly, forcing modeset (expected %s, found %s)\n", \
11700                           yesno(current_config->name), \
11701                           yesno(pipe_config->name)); \
11702                 ret = false; \
11703         } \
11704 } while (0)
11705
11706 #define PIPE_CONF_CHECK_P(name) do { \
11707         if (current_config->name != pipe_config->name) { \
11708                 pipe_config_err(adjust, __stringify(name), \
11709                           "(expected %p, found %p)\n", \
11710                           current_config->name, \
11711                           pipe_config->name); \
11712                 ret = false; \
11713         } \
11714 } while (0)
11715
11716 #define PIPE_CONF_CHECK_M_N(name) do { \
11717         if (!intel_compare_link_m_n(&current_config->name, \
11718                                     &pipe_config->name,\
11719                                     adjust)) { \
11720                 pipe_config_err(adjust, __stringify(name), \
11721                           "(expected tu %i gmch %i/%i link %i/%i, " \
11722                           "found tu %i, gmch %i/%i link %i/%i)\n", \
11723                           current_config->name.tu, \
11724                           current_config->name.gmch_m, \
11725                           current_config->name.gmch_n, \
11726                           current_config->name.link_m, \
11727                           current_config->name.link_n, \
11728                           pipe_config->name.tu, \
11729                           pipe_config->name.gmch_m, \
11730                           pipe_config->name.gmch_n, \
11731                           pipe_config->name.link_m, \
11732                           pipe_config->name.link_n); \
11733                 ret = false; \
11734         } \
11735 } while (0)
11736
11737 /* This is required for BDW+ where there is only one set of registers for
11738  * switching between high and low RR.
11739  * This macro can be used whenever a comparison has to be made between one
11740  * hw state and multiple sw state variables.
11741  */
11742 #define PIPE_CONF_CHECK_M_N_ALT(name, alt_name) do { \
11743         if (!intel_compare_link_m_n(&current_config->name, \
11744                                     &pipe_config->name, adjust) && \
11745             !intel_compare_link_m_n(&current_config->alt_name, \
11746                                     &pipe_config->name, adjust)) { \
11747                 pipe_config_err(adjust, __stringify(name), \
11748                           "(expected tu %i gmch %i/%i link %i/%i, " \
11749                           "or tu %i gmch %i/%i link %i/%i, " \
11750                           "found tu %i, gmch %i/%i link %i/%i)\n", \
11751                           current_config->name.tu, \
11752                           current_config->name.gmch_m, \
11753                           current_config->name.gmch_n, \
11754                           current_config->name.link_m, \
11755                           current_config->name.link_n, \
11756                           current_config->alt_name.tu, \
11757                           current_config->alt_name.gmch_m, \
11758                           current_config->alt_name.gmch_n, \
11759                           current_config->alt_name.link_m, \
11760                           current_config->alt_name.link_n, \
11761                           pipe_config->name.tu, \
11762                           pipe_config->name.gmch_m, \
11763                           pipe_config->name.gmch_n, \
11764                           pipe_config->name.link_m, \
11765                           pipe_config->name.link_n); \
11766                 ret = false; \
11767         } \
11768 } while (0)
11769
11770 #define PIPE_CONF_CHECK_FLAGS(name, mask) do { \
11771         if ((current_config->name ^ pipe_config->name) & (mask)) { \
11772                 pipe_config_err(adjust, __stringify(name), \
11773                           "(%x) (expected %i, found %i)\n", \
11774                           (mask), \
11775                           current_config->name & (mask), \
11776                           pipe_config->name & (mask)); \
11777                 ret = false; \
11778         } \
11779 } while (0)
11780
11781 #define PIPE_CONF_CHECK_CLOCK_FUZZY(name) do { \
11782         if (!intel_fuzzy_clock_check(current_config->name, pipe_config->name)) { \
11783                 pipe_config_err(adjust, __stringify(name), \
11784                           "(expected %i, found %i)\n", \
11785                           current_config->name, \
11786                           pipe_config->name); \
11787                 ret = false; \
11788         } \
11789 } while (0)
11790
11791 #define PIPE_CONF_QUIRK(quirk)  \
11792         ((current_config->quirks | pipe_config->quirks) & (quirk))
11793
11794         PIPE_CONF_CHECK_I(cpu_transcoder);
11795
11796         PIPE_CONF_CHECK_BOOL(has_pch_encoder);
11797         PIPE_CONF_CHECK_I(fdi_lanes);
11798         PIPE_CONF_CHECK_M_N(fdi_m_n);
11799
11800         PIPE_CONF_CHECK_I(lane_count);
11801         PIPE_CONF_CHECK_X(lane_lat_optim_mask);
11802
11803         if (INTEL_GEN(dev_priv) < 8) {
11804                 PIPE_CONF_CHECK_M_N(dp_m_n);
11805
11806                 if (current_config->has_drrs)
11807                         PIPE_CONF_CHECK_M_N(dp_m2_n2);
11808         } else
11809                 PIPE_CONF_CHECK_M_N_ALT(dp_m_n, dp_m2_n2);
11810
11811         PIPE_CONF_CHECK_X(output_types);
11812
11813         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_hdisplay);
11814         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_htotal);
11815         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_hblank_start);
11816         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_hblank_end);
11817         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_hsync_start);
11818         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_hsync_end);
11819
11820         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vdisplay);
11821         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vtotal);
11822         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vblank_start);
11823         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vblank_end);
11824         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vsync_start);
11825         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vsync_end);
11826
11827         PIPE_CONF_CHECK_I(pixel_multiplier);
11828         PIPE_CONF_CHECK_I(output_format);
11829         PIPE_CONF_CHECK_BOOL(has_hdmi_sink);
11830         if ((INTEL_GEN(dev_priv) < 8 && !IS_HASWELL(dev_priv)) ||
11831             IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
11832                 PIPE_CONF_CHECK_BOOL(limited_color_range);
11833
11834         PIPE_CONF_CHECK_BOOL(hdmi_scrambling);
11835         PIPE_CONF_CHECK_BOOL(hdmi_high_tmds_clock_ratio);
11836         PIPE_CONF_CHECK_BOOL_INCOMPLETE(has_infoframe);
11837
11838         PIPE_CONF_CHECK_BOOL_INCOMPLETE(has_audio);
11839
11840         PIPE_CONF_CHECK_FLAGS(base.adjusted_mode.flags,
11841                               DRM_MODE_FLAG_INTERLACE);
11842
11843         if (!PIPE_CONF_QUIRK(PIPE_CONFIG_QUIRK_MODE_SYNC_FLAGS)) {
11844                 PIPE_CONF_CHECK_FLAGS(base.adjusted_mode.flags,
11845                                       DRM_MODE_FLAG_PHSYNC);
11846                 PIPE_CONF_CHECK_FLAGS(base.adjusted_mode.flags,
11847                                       DRM_MODE_FLAG_NHSYNC);
11848                 PIPE_CONF_CHECK_FLAGS(base.adjusted_mode.flags,
11849                                       DRM_MODE_FLAG_PVSYNC);
11850                 PIPE_CONF_CHECK_FLAGS(base.adjusted_mode.flags,
11851                                       DRM_MODE_FLAG_NVSYNC);
11852         }
11853
11854         PIPE_CONF_CHECK_X(gmch_pfit.control);
11855         /* pfit ratios are autocomputed by the hw on gen4+ */
11856         if (INTEL_GEN(dev_priv) < 4)
11857                 PIPE_CONF_CHECK_X(gmch_pfit.pgm_ratios);
11858         PIPE_CONF_CHECK_X(gmch_pfit.lvds_border_bits);
11859
11860         if (!adjust) {
11861                 PIPE_CONF_CHECK_I(pipe_src_w);
11862                 PIPE_CONF_CHECK_I(pipe_src_h);
11863
11864                 PIPE_CONF_CHECK_BOOL(pch_pfit.enabled);
11865                 if (current_config->pch_pfit.enabled) {
11866                         PIPE_CONF_CHECK_X(pch_pfit.pos);
11867                         PIPE_CONF_CHECK_X(pch_pfit.size);
11868                 }
11869
11870                 PIPE_CONF_CHECK_I(scaler_state.scaler_id);
11871                 PIPE_CONF_CHECK_CLOCK_FUZZY(pixel_rate);
11872         }
11873
11874         PIPE_CONF_CHECK_BOOL(double_wide);
11875
11876         PIPE_CONF_CHECK_P(shared_dpll);
11877         PIPE_CONF_CHECK_X(dpll_hw_state.dpll);
11878         PIPE_CONF_CHECK_X(dpll_hw_state.dpll_md);
11879         PIPE_CONF_CHECK_X(dpll_hw_state.fp0);
11880         PIPE_CONF_CHECK_X(dpll_hw_state.fp1);
11881         PIPE_CONF_CHECK_X(dpll_hw_state.wrpll);
11882         PIPE_CONF_CHECK_X(dpll_hw_state.spll);
11883         PIPE_CONF_CHECK_X(dpll_hw_state.ctrl1);
11884         PIPE_CONF_CHECK_X(dpll_hw_state.cfgcr1);
11885         PIPE_CONF_CHECK_X(dpll_hw_state.cfgcr2);
11886         PIPE_CONF_CHECK_X(dpll_hw_state.cfgcr0);
11887         PIPE_CONF_CHECK_X(dpll_hw_state.ebb0);
11888         PIPE_CONF_CHECK_X(dpll_hw_state.ebb4);
11889         PIPE_CONF_CHECK_X(dpll_hw_state.pll0);
11890         PIPE_CONF_CHECK_X(dpll_hw_state.pll1);
11891         PIPE_CONF_CHECK_X(dpll_hw_state.pll2);
11892         PIPE_CONF_CHECK_X(dpll_hw_state.pll3);
11893         PIPE_CONF_CHECK_X(dpll_hw_state.pll6);
11894         PIPE_CONF_CHECK_X(dpll_hw_state.pll8);
11895         PIPE_CONF_CHECK_X(dpll_hw_state.pll9);
11896         PIPE_CONF_CHECK_X(dpll_hw_state.pll10);
11897         PIPE_CONF_CHECK_X(dpll_hw_state.pcsdw12);
11898         PIPE_CONF_CHECK_X(dpll_hw_state.mg_refclkin_ctl);
11899         PIPE_CONF_CHECK_X(dpll_hw_state.mg_clktop2_coreclkctl1);
11900         PIPE_CONF_CHECK_X(dpll_hw_state.mg_clktop2_hsclkctl);
11901         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_div0);
11902         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_div1);
11903         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_lf);
11904         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_frac_lock);
11905         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_ssc);
11906         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_bias);
11907         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_tdc_coldst_bias);
11908
11909         PIPE_CONF_CHECK_X(dsi_pll.ctrl);
11910         PIPE_CONF_CHECK_X(dsi_pll.div);
11911
11912         if (IS_G4X(dev_priv) || INTEL_GEN(dev_priv) >= 5)
11913                 PIPE_CONF_CHECK_I(pipe_bpp);
11914
11915         PIPE_CONF_CHECK_CLOCK_FUZZY(base.adjusted_mode.crtc_clock);
11916         PIPE_CONF_CHECK_CLOCK_FUZZY(port_clock);
11917
11918         PIPE_CONF_CHECK_I(min_voltage_level);
11919
11920 #undef PIPE_CONF_CHECK_X
11921 #undef PIPE_CONF_CHECK_I
11922 #undef PIPE_CONF_CHECK_BOOL
11923 #undef PIPE_CONF_CHECK_BOOL_INCOMPLETE
11924 #undef PIPE_CONF_CHECK_P
11925 #undef PIPE_CONF_CHECK_FLAGS
11926 #undef PIPE_CONF_CHECK_CLOCK_FUZZY
11927 #undef PIPE_CONF_QUIRK
11928
11929         return ret;
11930 }
11931
11932 static void intel_pipe_config_sanity_check(struct drm_i915_private *dev_priv,
11933                                            const struct intel_crtc_state *pipe_config)
11934 {
11935         if (pipe_config->has_pch_encoder) {
11936                 int fdi_dotclock = intel_dotclock_calculate(intel_fdi_link_freq(dev_priv, pipe_config),
11937                                                             &pipe_config->fdi_m_n);
11938                 int dotclock = pipe_config->base.adjusted_mode.crtc_clock;
11939
11940                 /*
11941                  * FDI already provided one idea for the dotclock.
11942                  * Yell if the encoder disagrees.
11943                  */
11944                 WARN(!intel_fuzzy_clock_check(fdi_dotclock, dotclock),
11945                      "FDI dotclock and encoder dotclock mismatch, fdi: %i, encoder: %i\n",
11946                      fdi_dotclock, dotclock);
11947         }
11948 }
11949
11950 static void verify_wm_state(struct drm_crtc *crtc,
11951                             struct drm_crtc_state *new_state)
11952 {
11953         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
11954         struct skl_ddb_allocation hw_ddb, *sw_ddb;
11955         struct skl_pipe_wm hw_wm, *sw_wm;
11956         struct skl_plane_wm *hw_plane_wm, *sw_plane_wm;
11957         struct skl_ddb_entry *hw_ddb_entry, *sw_ddb_entry;
11958         struct skl_ddb_entry hw_ddb_y[I915_MAX_PLANES];
11959         struct skl_ddb_entry hw_ddb_uv[I915_MAX_PLANES];
11960         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
11961         const enum pipe pipe = intel_crtc->pipe;
11962         int plane, level, max_level = ilk_wm_max_level(dev_priv);
11963
11964         if (INTEL_GEN(dev_priv) < 9 || !new_state->active)
11965                 return;
11966
11967         skl_pipe_wm_get_hw_state(crtc, &hw_wm);
11968         sw_wm = &to_intel_crtc_state(new_state)->wm.skl.optimal;
11969
11970         skl_pipe_ddb_get_hw_state(intel_crtc, hw_ddb_y, hw_ddb_uv);
11971
11972         skl_ddb_get_hw_state(dev_priv, &hw_ddb);
11973         sw_ddb = &dev_priv->wm.skl_hw.ddb;
11974
11975         if (INTEL_GEN(dev_priv) >= 11)
11976                 if (hw_ddb.enabled_slices != sw_ddb->enabled_slices)
11977                         DRM_ERROR("mismatch in DBUF Slices (expected %u, got %u)\n",
11978                                   sw_ddb->enabled_slices,
11979                                   hw_ddb.enabled_slices);
11980         /* planes */
11981         for_each_universal_plane(dev_priv, pipe, plane) {
11982                 hw_plane_wm = &hw_wm.planes[plane];
11983                 sw_plane_wm = &sw_wm->planes[plane];
11984
11985                 /* Watermarks */
11986                 for (level = 0; level <= max_level; level++) {
11987                         if (skl_wm_level_equals(&hw_plane_wm->wm[level],
11988                                                 &sw_plane_wm->wm[level]))
11989                                 continue;
11990
11991                         DRM_ERROR("mismatch in WM pipe %c plane %d level %d (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
11992                                   pipe_name(pipe), plane + 1, level,
11993                                   sw_plane_wm->wm[level].plane_en,
11994                                   sw_plane_wm->wm[level].plane_res_b,
11995                                   sw_plane_wm->wm[level].plane_res_l,
11996                                   hw_plane_wm->wm[level].plane_en,
11997                                   hw_plane_wm->wm[level].plane_res_b,
11998                                   hw_plane_wm->wm[level].plane_res_l);
11999                 }
12000
12001                 if (!skl_wm_level_equals(&hw_plane_wm->trans_wm,
12002                                          &sw_plane_wm->trans_wm)) {
12003                         DRM_ERROR("mismatch in trans WM pipe %c plane %d (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
12004                                   pipe_name(pipe), plane + 1,
12005                                   sw_plane_wm->trans_wm.plane_en,
12006                                   sw_plane_wm->trans_wm.plane_res_b,
12007                                   sw_plane_wm->trans_wm.plane_res_l,
12008                                   hw_plane_wm->trans_wm.plane_en,
12009                                   hw_plane_wm->trans_wm.plane_res_b,
12010                                   hw_plane_wm->trans_wm.plane_res_l);
12011                 }
12012
12013                 /* DDB */
12014                 hw_ddb_entry = &hw_ddb_y[plane];
12015                 sw_ddb_entry = &to_intel_crtc_state(new_state)->wm.skl.plane_ddb_y[plane];
12016
12017                 if (!skl_ddb_entry_equal(hw_ddb_entry, sw_ddb_entry)) {
12018                         DRM_ERROR("mismatch in DDB state pipe %c plane %d (expected (%u,%u), found (%u,%u))\n",
12019                                   pipe_name(pipe), plane + 1,
12020                                   sw_ddb_entry->start, sw_ddb_entry->end,
12021                                   hw_ddb_entry->start, hw_ddb_entry->end);
12022                 }
12023         }
12024
12025         /*
12026          * cursor
12027          * If the cursor plane isn't active, we may not have updated it's ddb
12028          * allocation. In that case since the ddb allocation will be updated
12029          * once the plane becomes visible, we can skip this check
12030          */
12031         if (1) {
12032                 hw_plane_wm = &hw_wm.planes[PLANE_CURSOR];
12033                 sw_plane_wm = &sw_wm->planes[PLANE_CURSOR];
12034
12035                 /* Watermarks */
12036                 for (level = 0; level <= max_level; level++) {
12037                         if (skl_wm_level_equals(&hw_plane_wm->wm[level],
12038                                                 &sw_plane_wm->wm[level]))
12039                                 continue;
12040
12041                         DRM_ERROR("mismatch in WM pipe %c cursor level %d (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
12042                                   pipe_name(pipe), level,
12043                                   sw_plane_wm->wm[level].plane_en,
12044                                   sw_plane_wm->wm[level].plane_res_b,
12045                                   sw_plane_wm->wm[level].plane_res_l,
12046                                   hw_plane_wm->wm[level].plane_en,
12047                                   hw_plane_wm->wm[level].plane_res_b,
12048                                   hw_plane_wm->wm[level].plane_res_l);
12049                 }
12050
12051                 if (!skl_wm_level_equals(&hw_plane_wm->trans_wm,
12052                                          &sw_plane_wm->trans_wm)) {
12053                         DRM_ERROR("mismatch in trans WM pipe %c cursor (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
12054                                   pipe_name(pipe),
12055                                   sw_plane_wm->trans_wm.plane_en,
12056                                   sw_plane_wm->trans_wm.plane_res_b,
12057                                   sw_plane_wm->trans_wm.plane_res_l,
12058                                   hw_plane_wm->trans_wm.plane_en,
12059                                   hw_plane_wm->trans_wm.plane_res_b,
12060                                   hw_plane_wm->trans_wm.plane_res_l);
12061                 }
12062
12063                 /* DDB */
12064                 hw_ddb_entry = &hw_ddb_y[PLANE_CURSOR];
12065                 sw_ddb_entry = &to_intel_crtc_state(new_state)->wm.skl.plane_ddb_y[PLANE_CURSOR];
12066
12067                 if (!skl_ddb_entry_equal(hw_ddb_entry, sw_ddb_entry)) {
12068                         DRM_ERROR("mismatch in DDB state pipe %c cursor (expected (%u,%u), found (%u,%u))\n",
12069                                   pipe_name(pipe),
12070                                   sw_ddb_entry->start, sw_ddb_entry->end,
12071                                   hw_ddb_entry->start, hw_ddb_entry->end);
12072                 }
12073         }
12074 }
12075
12076 static void
12077 verify_connector_state(struct drm_device *dev,
12078                        struct drm_atomic_state *state,
12079                        struct drm_crtc *crtc)
12080 {
12081         struct drm_connector *connector;
12082         struct drm_connector_state *new_conn_state;
12083         int i;
12084
12085         for_each_new_connector_in_state(state, connector, new_conn_state, i) {
12086                 struct drm_encoder *encoder = connector->encoder;
12087                 struct drm_crtc_state *crtc_state = NULL;
12088
12089                 if (new_conn_state->crtc != crtc)
12090                         continue;
12091
12092                 if (crtc)
12093                         crtc_state = drm_atomic_get_new_crtc_state(state, new_conn_state->crtc);
12094
12095                 intel_connector_verify_state(crtc_state, new_conn_state);
12096
12097                 I915_STATE_WARN(new_conn_state->best_encoder != encoder,
12098                      "connector's atomic encoder doesn't match legacy encoder\n");
12099         }
12100 }
12101
12102 static void
12103 verify_encoder_state(struct drm_device *dev, struct drm_atomic_state *state)
12104 {
12105         struct intel_encoder *encoder;
12106         struct drm_connector *connector;
12107         struct drm_connector_state *old_conn_state, *new_conn_state;
12108         int i;
12109
12110         for_each_intel_encoder(dev, encoder) {
12111                 bool enabled = false, found = false;
12112                 enum pipe pipe;
12113
12114                 DRM_DEBUG_KMS("[ENCODER:%d:%s]\n",
12115                               encoder->base.base.id,
12116                               encoder->base.name);
12117
12118                 for_each_oldnew_connector_in_state(state, connector, old_conn_state,
12119                                                    new_conn_state, i) {
12120                         if (old_conn_state->best_encoder == &encoder->base)
12121                                 found = true;
12122
12123                         if (new_conn_state->best_encoder != &encoder->base)
12124                                 continue;
12125                         found = enabled = true;
12126
12127                         I915_STATE_WARN(new_conn_state->crtc !=
12128                                         encoder->base.crtc,
12129                              "connector's crtc doesn't match encoder crtc\n");
12130                 }
12131
12132                 if (!found)
12133                         continue;
12134
12135                 I915_STATE_WARN(!!encoder->base.crtc != enabled,
12136                      "encoder's enabled state mismatch "
12137                      "(expected %i, found %i)\n",
12138                      !!encoder->base.crtc, enabled);
12139
12140                 if (!encoder->base.crtc) {
12141                         bool active;
12142
12143                         active = encoder->get_hw_state(encoder, &pipe);
12144                         I915_STATE_WARN(active,
12145                              "encoder detached but still enabled on pipe %c.\n",
12146                              pipe_name(pipe));
12147                 }
12148         }
12149 }
12150
12151 static void
12152 verify_crtc_state(struct drm_crtc *crtc,
12153                   struct drm_crtc_state *old_crtc_state,
12154                   struct drm_crtc_state *new_crtc_state)
12155 {
12156         struct drm_device *dev = crtc->dev;
12157         struct drm_i915_private *dev_priv = to_i915(dev);
12158         struct intel_encoder *encoder;
12159         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
12160         struct intel_crtc_state *pipe_config, *sw_config;
12161         struct drm_atomic_state *old_state;
12162         bool active;
12163
12164         old_state = old_crtc_state->state;
12165         __drm_atomic_helper_crtc_destroy_state(old_crtc_state);
12166         pipe_config = to_intel_crtc_state(old_crtc_state);
12167         memset(pipe_config, 0, sizeof(*pipe_config));
12168         pipe_config->base.crtc = crtc;
12169         pipe_config->base.state = old_state;
12170
12171         DRM_DEBUG_KMS("[CRTC:%d:%s]\n", crtc->base.id, crtc->name);
12172
12173         active = dev_priv->display.get_pipe_config(intel_crtc, pipe_config);
12174
12175         /* we keep both pipes enabled on 830 */
12176         if (IS_I830(dev_priv))
12177                 active = new_crtc_state->active;
12178
12179         I915_STATE_WARN(new_crtc_state->active != active,
12180              "crtc active state doesn't match with hw state "
12181              "(expected %i, found %i)\n", new_crtc_state->active, active);
12182
12183         I915_STATE_WARN(intel_crtc->active != new_crtc_state->active,
12184              "transitional active state does not match atomic hw state "
12185              "(expected %i, found %i)\n", new_crtc_state->active, intel_crtc->active);
12186
12187         for_each_encoder_on_crtc(dev, crtc, encoder) {
12188                 enum pipe pipe;
12189
12190                 active = encoder->get_hw_state(encoder, &pipe);
12191                 I915_STATE_WARN(active != new_crtc_state->active,
12192                         "[ENCODER:%i] active %i with crtc active %i\n",
12193                         encoder->base.base.id, active, new_crtc_state->active);
12194
12195                 I915_STATE_WARN(active && intel_crtc->pipe != pipe,
12196                                 "Encoder connected to wrong pipe %c\n",
12197                                 pipe_name(pipe));
12198
12199                 if (active)
12200                         encoder->get_config(encoder, pipe_config);
12201         }
12202
12203         intel_crtc_compute_pixel_rate(pipe_config);
12204
12205         if (!new_crtc_state->active)
12206                 return;
12207
12208         intel_pipe_config_sanity_check(dev_priv, pipe_config);
12209
12210         sw_config = to_intel_crtc_state(new_crtc_state);
12211         if (!intel_pipe_config_compare(dev_priv, sw_config,
12212                                        pipe_config, false)) {
12213                 I915_STATE_WARN(1, "pipe state doesn't match!\n");
12214                 intel_dump_pipe_config(intel_crtc, pipe_config,
12215                                        "[hw state]");
12216                 intel_dump_pipe_config(intel_crtc, sw_config,
12217                                        "[sw state]");
12218         }
12219 }
12220
12221 static void
12222 intel_verify_planes(struct intel_atomic_state *state)
12223 {
12224         struct intel_plane *plane;
12225         const struct intel_plane_state *plane_state;
12226         int i;
12227
12228         for_each_new_intel_plane_in_state(state, plane,
12229                                           plane_state, i)
12230                 assert_plane(plane, plane_state->base.visible);
12231 }
12232
12233 static void
12234 verify_single_dpll_state(struct drm_i915_private *dev_priv,
12235                          struct intel_shared_dpll *pll,
12236                          struct drm_crtc *crtc,
12237                          struct drm_crtc_state *new_state)
12238 {
12239         struct intel_dpll_hw_state dpll_hw_state;
12240         unsigned int crtc_mask;
12241         bool active;
12242
12243         memset(&dpll_hw_state, 0, sizeof(dpll_hw_state));
12244
12245         DRM_DEBUG_KMS("%s\n", pll->info->name);
12246
12247         active = pll->info->funcs->get_hw_state(dev_priv, pll, &dpll_hw_state);
12248
12249         if (!(pll->info->flags & INTEL_DPLL_ALWAYS_ON)) {
12250                 I915_STATE_WARN(!pll->on && pll->active_mask,
12251                      "pll in active use but not on in sw tracking\n");
12252                 I915_STATE_WARN(pll->on && !pll->active_mask,
12253                      "pll is on but not used by any active crtc\n");
12254                 I915_STATE_WARN(pll->on != active,
12255                      "pll on state mismatch (expected %i, found %i)\n",
12256                      pll->on, active);
12257         }
12258
12259         if (!crtc) {
12260                 I915_STATE_WARN(pll->active_mask & ~pll->state.crtc_mask,
12261                                 "more active pll users than references: %x vs %x\n",
12262                                 pll->active_mask, pll->state.crtc_mask);
12263
12264                 return;
12265         }
12266
12267         crtc_mask = drm_crtc_mask(crtc);
12268
12269         if (new_state->active)
12270                 I915_STATE_WARN(!(pll->active_mask & crtc_mask),
12271                                 "pll active mismatch (expected pipe %c in active mask 0x%02x)\n",
12272                                 pipe_name(drm_crtc_index(crtc)), pll->active_mask);
12273         else
12274                 I915_STATE_WARN(pll->active_mask & crtc_mask,
12275                                 "pll active mismatch (didn't expect pipe %c in active mask 0x%02x)\n",
12276                                 pipe_name(drm_crtc_index(crtc)), pll->active_mask);
12277
12278         I915_STATE_WARN(!(pll->state.crtc_mask & crtc_mask),
12279                         "pll enabled crtcs mismatch (expected 0x%x in 0x%02x)\n",
12280                         crtc_mask, pll->state.crtc_mask);
12281
12282         I915_STATE_WARN(pll->on && memcmp(&pll->state.hw_state,
12283                                           &dpll_hw_state,
12284                                           sizeof(dpll_hw_state)),
12285                         "pll hw state mismatch\n");
12286 }
12287
12288 static void
12289 verify_shared_dpll_state(struct drm_device *dev, struct drm_crtc *crtc,
12290                          struct drm_crtc_state *old_crtc_state,
12291                          struct drm_crtc_state *new_crtc_state)
12292 {
12293         struct drm_i915_private *dev_priv = to_i915(dev);
12294         struct intel_crtc_state *old_state = to_intel_crtc_state(old_crtc_state);
12295         struct intel_crtc_state *new_state = to_intel_crtc_state(new_crtc_state);
12296
12297         if (new_state->shared_dpll)
12298                 verify_single_dpll_state(dev_priv, new_state->shared_dpll, crtc, new_crtc_state);
12299
12300         if (old_state->shared_dpll &&
12301             old_state->shared_dpll != new_state->shared_dpll) {
12302                 unsigned int crtc_mask = drm_crtc_mask(crtc);
12303                 struct intel_shared_dpll *pll = old_state->shared_dpll;
12304
12305                 I915_STATE_WARN(pll->active_mask & crtc_mask,
12306                                 "pll active mismatch (didn't expect pipe %c in active mask)\n",
12307                                 pipe_name(drm_crtc_index(crtc)));
12308                 I915_STATE_WARN(pll->state.crtc_mask & crtc_mask,
12309                                 "pll enabled crtcs mismatch (found %x in enabled mask)\n",
12310                                 pipe_name(drm_crtc_index(crtc)));
12311         }
12312 }
12313
12314 static void
12315 intel_modeset_verify_crtc(struct drm_crtc *crtc,
12316                           struct drm_atomic_state *state,
12317                           struct drm_crtc_state *old_state,
12318                           struct drm_crtc_state *new_state)
12319 {
12320         if (!needs_modeset(new_state) &&
12321             !to_intel_crtc_state(new_state)->update_pipe)
12322                 return;
12323
12324         verify_wm_state(crtc, new_state);
12325         verify_connector_state(crtc->dev, state, crtc);
12326         verify_crtc_state(crtc, old_state, new_state);
12327         verify_shared_dpll_state(crtc->dev, crtc, old_state, new_state);
12328 }
12329
12330 static void
12331 verify_disabled_dpll_state(struct drm_device *dev)
12332 {
12333         struct drm_i915_private *dev_priv = to_i915(dev);
12334         int i;
12335
12336         for (i = 0; i < dev_priv->num_shared_dpll; i++)
12337                 verify_single_dpll_state(dev_priv, &dev_priv->shared_dplls[i], NULL, NULL);
12338 }
12339
12340 static void
12341 intel_modeset_verify_disabled(struct drm_device *dev,
12342                               struct drm_atomic_state *state)
12343 {
12344         verify_encoder_state(dev, state);
12345         verify_connector_state(dev, state, NULL);
12346         verify_disabled_dpll_state(dev);
12347 }
12348
12349 static void update_scanline_offset(const struct intel_crtc_state *crtc_state)
12350 {
12351         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
12352         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
12353
12354         /*
12355          * The scanline counter increments at the leading edge of hsync.
12356          *
12357          * On most platforms it starts counting from vtotal-1 on the
12358          * first active line. That means the scanline counter value is
12359          * always one less than what we would expect. Ie. just after
12360          * start of vblank, which also occurs at start of hsync (on the
12361          * last active line), the scanline counter will read vblank_start-1.
12362          *
12363          * On gen2 the scanline counter starts counting from 1 instead
12364          * of vtotal-1, so we have to subtract one (or rather add vtotal-1
12365          * to keep the value positive), instead of adding one.
12366          *
12367          * On HSW+ the behaviour of the scanline counter depends on the output
12368          * type. For DP ports it behaves like most other platforms, but on HDMI
12369          * there's an extra 1 line difference. So we need to add two instead of
12370          * one to the value.
12371          *
12372          * On VLV/CHV DSI the scanline counter would appear to increment
12373          * approx. 1/3 of a scanline before start of vblank. Unfortunately
12374          * that means we can't tell whether we're in vblank or not while
12375          * we're on that particular line. We must still set scanline_offset
12376          * to 1 so that the vblank timestamps come out correct when we query
12377          * the scanline counter from within the vblank interrupt handler.
12378          * However if queried just before the start of vblank we'll get an
12379          * answer that's slightly in the future.
12380          */
12381         if (IS_GEN2(dev_priv)) {
12382                 const struct drm_display_mode *adjusted_mode = &crtc_state->base.adjusted_mode;
12383                 int vtotal;
12384
12385                 vtotal = adjusted_mode->crtc_vtotal;
12386                 if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE)
12387                         vtotal /= 2;
12388
12389                 crtc->scanline_offset = vtotal - 1;
12390         } else if (HAS_DDI(dev_priv) &&
12391                    intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) {
12392                 crtc->scanline_offset = 2;
12393         } else
12394                 crtc->scanline_offset = 1;
12395 }
12396
12397 static void intel_modeset_clear_plls(struct drm_atomic_state *state)
12398 {
12399         struct drm_device *dev = state->dev;
12400         struct drm_i915_private *dev_priv = to_i915(dev);
12401         struct drm_crtc *crtc;
12402         struct drm_crtc_state *old_crtc_state, *new_crtc_state;
12403         int i;
12404
12405         if (!dev_priv->display.crtc_compute_clock)
12406                 return;
12407
12408         for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
12409                 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
12410                 struct intel_shared_dpll *old_dpll =
12411                         to_intel_crtc_state(old_crtc_state)->shared_dpll;
12412
12413                 if (!needs_modeset(new_crtc_state))
12414                         continue;
12415
12416                 to_intel_crtc_state(new_crtc_state)->shared_dpll = NULL;
12417
12418                 if (!old_dpll)
12419                         continue;
12420
12421                 intel_release_shared_dpll(old_dpll, intel_crtc, state);
12422         }
12423 }
12424
12425 /*
12426  * This implements the workaround described in the "notes" section of the mode
12427  * set sequence documentation. When going from no pipes or single pipe to
12428  * multiple pipes, and planes are enabled after the pipe, we need to wait at
12429  * least 2 vblanks on the first pipe before enabling planes on the second pipe.
12430  */
12431 static int haswell_mode_set_planes_workaround(struct drm_atomic_state *state)
12432 {
12433         struct drm_crtc_state *crtc_state;
12434         struct intel_crtc *intel_crtc;
12435         struct drm_crtc *crtc;
12436         struct intel_crtc_state *first_crtc_state = NULL;
12437         struct intel_crtc_state *other_crtc_state = NULL;
12438         enum pipe first_pipe = INVALID_PIPE, enabled_pipe = INVALID_PIPE;
12439         int i;
12440
12441         /* look at all crtc's that are going to be enabled in during modeset */
12442         for_each_new_crtc_in_state(state, crtc, crtc_state, i) {
12443                 intel_crtc = to_intel_crtc(crtc);
12444
12445                 if (!crtc_state->active || !needs_modeset(crtc_state))
12446                         continue;
12447
12448                 if (first_crtc_state) {
12449                         other_crtc_state = to_intel_crtc_state(crtc_state);
12450                         break;
12451                 } else {
12452                         first_crtc_state = to_intel_crtc_state(crtc_state);
12453                         first_pipe = intel_crtc->pipe;
12454                 }
12455         }
12456
12457         /* No workaround needed? */
12458         if (!first_crtc_state)
12459                 return 0;
12460
12461         /* w/a possibly needed, check how many crtc's are already enabled. */
12462         for_each_intel_crtc(state->dev, intel_crtc) {
12463                 struct intel_crtc_state *pipe_config;
12464
12465                 pipe_config = intel_atomic_get_crtc_state(state, intel_crtc);
12466                 if (IS_ERR(pipe_config))
12467                         return PTR_ERR(pipe_config);
12468
12469                 pipe_config->hsw_workaround_pipe = INVALID_PIPE;
12470
12471                 if (!pipe_config->base.active ||
12472                     needs_modeset(&pipe_config->base))
12473                         continue;
12474
12475                 /* 2 or more enabled crtcs means no need for w/a */
12476                 if (enabled_pipe != INVALID_PIPE)
12477                         return 0;
12478
12479                 enabled_pipe = intel_crtc->pipe;
12480         }
12481
12482         if (enabled_pipe != INVALID_PIPE)
12483                 first_crtc_state->hsw_workaround_pipe = enabled_pipe;
12484         else if (other_crtc_state)
12485                 other_crtc_state->hsw_workaround_pipe = first_pipe;
12486
12487         return 0;
12488 }
12489
12490 static int intel_lock_all_pipes(struct drm_atomic_state *state)
12491 {
12492         struct drm_crtc *crtc;
12493
12494         /* Add all pipes to the state */
12495         for_each_crtc(state->dev, crtc) {
12496                 struct drm_crtc_state *crtc_state;
12497
12498                 crtc_state = drm_atomic_get_crtc_state(state, crtc);
12499                 if (IS_ERR(crtc_state))
12500                         return PTR_ERR(crtc_state);
12501         }
12502
12503         return 0;
12504 }
12505
12506 static int intel_modeset_all_pipes(struct drm_atomic_state *state)
12507 {
12508         struct drm_crtc *crtc;
12509
12510         /*
12511          * Add all pipes to the state, and force
12512          * a modeset on all the active ones.
12513          */
12514         for_each_crtc(state->dev, crtc) {
12515                 struct drm_crtc_state *crtc_state;
12516                 int ret;
12517
12518                 crtc_state = drm_atomic_get_crtc_state(state, crtc);
12519                 if (IS_ERR(crtc_state))
12520                         return PTR_ERR(crtc_state);
12521
12522                 if (!crtc_state->active || needs_modeset(crtc_state))
12523                         continue;
12524
12525                 crtc_state->mode_changed = true;
12526
12527                 ret = drm_atomic_add_affected_connectors(state, crtc);
12528                 if (ret)
12529                         return ret;
12530
12531                 ret = drm_atomic_add_affected_planes(state, crtc);
12532                 if (ret)
12533                         return ret;
12534         }
12535
12536         return 0;
12537 }
12538
12539 static int intel_modeset_checks(struct drm_atomic_state *state)
12540 {
12541         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
12542         struct drm_i915_private *dev_priv = to_i915(state->dev);
12543         struct drm_crtc *crtc;
12544         struct drm_crtc_state *old_crtc_state, *new_crtc_state;
12545         int ret = 0, i;
12546
12547         if (!check_digital_port_conflicts(state)) {
12548                 DRM_DEBUG_KMS("rejecting conflicting digital port configuration\n");
12549                 return -EINVAL;
12550         }
12551
12552         intel_state->modeset = true;
12553         intel_state->active_crtcs = dev_priv->active_crtcs;
12554         intel_state->cdclk.logical = dev_priv->cdclk.logical;
12555         intel_state->cdclk.actual = dev_priv->cdclk.actual;
12556
12557         for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
12558                 if (new_crtc_state->active)
12559                         intel_state->active_crtcs |= 1 << i;
12560                 else
12561                         intel_state->active_crtcs &= ~(1 << i);
12562
12563                 if (old_crtc_state->active != new_crtc_state->active)
12564                         intel_state->active_pipe_changes |= drm_crtc_mask(crtc);
12565         }
12566
12567         /*
12568          * See if the config requires any additional preparation, e.g.
12569          * to adjust global state with pipes off.  We need to do this
12570          * here so we can get the modeset_pipe updated config for the new
12571          * mode set on this crtc.  For other crtcs we need to use the
12572          * adjusted_mode bits in the crtc directly.
12573          */
12574         if (dev_priv->display.modeset_calc_cdclk) {
12575                 ret = dev_priv->display.modeset_calc_cdclk(state);
12576                 if (ret < 0)
12577                         return ret;
12578
12579                 /*
12580                  * Writes to dev_priv->cdclk.logical must protected by
12581                  * holding all the crtc locks, even if we don't end up
12582                  * touching the hardware
12583                  */
12584                 if (intel_cdclk_changed(&dev_priv->cdclk.logical,
12585                                         &intel_state->cdclk.logical)) {
12586                         ret = intel_lock_all_pipes(state);
12587                         if (ret < 0)
12588                                 return ret;
12589                 }
12590
12591                 /* All pipes must be switched off while we change the cdclk. */
12592                 if (intel_cdclk_needs_modeset(&dev_priv->cdclk.actual,
12593                                               &intel_state->cdclk.actual)) {
12594                         ret = intel_modeset_all_pipes(state);
12595                         if (ret < 0)
12596                                 return ret;
12597                 }
12598
12599                 DRM_DEBUG_KMS("New cdclk calculated to be logical %u kHz, actual %u kHz\n",
12600                               intel_state->cdclk.logical.cdclk,
12601                               intel_state->cdclk.actual.cdclk);
12602                 DRM_DEBUG_KMS("New voltage level calculated to be logical %u, actual %u\n",
12603                               intel_state->cdclk.logical.voltage_level,
12604                               intel_state->cdclk.actual.voltage_level);
12605         } else {
12606                 to_intel_atomic_state(state)->cdclk.logical = dev_priv->cdclk.logical;
12607         }
12608
12609         intel_modeset_clear_plls(state);
12610
12611         if (IS_HASWELL(dev_priv))
12612                 return haswell_mode_set_planes_workaround(state);
12613
12614         return 0;
12615 }
12616
12617 /*
12618  * Handle calculation of various watermark data at the end of the atomic check
12619  * phase.  The code here should be run after the per-crtc and per-plane 'check'
12620  * handlers to ensure that all derived state has been updated.
12621  */
12622 static int calc_watermark_data(struct drm_atomic_state *state)
12623 {
12624         struct drm_device *dev = state->dev;
12625         struct drm_i915_private *dev_priv = to_i915(dev);
12626
12627         /* Is there platform-specific watermark information to calculate? */
12628         if (dev_priv->display.compute_global_watermarks)
12629                 return dev_priv->display.compute_global_watermarks(state);
12630
12631         return 0;
12632 }
12633
12634 /**
12635  * intel_atomic_check - validate state object
12636  * @dev: drm device
12637  * @state: state to validate
12638  */
12639 static int intel_atomic_check(struct drm_device *dev,
12640                               struct drm_atomic_state *state)
12641 {
12642         struct drm_i915_private *dev_priv = to_i915(dev);
12643         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
12644         struct drm_crtc *crtc;
12645         struct drm_crtc_state *old_crtc_state, *crtc_state;
12646         int ret, i;
12647         bool any_ms = false;
12648
12649         /* Catch I915_MODE_FLAG_INHERITED */
12650         for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state,
12651                                       crtc_state, i) {
12652                 if (crtc_state->mode.private_flags !=
12653                     old_crtc_state->mode.private_flags)
12654                         crtc_state->mode_changed = true;
12655         }
12656
12657         ret = drm_atomic_helper_check_modeset(dev, state);
12658         if (ret)
12659                 return ret;
12660
12661         for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, crtc_state, i) {
12662                 struct intel_crtc_state *pipe_config =
12663                         to_intel_crtc_state(crtc_state);
12664
12665                 if (!needs_modeset(crtc_state))
12666                         continue;
12667
12668                 if (!crtc_state->enable) {
12669                         any_ms = true;
12670                         continue;
12671                 }
12672
12673                 ret = intel_modeset_pipe_config(crtc, pipe_config);
12674                 if (ret == -EDEADLK)
12675                         return ret;
12676                 if (ret) {
12677                         intel_dump_pipe_config(to_intel_crtc(crtc),
12678                                                pipe_config, "[failed]");
12679                         return ret;
12680                 }
12681
12682                 if (i915_modparams.fastboot &&
12683                     intel_pipe_config_compare(dev_priv,
12684                                         to_intel_crtc_state(old_crtc_state),
12685                                         pipe_config, true)) {
12686                         crtc_state->mode_changed = false;
12687                         pipe_config->update_pipe = true;
12688                 }
12689
12690                 if (needs_modeset(crtc_state))
12691                         any_ms = true;
12692
12693                 intel_dump_pipe_config(to_intel_crtc(crtc), pipe_config,
12694                                        needs_modeset(crtc_state) ?
12695                                        "[modeset]" : "[fastset]");
12696         }
12697
12698         if (any_ms) {
12699                 ret = intel_modeset_checks(state);
12700
12701                 if (ret)
12702                         return ret;
12703         } else {
12704                 intel_state->cdclk.logical = dev_priv->cdclk.logical;
12705         }
12706
12707         ret = icl_add_linked_planes(intel_state);
12708         if (ret)
12709                 return ret;
12710
12711         ret = drm_atomic_helper_check_planes(dev, state);
12712         if (ret)
12713                 return ret;
12714
12715         intel_fbc_choose_crtc(dev_priv, intel_state);
12716         return calc_watermark_data(state);
12717 }
12718
12719 static int intel_atomic_prepare_commit(struct drm_device *dev,
12720                                        struct drm_atomic_state *state)
12721 {
12722         return drm_atomic_helper_prepare_planes(dev, state);
12723 }
12724
12725 u32 intel_crtc_get_vblank_counter(struct intel_crtc *crtc)
12726 {
12727         struct drm_device *dev = crtc->base.dev;
12728
12729         if (!dev->max_vblank_count)
12730                 return (u32)drm_crtc_accurate_vblank_count(&crtc->base);
12731
12732         return dev->driver->get_vblank_counter(dev, crtc->pipe);
12733 }
12734
12735 static void intel_update_crtc(struct drm_crtc *crtc,
12736                               struct drm_atomic_state *state,
12737                               struct drm_crtc_state *old_crtc_state,
12738                               struct drm_crtc_state *new_crtc_state)
12739 {
12740         struct drm_device *dev = crtc->dev;
12741         struct drm_i915_private *dev_priv = to_i915(dev);
12742         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
12743         struct intel_crtc_state *pipe_config = to_intel_crtc_state(new_crtc_state);
12744         bool modeset = needs_modeset(new_crtc_state);
12745         struct intel_plane_state *new_plane_state =
12746                 intel_atomic_get_new_plane_state(to_intel_atomic_state(state),
12747                                                  to_intel_plane(crtc->primary));
12748
12749         if (modeset) {
12750                 update_scanline_offset(pipe_config);
12751                 dev_priv->display.crtc_enable(pipe_config, state);
12752
12753                 /* vblanks work again, re-enable pipe CRC. */
12754                 intel_crtc_enable_pipe_crc(intel_crtc);
12755         } else {
12756                 intel_pre_plane_update(to_intel_crtc_state(old_crtc_state),
12757                                        pipe_config);
12758         }
12759
12760         if (new_plane_state)
12761                 intel_fbc_enable(intel_crtc, pipe_config, new_plane_state);
12762
12763         intel_begin_crtc_commit(crtc, old_crtc_state);
12764
12765         if (INTEL_GEN(dev_priv) >= 9)
12766                 skl_update_planes_on_crtc(to_intel_atomic_state(state), intel_crtc);
12767         else
12768                 i9xx_update_planes_on_crtc(to_intel_atomic_state(state), intel_crtc);
12769
12770         intel_finish_crtc_commit(crtc, old_crtc_state);
12771 }
12772
12773 static void intel_update_crtcs(struct drm_atomic_state *state)
12774 {
12775         struct drm_crtc *crtc;
12776         struct drm_crtc_state *old_crtc_state, *new_crtc_state;
12777         int i;
12778
12779         for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
12780                 if (!new_crtc_state->active)
12781                         continue;
12782
12783                 intel_update_crtc(crtc, state, old_crtc_state,
12784                                   new_crtc_state);
12785         }
12786 }
12787
12788 static void skl_update_crtcs(struct drm_atomic_state *state)
12789 {
12790         struct drm_i915_private *dev_priv = to_i915(state->dev);
12791         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
12792         struct drm_crtc *crtc;
12793         struct intel_crtc *intel_crtc;
12794         struct drm_crtc_state *old_crtc_state, *new_crtc_state;
12795         struct intel_crtc_state *cstate;
12796         unsigned int updated = 0;
12797         bool progress;
12798         enum pipe pipe;
12799         int i;
12800         u8 hw_enabled_slices = dev_priv->wm.skl_hw.ddb.enabled_slices;
12801         u8 required_slices = intel_state->wm_results.ddb.enabled_slices;
12802         struct skl_ddb_entry entries[I915_MAX_PIPES] = {};
12803
12804         for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i)
12805                 /* ignore allocations for crtc's that have been turned off. */
12806                 if (new_crtc_state->active)
12807                         entries[i] = to_intel_crtc_state(old_crtc_state)->wm.skl.ddb;
12808
12809         /* If 2nd DBuf slice required, enable it here */
12810         if (INTEL_GEN(dev_priv) >= 11 && required_slices > hw_enabled_slices)
12811                 icl_dbuf_slices_update(dev_priv, required_slices);
12812
12813         /*
12814          * Whenever the number of active pipes changes, we need to make sure we
12815          * update the pipes in the right order so that their ddb allocations
12816          * never overlap with eachother inbetween CRTC updates. Otherwise we'll
12817          * cause pipe underruns and other bad stuff.
12818          */
12819         do {
12820                 progress = false;
12821
12822                 for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
12823                         bool vbl_wait = false;
12824                         unsigned int cmask = drm_crtc_mask(crtc);
12825
12826                         intel_crtc = to_intel_crtc(crtc);
12827                         cstate = to_intel_crtc_state(new_crtc_state);
12828                         pipe = intel_crtc->pipe;
12829
12830                         if (updated & cmask || !cstate->base.active)
12831                                 continue;
12832
12833                         if (skl_ddb_allocation_overlaps(&cstate->wm.skl.ddb,
12834                                                         entries,
12835                                                         INTEL_INFO(dev_priv)->num_pipes, i))
12836                                 continue;
12837
12838                         updated |= cmask;
12839                         entries[i] = cstate->wm.skl.ddb;
12840
12841                         /*
12842                          * If this is an already active pipe, it's DDB changed,
12843                          * and this isn't the last pipe that needs updating
12844                          * then we need to wait for a vblank to pass for the
12845                          * new ddb allocation to take effect.
12846                          */
12847                         if (!skl_ddb_entry_equal(&cstate->wm.skl.ddb,
12848                                                  &to_intel_crtc_state(old_crtc_state)->wm.skl.ddb) &&
12849                             !new_crtc_state->active_changed &&
12850                             intel_state->wm_results.dirty_pipes != updated)
12851                                 vbl_wait = true;
12852
12853                         intel_update_crtc(crtc, state, old_crtc_state,
12854                                           new_crtc_state);
12855
12856                         if (vbl_wait)
12857                                 intel_wait_for_vblank(dev_priv, pipe);
12858
12859                         progress = true;
12860                 }
12861         } while (progress);
12862
12863         /* If 2nd DBuf slice is no more required disable it */
12864         if (INTEL_GEN(dev_priv) >= 11 && required_slices < hw_enabled_slices)
12865                 icl_dbuf_slices_update(dev_priv, required_slices);
12866 }
12867
12868 static void intel_atomic_helper_free_state(struct drm_i915_private *dev_priv)
12869 {
12870         struct intel_atomic_state *state, *next;
12871         struct llist_node *freed;
12872
12873         freed = llist_del_all(&dev_priv->atomic_helper.free_list);
12874         llist_for_each_entry_safe(state, next, freed, freed)
12875                 drm_atomic_state_put(&state->base);
12876 }
12877
12878 static void intel_atomic_helper_free_state_worker(struct work_struct *work)
12879 {
12880         struct drm_i915_private *dev_priv =
12881                 container_of(work, typeof(*dev_priv), atomic_helper.free_work);
12882
12883         intel_atomic_helper_free_state(dev_priv);
12884 }
12885
12886 static void intel_atomic_commit_fence_wait(struct intel_atomic_state *intel_state)
12887 {
12888         struct wait_queue_entry wait_fence, wait_reset;
12889         struct drm_i915_private *dev_priv = to_i915(intel_state->base.dev);
12890
12891         init_wait_entry(&wait_fence, 0);
12892         init_wait_entry(&wait_reset, 0);
12893         for (;;) {
12894                 prepare_to_wait(&intel_state->commit_ready.wait,
12895                                 &wait_fence, TASK_UNINTERRUPTIBLE);
12896                 prepare_to_wait(&dev_priv->gpu_error.wait_queue,
12897                                 &wait_reset, TASK_UNINTERRUPTIBLE);
12898
12899
12900                 if (i915_sw_fence_done(&intel_state->commit_ready)
12901                     || test_bit(I915_RESET_MODESET, &dev_priv->gpu_error.flags))
12902                         break;
12903
12904                 schedule();
12905         }
12906         finish_wait(&intel_state->commit_ready.wait, &wait_fence);
12907         finish_wait(&dev_priv->gpu_error.wait_queue, &wait_reset);
12908 }
12909
12910 static void intel_atomic_cleanup_work(struct work_struct *work)
12911 {
12912         struct drm_atomic_state *state =
12913                 container_of(work, struct drm_atomic_state, commit_work);
12914         struct drm_i915_private *i915 = to_i915(state->dev);
12915
12916         drm_atomic_helper_cleanup_planes(&i915->drm, state);
12917         drm_atomic_helper_commit_cleanup_done(state);
12918         drm_atomic_state_put(state);
12919
12920         intel_atomic_helper_free_state(i915);
12921 }
12922
12923 static void intel_atomic_commit_tail(struct drm_atomic_state *state)
12924 {
12925         struct drm_device *dev = state->dev;
12926         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
12927         struct drm_i915_private *dev_priv = to_i915(dev);
12928         struct drm_crtc_state *old_crtc_state, *new_crtc_state;
12929         struct intel_crtc_state *new_intel_crtc_state, *old_intel_crtc_state;
12930         struct drm_crtc *crtc;
12931         struct intel_crtc *intel_crtc;
12932         u64 put_domains[I915_MAX_PIPES] = {};
12933         int i;
12934
12935         intel_atomic_commit_fence_wait(intel_state);
12936
12937         drm_atomic_helper_wait_for_dependencies(state);
12938
12939         if (intel_state->modeset)
12940                 intel_display_power_get(dev_priv, POWER_DOMAIN_MODESET);
12941
12942         for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
12943                 old_intel_crtc_state = to_intel_crtc_state(old_crtc_state);
12944                 new_intel_crtc_state = to_intel_crtc_state(new_crtc_state);
12945                 intel_crtc = to_intel_crtc(crtc);
12946
12947                 if (needs_modeset(new_crtc_state) ||
12948                     to_intel_crtc_state(new_crtc_state)->update_pipe) {
12949
12950                         put_domains[intel_crtc->pipe] =
12951                                 modeset_get_crtc_power_domains(crtc,
12952                                         new_intel_crtc_state);
12953                 }
12954
12955                 if (!needs_modeset(new_crtc_state))
12956                         continue;
12957
12958                 intel_pre_plane_update(old_intel_crtc_state, new_intel_crtc_state);
12959
12960                 if (old_crtc_state->active) {
12961                         intel_crtc_disable_planes(intel_state, intel_crtc);
12962
12963                         /*
12964                          * We need to disable pipe CRC before disabling the pipe,
12965                          * or we race against vblank off.
12966                          */
12967                         intel_crtc_disable_pipe_crc(intel_crtc);
12968
12969                         dev_priv->display.crtc_disable(old_intel_crtc_state, state);
12970                         intel_crtc->active = false;
12971                         intel_fbc_disable(intel_crtc);
12972                         intel_disable_shared_dpll(old_intel_crtc_state);
12973
12974                         /*
12975                          * Underruns don't always raise
12976                          * interrupts, so check manually.
12977                          */
12978                         intel_check_cpu_fifo_underruns(dev_priv);
12979                         intel_check_pch_fifo_underruns(dev_priv);
12980
12981                         /* FIXME unify this for all platforms */
12982                         if (!new_crtc_state->active &&
12983                             !HAS_GMCH_DISPLAY(dev_priv) &&
12984                             dev_priv->display.initial_watermarks)
12985                                 dev_priv->display.initial_watermarks(intel_state,
12986                                                                      new_intel_crtc_state);
12987                 }
12988         }
12989
12990         /* FIXME: Eventually get rid of our intel_crtc->config pointer */
12991         for_each_new_crtc_in_state(state, crtc, new_crtc_state, i)
12992                 to_intel_crtc(crtc)->config = to_intel_crtc_state(new_crtc_state);
12993
12994         if (intel_state->modeset) {
12995                 drm_atomic_helper_update_legacy_modeset_state(state->dev, state);
12996
12997                 intel_set_cdclk(dev_priv, &dev_priv->cdclk.actual);
12998
12999                 /*
13000                  * SKL workaround: bspec recommends we disable the SAGV when we
13001                  * have more then one pipe enabled
13002                  */
13003                 if (!intel_can_enable_sagv(state))
13004                         intel_disable_sagv(dev_priv);
13005
13006                 intel_modeset_verify_disabled(dev, state);
13007         }
13008
13009         /* Complete the events for pipes that have now been disabled */
13010         for_each_new_crtc_in_state(state, crtc, new_crtc_state, i) {
13011                 bool modeset = needs_modeset(new_crtc_state);
13012
13013                 /* Complete events for now disable pipes here. */
13014                 if (modeset && !new_crtc_state->active && new_crtc_state->event) {
13015                         spin_lock_irq(&dev->event_lock);
13016                         drm_crtc_send_vblank_event(crtc, new_crtc_state->event);
13017                         spin_unlock_irq(&dev->event_lock);
13018
13019                         new_crtc_state->event = NULL;
13020                 }
13021         }
13022
13023         /* Now enable the clocks, plane, pipe, and connectors that we set up. */
13024         dev_priv->display.update_crtcs(state);
13025
13026         /* FIXME: We should call drm_atomic_helper_commit_hw_done() here
13027          * already, but still need the state for the delayed optimization. To
13028          * fix this:
13029          * - wrap the optimization/post_plane_update stuff into a per-crtc work.
13030          * - schedule that vblank worker _before_ calling hw_done
13031          * - at the start of commit_tail, cancel it _synchrously
13032          * - switch over to the vblank wait helper in the core after that since
13033          *   we don't need out special handling any more.
13034          */
13035         drm_atomic_helper_wait_for_flip_done(dev, state);
13036
13037         /*
13038          * Now that the vblank has passed, we can go ahead and program the
13039          * optimal watermarks on platforms that need two-step watermark
13040          * programming.
13041          *
13042          * TODO: Move this (and other cleanup) to an async worker eventually.
13043          */
13044         for_each_new_crtc_in_state(state, crtc, new_crtc_state, i) {
13045                 new_intel_crtc_state = to_intel_crtc_state(new_crtc_state);
13046
13047                 if (dev_priv->display.optimize_watermarks)
13048                         dev_priv->display.optimize_watermarks(intel_state,
13049                                                               new_intel_crtc_state);
13050         }
13051
13052         for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
13053                 intel_post_plane_update(to_intel_crtc_state(old_crtc_state));
13054
13055                 if (put_domains[i])
13056                         modeset_put_power_domains(dev_priv, put_domains[i]);
13057
13058                 intel_modeset_verify_crtc(crtc, state, old_crtc_state, new_crtc_state);
13059         }
13060
13061         if (intel_state->modeset)
13062                 intel_verify_planes(intel_state);
13063
13064         if (intel_state->modeset && intel_can_enable_sagv(state))
13065                 intel_enable_sagv(dev_priv);
13066
13067         drm_atomic_helper_commit_hw_done(state);
13068
13069         if (intel_state->modeset) {
13070                 /* As one of the primary mmio accessors, KMS has a high
13071                  * likelihood of triggering bugs in unclaimed access. After we
13072                  * finish modesetting, see if an error has been flagged, and if
13073                  * so enable debugging for the next modeset - and hope we catch
13074                  * the culprit.
13075                  */
13076                 intel_uncore_arm_unclaimed_mmio_detection(dev_priv);
13077                 intel_display_power_put(dev_priv, POWER_DOMAIN_MODESET);
13078         }
13079
13080         /*
13081          * Defer the cleanup of the old state to a separate worker to not
13082          * impede the current task (userspace for blocking modesets) that
13083          * are executed inline. For out-of-line asynchronous modesets/flips,
13084          * deferring to a new worker seems overkill, but we would place a
13085          * schedule point (cond_resched()) here anyway to keep latencies
13086          * down.
13087          */
13088         INIT_WORK(&state->commit_work, intel_atomic_cleanup_work);
13089         queue_work(system_highpri_wq, &state->commit_work);
13090 }
13091
13092 static void intel_atomic_commit_work(struct work_struct *work)
13093 {
13094         struct drm_atomic_state *state =
13095                 container_of(work, struct drm_atomic_state, commit_work);
13096
13097         intel_atomic_commit_tail(state);
13098 }
13099
13100 static int __i915_sw_fence_call
13101 intel_atomic_commit_ready(struct i915_sw_fence *fence,
13102                           enum i915_sw_fence_notify notify)
13103 {
13104         struct intel_atomic_state *state =
13105                 container_of(fence, struct intel_atomic_state, commit_ready);
13106
13107         switch (notify) {
13108         case FENCE_COMPLETE:
13109                 /* we do blocking waits in the worker, nothing to do here */
13110                 break;
13111         case FENCE_FREE:
13112                 {
13113                         struct intel_atomic_helper *helper =
13114                                 &to_i915(state->base.dev)->atomic_helper;
13115
13116                         if (llist_add(&state->freed, &helper->free_list))
13117                                 schedule_work(&helper->free_work);
13118                         break;
13119                 }
13120         }
13121
13122         return NOTIFY_DONE;
13123 }
13124
13125 static void intel_atomic_track_fbs(struct drm_atomic_state *state)
13126 {
13127         struct drm_plane_state *old_plane_state, *new_plane_state;
13128         struct drm_plane *plane;
13129         int i;
13130
13131         for_each_oldnew_plane_in_state(state, plane, old_plane_state, new_plane_state, i)
13132                 i915_gem_track_fb(intel_fb_obj(old_plane_state->fb),
13133                                   intel_fb_obj(new_plane_state->fb),
13134                                   to_intel_plane(plane)->frontbuffer_bit);
13135 }
13136
13137 /**
13138  * intel_atomic_commit - commit validated state object
13139  * @dev: DRM device
13140  * @state: the top-level driver state object
13141  * @nonblock: nonblocking commit
13142  *
13143  * This function commits a top-level state object that has been validated
13144  * with drm_atomic_helper_check().
13145  *
13146  * RETURNS
13147  * Zero for success or -errno.
13148  */
13149 static int intel_atomic_commit(struct drm_device *dev,
13150                                struct drm_atomic_state *state,
13151                                bool nonblock)
13152 {
13153         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
13154         struct drm_i915_private *dev_priv = to_i915(dev);
13155         int ret = 0;
13156
13157         drm_atomic_state_get(state);
13158         i915_sw_fence_init(&intel_state->commit_ready,
13159                            intel_atomic_commit_ready);
13160
13161         /*
13162          * The intel_legacy_cursor_update() fast path takes care
13163          * of avoiding the vblank waits for simple cursor
13164          * movement and flips. For cursor on/off and size changes,
13165          * we want to perform the vblank waits so that watermark
13166          * updates happen during the correct frames. Gen9+ have
13167          * double buffered watermarks and so shouldn't need this.
13168          *
13169          * Unset state->legacy_cursor_update before the call to
13170          * drm_atomic_helper_setup_commit() because otherwise
13171          * drm_atomic_helper_wait_for_flip_done() is a noop and
13172          * we get FIFO underruns because we didn't wait
13173          * for vblank.
13174          *
13175          * FIXME doing watermarks and fb cleanup from a vblank worker
13176          * (assuming we had any) would solve these problems.
13177          */
13178         if (INTEL_GEN(dev_priv) < 9 && state->legacy_cursor_update) {
13179                 struct intel_crtc_state *new_crtc_state;
13180                 struct intel_crtc *crtc;
13181                 int i;
13182
13183                 for_each_new_intel_crtc_in_state(intel_state, crtc, new_crtc_state, i)
13184                         if (new_crtc_state->wm.need_postvbl_update ||
13185                             new_crtc_state->update_wm_post)
13186                                 state->legacy_cursor_update = false;
13187         }
13188
13189         ret = intel_atomic_prepare_commit(dev, state);
13190         if (ret) {
13191                 DRM_DEBUG_ATOMIC("Preparing state failed with %i\n", ret);
13192                 i915_sw_fence_commit(&intel_state->commit_ready);
13193                 return ret;
13194         }
13195
13196         ret = drm_atomic_helper_setup_commit(state, nonblock);
13197         if (!ret)
13198                 ret = drm_atomic_helper_swap_state(state, true);
13199
13200         if (ret) {
13201                 i915_sw_fence_commit(&intel_state->commit_ready);
13202
13203                 drm_atomic_helper_cleanup_planes(dev, state);
13204                 return ret;
13205         }
13206         dev_priv->wm.distrust_bios_wm = false;
13207         intel_shared_dpll_swap_state(state);
13208         intel_atomic_track_fbs(state);
13209
13210         if (intel_state->modeset) {
13211                 memcpy(dev_priv->min_cdclk, intel_state->min_cdclk,
13212                        sizeof(intel_state->min_cdclk));
13213                 memcpy(dev_priv->min_voltage_level,
13214                        intel_state->min_voltage_level,
13215                        sizeof(intel_state->min_voltage_level));
13216                 dev_priv->active_crtcs = intel_state->active_crtcs;
13217                 dev_priv->cdclk.logical = intel_state->cdclk.logical;
13218                 dev_priv->cdclk.actual = intel_state->cdclk.actual;
13219         }
13220
13221         drm_atomic_state_get(state);
13222         INIT_WORK(&state->commit_work, intel_atomic_commit_work);
13223
13224         i915_sw_fence_commit(&intel_state->commit_ready);
13225         if (nonblock && intel_state->modeset) {
13226                 queue_work(dev_priv->modeset_wq, &state->commit_work);
13227         } else if (nonblock) {
13228                 queue_work(system_unbound_wq, &state->commit_work);
13229         } else {
13230                 if (intel_state->modeset)
13231                         flush_workqueue(dev_priv->modeset_wq);
13232                 intel_atomic_commit_tail(state);
13233         }
13234
13235         return 0;
13236 }
13237
13238 static const struct drm_crtc_funcs intel_crtc_funcs = {
13239         .gamma_set = drm_atomic_helper_legacy_gamma_set,
13240         .set_config = drm_atomic_helper_set_config,
13241         .destroy = intel_crtc_destroy,
13242         .page_flip = drm_atomic_helper_page_flip,
13243         .atomic_duplicate_state = intel_crtc_duplicate_state,
13244         .atomic_destroy_state = intel_crtc_destroy_state,
13245         .set_crc_source = intel_crtc_set_crc_source,
13246         .verify_crc_source = intel_crtc_verify_crc_source,
13247         .get_crc_sources = intel_crtc_get_crc_sources,
13248 };
13249
13250 struct wait_rps_boost {
13251         struct wait_queue_entry wait;
13252
13253         struct drm_crtc *crtc;
13254         struct i915_request *request;
13255 };
13256
13257 static int do_rps_boost(struct wait_queue_entry *_wait,
13258                         unsigned mode, int sync, void *key)
13259 {
13260         struct wait_rps_boost *wait = container_of(_wait, typeof(*wait), wait);
13261         struct i915_request *rq = wait->request;
13262
13263         /*
13264          * If we missed the vblank, but the request is already running it
13265          * is reasonable to assume that it will complete before the next
13266          * vblank without our intervention, so leave RPS alone.
13267          */
13268         if (!i915_request_started(rq))
13269                 gen6_rps_boost(rq, NULL);
13270         i915_request_put(rq);
13271
13272         drm_crtc_vblank_put(wait->crtc);
13273
13274         list_del(&wait->wait.entry);
13275         kfree(wait);
13276         return 1;
13277 }
13278
13279 static void add_rps_boost_after_vblank(struct drm_crtc *crtc,
13280                                        struct dma_fence *fence)
13281 {
13282         struct wait_rps_boost *wait;
13283
13284         if (!dma_fence_is_i915(fence))
13285                 return;
13286
13287         if (INTEL_GEN(to_i915(crtc->dev)) < 6)
13288                 return;
13289
13290         if (drm_crtc_vblank_get(crtc))
13291                 return;
13292
13293         wait = kmalloc(sizeof(*wait), GFP_KERNEL);
13294         if (!wait) {
13295                 drm_crtc_vblank_put(crtc);
13296                 return;
13297         }
13298
13299         wait->request = to_request(dma_fence_get(fence));
13300         wait->crtc = crtc;
13301
13302         wait->wait.func = do_rps_boost;
13303         wait->wait.flags = 0;
13304
13305         add_wait_queue(drm_crtc_vblank_waitqueue(crtc), &wait->wait);
13306 }
13307
13308 static int intel_plane_pin_fb(struct intel_plane_state *plane_state)
13309 {
13310         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
13311         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
13312         struct drm_framebuffer *fb = plane_state->base.fb;
13313         struct i915_vma *vma;
13314
13315         if (plane->id == PLANE_CURSOR &&
13316             INTEL_INFO(dev_priv)->display.cursor_needs_physical) {
13317                 struct drm_i915_gem_object *obj = intel_fb_obj(fb);
13318                 const int align = intel_cursor_alignment(dev_priv);
13319                 int err;
13320
13321                 err = i915_gem_object_attach_phys(obj, align);
13322                 if (err)
13323                         return err;
13324         }
13325
13326         vma = intel_pin_and_fence_fb_obj(fb,
13327                                          &plane_state->view,
13328                                          intel_plane_uses_fence(plane_state),
13329                                          &plane_state->flags);
13330         if (IS_ERR(vma))
13331                 return PTR_ERR(vma);
13332
13333         plane_state->vma = vma;
13334
13335         return 0;
13336 }
13337
13338 static void intel_plane_unpin_fb(struct intel_plane_state *old_plane_state)
13339 {
13340         struct i915_vma *vma;
13341
13342         vma = fetch_and_zero(&old_plane_state->vma);
13343         if (vma)
13344                 intel_unpin_fb_vma(vma, old_plane_state->flags);
13345 }
13346
13347 static void fb_obj_bump_render_priority(struct drm_i915_gem_object *obj)
13348 {
13349         struct i915_sched_attr attr = {
13350                 .priority = I915_PRIORITY_DISPLAY,
13351         };
13352
13353         i915_gem_object_wait_priority(obj, 0, &attr);
13354 }
13355
13356 /**
13357  * intel_prepare_plane_fb - Prepare fb for usage on plane
13358  * @plane: drm plane to prepare for
13359  * @new_state: the plane state being prepared
13360  *
13361  * Prepares a framebuffer for usage on a display plane.  Generally this
13362  * involves pinning the underlying object and updating the frontbuffer tracking
13363  * bits.  Some older platforms need special physical address handling for
13364  * cursor planes.
13365  *
13366  * Must be called with struct_mutex held.
13367  *
13368  * Returns 0 on success, negative error code on failure.
13369  */
13370 int
13371 intel_prepare_plane_fb(struct drm_plane *plane,
13372                        struct drm_plane_state *new_state)
13373 {
13374         struct intel_atomic_state *intel_state =
13375                 to_intel_atomic_state(new_state->state);
13376         struct drm_i915_private *dev_priv = to_i915(plane->dev);
13377         struct drm_framebuffer *fb = new_state->fb;
13378         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
13379         struct drm_i915_gem_object *old_obj = intel_fb_obj(plane->state->fb);
13380         int ret;
13381
13382         if (old_obj) {
13383                 struct drm_crtc_state *crtc_state =
13384                         drm_atomic_get_new_crtc_state(new_state->state,
13385                                                       plane->state->crtc);
13386
13387                 /* Big Hammer, we also need to ensure that any pending
13388                  * MI_WAIT_FOR_EVENT inside a user batch buffer on the
13389                  * current scanout is retired before unpinning the old
13390                  * framebuffer. Note that we rely on userspace rendering
13391                  * into the buffer attached to the pipe they are waiting
13392                  * on. If not, userspace generates a GPU hang with IPEHR
13393                  * point to the MI_WAIT_FOR_EVENT.
13394                  *
13395                  * This should only fail upon a hung GPU, in which case we
13396                  * can safely continue.
13397                  */
13398                 if (needs_modeset(crtc_state)) {
13399                         ret = i915_sw_fence_await_reservation(&intel_state->commit_ready,
13400                                                               old_obj->resv, NULL,
13401                                                               false, 0,
13402                                                               GFP_KERNEL);
13403                         if (ret < 0)
13404                                 return ret;
13405                 }
13406         }
13407
13408         if (new_state->fence) { /* explicit fencing */
13409                 ret = i915_sw_fence_await_dma_fence(&intel_state->commit_ready,
13410                                                     new_state->fence,
13411                                                     I915_FENCE_TIMEOUT,
13412                                                     GFP_KERNEL);
13413                 if (ret < 0)
13414                         return ret;
13415         }
13416
13417         if (!obj)
13418                 return 0;
13419
13420         ret = i915_gem_object_pin_pages(obj);
13421         if (ret)
13422                 return ret;
13423
13424         ret = mutex_lock_interruptible(&dev_priv->drm.struct_mutex);
13425         if (ret) {
13426                 i915_gem_object_unpin_pages(obj);
13427                 return ret;
13428         }
13429
13430         ret = intel_plane_pin_fb(to_intel_plane_state(new_state));
13431
13432         mutex_unlock(&dev_priv->drm.struct_mutex);
13433         i915_gem_object_unpin_pages(obj);
13434         if (ret)
13435                 return ret;
13436
13437         fb_obj_bump_render_priority(obj);
13438         intel_fb_obj_flush(obj, ORIGIN_DIRTYFB);
13439
13440         if (!new_state->fence) { /* implicit fencing */
13441                 struct dma_fence *fence;
13442
13443                 ret = i915_sw_fence_await_reservation(&intel_state->commit_ready,
13444                                                       obj->resv, NULL,
13445                                                       false, I915_FENCE_TIMEOUT,
13446                                                       GFP_KERNEL);
13447                 if (ret < 0)
13448                         return ret;
13449
13450                 fence = reservation_object_get_excl_rcu(obj->resv);
13451                 if (fence) {
13452                         add_rps_boost_after_vblank(new_state->crtc, fence);
13453                         dma_fence_put(fence);
13454                 }
13455         } else {
13456                 add_rps_boost_after_vblank(new_state->crtc, new_state->fence);
13457         }
13458
13459         /*
13460          * We declare pageflips to be interactive and so merit a small bias
13461          * towards upclocking to deliver the frame on time. By only changing
13462          * the RPS thresholds to sample more regularly and aim for higher
13463          * clocks we can hopefully deliver low power workloads (like kodi)
13464          * that are not quite steady state without resorting to forcing
13465          * maximum clocks following a vblank miss (see do_rps_boost()).
13466          */
13467         if (!intel_state->rps_interactive) {
13468                 intel_rps_mark_interactive(dev_priv, true);
13469                 intel_state->rps_interactive = true;
13470         }
13471
13472         return 0;
13473 }
13474
13475 /**
13476  * intel_cleanup_plane_fb - Cleans up an fb after plane use
13477  * @plane: drm plane to clean up for
13478  * @old_state: the state from the previous modeset
13479  *
13480  * Cleans up a framebuffer that has just been removed from a plane.
13481  *
13482  * Must be called with struct_mutex held.
13483  */
13484 void
13485 intel_cleanup_plane_fb(struct drm_plane *plane,
13486                        struct drm_plane_state *old_state)
13487 {
13488         struct intel_atomic_state *intel_state =
13489                 to_intel_atomic_state(old_state->state);
13490         struct drm_i915_private *dev_priv = to_i915(plane->dev);
13491
13492         if (intel_state->rps_interactive) {
13493                 intel_rps_mark_interactive(dev_priv, false);
13494                 intel_state->rps_interactive = false;
13495         }
13496
13497         /* Should only be called after a successful intel_prepare_plane_fb()! */
13498         mutex_lock(&dev_priv->drm.struct_mutex);
13499         intel_plane_unpin_fb(to_intel_plane_state(old_state));
13500         mutex_unlock(&dev_priv->drm.struct_mutex);
13501 }
13502
13503 int
13504 skl_max_scale(const struct intel_crtc_state *crtc_state,
13505               u32 pixel_format)
13506 {
13507         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
13508         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
13509         int max_scale, mult;
13510         int crtc_clock, max_dotclk, tmpclk1, tmpclk2;
13511
13512         if (!crtc_state->base.enable)
13513                 return DRM_PLANE_HELPER_NO_SCALING;
13514
13515         crtc_clock = crtc_state->base.adjusted_mode.crtc_clock;
13516         max_dotclk = to_intel_atomic_state(crtc_state->base.state)->cdclk.logical.cdclk;
13517
13518         if (IS_GEMINILAKE(dev_priv) || INTEL_GEN(dev_priv) >= 10)
13519                 max_dotclk *= 2;
13520
13521         if (WARN_ON_ONCE(!crtc_clock || max_dotclk < crtc_clock))
13522                 return DRM_PLANE_HELPER_NO_SCALING;
13523
13524         /*
13525          * skl max scale is lower of:
13526          *    close to 3 but not 3, -1 is for that purpose
13527          *            or
13528          *    cdclk/crtc_clock
13529          */
13530         mult = pixel_format == DRM_FORMAT_NV12 ? 2 : 3;
13531         tmpclk1 = (1 << 16) * mult - 1;
13532         tmpclk2 = (1 << 8) * ((max_dotclk << 8) / crtc_clock);
13533         max_scale = min(tmpclk1, tmpclk2);
13534
13535         return max_scale;
13536 }
13537
13538 static void intel_begin_crtc_commit(struct drm_crtc *crtc,
13539                                     struct drm_crtc_state *old_crtc_state)
13540 {
13541         struct drm_device *dev = crtc->dev;
13542         struct drm_i915_private *dev_priv = to_i915(dev);
13543         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
13544         struct intel_crtc_state *old_intel_cstate =
13545                 to_intel_crtc_state(old_crtc_state);
13546         struct intel_atomic_state *old_intel_state =
13547                 to_intel_atomic_state(old_crtc_state->state);
13548         struct intel_crtc_state *intel_cstate =
13549                 intel_atomic_get_new_crtc_state(old_intel_state, intel_crtc);
13550         bool modeset = needs_modeset(&intel_cstate->base);
13551
13552         if (!modeset &&
13553             (intel_cstate->base.color_mgmt_changed ||
13554              intel_cstate->update_pipe)) {
13555                 intel_color_set_csc(&intel_cstate->base);
13556                 intel_color_load_luts(&intel_cstate->base);
13557         }
13558
13559         /* Perform vblank evasion around commit operation */
13560         intel_pipe_update_start(intel_cstate);
13561
13562         if (modeset)
13563                 goto out;
13564
13565         if (intel_cstate->update_pipe)
13566                 intel_update_pipe_config(old_intel_cstate, intel_cstate);
13567         else if (INTEL_GEN(dev_priv) >= 9)
13568                 skl_detach_scalers(intel_cstate);
13569
13570 out:
13571         if (dev_priv->display.atomic_update_watermarks)
13572                 dev_priv->display.atomic_update_watermarks(old_intel_state,
13573                                                            intel_cstate);
13574 }
13575
13576 void intel_crtc_arm_fifo_underrun(struct intel_crtc *crtc,
13577                                   struct intel_crtc_state *crtc_state)
13578 {
13579         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
13580
13581         if (!IS_GEN2(dev_priv))
13582                 intel_set_cpu_fifo_underrun_reporting(dev_priv, crtc->pipe, true);
13583
13584         if (crtc_state->has_pch_encoder) {
13585                 enum pipe pch_transcoder =
13586                         intel_crtc_pch_transcoder(crtc);
13587
13588                 intel_set_pch_fifo_underrun_reporting(dev_priv, pch_transcoder, true);
13589         }
13590 }
13591
13592 static void intel_finish_crtc_commit(struct drm_crtc *crtc,
13593                                      struct drm_crtc_state *old_crtc_state)
13594 {
13595         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
13596         struct intel_atomic_state *old_intel_state =
13597                 to_intel_atomic_state(old_crtc_state->state);
13598         struct intel_crtc_state *new_crtc_state =
13599                 intel_atomic_get_new_crtc_state(old_intel_state, intel_crtc);
13600
13601         intel_pipe_update_end(new_crtc_state);
13602
13603         if (new_crtc_state->update_pipe &&
13604             !needs_modeset(&new_crtc_state->base) &&
13605             old_crtc_state->mode.private_flags & I915_MODE_FLAG_INHERITED)
13606                 intel_crtc_arm_fifo_underrun(intel_crtc, new_crtc_state);
13607 }
13608
13609 /**
13610  * intel_plane_destroy - destroy a plane
13611  * @plane: plane to destroy
13612  *
13613  * Common destruction function for all types of planes (primary, cursor,
13614  * sprite).
13615  */
13616 void intel_plane_destroy(struct drm_plane *plane)
13617 {
13618         drm_plane_cleanup(plane);
13619         kfree(to_intel_plane(plane));
13620 }
13621
13622 static bool i8xx_plane_format_mod_supported(struct drm_plane *_plane,
13623                                             u32 format, u64 modifier)
13624 {
13625         switch (modifier) {
13626         case DRM_FORMAT_MOD_LINEAR:
13627         case I915_FORMAT_MOD_X_TILED:
13628                 break;
13629         default:
13630                 return false;
13631         }
13632
13633         switch (format) {
13634         case DRM_FORMAT_C8:
13635         case DRM_FORMAT_RGB565:
13636         case DRM_FORMAT_XRGB1555:
13637         case DRM_FORMAT_XRGB8888:
13638                 return modifier == DRM_FORMAT_MOD_LINEAR ||
13639                         modifier == I915_FORMAT_MOD_X_TILED;
13640         default:
13641                 return false;
13642         }
13643 }
13644
13645 static bool i965_plane_format_mod_supported(struct drm_plane *_plane,
13646                                             u32 format, u64 modifier)
13647 {
13648         switch (modifier) {
13649         case DRM_FORMAT_MOD_LINEAR:
13650         case I915_FORMAT_MOD_X_TILED:
13651                 break;
13652         default:
13653                 return false;
13654         }
13655
13656         switch (format) {
13657         case DRM_FORMAT_C8:
13658         case DRM_FORMAT_RGB565:
13659         case DRM_FORMAT_XRGB8888:
13660         case DRM_FORMAT_XBGR8888:
13661         case DRM_FORMAT_XRGB2101010:
13662         case DRM_FORMAT_XBGR2101010:
13663                 return modifier == DRM_FORMAT_MOD_LINEAR ||
13664                         modifier == I915_FORMAT_MOD_X_TILED;
13665         default:
13666                 return false;
13667         }
13668 }
13669
13670 static bool intel_cursor_format_mod_supported(struct drm_plane *_plane,
13671                                               u32 format, u64 modifier)
13672 {
13673         return modifier == DRM_FORMAT_MOD_LINEAR &&
13674                 format == DRM_FORMAT_ARGB8888;
13675 }
13676
13677 static const struct drm_plane_funcs i965_plane_funcs = {
13678         .update_plane = drm_atomic_helper_update_plane,
13679         .disable_plane = drm_atomic_helper_disable_plane,
13680         .destroy = intel_plane_destroy,
13681         .atomic_get_property = intel_plane_atomic_get_property,
13682         .atomic_set_property = intel_plane_atomic_set_property,
13683         .atomic_duplicate_state = intel_plane_duplicate_state,
13684         .atomic_destroy_state = intel_plane_destroy_state,
13685         .format_mod_supported = i965_plane_format_mod_supported,
13686 };
13687
13688 static const struct drm_plane_funcs i8xx_plane_funcs = {
13689         .update_plane = drm_atomic_helper_update_plane,
13690         .disable_plane = drm_atomic_helper_disable_plane,
13691         .destroy = intel_plane_destroy,
13692         .atomic_get_property = intel_plane_atomic_get_property,
13693         .atomic_set_property = intel_plane_atomic_set_property,
13694         .atomic_duplicate_state = intel_plane_duplicate_state,
13695         .atomic_destroy_state = intel_plane_destroy_state,
13696         .format_mod_supported = i8xx_plane_format_mod_supported,
13697 };
13698
13699 static int
13700 intel_legacy_cursor_update(struct drm_plane *plane,
13701                            struct drm_crtc *crtc,
13702                            struct drm_framebuffer *fb,
13703                            int crtc_x, int crtc_y,
13704                            unsigned int crtc_w, unsigned int crtc_h,
13705                            uint32_t src_x, uint32_t src_y,
13706                            uint32_t src_w, uint32_t src_h,
13707                            struct drm_modeset_acquire_ctx *ctx)
13708 {
13709         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
13710         int ret;
13711         struct drm_plane_state *old_plane_state, *new_plane_state;
13712         struct intel_plane *intel_plane = to_intel_plane(plane);
13713         struct drm_framebuffer *old_fb;
13714         struct intel_crtc_state *crtc_state =
13715                 to_intel_crtc_state(crtc->state);
13716         struct intel_crtc_state *new_crtc_state;
13717
13718         /*
13719          * When crtc is inactive or there is a modeset pending,
13720          * wait for it to complete in the slowpath
13721          */
13722         if (!crtc_state->base.active || needs_modeset(&crtc_state->base) ||
13723             crtc_state->update_pipe)
13724                 goto slow;
13725
13726         old_plane_state = plane->state;
13727         /*
13728          * Don't do an async update if there is an outstanding commit modifying
13729          * the plane.  This prevents our async update's changes from getting
13730          * overridden by a previous synchronous update's state.
13731          */
13732         if (old_plane_state->commit &&
13733             !try_wait_for_completion(&old_plane_state->commit->hw_done))
13734                 goto slow;
13735
13736         /*
13737          * If any parameters change that may affect watermarks,
13738          * take the slowpath. Only changing fb or position should be
13739          * in the fastpath.
13740          */
13741         if (old_plane_state->crtc != crtc ||
13742             old_plane_state->src_w != src_w ||
13743             old_plane_state->src_h != src_h ||
13744             old_plane_state->crtc_w != crtc_w ||
13745             old_plane_state->crtc_h != crtc_h ||
13746             !old_plane_state->fb != !fb)
13747                 goto slow;
13748
13749         new_plane_state = intel_plane_duplicate_state(plane);
13750         if (!new_plane_state)
13751                 return -ENOMEM;
13752
13753         new_crtc_state = to_intel_crtc_state(intel_crtc_duplicate_state(crtc));
13754         if (!new_crtc_state) {
13755                 ret = -ENOMEM;
13756                 goto out_free;
13757         }
13758
13759         drm_atomic_set_fb_for_plane(new_plane_state, fb);
13760
13761         new_plane_state->src_x = src_x;
13762         new_plane_state->src_y = src_y;
13763         new_plane_state->src_w = src_w;
13764         new_plane_state->src_h = src_h;
13765         new_plane_state->crtc_x = crtc_x;
13766         new_plane_state->crtc_y = crtc_y;
13767         new_plane_state->crtc_w = crtc_w;
13768         new_plane_state->crtc_h = crtc_h;
13769
13770         ret = intel_plane_atomic_check_with_state(crtc_state, new_crtc_state,
13771                                                   to_intel_plane_state(old_plane_state),
13772                                                   to_intel_plane_state(new_plane_state));
13773         if (ret)
13774                 goto out_free;
13775
13776         ret = mutex_lock_interruptible(&dev_priv->drm.struct_mutex);
13777         if (ret)
13778                 goto out_free;
13779
13780         ret = intel_plane_pin_fb(to_intel_plane_state(new_plane_state));
13781         if (ret)
13782                 goto out_unlock;
13783
13784         intel_fb_obj_flush(intel_fb_obj(fb), ORIGIN_FLIP);
13785
13786         old_fb = old_plane_state->fb;
13787         i915_gem_track_fb(intel_fb_obj(old_fb), intel_fb_obj(fb),
13788                           intel_plane->frontbuffer_bit);
13789
13790         /* Swap plane state */
13791         plane->state = new_plane_state;
13792
13793         /*
13794          * We cannot swap crtc_state as it may be in use by an atomic commit or
13795          * page flip that's running simultaneously. If we swap crtc_state and
13796          * destroy the old state, we will cause a use-after-free there.
13797          *
13798          * Only update active_planes, which is needed for our internal
13799          * bookkeeping. Either value will do the right thing when updating
13800          * planes atomically. If the cursor was part of the atomic update then
13801          * we would have taken the slowpath.
13802          */
13803         crtc_state->active_planes = new_crtc_state->active_planes;
13804
13805         if (plane->state->visible) {
13806                 trace_intel_update_plane(plane, to_intel_crtc(crtc));
13807                 intel_plane->update_plane(intel_plane, crtc_state,
13808                                           to_intel_plane_state(plane->state));
13809         } else {
13810                 trace_intel_disable_plane(plane, to_intel_crtc(crtc));
13811                 intel_plane->disable_plane(intel_plane, crtc_state);
13812         }
13813
13814         intel_plane_unpin_fb(to_intel_plane_state(old_plane_state));
13815
13816 out_unlock:
13817         mutex_unlock(&dev_priv->drm.struct_mutex);
13818 out_free:
13819         if (new_crtc_state)
13820                 intel_crtc_destroy_state(crtc, &new_crtc_state->base);
13821         if (ret)
13822                 intel_plane_destroy_state(plane, new_plane_state);
13823         else
13824                 intel_plane_destroy_state(plane, old_plane_state);
13825         return ret;
13826
13827 slow:
13828         return drm_atomic_helper_update_plane(plane, crtc, fb,
13829                                               crtc_x, crtc_y, crtc_w, crtc_h,
13830                                               src_x, src_y, src_w, src_h, ctx);
13831 }
13832
13833 static const struct drm_plane_funcs intel_cursor_plane_funcs = {
13834         .update_plane = intel_legacy_cursor_update,
13835         .disable_plane = drm_atomic_helper_disable_plane,
13836         .destroy = intel_plane_destroy,
13837         .atomic_get_property = intel_plane_atomic_get_property,
13838         .atomic_set_property = intel_plane_atomic_set_property,
13839         .atomic_duplicate_state = intel_plane_duplicate_state,
13840         .atomic_destroy_state = intel_plane_destroy_state,
13841         .format_mod_supported = intel_cursor_format_mod_supported,
13842 };
13843
13844 static bool i9xx_plane_has_fbc(struct drm_i915_private *dev_priv,
13845                                enum i9xx_plane_id i9xx_plane)
13846 {
13847         if (!HAS_FBC(dev_priv))
13848                 return false;
13849
13850         if (IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
13851                 return i9xx_plane == PLANE_A; /* tied to pipe A */
13852         else if (IS_IVYBRIDGE(dev_priv))
13853                 return i9xx_plane == PLANE_A || i9xx_plane == PLANE_B ||
13854                         i9xx_plane == PLANE_C;
13855         else if (INTEL_GEN(dev_priv) >= 4)
13856                 return i9xx_plane == PLANE_A || i9xx_plane == PLANE_B;
13857         else
13858                 return i9xx_plane == PLANE_A;
13859 }
13860
13861 static struct intel_plane *
13862 intel_primary_plane_create(struct drm_i915_private *dev_priv, enum pipe pipe)
13863 {
13864         struct intel_plane *plane;
13865         const struct drm_plane_funcs *plane_funcs;
13866         unsigned int supported_rotations;
13867         unsigned int possible_crtcs;
13868         const u64 *modifiers;
13869         const u32 *formats;
13870         int num_formats;
13871         int ret;
13872
13873         if (INTEL_GEN(dev_priv) >= 9)
13874                 return skl_universal_plane_create(dev_priv, pipe,
13875                                                   PLANE_PRIMARY);
13876
13877         plane = intel_plane_alloc();
13878         if (IS_ERR(plane))
13879                 return plane;
13880
13881         plane->pipe = pipe;
13882         /*
13883          * On gen2/3 only plane A can do FBC, but the panel fitter and LVDS
13884          * port is hooked to pipe B. Hence we want plane A feeding pipe B.
13885          */
13886         if (HAS_FBC(dev_priv) && INTEL_GEN(dev_priv) < 4)
13887                 plane->i9xx_plane = (enum i9xx_plane_id) !pipe;
13888         else
13889                 plane->i9xx_plane = (enum i9xx_plane_id) pipe;
13890         plane->id = PLANE_PRIMARY;
13891         plane->frontbuffer_bit = INTEL_FRONTBUFFER(pipe, plane->id);
13892
13893         plane->has_fbc = i9xx_plane_has_fbc(dev_priv, plane->i9xx_plane);
13894         if (plane->has_fbc) {
13895                 struct intel_fbc *fbc = &dev_priv->fbc;
13896
13897                 fbc->possible_framebuffer_bits |= plane->frontbuffer_bit;
13898         }
13899
13900         if (INTEL_GEN(dev_priv) >= 4) {
13901                 formats = i965_primary_formats;
13902                 num_formats = ARRAY_SIZE(i965_primary_formats);
13903                 modifiers = i9xx_format_modifiers;
13904
13905                 plane->max_stride = i9xx_plane_max_stride;
13906                 plane->update_plane = i9xx_update_plane;
13907                 plane->disable_plane = i9xx_disable_plane;
13908                 plane->get_hw_state = i9xx_plane_get_hw_state;
13909                 plane->check_plane = i9xx_plane_check;
13910
13911                 plane_funcs = &i965_plane_funcs;
13912         } else {
13913                 formats = i8xx_primary_formats;
13914                 num_formats = ARRAY_SIZE(i8xx_primary_formats);
13915                 modifiers = i9xx_format_modifiers;
13916
13917                 plane->max_stride = i9xx_plane_max_stride;
13918                 plane->update_plane = i9xx_update_plane;
13919                 plane->disable_plane = i9xx_disable_plane;
13920                 plane->get_hw_state = i9xx_plane_get_hw_state;
13921                 plane->check_plane = i9xx_plane_check;
13922
13923                 plane_funcs = &i8xx_plane_funcs;
13924         }
13925
13926         possible_crtcs = BIT(pipe);
13927
13928         if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv))
13929                 ret = drm_universal_plane_init(&dev_priv->drm, &plane->base,
13930                                                possible_crtcs, plane_funcs,
13931                                                formats, num_formats, modifiers,
13932                                                DRM_PLANE_TYPE_PRIMARY,
13933                                                "primary %c", pipe_name(pipe));
13934         else
13935                 ret = drm_universal_plane_init(&dev_priv->drm, &plane->base,
13936                                                possible_crtcs, plane_funcs,
13937                                                formats, num_formats, modifiers,
13938                                                DRM_PLANE_TYPE_PRIMARY,
13939                                                "plane %c",
13940                                                plane_name(plane->i9xx_plane));
13941         if (ret)
13942                 goto fail;
13943
13944         if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_B) {
13945                 supported_rotations =
13946                         DRM_MODE_ROTATE_0 | DRM_MODE_ROTATE_180 |
13947                         DRM_MODE_REFLECT_X;
13948         } else if (INTEL_GEN(dev_priv) >= 4) {
13949                 supported_rotations =
13950                         DRM_MODE_ROTATE_0 | DRM_MODE_ROTATE_180;
13951         } else {
13952                 supported_rotations = DRM_MODE_ROTATE_0;
13953         }
13954
13955         if (INTEL_GEN(dev_priv) >= 4)
13956                 drm_plane_create_rotation_property(&plane->base,
13957                                                    DRM_MODE_ROTATE_0,
13958                                                    supported_rotations);
13959
13960         drm_plane_helper_add(&plane->base, &intel_plane_helper_funcs);
13961
13962         return plane;
13963
13964 fail:
13965         intel_plane_free(plane);
13966
13967         return ERR_PTR(ret);
13968 }
13969
13970 static struct intel_plane *
13971 intel_cursor_plane_create(struct drm_i915_private *dev_priv,
13972                           enum pipe pipe)
13973 {
13974         unsigned int possible_crtcs;
13975         struct intel_plane *cursor;
13976         int ret;
13977
13978         cursor = intel_plane_alloc();
13979         if (IS_ERR(cursor))
13980                 return cursor;
13981
13982         cursor->pipe = pipe;
13983         cursor->i9xx_plane = (enum i9xx_plane_id) pipe;
13984         cursor->id = PLANE_CURSOR;
13985         cursor->frontbuffer_bit = INTEL_FRONTBUFFER(pipe, cursor->id);
13986
13987         if (IS_I845G(dev_priv) || IS_I865G(dev_priv)) {
13988                 cursor->max_stride = i845_cursor_max_stride;
13989                 cursor->update_plane = i845_update_cursor;
13990                 cursor->disable_plane = i845_disable_cursor;
13991                 cursor->get_hw_state = i845_cursor_get_hw_state;
13992                 cursor->check_plane = i845_check_cursor;
13993         } else {
13994                 cursor->max_stride = i9xx_cursor_max_stride;
13995                 cursor->update_plane = i9xx_update_cursor;
13996                 cursor->disable_plane = i9xx_disable_cursor;
13997                 cursor->get_hw_state = i9xx_cursor_get_hw_state;
13998                 cursor->check_plane = i9xx_check_cursor;
13999         }
14000
14001         cursor->cursor.base = ~0;
14002         cursor->cursor.cntl = ~0;
14003
14004         if (IS_I845G(dev_priv) || IS_I865G(dev_priv) || HAS_CUR_FBC(dev_priv))
14005                 cursor->cursor.size = ~0;
14006
14007         possible_crtcs = BIT(pipe);
14008
14009         ret = drm_universal_plane_init(&dev_priv->drm, &cursor->base,
14010                                        possible_crtcs, &intel_cursor_plane_funcs,
14011                                        intel_cursor_formats,
14012                                        ARRAY_SIZE(intel_cursor_formats),
14013                                        cursor_format_modifiers,
14014                                        DRM_PLANE_TYPE_CURSOR,
14015                                        "cursor %c", pipe_name(pipe));
14016         if (ret)
14017                 goto fail;
14018
14019         if (INTEL_GEN(dev_priv) >= 4)
14020                 drm_plane_create_rotation_property(&cursor->base,
14021                                                    DRM_MODE_ROTATE_0,
14022                                                    DRM_MODE_ROTATE_0 |
14023                                                    DRM_MODE_ROTATE_180);
14024
14025         drm_plane_helper_add(&cursor->base, &intel_plane_helper_funcs);
14026
14027         return cursor;
14028
14029 fail:
14030         intel_plane_free(cursor);
14031
14032         return ERR_PTR(ret);
14033 }
14034
14035 static void intel_crtc_init_scalers(struct intel_crtc *crtc,
14036                                     struct intel_crtc_state *crtc_state)
14037 {
14038         struct intel_crtc_scaler_state *scaler_state =
14039                 &crtc_state->scaler_state;
14040         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
14041         int i;
14042
14043         crtc->num_scalers = dev_priv->info.num_scalers[crtc->pipe];
14044         if (!crtc->num_scalers)
14045                 return;
14046
14047         for (i = 0; i < crtc->num_scalers; i++) {
14048                 struct intel_scaler *scaler = &scaler_state->scalers[i];
14049
14050                 scaler->in_use = 0;
14051                 scaler->mode = 0;
14052         }
14053
14054         scaler_state->scaler_id = -1;
14055 }
14056
14057 static int intel_crtc_init(struct drm_i915_private *dev_priv, enum pipe pipe)
14058 {
14059         struct intel_crtc *intel_crtc;
14060         struct intel_crtc_state *crtc_state = NULL;
14061         struct intel_plane *primary = NULL;
14062         struct intel_plane *cursor = NULL;
14063         int sprite, ret;
14064
14065         intel_crtc = kzalloc(sizeof(*intel_crtc), GFP_KERNEL);
14066         if (!intel_crtc)
14067                 return -ENOMEM;
14068
14069         crtc_state = kzalloc(sizeof(*crtc_state), GFP_KERNEL);
14070         if (!crtc_state) {
14071                 ret = -ENOMEM;
14072                 goto fail;
14073         }
14074         intel_crtc->config = crtc_state;
14075         intel_crtc->base.state = &crtc_state->base;
14076         crtc_state->base.crtc = &intel_crtc->base;
14077
14078         primary = intel_primary_plane_create(dev_priv, pipe);
14079         if (IS_ERR(primary)) {
14080                 ret = PTR_ERR(primary);
14081                 goto fail;
14082         }
14083         intel_crtc->plane_ids_mask |= BIT(primary->id);
14084
14085         for_each_sprite(dev_priv, pipe, sprite) {
14086                 struct intel_plane *plane;
14087
14088                 plane = intel_sprite_plane_create(dev_priv, pipe, sprite);
14089                 if (IS_ERR(plane)) {
14090                         ret = PTR_ERR(plane);
14091                         goto fail;
14092                 }
14093                 intel_crtc->plane_ids_mask |= BIT(plane->id);
14094         }
14095
14096         cursor = intel_cursor_plane_create(dev_priv, pipe);
14097         if (IS_ERR(cursor)) {
14098                 ret = PTR_ERR(cursor);
14099                 goto fail;
14100         }
14101         intel_crtc->plane_ids_mask |= BIT(cursor->id);
14102
14103         ret = drm_crtc_init_with_planes(&dev_priv->drm, &intel_crtc->base,
14104                                         &primary->base, &cursor->base,
14105                                         &intel_crtc_funcs,
14106                                         "pipe %c", pipe_name(pipe));
14107         if (ret)
14108                 goto fail;
14109
14110         intel_crtc->pipe = pipe;
14111
14112         /* initialize shared scalers */
14113         intel_crtc_init_scalers(intel_crtc, crtc_state);
14114
14115         BUG_ON(pipe >= ARRAY_SIZE(dev_priv->pipe_to_crtc_mapping) ||
14116                dev_priv->pipe_to_crtc_mapping[pipe] != NULL);
14117         dev_priv->pipe_to_crtc_mapping[pipe] = intel_crtc;
14118
14119         if (INTEL_GEN(dev_priv) < 9) {
14120                 enum i9xx_plane_id i9xx_plane = primary->i9xx_plane;
14121
14122                 BUG_ON(i9xx_plane >= ARRAY_SIZE(dev_priv->plane_to_crtc_mapping) ||
14123                        dev_priv->plane_to_crtc_mapping[i9xx_plane] != NULL);
14124                 dev_priv->plane_to_crtc_mapping[i9xx_plane] = intel_crtc;
14125         }
14126
14127         drm_crtc_helper_add(&intel_crtc->base, &intel_helper_funcs);
14128
14129         intel_color_init(&intel_crtc->base);
14130
14131         WARN_ON(drm_crtc_index(&intel_crtc->base) != intel_crtc->pipe);
14132
14133         return 0;
14134
14135 fail:
14136         /*
14137          * drm_mode_config_cleanup() will free up any
14138          * crtcs/planes already initialized.
14139          */
14140         kfree(crtc_state);
14141         kfree(intel_crtc);
14142
14143         return ret;
14144 }
14145
14146 int intel_get_pipe_from_crtc_id_ioctl(struct drm_device *dev, void *data,
14147                                       struct drm_file *file)
14148 {
14149         struct drm_i915_get_pipe_from_crtc_id *pipe_from_crtc_id = data;
14150         struct drm_crtc *drmmode_crtc;
14151         struct intel_crtc *crtc;
14152
14153         drmmode_crtc = drm_crtc_find(dev, file, pipe_from_crtc_id->crtc_id);
14154         if (!drmmode_crtc)
14155                 return -ENOENT;
14156
14157         crtc = to_intel_crtc(drmmode_crtc);
14158         pipe_from_crtc_id->pipe = crtc->pipe;
14159
14160         return 0;
14161 }
14162
14163 static int intel_encoder_clones(struct intel_encoder *encoder)
14164 {
14165         struct drm_device *dev = encoder->base.dev;
14166         struct intel_encoder *source_encoder;
14167         int index_mask = 0;
14168         int entry = 0;
14169
14170         for_each_intel_encoder(dev, source_encoder) {
14171                 if (encoders_cloneable(encoder, source_encoder))
14172                         index_mask |= (1 << entry);
14173
14174                 entry++;
14175         }
14176
14177         return index_mask;
14178 }
14179
14180 static bool has_edp_a(struct drm_i915_private *dev_priv)
14181 {
14182         if (!IS_MOBILE(dev_priv))
14183                 return false;
14184
14185         if ((I915_READ(DP_A) & DP_DETECTED) == 0)
14186                 return false;
14187
14188         if (IS_GEN5(dev_priv) && (I915_READ(FUSE_STRAP) & ILK_eDP_A_DISABLE))
14189                 return false;
14190
14191         return true;
14192 }
14193
14194 static bool intel_crt_present(struct drm_i915_private *dev_priv)
14195 {
14196         if (INTEL_GEN(dev_priv) >= 9)
14197                 return false;
14198
14199         if (IS_HSW_ULT(dev_priv) || IS_BDW_ULT(dev_priv))
14200                 return false;
14201
14202         if (IS_CHERRYVIEW(dev_priv))
14203                 return false;
14204
14205         if (HAS_PCH_LPT_H(dev_priv) &&
14206             I915_READ(SFUSE_STRAP) & SFUSE_STRAP_CRT_DISABLED)
14207                 return false;
14208
14209         /* DDI E can't be used if DDI A requires 4 lanes */
14210         if (HAS_DDI(dev_priv) && I915_READ(DDI_BUF_CTL(PORT_A)) & DDI_A_4_LANES)
14211                 return false;
14212
14213         if (!dev_priv->vbt.int_crt_support)
14214                 return false;
14215
14216         return true;
14217 }
14218
14219 void intel_pps_unlock_regs_wa(struct drm_i915_private *dev_priv)
14220 {
14221         int pps_num;
14222         int pps_idx;
14223
14224         if (HAS_DDI(dev_priv))
14225                 return;
14226         /*
14227          * This w/a is needed at least on CPT/PPT, but to be sure apply it
14228          * everywhere where registers can be write protected.
14229          */
14230         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
14231                 pps_num = 2;
14232         else
14233                 pps_num = 1;
14234
14235         for (pps_idx = 0; pps_idx < pps_num; pps_idx++) {
14236                 u32 val = I915_READ(PP_CONTROL(pps_idx));
14237
14238                 val = (val & ~PANEL_UNLOCK_MASK) | PANEL_UNLOCK_REGS;
14239                 I915_WRITE(PP_CONTROL(pps_idx), val);
14240         }
14241 }
14242
14243 static void intel_pps_init(struct drm_i915_private *dev_priv)
14244 {
14245         if (HAS_PCH_SPLIT(dev_priv) || IS_GEN9_LP(dev_priv))
14246                 dev_priv->pps_mmio_base = PCH_PPS_BASE;
14247         else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
14248                 dev_priv->pps_mmio_base = VLV_PPS_BASE;
14249         else
14250                 dev_priv->pps_mmio_base = PPS_BASE;
14251
14252         intel_pps_unlock_regs_wa(dev_priv);
14253 }
14254
14255 static void intel_setup_outputs(struct drm_i915_private *dev_priv)
14256 {
14257         struct intel_encoder *encoder;
14258         bool dpd_is_edp = false;
14259
14260         intel_pps_init(dev_priv);
14261
14262         if (!HAS_DISPLAY(dev_priv))
14263                 return;
14264
14265         /*
14266          * intel_edp_init_connector() depends on this completing first, to
14267          * prevent the registeration of both eDP and LVDS and the incorrect
14268          * sharing of the PPS.
14269          */
14270         intel_lvds_init(dev_priv);
14271
14272         if (intel_crt_present(dev_priv))
14273                 intel_crt_init(dev_priv);
14274
14275         if (IS_ICELAKE(dev_priv)) {
14276                 intel_ddi_init(dev_priv, PORT_A);
14277                 intel_ddi_init(dev_priv, PORT_B);
14278                 intel_ddi_init(dev_priv, PORT_C);
14279                 intel_ddi_init(dev_priv, PORT_D);
14280                 intel_ddi_init(dev_priv, PORT_E);
14281                 intel_ddi_init(dev_priv, PORT_F);
14282                 icl_dsi_init(dev_priv);
14283         } else if (IS_GEN9_LP(dev_priv)) {
14284                 /*
14285                  * FIXME: Broxton doesn't support port detection via the
14286                  * DDI_BUF_CTL_A or SFUSE_STRAP registers, find another way to
14287                  * detect the ports.
14288                  */
14289                 intel_ddi_init(dev_priv, PORT_A);
14290                 intel_ddi_init(dev_priv, PORT_B);
14291                 intel_ddi_init(dev_priv, PORT_C);
14292
14293                 vlv_dsi_init(dev_priv);
14294         } else if (HAS_DDI(dev_priv)) {
14295                 int found;
14296
14297                 /*
14298                  * Haswell uses DDI functions to detect digital outputs.
14299                  * On SKL pre-D0 the strap isn't connected, so we assume
14300                  * it's there.
14301                  */
14302                 found = I915_READ(DDI_BUF_CTL(PORT_A)) & DDI_INIT_DISPLAY_DETECTED;
14303                 /* WaIgnoreDDIAStrap: skl */
14304                 if (found || IS_GEN9_BC(dev_priv))
14305                         intel_ddi_init(dev_priv, PORT_A);
14306
14307                 /* DDI B, C, D, and F detection is indicated by the SFUSE_STRAP
14308                  * register */
14309                 found = I915_READ(SFUSE_STRAP);
14310
14311                 if (found & SFUSE_STRAP_DDIB_DETECTED)
14312                         intel_ddi_init(dev_priv, PORT_B);
14313                 if (found & SFUSE_STRAP_DDIC_DETECTED)
14314                         intel_ddi_init(dev_priv, PORT_C);
14315                 if (found & SFUSE_STRAP_DDID_DETECTED)
14316                         intel_ddi_init(dev_priv, PORT_D);
14317                 if (found & SFUSE_STRAP_DDIF_DETECTED)
14318                         intel_ddi_init(dev_priv, PORT_F);
14319                 /*
14320                  * On SKL we don't have a way to detect DDI-E so we rely on VBT.
14321                  */
14322                 if (IS_GEN9_BC(dev_priv) &&
14323                     (dev_priv->vbt.ddi_port_info[PORT_E].supports_dp ||
14324                      dev_priv->vbt.ddi_port_info[PORT_E].supports_dvi ||
14325                      dev_priv->vbt.ddi_port_info[PORT_E].supports_hdmi))
14326                         intel_ddi_init(dev_priv, PORT_E);
14327
14328         } else if (HAS_PCH_SPLIT(dev_priv)) {
14329                 int found;
14330                 dpd_is_edp = intel_dp_is_port_edp(dev_priv, PORT_D);
14331
14332                 if (has_edp_a(dev_priv))
14333                         intel_dp_init(dev_priv, DP_A, PORT_A);
14334
14335                 if (I915_READ(PCH_HDMIB) & SDVO_DETECTED) {
14336                         /* PCH SDVOB multiplex with HDMIB */
14337                         found = intel_sdvo_init(dev_priv, PCH_SDVOB, PORT_B);
14338                         if (!found)
14339                                 intel_hdmi_init(dev_priv, PCH_HDMIB, PORT_B);
14340                         if (!found && (I915_READ(PCH_DP_B) & DP_DETECTED))
14341                                 intel_dp_init(dev_priv, PCH_DP_B, PORT_B);
14342                 }
14343
14344                 if (I915_READ(PCH_HDMIC) & SDVO_DETECTED)
14345                         intel_hdmi_init(dev_priv, PCH_HDMIC, PORT_C);
14346
14347                 if (!dpd_is_edp && I915_READ(PCH_HDMID) & SDVO_DETECTED)
14348                         intel_hdmi_init(dev_priv, PCH_HDMID, PORT_D);
14349
14350                 if (I915_READ(PCH_DP_C) & DP_DETECTED)
14351                         intel_dp_init(dev_priv, PCH_DP_C, PORT_C);
14352
14353                 if (I915_READ(PCH_DP_D) & DP_DETECTED)
14354                         intel_dp_init(dev_priv, PCH_DP_D, PORT_D);
14355         } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
14356                 bool has_edp, has_port;
14357
14358                 /*
14359                  * The DP_DETECTED bit is the latched state of the DDC
14360                  * SDA pin at boot. However since eDP doesn't require DDC
14361                  * (no way to plug in a DP->HDMI dongle) the DDC pins for
14362                  * eDP ports may have been muxed to an alternate function.
14363                  * Thus we can't rely on the DP_DETECTED bit alone to detect
14364                  * eDP ports. Consult the VBT as well as DP_DETECTED to
14365                  * detect eDP ports.
14366                  *
14367                  * Sadly the straps seem to be missing sometimes even for HDMI
14368                  * ports (eg. on Voyo V3 - CHT x7-Z8700), so check both strap
14369                  * and VBT for the presence of the port. Additionally we can't
14370                  * trust the port type the VBT declares as we've seen at least
14371                  * HDMI ports that the VBT claim are DP or eDP.
14372                  */
14373                 has_edp = intel_dp_is_port_edp(dev_priv, PORT_B);
14374                 has_port = intel_bios_is_port_present(dev_priv, PORT_B);
14375                 if (I915_READ(VLV_DP_B) & DP_DETECTED || has_port)
14376                         has_edp &= intel_dp_init(dev_priv, VLV_DP_B, PORT_B);
14377                 if ((I915_READ(VLV_HDMIB) & SDVO_DETECTED || has_port) && !has_edp)
14378                         intel_hdmi_init(dev_priv, VLV_HDMIB, PORT_B);
14379
14380                 has_edp = intel_dp_is_port_edp(dev_priv, PORT_C);
14381                 has_port = intel_bios_is_port_present(dev_priv, PORT_C);
14382                 if (I915_READ(VLV_DP_C) & DP_DETECTED || has_port)
14383                         has_edp &= intel_dp_init(dev_priv, VLV_DP_C, PORT_C);
14384                 if ((I915_READ(VLV_HDMIC) & SDVO_DETECTED || has_port) && !has_edp)
14385                         intel_hdmi_init(dev_priv, VLV_HDMIC, PORT_C);
14386
14387                 if (IS_CHERRYVIEW(dev_priv)) {
14388                         /*
14389                          * eDP not supported on port D,
14390                          * so no need to worry about it
14391                          */
14392                         has_port = intel_bios_is_port_present(dev_priv, PORT_D);
14393                         if (I915_READ(CHV_DP_D) & DP_DETECTED || has_port)
14394                                 intel_dp_init(dev_priv, CHV_DP_D, PORT_D);
14395                         if (I915_READ(CHV_HDMID) & SDVO_DETECTED || has_port)
14396                                 intel_hdmi_init(dev_priv, CHV_HDMID, PORT_D);
14397                 }
14398
14399                 vlv_dsi_init(dev_priv);
14400         } else if (!IS_GEN2(dev_priv) && !IS_PINEVIEW(dev_priv)) {
14401                 bool found = false;
14402
14403                 if (I915_READ(GEN3_SDVOB) & SDVO_DETECTED) {
14404                         DRM_DEBUG_KMS("probing SDVOB\n");
14405                         found = intel_sdvo_init(dev_priv, GEN3_SDVOB, PORT_B);
14406                         if (!found && IS_G4X(dev_priv)) {
14407                                 DRM_DEBUG_KMS("probing HDMI on SDVOB\n");
14408                                 intel_hdmi_init(dev_priv, GEN4_HDMIB, PORT_B);
14409                         }
14410
14411                         if (!found && IS_G4X(dev_priv))
14412                                 intel_dp_init(dev_priv, DP_B, PORT_B);
14413                 }
14414
14415                 /* Before G4X SDVOC doesn't have its own detect register */
14416
14417                 if (I915_READ(GEN3_SDVOB) & SDVO_DETECTED) {
14418                         DRM_DEBUG_KMS("probing SDVOC\n");
14419                         found = intel_sdvo_init(dev_priv, GEN3_SDVOC, PORT_C);
14420                 }
14421
14422                 if (!found && (I915_READ(GEN3_SDVOC) & SDVO_DETECTED)) {
14423
14424                         if (IS_G4X(dev_priv)) {
14425                                 DRM_DEBUG_KMS("probing HDMI on SDVOC\n");
14426                                 intel_hdmi_init(dev_priv, GEN4_HDMIC, PORT_C);
14427                         }
14428                         if (IS_G4X(dev_priv))
14429                                 intel_dp_init(dev_priv, DP_C, PORT_C);
14430                 }
14431
14432                 if (IS_G4X(dev_priv) && (I915_READ(DP_D) & DP_DETECTED))
14433                         intel_dp_init(dev_priv, DP_D, PORT_D);
14434         } else if (IS_GEN2(dev_priv))
14435                 intel_dvo_init(dev_priv);
14436
14437         if (SUPPORTS_TV(dev_priv))
14438                 intel_tv_init(dev_priv);
14439
14440         intel_psr_init(dev_priv);
14441
14442         for_each_intel_encoder(&dev_priv->drm, encoder) {
14443                 encoder->base.possible_crtcs = encoder->crtc_mask;
14444                 encoder->base.possible_clones =
14445                         intel_encoder_clones(encoder);
14446         }
14447
14448         intel_init_pch_refclk(dev_priv);
14449
14450         drm_helper_move_panel_connectors_to_head(&dev_priv->drm);
14451 }
14452
14453 static void intel_user_framebuffer_destroy(struct drm_framebuffer *fb)
14454 {
14455         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
14456         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
14457
14458         drm_framebuffer_cleanup(fb);
14459
14460         i915_gem_object_lock(obj);
14461         WARN_ON(!obj->framebuffer_references--);
14462         i915_gem_object_unlock(obj);
14463
14464         i915_gem_object_put(obj);
14465
14466         kfree(intel_fb);
14467 }
14468
14469 static int intel_user_framebuffer_create_handle(struct drm_framebuffer *fb,
14470                                                 struct drm_file *file,
14471                                                 unsigned int *handle)
14472 {
14473         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
14474
14475         if (obj->userptr.mm) {
14476                 DRM_DEBUG("attempting to use a userptr for a framebuffer, denied\n");
14477                 return -EINVAL;
14478         }
14479
14480         return drm_gem_handle_create(file, &obj->base, handle);
14481 }
14482
14483 static int intel_user_framebuffer_dirty(struct drm_framebuffer *fb,
14484                                         struct drm_file *file,
14485                                         unsigned flags, unsigned color,
14486                                         struct drm_clip_rect *clips,
14487                                         unsigned num_clips)
14488 {
14489         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
14490
14491         i915_gem_object_flush_if_display(obj);
14492         intel_fb_obj_flush(obj, ORIGIN_DIRTYFB);
14493
14494         return 0;
14495 }
14496
14497 static const struct drm_framebuffer_funcs intel_fb_funcs = {
14498         .destroy = intel_user_framebuffer_destroy,
14499         .create_handle = intel_user_framebuffer_create_handle,
14500         .dirty = intel_user_framebuffer_dirty,
14501 };
14502
14503 static
14504 u32 intel_fb_pitch_limit(struct drm_i915_private *dev_priv,
14505                          u32 pixel_format, u64 fb_modifier)
14506 {
14507         struct intel_crtc *crtc;
14508         struct intel_plane *plane;
14509
14510         /*
14511          * We assume the primary plane for pipe A has
14512          * the highest stride limits of them all.
14513          */
14514         crtc = intel_get_crtc_for_pipe(dev_priv, PIPE_A);
14515         plane = to_intel_plane(crtc->base.primary);
14516
14517         return plane->max_stride(plane, pixel_format, fb_modifier,
14518                                  DRM_MODE_ROTATE_0);
14519 }
14520
14521 static int intel_framebuffer_init(struct intel_framebuffer *intel_fb,
14522                                   struct drm_i915_gem_object *obj,
14523                                   struct drm_mode_fb_cmd2 *mode_cmd)
14524 {
14525         struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
14526         struct drm_framebuffer *fb = &intel_fb->base;
14527         u32 pitch_limit;
14528         unsigned int tiling, stride;
14529         int ret = -EINVAL;
14530         int i;
14531
14532         i915_gem_object_lock(obj);
14533         obj->framebuffer_references++;
14534         tiling = i915_gem_object_get_tiling(obj);
14535         stride = i915_gem_object_get_stride(obj);
14536         i915_gem_object_unlock(obj);
14537
14538         if (mode_cmd->flags & DRM_MODE_FB_MODIFIERS) {
14539                 /*
14540                  * If there's a fence, enforce that
14541                  * the fb modifier and tiling mode match.
14542                  */
14543                 if (tiling != I915_TILING_NONE &&
14544                     tiling != intel_fb_modifier_to_tiling(mode_cmd->modifier[0])) {
14545                         DRM_DEBUG_KMS("tiling_mode doesn't match fb modifier\n");
14546                         goto err;
14547                 }
14548         } else {
14549                 if (tiling == I915_TILING_X) {
14550                         mode_cmd->modifier[0] = I915_FORMAT_MOD_X_TILED;
14551                 } else if (tiling == I915_TILING_Y) {
14552                         DRM_DEBUG_KMS("No Y tiling for legacy addfb\n");
14553                         goto err;
14554                 }
14555         }
14556
14557         if (!drm_any_plane_has_format(&dev_priv->drm,
14558                                       mode_cmd->pixel_format,
14559                                       mode_cmd->modifier[0])) {
14560                 struct drm_format_name_buf format_name;
14561
14562                 DRM_DEBUG_KMS("unsupported pixel format %s / modifier 0x%llx\n",
14563                               drm_get_format_name(mode_cmd->pixel_format,
14564                                                   &format_name),
14565                               mode_cmd->modifier[0]);
14566                 goto err;
14567         }
14568
14569         /*
14570          * gen2/3 display engine uses the fence if present,
14571          * so the tiling mode must match the fb modifier exactly.
14572          */
14573         if (INTEL_GEN(dev_priv) < 4 &&
14574             tiling != intel_fb_modifier_to_tiling(mode_cmd->modifier[0])) {
14575                 DRM_DEBUG_KMS("tiling_mode must match fb modifier exactly on gen2/3\n");
14576                 goto err;
14577         }
14578
14579         pitch_limit = intel_fb_pitch_limit(dev_priv, mode_cmd->pixel_format,
14580                                            mode_cmd->modifier[0]);
14581         if (mode_cmd->pitches[0] > pitch_limit) {
14582                 DRM_DEBUG_KMS("%s pitch (%u) must be at most %d\n",
14583                               mode_cmd->modifier[0] != DRM_FORMAT_MOD_LINEAR ?
14584                               "tiled" : "linear",
14585                               mode_cmd->pitches[0], pitch_limit);
14586                 goto err;
14587         }
14588
14589         /*
14590          * If there's a fence, enforce that
14591          * the fb pitch and fence stride match.
14592          */
14593         if (tiling != I915_TILING_NONE && mode_cmd->pitches[0] != stride) {
14594                 DRM_DEBUG_KMS("pitch (%d) must match tiling stride (%d)\n",
14595                               mode_cmd->pitches[0], stride);
14596                 goto err;
14597         }
14598
14599         /* FIXME need to adjust LINOFF/TILEOFF accordingly. */
14600         if (mode_cmd->offsets[0] != 0)
14601                 goto err;
14602
14603         drm_helper_mode_fill_fb_struct(&dev_priv->drm, fb, mode_cmd);
14604
14605         if (fb->format->format == DRM_FORMAT_NV12 &&
14606             (fb->width < SKL_MIN_YUV_420_SRC_W ||
14607              fb->height < SKL_MIN_YUV_420_SRC_H ||
14608              (fb->width % 4) != 0 || (fb->height % 4) != 0)) {
14609                 DRM_DEBUG_KMS("src dimensions not correct for NV12\n");
14610                 goto err;
14611         }
14612
14613         for (i = 0; i < fb->format->num_planes; i++) {
14614                 u32 stride_alignment;
14615
14616                 if (mode_cmd->handles[i] != mode_cmd->handles[0]) {
14617                         DRM_DEBUG_KMS("bad plane %d handle\n", i);
14618                         goto err;
14619                 }
14620
14621                 stride_alignment = intel_fb_stride_alignment(fb, i);
14622
14623                 /*
14624                  * Display WA #0531: skl,bxt,kbl,glk
14625                  *
14626                  * Render decompression and plane width > 3840
14627                  * combined with horizontal panning requires the
14628                  * plane stride to be a multiple of 4. We'll just
14629                  * require the entire fb to accommodate that to avoid
14630                  * potential runtime errors at plane configuration time.
14631                  */
14632                 if (IS_GEN9(dev_priv) && i == 0 && fb->width > 3840 &&
14633                     is_ccs_modifier(fb->modifier))
14634                         stride_alignment *= 4;
14635
14636                 if (fb->pitches[i] & (stride_alignment - 1)) {
14637                         DRM_DEBUG_KMS("plane %d pitch (%d) must be at least %u byte aligned\n",
14638                                       i, fb->pitches[i], stride_alignment);
14639                         goto err;
14640                 }
14641
14642                 fb->obj[i] = &obj->base;
14643         }
14644
14645         ret = intel_fill_fb_info(dev_priv, fb);
14646         if (ret)
14647                 goto err;
14648
14649         ret = drm_framebuffer_init(&dev_priv->drm, fb, &intel_fb_funcs);
14650         if (ret) {
14651                 DRM_ERROR("framebuffer init failed %d\n", ret);
14652                 goto err;
14653         }
14654
14655         return 0;
14656
14657 err:
14658         i915_gem_object_lock(obj);
14659         obj->framebuffer_references--;
14660         i915_gem_object_unlock(obj);
14661         return ret;
14662 }
14663
14664 static struct drm_framebuffer *
14665 intel_user_framebuffer_create(struct drm_device *dev,
14666                               struct drm_file *filp,
14667                               const struct drm_mode_fb_cmd2 *user_mode_cmd)
14668 {
14669         struct drm_framebuffer *fb;
14670         struct drm_i915_gem_object *obj;
14671         struct drm_mode_fb_cmd2 mode_cmd = *user_mode_cmd;
14672
14673         obj = i915_gem_object_lookup(filp, mode_cmd.handles[0]);
14674         if (!obj)
14675                 return ERR_PTR(-ENOENT);
14676
14677         fb = intel_framebuffer_create(obj, &mode_cmd);
14678         if (IS_ERR(fb))
14679                 i915_gem_object_put(obj);
14680
14681         return fb;
14682 }
14683
14684 static void intel_atomic_state_free(struct drm_atomic_state *state)
14685 {
14686         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
14687
14688         drm_atomic_state_default_release(state);
14689
14690         i915_sw_fence_fini(&intel_state->commit_ready);
14691
14692         kfree(state);
14693 }
14694
14695 static enum drm_mode_status
14696 intel_mode_valid(struct drm_device *dev,
14697                  const struct drm_display_mode *mode)
14698 {
14699         struct drm_i915_private *dev_priv = to_i915(dev);
14700         int hdisplay_max, htotal_max;
14701         int vdisplay_max, vtotal_max;
14702
14703         /*
14704          * Can't reject DBLSCAN here because Xorg ddxen can add piles
14705          * of DBLSCAN modes to the output's mode list when they detect
14706          * the scaling mode property on the connector. And they don't
14707          * ask the kernel to validate those modes in any way until
14708          * modeset time at which point the client gets a protocol error.
14709          * So in order to not upset those clients we silently ignore the
14710          * DBLSCAN flag on such connectors. For other connectors we will
14711          * reject modes with the DBLSCAN flag in encoder->compute_config().
14712          * And we always reject DBLSCAN modes in connector->mode_valid()
14713          * as we never want such modes on the connector's mode list.
14714          */
14715
14716         if (mode->vscan > 1)
14717                 return MODE_NO_VSCAN;
14718
14719         if (mode->flags & DRM_MODE_FLAG_HSKEW)
14720                 return MODE_H_ILLEGAL;
14721
14722         if (mode->flags & (DRM_MODE_FLAG_CSYNC |
14723                            DRM_MODE_FLAG_NCSYNC |
14724                            DRM_MODE_FLAG_PCSYNC))
14725                 return MODE_HSYNC;
14726
14727         if (mode->flags & (DRM_MODE_FLAG_BCAST |
14728                            DRM_MODE_FLAG_PIXMUX |
14729                            DRM_MODE_FLAG_CLKDIV2))
14730                 return MODE_BAD;
14731
14732         if (INTEL_GEN(dev_priv) >= 9 ||
14733             IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv)) {
14734                 hdisplay_max = 8192; /* FDI max 4096 handled elsewhere */
14735                 vdisplay_max = 4096;
14736                 htotal_max = 8192;
14737                 vtotal_max = 8192;
14738         } else if (INTEL_GEN(dev_priv) >= 3) {
14739                 hdisplay_max = 4096;
14740                 vdisplay_max = 4096;
14741                 htotal_max = 8192;
14742                 vtotal_max = 8192;
14743         } else {
14744                 hdisplay_max = 2048;
14745                 vdisplay_max = 2048;
14746                 htotal_max = 4096;
14747                 vtotal_max = 4096;
14748         }
14749
14750         if (mode->hdisplay > hdisplay_max ||
14751             mode->hsync_start > htotal_max ||
14752             mode->hsync_end > htotal_max ||
14753             mode->htotal > htotal_max)
14754                 return MODE_H_ILLEGAL;
14755
14756         if (mode->vdisplay > vdisplay_max ||
14757             mode->vsync_start > vtotal_max ||
14758             mode->vsync_end > vtotal_max ||
14759             mode->vtotal > vtotal_max)
14760                 return MODE_V_ILLEGAL;
14761
14762         return MODE_OK;
14763 }
14764
14765 static const struct drm_mode_config_funcs intel_mode_funcs = {
14766         .fb_create = intel_user_framebuffer_create,
14767         .get_format_info = intel_get_format_info,
14768         .output_poll_changed = intel_fbdev_output_poll_changed,
14769         .mode_valid = intel_mode_valid,
14770         .atomic_check = intel_atomic_check,
14771         .atomic_commit = intel_atomic_commit,
14772         .atomic_state_alloc = intel_atomic_state_alloc,
14773         .atomic_state_clear = intel_atomic_state_clear,
14774         .atomic_state_free = intel_atomic_state_free,
14775 };
14776
14777 /**
14778  * intel_init_display_hooks - initialize the display modesetting hooks
14779  * @dev_priv: device private
14780  */
14781 void intel_init_display_hooks(struct drm_i915_private *dev_priv)
14782 {
14783         intel_init_cdclk_hooks(dev_priv);
14784
14785         if (INTEL_GEN(dev_priv) >= 9) {
14786                 dev_priv->display.get_pipe_config = haswell_get_pipe_config;
14787                 dev_priv->display.get_initial_plane_config =
14788                         skylake_get_initial_plane_config;
14789                 dev_priv->display.crtc_compute_clock =
14790                         haswell_crtc_compute_clock;
14791                 dev_priv->display.crtc_enable = haswell_crtc_enable;
14792                 dev_priv->display.crtc_disable = haswell_crtc_disable;
14793         } else if (HAS_DDI(dev_priv)) {
14794                 dev_priv->display.get_pipe_config = haswell_get_pipe_config;
14795                 dev_priv->display.get_initial_plane_config =
14796                         i9xx_get_initial_plane_config;
14797                 dev_priv->display.crtc_compute_clock =
14798                         haswell_crtc_compute_clock;
14799                 dev_priv->display.crtc_enable = haswell_crtc_enable;
14800                 dev_priv->display.crtc_disable = haswell_crtc_disable;
14801         } else if (HAS_PCH_SPLIT(dev_priv)) {
14802                 dev_priv->display.get_pipe_config = ironlake_get_pipe_config;
14803                 dev_priv->display.get_initial_plane_config =
14804                         i9xx_get_initial_plane_config;
14805                 dev_priv->display.crtc_compute_clock =
14806                         ironlake_crtc_compute_clock;
14807                 dev_priv->display.crtc_enable = ironlake_crtc_enable;
14808                 dev_priv->display.crtc_disable = ironlake_crtc_disable;
14809         } else if (IS_CHERRYVIEW(dev_priv)) {
14810                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
14811                 dev_priv->display.get_initial_plane_config =
14812                         i9xx_get_initial_plane_config;
14813                 dev_priv->display.crtc_compute_clock = chv_crtc_compute_clock;
14814                 dev_priv->display.crtc_enable = valleyview_crtc_enable;
14815                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
14816         } else if (IS_VALLEYVIEW(dev_priv)) {
14817                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
14818                 dev_priv->display.get_initial_plane_config =
14819                         i9xx_get_initial_plane_config;
14820                 dev_priv->display.crtc_compute_clock = vlv_crtc_compute_clock;
14821                 dev_priv->display.crtc_enable = valleyview_crtc_enable;
14822                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
14823         } else if (IS_G4X(dev_priv)) {
14824                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
14825                 dev_priv->display.get_initial_plane_config =
14826                         i9xx_get_initial_plane_config;
14827                 dev_priv->display.crtc_compute_clock = g4x_crtc_compute_clock;
14828                 dev_priv->display.crtc_enable = i9xx_crtc_enable;
14829                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
14830         } else if (IS_PINEVIEW(dev_priv)) {
14831                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
14832                 dev_priv->display.get_initial_plane_config =
14833                         i9xx_get_initial_plane_config;
14834                 dev_priv->display.crtc_compute_clock = pnv_crtc_compute_clock;
14835                 dev_priv->display.crtc_enable = i9xx_crtc_enable;
14836                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
14837         } else if (!IS_GEN2(dev_priv)) {
14838                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
14839                 dev_priv->display.get_initial_plane_config =
14840                         i9xx_get_initial_plane_config;
14841                 dev_priv->display.crtc_compute_clock = i9xx_crtc_compute_clock;
14842                 dev_priv->display.crtc_enable = i9xx_crtc_enable;
14843                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
14844         } else {
14845                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
14846                 dev_priv->display.get_initial_plane_config =
14847                         i9xx_get_initial_plane_config;
14848                 dev_priv->display.crtc_compute_clock = i8xx_crtc_compute_clock;
14849                 dev_priv->display.crtc_enable = i9xx_crtc_enable;
14850                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
14851         }
14852
14853         if (IS_GEN5(dev_priv)) {
14854                 dev_priv->display.fdi_link_train = ironlake_fdi_link_train;
14855         } else if (IS_GEN6(dev_priv)) {
14856                 dev_priv->display.fdi_link_train = gen6_fdi_link_train;
14857         } else if (IS_IVYBRIDGE(dev_priv)) {
14858                 /* FIXME: detect B0+ stepping and use auto training */
14859                 dev_priv->display.fdi_link_train = ivb_manual_fdi_link_train;
14860         } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
14861                 dev_priv->display.fdi_link_train = hsw_fdi_link_train;
14862         }
14863
14864         if (INTEL_GEN(dev_priv) >= 9)
14865                 dev_priv->display.update_crtcs = skl_update_crtcs;
14866         else
14867                 dev_priv->display.update_crtcs = intel_update_crtcs;
14868 }
14869
14870 /* Disable the VGA plane that we never use */
14871 static void i915_disable_vga(struct drm_i915_private *dev_priv)
14872 {
14873         struct pci_dev *pdev = dev_priv->drm.pdev;
14874         u8 sr1;
14875         i915_reg_t vga_reg = i915_vgacntrl_reg(dev_priv);
14876
14877         /* WaEnableVGAAccessThroughIOPort:ctg,elk,ilk,snb,ivb,vlv,hsw */
14878         vga_get_uninterruptible(pdev, VGA_RSRC_LEGACY_IO);
14879         outb(SR01, VGA_SR_INDEX);
14880         sr1 = inb(VGA_SR_DATA);
14881         outb(sr1 | 1<<5, VGA_SR_DATA);
14882         vga_put(pdev, VGA_RSRC_LEGACY_IO);
14883         udelay(300);
14884
14885         I915_WRITE(vga_reg, VGA_DISP_DISABLE);
14886         POSTING_READ(vga_reg);
14887 }
14888
14889 void intel_modeset_init_hw(struct drm_device *dev)
14890 {
14891         struct drm_i915_private *dev_priv = to_i915(dev);
14892
14893         intel_update_cdclk(dev_priv);
14894         intel_dump_cdclk_state(&dev_priv->cdclk.hw, "Current CDCLK");
14895         dev_priv->cdclk.logical = dev_priv->cdclk.actual = dev_priv->cdclk.hw;
14896 }
14897
14898 /*
14899  * Calculate what we think the watermarks should be for the state we've read
14900  * out of the hardware and then immediately program those watermarks so that
14901  * we ensure the hardware settings match our internal state.
14902  *
14903  * We can calculate what we think WM's should be by creating a duplicate of the
14904  * current state (which was constructed during hardware readout) and running it
14905  * through the atomic check code to calculate new watermark values in the
14906  * state object.
14907  */
14908 static void sanitize_watermarks(struct drm_device *dev)
14909 {
14910         struct drm_i915_private *dev_priv = to_i915(dev);
14911         struct drm_atomic_state *state;
14912         struct intel_atomic_state *intel_state;
14913         struct drm_crtc *crtc;
14914         struct drm_crtc_state *cstate;
14915         struct drm_modeset_acquire_ctx ctx;
14916         int ret;
14917         int i;
14918
14919         /* Only supported on platforms that use atomic watermark design */
14920         if (!dev_priv->display.optimize_watermarks)
14921                 return;
14922
14923         /*
14924          * We need to hold connection_mutex before calling duplicate_state so
14925          * that the connector loop is protected.
14926          */
14927         drm_modeset_acquire_init(&ctx, 0);
14928 retry:
14929         ret = drm_modeset_lock_all_ctx(dev, &ctx);
14930         if (ret == -EDEADLK) {
14931                 drm_modeset_backoff(&ctx);
14932                 goto retry;
14933         } else if (WARN_ON(ret)) {
14934                 goto fail;
14935         }
14936
14937         state = drm_atomic_helper_duplicate_state(dev, &ctx);
14938         if (WARN_ON(IS_ERR(state)))
14939                 goto fail;
14940
14941         intel_state = to_intel_atomic_state(state);
14942
14943         /*
14944          * Hardware readout is the only time we don't want to calculate
14945          * intermediate watermarks (since we don't trust the current
14946          * watermarks).
14947          */
14948         if (!HAS_GMCH_DISPLAY(dev_priv))
14949                 intel_state->skip_intermediate_wm = true;
14950
14951         ret = intel_atomic_check(dev, state);
14952         if (ret) {
14953                 /*
14954                  * If we fail here, it means that the hardware appears to be
14955                  * programmed in a way that shouldn't be possible, given our
14956                  * understanding of watermark requirements.  This might mean a
14957                  * mistake in the hardware readout code or a mistake in the
14958                  * watermark calculations for a given platform.  Raise a WARN
14959                  * so that this is noticeable.
14960                  *
14961                  * If this actually happens, we'll have to just leave the
14962                  * BIOS-programmed watermarks untouched and hope for the best.
14963                  */
14964                 WARN(true, "Could not determine valid watermarks for inherited state\n");
14965                 goto put_state;
14966         }
14967
14968         /* Write calculated watermark values back */
14969         for_each_new_crtc_in_state(state, crtc, cstate, i) {
14970                 struct intel_crtc_state *cs = to_intel_crtc_state(cstate);
14971
14972                 cs->wm.need_postvbl_update = true;
14973                 dev_priv->display.optimize_watermarks(intel_state, cs);
14974
14975                 to_intel_crtc_state(crtc->state)->wm = cs->wm;
14976         }
14977
14978 put_state:
14979         drm_atomic_state_put(state);
14980 fail:
14981         drm_modeset_drop_locks(&ctx);
14982         drm_modeset_acquire_fini(&ctx);
14983 }
14984
14985 static void intel_update_fdi_pll_freq(struct drm_i915_private *dev_priv)
14986 {
14987         if (IS_GEN5(dev_priv)) {
14988                 u32 fdi_pll_clk =
14989                         I915_READ(FDI_PLL_BIOS_0) & FDI_PLL_FB_CLOCK_MASK;
14990
14991                 dev_priv->fdi_pll_freq = (fdi_pll_clk + 2) * 10000;
14992         } else if (IS_GEN6(dev_priv) || IS_IVYBRIDGE(dev_priv)) {
14993                 dev_priv->fdi_pll_freq = 270000;
14994         } else {
14995                 return;
14996         }
14997
14998         DRM_DEBUG_DRIVER("FDI PLL freq=%d\n", dev_priv->fdi_pll_freq);
14999 }
15000
15001 static int intel_initial_commit(struct drm_device *dev)
15002 {
15003         struct drm_atomic_state *state = NULL;
15004         struct drm_modeset_acquire_ctx ctx;
15005         struct drm_crtc *crtc;
15006         struct drm_crtc_state *crtc_state;
15007         int ret = 0;
15008
15009         state = drm_atomic_state_alloc(dev);
15010         if (!state)
15011                 return -ENOMEM;
15012
15013         drm_modeset_acquire_init(&ctx, 0);
15014
15015 retry:
15016         state->acquire_ctx = &ctx;
15017
15018         drm_for_each_crtc(crtc, dev) {
15019                 crtc_state = drm_atomic_get_crtc_state(state, crtc);
15020                 if (IS_ERR(crtc_state)) {
15021                         ret = PTR_ERR(crtc_state);
15022                         goto out;
15023                 }
15024
15025                 if (crtc_state->active) {
15026                         ret = drm_atomic_add_affected_planes(state, crtc);
15027                         if (ret)
15028                                 goto out;
15029
15030                         /*
15031                          * FIXME hack to force a LUT update to avoid the
15032                          * plane update forcing the pipe gamma on without
15033                          * having a proper LUT loaded. Remove once we
15034                          * have readout for pipe gamma enable.
15035                          */
15036                         crtc_state->color_mgmt_changed = true;
15037                 }
15038         }
15039
15040         ret = drm_atomic_commit(state);
15041
15042 out:
15043         if (ret == -EDEADLK) {
15044                 drm_atomic_state_clear(state);
15045                 drm_modeset_backoff(&ctx);
15046                 goto retry;
15047         }
15048
15049         drm_atomic_state_put(state);
15050
15051         drm_modeset_drop_locks(&ctx);
15052         drm_modeset_acquire_fini(&ctx);
15053
15054         return ret;
15055 }
15056
15057 int intel_modeset_init(struct drm_device *dev)
15058 {
15059         struct drm_i915_private *dev_priv = to_i915(dev);
15060         struct i915_ggtt *ggtt = &dev_priv->ggtt;
15061         enum pipe pipe;
15062         struct intel_crtc *crtc;
15063         int ret;
15064
15065         dev_priv->modeset_wq = alloc_ordered_workqueue("i915_modeset", 0);
15066
15067         drm_mode_config_init(dev);
15068
15069         dev->mode_config.min_width = 0;
15070         dev->mode_config.min_height = 0;
15071
15072         dev->mode_config.preferred_depth = 24;
15073         dev->mode_config.prefer_shadow = 1;
15074
15075         dev->mode_config.allow_fb_modifiers = true;
15076
15077         dev->mode_config.funcs = &intel_mode_funcs;
15078
15079         init_llist_head(&dev_priv->atomic_helper.free_list);
15080         INIT_WORK(&dev_priv->atomic_helper.free_work,
15081                   intel_atomic_helper_free_state_worker);
15082
15083         intel_init_quirks(dev_priv);
15084
15085         intel_fbc_init(dev_priv);
15086
15087         intel_init_pm(dev_priv);
15088
15089         /*
15090          * There may be no VBT; and if the BIOS enabled SSC we can
15091          * just keep using it to avoid unnecessary flicker.  Whereas if the
15092          * BIOS isn't using it, don't assume it will work even if the VBT
15093          * indicates as much.
15094          */
15095         if (HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv)) {
15096                 bool bios_lvds_use_ssc = !!(I915_READ(PCH_DREF_CONTROL) &
15097                                             DREF_SSC1_ENABLE);
15098
15099                 if (dev_priv->vbt.lvds_use_ssc != bios_lvds_use_ssc) {
15100                         DRM_DEBUG_KMS("SSC %sabled by BIOS, overriding VBT which says %sabled\n",
15101                                      bios_lvds_use_ssc ? "en" : "dis",
15102                                      dev_priv->vbt.lvds_use_ssc ? "en" : "dis");
15103                         dev_priv->vbt.lvds_use_ssc = bios_lvds_use_ssc;
15104                 }
15105         }
15106
15107         /* maximum framebuffer dimensions */
15108         if (IS_GEN2(dev_priv)) {
15109                 dev->mode_config.max_width = 2048;
15110                 dev->mode_config.max_height = 2048;
15111         } else if (IS_GEN3(dev_priv)) {
15112                 dev->mode_config.max_width = 4096;
15113                 dev->mode_config.max_height = 4096;
15114         } else {
15115                 dev->mode_config.max_width = 8192;
15116                 dev->mode_config.max_height = 8192;
15117         }
15118
15119         if (IS_I845G(dev_priv) || IS_I865G(dev_priv)) {
15120                 dev->mode_config.cursor_width = IS_I845G(dev_priv) ? 64 : 512;
15121                 dev->mode_config.cursor_height = 1023;
15122         } else if (IS_GEN2(dev_priv)) {
15123                 dev->mode_config.cursor_width = 64;
15124                 dev->mode_config.cursor_height = 64;
15125         } else {
15126                 dev->mode_config.cursor_width = 256;
15127                 dev->mode_config.cursor_height = 256;
15128         }
15129
15130         dev->mode_config.fb_base = ggtt->gmadr.start;
15131
15132         DRM_DEBUG_KMS("%d display pipe%s available.\n",
15133                       INTEL_INFO(dev_priv)->num_pipes,
15134                       INTEL_INFO(dev_priv)->num_pipes > 1 ? "s" : "");
15135
15136         for_each_pipe(dev_priv, pipe) {
15137                 ret = intel_crtc_init(dev_priv, pipe);
15138                 if (ret) {
15139                         drm_mode_config_cleanup(dev);
15140                         return ret;
15141                 }
15142         }
15143
15144         intel_shared_dpll_init(dev);
15145         intel_update_fdi_pll_freq(dev_priv);
15146
15147         intel_update_czclk(dev_priv);
15148         intel_modeset_init_hw(dev);
15149
15150         if (dev_priv->max_cdclk_freq == 0)
15151                 intel_update_max_cdclk(dev_priv);
15152
15153         /* Just disable it once at startup */
15154         i915_disable_vga(dev_priv);
15155         intel_setup_outputs(dev_priv);
15156
15157         drm_modeset_lock_all(dev);
15158         intel_modeset_setup_hw_state(dev, dev->mode_config.acquire_ctx);
15159         drm_modeset_unlock_all(dev);
15160
15161         for_each_intel_crtc(dev, crtc) {
15162                 struct intel_initial_plane_config plane_config = {};
15163
15164                 if (!crtc->active)
15165                         continue;
15166
15167                 /*
15168                  * Note that reserving the BIOS fb up front prevents us
15169                  * from stuffing other stolen allocations like the ring
15170                  * on top.  This prevents some ugliness at boot time, and
15171                  * can even allow for smooth boot transitions if the BIOS
15172                  * fb is large enough for the active pipe configuration.
15173                  */
15174                 dev_priv->display.get_initial_plane_config(crtc,
15175                                                            &plane_config);
15176
15177                 /*
15178                  * If the fb is shared between multiple heads, we'll
15179                  * just get the first one.
15180                  */
15181                 intel_find_initial_plane_obj(crtc, &plane_config);
15182         }
15183
15184         /*
15185          * Make sure hardware watermarks really match the state we read out.
15186          * Note that we need to do this after reconstructing the BIOS fb's
15187          * since the watermark calculation done here will use pstate->fb.
15188          */
15189         if (!HAS_GMCH_DISPLAY(dev_priv))
15190                 sanitize_watermarks(dev);
15191
15192         /*
15193          * Force all active planes to recompute their states. So that on
15194          * mode_setcrtc after probe, all the intel_plane_state variables
15195          * are already calculated and there is no assert_plane warnings
15196          * during bootup.
15197          */
15198         ret = intel_initial_commit(dev);
15199         if (ret)
15200                 DRM_DEBUG_KMS("Initial commit in probe failed.\n");
15201
15202         return 0;
15203 }
15204
15205 void i830_enable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe)
15206 {
15207         struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
15208         /* 640x480@60Hz, ~25175 kHz */
15209         struct dpll clock = {
15210                 .m1 = 18,
15211                 .m2 = 7,
15212                 .p1 = 13,
15213                 .p2 = 4,
15214                 .n = 2,
15215         };
15216         u32 dpll, fp;
15217         int i;
15218
15219         WARN_ON(i9xx_calc_dpll_params(48000, &clock) != 25154);
15220
15221         DRM_DEBUG_KMS("enabling pipe %c due to force quirk (vco=%d dot=%d)\n",
15222                       pipe_name(pipe), clock.vco, clock.dot);
15223
15224         fp = i9xx_dpll_compute_fp(&clock);
15225         dpll = (I915_READ(DPLL(pipe)) & DPLL_DVO_2X_MODE) |
15226                 DPLL_VGA_MODE_DIS |
15227                 ((clock.p1 - 2) << DPLL_FPA01_P1_POST_DIV_SHIFT) |
15228                 PLL_P2_DIVIDE_BY_4 |
15229                 PLL_REF_INPUT_DREFCLK |
15230                 DPLL_VCO_ENABLE;
15231
15232         I915_WRITE(FP0(pipe), fp);
15233         I915_WRITE(FP1(pipe), fp);
15234
15235         I915_WRITE(HTOTAL(pipe), (640 - 1) | ((800 - 1) << 16));
15236         I915_WRITE(HBLANK(pipe), (640 - 1) | ((800 - 1) << 16));
15237         I915_WRITE(HSYNC(pipe), (656 - 1) | ((752 - 1) << 16));
15238         I915_WRITE(VTOTAL(pipe), (480 - 1) | ((525 - 1) << 16));
15239         I915_WRITE(VBLANK(pipe), (480 - 1) | ((525 - 1) << 16));
15240         I915_WRITE(VSYNC(pipe), (490 - 1) | ((492 - 1) << 16));
15241         I915_WRITE(PIPESRC(pipe), ((640 - 1) << 16) | (480 - 1));
15242
15243         /*
15244          * Apparently we need to have VGA mode enabled prior to changing
15245          * the P1/P2 dividers. Otherwise the DPLL will keep using the old
15246          * dividers, even though the register value does change.
15247          */
15248         I915_WRITE(DPLL(pipe), dpll & ~DPLL_VGA_MODE_DIS);
15249         I915_WRITE(DPLL(pipe), dpll);
15250
15251         /* Wait for the clocks to stabilize. */
15252         POSTING_READ(DPLL(pipe));
15253         udelay(150);
15254
15255         /* The pixel multiplier can only be updated once the
15256          * DPLL is enabled and the clocks are stable.
15257          *
15258          * So write it again.
15259          */
15260         I915_WRITE(DPLL(pipe), dpll);
15261
15262         /* We do this three times for luck */
15263         for (i = 0; i < 3 ; i++) {
15264                 I915_WRITE(DPLL(pipe), dpll);
15265                 POSTING_READ(DPLL(pipe));
15266                 udelay(150); /* wait for warmup */
15267         }
15268
15269         I915_WRITE(PIPECONF(pipe), PIPECONF_ENABLE | PIPECONF_PROGRESSIVE);
15270         POSTING_READ(PIPECONF(pipe));
15271
15272         intel_wait_for_pipe_scanline_moving(crtc);
15273 }
15274
15275 void i830_disable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe)
15276 {
15277         struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
15278
15279         DRM_DEBUG_KMS("disabling pipe %c due to force quirk\n",
15280                       pipe_name(pipe));
15281
15282         WARN_ON(I915_READ(DSPCNTR(PLANE_A)) & DISPLAY_PLANE_ENABLE);
15283         WARN_ON(I915_READ(DSPCNTR(PLANE_B)) & DISPLAY_PLANE_ENABLE);
15284         WARN_ON(I915_READ(DSPCNTR(PLANE_C)) & DISPLAY_PLANE_ENABLE);
15285         WARN_ON(I915_READ(CURCNTR(PIPE_A)) & MCURSOR_MODE);
15286         WARN_ON(I915_READ(CURCNTR(PIPE_B)) & MCURSOR_MODE);
15287
15288         I915_WRITE(PIPECONF(pipe), 0);
15289         POSTING_READ(PIPECONF(pipe));
15290
15291         intel_wait_for_pipe_scanline_stopped(crtc);
15292
15293         I915_WRITE(DPLL(pipe), DPLL_VGA_MODE_DIS);
15294         POSTING_READ(DPLL(pipe));
15295 }
15296
15297 static void
15298 intel_sanitize_plane_mapping(struct drm_i915_private *dev_priv)
15299 {
15300         struct intel_crtc *crtc;
15301
15302         if (INTEL_GEN(dev_priv) >= 4)
15303                 return;
15304
15305         for_each_intel_crtc(&dev_priv->drm, crtc) {
15306                 struct intel_plane *plane =
15307                         to_intel_plane(crtc->base.primary);
15308                 struct intel_crtc *plane_crtc;
15309                 enum pipe pipe;
15310
15311                 if (!plane->get_hw_state(plane, &pipe))
15312                         continue;
15313
15314                 if (pipe == crtc->pipe)
15315                         continue;
15316
15317                 DRM_DEBUG_KMS("[PLANE:%d:%s] attached to the wrong pipe, disabling plane\n",
15318                               plane->base.base.id, plane->base.name);
15319
15320                 plane_crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
15321                 intel_plane_disable_noatomic(plane_crtc, plane);
15322         }
15323 }
15324
15325 static bool intel_crtc_has_encoders(struct intel_crtc *crtc)
15326 {
15327         struct drm_device *dev = crtc->base.dev;
15328         struct intel_encoder *encoder;
15329
15330         for_each_encoder_on_crtc(dev, &crtc->base, encoder)
15331                 return true;
15332
15333         return false;
15334 }
15335
15336 static struct intel_connector *intel_encoder_find_connector(struct intel_encoder *encoder)
15337 {
15338         struct drm_device *dev = encoder->base.dev;
15339         struct intel_connector *connector;
15340
15341         for_each_connector_on_encoder(dev, &encoder->base, connector)
15342                 return connector;
15343
15344         return NULL;
15345 }
15346
15347 static bool has_pch_trancoder(struct drm_i915_private *dev_priv,
15348                               enum pipe pch_transcoder)
15349 {
15350         return HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv) ||
15351                 (HAS_PCH_LPT_H(dev_priv) && pch_transcoder == PIPE_A);
15352 }
15353
15354 static void intel_sanitize_crtc(struct intel_crtc *crtc,
15355                                 struct drm_modeset_acquire_ctx *ctx)
15356 {
15357         struct drm_device *dev = crtc->base.dev;
15358         struct drm_i915_private *dev_priv = to_i915(dev);
15359         struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state);
15360         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
15361
15362         /* Clear any frame start delays used for debugging left by the BIOS */
15363         if (crtc->active && !transcoder_is_dsi(cpu_transcoder)) {
15364                 i915_reg_t reg = PIPECONF(cpu_transcoder);
15365
15366                 I915_WRITE(reg,
15367                            I915_READ(reg) & ~PIPECONF_FRAME_START_DELAY_MASK);
15368         }
15369
15370         if (crtc_state->base.active) {
15371                 struct intel_plane *plane;
15372
15373                 /* Disable everything but the primary plane */
15374                 for_each_intel_plane_on_crtc(dev, crtc, plane) {
15375                         const struct intel_plane_state *plane_state =
15376                                 to_intel_plane_state(plane->base.state);
15377
15378                         if (plane_state->base.visible &&
15379                             plane->base.type != DRM_PLANE_TYPE_PRIMARY)
15380                                 intel_plane_disable_noatomic(crtc, plane);
15381                 }
15382         }
15383
15384         /* Adjust the state of the output pipe according to whether we
15385          * have active connectors/encoders. */
15386         if (crtc_state->base.active && !intel_crtc_has_encoders(crtc))
15387                 intel_crtc_disable_noatomic(&crtc->base, ctx);
15388
15389         if (crtc_state->base.active || HAS_GMCH_DISPLAY(dev_priv)) {
15390                 /*
15391                  * We start out with underrun reporting disabled to avoid races.
15392                  * For correct bookkeeping mark this on active crtcs.
15393                  *
15394                  * Also on gmch platforms we dont have any hardware bits to
15395                  * disable the underrun reporting. Which means we need to start
15396                  * out with underrun reporting disabled also on inactive pipes,
15397                  * since otherwise we'll complain about the garbage we read when
15398                  * e.g. coming up after runtime pm.
15399                  *
15400                  * No protection against concurrent access is required - at
15401                  * worst a fifo underrun happens which also sets this to false.
15402                  */
15403                 crtc->cpu_fifo_underrun_disabled = true;
15404                 /*
15405                  * We track the PCH trancoder underrun reporting state
15406                  * within the crtc. With crtc for pipe A housing the underrun
15407                  * reporting state for PCH transcoder A, crtc for pipe B housing
15408                  * it for PCH transcoder B, etc. LPT-H has only PCH transcoder A,
15409                  * and marking underrun reporting as disabled for the non-existing
15410                  * PCH transcoders B and C would prevent enabling the south
15411                  * error interrupt (see cpt_can_enable_serr_int()).
15412                  */
15413                 if (has_pch_trancoder(dev_priv, crtc->pipe))
15414                         crtc->pch_fifo_underrun_disabled = true;
15415         }
15416 }
15417
15418 static bool has_bogus_dpll_config(const struct intel_crtc_state *crtc_state)
15419 {
15420         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
15421
15422         /*
15423          * Some SNB BIOSen (eg. ASUS K53SV) are known to misprogram
15424          * the hardware when a high res displays plugged in. DPLL P
15425          * divider is zero, and the pipe timings are bonkers. We'll
15426          * try to disable everything in that case.
15427          *
15428          * FIXME would be nice to be able to sanitize this state
15429          * without several WARNs, but for now let's take the easy
15430          * road.
15431          */
15432         return IS_GEN6(dev_priv) &&
15433                 crtc_state->base.active &&
15434                 crtc_state->shared_dpll &&
15435                 crtc_state->port_clock == 0;
15436 }
15437
15438 static void intel_sanitize_encoder(struct intel_encoder *encoder)
15439 {
15440         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
15441         struct intel_connector *connector;
15442         struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
15443         struct intel_crtc_state *crtc_state = crtc ?
15444                 to_intel_crtc_state(crtc->base.state) : NULL;
15445
15446         /* We need to check both for a crtc link (meaning that the
15447          * encoder is active and trying to read from a pipe) and the
15448          * pipe itself being active. */
15449         bool has_active_crtc = crtc_state &&
15450                 crtc_state->base.active;
15451
15452         if (crtc_state && has_bogus_dpll_config(crtc_state)) {
15453                 DRM_DEBUG_KMS("BIOS has misprogrammed the hardware. Disabling pipe %c\n",
15454                               pipe_name(crtc->pipe));
15455                 has_active_crtc = false;
15456         }
15457
15458         connector = intel_encoder_find_connector(encoder);
15459         if (connector && !has_active_crtc) {
15460                 DRM_DEBUG_KMS("[ENCODER:%d:%s] has active connectors but no active pipe!\n",
15461                               encoder->base.base.id,
15462                               encoder->base.name);
15463
15464                 /* Connector is active, but has no active pipe. This is
15465                  * fallout from our resume register restoring. Disable
15466                  * the encoder manually again. */
15467                 if (crtc_state) {
15468                         struct drm_encoder *best_encoder;
15469
15470                         DRM_DEBUG_KMS("[ENCODER:%d:%s] manually disabled\n",
15471                                       encoder->base.base.id,
15472                                       encoder->base.name);
15473
15474                         /* avoid oopsing in case the hooks consult best_encoder */
15475                         best_encoder = connector->base.state->best_encoder;
15476                         connector->base.state->best_encoder = &encoder->base;
15477
15478                         if (encoder->disable)
15479                                 encoder->disable(encoder, crtc_state,
15480                                                  connector->base.state);
15481                         if (encoder->post_disable)
15482                                 encoder->post_disable(encoder, crtc_state,
15483                                                       connector->base.state);
15484
15485                         connector->base.state->best_encoder = best_encoder;
15486                 }
15487                 encoder->base.crtc = NULL;
15488
15489                 /* Inconsistent output/port/pipe state happens presumably due to
15490                  * a bug in one of the get_hw_state functions. Or someplace else
15491                  * in our code, like the register restore mess on resume. Clamp
15492                  * things to off as a safer default. */
15493
15494                 connector->base.dpms = DRM_MODE_DPMS_OFF;
15495                 connector->base.encoder = NULL;
15496         }
15497
15498         /* notify opregion of the sanitized encoder state */
15499         intel_opregion_notify_encoder(encoder, connector && has_active_crtc);
15500
15501         if (INTEL_GEN(dev_priv) >= 11)
15502                 icl_sanitize_encoder_pll_mapping(encoder);
15503 }
15504
15505 void i915_redisable_vga_power_on(struct drm_i915_private *dev_priv)
15506 {
15507         i915_reg_t vga_reg = i915_vgacntrl_reg(dev_priv);
15508
15509         if (!(I915_READ(vga_reg) & VGA_DISP_DISABLE)) {
15510                 DRM_DEBUG_KMS("Something enabled VGA plane, disabling it\n");
15511                 i915_disable_vga(dev_priv);
15512         }
15513 }
15514
15515 void i915_redisable_vga(struct drm_i915_private *dev_priv)
15516 {
15517         /* This function can be called both from intel_modeset_setup_hw_state or
15518          * at a very early point in our resume sequence, where the power well
15519          * structures are not yet restored. Since this function is at a very
15520          * paranoid "someone might have enabled VGA while we were not looking"
15521          * level, just check if the power well is enabled instead of trying to
15522          * follow the "don't touch the power well if we don't need it" policy
15523          * the rest of the driver uses. */
15524         if (!intel_display_power_get_if_enabled(dev_priv, POWER_DOMAIN_VGA))
15525                 return;
15526
15527         i915_redisable_vga_power_on(dev_priv);
15528
15529         intel_display_power_put(dev_priv, POWER_DOMAIN_VGA);
15530 }
15531
15532 /* FIXME read out full plane state for all planes */
15533 static void readout_plane_state(struct drm_i915_private *dev_priv)
15534 {
15535         struct intel_plane *plane;
15536         struct intel_crtc *crtc;
15537
15538         for_each_intel_plane(&dev_priv->drm, plane) {
15539                 struct intel_plane_state *plane_state =
15540                         to_intel_plane_state(plane->base.state);
15541                 struct intel_crtc_state *crtc_state;
15542                 enum pipe pipe = PIPE_A;
15543                 bool visible;
15544
15545                 visible = plane->get_hw_state(plane, &pipe);
15546
15547                 crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
15548                 crtc_state = to_intel_crtc_state(crtc->base.state);
15549
15550                 intel_set_plane_visible(crtc_state, plane_state, visible);
15551
15552                 DRM_DEBUG_KMS("[PLANE:%d:%s] hw state readout: %s, pipe %c\n",
15553                               plane->base.base.id, plane->base.name,
15554                               enableddisabled(visible), pipe_name(pipe));
15555         }
15556
15557         for_each_intel_crtc(&dev_priv->drm, crtc) {
15558                 struct intel_crtc_state *crtc_state =
15559                         to_intel_crtc_state(crtc->base.state);
15560
15561                 fixup_active_planes(crtc_state);
15562         }
15563 }
15564
15565 static void intel_modeset_readout_hw_state(struct drm_device *dev)
15566 {
15567         struct drm_i915_private *dev_priv = to_i915(dev);
15568         enum pipe pipe;
15569         struct intel_crtc *crtc;
15570         struct intel_encoder *encoder;
15571         struct intel_connector *connector;
15572         struct drm_connector_list_iter conn_iter;
15573         int i;
15574
15575         dev_priv->active_crtcs = 0;
15576
15577         for_each_intel_crtc(dev, crtc) {
15578                 struct intel_crtc_state *crtc_state =
15579                         to_intel_crtc_state(crtc->base.state);
15580
15581                 __drm_atomic_helper_crtc_destroy_state(&crtc_state->base);
15582                 memset(crtc_state, 0, sizeof(*crtc_state));
15583                 crtc_state->base.crtc = &crtc->base;
15584
15585                 crtc_state->base.active = crtc_state->base.enable =
15586                         dev_priv->display.get_pipe_config(crtc, crtc_state);
15587
15588                 crtc->base.enabled = crtc_state->base.enable;
15589                 crtc->active = crtc_state->base.active;
15590
15591                 if (crtc_state->base.active)
15592                         dev_priv->active_crtcs |= 1 << crtc->pipe;
15593
15594                 DRM_DEBUG_KMS("[CRTC:%d:%s] hw state readout: %s\n",
15595                               crtc->base.base.id, crtc->base.name,
15596                               enableddisabled(crtc_state->base.active));
15597         }
15598
15599         readout_plane_state(dev_priv);
15600
15601         for (i = 0; i < dev_priv->num_shared_dpll; i++) {
15602                 struct intel_shared_dpll *pll = &dev_priv->shared_dplls[i];
15603
15604                 pll->on = pll->info->funcs->get_hw_state(dev_priv, pll,
15605                                                         &pll->state.hw_state);
15606                 pll->state.crtc_mask = 0;
15607                 for_each_intel_crtc(dev, crtc) {
15608                         struct intel_crtc_state *crtc_state =
15609                                 to_intel_crtc_state(crtc->base.state);
15610
15611                         if (crtc_state->base.active &&
15612                             crtc_state->shared_dpll == pll)
15613                                 pll->state.crtc_mask |= 1 << crtc->pipe;
15614                 }
15615                 pll->active_mask = pll->state.crtc_mask;
15616
15617                 DRM_DEBUG_KMS("%s hw state readout: crtc_mask 0x%08x, on %i\n",
15618                               pll->info->name, pll->state.crtc_mask, pll->on);
15619         }
15620
15621         for_each_intel_encoder(dev, encoder) {
15622                 pipe = 0;
15623
15624                 if (encoder->get_hw_state(encoder, &pipe)) {
15625                         struct intel_crtc_state *crtc_state;
15626
15627                         crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
15628                         crtc_state = to_intel_crtc_state(crtc->base.state);
15629
15630                         encoder->base.crtc = &crtc->base;
15631                         encoder->get_config(encoder, crtc_state);
15632                 } else {
15633                         encoder->base.crtc = NULL;
15634                 }
15635
15636                 DRM_DEBUG_KMS("[ENCODER:%d:%s] hw state readout: %s, pipe %c\n",
15637                               encoder->base.base.id, encoder->base.name,
15638                               enableddisabled(encoder->base.crtc),
15639                               pipe_name(pipe));
15640         }
15641
15642         drm_connector_list_iter_begin(dev, &conn_iter);
15643         for_each_intel_connector_iter(connector, &conn_iter) {
15644                 if (connector->get_hw_state(connector)) {
15645                         connector->base.dpms = DRM_MODE_DPMS_ON;
15646
15647                         encoder = connector->encoder;
15648                         connector->base.encoder = &encoder->base;
15649
15650                         if (encoder->base.crtc &&
15651                             encoder->base.crtc->state->active) {
15652                                 /*
15653                                  * This has to be done during hardware readout
15654                                  * because anything calling .crtc_disable may
15655                                  * rely on the connector_mask being accurate.
15656                                  */
15657                                 encoder->base.crtc->state->connector_mask |=
15658                                         drm_connector_mask(&connector->base);
15659                                 encoder->base.crtc->state->encoder_mask |=
15660                                         drm_encoder_mask(&encoder->base);
15661                         }
15662
15663                 } else {
15664                         connector->base.dpms = DRM_MODE_DPMS_OFF;
15665                         connector->base.encoder = NULL;
15666                 }
15667                 DRM_DEBUG_KMS("[CONNECTOR:%d:%s] hw state readout: %s\n",
15668                               connector->base.base.id, connector->base.name,
15669                               enableddisabled(connector->base.encoder));
15670         }
15671         drm_connector_list_iter_end(&conn_iter);
15672
15673         for_each_intel_crtc(dev, crtc) {
15674                 struct intel_crtc_state *crtc_state =
15675                         to_intel_crtc_state(crtc->base.state);
15676                 int min_cdclk = 0;
15677
15678                 memset(&crtc->base.mode, 0, sizeof(crtc->base.mode));
15679                 if (crtc_state->base.active) {
15680                         intel_mode_from_pipe_config(&crtc->base.mode, crtc_state);
15681                         crtc->base.mode.hdisplay = crtc_state->pipe_src_w;
15682                         crtc->base.mode.vdisplay = crtc_state->pipe_src_h;
15683                         intel_mode_from_pipe_config(&crtc_state->base.adjusted_mode, crtc_state);
15684                         WARN_ON(drm_atomic_set_mode_for_crtc(crtc->base.state, &crtc->base.mode));
15685
15686                         /*
15687                          * The initial mode needs to be set in order to keep
15688                          * the atomic core happy. It wants a valid mode if the
15689                          * crtc's enabled, so we do the above call.
15690                          *
15691                          * But we don't set all the derived state fully, hence
15692                          * set a flag to indicate that a full recalculation is
15693                          * needed on the next commit.
15694                          */
15695                         crtc_state->base.mode.private_flags = I915_MODE_FLAG_INHERITED;
15696
15697                         intel_crtc_compute_pixel_rate(crtc_state);
15698
15699                         if (dev_priv->display.modeset_calc_cdclk) {
15700                                 min_cdclk = intel_crtc_compute_min_cdclk(crtc_state);
15701                                 if (WARN_ON(min_cdclk < 0))
15702                                         min_cdclk = 0;
15703                         }
15704
15705                         drm_calc_timestamping_constants(&crtc->base,
15706                                                         &crtc_state->base.adjusted_mode);
15707                         update_scanline_offset(crtc_state);
15708                 }
15709
15710                 dev_priv->min_cdclk[crtc->pipe] = min_cdclk;
15711                 dev_priv->min_voltage_level[crtc->pipe] =
15712                         crtc_state->min_voltage_level;
15713
15714                 intel_pipe_config_sanity_check(dev_priv, crtc_state);
15715         }
15716 }
15717
15718 static void
15719 get_encoder_power_domains(struct drm_i915_private *dev_priv)
15720 {
15721         struct intel_encoder *encoder;
15722
15723         for_each_intel_encoder(&dev_priv->drm, encoder) {
15724                 u64 get_domains;
15725                 enum intel_display_power_domain domain;
15726                 struct intel_crtc_state *crtc_state;
15727
15728                 if (!encoder->get_power_domains)
15729                         continue;
15730
15731                 /*
15732                  * MST-primary and inactive encoders don't have a crtc state
15733                  * and neither of these require any power domain references.
15734                  */
15735                 if (!encoder->base.crtc)
15736                         continue;
15737
15738                 crtc_state = to_intel_crtc_state(encoder->base.crtc->state);
15739                 get_domains = encoder->get_power_domains(encoder, crtc_state);
15740                 for_each_power_domain(domain, get_domains)
15741                         intel_display_power_get(dev_priv, domain);
15742         }
15743 }
15744
15745 static void intel_early_display_was(struct drm_i915_private *dev_priv)
15746 {
15747         /* Display WA #1185 WaDisableDARBFClkGating:cnl,glk */
15748         if (IS_CANNONLAKE(dev_priv) || IS_GEMINILAKE(dev_priv))
15749                 I915_WRITE(GEN9_CLKGATE_DIS_0, I915_READ(GEN9_CLKGATE_DIS_0) |
15750                            DARBF_GATING_DIS);
15751
15752         if (IS_HASWELL(dev_priv)) {
15753                 /*
15754                  * WaRsPkgCStateDisplayPMReq:hsw
15755                  * System hang if this isn't done before disabling all planes!
15756                  */
15757                 I915_WRITE(CHICKEN_PAR1_1,
15758                            I915_READ(CHICKEN_PAR1_1) | FORCE_ARB_IDLE_PLANES);
15759         }
15760 }
15761
15762 static void ibx_sanitize_pch_hdmi_port(struct drm_i915_private *dev_priv,
15763                                        enum port port, i915_reg_t hdmi_reg)
15764 {
15765         u32 val = I915_READ(hdmi_reg);
15766
15767         if (val & SDVO_ENABLE ||
15768             (val & SDVO_PIPE_SEL_MASK) == SDVO_PIPE_SEL(PIPE_A))
15769                 return;
15770
15771         DRM_DEBUG_KMS("Sanitizing transcoder select for HDMI %c\n",
15772                       port_name(port));
15773
15774         val &= ~SDVO_PIPE_SEL_MASK;
15775         val |= SDVO_PIPE_SEL(PIPE_A);
15776
15777         I915_WRITE(hdmi_reg, val);
15778 }
15779
15780 static void ibx_sanitize_pch_dp_port(struct drm_i915_private *dev_priv,
15781                                      enum port port, i915_reg_t dp_reg)
15782 {
15783         u32 val = I915_READ(dp_reg);
15784
15785         if (val & DP_PORT_EN ||
15786             (val & DP_PIPE_SEL_MASK) == DP_PIPE_SEL(PIPE_A))
15787                 return;
15788
15789         DRM_DEBUG_KMS("Sanitizing transcoder select for DP %c\n",
15790                       port_name(port));
15791
15792         val &= ~DP_PIPE_SEL_MASK;
15793         val |= DP_PIPE_SEL(PIPE_A);
15794
15795         I915_WRITE(dp_reg, val);
15796 }
15797
15798 static void ibx_sanitize_pch_ports(struct drm_i915_private *dev_priv)
15799 {
15800         /*
15801          * The BIOS may select transcoder B on some of the PCH
15802          * ports even it doesn't enable the port. This would trip
15803          * assert_pch_dp_disabled() and assert_pch_hdmi_disabled().
15804          * Sanitize the transcoder select bits to prevent that. We
15805          * assume that the BIOS never actually enabled the port,
15806          * because if it did we'd actually have to toggle the port
15807          * on and back off to make the transcoder A select stick
15808          * (see. intel_dp_link_down(), intel_disable_hdmi(),
15809          * intel_disable_sdvo()).
15810          */
15811         ibx_sanitize_pch_dp_port(dev_priv, PORT_B, PCH_DP_B);
15812         ibx_sanitize_pch_dp_port(dev_priv, PORT_C, PCH_DP_C);
15813         ibx_sanitize_pch_dp_port(dev_priv, PORT_D, PCH_DP_D);
15814
15815         /* PCH SDVOB multiplex with HDMIB */
15816         ibx_sanitize_pch_hdmi_port(dev_priv, PORT_B, PCH_HDMIB);
15817         ibx_sanitize_pch_hdmi_port(dev_priv, PORT_C, PCH_HDMIC);
15818         ibx_sanitize_pch_hdmi_port(dev_priv, PORT_D, PCH_HDMID);
15819 }
15820
15821 /* Scan out the current hw modeset state,
15822  * and sanitizes it to the current state
15823  */
15824 static void
15825 intel_modeset_setup_hw_state(struct drm_device *dev,
15826                              struct drm_modeset_acquire_ctx *ctx)
15827 {
15828         struct drm_i915_private *dev_priv = to_i915(dev);
15829         struct intel_crtc *crtc;
15830         struct intel_crtc_state *crtc_state;
15831         struct intel_encoder *encoder;
15832         int i;
15833
15834         intel_display_power_get(dev_priv, POWER_DOMAIN_INIT);
15835
15836         intel_early_display_was(dev_priv);
15837         intel_modeset_readout_hw_state(dev);
15838
15839         /* HW state is read out, now we need to sanitize this mess. */
15840         get_encoder_power_domains(dev_priv);
15841
15842         if (HAS_PCH_IBX(dev_priv))
15843                 ibx_sanitize_pch_ports(dev_priv);
15844
15845         /*
15846          * intel_sanitize_plane_mapping() may need to do vblank
15847          * waits, so we need vblank interrupts restored beforehand.
15848          */
15849         for_each_intel_crtc(&dev_priv->drm, crtc) {
15850                 drm_crtc_vblank_reset(&crtc->base);
15851
15852                 if (crtc->base.state->active)
15853                         drm_crtc_vblank_on(&crtc->base);
15854         }
15855
15856         intel_sanitize_plane_mapping(dev_priv);
15857
15858         for_each_intel_encoder(dev, encoder)
15859                 intel_sanitize_encoder(encoder);
15860
15861         for_each_intel_crtc(&dev_priv->drm, crtc) {
15862                 crtc_state = to_intel_crtc_state(crtc->base.state);
15863                 intel_sanitize_crtc(crtc, ctx);
15864                 intel_dump_pipe_config(crtc, crtc_state,
15865                                        "[setup_hw_state]");
15866         }
15867
15868         intel_modeset_update_connector_atomic_state(dev);
15869
15870         for (i = 0; i < dev_priv->num_shared_dpll; i++) {
15871                 struct intel_shared_dpll *pll = &dev_priv->shared_dplls[i];
15872
15873                 if (!pll->on || pll->active_mask)
15874                         continue;
15875
15876                 DRM_DEBUG_KMS("%s enabled but not in use, disabling\n",
15877                               pll->info->name);
15878
15879                 pll->info->funcs->disable(dev_priv, pll);
15880                 pll->on = false;
15881         }
15882
15883         if (IS_G4X(dev_priv)) {
15884                 g4x_wm_get_hw_state(dev);
15885                 g4x_wm_sanitize(dev_priv);
15886         } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
15887                 vlv_wm_get_hw_state(dev);
15888                 vlv_wm_sanitize(dev_priv);
15889         } else if (INTEL_GEN(dev_priv) >= 9) {
15890                 skl_wm_get_hw_state(dev);
15891         } else if (HAS_PCH_SPLIT(dev_priv)) {
15892                 ilk_wm_get_hw_state(dev);
15893         }
15894
15895         for_each_intel_crtc(dev, crtc) {
15896                 u64 put_domains;
15897
15898                 crtc_state = to_intel_crtc_state(crtc->base.state);
15899                 put_domains = modeset_get_crtc_power_domains(&crtc->base, crtc_state);
15900                 if (WARN_ON(put_domains))
15901                         modeset_put_power_domains(dev_priv, put_domains);
15902         }
15903
15904         intel_display_power_put(dev_priv, POWER_DOMAIN_INIT);
15905
15906         intel_fbc_init_pipe_state(dev_priv);
15907 }
15908
15909 void intel_display_resume(struct drm_device *dev)
15910 {
15911         struct drm_i915_private *dev_priv = to_i915(dev);
15912         struct drm_atomic_state *state = dev_priv->modeset_restore_state;
15913         struct drm_modeset_acquire_ctx ctx;
15914         int ret;
15915
15916         dev_priv->modeset_restore_state = NULL;
15917         if (state)
15918                 state->acquire_ctx = &ctx;
15919
15920         drm_modeset_acquire_init(&ctx, 0);
15921
15922         while (1) {
15923                 ret = drm_modeset_lock_all_ctx(dev, &ctx);
15924                 if (ret != -EDEADLK)
15925                         break;
15926
15927                 drm_modeset_backoff(&ctx);
15928         }
15929
15930         if (!ret)
15931                 ret = __intel_display_resume(dev, state, &ctx);
15932
15933         intel_enable_ipc(dev_priv);
15934         drm_modeset_drop_locks(&ctx);
15935         drm_modeset_acquire_fini(&ctx);
15936
15937         if (ret)
15938                 DRM_ERROR("Restoring old state failed with %i\n", ret);
15939         if (state)
15940                 drm_atomic_state_put(state);
15941 }
15942
15943 static void intel_hpd_poll_fini(struct drm_device *dev)
15944 {
15945         struct intel_connector *connector;
15946         struct drm_connector_list_iter conn_iter;
15947
15948         /* Kill all the work that may have been queued by hpd. */
15949         drm_connector_list_iter_begin(dev, &conn_iter);
15950         for_each_intel_connector_iter(connector, &conn_iter) {
15951                 if (connector->modeset_retry_work.func)
15952                         cancel_work_sync(&connector->modeset_retry_work);
15953                 if (connector->hdcp.shim) {
15954                         cancel_delayed_work_sync(&connector->hdcp.check_work);
15955                         cancel_work_sync(&connector->hdcp.prop_work);
15956                 }
15957         }
15958         drm_connector_list_iter_end(&conn_iter);
15959 }
15960
15961 void intel_modeset_cleanup(struct drm_device *dev)
15962 {
15963         struct drm_i915_private *dev_priv = to_i915(dev);
15964
15965         flush_workqueue(dev_priv->modeset_wq);
15966
15967         flush_work(&dev_priv->atomic_helper.free_work);
15968         WARN_ON(!llist_empty(&dev_priv->atomic_helper.free_list));
15969
15970         /*
15971          * Interrupts and polling as the first thing to avoid creating havoc.
15972          * Too much stuff here (turning of connectors, ...) would
15973          * experience fancy races otherwise.
15974          */
15975         intel_irq_uninstall(dev_priv);
15976
15977         /*
15978          * Due to the hpd irq storm handling the hotplug work can re-arm the
15979          * poll handlers. Hence disable polling after hpd handling is shut down.
15980          */
15981         intel_hpd_poll_fini(dev);
15982
15983         /* poll work can call into fbdev, hence clean that up afterwards */
15984         intel_fbdev_fini(dev_priv);
15985
15986         intel_unregister_dsm_handler();
15987
15988         intel_fbc_global_disable(dev_priv);
15989
15990         /* flush any delayed tasks or pending work */
15991         flush_scheduled_work();
15992
15993         drm_mode_config_cleanup(dev);
15994
15995         intel_overlay_cleanup(dev_priv);
15996
15997         intel_teardown_gmbus(dev_priv);
15998
15999         destroy_workqueue(dev_priv->modeset_wq);
16000
16001         intel_fbc_cleanup_cfb(dev_priv);
16002 }
16003
16004 /*
16005  * set vga decode state - true == enable VGA decode
16006  */
16007 int intel_modeset_vga_set_state(struct drm_i915_private *dev_priv, bool state)
16008 {
16009         unsigned reg = INTEL_GEN(dev_priv) >= 6 ? SNB_GMCH_CTRL : INTEL_GMCH_CTRL;
16010         u16 gmch_ctrl;
16011
16012         if (pci_read_config_word(dev_priv->bridge_dev, reg, &gmch_ctrl)) {
16013                 DRM_ERROR("failed to read control word\n");
16014                 return -EIO;
16015         }
16016
16017         if (!!(gmch_ctrl & INTEL_GMCH_VGA_DISABLE) == !state)
16018                 return 0;
16019
16020         if (state)
16021                 gmch_ctrl &= ~INTEL_GMCH_VGA_DISABLE;
16022         else
16023                 gmch_ctrl |= INTEL_GMCH_VGA_DISABLE;
16024
16025         if (pci_write_config_word(dev_priv->bridge_dev, reg, gmch_ctrl)) {
16026                 DRM_ERROR("failed to write control word\n");
16027                 return -EIO;
16028         }
16029
16030         return 0;
16031 }
16032
16033 #if IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR)
16034
16035 struct intel_display_error_state {
16036
16037         u32 power_well_driver;
16038
16039         int num_transcoders;
16040
16041         struct intel_cursor_error_state {
16042                 u32 control;
16043                 u32 position;
16044                 u32 base;
16045                 u32 size;
16046         } cursor[I915_MAX_PIPES];
16047
16048         struct intel_pipe_error_state {
16049                 bool power_domain_on;
16050                 u32 source;
16051                 u32 stat;
16052         } pipe[I915_MAX_PIPES];
16053
16054         struct intel_plane_error_state {
16055                 u32 control;
16056                 u32 stride;
16057                 u32 size;
16058                 u32 pos;
16059                 u32 addr;
16060                 u32 surface;
16061                 u32 tile_offset;
16062         } plane[I915_MAX_PIPES];
16063
16064         struct intel_transcoder_error_state {
16065                 bool power_domain_on;
16066                 enum transcoder cpu_transcoder;
16067
16068                 u32 conf;
16069
16070                 u32 htotal;
16071                 u32 hblank;
16072                 u32 hsync;
16073                 u32 vtotal;
16074                 u32 vblank;
16075                 u32 vsync;
16076         } transcoder[4];
16077 };
16078
16079 struct intel_display_error_state *
16080 intel_display_capture_error_state(struct drm_i915_private *dev_priv)
16081 {
16082         struct intel_display_error_state *error;
16083         int transcoders[] = {
16084                 TRANSCODER_A,
16085                 TRANSCODER_B,
16086                 TRANSCODER_C,
16087                 TRANSCODER_EDP,
16088         };
16089         int i;
16090
16091         if (!HAS_DISPLAY(dev_priv))
16092                 return NULL;
16093
16094         error = kzalloc(sizeof(*error), GFP_ATOMIC);
16095         if (error == NULL)
16096                 return NULL;
16097
16098         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
16099                 error->power_well_driver = I915_READ(HSW_PWR_WELL_CTL2);
16100
16101         for_each_pipe(dev_priv, i) {
16102                 error->pipe[i].power_domain_on =
16103                         __intel_display_power_is_enabled(dev_priv,
16104                                                          POWER_DOMAIN_PIPE(i));
16105                 if (!error->pipe[i].power_domain_on)
16106                         continue;
16107
16108                 error->cursor[i].control = I915_READ(CURCNTR(i));
16109                 error->cursor[i].position = I915_READ(CURPOS(i));
16110                 error->cursor[i].base = I915_READ(CURBASE(i));
16111
16112                 error->plane[i].control = I915_READ(DSPCNTR(i));
16113                 error->plane[i].stride = I915_READ(DSPSTRIDE(i));
16114                 if (INTEL_GEN(dev_priv) <= 3) {
16115                         error->plane[i].size = I915_READ(DSPSIZE(i));
16116                         error->plane[i].pos = I915_READ(DSPPOS(i));
16117                 }
16118                 if (INTEL_GEN(dev_priv) <= 7 && !IS_HASWELL(dev_priv))
16119                         error->plane[i].addr = I915_READ(DSPADDR(i));
16120                 if (INTEL_GEN(dev_priv) >= 4) {
16121                         error->plane[i].surface = I915_READ(DSPSURF(i));
16122                         error->plane[i].tile_offset = I915_READ(DSPTILEOFF(i));
16123                 }
16124
16125                 error->pipe[i].source = I915_READ(PIPESRC(i));
16126
16127                 if (HAS_GMCH_DISPLAY(dev_priv))
16128                         error->pipe[i].stat = I915_READ(PIPESTAT(i));
16129         }
16130
16131         /* Note: this does not include DSI transcoders. */
16132         error->num_transcoders = INTEL_INFO(dev_priv)->num_pipes;
16133         if (HAS_DDI(dev_priv))
16134                 error->num_transcoders++; /* Account for eDP. */
16135
16136         for (i = 0; i < error->num_transcoders; i++) {
16137                 enum transcoder cpu_transcoder = transcoders[i];
16138
16139                 error->transcoder[i].power_domain_on =
16140                         __intel_display_power_is_enabled(dev_priv,
16141                                 POWER_DOMAIN_TRANSCODER(cpu_transcoder));
16142                 if (!error->transcoder[i].power_domain_on)
16143                         continue;
16144
16145                 error->transcoder[i].cpu_transcoder = cpu_transcoder;
16146
16147                 error->transcoder[i].conf = I915_READ(PIPECONF(cpu_transcoder));
16148                 error->transcoder[i].htotal = I915_READ(HTOTAL(cpu_transcoder));
16149                 error->transcoder[i].hblank = I915_READ(HBLANK(cpu_transcoder));
16150                 error->transcoder[i].hsync = I915_READ(HSYNC(cpu_transcoder));
16151                 error->transcoder[i].vtotal = I915_READ(VTOTAL(cpu_transcoder));
16152                 error->transcoder[i].vblank = I915_READ(VBLANK(cpu_transcoder));
16153                 error->transcoder[i].vsync = I915_READ(VSYNC(cpu_transcoder));
16154         }
16155
16156         return error;
16157 }
16158
16159 #define err_printf(e, ...) i915_error_printf(e, __VA_ARGS__)
16160
16161 void
16162 intel_display_print_error_state(struct drm_i915_error_state_buf *m,
16163                                 struct intel_display_error_state *error)
16164 {
16165         struct drm_i915_private *dev_priv = m->i915;
16166         int i;
16167
16168         if (!error)
16169                 return;
16170
16171         err_printf(m, "Num Pipes: %d\n", INTEL_INFO(dev_priv)->num_pipes);
16172         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
16173                 err_printf(m, "PWR_WELL_CTL2: %08x\n",
16174                            error->power_well_driver);
16175         for_each_pipe(dev_priv, i) {
16176                 err_printf(m, "Pipe [%d]:\n", i);
16177                 err_printf(m, "  Power: %s\n",
16178                            onoff(error->pipe[i].power_domain_on));
16179                 err_printf(m, "  SRC: %08x\n", error->pipe[i].source);
16180                 err_printf(m, "  STAT: %08x\n", error->pipe[i].stat);
16181
16182                 err_printf(m, "Plane [%d]:\n", i);
16183                 err_printf(m, "  CNTR: %08x\n", error->plane[i].control);
16184                 err_printf(m, "  STRIDE: %08x\n", error->plane[i].stride);
16185                 if (INTEL_GEN(dev_priv) <= 3) {
16186                         err_printf(m, "  SIZE: %08x\n", error->plane[i].size);
16187                         err_printf(m, "  POS: %08x\n", error->plane[i].pos);
16188                 }
16189                 if (INTEL_GEN(dev_priv) <= 7 && !IS_HASWELL(dev_priv))
16190                         err_printf(m, "  ADDR: %08x\n", error->plane[i].addr);
16191                 if (INTEL_GEN(dev_priv) >= 4) {
16192                         err_printf(m, "  SURF: %08x\n", error->plane[i].surface);
16193                         err_printf(m, "  TILEOFF: %08x\n", error->plane[i].tile_offset);
16194                 }
16195
16196                 err_printf(m, "Cursor [%d]:\n", i);
16197                 err_printf(m, "  CNTR: %08x\n", error->cursor[i].control);
16198                 err_printf(m, "  POS: %08x\n", error->cursor[i].position);
16199                 err_printf(m, "  BASE: %08x\n", error->cursor[i].base);
16200         }
16201
16202         for (i = 0; i < error->num_transcoders; i++) {
16203                 err_printf(m, "CPU transcoder: %s\n",
16204                            transcoder_name(error->transcoder[i].cpu_transcoder));
16205                 err_printf(m, "  Power: %s\n",
16206                            onoff(error->transcoder[i].power_domain_on));
16207                 err_printf(m, "  CONF: %08x\n", error->transcoder[i].conf);
16208                 err_printf(m, "  HTOTAL: %08x\n", error->transcoder[i].htotal);
16209                 err_printf(m, "  HBLANK: %08x\n", error->transcoder[i].hblank);
16210                 err_printf(m, "  HSYNC: %08x\n", error->transcoder[i].hsync);
16211                 err_printf(m, "  VTOTAL: %08x\n", error->transcoder[i].vtotal);
16212                 err_printf(m, "  VBLANK: %08x\n", error->transcoder[i].vblank);
16213                 err_printf(m, "  VSYNC: %08x\n", error->transcoder[i].vsync);
16214         }
16215 }
16216
16217 #endif