drivers/gpu/drm/i915/intel_display.c

   1 /*
   2  * Copyright © 2006-2007 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  21  * DEALINGS IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *      Eric Anholt <eric@anholt.net>
  25  */
  26
  27 #include <linux/dmi.h>
  28 #include <linux/module.h>
  29 #include <linux/input.h>
  30 #include <linux/i2c.h>
  31 #include <linux/kernel.h>
  32 #include <linux/slab.h>
  33 #include <linux/vgaarb.h>
  34 #include <drm/drm_edid.h>
  35 #include <drm/drmP.h>
  36 #include "intel_drv.h"
  37 #include "intel_frontbuffer.h"
  38 #include <drm/i915_drm.h>
  39 #include "i915_drv.h"
  40 #include "i915_gem_clflush.h"
  41 #include "intel_dsi.h"
  42 #include "i915_trace.h"
  43 #include <drm/drm_atomic.h>
  44 #include <drm/drm_atomic_helper.h>
  45 #include <drm/drm_dp_helper.h>
  46 #include <drm/drm_crtc_helper.h>
  47 #include <drm/drm_plane_helper.h>
  48 #include <drm/drm_rect.h>
  49 #include <drm/drm_atomic_uapi.h>
  50 #include <linux/dma_remapping.h>
  51 #include <linux/reservation.h>
  52
  53 /* Primary plane formats for gen <= 3 */
  54 static const uint32_t i8xx_primary_formats[] = {
  55         DRM_FORMAT_C8,
  56         DRM_FORMAT_RGB565,
  57         DRM_FORMAT_XRGB1555,
  58         DRM_FORMAT_XRGB8888,
  59 };
  60
  61 /* Primary plane formats for gen >= 4 */
  62 static const uint32_t i965_primary_formats[] = {
  63         DRM_FORMAT_C8,
  64         DRM_FORMAT_RGB565,
  65         DRM_FORMAT_XRGB8888,
  66         DRM_FORMAT_XBGR8888,
  67         DRM_FORMAT_XRGB2101010,
  68         DRM_FORMAT_XBGR2101010,
  69 };
  70
  71 static const uint64_t i9xx_format_modifiers[] = {
  72         I915_FORMAT_MOD_X_TILED,
  73         DRM_FORMAT_MOD_LINEAR,
  74         DRM_FORMAT_MOD_INVALID
  75 };
  76
  77 static const uint32_t skl_primary_formats[] = {
  78         DRM_FORMAT_C8,
  79         DRM_FORMAT_RGB565,
  80         DRM_FORMAT_XRGB8888,
  81         DRM_FORMAT_XBGR8888,
  82         DRM_FORMAT_ARGB8888,
  83         DRM_FORMAT_ABGR8888,
  84         DRM_FORMAT_XRGB2101010,
  85         DRM_FORMAT_XBGR2101010,
  86         DRM_FORMAT_YUYV,
  87         DRM_FORMAT_YVYU,
  88         DRM_FORMAT_UYVY,
  89         DRM_FORMAT_VYUY,
  90 };
  91
  92 static const uint32_t skl_pri_planar_formats[] = {
  93         DRM_FORMAT_C8,
  94         DRM_FORMAT_RGB565,
  95         DRM_FORMAT_XRGB8888,
  96         DRM_FORMAT_XBGR8888,
  97         DRM_FORMAT_ARGB8888,
  98         DRM_FORMAT_ABGR8888,
  99         DRM_FORMAT_XRGB2101010,
 100         DRM_FORMAT_XBGR2101010,
 101         DRM_FORMAT_YUYV,
 102         DRM_FORMAT_YVYU,
 103         DRM_FORMAT_UYVY,
 104         DRM_FORMAT_VYUY,
 105         DRM_FORMAT_NV12,
 106 };
 107
 108 static const uint64_t skl_format_modifiers_noccs[] = {
 109         I915_FORMAT_MOD_Yf_TILED,
 110         I915_FORMAT_MOD_Y_TILED,
 111         I915_FORMAT_MOD_X_TILED,
 112         DRM_FORMAT_MOD_LINEAR,
 113         DRM_FORMAT_MOD_INVALID
 114 };
 115
 116 static const uint64_t skl_format_modifiers_ccs[] = {
 117         I915_FORMAT_MOD_Yf_TILED_CCS,
 118         I915_FORMAT_MOD_Y_TILED_CCS,
 119         I915_FORMAT_MOD_Yf_TILED,
 120         I915_FORMAT_MOD_Y_TILED,
 121         I915_FORMAT_MOD_X_TILED,
 122         DRM_FORMAT_MOD_LINEAR,
 123         DRM_FORMAT_MOD_INVALID
 124 };
 125
 126 /* Cursor formats */
 127 static const uint32_t intel_cursor_formats[] = {
 128         DRM_FORMAT_ARGB8888,
 129 };
 130
 131 static const uint64_t cursor_format_modifiers[] = {
 132         DRM_FORMAT_MOD_LINEAR,
 133         DRM_FORMAT_MOD_INVALID
 134 };
 135
 136 static void i9xx_crtc_clock_get(struct intel_crtc *crtc,
 137                                 struct intel_crtc_state *pipe_config);
 138 static void ironlake_pch_clock_get(struct intel_crtc *crtc,
 139                                    struct intel_crtc_state *pipe_config);
 140
 141 static int intel_framebuffer_init(struct intel_framebuffer *ifb,
 142                                   struct drm_i915_gem_object *obj,
 143                                   struct drm_mode_fb_cmd2 *mode_cmd);
 144 static void i9xx_set_pipeconf(struct intel_crtc *intel_crtc);
 145 static void intel_set_pipe_timings(struct intel_crtc *intel_crtc);
 146 static void intel_set_pipe_src_size(struct intel_crtc *intel_crtc);
 147 static void intel_cpu_transcoder_set_m_n(struct intel_crtc *crtc,
 148                                          struct intel_link_m_n *m_n,
 149                                          struct intel_link_m_n *m2_n2);
 150 static void ironlake_set_pipeconf(struct drm_crtc *crtc);
 151 static void haswell_set_pipeconf(struct drm_crtc *crtc);
 152 static void haswell_set_pipemisc(struct drm_crtc *crtc);
 153 static void vlv_prepare_pll(struct intel_crtc *crtc,
 154                             const struct intel_crtc_state *pipe_config);
 155 static void chv_prepare_pll(struct intel_crtc *crtc,
 156                             const struct intel_crtc_state *pipe_config);
 157 static void intel_begin_crtc_commit(struct drm_crtc *, struct drm_crtc_state *);
 158 static void intel_finish_crtc_commit(struct drm_crtc *, struct drm_crtc_state *);
 159 static void intel_crtc_init_scalers(struct intel_crtc *crtc,
 160                                     struct intel_crtc_state *crtc_state);
 161 static void skylake_pfit_enable(struct intel_crtc *crtc);
 162 static void ironlake_pfit_disable(struct intel_crtc *crtc, bool force);
 163 static void ironlake_pfit_enable(struct intel_crtc *crtc);
 164 static void intel_modeset_setup_hw_state(struct drm_device *dev,
 165                                          struct drm_modeset_acquire_ctx *ctx);
 166 static void intel_pre_disable_primary_noatomic(struct drm_crtc *crtc);
 167
 168 struct intel_limit {
 169         struct {
 170                 int min, max;
 171         } dot, vco, n, m, m1, m2, p, p1;
 172
 173         struct {
 174                 int dot_limit;
 175                 int p2_slow, p2_fast;
 176         } p2;
 177 };
 178
 179 /* returns HPLL frequency in kHz */
 180 int vlv_get_hpll_vco(struct drm_i915_private *dev_priv)
 181 {
 182         int hpll_freq, vco_freq[] = { 800, 1600, 2000, 2400 };
 183
 184         /* Obtain SKU information */
 185         mutex_lock(&dev_priv->sb_lock);
 186         hpll_freq = vlv_cck_read(dev_priv, CCK_FUSE_REG) &
 187                 CCK_FUSE_HPLL_FREQ_MASK;
 188         mutex_unlock(&dev_priv->sb_lock);
 189
 190         return vco_freq[hpll_freq] * 1000;
 191 }
 192
 193 int vlv_get_cck_clock(struct drm_i915_private *dev_priv,
 194                       const char *name, u32 reg, int ref_freq)
 195 {
 196         u32 val;
 197         int divider;
 198
 199         mutex_lock(&dev_priv->sb_lock);
 200         val = vlv_cck_read(dev_priv, reg);
 201         mutex_unlock(&dev_priv->sb_lock);
 202
 203         divider = val & CCK_FREQUENCY_VALUES;
 204
 205         WARN((val & CCK_FREQUENCY_STATUS) !=
 206              (divider << CCK_FREQUENCY_STATUS_SHIFT),
 207              "%s change in progress\n", name);
 208
 209         return DIV_ROUND_CLOSEST(ref_freq << 1, divider + 1);
 210 }
 211
 212 int vlv_get_cck_clock_hpll(struct drm_i915_private *dev_priv,
 213                            const char *name, u32 reg)
 214 {
 215         if (dev_priv->hpll_freq == 0)
 216                 dev_priv->hpll_freq = vlv_get_hpll_vco(dev_priv);
 217
 218         return vlv_get_cck_clock(dev_priv, name, reg,
 219                                  dev_priv->hpll_freq);
 220 }
 221
 222 static void intel_update_czclk(struct drm_i915_private *dev_priv)
 223 {
 224         if (!(IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)))
 225                 return;
 226
 227         dev_priv->czclk_freq = vlv_get_cck_clock_hpll(dev_priv, "czclk",
 228                                                       CCK_CZ_CLOCK_CONTROL);
 229
 230         DRM_DEBUG_DRIVER("CZ clock rate: %d kHz\n", dev_priv->czclk_freq);
 231 }
 232
 233 static inline u32 /* units of 100MHz */
 234 intel_fdi_link_freq(struct drm_i915_private *dev_priv,
 235                     const struct intel_crtc_state *pipe_config)
 236 {
 237         if (HAS_DDI(dev_priv))
 238                 return pipe_config->port_clock; /* SPLL */
 239         else
 240                 return dev_priv->fdi_pll_freq;
 241 }
 242
 243 static const struct intel_limit intel_limits_i8xx_dac = {
 244         .dot = { .min = 25000, .max = 350000 },
 245         .vco = { .min = 908000, .max = 1512000 },
 246         .n = { .min = 2, .max = 16 },
 247         .m = { .min = 96, .max = 140 },
 248         .m1 = { .min = 18, .max = 26 },
 249         .m2 = { .min = 6, .max = 16 },
 250         .p = { .min = 4, .max = 128 },
 251         .p1 = { .min = 2, .max = 33 },
 252         .p2 = { .dot_limit = 165000,
 253                 .p2_slow = 4, .p2_fast = 2 },
 254 };
 255
 256 static const struct intel_limit intel_limits_i8xx_dvo = {
 257         .dot = { .min = 25000, .max = 350000 },
 258         .vco = { .min = 908000, .max = 1512000 },
 259         .n = { .min = 2, .max = 16 },
 260         .m = { .min = 96, .max = 140 },
 261         .m1 = { .min = 18, .max = 26 },
 262         .m2 = { .min = 6, .max = 16 },
 263         .p = { .min = 4, .max = 128 },
 264         .p1 = { .min = 2, .max = 33 },
 265         .p2 = { .dot_limit = 165000,
 266                 .p2_slow = 4, .p2_fast = 4 },
 267 };
 268
 269 static const struct intel_limit intel_limits_i8xx_lvds = {
 270         .dot = { .min = 25000, .max = 350000 },
 271         .vco = { .min = 908000, .max = 1512000 },
 272         .n = { .min = 2, .max = 16 },
 273         .m = { .min = 96, .max = 140 },
 274         .m1 = { .min = 18, .max = 26 },
 275         .m2 = { .min = 6, .max = 16 },
 276         .p = { .min = 4, .max = 128 },
 277         .p1 = { .min = 1, .max = 6 },
 278         .p2 = { .dot_limit = 165000,
 279                 .p2_slow = 14, .p2_fast = 7 },
 280 };
 281
 282 static const struct intel_limit intel_limits_i9xx_sdvo = {
 283         .dot = { .min = 20000, .max = 400000 },
 284         .vco = { .min = 1400000, .max = 2800000 },
 285         .n = { .min = 1, .max = 6 },
 286         .m = { .min = 70, .max = 120 },
 287         .m1 = { .min = 8, .max = 18 },
 288         .m2 = { .min = 3, .max = 7 },
 289         .p = { .min = 5, .max = 80 },
 290         .p1 = { .min = 1, .max = 8 },
 291         .p2 = { .dot_limit = 200000,
 292                 .p2_slow = 10, .p2_fast = 5 },
 293 };
 294
 295 static const struct intel_limit intel_limits_i9xx_lvds = {
 296         .dot = { .min = 20000, .max = 400000 },
 297         .vco = { .min = 1400000, .max = 2800000 },
 298         .n = { .min = 1, .max = 6 },
 299         .m = { .min = 70, .max = 120 },
 300         .m1 = { .min = 8, .max = 18 },
 301         .m2 = { .min = 3, .max = 7 },
 302         .p = { .min = 7, .max = 98 },
 303         .p1 = { .min = 1, .max = 8 },
 304         .p2 = { .dot_limit = 112000,
 305                 .p2_slow = 14, .p2_fast = 7 },
 306 };
 307
 308
 309 static const struct intel_limit intel_limits_g4x_sdvo = {
 310         .dot = { .min = 25000, .max = 270000 },
 311         .vco = { .min = 1750000, .max = 3500000},
 312         .n = { .min = 1, .max = 4 },
 313         .m = { .min = 104, .max = 138 },
 314         .m1 = { .min = 17, .max = 23 },
 315         .m2 = { .min = 5, .max = 11 },
 316         .p = { .min = 10, .max = 30 },
 317         .p1 = { .min = 1, .max = 3},
 318         .p2 = { .dot_limit = 270000,
 319                 .p2_slow = 10,
 320                 .p2_fast = 10
 321         },
 322 };
 323
 324 static const struct intel_limit intel_limits_g4x_hdmi = {
 325         .dot = { .min = 22000, .max = 400000 },
 326         .vco = { .min = 1750000, .max = 3500000},
 327         .n = { .min = 1, .max = 4 },
 328         .m = { .min = 104, .max = 138 },
 329         .m1 = { .min = 16, .max = 23 },
 330         .m2 = { .min = 5, .max = 11 },
 331         .p = { .min = 5, .max = 80 },
 332         .p1 = { .min = 1, .max = 8},
 333         .p2 = { .dot_limit = 165000,
 334                 .p2_slow = 10, .p2_fast = 5 },
 335 };
 336
 337 static const struct intel_limit intel_limits_g4x_single_channel_lvds = {
 338         .dot = { .min = 20000, .max = 115000 },
 339         .vco = { .min = 1750000, .max = 3500000 },
 340         .n = { .min = 1, .max = 3 },
 341         .m = { .min = 104, .max = 138 },
 342         .m1 = { .min = 17, .max = 23 },
 343         .m2 = { .min = 5, .max = 11 },
 344         .p = { .min = 28, .max = 112 },
 345         .p1 = { .min = 2, .max = 8 },
 346         .p2 = { .dot_limit = 0,
 347                 .p2_slow = 14, .p2_fast = 14
 348         },
 349 };
 350
 351 static const struct intel_limit intel_limits_g4x_dual_channel_lvds = {
 352         .dot = { .min = 80000, .max = 224000 },
 353         .vco = { .min = 1750000, .max = 3500000 },
 354         .n = { .min = 1, .max = 3 },
 355         .m = { .min = 104, .max = 138 },
 356         .m1 = { .min = 17, .max = 23 },
 357         .m2 = { .min = 5, .max = 11 },
 358         .p = { .min = 14, .max = 42 },
 359         .p1 = { .min = 2, .max = 6 },
 360         .p2 = { .dot_limit = 0,
 361                 .p2_slow = 7, .p2_fast = 7
 362         },
 363 };
 364
 365 static const struct intel_limit intel_limits_pineview_sdvo = {
 366         .dot = { .min = 20000, .max = 400000},
 367         .vco = { .min = 1700000, .max = 3500000 },
 368         /* Pineview's Ncounter is a ring counter */
 369         .n = { .min = 3, .max = 6 },
 370         .m = { .min = 2, .max = 256 },
 371         /* Pineview only has one combined m divider, which we treat as m2. */
 372         .m1 = { .min = 0, .max = 0 },
 373         .m2 = { .min = 0, .max = 254 },
 374         .p = { .min = 5, .max = 80 },
 375         .p1 = { .min = 1, .max = 8 },
 376         .p2 = { .dot_limit = 200000,
 377                 .p2_slow = 10, .p2_fast = 5 },
 378 };
 379
 380 static const struct intel_limit intel_limits_pineview_lvds = {
 381         .dot = { .min = 20000, .max = 400000 },
 382         .vco = { .min = 1700000, .max = 3500000 },
 383         .n = { .min = 3, .max = 6 },
 384         .m = { .min = 2, .max = 256 },
 385         .m1 = { .min = 0, .max = 0 },
 386         .m2 = { .min = 0, .max = 254 },
 387         .p = { .min = 7, .max = 112 },
 388         .p1 = { .min = 1, .max = 8 },
 389         .p2 = { .dot_limit = 112000,
 390                 .p2_slow = 14, .p2_fast = 14 },
 391 };
 392
 393 /* Ironlake / Sandybridge
 394  *
 395  * We calculate clock using (register_value + 2) for N/M1/M2, so here
 396  * the range value for them is (actual_value - 2).
 397  */
 398 static const struct intel_limit intel_limits_ironlake_dac = {
 399         .dot = { .min = 25000, .max = 350000 },
 400         .vco = { .min = 1760000, .max = 3510000 },
 401         .n = { .min = 1, .max = 5 },
 402         .m = { .min = 79, .max = 127 },
 403         .m1 = { .min = 12, .max = 22 },
 404         .m2 = { .min = 5, .max = 9 },
 405         .p = { .min = 5, .max = 80 },
 406         .p1 = { .min = 1, .max = 8 },
 407         .p2 = { .dot_limit = 225000,
 408                 .p2_slow = 10, .p2_fast = 5 },
 409 };
 410
 411 static const struct intel_limit intel_limits_ironlake_single_lvds = {
 412         .dot = { .min = 25000, .max = 350000 },
 413         .vco = { .min = 1760000, .max = 3510000 },
 414         .n = { .min = 1, .max = 3 },
 415         .m = { .min = 79, .max = 118 },
 416         .m1 = { .min = 12, .max = 22 },
 417         .m2 = { .min = 5, .max = 9 },
 418         .p = { .min = 28, .max = 112 },
 419         .p1 = { .min = 2, .max = 8 },
 420         .p2 = { .dot_limit = 225000,
 421                 .p2_slow = 14, .p2_fast = 14 },
 422 };
 423
 424 static const struct intel_limit intel_limits_ironlake_dual_lvds = {
 425         .dot = { .min = 25000, .max = 350000 },
 426         .vco = { .min = 1760000, .max = 3510000 },
 427         .n = { .min = 1, .max = 3 },
 428         .m = { .min = 79, .max = 127 },
 429         .m1 = { .min = 12, .max = 22 },
 430         .m2 = { .min = 5, .max = 9 },
 431         .p = { .min = 14, .max = 56 },
 432         .p1 = { .min = 2, .max = 8 },
 433         .p2 = { .dot_limit = 225000,
 434                 .p2_slow = 7, .p2_fast = 7 },
 435 };
 436
 437 /* LVDS 100mhz refclk limits. */
 438 static const struct intel_limit intel_limits_ironlake_single_lvds_100m = {
 439         .dot = { .min = 25000, .max = 350000 },
 440         .vco = { .min = 1760000, .max = 3510000 },
 441         .n = { .min = 1, .max = 2 },
 442         .m = { .min = 79, .max = 126 },
 443         .m1 = { .min = 12, .max = 22 },
 444         .m2 = { .min = 5, .max = 9 },
 445         .p = { .min = 28, .max = 112 },
 446         .p1 = { .min = 2, .max = 8 },
 447         .p2 = { .dot_limit = 225000,
 448                 .p2_slow = 14, .p2_fast = 14 },
 449 };
 450
 451 static const struct intel_limit intel_limits_ironlake_dual_lvds_100m = {
 452         .dot = { .min = 25000, .max = 350000 },
 453         .vco = { .min = 1760000, .max = 3510000 },
 454         .n = { .min = 1, .max = 3 },
 455         .m = { .min = 79, .max = 126 },
 456         .m1 = { .min = 12, .max = 22 },
 457         .m2 = { .min = 5, .max = 9 },
 458         .p = { .min = 14, .max = 42 },
 459         .p1 = { .min = 2, .max = 6 },
 460         .p2 = { .dot_limit = 225000,
 461                 .p2_slow = 7, .p2_fast = 7 },
 462 };
 463
 464 static const struct intel_limit intel_limits_vlv = {
 465          /*
 466           * These are the data rate limits (measured in fast clocks)
 467           * since those are the strictest limits we have. The fast
 468           * clock and actual rate limits are more relaxed, so checking
 469           * them would make no difference.
 470           */
 471         .dot = { .min = 25000 * 5, .max = 270000 * 5 },
 472         .vco = { .min = 4000000, .max = 6000000 },
 473         .n = { .min = 1, .max = 7 },
 474         .m1 = { .min = 2, .max = 3 },
 475         .m2 = { .min = 11, .max = 156 },
 476         .p1 = { .min = 2, .max = 3 },
 477         .p2 = { .p2_slow = 2, .p2_fast = 20 }, /* slow=min, fast=max */
 478 };
 479
 480 static const struct intel_limit intel_limits_chv = {
 481         /*
 482          * These are the data rate limits (measured in fast clocks)
 483          * since those are the strictest limits we have.  The fast
 484          * clock and actual rate limits are more relaxed, so checking
 485          * them would make no difference.
 486          */
 487         .dot = { .min = 25000 * 5, .max = 540000 * 5},
 488         .vco = { .min = 4800000, .max = 6480000 },
 489         .n = { .min = 1, .max = 1 },
 490         .m1 = { .min = 2, .max = 2 },
 491         .m2 = { .min = 24 << 22, .max = 175 << 22 },
 492         .p1 = { .min = 2, .max = 4 },
 493         .p2 = { .p2_slow = 1, .p2_fast = 14 },
 494 };
 495
 496 static const struct intel_limit intel_limits_bxt = {
 497         /* FIXME: find real dot limits */
 498         .dot = { .min = 0, .max = INT_MAX },
 499         .vco = { .min = 4800000, .max = 6700000 },
 500         .n = { .min = 1, .max = 1 },
 501         .m1 = { .min = 2, .max = 2 },
 502         /* FIXME: find real m2 limits */
 503         .m2 = { .min = 2 << 22, .max = 255 << 22 },
 504         .p1 = { .min = 2, .max = 4 },
 505         .p2 = { .p2_slow = 1, .p2_fast = 20 },
 506 };
 507
 508 static void
 509 skl_wa_528(struct drm_i915_private *dev_priv, int pipe, bool enable)
 510 {
 511         if (IS_SKYLAKE(dev_priv) || IS_BROXTON(dev_priv))
 512                 return;
 513
 514         if (enable)
 515                 I915_WRITE(CHICKEN_PIPESL_1(pipe), HSW_FBCQ_DIS);
 516         else
 517                 I915_WRITE(CHICKEN_PIPESL_1(pipe), 0);
 518 }
 519
 520 static void
 521 skl_wa_clkgate(struct drm_i915_private *dev_priv, int pipe, bool enable)
 522 {
 523         if (IS_SKYLAKE(dev_priv) || IS_BROXTON(dev_priv))
 524                 return;
 525
 526         if (enable)
 527                 I915_WRITE(CLKGATE_DIS_PSL(pipe),
 528                            DUPS1_GATING_DIS | DUPS2_GATING_DIS);
 529         else
 530                 I915_WRITE(CLKGATE_DIS_PSL(pipe),
 531                            I915_READ(CLKGATE_DIS_PSL(pipe)) &
 532                            ~(DUPS1_GATING_DIS | DUPS2_GATING_DIS));
 533 }
 534
 535 static bool
 536 needs_modeset(const struct drm_crtc_state *state)
 537 {
 538         return drm_atomic_crtc_needs_modeset(state);
 539 }
 540
 541 /*
 542  * Platform specific helpers to calculate the port PLL loopback- (clock.m),
 543  * and post-divider (clock.p) values, pre- (clock.vco) and post-divided fast
 544  * (clock.dot) clock rates. This fast dot clock is fed to the port's IO logic.
 545  * The helpers' return value is the rate of the clock that is fed to the
 546  * display engine's pipe which can be the above fast dot clock rate or a
 547  * divided-down version of it.
 548  */
 549 /* m1 is reserved as 0 in Pineview, n is a ring counter */
 550 static int pnv_calc_dpll_params(int refclk, struct dpll *clock)
 551 {
 552         clock->m = clock->m2 + 2;
 553         clock->p = clock->p1 * clock->p2;
 554         if (WARN_ON(clock->n == 0 || clock->p == 0))
 555                 return 0;
 556         clock->vco = DIV_ROUND_CLOSEST(refclk * clock->m, clock->n);
 557         clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p);
 558
 559         return clock->dot;
 560 }
 561
 562 static uint32_t i9xx_dpll_compute_m(struct dpll *dpll)
 563 {
 564         return 5 * (dpll->m1 + 2) + (dpll->m2 + 2);
 565 }
 566
 567 static int i9xx_calc_dpll_params(int refclk, struct dpll *clock)
 568 {
 569         clock->m = i9xx_dpll_compute_m(clock);
 570         clock->p = clock->p1 * clock->p2;
 571         if (WARN_ON(clock->n + 2 == 0 || clock->p == 0))
 572                 return 0;
 573         clock->vco = DIV_ROUND_CLOSEST(refclk * clock->m, clock->n + 2);
 574         clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p);
 575
 576         return clock->dot;
 577 }
 578
 579 static int vlv_calc_dpll_params(int refclk, struct dpll *clock)
 580 {
 581         clock->m = clock->m1 * clock->m2;
 582         clock->p = clock->p1 * clock->p2;
 583         if (WARN_ON(clock->n == 0 || clock->p == 0))
 584                 return 0;
 585         clock->vco = DIV_ROUND_CLOSEST(refclk * clock->m, clock->n);
 586         clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p);
 587
 588         return clock->dot / 5;
 589 }
 590
 591 int chv_calc_dpll_params(int refclk, struct dpll *clock)
 592 {
 593         clock->m = clock->m1 * clock->m2;
 594         clock->p = clock->p1 * clock->p2;
 595         if (WARN_ON(clock->n == 0 || clock->p == 0))
 596                 return 0;
 597         clock->vco = DIV_ROUND_CLOSEST_ULL((uint64_t)refclk * clock->m,
 598                         clock->n << 22);
 599         clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p);
 600
 601         return clock->dot / 5;
 602 }
 603
 604 #define INTELPllInvalid(s)   do { /* DRM_DEBUG(s); */ return false; } while (0)
 605
 606 /*
 607  * Returns whether the given set of divisors are valid for a given refclk with
 608  * the given connectors.
 609  */
 610 static bool intel_PLL_is_valid(struct drm_i915_private *dev_priv,
 611                                const struct intel_limit *limit,
 612                                const struct dpll *clock)
 613 {
 614         if (clock->n   < limit->n.min   || limit->n.max   < clock->n)
 615                 INTELPllInvalid("n out of range\n");
 616         if (clock->p1  < limit->p1.min  || limit->p1.max  < clock->p1)
 617                 INTELPllInvalid("p1 out of range\n");
 618         if (clock->m2  < limit->m2.min  || limit->m2.max  < clock->m2)
 619                 INTELPllInvalid("m2 out of range\n");
 620         if (clock->m1  < limit->m1.min  || limit->m1.max  < clock->m1)
 621                 INTELPllInvalid("m1 out of range\n");
 622
 623         if (!IS_PINEVIEW(dev_priv) && !IS_VALLEYVIEW(dev_priv) &&
 624             !IS_CHERRYVIEW(dev_priv) && !IS_GEN9_LP(dev_priv))
 625                 if (clock->m1 <= clock->m2)
 626                         INTELPllInvalid("m1 <= m2\n");
 627
 628         if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv) &&
 629             !IS_GEN9_LP(dev_priv)) {
 630                 if (clock->p < limit->p.min || limit->p.max < clock->p)
 631                         INTELPllInvalid("p out of range\n");
 632                 if (clock->m < limit->m.min || limit->m.max < clock->m)
 633                         INTELPllInvalid("m out of range\n");
 634         }
 635
 636         if (clock->vco < limit->vco.min || limit->vco.max < clock->vco)
 637                 INTELPllInvalid("vco out of range\n");
 638         /* XXX: We may need to be checking "Dot clock" depending on the multiplier,
 639          * connector, etc., rather than just a single range.
 640          */
 641         if (clock->dot < limit->dot.min || limit->dot.max < clock->dot)
 642                 INTELPllInvalid("dot out of range\n");
 643
 644         return true;
 645 }
 646
 647 static int
 648 i9xx_select_p2_div(const struct intel_limit *limit,
 649                    const struct intel_crtc_state *crtc_state,
 650                    int target)
 651 {
 652         struct drm_device *dev = crtc_state->base.crtc->dev;
 653
 654         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
 655                 /*
 656                  * For LVDS just rely on its current settings for dual-channel.
 657                  * We haven't figured out how to reliably set up different
 658                  * single/dual channel state, if we even can.
 659                  */
 660                 if (intel_is_dual_link_lvds(dev))
 661                         return limit->p2.p2_fast;
 662                 else
 663                         return limit->p2.p2_slow;
 664         } else {
 665                 if (target < limit->p2.dot_limit)
 666                         return limit->p2.p2_slow;
 667                 else
 668                         return limit->p2.p2_fast;
 669         }
 670 }
 671
 672 /*
 673  * Returns a set of divisors for the desired target clock with the given
 674  * refclk, or FALSE.  The returned values represent the clock equation:
 675  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
 676  *
 677  * Target and reference clocks are specified in kHz.
 678  *
 679  * If match_clock is provided, then best_clock P divider must match the P
 680  * divider from @match_clock used for LVDS downclocking.
 681  */
 682 static bool
 683 i9xx_find_best_dpll(const struct intel_limit *limit,
 684                     struct intel_crtc_state *crtc_state,
 685                     int target, int refclk, struct dpll *match_clock,
 686                     struct dpll *best_clock)
 687 {
 688         struct drm_device *dev = crtc_state->base.crtc->dev;
 689         struct dpll clock;
 690         int err = target;
 691
 692         memset(best_clock, 0, sizeof(*best_clock));
 693
 694         clock.p2 = i9xx_select_p2_div(limit, crtc_state, target);
 695
 696         for (clock.m1 = limit->m1.min; clock.m1 <= limit->m1.max;
 697              clock.m1++) {
 698                 for (clock.m2 = limit->m2.min;
 699                      clock.m2 <= limit->m2.max; clock.m2++) {
 700                         if (clock.m2 >= clock.m1)
 701                                 break;
 702                         for (clock.n = limit->n.min;
 703                              clock.n <= limit->n.max; clock.n++) {
 704                                 for (clock.p1 = limit->p1.min;
 705                                         clock.p1 <= limit->p1.max; clock.p1++) {
 706                                         int this_err;
 707
 708                                         i9xx_calc_dpll_params(refclk, &clock);
 709                                         if (!intel_PLL_is_valid(to_i915(dev),
 710                                                                 limit,
 711                                                                 &clock))
 712                                                 continue;
 713                                         if (match_clock &&
 714                                             clock.p != match_clock->p)
 715                                                 continue;
 716
 717                                         this_err = abs(clock.dot - target);
 718                                         if (this_err < err) {
 719                                                 *best_clock = clock;
 720                                                 err = this_err;
 721                                         }
 722                                 }
 723                         }
 724                 }
 725         }
 726
 727         return (err != target);
 728 }
 729
 730 /*
 731  * Returns a set of divisors for the desired target clock with the given
 732  * refclk, or FALSE.  The returned values represent the clock equation:
 733  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
 734  *
 735  * Target and reference clocks are specified in kHz.
 736  *
 737  * If match_clock is provided, then best_clock P divider must match the P
 738  * divider from @match_clock used for LVDS downclocking.
 739  */
 740 static bool
 741 pnv_find_best_dpll(const struct intel_limit *limit,
 742                    struct intel_crtc_state *crtc_state,
 743                    int target, int refclk, struct dpll *match_clock,
 744                    struct dpll *best_clock)
 745 {
 746         struct drm_device *dev = crtc_state->base.crtc->dev;
 747         struct dpll clock;
 748         int err = target;
 749
 750         memset(best_clock, 0, sizeof(*best_clock));
 751
 752         clock.p2 = i9xx_select_p2_div(limit, crtc_state, target);
 753
 754         for (clock.m1 = limit->m1.min; clock.m1 <= limit->m1.max;
 755              clock.m1++) {
 756                 for (clock.m2 = limit->m2.min;
 757                      clock.m2 <= limit->m2.max; clock.m2++) {
 758                         for (clock.n = limit->n.min;
 759                              clock.n <= limit->n.max; clock.n++) {
 760                                 for (clock.p1 = limit->p1.min;
 761                                         clock.p1 <= limit->p1.max; clock.p1++) {
 762                                         int this_err;
 763
 764                                         pnv_calc_dpll_params(refclk, &clock);
 765                                         if (!intel_PLL_is_valid(to_i915(dev),
 766                                                                 limit,
 767                                                                 &clock))
 768                                                 continue;
 769                                         if (match_clock &&
 770                                             clock.p != match_clock->p)
 771                                                 continue;
 772
 773                                         this_err = abs(clock.dot - target);
 774                                         if (this_err < err) {
 775                                                 *best_clock = clock;
 776                                                 err = this_err;
 777                                         }
 778                                 }
 779                         }
 780                 }
 781         }
 782
 783         return (err != target);
 784 }
 785
 786 /*
 787  * Returns a set of divisors for the desired target clock with the given
 788  * refclk, or FALSE.  The returned values represent the clock equation:
 789  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
 790  *
 791  * Target and reference clocks are specified in kHz.
 792  *
 793  * If match_clock is provided, then best_clock P divider must match the P
 794  * divider from @match_clock used for LVDS downclocking.
 795  */
 796 static bool
 797 g4x_find_best_dpll(const struct intel_limit *limit,
 798                    struct intel_crtc_state *crtc_state,
 799                    int target, int refclk, struct dpll *match_clock,
 800                    struct dpll *best_clock)
 801 {
 802         struct drm_device *dev = crtc_state->base.crtc->dev;
 803         struct dpll clock;
 804         int max_n;
 805         bool found = false;
 806         /* approximately equals target * 0.00585 */
 807         int err_most = (target >> 8) + (target >> 9);
 808
 809         memset(best_clock, 0, sizeof(*best_clock));
 810
 811         clock.p2 = i9xx_select_p2_div(limit, crtc_state, target);
 812
 813         max_n = limit->n.max;
 814         /* based on hardware requirement, prefer smaller n to precision */
 815         for (clock.n = limit->n.min; clock.n <= max_n; clock.n++) {
 816                 /* based on hardware requirement, prefere larger m1,m2 */
 817                 for (clock.m1 = limit->m1.max;
 818                      clock.m1 >= limit->m1.min; clock.m1--) {
 819                         for (clock.m2 = limit->m2.max;
 820                              clock.m2 >= limit->m2.min; clock.m2--) {
 821                                 for (clock.p1 = limit->p1.max;
 822                                      clock.p1 >= limit->p1.min; clock.p1--) {
 823                                         int this_err;
 824
 825                                         i9xx_calc_dpll_params(refclk, &clock);
 826                                         if (!intel_PLL_is_valid(to_i915(dev),
 827                                                                 limit,
 828                                                                 &clock))
 829                                                 continue;
 830
 831                                         this_err = abs(clock.dot - target);
 832                                         if (this_err < err_most) {
 833                                                 *best_clock = clock;
 834                                                 err_most = this_err;
 835                                                 max_n = clock.n;
 836                                                 found = true;
 837                                         }
 838                                 }
 839                         }
 840                 }
 841         }
 842         return found;
 843 }
 844
 845 /*
 846  * Check if the calculated PLL configuration is more optimal compared to the
 847  * best configuration and error found so far. Return the calculated error.
 848  */
 849 static bool vlv_PLL_is_optimal(struct drm_device *dev, int target_freq,
 850                                const struct dpll *calculated_clock,
 851                                const struct dpll *best_clock,
 852                                unsigned int best_error_ppm,
 853                                unsigned int *error_ppm)
 854 {
 855         /*
 856          * For CHV ignore the error and consider only the P value.
 857          * Prefer a bigger P value based on HW requirements.
 858          */
 859         if (IS_CHERRYVIEW(to_i915(dev))) {
 860                 *error_ppm = 0;
 861
 862                 return calculated_clock->p > best_clock->p;
 863         }
 864
 865         if (WARN_ON_ONCE(!target_freq))
 866                 return false;
 867
 868         *error_ppm = div_u64(1000000ULL *
 869                                 abs(target_freq - calculated_clock->dot),
 870                              target_freq);
 871         /*
 872          * Prefer a better P value over a better (smaller) error if the error
 873          * is small. Ensure this preference for future configurations too by
 874          * setting the error to 0.
 875          */
 876         if (*error_ppm < 100 && calculated_clock->p > best_clock->p) {
 877                 *error_ppm = 0;
 878
 879                 return true;
 880         }
 881
 882         return *error_ppm + 10 < best_error_ppm;
 883 }
 884
 885 /*
 886  * Returns a set of divisors for the desired target clock with the given
 887  * refclk, or FALSE.  The returned values represent the clock equation:
 888  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
 889  */
 890 static bool
 891 vlv_find_best_dpll(const struct intel_limit *limit,
 892                    struct intel_crtc_state *crtc_state,
 893                    int target, int refclk, struct dpll *match_clock,
 894                    struct dpll *best_clock)
 895 {
 896         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
 897         struct drm_device *dev = crtc->base.dev;
 898         struct dpll clock;
 899         unsigned int bestppm = 1000000;
 900         /* min update 19.2 MHz */
 901         int max_n = min(limit->n.max, refclk / 19200);
 902         bool found = false;
 903
 904         target *= 5; /* fast clock */
 905
 906         memset(best_clock, 0, sizeof(*best_clock));
 907
 908         /* based on hardware requirement, prefer smaller n to precision */
 909         for (clock.n = limit->n.min; clock.n <= max_n; clock.n++) {
 910                 for (clock.p1 = limit->p1.max; clock.p1 >= limit->p1.min; clock.p1--) {
 911                         for (clock.p2 = limit->p2.p2_fast; clock.p2 >= limit->p2.p2_slow;
 912                              clock.p2 -= clock.p2 > 10 ? 2 : 1) {
 913                                 clock.p = clock.p1 * clock.p2;
 914                                 /* based on hardware requirement, prefer bigger m1,m2 values */
 915                                 for (clock.m1 = limit->m1.min; clock.m1 <= limit->m1.max; clock.m1++) {
 916                                         unsigned int ppm;
 917
 918                                         clock.m2 = DIV_ROUND_CLOSEST(target * clock.p * clock.n,
 919                                                                      refclk * clock.m1);
 920
 921                                         vlv_calc_dpll_params(refclk, &clock);
 922
 923                                         if (!intel_PLL_is_valid(to_i915(dev),
 924                                                                 limit,
 925                                                                 &clock))
 926                                                 continue;
 927
 928                                         if (!vlv_PLL_is_optimal(dev, target,
 929                                                                 &clock,
 930                                                                 best_clock,
 931                                                                 bestppm, &ppm))
 932                                                 continue;
 933
 934                                         *best_clock = clock;
 935                                         bestppm = ppm;
 936                                         found = true;
 937                                 }
 938                         }
 939                 }
 940         }
 941
 942         return found;
 943 }
 944
 945 /*
 946  * Returns a set of divisors for the desired target clock with the given
 947  * refclk, or FALSE.  The returned values represent the clock equation:
 948  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
 949  */
 950 static bool
 951 chv_find_best_dpll(const struct intel_limit *limit,
 952                    struct intel_crtc_state *crtc_state,
 953                    int target, int refclk, struct dpll *match_clock,
 954                    struct dpll *best_clock)
 955 {
 956         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
 957         struct drm_device *dev = crtc->base.dev;
 958         unsigned int best_error_ppm;
 959         struct dpll clock;
 960         uint64_t m2;
 961         int found = false;
 962
 963         memset(best_clock, 0, sizeof(*best_clock));
 964         best_error_ppm = 1000000;
 965
 966         /*
 967          * Based on hardware doc, the n always set to 1, and m1 always
 968          * set to 2.  If requires to support 200Mhz refclk, we need to
 969          * revisit this because n may not 1 anymore.
 970          */
 971         clock.n = 1, clock.m1 = 2;
 972         target *= 5;    /* fast clock */
 973
 974         for (clock.p1 = limit->p1.max; clock.p1 >= limit->p1.min; clock.p1--) {
 975                 for (clock.p2 = limit->p2.p2_fast;
 976                                 clock.p2 >= limit->p2.p2_slow;
 977                                 clock.p2 -= clock.p2 > 10 ? 2 : 1) {
 978                         unsigned int error_ppm;
 979
 980                         clock.p = clock.p1 * clock.p2;
 981
 982                         m2 = DIV_ROUND_CLOSEST_ULL(((uint64_t)target * clock.p *
 983                                         clock.n) << 22, refclk * clock.m1);
 984
 985                         if (m2 > INT_MAX/clock.m1)
 986                                 continue;
 987
 988                         clock.m2 = m2;
 989
 990                         chv_calc_dpll_params(refclk, &clock);
 991
 992                         if (!intel_PLL_is_valid(to_i915(dev), limit, &clock))
 993                                 continue;
 994
 995                         if (!vlv_PLL_is_optimal(dev, target, &clock, best_clock,
 996                                                 best_error_ppm, &error_ppm))
 997                                 continue;
 998
 999                         *best_clock = clock;
1000                         best_error_ppm = error_ppm;
1001                         found = true;
1002                 }
1003         }
1004
1005         return found;
1006 }
1007
1008 bool bxt_find_best_dpll(struct intel_crtc_state *crtc_state, int target_clock,
1009                         struct dpll *best_clock)
1010 {
1011         int refclk = 100000;
1012         const struct intel_limit *limit = &intel_limits_bxt;
1013
1014         return chv_find_best_dpll(limit, crtc_state,
1015                                   target_clock, refclk, NULL, best_clock);
1016 }
1017
1018 bool intel_crtc_active(struct intel_crtc *crtc)
1019 {
1020         /* Be paranoid as we can arrive here with only partial
1021          * state retrieved from the hardware during setup.
1022          *
1023          * We can ditch the adjusted_mode.crtc_clock check as soon
1024          * as Haswell has gained clock readout/fastboot support.
1025          *
1026          * We can ditch the crtc->primary->state->fb check as soon as we can
1027          * properly reconstruct framebuffers.
1028          *
1029          * FIXME: The intel_crtc->active here should be switched to
1030          * crtc->state->active once we have proper CRTC states wired up
1031          * for atomic.
1032          */
1033         return crtc->active && crtc->base.primary->state->fb &&
1034                 crtc->config->base.adjusted_mode.crtc_clock;
1035 }
1036
1037 enum transcoder intel_pipe_to_cpu_transcoder(struct drm_i915_private *dev_priv,
1038                                              enum pipe pipe)
1039 {
1040         struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
1041
1042         return crtc->config->cpu_transcoder;
1043 }
1044
1045 static bool pipe_scanline_is_moving(struct drm_i915_private *dev_priv,
1046                                     enum pipe pipe)
1047 {
1048         i915_reg_t reg = PIPEDSL(pipe);
1049         u32 line1, line2;
1050         u32 line_mask;
1051
1052         if (IS_GEN2(dev_priv))
1053                 line_mask = DSL_LINEMASK_GEN2;
1054         else
1055                 line_mask = DSL_LINEMASK_GEN3;
1056
1057         line1 = I915_READ(reg) & line_mask;
1058         msleep(5);
1059         line2 = I915_READ(reg) & line_mask;
1060
1061         return line1 != line2;
1062 }
1063
1064 static void wait_for_pipe_scanline_moving(struct intel_crtc *crtc, bool state)
1065 {
1066         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1067         enum pipe pipe = crtc->pipe;
1068
1069         /* Wait for the display line to settle/start moving */
1070         if (wait_for(pipe_scanline_is_moving(dev_priv, pipe) == state, 100))
1071                 DRM_ERROR("pipe %c scanline %s wait timed out\n",
1072                           pipe_name(pipe), onoff(state));
1073 }
1074
1075 static void intel_wait_for_pipe_scanline_stopped(struct intel_crtc *crtc)
1076 {
1077         wait_for_pipe_scanline_moving(crtc, false);
1078 }
1079
1080 static void intel_wait_for_pipe_scanline_moving(struct intel_crtc *crtc)
1081 {
1082         wait_for_pipe_scanline_moving(crtc, true);
1083 }
1084
1085 static void
1086 intel_wait_for_pipe_off(const struct intel_crtc_state *old_crtc_state)
1087 {
1088         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc);
1089         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1090
1091         if (INTEL_GEN(dev_priv) >= 4) {
1092                 enum transcoder cpu_transcoder = old_crtc_state->cpu_transcoder;
1093                 i915_reg_t reg = PIPECONF(cpu_transcoder);
1094
1095                 /* Wait for the Pipe State to go off */
1096                 if (intel_wait_for_register(dev_priv,
1097                                             reg, I965_PIPECONF_ACTIVE, 0,
1098                                             100))
1099                         WARN(1, "pipe_off wait timed out\n");
1100         } else {
1101                 intel_wait_for_pipe_scanline_stopped(crtc);
1102         }
1103 }
1104
1105 /* Only for pre-ILK configs */
1106 void assert_pll(struct drm_i915_private *dev_priv,
1107                 enum pipe pipe, bool state)
1108 {
1109         u32 val;
1110         bool cur_state;
1111
1112         val = I915_READ(DPLL(pipe));
1113         cur_state = !!(val & DPLL_VCO_ENABLE);
1114         I915_STATE_WARN(cur_state != state,
1115              "PLL state assertion failure (expected %s, current %s)\n",
1116                         onoff(state), onoff(cur_state));
1117 }
1118
1119 /* XXX: the dsi pll is shared between MIPI DSI ports */
1120 void assert_dsi_pll(struct drm_i915_private *dev_priv, bool state)
1121 {
1122         u32 val;
1123         bool cur_state;
1124
1125         mutex_lock(&dev_priv->sb_lock);
1126         val = vlv_cck_read(dev_priv, CCK_REG_DSI_PLL_CONTROL);
1127         mutex_unlock(&dev_priv->sb_lock);
1128
1129         cur_state = val & DSI_PLL_VCO_EN;
1130         I915_STATE_WARN(cur_state != state,
1131              "DSI PLL state assertion failure (expected %s, current %s)\n",
1132                         onoff(state), onoff(cur_state));
1133 }
1134
1135 static void assert_fdi_tx(struct drm_i915_private *dev_priv,
1136                           enum pipe pipe, bool state)
1137 {
1138         bool cur_state;
1139         enum transcoder cpu_transcoder = intel_pipe_to_cpu_transcoder(dev_priv,
1140                                                                       pipe);
1141
1142         if (HAS_DDI(dev_priv)) {
1143                 /* DDI does not have a specific FDI_TX register */
1144                 u32 val = I915_READ(TRANS_DDI_FUNC_CTL(cpu_transcoder));
1145                 cur_state = !!(val & TRANS_DDI_FUNC_ENABLE);
1146         } else {
1147                 u32 val = I915_READ(FDI_TX_CTL(pipe));
1148                 cur_state = !!(val & FDI_TX_ENABLE);
1149         }
1150         I915_STATE_WARN(cur_state != state,
1151              "FDI TX state assertion failure (expected %s, current %s)\n",
1152                         onoff(state), onoff(cur_state));
1153 }
1154 #define assert_fdi_tx_enabled(d, p) assert_fdi_tx(d, p, true)
1155 #define assert_fdi_tx_disabled(d, p) assert_fdi_tx(d, p, false)
1156
1157 static void assert_fdi_rx(struct drm_i915_private *dev_priv,
1158                           enum pipe pipe, bool state)
1159 {
1160         u32 val;
1161         bool cur_state;
1162
1163         val = I915_READ(FDI_RX_CTL(pipe));
1164         cur_state = !!(val & FDI_RX_ENABLE);
1165         I915_STATE_WARN(cur_state != state,
1166              "FDI RX state assertion failure (expected %s, current %s)\n",
1167                         onoff(state), onoff(cur_state));
1168 }
1169 #define assert_fdi_rx_enabled(d, p) assert_fdi_rx(d, p, true)
1170 #define assert_fdi_rx_disabled(d, p) assert_fdi_rx(d, p, false)
1171
1172 static void assert_fdi_tx_pll_enabled(struct drm_i915_private *dev_priv,
1173                                       enum pipe pipe)
1174 {
1175         u32 val;
1176
1177         /* ILK FDI PLL is always enabled */
1178         if (IS_GEN5(dev_priv))
1179                 return;
1180
1181         /* On Haswell, DDI ports are responsible for the FDI PLL setup */
1182         if (HAS_DDI(dev_priv))
1183                 return;
1184
1185         val = I915_READ(FDI_TX_CTL(pipe));
1186         I915_STATE_WARN(!(val & FDI_TX_PLL_ENABLE), "FDI TX PLL assertion failure, should be active but is disabled\n");
1187 }
1188
1189 void assert_fdi_rx_pll(struct drm_i915_private *dev_priv,
1190                        enum pipe pipe, bool state)
1191 {
1192         u32 val;
1193         bool cur_state;
1194
1195         val = I915_READ(FDI_RX_CTL(pipe));
1196         cur_state = !!(val & FDI_RX_PLL_ENABLE);
1197         I915_STATE_WARN(cur_state != state,
1198              "FDI RX PLL assertion failure (expected %s, current %s)\n",
1199                         onoff(state), onoff(cur_state));
1200 }
1201
1202 void assert_panel_unlocked(struct drm_i915_private *dev_priv, enum pipe pipe)
1203 {
1204         i915_reg_t pp_reg;
1205         u32 val;
1206         enum pipe panel_pipe = INVALID_PIPE;
1207         bool locked = true;
1208
1209         if (WARN_ON(HAS_DDI(dev_priv)))
1210                 return;
1211
1212         if (HAS_PCH_SPLIT(dev_priv)) {
1213                 u32 port_sel;
1214
1215                 pp_reg = PP_CONTROL(0);
1216                 port_sel = I915_READ(PP_ON_DELAYS(0)) & PANEL_PORT_SELECT_MASK;
1217
1218                 switch (port_sel) {
1219                 case PANEL_PORT_SELECT_LVDS:
1220                         intel_lvds_port_enabled(dev_priv, PCH_LVDS, &panel_pipe);
1221                         break;
1222                 case PANEL_PORT_SELECT_DPA:
1223                         intel_dp_port_enabled(dev_priv, DP_A, PORT_A, &panel_pipe);
1224                         break;
1225                 case PANEL_PORT_SELECT_DPC:
1226                         intel_dp_port_enabled(dev_priv, PCH_DP_C, PORT_C, &panel_pipe);
1227                         break;
1228                 case PANEL_PORT_SELECT_DPD:
1229                         intel_dp_port_enabled(dev_priv, PCH_DP_D, PORT_D, &panel_pipe);
1230                         break;
1231                 default:
1232                         MISSING_CASE(port_sel);
1233                         break;
1234                 }
1235         } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
1236                 /* presumably write lock depends on pipe, not port select */
1237                 pp_reg = PP_CONTROL(pipe);
1238                 panel_pipe = pipe;
1239         } else {
1240                 u32 port_sel;
1241
1242                 pp_reg = PP_CONTROL(0);
1243                 port_sel = I915_READ(PP_ON_DELAYS(0)) & PANEL_PORT_SELECT_MASK;
1244
1245                 WARN_ON(port_sel != PANEL_PORT_SELECT_LVDS);
1246                 intel_lvds_port_enabled(dev_priv, LVDS, &panel_pipe);
1247         }
1248
1249         val = I915_READ(pp_reg);
1250         if (!(val & PANEL_POWER_ON) ||
1251             ((val & PANEL_UNLOCK_MASK) == PANEL_UNLOCK_REGS))
1252                 locked = false;
1253
1254         I915_STATE_WARN(panel_pipe == pipe && locked,
1255              "panel assertion failure, pipe %c regs locked\n",
1256              pipe_name(pipe));
1257 }
1258
1259 void assert_pipe(struct drm_i915_private *dev_priv,
1260                  enum pipe pipe, bool state)
1261 {
1262         bool cur_state;
1263         enum transcoder cpu_transcoder = intel_pipe_to_cpu_transcoder(dev_priv,
1264                                                                       pipe);
1265         enum intel_display_power_domain power_domain;
1266
1267         /* we keep both pipes enabled on 830 */
1268         if (IS_I830(dev_priv))
1269                 state = true;
1270
1271         power_domain = POWER_DOMAIN_TRANSCODER(cpu_transcoder);
1272         if (intel_display_power_get_if_enabled(dev_priv, power_domain)) {
1273                 u32 val = I915_READ(PIPECONF(cpu_transcoder));
1274                 cur_state = !!(val & PIPECONF_ENABLE);
1275
1276                 intel_display_power_put(dev_priv, power_domain);
1277         } else {
1278                 cur_state = false;
1279         }
1280
1281         I915_STATE_WARN(cur_state != state,
1282              "pipe %c assertion failure (expected %s, current %s)\n",
1283                         pipe_name(pipe), onoff(state), onoff(cur_state));
1284 }
1285
1286 static void assert_plane(struct intel_plane *plane, bool state)
1287 {
1288         enum pipe pipe;
1289         bool cur_state;
1290
1291         cur_state = plane->get_hw_state(plane, &pipe);
1292
1293         I915_STATE_WARN(cur_state != state,
1294                         "%s assertion failure (expected %s, current %s)\n",
1295                         plane->base.name, onoff(state), onoff(cur_state));
1296 }
1297
1298 #define assert_plane_enabled(p) assert_plane(p, true)
1299 #define assert_plane_disabled(p) assert_plane(p, false)
1300
1301 static void assert_planes_disabled(struct intel_crtc *crtc)
1302 {
1303         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1304         struct intel_plane *plane;
1305
1306         for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane)
1307                 assert_plane_disabled(plane);
1308 }
1309
1310 static void assert_vblank_disabled(struct drm_crtc *crtc)
1311 {
1312         if (I915_STATE_WARN_ON(drm_crtc_vblank_get(crtc) == 0))
1313                 drm_crtc_vblank_put(crtc);
1314 }
1315
1316 void assert_pch_transcoder_disabled(struct drm_i915_private *dev_priv,
1317                                     enum pipe pipe)
1318 {
1319         u32 val;
1320         bool enabled;
1321
1322         val = I915_READ(PCH_TRANSCONF(pipe));
1323         enabled = !!(val & TRANS_ENABLE);
1324         I915_STATE_WARN(enabled,
1325              "transcoder assertion failed, should be off on pipe %c but is still active\n",
1326              pipe_name(pipe));
1327 }
1328
1329 static void assert_pch_dp_disabled(struct drm_i915_private *dev_priv,
1330                                    enum pipe pipe, enum port port,
1331                                    i915_reg_t dp_reg)
1332 {
1333         enum pipe port_pipe;
1334         bool state;
1335
1336         state = intel_dp_port_enabled(dev_priv, dp_reg, port, &port_pipe);
1337
1338         I915_STATE_WARN(state && port_pipe == pipe,
1339                         "PCH DP %c enabled on transcoder %c, should be disabled\n",
1340                         port_name(port), pipe_name(pipe));
1341
1342         I915_STATE_WARN(HAS_PCH_IBX(dev_priv) && !state && port_pipe == PIPE_B,
1343                         "IBX PCH DP %c still using transcoder B\n",
1344                         port_name(port));
1345 }
1346
1347 static void assert_pch_hdmi_disabled(struct drm_i915_private *dev_priv,
1348                                      enum pipe pipe, enum port port,
1349                                      i915_reg_t hdmi_reg)
1350 {
1351         enum pipe port_pipe;
1352         bool state;
1353
1354         state = intel_sdvo_port_enabled(dev_priv, hdmi_reg, &port_pipe);
1355
1356         I915_STATE_WARN(state && port_pipe == pipe,
1357                         "PCH HDMI %c enabled on transcoder %c, should be disabled\n",
1358                         port_name(port), pipe_name(pipe));
1359
1360         I915_STATE_WARN(HAS_PCH_IBX(dev_priv) && !state && port_pipe == PIPE_B,
1361                         "IBX PCH HDMI %c still using transcoder B\n",
1362                         port_name(port));
1363 }
1364
1365 static void assert_pch_ports_disabled(struct drm_i915_private *dev_priv,
1366                                       enum pipe pipe)
1367 {
1368         enum pipe port_pipe;
1369
1370         assert_pch_dp_disabled(dev_priv, pipe, PORT_B, PCH_DP_B);
1371         assert_pch_dp_disabled(dev_priv, pipe, PORT_C, PCH_DP_C);
1372         assert_pch_dp_disabled(dev_priv, pipe, PORT_D, PCH_DP_D);
1373
1374         I915_STATE_WARN(intel_crt_port_enabled(dev_priv, PCH_ADPA, &port_pipe) &&
1375                         port_pipe == pipe,
1376                         "PCH VGA enabled on transcoder %c, should be disabled\n",
1377                         pipe_name(pipe));
1378
1379         I915_STATE_WARN(intel_lvds_port_enabled(dev_priv, PCH_LVDS, &port_pipe) &&
1380                         port_pipe == pipe,
1381                         "PCH LVDS enabled on transcoder %c, should be disabled\n",
1382                         pipe_name(pipe));
1383
1384         assert_pch_hdmi_disabled(dev_priv, pipe, PORT_B, PCH_HDMIB);
1385         assert_pch_hdmi_disabled(dev_priv, pipe, PORT_C, PCH_HDMIC);
1386         assert_pch_hdmi_disabled(dev_priv, pipe, PORT_D, PCH_HDMID);
1387 }
1388
1389 static void _vlv_enable_pll(struct intel_crtc *crtc,
1390                             const struct intel_crtc_state *pipe_config)
1391 {
1392         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1393         enum pipe pipe = crtc->pipe;
1394
1395         I915_WRITE(DPLL(pipe), pipe_config->dpll_hw_state.dpll);
1396         POSTING_READ(DPLL(pipe));
1397         udelay(150);
1398
1399         if (intel_wait_for_register(dev_priv,
1400                                     DPLL(pipe),
1401                                     DPLL_LOCK_VLV,
1402                                     DPLL_LOCK_VLV,
1403                                     1))
1404                 DRM_ERROR("DPLL %d failed to lock\n", pipe);
1405 }
1406
1407 static void vlv_enable_pll(struct intel_crtc *crtc,
1408                            const struct intel_crtc_state *pipe_config)
1409 {
1410         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1411         enum pipe pipe = crtc->pipe;
1412
1413         assert_pipe_disabled(dev_priv, pipe);
1414
1415         /* PLL is protected by panel, make sure we can write it */
1416         assert_panel_unlocked(dev_priv, pipe);
1417
1418         if (pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE)
1419                 _vlv_enable_pll(crtc, pipe_config);
1420
1421         I915_WRITE(DPLL_MD(pipe), pipe_config->dpll_hw_state.dpll_md);
1422         POSTING_READ(DPLL_MD(pipe));
1423 }
1424
1425
1426 static void _chv_enable_pll(struct intel_crtc *crtc,
1427                             const struct intel_crtc_state *pipe_config)
1428 {
1429         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1430         enum pipe pipe = crtc->pipe;
1431         enum dpio_channel port = vlv_pipe_to_channel(pipe);
1432         u32 tmp;
1433
1434         mutex_lock(&dev_priv->sb_lock);
1435
1436         /* Enable back the 10bit clock to display controller */
1437         tmp = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW14(port));
1438         tmp |= DPIO_DCLKP_EN;
1439         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW14(port), tmp);
1440
1441         mutex_unlock(&dev_priv->sb_lock);
1442
1443         /*
1444          * Need to wait > 100ns between dclkp clock enable bit and PLL enable.
1445          */
1446         udelay(1);
1447
1448         /* Enable PLL */
1449         I915_WRITE(DPLL(pipe), pipe_config->dpll_hw_state.dpll);
1450
1451         /* Check PLL is locked */
1452         if (intel_wait_for_register(dev_priv,
1453                                     DPLL(pipe), DPLL_LOCK_VLV, DPLL_LOCK_VLV,
1454                                     1))
1455                 DRM_ERROR("PLL %d failed to lock\n", pipe);
1456 }
1457
1458 static void chv_enable_pll(struct intel_crtc *crtc,
1459                            const struct intel_crtc_state *pipe_config)
1460 {
1461         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1462         enum pipe pipe = crtc->pipe;
1463
1464         assert_pipe_disabled(dev_priv, pipe);
1465
1466         /* PLL is protected by panel, make sure we can write it */
1467         assert_panel_unlocked(dev_priv, pipe);
1468
1469         if (pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE)
1470                 _chv_enable_pll(crtc, pipe_config);
1471
1472         if (pipe != PIPE_A) {
1473                 /*
1474                  * WaPixelRepeatModeFixForC0:chv
1475                  *
1476                  * DPLLCMD is AWOL. Use chicken bits to propagate
1477                  * the value from DPLLBMD to either pipe B or C.
1478                  */
1479                 I915_WRITE(CBR4_VLV, CBR_DPLLBMD_PIPE(pipe));
1480                 I915_WRITE(DPLL_MD(PIPE_B), pipe_config->dpll_hw_state.dpll_md);
1481                 I915_WRITE(CBR4_VLV, 0);
1482                 dev_priv->chv_dpll_md[pipe] = pipe_config->dpll_hw_state.dpll_md;
1483
1484                 /*
1485                  * DPLLB VGA mode also seems to cause problems.
1486                  * We should always have it disabled.
1487                  */
1488                 WARN_ON((I915_READ(DPLL(PIPE_B)) & DPLL_VGA_MODE_DIS) == 0);
1489         } else {
1490                 I915_WRITE(DPLL_MD(pipe), pipe_config->dpll_hw_state.dpll_md);
1491                 POSTING_READ(DPLL_MD(pipe));
1492         }
1493 }
1494
1495 static int intel_num_dvo_pipes(struct drm_i915_private *dev_priv)
1496 {
1497         struct intel_crtc *crtc;
1498         int count = 0;
1499
1500         for_each_intel_crtc(&dev_priv->drm, crtc) {
1501                 count += crtc->base.state->active &&
1502                         intel_crtc_has_type(crtc->config, INTEL_OUTPUT_DVO);
1503         }
1504
1505         return count;
1506 }
1507
1508 static void i9xx_enable_pll(struct intel_crtc *crtc,
1509                             const struct intel_crtc_state *crtc_state)
1510 {
1511         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1512         i915_reg_t reg = DPLL(crtc->pipe);
1513         u32 dpll = crtc_state->dpll_hw_state.dpll;
1514         int i;
1515
1516         assert_pipe_disabled(dev_priv, crtc->pipe);
1517
1518         /* PLL is protected by panel, make sure we can write it */
1519         if (IS_MOBILE(dev_priv) && !IS_I830(dev_priv))
1520                 assert_panel_unlocked(dev_priv, crtc->pipe);
1521
1522         /* Enable DVO 2x clock on both PLLs if necessary */
1523         if (IS_I830(dev_priv) && intel_num_dvo_pipes(dev_priv) > 0) {
1524                 /*
1525                  * It appears to be important that we don't enable this
1526                  * for the current pipe before otherwise configuring the
1527                  * PLL. No idea how this should be handled if multiple
1528                  * DVO outputs are enabled simultaneosly.
1529                  */
1530                 dpll |= DPLL_DVO_2X_MODE;
1531                 I915_WRITE(DPLL(!crtc->pipe),
1532                            I915_READ(DPLL(!crtc->pipe)) | DPLL_DVO_2X_MODE);
1533         }
1534
1535         /*
1536          * Apparently we need to have VGA mode enabled prior to changing
1537          * the P1/P2 dividers. Otherwise the DPLL will keep using the old
1538          * dividers, even though the register value does change.
1539          */
1540         I915_WRITE(reg, 0);
1541
1542         I915_WRITE(reg, dpll);
1543
1544         /* Wait for the clocks to stabilize. */
1545         POSTING_READ(reg);
1546         udelay(150);
1547
1548         if (INTEL_GEN(dev_priv) >= 4) {
1549                 I915_WRITE(DPLL_MD(crtc->pipe),
1550                            crtc_state->dpll_hw_state.dpll_md);
1551         } else {
1552                 /* The pixel multiplier can only be updated once the
1553                  * DPLL is enabled and the clocks are stable.
1554                  *
1555                  * So write it again.
1556                  */
1557                 I915_WRITE(reg, dpll);
1558         }
1559
1560         /* We do this three times for luck */
1561         for (i = 0; i < 3; i++) {
1562                 I915_WRITE(reg, dpll);
1563                 POSTING_READ(reg);
1564                 udelay(150); /* wait for warmup */
1565         }
1566 }
1567
1568 static void i9xx_disable_pll(struct intel_crtc *crtc)
1569 {
1570         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1571         enum pipe pipe = crtc->pipe;
1572
1573         /* Disable DVO 2x clock on both PLLs if necessary */
1574         if (IS_I830(dev_priv) &&
1575             intel_crtc_has_type(crtc->config, INTEL_OUTPUT_DVO) &&
1576             !intel_num_dvo_pipes(dev_priv)) {
1577                 I915_WRITE(DPLL(PIPE_B),
1578                            I915_READ(DPLL(PIPE_B)) & ~DPLL_DVO_2X_MODE);
1579                 I915_WRITE(DPLL(PIPE_A),
1580                            I915_READ(DPLL(PIPE_A)) & ~DPLL_DVO_2X_MODE);
1581         }
1582
1583         /* Don't disable pipe or pipe PLLs if needed */
1584         if (IS_I830(dev_priv))
1585                 return;
1586
1587         /* Make sure the pipe isn't still relying on us */
1588         assert_pipe_disabled(dev_priv, pipe);
1589
1590         I915_WRITE(DPLL(pipe), DPLL_VGA_MODE_DIS);
1591         POSTING_READ(DPLL(pipe));
1592 }
1593
1594 static void vlv_disable_pll(struct drm_i915_private *dev_priv, enum pipe pipe)
1595 {
1596         u32 val;
1597
1598         /* Make sure the pipe isn't still relying on us */
1599         assert_pipe_disabled(dev_priv, pipe);
1600
1601         val = DPLL_INTEGRATED_REF_CLK_VLV |
1602                 DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
1603         if (pipe != PIPE_A)
1604                 val |= DPLL_INTEGRATED_CRI_CLK_VLV;
1605
1606         I915_WRITE(DPLL(pipe), val);
1607         POSTING_READ(DPLL(pipe));
1608 }
1609
1610 static void chv_disable_pll(struct drm_i915_private *dev_priv, enum pipe pipe)
1611 {
1612         enum dpio_channel port = vlv_pipe_to_channel(pipe);
1613         u32 val;
1614
1615         /* Make sure the pipe isn't still relying on us */
1616         assert_pipe_disabled(dev_priv, pipe);
1617
1618         val = DPLL_SSC_REF_CLK_CHV |
1619                 DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
1620         if (pipe != PIPE_A)
1621                 val |= DPLL_INTEGRATED_CRI_CLK_VLV;
1622
1623         I915_WRITE(DPLL(pipe), val);
1624         POSTING_READ(DPLL(pipe));
1625
1626         mutex_lock(&dev_priv->sb_lock);
1627
1628         /* Disable 10bit clock to display controller */
1629         val = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW14(port));
1630         val &= ~DPIO_DCLKP_EN;
1631         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW14(port), val);
1632
1633         mutex_unlock(&dev_priv->sb_lock);
1634 }
1635
1636 void vlv_wait_port_ready(struct drm_i915_private *dev_priv,
1637                          struct intel_digital_port *dport,
1638                          unsigned int expected_mask)
1639 {
1640         u32 port_mask;
1641         i915_reg_t dpll_reg;
1642
1643         switch (dport->base.port) {
1644         case PORT_B:
1645                 port_mask = DPLL_PORTB_READY_MASK;
1646                 dpll_reg = DPLL(0);
1647                 break;
1648         case PORT_C:
1649                 port_mask = DPLL_PORTC_READY_MASK;
1650                 dpll_reg = DPLL(0);
1651                 expected_mask <<= 4;
1652                 break;
1653         case PORT_D:
1654                 port_mask = DPLL_PORTD_READY_MASK;
1655                 dpll_reg = DPIO_PHY_STATUS;
1656                 break;
1657         default:
1658                 BUG();
1659         }
1660
1661         if (intel_wait_for_register(dev_priv,
1662                                     dpll_reg, port_mask, expected_mask,
1663                                     1000))
1664                 WARN(1, "timed out waiting for port %c ready: got 0x%x, expected 0x%x\n",
1665                      port_name(dport->base.port),
1666                      I915_READ(dpll_reg) & port_mask, expected_mask);
1667 }
1668
1669 static void ironlake_enable_pch_transcoder(struct drm_i915_private *dev_priv,
1670                                            enum pipe pipe)
1671 {
1672         struct intel_crtc *intel_crtc = intel_get_crtc_for_pipe(dev_priv,
1673                                                                 pipe);
1674         i915_reg_t reg;
1675         uint32_t val, pipeconf_val;
1676
1677         /* Make sure PCH DPLL is enabled */
1678         assert_shared_dpll_enabled(dev_priv, intel_crtc->config->shared_dpll);
1679
1680         /* FDI must be feeding us bits for PCH ports */
1681         assert_fdi_tx_enabled(dev_priv, pipe);
1682         assert_fdi_rx_enabled(dev_priv, pipe);
1683
1684         if (HAS_PCH_CPT(dev_priv)) {
1685                 /* Workaround: Set the timing override bit before enabling the
1686                  * pch transcoder. */
1687                 reg = TRANS_CHICKEN2(pipe);
1688                 val = I915_READ(reg);
1689                 val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
1690                 I915_WRITE(reg, val);
1691         }
1692
1693         reg = PCH_TRANSCONF(pipe);
1694         val = I915_READ(reg);
1695         pipeconf_val = I915_READ(PIPECONF(pipe));
1696
1697         if (HAS_PCH_IBX(dev_priv)) {
1698                 /*
1699                  * Make the BPC in transcoder be consistent with
1700                  * that in pipeconf reg. For HDMI we must use 8bpc
1701                  * here for both 8bpc and 12bpc.
1702                  */
1703                 val &= ~PIPECONF_BPC_MASK;
1704                 if (intel_crtc_has_type(intel_crtc->config, INTEL_OUTPUT_HDMI))
1705                         val |= PIPECONF_8BPC;
1706                 else
1707                         val |= pipeconf_val & PIPECONF_BPC_MASK;
1708         }
1709
1710         val &= ~TRANS_INTERLACE_MASK;
1711         if ((pipeconf_val & PIPECONF_INTERLACE_MASK) == PIPECONF_INTERLACED_ILK)
1712                 if (HAS_PCH_IBX(dev_priv) &&
1713                     intel_crtc_has_type(intel_crtc->config, INTEL_OUTPUT_SDVO))
1714                         val |= TRANS_LEGACY_INTERLACED_ILK;
1715                 else
1716                         val |= TRANS_INTERLACED;
1717         else
1718                 val |= TRANS_PROGRESSIVE;
1719
1720         I915_WRITE(reg, val | TRANS_ENABLE);
1721         if (intel_wait_for_register(dev_priv,
1722                                     reg, TRANS_STATE_ENABLE, TRANS_STATE_ENABLE,
1723                                     100))
1724                 DRM_ERROR("failed to enable transcoder %c\n", pipe_name(pipe));
1725 }
1726
1727 static void lpt_enable_pch_transcoder(struct drm_i915_private *dev_priv,
1728                                       enum transcoder cpu_transcoder)
1729 {
1730         u32 val, pipeconf_val;
1731
1732         /* FDI must be feeding us bits for PCH ports */
1733         assert_fdi_tx_enabled(dev_priv, (enum pipe) cpu_transcoder);
1734         assert_fdi_rx_enabled(dev_priv, PIPE_A);
1735
1736         /* Workaround: set timing override bit. */
1737         val = I915_READ(TRANS_CHICKEN2(PIPE_A));
1738         val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
1739         I915_WRITE(TRANS_CHICKEN2(PIPE_A), val);
1740
1741         val = TRANS_ENABLE;
1742         pipeconf_val = I915_READ(PIPECONF(cpu_transcoder));
1743
1744         if ((pipeconf_val & PIPECONF_INTERLACE_MASK_HSW) ==
1745             PIPECONF_INTERLACED_ILK)
1746                 val |= TRANS_INTERLACED;
1747         else
1748                 val |= TRANS_PROGRESSIVE;
1749
1750         I915_WRITE(LPT_TRANSCONF, val);
1751         if (intel_wait_for_register(dev_priv,
1752                                     LPT_TRANSCONF,
1753                                     TRANS_STATE_ENABLE,
1754                                     TRANS_STATE_ENABLE,
1755                                     100))
1756                 DRM_ERROR("Failed to enable PCH transcoder\n");
1757 }
1758
1759 static void ironlake_disable_pch_transcoder(struct drm_i915_private *dev_priv,
1760                                             enum pipe pipe)
1761 {
1762         i915_reg_t reg;
1763         uint32_t val;
1764
1765         /* FDI relies on the transcoder */
1766         assert_fdi_tx_disabled(dev_priv, pipe);
1767         assert_fdi_rx_disabled(dev_priv, pipe);
1768
1769         /* Ports must be off as well */
1770         assert_pch_ports_disabled(dev_priv, pipe);
1771
1772         reg = PCH_TRANSCONF(pipe);
1773         val = I915_READ(reg);
1774         val &= ~TRANS_ENABLE;
1775         I915_WRITE(reg, val);
1776         /* wait for PCH transcoder off, transcoder state */
1777         if (intel_wait_for_register(dev_priv,
1778                                     reg, TRANS_STATE_ENABLE, 0,
1779                                     50))
1780                 DRM_ERROR("failed to disable transcoder %c\n", pipe_name(pipe));
1781
1782         if (HAS_PCH_CPT(dev_priv)) {
1783                 /* Workaround: Clear the timing override chicken bit again. */
1784                 reg = TRANS_CHICKEN2(pipe);
1785                 val = I915_READ(reg);
1786                 val &= ~TRANS_CHICKEN2_TIMING_OVERRIDE;
1787                 I915_WRITE(reg, val);
1788         }
1789 }
1790
1791 void lpt_disable_pch_transcoder(struct drm_i915_private *dev_priv)
1792 {
1793         u32 val;
1794
1795         val = I915_READ(LPT_TRANSCONF);
1796         val &= ~TRANS_ENABLE;
1797         I915_WRITE(LPT_TRANSCONF, val);
1798         /* wait for PCH transcoder off, transcoder state */
1799         if (intel_wait_for_register(dev_priv,
1800                                     LPT_TRANSCONF, TRANS_STATE_ENABLE, 0,
1801                                     50))
1802                 DRM_ERROR("Failed to disable PCH transcoder\n");
1803
1804         /* Workaround: clear timing override bit. */
1805         val = I915_READ(TRANS_CHICKEN2(PIPE_A));
1806         val &= ~TRANS_CHICKEN2_TIMING_OVERRIDE;
1807         I915_WRITE(TRANS_CHICKEN2(PIPE_A), val);
1808 }
1809
1810 enum pipe intel_crtc_pch_transcoder(struct intel_crtc *crtc)
1811 {
1812         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1813
1814         if (HAS_PCH_LPT(dev_priv))
1815                 return PIPE_A;
1816         else
1817                 return crtc->pipe;
1818 }
1819
1820 static void intel_enable_pipe(const struct intel_crtc_state *new_crtc_state)
1821 {
1822         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->base.crtc);
1823         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1824         enum transcoder cpu_transcoder = new_crtc_state->cpu_transcoder;
1825         enum pipe pipe = crtc->pipe;
1826         i915_reg_t reg;
1827         u32 val;
1828
1829         DRM_DEBUG_KMS("enabling pipe %c\n", pipe_name(pipe));
1830
1831         assert_planes_disabled(crtc);
1832
1833         /*
1834          * A pipe without a PLL won't actually be able to drive bits from
1835          * a plane.  On ILK+ the pipe PLLs are integrated, so we don't
1836          * need the check.
1837          */
1838         if (HAS_GMCH_DISPLAY(dev_priv)) {
1839                 if (intel_crtc_has_type(new_crtc_state, INTEL_OUTPUT_DSI))
1840                         assert_dsi_pll_enabled(dev_priv);
1841                 else
1842                         assert_pll_enabled(dev_priv, pipe);
1843         } else {
1844                 if (new_crtc_state->has_pch_encoder) {
1845                         /* if driving the PCH, we need FDI enabled */
1846                         assert_fdi_rx_pll_enabled(dev_priv,
1847                                                   intel_crtc_pch_transcoder(crtc));
1848                         assert_fdi_tx_pll_enabled(dev_priv,
1849                                                   (enum pipe) cpu_transcoder);
1850                 }
1851                 /* FIXME: assert CPU port conditions for SNB+ */
1852         }
1853
1854         reg = PIPECONF(cpu_transcoder);
1855         val = I915_READ(reg);
1856         if (val & PIPECONF_ENABLE) {
1857                 /* we keep both pipes enabled on 830 */
1858                 WARN_ON(!IS_I830(dev_priv));
1859                 return;
1860         }
1861
1862         I915_WRITE(reg, val | PIPECONF_ENABLE);
1863         POSTING_READ(reg);
1864
1865         /*
1866          * Until the pipe starts PIPEDSL reads will return a stale value,
1867          * which causes an apparent vblank timestamp jump when PIPEDSL
1868          * resets to its proper value. That also messes up the frame count
1869          * when it's derived from the timestamps. So let's wait for the
1870          * pipe to start properly before we call drm_crtc_vblank_on()
1871          */
1872         if (dev_priv->drm.max_vblank_count == 0)
1873                 intel_wait_for_pipe_scanline_moving(crtc);
1874 }
1875
1876 static void intel_disable_pipe(const struct intel_crtc_state *old_crtc_state)
1877 {
1878         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc);
1879         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1880         enum transcoder cpu_transcoder = old_crtc_state->cpu_transcoder;
1881         enum pipe pipe = crtc->pipe;
1882         i915_reg_t reg;
1883         u32 val;
1884
1885         DRM_DEBUG_KMS("disabling pipe %c\n", pipe_name(pipe));
1886
1887         /*
1888          * Make sure planes won't keep trying to pump pixels to us,
1889          * or we might hang the display.
1890          */
1891         assert_planes_disabled(crtc);
1892
1893         reg = PIPECONF(cpu_transcoder);
1894         val = I915_READ(reg);
1895         if ((val & PIPECONF_ENABLE) == 0)
1896                 return;
1897
1898         /*
1899          * Double wide has implications for planes
1900          * so best keep it disabled when not needed.
1901          */
1902         if (old_crtc_state->double_wide)
1903                 val &= ~PIPECONF_DOUBLE_WIDE;
1904
1905         /* Don't disable pipe or pipe PLLs if needed */
1906         if (!IS_I830(dev_priv))
1907                 val &= ~PIPECONF_ENABLE;
1908
1909         I915_WRITE(reg, val);
1910         if ((val & PIPECONF_ENABLE) == 0)
1911                 intel_wait_for_pipe_off(old_crtc_state);
1912 }
1913
1914 static unsigned int intel_tile_size(const struct drm_i915_private *dev_priv)
1915 {
1916         return IS_GEN2(dev_priv) ? 2048 : 4096;
1917 }
1918
1919 static unsigned int
1920 intel_tile_width_bytes(const struct drm_framebuffer *fb, int color_plane)
1921 {
1922         struct drm_i915_private *dev_priv = to_i915(fb->dev);
1923         unsigned int cpp = fb->format->cpp[color_plane];
1924
1925         switch (fb->modifier) {
1926         case DRM_FORMAT_MOD_LINEAR:
1927                 return cpp;
1928         case I915_FORMAT_MOD_X_TILED:
1929                 if (IS_GEN2(dev_priv))
1930                         return 128;
1931                 else
1932                         return 512;
1933         case I915_FORMAT_MOD_Y_TILED_CCS:
1934                 if (color_plane == 1)
1935                         return 128;
1936                 /* fall through */
1937         case I915_FORMAT_MOD_Y_TILED:
1938                 if (IS_GEN2(dev_priv) || HAS_128_BYTE_Y_TILING(dev_priv))
1939                         return 128;
1940                 else
1941                         return 512;
1942         case I915_FORMAT_MOD_Yf_TILED_CCS:
1943                 if (color_plane == 1)
1944                         return 128;
1945                 /* fall through */
1946         case I915_FORMAT_MOD_Yf_TILED:
1947                 switch (cpp) {
1948                 case 1:
1949                         return 64;
1950                 case 2:
1951                 case 4:
1952                         return 128;
1953                 case 8:
1954                 case 16:
1955                         return 256;
1956                 default:
1957                         MISSING_CASE(cpp);
1958                         return cpp;
1959                 }
1960                 break;
1961         default:
1962                 MISSING_CASE(fb->modifier);
1963                 return cpp;
1964         }
1965 }
1966
1967 static unsigned int
1968 intel_tile_height(const struct drm_framebuffer *fb, int color_plane)
1969 {
1970         if (fb->modifier == DRM_FORMAT_MOD_LINEAR)
1971                 return 1;
1972         else
1973                 return intel_tile_size(to_i915(fb->dev)) /
1974                         intel_tile_width_bytes(fb, color_plane);
1975 }
1976
1977 /* Return the tile dimensions in pixel units */
1978 static void intel_tile_dims(const struct drm_framebuffer *fb, int color_plane,
1979                             unsigned int *tile_width,
1980                             unsigned int *tile_height)
1981 {
1982         unsigned int tile_width_bytes = intel_tile_width_bytes(fb, color_plane);
1983         unsigned int cpp = fb->format->cpp[color_plane];
1984
1985         *tile_width = tile_width_bytes / cpp;
1986         *tile_height = intel_tile_size(to_i915(fb->dev)) / tile_width_bytes;
1987 }
1988
1989 unsigned int
1990 intel_fb_align_height(const struct drm_framebuffer *fb,
1991                       int color_plane, unsigned int height)
1992 {
1993         unsigned int tile_height = intel_tile_height(fb, color_plane);
1994
1995         return ALIGN(height, tile_height);
1996 }
1997
1998 unsigned int intel_rotation_info_size(const struct intel_rotation_info *rot_info)
1999 {
2000         unsigned int size = 0;
2001         int i;
2002
2003         for (i = 0 ; i < ARRAY_SIZE(rot_info->plane); i++)
2004                 size += rot_info->plane[i].width * rot_info->plane[i].height;
2005
2006         return size;
2007 }
2008
2009 static void
2010 intel_fill_fb_ggtt_view(struct i915_ggtt_view *view,
2011                         const struct drm_framebuffer *fb,
2012                         unsigned int rotation)
2013 {
2014         view->type = I915_GGTT_VIEW_NORMAL;
2015         if (drm_rotation_90_or_270(rotation)) {
2016                 view->type = I915_GGTT_VIEW_ROTATED;
2017                 view->rotated = to_intel_framebuffer(fb)->rot_info;
2018         }
2019 }
2020
2021 static unsigned int intel_cursor_alignment(const struct drm_i915_private *dev_priv)
2022 {
2023         if (IS_I830(dev_priv))
2024                 return 16 * 1024;
2025         else if (IS_I85X(dev_priv))
2026                 return 256;
2027         else if (IS_I845G(dev_priv) || IS_I865G(dev_priv))
2028                 return 32;
2029         else
2030                 return 4 * 1024;
2031 }
2032
2033 static unsigned int intel_linear_alignment(const struct drm_i915_private *dev_priv)
2034 {
2035         if (INTEL_GEN(dev_priv) >= 9)
2036                 return 256 * 1024;
2037         else if (IS_I965G(dev_priv) || IS_I965GM(dev_priv) ||
2038                  IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
2039                 return 128 * 1024;
2040         else if (INTEL_GEN(dev_priv) >= 4)
2041                 return 4 * 1024;
2042         else
2043                 return 0;
2044 }
2045
2046 static unsigned int intel_surf_alignment(const struct drm_framebuffer *fb,
2047                                          int color_plane)
2048 {
2049         struct drm_i915_private *dev_priv = to_i915(fb->dev);
2050
2051         /* AUX_DIST needs only 4K alignment */
2052         if (color_plane == 1)
2053                 return 4096;
2054
2055         switch (fb->modifier) {
2056         case DRM_FORMAT_MOD_LINEAR:
2057                 return intel_linear_alignment(dev_priv);
2058         case I915_FORMAT_MOD_X_TILED:
2059                 if (INTEL_GEN(dev_priv) >= 9)
2060                         return 256 * 1024;
2061                 return 0;
2062         case I915_FORMAT_MOD_Y_TILED_CCS:
2063         case I915_FORMAT_MOD_Yf_TILED_CCS:
2064         case I915_FORMAT_MOD_Y_TILED:
2065         case I915_FORMAT_MOD_Yf_TILED:
2066                 return 1 * 1024 * 1024;
2067         default:
2068                 MISSING_CASE(fb->modifier);
2069                 return 0;
2070         }
2071 }
2072
2073 static bool intel_plane_uses_fence(const struct intel_plane_state *plane_state)
2074 {
2075         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
2076         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
2077
2078         return INTEL_GEN(dev_priv) < 4 || plane->has_fbc;
2079 }
2080
2081 struct i915_vma *
2082 intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb,
2083                            const struct i915_ggtt_view *view,
2084                            bool uses_fence,
2085                            unsigned long *out_flags)
2086 {
2087         struct drm_device *dev = fb->dev;
2088         struct drm_i915_private *dev_priv = to_i915(dev);
2089         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
2090         struct i915_vma *vma;
2091         unsigned int pinctl;
2092         u32 alignment;
2093
2094         WARN_ON(!mutex_is_locked(&dev->struct_mutex));
2095
2096         alignment = intel_surf_alignment(fb, 0);
2097
2098         /* Note that the w/a also requires 64 PTE of padding following the
2099          * bo. We currently fill all unused PTE with the shadow page and so
2100          * we should always have valid PTE following the scanout preventing
2101          * the VT-d warning.
2102          */
2103         if (intel_scanout_needs_vtd_wa(dev_priv) && alignment < 256 * 1024)
2104                 alignment = 256 * 1024;
2105
2106         /*
2107          * Global gtt pte registers are special registers which actually forward
2108          * writes to a chunk of system memory. Which means that there is no risk
2109          * that the register values disappear as soon as we call
2110          * intel_runtime_pm_put(), so it is correct to wrap only the
2111          * pin/unpin/fence and not more.
2112          */
2113         intel_runtime_pm_get(dev_priv);
2114
2115         atomic_inc(&dev_priv->gpu_error.pending_fb_pin);
2116
2117         pinctl = 0;
2118
2119         /* Valleyview is definitely limited to scanning out the first
2120          * 512MiB. Lets presume this behaviour was inherited from the
2121          * g4x display engine and that all earlier gen are similarly
2122          * limited. Testing suggests that it is a little more
2123          * complicated than this. For example, Cherryview appears quite
2124          * happy to scanout from anywhere within its global aperture.
2125          */
2126         if (HAS_GMCH_DISPLAY(dev_priv))
2127                 pinctl |= PIN_MAPPABLE;
2128
2129         vma = i915_gem_object_pin_to_display_plane(obj,
2130                                                    alignment, view, pinctl);
2131         if (IS_ERR(vma))
2132                 goto err;
2133
2134         if (uses_fence && i915_vma_is_map_and_fenceable(vma)) {
2135                 int ret;
2136
2137                 /* Install a fence for tiled scan-out. Pre-i965 always needs a
2138                  * fence, whereas 965+ only requires a fence if using
2139                  * framebuffer compression.  For simplicity, we always, when
2140                  * possible, install a fence as the cost is not that onerous.
2141                  *
2142                  * If we fail to fence the tiled scanout, then either the
2143                  * modeset will reject the change (which is highly unlikely as
2144                  * the affected systems, all but one, do not have unmappable
2145                  * space) or we will not be able to enable full powersaving
2146                  * techniques (also likely not to apply due to various limits
2147                  * FBC and the like impose on the size of the buffer, which
2148                  * presumably we violated anyway with this unmappable buffer).
2149                  * Anyway, it is presumably better to stumble onwards with
2150                  * something and try to run the system in a "less than optimal"
2151                  * mode that matches the user configuration.
2152                  */
2153                 ret = i915_vma_pin_fence(vma);
2154                 if (ret != 0 && INTEL_GEN(dev_priv) < 4) {
2155                         i915_gem_object_unpin_from_display_plane(vma);
2156                         vma = ERR_PTR(ret);
2157                         goto err;
2158                 }
2159
2160                 if (ret == 0 && vma->fence)
2161                         *out_flags |= PLANE_HAS_FENCE;
2162         }
2163
2164         i915_vma_get(vma);
2165 err:
2166         atomic_dec(&dev_priv->gpu_error.pending_fb_pin);
2167
2168         intel_runtime_pm_put(dev_priv);
2169         return vma;
2170 }
2171
2172 void intel_unpin_fb_vma(struct i915_vma *vma, unsigned long flags)
2173 {
2174         lockdep_assert_held(&vma->vm->i915->drm.struct_mutex);
2175
2176         if (flags & PLANE_HAS_FENCE)
2177                 i915_vma_unpin_fence(vma);
2178         i915_gem_object_unpin_from_display_plane(vma);
2179         i915_vma_put(vma);
2180 }
2181
2182 static int intel_fb_pitch(const struct drm_framebuffer *fb, int color_plane,
2183                           unsigned int rotation)
2184 {
2185         if (drm_rotation_90_or_270(rotation))
2186                 return to_intel_framebuffer(fb)->rotated[color_plane].pitch;
2187         else
2188                 return fb->pitches[color_plane];
2189 }
2190
2191 /*
2192  * Convert the x/y offsets into a linear offset.
2193  * Only valid with 0/180 degree rotation, which is fine since linear
2194  * offset is only used with linear buffers on pre-hsw and tiled buffers
2195  * with gen2/3, and 90/270 degree rotations isn't supported on any of them.
2196  */
2197 u32 intel_fb_xy_to_linear(int x, int y,
2198                           const struct intel_plane_state *state,
2199                           int color_plane)
2200 {
2201         const struct drm_framebuffer *fb = state->base.fb;
2202         unsigned int cpp = fb->format->cpp[color_plane];
2203         unsigned int pitch = state->color_plane[color_plane].stride;
2204
2205         return y * pitch + x * cpp;
2206 }
2207
2208 /*
2209  * Add the x/y offsets derived from fb->offsets[] to the user
2210  * specified plane src x/y offsets. The resulting x/y offsets
2211  * specify the start of scanout from the beginning of the gtt mapping.
2212  */
2213 void intel_add_fb_offsets(int *x, int *y,
2214                           const struct intel_plane_state *state,
2215                           int color_plane)
2216
2217 {
2218         const struct intel_framebuffer *intel_fb = to_intel_framebuffer(state->base.fb);
2219         unsigned int rotation = state->base.rotation;
2220
2221         if (drm_rotation_90_or_270(rotation)) {
2222                 *x += intel_fb->rotated[color_plane].x;
2223                 *y += intel_fb->rotated[color_plane].y;
2224         } else {
2225                 *x += intel_fb->normal[color_plane].x;
2226                 *y += intel_fb->normal[color_plane].y;
2227         }
2228 }
2229
2230 static u32 intel_adjust_tile_offset(int *x, int *y,
2231                                     unsigned int tile_width,
2232                                     unsigned int tile_height,
2233                                     unsigned int tile_size,
2234                                     unsigned int pitch_tiles,
2235                                     u32 old_offset,
2236                                     u32 new_offset)
2237 {
2238         unsigned int pitch_pixels = pitch_tiles * tile_width;
2239         unsigned int tiles;
2240
2241         WARN_ON(old_offset & (tile_size - 1));
2242         WARN_ON(new_offset & (tile_size - 1));
2243         WARN_ON(new_offset > old_offset);
2244
2245         tiles = (old_offset - new_offset) / tile_size;
2246
2247         *y += tiles / pitch_tiles * tile_height;
2248         *x += tiles % pitch_tiles * tile_width;
2249
2250         /* minimize x in case it got needlessly big */
2251         *y += *x / pitch_pixels * tile_height;
2252         *x %= pitch_pixels;
2253
2254         return new_offset;
2255 }
2256
2257 static u32 intel_adjust_aligned_offset(int *x, int *y,
2258                                        const struct drm_framebuffer *fb,
2259                                        int color_plane,
2260                                        unsigned int rotation,
2261                                        unsigned int pitch,
2262                                        u32 old_offset, u32 new_offset)
2263 {
2264         struct drm_i915_private *dev_priv = to_i915(fb->dev);
2265         unsigned int cpp = fb->format->cpp[color_plane];
2266
2267         WARN_ON(new_offset > old_offset);
2268
2269         if (fb->modifier != DRM_FORMAT_MOD_LINEAR) {
2270                 unsigned int tile_size, tile_width, tile_height;
2271                 unsigned int pitch_tiles;
2272
2273                 tile_size = intel_tile_size(dev_priv);
2274                 intel_tile_dims(fb, color_plane, &tile_width, &tile_height);
2275
2276                 if (drm_rotation_90_or_270(rotation)) {
2277                         pitch_tiles = pitch / tile_height;
2278                         swap(tile_width, tile_height);
2279                 } else {
2280                         pitch_tiles = pitch / (tile_width * cpp);
2281                 }
2282
2283                 intel_adjust_tile_offset(x, y, tile_width, tile_height,
2284                                          tile_size, pitch_tiles,
2285                                          old_offset, new_offset);
2286         } else {
2287                 old_offset += *y * pitch + *x * cpp;
2288
2289                 *y = (old_offset - new_offset) / pitch;
2290                 *x = ((old_offset - new_offset) - *y * pitch) / cpp;
2291         }
2292
2293         return new_offset;
2294 }
2295
2296 /*
2297  * Adjust the tile offset by moving the difference into
2298  * the x/y offsets.
2299  */
2300 static u32 intel_plane_adjust_aligned_offset(int *x, int *y,
2301                                              const struct intel_plane_state *state,
2302                                              int color_plane,
2303                                              u32 old_offset, u32 new_offset)
2304 {
2305         return intel_adjust_aligned_offset(x, y, state->base.fb, color_plane,
2306                                            state->base.rotation,
2307                                            state->color_plane[color_plane].stride,
2308                                            old_offset, new_offset);
2309 }
2310
2311 /*
2312  * Computes the aligned offset to the base tile and adjusts
2313  * x, y. bytes per pixel is assumed to be a power-of-two.
2314  *
2315  * In the 90/270 rotated case, x and y are assumed
2316  * to be already rotated to match the rotated GTT view, and
2317  * pitch is the tile_height aligned framebuffer height.
2318  *
2319  * This function is used when computing the derived information
2320  * under intel_framebuffer, so using any of that information
2321  * here is not allowed. Anything under drm_framebuffer can be
2322  * used. This is why the user has to pass in the pitch since it
2323  * is specified in the rotated orientation.
2324  */
2325 static u32 intel_compute_aligned_offset(struct drm_i915_private *dev_priv,
2326                                         int *x, int *y,
2327                                         const struct drm_framebuffer *fb,
2328                                         int color_plane,
2329                                         unsigned int pitch,
2330                                         unsigned int rotation,
2331                                         u32 alignment)
2332 {
2333         uint64_t fb_modifier = fb->modifier;
2334         unsigned int cpp = fb->format->cpp[color_plane];
2335         u32 offset, offset_aligned;
2336
2337         if (alignment)
2338                 alignment--;
2339
2340         if (fb_modifier != DRM_FORMAT_MOD_LINEAR) {
2341                 unsigned int tile_size, tile_width, tile_height;
2342                 unsigned int tile_rows, tiles, pitch_tiles;
2343
2344                 tile_size = intel_tile_size(dev_priv);
2345                 intel_tile_dims(fb, color_plane, &tile_width, &tile_height);
2346
2347                 if (drm_rotation_90_or_270(rotation)) {
2348                         pitch_tiles = pitch / tile_height;
2349                         swap(tile_width, tile_height);
2350                 } else {
2351                         pitch_tiles = pitch / (tile_width * cpp);
2352                 }
2353
2354                 tile_rows = *y / tile_height;
2355                 *y %= tile_height;
2356
2357                 tiles = *x / tile_width;
2358                 *x %= tile_width;
2359
2360                 offset = (tile_rows * pitch_tiles + tiles) * tile_size;
2361                 offset_aligned = offset & ~alignment;
2362
2363                 intel_adjust_tile_offset(x, y, tile_width, tile_height,
2364                                          tile_size, pitch_tiles,
2365                                          offset, offset_aligned);
2366         } else {
2367                 offset = *y * pitch + *x * cpp;
2368                 offset_aligned = offset & ~alignment;
2369
2370                 *y = (offset & alignment) / pitch;
2371                 *x = ((offset & alignment) - *y * pitch) / cpp;
2372         }
2373
2374         return offset_aligned;
2375 }
2376
2377 static u32 intel_plane_compute_aligned_offset(int *x, int *y,
2378                                               const struct intel_plane_state *state,
2379                                               int color_plane)
2380 {
2381         struct intel_plane *intel_plane = to_intel_plane(state->base.plane);
2382         struct drm_i915_private *dev_priv = to_i915(intel_plane->base.dev);
2383         const struct drm_framebuffer *fb = state->base.fb;
2384         unsigned int rotation = state->base.rotation;
2385         int pitch = state->color_plane[color_plane].stride;
2386         u32 alignment;
2387
2388         if (intel_plane->id == PLANE_CURSOR)
2389                 alignment = intel_cursor_alignment(dev_priv);
2390         else
2391                 alignment = intel_surf_alignment(fb, color_plane);
2392
2393         return intel_compute_aligned_offset(dev_priv, x, y, fb, color_plane,
2394                                             pitch, rotation, alignment);
2395 }
2396
2397 /* Convert the fb->offset[] into x/y offsets */
2398 static int intel_fb_offset_to_xy(int *x, int *y,
2399                                  const struct drm_framebuffer *fb,
2400                                  int color_plane)
2401 {
2402         struct drm_i915_private *dev_priv = to_i915(fb->dev);
2403
2404         if (fb->modifier != DRM_FORMAT_MOD_LINEAR &&
2405             fb->offsets[color_plane] % intel_tile_size(dev_priv))
2406                 return -EINVAL;
2407
2408         *x = 0;
2409         *y = 0;
2410
2411         intel_adjust_aligned_offset(x, y,
2412                                     fb, color_plane, DRM_MODE_ROTATE_0,
2413                                     fb->pitches[color_plane],
2414                                     fb->offsets[color_plane], 0);
2415
2416         return 0;
2417 }
2418
2419 static unsigned int intel_fb_modifier_to_tiling(uint64_t fb_modifier)
2420 {
2421         switch (fb_modifier) {
2422         case I915_FORMAT_MOD_X_TILED:
2423                 return I915_TILING_X;
2424         case I915_FORMAT_MOD_Y_TILED:
2425         case I915_FORMAT_MOD_Y_TILED_CCS:
2426                 return I915_TILING_Y;
2427         default:
2428                 return I915_TILING_NONE;
2429         }
2430 }
2431
2432 /*
2433  * From the Sky Lake PRM:
2434  * "The Color Control Surface (CCS) contains the compression status of
2435  *  the cache-line pairs. The compression state of the cache-line pair
2436  *  is specified by 2 bits in the CCS. Each CCS cache-line represents
2437  *  an area on the main surface of 16 x16 sets of 128 byte Y-tiled
2438  *  cache-line-pairs. CCS is always Y tiled."
2439  *
2440  * Since cache line pairs refers to horizontally adjacent cache lines,
2441  * each cache line in the CCS corresponds to an area of 32x16 cache
2442  * lines on the main surface. Since each pixel is 4 bytes, this gives
2443  * us a ratio of one byte in the CCS for each 8x16 pixels in the
2444  * main surface.
2445  */
2446 static const struct drm_format_info ccs_formats[] = {
2447         { .format = DRM_FORMAT_XRGB8888, .depth = 24, .num_planes = 2, .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, },
2448         { .format = DRM_FORMAT_XBGR8888, .depth = 24, .num_planes = 2, .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, },
2449         { .format = DRM_FORMAT_ARGB8888, .depth = 32, .num_planes = 2, .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, },
2450         { .format = DRM_FORMAT_ABGR8888, .depth = 32, .num_planes = 2, .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, },
2451 };
2452
2453 static const struct drm_format_info *
2454 lookup_format_info(const struct drm_format_info formats[],
2455                    int num_formats, u32 format)
2456 {
2457         int i;
2458
2459         for (i = 0; i < num_formats; i++) {
2460                 if (formats[i].format == format)
2461                         return &formats[i];
2462         }
2463
2464         return NULL;
2465 }
2466
2467 static const struct drm_format_info *
2468 intel_get_format_info(const struct drm_mode_fb_cmd2 *cmd)
2469 {
2470         switch (cmd->modifier[0]) {
2471         case I915_FORMAT_MOD_Y_TILED_CCS:
2472         case I915_FORMAT_MOD_Yf_TILED_CCS:
2473                 return lookup_format_info(ccs_formats,
2474                                           ARRAY_SIZE(ccs_formats),
2475                                           cmd->pixel_format);
2476         default:
2477                 return NULL;
2478         }
2479 }
2480
2481 bool is_ccs_modifier(u64 modifier)
2482 {
2483         return modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
2484                modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
2485 }
2486
2487 static int
2488 intel_fill_fb_info(struct drm_i915_private *dev_priv,
2489                    struct drm_framebuffer *fb)
2490 {
2491         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
2492         struct intel_rotation_info *rot_info = &intel_fb->rot_info;
2493         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
2494         u32 gtt_offset_rotated = 0;
2495         unsigned int max_size = 0;
2496         int i, num_planes = fb->format->num_planes;
2497         unsigned int tile_size = intel_tile_size(dev_priv);
2498
2499         for (i = 0; i < num_planes; i++) {
2500                 unsigned int width, height;
2501                 unsigned int cpp, size;
2502                 u32 offset;
2503                 int x, y;
2504                 int ret;
2505
2506                 cpp = fb->format->cpp[i];
2507                 width = drm_framebuffer_plane_width(fb->width, fb, i);
2508                 height = drm_framebuffer_plane_height(fb->height, fb, i);
2509
2510                 ret = intel_fb_offset_to_xy(&x, &y, fb, i);
2511                 if (ret) {
2512                         DRM_DEBUG_KMS("bad fb plane %d offset: 0x%x\n",
2513                                       i, fb->offsets[i]);
2514                         return ret;
2515                 }
2516
2517                 if (is_ccs_modifier(fb->modifier) && i == 1) {
2518                         int hsub = fb->format->hsub;
2519                         int vsub = fb->format->vsub;
2520                         int tile_width, tile_height;
2521                         int main_x, main_y;
2522                         int ccs_x, ccs_y;
2523
2524                         intel_tile_dims(fb, i, &tile_width, &tile_height);
2525                         tile_width *= hsub;
2526                         tile_height *= vsub;
2527
2528                         ccs_x = (x * hsub) % tile_width;
2529                         ccs_y = (y * vsub) % tile_height;
2530                         main_x = intel_fb->normal[0].x % tile_width;
2531                         main_y = intel_fb->normal[0].y % tile_height;
2532
2533                         /*
2534                          * CCS doesn't have its own x/y offset register, so the intra CCS tile
2535                          * x/y offsets must match between CCS and the main surface.
2536                          */
2537                         if (main_x != ccs_x || main_y != ccs_y) {
2538                                 DRM_DEBUG_KMS("Bad CCS x/y (main %d,%d ccs %d,%d) full (main %d,%d ccs %d,%d)\n",
2539                                               main_x, main_y,
2540                                               ccs_x, ccs_y,
2541                                               intel_fb->normal[0].x,
2542                                               intel_fb->normal[0].y,
2543                                               x, y);
2544                                 return -EINVAL;
2545                         }
2546                 }
2547
2548                 /*
2549                  * The fence (if used) is aligned to the start of the object
2550                  * so having the framebuffer wrap around across the edge of the
2551                  * fenced region doesn't really work. We have no API to configure
2552                  * the fence start offset within the object (nor could we probably
2553                  * on gen2/3). So it's just easier if we just require that the
2554                  * fb layout agrees with the fence layout. We already check that the
2555                  * fb stride matches the fence stride elsewhere.
2556                  */
2557                 if (i == 0 && i915_gem_object_is_tiled(obj) &&
2558                     (x + width) * cpp > fb->pitches[i]) {
2559                         DRM_DEBUG_KMS("bad fb plane %d offset: 0x%x\n",
2560                                       i, fb->offsets[i]);
2561                         return -EINVAL;
2562                 }
2563
2564                 /*
2565                  * First pixel of the framebuffer from
2566                  * the start of the normal gtt mapping.
2567                  */
2568                 intel_fb->normal[i].x = x;
2569                 intel_fb->normal[i].y = y;
2570
2571                 offset = intel_compute_aligned_offset(dev_priv, &x, &y, fb, i,
2572                                                       fb->pitches[i],
2573                                                       DRM_MODE_ROTATE_0,
2574                                                       tile_size);
2575                 offset /= tile_size;
2576
2577                 if (fb->modifier != DRM_FORMAT_MOD_LINEAR) {
2578                         unsigned int tile_width, tile_height;
2579                         unsigned int pitch_tiles;
2580                         struct drm_rect r;
2581
2582                         intel_tile_dims(fb, i, &tile_width, &tile_height);
2583
2584                         rot_info->plane[i].offset = offset;
2585                         rot_info->plane[i].stride = DIV_ROUND_UP(fb->pitches[i], tile_width * cpp);
2586                         rot_info->plane[i].width = DIV_ROUND_UP(x + width, tile_width);
2587                         rot_info->plane[i].height = DIV_ROUND_UP(y + height, tile_height);
2588
2589                         intel_fb->rotated[i].pitch =
2590                                 rot_info->plane[i].height * tile_height;
2591
2592                         /* how many tiles does this plane need */
2593                         size = rot_info->plane[i].stride * rot_info->plane[i].height;
2594                         /*
2595                          * If the plane isn't horizontally tile aligned,
2596                          * we need one more tile.
2597                          */
2598                         if (x != 0)
2599                                 size++;
2600
2601                         /* rotate the x/y offsets to match the GTT view */
2602                         r.x1 = x;
2603                         r.y1 = y;
2604                         r.x2 = x + width;
2605                         r.y2 = y + height;
2606                         drm_rect_rotate(&r,
2607                                         rot_info->plane[i].width * tile_width,
2608                                         rot_info->plane[i].height * tile_height,
2609                                         DRM_MODE_ROTATE_270);
2610                         x = r.x1;
2611                         y = r.y1;
2612
2613                         /* rotate the tile dimensions to match the GTT view */
2614                         pitch_tiles = intel_fb->rotated[i].pitch / tile_height;
2615                         swap(tile_width, tile_height);
2616
2617                         /*
2618                          * We only keep the x/y offsets, so push all of the
2619                          * gtt offset into the x/y offsets.
2620                          */
2621                         intel_adjust_tile_offset(&x, &y,
2622                                                  tile_width, tile_height,
2623                                                  tile_size, pitch_tiles,
2624                                                  gtt_offset_rotated * tile_size, 0);
2625
2626                         gtt_offset_rotated += rot_info->plane[i].width * rot_info->plane[i].height;
2627
2628                         /*
2629                          * First pixel of the framebuffer from
2630                          * the start of the rotated gtt mapping.
2631                          */
2632                         intel_fb->rotated[i].x = x;
2633                         intel_fb->rotated[i].y = y;
2634                 } else {
2635                         size = DIV_ROUND_UP((y + height) * fb->pitches[i] +
2636                                             x * cpp, tile_size);
2637                 }
2638
2639                 /* how many tiles in total needed in the bo */
2640                 max_size = max(max_size, offset + size);
2641         }
2642
2643         if (mul_u32_u32(max_size, tile_size) > obj->base.size) {
2644                 DRM_DEBUG_KMS("fb too big for bo (need %llu bytes, have %zu bytes)\n",
2645                               mul_u32_u32(max_size, tile_size), obj->base.size);
2646                 return -EINVAL;
2647         }
2648
2649         return 0;
2650 }
2651
2652 static int i9xx_format_to_fourcc(int format)
2653 {
2654         switch (format) {
2655         case DISPPLANE_8BPP:
2656                 return DRM_FORMAT_C8;
2657         case DISPPLANE_BGRX555:
2658                 return DRM_FORMAT_XRGB1555;
2659         case DISPPLANE_BGRX565:
2660                 return DRM_FORMAT_RGB565;
2661         default:
2662         case DISPPLANE_BGRX888:
2663                 return DRM_FORMAT_XRGB8888;
2664         case DISPPLANE_RGBX888:
2665                 return DRM_FORMAT_XBGR8888;
2666         case DISPPLANE_BGRX101010:
2667                 return DRM_FORMAT_XRGB2101010;
2668         case DISPPLANE_RGBX101010:
2669                 return DRM_FORMAT_XBGR2101010;
2670         }
2671 }
2672
2673 int skl_format_to_fourcc(int format, bool rgb_order, bool alpha)
2674 {
2675         switch (format) {
2676         case PLANE_CTL_FORMAT_RGB_565:
2677                 return DRM_FORMAT_RGB565;
2678         case PLANE_CTL_FORMAT_NV12:
2679                 return DRM_FORMAT_NV12;
2680         default:
2681         case PLANE_CTL_FORMAT_XRGB_8888:
2682                 if (rgb_order) {
2683                         if (alpha)
2684                                 return DRM_FORMAT_ABGR8888;
2685                         else
2686                                 return DRM_FORMAT_XBGR8888;
2687                 } else {
2688                         if (alpha)
2689                                 return DRM_FORMAT_ARGB8888;
2690                         else
2691                                 return DRM_FORMAT_XRGB8888;
2692                 }
2693         case PLANE_CTL_FORMAT_XRGB_2101010:
2694                 if (rgb_order)
2695                         return DRM_FORMAT_XBGR2101010;
2696                 else
2697                         return DRM_FORMAT_XRGB2101010;
2698         }
2699 }
2700
2701 static bool
2702 intel_alloc_initial_plane_obj(struct intel_crtc *crtc,
2703                               struct intel_initial_plane_config *plane_config)
2704 {
2705         struct drm_device *dev = crtc->base.dev;
2706         struct drm_i915_private *dev_priv = to_i915(dev);
2707         struct drm_i915_gem_object *obj = NULL;
2708         struct drm_mode_fb_cmd2 mode_cmd = { 0 };
2709         struct drm_framebuffer *fb = &plane_config->fb->base;
2710         u32 base_aligned = round_down(plane_config->base, PAGE_SIZE);
2711         u32 size_aligned = round_up(plane_config->base + plane_config->size,
2712                                     PAGE_SIZE);
2713
2714         size_aligned -= base_aligned;
2715
2716         if (plane_config->size == 0)
2717                 return false;
2718
2719         /* If the FB is too big, just don't use it since fbdev is not very
2720          * important and we should probably use that space with FBC or other
2721          * features. */
2722         if (size_aligned * 2 > dev_priv->stolen_usable_size)
2723                 return false;
2724
2725         switch (fb->modifier) {
2726         case DRM_FORMAT_MOD_LINEAR:
2727         case I915_FORMAT_MOD_X_TILED:
2728         case I915_FORMAT_MOD_Y_TILED:
2729                 break;
2730         default:
2731                 DRM_DEBUG_DRIVER("Unsupported modifier for initial FB: 0x%llx\n",
2732                                  fb->modifier);
2733                 return false;
2734         }
2735
2736         mutex_lock(&dev->struct_mutex);
2737         obj = i915_gem_object_create_stolen_for_preallocated(dev_priv,
2738                                                              base_aligned,
2739                                                              base_aligned,
2740                                                              size_aligned);
2741         mutex_unlock(&dev->struct_mutex);
2742         if (!obj)
2743                 return false;
2744
2745         switch (plane_config->tiling) {
2746         case I915_TILING_NONE:
2747                 break;
2748         case I915_TILING_X:
2749         case I915_TILING_Y:
2750                 obj->tiling_and_stride = fb->pitches[0] | plane_config->tiling;
2751                 break;
2752         default:
2753                 MISSING_CASE(plane_config->tiling);
2754                 return false;
2755         }
2756
2757         mode_cmd.pixel_format = fb->format->format;
2758         mode_cmd.width = fb->width;
2759         mode_cmd.height = fb->height;
2760         mode_cmd.pitches[0] = fb->pitches[0];
2761         mode_cmd.modifier[0] = fb->modifier;
2762         mode_cmd.flags = DRM_MODE_FB_MODIFIERS;
2763
2764         if (intel_framebuffer_init(to_intel_framebuffer(fb), obj, &mode_cmd)) {
2765                 DRM_DEBUG_KMS("intel fb init failed\n");
2766                 goto out_unref_obj;
2767         }
2768
2769
2770         DRM_DEBUG_KMS("initial plane fb obj %p\n", obj);
2771         return true;
2772
2773 out_unref_obj:
2774         i915_gem_object_put(obj);
2775         return false;
2776 }
2777
2778 static void
2779 intel_set_plane_visible(struct intel_crtc_state *crtc_state,
2780                         struct intel_plane_state *plane_state,
2781                         bool visible)
2782 {
2783         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
2784
2785         plane_state->base.visible = visible;
2786
2787         if (visible)
2788                 crtc_state->base.plane_mask |= drm_plane_mask(&plane->base);
2789         else
2790                 crtc_state->base.plane_mask &= ~drm_plane_mask(&plane->base);
2791
2792         DRM_DEBUG_KMS("%s active planes 0x%x\n",
2793                       crtc_state->base.crtc->name,
2794                       crtc_state->active_planes);
2795 }
2796
2797 static void fixup_active_planes(struct intel_crtc_state *crtc_state)
2798 {
2799         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
2800         struct drm_plane *plane;
2801
2802         /*
2803          * Active_planes aliases if multiple "primary" or cursor planes
2804          * have been used on the same (or wrong) pipe. plane_mask uses
2805          * unique ids, hence we can use that to reconstruct active_planes.
2806          */
2807         crtc_state->active_planes = 0;
2808
2809         drm_for_each_plane_mask(plane, &dev_priv->drm,
2810                                 crtc_state->base.plane_mask)
2811                 crtc_state->active_planes |= BIT(to_intel_plane(plane)->id);
2812 }
2813
2814 static void intel_plane_disable_noatomic(struct intel_crtc *crtc,
2815                                          struct intel_plane *plane)
2816 {
2817         struct intel_crtc_state *crtc_state =
2818                 to_intel_crtc_state(crtc->base.state);
2819         struct intel_plane_state *plane_state =
2820                 to_intel_plane_state(plane->base.state);
2821
2822         intel_set_plane_visible(crtc_state, plane_state, false);
2823         fixup_active_planes(crtc_state);
2824
2825         if (plane->id == PLANE_PRIMARY)
2826                 intel_pre_disable_primary_noatomic(&crtc->base);
2827
2828         trace_intel_disable_plane(&plane->base, crtc);
2829         plane->disable_plane(plane, crtc);
2830 }
2831
2832 static void
2833 intel_find_initial_plane_obj(struct intel_crtc *intel_crtc,
2834                              struct intel_initial_plane_config *plane_config)
2835 {
2836         struct drm_device *dev = intel_crtc->base.dev;
2837         struct drm_i915_private *dev_priv = to_i915(dev);
2838         struct drm_crtc *c;
2839         struct drm_i915_gem_object *obj;
2840         struct drm_plane *primary = intel_crtc->base.primary;
2841         struct drm_plane_state *plane_state = primary->state;
2842         struct intel_plane *intel_plane = to_intel_plane(primary);
2843         struct intel_plane_state *intel_state =
2844                 to_intel_plane_state(plane_state);
2845         struct drm_framebuffer *fb;
2846
2847         if (!plane_config->fb)
2848                 return;
2849
2850         if (intel_alloc_initial_plane_obj(intel_crtc, plane_config)) {
2851                 fb = &plane_config->fb->base;
2852                 goto valid_fb;
2853         }
2854
2855         kfree(plane_config->fb);
2856
2857         /*
2858          * Failed to alloc the obj, check to see if we should share
2859          * an fb with another CRTC instead
2860          */
2861         for_each_crtc(dev, c) {
2862                 struct intel_plane_state *state;
2863
2864                 if (c == &intel_crtc->base)
2865                         continue;
2866
2867                 if (!to_intel_crtc(c)->active)
2868                         continue;
2869
2870                 state = to_intel_plane_state(c->primary->state);
2871                 if (!state->vma)
2872                         continue;
2873
2874                 if (intel_plane_ggtt_offset(state) == plane_config->base) {
2875                         fb = state->base.fb;
2876                         drm_framebuffer_get(fb);
2877                         goto valid_fb;
2878                 }
2879         }
2880
2881         /*
2882          * We've failed to reconstruct the BIOS FB.  Current display state
2883          * indicates that the primary plane is visible, but has a NULL FB,
2884          * which will lead to problems later if we don't fix it up.  The
2885          * simplest solution is to just disable the primary plane now and
2886          * pretend the BIOS never had it enabled.
2887          */
2888         intel_plane_disable_noatomic(intel_crtc, intel_plane);
2889
2890         return;
2891
2892 valid_fb:
2893         intel_fill_fb_ggtt_view(&intel_state->view, fb,
2894                                 intel_state->base.rotation);
2895         intel_state->color_plane[0].stride =
2896                 intel_fb_pitch(fb, 0, intel_state->base.rotation);
2897
2898         mutex_lock(&dev->struct_mutex);
2899         intel_state->vma =
2900                 intel_pin_and_fence_fb_obj(fb,
2901                                            &intel_state->view,
2902                                            intel_plane_uses_fence(intel_state),
2903                                            &intel_state->flags);
2904         mutex_unlock(&dev->struct_mutex);
2905         if (IS_ERR(intel_state->vma)) {
2906                 DRM_ERROR("failed to pin boot fb on pipe %d: %li\n",
2907                           intel_crtc->pipe, PTR_ERR(intel_state->vma));
2908
2909                 intel_state->vma = NULL;
2910                 drm_framebuffer_put(fb);
2911                 return;
2912         }
2913
2914         obj = intel_fb_obj(fb);
2915         intel_fb_obj_flush(obj, ORIGIN_DIRTYFB);
2916
2917         plane_state->src_x = 0;
2918         plane_state->src_y = 0;
2919         plane_state->src_w = fb->width << 16;
2920         plane_state->src_h = fb->height << 16;
2921
2922         plane_state->crtc_x = 0;
2923         plane_state->crtc_y = 0;
2924         plane_state->crtc_w = fb->width;
2925         plane_state->crtc_h = fb->height;
2926
2927         intel_state->base.src = drm_plane_state_src(plane_state);
2928         intel_state->base.dst = drm_plane_state_dest(plane_state);
2929
2930         if (i915_gem_object_is_tiled(obj))
2931                 dev_priv->preserve_bios_swizzle = true;
2932
2933         plane_state->fb = fb;
2934         plane_state->crtc = &intel_crtc->base;
2935
2936         atomic_or(to_intel_plane(primary)->frontbuffer_bit,
2937                   &obj->frontbuffer_bits);
2938 }
2939
2940 static int skl_max_plane_width(const struct drm_framebuffer *fb,
2941                                int color_plane,
2942                                unsigned int rotation)
2943 {
2944         int cpp = fb->format->cpp[color_plane];
2945
2946         switch (fb->modifier) {
2947         case DRM_FORMAT_MOD_LINEAR:
2948         case I915_FORMAT_MOD_X_TILED:
2949                 switch (cpp) {
2950                 case 8:
2951                         return 4096;
2952                 case 4:
2953                 case 2:
2954                 case 1:
2955                         return 8192;
2956                 default:
2957                         MISSING_CASE(cpp);
2958                         break;
2959                 }
2960                 break;
2961         case I915_FORMAT_MOD_Y_TILED_CCS:
2962         case I915_FORMAT_MOD_Yf_TILED_CCS:
2963                 /* FIXME AUX plane? */
2964         case I915_FORMAT_MOD_Y_TILED:
2965         case I915_FORMAT_MOD_Yf_TILED:
2966                 switch (cpp) {
2967                 case 8:
2968                         return 2048;
2969                 case 4:
2970                         return 4096;
2971                 case 2:
2972                 case 1:
2973                         return 8192;
2974                 default:
2975                         MISSING_CASE(cpp);
2976                         break;
2977                 }
2978                 break;
2979         default:
2980                 MISSING_CASE(fb->modifier);
2981         }
2982
2983         return 2048;
2984 }
2985
2986 static bool skl_check_main_ccs_coordinates(struct intel_plane_state *plane_state,
2987                                            int main_x, int main_y, u32 main_offset)
2988 {
2989         const struct drm_framebuffer *fb = plane_state->base.fb;
2990         int hsub = fb->format->hsub;
2991         int vsub = fb->format->vsub;
2992         int aux_x = plane_state->color_plane[1].x;
2993         int aux_y = plane_state->color_plane[1].y;
2994         u32 aux_offset = plane_state->color_plane[1].offset;
2995         u32 alignment = intel_surf_alignment(fb, 1);
2996
2997         while (aux_offset >= main_offset && aux_y <= main_y) {
2998                 int x, y;
2999
3000                 if (aux_x == main_x && aux_y == main_y)
3001                         break;
3002
3003                 if (aux_offset == 0)
3004                         break;
3005
3006                 x = aux_x / hsub;
3007                 y = aux_y / vsub;
3008                 aux_offset = intel_plane_adjust_aligned_offset(&x, &y, plane_state, 1,
3009                                                                aux_offset, aux_offset - alignment);
3010                 aux_x = x * hsub + aux_x % hsub;
3011                 aux_y = y * vsub + aux_y % vsub;
3012         }
3013
3014         if (aux_x != main_x || aux_y != main_y)
3015                 return false;
3016
3017         plane_state->color_plane[1].offset = aux_offset;
3018         plane_state->color_plane[1].x = aux_x;
3019         plane_state->color_plane[1].y = aux_y;
3020
3021         return true;
3022 }
3023
3024 static int skl_check_main_surface(struct intel_plane_state *plane_state)
3025 {
3026         const struct drm_framebuffer *fb = plane_state->base.fb;
3027         unsigned int rotation = plane_state->base.rotation;
3028         int x = plane_state->base.src.x1 >> 16;
3029         int y = plane_state->base.src.y1 >> 16;
3030         int w = drm_rect_width(&plane_state->base.src) >> 16;
3031         int h = drm_rect_height(&plane_state->base.src) >> 16;
3032         int max_width = skl_max_plane_width(fb, 0, rotation);
3033         int max_height = 4096;
3034         u32 alignment, offset, aux_offset = plane_state->color_plane[1].offset;
3035
3036         if (w > max_width || h > max_height) {
3037                 DRM_DEBUG_KMS("requested Y/RGB source size %dx%d too big (limit %dx%d)\n",
3038                               w, h, max_width, max_height);
3039                 return -EINVAL;
3040         }
3041
3042         intel_add_fb_offsets(&x, &y, plane_state, 0);
3043         offset = intel_plane_compute_aligned_offset(&x, &y, plane_state, 0);
3044         alignment = intel_surf_alignment(fb, 0);
3045
3046         /*
3047          * AUX surface offset is specified as the distance from the
3048          * main surface offset, and it must be non-negative. Make
3049          * sure that is what we will get.
3050          */
3051         if (offset > aux_offset)
3052                 offset = intel_plane_adjust_aligned_offset(&x, &y, plane_state, 0,
3053                                                            offset, aux_offset & ~(alignment - 1));
3054
3055         /*
3056          * When using an X-tiled surface, the plane blows up
3057          * if the x offset + width exceed the stride.
3058          *
3059          * TODO: linear and Y-tiled seem fine, Yf untested,
3060          */
3061         if (fb->modifier == I915_FORMAT_MOD_X_TILED) {
3062                 int cpp = fb->format->cpp[0];
3063
3064                 while ((x + w) * cpp > plane_state->color_plane[0].stride) {
3065                         if (offset == 0) {
3066                                 DRM_DEBUG_KMS("Unable to find suitable display surface offset due to X-tiling\n");
3067                                 return -EINVAL;
3068                         }
3069
3070                         offset = intel_plane_adjust_aligned_offset(&x, &y, plane_state, 0,
3071                                                                    offset, offset - alignment);
3072                 }
3073         }
3074
3075         /*
3076          * CCS AUX surface doesn't have its own x/y offsets, we must make sure
3077          * they match with the main surface x/y offsets.
3078          */
3079         if (is_ccs_modifier(fb->modifier)) {
3080                 while (!skl_check_main_ccs_coordinates(plane_state, x, y, offset)) {
3081                         if (offset == 0)
3082                                 break;
3083
3084                         offset = intel_plane_adjust_aligned_offset(&x, &y, plane_state, 0,
3085                                                                    offset, offset - alignment);
3086                 }
3087
3088                 if (x != plane_state->color_plane[1].x || y != plane_state->color_plane[1].y) {
3089                         DRM_DEBUG_KMS("Unable to find suitable display surface offset due to CCS\n");
3090                         return -EINVAL;
3091                 }
3092         }
3093
3094         plane_state->color_plane[0].offset = offset;
3095         plane_state->color_plane[0].x = x;
3096         plane_state->color_plane[0].y = y;
3097
3098         return 0;
3099 }
3100
3101 static int
3102 skl_check_nv12_surface(struct intel_plane_state *plane_state)
3103 {
3104         /* Display WA #1106 */
3105         if (plane_state->base.rotation !=
3106             (DRM_MODE_REFLECT_X | DRM_MODE_ROTATE_90) &&
3107             plane_state->base.rotation != DRM_MODE_ROTATE_270)
3108                 return 0;
3109
3110         /*
3111          * src coordinates are rotated here.
3112          * We check height but report it as width
3113          */
3114         if (((drm_rect_height(&plane_state->base.src) >> 16) % 4) != 0) {
3115                 DRM_DEBUG_KMS("src width must be multiple "
3116                               "of 4 for rotated NV12\n");
3117                 return -EINVAL;
3118         }
3119
3120         return 0;
3121 }
3122
3123 static int skl_check_nv12_aux_surface(struct intel_plane_state *plane_state)
3124 {
3125         const struct drm_framebuffer *fb = plane_state->base.fb;
3126         unsigned int rotation = plane_state->base.rotation;
3127         int max_width = skl_max_plane_width(fb, 1, rotation);
3128         int max_height = 4096;
3129         int x = plane_state->base.src.x1 >> 17;
3130         int y = plane_state->base.src.y1 >> 17;
3131         int w = drm_rect_width(&plane_state->base.src) >> 17;
3132         int h = drm_rect_height(&plane_state->base.src) >> 17;
3133         u32 offset;
3134
3135         intel_add_fb_offsets(&x, &y, plane_state, 1);
3136         offset = intel_plane_compute_aligned_offset(&x, &y, plane_state, 1);
3137
3138         /* FIXME not quite sure how/if these apply to the chroma plane */
3139         if (w > max_width || h > max_height) {
3140                 DRM_DEBUG_KMS("CbCr source size %dx%d too big (limit %dx%d)\n",
3141                               w, h, max_width, max_height);
3142                 return -EINVAL;
3143         }
3144
3145         plane_state->color_plane[1].offset = offset;
3146         plane_state->color_plane[1].x = x;
3147         plane_state->color_plane[1].y = y;
3148
3149         return 0;
3150 }
3151
3152 static int skl_check_ccs_aux_surface(struct intel_plane_state *plane_state)
3153 {
3154         const struct drm_framebuffer *fb = plane_state->base.fb;
3155         int src_x = plane_state->base.src.x1 >> 16;
3156         int src_y = plane_state->base.src.y1 >> 16;
3157         int hsub = fb->format->hsub;
3158         int vsub = fb->format->vsub;
3159         int x = src_x / hsub;
3160         int y = src_y / vsub;
3161         u32 offset;
3162
3163         intel_add_fb_offsets(&x, &y, plane_state, 1);
3164         offset = intel_plane_compute_aligned_offset(&x, &y, plane_state, 1);
3165
3166         plane_state->color_plane[1].offset = offset;
3167         plane_state->color_plane[1].x = x * hsub + src_x % hsub;
3168         plane_state->color_plane[1].y = y * vsub + src_y % vsub;
3169
3170         return 0;
3171 }
3172
3173 int skl_check_plane_surface(struct intel_plane_state *plane_state)
3174 {
3175         const struct drm_framebuffer *fb = plane_state->base.fb;
3176         unsigned int rotation = plane_state->base.rotation;
3177         int ret;
3178
3179         intel_fill_fb_ggtt_view(&plane_state->view, fb, rotation);
3180         plane_state->color_plane[0].stride = intel_fb_pitch(fb, 0, rotation);
3181         plane_state->color_plane[1].stride = intel_fb_pitch(fb, 1, rotation);
3182
3183         ret = intel_plane_check_stride(plane_state);
3184         if (ret)
3185                 return ret;
3186
3187         if (!plane_state->base.visible)
3188                 return 0;
3189
3190         /* Rotate src coordinates to match rotated GTT view */
3191         if (drm_rotation_90_or_270(rotation))
3192                 drm_rect_rotate(&plane_state->base.src,
3193                                 fb->width << 16, fb->height << 16,
3194                                 DRM_MODE_ROTATE_270);
3195
3196         /*
3197          * Handle the AUX surface first since
3198          * the main surface setup depends on it.
3199          */
3200         if (fb->format->format == DRM_FORMAT_NV12) {
3201                 ret = skl_check_nv12_surface(plane_state);
3202                 if (ret)
3203                         return ret;
3204                 ret = skl_check_nv12_aux_surface(plane_state);
3205                 if (ret)
3206                         return ret;
3207         } else if (is_ccs_modifier(fb->modifier)) {
3208                 ret = skl_check_ccs_aux_surface(plane_state);
3209                 if (ret)
3210                         return ret;
3211         } else {
3212                 plane_state->color_plane[1].offset = ~0xfff;
3213                 plane_state->color_plane[1].x = 0;
3214                 plane_state->color_plane[1].y = 0;
3215         }
3216
3217         ret = skl_check_main_surface(plane_state);
3218         if (ret)
3219                 return ret;
3220
3221         return 0;
3222 }
3223
3224 unsigned int
3225 i9xx_plane_max_stride(struct intel_plane *plane,
3226                       u32 pixel_format, u64 modifier,
3227                       unsigned int rotation)
3228 {
3229         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
3230
3231         if (!HAS_GMCH_DISPLAY(dev_priv)) {
3232                 return 32*1024;
3233         } else if (INTEL_GEN(dev_priv) >= 4) {
3234                 if (modifier == I915_FORMAT_MOD_X_TILED)
3235                         return 16*1024;
3236                 else
3237                         return 32*1024;
3238         } else if (INTEL_GEN(dev_priv) >= 3) {
3239                 if (modifier == I915_FORMAT_MOD_X_TILED)
3240                         return 8*1024;
3241                 else
3242                         return 16*1024;
3243         } else {
3244                 if (plane->i9xx_plane == PLANE_C)
3245                         return 4*1024;
3246                 else
3247                         return 8*1024;
3248         }
3249 }
3250
3251 static u32 i9xx_plane_ctl(const struct intel_crtc_state *crtc_state,
3252                           const struct intel_plane_state *plane_state)
3253 {
3254         struct drm_i915_private *dev_priv =
3255                 to_i915(plane_state->base.plane->dev);
3256         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
3257         const struct drm_framebuffer *fb = plane_state->base.fb;
3258         unsigned int rotation = plane_state->base.rotation;
3259         u32 dspcntr;
3260
3261         dspcntr = DISPLAY_PLANE_ENABLE | DISPPLANE_GAMMA_ENABLE;
3262
3263         if (IS_G4X(dev_priv) || IS_GEN5(dev_priv) ||
3264             IS_GEN6(dev_priv) || IS_IVYBRIDGE(dev_priv))
3265                 dspcntr |= DISPPLANE_TRICKLE_FEED_DISABLE;
3266
3267         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3268                 dspcntr |= DISPPLANE_PIPE_CSC_ENABLE;
3269
3270         if (INTEL_GEN(dev_priv) < 5)
3271                 dspcntr |= DISPPLANE_SEL_PIPE(crtc->pipe);
3272
3273         switch (fb->format->format) {
3274         case DRM_FORMAT_C8:
3275                 dspcntr |= DISPPLANE_8BPP;
3276                 break;
3277         case DRM_FORMAT_XRGB1555:
3278                 dspcntr |= DISPPLANE_BGRX555;
3279                 break;
3280         case DRM_FORMAT_RGB565:
3281                 dspcntr |= DISPPLANE_BGRX565;
3282                 break;
3283         case DRM_FORMAT_XRGB8888:
3284                 dspcntr |= DISPPLANE_BGRX888;
3285                 break;
3286         case DRM_FORMAT_XBGR8888:
3287                 dspcntr |= DISPPLANE_RGBX888;
3288                 break;
3289         case DRM_FORMAT_XRGB2101010:
3290                 dspcntr |= DISPPLANE_BGRX101010;
3291                 break;
3292         case DRM_FORMAT_XBGR2101010:
3293                 dspcntr |= DISPPLANE_RGBX101010;
3294                 break;
3295         default:
3296                 MISSING_CASE(fb->format->format);
3297                 return 0;
3298         }
3299
3300         if (INTEL_GEN(dev_priv) >= 4 &&
3301             fb->modifier == I915_FORMAT_MOD_X_TILED)
3302                 dspcntr |= DISPPLANE_TILED;
3303
3304         if (rotation & DRM_MODE_ROTATE_180)
3305                 dspcntr |= DISPPLANE_ROTATE_180;
3306
3307         if (rotation & DRM_MODE_REFLECT_X)
3308                 dspcntr |= DISPPLANE_MIRROR;
3309
3310         return dspcntr;
3311 }
3312
3313 int i9xx_check_plane_surface(struct intel_plane_state *plane_state)
3314 {
3315         struct drm_i915_private *dev_priv =
3316                 to_i915(plane_state->base.plane->dev);
3317         const struct drm_framebuffer *fb = plane_state->base.fb;
3318         unsigned int rotation = plane_state->base.rotation;
3319         int src_x = plane_state->base.src.x1 >> 16;
3320         int src_y = plane_state->base.src.y1 >> 16;
3321         u32 offset;
3322         int ret;
3323
3324         intel_fill_fb_ggtt_view(&plane_state->view, fb, rotation);
3325         plane_state->color_plane[0].stride = intel_fb_pitch(fb, 0, rotation);
3326
3327         ret = intel_plane_check_stride(plane_state);
3328         if (ret)
3329                 return ret;
3330
3331         intel_add_fb_offsets(&src_x, &src_y, plane_state, 0);
3332
3333         if (INTEL_GEN(dev_priv) >= 4)
3334                 offset = intel_plane_compute_aligned_offset(&src_x, &src_y,
3335                                                             plane_state, 0);
3336         else
3337                 offset = 0;
3338
3339         /* HSW/BDW do this automagically in hardware */
3340         if (!IS_HASWELL(dev_priv) && !IS_BROADWELL(dev_priv)) {
3341                 int src_w = drm_rect_width(&plane_state->base.src) >> 16;
3342                 int src_h = drm_rect_height(&plane_state->base.src) >> 16;
3343
3344                 if (rotation & DRM_MODE_ROTATE_180) {
3345                         src_x += src_w - 1;
3346                         src_y += src_h - 1;
3347                 } else if (rotation & DRM_MODE_REFLECT_X) {
3348                         src_x += src_w - 1;
3349                 }
3350         }
3351
3352         plane_state->color_plane[0].offset = offset;
3353         plane_state->color_plane[0].x = src_x;
3354         plane_state->color_plane[0].y = src_y;
3355
3356         return 0;
3357 }
3358
3359 static int
3360 i9xx_plane_check(struct intel_crtc_state *crtc_state,
3361                  struct intel_plane_state *plane_state)
3362 {
3363         int ret;
3364
3365         ret = chv_plane_check_rotation(plane_state);
3366         if (ret)
3367                 return ret;
3368
3369         ret = drm_atomic_helper_check_plane_state(&plane_state->base,
3370                                                   &crtc_state->base,
3371                                                   DRM_PLANE_HELPER_NO_SCALING,
3372                                                   DRM_PLANE_HELPER_NO_SCALING,
3373                                                   false, true);
3374         if (ret)
3375                 return ret;
3376
3377         if (!plane_state->base.visible)
3378                 return 0;
3379
3380         ret = intel_plane_check_src_coordinates(plane_state);
3381         if (ret)
3382                 return ret;
3383
3384         ret = i9xx_check_plane_surface(plane_state);
3385         if (ret)
3386                 return ret;
3387
3388         plane_state->ctl = i9xx_plane_ctl(crtc_state, plane_state);
3389
3390         return 0;
3391 }
3392
3393 static void i9xx_update_plane(struct intel_plane *plane,
3394                               const struct intel_crtc_state *crtc_state,
3395                               const struct intel_plane_state *plane_state)
3396 {
3397         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
3398         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
3399         u32 linear_offset;
3400         u32 dspcntr = plane_state->ctl;
3401         i915_reg_t reg = DSPCNTR(i9xx_plane);
3402         int x = plane_state->color_plane[0].x;
3403         int y = plane_state->color_plane[0].y;
3404         unsigned long irqflags;
3405         u32 dspaddr_offset;
3406
3407         linear_offset = intel_fb_xy_to_linear(x, y, plane_state, 0);
3408
3409         if (INTEL_GEN(dev_priv) >= 4)
3410                 dspaddr_offset = plane_state->color_plane[0].offset;
3411         else
3412                 dspaddr_offset = linear_offset;
3413
3414         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
3415
3416         if (INTEL_GEN(dev_priv) < 4) {
3417                 /* pipesrc and dspsize control the size that is scaled from,
3418                  * which should always be the user's requested size.
3419                  */
3420                 I915_WRITE_FW(DSPSIZE(i9xx_plane),
3421                               ((crtc_state->pipe_src_h - 1) << 16) |
3422                               (crtc_state->pipe_src_w - 1));
3423                 I915_WRITE_FW(DSPPOS(i9xx_plane), 0);
3424         } else if (IS_CHERRYVIEW(dev_priv) && i9xx_plane == PLANE_B) {
3425                 I915_WRITE_FW(PRIMSIZE(i9xx_plane),
3426                               ((crtc_state->pipe_src_h - 1) << 16) |
3427                               (crtc_state->pipe_src_w - 1));
3428                 I915_WRITE_FW(PRIMPOS(i9xx_plane), 0);
3429                 I915_WRITE_FW(PRIMCNSTALPHA(i9xx_plane), 0);
3430         }
3431
3432         I915_WRITE_FW(reg, dspcntr);
3433
3434         I915_WRITE_FW(DSPSTRIDE(i9xx_plane), plane_state->color_plane[0].stride);
3435         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
3436                 I915_WRITE_FW(DSPSURF(i9xx_plane),
3437                               intel_plane_ggtt_offset(plane_state) +
3438                               dspaddr_offset);
3439                 I915_WRITE_FW(DSPOFFSET(i9xx_plane), (y << 16) | x);
3440         } else if (INTEL_GEN(dev_priv) >= 4) {
3441                 I915_WRITE_FW(DSPSURF(i9xx_plane),
3442                               intel_plane_ggtt_offset(plane_state) +
3443                               dspaddr_offset);
3444                 I915_WRITE_FW(DSPTILEOFF(i9xx_plane), (y << 16) | x);
3445                 I915_WRITE_FW(DSPLINOFF(i9xx_plane), linear_offset);
3446         } else {
3447                 I915_WRITE_FW(DSPADDR(i9xx_plane),
3448                               intel_plane_ggtt_offset(plane_state) +
3449                               dspaddr_offset);
3450         }
3451         POSTING_READ_FW(reg);
3452
3453         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
3454 }
3455
3456 static void i9xx_disable_plane(struct intel_plane *plane,
3457                                struct intel_crtc *crtc)
3458 {
3459         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
3460         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
3461         unsigned long irqflags;
3462
3463         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
3464
3465         I915_WRITE_FW(DSPCNTR(i9xx_plane), 0);
3466         if (INTEL_GEN(dev_priv) >= 4)
3467                 I915_WRITE_FW(DSPSURF(i9xx_plane), 0);
3468         else
3469                 I915_WRITE_FW(DSPADDR(i9xx_plane), 0);
3470         POSTING_READ_FW(DSPCNTR(i9xx_plane));
3471
3472         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
3473 }
3474
3475 static bool i9xx_plane_get_hw_state(struct intel_plane *plane,
3476                                     enum pipe *pipe)
3477 {
3478         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
3479         enum intel_display_power_domain power_domain;
3480         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
3481         bool ret;
3482         u32 val;
3483
3484         /*
3485          * Not 100% correct for planes that can move between pipes,
3486          * but that's only the case for gen2-4 which don't have any
3487          * display power wells.
3488          */
3489         power_domain = POWER_DOMAIN_PIPE(plane->pipe);
3490         if (!intel_display_power_get_if_enabled(dev_priv, power_domain))
3491                 return false;
3492
3493         val = I915_READ(DSPCNTR(i9xx_plane));
3494
3495         ret = val & DISPLAY_PLANE_ENABLE;
3496
3497         if (INTEL_GEN(dev_priv) >= 5)
3498                 *pipe = plane->pipe;
3499         else
3500                 *pipe = (val & DISPPLANE_SEL_PIPE_MASK) >>
3501                         DISPPLANE_SEL_PIPE_SHIFT;
3502
3503         intel_display_power_put(dev_priv, power_domain);
3504
3505         return ret;
3506 }
3507
3508 static u32
3509 intel_fb_stride_alignment(const struct drm_framebuffer *fb, int color_plane)
3510 {
3511         if (fb->modifier == DRM_FORMAT_MOD_LINEAR)
3512                 return 64;
3513         else
3514                 return intel_tile_width_bytes(fb, color_plane);
3515 }
3516
3517 static void skl_detach_scaler(struct intel_crtc *intel_crtc, int id)
3518 {
3519         struct drm_device *dev = intel_crtc->base.dev;
3520         struct drm_i915_private *dev_priv = to_i915(dev);
3521
3522         I915_WRITE(SKL_PS_CTRL(intel_crtc->pipe, id), 0);
3523         I915_WRITE(SKL_PS_WIN_POS(intel_crtc->pipe, id), 0);
3524         I915_WRITE(SKL_PS_WIN_SZ(intel_crtc->pipe, id), 0);
3525 }
3526
3527 /*
3528  * This function detaches (aka. unbinds) unused scalers in hardware
3529  */
3530 static void skl_detach_scalers(struct intel_crtc *intel_crtc)
3531 {
3532         struct intel_crtc_scaler_state *scaler_state;
3533         int i;
3534
3535         scaler_state = &intel_crtc->config->scaler_state;
3536
3537         /* loop through and disable scalers that aren't in use */
3538         for (i = 0; i < intel_crtc->num_scalers; i++) {
3539                 if (!scaler_state->scalers[i].in_use)
3540                         skl_detach_scaler(intel_crtc, i);
3541         }
3542 }
3543
3544 u32 skl_plane_stride(const struct intel_plane_state *plane_state,
3545                      int color_plane)
3546 {
3547         const struct drm_framebuffer *fb = plane_state->base.fb;
3548         unsigned int rotation = plane_state->base.rotation;
3549         u32 stride = plane_state->color_plane[color_plane].stride;
3550
3551         if (color_plane >= fb->format->num_planes)
3552                 return 0;
3553
3554         /*
3555          * The stride is either expressed as a multiple of 64 bytes chunks for
3556          * linear buffers or in number of tiles for tiled buffers.
3557          */
3558         if (drm_rotation_90_or_270(rotation))
3559                 stride /= intel_tile_height(fb, color_plane);
3560         else
3561                 stride /= intel_fb_stride_alignment(fb, color_plane);
3562
3563         return stride;
3564 }
3565
3566 static u32 skl_plane_ctl_format(uint32_t pixel_format)
3567 {
3568         switch (pixel_format) {
3569         case DRM_FORMAT_C8:
3570                 return PLANE_CTL_FORMAT_INDEXED;
3571         case DRM_FORMAT_RGB565:
3572                 return PLANE_CTL_FORMAT_RGB_565;
3573         case DRM_FORMAT_XBGR8888:
3574         case DRM_FORMAT_ABGR8888:
3575                 return PLANE_CTL_FORMAT_XRGB_8888 | PLANE_CTL_ORDER_RGBX;
3576         case DRM_FORMAT_XRGB8888:
3577         case DRM_FORMAT_ARGB8888:
3578                 return PLANE_CTL_FORMAT_XRGB_8888;
3579         case DRM_FORMAT_XRGB2101010:
3580                 return PLANE_CTL_FORMAT_XRGB_2101010;
3581         case DRM_FORMAT_XBGR2101010:
3582                 return PLANE_CTL_ORDER_RGBX | PLANE_CTL_FORMAT_XRGB_2101010;
3583         case DRM_FORMAT_YUYV:
3584                 return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_YUYV;
3585         case DRM_FORMAT_YVYU:
3586                 return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_YVYU;
3587         case DRM_FORMAT_UYVY:
3588                 return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_UYVY;
3589         case DRM_FORMAT_VYUY:
3590                 return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_VYUY;
3591         case DRM_FORMAT_NV12:
3592                 return PLANE_CTL_FORMAT_NV12;
3593         default:
3594                 MISSING_CASE(pixel_format);
3595         }
3596
3597         return 0;
3598 }
3599
3600 /*
3601  * XXX: For ARBG/ABGR formats we default to expecting scanout buffers
3602  * to be already pre-multiplied. We need to add a knob (or a different
3603  * DRM_FORMAT) for user-space to configure that.
3604  */
3605 static u32 skl_plane_ctl_alpha(uint32_t pixel_format)
3606 {
3607         switch (pixel_format) {
3608         case DRM_FORMAT_ABGR8888:
3609         case DRM_FORMAT_ARGB8888:
3610                 return PLANE_CTL_ALPHA_SW_PREMULTIPLY;
3611         default:
3612                 return PLANE_CTL_ALPHA_DISABLE;
3613         }
3614 }
3615
3616 static u32 glk_plane_color_ctl_alpha(uint32_t pixel_format)
3617 {
3618         switch (pixel_format) {
3619         case DRM_FORMAT_ABGR8888:
3620         case DRM_FORMAT_ARGB8888:
3621                 return PLANE_COLOR_ALPHA_SW_PREMULTIPLY;
3622         default:
3623                 return PLANE_COLOR_ALPHA_DISABLE;
3624         }
3625 }
3626
3627 static u32 skl_plane_ctl_tiling(uint64_t fb_modifier)
3628 {
3629         switch (fb_modifier) {
3630         case DRM_FORMAT_MOD_LINEAR:
3631                 break;
3632         case I915_FORMAT_MOD_X_TILED:
3633                 return PLANE_CTL_TILED_X;
3634         case I915_FORMAT_MOD_Y_TILED:
3635                 return PLANE_CTL_TILED_Y;
3636         case I915_FORMAT_MOD_Y_TILED_CCS:
3637                 return PLANE_CTL_TILED_Y | PLANE_CTL_RENDER_DECOMPRESSION_ENABLE;
3638         case I915_FORMAT_MOD_Yf_TILED:
3639                 return PLANE_CTL_TILED_YF;
3640         case I915_FORMAT_MOD_Yf_TILED_CCS:
3641                 return PLANE_CTL_TILED_YF | PLANE_CTL_RENDER_DECOMPRESSION_ENABLE;
3642         default:
3643                 MISSING_CASE(fb_modifier);
3644         }
3645
3646         return 0;
3647 }
3648
3649 static u32 skl_plane_ctl_rotate(unsigned int rotate)
3650 {
3651         switch (rotate) {
3652         case DRM_MODE_ROTATE_0:
3653                 break;
3654         /*
3655          * DRM_MODE_ROTATE_ is counter clockwise to stay compatible with Xrandr
3656          * while i915 HW rotation is clockwise, thats why this swapping.
3657          */
3658         case DRM_MODE_ROTATE_90:
3659                 return PLANE_CTL_ROTATE_270;
3660         case DRM_MODE_ROTATE_180:
3661                 return PLANE_CTL_ROTATE_180;
3662         case DRM_MODE_ROTATE_270:
3663                 return PLANE_CTL_ROTATE_90;
3664         default:
3665                 MISSING_CASE(rotate);
3666         }
3667
3668         return 0;
3669 }
3670
3671 static u32 cnl_plane_ctl_flip(unsigned int reflect)
3672 {
3673         switch (reflect) {
3674         case 0:
3675                 break;
3676         case DRM_MODE_REFLECT_X:
3677                 return PLANE_CTL_FLIP_HORIZONTAL;
3678         case DRM_MODE_REFLECT_Y:
3679         default:
3680                 MISSING_CASE(reflect);
3681         }
3682
3683         return 0;
3684 }
3685
3686 u32 skl_plane_ctl(const struct intel_crtc_state *crtc_state,
3687                   const struct intel_plane_state *plane_state)
3688 {
3689         struct drm_i915_private *dev_priv =
3690                 to_i915(plane_state->base.plane->dev);
3691         const struct drm_framebuffer *fb = plane_state->base.fb;
3692         unsigned int rotation = plane_state->base.rotation;
3693         const struct drm_intel_sprite_colorkey *key = &plane_state->ckey;
3694         u32 plane_ctl;
3695
3696         plane_ctl = PLANE_CTL_ENABLE;
3697
3698         if (INTEL_GEN(dev_priv) < 10 && !IS_GEMINILAKE(dev_priv)) {
3699                 plane_ctl |= skl_plane_ctl_alpha(fb->format->format);
3700                 plane_ctl |=
3701                         PLANE_CTL_PIPE_GAMMA_ENABLE |
3702                         PLANE_CTL_PIPE_CSC_ENABLE |
3703                         PLANE_CTL_PLANE_GAMMA_DISABLE;
3704
3705                 if (plane_state->base.color_encoding == DRM_COLOR_YCBCR_BT709)
3706                         plane_ctl |= PLANE_CTL_YUV_TO_RGB_CSC_FORMAT_BT709;
3707
3708                 if (plane_state->base.color_range == DRM_COLOR_YCBCR_FULL_RANGE)
3709                         plane_ctl |= PLANE_CTL_YUV_RANGE_CORRECTION_DISABLE;
3710         }
3711
3712         plane_ctl |= skl_plane_ctl_format(fb->format->format);
3713         plane_ctl |= skl_plane_ctl_tiling(fb->modifier);
3714         plane_ctl |= skl_plane_ctl_rotate(rotation & DRM_MODE_ROTATE_MASK);
3715
3716         if (INTEL_GEN(dev_priv) >= 10)
3717                 plane_ctl |= cnl_plane_ctl_flip(rotation &
3718                                                 DRM_MODE_REFLECT_MASK);
3719
3720         if (key->flags & I915_SET_COLORKEY_DESTINATION)
3721                 plane_ctl |= PLANE_CTL_KEY_ENABLE_DESTINATION;
3722         else if (key->flags & I915_SET_COLORKEY_SOURCE)
3723                 plane_ctl |= PLANE_CTL_KEY_ENABLE_SOURCE;
3724
3725         return plane_ctl;
3726 }
3727
3728 u32 glk_plane_color_ctl(const struct intel_crtc_state *crtc_state,
3729                         const struct intel_plane_state *plane_state)
3730 {
3731         struct drm_i915_private *dev_priv =
3732                 to_i915(plane_state->base.plane->dev);
3733         const struct drm_framebuffer *fb = plane_state->base.fb;
3734         u32 plane_color_ctl = 0;
3735
3736         if (INTEL_GEN(dev_priv) < 11) {
3737                 plane_color_ctl |= PLANE_COLOR_PIPE_GAMMA_ENABLE;
3738                 plane_color_ctl |= PLANE_COLOR_PIPE_CSC_ENABLE;
3739         }
3740         plane_color_ctl |= PLANE_COLOR_PLANE_GAMMA_DISABLE;
3741         plane_color_ctl |= glk_plane_color_ctl_alpha(fb->format->format);
3742
3743         if (fb->format->is_yuv) {
3744                 if (plane_state->base.color_encoding == DRM_COLOR_YCBCR_BT709)
3745                         plane_color_ctl |= PLANE_COLOR_CSC_MODE_YUV709_TO_RGB709;
3746                 else
3747                         plane_color_ctl |= PLANE_COLOR_CSC_MODE_YUV601_TO_RGB709;
3748
3749                 if (plane_state->base.color_range == DRM_COLOR_YCBCR_FULL_RANGE)
3750                         plane_color_ctl |= PLANE_COLOR_YUV_RANGE_CORRECTION_DISABLE;
3751         }
3752
3753         return plane_color_ctl;
3754 }
3755
3756 static int
3757 __intel_display_resume(struct drm_device *dev,
3758                        struct drm_atomic_state *state,
3759                        struct drm_modeset_acquire_ctx *ctx)
3760 {
3761         struct drm_crtc_state *crtc_state;
3762         struct drm_crtc *crtc;
3763         int i, ret;
3764
3765         intel_modeset_setup_hw_state(dev, ctx);
3766         i915_redisable_vga(to_i915(dev));
3767
3768         if (!state)
3769                 return 0;
3770
3771         /*
3772          * We've duplicated the state, pointers to the old state are invalid.
3773          *
3774          * Don't attempt to use the old state until we commit the duplicated state.
3775          */
3776         for_each_new_crtc_in_state(state, crtc, crtc_state, i) {
3777                 /*
3778                  * Force recalculation even if we restore
3779                  * current state. With fast modeset this may not result
3780                  * in a modeset when the state is compatible.
3781                  */
3782                 crtc_state->mode_changed = true;
3783         }
3784
3785         /* ignore any reset values/BIOS leftovers in the WM registers */
3786         if (!HAS_GMCH_DISPLAY(to_i915(dev)))
3787                 to_intel_atomic_state(state)->skip_intermediate_wm = true;
3788
3789         ret = drm_atomic_helper_commit_duplicated_state(state, ctx);
3790
3791         WARN_ON(ret == -EDEADLK);
3792         return ret;
3793 }
3794
3795 static bool gpu_reset_clobbers_display(struct drm_i915_private *dev_priv)
3796 {
3797         return intel_has_gpu_reset(dev_priv) &&
3798                 INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv);
3799 }
3800
3801 void intel_prepare_reset(struct drm_i915_private *dev_priv)
3802 {
3803         struct drm_device *dev = &dev_priv->drm;
3804         struct drm_modeset_acquire_ctx *ctx = &dev_priv->reset_ctx;
3805         struct drm_atomic_state *state;
3806         int ret;
3807
3808         /* reset doesn't touch the display */
3809         if (!i915_modparams.force_reset_modeset_test &&
3810             !gpu_reset_clobbers_display(dev_priv))
3811                 return;
3812
3813         /* We have a modeset vs reset deadlock, defensively unbreak it. */
3814         set_bit(I915_RESET_MODESET, &dev_priv->gpu_error.flags);
3815         wake_up_all(&dev_priv->gpu_error.wait_queue);
3816
3817         if (atomic_read(&dev_priv->gpu_error.pending_fb_pin)) {
3818                 DRM_DEBUG_KMS("Modeset potentially stuck, unbreaking through wedging\n");
3819                 i915_gem_set_wedged(dev_priv);
3820         }
3821
3822         /*
3823          * Need mode_config.mutex so that we don't
3824          * trample ongoing ->detect() and whatnot.
3825          */
3826         mutex_lock(&dev->mode_config.mutex);
3827         drm_modeset_acquire_init(ctx, 0);
3828         while (1) {
3829                 ret = drm_modeset_lock_all_ctx(dev, ctx);
3830                 if (ret != -EDEADLK)
3831                         break;
3832
3833                 drm_modeset_backoff(ctx);
3834         }
3835         /*
3836          * Disabling the crtcs gracefully seems nicer. Also the
3837          * g33 docs say we should at least disable all the planes.
3838          */
3839         state = drm_atomic_helper_duplicate_state(dev, ctx);
3840         if (IS_ERR(state)) {
3841                 ret = PTR_ERR(state);
3842                 DRM_ERROR("Duplicating state failed with %i\n", ret);
3843                 return;
3844         }
3845
3846         ret = drm_atomic_helper_disable_all(dev, ctx);
3847         if (ret) {
3848                 DRM_ERROR("Suspending crtc's failed with %i\n", ret);
3849                 drm_atomic_state_put(state);
3850                 return;
3851         }
3852
3853         dev_priv->modeset_restore_state = state;
3854         state->acquire_ctx = ctx;
3855 }
3856
3857 void intel_finish_reset(struct drm_i915_private *dev_priv)
3858 {
3859         struct drm_device *dev = &dev_priv->drm;
3860         struct drm_modeset_acquire_ctx *ctx = &dev_priv->reset_ctx;
3861         struct drm_atomic_state *state;
3862         int ret;
3863
3864         /* reset doesn't touch the display */
3865         if (!test_bit(I915_RESET_MODESET, &dev_priv->gpu_error.flags))
3866                 return;
3867
3868         state = fetch_and_zero(&dev_priv->modeset_restore_state);
3869         if (!state)
3870                 goto unlock;
3871
3872         /* reset doesn't touch the display */
3873         if (!gpu_reset_clobbers_display(dev_priv)) {
3874                 /* for testing only restore the display */
3875                 ret = __intel_display_resume(dev, state, ctx);
3876                 if (ret)
3877                         DRM_ERROR("Restoring old state failed with %i\n", ret);
3878         } else {
3879                 /*
3880                  * The display has been reset as well,
3881                  * so need a full re-initialization.
3882                  */
3883                 intel_runtime_pm_disable_interrupts(dev_priv);
3884                 intel_runtime_pm_enable_interrupts(dev_priv);
3885
3886                 intel_pps_unlock_regs_wa(dev_priv);
3887                 intel_modeset_init_hw(dev);
3888                 intel_init_clock_gating(dev_priv);
3889
3890                 spin_lock_irq(&dev_priv->irq_lock);
3891                 if (dev_priv->display.hpd_irq_setup)
3892                         dev_priv->display.hpd_irq_setup(dev_priv);
3893                 spin_unlock_irq(&dev_priv->irq_lock);
3894
3895                 ret = __intel_display_resume(dev, state, ctx);
3896                 if (ret)
3897                         DRM_ERROR("Restoring old state failed with %i\n", ret);
3898
3899                 intel_hpd_init(dev_priv);
3900         }
3901
3902         drm_atomic_state_put(state);
3903 unlock:
3904         drm_modeset_drop_locks(ctx);
3905         drm_modeset_acquire_fini(ctx);
3906         mutex_unlock(&dev->mode_config.mutex);
3907
3908         clear_bit(I915_RESET_MODESET, &dev_priv->gpu_error.flags);
3909 }
3910
3911 static void intel_update_pipe_config(const struct intel_crtc_state *old_crtc_state,
3912                                      const struct intel_crtc_state *new_crtc_state)
3913 {
3914         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->base.crtc);
3915         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3916
3917         /* drm_atomic_helper_update_legacy_modeset_state might not be called. */
3918         crtc->base.mode = new_crtc_state->base.mode;
3919
3920         /*
3921          * Update pipe size and adjust fitter if needed: the reason for this is
3922          * that in compute_mode_changes we check the native mode (not the pfit
3923          * mode) to see if we can flip rather than do a full mode set. In the
3924          * fastboot case, we'll flip, but if we don't update the pipesrc and
3925          * pfit state, we'll end up with a big fb scanned out into the wrong
3926          * sized surface.
3927          */
3928
3929         I915_WRITE(PIPESRC(crtc->pipe),
3930                    ((new_crtc_state->pipe_src_w - 1) << 16) |
3931                    (new_crtc_state->pipe_src_h - 1));
3932
3933         /* on skylake this is done by detaching scalers */
3934         if (INTEL_GEN(dev_priv) >= 9) {
3935                 skl_detach_scalers(crtc);
3936
3937                 if (new_crtc_state->pch_pfit.enabled)
3938                         skylake_pfit_enable(crtc);
3939         } else if (HAS_PCH_SPLIT(dev_priv)) {
3940                 if (new_crtc_state->pch_pfit.enabled)
3941                         ironlake_pfit_enable(crtc);
3942                 else if (old_crtc_state->pch_pfit.enabled)
3943                         ironlake_pfit_disable(crtc, true);
3944         }
3945 }
3946
3947 static void intel_fdi_normal_train(struct intel_crtc *crtc)
3948 {
3949         struct drm_device *dev = crtc->base.dev;
3950         struct drm_i915_private *dev_priv = to_i915(dev);
3951         int pipe = crtc->pipe;
3952         i915_reg_t reg;
3953         u32 temp;
3954
3955         /* enable normal train */
3956         reg = FDI_TX_CTL(pipe);
3957         temp = I915_READ(reg);
3958         if (IS_IVYBRIDGE(dev_priv)) {
3959                 temp &= ~FDI_LINK_TRAIN_NONE_IVB;
3960                 temp |= FDI_LINK_TRAIN_NONE_IVB | FDI_TX_ENHANCE_FRAME_ENABLE;
3961         } else {
3962                 temp &= ~FDI_LINK_TRAIN_NONE;
3963                 temp |= FDI_LINK_TRAIN_NONE | FDI_TX_ENHANCE_FRAME_ENABLE;
3964         }
3965         I915_WRITE(reg, temp);
3966
3967         reg = FDI_RX_CTL(pipe);
3968         temp = I915_READ(reg);
3969         if (HAS_PCH_CPT(dev_priv)) {
3970                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
3971                 temp |= FDI_LINK_TRAIN_NORMAL_CPT;
3972         } else {
3973                 temp &= ~FDI_LINK_TRAIN_NONE;
3974                 temp |= FDI_LINK_TRAIN_NONE;
3975         }
3976         I915_WRITE(reg, temp | FDI_RX_ENHANCE_FRAME_ENABLE);
3977
3978         /* wait one idle pattern time */
3979         POSTING_READ(reg);
3980         udelay(1000);
3981
3982         /* IVB wants error correction enabled */
3983         if (IS_IVYBRIDGE(dev_priv))
3984                 I915_WRITE(reg, I915_READ(reg) | FDI_FS_ERRC_ENABLE |
3985                            FDI_FE_ERRC_ENABLE);
3986 }
3987
3988 /* The FDI link training functions for ILK/Ibexpeak. */
3989 static void ironlake_fdi_link_train(struct intel_crtc *crtc,
3990                                     const struct intel_crtc_state *crtc_state)
3991 {
3992         struct drm_device *dev = crtc->base.dev;
3993         struct drm_i915_private *dev_priv = to_i915(dev);
3994         int pipe = crtc->pipe;
3995         i915_reg_t reg;
3996         u32 temp, tries;
3997
3998         /* FDI needs bits from pipe first */
3999         assert_pipe_enabled(dev_priv, pipe);
4000
4001         /* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit
4002            for train result */
4003         reg = FDI_RX_IMR(pipe);
4004         temp = I915_READ(reg);
4005         temp &= ~FDI_RX_SYMBOL_LOCK;
4006         temp &= ~FDI_RX_BIT_LOCK;
4007         I915_WRITE(reg, temp);
4008         I915_READ(reg);
4009         udelay(150);
4010
4011         /* enable CPU FDI TX and PCH FDI RX */
4012         reg = FDI_TX_CTL(pipe);
4013         temp = I915_READ(reg);
4014         temp &= ~FDI_DP_PORT_WIDTH_MASK;
4015         temp |= FDI_DP_PORT_WIDTH(crtc_state->fdi_lanes);
4016         temp &= ~FDI_LINK_TRAIN_NONE;
4017         temp |= FDI_LINK_TRAIN_PATTERN_1;
4018         I915_WRITE(reg, temp | FDI_TX_ENABLE);
4019
4020         reg = FDI_RX_CTL(pipe);
4021         temp = I915_READ(reg);
4022         temp &= ~FDI_LINK_TRAIN_NONE;
4023         temp |= FDI_LINK_TRAIN_PATTERN_1;
4024         I915_WRITE(reg, temp | FDI_RX_ENABLE);
4025
4026         POSTING_READ(reg);
4027         udelay(150);
4028
4029         /* Ironlake workaround, enable clock pointer after FDI enable*/
4030         I915_WRITE(FDI_RX_CHICKEN(pipe), FDI_RX_PHASE_SYNC_POINTER_OVR);
4031         I915_WRITE(FDI_RX_CHICKEN(pipe), FDI_RX_PHASE_SYNC_POINTER_OVR |
4032                    FDI_RX_PHASE_SYNC_POINTER_EN);
4033
4034         reg = FDI_RX_IIR(pipe);
4035         for (tries = 0; tries < 5; tries++) {
4036                 temp = I915_READ(reg);
4037                 DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
4038
4039                 if ((temp & FDI_RX_BIT_LOCK)) {
4040                         DRM_DEBUG_KMS("FDI train 1 done.\n");
4041                         I915_WRITE(reg, temp | FDI_RX_BIT_LOCK);
4042                         break;
4043                 }
4044         }
4045         if (tries == 5)
4046                 DRM_ERROR("FDI train 1 fail!\n");
4047
4048         /* Train 2 */
4049         reg = FDI_TX_CTL(pipe);
4050         temp = I915_READ(reg);
4051         temp &= ~FDI_LINK_TRAIN_NONE;
4052         temp |= FDI_LINK_TRAIN_PATTERN_2;
4053         I915_WRITE(reg, temp);
4054
4055         reg = FDI_RX_CTL(pipe);
4056         temp = I915_READ(reg);
4057         temp &= ~FDI_LINK_TRAIN_NONE;
4058         temp |= FDI_LINK_TRAIN_PATTERN_2;
4059         I915_WRITE(reg, temp);
4060
4061         POSTING_READ(reg);
4062         udelay(150);
4063
4064         reg = FDI_RX_IIR(pipe);
4065         for (tries = 0; tries < 5; tries++) {
4066                 temp = I915_READ(reg);
4067                 DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
4068
4069                 if (temp & FDI_RX_SYMBOL_LOCK) {
4070                         I915_WRITE(reg, temp | FDI_RX_SYMBOL_LOCK);
4071                         DRM_DEBUG_KMS("FDI train 2 done.\n");
4072                         break;
4073                 }
4074         }
4075         if (tries == 5)
4076                 DRM_ERROR("FDI train 2 fail!\n");
4077
4078         DRM_DEBUG_KMS("FDI train done\n");
4079
4080 }
4081
4082 static const int snb_b_fdi_train_param[] = {
4083         FDI_LINK_TRAIN_400MV_0DB_SNB_B,
4084         FDI_LINK_TRAIN_400MV_6DB_SNB_B,
4085         FDI_LINK_TRAIN_600MV_3_5DB_SNB_B,
4086         FDI_LINK_TRAIN_800MV_0DB_SNB_B,
4087 };
4088
4089 /* The FDI link training functions for SNB/Cougarpoint. */
4090 static void gen6_fdi_link_train(struct intel_crtc *crtc,
4091                                 const struct intel_crtc_state *crtc_state)
4092 {
4093         struct drm_device *dev = crtc->base.dev;
4094         struct drm_i915_private *dev_priv = to_i915(dev);
4095         int pipe = crtc->pipe;
4096         i915_reg_t reg;
4097         u32 temp, i, retry;
4098
4099         /* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit
4100            for train result */
4101         reg = FDI_RX_IMR(pipe);
4102         temp = I915_READ(reg);
4103         temp &= ~FDI_RX_SYMBOL_LOCK;
4104         temp &= ~FDI_RX_BIT_LOCK;
4105         I915_WRITE(reg, temp);
4106
4107         POSTING_READ(reg);
4108         udelay(150);
4109
4110         /* enable CPU FDI TX and PCH FDI RX */
4111         reg = FDI_TX_CTL(pipe);
4112         temp = I915_READ(reg);
4113         temp &= ~FDI_DP_PORT_WIDTH_MASK;
4114         temp |= FDI_DP_PORT_WIDTH(crtc_state->fdi_lanes);
4115         temp &= ~FDI_LINK_TRAIN_NONE;
4116         temp |= FDI_LINK_TRAIN_PATTERN_1;
4117         temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
4118         /* SNB-B */
4119         temp |= FDI_LINK_TRAIN_400MV_0DB_SNB_B;
4120         I915_WRITE(reg, temp | FDI_TX_ENABLE);
4121
4122         I915_WRITE(FDI_RX_MISC(pipe),
4123                    FDI_RX_TP1_TO_TP2_48 | FDI_RX_FDI_DELAY_90);
4124
4125         reg = FDI_RX_CTL(pipe);
4126         temp = I915_READ(reg);
4127         if (HAS_PCH_CPT(dev_priv)) {
4128                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
4129                 temp |= FDI_LINK_TRAIN_PATTERN_1_CPT;
4130         } else {
4131                 temp &= ~FDI_LINK_TRAIN_NONE;
4132                 temp |= FDI_LINK_TRAIN_PATTERN_1;
4133         }
4134         I915_WRITE(reg, temp | FDI_RX_ENABLE);
4135
4136         POSTING_READ(reg);
4137         udelay(150);
4138
4139         for (i = 0; i < 4; i++) {
4140                 reg = FDI_TX_CTL(pipe);
4141                 temp = I915_READ(reg);
4142                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
4143                 temp |= snb_b_fdi_train_param[i];
4144                 I915_WRITE(reg, temp);
4145
4146                 POSTING_READ(reg);
4147                 udelay(500);
4148
4149                 for (retry = 0; retry < 5; retry++) {
4150                         reg = FDI_RX_IIR(pipe);
4151                         temp = I915_READ(reg);
4152                         DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
4153                         if (temp & FDI_RX_BIT_LOCK) {
4154                                 I915_WRITE(reg, temp | FDI_RX_BIT_LOCK);
4155                                 DRM_DEBUG_KMS("FDI train 1 done.\n");
4156                                 break;
4157                         }
4158                         udelay(50);
4159                 }
4160                 if (retry < 5)
4161                         break;
4162         }
4163         if (i == 4)
4164                 DRM_ERROR("FDI train 1 fail!\n");
4165
4166         /* Train 2 */
4167         reg = FDI_TX_CTL(pipe);
4168         temp = I915_READ(reg);
4169         temp &= ~FDI_LINK_TRAIN_NONE;
4170         temp |= FDI_LINK_TRAIN_PATTERN_2;
4171         if (IS_GEN6(dev_priv)) {
4172                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
4173                 /* SNB-B */
4174                 temp |= FDI_LINK_TRAIN_400MV_0DB_SNB_B;
4175         }
4176         I915_WRITE(reg, temp);
4177
4178         reg = FDI_RX_CTL(pipe);
4179         temp = I915_READ(reg);
4180         if (HAS_PCH_CPT(dev_priv)) {
4181                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
4182                 temp |= FDI_LINK_TRAIN_PATTERN_2_CPT;
4183         } else {
4184                 temp &= ~FDI_LINK_TRAIN_NONE;
4185                 temp |= FDI_LINK_TRAIN_PATTERN_2;
4186         }
4187         I915_WRITE(reg, temp);
4188
4189         POSTING_READ(reg);
4190         udelay(150);
4191
4192         for (i = 0; i < 4; i++) {
4193                 reg = FDI_TX_CTL(pipe);
4194                 temp = I915_READ(reg);
4195                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
4196                 temp |= snb_b_fdi_train_param[i];
4197                 I915_WRITE(reg, temp);
4198
4199                 POSTING_READ(reg);
4200                 udelay(500);
4201
4202                 for (retry = 0; retry < 5; retry++) {
4203                         reg = FDI_RX_IIR(pipe);
4204                         temp = I915_READ(reg);
4205                         DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
4206                         if (temp & FDI_RX_SYMBOL_LOCK) {
4207                                 I915_WRITE(reg, temp | FDI_RX_SYMBOL_LOCK);
4208                                 DRM_DEBUG_KMS("FDI train 2 done.\n");
4209                                 break;
4210                         }
4211                         udelay(50);
4212                 }
4213                 if (retry < 5)
4214                         break;
4215         }
4216         if (i == 4)
4217                 DRM_ERROR("FDI train 2 fail!\n");
4218
4219         DRM_DEBUG_KMS("FDI train done.\n");
4220 }
4221
4222 /* Manual link training for Ivy Bridge A0 parts */
4223 static void ivb_manual_fdi_link_train(struct intel_crtc *crtc,
4224                                       const struct intel_crtc_state *crtc_state)
4225 {
4226         struct drm_device *dev = crtc->base.dev;
4227         struct drm_i915_private *dev_priv = to_i915(dev);
4228         int pipe = crtc->pipe;
4229         i915_reg_t reg;
4230         u32 temp, i, j;
4231
4232         /* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit
4233            for train result */
4234         reg = FDI_RX_IMR(pipe);
4235         temp = I915_READ(reg);
4236         temp &= ~FDI_RX_SYMBOL_LOCK;
4237         temp &= ~FDI_RX_BIT_LOCK;
4238         I915_WRITE(reg, temp);
4239
4240         POSTING_READ(reg);
4241         udelay(150);
4242
4243         DRM_DEBUG_KMS("FDI_RX_IIR before link train 0x%x\n",
4244                       I915_READ(FDI_RX_IIR(pipe)));
4245
4246         /* Try each vswing and preemphasis setting twice before moving on */
4247         for (j = 0; j < ARRAY_SIZE(snb_b_fdi_train_param) * 2; j++) {
4248                 /* disable first in case we need to retry */
4249                 reg = FDI_TX_CTL(pipe);
4250                 temp = I915_READ(reg);
4251                 temp &= ~(FDI_LINK_TRAIN_AUTO | FDI_LINK_TRAIN_NONE_IVB);
4252                 temp &= ~FDI_TX_ENABLE;
4253                 I915_WRITE(reg, temp);
4254
4255                 reg = FDI_RX_CTL(pipe);
4256                 temp = I915_READ(reg);
4257                 temp &= ~FDI_LINK_TRAIN_AUTO;
4258                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
4259                 temp &= ~FDI_RX_ENABLE;
4260                 I915_WRITE(reg, temp);
4261
4262                 /* enable CPU FDI TX and PCH FDI RX */
4263                 reg = FDI_TX_CTL(pipe);
4264                 temp = I915_READ(reg);
4265                 temp &= ~FDI_DP_PORT_WIDTH_MASK;
4266                 temp |= FDI_DP_PORT_WIDTH(crtc_state->fdi_lanes);
4267                 temp |= FDI_LINK_TRAIN_PATTERN_1_IVB;
4268                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
4269                 temp |= snb_b_fdi_train_param[j/2];
4270                 temp |= FDI_COMPOSITE_SYNC;
4271                 I915_WRITE(reg, temp | FDI_TX_ENABLE);
4272
4273                 I915_WRITE(FDI_RX_MISC(pipe),
4274                            FDI_RX_TP1_TO_TP2_48 | FDI_RX_FDI_DELAY_90);
4275
4276                 reg = FDI_RX_CTL(pipe);
4277                 temp = I915_READ(reg);
4278                 temp |= FDI_LINK_TRAIN_PATTERN_1_CPT;
4279                 temp |= FDI_COMPOSITE_SYNC;
4280                 I915_WRITE(reg, temp | FDI_RX_ENABLE);
4281
4282                 POSTING_READ(reg);
4283                 udelay(1); /* should be 0.5us */
4284
4285                 for (i = 0; i < 4; i++) {
4286                         reg = FDI_RX_IIR(pipe);
4287                         temp = I915_READ(reg);
4288                         DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
4289
4290                         if (temp & FDI_RX_BIT_LOCK ||
4291                             (I915_READ(reg) & FDI_RX_BIT_LOCK)) {
4292                                 I915_WRITE(reg, temp | FDI_RX_BIT_LOCK);
4293                                 DRM_DEBUG_KMS("FDI train 1 done, level %i.\n",
4294                                               i);
4295                                 break;
4296                         }
4297                         udelay(1); /* should be 0.5us */
4298                 }
4299                 if (i == 4) {
4300                         DRM_DEBUG_KMS("FDI train 1 fail on vswing %d\n", j / 2);
4301                         continue;
4302                 }
4303
4304                 /* Train 2 */
4305                 reg = FDI_TX_CTL(pipe);
4306                 temp = I915_READ(reg);
4307                 temp &= ~FDI_LINK_TRAIN_NONE_IVB;
4308                 temp |= FDI_LINK_TRAIN_PATTERN_2_IVB;
4309                 I915_WRITE(reg, temp);
4310
4311                 reg = FDI_RX_CTL(pipe);
4312                 temp = I915_READ(reg);
4313                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
4314                 temp |= FDI_LINK_TRAIN_PATTERN_2_CPT;
4315                 I915_WRITE(reg, temp);
4316
4317                 POSTING_READ(reg);
4318                 udelay(2); /* should be 1.5us */
4319
4320                 for (i = 0; i < 4; i++) {
4321                         reg = FDI_RX_IIR(pipe);
4322                         temp = I915_READ(reg);
4323                         DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
4324
4325                         if (temp & FDI_RX_SYMBOL_LOCK ||
4326                             (I915_READ(reg) & FDI_RX_SYMBOL_LOCK)) {
4327                                 I915_WRITE(reg, temp | FDI_RX_SYMBOL_LOCK);
4328                                 DRM_DEBUG_KMS("FDI train 2 done, level %i.\n",
4329                                               i);
4330                                 goto train_done;
4331                         }
4332                         udelay(2); /* should be 1.5us */
4333                 }
4334                 if (i == 4)
4335                         DRM_DEBUG_KMS("FDI train 2 fail on vswing %d\n", j / 2);
4336         }
4337
4338 train_done:
4339         DRM_DEBUG_KMS("FDI train done.\n");
4340 }
4341
4342 static void ironlake_fdi_pll_enable(struct intel_crtc *intel_crtc)
4343 {
4344         struct drm_device *dev = intel_crtc->base.dev;
4345         struct drm_i915_private *dev_priv = to_i915(dev);
4346         int pipe = intel_crtc->pipe;
4347         i915_reg_t reg;
4348         u32 temp;
4349
4350         /* enable PCH FDI RX PLL, wait warmup plus DMI latency */
4351         reg = FDI_RX_CTL(pipe);
4352         temp = I915_READ(reg);
4353         temp &= ~(FDI_DP_PORT_WIDTH_MASK | (0x7 << 16));
4354         temp |= FDI_DP_PORT_WIDTH(intel_crtc->config->fdi_lanes);
4355         temp |= (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) << 11;
4356         I915_WRITE(reg, temp | FDI_RX_PLL_ENABLE);
4357
4358         POSTING_READ(reg);
4359         udelay(200);
4360
4361         /* Switch from Rawclk to PCDclk */
4362         temp = I915_READ(reg);
4363         I915_WRITE(reg, temp | FDI_PCDCLK);
4364
4365         POSTING_READ(reg);
4366         udelay(200);
4367
4368         /* Enable CPU FDI TX PLL, always on for Ironlake */
4369         reg = FDI_TX_CTL(pipe);
4370         temp = I915_READ(reg);
4371         if ((temp & FDI_TX_PLL_ENABLE) == 0) {
4372                 I915_WRITE(reg, temp | FDI_TX_PLL_ENABLE);
4373
4374                 POSTING_READ(reg);
4375                 udelay(100);
4376         }
4377 }
4378
4379 static void ironlake_fdi_pll_disable(struct intel_crtc *intel_crtc)
4380 {
4381         struct drm_device *dev = intel_crtc->base.dev;
4382         struct drm_i915_private *dev_priv = to_i915(dev);
4383         int pipe = intel_crtc->pipe;
4384         i915_reg_t reg;
4385         u32 temp;
4386
4387         /* Switch from PCDclk to Rawclk */
4388         reg = FDI_RX_CTL(pipe);
4389         temp = I915_READ(reg);
4390         I915_WRITE(reg, temp & ~FDI_PCDCLK);
4391
4392         /* Disable CPU FDI TX PLL */
4393         reg = FDI_TX_CTL(pipe);
4394         temp = I915_READ(reg);
4395         I915_WRITE(reg, temp & ~FDI_TX_PLL_ENABLE);
4396
4397         POSTING_READ(reg);
4398         udelay(100);
4399
4400         reg = FDI_RX_CTL(pipe);
4401         temp = I915_READ(reg);
4402         I915_WRITE(reg, temp & ~FDI_RX_PLL_ENABLE);
4403
4404         /* Wait for the clocks to turn off. */
4405         POSTING_READ(reg);
4406         udelay(100);
4407 }
4408
4409 static void ironlake_fdi_disable(struct drm_crtc *crtc)
4410 {
4411         struct drm_device *dev = crtc->dev;
4412         struct drm_i915_private *dev_priv = to_i915(dev);
4413         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
4414         int pipe = intel_crtc->pipe;
4415         i915_reg_t reg;
4416         u32 temp;
4417
4418         /* disable CPU FDI tx and PCH FDI rx */
4419         reg = FDI_TX_CTL(pipe);
4420         temp = I915_READ(reg);
4421         I915_WRITE(reg, temp & ~FDI_TX_ENABLE);
4422         POSTING_READ(reg);
4423
4424         reg = FDI_RX_CTL(pipe);
4425         temp = I915_READ(reg);
4426         temp &= ~(0x7 << 16);
4427         temp |= (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) << 11;
4428         I915_WRITE(reg, temp & ~FDI_RX_ENABLE);
4429
4430         POSTING_READ(reg);
4431         udelay(100);
4432
4433         /* Ironlake workaround, disable clock pointer after downing FDI */
4434         if (HAS_PCH_IBX(dev_priv))
4435                 I915_WRITE(FDI_RX_CHICKEN(pipe), FDI_RX_PHASE_SYNC_POINTER_OVR);
4436
4437         /* still set train pattern 1 */
4438         reg = FDI_TX_CTL(pipe);
4439         temp = I915_READ(reg);
4440         temp &= ~FDI_LINK_TRAIN_NONE;
4441         temp |= FDI_LINK_TRAIN_PATTERN_1;
4442         I915_WRITE(reg, temp);
4443
4444         reg = FDI_RX_CTL(pipe);
4445         temp = I915_READ(reg);
4446         if (HAS_PCH_CPT(dev_priv)) {
4447                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
4448                 temp |= FDI_LINK_TRAIN_PATTERN_1_CPT;
4449         } else {
4450                 temp &= ~FDI_LINK_TRAIN_NONE;
4451                 temp |= FDI_LINK_TRAIN_PATTERN_1;
4452         }
4453         /* BPC in FDI rx is consistent with that in PIPECONF */
4454         temp &= ~(0x07 << 16);
4455         temp |= (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) << 11;
4456         I915_WRITE(reg, temp);
4457
4458         POSTING_READ(reg);
4459         udelay(100);
4460 }
4461
4462 bool intel_has_pending_fb_unpin(struct drm_i915_private *dev_priv)
4463 {
4464         struct drm_crtc *crtc;
4465         bool cleanup_done;
4466
4467         drm_for_each_crtc(crtc, &dev_priv->drm) {
4468                 struct drm_crtc_commit *commit;
4469                 spin_lock(&crtc->commit_lock);
4470                 commit = list_first_entry_or_null(&crtc->commit_list,
4471                                                   struct drm_crtc_commit, commit_entry);
4472                 cleanup_done = commit ?
4473                         try_wait_for_completion(&commit->cleanup_done) : true;
4474                 spin_unlock(&crtc->commit_lock);
4475
4476                 if (cleanup_done)
4477                         continue;
4478
4479                 drm_crtc_wait_one_vblank(crtc);
4480
4481                 return true;
4482         }
4483
4484         return false;
4485 }
4486
4487 void lpt_disable_iclkip(struct drm_i915_private *dev_priv)
4488 {
4489         u32 temp;
4490
4491         I915_WRITE(PIXCLK_GATE, PIXCLK_GATE_GATE);
4492
4493         mutex_lock(&dev_priv->sb_lock);
4494
4495         temp = intel_sbi_read(dev_priv, SBI_SSCCTL6, SBI_ICLK);
4496         temp |= SBI_SSCCTL_DISABLE;
4497         intel_sbi_write(dev_priv, SBI_SSCCTL6, temp, SBI_ICLK);
4498
4499         mutex_unlock(&dev_priv->sb_lock);
4500 }
4501
4502 /* Program iCLKIP clock to the desired frequency */
4503 static void lpt_program_iclkip(struct intel_crtc *crtc)
4504 {
4505         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4506         int clock = crtc->config->base.adjusted_mode.crtc_clock;
4507         u32 divsel, phaseinc, auxdiv, phasedir = 0;
4508         u32 temp;
4509
4510         lpt_disable_iclkip(dev_priv);
4511
4512         /* The iCLK virtual clock root frequency is in MHz,
4513          * but the adjusted_mode->crtc_clock in in KHz. To get the
4514          * divisors, it is necessary to divide one by another, so we
4515          * convert the virtual clock precision to KHz here for higher
4516          * precision.
4517          */
4518         for (auxdiv = 0; auxdiv < 2; auxdiv++) {
4519                 u32 iclk_virtual_root_freq = 172800 * 1000;
4520                 u32 iclk_pi_range = 64;
4521                 u32 desired_divisor;
4522
4523                 desired_divisor = DIV_ROUND_CLOSEST(iclk_virtual_root_freq,
4524                                                     clock << auxdiv);
4525                 divsel = (desired_divisor / iclk_pi_range) - 2;
4526                 phaseinc = desired_divisor % iclk_pi_range;
4527
4528                 /*
4529                  * Near 20MHz is a corner case which is
4530                  * out of range for the 7-bit divisor
4531                  */
4532                 if (divsel <= 0x7f)
4533                         break;
4534         }
4535
4536         /* This should not happen with any sane values */
4537         WARN_ON(SBI_SSCDIVINTPHASE_DIVSEL(divsel) &
4538                 ~SBI_SSCDIVINTPHASE_DIVSEL_MASK);
4539         WARN_ON(SBI_SSCDIVINTPHASE_DIR(phasedir) &
4540                 ~SBI_SSCDIVINTPHASE_INCVAL_MASK);
4541
4542         DRM_DEBUG_KMS("iCLKIP clock: found settings for %dKHz refresh rate: auxdiv=%x, divsel=%x, phasedir=%x, phaseinc=%x\n",
4543                         clock,
4544                         auxdiv,
4545                         divsel,
4546                         phasedir,
4547                         phaseinc);
4548
4549         mutex_lock(&dev_priv->sb_lock);
4550
4551         /* Program SSCDIVINTPHASE6 */
4552         temp = intel_sbi_read(dev_priv, SBI_SSCDIVINTPHASE6, SBI_ICLK);
4553         temp &= ~SBI_SSCDIVINTPHASE_DIVSEL_MASK;
4554         temp |= SBI_SSCDIVINTPHASE_DIVSEL(divsel);
4555         temp &= ~SBI_SSCDIVINTPHASE_INCVAL_MASK;
4556         temp |= SBI_SSCDIVINTPHASE_INCVAL(phaseinc);
4557         temp |= SBI_SSCDIVINTPHASE_DIR(phasedir);
4558         temp |= SBI_SSCDIVINTPHASE_PROPAGATE;
4559         intel_sbi_write(dev_priv, SBI_SSCDIVINTPHASE6, temp, SBI_ICLK);
4560
4561         /* Program SSCAUXDIV */
4562         temp = intel_sbi_read(dev_priv, SBI_SSCAUXDIV6, SBI_ICLK);
4563         temp &= ~SBI_SSCAUXDIV_FINALDIV2SEL(1);
4564         temp |= SBI_SSCAUXDIV_FINALDIV2SEL(auxdiv);
4565         intel_sbi_write(dev_priv, SBI_SSCAUXDIV6, temp, SBI_ICLK);
4566
4567         /* Enable modulator and associated divider */
4568         temp = intel_sbi_read(dev_priv, SBI_SSCCTL6, SBI_ICLK);
4569         temp &= ~SBI_SSCCTL_DISABLE;
4570         intel_sbi_write(dev_priv, SBI_SSCCTL6, temp, SBI_ICLK);
4571
4572         mutex_unlock(&dev_priv->sb_lock);
4573
4574         /* Wait for initialization time */
4575         udelay(24);
4576
4577         I915_WRITE(PIXCLK_GATE, PIXCLK_GATE_UNGATE);
4578 }
4579
4580 int lpt_get_iclkip(struct drm_i915_private *dev_priv)
4581 {
4582         u32 divsel, phaseinc, auxdiv;
4583         u32 iclk_virtual_root_freq = 172800 * 1000;
4584         u32 iclk_pi_range = 64;
4585         u32 desired_divisor;
4586         u32 temp;
4587
4588         if ((I915_READ(PIXCLK_GATE) & PIXCLK_GATE_UNGATE) == 0)
4589                 return 0;
4590
4591         mutex_lock(&dev_priv->sb_lock);
4592
4593         temp = intel_sbi_read(dev_priv, SBI_SSCCTL6, SBI_ICLK);
4594         if (temp & SBI_SSCCTL_DISABLE) {
4595                 mutex_unlock(&dev_priv->sb_lock);
4596                 return 0;
4597         }
4598
4599         temp = intel_sbi_read(dev_priv, SBI_SSCDIVINTPHASE6, SBI_ICLK);
4600         divsel = (temp & SBI_SSCDIVINTPHASE_DIVSEL_MASK) >>
4601                 SBI_SSCDIVINTPHASE_DIVSEL_SHIFT;
4602         phaseinc = (temp & SBI_SSCDIVINTPHASE_INCVAL_MASK) >>
4603                 SBI_SSCDIVINTPHASE_INCVAL_SHIFT;
4604
4605         temp = intel_sbi_read(dev_priv, SBI_SSCAUXDIV6, SBI_ICLK);
4606         auxdiv = (temp & SBI_SSCAUXDIV_FINALDIV2SEL_MASK) >>
4607                 SBI_SSCAUXDIV_FINALDIV2SEL_SHIFT;
4608
4609         mutex_unlock(&dev_priv->sb_lock);
4610
4611         desired_divisor = (divsel + 2) * iclk_pi_range + phaseinc;
4612
4613         return DIV_ROUND_CLOSEST(iclk_virtual_root_freq,
4614                                  desired_divisor << auxdiv);
4615 }
4616
4617 static void ironlake_pch_transcoder_set_timings(struct intel_crtc *crtc,
4618                                                 enum pipe pch_transcoder)
4619 {
4620         struct drm_device *dev = crtc->base.dev;
4621         struct drm_i915_private *dev_priv = to_i915(dev);
4622         enum transcoder cpu_transcoder = crtc->config->cpu_transcoder;
4623
4624         I915_WRITE(PCH_TRANS_HTOTAL(pch_transcoder),
4625                    I915_READ(HTOTAL(cpu_transcoder)));
4626         I915_WRITE(PCH_TRANS_HBLANK(pch_transcoder),
4627                    I915_READ(HBLANK(cpu_transcoder)));
4628         I915_WRITE(PCH_TRANS_HSYNC(pch_transcoder),
4629                    I915_READ(HSYNC(cpu_transcoder)));
4630
4631         I915_WRITE(PCH_TRANS_VTOTAL(pch_transcoder),
4632                    I915_READ(VTOTAL(cpu_transcoder)));
4633         I915_WRITE(PCH_TRANS_VBLANK(pch_transcoder),
4634                    I915_READ(VBLANK(cpu_transcoder)));
4635         I915_WRITE(PCH_TRANS_VSYNC(pch_transcoder),
4636                    I915_READ(VSYNC(cpu_transcoder)));
4637         I915_WRITE(PCH_TRANS_VSYNCSHIFT(pch_transcoder),
4638                    I915_READ(VSYNCSHIFT(cpu_transcoder)));
4639 }
4640
4641 static void cpt_set_fdi_bc_bifurcation(struct drm_device *dev, bool enable)
4642 {
4643         struct drm_i915_private *dev_priv = to_i915(dev);
4644         uint32_t temp;
4645
4646         temp = I915_READ(SOUTH_CHICKEN1);
4647         if (!!(temp & FDI_BC_BIFURCATION_SELECT) == enable)
4648                 return;
4649
4650         WARN_ON(I915_READ(FDI_RX_CTL(PIPE_B)) & FDI_RX_ENABLE);
4651         WARN_ON(I915_READ(FDI_RX_CTL(PIPE_C)) & FDI_RX_ENABLE);
4652
4653         temp &= ~FDI_BC_BIFURCATION_SELECT;
4654         if (enable)
4655                 temp |= FDI_BC_BIFURCATION_SELECT;
4656
4657         DRM_DEBUG_KMS("%sabling fdi C rx\n", enable ? "en" : "dis");
4658         I915_WRITE(SOUTH_CHICKEN1, temp);
4659         POSTING_READ(SOUTH_CHICKEN1);
4660 }
4661
4662 static void ivybridge_update_fdi_bc_bifurcation(struct intel_crtc *intel_crtc)
4663 {
4664         struct drm_device *dev = intel_crtc->base.dev;
4665
4666         switch (intel_crtc->pipe) {
4667         case PIPE_A:
4668                 break;
4669         case PIPE_B:
4670                 if (intel_crtc->config->fdi_lanes > 2)
4671                         cpt_set_fdi_bc_bifurcation(dev, false);
4672                 else
4673                         cpt_set_fdi_bc_bifurcation(dev, true);
4674
4675                 break;
4676         case PIPE_C:
4677                 cpt_set_fdi_bc_bifurcation(dev, true);
4678
4679                 break;
4680         default:
4681                 BUG();
4682         }
4683 }
4684
4685 /*
4686  * Finds the encoder associated with the given CRTC. This can only be
4687  * used when we know that the CRTC isn't feeding multiple encoders!
4688  */
4689 static struct intel_encoder *
4690 intel_get_crtc_new_encoder(const struct intel_atomic_state *state,
4691                            const struct intel_crtc_state *crtc_state)
4692 {
4693         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
4694         const struct drm_connector_state *connector_state;
4695         const struct drm_connector *connector;
4696         struct intel_encoder *encoder = NULL;
4697         int num_encoders = 0;
4698         int i;
4699
4700         for_each_new_connector_in_state(&state->base, connector, connector_state, i) {
4701                 if (connector_state->crtc != &crtc->base)
4702                         continue;
4703
4704                 encoder = to_intel_encoder(connector_state->best_encoder);
4705                 num_encoders++;
4706         }
4707
4708         WARN(num_encoders != 1, "%d encoders for pipe %c\n",
4709              num_encoders, pipe_name(crtc->pipe));
4710
4711         return encoder;
4712 }
4713
4714 /*
4715  * Enable PCH resources required for PCH ports:
4716  *   - PCH PLLs
4717  *   - FDI training & RX/TX
4718  *   - update transcoder timings
4719  *   - DP transcoding bits
4720  *   - transcoder
4721  */
4722 static void ironlake_pch_enable(const struct intel_atomic_state *state,
4723                                 const struct intel_crtc_state *crtc_state)
4724 {
4725         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
4726         struct drm_device *dev = crtc->base.dev;
4727         struct drm_i915_private *dev_priv = to_i915(dev);
4728         int pipe = crtc->pipe;
4729         u32 temp;
4730
4731         assert_pch_transcoder_disabled(dev_priv, pipe);
4732
4733         if (IS_IVYBRIDGE(dev_priv))
4734                 ivybridge_update_fdi_bc_bifurcation(crtc);
4735
4736         /* Write the TU size bits before fdi link training, so that error
4737          * detection works. */
4738         I915_WRITE(FDI_RX_TUSIZE1(pipe),
4739                    I915_READ(PIPE_DATA_M1(pipe)) & TU_SIZE_MASK);
4740
4741         /* For PCH output, training FDI link */
4742         dev_priv->display.fdi_link_train(crtc, crtc_state);
4743
4744         /* We need to program the right clock selection before writing the pixel
4745          * mutliplier into the DPLL. */
4746         if (HAS_PCH_CPT(dev_priv)) {
4747                 u32 sel;
4748
4749                 temp = I915_READ(PCH_DPLL_SEL);
4750                 temp |= TRANS_DPLL_ENABLE(pipe);
4751                 sel = TRANS_DPLLB_SEL(pipe);
4752                 if (crtc_state->shared_dpll ==
4753                     intel_get_shared_dpll_by_id(dev_priv, DPLL_ID_PCH_PLL_B))
4754                         temp |= sel;
4755                 else
4756                         temp &= ~sel;
4757                 I915_WRITE(PCH_DPLL_SEL, temp);
4758         }
4759
4760         /* XXX: pch pll's can be enabled any time before we enable the PCH
4761          * transcoder, and we actually should do this to not upset any PCH
4762          * transcoder that already use the clock when we share it.
4763          *
4764          * Note that enable_shared_dpll tries to do the right thing, but
4765          * get_shared_dpll unconditionally resets the pll - we need that to have
4766          * the right LVDS enable sequence. */
4767         intel_enable_shared_dpll(crtc);
4768
4769         /* set transcoder timing, panel must allow it */
4770         assert_panel_unlocked(dev_priv, pipe);
4771         ironlake_pch_transcoder_set_timings(crtc, pipe);
4772
4773         intel_fdi_normal_train(crtc);
4774
4775         /* For PCH DP, enable TRANS_DP_CTL */
4776         if (HAS_PCH_CPT(dev_priv) &&
4777             intel_crtc_has_dp_encoder(crtc_state)) {
4778                 const struct drm_display_mode *adjusted_mode =
4779                         &crtc_state->base.adjusted_mode;
4780                 u32 bpc = (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) >> 5;
4781                 i915_reg_t reg = TRANS_DP_CTL(pipe);
4782                 enum port port;
4783
4784                 temp = I915_READ(reg);
4785                 temp &= ~(TRANS_DP_PORT_SEL_MASK |
4786                           TRANS_DP_SYNC_MASK |
4787                           TRANS_DP_BPC_MASK);
4788                 temp |= TRANS_DP_OUTPUT_ENABLE;
4789                 temp |= bpc << 9; /* same format but at 11:9 */
4790
4791                 if (adjusted_mode->flags & DRM_MODE_FLAG_PHSYNC)
4792                         temp |= TRANS_DP_HSYNC_ACTIVE_HIGH;
4793                 if (adjusted_mode->flags & DRM_MODE_FLAG_PVSYNC)
4794                         temp |= TRANS_DP_VSYNC_ACTIVE_HIGH;
4795
4796                 port = intel_get_crtc_new_encoder(state, crtc_state)->port;
4797                 WARN_ON(port < PORT_B || port > PORT_D);
4798                 temp |= TRANS_DP_PORT_SEL(port);
4799
4800                 I915_WRITE(reg, temp);
4801         }
4802
4803         ironlake_enable_pch_transcoder(dev_priv, pipe);
4804 }
4805
4806 static void lpt_pch_enable(const struct intel_atomic_state *state,
4807                            const struct intel_crtc_state *crtc_state)
4808 {
4809         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
4810         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4811         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
4812
4813         assert_pch_transcoder_disabled(dev_priv, PIPE_A);
4814
4815         lpt_program_iclkip(crtc);
4816
4817         /* Set transcoder timing. */
4818         ironlake_pch_transcoder_set_timings(crtc, PIPE_A);
4819
4820         lpt_enable_pch_transcoder(dev_priv, cpu_transcoder);
4821 }
4822
4823 static void cpt_verify_modeset(struct drm_device *dev, int pipe)
4824 {
4825         struct drm_i915_private *dev_priv = to_i915(dev);
4826         i915_reg_t dslreg = PIPEDSL(pipe);
4827         u32 temp;
4828
4829         temp = I915_READ(dslreg);
4830         udelay(500);
4831         if (wait_for(I915_READ(dslreg) != temp, 5)) {
4832                 if (wait_for(I915_READ(dslreg) != temp, 5))
4833                         DRM_ERROR("mode set failed: pipe %c stuck\n", pipe_name(pipe));
4834         }
4835 }
4836
4837 /*
4838  * The hardware phase 0.0 refers to the center of the pixel.
4839  * We want to start from the top/left edge which is phase
4840  * -0.5. That matches how the hardware calculates the scaling
4841  * factors (from top-left of the first pixel to bottom-right
4842  * of the last pixel, as opposed to the pixel centers).
4843  *
4844  * For 4:2:0 subsampled chroma planes we obviously have to
4845  * adjust that so that the chroma sample position lands in
4846  * the right spot.
4847  *
4848  * Note that for packed YCbCr 4:2:2 formats there is no way to
4849  * control chroma siting. The hardware simply replicates the
4850  * chroma samples for both of the luma samples, and thus we don't
4851  * actually get the expected MPEG2 chroma siting convention :(
4852  * The same behaviour is observed on pre-SKL platforms as well.
4853  *
4854  * Theory behind the formula (note that we ignore sub-pixel
4855  * source coordinates):
4856  * s = source sample position
4857  * d = destination sample position
4858  *
4859  * Downscaling 4:1:
4860  * -0.5
4861  * | 0.0
4862  * | |     1.5 (initial phase)
4863  * | |     |
4864  * v v     v
4865  * | s | s | s | s |
4866  * |       d       |
4867  *
4868  * Upscaling 1:4:
4869  * -0.5
4870  * | -0.375 (initial phase)
4871  * | |     0.0
4872  * | |     |
4873  * v v     v
4874  * |       s       |
4875  * | d | d | d | d |
4876  */
4877 u16 skl_scaler_calc_phase(int sub, int scale, bool chroma_cosited)
4878 {
4879         int phase = -0x8000;
4880         u16 trip = 0;
4881
4882         if (chroma_cosited)
4883                 phase += (sub - 1) * 0x8000 / sub;
4884
4885         phase += scale / (2 * sub);
4886
4887         /*
4888          * Hardware initial phase limited to [-0.5:1.5].
4889          * Since the max hardware scale factor is 3.0, we
4890          * should never actually excdeed 1.0 here.
4891          */
4892         WARN_ON(phase < -0x8000 || phase > 0x18000);
4893
4894         if (phase < 0)
4895                 phase = 0x10000 + phase;
4896         else
4897                 trip = PS_PHASE_TRIP;
4898
4899         return ((phase >> 2) & PS_PHASE_MASK) | trip;
4900 }
4901
4902 static int
4903 skl_update_scaler(struct intel_crtc_state *crtc_state, bool force_detach,
4904                   unsigned int scaler_user, int *scaler_id,
4905                   int src_w, int src_h, int dst_w, int dst_h,
4906                   bool plane_scaler_check,
4907                   uint32_t pixel_format)
4908 {
4909         struct intel_crtc_scaler_state *scaler_state =
4910                 &crtc_state->scaler_state;
4911         struct intel_crtc *intel_crtc =
4912                 to_intel_crtc(crtc_state->base.crtc);
4913         struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
4914         const struct drm_display_mode *adjusted_mode =
4915                 &crtc_state->base.adjusted_mode;
4916         int need_scaling;
4917
4918         /*
4919          * Src coordinates are already rotated by 270 degrees for
4920          * the 90/270 degree plane rotation cases (to match the
4921          * GTT mapping), hence no need to account for rotation here.
4922          */
4923         need_scaling = src_w != dst_w || src_h != dst_h;
4924
4925         if (plane_scaler_check)
4926                 if (pixel_format == DRM_FORMAT_NV12)
4927                         need_scaling = true;
4928
4929         if (crtc_state->ycbcr420 && scaler_user == SKL_CRTC_INDEX)
4930                 need_scaling = true;
4931
4932         /*
4933          * Scaling/fitting not supported in IF-ID mode in GEN9+
4934          * TODO: Interlace fetch mode doesn't support YUV420 planar formats.
4935          * Once NV12 is enabled, handle it here while allocating scaler
4936          * for NV12.
4937          */
4938         if (INTEL_GEN(dev_priv) >= 9 && crtc_state->base.enable &&
4939             need_scaling && adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) {
4940                 DRM_DEBUG_KMS("Pipe/Plane scaling not supported with IF-ID mode\n");
4941                 return -EINVAL;
4942         }
4943
4944         /*
4945          * if plane is being disabled or scaler is no more required or force detach
4946          *  - free scaler binded to this plane/crtc
4947          *  - in order to do this, update crtc->scaler_usage
4948          *
4949          * Here scaler state in crtc_state is set free so that
4950          * scaler can be assigned to other user. Actual register
4951          * update to free the scaler is done in plane/panel-fit programming.
4952          * For this purpose crtc/plane_state->scaler_id isn't reset here.
4953          */
4954         if (force_detach || !need_scaling) {
4955                 if (*scaler_id >= 0) {
4956                         scaler_state->scaler_users &= ~(1 << scaler_user);
4957                         scaler_state->scalers[*scaler_id].in_use = 0;
4958
4959                         DRM_DEBUG_KMS("scaler_user index %u.%u: "
4960                                 "Staged freeing scaler id %d scaler_users = 0x%x\n",
4961                                 intel_crtc->pipe, scaler_user, *scaler_id,
4962                                 scaler_state->scaler_users);
4963                         *scaler_id = -1;
4964                 }
4965                 return 0;
4966         }
4967
4968         if (plane_scaler_check && pixel_format == DRM_FORMAT_NV12 &&
4969             (src_h < SKL_MIN_YUV_420_SRC_H || src_w < SKL_MIN_YUV_420_SRC_W)) {
4970                 DRM_DEBUG_KMS("NV12: src dimensions not met\n");
4971                 return -EINVAL;
4972         }
4973
4974         /* range checks */
4975         if (src_w < SKL_MIN_SRC_W || src_h < SKL_MIN_SRC_H ||
4976             dst_w < SKL_MIN_DST_W || dst_h < SKL_MIN_DST_H ||
4977             (IS_GEN11(dev_priv) &&
4978              (src_w > ICL_MAX_SRC_W || src_h > ICL_MAX_SRC_H ||
4979               dst_w > ICL_MAX_DST_W || dst_h > ICL_MAX_DST_H)) ||
4980             (!IS_GEN11(dev_priv) &&
4981              (src_w > SKL_MAX_SRC_W || src_h > SKL_MAX_SRC_H ||
4982               dst_w > SKL_MAX_DST_W || dst_h > SKL_MAX_DST_H))) {
4983                 DRM_DEBUG_KMS("scaler_user index %u.%u: src %ux%u dst %ux%u "
4984                         "size is out of scaler range\n",
4985                         intel_crtc->pipe, scaler_user, src_w, src_h, dst_w, dst_h);
4986                 return -EINVAL;
4987         }
4988
4989         /* mark this plane as a scaler user in crtc_state */
4990         scaler_state->scaler_users |= (1 << scaler_user);
4991         DRM_DEBUG_KMS("scaler_user index %u.%u: "
4992                 "staged scaling request for %ux%u->%ux%u scaler_users = 0x%x\n",
4993                 intel_crtc->pipe, scaler_user, src_w, src_h, dst_w, dst_h,
4994                 scaler_state->scaler_users);
4995
4996         return 0;
4997 }
4998
4999 /**
5000  * skl_update_scaler_crtc - Stages update to scaler state for a given crtc.
5001  *
5002  * @state: crtc's scaler state
5003  *
5004  * Return
5005  *     0 - scaler_usage updated successfully
5006  *    error - requested scaling cannot be supported or other error condition
5007  */
5008 int skl_update_scaler_crtc(struct intel_crtc_state *state)
5009 {
5010         const struct drm_display_mode *adjusted_mode = &state->base.adjusted_mode;
5011
5012         return skl_update_scaler(state, !state->base.active, SKL_CRTC_INDEX,
5013                                  &state->scaler_state.scaler_id,
5014                                  state->pipe_src_w, state->pipe_src_h,
5015                                  adjusted_mode->crtc_hdisplay,
5016                                  adjusted_mode->crtc_vdisplay, false, 0);
5017 }
5018
5019 /**
5020  * skl_update_scaler_plane - Stages update to scaler state for a given plane.
5021  * @crtc_state: crtc's scaler state
5022  * @plane_state: atomic plane state to update
5023  *
5024  * Return
5025  *     0 - scaler_usage updated successfully
5026  *    error - requested scaling cannot be supported or other error condition
5027  */
5028 static int skl_update_scaler_plane(struct intel_crtc_state *crtc_state,
5029                                    struct intel_plane_state *plane_state)
5030 {
5031
5032         struct intel_plane *intel_plane =
5033                 to_intel_plane(plane_state->base.plane);
5034         struct drm_framebuffer *fb = plane_state->base.fb;
5035         int ret;
5036
5037         bool force_detach = !fb || !plane_state->base.visible;
5038
5039         ret = skl_update_scaler(crtc_state, force_detach,
5040                                 drm_plane_index(&intel_plane->base),
5041                                 &plane_state->scaler_id,
5042                                 drm_rect_width(&plane_state->base.src) >> 16,
5043                                 drm_rect_height(&plane_state->base.src) >> 16,
5044                                 drm_rect_width(&plane_state->base.dst),
5045                                 drm_rect_height(&plane_state->base.dst),
5046                                 fb ? true : false, fb ? fb->format->format : 0);
5047
5048         if (ret || plane_state->scaler_id < 0)
5049                 return ret;
5050
5051         /* check colorkey */
5052         if (plane_state->ckey.flags) {
5053                 DRM_DEBUG_KMS("[PLANE:%d:%s] scaling with color key not allowed",
5054                               intel_plane->base.base.id,
5055                               intel_plane->base.name);
5056                 return -EINVAL;
5057         }
5058
5059         /* Check src format */
5060         switch (fb->format->format) {
5061         case DRM_FORMAT_RGB565:
5062         case DRM_FORMAT_XBGR8888:
5063         case DRM_FORMAT_XRGB8888:
5064         case DRM_FORMAT_ABGR8888:
5065         case DRM_FORMAT_ARGB8888:
5066         case DRM_FORMAT_XRGB2101010:
5067         case DRM_FORMAT_XBGR2101010:
5068         case DRM_FORMAT_YUYV:
5069         case DRM_FORMAT_YVYU:
5070         case DRM_FORMAT_UYVY:
5071         case DRM_FORMAT_VYUY:
5072         case DRM_FORMAT_NV12:
5073                 break;
5074         default:
5075                 DRM_DEBUG_KMS("[PLANE:%d:%s] FB:%d unsupported scaling format 0x%x\n",
5076                               intel_plane->base.base.id, intel_plane->base.name,
5077                               fb->base.id, fb->format->format);
5078                 return -EINVAL;
5079         }
5080
5081         return 0;
5082 }
5083
5084 static void skylake_scaler_disable(struct intel_crtc *crtc)
5085 {
5086         int i;
5087
5088         for (i = 0; i < crtc->num_scalers; i++)
5089                 skl_detach_scaler(crtc, i);
5090 }
5091
5092 static void skylake_pfit_enable(struct intel_crtc *crtc)
5093 {
5094         struct drm_device *dev = crtc->base.dev;
5095         struct drm_i915_private *dev_priv = to_i915(dev);
5096         int pipe = crtc->pipe;
5097         struct intel_crtc_scaler_state *scaler_state =
5098                 &crtc->config->scaler_state;
5099
5100         if (crtc->config->pch_pfit.enabled) {
5101                 u16 uv_rgb_hphase, uv_rgb_vphase;
5102                 int pfit_w, pfit_h, hscale, vscale;
5103                 int id;
5104
5105                 if (WARN_ON(crtc->config->scaler_state.scaler_id < 0))
5106                         return;
5107
5108                 pfit_w = (crtc->config->pch_pfit.size >> 16) & 0xFFFF;
5109                 pfit_h = crtc->config->pch_pfit.size & 0xFFFF;
5110
5111                 hscale = (crtc->config->pipe_src_w << 16) / pfit_w;
5112                 vscale = (crtc->config->pipe_src_h << 16) / pfit_h;
5113
5114                 uv_rgb_hphase = skl_scaler_calc_phase(1, hscale, false);
5115                 uv_rgb_vphase = skl_scaler_calc_phase(1, vscale, false);
5116
5117                 id = scaler_state->scaler_id;
5118                 I915_WRITE(SKL_PS_CTRL(pipe, id), PS_SCALER_EN |
5119                         PS_FILTER_MEDIUM | scaler_state->scalers[id].mode);
5120                 I915_WRITE_FW(SKL_PS_VPHASE(pipe, id),
5121                               PS_Y_PHASE(0) | PS_UV_RGB_PHASE(uv_rgb_vphase));
5122                 I915_WRITE_FW(SKL_PS_HPHASE(pipe, id),
5123                               PS_Y_PHASE(0) | PS_UV_RGB_PHASE(uv_rgb_hphase));
5124                 I915_WRITE(SKL_PS_WIN_POS(pipe, id), crtc->config->pch_pfit.pos);
5125                 I915_WRITE(SKL_PS_WIN_SZ(pipe, id), crtc->config->pch_pfit.size);
5126         }
5127 }
5128
5129 static void ironlake_pfit_enable(struct intel_crtc *crtc)
5130 {
5131         struct drm_device *dev = crtc->base.dev;
5132         struct drm_i915_private *dev_priv = to_i915(dev);
5133         int pipe = crtc->pipe;
5134
5135         if (crtc->config->pch_pfit.enabled) {
5136                 /* Force use of hard-coded filter coefficients
5137                  * as some pre-programmed values are broken,
5138                  * e.g. x201.
5139                  */
5140                 if (IS_IVYBRIDGE(dev_priv) || IS_HASWELL(dev_priv))
5141                         I915_WRITE(PF_CTL(pipe), PF_ENABLE | PF_FILTER_MED_3x3 |
5142                                                  PF_PIPE_SEL_IVB(pipe));
5143                 else
5144                         I915_WRITE(PF_CTL(pipe), PF_ENABLE | PF_FILTER_MED_3x3);
5145                 I915_WRITE(PF_WIN_POS(pipe), crtc->config->pch_pfit.pos);
5146                 I915_WRITE(PF_WIN_SZ(pipe), crtc->config->pch_pfit.size);
5147         }
5148 }
5149
5150 void hsw_enable_ips(const struct intel_crtc_state *crtc_state)
5151 {
5152         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
5153         struct drm_device *dev = crtc->base.dev;
5154         struct drm_i915_private *dev_priv = to_i915(dev);
5155
5156         if (!crtc_state->ips_enabled)
5157                 return;
5158
5159         /*
5160          * We can only enable IPS after we enable a plane and wait for a vblank
5161          * This function is called from post_plane_update, which is run after
5162          * a vblank wait.
5163          */
5164         WARN_ON(!(crtc_state->active_planes & ~BIT(PLANE_CURSOR)));
5165
5166         if (IS_BROADWELL(dev_priv)) {
5167                 mutex_lock(&dev_priv->pcu_lock);
5168                 WARN_ON(sandybridge_pcode_write(dev_priv, DISPLAY_IPS_CONTROL,
5169                                                 IPS_ENABLE | IPS_PCODE_CONTROL));
5170                 mutex_unlock(&dev_priv->pcu_lock);
5171                 /* Quoting Art Runyan: "its not safe to expect any particular
5172                  * value in IPS_CTL bit 31 after enabling IPS through the
5173                  * mailbox." Moreover, the mailbox may return a bogus state,
5174                  * so we need to just enable it and continue on.
5175                  */
5176         } else {
5177                 I915_WRITE(IPS_CTL, IPS_ENABLE);
5178                 /* The bit only becomes 1 in the next vblank, so this wait here
5179                  * is essentially intel_wait_for_vblank. If we don't have this
5180                  * and don't wait for vblanks until the end of crtc_enable, then
5181                  * the HW state readout code will complain that the expected
5182                  * IPS_CTL value is not the one we read. */
5183                 if (intel_wait_for_register(dev_priv,
5184                                             IPS_CTL, IPS_ENABLE, IPS_ENABLE,
5185                                             50))
5186                         DRM_ERROR("Timed out waiting for IPS enable\n");
5187         }
5188 }
5189
5190 void hsw_disable_ips(const struct intel_crtc_state *crtc_state)
5191 {
5192         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
5193         struct drm_device *dev = crtc->base.dev;
5194         struct drm_i915_private *dev_priv = to_i915(dev);
5195
5196         if (!crtc_state->ips_enabled)
5197                 return;
5198
5199         if (IS_BROADWELL(dev_priv)) {
5200                 mutex_lock(&dev_priv->pcu_lock);
5201                 WARN_ON(sandybridge_pcode_write(dev_priv, DISPLAY_IPS_CONTROL, 0));
5202                 mutex_unlock(&dev_priv->pcu_lock);
5203                 /*
5204                  * Wait for PCODE to finish disabling IPS. The BSpec specified
5205                  * 42ms timeout value leads to occasional timeouts so use 100ms
5206                  * instead.
5207                  */
5208                 if (intel_wait_for_register(dev_priv,
5209                                             IPS_CTL, IPS_ENABLE, 0,
5210                                             100))
5211                         DRM_ERROR("Timed out waiting for IPS disable\n");
5212         } else {
5213                 I915_WRITE(IPS_CTL, 0);
5214                 POSTING_READ(IPS_CTL);
5215         }
5216
5217         /* We need to wait for a vblank before we can disable the plane. */
5218         intel_wait_for_vblank(dev_priv, crtc->pipe);
5219 }
5220
5221 static void intel_crtc_dpms_overlay_disable(struct intel_crtc *intel_crtc)
5222 {
5223         if (intel_crtc->overlay) {
5224                 struct drm_device *dev = intel_crtc->base.dev;
5225
5226                 mutex_lock(&dev->struct_mutex);
5227                 (void) intel_overlay_switch_off(intel_crtc->overlay);
5228                 mutex_unlock(&dev->struct_mutex);
5229         }
5230
5231         /* Let userspace switch the overlay on again. In most cases userspace
5232          * has to recompute where to put it anyway.
5233          */
5234 }
5235
5236 /**
5237  * intel_post_enable_primary - Perform operations after enabling primary plane
5238  * @crtc: the CRTC whose primary plane was just enabled
5239  * @new_crtc_state: the enabling state
5240  *
5241  * Performs potentially sleeping operations that must be done after the primary
5242  * plane is enabled, such as updating FBC and IPS.  Note that this may be
5243  * called due to an explicit primary plane update, or due to an implicit
5244  * re-enable that is caused when a sprite plane is updated to no longer
5245  * completely hide the primary plane.
5246  */
5247 static void
5248 intel_post_enable_primary(struct drm_crtc *crtc,
5249                           const struct intel_crtc_state *new_crtc_state)
5250 {
5251         struct drm_device *dev = crtc->dev;
5252         struct drm_i915_private *dev_priv = to_i915(dev);
5253         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5254         int pipe = intel_crtc->pipe;
5255
5256         /*
5257          * Gen2 reports pipe underruns whenever all planes are disabled.
5258          * So don't enable underrun reporting before at least some planes
5259          * are enabled.
5260          * FIXME: Need to fix the logic to work when we turn off all planes
5261          * but leave the pipe running.
5262          */
5263         if (IS_GEN2(dev_priv))
5264                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
5265
5266         /* Underruns don't always raise interrupts, so check manually. */
5267         intel_check_cpu_fifo_underruns(dev_priv);
5268         intel_check_pch_fifo_underruns(dev_priv);
5269 }
5270
5271 /* FIXME get rid of this and use pre_plane_update */
5272 static void
5273 intel_pre_disable_primary_noatomic(struct drm_crtc *crtc)
5274 {
5275         struct drm_device *dev = crtc->dev;
5276         struct drm_i915_private *dev_priv = to_i915(dev);
5277         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5278         int pipe = intel_crtc->pipe;
5279
5280         /*
5281          * Gen2 reports pipe underruns whenever all planes are disabled.
5282          * So disable underrun reporting before all the planes get disabled.
5283          */
5284         if (IS_GEN2(dev_priv))
5285                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
5286
5287         hsw_disable_ips(to_intel_crtc_state(crtc->state));
5288
5289         /*
5290          * Vblank time updates from the shadow to live plane control register
5291          * are blocked if the memory self-refresh mode is active at that
5292          * moment. So to make sure the plane gets truly disabled, disable
5293          * first the self-refresh mode. The self-refresh enable bit in turn
5294          * will be checked/applied by the HW only at the next frame start
5295          * event which is after the vblank start event, so we need to have a
5296          * wait-for-vblank between disabling the plane and the pipe.
5297          */
5298         if (HAS_GMCH_DISPLAY(dev_priv) &&
5299             intel_set_memory_cxsr(dev_priv, false))
5300                 intel_wait_for_vblank(dev_priv, pipe);
5301 }
5302
5303 static bool hsw_pre_update_disable_ips(const struct intel_crtc_state *old_crtc_state,
5304                                        const struct intel_crtc_state *new_crtc_state)
5305 {
5306         if (!old_crtc_state->ips_enabled)
5307                 return false;
5308
5309         if (needs_modeset(&new_crtc_state->base))
5310                 return true;
5311
5312         return !new_crtc_state->ips_enabled;
5313 }
5314
5315 static bool hsw_post_update_enable_ips(const struct intel_crtc_state *old_crtc_state,
5316                                        const struct intel_crtc_state *new_crtc_state)
5317 {
5318         if (!new_crtc_state->ips_enabled)
5319                 return false;
5320
5321         if (needs_modeset(&new_crtc_state->base))
5322                 return true;
5323
5324         /*
5325          * We can't read out IPS on broadwell, assume the worst and
5326          * forcibly enable IPS on the first fastset.
5327          */
5328         if (new_crtc_state->update_pipe &&
5329             old_crtc_state->base.adjusted_mode.private_flags & I915_MODE_FLAG_INHERITED)
5330                 return true;
5331
5332         return !old_crtc_state->ips_enabled;
5333 }
5334
5335 static bool needs_nv12_wa(struct drm_i915_private *dev_priv,
5336                           const struct intel_crtc_state *crtc_state)
5337 {
5338         if (!crtc_state->nv12_planes)
5339                 return false;
5340
5341         if (IS_SKYLAKE(dev_priv) || IS_BROXTON(dev_priv))
5342                 return false;
5343
5344         if ((INTEL_GEN(dev_priv) == 9 && !IS_GEMINILAKE(dev_priv)) ||
5345             IS_CANNONLAKE(dev_priv))
5346                 return true;
5347
5348         return false;
5349 }
5350
5351 static void intel_post_plane_update(struct intel_crtc_state *old_crtc_state)
5352 {
5353         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc);
5354         struct drm_device *dev = crtc->base.dev;
5355         struct drm_i915_private *dev_priv = to_i915(dev);
5356         struct drm_atomic_state *old_state = old_crtc_state->base.state;
5357         struct intel_crtc_state *pipe_config =
5358                 intel_atomic_get_new_crtc_state(to_intel_atomic_state(old_state),
5359                                                 crtc);
5360         struct drm_plane *primary = crtc->base.primary;
5361         struct drm_plane_state *old_primary_state =
5362                 drm_atomic_get_old_plane_state(old_state, primary);
5363
5364         intel_frontbuffer_flip(to_i915(crtc->base.dev), pipe_config->fb_bits);
5365
5366         if (pipe_config->update_wm_post && pipe_config->base.active)
5367                 intel_update_watermarks(crtc);
5368
5369         if (hsw_post_update_enable_ips(old_crtc_state, pipe_config))
5370                 hsw_enable_ips(pipe_config);
5371
5372         if (old_primary_state) {
5373                 struct drm_plane_state *new_primary_state =
5374                         drm_atomic_get_new_plane_state(old_state, primary);
5375
5376                 intel_fbc_post_update(crtc);
5377
5378                 if (new_primary_state->visible &&
5379                     (needs_modeset(&pipe_config->base) ||
5380                      !old_primary_state->visible))
5381                         intel_post_enable_primary(&crtc->base, pipe_config);
5382         }
5383
5384         /* Display WA 827 */
5385         if (needs_nv12_wa(dev_priv, old_crtc_state) &&
5386             !needs_nv12_wa(dev_priv, pipe_config)) {
5387                 skl_wa_clkgate(dev_priv, crtc->pipe, false);
5388                 skl_wa_528(dev_priv, crtc->pipe, false);
5389         }
5390 }
5391
5392 static void intel_pre_plane_update(struct intel_crtc_state *old_crtc_state,
5393                                    struct intel_crtc_state *pipe_config)
5394 {
5395         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc);
5396         struct drm_device *dev = crtc->base.dev;
5397         struct drm_i915_private *dev_priv = to_i915(dev);
5398         struct drm_atomic_state *old_state = old_crtc_state->base.state;
5399         struct drm_plane *primary = crtc->base.primary;
5400         struct drm_plane_state *old_primary_state =
5401                 drm_atomic_get_old_plane_state(old_state, primary);
5402         bool modeset = needs_modeset(&pipe_config->base);
5403         struct intel_atomic_state *old_intel_state =
5404                 to_intel_atomic_state(old_state);
5405
5406         if (hsw_pre_update_disable_ips(old_crtc_state, pipe_config))
5407                 hsw_disable_ips(old_crtc_state);
5408
5409         if (old_primary_state) {
5410                 struct intel_plane_state *new_primary_state =
5411                         intel_atomic_get_new_plane_state(old_intel_state,
5412                                                          to_intel_plane(primary));
5413
5414                 intel_fbc_pre_update(crtc, pipe_config, new_primary_state);
5415                 /*
5416                  * Gen2 reports pipe underruns whenever all planes are disabled.
5417                  * So disable underrun reporting before all the planes get disabled.
5418                  */
5419                 if (IS_GEN2(dev_priv) && old_primary_state->visible &&
5420                     (modeset || !new_primary_state->base.visible))
5421                         intel_set_cpu_fifo_underrun_reporting(dev_priv, crtc->pipe, false);
5422         }
5423
5424         /* Display WA 827 */
5425         if (!needs_nv12_wa(dev_priv, old_crtc_state) &&
5426             needs_nv12_wa(dev_priv, pipe_config)) {
5427                 skl_wa_clkgate(dev_priv, crtc->pipe, true);
5428                 skl_wa_528(dev_priv, crtc->pipe, true);
5429         }
5430
5431         /*
5432          * Vblank time updates from the shadow to live plane control register
5433          * are blocked if the memory self-refresh mode is active at that
5434          * moment. So to make sure the plane gets truly disabled, disable
5435          * first the self-refresh mode. The self-refresh enable bit in turn
5436          * will be checked/applied by the HW only at the next frame start
5437          * event which is after the vblank start event, so we need to have a
5438          * wait-for-vblank between disabling the plane and the pipe.
5439          */
5440         if (HAS_GMCH_DISPLAY(dev_priv) && old_crtc_state->base.active &&
5441             pipe_config->disable_cxsr && intel_set_memory_cxsr(dev_priv, false))
5442                 intel_wait_for_vblank(dev_priv, crtc->pipe);
5443
5444         /*
5445          * IVB workaround: must disable low power watermarks for at least
5446          * one frame before enabling scaling.  LP watermarks can be re-enabled
5447          * when scaling is disabled.
5448          *
5449          * WaCxSRDisabledForSpriteScaling:ivb
5450          */
5451         if (pipe_config->disable_lp_wm && ilk_disable_lp_wm(dev))
5452                 intel_wait_for_vblank(dev_priv, crtc->pipe);
5453
5454         /*
5455          * If we're doing a modeset, we're done.  No need to do any pre-vblank
5456          * watermark programming here.
5457          */
5458         if (needs_modeset(&pipe_config->base))
5459                 return;
5460
5461         /*
5462          * For platforms that support atomic watermarks, program the
5463          * 'intermediate' watermarks immediately.  On pre-gen9 platforms, these
5464          * will be the intermediate values that are safe for both pre- and
5465          * post- vblank; when vblank happens, the 'active' values will be set
5466          * to the final 'target' values and we'll do this again to get the
5467          * optimal watermarks.  For gen9+ platforms, the values we program here
5468          * will be the final target values which will get automatically latched
5469          * at vblank time; no further programming will be necessary.
5470          *
5471          * If a platform hasn't been transitioned to atomic watermarks yet,
5472          * we'll continue to update watermarks the old way, if flags tell
5473          * us to.
5474          */
5475         if (dev_priv->display.initial_watermarks != NULL)
5476                 dev_priv->display.initial_watermarks(old_intel_state,
5477                                                      pipe_config);
5478         else if (pipe_config->update_wm_pre)
5479                 intel_update_watermarks(crtc);
5480 }
5481
5482 static void intel_crtc_disable_planes(struct drm_crtc *crtc, unsigned plane_mask)
5483 {
5484         struct drm_device *dev = crtc->dev;
5485         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5486         struct drm_plane *p;
5487         int pipe = intel_crtc->pipe;
5488
5489         intel_crtc_dpms_overlay_disable(intel_crtc);
5490
5491         drm_for_each_plane_mask(p, dev, plane_mask)
5492                 to_intel_plane(p)->disable_plane(to_intel_plane(p), intel_crtc);
5493
5494         /*
5495          * FIXME: Once we grow proper nuclear flip support out of this we need
5496          * to compute the mask of flip planes precisely. For the time being
5497          * consider this a flip to a NULL plane.
5498          */
5499         intel_frontbuffer_flip(to_i915(dev), INTEL_FRONTBUFFER_ALL_MASK(pipe));
5500 }
5501
5502 static void intel_encoders_pre_pll_enable(struct drm_crtc *crtc,
5503                                           struct intel_crtc_state *crtc_state,
5504                                           struct drm_atomic_state *old_state)
5505 {
5506         struct drm_connector_state *conn_state;
5507         struct drm_connector *conn;
5508         int i;
5509
5510         for_each_new_connector_in_state(old_state, conn, conn_state, i) {
5511                 struct intel_encoder *encoder =
5512                         to_intel_encoder(conn_state->best_encoder);
5513
5514                 if (conn_state->crtc != crtc)
5515                         continue;
5516
5517                 if (encoder->pre_pll_enable)
5518                         encoder->pre_pll_enable(encoder, crtc_state, conn_state);
5519         }
5520 }
5521
5522 static void intel_encoders_pre_enable(struct drm_crtc *crtc,
5523                                       struct intel_crtc_state *crtc_state,
5524                                       struct drm_atomic_state *old_state)
5525 {
5526         struct drm_connector_state *conn_state;
5527         struct drm_connector *conn;
5528         int i;
5529
5530         for_each_new_connector_in_state(old_state, conn, conn_state, i) {
5531                 struct intel_encoder *encoder =
5532                         to_intel_encoder(conn_state->best_encoder);
5533
5534                 if (conn_state->crtc != crtc)
5535                         continue;
5536
5537                 if (encoder->pre_enable)
5538                         encoder->pre_enable(encoder, crtc_state, conn_state);
5539         }
5540 }
5541
5542 static void intel_encoders_enable(struct drm_crtc *crtc,
5543                                   struct intel_crtc_state *crtc_state,
5544                                   struct drm_atomic_state *old_state)
5545 {
5546         struct drm_connector_state *conn_state;
5547         struct drm_connector *conn;
5548         int i;
5549
5550         for_each_new_connector_in_state(old_state, conn, conn_state, i) {
5551                 struct intel_encoder *encoder =
5552                         to_intel_encoder(conn_state->best_encoder);
5553
5554                 if (conn_state->crtc != crtc)
5555                         continue;
5556
5557                 encoder->enable(encoder, crtc_state, conn_state);
5558                 intel_opregion_notify_encoder(encoder, true);
5559         }
5560 }
5561
5562 static void intel_encoders_disable(struct drm_crtc *crtc,
5563                                    struct intel_crtc_state *old_crtc_state,
5564                                    struct drm_atomic_state *old_state)
5565 {
5566         struct drm_connector_state *old_conn_state;
5567         struct drm_connector *conn;
5568         int i;
5569
5570         for_each_old_connector_in_state(old_state, conn, old_conn_state, i) {
5571                 struct intel_encoder *encoder =
5572                         to_intel_encoder(old_conn_state->best_encoder);
5573
5574                 if (old_conn_state->crtc != crtc)
5575                         continue;
5576
5577                 intel_opregion_notify_encoder(encoder, false);
5578                 encoder->disable(encoder, old_crtc_state, old_conn_state);
5579         }
5580 }
5581
5582 static void intel_encoders_post_disable(struct drm_crtc *crtc,
5583                                         struct intel_crtc_state *old_crtc_state,
5584                                         struct drm_atomic_state *old_state)
5585 {
5586         struct drm_connector_state *old_conn_state;
5587         struct drm_connector *conn;
5588         int i;
5589
5590         for_each_old_connector_in_state(old_state, conn, old_conn_state, i) {
5591                 struct intel_encoder *encoder =
5592                         to_intel_encoder(old_conn_state->best_encoder);
5593
5594                 if (old_conn_state->crtc != crtc)
5595                         continue;
5596
5597                 if (encoder->post_disable)
5598                         encoder->post_disable(encoder, old_crtc_state, old_conn_state);
5599         }
5600 }
5601
5602 static void intel_encoders_post_pll_disable(struct drm_crtc *crtc,
5603                                             struct intel_crtc_state *old_crtc_state,
5604                                             struct drm_atomic_state *old_state)
5605 {
5606         struct drm_connector_state *old_conn_state;
5607         struct drm_connector *conn;
5608         int i;
5609
5610         for_each_old_connector_in_state(old_state, conn, old_conn_state, i) {
5611                 struct intel_encoder *encoder =
5612                         to_intel_encoder(old_conn_state->best_encoder);
5613
5614                 if (old_conn_state->crtc != crtc)
5615                         continue;
5616
5617                 if (encoder->post_pll_disable)
5618                         encoder->post_pll_disable(encoder, old_crtc_state, old_conn_state);
5619         }
5620 }
5621
5622 static void ironlake_crtc_enable(struct intel_crtc_state *pipe_config,
5623                                  struct drm_atomic_state *old_state)
5624 {
5625         struct drm_crtc *crtc = pipe_config->base.crtc;
5626         struct drm_device *dev = crtc->dev;
5627         struct drm_i915_private *dev_priv = to_i915(dev);
5628         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5629         int pipe = intel_crtc->pipe;
5630         struct intel_atomic_state *old_intel_state =
5631                 to_intel_atomic_state(old_state);
5632
5633         if (WARN_ON(intel_crtc->active))
5634                 return;
5635
5636         /*
5637          * Sometimes spurious CPU pipe underruns happen during FDI
5638          * training, at least with VGA+HDMI cloning. Suppress them.
5639          *
5640          * On ILK we get an occasional spurious CPU pipe underruns
5641          * between eDP port A enable and vdd enable. Also PCH port
5642          * enable seems to result in the occasional CPU pipe underrun.
5643          *
5644          * Spurious PCH underruns also occur during PCH enabling.
5645          */
5646         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
5647         intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, false);
5648
5649         if (intel_crtc->config->has_pch_encoder)
5650                 intel_prepare_shared_dpll(intel_crtc);
5651
5652         if (intel_crtc_has_dp_encoder(intel_crtc->config))
5653                 intel_dp_set_m_n(intel_crtc, M1_N1);
5654
5655         intel_set_pipe_timings(intel_crtc);
5656         intel_set_pipe_src_size(intel_crtc);
5657
5658         if (intel_crtc->config->has_pch_encoder) {
5659                 intel_cpu_transcoder_set_m_n(intel_crtc,
5660                                      &intel_crtc->config->fdi_m_n, NULL);
5661         }
5662
5663         ironlake_set_pipeconf(crtc);
5664
5665         intel_crtc->active = true;
5666
5667         intel_encoders_pre_enable(crtc, pipe_config, old_state);
5668
5669         if (intel_crtc->config->has_pch_encoder) {
5670                 /* Note: FDI PLL enabling _must_ be done before we enable the
5671                  * cpu pipes, hence this is separate from all the other fdi/pch
5672                  * enabling. */
5673                 ironlake_fdi_pll_enable(intel_crtc);
5674         } else {
5675                 assert_fdi_tx_disabled(dev_priv, pipe);
5676                 assert_fdi_rx_disabled(dev_priv, pipe);
5677         }
5678
5679         ironlake_pfit_enable(intel_crtc);
5680
5681         /*
5682          * On ILK+ LUT must be loaded before the pipe is running but with
5683          * clocks enabled
5684          */
5685         intel_color_load_luts(&pipe_config->base);
5686
5687         if (dev_priv->display.initial_watermarks != NULL)
5688                 dev_priv->display.initial_watermarks(old_intel_state, intel_crtc->config);
5689         intel_enable_pipe(pipe_config);
5690
5691         if (intel_crtc->config->has_pch_encoder)
5692                 ironlake_pch_enable(old_intel_state, pipe_config);
5693
5694         assert_vblank_disabled(crtc);
5695         drm_crtc_vblank_on(crtc);
5696
5697         intel_encoders_enable(crtc, pipe_config, old_state);
5698
5699         if (HAS_PCH_CPT(dev_priv))
5700                 cpt_verify_modeset(dev, intel_crtc->pipe);
5701
5702         /*
5703          * Must wait for vblank to avoid spurious PCH FIFO underruns.
5704          * And a second vblank wait is needed at least on ILK with
5705          * some interlaced HDMI modes. Let's do the double wait always
5706          * in case there are more corner cases we don't know about.
5707          */
5708         if (intel_crtc->config->has_pch_encoder) {
5709                 intel_wait_for_vblank(dev_priv, pipe);
5710                 intel_wait_for_vblank(dev_priv, pipe);
5711         }
5712         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
5713         intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, true);
5714 }
5715
5716 /* IPS only exists on ULT machines and is tied to pipe A. */
5717 static bool hsw_crtc_supports_ips(struct intel_crtc *crtc)
5718 {
5719         return HAS_IPS(to_i915(crtc->base.dev)) && crtc->pipe == PIPE_A;
5720 }
5721
5722 static void glk_pipe_scaler_clock_gating_wa(struct drm_i915_private *dev_priv,
5723                                             enum pipe pipe, bool apply)
5724 {
5725         u32 val = I915_READ(CLKGATE_DIS_PSL(pipe));
5726         u32 mask = DPF_GATING_DIS | DPF_RAM_GATING_DIS | DPFR_GATING_DIS;
5727
5728         if (apply)
5729                 val |= mask;
5730         else
5731                 val &= ~mask;
5732
5733         I915_WRITE(CLKGATE_DIS_PSL(pipe), val);
5734 }
5735
5736 static void icl_pipe_mbus_enable(struct intel_crtc *crtc)
5737 {
5738         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5739         enum pipe pipe = crtc->pipe;
5740         uint32_t val;
5741
5742         val = MBUS_DBOX_BW_CREDIT(1) | MBUS_DBOX_A_CREDIT(2);
5743
5744         /* Program B credit equally to all pipes */
5745         val |= MBUS_DBOX_B_CREDIT(24 / INTEL_INFO(dev_priv)->num_pipes);
5746
5747         I915_WRITE(PIPE_MBUS_DBOX_CTL(pipe), val);
5748 }
5749
5750 static void haswell_crtc_enable(struct intel_crtc_state *pipe_config,
5751                                 struct drm_atomic_state *old_state)
5752 {
5753         struct drm_crtc *crtc = pipe_config->base.crtc;
5754         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
5755         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5756         int pipe = intel_crtc->pipe, hsw_workaround_pipe;
5757         enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder;
5758         struct intel_atomic_state *old_intel_state =
5759                 to_intel_atomic_state(old_state);
5760         bool psl_clkgate_wa;
5761         u32 pipe_chicken;
5762
5763         if (WARN_ON(intel_crtc->active))
5764                 return;
5765
5766         intel_encoders_pre_pll_enable(crtc, pipe_config, old_state);
5767
5768         if (intel_crtc->config->shared_dpll)
5769                 intel_enable_shared_dpll(intel_crtc);
5770
5771         if (INTEL_GEN(dev_priv) >= 11)
5772                 icl_map_plls_to_ports(crtc, pipe_config, old_state);
5773
5774         intel_encoders_pre_enable(crtc, pipe_config, old_state);
5775
5776         if (intel_crtc_has_dp_encoder(intel_crtc->config))
5777                 intel_dp_set_m_n(intel_crtc, M1_N1);
5778
5779         if (!transcoder_is_dsi(cpu_transcoder))
5780                 intel_set_pipe_timings(intel_crtc);
5781
5782         intel_set_pipe_src_size(intel_crtc);
5783
5784         if (cpu_transcoder != TRANSCODER_EDP &&
5785             !transcoder_is_dsi(cpu_transcoder)) {
5786                 I915_WRITE(PIPE_MULT(cpu_transcoder),
5787                            intel_crtc->config->pixel_multiplier - 1);
5788         }
5789
5790         if (intel_crtc->config->has_pch_encoder) {
5791                 intel_cpu_transcoder_set_m_n(intel_crtc,
5792                                      &intel_crtc->config->fdi_m_n, NULL);
5793         }
5794
5795         if (!transcoder_is_dsi(cpu_transcoder))
5796                 haswell_set_pipeconf(crtc);
5797
5798         haswell_set_pipemisc(crtc);
5799
5800         intel_color_set_csc(&pipe_config->base);
5801
5802         intel_crtc->active = true;
5803
5804         /* Display WA #1180: WaDisableScalarClockGating: glk, cnl */
5805         psl_clkgate_wa = (IS_GEMINILAKE(dev_priv) || IS_CANNONLAKE(dev_priv)) &&
5806                          intel_crtc->config->pch_pfit.enabled;
5807         if (psl_clkgate_wa)
5808                 glk_pipe_scaler_clock_gating_wa(dev_priv, pipe, true);
5809
5810         if (INTEL_GEN(dev_priv) >= 9)
5811                 skylake_pfit_enable(intel_crtc);
5812         else
5813                 ironlake_pfit_enable(intel_crtc);
5814
5815         /*
5816          * On ILK+ LUT must be loaded before the pipe is running but with
5817          * clocks enabled
5818          */
5819         intel_color_load_luts(&pipe_config->base);
5820
5821         /*
5822          * Display WA #1153: enable hardware to bypass the alpha math
5823          * and rounding for per-pixel values 00 and 0xff
5824          */
5825         if (INTEL_GEN(dev_priv) >= 11) {
5826                 pipe_chicken = I915_READ(PIPE_CHICKEN(pipe));
5827                 if (!(pipe_chicken & PER_PIXEL_ALPHA_BYPASS_EN))
5828                         I915_WRITE_FW(PIPE_CHICKEN(pipe),
5829                                       pipe_chicken | PER_PIXEL_ALPHA_BYPASS_EN);
5830         }
5831
5832         intel_ddi_set_pipe_settings(pipe_config);
5833         if (!transcoder_is_dsi(cpu_transcoder))
5834                 intel_ddi_enable_transcoder_func(pipe_config);
5835
5836         if (dev_priv->display.initial_watermarks != NULL)
5837                 dev_priv->display.initial_watermarks(old_intel_state, pipe_config);
5838
5839         if (INTEL_GEN(dev_priv) >= 11)
5840                 icl_pipe_mbus_enable(intel_crtc);
5841
5842         /* XXX: Do the pipe assertions at the right place for BXT DSI. */
5843         if (!transcoder_is_dsi(cpu_transcoder))
5844                 intel_enable_pipe(pipe_config);
5845
5846         if (intel_crtc->config->has_pch_encoder)
5847                 lpt_pch_enable(old_intel_state, pipe_config);
5848
5849         if (intel_crtc_has_type(intel_crtc->config, INTEL_OUTPUT_DP_MST))
5850                 intel_ddi_set_vc_payload_alloc(pipe_config, true);
5851
5852         assert_vblank_disabled(crtc);
5853         drm_crtc_vblank_on(crtc);
5854
5855         intel_encoders_enable(crtc, pipe_config, old_state);
5856
5857         if (psl_clkgate_wa) {
5858                 intel_wait_for_vblank(dev_priv, pipe);
5859                 glk_pipe_scaler_clock_gating_wa(dev_priv, pipe, false);
5860         }
5861
5862         /* If we change the relative order between pipe/planes enabling, we need
5863          * to change the workaround. */
5864         hsw_workaround_pipe = pipe_config->hsw_workaround_pipe;
5865         if (IS_HASWELL(dev_priv) && hsw_workaround_pipe != INVALID_PIPE) {
5866                 intel_wait_for_vblank(dev_priv, hsw_workaround_pipe);
5867                 intel_wait_for_vblank(dev_priv, hsw_workaround_pipe);
5868         }
5869 }
5870
5871 static void ironlake_pfit_disable(struct intel_crtc *crtc, bool force)
5872 {
5873         struct drm_device *dev = crtc->base.dev;
5874         struct drm_i915_private *dev_priv = to_i915(dev);
5875         int pipe = crtc->pipe;
5876
5877         /* To avoid upsetting the power well on haswell only disable the pfit if
5878          * it's in use. The hw state code will make sure we get this right. */
5879         if (force || crtc->config->pch_pfit.enabled) {
5880                 I915_WRITE(PF_CTL(pipe), 0);
5881                 I915_WRITE(PF_WIN_POS(pipe), 0);
5882                 I915_WRITE(PF_WIN_SZ(pipe), 0);
5883         }
5884 }
5885
5886 static void ironlake_crtc_disable(struct intel_crtc_state *old_crtc_state,
5887                                   struct drm_atomic_state *old_state)
5888 {
5889         struct drm_crtc *crtc = old_crtc_state->base.crtc;
5890         struct drm_device *dev = crtc->dev;
5891         struct drm_i915_private *dev_priv = to_i915(dev);
5892         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5893         int pipe = intel_crtc->pipe;
5894
5895         /*
5896          * Sometimes spurious CPU pipe underruns happen when the
5897          * pipe is already disabled, but FDI RX/TX is still enabled.
5898          * Happens at least with VGA+HDMI cloning. Suppress them.
5899          */
5900         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
5901         intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, false);
5902
5903         intel_encoders_disable(crtc, old_crtc_state, old_state);
5904
5905         drm_crtc_vblank_off(crtc);
5906         assert_vblank_disabled(crtc);
5907
5908         intel_disable_pipe(old_crtc_state);
5909
5910         ironlake_pfit_disable(intel_crtc, false);
5911
5912         if (intel_crtc->config->has_pch_encoder)
5913                 ironlake_fdi_disable(crtc);
5914
5915         intel_encoders_post_disable(crtc, old_crtc_state, old_state);
5916
5917         if (intel_crtc->config->has_pch_encoder) {
5918                 ironlake_disable_pch_transcoder(dev_priv, pipe);
5919
5920                 if (HAS_PCH_CPT(dev_priv)) {
5921                         i915_reg_t reg;
5922                         u32 temp;
5923
5924                         /* disable TRANS_DP_CTL */
5925                         reg = TRANS_DP_CTL(pipe);
5926                         temp = I915_READ(reg);
5927                         temp &= ~(TRANS_DP_OUTPUT_ENABLE |
5928                                   TRANS_DP_PORT_SEL_MASK);
5929                         temp |= TRANS_DP_PORT_SEL_NONE;
5930                         I915_WRITE(reg, temp);
5931
5932                         /* disable DPLL_SEL */
5933                         temp = I915_READ(PCH_DPLL_SEL);
5934                         temp &= ~(TRANS_DPLL_ENABLE(pipe) | TRANS_DPLLB_SEL(pipe));
5935                         I915_WRITE(PCH_DPLL_SEL, temp);
5936                 }
5937
5938                 ironlake_fdi_pll_disable(intel_crtc);
5939         }
5940
5941         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
5942         intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, true);
5943 }
5944
5945 static void haswell_crtc_disable(struct intel_crtc_state *old_crtc_state,
5946                                  struct drm_atomic_state *old_state)
5947 {
5948         struct drm_crtc *crtc = old_crtc_state->base.crtc;
5949         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
5950         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5951         enum transcoder cpu_transcoder = old_crtc_state->cpu_transcoder;
5952
5953         intel_encoders_disable(crtc, old_crtc_state, old_state);
5954
5955         drm_crtc_vblank_off(crtc);
5956         assert_vblank_disabled(crtc);
5957
5958         /* XXX: Do the pipe assertions at the right place for BXT DSI. */
5959         if (!transcoder_is_dsi(cpu_transcoder))
5960                 intel_disable_pipe(old_crtc_state);
5961
5962         if (intel_crtc_has_type(old_crtc_state, INTEL_OUTPUT_DP_MST))
5963                 intel_ddi_set_vc_payload_alloc(old_crtc_state, false);
5964
5965         if (!transcoder_is_dsi(cpu_transcoder))
5966                 intel_ddi_disable_transcoder_func(old_crtc_state);
5967
5968         if (INTEL_GEN(dev_priv) >= 9)
5969                 skylake_scaler_disable(intel_crtc);
5970         else
5971                 ironlake_pfit_disable(intel_crtc, false);
5972
5973         intel_encoders_post_disable(crtc, old_crtc_state, old_state);
5974
5975         if (INTEL_GEN(dev_priv) >= 11)
5976                 icl_unmap_plls_to_ports(crtc, old_crtc_state, old_state);
5977 }
5978
5979 static void i9xx_pfit_enable(struct intel_crtc *crtc)
5980 {
5981         struct drm_device *dev = crtc->base.dev;
5982         struct drm_i915_private *dev_priv = to_i915(dev);
5983         struct intel_crtc_state *pipe_config = crtc->config;
5984
5985         if (!pipe_config->gmch_pfit.control)
5986                 return;
5987
5988         /*
5989          * The panel fitter should only be adjusted whilst the pipe is disabled,
5990          * according to register description and PRM.
5991          */
5992         WARN_ON(I915_READ(PFIT_CONTROL) & PFIT_ENABLE);
5993         assert_pipe_disabled(dev_priv, crtc->pipe);
5994
5995         I915_WRITE(PFIT_PGM_RATIOS, pipe_config->gmch_pfit.pgm_ratios);
5996         I915_WRITE(PFIT_CONTROL, pipe_config->gmch_pfit.control);
5997
5998         /* Border color in case we don't scale up to the full screen. Black by
5999          * default, change to something else for debugging. */
6000         I915_WRITE(BCLRPAT(crtc->pipe), 0);
6001 }
6002
6003 bool intel_port_is_combophy(struct drm_i915_private *dev_priv, enum port port)
6004 {
6005         if (port == PORT_NONE)
6006                 return false;
6007
6008         if (IS_ICELAKE(dev_priv))
6009                 return port <= PORT_B;
6010
6011         return false;
6012 }
6013
6014 bool intel_port_is_tc(struct drm_i915_private *dev_priv, enum port port)
6015 {
6016         if (IS_ICELAKE(dev_priv))
6017                 return port >= PORT_C && port <= PORT_F;
6018
6019         return false;
6020 }
6021
6022 enum tc_port intel_port_to_tc(struct drm_i915_private *dev_priv, enum port port)
6023 {
6024         if (!intel_port_is_tc(dev_priv, port))
6025                 return PORT_TC_NONE;
6026
6027         return port - PORT_C;
6028 }
6029
6030 enum intel_display_power_domain intel_port_to_power_domain(enum port port)
6031 {
6032         switch (port) {
6033         case PORT_A:
6034                 return POWER_DOMAIN_PORT_DDI_A_LANES;
6035         case PORT_B:
6036                 return POWER_DOMAIN_PORT_DDI_B_LANES;
6037         case PORT_C:
6038                 return POWER_DOMAIN_PORT_DDI_C_LANES;
6039         case PORT_D:
6040                 return POWER_DOMAIN_PORT_DDI_D_LANES;
6041         case PORT_E:
6042                 return POWER_DOMAIN_PORT_DDI_E_LANES;
6043         case PORT_F:
6044                 return POWER_DOMAIN_PORT_DDI_F_LANES;
6045         default:
6046                 MISSING_CASE(port);
6047                 return POWER_DOMAIN_PORT_OTHER;
6048         }
6049 }
6050
6051 static u64 get_crtc_power_domains(struct drm_crtc *crtc,
6052                                   struct intel_crtc_state *crtc_state)
6053 {
6054         struct drm_device *dev = crtc->dev;
6055         struct drm_i915_private *dev_priv = to_i915(dev);
6056         struct drm_encoder *encoder;
6057         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6058         enum pipe pipe = intel_crtc->pipe;
6059         u64 mask;
6060         enum transcoder transcoder = crtc_state->cpu_transcoder;
6061
6062         if (!crtc_state->base.active)
6063                 return 0;
6064
6065         mask = BIT_ULL(POWER_DOMAIN_PIPE(pipe));
6066         mask |= BIT_ULL(POWER_DOMAIN_TRANSCODER(transcoder));
6067         if (crtc_state->pch_pfit.enabled ||
6068             crtc_state->pch_pfit.force_thru)
6069                 mask |= BIT_ULL(POWER_DOMAIN_PIPE_PANEL_FITTER(pipe));
6070
6071         drm_for_each_encoder_mask(encoder, dev, crtc_state->base.encoder_mask) {
6072                 struct intel_encoder *intel_encoder = to_intel_encoder(encoder);
6073
6074                 mask |= BIT_ULL(intel_encoder->power_domain);
6075         }
6076
6077         if (HAS_DDI(dev_priv) && crtc_state->has_audio)
6078                 mask |= BIT_ULL(POWER_DOMAIN_AUDIO);
6079
6080         if (crtc_state->shared_dpll)
6081                 mask |= BIT_ULL(POWER_DOMAIN_PLLS);
6082
6083         return mask;
6084 }
6085
6086 static u64
6087 modeset_get_crtc_power_domains(struct drm_crtc *crtc,
6088                                struct intel_crtc_state *crtc_state)
6089 {
6090         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
6091         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6092         enum intel_display_power_domain domain;
6093         u64 domains, new_domains, old_domains;
6094
6095         old_domains = intel_crtc->enabled_power_domains;
6096         intel_crtc->enabled_power_domains = new_domains =
6097                 get_crtc_power_domains(crtc, crtc_state);
6098
6099         domains = new_domains & ~old_domains;
6100
6101         for_each_power_domain(domain, domains)
6102                 intel_display_power_get(dev_priv, domain);
6103
6104         return old_domains & ~new_domains;
6105 }
6106
6107 static void modeset_put_power_domains(struct drm_i915_private *dev_priv,
6108                                       u64 domains)
6109 {
6110         enum intel_display_power_domain domain;
6111
6112         for_each_power_domain(domain, domains)
6113                 intel_display_power_put(dev_priv, domain);
6114 }
6115
6116 static void valleyview_crtc_enable(struct intel_crtc_state *pipe_config,
6117                                    struct drm_atomic_state *old_state)
6118 {
6119         struct intel_atomic_state *old_intel_state =
6120                 to_intel_atomic_state(old_state);
6121         struct drm_crtc *crtc = pipe_config->base.crtc;
6122         struct drm_device *dev = crtc->dev;
6123         struct drm_i915_private *dev_priv = to_i915(dev);
6124         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6125         int pipe = intel_crtc->pipe;
6126
6127         if (WARN_ON(intel_crtc->active))
6128                 return;
6129
6130         if (intel_crtc_has_dp_encoder(intel_crtc->config))
6131                 intel_dp_set_m_n(intel_crtc, M1_N1);
6132
6133         intel_set_pipe_timings(intel_crtc);
6134         intel_set_pipe_src_size(intel_crtc);
6135
6136         if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_B) {
6137                 struct drm_i915_private *dev_priv = to_i915(dev);
6138
6139                 I915_WRITE(CHV_BLEND(pipe), CHV_BLEND_LEGACY);
6140                 I915_WRITE(CHV_CANVAS(pipe), 0);
6141         }
6142
6143         i9xx_set_pipeconf(intel_crtc);
6144
6145         intel_color_set_csc(&pipe_config->base);
6146
6147         intel_crtc->active = true;
6148
6149         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
6150
6151         intel_encoders_pre_pll_enable(crtc, pipe_config, old_state);
6152
6153         if (IS_CHERRYVIEW(dev_priv)) {
6154                 chv_prepare_pll(intel_crtc, intel_crtc->config);
6155                 chv_enable_pll(intel_crtc, intel_crtc->config);
6156         } else {
6157                 vlv_prepare_pll(intel_crtc, intel_crtc->config);
6158                 vlv_enable_pll(intel_crtc, intel_crtc->config);
6159         }
6160
6161         intel_encoders_pre_enable(crtc, pipe_config, old_state);
6162
6163         i9xx_pfit_enable(intel_crtc);
6164
6165         intel_color_load_luts(&pipe_config->base);
6166
6167         dev_priv->display.initial_watermarks(old_intel_state,
6168                                              pipe_config);
6169         intel_enable_pipe(pipe_config);
6170
6171         assert_vblank_disabled(crtc);
6172         drm_crtc_vblank_on(crtc);
6173
6174         intel_encoders_enable(crtc, pipe_config, old_state);
6175 }
6176
6177 static void i9xx_set_pll_dividers(struct intel_crtc *crtc)
6178 {
6179         struct drm_device *dev = crtc->base.dev;
6180         struct drm_i915_private *dev_priv = to_i915(dev);
6181
6182         I915_WRITE(FP0(crtc->pipe), crtc->config->dpll_hw_state.fp0);
6183         I915_WRITE(FP1(crtc->pipe), crtc->config->dpll_hw_state.fp1);
6184 }
6185
6186 static void i9xx_crtc_enable(struct intel_crtc_state *pipe_config,
6187                              struct drm_atomic_state *old_state)
6188 {
6189         struct intel_atomic_state *old_intel_state =
6190                 to_intel_atomic_state(old_state);
6191         struct drm_crtc *crtc = pipe_config->base.crtc;
6192         struct drm_device *dev = crtc->dev;
6193         struct drm_i915_private *dev_priv = to_i915(dev);
6194         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6195         enum pipe pipe = intel_crtc->pipe;
6196
6197         if (WARN_ON(intel_crtc->active))
6198                 return;
6199
6200         i9xx_set_pll_dividers(intel_crtc);
6201
6202         if (intel_crtc_has_dp_encoder(intel_crtc->config))
6203                 intel_dp_set_m_n(intel_crtc, M1_N1);
6204
6205         intel_set_pipe_timings(intel_crtc);
6206         intel_set_pipe_src_size(intel_crtc);
6207
6208         i9xx_set_pipeconf(intel_crtc);
6209
6210         intel_crtc->active = true;
6211
6212         if (!IS_GEN2(dev_priv))
6213                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
6214
6215         intel_encoders_pre_enable(crtc, pipe_config, old_state);
6216
6217         i9xx_enable_pll(intel_crtc, pipe_config);
6218
6219         i9xx_pfit_enable(intel_crtc);
6220
6221         intel_color_load_luts(&pipe_config->base);
6222
6223         if (dev_priv->display.initial_watermarks != NULL)
6224                 dev_priv->display.initial_watermarks(old_intel_state,
6225                                                      intel_crtc->config);
6226         else
6227                 intel_update_watermarks(intel_crtc);
6228         intel_enable_pipe(pipe_config);
6229
6230         assert_vblank_disabled(crtc);
6231         drm_crtc_vblank_on(crtc);
6232
6233         intel_encoders_enable(crtc, pipe_config, old_state);
6234 }
6235
6236 static void i9xx_pfit_disable(struct intel_crtc *crtc)
6237 {
6238         struct drm_device *dev = crtc->base.dev;
6239         struct drm_i915_private *dev_priv = to_i915(dev);
6240
6241         if (!crtc->config->gmch_pfit.control)
6242                 return;
6243
6244         assert_pipe_disabled(dev_priv, crtc->pipe);
6245
6246         DRM_DEBUG_KMS("disabling pfit, current: 0x%08x\n",
6247                       I915_READ(PFIT_CONTROL));
6248         I915_WRITE(PFIT_CONTROL, 0);
6249 }
6250
6251 static void i9xx_crtc_disable(struct intel_crtc_state *old_crtc_state,
6252                               struct drm_atomic_state *old_state)
6253 {
6254         struct drm_crtc *crtc = old_crtc_state->base.crtc;
6255         struct drm_device *dev = crtc->dev;
6256         struct drm_i915_private *dev_priv = to_i915(dev);
6257         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6258         int pipe = intel_crtc->pipe;
6259
6260         /*
6261          * On gen2 planes are double buffered but the pipe isn't, so we must
6262          * wait for planes to fully turn off before disabling the pipe.
6263          */
6264         if (IS_GEN2(dev_priv))
6265                 intel_wait_for_vblank(dev_priv, pipe);
6266
6267         intel_encoders_disable(crtc, old_crtc_state, old_state);
6268
6269         drm_crtc_vblank_off(crtc);
6270         assert_vblank_disabled(crtc);
6271
6272         intel_disable_pipe(old_crtc_state);
6273
6274         i9xx_pfit_disable(intel_crtc);
6275
6276         intel_encoders_post_disable(crtc, old_crtc_state, old_state);
6277
6278         if (!intel_crtc_has_type(intel_crtc->config, INTEL_OUTPUT_DSI)) {
6279                 if (IS_CHERRYVIEW(dev_priv))
6280                         chv_disable_pll(dev_priv, pipe);
6281                 else if (IS_VALLEYVIEW(dev_priv))
6282                         vlv_disable_pll(dev_priv, pipe);
6283                 else
6284                         i9xx_disable_pll(intel_crtc);
6285         }
6286
6287         intel_encoders_post_pll_disable(crtc, old_crtc_state, old_state);
6288
6289         if (!IS_GEN2(dev_priv))
6290                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
6291
6292         if (!dev_priv->display.initial_watermarks)
6293                 intel_update_watermarks(intel_crtc);
6294
6295         /* clock the pipe down to 640x480@60 to potentially save power */
6296         if (IS_I830(dev_priv))
6297                 i830_enable_pipe(dev_priv, pipe);
6298 }
6299
6300 static void intel_crtc_disable_noatomic(struct drm_crtc *crtc,
6301                                         struct drm_modeset_acquire_ctx *ctx)
6302 {
6303         struct intel_encoder *encoder;
6304         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6305         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
6306         enum intel_display_power_domain domain;
6307         struct intel_plane *plane;
6308         u64 domains;
6309         struct drm_atomic_state *state;
6310         struct intel_crtc_state *crtc_state;
6311         int ret;
6312
6313         if (!intel_crtc->active)
6314                 return;
6315
6316         for_each_intel_plane_on_crtc(&dev_priv->drm, intel_crtc, plane) {
6317                 const struct intel_plane_state *plane_state =
6318                         to_intel_plane_state(plane->base.state);
6319
6320                 if (plane_state->base.visible)
6321                         intel_plane_disable_noatomic(intel_crtc, plane);
6322         }
6323
6324         state = drm_atomic_state_alloc(crtc->dev);
6325         if (!state) {
6326                 DRM_DEBUG_KMS("failed to disable [CRTC:%d:%s], out of memory",
6327                               crtc->base.id, crtc->name);
6328                 return;
6329         }
6330
6331         state->acquire_ctx = ctx;
6332
6333         /* Everything's already locked, -EDEADLK can't happen. */
6334         crtc_state = intel_atomic_get_crtc_state(state, intel_crtc);
6335         ret = drm_atomic_add_affected_connectors(state, crtc);
6336
6337         WARN_ON(IS_ERR(crtc_state) || ret);
6338
6339         dev_priv->display.crtc_disable(crtc_state, state);
6340
6341         drm_atomic_state_put(state);
6342
6343         DRM_DEBUG_KMS("[CRTC:%d:%s] hw state adjusted, was enabled, now disabled\n",
6344                       crtc->base.id, crtc->name);
6345
6346         WARN_ON(drm_atomic_set_mode_for_crtc(crtc->state, NULL) < 0);
6347         crtc->state->active = false;
6348         intel_crtc->active = false;
6349         crtc->enabled = false;
6350         crtc->state->connector_mask = 0;
6351         crtc->state->encoder_mask = 0;
6352
6353         for_each_encoder_on_crtc(crtc->dev, crtc, encoder)
6354                 encoder->base.crtc = NULL;
6355
6356         intel_fbc_disable(intel_crtc);
6357         intel_update_watermarks(intel_crtc);
6358         intel_disable_shared_dpll(intel_crtc);
6359
6360         domains = intel_crtc->enabled_power_domains;
6361         for_each_power_domain(domain, domains)
6362                 intel_display_power_put(dev_priv, domain);
6363         intel_crtc->enabled_power_domains = 0;
6364
6365         dev_priv->active_crtcs &= ~(1 << intel_crtc->pipe);
6366         dev_priv->min_cdclk[intel_crtc->pipe] = 0;
6367         dev_priv->min_voltage_level[intel_crtc->pipe] = 0;
6368 }
6369
6370 /*
6371  * turn all crtc's off, but do not adjust state
6372  * This has to be paired with a call to intel_modeset_setup_hw_state.
6373  */
6374 int intel_display_suspend(struct drm_device *dev)
6375 {
6376         struct drm_i915_private *dev_priv = to_i915(dev);
6377         struct drm_atomic_state *state;
6378         int ret;
6379
6380         state = drm_atomic_helper_suspend(dev);
6381         ret = PTR_ERR_OR_ZERO(state);
6382         if (ret)
6383                 DRM_ERROR("Suspending crtc's failed with %i\n", ret);
6384         else
6385                 dev_priv->modeset_restore_state = state;
6386         return ret;
6387 }
6388
6389 void intel_encoder_destroy(struct drm_encoder *encoder)
6390 {
6391         struct intel_encoder *intel_encoder = to_intel_encoder(encoder);
6392
6393         drm_encoder_cleanup(encoder);
6394         kfree(intel_encoder);
6395 }
6396
6397 /* Cross check the actual hw state with our own modeset state tracking (and it's
6398  * internal consistency). */
6399 static void intel_connector_verify_state(struct drm_crtc_state *crtc_state,
6400                                          struct drm_connector_state *conn_state)
6401 {
6402         struct intel_connector *connector = to_intel_connector(conn_state->connector);
6403
6404         DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n",
6405                       connector->base.base.id,
6406                       connector->base.name);
6407
6408         if (connector->get_hw_state(connector)) {
6409                 struct intel_encoder *encoder = connector->encoder;
6410
6411                 I915_STATE_WARN(!crtc_state,
6412                          "connector enabled without attached crtc\n");
6413
6414                 if (!crtc_state)
6415                         return;
6416
6417                 I915_STATE_WARN(!crtc_state->active,
6418                       "connector is active, but attached crtc isn't\n");
6419
6420                 if (!encoder || encoder->type == INTEL_OUTPUT_DP_MST)
6421                         return;
6422
6423                 I915_STATE_WARN(conn_state->best_encoder != &encoder->base,
6424                         "atomic encoder doesn't match attached encoder\n");
6425
6426                 I915_STATE_WARN(conn_state->crtc != encoder->base.crtc,
6427                         "attached encoder crtc differs from connector crtc\n");
6428         } else {
6429                 I915_STATE_WARN(crtc_state && crtc_state->active,
6430                         "attached crtc is active, but connector isn't\n");
6431                 I915_STATE_WARN(!crtc_state && conn_state->best_encoder,
6432                         "best encoder set without crtc!\n");
6433         }
6434 }
6435
6436 int intel_connector_init(struct intel_connector *connector)
6437 {
6438         struct intel_digital_connector_state *conn_state;
6439
6440         /*
6441          * Allocate enough memory to hold intel_digital_connector_state,
6442          * This might be a few bytes too many, but for connectors that don't
6443          * need it we'll free the state and allocate a smaller one on the first
6444          * succesful commit anyway.
6445          */
6446         conn_state = kzalloc(sizeof(*conn_state), GFP_KERNEL);
6447         if (!conn_state)
6448                 return -ENOMEM;
6449
6450         __drm_atomic_helper_connector_reset(&connector->base,
6451                                             &conn_state->base);
6452
6453         return 0;
6454 }
6455
6456 struct intel_connector *intel_connector_alloc(void)
6457 {
6458         struct intel_connector *connector;
6459
6460         connector = kzalloc(sizeof *connector, GFP_KERNEL);
6461         if (!connector)
6462                 return NULL;
6463
6464         if (intel_connector_init(connector) < 0) {
6465                 kfree(connector);
6466                 return NULL;
6467         }
6468
6469         return connector;
6470 }
6471
6472 /*
6473  * Free the bits allocated by intel_connector_alloc.
6474  * This should only be used after intel_connector_alloc has returned
6475  * successfully, and before drm_connector_init returns successfully.
6476  * Otherwise the destroy callbacks for the connector and the state should
6477  * take care of proper cleanup/free
6478  */
6479 void intel_connector_free(struct intel_connector *connector)
6480 {
6481         kfree(to_intel_digital_connector_state(connector->base.state));
6482         kfree(connector);
6483 }
6484
6485 /* Simple connector->get_hw_state implementation for encoders that support only
6486  * one connector and no cloning and hence the encoder state determines the state
6487  * of the connector. */
6488 bool intel_connector_get_hw_state(struct intel_connector *connector)
6489 {
6490         enum pipe pipe = 0;
6491         struct intel_encoder *encoder = connector->encoder;
6492
6493         return encoder->get_hw_state(encoder, &pipe);
6494 }
6495
6496 static int pipe_required_fdi_lanes(struct intel_crtc_state *crtc_state)
6497 {
6498         if (crtc_state->base.enable && crtc_state->has_pch_encoder)
6499                 return crtc_state->fdi_lanes;
6500
6501         return 0;
6502 }
6503
6504 static int ironlake_check_fdi_lanes(struct drm_device *dev, enum pipe pipe,
6505                                      struct intel_crtc_state *pipe_config)
6506 {
6507         struct drm_i915_private *dev_priv = to_i915(dev);
6508         struct drm_atomic_state *state = pipe_config->base.state;
6509         struct intel_crtc *other_crtc;
6510         struct intel_crtc_state *other_crtc_state;
6511
6512         DRM_DEBUG_KMS("checking fdi config on pipe %c, lanes %i\n",
6513                       pipe_name(pipe), pipe_config->fdi_lanes);
6514         if (pipe_config->fdi_lanes > 4) {
6515                 DRM_DEBUG_KMS("invalid fdi lane config on pipe %c: %i lanes\n",
6516                               pipe_name(pipe), pipe_config->fdi_lanes);
6517                 return -EINVAL;
6518         }
6519
6520         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
6521                 if (pipe_config->fdi_lanes > 2) {
6522                         DRM_DEBUG_KMS("only 2 lanes on haswell, required: %i lanes\n",
6523                                       pipe_config->fdi_lanes);
6524                         return -EINVAL;
6525                 } else {
6526                         return 0;
6527                 }
6528         }
6529
6530         if (INTEL_INFO(dev_priv)->num_pipes == 2)
6531                 return 0;
6532
6533         /* Ivybridge 3 pipe is really complicated */
6534         switch (pipe) {
6535         case PIPE_A:
6536                 return 0;
6537         case PIPE_B:
6538                 if (pipe_config->fdi_lanes <= 2)
6539                         return 0;
6540
6541                 other_crtc = intel_get_crtc_for_pipe(dev_priv, PIPE_C);
6542                 other_crtc_state =
6543                         intel_atomic_get_crtc_state(state, other_crtc);
6544                 if (IS_ERR(other_crtc_state))
6545                         return PTR_ERR(other_crtc_state);
6546
6547                 if (pipe_required_fdi_lanes(other_crtc_state) > 0) {
6548                         DRM_DEBUG_KMS("invalid shared fdi lane config on pipe %c: %i lanes\n",
6549                                       pipe_name(pipe), pipe_config->fdi_lanes);
6550                         return -EINVAL;
6551                 }
6552                 return 0;
6553         case PIPE_C:
6554                 if (pipe_config->fdi_lanes > 2) {
6555                         DRM_DEBUG_KMS("only 2 lanes on pipe %c: required %i lanes\n",
6556                                       pipe_name(pipe), pipe_config->fdi_lanes);
6557                         return -EINVAL;
6558                 }
6559
6560                 other_crtc = intel_get_crtc_for_pipe(dev_priv, PIPE_B);
6561                 other_crtc_state =
6562                         intel_atomic_get_crtc_state(state, other_crtc);
6563                 if (IS_ERR(other_crtc_state))
6564                         return PTR_ERR(other_crtc_state);
6565
6566                 if (pipe_required_fdi_lanes(other_crtc_state) > 2) {
6567                         DRM_DEBUG_KMS("fdi link B uses too many lanes to enable link C\n");
6568                         return -EINVAL;
6569                 }
6570                 return 0;
6571         default:
6572                 BUG();
6573         }
6574 }
6575
6576 #define RETRY 1
6577 static int ironlake_fdi_compute_config(struct intel_crtc *intel_crtc,
6578                                        struct intel_crtc_state *pipe_config)
6579 {
6580         struct drm_device *dev = intel_crtc->base.dev;
6581         const struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode;
6582         int lane, link_bw, fdi_dotclock, ret;
6583         bool needs_recompute = false;
6584
6585 retry:
6586         /* FDI is a binary signal running at ~2.7GHz, encoding
6587          * each output octet as 10 bits. The actual frequency
6588          * is stored as a divider into a 100MHz clock, and the
6589          * mode pixel clock is stored in units of 1KHz.
6590          * Hence the bw of each lane in terms of the mode signal
6591          * is:
6592          */
6593         link_bw = intel_fdi_link_freq(to_i915(dev), pipe_config);
6594
6595         fdi_dotclock = adjusted_mode->crtc_clock;
6596
6597         lane = ironlake_get_lanes_required(fdi_dotclock, link_bw,
6598                                            pipe_config->pipe_bpp);
6599
6600         pipe_config->fdi_lanes = lane;
6601
6602         intel_link_compute_m_n(pipe_config->pipe_bpp, lane, fdi_dotclock,
6603                                link_bw, &pipe_config->fdi_m_n, false);
6604
6605         ret = ironlake_check_fdi_lanes(dev, intel_crtc->pipe, pipe_config);
6606         if (ret == -EINVAL && pipe_config->pipe_bpp > 6*3) {
6607                 pipe_config->pipe_bpp -= 2*3;
6608                 DRM_DEBUG_KMS("fdi link bw constraint, reducing pipe bpp to %i\n",
6609                               pipe_config->pipe_bpp);
6610                 needs_recompute = true;
6611                 pipe_config->bw_constrained = true;
6612
6613                 goto retry;
6614         }
6615
6616         if (needs_recompute)
6617                 return RETRY;
6618
6619         return ret;
6620 }
6621
6622 bool hsw_crtc_state_ips_capable(const struct intel_crtc_state *crtc_state)
6623 {
6624         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
6625         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6626
6627         /* IPS only exists on ULT machines and is tied to pipe A. */
6628         if (!hsw_crtc_supports_ips(crtc))
6629                 return false;
6630
6631         if (!i915_modparams.enable_ips)
6632                 return false;
6633
6634         if (crtc_state->pipe_bpp > 24)
6635                 return false;
6636
6637         /*
6638          * We compare against max which means we must take
6639          * the increased cdclk requirement into account when
6640          * calculating the new cdclk.
6641          *
6642          * Should measure whether using a lower cdclk w/o IPS
6643          */
6644         if (IS_BROADWELL(dev_priv) &&
6645             crtc_state->pixel_rate > dev_priv->max_cdclk_freq * 95 / 100)
6646                 return false;
6647
6648         return true;
6649 }
6650
6651 static bool hsw_compute_ips_config(struct intel_crtc_state *crtc_state)
6652 {
6653         struct drm_i915_private *dev_priv =
6654                 to_i915(crtc_state->base.crtc->dev);
6655         struct intel_atomic_state *intel_state =
6656                 to_intel_atomic_state(crtc_state->base.state);
6657
6658         if (!hsw_crtc_state_ips_capable(crtc_state))
6659                 return false;
6660
6661         if (crtc_state->ips_force_disable)
6662                 return false;
6663
6664         /* IPS should be fine as long as at least one plane is enabled. */
6665         if (!(crtc_state->active_planes & ~BIT(PLANE_CURSOR)))
6666                 return false;
6667
6668         /* pixel rate mustn't exceed 95% of cdclk with IPS on BDW */
6669         if (IS_BROADWELL(dev_priv) &&
6670             crtc_state->pixel_rate > intel_state->cdclk.logical.cdclk * 95 / 100)
6671                 return false;
6672
6673         return true;
6674 }
6675
6676 static bool intel_crtc_supports_double_wide(const struct intel_crtc *crtc)
6677 {
6678         const struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6679
6680         /* GDG double wide on either pipe, otherwise pipe A only */
6681         return INTEL_GEN(dev_priv) < 4 &&
6682                 (crtc->pipe == PIPE_A || IS_I915G(dev_priv));
6683 }
6684
6685 static uint32_t ilk_pipe_pixel_rate(const struct intel_crtc_state *pipe_config)
6686 {
6687         uint32_t pixel_rate;
6688
6689         pixel_rate = pipe_config->base.adjusted_mode.crtc_clock;
6690
6691         /*
6692          * We only use IF-ID interlacing. If we ever use
6693          * PF-ID we'll need to adjust the pixel_rate here.
6694          */
6695
6696         if (pipe_config->pch_pfit.enabled) {
6697                 uint64_t pipe_w, pipe_h, pfit_w, pfit_h;
6698                 uint32_t pfit_size = pipe_config->pch_pfit.size;
6699
6700                 pipe_w = pipe_config->pipe_src_w;
6701                 pipe_h = pipe_config->pipe_src_h;
6702
6703                 pfit_w = (pfit_size >> 16) & 0xFFFF;
6704                 pfit_h = pfit_size & 0xFFFF;
6705                 if (pipe_w < pfit_w)
6706                         pipe_w = pfit_w;
6707                 if (pipe_h < pfit_h)
6708                         pipe_h = pfit_h;
6709
6710                 if (WARN_ON(!pfit_w || !pfit_h))
6711                         return pixel_rate;
6712
6713                 pixel_rate = div_u64((uint64_t) pixel_rate * pipe_w * pipe_h,
6714                                      pfit_w * pfit_h);
6715         }
6716
6717         return pixel_rate;
6718 }
6719
6720 static void intel_crtc_compute_pixel_rate(struct intel_crtc_state *crtc_state)
6721 {
6722         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
6723
6724         if (HAS_GMCH_DISPLAY(dev_priv))
6725                 /* FIXME calculate proper pipe pixel rate for GMCH pfit */
6726                 crtc_state->pixel_rate =
6727                         crtc_state->base.adjusted_mode.crtc_clock;
6728         else
6729                 crtc_state->pixel_rate =
6730                         ilk_pipe_pixel_rate(crtc_state);
6731 }
6732
6733 static int intel_crtc_compute_config(struct intel_crtc *crtc,
6734                                      struct intel_crtc_state *pipe_config)
6735 {
6736         struct drm_device *dev = crtc->base.dev;
6737         struct drm_i915_private *dev_priv = to_i915(dev);
6738         const struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode;
6739         int clock_limit = dev_priv->max_dotclk_freq;
6740
6741         if (INTEL_GEN(dev_priv) < 4) {
6742                 clock_limit = dev_priv->max_cdclk_freq * 9 / 10;
6743
6744                 /*
6745                  * Enable double wide mode when the dot clock
6746                  * is > 90% of the (display) core speed.
6747                  */
6748                 if (intel_crtc_supports_double_wide(crtc) &&
6749                     adjusted_mode->crtc_clock > clock_limit) {
6750                         clock_limit = dev_priv->max_dotclk_freq;
6751                         pipe_config->double_wide = true;
6752                 }
6753         }
6754
6755         if (adjusted_mode->crtc_clock > clock_limit) {
6756                 DRM_DEBUG_KMS("requested pixel clock (%d kHz) too high (max: %d kHz, double wide: %s)\n",
6757                               adjusted_mode->crtc_clock, clock_limit,
6758                               yesno(pipe_config->double_wide));
6759                 return -EINVAL;
6760         }
6761
6762         if (pipe_config->ycbcr420 && pipe_config->base.ctm) {
6763                 /*
6764                  * There is only one pipe CSC unit per pipe, and we need that
6765                  * for output conversion from RGB->YCBCR. So if CTM is already
6766                  * applied we can't support YCBCR420 output.
6767                  */
6768                 DRM_DEBUG_KMS("YCBCR420 and CTM together are not possible\n");
6769                 return -EINVAL;
6770         }
6771
6772         /*
6773          * Pipe horizontal size must be even in:
6774          * - DVO ganged mode
6775          * - LVDS dual channel mode
6776          * - Double wide pipe
6777          */
6778         if (pipe_config->pipe_src_w & 1) {
6779                 if (pipe_config->double_wide) {
6780                         DRM_DEBUG_KMS("Odd pipe source width not supported with double wide pipe\n");
6781                         return -EINVAL;
6782                 }
6783
6784                 if (intel_crtc_has_type(pipe_config, INTEL_OUTPUT_LVDS) &&
6785                     intel_is_dual_link_lvds(dev)) {
6786                         DRM_DEBUG_KMS("Odd pipe source width not supported with dual link LVDS\n");
6787                         return -EINVAL;
6788                 }
6789         }
6790
6791         /* Cantiga+ cannot handle modes with a hsync front porch of 0.
6792          * WaPruneModeWithIncorrectHsyncOffset:ctg,elk,ilk,snb,ivb,vlv,hsw.
6793          */
6794         if ((INTEL_GEN(dev_priv) > 4 || IS_G4X(dev_priv)) &&
6795                 adjusted_mode->crtc_hsync_start == adjusted_mode->crtc_hdisplay)
6796                 return -EINVAL;
6797
6798         intel_crtc_compute_pixel_rate(pipe_config);
6799
6800         if (pipe_config->has_pch_encoder)
6801                 return ironlake_fdi_compute_config(crtc, pipe_config);
6802
6803         return 0;
6804 }
6805
6806 static void
6807 intel_reduce_m_n_ratio(uint32_t *num, uint32_t *den)
6808 {
6809         while (*num > DATA_LINK_M_N_MASK ||
6810                *den > DATA_LINK_M_N_MASK) {
6811                 *num >>= 1;
6812                 *den >>= 1;
6813         }
6814 }
6815
6816 static void compute_m_n(unsigned int m, unsigned int n,
6817                         uint32_t *ret_m, uint32_t *ret_n,
6818                         bool constant_n)
6819 {
6820         /*
6821          * Several DP dongles in particular seem to be fussy about
6822          * too large link M/N values. Give N value as 0x8000 that
6823          * should be acceptable by specific devices. 0x8000 is the
6824          * specified fixed N value for asynchronous clock mode,
6825          * which the devices expect also in synchronous clock mode.
6826          */
6827         if (constant_n)
6828                 *ret_n = 0x8000;
6829         else
6830                 *ret_n = min_t(unsigned int, roundup_pow_of_two(n), DATA_LINK_N_MAX);
6831
6832         *ret_m = div_u64((uint64_t) m * *ret_n, n);
6833         intel_reduce_m_n_ratio(ret_m, ret_n);
6834 }
6835
6836 void
6837 intel_link_compute_m_n(int bits_per_pixel, int nlanes,
6838                        int pixel_clock, int link_clock,
6839                        struct intel_link_m_n *m_n,
6840                        bool constant_n)
6841 {
6842         m_n->tu = 64;
6843
6844         compute_m_n(bits_per_pixel * pixel_clock,
6845                     link_clock * nlanes * 8,
6846                     &m_n->gmch_m, &m_n->gmch_n,
6847                     constant_n);
6848
6849         compute_m_n(pixel_clock, link_clock,
6850                     &m_n->link_m, &m_n->link_n,
6851                     constant_n);
6852 }
6853
6854 static inline bool intel_panel_use_ssc(struct drm_i915_private *dev_priv)
6855 {
6856         if (i915_modparams.panel_use_ssc >= 0)
6857                 return i915_modparams.panel_use_ssc != 0;
6858         return dev_priv->vbt.lvds_use_ssc
6859                 && !(dev_priv->quirks & QUIRK_LVDS_SSC_DISABLE);
6860 }
6861
6862 static uint32_t pnv_dpll_compute_fp(struct dpll *dpll)
6863 {
6864         return (1 << dpll->n) << 16 | dpll->m2;
6865 }
6866
6867 static uint32_t i9xx_dpll_compute_fp(struct dpll *dpll)
6868 {
6869         return dpll->n << 16 | dpll->m1 << 8 | dpll->m2;
6870 }
6871
6872 static void i9xx_update_pll_dividers(struct intel_crtc *crtc,
6873                                      struct intel_crtc_state *crtc_state,
6874                                      struct dpll *reduced_clock)
6875 {
6876         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6877         u32 fp, fp2 = 0;
6878
6879         if (IS_PINEVIEW(dev_priv)) {
6880                 fp = pnv_dpll_compute_fp(&crtc_state->dpll);
6881                 if (reduced_clock)
6882                         fp2 = pnv_dpll_compute_fp(reduced_clock);
6883         } else {
6884                 fp = i9xx_dpll_compute_fp(&crtc_state->dpll);
6885                 if (reduced_clock)
6886                         fp2 = i9xx_dpll_compute_fp(reduced_clock);
6887         }
6888
6889         crtc_state->dpll_hw_state.fp0 = fp;
6890
6891         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS) &&
6892             reduced_clock) {
6893                 crtc_state->dpll_hw_state.fp1 = fp2;
6894         } else {
6895                 crtc_state->dpll_hw_state.fp1 = fp;
6896         }
6897 }
6898
6899 static void vlv_pllb_recal_opamp(struct drm_i915_private *dev_priv, enum pipe
6900                 pipe)
6901 {
6902         u32 reg_val;
6903
6904         /*
6905          * PLLB opamp always calibrates to max value of 0x3f, force enable it
6906          * and set it to a reasonable value instead.
6907          */
6908         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW9(1));
6909         reg_val &= 0xffffff00;
6910         reg_val |= 0x00000030;
6911         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW9(1), reg_val);
6912
6913         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_REF_DW13);
6914         reg_val &= 0x00ffffff;
6915         reg_val |= 0x8c000000;
6916         vlv_dpio_write(dev_priv, pipe, VLV_REF_DW13, reg_val);
6917
6918         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW9(1));
6919         reg_val &= 0xffffff00;
6920         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW9(1), reg_val);
6921
6922         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_REF_DW13);
6923         reg_val &= 0x00ffffff;
6924         reg_val |= 0xb0000000;
6925         vlv_dpio_write(dev_priv, pipe, VLV_REF_DW13, reg_val);
6926 }
6927
6928 static void intel_pch_transcoder_set_m_n(struct intel_crtc *crtc,
6929                                          struct intel_link_m_n *m_n)
6930 {
6931         struct drm_device *dev = crtc->base.dev;
6932         struct drm_i915_private *dev_priv = to_i915(dev);
6933         int pipe = crtc->pipe;
6934
6935         I915_WRITE(PCH_TRANS_DATA_M1(pipe), TU_SIZE(m_n->tu) | m_n->gmch_m);
6936         I915_WRITE(PCH_TRANS_DATA_N1(pipe), m_n->gmch_n);
6937         I915_WRITE(PCH_TRANS_LINK_M1(pipe), m_n->link_m);
6938         I915_WRITE(PCH_TRANS_LINK_N1(pipe), m_n->link_n);
6939 }
6940
6941 static void intel_cpu_transcoder_set_m_n(struct intel_crtc *crtc,
6942                                          struct intel_link_m_n *m_n,
6943                                          struct intel_link_m_n *m2_n2)
6944 {
6945         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6946         int pipe = crtc->pipe;
6947         enum transcoder transcoder = crtc->config->cpu_transcoder;
6948
6949         if (INTEL_GEN(dev_priv) >= 5) {
6950                 I915_WRITE(PIPE_DATA_M1(transcoder), TU_SIZE(m_n->tu) | m_n->gmch_m);
6951                 I915_WRITE(PIPE_DATA_N1(transcoder), m_n->gmch_n);
6952                 I915_WRITE(PIPE_LINK_M1(transcoder), m_n->link_m);
6953                 I915_WRITE(PIPE_LINK_N1(transcoder), m_n->link_n);
6954                 /* M2_N2 registers to be set only for gen < 8 (M2_N2 available
6955                  * for gen < 8) and if DRRS is supported (to make sure the
6956                  * registers are not unnecessarily accessed).
6957                  */
6958                 if (m2_n2 && (IS_CHERRYVIEW(dev_priv) ||
6959                     INTEL_GEN(dev_priv) < 8) && crtc->config->has_drrs) {
6960                         I915_WRITE(PIPE_DATA_M2(transcoder),
6961                                         TU_SIZE(m2_n2->tu) | m2_n2->gmch_m);
6962                         I915_WRITE(PIPE_DATA_N2(transcoder), m2_n2->gmch_n);
6963                         I915_WRITE(PIPE_LINK_M2(transcoder), m2_n2->link_m);
6964                         I915_WRITE(PIPE_LINK_N2(transcoder), m2_n2->link_n);
6965                 }
6966         } else {
6967                 I915_WRITE(PIPE_DATA_M_G4X(pipe), TU_SIZE(m_n->tu) | m_n->gmch_m);
6968                 I915_WRITE(PIPE_DATA_N_G4X(pipe), m_n->gmch_n);
6969                 I915_WRITE(PIPE_LINK_M_G4X(pipe), m_n->link_m);
6970                 I915_WRITE(PIPE_LINK_N_G4X(pipe), m_n->link_n);
6971         }
6972 }
6973
6974 void intel_dp_set_m_n(struct intel_crtc *crtc, enum link_m_n_set m_n)
6975 {
6976         struct intel_link_m_n *dp_m_n, *dp_m2_n2 = NULL;
6977
6978         if (m_n == M1_N1) {
6979                 dp_m_n = &crtc->config->dp_m_n;
6980                 dp_m2_n2 = &crtc->config->dp_m2_n2;
6981         } else if (m_n == M2_N2) {
6982
6983                 /*
6984                  * M2_N2 registers are not supported. Hence m2_n2 divider value
6985                  * needs to be programmed into M1_N1.
6986                  */
6987                 dp_m_n = &crtc->config->dp_m2_n2;
6988         } else {
6989                 DRM_ERROR("Unsupported divider value\n");
6990                 return;
6991         }
6992
6993         if (crtc->config->has_pch_encoder)
6994                 intel_pch_transcoder_set_m_n(crtc, &crtc->config->dp_m_n);
6995         else
6996                 intel_cpu_transcoder_set_m_n(crtc, dp_m_n, dp_m2_n2);
6997 }
6998
6999 static void vlv_compute_dpll(struct intel_crtc *crtc,
7000                              struct intel_crtc_state *pipe_config)
7001 {
7002         pipe_config->dpll_hw_state.dpll = DPLL_INTEGRATED_REF_CLK_VLV |
7003                 DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
7004         if (crtc->pipe != PIPE_A)
7005                 pipe_config->dpll_hw_state.dpll |= DPLL_INTEGRATED_CRI_CLK_VLV;
7006
7007         /* DPLL not used with DSI, but still need the rest set up */
7008         if (!intel_crtc_has_type(pipe_config, INTEL_OUTPUT_DSI))
7009                 pipe_config->dpll_hw_state.dpll |= DPLL_VCO_ENABLE |
7010                         DPLL_EXT_BUFFER_ENABLE_VLV;
7011
7012         pipe_config->dpll_hw_state.dpll_md =
7013                 (pipe_config->pixel_multiplier - 1) << DPLL_MD_UDI_MULTIPLIER_SHIFT;
7014 }
7015
7016 static void chv_compute_dpll(struct intel_crtc *crtc,
7017                              struct intel_crtc_state *pipe_config)
7018 {
7019         pipe_config->dpll_hw_state.dpll = DPLL_SSC_REF_CLK_CHV |
7020                 DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
7021         if (crtc->pipe != PIPE_A)
7022                 pipe_config->dpll_hw_state.dpll |= DPLL_INTEGRATED_CRI_CLK_VLV;
7023
7024         /* DPLL not used with DSI, but still need the rest set up */
7025         if (!intel_crtc_has_type(pipe_config, INTEL_OUTPUT_DSI))
7026                 pipe_config->dpll_hw_state.dpll |= DPLL_VCO_ENABLE;
7027
7028         pipe_config->dpll_hw_state.dpll_md =
7029                 (pipe_config->pixel_multiplier - 1) << DPLL_MD_UDI_MULTIPLIER_SHIFT;
7030 }
7031
7032 static void vlv_prepare_pll(struct intel_crtc *crtc,
7033                             const struct intel_crtc_state *pipe_config)
7034 {
7035         struct drm_device *dev = crtc->base.dev;
7036         struct drm_i915_private *dev_priv = to_i915(dev);
7037         enum pipe pipe = crtc->pipe;
7038         u32 mdiv;
7039         u32 bestn, bestm1, bestm2, bestp1, bestp2;
7040         u32 coreclk, reg_val;
7041
7042         /* Enable Refclk */
7043         I915_WRITE(DPLL(pipe),
7044                    pipe_config->dpll_hw_state.dpll &
7045                    ~(DPLL_VCO_ENABLE | DPLL_EXT_BUFFER_ENABLE_VLV));
7046
7047         /* No need to actually set up the DPLL with DSI */
7048         if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0)
7049                 return;
7050
7051         mutex_lock(&dev_priv->sb_lock);
7052
7053         bestn = pipe_config->dpll.n;
7054         bestm1 = pipe_config->dpll.m1;
7055         bestm2 = pipe_config->dpll.m2;
7056         bestp1 = pipe_config->dpll.p1;
7057         bestp2 = pipe_config->dpll.p2;
7058
7059         /* See eDP HDMI DPIO driver vbios notes doc */
7060
7061         /* PLL B needs special handling */
7062         if (pipe == PIPE_B)
7063                 vlv_pllb_recal_opamp(dev_priv, pipe);
7064
7065         /* Set up Tx target for periodic Rcomp update */
7066         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW9_BCAST, 0x0100000f);
7067
7068         /* Disable target IRef on PLL */
7069         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW8(pipe));
7070         reg_val &= 0x00ffffff;
7071         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW8(pipe), reg_val);
7072
7073         /* Disable fast lock */
7074         vlv_dpio_write(dev_priv, pipe, VLV_CMN_DW0, 0x610);
7075
7076         /* Set idtafcrecal before PLL is enabled */
7077         mdiv = ((bestm1 << DPIO_M1DIV_SHIFT) | (bestm2 & DPIO_M2DIV_MASK));
7078         mdiv |= ((bestp1 << DPIO_P1_SHIFT) | (bestp2 << DPIO_P2_SHIFT));
7079         mdiv |= ((bestn << DPIO_N_SHIFT));
7080         mdiv |= (1 << DPIO_K_SHIFT);
7081
7082         /*
7083          * Post divider depends on pixel clock rate, DAC vs digital (and LVDS,
7084          * but we don't support that).
7085          * Note: don't use the DAC post divider as it seems unstable.
7086          */
7087         mdiv |= (DPIO_POST_DIV_HDMIDP << DPIO_POST_DIV_SHIFT);
7088         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW3(pipe), mdiv);
7089
7090         mdiv |= DPIO_ENABLE_CALIBRATION;
7091         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW3(pipe), mdiv);
7092
7093         /* Set HBR and RBR LPF coefficients */
7094         if (pipe_config->port_clock == 162000 ||
7095             intel_crtc_has_type(crtc->config, INTEL_OUTPUT_ANALOG) ||
7096             intel_crtc_has_type(crtc->config, INTEL_OUTPUT_HDMI))
7097                 vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW10(pipe),
7098                                  0x009f0003);
7099         else
7100                 vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW10(pipe),
7101                                  0x00d0000f);
7102
7103         if (intel_crtc_has_dp_encoder(pipe_config)) {
7104                 /* Use SSC source */
7105                 if (pipe == PIPE_A)
7106                         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
7107                                          0x0df40000);
7108                 else
7109                         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
7110                                          0x0df70000);
7111         } else { /* HDMI or VGA */
7112                 /* Use bend source */
7113                 if (pipe == PIPE_A)
7114                         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
7115                                          0x0df70000);
7116                 else
7117                         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
7118                                          0x0df40000);
7119         }
7120
7121         coreclk = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW7(pipe));
7122         coreclk = (coreclk & 0x0000ff00) | 0x01c00000;
7123         if (intel_crtc_has_dp_encoder(crtc->config))
7124                 coreclk |= 0x01000000;
7125         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW7(pipe), coreclk);
7126
7127         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW11(pipe), 0x87871000);
7128         mutex_unlock(&dev_priv->sb_lock);
7129 }
7130
7131 static void chv_prepare_pll(struct intel_crtc *crtc,
7132                             const struct intel_crtc_state *pipe_config)
7133 {
7134         struct drm_device *dev = crtc->base.dev;
7135         struct drm_i915_private *dev_priv = to_i915(dev);
7136         enum pipe pipe = crtc->pipe;
7137         enum dpio_channel port = vlv_pipe_to_channel(pipe);
7138         u32 loopfilter, tribuf_calcntr;
7139         u32 bestn, bestm1, bestm2, bestp1, bestp2, bestm2_frac;
7140         u32 dpio_val;
7141         int vco;
7142
7143         /* Enable Refclk and SSC */
7144         I915_WRITE(DPLL(pipe),
7145                    pipe_config->dpll_hw_state.dpll & ~DPLL_VCO_ENABLE);
7146
7147         /* No need to actually set up the DPLL with DSI */
7148         if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0)
7149                 return;
7150
7151         bestn = pipe_config->dpll.n;
7152         bestm2_frac = pipe_config->dpll.m2 & 0x3fffff;
7153         bestm1 = pipe_config->dpll.m1;
7154         bestm2 = pipe_config->dpll.m2 >> 22;
7155         bestp1 = pipe_config->dpll.p1;
7156         bestp2 = pipe_config->dpll.p2;
7157         vco = pipe_config->dpll.vco;
7158         dpio_val = 0;
7159         loopfilter = 0;
7160
7161         mutex_lock(&dev_priv->sb_lock);
7162
7163         /* p1 and p2 divider */
7164         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW13(port),
7165                         5 << DPIO_CHV_S1_DIV_SHIFT |
7166                         bestp1 << DPIO_CHV_P1_DIV_SHIFT |
7167                         bestp2 << DPIO_CHV_P2_DIV_SHIFT |
7168                         1 << DPIO_CHV_K_DIV_SHIFT);
7169
7170         /* Feedback post-divider - m2 */
7171         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW0(port), bestm2);
7172
7173         /* Feedback refclk divider - n and m1 */
7174         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW1(port),
7175                         DPIO_CHV_M1_DIV_BY_2 |
7176                         1 << DPIO_CHV_N_DIV_SHIFT);
7177
7178         /* M2 fraction division */
7179         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW2(port), bestm2_frac);
7180
7181         /* M2 fraction division enable */
7182         dpio_val = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW3(port));
7183         dpio_val &= ~(DPIO_CHV_FEEDFWD_GAIN_MASK | DPIO_CHV_FRAC_DIV_EN);
7184         dpio_val |= (2 << DPIO_CHV_FEEDFWD_GAIN_SHIFT);
7185         if (bestm2_frac)
7186                 dpio_val |= DPIO_CHV_FRAC_DIV_EN;
7187         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW3(port), dpio_val);
7188
7189         /* Program digital lock detect threshold */
7190         dpio_val = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW9(port));
7191         dpio_val &= ~(DPIO_CHV_INT_LOCK_THRESHOLD_MASK |
7192                                         DPIO_CHV_INT_LOCK_THRESHOLD_SEL_COARSE);
7193         dpio_val |= (0x5 << DPIO_CHV_INT_LOCK_THRESHOLD_SHIFT);
7194         if (!bestm2_frac)
7195                 dpio_val |= DPIO_CHV_INT_LOCK_THRESHOLD_SEL_COARSE;
7196         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW9(port), dpio_val);
7197
7198         /* Loop filter */
7199         if (vco == 5400000) {
7200                 loopfilter |= (0x3 << DPIO_CHV_PROP_COEFF_SHIFT);
7201                 loopfilter |= (0x8 << DPIO_CHV_INT_COEFF_SHIFT);
7202                 loopfilter |= (0x1 << DPIO_CHV_GAIN_CTRL_SHIFT);
7203                 tribuf_calcntr = 0x9;
7204         } else if (vco <= 6200000) {
7205                 loopfilter |= (0x5 << DPIO_CHV_PROP_COEFF_SHIFT);
7206                 loopfilter |= (0xB << DPIO_CHV_INT_COEFF_SHIFT);
7207                 loopfilter |= (0x3 << DPIO_CHV_GAIN_CTRL_SHIFT);
7208                 tribuf_calcntr = 0x9;
7209         } else if (vco <= 6480000) {
7210                 loopfilter |= (0x4 << DPIO_CHV_PROP_COEFF_SHIFT);
7211                 loopfilter |= (0x9 << DPIO_CHV_INT_COEFF_SHIFT);
7212                 loopfilter |= (0x3 << DPIO_CHV_GAIN_CTRL_SHIFT);
7213                 tribuf_calcntr = 0x8;
7214         } else {
7215                 /* Not supported. Apply the same limits as in the max case */
7216                 loopfilter |= (0x4 << DPIO_CHV_PROP_COEFF_SHIFT);
7217                 loopfilter |= (0x9 << DPIO_CHV_INT_COEFF_SHIFT);
7218                 loopfilter |= (0x3 << DPIO_CHV_GAIN_CTRL_SHIFT);
7219                 tribuf_calcntr = 0;
7220         }
7221         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW6(port), loopfilter);
7222
7223         dpio_val = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW8(port));
7224         dpio_val &= ~DPIO_CHV_TDC_TARGET_CNT_MASK;
7225         dpio_val |= (tribuf_calcntr << DPIO_CHV_TDC_TARGET_CNT_SHIFT);
7226         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW8(port), dpio_val);
7227
7228         /* AFC Recal */
7229         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW14(port),
7230                         vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW14(port)) |
7231                         DPIO_AFC_RECAL);
7232
7233         mutex_unlock(&dev_priv->sb_lock);
7234 }
7235
7236 /**
7237  * vlv_force_pll_on - forcibly enable just the PLL
7238  * @dev_priv: i915 private structure
7239  * @pipe: pipe PLL to enable
7240  * @dpll: PLL configuration
7241  *
7242  * Enable the PLL for @pipe using the supplied @dpll config. To be used
7243  * in cases where we need the PLL enabled even when @pipe is not going to
7244  * be enabled.
7245  */
7246 int vlv_force_pll_on(struct drm_i915_private *dev_priv, enum pipe pipe,
7247                      const struct dpll *dpll)
7248 {
7249         struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
7250         struct intel_crtc_state *pipe_config;
7251
7252         pipe_config = kzalloc(sizeof(*pipe_config), GFP_KERNEL);
7253         if (!pipe_config)
7254                 return -ENOMEM;
7255
7256         pipe_config->base.crtc = &crtc->base;
7257         pipe_config->pixel_multiplier = 1;
7258         pipe_config->dpll = *dpll;
7259
7260         if (IS_CHERRYVIEW(dev_priv)) {
7261                 chv_compute_dpll(crtc, pipe_config);
7262                 chv_prepare_pll(crtc, pipe_config);
7263                 chv_enable_pll(crtc, pipe_config);
7264         } else {
7265                 vlv_compute_dpll(crtc, pipe_config);
7266                 vlv_prepare_pll(crtc, pipe_config);
7267                 vlv_enable_pll(crtc, pipe_config);
7268         }
7269
7270         kfree(pipe_config);
7271
7272         return 0;
7273 }
7274
7275 /**
7276  * vlv_force_pll_off - forcibly disable just the PLL
7277  * @dev_priv: i915 private structure
7278  * @pipe: pipe PLL to disable
7279  *
7280  * Disable the PLL for @pipe. To be used in cases where we need
7281  * the PLL enabled even when @pipe is not going to be enabled.
7282  */
7283 void vlv_force_pll_off(struct drm_i915_private *dev_priv, enum pipe pipe)
7284 {
7285         if (IS_CHERRYVIEW(dev_priv))
7286                 chv_disable_pll(dev_priv, pipe);
7287         else
7288                 vlv_disable_pll(dev_priv, pipe);
7289 }
7290
7291 static void i9xx_compute_dpll(struct intel_crtc *crtc,
7292                               struct intel_crtc_state *crtc_state,
7293                               struct dpll *reduced_clock)
7294 {
7295         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7296         u32 dpll;
7297         struct dpll *clock = &crtc_state->dpll;
7298
7299         i9xx_update_pll_dividers(crtc, crtc_state, reduced_clock);
7300
7301         dpll = DPLL_VGA_MODE_DIS;
7302
7303         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS))
7304                 dpll |= DPLLB_MODE_LVDS;
7305         else
7306                 dpll |= DPLLB_MODE_DAC_SERIAL;
7307
7308         if (IS_I945G(dev_priv) || IS_I945GM(dev_priv) ||
7309             IS_G33(dev_priv) || IS_PINEVIEW(dev_priv)) {
7310                 dpll |= (crtc_state->pixel_multiplier - 1)
7311                         << SDVO_MULTIPLIER_SHIFT_HIRES;
7312         }
7313
7314         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO) ||
7315             intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI))
7316                 dpll |= DPLL_SDVO_HIGH_SPEED;
7317
7318         if (intel_crtc_has_dp_encoder(crtc_state))
7319                 dpll |= DPLL_SDVO_HIGH_SPEED;
7320
7321         /* compute bitmask from p1 value */
7322         if (IS_PINEVIEW(dev_priv))
7323                 dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT_PINEVIEW;
7324         else {
7325                 dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
7326                 if (IS_G4X(dev_priv) && reduced_clock)
7327                         dpll |= (1 << (reduced_clock->p1 - 1)) << DPLL_FPA1_P1_POST_DIV_SHIFT;
7328         }
7329         switch (clock->p2) {
7330         case 5:
7331                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_5;
7332                 break;
7333         case 7:
7334                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_7;
7335                 break;
7336         case 10:
7337                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_10;
7338                 break;
7339         case 14:
7340                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_14;
7341                 break;
7342         }
7343         if (INTEL_GEN(dev_priv) >= 4)
7344                 dpll |= (6 << PLL_LOAD_PULSE_PHASE_SHIFT);
7345
7346         if (crtc_state->sdvo_tv_clock)
7347                 dpll |= PLL_REF_INPUT_TVCLKINBC;
7348         else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS) &&
7349                  intel_panel_use_ssc(dev_priv))
7350                 dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN;
7351         else
7352                 dpll |= PLL_REF_INPUT_DREFCLK;
7353
7354         dpll |= DPLL_VCO_ENABLE;
7355         crtc_state->dpll_hw_state.dpll = dpll;
7356
7357         if (INTEL_GEN(dev_priv) >= 4) {
7358                 u32 dpll_md = (crtc_state->pixel_multiplier - 1)
7359                         << DPLL_MD_UDI_MULTIPLIER_SHIFT;
7360                 crtc_state->dpll_hw_state.dpll_md = dpll_md;
7361         }
7362 }
7363
7364 static void i8xx_compute_dpll(struct intel_crtc *crtc,
7365                               struct intel_crtc_state *crtc_state,
7366                               struct dpll *reduced_clock)
7367 {
7368         struct drm_device *dev = crtc->base.dev;
7369         struct drm_i915_private *dev_priv = to_i915(dev);
7370         u32 dpll;
7371         struct dpll *clock = &crtc_state->dpll;
7372
7373         i9xx_update_pll_dividers(crtc, crtc_state, reduced_clock);
7374
7375         dpll = DPLL_VGA_MODE_DIS;
7376
7377         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
7378                 dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
7379         } else {
7380                 if (clock->p1 == 2)
7381                         dpll |= PLL_P1_DIVIDE_BY_TWO;
7382                 else
7383                         dpll |= (clock->p1 - 2) << DPLL_FPA01_P1_POST_DIV_SHIFT;
7384                 if (clock->p2 == 4)
7385                         dpll |= PLL_P2_DIVIDE_BY_4;
7386         }
7387
7388         if (!IS_I830(dev_priv) &&
7389             intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DVO))
7390                 dpll |= DPLL_DVO_2X_MODE;
7391
7392         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS) &&
7393             intel_panel_use_ssc(dev_priv))
7394                 dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN;
7395         else
7396                 dpll |= PLL_REF_INPUT_DREFCLK;
7397
7398         dpll |= DPLL_VCO_ENABLE;
7399         crtc_state->dpll_hw_state.dpll = dpll;
7400 }
7401
7402 static void intel_set_pipe_timings(struct intel_crtc *intel_crtc)
7403 {
7404         struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
7405         enum pipe pipe = intel_crtc->pipe;
7406         enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder;
7407         const struct drm_display_mode *adjusted_mode = &intel_crtc->config->base.adjusted_mode;
7408         uint32_t crtc_vtotal, crtc_vblank_end;
7409         int vsyncshift = 0;
7410
7411         /* We need to be careful not to changed the adjusted mode, for otherwise
7412          * the hw state checker will get angry at the mismatch. */
7413         crtc_vtotal = adjusted_mode->crtc_vtotal;
7414         crtc_vblank_end = adjusted_mode->crtc_vblank_end;
7415
7416         if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) {
7417                 /* the chip adds 2 halflines automatically */
7418                 crtc_vtotal -= 1;
7419                 crtc_vblank_end -= 1;
7420
7421                 if (intel_crtc_has_type(intel_crtc->config, INTEL_OUTPUT_SDVO))
7422                         vsyncshift = (adjusted_mode->crtc_htotal - 1) / 2;
7423                 else
7424                         vsyncshift = adjusted_mode->crtc_hsync_start -
7425                                 adjusted_mode->crtc_htotal / 2;
7426                 if (vsyncshift < 0)
7427                         vsyncshift += adjusted_mode->crtc_htotal;
7428         }
7429
7430         if (INTEL_GEN(dev_priv) > 3)
7431                 I915_WRITE(VSYNCSHIFT(cpu_transcoder), vsyncshift);
7432
7433         I915_WRITE(HTOTAL(cpu_transcoder),
7434                    (adjusted_mode->crtc_hdisplay - 1) |
7435                    ((adjusted_mode->crtc_htotal - 1) << 16));
7436         I915_WRITE(HBLANK(cpu_transcoder),
7437                    (adjusted_mode->crtc_hblank_start - 1) |
7438                    ((adjusted_mode->crtc_hblank_end - 1) << 16));
7439         I915_WRITE(HSYNC(cpu_transcoder),
7440                    (adjusted_mode->crtc_hsync_start - 1) |
7441                    ((adjusted_mode->crtc_hsync_end - 1) << 16));
7442
7443         I915_WRITE(VTOTAL(cpu_transcoder),
7444                    (adjusted_mode->crtc_vdisplay - 1) |
7445                    ((crtc_vtotal - 1) << 16));
7446         I915_WRITE(VBLANK(cpu_transcoder),
7447                    (adjusted_mode->crtc_vblank_start - 1) |
7448                    ((crtc_vblank_end - 1) << 16));
7449         I915_WRITE(VSYNC(cpu_transcoder),
7450                    (adjusted_mode->crtc_vsync_start - 1) |
7451                    ((adjusted_mode->crtc_vsync_end - 1) << 16));
7452
7453         /* Workaround: when the EDP input selection is B, the VTOTAL_B must be
7454          * programmed with the VTOTAL_EDP value. Same for VTOTAL_C. This is
7455          * documented on the DDI_FUNC_CTL register description, EDP Input Select
7456          * bits. */
7457         if (IS_HASWELL(dev_priv) && cpu_transcoder == TRANSCODER_EDP &&
7458             (pipe == PIPE_B || pipe == PIPE_C))
7459                 I915_WRITE(VTOTAL(pipe), I915_READ(VTOTAL(cpu_transcoder)));
7460
7461 }
7462
7463 static void intel_set_pipe_src_size(struct intel_crtc *intel_crtc)
7464 {
7465         struct drm_device *dev = intel_crtc->base.dev;
7466         struct drm_i915_private *dev_priv = to_i915(dev);
7467         enum pipe pipe = intel_crtc->pipe;
7468
7469         /* pipesrc controls the size that is scaled from, which should
7470          * always be the user's requested size.
7471          */
7472         I915_WRITE(PIPESRC(pipe),
7473                    ((intel_crtc->config->pipe_src_w - 1) << 16) |
7474                    (intel_crtc->config->pipe_src_h - 1));
7475 }
7476
7477 static void intel_get_pipe_timings(struct intel_crtc *crtc,
7478                                    struct intel_crtc_state *pipe_config)
7479 {
7480         struct drm_device *dev = crtc->base.dev;
7481         struct drm_i915_private *dev_priv = to_i915(dev);
7482         enum transcoder cpu_transcoder = pipe_config->cpu_transcoder;
7483         uint32_t tmp;
7484
7485         tmp = I915_READ(HTOTAL(cpu_transcoder));
7486         pipe_config->base.adjusted_mode.crtc_hdisplay = (tmp & 0xffff) + 1;
7487         pipe_config->base.adjusted_mode.crtc_htotal = ((tmp >> 16) & 0xffff) + 1;
7488         tmp = I915_READ(HBLANK(cpu_transcoder));
7489         pipe_config->base.adjusted_mode.crtc_hblank_start = (tmp & 0xffff) + 1;
7490         pipe_config->base.adjusted_mode.crtc_hblank_end = ((tmp >> 16) & 0xffff) + 1;
7491         tmp = I915_READ(HSYNC(cpu_transcoder));
7492         pipe_config->base.adjusted_mode.crtc_hsync_start = (tmp & 0xffff) + 1;
7493         pipe_config->base.adjusted_mode.crtc_hsync_end = ((tmp >> 16) & 0xffff) + 1;
7494
7495         tmp = I915_READ(VTOTAL(cpu_transcoder));
7496         pipe_config->base.adjusted_mode.crtc_vdisplay = (tmp & 0xffff) + 1;
7497         pipe_config->base.adjusted_mode.crtc_vtotal = ((tmp >> 16) & 0xffff) + 1;
7498         tmp = I915_READ(VBLANK(cpu_transcoder));
7499         pipe_config->base.adjusted_mode.crtc_vblank_start = (tmp & 0xffff) + 1;
7500         pipe_config->base.adjusted_mode.crtc_vblank_end = ((tmp >> 16) & 0xffff) + 1;
7501         tmp = I915_READ(VSYNC(cpu_transcoder));
7502         pipe_config->base.adjusted_mode.crtc_vsync_start = (tmp & 0xffff) + 1;
7503         pipe_config->base.adjusted_mode.crtc_vsync_end = ((tmp >> 16) & 0xffff) + 1;
7504
7505         if (I915_READ(PIPECONF(cpu_transcoder)) & PIPECONF_INTERLACE_MASK) {
7506                 pipe_config->base.adjusted_mode.flags |= DRM_MODE_FLAG_INTERLACE;
7507                 pipe_config->base.adjusted_mode.crtc_vtotal += 1;
7508                 pipe_config->base.adjusted_mode.crtc_vblank_end += 1;
7509         }
7510 }
7511
7512 static void intel_get_pipe_src_size(struct intel_crtc *crtc,
7513                                     struct intel_crtc_state *pipe_config)
7514 {
7515         struct drm_device *dev = crtc->base.dev;
7516         struct drm_i915_private *dev_priv = to_i915(dev);
7517         u32 tmp;
7518
7519         tmp = I915_READ(PIPESRC(crtc->pipe));
7520         pipe_config->pipe_src_h = (tmp & 0xffff) + 1;
7521         pipe_config->pipe_src_w = ((tmp >> 16) & 0xffff) + 1;
7522
7523         pipe_config->base.mode.vdisplay = pipe_config->pipe_src_h;
7524         pipe_config->base.mode.hdisplay = pipe_config->pipe_src_w;
7525 }
7526
7527 void intel_mode_from_pipe_config(struct drm_display_mode *mode,
7528                                  struct intel_crtc_state *pipe_config)
7529 {
7530         mode->hdisplay = pipe_config->base.adjusted_mode.crtc_hdisplay;
7531         mode->htotal = pipe_config->base.adjusted_mode.crtc_htotal;
7532         mode->hsync_start = pipe_config->base.adjusted_mode.crtc_hsync_start;
7533         mode->hsync_end = pipe_config->base.adjusted_mode.crtc_hsync_end;
7534
7535         mode->vdisplay = pipe_config->base.adjusted_mode.crtc_vdisplay;
7536         mode->vtotal = pipe_config->base.adjusted_mode.crtc_vtotal;
7537         mode->vsync_start = pipe_config->base.adjusted_mode.crtc_vsync_start;
7538         mode->vsync_end = pipe_config->base.adjusted_mode.crtc_vsync_end;
7539
7540         mode->flags = pipe_config->base.adjusted_mode.flags;
7541         mode->type = DRM_MODE_TYPE_DRIVER;
7542
7543         mode->clock = pipe_config->base.adjusted_mode.crtc_clock;
7544
7545         mode->hsync = drm_mode_hsync(mode);
7546         mode->vrefresh = drm_mode_vrefresh(mode);
7547         drm_mode_set_name(mode);
7548 }
7549
7550 static void i9xx_set_pipeconf(struct intel_crtc *intel_crtc)
7551 {
7552         struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
7553         uint32_t pipeconf;
7554
7555         pipeconf = 0;
7556
7557         /* we keep both pipes enabled on 830 */
7558         if (IS_I830(dev_priv))
7559                 pipeconf |= I915_READ(PIPECONF(intel_crtc->pipe)) & PIPECONF_ENABLE;
7560
7561         if (intel_crtc->config->double_wide)
7562                 pipeconf |= PIPECONF_DOUBLE_WIDE;
7563
7564         /* only g4x and later have fancy bpc/dither controls */
7565         if (IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
7566             IS_CHERRYVIEW(dev_priv)) {
7567                 /* Bspec claims that we can't use dithering for 30bpp pipes. */
7568                 if (intel_crtc->config->dither && intel_crtc->config->pipe_bpp != 30)
7569                         pipeconf |= PIPECONF_DITHER_EN |
7570                                     PIPECONF_DITHER_TYPE_SP;
7571
7572                 switch (intel_crtc->config->pipe_bpp) {
7573                 case 18:
7574                         pipeconf |= PIPECONF_6BPC;
7575                         break;
7576                 case 24:
7577                         pipeconf |= PIPECONF_8BPC;
7578                         break;
7579                 case 30:
7580                         pipeconf |= PIPECONF_10BPC;
7581                         break;
7582                 default:
7583                         /* Case prevented by intel_choose_pipe_bpp_dither. */
7584                         BUG();
7585                 }
7586         }
7587
7588         if (intel_crtc->config->base.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE) {
7589                 if (INTEL_GEN(dev_priv) < 4 ||
7590                     intel_crtc_has_type(intel_crtc->config, INTEL_OUTPUT_SDVO))
7591                         pipeconf |= PIPECONF_INTERLACE_W_FIELD_INDICATION;
7592                 else
7593                         pipeconf |= PIPECONF_INTERLACE_W_SYNC_SHIFT;
7594         } else
7595                 pipeconf |= PIPECONF_PROGRESSIVE;
7596
7597         if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
7598              intel_crtc->config->limited_color_range)
7599                 pipeconf |= PIPECONF_COLOR_RANGE_SELECT;
7600
7601         I915_WRITE(PIPECONF(intel_crtc->pipe), pipeconf);
7602         POSTING_READ(PIPECONF(intel_crtc->pipe));
7603 }
7604
7605 static int i8xx_crtc_compute_clock(struct intel_crtc *crtc,
7606                                    struct intel_crtc_state *crtc_state)
7607 {
7608         struct drm_device *dev = crtc->base.dev;
7609         struct drm_i915_private *dev_priv = to_i915(dev);
7610         const struct intel_limit *limit;
7611         int refclk = 48000;
7612
7613         memset(&crtc_state->dpll_hw_state, 0,
7614                sizeof(crtc_state->dpll_hw_state));
7615
7616         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
7617                 if (intel_panel_use_ssc(dev_priv)) {
7618                         refclk = dev_priv->vbt.lvds_ssc_freq;
7619                         DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n", refclk);
7620                 }
7621
7622                 limit = &intel_limits_i8xx_lvds;
7623         } else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DVO)) {
7624                 limit = &intel_limits_i8xx_dvo;
7625         } else {
7626                 limit = &intel_limits_i8xx_dac;
7627         }
7628
7629         if (!crtc_state->clock_set &&
7630             !i9xx_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
7631                                  refclk, NULL, &crtc_state->dpll)) {
7632                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
7633                 return -EINVAL;
7634         }
7635
7636         i8xx_compute_dpll(crtc, crtc_state, NULL);
7637
7638         return 0;
7639 }
7640
7641 static int g4x_crtc_compute_clock(struct intel_crtc *crtc,
7642                                   struct intel_crtc_state *crtc_state)
7643 {
7644         struct drm_device *dev = crtc->base.dev;
7645         struct drm_i915_private *dev_priv = to_i915(dev);
7646         const struct intel_limit *limit;
7647         int refclk = 96000;
7648
7649         memset(&crtc_state->dpll_hw_state, 0,
7650                sizeof(crtc_state->dpll_hw_state));
7651
7652         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
7653                 if (intel_panel_use_ssc(dev_priv)) {
7654                         refclk = dev_priv->vbt.lvds_ssc_freq;
7655                         DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n", refclk);
7656                 }
7657
7658                 if (intel_is_dual_link_lvds(dev))
7659                         limit = &intel_limits_g4x_dual_channel_lvds;
7660                 else
7661                         limit = &intel_limits_g4x_single_channel_lvds;
7662         } else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI) ||
7663                    intel_crtc_has_type(crtc_state, INTEL_OUTPUT_ANALOG)) {
7664                 limit = &intel_limits_g4x_hdmi;
7665         } else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO)) {
7666                 limit = &intel_limits_g4x_sdvo;
7667         } else {
7668                 /* The option is for other outputs */
7669                 limit = &intel_limits_i9xx_sdvo;
7670         }
7671
7672         if (!crtc_state->clock_set &&
7673             !g4x_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
7674                                 refclk, NULL, &crtc_state->dpll)) {
7675                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
7676                 return -EINVAL;
7677         }
7678
7679         i9xx_compute_dpll(crtc, crtc_state, NULL);
7680
7681         return 0;
7682 }
7683
7684 static int pnv_crtc_compute_clock(struct intel_crtc *crtc,
7685                                   struct intel_crtc_state *crtc_state)
7686 {
7687         struct drm_device *dev = crtc->base.dev;
7688         struct drm_i915_private *dev_priv = to_i915(dev);
7689         const struct intel_limit *limit;
7690         int refclk = 96000;
7691
7692         memset(&crtc_state->dpll_hw_state, 0,
7693                sizeof(crtc_state->dpll_hw_state));
7694
7695         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
7696                 if (intel_panel_use_ssc(dev_priv)) {
7697                         refclk = dev_priv->vbt.lvds_ssc_freq;
7698                         DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n", refclk);
7699                 }
7700
7701                 limit = &intel_limits_pineview_lvds;
7702         } else {
7703                 limit = &intel_limits_pineview_sdvo;
7704         }
7705
7706         if (!crtc_state->clock_set &&
7707             !pnv_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
7708                                 refclk, NULL, &crtc_state->dpll)) {
7709                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
7710                 return -EINVAL;
7711         }
7712
7713         i9xx_compute_dpll(crtc, crtc_state, NULL);
7714
7715         return 0;
7716 }
7717
7718 static int i9xx_crtc_compute_clock(struct intel_crtc *crtc,
7719                                    struct intel_crtc_state *crtc_state)
7720 {
7721         struct drm_device *dev = crtc->base.dev;
7722         struct drm_i915_private *dev_priv = to_i915(dev);
7723         const struct intel_limit *limit;
7724         int refclk = 96000;
7725
7726         memset(&crtc_state->dpll_hw_state, 0,
7727                sizeof(crtc_state->dpll_hw_state));
7728
7729         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
7730                 if (intel_panel_use_ssc(dev_priv)) {
7731                         refclk = dev_priv->vbt.lvds_ssc_freq;
7732                         DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n", refclk);
7733                 }
7734
7735                 limit = &intel_limits_i9xx_lvds;
7736         } else {
7737                 limit = &intel_limits_i9xx_sdvo;
7738         }
7739
7740         if (!crtc_state->clock_set &&
7741             !i9xx_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
7742                                  refclk, NULL, &crtc_state->dpll)) {
7743                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
7744                 return -EINVAL;
7745         }
7746
7747         i9xx_compute_dpll(crtc, crtc_state, NULL);
7748
7749         return 0;
7750 }
7751
7752 static int chv_crtc_compute_clock(struct intel_crtc *crtc,
7753                                   struct intel_crtc_state *crtc_state)
7754 {
7755         int refclk = 100000;
7756         const struct intel_limit *limit = &intel_limits_chv;
7757
7758         memset(&crtc_state->dpll_hw_state, 0,
7759                sizeof(crtc_state->dpll_hw_state));
7760
7761         if (!crtc_state->clock_set &&
7762             !chv_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
7763                                 refclk, NULL, &crtc_state->dpll)) {
7764                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
7765                 return -EINVAL;
7766         }
7767
7768         chv_compute_dpll(crtc, crtc_state);
7769
7770         return 0;
7771 }
7772
7773 static int vlv_crtc_compute_clock(struct intel_crtc *crtc,
7774                                   struct intel_crtc_state *crtc_state)
7775 {
7776         int refclk = 100000;
7777         const struct intel_limit *limit = &intel_limits_vlv;
7778
7779         memset(&crtc_state->dpll_hw_state, 0,
7780                sizeof(crtc_state->dpll_hw_state));
7781
7782         if (!crtc_state->clock_set &&
7783             !vlv_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
7784                                 refclk, NULL, &crtc_state->dpll)) {
7785                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
7786                 return -EINVAL;
7787         }
7788
7789         vlv_compute_dpll(crtc, crtc_state);
7790
7791         return 0;
7792 }
7793
7794 static void i9xx_get_pfit_config(struct intel_crtc *crtc,
7795                                  struct intel_crtc_state *pipe_config)
7796 {
7797         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7798         uint32_t tmp;
7799
7800         if (INTEL_GEN(dev_priv) <= 3 &&
7801             (IS_I830(dev_priv) || !IS_MOBILE(dev_priv)))
7802                 return;
7803
7804         tmp = I915_READ(PFIT_CONTROL);
7805         if (!(tmp & PFIT_ENABLE))
7806                 return;
7807
7808         /* Check whether the pfit is attached to our pipe. */
7809         if (INTEL_GEN(dev_priv) < 4) {
7810                 if (crtc->pipe != PIPE_B)
7811                         return;
7812         } else {
7813                 if ((tmp & PFIT_PIPE_MASK) != (crtc->pipe << PFIT_PIPE_SHIFT))
7814                         return;
7815         }
7816
7817         pipe_config->gmch_pfit.control = tmp;
7818         pipe_config->gmch_pfit.pgm_ratios = I915_READ(PFIT_PGM_RATIOS);
7819 }
7820
7821 static void vlv_crtc_clock_get(struct intel_crtc *crtc,
7822                                struct intel_crtc_state *pipe_config)
7823 {
7824         struct drm_device *dev = crtc->base.dev;
7825         struct drm_i915_private *dev_priv = to_i915(dev);
7826         int pipe = pipe_config->cpu_transcoder;
7827         struct dpll clock;
7828         u32 mdiv;
7829         int refclk = 100000;
7830
7831         /* In case of DSI, DPLL will not be used */
7832         if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0)
7833                 return;
7834
7835         mutex_lock(&dev_priv->sb_lock);
7836         mdiv = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW3(pipe));
7837         mutex_unlock(&dev_priv->sb_lock);
7838
7839         clock.m1 = (mdiv >> DPIO_M1DIV_SHIFT) & 7;
7840         clock.m2 = mdiv & DPIO_M2DIV_MASK;
7841         clock.n = (mdiv >> DPIO_N_SHIFT) & 0xf;
7842         clock.p1 = (mdiv >> DPIO_P1_SHIFT) & 7;
7843         clock.p2 = (mdiv >> DPIO_P2_SHIFT) & 0x1f;
7844
7845         pipe_config->port_clock = vlv_calc_dpll_params(refclk, &clock);
7846 }
7847
7848 static void
7849 i9xx_get_initial_plane_config(struct intel_crtc *crtc,
7850                               struct intel_initial_plane_config *plane_config)
7851 {
7852         struct drm_device *dev = crtc->base.dev;
7853         struct drm_i915_private *dev_priv = to_i915(dev);
7854         struct intel_plane *plane = to_intel_plane(crtc->base.primary);
7855         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
7856         enum pipe pipe;
7857         u32 val, base, offset;
7858         int fourcc, pixel_format;
7859         unsigned int aligned_height;
7860         struct drm_framebuffer *fb;
7861         struct intel_framebuffer *intel_fb;
7862
7863         if (!plane->get_hw_state(plane, &pipe))
7864                 return;
7865
7866         WARN_ON(pipe != crtc->pipe);
7867
7868         intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL);
7869         if (!intel_fb) {
7870                 DRM_DEBUG_KMS("failed to alloc fb\n");
7871                 return;
7872         }
7873
7874         fb = &intel_fb->base;
7875
7876         fb->dev = dev;
7877
7878         val = I915_READ(DSPCNTR(i9xx_plane));
7879
7880         if (INTEL_GEN(dev_priv) >= 4) {
7881                 if (val & DISPPLANE_TILED) {
7882                         plane_config->tiling = I915_TILING_X;
7883                         fb->modifier = I915_FORMAT_MOD_X_TILED;
7884                 }
7885         }
7886
7887         pixel_format = val & DISPPLANE_PIXFORMAT_MASK;
7888         fourcc = i9xx_format_to_fourcc(pixel_format);
7889         fb->format = drm_format_info(fourcc);
7890
7891         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
7892                 offset = I915_READ(DSPOFFSET(i9xx_plane));
7893                 base = I915_READ(DSPSURF(i9xx_plane)) & 0xfffff000;
7894         } else if (INTEL_GEN(dev_priv) >= 4) {
7895                 if (plane_config->tiling)
7896                         offset = I915_READ(DSPTILEOFF(i9xx_plane));
7897                 else
7898                         offset = I915_READ(DSPLINOFF(i9xx_plane));
7899                 base = I915_READ(DSPSURF(i9xx_plane)) & 0xfffff000;
7900         } else {
7901                 base = I915_READ(DSPADDR(i9xx_plane));
7902         }
7903         plane_config->base = base;
7904
7905         val = I915_READ(PIPESRC(pipe));
7906         fb->width = ((val >> 16) & 0xfff) + 1;
7907         fb->height = ((val >> 0) & 0xfff) + 1;
7908
7909         val = I915_READ(DSPSTRIDE(i9xx_plane));
7910         fb->pitches[0] = val & 0xffffffc0;
7911
7912         aligned_height = intel_fb_align_height(fb, 0, fb->height);
7913
7914         plane_config->size = fb->pitches[0] * aligned_height;
7915
7916         DRM_DEBUG_KMS("%s/%s with fb: size=%dx%d@%d, offset=%x, pitch %d, size 0x%x\n",
7917                       crtc->base.name, plane->base.name, fb->width, fb->height,
7918                       fb->format->cpp[0] * 8, base, fb->pitches[0],
7919                       plane_config->size);
7920
7921         plane_config->fb = intel_fb;
7922 }
7923
7924 static void chv_crtc_clock_get(struct intel_crtc *crtc,
7925                                struct intel_crtc_state *pipe_config)
7926 {
7927         struct drm_device *dev = crtc->base.dev;
7928         struct drm_i915_private *dev_priv = to_i915(dev);
7929         int pipe = pipe_config->cpu_transcoder;
7930         enum dpio_channel port = vlv_pipe_to_channel(pipe);
7931         struct dpll clock;
7932         u32 cmn_dw13, pll_dw0, pll_dw1, pll_dw2, pll_dw3;
7933         int refclk = 100000;
7934
7935         /* In case of DSI, DPLL will not be used */
7936         if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0)
7937                 return;
7938
7939         mutex_lock(&dev_priv->sb_lock);
7940         cmn_dw13 = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW13(port));
7941         pll_dw0 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW0(port));
7942         pll_dw1 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW1(port));
7943         pll_dw2 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW2(port));
7944         pll_dw3 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW3(port));
7945         mutex_unlock(&dev_priv->sb_lock);
7946
7947         clock.m1 = (pll_dw1 & 0x7) == DPIO_CHV_M1_DIV_BY_2 ? 2 : 0;
7948         clock.m2 = (pll_dw0 & 0xff) << 22;
7949         if (pll_dw3 & DPIO_CHV_FRAC_DIV_EN)
7950                 clock.m2 |= pll_dw2 & 0x3fffff;
7951         clock.n = (pll_dw1 >> DPIO_CHV_N_DIV_SHIFT) & 0xf;
7952         clock.p1 = (cmn_dw13 >> DPIO_CHV_P1_DIV_SHIFT) & 0x7;
7953         clock.p2 = (cmn_dw13 >> DPIO_CHV_P2_DIV_SHIFT) & 0x1f;
7954
7955         pipe_config->port_clock = chv_calc_dpll_params(refclk, &clock);
7956 }
7957
7958 static bool i9xx_get_pipe_config(struct intel_crtc *crtc,
7959                                  struct intel_crtc_state *pipe_config)
7960 {
7961         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7962         enum intel_display_power_domain power_domain;
7963         uint32_t tmp;
7964         bool ret;
7965
7966         power_domain = POWER_DOMAIN_PIPE(crtc->pipe);
7967         if (!intel_display_power_get_if_enabled(dev_priv, power_domain))
7968                 return false;
7969
7970         pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe;
7971         pipe_config->shared_dpll = NULL;
7972
7973         ret = false;
7974
7975         tmp = I915_READ(PIPECONF(crtc->pipe));
7976         if (!(tmp & PIPECONF_ENABLE))
7977                 goto out;
7978
7979         if (IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
7980             IS_CHERRYVIEW(dev_priv)) {
7981                 switch (tmp & PIPECONF_BPC_MASK) {
7982                 case PIPECONF_6BPC:
7983                         pipe_config->pipe_bpp = 18;
7984                         break;
7985                 case PIPECONF_8BPC:
7986                         pipe_config->pipe_bpp = 24;
7987                         break;
7988                 case PIPECONF_10BPC:
7989                         pipe_config->pipe_bpp = 30;
7990                         break;
7991                 default:
7992                         break;
7993                 }
7994         }
7995
7996         if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
7997             (tmp & PIPECONF_COLOR_RANGE_SELECT))
7998                 pipe_config->limited_color_range = true;
7999
8000         if (INTEL_GEN(dev_priv) < 4)
8001                 pipe_config->double_wide = tmp & PIPECONF_DOUBLE_WIDE;
8002
8003         intel_get_pipe_timings(crtc, pipe_config);
8004         intel_get_pipe_src_size(crtc, pipe_config);
8005
8006         i9xx_get_pfit_config(crtc, pipe_config);
8007
8008         if (INTEL_GEN(dev_priv) >= 4) {
8009                 /* No way to read it out on pipes B and C */
8010                 if (IS_CHERRYVIEW(dev_priv) && crtc->pipe != PIPE_A)
8011                         tmp = dev_priv->chv_dpll_md[crtc->pipe];
8012                 else
8013                         tmp = I915_READ(DPLL_MD(crtc->pipe));
8014                 pipe_config->pixel_multiplier =
8015                         ((tmp & DPLL_MD_UDI_MULTIPLIER_MASK)
8016                          >> DPLL_MD_UDI_MULTIPLIER_SHIFT) + 1;
8017                 pipe_config->dpll_hw_state.dpll_md = tmp;
8018         } else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv) ||
8019                    IS_G33(dev_priv) || IS_PINEVIEW(dev_priv)) {
8020                 tmp = I915_READ(DPLL(crtc->pipe));
8021                 pipe_config->pixel_multiplier =
8022                         ((tmp & SDVO_MULTIPLIER_MASK)
8023                          >> SDVO_MULTIPLIER_SHIFT_HIRES) + 1;
8024         } else {
8025                 /* Note that on i915G/GM the pixel multiplier is in the sdvo
8026                  * port and will be fixed up in the encoder->get_config
8027                  * function. */
8028                 pipe_config->pixel_multiplier = 1;
8029         }
8030         pipe_config->dpll_hw_state.dpll = I915_READ(DPLL(crtc->pipe));
8031         if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv)) {
8032                 /*
8033                  * DPLL_DVO_2X_MODE must be enabled for both DPLLs
8034                  * on 830. Filter it out here so that we don't
8035                  * report errors due to that.
8036                  */
8037                 if (IS_I830(dev_priv))
8038                         pipe_config->dpll_hw_state.dpll &= ~DPLL_DVO_2X_MODE;
8039
8040                 pipe_config->dpll_hw_state.fp0 = I915_READ(FP0(crtc->pipe));
8041                 pipe_config->dpll_hw_state.fp1 = I915_READ(FP1(crtc->pipe));
8042         } else {
8043                 /* Mask out read-only status bits. */
8044                 pipe_config->dpll_hw_state.dpll &= ~(DPLL_LOCK_VLV |
8045                                                      DPLL_PORTC_READY_MASK |
8046                                                      DPLL_PORTB_READY_MASK);
8047         }
8048
8049         if (IS_CHERRYVIEW(dev_priv))
8050                 chv_crtc_clock_get(crtc, pipe_config);
8051         else if (IS_VALLEYVIEW(dev_priv))
8052                 vlv_crtc_clock_get(crtc, pipe_config);
8053         else
8054                 i9xx_crtc_clock_get(crtc, pipe_config);
8055
8056         /*
8057          * Normally the dotclock is filled in by the encoder .get_config()
8058          * but in case the pipe is enabled w/o any ports we need a sane
8059          * default.
8060          */
8061         pipe_config->base.adjusted_mode.crtc_clock =
8062                 pipe_config->port_clock / pipe_config->pixel_multiplier;
8063
8064         ret = true;
8065
8066 out:
8067         intel_display_power_put(dev_priv, power_domain);
8068
8069         return ret;
8070 }
8071
8072 static void ironlake_init_pch_refclk(struct drm_i915_private *dev_priv)
8073 {
8074         struct intel_encoder *encoder;
8075         int i;
8076         u32 val, final;
8077         bool has_lvds = false;
8078         bool has_cpu_edp = false;
8079         bool has_panel = false;
8080         bool has_ck505 = false;
8081         bool can_ssc = false;
8082         bool using_ssc_source = false;
8083
8084         /* We need to take the global config into account */
8085         for_each_intel_encoder(&dev_priv->drm, encoder) {
8086                 switch (encoder->type) {
8087                 case INTEL_OUTPUT_LVDS:
8088                         has_panel = true;
8089                         has_lvds = true;
8090                         break;
8091                 case INTEL_OUTPUT_EDP:
8092                         has_panel = true;
8093                         if (encoder->port == PORT_A)
8094                                 has_cpu_edp = true;
8095                         break;
8096                 default:
8097                         break;
8098                 }
8099         }
8100
8101         if (HAS_PCH_IBX(dev_priv)) {
8102                 has_ck505 = dev_priv->vbt.display_clock_mode;
8103                 can_ssc = has_ck505;
8104         } else {
8105                 has_ck505 = false;
8106                 can_ssc = true;
8107         }
8108
8109         /* Check if any DPLLs are using the SSC source */
8110         for (i = 0; i < dev_priv->num_shared_dpll; i++) {
8111                 u32 temp = I915_READ(PCH_DPLL(i));
8112
8113                 if (!(temp & DPLL_VCO_ENABLE))
8114                         continue;
8115
8116                 if ((temp & PLL_REF_INPUT_MASK) ==
8117                     PLLB_REF_INPUT_SPREADSPECTRUMIN) {
8118                         using_ssc_source = true;
8119                         break;
8120                 }
8121         }
8122
8123         DRM_DEBUG_KMS("has_panel %d has_lvds %d has_ck505 %d using_ssc_source %d\n",
8124                       has_panel, has_lvds, has_ck505, using_ssc_source);
8125
8126         /* Ironlake: try to setup display ref clock before DPLL
8127          * enabling. This is only under driver's control after
8128          * PCH B stepping, previous chipset stepping should be
8129          * ignoring this setting.
8130          */
8131         val = I915_READ(PCH_DREF_CONTROL);
8132
8133         /* As we must carefully and slowly disable/enable each source in turn,
8134          * compute the final state we want first and check if we need to
8135          * make any changes at all.
8136          */
8137         final = val;
8138         final &= ~DREF_NONSPREAD_SOURCE_MASK;
8139         if (has_ck505)
8140                 final |= DREF_NONSPREAD_CK505_ENABLE;
8141         else
8142                 final |= DREF_NONSPREAD_SOURCE_ENABLE;
8143
8144         final &= ~DREF_SSC_SOURCE_MASK;
8145         final &= ~DREF_CPU_SOURCE_OUTPUT_MASK;
8146         final &= ~DREF_SSC1_ENABLE;
8147
8148         if (has_panel) {
8149                 final |= DREF_SSC_SOURCE_ENABLE;
8150
8151                 if (intel_panel_use_ssc(dev_priv) && can_ssc)
8152                         final |= DREF_SSC1_ENABLE;
8153
8154                 if (has_cpu_edp) {
8155                         if (intel_panel_use_ssc(dev_priv) && can_ssc)
8156                                 final |= DREF_CPU_SOURCE_OUTPUT_DOWNSPREAD;
8157                         else
8158                                 final |= DREF_CPU_SOURCE_OUTPUT_NONSPREAD;
8159                 } else
8160                         final |= DREF_CPU_SOURCE_OUTPUT_DISABLE;
8161         } else if (using_ssc_source) {
8162                 final |= DREF_SSC_SOURCE_ENABLE;
8163                 final |= DREF_SSC1_ENABLE;
8164         }
8165
8166         if (final == val)
8167                 return;
8168
8169         /* Always enable nonspread source */
8170         val &= ~DREF_NONSPREAD_SOURCE_MASK;
8171
8172         if (has_ck505)
8173                 val |= DREF_NONSPREAD_CK505_ENABLE;
8174         else
8175                 val |= DREF_NONSPREAD_SOURCE_ENABLE;
8176
8177         if (has_panel) {
8178                 val &= ~DREF_SSC_SOURCE_MASK;
8179                 val |= DREF_SSC_SOURCE_ENABLE;
8180
8181                 /* SSC must be turned on before enabling the CPU output  */
8182                 if (intel_panel_use_ssc(dev_priv) && can_ssc) {
8183                         DRM_DEBUG_KMS("Using SSC on panel\n");
8184                         val |= DREF_SSC1_ENABLE;
8185                 } else
8186                         val &= ~DREF_SSC1_ENABLE;
8187
8188                 /* Get SSC going before enabling the outputs */
8189                 I915_WRITE(PCH_DREF_CONTROL, val);
8190                 POSTING_READ(PCH_DREF_CONTROL);
8191                 udelay(200);
8192
8193                 val &= ~DREF_CPU_SOURCE_OUTPUT_MASK;
8194
8195                 /* Enable CPU source on CPU attached eDP */
8196                 if (has_cpu_edp) {
8197                         if (intel_panel_use_ssc(dev_priv) && can_ssc) {
8198                                 DRM_DEBUG_KMS("Using SSC on eDP\n");
8199                                 val |= DREF_CPU_SOURCE_OUTPUT_DOWNSPREAD;
8200                         } else
8201                                 val |= DREF_CPU_SOURCE_OUTPUT_NONSPREAD;
8202                 } else
8203                         val |= DREF_CPU_SOURCE_OUTPUT_DISABLE;
8204
8205                 I915_WRITE(PCH_DREF_CONTROL, val);
8206                 POSTING_READ(PCH_DREF_CONTROL);
8207                 udelay(200);
8208         } else {
8209                 DRM_DEBUG_KMS("Disabling CPU source output\n");
8210
8211                 val &= ~DREF_CPU_SOURCE_OUTPUT_MASK;
8212
8213                 /* Turn off CPU output */
8214                 val |= DREF_CPU_SOURCE_OUTPUT_DISABLE;
8215
8216                 I915_WRITE(PCH_DREF_CONTROL, val);
8217                 POSTING_READ(PCH_DREF_CONTROL);
8218                 udelay(200);
8219
8220                 if (!using_ssc_source) {
8221                         DRM_DEBUG_KMS("Disabling SSC source\n");
8222
8223                         /* Turn off the SSC source */
8224                         val &= ~DREF_SSC_SOURCE_MASK;
8225                         val |= DREF_SSC_SOURCE_DISABLE;
8226
8227                         /* Turn off SSC1 */
8228                         val &= ~DREF_SSC1_ENABLE;
8229
8230                         I915_WRITE(PCH_DREF_CONTROL, val);
8231                         POSTING_READ(PCH_DREF_CONTROL);
8232                         udelay(200);
8233                 }
8234         }
8235
8236         BUG_ON(val != final);
8237 }
8238
8239 static void lpt_reset_fdi_mphy(struct drm_i915_private *dev_priv)
8240 {
8241         uint32_t tmp;
8242
8243         tmp = I915_READ(SOUTH_CHICKEN2);
8244         tmp |= FDI_MPHY_IOSFSB_RESET_CTL;
8245         I915_WRITE(SOUTH_CHICKEN2, tmp);
8246
8247         if (wait_for_us(I915_READ(SOUTH_CHICKEN2) &
8248                         FDI_MPHY_IOSFSB_RESET_STATUS, 100))
8249                 DRM_ERROR("FDI mPHY reset assert timeout\n");
8250
8251         tmp = I915_READ(SOUTH_CHICKEN2);
8252         tmp &= ~FDI_MPHY_IOSFSB_RESET_CTL;
8253         I915_WRITE(SOUTH_CHICKEN2, tmp);
8254
8255         if (wait_for_us((I915_READ(SOUTH_CHICKEN2) &
8256                          FDI_MPHY_IOSFSB_RESET_STATUS) == 0, 100))
8257                 DRM_ERROR("FDI mPHY reset de-assert timeout\n");
8258 }
8259
8260 /* WaMPhyProgramming:hsw */
8261 static void lpt_program_fdi_mphy(struct drm_i915_private *dev_priv)
8262 {
8263         uint32_t tmp;
8264
8265         tmp = intel_sbi_read(dev_priv, 0x8008, SBI_MPHY);
8266         tmp &= ~(0xFF << 24);
8267         tmp |= (0x12 << 24);
8268         intel_sbi_write(dev_priv, 0x8008, tmp, SBI_MPHY);
8269
8270         tmp = intel_sbi_read(dev_priv, 0x2008, SBI_MPHY);
8271         tmp |= (1 << 11);
8272         intel_sbi_write(dev_priv, 0x2008, tmp, SBI_MPHY);
8273
8274         tmp = intel_sbi_read(dev_priv, 0x2108, SBI_MPHY);
8275         tmp |= (1 << 11);
8276         intel_sbi_write(dev_priv, 0x2108, tmp, SBI_MPHY);
8277
8278         tmp = intel_sbi_read(dev_priv, 0x206C, SBI_MPHY);
8279         tmp |= (1 << 24) | (1 << 21) | (1 << 18);
8280         intel_sbi_write(dev_priv, 0x206C, tmp, SBI_MPHY);
8281
8282         tmp = intel_sbi_read(dev_priv, 0x216C, SBI_MPHY);
8283         tmp |= (1 << 24) | (1 << 21) | (1 << 18);
8284         intel_sbi_write(dev_priv, 0x216C, tmp, SBI_MPHY);
8285
8286         tmp = intel_sbi_read(dev_priv, 0x2080, SBI_MPHY);
8287         tmp &= ~(7 << 13);
8288         tmp |= (5 << 13);
8289         intel_sbi_write(dev_priv, 0x2080, tmp, SBI_MPHY);
8290
8291         tmp = intel_sbi_read(dev_priv, 0x2180, SBI_MPHY);
8292         tmp &= ~(7 << 13);
8293         tmp |= (5 << 13);
8294         intel_sbi_write(dev_priv, 0x2180, tmp, SBI_MPHY);
8295
8296         tmp = intel_sbi_read(dev_priv, 0x208C, SBI_MPHY);
8297         tmp &= ~0xFF;
8298         tmp |= 0x1C;
8299         intel_sbi_write(dev_priv, 0x208C, tmp, SBI_MPHY);
8300
8301         tmp = intel_sbi_read(dev_priv, 0x218C, SBI_MPHY);
8302         tmp &= ~0xFF;
8303         tmp |= 0x1C;
8304         intel_sbi_write(dev_priv, 0x218C, tmp, SBI_MPHY);
8305
8306         tmp = intel_sbi_read(dev_priv, 0x2098, SBI_MPHY);
8307         tmp &= ~(0xFF << 16);
8308         tmp |= (0x1C << 16);
8309         intel_sbi_write(dev_priv, 0x2098, tmp, SBI_MPHY);
8310
8311         tmp = intel_sbi_read(dev_priv, 0x2198, SBI_MPHY);
8312         tmp &= ~(0xFF << 16);
8313         tmp |= (0x1C << 16);
8314         intel_sbi_write(dev_priv, 0x2198, tmp, SBI_MPHY);
8315
8316         tmp = intel_sbi_read(dev_priv, 0x20C4, SBI_MPHY);
8317         tmp |= (1 << 27);
8318         intel_sbi_write(dev_priv, 0x20C4, tmp, SBI_MPHY);
8319
8320         tmp = intel_sbi_read(dev_priv, 0x21C4, SBI_MPHY);
8321         tmp |= (1 << 27);
8322         intel_sbi_write(dev_priv, 0x21C4, tmp, SBI_MPHY);
8323
8324         tmp = intel_sbi_read(dev_priv, 0x20EC, SBI_MPHY);
8325         tmp &= ~(0xF << 28);
8326         tmp |= (4 << 28);
8327         intel_sbi_write(dev_priv, 0x20EC, tmp, SBI_MPHY);
8328
8329         tmp = intel_sbi_read(dev_priv, 0x21EC, SBI_MPHY);
8330         tmp &= ~(0xF << 28);
8331         tmp |= (4 << 28);
8332         intel_sbi_write(dev_priv, 0x21EC, tmp, SBI_MPHY);
8333 }
8334
8335 /* Implements 3 different sequences from BSpec chapter "Display iCLK
8336  * Programming" based on the parameters passed:
8337  * - Sequence to enable CLKOUT_DP
8338  * - Sequence to enable CLKOUT_DP without spread
8339  * - Sequence to enable CLKOUT_DP for FDI usage and configure PCH FDI I/O
8340  */
8341 static void lpt_enable_clkout_dp(struct drm_i915_private *dev_priv,
8342                                  bool with_spread, bool with_fdi)
8343 {
8344         uint32_t reg, tmp;
8345
8346         if (WARN(with_fdi && !with_spread, "FDI requires downspread\n"))
8347                 with_spread = true;
8348         if (WARN(HAS_PCH_LPT_LP(dev_priv) &&
8349             with_fdi, "LP PCH doesn't have FDI\n"))
8350                 with_fdi = false;
8351
8352         mutex_lock(&dev_priv->sb_lock);
8353
8354         tmp = intel_sbi_read(dev_priv, SBI_SSCCTL, SBI_ICLK);
8355         tmp &= ~SBI_SSCCTL_DISABLE;
8356         tmp |= SBI_SSCCTL_PATHALT;
8357         intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
8358
8359         udelay(24);
8360
8361         if (with_spread) {
8362                 tmp = intel_sbi_read(dev_priv, SBI_SSCCTL, SBI_ICLK);
8363                 tmp &= ~SBI_SSCCTL_PATHALT;
8364                 intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
8365
8366                 if (with_fdi) {
8367                         lpt_reset_fdi_mphy(dev_priv);
8368                         lpt_program_fdi_mphy(dev_priv);
8369                 }
8370         }
8371
8372         reg = HAS_PCH_LPT_LP(dev_priv) ? SBI_GEN0 : SBI_DBUFF0;
8373         tmp = intel_sbi_read(dev_priv, reg, SBI_ICLK);
8374         tmp |= SBI_GEN0_CFG_BUFFENABLE_DISABLE;
8375         intel_sbi_write(dev_priv, reg, tmp, SBI_ICLK);
8376
8377         mutex_unlock(&dev_priv->sb_lock);
8378 }
8379
8380 /* Sequence to disable CLKOUT_DP */
8381 static void lpt_disable_clkout_dp(struct drm_i915_private *dev_priv)
8382 {
8383         uint32_t reg, tmp;
8384
8385         mutex_lock(&dev_priv->sb_lock);
8386
8387         reg = HAS_PCH_LPT_LP(dev_priv) ? SBI_GEN0 : SBI_DBUFF0;
8388         tmp = intel_sbi_read(dev_priv, reg, SBI_ICLK);
8389         tmp &= ~SBI_GEN0_CFG_BUFFENABLE_DISABLE;
8390         intel_sbi_write(dev_priv, reg, tmp, SBI_ICLK);
8391
8392         tmp = intel_sbi_read(dev_priv, SBI_SSCCTL, SBI_ICLK);
8393         if (!(tmp & SBI_SSCCTL_DISABLE)) {
8394                 if (!(tmp & SBI_SSCCTL_PATHALT)) {
8395                         tmp |= SBI_SSCCTL_PATHALT;
8396                         intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
8397                         udelay(32);
8398                 }
8399                 tmp |= SBI_SSCCTL_DISABLE;
8400                 intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
8401         }
8402
8403         mutex_unlock(&dev_priv->sb_lock);
8404 }
8405
8406 #define BEND_IDX(steps) ((50 + (steps)) / 5)
8407
8408 static const uint16_t sscdivintphase[] = {
8409         [BEND_IDX( 50)] = 0x3B23,
8410         [BEND_IDX( 45)] = 0x3B23,
8411         [BEND_IDX( 40)] = 0x3C23,
8412         [BEND_IDX( 35)] = 0x3C23,
8413         [BEND_IDX( 30)] = 0x3D23,
8414         [BEND_IDX( 25)] = 0x3D23,
8415         [BEND_IDX( 20)] = 0x3E23,
8416         [BEND_IDX( 15)] = 0x3E23,
8417         [BEND_IDX( 10)] = 0x3F23,
8418         [BEND_IDX(  5)] = 0x3F23,
8419         [BEND_IDX(  0)] = 0x0025,
8420         [BEND_IDX( -5)] = 0x0025,
8421         [BEND_IDX(-10)] = 0x0125,
8422         [BEND_IDX(-15)] = 0x0125,
8423         [BEND_IDX(-20)] = 0x0225,
8424         [BEND_IDX(-25)] = 0x0225,
8425         [BEND_IDX(-30)] = 0x0325,
8426         [BEND_IDX(-35)] = 0x0325,
8427         [BEND_IDX(-40)] = 0x0425,
8428         [BEND_IDX(-45)] = 0x0425,
8429         [BEND_IDX(-50)] = 0x0525,
8430 };
8431
8432 /*
8433  * Bend CLKOUT_DP
8434  * steps -50 to 50 inclusive, in steps of 5
8435  * < 0 slow down the clock, > 0 speed up the clock, 0 == no bend (135MHz)
8436  * change in clock period = -(steps / 10) * 5.787 ps
8437  */
8438 static void lpt_bend_clkout_dp(struct drm_i915_private *dev_priv, int steps)
8439 {
8440         uint32_t tmp;
8441         int idx = BEND_IDX(steps);
8442
8443         if (WARN_ON(steps % 5 != 0))
8444                 return;
8445
8446         if (WARN_ON(idx >= ARRAY_SIZE(sscdivintphase)))
8447                 return;
8448
8449         mutex_lock(&dev_priv->sb_lock);
8450
8451         if (steps % 10 != 0)
8452                 tmp = 0xAAAAAAAB;
8453         else
8454                 tmp = 0x00000000;
8455         intel_sbi_write(dev_priv, SBI_SSCDITHPHASE, tmp, SBI_ICLK);
8456
8457         tmp = intel_sbi_read(dev_priv, SBI_SSCDIVINTPHASE, SBI_ICLK);
8458         tmp &= 0xffff0000;
8459         tmp |= sscdivintphase[idx];
8460         intel_sbi_write(dev_priv, SBI_SSCDIVINTPHASE, tmp, SBI_ICLK);
8461
8462         mutex_unlock(&dev_priv->sb_lock);
8463 }
8464
8465 #undef BEND_IDX
8466
8467 static void lpt_init_pch_refclk(struct drm_i915_private *dev_priv)
8468 {
8469         struct intel_encoder *encoder;
8470         bool has_vga = false;
8471
8472         for_each_intel_encoder(&dev_priv->drm, encoder) {
8473                 switch (encoder->type) {
8474                 case INTEL_OUTPUT_ANALOG:
8475                         has_vga = true;
8476                         break;
8477                 default:
8478                         break;
8479                 }
8480         }
8481
8482         if (has_vga) {
8483                 lpt_bend_clkout_dp(dev_priv, 0);
8484                 lpt_enable_clkout_dp(dev_priv, true, true);
8485         } else {
8486                 lpt_disable_clkout_dp(dev_priv);
8487         }
8488 }
8489
8490 /*
8491  * Initialize reference clocks when the driver loads
8492  */
8493 void intel_init_pch_refclk(struct drm_i915_private *dev_priv)
8494 {
8495         if (HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv))
8496                 ironlake_init_pch_refclk(dev_priv);
8497         else if (HAS_PCH_LPT(dev_priv))
8498                 lpt_init_pch_refclk(dev_priv);
8499 }
8500
8501 static void ironlake_set_pipeconf(struct drm_crtc *crtc)
8502 {
8503         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
8504         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
8505         int pipe = intel_crtc->pipe;
8506         uint32_t val;
8507
8508         val = 0;
8509
8510         switch (intel_crtc->config->pipe_bpp) {
8511         case 18:
8512                 val |= PIPECONF_6BPC;
8513                 break;
8514         case 24:
8515                 val |= PIPECONF_8BPC;
8516                 break;
8517         case 30:
8518                 val |= PIPECONF_10BPC;
8519                 break;
8520         case 36:
8521                 val |= PIPECONF_12BPC;
8522                 break;
8523         default:
8524                 /* Case prevented by intel_choose_pipe_bpp_dither. */
8525                 BUG();
8526         }
8527
8528         if (intel_crtc->config->dither)
8529                 val |= (PIPECONF_DITHER_EN | PIPECONF_DITHER_TYPE_SP);
8530
8531         if (intel_crtc->config->base.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE)
8532                 val |= PIPECONF_INTERLACED_ILK;
8533         else
8534                 val |= PIPECONF_PROGRESSIVE;
8535
8536         if (intel_crtc->config->limited_color_range)
8537                 val |= PIPECONF_COLOR_RANGE_SELECT;
8538
8539         I915_WRITE(PIPECONF(pipe), val);
8540         POSTING_READ(PIPECONF(pipe));
8541 }
8542
8543 static void haswell_set_pipeconf(struct drm_crtc *crtc)
8544 {
8545         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
8546         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
8547         enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder;
8548         u32 val = 0;
8549
8550         if (IS_HASWELL(dev_priv) && intel_crtc->config->dither)
8551                 val |= (PIPECONF_DITHER_EN | PIPECONF_DITHER_TYPE_SP);
8552
8553         if (intel_crtc->config->base.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE)
8554                 val |= PIPECONF_INTERLACED_ILK;
8555         else
8556                 val |= PIPECONF_PROGRESSIVE;
8557
8558         I915_WRITE(PIPECONF(cpu_transcoder), val);
8559         POSTING_READ(PIPECONF(cpu_transcoder));
8560 }
8561
8562 static void haswell_set_pipemisc(struct drm_crtc *crtc)
8563 {
8564         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
8565         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
8566         struct intel_crtc_state *config = intel_crtc->config;
8567
8568         if (IS_BROADWELL(dev_priv) || INTEL_GEN(dev_priv) >= 9) {
8569                 u32 val = 0;
8570
8571                 switch (intel_crtc->config->pipe_bpp) {
8572                 case 18:
8573                         val |= PIPEMISC_DITHER_6_BPC;
8574                         break;
8575                 case 24:
8576                         val |= PIPEMISC_DITHER_8_BPC;
8577                         break;
8578                 case 30:
8579                         val |= PIPEMISC_DITHER_10_BPC;
8580                         break;
8581                 case 36:
8582                         val |= PIPEMISC_DITHER_12_BPC;
8583                         break;
8584                 default:
8585                         /* Case prevented by pipe_config_set_bpp. */
8586                         BUG();
8587                 }
8588
8589                 if (intel_crtc->config->dither)
8590                         val |= PIPEMISC_DITHER_ENABLE | PIPEMISC_DITHER_TYPE_SP;
8591
8592                 if (config->ycbcr420) {
8593                         val |= PIPEMISC_OUTPUT_COLORSPACE_YUV |
8594                                 PIPEMISC_YUV420_ENABLE |
8595                                 PIPEMISC_YUV420_MODE_FULL_BLEND;
8596                 }
8597
8598                 I915_WRITE(PIPEMISC(intel_crtc->pipe), val);
8599         }
8600 }
8601
8602 int ironlake_get_lanes_required(int target_clock, int link_bw, int bpp)
8603 {
8604         /*
8605          * Account for spread spectrum to avoid
8606          * oversubscribing the link. Max center spread
8607          * is 2.5%; use 5% for safety's sake.
8608          */
8609         u32 bps = target_clock * bpp * 21 / 20;
8610         return DIV_ROUND_UP(bps, link_bw * 8);
8611 }
8612
8613 static bool ironlake_needs_fb_cb_tune(struct dpll *dpll, int factor)
8614 {
8615         return i9xx_dpll_compute_m(dpll) < factor * dpll->n;
8616 }
8617
8618 static void ironlake_compute_dpll(struct intel_crtc *intel_crtc,
8619                                   struct intel_crtc_state *crtc_state,
8620                                   struct dpll *reduced_clock)
8621 {
8622         struct drm_crtc *crtc = &intel_crtc->base;
8623         struct drm_device *dev = crtc->dev;
8624         struct drm_i915_private *dev_priv = to_i915(dev);
8625         u32 dpll, fp, fp2;
8626         int factor;
8627
8628         /* Enable autotuning of the PLL clock (if permissible) */
8629         factor = 21;
8630         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
8631                 if ((intel_panel_use_ssc(dev_priv) &&
8632                      dev_priv->vbt.lvds_ssc_freq == 100000) ||
8633                     (HAS_PCH_IBX(dev_priv) && intel_is_dual_link_lvds(dev)))
8634                         factor = 25;
8635         } else if (crtc_state->sdvo_tv_clock)
8636                 factor = 20;
8637
8638         fp = i9xx_dpll_compute_fp(&crtc_state->dpll);
8639
8640         if (ironlake_needs_fb_cb_tune(&crtc_state->dpll, factor))
8641                 fp |= FP_CB_TUNE;
8642
8643         if (reduced_clock) {
8644                 fp2 = i9xx_dpll_compute_fp(reduced_clock);
8645
8646                 if (reduced_clock->m < factor * reduced_clock->n)
8647                         fp2 |= FP_CB_TUNE;
8648         } else {
8649                 fp2 = fp;
8650         }
8651
8652         dpll = 0;
8653
8654         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS))
8655                 dpll |= DPLLB_MODE_LVDS;
8656         else
8657                 dpll |= DPLLB_MODE_DAC_SERIAL;
8658
8659         dpll |= (crtc_state->pixel_multiplier - 1)
8660                 << PLL_REF_SDVO_HDMI_MULTIPLIER_SHIFT;
8661
8662         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO) ||
8663             intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI))
8664                 dpll |= DPLL_SDVO_HIGH_SPEED;
8665
8666         if (intel_crtc_has_dp_encoder(crtc_state))
8667                 dpll |= DPLL_SDVO_HIGH_SPEED;
8668
8669         /*
8670          * The high speed IO clock is only really required for
8671          * SDVO/HDMI/DP, but we also enable it for CRT to make it
8672          * possible to share the DPLL between CRT and HDMI. Enabling
8673          * the clock needlessly does no real harm, except use up a
8674          * bit of power potentially.
8675          *
8676          * We'll limit this to IVB with 3 pipes, since it has only two
8677          * DPLLs and so DPLL sharing is the only way to get three pipes
8678          * driving PCH ports at the same time. On SNB we could do this,
8679          * and potentially avoid enabling the second DPLL, but it's not
8680          * clear if it''s a win or loss power wise. No point in doing
8681          * this on ILK at all since it has a fixed DPLL<->pipe mapping.
8682          */
8683         if (INTEL_INFO(dev_priv)->num_pipes == 3 &&
8684             intel_crtc_has_type(crtc_state, INTEL_OUTPUT_ANALOG))
8685                 dpll |= DPLL_SDVO_HIGH_SPEED;
8686
8687         /* compute bitmask from p1 value */
8688         dpll |= (1 << (crtc_state->dpll.p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
8689         /* also FPA1 */
8690         dpll |= (1 << (crtc_state->dpll.p1 - 1)) << DPLL_FPA1_P1_POST_DIV_SHIFT;
8691
8692         switch (crtc_state->dpll.p2) {
8693         case 5:
8694                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_5;
8695                 break;
8696         case 7:
8697                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_7;
8698                 break;
8699         case 10:
8700                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_10;
8701                 break;
8702         case 14:
8703                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_14;
8704                 break;
8705         }
8706
8707         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS) &&
8708             intel_panel_use_ssc(dev_priv))
8709                 dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN;
8710         else
8711                 dpll |= PLL_REF_INPUT_DREFCLK;
8712
8713         dpll |= DPLL_VCO_ENABLE;
8714
8715         crtc_state->dpll_hw_state.dpll = dpll;
8716         crtc_state->dpll_hw_state.fp0 = fp;
8717         crtc_state->dpll_hw_state.fp1 = fp2;
8718 }
8719
8720 static int ironlake_crtc_compute_clock(struct intel_crtc *crtc,
8721                                        struct intel_crtc_state *crtc_state)
8722 {
8723         struct drm_device *dev = crtc->base.dev;
8724         struct drm_i915_private *dev_priv = to_i915(dev);
8725         const struct intel_limit *limit;
8726         int refclk = 120000;
8727
8728         memset(&crtc_state->dpll_hw_state, 0,
8729                sizeof(crtc_state->dpll_hw_state));
8730
8731         /* CPU eDP is the only output that doesn't need a PCH PLL of its own. */
8732         if (!crtc_state->has_pch_encoder)
8733                 return 0;
8734
8735         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
8736                 if (intel_panel_use_ssc(dev_priv)) {
8737                         DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n",
8738                                       dev_priv->vbt.lvds_ssc_freq);
8739                         refclk = dev_priv->vbt.lvds_ssc_freq;
8740                 }
8741
8742                 if (intel_is_dual_link_lvds(dev)) {
8743                         if (refclk == 100000)
8744                                 limit = &intel_limits_ironlake_dual_lvds_100m;
8745                         else
8746                                 limit = &intel_limits_ironlake_dual_lvds;
8747                 } else {
8748                         if (refclk == 100000)
8749                                 limit = &intel_limits_ironlake_single_lvds_100m;
8750                         else
8751                                 limit = &intel_limits_ironlake_single_lvds;
8752                 }
8753         } else {
8754                 limit = &intel_limits_ironlake_dac;
8755         }
8756
8757         if (!crtc_state->clock_set &&
8758             !g4x_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
8759                                 refclk, NULL, &crtc_state->dpll)) {
8760                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
8761                 return -EINVAL;
8762         }
8763
8764         ironlake_compute_dpll(crtc, crtc_state, NULL);
8765
8766         if (!intel_get_shared_dpll(crtc, crtc_state, NULL)) {
8767                 DRM_DEBUG_KMS("failed to find PLL for pipe %c\n",
8768                               pipe_name(crtc->pipe));
8769                 return -EINVAL;
8770         }
8771
8772         return 0;
8773 }
8774
8775 static void intel_pch_transcoder_get_m_n(struct intel_crtc *crtc,
8776                                          struct intel_link_m_n *m_n)
8777 {
8778         struct drm_device *dev = crtc->base.dev;
8779         struct drm_i915_private *dev_priv = to_i915(dev);
8780         enum pipe pipe = crtc->pipe;
8781
8782         m_n->link_m = I915_READ(PCH_TRANS_LINK_M1(pipe));
8783         m_n->link_n = I915_READ(PCH_TRANS_LINK_N1(pipe));
8784         m_n->gmch_m = I915_READ(PCH_TRANS_DATA_M1(pipe))
8785                 & ~TU_SIZE_MASK;
8786         m_n->gmch_n = I915_READ(PCH_TRANS_DATA_N1(pipe));
8787         m_n->tu = ((I915_READ(PCH_TRANS_DATA_M1(pipe))
8788                     & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
8789 }
8790
8791 static void intel_cpu_transcoder_get_m_n(struct intel_crtc *crtc,
8792                                          enum transcoder transcoder,
8793                                          struct intel_link_m_n *m_n,
8794                                          struct intel_link_m_n *m2_n2)
8795 {
8796         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8797         enum pipe pipe = crtc->pipe;
8798
8799         if (INTEL_GEN(dev_priv) >= 5) {
8800                 m_n->link_m = I915_READ(PIPE_LINK_M1(transcoder));
8801                 m_n->link_n = I915_READ(PIPE_LINK_N1(transcoder));
8802                 m_n->gmch_m = I915_READ(PIPE_DATA_M1(transcoder))
8803                         & ~TU_SIZE_MASK;
8804                 m_n->gmch_n = I915_READ(PIPE_DATA_N1(transcoder));
8805                 m_n->tu = ((I915_READ(PIPE_DATA_M1(transcoder))
8806                             & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
8807                 /* Read M2_N2 registers only for gen < 8 (M2_N2 available for
8808                  * gen < 8) and if DRRS is supported (to make sure the
8809                  * registers are not unnecessarily read).
8810                  */
8811                 if (m2_n2 && INTEL_GEN(dev_priv) < 8 &&
8812                         crtc->config->has_drrs) {
8813                         m2_n2->link_m = I915_READ(PIPE_LINK_M2(transcoder));
8814                         m2_n2->link_n = I915_READ(PIPE_LINK_N2(transcoder));
8815                         m2_n2->gmch_m = I915_READ(PIPE_DATA_M2(transcoder))
8816                                         & ~TU_SIZE_MASK;
8817                         m2_n2->gmch_n = I915_READ(PIPE_DATA_N2(transcoder));
8818                         m2_n2->tu = ((I915_READ(PIPE_DATA_M2(transcoder))
8819                                         & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
8820                 }
8821         } else {
8822                 m_n->link_m = I915_READ(PIPE_LINK_M_G4X(pipe));
8823                 m_n->link_n = I915_READ(PIPE_LINK_N_G4X(pipe));
8824                 m_n->gmch_m = I915_READ(PIPE_DATA_M_G4X(pipe))
8825                         & ~TU_SIZE_MASK;
8826                 m_n->gmch_n = I915_READ(PIPE_DATA_N_G4X(pipe));
8827                 m_n->tu = ((I915_READ(PIPE_DATA_M_G4X(pipe))
8828                             & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
8829         }
8830 }
8831
8832 void intel_dp_get_m_n(struct intel_crtc *crtc,
8833                       struct intel_crtc_state *pipe_config)
8834 {
8835         if (pipe_config->has_pch_encoder)
8836                 intel_pch_transcoder_get_m_n(crtc, &pipe_config->dp_m_n);
8837         else
8838                 intel_cpu_transcoder_get_m_n(crtc, pipe_config->cpu_transcoder,
8839                                              &pipe_config->dp_m_n,
8840                                              &pipe_config->dp_m2_n2);
8841 }
8842
8843 static void ironlake_get_fdi_m_n_config(struct intel_crtc *crtc,
8844                                         struct intel_crtc_state *pipe_config)
8845 {
8846         intel_cpu_transcoder_get_m_n(crtc, pipe_config->cpu_transcoder,
8847                                      &pipe_config->fdi_m_n, NULL);
8848 }
8849
8850 static void skylake_get_pfit_config(struct intel_crtc *crtc,
8851                                     struct intel_crtc_state *pipe_config)
8852 {
8853         struct drm_device *dev = crtc->base.dev;
8854         struct drm_i915_private *dev_priv = to_i915(dev);
8855         struct intel_crtc_scaler_state *scaler_state = &pipe_config->scaler_state;
8856         uint32_t ps_ctrl = 0;
8857         int id = -1;
8858         int i;
8859
8860         /* find scaler attached to this pipe */
8861         for (i = 0; i < crtc->num_scalers; i++) {
8862                 ps_ctrl = I915_READ(SKL_PS_CTRL(crtc->pipe, i));
8863                 if (ps_ctrl & PS_SCALER_EN && !(ps_ctrl & PS_PLANE_SEL_MASK)) {
8864                         id = i;
8865                         pipe_config->pch_pfit.enabled = true;
8866                         pipe_config->pch_pfit.pos = I915_READ(SKL_PS_WIN_POS(crtc->pipe, i));
8867                         pipe_config->pch_pfit.size = I915_READ(SKL_PS_WIN_SZ(crtc->pipe, i));
8868                         break;
8869                 }
8870         }
8871
8872         scaler_state->scaler_id = id;
8873         if (id >= 0) {
8874                 scaler_state->scaler_users |= (1 << SKL_CRTC_INDEX);
8875         } else {
8876                 scaler_state->scaler_users &= ~(1 << SKL_CRTC_INDEX);
8877         }
8878 }
8879
8880 static void
8881 skylake_get_initial_plane_config(struct intel_crtc *crtc,
8882                                  struct intel_initial_plane_config *plane_config)
8883 {
8884         struct drm_device *dev = crtc->base.dev;
8885         struct drm_i915_private *dev_priv = to_i915(dev);
8886         struct intel_plane *plane = to_intel_plane(crtc->base.primary);
8887         enum plane_id plane_id = plane->id;
8888         enum pipe pipe;
8889         u32 val, base, offset, stride_mult, tiling, alpha;
8890         int fourcc, pixel_format;
8891         unsigned int aligned_height;
8892         struct drm_framebuffer *fb;
8893         struct intel_framebuffer *intel_fb;
8894
8895         if (!plane->get_hw_state(plane, &pipe))
8896                 return;
8897
8898         WARN_ON(pipe != crtc->pipe);
8899
8900         intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL);
8901         if (!intel_fb) {
8902                 DRM_DEBUG_KMS("failed to alloc fb\n");
8903                 return;
8904         }
8905
8906         fb = &intel_fb->base;
8907
8908         fb->dev = dev;
8909
8910         val = I915_READ(PLANE_CTL(pipe, plane_id));
8911
8912         if (INTEL_GEN(dev_priv) >= 11)
8913                 pixel_format = val & ICL_PLANE_CTL_FORMAT_MASK;
8914         else
8915                 pixel_format = val & PLANE_CTL_FORMAT_MASK;
8916
8917         if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv)) {
8918                 alpha = I915_READ(PLANE_COLOR_CTL(pipe, plane_id));
8919                 alpha &= PLANE_COLOR_ALPHA_MASK;
8920         } else {
8921                 alpha = val & PLANE_CTL_ALPHA_MASK;
8922         }
8923
8924         fourcc = skl_format_to_fourcc(pixel_format,
8925                                       val & PLANE_CTL_ORDER_RGBX, alpha);
8926         fb->format = drm_format_info(fourcc);
8927
8928         tiling = val & PLANE_CTL_TILED_MASK;
8929         switch (tiling) {
8930         case PLANE_CTL_TILED_LINEAR:
8931                 fb->modifier = DRM_FORMAT_MOD_LINEAR;
8932                 break;
8933         case PLANE_CTL_TILED_X:
8934                 plane_config->tiling = I915_TILING_X;
8935                 fb->modifier = I915_FORMAT_MOD_X_TILED;
8936                 break;
8937         case PLANE_CTL_TILED_Y:
8938                 plane_config->tiling = I915_TILING_Y;
8939                 if (val & PLANE_CTL_RENDER_DECOMPRESSION_ENABLE)
8940                         fb->modifier = I915_FORMAT_MOD_Y_TILED_CCS;
8941                 else
8942                         fb->modifier = I915_FORMAT_MOD_Y_TILED;
8943                 break;
8944         case PLANE_CTL_TILED_YF:
8945                 if (val & PLANE_CTL_RENDER_DECOMPRESSION_ENABLE)
8946                         fb->modifier = I915_FORMAT_MOD_Yf_TILED_CCS;
8947                 else
8948                         fb->modifier = I915_FORMAT_MOD_Yf_TILED;
8949                 break;
8950         default:
8951                 MISSING_CASE(tiling);
8952                 goto error;
8953         }
8954
8955         base = I915_READ(PLANE_SURF(pipe, plane_id)) & 0xfffff000;
8956         plane_config->base = base;
8957
8958         offset = I915_READ(PLANE_OFFSET(pipe, plane_id));
8959
8960         val = I915_READ(PLANE_SIZE(pipe, plane_id));
8961         fb->height = ((val >> 16) & 0xfff) + 1;
8962         fb->width = ((val >> 0) & 0x1fff) + 1;
8963
8964         val = I915_READ(PLANE_STRIDE(pipe, plane_id));
8965         stride_mult = intel_fb_stride_alignment(fb, 0);
8966         fb->pitches[0] = (val & 0x3ff) * stride_mult;
8967
8968         aligned_height = intel_fb_align_height(fb, 0, fb->height);
8969
8970         plane_config->size = fb->pitches[0] * aligned_height;
8971
8972         DRM_DEBUG_KMS("%s/%s with fb: size=%dx%d@%d, offset=%x, pitch %d, size 0x%x\n",
8973                       crtc->base.name, plane->base.name, fb->width, fb->height,
8974                       fb->format->cpp[0] * 8, base, fb->pitches[0],
8975                       plane_config->size);
8976
8977         plane_config->fb = intel_fb;
8978         return;
8979
8980 error:
8981         kfree(intel_fb);
8982 }
8983
8984 static void ironlake_get_pfit_config(struct intel_crtc *crtc,
8985                                      struct intel_crtc_state *pipe_config)
8986 {
8987         struct drm_device *dev = crtc->base.dev;
8988         struct drm_i915_private *dev_priv = to_i915(dev);
8989         uint32_t tmp;
8990
8991         tmp = I915_READ(PF_CTL(crtc->pipe));
8992
8993         if (tmp & PF_ENABLE) {
8994                 pipe_config->pch_pfit.enabled = true;
8995                 pipe_config->pch_pfit.pos = I915_READ(PF_WIN_POS(crtc->pipe));
8996                 pipe_config->pch_pfit.size = I915_READ(PF_WIN_SZ(crtc->pipe));
8997
8998                 /* We currently do not free assignements of panel fitters on
8999                  * ivb/hsw (since we don't use the higher upscaling modes which
9000                  * differentiates them) so just WARN about this case for now. */
9001                 if (IS_GEN7(dev_priv)) {
9002                         WARN_ON((tmp & PF_PIPE_SEL_MASK_IVB) !=
9003                                 PF_PIPE_SEL_IVB(crtc->pipe));
9004                 }
9005         }
9006 }
9007
9008 static bool ironlake_get_pipe_config(struct intel_crtc *crtc,
9009                                      struct intel_crtc_state *pipe_config)
9010 {
9011         struct drm_device *dev = crtc->base.dev;
9012         struct drm_i915_private *dev_priv = to_i915(dev);
9013         enum intel_display_power_domain power_domain;
9014         uint32_t tmp;
9015         bool ret;
9016
9017         power_domain = POWER_DOMAIN_PIPE(crtc->pipe);
9018         if (!intel_display_power_get_if_enabled(dev_priv, power_domain))
9019                 return false;
9020
9021         pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe;
9022         pipe_config->shared_dpll = NULL;
9023
9024         ret = false;
9025         tmp = I915_READ(PIPECONF(crtc->pipe));
9026         if (!(tmp & PIPECONF_ENABLE))
9027                 goto out;
9028
9029         switch (tmp & PIPECONF_BPC_MASK) {
9030         case PIPECONF_6BPC:
9031                 pipe_config->pipe_bpp = 18;
9032                 break;
9033         case PIPECONF_8BPC:
9034                 pipe_config->pipe_bpp = 24;
9035                 break;
9036         case PIPECONF_10BPC:
9037                 pipe_config->pipe_bpp = 30;
9038                 break;
9039         case PIPECONF_12BPC:
9040                 pipe_config->pipe_bpp = 36;
9041                 break;
9042         default:
9043                 break;
9044         }
9045
9046         if (tmp & PIPECONF_COLOR_RANGE_SELECT)
9047                 pipe_config->limited_color_range = true;
9048
9049         if (I915_READ(PCH_TRANSCONF(crtc->pipe)) & TRANS_ENABLE) {
9050                 struct intel_shared_dpll *pll;
9051                 enum intel_dpll_id pll_id;
9052
9053                 pipe_config->has_pch_encoder = true;
9054
9055                 tmp = I915_READ(FDI_RX_CTL(crtc->pipe));
9056                 pipe_config->fdi_lanes = ((FDI_DP_PORT_WIDTH_MASK & tmp) >>
9057                                           FDI_DP_PORT_WIDTH_SHIFT) + 1;
9058
9059                 ironlake_get_fdi_m_n_config(crtc, pipe_config);
9060
9061                 if (HAS_PCH_IBX(dev_priv)) {
9062                         /*
9063                          * The pipe->pch transcoder and pch transcoder->pll
9064                          * mapping is fixed.
9065                          */
9066                         pll_id = (enum intel_dpll_id) crtc->pipe;
9067                 } else {
9068                         tmp = I915_READ(PCH_DPLL_SEL);
9069                         if (tmp & TRANS_DPLLB_SEL(crtc->pipe))
9070                                 pll_id = DPLL_ID_PCH_PLL_B;
9071                         else
9072                                 pll_id= DPLL_ID_PCH_PLL_A;
9073                 }
9074
9075                 pipe_config->shared_dpll =
9076                         intel_get_shared_dpll_by_id(dev_priv, pll_id);
9077                 pll = pipe_config->shared_dpll;
9078
9079                 WARN_ON(!pll->info->funcs->get_hw_state(dev_priv, pll,
9080                                                 &pipe_config->dpll_hw_state));
9081
9082                 tmp = pipe_config->dpll_hw_state.dpll;
9083                 pipe_config->pixel_multiplier =
9084                         ((tmp & PLL_REF_SDVO_HDMI_MULTIPLIER_MASK)
9085                          >> PLL_REF_SDVO_HDMI_MULTIPLIER_SHIFT) + 1;
9086
9087                 ironlake_pch_clock_get(crtc, pipe_config);
9088         } else {
9089                 pipe_config->pixel_multiplier = 1;
9090         }
9091
9092         intel_get_pipe_timings(crtc, pipe_config);
9093         intel_get_pipe_src_size(crtc, pipe_config);
9094
9095         ironlake_get_pfit_config(crtc, pipe_config);
9096
9097         ret = true;
9098
9099 out:
9100         intel_display_power_put(dev_priv, power_domain);
9101
9102         return ret;
9103 }
9104
9105 static void assert_can_disable_lcpll(struct drm_i915_private *dev_priv)
9106 {
9107         struct drm_device *dev = &dev_priv->drm;
9108         struct intel_crtc *crtc;
9109
9110         for_each_intel_crtc(dev, crtc)
9111                 I915_STATE_WARN(crtc->active, "CRTC for pipe %c enabled\n",
9112                      pipe_name(crtc->pipe));
9113
9114         I915_STATE_WARN(I915_READ(HSW_PWR_WELL_CTL2),
9115                         "Display power well on\n");
9116         I915_STATE_WARN(I915_READ(SPLL_CTL) & SPLL_PLL_ENABLE, "SPLL enabled\n");
9117         I915_STATE_WARN(I915_READ(WRPLL_CTL(0)) & WRPLL_PLL_ENABLE, "WRPLL1 enabled\n");
9118         I915_STATE_WARN(I915_READ(WRPLL_CTL(1)) & WRPLL_PLL_ENABLE, "WRPLL2 enabled\n");
9119         I915_STATE_WARN(I915_READ(PP_STATUS(0)) & PP_ON, "Panel power on\n");
9120         I915_STATE_WARN(I915_READ(BLC_PWM_CPU_CTL2) & BLM_PWM_ENABLE,
9121              "CPU PWM1 enabled\n");
9122         if (IS_HASWELL(dev_priv))
9123                 I915_STATE_WARN(I915_READ(HSW_BLC_PWM2_CTL) & BLM_PWM_ENABLE,
9124                      "CPU PWM2 enabled\n");
9125         I915_STATE_WARN(I915_READ(BLC_PWM_PCH_CTL1) & BLM_PCH_PWM_ENABLE,
9126              "PCH PWM1 enabled\n");
9127         I915_STATE_WARN(I915_READ(UTIL_PIN_CTL) & UTIL_PIN_ENABLE,
9128              "Utility pin enabled\n");
9129         I915_STATE_WARN(I915_READ(PCH_GTC_CTL) & PCH_GTC_ENABLE, "PCH GTC enabled\n");
9130
9131         /*
9132          * In theory we can still leave IRQs enabled, as long as only the HPD
9133          * interrupts remain enabled. We used to check for that, but since it's
9134          * gen-specific and since we only disable LCPLL after we fully disable
9135          * the interrupts, the check below should be enough.
9136          */
9137         I915_STATE_WARN(intel_irqs_enabled(dev_priv), "IRQs enabled\n");
9138 }
9139
9140 static uint32_t hsw_read_dcomp(struct drm_i915_private *dev_priv)
9141 {
9142         if (IS_HASWELL(dev_priv))
9143                 return I915_READ(D_COMP_HSW);
9144         else
9145                 return I915_READ(D_COMP_BDW);
9146 }
9147
9148 static void hsw_write_dcomp(struct drm_i915_private *dev_priv, uint32_t val)
9149 {
9150         if (IS_HASWELL(dev_priv)) {
9151                 mutex_lock(&dev_priv->pcu_lock);
9152                 if (sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_D_COMP,
9153                                             val))
9154                         DRM_DEBUG_KMS("Failed to write to D_COMP\n");
9155                 mutex_unlock(&dev_priv->pcu_lock);
9156         } else {
9157                 I915_WRITE(D_COMP_BDW, val);
9158                 POSTING_READ(D_COMP_BDW);
9159         }
9160 }
9161
9162 /*
9163  * This function implements pieces of two sequences from BSpec:
9164  * - Sequence for display software to disable LCPLL
9165  * - Sequence for display software to allow package C8+
9166  * The steps implemented here are just the steps that actually touch the LCPLL
9167  * register. Callers should take care of disabling all the display engine
9168  * functions, doing the mode unset, fixing interrupts, etc.
9169  */
9170 static void hsw_disable_lcpll(struct drm_i915_private *dev_priv,
9171                               bool switch_to_fclk, bool allow_power_down)
9172 {
9173         uint32_t val;
9174
9175         assert_can_disable_lcpll(dev_priv);
9176
9177         val = I915_READ(LCPLL_CTL);
9178
9179         if (switch_to_fclk) {
9180                 val |= LCPLL_CD_SOURCE_FCLK;
9181                 I915_WRITE(LCPLL_CTL, val);
9182
9183                 if (wait_for_us(I915_READ(LCPLL_CTL) &
9184                                 LCPLL_CD_SOURCE_FCLK_DONE, 1))
9185                         DRM_ERROR("Switching to FCLK failed\n");
9186
9187                 val = I915_READ(LCPLL_CTL);
9188         }
9189
9190         val |= LCPLL_PLL_DISABLE;
9191         I915_WRITE(LCPLL_CTL, val);
9192         POSTING_READ(LCPLL_CTL);
9193
9194         if (intel_wait_for_register(dev_priv, LCPLL_CTL, LCPLL_PLL_LOCK, 0, 1))
9195                 DRM_ERROR("LCPLL still locked\n");
9196
9197         val = hsw_read_dcomp(dev_priv);
9198         val |= D_COMP_COMP_DISABLE;
9199         hsw_write_dcomp(dev_priv, val);
9200         ndelay(100);
9201
9202         if (wait_for((hsw_read_dcomp(dev_priv) & D_COMP_RCOMP_IN_PROGRESS) == 0,
9203                      1))
9204                 DRM_ERROR("D_COMP RCOMP still in progress\n");
9205
9206         if (allow_power_down) {
9207                 val = I915_READ(LCPLL_CTL);
9208                 val |= LCPLL_POWER_DOWN_ALLOW;
9209                 I915_WRITE(LCPLL_CTL, val);
9210                 POSTING_READ(LCPLL_CTL);
9211         }
9212 }
9213
9214 /*
9215  * Fully restores LCPLL, disallowing power down and switching back to LCPLL
9216  * source.
9217  */
9218 static void hsw_restore_lcpll(struct drm_i915_private *dev_priv)
9219 {
9220         uint32_t val;
9221
9222         val = I915_READ(LCPLL_CTL);
9223
9224         if ((val & (LCPLL_PLL_LOCK | LCPLL_PLL_DISABLE | LCPLL_CD_SOURCE_FCLK |
9225                     LCPLL_POWER_DOWN_ALLOW)) == LCPLL_PLL_LOCK)
9226                 return;
9227
9228         /*
9229          * Make sure we're not on PC8 state before disabling PC8, otherwise
9230          * we'll hang the machine. To prevent PC8 state, just enable force_wake.
9231          */
9232         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
9233
9234         if (val & LCPLL_POWER_DOWN_ALLOW) {
9235                 val &= ~LCPLL_POWER_DOWN_ALLOW;
9236                 I915_WRITE(LCPLL_CTL, val);
9237                 POSTING_READ(LCPLL_CTL);
9238         }
9239
9240         val = hsw_read_dcomp(dev_priv);
9241         val |= D_COMP_COMP_FORCE;
9242         val &= ~D_COMP_COMP_DISABLE;
9243         hsw_write_dcomp(dev_priv, val);
9244
9245         val = I915_READ(LCPLL_CTL);
9246         val &= ~LCPLL_PLL_DISABLE;
9247         I915_WRITE(LCPLL_CTL, val);
9248
9249         if (intel_wait_for_register(dev_priv,
9250                                     LCPLL_CTL, LCPLL_PLL_LOCK, LCPLL_PLL_LOCK,
9251                                     5))
9252                 DRM_ERROR("LCPLL not locked yet\n");
9253
9254         if (val & LCPLL_CD_SOURCE_FCLK) {
9255                 val = I915_READ(LCPLL_CTL);
9256                 val &= ~LCPLL_CD_SOURCE_FCLK;
9257                 I915_WRITE(LCPLL_CTL, val);
9258
9259                 if (wait_for_us((I915_READ(LCPLL_CTL) &
9260                                  LCPLL_CD_SOURCE_FCLK_DONE) == 0, 1))
9261                         DRM_ERROR("Switching back to LCPLL failed\n");
9262         }
9263
9264         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
9265
9266         intel_update_cdclk(dev_priv);
9267         intel_dump_cdclk_state(&dev_priv->cdclk.hw, "Current CDCLK");
9268 }
9269
9270 /*
9271  * Package states C8 and deeper are really deep PC states that can only be
9272  * reached when all the devices on the system allow it, so even if the graphics
9273  * device allows PC8+, it doesn't mean the system will actually get to these
9274  * states. Our driver only allows PC8+ when going into runtime PM.
9275  *
9276  * The requirements for PC8+ are that all the outputs are disabled, the power
9277  * well is disabled and most interrupts are disabled, and these are also
9278  * requirements for runtime PM. When these conditions are met, we manually do
9279  * the other conditions: disable the interrupts, clocks and switch LCPLL refclk
9280  * to Fclk. If we're in PC8+ and we get an non-hotplug interrupt, we can hard
9281  * hang the machine.
9282  *
9283  * When we really reach PC8 or deeper states (not just when we allow it) we lose
9284  * the state of some registers, so when we come back from PC8+ we need to
9285  * restore this state. We don't get into PC8+ if we're not in RC6, so we don't
9286  * need to take care of the registers kept by RC6. Notice that this happens even
9287  * if we don't put the device in PCI D3 state (which is what currently happens
9288  * because of the runtime PM support).
9289  *
9290  * For more, read "Display Sequences for Package C8" on the hardware
9291  * documentation.
9292  */
9293 void hsw_enable_pc8(struct drm_i915_private *dev_priv)
9294 {
9295         uint32_t val;
9296
9297         DRM_DEBUG_KMS("Enabling package C8+\n");
9298
9299         if (HAS_PCH_LPT_LP(dev_priv)) {
9300                 val = I915_READ(SOUTH_DSPCLK_GATE_D);
9301                 val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
9302                 I915_WRITE(SOUTH_DSPCLK_GATE_D, val);
9303         }
9304
9305         lpt_disable_clkout_dp(dev_priv);
9306         hsw_disable_lcpll(dev_priv, true, true);
9307 }
9308
9309 void hsw_disable_pc8(struct drm_i915_private *dev_priv)
9310 {
9311         uint32_t val;
9312
9313         DRM_DEBUG_KMS("Disabling package C8+\n");
9314
9315         hsw_restore_lcpll(dev_priv);
9316         lpt_init_pch_refclk(dev_priv);
9317
9318         if (HAS_PCH_LPT_LP(dev_priv)) {
9319                 val = I915_READ(SOUTH_DSPCLK_GATE_D);
9320                 val |= PCH_LP_PARTITION_LEVEL_DISABLE;
9321                 I915_WRITE(SOUTH_DSPCLK_GATE_D, val);
9322         }
9323 }
9324
9325 static int haswell_crtc_compute_clock(struct intel_crtc *crtc,
9326                                       struct intel_crtc_state *crtc_state)
9327 {
9328         struct intel_atomic_state *state =
9329                 to_intel_atomic_state(crtc_state->base.state);
9330
9331         if (!intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DSI)) {
9332                 struct intel_encoder *encoder =
9333                         intel_get_crtc_new_encoder(state, crtc_state);
9334
9335                 if (!intel_get_shared_dpll(crtc, crtc_state, encoder)) {
9336                         DRM_DEBUG_KMS("failed to find PLL for pipe %c\n",
9337                                       pipe_name(crtc->pipe));
9338                         return -EINVAL;
9339                 }
9340         }
9341
9342         return 0;
9343 }
9344
9345 static void cannonlake_get_ddi_pll(struct drm_i915_private *dev_priv,
9346                                    enum port port,
9347                                    struct intel_crtc_state *pipe_config)
9348 {
9349         enum intel_dpll_id id;
9350         u32 temp;
9351
9352         temp = I915_READ(DPCLKA_CFGCR0) & DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(port);
9353         id = temp >> DPCLKA_CFGCR0_DDI_CLK_SEL_SHIFT(port);
9354
9355         if (WARN_ON(id < SKL_DPLL0 || id > SKL_DPLL2))
9356                 return;
9357
9358         pipe_config->shared_dpll = intel_get_shared_dpll_by_id(dev_priv, id);
9359 }
9360
9361 static void icelake_get_ddi_pll(struct drm_i915_private *dev_priv,
9362                                 enum port port,
9363                                 struct intel_crtc_state *pipe_config)
9364 {
9365         enum intel_dpll_id id;
9366         u32 temp;
9367
9368         /* TODO: TBT pll not implemented. */
9369         switch (port) {
9370         case PORT_A:
9371         case PORT_B:
9372                 temp = I915_READ(DPCLKA_CFGCR0_ICL) &
9373                        DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(port);
9374                 id = temp >> DPCLKA_CFGCR0_DDI_CLK_SEL_SHIFT(port);
9375
9376                 if (WARN_ON(id != DPLL_ID_ICL_DPLL0 && id != DPLL_ID_ICL_DPLL1))
9377                         return;
9378                 break;
9379         case PORT_C:
9380                 id = DPLL_ID_ICL_MGPLL1;
9381                 break;
9382         case PORT_D:
9383                 id = DPLL_ID_ICL_MGPLL2;
9384                 break;
9385         case PORT_E:
9386                 id = DPLL_ID_ICL_MGPLL3;
9387                 break;
9388         case PORT_F:
9389                 id = DPLL_ID_ICL_MGPLL4;
9390                 break;
9391         default:
9392                 MISSING_CASE(port);
9393                 return;
9394         }
9395
9396         pipe_config->shared_dpll = intel_get_shared_dpll_by_id(dev_priv, id);
9397 }
9398
9399 static void bxt_get_ddi_pll(struct drm_i915_private *dev_priv,
9400                                 enum port port,
9401                                 struct intel_crtc_state *pipe_config)
9402 {
9403         enum intel_dpll_id id;
9404
9405         switch (port) {
9406         case PORT_A:
9407                 id = DPLL_ID_SKL_DPLL0;
9408                 break;
9409         case PORT_B:
9410                 id = DPLL_ID_SKL_DPLL1;
9411                 break;
9412         case PORT_C:
9413                 id = DPLL_ID_SKL_DPLL2;
9414                 break;
9415         default:
9416                 DRM_ERROR("Incorrect port type\n");
9417                 return;
9418         }
9419
9420         pipe_config->shared_dpll = intel_get_shared_dpll_by_id(dev_priv, id);
9421 }
9422
9423 static void skylake_get_ddi_pll(struct drm_i915_private *dev_priv,
9424                                 enum port port,
9425                                 struct intel_crtc_state *pipe_config)
9426 {
9427         enum intel_dpll_id id;
9428         u32 temp;
9429
9430         temp = I915_READ(DPLL_CTRL2) & DPLL_CTRL2_DDI_CLK_SEL_MASK(port);
9431         id = temp >> (port * 3 + 1);
9432
9433         if (WARN_ON(id < SKL_DPLL0 || id > SKL_DPLL3))
9434                 return;
9435
9436         pipe_config->shared_dpll = intel_get_shared_dpll_by_id(dev_priv, id);
9437 }
9438
9439 static void haswell_get_ddi_pll(struct drm_i915_private *dev_priv,
9440                                 enum port port,
9441                                 struct intel_crtc_state *pipe_config)
9442 {
9443         enum intel_dpll_id id;
9444         uint32_t ddi_pll_sel = I915_READ(PORT_CLK_SEL(port));
9445
9446         switch (ddi_pll_sel) {
9447         case PORT_CLK_SEL_WRPLL1:
9448                 id = DPLL_ID_WRPLL1;
9449                 break;
9450         case PORT_CLK_SEL_WRPLL2:
9451                 id = DPLL_ID_WRPLL2;
9452                 break;
9453         case PORT_CLK_SEL_SPLL:
9454                 id = DPLL_ID_SPLL;
9455                 break;
9456         case PORT_CLK_SEL_LCPLL_810:
9457                 id = DPLL_ID_LCPLL_810;
9458                 break;
9459         case PORT_CLK_SEL_LCPLL_1350:
9460                 id = DPLL_ID_LCPLL_1350;
9461                 break;
9462         case PORT_CLK_SEL_LCPLL_2700:
9463                 id = DPLL_ID_LCPLL_2700;
9464                 break;
9465         default:
9466                 MISSING_CASE(ddi_pll_sel);
9467                 /* fall through */
9468         case PORT_CLK_SEL_NONE:
9469                 return;
9470         }
9471
9472         pipe_config->shared_dpll = intel_get_shared_dpll_by_id(dev_priv, id);
9473 }
9474
9475 static bool hsw_get_transcoder_state(struct intel_crtc *crtc,
9476                                      struct intel_crtc_state *pipe_config,
9477                                      u64 *power_domain_mask)
9478 {
9479         struct drm_device *dev = crtc->base.dev;
9480         struct drm_i915_private *dev_priv = to_i915(dev);
9481         enum intel_display_power_domain power_domain;
9482         u32 tmp;
9483
9484         /*
9485          * The pipe->transcoder mapping is fixed with the exception of the eDP
9486          * transcoder handled below.
9487          */
9488         pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe;
9489
9490         /*
9491          * XXX: Do intel_display_power_get_if_enabled before reading this (for
9492          * consistency and less surprising code; it's in always on power).
9493          */
9494         tmp = I915_READ(TRANS_DDI_FUNC_CTL(TRANSCODER_EDP));
9495         if (tmp & TRANS_DDI_FUNC_ENABLE) {
9496                 enum pipe trans_edp_pipe;
9497                 switch (tmp & TRANS_DDI_EDP_INPUT_MASK) {
9498                 default:
9499                         WARN(1, "unknown pipe linked to edp transcoder\n");
9500                         /* fall through */
9501                 case TRANS_DDI_EDP_INPUT_A_ONOFF:
9502                 case TRANS_DDI_EDP_INPUT_A_ON:
9503                         trans_edp_pipe = PIPE_A;
9504                         break;
9505                 case TRANS_DDI_EDP_INPUT_B_ONOFF:
9506                         trans_edp_pipe = PIPE_B;
9507                         break;
9508                 case TRANS_DDI_EDP_INPUT_C_ONOFF:
9509                         trans_edp_pipe = PIPE_C;
9510                         break;
9511                 }
9512
9513                 if (trans_edp_pipe == crtc->pipe)
9514                         pipe_config->cpu_transcoder = TRANSCODER_EDP;
9515         }
9516
9517         power_domain = POWER_DOMAIN_TRANSCODER(pipe_config->cpu_transcoder);
9518         if (!intel_display_power_get_if_enabled(dev_priv, power_domain))
9519                 return false;
9520         *power_domain_mask |= BIT_ULL(power_domain);
9521
9522         tmp = I915_READ(PIPECONF(pipe_config->cpu_transcoder));
9523
9524         return tmp & PIPECONF_ENABLE;
9525 }
9526
9527 static bool bxt_get_dsi_transcoder_state(struct intel_crtc *crtc,
9528                                          struct intel_crtc_state *pipe_config,
9529                                          u64 *power_domain_mask)
9530 {
9531         struct drm_device *dev = crtc->base.dev;
9532         struct drm_i915_private *dev_priv = to_i915(dev);
9533         enum intel_display_power_domain power_domain;
9534         enum port port;
9535         enum transcoder cpu_transcoder;
9536         u32 tmp;
9537
9538         for_each_port_masked(port, BIT(PORT_A) | BIT(PORT_C)) {
9539                 if (port == PORT_A)
9540                         cpu_transcoder = TRANSCODER_DSI_A;
9541                 else
9542                         cpu_transcoder = TRANSCODER_DSI_C;
9543
9544                 power_domain = POWER_DOMAIN_TRANSCODER(cpu_transcoder);
9545                 if (!intel_display_power_get_if_enabled(dev_priv, power_domain))
9546                         continue;
9547                 *power_domain_mask |= BIT_ULL(power_domain);
9548
9549                 /*
9550                  * The PLL needs to be enabled with a valid divider
9551                  * configuration, otherwise accessing DSI registers will hang
9552                  * the machine. See BSpec North Display Engine
9553                  * registers/MIPI[BXT]. We can break out here early, since we
9554                  * need the same DSI PLL to be enabled for both DSI ports.
9555                  */
9556                 if (!bxt_dsi_pll_is_enabled(dev_priv))
9557                         break;
9558
9559                 /* XXX: this works for video mode only */
9560                 tmp = I915_READ(BXT_MIPI_PORT_CTRL(port));
9561                 if (!(tmp & DPI_ENABLE))
9562                         continue;
9563
9564                 tmp = I915_READ(MIPI_CTRL(port));
9565                 if ((tmp & BXT_PIPE_SELECT_MASK) != BXT_PIPE_SELECT(crtc->pipe))
9566                         continue;
9567
9568                 pipe_config->cpu_transcoder = cpu_transcoder;
9569                 break;
9570         }
9571
9572         return transcoder_is_dsi(pipe_config->cpu_transcoder);
9573 }
9574
9575 static void haswell_get_ddi_port_state(struct intel_crtc *crtc,
9576                                        struct intel_crtc_state *pipe_config)
9577 {
9578         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9579         struct intel_shared_dpll *pll;
9580         enum port port;
9581         uint32_t tmp;
9582
9583         tmp = I915_READ(TRANS_DDI_FUNC_CTL(pipe_config->cpu_transcoder));
9584
9585         port = (tmp & TRANS_DDI_PORT_MASK) >> TRANS_DDI_PORT_SHIFT;
9586
9587         if (IS_ICELAKE(dev_priv))
9588                 icelake_get_ddi_pll(dev_priv, port, pipe_config);
9589         else if (IS_CANNONLAKE(dev_priv))
9590                 cannonlake_get_ddi_pll(dev_priv, port, pipe_config);
9591         else if (IS_GEN9_BC(dev_priv))
9592                 skylake_get_ddi_pll(dev_priv, port, pipe_config);
9593         else if (IS_GEN9_LP(dev_priv))
9594                 bxt_get_ddi_pll(dev_priv, port, pipe_config);
9595         else
9596                 haswell_get_ddi_pll(dev_priv, port, pipe_config);
9597
9598         pll = pipe_config->shared_dpll;
9599         if (pll) {
9600                 WARN_ON(!pll->info->funcs->get_hw_state(dev_priv, pll,
9601                                                 &pipe_config->dpll_hw_state));
9602         }
9603
9604         /*
9605          * Haswell has only FDI/PCH transcoder A. It is which is connected to
9606          * DDI E. So just check whether this pipe is wired to DDI E and whether
9607          * the PCH transcoder is on.
9608          */
9609         if (INTEL_GEN(dev_priv) < 9 &&
9610             (port == PORT_E) && I915_READ(LPT_TRANSCONF) & TRANS_ENABLE) {
9611                 pipe_config->has_pch_encoder = true;
9612
9613                 tmp = I915_READ(FDI_RX_CTL(PIPE_A));
9614                 pipe_config->fdi_lanes = ((FDI_DP_PORT_WIDTH_MASK & tmp) >>
9615                                           FDI_DP_PORT_WIDTH_SHIFT) + 1;
9616
9617                 ironlake_get_fdi_m_n_config(crtc, pipe_config);
9618         }
9619 }
9620
9621 static bool haswell_get_pipe_config(struct intel_crtc *crtc,
9622                                     struct intel_crtc_state *pipe_config)
9623 {
9624         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9625         enum intel_display_power_domain power_domain;
9626         u64 power_domain_mask;
9627         bool active;
9628
9629         intel_crtc_init_scalers(crtc, pipe_config);
9630
9631         power_domain = POWER_DOMAIN_PIPE(crtc->pipe);
9632         if (!intel_display_power_get_if_enabled(dev_priv, power_domain))
9633                 return false;
9634         power_domain_mask = BIT_ULL(power_domain);
9635
9636         pipe_config->shared_dpll = NULL;
9637
9638         active = hsw_get_transcoder_state(crtc, pipe_config, &power_domain_mask);
9639
9640         if (IS_GEN9_LP(dev_priv) &&
9641             bxt_get_dsi_transcoder_state(crtc, pipe_config, &power_domain_mask)) {
9642                 WARN_ON(active);
9643                 active = true;
9644         }
9645
9646         if (!active)
9647                 goto out;
9648
9649         if (!transcoder_is_dsi(pipe_config->cpu_transcoder)) {
9650                 haswell_get_ddi_port_state(crtc, pipe_config);
9651                 intel_get_pipe_timings(crtc, pipe_config);
9652         }
9653
9654         intel_get_pipe_src_size(crtc, pipe_config);
9655
9656         pipe_config->gamma_mode =
9657                 I915_READ(GAMMA_MODE(crtc->pipe)) & GAMMA_MODE_MODE_MASK;
9658
9659         if (IS_BROADWELL(dev_priv) || INTEL_GEN(dev_priv) >= 9) {
9660                 u32 tmp = I915_READ(PIPEMISC(crtc->pipe));
9661                 bool clrspace_yuv = tmp & PIPEMISC_OUTPUT_COLORSPACE_YUV;
9662
9663                 if (IS_GEMINILAKE(dev_priv) || INTEL_GEN(dev_priv) >= 10) {
9664                         bool blend_mode_420 = tmp &
9665                                               PIPEMISC_YUV420_MODE_FULL_BLEND;
9666
9667                         pipe_config->ycbcr420 = tmp & PIPEMISC_YUV420_ENABLE;
9668                         if (pipe_config->ycbcr420 != clrspace_yuv ||
9669                             pipe_config->ycbcr420 != blend_mode_420)
9670                                 DRM_DEBUG_KMS("Bad 4:2:0 mode (%08x)\n", tmp);
9671                 } else if (clrspace_yuv) {
9672                         DRM_DEBUG_KMS("YCbCr 4:2:0 Unsupported\n");
9673                 }
9674         }
9675
9676         power_domain = POWER_DOMAIN_PIPE_PANEL_FITTER(crtc->pipe);
9677         if (intel_display_power_get_if_enabled(dev_priv, power_domain)) {
9678                 power_domain_mask |= BIT_ULL(power_domain);
9679                 if (INTEL_GEN(dev_priv) >= 9)
9680                         skylake_get_pfit_config(crtc, pipe_config);
9681                 else
9682                         ironlake_get_pfit_config(crtc, pipe_config);
9683         }
9684
9685         if (hsw_crtc_supports_ips(crtc)) {
9686                 if (IS_HASWELL(dev_priv))
9687                         pipe_config->ips_enabled = I915_READ(IPS_CTL) & IPS_ENABLE;
9688                 else {
9689                         /*
9690                          * We cannot readout IPS state on broadwell, set to
9691                          * true so we can set it to a defined state on first
9692                          * commit.
9693                          */
9694                         pipe_config->ips_enabled = true;
9695                 }
9696         }
9697
9698         if (pipe_config->cpu_transcoder != TRANSCODER_EDP &&
9699             !transcoder_is_dsi(pipe_config->cpu_transcoder)) {
9700                 pipe_config->pixel_multiplier =
9701                         I915_READ(PIPE_MULT(pipe_config->cpu_transcoder)) + 1;
9702         } else {
9703                 pipe_config->pixel_multiplier = 1;
9704         }
9705
9706 out:
9707         for_each_power_domain(power_domain, power_domain_mask)
9708                 intel_display_power_put(dev_priv, power_domain);
9709
9710         return active;
9711 }
9712
9713 static u32 intel_cursor_base(const struct intel_plane_state *plane_state)
9714 {
9715         struct drm_i915_private *dev_priv =
9716                 to_i915(plane_state->base.plane->dev);
9717         const struct drm_framebuffer *fb = plane_state->base.fb;
9718         const struct drm_i915_gem_object *obj = intel_fb_obj(fb);
9719         u32 base;
9720
9721         if (INTEL_INFO(dev_priv)->cursor_needs_physical)
9722                 base = obj->phys_handle->busaddr;
9723         else
9724                 base = intel_plane_ggtt_offset(plane_state);
9725
9726         base += plane_state->color_plane[0].offset;
9727
9728         /* ILK+ do this automagically */
9729         if (HAS_GMCH_DISPLAY(dev_priv) &&
9730             plane_state->base.rotation & DRM_MODE_ROTATE_180)
9731                 base += (plane_state->base.crtc_h *
9732                          plane_state->base.crtc_w - 1) * fb->format->cpp[0];
9733
9734         return base;
9735 }
9736
9737 static u32 intel_cursor_position(const struct intel_plane_state *plane_state)
9738 {
9739         int x = plane_state->base.crtc_x;
9740         int y = plane_state->base.crtc_y;
9741         u32 pos = 0;
9742
9743         if (x < 0) {
9744                 pos |= CURSOR_POS_SIGN << CURSOR_X_SHIFT;
9745                 x = -x;
9746         }
9747         pos |= x << CURSOR_X_SHIFT;
9748
9749         if (y < 0) {
9750                 pos |= CURSOR_POS_SIGN << CURSOR_Y_SHIFT;
9751                 y = -y;
9752         }
9753         pos |= y << CURSOR_Y_SHIFT;
9754
9755         return pos;
9756 }
9757
9758 static bool intel_cursor_size_ok(const struct intel_plane_state *plane_state)
9759 {
9760         const struct drm_mode_config *config =
9761                 &plane_state->base.plane->dev->mode_config;
9762         int width = plane_state->base.crtc_w;
9763         int height = plane_state->base.crtc_h;
9764
9765         return width > 0 && width <= config->cursor_width &&
9766                 height > 0 && height <= config->cursor_height;
9767 }
9768
9769 static int intel_cursor_check_surface(struct intel_plane_state *plane_state)
9770 {
9771         const struct drm_framebuffer *fb = plane_state->base.fb;
9772         unsigned int rotation = plane_state->base.rotation;
9773         int src_x, src_y;
9774         u32 offset;
9775         int ret;
9776
9777         intel_fill_fb_ggtt_view(&plane_state->view, fb, rotation);
9778         plane_state->color_plane[0].stride = intel_fb_pitch(fb, 0, rotation);
9779
9780         ret = intel_plane_check_stride(plane_state);
9781         if (ret)
9782                 return ret;
9783
9784         src_x = plane_state->base.src_x >> 16;
9785         src_y = plane_state->base.src_y >> 16;
9786
9787         intel_add_fb_offsets(&src_x, &src_y, plane_state, 0);
9788         offset = intel_plane_compute_aligned_offset(&src_x, &src_y,
9789                                                     plane_state, 0);
9790
9791         if (src_x != 0 || src_y != 0) {
9792                 DRM_DEBUG_KMS("Arbitrary cursor panning not supported\n");
9793                 return -EINVAL;
9794         }
9795
9796         plane_state->color_plane[0].offset = offset;
9797
9798         return 0;
9799 }
9800
9801 static int intel_check_cursor(struct intel_crtc_state *crtc_state,
9802                               struct intel_plane_state *plane_state)
9803 {
9804         const struct drm_framebuffer *fb = plane_state->base.fb;
9805         int ret;
9806
9807         if (fb && fb->modifier != DRM_FORMAT_MOD_LINEAR) {
9808                 DRM_DEBUG_KMS("cursor cannot be tiled\n");
9809                 return -EINVAL;
9810         }
9811
9812         ret = drm_atomic_helper_check_plane_state(&plane_state->base,
9813                                                   &crtc_state->base,
9814                                                   DRM_PLANE_HELPER_NO_SCALING,
9815                                                   DRM_PLANE_HELPER_NO_SCALING,
9816                                                   true, true);
9817         if (ret)
9818                 return ret;
9819
9820         if (!plane_state->base.visible)
9821                 return 0;
9822
9823         ret = intel_plane_check_src_coordinates(plane_state);
9824         if (ret)
9825                 return ret;
9826
9827         ret = intel_cursor_check_surface(plane_state);
9828         if (ret)
9829                 return ret;
9830
9831         return 0;
9832 }
9833
9834 static unsigned int
9835 i845_cursor_max_stride(struct intel_plane *plane,
9836                        u32 pixel_format, u64 modifier,
9837                        unsigned int rotation)
9838 {
9839         return 2048;
9840 }
9841
9842 static u32 i845_cursor_ctl(const struct intel_crtc_state *crtc_state,
9843                            const struct intel_plane_state *plane_state)
9844 {
9845         return CURSOR_ENABLE |
9846                 CURSOR_GAMMA_ENABLE |
9847                 CURSOR_FORMAT_ARGB |
9848                 CURSOR_STRIDE(plane_state->color_plane[0].stride);
9849 }
9850
9851 static bool i845_cursor_size_ok(const struct intel_plane_state *plane_state)
9852 {
9853         int width = plane_state->base.crtc_w;
9854
9855         /*
9856          * 845g/865g are only limited by the width of their cursors,
9857          * the height is arbitrary up to the precision of the register.
9858          */
9859         return intel_cursor_size_ok(plane_state) && IS_ALIGNED(width, 64);
9860 }
9861
9862 static int i845_check_cursor(struct intel_crtc_state *crtc_state,
9863                              struct intel_plane_state *plane_state)
9864 {
9865         const struct drm_framebuffer *fb = plane_state->base.fb;
9866         int ret;
9867
9868         ret = intel_check_cursor(crtc_state, plane_state);
9869         if (ret)
9870                 return ret;
9871
9872         /* if we want to turn off the cursor ignore width and height */
9873         if (!fb)
9874                 return 0;
9875
9876         /* Check for which cursor types we support */
9877         if (!i845_cursor_size_ok(plane_state)) {
9878                 DRM_DEBUG("Cursor dimension %dx%d not supported\n",
9879                           plane_state->base.crtc_w,
9880                           plane_state->base.crtc_h);
9881                 return -EINVAL;
9882         }
9883
9884         WARN_ON(plane_state->base.visible &&
9885                 plane_state->color_plane[0].stride != fb->pitches[0]);
9886
9887         switch (fb->pitches[0]) {
9888         case 256:
9889         case 512:
9890         case 1024:
9891         case 2048:
9892                 break;
9893         default:
9894                 DRM_DEBUG_KMS("Invalid cursor stride (%u)\n",
9895                               fb->pitches[0]);
9896                 return -EINVAL;
9897         }
9898
9899         plane_state->ctl = i845_cursor_ctl(crtc_state, plane_state);
9900
9901         return 0;
9902 }
9903
9904 static void i845_update_cursor(struct intel_plane *plane,
9905                                const struct intel_crtc_state *crtc_state,
9906                                const struct intel_plane_state *plane_state)
9907 {
9908         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
9909         u32 cntl = 0, base = 0, pos = 0, size = 0;
9910         unsigned long irqflags;
9911
9912         if (plane_state && plane_state->base.visible) {
9913                 unsigned int width = plane_state->base.crtc_w;
9914                 unsigned int height = plane_state->base.crtc_h;
9915
9916                 cntl = plane_state->ctl;
9917                 size = (height << 12) | width;
9918
9919                 base = intel_cursor_base(plane_state);
9920                 pos = intel_cursor_position(plane_state);
9921         }
9922
9923         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
9924
9925         /* On these chipsets we can only modify the base/size/stride
9926          * whilst the cursor is disabled.
9927          */
9928         if (plane->cursor.base != base ||
9929             plane->cursor.size != size ||
9930             plane->cursor.cntl != cntl) {
9931                 I915_WRITE_FW(CURCNTR(PIPE_A), 0);
9932                 I915_WRITE_FW(CURBASE(PIPE_A), base);
9933                 I915_WRITE_FW(CURSIZE, size);
9934                 I915_WRITE_FW(CURPOS(PIPE_A), pos);
9935                 I915_WRITE_FW(CURCNTR(PIPE_A), cntl);
9936
9937                 plane->cursor.base = base;
9938                 plane->cursor.size = size;
9939                 plane->cursor.cntl = cntl;
9940         } else {
9941                 I915_WRITE_FW(CURPOS(PIPE_A), pos);
9942         }
9943
9944         POSTING_READ_FW(CURCNTR(PIPE_A));
9945
9946         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
9947 }
9948
9949 static void i845_disable_cursor(struct intel_plane *plane,
9950                                 struct intel_crtc *crtc)
9951 {
9952         i845_update_cursor(plane, NULL, NULL);
9953 }
9954
9955 static bool i845_cursor_get_hw_state(struct intel_plane *plane,
9956                                      enum pipe *pipe)
9957 {
9958         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
9959         enum intel_display_power_domain power_domain;
9960         bool ret;
9961
9962         power_domain = POWER_DOMAIN_PIPE(PIPE_A);
9963         if (!intel_display_power_get_if_enabled(dev_priv, power_domain))
9964                 return false;
9965
9966         ret = I915_READ(CURCNTR(PIPE_A)) & CURSOR_ENABLE;
9967
9968         *pipe = PIPE_A;
9969
9970         intel_display_power_put(dev_priv, power_domain);
9971
9972         return ret;
9973 }
9974
9975 static unsigned int
9976 i9xx_cursor_max_stride(struct intel_plane *plane,
9977                        u32 pixel_format, u64 modifier,
9978                        unsigned int rotation)
9979 {
9980         return plane->base.dev->mode_config.cursor_width * 4;
9981 }
9982
9983 static u32 i9xx_cursor_ctl(const struct intel_crtc_state *crtc_state,
9984                            const struct intel_plane_state *plane_state)
9985 {
9986         struct drm_i915_private *dev_priv =
9987                 to_i915(plane_state->base.plane->dev);
9988         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
9989         u32 cntl = 0;
9990
9991         if (IS_GEN6(dev_priv) || IS_IVYBRIDGE(dev_priv))
9992                 cntl |= MCURSOR_TRICKLE_FEED_DISABLE;
9993
9994         if (INTEL_GEN(dev_priv) <= 10) {
9995                 cntl |= MCURSOR_GAMMA_ENABLE;
9996
9997                 if (HAS_DDI(dev_priv))
9998                         cntl |= MCURSOR_PIPE_CSC_ENABLE;
9999         }
10000
10001         if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv))
10002                 cntl |= MCURSOR_PIPE_SELECT(crtc->pipe);
10003
10004         switch (plane_state->base.crtc_w) {
10005         case 64:
10006                 cntl |= MCURSOR_MODE_64_ARGB_AX;
10007                 break;
10008         case 128:
10009                 cntl |= MCURSOR_MODE_128_ARGB_AX;
10010                 break;
10011         case 256:
10012                 cntl |= MCURSOR_MODE_256_ARGB_AX;
10013                 break;
10014         default:
10015                 MISSING_CASE(plane_state->base.crtc_w);
10016                 return 0;
10017         }
10018
10019         if (plane_state->base.rotation & DRM_MODE_ROTATE_180)
10020                 cntl |= MCURSOR_ROTATE_180;
10021
10022         return cntl;
10023 }
10024
10025 static bool i9xx_cursor_size_ok(const struct intel_plane_state *plane_state)
10026 {
10027         struct drm_i915_private *dev_priv =
10028                 to_i915(plane_state->base.plane->dev);
10029         int width = plane_state->base.crtc_w;
10030         int height = plane_state->base.crtc_h;
10031
10032         if (!intel_cursor_size_ok(plane_state))
10033                 return false;
10034
10035         /* Cursor width is limited to a few power-of-two sizes */
10036         switch (width) {
10037         case 256:
10038         case 128:
10039         case 64:
10040                 break;
10041         default:
10042                 return false;
10043         }
10044
10045         /*
10046          * IVB+ have CUR_FBC_CTL which allows an arbitrary cursor
10047          * height from 8 lines up to the cursor width, when the
10048          * cursor is not rotated. Everything else requires square
10049          * cursors.
10050          */
10051         if (HAS_CUR_FBC(dev_priv) &&
10052             plane_state->base.rotation & DRM_MODE_ROTATE_0) {
10053                 if (height < 8 || height > width)
10054                         return false;
10055         } else {
10056                 if (height != width)
10057                         return false;
10058         }
10059
10060         return true;
10061 }
10062
10063 static int i9xx_check_cursor(struct intel_crtc_state *crtc_state,
10064                              struct intel_plane_state *plane_state)
10065 {
10066         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
10067         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
10068         const struct drm_framebuffer *fb = plane_state->base.fb;
10069         enum pipe pipe = plane->pipe;
10070         int ret;
10071
10072         ret = intel_check_cursor(crtc_state, plane_state);
10073         if (ret)
10074                 return ret;
10075
10076         /* if we want to turn off the cursor ignore width and height */
10077         if (!fb)
10078                 return 0;
10079
10080         /* Check for which cursor types we support */
10081         if (!i9xx_cursor_size_ok(plane_state)) {
10082                 DRM_DEBUG("Cursor dimension %dx%d not supported\n",
10083                           plane_state->base.crtc_w,
10084                           plane_state->base.crtc_h);
10085                 return -EINVAL;
10086         }
10087
10088         WARN_ON(plane_state->base.visible &&
10089                 plane_state->color_plane[0].stride != fb->pitches[0]);
10090
10091         if (fb->pitches[0] != plane_state->base.crtc_w * fb->format->cpp[0]) {
10092                 DRM_DEBUG_KMS("Invalid cursor stride (%u) (cursor width %d)\n",
10093                               fb->pitches[0], plane_state->base.crtc_w);
10094                 return -EINVAL;
10095         }
10096
10097         /*
10098          * There's something wrong with the cursor on CHV pipe C.
10099          * If it straddles the left edge of the screen then
10100          * moving it away from the edge or disabling it often
10101          * results in a pipe underrun, and often that can lead to
10102          * dead pipe (constant underrun reported, and it scans
10103          * out just a solid color). To recover from that, the
10104          * display power well must be turned off and on again.
10105          * Refuse the put the cursor into that compromised position.
10106          */
10107         if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_C &&
10108             plane_state->base.visible && plane_state->base.crtc_x < 0) {
10109                 DRM_DEBUG_KMS("CHV cursor C not allowed to straddle the left screen edge\n");
10110                 return -EINVAL;
10111         }
10112
10113         plane_state->ctl = i9xx_cursor_ctl(crtc_state, plane_state);
10114
10115         return 0;
10116 }
10117
10118 static void i9xx_update_cursor(struct intel_plane *plane,
10119                                const struct intel_crtc_state *crtc_state,
10120                                const struct intel_plane_state *plane_state)
10121 {
10122         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
10123         enum pipe pipe = plane->pipe;
10124         u32 cntl = 0, base = 0, pos = 0, fbc_ctl = 0;
10125         unsigned long irqflags;
10126
10127         if (plane_state && plane_state->base.visible) {
10128                 cntl = plane_state->ctl;
10129
10130                 if (plane_state->base.crtc_h != plane_state->base.crtc_w)
10131                         fbc_ctl = CUR_FBC_CTL_EN | (plane_state->base.crtc_h - 1);
10132
10133                 base = intel_cursor_base(plane_state);
10134                 pos = intel_cursor_position(plane_state);
10135         }
10136
10137         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
10138
10139         /*
10140          * On some platforms writing CURCNTR first will also
10141          * cause CURPOS to be armed by the CURBASE write.
10142          * Without the CURCNTR write the CURPOS write would
10143          * arm itself. Thus we always start the full update
10144          * with a CURCNTR write.
10145          *
10146          * On other platforms CURPOS always requires the
10147          * CURBASE write to arm the update. Additonally
10148          * a write to any of the cursor register will cancel
10149          * an already armed cursor update. Thus leaving out
10150          * the CURBASE write after CURPOS could lead to a
10151          * cursor that doesn't appear to move, or even change
10152          * shape. Thus we always write CURBASE.
10153          *
10154          * CURCNTR and CUR_FBC_CTL are always
10155          * armed by the CURBASE write only.
10156          */
10157         if (plane->cursor.base != base ||
10158             plane->cursor.size != fbc_ctl ||
10159             plane->cursor.cntl != cntl) {
10160                 I915_WRITE_FW(CURCNTR(pipe), cntl);
10161                 if (HAS_CUR_FBC(dev_priv))
10162                         I915_WRITE_FW(CUR_FBC_CTL(pipe), fbc_ctl);
10163                 I915_WRITE_FW(CURPOS(pipe), pos);
10164                 I915_WRITE_FW(CURBASE(pipe), base);
10165
10166                 plane->cursor.base = base;
10167                 plane->cursor.size = fbc_ctl;
10168                 plane->cursor.cntl = cntl;
10169         } else {
10170                 I915_WRITE_FW(CURPOS(pipe), pos);
10171                 I915_WRITE_FW(CURBASE(pipe), base);
10172         }
10173
10174         POSTING_READ_FW(CURBASE(pipe));
10175
10176         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
10177 }
10178
10179 static void i9xx_disable_cursor(struct intel_plane *plane,
10180                                 struct intel_crtc *crtc)
10181 {
10182         i9xx_update_cursor(plane, NULL, NULL);
10183 }
10184
10185 static bool i9xx_cursor_get_hw_state(struct intel_plane *plane,
10186                                      enum pipe *pipe)
10187 {
10188         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
10189         enum intel_display_power_domain power_domain;
10190         bool ret;
10191         u32 val;
10192
10193         /*
10194          * Not 100% correct for planes that can move between pipes,
10195          * but that's only the case for gen2-3 which don't have any
10196          * display power wells.
10197          */
10198         power_domain = POWER_DOMAIN_PIPE(plane->pipe);
10199         if (!intel_display_power_get_if_enabled(dev_priv, power_domain))
10200                 return false;
10201
10202         val = I915_READ(CURCNTR(plane->pipe));
10203
10204         ret = val & MCURSOR_MODE;
10205
10206         if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv))
10207                 *pipe = plane->pipe;
10208         else
10209                 *pipe = (val & MCURSOR_PIPE_SELECT_MASK) >>
10210                         MCURSOR_PIPE_SELECT_SHIFT;
10211
10212         intel_display_power_put(dev_priv, power_domain);
10213
10214         return ret;
10215 }
10216
10217 /* VESA 640x480x72Hz mode to set on the pipe */
10218 static const struct drm_display_mode load_detect_mode = {
10219         DRM_MODE("640x480", DRM_MODE_TYPE_DEFAULT, 31500, 640, 664,
10220                  704, 832, 0, 480, 489, 491, 520, 0, DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
10221 };
10222
10223 struct drm_framebuffer *
10224 intel_framebuffer_create(struct drm_i915_gem_object *obj,
10225                          struct drm_mode_fb_cmd2 *mode_cmd)
10226 {
10227         struct intel_framebuffer *intel_fb;
10228         int ret;
10229
10230         intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL);
10231         if (!intel_fb)
10232                 return ERR_PTR(-ENOMEM);
10233
10234         ret = intel_framebuffer_init(intel_fb, obj, mode_cmd);
10235         if (ret)
10236                 goto err;
10237
10238         return &intel_fb->base;
10239
10240 err:
10241         kfree(intel_fb);
10242         return ERR_PTR(ret);
10243 }
10244
10245 static int intel_modeset_disable_planes(struct drm_atomic_state *state,
10246                                         struct drm_crtc *crtc)
10247 {
10248         struct drm_plane *plane;
10249         struct drm_plane_state *plane_state;
10250         int ret, i;
10251
10252         ret = drm_atomic_add_affected_planes(state, crtc);
10253         if (ret)
10254                 return ret;
10255
10256         for_each_new_plane_in_state(state, plane, plane_state, i) {
10257                 if (plane_state->crtc != crtc)
10258                         continue;
10259
10260                 ret = drm_atomic_set_crtc_for_plane(plane_state, NULL);
10261                 if (ret)
10262                         return ret;
10263
10264                 drm_atomic_set_fb_for_plane(plane_state, NULL);
10265         }
10266
10267         return 0;
10268 }
10269
10270 int intel_get_load_detect_pipe(struct drm_connector *connector,
10271                                const struct drm_display_mode *mode,
10272                                struct intel_load_detect_pipe *old,
10273                                struct drm_modeset_acquire_ctx *ctx)
10274 {
10275         struct intel_crtc *intel_crtc;
10276         struct intel_encoder *intel_encoder =
10277                 intel_attached_encoder(connector);
10278         struct drm_crtc *possible_crtc;
10279         struct drm_encoder *encoder = &intel_encoder->base;
10280         struct drm_crtc *crtc = NULL;
10281         struct drm_device *dev = encoder->dev;
10282         struct drm_i915_private *dev_priv = to_i915(dev);
10283         struct drm_mode_config *config = &dev->mode_config;
10284         struct drm_atomic_state *state = NULL, *restore_state = NULL;
10285         struct drm_connector_state *connector_state;
10286         struct intel_crtc_state *crtc_state;
10287         int ret, i = -1;
10288
10289         DRM_DEBUG_KMS("[CONNECTOR:%d:%s], [ENCODER:%d:%s]\n",
10290                       connector->base.id, connector->name,
10291                       encoder->base.id, encoder->name);
10292
10293         old->restore_state = NULL;
10294
10295         WARN_ON(!drm_modeset_is_locked(&config->connection_mutex));
10296
10297         /*
10298          * Algorithm gets a little messy:
10299          *
10300          *   - if the connector already has an assigned crtc, use it (but make
10301          *     sure it's on first)
10302          *
10303          *   - try to find the first unused crtc that can drive this connector,
10304          *     and use that if we find one
10305          */
10306
10307         /* See if we already have a CRTC for this connector */
10308         if (connector->state->crtc) {
10309                 crtc = connector->state->crtc;
10310
10311                 ret = drm_modeset_lock(&crtc->mutex, ctx);
10312                 if (ret)
10313                         goto fail;
10314
10315                 /* Make sure the crtc and connector are running */
10316                 goto found;
10317         }
10318
10319         /* Find an unused one (if possible) */
10320         for_each_crtc(dev, possible_crtc) {
10321                 i++;
10322                 if (!(encoder->possible_crtcs & (1 << i)))
10323                         continue;
10324
10325                 ret = drm_modeset_lock(&possible_crtc->mutex, ctx);
10326                 if (ret)
10327                         goto fail;
10328
10329                 if (possible_crtc->state->enable) {
10330                         drm_modeset_unlock(&possible_crtc->mutex);
10331                         continue;
10332                 }
10333
10334                 crtc = possible_crtc;
10335                 break;
10336         }
10337
10338         /*
10339          * If we didn't find an unused CRTC, don't use any.
10340          */
10341         if (!crtc) {
10342                 DRM_DEBUG_KMS("no pipe available for load-detect\n");
10343                 ret = -ENODEV;
10344                 goto fail;
10345         }
10346
10347 found:
10348         intel_crtc = to_intel_crtc(crtc);
10349
10350         state = drm_atomic_state_alloc(dev);
10351         restore_state = drm_atomic_state_alloc(dev);
10352         if (!state || !restore_state) {
10353                 ret = -ENOMEM;
10354                 goto fail;
10355         }
10356
10357         state->acquire_ctx = ctx;
10358         restore_state->acquire_ctx = ctx;
10359
10360         connector_state = drm_atomic_get_connector_state(state, connector);
10361         if (IS_ERR(connector_state)) {
10362                 ret = PTR_ERR(connector_state);
10363                 goto fail;
10364         }
10365
10366         ret = drm_atomic_set_crtc_for_connector(connector_state, crtc);
10367         if (ret)
10368                 goto fail;
10369
10370         crtc_state = intel_atomic_get_crtc_state(state, intel_crtc);
10371         if (IS_ERR(crtc_state)) {
10372                 ret = PTR_ERR(crtc_state);
10373                 goto fail;
10374         }
10375
10376         crtc_state->base.active = crtc_state->base.enable = true;
10377
10378         if (!mode)
10379                 mode = &load_detect_mode;
10380
10381         ret = drm_atomic_set_mode_for_crtc(&crtc_state->base, mode);
10382         if (ret)
10383                 goto fail;
10384
10385         ret = intel_modeset_disable_planes(state, crtc);
10386         if (ret)
10387                 goto fail;
10388
10389         ret = PTR_ERR_OR_ZERO(drm_atomic_get_connector_state(restore_state, connector));
10390         if (!ret)
10391                 ret = PTR_ERR_OR_ZERO(drm_atomic_get_crtc_state(restore_state, crtc));
10392         if (!ret)
10393                 ret = drm_atomic_add_affected_planes(restore_state, crtc);
10394         if (ret) {
10395                 DRM_DEBUG_KMS("Failed to create a copy of old state to restore: %i\n", ret);
10396                 goto fail;
10397         }
10398
10399         ret = drm_atomic_commit(state);
10400         if (ret) {
10401                 DRM_DEBUG_KMS("failed to set mode on load-detect pipe\n");
10402                 goto fail;
10403         }
10404
10405         old->restore_state = restore_state;
10406         drm_atomic_state_put(state);
10407
10408         /* let the connector get through one full cycle before testing */
10409         intel_wait_for_vblank(dev_priv, intel_crtc->pipe);
10410         return true;
10411
10412 fail:
10413         if (state) {
10414                 drm_atomic_state_put(state);
10415                 state = NULL;
10416         }
10417         if (restore_state) {
10418                 drm_atomic_state_put(restore_state);
10419                 restore_state = NULL;
10420         }
10421
10422         if (ret == -EDEADLK)
10423                 return ret;
10424
10425         return false;
10426 }
10427
10428 void intel_release_load_detect_pipe(struct drm_connector *connector,
10429                                     struct intel_load_detect_pipe *old,
10430                                     struct drm_modeset_acquire_ctx *ctx)
10431 {
10432         struct intel_encoder *intel_encoder =
10433                 intel_attached_encoder(connector);
10434         struct drm_encoder *encoder = &intel_encoder->base;
10435         struct drm_atomic_state *state = old->restore_state;
10436         int ret;
10437
10438         DRM_DEBUG_KMS("[CONNECTOR:%d:%s], [ENCODER:%d:%s]\n",
10439                       connector->base.id, connector->name,
10440                       encoder->base.id, encoder->name);
10441
10442         if (!state)
10443                 return;
10444
10445         ret = drm_atomic_helper_commit_duplicated_state(state, ctx);
10446         if (ret)
10447                 DRM_DEBUG_KMS("Couldn't release load detect pipe: %i\n", ret);
10448         drm_atomic_state_put(state);
10449 }
10450
10451 static int i9xx_pll_refclk(struct drm_device *dev,
10452                            const struct intel_crtc_state *pipe_config)
10453 {
10454         struct drm_i915_private *dev_priv = to_i915(dev);
10455         u32 dpll = pipe_config->dpll_hw_state.dpll;
10456
10457         if ((dpll & PLL_REF_INPUT_MASK) == PLLB_REF_INPUT_SPREADSPECTRUMIN)
10458                 return dev_priv->vbt.lvds_ssc_freq;
10459         else if (HAS_PCH_SPLIT(dev_priv))
10460                 return 120000;
10461         else if (!IS_GEN2(dev_priv))
10462                 return 96000;
10463         else
10464                 return 48000;
10465 }
10466
10467 /* Returns the clock of the currently programmed mode of the given pipe. */
10468 static void i9xx_crtc_clock_get(struct intel_crtc *crtc,
10469                                 struct intel_crtc_state *pipe_config)
10470 {
10471         struct drm_device *dev = crtc->base.dev;
10472         struct drm_i915_private *dev_priv = to_i915(dev);
10473         int pipe = pipe_config->cpu_transcoder;
10474         u32 dpll = pipe_config->dpll_hw_state.dpll;
10475         u32 fp;
10476         struct dpll clock;
10477         int port_clock;
10478         int refclk = i9xx_pll_refclk(dev, pipe_config);
10479
10480         if ((dpll & DISPLAY_RATE_SELECT_FPA1) == 0)
10481                 fp = pipe_config->dpll_hw_state.fp0;
10482         else
10483                 fp = pipe_config->dpll_hw_state.fp1;
10484
10485         clock.m1 = (fp & FP_M1_DIV_MASK) >> FP_M1_DIV_SHIFT;
10486         if (IS_PINEVIEW(dev_priv)) {
10487                 clock.n = ffs((fp & FP_N_PINEVIEW_DIV_MASK) >> FP_N_DIV_SHIFT) - 1;
10488                 clock.m2 = (fp & FP_M2_PINEVIEW_DIV_MASK) >> FP_M2_DIV_SHIFT;
10489         } else {
10490                 clock.n = (fp & FP_N_DIV_MASK) >> FP_N_DIV_SHIFT;
10491                 clock.m2 = (fp & FP_M2_DIV_MASK) >> FP_M2_DIV_SHIFT;
10492         }
10493
10494         if (!IS_GEN2(dev_priv)) {
10495                 if (IS_PINEVIEW(dev_priv))
10496                         clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK_PINEVIEW) >>
10497                                 DPLL_FPA01_P1_POST_DIV_SHIFT_PINEVIEW);
10498                 else
10499                         clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK) >>
10500                                DPLL_FPA01_P1_POST_DIV_SHIFT);
10501
10502                 switch (dpll & DPLL_MODE_MASK) {
10503                 case DPLLB_MODE_DAC_SERIAL:
10504                         clock.p2 = dpll & DPLL_DAC_SERIAL_P2_CLOCK_DIV_5 ?
10505                                 5 : 10;
10506                         break;
10507                 case DPLLB_MODE_LVDS:
10508                         clock.p2 = dpll & DPLLB_LVDS_P2_CLOCK_DIV_7 ?
10509                                 7 : 14;
10510                         break;
10511                 default:
10512                         DRM_DEBUG_KMS("Unknown DPLL mode %08x in programmed "
10513                                   "mode\n", (int)(dpll & DPLL_MODE_MASK));
10514                         return;
10515                 }
10516
10517                 if (IS_PINEVIEW(dev_priv))
10518                         port_clock = pnv_calc_dpll_params(refclk, &clock);
10519                 else
10520                         port_clock = i9xx_calc_dpll_params(refclk, &clock);
10521         } else {
10522                 u32 lvds = IS_I830(dev_priv) ? 0 : I915_READ(LVDS);
10523                 bool is_lvds = (pipe == 1) && (lvds & LVDS_PORT_EN);
10524
10525                 if (is_lvds) {
10526                         clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK_I830_LVDS) >>
10527                                        DPLL_FPA01_P1_POST_DIV_SHIFT);
10528
10529                         if (lvds & LVDS_CLKB_POWER_UP)
10530                                 clock.p2 = 7;
10531                         else
10532                                 clock.p2 = 14;
10533                 } else {
10534                         if (dpll & PLL_P1_DIVIDE_BY_TWO)
10535                                 clock.p1 = 2;
10536                         else {
10537                                 clock.p1 = ((dpll & DPLL_FPA01_P1_POST_DIV_MASK_I830) >>
10538                                             DPLL_FPA01_P1_POST_DIV_SHIFT) + 2;
10539                         }
10540                         if (dpll & PLL_P2_DIVIDE_BY_4)
10541                                 clock.p2 = 4;
10542                         else
10543                                 clock.p2 = 2;
10544                 }
10545
10546                 port_clock = i9xx_calc_dpll_params(refclk, &clock);
10547         }
10548
10549         /*
10550          * This value includes pixel_multiplier. We will use
10551          * port_clock to compute adjusted_mode.crtc_clock in the
10552          * encoder's get_config() function.
10553          */
10554         pipe_config->port_clock = port_clock;
10555 }
10556
10557 int intel_dotclock_calculate(int link_freq,
10558                              const struct intel_link_m_n *m_n)
10559 {
10560         /*
10561          * The calculation for the data clock is:
10562          * pixel_clock = ((m/n)*(link_clock * nr_lanes))/bpp
10563          * But we want to avoid losing precison if possible, so:
10564          * pixel_clock = ((m * link_clock * nr_lanes)/(n*bpp))
10565          *
10566          * and the link clock is simpler:
10567          * link_clock = (m * link_clock) / n
10568          */
10569
10570         if (!m_n->link_n)
10571                 return 0;
10572
10573         return div_u64(mul_u32_u32(m_n->link_m, link_freq), m_n->link_n);
10574 }
10575
10576 static void ironlake_pch_clock_get(struct intel_crtc *crtc,
10577                                    struct intel_crtc_state *pipe_config)
10578 {
10579         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
10580
10581         /* read out port_clock from the DPLL */
10582         i9xx_crtc_clock_get(crtc, pipe_config);
10583
10584         /*
10585          * In case there is an active pipe without active ports,
10586          * we may need some idea for the dotclock anyway.
10587          * Calculate one based on the FDI configuration.
10588          */
10589         pipe_config->base.adjusted_mode.crtc_clock =
10590                 intel_dotclock_calculate(intel_fdi_link_freq(dev_priv, pipe_config),
10591                                          &pipe_config->fdi_m_n);
10592 }
10593
10594 /* Returns the currently programmed mode of the given encoder. */
10595 struct drm_display_mode *
10596 intel_encoder_current_mode(struct intel_encoder *encoder)
10597 {
10598         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
10599         struct intel_crtc_state *crtc_state;
10600         struct drm_display_mode *mode;
10601         struct intel_crtc *crtc;
10602         enum pipe pipe;
10603
10604         if (!encoder->get_hw_state(encoder, &pipe))
10605                 return NULL;
10606
10607         crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
10608
10609         mode = kzalloc(sizeof(*mode), GFP_KERNEL);
10610         if (!mode)
10611                 return NULL;
10612
10613         crtc_state = kzalloc(sizeof(*crtc_state), GFP_KERNEL);
10614         if (!crtc_state) {
10615                 kfree(mode);
10616                 return NULL;
10617         }
10618
10619         crtc_state->base.crtc = &crtc->base;
10620
10621         if (!dev_priv->display.get_pipe_config(crtc, crtc_state)) {
10622                 kfree(crtc_state);
10623                 kfree(mode);
10624                 return NULL;
10625         }
10626
10627         encoder->get_config(encoder, crtc_state);
10628
10629         intel_mode_from_pipe_config(mode, crtc_state);
10630
10631         kfree(crtc_state);
10632
10633         return mode;
10634 }
10635
10636 static void intel_crtc_destroy(struct drm_crtc *crtc)
10637 {
10638         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
10639
10640         drm_crtc_cleanup(crtc);
10641         kfree(intel_crtc);
10642 }
10643
10644 /**
10645  * intel_wm_need_update - Check whether watermarks need updating
10646  * @plane: drm plane
10647  * @state: new plane state
10648  *
10649  * Check current plane state versus the new one to determine whether
10650  * watermarks need to be recalculated.
10651  *
10652  * Returns true or false.
10653  */
10654 static bool intel_wm_need_update(struct drm_plane *plane,
10655                                  struct drm_plane_state *state)
10656 {
10657         struct intel_plane_state *new = to_intel_plane_state(state);
10658         struct intel_plane_state *cur = to_intel_plane_state(plane->state);
10659
10660         /* Update watermarks on tiling or size changes. */
10661         if (new->base.visible != cur->base.visible)
10662                 return true;
10663
10664         if (!cur->base.fb || !new->base.fb)
10665                 return false;
10666
10667         if (cur->base.fb->modifier != new->base.fb->modifier ||
10668             cur->base.rotation != new->base.rotation ||
10669             drm_rect_width(&new->base.src) != drm_rect_width(&cur->base.src) ||
10670             drm_rect_height(&new->base.src) != drm_rect_height(&cur->base.src) ||
10671             drm_rect_width(&new->base.dst) != drm_rect_width(&cur->base.dst) ||
10672             drm_rect_height(&new->base.dst) != drm_rect_height(&cur->base.dst))
10673                 return true;
10674
10675         return false;
10676 }
10677
10678 static bool needs_scaling(const struct intel_plane_state *state)
10679 {
10680         int src_w = drm_rect_width(&state->base.src) >> 16;
10681         int src_h = drm_rect_height(&state->base.src) >> 16;
10682         int dst_w = drm_rect_width(&state->base.dst);
10683         int dst_h = drm_rect_height(&state->base.dst);
10684
10685         return (src_w != dst_w || src_h != dst_h);
10686 }
10687
10688 int intel_plane_atomic_calc_changes(const struct intel_crtc_state *old_crtc_state,
10689                                     struct drm_crtc_state *crtc_state,
10690                                     const struct intel_plane_state *old_plane_state,
10691                                     struct drm_plane_state *plane_state)
10692 {
10693         struct intel_crtc_state *pipe_config = to_intel_crtc_state(crtc_state);
10694         struct drm_crtc *crtc = crtc_state->crtc;
10695         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
10696         struct intel_plane *plane = to_intel_plane(plane_state->plane);
10697         struct drm_device *dev = crtc->dev;
10698         struct drm_i915_private *dev_priv = to_i915(dev);
10699         bool mode_changed = needs_modeset(crtc_state);
10700         bool was_crtc_enabled = old_crtc_state->base.active;
10701         bool is_crtc_enabled = crtc_state->active;
10702         bool turn_off, turn_on, visible, was_visible;
10703         struct drm_framebuffer *fb = plane_state->fb;
10704         int ret;
10705
10706         if (INTEL_GEN(dev_priv) >= 9 && plane->id != PLANE_CURSOR) {
10707                 ret = skl_update_scaler_plane(
10708                         to_intel_crtc_state(crtc_state),
10709                         to_intel_plane_state(plane_state));
10710                 if (ret)
10711                         return ret;
10712         }
10713
10714         was_visible = old_plane_state->base.visible;
10715         visible = plane_state->visible;
10716
10717         if (!was_crtc_enabled && WARN_ON(was_visible))
10718                 was_visible = false;
10719
10720         /*
10721          * Visibility is calculated as if the crtc was on, but
10722          * after scaler setup everything depends on it being off
10723          * when the crtc isn't active.
10724          *
10725          * FIXME this is wrong for watermarks. Watermarks should also
10726          * be computed as if the pipe would be active. Perhaps move
10727          * per-plane wm computation to the .check_plane() hook, and
10728          * only combine the results from all planes in the current place?
10729          */
10730         if (!is_crtc_enabled) {
10731                 plane_state->visible = visible = false;
10732                 to_intel_crtc_state(crtc_state)->active_planes &= ~BIT(plane->id);
10733         }
10734
10735         if (!was_visible && !visible)
10736                 return 0;
10737
10738         if (fb != old_plane_state->base.fb)
10739                 pipe_config->fb_changed = true;
10740
10741         turn_off = was_visible && (!visible || mode_changed);
10742         turn_on = visible && (!was_visible || mode_changed);
10743
10744         DRM_DEBUG_ATOMIC("[CRTC:%d:%s] has [PLANE:%d:%s] with fb %i\n",
10745                          intel_crtc->base.base.id, intel_crtc->base.name,
10746                          plane->base.base.id, plane->base.name,
10747                          fb ? fb->base.id : -1);
10748
10749         DRM_DEBUG_ATOMIC("[PLANE:%d:%s] visible %i -> %i, off %i, on %i, ms %i\n",
10750                          plane->base.base.id, plane->base.name,
10751                          was_visible, visible,
10752                          turn_off, turn_on, mode_changed);
10753
10754         if (turn_on) {
10755                 if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv))
10756                         pipe_config->update_wm_pre = true;
10757
10758                 /* must disable cxsr around plane enable/disable */
10759                 if (plane->id != PLANE_CURSOR)
10760                         pipe_config->disable_cxsr = true;
10761         } else if (turn_off) {
10762                 if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv))
10763                         pipe_config->update_wm_post = true;
10764
10765                 /* must disable cxsr around plane enable/disable */
10766                 if (plane->id != PLANE_CURSOR)
10767                         pipe_config->disable_cxsr = true;
10768         } else if (intel_wm_need_update(&plane->base, plane_state)) {
10769                 if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv)) {
10770                         /* FIXME bollocks */
10771                         pipe_config->update_wm_pre = true;
10772                         pipe_config->update_wm_post = true;
10773                 }
10774         }
10775
10776         if (visible || was_visible)
10777                 pipe_config->fb_bits |= plane->frontbuffer_bit;
10778
10779         /*
10780          * WaCxSRDisabledForSpriteScaling:ivb
10781          *
10782          * cstate->update_wm was already set above, so this flag will
10783          * take effect when we commit and program watermarks.
10784          */
10785         if (plane->id == PLANE_SPRITE0 && IS_IVYBRIDGE(dev_priv) &&
10786             needs_scaling(to_intel_plane_state(plane_state)) &&
10787             !needs_scaling(old_plane_state))
10788                 pipe_config->disable_lp_wm = true;
10789
10790         return 0;
10791 }
10792
10793 static bool encoders_cloneable(const struct intel_encoder *a,
10794                                const struct intel_encoder *b)
10795 {
10796         /* masks could be asymmetric, so check both ways */
10797         return a == b || (a->cloneable & (1 << b->type) &&
10798                           b->cloneable & (1 << a->type));
10799 }
10800
10801 static bool check_single_encoder_cloning(struct drm_atomic_state *state,
10802                                          struct intel_crtc *crtc,
10803                                          struct intel_encoder *encoder)
10804 {
10805         struct intel_encoder *source_encoder;
10806         struct drm_connector *connector;
10807         struct drm_connector_state *connector_state;
10808         int i;
10809
10810         for_each_new_connector_in_state(state, connector, connector_state, i) {
10811                 if (connector_state->crtc != &crtc->base)
10812                         continue;
10813
10814                 source_encoder =
10815                         to_intel_encoder(connector_state->best_encoder);
10816                 if (!encoders_cloneable(encoder, source_encoder))
10817                         return false;
10818         }
10819
10820         return true;
10821 }
10822
10823 static int intel_crtc_atomic_check(struct drm_crtc *crtc,
10824                                    struct drm_crtc_state *crtc_state)
10825 {
10826         struct drm_device *dev = crtc->dev;
10827         struct drm_i915_private *dev_priv = to_i915(dev);
10828         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
10829         struct intel_crtc_state *pipe_config =
10830                 to_intel_crtc_state(crtc_state);
10831         struct drm_atomic_state *state = crtc_state->state;
10832         int ret;
10833         bool mode_changed = needs_modeset(crtc_state);
10834
10835         if (mode_changed && !crtc_state->active)
10836                 pipe_config->update_wm_post = true;
10837
10838         if (mode_changed && crtc_state->enable &&
10839             dev_priv->display.crtc_compute_clock &&
10840             !WARN_ON(pipe_config->shared_dpll)) {
10841                 ret = dev_priv->display.crtc_compute_clock(intel_crtc,
10842                                                            pipe_config);
10843                 if (ret)
10844                         return ret;
10845         }
10846
10847         if (crtc_state->color_mgmt_changed) {
10848                 ret = intel_color_check(crtc, crtc_state);
10849                 if (ret)
10850                         return ret;
10851
10852                 /*
10853                  * Changing color management on Intel hardware is
10854                  * handled as part of planes update.
10855                  */
10856                 crtc_state->planes_changed = true;
10857         }
10858
10859         ret = 0;
10860         if (dev_priv->display.compute_pipe_wm) {
10861                 ret = dev_priv->display.compute_pipe_wm(pipe_config);
10862                 if (ret) {
10863                         DRM_DEBUG_KMS("Target pipe watermarks are invalid\n");
10864                         return ret;
10865                 }
10866         }
10867
10868         if (dev_priv->display.compute_intermediate_wm &&
10869             !to_intel_atomic_state(state)->skip_intermediate_wm) {
10870                 if (WARN_ON(!dev_priv->display.compute_pipe_wm))
10871                         return 0;
10872
10873                 /*
10874                  * Calculate 'intermediate' watermarks that satisfy both the
10875                  * old state and the new state.  We can program these
10876                  * immediately.
10877                  */
10878                 ret = dev_priv->display.compute_intermediate_wm(dev,
10879                                                                 intel_crtc,
10880                                                                 pipe_config);
10881                 if (ret) {
10882                         DRM_DEBUG_KMS("No valid intermediate pipe watermarks are possible\n");
10883                         return ret;
10884                 }
10885         } else if (dev_priv->display.compute_intermediate_wm) {
10886                 if (HAS_PCH_SPLIT(dev_priv) && INTEL_GEN(dev_priv) < 9)
10887                         pipe_config->wm.ilk.intermediate = pipe_config->wm.ilk.optimal;
10888         }
10889
10890         if (INTEL_GEN(dev_priv) >= 9) {
10891                 if (mode_changed)
10892                         ret = skl_update_scaler_crtc(pipe_config);
10893
10894                 if (!ret)
10895                         ret = skl_check_pipe_max_pixel_rate(intel_crtc,
10896                                                             pipe_config);
10897                 if (!ret)
10898                         ret = intel_atomic_setup_scalers(dev_priv, intel_crtc,
10899                                                          pipe_config);
10900         }
10901
10902         if (HAS_IPS(dev_priv))
10903                 pipe_config->ips_enabled = hsw_compute_ips_config(pipe_config);
10904
10905         return ret;
10906 }
10907
10908 static const struct drm_crtc_helper_funcs intel_helper_funcs = {
10909         .atomic_begin = intel_begin_crtc_commit,
10910         .atomic_flush = intel_finish_crtc_commit,
10911         .atomic_check = intel_crtc_atomic_check,
10912 };
10913
10914 static void intel_modeset_update_connector_atomic_state(struct drm_device *dev)
10915 {
10916         struct intel_connector *connector;
10917         struct drm_connector_list_iter conn_iter;
10918
10919         drm_connector_list_iter_begin(dev, &conn_iter);
10920         for_each_intel_connector_iter(connector, &conn_iter) {
10921                 if (connector->base.state->crtc)
10922                         drm_connector_put(&connector->base);
10923
10924                 if (connector->base.encoder) {
10925                         connector->base.state->best_encoder =
10926                                 connector->base.encoder;
10927                         connector->base.state->crtc =
10928                                 connector->base.encoder->crtc;
10929
10930                         drm_connector_get(&connector->base);
10931                 } else {
10932                         connector->base.state->best_encoder = NULL;
10933                         connector->base.state->crtc = NULL;
10934                 }
10935         }
10936         drm_connector_list_iter_end(&conn_iter);
10937 }
10938
10939 static void
10940 connected_sink_compute_bpp(struct intel_connector *connector,
10941                            struct intel_crtc_state *pipe_config)
10942 {
10943         const struct drm_display_info *info = &connector->base.display_info;
10944         int bpp = pipe_config->pipe_bpp;
10945
10946         DRM_DEBUG_KMS("[CONNECTOR:%d:%s] checking for sink bpp constrains\n",
10947                       connector->base.base.id,
10948                       connector->base.name);
10949
10950         /* Don't use an invalid EDID bpc value */
10951         if (info->bpc != 0 && info->bpc * 3 < bpp) {
10952                 DRM_DEBUG_KMS("clamping display bpp (was %d) to EDID reported max of %d\n",
10953                               bpp, info->bpc * 3);
10954                 pipe_config->pipe_bpp = info->bpc * 3;
10955         }
10956
10957         /* Clamp bpp to 8 on screens without EDID 1.4 */
10958         if (info->bpc == 0 && bpp > 24) {
10959                 DRM_DEBUG_KMS("clamping display bpp (was %d) to default limit of 24\n",
10960                               bpp);
10961                 pipe_config->pipe_bpp = 24;
10962         }
10963 }
10964
10965 static int
10966 compute_baseline_pipe_bpp(struct intel_crtc *crtc,
10967                           struct intel_crtc_state *pipe_config)
10968 {
10969         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
10970         struct drm_atomic_state *state;
10971         struct drm_connector *connector;
10972         struct drm_connector_state *connector_state;
10973         int bpp, i;
10974
10975         if ((IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
10976             IS_CHERRYVIEW(dev_priv)))
10977                 bpp = 10*3;
10978         else if (INTEL_GEN(dev_priv) >= 5)
10979                 bpp = 12*3;
10980         else
10981                 bpp = 8*3;
10982
10983
10984         pipe_config->pipe_bpp = bpp;
10985
10986         state = pipe_config->base.state;
10987
10988         /* Clamp display bpp to EDID value */
10989         for_each_new_connector_in_state(state, connector, connector_state, i) {
10990                 if (connector_state->crtc != &crtc->base)
10991                         continue;
10992
10993                 connected_sink_compute_bpp(to_intel_connector(connector),
10994                                            pipe_config);
10995         }
10996
10997         return bpp;
10998 }
10999
11000 static void intel_dump_crtc_timings(const struct drm_display_mode *mode)
11001 {
11002         DRM_DEBUG_KMS("crtc timings: %d %d %d %d %d %d %d %d %d, "
11003                         "type: 0x%x flags: 0x%x\n",
11004                 mode->crtc_clock,
11005                 mode->crtc_hdisplay, mode->crtc_hsync_start,
11006                 mode->crtc_hsync_end, mode->crtc_htotal,
11007                 mode->crtc_vdisplay, mode->crtc_vsync_start,
11008                 mode->crtc_vsync_end, mode->crtc_vtotal, mode->type, mode->flags);
11009 }
11010
11011 static inline void
11012 intel_dump_m_n_config(struct intel_crtc_state *pipe_config, char *id,
11013                       unsigned int lane_count, struct intel_link_m_n *m_n)
11014 {
11015         DRM_DEBUG_KMS("%s: lanes: %i; gmch_m: %u, gmch_n: %u, link_m: %u, link_n: %u, tu: %u\n",
11016                       id, lane_count,
11017                       m_n->gmch_m, m_n->gmch_n,
11018                       m_n->link_m, m_n->link_n, m_n->tu);
11019 }
11020
11021 #define OUTPUT_TYPE(x) [INTEL_OUTPUT_ ## x] = #x
11022
11023 static const char * const output_type_str[] = {
11024         OUTPUT_TYPE(UNUSED),
11025         OUTPUT_TYPE(ANALOG),
11026         OUTPUT_TYPE(DVO),
11027         OUTPUT_TYPE(SDVO),
11028         OUTPUT_TYPE(LVDS),
11029         OUTPUT_TYPE(TVOUT),
11030         OUTPUT_TYPE(HDMI),
11031         OUTPUT_TYPE(DP),
11032         OUTPUT_TYPE(EDP),
11033         OUTPUT_TYPE(DSI),
11034         OUTPUT_TYPE(DDI),
11035         OUTPUT_TYPE(DP_MST),
11036 };
11037
11038 #undef OUTPUT_TYPE
11039
11040 static void snprintf_output_types(char *buf, size_t len,
11041                                   unsigned int output_types)
11042 {
11043         char *str = buf;
11044         int i;
11045
11046         str[0] = '\0';
11047
11048         for (i = 0; i < ARRAY_SIZE(output_type_str); i++) {
11049                 int r;
11050
11051                 if ((output_types & BIT(i)) == 0)
11052                         continue;
11053
11054                 r = snprintf(str, len, "%s%s",
11055                              str != buf ? "," : "", output_type_str[i]);
11056                 if (r >= len)
11057                         break;
11058                 str += r;
11059                 len -= r;
11060
11061                 output_types &= ~BIT(i);
11062         }
11063
11064         WARN_ON_ONCE(output_types != 0);
11065 }
11066
11067 static void intel_dump_pipe_config(struct intel_crtc *crtc,
11068                                    struct intel_crtc_state *pipe_config,
11069                                    const char *context)
11070 {
11071         struct drm_device *dev = crtc->base.dev;
11072         struct drm_i915_private *dev_priv = to_i915(dev);
11073         struct drm_plane *plane;
11074         struct intel_plane *intel_plane;
11075         struct intel_plane_state *state;
11076         struct drm_framebuffer *fb;
11077         char buf[64];
11078
11079         DRM_DEBUG_KMS("[CRTC:%d:%s]%s\n",
11080                       crtc->base.base.id, crtc->base.name, context);
11081
11082         snprintf_output_types(buf, sizeof(buf), pipe_config->output_types);
11083         DRM_DEBUG_KMS("output_types: %s (0x%x)\n",
11084                       buf, pipe_config->output_types);
11085
11086         DRM_DEBUG_KMS("cpu_transcoder: %s, pipe bpp: %i, dithering: %i\n",
11087                       transcoder_name(pipe_config->cpu_transcoder),
11088                       pipe_config->pipe_bpp, pipe_config->dither);
11089
11090         if (pipe_config->has_pch_encoder)
11091                 intel_dump_m_n_config(pipe_config, "fdi",
11092                                       pipe_config->fdi_lanes,
11093                                       &pipe_config->fdi_m_n);
11094
11095         if (pipe_config->ycbcr420)
11096                 DRM_DEBUG_KMS("YCbCr 4:2:0 output enabled\n");
11097
11098         if (intel_crtc_has_dp_encoder(pipe_config)) {
11099                 intel_dump_m_n_config(pipe_config, "dp m_n",
11100                                 pipe_config->lane_count, &pipe_config->dp_m_n);
11101                 if (pipe_config->has_drrs)
11102                         intel_dump_m_n_config(pipe_config, "dp m2_n2",
11103                                               pipe_config->lane_count,
11104                                               &pipe_config->dp_m2_n2);
11105         }
11106
11107         DRM_DEBUG_KMS("audio: %i, infoframes: %i\n",
11108                       pipe_config->has_audio, pipe_config->has_infoframe);
11109
11110         DRM_DEBUG_KMS("requested mode:\n");
11111         drm_mode_debug_printmodeline(&pipe_config->base.mode);
11112         DRM_DEBUG_KMS("adjusted mode:\n");
11113         drm_mode_debug_printmodeline(&pipe_config->base.adjusted_mode);
11114         intel_dump_crtc_timings(&pipe_config->base.adjusted_mode);
11115         DRM_DEBUG_KMS("port clock: %d, pipe src size: %dx%d, pixel rate %d\n",
11116                       pipe_config->port_clock,
11117                       pipe_config->pipe_src_w, pipe_config->pipe_src_h,
11118                       pipe_config->pixel_rate);
11119
11120         if (INTEL_GEN(dev_priv) >= 9)
11121                 DRM_DEBUG_KMS("num_scalers: %d, scaler_users: 0x%x, scaler_id: %d\n",
11122                               crtc->num_scalers,
11123                               pipe_config->scaler_state.scaler_users,
11124                               pipe_config->scaler_state.scaler_id);
11125
11126         if (HAS_GMCH_DISPLAY(dev_priv))
11127                 DRM_DEBUG_KMS("gmch pfit: control: 0x%08x, ratios: 0x%08x, lvds border: 0x%08x\n",
11128                               pipe_config->gmch_pfit.control,
11129                               pipe_config->gmch_pfit.pgm_ratios,
11130                               pipe_config->gmch_pfit.lvds_border_bits);
11131         else
11132                 DRM_DEBUG_KMS("pch pfit: pos: 0x%08x, size: 0x%08x, %s\n",
11133                               pipe_config->pch_pfit.pos,
11134                               pipe_config->pch_pfit.size,
11135                               enableddisabled(pipe_config->pch_pfit.enabled));
11136
11137         DRM_DEBUG_KMS("ips: %i, double wide: %i\n",
11138                       pipe_config->ips_enabled, pipe_config->double_wide);
11139
11140         intel_dpll_dump_hw_state(dev_priv, &pipe_config->dpll_hw_state);
11141
11142         DRM_DEBUG_KMS("planes on this crtc\n");
11143         list_for_each_entry(plane, &dev->mode_config.plane_list, head) {
11144                 struct drm_format_name_buf format_name;
11145                 intel_plane = to_intel_plane(plane);
11146                 if (intel_plane->pipe != crtc->pipe)
11147                         continue;
11148
11149                 state = to_intel_plane_state(plane->state);
11150                 fb = state->base.fb;
11151                 if (!fb) {
11152                         DRM_DEBUG_KMS("[PLANE:%d:%s] disabled, scaler_id = %d\n",
11153                                       plane->base.id, plane->name, state->scaler_id);
11154                         continue;
11155                 }
11156
11157                 DRM_DEBUG_KMS("[PLANE:%d:%s] FB:%d, fb = %ux%u format = %s\n",
11158                               plane->base.id, plane->name,
11159                               fb->base.id, fb->width, fb->height,
11160                               drm_get_format_name(fb->format->format, &format_name));
11161                 if (INTEL_GEN(dev_priv) >= 9)
11162                         DRM_DEBUG_KMS("\tscaler:%d src %dx%d+%d+%d dst %dx%d+%d+%d\n",
11163                                       state->scaler_id,
11164                                       state->base.src.x1 >> 16,
11165                                       state->base.src.y1 >> 16,
11166                                       drm_rect_width(&state->base.src) >> 16,
11167                                       drm_rect_height(&state->base.src) >> 16,
11168                                       state->base.dst.x1, state->base.dst.y1,
11169                                       drm_rect_width(&state->base.dst),
11170                                       drm_rect_height(&state->base.dst));
11171         }
11172 }
11173
11174 static bool check_digital_port_conflicts(struct drm_atomic_state *state)
11175 {
11176         struct drm_device *dev = state->dev;
11177         struct drm_connector *connector;
11178         struct drm_connector_list_iter conn_iter;
11179         unsigned int used_ports = 0;
11180         unsigned int used_mst_ports = 0;
11181         bool ret = true;
11182
11183         /*
11184          * Walk the connector list instead of the encoder
11185          * list to detect the problem on ddi platforms
11186          * where there's just one encoder per digital port.
11187          */
11188         drm_connector_list_iter_begin(dev, &conn_iter);
11189         drm_for_each_connector_iter(connector, &conn_iter) {
11190                 struct drm_connector_state *connector_state;
11191                 struct intel_encoder *encoder;
11192
11193                 connector_state = drm_atomic_get_new_connector_state(state, connector);
11194                 if (!connector_state)
11195                         connector_state = connector->state;
11196
11197                 if (!connector_state->best_encoder)
11198                         continue;
11199
11200                 encoder = to_intel_encoder(connector_state->best_encoder);
11201
11202                 WARN_ON(!connector_state->crtc);
11203
11204                 switch (encoder->type) {
11205                         unsigned int port_mask;
11206                 case INTEL_OUTPUT_DDI:
11207                         if (WARN_ON(!HAS_DDI(to_i915(dev))))
11208                                 break;
11209                         /* else: fall through */
11210                 case INTEL_OUTPUT_DP:
11211                 case INTEL_OUTPUT_HDMI:
11212                 case INTEL_OUTPUT_EDP:
11213                         port_mask = 1 << encoder->port;
11214
11215                         /* the same port mustn't appear more than once */
11216                         if (used_ports & port_mask)
11217                                 ret = false;
11218
11219                         used_ports |= port_mask;
11220                         break;
11221                 case INTEL_OUTPUT_DP_MST:
11222                         used_mst_ports |=
11223                                 1 << encoder->port;
11224                         break;
11225                 default:
11226                         break;
11227                 }
11228         }
11229         drm_connector_list_iter_end(&conn_iter);
11230
11231         /* can't mix MST and SST/HDMI on the same port */
11232         if (used_ports & used_mst_ports)
11233                 return false;
11234
11235         return ret;
11236 }
11237
11238 static void
11239 clear_intel_crtc_state(struct intel_crtc_state *crtc_state)
11240 {
11241         struct drm_i915_private *dev_priv =
11242                 to_i915(crtc_state->base.crtc->dev);
11243         struct intel_crtc_scaler_state scaler_state;
11244         struct intel_dpll_hw_state dpll_hw_state;
11245         struct intel_shared_dpll *shared_dpll;
11246         struct intel_crtc_wm_state wm_state;
11247         bool force_thru, ips_force_disable;
11248
11249         /* FIXME: before the switch to atomic started, a new pipe_config was
11250          * kzalloc'd. Code that depends on any field being zero should be
11251          * fixed, so that the crtc_state can be safely duplicated. For now,
11252          * only fields that are know to not cause problems are preserved. */
11253
11254         scaler_state = crtc_state->scaler_state;
11255         shared_dpll = crtc_state->shared_dpll;
11256         dpll_hw_state = crtc_state->dpll_hw_state;
11257         force_thru = crtc_state->pch_pfit.force_thru;
11258         ips_force_disable = crtc_state->ips_force_disable;
11259         if (IS_G4X(dev_priv) ||
11260             IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
11261                 wm_state = crtc_state->wm;
11262
11263         /* Keep base drm_crtc_state intact, only clear our extended struct */
11264         BUILD_BUG_ON(offsetof(struct intel_crtc_state, base));
11265         memset(&crtc_state->base + 1, 0,
11266                sizeof(*crtc_state) - sizeof(crtc_state->base));
11267
11268         crtc_state->scaler_state = scaler_state;
11269         crtc_state->shared_dpll = shared_dpll;
11270         crtc_state->dpll_hw_state = dpll_hw_state;
11271         crtc_state->pch_pfit.force_thru = force_thru;
11272         crtc_state->ips_force_disable = ips_force_disable;
11273         if (IS_G4X(dev_priv) ||
11274             IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
11275                 crtc_state->wm = wm_state;
11276 }
11277
11278 static int
11279 intel_modeset_pipe_config(struct drm_crtc *crtc,
11280                           struct intel_crtc_state *pipe_config)
11281 {
11282         struct drm_atomic_state *state = pipe_config->base.state;
11283         struct intel_encoder *encoder;
11284         struct drm_connector *connector;
11285         struct drm_connector_state *connector_state;
11286         int base_bpp, ret = -EINVAL;
11287         int i;
11288         bool retry = true;
11289
11290         clear_intel_crtc_state(pipe_config);
11291
11292         pipe_config->cpu_transcoder =
11293                 (enum transcoder) to_intel_crtc(crtc)->pipe;
11294
11295         /*
11296          * Sanitize sync polarity flags based on requested ones. If neither
11297          * positive or negative polarity is requested, treat this as meaning
11298          * negative polarity.
11299          */
11300         if (!(pipe_config->base.adjusted_mode.flags &
11301               (DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_NHSYNC)))
11302                 pipe_config->base.adjusted_mode.flags |= DRM_MODE_FLAG_NHSYNC;
11303
11304         if (!(pipe_config->base.adjusted_mode.flags &
11305               (DRM_MODE_FLAG_PVSYNC | DRM_MODE_FLAG_NVSYNC)))
11306                 pipe_config->base.adjusted_mode.flags |= DRM_MODE_FLAG_NVSYNC;
11307
11308         base_bpp = compute_baseline_pipe_bpp(to_intel_crtc(crtc),
11309                                              pipe_config);
11310         if (base_bpp < 0)
11311                 goto fail;
11312
11313         /*
11314          * Determine the real pipe dimensions. Note that stereo modes can
11315          * increase the actual pipe size due to the frame doubling and
11316          * insertion of additional space for blanks between the frame. This
11317          * is stored in the crtc timings. We use the requested mode to do this
11318          * computation to clearly distinguish it from the adjusted mode, which
11319          * can be changed by the connectors in the below retry loop.
11320          */
11321         drm_mode_get_hv_timing(&pipe_config->base.mode,
11322                                &pipe_config->pipe_src_w,
11323                                &pipe_config->pipe_src_h);
11324
11325         for_each_new_connector_in_state(state, connector, connector_state, i) {
11326                 if (connector_state->crtc != crtc)
11327                         continue;
11328
11329                 encoder = to_intel_encoder(connector_state->best_encoder);
11330
11331                 if (!check_single_encoder_cloning(state, to_intel_crtc(crtc), encoder)) {
11332                         DRM_DEBUG_KMS("rejecting invalid cloning configuration\n");
11333                         goto fail;
11334                 }
11335
11336                 /*
11337                  * Determine output_types before calling the .compute_config()
11338                  * hooks so that the hooks can use this information safely.
11339                  */
11340                 if (encoder->compute_output_type)
11341                         pipe_config->output_types |=
11342                                 BIT(encoder->compute_output_type(encoder, pipe_config,
11343                                                                  connector_state));
11344                 else
11345                         pipe_config->output_types |= BIT(encoder->type);
11346         }
11347
11348 encoder_retry:
11349         /* Ensure the port clock defaults are reset when retrying. */
11350         pipe_config->port_clock = 0;
11351         pipe_config->pixel_multiplier = 1;
11352
11353         /* Fill in default crtc timings, allow encoders to overwrite them. */
11354         drm_mode_set_crtcinfo(&pipe_config->base.adjusted_mode,
11355                               CRTC_STEREO_DOUBLE);
11356
11357         /* Pass our mode to the connectors and the CRTC to give them a chance to
11358          * adjust it according to limitations or connector properties, and also
11359          * a chance to reject the mode entirely.
11360          */
11361         for_each_new_connector_in_state(state, connector, connector_state, i) {
11362                 if (connector_state->crtc != crtc)
11363                         continue;
11364
11365                 encoder = to_intel_encoder(connector_state->best_encoder);
11366
11367                 if (!(encoder->compute_config(encoder, pipe_config, connector_state))) {
11368                         DRM_DEBUG_KMS("Encoder config failure\n");
11369                         goto fail;
11370                 }
11371         }
11372
11373         /* Set default port clock if not overwritten by the encoder. Needs to be
11374          * done afterwards in case the encoder adjusts the mode. */
11375         if (!pipe_config->port_clock)
11376                 pipe_config->port_clock = pipe_config->base.adjusted_mode.crtc_clock
11377                         * pipe_config->pixel_multiplier;
11378
11379         ret = intel_crtc_compute_config(to_intel_crtc(crtc), pipe_config);
11380         if (ret < 0) {
11381                 DRM_DEBUG_KMS("CRTC fixup failed\n");
11382                 goto fail;
11383         }
11384
11385         if (ret == RETRY) {
11386                 if (WARN(!retry, "loop in pipe configuration computation\n")) {
11387                         ret = -EINVAL;
11388                         goto fail;
11389                 }
11390
11391                 DRM_DEBUG_KMS("CRTC bw constrained, retrying\n");
11392                 retry = false;
11393                 goto encoder_retry;
11394         }
11395
11396         /* Dithering seems to not pass-through bits correctly when it should, so
11397          * only enable it on 6bpc panels and when its not a compliance
11398          * test requesting 6bpc video pattern.
11399          */
11400         pipe_config->dither = (pipe_config->pipe_bpp == 6*3) &&
11401                 !pipe_config->dither_force_disable;
11402         DRM_DEBUG_KMS("hw max bpp: %i, pipe bpp: %i, dithering: %i\n",
11403                       base_bpp, pipe_config->pipe_bpp, pipe_config->dither);
11404
11405 fail:
11406         return ret;
11407 }
11408
11409 static bool intel_fuzzy_clock_check(int clock1, int clock2)
11410 {
11411         int diff;
11412
11413         if (clock1 == clock2)
11414                 return true;
11415
11416         if (!clock1 || !clock2)
11417                 return false;
11418
11419         diff = abs(clock1 - clock2);
11420
11421         if (((((diff + clock1 + clock2) * 100)) / (clock1 + clock2)) < 105)
11422                 return true;
11423
11424         return false;
11425 }
11426
11427 static bool
11428 intel_compare_m_n(unsigned int m, unsigned int n,
11429                   unsigned int m2, unsigned int n2,
11430                   bool exact)
11431 {
11432         if (m == m2 && n == n2)
11433                 return true;
11434
11435         if (exact || !m || !n || !m2 || !n2)
11436                 return false;
11437
11438         BUILD_BUG_ON(DATA_LINK_M_N_MASK > INT_MAX);
11439
11440         if (n > n2) {
11441                 while (n > n2) {
11442                         m2 <<= 1;
11443                         n2 <<= 1;
11444                 }
11445         } else if (n < n2) {
11446                 while (n < n2) {
11447                         m <<= 1;
11448                         n <<= 1;
11449                 }
11450         }
11451
11452         if (n != n2)
11453                 return false;
11454
11455         return intel_fuzzy_clock_check(m, m2);
11456 }
11457
11458 static bool
11459 intel_compare_link_m_n(const struct intel_link_m_n *m_n,
11460                        struct intel_link_m_n *m2_n2,
11461                        bool adjust)
11462 {
11463         if (m_n->tu == m2_n2->tu &&
11464             intel_compare_m_n(m_n->gmch_m, m_n->gmch_n,
11465                               m2_n2->gmch_m, m2_n2->gmch_n, !adjust) &&
11466             intel_compare_m_n(m_n->link_m, m_n->link_n,
11467                               m2_n2->link_m, m2_n2->link_n, !adjust)) {
11468                 if (adjust)
11469                         *m2_n2 = *m_n;
11470
11471                 return true;
11472         }
11473
11474         return false;
11475 }
11476
11477 static void __printf(3, 4)
11478 pipe_config_err(bool adjust, const char *name, const char *format, ...)
11479 {
11480         struct va_format vaf;
11481         va_list args;
11482
11483         va_start(args, format);
11484         vaf.fmt = format;
11485         vaf.va = &args;
11486
11487         if (adjust)
11488                 drm_dbg(DRM_UT_KMS, "mismatch in %s %pV", name, &vaf);
11489         else
11490                 drm_err("mismatch in %s %pV", name, &vaf);
11491
11492         va_end(args);
11493 }
11494
11495 static bool
11496 intel_pipe_config_compare(struct drm_i915_private *dev_priv,
11497                           struct intel_crtc_state *current_config,
11498                           struct intel_crtc_state *pipe_config,
11499                           bool adjust)
11500 {
11501         bool ret = true;
11502         bool fixup_inherited = adjust &&
11503                 (current_config->base.mode.private_flags & I915_MODE_FLAG_INHERITED) &&
11504                 !(pipe_config->base.mode.private_flags & I915_MODE_FLAG_INHERITED);
11505
11506 #define PIPE_CONF_CHECK_X(name) do { \
11507         if (current_config->name != pipe_config->name) { \
11508                 pipe_config_err(adjust, __stringify(name), \
11509                           "(expected 0x%08x, found 0x%08x)\n", \
11510                           current_config->name, \
11511                           pipe_config->name); \
11512                 ret = false; \
11513         } \
11514 } while (0)
11515
11516 #define PIPE_CONF_CHECK_I(name) do { \
11517         if (current_config->name != pipe_config->name) { \
11518                 pipe_config_err(adjust, __stringify(name), \
11519                           "(expected %i, found %i)\n", \
11520                           current_config->name, \
11521                           pipe_config->name); \
11522                 ret = false; \
11523         } \
11524 } while (0)
11525
11526 #define PIPE_CONF_CHECK_BOOL(name) do { \
11527         if (current_config->name != pipe_config->name) { \
11528                 pipe_config_err(adjust, __stringify(name), \
11529                           "(expected %s, found %s)\n", \
11530                           yesno(current_config->name), \
11531                           yesno(pipe_config->name)); \
11532                 ret = false; \
11533         } \
11534 } while (0)
11535
11536 /*
11537  * Checks state where we only read out the enabling, but not the entire
11538  * state itself (like full infoframes or ELD for audio). These states
11539  * require a full modeset on bootup to fix up.
11540  */
11541 #define PIPE_CONF_CHECK_BOOL_INCOMPLETE(name) do { \
11542         if (!fixup_inherited || (!current_config->name && !pipe_config->name)) { \
11543                 PIPE_CONF_CHECK_BOOL(name); \
11544         } else { \
11545                 pipe_config_err(adjust, __stringify(name), \
11546                           "unable to verify whether state matches exactly, forcing modeset (expected %s, found %s)\n", \
11547                           yesno(current_config->name), \
11548                           yesno(pipe_config->name)); \
11549                 ret = false; \
11550         } \
11551 } while (0)
11552
11553 #define PIPE_CONF_CHECK_P(name) do { \
11554         if (current_config->name != pipe_config->name) { \
11555                 pipe_config_err(adjust, __stringify(name), \
11556                           "(expected %p, found %p)\n", \
11557                           current_config->name, \
11558                           pipe_config->name); \
11559                 ret = false; \
11560         } \
11561 } while (0)
11562
11563 #define PIPE_CONF_CHECK_M_N(name) do { \
11564         if (!intel_compare_link_m_n(&current_config->name, \
11565                                     &pipe_config->name,\
11566                                     adjust)) { \
11567                 pipe_config_err(adjust, __stringify(name), \
11568                           "(expected tu %i gmch %i/%i link %i/%i, " \
11569                           "found tu %i, gmch %i/%i link %i/%i)\n", \
11570                           current_config->name.tu, \
11571                           current_config->name.gmch_m, \
11572                           current_config->name.gmch_n, \
11573                           current_config->name.link_m, \
11574                           current_config->name.link_n, \
11575                           pipe_config->name.tu, \
11576                           pipe_config->name.gmch_m, \
11577                           pipe_config->name.gmch_n, \
11578                           pipe_config->name.link_m, \
11579                           pipe_config->name.link_n); \
11580                 ret = false; \
11581         } \
11582 } while (0)
11583
11584 /* This is required for BDW+ where there is only one set of registers for
11585  * switching between high and low RR.
11586  * This macro can be used whenever a comparison has to be made between one
11587  * hw state and multiple sw state variables.
11588  */
11589 #define PIPE_CONF_CHECK_M_N_ALT(name, alt_name) do { \
11590         if (!intel_compare_link_m_n(&current_config->name, \
11591                                     &pipe_config->name, adjust) && \
11592             !intel_compare_link_m_n(&current_config->alt_name, \
11593                                     &pipe_config->name, adjust)) { \
11594                 pipe_config_err(adjust, __stringify(name), \
11595                           "(expected tu %i gmch %i/%i link %i/%i, " \
11596                           "or tu %i gmch %i/%i link %i/%i, " \
11597                           "found tu %i, gmch %i/%i link %i/%i)\n", \
11598                           current_config->name.tu, \
11599                           current_config->name.gmch_m, \
11600                           current_config->name.gmch_n, \
11601                           current_config->name.link_m, \
11602                           current_config->name.link_n, \
11603                           current_config->alt_name.tu, \
11604                           current_config->alt_name.gmch_m, \
11605                           current_config->alt_name.gmch_n, \
11606                           current_config->alt_name.link_m, \
11607                           current_config->alt_name.link_n, \
11608                           pipe_config->name.tu, \
11609                           pipe_config->name.gmch_m, \
11610                           pipe_config->name.gmch_n, \
11611                           pipe_config->name.link_m, \
11612                           pipe_config->name.link_n); \
11613                 ret = false; \
11614         } \
11615 } while (0)
11616
11617 #define PIPE_CONF_CHECK_FLAGS(name, mask) do { \
11618         if ((current_config->name ^ pipe_config->name) & (mask)) { \
11619                 pipe_config_err(adjust, __stringify(name), \
11620                           "(%x) (expected %i, found %i)\n", \
11621                           (mask), \
11622                           current_config->name & (mask), \
11623                           pipe_config->name & (mask)); \
11624                 ret = false; \
11625         } \
11626 } while (0)
11627
11628 #define PIPE_CONF_CHECK_CLOCK_FUZZY(name) do { \
11629         if (!intel_fuzzy_clock_check(current_config->name, pipe_config->name)) { \
11630                 pipe_config_err(adjust, __stringify(name), \
11631                           "(expected %i, found %i)\n", \
11632                           current_config->name, \
11633                           pipe_config->name); \
11634                 ret = false; \
11635         } \
11636 } while (0)
11637
11638 #define PIPE_CONF_QUIRK(quirk)  \
11639         ((current_config->quirks | pipe_config->quirks) & (quirk))
11640
11641         PIPE_CONF_CHECK_I(cpu_transcoder);
11642
11643         PIPE_CONF_CHECK_BOOL(has_pch_encoder);
11644         PIPE_CONF_CHECK_I(fdi_lanes);
11645         PIPE_CONF_CHECK_M_N(fdi_m_n);
11646
11647         PIPE_CONF_CHECK_I(lane_count);
11648         PIPE_CONF_CHECK_X(lane_lat_optim_mask);
11649
11650         if (INTEL_GEN(dev_priv) < 8) {
11651                 PIPE_CONF_CHECK_M_N(dp_m_n);
11652
11653                 if (current_config->has_drrs)
11654                         PIPE_CONF_CHECK_M_N(dp_m2_n2);
11655         } else
11656                 PIPE_CONF_CHECK_M_N_ALT(dp_m_n, dp_m2_n2);
11657
11658         PIPE_CONF_CHECK_X(output_types);
11659
11660         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_hdisplay);
11661         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_htotal);
11662         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_hblank_start);
11663         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_hblank_end);
11664         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_hsync_start);
11665         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_hsync_end);
11666
11667         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vdisplay);
11668         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vtotal);
11669         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vblank_start);
11670         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vblank_end);
11671         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vsync_start);
11672         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vsync_end);
11673
11674         PIPE_CONF_CHECK_I(pixel_multiplier);
11675         PIPE_CONF_CHECK_BOOL(has_hdmi_sink);
11676         if ((INTEL_GEN(dev_priv) < 8 && !IS_HASWELL(dev_priv)) ||
11677             IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
11678                 PIPE_CONF_CHECK_BOOL(limited_color_range);
11679
11680         PIPE_CONF_CHECK_BOOL(hdmi_scrambling);
11681         PIPE_CONF_CHECK_BOOL(hdmi_high_tmds_clock_ratio);
11682         PIPE_CONF_CHECK_BOOL_INCOMPLETE(has_infoframe);
11683         PIPE_CONF_CHECK_BOOL(ycbcr420);
11684
11685         PIPE_CONF_CHECK_BOOL_INCOMPLETE(has_audio);
11686
11687         PIPE_CONF_CHECK_FLAGS(base.adjusted_mode.flags,
11688                               DRM_MODE_FLAG_INTERLACE);
11689
11690         if (!PIPE_CONF_QUIRK(PIPE_CONFIG_QUIRK_MODE_SYNC_FLAGS)) {
11691                 PIPE_CONF_CHECK_FLAGS(base.adjusted_mode.flags,
11692                                       DRM_MODE_FLAG_PHSYNC);
11693                 PIPE_CONF_CHECK_FLAGS(base.adjusted_mode.flags,
11694                                       DRM_MODE_FLAG_NHSYNC);
11695                 PIPE_CONF_CHECK_FLAGS(base.adjusted_mode.flags,
11696                                       DRM_MODE_FLAG_PVSYNC);
11697                 PIPE_CONF_CHECK_FLAGS(base.adjusted_mode.flags,
11698                                       DRM_MODE_FLAG_NVSYNC);
11699         }
11700
11701         PIPE_CONF_CHECK_X(gmch_pfit.control);
11702         /* pfit ratios are autocomputed by the hw on gen4+ */
11703         if (INTEL_GEN(dev_priv) < 4)
11704                 PIPE_CONF_CHECK_X(gmch_pfit.pgm_ratios);
11705         PIPE_CONF_CHECK_X(gmch_pfit.lvds_border_bits);
11706
11707         if (!adjust) {
11708                 PIPE_CONF_CHECK_I(pipe_src_w);
11709                 PIPE_CONF_CHECK_I(pipe_src_h);
11710
11711                 PIPE_CONF_CHECK_BOOL(pch_pfit.enabled);
11712                 if (current_config->pch_pfit.enabled) {
11713                         PIPE_CONF_CHECK_X(pch_pfit.pos);
11714                         PIPE_CONF_CHECK_X(pch_pfit.size);
11715                 }
11716
11717                 PIPE_CONF_CHECK_I(scaler_state.scaler_id);
11718                 PIPE_CONF_CHECK_CLOCK_FUZZY(pixel_rate);
11719         }
11720
11721         PIPE_CONF_CHECK_BOOL(double_wide);
11722
11723         PIPE_CONF_CHECK_P(shared_dpll);
11724         PIPE_CONF_CHECK_X(dpll_hw_state.dpll);
11725         PIPE_CONF_CHECK_X(dpll_hw_state.dpll_md);
11726         PIPE_CONF_CHECK_X(dpll_hw_state.fp0);
11727         PIPE_CONF_CHECK_X(dpll_hw_state.fp1);
11728         PIPE_CONF_CHECK_X(dpll_hw_state.wrpll);
11729         PIPE_CONF_CHECK_X(dpll_hw_state.spll);
11730         PIPE_CONF_CHECK_X(dpll_hw_state.ctrl1);
11731         PIPE_CONF_CHECK_X(dpll_hw_state.cfgcr1);
11732         PIPE_CONF_CHECK_X(dpll_hw_state.cfgcr2);
11733         PIPE_CONF_CHECK_X(dpll_hw_state.cfgcr0);
11734         PIPE_CONF_CHECK_X(dpll_hw_state.ebb0);
11735         PIPE_CONF_CHECK_X(dpll_hw_state.ebb4);
11736         PIPE_CONF_CHECK_X(dpll_hw_state.pll0);
11737         PIPE_CONF_CHECK_X(dpll_hw_state.pll1);
11738         PIPE_CONF_CHECK_X(dpll_hw_state.pll2);
11739         PIPE_CONF_CHECK_X(dpll_hw_state.pll3);
11740         PIPE_CONF_CHECK_X(dpll_hw_state.pll6);
11741         PIPE_CONF_CHECK_X(dpll_hw_state.pll8);
11742         PIPE_CONF_CHECK_X(dpll_hw_state.pll9);
11743         PIPE_CONF_CHECK_X(dpll_hw_state.pll10);
11744         PIPE_CONF_CHECK_X(dpll_hw_state.pcsdw12);
11745         PIPE_CONF_CHECK_X(dpll_hw_state.mg_refclkin_ctl);
11746         PIPE_CONF_CHECK_X(dpll_hw_state.mg_clktop2_coreclkctl1);
11747         PIPE_CONF_CHECK_X(dpll_hw_state.mg_clktop2_hsclkctl);
11748         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_div0);
11749         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_div1);
11750         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_lf);
11751         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_frac_lock);
11752         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_ssc);
11753         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_bias);
11754         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_tdc_coldst_bias);
11755
11756         PIPE_CONF_CHECK_X(dsi_pll.ctrl);
11757         PIPE_CONF_CHECK_X(dsi_pll.div);
11758
11759         if (IS_G4X(dev_priv) || INTEL_GEN(dev_priv) >= 5)
11760                 PIPE_CONF_CHECK_I(pipe_bpp);
11761
11762         PIPE_CONF_CHECK_CLOCK_FUZZY(base.adjusted_mode.crtc_clock);
11763         PIPE_CONF_CHECK_CLOCK_FUZZY(port_clock);
11764
11765         PIPE_CONF_CHECK_I(min_voltage_level);
11766
11767 #undef PIPE_CONF_CHECK_X
11768 #undef PIPE_CONF_CHECK_I
11769 #undef PIPE_CONF_CHECK_BOOL
11770 #undef PIPE_CONF_CHECK_BOOL_INCOMPLETE
11771 #undef PIPE_CONF_CHECK_P
11772 #undef PIPE_CONF_CHECK_FLAGS
11773 #undef PIPE_CONF_CHECK_CLOCK_FUZZY
11774 #undef PIPE_CONF_QUIRK
11775
11776         return ret;
11777 }
11778
11779 static void intel_pipe_config_sanity_check(struct drm_i915_private *dev_priv,
11780                                            const struct intel_crtc_state *pipe_config)
11781 {
11782         if (pipe_config->has_pch_encoder) {
11783                 int fdi_dotclock = intel_dotclock_calculate(intel_fdi_link_freq(dev_priv, pipe_config),
11784                                                             &pipe_config->fdi_m_n);
11785                 int dotclock = pipe_config->base.adjusted_mode.crtc_clock;
11786
11787                 /*
11788                  * FDI already provided one idea for the dotclock.
11789                  * Yell if the encoder disagrees.
11790                  */
11791                 WARN(!intel_fuzzy_clock_check(fdi_dotclock, dotclock),
11792                      "FDI dotclock and encoder dotclock mismatch, fdi: %i, encoder: %i\n",
11793                      fdi_dotclock, dotclock);
11794         }
11795 }
11796
11797 static void verify_wm_state(struct drm_crtc *crtc,
11798                             struct drm_crtc_state *new_state)
11799 {
11800         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
11801         struct skl_ddb_allocation hw_ddb, *sw_ddb;
11802         struct skl_pipe_wm hw_wm, *sw_wm;
11803         struct skl_plane_wm *hw_plane_wm, *sw_plane_wm;
11804         struct skl_ddb_entry *hw_ddb_entry, *sw_ddb_entry;
11805         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
11806         const enum pipe pipe = intel_crtc->pipe;
11807         int plane, level, max_level = ilk_wm_max_level(dev_priv);
11808
11809         if (INTEL_GEN(dev_priv) < 9 || !new_state->active)
11810                 return;
11811
11812         skl_pipe_wm_get_hw_state(crtc, &hw_wm);
11813         sw_wm = &to_intel_crtc_state(new_state)->wm.skl.optimal;
11814
11815         skl_ddb_get_hw_state(dev_priv, &hw_ddb);
11816         sw_ddb = &dev_priv->wm.skl_hw.ddb;
11817
11818         if (INTEL_GEN(dev_priv) >= 11)
11819                 if (hw_ddb.enabled_slices != sw_ddb->enabled_slices)
11820                         DRM_ERROR("mismatch in DBUF Slices (expected %u, got %u)\n",
11821                                   sw_ddb->enabled_slices,
11822                                   hw_ddb.enabled_slices);
11823         /* planes */
11824         for_each_universal_plane(dev_priv, pipe, plane) {
11825                 hw_plane_wm = &hw_wm.planes[plane];
11826                 sw_plane_wm = &sw_wm->planes[plane];
11827
11828                 /* Watermarks */
11829                 for (level = 0; level <= max_level; level++) {
11830                         if (skl_wm_level_equals(&hw_plane_wm->wm[level],
11831                                                 &sw_plane_wm->wm[level]))
11832                                 continue;
11833
11834                         DRM_ERROR("mismatch in WM pipe %c plane %d level %d (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
11835                                   pipe_name(pipe), plane + 1, level,
11836                                   sw_plane_wm->wm[level].plane_en,
11837                                   sw_plane_wm->wm[level].plane_res_b,
11838                                   sw_plane_wm->wm[level].plane_res_l,
11839                                   hw_plane_wm->wm[level].plane_en,
11840                                   hw_plane_wm->wm[level].plane_res_b,
11841                                   hw_plane_wm->wm[level].plane_res_l);
11842                 }
11843
11844                 if (!skl_wm_level_equals(&hw_plane_wm->trans_wm,
11845                                          &sw_plane_wm->trans_wm)) {
11846                         DRM_ERROR("mismatch in trans WM pipe %c plane %d (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
11847                                   pipe_name(pipe), plane + 1,
11848                                   sw_plane_wm->trans_wm.plane_en,
11849                                   sw_plane_wm->trans_wm.plane_res_b,
11850                                   sw_plane_wm->trans_wm.plane_res_l,
11851                                   hw_plane_wm->trans_wm.plane_en,
11852                                   hw_plane_wm->trans_wm.plane_res_b,
11853                                   hw_plane_wm->trans_wm.plane_res_l);
11854                 }
11855
11856                 /* DDB */
11857                 hw_ddb_entry = &hw_ddb.plane[pipe][plane];
11858                 sw_ddb_entry = &sw_ddb->plane[pipe][plane];
11859
11860                 if (!skl_ddb_entry_equal(hw_ddb_entry, sw_ddb_entry)) {
11861                         DRM_ERROR("mismatch in DDB state pipe %c plane %d (expected (%u,%u), found (%u,%u))\n",
11862                                   pipe_name(pipe), plane + 1,
11863                                   sw_ddb_entry->start, sw_ddb_entry->end,
11864                                   hw_ddb_entry->start, hw_ddb_entry->end);
11865                 }
11866         }
11867
11868         /*
11869          * cursor
11870          * If the cursor plane isn't active, we may not have updated it's ddb
11871          * allocation. In that case since the ddb allocation will be updated
11872          * once the plane becomes visible, we can skip this check
11873          */
11874         if (1) {
11875                 hw_plane_wm = &hw_wm.planes[PLANE_CURSOR];
11876                 sw_plane_wm = &sw_wm->planes[PLANE_CURSOR];
11877
11878                 /* Watermarks */
11879                 for (level = 0; level <= max_level; level++) {
11880                         if (skl_wm_level_equals(&hw_plane_wm->wm[level],
11881                                                 &sw_plane_wm->wm[level]))
11882                                 continue;
11883
11884                         DRM_ERROR("mismatch in WM pipe %c cursor level %d (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
11885                                   pipe_name(pipe), level,
11886                                   sw_plane_wm->wm[level].plane_en,
11887                                   sw_plane_wm->wm[level].plane_res_b,
11888                                   sw_plane_wm->wm[level].plane_res_l,
11889                                   hw_plane_wm->wm[level].plane_en,
11890                                   hw_plane_wm->wm[level].plane_res_b,
11891                                   hw_plane_wm->wm[level].plane_res_l);
11892                 }
11893
11894                 if (!skl_wm_level_equals(&hw_plane_wm->trans_wm,
11895                                          &sw_plane_wm->trans_wm)) {
11896                         DRM_ERROR("mismatch in trans WM pipe %c cursor (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
11897                                   pipe_name(pipe),
11898                                   sw_plane_wm->trans_wm.plane_en,
11899                                   sw_plane_wm->trans_wm.plane_res_b,
11900                                   sw_plane_wm->trans_wm.plane_res_l,
11901                                   hw_plane_wm->trans_wm.plane_en,
11902                                   hw_plane_wm->trans_wm.plane_res_b,
11903                                   hw_plane_wm->trans_wm.plane_res_l);
11904                 }
11905
11906                 /* DDB */
11907                 hw_ddb_entry = &hw_ddb.plane[pipe][PLANE_CURSOR];
11908                 sw_ddb_entry = &sw_ddb->plane[pipe][PLANE_CURSOR];
11909
11910                 if (!skl_ddb_entry_equal(hw_ddb_entry, sw_ddb_entry)) {
11911                         DRM_ERROR("mismatch in DDB state pipe %c cursor (expected (%u,%u), found (%u,%u))\n",
11912                                   pipe_name(pipe),
11913                                   sw_ddb_entry->start, sw_ddb_entry->end,
11914                                   hw_ddb_entry->start, hw_ddb_entry->end);
11915                 }
11916         }
11917 }
11918
11919 static void
11920 verify_connector_state(struct drm_device *dev,
11921                        struct drm_atomic_state *state,
11922                        struct drm_crtc *crtc)
11923 {
11924         struct drm_connector *connector;
11925         struct drm_connector_state *new_conn_state;
11926         int i;
11927
11928         for_each_new_connector_in_state(state, connector, new_conn_state, i) {
11929                 struct drm_encoder *encoder = connector->encoder;
11930                 struct drm_crtc_state *crtc_state = NULL;
11931
11932                 if (new_conn_state->crtc != crtc)
11933                         continue;
11934
11935                 if (crtc)
11936                         crtc_state = drm_atomic_get_new_crtc_state(state, new_conn_state->crtc);
11937
11938                 intel_connector_verify_state(crtc_state, new_conn_state);
11939
11940                 I915_STATE_WARN(new_conn_state->best_encoder != encoder,
11941                      "connector's atomic encoder doesn't match legacy encoder\n");
11942         }
11943 }
11944
11945 static void
11946 verify_encoder_state(struct drm_device *dev, struct drm_atomic_state *state)
11947 {
11948         struct intel_encoder *encoder;
11949         struct drm_connector *connector;
11950         struct drm_connector_state *old_conn_state, *new_conn_state;
11951         int i;
11952
11953         for_each_intel_encoder(dev, encoder) {
11954                 bool enabled = false, found = false;
11955                 enum pipe pipe;
11956
11957                 DRM_DEBUG_KMS("[ENCODER:%d:%s]\n",
11958                               encoder->base.base.id,
11959                               encoder->base.name);
11960
11961                 for_each_oldnew_connector_in_state(state, connector, old_conn_state,
11962                                                    new_conn_state, i) {
11963                         if (old_conn_state->best_encoder == &encoder->base)
11964                                 found = true;
11965
11966                         if (new_conn_state->best_encoder != &encoder->base)
11967                                 continue;
11968                         found = enabled = true;
11969
11970                         I915_STATE_WARN(new_conn_state->crtc !=
11971                                         encoder->base.crtc,
11972                              "connector's crtc doesn't match encoder crtc\n");
11973                 }
11974
11975                 if (!found)
11976                         continue;
11977
11978                 I915_STATE_WARN(!!encoder->base.crtc != enabled,
11979                      "encoder's enabled state mismatch "
11980                      "(expected %i, found %i)\n",
11981                      !!encoder->base.crtc, enabled);
11982
11983                 if (!encoder->base.crtc) {
11984                         bool active;
11985
11986                         active = encoder->get_hw_state(encoder, &pipe);
11987                         I915_STATE_WARN(active,
11988                              "encoder detached but still enabled on pipe %c.\n",
11989                              pipe_name(pipe));
11990                 }
11991         }
11992 }
11993
11994 static void
11995 verify_crtc_state(struct drm_crtc *crtc,
11996                   struct drm_crtc_state *old_crtc_state,
11997                   struct drm_crtc_state *new_crtc_state)
11998 {
11999         struct drm_device *dev = crtc->dev;
12000         struct drm_i915_private *dev_priv = to_i915(dev);
12001         struct intel_encoder *encoder;
12002         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
12003         struct intel_crtc_state *pipe_config, *sw_config;
12004         struct drm_atomic_state *old_state;
12005         bool active;
12006
12007         old_state = old_crtc_state->state;
12008         __drm_atomic_helper_crtc_destroy_state(old_crtc_state);
12009         pipe_config = to_intel_crtc_state(old_crtc_state);
12010         memset(pipe_config, 0, sizeof(*pipe_config));
12011         pipe_config->base.crtc = crtc;
12012         pipe_config->base.state = old_state;
12013
12014         DRM_DEBUG_KMS("[CRTC:%d:%s]\n", crtc->base.id, crtc->name);
12015
12016         active = dev_priv->display.get_pipe_config(intel_crtc, pipe_config);
12017
12018         /* we keep both pipes enabled on 830 */
12019         if (IS_I830(dev_priv))
12020                 active = new_crtc_state->active;
12021
12022         I915_STATE_WARN(new_crtc_state->active != active,
12023              "crtc active state doesn't match with hw state "
12024              "(expected %i, found %i)\n", new_crtc_state->active, active);
12025
12026         I915_STATE_WARN(intel_crtc->active != new_crtc_state->active,
12027              "transitional active state does not match atomic hw state "
12028              "(expected %i, found %i)\n", new_crtc_state->active, intel_crtc->active);
12029
12030         for_each_encoder_on_crtc(dev, crtc, encoder) {
12031                 enum pipe pipe;
12032
12033                 active = encoder->get_hw_state(encoder, &pipe);
12034                 I915_STATE_WARN(active != new_crtc_state->active,
12035                         "[ENCODER:%i] active %i with crtc active %i\n",
12036                         encoder->base.base.id, active, new_crtc_state->active);
12037
12038                 I915_STATE_WARN(active && intel_crtc->pipe != pipe,
12039                                 "Encoder connected to wrong pipe %c\n",
12040                                 pipe_name(pipe));
12041
12042                 if (active)
12043                         encoder->get_config(encoder, pipe_config);
12044         }
12045
12046         intel_crtc_compute_pixel_rate(pipe_config);
12047
12048         if (!new_crtc_state->active)
12049                 return;
12050
12051         intel_pipe_config_sanity_check(dev_priv, pipe_config);
12052
12053         sw_config = to_intel_crtc_state(new_crtc_state);
12054         if (!intel_pipe_config_compare(dev_priv, sw_config,
12055                                        pipe_config, false)) {
12056                 I915_STATE_WARN(1, "pipe state doesn't match!\n");
12057                 intel_dump_pipe_config(intel_crtc, pipe_config,
12058                                        "[hw state]");
12059                 intel_dump_pipe_config(intel_crtc, sw_config,
12060                                        "[sw state]");
12061         }
12062 }
12063
12064 static void
12065 intel_verify_planes(struct intel_atomic_state *state)
12066 {
12067         struct intel_plane *plane;
12068         const struct intel_plane_state *plane_state;
12069         int i;
12070
12071         for_each_new_intel_plane_in_state(state, plane,
12072                                           plane_state, i)
12073                 assert_plane(plane, plane_state->base.visible);
12074 }
12075
12076 static void
12077 verify_single_dpll_state(struct drm_i915_private *dev_priv,
12078                          struct intel_shared_dpll *pll,
12079                          struct drm_crtc *crtc,
12080                          struct drm_crtc_state *new_state)
12081 {
12082         struct intel_dpll_hw_state dpll_hw_state;
12083         unsigned int crtc_mask;
12084         bool active;
12085
12086         memset(&dpll_hw_state, 0, sizeof(dpll_hw_state));
12087
12088         DRM_DEBUG_KMS("%s\n", pll->info->name);
12089
12090         active = pll->info->funcs->get_hw_state(dev_priv, pll, &dpll_hw_state);
12091
12092         if (!(pll->info->flags & INTEL_DPLL_ALWAYS_ON)) {
12093                 I915_STATE_WARN(!pll->on && pll->active_mask,
12094                      "pll in active use but not on in sw tracking\n");
12095                 I915_STATE_WARN(pll->on && !pll->active_mask,
12096                      "pll is on but not used by any active crtc\n");
12097                 I915_STATE_WARN(pll->on != active,
12098                      "pll on state mismatch (expected %i, found %i)\n",
12099                      pll->on, active);
12100         }
12101
12102         if (!crtc) {
12103                 I915_STATE_WARN(pll->active_mask & ~pll->state.crtc_mask,
12104                                 "more active pll users than references: %x vs %x\n",
12105                                 pll->active_mask, pll->state.crtc_mask);
12106
12107                 return;
12108         }
12109
12110         crtc_mask = drm_crtc_mask(crtc);
12111
12112         if (new_state->active)
12113                 I915_STATE_WARN(!(pll->active_mask & crtc_mask),
12114                                 "pll active mismatch (expected pipe %c in active mask 0x%02x)\n",
12115                                 pipe_name(drm_crtc_index(crtc)), pll->active_mask);
12116         else
12117                 I915_STATE_WARN(pll->active_mask & crtc_mask,
12118                                 "pll active mismatch (didn't expect pipe %c in active mask 0x%02x)\n",
12119                                 pipe_name(drm_crtc_index(crtc)), pll->active_mask);
12120
12121         I915_STATE_WARN(!(pll->state.crtc_mask & crtc_mask),
12122                         "pll enabled crtcs mismatch (expected 0x%x in 0x%02x)\n",
12123                         crtc_mask, pll->state.crtc_mask);
12124
12125         I915_STATE_WARN(pll->on && memcmp(&pll->state.hw_state,
12126                                           &dpll_hw_state,
12127                                           sizeof(dpll_hw_state)),
12128                         "pll hw state mismatch\n");
12129 }
12130
12131 static void
12132 verify_shared_dpll_state(struct drm_device *dev, struct drm_crtc *crtc,
12133                          struct drm_crtc_state *old_crtc_state,
12134                          struct drm_crtc_state *new_crtc_state)
12135 {
12136         struct drm_i915_private *dev_priv = to_i915(dev);
12137         struct intel_crtc_state *old_state = to_intel_crtc_state(old_crtc_state);
12138         struct intel_crtc_state *new_state = to_intel_crtc_state(new_crtc_state);
12139
12140         if (new_state->shared_dpll)
12141                 verify_single_dpll_state(dev_priv, new_state->shared_dpll, crtc, new_crtc_state);
12142
12143         if (old_state->shared_dpll &&
12144             old_state->shared_dpll != new_state->shared_dpll) {
12145                 unsigned int crtc_mask = drm_crtc_mask(crtc);
12146                 struct intel_shared_dpll *pll = old_state->shared_dpll;
12147
12148                 I915_STATE_WARN(pll->active_mask & crtc_mask,
12149                                 "pll active mismatch (didn't expect pipe %c in active mask)\n",
12150                                 pipe_name(drm_crtc_index(crtc)));
12151                 I915_STATE_WARN(pll->state.crtc_mask & crtc_mask,
12152                                 "pll enabled crtcs mismatch (found %x in enabled mask)\n",
12153                                 pipe_name(drm_crtc_index(crtc)));
12154         }
12155 }
12156
12157 static void
12158 intel_modeset_verify_crtc(struct drm_crtc *crtc,
12159                           struct drm_atomic_state *state,
12160                           struct drm_crtc_state *old_state,
12161                           struct drm_crtc_state *new_state)
12162 {
12163         if (!needs_modeset(new_state) &&
12164             !to_intel_crtc_state(new_state)->update_pipe)
12165                 return;
12166
12167         verify_wm_state(crtc, new_state);
12168         verify_connector_state(crtc->dev, state, crtc);
12169         verify_crtc_state(crtc, old_state, new_state);
12170         verify_shared_dpll_state(crtc->dev, crtc, old_state, new_state);
12171 }
12172
12173 static void
12174 verify_disabled_dpll_state(struct drm_device *dev)
12175 {
12176         struct drm_i915_private *dev_priv = to_i915(dev);
12177         int i;
12178
12179         for (i = 0; i < dev_priv->num_shared_dpll; i++)
12180                 verify_single_dpll_state(dev_priv, &dev_priv->shared_dplls[i], NULL, NULL);
12181 }
12182
12183 static void
12184 intel_modeset_verify_disabled(struct drm_device *dev,
12185                               struct drm_atomic_state *state)
12186 {
12187         verify_encoder_state(dev, state);
12188         verify_connector_state(dev, state, NULL);
12189         verify_disabled_dpll_state(dev);
12190 }
12191
12192 static void update_scanline_offset(struct intel_crtc *crtc)
12193 {
12194         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
12195
12196         /*
12197          * The scanline counter increments at the leading edge of hsync.
12198          *
12199          * On most platforms it starts counting from vtotal-1 on the
12200          * first active line. That means the scanline counter value is
12201          * always one less than what we would expect. Ie. just after
12202          * start of vblank, which also occurs at start of hsync (on the
12203          * last active line), the scanline counter will read vblank_start-1.
12204          *
12205          * On gen2 the scanline counter starts counting from 1 instead
12206          * of vtotal-1, so we have to subtract one (or rather add vtotal-1
12207          * to keep the value positive), instead of adding one.
12208          *
12209          * On HSW+ the behaviour of the scanline counter depends on the output
12210          * type. For DP ports it behaves like most other platforms, but on HDMI
12211          * there's an extra 1 line difference. So we need to add two instead of
12212          * one to the value.
12213          *
12214          * On VLV/CHV DSI the scanline counter would appear to increment
12215          * approx. 1/3 of a scanline before start of vblank. Unfortunately
12216          * that means we can't tell whether we're in vblank or not while
12217          * we're on that particular line. We must still set scanline_offset
12218          * to 1 so that the vblank timestamps come out correct when we query
12219          * the scanline counter from within the vblank interrupt handler.
12220          * However if queried just before the start of vblank we'll get an
12221          * answer that's slightly in the future.
12222          */
12223         if (IS_GEN2(dev_priv)) {
12224                 const struct drm_display_mode *adjusted_mode = &crtc->config->base.adjusted_mode;
12225                 int vtotal;
12226
12227                 vtotal = adjusted_mode->crtc_vtotal;
12228                 if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE)
12229                         vtotal /= 2;
12230
12231                 crtc->scanline_offset = vtotal - 1;
12232         } else if (HAS_DDI(dev_priv) &&
12233                    intel_crtc_has_type(crtc->config, INTEL_OUTPUT_HDMI)) {
12234                 crtc->scanline_offset = 2;
12235         } else
12236                 crtc->scanline_offset = 1;
12237 }
12238
12239 static void intel_modeset_clear_plls(struct drm_atomic_state *state)
12240 {
12241         struct drm_device *dev = state->dev;
12242         struct drm_i915_private *dev_priv = to_i915(dev);
12243         struct drm_crtc *crtc;
12244         struct drm_crtc_state *old_crtc_state, *new_crtc_state;
12245         int i;
12246
12247         if (!dev_priv->display.crtc_compute_clock)
12248                 return;
12249
12250         for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
12251                 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
12252                 struct intel_shared_dpll *old_dpll =
12253                         to_intel_crtc_state(old_crtc_state)->shared_dpll;
12254
12255                 if (!needs_modeset(new_crtc_state))
12256                         continue;
12257
12258                 to_intel_crtc_state(new_crtc_state)->shared_dpll = NULL;
12259
12260                 if (!old_dpll)
12261                         continue;
12262
12263                 intel_release_shared_dpll(old_dpll, intel_crtc, state);
12264         }
12265 }
12266
12267 /*
12268  * This implements the workaround described in the "notes" section of the mode
12269  * set sequence documentation. When going from no pipes or single pipe to
12270  * multiple pipes, and planes are enabled after the pipe, we need to wait at
12271  * least 2 vblanks on the first pipe before enabling planes on the second pipe.
12272  */
12273 static int haswell_mode_set_planes_workaround(struct drm_atomic_state *state)
12274 {
12275         struct drm_crtc_state *crtc_state;
12276         struct intel_crtc *intel_crtc;
12277         struct drm_crtc *crtc;
12278         struct intel_crtc_state *first_crtc_state = NULL;
12279         struct intel_crtc_state *other_crtc_state = NULL;
12280         enum pipe first_pipe = INVALID_PIPE, enabled_pipe = INVALID_PIPE;
12281         int i;
12282
12283         /* look at all crtc's that are going to be enabled in during modeset */
12284         for_each_new_crtc_in_state(state, crtc, crtc_state, i) {
12285                 intel_crtc = to_intel_crtc(crtc);
12286
12287                 if (!crtc_state->active || !needs_modeset(crtc_state))
12288                         continue;
12289
12290                 if (first_crtc_state) {
12291                         other_crtc_state = to_intel_crtc_state(crtc_state);
12292                         break;
12293                 } else {
12294                         first_crtc_state = to_intel_crtc_state(crtc_state);
12295                         first_pipe = intel_crtc->pipe;
12296                 }
12297         }
12298
12299         /* No workaround needed? */
12300         if (!first_crtc_state)
12301                 return 0;
12302
12303         /* w/a possibly needed, check how many crtc's are already enabled. */
12304         for_each_intel_crtc(state->dev, intel_crtc) {
12305                 struct intel_crtc_state *pipe_config;
12306
12307                 pipe_config = intel_atomic_get_crtc_state(state, intel_crtc);
12308                 if (IS_ERR(pipe_config))
12309                         return PTR_ERR(pipe_config);
12310
12311                 pipe_config->hsw_workaround_pipe = INVALID_PIPE;
12312
12313                 if (!pipe_config->base.active ||
12314                     needs_modeset(&pipe_config->base))
12315                         continue;
12316
12317                 /* 2 or more enabled crtcs means no need for w/a */
12318                 if (enabled_pipe != INVALID_PIPE)
12319                         return 0;
12320
12321                 enabled_pipe = intel_crtc->pipe;
12322         }
12323
12324         if (enabled_pipe != INVALID_PIPE)
12325                 first_crtc_state->hsw_workaround_pipe = enabled_pipe;
12326         else if (other_crtc_state)
12327                 other_crtc_state->hsw_workaround_pipe = first_pipe;
12328
12329         return 0;
12330 }
12331
12332 static int intel_lock_all_pipes(struct drm_atomic_state *state)
12333 {
12334         struct drm_crtc *crtc;
12335
12336         /* Add all pipes to the state */
12337         for_each_crtc(state->dev, crtc) {
12338                 struct drm_crtc_state *crtc_state;
12339
12340                 crtc_state = drm_atomic_get_crtc_state(state, crtc);
12341                 if (IS_ERR(crtc_state))
12342                         return PTR_ERR(crtc_state);
12343         }
12344
12345         return 0;
12346 }
12347
12348 static int intel_modeset_all_pipes(struct drm_atomic_state *state)
12349 {
12350         struct drm_crtc *crtc;
12351
12352         /*
12353          * Add all pipes to the state, and force
12354          * a modeset on all the active ones.
12355          */
12356         for_each_crtc(state->dev, crtc) {
12357                 struct drm_crtc_state *crtc_state;
12358                 int ret;
12359
12360                 crtc_state = drm_atomic_get_crtc_state(state, crtc);
12361                 if (IS_ERR(crtc_state))
12362                         return PTR_ERR(crtc_state);
12363
12364                 if (!crtc_state->active || needs_modeset(crtc_state))
12365                         continue;
12366
12367                 crtc_state->mode_changed = true;
12368
12369                 ret = drm_atomic_add_affected_connectors(state, crtc);
12370                 if (ret)
12371                         return ret;
12372
12373                 ret = drm_atomic_add_affected_planes(state, crtc);
12374                 if (ret)
12375                         return ret;
12376         }
12377
12378         return 0;
12379 }
12380
12381 static int intel_modeset_checks(struct drm_atomic_state *state)
12382 {
12383         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
12384         struct drm_i915_private *dev_priv = to_i915(state->dev);
12385         struct drm_crtc *crtc;
12386         struct drm_crtc_state *old_crtc_state, *new_crtc_state;
12387         int ret = 0, i;
12388
12389         if (!check_digital_port_conflicts(state)) {
12390                 DRM_DEBUG_KMS("rejecting conflicting digital port configuration\n");
12391                 return -EINVAL;
12392         }
12393
12394         intel_state->modeset = true;
12395         intel_state->active_crtcs = dev_priv->active_crtcs;
12396         intel_state->cdclk.logical = dev_priv->cdclk.logical;
12397         intel_state->cdclk.actual = dev_priv->cdclk.actual;
12398
12399         for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
12400                 if (new_crtc_state->active)
12401                         intel_state->active_crtcs |= 1 << i;
12402                 else
12403                         intel_state->active_crtcs &= ~(1 << i);
12404
12405                 if (old_crtc_state->active != new_crtc_state->active)
12406                         intel_state->active_pipe_changes |= drm_crtc_mask(crtc);
12407         }
12408
12409         /*
12410          * See if the config requires any additional preparation, e.g.
12411          * to adjust global state with pipes off.  We need to do this
12412          * here so we can get the modeset_pipe updated config for the new
12413          * mode set on this crtc.  For other crtcs we need to use the
12414          * adjusted_mode bits in the crtc directly.
12415          */
12416         if (dev_priv->display.modeset_calc_cdclk) {
12417                 ret = dev_priv->display.modeset_calc_cdclk(state);
12418                 if (ret < 0)
12419                         return ret;
12420
12421                 /*
12422                  * Writes to dev_priv->cdclk.logical must protected by
12423                  * holding all the crtc locks, even if we don't end up
12424                  * touching the hardware
12425                  */
12426                 if (intel_cdclk_changed(&dev_priv->cdclk.logical,
12427                                         &intel_state->cdclk.logical)) {
12428                         ret = intel_lock_all_pipes(state);
12429                         if (ret < 0)
12430                                 return ret;
12431                 }
12432
12433                 /* All pipes must be switched off while we change the cdclk. */
12434                 if (intel_cdclk_needs_modeset(&dev_priv->cdclk.actual,
12435                                               &intel_state->cdclk.actual)) {
12436                         ret = intel_modeset_all_pipes(state);
12437                         if (ret < 0)
12438                                 return ret;
12439                 }
12440
12441                 DRM_DEBUG_KMS("New cdclk calculated to be logical %u kHz, actual %u kHz\n",
12442                               intel_state->cdclk.logical.cdclk,
12443                               intel_state->cdclk.actual.cdclk);
12444                 DRM_DEBUG_KMS("New voltage level calculated to be logical %u, actual %u\n",
12445                               intel_state->cdclk.logical.voltage_level,
12446                               intel_state->cdclk.actual.voltage_level);
12447         } else {
12448                 to_intel_atomic_state(state)->cdclk.logical = dev_priv->cdclk.logical;
12449         }
12450
12451         intel_modeset_clear_plls(state);
12452
12453         if (IS_HASWELL(dev_priv))
12454                 return haswell_mode_set_planes_workaround(state);
12455
12456         return 0;
12457 }
12458
12459 /*
12460  * Handle calculation of various watermark data at the end of the atomic check
12461  * phase.  The code here should be run after the per-crtc and per-plane 'check'
12462  * handlers to ensure that all derived state has been updated.
12463  */
12464 static int calc_watermark_data(struct drm_atomic_state *state)
12465 {
12466         struct drm_device *dev = state->dev;
12467         struct drm_i915_private *dev_priv = to_i915(dev);
12468
12469         /* Is there platform-specific watermark information to calculate? */
12470         if (dev_priv->display.compute_global_watermarks)
12471                 return dev_priv->display.compute_global_watermarks(state);
12472
12473         return 0;
12474 }
12475
12476 /**
12477  * intel_atomic_check - validate state object
12478  * @dev: drm device
12479  * @state: state to validate
12480  */
12481 static int intel_atomic_check(struct drm_device *dev,
12482                               struct drm_atomic_state *state)
12483 {
12484         struct drm_i915_private *dev_priv = to_i915(dev);
12485         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
12486         struct drm_crtc *crtc;
12487         struct drm_crtc_state *old_crtc_state, *crtc_state;
12488         int ret, i;
12489         bool any_ms = false;
12490
12491         /* Catch I915_MODE_FLAG_INHERITED */
12492         for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state,
12493                                       crtc_state, i) {
12494                 if (crtc_state->mode.private_flags !=
12495                     old_crtc_state->mode.private_flags)
12496                         crtc_state->mode_changed = true;
12497         }
12498
12499         ret = drm_atomic_helper_check_modeset(dev, state);
12500         if (ret)
12501                 return ret;
12502
12503         for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, crtc_state, i) {
12504                 struct intel_crtc_state *pipe_config =
12505                         to_intel_crtc_state(crtc_state);
12506
12507                 if (!needs_modeset(crtc_state))
12508                         continue;
12509
12510                 if (!crtc_state->enable) {
12511                         any_ms = true;
12512                         continue;
12513                 }
12514
12515                 ret = intel_modeset_pipe_config(crtc, pipe_config);
12516                 if (ret) {
12517                         intel_dump_pipe_config(to_intel_crtc(crtc),
12518                                                pipe_config, "[failed]");
12519                         return ret;
12520                 }
12521
12522                 if (i915_modparams.fastboot &&
12523                     intel_pipe_config_compare(dev_priv,
12524                                         to_intel_crtc_state(old_crtc_state),
12525                                         pipe_config, true)) {
12526                         crtc_state->mode_changed = false;
12527                         pipe_config->update_pipe = true;
12528                 }
12529
12530                 if (needs_modeset(crtc_state))
12531                         any_ms = true;
12532
12533                 intel_dump_pipe_config(to_intel_crtc(crtc), pipe_config,
12534                                        needs_modeset(crtc_state) ?
12535                                        "[modeset]" : "[fastset]");
12536         }
12537
12538         if (any_ms) {
12539                 ret = intel_modeset_checks(state);
12540
12541                 if (ret)
12542                         return ret;
12543         } else {
12544                 intel_state->cdclk.logical = dev_priv->cdclk.logical;
12545         }
12546
12547         ret = drm_atomic_helper_check_planes(dev, state);
12548         if (ret)
12549                 return ret;
12550
12551         intel_fbc_choose_crtc(dev_priv, intel_state);
12552         return calc_watermark_data(state);
12553 }
12554
12555 static int intel_atomic_prepare_commit(struct drm_device *dev,
12556                                        struct drm_atomic_state *state)
12557 {
12558         return drm_atomic_helper_prepare_planes(dev, state);
12559 }
12560
12561 u32 intel_crtc_get_vblank_counter(struct intel_crtc *crtc)
12562 {
12563         struct drm_device *dev = crtc->base.dev;
12564
12565         if (!dev->max_vblank_count)
12566                 return (u32)drm_crtc_accurate_vblank_count(&crtc->base);
12567
12568         return dev->driver->get_vblank_counter(dev, crtc->pipe);
12569 }
12570
12571 static void intel_update_crtc(struct drm_crtc *crtc,
12572                               struct drm_atomic_state *state,
12573                               struct drm_crtc_state *old_crtc_state,
12574                               struct drm_crtc_state *new_crtc_state)
12575 {
12576         struct drm_device *dev = crtc->dev;
12577         struct drm_i915_private *dev_priv = to_i915(dev);
12578         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
12579         struct intel_crtc_state *pipe_config = to_intel_crtc_state(new_crtc_state);
12580         bool modeset = needs_modeset(new_crtc_state);
12581         struct intel_plane_state *new_plane_state =
12582                 intel_atomic_get_new_plane_state(to_intel_atomic_state(state),
12583                                                  to_intel_plane(crtc->primary));
12584
12585         if (modeset) {
12586                 update_scanline_offset(intel_crtc);
12587                 dev_priv->display.crtc_enable(pipe_config, state);
12588
12589                 /* vblanks work again, re-enable pipe CRC. */
12590                 intel_crtc_enable_pipe_crc(intel_crtc);
12591         } else {
12592                 intel_pre_plane_update(to_intel_crtc_state(old_crtc_state),
12593                                        pipe_config);
12594         }
12595
12596         if (new_plane_state)
12597                 intel_fbc_enable(intel_crtc, pipe_config, new_plane_state);
12598
12599         drm_atomic_helper_commit_planes_on_crtc(old_crtc_state);
12600 }
12601
12602 static void intel_update_crtcs(struct drm_atomic_state *state)
12603 {
12604         struct drm_crtc *crtc;
12605         struct drm_crtc_state *old_crtc_state, *new_crtc_state;
12606         int i;
12607
12608         for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
12609                 if (!new_crtc_state->active)
12610                         continue;
12611
12612                 intel_update_crtc(crtc, state, old_crtc_state,
12613                                   new_crtc_state);
12614         }
12615 }
12616
12617 static void skl_update_crtcs(struct drm_atomic_state *state)
12618 {
12619         struct drm_i915_private *dev_priv = to_i915(state->dev);
12620         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
12621         struct drm_crtc *crtc;
12622         struct intel_crtc *intel_crtc;
12623         struct drm_crtc_state *old_crtc_state, *new_crtc_state;
12624         struct intel_crtc_state *cstate;
12625         unsigned int updated = 0;
12626         bool progress;
12627         enum pipe pipe;
12628         int i;
12629         u8 hw_enabled_slices = dev_priv->wm.skl_hw.ddb.enabled_slices;
12630         u8 required_slices = intel_state->wm_results.ddb.enabled_slices;
12631
12632         const struct skl_ddb_entry *entries[I915_MAX_PIPES] = {};
12633
12634         for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i)
12635                 /* ignore allocations for crtc's that have been turned off. */
12636                 if (new_crtc_state->active)
12637                         entries[i] = &to_intel_crtc_state(old_crtc_state)->wm.skl.ddb;
12638
12639         /* If 2nd DBuf slice required, enable it here */
12640         if (INTEL_GEN(dev_priv) >= 11 && required_slices > hw_enabled_slices)
12641                 icl_dbuf_slices_update(dev_priv, required_slices);
12642
12643         /*
12644          * Whenever the number of active pipes changes, we need to make sure we
12645          * update the pipes in the right order so that their ddb allocations
12646          * never overlap with eachother inbetween CRTC updates. Otherwise we'll
12647          * cause pipe underruns and other bad stuff.
12648          */
12649         do {
12650                 progress = false;
12651
12652                 for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
12653                         bool vbl_wait = false;
12654                         unsigned int cmask = drm_crtc_mask(crtc);
12655
12656                         intel_crtc = to_intel_crtc(crtc);
12657                         cstate = to_intel_crtc_state(new_crtc_state);
12658                         pipe = intel_crtc->pipe;
12659
12660                         if (updated & cmask || !cstate->base.active)
12661                                 continue;
12662
12663                         if (skl_ddb_allocation_overlaps(dev_priv,
12664                                                         entries,
12665                                                         &cstate->wm.skl.ddb,
12666                                                         i))
12667                                 continue;
12668
12669                         updated |= cmask;
12670                         entries[i] = &cstate->wm.skl.ddb;
12671
12672                         /*
12673                          * If this is an already active pipe, it's DDB changed,
12674                          * and this isn't the last pipe that needs updating
12675                          * then we need to wait for a vblank to pass for the
12676                          * new ddb allocation to take effect.
12677                          */
12678                         if (!skl_ddb_entry_equal(&cstate->wm.skl.ddb,
12679                                                  &to_intel_crtc_state(old_crtc_state)->wm.skl.ddb) &&
12680                             !new_crtc_state->active_changed &&
12681                             intel_state->wm_results.dirty_pipes != updated)
12682                                 vbl_wait = true;
12683
12684                         intel_update_crtc(crtc, state, old_crtc_state,
12685                                           new_crtc_state);
12686
12687                         if (vbl_wait)
12688                                 intel_wait_for_vblank(dev_priv, pipe);
12689
12690                         progress = true;
12691                 }
12692         } while (progress);
12693
12694         /* If 2nd DBuf slice is no more required disable it */
12695         if (INTEL_GEN(dev_priv) >= 11 && required_slices < hw_enabled_slices)
12696                 icl_dbuf_slices_update(dev_priv, required_slices);
12697 }
12698
12699 static void intel_atomic_helper_free_state(struct drm_i915_private *dev_priv)
12700 {
12701         struct intel_atomic_state *state, *next;
12702         struct llist_node *freed;
12703
12704         freed = llist_del_all(&dev_priv->atomic_helper.free_list);
12705         llist_for_each_entry_safe(state, next, freed, freed)
12706                 drm_atomic_state_put(&state->base);
12707 }
12708
12709 static void intel_atomic_helper_free_state_worker(struct work_struct *work)
12710 {
12711         struct drm_i915_private *dev_priv =
12712                 container_of(work, typeof(*dev_priv), atomic_helper.free_work);
12713
12714         intel_atomic_helper_free_state(dev_priv);
12715 }
12716
12717 static void intel_atomic_commit_fence_wait(struct intel_atomic_state *intel_state)
12718 {
12719         struct wait_queue_entry wait_fence, wait_reset;
12720         struct drm_i915_private *dev_priv = to_i915(intel_state->base.dev);
12721
12722         init_wait_entry(&wait_fence, 0);
12723         init_wait_entry(&wait_reset, 0);
12724         for (;;) {
12725                 prepare_to_wait(&intel_state->commit_ready.wait,
12726                                 &wait_fence, TASK_UNINTERRUPTIBLE);
12727                 prepare_to_wait(&dev_priv->gpu_error.wait_queue,
12728                                 &wait_reset, TASK_UNINTERRUPTIBLE);
12729
12730
12731                 if (i915_sw_fence_done(&intel_state->commit_ready)
12732                     || test_bit(I915_RESET_MODESET, &dev_priv->gpu_error.flags))
12733                         break;
12734
12735                 schedule();
12736         }
12737         finish_wait(&intel_state->commit_ready.wait, &wait_fence);
12738         finish_wait(&dev_priv->gpu_error.wait_queue, &wait_reset);
12739 }
12740
12741 static void intel_atomic_cleanup_work(struct work_struct *work)
12742 {
12743         struct drm_atomic_state *state =
12744                 container_of(work, struct drm_atomic_state, commit_work);
12745         struct drm_i915_private *i915 = to_i915(state->dev);
12746
12747         drm_atomic_helper_cleanup_planes(&i915->drm, state);
12748         drm_atomic_helper_commit_cleanup_done(state);
12749         drm_atomic_state_put(state);
12750
12751         intel_atomic_helper_free_state(i915);
12752 }
12753
12754 static void intel_atomic_commit_tail(struct drm_atomic_state *state)
12755 {
12756         struct drm_device *dev = state->dev;
12757         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
12758         struct drm_i915_private *dev_priv = to_i915(dev);
12759         struct drm_crtc_state *old_crtc_state, *new_crtc_state;
12760         struct drm_crtc *crtc;
12761         struct intel_crtc_state *intel_cstate;
12762         u64 put_domains[I915_MAX_PIPES] = {};
12763         int i;
12764
12765         intel_atomic_commit_fence_wait(intel_state);
12766
12767         drm_atomic_helper_wait_for_dependencies(state);
12768
12769         if (intel_state->modeset)
12770                 intel_display_power_get(dev_priv, POWER_DOMAIN_MODESET);
12771
12772         for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
12773                 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
12774
12775                 if (needs_modeset(new_crtc_state) ||
12776                     to_intel_crtc_state(new_crtc_state)->update_pipe) {
12777
12778                         put_domains[to_intel_crtc(crtc)->pipe] =
12779                                 modeset_get_crtc_power_domains(crtc,
12780                                         to_intel_crtc_state(new_crtc_state));
12781                 }
12782
12783                 if (!needs_modeset(new_crtc_state))
12784                         continue;
12785
12786                 intel_pre_plane_update(to_intel_crtc_state(old_crtc_state),
12787                                        to_intel_crtc_state(new_crtc_state));
12788
12789                 if (old_crtc_state->active) {
12790                         intel_crtc_disable_planes(crtc, old_crtc_state->plane_mask);
12791
12792                         /*
12793                          * We need to disable pipe CRC before disabling the pipe,
12794                          * or we race against vblank off.
12795                          */
12796                         intel_crtc_disable_pipe_crc(intel_crtc);
12797
12798                         dev_priv->display.crtc_disable(to_intel_crtc_state(old_crtc_state), state);
12799                         intel_crtc->active = false;
12800                         intel_fbc_disable(intel_crtc);
12801                         intel_disable_shared_dpll(intel_crtc);
12802
12803                         /*
12804                          * Underruns don't always raise
12805                          * interrupts, so check manually.
12806                          */
12807                         intel_check_cpu_fifo_underruns(dev_priv);
12808                         intel_check_pch_fifo_underruns(dev_priv);
12809
12810                         /* FIXME unify this for all platforms */
12811                         if (!new_crtc_state->active &&
12812                             !HAS_GMCH_DISPLAY(dev_priv) &&
12813                             dev_priv->display.initial_watermarks)
12814                                 dev_priv->display.initial_watermarks(intel_state,
12815                                                                      to_intel_crtc_state(new_crtc_state));
12816                 }
12817         }
12818
12819         /* FIXME: Eventually get rid of our intel_crtc->config pointer */
12820         for_each_new_crtc_in_state(state, crtc, new_crtc_state, i)
12821                 to_intel_crtc(crtc)->config = to_intel_crtc_state(new_crtc_state);
12822
12823         if (intel_state->modeset) {
12824                 drm_atomic_helper_update_legacy_modeset_state(state->dev, state);
12825
12826                 intel_set_cdclk(dev_priv, &dev_priv->cdclk.actual);
12827
12828                 /*
12829                  * SKL workaround: bspec recommends we disable the SAGV when we
12830                  * have more then one pipe enabled
12831                  */
12832                 if (!intel_can_enable_sagv(state))
12833                         intel_disable_sagv(dev_priv);
12834
12835                 intel_modeset_verify_disabled(dev, state);
12836         }
12837
12838         /* Complete the events for pipes that have now been disabled */
12839         for_each_new_crtc_in_state(state, crtc, new_crtc_state, i) {
12840                 bool modeset = needs_modeset(new_crtc_state);
12841
12842                 /* Complete events for now disable pipes here. */
12843                 if (modeset && !new_crtc_state->active && new_crtc_state->event) {
12844                         spin_lock_irq(&dev->event_lock);
12845                         drm_crtc_send_vblank_event(crtc, new_crtc_state->event);
12846                         spin_unlock_irq(&dev->event_lock);
12847
12848                         new_crtc_state->event = NULL;
12849                 }
12850         }
12851
12852         /* Now enable the clocks, plane, pipe, and connectors that we set up. */
12853         dev_priv->display.update_crtcs(state);
12854
12855         /* FIXME: We should call drm_atomic_helper_commit_hw_done() here
12856          * already, but still need the state for the delayed optimization. To
12857          * fix this:
12858          * - wrap the optimization/post_plane_update stuff into a per-crtc work.
12859          * - schedule that vblank worker _before_ calling hw_done
12860          * - at the start of commit_tail, cancel it _synchrously
12861          * - switch over to the vblank wait helper in the core after that since
12862          *   we don't need out special handling any more.
12863          */
12864         drm_atomic_helper_wait_for_flip_done(dev, state);
12865
12866         /*
12867          * Now that the vblank has passed, we can go ahead and program the
12868          * optimal watermarks on platforms that need two-step watermark
12869          * programming.
12870          *
12871          * TODO: Move this (and other cleanup) to an async worker eventually.
12872          */
12873         for_each_new_crtc_in_state(state, crtc, new_crtc_state, i) {
12874                 intel_cstate = to_intel_crtc_state(new_crtc_state);
12875
12876                 if (dev_priv->display.optimize_watermarks)
12877                         dev_priv->display.optimize_watermarks(intel_state,
12878                                                               intel_cstate);
12879         }
12880
12881         for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
12882                 intel_post_plane_update(to_intel_crtc_state(old_crtc_state));
12883
12884                 if (put_domains[i])
12885                         modeset_put_power_domains(dev_priv, put_domains[i]);
12886
12887                 intel_modeset_verify_crtc(crtc, state, old_crtc_state, new_crtc_state);
12888         }
12889
12890         if (intel_state->modeset)
12891                 intel_verify_planes(intel_state);
12892
12893         if (intel_state->modeset && intel_can_enable_sagv(state))
12894                 intel_enable_sagv(dev_priv);
12895
12896         drm_atomic_helper_commit_hw_done(state);
12897
12898         if (intel_state->modeset) {
12899                 /* As one of the primary mmio accessors, KMS has a high
12900                  * likelihood of triggering bugs in unclaimed access. After we
12901                  * finish modesetting, see if an error has been flagged, and if
12902                  * so enable debugging for the next modeset - and hope we catch
12903                  * the culprit.
12904                  */
12905                 intel_uncore_arm_unclaimed_mmio_detection(dev_priv);
12906                 intel_display_power_put(dev_priv, POWER_DOMAIN_MODESET);
12907         }
12908
12909         /*
12910          * Defer the cleanup of the old state to a separate worker to not
12911          * impede the current task (userspace for blocking modesets) that
12912          * are executed inline. For out-of-line asynchronous modesets/flips,
12913          * deferring to a new worker seems overkill, but we would place a
12914          * schedule point (cond_resched()) here anyway to keep latencies
12915          * down.
12916          */
12917         INIT_WORK(&state->commit_work, intel_atomic_cleanup_work);
12918         queue_work(system_highpri_wq, &state->commit_work);
12919 }
12920
12921 static void intel_atomic_commit_work(struct work_struct *work)
12922 {
12923         struct drm_atomic_state *state =
12924                 container_of(work, struct drm_atomic_state, commit_work);
12925
12926         intel_atomic_commit_tail(state);
12927 }
12928
12929 static int __i915_sw_fence_call
12930 intel_atomic_commit_ready(struct i915_sw_fence *fence,
12931                           enum i915_sw_fence_notify notify)
12932 {
12933         struct intel_atomic_state *state =
12934                 container_of(fence, struct intel_atomic_state, commit_ready);
12935
12936         switch (notify) {
12937         case FENCE_COMPLETE:
12938                 /* we do blocking waits in the worker, nothing to do here */
12939                 break;
12940         case FENCE_FREE:
12941                 {
12942                         struct intel_atomic_helper *helper =
12943                                 &to_i915(state->base.dev)->atomic_helper;
12944
12945                         if (llist_add(&state->freed, &helper->free_list))
12946                                 schedule_work(&helper->free_work);
12947                         break;
12948                 }
12949         }
12950
12951         return NOTIFY_DONE;
12952 }
12953
12954 static void intel_atomic_track_fbs(struct drm_atomic_state *state)
12955 {
12956         struct drm_plane_state *old_plane_state, *new_plane_state;
12957         struct drm_plane *plane;
12958         int i;
12959
12960         for_each_oldnew_plane_in_state(state, plane, old_plane_state, new_plane_state, i)
12961                 i915_gem_track_fb(intel_fb_obj(old_plane_state->fb),
12962                                   intel_fb_obj(new_plane_state->fb),
12963                                   to_intel_plane(plane)->frontbuffer_bit);
12964 }
12965
12966 /**
12967  * intel_atomic_commit - commit validated state object
12968  * @dev: DRM device
12969  * @state: the top-level driver state object
12970  * @nonblock: nonblocking commit
12971  *
12972  * This function commits a top-level state object that has been validated
12973  * with drm_atomic_helper_check().
12974  *
12975  * RETURNS
12976  * Zero for success or -errno.
12977  */
12978 static int intel_atomic_commit(struct drm_device *dev,
12979                                struct drm_atomic_state *state,
12980                                bool nonblock)
12981 {
12982         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
12983         struct drm_i915_private *dev_priv = to_i915(dev);
12984         int ret = 0;
12985
12986         drm_atomic_state_get(state);
12987         i915_sw_fence_init(&intel_state->commit_ready,
12988                            intel_atomic_commit_ready);
12989
12990         /*
12991          * The intel_legacy_cursor_update() fast path takes care
12992          * of avoiding the vblank waits for simple cursor
12993          * movement and flips. For cursor on/off and size changes,
12994          * we want to perform the vblank waits so that watermark
12995          * updates happen during the correct frames. Gen9+ have
12996          * double buffered watermarks and so shouldn't need this.
12997          *
12998          * Unset state->legacy_cursor_update before the call to
12999          * drm_atomic_helper_setup_commit() because otherwise
13000          * drm_atomic_helper_wait_for_flip_done() is a noop and
13001          * we get FIFO underruns because we didn't wait
13002          * for vblank.
13003          *
13004          * FIXME doing watermarks and fb cleanup from a vblank worker
13005          * (assuming we had any) would solve these problems.
13006          */
13007         if (INTEL_GEN(dev_priv) < 9 && state->legacy_cursor_update) {
13008                 struct intel_crtc_state *new_crtc_state;
13009                 struct intel_crtc *crtc;
13010                 int i;
13011
13012                 for_each_new_intel_crtc_in_state(intel_state, crtc, new_crtc_state, i)
13013                         if (new_crtc_state->wm.need_postvbl_update ||
13014                             new_crtc_state->update_wm_post)
13015                                 state->legacy_cursor_update = false;
13016         }
13017
13018         ret = intel_atomic_prepare_commit(dev, state);
13019         if (ret) {
13020                 DRM_DEBUG_ATOMIC("Preparing state failed with %i\n", ret);
13021                 i915_sw_fence_commit(&intel_state->commit_ready);
13022                 return ret;
13023         }
13024
13025         ret = drm_atomic_helper_setup_commit(state, nonblock);
13026         if (!ret)
13027                 ret = drm_atomic_helper_swap_state(state, true);
13028
13029         if (ret) {
13030                 i915_sw_fence_commit(&intel_state->commit_ready);
13031
13032                 drm_atomic_helper_cleanup_planes(dev, state);
13033                 return ret;
13034         }
13035         dev_priv->wm.distrust_bios_wm = false;
13036         intel_shared_dpll_swap_state(state);
13037         intel_atomic_track_fbs(state);
13038
13039         if (intel_state->modeset) {
13040                 memcpy(dev_priv->min_cdclk, intel_state->min_cdclk,
13041                        sizeof(intel_state->min_cdclk));
13042                 memcpy(dev_priv->min_voltage_level,
13043                        intel_state->min_voltage_level,
13044                        sizeof(intel_state->min_voltage_level));
13045                 dev_priv->active_crtcs = intel_state->active_crtcs;
13046                 dev_priv->cdclk.logical = intel_state->cdclk.logical;
13047                 dev_priv->cdclk.actual = intel_state->cdclk.actual;
13048         }
13049
13050         drm_atomic_state_get(state);
13051         INIT_WORK(&state->commit_work, intel_atomic_commit_work);
13052
13053         i915_sw_fence_commit(&intel_state->commit_ready);
13054         if (nonblock && intel_state->modeset) {
13055                 queue_work(dev_priv->modeset_wq, &state->commit_work);
13056         } else if (nonblock) {
13057                 queue_work(system_unbound_wq, &state->commit_work);
13058         } else {
13059                 if (intel_state->modeset)
13060                         flush_workqueue(dev_priv->modeset_wq);
13061                 intel_atomic_commit_tail(state);
13062         }
13063
13064         return 0;
13065 }
13066
13067 static const struct drm_crtc_funcs intel_crtc_funcs = {
13068         .gamma_set = drm_atomic_helper_legacy_gamma_set,
13069         .set_config = drm_atomic_helper_set_config,
13070         .destroy = intel_crtc_destroy,
13071         .page_flip = drm_atomic_helper_page_flip,
13072         .atomic_duplicate_state = intel_crtc_duplicate_state,
13073         .atomic_destroy_state = intel_crtc_destroy_state,
13074         .set_crc_source = intel_crtc_set_crc_source,
13075         .verify_crc_source = intel_crtc_verify_crc_source,
13076         .get_crc_sources = intel_crtc_get_crc_sources,
13077 };
13078
13079 struct wait_rps_boost {
13080         struct wait_queue_entry wait;
13081
13082         struct drm_crtc *crtc;
13083         struct i915_request *request;
13084 };
13085
13086 static int do_rps_boost(struct wait_queue_entry *_wait,
13087                         unsigned mode, int sync, void *key)
13088 {
13089         struct wait_rps_boost *wait = container_of(_wait, typeof(*wait), wait);
13090         struct i915_request *rq = wait->request;
13091
13092         /*
13093          * If we missed the vblank, but the request is already running it
13094          * is reasonable to assume that it will complete before the next
13095          * vblank without our intervention, so leave RPS alone.
13096          */
13097         if (!i915_request_started(rq))
13098                 gen6_rps_boost(rq, NULL);
13099         i915_request_put(rq);
13100
13101         drm_crtc_vblank_put(wait->crtc);
13102
13103         list_del(&wait->wait.entry);
13104         kfree(wait);
13105         return 1;
13106 }
13107
13108 static void add_rps_boost_after_vblank(struct drm_crtc *crtc,
13109                                        struct dma_fence *fence)
13110 {
13111         struct wait_rps_boost *wait;
13112
13113         if (!dma_fence_is_i915(fence))
13114                 return;
13115
13116         if (INTEL_GEN(to_i915(crtc->dev)) < 6)
13117                 return;
13118
13119         if (drm_crtc_vblank_get(crtc))
13120                 return;
13121
13122         wait = kmalloc(sizeof(*wait), GFP_KERNEL);
13123         if (!wait) {
13124                 drm_crtc_vblank_put(crtc);
13125                 return;
13126         }
13127
13128         wait->request = to_request(dma_fence_get(fence));
13129         wait->crtc = crtc;
13130
13131         wait->wait.func = do_rps_boost;
13132         wait->wait.flags = 0;
13133
13134         add_wait_queue(drm_crtc_vblank_waitqueue(crtc), &wait->wait);
13135 }
13136
13137 static int intel_plane_pin_fb(struct intel_plane_state *plane_state)
13138 {
13139         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
13140         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
13141         struct drm_framebuffer *fb = plane_state->base.fb;
13142         struct i915_vma *vma;
13143
13144         if (plane->id == PLANE_CURSOR &&
13145             INTEL_INFO(dev_priv)->cursor_needs_physical) {
13146                 struct drm_i915_gem_object *obj = intel_fb_obj(fb);
13147                 const int align = intel_cursor_alignment(dev_priv);
13148                 int err;
13149
13150                 err = i915_gem_object_attach_phys(obj, align);
13151                 if (err)
13152                         return err;
13153         }
13154
13155         vma = intel_pin_and_fence_fb_obj(fb,
13156                                          &plane_state->view,
13157                                          intel_plane_uses_fence(plane_state),
13158                                          &plane_state->flags);
13159         if (IS_ERR(vma))
13160                 return PTR_ERR(vma);
13161
13162         plane_state->vma = vma;
13163
13164         return 0;
13165 }
13166
13167 static void intel_plane_unpin_fb(struct intel_plane_state *old_plane_state)
13168 {
13169         struct i915_vma *vma;
13170
13171         vma = fetch_and_zero(&old_plane_state->vma);
13172         if (vma)
13173                 intel_unpin_fb_vma(vma, old_plane_state->flags);
13174 }
13175
13176 static void fb_obj_bump_render_priority(struct drm_i915_gem_object *obj)
13177 {
13178         struct i915_sched_attr attr = {
13179                 .priority = I915_PRIORITY_DISPLAY,
13180         };
13181
13182         i915_gem_object_wait_priority(obj, 0, &attr);
13183 }
13184
13185 /**
13186  * intel_prepare_plane_fb - Prepare fb for usage on plane
13187  * @plane: drm plane to prepare for
13188  * @new_state: the plane state being prepared
13189  *
13190  * Prepares a framebuffer for usage on a display plane.  Generally this
13191  * involves pinning the underlying object and updating the frontbuffer tracking
13192  * bits.  Some older platforms need special physical address handling for
13193  * cursor planes.
13194  *
13195  * Must be called with struct_mutex held.
13196  *
13197  * Returns 0 on success, negative error code on failure.
13198  */
13199 int
13200 intel_prepare_plane_fb(struct drm_plane *plane,
13201                        struct drm_plane_state *new_state)
13202 {
13203         struct intel_atomic_state *intel_state =
13204                 to_intel_atomic_state(new_state->state);
13205         struct drm_i915_private *dev_priv = to_i915(plane->dev);
13206         struct drm_framebuffer *fb = new_state->fb;
13207         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
13208         struct drm_i915_gem_object *old_obj = intel_fb_obj(plane->state->fb);
13209         int ret;
13210
13211         if (old_obj) {
13212                 struct drm_crtc_state *crtc_state =
13213                         drm_atomic_get_new_crtc_state(new_state->state,
13214                                                       plane->state->crtc);
13215
13216                 /* Big Hammer, we also need to ensure that any pending
13217                  * MI_WAIT_FOR_EVENT inside a user batch buffer on the
13218                  * current scanout is retired before unpinning the old
13219                  * framebuffer. Note that we rely on userspace rendering
13220                  * into the buffer attached to the pipe they are waiting
13221                  * on. If not, userspace generates a GPU hang with IPEHR
13222                  * point to the MI_WAIT_FOR_EVENT.
13223                  *
13224                  * This should only fail upon a hung GPU, in which case we
13225                  * can safely continue.
13226                  */
13227                 if (needs_modeset(crtc_state)) {
13228                         ret = i915_sw_fence_await_reservation(&intel_state->commit_ready,
13229                                                               old_obj->resv, NULL,
13230                                                               false, 0,
13231                                                               GFP_KERNEL);
13232                         if (ret < 0)
13233                                 return ret;
13234                 }
13235         }
13236
13237         if (new_state->fence) { /* explicit fencing */
13238                 ret = i915_sw_fence_await_dma_fence(&intel_state->commit_ready,
13239                                                     new_state->fence,
13240                                                     I915_FENCE_TIMEOUT,
13241                                                     GFP_KERNEL);
13242                 if (ret < 0)
13243                         return ret;
13244         }
13245
13246         if (!obj)
13247                 return 0;
13248
13249         ret = i915_gem_object_pin_pages(obj);
13250         if (ret)
13251                 return ret;
13252
13253         ret = mutex_lock_interruptible(&dev_priv->drm.struct_mutex);
13254         if (ret) {
13255                 i915_gem_object_unpin_pages(obj);
13256                 return ret;
13257         }
13258
13259         ret = intel_plane_pin_fb(to_intel_plane_state(new_state));
13260
13261         fb_obj_bump_render_priority(obj);
13262
13263         mutex_unlock(&dev_priv->drm.struct_mutex);
13264         i915_gem_object_unpin_pages(obj);
13265         if (ret)
13266                 return ret;
13267
13268         intel_fb_obj_flush(obj, ORIGIN_DIRTYFB);
13269
13270         if (!new_state->fence) { /* implicit fencing */
13271                 struct dma_fence *fence;
13272
13273                 ret = i915_sw_fence_await_reservation(&intel_state->commit_ready,
13274                                                       obj->resv, NULL,
13275                                                       false, I915_FENCE_TIMEOUT,
13276                                                       GFP_KERNEL);
13277                 if (ret < 0)
13278                         return ret;
13279
13280                 fence = reservation_object_get_excl_rcu(obj->resv);
13281                 if (fence) {
13282                         add_rps_boost_after_vblank(new_state->crtc, fence);
13283                         dma_fence_put(fence);
13284                 }
13285         } else {
13286                 add_rps_boost_after_vblank(new_state->crtc, new_state->fence);
13287         }
13288
13289         /*
13290          * We declare pageflips to be interactive and so merit a small bias
13291          * towards upclocking to deliver the frame on time. By only changing
13292          * the RPS thresholds to sample more regularly and aim for higher
13293          * clocks we can hopefully deliver low power workloads (like kodi)
13294          * that are not quite steady state without resorting to forcing
13295          * maximum clocks following a vblank miss (see do_rps_boost()).
13296          */
13297         if (!intel_state->rps_interactive) {
13298                 intel_rps_mark_interactive(dev_priv, true);
13299                 intel_state->rps_interactive = true;
13300         }
13301
13302         return 0;
13303 }
13304
13305 /**
13306  * intel_cleanup_plane_fb - Cleans up an fb after plane use
13307  * @plane: drm plane to clean up for
13308  * @old_state: the state from the previous modeset
13309  *
13310  * Cleans up a framebuffer that has just been removed from a plane.
13311  *
13312  * Must be called with struct_mutex held.
13313  */
13314 void
13315 intel_cleanup_plane_fb(struct drm_plane *plane,
13316                        struct drm_plane_state *old_state)
13317 {
13318         struct intel_atomic_state *intel_state =
13319                 to_intel_atomic_state(old_state->state);
13320         struct drm_i915_private *dev_priv = to_i915(plane->dev);
13321
13322         if (intel_state->rps_interactive) {
13323                 intel_rps_mark_interactive(dev_priv, false);
13324                 intel_state->rps_interactive = false;
13325         }
13326
13327         /* Should only be called after a successful intel_prepare_plane_fb()! */
13328         mutex_lock(&dev_priv->drm.struct_mutex);
13329         intel_plane_unpin_fb(to_intel_plane_state(old_state));
13330         mutex_unlock(&dev_priv->drm.struct_mutex);
13331 }
13332
13333 int
13334 skl_max_scale(const struct intel_crtc_state *crtc_state,
13335               u32 pixel_format)
13336 {
13337         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
13338         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
13339         int max_scale, mult;
13340         int crtc_clock, max_dotclk, tmpclk1, tmpclk2;
13341
13342         if (!crtc_state->base.enable)
13343                 return DRM_PLANE_HELPER_NO_SCALING;
13344
13345         crtc_clock = crtc_state->base.adjusted_mode.crtc_clock;
13346         max_dotclk = to_intel_atomic_state(crtc_state->base.state)->cdclk.logical.cdclk;
13347
13348         if (IS_GEMINILAKE(dev_priv) || INTEL_GEN(dev_priv) >= 10)
13349                 max_dotclk *= 2;
13350
13351         if (WARN_ON_ONCE(!crtc_clock || max_dotclk < crtc_clock))
13352                 return DRM_PLANE_HELPER_NO_SCALING;
13353
13354         /*
13355          * skl max scale is lower of:
13356          *    close to 3 but not 3, -1 is for that purpose
13357          *            or
13358          *    cdclk/crtc_clock
13359          */
13360         mult = pixel_format == DRM_FORMAT_NV12 ? 2 : 3;
13361         tmpclk1 = (1 << 16) * mult - 1;
13362         tmpclk2 = (1 << 8) * ((max_dotclk << 8) / crtc_clock);
13363         max_scale = min(tmpclk1, tmpclk2);
13364
13365         return max_scale;
13366 }
13367
13368 static void intel_begin_crtc_commit(struct drm_crtc *crtc,
13369                                     struct drm_crtc_state *old_crtc_state)
13370 {
13371         struct drm_device *dev = crtc->dev;
13372         struct drm_i915_private *dev_priv = to_i915(dev);
13373         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
13374         struct intel_crtc_state *old_intel_cstate =
13375                 to_intel_crtc_state(old_crtc_state);
13376         struct intel_atomic_state *old_intel_state =
13377                 to_intel_atomic_state(old_crtc_state->state);
13378         struct intel_crtc_state *intel_cstate =
13379                 intel_atomic_get_new_crtc_state(old_intel_state, intel_crtc);
13380         bool modeset = needs_modeset(&intel_cstate->base);
13381
13382         if (!modeset &&
13383             (intel_cstate->base.color_mgmt_changed ||
13384              intel_cstate->update_pipe)) {
13385                 intel_color_set_csc(&intel_cstate->base);
13386                 intel_color_load_luts(&intel_cstate->base);
13387         }
13388
13389         /* Perform vblank evasion around commit operation */
13390         intel_pipe_update_start(intel_cstate);
13391
13392         if (modeset)
13393                 goto out;
13394
13395         if (intel_cstate->update_pipe)
13396                 intel_update_pipe_config(old_intel_cstate, intel_cstate);
13397         else if (INTEL_GEN(dev_priv) >= 9)
13398                 skl_detach_scalers(intel_crtc);
13399
13400 out:
13401         if (dev_priv->display.atomic_update_watermarks)
13402                 dev_priv->display.atomic_update_watermarks(old_intel_state,
13403                                                            intel_cstate);
13404 }
13405
13406 void intel_crtc_arm_fifo_underrun(struct intel_crtc *crtc,
13407                                   struct intel_crtc_state *crtc_state)
13408 {
13409         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
13410
13411         if (!IS_GEN2(dev_priv))
13412                 intel_set_cpu_fifo_underrun_reporting(dev_priv, crtc->pipe, true);
13413
13414         if (crtc_state->has_pch_encoder) {
13415                 enum pipe pch_transcoder =
13416                         intel_crtc_pch_transcoder(crtc);
13417
13418                 intel_set_pch_fifo_underrun_reporting(dev_priv, pch_transcoder, true);
13419         }
13420 }
13421
13422 static void intel_finish_crtc_commit(struct drm_crtc *crtc,
13423                                      struct drm_crtc_state *old_crtc_state)
13424 {
13425         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
13426         struct intel_atomic_state *old_intel_state =
13427                 to_intel_atomic_state(old_crtc_state->state);
13428         struct intel_crtc_state *new_crtc_state =
13429                 intel_atomic_get_new_crtc_state(old_intel_state, intel_crtc);
13430
13431         intel_pipe_update_end(new_crtc_state);
13432
13433         if (new_crtc_state->update_pipe &&
13434             !needs_modeset(&new_crtc_state->base) &&
13435             old_crtc_state->mode.private_flags & I915_MODE_FLAG_INHERITED)
13436                 intel_crtc_arm_fifo_underrun(intel_crtc, new_crtc_state);
13437 }
13438
13439 /**
13440  * intel_plane_destroy - destroy a plane
13441  * @plane: plane to destroy
13442  *
13443  * Common destruction function for all types of planes (primary, cursor,
13444  * sprite).
13445  */
13446 void intel_plane_destroy(struct drm_plane *plane)
13447 {
13448         drm_plane_cleanup(plane);
13449         kfree(to_intel_plane(plane));
13450 }
13451
13452 static bool i8xx_plane_format_mod_supported(struct drm_plane *_plane,
13453                                             u32 format, u64 modifier)
13454 {
13455         switch (modifier) {
13456         case DRM_FORMAT_MOD_LINEAR:
13457         case I915_FORMAT_MOD_X_TILED:
13458                 break;
13459         default:
13460                 return false;
13461         }
13462
13463         switch (format) {
13464         case DRM_FORMAT_C8:
13465         case DRM_FORMAT_RGB565:
13466         case DRM_FORMAT_XRGB1555:
13467         case DRM_FORMAT_XRGB8888:
13468                 return modifier == DRM_FORMAT_MOD_LINEAR ||
13469                         modifier == I915_FORMAT_MOD_X_TILED;
13470         default:
13471                 return false;
13472         }
13473 }
13474
13475 static bool i965_plane_format_mod_supported(struct drm_plane *_plane,
13476                                             u32 format, u64 modifier)
13477 {
13478         switch (modifier) {
13479         case DRM_FORMAT_MOD_LINEAR:
13480         case I915_FORMAT_MOD_X_TILED:
13481                 break;
13482         default:
13483                 return false;
13484         }
13485
13486         switch (format) {
13487         case DRM_FORMAT_C8:
13488         case DRM_FORMAT_RGB565:
13489         case DRM_FORMAT_XRGB8888:
13490         case DRM_FORMAT_XBGR8888:
13491         case DRM_FORMAT_XRGB2101010:
13492         case DRM_FORMAT_XBGR2101010:
13493                 return modifier == DRM_FORMAT_MOD_LINEAR ||
13494                         modifier == I915_FORMAT_MOD_X_TILED;
13495         default:
13496                 return false;
13497         }
13498 }
13499
13500 static bool skl_plane_format_mod_supported(struct drm_plane *_plane,
13501                                            u32 format, u64 modifier)
13502 {
13503         struct intel_plane *plane = to_intel_plane(_plane);
13504
13505         switch (modifier) {
13506         case DRM_FORMAT_MOD_LINEAR:
13507         case I915_FORMAT_MOD_X_TILED:
13508         case I915_FORMAT_MOD_Y_TILED:
13509         case I915_FORMAT_MOD_Yf_TILED:
13510                 break;
13511         case I915_FORMAT_MOD_Y_TILED_CCS:
13512         case I915_FORMAT_MOD_Yf_TILED_CCS:
13513                 if (!plane->has_ccs)
13514                         return false;
13515                 break;
13516         default:
13517                 return false;
13518         }
13519
13520         switch (format) {
13521         case DRM_FORMAT_XRGB8888:
13522         case DRM_FORMAT_XBGR8888:
13523         case DRM_FORMAT_ARGB8888:
13524         case DRM_FORMAT_ABGR8888:
13525                 if (is_ccs_modifier(modifier))
13526                         return true;
13527                 /* fall through */
13528         case DRM_FORMAT_RGB565:
13529         case DRM_FORMAT_XRGB2101010:
13530         case DRM_FORMAT_XBGR2101010:
13531         case DRM_FORMAT_YUYV:
13532         case DRM_FORMAT_YVYU:
13533         case DRM_FORMAT_UYVY:
13534         case DRM_FORMAT_VYUY:
13535         case DRM_FORMAT_NV12:
13536                 if (modifier == I915_FORMAT_MOD_Yf_TILED)
13537                         return true;
13538                 /* fall through */
13539         case DRM_FORMAT_C8:
13540                 if (modifier == DRM_FORMAT_MOD_LINEAR ||
13541                     modifier == I915_FORMAT_MOD_X_TILED ||
13542                     modifier == I915_FORMAT_MOD_Y_TILED)
13543                         return true;
13544                 /* fall through */
13545         default:
13546                 return false;
13547         }
13548 }
13549
13550 static bool intel_cursor_format_mod_supported(struct drm_plane *_plane,
13551                                               u32 format, u64 modifier)
13552 {
13553         return modifier == DRM_FORMAT_MOD_LINEAR &&
13554                 format == DRM_FORMAT_ARGB8888;
13555 }
13556
13557 static struct drm_plane_funcs skl_plane_funcs = {
13558         .update_plane = drm_atomic_helper_update_plane,
13559         .disable_plane = drm_atomic_helper_disable_plane,
13560         .destroy = intel_plane_destroy,
13561         .atomic_get_property = intel_plane_atomic_get_property,
13562         .atomic_set_property = intel_plane_atomic_set_property,
13563         .atomic_duplicate_state = intel_plane_duplicate_state,
13564         .atomic_destroy_state = intel_plane_destroy_state,
13565         .format_mod_supported = skl_plane_format_mod_supported,
13566 };
13567
13568 static struct drm_plane_funcs i965_plane_funcs = {
13569         .update_plane = drm_atomic_helper_update_plane,
13570         .disable_plane = drm_atomic_helper_disable_plane,
13571         .destroy = intel_plane_destroy,
13572         .atomic_get_property = intel_plane_atomic_get_property,
13573         .atomic_set_property = intel_plane_atomic_set_property,
13574         .atomic_duplicate_state = intel_plane_duplicate_state,
13575         .atomic_destroy_state = intel_plane_destroy_state,
13576         .format_mod_supported = i965_plane_format_mod_supported,
13577 };
13578
13579 static struct drm_plane_funcs i8xx_plane_funcs = {
13580         .update_plane = drm_atomic_helper_update_plane,
13581         .disable_plane = drm_atomic_helper_disable_plane,
13582         .destroy = intel_plane_destroy,
13583         .atomic_get_property = intel_plane_atomic_get_property,
13584         .atomic_set_property = intel_plane_atomic_set_property,
13585         .atomic_duplicate_state = intel_plane_duplicate_state,
13586         .atomic_destroy_state = intel_plane_destroy_state,
13587         .format_mod_supported = i8xx_plane_format_mod_supported,
13588 };
13589
13590 static int
13591 intel_legacy_cursor_update(struct drm_plane *plane,
13592                            struct drm_crtc *crtc,
13593                            struct drm_framebuffer *fb,
13594                            int crtc_x, int crtc_y,
13595                            unsigned int crtc_w, unsigned int crtc_h,
13596                            uint32_t src_x, uint32_t src_y,
13597                            uint32_t src_w, uint32_t src_h,
13598                            struct drm_modeset_acquire_ctx *ctx)
13599 {
13600         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
13601         int ret;
13602         struct drm_plane_state *old_plane_state, *new_plane_state;
13603         struct intel_plane *intel_plane = to_intel_plane(plane);
13604         struct drm_framebuffer *old_fb;
13605         struct drm_crtc_state *crtc_state = crtc->state;
13606
13607         /*
13608          * When crtc is inactive or there is a modeset pending,
13609          * wait for it to complete in the slowpath
13610          */
13611         if (!crtc_state->active || needs_modeset(crtc_state) ||
13612             to_intel_crtc_state(crtc_state)->update_pipe)
13613                 goto slow;
13614
13615         old_plane_state = plane->state;
13616         /*
13617          * Don't do an async update if there is an outstanding commit modifying
13618          * the plane.  This prevents our async update's changes from getting
13619          * overridden by a previous synchronous update's state.
13620          */
13621         if (old_plane_state->commit &&
13622             !try_wait_for_completion(&old_plane_state->commit->hw_done))
13623                 goto slow;
13624
13625         /*
13626          * If any parameters change that may affect watermarks,
13627          * take the slowpath. Only changing fb or position should be
13628          * in the fastpath.
13629          */
13630         if (old_plane_state->crtc != crtc ||
13631             old_plane_state->src_w != src_w ||
13632             old_plane_state->src_h != src_h ||
13633             old_plane_state->crtc_w != crtc_w ||
13634             old_plane_state->crtc_h != crtc_h ||
13635             !old_plane_state->fb != !fb)
13636                 goto slow;
13637
13638         new_plane_state = intel_plane_duplicate_state(plane);
13639         if (!new_plane_state)
13640                 return -ENOMEM;
13641
13642         drm_atomic_set_fb_for_plane(new_plane_state, fb);
13643
13644         new_plane_state->src_x = src_x;
13645         new_plane_state->src_y = src_y;
13646         new_plane_state->src_w = src_w;
13647         new_plane_state->src_h = src_h;
13648         new_plane_state->crtc_x = crtc_x;
13649         new_plane_state->crtc_y = crtc_y;
13650         new_plane_state->crtc_w = crtc_w;
13651         new_plane_state->crtc_h = crtc_h;
13652
13653         ret = intel_plane_atomic_check_with_state(to_intel_crtc_state(crtc->state),
13654                                                   to_intel_crtc_state(crtc->state), /* FIXME need a new crtc state? */
13655                                                   to_intel_plane_state(plane->state),
13656                                                   to_intel_plane_state(new_plane_state));
13657         if (ret)
13658                 goto out_free;
13659
13660         ret = mutex_lock_interruptible(&dev_priv->drm.struct_mutex);
13661         if (ret)
13662                 goto out_free;
13663
13664         ret = intel_plane_pin_fb(to_intel_plane_state(new_plane_state));
13665         if (ret)
13666                 goto out_unlock;
13667
13668         intel_fb_obj_flush(intel_fb_obj(fb), ORIGIN_FLIP);
13669
13670         old_fb = old_plane_state->fb;
13671         i915_gem_track_fb(intel_fb_obj(old_fb), intel_fb_obj(fb),
13672                           intel_plane->frontbuffer_bit);
13673
13674         /* Swap plane state */
13675         plane->state = new_plane_state;
13676
13677         if (plane->state->visible) {
13678                 trace_intel_update_plane(plane, to_intel_crtc(crtc));
13679                 intel_plane->update_plane(intel_plane,
13680                                           to_intel_crtc_state(crtc->state),
13681                                           to_intel_plane_state(plane->state));
13682         } else {
13683                 trace_intel_disable_plane(plane, to_intel_crtc(crtc));
13684                 intel_plane->disable_plane(intel_plane, to_intel_crtc(crtc));
13685         }
13686
13687         intel_plane_unpin_fb(to_intel_plane_state(old_plane_state));
13688
13689 out_unlock:
13690         mutex_unlock(&dev_priv->drm.struct_mutex);
13691 out_free:
13692         if (ret)
13693                 intel_plane_destroy_state(plane, new_plane_state);
13694         else
13695                 intel_plane_destroy_state(plane, old_plane_state);
13696         return ret;
13697
13698 slow:
13699         return drm_atomic_helper_update_plane(plane, crtc, fb,
13700                                               crtc_x, crtc_y, crtc_w, crtc_h,
13701                                               src_x, src_y, src_w, src_h, ctx);
13702 }
13703
13704 static const struct drm_plane_funcs intel_cursor_plane_funcs = {
13705         .update_plane = intel_legacy_cursor_update,
13706         .disable_plane = drm_atomic_helper_disable_plane,
13707         .destroy = intel_plane_destroy,
13708         .atomic_get_property = intel_plane_atomic_get_property,
13709         .atomic_set_property = intel_plane_atomic_set_property,
13710         .atomic_duplicate_state = intel_plane_duplicate_state,
13711         .atomic_destroy_state = intel_plane_destroy_state,
13712         .format_mod_supported = intel_cursor_format_mod_supported,
13713 };
13714
13715 static bool i9xx_plane_has_fbc(struct drm_i915_private *dev_priv,
13716                                enum i9xx_plane_id i9xx_plane)
13717 {
13718         if (!HAS_FBC(dev_priv))
13719                 return false;
13720
13721         if (IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
13722                 return i9xx_plane == PLANE_A; /* tied to pipe A */
13723         else if (IS_IVYBRIDGE(dev_priv))
13724                 return i9xx_plane == PLANE_A || i9xx_plane == PLANE_B ||
13725                         i9xx_plane == PLANE_C;
13726         else if (INTEL_GEN(dev_priv) >= 4)
13727                 return i9xx_plane == PLANE_A || i9xx_plane == PLANE_B;
13728         else
13729                 return i9xx_plane == PLANE_A;
13730 }
13731
13732 static bool skl_plane_has_fbc(struct drm_i915_private *dev_priv,
13733                               enum pipe pipe, enum plane_id plane_id)
13734 {
13735         if (!HAS_FBC(dev_priv))
13736                 return false;
13737
13738         return pipe == PIPE_A && plane_id == PLANE_PRIMARY;
13739 }
13740
13741 bool skl_plane_has_planar(struct drm_i915_private *dev_priv,
13742                           enum pipe pipe, enum plane_id plane_id)
13743 {
13744         /*
13745          * FIXME: ICL requires two hardware planes for scanning out NV12
13746          * framebuffers. Do not advertize support until this is implemented.
13747          */
13748         if (INTEL_GEN(dev_priv) >= 11)
13749                 return false;
13750
13751         if (IS_SKYLAKE(dev_priv) || IS_BROXTON(dev_priv))
13752                 return false;
13753
13754         if (INTEL_GEN(dev_priv) == 9 && !IS_GEMINILAKE(dev_priv) && pipe == PIPE_C)
13755                 return false;
13756
13757         if (plane_id != PLANE_PRIMARY && plane_id != PLANE_SPRITE0)
13758                 return false;
13759
13760         return true;
13761 }
13762
13763 static struct intel_plane *
13764 intel_primary_plane_create(struct drm_i915_private *dev_priv, enum pipe pipe)
13765 {
13766         struct intel_plane *primary = NULL;
13767         struct intel_plane_state *state = NULL;
13768         const struct drm_plane_funcs *plane_funcs;
13769         const uint32_t *intel_primary_formats;
13770         unsigned int supported_rotations;
13771         unsigned int num_formats;
13772         const uint64_t *modifiers;
13773         int ret;
13774
13775         primary = kzalloc(sizeof(*primary), GFP_KERNEL);
13776         if (!primary) {
13777                 ret = -ENOMEM;
13778                 goto fail;
13779         }
13780
13781         state = intel_create_plane_state(&primary->base);
13782         if (!state) {
13783                 ret = -ENOMEM;
13784                 goto fail;
13785         }
13786
13787         primary->base.state = &state->base;
13788
13789         if (INTEL_GEN(dev_priv) >= 9)
13790                 state->scaler_id = -1;
13791         primary->pipe = pipe;
13792         /*
13793          * On gen2/3 only plane A can do FBC, but the panel fitter and LVDS
13794          * port is hooked to pipe B. Hence we want plane A feeding pipe B.
13795          */
13796         if (HAS_FBC(dev_priv) && INTEL_GEN(dev_priv) < 4)
13797                 primary->i9xx_plane = (enum i9xx_plane_id) !pipe;
13798         else
13799                 primary->i9xx_plane = (enum i9xx_plane_id) pipe;
13800         primary->id = PLANE_PRIMARY;
13801         primary->frontbuffer_bit = INTEL_FRONTBUFFER(pipe, primary->id);
13802
13803         if (INTEL_GEN(dev_priv) >= 9)
13804                 primary->has_fbc = skl_plane_has_fbc(dev_priv,
13805                                                      primary->pipe,
13806                                                      primary->id);
13807         else
13808                 primary->has_fbc = i9xx_plane_has_fbc(dev_priv,
13809                                                       primary->i9xx_plane);
13810
13811         if (primary->has_fbc) {
13812                 struct intel_fbc *fbc = &dev_priv->fbc;
13813
13814                 fbc->possible_framebuffer_bits |= primary->frontbuffer_bit;
13815         }
13816
13817         if (INTEL_GEN(dev_priv) >= 9) {
13818                 primary->has_ccs = skl_plane_has_ccs(dev_priv, pipe,
13819                                                      PLANE_PRIMARY);
13820
13821                 if (skl_plane_has_planar(dev_priv, pipe, PLANE_PRIMARY)) {
13822                         intel_primary_formats = skl_pri_planar_formats;
13823                         num_formats = ARRAY_SIZE(skl_pri_planar_formats);
13824                 } else {
13825                         intel_primary_formats = skl_primary_formats;
13826                         num_formats = ARRAY_SIZE(skl_primary_formats);
13827                 }
13828
13829                 if (primary->has_ccs)
13830                         modifiers = skl_format_modifiers_ccs;
13831                 else
13832                         modifiers = skl_format_modifiers_noccs;
13833
13834                 primary->max_stride = skl_plane_max_stride;
13835                 primary->update_plane = skl_update_plane;
13836                 primary->disable_plane = skl_disable_plane;
13837                 primary->get_hw_state = skl_plane_get_hw_state;
13838                 primary->check_plane = skl_plane_check;
13839
13840                 plane_funcs = &skl_plane_funcs;
13841         } else if (INTEL_GEN(dev_priv) >= 4) {
13842                 intel_primary_formats = i965_primary_formats;
13843                 num_formats = ARRAY_SIZE(i965_primary_formats);
13844                 modifiers = i9xx_format_modifiers;
13845
13846                 primary->max_stride = i9xx_plane_max_stride;
13847                 primary->update_plane = i9xx_update_plane;
13848                 primary->disable_plane = i9xx_disable_plane;
13849                 primary->get_hw_state = i9xx_plane_get_hw_state;
13850                 primary->check_plane = i9xx_plane_check;
13851
13852                 plane_funcs = &i965_plane_funcs;
13853         } else {
13854                 intel_primary_formats = i8xx_primary_formats;
13855                 num_formats = ARRAY_SIZE(i8xx_primary_formats);
13856                 modifiers = i9xx_format_modifiers;
13857
13858                 primary->max_stride = i9xx_plane_max_stride;
13859                 primary->update_plane = i9xx_update_plane;
13860                 primary->disable_plane = i9xx_disable_plane;
13861                 primary->get_hw_state = i9xx_plane_get_hw_state;
13862                 primary->check_plane = i9xx_plane_check;
13863
13864                 plane_funcs = &i8xx_plane_funcs;
13865         }
13866
13867         if (INTEL_GEN(dev_priv) >= 9)
13868                 ret = drm_universal_plane_init(&dev_priv->drm, &primary->base,
13869                                                0, plane_funcs,
13870                                                intel_primary_formats, num_formats,
13871                                                modifiers,
13872                                                DRM_PLANE_TYPE_PRIMARY,
13873                                                "plane 1%c", pipe_name(pipe));
13874         else if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv))
13875                 ret = drm_universal_plane_init(&dev_priv->drm, &primary->base,
13876                                                0, plane_funcs,
13877                                                intel_primary_formats, num_formats,
13878                                                modifiers,
13879                                                DRM_PLANE_TYPE_PRIMARY,
13880                                                "primary %c", pipe_name(pipe));
13881         else
13882                 ret = drm_universal_plane_init(&dev_priv->drm, &primary->base,
13883                                                0, plane_funcs,
13884                                                intel_primary_formats, num_formats,
13885                                                modifiers,
13886                                                DRM_PLANE_TYPE_PRIMARY,
13887                                                "plane %c",
13888                                                plane_name(primary->i9xx_plane));
13889         if (ret)
13890                 goto fail;
13891
13892         if (INTEL_GEN(dev_priv) >= 10) {
13893                 supported_rotations =
13894                         DRM_MODE_ROTATE_0 | DRM_MODE_ROTATE_90 |
13895                         DRM_MODE_ROTATE_180 | DRM_MODE_ROTATE_270 |
13896                         DRM_MODE_REFLECT_X;
13897         } else if (INTEL_GEN(dev_priv) >= 9) {
13898                 supported_rotations =
13899                         DRM_MODE_ROTATE_0 | DRM_MODE_ROTATE_90 |
13900                         DRM_MODE_ROTATE_180 | DRM_MODE_ROTATE_270;
13901         } else if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_B) {
13902                 supported_rotations =
13903                         DRM_MODE_ROTATE_0 | DRM_MODE_ROTATE_180 |
13904                         DRM_MODE_REFLECT_X;
13905         } else if (INTEL_GEN(dev_priv) >= 4) {
13906                 supported_rotations =
13907                         DRM_MODE_ROTATE_0 | DRM_MODE_ROTATE_180;
13908         } else {
13909                 supported_rotations = DRM_MODE_ROTATE_0;
13910         }
13911
13912         if (INTEL_GEN(dev_priv) >= 4)
13913                 drm_plane_create_rotation_property(&primary->base,
13914                                                    DRM_MODE_ROTATE_0,
13915                                                    supported_rotations);
13916
13917         if (INTEL_GEN(dev_priv) >= 9)
13918                 drm_plane_create_color_properties(&primary->base,
13919                                                   BIT(DRM_COLOR_YCBCR_BT601) |
13920                                                   BIT(DRM_COLOR_YCBCR_BT709),
13921                                                   BIT(DRM_COLOR_YCBCR_LIMITED_RANGE) |
13922                                                   BIT(DRM_COLOR_YCBCR_FULL_RANGE),
13923                                                   DRM_COLOR_YCBCR_BT709,
13924                                                   DRM_COLOR_YCBCR_LIMITED_RANGE);
13925
13926         drm_plane_helper_add(&primary->base, &intel_plane_helper_funcs);
13927
13928         return primary;
13929
13930 fail:
13931         kfree(state);
13932         kfree(primary);
13933
13934         return ERR_PTR(ret);
13935 }
13936
13937 static struct intel_plane *
13938 intel_cursor_plane_create(struct drm_i915_private *dev_priv,
13939                           enum pipe pipe)
13940 {
13941         struct intel_plane *cursor = NULL;
13942         struct intel_plane_state *state = NULL;
13943         int ret;
13944
13945         cursor = kzalloc(sizeof(*cursor), GFP_KERNEL);
13946         if (!cursor) {
13947                 ret = -ENOMEM;
13948                 goto fail;
13949         }
13950
13951         state = intel_create_plane_state(&cursor->base);
13952         if (!state) {
13953                 ret = -ENOMEM;
13954                 goto fail;
13955         }
13956
13957         cursor->base.state = &state->base;
13958
13959         cursor->pipe = pipe;
13960         cursor->i9xx_plane = (enum i9xx_plane_id) pipe;
13961         cursor->id = PLANE_CURSOR;
13962         cursor->frontbuffer_bit = INTEL_FRONTBUFFER(pipe, cursor->id);
13963
13964         if (IS_I845G(dev_priv) || IS_I865G(dev_priv)) {
13965                 cursor->max_stride = i845_cursor_max_stride;
13966                 cursor->update_plane = i845_update_cursor;
13967                 cursor->disable_plane = i845_disable_cursor;
13968                 cursor->get_hw_state = i845_cursor_get_hw_state;
13969                 cursor->check_plane = i845_check_cursor;
13970         } else {
13971                 cursor->max_stride = i9xx_cursor_max_stride;
13972                 cursor->update_plane = i9xx_update_cursor;
13973                 cursor->disable_plane = i9xx_disable_cursor;
13974                 cursor->get_hw_state = i9xx_cursor_get_hw_state;
13975                 cursor->check_plane = i9xx_check_cursor;
13976         }
13977
13978         cursor->cursor.base = ~0;
13979         cursor->cursor.cntl = ~0;
13980
13981         if (IS_I845G(dev_priv) || IS_I865G(dev_priv) || HAS_CUR_FBC(dev_priv))
13982                 cursor->cursor.size = ~0;
13983
13984         ret = drm_universal_plane_init(&dev_priv->drm, &cursor->base,
13985                                        0, &intel_cursor_plane_funcs,
13986                                        intel_cursor_formats,
13987                                        ARRAY_SIZE(intel_cursor_formats),
13988                                        cursor_format_modifiers,
13989                                        DRM_PLANE_TYPE_CURSOR,
13990                                        "cursor %c", pipe_name(pipe));
13991         if (ret)
13992                 goto fail;
13993
13994         if (INTEL_GEN(dev_priv) >= 4)
13995                 drm_plane_create_rotation_property(&cursor->base,
13996                                                    DRM_MODE_ROTATE_0,
13997                                                    DRM_MODE_ROTATE_0 |
13998                                                    DRM_MODE_ROTATE_180);
13999
14000         if (INTEL_GEN(dev_priv) >= 9)
14001                 state->scaler_id = -1;
14002
14003         drm_plane_helper_add(&cursor->base, &intel_plane_helper_funcs);
14004
14005         return cursor;
14006
14007 fail:
14008         kfree(state);
14009         kfree(cursor);
14010
14011         return ERR_PTR(ret);
14012 }
14013
14014 static void intel_crtc_init_scalers(struct intel_crtc *crtc,
14015                                     struct intel_crtc_state *crtc_state)
14016 {
14017         struct intel_crtc_scaler_state *scaler_state =
14018                 &crtc_state->scaler_state;
14019         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
14020         int i;
14021
14022         crtc->num_scalers = dev_priv->info.num_scalers[crtc->pipe];
14023         if (!crtc->num_scalers)
14024                 return;
14025
14026         for (i = 0; i < crtc->num_scalers; i++) {
14027                 struct intel_scaler *scaler = &scaler_state->scalers[i];
14028
14029                 scaler->in_use = 0;
14030                 scaler->mode = PS_SCALER_MODE_DYN;
14031         }
14032
14033         scaler_state->scaler_id = -1;
14034 }
14035
14036 static int intel_crtc_init(struct drm_i915_private *dev_priv, enum pipe pipe)
14037 {
14038         struct intel_crtc *intel_crtc;
14039         struct intel_crtc_state *crtc_state = NULL;
14040         struct intel_plane *primary = NULL;
14041         struct intel_plane *cursor = NULL;
14042         int sprite, ret;
14043
14044         intel_crtc = kzalloc(sizeof(*intel_crtc), GFP_KERNEL);
14045         if (!intel_crtc)
14046                 return -ENOMEM;
14047
14048         crtc_state = kzalloc(sizeof(*crtc_state), GFP_KERNEL);
14049         if (!crtc_state) {
14050                 ret = -ENOMEM;
14051                 goto fail;
14052         }
14053         intel_crtc->config = crtc_state;
14054         intel_crtc->base.state = &crtc_state->base;
14055         crtc_state->base.crtc = &intel_crtc->base;
14056
14057         primary = intel_primary_plane_create(dev_priv, pipe);
14058         if (IS_ERR(primary)) {
14059                 ret = PTR_ERR(primary);
14060                 goto fail;
14061         }
14062         intel_crtc->plane_ids_mask |= BIT(primary->id);
14063
14064         for_each_sprite(dev_priv, pipe, sprite) {
14065                 struct intel_plane *plane;
14066
14067                 plane = intel_sprite_plane_create(dev_priv, pipe, sprite);
14068                 if (IS_ERR(plane)) {
14069                         ret = PTR_ERR(plane);
14070                         goto fail;
14071                 }
14072                 intel_crtc->plane_ids_mask |= BIT(plane->id);
14073         }
14074
14075         cursor = intel_cursor_plane_create(dev_priv, pipe);
14076         if (IS_ERR(cursor)) {
14077                 ret = PTR_ERR(cursor);
14078                 goto fail;
14079         }
14080         intel_crtc->plane_ids_mask |= BIT(cursor->id);
14081
14082         ret = drm_crtc_init_with_planes(&dev_priv->drm, &intel_crtc->base,
14083                                         &primary->base, &cursor->base,
14084                                         &intel_crtc_funcs,
14085                                         "pipe %c", pipe_name(pipe));
14086         if (ret)
14087                 goto fail;
14088
14089         intel_crtc->pipe = pipe;
14090
14091         /* initialize shared scalers */
14092         intel_crtc_init_scalers(intel_crtc, crtc_state);
14093
14094         BUG_ON(pipe >= ARRAY_SIZE(dev_priv->pipe_to_crtc_mapping) ||
14095                dev_priv->pipe_to_crtc_mapping[pipe] != NULL);
14096         dev_priv->pipe_to_crtc_mapping[pipe] = intel_crtc;
14097
14098         if (INTEL_GEN(dev_priv) < 9) {
14099                 enum i9xx_plane_id i9xx_plane = primary->i9xx_plane;
14100
14101                 BUG_ON(i9xx_plane >= ARRAY_SIZE(dev_priv->plane_to_crtc_mapping) ||
14102                        dev_priv->plane_to_crtc_mapping[i9xx_plane] != NULL);
14103                 dev_priv->plane_to_crtc_mapping[i9xx_plane] = intel_crtc;
14104         }
14105
14106         drm_crtc_helper_add(&intel_crtc->base, &intel_helper_funcs);
14107
14108         intel_color_init(&intel_crtc->base);
14109
14110         WARN_ON(drm_crtc_index(&intel_crtc->base) != intel_crtc->pipe);
14111
14112         return 0;
14113
14114 fail:
14115         /*
14116          * drm_mode_config_cleanup() will free up any
14117          * crtcs/planes already initialized.
14118          */
14119         kfree(crtc_state);
14120         kfree(intel_crtc);
14121
14122         return ret;
14123 }
14124
14125 enum pipe intel_get_pipe_from_connector(struct intel_connector *connector)
14126 {
14127         struct drm_device *dev = connector->base.dev;
14128
14129         WARN_ON(!drm_modeset_is_locked(&dev->mode_config.connection_mutex));
14130
14131         if (!connector->base.state->crtc)
14132                 return INVALID_PIPE;
14133
14134         return to_intel_crtc(connector->base.state->crtc)->pipe;
14135 }
14136
14137 int intel_get_pipe_from_crtc_id_ioctl(struct drm_device *dev, void *data,
14138                                       struct drm_file *file)
14139 {
14140         struct drm_i915_get_pipe_from_crtc_id *pipe_from_crtc_id = data;
14141         struct drm_crtc *drmmode_crtc;
14142         struct intel_crtc *crtc;
14143
14144         drmmode_crtc = drm_crtc_find(dev, file, pipe_from_crtc_id->crtc_id);
14145         if (!drmmode_crtc)
14146                 return -ENOENT;
14147
14148         crtc = to_intel_crtc(drmmode_crtc);
14149         pipe_from_crtc_id->pipe = crtc->pipe;
14150
14151         return 0;
14152 }
14153
14154 static int intel_encoder_clones(struct intel_encoder *encoder)
14155 {
14156         struct drm_device *dev = encoder->base.dev;
14157         struct intel_encoder *source_encoder;
14158         int index_mask = 0;
14159         int entry = 0;
14160
14161         for_each_intel_encoder(dev, source_encoder) {
14162                 if (encoders_cloneable(encoder, source_encoder))
14163                         index_mask |= (1 << entry);
14164
14165                 entry++;
14166         }
14167
14168         return index_mask;
14169 }
14170
14171 static bool has_edp_a(struct drm_i915_private *dev_priv)
14172 {
14173         if (!IS_MOBILE(dev_priv))
14174                 return false;
14175
14176         if ((I915_READ(DP_A) & DP_DETECTED) == 0)
14177                 return false;
14178
14179         if (IS_GEN5(dev_priv) && (I915_READ(FUSE_STRAP) & ILK_eDP_A_DISABLE))
14180                 return false;
14181
14182         return true;
14183 }
14184
14185 static bool intel_crt_present(struct drm_i915_private *dev_priv)
14186 {
14187         if (INTEL_GEN(dev_priv) >= 9)
14188                 return false;
14189
14190         if (IS_HSW_ULT(dev_priv) || IS_BDW_ULT(dev_priv))
14191                 return false;
14192
14193         if (IS_CHERRYVIEW(dev_priv))
14194                 return false;
14195
14196         if (HAS_PCH_LPT_H(dev_priv) &&
14197             I915_READ(SFUSE_STRAP) & SFUSE_STRAP_CRT_DISABLED)
14198                 return false;
14199
14200         /* DDI E can't be used if DDI A requires 4 lanes */
14201         if (HAS_DDI(dev_priv) && I915_READ(DDI_BUF_CTL(PORT_A)) & DDI_A_4_LANES)
14202                 return false;
14203
14204         if (!dev_priv->vbt.int_crt_support)
14205                 return false;
14206
14207         return true;
14208 }
14209
14210 void intel_pps_unlock_regs_wa(struct drm_i915_private *dev_priv)
14211 {
14212         int pps_num;
14213         int pps_idx;
14214
14215         if (HAS_DDI(dev_priv))
14216                 return;
14217         /*
14218          * This w/a is needed at least on CPT/PPT, but to be sure apply it
14219          * everywhere where registers can be write protected.
14220          */
14221         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
14222                 pps_num = 2;
14223         else
14224                 pps_num = 1;
14225
14226         for (pps_idx = 0; pps_idx < pps_num; pps_idx++) {
14227                 u32 val = I915_READ(PP_CONTROL(pps_idx));
14228
14229                 val = (val & ~PANEL_UNLOCK_MASK) | PANEL_UNLOCK_REGS;
14230                 I915_WRITE(PP_CONTROL(pps_idx), val);
14231         }
14232 }
14233
14234 static void intel_pps_init(struct drm_i915_private *dev_priv)
14235 {
14236         if (HAS_PCH_SPLIT(dev_priv) || IS_GEN9_LP(dev_priv))
14237                 dev_priv->pps_mmio_base = PCH_PPS_BASE;
14238         else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
14239                 dev_priv->pps_mmio_base = VLV_PPS_BASE;
14240         else
14241                 dev_priv->pps_mmio_base = PPS_BASE;
14242
14243         intel_pps_unlock_regs_wa(dev_priv);
14244 }
14245
14246 static void intel_setup_outputs(struct drm_i915_private *dev_priv)
14247 {
14248         struct intel_encoder *encoder;
14249         bool dpd_is_edp = false;
14250
14251         intel_pps_init(dev_priv);
14252
14253         if (INTEL_INFO(dev_priv)->num_pipes == 0)
14254                 return;
14255
14256         /*
14257          * intel_edp_init_connector() depends on this completing first, to
14258          * prevent the registeration of both eDP and LVDS and the incorrect
14259          * sharing of the PPS.
14260          */
14261         intel_lvds_init(dev_priv);
14262
14263         if (intel_crt_present(dev_priv))
14264                 intel_crt_init(dev_priv);
14265
14266         if (IS_ICELAKE(dev_priv)) {
14267                 intel_ddi_init(dev_priv, PORT_A);
14268                 intel_ddi_init(dev_priv, PORT_B);
14269                 intel_ddi_init(dev_priv, PORT_C);
14270                 intel_ddi_init(dev_priv, PORT_D);
14271                 intel_ddi_init(dev_priv, PORT_E);
14272                 intel_ddi_init(dev_priv, PORT_F);
14273         } else if (IS_GEN9_LP(dev_priv)) {
14274                 /*
14275                  * FIXME: Broxton doesn't support port detection via the
14276                  * DDI_BUF_CTL_A or SFUSE_STRAP registers, find another way to
14277                  * detect the ports.
14278                  */
14279                 intel_ddi_init(dev_priv, PORT_A);
14280                 intel_ddi_init(dev_priv, PORT_B);
14281                 intel_ddi_init(dev_priv, PORT_C);
14282
14283                 vlv_dsi_init(dev_priv);
14284         } else if (HAS_DDI(dev_priv)) {
14285                 int found;
14286
14287                 /*
14288                  * Haswell uses DDI functions to detect digital outputs.
14289                  * On SKL pre-D0 the strap isn't connected, so we assume
14290                  * it's there.
14291                  */
14292                 found = I915_READ(DDI_BUF_CTL(PORT_A)) & DDI_INIT_DISPLAY_DETECTED;
14293                 /* WaIgnoreDDIAStrap: skl */
14294                 if (found || IS_GEN9_BC(dev_priv))
14295                         intel_ddi_init(dev_priv, PORT_A);
14296
14297                 /* DDI B, C, D, and F detection is indicated by the SFUSE_STRAP
14298                  * register */
14299                 found = I915_READ(SFUSE_STRAP);
14300
14301                 if (found & SFUSE_STRAP_DDIB_DETECTED)
14302                         intel_ddi_init(dev_priv, PORT_B);
14303                 if (found & SFUSE_STRAP_DDIC_DETECTED)
14304                         intel_ddi_init(dev_priv, PORT_C);
14305                 if (found & SFUSE_STRAP_DDID_DETECTED)
14306                         intel_ddi_init(dev_priv, PORT_D);
14307                 if (found & SFUSE_STRAP_DDIF_DETECTED)
14308                         intel_ddi_init(dev_priv, PORT_F);
14309                 /*
14310                  * On SKL we don't have a way to detect DDI-E so we rely on VBT.
14311                  */
14312                 if (IS_GEN9_BC(dev_priv) &&
14313                     (dev_priv->vbt.ddi_port_info[PORT_E].supports_dp ||
14314                      dev_priv->vbt.ddi_port_info[PORT_E].supports_dvi ||
14315                      dev_priv->vbt.ddi_port_info[PORT_E].supports_hdmi))
14316                         intel_ddi_init(dev_priv, PORT_E);
14317
14318         } else if (HAS_PCH_SPLIT(dev_priv)) {
14319                 int found;
14320                 dpd_is_edp = intel_dp_is_port_edp(dev_priv, PORT_D);
14321
14322                 if (has_edp_a(dev_priv))
14323                         intel_dp_init(dev_priv, DP_A, PORT_A);
14324
14325                 if (I915_READ(PCH_HDMIB) & SDVO_DETECTED) {
14326                         /* PCH SDVOB multiplex with HDMIB */
14327                         found = intel_sdvo_init(dev_priv, PCH_SDVOB, PORT_B);
14328                         if (!found)
14329                                 intel_hdmi_init(dev_priv, PCH_HDMIB, PORT_B);
14330                         if (!found && (I915_READ(PCH_DP_B) & DP_DETECTED))
14331                                 intel_dp_init(dev_priv, PCH_DP_B, PORT_B);
14332                 }
14333
14334                 if (I915_READ(PCH_HDMIC) & SDVO_DETECTED)
14335                         intel_hdmi_init(dev_priv, PCH_HDMIC, PORT_C);
14336
14337                 if (!dpd_is_edp && I915_READ(PCH_HDMID) & SDVO_DETECTED)
14338                         intel_hdmi_init(dev_priv, PCH_HDMID, PORT_D);
14339
14340                 if (I915_READ(PCH_DP_C) & DP_DETECTED)
14341                         intel_dp_init(dev_priv, PCH_DP_C, PORT_C);
14342
14343                 if (I915_READ(PCH_DP_D) & DP_DETECTED)
14344                         intel_dp_init(dev_priv, PCH_DP_D, PORT_D);
14345         } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
14346                 bool has_edp, has_port;
14347
14348                 /*
14349                  * The DP_DETECTED bit is the latched state of the DDC
14350                  * SDA pin at boot. However since eDP doesn't require DDC
14351                  * (no way to plug in a DP->HDMI dongle) the DDC pins for
14352                  * eDP ports may have been muxed to an alternate function.
14353                  * Thus we can't rely on the DP_DETECTED bit alone to detect
14354                  * eDP ports. Consult the VBT as well as DP_DETECTED to
14355                  * detect eDP ports.
14356                  *
14357                  * Sadly the straps seem to be missing sometimes even for HDMI
14358                  * ports (eg. on Voyo V3 - CHT x7-Z8700), so check both strap
14359                  * and VBT for the presence of the port. Additionally we can't
14360                  * trust the port type the VBT declares as we've seen at least
14361                  * HDMI ports that the VBT claim are DP or eDP.
14362                  */
14363                 has_edp = intel_dp_is_port_edp(dev_priv, PORT_B);
14364                 has_port = intel_bios_is_port_present(dev_priv, PORT_B);
14365                 if (I915_READ(VLV_DP_B) & DP_DETECTED || has_port)
14366                         has_edp &= intel_dp_init(dev_priv, VLV_DP_B, PORT_B);
14367                 if ((I915_READ(VLV_HDMIB) & SDVO_DETECTED || has_port) && !has_edp)
14368                         intel_hdmi_init(dev_priv, VLV_HDMIB, PORT_B);
14369
14370                 has_edp = intel_dp_is_port_edp(dev_priv, PORT_C);
14371                 has_port = intel_bios_is_port_present(dev_priv, PORT_C);
14372                 if (I915_READ(VLV_DP_C) & DP_DETECTED || has_port)
14373                         has_edp &= intel_dp_init(dev_priv, VLV_DP_C, PORT_C);
14374                 if ((I915_READ(VLV_HDMIC) & SDVO_DETECTED || has_port) && !has_edp)
14375                         intel_hdmi_init(dev_priv, VLV_HDMIC, PORT_C);
14376
14377                 if (IS_CHERRYVIEW(dev_priv)) {
14378                         /*
14379                          * eDP not supported on port D,
14380                          * so no need to worry about it
14381                          */
14382                         has_port = intel_bios_is_port_present(dev_priv, PORT_D);
14383                         if (I915_READ(CHV_DP_D) & DP_DETECTED || has_port)
14384                                 intel_dp_init(dev_priv, CHV_DP_D, PORT_D);
14385                         if (I915_READ(CHV_HDMID) & SDVO_DETECTED || has_port)
14386                                 intel_hdmi_init(dev_priv, CHV_HDMID, PORT_D);
14387                 }
14388
14389                 vlv_dsi_init(dev_priv);
14390         } else if (!IS_GEN2(dev_priv) && !IS_PINEVIEW(dev_priv)) {
14391                 bool found = false;
14392
14393                 if (I915_READ(GEN3_SDVOB) & SDVO_DETECTED) {
14394                         DRM_DEBUG_KMS("probing SDVOB\n");
14395                         found = intel_sdvo_init(dev_priv, GEN3_SDVOB, PORT_B);
14396                         if (!found && IS_G4X(dev_priv)) {
14397                                 DRM_DEBUG_KMS("probing HDMI on SDVOB\n");
14398                                 intel_hdmi_init(dev_priv, GEN4_HDMIB, PORT_B);
14399                         }
14400
14401                         if (!found && IS_G4X(dev_priv))
14402                                 intel_dp_init(dev_priv, DP_B, PORT_B);
14403                 }
14404
14405                 /* Before G4X SDVOC doesn't have its own detect register */
14406
14407                 if (I915_READ(GEN3_SDVOB) & SDVO_DETECTED) {
14408                         DRM_DEBUG_KMS("probing SDVOC\n");
14409                         found = intel_sdvo_init(dev_priv, GEN3_SDVOC, PORT_C);
14410                 }
14411
14412                 if (!found && (I915_READ(GEN3_SDVOC) & SDVO_DETECTED)) {
14413
14414                         if (IS_G4X(dev_priv)) {
14415                                 DRM_DEBUG_KMS("probing HDMI on SDVOC\n");
14416                                 intel_hdmi_init(dev_priv, GEN4_HDMIC, PORT_C);
14417                         }
14418                         if (IS_G4X(dev_priv))
14419                                 intel_dp_init(dev_priv, DP_C, PORT_C);
14420                 }
14421
14422                 if (IS_G4X(dev_priv) && (I915_READ(DP_D) & DP_DETECTED))
14423                         intel_dp_init(dev_priv, DP_D, PORT_D);
14424         } else if (IS_GEN2(dev_priv))
14425                 intel_dvo_init(dev_priv);
14426
14427         if (SUPPORTS_TV(dev_priv))
14428                 intel_tv_init(dev_priv);
14429
14430         intel_psr_init(dev_priv);
14431
14432         for_each_intel_encoder(&dev_priv->drm, encoder) {
14433                 encoder->base.possible_crtcs = encoder->crtc_mask;
14434                 encoder->base.possible_clones =
14435                         intel_encoder_clones(encoder);
14436         }
14437
14438         intel_init_pch_refclk(dev_priv);
14439
14440         drm_helper_move_panel_connectors_to_head(&dev_priv->drm);
14441 }
14442
14443 static void intel_user_framebuffer_destroy(struct drm_framebuffer *fb)
14444 {
14445         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
14446         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
14447
14448         drm_framebuffer_cleanup(fb);
14449
14450         i915_gem_object_lock(obj);
14451         WARN_ON(!obj->framebuffer_references--);
14452         i915_gem_object_unlock(obj);
14453
14454         i915_gem_object_put(obj);
14455
14456         kfree(intel_fb);
14457 }
14458
14459 static int intel_user_framebuffer_create_handle(struct drm_framebuffer *fb,
14460                                                 struct drm_file *file,
14461                                                 unsigned int *handle)
14462 {
14463         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
14464
14465         if (obj->userptr.mm) {
14466                 DRM_DEBUG("attempting to use a userptr for a framebuffer, denied\n");
14467                 return -EINVAL;
14468         }
14469
14470         return drm_gem_handle_create(file, &obj->base, handle);
14471 }
14472
14473 static int intel_user_framebuffer_dirty(struct drm_framebuffer *fb,
14474                                         struct drm_file *file,
14475                                         unsigned flags, unsigned color,
14476                                         struct drm_clip_rect *clips,
14477                                         unsigned num_clips)
14478 {
14479         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
14480
14481         i915_gem_object_flush_if_display(obj);
14482         intel_fb_obj_flush(obj, ORIGIN_DIRTYFB);
14483
14484         return 0;
14485 }
14486
14487 static const struct drm_framebuffer_funcs intel_fb_funcs = {
14488         .destroy = intel_user_framebuffer_destroy,
14489         .create_handle = intel_user_framebuffer_create_handle,
14490         .dirty = intel_user_framebuffer_dirty,
14491 };
14492
14493 static
14494 u32 intel_fb_pitch_limit(struct drm_i915_private *dev_priv,
14495                          uint64_t fb_modifier, uint32_t pixel_format)
14496 {
14497         struct intel_crtc *crtc;
14498         struct intel_plane *plane;
14499
14500         /*
14501          * We assume the primary plane for pipe A has
14502          * the highest stride limits of them all.
14503          */
14504         crtc = intel_get_crtc_for_pipe(dev_priv, PIPE_A);
14505         plane = to_intel_plane(crtc->base.primary);
14506
14507         return plane->max_stride(plane, pixel_format, fb_modifier,
14508                                  DRM_MODE_ROTATE_0);
14509 }
14510
14511 static int intel_framebuffer_init(struct intel_framebuffer *intel_fb,
14512                                   struct drm_i915_gem_object *obj,
14513                                   struct drm_mode_fb_cmd2 *mode_cmd)
14514 {
14515         struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
14516         struct drm_framebuffer *fb = &intel_fb->base;
14517         struct drm_format_name_buf format_name;
14518         u32 pitch_limit;
14519         unsigned int tiling, stride;
14520         int ret = -EINVAL;
14521         int i;
14522
14523         i915_gem_object_lock(obj);
14524         obj->framebuffer_references++;
14525         tiling = i915_gem_object_get_tiling(obj);
14526         stride = i915_gem_object_get_stride(obj);
14527         i915_gem_object_unlock(obj);
14528
14529         if (mode_cmd->flags & DRM_MODE_FB_MODIFIERS) {
14530                 /*
14531                  * If there's a fence, enforce that
14532                  * the fb modifier and tiling mode match.
14533                  */
14534                 if (tiling != I915_TILING_NONE &&
14535                     tiling != intel_fb_modifier_to_tiling(mode_cmd->modifier[0])) {
14536                         DRM_DEBUG_KMS("tiling_mode doesn't match fb modifier\n");
14537                         goto err;
14538                 }
14539         } else {
14540                 if (tiling == I915_TILING_X) {
14541                         mode_cmd->modifier[0] = I915_FORMAT_MOD_X_TILED;
14542                 } else if (tiling == I915_TILING_Y) {
14543                         DRM_DEBUG_KMS("No Y tiling for legacy addfb\n");
14544                         goto err;
14545                 }
14546         }
14547
14548         /* Passed in modifier sanity checking. */
14549         switch (mode_cmd->modifier[0]) {
14550         case I915_FORMAT_MOD_Y_TILED_CCS:
14551         case I915_FORMAT_MOD_Yf_TILED_CCS:
14552                 switch (mode_cmd->pixel_format) {
14553                 case DRM_FORMAT_XBGR8888:
14554                 case DRM_FORMAT_ABGR8888:
14555                 case DRM_FORMAT_XRGB8888:
14556                 case DRM_FORMAT_ARGB8888:
14557                         break;
14558                 default:
14559                         DRM_DEBUG_KMS("RC supported only with RGB8888 formats\n");
14560                         goto err;
14561                 }
14562                 /* fall through */
14563         case I915_FORMAT_MOD_Y_TILED:
14564         case I915_FORMAT_MOD_Yf_TILED:
14565                 if (INTEL_GEN(dev_priv) < 9) {
14566                         DRM_DEBUG_KMS("Unsupported tiling 0x%llx!\n",
14567                                       mode_cmd->modifier[0]);
14568                         goto err;
14569                 }
14570         case DRM_FORMAT_MOD_LINEAR:
14571         case I915_FORMAT_MOD_X_TILED:
14572                 break;
14573         default:
14574                 DRM_DEBUG_KMS("Unsupported fb modifier 0x%llx!\n",
14575                               mode_cmd->modifier[0]);
14576                 goto err;
14577         }
14578
14579         /*
14580          * gen2/3 display engine uses the fence if present,
14581          * so the tiling mode must match the fb modifier exactly.
14582          */
14583         if (INTEL_GEN(dev_priv) < 4 &&
14584             tiling != intel_fb_modifier_to_tiling(mode_cmd->modifier[0])) {
14585                 DRM_DEBUG_KMS("tiling_mode must match fb modifier exactly on gen2/3\n");
14586                 goto err;
14587         }
14588
14589         pitch_limit = intel_fb_pitch_limit(dev_priv, mode_cmd->modifier[0],
14590                                            mode_cmd->pixel_format);
14591         if (mode_cmd->pitches[0] > pitch_limit) {
14592                 DRM_DEBUG_KMS("%s pitch (%u) must be at most %d\n",
14593                               mode_cmd->modifier[0] != DRM_FORMAT_MOD_LINEAR ?
14594                               "tiled" : "linear",
14595                               mode_cmd->pitches[0], pitch_limit);
14596                 goto err;
14597         }
14598
14599         /*
14600          * If there's a fence, enforce that
14601          * the fb pitch and fence stride match.
14602          */
14603         if (tiling != I915_TILING_NONE && mode_cmd->pitches[0] != stride) {
14604                 DRM_DEBUG_KMS("pitch (%d) must match tiling stride (%d)\n",
14605                               mode_cmd->pitches[0], stride);
14606                 goto err;
14607         }
14608
14609         /* Reject formats not supported by any plane early. */
14610         switch (mode_cmd->pixel_format) {
14611         case DRM_FORMAT_C8:
14612         case DRM_FORMAT_RGB565:
14613         case DRM_FORMAT_XRGB8888:
14614         case DRM_FORMAT_ARGB8888:
14615                 break;
14616         case DRM_FORMAT_XRGB1555:
14617                 if (INTEL_GEN(dev_priv) > 3) {
14618                         DRM_DEBUG_KMS("unsupported pixel format: %s\n",
14619                                       drm_get_format_name(mode_cmd->pixel_format, &format_name));
14620                         goto err;
14621                 }
14622                 break;
14623         case DRM_FORMAT_ABGR8888:
14624                 if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv) &&
14625                     INTEL_GEN(dev_priv) < 9) {
14626                         DRM_DEBUG_KMS("unsupported pixel format: %s\n",
14627                                       drm_get_format_name(mode_cmd->pixel_format, &format_name));
14628                         goto err;
14629                 }
14630                 break;
14631         case DRM_FORMAT_XBGR8888:
14632         case DRM_FORMAT_XRGB2101010:
14633         case DRM_FORMAT_XBGR2101010:
14634                 if (INTEL_GEN(dev_priv) < 4) {
14635                         DRM_DEBUG_KMS("unsupported pixel format: %s\n",
14636                                       drm_get_format_name(mode_cmd->pixel_format, &format_name));
14637                         goto err;
14638                 }
14639                 break;
14640         case DRM_FORMAT_ABGR2101010:
14641                 if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv)) {
14642                         DRM_DEBUG_KMS("unsupported pixel format: %s\n",
14643                                       drm_get_format_name(mode_cmd->pixel_format, &format_name));
14644                         goto err;
14645                 }
14646                 break;
14647         case DRM_FORMAT_YUYV:
14648         case DRM_FORMAT_UYVY:
14649         case DRM_FORMAT_YVYU:
14650         case DRM_FORMAT_VYUY:
14651                 if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv)) {
14652                         DRM_DEBUG_KMS("unsupported pixel format: %s\n",
14653                                       drm_get_format_name(mode_cmd->pixel_format, &format_name));
14654                         goto err;
14655                 }
14656                 break;
14657         case DRM_FORMAT_NV12:
14658                 if (INTEL_GEN(dev_priv) < 9 || IS_SKYLAKE(dev_priv) ||
14659                     IS_BROXTON(dev_priv) || INTEL_GEN(dev_priv) >= 11) {
14660                         DRM_DEBUG_KMS("unsupported pixel format: %s\n",
14661                                       drm_get_format_name(mode_cmd->pixel_format,
14662                                                           &format_name));
14663                         goto err;
14664                 }
14665                 break;
14666         default:
14667                 DRM_DEBUG_KMS("unsupported pixel format: %s\n",
14668                               drm_get_format_name(mode_cmd->pixel_format, &format_name));
14669                 goto err;
14670         }
14671
14672         /* FIXME need to adjust LINOFF/TILEOFF accordingly. */
14673         if (mode_cmd->offsets[0] != 0)
14674                 goto err;
14675
14676         drm_helper_mode_fill_fb_struct(&dev_priv->drm, fb, mode_cmd);
14677
14678         if (fb->format->format == DRM_FORMAT_NV12 &&
14679             (fb->width < SKL_MIN_YUV_420_SRC_W ||
14680              fb->height < SKL_MIN_YUV_420_SRC_H ||
14681              (fb->width % 4) != 0 || (fb->height % 4) != 0)) {
14682                 DRM_DEBUG_KMS("src dimensions not correct for NV12\n");
14683                 goto err;
14684         }
14685
14686         for (i = 0; i < fb->format->num_planes; i++) {
14687                 u32 stride_alignment;
14688
14689                 if (mode_cmd->handles[i] != mode_cmd->handles[0]) {
14690                         DRM_DEBUG_KMS("bad plane %d handle\n", i);
14691                         goto err;
14692                 }
14693
14694                 stride_alignment = intel_fb_stride_alignment(fb, i);
14695
14696                 /*
14697                  * Display WA #0531: skl,bxt,kbl,glk
14698                  *
14699                  * Render decompression and plane width > 3840
14700                  * combined with horizontal panning requires the
14701                  * plane stride to be a multiple of 4. We'll just
14702                  * require the entire fb to accommodate that to avoid
14703                  * potential runtime errors at plane configuration time.
14704                  */
14705                 if (IS_GEN9(dev_priv) && i == 0 && fb->width > 3840 &&
14706                     is_ccs_modifier(fb->modifier))
14707                         stride_alignment *= 4;
14708
14709                 if (fb->pitches[i] & (stride_alignment - 1)) {
14710                         DRM_DEBUG_KMS("plane %d pitch (%d) must be at least %u byte aligned\n",
14711                                       i, fb->pitches[i], stride_alignment);
14712                         goto err;
14713                 }
14714
14715                 fb->obj[i] = &obj->base;
14716         }
14717
14718         ret = intel_fill_fb_info(dev_priv, fb);
14719         if (ret)
14720                 goto err;
14721
14722         ret = drm_framebuffer_init(&dev_priv->drm, fb, &intel_fb_funcs);
14723         if (ret) {
14724                 DRM_ERROR("framebuffer init failed %d\n", ret);
14725                 goto err;
14726         }
14727
14728         return 0;
14729
14730 err:
14731         i915_gem_object_lock(obj);
14732         obj->framebuffer_references--;
14733         i915_gem_object_unlock(obj);
14734         return ret;
14735 }
14736
14737 static struct drm_framebuffer *
14738 intel_user_framebuffer_create(struct drm_device *dev,
14739                               struct drm_file *filp,
14740                               const struct drm_mode_fb_cmd2 *user_mode_cmd)
14741 {
14742         struct drm_framebuffer *fb;
14743         struct drm_i915_gem_object *obj;
14744         struct drm_mode_fb_cmd2 mode_cmd = *user_mode_cmd;
14745
14746         obj = i915_gem_object_lookup(filp, mode_cmd.handles[0]);
14747         if (!obj)
14748                 return ERR_PTR(-ENOENT);
14749
14750         fb = intel_framebuffer_create(obj, &mode_cmd);
14751         if (IS_ERR(fb))
14752                 i915_gem_object_put(obj);
14753
14754         return fb;
14755 }
14756
14757 static void intel_atomic_state_free(struct drm_atomic_state *state)
14758 {
14759         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
14760
14761         drm_atomic_state_default_release(state);
14762
14763         i915_sw_fence_fini(&intel_state->commit_ready);
14764
14765         kfree(state);
14766 }
14767
14768 static enum drm_mode_status
14769 intel_mode_valid(struct drm_device *dev,
14770                  const struct drm_display_mode *mode)
14771 {
14772         struct drm_i915_private *dev_priv = to_i915(dev);
14773         int hdisplay_max, htotal_max;
14774         int vdisplay_max, vtotal_max;
14775
14776         /*
14777          * Can't reject DBLSCAN here because Xorg ddxen can add piles
14778          * of DBLSCAN modes to the output's mode list when they detect
14779          * the scaling mode property on the connector. And they don't
14780          * ask the kernel to validate those modes in any way until
14781          * modeset time at which point the client gets a protocol error.
14782          * So in order to not upset those clients we silently ignore the
14783          * DBLSCAN flag on such connectors. For other connectors we will
14784          * reject modes with the DBLSCAN flag in encoder->compute_config().
14785          * And we always reject DBLSCAN modes in connector->mode_valid()
14786          * as we never want such modes on the connector's mode list.
14787          */
14788
14789         if (mode->vscan > 1)
14790                 return MODE_NO_VSCAN;
14791
14792         if (mode->flags & DRM_MODE_FLAG_HSKEW)
14793                 return MODE_H_ILLEGAL;
14794
14795         if (mode->flags & (DRM_MODE_FLAG_CSYNC |
14796                            DRM_MODE_FLAG_NCSYNC |
14797                            DRM_MODE_FLAG_PCSYNC))
14798                 return MODE_HSYNC;
14799
14800         if (mode->flags & (DRM_MODE_FLAG_BCAST |
14801                            DRM_MODE_FLAG_PIXMUX |
14802                            DRM_MODE_FLAG_CLKDIV2))
14803                 return MODE_BAD;
14804
14805         if (INTEL_GEN(dev_priv) >= 9 ||
14806             IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv)) {
14807                 hdisplay_max = 8192; /* FDI max 4096 handled elsewhere */
14808                 vdisplay_max = 4096;
14809                 htotal_max = 8192;
14810                 vtotal_max = 8192;
14811         } else if (INTEL_GEN(dev_priv) >= 3) {
14812                 hdisplay_max = 4096;
14813                 vdisplay_max = 4096;
14814                 htotal_max = 8192;
14815                 vtotal_max = 8192;
14816         } else {
14817                 hdisplay_max = 2048;
14818                 vdisplay_max = 2048;
14819                 htotal_max = 4096;
14820                 vtotal_max = 4096;
14821         }
14822
14823         if (mode->hdisplay > hdisplay_max ||
14824             mode->hsync_start > htotal_max ||
14825             mode->hsync_end > htotal_max ||
14826             mode->htotal > htotal_max)
14827                 return MODE_H_ILLEGAL;
14828
14829         if (mode->vdisplay > vdisplay_max ||
14830             mode->vsync_start > vtotal_max ||
14831             mode->vsync_end > vtotal_max ||
14832             mode->vtotal > vtotal_max)
14833                 return MODE_V_ILLEGAL;
14834
14835         return MODE_OK;
14836 }
14837
14838 static const struct drm_mode_config_funcs intel_mode_funcs = {
14839         .fb_create = intel_user_framebuffer_create,
14840         .get_format_info = intel_get_format_info,
14841         .output_poll_changed = intel_fbdev_output_poll_changed,
14842         .mode_valid = intel_mode_valid,
14843         .atomic_check = intel_atomic_check,
14844         .atomic_commit = intel_atomic_commit,
14845         .atomic_state_alloc = intel_atomic_state_alloc,
14846         .atomic_state_clear = intel_atomic_state_clear,
14847         .atomic_state_free = intel_atomic_state_free,
14848 };
14849
14850 /**
14851  * intel_init_display_hooks - initialize the display modesetting hooks
14852  * @dev_priv: device private
14853  */
14854 void intel_init_display_hooks(struct drm_i915_private *dev_priv)
14855 {
14856         intel_init_cdclk_hooks(dev_priv);
14857
14858         if (INTEL_GEN(dev_priv) >= 9) {
14859                 dev_priv->display.get_pipe_config = haswell_get_pipe_config;
14860                 dev_priv->display.get_initial_plane_config =
14861                         skylake_get_initial_plane_config;
14862                 dev_priv->display.crtc_compute_clock =
14863                         haswell_crtc_compute_clock;
14864                 dev_priv->display.crtc_enable = haswell_crtc_enable;
14865                 dev_priv->display.crtc_disable = haswell_crtc_disable;
14866         } else if (HAS_DDI(dev_priv)) {
14867                 dev_priv->display.get_pipe_config = haswell_get_pipe_config;
14868                 dev_priv->display.get_initial_plane_config =
14869                         i9xx_get_initial_plane_config;
14870                 dev_priv->display.crtc_compute_clock =
14871                         haswell_crtc_compute_clock;
14872                 dev_priv->display.crtc_enable = haswell_crtc_enable;
14873                 dev_priv->display.crtc_disable = haswell_crtc_disable;
14874         } else if (HAS_PCH_SPLIT(dev_priv)) {
14875                 dev_priv->display.get_pipe_config = ironlake_get_pipe_config;
14876                 dev_priv->display.get_initial_plane_config =
14877                         i9xx_get_initial_plane_config;
14878                 dev_priv->display.crtc_compute_clock =
14879                         ironlake_crtc_compute_clock;
14880                 dev_priv->display.crtc_enable = ironlake_crtc_enable;
14881                 dev_priv->display.crtc_disable = ironlake_crtc_disable;
14882         } else if (IS_CHERRYVIEW(dev_priv)) {
14883                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
14884                 dev_priv->display.get_initial_plane_config =
14885                         i9xx_get_initial_plane_config;
14886                 dev_priv->display.crtc_compute_clock = chv_crtc_compute_clock;
14887                 dev_priv->display.crtc_enable = valleyview_crtc_enable;
14888                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
14889         } else if (IS_VALLEYVIEW(dev_priv)) {
14890                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
14891                 dev_priv->display.get_initial_plane_config =
14892                         i9xx_get_initial_plane_config;
14893                 dev_priv->display.crtc_compute_clock = vlv_crtc_compute_clock;
14894                 dev_priv->display.crtc_enable = valleyview_crtc_enable;
14895                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
14896         } else if (IS_G4X(dev_priv)) {
14897                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
14898                 dev_priv->display.get_initial_plane_config =
14899                         i9xx_get_initial_plane_config;
14900                 dev_priv->display.crtc_compute_clock = g4x_crtc_compute_clock;
14901                 dev_priv->display.crtc_enable = i9xx_crtc_enable;
14902                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
14903         } else if (IS_PINEVIEW(dev_priv)) {
14904                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
14905                 dev_priv->display.get_initial_plane_config =
14906                         i9xx_get_initial_plane_config;
14907                 dev_priv->display.crtc_compute_clock = pnv_crtc_compute_clock;
14908                 dev_priv->display.crtc_enable = i9xx_crtc_enable;
14909                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
14910         } else if (!IS_GEN2(dev_priv)) {
14911                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
14912                 dev_priv->display.get_initial_plane_config =
14913                         i9xx_get_initial_plane_config;
14914                 dev_priv->display.crtc_compute_clock = i9xx_crtc_compute_clock;
14915                 dev_priv->display.crtc_enable = i9xx_crtc_enable;
14916                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
14917         } else {
14918                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
14919                 dev_priv->display.get_initial_plane_config =
14920                         i9xx_get_initial_plane_config;
14921                 dev_priv->display.crtc_compute_clock = i8xx_crtc_compute_clock;
14922                 dev_priv->display.crtc_enable = i9xx_crtc_enable;
14923                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
14924         }
14925
14926         if (IS_GEN5(dev_priv)) {
14927                 dev_priv->display.fdi_link_train = ironlake_fdi_link_train;
14928         } else if (IS_GEN6(dev_priv)) {
14929                 dev_priv->display.fdi_link_train = gen6_fdi_link_train;
14930         } else if (IS_IVYBRIDGE(dev_priv)) {
14931                 /* FIXME: detect B0+ stepping and use auto training */
14932                 dev_priv->display.fdi_link_train = ivb_manual_fdi_link_train;
14933         } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
14934                 dev_priv->display.fdi_link_train = hsw_fdi_link_train;
14935         }
14936
14937         if (INTEL_GEN(dev_priv) >= 9)
14938                 dev_priv->display.update_crtcs = skl_update_crtcs;
14939         else
14940                 dev_priv->display.update_crtcs = intel_update_crtcs;
14941 }
14942
14943 /*
14944  * Some machines (Lenovo U160) do not work with SSC on LVDS for some reason
14945  */
14946 static void quirk_ssc_force_disable(struct drm_device *dev)
14947 {
14948         struct drm_i915_private *dev_priv = to_i915(dev);
14949         dev_priv->quirks |= QUIRK_LVDS_SSC_DISABLE;
14950         DRM_INFO("applying lvds SSC disable quirk\n");
14951 }
14952
14953 /*
14954  * A machine (e.g. Acer Aspire 5734Z) may need to invert the panel backlight
14955  * brightness value
14956  */
14957 static void quirk_invert_brightness(struct drm_device *dev)
14958 {
14959         struct drm_i915_private *dev_priv = to_i915(dev);
14960         dev_priv->quirks |= QUIRK_INVERT_BRIGHTNESS;
14961         DRM_INFO("applying inverted panel brightness quirk\n");
14962 }
14963
14964 /* Some VBT's incorrectly indicate no backlight is present */
14965 static void quirk_backlight_present(struct drm_device *dev)
14966 {
14967         struct drm_i915_private *dev_priv = to_i915(dev);
14968         dev_priv->quirks |= QUIRK_BACKLIGHT_PRESENT;
14969         DRM_INFO("applying backlight present quirk\n");
14970 }
14971
14972 /* Toshiba Satellite P50-C-18C requires T12 delay to be min 800ms
14973  * which is 300 ms greater than eDP spec T12 min.
14974  */
14975 static void quirk_increase_t12_delay(struct drm_device *dev)
14976 {
14977         struct drm_i915_private *dev_priv = to_i915(dev);
14978
14979         dev_priv->quirks |= QUIRK_INCREASE_T12_DELAY;
14980         DRM_INFO("Applying T12 delay quirk\n");
14981 }
14982
14983 /*
14984  * GeminiLake NUC HDMI outputs require additional off time
14985  * this allows the onboard retimer to correctly sync to signal
14986  */
14987 static void quirk_increase_ddi_disabled_time(struct drm_device *dev)
14988 {
14989         struct drm_i915_private *dev_priv = to_i915(dev);
14990
14991         dev_priv->quirks |= QUIRK_INCREASE_DDI_DISABLED_TIME;
14992         DRM_INFO("Applying Increase DDI Disabled quirk\n");
14993 }
14994
14995 struct intel_quirk {
14996         int device;
14997         int subsystem_vendor;
14998         int subsystem_device;
14999         void (*hook)(struct drm_device *dev);
15000 };
15001
15002 /* For systems that don't have a meaningful PCI subdevice/subvendor ID */
15003 struct intel_dmi_quirk {
15004         void (*hook)(struct drm_device *dev);
15005         const struct dmi_system_id (*dmi_id_list)[];
15006 };
15007
15008 static int intel_dmi_reverse_brightness(const struct dmi_system_id *id)
15009 {
15010         DRM_INFO("Backlight polarity reversed on %s\n", id->ident);
15011         return 1;
15012 }
15013
15014 static const struct intel_dmi_quirk intel_dmi_quirks[] = {
15015         {
15016                 .dmi_id_list = &(const struct dmi_system_id[]) {
15017                         {
15018                                 .callback = intel_dmi_reverse_brightness,
15019                                 .ident = "NCR Corporation",
15020                                 .matches = {DMI_MATCH(DMI_SYS_VENDOR, "NCR Corporation"),
15021                                             DMI_MATCH(DMI_PRODUCT_NAME, ""),
15022                                 },
15023                         },
15024                         { }  /* terminating entry */
15025                 },
15026                 .hook = quirk_invert_brightness,
15027         },
15028 };
15029
15030 static struct intel_quirk intel_quirks[] = {
15031         /* Lenovo U160 cannot use SSC on LVDS */
15032         { 0x0046, 0x17aa, 0x3920, quirk_ssc_force_disable },
15033
15034         /* Sony Vaio Y cannot use SSC on LVDS */
15035         { 0x0046, 0x104d, 0x9076, quirk_ssc_force_disable },
15036
15037         /* Acer Aspire 5734Z must invert backlight brightness */
15038         { 0x2a42, 0x1025, 0x0459, quirk_invert_brightness },
15039
15040         /* Acer/eMachines G725 */
15041         { 0x2a42, 0x1025, 0x0210, quirk_invert_brightness },
15042
15043         /* Acer/eMachines e725 */
15044         { 0x2a42, 0x1025, 0x0212, quirk_invert_brightness },
15045
15046         /* Acer/Packard Bell NCL20 */
15047         { 0x2a42, 0x1025, 0x034b, quirk_invert_brightness },
15048
15049         /* Acer Aspire 4736Z */
15050         { 0x2a42, 0x1025, 0x0260, quirk_invert_brightness },
15051
15052         /* Acer Aspire 5336 */
15053         { 0x2a42, 0x1025, 0x048a, quirk_invert_brightness },
15054
15055         /* Acer C720 and C720P Chromebooks (Celeron 2955U) have backlights */
15056         { 0x0a06, 0x1025, 0x0a11, quirk_backlight_present },
15057
15058         /* Acer C720 Chromebook (Core i3 4005U) */
15059         { 0x0a16, 0x1025, 0x0a11, quirk_backlight_present },
15060
15061         /* Apple Macbook 2,1 (Core 2 T7400) */
15062         { 0x27a2, 0x8086, 0x7270, quirk_backlight_present },
15063
15064         /* Apple Macbook 4,1 */
15065         { 0x2a02, 0x106b, 0x00a1, quirk_backlight_present },
15066
15067         /* Toshiba CB35 Chromebook (Celeron 2955U) */
15068         { 0x0a06, 0x1179, 0x0a88, quirk_backlight_present },
15069
15070         /* HP Chromebook 14 (Celeron 2955U) */
15071         { 0x0a06, 0x103c, 0x21ed, quirk_backlight_present },
15072
15073         /* Dell Chromebook 11 */
15074         { 0x0a06, 0x1028, 0x0a35, quirk_backlight_present },
15075
15076         /* Dell Chromebook 11 (2015 version) */
15077         { 0x0a16, 0x1028, 0x0a35, quirk_backlight_present },
15078
15079         /* Toshiba Satellite P50-C-18C */
15080         { 0x191B, 0x1179, 0xF840, quirk_increase_t12_delay },
15081
15082         /* GeminiLake NUC */
15083         { 0x3185, 0x8086, 0x2072, quirk_increase_ddi_disabled_time },
15084         { 0x3184, 0x8086, 0x2072, quirk_increase_ddi_disabled_time },
15085         /* ASRock ITX*/
15086         { 0x3185, 0x1849, 0x2212, quirk_increase_ddi_disabled_time },
15087         { 0x3184, 0x1849, 0x2212, quirk_increase_ddi_disabled_time },
15088 };
15089
15090 static void intel_init_quirks(struct drm_device *dev)
15091 {
15092         struct pci_dev *d = dev->pdev;
15093         int i;
15094
15095         for (i = 0; i < ARRAY_SIZE(intel_quirks); i++) {
15096                 struct intel_quirk *q = &intel_quirks[i];
15097
15098                 if (d->device == q->device &&
15099                     (d->subsystem_vendor == q->subsystem_vendor ||
15100                      q->subsystem_vendor == PCI_ANY_ID) &&
15101                     (d->subsystem_device == q->subsystem_device ||
15102                      q->subsystem_device == PCI_ANY_ID))
15103                         q->hook(dev);
15104         }
15105         for (i = 0; i < ARRAY_SIZE(intel_dmi_quirks); i++) {
15106                 if (dmi_check_system(*intel_dmi_quirks[i].dmi_id_list) != 0)
15107                         intel_dmi_quirks[i].hook(dev);
15108         }
15109 }
15110
15111 /* Disable the VGA plane that we never use */
15112 static void i915_disable_vga(struct drm_i915_private *dev_priv)
15113 {
15114         struct pci_dev *pdev = dev_priv->drm.pdev;
15115         u8 sr1;
15116         i915_reg_t vga_reg = i915_vgacntrl_reg(dev_priv);
15117
15118         /* WaEnableVGAAccessThroughIOPort:ctg,elk,ilk,snb,ivb,vlv,hsw */
15119         vga_get_uninterruptible(pdev, VGA_RSRC_LEGACY_IO);
15120         outb(SR01, VGA_SR_INDEX);
15121         sr1 = inb(VGA_SR_DATA);
15122         outb(sr1 | 1<<5, VGA_SR_DATA);
15123         vga_put(pdev, VGA_RSRC_LEGACY_IO);
15124         udelay(300);
15125
15126         I915_WRITE(vga_reg, VGA_DISP_DISABLE);
15127         POSTING_READ(vga_reg);
15128 }
15129
15130 void intel_modeset_init_hw(struct drm_device *dev)
15131 {
15132         struct drm_i915_private *dev_priv = to_i915(dev);
15133
15134         intel_update_cdclk(dev_priv);
15135         intel_dump_cdclk_state(&dev_priv->cdclk.hw, "Current CDCLK");
15136         dev_priv->cdclk.logical = dev_priv->cdclk.actual = dev_priv->cdclk.hw;
15137 }
15138
15139 /*
15140  * Calculate what we think the watermarks should be for the state we've read
15141  * out of the hardware and then immediately program those watermarks so that
15142  * we ensure the hardware settings match our internal state.
15143  *
15144  * We can calculate what we think WM's should be by creating a duplicate of the
15145  * current state (which was constructed during hardware readout) and running it
15146  * through the atomic check code to calculate new watermark values in the
15147  * state object.
15148  */
15149 static void sanitize_watermarks(struct drm_device *dev)
15150 {
15151         struct drm_i915_private *dev_priv = to_i915(dev);
15152         struct drm_atomic_state *state;
15153         struct intel_atomic_state *intel_state;
15154         struct drm_crtc *crtc;
15155         struct drm_crtc_state *cstate;
15156         struct drm_modeset_acquire_ctx ctx;
15157         int ret;
15158         int i;
15159
15160         /* Only supported on platforms that use atomic watermark design */
15161         if (!dev_priv->display.optimize_watermarks)
15162                 return;
15163
15164         /*
15165          * We need to hold connection_mutex before calling duplicate_state so
15166          * that the connector loop is protected.
15167          */
15168         drm_modeset_acquire_init(&ctx, 0);
15169 retry:
15170         ret = drm_modeset_lock_all_ctx(dev, &ctx);
15171         if (ret == -EDEADLK) {
15172                 drm_modeset_backoff(&ctx);
15173                 goto retry;
15174         } else if (WARN_ON(ret)) {
15175                 goto fail;
15176         }
15177
15178         state = drm_atomic_helper_duplicate_state(dev, &ctx);
15179         if (WARN_ON(IS_ERR(state)))
15180                 goto fail;
15181
15182         intel_state = to_intel_atomic_state(state);
15183
15184         /*
15185          * Hardware readout is the only time we don't want to calculate
15186          * intermediate watermarks (since we don't trust the current
15187          * watermarks).
15188          */
15189         if (!HAS_GMCH_DISPLAY(dev_priv))
15190                 intel_state->skip_intermediate_wm = true;
15191
15192         ret = intel_atomic_check(dev, state);
15193         if (ret) {
15194                 /*
15195                  * If we fail here, it means that the hardware appears to be
15196                  * programmed in a way that shouldn't be possible, given our
15197                  * understanding of watermark requirements.  This might mean a
15198                  * mistake in the hardware readout code or a mistake in the
15199                  * watermark calculations for a given platform.  Raise a WARN
15200                  * so that this is noticeable.
15201                  *
15202                  * If this actually happens, we'll have to just leave the
15203                  * BIOS-programmed watermarks untouched and hope for the best.
15204                  */
15205                 WARN(true, "Could not determine valid watermarks for inherited state\n");
15206                 goto put_state;
15207         }
15208
15209         /* Write calculated watermark values back */
15210         for_each_new_crtc_in_state(state, crtc, cstate, i) {
15211                 struct intel_crtc_state *cs = to_intel_crtc_state(cstate);
15212
15213                 cs->wm.need_postvbl_update = true;
15214                 dev_priv->display.optimize_watermarks(intel_state, cs);
15215
15216                 to_intel_crtc_state(crtc->state)->wm = cs->wm;
15217         }
15218
15219 put_state:
15220         drm_atomic_state_put(state);
15221 fail:
15222         drm_modeset_drop_locks(&ctx);
15223         drm_modeset_acquire_fini(&ctx);
15224 }
15225
15226 static void intel_update_fdi_pll_freq(struct drm_i915_private *dev_priv)
15227 {
15228         if (IS_GEN5(dev_priv)) {
15229                 u32 fdi_pll_clk =
15230                         I915_READ(FDI_PLL_BIOS_0) & FDI_PLL_FB_CLOCK_MASK;
15231
15232                 dev_priv->fdi_pll_freq = (fdi_pll_clk + 2) * 10000;
15233         } else if (IS_GEN6(dev_priv) || IS_IVYBRIDGE(dev_priv)) {
15234                 dev_priv->fdi_pll_freq = 270000;
15235         } else {
15236                 return;
15237         }
15238
15239         DRM_DEBUG_DRIVER("FDI PLL freq=%d\n", dev_priv->fdi_pll_freq);
15240 }
15241
15242 static int intel_initial_commit(struct drm_device *dev)
15243 {
15244         struct drm_atomic_state *state = NULL;
15245         struct drm_modeset_acquire_ctx ctx;
15246         struct drm_crtc *crtc;
15247         struct drm_crtc_state *crtc_state;
15248         int ret = 0;
15249
15250         state = drm_atomic_state_alloc(dev);
15251         if (!state)
15252                 return -ENOMEM;
15253
15254         drm_modeset_acquire_init(&ctx, 0);
15255
15256 retry:
15257         state->acquire_ctx = &ctx;
15258
15259         drm_for_each_crtc(crtc, dev) {
15260                 crtc_state = drm_atomic_get_crtc_state(state, crtc);
15261                 if (IS_ERR(crtc_state)) {
15262                         ret = PTR_ERR(crtc_state);
15263                         goto out;
15264                 }
15265
15266                 if (crtc_state->active) {
15267                         ret = drm_atomic_add_affected_planes(state, crtc);
15268                         if (ret)
15269                                 goto out;
15270
15271                         /*
15272                          * FIXME hack to force a LUT update to avoid the
15273                          * plane update forcing the pipe gamma on without
15274                          * having a proper LUT loaded. Remove once we
15275                          * have readout for pipe gamma enable.
15276                          */
15277                         crtc_state->color_mgmt_changed = true;
15278                 }
15279         }
15280
15281         ret = drm_atomic_commit(state);
15282
15283 out:
15284         if (ret == -EDEADLK) {
15285                 drm_atomic_state_clear(state);
15286                 drm_modeset_backoff(&ctx);
15287                 goto retry;
15288         }
15289
15290         drm_atomic_state_put(state);
15291
15292         drm_modeset_drop_locks(&ctx);
15293         drm_modeset_acquire_fini(&ctx);
15294
15295         return ret;
15296 }
15297
15298 int intel_modeset_init(struct drm_device *dev)
15299 {
15300         struct drm_i915_private *dev_priv = to_i915(dev);
15301         struct i915_ggtt *ggtt = &dev_priv->ggtt;
15302         enum pipe pipe;
15303         struct intel_crtc *crtc;
15304         int ret;
15305
15306         dev_priv->modeset_wq = alloc_ordered_workqueue("i915_modeset", 0);
15307
15308         drm_mode_config_init(dev);
15309
15310         dev->mode_config.min_width = 0;
15311         dev->mode_config.min_height = 0;
15312
15313         dev->mode_config.preferred_depth = 24;
15314         dev->mode_config.prefer_shadow = 1;
15315
15316         dev->mode_config.allow_fb_modifiers = true;
15317
15318         dev->mode_config.funcs = &intel_mode_funcs;
15319
15320         init_llist_head(&dev_priv->atomic_helper.free_list);
15321         INIT_WORK(&dev_priv->atomic_helper.free_work,
15322                   intel_atomic_helper_free_state_worker);
15323
15324         intel_init_quirks(dev);
15325
15326         intel_init_pm(dev_priv);
15327
15328         /*
15329          * There may be no VBT; and if the BIOS enabled SSC we can
15330          * just keep using it to avoid unnecessary flicker.  Whereas if the
15331          * BIOS isn't using it, don't assume it will work even if the VBT
15332          * indicates as much.
15333          */
15334         if (HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv)) {
15335                 bool bios_lvds_use_ssc = !!(I915_READ(PCH_DREF_CONTROL) &
15336                                             DREF_SSC1_ENABLE);
15337
15338                 if (dev_priv->vbt.lvds_use_ssc != bios_lvds_use_ssc) {
15339                         DRM_DEBUG_KMS("SSC %sabled by BIOS, overriding VBT which says %sabled\n",
15340                                      bios_lvds_use_ssc ? "en" : "dis",
15341                                      dev_priv->vbt.lvds_use_ssc ? "en" : "dis");
15342                         dev_priv->vbt.lvds_use_ssc = bios_lvds_use_ssc;
15343                 }
15344         }
15345
15346         /* maximum framebuffer dimensions */
15347         if (IS_GEN2(dev_priv)) {
15348                 dev->mode_config.max_width = 2048;
15349                 dev->mode_config.max_height = 2048;
15350         } else if (IS_GEN3(dev_priv)) {
15351                 dev->mode_config.max_width = 4096;
15352                 dev->mode_config.max_height = 4096;
15353         } else {
15354                 dev->mode_config.max_width = 8192;
15355                 dev->mode_config.max_height = 8192;
15356         }
15357
15358         if (IS_I845G(dev_priv) || IS_I865G(dev_priv)) {
15359                 dev->mode_config.cursor_width = IS_I845G(dev_priv) ? 64 : 512;
15360                 dev->mode_config.cursor_height = 1023;
15361         } else if (IS_GEN2(dev_priv)) {
15362                 dev->mode_config.cursor_width = 64;
15363                 dev->mode_config.cursor_height = 64;
15364         } else {
15365                 dev->mode_config.cursor_width = 256;
15366                 dev->mode_config.cursor_height = 256;
15367         }
15368
15369         dev->mode_config.fb_base = ggtt->gmadr.start;
15370
15371         DRM_DEBUG_KMS("%d display pipe%s available.\n",
15372                       INTEL_INFO(dev_priv)->num_pipes,
15373                       INTEL_INFO(dev_priv)->num_pipes > 1 ? "s" : "");
15374
15375         for_each_pipe(dev_priv, pipe) {
15376                 ret = intel_crtc_init(dev_priv, pipe);
15377                 if (ret) {
15378                         drm_mode_config_cleanup(dev);
15379                         return ret;
15380                 }
15381         }
15382
15383         intel_shared_dpll_init(dev);
15384         intel_update_fdi_pll_freq(dev_priv);
15385
15386         intel_update_czclk(dev_priv);
15387         intel_modeset_init_hw(dev);
15388
15389         if (dev_priv->max_cdclk_freq == 0)
15390                 intel_update_max_cdclk(dev_priv);
15391
15392         /* Just disable it once at startup */
15393         i915_disable_vga(dev_priv);
15394         intel_setup_outputs(dev_priv);
15395
15396         drm_modeset_lock_all(dev);
15397         intel_modeset_setup_hw_state(dev, dev->mode_config.acquire_ctx);
15398         drm_modeset_unlock_all(dev);
15399
15400         for_each_intel_crtc(dev, crtc) {
15401                 struct intel_initial_plane_config plane_config = {};
15402
15403                 if (!crtc->active)
15404                         continue;
15405
15406                 /*
15407                  * Note that reserving the BIOS fb up front prevents us
15408                  * from stuffing other stolen allocations like the ring
15409                  * on top.  This prevents some ugliness at boot time, and
15410                  * can even allow for smooth boot transitions if the BIOS
15411                  * fb is large enough for the active pipe configuration.
15412                  */
15413                 dev_priv->display.get_initial_plane_config(crtc,
15414                                                            &plane_config);
15415
15416                 /*
15417                  * If the fb is shared between multiple heads, we'll
15418                  * just get the first one.
15419                  */
15420                 intel_find_initial_plane_obj(crtc, &plane_config);
15421         }
15422
15423         /*
15424          * Make sure hardware watermarks really match the state we read out.
15425          * Note that we need to do this after reconstructing the BIOS fb's
15426          * since the watermark calculation done here will use pstate->fb.
15427          */
15428         if (!HAS_GMCH_DISPLAY(dev_priv))
15429                 sanitize_watermarks(dev);
15430
15431         /*
15432          * Force all active planes to recompute their states. So that on
15433          * mode_setcrtc after probe, all the intel_plane_state variables
15434          * are already calculated and there is no assert_plane warnings
15435          * during bootup.
15436          */
15437         ret = intel_initial_commit(dev);
15438         if (ret)
15439                 DRM_DEBUG_KMS("Initial commit in probe failed.\n");
15440
15441         return 0;
15442 }
15443
15444 void i830_enable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe)
15445 {
15446         struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
15447         /* 640x480@60Hz, ~25175 kHz */
15448         struct dpll clock = {
15449                 .m1 = 18,
15450                 .m2 = 7,
15451                 .p1 = 13,
15452                 .p2 = 4,
15453                 .n = 2,
15454         };
15455         u32 dpll, fp;
15456         int i;
15457
15458         WARN_ON(i9xx_calc_dpll_params(48000, &clock) != 25154);
15459
15460         DRM_DEBUG_KMS("enabling pipe %c due to force quirk (vco=%d dot=%d)\n",
15461                       pipe_name(pipe), clock.vco, clock.dot);
15462
15463         fp = i9xx_dpll_compute_fp(&clock);
15464         dpll = (I915_READ(DPLL(pipe)) & DPLL_DVO_2X_MODE) |
15465                 DPLL_VGA_MODE_DIS |
15466                 ((clock.p1 - 2) << DPLL_FPA01_P1_POST_DIV_SHIFT) |
15467                 PLL_P2_DIVIDE_BY_4 |
15468                 PLL_REF_INPUT_DREFCLK |
15469                 DPLL_VCO_ENABLE;
15470
15471         I915_WRITE(FP0(pipe), fp);
15472         I915_WRITE(FP1(pipe), fp);
15473
15474         I915_WRITE(HTOTAL(pipe), (640 - 1) | ((800 - 1) << 16));
15475         I915_WRITE(HBLANK(pipe), (640 - 1) | ((800 - 1) << 16));
15476         I915_WRITE(HSYNC(pipe), (656 - 1) | ((752 - 1) << 16));
15477         I915_WRITE(VTOTAL(pipe), (480 - 1) | ((525 - 1) << 16));
15478         I915_WRITE(VBLANK(pipe), (480 - 1) | ((525 - 1) << 16));
15479         I915_WRITE(VSYNC(pipe), (490 - 1) | ((492 - 1) << 16));
15480         I915_WRITE(PIPESRC(pipe), ((640 - 1) << 16) | (480 - 1));
15481
15482         /*
15483          * Apparently we need to have VGA mode enabled prior to changing
15484          * the P1/P2 dividers. Otherwise the DPLL will keep using the old
15485          * dividers, even though the register value does change.
15486          */
15487         I915_WRITE(DPLL(pipe), dpll & ~DPLL_VGA_MODE_DIS);
15488         I915_WRITE(DPLL(pipe), dpll);
15489
15490         /* Wait for the clocks to stabilize. */
15491         POSTING_READ(DPLL(pipe));
15492         udelay(150);
15493
15494         /* The pixel multiplier can only be updated once the
15495          * DPLL is enabled and the clocks are stable.
15496          *
15497          * So write it again.
15498          */
15499         I915_WRITE(DPLL(pipe), dpll);
15500
15501         /* We do this three times for luck */
15502         for (i = 0; i < 3 ; i++) {
15503                 I915_WRITE(DPLL(pipe), dpll);
15504                 POSTING_READ(DPLL(pipe));
15505                 udelay(150); /* wait for warmup */
15506         }
15507
15508         I915_WRITE(PIPECONF(pipe), PIPECONF_ENABLE | PIPECONF_PROGRESSIVE);
15509         POSTING_READ(PIPECONF(pipe));
15510
15511         intel_wait_for_pipe_scanline_moving(crtc);
15512 }
15513
15514 void i830_disable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe)
15515 {
15516         struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
15517
15518         DRM_DEBUG_KMS("disabling pipe %c due to force quirk\n",
15519                       pipe_name(pipe));
15520
15521         WARN_ON(I915_READ(DSPCNTR(PLANE_A)) & DISPLAY_PLANE_ENABLE);
15522         WARN_ON(I915_READ(DSPCNTR(PLANE_B)) & DISPLAY_PLANE_ENABLE);
15523         WARN_ON(I915_READ(DSPCNTR(PLANE_C)) & DISPLAY_PLANE_ENABLE);
15524         WARN_ON(I915_READ(CURCNTR(PIPE_A)) & MCURSOR_MODE);
15525         WARN_ON(I915_READ(CURCNTR(PIPE_B)) & MCURSOR_MODE);
15526
15527         I915_WRITE(PIPECONF(pipe), 0);
15528         POSTING_READ(PIPECONF(pipe));
15529
15530         intel_wait_for_pipe_scanline_stopped(crtc);
15531
15532         I915_WRITE(DPLL(pipe), DPLL_VGA_MODE_DIS);
15533         POSTING_READ(DPLL(pipe));
15534 }
15535
15536 static void
15537 intel_sanitize_plane_mapping(struct drm_i915_private *dev_priv)
15538 {
15539         struct intel_crtc *crtc;
15540
15541         if (INTEL_GEN(dev_priv) >= 4)
15542                 return;
15543
15544         for_each_intel_crtc(&dev_priv->drm, crtc) {
15545                 struct intel_plane *plane =
15546                         to_intel_plane(crtc->base.primary);
15547                 struct intel_crtc *plane_crtc;
15548                 enum pipe pipe;
15549
15550                 if (!plane->get_hw_state(plane, &pipe))
15551                         continue;
15552
15553                 if (pipe == crtc->pipe)
15554                         continue;
15555
15556                 DRM_DEBUG_KMS("%s attached to the wrong pipe, disabling plane\n",
15557                               plane->base.name);
15558
15559                 plane_crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
15560                 intel_plane_disable_noatomic(plane_crtc, plane);
15561         }
15562 }
15563
15564 static bool intel_crtc_has_encoders(struct intel_crtc *crtc)
15565 {
15566         struct drm_device *dev = crtc->base.dev;
15567         struct intel_encoder *encoder;
15568
15569         for_each_encoder_on_crtc(dev, &crtc->base, encoder)
15570                 return true;
15571
15572         return false;
15573 }
15574
15575 static struct intel_connector *intel_encoder_find_connector(struct intel_encoder *encoder)
15576 {
15577         struct drm_device *dev = encoder->base.dev;
15578         struct intel_connector *connector;
15579
15580         for_each_connector_on_encoder(dev, &encoder->base, connector)
15581                 return connector;
15582
15583         return NULL;
15584 }
15585
15586 static bool has_pch_trancoder(struct drm_i915_private *dev_priv,
15587                               enum pipe pch_transcoder)
15588 {
15589         return HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv) ||
15590                 (HAS_PCH_LPT_H(dev_priv) && pch_transcoder == PIPE_A);
15591 }
15592
15593 static void intel_sanitize_crtc(struct intel_crtc *crtc,
15594                                 struct drm_modeset_acquire_ctx *ctx)
15595 {
15596         struct drm_device *dev = crtc->base.dev;
15597         struct drm_i915_private *dev_priv = to_i915(dev);
15598         enum transcoder cpu_transcoder = crtc->config->cpu_transcoder;
15599
15600         /* Clear any frame start delays used for debugging left by the BIOS */
15601         if (crtc->active && !transcoder_is_dsi(cpu_transcoder)) {
15602                 i915_reg_t reg = PIPECONF(cpu_transcoder);
15603
15604                 I915_WRITE(reg,
15605                            I915_READ(reg) & ~PIPECONF_FRAME_START_DELAY_MASK);
15606         }
15607
15608         if (crtc->active) {
15609                 struct intel_plane *plane;
15610
15611                 /* Disable everything but the primary plane */
15612                 for_each_intel_plane_on_crtc(dev, crtc, plane) {
15613                         const struct intel_plane_state *plane_state =
15614                                 to_intel_plane_state(plane->base.state);
15615
15616                         if (plane_state->base.visible &&
15617                             plane->base.type != DRM_PLANE_TYPE_PRIMARY)
15618                                 intel_plane_disable_noatomic(crtc, plane);
15619                 }
15620         }
15621
15622         /* Adjust the state of the output pipe according to whether we
15623          * have active connectors/encoders. */
15624         if (crtc->active && !intel_crtc_has_encoders(crtc))
15625                 intel_crtc_disable_noatomic(&crtc->base, ctx);
15626
15627         if (crtc->active || HAS_GMCH_DISPLAY(dev_priv)) {
15628                 /*
15629                  * We start out with underrun reporting disabled to avoid races.
15630                  * For correct bookkeeping mark this on active crtcs.
15631                  *
15632                  * Also on gmch platforms we dont have any hardware bits to
15633                  * disable the underrun reporting. Which means we need to start
15634                  * out with underrun reporting disabled also on inactive pipes,
15635                  * since otherwise we'll complain about the garbage we read when
15636                  * e.g. coming up after runtime pm.
15637                  *
15638                  * No protection against concurrent access is required - at
15639                  * worst a fifo underrun happens which also sets this to false.
15640                  */
15641                 crtc->cpu_fifo_underrun_disabled = true;
15642                 /*
15643                  * We track the PCH trancoder underrun reporting state
15644                  * within the crtc. With crtc for pipe A housing the underrun
15645                  * reporting state for PCH transcoder A, crtc for pipe B housing
15646                  * it for PCH transcoder B, etc. LPT-H has only PCH transcoder A,
15647                  * and marking underrun reporting as disabled for the non-existing
15648                  * PCH transcoders B and C would prevent enabling the south
15649                  * error interrupt (see cpt_can_enable_serr_int()).
15650                  */
15651                 if (has_pch_trancoder(dev_priv, crtc->pipe))
15652                         crtc->pch_fifo_underrun_disabled = true;
15653         }
15654 }
15655
15656 static void intel_sanitize_encoder(struct intel_encoder *encoder)
15657 {
15658         struct intel_connector *connector;
15659
15660         /* We need to check both for a crtc link (meaning that the
15661          * encoder is active and trying to read from a pipe) and the
15662          * pipe itself being active. */
15663         bool has_active_crtc = encoder->base.crtc &&
15664                 to_intel_crtc(encoder->base.crtc)->active;
15665
15666         connector = intel_encoder_find_connector(encoder);
15667         if (connector && !has_active_crtc) {
15668                 DRM_DEBUG_KMS("[ENCODER:%d:%s] has active connectors but no active pipe!\n",
15669                               encoder->base.base.id,
15670                               encoder->base.name);
15671
15672                 /* Connector is active, but has no active pipe. This is
15673                  * fallout from our resume register restoring. Disable
15674                  * the encoder manually again. */
15675                 if (encoder->base.crtc) {
15676                         struct drm_crtc_state *crtc_state = encoder->base.crtc->state;
15677
15678                         DRM_DEBUG_KMS("[ENCODER:%d:%s] manually disabled\n",
15679                                       encoder->base.base.id,
15680                                       encoder->base.name);
15681                         encoder->disable(encoder, to_intel_crtc_state(crtc_state), connector->base.state);
15682                         if (encoder->post_disable)
15683                                 encoder->post_disable(encoder, to_intel_crtc_state(crtc_state), connector->base.state);
15684                 }
15685                 encoder->base.crtc = NULL;
15686
15687                 /* Inconsistent output/port/pipe state happens presumably due to
15688                  * a bug in one of the get_hw_state functions. Or someplace else
15689                  * in our code, like the register restore mess on resume. Clamp
15690                  * things to off as a safer default. */
15691
15692                 connector->base.dpms = DRM_MODE_DPMS_OFF;
15693                 connector->base.encoder = NULL;
15694         }
15695
15696         /* notify opregion of the sanitized encoder state */
15697         intel_opregion_notify_encoder(encoder, connector && has_active_crtc);
15698 }
15699
15700 void i915_redisable_vga_power_on(struct drm_i915_private *dev_priv)
15701 {
15702         i915_reg_t vga_reg = i915_vgacntrl_reg(dev_priv);
15703
15704         if (!(I915_READ(vga_reg) & VGA_DISP_DISABLE)) {
15705                 DRM_DEBUG_KMS("Something enabled VGA plane, disabling it\n");
15706                 i915_disable_vga(dev_priv);
15707         }
15708 }
15709
15710 void i915_redisable_vga(struct drm_i915_private *dev_priv)
15711 {
15712         /* This function can be called both from intel_modeset_setup_hw_state or
15713          * at a very early point in our resume sequence, where the power well
15714          * structures are not yet restored. Since this function is at a very
15715          * paranoid "someone might have enabled VGA while we were not looking"
15716          * level, just check if the power well is enabled instead of trying to
15717          * follow the "don't touch the power well if we don't need it" policy
15718          * the rest of the driver uses. */
15719         if (!intel_display_power_get_if_enabled(dev_priv, POWER_DOMAIN_VGA))
15720                 return;
15721
15722         i915_redisable_vga_power_on(dev_priv);
15723
15724         intel_display_power_put(dev_priv, POWER_DOMAIN_VGA);
15725 }
15726
15727 /* FIXME read out full plane state for all planes */
15728 static void readout_plane_state(struct drm_i915_private *dev_priv)
15729 {
15730         struct intel_plane *plane;
15731         struct intel_crtc *crtc;
15732
15733         for_each_intel_plane(&dev_priv->drm, plane) {
15734                 struct intel_plane_state *plane_state =
15735                         to_intel_plane_state(plane->base.state);
15736                 struct intel_crtc_state *crtc_state;
15737                 enum pipe pipe = PIPE_A;
15738                 bool visible;
15739
15740                 visible = plane->get_hw_state(plane, &pipe);
15741
15742                 crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
15743                 crtc_state = to_intel_crtc_state(crtc->base.state);
15744
15745                 intel_set_plane_visible(crtc_state, plane_state, visible);
15746         }
15747
15748         for_each_intel_crtc(&dev_priv->drm, crtc) {
15749                 struct intel_crtc_state *crtc_state =
15750                         to_intel_crtc_state(crtc->base.state);
15751
15752                 fixup_active_planes(crtc_state);
15753         }
15754 }
15755
15756 static void intel_modeset_readout_hw_state(struct drm_device *dev)
15757 {
15758         struct drm_i915_private *dev_priv = to_i915(dev);
15759         enum pipe pipe;
15760         struct intel_crtc *crtc;
15761         struct intel_encoder *encoder;
15762         struct intel_connector *connector;
15763         struct drm_connector_list_iter conn_iter;
15764         int i;
15765
15766         dev_priv->active_crtcs = 0;
15767
15768         for_each_intel_crtc(dev, crtc) {
15769                 struct intel_crtc_state *crtc_state =
15770                         to_intel_crtc_state(crtc->base.state);
15771
15772                 __drm_atomic_helper_crtc_destroy_state(&crtc_state->base);
15773                 memset(crtc_state, 0, sizeof(*crtc_state));
15774                 crtc_state->base.crtc = &crtc->base;
15775
15776                 crtc_state->base.active = crtc_state->base.enable =
15777                         dev_priv->display.get_pipe_config(crtc, crtc_state);
15778
15779                 crtc->base.enabled = crtc_state->base.enable;
15780                 crtc->active = crtc_state->base.active;
15781
15782                 if (crtc_state->base.active)
15783                         dev_priv->active_crtcs |= 1 << crtc->pipe;
15784
15785                 DRM_DEBUG_KMS("[CRTC:%d:%s] hw state readout: %s\n",
15786                               crtc->base.base.id, crtc->base.name,
15787                               enableddisabled(crtc_state->base.active));
15788         }
15789
15790         readout_plane_state(dev_priv);
15791
15792         for (i = 0; i < dev_priv->num_shared_dpll; i++) {
15793                 struct intel_shared_dpll *pll = &dev_priv->shared_dplls[i];
15794
15795                 pll->on = pll->info->funcs->get_hw_state(dev_priv, pll,
15796                                                         &pll->state.hw_state);
15797                 pll->state.crtc_mask = 0;
15798                 for_each_intel_crtc(dev, crtc) {
15799                         struct intel_crtc_state *crtc_state =
15800                                 to_intel_crtc_state(crtc->base.state);
15801
15802                         if (crtc_state->base.active &&
15803                             crtc_state->shared_dpll == pll)
15804                                 pll->state.crtc_mask |= 1 << crtc->pipe;
15805                 }
15806                 pll->active_mask = pll->state.crtc_mask;
15807
15808                 DRM_DEBUG_KMS("%s hw state readout: crtc_mask 0x%08x, on %i\n",
15809                               pll->info->name, pll->state.crtc_mask, pll->on);
15810         }
15811
15812         for_each_intel_encoder(dev, encoder) {
15813                 pipe = 0;
15814
15815                 if (encoder->get_hw_state(encoder, &pipe)) {
15816                         struct intel_crtc_state *crtc_state;
15817
15818                         crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
15819                         crtc_state = to_intel_crtc_state(crtc->base.state);
15820
15821                         encoder->base.crtc = &crtc->base;
15822                         encoder->get_config(encoder, crtc_state);
15823                 } else {
15824                         encoder->base.crtc = NULL;
15825                 }
15826
15827                 DRM_DEBUG_KMS("[ENCODER:%d:%s] hw state readout: %s, pipe %c\n",
15828                               encoder->base.base.id, encoder->base.name,
15829                               enableddisabled(encoder->base.crtc),
15830                               pipe_name(pipe));
15831         }
15832
15833         drm_connector_list_iter_begin(dev, &conn_iter);
15834         for_each_intel_connector_iter(connector, &conn_iter) {
15835                 if (connector->get_hw_state(connector)) {
15836                         connector->base.dpms = DRM_MODE_DPMS_ON;
15837
15838                         encoder = connector->encoder;
15839                         connector->base.encoder = &encoder->base;
15840
15841                         if (encoder->base.crtc &&
15842                             encoder->base.crtc->state->active) {
15843                                 /*
15844                                  * This has to be done during hardware readout
15845                                  * because anything calling .crtc_disable may
15846                                  * rely on the connector_mask being accurate.
15847                                  */
15848                                 encoder->base.crtc->state->connector_mask |=
15849                                         drm_connector_mask(&connector->base);
15850                                 encoder->base.crtc->state->encoder_mask |=
15851                                         drm_encoder_mask(&encoder->base);
15852                         }
15853
15854                 } else {
15855                         connector->base.dpms = DRM_MODE_DPMS_OFF;
15856                         connector->base.encoder = NULL;
15857                 }
15858                 DRM_DEBUG_KMS("[CONNECTOR:%d:%s] hw state readout: %s\n",
15859                               connector->base.base.id, connector->base.name,
15860                               enableddisabled(connector->base.encoder));
15861         }
15862         drm_connector_list_iter_end(&conn_iter);
15863
15864         for_each_intel_crtc(dev, crtc) {
15865                 struct intel_crtc_state *crtc_state =
15866                         to_intel_crtc_state(crtc->base.state);
15867                 int min_cdclk = 0;
15868
15869                 memset(&crtc->base.mode, 0, sizeof(crtc->base.mode));
15870                 if (crtc_state->base.active) {
15871                         intel_mode_from_pipe_config(&crtc->base.mode, crtc_state);
15872                         crtc->base.mode.hdisplay = crtc_state->pipe_src_w;
15873                         crtc->base.mode.vdisplay = crtc_state->pipe_src_h;
15874                         intel_mode_from_pipe_config(&crtc_state->base.adjusted_mode, crtc_state);
15875                         WARN_ON(drm_atomic_set_mode_for_crtc(crtc->base.state, &crtc->base.mode));
15876
15877                         /*
15878                          * The initial mode needs to be set in order to keep
15879                          * the atomic core happy. It wants a valid mode if the
15880                          * crtc's enabled, so we do the above call.
15881                          *
15882                          * But we don't set all the derived state fully, hence
15883                          * set a flag to indicate that a full recalculation is
15884                          * needed on the next commit.
15885                          */
15886                         crtc_state->base.mode.private_flags = I915_MODE_FLAG_INHERITED;
15887
15888                         intel_crtc_compute_pixel_rate(crtc_state);
15889
15890                         if (dev_priv->display.modeset_calc_cdclk) {
15891                                 min_cdclk = intel_crtc_compute_min_cdclk(crtc_state);
15892                                 if (WARN_ON(min_cdclk < 0))
15893                                         min_cdclk = 0;
15894                         }
15895
15896                         drm_calc_timestamping_constants(&crtc->base,
15897                                                         &crtc_state->base.adjusted_mode);
15898                         update_scanline_offset(crtc);
15899                 }
15900
15901                 dev_priv->min_cdclk[crtc->pipe] = min_cdclk;
15902                 dev_priv->min_voltage_level[crtc->pipe] =
15903                         crtc_state->min_voltage_level;
15904
15905                 intel_pipe_config_sanity_check(dev_priv, crtc_state);
15906         }
15907 }
15908
15909 static void
15910 get_encoder_power_domains(struct drm_i915_private *dev_priv)
15911 {
15912         struct intel_encoder *encoder;
15913
15914         for_each_intel_encoder(&dev_priv->drm, encoder) {
15915                 u64 get_domains;
15916                 enum intel_display_power_domain domain;
15917                 struct intel_crtc_state *crtc_state;
15918
15919                 if (!encoder->get_power_domains)
15920                         continue;
15921
15922                 /*
15923                  * MST-primary and inactive encoders don't have a crtc state
15924                  * and neither of these require any power domain references.
15925                  */
15926                 if (!encoder->base.crtc)
15927                         continue;
15928
15929                 crtc_state = to_intel_crtc_state(encoder->base.crtc->state);
15930                 get_domains = encoder->get_power_domains(encoder, crtc_state);
15931                 for_each_power_domain(domain, get_domains)
15932                         intel_display_power_get(dev_priv, domain);
15933         }
15934 }
15935
15936 static void intel_early_display_was(struct drm_i915_private *dev_priv)
15937 {
15938         /* Display WA #1185 WaDisableDARBFClkGating:cnl,glk */
15939         if (IS_CANNONLAKE(dev_priv) || IS_GEMINILAKE(dev_priv))
15940                 I915_WRITE(GEN9_CLKGATE_DIS_0, I915_READ(GEN9_CLKGATE_DIS_0) |
15941                            DARBF_GATING_DIS);
15942
15943         if (IS_HASWELL(dev_priv)) {
15944                 /*
15945                  * WaRsPkgCStateDisplayPMReq:hsw
15946                  * System hang if this isn't done before disabling all planes!
15947                  */
15948                 I915_WRITE(CHICKEN_PAR1_1,
15949                            I915_READ(CHICKEN_PAR1_1) | FORCE_ARB_IDLE_PLANES);
15950         }
15951 }
15952
15953 /* Scan out the current hw modeset state,
15954  * and sanitizes it to the current state
15955  */
15956 static void
15957 intel_modeset_setup_hw_state(struct drm_device *dev,
15958                              struct drm_modeset_acquire_ctx *ctx)
15959 {
15960         struct drm_i915_private *dev_priv = to_i915(dev);
15961         struct intel_crtc *crtc;
15962         struct intel_encoder *encoder;
15963         int i;
15964
15965         intel_display_power_get(dev_priv, POWER_DOMAIN_INIT);
15966
15967         intel_early_display_was(dev_priv);
15968         intel_modeset_readout_hw_state(dev);
15969
15970         /* HW state is read out, now we need to sanitize this mess. */
15971         get_encoder_power_domains(dev_priv);
15972
15973         /*
15974          * intel_sanitize_plane_mapping() may need to do vblank
15975          * waits, so we need vblank interrupts restored beforehand.
15976          */
15977         for_each_intel_crtc(&dev_priv->drm, crtc) {
15978                 drm_crtc_vblank_reset(&crtc->base);
15979
15980                 if (crtc->active)
15981                         drm_crtc_vblank_on(&crtc->base);
15982         }
15983
15984         intel_sanitize_plane_mapping(dev_priv);
15985
15986         for_each_intel_encoder(dev, encoder)
15987                 intel_sanitize_encoder(encoder);
15988
15989         for_each_intel_crtc(&dev_priv->drm, crtc) {
15990                 intel_sanitize_crtc(crtc, ctx);
15991                 intel_dump_pipe_config(crtc, crtc->config,
15992                                        "[setup_hw_state]");
15993         }
15994
15995         intel_modeset_update_connector_atomic_state(dev);
15996
15997         for (i = 0; i < dev_priv->num_shared_dpll; i++) {
15998                 struct intel_shared_dpll *pll = &dev_priv->shared_dplls[i];
15999
16000                 if (!pll->on || pll->active_mask)
16001                         continue;
16002
16003                 DRM_DEBUG_KMS("%s enabled but not in use, disabling\n",
16004                               pll->info->name);
16005
16006                 pll->info->funcs->disable(dev_priv, pll);
16007                 pll->on = false;
16008         }
16009
16010         if (IS_G4X(dev_priv)) {
16011                 g4x_wm_get_hw_state(dev);
16012                 g4x_wm_sanitize(dev_priv);
16013         } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
16014                 vlv_wm_get_hw_state(dev);
16015                 vlv_wm_sanitize(dev_priv);
16016         } else if (INTEL_GEN(dev_priv) >= 9) {
16017                 skl_wm_get_hw_state(dev);
16018         } else if (HAS_PCH_SPLIT(dev_priv)) {
16019                 ilk_wm_get_hw_state(dev);
16020         }
16021
16022         for_each_intel_crtc(dev, crtc) {
16023                 u64 put_domains;
16024
16025                 put_domains = modeset_get_crtc_power_domains(&crtc->base, crtc->config);
16026                 if (WARN_ON(put_domains))
16027                         modeset_put_power_domains(dev_priv, put_domains);
16028         }
16029
16030         intel_display_power_put(dev_priv, POWER_DOMAIN_INIT);
16031
16032         intel_fbc_init_pipe_state(dev_priv);
16033 }
16034
16035 void intel_display_resume(struct drm_device *dev)
16036 {
16037         struct drm_i915_private *dev_priv = to_i915(dev);
16038         struct drm_atomic_state *state = dev_priv->modeset_restore_state;
16039         struct drm_modeset_acquire_ctx ctx;
16040         int ret;
16041
16042         dev_priv->modeset_restore_state = NULL;
16043         if (state)
16044                 state->acquire_ctx = &ctx;
16045
16046         drm_modeset_acquire_init(&ctx, 0);
16047
16048         while (1) {
16049                 ret = drm_modeset_lock_all_ctx(dev, &ctx);
16050                 if (ret != -EDEADLK)
16051                         break;
16052
16053                 drm_modeset_backoff(&ctx);
16054         }
16055
16056         if (!ret)
16057                 ret = __intel_display_resume(dev, state, &ctx);
16058
16059         intel_enable_ipc(dev_priv);
16060         drm_modeset_drop_locks(&ctx);
16061         drm_modeset_acquire_fini(&ctx);
16062
16063         if (ret)
16064                 DRM_ERROR("Restoring old state failed with %i\n", ret);
16065         if (state)
16066                 drm_atomic_state_put(state);
16067 }
16068
16069 int intel_connector_register(struct drm_connector *connector)
16070 {
16071         struct intel_connector *intel_connector = to_intel_connector(connector);
16072         int ret;
16073
16074         ret = intel_backlight_device_register(intel_connector);
16075         if (ret)
16076                 goto err;
16077
16078         return 0;
16079
16080 err:
16081         return ret;
16082 }
16083
16084 void intel_connector_unregister(struct drm_connector *connector)
16085 {
16086         struct intel_connector *intel_connector = to_intel_connector(connector);
16087
16088         intel_backlight_device_unregister(intel_connector);
16089         intel_panel_destroy_backlight(connector);
16090 }
16091
16092 static void intel_hpd_poll_fini(struct drm_device *dev)
16093 {
16094         struct intel_connector *connector;
16095         struct drm_connector_list_iter conn_iter;
16096
16097         /* Kill all the work that may have been queued by hpd. */
16098         drm_connector_list_iter_begin(dev, &conn_iter);
16099         for_each_intel_connector_iter(connector, &conn_iter) {
16100                 if (connector->modeset_retry_work.func)
16101                         cancel_work_sync(&connector->modeset_retry_work);
16102                 if (connector->hdcp_shim) {
16103                         cancel_delayed_work_sync(&connector->hdcp_check_work);
16104                         cancel_work_sync(&connector->hdcp_prop_work);
16105                 }
16106         }
16107         drm_connector_list_iter_end(&conn_iter);
16108 }
16109
16110 void intel_modeset_cleanup(struct drm_device *dev)
16111 {
16112         struct drm_i915_private *dev_priv = to_i915(dev);
16113
16114         flush_workqueue(dev_priv->modeset_wq);
16115
16116         flush_work(&dev_priv->atomic_helper.free_work);
16117         WARN_ON(!llist_empty(&dev_priv->atomic_helper.free_list));
16118
16119         /*
16120          * Interrupts and polling as the first thing to avoid creating havoc.
16121          * Too much stuff here (turning of connectors, ...) would
16122          * experience fancy races otherwise.
16123          */
16124         intel_irq_uninstall(dev_priv);
16125
16126         /*
16127          * Due to the hpd irq storm handling the hotplug work can re-arm the
16128          * poll handlers. Hence disable polling after hpd handling is shut down.
16129          */
16130         intel_hpd_poll_fini(dev);
16131
16132         /* poll work can call into fbdev, hence clean that up afterwards */
16133         intel_fbdev_fini(dev_priv);
16134
16135         intel_unregister_dsm_handler();
16136
16137         intel_fbc_global_disable(dev_priv);
16138
16139         /* flush any delayed tasks or pending work */
16140         flush_scheduled_work();
16141
16142         drm_mode_config_cleanup(dev);
16143
16144         intel_cleanup_overlay(dev_priv);
16145
16146         intel_teardown_gmbus(dev_priv);
16147
16148         destroy_workqueue(dev_priv->modeset_wq);
16149 }
16150
16151 void intel_connector_attach_encoder(struct intel_connector *connector,
16152                                     struct intel_encoder *encoder)
16153 {
16154         connector->encoder = encoder;
16155         drm_connector_attach_encoder(&connector->base, &encoder->base);
16156 }
16157
16158 /*
16159  * set vga decode state - true == enable VGA decode
16160  */
16161 int intel_modeset_vga_set_state(struct drm_i915_private *dev_priv, bool state)
16162 {
16163         unsigned reg = INTEL_GEN(dev_priv) >= 6 ? SNB_GMCH_CTRL : INTEL_GMCH_CTRL;
16164         u16 gmch_ctrl;
16165
16166         if (pci_read_config_word(dev_priv->bridge_dev, reg, &gmch_ctrl)) {
16167                 DRM_ERROR("failed to read control word\n");
16168                 return -EIO;
16169         }
16170
16171         if (!!(gmch_ctrl & INTEL_GMCH_VGA_DISABLE) == !state)
16172                 return 0;
16173
16174         if (state)
16175                 gmch_ctrl &= ~INTEL_GMCH_VGA_DISABLE;
16176         else
16177                 gmch_ctrl |= INTEL_GMCH_VGA_DISABLE;
16178
16179         if (pci_write_config_word(dev_priv->bridge_dev, reg, gmch_ctrl)) {
16180                 DRM_ERROR("failed to write control word\n");
16181                 return -EIO;
16182         }
16183
16184         return 0;
16185 }
16186
16187 #if IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR)
16188
16189 struct intel_display_error_state {
16190
16191         u32 power_well_driver;
16192
16193         int num_transcoders;
16194
16195         struct intel_cursor_error_state {
16196                 u32 control;
16197                 u32 position;
16198                 u32 base;
16199                 u32 size;
16200         } cursor[I915_MAX_PIPES];
16201
16202         struct intel_pipe_error_state {
16203                 bool power_domain_on;
16204                 u32 source;
16205                 u32 stat;
16206         } pipe[I915_MAX_PIPES];
16207
16208         struct intel_plane_error_state {
16209                 u32 control;
16210                 u32 stride;
16211                 u32 size;
16212                 u32 pos;
16213                 u32 addr;
16214                 u32 surface;
16215                 u32 tile_offset;
16216         } plane[I915_MAX_PIPES];
16217
16218         struct intel_transcoder_error_state {
16219                 bool power_domain_on;
16220                 enum transcoder cpu_transcoder;
16221
16222                 u32 conf;
16223
16224                 u32 htotal;
16225                 u32 hblank;
16226                 u32 hsync;
16227                 u32 vtotal;
16228                 u32 vblank;
16229                 u32 vsync;
16230         } transcoder[4];
16231 };
16232
16233 struct intel_display_error_state *
16234 intel_display_capture_error_state(struct drm_i915_private *dev_priv)
16235 {
16236         struct intel_display_error_state *error;
16237         int transcoders[] = {
16238                 TRANSCODER_A,
16239                 TRANSCODER_B,
16240                 TRANSCODER_C,
16241                 TRANSCODER_EDP,
16242         };
16243         int i;
16244
16245         if (INTEL_INFO(dev_priv)->num_pipes == 0)
16246                 return NULL;
16247
16248         error = kzalloc(sizeof(*error), GFP_ATOMIC);
16249         if (error == NULL)
16250                 return NULL;
16251
16252         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
16253                 error->power_well_driver = I915_READ(HSW_PWR_WELL_CTL2);
16254
16255         for_each_pipe(dev_priv, i) {
16256                 error->pipe[i].power_domain_on =
16257                         __intel_display_power_is_enabled(dev_priv,
16258                                                          POWER_DOMAIN_PIPE(i));
16259                 if (!error->pipe[i].power_domain_on)
16260                         continue;
16261
16262                 error->cursor[i].control = I915_READ(CURCNTR(i));
16263                 error->cursor[i].position = I915_READ(CURPOS(i));
16264                 error->cursor[i].base = I915_READ(CURBASE(i));
16265
16266                 error->plane[i].control = I915_READ(DSPCNTR(i));
16267                 error->plane[i].stride = I915_READ(DSPSTRIDE(i));
16268                 if (INTEL_GEN(dev_priv) <= 3) {
16269                         error->plane[i].size = I915_READ(DSPSIZE(i));
16270                         error->plane[i].pos = I915_READ(DSPPOS(i));
16271                 }
16272                 if (INTEL_GEN(dev_priv) <= 7 && !IS_HASWELL(dev_priv))
16273                         error->plane[i].addr = I915_READ(DSPADDR(i));
16274                 if (INTEL_GEN(dev_priv) >= 4) {
16275                         error->plane[i].surface = I915_READ(DSPSURF(i));
16276                         error->plane[i].tile_offset = I915_READ(DSPTILEOFF(i));
16277                 }
16278
16279                 error->pipe[i].source = I915_READ(PIPESRC(i));
16280
16281                 if (HAS_GMCH_DISPLAY(dev_priv))
16282                         error->pipe[i].stat = I915_READ(PIPESTAT(i));
16283         }
16284
16285         /* Note: this does not include DSI transcoders. */
16286         error->num_transcoders = INTEL_INFO(dev_priv)->num_pipes;
16287         if (HAS_DDI(dev_priv))
16288                 error->num_transcoders++; /* Account for eDP. */
16289
16290         for (i = 0; i < error->num_transcoders; i++) {
16291                 enum transcoder cpu_transcoder = transcoders[i];
16292
16293                 error->transcoder[i].power_domain_on =
16294                         __intel_display_power_is_enabled(dev_priv,
16295                                 POWER_DOMAIN_TRANSCODER(cpu_transcoder));
16296                 if (!error->transcoder[i].power_domain_on)
16297                         continue;
16298
16299                 error->transcoder[i].cpu_transcoder = cpu_transcoder;
16300
16301                 error->transcoder[i].conf = I915_READ(PIPECONF(cpu_transcoder));
16302                 error->transcoder[i].htotal = I915_READ(HTOTAL(cpu_transcoder));
16303                 error->transcoder[i].hblank = I915_READ(HBLANK(cpu_transcoder));
16304                 error->transcoder[i].hsync = I915_READ(HSYNC(cpu_transcoder));
16305                 error->transcoder[i].vtotal = I915_READ(VTOTAL(cpu_transcoder));
16306                 error->transcoder[i].vblank = I915_READ(VBLANK(cpu_transcoder));
16307                 error->transcoder[i].vsync = I915_READ(VSYNC(cpu_transcoder));
16308         }
16309
16310         return error;
16311 }
16312
16313 #define err_printf(e, ...) i915_error_printf(e, __VA_ARGS__)
16314
16315 void
16316 intel_display_print_error_state(struct drm_i915_error_state_buf *m,
16317                                 struct intel_display_error_state *error)
16318 {
16319         struct drm_i915_private *dev_priv = m->i915;
16320         int i;
16321
16322         if (!error)
16323                 return;
16324
16325         err_printf(m, "Num Pipes: %d\n", INTEL_INFO(dev_priv)->num_pipes);
16326         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
16327                 err_printf(m, "PWR_WELL_CTL2: %08x\n",
16328                            error->power_well_driver);
16329         for_each_pipe(dev_priv, i) {
16330                 err_printf(m, "Pipe [%d]:\n", i);
16331                 err_printf(m, "  Power: %s\n",
16332                            onoff(error->pipe[i].power_domain_on));
16333                 err_printf(m, "  SRC: %08x\n", error->pipe[i].source);
16334                 err_printf(m, "  STAT: %08x\n", error->pipe[i].stat);
16335
16336                 err_printf(m, "Plane [%d]:\n", i);
16337                 err_printf(m, "  CNTR: %08x\n", error->plane[i].control);
16338                 err_printf(m, "  STRIDE: %08x\n", error->plane[i].stride);
16339                 if (INTEL_GEN(dev_priv) <= 3) {
16340                         err_printf(m, "  SIZE: %08x\n", error->plane[i].size);
16341                         err_printf(m, "  POS: %08x\n", error->plane[i].pos);
16342                 }
16343                 if (INTEL_GEN(dev_priv) <= 7 && !IS_HASWELL(dev_priv))
16344                         err_printf(m, "  ADDR: %08x\n", error->plane[i].addr);
16345                 if (INTEL_GEN(dev_priv) >= 4) {
16346                         err_printf(m, "  SURF: %08x\n", error->plane[i].surface);
16347                         err_printf(m, "  TILEOFF: %08x\n", error->plane[i].tile_offset);
16348                 }
16349
16350                 err_printf(m, "Cursor [%d]:\n", i);
16351                 err_printf(m, "  CNTR: %08x\n", error->cursor[i].control);
16352                 err_printf(m, "  POS: %08x\n", error->cursor[i].position);
16353                 err_printf(m, "  BASE: %08x\n", error->cursor[i].base);
16354         }
16355
16356         for (i = 0; i < error->num_transcoders; i++) {
16357                 err_printf(m, "CPU transcoder: %s\n",
16358                            transcoder_name(error->transcoder[i].cpu_transcoder));
16359                 err_printf(m, "  Power: %s\n",
16360                            onoff(error->transcoder[i].power_domain_on));
16361                 err_printf(m, "  CONF: %08x\n", error->transcoder[i].conf);
16362                 err_printf(m, "  HTOTAL: %08x\n", error->transcoder[i].htotal);
16363                 err_printf(m, "  HBLANK: %08x\n", error->transcoder[i].hblank);
16364                 err_printf(m, "  HSYNC: %08x\n", error->transcoder[i].hsync);
16365                 err_printf(m, "  VTOTAL: %08x\n", error->transcoder[i].vtotal);
16366                 err_printf(m, "  VBLANK: %08x\n", error->transcoder[i].vblank);
16367                 err_printf(m, "  VSYNC: %08x\n", error->transcoder[i].vsync);
16368         }
16369 }
16370
16371 #endif