Merge drm/drm-next into drm-intel-next-queued
[sfrench/cifs-2.6.git] / drivers / gpu / drm / i915 / display / intel_display.c
1 /*
2  * Copyright © 2006-2007 Intel Corporation
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice (including the next
12  * paragraph) shall be included in all copies or substantial portions of the
13  * Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21  * DEALINGS IN THE SOFTWARE.
22  *
23  * Authors:
24  *      Eric Anholt <eric@anholt.net>
25  */
26
27 #include <linux/i2c.h>
28 #include <linux/input.h>
29 #include <linux/intel-iommu.h>
30 #include <linux/kernel.h>
31 #include <linux/module.h>
32 #include <linux/reservation.h>
33 #include <linux/slab.h>
34 #include <linux/vgaarb.h>
35
36 #include <drm/drm_atomic.h>
37 #include <drm/drm_atomic_helper.h>
38 #include <drm/drm_atomic_uapi.h>
39 #include <drm/drm_dp_helper.h>
40 #include <drm/drm_edid.h>
41 #include <drm/drm_fourcc.h>
42 #include <drm/drm_plane_helper.h>
43 #include <drm/drm_probe_helper.h>
44 #include <drm/drm_rect.h>
45 #include <drm/i915_drm.h>
46
47 #include "display/intel_crt.h"
48 #include "display/intel_ddi.h"
49 #include "display/intel_dp.h"
50 #include "display/intel_dsi.h"
51 #include "display/intel_dvo.h"
52 #include "display/intel_gmbus.h"
53 #include "display/intel_hdmi.h"
54 #include "display/intel_lvds.h"
55 #include "display/intel_sdvo.h"
56 #include "display/intel_tv.h"
57 #include "display/intel_vdsc.h"
58
59 #include "i915_drv.h"
60 #include "i915_trace.h"
61 #include "intel_acpi.h"
62 #include "intel_atomic.h"
63 #include "intel_atomic_plane.h"
64 #include "intel_bw.h"
65 #include "intel_color.h"
66 #include "intel_cdclk.h"
67 #include "intel_drv.h"
68 #include "intel_fbc.h"
69 #include "intel_fbdev.h"
70 #include "intel_fifo_underrun.h"
71 #include "intel_frontbuffer.h"
72 #include "intel_hdcp.h"
73 #include "intel_hotplug.h"
74 #include "intel_overlay.h"
75 #include "intel_pipe_crc.h"
76 #include "intel_pm.h"
77 #include "intel_psr.h"
78 #include "intel_quirks.h"
79 #include "intel_sideband.h"
80 #include "intel_sprite.h"
81 #include "intel_tc.h"
82
83 /* Primary plane formats for gen <= 3 */
84 static const u32 i8xx_primary_formats[] = {
85         DRM_FORMAT_C8,
86         DRM_FORMAT_RGB565,
87         DRM_FORMAT_XRGB1555,
88         DRM_FORMAT_XRGB8888,
89 };
90
91 /* Primary plane formats for gen >= 4 */
92 static const u32 i965_primary_formats[] = {
93         DRM_FORMAT_C8,
94         DRM_FORMAT_RGB565,
95         DRM_FORMAT_XRGB8888,
96         DRM_FORMAT_XBGR8888,
97         DRM_FORMAT_XRGB2101010,
98         DRM_FORMAT_XBGR2101010,
99 };
100
101 static const u64 i9xx_format_modifiers[] = {
102         I915_FORMAT_MOD_X_TILED,
103         DRM_FORMAT_MOD_LINEAR,
104         DRM_FORMAT_MOD_INVALID
105 };
106
107 /* Cursor formats */
108 static const u32 intel_cursor_formats[] = {
109         DRM_FORMAT_ARGB8888,
110 };
111
112 static const u64 cursor_format_modifiers[] = {
113         DRM_FORMAT_MOD_LINEAR,
114         DRM_FORMAT_MOD_INVALID
115 };
116
117 static void i9xx_crtc_clock_get(struct intel_crtc *crtc,
118                                 struct intel_crtc_state *pipe_config);
119 static void ironlake_pch_clock_get(struct intel_crtc *crtc,
120                                    struct intel_crtc_state *pipe_config);
121
122 static int intel_framebuffer_init(struct intel_framebuffer *ifb,
123                                   struct drm_i915_gem_object *obj,
124                                   struct drm_mode_fb_cmd2 *mode_cmd);
125 static void intel_set_pipe_timings(const struct intel_crtc_state *crtc_state);
126 static void intel_set_pipe_src_size(const struct intel_crtc_state *crtc_state);
127 static void intel_cpu_transcoder_set_m_n(const struct intel_crtc_state *crtc_state,
128                                          const struct intel_link_m_n *m_n,
129                                          const struct intel_link_m_n *m2_n2);
130 static void i9xx_set_pipeconf(const struct intel_crtc_state *crtc_state);
131 static void ironlake_set_pipeconf(const struct intel_crtc_state *crtc_state);
132 static void haswell_set_pipeconf(const struct intel_crtc_state *crtc_state);
133 static void bdw_set_pipemisc(const struct intel_crtc_state *crtc_state);
134 static void vlv_prepare_pll(struct intel_crtc *crtc,
135                             const struct intel_crtc_state *pipe_config);
136 static void chv_prepare_pll(struct intel_crtc *crtc,
137                             const struct intel_crtc_state *pipe_config);
138 static void intel_begin_crtc_commit(struct intel_atomic_state *, struct intel_crtc *);
139 static void intel_finish_crtc_commit(struct intel_atomic_state *, struct intel_crtc *);
140 static void intel_crtc_init_scalers(struct intel_crtc *crtc,
141                                     struct intel_crtc_state *crtc_state);
142 static void skylake_pfit_enable(const struct intel_crtc_state *crtc_state);
143 static void ironlake_pfit_disable(const struct intel_crtc_state *old_crtc_state);
144 static void ironlake_pfit_enable(const struct intel_crtc_state *crtc_state);
145 static void intel_modeset_setup_hw_state(struct drm_device *dev,
146                                          struct drm_modeset_acquire_ctx *ctx);
147 static void intel_pre_disable_primary_noatomic(struct drm_crtc *crtc);
148
149 struct intel_limit {
150         struct {
151                 int min, max;
152         } dot, vco, n, m, m1, m2, p, p1;
153
154         struct {
155                 int dot_limit;
156                 int p2_slow, p2_fast;
157         } p2;
158 };
159
160 /* returns HPLL frequency in kHz */
161 int vlv_get_hpll_vco(struct drm_i915_private *dev_priv)
162 {
163         int hpll_freq, vco_freq[] = { 800, 1600, 2000, 2400 };
164
165         /* Obtain SKU information */
166         hpll_freq = vlv_cck_read(dev_priv, CCK_FUSE_REG) &
167                 CCK_FUSE_HPLL_FREQ_MASK;
168
169         return vco_freq[hpll_freq] * 1000;
170 }
171
172 int vlv_get_cck_clock(struct drm_i915_private *dev_priv,
173                       const char *name, u32 reg, int ref_freq)
174 {
175         u32 val;
176         int divider;
177
178         val = vlv_cck_read(dev_priv, reg);
179         divider = val & CCK_FREQUENCY_VALUES;
180
181         WARN((val & CCK_FREQUENCY_STATUS) !=
182              (divider << CCK_FREQUENCY_STATUS_SHIFT),
183              "%s change in progress\n", name);
184
185         return DIV_ROUND_CLOSEST(ref_freq << 1, divider + 1);
186 }
187
188 int vlv_get_cck_clock_hpll(struct drm_i915_private *dev_priv,
189                            const char *name, u32 reg)
190 {
191         int hpll;
192
193         vlv_cck_get(dev_priv);
194
195         if (dev_priv->hpll_freq == 0)
196                 dev_priv->hpll_freq = vlv_get_hpll_vco(dev_priv);
197
198         hpll = vlv_get_cck_clock(dev_priv, name, reg, dev_priv->hpll_freq);
199
200         vlv_cck_put(dev_priv);
201
202         return hpll;
203 }
204
205 static void intel_update_czclk(struct drm_i915_private *dev_priv)
206 {
207         if (!(IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)))
208                 return;
209
210         dev_priv->czclk_freq = vlv_get_cck_clock_hpll(dev_priv, "czclk",
211                                                       CCK_CZ_CLOCK_CONTROL);
212
213         DRM_DEBUG_DRIVER("CZ clock rate: %d kHz\n", dev_priv->czclk_freq);
214 }
215
216 static inline u32 /* units of 100MHz */
217 intel_fdi_link_freq(struct drm_i915_private *dev_priv,
218                     const struct intel_crtc_state *pipe_config)
219 {
220         if (HAS_DDI(dev_priv))
221                 return pipe_config->port_clock; /* SPLL */
222         else
223                 return dev_priv->fdi_pll_freq;
224 }
225
226 static const struct intel_limit intel_limits_i8xx_dac = {
227         .dot = { .min = 25000, .max = 350000 },
228         .vco = { .min = 908000, .max = 1512000 },
229         .n = { .min = 2, .max = 16 },
230         .m = { .min = 96, .max = 140 },
231         .m1 = { .min = 18, .max = 26 },
232         .m2 = { .min = 6, .max = 16 },
233         .p = { .min = 4, .max = 128 },
234         .p1 = { .min = 2, .max = 33 },
235         .p2 = { .dot_limit = 165000,
236                 .p2_slow = 4, .p2_fast = 2 },
237 };
238
239 static const struct intel_limit intel_limits_i8xx_dvo = {
240         .dot = { .min = 25000, .max = 350000 },
241         .vco = { .min = 908000, .max = 1512000 },
242         .n = { .min = 2, .max = 16 },
243         .m = { .min = 96, .max = 140 },
244         .m1 = { .min = 18, .max = 26 },
245         .m2 = { .min = 6, .max = 16 },
246         .p = { .min = 4, .max = 128 },
247         .p1 = { .min = 2, .max = 33 },
248         .p2 = { .dot_limit = 165000,
249                 .p2_slow = 4, .p2_fast = 4 },
250 };
251
252 static const struct intel_limit intel_limits_i8xx_lvds = {
253         .dot = { .min = 25000, .max = 350000 },
254         .vco = { .min = 908000, .max = 1512000 },
255         .n = { .min = 2, .max = 16 },
256         .m = { .min = 96, .max = 140 },
257         .m1 = { .min = 18, .max = 26 },
258         .m2 = { .min = 6, .max = 16 },
259         .p = { .min = 4, .max = 128 },
260         .p1 = { .min = 1, .max = 6 },
261         .p2 = { .dot_limit = 165000,
262                 .p2_slow = 14, .p2_fast = 7 },
263 };
264
265 static const struct intel_limit intel_limits_i9xx_sdvo = {
266         .dot = { .min = 20000, .max = 400000 },
267         .vco = { .min = 1400000, .max = 2800000 },
268         .n = { .min = 1, .max = 6 },
269         .m = { .min = 70, .max = 120 },
270         .m1 = { .min = 8, .max = 18 },
271         .m2 = { .min = 3, .max = 7 },
272         .p = { .min = 5, .max = 80 },
273         .p1 = { .min = 1, .max = 8 },
274         .p2 = { .dot_limit = 200000,
275                 .p2_slow = 10, .p2_fast = 5 },
276 };
277
278 static const struct intel_limit intel_limits_i9xx_lvds = {
279         .dot = { .min = 20000, .max = 400000 },
280         .vco = { .min = 1400000, .max = 2800000 },
281         .n = { .min = 1, .max = 6 },
282         .m = { .min = 70, .max = 120 },
283         .m1 = { .min = 8, .max = 18 },
284         .m2 = { .min = 3, .max = 7 },
285         .p = { .min = 7, .max = 98 },
286         .p1 = { .min = 1, .max = 8 },
287         .p2 = { .dot_limit = 112000,
288                 .p2_slow = 14, .p2_fast = 7 },
289 };
290
291
292 static const struct intel_limit intel_limits_g4x_sdvo = {
293         .dot = { .min = 25000, .max = 270000 },
294         .vco = { .min = 1750000, .max = 3500000},
295         .n = { .min = 1, .max = 4 },
296         .m = { .min = 104, .max = 138 },
297         .m1 = { .min = 17, .max = 23 },
298         .m2 = { .min = 5, .max = 11 },
299         .p = { .min = 10, .max = 30 },
300         .p1 = { .min = 1, .max = 3},
301         .p2 = { .dot_limit = 270000,
302                 .p2_slow = 10,
303                 .p2_fast = 10
304         },
305 };
306
307 static const struct intel_limit intel_limits_g4x_hdmi = {
308         .dot = { .min = 22000, .max = 400000 },
309         .vco = { .min = 1750000, .max = 3500000},
310         .n = { .min = 1, .max = 4 },
311         .m = { .min = 104, .max = 138 },
312         .m1 = { .min = 16, .max = 23 },
313         .m2 = { .min = 5, .max = 11 },
314         .p = { .min = 5, .max = 80 },
315         .p1 = { .min = 1, .max = 8},
316         .p2 = { .dot_limit = 165000,
317                 .p2_slow = 10, .p2_fast = 5 },
318 };
319
320 static const struct intel_limit intel_limits_g4x_single_channel_lvds = {
321         .dot = { .min = 20000, .max = 115000 },
322         .vco = { .min = 1750000, .max = 3500000 },
323         .n = { .min = 1, .max = 3 },
324         .m = { .min = 104, .max = 138 },
325         .m1 = { .min = 17, .max = 23 },
326         .m2 = { .min = 5, .max = 11 },
327         .p = { .min = 28, .max = 112 },
328         .p1 = { .min = 2, .max = 8 },
329         .p2 = { .dot_limit = 0,
330                 .p2_slow = 14, .p2_fast = 14
331         },
332 };
333
334 static const struct intel_limit intel_limits_g4x_dual_channel_lvds = {
335         .dot = { .min = 80000, .max = 224000 },
336         .vco = { .min = 1750000, .max = 3500000 },
337         .n = { .min = 1, .max = 3 },
338         .m = { .min = 104, .max = 138 },
339         .m1 = { .min = 17, .max = 23 },
340         .m2 = { .min = 5, .max = 11 },
341         .p = { .min = 14, .max = 42 },
342         .p1 = { .min = 2, .max = 6 },
343         .p2 = { .dot_limit = 0,
344                 .p2_slow = 7, .p2_fast = 7
345         },
346 };
347
348 static const struct intel_limit intel_limits_pineview_sdvo = {
349         .dot = { .min = 20000, .max = 400000},
350         .vco = { .min = 1700000, .max = 3500000 },
351         /* Pineview's Ncounter is a ring counter */
352         .n = { .min = 3, .max = 6 },
353         .m = { .min = 2, .max = 256 },
354         /* Pineview only has one combined m divider, which we treat as m2. */
355         .m1 = { .min = 0, .max = 0 },
356         .m2 = { .min = 0, .max = 254 },
357         .p = { .min = 5, .max = 80 },
358         .p1 = { .min = 1, .max = 8 },
359         .p2 = { .dot_limit = 200000,
360                 .p2_slow = 10, .p2_fast = 5 },
361 };
362
363 static const struct intel_limit intel_limits_pineview_lvds = {
364         .dot = { .min = 20000, .max = 400000 },
365         .vco = { .min = 1700000, .max = 3500000 },
366         .n = { .min = 3, .max = 6 },
367         .m = { .min = 2, .max = 256 },
368         .m1 = { .min = 0, .max = 0 },
369         .m2 = { .min = 0, .max = 254 },
370         .p = { .min = 7, .max = 112 },
371         .p1 = { .min = 1, .max = 8 },
372         .p2 = { .dot_limit = 112000,
373                 .p2_slow = 14, .p2_fast = 14 },
374 };
375
376 /* Ironlake / Sandybridge
377  *
378  * We calculate clock using (register_value + 2) for N/M1/M2, so here
379  * the range value for them is (actual_value - 2).
380  */
381 static const struct intel_limit intel_limits_ironlake_dac = {
382         .dot = { .min = 25000, .max = 350000 },
383         .vco = { .min = 1760000, .max = 3510000 },
384         .n = { .min = 1, .max = 5 },
385         .m = { .min = 79, .max = 127 },
386         .m1 = { .min = 12, .max = 22 },
387         .m2 = { .min = 5, .max = 9 },
388         .p = { .min = 5, .max = 80 },
389         .p1 = { .min = 1, .max = 8 },
390         .p2 = { .dot_limit = 225000,
391                 .p2_slow = 10, .p2_fast = 5 },
392 };
393
394 static const struct intel_limit intel_limits_ironlake_single_lvds = {
395         .dot = { .min = 25000, .max = 350000 },
396         .vco = { .min = 1760000, .max = 3510000 },
397         .n = { .min = 1, .max = 3 },
398         .m = { .min = 79, .max = 118 },
399         .m1 = { .min = 12, .max = 22 },
400         .m2 = { .min = 5, .max = 9 },
401         .p = { .min = 28, .max = 112 },
402         .p1 = { .min = 2, .max = 8 },
403         .p2 = { .dot_limit = 225000,
404                 .p2_slow = 14, .p2_fast = 14 },
405 };
406
407 static const struct intel_limit intel_limits_ironlake_dual_lvds = {
408         .dot = { .min = 25000, .max = 350000 },
409         .vco = { .min = 1760000, .max = 3510000 },
410         .n = { .min = 1, .max = 3 },
411         .m = { .min = 79, .max = 127 },
412         .m1 = { .min = 12, .max = 22 },
413         .m2 = { .min = 5, .max = 9 },
414         .p = { .min = 14, .max = 56 },
415         .p1 = { .min = 2, .max = 8 },
416         .p2 = { .dot_limit = 225000,
417                 .p2_slow = 7, .p2_fast = 7 },
418 };
419
420 /* LVDS 100mhz refclk limits. */
421 static const struct intel_limit intel_limits_ironlake_single_lvds_100m = {
422         .dot = { .min = 25000, .max = 350000 },
423         .vco = { .min = 1760000, .max = 3510000 },
424         .n = { .min = 1, .max = 2 },
425         .m = { .min = 79, .max = 126 },
426         .m1 = { .min = 12, .max = 22 },
427         .m2 = { .min = 5, .max = 9 },
428         .p = { .min = 28, .max = 112 },
429         .p1 = { .min = 2, .max = 8 },
430         .p2 = { .dot_limit = 225000,
431                 .p2_slow = 14, .p2_fast = 14 },
432 };
433
434 static const struct intel_limit intel_limits_ironlake_dual_lvds_100m = {
435         .dot = { .min = 25000, .max = 350000 },
436         .vco = { .min = 1760000, .max = 3510000 },
437         .n = { .min = 1, .max = 3 },
438         .m = { .min = 79, .max = 126 },
439         .m1 = { .min = 12, .max = 22 },
440         .m2 = { .min = 5, .max = 9 },
441         .p = { .min = 14, .max = 42 },
442         .p1 = { .min = 2, .max = 6 },
443         .p2 = { .dot_limit = 225000,
444                 .p2_slow = 7, .p2_fast = 7 },
445 };
446
447 static const struct intel_limit intel_limits_vlv = {
448          /*
449           * These are the data rate limits (measured in fast clocks)
450           * since those are the strictest limits we have. The fast
451           * clock and actual rate limits are more relaxed, so checking
452           * them would make no difference.
453           */
454         .dot = { .min = 25000 * 5, .max = 270000 * 5 },
455         .vco = { .min = 4000000, .max = 6000000 },
456         .n = { .min = 1, .max = 7 },
457         .m1 = { .min = 2, .max = 3 },
458         .m2 = { .min = 11, .max = 156 },
459         .p1 = { .min = 2, .max = 3 },
460         .p2 = { .p2_slow = 2, .p2_fast = 20 }, /* slow=min, fast=max */
461 };
462
463 static const struct intel_limit intel_limits_chv = {
464         /*
465          * These are the data rate limits (measured in fast clocks)
466          * since those are the strictest limits we have.  The fast
467          * clock and actual rate limits are more relaxed, so checking
468          * them would make no difference.
469          */
470         .dot = { .min = 25000 * 5, .max = 540000 * 5},
471         .vco = { .min = 4800000, .max = 6480000 },
472         .n = { .min = 1, .max = 1 },
473         .m1 = { .min = 2, .max = 2 },
474         .m2 = { .min = 24 << 22, .max = 175 << 22 },
475         .p1 = { .min = 2, .max = 4 },
476         .p2 = { .p2_slow = 1, .p2_fast = 14 },
477 };
478
479 static const struct intel_limit intel_limits_bxt = {
480         /* FIXME: find real dot limits */
481         .dot = { .min = 0, .max = INT_MAX },
482         .vco = { .min = 4800000, .max = 6700000 },
483         .n = { .min = 1, .max = 1 },
484         .m1 = { .min = 2, .max = 2 },
485         /* FIXME: find real m2 limits */
486         .m2 = { .min = 2 << 22, .max = 255 << 22 },
487         .p1 = { .min = 2, .max = 4 },
488         .p2 = { .p2_slow = 1, .p2_fast = 20 },
489 };
490
491 /* WA Display #0827: Gen9:all */
492 static void
493 skl_wa_827(struct drm_i915_private *dev_priv, int pipe, bool enable)
494 {
495         if (enable)
496                 I915_WRITE(CLKGATE_DIS_PSL(pipe),
497                            I915_READ(CLKGATE_DIS_PSL(pipe)) |
498                            DUPS1_GATING_DIS | DUPS2_GATING_DIS);
499         else
500                 I915_WRITE(CLKGATE_DIS_PSL(pipe),
501                            I915_READ(CLKGATE_DIS_PSL(pipe)) &
502                            ~(DUPS1_GATING_DIS | DUPS2_GATING_DIS));
503 }
504
505 /* Wa_2006604312:icl */
506 static void
507 icl_wa_scalerclkgating(struct drm_i915_private *dev_priv, enum pipe pipe,
508                        bool enable)
509 {
510         if (enable)
511                 I915_WRITE(CLKGATE_DIS_PSL(pipe),
512                            I915_READ(CLKGATE_DIS_PSL(pipe)) | DPFR_GATING_DIS);
513         else
514                 I915_WRITE(CLKGATE_DIS_PSL(pipe),
515                            I915_READ(CLKGATE_DIS_PSL(pipe)) & ~DPFR_GATING_DIS);
516 }
517
518 static bool
519 needs_modeset(const struct intel_crtc_state *state)
520 {
521         return drm_atomic_crtc_needs_modeset(&state->base);
522 }
523
524 /*
525  * Platform specific helpers to calculate the port PLL loopback- (clock.m),
526  * and post-divider (clock.p) values, pre- (clock.vco) and post-divided fast
527  * (clock.dot) clock rates. This fast dot clock is fed to the port's IO logic.
528  * The helpers' return value is the rate of the clock that is fed to the
529  * display engine's pipe which can be the above fast dot clock rate or a
530  * divided-down version of it.
531  */
532 /* m1 is reserved as 0 in Pineview, n is a ring counter */
533 static int pnv_calc_dpll_params(int refclk, struct dpll *clock)
534 {
535         clock->m = clock->m2 + 2;
536         clock->p = clock->p1 * clock->p2;
537         if (WARN_ON(clock->n == 0 || clock->p == 0))
538                 return 0;
539         clock->vco = DIV_ROUND_CLOSEST(refclk * clock->m, clock->n);
540         clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p);
541
542         return clock->dot;
543 }
544
545 static u32 i9xx_dpll_compute_m(struct dpll *dpll)
546 {
547         return 5 * (dpll->m1 + 2) + (dpll->m2 + 2);
548 }
549
550 static int i9xx_calc_dpll_params(int refclk, struct dpll *clock)
551 {
552         clock->m = i9xx_dpll_compute_m(clock);
553         clock->p = clock->p1 * clock->p2;
554         if (WARN_ON(clock->n + 2 == 0 || clock->p == 0))
555                 return 0;
556         clock->vco = DIV_ROUND_CLOSEST(refclk * clock->m, clock->n + 2);
557         clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p);
558
559         return clock->dot;
560 }
561
562 static int vlv_calc_dpll_params(int refclk, struct dpll *clock)
563 {
564         clock->m = clock->m1 * clock->m2;
565         clock->p = clock->p1 * clock->p2;
566         if (WARN_ON(clock->n == 0 || clock->p == 0))
567                 return 0;
568         clock->vco = DIV_ROUND_CLOSEST(refclk * clock->m, clock->n);
569         clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p);
570
571         return clock->dot / 5;
572 }
573
574 int chv_calc_dpll_params(int refclk, struct dpll *clock)
575 {
576         clock->m = clock->m1 * clock->m2;
577         clock->p = clock->p1 * clock->p2;
578         if (WARN_ON(clock->n == 0 || clock->p == 0))
579                 return 0;
580         clock->vco = DIV_ROUND_CLOSEST_ULL(mul_u32_u32(refclk, clock->m),
581                                            clock->n << 22);
582         clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p);
583
584         return clock->dot / 5;
585 }
586
587 #define INTELPllInvalid(s)   do { /* DRM_DEBUG(s); */ return false; } while (0)
588
589 /*
590  * Returns whether the given set of divisors are valid for a given refclk with
591  * the given connectors.
592  */
593 static bool intel_PLL_is_valid(struct drm_i915_private *dev_priv,
594                                const struct intel_limit *limit,
595                                const struct dpll *clock)
596 {
597         if (clock->n   < limit->n.min   || limit->n.max   < clock->n)
598                 INTELPllInvalid("n out of range\n");
599         if (clock->p1  < limit->p1.min  || limit->p1.max  < clock->p1)
600                 INTELPllInvalid("p1 out of range\n");
601         if (clock->m2  < limit->m2.min  || limit->m2.max  < clock->m2)
602                 INTELPllInvalid("m2 out of range\n");
603         if (clock->m1  < limit->m1.min  || limit->m1.max  < clock->m1)
604                 INTELPllInvalid("m1 out of range\n");
605
606         if (!IS_PINEVIEW(dev_priv) && !IS_VALLEYVIEW(dev_priv) &&
607             !IS_CHERRYVIEW(dev_priv) && !IS_GEN9_LP(dev_priv))
608                 if (clock->m1 <= clock->m2)
609                         INTELPllInvalid("m1 <= m2\n");
610
611         if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv) &&
612             !IS_GEN9_LP(dev_priv)) {
613                 if (clock->p < limit->p.min || limit->p.max < clock->p)
614                         INTELPllInvalid("p out of range\n");
615                 if (clock->m < limit->m.min || limit->m.max < clock->m)
616                         INTELPllInvalid("m out of range\n");
617         }
618
619         if (clock->vco < limit->vco.min || limit->vco.max < clock->vco)
620                 INTELPllInvalid("vco out of range\n");
621         /* XXX: We may need to be checking "Dot clock" depending on the multiplier,
622          * connector, etc., rather than just a single range.
623          */
624         if (clock->dot < limit->dot.min || limit->dot.max < clock->dot)
625                 INTELPllInvalid("dot out of range\n");
626
627         return true;
628 }
629
630 static int
631 i9xx_select_p2_div(const struct intel_limit *limit,
632                    const struct intel_crtc_state *crtc_state,
633                    int target)
634 {
635         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
636
637         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
638                 /*
639                  * For LVDS just rely on its current settings for dual-channel.
640                  * We haven't figured out how to reliably set up different
641                  * single/dual channel state, if we even can.
642                  */
643                 if (intel_is_dual_link_lvds(dev_priv))
644                         return limit->p2.p2_fast;
645                 else
646                         return limit->p2.p2_slow;
647         } else {
648                 if (target < limit->p2.dot_limit)
649                         return limit->p2.p2_slow;
650                 else
651                         return limit->p2.p2_fast;
652         }
653 }
654
655 /*
656  * Returns a set of divisors for the desired target clock with the given
657  * refclk, or FALSE.  The returned values represent the clock equation:
658  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
659  *
660  * Target and reference clocks are specified in kHz.
661  *
662  * If match_clock is provided, then best_clock P divider must match the P
663  * divider from @match_clock used for LVDS downclocking.
664  */
665 static bool
666 i9xx_find_best_dpll(const struct intel_limit *limit,
667                     struct intel_crtc_state *crtc_state,
668                     int target, int refclk, struct dpll *match_clock,
669                     struct dpll *best_clock)
670 {
671         struct drm_device *dev = crtc_state->base.crtc->dev;
672         struct dpll clock;
673         int err = target;
674
675         memset(best_clock, 0, sizeof(*best_clock));
676
677         clock.p2 = i9xx_select_p2_div(limit, crtc_state, target);
678
679         for (clock.m1 = limit->m1.min; clock.m1 <= limit->m1.max;
680              clock.m1++) {
681                 for (clock.m2 = limit->m2.min;
682                      clock.m2 <= limit->m2.max; clock.m2++) {
683                         if (clock.m2 >= clock.m1)
684                                 break;
685                         for (clock.n = limit->n.min;
686                              clock.n <= limit->n.max; clock.n++) {
687                                 for (clock.p1 = limit->p1.min;
688                                         clock.p1 <= limit->p1.max; clock.p1++) {
689                                         int this_err;
690
691                                         i9xx_calc_dpll_params(refclk, &clock);
692                                         if (!intel_PLL_is_valid(to_i915(dev),
693                                                                 limit,
694                                                                 &clock))
695                                                 continue;
696                                         if (match_clock &&
697                                             clock.p != match_clock->p)
698                                                 continue;
699
700                                         this_err = abs(clock.dot - target);
701                                         if (this_err < err) {
702                                                 *best_clock = clock;
703                                                 err = this_err;
704                                         }
705                                 }
706                         }
707                 }
708         }
709
710         return (err != target);
711 }
712
713 /*
714  * Returns a set of divisors for the desired target clock with the given
715  * refclk, or FALSE.  The returned values represent the clock equation:
716  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
717  *
718  * Target and reference clocks are specified in kHz.
719  *
720  * If match_clock is provided, then best_clock P divider must match the P
721  * divider from @match_clock used for LVDS downclocking.
722  */
723 static bool
724 pnv_find_best_dpll(const struct intel_limit *limit,
725                    struct intel_crtc_state *crtc_state,
726                    int target, int refclk, struct dpll *match_clock,
727                    struct dpll *best_clock)
728 {
729         struct drm_device *dev = crtc_state->base.crtc->dev;
730         struct dpll clock;
731         int err = target;
732
733         memset(best_clock, 0, sizeof(*best_clock));
734
735         clock.p2 = i9xx_select_p2_div(limit, crtc_state, target);
736
737         for (clock.m1 = limit->m1.min; clock.m1 <= limit->m1.max;
738              clock.m1++) {
739                 for (clock.m2 = limit->m2.min;
740                      clock.m2 <= limit->m2.max; clock.m2++) {
741                         for (clock.n = limit->n.min;
742                              clock.n <= limit->n.max; clock.n++) {
743                                 for (clock.p1 = limit->p1.min;
744                                         clock.p1 <= limit->p1.max; clock.p1++) {
745                                         int this_err;
746
747                                         pnv_calc_dpll_params(refclk, &clock);
748                                         if (!intel_PLL_is_valid(to_i915(dev),
749                                                                 limit,
750                                                                 &clock))
751                                                 continue;
752                                         if (match_clock &&
753                                             clock.p != match_clock->p)
754                                                 continue;
755
756                                         this_err = abs(clock.dot - target);
757                                         if (this_err < err) {
758                                                 *best_clock = clock;
759                                                 err = this_err;
760                                         }
761                                 }
762                         }
763                 }
764         }
765
766         return (err != target);
767 }
768
769 /*
770  * Returns a set of divisors for the desired target clock with the given
771  * refclk, or FALSE.  The returned values represent the clock equation:
772  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
773  *
774  * Target and reference clocks are specified in kHz.
775  *
776  * If match_clock is provided, then best_clock P divider must match the P
777  * divider from @match_clock used for LVDS downclocking.
778  */
779 static bool
780 g4x_find_best_dpll(const struct intel_limit *limit,
781                    struct intel_crtc_state *crtc_state,
782                    int target, int refclk, struct dpll *match_clock,
783                    struct dpll *best_clock)
784 {
785         struct drm_device *dev = crtc_state->base.crtc->dev;
786         struct dpll clock;
787         int max_n;
788         bool found = false;
789         /* approximately equals target * 0.00585 */
790         int err_most = (target >> 8) + (target >> 9);
791
792         memset(best_clock, 0, sizeof(*best_clock));
793
794         clock.p2 = i9xx_select_p2_div(limit, crtc_state, target);
795
796         max_n = limit->n.max;
797         /* based on hardware requirement, prefer smaller n to precision */
798         for (clock.n = limit->n.min; clock.n <= max_n; clock.n++) {
799                 /* based on hardware requirement, prefere larger m1,m2 */
800                 for (clock.m1 = limit->m1.max;
801                      clock.m1 >= limit->m1.min; clock.m1--) {
802                         for (clock.m2 = limit->m2.max;
803                              clock.m2 >= limit->m2.min; clock.m2--) {
804                                 for (clock.p1 = limit->p1.max;
805                                      clock.p1 >= limit->p1.min; clock.p1--) {
806                                         int this_err;
807
808                                         i9xx_calc_dpll_params(refclk, &clock);
809                                         if (!intel_PLL_is_valid(to_i915(dev),
810                                                                 limit,
811                                                                 &clock))
812                                                 continue;
813
814                                         this_err = abs(clock.dot - target);
815                                         if (this_err < err_most) {
816                                                 *best_clock = clock;
817                                                 err_most = this_err;
818                                                 max_n = clock.n;
819                                                 found = true;
820                                         }
821                                 }
822                         }
823                 }
824         }
825         return found;
826 }
827
828 /*
829  * Check if the calculated PLL configuration is more optimal compared to the
830  * best configuration and error found so far. Return the calculated error.
831  */
832 static bool vlv_PLL_is_optimal(struct drm_device *dev, int target_freq,
833                                const struct dpll *calculated_clock,
834                                const struct dpll *best_clock,
835                                unsigned int best_error_ppm,
836                                unsigned int *error_ppm)
837 {
838         /*
839          * For CHV ignore the error and consider only the P value.
840          * Prefer a bigger P value based on HW requirements.
841          */
842         if (IS_CHERRYVIEW(to_i915(dev))) {
843                 *error_ppm = 0;
844
845                 return calculated_clock->p > best_clock->p;
846         }
847
848         if (WARN_ON_ONCE(!target_freq))
849                 return false;
850
851         *error_ppm = div_u64(1000000ULL *
852                                 abs(target_freq - calculated_clock->dot),
853                              target_freq);
854         /*
855          * Prefer a better P value over a better (smaller) error if the error
856          * is small. Ensure this preference for future configurations too by
857          * setting the error to 0.
858          */
859         if (*error_ppm < 100 && calculated_clock->p > best_clock->p) {
860                 *error_ppm = 0;
861
862                 return true;
863         }
864
865         return *error_ppm + 10 < best_error_ppm;
866 }
867
868 /*
869  * Returns a set of divisors for the desired target clock with the given
870  * refclk, or FALSE.  The returned values represent the clock equation:
871  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
872  */
873 static bool
874 vlv_find_best_dpll(const struct intel_limit *limit,
875                    struct intel_crtc_state *crtc_state,
876                    int target, int refclk, struct dpll *match_clock,
877                    struct dpll *best_clock)
878 {
879         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
880         struct drm_device *dev = crtc->base.dev;
881         struct dpll clock;
882         unsigned int bestppm = 1000000;
883         /* min update 19.2 MHz */
884         int max_n = min(limit->n.max, refclk / 19200);
885         bool found = false;
886
887         target *= 5; /* fast clock */
888
889         memset(best_clock, 0, sizeof(*best_clock));
890
891         /* based on hardware requirement, prefer smaller n to precision */
892         for (clock.n = limit->n.min; clock.n <= max_n; clock.n++) {
893                 for (clock.p1 = limit->p1.max; clock.p1 >= limit->p1.min; clock.p1--) {
894                         for (clock.p2 = limit->p2.p2_fast; clock.p2 >= limit->p2.p2_slow;
895                              clock.p2 -= clock.p2 > 10 ? 2 : 1) {
896                                 clock.p = clock.p1 * clock.p2;
897                                 /* based on hardware requirement, prefer bigger m1,m2 values */
898                                 for (clock.m1 = limit->m1.min; clock.m1 <= limit->m1.max; clock.m1++) {
899                                         unsigned int ppm;
900
901                                         clock.m2 = DIV_ROUND_CLOSEST(target * clock.p * clock.n,
902                                                                      refclk * clock.m1);
903
904                                         vlv_calc_dpll_params(refclk, &clock);
905
906                                         if (!intel_PLL_is_valid(to_i915(dev),
907                                                                 limit,
908                                                                 &clock))
909                                                 continue;
910
911                                         if (!vlv_PLL_is_optimal(dev, target,
912                                                                 &clock,
913                                                                 best_clock,
914                                                                 bestppm, &ppm))
915                                                 continue;
916
917                                         *best_clock = clock;
918                                         bestppm = ppm;
919                                         found = true;
920                                 }
921                         }
922                 }
923         }
924
925         return found;
926 }
927
928 /*
929  * Returns a set of divisors for the desired target clock with the given
930  * refclk, or FALSE.  The returned values represent the clock equation:
931  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
932  */
933 static bool
934 chv_find_best_dpll(const struct intel_limit *limit,
935                    struct intel_crtc_state *crtc_state,
936                    int target, int refclk, struct dpll *match_clock,
937                    struct dpll *best_clock)
938 {
939         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
940         struct drm_device *dev = crtc->base.dev;
941         unsigned int best_error_ppm;
942         struct dpll clock;
943         u64 m2;
944         int found = false;
945
946         memset(best_clock, 0, sizeof(*best_clock));
947         best_error_ppm = 1000000;
948
949         /*
950          * Based on hardware doc, the n always set to 1, and m1 always
951          * set to 2.  If requires to support 200Mhz refclk, we need to
952          * revisit this because n may not 1 anymore.
953          */
954         clock.n = 1, clock.m1 = 2;
955         target *= 5;    /* fast clock */
956
957         for (clock.p1 = limit->p1.max; clock.p1 >= limit->p1.min; clock.p1--) {
958                 for (clock.p2 = limit->p2.p2_fast;
959                                 clock.p2 >= limit->p2.p2_slow;
960                                 clock.p2 -= clock.p2 > 10 ? 2 : 1) {
961                         unsigned int error_ppm;
962
963                         clock.p = clock.p1 * clock.p2;
964
965                         m2 = DIV_ROUND_CLOSEST_ULL(mul_u32_u32(target, clock.p * clock.n) << 22,
966                                                    refclk * clock.m1);
967
968                         if (m2 > INT_MAX/clock.m1)
969                                 continue;
970
971                         clock.m2 = m2;
972
973                         chv_calc_dpll_params(refclk, &clock);
974
975                         if (!intel_PLL_is_valid(to_i915(dev), limit, &clock))
976                                 continue;
977
978                         if (!vlv_PLL_is_optimal(dev, target, &clock, best_clock,
979                                                 best_error_ppm, &error_ppm))
980                                 continue;
981
982                         *best_clock = clock;
983                         best_error_ppm = error_ppm;
984                         found = true;
985                 }
986         }
987
988         return found;
989 }
990
991 bool bxt_find_best_dpll(struct intel_crtc_state *crtc_state,
992                         struct dpll *best_clock)
993 {
994         int refclk = 100000;
995         const struct intel_limit *limit = &intel_limits_bxt;
996
997         return chv_find_best_dpll(limit, crtc_state,
998                                   crtc_state->port_clock, refclk,
999                                   NULL, best_clock);
1000 }
1001
1002 bool intel_crtc_active(struct intel_crtc *crtc)
1003 {
1004         /* Be paranoid as we can arrive here with only partial
1005          * state retrieved from the hardware during setup.
1006          *
1007          * We can ditch the adjusted_mode.crtc_clock check as soon
1008          * as Haswell has gained clock readout/fastboot support.
1009          *
1010          * We can ditch the crtc->primary->state->fb check as soon as we can
1011          * properly reconstruct framebuffers.
1012          *
1013          * FIXME: The intel_crtc->active here should be switched to
1014          * crtc->state->active once we have proper CRTC states wired up
1015          * for atomic.
1016          */
1017         return crtc->active && crtc->base.primary->state->fb &&
1018                 crtc->config->base.adjusted_mode.crtc_clock;
1019 }
1020
1021 enum transcoder intel_pipe_to_cpu_transcoder(struct drm_i915_private *dev_priv,
1022                                              enum pipe pipe)
1023 {
1024         struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
1025
1026         return crtc->config->cpu_transcoder;
1027 }
1028
1029 static bool pipe_scanline_is_moving(struct drm_i915_private *dev_priv,
1030                                     enum pipe pipe)
1031 {
1032         i915_reg_t reg = PIPEDSL(pipe);
1033         u32 line1, line2;
1034         u32 line_mask;
1035
1036         if (IS_GEN(dev_priv, 2))
1037                 line_mask = DSL_LINEMASK_GEN2;
1038         else
1039                 line_mask = DSL_LINEMASK_GEN3;
1040
1041         line1 = I915_READ(reg) & line_mask;
1042         msleep(5);
1043         line2 = I915_READ(reg) & line_mask;
1044
1045         return line1 != line2;
1046 }
1047
1048 static void wait_for_pipe_scanline_moving(struct intel_crtc *crtc, bool state)
1049 {
1050         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1051         enum pipe pipe = crtc->pipe;
1052
1053         /* Wait for the display line to settle/start moving */
1054         if (wait_for(pipe_scanline_is_moving(dev_priv, pipe) == state, 100))
1055                 DRM_ERROR("pipe %c scanline %s wait timed out\n",
1056                           pipe_name(pipe), onoff(state));
1057 }
1058
1059 static void intel_wait_for_pipe_scanline_stopped(struct intel_crtc *crtc)
1060 {
1061         wait_for_pipe_scanline_moving(crtc, false);
1062 }
1063
1064 static void intel_wait_for_pipe_scanline_moving(struct intel_crtc *crtc)
1065 {
1066         wait_for_pipe_scanline_moving(crtc, true);
1067 }
1068
1069 static void
1070 intel_wait_for_pipe_off(const struct intel_crtc_state *old_crtc_state)
1071 {
1072         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc);
1073         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1074
1075         if (INTEL_GEN(dev_priv) >= 4) {
1076                 enum transcoder cpu_transcoder = old_crtc_state->cpu_transcoder;
1077                 i915_reg_t reg = PIPECONF(cpu_transcoder);
1078
1079                 /* Wait for the Pipe State to go off */
1080                 if (intel_wait_for_register(&dev_priv->uncore,
1081                                             reg, I965_PIPECONF_ACTIVE, 0,
1082                                             100))
1083                         WARN(1, "pipe_off wait timed out\n");
1084         } else {
1085                 intel_wait_for_pipe_scanline_stopped(crtc);
1086         }
1087 }
1088
1089 /* Only for pre-ILK configs */
1090 void assert_pll(struct drm_i915_private *dev_priv,
1091                 enum pipe pipe, bool state)
1092 {
1093         u32 val;
1094         bool cur_state;
1095
1096         val = I915_READ(DPLL(pipe));
1097         cur_state = !!(val & DPLL_VCO_ENABLE);
1098         I915_STATE_WARN(cur_state != state,
1099              "PLL state assertion failure (expected %s, current %s)\n",
1100                         onoff(state), onoff(cur_state));
1101 }
1102
1103 /* XXX: the dsi pll is shared between MIPI DSI ports */
1104 void assert_dsi_pll(struct drm_i915_private *dev_priv, bool state)
1105 {
1106         u32 val;
1107         bool cur_state;
1108
1109         vlv_cck_get(dev_priv);
1110         val = vlv_cck_read(dev_priv, CCK_REG_DSI_PLL_CONTROL);
1111         vlv_cck_put(dev_priv);
1112
1113         cur_state = val & DSI_PLL_VCO_EN;
1114         I915_STATE_WARN(cur_state != state,
1115              "DSI PLL state assertion failure (expected %s, current %s)\n",
1116                         onoff(state), onoff(cur_state));
1117 }
1118
1119 static void assert_fdi_tx(struct drm_i915_private *dev_priv,
1120                           enum pipe pipe, bool state)
1121 {
1122         bool cur_state;
1123         enum transcoder cpu_transcoder = intel_pipe_to_cpu_transcoder(dev_priv,
1124                                                                       pipe);
1125
1126         if (HAS_DDI(dev_priv)) {
1127                 /* DDI does not have a specific FDI_TX register */
1128                 u32 val = I915_READ(TRANS_DDI_FUNC_CTL(cpu_transcoder));
1129                 cur_state = !!(val & TRANS_DDI_FUNC_ENABLE);
1130         } else {
1131                 u32 val = I915_READ(FDI_TX_CTL(pipe));
1132                 cur_state = !!(val & FDI_TX_ENABLE);
1133         }
1134         I915_STATE_WARN(cur_state != state,
1135              "FDI TX state assertion failure (expected %s, current %s)\n",
1136                         onoff(state), onoff(cur_state));
1137 }
1138 #define assert_fdi_tx_enabled(d, p) assert_fdi_tx(d, p, true)
1139 #define assert_fdi_tx_disabled(d, p) assert_fdi_tx(d, p, false)
1140
1141 static void assert_fdi_rx(struct drm_i915_private *dev_priv,
1142                           enum pipe pipe, bool state)
1143 {
1144         u32 val;
1145         bool cur_state;
1146
1147         val = I915_READ(FDI_RX_CTL(pipe));
1148         cur_state = !!(val & FDI_RX_ENABLE);
1149         I915_STATE_WARN(cur_state != state,
1150              "FDI RX state assertion failure (expected %s, current %s)\n",
1151                         onoff(state), onoff(cur_state));
1152 }
1153 #define assert_fdi_rx_enabled(d, p) assert_fdi_rx(d, p, true)
1154 #define assert_fdi_rx_disabled(d, p) assert_fdi_rx(d, p, false)
1155
1156 static void assert_fdi_tx_pll_enabled(struct drm_i915_private *dev_priv,
1157                                       enum pipe pipe)
1158 {
1159         u32 val;
1160
1161         /* ILK FDI PLL is always enabled */
1162         if (IS_GEN(dev_priv, 5))
1163                 return;
1164
1165         /* On Haswell, DDI ports are responsible for the FDI PLL setup */
1166         if (HAS_DDI(dev_priv))
1167                 return;
1168
1169         val = I915_READ(FDI_TX_CTL(pipe));
1170         I915_STATE_WARN(!(val & FDI_TX_PLL_ENABLE), "FDI TX PLL assertion failure, should be active but is disabled\n");
1171 }
1172
1173 void assert_fdi_rx_pll(struct drm_i915_private *dev_priv,
1174                        enum pipe pipe, bool state)
1175 {
1176         u32 val;
1177         bool cur_state;
1178
1179         val = I915_READ(FDI_RX_CTL(pipe));
1180         cur_state = !!(val & FDI_RX_PLL_ENABLE);
1181         I915_STATE_WARN(cur_state != state,
1182              "FDI RX PLL assertion failure (expected %s, current %s)\n",
1183                         onoff(state), onoff(cur_state));
1184 }
1185
1186 void assert_panel_unlocked(struct drm_i915_private *dev_priv, enum pipe pipe)
1187 {
1188         i915_reg_t pp_reg;
1189         u32 val;
1190         enum pipe panel_pipe = INVALID_PIPE;
1191         bool locked = true;
1192
1193         if (WARN_ON(HAS_DDI(dev_priv)))
1194                 return;
1195
1196         if (HAS_PCH_SPLIT(dev_priv)) {
1197                 u32 port_sel;
1198
1199                 pp_reg = PP_CONTROL(0);
1200                 port_sel = I915_READ(PP_ON_DELAYS(0)) & PANEL_PORT_SELECT_MASK;
1201
1202                 switch (port_sel) {
1203                 case PANEL_PORT_SELECT_LVDS:
1204                         intel_lvds_port_enabled(dev_priv, PCH_LVDS, &panel_pipe);
1205                         break;
1206                 case PANEL_PORT_SELECT_DPA:
1207                         intel_dp_port_enabled(dev_priv, DP_A, PORT_A, &panel_pipe);
1208                         break;
1209                 case PANEL_PORT_SELECT_DPC:
1210                         intel_dp_port_enabled(dev_priv, PCH_DP_C, PORT_C, &panel_pipe);
1211                         break;
1212                 case PANEL_PORT_SELECT_DPD:
1213                         intel_dp_port_enabled(dev_priv, PCH_DP_D, PORT_D, &panel_pipe);
1214                         break;
1215                 default:
1216                         MISSING_CASE(port_sel);
1217                         break;
1218                 }
1219         } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
1220                 /* presumably write lock depends on pipe, not port select */
1221                 pp_reg = PP_CONTROL(pipe);
1222                 panel_pipe = pipe;
1223         } else {
1224                 u32 port_sel;
1225
1226                 pp_reg = PP_CONTROL(0);
1227                 port_sel = I915_READ(PP_ON_DELAYS(0)) & PANEL_PORT_SELECT_MASK;
1228
1229                 WARN_ON(port_sel != PANEL_PORT_SELECT_LVDS);
1230                 intel_lvds_port_enabled(dev_priv, LVDS, &panel_pipe);
1231         }
1232
1233         val = I915_READ(pp_reg);
1234         if (!(val & PANEL_POWER_ON) ||
1235             ((val & PANEL_UNLOCK_MASK) == PANEL_UNLOCK_REGS))
1236                 locked = false;
1237
1238         I915_STATE_WARN(panel_pipe == pipe && locked,
1239              "panel assertion failure, pipe %c regs locked\n",
1240              pipe_name(pipe));
1241 }
1242
1243 void assert_pipe(struct drm_i915_private *dev_priv,
1244                  enum pipe pipe, bool state)
1245 {
1246         bool cur_state;
1247         enum transcoder cpu_transcoder = intel_pipe_to_cpu_transcoder(dev_priv,
1248                                                                       pipe);
1249         enum intel_display_power_domain power_domain;
1250         intel_wakeref_t wakeref;
1251
1252         /* we keep both pipes enabled on 830 */
1253         if (IS_I830(dev_priv))
1254                 state = true;
1255
1256         power_domain = POWER_DOMAIN_TRANSCODER(cpu_transcoder);
1257         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
1258         if (wakeref) {
1259                 u32 val = I915_READ(PIPECONF(cpu_transcoder));
1260                 cur_state = !!(val & PIPECONF_ENABLE);
1261
1262                 intel_display_power_put(dev_priv, power_domain, wakeref);
1263         } else {
1264                 cur_state = false;
1265         }
1266
1267         I915_STATE_WARN(cur_state != state,
1268              "pipe %c assertion failure (expected %s, current %s)\n",
1269                         pipe_name(pipe), onoff(state), onoff(cur_state));
1270 }
1271
1272 static void assert_plane(struct intel_plane *plane, bool state)
1273 {
1274         enum pipe pipe;
1275         bool cur_state;
1276
1277         cur_state = plane->get_hw_state(plane, &pipe);
1278
1279         I915_STATE_WARN(cur_state != state,
1280                         "%s assertion failure (expected %s, current %s)\n",
1281                         plane->base.name, onoff(state), onoff(cur_state));
1282 }
1283
1284 #define assert_plane_enabled(p) assert_plane(p, true)
1285 #define assert_plane_disabled(p) assert_plane(p, false)
1286
1287 static void assert_planes_disabled(struct intel_crtc *crtc)
1288 {
1289         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1290         struct intel_plane *plane;
1291
1292         for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane)
1293                 assert_plane_disabled(plane);
1294 }
1295
1296 static void assert_vblank_disabled(struct drm_crtc *crtc)
1297 {
1298         if (I915_STATE_WARN_ON(drm_crtc_vblank_get(crtc) == 0))
1299                 drm_crtc_vblank_put(crtc);
1300 }
1301
1302 void assert_pch_transcoder_disabled(struct drm_i915_private *dev_priv,
1303                                     enum pipe pipe)
1304 {
1305         u32 val;
1306         bool enabled;
1307
1308         val = I915_READ(PCH_TRANSCONF(pipe));
1309         enabled = !!(val & TRANS_ENABLE);
1310         I915_STATE_WARN(enabled,
1311              "transcoder assertion failed, should be off on pipe %c but is still active\n",
1312              pipe_name(pipe));
1313 }
1314
1315 static void assert_pch_dp_disabled(struct drm_i915_private *dev_priv,
1316                                    enum pipe pipe, enum port port,
1317                                    i915_reg_t dp_reg)
1318 {
1319         enum pipe port_pipe;
1320         bool state;
1321
1322         state = intel_dp_port_enabled(dev_priv, dp_reg, port, &port_pipe);
1323
1324         I915_STATE_WARN(state && port_pipe == pipe,
1325                         "PCH DP %c enabled on transcoder %c, should be disabled\n",
1326                         port_name(port), pipe_name(pipe));
1327
1328         I915_STATE_WARN(HAS_PCH_IBX(dev_priv) && !state && port_pipe == PIPE_B,
1329                         "IBX PCH DP %c still using transcoder B\n",
1330                         port_name(port));
1331 }
1332
1333 static void assert_pch_hdmi_disabled(struct drm_i915_private *dev_priv,
1334                                      enum pipe pipe, enum port port,
1335                                      i915_reg_t hdmi_reg)
1336 {
1337         enum pipe port_pipe;
1338         bool state;
1339
1340         state = intel_sdvo_port_enabled(dev_priv, hdmi_reg, &port_pipe);
1341
1342         I915_STATE_WARN(state && port_pipe == pipe,
1343                         "PCH HDMI %c enabled on transcoder %c, should be disabled\n",
1344                         port_name(port), pipe_name(pipe));
1345
1346         I915_STATE_WARN(HAS_PCH_IBX(dev_priv) && !state && port_pipe == PIPE_B,
1347                         "IBX PCH HDMI %c still using transcoder B\n",
1348                         port_name(port));
1349 }
1350
1351 static void assert_pch_ports_disabled(struct drm_i915_private *dev_priv,
1352                                       enum pipe pipe)
1353 {
1354         enum pipe port_pipe;
1355
1356         assert_pch_dp_disabled(dev_priv, pipe, PORT_B, PCH_DP_B);
1357         assert_pch_dp_disabled(dev_priv, pipe, PORT_C, PCH_DP_C);
1358         assert_pch_dp_disabled(dev_priv, pipe, PORT_D, PCH_DP_D);
1359
1360         I915_STATE_WARN(intel_crt_port_enabled(dev_priv, PCH_ADPA, &port_pipe) &&
1361                         port_pipe == pipe,
1362                         "PCH VGA enabled on transcoder %c, should be disabled\n",
1363                         pipe_name(pipe));
1364
1365         I915_STATE_WARN(intel_lvds_port_enabled(dev_priv, PCH_LVDS, &port_pipe) &&
1366                         port_pipe == pipe,
1367                         "PCH LVDS enabled on transcoder %c, should be disabled\n",
1368                         pipe_name(pipe));
1369
1370         /* PCH SDVOB multiplex with HDMIB */
1371         assert_pch_hdmi_disabled(dev_priv, pipe, PORT_B, PCH_HDMIB);
1372         assert_pch_hdmi_disabled(dev_priv, pipe, PORT_C, PCH_HDMIC);
1373         assert_pch_hdmi_disabled(dev_priv, pipe, PORT_D, PCH_HDMID);
1374 }
1375
1376 static void _vlv_enable_pll(struct intel_crtc *crtc,
1377                             const struct intel_crtc_state *pipe_config)
1378 {
1379         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1380         enum pipe pipe = crtc->pipe;
1381
1382         I915_WRITE(DPLL(pipe), pipe_config->dpll_hw_state.dpll);
1383         POSTING_READ(DPLL(pipe));
1384         udelay(150);
1385
1386         if (intel_wait_for_register(&dev_priv->uncore,
1387                                     DPLL(pipe),
1388                                     DPLL_LOCK_VLV,
1389                                     DPLL_LOCK_VLV,
1390                                     1))
1391                 DRM_ERROR("DPLL %d failed to lock\n", pipe);
1392 }
1393
1394 static void vlv_enable_pll(struct intel_crtc *crtc,
1395                            const struct intel_crtc_state *pipe_config)
1396 {
1397         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1398         enum pipe pipe = crtc->pipe;
1399
1400         assert_pipe_disabled(dev_priv, pipe);
1401
1402         /* PLL is protected by panel, make sure we can write it */
1403         assert_panel_unlocked(dev_priv, pipe);
1404
1405         if (pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE)
1406                 _vlv_enable_pll(crtc, pipe_config);
1407
1408         I915_WRITE(DPLL_MD(pipe), pipe_config->dpll_hw_state.dpll_md);
1409         POSTING_READ(DPLL_MD(pipe));
1410 }
1411
1412
1413 static void _chv_enable_pll(struct intel_crtc *crtc,
1414                             const struct intel_crtc_state *pipe_config)
1415 {
1416         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1417         enum pipe pipe = crtc->pipe;
1418         enum dpio_channel port = vlv_pipe_to_channel(pipe);
1419         u32 tmp;
1420
1421         vlv_dpio_get(dev_priv);
1422
1423         /* Enable back the 10bit clock to display controller */
1424         tmp = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW14(port));
1425         tmp |= DPIO_DCLKP_EN;
1426         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW14(port), tmp);
1427
1428         vlv_dpio_put(dev_priv);
1429
1430         /*
1431          * Need to wait > 100ns between dclkp clock enable bit and PLL enable.
1432          */
1433         udelay(1);
1434
1435         /* Enable PLL */
1436         I915_WRITE(DPLL(pipe), pipe_config->dpll_hw_state.dpll);
1437
1438         /* Check PLL is locked */
1439         if (intel_wait_for_register(&dev_priv->uncore,
1440                                     DPLL(pipe), DPLL_LOCK_VLV, DPLL_LOCK_VLV,
1441                                     1))
1442                 DRM_ERROR("PLL %d failed to lock\n", pipe);
1443 }
1444
1445 static void chv_enable_pll(struct intel_crtc *crtc,
1446                            const struct intel_crtc_state *pipe_config)
1447 {
1448         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1449         enum pipe pipe = crtc->pipe;
1450
1451         assert_pipe_disabled(dev_priv, pipe);
1452
1453         /* PLL is protected by panel, make sure we can write it */
1454         assert_panel_unlocked(dev_priv, pipe);
1455
1456         if (pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE)
1457                 _chv_enable_pll(crtc, pipe_config);
1458
1459         if (pipe != PIPE_A) {
1460                 /*
1461                  * WaPixelRepeatModeFixForC0:chv
1462                  *
1463                  * DPLLCMD is AWOL. Use chicken bits to propagate
1464                  * the value from DPLLBMD to either pipe B or C.
1465                  */
1466                 I915_WRITE(CBR4_VLV, CBR_DPLLBMD_PIPE(pipe));
1467                 I915_WRITE(DPLL_MD(PIPE_B), pipe_config->dpll_hw_state.dpll_md);
1468                 I915_WRITE(CBR4_VLV, 0);
1469                 dev_priv->chv_dpll_md[pipe] = pipe_config->dpll_hw_state.dpll_md;
1470
1471                 /*
1472                  * DPLLB VGA mode also seems to cause problems.
1473                  * We should always have it disabled.
1474                  */
1475                 WARN_ON((I915_READ(DPLL(PIPE_B)) & DPLL_VGA_MODE_DIS) == 0);
1476         } else {
1477                 I915_WRITE(DPLL_MD(pipe), pipe_config->dpll_hw_state.dpll_md);
1478                 POSTING_READ(DPLL_MD(pipe));
1479         }
1480 }
1481
1482 static bool i9xx_has_pps(struct drm_i915_private *dev_priv)
1483 {
1484         if (IS_I830(dev_priv))
1485                 return false;
1486
1487         return IS_PINEVIEW(dev_priv) || IS_MOBILE(dev_priv);
1488 }
1489
1490 static void i9xx_enable_pll(struct intel_crtc *crtc,
1491                             const struct intel_crtc_state *crtc_state)
1492 {
1493         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1494         i915_reg_t reg = DPLL(crtc->pipe);
1495         u32 dpll = crtc_state->dpll_hw_state.dpll;
1496         int i;
1497
1498         assert_pipe_disabled(dev_priv, crtc->pipe);
1499
1500         /* PLL is protected by panel, make sure we can write it */
1501         if (i9xx_has_pps(dev_priv))
1502                 assert_panel_unlocked(dev_priv, crtc->pipe);
1503
1504         /*
1505          * Apparently we need to have VGA mode enabled prior to changing
1506          * the P1/P2 dividers. Otherwise the DPLL will keep using the old
1507          * dividers, even though the register value does change.
1508          */
1509         I915_WRITE(reg, dpll & ~DPLL_VGA_MODE_DIS);
1510         I915_WRITE(reg, dpll);
1511
1512         /* Wait for the clocks to stabilize. */
1513         POSTING_READ(reg);
1514         udelay(150);
1515
1516         if (INTEL_GEN(dev_priv) >= 4) {
1517                 I915_WRITE(DPLL_MD(crtc->pipe),
1518                            crtc_state->dpll_hw_state.dpll_md);
1519         } else {
1520                 /* The pixel multiplier can only be updated once the
1521                  * DPLL is enabled and the clocks are stable.
1522                  *
1523                  * So write it again.
1524                  */
1525                 I915_WRITE(reg, dpll);
1526         }
1527
1528         /* We do this three times for luck */
1529         for (i = 0; i < 3; i++) {
1530                 I915_WRITE(reg, dpll);
1531                 POSTING_READ(reg);
1532                 udelay(150); /* wait for warmup */
1533         }
1534 }
1535
1536 static void i9xx_disable_pll(const struct intel_crtc_state *crtc_state)
1537 {
1538         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
1539         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1540         enum pipe pipe = crtc->pipe;
1541
1542         /* Don't disable pipe or pipe PLLs if needed */
1543         if (IS_I830(dev_priv))
1544                 return;
1545
1546         /* Make sure the pipe isn't still relying on us */
1547         assert_pipe_disabled(dev_priv, pipe);
1548
1549         I915_WRITE(DPLL(pipe), DPLL_VGA_MODE_DIS);
1550         POSTING_READ(DPLL(pipe));
1551 }
1552
1553 static void vlv_disable_pll(struct drm_i915_private *dev_priv, enum pipe pipe)
1554 {
1555         u32 val;
1556
1557         /* Make sure the pipe isn't still relying on us */
1558         assert_pipe_disabled(dev_priv, pipe);
1559
1560         val = DPLL_INTEGRATED_REF_CLK_VLV |
1561                 DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
1562         if (pipe != PIPE_A)
1563                 val |= DPLL_INTEGRATED_CRI_CLK_VLV;
1564
1565         I915_WRITE(DPLL(pipe), val);
1566         POSTING_READ(DPLL(pipe));
1567 }
1568
1569 static void chv_disable_pll(struct drm_i915_private *dev_priv, enum pipe pipe)
1570 {
1571         enum dpio_channel port = vlv_pipe_to_channel(pipe);
1572         u32 val;
1573
1574         /* Make sure the pipe isn't still relying on us */
1575         assert_pipe_disabled(dev_priv, pipe);
1576
1577         val = DPLL_SSC_REF_CLK_CHV |
1578                 DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
1579         if (pipe != PIPE_A)
1580                 val |= DPLL_INTEGRATED_CRI_CLK_VLV;
1581
1582         I915_WRITE(DPLL(pipe), val);
1583         POSTING_READ(DPLL(pipe));
1584
1585         vlv_dpio_get(dev_priv);
1586
1587         /* Disable 10bit clock to display controller */
1588         val = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW14(port));
1589         val &= ~DPIO_DCLKP_EN;
1590         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW14(port), val);
1591
1592         vlv_dpio_put(dev_priv);
1593 }
1594
1595 void vlv_wait_port_ready(struct drm_i915_private *dev_priv,
1596                          struct intel_digital_port *dport,
1597                          unsigned int expected_mask)
1598 {
1599         u32 port_mask;
1600         i915_reg_t dpll_reg;
1601
1602         switch (dport->base.port) {
1603         case PORT_B:
1604                 port_mask = DPLL_PORTB_READY_MASK;
1605                 dpll_reg = DPLL(0);
1606                 break;
1607         case PORT_C:
1608                 port_mask = DPLL_PORTC_READY_MASK;
1609                 dpll_reg = DPLL(0);
1610                 expected_mask <<= 4;
1611                 break;
1612         case PORT_D:
1613                 port_mask = DPLL_PORTD_READY_MASK;
1614                 dpll_reg = DPIO_PHY_STATUS;
1615                 break;
1616         default:
1617                 BUG();
1618         }
1619
1620         if (intel_wait_for_register(&dev_priv->uncore,
1621                                     dpll_reg, port_mask, expected_mask,
1622                                     1000))
1623                 WARN(1, "timed out waiting for port %c ready: got 0x%x, expected 0x%x\n",
1624                      port_name(dport->base.port),
1625                      I915_READ(dpll_reg) & port_mask, expected_mask);
1626 }
1627
1628 static void ironlake_enable_pch_transcoder(const struct intel_crtc_state *crtc_state)
1629 {
1630         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
1631         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1632         enum pipe pipe = crtc->pipe;
1633         i915_reg_t reg;
1634         u32 val, pipeconf_val;
1635
1636         /* Make sure PCH DPLL is enabled */
1637         assert_shared_dpll_enabled(dev_priv, crtc_state->shared_dpll);
1638
1639         /* FDI must be feeding us bits for PCH ports */
1640         assert_fdi_tx_enabled(dev_priv, pipe);
1641         assert_fdi_rx_enabled(dev_priv, pipe);
1642
1643         if (HAS_PCH_CPT(dev_priv)) {
1644                 /* Workaround: Set the timing override bit before enabling the
1645                  * pch transcoder. */
1646                 reg = TRANS_CHICKEN2(pipe);
1647                 val = I915_READ(reg);
1648                 val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
1649                 I915_WRITE(reg, val);
1650         }
1651
1652         reg = PCH_TRANSCONF(pipe);
1653         val = I915_READ(reg);
1654         pipeconf_val = I915_READ(PIPECONF(pipe));
1655
1656         if (HAS_PCH_IBX(dev_priv)) {
1657                 /*
1658                  * Make the BPC in transcoder be consistent with
1659                  * that in pipeconf reg. For HDMI we must use 8bpc
1660                  * here for both 8bpc and 12bpc.
1661                  */
1662                 val &= ~PIPECONF_BPC_MASK;
1663                 if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI))
1664                         val |= PIPECONF_8BPC;
1665                 else
1666                         val |= pipeconf_val & PIPECONF_BPC_MASK;
1667         }
1668
1669         val &= ~TRANS_INTERLACE_MASK;
1670         if ((pipeconf_val & PIPECONF_INTERLACE_MASK) == PIPECONF_INTERLACED_ILK) {
1671                 if (HAS_PCH_IBX(dev_priv) &&
1672                     intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO))
1673                         val |= TRANS_LEGACY_INTERLACED_ILK;
1674                 else
1675                         val |= TRANS_INTERLACED;
1676         } else {
1677                 val |= TRANS_PROGRESSIVE;
1678         }
1679
1680         I915_WRITE(reg, val | TRANS_ENABLE);
1681         if (intel_wait_for_register(&dev_priv->uncore,
1682                                     reg, TRANS_STATE_ENABLE, TRANS_STATE_ENABLE,
1683                                     100))
1684                 DRM_ERROR("failed to enable transcoder %c\n", pipe_name(pipe));
1685 }
1686
1687 static void lpt_enable_pch_transcoder(struct drm_i915_private *dev_priv,
1688                                       enum transcoder cpu_transcoder)
1689 {
1690         u32 val, pipeconf_val;
1691
1692         /* FDI must be feeding us bits for PCH ports */
1693         assert_fdi_tx_enabled(dev_priv, (enum pipe) cpu_transcoder);
1694         assert_fdi_rx_enabled(dev_priv, PIPE_A);
1695
1696         /* Workaround: set timing override bit. */
1697         val = I915_READ(TRANS_CHICKEN2(PIPE_A));
1698         val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
1699         I915_WRITE(TRANS_CHICKEN2(PIPE_A), val);
1700
1701         val = TRANS_ENABLE;
1702         pipeconf_val = I915_READ(PIPECONF(cpu_transcoder));
1703
1704         if ((pipeconf_val & PIPECONF_INTERLACE_MASK_HSW) ==
1705             PIPECONF_INTERLACED_ILK)
1706                 val |= TRANS_INTERLACED;
1707         else
1708                 val |= TRANS_PROGRESSIVE;
1709
1710         I915_WRITE(LPT_TRANSCONF, val);
1711         if (intel_wait_for_register(&dev_priv->uncore,
1712                                     LPT_TRANSCONF,
1713                                     TRANS_STATE_ENABLE,
1714                                     TRANS_STATE_ENABLE,
1715                                     100))
1716                 DRM_ERROR("Failed to enable PCH transcoder\n");
1717 }
1718
1719 static void ironlake_disable_pch_transcoder(struct drm_i915_private *dev_priv,
1720                                             enum pipe pipe)
1721 {
1722         i915_reg_t reg;
1723         u32 val;
1724
1725         /* FDI relies on the transcoder */
1726         assert_fdi_tx_disabled(dev_priv, pipe);
1727         assert_fdi_rx_disabled(dev_priv, pipe);
1728
1729         /* Ports must be off as well */
1730         assert_pch_ports_disabled(dev_priv, pipe);
1731
1732         reg = PCH_TRANSCONF(pipe);
1733         val = I915_READ(reg);
1734         val &= ~TRANS_ENABLE;
1735         I915_WRITE(reg, val);
1736         /* wait for PCH transcoder off, transcoder state */
1737         if (intel_wait_for_register(&dev_priv->uncore,
1738                                     reg, TRANS_STATE_ENABLE, 0,
1739                                     50))
1740                 DRM_ERROR("failed to disable transcoder %c\n", pipe_name(pipe));
1741
1742         if (HAS_PCH_CPT(dev_priv)) {
1743                 /* Workaround: Clear the timing override chicken bit again. */
1744                 reg = TRANS_CHICKEN2(pipe);
1745                 val = I915_READ(reg);
1746                 val &= ~TRANS_CHICKEN2_TIMING_OVERRIDE;
1747                 I915_WRITE(reg, val);
1748         }
1749 }
1750
1751 void lpt_disable_pch_transcoder(struct drm_i915_private *dev_priv)
1752 {
1753         u32 val;
1754
1755         val = I915_READ(LPT_TRANSCONF);
1756         val &= ~TRANS_ENABLE;
1757         I915_WRITE(LPT_TRANSCONF, val);
1758         /* wait for PCH transcoder off, transcoder state */
1759         if (intel_wait_for_register(&dev_priv->uncore,
1760                                     LPT_TRANSCONF, TRANS_STATE_ENABLE, 0,
1761                                     50))
1762                 DRM_ERROR("Failed to disable PCH transcoder\n");
1763
1764         /* Workaround: clear timing override bit. */
1765         val = I915_READ(TRANS_CHICKEN2(PIPE_A));
1766         val &= ~TRANS_CHICKEN2_TIMING_OVERRIDE;
1767         I915_WRITE(TRANS_CHICKEN2(PIPE_A), val);
1768 }
1769
1770 enum pipe intel_crtc_pch_transcoder(struct intel_crtc *crtc)
1771 {
1772         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1773
1774         if (HAS_PCH_LPT(dev_priv))
1775                 return PIPE_A;
1776         else
1777                 return crtc->pipe;
1778 }
1779
1780 static u32 intel_crtc_max_vblank_count(const struct intel_crtc_state *crtc_state)
1781 {
1782         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
1783
1784         /*
1785          * On i965gm the hardware frame counter reads
1786          * zero when the TV encoder is enabled :(
1787          */
1788         if (IS_I965GM(dev_priv) &&
1789             (crtc_state->output_types & BIT(INTEL_OUTPUT_TVOUT)))
1790                 return 0;
1791
1792         if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv))
1793                 return 0xffffffff; /* full 32 bit counter */
1794         else if (INTEL_GEN(dev_priv) >= 3)
1795                 return 0xffffff; /* only 24 bits of frame count */
1796         else
1797                 return 0; /* Gen2 doesn't have a hardware frame counter */
1798 }
1799
1800 static void intel_crtc_vblank_on(const struct intel_crtc_state *crtc_state)
1801 {
1802         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
1803
1804         drm_crtc_set_max_vblank_count(&crtc->base,
1805                                       intel_crtc_max_vblank_count(crtc_state));
1806         drm_crtc_vblank_on(&crtc->base);
1807 }
1808
1809 static void intel_enable_pipe(const struct intel_crtc_state *new_crtc_state)
1810 {
1811         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->base.crtc);
1812         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1813         enum transcoder cpu_transcoder = new_crtc_state->cpu_transcoder;
1814         enum pipe pipe = crtc->pipe;
1815         i915_reg_t reg;
1816         u32 val;
1817
1818         DRM_DEBUG_KMS("enabling pipe %c\n", pipe_name(pipe));
1819
1820         assert_planes_disabled(crtc);
1821
1822         /*
1823          * A pipe without a PLL won't actually be able to drive bits from
1824          * a plane.  On ILK+ the pipe PLLs are integrated, so we don't
1825          * need the check.
1826          */
1827         if (HAS_GMCH(dev_priv)) {
1828                 if (intel_crtc_has_type(new_crtc_state, INTEL_OUTPUT_DSI))
1829                         assert_dsi_pll_enabled(dev_priv);
1830                 else
1831                         assert_pll_enabled(dev_priv, pipe);
1832         } else {
1833                 if (new_crtc_state->has_pch_encoder) {
1834                         /* if driving the PCH, we need FDI enabled */
1835                         assert_fdi_rx_pll_enabled(dev_priv,
1836                                                   intel_crtc_pch_transcoder(crtc));
1837                         assert_fdi_tx_pll_enabled(dev_priv,
1838                                                   (enum pipe) cpu_transcoder);
1839                 }
1840                 /* FIXME: assert CPU port conditions for SNB+ */
1841         }
1842
1843         trace_intel_pipe_enable(crtc);
1844
1845         reg = PIPECONF(cpu_transcoder);
1846         val = I915_READ(reg);
1847         if (val & PIPECONF_ENABLE) {
1848                 /* we keep both pipes enabled on 830 */
1849                 WARN_ON(!IS_I830(dev_priv));
1850                 return;
1851         }
1852
1853         I915_WRITE(reg, val | PIPECONF_ENABLE);
1854         POSTING_READ(reg);
1855
1856         /*
1857          * Until the pipe starts PIPEDSL reads will return a stale value,
1858          * which causes an apparent vblank timestamp jump when PIPEDSL
1859          * resets to its proper value. That also messes up the frame count
1860          * when it's derived from the timestamps. So let's wait for the
1861          * pipe to start properly before we call drm_crtc_vblank_on()
1862          */
1863         if (intel_crtc_max_vblank_count(new_crtc_state) == 0)
1864                 intel_wait_for_pipe_scanline_moving(crtc);
1865 }
1866
1867 static void intel_disable_pipe(const struct intel_crtc_state *old_crtc_state)
1868 {
1869         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc);
1870         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1871         enum transcoder cpu_transcoder = old_crtc_state->cpu_transcoder;
1872         enum pipe pipe = crtc->pipe;
1873         i915_reg_t reg;
1874         u32 val;
1875
1876         DRM_DEBUG_KMS("disabling pipe %c\n", pipe_name(pipe));
1877
1878         /*
1879          * Make sure planes won't keep trying to pump pixels to us,
1880          * or we might hang the display.
1881          */
1882         assert_planes_disabled(crtc);
1883
1884         trace_intel_pipe_disable(crtc);
1885
1886         reg = PIPECONF(cpu_transcoder);
1887         val = I915_READ(reg);
1888         if ((val & PIPECONF_ENABLE) == 0)
1889                 return;
1890
1891         /*
1892          * Double wide has implications for planes
1893          * so best keep it disabled when not needed.
1894          */
1895         if (old_crtc_state->double_wide)
1896                 val &= ~PIPECONF_DOUBLE_WIDE;
1897
1898         /* Don't disable pipe or pipe PLLs if needed */
1899         if (!IS_I830(dev_priv))
1900                 val &= ~PIPECONF_ENABLE;
1901
1902         I915_WRITE(reg, val);
1903         if ((val & PIPECONF_ENABLE) == 0)
1904                 intel_wait_for_pipe_off(old_crtc_state);
1905 }
1906
1907 static unsigned int intel_tile_size(const struct drm_i915_private *dev_priv)
1908 {
1909         return IS_GEN(dev_priv, 2) ? 2048 : 4096;
1910 }
1911
1912 static unsigned int
1913 intel_tile_width_bytes(const struct drm_framebuffer *fb, int color_plane)
1914 {
1915         struct drm_i915_private *dev_priv = to_i915(fb->dev);
1916         unsigned int cpp = fb->format->cpp[color_plane];
1917
1918         switch (fb->modifier) {
1919         case DRM_FORMAT_MOD_LINEAR:
1920                 return intel_tile_size(dev_priv);
1921         case I915_FORMAT_MOD_X_TILED:
1922                 if (IS_GEN(dev_priv, 2))
1923                         return 128;
1924                 else
1925                         return 512;
1926         case I915_FORMAT_MOD_Y_TILED_CCS:
1927                 if (color_plane == 1)
1928                         return 128;
1929                 /* fall through */
1930         case I915_FORMAT_MOD_Y_TILED:
1931                 if (IS_GEN(dev_priv, 2) || HAS_128_BYTE_Y_TILING(dev_priv))
1932                         return 128;
1933                 else
1934                         return 512;
1935         case I915_FORMAT_MOD_Yf_TILED_CCS:
1936                 if (color_plane == 1)
1937                         return 128;
1938                 /* fall through */
1939         case I915_FORMAT_MOD_Yf_TILED:
1940                 switch (cpp) {
1941                 case 1:
1942                         return 64;
1943                 case 2:
1944                 case 4:
1945                         return 128;
1946                 case 8:
1947                 case 16:
1948                         return 256;
1949                 default:
1950                         MISSING_CASE(cpp);
1951                         return cpp;
1952                 }
1953                 break;
1954         default:
1955                 MISSING_CASE(fb->modifier);
1956                 return cpp;
1957         }
1958 }
1959
1960 static unsigned int
1961 intel_tile_height(const struct drm_framebuffer *fb, int color_plane)
1962 {
1963         return intel_tile_size(to_i915(fb->dev)) /
1964                 intel_tile_width_bytes(fb, color_plane);
1965 }
1966
1967 /* Return the tile dimensions in pixel units */
1968 static void intel_tile_dims(const struct drm_framebuffer *fb, int color_plane,
1969                             unsigned int *tile_width,
1970                             unsigned int *tile_height)
1971 {
1972         unsigned int tile_width_bytes = intel_tile_width_bytes(fb, color_plane);
1973         unsigned int cpp = fb->format->cpp[color_plane];
1974
1975         *tile_width = tile_width_bytes / cpp;
1976         *tile_height = intel_tile_size(to_i915(fb->dev)) / tile_width_bytes;
1977 }
1978
1979 unsigned int
1980 intel_fb_align_height(const struct drm_framebuffer *fb,
1981                       int color_plane, unsigned int height)
1982 {
1983         unsigned int tile_height = intel_tile_height(fb, color_plane);
1984
1985         return ALIGN(height, tile_height);
1986 }
1987
1988 unsigned int intel_rotation_info_size(const struct intel_rotation_info *rot_info)
1989 {
1990         unsigned int size = 0;
1991         int i;
1992
1993         for (i = 0 ; i < ARRAY_SIZE(rot_info->plane); i++)
1994                 size += rot_info->plane[i].width * rot_info->plane[i].height;
1995
1996         return size;
1997 }
1998
1999 unsigned int intel_remapped_info_size(const struct intel_remapped_info *rem_info)
2000 {
2001         unsigned int size = 0;
2002         int i;
2003
2004         for (i = 0 ; i < ARRAY_SIZE(rem_info->plane); i++)
2005                 size += rem_info->plane[i].width * rem_info->plane[i].height;
2006
2007         return size;
2008 }
2009
2010 static void
2011 intel_fill_fb_ggtt_view(struct i915_ggtt_view *view,
2012                         const struct drm_framebuffer *fb,
2013                         unsigned int rotation)
2014 {
2015         view->type = I915_GGTT_VIEW_NORMAL;
2016         if (drm_rotation_90_or_270(rotation)) {
2017                 view->type = I915_GGTT_VIEW_ROTATED;
2018                 view->rotated = to_intel_framebuffer(fb)->rot_info;
2019         }
2020 }
2021
2022 static unsigned int intel_cursor_alignment(const struct drm_i915_private *dev_priv)
2023 {
2024         if (IS_I830(dev_priv))
2025                 return 16 * 1024;
2026         else if (IS_I85X(dev_priv))
2027                 return 256;
2028         else if (IS_I845G(dev_priv) || IS_I865G(dev_priv))
2029                 return 32;
2030         else
2031                 return 4 * 1024;
2032 }
2033
2034 static unsigned int intel_linear_alignment(const struct drm_i915_private *dev_priv)
2035 {
2036         if (INTEL_GEN(dev_priv) >= 9)
2037                 return 256 * 1024;
2038         else if (IS_I965G(dev_priv) || IS_I965GM(dev_priv) ||
2039                  IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
2040                 return 128 * 1024;
2041         else if (INTEL_GEN(dev_priv) >= 4)
2042                 return 4 * 1024;
2043         else
2044                 return 0;
2045 }
2046
2047 static unsigned int intel_surf_alignment(const struct drm_framebuffer *fb,
2048                                          int color_plane)
2049 {
2050         struct drm_i915_private *dev_priv = to_i915(fb->dev);
2051
2052         /* AUX_DIST needs only 4K alignment */
2053         if (color_plane == 1)
2054                 return 4096;
2055
2056         switch (fb->modifier) {
2057         case DRM_FORMAT_MOD_LINEAR:
2058                 return intel_linear_alignment(dev_priv);
2059         case I915_FORMAT_MOD_X_TILED:
2060                 if (INTEL_GEN(dev_priv) >= 9)
2061                         return 256 * 1024;
2062                 return 0;
2063         case I915_FORMAT_MOD_Y_TILED_CCS:
2064         case I915_FORMAT_MOD_Yf_TILED_CCS:
2065         case I915_FORMAT_MOD_Y_TILED:
2066         case I915_FORMAT_MOD_Yf_TILED:
2067                 return 1 * 1024 * 1024;
2068         default:
2069                 MISSING_CASE(fb->modifier);
2070                 return 0;
2071         }
2072 }
2073
2074 static bool intel_plane_uses_fence(const struct intel_plane_state *plane_state)
2075 {
2076         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
2077         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
2078
2079         return INTEL_GEN(dev_priv) < 4 ||
2080                 (plane->has_fbc &&
2081                  plane_state->view.type == I915_GGTT_VIEW_NORMAL);
2082 }
2083
2084 struct i915_vma *
2085 intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb,
2086                            const struct i915_ggtt_view *view,
2087                            bool uses_fence,
2088                            unsigned long *out_flags)
2089 {
2090         struct drm_device *dev = fb->dev;
2091         struct drm_i915_private *dev_priv = to_i915(dev);
2092         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
2093         intel_wakeref_t wakeref;
2094         struct i915_vma *vma;
2095         unsigned int pinctl;
2096         u32 alignment;
2097
2098         WARN_ON(!mutex_is_locked(&dev->struct_mutex));
2099
2100         alignment = intel_surf_alignment(fb, 0);
2101
2102         /* Note that the w/a also requires 64 PTE of padding following the
2103          * bo. We currently fill all unused PTE with the shadow page and so
2104          * we should always have valid PTE following the scanout preventing
2105          * the VT-d warning.
2106          */
2107         if (intel_scanout_needs_vtd_wa(dev_priv) && alignment < 256 * 1024)
2108                 alignment = 256 * 1024;
2109
2110         /*
2111          * Global gtt pte registers are special registers which actually forward
2112          * writes to a chunk of system memory. Which means that there is no risk
2113          * that the register values disappear as soon as we call
2114          * intel_runtime_pm_put(), so it is correct to wrap only the
2115          * pin/unpin/fence and not more.
2116          */
2117         wakeref = intel_runtime_pm_get(&dev_priv->runtime_pm);
2118         i915_gem_object_lock(obj);
2119
2120         atomic_inc(&dev_priv->gpu_error.pending_fb_pin);
2121
2122         pinctl = 0;
2123
2124         /* Valleyview is definitely limited to scanning out the first
2125          * 512MiB. Lets presume this behaviour was inherited from the
2126          * g4x display engine and that all earlier gen are similarly
2127          * limited. Testing suggests that it is a little more
2128          * complicated than this. For example, Cherryview appears quite
2129          * happy to scanout from anywhere within its global aperture.
2130          */
2131         if (HAS_GMCH(dev_priv))
2132                 pinctl |= PIN_MAPPABLE;
2133
2134         vma = i915_gem_object_pin_to_display_plane(obj,
2135                                                    alignment, view, pinctl);
2136         if (IS_ERR(vma))
2137                 goto err;
2138
2139         if (uses_fence && i915_vma_is_map_and_fenceable(vma)) {
2140                 int ret;
2141
2142                 /* Install a fence for tiled scan-out. Pre-i965 always needs a
2143                  * fence, whereas 965+ only requires a fence if using
2144                  * framebuffer compression.  For simplicity, we always, when
2145                  * possible, install a fence as the cost is not that onerous.
2146                  *
2147                  * If we fail to fence the tiled scanout, then either the
2148                  * modeset will reject the change (which is highly unlikely as
2149                  * the affected systems, all but one, do not have unmappable
2150                  * space) or we will not be able to enable full powersaving
2151                  * techniques (also likely not to apply due to various limits
2152                  * FBC and the like impose on the size of the buffer, which
2153                  * presumably we violated anyway with this unmappable buffer).
2154                  * Anyway, it is presumably better to stumble onwards with
2155                  * something and try to run the system in a "less than optimal"
2156                  * mode that matches the user configuration.
2157                  */
2158                 ret = i915_vma_pin_fence(vma);
2159                 if (ret != 0 && INTEL_GEN(dev_priv) < 4) {
2160                         i915_gem_object_unpin_from_display_plane(vma);
2161                         vma = ERR_PTR(ret);
2162                         goto err;
2163                 }
2164
2165                 if (ret == 0 && vma->fence)
2166                         *out_flags |= PLANE_HAS_FENCE;
2167         }
2168
2169         i915_vma_get(vma);
2170 err:
2171         atomic_dec(&dev_priv->gpu_error.pending_fb_pin);
2172
2173         i915_gem_object_unlock(obj);
2174         intel_runtime_pm_put(&dev_priv->runtime_pm, wakeref);
2175         return vma;
2176 }
2177
2178 void intel_unpin_fb_vma(struct i915_vma *vma, unsigned long flags)
2179 {
2180         lockdep_assert_held(&vma->vm->i915->drm.struct_mutex);
2181
2182         i915_gem_object_lock(vma->obj);
2183         if (flags & PLANE_HAS_FENCE)
2184                 i915_vma_unpin_fence(vma);
2185         i915_gem_object_unpin_from_display_plane(vma);
2186         i915_gem_object_unlock(vma->obj);
2187
2188         i915_vma_put(vma);
2189 }
2190
2191 static int intel_fb_pitch(const struct drm_framebuffer *fb, int color_plane,
2192                           unsigned int rotation)
2193 {
2194         if (drm_rotation_90_or_270(rotation))
2195                 return to_intel_framebuffer(fb)->rotated[color_plane].pitch;
2196         else
2197                 return fb->pitches[color_plane];
2198 }
2199
2200 /*
2201  * Convert the x/y offsets into a linear offset.
2202  * Only valid with 0/180 degree rotation, which is fine since linear
2203  * offset is only used with linear buffers on pre-hsw and tiled buffers
2204  * with gen2/3, and 90/270 degree rotations isn't supported on any of them.
2205  */
2206 u32 intel_fb_xy_to_linear(int x, int y,
2207                           const struct intel_plane_state *state,
2208                           int color_plane)
2209 {
2210         const struct drm_framebuffer *fb = state->base.fb;
2211         unsigned int cpp = fb->format->cpp[color_plane];
2212         unsigned int pitch = state->color_plane[color_plane].stride;
2213
2214         return y * pitch + x * cpp;
2215 }
2216
2217 /*
2218  * Add the x/y offsets derived from fb->offsets[] to the user
2219  * specified plane src x/y offsets. The resulting x/y offsets
2220  * specify the start of scanout from the beginning of the gtt mapping.
2221  */
2222 void intel_add_fb_offsets(int *x, int *y,
2223                           const struct intel_plane_state *state,
2224                           int color_plane)
2225
2226 {
2227         *x += state->color_plane[color_plane].x;
2228         *y += state->color_plane[color_plane].y;
2229 }
2230
2231 static u32 intel_adjust_tile_offset(int *x, int *y,
2232                                     unsigned int tile_width,
2233                                     unsigned int tile_height,
2234                                     unsigned int tile_size,
2235                                     unsigned int pitch_tiles,
2236                                     u32 old_offset,
2237                                     u32 new_offset)
2238 {
2239         unsigned int pitch_pixels = pitch_tiles * tile_width;
2240         unsigned int tiles;
2241
2242         WARN_ON(old_offset & (tile_size - 1));
2243         WARN_ON(new_offset & (tile_size - 1));
2244         WARN_ON(new_offset > old_offset);
2245
2246         tiles = (old_offset - new_offset) / tile_size;
2247
2248         *y += tiles / pitch_tiles * tile_height;
2249         *x += tiles % pitch_tiles * tile_width;
2250
2251         /* minimize x in case it got needlessly big */
2252         *y += *x / pitch_pixels * tile_height;
2253         *x %= pitch_pixels;
2254
2255         return new_offset;
2256 }
2257
2258 static bool is_surface_linear(u64 modifier, int color_plane)
2259 {
2260         return modifier == DRM_FORMAT_MOD_LINEAR;
2261 }
2262
2263 static u32 intel_adjust_aligned_offset(int *x, int *y,
2264                                        const struct drm_framebuffer *fb,
2265                                        int color_plane,
2266                                        unsigned int rotation,
2267                                        unsigned int pitch,
2268                                        u32 old_offset, u32 new_offset)
2269 {
2270         struct drm_i915_private *dev_priv = to_i915(fb->dev);
2271         unsigned int cpp = fb->format->cpp[color_plane];
2272
2273         WARN_ON(new_offset > old_offset);
2274
2275         if (!is_surface_linear(fb->modifier, color_plane)) {
2276                 unsigned int tile_size, tile_width, tile_height;
2277                 unsigned int pitch_tiles;
2278
2279                 tile_size = intel_tile_size(dev_priv);
2280                 intel_tile_dims(fb, color_plane, &tile_width, &tile_height);
2281
2282                 if (drm_rotation_90_or_270(rotation)) {
2283                         pitch_tiles = pitch / tile_height;
2284                         swap(tile_width, tile_height);
2285                 } else {
2286                         pitch_tiles = pitch / (tile_width * cpp);
2287                 }
2288
2289                 intel_adjust_tile_offset(x, y, tile_width, tile_height,
2290                                          tile_size, pitch_tiles,
2291                                          old_offset, new_offset);
2292         } else {
2293                 old_offset += *y * pitch + *x * cpp;
2294
2295                 *y = (old_offset - new_offset) / pitch;
2296                 *x = ((old_offset - new_offset) - *y * pitch) / cpp;
2297         }
2298
2299         return new_offset;
2300 }
2301
2302 /*
2303  * Adjust the tile offset by moving the difference into
2304  * the x/y offsets.
2305  */
2306 static u32 intel_plane_adjust_aligned_offset(int *x, int *y,
2307                                              const struct intel_plane_state *state,
2308                                              int color_plane,
2309                                              u32 old_offset, u32 new_offset)
2310 {
2311         return intel_adjust_aligned_offset(x, y, state->base.fb, color_plane,
2312                                            state->base.rotation,
2313                                            state->color_plane[color_plane].stride,
2314                                            old_offset, new_offset);
2315 }
2316
2317 /*
2318  * Computes the aligned offset to the base tile and adjusts
2319  * x, y. bytes per pixel is assumed to be a power-of-two.
2320  *
2321  * In the 90/270 rotated case, x and y are assumed
2322  * to be already rotated to match the rotated GTT view, and
2323  * pitch is the tile_height aligned framebuffer height.
2324  *
2325  * This function is used when computing the derived information
2326  * under intel_framebuffer, so using any of that information
2327  * here is not allowed. Anything under drm_framebuffer can be
2328  * used. This is why the user has to pass in the pitch since it
2329  * is specified in the rotated orientation.
2330  */
2331 static u32 intel_compute_aligned_offset(struct drm_i915_private *dev_priv,
2332                                         int *x, int *y,
2333                                         const struct drm_framebuffer *fb,
2334                                         int color_plane,
2335                                         unsigned int pitch,
2336                                         unsigned int rotation,
2337                                         u32 alignment)
2338 {
2339         unsigned int cpp = fb->format->cpp[color_plane];
2340         u32 offset, offset_aligned;
2341
2342         if (alignment)
2343                 alignment--;
2344
2345         if (!is_surface_linear(fb->modifier, color_plane)) {
2346                 unsigned int tile_size, tile_width, tile_height;
2347                 unsigned int tile_rows, tiles, pitch_tiles;
2348
2349                 tile_size = intel_tile_size(dev_priv);
2350                 intel_tile_dims(fb, color_plane, &tile_width, &tile_height);
2351
2352                 if (drm_rotation_90_or_270(rotation)) {
2353                         pitch_tiles = pitch / tile_height;
2354                         swap(tile_width, tile_height);
2355                 } else {
2356                         pitch_tiles = pitch / (tile_width * cpp);
2357                 }
2358
2359                 tile_rows = *y / tile_height;
2360                 *y %= tile_height;
2361
2362                 tiles = *x / tile_width;
2363                 *x %= tile_width;
2364
2365                 offset = (tile_rows * pitch_tiles + tiles) * tile_size;
2366                 offset_aligned = offset & ~alignment;
2367
2368                 intel_adjust_tile_offset(x, y, tile_width, tile_height,
2369                                          tile_size, pitch_tiles,
2370                                          offset, offset_aligned);
2371         } else {
2372                 offset = *y * pitch + *x * cpp;
2373                 offset_aligned = offset & ~alignment;
2374
2375                 *y = (offset & alignment) / pitch;
2376                 *x = ((offset & alignment) - *y * pitch) / cpp;
2377         }
2378
2379         return offset_aligned;
2380 }
2381
2382 static u32 intel_plane_compute_aligned_offset(int *x, int *y,
2383                                               const struct intel_plane_state *state,
2384                                               int color_plane)
2385 {
2386         struct intel_plane *intel_plane = to_intel_plane(state->base.plane);
2387         struct drm_i915_private *dev_priv = to_i915(intel_plane->base.dev);
2388         const struct drm_framebuffer *fb = state->base.fb;
2389         unsigned int rotation = state->base.rotation;
2390         int pitch = state->color_plane[color_plane].stride;
2391         u32 alignment;
2392
2393         if (intel_plane->id == PLANE_CURSOR)
2394                 alignment = intel_cursor_alignment(dev_priv);
2395         else
2396                 alignment = intel_surf_alignment(fb, color_plane);
2397
2398         return intel_compute_aligned_offset(dev_priv, x, y, fb, color_plane,
2399                                             pitch, rotation, alignment);
2400 }
2401
2402 /* Convert the fb->offset[] into x/y offsets */
2403 static int intel_fb_offset_to_xy(int *x, int *y,
2404                                  const struct drm_framebuffer *fb,
2405                                  int color_plane)
2406 {
2407         struct drm_i915_private *dev_priv = to_i915(fb->dev);
2408         unsigned int height;
2409
2410         if (fb->modifier != DRM_FORMAT_MOD_LINEAR &&
2411             fb->offsets[color_plane] % intel_tile_size(dev_priv)) {
2412                 DRM_DEBUG_KMS("Misaligned offset 0x%08x for color plane %d\n",
2413                               fb->offsets[color_plane], color_plane);
2414                 return -EINVAL;
2415         }
2416
2417         height = drm_framebuffer_plane_height(fb->height, fb, color_plane);
2418         height = ALIGN(height, intel_tile_height(fb, color_plane));
2419
2420         /* Catch potential overflows early */
2421         if (add_overflows_t(u32, mul_u32_u32(height, fb->pitches[color_plane]),
2422                             fb->offsets[color_plane])) {
2423                 DRM_DEBUG_KMS("Bad offset 0x%08x or pitch %d for color plane %d\n",
2424                               fb->offsets[color_plane], fb->pitches[color_plane],
2425                               color_plane);
2426                 return -ERANGE;
2427         }
2428
2429         *x = 0;
2430         *y = 0;
2431
2432         intel_adjust_aligned_offset(x, y,
2433                                     fb, color_plane, DRM_MODE_ROTATE_0,
2434                                     fb->pitches[color_plane],
2435                                     fb->offsets[color_plane], 0);
2436
2437         return 0;
2438 }
2439
2440 static unsigned int intel_fb_modifier_to_tiling(u64 fb_modifier)
2441 {
2442         switch (fb_modifier) {
2443         case I915_FORMAT_MOD_X_TILED:
2444                 return I915_TILING_X;
2445         case I915_FORMAT_MOD_Y_TILED:
2446         case I915_FORMAT_MOD_Y_TILED_CCS:
2447                 return I915_TILING_Y;
2448         default:
2449                 return I915_TILING_NONE;
2450         }
2451 }
2452
2453 /*
2454  * From the Sky Lake PRM:
2455  * "The Color Control Surface (CCS) contains the compression status of
2456  *  the cache-line pairs. The compression state of the cache-line pair
2457  *  is specified by 2 bits in the CCS. Each CCS cache-line represents
2458  *  an area on the main surface of 16 x16 sets of 128 byte Y-tiled
2459  *  cache-line-pairs. CCS is always Y tiled."
2460  *
2461  * Since cache line pairs refers to horizontally adjacent cache lines,
2462  * each cache line in the CCS corresponds to an area of 32x16 cache
2463  * lines on the main surface. Since each pixel is 4 bytes, this gives
2464  * us a ratio of one byte in the CCS for each 8x16 pixels in the
2465  * main surface.
2466  */
2467 static const struct drm_format_info ccs_formats[] = {
2468         { .format = DRM_FORMAT_XRGB8888, .depth = 24, .num_planes = 2,
2469           .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, },
2470         { .format = DRM_FORMAT_XBGR8888, .depth = 24, .num_planes = 2,
2471           .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, },
2472         { .format = DRM_FORMAT_ARGB8888, .depth = 32, .num_planes = 2,
2473           .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, .has_alpha = true, },
2474         { .format = DRM_FORMAT_ABGR8888, .depth = 32, .num_planes = 2,
2475           .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, .has_alpha = true, },
2476 };
2477
2478 static const struct drm_format_info *
2479 lookup_format_info(const struct drm_format_info formats[],
2480                    int num_formats, u32 format)
2481 {
2482         int i;
2483
2484         for (i = 0; i < num_formats; i++) {
2485                 if (formats[i].format == format)
2486                         return &formats[i];
2487         }
2488
2489         return NULL;
2490 }
2491
2492 static const struct drm_format_info *
2493 intel_get_format_info(const struct drm_mode_fb_cmd2 *cmd)
2494 {
2495         switch (cmd->modifier[0]) {
2496         case I915_FORMAT_MOD_Y_TILED_CCS:
2497         case I915_FORMAT_MOD_Yf_TILED_CCS:
2498                 return lookup_format_info(ccs_formats,
2499                                           ARRAY_SIZE(ccs_formats),
2500                                           cmd->pixel_format);
2501         default:
2502                 return NULL;
2503         }
2504 }
2505
2506 bool is_ccs_modifier(u64 modifier)
2507 {
2508         return modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
2509                modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
2510 }
2511
2512 u32 intel_plane_fb_max_stride(struct drm_i915_private *dev_priv,
2513                               u32 pixel_format, u64 modifier)
2514 {
2515         struct intel_crtc *crtc;
2516         struct intel_plane *plane;
2517
2518         /*
2519          * We assume the primary plane for pipe A has
2520          * the highest stride limits of them all.
2521          */
2522         crtc = intel_get_crtc_for_pipe(dev_priv, PIPE_A);
2523         plane = to_intel_plane(crtc->base.primary);
2524
2525         return plane->max_stride(plane, pixel_format, modifier,
2526                                  DRM_MODE_ROTATE_0);
2527 }
2528
2529 static
2530 u32 intel_fb_max_stride(struct drm_i915_private *dev_priv,
2531                         u32 pixel_format, u64 modifier)
2532 {
2533         /*
2534          * Arbitrary limit for gen4+ chosen to match the
2535          * render engine max stride.
2536          *
2537          * The new CCS hash mode makes remapping impossible
2538          */
2539         if (!is_ccs_modifier(modifier)) {
2540                 if (INTEL_GEN(dev_priv) >= 7)
2541                         return 256*1024;
2542                 else if (INTEL_GEN(dev_priv) >= 4)
2543                         return 128*1024;
2544         }
2545
2546         return intel_plane_fb_max_stride(dev_priv, pixel_format, modifier);
2547 }
2548
2549 static u32
2550 intel_fb_stride_alignment(const struct drm_framebuffer *fb, int color_plane)
2551 {
2552         struct drm_i915_private *dev_priv = to_i915(fb->dev);
2553
2554         if (fb->modifier == DRM_FORMAT_MOD_LINEAR) {
2555                 u32 max_stride = intel_plane_fb_max_stride(dev_priv,
2556                                                            fb->format->format,
2557                                                            fb->modifier);
2558
2559                 /*
2560                  * To make remapping with linear generally feasible
2561                  * we need the stride to be page aligned.
2562                  */
2563                 if (fb->pitches[color_plane] > max_stride)
2564                         return intel_tile_size(dev_priv);
2565                 else
2566                         return 64;
2567         } else {
2568                 return intel_tile_width_bytes(fb, color_plane);
2569         }
2570 }
2571
2572 bool intel_plane_can_remap(const struct intel_plane_state *plane_state)
2573 {
2574         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
2575         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
2576         const struct drm_framebuffer *fb = plane_state->base.fb;
2577         int i;
2578
2579         /* We don't want to deal with remapping with cursors */
2580         if (plane->id == PLANE_CURSOR)
2581                 return false;
2582
2583         /*
2584          * The display engine limits already match/exceed the
2585          * render engine limits, so not much point in remapping.
2586          * Would also need to deal with the fence POT alignment
2587          * and gen2 2KiB GTT tile size.
2588          */
2589         if (INTEL_GEN(dev_priv) < 4)
2590                 return false;
2591
2592         /*
2593          * The new CCS hash mode isn't compatible with remapping as
2594          * the virtual address of the pages affects the compressed data.
2595          */
2596         if (is_ccs_modifier(fb->modifier))
2597                 return false;
2598
2599         /* Linear needs a page aligned stride for remapping */
2600         if (fb->modifier == DRM_FORMAT_MOD_LINEAR) {
2601                 unsigned int alignment = intel_tile_size(dev_priv) - 1;
2602
2603                 for (i = 0; i < fb->format->num_planes; i++) {
2604                         if (fb->pitches[i] & alignment)
2605                                 return false;
2606                 }
2607         }
2608
2609         return true;
2610 }
2611
2612 static bool intel_plane_needs_remap(const struct intel_plane_state *plane_state)
2613 {
2614         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
2615         const struct drm_framebuffer *fb = plane_state->base.fb;
2616         unsigned int rotation = plane_state->base.rotation;
2617         u32 stride, max_stride;
2618
2619         /*
2620          * No remapping for invisible planes since we don't have
2621          * an actual source viewport to remap.
2622          */
2623         if (!plane_state->base.visible)
2624                 return false;
2625
2626         if (!intel_plane_can_remap(plane_state))
2627                 return false;
2628
2629         /*
2630          * FIXME: aux plane limits on gen9+ are
2631          * unclear in Bspec, for now no checking.
2632          */
2633         stride = intel_fb_pitch(fb, 0, rotation);
2634         max_stride = plane->max_stride(plane, fb->format->format,
2635                                        fb->modifier, rotation);
2636
2637         return stride > max_stride;
2638 }
2639
2640 static int
2641 intel_fill_fb_info(struct drm_i915_private *dev_priv,
2642                    struct drm_framebuffer *fb)
2643 {
2644         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
2645         struct intel_rotation_info *rot_info = &intel_fb->rot_info;
2646         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
2647         u32 gtt_offset_rotated = 0;
2648         unsigned int max_size = 0;
2649         int i, num_planes = fb->format->num_planes;
2650         unsigned int tile_size = intel_tile_size(dev_priv);
2651
2652         for (i = 0; i < num_planes; i++) {
2653                 unsigned int width, height;
2654                 unsigned int cpp, size;
2655                 u32 offset;
2656                 int x, y;
2657                 int ret;
2658
2659                 cpp = fb->format->cpp[i];
2660                 width = drm_framebuffer_plane_width(fb->width, fb, i);
2661                 height = drm_framebuffer_plane_height(fb->height, fb, i);
2662
2663                 ret = intel_fb_offset_to_xy(&x, &y, fb, i);
2664                 if (ret) {
2665                         DRM_DEBUG_KMS("bad fb plane %d offset: 0x%x\n",
2666                                       i, fb->offsets[i]);
2667                         return ret;
2668                 }
2669
2670                 if (is_ccs_modifier(fb->modifier) && i == 1) {
2671                         int hsub = fb->format->hsub;
2672                         int vsub = fb->format->vsub;
2673                         int tile_width, tile_height;
2674                         int main_x, main_y;
2675                         int ccs_x, ccs_y;
2676
2677                         intel_tile_dims(fb, i, &tile_width, &tile_height);
2678                         tile_width *= hsub;
2679                         tile_height *= vsub;
2680
2681                         ccs_x = (x * hsub) % tile_width;
2682                         ccs_y = (y * vsub) % tile_height;
2683                         main_x = intel_fb->normal[0].x % tile_width;
2684                         main_y = intel_fb->normal[0].y % tile_height;
2685
2686                         /*
2687                          * CCS doesn't have its own x/y offset register, so the intra CCS tile
2688                          * x/y offsets must match between CCS and the main surface.
2689                          */
2690                         if (main_x != ccs_x || main_y != ccs_y) {
2691                                 DRM_DEBUG_KMS("Bad CCS x/y (main %d,%d ccs %d,%d) full (main %d,%d ccs %d,%d)\n",
2692                                               main_x, main_y,
2693                                               ccs_x, ccs_y,
2694                                               intel_fb->normal[0].x,
2695                                               intel_fb->normal[0].y,
2696                                               x, y);
2697                                 return -EINVAL;
2698                         }
2699                 }
2700
2701                 /*
2702                  * The fence (if used) is aligned to the start of the object
2703                  * so having the framebuffer wrap around across the edge of the
2704                  * fenced region doesn't really work. We have no API to configure
2705                  * the fence start offset within the object (nor could we probably
2706                  * on gen2/3). So it's just easier if we just require that the
2707                  * fb layout agrees with the fence layout. We already check that the
2708                  * fb stride matches the fence stride elsewhere.
2709                  */
2710                 if (i == 0 && i915_gem_object_is_tiled(obj) &&
2711                     (x + width) * cpp > fb->pitches[i]) {
2712                         DRM_DEBUG_KMS("bad fb plane %d offset: 0x%x\n",
2713                                       i, fb->offsets[i]);
2714                         return -EINVAL;
2715                 }
2716
2717                 /*
2718                  * First pixel of the framebuffer from
2719                  * the start of the normal gtt mapping.
2720                  */
2721                 intel_fb->normal[i].x = x;
2722                 intel_fb->normal[i].y = y;
2723
2724                 offset = intel_compute_aligned_offset(dev_priv, &x, &y, fb, i,
2725                                                       fb->pitches[i],
2726                                                       DRM_MODE_ROTATE_0,
2727                                                       tile_size);
2728                 offset /= tile_size;
2729
2730                 if (!is_surface_linear(fb->modifier, i)) {
2731                         unsigned int tile_width, tile_height;
2732                         unsigned int pitch_tiles;
2733                         struct drm_rect r;
2734
2735                         intel_tile_dims(fb, i, &tile_width, &tile_height);
2736
2737                         rot_info->plane[i].offset = offset;
2738                         rot_info->plane[i].stride = DIV_ROUND_UP(fb->pitches[i], tile_width * cpp);
2739                         rot_info->plane[i].width = DIV_ROUND_UP(x + width, tile_width);
2740                         rot_info->plane[i].height = DIV_ROUND_UP(y + height, tile_height);
2741
2742                         intel_fb->rotated[i].pitch =
2743                                 rot_info->plane[i].height * tile_height;
2744
2745                         /* how many tiles does this plane need */
2746                         size = rot_info->plane[i].stride * rot_info->plane[i].height;
2747                         /*
2748                          * If the plane isn't horizontally tile aligned,
2749                          * we need one more tile.
2750                          */
2751                         if (x != 0)
2752                                 size++;
2753
2754                         /* rotate the x/y offsets to match the GTT view */
2755                         r.x1 = x;
2756                         r.y1 = y;
2757                         r.x2 = x + width;
2758                         r.y2 = y + height;
2759                         drm_rect_rotate(&r,
2760                                         rot_info->plane[i].width * tile_width,
2761                                         rot_info->plane[i].height * tile_height,
2762                                         DRM_MODE_ROTATE_270);
2763                         x = r.x1;
2764                         y = r.y1;
2765
2766                         /* rotate the tile dimensions to match the GTT view */
2767                         pitch_tiles = intel_fb->rotated[i].pitch / tile_height;
2768                         swap(tile_width, tile_height);
2769
2770                         /*
2771                          * We only keep the x/y offsets, so push all of the
2772                          * gtt offset into the x/y offsets.
2773                          */
2774                         intel_adjust_tile_offset(&x, &y,
2775                                                  tile_width, tile_height,
2776                                                  tile_size, pitch_tiles,
2777                                                  gtt_offset_rotated * tile_size, 0);
2778
2779                         gtt_offset_rotated += rot_info->plane[i].width * rot_info->plane[i].height;
2780
2781                         /*
2782                          * First pixel of the framebuffer from
2783                          * the start of the rotated gtt mapping.
2784                          */
2785                         intel_fb->rotated[i].x = x;
2786                         intel_fb->rotated[i].y = y;
2787                 } else {
2788                         size = DIV_ROUND_UP((y + height) * fb->pitches[i] +
2789                                             x * cpp, tile_size);
2790                 }
2791
2792                 /* how many tiles in total needed in the bo */
2793                 max_size = max(max_size, offset + size);
2794         }
2795
2796         if (mul_u32_u32(max_size, tile_size) > obj->base.size) {
2797                 DRM_DEBUG_KMS("fb too big for bo (need %llu bytes, have %zu bytes)\n",
2798                               mul_u32_u32(max_size, tile_size), obj->base.size);
2799                 return -EINVAL;
2800         }
2801
2802         return 0;
2803 }
2804
2805 static void
2806 intel_plane_remap_gtt(struct intel_plane_state *plane_state)
2807 {
2808         struct drm_i915_private *dev_priv =
2809                 to_i915(plane_state->base.plane->dev);
2810         struct drm_framebuffer *fb = plane_state->base.fb;
2811         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
2812         struct intel_rotation_info *info = &plane_state->view.rotated;
2813         unsigned int rotation = plane_state->base.rotation;
2814         int i, num_planes = fb->format->num_planes;
2815         unsigned int tile_size = intel_tile_size(dev_priv);
2816         unsigned int src_x, src_y;
2817         unsigned int src_w, src_h;
2818         u32 gtt_offset = 0;
2819
2820         memset(&plane_state->view, 0, sizeof(plane_state->view));
2821         plane_state->view.type = drm_rotation_90_or_270(rotation) ?
2822                 I915_GGTT_VIEW_ROTATED : I915_GGTT_VIEW_REMAPPED;
2823
2824         src_x = plane_state->base.src.x1 >> 16;
2825         src_y = plane_state->base.src.y1 >> 16;
2826         src_w = drm_rect_width(&plane_state->base.src) >> 16;
2827         src_h = drm_rect_height(&plane_state->base.src) >> 16;
2828
2829         WARN_ON(is_ccs_modifier(fb->modifier));
2830
2831         /* Make src coordinates relative to the viewport */
2832         drm_rect_translate(&plane_state->base.src,
2833                            -(src_x << 16), -(src_y << 16));
2834
2835         /* Rotate src coordinates to match rotated GTT view */
2836         if (drm_rotation_90_or_270(rotation))
2837                 drm_rect_rotate(&plane_state->base.src,
2838                                 src_w << 16, src_h << 16,
2839                                 DRM_MODE_ROTATE_270);
2840
2841         for (i = 0; i < num_planes; i++) {
2842                 unsigned int hsub = i ? fb->format->hsub : 1;
2843                 unsigned int vsub = i ? fb->format->vsub : 1;
2844                 unsigned int cpp = fb->format->cpp[i];
2845                 unsigned int tile_width, tile_height;
2846                 unsigned int width, height;
2847                 unsigned int pitch_tiles;
2848                 unsigned int x, y;
2849                 u32 offset;
2850
2851                 intel_tile_dims(fb, i, &tile_width, &tile_height);
2852
2853                 x = src_x / hsub;
2854                 y = src_y / vsub;
2855                 width = src_w / hsub;
2856                 height = src_h / vsub;
2857
2858                 /*
2859                  * First pixel of the src viewport from the
2860                  * start of the normal gtt mapping.
2861                  */
2862                 x += intel_fb->normal[i].x;
2863                 y += intel_fb->normal[i].y;
2864
2865                 offset = intel_compute_aligned_offset(dev_priv, &x, &y,
2866                                                       fb, i, fb->pitches[i],
2867                                                       DRM_MODE_ROTATE_0, tile_size);
2868                 offset /= tile_size;
2869
2870                 info->plane[i].offset = offset;
2871                 info->plane[i].stride = DIV_ROUND_UP(fb->pitches[i],
2872                                                      tile_width * cpp);
2873                 info->plane[i].width = DIV_ROUND_UP(x + width, tile_width);
2874                 info->plane[i].height = DIV_ROUND_UP(y + height, tile_height);
2875
2876                 if (drm_rotation_90_or_270(rotation)) {
2877                         struct drm_rect r;
2878
2879                         /* rotate the x/y offsets to match the GTT view */
2880                         r.x1 = x;
2881                         r.y1 = y;
2882                         r.x2 = x + width;
2883                         r.y2 = y + height;
2884                         drm_rect_rotate(&r,
2885                                         info->plane[i].width * tile_width,
2886                                         info->plane[i].height * tile_height,
2887                                         DRM_MODE_ROTATE_270);
2888                         x = r.x1;
2889                         y = r.y1;
2890
2891                         pitch_tiles = info->plane[i].height;
2892                         plane_state->color_plane[i].stride = pitch_tiles * tile_height;
2893
2894                         /* rotate the tile dimensions to match the GTT view */
2895                         swap(tile_width, tile_height);
2896                 } else {
2897                         pitch_tiles = info->plane[i].width;
2898                         plane_state->color_plane[i].stride = pitch_tiles * tile_width * cpp;
2899                 }
2900
2901                 /*
2902                  * We only keep the x/y offsets, so push all of the
2903                  * gtt offset into the x/y offsets.
2904                  */
2905                 intel_adjust_tile_offset(&x, &y,
2906                                          tile_width, tile_height,
2907                                          tile_size, pitch_tiles,
2908                                          gtt_offset * tile_size, 0);
2909
2910                 gtt_offset += info->plane[i].width * info->plane[i].height;
2911
2912                 plane_state->color_plane[i].offset = 0;
2913                 plane_state->color_plane[i].x = x;
2914                 plane_state->color_plane[i].y = y;
2915         }
2916 }
2917
2918 static int
2919 intel_plane_compute_gtt(struct intel_plane_state *plane_state)
2920 {
2921         const struct intel_framebuffer *fb =
2922                 to_intel_framebuffer(plane_state->base.fb);
2923         unsigned int rotation = plane_state->base.rotation;
2924         int i, num_planes;
2925
2926         if (!fb)
2927                 return 0;
2928
2929         num_planes = fb->base.format->num_planes;
2930
2931         if (intel_plane_needs_remap(plane_state)) {
2932                 intel_plane_remap_gtt(plane_state);
2933
2934                 /*
2935                  * Sometimes even remapping can't overcome
2936                  * the stride limitations :( Can happen with
2937                  * big plane sizes and suitably misaligned
2938                  * offsets.
2939                  */
2940                 return intel_plane_check_stride(plane_state);
2941         }
2942
2943         intel_fill_fb_ggtt_view(&plane_state->view, &fb->base, rotation);
2944
2945         for (i = 0; i < num_planes; i++) {
2946                 plane_state->color_plane[i].stride = intel_fb_pitch(&fb->base, i, rotation);
2947                 plane_state->color_plane[i].offset = 0;
2948
2949                 if (drm_rotation_90_or_270(rotation)) {
2950                         plane_state->color_plane[i].x = fb->rotated[i].x;