drivers/gpu/drm/i915/display/intel_psr.c

   1 /*
   2  * Copyright © 2014 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  21  * DEALINGS IN THE SOFTWARE.
  22  */
  23
  24 #include <drm/drm_atomic_helper.h>
  25 #include <drm/drm_damage_helper.h>
  26 #include <drm/drm_debugfs.h>
  27
  28 #include "i915_drv.h"
  29 #include "i915_reg.h"
  30 #include "intel_atomic.h"
  31 #include "intel_crtc.h"
  32 #include "intel_ddi.h"
  33 #include "intel_de.h"
  34 #include "intel_display_types.h"
  35 #include "intel_dp.h"
  36 #include "intel_dp_aux.h"
  37 #include "intel_frontbuffer.h"
  38 #include "intel_hdmi.h"
  39 #include "intel_psr.h"
  40 #include "intel_psr_regs.h"
  41 #include "intel_snps_phy.h"
  42 #include "skl_universal_plane.h"
  43
  44 /**
  45  * DOC: Panel Self Refresh (PSR/SRD)
  46  *
  47  * Since Haswell Display controller supports Panel Self-Refresh on display
  48  * panels witch have a remote frame buffer (RFB) implemented according to PSR
  49  * spec in eDP1.3. PSR feature allows the display to go to lower standby states
  50  * when system is idle but display is on as it eliminates display refresh
  51  * request to DDR memory completely as long as the frame buffer for that
  52  * display is unchanged.
  53  *
  54  * Panel Self Refresh must be supported by both Hardware (source) and
  55  * Panel (sink).
  56  *
  57  * PSR saves power by caching the framebuffer in the panel RFB, which allows us
  58  * to power down the link and memory controller. For DSI panels the same idea
  59  * is called "manual mode".
  60  *
  61  * The implementation uses the hardware-based PSR support which automatically
  62  * enters/exits self-refresh mode. The hardware takes care of sending the
  63  * required DP aux message and could even retrain the link (that part isn't
  64  * enabled yet though). The hardware also keeps track of any frontbuffer
  65  * changes to know when to exit self-refresh mode again. Unfortunately that
  66  * part doesn't work too well, hence why the i915 PSR support uses the
  67  * software frontbuffer tracking to make sure it doesn't miss a screen
  68  * update. For this integration intel_psr_invalidate() and intel_psr_flush()
  69  * get called by the frontbuffer tracking code. Note that because of locking
  70  * issues the self-refresh re-enable code is done from a work queue, which
  71  * must be correctly synchronized/cancelled when shutting down the pipe."
  72  *
  73  * DC3CO (DC3 clock off)
  74  *
  75  * On top of PSR2, GEN12 adds a intermediate power savings state that turns
  76  * clock off automatically during PSR2 idle state.
  77  * The smaller overhead of DC3co entry/exit vs. the overhead of PSR2 deep sleep
  78  * entry/exit allows the HW to enter a low-power state even when page flipping
  79  * periodically (for instance a 30fps video playback scenario).
  80  *
  81  * Every time a flips occurs PSR2 will get out of deep sleep state(if it was),
  82  * so DC3CO is enabled and tgl_dc3co_disable_work is schedule to run after 6
  83  * frames, if no other flip occurs and the function above is executed, DC3CO is
  84  * disabled and PSR2 is configured to enter deep sleep, resetting again in case
  85  * of another flip.
  86  * Front buffer modifications do not trigger DC3CO activation on purpose as it
  87  * would bring a lot of complexity and most of the moderns systems will only
  88  * use page flips.
  89  */
  90
  91 /*
  92  * Description of PSR mask bits:
  93  *
  94  * EDP_PSR_DEBUG[16]/EDP_PSR_DEBUG_MASK_DISP_REG_WRITE (hsw-skl):
  95  *
  96  *  When unmasked (nearly) all display register writes (eg. even
  97  *  SWF) trigger a PSR exit. Some registers are excluded from this
  98  *  and they have a more specific mask (described below). On icl+
  99  *  this bit no longer exists and is effectively always set.
 100  *
 101  * PIPE_MISC[21]/PIPE_MISC_PSR_MASK_PIPE_REG_WRITE (skl+):
 102  *
 103  *  When unmasked (nearly) all pipe/plane register writes
 104  *  trigger a PSR exit. Some plane registers are excluded from this
 105  *  and they have a more specific mask (described below).
 106  *
 107  * CHICKEN_PIPESL_1[11]/SKL_PSR_MASK_PLANE_FLIP (skl+):
 108  * PIPE_MISC[23]/PIPE_MISC_PSR_MASK_PRIMARY_FLIP (bdw):
 109  * EDP_PSR_DEBUG[23]/EDP_PSR_DEBUG_MASK_PRIMARY_FLIP (hsw):
 110  *
 111  *  When unmasked PRI_SURF/PLANE_SURF writes trigger a PSR exit.
 112  *  SPR_SURF/CURBASE are not included in this and instead are
 113  *  controlled by PIPE_MISC_PSR_MASK_PIPE_REG_WRITE (skl+) or
 114  *  EDP_PSR_DEBUG_MASK_DISP_REG_WRITE (hsw/bdw).
 115  *
 116  * PIPE_MISC[22]/PIPE_MISC_PSR_MASK_SPRITE_ENABLE (bdw):
 117  * EDP_PSR_DEBUG[21]/EDP_PSR_DEBUG_MASK_SPRITE_ENABLE (hsw):
 118  *
 119  *  When unmasked PSR is blocked as long as the sprite
 120  *  plane is enabled. skl+ with their universal planes no
 121  *  longer have a mask bit like this, and no plane being
 122  *  enabledb blocks PSR.
 123  *
 124  * PIPE_MISC[21]/PIPE_MISC_PSR_MASK_CURSOR_MOVE (bdw):
 125  * EDP_PSR_DEBUG[20]/EDP_PSR_DEBUG_MASK_CURSOR_MOVE (hsw):
 126  *
 127  *  When umasked CURPOS writes trigger a PSR exit. On skl+
 128  *  this doesn't exit but CURPOS is included in the
 129  *  PIPE_MISC_PSR_MASK_PIPE_REG_WRITE mask.
 130  *
 131  * PIPE_MISC[20]/PIPE_MISC_PSR_MASK_VBLANK_VSYNC_INT (bdw+):
 132  * EDP_PSR_DEBUG[19]/EDP_PSR_DEBUG_MASK_VBLANK_VSYNC_INT (hsw):
 133  *
 134  *  When unmasked PSR is blocked as long as vblank and/or vsync
 135  *  interrupt is unmasked in IMR *and* enabled in IER.
 136  *
 137  * CHICKEN_TRANS[30]/SKL_UNMASK_VBL_TO_PIPE_IN_SRD (skl+):
 138  * CHICKEN_PAR1_1[15]/HSW_MASK_VBL_TO_PIPE_IN_SRD (hsw/bdw):
 139  *
 140  *  Selectcs whether PSR exit generates an extra vblank before
 141  *  the first frame is transmitted. Also note the opposite polarity
 142  *  if the bit on hsw/bdw vs. skl+ (masked==generate the extra vblank,
 143  *  unmasked==do not generate the extra vblank).
 144  *
 145  *  With DC states enabled the extra vblank happens after link training,
 146  *  with DC states disabled it happens immediately upuon PSR exit trigger.
 147  *  No idea as of now why there is a difference. HSW/BDW (which don't
 148  *  even have DMC) always generate it after link training. Go figure.
 149  *
 150  *  Unfortunately CHICKEN_TRANS itself seems to be double buffered
 151  *  and thus won't latch until the first vblank. So with DC states
 152  *  enabled the register effctively uses the reset value during DC5
 153  *  exit+PSR exit sequence, and thus the bit does nothing until
 154  *  latched by the vblank that it was trying to prevent from being
 155  *  generated in the first place. So we should probably call this
 156  *  one a chicken/egg bit instead on skl+.
 157  *
 158  *  In standby mode (as opposed to link-off) this makes no difference
 159  *  as the timing generator keeps running the whole time generating
 160  *  normal periodic vblanks.
 161  *
 162  *  WaPsrDPAMaskVBlankInSRD asks us to set the bit on hsw/bdw,
 163  *  and doing so makes the behaviour match the skl+ reset value.
 164  *
 165  * CHICKEN_PIPESL_1[0]/BDW_UNMASK_VBL_TO_REGS_IN_SRD (bdw):
 166  * CHICKEN_PIPESL_1[15]/HSW_UNMASK_VBL_TO_REGS_IN_SRD (hsw):
 167  *
 168  *  On BDW without this bit is no vblanks whatsoever are
 169  *  generated after PSR exit. On HSW this has no apparant effect.
 170  *  WaPsrDPRSUnmaskVBlankInSRD says to set this.
 171  *
 172  * The rest of the bits are more self-explanatory and/or
 173  * irrelevant for normal operation.
 174  */
 175
 176 bool intel_encoder_can_psr(struct intel_encoder *encoder)
 177 {
 178         if (intel_encoder_is_dp(encoder) || encoder->type == INTEL_OUTPUT_DP_MST)
 179                 return CAN_PSR(enc_to_intel_dp(encoder)) ||
 180                        CAN_PANEL_REPLAY(enc_to_intel_dp(encoder));
 181         else
 182                 return false;
 183 }
 184
 185 static bool psr_global_enabled(struct intel_dp *intel_dp)
 186 {
 187         struct intel_connector *connector = intel_dp->attached_connector;
 188         struct drm_i915_private *i915 = dp_to_i915(intel_dp);
 189
 190         switch (intel_dp->psr.debug & I915_PSR_DEBUG_MODE_MASK) {
 191         case I915_PSR_DEBUG_DEFAULT:
 192                 if (i915->display.params.enable_psr == -1)
 193                         return connector->panel.vbt.psr.enable;
 194                 return i915->display.params.enable_psr;
 195         case I915_PSR_DEBUG_DISABLE:
 196                 return false;
 197         default:
 198                 return true;
 199         }
 200 }
 201
 202 static bool psr2_global_enabled(struct intel_dp *intel_dp)
 203 {
 204         struct drm_i915_private *i915 = dp_to_i915(intel_dp);
 205
 206         switch (intel_dp->psr.debug & I915_PSR_DEBUG_MODE_MASK) {
 207         case I915_PSR_DEBUG_DISABLE:
 208         case I915_PSR_DEBUG_FORCE_PSR1:
 209                 return false;
 210         default:
 211                 if (i915->display.params.enable_psr == 1)
 212                         return false;
 213                 return true;
 214         }
 215 }
 216
 217 static u32 psr_irq_psr_error_bit_get(struct intel_dp *intel_dp)
 218 {
 219         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 220
 221         return DISPLAY_VER(dev_priv) >= 12 ? TGL_PSR_ERROR :
 222                 EDP_PSR_ERROR(intel_dp->psr.transcoder);
 223 }
 224
 225 static u32 psr_irq_post_exit_bit_get(struct intel_dp *intel_dp)
 226 {
 227         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 228
 229         return DISPLAY_VER(dev_priv) >= 12 ? TGL_PSR_POST_EXIT :
 230                 EDP_PSR_POST_EXIT(intel_dp->psr.transcoder);
 231 }
 232
 233 static u32 psr_irq_pre_entry_bit_get(struct intel_dp *intel_dp)
 234 {
 235         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 236
 237         return DISPLAY_VER(dev_priv) >= 12 ? TGL_PSR_PRE_ENTRY :
 238                 EDP_PSR_PRE_ENTRY(intel_dp->psr.transcoder);
 239 }
 240
 241 static u32 psr_irq_mask_get(struct intel_dp *intel_dp)
 242 {
 243         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 244
 245         return DISPLAY_VER(dev_priv) >= 12 ? TGL_PSR_MASK :
 246                 EDP_PSR_MASK(intel_dp->psr.transcoder);
 247 }
 248
 249 static i915_reg_t psr_ctl_reg(struct drm_i915_private *dev_priv,
 250                               enum transcoder cpu_transcoder)
 251 {
 252         if (DISPLAY_VER(dev_priv) >= 8)
 253                 return EDP_PSR_CTL(cpu_transcoder);
 254         else
 255                 return HSW_SRD_CTL;
 256 }
 257
 258 static i915_reg_t psr_debug_reg(struct drm_i915_private *dev_priv,
 259                                 enum transcoder cpu_transcoder)
 260 {
 261         if (DISPLAY_VER(dev_priv) >= 8)
 262                 return EDP_PSR_DEBUG(cpu_transcoder);
 263         else
 264                 return HSW_SRD_DEBUG;
 265 }
 266
 267 static i915_reg_t psr_perf_cnt_reg(struct drm_i915_private *dev_priv,
 268                                    enum transcoder cpu_transcoder)
 269 {
 270         if (DISPLAY_VER(dev_priv) >= 8)
 271                 return EDP_PSR_PERF_CNT(cpu_transcoder);
 272         else
 273                 return HSW_SRD_PERF_CNT;
 274 }
 275
 276 static i915_reg_t psr_status_reg(struct drm_i915_private *dev_priv,
 277                                  enum transcoder cpu_transcoder)
 278 {
 279         if (DISPLAY_VER(dev_priv) >= 8)
 280                 return EDP_PSR_STATUS(cpu_transcoder);
 281         else
 282                 return HSW_SRD_STATUS;
 283 }
 284
 285 static i915_reg_t psr_imr_reg(struct drm_i915_private *dev_priv,
 286                               enum transcoder cpu_transcoder)
 287 {
 288         if (DISPLAY_VER(dev_priv) >= 12)
 289                 return TRANS_PSR_IMR(cpu_transcoder);
 290         else
 291                 return EDP_PSR_IMR;
 292 }
 293
 294 static i915_reg_t psr_iir_reg(struct drm_i915_private *dev_priv,
 295                               enum transcoder cpu_transcoder)
 296 {
 297         if (DISPLAY_VER(dev_priv) >= 12)
 298                 return TRANS_PSR_IIR(cpu_transcoder);
 299         else
 300                 return EDP_PSR_IIR;
 301 }
 302
 303 static i915_reg_t psr_aux_ctl_reg(struct drm_i915_private *dev_priv,
 304                                   enum transcoder cpu_transcoder)
 305 {
 306         if (DISPLAY_VER(dev_priv) >= 8)
 307                 return EDP_PSR_AUX_CTL(cpu_transcoder);
 308         else
 309                 return HSW_SRD_AUX_CTL;
 310 }
 311
 312 static i915_reg_t psr_aux_data_reg(struct drm_i915_private *dev_priv,
 313                                    enum transcoder cpu_transcoder, int i)
 314 {
 315         if (DISPLAY_VER(dev_priv) >= 8)
 316                 return EDP_PSR_AUX_DATA(cpu_transcoder, i);
 317         else
 318                 return HSW_SRD_AUX_DATA(i);
 319 }
 320
 321 static void psr_irq_control(struct intel_dp *intel_dp)
 322 {
 323         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 324         enum transcoder cpu_transcoder = intel_dp->psr.transcoder;
 325         u32 mask;
 326
 327         mask = psr_irq_psr_error_bit_get(intel_dp);
 328         if (intel_dp->psr.debug & I915_PSR_DEBUG_IRQ)
 329                 mask |= psr_irq_post_exit_bit_get(intel_dp) |
 330                         psr_irq_pre_entry_bit_get(intel_dp);
 331
 332         intel_de_rmw(dev_priv, psr_imr_reg(dev_priv, cpu_transcoder),
 333                      psr_irq_mask_get(intel_dp), ~mask);
 334 }
 335
 336 static void psr_event_print(struct drm_i915_private *i915,
 337                             u32 val, bool psr2_enabled)
 338 {
 339         drm_dbg_kms(&i915->drm, "PSR exit events: 0x%x\n", val);
 340         if (val & PSR_EVENT_PSR2_WD_TIMER_EXPIRE)
 341                 drm_dbg_kms(&i915->drm, "\tPSR2 watchdog timer expired\n");
 342         if ((val & PSR_EVENT_PSR2_DISABLED) && psr2_enabled)
 343                 drm_dbg_kms(&i915->drm, "\tPSR2 disabled\n");
 344         if (val & PSR_EVENT_SU_DIRTY_FIFO_UNDERRUN)
 345                 drm_dbg_kms(&i915->drm, "\tSU dirty FIFO underrun\n");
 346         if (val & PSR_EVENT_SU_CRC_FIFO_UNDERRUN)
 347                 drm_dbg_kms(&i915->drm, "\tSU CRC FIFO underrun\n");
 348         if (val & PSR_EVENT_GRAPHICS_RESET)
 349                 drm_dbg_kms(&i915->drm, "\tGraphics reset\n");
 350         if (val & PSR_EVENT_PCH_INTERRUPT)
 351                 drm_dbg_kms(&i915->drm, "\tPCH interrupt\n");
 352         if (val & PSR_EVENT_MEMORY_UP)
 353                 drm_dbg_kms(&i915->drm, "\tMemory up\n");
 354         if (val & PSR_EVENT_FRONT_BUFFER_MODIFY)
 355                 drm_dbg_kms(&i915->drm, "\tFront buffer modification\n");
 356         if (val & PSR_EVENT_WD_TIMER_EXPIRE)
 357                 drm_dbg_kms(&i915->drm, "\tPSR watchdog timer expired\n");
 358         if (val & PSR_EVENT_PIPE_REGISTERS_UPDATE)
 359                 drm_dbg_kms(&i915->drm, "\tPIPE registers updated\n");
 360         if (val & PSR_EVENT_REGISTER_UPDATE)
 361                 drm_dbg_kms(&i915->drm, "\tRegister updated\n");
 362         if (val & PSR_EVENT_HDCP_ENABLE)
 363                 drm_dbg_kms(&i915->drm, "\tHDCP enabled\n");
 364         if (val & PSR_EVENT_KVMR_SESSION_ENABLE)
 365                 drm_dbg_kms(&i915->drm, "\tKVMR session enabled\n");
 366         if (val & PSR_EVENT_VBI_ENABLE)
 367                 drm_dbg_kms(&i915->drm, "\tVBI enabled\n");
 368         if (val & PSR_EVENT_LPSP_MODE_EXIT)
 369                 drm_dbg_kms(&i915->drm, "\tLPSP mode exited\n");
 370         if ((val & PSR_EVENT_PSR_DISABLE) && !psr2_enabled)
 371                 drm_dbg_kms(&i915->drm, "\tPSR disabled\n");
 372 }
 373
 374 void intel_psr_irq_handler(struct intel_dp *intel_dp, u32 psr_iir)
 375 {
 376         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 377         enum transcoder cpu_transcoder = intel_dp->psr.transcoder;
 378         ktime_t time_ns =  ktime_get();
 379
 380         if (psr_iir & psr_irq_pre_entry_bit_get(intel_dp)) {
 381                 intel_dp->psr.last_entry_attempt = time_ns;
 382                 drm_dbg_kms(&dev_priv->drm,
 383                             "[transcoder %s] PSR entry attempt in 2 vblanks\n",
 384                             transcoder_name(cpu_transcoder));
 385         }
 386
 387         if (psr_iir & psr_irq_post_exit_bit_get(intel_dp)) {
 388                 intel_dp->psr.last_exit = time_ns;
 389                 drm_dbg_kms(&dev_priv->drm,
 390                             "[transcoder %s] PSR exit completed\n",
 391                             transcoder_name(cpu_transcoder));
 392
 393                 if (DISPLAY_VER(dev_priv) >= 9) {
 394                         u32 val;
 395
 396                         val = intel_de_rmw(dev_priv, PSR_EVENT(cpu_transcoder), 0, 0);
 397
 398                         psr_event_print(dev_priv, val, intel_dp->psr.psr2_enabled);
 399                 }
 400         }
 401
 402         if (psr_iir & psr_irq_psr_error_bit_get(intel_dp)) {
 403                 drm_warn(&dev_priv->drm, "[transcoder %s] PSR aux error\n",
 404                          transcoder_name(cpu_transcoder));
 405
 406                 intel_dp->psr.irq_aux_error = true;
 407
 408                 /*
 409                  * If this interruption is not masked it will keep
 410                  * interrupting so fast that it prevents the scheduled
 411                  * work to run.
 412                  * Also after a PSR error, we don't want to arm PSR
 413                  * again so we don't care about unmask the interruption
 414                  * or unset irq_aux_error.
 415                  */
 416                 intel_de_rmw(dev_priv, psr_imr_reg(dev_priv, cpu_transcoder),
 417                              0, psr_irq_psr_error_bit_get(intel_dp));
 418
 419                 queue_work(dev_priv->unordered_wq, &intel_dp->psr.work);
 420         }
 421 }
 422
 423 static bool intel_dp_get_alpm_status(struct intel_dp *intel_dp)
 424 {
 425         u8 alpm_caps = 0;
 426
 427         if (drm_dp_dpcd_readb(&intel_dp->aux, DP_RECEIVER_ALPM_CAP,
 428                               &alpm_caps) != 1)
 429                 return false;
 430         return alpm_caps & DP_ALPM_CAP;
 431 }
 432
 433 static u8 intel_dp_get_sink_sync_latency(struct intel_dp *intel_dp)
 434 {
 435         struct drm_i915_private *i915 = dp_to_i915(intel_dp);
 436         u8 val = 8; /* assume the worst if we can't read the value */
 437
 438         if (drm_dp_dpcd_readb(&intel_dp->aux,
 439                               DP_SYNCHRONIZATION_LATENCY_IN_SINK, &val) == 1)
 440                 val &= DP_MAX_RESYNC_FRAME_COUNT_MASK;
 441         else
 442                 drm_dbg_kms(&i915->drm,
 443                             "Unable to get sink synchronization latency, assuming 8 frames\n");
 444         return val;
 445 }
 446
 447 static void intel_dp_get_su_granularity(struct intel_dp *intel_dp)
 448 {
 449         struct drm_i915_private *i915 = dp_to_i915(intel_dp);
 450         ssize_t r;
 451         u16 w;
 452         u8 y;
 453
 454         /* If sink don't have specific granularity requirements set legacy ones */
 455         if (!(intel_dp->psr_dpcd[1] & DP_PSR2_SU_GRANULARITY_REQUIRED)) {
 456                 /* As PSR2 HW sends full lines, we do not care about x granularity */
 457                 w = 4;
 458                 y = 4;
 459                 goto exit;
 460         }
 461
 462         r = drm_dp_dpcd_read(&intel_dp->aux, DP_PSR2_SU_X_GRANULARITY, &w, 2);
 463         if (r != 2)
 464                 drm_dbg_kms(&i915->drm,
 465                             "Unable to read DP_PSR2_SU_X_GRANULARITY\n");
 466         /*
 467          * Spec says that if the value read is 0 the default granularity should
 468          * be used instead.
 469          */
 470         if (r != 2 || w == 0)
 471                 w = 4;
 472
 473         r = drm_dp_dpcd_read(&intel_dp->aux, DP_PSR2_SU_Y_GRANULARITY, &y, 1);
 474         if (r != 1) {
 475                 drm_dbg_kms(&i915->drm,
 476                             "Unable to read DP_PSR2_SU_Y_GRANULARITY\n");
 477                 y = 4;
 478         }
 479         if (y == 0)
 480                 y = 1;
 481
 482 exit:
 483         intel_dp->psr.su_w_granularity = w;
 484         intel_dp->psr.su_y_granularity = y;
 485 }
 486
 487 static void _panel_replay_init_dpcd(struct intel_dp *intel_dp)
 488 {
 489         struct drm_i915_private *i915 = dp_to_i915(intel_dp);
 490         u8 pr_dpcd = 0;
 491
 492         intel_dp->psr.sink_panel_replay_support = false;
 493         drm_dp_dpcd_readb(&intel_dp->aux, DP_PANEL_REPLAY_CAP, &pr_dpcd);
 494
 495         if (!(pr_dpcd & DP_PANEL_REPLAY_SUPPORT)) {
 496                 drm_dbg_kms(&i915->drm,
 497                             "Panel replay is not supported by panel\n");
 498                 return;
 499         }
 500
 501         drm_dbg_kms(&i915->drm,
 502                     "Panel replay is supported by panel\n");
 503         intel_dp->psr.sink_panel_replay_support = true;
 504 }
 505
 506 static void _psr_init_dpcd(struct intel_dp *intel_dp)
 507 {
 508         struct drm_i915_private *i915 =
 509                 to_i915(dp_to_dig_port(intel_dp)->base.base.dev);
 510
 511         drm_dbg_kms(&i915->drm, "eDP panel supports PSR version %x\n",
 512                     intel_dp->psr_dpcd[0]);
 513
 514         if (drm_dp_has_quirk(&intel_dp->desc, DP_DPCD_QUIRK_NO_PSR)) {
 515                 drm_dbg_kms(&i915->drm,
 516                             "PSR support not currently available for this panel\n");
 517                 return;
 518         }
 519
 520         if (!(intel_dp->edp_dpcd[1] & DP_EDP_SET_POWER_CAP)) {
 521                 drm_dbg_kms(&i915->drm,
 522                             "Panel lacks power state control, PSR cannot be enabled\n");
 523                 return;
 524         }
 525
 526         intel_dp->psr.sink_support = true;
 527         intel_dp->psr.sink_sync_latency =
 528                 intel_dp_get_sink_sync_latency(intel_dp);
 529
 530         if (DISPLAY_VER(i915) >= 9 &&
 531             intel_dp->psr_dpcd[0] == DP_PSR2_WITH_Y_COORD_IS_SUPPORTED) {
 532                 bool y_req = intel_dp->psr_dpcd[1] &
 533                              DP_PSR2_SU_Y_COORDINATE_REQUIRED;
 534                 bool alpm = intel_dp_get_alpm_status(intel_dp);
 535
 536                 /*
 537                  * All panels that supports PSR version 03h (PSR2 +
 538                  * Y-coordinate) can handle Y-coordinates in VSC but we are
 539                  * only sure that it is going to be used when required by the
 540                  * panel. This way panel is capable to do selective update
 541                  * without a aux frame sync.
 542                  *
 543                  * To support PSR version 02h and PSR version 03h without
 544                  * Y-coordinate requirement panels we would need to enable
 545                  * GTC first.
 546                  */
 547                 intel_dp->psr.sink_psr2_support = y_req && alpm;
 548                 drm_dbg_kms(&i915->drm, "PSR2 %ssupported\n",
 549                             intel_dp->psr.sink_psr2_support ? "" : "not ");
 550         }
 551 }
 552
 553 void intel_psr_init_dpcd(struct intel_dp *intel_dp)
 554 {
 555         _panel_replay_init_dpcd(intel_dp);
 556
 557         drm_dp_dpcd_read(&intel_dp->aux, DP_PSR_SUPPORT, intel_dp->psr_dpcd,
 558                          sizeof(intel_dp->psr_dpcd));
 559
 560         if (intel_dp->psr_dpcd[0])
 561                 _psr_init_dpcd(intel_dp);
 562
 563         if (intel_dp->psr.sink_psr2_support) {
 564                 intel_dp->psr.colorimetry_support =
 565                         intel_dp_get_colorimetry_status(intel_dp);
 566                 intel_dp_get_su_granularity(intel_dp);
 567         }
 568 }
 569
 570 static void hsw_psr_setup_aux(struct intel_dp *intel_dp)
 571 {
 572         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 573         enum transcoder cpu_transcoder = intel_dp->psr.transcoder;
 574         u32 aux_clock_divider, aux_ctl;
 575         /* write DP_SET_POWER=D0 */
 576         static const u8 aux_msg[] = {
 577                 [0] = (DP_AUX_NATIVE_WRITE << 4) | ((DP_SET_POWER >> 16) & 0xf),
 578                 [1] = (DP_SET_POWER >> 8) & 0xff,
 579                 [2] = DP_SET_POWER & 0xff,
 580                 [3] = 1 - 1,
 581                 [4] = DP_SET_POWER_D0,
 582         };
 583         int i;
 584
 585         BUILD_BUG_ON(sizeof(aux_msg) > 20);
 586         for (i = 0; i < sizeof(aux_msg); i += 4)
 587                 intel_de_write(dev_priv,
 588                                psr_aux_data_reg(dev_priv, cpu_transcoder, i >> 2),
 589                                intel_dp_aux_pack(&aux_msg[i], sizeof(aux_msg) - i));
 590
 591         aux_clock_divider = intel_dp->get_aux_clock_divider(intel_dp, 0);
 592
 593         /* Start with bits set for DDI_AUX_CTL register */
 594         aux_ctl = intel_dp->get_aux_send_ctl(intel_dp, sizeof(aux_msg),
 595                                              aux_clock_divider);
 596
 597         /* Select only valid bits for SRD_AUX_CTL */
 598         aux_ctl &= EDP_PSR_AUX_CTL_TIME_OUT_MASK |
 599                 EDP_PSR_AUX_CTL_MESSAGE_SIZE_MASK |
 600                 EDP_PSR_AUX_CTL_PRECHARGE_2US_MASK |
 601                 EDP_PSR_AUX_CTL_BIT_CLOCK_2X_MASK;
 602
 603         intel_de_write(dev_priv, psr_aux_ctl_reg(dev_priv, cpu_transcoder),
 604                        aux_ctl);
 605 }
 606
 607 static void intel_psr_enable_sink(struct intel_dp *intel_dp)
 608 {
 609         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 610         u8 dpcd_val = DP_PSR_ENABLE;
 611
 612         if (intel_dp->psr.panel_replay_enabled)
 613                 return;
 614
 615         if (intel_dp->psr.psr2_enabled) {
 616                 /* Enable ALPM at sink for psr2 */
 617                 drm_dp_dpcd_writeb(&intel_dp->aux, DP_RECEIVER_ALPM_CONFIG,
 618                                    DP_ALPM_ENABLE |
 619                                    DP_ALPM_LOCK_ERROR_IRQ_HPD_ENABLE);
 620
 621                 dpcd_val |= DP_PSR_ENABLE_PSR2 | DP_PSR_IRQ_HPD_WITH_CRC_ERRORS;
 622         } else {
 623                 if (intel_dp->psr.link_standby)
 624                         dpcd_val |= DP_PSR_MAIN_LINK_ACTIVE;
 625
 626                 if (DISPLAY_VER(dev_priv) >= 8)
 627                         dpcd_val |= DP_PSR_CRC_VERIFICATION;
 628         }
 629
 630         if (intel_dp->psr.req_psr2_sdp_prior_scanline)
 631                 dpcd_val |= DP_PSR_SU_REGION_SCANLINE_CAPTURE;
 632
 633         if (intel_dp->psr.entry_setup_frames > 0)
 634                 dpcd_val |= DP_PSR_FRAME_CAPTURE;
 635
 636         drm_dp_dpcd_writeb(&intel_dp->aux, DP_PSR_EN_CFG, dpcd_val);
 637
 638         drm_dp_dpcd_writeb(&intel_dp->aux, DP_SET_POWER, DP_SET_POWER_D0);
 639 }
 640
 641 static u32 intel_psr1_get_tp_time(struct intel_dp *intel_dp)
 642 {
 643         struct intel_connector *connector = intel_dp->attached_connector;
 644         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 645         u32 val = 0;
 646
 647         if (DISPLAY_VER(dev_priv) >= 11)
 648                 val |= EDP_PSR_TP4_TIME_0us;
 649
 650         if (dev_priv->display.params.psr_safest_params) {
 651                 val |= EDP_PSR_TP1_TIME_2500us;
 652                 val |= EDP_PSR_TP2_TP3_TIME_2500us;
 653                 goto check_tp3_sel;
 654         }
 655
 656         if (connector->panel.vbt.psr.tp1_wakeup_time_us == 0)
 657                 val |= EDP_PSR_TP1_TIME_0us;
 658         else if (connector->panel.vbt.psr.tp1_wakeup_time_us <= 100)
 659                 val |= EDP_PSR_TP1_TIME_100us;
 660         else if (connector->panel.vbt.psr.tp1_wakeup_time_us <= 500)
 661                 val |= EDP_PSR_TP1_TIME_500us;
 662         else
 663                 val |= EDP_PSR_TP1_TIME_2500us;
 664
 665         if (connector->panel.vbt.psr.tp2_tp3_wakeup_time_us == 0)
 666                 val |= EDP_PSR_TP2_TP3_TIME_0us;
 667         else if (connector->panel.vbt.psr.tp2_tp3_wakeup_time_us <= 100)
 668                 val |= EDP_PSR_TP2_TP3_TIME_100us;
 669         else if (connector->panel.vbt.psr.tp2_tp3_wakeup_time_us <= 500)
 670                 val |= EDP_PSR_TP2_TP3_TIME_500us;
 671         else
 672                 val |= EDP_PSR_TP2_TP3_TIME_2500us;
 673
 674         /*
 675          * WA 0479: hsw,bdw
 676          * "Do not skip both TP1 and TP2/TP3"
 677          */
 678         if (DISPLAY_VER(dev_priv) < 9 &&
 679             connector->panel.vbt.psr.tp1_wakeup_time_us == 0 &&
 680             connector->panel.vbt.psr.tp2_tp3_wakeup_time_us == 0)
 681                 val |= EDP_PSR_TP2_TP3_TIME_100us;
 682
 683 check_tp3_sel:
 684         if (intel_dp_source_supports_tps3(dev_priv) &&
 685             drm_dp_tps3_supported(intel_dp->dpcd))
 686                 val |= EDP_PSR_TP_TP1_TP3;
 687         else
 688                 val |= EDP_PSR_TP_TP1_TP2;
 689
 690         return val;
 691 }
 692
 693 static u8 psr_compute_idle_frames(struct intel_dp *intel_dp)
 694 {
 695         struct intel_connector *connector = intel_dp->attached_connector;
 696         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 697         int idle_frames;
 698
 699         /* Let's use 6 as the minimum to cover all known cases including the
 700          * off-by-one issue that HW has in some cases.
 701          */
 702         idle_frames = max(6, connector->panel.vbt.psr.idle_frames);
 703         idle_frames = max(idle_frames, intel_dp->psr.sink_sync_latency + 1);
 704
 705         if (drm_WARN_ON(&dev_priv->drm, idle_frames > 0xf))
 706                 idle_frames = 0xf;
 707
 708         return idle_frames;
 709 }
 710
 711 static void hsw_activate_psr1(struct intel_dp *intel_dp)
 712 {
 713         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 714         enum transcoder cpu_transcoder = intel_dp->psr.transcoder;
 715         u32 max_sleep_time = 0x1f;
 716         u32 val = EDP_PSR_ENABLE;
 717
 718         val |= EDP_PSR_IDLE_FRAMES(psr_compute_idle_frames(intel_dp));
 719
 720         if (DISPLAY_VER(dev_priv) < 20)
 721                 val |= EDP_PSR_MAX_SLEEP_TIME(max_sleep_time);
 722
 723         if (IS_HASWELL(dev_priv))
 724                 val |= EDP_PSR_MIN_LINK_ENTRY_TIME_8_LINES;
 725
 726         if (intel_dp->psr.link_standby)
 727                 val |= EDP_PSR_LINK_STANDBY;
 728
 729         val |= intel_psr1_get_tp_time(intel_dp);
 730
 731         if (DISPLAY_VER(dev_priv) >= 8)
 732                 val |= EDP_PSR_CRC_ENABLE;
 733
 734         if (DISPLAY_VER(dev_priv) >= 20)
 735                 val |= LNL_EDP_PSR_ENTRY_SETUP_FRAMES(intel_dp->psr.entry_setup_frames);
 736
 737         intel_de_rmw(dev_priv, psr_ctl_reg(dev_priv, cpu_transcoder),
 738                      ~EDP_PSR_RESTORE_PSR_ACTIVE_CTX_MASK, val);
 739 }
 740
 741 static u32 intel_psr2_get_tp_time(struct intel_dp *intel_dp)
 742 {
 743         struct intel_connector *connector = intel_dp->attached_connector;
 744         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 745         u32 val = 0;
 746
 747         if (dev_priv->display.params.psr_safest_params)
 748                 return EDP_PSR2_TP2_TIME_2500us;
 749
 750         if (connector->panel.vbt.psr.psr2_tp2_tp3_wakeup_time_us >= 0 &&
 751             connector->panel.vbt.psr.psr2_tp2_tp3_wakeup_time_us <= 50)
 752                 val |= EDP_PSR2_TP2_TIME_50us;
 753         else if (connector->panel.vbt.psr.psr2_tp2_tp3_wakeup_time_us <= 100)
 754                 val |= EDP_PSR2_TP2_TIME_100us;
 755         else if (connector->panel.vbt.psr.psr2_tp2_tp3_wakeup_time_us <= 500)
 756                 val |= EDP_PSR2_TP2_TIME_500us;
 757         else
 758                 val |= EDP_PSR2_TP2_TIME_2500us;
 759
 760         return val;
 761 }
 762
 763 static int psr2_block_count_lines(struct intel_dp *intel_dp)
 764 {
 765         return intel_dp->psr.io_wake_lines < 9 &&
 766                 intel_dp->psr.fast_wake_lines < 9 ? 8 : 12;
 767 }
 768
 769 static int psr2_block_count(struct intel_dp *intel_dp)
 770 {
 771         return psr2_block_count_lines(intel_dp) / 4;
 772 }
 773
 774 static u8 frames_before_su_entry(struct intel_dp *intel_dp)
 775 {
 776         u8 frames_before_su_entry;
 777
 778         frames_before_su_entry = max_t(u8,
 779                                        intel_dp->psr.sink_sync_latency + 1,
 780                                        2);
 781
 782         /* Entry setup frames must be at least 1 less than frames before SU entry */
 783         if (intel_dp->psr.entry_setup_frames >= frames_before_su_entry)
 784                 frames_before_su_entry = intel_dp->psr.entry_setup_frames + 1;
 785
 786         return frames_before_su_entry;
 787 }
 788
 789 static void dg2_activate_panel_replay(struct intel_dp *intel_dp)
 790 {
 791         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 792
 793         intel_de_rmw(dev_priv, PSR2_MAN_TRK_CTL(intel_dp->psr.transcoder),
 794                      0, ADLP_PSR2_MAN_TRK_CTL_SF_CONTINUOS_FULL_FRAME);
 795
 796         intel_de_rmw(dev_priv, TRANS_DP2_CTL(intel_dp->psr.transcoder), 0,
 797                      TRANS_DP2_PANEL_REPLAY_ENABLE);
 798 }
 799
 800 static void hsw_activate_psr2(struct intel_dp *intel_dp)
 801 {
 802         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 803         enum transcoder cpu_transcoder = intel_dp->psr.transcoder;
 804         u32 val = EDP_PSR2_ENABLE;
 805         u32 psr_val = 0;
 806
 807         val |= EDP_PSR2_IDLE_FRAMES(psr_compute_idle_frames(intel_dp));
 808
 809         if (DISPLAY_VER(dev_priv) < 14 && !IS_ALDERLAKE_P(dev_priv))
 810                 val |= EDP_SU_TRACK_ENABLE;
 811
 812         if (DISPLAY_VER(dev_priv) >= 10 && DISPLAY_VER(dev_priv) < 13)
 813                 val |= EDP_Y_COORDINATE_ENABLE;
 814
 815         val |= EDP_PSR2_FRAME_BEFORE_SU(frames_before_su_entry(intel_dp));
 816
 817         val |= intel_psr2_get_tp_time(intel_dp);
 818
 819         if (DISPLAY_VER(dev_priv) >= 12) {
 820                 if (psr2_block_count(intel_dp) > 2)
 821                         val |= TGL_EDP_PSR2_BLOCK_COUNT_NUM_3;
 822                 else
 823                         val |= TGL_EDP_PSR2_BLOCK_COUNT_NUM_2;
 824         }
 825
 826         /* Wa_22012278275:adl-p */
 827         if (IS_ALDERLAKE_P(dev_priv) && IS_DISPLAY_STEP(dev_priv, STEP_A0, STEP_E0)) {
 828                 static const u8 map[] = {
 829                         2, /* 5 lines */
 830                         1, /* 6 lines */
 831                         0, /* 7 lines */
 832                         3, /* 8 lines */
 833                         6, /* 9 lines */
 834                         5, /* 10 lines */
 835                         4, /* 11 lines */
 836                         7, /* 12 lines */
 837                 };
 838                 /*
 839                  * Still using the default IO_BUFFER_WAKE and FAST_WAKE, see
 840                  * comments bellow for more information
 841                  */
 842                 int tmp;
 843
 844                 tmp = map[intel_dp->psr.io_wake_lines - TGL_EDP_PSR2_IO_BUFFER_WAKE_MIN_LINES];
 845                 val |= TGL_EDP_PSR2_IO_BUFFER_WAKE(tmp + TGL_EDP_PSR2_IO_BUFFER_WAKE_MIN_LINES);
 846
 847                 tmp = map[intel_dp->psr.fast_wake_lines - TGL_EDP_PSR2_FAST_WAKE_MIN_LINES];
 848                 val |= TGL_EDP_PSR2_FAST_WAKE(tmp + TGL_EDP_PSR2_FAST_WAKE_MIN_LINES);
 849         } else if (DISPLAY_VER(dev_priv) >= 12) {
 850                 val |= TGL_EDP_PSR2_IO_BUFFER_WAKE(intel_dp->psr.io_wake_lines);
 851                 val |= TGL_EDP_PSR2_FAST_WAKE(intel_dp->psr.fast_wake_lines);
 852         } else if (DISPLAY_VER(dev_priv) >= 9) {
 853                 val |= EDP_PSR2_IO_BUFFER_WAKE(intel_dp->psr.io_wake_lines);
 854                 val |= EDP_PSR2_FAST_WAKE(intel_dp->psr.fast_wake_lines);
 855         }
 856
 857         if (intel_dp->psr.req_psr2_sdp_prior_scanline)
 858                 val |= EDP_PSR2_SU_SDP_SCANLINE;
 859
 860         if (DISPLAY_VER(dev_priv) >= 20)
 861                 psr_val |= LNL_EDP_PSR_ENTRY_SETUP_FRAMES(intel_dp->psr.entry_setup_frames);
 862
 863         if (intel_dp->psr.psr2_sel_fetch_enabled) {
 864                 u32 tmp;
 865
 866                 tmp = intel_de_read(dev_priv, PSR2_MAN_TRK_CTL(cpu_transcoder));
 867                 drm_WARN_ON(&dev_priv->drm, !(tmp & PSR2_MAN_TRK_CTL_ENABLE));
 868         } else if (HAS_PSR2_SEL_FETCH(dev_priv)) {
 869                 intel_de_write(dev_priv, PSR2_MAN_TRK_CTL(cpu_transcoder), 0);
 870         }
 871
 872         /*
 873          * PSR2 HW is incorrectly using EDP_PSR_TP1_TP3_SEL and BSpec is
 874          * recommending keep this bit unset while PSR2 is enabled.
 875          */
 876         intel_de_write(dev_priv, psr_ctl_reg(dev_priv, cpu_transcoder), psr_val);
 877
 878         intel_de_write(dev_priv, EDP_PSR2_CTL(cpu_transcoder), val);
 879 }
 880
 881 static bool
 882 transcoder_has_psr2(struct drm_i915_private *dev_priv, enum transcoder cpu_transcoder)
 883 {
 884         if (IS_ALDERLAKE_P(dev_priv) || DISPLAY_VER(dev_priv) >= 14)
 885                 return cpu_transcoder == TRANSCODER_A || cpu_transcoder == TRANSCODER_B;
 886         else if (DISPLAY_VER(dev_priv) >= 12)
 887                 return cpu_transcoder == TRANSCODER_A;
 888         else if (DISPLAY_VER(dev_priv) >= 9)
 889                 return cpu_transcoder == TRANSCODER_EDP;
 890         else
 891                 return false;
 892 }
 893
 894 static u32 intel_get_frame_time_us(const struct intel_crtc_state *crtc_state)
 895 {
 896         if (!crtc_state->hw.active)
 897                 return 0;
 898
 899         return DIV_ROUND_UP(1000 * 1000,
 900                             drm_mode_vrefresh(&crtc_state->hw.adjusted_mode));
 901 }
 902
 903 static void psr2_program_idle_frames(struct intel_dp *intel_dp,
 904                                      u32 idle_frames)
 905 {
 906         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 907         enum transcoder cpu_transcoder = intel_dp->psr.transcoder;
 908
 909         intel_de_rmw(dev_priv, EDP_PSR2_CTL(cpu_transcoder),
 910                      EDP_PSR2_IDLE_FRAMES_MASK,
 911                      EDP_PSR2_IDLE_FRAMES(idle_frames));
 912 }
 913
 914 static void tgl_psr2_enable_dc3co(struct intel_dp *intel_dp)
 915 {
 916         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 917
 918         psr2_program_idle_frames(intel_dp, 0);
 919         intel_display_power_set_target_dc_state(dev_priv, DC_STATE_EN_DC3CO);
 920 }
 921
 922 static void tgl_psr2_disable_dc3co(struct intel_dp *intel_dp)
 923 {
 924         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 925
 926         intel_display_power_set_target_dc_state(dev_priv, DC_STATE_EN_UPTO_DC6);
 927         psr2_program_idle_frames(intel_dp, psr_compute_idle_frames(intel_dp));
 928 }
 929
 930 static void tgl_dc3co_disable_work(struct work_struct *work)
 931 {
 932         struct intel_dp *intel_dp =
 933                 container_of(work, typeof(*intel_dp), psr.dc3co_work.work);
 934
 935         mutex_lock(&intel_dp->psr.lock);
 936         /* If delayed work is pending, it is not idle */
 937         if (delayed_work_pending(&intel_dp->psr.dc3co_work))
 938                 goto unlock;
 939
 940         tgl_psr2_disable_dc3co(intel_dp);
 941 unlock:
 942         mutex_unlock(&intel_dp->psr.lock);
 943 }
 944
 945 static void tgl_disallow_dc3co_on_psr2_exit(struct intel_dp *intel_dp)
 946 {
 947         if (!intel_dp->psr.dc3co_exitline)
 948                 return;
 949
 950         cancel_delayed_work(&intel_dp->psr.dc3co_work);
 951         /* Before PSR2 exit disallow dc3co*/
 952         tgl_psr2_disable_dc3co(intel_dp);
 953 }
 954
 955 static bool
 956 dc3co_is_pipe_port_compatible(struct intel_dp *intel_dp,
 957                               struct intel_crtc_state *crtc_state)
 958 {
 959         struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
 960         enum pipe pipe = to_intel_crtc(crtc_state->uapi.crtc)->pipe;
 961         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 962         enum port port = dig_port->base.port;
 963
 964         if (IS_ALDERLAKE_P(dev_priv) || DISPLAY_VER(dev_priv) >= 14)
 965                 return pipe <= PIPE_B && port <= PORT_B;
 966         else
 967                 return pipe == PIPE_A && port == PORT_A;
 968 }
 969
 970 static void
 971 tgl_dc3co_exitline_compute_config(struct intel_dp *intel_dp,
 972                                   struct intel_crtc_state *crtc_state)
 973 {
 974         const u32 crtc_vdisplay = crtc_state->uapi.adjusted_mode.crtc_vdisplay;
 975         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 976         struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
 977         u32 exit_scanlines;
 978
 979         /*
 980          * FIXME: Due to the changed sequence of activating/deactivating DC3CO,
 981          * disable DC3CO until the changed dc3co activating/deactivating sequence
 982          * is applied. B.Specs:49196
 983          */
 984         return;
 985
 986         /*
 987          * DMC's DC3CO exit mechanism has an issue with Selective Fecth
 988          * TODO: when the issue is addressed, this restriction should be removed.
 989          */
 990         if (crtc_state->enable_psr2_sel_fetch)
 991                 return;
 992
 993         if (!(power_domains->allowed_dc_mask & DC_STATE_EN_DC3CO))
 994                 return;
 995
 996         if (!dc3co_is_pipe_port_compatible(intel_dp, crtc_state))
 997                 return;
 998
 999         /* Wa_16011303918:adl-p */
1000         if (IS_ALDERLAKE_P(dev_priv) && IS_DISPLAY_STEP(dev_priv, STEP_A0, STEP_B0))
1001                 return;
1002
1003         /*
1004          * DC3CO Exit time 200us B.Spec 49196
1005          * PSR2 transcoder Early Exit scanlines = ROUNDUP(200 / line time) + 1
1006          */
1007         exit_scanlines =
1008                 intel_usecs_to_scanlines(&crtc_state->uapi.adjusted_mode, 200) + 1;
1009
1010         if (drm_WARN_ON(&dev_priv->drm, exit_scanlines > crtc_vdisplay))
1011                 return;
1012
1013         crtc_state->dc3co_exitline = crtc_vdisplay - exit_scanlines;
1014 }
1015
1016 static bool intel_psr2_sel_fetch_config_valid(struct intel_dp *intel_dp,
1017                                               struct intel_crtc_state *crtc_state)
1018 {
1019         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
1020
1021         if (!dev_priv->display.params.enable_psr2_sel_fetch &&
1022             intel_dp->psr.debug != I915_PSR_DEBUG_ENABLE_SEL_FETCH) {
1023                 drm_dbg_kms(&dev_priv->drm,
1024                             "PSR2 sel fetch not enabled, disabled by parameter\n");
1025                 return false;
1026         }
1027
1028         if (crtc_state->uapi.async_flip) {
1029                 drm_dbg_kms(&dev_priv->drm,
1030                             "PSR2 sel fetch not enabled, async flip enabled\n");
1031                 return false;
1032         }
1033
1034         return crtc_state->enable_psr2_sel_fetch = true;
1035 }
1036
1037 static bool psr2_granularity_check(struct intel_dp *intel_dp,
1038                                    struct intel_crtc_state *crtc_state)
1039 {
1040         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
1041         const struct drm_dsc_config *vdsc_cfg = &crtc_state->dsc.config;
1042         const int crtc_hdisplay = crtc_state->hw.adjusted_mode.crtc_hdisplay;
1043         const int crtc_vdisplay = crtc_state->hw.adjusted_mode.crtc_vdisplay;
1044         u16 y_granularity = 0;
1045
1046         /* PSR2 HW only send full lines so we only need to validate the width */
1047         if (crtc_hdisplay % intel_dp->psr.su_w_granularity)
1048                 return false;
1049
1050         if (crtc_vdisplay % intel_dp->psr.su_y_granularity)
1051                 return false;
1052
1053         /* HW tracking is only aligned to 4 lines */
1054         if (!crtc_state->enable_psr2_sel_fetch)
1055                 return intel_dp->psr.su_y_granularity == 4;
1056
1057         /*
1058          * adl_p and mtl platforms have 1 line granularity.
1059          * For other platforms with SW tracking we can adjust the y coordinates
1060          * to match sink requirement if multiple of 4.
1061          */
1062         if (IS_ALDERLAKE_P(dev_priv) || DISPLAY_VER(dev_priv) >= 14)
1063                 y_granularity = intel_dp->psr.su_y_granularity;
1064         else if (intel_dp->psr.su_y_granularity <= 2)
1065                 y_granularity = 4;
1066         else if ((intel_dp->psr.su_y_granularity % 4) == 0)
1067                 y_granularity = intel_dp->psr.su_y_granularity;
1068
1069         if (y_granularity == 0 || crtc_vdisplay % y_granularity)
1070                 return false;
1071
1072         if (crtc_state->dsc.compression_enable &&
1073             vdsc_cfg->slice_height % y_granularity)
1074                 return false;
1075
1076         crtc_state->su_y_granularity = y_granularity;
1077         return true;
1078 }
1079
1080 static bool _compute_psr2_sdp_prior_scanline_indication(struct intel_dp *intel_dp,
1081                                                         struct intel_crtc_state *crtc_state)
1082 {
1083         const struct drm_display_mode *adjusted_mode = &crtc_state->uapi.adjusted_mode;
1084         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
1085         u32 hblank_total, hblank_ns, req_ns;
1086
1087         hblank_total = adjusted_mode->crtc_hblank_end - adjusted_mode->crtc_hblank_start;
1088         hblank_ns = div_u64(1000000ULL * hblank_total, adjusted_mode->crtc_clock);
1089
1090         /* From spec: ((60 / number of lanes) + 11) * 1000 / symbol clock frequency MHz */
1091         req_ns = ((60 / crtc_state->lane_count) + 11) * 1000 / (crtc_state->port_clock / 1000);
1092
1093         if ((hblank_ns - req_ns) > 100)
1094                 return true;
1095
1096         /* Not supported <13 / Wa_22012279113:adl-p */
1097         if (DISPLAY_VER(dev_priv) < 14 || intel_dp->edp_dpcd[0] < DP_EDP_14b)
1098                 return false;
1099
1100         crtc_state->req_psr2_sdp_prior_scanline = true;
1101         return true;
1102 }
1103
1104 static bool _compute_psr2_wake_times(struct intel_dp *intel_dp,
1105                                      struct intel_crtc_state *crtc_state)
1106 {
1107         struct drm_i915_private *i915 = dp_to_i915(intel_dp);
1108         int io_wake_lines, io_wake_time, fast_wake_lines, fast_wake_time;
1109         u8 max_wake_lines;
1110
1111         if (DISPLAY_VER(i915) >= 12) {
1112                 io_wake_time = 42;
1113                 /*
1114                  * According to Bspec it's 42us, but based on testing
1115                  * it is not enough -> use 45 us.
1116                  */
1117                 fast_wake_time = 45;
1118                 max_wake_lines = 12;
1119         } else {
1120                 io_wake_time = 50;
1121                 fast_wake_time = 32;
1122                 max_wake_lines = 8;
1123         }
1124
1125         io_wake_lines = intel_usecs_to_scanlines(
1126                 &crtc_state->hw.adjusted_mode, io_wake_time);
1127         fast_wake_lines = intel_usecs_to_scanlines(
1128                 &crtc_state->hw.adjusted_mode, fast_wake_time);
1129
1130         if (io_wake_lines > max_wake_lines ||
1131             fast_wake_lines > max_wake_lines)
1132                 return false;
1133
1134         if (i915->display.params.psr_safest_params)
1135                 io_wake_lines = fast_wake_lines = max_wake_lines;
1136
1137         /* According to Bspec lower limit should be set as 7 lines. */
1138         intel_dp->psr.io_wake_lines = max(io_wake_lines, 7);
1139         intel_dp->psr.fast_wake_lines = max(fast_wake_lines, 7);
1140
1141         return true;
1142 }
1143
1144 static int intel_psr_entry_setup_frames(struct intel_dp *intel_dp,
1145                                         const struct drm_display_mode *adjusted_mode)
1146 {
1147         struct drm_i915_private *i915 = dp_to_i915(intel_dp);
1148         int psr_setup_time = drm_dp_psr_setup_time(intel_dp->psr_dpcd);
1149         int entry_setup_frames = 0;
1150
1151         if (psr_setup_time < 0) {
1152                 drm_dbg_kms(&i915->drm,
1153                             "PSR condition failed: Invalid PSR setup time (0x%02x)\n",
1154                             intel_dp->psr_dpcd[1]);
1155                 return -ETIME;
1156         }
1157
1158         if (intel_usecs_to_scanlines(adjusted_mode, psr_setup_time) >
1159             adjusted_mode->crtc_vtotal - adjusted_mode->crtc_vdisplay - 1) {
1160                 if (DISPLAY_VER(i915) >= 20) {
1161                         /* setup entry frames can be up to 3 frames */
1162                         entry_setup_frames = 1;
1163                         drm_dbg_kms(&i915->drm,
1164                                     "PSR setup entry frames %d\n",
1165                                     entry_setup_frames);
1166                 } else {
1167                         drm_dbg_kms(&i915->drm,
1168                                     "PSR condition failed: PSR setup time (%d us) too long\n",
1169                                     psr_setup_time);
1170                         return -ETIME;
1171                 }
1172         }
1173
1174         return entry_setup_frames;
1175 }
1176
1177 static bool intel_psr2_config_valid(struct intel_dp *intel_dp,
1178                                     struct intel_crtc_state *crtc_state)
1179 {
1180         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
1181         int crtc_hdisplay = crtc_state->hw.adjusted_mode.crtc_hdisplay;
1182         int crtc_vdisplay = crtc_state->hw.adjusted_mode.crtc_vdisplay;
1183         int psr_max_h = 0, psr_max_v = 0, max_bpp = 0;
1184
1185         if (!intel_dp->psr.sink_psr2_support)
1186                 return false;
1187
1188         /* JSL and EHL only supports eDP 1.3 */
1189         if (IS_JASPERLAKE(dev_priv) || IS_ELKHARTLAKE(dev_priv)) {
1190                 drm_dbg_kms(&dev_priv->drm, "PSR2 not supported by phy\n");
1191                 return false;
1192         }
1193
1194         /* Wa_16011181250 */
1195         if (IS_ROCKETLAKE(dev_priv) || IS_ALDERLAKE_S(dev_priv) ||
1196             IS_DG2(dev_priv)) {
1197                 drm_dbg_kms(&dev_priv->drm, "PSR2 is defeatured for this platform\n");
1198                 return false;
1199         }
1200
1201         if (IS_ALDERLAKE_P(dev_priv) && IS_DISPLAY_STEP(dev_priv, STEP_A0, STEP_B0)) {
1202                 drm_dbg_kms(&dev_priv->drm, "PSR2 not completely functional in this stepping\n");
1203                 return false;
1204         }
1205
1206         if (!transcoder_has_psr2(dev_priv, crtc_state->cpu_transcoder)) {
1207                 drm_dbg_kms(&dev_priv->drm,
1208                             "PSR2 not supported in transcoder %s\n",
1209                             transcoder_name(crtc_state->cpu_transcoder));
1210                 return false;
1211         }
1212
1213         if (!psr2_global_enabled(intel_dp)) {
1214                 drm_dbg_kms(&dev_priv->drm, "PSR2 disabled by flag\n");
1215                 return false;
1216         }
1217
1218         /*
1219          * DSC and PSR2 cannot be enabled simultaneously. If a requested
1220          * resolution requires DSC to be enabled, priority is given to DSC
1221          * over PSR2.
1222          */
1223         if (crtc_state->dsc.compression_enable &&
1224             (DISPLAY_VER(dev_priv) < 14 && !IS_ALDERLAKE_P(dev_priv))) {
1225                 drm_dbg_kms(&dev_priv->drm,
1226                             "PSR2 cannot be enabled since DSC is enabled\n");
1227                 return false;
1228         }
1229
1230         if (crtc_state->crc_enabled) {
1231                 drm_dbg_kms(&dev_priv->drm,
1232                             "PSR2 not enabled because it would inhibit pipe CRC calculation\n");
1233                 return false;
1234         }
1235
1236         if (DISPLAY_VER(dev_priv) >= 12) {
1237                 psr_max_h = 5120;
1238                 psr_max_v = 3200;
1239                 max_bpp = 30;
1240         } else if (DISPLAY_VER(dev_priv) >= 10) {
1241                 psr_max_h = 4096;
1242                 psr_max_v = 2304;
1243                 max_bpp = 24;
1244         } else if (DISPLAY_VER(dev_priv) == 9) {
1245                 psr_max_h = 3640;
1246                 psr_max_v = 2304;
1247                 max_bpp = 24;
1248         }
1249
1250         if (crtc_state->pipe_bpp > max_bpp) {
1251                 drm_dbg_kms(&dev_priv->drm,
1252                             "PSR2 not enabled, pipe bpp %d > max supported %d\n",
1253                             crtc_state->pipe_bpp, max_bpp);
1254                 return false;
1255         }
1256
1257         /* Wa_16011303918:adl-p */
1258         if (crtc_state->vrr.enable &&
1259             IS_ALDERLAKE_P(dev_priv) && IS_DISPLAY_STEP(dev_priv, STEP_A0, STEP_B0)) {
1260                 drm_dbg_kms(&dev_priv->drm,
1261                             "PSR2 not enabled, not compatible with HW stepping + VRR\n");
1262                 return false;
1263         }
1264
1265         if (!_compute_psr2_sdp_prior_scanline_indication(intel_dp, crtc_state)) {
1266                 drm_dbg_kms(&dev_priv->drm,
1267                             "PSR2 not enabled, PSR2 SDP indication do not fit in hblank\n");
1268                 return false;
1269         }
1270
1271         if (!_compute_psr2_wake_times(intel_dp, crtc_state)) {
1272                 drm_dbg_kms(&dev_priv->drm,
1273                             "PSR2 not enabled, Unable to use long enough wake times\n");
1274                 return false;
1275         }
1276
1277         /* Vblank >= PSR2_CTL Block Count Number maximum line count */
1278         if (crtc_state->hw.adjusted_mode.crtc_vblank_end -
1279             crtc_state->hw.adjusted_mode.crtc_vblank_start <
1280             psr2_block_count_lines(intel_dp)) {
1281                 drm_dbg_kms(&dev_priv->drm,
1282                             "PSR2 not enabled, too short vblank time\n");
1283                 return false;
1284         }
1285
1286         if (HAS_PSR2_SEL_FETCH(dev_priv)) {
1287                 if (!intel_psr2_sel_fetch_config_valid(intel_dp, crtc_state) &&
1288                     !HAS_PSR_HW_TRACKING(dev_priv)) {
1289                         drm_dbg_kms(&dev_priv->drm,
1290                                     "PSR2 not enabled, selective fetch not valid and no HW tracking available\n");
1291                         return false;
1292                 }
1293         }
1294
1295         if (!psr2_granularity_check(intel_dp, crtc_state)) {
1296                 drm_dbg_kms(&dev_priv->drm, "PSR2 not enabled, SU granularity not compatible\n");
1297                 goto unsupported;
1298         }
1299
1300         if (!crtc_state->enable_psr2_sel_fetch &&
1301             (crtc_hdisplay > psr_max_h || crtc_vdisplay > psr_max_v)) {
1302                 drm_dbg_kms(&dev_priv->drm,
1303                             "PSR2 not enabled, resolution %dx%d > max supported %dx%d\n",
1304                             crtc_hdisplay, crtc_vdisplay,
1305                             psr_max_h, psr_max_v);
1306                 goto unsupported;
1307         }
1308
1309         tgl_dc3co_exitline_compute_config(intel_dp, crtc_state);
1310         return true;
1311
1312 unsupported:
1313         crtc_state->enable_psr2_sel_fetch = false;
1314         return false;
1315 }
1316
1317 static bool _psr_compute_config(struct intel_dp *intel_dp,
1318                                 struct intel_crtc_state *crtc_state)
1319 {
1320         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
1321         const struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode;
1322         int entry_setup_frames;
1323
1324         /*
1325          * Current PSR panels don't work reliably with VRR enabled
1326          * So if VRR is enabled, do not enable PSR.
1327          */
1328         if (crtc_state->vrr.enable)
1329                 return false;
1330
1331         if (!CAN_PSR(intel_dp))
1332                 return false;
1333
1334         entry_setup_frames = intel_psr_entry_setup_frames(intel_dp, adjusted_mode);
1335
1336         if (entry_setup_frames >= 0) {
1337                 intel_dp->psr.entry_setup_frames = entry_setup_frames;
1338         } else {
1339                 drm_dbg_kms(&dev_priv->drm,
1340                             "PSR condition failed: PSR setup timing not met\n");
1341                 return false;
1342         }
1343
1344         return true;
1345 }
1346
1347 void intel_psr_compute_config(struct intel_dp *intel_dp,
1348                               struct intel_crtc_state *crtc_state,
1349                               struct drm_connector_state *conn_state)
1350 {
1351         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
1352         const struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode;
1353
1354         if (!psr_global_enabled(intel_dp)) {
1355                 drm_dbg_kms(&dev_priv->drm, "PSR disabled by flag\n");
1356                 return;
1357         }
1358
1359         if (intel_dp->psr.sink_not_reliable) {
1360                 drm_dbg_kms(&dev_priv->drm,
1361                             "PSR sink implementation is not reliable\n");
1362                 return;
1363         }
1364
1365         if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) {
1366                 drm_dbg_kms(&dev_priv->drm,
1367                             "PSR condition failed: Interlaced mode enabled\n");
1368                 return;
1369         }
1370
1371         if (CAN_PANEL_REPLAY(intel_dp))
1372                 crtc_state->has_panel_replay = true;
1373         else
1374                 crtc_state->has_psr = _psr_compute_config(intel_dp, crtc_state);
1375
1376         if (!(crtc_state->has_panel_replay || crtc_state->has_psr))
1377                 return;
1378
1379         crtc_state->has_psr2 = intel_psr2_config_valid(intel_dp, crtc_state);
1380
1381         crtc_state->infoframes.enable |= intel_hdmi_infoframe_enable(DP_SDP_VSC);
1382         intel_dp_compute_psr_vsc_sdp(intel_dp, crtc_state, conn_state,
1383                                      &crtc_state->psr_vsc);
1384 }
1385
1386 void intel_psr_get_config(struct intel_encoder *encoder,
1387                           struct intel_crtc_state *pipe_config)
1388 {
1389         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
1390         struct intel_digital_port *dig_port = enc_to_dig_port(encoder);
1391         enum transcoder cpu_transcoder = pipe_config->cpu_transcoder;
1392         struct intel_dp *intel_dp;
1393         u32 val;
1394
1395         if (!dig_port)
1396                 return;
1397
1398         intel_dp = &dig_port->dp;
1399         if (!(CAN_PSR(intel_dp) || CAN_PANEL_REPLAY(intel_dp)))
1400                 return;
1401
1402         mutex_lock(&intel_dp->psr.lock);
1403         if (!intel_dp->psr.enabled)
1404                 goto unlock;
1405
1406         if (intel_dp->psr.panel_replay_enabled) {
1407                 pipe_config->has_panel_replay = true;
1408         } else {
1409                 /*
1410                  * Not possible to read EDP_PSR/PSR2_CTL registers as it is
1411                  * enabled/disabled because of frontbuffer tracking and others.
1412                  */
1413                 pipe_config->has_psr = true;
1414         }
1415
1416         pipe_config->has_psr2 = intel_dp->psr.psr2_enabled;
1417         pipe_config->infoframes.enable |= intel_hdmi_infoframe_enable(DP_SDP_VSC);
1418
1419         if (!intel_dp->psr.psr2_enabled)
1420                 goto unlock;
1421
1422         if (HAS_PSR2_SEL_FETCH(dev_priv)) {
1423                 val = intel_de_read(dev_priv, PSR2_MAN_TRK_CTL(cpu_transcoder));
1424                 if (val & PSR2_MAN_TRK_CTL_ENABLE)
1425                         pipe_config->enable_psr2_sel_fetch = true;
1426         }
1427
1428         if (DISPLAY_VER(dev_priv) >= 12) {
1429                 val = intel_de_read(dev_priv, TRANS_EXITLINE(cpu_transcoder));
1430                 pipe_config->dc3co_exitline = REG_FIELD_GET(EXITLINE_MASK, val);
1431         }
1432 unlock:
1433         mutex_unlock(&intel_dp->psr.lock);
1434 }
1435
1436 static void intel_psr_activate(struct intel_dp *intel_dp)
1437 {
1438         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
1439         enum transcoder cpu_transcoder = intel_dp->psr.transcoder;
1440
1441         drm_WARN_ON(&dev_priv->drm,
1442                     transcoder_has_psr2(dev_priv, cpu_transcoder) &&
1443                     intel_de_read(dev_priv, EDP_PSR2_CTL(cpu_transcoder)) & EDP_PSR2_ENABLE);
1444
1445         drm_WARN_ON(&dev_priv->drm,
1446                     intel_de_read(dev_priv, psr_ctl_reg(dev_priv, cpu_transcoder)) & EDP_PSR_ENABLE);
1447
1448         drm_WARN_ON(&dev_priv->drm, intel_dp->psr.active);
1449
1450         lockdep_assert_held(&intel_dp->psr.lock);
1451
1452         /* psr1, psr2 and panel-replay are mutually exclusive.*/
1453         if (intel_dp->psr.panel_replay_enabled)
1454                 dg2_activate_panel_replay(intel_dp);
1455         else if (intel_dp->psr.psr2_enabled)
1456                 hsw_activate_psr2(intel_dp);
1457         else
1458                 hsw_activate_psr1(intel_dp);
1459
1460         intel_dp->psr.active = true;
1461 }
1462
1463 static u32 wa_16013835468_bit_get(struct intel_dp *intel_dp)
1464 {
1465         switch (intel_dp->psr.pipe) {
1466         case PIPE_A:
1467                 return LATENCY_REPORTING_REMOVED_PIPE_A;
1468         case PIPE_B:
1469                 return LATENCY_REPORTING_REMOVED_PIPE_B;
1470         case PIPE_C:
1471                 return LATENCY_REPORTING_REMOVED_PIPE_C;
1472         case PIPE_D:
1473                 return LATENCY_REPORTING_REMOVED_PIPE_D;
1474         default:
1475                 MISSING_CASE(intel_dp->psr.pipe);
1476                 return 0;
1477         }
1478 }
1479
1480 /*
1481  * Wa_16013835468
1482  * Wa_14015648006
1483  */
1484 static void wm_optimization_wa(struct intel_dp *intel_dp,
1485                                const struct intel_crtc_state *crtc_state)
1486 {
1487         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
1488         bool set_wa_bit = false;
1489
1490         /* Wa_14015648006 */
1491         if (IS_DISPLAY_VER(dev_priv, 11, 14))
1492                 set_wa_bit |= crtc_state->wm_level_disabled;
1493
1494         /* Wa_16013835468 */
1495         if (DISPLAY_VER(dev_priv) == 12)
1496                 set_wa_bit |= crtc_state->hw.adjusted_mode.crtc_vblank_start !=
1497                         crtc_state->hw.adjusted_mode.crtc_vdisplay;
1498
1499         if (set_wa_bit)
1500                 intel_de_rmw(dev_priv, GEN8_CHICKEN_DCPR_1,
1501                              0, wa_16013835468_bit_get(intel_dp));
1502         else
1503                 intel_de_rmw(dev_priv, GEN8_CHICKEN_DCPR_1,
1504                              wa_16013835468_bit_get(intel_dp), 0);
1505 }
1506
1507 static void intel_psr_enable_source(struct intel_dp *intel_dp,
1508                                     const struct intel_crtc_state *crtc_state)
1509 {
1510         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
1511         enum transcoder cpu_transcoder = intel_dp->psr.transcoder;
1512         u32 mask;
1513
1514         /*
1515          * Only HSW and BDW have PSR AUX registers that need to be setup.
1516          * SKL+ use hardcoded values PSR AUX transactions
1517          */
1518         if (DISPLAY_VER(dev_priv) < 9)
1519                 hsw_psr_setup_aux(intel_dp);
1520
1521         /*
1522          * Per Spec: Avoid continuous PSR exit by masking MEMUP and HPD also
1523          * mask LPSP to avoid dependency on other drivers that might block
1524          * runtime_pm besides preventing  other hw tracking issues now we
1525          * can rely on frontbuffer tracking.
1526          */
1527         mask = EDP_PSR_DEBUG_MASK_MEMUP |
1528                EDP_PSR_DEBUG_MASK_HPD |
1529                EDP_PSR_DEBUG_MASK_LPSP;
1530
1531         if (DISPLAY_VER(dev_priv) < 20)
1532                 mask |= EDP_PSR_DEBUG_MASK_MAX_SLEEP;
1533
1534         /*
1535          * No separate pipe reg write mask on hsw/bdw, so have to unmask all
1536          * registers in order to keep the CURSURFLIVE tricks working :(
1537          */
1538         if (IS_DISPLAY_VER(dev_priv, 9, 10))
1539                 mask |= EDP_PSR_DEBUG_MASK_DISP_REG_WRITE;
1540
1541         /* allow PSR with sprite enabled */
1542         if (IS_HASWELL(dev_priv))
1543                 mask |= EDP_PSR_DEBUG_MASK_SPRITE_ENABLE;
1544
1545         intel_de_write(dev_priv, psr_debug_reg(dev_priv, cpu_transcoder), mask);
1546
1547         psr_irq_control(intel_dp);
1548
1549         /*
1550          * TODO: if future platforms supports DC3CO in more than one
1551          * transcoder, EXITLINE will need to be unset when disabling PSR
1552          */
1553         if (intel_dp->psr.dc3co_exitline)
1554                 intel_de_rmw(dev_priv, TRANS_EXITLINE(cpu_transcoder), EXITLINE_MASK,
1555                              intel_dp->psr.dc3co_exitline << EXITLINE_SHIFT | EXITLINE_ENABLE);
1556
1557         if (HAS_PSR_HW_TRACKING(dev_priv) && HAS_PSR2_SEL_FETCH(dev_priv))
1558                 intel_de_rmw(dev_priv, CHICKEN_PAR1_1, IGNORE_PSR2_HW_TRACKING,
1559                              intel_dp->psr.psr2_sel_fetch_enabled ?
1560                              IGNORE_PSR2_HW_TRACKING : 0);
1561
1562         /*
1563          * Wa_16013835468
1564          * Wa_14015648006
1565          */
1566         wm_optimization_wa(intel_dp, crtc_state);
1567
1568         if (intel_dp->psr.psr2_enabled) {
1569                 if (DISPLAY_VER(dev_priv) == 9)
1570                         intel_de_rmw(dev_priv, CHICKEN_TRANS(cpu_transcoder), 0,
1571                                      PSR2_VSC_ENABLE_PROG_HEADER |
1572                                      PSR2_ADD_VERTICAL_LINE_COUNT);
1573
1574                 /*
1575                  * Wa_16014451276:adlp,mtl[a0,b0]
1576                  * All supported adlp panels have 1-based X granularity, this may
1577                  * cause issues if non-supported panels are used.
1578                  */
1579                 if (IS_DISPLAY_IP_STEP(dev_priv, IP_VER(14, 0), STEP_A0, STEP_B0) ||
1580                     IS_ALDERLAKE_P(dev_priv))
1581                         intel_de_rmw(dev_priv, hsw_chicken_trans_reg(dev_priv, cpu_transcoder),
1582                                      0, ADLP_1_BASED_X_GRANULARITY);
1583
1584                 /* Wa_16012604467:adlp,mtl[a0,b0] */
1585                 if (IS_DISPLAY_IP_STEP(dev_priv, IP_VER(14, 0), STEP_A0, STEP_B0))
1586                         intel_de_rmw(dev_priv,
1587                                      MTL_CLKGATE_DIS_TRANS(cpu_transcoder), 0,
1588                                      MTL_CLKGATE_DIS_TRANS_DMASC_GATING_DIS);
1589                 else if (IS_ALDERLAKE_P(dev_priv))
1590                         intel_de_rmw(dev_priv, CLKGATE_DIS_MISC, 0,
1591                                      CLKGATE_DIS_MISC_DMASC_GATING_DIS);
1592         }
1593 }
1594
1595 static bool psr_interrupt_error_check(struct intel_dp *intel_dp)
1596 {
1597         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
1598         enum transcoder cpu_transcoder = intel_dp->psr.transcoder;
1599         u32 val;
1600
1601         /*
1602          * If a PSR error happened and the driver is reloaded, the EDP_PSR_IIR
1603          * will still keep the error set even after the reset done in the
1604          * irq_preinstall and irq_uninstall hooks.
1605          * And enabling in this situation cause the screen to freeze in the
1606          * first time that PSR HW tries to activate so lets keep PSR disabled
1607          * to avoid any rendering problems.
1608          */
1609         val = intel_de_read(dev_priv, psr_iir_reg(dev_priv, cpu_transcoder));
1610         val &= psr_irq_psr_error_bit_get(intel_dp);
1611         if (val) {
1612                 intel_dp->psr.sink_not_reliable = true;
1613                 drm_dbg_kms(&dev_priv->drm,
1614                             "PSR interruption error set, not enabling PSR\n");
1615                 return false;
1616         }
1617
1618         return true;
1619 }
1620
1621 static void intel_psr_enable_locked(struct intel_dp *intel_dp,
1622                                     const struct intel_crtc_state *crtc_state)
1623 {
1624         struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
1625         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
1626         enum phy phy = intel_port_to_phy(dev_priv, dig_port->base.port);
1627         struct intel_encoder *encoder = &dig_port->base;
1628         u32 val;
1629
1630         drm_WARN_ON(&dev_priv->drm, intel_dp->psr.enabled);
1631
1632         intel_dp->psr.psr2_enabled = crtc_state->has_psr2;
1633         intel_dp->psr.panel_replay_enabled = crtc_state->has_panel_replay;
1634         intel_dp->psr.busy_frontbuffer_bits = 0;
1635         intel_dp->psr.pipe = to_intel_crtc(crtc_state->uapi.crtc)->pipe;
1636         intel_dp->psr.transcoder = crtc_state->cpu_transcoder;
1637         /* DC5/DC6 requires at least 6 idle frames */
1638         val = usecs_to_jiffies(intel_get_frame_time_us(crtc_state) * 6);
1639         intel_dp->psr.dc3co_exit_delay = val;
1640         intel_dp->psr.dc3co_exitline = crtc_state->dc3co_exitline;
1641         intel_dp->psr.psr2_sel_fetch_enabled = crtc_state->enable_psr2_sel_fetch;
1642         intel_dp->psr.psr2_sel_fetch_cff_enabled = false;
1643         intel_dp->psr.req_psr2_sdp_prior_scanline =
1644                 crtc_state->req_psr2_sdp_prior_scanline;
1645
1646         if (!psr_interrupt_error_check(intel_dp))
1647                 return;
1648
1649         if (intel_dp->psr.panel_replay_enabled)
1650                 drm_dbg_kms(&dev_priv->drm, "Enabling Panel Replay\n");
1651         else
1652                 drm_dbg_kms(&dev_priv->drm, "Enabling PSR%s\n",
1653                             intel_dp->psr.psr2_enabled ? "2" : "1");
1654
1655         intel_write_dp_vsc_sdp(encoder, crtc_state, &crtc_state->psr_vsc);
1656         intel_snps_phy_update_psr_power_state(dev_priv, phy, true);
1657         intel_psr_enable_sink(intel_dp);
1658         intel_psr_enable_source(intel_dp, crtc_state);
1659         intel_dp->psr.enabled = true;
1660         intel_dp->psr.paused = false;
1661
1662         intel_psr_activate(intel_dp);
1663 }
1664
1665 static void intel_psr_exit(struct intel_dp *intel_dp)
1666 {
1667         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
1668         enum transcoder cpu_transcoder = intel_dp->psr.transcoder;
1669         u32 val;
1670
1671         if (!intel_dp->psr.active) {
1672                 if (transcoder_has_psr2(dev_priv, cpu_transcoder)) {
1673                         val = intel_de_read(dev_priv, EDP_PSR2_CTL(cpu_transcoder));
1674                         drm_WARN_ON(&dev_priv->drm, val & EDP_PSR2_ENABLE);
1675                 }
1676
1677                 val = intel_de_read(dev_priv, psr_ctl_reg(dev_priv, cpu_transcoder));
1678                 drm_WARN_ON(&dev_priv->drm, val & EDP_PSR_ENABLE);
1679
1680                 return;
1681         }
1682
1683         if (intel_dp->psr.panel_replay_enabled) {
1684                 intel_de_rmw(dev_priv, TRANS_DP2_CTL(intel_dp->psr.transcoder),
1685                              TRANS_DP2_PANEL_REPLAY_ENABLE, 0);
1686         } else if (intel_dp->psr.psr2_enabled) {
1687                 tgl_disallow_dc3co_on_psr2_exit(intel_dp);
1688
1689                 val = intel_de_rmw(dev_priv, EDP_PSR2_CTL(cpu_transcoder),
1690                                    EDP_PSR2_ENABLE, 0);
1691
1692                 drm_WARN_ON(&dev_priv->drm, !(val & EDP_PSR2_ENABLE));
1693         } else {
1694                 val = intel_de_rmw(dev_priv, psr_ctl_reg(dev_priv, cpu_transcoder),
1695                                    EDP_PSR_ENABLE, 0);
1696
1697                 drm_WARN_ON(&dev_priv->drm, !(val & EDP_PSR_ENABLE));
1698         }
1699         intel_dp->psr.active = false;
1700 }
1701
1702 static void intel_psr_wait_exit_locked(struct intel_dp *intel_dp)
1703 {
1704         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
1705         enum transcoder cpu_transcoder = intel_dp->psr.transcoder;
1706         i915_reg_t psr_status;
1707         u32 psr_status_mask;
1708
1709         if (intel_dp->psr.psr2_enabled) {
1710                 psr_status = EDP_PSR2_STATUS(cpu_transcoder);
1711                 psr_status_mask = EDP_PSR2_STATUS_STATE_MASK;
1712         } else {
1713                 psr_status = psr_status_reg(dev_priv, cpu_transcoder);
1714                 psr_status_mask = EDP_PSR_STATUS_STATE_MASK;
1715         }
1716
1717         /* Wait till PSR is idle */
1718         if (intel_de_wait_for_clear(dev_priv, psr_status,
1719                                     psr_status_mask, 2000))
1720                 drm_err(&dev_priv->drm, "Timed out waiting PSR idle state\n");
1721 }
1722
1723 static void intel_psr_disable_locked(struct intel_dp *intel_dp)
1724 {
1725         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
1726         enum transcoder cpu_transcoder = intel_dp->psr.transcoder;
1727         enum phy phy = intel_port_to_phy(dev_priv,
1728                                          dp_to_dig_port(intel_dp)->base.port);
1729
1730         lockdep_assert_held(&intel_dp->psr.lock);
1731
1732         if (!intel_dp->psr.enabled)
1733                 return;
1734
1735         if (intel_dp->psr.panel_replay_enabled)
1736                 drm_dbg_kms(&dev_priv->drm, "Disabling Panel Replay\n");
1737         else
1738                 drm_dbg_kms(&dev_priv->drm, "Disabling PSR%s\n",
1739                             intel_dp->psr.psr2_enabled ? "2" : "1");
1740
1741         intel_psr_exit(intel_dp);
1742         intel_psr_wait_exit_locked(intel_dp);
1743
1744         /*
1745          * Wa_16013835468
1746          * Wa_14015648006
1747          */
1748         if (DISPLAY_VER(dev_priv) >= 11)
1749                 intel_de_rmw(dev_priv, GEN8_CHICKEN_DCPR_1,
1750                              wa_16013835468_bit_get(intel_dp), 0);
1751
1752         if (intel_dp->psr.psr2_enabled) {
1753                 /* Wa_16012604467:adlp,mtl[a0,b0] */
1754                 if (IS_DISPLAY_IP_STEP(dev_priv, IP_VER(14, 0), STEP_A0, STEP_B0))
1755                         intel_de_rmw(dev_priv,
1756                                      MTL_CLKGATE_DIS_TRANS(cpu_transcoder),
1757                                      MTL_CLKGATE_DIS_TRANS_DMASC_GATING_DIS, 0);
1758                 else if (IS_ALDERLAKE_P(dev_priv))
1759                         intel_de_rmw(dev_priv, CLKGATE_DIS_MISC,
1760                                      CLKGATE_DIS_MISC_DMASC_GATING_DIS, 0);
1761         }
1762
1763         intel_snps_phy_update_psr_power_state(dev_priv, phy, false);
1764
1765         /* Disable PSR on Sink */
1766         drm_dp_dpcd_writeb(&intel_dp->aux, DP_PSR_EN_CFG, 0);
1767
1768         if (intel_dp->psr.psr2_enabled)
1769                 drm_dp_dpcd_writeb(&intel_dp->aux, DP_RECEIVER_ALPM_CONFIG, 0);
1770
1771         intel_dp->psr.enabled = false;
1772         intel_dp->psr.panel_replay_enabled = false;
1773         intel_dp->psr.psr2_enabled = false;
1774         intel_dp->psr.psr2_sel_fetch_enabled = false;
1775         intel_dp->psr.psr2_sel_fetch_cff_enabled = false;
1776 }
1777
1778 /**
1779  * intel_psr_disable - Disable PSR
1780  * @intel_dp: Intel DP
1781  * @old_crtc_state: old CRTC state
1782  *
1783  * This function needs to be called before disabling pipe.
1784  */
1785 void intel_psr_disable(struct intel_dp *intel_dp,
1786                        const struct intel_crtc_state *old_crtc_state)
1787 {
1788         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
1789
1790         if (!old_crtc_state->has_psr)
1791                 return;
1792
1793         if (drm_WARN_ON(&dev_priv->drm, !CAN_PSR(intel_dp)))
1794                 return;
1795
1796         mutex_lock(&intel_dp->psr.lock);
1797
1798         intel_psr_disable_locked(intel_dp);
1799
1800         mutex_unlock(&intel_dp->psr.lock);
1801         cancel_work_sync(&intel_dp->psr.work);
1802         cancel_delayed_work_sync(&intel_dp->psr.dc3co_work);
1803 }
1804
1805 /**
1806  * intel_psr_pause - Pause PSR
1807  * @intel_dp: Intel DP
1808  *
1809  * This function need to be called after enabling psr.
1810  */
1811 void intel_psr_pause(struct intel_dp *intel_dp)
1812 {
1813         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
1814         struct intel_psr *psr = &intel_dp->psr;
1815
1816         if (!CAN_PSR(intel_dp))
1817                 return;
1818
1819         mutex_lock(&psr->lock);
1820
1821         if (!psr->enabled) {
1822                 mutex_unlock(&psr->lock);
1823                 return;
1824         }
1825
1826         /* If we ever hit this, we will need to add refcount to pause/resume */
1827         drm_WARN_ON(&dev_priv->drm, psr->paused);
1828
1829         intel_psr_exit(intel_dp);
1830         intel_psr_wait_exit_locked(intel_dp);
1831         psr->paused = true;
1832
1833         mutex_unlock(&psr->lock);
1834
1835         cancel_work_sync(&psr->work);
1836         cancel_delayed_work_sync(&psr->dc3co_work);
1837 }
1838
1839 /**
1840  * intel_psr_resume - Resume PSR
1841  * @intel_dp: Intel DP
1842  *
1843  * This function need to be called after pausing psr.
1844  */
1845 void intel_psr_resume(struct intel_dp *intel_dp)
1846 {
1847         struct intel_psr *psr = &intel_dp->psr;
1848
1849         if (!CAN_PSR(intel_dp))
1850                 return;
1851
1852         mutex_lock(&psr->lock);
1853
1854         if (!psr->paused)
1855                 goto unlock;
1856
1857         psr->paused = false;
1858         intel_psr_activate(intel_dp);
1859
1860 unlock:
1861         mutex_unlock(&psr->lock);
1862 }
1863
1864 static u32 man_trk_ctl_enable_bit_get(struct drm_i915_private *dev_priv)
1865 {
1866         return IS_ALDERLAKE_P(dev_priv) || DISPLAY_VER(dev_priv) >= 14 ? 0 :
1867                 PSR2_MAN_TRK_CTL_ENABLE;
1868 }
1869
1870 static u32 man_trk_ctl_single_full_frame_bit_get(struct drm_i915_private *dev_priv)
1871 {
1872         return IS_ALDERLAKE_P(dev_priv) || DISPLAY_VER(dev_priv) >= 14 ?
1873                ADLP_PSR2_MAN_TRK_CTL_SF_SINGLE_FULL_FRAME :
1874                PSR2_MAN_TRK_CTL_SF_SINGLE_FULL_FRAME;
1875 }
1876
1877 static u32 man_trk_ctl_partial_frame_bit_get(struct drm_i915_private *dev_priv)
1878 {
1879         return IS_ALDERLAKE_P(dev_priv) || DISPLAY_VER(dev_priv) >= 14 ?
1880                ADLP_PSR2_MAN_TRK_CTL_SF_PARTIAL_FRAME_UPDATE :
1881                PSR2_MAN_TRK_CTL_SF_PARTIAL_FRAME_UPDATE;
1882 }
1883
1884 static u32 man_trk_ctl_continuos_full_frame(struct drm_i915_private *dev_priv)
1885 {
1886         return IS_ALDERLAKE_P(dev_priv) || DISPLAY_VER(dev_priv) >= 14 ?
1887                ADLP_PSR2_MAN_TRK_CTL_SF_CONTINUOS_FULL_FRAME :
1888                PSR2_MAN_TRK_CTL_SF_CONTINUOS_FULL_FRAME;
1889 }
1890
1891 static void psr_force_hw_tracking_exit(struct intel_dp *intel_dp)
1892 {
1893         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
1894         enum transcoder cpu_transcoder = intel_dp->psr.transcoder;
1895
1896         if (intel_dp->psr.psr2_sel_fetch_enabled)
1897                 intel_de_write(dev_priv,
1898                                PSR2_MAN_TRK_CTL(cpu_transcoder),
1899                                man_trk_ctl_enable_bit_get(dev_priv) |
1900                                man_trk_ctl_partial_frame_bit_get(dev_priv) |
1901                                man_trk_ctl_single_full_frame_bit_get(dev_priv) |
1902                                man_trk_ctl_continuos_full_frame(dev_priv));
1903
1904         /*
1905          * Display WA #0884: skl+
1906          * This documented WA for bxt can be safely applied
1907          * broadly so we can force HW tracking to exit PSR
1908          * instead of disabling and re-enabling.
1909          * Workaround tells us to write 0 to CUR_SURFLIVE_A,
1910          * but it makes more sense write to the current active
1911          * pipe.
1912          *
1913          * This workaround do not exist for platforms with display 10 or newer
1914          * but testing proved that it works for up display 13, for newer
1915          * than that testing will be needed.
1916          */
1917         intel_de_write(dev_priv, CURSURFLIVE(intel_dp->psr.pipe), 0);
1918 }
1919
1920 void intel_psr2_program_trans_man_trk_ctl(const struct intel_crtc_state *crtc_state)
1921 {
1922         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1923         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
1924         struct intel_encoder *encoder;
1925
1926         if (!crtc_state->enable_psr2_sel_fetch)
1927                 return;
1928
1929         for_each_intel_encoder_mask_with_psr(&dev_priv->drm, encoder,
1930                                              crtc_state->uapi.encoder_mask) {
1931                 struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
1932
1933                 lockdep_assert_held(&intel_dp->psr.lock);
1934                 if (intel_dp->psr.psr2_sel_fetch_cff_enabled)
1935                         return;
1936                 break;
1937         }
1938
1939         intel_de_write(dev_priv, PSR2_MAN_TRK_CTL(cpu_transcoder),
1940                        crtc_state->psr2_man_track_ctl);
1941 }
1942
1943 static void psr2_man_trk_ctl_calc(struct intel_crtc_state *crtc_state,
1944                                   struct drm_rect *clip, bool full_update)
1945 {
1946         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1947         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1948         u32 val = man_trk_ctl_enable_bit_get(dev_priv);
1949
1950         /* SF partial frame enable has to be set even on full update */
1951         val |= man_trk_ctl_partial_frame_bit_get(dev_priv);
1952
1953         if (full_update) {
1954                 val |= man_trk_ctl_single_full_frame_bit_get(dev_priv);
1955                 val |= man_trk_ctl_continuos_full_frame(dev_priv);
1956                 goto exit;
1957         }
1958
1959         if (clip->y1 == -1)
1960                 goto exit;
1961
1962         if (IS_ALDERLAKE_P(dev_priv) || DISPLAY_VER(dev_priv) >= 14) {
1963                 val |= ADLP_PSR2_MAN_TRK_CTL_SU_REGION_START_ADDR(clip->y1);
1964                 val |= ADLP_PSR2_MAN_TRK_CTL_SU_REGION_END_ADDR(clip->y2 - 1);
1965         } else {
1966                 drm_WARN_ON(crtc_state->uapi.crtc->dev, clip->y1 % 4 || clip->y2 % 4);
1967
1968                 val |= PSR2_MAN_TRK_CTL_SU_REGION_START_ADDR(clip->y1 / 4 + 1);
1969                 val |= PSR2_MAN_TRK_CTL_SU_REGION_END_ADDR(clip->y2 / 4 + 1);
1970         }
1971 exit:
1972         crtc_state->psr2_man_track_ctl = val;
1973 }
1974
1975 static void clip_area_update(struct drm_rect *overlap_damage_area,
1976                              struct drm_rect *damage_area,
1977                              struct drm_rect *pipe_src)
1978 {
1979         if (!drm_rect_intersect(damage_area, pipe_src))
1980                 return;
1981
1982         if (overlap_damage_area->y1 == -1) {
1983                 overlap_damage_area->y1 = damage_area->y1;
1984                 overlap_damage_area->y2 = damage_area->y2;
1985                 return;
1986         }
1987
1988         if (damage_area->y1 < overlap_damage_area->y1)
1989                 overlap_damage_area->y1 = damage_area->y1;
1990
1991         if (damage_area->y2 > overlap_damage_area->y2)
1992                 overlap_damage_area->y2 = damage_area->y2;
1993 }
1994
1995 static void intel_psr2_sel_fetch_pipe_alignment(const struct intel_crtc_state *crtc_state,
1996                                                 struct drm_rect *pipe_clip)
1997 {
1998         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1999         const struct drm_dsc_config *vdsc_cfg = &crtc_state->dsc.config;
2000         u16 y_alignment;
2001
2002         /* ADLP aligns the SU region to vdsc slice height in case dsc is enabled */
2003         if (crtc_state->dsc.compression_enable &&
2004             (IS_ALDERLAKE_P(dev_priv) || DISPLAY_VER(dev_priv) >= 14))
2005                 y_alignment = vdsc_cfg->slice_height;
2006         else
2007                 y_alignment = crtc_state->su_y_granularity;
2008
2009         pipe_clip->y1 -= pipe_clip->y1 % y_alignment;
2010         if (pipe_clip->y2 % y_alignment)
2011                 pipe_clip->y2 = ((pipe_clip->y2 / y_alignment) + 1) * y_alignment;
2012 }
2013
2014 /*
2015  * TODO: Not clear how to handle planes with negative position,
2016  * also planes are not updated if they have a negative X
2017  * position so for now doing a full update in this cases
2018  *
2019  * Plane scaling and rotation is not supported by selective fetch and both
2020  * properties can change without a modeset, so need to be check at every
2021  * atomic commit.
2022  */
2023 static bool psr2_sel_fetch_plane_state_supported(const struct intel_plane_state *plane_state)
2024 {
2025         if (plane_state->uapi.dst.y1 < 0 ||
2026             plane_state->uapi.dst.x1 < 0 ||
2027             plane_state->scaler_id >= 0 ||
2028             plane_state->uapi.rotation != DRM_MODE_ROTATE_0)
2029                 return false;
2030
2031         return true;
2032 }
2033
2034 /*
2035  * Check for pipe properties that is not supported by selective fetch.
2036  *
2037  * TODO: pipe scaling causes a modeset but skl_update_scaler_crtc() is executed
2038  * after intel_psr_compute_config(), so for now keeping PSR2 selective fetch
2039  * enabled and going to the full update path.
2040  */
2041 static bool psr2_sel_fetch_pipe_state_supported(const struct intel_crtc_state *crtc_state)
2042 {
2043         if (crtc_state->scaler_state.scaler_id >= 0)
2044                 return false;
2045
2046         return true;
2047 }
2048
2049 int intel_psr2_sel_fetch_update(struct intel_atomic_state *state,
2050                                 struct intel_crtc *crtc)
2051 {
2052         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2053         struct intel_crtc_state *crtc_state = intel_atomic_get_new_crtc_state(state, crtc);
2054         struct drm_rect pipe_clip = { .x1 = 0, .y1 = -1, .x2 = INT_MAX, .y2 = -1 };
2055         struct intel_plane_state *new_plane_state, *old_plane_state;
2056         struct intel_plane *plane;
2057         bool full_update = false;
2058         int i, ret;
2059
2060         if (!crtc_state->enable_psr2_sel_fetch)
2061                 return 0;
2062
2063         if (!psr2_sel_fetch_pipe_state_supported(crtc_state)) {
2064                 full_update = true;
2065                 goto skip_sel_fetch_set_loop;
2066         }
2067
2068         /*
2069          * Calculate minimal selective fetch area of each plane and calculate
2070          * the pipe damaged area.
2071          * In the next loop the plane selective fetch area will actually be set
2072          * using whole pipe damaged area.
2073          */
2074         for_each_oldnew_intel_plane_in_state(state, plane, old_plane_state,
2075                                              new_plane_state, i) {
2076                 struct drm_rect src, damaged_area = { .x1 = 0, .y1 = -1,
2077                                                       .x2 = INT_MAX };
2078
2079                 if (new_plane_state->uapi.crtc != crtc_state->uapi.crtc)
2080                         continue;
2081
2082                 if (!new_plane_state->uapi.visible &&
2083                     !old_plane_state->uapi.visible)
2084                         continue;
2085
2086                 if (!psr2_sel_fetch_plane_state_supported(new_plane_state)) {
2087                         full_update = true;
2088                         break;
2089                 }
2090
2091                 /*
2092                  * If visibility or plane moved, mark the whole plane area as
2093                  * damaged as it needs to be complete redraw in the new and old
2094                  * position.
2095                  */
2096                 if (new_plane_state->uapi.visible != old_plane_state->uapi.visible ||
2097                     !drm_rect_equals(&new_plane_state->uapi.dst,
2098                                      &old_plane_state->uapi.dst)) {
2099                         if (old_plane_state->uapi.visible) {
2100                                 damaged_area.y1 = old_plane_state->uapi.dst.y1;
2101                                 damaged_area.y2 = old_plane_state->uapi.dst.y2;
2102                                 clip_area_update(&pipe_clip, &damaged_area,
2103                                                  &crtc_state->pipe_src);
2104                         }
2105
2106                         if (new_plane_state->uapi.visible) {
2107                                 damaged_area.y1 = new_plane_state->uapi.dst.y1;
2108                                 damaged_area.y2 = new_plane_state->uapi.dst.y2;
2109                                 clip_area_update(&pipe_clip, &damaged_area,
2110                                                  &crtc_state->pipe_src);
2111                         }
2112                         continue;
2113                 } else if (new_plane_state->uapi.alpha != old_plane_state->uapi.alpha) {
2114                         /* If alpha changed mark the whole plane area as damaged */
2115                         damaged_area.y1 = new_plane_state->uapi.dst.y1;
2116                         damaged_area.y2 = new_plane_state->uapi.dst.y2;
2117                         clip_area_update(&pipe_clip, &damaged_area,
2118                                          &crtc_state->pipe_src);
2119                         continue;
2120                 }
2121
2122                 src = drm_plane_state_src(&new_plane_state->uapi);
2123                 drm_rect_fp_to_int(&src, &src);
2124
2125                 if (!drm_atomic_helper_damage_merged(&old_plane_state->uapi,
2126                                                      &new_plane_state->uapi, &damaged_area))
2127                         continue;
2128
2129                 damaged_area.y1 += new_plane_state->uapi.dst.y1 - src.y1;
2130                 damaged_area.y2 += new_plane_state->uapi.dst.y1 - src.y1;
2131                 damaged_area.x1 += new_plane_state->uapi.dst.x1 - src.x1;
2132                 damaged_area.x2 += new_plane_state->uapi.dst.x1 - src.x1;
2133
2134                 clip_area_update(&pipe_clip, &damaged_area, &crtc_state->pipe_src);
2135         }
2136
2137         /*
2138          * TODO: For now we are just using full update in case
2139          * selective fetch area calculation fails. To optimize this we
2140          * should identify cases where this happens and fix the area
2141          * calculation for those.
2142          */
2143         if (pipe_clip.y1 == -1) {
2144                 drm_info_once(&dev_priv->drm,
2145                               "Selective fetch area calculation failed in pipe %c\n",
2146                               pipe_name(crtc->pipe));
2147                 full_update = true;
2148         }
2149
2150         if (full_update)
2151                 goto skip_sel_fetch_set_loop;
2152
2153         /* Wa_14014971492 */
2154         if ((IS_DISPLAY_IP_STEP(dev_priv, IP_VER(14, 0), STEP_A0, STEP_B0) ||
2155              IS_ALDERLAKE_P(dev_priv) || IS_TIGERLAKE(dev_priv)) &&
2156             crtc_state->splitter.enable)
2157                 pipe_clip.y1 = 0;
2158
2159         ret = drm_atomic_add_affected_planes(&state->base, &crtc->base);
2160         if (ret)
2161                 return ret;
2162
2163         intel_psr2_sel_fetch_pipe_alignment(crtc_state, &pipe_clip);
2164
2165         /*
2166          * Now that we have the pipe damaged area check if it intersect with
2167          * every plane, if it does set the plane selective fetch area.
2168          */
2169         for_each_oldnew_intel_plane_in_state(state, plane, old_plane_state,
2170                                              new_plane_state, i) {
2171                 struct drm_rect *sel_fetch_area, inter;
2172                 struct intel_plane *linked = new_plane_state->planar_linked_plane;
2173
2174                 if (new_plane_state->uapi.crtc != crtc_state->uapi.crtc ||
2175                     !new_plane_state->uapi.visible)
2176                         continue;
2177
2178                 inter = pipe_clip;
2179                 sel_fetch_area = &new_plane_state->psr2_sel_fetch_area;
2180                 if (!drm_rect_intersect(&inter, &new_plane_state->uapi.dst)) {
2181                         sel_fetch_area->y1 = -1;
2182                         sel_fetch_area->y2 = -1;
2183                         /*
2184                          * if plane sel fetch was previously enabled ->
2185                          * disable it
2186                          */
2187                         if (drm_rect_height(&old_plane_state->psr2_sel_fetch_area) > 0)
2188                                 crtc_state->update_planes |= BIT(plane->id);
2189
2190                         continue;
2191                 }
2192
2193                 if (!psr2_sel_fetch_plane_state_supported(new_plane_state)) {
2194                         full_update = true;
2195                         break;
2196                 }
2197
2198                 sel_fetch_area = &new_plane_state->psr2_sel_fetch_area;
2199                 sel_fetch_area->y1 = inter.y1 - new_plane_state->uapi.dst.y1;
2200                 sel_fetch_area->y2 = inter.y2 - new_plane_state->uapi.dst.y1;
2201                 crtc_state->update_planes |= BIT(plane->id);
2202
2203                 /*
2204                  * Sel_fetch_area is calculated for UV plane. Use
2205                  * same area for Y plane as well.
2206                  */
2207                 if (linked) {
2208                         struct intel_plane_state *linked_new_plane_state;
2209                         struct drm_rect *linked_sel_fetch_area;
2210
2211                         linked_new_plane_state = intel_atomic_get_plane_state(state, linked);
2212                         if (IS_ERR(linked_new_plane_state))
2213                                 return PTR_ERR(linked_new_plane_state);
2214
2215                         linked_sel_fetch_area = &linked_new_plane_state->psr2_sel_fetch_area;
2216                         linked_sel_fetch_area->y1 = sel_fetch_area->y1;
2217                         linked_sel_fetch_area->y2 = sel_fetch_area->y2;
2218                         crtc_state->update_planes |= BIT(linked->id);
2219                 }
2220         }
2221
2222 skip_sel_fetch_set_loop:
2223         psr2_man_trk_ctl_calc(crtc_state, &pipe_clip, full_update);
2224         return 0;
2225 }
2226
2227 void intel_psr_pre_plane_update(struct intel_atomic_state *state,
2228                                 struct intel_crtc *crtc)
2229 {
2230         struct drm_i915_private *i915 = to_i915(state->base.dev);
2231         const struct intel_crtc_state *old_crtc_state =
2232                 intel_atomic_get_old_crtc_state(state, crtc);
2233         const struct intel_crtc_state *new_crtc_state =
2234                 intel_atomic_get_new_crtc_state(state, crtc);
2235         struct intel_encoder *encoder;
2236
2237         if (!HAS_PSR(i915))
2238                 return;
2239
2240         for_each_intel_encoder_mask_with_psr(state->base.dev, encoder,
2241                                              old_crtc_state->uapi.encoder_mask) {
2242                 struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
2243                 struct intel_psr *psr = &intel_dp->psr;
2244                 bool needs_to_disable = false;
2245
2246                 mutex_lock(&psr->lock);
2247
2248                 /*
2249                  * Reasons to disable:
2250                  * - PSR disabled in new state
2251                  * - All planes will go inactive
2252                  * - Changing between PSR versions
2253                  * - Display WA #1136: skl, bxt
2254                  */
2255                 needs_to_disable |= intel_crtc_needs_modeset(new_crtc_state);
2256                 needs_to_disable |= !new_crtc_state->has_psr;
2257                 needs_to_disable |= !new_crtc_state->active_planes;
2258                 needs_to_disable |= new_crtc_state->has_psr2 != psr->psr2_enabled;
2259                 needs_to_disable |= DISPLAY_VER(i915) < 11 &&
2260                         new_crtc_state->wm_level_disabled;
2261
2262                 if (psr->enabled && needs_to_disable)
2263                         intel_psr_disable_locked(intel_dp);
2264                 else if (psr->enabled && new_crtc_state->wm_level_disabled)
2265                         /* Wa_14015648006 */
2266                         wm_optimization_wa(intel_dp, new_crtc_state);
2267
2268                 mutex_unlock(&psr->lock);
2269         }
2270 }
2271
2272 void intel_psr_post_plane_update(struct intel_atomic_state *state,
2273                                  struct intel_crtc *crtc)
2274 {
2275         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2276         const struct intel_crtc_state *crtc_state =
2277                 intel_atomic_get_new_crtc_state(state, crtc);
2278         struct intel_encoder *encoder;
2279
2280         if (!(crtc_state->has_psr || crtc_state->has_panel_replay))
2281                 return;
2282
2283         for_each_intel_encoder_mask_with_psr(state->base.dev, encoder,
2284                                              crtc_state->uapi.encoder_mask) {
2285                 struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
2286                 struct intel_psr *psr = &intel_dp->psr;
2287                 bool keep_disabled = false;
2288
2289                 mutex_lock(&psr->lock);
2290
2291                 drm_WARN_ON(&dev_priv->drm, psr->enabled && !crtc_state->active_planes);
2292
2293                 keep_disabled |= psr->sink_not_reliable;
2294                 keep_disabled |= !crtc_state->active_planes;
2295
2296                 /* Display WA #1136: skl, bxt */
2297                 keep_disabled |= DISPLAY_VER(dev_priv) < 11 &&
2298                         crtc_state->wm_level_disabled;
2299
2300                 if (!psr->enabled && !keep_disabled)
2301                         intel_psr_enable_locked(intel_dp, crtc_state);
2302                 else if (psr->enabled && !crtc_state->wm_level_disabled)
2303                         /* Wa_14015648006 */
2304                         wm_optimization_wa(intel_dp, crtc_state);
2305
2306                 /* Force a PSR exit when enabling CRC to avoid CRC timeouts */
2307                 if (crtc_state->crc_enabled && psr->enabled)
2308                         psr_force_hw_tracking_exit(intel_dp);
2309
2310                 /*
2311                  * Clear possible busy bits in case we have
2312                  * invalidate -> flip -> flush sequence.
2313                  */
2314                 intel_dp->psr.busy_frontbuffer_bits = 0;
2315
2316                 mutex_unlock(&psr->lock);
2317         }
2318 }
2319
2320 static int _psr2_ready_for_pipe_update_locked(struct intel_dp *intel_dp)
2321 {
2322         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
2323         enum transcoder cpu_transcoder = intel_dp->psr.transcoder;
2324
2325         /*
2326          * Any state lower than EDP_PSR2_STATUS_STATE_DEEP_SLEEP is enough.
2327          * As all higher states has bit 4 of PSR2 state set we can just wait for
2328          * EDP_PSR2_STATUS_STATE_DEEP_SLEEP to be cleared.
2329          */
2330         return intel_de_wait_for_clear(dev_priv,
2331                                        EDP_PSR2_STATUS(cpu_transcoder),
2332                                        EDP_PSR2_STATUS_STATE_DEEP_SLEEP, 50);
2333 }
2334
2335 static int _psr1_ready_for_pipe_update_locked(struct intel_dp *intel_dp)
2336 {
2337         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
2338         enum transcoder cpu_transcoder = intel_dp->psr.transcoder;
2339
2340         /*
2341          * From bspec: Panel Self Refresh (BDW+)
2342          * Max. time for PSR to idle = Inverse of the refresh rate + 6 ms of
2343          * exit training time + 1.5 ms of aux channel handshake. 50 ms is
2344          * defensive enough to cover everything.
2345          */
2346         return intel_de_wait_for_clear(dev_priv,
2347                                        psr_status_reg(dev_priv, cpu_transcoder),
2348                                        EDP_PSR_STATUS_STATE_MASK, 50);
2349 }
2350
2351 /**
2352  * intel_psr_wait_for_idle_locked - wait for PSR be ready for a pipe update
2353  * @new_crtc_state: new CRTC state
2354  *
2355  * This function is expected to be called from pipe_update_start() where it is
2356  * not expected to race with PSR enable or disable.
2357  */
2358 void intel_psr_wait_for_idle_locked(const struct intel_crtc_state *new_crtc_state)
2359 {
2360         struct drm_i915_private *dev_priv = to_i915(new_crtc_state->uapi.crtc->dev);
2361         struct intel_encoder *encoder;
2362
2363         if (!new_crtc_state->has_psr)
2364                 return;
2365
2366         for_each_intel_encoder_mask_with_psr(&dev_priv->drm, encoder,
2367                                              new_crtc_state->uapi.encoder_mask) {
2368                 struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
2369                 int ret;
2370
2371                 lockdep_assert_held(&intel_dp->psr.lock);
2372
2373                 if (!intel_dp->psr.enabled)
2374                         continue;
2375
2376                 if (intel_dp->psr.psr2_enabled)
2377                         ret = _psr2_ready_for_pipe_update_locked(intel_dp);
2378                 else
2379                         ret = _psr1_ready_for_pipe_update_locked(intel_dp);
2380
2381                 if (ret)
2382                         drm_err(&dev_priv->drm, "PSR wait timed out, atomic update may fail\n");
2383         }
2384 }
2385
2386 static bool __psr_wait_for_idle_locked(struct intel_dp *intel_dp)
2387 {
2388         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
2389         enum transcoder cpu_transcoder = intel_dp->psr.transcoder;
2390         i915_reg_t reg;
2391         u32 mask;
2392         int err;
2393
2394         if (!intel_dp->psr.enabled)
2395                 return false;
2396
2397         if (intel_dp->psr.psr2_enabled) {
2398                 reg = EDP_PSR2_STATUS(cpu_transcoder);
2399                 mask = EDP_PSR2_STATUS_STATE_MASK;
2400         } else {
2401                 reg = psr_status_reg(dev_priv, cpu_transcoder);
2402                 mask = EDP_PSR_STATUS_STATE_MASK;
2403         }
2404
2405         mutex_unlock(&intel_dp->psr.lock);
2406
2407         err = intel_de_wait_for_clear(dev_priv, reg, mask, 50);
2408         if (err)
2409                 drm_err(&dev_priv->drm,
2410                         "Timed out waiting for PSR Idle for re-enable\n");
2411
2412         /* After the unlocked wait, verify that PSR is still wanted! */
2413         mutex_lock(&intel_dp->psr.lock);
2414         return err == 0 && intel_dp->psr.enabled;
2415 }
2416
2417 static int intel_psr_fastset_force(struct drm_i915_private *dev_priv)
2418 {
2419         struct drm_connector_list_iter conn_iter;
2420         struct drm_modeset_acquire_ctx ctx;
2421         struct drm_atomic_state *state;
2422         struct drm_connector *conn;
2423         int err = 0;
2424
2425         state = drm_atomic_state_alloc(&dev_priv->drm);
2426         if (!state)
2427                 return -ENOMEM;
2428
2429         drm_modeset_acquire_init(&ctx, DRM_MODESET_ACQUIRE_INTERRUPTIBLE);
2430
2431         state->acquire_ctx = &ctx;
2432         to_intel_atomic_state(state)->internal = true;
2433
2434 retry:
2435         drm_connector_list_iter_begin(&dev_priv->drm, &conn_iter);
2436         drm_for_each_connector_iter(conn, &conn_iter) {
2437                 struct drm_connector_state *conn_state;
2438                 struct drm_crtc_state *crtc_state;
2439
2440                 if (conn->connector_type != DRM_MODE_CONNECTOR_eDP)
2441                         continue;
2442
2443                 conn_state = drm_atomic_get_connector_state(state, conn);
2444                 if (IS_ERR(conn_state)) {
2445                         err = PTR_ERR(conn_state);
2446                         break;
2447                 }
2448
2449                 if (!conn_state->crtc)
2450                         continue;
2451
2452                 crtc_state = drm_atomic_get_crtc_state(state, conn_state->crtc);
2453                 if (IS_ERR(crtc_state)) {
2454                         err = PTR_ERR(crtc_state);
2455                         break;
2456                 }
2457
2458                 /* Mark mode as changed to trigger a pipe->update() */
2459                 crtc_state->mode_changed = true;
2460         }
2461         drm_connector_list_iter_end(&conn_iter);
2462
2463         if (err == 0)
2464                 err = drm_atomic_commit(state);
2465
2466         if (err == -EDEADLK) {
2467                 drm_atomic_state_clear(state);
2468                 err = drm_modeset_backoff(&ctx);
2469                 if (!err)
2470                         goto retry;
2471         }
2472
2473         drm_modeset_drop_locks(&ctx);
2474         drm_modeset_acquire_fini(&ctx);
2475         drm_atomic_state_put(state);
2476
2477         return err;
2478 }
2479
2480 int intel_psr_debug_set(struct intel_dp *intel_dp, u64 val)
2481 {
2482         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
2483         const u32 mode = val & I915_PSR_DEBUG_MODE_MASK;
2484         u32 old_mode;
2485         int ret;
2486
2487         if (val & ~(I915_PSR_DEBUG_IRQ | I915_PSR_DEBUG_MODE_MASK) ||
2488             mode > I915_PSR_DEBUG_ENABLE_SEL_FETCH) {
2489                 drm_dbg_kms(&dev_priv->drm, "Invalid debug mask %llx\n", val);
2490                 return -EINVAL;
2491         }
2492
2493         ret = mutex_lock_interruptible(&intel_dp->psr.lock);
2494         if (ret)
2495                 return ret;
2496
2497         old_mode = intel_dp->psr.debug & I915_PSR_DEBUG_MODE_MASK;
2498         intel_dp->psr.debug = val;
2499
2500         /*
2501          * Do it right away if it's already enabled, otherwise it will be done
2502          * when enabling the source.
2503          */
2504         if (intel_dp->psr.enabled)
2505                 psr_irq_control(intel_dp);
2506
2507         mutex_unlock(&intel_dp->psr.lock);
2508
2509         if (old_mode != mode)
2510                 ret = intel_psr_fastset_force(dev_priv);
2511
2512         return ret;
2513 }
2514
2515 static void intel_psr_handle_irq(struct intel_dp *intel_dp)
2516 {
2517         struct intel_psr *psr = &intel_dp->psr;
2518
2519         intel_psr_disable_locked(intel_dp);
2520         psr->sink_not_reliable = true;
2521         /* let's make sure that sink is awaken */
2522         drm_dp_dpcd_writeb(&intel_dp->aux, DP_SET_POWER, DP_SET_POWER_D0);
2523 }
2524
2525 static void intel_psr_work(struct work_struct *work)
2526 {
2527         struct intel_dp *intel_dp =
2528                 container_of(work, typeof(*intel_dp), psr.work);
2529
2530         mutex_lock(&intel_dp->psr.lock);
2531
2532         if (!intel_dp->psr.enabled)
2533                 goto unlock;
2534
2535         if (READ_ONCE(intel_dp->psr.irq_aux_error))
2536                 intel_psr_handle_irq(intel_dp);
2537
2538         /*
2539          * We have to make sure PSR is ready for re-enable
2540          * otherwise it keeps disabled until next full enable/disable cycle.
2541          * PSR might take some time to get fully disabled
2542          * and be ready for re-enable.
2543          */
2544         if (!__psr_wait_for_idle_locked(intel_dp))
2545                 goto unlock;
2546
2547         /*
2548          * The delayed work can race with an invalidate hence we need to
2549          * recheck. Since psr_flush first clears this and then reschedules we
2550          * won't ever miss a flush when bailing out here.
2551          */
2552         if (intel_dp->psr.busy_frontbuffer_bits || intel_dp->psr.active)
2553                 goto unlock;
2554
2555         intel_psr_activate(intel_dp);
2556 unlock:
2557         mutex_unlock(&intel_dp->psr.lock);
2558 }
2559
2560 static void _psr_invalidate_handle(struct intel_dp *intel_dp)
2561 {
2562         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
2563         enum transcoder cpu_transcoder = intel_dp->psr.transcoder;
2564
2565         if (intel_dp->psr.psr2_sel_fetch_enabled) {
2566                 u32 val;
2567
2568                 if (intel_dp->psr.psr2_sel_fetch_cff_enabled) {
2569                         /* Send one update otherwise lag is observed in screen */
2570                         intel_de_write(dev_priv, CURSURFLIVE(intel_dp->psr.pipe), 0);
2571                         return;
2572                 }
2573
2574                 val = man_trk_ctl_enable_bit_get(dev_priv) |
2575                       man_trk_ctl_partial_frame_bit_get(dev_priv) |
2576                       man_trk_ctl_continuos_full_frame(dev_priv);
2577                 intel_de_write(dev_priv, PSR2_MAN_TRK_CTL(cpu_transcoder), val);
2578                 intel_de_write(dev_priv, CURSURFLIVE(intel_dp->psr.pipe), 0);
2579                 intel_dp->psr.psr2_sel_fetch_cff_enabled = true;
2580         } else {
2581                 intel_psr_exit(intel_dp);
2582         }
2583 }
2584
2585 /**
2586  * intel_psr_invalidate - Invalidate PSR
2587  * @dev_priv: i915 device
2588  * @frontbuffer_bits: frontbuffer plane tracking bits
2589  * @origin: which operation caused the invalidate
2590  *
2591  * Since the hardware frontbuffer tracking has gaps we need to integrate
2592  * with the software frontbuffer tracking. This function gets called every
2593  * time frontbuffer rendering starts and a buffer gets dirtied. PSR must be
2594  * disabled if the frontbuffer mask contains a buffer relevant to PSR.
2595  *
2596  * Dirty frontbuffers relevant to PSR are tracked in busy_frontbuffer_bits."
2597  */
2598 void intel_psr_invalidate(struct drm_i915_private *dev_priv,
2599                           unsigned frontbuffer_bits, enum fb_op_origin origin)
2600 {
2601         struct intel_encoder *encoder;
2602
2603         if (origin == ORIGIN_FLIP)
2604                 return;
2605
2606         for_each_intel_encoder_with_psr(&dev_priv->drm, encoder) {
2607                 unsigned int pipe_frontbuffer_bits = frontbuffer_bits;
2608                 struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
2609
2610                 mutex_lock(&intel_dp->psr.lock);
2611                 if (!intel_dp->psr.enabled) {
2612                         mutex_unlock(&intel_dp->psr.lock);
2613                         continue;
2614                 }
2615
2616                 pipe_frontbuffer_bits &=
2617                         INTEL_FRONTBUFFER_ALL_MASK(intel_dp->psr.pipe);
2618                 intel_dp->psr.busy_frontbuffer_bits |= pipe_frontbuffer_bits;
2619
2620                 if (pipe_frontbuffer_bits)
2621                         _psr_invalidate_handle(intel_dp);
2622
2623                 mutex_unlock(&intel_dp->psr.lock);
2624         }
2625 }
2626 /*
2627  * When we will be completely rely on PSR2 S/W tracking in future,
2628  * intel_psr_flush() will invalidate and flush the PSR for ORIGIN_FLIP
2629  * event also therefore tgl_dc3co_flush_locked() require to be changed
2630  * accordingly in future.
2631  */
2632 static void
2633 tgl_dc3co_flush_locked(struct intel_dp *intel_dp, unsigned int frontbuffer_bits,
2634                        enum fb_op_origin origin)
2635 {
2636         struct drm_i915_private *i915 = dp_to_i915(intel_dp);
2637
2638         if (!intel_dp->psr.dc3co_exitline || !intel_dp->psr.psr2_enabled ||
2639             !intel_dp->psr.active)
2640                 return;
2641
2642         /*
2643          * At every frontbuffer flush flip event modified delay of delayed work,
2644          * when delayed work schedules that means display has been idle.
2645          */
2646         if (!(frontbuffer_bits &
2647             INTEL_FRONTBUFFER_ALL_MASK(intel_dp->psr.pipe)))
2648                 return;
2649
2650         tgl_psr2_enable_dc3co(intel_dp);
2651         mod_delayed_work(i915->unordered_wq, &intel_dp->psr.dc3co_work,
2652                          intel_dp->psr.dc3co_exit_delay);
2653 }
2654
2655 static void _psr_flush_handle(struct intel_dp *intel_dp)
2656 {
2657         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
2658         enum transcoder cpu_transcoder = intel_dp->psr.transcoder;
2659
2660         if (intel_dp->psr.psr2_sel_fetch_enabled) {
2661                 if (intel_dp->psr.psr2_sel_fetch_cff_enabled) {
2662                         /* can we turn CFF off? */
2663                         if (intel_dp->psr.busy_frontbuffer_bits == 0) {
2664                                 u32 val = man_trk_ctl_enable_bit_get(dev_priv) |
2665                                         man_trk_ctl_partial_frame_bit_get(dev_priv) |
2666                                         man_trk_ctl_single_full_frame_bit_get(dev_priv) |
2667                                         man_trk_ctl_continuos_full_frame(dev_priv);
2668
2669                                 /*
2670                                  * Set psr2_sel_fetch_cff_enabled as false to allow selective
2671                                  * updates. Still keep cff bit enabled as we don't have proper
2672                                  * SU configuration in case update is sent for any reason after
2673                                  * sff bit gets cleared by the HW on next vblank.
2674                                  */
2675                                 intel_de_write(dev_priv, PSR2_MAN_TRK_CTL(cpu_transcoder),
2676                                                val);
2677                                 intel_de_write(dev_priv, CURSURFLIVE(intel_dp->psr.pipe), 0);
2678                                 intel_dp->psr.psr2_sel_fetch_cff_enabled = false;
2679                         }
2680                 } else {
2681                         /*
2682                          * continuous full frame is disabled, only a single full
2683                          * frame is required
2684                          */
2685                         psr_force_hw_tracking_exit(intel_dp);
2686                 }
2687         } else {
2688                 psr_force_hw_tracking_exit(intel_dp);
2689
2690                 if (!intel_dp->psr.active && !intel_dp->psr.busy_frontbuffer_bits)
2691                         queue_work(dev_priv->unordered_wq, &intel_dp->psr.work);
2692         }
2693 }
2694
2695 /**
2696  * intel_psr_flush - Flush PSR
2697  * @dev_priv: i915 device
2698  * @frontbuffer_bits: frontbuffer plane tracking bits
2699  * @origin: which operation caused the flush
2700  *
2701  * Since the hardware frontbuffer tracking has gaps we need to integrate
2702  * with the software frontbuffer tracking. This function gets called every
2703  * time frontbuffer rendering has completed and flushed out to memory. PSR
2704  * can be enabled again if no other frontbuffer relevant to PSR is dirty.
2705  *
2706  * Dirty frontbuffers relevant to PSR are tracked in busy_frontbuffer_bits.
2707  */
2708 void intel_psr_flush(struct drm_i915_private *dev_priv,
2709                      unsigned frontbuffer_bits, enum fb_op_origin origin)
2710 {
2711         struct intel_encoder *encoder;
2712
2713         for_each_intel_encoder_with_psr(&dev_priv->drm, encoder) {
2714                 unsigned int pipe_frontbuffer_bits = frontbuffer_bits;
2715                 struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
2716
2717                 mutex_lock(&intel_dp->psr.lock);
2718                 if (!intel_dp->psr.enabled) {
2719                         mutex_unlock(&intel_dp->psr.lock);
2720                         continue;
2721                 }
2722
2723                 pipe_frontbuffer_bits &=
2724                         INTEL_FRONTBUFFER_ALL_MASK(intel_dp->psr.pipe);
2725                 intel_dp->psr.busy_frontbuffer_bits &= ~pipe_frontbuffer_bits;
2726
2727                 /*
2728                  * If the PSR is paused by an explicit intel_psr_paused() call,
2729                  * we have to ensure that the PSR is not activated until
2730                  * intel_psr_resume() is called.
2731                  */
2732                 if (intel_dp->psr.paused)
2733                         goto unlock;
2734
2735                 if (origin == ORIGIN_FLIP ||
2736                     (origin == ORIGIN_CURSOR_UPDATE &&
2737                      !intel_dp->psr.psr2_sel_fetch_enabled)) {
2738                         tgl_dc3co_flush_locked(intel_dp, frontbuffer_bits, origin);
2739                         goto unlock;
2740                 }
2741
2742                 if (pipe_frontbuffer_bits == 0)
2743                         goto unlock;
2744
2745                 /* By definition flush = invalidate + flush */
2746                 _psr_flush_handle(intel_dp);
2747 unlock:
2748                 mutex_unlock(&intel_dp->psr.lock);
2749         }
2750 }
2751
2752 /**
2753  * intel_psr_init - Init basic PSR work and mutex.
2754  * @intel_dp: Intel DP
2755  *
2756  * This function is called after the initializing connector.
2757  * (the initializing of connector treats the handling of connector capabilities)
2758  * And it initializes basic PSR stuff for each DP Encoder.
2759  */
2760 void intel_psr_init(struct intel_dp *intel_dp)
2761 {
2762         struct intel_connector *connector = intel_dp->attached_connector;
2763         struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
2764         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
2765
2766         if (!(HAS_PSR(dev_priv) || HAS_DP20(dev_priv)))
2767                 return;
2768
2769         if (!intel_dp_is_edp(intel_dp))
2770                 intel_psr_init_dpcd(intel_dp);
2771
2772         /*
2773          * HSW spec explicitly says PSR is tied to port A.
2774          * BDW+ platforms have a instance of PSR registers per transcoder but
2775          * BDW, GEN9 and GEN11 are not validated by HW team in other transcoder
2776          * than eDP one.
2777          * For now it only supports one instance of PSR for BDW, GEN9 and GEN11.
2778          * So lets keep it hardcoded to PORT_A for BDW, GEN9 and GEN11.
2779          * But GEN12 supports a instance of PSR registers per transcoder.
2780          */
2781         if (DISPLAY_VER(dev_priv) < 12 && dig_port->base.port != PORT_A) {
2782                 drm_dbg_kms(&dev_priv->drm,
2783                             "PSR condition failed: Port not supported\n");
2784                 return;
2785         }
2786
2787         if (HAS_DP20(dev_priv) && !intel_dp_is_edp(intel_dp))
2788                 intel_dp->psr.source_panel_replay_support = true;
2789         else
2790                 intel_dp->psr.source_support = true;
2791
2792         /* Set link_standby x link_off defaults */
2793         if (DISPLAY_VER(dev_priv) < 12)
2794                 /* For new platforms up to TGL let's respect VBT back again */
2795                 intel_dp->psr.link_standby = connector->panel.vbt.psr.full_link;
2796
2797         INIT_WORK(&intel_dp->psr.work, intel_psr_work);
2798         INIT_DELAYED_WORK(&intel_dp->psr.dc3co_work, tgl_dc3co_disable_work);
2799         mutex_init(&intel_dp->psr.lock);
2800 }
2801
2802 static int psr_get_status_and_error_status(struct intel_dp *intel_dp,
2803                                            u8 *status, u8 *error_status)
2804 {
2805         struct drm_dp_aux *aux = &intel_dp->aux;
2806         int ret;
2807         unsigned int offset;
2808
2809         offset = intel_dp->psr.panel_replay_enabled ?
2810                  DP_SINK_DEVICE_PR_AND_FRAME_LOCK_STATUS : DP_PSR_STATUS;
2811
2812         ret = drm_dp_dpcd_readb(aux, offset, status);
2813         if (ret != 1)
2814                 return ret;
2815
2816         offset = intel_dp->psr.panel_replay_enabled ?
2817                  DP_PANEL_REPLAY_ERROR_STATUS : DP_PSR_ERROR_STATUS;
2818
2819         ret = drm_dp_dpcd_readb(aux, offset, error_status);
2820         if (ret != 1)
2821                 return ret;
2822
2823         *status = *status & DP_PSR_SINK_STATE_MASK;
2824
2825         return 0;
2826 }
2827
2828 static void psr_alpm_check(struct intel_dp *intel_dp)
2829 {
2830         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
2831         struct drm_dp_aux *aux = &intel_dp->aux;
2832         struct intel_psr *psr = &intel_dp->psr;
2833         u8 val;
2834         int r;
2835
2836         if (!psr->psr2_enabled)
2837                 return;
2838
2839         r = drm_dp_dpcd_readb(aux, DP_RECEIVER_ALPM_STATUS, &val);
2840         if (r != 1) {
2841                 drm_err(&dev_priv->drm, "Error reading ALPM status\n");
2842                 return;
2843         }
2844
2845         if (val & DP_ALPM_LOCK_TIMEOUT_ERROR) {
2846                 intel_psr_disable_locked(intel_dp);
2847                 psr->sink_not_reliable = true;
2848                 drm_dbg_kms(&dev_priv->drm,
2849                             "ALPM lock timeout error, disabling PSR\n");
2850
2851                 /* Clearing error */
2852                 drm_dp_dpcd_writeb(aux, DP_RECEIVER_ALPM_STATUS, val);
2853         }
2854 }
2855
2856 static void psr_capability_changed_check(struct intel_dp *intel_dp)
2857 {
2858         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
2859         struct intel_psr *psr = &intel_dp->psr;
2860         u8 val;
2861         int r;
2862
2863         r = drm_dp_dpcd_readb(&intel_dp->aux, DP_PSR_ESI, &val);
2864         if (r != 1) {
2865                 drm_err(&dev_priv->drm, "Error reading DP_PSR_ESI\n");
2866                 return;
2867         }
2868
2869         if (val & DP_PSR_CAPS_CHANGE) {
2870                 intel_psr_disable_locked(intel_dp);
2871                 psr->sink_not_reliable = true;
2872                 drm_dbg_kms(&dev_priv->drm,
2873                             "Sink PSR capability changed, disabling PSR\n");
2874
2875                 /* Clearing it */
2876                 drm_dp_dpcd_writeb(&intel_dp->aux, DP_PSR_ESI, val);
2877         }
2878 }
2879
2880 void intel_psr_short_pulse(struct intel_dp *intel_dp)
2881 {
2882         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
2883         struct intel_psr *psr = &intel_dp->psr;
2884         u8 status, error_status;
2885         const u8 errors = DP_PSR_RFB_STORAGE_ERROR |
2886                           DP_PSR_VSC_SDP_UNCORRECTABLE_ERROR |
2887                           DP_PSR_LINK_CRC_ERROR;
2888
2889         if (!CAN_PSR(intel_dp))
2890                 return;
2891
2892         mutex_lock(&psr->lock);
2893
2894         if (!psr->enabled)
2895                 goto exit;
2896
2897         if (psr_get_status_and_error_status(intel_dp, &status, &error_status)) {
2898                 drm_err(&dev_priv->drm,
2899                         "Error reading PSR status or error status\n");
2900                 goto exit;
2901         }
2902
2903         if (status == DP_PSR_SINK_INTERNAL_ERROR || (error_status & errors)) {
2904                 intel_psr_disable_locked(intel_dp);
2905                 psr->sink_not_reliable = true;
2906         }
2907
2908         if (status == DP_PSR_SINK_INTERNAL_ERROR && !error_status)
2909                 drm_dbg_kms(&dev_priv->drm,
2910                             "PSR sink internal error, disabling PSR\n");
2911         if (error_status & DP_PSR_RFB_STORAGE_ERROR)
2912                 drm_dbg_kms(&dev_priv->drm,
2913                             "PSR RFB storage error, disabling PSR\n");
2914         if (error_status & DP_PSR_VSC_SDP_UNCORRECTABLE_ERROR)
2915                 drm_dbg_kms(&dev_priv->drm,
2916                             "PSR VSC SDP uncorrectable error, disabling PSR\n");
2917         if (error_status & DP_PSR_LINK_CRC_ERROR)
2918                 drm_dbg_kms(&dev_priv->drm,
2919                             "PSR Link CRC error, disabling PSR\n");
2920
2921         if (error_status & ~errors)
2922                 drm_err(&dev_priv->drm,
2923                         "PSR_ERROR_STATUS unhandled errors %x\n",
2924                         error_status & ~errors);
2925         /* clear status register */
2926         drm_dp_dpcd_writeb(&intel_dp->aux, DP_PSR_ERROR_STATUS, error_status);
2927
2928         psr_alpm_check(intel_dp);
2929         psr_capability_changed_check(intel_dp);
2930
2931 exit:
2932         mutex_unlock(&psr->lock);
2933 }
2934
2935 bool intel_psr_enabled(struct intel_dp *intel_dp)
2936 {
2937         bool ret;
2938
2939         if (!CAN_PSR(intel_dp))
2940                 return false;
2941
2942         mutex_lock(&intel_dp->psr.lock);
2943         ret = intel_dp->psr.enabled;
2944         mutex_unlock(&intel_dp->psr.lock);
2945
2946         return ret;
2947 }
2948
2949 /**
2950  * intel_psr_lock - grab PSR lock
2951  * @crtc_state: the crtc state
2952  *
2953  * This is initially meant to be used by around CRTC update, when
2954  * vblank sensitive registers are updated and we need grab the lock
2955  * before it to avoid vblank evasion.
2956  */
2957 void intel_psr_lock(const struct intel_crtc_state *crtc_state)
2958 {
2959         struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
2960         struct intel_encoder *encoder;
2961
2962         if (!crtc_state->has_psr)
2963                 return;
2964
2965         for_each_intel_encoder_mask_with_psr(&i915->drm, encoder,
2966                                              crtc_state->uapi.encoder_mask) {
2967                 struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
2968
2969                 mutex_lock(&intel_dp->psr.lock);
2970                 break;
2971         }
2972 }
2973
2974 /**
2975  * intel_psr_unlock - release PSR lock
2976  * @crtc_state: the crtc state
2977  *
2978  * Release the PSR lock that was held during pipe update.
2979  */
2980 void intel_psr_unlock(const struct intel_crtc_state *crtc_state)
2981 {
2982         struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
2983         struct intel_encoder *encoder;
2984
2985         if (!crtc_state->has_psr)
2986                 return;
2987
2988         for_each_intel_encoder_mask_with_psr(&i915->drm, encoder,
2989                                              crtc_state->uapi.encoder_mask) {
2990                 struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
2991
2992                 mutex_unlock(&intel_dp->psr.lock);
2993                 break;
2994         }
2995 }
2996
2997 static void
2998 psr_source_status(struct intel_dp *intel_dp, struct seq_file *m)
2999 {
3000         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
3001         enum transcoder cpu_transcoder = intel_dp->psr.transcoder;
3002         const char *status = "unknown";
3003         u32 val, status_val;
3004
3005         if (intel_dp->psr.psr2_enabled) {
3006                 static const char * const live_status[] = {
3007                         "IDLE",
3008                         "CAPTURE",
3009                         "CAPTURE_FS",
3010                         "SLEEP",
3011                         "BUFON_FW",
3012                         "ML_UP",
3013                         "SU_STANDBY",
3014                         "FAST_SLEEP",
3015                         "DEEP_SLEEP",
3016                         "BUF_ON",
3017                         "TG_ON"
3018                 };
3019                 val = intel_de_read(dev_priv, EDP_PSR2_STATUS(cpu_transcoder));
3020                 status_val = REG_FIELD_GET(EDP_PSR2_STATUS_STATE_MASK, val);
3021                 if (status_val < ARRAY_SIZE(live_status))
3022                         status = live_status[status_val];
3023         } else {
3024                 static const char * const live_status[] = {
3025                         "IDLE",
3026                         "SRDONACK",
3027                         "SRDENT",
3028                         "BUFOFF",
3029                         "BUFON",
3030                         "AUXACK",
3031                         "SRDOFFACK",
3032                         "SRDENT_ON",
3033                 };
3034                 val = intel_de_read(dev_priv, psr_status_reg(dev_priv, cpu_transcoder));
3035                 status_val = REG_FIELD_GET(EDP_PSR_STATUS_STATE_MASK, val);
3036                 if (status_val < ARRAY_SIZE(live_status))
3037                         status = live_status[status_val];
3038         }
3039
3040         seq_printf(m, "Source PSR/PanelReplay status: %s [0x%08x]\n", status, val);
3041 }
3042
3043 static int intel_psr_status(struct seq_file *m, struct intel_dp *intel_dp)
3044 {
3045         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
3046         enum transcoder cpu_transcoder = intel_dp->psr.transcoder;
3047         struct intel_psr *psr = &intel_dp->psr;
3048         intel_wakeref_t wakeref;
3049         const char *status;
3050         bool enabled;
3051         u32 val;
3052
3053         seq_printf(m, "Sink support: PSR = %s",
3054                    str_yes_no(psr->sink_support));
3055
3056         if (psr->sink_support)
3057                 seq_printf(m, " [0x%02x]", intel_dp->psr_dpcd[0]);
3058         seq_printf(m, ", Panel Replay = %s\n", str_yes_no(psr->sink_panel_replay_support));
3059
3060         if (!(psr->sink_support || psr->sink_panel_replay_support))
3061                 return 0;
3062
3063         wakeref = intel_runtime_pm_get(&dev_priv->runtime_pm);
3064         mutex_lock(&psr->lock);
3065
3066         if (psr->panel_replay_enabled)
3067                 status = "Panel Replay Enabled";
3068         else if (psr->enabled)
3069                 status = psr->psr2_enabled ? "PSR2 enabled" : "PSR1 enabled";
3070         else
3071                 status = "disabled";
3072         seq_printf(m, "PSR mode: %s\n", status);
3073
3074         if (!psr->enabled) {
3075                 seq_printf(m, "PSR sink not reliable: %s\n",
3076                            str_yes_no(psr->sink_not_reliable));
3077
3078                 goto unlock;
3079         }
3080
3081         if (psr->panel_replay_enabled) {
3082                 val = intel_de_read(dev_priv, TRANS_DP2_CTL(cpu_transcoder));
3083                 enabled = val & TRANS_DP2_PANEL_REPLAY_ENABLE;
3084         } else if (psr->psr2_enabled) {
3085                 val = intel_de_read(dev_priv, EDP_PSR2_CTL(cpu_transcoder));
3086                 enabled = val & EDP_PSR2_ENABLE;
3087         } else {
3088                 val = intel_de_read(dev_priv, psr_ctl_reg(dev_priv, cpu_transcoder));
3089                 enabled = val & EDP_PSR_ENABLE;
3090         }
3091         seq_printf(m, "Source PSR/PanelReplay ctl: %s [0x%08x]\n",
3092                    str_enabled_disabled(enabled), val);
3093         psr_source_status(intel_dp, m);
3094         seq_printf(m, "Busy frontbuffer bits: 0x%08x\n",
3095                    psr->busy_frontbuffer_bits);
3096
3097         /*
3098          * SKL+ Perf counter is reset to 0 everytime DC state is entered
3099          */
3100         val = intel_de_read(dev_priv, psr_perf_cnt_reg(dev_priv, cpu_transcoder));
3101         seq_printf(m, "Performance counter: %u\n",
3102                    REG_FIELD_GET(EDP_PSR_PERF_CNT_MASK, val));
3103
3104         if (psr->debug & I915_PSR_DEBUG_IRQ) {
3105                 seq_printf(m, "Last attempted entry at: %lld\n",
3106                            psr->last_entry_attempt);
3107                 seq_printf(m, "Last exit at: %lld\n", psr->last_exit);
3108         }
3109
3110         if (psr->psr2_enabled) {
3111                 u32 su_frames_val[3];
3112                 int frame;
3113
3114                 /*
3115                  * Reading all 3 registers before hand to minimize crossing a
3116                  * frame boundary between register reads
3117                  */
3118                 for (frame = 0; frame < PSR2_SU_STATUS_FRAMES; frame += 3) {
3119                         val = intel_de_read(dev_priv, PSR2_SU_STATUS(cpu_transcoder, frame));
3120                         su_frames_val[frame / 3] = val;
3121                 }
3122
3123                 seq_puts(m, "Frame:\tPSR2 SU blocks:\n");
3124
3125                 for (frame = 0; frame < PSR2_SU_STATUS_FRAMES; frame++) {
3126                         u32 su_blocks;
3127
3128                         su_blocks = su_frames_val[frame / 3] &
3129                                     PSR2_SU_STATUS_MASK(frame);
3130                         su_blocks = su_blocks >> PSR2_SU_STATUS_SHIFT(frame);
3131                         seq_printf(m, "%d\t%d\n", frame, su_blocks);
3132                 }
3133
3134                 seq_printf(m, "PSR2 selective fetch: %s\n",
3135                            str_enabled_disabled(psr->psr2_sel_fetch_enabled));
3136         }
3137
3138 unlock:
3139         mutex_unlock(&psr->lock);
3140         intel_runtime_pm_put(&dev_priv->runtime_pm, wakeref);
3141
3142         return 0;
3143 }
3144
3145 static int i915_edp_psr_status_show(struct seq_file *m, void *data)
3146 {
3147         struct drm_i915_private *dev_priv = m->private;
3148         struct intel_dp *intel_dp = NULL;
3149         struct intel_encoder *encoder;
3150
3151         if (!HAS_PSR(dev_priv))
3152                 return -ENODEV;
3153
3154         /* Find the first EDP which supports PSR */
3155         for_each_intel_encoder_with_psr(&dev_priv->drm, encoder) {
3156                 intel_dp = enc_to_intel_dp(encoder);
3157                 break;
3158         }
3159
3160         if (!intel_dp)
3161                 return -ENODEV;
3162
3163         return intel_psr_status(m, intel_dp);
3164 }
3165 DEFINE_SHOW_ATTRIBUTE(i915_edp_psr_status);
3166
3167 static int
3168 i915_edp_psr_debug_set(void *data, u64 val)
3169 {
3170         struct drm_i915_private *dev_priv = data;
3171         struct intel_encoder *encoder;
3172         intel_wakeref_t wakeref;
3173         int ret = -ENODEV;
3174
3175         if (!HAS_PSR(dev_priv))
3176                 return ret;
3177
3178         for_each_intel_encoder_with_psr(&dev_priv->drm, encoder) {
3179                 struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
3180
3181                 drm_dbg_kms(&dev_priv->drm, "Setting PSR debug to %llx\n", val);
3182
3183                 wakeref = intel_runtime_pm_get(&dev_priv->runtime_pm);
3184
3185                 // TODO: split to each transcoder's PSR debug state
3186                 ret = intel_psr_debug_set(intel_dp, val);
3187
3188                 intel_runtime_pm_put(&dev_priv->runtime_pm, wakeref);
3189         }
3190
3191         return ret;
3192 }
3193
3194 static int
3195 i915_edp_psr_debug_get(void *data, u64 *val)
3196 {
3197         struct drm_i915_private *dev_priv = data;
3198         struct intel_encoder *encoder;
3199
3200         if (!HAS_PSR(dev_priv))
3201                 return -ENODEV;
3202
3203         for_each_intel_encoder_with_psr(&dev_priv->drm, encoder) {
3204                 struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
3205
3206                 // TODO: split to each transcoder's PSR debug state
3207                 *val = READ_ONCE(intel_dp->psr.debug);
3208                 return 0;
3209         }
3210
3211         return -ENODEV;
3212 }
3213
3214 DEFINE_SIMPLE_ATTRIBUTE(i915_edp_psr_debug_fops,
3215                         i915_edp_psr_debug_get, i915_edp_psr_debug_set,
3216                         "%llu\n");
3217
3218 void intel_psr_debugfs_register(struct drm_i915_private *i915)
3219 {
3220         struct drm_minor *minor = i915->drm.primary;
3221
3222         debugfs_create_file("i915_edp_psr_debug", 0644, minor->debugfs_root,
3223                             i915, &i915_edp_psr_debug_fops);
3224
3225         debugfs_create_file("i915_edp_psr_status", 0444, minor->debugfs_root,
3226                             i915, &i915_edp_psr_status_fops);
3227 }
3228
3229 static const char *psr_mode_str(struct intel_dp *intel_dp)
3230 {
3231         if (intel_dp->psr.panel_replay_enabled)
3232                 return "PANEL-REPLAY";
3233         else if (intel_dp->psr.enabled)
3234                 return "PSR";
3235
3236         return "unknown";
3237 }
3238
3239 static int i915_psr_sink_status_show(struct seq_file *m, void *data)
3240 {
3241         struct intel_connector *connector = m->private;
3242         struct intel_dp *intel_dp = intel_attached_dp(connector);
3243         static const char * const sink_status[] = {
3244                 "inactive",
3245                 "transition to active, capture and display",
3246                 "active, display from RFB",
3247                 "active, capture and display on sink device timings",
3248                 "transition to inactive, capture and display, timing re-sync",
3249                 "reserved",
3250                 "reserved",
3251                 "sink internal error",
3252         };
3253         static const char * const panel_replay_status[] = {
3254                 "Sink device frame is locked to the Source device",
3255                 "Sink device is coasting, using the VTotal target",
3256                 "Sink device is governing the frame rate (frame rate unlock is granted)",
3257                 "Sink device in the process of re-locking with the Source device",
3258         };
3259         const char *str;
3260         int ret;
3261         u8 status, error_status;
3262         u32 idx;
3263
3264         if (!(CAN_PSR(intel_dp) || CAN_PANEL_REPLAY(intel_dp))) {
3265                 seq_puts(m, "PSR/Panel-Replay Unsupported\n");
3266                 return -ENODEV;
3267         }
3268
3269         if (connector->base.status != connector_status_connected)
3270                 return -ENODEV;
3271
3272         ret = psr_get_status_and_error_status(intel_dp, &status, &error_status);
3273         if (ret)
3274                 return ret;
3275
3276         str = "unknown";
3277         if (intel_dp->psr.panel_replay_enabled) {
3278                 idx = (status & DP_SINK_FRAME_LOCKED_MASK) >> DP_SINK_FRAME_LOCKED_SHIFT;
3279                 if (idx < ARRAY_SIZE(panel_replay_status))
3280                         str = panel_replay_status[idx];
3281         } else if (intel_dp->psr.enabled) {
3282                 idx = status & DP_PSR_SINK_STATE_MASK;
3283                 if (idx < ARRAY_SIZE(sink_status))
3284                         str = sink_status[idx];
3285         }
3286
3287         seq_printf(m, "Sink %s status: 0x%x [%s]\n", psr_mode_str(intel_dp), status, str);
3288
3289         seq_printf(m, "Sink %s error status: 0x%x", psr_mode_str(intel_dp), error_status);
3290
3291         if (error_status & (DP_PSR_RFB_STORAGE_ERROR |
3292                             DP_PSR_VSC_SDP_UNCORRECTABLE_ERROR |
3293                             DP_PSR_LINK_CRC_ERROR))
3294                 seq_puts(m, ":\n");
3295         else
3296                 seq_puts(m, "\n");
3297         if (error_status & DP_PSR_RFB_STORAGE_ERROR)
3298                 seq_printf(m, "\t%s RFB storage error\n", psr_mode_str(intel_dp));
3299         if (error_status & DP_PSR_VSC_SDP_UNCORRECTABLE_ERROR)
3300                 seq_printf(m, "\t%s VSC SDP uncorrectable error\n", psr_mode_str(intel_dp));
3301         if (error_status & DP_PSR_LINK_CRC_ERROR)
3302                 seq_printf(m, "\t%s Link CRC error\n", psr_mode_str(intel_dp));
3303
3304         return ret;
3305 }
3306 DEFINE_SHOW_ATTRIBUTE(i915_psr_sink_status);
3307
3308 static int i915_psr_status_show(struct seq_file *m, void *data)
3309 {
3310         struct intel_connector *connector = m->private;
3311         struct intel_dp *intel_dp = intel_attached_dp(connector);
3312
3313         return intel_psr_status(m, intel_dp);
3314 }
3315 DEFINE_SHOW_ATTRIBUTE(i915_psr_status);
3316
3317 void intel_psr_connector_debugfs_add(struct intel_connector *connector)
3318 {
3319         struct drm_i915_private *i915 = to_i915(connector->base.dev);
3320         struct dentry *root = connector->base.debugfs_entry;
3321
3322         /* TODO: Add support for MST connectors as well. */
3323         if ((connector->base.connector_type != DRM_MODE_CONNECTOR_eDP &&
3324              connector->base.connector_type != DRM_MODE_CONNECTOR_DisplayPort) ||
3325             connector->mst_port)
3326                 return;
3327
3328         debugfs_create_file("i915_psr_sink_status", 0444, root,
3329                             connector, &i915_psr_sink_status_fops);
3330
3331         if (HAS_PSR(i915) || HAS_DP20(i915))
3332                 debugfs_create_file("i915_psr_status", 0444, root,
3333                                     connector, &i915_psr_status_fops);
3334 }