drivers/gpu/drm/i915/i915_reset.c

   1 /*
   2  * SPDX-License-Identifier: MIT
   3  *
   4  * Copyright © 2008-2018 Intel Corporation
   5  */
   6
   7 #include <linux/sched/mm.h>
   8 #include <linux/stop_machine.h>
   9
  10 #include "i915_drv.h"
  11 #include "i915_gpu_error.h"
  12 #include "i915_reset.h"
  13
  14 #include "intel_guc.h"
  15
  16 #define RESET_MAX_RETRIES 3
  17
  18 /* XXX How to handle concurrent GGTT updates using tiling registers? */
  19 #define RESET_UNDER_STOP_MACHINE 0
  20
  21 static void engine_skip_context(struct i915_request *rq)
  22 {
  23         struct intel_engine_cs *engine = rq->engine;
  24         struct i915_gem_context *hung_ctx = rq->gem_context;
  25
  26         lockdep_assert_held(&engine->timeline.lock);
  27
  28         if (!i915_request_is_active(rq))
  29                 return;
  30
  31         list_for_each_entry_continue(rq, &engine->timeline.requests, link)
  32                 if (rq->gem_context == hung_ctx)
  33                         i915_request_skip(rq, -EIO);
  34 }
  35
  36 static void client_mark_guilty(struct drm_i915_file_private *file_priv,
  37                                const struct i915_gem_context *ctx)
  38 {
  39         unsigned int score;
  40         unsigned long prev_hang;
  41
  42         if (i915_gem_context_is_banned(ctx))
  43                 score = I915_CLIENT_SCORE_CONTEXT_BAN;
  44         else
  45                 score = 0;
  46
  47         prev_hang = xchg(&file_priv->hang_timestamp, jiffies);
  48         if (time_before(jiffies, prev_hang + I915_CLIENT_FAST_HANG_JIFFIES))
  49                 score += I915_CLIENT_SCORE_HANG_FAST;
  50
  51         if (score) {
  52                 atomic_add(score, &file_priv->ban_score);
  53
  54                 DRM_DEBUG_DRIVER("client %s: gained %u ban score, now %u\n",
  55                                  ctx->name, score,
  56                                  atomic_read(&file_priv->ban_score));
  57         }
  58 }
  59
  60 static bool context_mark_guilty(struct i915_gem_context *ctx)
  61 {
  62         unsigned long prev_hang;
  63         bool banned;
  64         int i;
  65
  66         atomic_inc(&ctx->guilty_count);
  67
  68         /* Cool contexts are too cool to be banned! (Used for reset testing.) */
  69         if (!i915_gem_context_is_bannable(ctx))
  70                 return false;
  71
  72         /* Record the timestamp for the last N hangs */
  73         prev_hang = ctx->hang_timestamp[0];
  74         for (i = 0; i < ARRAY_SIZE(ctx->hang_timestamp) - 1; i++)
  75                 ctx->hang_timestamp[i] = ctx->hang_timestamp[i + 1];
  76         ctx->hang_timestamp[i] = jiffies;
  77
  78         /* If we have hung N+1 times in rapid succession, we ban the context! */
  79         banned = !i915_gem_context_is_recoverable(ctx);
  80         if (time_before(jiffies, prev_hang + CONTEXT_FAST_HANG_JIFFIES))
  81                 banned = true;
  82         if (banned) {
  83                 DRM_DEBUG_DRIVER("context %s: guilty %d, banned\n",
  84                                  ctx->name, atomic_read(&ctx->guilty_count));
  85                 i915_gem_context_set_banned(ctx);
  86         }
  87
  88         if (!IS_ERR_OR_NULL(ctx->file_priv))
  89                 client_mark_guilty(ctx->file_priv, ctx);
  90
  91         return banned;
  92 }
  93
  94 static void context_mark_innocent(struct i915_gem_context *ctx)
  95 {
  96         atomic_inc(&ctx->active_count);
  97 }
  98
  99 void i915_reset_request(struct i915_request *rq, bool guilty)
 100 {
 101         lockdep_assert_held(&rq->engine->timeline.lock);
 102         GEM_BUG_ON(i915_request_completed(rq));
 103
 104         if (guilty) {
 105                 i915_request_skip(rq, -EIO);
 106                 if (context_mark_guilty(rq->gem_context))
 107                         engine_skip_context(rq);
 108         } else {
 109                 dma_fence_set_error(&rq->fence, -EAGAIN);
 110                 context_mark_innocent(rq->gem_context);
 111         }
 112 }
 113
 114 static void gen3_stop_engine(struct intel_engine_cs *engine)
 115 {
 116         struct drm_i915_private *dev_priv = engine->i915;
 117         const u32 base = engine->mmio_base;
 118
 119         GEM_TRACE("%s\n", engine->name);
 120
 121         if (intel_engine_stop_cs(engine))
 122                 GEM_TRACE("%s: timed out on STOP_RING\n", engine->name);
 123
 124         I915_WRITE_FW(RING_HEAD(base), I915_READ_FW(RING_TAIL(base)));
 125         POSTING_READ_FW(RING_HEAD(base)); /* paranoia */
 126
 127         I915_WRITE_FW(RING_HEAD(base), 0);
 128         I915_WRITE_FW(RING_TAIL(base), 0);
 129         POSTING_READ_FW(RING_TAIL(base));
 130
 131         /* The ring must be empty before it is disabled */
 132         I915_WRITE_FW(RING_CTL(base), 0);
 133
 134         /* Check acts as a post */
 135         if (I915_READ_FW(RING_HEAD(base)))
 136                 GEM_TRACE("%s: ring head [%x] not parked\n",
 137                           engine->name, I915_READ_FW(RING_HEAD(base)));
 138 }
 139
 140 static void i915_stop_engines(struct drm_i915_private *i915,
 141                               unsigned int engine_mask)
 142 {
 143         struct intel_engine_cs *engine;
 144         enum intel_engine_id id;
 145
 146         if (INTEL_GEN(i915) < 3)
 147                 return;
 148
 149         for_each_engine_masked(engine, i915, engine_mask, id)
 150                 gen3_stop_engine(engine);
 151 }
 152
 153 static bool i915_in_reset(struct pci_dev *pdev)
 154 {
 155         u8 gdrst;
 156
 157         pci_read_config_byte(pdev, I915_GDRST, &gdrst);
 158         return gdrst & GRDOM_RESET_STATUS;
 159 }
 160
 161 static int i915_do_reset(struct drm_i915_private *i915,
 162                          unsigned int engine_mask,
 163                          unsigned int retry)
 164 {
 165         struct pci_dev *pdev = i915->drm.pdev;
 166         int err;
 167
 168         /* Assert reset for at least 20 usec, and wait for acknowledgement. */
 169         pci_write_config_byte(pdev, I915_GDRST, GRDOM_RESET_ENABLE);
 170         udelay(50);
 171         err = wait_for_atomic(i915_in_reset(pdev), 50);
 172
 173         /* Clear the reset request. */
 174         pci_write_config_byte(pdev, I915_GDRST, 0);
 175         udelay(50);
 176         if (!err)
 177                 err = wait_for_atomic(!i915_in_reset(pdev), 50);
 178
 179         return err;
 180 }
 181
 182 static bool g4x_reset_complete(struct pci_dev *pdev)
 183 {
 184         u8 gdrst;
 185
 186         pci_read_config_byte(pdev, I915_GDRST, &gdrst);
 187         return (gdrst & GRDOM_RESET_ENABLE) == 0;
 188 }
 189
 190 static int g33_do_reset(struct drm_i915_private *i915,
 191                         unsigned int engine_mask,
 192                         unsigned int retry)
 193 {
 194         struct pci_dev *pdev = i915->drm.pdev;
 195
 196         pci_write_config_byte(pdev, I915_GDRST, GRDOM_RESET_ENABLE);
 197         return wait_for_atomic(g4x_reset_complete(pdev), 50);
 198 }
 199
 200 static int g4x_do_reset(struct drm_i915_private *dev_priv,
 201                         unsigned int engine_mask,
 202                         unsigned int retry)
 203 {
 204         struct pci_dev *pdev = dev_priv->drm.pdev;
 205         int ret;
 206
 207         /* WaVcpClkGateDisableForMediaReset:ctg,elk */
 208         I915_WRITE_FW(VDECCLK_GATE_D,
 209                       I915_READ(VDECCLK_GATE_D) | VCP_UNIT_CLOCK_GATE_DISABLE);
 210         POSTING_READ_FW(VDECCLK_GATE_D);
 211
 212         pci_write_config_byte(pdev, I915_GDRST,
 213                               GRDOM_MEDIA | GRDOM_RESET_ENABLE);
 214         ret =  wait_for_atomic(g4x_reset_complete(pdev), 50);
 215         if (ret) {
 216                 DRM_DEBUG_DRIVER("Wait for media reset failed\n");
 217                 goto out;
 218         }
 219
 220         pci_write_config_byte(pdev, I915_GDRST,
 221                               GRDOM_RENDER | GRDOM_RESET_ENABLE);
 222         ret =  wait_for_atomic(g4x_reset_complete(pdev), 50);
 223         if (ret) {
 224                 DRM_DEBUG_DRIVER("Wait for render reset failed\n");
 225                 goto out;
 226         }
 227
 228 out:
 229         pci_write_config_byte(pdev, I915_GDRST, 0);
 230
 231         I915_WRITE_FW(VDECCLK_GATE_D,
 232                       I915_READ(VDECCLK_GATE_D) & ~VCP_UNIT_CLOCK_GATE_DISABLE);
 233         POSTING_READ_FW(VDECCLK_GATE_D);
 234
 235         return ret;
 236 }
 237
 238 static int ironlake_do_reset(struct drm_i915_private *dev_priv,
 239                              unsigned int engine_mask,
 240                              unsigned int retry)
 241 {
 242         int ret;
 243
 244         I915_WRITE_FW(ILK_GDSR, ILK_GRDOM_RENDER | ILK_GRDOM_RESET_ENABLE);
 245         ret = __intel_wait_for_register_fw(dev_priv, ILK_GDSR,
 246                                            ILK_GRDOM_RESET_ENABLE, 0,
 247                                            5000, 0,
 248                                            NULL);
 249         if (ret) {
 250                 DRM_DEBUG_DRIVER("Wait for render reset failed\n");
 251                 goto out;
 252         }
 253
 254         I915_WRITE_FW(ILK_GDSR, ILK_GRDOM_MEDIA | ILK_GRDOM_RESET_ENABLE);
 255         ret = __intel_wait_for_register_fw(dev_priv, ILK_GDSR,
 256                                            ILK_GRDOM_RESET_ENABLE, 0,
 257                                            5000, 0,
 258                                            NULL);
 259         if (ret) {
 260                 DRM_DEBUG_DRIVER("Wait for media reset failed\n");
 261                 goto out;
 262         }
 263
 264 out:
 265         I915_WRITE_FW(ILK_GDSR, 0);
 266         POSTING_READ_FW(ILK_GDSR);
 267         return ret;
 268 }
 269
 270 /* Reset the hardware domains (GENX_GRDOM_*) specified by mask */
 271 static int gen6_hw_domain_reset(struct drm_i915_private *dev_priv,
 272                                 u32 hw_domain_mask)
 273 {
 274         int err;
 275
 276         /*
 277          * GEN6_GDRST is not in the gt power well, no need to check
 278          * for fifo space for the write or forcewake the chip for
 279          * the read
 280          */
 281         I915_WRITE_FW(GEN6_GDRST, hw_domain_mask);
 282
 283         /* Wait for the device to ack the reset requests */
 284         err = __intel_wait_for_register_fw(dev_priv,
 285                                            GEN6_GDRST, hw_domain_mask, 0,
 286                                            500, 0,
 287                                            NULL);
 288         if (err)
 289                 DRM_DEBUG_DRIVER("Wait for 0x%08x engines reset failed\n",
 290                                  hw_domain_mask);
 291
 292         return err;
 293 }
 294
 295 static int gen6_reset_engines(struct drm_i915_private *i915,
 296                               unsigned int engine_mask,
 297                               unsigned int retry)
 298 {
 299         struct intel_engine_cs *engine;
 300         const u32 hw_engine_mask[] = {
 301                 [RCS0]  = GEN6_GRDOM_RENDER,
 302                 [BCS0]  = GEN6_GRDOM_BLT,
 303                 [VCS0]  = GEN6_GRDOM_MEDIA,
 304                 [VCS1]  = GEN8_GRDOM_MEDIA2,
 305                 [VECS0] = GEN6_GRDOM_VECS,
 306         };
 307         u32 hw_mask;
 308
 309         if (engine_mask == ALL_ENGINES) {
 310                 hw_mask = GEN6_GRDOM_FULL;
 311         } else {
 312                 unsigned int tmp;
 313
 314                 hw_mask = 0;
 315                 for_each_engine_masked(engine, i915, engine_mask, tmp) {
 316                         GEM_BUG_ON(engine->id >= ARRAY_SIZE(hw_engine_mask));
 317                         hw_mask |= hw_engine_mask[engine->id];
 318                 }
 319         }
 320
 321         return gen6_hw_domain_reset(i915, hw_mask);
 322 }
 323
 324 static u32 gen11_lock_sfc(struct drm_i915_private *dev_priv,
 325                           struct intel_engine_cs *engine)
 326 {
 327         u8 vdbox_sfc_access = RUNTIME_INFO(dev_priv)->vdbox_sfc_access;
 328         i915_reg_t sfc_forced_lock, sfc_forced_lock_ack;
 329         u32 sfc_forced_lock_bit, sfc_forced_lock_ack_bit;
 330         i915_reg_t sfc_usage;
 331         u32 sfc_usage_bit;
 332         u32 sfc_reset_bit;
 333
 334         switch (engine->class) {
 335         case VIDEO_DECODE_CLASS:
 336                 if ((BIT(engine->instance) & vdbox_sfc_access) == 0)
 337                         return 0;
 338
 339                 sfc_forced_lock = GEN11_VCS_SFC_FORCED_LOCK(engine);
 340                 sfc_forced_lock_bit = GEN11_VCS_SFC_FORCED_LOCK_BIT;
 341
 342                 sfc_forced_lock_ack = GEN11_VCS_SFC_LOCK_STATUS(engine);
 343                 sfc_forced_lock_ack_bit  = GEN11_VCS_SFC_LOCK_ACK_BIT;
 344
 345                 sfc_usage = GEN11_VCS_SFC_LOCK_STATUS(engine);
 346                 sfc_usage_bit = GEN11_VCS_SFC_USAGE_BIT;
 347                 sfc_reset_bit = GEN11_VCS_SFC_RESET_BIT(engine->instance);
 348                 break;
 349
 350         case VIDEO_ENHANCEMENT_CLASS:
 351                 sfc_forced_lock = GEN11_VECS_SFC_FORCED_LOCK(engine);
 352                 sfc_forced_lock_bit = GEN11_VECS_SFC_FORCED_LOCK_BIT;
 353
 354                 sfc_forced_lock_ack = GEN11_VECS_SFC_LOCK_ACK(engine);
 355                 sfc_forced_lock_ack_bit  = GEN11_VECS_SFC_LOCK_ACK_BIT;
 356
 357                 sfc_usage = GEN11_VECS_SFC_USAGE(engine);
 358                 sfc_usage_bit = GEN11_VECS_SFC_USAGE_BIT;
 359                 sfc_reset_bit = GEN11_VECS_SFC_RESET_BIT(engine->instance);
 360                 break;
 361
 362         default:
 363                 return 0;
 364         }
 365
 366         /*
 367          * Tell the engine that a software reset is going to happen. The engine
 368          * will then try to force lock the SFC (if currently locked, it will
 369          * remain so until we tell the engine it is safe to unlock; if currently
 370          * unlocked, it will ignore this and all new lock requests). If SFC
 371          * ends up being locked to the engine we want to reset, we have to reset
 372          * it as well (we will unlock it once the reset sequence is completed).
 373          */
 374         I915_WRITE_FW(sfc_forced_lock,
 375                       I915_READ_FW(sfc_forced_lock) | sfc_forced_lock_bit);
 376
 377         if (__intel_wait_for_register_fw(dev_priv,
 378                                          sfc_forced_lock_ack,
 379                                          sfc_forced_lock_ack_bit,
 380                                          sfc_forced_lock_ack_bit,
 381                                          1000, 0, NULL)) {
 382                 DRM_DEBUG_DRIVER("Wait for SFC forced lock ack failed\n");
 383                 return 0;
 384         }
 385
 386         if (I915_READ_FW(sfc_usage) & sfc_usage_bit)
 387                 return sfc_reset_bit;
 388
 389         return 0;
 390 }
 391
 392 static void gen11_unlock_sfc(struct drm_i915_private *dev_priv,
 393                              struct intel_engine_cs *engine)
 394 {
 395         u8 vdbox_sfc_access = RUNTIME_INFO(dev_priv)->vdbox_sfc_access;
 396         i915_reg_t sfc_forced_lock;
 397         u32 sfc_forced_lock_bit;
 398
 399         switch (engine->class) {
 400         case VIDEO_DECODE_CLASS:
 401                 if ((BIT(engine->instance) & vdbox_sfc_access) == 0)
 402                         return;
 403
 404                 sfc_forced_lock = GEN11_VCS_SFC_FORCED_LOCK(engine);
 405                 sfc_forced_lock_bit = GEN11_VCS_SFC_FORCED_LOCK_BIT;
 406                 break;
 407
 408         case VIDEO_ENHANCEMENT_CLASS:
 409                 sfc_forced_lock = GEN11_VECS_SFC_FORCED_LOCK(engine);
 410                 sfc_forced_lock_bit = GEN11_VECS_SFC_FORCED_LOCK_BIT;
 411                 break;
 412
 413         default:
 414                 return;
 415         }
 416
 417         I915_WRITE_FW(sfc_forced_lock,
 418                       I915_READ_FW(sfc_forced_lock) & ~sfc_forced_lock_bit);
 419 }
 420
 421 static int gen11_reset_engines(struct drm_i915_private *i915,
 422                                unsigned int engine_mask,
 423                                unsigned int retry)
 424 {
 425         const u32 hw_engine_mask[] = {
 426                 [RCS0]  = GEN11_GRDOM_RENDER,
 427                 [BCS0]  = GEN11_GRDOM_BLT,
 428                 [VCS0]  = GEN11_GRDOM_MEDIA,
 429                 [VCS1]  = GEN11_GRDOM_MEDIA2,
 430                 [VCS2]  = GEN11_GRDOM_MEDIA3,
 431                 [VCS3]  = GEN11_GRDOM_MEDIA4,
 432                 [VECS0] = GEN11_GRDOM_VECS,
 433                 [VECS1] = GEN11_GRDOM_VECS2,
 434         };
 435         struct intel_engine_cs *engine;
 436         unsigned int tmp;
 437         u32 hw_mask;
 438         int ret;
 439
 440         if (engine_mask == ALL_ENGINES) {
 441                 hw_mask = GEN11_GRDOM_FULL;
 442         } else {
 443                 hw_mask = 0;
 444                 for_each_engine_masked(engine, i915, engine_mask, tmp) {
 445                         GEM_BUG_ON(engine->id >= ARRAY_SIZE(hw_engine_mask));
 446                         hw_mask |= hw_engine_mask[engine->id];
 447                         hw_mask |= gen11_lock_sfc(i915, engine);
 448                 }
 449         }
 450
 451         ret = gen6_hw_domain_reset(i915, hw_mask);
 452
 453         if (engine_mask != ALL_ENGINES)
 454                 for_each_engine_masked(engine, i915, engine_mask, tmp)
 455                         gen11_unlock_sfc(i915, engine);
 456
 457         return ret;
 458 }
 459
 460 static int gen8_engine_reset_prepare(struct intel_engine_cs *engine)
 461 {
 462         struct drm_i915_private *dev_priv = engine->i915;
 463         int ret;
 464
 465         I915_WRITE_FW(RING_RESET_CTL(engine->mmio_base),
 466                       _MASKED_BIT_ENABLE(RESET_CTL_REQUEST_RESET));
 467
 468         ret = __intel_wait_for_register_fw(dev_priv,
 469                                            RING_RESET_CTL(engine->mmio_base),
 470                                            RESET_CTL_READY_TO_RESET,
 471                                            RESET_CTL_READY_TO_RESET,
 472                                            700, 0,
 473                                            NULL);
 474         if (ret)
 475                 DRM_ERROR("%s: reset request timeout\n", engine->name);
 476
 477         return ret;
 478 }
 479
 480 static void gen8_engine_reset_cancel(struct intel_engine_cs *engine)
 481 {
 482         struct drm_i915_private *dev_priv = engine->i915;
 483
 484         I915_WRITE_FW(RING_RESET_CTL(engine->mmio_base),
 485                       _MASKED_BIT_DISABLE(RESET_CTL_REQUEST_RESET));
 486 }
 487
 488 static int gen8_reset_engines(struct drm_i915_private *i915,
 489                               unsigned int engine_mask,
 490                               unsigned int retry)
 491 {
 492         struct intel_engine_cs *engine;
 493         const bool reset_non_ready = retry >= 1;
 494         unsigned int tmp;
 495         int ret;
 496
 497         for_each_engine_masked(engine, i915, engine_mask, tmp) {
 498                 ret = gen8_engine_reset_prepare(engine);
 499                 if (ret && !reset_non_ready)
 500                         goto skip_reset;
 501
 502                 /*
 503                  * If this is not the first failed attempt to prepare,
 504                  * we decide to proceed anyway.
 505                  *
 506                  * By doing so we risk context corruption and with
 507                  * some gens (kbl), possible system hang if reset
 508                  * happens during active bb execution.
 509                  *
 510                  * We rather take context corruption instead of
 511                  * failed reset with a wedged driver/gpu. And
 512                  * active bb execution case should be covered by
 513                  * i915_stop_engines we have before the reset.
 514                  */
 515         }
 516
 517         if (INTEL_GEN(i915) >= 11)
 518                 ret = gen11_reset_engines(i915, engine_mask, retry);
 519         else
 520                 ret = gen6_reset_engines(i915, engine_mask, retry);
 521
 522 skip_reset:
 523         for_each_engine_masked(engine, i915, engine_mask, tmp)
 524                 gen8_engine_reset_cancel(engine);
 525
 526         return ret;
 527 }
 528
 529 typedef int (*reset_func)(struct drm_i915_private *,
 530                           unsigned int engine_mask,
 531                           unsigned int retry);
 532
 533 static reset_func intel_get_gpu_reset(struct drm_i915_private *i915)
 534 {
 535         if (INTEL_GEN(i915) >= 8)
 536                 return gen8_reset_engines;
 537         else if (INTEL_GEN(i915) >= 6)
 538                 return gen6_reset_engines;
 539         else if (INTEL_GEN(i915) >= 5)
 540                 return ironlake_do_reset;
 541         else if (IS_G4X(i915))
 542                 return g4x_do_reset;
 543         else if (IS_G33(i915) || IS_PINEVIEW(i915))
 544                 return g33_do_reset;
 545         else if (INTEL_GEN(i915) >= 3)
 546                 return i915_do_reset;
 547         else
 548                 return NULL;
 549 }
 550
 551 int intel_gpu_reset(struct drm_i915_private *i915, unsigned int engine_mask)
 552 {
 553         const int retries = engine_mask == ALL_ENGINES ? RESET_MAX_RETRIES : 1;
 554         reset_func reset;
 555         int ret = -ETIMEDOUT;
 556         int retry;
 557
 558         reset = intel_get_gpu_reset(i915);
 559         if (!reset)
 560                 return -ENODEV;
 561
 562         /*
 563          * If the power well sleeps during the reset, the reset
 564          * request may be dropped and never completes (causing -EIO).
 565          */
 566         intel_uncore_forcewake_get(i915, FORCEWAKE_ALL);
 567         for (retry = 0; ret == -ETIMEDOUT && retry < retries; retry++) {
 568                 /*
 569                  * We stop engines, otherwise we might get failed reset and a
 570                  * dead gpu (on elk). Also as modern gpu as kbl can suffer
 571                  * from system hang if batchbuffer is progressing when
 572                  * the reset is issued, regardless of READY_TO_RESET ack.
 573                  * Thus assume it is best to stop engines on all gens
 574                  * where we have a gpu reset.
 575                  *
 576                  * WaKBLVECSSemaphoreWaitPoll:kbl (on ALL_ENGINES)
 577                  *
 578                  * WaMediaResetMainRingCleanup:ctg,elk (presumably)
 579                  *
 580                  * FIXME: Wa for more modern gens needs to be validated
 581                  */
 582                 if (retry)
 583                         i915_stop_engines(i915, engine_mask);
 584
 585                 GEM_TRACE("engine_mask=%x\n", engine_mask);
 586                 preempt_disable();
 587                 ret = reset(i915, engine_mask, retry);
 588                 preempt_enable();
 589         }
 590         intel_uncore_forcewake_put(i915, FORCEWAKE_ALL);
 591
 592         return ret;
 593 }
 594
 595 bool intel_has_gpu_reset(struct drm_i915_private *i915)
 596 {
 597         if (USES_GUC(i915))
 598                 return false;
 599
 600         if (!i915_modparams.reset)
 601                 return NULL;
 602
 603         return intel_get_gpu_reset(i915);
 604 }
 605
 606 bool intel_has_reset_engine(struct drm_i915_private *i915)
 607 {
 608         return INTEL_INFO(i915)->has_reset_engine && i915_modparams.reset >= 2;
 609 }
 610
 611 int intel_reset_guc(struct drm_i915_private *i915)
 612 {
 613         u32 guc_domain =
 614                 INTEL_GEN(i915) >= 11 ? GEN11_GRDOM_GUC : GEN9_GRDOM_GUC;
 615         int ret;
 616
 617         GEM_BUG_ON(!HAS_GUC(i915));
 618
 619         intel_uncore_forcewake_get(i915, FORCEWAKE_ALL);
 620         ret = gen6_hw_domain_reset(i915, guc_domain);
 621         intel_uncore_forcewake_put(i915, FORCEWAKE_ALL);
 622
 623         return ret;
 624 }
 625
 626 /*
 627  * Ensure irq handler finishes, and not run again.
 628  * Also return the active request so that we only search for it once.
 629  */
 630 static void reset_prepare_engine(struct intel_engine_cs *engine)
 631 {
 632         /*
 633          * During the reset sequence, we must prevent the engine from
 634          * entering RC6. As the context state is undefined until we restart
 635          * the engine, if it does enter RC6 during the reset, the state
 636          * written to the powercontext is undefined and so we may lose
 637          * GPU state upon resume, i.e. fail to restart after a reset.
 638          */
 639         intel_uncore_forcewake_get(engine->i915, FORCEWAKE_ALL);
 640         engine->reset.prepare(engine);
 641 }
 642
 643 static void revoke_mmaps(struct drm_i915_private *i915)
 644 {
 645         int i;
 646
 647         for (i = 0; i < i915->num_fence_regs; i++) {
 648                 struct drm_vma_offset_node *node;
 649                 struct i915_vma *vma;
 650                 u64 vma_offset;
 651
 652                 vma = READ_ONCE(i915->fence_regs[i].vma);
 653                 if (!vma)
 654                         continue;
 655
 656                 if (!i915_vma_has_userfault(vma))
 657                         continue;
 658
 659                 GEM_BUG_ON(vma->fence != &i915->fence_regs[i]);
 660                 node = &vma->obj->base.vma_node;
 661                 vma_offset = vma->ggtt_view.partial.offset << PAGE_SHIFT;
 662                 unmap_mapping_range(i915->drm.anon_inode->i_mapping,
 663                                     drm_vma_node_offset_addr(node) + vma_offset,
 664                                     vma->size,
 665                                     1);
 666         }
 667 }
 668
 669 static void reset_prepare(struct drm_i915_private *i915)
 670 {
 671         struct intel_engine_cs *engine;
 672         enum intel_engine_id id;
 673
 674         for_each_engine(engine, i915, id)
 675                 reset_prepare_engine(engine);
 676
 677         intel_uc_reset_prepare(i915);
 678         revoke_mmaps(i915);
 679 }
 680
 681 static int gt_reset(struct drm_i915_private *i915, unsigned int stalled_mask)
 682 {
 683         struct intel_engine_cs *engine;
 684         enum intel_engine_id id;
 685         int err;
 686
 687         /*
 688          * Everything depends on having the GTT running, so we need to start
 689          * there.
 690          */
 691         err = i915_ggtt_enable_hw(i915);
 692         if (err)
 693                 return err;
 694
 695         for_each_engine(engine, i915, id)
 696                 intel_engine_reset(engine, stalled_mask & engine->mask);
 697
 698         i915_gem_restore_fences(i915);
 699
 700         return err;
 701 }
 702
 703 static void reset_finish_engine(struct intel_engine_cs *engine)
 704 {
 705         engine->reset.finish(engine);
 706         intel_uncore_forcewake_put(engine->i915, FORCEWAKE_ALL);
 707 }
 708
 709 struct i915_gpu_restart {
 710         struct work_struct work;
 711         struct drm_i915_private *i915;
 712 };
 713
 714 static void restart_work(struct work_struct *work)
 715 {
 716         struct i915_gpu_restart *arg = container_of(work, typeof(*arg), work);
 717         struct drm_i915_private *i915 = arg->i915;
 718         struct intel_engine_cs *engine;
 719         enum intel_engine_id id;
 720         intel_wakeref_t wakeref;
 721
 722         wakeref = intel_runtime_pm_get(i915);
 723         mutex_lock(&i915->drm.struct_mutex);
 724         WRITE_ONCE(i915->gpu_error.restart, NULL);
 725
 726         for_each_engine(engine, i915, id) {
 727                 struct i915_request *rq;
 728
 729                 /*
 730                  * Ostensibily, we always want a context loaded for powersaving,
 731                  * so if the engine is idle after the reset, send a request
 732                  * to load our scratch kernel_context.
 733                  */
 734                 if (!intel_engine_is_idle(engine))
 735                         continue;
 736
 737                 rq = i915_request_alloc(engine, i915->kernel_context);
 738                 if (!IS_ERR(rq))
 739                         i915_request_add(rq);
 740         }
 741
 742         mutex_unlock(&i915->drm.struct_mutex);
 743         intel_runtime_pm_put(i915, wakeref);
 744
 745         kfree(arg);
 746 }
 747
 748 static void reset_finish(struct drm_i915_private *i915)
 749 {
 750         struct intel_engine_cs *engine;
 751         enum intel_engine_id id;
 752
 753         for_each_engine(engine, i915, id)
 754                 reset_finish_engine(engine);
 755 }
 756
 757 static void reset_restart(struct drm_i915_private *i915)
 758 {
 759         struct i915_gpu_restart *arg;
 760
 761         /*
 762          * Following the reset, ensure that we always reload context for
 763          * powersaving, and to correct engine->last_retired_context. Since
 764          * this requires us to submit a request, queue a worker to do that
 765          * task for us to evade any locking here.
 766          */
 767         if (READ_ONCE(i915->gpu_error.restart))
 768                 return;
 769
 770         arg = kmalloc(sizeof(*arg), GFP_KERNEL);
 771         if (arg) {
 772                 arg->i915 = i915;
 773                 INIT_WORK(&arg->work, restart_work);
 774
 775                 WRITE_ONCE(i915->gpu_error.restart, arg);
 776                 queue_work(i915->wq, &arg->work);
 777         }
 778 }
 779
 780 static void nop_submit_request(struct i915_request *request)
 781 {
 782         struct intel_engine_cs *engine = request->engine;
 783         unsigned long flags;
 784
 785         GEM_TRACE("%s fence %llx:%lld -> -EIO\n",
 786                   engine->name, request->fence.context, request->fence.seqno);
 787         dma_fence_set_error(&request->fence, -EIO);
 788
 789         spin_lock_irqsave(&engine->timeline.lock, flags);
 790         __i915_request_submit(request);
 791         i915_request_mark_complete(request);
 792         spin_unlock_irqrestore(&engine->timeline.lock, flags);
 793
 794         intel_engine_queue_breadcrumbs(engine);
 795 }
 796
 797 static void __i915_gem_set_wedged(struct drm_i915_private *i915)
 798 {
 799         struct i915_gpu_error *error = &i915->gpu_error;
 800         struct intel_engine_cs *engine;
 801         enum intel_engine_id id;
 802
 803         if (test_bit(I915_WEDGED, &error->flags))
 804                 return;
 805
 806         if (GEM_SHOW_DEBUG() && !intel_engines_are_idle(i915)) {
 807                 struct drm_printer p = drm_debug_printer(__func__);
 808
 809                 for_each_engine(engine, i915, id)
 810                         intel_engine_dump(engine, &p, "%s\n", engine->name);
 811         }
 812
 813         GEM_TRACE("start\n");
 814
 815         /*
 816          * First, stop submission to hw, but do not yet complete requests by
 817          * rolling the global seqno forward (since this would complete requests
 818          * for which we haven't set the fence error to EIO yet).
 819          */
 820         for_each_engine(engine, i915, id)
 821                 reset_prepare_engine(engine);
 822
 823         /* Even if the GPU reset fails, it should still stop the engines */
 824         if (!INTEL_INFO(i915)->gpu_reset_clobbers_display)
 825                 intel_gpu_reset(i915, ALL_ENGINES);
 826
 827         for_each_engine(engine, i915, id) {
 828                 engine->submit_request = nop_submit_request;
 829                 engine->schedule = NULL;
 830         }
 831         i915->caps.scheduler = 0;
 832
 833         /*
 834          * Make sure no request can slip through without getting completed by
 835          * either this call here to intel_engine_write_global_seqno, or the one
 836          * in nop_submit_request.
 837          */
 838         synchronize_rcu_expedited();
 839
 840         /* Mark all executing requests as skipped */
 841         for_each_engine(engine, i915, id)
 842                 engine->cancel_requests(engine);
 843
 844         for_each_engine(engine, i915, id) {
 845                 reset_finish_engine(engine);
 846                 intel_engine_signal_breadcrumbs(engine);
 847         }
 848
 849         smp_mb__before_atomic();
 850         set_bit(I915_WEDGED, &error->flags);
 851
 852         GEM_TRACE("end\n");
 853 }
 854
 855 void i915_gem_set_wedged(struct drm_i915_private *i915)
 856 {
 857         struct i915_gpu_error *error = &i915->gpu_error;
 858
 859         mutex_lock(&error->wedge_mutex);
 860         __i915_gem_set_wedged(i915);
 861         mutex_unlock(&error->wedge_mutex);
 862 }
 863
 864 static bool __i915_gem_unset_wedged(struct drm_i915_private *i915)
 865 {
 866         struct i915_gpu_error *error = &i915->gpu_error;
 867         struct i915_timeline *tl;
 868
 869         if (!test_bit(I915_WEDGED, &error->flags))
 870                 return true;
 871
 872         if (!i915->gt.scratch) /* Never full initialised, recovery impossible */
 873                 return false;
 874
 875         GEM_TRACE("start\n");
 876
 877         /*
 878          * Before unwedging, make sure that all pending operations
 879          * are flushed and errored out - we may have requests waiting upon
 880          * third party fences. We marked all inflight requests as EIO, and
 881          * every execbuf since returned EIO, for consistency we want all
 882          * the currently pending requests to also be marked as EIO, which
 883          * is done inside our nop_submit_request - and so we must wait.
 884          *
 885          * No more can be submitted until we reset the wedged bit.
 886          */
 887         mutex_lock(&i915->gt.timelines.mutex);
 888         list_for_each_entry(tl, &i915->gt.timelines.active_list, link) {
 889                 struct i915_request *rq;
 890
 891                 rq = i915_active_request_get_unlocked(&tl->last_request);
 892                 if (!rq)
 893                         continue;
 894
 895                 /*
 896                  * All internal dependencies (i915_requests) will have
 897                  * been flushed by the set-wedge, but we may be stuck waiting
 898                  * for external fences. These should all be capped to 10s
 899                  * (I915_FENCE_TIMEOUT) so this wait should not be unbounded
 900                  * in the worst case.
 901                  */
 902                 dma_fence_default_wait(&rq->fence, false, MAX_SCHEDULE_TIMEOUT);
 903                 i915_request_put(rq);
 904         }
 905         mutex_unlock(&i915->gt.timelines.mutex);
 906
 907         intel_engines_sanitize(i915, false);
 908
 909         /*
 910          * Undo nop_submit_request. We prevent all new i915 requests from
 911          * being queued (by disallowing execbuf whilst wedged) so having
 912          * waited for all active requests above, we know the system is idle
 913          * and do not have to worry about a thread being inside
 914          * engine->submit_request() as we swap over. So unlike installing
 915          * the nop_submit_request on reset, we can do this from normal
 916          * context and do not require stop_machine().
 917          */
 918         intel_engines_reset_default_submission(i915);
 919
 920         GEM_TRACE("end\n");
 921
 922         smp_mb__before_atomic(); /* complete takeover before enabling execbuf */
 923         clear_bit(I915_WEDGED, &i915->gpu_error.flags);
 924
 925         return true;
 926 }
 927
 928 bool i915_gem_unset_wedged(struct drm_i915_private *i915)
 929 {
 930         struct i915_gpu_error *error = &i915->gpu_error;
 931         bool result;
 932
 933         mutex_lock(&error->wedge_mutex);
 934         result = __i915_gem_unset_wedged(i915);
 935         mutex_unlock(&error->wedge_mutex);
 936
 937         return result;
 938 }
 939
 940 static int do_reset(struct drm_i915_private *i915, unsigned int stalled_mask)
 941 {
 942         int err, i;
 943
 944         err = intel_gpu_reset(i915, ALL_ENGINES);
 945         for (i = 0; err && i < RESET_MAX_RETRIES; i++) {
 946                 msleep(10 * (i + 1));
 947                 err = intel_gpu_reset(i915, ALL_ENGINES);
 948         }
 949         if (err)
 950                 return err;
 951
 952         return gt_reset(i915, stalled_mask);
 953 }
 954
 955 /**
 956  * i915_reset - reset chip after a hang
 957  * @i915: #drm_i915_private to reset
 958  * @stalled_mask: mask of the stalled engines with the guilty requests
 959  * @reason: user error message for why we are resetting
 960  *
 961  * Reset the chip.  Useful if a hang is detected. Marks the device as wedged
 962  * on failure.
 963  *
 964  * Procedure is fairly simple:
 965  *   - reset the chip using the reset reg
 966  *   - re-init context state
 967  *   - re-init hardware status page
 968  *   - re-init ring buffer
 969  *   - re-init interrupt state
 970  *   - re-init display
 971  */
 972 void i915_reset(struct drm_i915_private *i915,
 973                 unsigned int stalled_mask,
 974                 const char *reason)
 975 {
 976         struct i915_gpu_error *error = &i915->gpu_error;
 977         int ret;
 978
 979         GEM_TRACE("flags=%lx\n", error->flags);
 980
 981         might_sleep();
 982         assert_rpm_wakelock_held(i915);
 983         GEM_BUG_ON(!test_bit(I915_RESET_BACKOFF, &error->flags));
 984
 985         /* Clear any previous failed attempts at recovery. Time to try again. */
 986         if (!__i915_gem_unset_wedged(i915))
 987                 return;
 988
 989         if (reason)
 990                 dev_notice(i915->drm.dev, "Resetting chip for %s\n", reason);
 991         error->reset_count++;
 992
 993         reset_prepare(i915);
 994
 995         if (!intel_has_gpu_reset(i915)) {
 996                 if (i915_modparams.reset)
 997                         dev_err(i915->drm.dev, "GPU reset not supported\n");
 998                 else
 999                         DRM_DEBUG_DRIVER("GPU reset disabled\n");
1000                 goto error;
1001         }
1002
1003         if (INTEL_INFO(i915)->gpu_reset_clobbers_display)
1004                 intel_runtime_pm_disable_interrupts(i915);
1005
1006         if (do_reset(i915, stalled_mask)) {
1007                 dev_err(i915->drm.dev, "Failed to reset chip\n");
1008                 goto taint;
1009         }
1010
1011         if (INTEL_INFO(i915)->gpu_reset_clobbers_display)
1012                 intel_runtime_pm_enable_interrupts(i915);
1013
1014         intel_overlay_reset(i915);
1015
1016         /*
1017          * Next we need to restore the context, but we don't use those
1018          * yet either...
1019          *
1020          * Ring buffer needs to be re-initialized in the KMS case, or if X
1021          * was running at the time of the reset (i.e. we weren't VT
1022          * switched away).
1023          */
1024         ret = i915_gem_init_hw(i915);
1025         if (ret) {
1026                 DRM_ERROR("Failed to initialise HW following reset (%d)\n",
1027                           ret);
1028                 goto error;
1029         }
1030
1031         i915_queue_hangcheck(i915);
1032
1033 finish:
1034         reset_finish(i915);
1035         if (!__i915_wedged(error))
1036                 reset_restart(i915);
1037         return;
1038
1039 taint:
1040         /*
1041          * History tells us that if we cannot reset the GPU now, we
1042          * never will. This then impacts everything that is run
1043          * subsequently. On failing the reset, we mark the driver
1044          * as wedged, preventing further execution on the GPU.
1045          * We also want to go one step further and add a taint to the
1046          * kernel so that any subsequent faults can be traced back to
1047          * this failure. This is important for CI, where if the
1048          * GPU/driver fails we would like to reboot and restart testing
1049          * rather than continue on into oblivion. For everyone else,
1050          * the system should still plod along, but they have been warned!
1051          */
1052         add_taint(TAINT_WARN, LOCKDEP_STILL_OK);
1053 error:
1054         __i915_gem_set_wedged(i915);
1055         goto finish;
1056 }
1057
1058 static inline int intel_gt_reset_engine(struct drm_i915_private *i915,
1059                                         struct intel_engine_cs *engine)
1060 {
1061         return intel_gpu_reset(i915, engine->mask);
1062 }
1063
1064 /**
1065  * i915_reset_engine - reset GPU engine to recover from a hang
1066  * @engine: engine to reset
1067  * @msg: reason for GPU reset; or NULL for no dev_notice()
1068  *
1069  * Reset a specific GPU engine. Useful if a hang is detected.
1070  * Returns zero on successful reset or otherwise an error code.
1071  *
1072  * Procedure is:
1073  *  - identifies the request that caused the hang and it is dropped
1074  *  - reset engine (which will force the engine to idle)
1075  *  - re-init/configure engine
1076  */
1077 int i915_reset_engine(struct intel_engine_cs *engine, const char *msg)
1078 {
1079         struct i915_gpu_error *error = &engine->i915->gpu_error;
1080         int ret;
1081
1082         GEM_TRACE("%s flags=%lx\n", engine->name, error->flags);
1083         GEM_BUG_ON(!test_bit(I915_RESET_ENGINE + engine->id, &error->flags));
1084
1085         reset_prepare_engine(engine);
1086
1087         if (msg)
1088                 dev_notice(engine->i915->drm.dev,
1089                            "Resetting %s for %s\n", engine->name, msg);
1090         error->reset_engine_count[engine->id]++;
1091
1092         if (!engine->i915->guc.execbuf_client)
1093                 ret = intel_gt_reset_engine(engine->i915, engine);
1094         else
1095                 ret = intel_guc_reset_engine(&engine->i915->guc, engine);
1096         if (ret) {
1097                 /* If we fail here, we expect to fallback to a global reset */
1098                 DRM_DEBUG_DRIVER("%sFailed to reset %s, ret=%d\n",
1099                                  engine->i915->guc.execbuf_client ? "GuC " : "",
1100                                  engine->name, ret);
1101                 goto out;
1102         }
1103
1104         /*
1105          * The request that caused the hang is stuck on elsp, we know the
1106          * active request and can drop it, adjust head to skip the offending
1107          * request to resume executing remaining requests in the queue.
1108          */
1109         intel_engine_reset(engine, true);
1110
1111         /*
1112          * The engine and its registers (and workarounds in case of render)
1113          * have been reset to their default values. Follow the init_ring
1114          * process to program RING_MODE, HWSP and re-enable submission.
1115          */
1116         ret = engine->init_hw(engine);
1117         if (ret)
1118                 goto out;
1119
1120 out:
1121         intel_engine_cancel_stop_cs(engine);
1122         reset_finish_engine(engine);
1123         return ret;
1124 }
1125
1126 static void i915_reset_device(struct drm_i915_private *i915,
1127                               u32 engine_mask,
1128                               const char *reason)
1129 {
1130         struct i915_gpu_error *error = &i915->gpu_error;
1131         struct kobject *kobj = &i915->drm.primary->kdev->kobj;
1132         char *error_event[] = { I915_ERROR_UEVENT "=1", NULL };
1133         char *reset_event[] = { I915_RESET_UEVENT "=1", NULL };
1134         char *reset_done_event[] = { I915_ERROR_UEVENT "=0", NULL };
1135         struct i915_wedge_me w;
1136
1137         kobject_uevent_env(kobj, KOBJ_CHANGE, error_event);
1138
1139         DRM_DEBUG_DRIVER("resetting chip\n");
1140         kobject_uevent_env(kobj, KOBJ_CHANGE, reset_event);
1141
1142         /* Use a watchdog to ensure that our reset completes */
1143         i915_wedge_on_timeout(&w, i915, 5 * HZ) {
1144                 intel_prepare_reset(i915);
1145
1146                 /* Flush everyone using a resource about to be clobbered */
1147                 synchronize_srcu_expedited(&error->reset_backoff_srcu);
1148
1149                 mutex_lock(&error->wedge_mutex);
1150                 i915_reset(i915, engine_mask, reason);
1151                 mutex_unlock(&error->wedge_mutex);
1152
1153                 intel_finish_reset(i915);
1154         }
1155
1156         if (!test_bit(I915_WEDGED, &error->flags))
1157                 kobject_uevent_env(kobj, KOBJ_CHANGE, reset_done_event);
1158 }
1159
1160 void i915_clear_error_registers(struct drm_i915_private *dev_priv)
1161 {
1162         u32 eir;
1163
1164         if (!IS_GEN(dev_priv, 2))
1165                 I915_WRITE(PGTBL_ER, I915_READ(PGTBL_ER));
1166
1167         if (INTEL_GEN(dev_priv) < 4)
1168                 I915_WRITE(IPEIR, I915_READ(IPEIR));
1169         else
1170                 I915_WRITE(IPEIR_I965, I915_READ(IPEIR_I965));
1171
1172         I915_WRITE(EIR, I915_READ(EIR));
1173         eir = I915_READ(EIR);
1174         if (eir) {
1175                 /*
1176                  * some errors might have become stuck,
1177                  * mask them.
1178                  */
1179                 DRM_DEBUG_DRIVER("EIR stuck: 0x%08x, masking\n", eir);
1180                 I915_WRITE(EMR, I915_READ(EMR) | eir);
1181                 I915_WRITE(IIR, I915_MASTER_ERROR_INTERRUPT);
1182         }
1183
1184         if (INTEL_GEN(dev_priv) >= 8) {
1185                 I915_WRITE(GEN8_RING_FAULT_REG,
1186                            I915_READ(GEN8_RING_FAULT_REG) & ~RING_FAULT_VALID);
1187                 POSTING_READ(GEN8_RING_FAULT_REG);
1188         } else if (INTEL_GEN(dev_priv) >= 6) {
1189                 struct intel_engine_cs *engine;
1190                 enum intel_engine_id id;
1191
1192                 for_each_engine(engine, dev_priv, id) {
1193                         I915_WRITE(RING_FAULT_REG(engine),
1194                                    I915_READ(RING_FAULT_REG(engine)) &
1195                                    ~RING_FAULT_VALID);
1196                 }
1197                 POSTING_READ(RING_FAULT_REG(dev_priv->engine[RCS0]));
1198         }
1199 }
1200
1201 /**
1202  * i915_handle_error - handle a gpu error
1203  * @i915: i915 device private
1204  * @engine_mask: mask representing engines that are hung
1205  * @flags: control flags
1206  * @fmt: Error message format string
1207  *
1208  * Do some basic checking of register state at error time and
1209  * dump it to the syslog.  Also call i915_capture_error_state() to make
1210  * sure we get a record and make it available in debugfs.  Fire a uevent
1211  * so userspace knows something bad happened (should trigger collection
1212  * of a ring dump etc.).
1213  */
1214 void i915_handle_error(struct drm_i915_private *i915,
1215                        u32 engine_mask,
1216                        unsigned long flags,
1217                        const char *fmt, ...)
1218 {
1219         struct i915_gpu_error *error = &i915->gpu_error;
1220         struct intel_engine_cs *engine;
1221         intel_wakeref_t wakeref;
1222         unsigned int tmp;
1223         char error_msg[80];
1224         char *msg = NULL;
1225
1226         if (fmt) {
1227                 va_list args;
1228
1229                 va_start(args, fmt);
1230                 vscnprintf(error_msg, sizeof(error_msg), fmt, args);
1231                 va_end(args);
1232
1233                 msg = error_msg;
1234         }
1235
1236         /*
1237          * In most cases it's guaranteed that we get here with an RPM
1238          * reference held, for example because there is a pending GPU
1239          * request that won't finish until the reset is done. This
1240          * isn't the case at least when we get here by doing a
1241          * simulated reset via debugfs, so get an RPM reference.
1242          */
1243         wakeref = intel_runtime_pm_get(i915);
1244
1245         engine_mask &= INTEL_INFO(i915)->engine_mask;
1246
1247         if (flags & I915_ERROR_CAPTURE) {
1248                 i915_capture_error_state(i915, engine_mask, msg);
1249                 i915_clear_error_registers(i915);
1250         }
1251
1252         /*
1253          * Try engine reset when available. We fall back to full reset if
1254          * single reset fails.
1255          */
1256         if (intel_has_reset_engine(i915) && !__i915_wedged(error)) {
1257                 for_each_engine_masked(engine, i915, engine_mask, tmp) {
1258                         BUILD_BUG_ON(I915_RESET_MODESET >= I915_RESET_ENGINE);
1259                         if (test_and_set_bit(I915_RESET_ENGINE + engine->id,
1260                                              &error->flags))
1261                                 continue;
1262
1263                         if (i915_reset_engine(engine, msg) == 0)
1264                                 engine_mask &= ~engine->mask;
1265
1266                         clear_bit(I915_RESET_ENGINE + engine->id,
1267                                   &error->flags);
1268                         wake_up_bit(&error->flags,
1269                                     I915_RESET_ENGINE + engine->id);
1270                 }
1271         }
1272
1273         if (!engine_mask)
1274                 goto out;
1275
1276         /* Full reset needs the mutex, stop any other user trying to do so. */
1277         if (test_and_set_bit(I915_RESET_BACKOFF, &error->flags)) {
1278                 wait_event(error->reset_queue,
1279                            !test_bit(I915_RESET_BACKOFF, &error->flags));
1280                 goto out; /* piggy-back on the other reset */
1281         }
1282
1283         /* Make sure i915_reset_trylock() sees the I915_RESET_BACKOFF */
1284         synchronize_rcu_expedited();
1285
1286         /* Prevent any other reset-engine attempt. */
1287         for_each_engine(engine, i915, tmp) {
1288                 while (test_and_set_bit(I915_RESET_ENGINE + engine->id,
1289                                         &error->flags))
1290                         wait_on_bit(&error->flags,
1291                                     I915_RESET_ENGINE + engine->id,
1292                                     TASK_UNINTERRUPTIBLE);
1293         }
1294
1295         i915_reset_device(i915, engine_mask, msg);
1296
1297         for_each_engine(engine, i915, tmp) {
1298                 clear_bit(I915_RESET_ENGINE + engine->id,
1299                           &error->flags);
1300         }
1301
1302         clear_bit(I915_RESET_BACKOFF, &error->flags);
1303         wake_up_all(&error->reset_queue);
1304
1305 out:
1306         intel_runtime_pm_put(i915, wakeref);
1307 }
1308
1309 int i915_reset_trylock(struct drm_i915_private *i915)
1310 {
1311         struct i915_gpu_error *error = &i915->gpu_error;
1312         int srcu;
1313
1314         might_lock(&error->reset_backoff_srcu);
1315         might_sleep();
1316
1317         rcu_read_lock();
1318         while (test_bit(I915_RESET_BACKOFF, &error->flags)) {
1319                 rcu_read_unlock();
1320
1321                 if (wait_event_interruptible(error->reset_queue,
1322                                              !test_bit(I915_RESET_BACKOFF,
1323                                                        &error->flags)))
1324                         return -EINTR;
1325
1326                 rcu_read_lock();
1327         }
1328         srcu = srcu_read_lock(&error->reset_backoff_srcu);
1329         rcu_read_unlock();
1330
1331         return srcu;
1332 }
1333
1334 void i915_reset_unlock(struct drm_i915_private *i915, int tag)
1335 __releases(&i915->gpu_error.reset_backoff_srcu)
1336 {
1337         struct i915_gpu_error *error = &i915->gpu_error;
1338
1339         srcu_read_unlock(&error->reset_backoff_srcu, tag);
1340 }
1341
1342 int i915_terminally_wedged(struct drm_i915_private *i915)
1343 {
1344         struct i915_gpu_error *error = &i915->gpu_error;
1345
1346         might_sleep();
1347
1348         if (!__i915_wedged(error))
1349                 return 0;
1350
1351         /* Reset still in progress? Maybe we will recover? */
1352         if (!test_bit(I915_RESET_BACKOFF, &error->flags))
1353                 return -EIO;
1354
1355         /* XXX intel_reset_finish() still takes struct_mutex!!! */
1356         if (mutex_is_locked(&i915->drm.struct_mutex))
1357                 return -EAGAIN;
1358
1359         if (wait_event_interruptible(error->reset_queue,
1360                                      !test_bit(I915_RESET_BACKOFF,
1361                                                &error->flags)))
1362                 return -EINTR;
1363
1364         return __i915_wedged(error) ? -EIO : 0;
1365 }
1366
1367 bool i915_reset_flush(struct drm_i915_private *i915)
1368 {
1369         int err;
1370
1371         cancel_delayed_work_sync(&i915->gpu_error.hangcheck_work);
1372
1373         flush_workqueue(i915->wq);
1374         GEM_BUG_ON(READ_ONCE(i915->gpu_error.restart));
1375
1376         mutex_lock(&i915->drm.struct_mutex);
1377         err = i915_gem_wait_for_idle(i915,
1378                                      I915_WAIT_LOCKED |
1379                                      I915_WAIT_FOR_IDLE_BOOST,
1380                                      MAX_SCHEDULE_TIMEOUT);
1381         mutex_unlock(&i915->drm.struct_mutex);
1382
1383         return !err;
1384 }
1385
1386 static void i915_wedge_me(struct work_struct *work)
1387 {
1388         struct i915_wedge_me *w = container_of(work, typeof(*w), work.work);
1389
1390         dev_err(w->i915->drm.dev,
1391                 "%s timed out, cancelling all in-flight rendering.\n",
1392                 w->name);
1393         i915_gem_set_wedged(w->i915);
1394 }
1395
1396 void __i915_init_wedge(struct i915_wedge_me *w,
1397                        struct drm_i915_private *i915,
1398                        long timeout,
1399                        const char *name)
1400 {
1401         w->i915 = i915;
1402         w->name = name;
1403
1404         INIT_DELAYED_WORK_ONSTACK(&w->work, i915_wedge_me);
1405         schedule_delayed_work(&w->work, timeout);
1406 }
1407
1408 void __i915_fini_wedge(struct i915_wedge_me *w)
1409 {
1410         cancel_delayed_work_sync(&w->work);
1411         destroy_delayed_work_on_stack(&w->work);
1412         w->i915 = NULL;
1413 }