drivers/gpu/drm/i915/intel_uncore.c

   1 /*
   2  * Copyright © 2013 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #include <drm/drm_managed.h>
  25 #include <linux/pm_runtime.h>
  26
  27 #include "gt/intel_engine_regs.h"
  28 #include "gt/intel_gt_regs.h"
  29
  30 #include "i915_drv.h"
  31 #include "i915_iosf_mbi.h"
  32 #include "i915_reg.h"
  33 #include "i915_trace.h"
  34 #include "i915_vgpu.h"
  35 #include "intel_pm.h"
  36
  37 #define FORCEWAKE_ACK_TIMEOUT_MS 50
  38 #define GT_FIFO_TIMEOUT_MS       10
  39
  40 #define __raw_posting_read(...) ((void)__raw_uncore_read32(__VA_ARGS__))
  41
  42 static void
  43 fw_domains_get(struct intel_uncore *uncore, enum forcewake_domains fw_domains)
  44 {
  45         uncore->fw_get_funcs->force_wake_get(uncore, fw_domains);
  46 }
  47
  48 void
  49 intel_uncore_mmio_debug_init_early(struct drm_i915_private *i915)
  50 {
  51         spin_lock_init(&i915->mmio_debug.lock);
  52         i915->mmio_debug.unclaimed_mmio_check = 1;
  53
  54         i915->uncore.debug = &i915->mmio_debug;
  55 }
  56
  57 static void mmio_debug_suspend(struct intel_uncore *uncore)
  58 {
  59         if (!uncore->debug)
  60                 return;
  61
  62         spin_lock(&uncore->debug->lock);
  63
  64         /* Save and disable mmio debugging for the user bypass */
  65         if (!uncore->debug->suspend_count++) {
  66                 uncore->debug->saved_mmio_check = uncore->debug->unclaimed_mmio_check;
  67                 uncore->debug->unclaimed_mmio_check = 0;
  68         }
  69
  70         spin_unlock(&uncore->debug->lock);
  71 }
  72
  73 static bool check_for_unclaimed_mmio(struct intel_uncore *uncore);
  74
  75 static void mmio_debug_resume(struct intel_uncore *uncore)
  76 {
  77         if (!uncore->debug)
  78                 return;
  79
  80         spin_lock(&uncore->debug->lock);
  81
  82         if (!--uncore->debug->suspend_count)
  83                 uncore->debug->unclaimed_mmio_check = uncore->debug->saved_mmio_check;
  84
  85         if (check_for_unclaimed_mmio(uncore))
  86                 drm_info(&uncore->i915->drm,
  87                          "Invalid mmio detected during user access\n");
  88
  89         spin_unlock(&uncore->debug->lock);
  90 }
  91
  92 static const char * const forcewake_domain_names[] = {
  93         "render",
  94         "gt",
  95         "media",
  96         "vdbox0",
  97         "vdbox1",
  98         "vdbox2",
  99         "vdbox3",
 100         "vdbox4",
 101         "vdbox5",
 102         "vdbox6",
 103         "vdbox7",
 104         "vebox0",
 105         "vebox1",
 106         "vebox2",
 107         "vebox3",
 108         "gsc",
 109 };
 110
 111 const char *
 112 intel_uncore_forcewake_domain_to_str(const enum forcewake_domain_id id)
 113 {
 114         BUILD_BUG_ON(ARRAY_SIZE(forcewake_domain_names) != FW_DOMAIN_ID_COUNT);
 115
 116         if (id >= 0 && id < FW_DOMAIN_ID_COUNT)
 117                 return forcewake_domain_names[id];
 118
 119         WARN_ON(id);
 120
 121         return "unknown";
 122 }
 123
 124 #define fw_ack(d) readl((d)->reg_ack)
 125 #define fw_set(d, val) writel(_MASKED_BIT_ENABLE((val)), (d)->reg_set)
 126 #define fw_clear(d, val) writel(_MASKED_BIT_DISABLE((val)), (d)->reg_set)
 127
 128 static inline void
 129 fw_domain_reset(const struct intel_uncore_forcewake_domain *d)
 130 {
 131         /*
 132          * We don't really know if the powerwell for the forcewake domain we are
 133          * trying to reset here does exist at this point (engines could be fused
 134          * off in ICL+), so no waiting for acks
 135          */
 136         /* WaRsClearFWBitsAtReset */
 137         if (GRAPHICS_VER(d->uncore->i915) >= 12)
 138                 fw_clear(d, 0xefff);
 139         else
 140                 fw_clear(d, 0xffff);
 141 }
 142
 143 static inline void
 144 fw_domain_arm_timer(struct intel_uncore_forcewake_domain *d)
 145 {
 146         GEM_BUG_ON(d->uncore->fw_domains_timer & d->mask);
 147         d->uncore->fw_domains_timer |= d->mask;
 148         d->wake_count++;
 149         hrtimer_start_range_ns(&d->timer,
 150                                NSEC_PER_MSEC,
 151                                NSEC_PER_MSEC,
 152                                HRTIMER_MODE_REL);
 153 }
 154
 155 static inline int
 156 __wait_for_ack(const struct intel_uncore_forcewake_domain *d,
 157                const u32 ack,
 158                const u32 value)
 159 {
 160         return wait_for_atomic((fw_ack(d) & ack) == value,
 161                                FORCEWAKE_ACK_TIMEOUT_MS);
 162 }
 163
 164 static inline int
 165 wait_ack_clear(const struct intel_uncore_forcewake_domain *d,
 166                const u32 ack)
 167 {
 168         return __wait_for_ack(d, ack, 0);
 169 }
 170
 171 static inline int
 172 wait_ack_set(const struct intel_uncore_forcewake_domain *d,
 173              const u32 ack)
 174 {
 175         return __wait_for_ack(d, ack, ack);
 176 }
 177
 178 static inline void
 179 fw_domain_wait_ack_clear(const struct intel_uncore_forcewake_domain *d)
 180 {
 181         if (wait_ack_clear(d, FORCEWAKE_KERNEL)) {
 182                 DRM_ERROR("%s: timed out waiting for forcewake ack to clear.\n",
 183                           intel_uncore_forcewake_domain_to_str(d->id));
 184                 add_taint_for_CI(d->uncore->i915, TAINT_WARN); /* CI now unreliable */
 185         }
 186 }
 187
 188 enum ack_type {
 189         ACK_CLEAR = 0,
 190         ACK_SET
 191 };
 192
 193 static int
 194 fw_domain_wait_ack_with_fallback(const struct intel_uncore_forcewake_domain *d,
 195                                  const enum ack_type type)
 196 {
 197         const u32 ack_bit = FORCEWAKE_KERNEL;
 198         const u32 value = type == ACK_SET ? ack_bit : 0;
 199         unsigned int pass;
 200         bool ack_detected;
 201
 202         /*
 203          * There is a possibility of driver's wake request colliding
 204          * with hardware's own wake requests and that can cause
 205          * hardware to not deliver the driver's ack message.
 206          *
 207          * Use a fallback bit toggle to kick the gpu state machine
 208          * in the hope that the original ack will be delivered along with
 209          * the fallback ack.
 210          *
 211          * This workaround is described in HSDES #1604254524 and it's known as:
 212          * WaRsForcewakeAddDelayForAck:skl,bxt,kbl,glk,cfl,cnl,icl
 213          * although the name is a bit misleading.
 214          */
 215
 216         pass = 1;
 217         do {
 218                 wait_ack_clear(d, FORCEWAKE_KERNEL_FALLBACK);
 219
 220                 fw_set(d, FORCEWAKE_KERNEL_FALLBACK);
 221                 /* Give gt some time to relax before the polling frenzy */
 222                 udelay(10 * pass);
 223                 wait_ack_set(d, FORCEWAKE_KERNEL_FALLBACK);
 224
 225                 ack_detected = (fw_ack(d) & ack_bit) == value;
 226
 227                 fw_clear(d, FORCEWAKE_KERNEL_FALLBACK);
 228         } while (!ack_detected && pass++ < 10);
 229
 230         DRM_DEBUG_DRIVER("%s had to use fallback to %s ack, 0x%x (passes %u)\n",
 231                          intel_uncore_forcewake_domain_to_str(d->id),
 232                          type == ACK_SET ? "set" : "clear",
 233                          fw_ack(d),
 234                          pass);
 235
 236         return ack_detected ? 0 : -ETIMEDOUT;
 237 }
 238
 239 static inline void
 240 fw_domain_wait_ack_clear_fallback(const struct intel_uncore_forcewake_domain *d)
 241 {
 242         if (likely(!wait_ack_clear(d, FORCEWAKE_KERNEL)))
 243                 return;
 244
 245         if (fw_domain_wait_ack_with_fallback(d, ACK_CLEAR))
 246                 fw_domain_wait_ack_clear(d);
 247 }
 248
 249 static inline void
 250 fw_domain_get(const struct intel_uncore_forcewake_domain *d)
 251 {
 252         fw_set(d, FORCEWAKE_KERNEL);
 253 }
 254
 255 static inline void
 256 fw_domain_wait_ack_set(const struct intel_uncore_forcewake_domain *d)
 257 {
 258         if (wait_ack_set(d, FORCEWAKE_KERNEL)) {
 259                 DRM_ERROR("%s: timed out waiting for forcewake ack request.\n",
 260                           intel_uncore_forcewake_domain_to_str(d->id));
 261                 add_taint_for_CI(d->uncore->i915, TAINT_WARN); /* CI now unreliable */
 262         }
 263 }
 264
 265 static inline void
 266 fw_domain_wait_ack_set_fallback(const struct intel_uncore_forcewake_domain *d)
 267 {
 268         if (likely(!wait_ack_set(d, FORCEWAKE_KERNEL)))
 269                 return;
 270
 271         if (fw_domain_wait_ack_with_fallback(d, ACK_SET))
 272                 fw_domain_wait_ack_set(d);
 273 }
 274
 275 static inline void
 276 fw_domain_put(const struct intel_uncore_forcewake_domain *d)
 277 {
 278         fw_clear(d, FORCEWAKE_KERNEL);
 279 }
 280
 281 static void
 282 fw_domains_get_normal(struct intel_uncore *uncore, enum forcewake_domains fw_domains)
 283 {
 284         struct intel_uncore_forcewake_domain *d;
 285         unsigned int tmp;
 286
 287         GEM_BUG_ON(fw_domains & ~uncore->fw_domains);
 288
 289         for_each_fw_domain_masked(d, fw_domains, uncore, tmp) {
 290                 fw_domain_wait_ack_clear(d);
 291                 fw_domain_get(d);
 292         }
 293
 294         for_each_fw_domain_masked(d, fw_domains, uncore, tmp)
 295                 fw_domain_wait_ack_set(d);
 296
 297         uncore->fw_domains_active |= fw_domains;
 298 }
 299
 300 static void
 301 fw_domains_get_with_fallback(struct intel_uncore *uncore,
 302                              enum forcewake_domains fw_domains)
 303 {
 304         struct intel_uncore_forcewake_domain *d;
 305         unsigned int tmp;
 306
 307         GEM_BUG_ON(fw_domains & ~uncore->fw_domains);
 308
 309         for_each_fw_domain_masked(d, fw_domains, uncore, tmp) {
 310                 fw_domain_wait_ack_clear_fallback(d);
 311                 fw_domain_get(d);
 312         }
 313
 314         for_each_fw_domain_masked(d, fw_domains, uncore, tmp)
 315                 fw_domain_wait_ack_set_fallback(d);
 316
 317         uncore->fw_domains_active |= fw_domains;
 318 }
 319
 320 static void
 321 fw_domains_put(struct intel_uncore *uncore, enum forcewake_domains fw_domains)
 322 {
 323         struct intel_uncore_forcewake_domain *d;
 324         unsigned int tmp;
 325
 326         GEM_BUG_ON(fw_domains & ~uncore->fw_domains);
 327
 328         for_each_fw_domain_masked(d, fw_domains, uncore, tmp)
 329                 fw_domain_put(d);
 330
 331         uncore->fw_domains_active &= ~fw_domains;
 332 }
 333
 334 static void
 335 fw_domains_reset(struct intel_uncore *uncore,
 336                  enum forcewake_domains fw_domains)
 337 {
 338         struct intel_uncore_forcewake_domain *d;
 339         unsigned int tmp;
 340
 341         if (!fw_domains)
 342                 return;
 343
 344         GEM_BUG_ON(fw_domains & ~uncore->fw_domains);
 345
 346         for_each_fw_domain_masked(d, fw_domains, uncore, tmp)
 347                 fw_domain_reset(d);
 348 }
 349
 350 static inline u32 gt_thread_status(struct intel_uncore *uncore)
 351 {
 352         u32 val;
 353
 354         val = __raw_uncore_read32(uncore, GEN6_GT_THREAD_STATUS_REG);
 355         val &= GEN6_GT_THREAD_STATUS_CORE_MASK;
 356
 357         return val;
 358 }
 359
 360 static void __gen6_gt_wait_for_thread_c0(struct intel_uncore *uncore)
 361 {
 362         /*
 363          * w/a for a sporadic read returning 0 by waiting for the GT
 364          * thread to wake up.
 365          */
 366         drm_WARN_ONCE(&uncore->i915->drm,
 367                       wait_for_atomic_us(gt_thread_status(uncore) == 0, 5000),
 368                       "GT thread status wait timed out\n");
 369 }
 370
 371 static void fw_domains_get_with_thread_status(struct intel_uncore *uncore,
 372                                               enum forcewake_domains fw_domains)
 373 {
 374         fw_domains_get_normal(uncore, fw_domains);
 375
 376         /* WaRsForcewakeWaitTC0:snb,ivb,hsw,bdw,vlv */
 377         __gen6_gt_wait_for_thread_c0(uncore);
 378 }
 379
 380 static inline u32 fifo_free_entries(struct intel_uncore *uncore)
 381 {
 382         u32 count = __raw_uncore_read32(uncore, GTFIFOCTL);
 383
 384         return count & GT_FIFO_FREE_ENTRIES_MASK;
 385 }
 386
 387 static void __gen6_gt_wait_for_fifo(struct intel_uncore *uncore)
 388 {
 389         u32 n;
 390
 391         /* On VLV, FIFO will be shared by both SW and HW.
 392          * So, we need to read the FREE_ENTRIES everytime */
 393         if (IS_VALLEYVIEW(uncore->i915))
 394                 n = fifo_free_entries(uncore);
 395         else
 396                 n = uncore->fifo_count;
 397
 398         if (n <= GT_FIFO_NUM_RESERVED_ENTRIES) {
 399                 if (wait_for_atomic((n = fifo_free_entries(uncore)) >
 400                                     GT_FIFO_NUM_RESERVED_ENTRIES,
 401                                     GT_FIFO_TIMEOUT_MS)) {
 402                         drm_dbg(&uncore->i915->drm,
 403                                 "GT_FIFO timeout, entries: %u\n", n);
 404                         return;
 405                 }
 406         }
 407
 408         uncore->fifo_count = n - 1;
 409 }
 410
 411 static enum hrtimer_restart
 412 intel_uncore_fw_release_timer(struct hrtimer *timer)
 413 {
 414         struct intel_uncore_forcewake_domain *domain =
 415                container_of(timer, struct intel_uncore_forcewake_domain, timer);
 416         struct intel_uncore *uncore = domain->uncore;
 417         unsigned long irqflags;
 418
 419         assert_rpm_device_not_suspended(uncore->rpm);
 420
 421         if (xchg(&domain->active, false))
 422                 return HRTIMER_RESTART;
 423
 424         spin_lock_irqsave(&uncore->lock, irqflags);
 425
 426         uncore->fw_domains_timer &= ~domain->mask;
 427
 428         GEM_BUG_ON(!domain->wake_count);
 429         if (--domain->wake_count == 0)
 430                 fw_domains_put(uncore, domain->mask);
 431
 432         spin_unlock_irqrestore(&uncore->lock, irqflags);
 433
 434         return HRTIMER_NORESTART;
 435 }
 436
 437 /* Note callers must have acquired the PUNIT->PMIC bus, before calling this. */
 438 static unsigned int
 439 intel_uncore_forcewake_reset(struct intel_uncore *uncore)
 440 {
 441         unsigned long irqflags;
 442         struct intel_uncore_forcewake_domain *domain;
 443         int retry_count = 100;
 444         enum forcewake_domains fw, active_domains;
 445
 446         iosf_mbi_assert_punit_acquired();
 447
 448         /* Hold uncore.lock across reset to prevent any register access
 449          * with forcewake not set correctly. Wait until all pending
 450          * timers are run before holding.
 451          */
 452         while (1) {
 453                 unsigned int tmp;
 454
 455                 active_domains = 0;
 456
 457                 for_each_fw_domain(domain, uncore, tmp) {
 458                         smp_store_mb(domain->active, false);
 459                         if (hrtimer_cancel(&domain->timer) == 0)
 460                                 continue;
 461
 462                         intel_uncore_fw_release_timer(&domain->timer);
 463                 }
 464
 465                 spin_lock_irqsave(&uncore->lock, irqflags);
 466
 467                 for_each_fw_domain(domain, uncore, tmp) {
 468                         if (hrtimer_active(&domain->timer))
 469                                 active_domains |= domain->mask;
 470                 }
 471
 472                 if (active_domains == 0)
 473                         break;
 474
 475                 if (--retry_count == 0) {
 476                         drm_err(&uncore->i915->drm, "Timed out waiting for forcewake timers to finish\n");
 477                         break;
 478                 }
 479
 480                 spin_unlock_irqrestore(&uncore->lock, irqflags);
 481                 cond_resched();
 482         }
 483
 484         drm_WARN_ON(&uncore->i915->drm, active_domains);
 485
 486         fw = uncore->fw_domains_active;
 487         if (fw)
 488                 fw_domains_put(uncore, fw);
 489
 490         fw_domains_reset(uncore, uncore->fw_domains);
 491         assert_forcewakes_inactive(uncore);
 492
 493         spin_unlock_irqrestore(&uncore->lock, irqflags);
 494
 495         return fw; /* track the lost user forcewake domains */
 496 }
 497
 498 static bool
 499 fpga_check_for_unclaimed_mmio(struct intel_uncore *uncore)
 500 {
 501         u32 dbg;
 502
 503         dbg = __raw_uncore_read32(uncore, FPGA_DBG);
 504         if (likely(!(dbg & FPGA_DBG_RM_NOCLAIM)))
 505                 return false;
 506
 507         /*
 508          * Bugs in PCI programming (or failing hardware) can occasionally cause
 509          * us to lose access to the MMIO BAR.  When this happens, register
 510          * reads will come back with 0xFFFFFFFF for every register and things
 511          * go bad very quickly.  Let's try to detect that special case and at
 512          * least try to print a more informative message about what has
 513          * happened.
 514          *
 515          * During normal operation the FPGA_DBG register has several unused
 516          * bits that will always read back as 0's so we can use them as canaries
 517          * to recognize when MMIO accesses are just busted.
 518          */
 519         if (unlikely(dbg == ~0))
 520                 drm_err(&uncore->i915->drm,
 521                         "Lost access to MMIO BAR; all registers now read back as 0xFFFFFFFF!\n");
 522
 523         __raw_uncore_write32(uncore, FPGA_DBG, FPGA_DBG_RM_NOCLAIM);
 524
 525         return true;
 526 }
 527
 528 static bool
 529 vlv_check_for_unclaimed_mmio(struct intel_uncore *uncore)
 530 {
 531         u32 cer;
 532
 533         cer = __raw_uncore_read32(uncore, CLAIM_ER);
 534         if (likely(!(cer & (CLAIM_ER_OVERFLOW | CLAIM_ER_CTR_MASK))))
 535                 return false;
 536
 537         __raw_uncore_write32(uncore, CLAIM_ER, CLAIM_ER_CLR);
 538
 539         return true;
 540 }
 541
 542 static bool
 543 gen6_check_for_fifo_debug(struct intel_uncore *uncore)
 544 {
 545         u32 fifodbg;
 546
 547         fifodbg = __raw_uncore_read32(uncore, GTFIFODBG);
 548
 549         if (unlikely(fifodbg)) {
 550                 drm_dbg(&uncore->i915->drm, "GTFIFODBG = 0x08%x\n", fifodbg);
 551                 __raw_uncore_write32(uncore, GTFIFODBG, fifodbg);
 552         }
 553
 554         return fifodbg;
 555 }
 556
 557 static bool
 558 check_for_unclaimed_mmio(struct intel_uncore *uncore)
 559 {
 560         bool ret = false;
 561
 562         lockdep_assert_held(&uncore->debug->lock);
 563
 564         if (uncore->debug->suspend_count)
 565                 return false;
 566
 567         if (intel_uncore_has_fpga_dbg_unclaimed(uncore))
 568                 ret |= fpga_check_for_unclaimed_mmio(uncore);
 569
 570         if (intel_uncore_has_dbg_unclaimed(uncore))
 571                 ret |= vlv_check_for_unclaimed_mmio(uncore);
 572
 573         if (intel_uncore_has_fifo(uncore))
 574                 ret |= gen6_check_for_fifo_debug(uncore);
 575
 576         return ret;
 577 }
 578
 579 static void forcewake_early_sanitize(struct intel_uncore *uncore,
 580                                      unsigned int restore_forcewake)
 581 {
 582         GEM_BUG_ON(!intel_uncore_has_forcewake(uncore));
 583
 584         /* WaDisableShadowRegForCpd:chv */
 585         if (IS_CHERRYVIEW(uncore->i915)) {
 586                 __raw_uncore_write32(uncore, GTFIFOCTL,
 587                                      __raw_uncore_read32(uncore, GTFIFOCTL) |
 588                                      GT_FIFO_CTL_BLOCK_ALL_POLICY_STALL |
 589                                      GT_FIFO_CTL_RC6_POLICY_STALL);
 590         }
 591
 592         iosf_mbi_punit_acquire();
 593         intel_uncore_forcewake_reset(uncore);
 594         if (restore_forcewake) {
 595                 spin_lock_irq(&uncore->lock);
 596                 fw_domains_get(uncore, restore_forcewake);
 597
 598                 if (intel_uncore_has_fifo(uncore))
 599                         uncore->fifo_count = fifo_free_entries(uncore);
 600                 spin_unlock_irq(&uncore->lock);
 601         }
 602         iosf_mbi_punit_release();
 603 }
 604
 605 void intel_uncore_suspend(struct intel_uncore *uncore)
 606 {
 607         if (!intel_uncore_has_forcewake(uncore))
 608                 return;
 609
 610         iosf_mbi_punit_acquire();
 611         iosf_mbi_unregister_pmic_bus_access_notifier_unlocked(
 612                 &uncore->pmic_bus_access_nb);
 613         uncore->fw_domains_saved = intel_uncore_forcewake_reset(uncore);
 614         iosf_mbi_punit_release();
 615 }
 616
 617 void intel_uncore_resume_early(struct intel_uncore *uncore)
 618 {
 619         unsigned int restore_forcewake;
 620
 621         if (intel_uncore_unclaimed_mmio(uncore))
 622                 drm_dbg(&uncore->i915->drm, "unclaimed mmio detected on resume, clearing\n");
 623
 624         if (!intel_uncore_has_forcewake(uncore))
 625                 return;
 626
 627         restore_forcewake = fetch_and_zero(&uncore->fw_domains_saved);
 628         forcewake_early_sanitize(uncore, restore_forcewake);
 629
 630         iosf_mbi_register_pmic_bus_access_notifier(&uncore->pmic_bus_access_nb);
 631 }
 632
 633 void intel_uncore_runtime_resume(struct intel_uncore *uncore)
 634 {
 635         if (!intel_uncore_has_forcewake(uncore))
 636                 return;
 637
 638         iosf_mbi_register_pmic_bus_access_notifier(&uncore->pmic_bus_access_nb);
 639 }
 640
 641 static void __intel_uncore_forcewake_get(struct intel_uncore *uncore,
 642                                          enum forcewake_domains fw_domains)
 643 {
 644         struct intel_uncore_forcewake_domain *domain;
 645         unsigned int tmp;
 646
 647         fw_domains &= uncore->fw_domains;
 648
 649         for_each_fw_domain_masked(domain, fw_domains, uncore, tmp) {
 650                 if (domain->wake_count++) {
 651                         fw_domains &= ~domain->mask;
 652                         domain->active = true;
 653                 }
 654         }
 655
 656         if (fw_domains)
 657                 fw_domains_get(uncore, fw_domains);
 658 }
 659
 660 /**
 661  * intel_uncore_forcewake_get - grab forcewake domain references
 662  * @uncore: the intel_uncore structure
 663  * @fw_domains: forcewake domains to get reference on
 664  *
 665  * This function can be used get GT's forcewake domain references.
 666  * Normal register access will handle the forcewake domains automatically.
 667  * However if some sequence requires the GT to not power down a particular
 668  * forcewake domains this function should be called at the beginning of the
 669  * sequence. And subsequently the reference should be dropped by symmetric
 670  * call to intel_unforce_forcewake_put(). Usually caller wants all the domains
 671  * to be kept awake so the @fw_domains would be then FORCEWAKE_ALL.
 672  */
 673 void intel_uncore_forcewake_get(struct intel_uncore *uncore,
 674                                 enum forcewake_domains fw_domains)
 675 {
 676         unsigned long irqflags;
 677
 678         if (!uncore->fw_get_funcs)
 679                 return;
 680
 681         assert_rpm_wakelock_held(uncore->rpm);
 682
 683         spin_lock_irqsave(&uncore->lock, irqflags);
 684         __intel_uncore_forcewake_get(uncore, fw_domains);
 685         spin_unlock_irqrestore(&uncore->lock, irqflags);
 686 }
 687
 688 /**
 689  * intel_uncore_forcewake_user_get - claim forcewake on behalf of userspace
 690  * @uncore: the intel_uncore structure
 691  *
 692  * This function is a wrapper around intel_uncore_forcewake_get() to acquire
 693  * the GT powerwell and in the process disable our debugging for the
 694  * duration of userspace's bypass.
 695  */
 696 void intel_uncore_forcewake_user_get(struct intel_uncore *uncore)
 697 {
 698         spin_lock_irq(&uncore->lock);
 699         if (!uncore->user_forcewake_count++) {
 700                 intel_uncore_forcewake_get__locked(uncore, FORCEWAKE_ALL);
 701                 mmio_debug_suspend(uncore);
 702         }
 703         spin_unlock_irq(&uncore->lock);
 704 }
 705
 706 /**
 707  * intel_uncore_forcewake_user_put - release forcewake on behalf of userspace
 708  * @uncore: the intel_uncore structure
 709  *
 710  * This function complements intel_uncore_forcewake_user_get() and releases
 711  * the GT powerwell taken on behalf of the userspace bypass.
 712  */
 713 void intel_uncore_forcewake_user_put(struct intel_uncore *uncore)
 714 {
 715         spin_lock_irq(&uncore->lock);
 716         if (!--uncore->user_forcewake_count) {
 717                 mmio_debug_resume(uncore);
 718                 intel_uncore_forcewake_put__locked(uncore, FORCEWAKE_ALL);
 719         }
 720         spin_unlock_irq(&uncore->lock);
 721 }
 722
 723 /**
 724  * intel_uncore_forcewake_get__locked - grab forcewake domain references
 725  * @uncore: the intel_uncore structure
 726  * @fw_domains: forcewake domains to get reference on
 727  *
 728  * See intel_uncore_forcewake_get(). This variant places the onus
 729  * on the caller to explicitly handle the dev_priv->uncore.lock spinlock.
 730  */
 731 void intel_uncore_forcewake_get__locked(struct intel_uncore *uncore,
 732                                         enum forcewake_domains fw_domains)
 733 {
 734         lockdep_assert_held(&uncore->lock);
 735
 736         if (!uncore->fw_get_funcs)
 737                 return;
 738
 739         __intel_uncore_forcewake_get(uncore, fw_domains);
 740 }
 741
 742 static void __intel_uncore_forcewake_put(struct intel_uncore *uncore,
 743                                          enum forcewake_domains fw_domains,
 744                                          bool delayed)
 745 {
 746         struct intel_uncore_forcewake_domain *domain;
 747         unsigned int tmp;
 748
 749         fw_domains &= uncore->fw_domains;
 750
 751         for_each_fw_domain_masked(domain, fw_domains, uncore, tmp) {
 752                 GEM_BUG_ON(!domain->wake_count);
 753
 754                 if (--domain->wake_count) {
 755                         domain->active = true;
 756                         continue;
 757                 }
 758
 759                 if (delayed &&
 760                     !(domain->uncore->fw_domains_timer & domain->mask))
 761                         fw_domain_arm_timer(domain);
 762                 else
 763                         fw_domains_put(uncore, domain->mask);
 764         }
 765 }
 766
 767 /**
 768  * intel_uncore_forcewake_put - release a forcewake domain reference
 769  * @uncore: the intel_uncore structure
 770  * @fw_domains: forcewake domains to put references
 771  *
 772  * This function drops the device-level forcewakes for specified
 773  * domains obtained by intel_uncore_forcewake_get().
 774  */
 775 void intel_uncore_forcewake_put(struct intel_uncore *uncore,
 776                                 enum forcewake_domains fw_domains)
 777 {
 778         unsigned long irqflags;
 779
 780         if (!uncore->fw_get_funcs)
 781                 return;
 782
 783         spin_lock_irqsave(&uncore->lock, irqflags);
 784         __intel_uncore_forcewake_put(uncore, fw_domains, false);
 785         spin_unlock_irqrestore(&uncore->lock, irqflags);
 786 }
 787
 788 void intel_uncore_forcewake_put_delayed(struct intel_uncore *uncore,
 789                                         enum forcewake_domains fw_domains)
 790 {
 791         unsigned long irqflags;
 792
 793         if (!uncore->fw_get_funcs)
 794                 return;
 795
 796         spin_lock_irqsave(&uncore->lock, irqflags);
 797         __intel_uncore_forcewake_put(uncore, fw_domains, true);
 798         spin_unlock_irqrestore(&uncore->lock, irqflags);
 799 }
 800
 801 /**
 802  * intel_uncore_forcewake_flush - flush the delayed release
 803  * @uncore: the intel_uncore structure
 804  * @fw_domains: forcewake domains to flush
 805  */
 806 void intel_uncore_forcewake_flush(struct intel_uncore *uncore,
 807                                   enum forcewake_domains fw_domains)
 808 {
 809         struct intel_uncore_forcewake_domain *domain;
 810         unsigned int tmp;
 811
 812         if (!uncore->fw_get_funcs)
 813                 return;
 814
 815         fw_domains &= uncore->fw_domains;
 816         for_each_fw_domain_masked(domain, fw_domains, uncore, tmp) {
 817                 WRITE_ONCE(domain->active, false);
 818                 if (hrtimer_cancel(&domain->timer))
 819                         intel_uncore_fw_release_timer(&domain->timer);
 820         }
 821 }
 822
 823 /**
 824  * intel_uncore_forcewake_put__locked - grab forcewake domain references
 825  * @uncore: the intel_uncore structure
 826  * @fw_domains: forcewake domains to get reference on
 827  *
 828  * See intel_uncore_forcewake_put(). This variant places the onus
 829  * on the caller to explicitly handle the dev_priv->uncore.lock spinlock.
 830  */
 831 void intel_uncore_forcewake_put__locked(struct intel_uncore *uncore,
 832                                         enum forcewake_domains fw_domains)
 833 {
 834         lockdep_assert_held(&uncore->lock);
 835
 836         if (!uncore->fw_get_funcs)
 837                 return;
 838
 839         __intel_uncore_forcewake_put(uncore, fw_domains, false);
 840 }
 841
 842 void assert_forcewakes_inactive(struct intel_uncore *uncore)
 843 {
 844         if (!uncore->fw_get_funcs)
 845                 return;
 846
 847         drm_WARN(&uncore->i915->drm, uncore->fw_domains_active,
 848                  "Expected all fw_domains to be inactive, but %08x are still on\n",
 849                  uncore->fw_domains_active);
 850 }
 851
 852 void assert_forcewakes_active(struct intel_uncore *uncore,
 853                               enum forcewake_domains fw_domains)
 854 {
 855         struct intel_uncore_forcewake_domain *domain;
 856         unsigned int tmp;
 857
 858         if (!IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM))
 859                 return;
 860
 861         if (!uncore->fw_get_funcs)
 862                 return;
 863
 864         spin_lock_irq(&uncore->lock);
 865
 866         assert_rpm_wakelock_held(uncore->rpm);
 867
 868         fw_domains &= uncore->fw_domains;
 869         drm_WARN(&uncore->i915->drm, fw_domains & ~uncore->fw_domains_active,
 870                  "Expected %08x fw_domains to be active, but %08x are off\n",
 871                  fw_domains, fw_domains & ~uncore->fw_domains_active);
 872
 873         /*
 874          * Check that the caller has an explicit wakeref and we don't mistake
 875          * it for the auto wakeref.
 876          */
 877         for_each_fw_domain_masked(domain, fw_domains, uncore, tmp) {
 878                 unsigned int actual = READ_ONCE(domain->wake_count);
 879                 unsigned int expect = 1;
 880
 881                 if (uncore->fw_domains_timer & domain->mask)
 882                         expect++; /* pending automatic release */
 883
 884                 if (drm_WARN(&uncore->i915->drm, actual < expect,
 885                              "Expected domain %d to be held awake by caller, count=%d\n",
 886                              domain->id, actual))
 887                         break;
 888         }
 889
 890         spin_unlock_irq(&uncore->lock);
 891 }
 892
 893 /*
 894  * We give fast paths for the really cool registers.  The second range includes
 895  * media domains (and the GSC starting from Xe_LPM+)
 896  */
 897 #define NEEDS_FORCE_WAKE(reg) ({ \
 898         u32 __reg = (reg); \
 899         __reg < 0x40000 || __reg >= 0x116000; \
 900 })
 901
 902 static int fw_range_cmp(u32 offset, const struct intel_forcewake_range *entry)
 903 {
 904         if (offset < entry->start)
 905                 return -1;
 906         else if (offset > entry->end)
 907                 return 1;
 908         else
 909                 return 0;
 910 }
 911
 912 /* Copied and "macroized" from lib/bsearch.c */
 913 #define BSEARCH(key, base, num, cmp) ({                                 \
 914         unsigned int start__ = 0, end__ = (num);                        \
 915         typeof(base) result__ = NULL;                                   \
 916         while (start__ < end__) {                                       \
 917                 unsigned int mid__ = start__ + (end__ - start__) / 2;   \
 918                 int ret__ = (cmp)((key), (base) + mid__);               \
 919                 if (ret__ < 0) {                                        \
 920                         end__ = mid__;                                  \
 921                 } else if (ret__ > 0) {                                 \
 922                         start__ = mid__ + 1;                            \
 923                 } else {                                                \
 924                         result__ = (base) + mid__;                      \
 925                         break;                                          \
 926                 }                                                       \
 927         }                                                               \
 928         result__;                                                       \
 929 })
 930
 931 static enum forcewake_domains
 932 find_fw_domain(struct intel_uncore *uncore, u32 offset)
 933 {
 934         const struct intel_forcewake_range *entry;
 935
 936         if (IS_GSI_REG(offset))
 937                 offset += uncore->gsi_offset;
 938
 939         entry = BSEARCH(offset,
 940                         uncore->fw_domains_table,
 941                         uncore->fw_domains_table_entries,
 942                         fw_range_cmp);
 943
 944         if (!entry)
 945                 return 0;
 946
 947         /*
 948          * The list of FW domains depends on the SKU in gen11+ so we
 949          * can't determine it statically. We use FORCEWAKE_ALL and
 950          * translate it here to the list of available domains.
 951          */
 952         if (entry->domains == FORCEWAKE_ALL)
 953                 return uncore->fw_domains;
 954
 955         drm_WARN(&uncore->i915->drm, entry->domains & ~uncore->fw_domains,
 956                  "Uninitialized forcewake domain(s) 0x%x accessed at 0x%x\n",
 957                  entry->domains & ~uncore->fw_domains, offset);
 958
 959         return entry->domains;
 960 }
 961
 962 /*
 963  * Shadowed register tables describe special register ranges that i915 is
 964  * allowed to write to without acquiring forcewake.  If these registers' power
 965  * wells are down, the hardware will save values written by i915 to a shadow
 966  * copy and automatically transfer them into the real register the next time
 967  * the power well is woken up.  Shadowing only applies to writes; forcewake
 968  * must still be acquired when reading from registers in these ranges.
 969  *
 970  * The documentation for shadowed registers is somewhat spotty on older
 971  * platforms.  However missing registers from these lists is non-fatal; it just
 972  * means we'll wake up the hardware for some register accesses where we didn't
 973  * really need to.
 974  *
 975  * The ranges listed in these tables must be sorted by offset.
 976  *
 977  * When adding new tables here, please also add them to
 978  * intel_shadow_table_check() in selftests/intel_uncore.c so that they will be
 979  * scanned for obvious mistakes or typos by the selftests.
 980  */
 981
 982 static const struct i915_range gen8_shadowed_regs[] = {
 983         { .start =  0x2030, .end =  0x2030 },
 984         { .start =  0xA008, .end =  0xA00C },
 985         { .start = 0x12030, .end = 0x12030 },
 986         { .start = 0x1a030, .end = 0x1a030 },
 987         { .start = 0x22030, .end = 0x22030 },
 988 };
 989
 990 static const struct i915_range gen11_shadowed_regs[] = {
 991         { .start =   0x2030, .end =   0x2030 },
 992         { .start =   0x2550, .end =   0x2550 },
 993         { .start =   0xA008, .end =   0xA00C },
 994         { .start =  0x22030, .end =  0x22030 },
 995         { .start =  0x22230, .end =  0x22230 },
 996         { .start =  0x22510, .end =  0x22550 },
 997         { .start = 0x1C0030, .end = 0x1C0030 },
 998         { .start = 0x1C0230, .end = 0x1C0230 },
 999         { .start = 0x1C0510, .end = 0x1C0550 },
1000         { .start = 0x1C4030, .end = 0x1C4030 },
1001         { .start = 0x1C4230, .end = 0x1C4230 },
1002         { .start = 0x1C4510, .end = 0x1C4550 },
1003         { .start = 0x1C8030, .end = 0x1C8030 },
1004         { .start = 0x1C8230, .end = 0x1C8230 },
1005         { .start = 0x1C8510, .end = 0x1C8550 },
1006         { .start = 0x1D0030, .end = 0x1D0030 },
1007         { .start = 0x1D0230, .end = 0x1D0230 },
1008         { .start = 0x1D0510, .end = 0x1D0550 },
1009         { .start = 0x1D4030, .end = 0x1D4030 },
1010         { .start = 0x1D4230, .end = 0x1D4230 },
1011         { .start = 0x1D4510, .end = 0x1D4550 },
1012         { .start = 0x1D8030, .end = 0x1D8030 },
1013         { .start = 0x1D8230, .end = 0x1D8230 },
1014         { .start = 0x1D8510, .end = 0x1D8550 },
1015 };
1016
1017 static const struct i915_range gen12_shadowed_regs[] = {
1018         { .start =   0x2030, .end =   0x2030 },
1019         { .start =   0x2510, .end =   0x2550 },
1020         { .start =   0xA008, .end =   0xA00C },
1021         { .start =   0xA188, .end =   0xA188 },
1022         { .start =   0xA278, .end =   0xA278 },
1023         { .start =   0xA540, .end =   0xA56C },
1024         { .start =   0xC4C8, .end =   0xC4C8 },
1025         { .start =   0xC4D4, .end =   0xC4D4 },
1026         { .start =   0xC600, .end =   0xC600 },
1027         { .start =  0x22030, .end =  0x22030 },
1028         { .start =  0x22510, .end =  0x22550 },
1029         { .start = 0x1C0030, .end = 0x1C0030 },
1030         { .start = 0x1C0510, .end = 0x1C0550 },
1031         { .start = 0x1C4030, .end = 0x1C4030 },
1032         { .start = 0x1C4510, .end = 0x1C4550 },
1033         { .start = 0x1C8030, .end = 0x1C8030 },
1034         { .start = 0x1C8510, .end = 0x1C8550 },
1035         { .start = 0x1D0030, .end = 0x1D0030 },
1036         { .start = 0x1D0510, .end = 0x1D0550 },
1037         { .start = 0x1D4030, .end = 0x1D4030 },
1038         { .start = 0x1D4510, .end = 0x1D4550 },
1039         { .start = 0x1D8030, .end = 0x1D8030 },
1040         { .start = 0x1D8510, .end = 0x1D8550 },
1041
1042         /*
1043          * The rest of these ranges are specific to Xe_HP and beyond, but
1044          * are reserved/unused ranges on earlier gen12 platforms, so they can
1045          * be safely added to the gen12 table.
1046          */
1047         { .start = 0x1E0030, .end = 0x1E0030 },
1048         { .start = 0x1E0510, .end = 0x1E0550 },
1049         { .start = 0x1E4030, .end = 0x1E4030 },
1050         { .start = 0x1E4510, .end = 0x1E4550 },
1051         { .start = 0x1E8030, .end = 0x1E8030 },
1052         { .start = 0x1E8510, .end = 0x1E8550 },
1053         { .start = 0x1F0030, .end = 0x1F0030 },
1054         { .start = 0x1F0510, .end = 0x1F0550 },
1055         { .start = 0x1F4030, .end = 0x1F4030 },
1056         { .start = 0x1F4510, .end = 0x1F4550 },
1057         { .start = 0x1F8030, .end = 0x1F8030 },
1058         { .start = 0x1F8510, .end = 0x1F8550 },
1059 };
1060
1061 static const struct i915_range dg2_shadowed_regs[] = {
1062         { .start =   0x2030, .end =   0x2030 },
1063         { .start =   0x2510, .end =   0x2550 },
1064         { .start =   0xA008, .end =   0xA00C },
1065         { .start =   0xA188, .end =   0xA188 },
1066         { .start =   0xA278, .end =   0xA278 },
1067         { .start =   0xA540, .end =   0xA56C },
1068         { .start =   0xC4C8, .end =   0xC4C8 },
1069         { .start =   0xC4E0, .end =   0xC4E0 },
1070         { .start =   0xC600, .end =   0xC600 },
1071         { .start =   0xC658, .end =   0xC658 },
1072         { .start =  0x22030, .end =  0x22030 },
1073         { .start =  0x22510, .end =  0x22550 },
1074         { .start = 0x1C0030, .end = 0x1C0030 },
1075         { .start = 0x1C0510, .end = 0x1C0550 },
1076         { .start = 0x1C4030, .end = 0x1C4030 },
1077         { .start = 0x1C4510, .end = 0x1C4550 },
1078         { .start = 0x1C8030, .end = 0x1C8030 },
1079         { .start = 0x1C8510, .end = 0x1C8550 },
1080         { .start = 0x1D0030, .end = 0x1D0030 },
1081         { .start = 0x1D0510, .end = 0x1D0550 },
1082         { .start = 0x1D4030, .end = 0x1D4030 },
1083         { .start = 0x1D4510, .end = 0x1D4550 },
1084         { .start = 0x1D8030, .end = 0x1D8030 },
1085         { .start = 0x1D8510, .end = 0x1D8550 },
1086         { .start = 0x1E0030, .end = 0x1E0030 },
1087         { .start = 0x1E0510, .end = 0x1E0550 },
1088         { .start = 0x1E4030, .end = 0x1E4030 },
1089         { .start = 0x1E4510, .end = 0x1E4550 },
1090         { .start = 0x1E8030, .end = 0x1E8030 },
1091         { .start = 0x1E8510, .end = 0x1E8550 },
1092         { .start = 0x1F0030, .end = 0x1F0030 },
1093         { .start = 0x1F0510, .end = 0x1F0550 },
1094         { .start = 0x1F4030, .end = 0x1F4030 },
1095         { .start = 0x1F4510, .end = 0x1F4550 },
1096         { .start = 0x1F8030, .end = 0x1F8030 },
1097         { .start = 0x1F8510, .end = 0x1F8550 },
1098 };
1099
1100 static const struct i915_range pvc_shadowed_regs[] = {
1101         { .start =   0x2030, .end =   0x2030 },
1102         { .start =   0x2510, .end =   0x2550 },
1103         { .start =   0xA008, .end =   0xA00C },
1104         { .start =   0xA188, .end =   0xA188 },
1105         { .start =   0xA278, .end =   0xA278 },
1106         { .start =   0xA540, .end =   0xA56C },
1107         { .start =   0xC4C8, .end =   0xC4C8 },
1108         { .start =   0xC4E0, .end =   0xC4E0 },
1109         { .start =   0xC600, .end =   0xC600 },
1110         { .start =   0xC658, .end =   0xC658 },
1111         { .start =  0x22030, .end =  0x22030 },
1112         { .start =  0x22510, .end =  0x22550 },
1113         { .start = 0x1C0030, .end = 0x1C0030 },
1114         { .start = 0x1C0510, .end = 0x1C0550 },
1115         { .start = 0x1C4030, .end = 0x1C4030 },
1116         { .start = 0x1C4510, .end = 0x1C4550 },
1117         { .start = 0x1C8030, .end = 0x1C8030 },
1118         { .start = 0x1C8510, .end = 0x1C8550 },
1119         { .start = 0x1D0030, .end = 0x1D0030 },
1120         { .start = 0x1D0510, .end = 0x1D0550 },
1121         { .start = 0x1D4030, .end = 0x1D4030 },
1122         { .start = 0x1D4510, .end = 0x1D4550 },
1123         { .start = 0x1D8030, .end = 0x1D8030 },
1124         { .start = 0x1D8510, .end = 0x1D8550 },
1125         { .start = 0x1E0030, .end = 0x1E0030 },
1126         { .start = 0x1E0510, .end = 0x1E0550 },
1127         { .start = 0x1E4030, .end = 0x1E4030 },
1128         { .start = 0x1E4510, .end = 0x1E4550 },
1129         { .start = 0x1E8030, .end = 0x1E8030 },
1130         { .start = 0x1E8510, .end = 0x1E8550 },
1131         { .start = 0x1F0030, .end = 0x1F0030 },
1132         { .start = 0x1F0510, .end = 0x1F0550 },
1133         { .start = 0x1F4030, .end = 0x1F4030 },
1134         { .start = 0x1F4510, .end = 0x1F4550 },
1135         { .start = 0x1F8030, .end = 0x1F8030 },
1136         { .start = 0x1F8510, .end = 0x1F8550 },
1137 };
1138
1139 static const struct i915_range mtl_shadowed_regs[] = {
1140         { .start =   0x2030, .end =   0x2030 },
1141         { .start =   0x2510, .end =   0x2550 },
1142         { .start =   0xA008, .end =   0xA00C },
1143         { .start =   0xA188, .end =   0xA188 },
1144         { .start =   0xA278, .end =   0xA278 },
1145         { .start =   0xA540, .end =   0xA56C },
1146         { .start =   0xC050, .end =   0xC050 },
1147         { .start =   0xC340, .end =   0xC340 },
1148         { .start =   0xC4C8, .end =   0xC4C8 },
1149         { .start =   0xC4E0, .end =   0xC4E0 },
1150         { .start =   0xC600, .end =   0xC600 },
1151         { .start =   0xC658, .end =   0xC658 },
1152         { .start =   0xCFD4, .end =   0xCFDC },
1153         { .start =  0x22030, .end =  0x22030 },
1154         { .start =  0x22510, .end =  0x22550 },
1155 };
1156
1157 static const struct i915_range xelpmp_shadowed_regs[] = {
1158         { .start = 0x1C0030, .end = 0x1C0030 },
1159         { .start = 0x1C0510, .end = 0x1C0550 },
1160         { .start = 0x1C8030, .end = 0x1C8030 },
1161         { .start = 0x1C8510, .end = 0x1C8550 },
1162         { .start = 0x1D0030, .end = 0x1D0030 },
1163         { .start = 0x1D0510, .end = 0x1D0550 },
1164         { .start = 0x38A008, .end = 0x38A00C },
1165         { .start = 0x38A188, .end = 0x38A188 },
1166         { .start = 0x38A278, .end = 0x38A278 },
1167         { .start = 0x38A540, .end = 0x38A56C },
1168         { .start = 0x38A618, .end = 0x38A618 },
1169         { .start = 0x38C050, .end = 0x38C050 },
1170         { .start = 0x38C340, .end = 0x38C340 },
1171         { .start = 0x38C4C8, .end = 0x38C4C8 },
1172         { .start = 0x38C4E0, .end = 0x38C4E4 },
1173         { .start = 0x38C600, .end = 0x38C600 },
1174         { .start = 0x38C658, .end = 0x38C658 },
1175         { .start = 0x38CFD4, .end = 0x38CFDC },
1176 };
1177
1178 static int mmio_range_cmp(u32 key, const struct i915_range *range)
1179 {
1180         if (key < range->start)
1181                 return -1;
1182         else if (key > range->end)
1183                 return 1;
1184         else
1185                 return 0;
1186 }
1187
1188 static bool is_shadowed(struct intel_uncore *uncore, u32 offset)
1189 {
1190         if (drm_WARN_ON(&uncore->i915->drm, !uncore->shadowed_reg_table))
1191                 return false;
1192
1193         if (IS_GSI_REG(offset))
1194                 offset += uncore->gsi_offset;
1195
1196         return BSEARCH(offset,
1197                        uncore->shadowed_reg_table,
1198                        uncore->shadowed_reg_table_entries,
1199                        mmio_range_cmp);
1200 }
1201
1202 static enum forcewake_domains
1203 gen6_reg_write_fw_domains(struct intel_uncore *uncore, i915_reg_t reg)
1204 {
1205         return FORCEWAKE_RENDER;
1206 }
1207
1208 #define __fwtable_reg_read_fw_domains(uncore, offset) \
1209 ({ \
1210         enum forcewake_domains __fwd = 0; \
1211         if (NEEDS_FORCE_WAKE((offset))) \
1212                 __fwd = find_fw_domain(uncore, offset); \
1213         __fwd; \
1214 })
1215
1216 #define __fwtable_reg_write_fw_domains(uncore, offset) \
1217 ({ \
1218         enum forcewake_domains __fwd = 0; \
1219         const u32 __offset = (offset); \
1220         if (NEEDS_FORCE_WAKE((__offset)) && !is_shadowed(uncore, __offset)) \
1221                 __fwd = find_fw_domain(uncore, __offset); \
1222         __fwd; \
1223 })
1224
1225 #define GEN_FW_RANGE(s, e, d) \
1226         { .start = (s), .end = (e), .domains = (d) }
1227
1228 /*
1229  * All platforms' forcewake tables below must be sorted by offset ranges.
1230  * Furthermore, new forcewake tables added should be "watertight" and have
1231  * no gaps between ranges.
1232  *
1233  * When there are multiple consecutive ranges listed in the bspec with
1234  * the same forcewake domain, it is customary to combine them into a single
1235  * row in the tables below to keep the tables small and lookups fast.
1236  * Likewise, reserved/unused ranges may be combined with the preceding and/or
1237  * following ranges since the driver will never be making MMIO accesses in
1238  * those ranges.
1239  *
1240  * For example, if the bspec were to list:
1241  *
1242  *    ...
1243  *    0x1000 - 0x1fff:  GT
1244  *    0x2000 - 0x2cff:  GT
1245  *    0x2d00 - 0x2fff:  unused/reserved
1246  *    0x3000 - 0xffff:  GT
1247  *    ...
1248  *
1249  * these could all be represented by a single line in the code:
1250  *
1251  *   GEN_FW_RANGE(0x1000, 0xffff, FORCEWAKE_GT)
1252  *
1253  * When adding new forcewake tables here, please also add them to
1254  * intel_uncore_mock_selftests in selftests/intel_uncore.c so that they will be
1255  * scanned for obvious mistakes or typos by the selftests.
1256  */
1257
1258 static const struct intel_forcewake_range __gen6_fw_ranges[] = {
1259         GEN_FW_RANGE(0x0, 0x3ffff, FORCEWAKE_RENDER),
1260 };
1261
1262 static const struct intel_forcewake_range __vlv_fw_ranges[] = {
1263         GEN_FW_RANGE(0x2000, 0x3fff, FORCEWAKE_RENDER),
1264         GEN_FW_RANGE(0x5000, 0x7fff, FORCEWAKE_RENDER),
1265         GEN_FW_RANGE(0xb000, 0x11fff, FORCEWAKE_RENDER),
1266         GEN_FW_RANGE(0x12000, 0x13fff, FORCEWAKE_MEDIA),
1267         GEN_FW_RANGE(0x22000, 0x23fff, FORCEWAKE_MEDIA),
1268         GEN_FW_RANGE(0x2e000, 0x2ffff, FORCEWAKE_RENDER),
1269         GEN_FW_RANGE(0x30000, 0x3ffff, FORCEWAKE_MEDIA),
1270 };
1271
1272 static const struct intel_forcewake_range __chv_fw_ranges[] = {
1273         GEN_FW_RANGE(0x2000, 0x3fff, FORCEWAKE_RENDER),
1274         GEN_FW_RANGE(0x4000, 0x4fff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
1275         GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER),
1276         GEN_FW_RANGE(0x8000, 0x82ff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
1277         GEN_FW_RANGE(0x8300, 0x84ff, FORCEWAKE_RENDER),
1278         GEN_FW_RANGE(0x8500, 0x85ff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
1279         GEN_FW_RANGE(0x8800, 0x88ff, FORCEWAKE_MEDIA),
1280         GEN_FW_RANGE(0x9000, 0xafff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
1281         GEN_FW_RANGE(0xb000, 0xb47f, FORCEWAKE_RENDER),
1282         GEN_FW_RANGE(0xd000, 0xd7ff, FORCEWAKE_MEDIA),
1283         GEN_FW_RANGE(0xe000, 0xe7ff, FORCEWAKE_RENDER),
1284         GEN_FW_RANGE(0xf000, 0xffff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
1285         GEN_FW_RANGE(0x12000, 0x13fff, FORCEWAKE_MEDIA),
1286         GEN_FW_RANGE(0x1a000, 0x1bfff, FORCEWAKE_MEDIA),
1287         GEN_FW_RANGE(0x1e800, 0x1e9ff, FORCEWAKE_MEDIA),
1288         GEN_FW_RANGE(0x30000, 0x37fff, FORCEWAKE_MEDIA),
1289 };
1290
1291 static const struct intel_forcewake_range __gen9_fw_ranges[] = {
1292         GEN_FW_RANGE(0x0, 0xaff, FORCEWAKE_GT),
1293         GEN_FW_RANGE(0xb00, 0x1fff, 0), /* uncore range */
1294         GEN_FW_RANGE(0x2000, 0x26ff, FORCEWAKE_RENDER),
1295         GEN_FW_RANGE(0x2700, 0x2fff, FORCEWAKE_GT),
1296         GEN_FW_RANGE(0x3000, 0x3fff, FORCEWAKE_RENDER),
1297         GEN_FW_RANGE(0x4000, 0x51ff, FORCEWAKE_GT),
1298         GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER),
1299         GEN_FW_RANGE(0x8000, 0x812f, FORCEWAKE_GT),
1300         GEN_FW_RANGE(0x8130, 0x813f, FORCEWAKE_MEDIA),
1301         GEN_FW_RANGE(0x8140, 0x815f, FORCEWAKE_RENDER),
1302         GEN_FW_RANGE(0x8160, 0x82ff, FORCEWAKE_GT),
1303         GEN_FW_RANGE(0x8300, 0x84ff, FORCEWAKE_RENDER),
1304         GEN_FW_RANGE(0x8500, 0x87ff, FORCEWAKE_GT),
1305         GEN_FW_RANGE(0x8800, 0x89ff, FORCEWAKE_MEDIA),
1306         GEN_FW_RANGE(0x8a00, 0x8bff, FORCEWAKE_GT),
1307         GEN_FW_RANGE(0x8c00, 0x8cff, FORCEWAKE_RENDER),
1308         GEN_FW_RANGE(0x8d00, 0x93ff, FORCEWAKE_GT),
1309         GEN_FW_RANGE(0x9400, 0x97ff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
1310         GEN_FW_RANGE(0x9800, 0xafff, FORCEWAKE_GT),
1311         GEN_FW_RANGE(0xb000, 0xb47f, FORCEWAKE_RENDER),
1312         GEN_FW_RANGE(0xb480, 0xcfff, FORCEWAKE_GT),
1313         GEN_FW_RANGE(0xd000, 0xd7ff, FORCEWAKE_MEDIA),
1314         GEN_FW_RANGE(0xd800, 0xdfff, FORCEWAKE_GT),
1315         GEN_FW_RANGE(0xe000, 0xe8ff, FORCEWAKE_RENDER),
1316         GEN_FW_RANGE(0xe900, 0x11fff, FORCEWAKE_GT),
1317         GEN_FW_RANGE(0x12000, 0x13fff, FORCEWAKE_MEDIA),
1318         GEN_FW_RANGE(0x14000, 0x19fff, FORCEWAKE_GT),
1319         GEN_FW_RANGE(0x1a000, 0x1e9ff, FORCEWAKE_MEDIA),
1320         GEN_FW_RANGE(0x1ea00, 0x243ff, FORCEWAKE_GT),
1321         GEN_FW_RANGE(0x24400, 0x247ff, FORCEWAKE_RENDER),
1322         GEN_FW_RANGE(0x24800, 0x2ffff, FORCEWAKE_GT),
1323         GEN_FW_RANGE(0x30000, 0x3ffff, FORCEWAKE_MEDIA),
1324 };
1325
1326 static const struct intel_forcewake_range __gen11_fw_ranges[] = {
1327         GEN_FW_RANGE(0x0, 0x1fff, 0), /* uncore range */
1328         GEN_FW_RANGE(0x2000, 0x26ff, FORCEWAKE_RENDER),
1329         GEN_FW_RANGE(0x2700, 0x2fff, FORCEWAKE_GT),
1330         GEN_FW_RANGE(0x3000, 0x3fff, FORCEWAKE_RENDER),
1331         GEN_FW_RANGE(0x4000, 0x51ff, FORCEWAKE_GT),
1332         GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER),
1333         GEN_FW_RANGE(0x8000, 0x813f, FORCEWAKE_GT),
1334         GEN_FW_RANGE(0x8140, 0x815f, FORCEWAKE_RENDER),
1335         GEN_FW_RANGE(0x8160, 0x82ff, FORCEWAKE_GT),
1336         GEN_FW_RANGE(0x8300, 0x84ff, FORCEWAKE_RENDER),
1337         GEN_FW_RANGE(0x8500, 0x87ff, FORCEWAKE_GT),
1338         GEN_FW_RANGE(0x8800, 0x8bff, 0),
1339         GEN_FW_RANGE(0x8c00, 0x8cff, FORCEWAKE_RENDER),
1340         GEN_FW_RANGE(0x8d00, 0x94cf, FORCEWAKE_GT),
1341         GEN_FW_RANGE(0x94d0, 0x955f, FORCEWAKE_RENDER),
1342         GEN_FW_RANGE(0x9560, 0x95ff, 0),
1343         GEN_FW_RANGE(0x9600, 0xafff, FORCEWAKE_GT),
1344         GEN_FW_RANGE(0xb000, 0xb47f, FORCEWAKE_RENDER),
1345         GEN_FW_RANGE(0xb480, 0xdeff, FORCEWAKE_GT),
1346         GEN_FW_RANGE(0xdf00, 0xe8ff, FORCEWAKE_RENDER),
1347         GEN_FW_RANGE(0xe900, 0x16dff, FORCEWAKE_GT),
1348         GEN_FW_RANGE(0x16e00, 0x19fff, FORCEWAKE_RENDER),
1349         GEN_FW_RANGE(0x1a000, 0x23fff, FORCEWAKE_GT),
1350         GEN_FW_RANGE(0x24000, 0x2407f, 0),
1351         GEN_FW_RANGE(0x24080, 0x2417f, FORCEWAKE_GT),
1352         GEN_FW_RANGE(0x24180, 0x242ff, FORCEWAKE_RENDER),
1353         GEN_FW_RANGE(0x24300, 0x243ff, FORCEWAKE_GT),
1354         GEN_FW_RANGE(0x24400, 0x24fff, FORCEWAKE_RENDER),
1355         GEN_FW_RANGE(0x25000, 0x3ffff, FORCEWAKE_GT),
1356         GEN_FW_RANGE(0x40000, 0x1bffff, 0),
1357         GEN_FW_RANGE(0x1c0000, 0x1c3fff, FORCEWAKE_MEDIA_VDBOX0),
1358         GEN_FW_RANGE(0x1c4000, 0x1c7fff, 0),
1359         GEN_FW_RANGE(0x1c8000, 0x1cffff, FORCEWAKE_MEDIA_VEBOX0),
1360         GEN_FW_RANGE(0x1d0000, 0x1d3fff, FORCEWAKE_MEDIA_VDBOX2),
1361         GEN_FW_RANGE(0x1d4000, 0x1dbfff, 0)
1362 };
1363
1364 static const struct intel_forcewake_range __gen12_fw_ranges[] = {
1365         GEN_FW_RANGE(0x0, 0x1fff, 0), /*
1366                 0x0   -  0xaff: reserved
1367                 0xb00 - 0x1fff: always on */
1368         GEN_FW_RANGE(0x2000, 0x26ff, FORCEWAKE_RENDER),
1369         GEN_FW_RANGE(0x2700, 0x27ff, FORCEWAKE_GT),
1370         GEN_FW_RANGE(0x2800, 0x2aff, FORCEWAKE_RENDER),
1371         GEN_FW_RANGE(0x2b00, 0x2fff, FORCEWAKE_GT),
1372         GEN_FW_RANGE(0x3000, 0x3fff, FORCEWAKE_RENDER),
1373         GEN_FW_RANGE(0x4000, 0x51ff, FORCEWAKE_GT), /*
1374                 0x4000 - 0x48ff: gt
1375                 0x4900 - 0x51ff: reserved */
1376         GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER), /*
1377                 0x5200 - 0x53ff: render
1378                 0x5400 - 0x54ff: reserved
1379                 0x5500 - 0x7fff: render */
1380         GEN_FW_RANGE(0x8000, 0x813f, FORCEWAKE_GT),
1381         GEN_FW_RANGE(0x8140, 0x815f, FORCEWAKE_RENDER),
1382         GEN_FW_RANGE(0x8160, 0x81ff, 0), /*
1383                 0x8160 - 0x817f: reserved
1384                 0x8180 - 0x81ff: always on */
1385         GEN_FW_RANGE(0x8200, 0x82ff, FORCEWAKE_GT),
1386         GEN_FW_RANGE(0x8300, 0x84ff, FORCEWAKE_RENDER),
1387         GEN_FW_RANGE(0x8500, 0x94cf, FORCEWAKE_GT), /*
1388                 0x8500 - 0x87ff: gt
1389                 0x8800 - 0x8fff: reserved
1390                 0x9000 - 0x947f: gt
1391                 0x9480 - 0x94cf: reserved */
1392         GEN_FW_RANGE(0x94d0, 0x955f, FORCEWAKE_RENDER),
1393         GEN_FW_RANGE(0x9560, 0x97ff, 0), /*
1394                 0x9560 - 0x95ff: always on
1395                 0x9600 - 0x97ff: reserved */
1396         GEN_FW_RANGE(0x9800, 0xafff, FORCEWAKE_GT),
1397         GEN_FW_RANGE(0xb000, 0xb3ff, FORCEWAKE_RENDER),
1398         GEN_FW_RANGE(0xb400, 0xcfff, FORCEWAKE_GT), /*
1399                 0xb400 - 0xbf7f: gt
1400                 0xb480 - 0xbfff: reserved
1401                 0xc000 - 0xcfff: gt */
1402         GEN_FW_RANGE(0xd000, 0xd7ff, 0),
1403         GEN_FW_RANGE(0xd800, 0xd8ff, FORCEWAKE_RENDER),
1404         GEN_FW_RANGE(0xd900, 0xdbff, FORCEWAKE_GT),
1405         GEN_FW_RANGE(0xdc00, 0xefff, FORCEWAKE_RENDER), /*
1406                 0xdc00 - 0xddff: render
1407                 0xde00 - 0xde7f: reserved
1408                 0xde80 - 0xe8ff: render
1409                 0xe900 - 0xefff: reserved */
1410         GEN_FW_RANGE(0xf000, 0x147ff, FORCEWAKE_GT), /*
1411                  0xf000 - 0xffff: gt
1412                 0x10000 - 0x147ff: reserved */
1413         GEN_FW_RANGE(0x14800, 0x1ffff, FORCEWAKE_RENDER), /*
1414                 0x14800 - 0x14fff: render
1415                 0x15000 - 0x16dff: reserved
1416                 0x16e00 - 0x1bfff: render
1417                 0x1c000 - 0x1ffff: reserved */
1418         GEN_FW_RANGE(0x20000, 0x20fff, FORCEWAKE_MEDIA_VDBOX0),
1419         GEN_FW_RANGE(0x21000, 0x21fff, FORCEWAKE_MEDIA_VDBOX2),
1420         GEN_FW_RANGE(0x22000, 0x23fff, FORCEWAKE_GT),
1421         GEN_FW_RANGE(0x24000, 0x2417f, 0), /*
1422                 0x24000 - 0x2407f: always on
1423                 0x24080 - 0x2417f: reserved */
1424         GEN_FW_RANGE(0x24180, 0x249ff, FORCEWAKE_GT), /*
1425                 0x24180 - 0x241ff: gt
1426                 0x24200 - 0x249ff: reserved */
1427         GEN_FW_RANGE(0x24a00, 0x251ff, FORCEWAKE_RENDER), /*
1428                 0x24a00 - 0x24a7f: render
1429                 0x24a80 - 0x251ff: reserved */
1430         GEN_FW_RANGE(0x25200, 0x255ff, FORCEWAKE_GT), /*
1431                 0x25200 - 0x252ff: gt
1432                 0x25300 - 0x255ff: reserved */
1433         GEN_FW_RANGE(0x25600, 0x2567f, FORCEWAKE_MEDIA_VDBOX0),
1434         GEN_FW_RANGE(0x25680, 0x259ff, FORCEWAKE_MEDIA_VDBOX2), /*
1435                 0x25680 - 0x256ff: VD2
1436                 0x25700 - 0x259ff: reserved */
1437         GEN_FW_RANGE(0x25a00, 0x25a7f, FORCEWAKE_MEDIA_VDBOX0),
1438         GEN_FW_RANGE(0x25a80, 0x2ffff, FORCEWAKE_MEDIA_VDBOX2), /*
1439                 0x25a80 - 0x25aff: VD2
1440                 0x25b00 - 0x2ffff: reserved */
1441         GEN_FW_RANGE(0x30000, 0x3ffff, FORCEWAKE_GT),
1442         GEN_FW_RANGE(0x40000, 0x1bffff, 0),
1443         GEN_FW_RANGE(0x1c0000, 0x1c3fff, FORCEWAKE_MEDIA_VDBOX0), /*
1444                 0x1c0000 - 0x1c2bff: VD0
1445                 0x1c2c00 - 0x1c2cff: reserved
1446                 0x1c2d00 - 0x1c2dff: VD0
1447                 0x1c2e00 - 0x1c3eff: reserved
1448                 0x1c3f00 - 0x1c3fff: VD0 */
1449         GEN_FW_RANGE(0x1c4000, 0x1c7fff, 0),
1450         GEN_FW_RANGE(0x1c8000, 0x1cbfff, FORCEWAKE_MEDIA_VEBOX0), /*
1451                 0x1c8000 - 0x1ca0ff: VE0
1452                 0x1ca100 - 0x1cbeff: reserved
1453                 0x1cbf00 - 0x1cbfff: VE0 */
1454         GEN_FW_RANGE(0x1cc000, 0x1cffff, FORCEWAKE_MEDIA_VDBOX0), /*
1455                 0x1cc000 - 0x1ccfff: VD0
1456                 0x1cd000 - 0x1cffff: reserved */
1457         GEN_FW_RANGE(0x1d0000, 0x1d3fff, FORCEWAKE_MEDIA_VDBOX2), /*
1458                 0x1d0000 - 0x1d2bff: VD2
1459                 0x1d2c00 - 0x1d2cff: reserved
1460                 0x1d2d00 - 0x1d2dff: VD2
1461                 0x1d2e00 - 0x1d3eff: reserved
1462                 0x1d3f00 - 0x1d3fff: VD2 */
1463 };
1464
1465 /*
1466  * Graphics IP version 12.55 brings a slight change to the 0xd800 range,
1467  * switching it from the GT domain to the render domain.
1468  */
1469 #define XEHP_FWRANGES(FW_RANGE_D800)                                    \
1470         GEN_FW_RANGE(0x0, 0x1fff, 0), /*                                        \
1471                   0x0 -  0xaff: reserved                                        \
1472                 0xb00 - 0x1fff: always on */                                    \
1473         GEN_FW_RANGE(0x2000, 0x26ff, FORCEWAKE_RENDER),                         \
1474         GEN_FW_RANGE(0x2700, 0x4aff, FORCEWAKE_GT),                             \
1475         GEN_FW_RANGE(0x4b00, 0x51ff, 0), /*                                     \
1476                 0x4b00 - 0x4fff: reserved                                       \
1477                 0x5000 - 0x51ff: always on */                                   \
1478         GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER),                         \
1479         GEN_FW_RANGE(0x8000, 0x813f, FORCEWAKE_GT),                             \
1480         GEN_FW_RANGE(0x8140, 0x815f, FORCEWAKE_RENDER),                         \
1481         GEN_FW_RANGE(0x8160, 0x81ff, 0), /*                                     \
1482                 0x8160 - 0x817f: reserved                                       \
1483                 0x8180 - 0x81ff: always on */                                   \
1484         GEN_FW_RANGE(0x8200, 0x82ff, FORCEWAKE_GT),                             \
1485         GEN_FW_RANGE(0x8300, 0x84ff, FORCEWAKE_RENDER),                         \
1486         GEN_FW_RANGE(0x8500, 0x8cff, FORCEWAKE_GT), /*                          \
1487                 0x8500 - 0x87ff: gt                                             \
1488                 0x8800 - 0x8c7f: reserved                                       \
1489                 0x8c80 - 0x8cff: gt (DG2 only) */                               \
1490         GEN_FW_RANGE(0x8d00, 0x8fff, FORCEWAKE_RENDER), /*                      \
1491                 0x8d00 - 0x8dff: render (DG2 only)                              \
1492                 0x8e00 - 0x8fff: reserved */                                    \
1493         GEN_FW_RANGE(0x9000, 0x94cf, FORCEWAKE_GT), /*                          \
1494                 0x9000 - 0x947f: gt                                             \
1495                 0x9480 - 0x94cf: reserved */                                    \
1496         GEN_FW_RANGE(0x94d0, 0x955f, FORCEWAKE_RENDER),                         \
1497         GEN_FW_RANGE(0x9560, 0x967f, 0), /*                                     \
1498                 0x9560 - 0x95ff: always on                                      \
1499                 0x9600 - 0x967f: reserved */                                    \
1500         GEN_FW_RANGE(0x9680, 0x97ff, FORCEWAKE_RENDER), /*                      \
1501                 0x9680 - 0x96ff: render (DG2 only)                              \
1502                 0x9700 - 0x97ff: reserved */                                    \
1503         GEN_FW_RANGE(0x9800, 0xcfff, FORCEWAKE_GT), /*                          \
1504                 0x9800 - 0xb4ff: gt                                             \
1505                 0xb500 - 0xbfff: reserved                                       \
1506                 0xc000 - 0xcfff: gt */                                          \
1507         GEN_FW_RANGE(0xd000, 0xd7ff, 0),                                        \
1508         GEN_FW_RANGE(0xd800, 0xd87f, FW_RANGE_D800),                    \
1509         GEN_FW_RANGE(0xd880, 0xdbff, FORCEWAKE_GT),                             \
1510         GEN_FW_RANGE(0xdc00, 0xdcff, FORCEWAKE_RENDER),                         \
1511         GEN_FW_RANGE(0xdd00, 0xde7f, FORCEWAKE_GT), /*                          \
1512                 0xdd00 - 0xddff: gt                                             \
1513                 0xde00 - 0xde7f: reserved */                                    \
1514         GEN_FW_RANGE(0xde80, 0xe8ff, FORCEWAKE_RENDER), /*                      \
1515                 0xde80 - 0xdfff: render                                         \
1516                 0xe000 - 0xe0ff: reserved                                       \
1517                 0xe100 - 0xe8ff: render */                                      \
1518         GEN_FW_RANGE(0xe900, 0xffff, FORCEWAKE_GT), /*                          \
1519                 0xe900 - 0xe9ff: gt                                             \
1520                 0xea00 - 0xefff: reserved                                       \
1521                 0xf000 - 0xffff: gt */                                          \
1522         GEN_FW_RANGE(0x10000, 0x12fff, 0), /*                                   \
1523                 0x10000 - 0x11fff: reserved                                     \
1524                 0x12000 - 0x127ff: always on                                    \
1525                 0x12800 - 0x12fff: reserved */                                  \
1526         GEN_FW_RANGE(0x13000, 0x131ff, FORCEWAKE_MEDIA_VDBOX0), /* DG2 only */  \
1527         GEN_FW_RANGE(0x13200, 0x13fff, FORCEWAKE_MEDIA_VDBOX2), /*              \
1528                 0x13200 - 0x133ff: VD2 (DG2 only)                               \
1529                 0x13400 - 0x13fff: reserved */                                  \
1530         GEN_FW_RANGE(0x14000, 0x141ff, FORCEWAKE_MEDIA_VDBOX0), /* XEHPSDV only */      \
1531         GEN_FW_RANGE(0x14200, 0x143ff, FORCEWAKE_MEDIA_VDBOX2), /* XEHPSDV only */      \
1532         GEN_FW_RANGE(0x14400, 0x145ff, FORCEWAKE_MEDIA_VDBOX4), /* XEHPSDV only */      \
1533         GEN_FW_RANGE(0x14600, 0x147ff, FORCEWAKE_MEDIA_VDBOX6), /* XEHPSDV only */      \
1534         GEN_FW_RANGE(0x14800, 0x14fff, FORCEWAKE_RENDER),                       \
1535         GEN_FW_RANGE(0x15000, 0x16dff, FORCEWAKE_GT), /*                        \
1536                 0x15000 - 0x15fff: gt (DG2 only)                                \
1537                 0x16000 - 0x16dff: reserved */                                  \
1538         GEN_FW_RANGE(0x16e00, 0x1ffff, FORCEWAKE_RENDER),                       \
1539         GEN_FW_RANGE(0x20000, 0x21fff, FORCEWAKE_MEDIA_VDBOX0), /*              \
1540                 0x20000 - 0x20fff: VD0 (XEHPSDV only)                           \
1541                 0x21000 - 0x21fff: reserved */                                  \
1542         GEN_FW_RANGE(0x22000, 0x23fff, FORCEWAKE_GT),                           \
1543         GEN_FW_RANGE(0x24000, 0x2417f, 0), /*                                   \
1544                 0x24000 - 0x2407f: always on                                    \
1545                 0x24080 - 0x2417f: reserved */                                  \
1546         GEN_FW_RANGE(0x24180, 0x249ff, FORCEWAKE_GT), /*                        \
1547                 0x24180 - 0x241ff: gt                                           \
1548                 0x24200 - 0x249ff: reserved */                                  \
1549         GEN_FW_RANGE(0x24a00, 0x251ff, FORCEWAKE_RENDER), /*                    \
1550                 0x24a00 - 0x24a7f: render                                       \
1551                 0x24a80 - 0x251ff: reserved */                                  \
1552         GEN_FW_RANGE(0x25200, 0x25fff, FORCEWAKE_GT), /*                        \
1553                 0x25200 - 0x252ff: gt                                           \
1554                 0x25300 - 0x25fff: reserved */                                  \
1555         GEN_FW_RANGE(0x26000, 0x2ffff, FORCEWAKE_RENDER), /*                    \
1556                 0x26000 - 0x27fff: render                                       \
1557                 0x28000 - 0x29fff: reserved                                     \
1558                 0x2a000 - 0x2ffff: undocumented */                              \
1559         GEN_FW_RANGE(0x30000, 0x3ffff, FORCEWAKE_GT),                           \
1560         GEN_FW_RANGE(0x40000, 0x1bffff, 0),                                     \
1561         GEN_FW_RANGE(0x1c0000, 0x1c3fff, FORCEWAKE_MEDIA_VDBOX0), /*            \
1562                 0x1c0000 - 0x1c2bff: VD0                                        \
1563                 0x1c2c00 - 0x1c2cff: reserved                                   \
1564                 0x1c2d00 - 0x1c2dff: VD0                                        \
1565                 0x1c2e00 - 0x1c3eff: VD0 (DG2 only)                             \
1566                 0x1c3f00 - 0x1c3fff: VD0 */                                     \
1567         GEN_FW_RANGE(0x1c4000, 0x1c7fff, FORCEWAKE_MEDIA_VDBOX1), /*            \
1568                 0x1c4000 - 0x1c6bff: VD1                                        \
1569                 0x1c6c00 - 0x1c6cff: reserved                                   \
1570                 0x1c6d00 - 0x1c6dff: VD1                                        \
1571                 0x1c6e00 - 0x1c7fff: reserved */                                \
1572         GEN_FW_RANGE(0x1c8000, 0x1cbfff, FORCEWAKE_MEDIA_VEBOX0), /*            \
1573                 0x1c8000 - 0x1ca0ff: VE0                                        \
1574                 0x1ca100 - 0x1cbfff: reserved */                                \
1575         GEN_FW_RANGE(0x1cc000, 0x1ccfff, FORCEWAKE_MEDIA_VDBOX0),               \
1576         GEN_FW_RANGE(0x1cd000, 0x1cdfff, FORCEWAKE_MEDIA_VDBOX2),               \
1577         GEN_FW_RANGE(0x1ce000, 0x1cefff, FORCEWAKE_MEDIA_VDBOX4),               \
1578         GEN_FW_RANGE(0x1cf000, 0x1cffff, FORCEWAKE_MEDIA_VDBOX6),               \
1579         GEN_FW_RANGE(0x1d0000, 0x1d3fff, FORCEWAKE_MEDIA_VDBOX2), /*            \
1580                 0x1d0000 - 0x1d2bff: VD2                                        \
1581                 0x1d2c00 - 0x1d2cff: reserved                                   \
1582                 0x1d2d00 - 0x1d2dff: VD2                                        \
1583                 0x1d2e00 - 0x1d3dff: VD2 (DG2 only)                             \
1584                 0x1d3e00 - 0x1d3eff: reserved                                   \
1585                 0x1d3f00 - 0x1d3fff: VD2 */                                     \
1586         GEN_FW_RANGE(0x1d4000, 0x1d7fff, FORCEWAKE_MEDIA_VDBOX3), /*            \
1587                 0x1d4000 - 0x1d6bff: VD3                                        \
1588                 0x1d6c00 - 0x1d6cff: reserved                                   \
1589                 0x1d6d00 - 0x1d6dff: VD3                                        \
1590                 0x1d6e00 - 0x1d7fff: reserved */                                \
1591         GEN_FW_RANGE(0x1d8000, 0x1dffff, FORCEWAKE_MEDIA_VEBOX1), /*            \
1592                 0x1d8000 - 0x1da0ff: VE1                                        \
1593                 0x1da100 - 0x1dffff: reserved */                                \
1594         GEN_FW_RANGE(0x1e0000, 0x1e3fff, FORCEWAKE_MEDIA_VDBOX4), /*            \
1595                 0x1e0000 - 0x1e2bff: VD4                                        \
1596                 0x1e2c00 - 0x1e2cff: reserved                                   \
1597                 0x1e2d00 - 0x1e2dff: VD4                                        \
1598                 0x1e2e00 - 0x1e3eff: reserved                                   \
1599                 0x1e3f00 - 0x1e3fff: VD4 */                                     \
1600         GEN_FW_RANGE(0x1e4000, 0x1e7fff, FORCEWAKE_MEDIA_VDBOX5), /*            \
1601                 0x1e4000 - 0x1e6bff: VD5                                        \
1602                 0x1e6c00 - 0x1e6cff: reserved                                   \
1603                 0x1e6d00 - 0x1e6dff: VD5                                        \
1604                 0x1e6e00 - 0x1e7fff: reserved */                                \
1605         GEN_FW_RANGE(0x1e8000, 0x1effff, FORCEWAKE_MEDIA_VEBOX2), /*            \
1606                 0x1e8000 - 0x1ea0ff: VE2                                        \
1607                 0x1ea100 - 0x1effff: reserved */                                \
1608         GEN_FW_RANGE(0x1f0000, 0x1f3fff, FORCEWAKE_MEDIA_VDBOX6), /*            \
1609                 0x1f0000 - 0x1f2bff: VD6                                        \
1610                 0x1f2c00 - 0x1f2cff: reserved                                   \
1611                 0x1f2d00 - 0x1f2dff: VD6                                        \
1612                 0x1f2e00 - 0x1f3eff: reserved                                   \
1613                 0x1f3f00 - 0x1f3fff: VD6 */                                     \
1614         GEN_FW_RANGE(0x1f4000, 0x1f7fff, FORCEWAKE_MEDIA_VDBOX7), /*            \
1615                 0x1f4000 - 0x1f6bff: VD7                                        \
1616                 0x1f6c00 - 0x1f6cff: reserved                                   \
1617                 0x1f6d00 - 0x1f6dff: VD7                                        \
1618                 0x1f6e00 - 0x1f7fff: reserved */                                \
1619         GEN_FW_RANGE(0x1f8000, 0x1fa0ff, FORCEWAKE_MEDIA_VEBOX3),
1620
1621 static const struct intel_forcewake_range __xehp_fw_ranges[] = {
1622         XEHP_FWRANGES(FORCEWAKE_GT)
1623 };
1624
1625 static const struct intel_forcewake_range __dg2_fw_ranges[] = {
1626         XEHP_FWRANGES(FORCEWAKE_RENDER)
1627 };
1628
1629 static const struct intel_forcewake_range __pvc_fw_ranges[] = {
1630         GEN_FW_RANGE(0x0, 0xaff, 0),
1631         GEN_FW_RANGE(0xb00, 0xbff, FORCEWAKE_GT),
1632         GEN_FW_RANGE(0xc00, 0xfff, 0),
1633         GEN_FW_RANGE(0x1000, 0x1fff, FORCEWAKE_GT),
1634         GEN_FW_RANGE(0x2000, 0x26ff, FORCEWAKE_RENDER),
1635         GEN_FW_RANGE(0x2700, 0x2fff, FORCEWAKE_GT),
1636         GEN_FW_RANGE(0x3000, 0x3fff, FORCEWAKE_RENDER),
1637         GEN_FW_RANGE(0x4000, 0x813f, FORCEWAKE_GT), /*
1638                 0x4000 - 0x4aff: gt
1639                 0x4b00 - 0x4fff: reserved
1640                 0x5000 - 0x51ff: gt
1641                 0x5200 - 0x52ff: reserved
1642                 0x5300 - 0x53ff: gt
1643                 0x5400 - 0x7fff: reserved
1644                 0x8000 - 0x813f: gt */
1645         GEN_FW_RANGE(0x8140, 0x817f, FORCEWAKE_RENDER),
1646         GEN_FW_RANGE(0x8180, 0x81ff, 0),
1647         GEN_FW_RANGE(0x8200, 0x94cf, FORCEWAKE_GT), /*
1648                 0x8200 - 0x82ff: gt
1649                 0x8300 - 0x84ff: reserved
1650                 0x8500 - 0x887f: gt
1651                 0x8880 - 0x8a7f: reserved
1652                 0x8a80 - 0x8aff: gt
1653                 0x8b00 - 0x8fff: reserved
1654                 0x9000 - 0x947f: gt
1655                 0x9480 - 0x94cf: reserved */
1656         GEN_FW_RANGE(0x94d0, 0x955f, FORCEWAKE_RENDER),
1657         GEN_FW_RANGE(0x9560, 0x967f, 0), /*
1658                 0x9560 - 0x95ff: always on
1659                 0x9600 - 0x967f: reserved */
1660         GEN_FW_RANGE(0x9680, 0x97ff, FORCEWAKE_RENDER), /*
1661                 0x9680 - 0x96ff: render
1662                 0x9700 - 0x97ff: reserved */
1663         GEN_FW_RANGE(0x9800, 0xcfff, FORCEWAKE_GT), /*
1664                 0x9800 - 0xb4ff: gt
1665                 0xb500 - 0xbfff: reserved
1666                 0xc000 - 0xcfff: gt */
1667         GEN_FW_RANGE(0xd000, 0xd3ff, 0),
1668         GEN_FW_RANGE(0xd400, 0xdbff, FORCEWAKE_GT),
1669         GEN_FW_RANGE(0xdc00, 0xdcff, FORCEWAKE_RENDER),
1670         GEN_FW_RANGE(0xdd00, 0xde7f, FORCEWAKE_GT), /*
1671                 0xdd00 - 0xddff: gt
1672                 0xde00 - 0xde7f: reserved */
1673         GEN_FW_RANGE(0xde80, 0xe8ff, FORCEWAKE_RENDER), /*
1674                 0xde80 - 0xdeff: render
1675                 0xdf00 - 0xe1ff: reserved
1676                 0xe200 - 0xe7ff: render
1677                 0xe800 - 0xe8ff: reserved */
1678         GEN_FW_RANGE(0xe900, 0x11fff, FORCEWAKE_GT), /*
1679                  0xe900 -  0xe9ff: gt
1680                  0xea00 -  0xebff: reserved
1681                  0xec00 -  0xffff: gt
1682                 0x10000 - 0x11fff: reserved */
1683         GEN_FW_RANGE(0x12000, 0x12fff, 0), /*
1684                 0x12000 - 0x127ff: always on
1685                 0x12800 - 0x12fff: reserved */
1686         GEN_FW_RANGE(0x13000, 0x19fff, FORCEWAKE_GT), /*
1687                 0x13000 - 0x135ff: gt
1688                 0x13600 - 0x147ff: reserved
1689                 0x14800 - 0x153ff: gt
1690                 0x15400 - 0x19fff: reserved */
1691         GEN_FW_RANGE(0x1a000, 0x21fff, FORCEWAKE_RENDER), /*
1692                 0x1a000 - 0x1ffff: render
1693                 0x20000 - 0x21fff: reserved */
1694         GEN_FW_RANGE(0x22000, 0x23fff, FORCEWAKE_GT),
1695         GEN_FW_RANGE(0x24000, 0x2417f, 0), /*
1696                 24000 - 0x2407f: always on
1697                 24080 - 0x2417f: reserved */
1698         GEN_FW_RANGE(0x24180, 0x25fff, FORCEWAKE_GT), /*
1699                 0x24180 - 0x241ff: gt
1700                 0x24200 - 0x251ff: reserved
1701                 0x25200 - 0x252ff: gt
1702                 0x25300 - 0x25fff: reserved */
1703         GEN_FW_RANGE(0x26000, 0x2ffff, FORCEWAKE_RENDER), /*
1704                 0x26000 - 0x27fff: render
1705                 0x28000 - 0x2ffff: reserved */
1706         GEN_FW_RANGE(0x30000, 0x3ffff, FORCEWAKE_GT),
1707         GEN_FW_RANGE(0x40000, 0x1bffff, 0),
1708         GEN_FW_RANGE(0x1c0000, 0x1c3fff, FORCEWAKE_MEDIA_VDBOX0), /*
1709                 0x1c0000 - 0x1c2bff: VD0
1710                 0x1c2c00 - 0x1c2cff: reserved
1711                 0x1c2d00 - 0x1c2dff: VD0
1712                 0x1c2e00 - 0x1c3eff: reserved
1713                 0x1c3f00 - 0x1c3fff: VD0 */
1714         GEN_FW_RANGE(0x1c4000, 0x1cffff, FORCEWAKE_MEDIA_VDBOX1), /*
1715                 0x1c4000 - 0x1c6aff: VD1
1716                 0x1c6b00 - 0x1c7eff: reserved
1717                 0x1c7f00 - 0x1c7fff: VD1
1718                 0x1c8000 - 0x1cffff: reserved */
1719         GEN_FW_RANGE(0x1d0000, 0x23ffff, FORCEWAKE_MEDIA_VDBOX2), /*
1720                 0x1d0000 - 0x1d2aff: VD2
1721                 0x1d2b00 - 0x1d3eff: reserved
1722                 0x1d3f00 - 0x1d3fff: VD2
1723                 0x1d4000 - 0x23ffff: reserved */
1724         GEN_FW_RANGE(0x240000, 0x3dffff, 0),
1725         GEN_FW_RANGE(0x3e0000, 0x3effff, FORCEWAKE_GT),
1726 };
1727
1728 static const struct intel_forcewake_range __mtl_fw_ranges[] = {
1729         GEN_FW_RANGE(0x0, 0xaff, 0),
1730         GEN_FW_RANGE(0xb00, 0xbff, FORCEWAKE_GT),
1731         GEN_FW_RANGE(0xc00, 0xfff, 0),
1732         GEN_FW_RANGE(0x1000, 0x1fff, FORCEWAKE_GT),
1733         GEN_FW_RANGE(0x2000, 0x26ff, FORCEWAKE_RENDER),
1734         GEN_FW_RANGE(0x2700, 0x2fff, FORCEWAKE_GT),
1735         GEN_FW_RANGE(0x3000, 0x3fff, FORCEWAKE_RENDER),
1736         GEN_FW_RANGE(0x4000, 0x51ff, FORCEWAKE_GT), /*
1737                 0x4000 - 0x48ff: render
1738                 0x4900 - 0x51ff: reserved */
1739         GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER), /*
1740                 0x5200 - 0x53ff: render
1741                 0x5400 - 0x54ff: reserved
1742                 0x5500 - 0x7fff: render */
1743         GEN_FW_RANGE(0x8000, 0x813f, FORCEWAKE_GT),
1744         GEN_FW_RANGE(0x8140, 0x817f, FORCEWAKE_RENDER), /*
1745                 0x8140 - 0x815f: render
1746                 0x8160 - 0x817f: reserved */
1747         GEN_FW_RANGE(0x8180, 0x81ff, 0),
1748         GEN_FW_RANGE(0x8200, 0x94cf, FORCEWAKE_GT), /*
1749                 0x8200 - 0x87ff: gt
1750                 0x8800 - 0x8dff: reserved
1751                 0x8e00 - 0x8f7f: gt
1752                 0x8f80 - 0x8fff: reserved
1753                 0x9000 - 0x947f: gt
1754                 0x9480 - 0x94cf: reserved */
1755         GEN_FW_RANGE(0x94d0, 0x955f, FORCEWAKE_RENDER),
1756         GEN_FW_RANGE(0x9560, 0x967f, 0), /*
1757                 0x9560 - 0x95ff: always on
1758                 0x9600 - 0x967f: reserved */
1759         GEN_FW_RANGE(0x9680, 0x97ff, FORCEWAKE_RENDER), /*
1760                 0x9680 - 0x96ff: render
1761                 0x9700 - 0x97ff: reserved */
1762         GEN_FW_RANGE(0x9800, 0xcfff, FORCEWAKE_GT), /*
1763                 0x9800 - 0xb4ff: gt
1764                 0xb500 - 0xbfff: reserved
1765                 0xc000 - 0xcfff: gt */
1766         GEN_FW_RANGE(0xd000, 0xd7ff, 0), /*
1767                 0xd000 - 0xd3ff: always on
1768                 0xd400 - 0xd7ff: reserved */
1769         GEN_FW_RANGE(0xd800, 0xd87f, FORCEWAKE_RENDER),
1770         GEN_FW_RANGE(0xd880, 0xdbff, FORCEWAKE_GT),
1771         GEN_FW_RANGE(0xdc00, 0xdcff, FORCEWAKE_RENDER),
1772         GEN_FW_RANGE(0xdd00, 0xde7f, FORCEWAKE_GT), /*
1773                 0xdd00 - 0xddff: gt
1774                 0xde00 - 0xde7f: reserved */
1775         GEN_FW_RANGE(0xde80, 0xe8ff, FORCEWAKE_RENDER), /*
1776                 0xde80 - 0xdfff: render
1777                 0xe000 - 0xe0ff: reserved
1778                 0xe100 - 0xe8ff: render */
1779         GEN_FW_RANGE(0xe900, 0xe9ff, FORCEWAKE_GT),
1780         GEN_FW_RANGE(0xea00, 0x147ff, 0), /*
1781                  0xea00 - 0x11fff: reserved
1782                 0x12000 - 0x127ff: always on
1783                 0x12800 - 0x147ff: reserved */
1784         GEN_FW_RANGE(0x14800, 0x19fff, FORCEWAKE_GT), /*
1785                 0x14800 - 0x153ff: gt
1786                 0x15400 - 0x19fff: reserved */
1787         GEN_FW_RANGE(0x1a000, 0x21fff, FORCEWAKE_RENDER), /*
1788                 0x1a000 - 0x1bfff: render
1789                 0x1c000 - 0x21fff: reserved */
1790         GEN_FW_RANGE(0x22000, 0x23fff, FORCEWAKE_GT),
1791         GEN_FW_RANGE(0x24000, 0x2ffff, 0), /*
1792                 0x24000 - 0x2407f: always on
1793                 0x24080 - 0x2ffff: reserved */
1794         GEN_FW_RANGE(0x30000, 0x3ffff, FORCEWAKE_GT)
1795 };
1796
1797 /*
1798  * Note that the register ranges here are the final offsets after
1799  * translation of the GSI block to the 0x380000 offset.
1800  *
1801  * NOTE:  There are a couple MCR ranges near the bottom of this table
1802  * that need to power up either VD0 or VD2 depending on which replicated
1803  * instance of the register we're trying to access.  Our forcewake logic
1804  * at the moment doesn't have a good way to take steering into consideration,
1805  * and the driver doesn't even access any registers in those ranges today,
1806  * so for now we just mark those ranges as FORCEWAKE_ALL.  That will ensure
1807  * proper operation if we do start using the ranges in the future, and we
1808  * can determine at that time whether it's worth adding extra complexity to
1809  * the forcewake handling to take steering into consideration.
1810  */
1811 static const struct intel_forcewake_range __xelpmp_fw_ranges[] = {
1812         GEN_FW_RANGE(0x0, 0x115fff, 0), /* render GT range */
1813         GEN_FW_RANGE(0x116000, 0x11ffff, FORCEWAKE_GSC), /*
1814                 0x116000 - 0x117fff: gsc
1815                 0x118000 - 0x119fff: reserved
1816                 0x11a000 - 0x11efff: gsc
1817                 0x11f000 - 0x11ffff: reserved */
1818         GEN_FW_RANGE(0x120000, 0x1bffff, 0), /* non-GT range */
1819         GEN_FW_RANGE(0x1c0000, 0x1c7fff, FORCEWAKE_MEDIA_VDBOX0), /*
1820                 0x1c0000 - 0x1c3dff: VD0
1821                 0x1c3e00 - 0x1c3eff: reserved
1822                 0x1c3f00 - 0x1c3fff: VD0
1823                 0x1c4000 - 0x1c7fff: reserved */
1824         GEN_FW_RANGE(0x1c8000, 0x1cbfff, FORCEWAKE_MEDIA_VEBOX0), /*
1825                 0x1c8000 - 0x1ca0ff: VE0
1826                 0x1ca100 - 0x1cbfff: reserved */
1827         GEN_FW_RANGE(0x1cc000, 0x1cffff, FORCEWAKE_MEDIA_VDBOX0), /*
1828                 0x1cc000 - 0x1cdfff: VD0
1829                 0x1ce000 - 0x1cffff: reserved */
1830         GEN_FW_RANGE(0x1d0000, 0x1d7fff, FORCEWAKE_MEDIA_VDBOX2), /*
1831                 0x1d0000 - 0x1d3dff: VD2
1832                 0x1d3e00 - 0x1d3eff: reserved
1833                 0x1d4000 - 0x1d7fff: VD2 */
1834         GEN_FW_RANGE(0x1d8000, 0x1da0ff, FORCEWAKE_MEDIA_VEBOX1),
1835         GEN_FW_RANGE(0x1da100, 0x380aff, 0), /*
1836                 0x1da100 - 0x23ffff: reserved
1837                 0x240000 - 0x37ffff: non-GT range
1838                 0x380000 - 0x380aff: reserved */
1839         GEN_FW_RANGE(0x380b00, 0x380bff, FORCEWAKE_GT),
1840         GEN_FW_RANGE(0x380c00, 0x380fff, 0),
1841         GEN_FW_RANGE(0x381000, 0x38817f, FORCEWAKE_GT), /*
1842                 0x381000 - 0x381fff: gt
1843                 0x382000 - 0x383fff: reserved
1844                 0x384000 - 0x384aff: gt
1845                 0x384b00 - 0x3851ff: reserved
1846                 0x385200 - 0x3871ff: gt
1847                 0x387200 - 0x387fff: reserved
1848                 0x388000 - 0x38813f: gt
1849                 0x388140 - 0x38817f: reserved */
1850         GEN_FW_RANGE(0x388180, 0x3882ff, 0), /*
1851                 0x388180 - 0x3881ff: always on
1852                 0x388200 - 0x3882ff: reserved */
1853         GEN_FW_RANGE(0x388300, 0x38955f, FORCEWAKE_GT), /*
1854                 0x388300 - 0x38887f: gt
1855                 0x388880 - 0x388fff: reserved
1856                 0x389000 - 0x38947f: gt
1857                 0x389480 - 0x38955f: reserved */
1858         GEN_FW_RANGE(0x389560, 0x389fff, 0), /*
1859                 0x389560 - 0x3895ff: always on
1860                 0x389600 - 0x389fff: reserved */
1861         GEN_FW_RANGE(0x38a000, 0x38cfff, FORCEWAKE_GT), /*
1862                 0x38a000 - 0x38afff: gt
1863                 0x38b000 - 0x38bfff: reserved
1864                 0x38c000 - 0x38cfff: gt */
1865         GEN_FW_RANGE(0x38d000, 0x38d11f, 0),
1866         GEN_FW_RANGE(0x38d120, 0x391fff, FORCEWAKE_GT), /*
1867                 0x38d120 - 0x38dfff: gt
1868                 0x38e000 - 0x38efff: reserved
1869                 0x38f000 - 0x38ffff: gt
1870                 0x389000 - 0x391fff: reserved */
1871         GEN_FW_RANGE(0x392000, 0x392fff, 0), /*
1872                 0x392000 - 0x3927ff: always on
1873                 0x392800 - 0x292fff: reserved */
1874         GEN_FW_RANGE(0x393000, 0x3931ff, FORCEWAKE_GT),
1875         GEN_FW_RANGE(0x393200, 0x39323f, FORCEWAKE_ALL), /* instance-based, see note above */
1876         GEN_FW_RANGE(0x393240, 0x3933ff, FORCEWAKE_GT),
1877         GEN_FW_RANGE(0x393400, 0x3934ff, FORCEWAKE_ALL), /* instance-based, see note above */
1878         GEN_FW_RANGE(0x393500, 0x393c7f, 0), /*
1879                 0x393500 - 0x393bff: reserved
1880                 0x393c00 - 0x393c7f: always on */
1881         GEN_FW_RANGE(0x393c80, 0x393dff, FORCEWAKE_GT),
1882 };
1883
1884 static void
1885 ilk_dummy_write(struct intel_uncore *uncore)
1886 {
1887         /* WaIssueDummyWriteToWakeupFromRC6:ilk Issue a dummy write to wake up
1888          * the chip from rc6 before touching it for real. MI_MODE is masked,
1889          * hence harmless to write 0 into. */
1890         __raw_uncore_write32(uncore, RING_MI_MODE(RENDER_RING_BASE), 0);
1891 }
1892
1893 static void
1894 __unclaimed_reg_debug(struct intel_uncore *uncore,
1895                       const i915_reg_t reg,
1896                       const bool read)
1897 {
1898         if (drm_WARN(&uncore->i915->drm,
1899                      check_for_unclaimed_mmio(uncore),
1900                      "Unclaimed %s register 0x%x\n",
1901                      read ? "read from" : "write to",
1902                      i915_mmio_reg_offset(reg)))
1903                 /* Only report the first N failures */
1904                 uncore->i915->params.mmio_debug--;
1905 }
1906
1907 static void
1908 __unclaimed_previous_reg_debug(struct intel_uncore *uncore,
1909                                const i915_reg_t reg,
1910                                const bool read)
1911 {
1912         if (check_for_unclaimed_mmio(uncore))
1913                 drm_dbg(&uncore->i915->drm,
1914                         "Unclaimed access detected before %s register 0x%x\n",
1915                         read ? "read from" : "write to",
1916                         i915_mmio_reg_offset(reg));
1917 }
1918
1919 static inline void
1920 unclaimed_reg_debug(struct intel_uncore *uncore,
1921                     const i915_reg_t reg,
1922                     const bool read,
1923                     const bool before)
1924 {
1925         if (likely(!uncore->i915->params.mmio_debug) || !uncore->debug)
1926                 return;
1927
1928         /* interrupts are disabled and re-enabled around uncore->lock usage */
1929         lockdep_assert_held(&uncore->lock);
1930
1931         if (before) {
1932                 spin_lock(&uncore->debug->lock);
1933                 __unclaimed_previous_reg_debug(uncore, reg, read);
1934         } else {
1935                 __unclaimed_reg_debug(uncore, reg, read);
1936                 spin_unlock(&uncore->debug->lock);
1937         }
1938 }
1939
1940 #define __vgpu_read(x) \
1941 static u##x \
1942 vgpu_read##x(struct intel_uncore *uncore, i915_reg_t reg, bool trace) { \
1943         u##x val = __raw_uncore_read##x(uncore, reg); \
1944         trace_i915_reg_rw(false, reg, val, sizeof(val), trace); \
1945         return val; \
1946 }
1947 __vgpu_read(8)
1948 __vgpu_read(16)
1949 __vgpu_read(32)
1950 __vgpu_read(64)
1951
1952 #define GEN2_READ_HEADER(x) \
1953         u##x val = 0; \
1954         assert_rpm_wakelock_held(uncore->rpm);
1955
1956 #define GEN2_READ_FOOTER \
1957         trace_i915_reg_rw(false, reg, val, sizeof(val), trace); \
1958         return val
1959
1960 #define __gen2_read(x) \
1961 static u##x \
1962 gen2_read##x(struct intel_uncore *uncore, i915_reg_t reg, bool trace) { \
1963         GEN2_READ_HEADER(x); \
1964         val = __raw_uncore_read##x(uncore, reg); \
1965         GEN2_READ_FOOTER; \
1966 }
1967
1968 #define __gen5_read(x) \
1969 static u##x \
1970 gen5_read##x(struct intel_uncore *uncore, i915_reg_t reg, bool trace) { \
1971         GEN2_READ_HEADER(x); \
1972         ilk_dummy_write(uncore); \
1973         val = __raw_uncore_read##x(uncore, reg); \
1974         GEN2_READ_FOOTER; \
1975 }
1976
1977 __gen5_read(8)
1978 __gen5_read(16)
1979 __gen5_read(32)
1980 __gen5_read(64)
1981 __gen2_read(8)
1982 __gen2_read(16)
1983 __gen2_read(32)
1984 __gen2_read(64)
1985
1986 #undef __gen5_read
1987 #undef __gen2_read
1988
1989 #undef GEN2_READ_FOOTER
1990 #undef GEN2_READ_HEADER
1991
1992 #define GEN6_READ_HEADER(x) \
1993         u32 offset = i915_mmio_reg_offset(reg); \
1994         unsigned long irqflags; \
1995         u##x val = 0; \
1996         assert_rpm_wakelock_held(uncore->rpm); \
1997         spin_lock_irqsave(&uncore->lock, irqflags); \
1998         unclaimed_reg_debug(uncore, reg, true, true)
1999
2000 #define GEN6_READ_FOOTER \
2001         unclaimed_reg_debug(uncore, reg, true, false); \
2002         spin_unlock_irqrestore(&uncore->lock, irqflags); \
2003         trace_i915_reg_rw(false, reg, val, sizeof(val), trace); \
2004         return val
2005
2006 static noinline void ___force_wake_auto(struct intel_uncore *uncore,
2007                                         enum forcewake_domains fw_domains)
2008 {
2009         struct intel_uncore_forcewake_domain *domain;
2010         unsigned int tmp;
2011
2012         GEM_BUG_ON(fw_domains & ~uncore->fw_domains);
2013
2014         for_each_fw_domain_masked(domain, fw_domains, uncore, tmp)
2015                 fw_domain_arm_timer(domain);
2016
2017         fw_domains_get(uncore, fw_domains);
2018 }
2019
2020 static inline void __force_wake_auto(struct intel_uncore *uncore,
2021                                      enum forcewake_domains fw_domains)
2022 {
2023         GEM_BUG_ON(!fw_domains);
2024
2025         /* Turn on all requested but inactive supported forcewake domains. */
2026         fw_domains &= uncore->fw_domains;
2027         fw_domains &= ~uncore->fw_domains_active;
2028
2029         if (fw_domains)
2030                 ___force_wake_auto(uncore, fw_domains);
2031 }
2032
2033 #define __gen_fwtable_read(x) \
2034 static u##x \
2035 fwtable_read##x(struct intel_uncore *uncore, i915_reg_t reg, bool trace) \
2036 { \
2037         enum forcewake_domains fw_engine; \
2038         GEN6_READ_HEADER(x); \
2039         fw_engine = __fwtable_reg_read_fw_domains(uncore, offset); \
2040         if (fw_engine) \
2041                 __force_wake_auto(uncore, fw_engine); \
2042         val = __raw_uncore_read##x(uncore, reg); \
2043         GEN6_READ_FOOTER; \
2044 }
2045
2046 static enum forcewake_domains
2047 fwtable_reg_read_fw_domains(struct intel_uncore *uncore, i915_reg_t reg) {
2048         return __fwtable_reg_read_fw_domains(uncore, i915_mmio_reg_offset(reg));
2049 }
2050
2051 __gen_fwtable_read(8)
2052 __gen_fwtable_read(16)
2053 __gen_fwtable_read(32)
2054 __gen_fwtable_read(64)
2055
2056 #undef __gen_fwtable_read
2057 #undef GEN6_READ_FOOTER
2058 #undef GEN6_READ_HEADER
2059
2060 #define GEN2_WRITE_HEADER \
2061         trace_i915_reg_rw(true, reg, val, sizeof(val), trace); \
2062         assert_rpm_wakelock_held(uncore->rpm); \
2063
2064 #define GEN2_WRITE_FOOTER
2065
2066 #define __gen2_write(x) \
2067 static void \
2068 gen2_write##x(struct intel_uncore *uncore, i915_reg_t reg, u##x val, bool trace) { \
2069         GEN2_WRITE_HEADER; \
2070         __raw_uncore_write##x(uncore, reg, val); \
2071         GEN2_WRITE_FOOTER; \
2072 }
2073
2074 #define __gen5_write(x) \
2075 static void \
2076 gen5_write##x(struct intel_uncore *uncore, i915_reg_t reg, u##x val, bool trace) { \
2077         GEN2_WRITE_HEADER; \
2078         ilk_dummy_write(uncore); \
2079         __raw_uncore_write##x(uncore, reg, val); \
2080         GEN2_WRITE_FOOTER; \
2081 }
2082
2083 __gen5_write(8)
2084 __gen5_write(16)
2085 __gen5_write(32)
2086 __gen2_write(8)
2087 __gen2_write(16)
2088 __gen2_write(32)
2089
2090 #undef __gen5_write
2091 #undef __gen2_write
2092
2093 #undef GEN2_WRITE_FOOTER
2094 #undef GEN2_WRITE_HEADER
2095
2096 #define GEN6_WRITE_HEADER \
2097         u32 offset = i915_mmio_reg_offset(reg); \
2098         unsigned long irqflags; \
2099         trace_i915_reg_rw(true, reg, val, sizeof(val), trace); \
2100         assert_rpm_wakelock_held(uncore->rpm); \
2101         spin_lock_irqsave(&uncore->lock, irqflags); \
2102         unclaimed_reg_debug(uncore, reg, false, true)
2103
2104 #define GEN6_WRITE_FOOTER \
2105         unclaimed_reg_debug(uncore, reg, false, false); \
2106         spin_unlock_irqrestore(&uncore->lock, irqflags)
2107
2108 #define __gen6_write(x) \
2109 static void \
2110 gen6_write##x(struct intel_uncore *uncore, i915_reg_t reg, u##x val, bool trace) { \
2111         GEN6_WRITE_HEADER; \
2112         if (NEEDS_FORCE_WAKE(offset)) \
2113                 __gen6_gt_wait_for_fifo(uncore); \
2114         __raw_uncore_write##x(uncore, reg, val); \
2115         GEN6_WRITE_FOOTER; \
2116 }
2117 __gen6_write(8)
2118 __gen6_write(16)
2119 __gen6_write(32)
2120
2121 #define __gen_fwtable_write(x) \
2122 static void \
2123 fwtable_write##x(struct intel_uncore *uncore, i915_reg_t reg, u##x val, bool trace) { \
2124         enum forcewake_domains fw_engine; \
2125         GEN6_WRITE_HEADER; \
2126         fw_engine = __fwtable_reg_write_fw_domains(uncore, offset); \
2127         if (fw_engine) \
2128                 __force_wake_auto(uncore, fw_engine); \
2129         __raw_uncore_write##x(uncore, reg, val); \
2130         GEN6_WRITE_FOOTER; \
2131 }
2132
2133 static enum forcewake_domains
2134 fwtable_reg_write_fw_domains(struct intel_uncore *uncore, i915_reg_t reg)
2135 {
2136         return __fwtable_reg_write_fw_domains(uncore, i915_mmio_reg_offset(reg));
2137 }
2138
2139 __gen_fwtable_write(8)
2140 __gen_fwtable_write(16)
2141 __gen_fwtable_write(32)
2142
2143 #undef __gen_fwtable_write
2144 #undef GEN6_WRITE_FOOTER
2145 #undef GEN6_WRITE_HEADER
2146
2147 #define __vgpu_write(x) \
2148 static void \
2149 vgpu_write##x(struct intel_uncore *uncore, i915_reg_t reg, u##x val, bool trace) { \
2150         trace_i915_reg_rw(true, reg, val, sizeof(val), trace); \
2151         __raw_uncore_write##x(uncore, reg, val); \
2152 }
2153 __vgpu_write(8)
2154 __vgpu_write(16)
2155 __vgpu_write(32)
2156
2157 #define ASSIGN_RAW_WRITE_MMIO_VFUNCS(uncore, x) \
2158 do { \
2159         (uncore)->funcs.mmio_writeb = x##_write8; \
2160         (uncore)->funcs.mmio_writew = x##_write16; \
2161         (uncore)->funcs.mmio_writel = x##_write32; \
2162 } while (0)
2163
2164 #define ASSIGN_RAW_READ_MMIO_VFUNCS(uncore, x) \
2165 do { \
2166         (uncore)->funcs.mmio_readb = x##_read8; \
2167         (uncore)->funcs.mmio_readw = x##_read16; \
2168         (uncore)->funcs.mmio_readl = x##_read32; \
2169         (uncore)->funcs.mmio_readq = x##_read64; \
2170 } while (0)
2171
2172 #define ASSIGN_WRITE_MMIO_VFUNCS(uncore, x) \
2173 do { \
2174         ASSIGN_RAW_WRITE_MMIO_VFUNCS((uncore), x); \
2175         (uncore)->funcs.write_fw_domains = x##_reg_write_fw_domains; \
2176 } while (0)
2177
2178 #define ASSIGN_READ_MMIO_VFUNCS(uncore, x) \
2179 do { \
2180         ASSIGN_RAW_READ_MMIO_VFUNCS(uncore, x); \
2181         (uncore)->funcs.read_fw_domains = x##_reg_read_fw_domains; \
2182 } while (0)
2183
2184 static int __fw_domain_init(struct intel_uncore *uncore,
2185                             enum forcewake_domain_id domain_id,
2186                             i915_reg_t reg_set,
2187                             i915_reg_t reg_ack)
2188 {
2189         struct intel_uncore_forcewake_domain *d;
2190
2191         GEM_BUG_ON(domain_id >= FW_DOMAIN_ID_COUNT);
2192         GEM_BUG_ON(uncore->fw_domain[domain_id]);
2193
2194         if (i915_inject_probe_failure(uncore->i915))
2195                 return -ENOMEM;
2196
2197         d = kzalloc(sizeof(*d), GFP_KERNEL);
2198         if (!d)
2199                 return -ENOMEM;
2200
2201         drm_WARN_ON(&uncore->i915->drm, !i915_mmio_reg_valid(reg_set));
2202         drm_WARN_ON(&uncore->i915->drm, !i915_mmio_reg_valid(reg_ack));
2203
2204         d->uncore = uncore;
2205         d->wake_count = 0;
2206         d->reg_set = uncore->regs + i915_mmio_reg_offset(reg_set) + uncore->gsi_offset;
2207         d->reg_ack = uncore->regs + i915_mmio_reg_offset(reg_ack) + uncore->gsi_offset;
2208
2209         d->id = domain_id;
2210
2211         BUILD_BUG_ON(FORCEWAKE_RENDER != (1 << FW_DOMAIN_ID_RENDER));
2212         BUILD_BUG_ON(FORCEWAKE_GT != (1 << FW_DOMAIN_ID_GT));
2213         BUILD_BUG_ON(FORCEWAKE_MEDIA != (1 << FW_DOMAIN_ID_MEDIA));
2214         BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX0 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX0));
2215         BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX1 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX1));
2216         BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX2 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX2));
2217         BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX3 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX3));
2218         BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX4 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX4));
2219         BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX5 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX5));
2220         BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX6 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX6));
2221         BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX7 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX7));
2222         BUILD_BUG_ON(FORCEWAKE_MEDIA_VEBOX0 != (1 << FW_DOMAIN_ID_MEDIA_VEBOX0));
2223         BUILD_BUG_ON(FORCEWAKE_MEDIA_VEBOX1 != (1 << FW_DOMAIN_ID_MEDIA_VEBOX1));
2224         BUILD_BUG_ON(FORCEWAKE_MEDIA_VEBOX2 != (1 << FW_DOMAIN_ID_MEDIA_VEBOX2));
2225         BUILD_BUG_ON(FORCEWAKE_MEDIA_VEBOX3 != (1 << FW_DOMAIN_ID_MEDIA_VEBOX3));
2226         BUILD_BUG_ON(FORCEWAKE_GSC != (1 << FW_DOMAIN_ID_GSC));
2227
2228         d->mask = BIT(domain_id);
2229
2230         hrtimer_init(&d->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
2231         d->timer.function = intel_uncore_fw_release_timer;
2232
2233         uncore->fw_domains |= BIT(domain_id);
2234
2235         fw_domain_reset(d);
2236
2237         uncore->fw_domain[domain_id] = d;
2238
2239         return 0;
2240 }
2241
2242 static void fw_domain_fini(struct intel_uncore *uncore,
2243                            enum forcewake_domain_id domain_id)
2244 {
2245         struct intel_uncore_forcewake_domain *d;
2246
2247         GEM_BUG_ON(domain_id >= FW_DOMAIN_ID_COUNT);
2248
2249         d = fetch_and_zero(&uncore->fw_domain[domain_id]);
2250         if (!d)
2251                 return;
2252
2253         uncore->fw_domains &= ~BIT(domain_id);
2254         drm_WARN_ON(&uncore->i915->drm, d->wake_count);
2255         drm_WARN_ON(&uncore->i915->drm, hrtimer_cancel(&d->timer));
2256         kfree(d);
2257 }
2258
2259 static void intel_uncore_fw_domains_fini(struct intel_uncore *uncore)
2260 {
2261         struct intel_uncore_forcewake_domain *d;
2262         int tmp;
2263
2264         for_each_fw_domain(d, uncore, tmp)
2265                 fw_domain_fini(uncore, d->id);
2266 }
2267
2268 static const struct intel_uncore_fw_get uncore_get_fallback = {
2269         .force_wake_get = fw_domains_get_with_fallback
2270 };
2271
2272 static const struct intel_uncore_fw_get uncore_get_normal = {
2273         .force_wake_get = fw_domains_get_normal,
2274 };
2275
2276 static const struct intel_uncore_fw_get uncore_get_thread_status = {
2277         .force_wake_get = fw_domains_get_with_thread_status
2278 };
2279
2280 static int intel_uncore_fw_domains_init(struct intel_uncore *uncore)
2281 {
2282         struct drm_i915_private *i915 = uncore->i915;
2283         int ret = 0;
2284
2285         GEM_BUG_ON(!intel_uncore_has_forcewake(uncore));
2286
2287 #define fw_domain_init(uncore__, id__, set__, ack__) \
2288         (ret ?: (ret = __fw_domain_init((uncore__), (id__), (set__), (ack__))))
2289
2290         if (GRAPHICS_VER(i915) >= 11) {
2291                 intel_engine_mask_t emask;
2292                 int i;
2293
2294                 /* we'll prune the domains of missing engines later */
2295                 emask = uncore->gt->info.engine_mask;
2296
2297                 uncore->fw_get_funcs = &uncore_get_fallback;
2298                 if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70))
2299                         fw_domain_init(uncore, FW_DOMAIN_ID_GT,
2300                                        FORCEWAKE_GT_GEN9,
2301                                        FORCEWAKE_ACK_GT_MTL);
2302                 else
2303                         fw_domain_init(uncore, FW_DOMAIN_ID_GT,
2304                                        FORCEWAKE_GT_GEN9,
2305                                        FORCEWAKE_ACK_GT_GEN9);
2306
2307                 if (RCS_MASK(uncore->gt) || CCS_MASK(uncore->gt))
2308                         fw_domain_init(uncore, FW_DOMAIN_ID_RENDER,
2309                                        FORCEWAKE_RENDER_GEN9,
2310                                        FORCEWAKE_ACK_RENDER_GEN9);
2311
2312                 for (i = 0; i < I915_MAX_VCS; i++) {
2313                         if (!__HAS_ENGINE(emask, _VCS(i)))
2314                                 continue;
2315
2316                         fw_domain_init(uncore, FW_DOMAIN_ID_MEDIA_VDBOX0 + i,
2317                                        FORCEWAKE_MEDIA_VDBOX_GEN11(i),
2318                                        FORCEWAKE_ACK_MEDIA_VDBOX_GEN11(i));
2319                 }
2320                 for (i = 0; i < I915_MAX_VECS; i++) {
2321                         if (!__HAS_ENGINE(emask, _VECS(i)))
2322                                 continue;
2323
2324                         fw_domain_init(uncore, FW_DOMAIN_ID_MEDIA_VEBOX0 + i,
2325                                        FORCEWAKE_MEDIA_VEBOX_GEN11(i),
2326                                        FORCEWAKE_ACK_MEDIA_VEBOX_GEN11(i));
2327                 }
2328
2329                 if (uncore->gt->type == GT_MEDIA)
2330                         fw_domain_init(uncore, FW_DOMAIN_ID_GSC,
2331                                        FORCEWAKE_REQ_GSC, FORCEWAKE_ACK_GSC);
2332         } else if (IS_GRAPHICS_VER(i915, 9, 10)) {
2333                 uncore->fw_get_funcs = &uncore_get_fallback;
2334                 fw_domain_init(uncore, FW_DOMAIN_ID_RENDER,
2335                                FORCEWAKE_RENDER_GEN9,
2336                                FORCEWAKE_ACK_RENDER_GEN9);
2337                 fw_domain_init(uncore, FW_DOMAIN_ID_GT,
2338                                FORCEWAKE_GT_GEN9,
2339                                FORCEWAKE_ACK_GT_GEN9);
2340                 fw_domain_init(uncore, FW_DOMAIN_ID_MEDIA,
2341                                FORCEWAKE_MEDIA_GEN9, FORCEWAKE_ACK_MEDIA_GEN9);
2342         } else if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) {
2343                 uncore->fw_get_funcs = &uncore_get_normal;
2344                 fw_domain_init(uncore, FW_DOMAIN_ID_RENDER,
2345                                FORCEWAKE_VLV, FORCEWAKE_ACK_VLV);
2346                 fw_domain_init(uncore, FW_DOMAIN_ID_MEDIA,
2347                                FORCEWAKE_MEDIA_VLV, FORCEWAKE_ACK_MEDIA_VLV);
2348         } else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) {
2349                 uncore->fw_get_funcs = &uncore_get_thread_status;
2350                 fw_domain_init(uncore, FW_DOMAIN_ID_RENDER,
2351                                FORCEWAKE_MT, FORCEWAKE_ACK_HSW);
2352         } else if (IS_IVYBRIDGE(i915)) {
2353                 u32 ecobus;
2354
2355                 /* IVB configs may use multi-threaded forcewake */
2356
2357                 /* A small trick here - if the bios hasn't configured
2358                  * MT forcewake, and if the device is in RC6, then
2359                  * force_wake_mt_get will not wake the device and the
2360                  * ECOBUS read will return zero. Which will be
2361                  * (correctly) interpreted by the test below as MT
2362                  * forcewake being disabled.
2363                  */
2364                 uncore->fw_get_funcs = &uncore_get_thread_status;
2365
2366                 /* We need to init first for ECOBUS access and then
2367                  * determine later if we want to reinit, in case of MT access is
2368                  * not working. In this stage we don't know which flavour this
2369                  * ivb is, so it is better to reset also the gen6 fw registers
2370                  * before the ecobus check.
2371                  */
2372
2373                 __raw_uncore_write32(uncore, FORCEWAKE, 0);
2374                 __raw_posting_read(uncore, ECOBUS);
2375
2376                 ret = __fw_domain_init(uncore, FW_DOMAIN_ID_RENDER,
2377                                        FORCEWAKE_MT, FORCEWAKE_MT_ACK);
2378                 if (ret)
2379                         goto out;
2380
2381                 spin_lock_irq(&uncore->lock);
2382                 fw_domains_get_with_thread_status(uncore, FORCEWAKE_RENDER);
2383                 ecobus = __raw_uncore_read32(uncore, ECOBUS);
2384                 fw_domains_put(uncore, FORCEWAKE_RENDER);
2385                 spin_unlock_irq(&uncore->lock);
2386
2387                 if (!(ecobus & FORCEWAKE_MT_ENABLE)) {
2388                         drm_info(&i915->drm, "No MT forcewake available on Ivybridge, this can result in issues\n");
2389                         drm_info(&i915->drm, "when using vblank-synced partial screen updates.\n");
2390                         fw_domain_fini(uncore, FW_DOMAIN_ID_RENDER);
2391                         fw_domain_init(uncore, FW_DOMAIN_ID_RENDER,
2392                                        FORCEWAKE, FORCEWAKE_ACK);
2393                 }
2394         } else if (GRAPHICS_VER(i915) == 6) {
2395                 uncore->fw_get_funcs = &uncore_get_thread_status;
2396                 fw_domain_init(uncore, FW_DOMAIN_ID_RENDER,
2397                                FORCEWAKE, FORCEWAKE_ACK);
2398         }
2399
2400 #undef fw_domain_init
2401
2402         /* All future platforms are expected to require complex power gating */
2403         drm_WARN_ON(&i915->drm, !ret && uncore->fw_domains == 0);
2404
2405 out:
2406         if (ret)
2407                 intel_uncore_fw_domains_fini(uncore);
2408
2409         return ret;
2410 }
2411
2412 #define ASSIGN_FW_DOMAINS_TABLE(uncore, d) \
2413 { \
2414         (uncore)->fw_domains_table = \
2415                         (struct intel_forcewake_range *)(d); \
2416         (uncore)->fw_domains_table_entries = ARRAY_SIZE((d)); \
2417 }
2418
2419 #define ASSIGN_SHADOW_TABLE(uncore, d) \
2420 { \
2421         (uncore)->shadowed_reg_table = d; \
2422         (uncore)->shadowed_reg_table_entries = ARRAY_SIZE((d)); \
2423 }
2424
2425 static int i915_pmic_bus_access_notifier(struct notifier_block *nb,
2426                                          unsigned long action, void *data)
2427 {
2428         struct intel_uncore *uncore = container_of(nb,
2429                         struct intel_uncore, pmic_bus_access_nb);
2430
2431         switch (action) {
2432         case MBI_PMIC_BUS_ACCESS_BEGIN:
2433                 /*
2434                  * forcewake all now to make sure that we don't need to do a
2435                  * forcewake later which on systems where this notifier gets
2436                  * called requires the punit to access to the shared pmic i2c
2437                  * bus, which will be busy after this notification, leading to:
2438                  * "render: timed out waiting for forcewake ack request."
2439                  * errors.
2440                  *
2441                  * The notifier is unregistered during intel_runtime_suspend(),
2442                  * so it's ok to access the HW here without holding a RPM
2443                  * wake reference -> disable wakeref asserts for the time of
2444                  * the access.
2445                  */
2446                 disable_rpm_wakeref_asserts(uncore->rpm);
2447                 intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL);
2448                 enable_rpm_wakeref_asserts(uncore->rpm);
2449                 break;
2450         case MBI_PMIC_BUS_ACCESS_END:
2451                 intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL);
2452                 break;
2453         }
2454
2455         return NOTIFY_OK;
2456 }
2457
2458 static void uncore_unmap_mmio(struct drm_device *drm, void *regs)
2459 {
2460         iounmap(regs);
2461 }
2462
2463 int intel_uncore_setup_mmio(struct intel_uncore *uncore, phys_addr_t phys_addr)
2464 {
2465         struct drm_i915_private *i915 = uncore->i915;
2466         int mmio_size;
2467
2468         /*
2469          * Before gen4, the registers and the GTT are behind different BARs.
2470          * However, from gen4 onwards, the registers and the GTT are shared
2471          * in the same BAR, so we want to restrict this ioremap from
2472          * clobbering the GTT which we want ioremap_wc instead. Fortunately,
2473          * the register BAR remains the same size for all the earlier
2474          * generations up to Ironlake.
2475          * For dgfx chips register range is expanded to 4MB, and this larger
2476          * range is also used for integrated gpus beginning with Meteor Lake.
2477          */
2478         if (IS_DGFX(i915) || GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70))
2479                 mmio_size = 4 * 1024 * 1024;
2480         else if (GRAPHICS_VER(i915) >= 5)
2481                 mmio_size = 2 * 1024 * 1024;
2482         else
2483                 mmio_size = 512 * 1024;
2484
2485         uncore->regs = ioremap(phys_addr, mmio_size);
2486         if (uncore->regs == NULL) {
2487                 drm_err(&i915->drm, "failed to map registers\n");
2488                 return -EIO;
2489         }
2490
2491         return drmm_add_action_or_reset(&i915->drm, uncore_unmap_mmio, uncore->regs);
2492 }
2493
2494 void intel_uncore_init_early(struct intel_uncore *uncore,
2495                              struct intel_gt *gt)
2496 {
2497         spin_lock_init(&uncore->lock);
2498         uncore->i915 = gt->i915;
2499         uncore->gt = gt;
2500         uncore->rpm = &gt->i915->runtime_pm;
2501 }
2502
2503 static void uncore_raw_init(struct intel_uncore *uncore)
2504 {
2505         GEM_BUG_ON(intel_uncore_has_forcewake(uncore));
2506
2507         if (intel_vgpu_active(uncore->i915)) {
2508                 ASSIGN_RAW_WRITE_MMIO_VFUNCS(uncore, vgpu);
2509                 ASSIGN_RAW_READ_MMIO_VFUNCS(uncore, vgpu);
2510         } else if (GRAPHICS_VER(uncore->i915) == 5) {
2511                 ASSIGN_RAW_WRITE_MMIO_VFUNCS(uncore, gen5);
2512                 ASSIGN_RAW_READ_MMIO_VFUNCS(uncore, gen5);
2513         } else {
2514                 ASSIGN_RAW_WRITE_MMIO_VFUNCS(uncore, gen2);
2515                 ASSIGN_RAW_READ_MMIO_VFUNCS(uncore, gen2);
2516         }
2517 }
2518
2519 static int uncore_media_forcewake_init(struct intel_uncore *uncore)
2520 {
2521         struct drm_i915_private *i915 = uncore->i915;
2522
2523         if (MEDIA_VER(i915) >= 13) {
2524                 ASSIGN_FW_DOMAINS_TABLE(uncore, __xelpmp_fw_ranges);
2525                 ASSIGN_SHADOW_TABLE(uncore, xelpmp_shadowed_regs);
2526                 ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable);
2527         } else {
2528                 MISSING_CASE(MEDIA_VER(i915));
2529                 return -ENODEV;
2530         }
2531
2532         return 0;
2533 }
2534
2535 static int uncore_forcewake_init(struct intel_uncore *uncore)
2536 {
2537         struct drm_i915_private *i915 = uncore->i915;
2538         int ret;
2539
2540         GEM_BUG_ON(!intel_uncore_has_forcewake(uncore));
2541
2542         ret = intel_uncore_fw_domains_init(uncore);
2543         if (ret)
2544                 return ret;
2545         forcewake_early_sanitize(uncore, 0);
2546
2547         ASSIGN_READ_MMIO_VFUNCS(uncore, fwtable);
2548
2549         if (uncore->gt->type == GT_MEDIA)
2550                 return uncore_media_forcewake_init(uncore);
2551
2552         if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70)) {
2553                 ASSIGN_FW_DOMAINS_TABLE(uncore, __mtl_fw_ranges);
2554                 ASSIGN_SHADOW_TABLE(uncore, mtl_shadowed_regs);
2555                 ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable);
2556         } else if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 60)) {
2557                 ASSIGN_FW_DOMAINS_TABLE(uncore, __pvc_fw_ranges);
2558                 ASSIGN_SHADOW_TABLE(uncore, pvc_shadowed_regs);
2559                 ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable);
2560         } else if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 55)) {
2561                 ASSIGN_FW_DOMAINS_TABLE(uncore, __dg2_fw_ranges);
2562                 ASSIGN_SHADOW_TABLE(uncore, dg2_shadowed_regs);
2563                 ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable);
2564         } else if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 50)) {
2565                 ASSIGN_FW_DOMAINS_TABLE(uncore, __xehp_fw_ranges);
2566                 ASSIGN_SHADOW_TABLE(uncore, gen12_shadowed_regs);
2567                 ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable);
2568         } else if (GRAPHICS_VER(i915) >= 12) {
2569                 ASSIGN_FW_DOMAINS_TABLE(uncore, __gen12_fw_ranges);
2570                 ASSIGN_SHADOW_TABLE(uncore, gen12_shadowed_regs);
2571                 ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable);
2572         } else if (GRAPHICS_VER(i915) == 11) {
2573                 ASSIGN_FW_DOMAINS_TABLE(uncore, __gen11_fw_ranges);
2574                 ASSIGN_SHADOW_TABLE(uncore, gen11_shadowed_regs);
2575                 ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable);
2576         } else if (IS_GRAPHICS_VER(i915, 9, 10)) {
2577                 ASSIGN_FW_DOMAINS_TABLE(uncore, __gen9_fw_ranges);
2578                 ASSIGN_SHADOW_TABLE(uncore, gen8_shadowed_regs);
2579                 ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable);
2580         } else if (IS_CHERRYVIEW(i915)) {
2581                 ASSIGN_FW_DOMAINS_TABLE(uncore, __chv_fw_ranges);
2582                 ASSIGN_SHADOW_TABLE(uncore, gen8_shadowed_regs);
2583                 ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable);
2584         } else if (GRAPHICS_VER(i915) == 8) {
2585                 ASSIGN_FW_DOMAINS_TABLE(uncore, __gen6_fw_ranges);
2586                 ASSIGN_SHADOW_TABLE(uncore, gen8_shadowed_regs);
2587                 ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable);
2588         } else if (IS_VALLEYVIEW(i915)) {
2589                 ASSIGN_FW_DOMAINS_TABLE(uncore, __vlv_fw_ranges);
2590                 ASSIGN_WRITE_MMIO_VFUNCS(uncore, gen6);
2591         } else if (IS_GRAPHICS_VER(i915, 6, 7)) {
2592                 ASSIGN_FW_DOMAINS_TABLE(uncore, __gen6_fw_ranges);
2593                 ASSIGN_WRITE_MMIO_VFUNCS(uncore, gen6);
2594         }
2595
2596         uncore->pmic_bus_access_nb.notifier_call = i915_pmic_bus_access_notifier;
2597         iosf_mbi_register_pmic_bus_access_notifier(&uncore->pmic_bus_access_nb);
2598
2599         return 0;
2600 }
2601
2602 int intel_uncore_init_mmio(struct intel_uncore *uncore)
2603 {
2604         struct drm_i915_private *i915 = uncore->i915;
2605         int ret;
2606
2607         /*
2608          * The boot firmware initializes local memory and assesses its health.
2609          * If memory training fails, the punit will have been instructed to
2610          * keep the GT powered down; we won't be able to communicate with it
2611          * and we should not continue with driver initialization.
2612          */
2613         if (IS_DGFX(i915) &&
2614             !(__raw_uncore_read32(uncore, GU_CNTL) & LMEM_INIT)) {
2615                 drm_err(&i915->drm, "LMEM not initialized by firmware\n");
2616                 return -ENODEV;
2617         }
2618
2619         if (GRAPHICS_VER(i915) > 5 && !intel_vgpu_active(i915))
2620                 uncore->flags |= UNCORE_HAS_FORCEWAKE;
2621
2622         if (!intel_uncore_has_forcewake(uncore)) {
2623                 uncore_raw_init(uncore);
2624         } else {
2625                 ret = uncore_forcewake_init(uncore);
2626                 if (ret)
2627                         return ret;
2628         }
2629
2630         /* make sure fw funcs are set if and only if we have fw*/
2631         GEM_BUG_ON(intel_uncore_has_forcewake(uncore) != !!uncore->fw_get_funcs);
2632         GEM_BUG_ON(intel_uncore_has_forcewake(uncore) != !!uncore->funcs.read_fw_domains);
2633         GEM_BUG_ON(intel_uncore_has_forcewake(uncore) != !!uncore->funcs.write_fw_domains);
2634
2635         if (HAS_FPGA_DBG_UNCLAIMED(i915))
2636                 uncore->flags |= UNCORE_HAS_FPGA_DBG_UNCLAIMED;
2637
2638         if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915))
2639                 uncore->flags |= UNCORE_HAS_DBG_UNCLAIMED;
2640
2641         if (IS_GRAPHICS_VER(i915, 6, 7))
2642                 uncore->flags |= UNCORE_HAS_FIFO;
2643
2644         /* clear out unclaimed reg detection bit */
2645         if (intel_uncore_unclaimed_mmio(uncore))
2646                 drm_dbg(&i915->drm, "unclaimed mmio detected on uncore init, clearing\n");
2647
2648         return 0;
2649 }
2650
2651 /*
2652  * We might have detected that some engines are fused off after we initialized
2653  * the forcewake domains. Prune them, to make sure they only reference existing
2654  * engines.
2655  */
2656 void intel_uncore_prune_engine_fw_domains(struct intel_uncore *uncore,
2657                                           struct intel_gt *gt)
2658 {
2659         enum forcewake_domains fw_domains = uncore->fw_domains;
2660         enum forcewake_domain_id domain_id;
2661         int i;
2662
2663         if (!intel_uncore_has_forcewake(uncore) || GRAPHICS_VER(uncore->i915) < 11)
2664                 return;
2665
2666         for (i = 0; i < I915_MAX_VCS; i++) {
2667                 domain_id = FW_DOMAIN_ID_MEDIA_VDBOX0 + i;
2668
2669                 if (HAS_ENGINE(gt, _VCS(i)))
2670                         continue;
2671
2672                 /*
2673                  * Starting with XeHP, the power well for an even-numbered
2674                  * VDBOX is also used for shared units within the
2675                  * media slice such as SFC.  So even if the engine
2676                  * itself is fused off, we still need to initialize
2677                  * the forcewake domain if any of the other engines
2678                  * in the same media slice are present.
2679                  */
2680                 if (GRAPHICS_VER_FULL(uncore->i915) >= IP_VER(12, 50) && i % 2 == 0) {
2681                         if ((i + 1 < I915_MAX_VCS) && HAS_ENGINE(gt, _VCS(i + 1)))
2682                                 continue;
2683
2684                         if (HAS_ENGINE(gt, _VECS(i / 2)))
2685                                 continue;
2686                 }
2687
2688                 if (fw_domains & BIT(domain_id))
2689                         fw_domain_fini(uncore, domain_id);
2690         }
2691
2692         for (i = 0; i < I915_MAX_VECS; i++) {
2693                 domain_id = FW_DOMAIN_ID_MEDIA_VEBOX0 + i;
2694
2695                 if (HAS_ENGINE(gt, _VECS(i)))
2696                         continue;
2697
2698                 if (fw_domains & BIT(domain_id))
2699                         fw_domain_fini(uncore, domain_id);
2700         }
2701 }
2702
2703 /* Called via drm-managed action */
2704 void intel_uncore_fini_mmio(struct drm_device *dev, void *data)
2705 {
2706         struct intel_uncore *uncore = data;
2707
2708         if (intel_uncore_has_forcewake(uncore)) {
2709                 iosf_mbi_punit_acquire();
2710                 iosf_mbi_unregister_pmic_bus_access_notifier_unlocked(
2711                         &uncore->pmic_bus_access_nb);
2712                 intel_uncore_forcewake_reset(uncore);
2713                 intel_uncore_fw_domains_fini(uncore);
2714                 iosf_mbi_punit_release();
2715         }
2716 }
2717
2718 /**
2719  * __intel_wait_for_register_fw - wait until register matches expected state
2720  * @uncore: the struct intel_uncore
2721  * @reg: the register to read
2722  * @mask: mask to apply to register value
2723  * @value: expected value
2724  * @fast_timeout_us: fast timeout in microsecond for atomic/tight wait
2725  * @slow_timeout_ms: slow timeout in millisecond
2726  * @out_value: optional placeholder to hold registry value
2727  *
2728  * This routine waits until the target register @reg contains the expected
2729  * @value after applying the @mask, i.e. it waits until ::
2730  *
2731  *     (intel_uncore_read_fw(uncore, reg) & mask) == value
2732  *
2733  * Otherwise, the wait will timeout after @slow_timeout_ms milliseconds.
2734  * For atomic context @slow_timeout_ms must be zero and @fast_timeout_us
2735  * must be not larger than 20,0000 microseconds.
2736  *
2737  * Note that this routine assumes the caller holds forcewake asserted, it is
2738  * not suitable for very long waits. See intel_wait_for_register() if you
2739  * wish to wait without holding forcewake for the duration (i.e. you expect
2740  * the wait to be slow).
2741  *
2742  * Return: 0 if the register matches the desired condition, or -ETIMEDOUT.
2743  */
2744 int __intel_wait_for_register_fw(struct intel_uncore *uncore,
2745                                  i915_reg_t reg,
2746                                  u32 mask,
2747                                  u32 value,
2748                                  unsigned int fast_timeout_us,
2749                                  unsigned int slow_timeout_ms,
2750                                  u32 *out_value)
2751 {
2752         u32 reg_value = 0;
2753 #define done (((reg_value = intel_uncore_read_fw(uncore, reg)) & mask) == value)
2754         int ret;
2755
2756         /* Catch any overuse of this function */
2757         might_sleep_if(slow_timeout_ms);
2758         GEM_BUG_ON(fast_timeout_us > 20000);
2759         GEM_BUG_ON(!fast_timeout_us && !slow_timeout_ms);
2760
2761         ret = -ETIMEDOUT;
2762         if (fast_timeout_us && fast_timeout_us <= 20000)
2763                 ret = _wait_for_atomic(done, fast_timeout_us, 0);
2764         if (ret && slow_timeout_ms)
2765                 ret = wait_for(done, slow_timeout_ms);
2766
2767         if (out_value)
2768                 *out_value = reg_value;
2769
2770         return ret;
2771 #undef done
2772 }
2773
2774 /**
2775  * __intel_wait_for_register - wait until register matches expected state
2776  * @uncore: the struct intel_uncore
2777  * @reg: the register to read
2778  * @mask: mask to apply to register value
2779  * @value: expected value
2780  * @fast_timeout_us: fast timeout in microsecond for atomic/tight wait
2781  * @slow_timeout_ms: slow timeout in millisecond
2782  * @out_value: optional placeholder to hold registry value
2783  *
2784  * This routine waits until the target register @reg contains the expected
2785  * @value after applying the @mask, i.e. it waits until ::
2786  *
2787  *     (intel_uncore_read(uncore, reg) & mask) == value
2788  *
2789  * Otherwise, the wait will timeout after @timeout_ms milliseconds.
2790  *
2791  * Return: 0 if the register matches the desired condition, or -ETIMEDOUT.
2792  */
2793 int __intel_wait_for_register(struct intel_uncore *uncore,
2794                               i915_reg_t reg,
2795                               u32 mask,
2796                               u32 value,
2797                               unsigned int fast_timeout_us,
2798                               unsigned int slow_timeout_ms,
2799                               u32 *out_value)
2800 {
2801         unsigned fw =
2802                 intel_uncore_forcewake_for_reg(uncore, reg, FW_REG_READ);
2803         u32 reg_value;
2804         int ret;
2805
2806         might_sleep_if(slow_timeout_ms);
2807
2808         spin_lock_irq(&uncore->lock);
2809         intel_uncore_forcewake_get__locked(uncore, fw);
2810
2811         ret = __intel_wait_for_register_fw(uncore,
2812                                            reg, mask, value,
2813                                            fast_timeout_us, 0, &reg_value);
2814
2815         intel_uncore_forcewake_put__locked(uncore, fw);
2816         spin_unlock_irq(&uncore->lock);
2817
2818         if (ret && slow_timeout_ms)
2819                 ret = __wait_for(reg_value = intel_uncore_read_notrace(uncore,
2820                                                                        reg),
2821                                  (reg_value & mask) == value,
2822                                  slow_timeout_ms * 1000, 10, 1000);
2823
2824         /* just trace the final value */
2825         trace_i915_reg_rw(false, reg, reg_value, sizeof(reg_value), true);
2826
2827         if (out_value)
2828                 *out_value = reg_value;
2829
2830         return ret;
2831 }
2832
2833 bool intel_uncore_unclaimed_mmio(struct intel_uncore *uncore)
2834 {
2835         bool ret;
2836
2837         if (!uncore->debug)
2838                 return false;
2839
2840         spin_lock_irq(&uncore->debug->lock);
2841         ret = check_for_unclaimed_mmio(uncore);
2842         spin_unlock_irq(&uncore->debug->lock);
2843
2844         return ret;
2845 }
2846
2847 bool
2848 intel_uncore_arm_unclaimed_mmio_detection(struct intel_uncore *uncore)
2849 {
2850         bool ret = false;
2851
2852         if (drm_WARN_ON(&uncore->i915->drm, !uncore->debug))
2853                 return false;
2854
2855         spin_lock_irq(&uncore->debug->lock);
2856
2857         if (unlikely(uncore->debug->unclaimed_mmio_check <= 0))
2858                 goto out;
2859
2860         if (unlikely(check_for_unclaimed_mmio(uncore))) {
2861                 if (!uncore->i915->params.mmio_debug) {
2862                         drm_dbg(&uncore->i915->drm,
2863                                 "Unclaimed register detected, "
2864                                 "enabling oneshot unclaimed register reporting. "
2865                                 "Please use i915.mmio_debug=N for more information.\n");
2866                         uncore->i915->params.mmio_debug++;
2867                 }
2868                 uncore->debug->unclaimed_mmio_check--;
2869                 ret = true;
2870         }
2871
2872 out:
2873         spin_unlock_irq(&uncore->debug->lock);
2874
2875         return ret;
2876 }
2877
2878 /**
2879  * intel_uncore_forcewake_for_reg - which forcewake domains are needed to access
2880  *                                  a register
2881  * @uncore: pointer to struct intel_uncore
2882  * @reg: register in question
2883  * @op: operation bitmask of FW_REG_READ and/or FW_REG_WRITE
2884  *
2885  * Returns a set of forcewake domains required to be taken with for example
2886  * intel_uncore_forcewake_get for the specified register to be accessible in the
2887  * specified mode (read, write or read/write) with raw mmio accessors.
2888  *
2889  * NOTE: On Gen6 and Gen7 write forcewake domain (FORCEWAKE_RENDER) requires the
2890  * callers to do FIFO management on their own or risk losing writes.
2891  */
2892 enum forcewake_domains
2893 intel_uncore_forcewake_for_reg(struct intel_uncore *uncore,
2894                                i915_reg_t reg, unsigned int op)
2895 {
2896         enum forcewake_domains fw_domains = 0;
2897
2898         drm_WARN_ON(&uncore->i915->drm, !op);
2899
2900         if (!intel_uncore_has_forcewake(uncore))
2901                 return 0;
2902
2903         if (op & FW_REG_READ)
2904                 fw_domains = uncore->funcs.read_fw_domains(uncore, reg);
2905
2906         if (op & FW_REG_WRITE)
2907                 fw_domains |= uncore->funcs.write_fw_domains(uncore, reg);
2908
2909         drm_WARN_ON(&uncore->i915->drm, fw_domains & ~uncore->fw_domains);
2910
2911         return fw_domains;
2912 }
2913
2914 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
2915 #include "selftests/mock_uncore.c"
2916 #include "selftests/intel_uncore.c"
2917 #endif