drivers/gpu/drm/i915/i915_gem_gtt.c

   1 /*
   2  * Copyright © 2010 Daniel Vetter
   3  * Copyright © 2011-2014 Intel Corporation
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9  * and/or sell copies of the Software, and to permit persons to whom the
  10  * Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice (including the next
  13  * paragraph) shall be included in all copies or substantial portions of the
  14  * Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  21  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  22  * IN THE SOFTWARE.
  23  *
  24  */
  25
  26 #include <linux/slab.h> /* fault-inject.h is not standalone! */
  27
  28 #include <linux/fault-inject.h>
  29 #include <linux/log2.h>
  30 #include <linux/random.h>
  31 #include <linux/seq_file.h>
  32 #include <linux/stop_machine.h>
  33
  34 #include <asm/set_memory.h>
  35
  36 #include <drm/drmP.h>
  37 #include <drm/i915_drm.h>
  38
  39 #include "i915_drv.h"
  40 #include "i915_vgpu.h"
  41 #include "i915_trace.h"
  42 #include "intel_drv.h"
  43 #include "intel_frontbuffer.h"
  44
  45 #define I915_GFP_DMA (GFP_KERNEL | __GFP_HIGHMEM)
  46
  47 /**
  48  * DOC: Global GTT views
  49  *
  50  * Background and previous state
  51  *
  52  * Historically objects could exists (be bound) in global GTT space only as
  53  * singular instances with a view representing all of the object's backing pages
  54  * in a linear fashion. This view will be called a normal view.
  55  *
  56  * To support multiple views of the same object, where the number of mapped
  57  * pages is not equal to the backing store, or where the layout of the pages
  58  * is not linear, concept of a GGTT view was added.
  59  *
  60  * One example of an alternative view is a stereo display driven by a single
  61  * image. In this case we would have a framebuffer looking like this
  62  * (2x2 pages):
  63  *
  64  *    12
  65  *    34
  66  *
  67  * Above would represent a normal GGTT view as normally mapped for GPU or CPU
  68  * rendering. In contrast, fed to the display engine would be an alternative
  69  * view which could look something like this:
  70  *
  71  *   1212
  72  *   3434
  73  *
  74  * In this example both the size and layout of pages in the alternative view is
  75  * different from the normal view.
  76  *
  77  * Implementation and usage
  78  *
  79  * GGTT views are implemented using VMAs and are distinguished via enum
  80  * i915_ggtt_view_type and struct i915_ggtt_view.
  81  *
  82  * A new flavour of core GEM functions which work with GGTT bound objects were
  83  * added with the _ggtt_ infix, and sometimes with _view postfix to avoid
  84  * renaming  in large amounts of code. They take the struct i915_ggtt_view
  85  * parameter encapsulating all metadata required to implement a view.
  86  *
  87  * As a helper for callers which are only interested in the normal view,
  88  * globally const i915_ggtt_view_normal singleton instance exists. All old core
  89  * GEM API functions, the ones not taking the view parameter, are operating on,
  90  * or with the normal GGTT view.
  91  *
  92  * Code wanting to add or use a new GGTT view needs to:
  93  *
  94  * 1. Add a new enum with a suitable name.
  95  * 2. Extend the metadata in the i915_ggtt_view structure if required.
  96  * 3. Add support to i915_get_vma_pages().
  97  *
  98  * New views are required to build a scatter-gather table from within the
  99  * i915_get_vma_pages function. This table is stored in the vma.ggtt_view and
 100  * exists for the lifetime of an VMA.
 101  *
 102  * Core API is designed to have copy semantics which means that passed in
 103  * struct i915_ggtt_view does not need to be persistent (left around after
 104  * calling the core API functions).
 105  *
 106  */
 107
 108 static int
 109 i915_get_ggtt_vma_pages(struct i915_vma *vma);
 110
 111 static void gen6_ggtt_invalidate(struct drm_i915_private *dev_priv)
 112 {
 113         /* Note that as an uncached mmio write, this should flush the
 114          * WCB of the writes into the GGTT before it triggers the invalidate.
 115          */
 116         I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
 117 }
 118
 119 static void guc_ggtt_invalidate(struct drm_i915_private *dev_priv)
 120 {
 121         gen6_ggtt_invalidate(dev_priv);
 122         I915_WRITE(GEN8_GTCR, GEN8_GTCR_INVALIDATE);
 123 }
 124
 125 static void gmch_ggtt_invalidate(struct drm_i915_private *dev_priv)
 126 {
 127         intel_gtt_chipset_flush();
 128 }
 129
 130 static inline void i915_ggtt_invalidate(struct drm_i915_private *i915)
 131 {
 132         i915->ggtt.invalidate(i915);
 133 }
 134
 135 int intel_sanitize_enable_ppgtt(struct drm_i915_private *dev_priv,
 136                                 int enable_ppgtt)
 137 {
 138         bool has_aliasing_ppgtt;
 139         bool has_full_ppgtt;
 140         bool has_full_48bit_ppgtt;
 141
 142         has_aliasing_ppgtt = dev_priv->info.has_aliasing_ppgtt;
 143         has_full_ppgtt = dev_priv->info.has_full_ppgtt;
 144         has_full_48bit_ppgtt = dev_priv->info.has_full_48bit_ppgtt;
 145
 146         if (intel_vgpu_active(dev_priv)) {
 147                 /* emulation is too hard */
 148                 has_full_ppgtt = false;
 149                 has_full_48bit_ppgtt = false;
 150         }
 151
 152         if (!has_aliasing_ppgtt)
 153                 return 0;
 154
 155         /*
 156          * We don't allow disabling PPGTT for gen9+ as it's a requirement for
 157          * execlists, the sole mechanism available to submit work.
 158          */
 159         if (enable_ppgtt == 0 && INTEL_GEN(dev_priv) < 9)
 160                 return 0;
 161
 162         if (enable_ppgtt == 1)
 163                 return 1;
 164
 165         if (enable_ppgtt == 2 && has_full_ppgtt)
 166                 return 2;
 167
 168         if (enable_ppgtt == 3 && has_full_48bit_ppgtt)
 169                 return 3;
 170
 171 #ifdef CONFIG_INTEL_IOMMU
 172         /* Disable ppgtt on SNB if VT-d is on. */
 173         if (IS_GEN6(dev_priv) && intel_iommu_gfx_mapped) {
 174                 DRM_INFO("Disabling PPGTT because VT-d is on\n");
 175                 return 0;
 176         }
 177 #endif
 178
 179         /* Early VLV doesn't have this */
 180         if (IS_VALLEYVIEW(dev_priv) && dev_priv->drm.pdev->revision < 0xb) {
 181                 DRM_DEBUG_DRIVER("disabling PPGTT on pre-B3 step VLV\n");
 182                 return 0;
 183         }
 184
 185         if (INTEL_GEN(dev_priv) >= 8 && i915.enable_execlists && has_full_ppgtt)
 186                 return has_full_48bit_ppgtt ? 3 : 2;
 187         else
 188                 return has_aliasing_ppgtt ? 1 : 0;
 189 }
 190
 191 static int ppgtt_bind_vma(struct i915_vma *vma,
 192                           enum i915_cache_level cache_level,
 193                           u32 unused)
 194 {
 195         u32 pte_flags;
 196         int ret;
 197
 198         ret = vma->vm->allocate_va_range(vma->vm, vma->node.start, vma->size);
 199         if (ret)
 200                 return ret;
 201
 202         vma->pages = vma->obj->mm.pages;
 203
 204         /* Currently applicable only to VLV */
 205         pte_flags = 0;
 206         if (vma->obj->gt_ro)
 207                 pte_flags |= PTE_READ_ONLY;
 208
 209         vma->vm->insert_entries(vma->vm, vma->pages, vma->node.start,
 210                                 cache_level, pte_flags);
 211
 212         return 0;
 213 }
 214
 215 static void ppgtt_unbind_vma(struct i915_vma *vma)
 216 {
 217         vma->vm->clear_range(vma->vm, vma->node.start, vma->size);
 218 }
 219
 220 static gen8_pte_t gen8_pte_encode(dma_addr_t addr,
 221                                   enum i915_cache_level level)
 222 {
 223         gen8_pte_t pte = _PAGE_PRESENT | _PAGE_RW;
 224         pte |= addr;
 225
 226         switch (level) {
 227         case I915_CACHE_NONE:
 228                 pte |= PPAT_UNCACHED_INDEX;
 229                 break;
 230         case I915_CACHE_WT:
 231                 pte |= PPAT_DISPLAY_ELLC_INDEX;
 232                 break;
 233         default:
 234                 pte |= PPAT_CACHED_INDEX;
 235                 break;
 236         }
 237
 238         return pte;
 239 }
 240
 241 static gen8_pde_t gen8_pde_encode(const dma_addr_t addr,
 242                                   const enum i915_cache_level level)
 243 {
 244         gen8_pde_t pde = _PAGE_PRESENT | _PAGE_RW;
 245         pde |= addr;
 246         if (level != I915_CACHE_NONE)
 247                 pde |= PPAT_CACHED_PDE_INDEX;
 248         else
 249                 pde |= PPAT_UNCACHED_INDEX;
 250         return pde;
 251 }
 252
 253 #define gen8_pdpe_encode gen8_pde_encode
 254 #define gen8_pml4e_encode gen8_pde_encode
 255
 256 static gen6_pte_t snb_pte_encode(dma_addr_t addr,
 257                                  enum i915_cache_level level,
 258                                  u32 unused)
 259 {
 260         gen6_pte_t pte = GEN6_PTE_VALID;
 261         pte |= GEN6_PTE_ADDR_ENCODE(addr);
 262
 263         switch (level) {
 264         case I915_CACHE_L3_LLC:
 265         case I915_CACHE_LLC:
 266                 pte |= GEN6_PTE_CACHE_LLC;
 267                 break;
 268         case I915_CACHE_NONE:
 269                 pte |= GEN6_PTE_UNCACHED;
 270                 break;
 271         default:
 272                 MISSING_CASE(level);
 273         }
 274
 275         return pte;
 276 }
 277
 278 static gen6_pte_t ivb_pte_encode(dma_addr_t addr,
 279                                  enum i915_cache_level level,
 280                                  u32 unused)
 281 {
 282         gen6_pte_t pte = GEN6_PTE_VALID;
 283         pte |= GEN6_PTE_ADDR_ENCODE(addr);
 284
 285         switch (level) {
 286         case I915_CACHE_L3_LLC:
 287                 pte |= GEN7_PTE_CACHE_L3_LLC;
 288                 break;
 289         case I915_CACHE_LLC:
 290                 pte |= GEN6_PTE_CACHE_LLC;
 291                 break;
 292         case I915_CACHE_NONE:
 293                 pte |= GEN6_PTE_UNCACHED;
 294                 break;
 295         default:
 296                 MISSING_CASE(level);
 297         }
 298
 299         return pte;
 300 }
 301
 302 static gen6_pte_t byt_pte_encode(dma_addr_t addr,
 303                                  enum i915_cache_level level,
 304                                  u32 flags)
 305 {
 306         gen6_pte_t pte = GEN6_PTE_VALID;
 307         pte |= GEN6_PTE_ADDR_ENCODE(addr);
 308
 309         if (!(flags & PTE_READ_ONLY))
 310                 pte |= BYT_PTE_WRITEABLE;
 311
 312         if (level != I915_CACHE_NONE)
 313                 pte |= BYT_PTE_SNOOPED_BY_CPU_CACHES;
 314
 315         return pte;
 316 }
 317
 318 static gen6_pte_t hsw_pte_encode(dma_addr_t addr,
 319                                  enum i915_cache_level level,
 320                                  u32 unused)
 321 {
 322         gen6_pte_t pte = GEN6_PTE_VALID;
 323         pte |= HSW_PTE_ADDR_ENCODE(addr);
 324
 325         if (level != I915_CACHE_NONE)
 326                 pte |= HSW_WB_LLC_AGE3;
 327
 328         return pte;
 329 }
 330
 331 static gen6_pte_t iris_pte_encode(dma_addr_t addr,
 332                                   enum i915_cache_level level,
 333                                   u32 unused)
 334 {
 335         gen6_pte_t pte = GEN6_PTE_VALID;
 336         pte |= HSW_PTE_ADDR_ENCODE(addr);
 337
 338         switch (level) {
 339         case I915_CACHE_NONE:
 340                 break;
 341         case I915_CACHE_WT:
 342                 pte |= HSW_WT_ELLC_LLC_AGE3;
 343                 break;
 344         default:
 345                 pte |= HSW_WB_ELLC_LLC_AGE3;
 346                 break;
 347         }
 348
 349         return pte;
 350 }
 351
 352 static struct page *vm_alloc_page(struct i915_address_space *vm, gfp_t gfp)
 353 {
 354         struct page *page;
 355
 356         if (I915_SELFTEST_ONLY(should_fail(&vm->fault_attr, 1)))
 357                 i915_gem_shrink_all(vm->i915);
 358
 359         if (vm->free_pages.nr)
 360                 return vm->free_pages.pages[--vm->free_pages.nr];
 361
 362         page = alloc_page(gfp);
 363         if (!page)
 364                 return NULL;
 365
 366         if (vm->pt_kmap_wc)
 367                 set_pages_array_wc(&page, 1);
 368
 369         return page;
 370 }
 371
 372 static void vm_free_pages_release(struct i915_address_space *vm)
 373 {
 374         GEM_BUG_ON(!pagevec_count(&vm->free_pages));
 375
 376         if (vm->pt_kmap_wc)
 377                 set_pages_array_wb(vm->free_pages.pages,
 378                                    pagevec_count(&vm->free_pages));
 379
 380         __pagevec_release(&vm->free_pages);
 381 }
 382
 383 static void vm_free_page(struct i915_address_space *vm, struct page *page)
 384 {
 385         if (!pagevec_add(&vm->free_pages, page))
 386                 vm_free_pages_release(vm);
 387 }
 388
 389 static int __setup_page_dma(struct i915_address_space *vm,
 390                             struct i915_page_dma *p,
 391                             gfp_t gfp)
 392 {
 393         p->page = vm_alloc_page(vm, gfp | __GFP_NOWARN | __GFP_NORETRY);
 394         if (unlikely(!p->page))
 395                 return -ENOMEM;
 396
 397         p->daddr = dma_map_page(vm->dma, p->page, 0, PAGE_SIZE,
 398                                 PCI_DMA_BIDIRECTIONAL);
 399         if (unlikely(dma_mapping_error(vm->dma, p->daddr))) {
 400                 vm_free_page(vm, p->page);
 401                 return -ENOMEM;
 402         }
 403
 404         return 0;
 405 }
 406
 407 static int setup_page_dma(struct i915_address_space *vm,
 408                           struct i915_page_dma *p)
 409 {
 410         return __setup_page_dma(vm, p, I915_GFP_DMA);
 411 }
 412
 413 static void cleanup_page_dma(struct i915_address_space *vm,
 414                              struct i915_page_dma *p)
 415 {
 416         dma_unmap_page(vm->dma, p->daddr, PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
 417         vm_free_page(vm, p->page);
 418 }
 419
 420 #define kmap_atomic_px(px) kmap_atomic(px_base(px)->page)
 421
 422 #define setup_px(vm, px) setup_page_dma((vm), px_base(px))
 423 #define cleanup_px(vm, px) cleanup_page_dma((vm), px_base(px))
 424 #define fill_px(ppgtt, px, v) fill_page_dma((vm), px_base(px), (v))
 425 #define fill32_px(ppgtt, px, v) fill_page_dma_32((vm), px_base(px), (v))
 426
 427 static void fill_page_dma(struct i915_address_space *vm,
 428                           struct i915_page_dma *p,
 429                           const u64 val)
 430 {
 431         u64 * const vaddr = kmap_atomic(p->page);
 432         int i;
 433
 434         for (i = 0; i < 512; i++)
 435                 vaddr[i] = val;
 436
 437         kunmap_atomic(vaddr);
 438 }
 439
 440 static void fill_page_dma_32(struct i915_address_space *vm,
 441                              struct i915_page_dma *p,
 442                              const u32 v)
 443 {
 444         fill_page_dma(vm, p, (u64)v << 32 | v);
 445 }
 446
 447 static int
 448 setup_scratch_page(struct i915_address_space *vm, gfp_t gfp)
 449 {
 450         return __setup_page_dma(vm, &vm->scratch_page, gfp | __GFP_ZERO);
 451 }
 452
 453 static void cleanup_scratch_page(struct i915_address_space *vm)
 454 {
 455         cleanup_page_dma(vm, &vm->scratch_page);
 456 }
 457
 458 static struct i915_page_table *alloc_pt(struct i915_address_space *vm)
 459 {
 460         struct i915_page_table *pt;
 461
 462         pt = kmalloc(sizeof(*pt), GFP_KERNEL | __GFP_NOWARN);
 463         if (unlikely(!pt))
 464                 return ERR_PTR(-ENOMEM);
 465
 466         if (unlikely(setup_px(vm, pt))) {
 467                 kfree(pt);
 468                 return ERR_PTR(-ENOMEM);
 469         }
 470
 471         pt->used_ptes = 0;
 472         return pt;
 473 }
 474
 475 static void free_pt(struct i915_address_space *vm, struct i915_page_table *pt)
 476 {
 477         cleanup_px(vm, pt);
 478         kfree(pt);
 479 }
 480
 481 static void gen8_initialize_pt(struct i915_address_space *vm,
 482                                struct i915_page_table *pt)
 483 {
 484         fill_px(vm, pt,
 485                 gen8_pte_encode(vm->scratch_page.daddr, I915_CACHE_LLC));
 486 }
 487
 488 static void gen6_initialize_pt(struct i915_address_space *vm,
 489                                struct i915_page_table *pt)
 490 {
 491         fill32_px(vm, pt,
 492                   vm->pte_encode(vm->scratch_page.daddr, I915_CACHE_LLC, 0));
 493 }
 494
 495 static struct i915_page_directory *alloc_pd(struct i915_address_space *vm)
 496 {
 497         struct i915_page_directory *pd;
 498
 499         pd = kzalloc(sizeof(*pd), GFP_KERNEL | __GFP_NOWARN);
 500         if (unlikely(!pd))
 501                 return ERR_PTR(-ENOMEM);
 502
 503         if (unlikely(setup_px(vm, pd))) {
 504                 kfree(pd);
 505                 return ERR_PTR(-ENOMEM);
 506         }
 507
 508         pd->used_pdes = 0;
 509         return pd;
 510 }
 511
 512 static void free_pd(struct i915_address_space *vm,
 513                     struct i915_page_directory *pd)
 514 {
 515         cleanup_px(vm, pd);
 516         kfree(pd);
 517 }
 518
 519 static void gen8_initialize_pd(struct i915_address_space *vm,
 520                                struct i915_page_directory *pd)
 521 {
 522         unsigned int i;
 523
 524         fill_px(vm, pd,
 525                 gen8_pde_encode(px_dma(vm->scratch_pt), I915_CACHE_LLC));
 526         for (i = 0; i < I915_PDES; i++)
 527                 pd->page_table[i] = vm->scratch_pt;
 528 }
 529
 530 static int __pdp_init(struct i915_address_space *vm,
 531                       struct i915_page_directory_pointer *pdp)
 532 {
 533         const unsigned int pdpes = i915_pdpes_per_pdp(vm);
 534         unsigned int i;
 535
 536         pdp->page_directory = kmalloc_array(pdpes, sizeof(*pdp->page_directory),
 537                                             GFP_KERNEL | __GFP_NOWARN);
 538         if (unlikely(!pdp->page_directory))
 539                 return -ENOMEM;
 540
 541         for (i = 0; i < pdpes; i++)
 542                 pdp->page_directory[i] = vm->scratch_pd;
 543
 544         return 0;
 545 }
 546
 547 static void __pdp_fini(struct i915_page_directory_pointer *pdp)
 548 {
 549         kfree(pdp->page_directory);
 550         pdp->page_directory = NULL;
 551 }
 552
 553 static inline bool use_4lvl(const struct i915_address_space *vm)
 554 {
 555         return i915_vm_is_48bit(vm);
 556 }
 557
 558 static struct i915_page_directory_pointer *
 559 alloc_pdp(struct i915_address_space *vm)
 560 {
 561         struct i915_page_directory_pointer *pdp;
 562         int ret = -ENOMEM;
 563
 564         WARN_ON(!use_4lvl(vm));
 565
 566         pdp = kzalloc(sizeof(*pdp), GFP_KERNEL);
 567         if (!pdp)
 568                 return ERR_PTR(-ENOMEM);
 569
 570         ret = __pdp_init(vm, pdp);
 571         if (ret)
 572                 goto fail_bitmap;
 573
 574         ret = setup_px(vm, pdp);
 575         if (ret)
 576                 goto fail_page_m;
 577
 578         return pdp;
 579
 580 fail_page_m:
 581         __pdp_fini(pdp);
 582 fail_bitmap:
 583         kfree(pdp);
 584
 585         return ERR_PTR(ret);
 586 }
 587
 588 static void free_pdp(struct i915_address_space *vm,
 589                      struct i915_page_directory_pointer *pdp)
 590 {
 591         __pdp_fini(pdp);
 592
 593         if (!use_4lvl(vm))
 594                 return;
 595
 596         cleanup_px(vm, pdp);
 597         kfree(pdp);
 598 }
 599
 600 static void gen8_initialize_pdp(struct i915_address_space *vm,
 601                                 struct i915_page_directory_pointer *pdp)
 602 {
 603         gen8_ppgtt_pdpe_t scratch_pdpe;
 604
 605         scratch_pdpe = gen8_pdpe_encode(px_dma(vm->scratch_pd), I915_CACHE_LLC);
 606
 607         fill_px(vm, pdp, scratch_pdpe);
 608 }
 609
 610 static void gen8_initialize_pml4(struct i915_address_space *vm,
 611                                  struct i915_pml4 *pml4)
 612 {
 613         unsigned int i;
 614
 615         fill_px(vm, pml4,
 616                 gen8_pml4e_encode(px_dma(vm->scratch_pdp), I915_CACHE_LLC));
 617         for (i = 0; i < GEN8_PML4ES_PER_PML4; i++)
 618                 pml4->pdps[i] = vm->scratch_pdp;
 619 }
 620
 621 /* Broadwell Page Directory Pointer Descriptors */
 622 static int gen8_write_pdp(struct drm_i915_gem_request *req,
 623                           unsigned entry,
 624                           dma_addr_t addr)
 625 {
 626         struct intel_engine_cs *engine = req->engine;
 627         u32 *cs;
 628
 629         BUG_ON(entry >= 4);
 630
 631         cs = intel_ring_begin(req, 6);
 632         if (IS_ERR(cs))
 633                 return PTR_ERR(cs);
 634
 635         *cs++ = MI_LOAD_REGISTER_IMM(1);
 636         *cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_UDW(engine, entry));
 637         *cs++ = upper_32_bits(addr);
 638         *cs++ = MI_LOAD_REGISTER_IMM(1);
 639         *cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_LDW(engine, entry));
 640         *cs++ = lower_32_bits(addr);
 641         intel_ring_advance(req, cs);
 642
 643         return 0;
 644 }
 645
 646 static int gen8_mm_switch_3lvl(struct i915_hw_ppgtt *ppgtt,
 647                                struct drm_i915_gem_request *req)
 648 {
 649         int i, ret;
 650
 651         for (i = GEN8_3LVL_PDPES - 1; i >= 0; i--) {
 652                 const dma_addr_t pd_daddr = i915_page_dir_dma_addr(ppgtt, i);
 653
 654                 ret = gen8_write_pdp(req, i, pd_daddr);
 655                 if (ret)
 656                         return ret;
 657         }
 658
 659         return 0;
 660 }
 661
 662 static int gen8_mm_switch_4lvl(struct i915_hw_ppgtt *ppgtt,
 663                                struct drm_i915_gem_request *req)
 664 {
 665         return gen8_write_pdp(req, 0, px_dma(&ppgtt->pml4));
 666 }
 667
 668 /* PDE TLBs are a pain to invalidate on GEN8+. When we modify
 669  * the page table structures, we mark them dirty so that
 670  * context switching/execlist queuing code takes extra steps
 671  * to ensure that tlbs are flushed.
 672  */
 673 static void mark_tlbs_dirty(struct i915_hw_ppgtt *ppgtt)
 674 {
 675         ppgtt->pd_dirty_rings = INTEL_INFO(ppgtt->base.i915)->ring_mask;
 676 }
 677
 678 /* Removes entries from a single page table, releasing it if it's empty.
 679  * Caller can use the return value to update higher-level entries.
 680  */
 681 static bool gen8_ppgtt_clear_pt(struct i915_address_space *vm,
 682                                 struct i915_page_table *pt,
 683                                 u64 start, u64 length)
 684 {
 685         unsigned int num_entries = gen8_pte_count(start, length);
 686         unsigned int pte = gen8_pte_index(start);
 687         unsigned int pte_end = pte + num_entries;
 688         const gen8_pte_t scratch_pte =
 689                 gen8_pte_encode(vm->scratch_page.daddr, I915_CACHE_LLC);
 690         gen8_pte_t *vaddr;
 691
 692         GEM_BUG_ON(num_entries > pt->used_ptes);
 693
 694         pt->used_ptes -= num_entries;
 695         if (!pt->used_ptes)
 696                 return true;
 697
 698         vaddr = kmap_atomic_px(pt);
 699         while (pte < pte_end)
 700                 vaddr[pte++] = scratch_pte;
 701         kunmap_atomic(vaddr);
 702
 703         return false;
 704 }
 705
 706 static void gen8_ppgtt_set_pde(struct i915_address_space *vm,
 707                                struct i915_page_directory *pd,
 708                                struct i915_page_table *pt,
 709                                unsigned int pde)
 710 {
 711         gen8_pde_t *vaddr;
 712
 713         pd->page_table[pde] = pt;
 714
 715         vaddr = kmap_atomic_px(pd);
 716         vaddr[pde] = gen8_pde_encode(px_dma(pt), I915_CACHE_LLC);
 717         kunmap_atomic(vaddr);
 718 }
 719
 720 static bool gen8_ppgtt_clear_pd(struct i915_address_space *vm,
 721                                 struct i915_page_directory *pd,
 722                                 u64 start, u64 length)
 723 {
 724         struct i915_page_table *pt;
 725         u32 pde;
 726
 727         gen8_for_each_pde(pt, pd, start, length, pde) {
 728                 GEM_BUG_ON(pt == vm->scratch_pt);
 729
 730                 if (!gen8_ppgtt_clear_pt(vm, pt, start, length))
 731                         continue;
 732
 733                 gen8_ppgtt_set_pde(vm, pd, vm->scratch_pt, pde);
 734                 GEM_BUG_ON(!pd->used_pdes);
 735                 pd->used_pdes--;
 736
 737                 free_pt(vm, pt);
 738         }
 739
 740         return !pd->used_pdes;
 741 }
 742
 743 static void gen8_ppgtt_set_pdpe(struct i915_address_space *vm,
 744                                 struct i915_page_directory_pointer *pdp,
 745                                 struct i915_page_directory *pd,
 746                                 unsigned int pdpe)
 747 {
 748         gen8_ppgtt_pdpe_t *vaddr;
 749
 750         pdp->page_directory[pdpe] = pd;
 751         if (!use_4lvl(vm))
 752                 return;
 753
 754         vaddr = kmap_atomic_px(pdp);
 755         vaddr[pdpe] = gen8_pdpe_encode(px_dma(pd), I915_CACHE_LLC);
 756         kunmap_atomic(vaddr);
 757 }
 758
 759 /* Removes entries from a single page dir pointer, releasing it if it's empty.
 760  * Caller can use the return value to update higher-level entries
 761  */
 762 static bool gen8_ppgtt_clear_pdp(struct i915_address_space *vm,
 763                                  struct i915_page_directory_pointer *pdp,
 764                                  u64 start, u64 length)
 765 {
 766         struct i915_page_directory *pd;
 767         unsigned int pdpe;
 768
 769         gen8_for_each_pdpe(pd, pdp, start, length, pdpe) {
 770                 GEM_BUG_ON(pd == vm->scratch_pd);
 771
 772                 if (!gen8_ppgtt_clear_pd(vm, pd, start, length))
 773                         continue;
 774
 775                 gen8_ppgtt_set_pdpe(vm, pdp, vm->scratch_pd, pdpe);
 776                 GEM_BUG_ON(!pdp->used_pdpes);
 777                 pdp->used_pdpes--;
 778
 779                 free_pd(vm, pd);
 780         }
 781
 782         return !pdp->used_pdpes;
 783 }
 784
 785 static void gen8_ppgtt_clear_3lvl(struct i915_address_space *vm,
 786                                   u64 start, u64 length)
 787 {
 788         gen8_ppgtt_clear_pdp(vm, &i915_vm_to_ppgtt(vm)->pdp, start, length);
 789 }
 790
 791 static void gen8_ppgtt_set_pml4e(struct i915_pml4 *pml4,
 792                                  struct i915_page_directory_pointer *pdp,
 793                                  unsigned int pml4e)
 794 {
 795         gen8_ppgtt_pml4e_t *vaddr;
 796
 797         pml4->pdps[pml4e] = pdp;
 798
 799         vaddr = kmap_atomic_px(pml4);
 800         vaddr[pml4e] = gen8_pml4e_encode(px_dma(pdp), I915_CACHE_LLC);
 801         kunmap_atomic(vaddr);
 802 }
 803
 804 /* Removes entries from a single pml4.
 805  * This is the top-level structure in 4-level page tables used on gen8+.
 806  * Empty entries are always scratch pml4e.
 807  */
 808 static void gen8_ppgtt_clear_4lvl(struct i915_address_space *vm,
 809                                   u64 start, u64 length)
 810 {
 811         struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
 812         struct i915_pml4 *pml4 = &ppgtt->pml4;
 813         struct i915_page_directory_pointer *pdp;
 814         unsigned int pml4e;
 815
 816         GEM_BUG_ON(!use_4lvl(vm));
 817
 818         gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) {
 819                 GEM_BUG_ON(pdp == vm->scratch_pdp);
 820
 821                 if (!gen8_ppgtt_clear_pdp(vm, pdp, start, length))
 822                         continue;
 823
 824                 gen8_ppgtt_set_pml4e(pml4, vm->scratch_pdp, pml4e);
 825
 826                 free_pdp(vm, pdp);
 827         }
 828 }
 829
 830 struct sgt_dma {
 831         struct scatterlist *sg;
 832         dma_addr_t dma, max;
 833 };
 834
 835 struct gen8_insert_pte {
 836         u16 pml4e;
 837         u16 pdpe;
 838         u16 pde;
 839         u16 pte;
 840 };
 841
 842 static __always_inline struct gen8_insert_pte gen8_insert_pte(u64 start)
 843 {
 844         return (struct gen8_insert_pte) {
 845                  gen8_pml4e_index(start),
 846                  gen8_pdpe_index(start),
 847                  gen8_pde_index(start),
 848                  gen8_pte_index(start),
 849         };
 850 }
 851
 852 static __always_inline bool
 853 gen8_ppgtt_insert_pte_entries(struct i915_hw_ppgtt *ppgtt,
 854                               struct i915_page_directory_pointer *pdp,
 855                               struct sgt_dma *iter,
 856                               struct gen8_insert_pte *idx,
 857                               enum i915_cache_level cache_level)
 858 {
 859         struct i915_page_directory *pd;
 860         const gen8_pte_t pte_encode = gen8_pte_encode(0, cache_level);
 861         gen8_pte_t *vaddr;
 862         bool ret;
 863
 864         GEM_BUG_ON(idx->pdpe >= i915_pdpes_per_pdp(&ppgtt->base));
 865         pd = pdp->page_directory[idx->pdpe];
 866         vaddr = kmap_atomic_px(pd->page_table[idx->pde]);
 867         do {
 868                 vaddr[idx->pte] = pte_encode | iter->dma;
 869
 870                 iter->dma += PAGE_SIZE;
 871                 if (iter->dma >= iter->max) {
 872                         iter->sg = __sg_next(iter->sg);
 873                         if (!iter->sg) {
 874                                 ret = false;
 875                                 break;
 876                         }
 877
 878                         iter->dma = sg_dma_address(iter->sg);
 879                         iter->max = iter->dma + iter->sg->length;
 880                 }
 881
 882                 if (++idx->pte == GEN8_PTES) {
 883                         idx->pte = 0;
 884
 885                         if (++idx->pde == I915_PDES) {
 886                                 idx->pde = 0;
 887
 888                                 /* Limited by sg length for 3lvl */
 889                                 if (++idx->pdpe == GEN8_PML4ES_PER_PML4) {
 890                                         idx->pdpe = 0;
 891                                         ret = true;
 892                                         break;
 893                                 }
 894
 895                                 GEM_BUG_ON(idx->pdpe >= i915_pdpes_per_pdp(&ppgtt->base));
 896                                 pd = pdp->page_directory[idx->pdpe];
 897                         }
 898
 899                         kunmap_atomic(vaddr);
 900                         vaddr = kmap_atomic_px(pd->page_table[idx->pde]);
 901                 }
 902         } while (1);
 903         kunmap_atomic(vaddr);
 904
 905         return ret;
 906 }
 907
 908 static void gen8_ppgtt_insert_3lvl(struct i915_address_space *vm,
 909                                    struct sg_table *pages,
 910                                    u64 start,
 911                                    enum i915_cache_level cache_level,
 912                                    u32 unused)
 913 {
 914         struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
 915         struct sgt_dma iter = {
 916                 .sg = pages->sgl,
 917                 .dma = sg_dma_address(iter.sg),
 918                 .max = iter.dma + iter.sg->length,
 919         };
 920         struct gen8_insert_pte idx = gen8_insert_pte(start);
 921
 922         gen8_ppgtt_insert_pte_entries(ppgtt, &ppgtt->pdp, &iter, &idx,
 923                                       cache_level);
 924 }
 925
 926 static void gen8_ppgtt_insert_4lvl(struct i915_address_space *vm,
 927                                    struct sg_table *pages,
 928                                    u64 start,
 929                                    enum i915_cache_level cache_level,
 930                                    u32 unused)
 931 {
 932         struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
 933         struct sgt_dma iter = {
 934                 .sg = pages->sgl,
 935                 .dma = sg_dma_address(iter.sg),
 936                 .max = iter.dma + iter.sg->length,
 937         };
 938         struct i915_page_directory_pointer **pdps = ppgtt->pml4.pdps;
 939         struct gen8_insert_pte idx = gen8_insert_pte(start);
 940
 941         while (gen8_ppgtt_insert_pte_entries(ppgtt, pdps[idx.pml4e++], &iter,
 942                                              &idx, cache_level))
 943                 GEM_BUG_ON(idx.pml4e >= GEN8_PML4ES_PER_PML4);
 944 }
 945
 946 static void gen8_free_page_tables(struct i915_address_space *vm,
 947                                   struct i915_page_directory *pd)
 948 {
 949         int i;
 950
 951         if (!px_page(pd))
 952                 return;
 953
 954         for (i = 0; i < I915_PDES; i++) {
 955                 if (pd->page_table[i] != vm->scratch_pt)
 956                         free_pt(vm, pd->page_table[i]);
 957         }
 958 }
 959
 960 static int gen8_init_scratch(struct i915_address_space *vm)
 961 {
 962         int ret;
 963
 964         ret = setup_scratch_page(vm, I915_GFP_DMA);
 965         if (ret)
 966                 return ret;
 967
 968         vm->scratch_pt = alloc_pt(vm);
 969         if (IS_ERR(vm->scratch_pt)) {
 970                 ret = PTR_ERR(vm->scratch_pt);
 971                 goto free_scratch_page;
 972         }
 973
 974         vm->scratch_pd = alloc_pd(vm);
 975         if (IS_ERR(vm->scratch_pd)) {
 976                 ret = PTR_ERR(vm->scratch_pd);
 977                 goto free_pt;
 978         }
 979
 980         if (use_4lvl(vm)) {
 981                 vm->scratch_pdp = alloc_pdp(vm);
 982                 if (IS_ERR(vm->scratch_pdp)) {
 983                         ret = PTR_ERR(vm->scratch_pdp);
 984                         goto free_pd;
 985                 }
 986         }
 987
 988         gen8_initialize_pt(vm, vm->scratch_pt);
 989         gen8_initialize_pd(vm, vm->scratch_pd);
 990         if (use_4lvl(vm))
 991                 gen8_initialize_pdp(vm, vm->scratch_pdp);
 992
 993         return 0;
 994
 995 free_pd:
 996         free_pd(vm, vm->scratch_pd);
 997 free_pt:
 998         free_pt(vm, vm->scratch_pt);
 999 free_scratch_page:
1000         cleanup_scratch_page(vm);
1001
1002         return ret;
1003 }
1004
1005 static int gen8_ppgtt_notify_vgt(struct i915_hw_ppgtt *ppgtt, bool create)
1006 {
1007         struct i915_address_space *vm = &ppgtt->base;
1008         struct drm_i915_private *dev_priv = vm->i915;
1009         enum vgt_g2v_type msg;
1010         int i;
1011
1012         if (use_4lvl(vm)) {
1013                 const u64 daddr = px_dma(&ppgtt->pml4);
1014
1015                 I915_WRITE(vgtif_reg(pdp[0].lo), lower_32_bits(daddr));
1016                 I915_WRITE(vgtif_reg(pdp[0].hi), upper_32_bits(daddr));
1017
1018                 msg = (create ? VGT_G2V_PPGTT_L4_PAGE_TABLE_CREATE :
1019                                 VGT_G2V_PPGTT_L4_PAGE_TABLE_DESTROY);
1020         } else {
1021                 for (i = 0; i < GEN8_3LVL_PDPES; i++) {
1022                         const u64 daddr = i915_page_dir_dma_addr(ppgtt, i);
1023
1024                         I915_WRITE(vgtif_reg(pdp[i].lo), lower_32_bits(daddr));
1025                         I915_WRITE(vgtif_reg(pdp[i].hi), upper_32_bits(daddr));
1026                 }
1027
1028                 msg = (create ? VGT_G2V_PPGTT_L3_PAGE_TABLE_CREATE :
1029                                 VGT_G2V_PPGTT_L3_PAGE_TABLE_DESTROY);
1030         }
1031
1032         I915_WRITE(vgtif_reg(g2v_notify), msg);
1033
1034         return 0;
1035 }
1036
1037 static void gen8_free_scratch(struct i915_address_space *vm)
1038 {
1039         if (use_4lvl(vm))
1040                 free_pdp(vm, vm->scratch_pdp);
1041         free_pd(vm, vm->scratch_pd);
1042         free_pt(vm, vm->scratch_pt);
1043         cleanup_scratch_page(vm);
1044 }
1045
1046 static void gen8_ppgtt_cleanup_3lvl(struct i915_address_space *vm,
1047                                     struct i915_page_directory_pointer *pdp)
1048 {
1049         const unsigned int pdpes = i915_pdpes_per_pdp(vm);
1050         int i;
1051
1052         for (i = 0; i < pdpes; i++) {
1053                 if (pdp->page_directory[i] == vm->scratch_pd)
1054                         continue;
1055
1056                 gen8_free_page_tables(vm, pdp->page_directory[i]);
1057                 free_pd(vm, pdp->page_directory[i]);
1058         }
1059
1060         free_pdp(vm, pdp);
1061 }
1062
1063 static void gen8_ppgtt_cleanup_4lvl(struct i915_hw_ppgtt *ppgtt)
1064 {
1065         int i;
1066
1067         for (i = 0; i < GEN8_PML4ES_PER_PML4; i++) {
1068                 if (ppgtt->pml4.pdps[i] == ppgtt->base.scratch_pdp)
1069                         continue;
1070
1071                 gen8_ppgtt_cleanup_3lvl(&ppgtt->base, ppgtt->pml4.pdps[i]);
1072         }
1073
1074         cleanup_px(&ppgtt->base, &ppgtt->pml4);
1075 }
1076
1077 static void gen8_ppgtt_cleanup(struct i915_address_space *vm)
1078 {
1079         struct drm_i915_private *dev_priv = vm->i915;
1080         struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
1081
1082         if (intel_vgpu_active(dev_priv))
1083                 gen8_ppgtt_notify_vgt(ppgtt, false);
1084
1085         if (use_4lvl(vm))
1086                 gen8_ppgtt_cleanup_4lvl(ppgtt);
1087         else
1088                 gen8_ppgtt_cleanup_3lvl(&ppgtt->base, &ppgtt->pdp);
1089
1090         gen8_free_scratch(vm);
1091 }
1092
1093 static int gen8_ppgtt_alloc_pd(struct i915_address_space *vm,
1094                                struct i915_page_directory *pd,
1095                                u64 start, u64 length)
1096 {
1097         struct i915_page_table *pt;
1098         u64 from = start;
1099         unsigned int pde;
1100
1101         gen8_for_each_pde(pt, pd, start, length, pde) {
1102                 if (pt == vm->scratch_pt) {
1103                         pt = alloc_pt(vm);
1104                         if (IS_ERR(pt))
1105                                 goto unwind;
1106
1107                         gen8_initialize_pt(vm, pt);
1108
1109                         gen8_ppgtt_set_pde(vm, pd, pt, pde);
1110                         pd->used_pdes++;
1111                         GEM_BUG_ON(pd->used_pdes > I915_PDES);
1112                 }
1113
1114                 pt->used_ptes += gen8_pte_count(start, length);
1115         }
1116         return 0;
1117
1118 unwind:
1119         gen8_ppgtt_clear_pd(vm, pd, from, start - from);
1120         return -ENOMEM;
1121 }
1122
1123 static int gen8_ppgtt_alloc_pdp(struct i915_address_space *vm,
1124                                 struct i915_page_directory_pointer *pdp,
1125                                 u64 start, u64 length)
1126 {
1127         struct i915_page_directory *pd;
1128         u64 from = start;
1129         unsigned int pdpe;
1130         int ret;
1131
1132         gen8_for_each_pdpe(pd, pdp, start, length, pdpe) {
1133                 if (pd == vm->scratch_pd) {
1134                         pd = alloc_pd(vm);
1135                         if (IS_ERR(pd))
1136                                 goto unwind;
1137
1138                         gen8_initialize_pd(vm, pd);
1139                         gen8_ppgtt_set_pdpe(vm, pdp, pd, pdpe);
1140                         pdp->used_pdpes++;
1141                         GEM_BUG_ON(pdp->used_pdpes > i915_pdpes_per_pdp(vm));
1142
1143                         mark_tlbs_dirty(i915_vm_to_ppgtt(vm));
1144                 }
1145
1146                 ret = gen8_ppgtt_alloc_pd(vm, pd, start, length);
1147                 if (unlikely(ret))
1148                         goto unwind_pd;
1149         }
1150
1151         return 0;
1152
1153 unwind_pd:
1154         if (!pd->used_pdes) {
1155                 gen8_ppgtt_set_pdpe(vm, pdp, vm->scratch_pd, pdpe);
1156                 GEM_BUG_ON(!pdp->used_pdpes);
1157                 pdp->used_pdpes--;
1158                 free_pd(vm, pd);
1159         }
1160 unwind:
1161         gen8_ppgtt_clear_pdp(vm, pdp, from, start - from);
1162         return -ENOMEM;
1163 }
1164
1165 static int gen8_ppgtt_alloc_3lvl(struct i915_address_space *vm,
1166                                  u64 start, u64 length)
1167 {
1168         return gen8_ppgtt_alloc_pdp(vm,
1169                                     &i915_vm_to_ppgtt(vm)->pdp, start, length);
1170 }
1171
1172 static int gen8_ppgtt_alloc_4lvl(struct i915_address_space *vm,
1173                                  u64 start, u64 length)
1174 {
1175         struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
1176         struct i915_pml4 *pml4 = &ppgtt->pml4;
1177         struct i915_page_directory_pointer *pdp;
1178         u64 from = start;
1179         u32 pml4e;
1180         int ret;
1181
1182         gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) {
1183                 if (pml4->pdps[pml4e] == vm->scratch_pdp) {
1184                         pdp = alloc_pdp(vm);
1185                         if (IS_ERR(pdp))
1186                                 goto unwind;
1187
1188                         gen8_initialize_pdp(vm, pdp);
1189                         gen8_ppgtt_set_pml4e(pml4, pdp, pml4e);
1190                 }
1191
1192                 ret = gen8_ppgtt_alloc_pdp(vm, pdp, start, length);
1193                 if (unlikely(ret))
1194                         goto unwind_pdp;
1195         }
1196
1197         return 0;
1198
1199 unwind_pdp:
1200         if (!pdp->used_pdpes) {
1201                 gen8_ppgtt_set_pml4e(pml4, vm->scratch_pdp, pml4e);
1202                 free_pdp(vm, pdp);
1203         }
1204 unwind:
1205         gen8_ppgtt_clear_4lvl(vm, from, start - from);
1206         return -ENOMEM;
1207 }
1208
1209 static void gen8_dump_pdp(struct i915_hw_ppgtt *ppgtt,
1210                           struct i915_page_directory_pointer *pdp,
1211                           u64 start, u64 length,
1212                           gen8_pte_t scratch_pte,
1213                           struct seq_file *m)
1214 {
1215         struct i915_address_space *vm = &ppgtt->base;
1216         struct i915_page_directory *pd;
1217         u32 pdpe;
1218
1219         gen8_for_each_pdpe(pd, pdp, start, length, pdpe) {
1220                 struct i915_page_table *pt;
1221                 u64 pd_len = length;
1222                 u64 pd_start = start;
1223                 u32 pde;
1224
1225                 if (pdp->page_directory[pdpe] == ppgtt->base.scratch_pd)
1226                         continue;
1227
1228                 seq_printf(m, "\tPDPE #%d\n", pdpe);
1229                 gen8_for_each_pde(pt, pd, pd_start, pd_len, pde) {
1230                         u32 pte;
1231                         gen8_pte_t *pt_vaddr;
1232
1233                         if (pd->page_table[pde] == ppgtt->base.scratch_pt)
1234                                 continue;
1235
1236                         pt_vaddr = kmap_atomic_px(pt);
1237                         for (pte = 0; pte < GEN8_PTES; pte += 4) {
1238                                 u64 va = (pdpe << GEN8_PDPE_SHIFT |
1239                                           pde << GEN8_PDE_SHIFT |
1240                                           pte << GEN8_PTE_SHIFT);
1241                                 int i;
1242                                 bool found = false;
1243
1244                                 for (i = 0; i < 4; i++)
1245                                         if (pt_vaddr[pte + i] != scratch_pte)
1246                                                 found = true;
1247                                 if (!found)
1248                                         continue;
1249
1250                                 seq_printf(m, "\t\t0x%llx [%03d,%03d,%04d]: =", va, pdpe, pde, pte);
1251                                 for (i = 0; i < 4; i++) {
1252                                         if (pt_vaddr[pte + i] != scratch_pte)
1253                                                 seq_printf(m, " %llx", pt_vaddr[pte + i]);
1254                                         else
1255                                                 seq_puts(m, "  SCRATCH ");
1256                                 }
1257                                 seq_puts(m, "\n");
1258                         }
1259                         kunmap_atomic(pt_vaddr);
1260                 }
1261         }
1262 }
1263
1264 static void gen8_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m)
1265 {
1266         struct i915_address_space *vm = &ppgtt->base;
1267         const gen8_pte_t scratch_pte =
1268                 gen8_pte_encode(vm->scratch_page.daddr, I915_CACHE_LLC);
1269         u64 start = 0, length = ppgtt->base.total;
1270
1271         if (use_4lvl(vm)) {
1272                 u64 pml4e;
1273                 struct i915_pml4 *pml4 = &ppgtt->pml4;
1274                 struct i915_page_directory_pointer *pdp;
1275
1276                 gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) {
1277                         if (pml4->pdps[pml4e] == ppgtt->base.scratch_pdp)
1278                                 continue;
1279
1280                         seq_printf(m, "    PML4E #%llu\n", pml4e);
1281                         gen8_dump_pdp(ppgtt, pdp, start, length, scratch_pte, m);
1282                 }
1283         } else {
1284                 gen8_dump_pdp(ppgtt, &ppgtt->pdp, start, length, scratch_pte, m);
1285         }
1286 }
1287
1288 static int gen8_preallocate_top_level_pdp(struct i915_hw_ppgtt *ppgtt)
1289 {
1290         struct i915_address_space *vm = &ppgtt->base;
1291         struct i915_page_directory_pointer *pdp = &ppgtt->pdp;
1292         struct i915_page_directory *pd;
1293         u64 start = 0, length = ppgtt->base.total;
1294         u64 from = start;
1295         unsigned int pdpe;
1296
1297         gen8_for_each_pdpe(pd, pdp, start, length, pdpe) {
1298                 pd = alloc_pd(vm);
1299                 if (IS_ERR(pd))
1300                         goto unwind;
1301
1302                 gen8_initialize_pd(vm, pd);
1303                 gen8_ppgtt_set_pdpe(vm, pdp, pd, pdpe);
1304                 pdp->used_pdpes++;
1305         }
1306
1307         pdp->used_pdpes++; /* never remove */
1308         return 0;
1309
1310 unwind:
1311         start -= from;
1312         gen8_for_each_pdpe(pd, pdp, from, start, pdpe) {
1313                 gen8_ppgtt_set_pdpe(vm, pdp, vm->scratch_pd, pdpe);
1314                 free_pd(vm, pd);
1315         }
1316         pdp->used_pdpes = 0;
1317         return -ENOMEM;
1318 }
1319
1320 /*
1321  * GEN8 legacy ppgtt programming is accomplished through a max 4 PDP registers
1322  * with a net effect resembling a 2-level page table in normal x86 terms. Each
1323  * PDP represents 1GB of memory 4 * 512 * 512 * 4096 = 4GB legacy 32b address
1324  * space.
1325  *
1326  */
1327 static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
1328 {
1329         struct i915_address_space *vm = &ppgtt->base;
1330         struct drm_i915_private *dev_priv = vm->i915;
1331         int ret;
1332
1333         ppgtt->base.total = USES_FULL_48BIT_PPGTT(dev_priv) ?
1334                 1ULL << 48 :
1335                 1ULL << 32;
1336
1337         ret = gen8_init_scratch(&ppgtt->base);
1338         if (ret) {
1339                 ppgtt->base.total = 0;
1340                 return ret;
1341         }
1342
1343         /* There are only few exceptions for gen >=6. chv and bxt.
1344          * And we are not sure about the latter so play safe for now.
1345          */
1346         if (IS_CHERRYVIEW(dev_priv) || IS_BROXTON(dev_priv))
1347                 ppgtt->base.pt_kmap_wc = true;
1348
1349         if (use_4lvl(vm)) {
1350                 ret = setup_px(&ppgtt->base, &ppgtt->pml4);
1351                 if (ret)
1352                         goto free_scratch;
1353
1354                 gen8_initialize_pml4(&ppgtt->base, &ppgtt->pml4);
1355
1356                 ppgtt->switch_mm = gen8_mm_switch_4lvl;
1357                 ppgtt->base.allocate_va_range = gen8_ppgtt_alloc_4lvl;
1358                 ppgtt->base.insert_entries = gen8_ppgtt_insert_4lvl;
1359                 ppgtt->base.clear_range = gen8_ppgtt_clear_4lvl;
1360         } else {
1361                 ret = __pdp_init(&ppgtt->base, &ppgtt->pdp);
1362                 if (ret)
1363                         goto free_scratch;
1364
1365                 if (intel_vgpu_active(dev_priv)) {
1366                         ret = gen8_preallocate_top_level_pdp(ppgtt);
1367                         if (ret) {
1368                                 __pdp_fini(&ppgtt->pdp);
1369                                 goto free_scratch;
1370                         }
1371                 }
1372
1373                 ppgtt->switch_mm = gen8_mm_switch_3lvl;
1374                 ppgtt->base.allocate_va_range = gen8_ppgtt_alloc_3lvl;
1375                 ppgtt->base.insert_entries = gen8_ppgtt_insert_3lvl;
1376                 ppgtt->base.clear_range = gen8_ppgtt_clear_3lvl;
1377         }
1378
1379         if (intel_vgpu_active(dev_priv))
1380                 gen8_ppgtt_notify_vgt(ppgtt, true);
1381
1382         ppgtt->base.cleanup = gen8_ppgtt_cleanup;
1383         ppgtt->base.unbind_vma = ppgtt_unbind_vma;
1384         ppgtt->base.bind_vma = ppgtt_bind_vma;
1385         ppgtt->debug_dump = gen8_dump_ppgtt;
1386
1387         return 0;
1388
1389 free_scratch:
1390         gen8_free_scratch(&ppgtt->base);
1391         return ret;
1392 }
1393
1394 static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m)
1395 {
1396         struct i915_address_space *vm = &ppgtt->base;
1397         struct i915_page_table *unused;
1398         gen6_pte_t scratch_pte;
1399         u32 pd_entry, pte, pde;
1400         u32 start = 0, length = ppgtt->base.total;
1401
1402         scratch_pte = vm->pte_encode(vm->scratch_page.daddr,
1403                                      I915_CACHE_LLC, 0);
1404
1405         gen6_for_each_pde(unused, &ppgtt->pd, start, length, pde) {
1406                 u32 expected;
1407                 gen6_pte_t *pt_vaddr;
1408                 const dma_addr_t pt_addr = px_dma(ppgtt->pd.page_table[pde]);
1409                 pd_entry = readl(ppgtt->pd_addr + pde);
1410                 expected = (GEN6_PDE_ADDR_ENCODE(pt_addr) | GEN6_PDE_VALID);
1411
1412                 if (pd_entry != expected)
1413                         seq_printf(m, "\tPDE #%d mismatch: Actual PDE: %x Expected PDE: %x\n",
1414                                    pde,
1415                                    pd_entry,
1416                                    expected);
1417                 seq_printf(m, "\tPDE: %x\n", pd_entry);
1418
1419                 pt_vaddr = kmap_atomic_px(ppgtt->pd.page_table[pde]);
1420
1421                 for (pte = 0; pte < GEN6_PTES; pte+=4) {
1422                         unsigned long va =
1423                                 (pde * PAGE_SIZE * GEN6_PTES) +
1424                                 (pte * PAGE_SIZE);
1425                         int i;
1426                         bool found = false;
1427                         for (i = 0; i < 4; i++)
1428                                 if (pt_vaddr[pte + i] != scratch_pte)
1429                                         found = true;
1430                         if (!found)
1431                                 continue;
1432
1433                         seq_printf(m, "\t\t0x%lx [%03d,%04d]: =", va, pde, pte);
1434                         for (i = 0; i < 4; i++) {
1435                                 if (pt_vaddr[pte + i] != scratch_pte)
1436                                         seq_printf(m, " %08x", pt_vaddr[pte + i]);
1437                                 else
1438                                         seq_puts(m, "  SCRATCH ");
1439                         }
1440                         seq_puts(m, "\n");
1441                 }
1442                 kunmap_atomic(pt_vaddr);
1443         }
1444 }
1445
1446 /* Write pde (index) from the page directory @pd to the page table @pt */
1447 static inline void gen6_write_pde(const struct i915_hw_ppgtt *ppgtt,
1448                                   const unsigned int pde,
1449                                   const struct i915_page_table *pt)
1450 {
1451         /* Caller needs to make sure the write completes if necessary */
1452         writel_relaxed(GEN6_PDE_ADDR_ENCODE(px_dma(pt)) | GEN6_PDE_VALID,
1453                        ppgtt->pd_addr + pde);
1454 }
1455
1456 /* Write all the page tables found in the ppgtt structure to incrementing page
1457  * directories. */
1458 static void gen6_write_page_range(struct i915_hw_ppgtt *ppgtt,
1459                                   u32 start, u32 length)
1460 {
1461         struct i915_page_table *pt;
1462         unsigned int pde;
1463
1464         gen6_for_each_pde(pt, &ppgtt->pd, start, length, pde)
1465                 gen6_write_pde(ppgtt, pde, pt);
1466
1467         mark_tlbs_dirty(ppgtt);
1468         wmb();
1469 }
1470
1471 static inline u32 get_pd_offset(struct i915_hw_ppgtt *ppgtt)
1472 {
1473         GEM_BUG_ON(ppgtt->pd.base.ggtt_offset & 0x3f);
1474         return ppgtt->pd.base.ggtt_offset << 10;
1475 }
1476
1477 static int hsw_mm_switch(struct i915_hw_ppgtt *ppgtt,
1478                          struct drm_i915_gem_request *req)
1479 {
1480         struct intel_engine_cs *engine = req->engine;
1481         u32 *cs;
1482
1483         /* NB: TLBs must be flushed and invalidated before a switch */
1484         cs = intel_ring_begin(req, 6);
1485         if (IS_ERR(cs))
1486                 return PTR_ERR(cs);
1487
1488         *cs++ = MI_LOAD_REGISTER_IMM(2);
1489         *cs++ = i915_mmio_reg_offset(RING_PP_DIR_DCLV(engine));
1490         *cs++ = PP_DIR_DCLV_2G;
1491         *cs++ = i915_mmio_reg_offset(RING_PP_DIR_BASE(engine));
1492         *cs++ = get_pd_offset(ppgtt);
1493         *cs++ = MI_NOOP;
1494         intel_ring_advance(req, cs);
1495
1496         return 0;
1497 }
1498
1499 static int gen7_mm_switch(struct i915_hw_ppgtt *ppgtt,
1500                           struct drm_i915_gem_request *req)
1501 {
1502         struct intel_engine_cs *engine = req->engine;
1503         u32 *cs;
1504
1505         /* NB: TLBs must be flushed and invalidated before a switch */
1506         cs = intel_ring_begin(req, 6);
1507         if (IS_ERR(cs))
1508                 return PTR_ERR(cs);
1509
1510         *cs++ = MI_LOAD_REGISTER_IMM(2);
1511         *cs++ = i915_mmio_reg_offset(RING_PP_DIR_DCLV(engine));
1512         *cs++ = PP_DIR_DCLV_2G;
1513         *cs++ = i915_mmio_reg_offset(RING_PP_DIR_BASE(engine));
1514         *cs++ = get_pd_offset(ppgtt);
1515         *cs++ = MI_NOOP;
1516         intel_ring_advance(req, cs);
1517
1518         return 0;
1519 }
1520
1521 static int gen6_mm_switch(struct i915_hw_ppgtt *ppgtt,
1522                           struct drm_i915_gem_request *req)
1523 {
1524         struct intel_engine_cs *engine = req->engine;
1525         struct drm_i915_private *dev_priv = req->i915;
1526
1527         I915_WRITE(RING_PP_DIR_DCLV(engine), PP_DIR_DCLV_2G);
1528         I915_WRITE(RING_PP_DIR_BASE(engine), get_pd_offset(ppgtt));
1529         return 0;
1530 }
1531
1532 static void gen8_ppgtt_enable(struct drm_i915_private *dev_priv)
1533 {
1534         struct intel_engine_cs *engine;
1535         enum intel_engine_id id;
1536
1537         for_each_engine(engine, dev_priv, id) {
1538                 u32 four_level = USES_FULL_48BIT_PPGTT(dev_priv) ?
1539                                  GEN8_GFX_PPGTT_48B : 0;
1540                 I915_WRITE(RING_MODE_GEN7(engine),
1541                            _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE | four_level));
1542         }
1543 }
1544
1545 static void gen7_ppgtt_enable(struct drm_i915_private *dev_priv)
1546 {
1547         struct intel_engine_cs *engine;
1548         u32 ecochk, ecobits;
1549         enum intel_engine_id id;
1550
1551         ecobits = I915_READ(GAC_ECO_BITS);
1552         I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B);
1553
1554         ecochk = I915_READ(GAM_ECOCHK);
1555         if (IS_HASWELL(dev_priv)) {
1556                 ecochk |= ECOCHK_PPGTT_WB_HSW;
1557         } else {
1558                 ecochk |= ECOCHK_PPGTT_LLC_IVB;
1559                 ecochk &= ~ECOCHK_PPGTT_GFDT_IVB;
1560         }
1561         I915_WRITE(GAM_ECOCHK, ecochk);
1562
1563         for_each_engine(engine, dev_priv, id) {
1564                 /* GFX_MODE is per-ring on gen7+ */
1565                 I915_WRITE(RING_MODE_GEN7(engine),
1566                            _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
1567         }
1568 }
1569
1570 static void gen6_ppgtt_enable(struct drm_i915_private *dev_priv)
1571 {
1572         u32 ecochk, gab_ctl, ecobits;
1573
1574         ecobits = I915_READ(GAC_ECO_BITS);
1575         I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_SNB_BIT |
1576                    ECOBITS_PPGTT_CACHE64B);
1577
1578         gab_ctl = I915_READ(GAB_CTL);
1579         I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT);
1580
1581         ecochk = I915_READ(GAM_ECOCHK);
1582         I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT | ECOCHK_PPGTT_CACHE64B);
1583
1584         I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
1585 }
1586
1587 /* PPGTT support for Sandybdrige/Gen6 and later */
1588 static void gen6_ppgtt_clear_range(struct i915_address_space *vm,
1589                                    u64 start, u64 length)
1590 {
1591         struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
1592         unsigned int first_entry = start >> PAGE_SHIFT;
1593         unsigned int pde = first_entry / GEN6_PTES;
1594         unsigned int pte = first_entry % GEN6_PTES;
1595         unsigned int num_entries = length >> PAGE_SHIFT;
1596         gen6_pte_t scratch_pte =
1597                 vm->pte_encode(vm->scratch_page.daddr, I915_CACHE_LLC, 0);
1598
1599         while (num_entries) {
1600                 struct i915_page_table *pt = ppgtt->pd.page_table[pde++];
1601                 unsigned int end = min(pte + num_entries, GEN6_PTES);
1602                 gen6_pte_t *vaddr;
1603
1604                 num_entries -= end - pte;
1605
1606                 /* Note that the hw doesn't support removing PDE on the fly
1607                  * (they are cached inside the context with no means to
1608                  * invalidate the cache), so we can only reset the PTE
1609                  * entries back to scratch.
1610                  */
1611
1612                 vaddr = kmap_atomic_px(pt);
1613                 do {
1614                         vaddr[pte++] = scratch_pte;
1615                 } while (pte < end);
1616                 kunmap_atomic(vaddr);
1617
1618                 pte = 0;
1619         }
1620 }
1621
1622 static void gen6_ppgtt_insert_entries(struct i915_address_space *vm,
1623                                       struct sg_table *pages,
1624                                       u64 start,
1625                                       enum i915_cache_level cache_level,
1626                                       u32 flags)
1627 {
1628         struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
1629         unsigned first_entry = start >> PAGE_SHIFT;
1630         unsigned act_pt = first_entry / GEN6_PTES;
1631         unsigned act_pte = first_entry % GEN6_PTES;
1632         const u32 pte_encode = vm->pte_encode(0, cache_level, flags);
1633         struct sgt_dma iter;
1634         gen6_pte_t *vaddr;
1635
1636         vaddr = kmap_atomic_px(ppgtt->pd.page_table[act_pt]);
1637         iter.sg = pages->sgl;
1638         iter.dma = sg_dma_address(iter.sg);
1639         iter.max = iter.dma + iter.sg->length;
1640         do {
1641                 vaddr[act_pte] = pte_encode | GEN6_PTE_ADDR_ENCODE(iter.dma);
1642
1643                 iter.dma += PAGE_SIZE;
1644                 if (iter.dma == iter.max) {
1645                         iter.sg = __sg_next(iter.sg);
1646                         if (!iter.sg)
1647                                 break;
1648
1649                         iter.dma = sg_dma_address(iter.sg);
1650                         iter.max = iter.dma + iter.sg->length;
1651                 }
1652
1653                 if (++act_pte == GEN6_PTES) {
1654                         kunmap_atomic(vaddr);
1655                         vaddr = kmap_atomic_px(ppgtt->pd.page_table[++act_pt]);
1656                         act_pte = 0;
1657                 }
1658         } while (1);
1659         kunmap_atomic(vaddr);
1660 }
1661
1662 static int gen6_alloc_va_range(struct i915_address_space *vm,
1663                                u64 start, u64 length)
1664 {
1665         struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
1666         struct i915_page_table *pt;
1667         u64 from = start;
1668         unsigned int pde;
1669         bool flush = false;
1670
1671         gen6_for_each_pde(pt, &ppgtt->pd, start, length, pde) {
1672                 if (pt == vm->scratch_pt) {
1673                         pt = alloc_pt(vm);
1674                         if (IS_ERR(pt))
1675                                 goto unwind_out;
1676
1677                         gen6_initialize_pt(vm, pt);
1678                         ppgtt->pd.page_table[pde] = pt;
1679                         gen6_write_pde(ppgtt, pde, pt);
1680                         flush = true;
1681                 }
1682         }
1683
1684         if (flush) {
1685                 mark_tlbs_dirty(ppgtt);
1686                 wmb();
1687         }
1688
1689         return 0;
1690
1691 unwind_out:
1692         gen6_ppgtt_clear_range(vm, from, start);
1693         return -ENOMEM;
1694 }
1695
1696 static int gen6_init_scratch(struct i915_address_space *vm)
1697 {
1698         int ret;
1699
1700         ret = setup_scratch_page(vm, I915_GFP_DMA);
1701         if (ret)
1702                 return ret;
1703
1704         vm->scratch_pt = alloc_pt(vm);
1705         if (IS_ERR(vm->scratch_pt)) {
1706                 cleanup_scratch_page(vm);
1707                 return PTR_ERR(vm->scratch_pt);
1708         }
1709
1710         gen6_initialize_pt(vm, vm->scratch_pt);
1711
1712         return 0;
1713 }
1714
1715 static void gen6_free_scratch(struct i915_address_space *vm)
1716 {
1717         free_pt(vm, vm->scratch_pt);
1718         cleanup_scratch_page(vm);
1719 }
1720
1721 static void gen6_ppgtt_cleanup(struct i915_address_space *vm)
1722 {
1723         struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
1724         struct i915_page_directory *pd = &ppgtt->pd;
1725         struct i915_page_table *pt;
1726         u32 pde;
1727
1728         drm_mm_remove_node(&ppgtt->node);
1729
1730         gen6_for_all_pdes(pt, pd, pde)
1731                 if (pt != vm->scratch_pt)
1732                         free_pt(vm, pt);
1733
1734         gen6_free_scratch(vm);
1735 }
1736
1737 static int gen6_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt)
1738 {
1739         struct i915_address_space *vm = &ppgtt->base;
1740         struct drm_i915_private *dev_priv = ppgtt->base.i915;
1741         struct i915_ggtt *ggtt = &dev_priv->ggtt;
1742         int ret;
1743
1744         /* PPGTT PDEs reside in the GGTT and consists of 512 entries. The
1745          * allocator works in address space sizes, so it's multiplied by page
1746          * size. We allocate at the top of the GTT to avoid fragmentation.
1747          */
1748         BUG_ON(!drm_mm_initialized(&ggtt->base.mm));
1749
1750         ret = gen6_init_scratch(vm);
1751         if (ret)
1752                 return ret;
1753
1754         ret = i915_gem_gtt_insert(&ggtt->base, &ppgtt->node,
1755                                   GEN6_PD_SIZE, GEN6_PD_ALIGN,
1756                                   I915_COLOR_UNEVICTABLE,
1757                                   0, ggtt->base.total,
1758                                   PIN_HIGH);
1759         if (ret)
1760                 goto err_out;
1761
1762         if (ppgtt->node.start < ggtt->mappable_end)
1763                 DRM_DEBUG("Forced to use aperture for PDEs\n");
1764
1765         ppgtt->pd.base.ggtt_offset =
1766                 ppgtt->node.start / PAGE_SIZE * sizeof(gen6_pte_t);
1767
1768         ppgtt->pd_addr = (gen6_pte_t __iomem *)ggtt->gsm +
1769                 ppgtt->pd.base.ggtt_offset / sizeof(gen6_pte_t);
1770
1771         return 0;
1772
1773 err_out:
1774         gen6_free_scratch(vm);
1775         return ret;
1776 }
1777
1778 static int gen6_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt)
1779 {
1780         return gen6_ppgtt_allocate_page_directories(ppgtt);
1781 }
1782
1783 static void gen6_scratch_va_range(struct i915_hw_ppgtt *ppgtt,
1784                                   u64 start, u64 length)
1785 {
1786         struct i915_page_table *unused;
1787         u32 pde;
1788
1789         gen6_for_each_pde(unused, &ppgtt->pd, start, length, pde)
1790                 ppgtt->pd.page_table[pde] = ppgtt->base.scratch_pt;
1791 }
1792
1793 static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
1794 {
1795         struct drm_i915_private *dev_priv = ppgtt->base.i915;
1796         struct i915_ggtt *ggtt = &dev_priv->ggtt;
1797         int ret;
1798
1799         ppgtt->base.pte_encode = ggtt->base.pte_encode;
1800         if (intel_vgpu_active(dev_priv) || IS_GEN6(dev_priv))
1801                 ppgtt->switch_mm = gen6_mm_switch;
1802         else if (IS_HASWELL(dev_priv))
1803                 ppgtt->switch_mm = hsw_mm_switch;
1804         else if (IS_GEN7(dev_priv))
1805                 ppgtt->switch_mm = gen7_mm_switch;
1806         else
1807                 BUG();
1808
1809         ret = gen6_ppgtt_alloc(ppgtt);
1810         if (ret)
1811                 return ret;
1812
1813         ppgtt->base.total = I915_PDES * GEN6_PTES * PAGE_SIZE;
1814
1815         gen6_scratch_va_range(ppgtt, 0, ppgtt->base.total);
1816         gen6_write_page_range(ppgtt, 0, ppgtt->base.total);
1817
1818         ret = gen6_alloc_va_range(&ppgtt->base, 0, ppgtt->base.total);
1819         if (ret) {
1820                 gen6_ppgtt_cleanup(&ppgtt->base);
1821                 return ret;
1822         }
1823
1824         ppgtt->base.clear_range = gen6_ppgtt_clear_range;
1825         ppgtt->base.insert_entries = gen6_ppgtt_insert_entries;
1826         ppgtt->base.unbind_vma = ppgtt_unbind_vma;
1827         ppgtt->base.bind_vma = ppgtt_bind_vma;
1828         ppgtt->base.cleanup = gen6_ppgtt_cleanup;
1829         ppgtt->debug_dump = gen6_dump_ppgtt;
1830
1831         DRM_DEBUG_DRIVER("Allocated pde space (%lldM) at GTT entry: %llx\n",
1832                          ppgtt->node.size >> 20,
1833                          ppgtt->node.start / PAGE_SIZE);
1834
1835         DRM_DEBUG_DRIVER("Adding PPGTT at offset %x\n",
1836                          ppgtt->pd.base.ggtt_offset << 10);
1837
1838         return 0;
1839 }
1840
1841 static int __hw_ppgtt_init(struct i915_hw_ppgtt *ppgtt,
1842                            struct drm_i915_private *dev_priv)
1843 {
1844         ppgtt->base.i915 = dev_priv;
1845         ppgtt->base.dma = &dev_priv->drm.pdev->dev;
1846
1847         if (INTEL_INFO(dev_priv)->gen < 8)
1848                 return gen6_ppgtt_init(ppgtt);
1849         else
1850                 return gen8_ppgtt_init(ppgtt);
1851 }
1852
1853 static void i915_address_space_init(struct i915_address_space *vm,
1854                                     struct drm_i915_private *dev_priv,
1855                                     const char *name)
1856 {
1857         i915_gem_timeline_init(dev_priv, &vm->timeline, name);
1858
1859         drm_mm_init(&vm->mm, 0, vm->total);
1860         vm->mm.head_node.color = I915_COLOR_UNEVICTABLE;
1861
1862         INIT_LIST_HEAD(&vm->active_list);
1863         INIT_LIST_HEAD(&vm->inactive_list);
1864         INIT_LIST_HEAD(&vm->unbound_list);
1865
1866         list_add_tail(&vm->global_link, &dev_priv->vm_list);
1867         pagevec_init(&vm->free_pages, false);
1868 }
1869
1870 static void i915_address_space_fini(struct i915_address_space *vm)
1871 {
1872         if (pagevec_count(&vm->free_pages))
1873                 vm_free_pages_release(vm);
1874
1875         i915_gem_timeline_fini(&vm->timeline);
1876         drm_mm_takedown(&vm->mm);
1877         list_del(&vm->global_link);
1878 }
1879
1880 static void gtt_write_workarounds(struct drm_i915_private *dev_priv)
1881 {
1882         /* This function is for gtt related workarounds. This function is
1883          * called on driver load and after a GPU reset, so you can place
1884          * workarounds here even if they get overwritten by GPU reset.
1885          */
1886         /* WaIncreaseDefaultTLBEntries:chv,bdw,skl,bxt,kbl,glk */
1887         if (IS_BROADWELL(dev_priv))
1888                 I915_WRITE(GEN8_L3_LRA_1_GPGPU, GEN8_L3_LRA_1_GPGPU_DEFAULT_VALUE_BDW);
1889         else if (IS_CHERRYVIEW(dev_priv))
1890                 I915_WRITE(GEN8_L3_LRA_1_GPGPU, GEN8_L3_LRA_1_GPGPU_DEFAULT_VALUE_CHV);
1891         else if (IS_GEN9_BC(dev_priv))
1892                 I915_WRITE(GEN8_L3_LRA_1_GPGPU, GEN9_L3_LRA_1_GPGPU_DEFAULT_VALUE_SKL);
1893         else if (IS_GEN9_LP(dev_priv))
1894                 I915_WRITE(GEN8_L3_LRA_1_GPGPU, GEN9_L3_LRA_1_GPGPU_DEFAULT_VALUE_BXT);
1895 }
1896
1897 int i915_ppgtt_init_hw(struct drm_i915_private *dev_priv)
1898 {
1899         gtt_write_workarounds(dev_priv);
1900
1901         /* In the case of execlists, PPGTT is enabled by the context descriptor
1902          * and the PDPs are contained within the context itself.  We don't
1903          * need to do anything here. */
1904         if (i915.enable_execlists)
1905                 return 0;
1906
1907         if (!USES_PPGTT(dev_priv))
1908                 return 0;
1909
1910         if (IS_GEN6(dev_priv))
1911                 gen6_ppgtt_enable(dev_priv);
1912         else if (IS_GEN7(dev_priv))
1913                 gen7_ppgtt_enable(dev_priv);
1914         else if (INTEL_GEN(dev_priv) >= 8)
1915                 gen8_ppgtt_enable(dev_priv);
1916         else
1917                 MISSING_CASE(INTEL_GEN(dev_priv));
1918
1919         return 0;
1920 }
1921
1922 struct i915_hw_ppgtt *
1923 i915_ppgtt_create(struct drm_i915_private *dev_priv,
1924                   struct drm_i915_file_private *fpriv,
1925                   const char *name)
1926 {
1927         struct i915_hw_ppgtt *ppgtt;
1928         int ret;
1929
1930         ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
1931         if (!ppgtt)
1932                 return ERR_PTR(-ENOMEM);
1933
1934         ret = __hw_ppgtt_init(ppgtt, dev_priv);
1935         if (ret) {
1936                 kfree(ppgtt);
1937                 return ERR_PTR(ret);
1938         }
1939
1940         kref_init(&ppgtt->ref);
1941         i915_address_space_init(&ppgtt->base, dev_priv, name);
1942         ppgtt->base.file = fpriv;
1943
1944         trace_i915_ppgtt_create(&ppgtt->base);
1945
1946         return ppgtt;
1947 }
1948
1949 void i915_ppgtt_close(struct i915_address_space *vm)
1950 {
1951         struct list_head *phases[] = {
1952                 &vm->active_list,
1953                 &vm->inactive_list,
1954                 &vm->unbound_list,
1955                 NULL,
1956         }, **phase;
1957
1958         GEM_BUG_ON(vm->closed);
1959         vm->closed = true;
1960
1961         for (phase = phases; *phase; phase++) {
1962                 struct i915_vma *vma, *vn;
1963
1964                 list_for_each_entry_safe(vma, vn, *phase, vm_link)
1965                         if (!i915_vma_is_closed(vma))
1966                                 i915_vma_close(vma);
1967         }
1968 }
1969
1970 void i915_ppgtt_release(struct kref *kref)
1971 {
1972         struct i915_hw_ppgtt *ppgtt =
1973                 container_of(kref, struct i915_hw_ppgtt, ref);
1974
1975         trace_i915_ppgtt_release(&ppgtt->base);
1976
1977         /* vmas should already be unbound and destroyed */
1978         WARN_ON(!list_empty(&ppgtt->base.active_list));
1979         WARN_ON(!list_empty(&ppgtt->base.inactive_list));
1980         WARN_ON(!list_empty(&ppgtt->base.unbound_list));
1981
1982         ppgtt->base.cleanup(&ppgtt->base);
1983         i915_address_space_fini(&ppgtt->base);
1984         kfree(ppgtt);
1985 }
1986
1987 /* Certain Gen5 chipsets require require idling the GPU before
1988  * unmapping anything from the GTT when VT-d is enabled.
1989  */
1990 static bool needs_idle_maps(struct drm_i915_private *dev_priv)
1991 {
1992 #ifdef CONFIG_INTEL_IOMMU
1993         /* Query intel_iommu to see if we need the workaround. Presumably that
1994          * was loaded first.
1995          */
1996         if (IS_GEN5(dev_priv) && IS_MOBILE(dev_priv) && intel_iommu_gfx_mapped)
1997                 return true;
1998 #endif
1999         return false;
2000 }
2001
2002 void i915_check_and_clear_faults(struct drm_i915_private *dev_priv)
2003 {
2004         struct intel_engine_cs *engine;
2005         enum intel_engine_id id;
2006
2007         if (INTEL_INFO(dev_priv)->gen < 6)
2008                 return;
2009
2010         for_each_engine(engine, dev_priv, id) {
2011                 u32 fault_reg;
2012                 fault_reg = I915_READ(RING_FAULT_REG(engine));
2013                 if (fault_reg & RING_FAULT_VALID) {
2014                         DRM_DEBUG_DRIVER("Unexpected fault\n"
2015                                          "\tAddr: 0x%08lx\n"
2016                                          "\tAddress space: %s\n"
2017                                          "\tSource ID: %d\n"
2018                                          "\tType: %d\n",
2019                                          fault_reg & PAGE_MASK,
2020                                          fault_reg & RING_FAULT_GTTSEL_MASK ? "GGTT" : "PPGTT",
2021                                          RING_FAULT_SRCID(fault_reg),
2022                                          RING_FAULT_FAULT_TYPE(fault_reg));
2023                         I915_WRITE(RING_FAULT_REG(engine),
2024                                    fault_reg & ~RING_FAULT_VALID);
2025                 }
2026         }
2027
2028         /* Engine specific init may not have been done till this point. */
2029         if (dev_priv->engine[RCS])
2030                 POSTING_READ(RING_FAULT_REG(dev_priv->engine[RCS]));
2031 }
2032
2033 void i915_gem_suspend_gtt_mappings(struct drm_i915_private *dev_priv)
2034 {
2035         struct i915_ggtt *ggtt = &dev_priv->ggtt;
2036
2037         /* Don't bother messing with faults pre GEN6 as we have little
2038          * documentation supporting that it's a good idea.
2039          */
2040         if (INTEL_GEN(dev_priv) < 6)
2041                 return;
2042
2043         i915_check_and_clear_faults(dev_priv);
2044
2045         ggtt->base.clear_range(&ggtt->base, 0, ggtt->base.total);
2046
2047         i915_ggtt_invalidate(dev_priv);
2048 }
2049
2050 int i915_gem_gtt_prepare_pages(struct drm_i915_gem_object *obj,
2051                                struct sg_table *pages)
2052 {
2053         do {
2054                 if (dma_map_sg(&obj->base.dev->pdev->dev,
2055                                pages->sgl, pages->nents,
2056                                PCI_DMA_BIDIRECTIONAL))
2057                         return 0;
2058
2059                 /* If the DMA remap fails, one cause can be that we have
2060                  * too many objects pinned in a small remapping table,
2061                  * such as swiotlb. Incrementally purge all other objects and
2062                  * try again - if there are no more pages to remove from
2063                  * the DMA remapper, i915_gem_shrink will return 0.
2064                  */
2065                 GEM_BUG_ON(obj->mm.pages == pages);
2066         } while (i915_gem_shrink(to_i915(obj->base.dev),
2067                                  obj->base.size >> PAGE_SHIFT,
2068                                  I915_SHRINK_BOUND |
2069                                  I915_SHRINK_UNBOUND |
2070                                  I915_SHRINK_ACTIVE));
2071
2072         return -ENOSPC;
2073 }
2074
2075 static void gen8_set_pte(void __iomem *addr, gen8_pte_t pte)
2076 {
2077         writeq(pte, addr);
2078 }
2079
2080 static void gen8_ggtt_insert_page(struct i915_address_space *vm,
2081                                   dma_addr_t addr,
2082                                   u64 offset,
2083                                   enum i915_cache_level level,
2084                                   u32 unused)
2085 {
2086         struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
2087         gen8_pte_t __iomem *pte =
2088                 (gen8_pte_t __iomem *)ggtt->gsm + (offset >> PAGE_SHIFT);
2089
2090         gen8_set_pte(pte, gen8_pte_encode(addr, level));
2091
2092         ggtt->invalidate(vm->i915);
2093 }
2094
2095 static void gen8_ggtt_insert_entries(struct i915_address_space *vm,
2096                                      struct sg_table *st,
2097                                      u64 start,
2098                                      enum i915_cache_level level,
2099                                      u32 unused)
2100 {
2101         struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
2102         struct sgt_iter sgt_iter;
2103         gen8_pte_t __iomem *gtt_entries;
2104         const gen8_pte_t pte_encode = gen8_pte_encode(0, level);
2105         dma_addr_t addr;
2106
2107         gtt_entries = (gen8_pte_t __iomem *)ggtt->gsm;
2108         gtt_entries += start >> PAGE_SHIFT;
2109         for_each_sgt_dma(addr, sgt_iter, st)
2110                 gen8_set_pte(gtt_entries++, pte_encode | addr);
2111
2112         wmb();
2113
2114         /* This next bit makes the above posting read even more important. We
2115          * want to flush the TLBs only after we're certain all the PTE updates
2116          * have finished.
2117          */
2118         ggtt->invalidate(vm->i915);
2119 }
2120
2121 static void gen6_ggtt_insert_page(struct i915_address_space *vm,
2122                                   dma_addr_t addr,
2123                                   u64 offset,
2124                                   enum i915_cache_level level,
2125                                   u32 flags)
2126 {
2127         struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
2128         gen6_pte_t __iomem *pte =
2129                 (gen6_pte_t __iomem *)ggtt->gsm + (offset >> PAGE_SHIFT);
2130
2131         iowrite32(vm->pte_encode(addr, level, flags), pte);
2132
2133         ggtt->invalidate(vm->i915);
2134 }
2135
2136 /*
2137  * Binds an object into the global gtt with the specified cache level. The object
2138  * will be accessible to the GPU via commands whose operands reference offsets
2139  * within the global GTT as well as accessible by the GPU through the GMADR
2140  * mapped BAR (dev_priv->mm.gtt->gtt).
2141  */
2142 static void gen6_ggtt_insert_entries(struct i915_address_space *vm,
2143                                      struct sg_table *st,
2144                                      u64 start,
2145                                      enum i915_cache_level level,
2146                                      u32 flags)
2147 {
2148         struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
2149         gen6_pte_t __iomem *entries = (gen6_pte_t __iomem *)ggtt->gsm;
2150         unsigned int i = start >> PAGE_SHIFT;
2151         struct sgt_iter iter;
2152         dma_addr_t addr;
2153         for_each_sgt_dma(addr, iter, st)
2154                 iowrite32(vm->pte_encode(addr, level, flags), &entries[i++]);
2155         wmb();
2156
2157         /* This next bit makes the above posting read even more important. We
2158          * want to flush the TLBs only after we're certain all the PTE updates
2159          * have finished.
2160          */
2161         ggtt->invalidate(vm->i915);
2162 }
2163
2164 static void nop_clear_range(struct i915_address_space *vm,
2165                             u64 start, u64 length)
2166 {
2167 }
2168
2169 static void gen8_ggtt_clear_range(struct i915_address_space *vm,
2170                                   u64 start, u64 length)
2171 {
2172         struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
2173         unsigned first_entry = start >> PAGE_SHIFT;
2174         unsigned num_entries = length >> PAGE_SHIFT;
2175         const gen8_pte_t scratch_pte =
2176                 gen8_pte_encode(vm->scratch_page.daddr, I915_CACHE_LLC);
2177         gen8_pte_t __iomem *gtt_base =
2178                 (gen8_pte_t __iomem *)ggtt->gsm + first_entry;
2179         const int max_entries = ggtt_total_entries(ggtt) - first_entry;
2180         int i;
2181
2182         if (WARN(num_entries > max_entries,
2183                  "First entry = %d; Num entries = %d (max=%d)\n",
2184                  first_entry, num_entries, max_entries))
2185                 num_entries = max_entries;
2186
2187         for (i = 0; i < num_entries; i++)
2188                 gen8_set_pte(&gtt_base[i], scratch_pte);
2189 }
2190
2191 static void gen6_ggtt_clear_range(struct i915_address_space *vm,
2192                                   u64 start, u64 length)
2193 {
2194         struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
2195         unsigned first_entry = start >> PAGE_SHIFT;
2196         unsigned num_entries = length >> PAGE_SHIFT;
2197         gen6_pte_t scratch_pte, __iomem *gtt_base =
2198                 (gen6_pte_t __iomem *)ggtt->gsm + first_entry;
2199         const int max_entries = ggtt_total_entries(ggtt) - first_entry;
2200         int i;
2201
2202         if (WARN(num_entries > max_entries,
2203                  "First entry = %d; Num entries = %d (max=%d)\n",
2204                  first_entry, num_entries, max_entries))
2205                 num_entries = max_entries;
2206
2207         scratch_pte = vm->pte_encode(vm->scratch_page.daddr,
2208                                      I915_CACHE_LLC, 0);
2209
2210         for (i = 0; i < num_entries; i++)
2211                 iowrite32(scratch_pte, &gtt_base[i]);
2212 }
2213
2214 static void i915_ggtt_insert_page(struct i915_address_space *vm,
2215                                   dma_addr_t addr,
2216                                   u64 offset,
2217                                   enum i915_cache_level cache_level,
2218                                   u32 unused)
2219 {
2220         unsigned int flags = (cache_level == I915_CACHE_NONE) ?
2221                 AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY;
2222
2223         intel_gtt_insert_page(addr, offset >> PAGE_SHIFT, flags);
2224 }
2225
2226 static void i915_ggtt_insert_entries(struct i915_address_space *vm,
2227                                      struct sg_table *pages,
2228                                      u64 start,
2229                                      enum i915_cache_level cache_level,
2230                                      u32 unused)
2231 {
2232         unsigned int flags = (cache_level == I915_CACHE_NONE) ?
2233                 AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY;
2234
2235         intel_gtt_insert_sg_entries(pages, start >> PAGE_SHIFT, flags);
2236 }
2237
2238 static void i915_ggtt_clear_range(struct i915_address_space *vm,
2239                                   u64 start, u64 length)
2240 {
2241         intel_gtt_clear_range(start >> PAGE_SHIFT, length >> PAGE_SHIFT);
2242 }
2243
2244 static int ggtt_bind_vma(struct i915_vma *vma,
2245                          enum i915_cache_level cache_level,
2246                          u32 flags)
2247 {
2248         struct drm_i915_private *i915 = vma->vm->i915;
2249         struct drm_i915_gem_object *obj = vma->obj;
2250         u32 pte_flags;
2251
2252         if (unlikely(!vma->pages)) {
2253                 int ret = i915_get_ggtt_vma_pages(vma);
2254                 if (ret)
2255                         return ret;
2256         }
2257
2258         /* Currently applicable only to VLV */
2259         pte_flags = 0;
2260         if (obj->gt_ro)
2261                 pte_flags |= PTE_READ_ONLY;
2262
2263         intel_runtime_pm_get(i915);
2264         vma->vm->insert_entries(vma->vm, vma->pages, vma->node.start,
2265                                 cache_level, pte_flags);
2266         intel_runtime_pm_put(i915);
2267
2268         /*
2269          * Without aliasing PPGTT there's no difference between
2270          * GLOBAL/LOCAL_BIND, it's all the same ptes. Hence unconditionally
2271          * upgrade to both bound if we bind either to avoid double-binding.
2272          */
2273         vma->flags |= I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND;
2274
2275         return 0;
2276 }
2277
2278 static void ggtt_unbind_vma(struct i915_vma *vma)
2279 {
2280         struct drm_i915_private *i915 = vma->vm->i915;
2281
2282         intel_runtime_pm_get(i915);
2283         vma->vm->clear_range(vma->vm, vma->node.start, vma->size);
2284         intel_runtime_pm_put(i915);
2285 }
2286
2287 static int aliasing_gtt_bind_vma(struct i915_vma *vma,
2288                                  enum i915_cache_level cache_level,
2289                                  u32 flags)
2290 {
2291         struct drm_i915_private *i915 = vma->vm->i915;
2292         u32 pte_flags;
2293         int ret;
2294
2295         if (unlikely(!vma->pages)) {
2296                 ret = i915_get_ggtt_vma_pages(vma);
2297                 if (ret)
2298                         return ret;
2299         }
2300
2301         /* Currently applicable only to VLV */
2302         pte_flags = 0;
2303         if (vma->obj->gt_ro)
2304                 pte_flags |= PTE_READ_ONLY;
2305
2306         if (flags & I915_VMA_LOCAL_BIND) {
2307                 struct i915_hw_ppgtt *appgtt = i915->mm.aliasing_ppgtt;
2308
2309                 if (appgtt->base.allocate_va_range) {
2310                         ret = appgtt->base.allocate_va_range(&appgtt->base,
2311                                                              vma->node.start,
2312                                                              vma->node.size);
2313                         if (ret)
2314                                 goto err_pages;
2315                 }
2316
2317                 appgtt->base.insert_entries(&appgtt->base,
2318                                             vma->pages, vma->node.start,
2319                                             cache_level, pte_flags);
2320         }
2321
2322         if (flags & I915_VMA_GLOBAL_BIND) {
2323                 intel_runtime_pm_get(i915);
2324                 vma->vm->insert_entries(vma->vm,
2325                                         vma->pages, vma->node.start,
2326                                         cache_level, pte_flags);
2327                 intel_runtime_pm_put(i915);
2328         }
2329
2330         return 0;
2331
2332 err_pages:
2333         if (!(vma->flags & (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND))) {
2334                 if (vma->pages != vma->obj->mm.pages) {
2335                         GEM_BUG_ON(!vma->pages);
2336                         sg_free_table(vma->pages);
2337                         kfree(vma->pages);
2338                 }
2339                 vma->pages = NULL;
2340         }
2341         return ret;
2342 }
2343
2344 static void aliasing_gtt_unbind_vma(struct i915_vma *vma)
2345 {
2346         struct drm_i915_private *i915 = vma->vm->i915;
2347
2348         if (vma->flags & I915_VMA_GLOBAL_BIND) {
2349                 intel_runtime_pm_get(i915);
2350                 vma->vm->clear_range(vma->vm, vma->node.start, vma->size);
2351                 intel_runtime_pm_put(i915);
2352         }
2353
2354         if (vma->flags & I915_VMA_LOCAL_BIND) {
2355                 struct i915_address_space *vm = &i915->mm.aliasing_ppgtt->base;
2356
2357                 vm->clear_range(vm, vma->node.start, vma->size);
2358         }
2359 }
2360
2361 void i915_gem_gtt_finish_pages(struct drm_i915_gem_object *obj,
2362                                struct sg_table *pages)
2363 {
2364         struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
2365         struct device *kdev = &dev_priv->drm.pdev->dev;
2366         struct i915_ggtt *ggtt = &dev_priv->ggtt;
2367
2368         if (unlikely(ggtt->do_idle_maps)) {
2369                 if (i915_gem_wait_for_idle(dev_priv, 0)) {
2370                         DRM_ERROR("Failed to wait for idle; VT'd may hang.\n");
2371                         /* Wait a bit, in hopes it avoids the hang */
2372                         udelay(10);
2373                 }
2374         }
2375
2376         dma_unmap_sg(kdev, pages->sgl, pages->nents, PCI_DMA_BIDIRECTIONAL);
2377 }
2378
2379 static void i915_gtt_color_adjust(const struct drm_mm_node *node,
2380                                   unsigned long color,
2381                                   u64 *start,
2382                                   u64 *end)
2383 {
2384         if (node->allocated && node->color != color)
2385                 *start += I915_GTT_PAGE_SIZE;
2386
2387         /* Also leave a space between the unallocated reserved node after the
2388          * GTT and any objects within the GTT, i.e. we use the color adjustment
2389          * to insert a guard page to prevent prefetches crossing over the
2390          * GTT boundary.
2391          */
2392         node = list_next_entry(node, node_list);
2393         if (node->color != color)
2394                 *end -= I915_GTT_PAGE_SIZE;
2395 }
2396
2397 int i915_gem_init_aliasing_ppgtt(struct drm_i915_private *i915)
2398 {
2399         struct i915_ggtt *ggtt = &i915->ggtt;
2400         struct i915_hw_ppgtt *ppgtt;
2401         int err;
2402
2403         ppgtt = i915_ppgtt_create(i915, ERR_PTR(-EPERM), "[alias]");
2404         if (IS_ERR(ppgtt))
2405                 return PTR_ERR(ppgtt);
2406
2407         if (WARN_ON(ppgtt->base.total < ggtt->base.total)) {
2408                 err = -ENODEV;
2409                 goto err_ppgtt;
2410         }
2411
2412         if (ppgtt->base.allocate_va_range) {
2413                 /* Note we only pre-allocate as far as the end of the global
2414                  * GTT. On 48b / 4-level page-tables, the difference is very,
2415                  * very significant! We have to preallocate as GVT/vgpu does
2416                  * not like the page directory disappearing.
2417                  */
2418                 err = ppgtt->base.allocate_va_range(&ppgtt->base,
2419                                                     0, ggtt->base.total);
2420                 if (err)
2421                         goto err_ppgtt;
2422         }
2423
2424         i915->mm.aliasing_ppgtt = ppgtt;
2425
2426         WARN_ON(ggtt->base.bind_vma != ggtt_bind_vma);
2427         ggtt->base.bind_vma = aliasing_gtt_bind_vma;
2428
2429         WARN_ON(ggtt->base.unbind_vma != ggtt_unbind_vma);
2430         ggtt->base.unbind_vma = aliasing_gtt_unbind_vma;
2431
2432         return 0;
2433
2434 err_ppgtt:
2435         i915_ppgtt_put(ppgtt);
2436         return err;
2437 }
2438
2439 void i915_gem_fini_aliasing_ppgtt(struct drm_i915_private *i915)
2440 {
2441         struct i915_ggtt *ggtt = &i915->ggtt;
2442         struct i915_hw_ppgtt *ppgtt;
2443
2444         ppgtt = fetch_and_zero(&i915->mm.aliasing_ppgtt);
2445         if (!ppgtt)
2446                 return;
2447
2448         i915_ppgtt_put(ppgtt);
2449
2450         ggtt->base.bind_vma = ggtt_bind_vma;
2451         ggtt->base.unbind_vma = ggtt_unbind_vma;
2452 }
2453
2454 int i915_gem_init_ggtt(struct drm_i915_private *dev_priv)
2455 {
2456         /* Let GEM Manage all of the aperture.
2457          *
2458          * However, leave one page at the end still bound to the scratch page.
2459          * There are a number of places where the hardware apparently prefetches
2460          * past the end of the object, and we've seen multiple hangs with the
2461          * GPU head pointer stuck in a batchbuffer bound at the last page of the
2462          * aperture.  One page should be enough to keep any prefetching inside
2463          * of the aperture.
2464          */
2465         struct i915_ggtt *ggtt = &dev_priv->ggtt;
2466         unsigned long hole_start, hole_end;
2467         struct drm_mm_node *entry;
2468         int ret;
2469
2470         ret = intel_vgt_balloon(dev_priv);
2471         if (ret)
2472                 return ret;
2473
2474         /* Reserve a mappable slot for our lockless error capture */
2475         ret = drm_mm_insert_node_in_range(&ggtt->base.mm, &ggtt->error_capture,
2476                                           PAGE_SIZE, 0, I915_COLOR_UNEVICTABLE,
2477                                           0, ggtt->mappable_end,
2478                                           DRM_MM_INSERT_LOW);
2479         if (ret)
2480                 return ret;
2481
2482         /* Clear any non-preallocated blocks */
2483         drm_mm_for_each_hole(entry, &ggtt->base.mm, hole_start, hole_end) {
2484                 DRM_DEBUG_KMS("clearing unused GTT space: [%lx, %lx]\n",
2485                               hole_start, hole_end);
2486                 ggtt->base.clear_range(&ggtt->base, hole_start,
2487                                        hole_end - hole_start);
2488         }
2489
2490         /* And finally clear the reserved guard page */
2491         ggtt->base.clear_range(&ggtt->base,
2492                                ggtt->base.total - PAGE_SIZE, PAGE_SIZE);
2493
2494         if (USES_PPGTT(dev_priv) && !USES_FULL_PPGTT(dev_priv)) {
2495                 ret = i915_gem_init_aliasing_ppgtt(dev_priv);
2496                 if (ret)
2497                         goto err;
2498         }
2499
2500         return 0;
2501
2502 err:
2503         drm_mm_remove_node(&ggtt->error_capture);
2504         return ret;
2505 }
2506
2507 /**
2508  * i915_ggtt_cleanup_hw - Clean up GGTT hardware initialization
2509  * @dev_priv: i915 device
2510  */
2511 void i915_ggtt_cleanup_hw(struct drm_i915_private *dev_priv)
2512 {
2513         struct i915_ggtt *ggtt = &dev_priv->ggtt;
2514         struct i915_vma *vma, *vn;
2515
2516         ggtt->base.closed = true;
2517
2518         mutex_lock(&dev_priv->drm.struct_mutex);
2519         WARN_ON(!list_empty(&ggtt->base.active_list));
2520         list_for_each_entry_safe(vma, vn, &ggtt->base.inactive_list, vm_link)
2521                 WARN_ON(i915_vma_unbind(vma));
2522         mutex_unlock(&dev_priv->drm.struct_mutex);
2523
2524         i915_gem_cleanup_stolen(&dev_priv->drm);
2525
2526         mutex_lock(&dev_priv->drm.struct_mutex);
2527         i915_gem_fini_aliasing_ppgtt(dev_priv);
2528
2529         if (drm_mm_node_allocated(&ggtt->error_capture))
2530                 drm_mm_remove_node(&ggtt->error_capture);
2531
2532         if (drm_mm_initialized(&ggtt->base.mm)) {
2533                 intel_vgt_deballoon(dev_priv);
2534                 i915_address_space_fini(&ggtt->base);
2535         }
2536
2537         ggtt->base.cleanup(&ggtt->base);
2538         mutex_unlock(&dev_priv->drm.struct_mutex);
2539
2540         arch_phys_wc_del(ggtt->mtrr);
2541         io_mapping_fini(&ggtt->mappable);
2542 }
2543
2544 static unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl)
2545 {
2546         snb_gmch_ctl >>= SNB_GMCH_GGMS_SHIFT;
2547         snb_gmch_ctl &= SNB_GMCH_GGMS_MASK;
2548         return snb_gmch_ctl << 20;
2549 }
2550
2551 static unsigned int gen8_get_total_gtt_size(u16 bdw_gmch_ctl)
2552 {
2553         bdw_gmch_ctl >>= BDW_GMCH_GGMS_SHIFT;
2554         bdw_gmch_ctl &= BDW_GMCH_GGMS_MASK;
2555         if (bdw_gmch_ctl)
2556                 bdw_gmch_ctl = 1 << bdw_gmch_ctl;
2557
2558 #ifdef CONFIG_X86_32
2559         /* Limit 32b platforms to a 2GB GGTT: 4 << 20 / pte size * PAGE_SIZE */
2560         if (bdw_gmch_ctl > 4)
2561                 bdw_gmch_ctl = 4;
2562 #endif
2563
2564         return bdw_gmch_ctl << 20;
2565 }
2566
2567 static unsigned int chv_get_total_gtt_size(u16 gmch_ctrl)
2568 {
2569         gmch_ctrl >>= SNB_GMCH_GGMS_SHIFT;
2570         gmch_ctrl &= SNB_GMCH_GGMS_MASK;
2571
2572         if (gmch_ctrl)
2573                 return 1 << (20 + gmch_ctrl);
2574
2575         return 0;
2576 }
2577
2578 static size_t gen6_get_stolen_size(u16 snb_gmch_ctl)
2579 {
2580         snb_gmch_ctl >>= SNB_GMCH_GMS_SHIFT;
2581         snb_gmch_ctl &= SNB_GMCH_GMS_MASK;
2582         return snb_gmch_ctl << 25; /* 32 MB units */
2583 }
2584
2585 static size_t gen8_get_stolen_size(u16 bdw_gmch_ctl)
2586 {
2587         bdw_gmch_ctl >>= BDW_GMCH_GMS_SHIFT;
2588         bdw_gmch_ctl &= BDW_GMCH_GMS_MASK;
2589         return bdw_gmch_ctl << 25; /* 32 MB units */
2590 }
2591
2592 static size_t chv_get_stolen_size(u16 gmch_ctrl)
2593 {
2594         gmch_ctrl >>= SNB_GMCH_GMS_SHIFT;
2595         gmch_ctrl &= SNB_GMCH_GMS_MASK;
2596
2597         /*
2598          * 0x0  to 0x10: 32MB increments starting at 0MB
2599          * 0x11 to 0x16: 4MB increments starting at 8MB
2600          * 0x17 to 0x1d: 4MB increments start at 36MB
2601          */
2602         if (gmch_ctrl < 0x11)
2603                 return gmch_ctrl << 25;
2604         else if (gmch_ctrl < 0x17)
2605                 return (gmch_ctrl - 0x11 + 2) << 22;
2606         else
2607                 return (gmch_ctrl - 0x17 + 9) << 22;
2608 }
2609
2610 static size_t gen9_get_stolen_size(u16 gen9_gmch_ctl)
2611 {
2612         gen9_gmch_ctl >>= BDW_GMCH_GMS_SHIFT;
2613         gen9_gmch_ctl &= BDW_GMCH_GMS_MASK;
2614
2615         if (gen9_gmch_ctl < 0xf0)
2616                 return gen9_gmch_ctl << 25; /* 32 MB units */
2617         else
2618                 /* 4MB increments starting at 0xf0 for 4MB */
2619                 return (gen9_gmch_ctl - 0xf0 + 1) << 22;
2620 }
2621
2622 static int ggtt_probe_common(struct i915_ggtt *ggtt, u64 size)
2623 {
2624         struct drm_i915_private *dev_priv = ggtt->base.i915;
2625         struct pci_dev *pdev = dev_priv->drm.pdev;
2626         phys_addr_t phys_addr;
2627         int ret;
2628
2629         /* For Modern GENs the PTEs and register space are split in the BAR */
2630         phys_addr = pci_resource_start(pdev, 0) + pci_resource_len(pdev, 0) / 2;
2631
2632         /*
2633          * On BXT writes larger than 64 bit to the GTT pagetable range will be
2634          * dropped. For WC mappings in general we have 64 byte burst writes
2635          * when the WC buffer is flushed, so we can't use it, but have to
2636          * resort to an uncached mapping. The WC issue is easily caught by the
2637          * readback check when writing GTT PTE entries.
2638          */
2639         if (IS_GEN9_LP(dev_priv))
2640                 ggtt->gsm = ioremap_nocache(phys_addr, size);
2641         else
2642                 ggtt->gsm = ioremap_wc(phys_addr, size);
2643         if (!ggtt->gsm) {
2644                 DRM_ERROR("Failed to map the ggtt page table\n");
2645                 return -ENOMEM;
2646         }
2647
2648         ret = setup_scratch_page(&ggtt->base, GFP_DMA32);
2649         if (ret) {
2650                 DRM_ERROR("Scratch setup failed\n");
2651                 /* iounmap will also get called at remove, but meh */
2652                 iounmap(ggtt->gsm);
2653                 return ret;
2654         }
2655
2656         return 0;
2657 }
2658
2659 /* The GGTT and PPGTT need a private PPAT setup in order to handle cacheability
2660  * bits. When using advanced contexts each context stores its own PAT, but
2661  * writing this data shouldn't be harmful even in those cases. */
2662 static void bdw_setup_private_ppat(struct drm_i915_private *dev_priv)
2663 {
2664         u64 pat;
2665
2666         pat = GEN8_PPAT(0, GEN8_PPAT_WB | GEN8_PPAT_LLC)     | /* for normal objects, no eLLC */
2667               GEN8_PPAT(1, GEN8_PPAT_WC | GEN8_PPAT_LLCELLC) | /* for something pointing to ptes? */
2668               GEN8_PPAT(2, GEN8_PPAT_WT | GEN8_PPAT_LLCELLC) | /* for scanout with eLLC */
2669               GEN8_PPAT(3, GEN8_PPAT_UC)                     | /* Uncached objects, mostly for scanout */
2670               GEN8_PPAT(4, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(0)) |
2671               GEN8_PPAT(5, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(1)) |
2672               GEN8_PPAT(6, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(2)) |
2673               GEN8_PPAT(7, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(3));
2674
2675         if (!USES_PPGTT(dev_priv))
2676                 /* Spec: "For GGTT, there is NO pat_sel[2:0] from the entry,
2677                  * so RTL will always use the value corresponding to
2678                  * pat_sel = 000".
2679                  * So let's disable cache for GGTT to avoid screen corruptions.
2680                  * MOCS still can be used though.
2681                  * - System agent ggtt writes (i.e. cpu gtt mmaps) already work
2682                  * before this patch, i.e. the same uncached + snooping access
2683                  * like on gen6/7 seems to be in effect.
2684                  * - So this just fixes blitter/render access. Again it looks
2685                  * like it's not just uncached access, but uncached + snooping.
2686                  * So we can still hold onto all our assumptions wrt cpu
2687                  * clflushing on LLC machines.
2688                  */
2689                 pat = GEN8_PPAT(0, GEN8_PPAT_UC);
2690
2691         /* XXX: spec defines this as 2 distinct registers. It's unclear if a 64b
2692          * write would work. */
2693         I915_WRITE(GEN8_PRIVATE_PAT_LO, pat);
2694         I915_WRITE(GEN8_PRIVATE_PAT_HI, pat >> 32);
2695 }
2696
2697 static void chv_setup_private_ppat(struct drm_i915_private *dev_priv)
2698 {
2699         u64 pat;
2700
2701         /*
2702          * Map WB on BDW to snooped on CHV.
2703          *
2704          * Only the snoop bit has meaning for CHV, the rest is
2705          * ignored.
2706          *
2707          * The hardware will never snoop for certain types of accesses:
2708          * - CPU GTT (GMADR->GGTT->no snoop->memory)
2709          * - PPGTT page tables
2710          * - some other special cycles
2711          *
2712          * As with BDW, we also need to consider the following for GT accesses:
2713          * "For GGTT, there is NO pat_sel[2:0] from the entry,
2714          * so RTL will always use the value corresponding to
2715          * pat_sel = 000".
2716          * Which means we must set the snoop bit in PAT entry 0
2717          * in order to keep the global status page working.
2718          */
2719         pat = GEN8_PPAT(0, CHV_PPAT_SNOOP) |
2720               GEN8_PPAT(1, 0) |
2721               GEN8_PPAT(2, 0) |
2722               GEN8_PPAT(3, 0) |
2723               GEN8_PPAT(4, CHV_PPAT_SNOOP) |
2724               GEN8_PPAT(5, CHV_PPAT_SNOOP) |
2725               GEN8_PPAT(6, CHV_PPAT_SNOOP) |
2726               GEN8_PPAT(7, CHV_PPAT_SNOOP);
2727
2728         I915_WRITE(GEN8_PRIVATE_PAT_LO, pat);
2729         I915_WRITE(GEN8_PRIVATE_PAT_HI, pat >> 32);
2730 }
2731
2732 static void gen6_gmch_remove(struct i915_address_space *vm)
2733 {
2734         struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
2735
2736         iounmap(ggtt->gsm);
2737         cleanup_scratch_page(vm);
2738 }
2739
2740 static int gen8_gmch_probe(struct i915_ggtt *ggtt)
2741 {
2742         struct drm_i915_private *dev_priv = ggtt->base.i915;
2743         struct pci_dev *pdev = dev_priv->drm.pdev;
2744         unsigned int size;
2745         u16 snb_gmch_ctl;
2746
2747         /* TODO: We're not aware of mappable constraints on gen8 yet */
2748         ggtt->mappable_base = pci_resource_start(pdev, 2);
2749         ggtt->mappable_end = pci_resource_len(pdev, 2);
2750
2751         if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(39)))
2752                 pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(39));
2753
2754         pci_read_config_word(pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
2755
2756         if (INTEL_GEN(dev_priv) >= 9) {
2757                 ggtt->stolen_size = gen9_get_stolen_size(snb_gmch_ctl);
2758                 size = gen8_get_total_gtt_size(snb_gmch_ctl);
2759         } else if (IS_CHERRYVIEW(dev_priv)) {
2760                 ggtt->stolen_size = chv_get_stolen_size(snb_gmch_ctl);
2761                 size = chv_get_total_gtt_size(snb_gmch_ctl);
2762         } else {
2763                 ggtt->stolen_size = gen8_get_stolen_size(snb_gmch_ctl);
2764                 size = gen8_get_total_gtt_size(snb_gmch_ctl);
2765         }
2766
2767         ggtt->base.total = (size / sizeof(gen8_pte_t)) << PAGE_SHIFT;
2768
2769         if (IS_CHERRYVIEW(dev_priv) || IS_GEN9_LP(dev_priv))
2770                 chv_setup_private_ppat(dev_priv);
2771         else
2772                 bdw_setup_private_ppat(dev_priv);
2773
2774         ggtt->base.cleanup = gen6_gmch_remove;
2775         ggtt->base.bind_vma = ggtt_bind_vma;
2776         ggtt->base.unbind_vma = ggtt_unbind_vma;
2777         ggtt->base.insert_page = gen8_ggtt_insert_page;
2778         ggtt->base.clear_range = nop_clear_range;
2779         if (!USES_FULL_PPGTT(dev_priv) || intel_scanout_needs_vtd_wa(dev_priv))
2780                 ggtt->base.clear_range = gen8_ggtt_clear_range;
2781
2782         ggtt->base.insert_entries = gen8_ggtt_insert_entries;
2783
2784         ggtt->invalidate = gen6_ggtt_invalidate;
2785
2786         return ggtt_probe_common(ggtt, size);
2787 }
2788
2789 static int gen6_gmch_probe(struct i915_ggtt *ggtt)
2790 {
2791         struct drm_i915_private *dev_priv = ggtt->base.i915;
2792         struct pci_dev *pdev = dev_priv->drm.pdev;
2793         unsigned int size;
2794         u16 snb_gmch_ctl;
2795
2796         ggtt->mappable_base = pci_resource_start(pdev, 2);
2797         ggtt->mappable_end = pci_resource_len(pdev, 2);
2798
2799         /* 64/512MB is the current min/max we actually know of, but this is just
2800          * a coarse sanity check.
2801          */
2802         if (ggtt->mappable_end < (64<<20) || ggtt->mappable_end > (512<<20)) {
2803                 DRM_ERROR("Unknown GMADR size (%llx)\n", ggtt->mappable_end);
2804                 return -ENXIO;
2805         }
2806
2807         if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(40)))
2808                 pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(40));
2809         pci_read_config_word(pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
2810
2811         ggtt->stolen_size = gen6_get_stolen_size(snb_gmch_ctl);
2812
2813         size = gen6_get_total_gtt_size(snb_gmch_ctl);
2814         ggtt->base.total = (size / sizeof(gen6_pte_t)) << PAGE_SHIFT;
2815
2816         ggtt->base.clear_range = gen6_ggtt_clear_range;
2817         ggtt->base.insert_page = gen6_ggtt_insert_page;
2818         ggtt->base.insert_entries = gen6_ggtt_insert_entries;
2819         ggtt->base.bind_vma = ggtt_bind_vma;
2820         ggtt->base.unbind_vma = ggtt_unbind_vma;
2821         ggtt->base.cleanup = gen6_gmch_remove;
2822
2823         ggtt->invalidate = gen6_ggtt_invalidate;
2824
2825         if (HAS_EDRAM(dev_priv))
2826                 ggtt->base.pte_encode = iris_pte_encode;
2827         else if (IS_HASWELL(dev_priv))
2828                 ggtt->base.pte_encode = hsw_pte_encode;
2829         else if (IS_VALLEYVIEW(dev_priv))
2830                 ggtt->base.pte_encode = byt_pte_encode;
2831         else if (INTEL_GEN(dev_priv) >= 7)
2832                 ggtt->base.pte_encode = ivb_pte_encode;
2833         else
2834                 ggtt->base.pte_encode = snb_pte_encode;
2835
2836         return ggtt_probe_common(ggtt, size);
2837 }
2838
2839 static void i915_gmch_remove(struct i915_address_space *vm)
2840 {
2841         intel_gmch_remove();
2842 }
2843
2844 static int i915_gmch_probe(struct i915_ggtt *ggtt)
2845 {
2846         struct drm_i915_private *dev_priv = ggtt->base.i915;
2847         int ret;
2848
2849         ret = intel_gmch_probe(dev_priv->bridge_dev, dev_priv->drm.pdev, NULL);
2850         if (!ret) {
2851                 DRM_ERROR("failed to set up gmch\n");
2852                 return -EIO;
2853         }
2854
2855         intel_gtt_get(&ggtt->base.total,
2856                       &ggtt->stolen_size,
2857                       &ggtt->mappable_base,
2858                       &ggtt->mappable_end);
2859
2860         ggtt->do_idle_maps = needs_idle_maps(dev_priv);
2861         ggtt->base.insert_page = i915_ggtt_insert_page;
2862         ggtt->base.insert_entries = i915_ggtt_insert_entries;
2863         ggtt->base.clear_range = i915_ggtt_clear_range;
2864         ggtt->base.bind_vma = ggtt_bind_vma;
2865         ggtt->base.unbind_vma = ggtt_unbind_vma;
2866         ggtt->base.cleanup = i915_gmch_remove;
2867
2868         ggtt->invalidate = gmch_ggtt_invalidate;
2869
2870         if (unlikely(ggtt->do_idle_maps))
2871                 DRM_INFO("applying Ironlake quirks for intel_iommu\n");
2872
2873         return 0;
2874 }
2875
2876 /**
2877  * i915_ggtt_probe_hw - Probe GGTT hardware location
2878  * @dev_priv: i915 device
2879  */
2880 int i915_ggtt_probe_hw(struct drm_i915_private *dev_priv)
2881 {
2882         struct i915_ggtt *ggtt = &dev_priv->ggtt;
2883         int ret;
2884
2885         ggtt->base.i915 = dev_priv;
2886         ggtt->base.dma = &dev_priv->drm.pdev->dev;
2887
2888         if (INTEL_GEN(dev_priv) <= 5)
2889                 ret = i915_gmch_probe(ggtt);
2890         else if (INTEL_GEN(dev_priv) < 8)
2891                 ret = gen6_gmch_probe(ggtt);
2892         else
2893                 ret = gen8_gmch_probe(ggtt);
2894         if (ret)
2895                 return ret;
2896
2897         /* Trim the GGTT to fit the GuC mappable upper range (when enabled).
2898          * This is easier than doing range restriction on the fly, as we
2899          * currently don't have any bits spare to pass in this upper
2900          * restriction!
2901          */
2902         if (HAS_GUC(dev_priv) && i915.enable_guc_loading) {
2903                 ggtt->base.total = min_t(u64, ggtt->base.total, GUC_GGTT_TOP);
2904                 ggtt->mappable_end = min(ggtt->mappable_end, ggtt->base.total);
2905         }
2906
2907         if ((ggtt->base.total - 1) >> 32) {
2908                 DRM_ERROR("We never expected a Global GTT with more than 32bits"
2909                           " of address space! Found %lldM!\n",
2910                           ggtt->base.total >> 20);
2911                 ggtt->base.total = 1ULL << 32;
2912                 ggtt->mappable_end = min(ggtt->mappable_end, ggtt->base.total);
2913         }
2914
2915         if (ggtt->mappable_end > ggtt->base.total) {
2916                 DRM_ERROR("mappable aperture extends past end of GGTT,"
2917                           " aperture=%llx, total=%llx\n",
2918                           ggtt->mappable_end, ggtt->base.total);
2919                 ggtt->mappable_end = ggtt->base.total;
2920         }
2921
2922         /* GMADR is the PCI mmio aperture into the global GTT. */
2923         DRM_INFO("Memory usable by graphics device = %lluM\n",
2924                  ggtt->base.total >> 20);
2925         DRM_DEBUG_DRIVER("GMADR size = %lldM\n", ggtt->mappable_end >> 20);
2926         DRM_DEBUG_DRIVER("GTT stolen size = %uM\n", ggtt->stolen_size >> 20);
2927 #ifdef CONFIG_INTEL_IOMMU
2928         if (intel_iommu_gfx_mapped)
2929                 DRM_INFO("VT-d active for gfx access\n");
2930 #endif
2931
2932         return 0;
2933 }
2934
2935 /**
2936  * i915_ggtt_init_hw - Initialize GGTT hardware
2937  * @dev_priv: i915 device
2938  */
2939 int i915_ggtt_init_hw(struct drm_i915_private *dev_priv)
2940 {
2941         struct i915_ggtt *ggtt = &dev_priv->ggtt;
2942         int ret;
2943
2944         INIT_LIST_HEAD(&dev_priv->vm_list);
2945
2946         /* Note that we use page colouring to enforce a guard page at the
2947          * end of the address space. This is required as the CS may prefetch
2948          * beyond the end of the batch buffer, across the page boundary,
2949          * and beyond the end of the GTT if we do not provide a guard.
2950          */
2951         mutex_lock(&dev_priv->drm.struct_mutex);
2952         i915_address_space_init(&ggtt->base, dev_priv, "[global]");
2953         if (!HAS_LLC(dev_priv) && !USES_PPGTT(dev_priv))
2954                 ggtt->base.mm.color_adjust = i915_gtt_color_adjust;
2955         mutex_unlock(&dev_priv->drm.struct_mutex);
2956
2957         if (!io_mapping_init_wc(&dev_priv->ggtt.mappable,
2958                                 dev_priv->ggtt.mappable_base,
2959                                 dev_priv->ggtt.mappable_end)) {
2960                 ret = -EIO;
2961                 goto out_gtt_cleanup;
2962         }
2963
2964         ggtt->mtrr = arch_phys_wc_add(ggtt->mappable_base, ggtt->mappable_end);
2965
2966         /*
2967          * Initialise stolen early so that we may reserve preallocated
2968          * objects for the BIOS to KMS transition.
2969          */
2970         ret = i915_gem_init_stolen(dev_priv);
2971         if (ret)
2972                 goto out_gtt_cleanup;
2973
2974         return 0;
2975
2976 out_gtt_cleanup:
2977         ggtt->base.cleanup(&ggtt->base);
2978         return ret;
2979 }
2980
2981 int i915_ggtt_enable_hw(struct drm_i915_private *dev_priv)
2982 {
2983         if (INTEL_GEN(dev_priv) < 6 && !intel_enable_gtt())
2984                 return -EIO;
2985
2986         return 0;
2987 }
2988
2989 void i915_ggtt_enable_guc(struct drm_i915_private *i915)
2990 {
2991         i915->ggtt.invalidate = guc_ggtt_invalidate;
2992 }
2993
2994 void i915_ggtt_disable_guc(struct drm_i915_private *i915)
2995 {
2996         i915->ggtt.invalidate = gen6_ggtt_invalidate;
2997 }
2998
2999 void i915_gem_restore_gtt_mappings(struct drm_i915_private *dev_priv)
3000 {
3001         struct i915_ggtt *ggtt = &dev_priv->ggtt;
3002         struct drm_i915_gem_object *obj, *on;
3003
3004         i915_check_and_clear_faults(dev_priv);
3005
3006         /* First fill our portion of the GTT with scratch pages */
3007         ggtt->base.clear_range(&ggtt->base, 0, ggtt->base.total);
3008
3009         ggtt->base.closed = true; /* skip rewriting PTE on VMA unbind */
3010
3011         /* clflush objects bound into the GGTT and rebind them. */
3012         list_for_each_entry_safe(obj, on,
3013                                  &dev_priv->mm.bound_list, global_link) {
3014                 bool ggtt_bound = false;
3015                 struct i915_vma *vma;
3016
3017                 list_for_each_entry(vma, &obj->vma_list, obj_link) {
3018                         if (vma->vm != &ggtt->base)
3019                                 continue;
3020
3021                         if (!i915_vma_unbind(vma))
3022                                 continue;
3023
3024                         WARN_ON(i915_vma_bind(vma, obj->cache_level,
3025                                               PIN_UPDATE));
3026                         ggtt_bound = true;
3027                 }
3028
3029                 if (ggtt_bound)
3030                         WARN_ON(i915_gem_object_set_to_gtt_domain(obj, false));
3031         }
3032
3033         ggtt->base.closed = false;
3034
3035         if (INTEL_GEN(dev_priv) >= 8) {
3036                 if (IS_CHERRYVIEW(dev_priv) || IS_GEN9_LP(dev_priv))
3037                         chv_setup_private_ppat(dev_priv);
3038                 else
3039                         bdw_setup_private_ppat(dev_priv);
3040
3041                 return;
3042         }
3043
3044         if (USES_PPGTT(dev_priv)) {
3045                 struct i915_address_space *vm;
3046
3047                 list_for_each_entry(vm, &dev_priv->vm_list, global_link) {
3048                         struct i915_hw_ppgtt *ppgtt;
3049
3050                         if (i915_is_ggtt(vm))
3051                                 ppgtt = dev_priv->mm.aliasing_ppgtt;
3052                         else
3053                                 ppgtt = i915_vm_to_ppgtt(vm);
3054
3055                         gen6_write_page_range(ppgtt, 0, ppgtt->base.total);
3056                 }
3057         }
3058
3059         i915_ggtt_invalidate(dev_priv);
3060 }
3061
3062 static struct scatterlist *
3063 rotate_pages(const dma_addr_t *in, unsigned int offset,
3064              unsigned int width, unsigned int height,
3065              unsigned int stride,
3066              struct sg_table *st, struct scatterlist *sg)
3067 {
3068         unsigned int column, row;
3069         unsigned int src_idx;
3070
3071         for (column = 0; column < width; column++) {
3072                 src_idx = stride * (height - 1) + column;
3073                 for (row = 0; row < height; row++) {
3074                         st->nents++;
3075                         /* We don't need the pages, but need to initialize
3076                          * the entries so the sg list can be happily traversed.
3077                          * The only thing we need are DMA addresses.
3078                          */
3079                         sg_set_page(sg, NULL, PAGE_SIZE, 0);
3080                         sg_dma_address(sg) = in[offset + src_idx];
3081                         sg_dma_len(sg) = PAGE_SIZE;
3082                         sg = sg_next(sg);
3083                         src_idx -= stride;
3084                 }
3085         }
3086
3087         return sg;
3088 }
3089
3090 static noinline struct sg_table *
3091 intel_rotate_pages(struct intel_rotation_info *rot_info,
3092                    struct drm_i915_gem_object *obj)
3093 {
3094         const unsigned long n_pages = obj->base.size / PAGE_SIZE;
3095         unsigned int size = intel_rotation_info_size(rot_info);
3096         struct sgt_iter sgt_iter;
3097         dma_addr_t dma_addr;
3098         unsigned long i;
3099         dma_addr_t *page_addr_list;
3100         struct sg_table *st;
3101         struct scatterlist *sg;
3102         int ret = -ENOMEM;
3103
3104         /* Allocate a temporary list of source pages for random access. */
3105         page_addr_list = drm_malloc_gfp(n_pages,
3106                                         sizeof(dma_addr_t),
3107                                         GFP_TEMPORARY);
3108         if (!page_addr_list)
3109                 return ERR_PTR(ret);
3110
3111         /* Allocate target SG list. */
3112         st = kmalloc(sizeof(*st), GFP_KERNEL);
3113         if (!st)
3114                 goto err_st_alloc;
3115
3116         ret = sg_alloc_table(st, size, GFP_KERNEL);
3117         if (ret)
3118                 goto err_sg_alloc;
3119
3120         /* Populate source page list from the object. */
3121         i = 0;
3122         for_each_sgt_dma(dma_addr, sgt_iter, obj->mm.pages)
3123                 page_addr_list[i++] = dma_addr;
3124
3125         GEM_BUG_ON(i != n_pages);
3126         st->nents = 0;
3127         sg = st->sgl;
3128
3129         for (i = 0 ; i < ARRAY_SIZE(rot_info->plane); i++) {
3130                 sg = rotate_pages(page_addr_list, rot_info->plane[i].offset,
3131                                   rot_info->plane[i].width, rot_info->plane[i].height,
3132                                   rot_info->plane[i].stride, st, sg);
3133         }
3134
3135         DRM_DEBUG_KMS("Created rotated page mapping for object size %zu (%ux%u tiles, %u pages)\n",
3136                       obj->base.size, rot_info->plane[0].width, rot_info->plane[0].height, size);
3137
3138         drm_free_large(page_addr_list);
3139
3140         return st;
3141
3142 err_sg_alloc:
3143         kfree(st);
3144 err_st_alloc:
3145         drm_free_large(page_addr_list);
3146
3147         DRM_DEBUG_KMS("Failed to create rotated mapping for object size %zu! (%ux%u tiles, %u pages)\n",
3148                       obj->base.size, rot_info->plane[0].width, rot_info->plane[0].height, size);
3149
3150         return ERR_PTR(ret);
3151 }
3152
3153 static noinline struct sg_table *
3154 intel_partial_pages(const struct i915_ggtt_view *view,
3155                     struct drm_i915_gem_object *obj)
3156 {
3157         struct sg_table *st;
3158         struct scatterlist *sg, *iter;
3159         unsigned int count = view->partial.size;
3160         unsigned int offset;
3161         int ret = -ENOMEM;
3162
3163         st = kmalloc(sizeof(*st), GFP_KERNEL);
3164         if (!st)
3165                 goto err_st_alloc;
3166
3167         ret = sg_alloc_table(st, count, GFP_KERNEL);
3168         if (ret)
3169                 goto err_sg_alloc;
3170
3171         iter = i915_gem_object_get_sg(obj, view->partial.offset, &offset);
3172         GEM_BUG_ON(!iter);
3173
3174         sg = st->sgl;
3175         st->nents = 0;
3176         do {
3177                 unsigned int len;
3178
3179                 len = min(iter->length - (offset << PAGE_SHIFT),
3180                           count << PAGE_SHIFT);
3181                 sg_set_page(sg, NULL, len, 0);
3182                 sg_dma_address(sg) =
3183                         sg_dma_address(iter) + (offset << PAGE_SHIFT);
3184                 sg_dma_len(sg) = len;
3185
3186                 st->nents++;
3187                 count -= len >> PAGE_SHIFT;
3188                 if (count == 0) {
3189                         sg_mark_end(sg);
3190                         return st;
3191                 }
3192
3193                 sg = __sg_next(sg);
3194                 iter = __sg_next(iter);
3195                 offset = 0;
3196         } while (1);
3197
3198 err_sg_alloc:
3199         kfree(st);
3200 err_st_alloc:
3201         return ERR_PTR(ret);
3202 }
3203
3204 static int
3205 i915_get_ggtt_vma_pages(struct i915_vma *vma)
3206 {
3207         int ret;
3208
3209         /* The vma->pages are only valid within the lifespan of the borrowed
3210          * obj->mm.pages. When the obj->mm.pages sg_table is regenerated, so
3211          * must be the vma->pages. A simple rule is that vma->pages must only
3212          * be accessed when the obj->mm.pages are pinned.
3213          */
3214         GEM_BUG_ON(!i915_gem_object_has_pinned_pages(vma->obj));
3215
3216         switch (vma->ggtt_view.type) {
3217         case I915_GGTT_VIEW_NORMAL:
3218                 vma->pages = vma->obj->mm.pages;
3219                 return 0;
3220
3221         case I915_GGTT_VIEW_ROTATED:
3222                 vma->pages =
3223                         intel_rotate_pages(&vma->ggtt_view.rotated, vma->obj);
3224                 break;
3225
3226         case I915_GGTT_VIEW_PARTIAL:
3227                 vma->pages = intel_partial_pages(&vma->ggtt_view, vma->obj);
3228                 break;
3229
3230         default:
3231                 WARN_ONCE(1, "GGTT view %u not implemented!\n",
3232                           vma->ggtt_view.type);
3233                 return -EINVAL;
3234         }
3235
3236         ret = 0;
3237         if (unlikely(IS_ERR(vma->pages))) {
3238                 ret = PTR_ERR(vma->pages);
3239                 vma->pages = NULL;
3240                 DRM_ERROR("Failed to get pages for VMA view type %u (%d)!\n",
3241                           vma->ggtt_view.type, ret);
3242         }
3243         return ret;
3244 }
3245
3246 /**
3247  * i915_gem_gtt_reserve - reserve a node in an address_space (GTT)
3248  * @vm: the &struct i915_address_space
3249  * @node: the &struct drm_mm_node (typically i915_vma.mode)
3250  * @size: how much space to allocate inside the GTT,
3251  *        must be #I915_GTT_PAGE_SIZE aligned
3252  * @offset: where to insert inside the GTT,
3253  *          must be #I915_GTT_MIN_ALIGNMENT aligned, and the node
3254  *          (@offset + @size) must fit within the address space
3255  * @color: color to apply to node, if this node is not from a VMA,
3256  *         color must be #I915_COLOR_UNEVICTABLE
3257  * @flags: control search and eviction behaviour
3258  *
3259  * i915_gem_gtt_reserve() tries to insert the @node at the exact @offset inside
3260  * the address space (using @size and @color). If the @node does not fit, it
3261  * tries to evict any overlapping nodes from the GTT, including any
3262  * neighbouring nodes if the colors do not match (to ensure guard pages between
3263  * differing domains). See i915_gem_evict_for_node() for the gory details
3264  * on the eviction algorithm. #PIN_NONBLOCK may used to prevent waiting on
3265  * evicting active overlapping objects, and any overlapping node that is pinned
3266  * or marked as unevictable will also result in failure.
3267  *
3268  * Returns: 0 on success, -ENOSPC if no suitable hole is found, -EINTR if
3269  * asked to wait for eviction and interrupted.
3270  */
3271 int i915_gem_gtt_reserve(struct i915_address_space *vm,
3272                          struct drm_mm_node *node,
3273                          u64 size, u64 offset, unsigned long color,
3274                          unsigned int flags)
3275 {
3276         int err;
3277
3278         GEM_BUG_ON(!size);
3279         GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE));
3280         GEM_BUG_ON(!IS_ALIGNED(offset, I915_GTT_MIN_ALIGNMENT));
3281         GEM_BUG_ON(range_overflows(offset, size, vm->total));
3282         GEM_BUG_ON(vm == &vm->i915->mm.aliasing_ppgtt->base);
3283         GEM_BUG_ON(drm_mm_node_allocated(node));
3284
3285         node->size = size;
3286         node->start = offset;
3287         node->color = color;
3288
3289         err = drm_mm_reserve_node(&vm->mm, node);
3290         if (err != -ENOSPC)
3291                 return err;
3292
3293         err = i915_gem_evict_for_node(vm, node, flags);
3294         if (err == 0)
3295                 err = drm_mm_reserve_node(&vm->mm, node);
3296
3297         return err;
3298 }
3299
3300 static u64 random_offset(u64 start, u64 end, u64 len, u64 align)
3301 {
3302         u64 range, addr;
3303
3304         GEM_BUG_ON(range_overflows(start, len, end));
3305         GEM_BUG_ON(round_up(start, align) > round_down(end - len, align));
3306
3307         range = round_down(end - len, align) - round_up(start, align);
3308         if (range) {
3309                 if (sizeof(unsigned long) == sizeof(u64)) {
3310                         addr = get_random_long();
3311                 } else {
3312                         addr = get_random_int();
3313                         if (range > U32_MAX) {
3314                                 addr <<= 32;
3315                                 addr |= get_random_int();
3316                         }
3317                 }
3318                 div64_u64_rem(addr, range, &addr);
3319                 start += addr;
3320         }
3321
3322         return round_up(start, align);
3323 }
3324
3325 /**
3326  * i915_gem_gtt_insert - insert a node into an address_space (GTT)
3327  * @vm: the &struct i915_address_space
3328  * @node: the &struct drm_mm_node (typically i915_vma.node)
3329  * @size: how much space to allocate inside the GTT,
3330  *        must be #I915_GTT_PAGE_SIZE aligned
3331  * @alignment: required alignment of starting offset, may be 0 but
3332  *             if specified, this must be a power-of-two and at least
3333  *             #I915_GTT_MIN_ALIGNMENT
3334  * @color: color to apply to node
3335  * @start: start of any range restriction inside GTT (0 for all),
3336  *         must be #I915_GTT_PAGE_SIZE aligned
3337  * @end: end of any range restriction inside GTT (U64_MAX for all),
3338  *       must be #I915_GTT_PAGE_SIZE aligned if not U64_MAX
3339  * @flags: control search and eviction behaviour
3340  *
3341  * i915_gem_gtt_insert() first searches for an available hole into which
3342  * is can insert the node. The hole address is aligned to @alignment and
3343  * its @size must then fit entirely within the [@start, @end] bounds. The
3344  * nodes on either side of the hole must match @color, or else a guard page
3345  * will be inserted between the two nodes (or the node evicted). If no
3346  * suitable hole is found, first a victim is randomly selected and tested
3347  * for eviction, otherwise then the LRU list of objects within the GTT
3348  * is scanned to find the first set of replacement nodes to create the hole.
3349  * Those old overlapping nodes are evicted from the GTT (and so must be
3350  * rebound before any future use). Any node that is currently pinned cannot
3351  * be evicted (see i915_vma_pin()). Similar if the node's VMA is currently
3352  * active and #PIN_NONBLOCK is specified, that node is also skipped when
3353  * searching for an eviction candidate. See i915_gem_evict_something() for
3354  * the gory details on the eviction algorithm.
3355  *
3356  * Returns: 0 on success, -ENOSPC if no suitable hole is found, -EINTR if
3357  * asked to wait for eviction and interrupted.
3358  */
3359 int i915_gem_gtt_insert(struct i915_address_space *vm,
3360                         struct drm_mm_node *node,
3361                         u64 size, u64 alignment, unsigned long color,
3362                         u64 start, u64 end, unsigned int flags)
3363 {
3364         enum drm_mm_insert_mode mode;
3365         u64 offset;
3366         int err;
3367
3368         lockdep_assert_held(&vm->i915->drm.struct_mutex);
3369         GEM_BUG_ON(!size);
3370         GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE));
3371         GEM_BUG_ON(alignment && !is_power_of_2(alignment));
3372         GEM_BUG_ON(alignment && !IS_ALIGNED(alignment, I915_GTT_MIN_ALIGNMENT));
3373         GEM_BUG_ON(start >= end);
3374         GEM_BUG_ON(start > 0  && !IS_ALIGNED(start, I915_GTT_PAGE_SIZE));
3375         GEM_BUG_ON(end < U64_MAX && !IS_ALIGNED(end, I915_GTT_PAGE_SIZE));
3376         GEM_BUG_ON(vm == &vm->i915->mm.aliasing_ppgtt->base);
3377         GEM_BUG_ON(drm_mm_node_allocated(node));
3378
3379         if (unlikely(range_overflows(start, size, end)))
3380                 return -ENOSPC;
3381
3382         if (unlikely(round_up(start, alignment) > round_down(end - size, alignment)))
3383                 return -ENOSPC;
3384
3385         mode = DRM_MM_INSERT_BEST;
3386         if (flags & PIN_HIGH)
3387                 mode = DRM_MM_INSERT_HIGH;
3388         if (flags & PIN_MAPPABLE)
3389                 mode = DRM_MM_INSERT_LOW;
3390
3391         /* We only allocate in PAGE_SIZE/GTT_PAGE_SIZE (4096) chunks,
3392          * so we know that we always have a minimum alignment of 4096.
3393          * The drm_mm range manager is optimised to return results
3394          * with zero alignment, so where possible use the optimal
3395          * path.
3396          */
3397         BUILD_BUG_ON(I915_GTT_MIN_ALIGNMENT > I915_GTT_PAGE_SIZE);
3398         if (alignment <= I915_GTT_MIN_ALIGNMENT)
3399                 alignment = 0;
3400
3401         err = drm_mm_insert_node_in_range(&vm->mm, node,
3402                                           size, alignment, color,
3403                                           start, end, mode);
3404         if (err != -ENOSPC)
3405                 return err;
3406
3407         /* No free space, pick a slot at random.
3408          *
3409          * There is a pathological case here using a GTT shared between
3410          * mmap and GPU (i.e. ggtt/aliasing_ppgtt but not full-ppgtt):
3411          *
3412          *    |<-- 256 MiB aperture -->||<-- 1792 MiB unmappable -->|
3413          *         (64k objects)             (448k objects)
3414          *
3415          * Now imagine that the eviction LRU is ordered top-down (just because
3416          * pathology meets real life), and that we need to evict an object to
3417          * make room inside the aperture. The eviction scan then has to walk
3418          * the 448k list before it finds one within range. And now imagine that
3419          * it has to search for a new hole between every byte inside the memcpy,
3420          * for several simultaneous clients.
3421          *
3422          * On a full-ppgtt system, if we have run out of available space, there
3423          * will be lots and lots of objects in the eviction list! Again,
3424          * searching that LRU list may be slow if we are also applying any
3425          * range restrictions (e.g. restriction to low 4GiB) and so, for
3426          * simplicity and similarilty between different GTT, try the single
3427          * random replacement first.
3428          */
3429         offset = random_offset(start, end,
3430                                size, alignment ?: I915_GTT_MIN_ALIGNMENT);
3431         err = i915_gem_gtt_reserve(vm, node, size, offset, color, flags);
3432         if (err != -ENOSPC)
3433                 return err;
3434
3435         /* Randomly selected placement is pinned, do a search */
3436         err = i915_gem_evict_something(vm, size, alignment, color,
3437                                        start, end, flags);
3438         if (err)
3439                 return err;
3440
3441         return drm_mm_insert_node_in_range(&vm->mm, node,
3442                                            size, alignment, color,
3443                                            start, end, DRM_MM_INSERT_EVICT);
3444 }
3445
3446 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
3447 #include "selftests/mock_gtt.c"
3448 #include "selftests/i915_gem_gtt.c"
3449 #endif