drivers/gpu/drm/i915/i915_gem_gtt.c

   1 /*
   2  * Copyright © 2010 Daniel Vetter
   3  * Copyright © 2011-2014 Intel Corporation
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9  * and/or sell copies of the Software, and to permit persons to whom the
  10  * Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice (including the next
  13  * paragraph) shall be included in all copies or substantial portions of the
  14  * Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  21  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  22  * IN THE SOFTWARE.
  23  *
  24  */
  25
  26 #include <linux/slab.h> /* fault-inject.h is not standalone! */
  27
  28 #include <linux/fault-inject.h>
  29 #include <linux/log2.h>
  30 #include <linux/random.h>
  31 #include <linux/seq_file.h>
  32 #include <linux/stop_machine.h>
  33
  34 #include <asm/set_memory.h>
  35
  36 #include <drm/drmP.h>
  37 #include <drm/i915_drm.h>
  38
  39 #include "i915_drv.h"
  40 #include "i915_vgpu.h"
  41 #include "i915_trace.h"
  42 #include "intel_drv.h"
  43 #include "intel_frontbuffer.h"
  44
  45 #define I915_GFP_DMA (GFP_KERNEL | __GFP_HIGHMEM)
  46
  47 /**
  48  * DOC: Global GTT views
  49  *
  50  * Background and previous state
  51  *
  52  * Historically objects could exists (be bound) in global GTT space only as
  53  * singular instances with a view representing all of the object's backing pages
  54  * in a linear fashion. This view will be called a normal view.
  55  *
  56  * To support multiple views of the same object, where the number of mapped
  57  * pages is not equal to the backing store, or where the layout of the pages
  58  * is not linear, concept of a GGTT view was added.
  59  *
  60  * One example of an alternative view is a stereo display driven by a single
  61  * image. In this case we would have a framebuffer looking like this
  62  * (2x2 pages):
  63  *
  64  *    12
  65  *    34
  66  *
  67  * Above would represent a normal GGTT view as normally mapped for GPU or CPU
  68  * rendering. In contrast, fed to the display engine would be an alternative
  69  * view which could look something like this:
  70  *
  71  *   1212
  72  *   3434
  73  *
  74  * In this example both the size and layout of pages in the alternative view is
  75  * different from the normal view.
  76  *
  77  * Implementation and usage
  78  *
  79  * GGTT views are implemented using VMAs and are distinguished via enum
  80  * i915_ggtt_view_type and struct i915_ggtt_view.
  81  *
  82  * A new flavour of core GEM functions which work with GGTT bound objects were
  83  * added with the _ggtt_ infix, and sometimes with _view postfix to avoid
  84  * renaming  in large amounts of code. They take the struct i915_ggtt_view
  85  * parameter encapsulating all metadata required to implement a view.
  86  *
  87  * As a helper for callers which are only interested in the normal view,
  88  * globally const i915_ggtt_view_normal singleton instance exists. All old core
  89  * GEM API functions, the ones not taking the view parameter, are operating on,
  90  * or with the normal GGTT view.
  91  *
  92  * Code wanting to add or use a new GGTT view needs to:
  93  *
  94  * 1. Add a new enum with a suitable name.
  95  * 2. Extend the metadata in the i915_ggtt_view structure if required.
  96  * 3. Add support to i915_get_vma_pages().
  97  *
  98  * New views are required to build a scatter-gather table from within the
  99  * i915_get_vma_pages function. This table is stored in the vma.ggtt_view and
 100  * exists for the lifetime of an VMA.
 101  *
 102  * Core API is designed to have copy semantics which means that passed in
 103  * struct i915_ggtt_view does not need to be persistent (left around after
 104  * calling the core API functions).
 105  *
 106  */
 107
 108 static int
 109 i915_get_ggtt_vma_pages(struct i915_vma *vma);
 110
 111 static void gen6_ggtt_invalidate(struct drm_i915_private *dev_priv)
 112 {
 113         /* Note that as an uncached mmio write, this should flush the
 114          * WCB of the writes into the GGTT before it triggers the invalidate.
 115          */
 116         I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
 117 }
 118
 119 static void guc_ggtt_invalidate(struct drm_i915_private *dev_priv)
 120 {
 121         gen6_ggtt_invalidate(dev_priv);
 122         I915_WRITE(GEN8_GTCR, GEN8_GTCR_INVALIDATE);
 123 }
 124
 125 static void gmch_ggtt_invalidate(struct drm_i915_private *dev_priv)
 126 {
 127         intel_gtt_chipset_flush();
 128 }
 129
 130 static inline void i915_ggtt_invalidate(struct drm_i915_private *i915)
 131 {
 132         i915->ggtt.invalidate(i915);
 133 }
 134
 135 int intel_sanitize_enable_ppgtt(struct drm_i915_private *dev_priv,
 136                                 int enable_ppgtt)
 137 {
 138         bool has_full_ppgtt;
 139         bool has_full_48bit_ppgtt;
 140
 141         if (!dev_priv->info.has_aliasing_ppgtt)
 142                 return 0;
 143
 144         has_full_ppgtt = dev_priv->info.has_full_ppgtt;
 145         has_full_48bit_ppgtt = dev_priv->info.has_full_48bit_ppgtt;
 146
 147         if (intel_vgpu_active(dev_priv)) {
 148                 /* GVT-g has no support for 32bit ppgtt */
 149                 has_full_ppgtt = false;
 150                 has_full_48bit_ppgtt = intel_vgpu_has_full_48bit_ppgtt(dev_priv);
 151         }
 152
 153         /*
 154          * We don't allow disabling PPGTT for gen9+ as it's a requirement for
 155          * execlists, the sole mechanism available to submit work.
 156          */
 157         if (enable_ppgtt == 0 && INTEL_GEN(dev_priv) < 9)
 158                 return 0;
 159
 160         if (enable_ppgtt == 1)
 161                 return 1;
 162
 163         if (enable_ppgtt == 2 && has_full_ppgtt)
 164                 return 2;
 165
 166         if (enable_ppgtt == 3 && has_full_48bit_ppgtt)
 167                 return 3;
 168
 169         /* Disable ppgtt on SNB if VT-d is on. */
 170         if (IS_GEN6(dev_priv) && intel_vtd_active()) {
 171                 DRM_INFO("Disabling PPGTT because VT-d is on\n");
 172                 return 0;
 173         }
 174
 175         /* Early VLV doesn't have this */
 176         if (IS_VALLEYVIEW(dev_priv) && dev_priv->drm.pdev->revision < 0xb) {
 177                 DRM_DEBUG_DRIVER("disabling PPGTT on pre-B3 step VLV\n");
 178                 return 0;
 179         }
 180
 181         if (HAS_LOGICAL_RING_CONTEXTS(dev_priv)) {
 182                 if (has_full_48bit_ppgtt)
 183                         return 3;
 184
 185                 if (has_full_ppgtt)
 186                         return 2;
 187         }
 188
 189         return 1;
 190 }
 191
 192 static int ppgtt_bind_vma(struct i915_vma *vma,
 193                           enum i915_cache_level cache_level,
 194                           u32 unused)
 195 {
 196         u32 pte_flags;
 197         int ret;
 198
 199         if (!(vma->flags & I915_VMA_LOCAL_BIND)) {
 200                 ret = vma->vm->allocate_va_range(vma->vm, vma->node.start,
 201                                                  vma->size);
 202                 if (ret)
 203                         return ret;
 204         }
 205
 206         /* Currently applicable only to VLV */
 207         pte_flags = 0;
 208         if (vma->obj->gt_ro)
 209                 pte_flags |= PTE_READ_ONLY;
 210
 211         vma->vm->insert_entries(vma->vm, vma, cache_level, pte_flags);
 212
 213         return 0;
 214 }
 215
 216 static void ppgtt_unbind_vma(struct i915_vma *vma)
 217 {
 218         vma->vm->clear_range(vma->vm, vma->node.start, vma->size);
 219 }
 220
 221 static int ppgtt_set_pages(struct i915_vma *vma)
 222 {
 223         GEM_BUG_ON(vma->pages);
 224
 225         vma->pages = vma->obj->mm.pages;
 226
 227         vma->page_sizes = vma->obj->mm.page_sizes;
 228
 229         return 0;
 230 }
 231
 232 static void clear_pages(struct i915_vma *vma)
 233 {
 234         GEM_BUG_ON(!vma->pages);
 235
 236         if (vma->pages != vma->obj->mm.pages) {
 237                 sg_free_table(vma->pages);
 238                 kfree(vma->pages);
 239         }
 240         vma->pages = NULL;
 241
 242         memset(&vma->page_sizes, 0, sizeof(vma->page_sizes));
 243 }
 244
 245 static gen8_pte_t gen8_pte_encode(dma_addr_t addr,
 246                                   enum i915_cache_level level)
 247 {
 248         gen8_pte_t pte = _PAGE_PRESENT | _PAGE_RW;
 249         pte |= addr;
 250
 251         switch (level) {
 252         case I915_CACHE_NONE:
 253                 pte |= PPAT_UNCACHED;
 254                 break;
 255         case I915_CACHE_WT:
 256                 pte |= PPAT_DISPLAY_ELLC;
 257                 break;
 258         default:
 259                 pte |= PPAT_CACHED;
 260                 break;
 261         }
 262
 263         return pte;
 264 }
 265
 266 static gen8_pde_t gen8_pde_encode(const dma_addr_t addr,
 267                                   const enum i915_cache_level level)
 268 {
 269         gen8_pde_t pde = _PAGE_PRESENT | _PAGE_RW;
 270         pde |= addr;
 271         if (level != I915_CACHE_NONE)
 272                 pde |= PPAT_CACHED_PDE;
 273         else
 274                 pde |= PPAT_UNCACHED;
 275         return pde;
 276 }
 277
 278 #define gen8_pdpe_encode gen8_pde_encode
 279 #define gen8_pml4e_encode gen8_pde_encode
 280
 281 static gen6_pte_t snb_pte_encode(dma_addr_t addr,
 282                                  enum i915_cache_level level,
 283                                  u32 unused)
 284 {
 285         gen6_pte_t pte = GEN6_PTE_VALID;
 286         pte |= GEN6_PTE_ADDR_ENCODE(addr);
 287
 288         switch (level) {
 289         case I915_CACHE_L3_LLC:
 290         case I915_CACHE_LLC:
 291                 pte |= GEN6_PTE_CACHE_LLC;
 292                 break;
 293         case I915_CACHE_NONE:
 294                 pte |= GEN6_PTE_UNCACHED;
 295                 break;
 296         default:
 297                 MISSING_CASE(level);
 298         }
 299
 300         return pte;
 301 }
 302
 303 static gen6_pte_t ivb_pte_encode(dma_addr_t addr,
 304                                  enum i915_cache_level level,
 305                                  u32 unused)
 306 {
 307         gen6_pte_t pte = GEN6_PTE_VALID;
 308         pte |= GEN6_PTE_ADDR_ENCODE(addr);
 309
 310         switch (level) {
 311         case I915_CACHE_L3_LLC:
 312                 pte |= GEN7_PTE_CACHE_L3_LLC;
 313                 break;
 314         case I915_CACHE_LLC:
 315                 pte |= GEN6_PTE_CACHE_LLC;
 316                 break;
 317         case I915_CACHE_NONE:
 318                 pte |= GEN6_PTE_UNCACHED;
 319                 break;
 320         default:
 321                 MISSING_CASE(level);
 322         }
 323
 324         return pte;
 325 }
 326
 327 static gen6_pte_t byt_pte_encode(dma_addr_t addr,
 328                                  enum i915_cache_level level,
 329                                  u32 flags)
 330 {
 331         gen6_pte_t pte = GEN6_PTE_VALID;
 332         pte |= GEN6_PTE_ADDR_ENCODE(addr);
 333
 334         if (!(flags & PTE_READ_ONLY))
 335                 pte |= BYT_PTE_WRITEABLE;
 336
 337         if (level != I915_CACHE_NONE)
 338                 pte |= BYT_PTE_SNOOPED_BY_CPU_CACHES;
 339
 340         return pte;
 341 }
 342
 343 static gen6_pte_t hsw_pte_encode(dma_addr_t addr,
 344                                  enum i915_cache_level level,
 345                                  u32 unused)
 346 {
 347         gen6_pte_t pte = GEN6_PTE_VALID;
 348         pte |= HSW_PTE_ADDR_ENCODE(addr);
 349
 350         if (level != I915_CACHE_NONE)
 351                 pte |= HSW_WB_LLC_AGE3;
 352
 353         return pte;
 354 }
 355
 356 static gen6_pte_t iris_pte_encode(dma_addr_t addr,
 357                                   enum i915_cache_level level,
 358                                   u32 unused)
 359 {
 360         gen6_pte_t pte = GEN6_PTE_VALID;
 361         pte |= HSW_PTE_ADDR_ENCODE(addr);
 362
 363         switch (level) {
 364         case I915_CACHE_NONE:
 365                 break;
 366         case I915_CACHE_WT:
 367                 pte |= HSW_WT_ELLC_LLC_AGE3;
 368                 break;
 369         default:
 370                 pte |= HSW_WB_ELLC_LLC_AGE3;
 371                 break;
 372         }
 373
 374         return pte;
 375 }
 376
 377 static struct page *vm_alloc_page(struct i915_address_space *vm, gfp_t gfp)
 378 {
 379         struct pagevec *pvec = &vm->free_pages;
 380         struct pagevec stash;
 381
 382         if (I915_SELFTEST_ONLY(should_fail(&vm->fault_attr, 1)))
 383                 i915_gem_shrink_all(vm->i915);
 384
 385         if (likely(pvec->nr))
 386                 return pvec->pages[--pvec->nr];
 387
 388         if (!vm->pt_kmap_wc)
 389                 return alloc_page(gfp);
 390
 391         /* A placeholder for a specific mutex to guard the WC stash */
 392         lockdep_assert_held(&vm->i915->drm.struct_mutex);
 393
 394         /* Look in our global stash of WC pages... */
 395         pvec = &vm->i915->mm.wc_stash;
 396         if (likely(pvec->nr))
 397                 return pvec->pages[--pvec->nr];
 398
 399         /*
 400          * Otherwise batch allocate pages to amoritize cost of set_pages_wc.
 401          *
 402          * We have to be careful as page allocation may trigger the shrinker
 403          * (via direct reclaim) which will fill up the WC stash underneath us.
 404          * So we add our WB pages into a temporary pvec on the stack and merge
 405          * them into the WC stash after all the allocations are complete.
 406          */
 407         pagevec_init(&stash);
 408         do {
 409                 struct page *page;
 410
 411                 page = alloc_page(gfp);
 412                 if (unlikely(!page))
 413                         break;
 414
 415                 stash.pages[stash.nr++] = page;
 416         } while (stash.nr < pagevec_space(pvec));
 417
 418         if (stash.nr) {
 419                 int nr = min_t(int, stash.nr, pagevec_space(pvec));
 420                 struct page **pages = stash.pages + stash.nr - nr;
 421
 422                 if (nr && !set_pages_array_wc(pages, nr)) {
 423                         memcpy(pvec->pages + pvec->nr,
 424                                pages, sizeof(pages[0]) * nr);
 425                         pvec->nr += nr;
 426                         stash.nr -= nr;
 427                 }
 428
 429                 pagevec_release(&stash);
 430         }
 431
 432         return likely(pvec->nr) ? pvec->pages[--pvec->nr] : NULL;
 433 }
 434
 435 static void vm_free_pages_release(struct i915_address_space *vm,
 436                                   bool immediate)
 437 {
 438         struct pagevec *pvec = &vm->free_pages;
 439
 440         GEM_BUG_ON(!pagevec_count(pvec));
 441
 442         if (vm->pt_kmap_wc) {
 443                 struct pagevec *stash = &vm->i915->mm.wc_stash;
 444
 445                 /* When we use WC, first fill up the global stash and then
 446                  * only if full immediately free the overflow.
 447                  */
 448
 449                 lockdep_assert_held(&vm->i915->drm.struct_mutex);
 450                 if (pagevec_space(stash)) {
 451                         do {
 452                                 stash->pages[stash->nr++] =
 453                                         pvec->pages[--pvec->nr];
 454                                 if (!pvec->nr)
 455                                         return;
 456                         } while (pagevec_space(stash));
 457
 458                         /* As we have made some room in the VM's free_pages,
 459                          * we can wait for it to fill again. Unless we are
 460                          * inside i915_address_space_fini() and must
 461                          * immediately release the pages!
 462                          */
 463                         if (!immediate)
 464                                 return;
 465                 }
 466
 467                 set_pages_array_wb(pvec->pages, pvec->nr);
 468         }
 469
 470         __pagevec_release(pvec);
 471 }
 472
 473 static void vm_free_page(struct i915_address_space *vm, struct page *page)
 474 {
 475         /*
 476          * On !llc, we need to change the pages back to WB. We only do so
 477          * in bulk, so we rarely need to change the page attributes here,
 478          * but doing so requires a stop_machine() from deep inside arch/x86/mm.
 479          * To make detection of the possible sleep more likely, use an
 480          * unconditional might_sleep() for everybody.
 481          */
 482         might_sleep();
 483         if (!pagevec_add(&vm->free_pages, page))
 484                 vm_free_pages_release(vm, false);
 485 }
 486
 487 static int __setup_page_dma(struct i915_address_space *vm,
 488                             struct i915_page_dma *p,
 489                             gfp_t gfp)
 490 {
 491         p->page = vm_alloc_page(vm, gfp | __GFP_NOWARN | __GFP_NORETRY);
 492         if (unlikely(!p->page))
 493                 return -ENOMEM;
 494
 495         p->daddr = dma_map_page(vm->dma, p->page, 0, PAGE_SIZE,
 496                                 PCI_DMA_BIDIRECTIONAL);
 497         if (unlikely(dma_mapping_error(vm->dma, p->daddr))) {
 498                 vm_free_page(vm, p->page);
 499                 return -ENOMEM;
 500         }
 501
 502         return 0;
 503 }
 504
 505 static int setup_page_dma(struct i915_address_space *vm,
 506                           struct i915_page_dma *p)
 507 {
 508         return __setup_page_dma(vm, p, I915_GFP_DMA);
 509 }
 510
 511 static void cleanup_page_dma(struct i915_address_space *vm,
 512                              struct i915_page_dma *p)
 513 {
 514         dma_unmap_page(vm->dma, p->daddr, PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
 515         vm_free_page(vm, p->page);
 516 }
 517
 518 #define kmap_atomic_px(px) kmap_atomic(px_base(px)->page)
 519
 520 #define setup_px(vm, px) setup_page_dma((vm), px_base(px))
 521 #define cleanup_px(vm, px) cleanup_page_dma((vm), px_base(px))
 522 #define fill_px(ppgtt, px, v) fill_page_dma((vm), px_base(px), (v))
 523 #define fill32_px(ppgtt, px, v) fill_page_dma_32((vm), px_base(px), (v))
 524
 525 static void fill_page_dma(struct i915_address_space *vm,
 526                           struct i915_page_dma *p,
 527                           const u64 val)
 528 {
 529         u64 * const vaddr = kmap_atomic(p->page);
 530
 531         memset64(vaddr, val, PAGE_SIZE / sizeof(val));
 532
 533         kunmap_atomic(vaddr);
 534 }
 535
 536 static void fill_page_dma_32(struct i915_address_space *vm,
 537                              struct i915_page_dma *p,
 538                              const u32 v)
 539 {
 540         fill_page_dma(vm, p, (u64)v << 32 | v);
 541 }
 542
 543 static int
 544 setup_scratch_page(struct i915_address_space *vm, gfp_t gfp)
 545 {
 546         unsigned long size;
 547
 548         /*
 549          * In order to utilize 64K pages for an object with a size < 2M, we will
 550          * need to support a 64K scratch page, given that every 16th entry for a
 551          * page-table operating in 64K mode must point to a properly aligned 64K
 552          * region, including any PTEs which happen to point to scratch.
 553          *
 554          * This is only relevant for the 48b PPGTT where we support
 555          * huge-gtt-pages, see also i915_vma_insert().
 556          *
 557          * TODO: we should really consider write-protecting the scratch-page and
 558          * sharing between ppgtt
 559          */
 560         size = I915_GTT_PAGE_SIZE_4K;
 561         if (i915_vm_is_48bit(vm) &&
 562             HAS_PAGE_SIZES(vm->i915, I915_GTT_PAGE_SIZE_64K)) {
 563                 size = I915_GTT_PAGE_SIZE_64K;
 564                 gfp |= __GFP_NOWARN;
 565         }
 566         gfp |= __GFP_ZERO | __GFP_RETRY_MAYFAIL;
 567
 568         do {
 569                 int order = get_order(size);
 570                 struct page *page;
 571                 dma_addr_t addr;
 572
 573                 page = alloc_pages(gfp, order);
 574                 if (unlikely(!page))
 575                         goto skip;
 576
 577                 addr = dma_map_page(vm->dma, page, 0, size,
 578                                     PCI_DMA_BIDIRECTIONAL);
 579                 if (unlikely(dma_mapping_error(vm->dma, addr)))
 580                         goto free_page;
 581
 582                 if (unlikely(!IS_ALIGNED(addr, size)))
 583                         goto unmap_page;
 584
 585                 vm->scratch_page.page = page;
 586                 vm->scratch_page.daddr = addr;
 587                 vm->scratch_page.order = order;
 588                 return 0;
 589
 590 unmap_page:
 591                 dma_unmap_page(vm->dma, addr, size, PCI_DMA_BIDIRECTIONAL);
 592 free_page:
 593                 __free_pages(page, order);
 594 skip:
 595                 if (size == I915_GTT_PAGE_SIZE_4K)
 596                         return -ENOMEM;
 597
 598                 size = I915_GTT_PAGE_SIZE_4K;
 599                 gfp &= ~__GFP_NOWARN;
 600         } while (1);
 601 }
 602
 603 static void cleanup_scratch_page(struct i915_address_space *vm)
 604 {
 605         struct i915_page_dma *p = &vm->scratch_page;
 606
 607         dma_unmap_page(vm->dma, p->daddr, BIT(p->order) << PAGE_SHIFT,
 608                        PCI_DMA_BIDIRECTIONAL);
 609         __free_pages(p->page, p->order);
 610 }
 611
 612 static struct i915_page_table *alloc_pt(struct i915_address_space *vm)
 613 {
 614         struct i915_page_table *pt;
 615
 616         pt = kmalloc(sizeof(*pt), GFP_KERNEL | __GFP_NOWARN);
 617         if (unlikely(!pt))
 618                 return ERR_PTR(-ENOMEM);
 619
 620         if (unlikely(setup_px(vm, pt))) {
 621                 kfree(pt);
 622                 return ERR_PTR(-ENOMEM);
 623         }
 624
 625         pt->used_ptes = 0;
 626         return pt;
 627 }
 628
 629 static void free_pt(struct i915_address_space *vm, struct i915_page_table *pt)
 630 {
 631         cleanup_px(vm, pt);
 632         kfree(pt);
 633 }
 634
 635 static void gen8_initialize_pt(struct i915_address_space *vm,
 636                                struct i915_page_table *pt)
 637 {
 638         fill_px(vm, pt,
 639                 gen8_pte_encode(vm->scratch_page.daddr, I915_CACHE_LLC));
 640 }
 641
 642 static void gen6_initialize_pt(struct i915_address_space *vm,
 643                                struct i915_page_table *pt)
 644 {
 645         fill32_px(vm, pt,
 646                   vm->pte_encode(vm->scratch_page.daddr, I915_CACHE_LLC, 0));
 647 }
 648
 649 static struct i915_page_directory *alloc_pd(struct i915_address_space *vm)
 650 {
 651         struct i915_page_directory *pd;
 652
 653         pd = kzalloc(sizeof(*pd), GFP_KERNEL | __GFP_NOWARN);
 654         if (unlikely(!pd))
 655                 return ERR_PTR(-ENOMEM);
 656
 657         if (unlikely(setup_px(vm, pd))) {
 658                 kfree(pd);
 659                 return ERR_PTR(-ENOMEM);
 660         }
 661
 662         pd->used_pdes = 0;
 663         return pd;
 664 }
 665
 666 static void free_pd(struct i915_address_space *vm,
 667                     struct i915_page_directory *pd)
 668 {
 669         cleanup_px(vm, pd);
 670         kfree(pd);
 671 }
 672
 673 static void gen8_initialize_pd(struct i915_address_space *vm,
 674                                struct i915_page_directory *pd)
 675 {
 676         fill_px(vm, pd,
 677                 gen8_pde_encode(px_dma(vm->scratch_pt), I915_CACHE_LLC));
 678         memset_p((void **)pd->page_table, vm->scratch_pt, I915_PDES);
 679 }
 680
 681 static int __pdp_init(struct i915_address_space *vm,
 682                       struct i915_page_directory_pointer *pdp)
 683 {
 684         const unsigned int pdpes = i915_pdpes_per_pdp(vm);
 685
 686         pdp->page_directory = kmalloc_array(pdpes, sizeof(*pdp->page_directory),
 687                                             GFP_KERNEL | __GFP_NOWARN);
 688         if (unlikely(!pdp->page_directory))
 689                 return -ENOMEM;
 690
 691         memset_p((void **)pdp->page_directory, vm->scratch_pd, pdpes);
 692
 693         return 0;
 694 }
 695
 696 static void __pdp_fini(struct i915_page_directory_pointer *pdp)
 697 {
 698         kfree(pdp->page_directory);
 699         pdp->page_directory = NULL;
 700 }
 701
 702 static inline bool use_4lvl(const struct i915_address_space *vm)
 703 {
 704         return i915_vm_is_48bit(vm);
 705 }
 706
 707 static struct i915_page_directory_pointer *
 708 alloc_pdp(struct i915_address_space *vm)
 709 {
 710         struct i915_page_directory_pointer *pdp;
 711         int ret = -ENOMEM;
 712
 713         GEM_BUG_ON(!use_4lvl(vm));
 714
 715         pdp = kzalloc(sizeof(*pdp), GFP_KERNEL);
 716         if (!pdp)
 717                 return ERR_PTR(-ENOMEM);
 718
 719         ret = __pdp_init(vm, pdp);
 720         if (ret)
 721                 goto fail_bitmap;
 722
 723         ret = setup_px(vm, pdp);
 724         if (ret)
 725                 goto fail_page_m;
 726
 727         return pdp;
 728
 729 fail_page_m:
 730         __pdp_fini(pdp);
 731 fail_bitmap:
 732         kfree(pdp);
 733
 734         return ERR_PTR(ret);
 735 }
 736
 737 static void free_pdp(struct i915_address_space *vm,
 738                      struct i915_page_directory_pointer *pdp)
 739 {
 740         __pdp_fini(pdp);
 741
 742         if (!use_4lvl(vm))
 743                 return;
 744
 745         cleanup_px(vm, pdp);
 746         kfree(pdp);
 747 }
 748
 749 static void gen8_initialize_pdp(struct i915_address_space *vm,
 750                                 struct i915_page_directory_pointer *pdp)
 751 {
 752         gen8_ppgtt_pdpe_t scratch_pdpe;
 753
 754         scratch_pdpe = gen8_pdpe_encode(px_dma(vm->scratch_pd), I915_CACHE_LLC);
 755
 756         fill_px(vm, pdp, scratch_pdpe);
 757 }
 758
 759 static void gen8_initialize_pml4(struct i915_address_space *vm,
 760                                  struct i915_pml4 *pml4)
 761 {
 762         fill_px(vm, pml4,
 763                 gen8_pml4e_encode(px_dma(vm->scratch_pdp), I915_CACHE_LLC));
 764         memset_p((void **)pml4->pdps, vm->scratch_pdp, GEN8_PML4ES_PER_PML4);
 765 }
 766
 767 /* Broadwell Page Directory Pointer Descriptors */
 768 static int gen8_write_pdp(struct i915_request *rq,
 769                           unsigned entry,
 770                           dma_addr_t addr)
 771 {
 772         struct intel_engine_cs *engine = rq->engine;
 773         u32 *cs;
 774
 775         BUG_ON(entry >= 4);
 776
 777         cs = intel_ring_begin(rq, 6);
 778         if (IS_ERR(cs))
 779                 return PTR_ERR(cs);
 780
 781         *cs++ = MI_LOAD_REGISTER_IMM(1);
 782         *cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_UDW(engine, entry));
 783         *cs++ = upper_32_bits(addr);
 784         *cs++ = MI_LOAD_REGISTER_IMM(1);
 785         *cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_LDW(engine, entry));
 786         *cs++ = lower_32_bits(addr);
 787         intel_ring_advance(rq, cs);
 788
 789         return 0;
 790 }
 791
 792 static int gen8_mm_switch_3lvl(struct i915_hw_ppgtt *ppgtt,
 793                                struct i915_request *rq)
 794 {
 795         int i, ret;
 796
 797         for (i = GEN8_3LVL_PDPES - 1; i >= 0; i--) {
 798                 const dma_addr_t pd_daddr = i915_page_dir_dma_addr(ppgtt, i);
 799
 800                 ret = gen8_write_pdp(rq, i, pd_daddr);
 801                 if (ret)
 802                         return ret;
 803         }
 804
 805         return 0;
 806 }
 807
 808 static int gen8_mm_switch_4lvl(struct i915_hw_ppgtt *ppgtt,
 809                                struct i915_request *rq)
 810 {
 811         return gen8_write_pdp(rq, 0, px_dma(&ppgtt->pml4));
 812 }
 813
 814 /* PDE TLBs are a pain to invalidate on GEN8+. When we modify
 815  * the page table structures, we mark them dirty so that
 816  * context switching/execlist queuing code takes extra steps
 817  * to ensure that tlbs are flushed.
 818  */
 819 static void mark_tlbs_dirty(struct i915_hw_ppgtt *ppgtt)
 820 {
 821         ppgtt->pd_dirty_rings = INTEL_INFO(ppgtt->base.i915)->ring_mask;
 822 }
 823
 824 /* Removes entries from a single page table, releasing it if it's empty.
 825  * Caller can use the return value to update higher-level entries.
 826  */
 827 static bool gen8_ppgtt_clear_pt(struct i915_address_space *vm,
 828                                 struct i915_page_table *pt,
 829                                 u64 start, u64 length)
 830 {
 831         unsigned int num_entries = gen8_pte_count(start, length);
 832         unsigned int pte = gen8_pte_index(start);
 833         unsigned int pte_end = pte + num_entries;
 834         const gen8_pte_t scratch_pte =
 835                 gen8_pte_encode(vm->scratch_page.daddr, I915_CACHE_LLC);
 836         gen8_pte_t *vaddr;
 837
 838         GEM_BUG_ON(num_entries > pt->used_ptes);
 839
 840         pt->used_ptes -= num_entries;
 841         if (!pt->used_ptes)
 842                 return true;
 843
 844         vaddr = kmap_atomic_px(pt);
 845         while (pte < pte_end)
 846                 vaddr[pte++] = scratch_pte;
 847         kunmap_atomic(vaddr);
 848
 849         return false;
 850 }
 851
 852 static void gen8_ppgtt_set_pde(struct i915_address_space *vm,
 853                                struct i915_page_directory *pd,
 854                                struct i915_page_table *pt,
 855                                unsigned int pde)
 856 {
 857         gen8_pde_t *vaddr;
 858
 859         pd->page_table[pde] = pt;
 860
 861         vaddr = kmap_atomic_px(pd);
 862         vaddr[pde] = gen8_pde_encode(px_dma(pt), I915_CACHE_LLC);
 863         kunmap_atomic(vaddr);
 864 }
 865
 866 static bool gen8_ppgtt_clear_pd(struct i915_address_space *vm,
 867                                 struct i915_page_directory *pd,
 868                                 u64 start, u64 length)
 869 {
 870         struct i915_page_table *pt;
 871         u32 pde;
 872
 873         gen8_for_each_pde(pt, pd, start, length, pde) {
 874                 GEM_BUG_ON(pt == vm->scratch_pt);
 875
 876                 if (!gen8_ppgtt_clear_pt(vm, pt, start, length))
 877                         continue;
 878
 879                 gen8_ppgtt_set_pde(vm, pd, vm->scratch_pt, pde);
 880                 GEM_BUG_ON(!pd->used_pdes);
 881                 pd->used_pdes--;
 882
 883                 free_pt(vm, pt);
 884         }
 885
 886         return !pd->used_pdes;
 887 }
 888
 889 static void gen8_ppgtt_set_pdpe(struct i915_address_space *vm,
 890                                 struct i915_page_directory_pointer *pdp,
 891                                 struct i915_page_directory *pd,
 892                                 unsigned int pdpe)
 893 {
 894         gen8_ppgtt_pdpe_t *vaddr;
 895
 896         pdp->page_directory[pdpe] = pd;
 897         if (!use_4lvl(vm))
 898                 return;
 899
 900         vaddr = kmap_atomic_px(pdp);
 901         vaddr[pdpe] = gen8_pdpe_encode(px_dma(pd), I915_CACHE_LLC);
 902         kunmap_atomic(vaddr);
 903 }
 904
 905 /* Removes entries from a single page dir pointer, releasing it if it's empty.
 906  * Caller can use the return value to update higher-level entries
 907  */
 908 static bool gen8_ppgtt_clear_pdp(struct i915_address_space *vm,
 909                                  struct i915_page_directory_pointer *pdp,
 910                                  u64 start, u64 length)
 911 {
 912         struct i915_page_directory *pd;
 913         unsigned int pdpe;
 914
 915         gen8_for_each_pdpe(pd, pdp, start, length, pdpe) {
 916                 GEM_BUG_ON(pd == vm->scratch_pd);
 917
 918                 if (!gen8_ppgtt_clear_pd(vm, pd, start, length))
 919                         continue;
 920
 921                 gen8_ppgtt_set_pdpe(vm, pdp, vm->scratch_pd, pdpe);
 922                 GEM_BUG_ON(!pdp->used_pdpes);
 923                 pdp->used_pdpes--;
 924
 925                 free_pd(vm, pd);
 926         }
 927
 928         return !pdp->used_pdpes;
 929 }
 930
 931 static void gen8_ppgtt_clear_3lvl(struct i915_address_space *vm,
 932                                   u64 start, u64 length)
 933 {
 934         gen8_ppgtt_clear_pdp(vm, &i915_vm_to_ppgtt(vm)->pdp, start, length);
 935 }
 936
 937 static void gen8_ppgtt_set_pml4e(struct i915_pml4 *pml4,
 938                                  struct i915_page_directory_pointer *pdp,
 939                                  unsigned int pml4e)
 940 {
 941         gen8_ppgtt_pml4e_t *vaddr;
 942
 943         pml4->pdps[pml4e] = pdp;
 944
 945         vaddr = kmap_atomic_px(pml4);
 946         vaddr[pml4e] = gen8_pml4e_encode(px_dma(pdp), I915_CACHE_LLC);
 947         kunmap_atomic(vaddr);
 948 }
 949
 950 /* Removes entries from a single pml4.
 951  * This is the top-level structure in 4-level page tables used on gen8+.
 952  * Empty entries are always scratch pml4e.
 953  */
 954 static void gen8_ppgtt_clear_4lvl(struct i915_address_space *vm,
 955                                   u64 start, u64 length)
 956 {
 957         struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
 958         struct i915_pml4 *pml4 = &ppgtt->pml4;
 959         struct i915_page_directory_pointer *pdp;
 960         unsigned int pml4e;
 961
 962         GEM_BUG_ON(!use_4lvl(vm));
 963
 964         gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) {
 965                 GEM_BUG_ON(pdp == vm->scratch_pdp);
 966
 967                 if (!gen8_ppgtt_clear_pdp(vm, pdp, start, length))
 968                         continue;
 969
 970                 gen8_ppgtt_set_pml4e(pml4, vm->scratch_pdp, pml4e);
 971
 972                 free_pdp(vm, pdp);
 973         }
 974 }
 975
 976 static inline struct sgt_dma {
 977         struct scatterlist *sg;
 978         dma_addr_t dma, max;
 979 } sgt_dma(struct i915_vma *vma) {
 980         struct scatterlist *sg = vma->pages->sgl;
 981         dma_addr_t addr = sg_dma_address(sg);
 982         return (struct sgt_dma) { sg, addr, addr + sg->length };
 983 }
 984
 985 struct gen8_insert_pte {
 986         u16 pml4e;
 987         u16 pdpe;
 988         u16 pde;
 989         u16 pte;
 990 };
 991
 992 static __always_inline struct gen8_insert_pte gen8_insert_pte(u64 start)
 993 {
 994         return (struct gen8_insert_pte) {
 995                  gen8_pml4e_index(start),
 996                  gen8_pdpe_index(start),
 997                  gen8_pde_index(start),
 998                  gen8_pte_index(start),
 999         };
1000 }
1001
1002 static __always_inline bool
1003 gen8_ppgtt_insert_pte_entries(struct i915_hw_ppgtt *ppgtt,
1004                               struct i915_page_directory_pointer *pdp,
1005                               struct sgt_dma *iter,
1006                               struct gen8_insert_pte *idx,
1007                               enum i915_cache_level cache_level)
1008 {
1009         struct i915_page_directory *pd;
1010         const gen8_pte_t pte_encode = gen8_pte_encode(0, cache_level);
1011         gen8_pte_t *vaddr;
1012         bool ret;
1013
1014         GEM_BUG_ON(idx->pdpe >= i915_pdpes_per_pdp(&ppgtt->base));
1015         pd = pdp->page_directory[idx->pdpe];
1016         vaddr = kmap_atomic_px(pd->page_table[idx->pde]);
1017         do {
1018                 vaddr[idx->pte] = pte_encode | iter->dma;
1019
1020                 iter->dma += PAGE_SIZE;
1021                 if (iter->dma >= iter->max) {
1022                         iter->sg = __sg_next(iter->sg);
1023                         if (!iter->sg) {
1024                                 ret = false;
1025                                 break;
1026                         }
1027
1028                         iter->dma = sg_dma_address(iter->sg);
1029                         iter->max = iter->dma + iter->sg->length;
1030                 }
1031
1032                 if (++idx->pte == GEN8_PTES) {
1033                         idx->pte = 0;
1034
1035                         if (++idx->pde == I915_PDES) {
1036                                 idx->pde = 0;
1037
1038                                 /* Limited by sg length for 3lvl */
1039                                 if (++idx->pdpe == GEN8_PML4ES_PER_PML4) {
1040                                         idx->pdpe = 0;
1041                                         ret = true;
1042                                         break;
1043                                 }
1044
1045                                 GEM_BUG_ON(idx->pdpe >= i915_pdpes_per_pdp(&ppgtt->base));
1046                                 pd = pdp->page_directory[idx->pdpe];
1047                         }
1048
1049                         kunmap_atomic(vaddr);
1050                         vaddr = kmap_atomic_px(pd->page_table[idx->pde]);
1051                 }
1052         } while (1);
1053         kunmap_atomic(vaddr);
1054
1055         return ret;
1056 }
1057
1058 static void gen8_ppgtt_insert_3lvl(struct i915_address_space *vm,
1059                                    struct i915_vma *vma,
1060                                    enum i915_cache_level cache_level,
1061                                    u32 unused)
1062 {
1063         struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
1064         struct sgt_dma iter = sgt_dma(vma);
1065         struct gen8_insert_pte idx = gen8_insert_pte(vma->node.start);
1066
1067         gen8_ppgtt_insert_pte_entries(ppgtt, &ppgtt->pdp, &iter, &idx,
1068                                       cache_level);
1069
1070         vma->page_sizes.gtt = I915_GTT_PAGE_SIZE;
1071 }
1072
1073 static void gen8_ppgtt_insert_huge_entries(struct i915_vma *vma,
1074                                            struct i915_page_directory_pointer **pdps,
1075                                            struct sgt_dma *iter,
1076                                            enum i915_cache_level cache_level)
1077 {
1078         const gen8_pte_t pte_encode = gen8_pte_encode(0, cache_level);
1079         u64 start = vma->node.start;
1080         dma_addr_t rem = iter->sg->length;
1081
1082         do {
1083                 struct gen8_insert_pte idx = gen8_insert_pte(start);
1084                 struct i915_page_directory_pointer *pdp = pdps[idx.pml4e];
1085                 struct i915_page_directory *pd = pdp->page_directory[idx.pdpe];
1086                 unsigned int page_size;
1087                 bool maybe_64K = false;
1088                 gen8_pte_t encode = pte_encode;
1089                 gen8_pte_t *vaddr;
1090                 u16 index, max;
1091
1092                 if (vma->page_sizes.sg & I915_GTT_PAGE_SIZE_2M &&
1093                     IS_ALIGNED(iter->dma, I915_GTT_PAGE_SIZE_2M) &&
1094                     rem >= I915_GTT_PAGE_SIZE_2M && !idx.pte) {
1095                         index = idx.pde;
1096                         max = I915_PDES;
1097                         page_size = I915_GTT_PAGE_SIZE_2M;
1098
1099                         encode |= GEN8_PDE_PS_2M;
1100
1101                         vaddr = kmap_atomic_px(pd);
1102                 } else {
1103                         struct i915_page_table *pt = pd->page_table[idx.pde];
1104
1105                         index = idx.pte;
1106                         max = GEN8_PTES;
1107                         page_size = I915_GTT_PAGE_SIZE;
1108
1109                         if (!index &&
1110                             vma->page_sizes.sg & I915_GTT_PAGE_SIZE_64K &&
1111                             IS_ALIGNED(iter->dma, I915_GTT_PAGE_SIZE_64K) &&
1112                             (IS_ALIGNED(rem, I915_GTT_PAGE_SIZE_64K) ||
1113                              rem >= (max - index) << PAGE_SHIFT))
1114                                 maybe_64K = true;
1115
1116                         vaddr = kmap_atomic_px(pt);
1117                 }
1118
1119                 do {
1120                         GEM_BUG_ON(iter->sg->length < page_size);
1121                         vaddr[index++] = encode | iter->dma;
1122
1123                         start += page_size;
1124                         iter->dma += page_size;
1125                         rem -= page_size;
1126                         if (iter->dma >= iter->max) {
1127                                 iter->sg = __sg_next(iter->sg);
1128                                 if (!iter->sg)
1129                                         break;
1130
1131                                 rem = iter->sg->length;
1132                                 iter->dma = sg_dma_address(iter->sg);
1133                                 iter->max = iter->dma + rem;
1134
1135                                 if (maybe_64K && index < max &&
1136                                     !(IS_ALIGNED(iter->dma, I915_GTT_PAGE_SIZE_64K) &&
1137                                       (IS_ALIGNED(rem, I915_GTT_PAGE_SIZE_64K) ||
1138                                        rem >= (max - index) << PAGE_SHIFT)))
1139                                         maybe_64K = false;
1140
1141                                 if (unlikely(!IS_ALIGNED(iter->dma, page_size)))
1142                                         break;
1143                         }
1144                 } while (rem >= page_size && index < max);
1145
1146                 kunmap_atomic(vaddr);
1147
1148                 /*
1149                  * Is it safe to mark the 2M block as 64K? -- Either we have
1150                  * filled whole page-table with 64K entries, or filled part of
1151                  * it and have reached the end of the sg table and we have
1152                  * enough padding.
1153                  */
1154                 if (maybe_64K &&
1155                     (index == max ||
1156                      (i915_vm_has_scratch_64K(vma->vm) &&
1157                       !iter->sg && IS_ALIGNED(vma->node.start +
1158                                               vma->node.size,
1159                                               I915_GTT_PAGE_SIZE_2M)))) {
1160                         vaddr = kmap_atomic_px(pd);
1161                         vaddr[idx.pde] |= GEN8_PDE_IPS_64K;
1162                         kunmap_atomic(vaddr);
1163                         page_size = I915_GTT_PAGE_SIZE_64K;
1164                 }
1165
1166                 vma->page_sizes.gtt |= page_size;
1167         } while (iter->sg);
1168 }
1169
1170 static void gen8_ppgtt_insert_4lvl(struct i915_address_space *vm,
1171                                    struct i915_vma *vma,
1172                                    enum i915_cache_level cache_level,
1173                                    u32 unused)
1174 {
1175         struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
1176         struct sgt_dma iter = sgt_dma(vma);
1177         struct i915_page_directory_pointer **pdps = ppgtt->pml4.pdps;
1178
1179         if (vma->page_sizes.sg > I915_GTT_PAGE_SIZE) {
1180                 gen8_ppgtt_insert_huge_entries(vma, pdps, &iter, cache_level);
1181         } else {
1182                 struct gen8_insert_pte idx = gen8_insert_pte(vma->node.start);
1183
1184                 while (gen8_ppgtt_insert_pte_entries(ppgtt, pdps[idx.pml4e++],
1185                                                      &iter, &idx, cache_level))
1186                         GEM_BUG_ON(idx.pml4e >= GEN8_PML4ES_PER_PML4);
1187
1188                 vma->page_sizes.gtt = I915_GTT_PAGE_SIZE;
1189         }
1190 }
1191
1192 static void gen8_free_page_tables(struct i915_address_space *vm,
1193                                   struct i915_page_directory *pd)
1194 {
1195         int i;
1196
1197         if (!px_page(pd))
1198                 return;
1199
1200         for (i = 0; i < I915_PDES; i++) {
1201                 if (pd->page_table[i] != vm->scratch_pt)
1202                         free_pt(vm, pd->page_table[i]);
1203         }
1204 }
1205
1206 static int gen8_init_scratch(struct i915_address_space *vm)
1207 {
1208         int ret;
1209
1210         ret = setup_scratch_page(vm, I915_GFP_DMA);
1211         if (ret)
1212                 return ret;
1213
1214         vm->scratch_pt = alloc_pt(vm);
1215         if (IS_ERR(vm->scratch_pt)) {
1216                 ret = PTR_ERR(vm->scratch_pt);
1217                 goto free_scratch_page;
1218         }
1219
1220         vm->scratch_pd = alloc_pd(vm);
1221         if (IS_ERR(vm->scratch_pd)) {
1222                 ret = PTR_ERR(vm->scratch_pd);
1223                 goto free_pt;
1224         }
1225
1226         if (use_4lvl(vm)) {
1227                 vm->scratch_pdp = alloc_pdp(vm);
1228                 if (IS_ERR(vm->scratch_pdp)) {
1229                         ret = PTR_ERR(vm->scratch_pdp);
1230                         goto free_pd;
1231                 }
1232         }
1233
1234         gen8_initialize_pt(vm, vm->scratch_pt);
1235         gen8_initialize_pd(vm, vm->scratch_pd);
1236         if (use_4lvl(vm))
1237                 gen8_initialize_pdp(vm, vm->scratch_pdp);
1238
1239         return 0;
1240
1241 free_pd:
1242         free_pd(vm, vm->scratch_pd);
1243 free_pt:
1244         free_pt(vm, vm->scratch_pt);
1245 free_scratch_page:
1246         cleanup_scratch_page(vm);
1247
1248         return ret;
1249 }
1250
1251 static int gen8_ppgtt_notify_vgt(struct i915_hw_ppgtt *ppgtt, bool create)
1252 {
1253         struct i915_address_space *vm = &ppgtt->base;
1254         struct drm_i915_private *dev_priv = vm->i915;
1255         enum vgt_g2v_type msg;
1256         int i;
1257
1258         if (use_4lvl(vm)) {
1259                 const u64 daddr = px_dma(&ppgtt->pml4);
1260
1261                 I915_WRITE(vgtif_reg(pdp[0].lo), lower_32_bits(daddr));
1262                 I915_WRITE(vgtif_reg(pdp[0].hi), upper_32_bits(daddr));
1263
1264                 msg = (create ? VGT_G2V_PPGTT_L4_PAGE_TABLE_CREATE :
1265                                 VGT_G2V_PPGTT_L4_PAGE_TABLE_DESTROY);
1266         } else {
1267                 for (i = 0; i < GEN8_3LVL_PDPES; i++) {
1268                         const u64 daddr = i915_page_dir_dma_addr(ppgtt, i);
1269
1270                         I915_WRITE(vgtif_reg(pdp[i].lo), lower_32_bits(daddr));
1271                         I915_WRITE(vgtif_reg(pdp[i].hi), upper_32_bits(daddr));
1272                 }
1273
1274                 msg = (create ? VGT_G2V_PPGTT_L3_PAGE_TABLE_CREATE :
1275                                 VGT_G2V_PPGTT_L3_PAGE_TABLE_DESTROY);
1276         }
1277
1278         I915_WRITE(vgtif_reg(g2v_notify), msg);
1279
1280         return 0;
1281 }
1282
1283 static void gen8_free_scratch(struct i915_address_space *vm)
1284 {
1285         if (use_4lvl(vm))
1286                 free_pdp(vm, vm->scratch_pdp);
1287         free_pd(vm, vm->scratch_pd);
1288         free_pt(vm, vm->scratch_pt);
1289         cleanup_scratch_page(vm);
1290 }
1291
1292 static void gen8_ppgtt_cleanup_3lvl(struct i915_address_space *vm,
1293                                     struct i915_page_directory_pointer *pdp)
1294 {
1295         const unsigned int pdpes = i915_pdpes_per_pdp(vm);
1296         int i;
1297
1298         for (i = 0; i < pdpes; i++) {
1299                 if (pdp->page_directory[i] == vm->scratch_pd)
1300                         continue;
1301
1302                 gen8_free_page_tables(vm, pdp->page_directory[i]);
1303                 free_pd(vm, pdp->page_directory[i]);
1304         }
1305
1306         free_pdp(vm, pdp);
1307 }
1308
1309 static void gen8_ppgtt_cleanup_4lvl(struct i915_hw_ppgtt *ppgtt)
1310 {
1311         int i;
1312
1313         for (i = 0; i < GEN8_PML4ES_PER_PML4; i++) {
1314                 if (ppgtt->pml4.pdps[i] == ppgtt->base.scratch_pdp)
1315                         continue;
1316
1317                 gen8_ppgtt_cleanup_3lvl(&ppgtt->base, ppgtt->pml4.pdps[i]);
1318         }
1319
1320         cleanup_px(&ppgtt->base, &ppgtt->pml4);
1321 }
1322
1323 static void gen8_ppgtt_cleanup(struct i915_address_space *vm)
1324 {
1325         struct drm_i915_private *dev_priv = vm->i915;
1326         struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
1327
1328         if (intel_vgpu_active(dev_priv))
1329                 gen8_ppgtt_notify_vgt(ppgtt, false);
1330
1331         if (use_4lvl(vm))
1332                 gen8_ppgtt_cleanup_4lvl(ppgtt);
1333         else
1334                 gen8_ppgtt_cleanup_3lvl(&ppgtt->base, &ppgtt->pdp);
1335
1336         gen8_free_scratch(vm);
1337 }
1338
1339 static int gen8_ppgtt_alloc_pd(struct i915_address_space *vm,
1340                                struct i915_page_directory *pd,
1341                                u64 start, u64 length)
1342 {
1343         struct i915_page_table *pt;
1344         u64 from = start;
1345         unsigned int pde;
1346
1347         gen8_for_each_pde(pt, pd, start, length, pde) {
1348                 int count = gen8_pte_count(start, length);
1349
1350                 if (pt == vm->scratch_pt) {
1351                         pd->used_pdes++;
1352
1353                         pt = alloc_pt(vm);
1354                         if (IS_ERR(pt)) {
1355                                 pd->used_pdes--;
1356                                 goto unwind;
1357                         }
1358
1359                         if (count < GEN8_PTES || intel_vgpu_active(vm->i915))
1360                                 gen8_initialize_pt(vm, pt);
1361
1362                         gen8_ppgtt_set_pde(vm, pd, pt, pde);
1363                         GEM_BUG_ON(pd->used_pdes > I915_PDES);
1364                 }
1365
1366                 pt->used_ptes += count;
1367         }
1368         return 0;
1369
1370 unwind:
1371         gen8_ppgtt_clear_pd(vm, pd, from, start - from);
1372         return -ENOMEM;
1373 }
1374
1375 static int gen8_ppgtt_alloc_pdp(struct i915_address_space *vm,
1376                                 struct i915_page_directory_pointer *pdp,
1377                                 u64 start, u64 length)
1378 {
1379         struct i915_page_directory *pd;
1380         u64 from = start;
1381         unsigned int pdpe;
1382         int ret;
1383
1384         gen8_for_each_pdpe(pd, pdp, start, length, pdpe) {
1385                 if (pd == vm->scratch_pd) {
1386                         pdp->used_pdpes++;
1387
1388                         pd = alloc_pd(vm);
1389                         if (IS_ERR(pd)) {
1390                                 pdp->used_pdpes--;
1391                                 goto unwind;
1392                         }
1393
1394                         gen8_initialize_pd(vm, pd);
1395                         gen8_ppgtt_set_pdpe(vm, pdp, pd, pdpe);
1396                         GEM_BUG_ON(pdp->used_pdpes > i915_pdpes_per_pdp(vm));
1397
1398                         mark_tlbs_dirty(i915_vm_to_ppgtt(vm));
1399                 }
1400
1401                 ret = gen8_ppgtt_alloc_pd(vm, pd, start, length);
1402                 if (unlikely(ret))
1403                         goto unwind_pd;
1404         }
1405
1406         return 0;
1407
1408 unwind_pd:
1409         if (!pd->used_pdes) {
1410                 gen8_ppgtt_set_pdpe(vm, pdp, vm->scratch_pd, pdpe);
1411                 GEM_BUG_ON(!pdp->used_pdpes);
1412                 pdp->used_pdpes--;
1413                 free_pd(vm, pd);
1414         }
1415 unwind:
1416         gen8_ppgtt_clear_pdp(vm, pdp, from, start - from);
1417         return -ENOMEM;
1418 }
1419
1420 static int gen8_ppgtt_alloc_3lvl(struct i915_address_space *vm,
1421                                  u64 start, u64 length)
1422 {
1423         return gen8_ppgtt_alloc_pdp(vm,
1424                                     &i915_vm_to_ppgtt(vm)->pdp, start, length);
1425 }
1426
1427 static int gen8_ppgtt_alloc_4lvl(struct i915_address_space *vm,
1428                                  u64 start, u64 length)
1429 {
1430         struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
1431         struct i915_pml4 *pml4 = &ppgtt->pml4;
1432         struct i915_page_directory_pointer *pdp;
1433         u64 from = start;
1434         u32 pml4e;
1435         int ret;
1436
1437         gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) {
1438                 if (pml4->pdps[pml4e] == vm->scratch_pdp) {
1439                         pdp = alloc_pdp(vm);
1440                         if (IS_ERR(pdp))
1441                                 goto unwind;
1442
1443                         gen8_initialize_pdp(vm, pdp);
1444                         gen8_ppgtt_set_pml4e(pml4, pdp, pml4e);
1445                 }
1446
1447                 ret = gen8_ppgtt_alloc_pdp(vm, pdp, start, length);
1448                 if (unlikely(ret))
1449                         goto unwind_pdp;
1450         }
1451
1452         return 0;
1453
1454 unwind_pdp:
1455         if (!pdp->used_pdpes) {
1456                 gen8_ppgtt_set_pml4e(pml4, vm->scratch_pdp, pml4e);
1457                 free_pdp(vm, pdp);
1458         }
1459 unwind:
1460         gen8_ppgtt_clear_4lvl(vm, from, start - from);
1461         return -ENOMEM;
1462 }
1463
1464 static void gen8_dump_pdp(struct i915_hw_ppgtt *ppgtt,
1465                           struct i915_page_directory_pointer *pdp,
1466                           u64 start, u64 length,
1467                           gen8_pte_t scratch_pte,
1468                           struct seq_file *m)
1469 {
1470         struct i915_address_space *vm = &ppgtt->base;
1471         struct i915_page_directory *pd;
1472         u32 pdpe;
1473
1474         gen8_for_each_pdpe(pd, pdp, start, length, pdpe) {
1475                 struct i915_page_table *pt;
1476                 u64 pd_len = length;
1477                 u64 pd_start = start;
1478                 u32 pde;
1479
1480                 if (pdp->page_directory[pdpe] == ppgtt->base.scratch_pd)
1481                         continue;
1482
1483                 seq_printf(m, "\tPDPE #%d\n", pdpe);
1484                 gen8_for_each_pde(pt, pd, pd_start, pd_len, pde) {
1485                         u32 pte;
1486                         gen8_pte_t *pt_vaddr;
1487
1488                         if (pd->page_table[pde] == ppgtt->base.scratch_pt)
1489                                 continue;
1490
1491                         pt_vaddr = kmap_atomic_px(pt);
1492                         for (pte = 0; pte < GEN8_PTES; pte += 4) {
1493                                 u64 va = (pdpe << GEN8_PDPE_SHIFT |
1494                                           pde << GEN8_PDE_SHIFT |
1495                                           pte << GEN8_PTE_SHIFT);
1496                                 int i;
1497                                 bool found = false;
1498
1499                                 for (i = 0; i < 4; i++)
1500                                         if (pt_vaddr[pte + i] != scratch_pte)
1501                                                 found = true;
1502                                 if (!found)
1503                                         continue;
1504
1505                                 seq_printf(m, "\t\t0x%llx [%03d,%03d,%04d]: =", va, pdpe, pde, pte);
1506                                 for (i = 0; i < 4; i++) {
1507                                         if (pt_vaddr[pte + i] != scratch_pte)
1508                                                 seq_printf(m, " %llx", pt_vaddr[pte + i]);
1509                                         else
1510                                                 seq_puts(m, "  SCRATCH ");
1511                                 }
1512                                 seq_puts(m, "\n");
1513                         }
1514                         kunmap_atomic(pt_vaddr);
1515                 }
1516         }
1517 }
1518
1519 static void gen8_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m)
1520 {
1521         struct i915_address_space *vm = &ppgtt->base;
1522         const gen8_pte_t scratch_pte =
1523                 gen8_pte_encode(vm->scratch_page.daddr, I915_CACHE_LLC);
1524         u64 start = 0, length = ppgtt->base.total;
1525
1526         if (use_4lvl(vm)) {
1527                 u64 pml4e;
1528                 struct i915_pml4 *pml4 = &ppgtt->pml4;
1529                 struct i915_page_directory_pointer *pdp;
1530
1531                 gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) {
1532                         if (pml4->pdps[pml4e] == ppgtt->base.scratch_pdp)
1533                                 continue;
1534
1535                         seq_printf(m, "    PML4E #%llu\n", pml4e);
1536                         gen8_dump_pdp(ppgtt, pdp, start, length, scratch_pte, m);
1537                 }
1538         } else {
1539                 gen8_dump_pdp(ppgtt, &ppgtt->pdp, start, length, scratch_pte, m);
1540         }
1541 }
1542
1543 static int gen8_preallocate_top_level_pdp(struct i915_hw_ppgtt *ppgtt)
1544 {
1545         struct i915_address_space *vm = &ppgtt->base;
1546         struct i915_page_directory_pointer *pdp = &ppgtt->pdp;
1547         struct i915_page_directory *pd;
1548         u64 start = 0, length = ppgtt->base.total;
1549         u64 from = start;
1550         unsigned int pdpe;
1551
1552         gen8_for_each_pdpe(pd, pdp, start, length, pdpe) {
1553                 pd = alloc_pd(vm);
1554                 if (IS_ERR(pd))
1555                         goto unwind;
1556
1557                 gen8_initialize_pd(vm, pd);
1558                 gen8_ppgtt_set_pdpe(vm, pdp, pd, pdpe);
1559                 pdp->used_pdpes++;
1560         }
1561
1562         pdp->used_pdpes++; /* never remove */
1563         return 0;
1564
1565 unwind:
1566         start -= from;
1567         gen8_for_each_pdpe(pd, pdp, from, start, pdpe) {
1568                 gen8_ppgtt_set_pdpe(vm, pdp, vm->scratch_pd, pdpe);
1569                 free_pd(vm, pd);
1570         }
1571         pdp->used_pdpes = 0;
1572         return -ENOMEM;
1573 }
1574
1575 /*
1576  * GEN8 legacy ppgtt programming is accomplished through a max 4 PDP registers
1577  * with a net effect resembling a 2-level page table in normal x86 terms. Each
1578  * PDP represents 1GB of memory 4 * 512 * 512 * 4096 = 4GB legacy 32b address
1579  * space.
1580  *
1581  */
1582 static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
1583 {
1584         struct i915_address_space *vm = &ppgtt->base;
1585         struct drm_i915_private *dev_priv = vm->i915;
1586         int ret;
1587
1588         ppgtt->base.total = USES_FULL_48BIT_PPGTT(dev_priv) ?
1589                 1ULL << 48 :
1590                 1ULL << 32;
1591
1592         /* There are only few exceptions for gen >=6. chv and bxt.
1593          * And we are not sure about the latter so play safe for now.
1594          */
1595         if (IS_CHERRYVIEW(dev_priv) || IS_BROXTON(dev_priv))
1596                 ppgtt->base.pt_kmap_wc = true;
1597
1598         ret = gen8_init_scratch(&ppgtt->base);
1599         if (ret) {
1600                 ppgtt->base.total = 0;
1601                 return ret;
1602         }
1603
1604         if (use_4lvl(vm)) {
1605                 ret = setup_px(&ppgtt->base, &ppgtt->pml4);
1606                 if (ret)
1607                         goto free_scratch;
1608
1609                 gen8_initialize_pml4(&ppgtt->base, &ppgtt->pml4);
1610
1611                 ppgtt->switch_mm = gen8_mm_switch_4lvl;
1612                 ppgtt->base.allocate_va_range = gen8_ppgtt_alloc_4lvl;
1613                 ppgtt->base.insert_entries = gen8_ppgtt_insert_4lvl;
1614                 ppgtt->base.clear_range = gen8_ppgtt_clear_4lvl;
1615         } else {
1616                 ret = __pdp_init(&ppgtt->base, &ppgtt->pdp);
1617                 if (ret)
1618                         goto free_scratch;
1619
1620                 if (intel_vgpu_active(dev_priv)) {
1621                         ret = gen8_preallocate_top_level_pdp(ppgtt);
1622                         if (ret) {
1623                                 __pdp_fini(&ppgtt->pdp);
1624                                 goto free_scratch;
1625                         }
1626                 }
1627
1628                 ppgtt->switch_mm = gen8_mm_switch_3lvl;
1629                 ppgtt->base.allocate_va_range = gen8_ppgtt_alloc_3lvl;
1630                 ppgtt->base.insert_entries = gen8_ppgtt_insert_3lvl;
1631                 ppgtt->base.clear_range = gen8_ppgtt_clear_3lvl;
1632         }
1633
1634         if (intel_vgpu_active(dev_priv))
1635                 gen8_ppgtt_notify_vgt(ppgtt, true);
1636
1637         ppgtt->base.cleanup = gen8_ppgtt_cleanup;
1638         ppgtt->base.unbind_vma = ppgtt_unbind_vma;
1639         ppgtt->base.bind_vma = ppgtt_bind_vma;
1640         ppgtt->base.set_pages = ppgtt_set_pages;
1641         ppgtt->base.clear_pages = clear_pages;
1642         ppgtt->debug_dump = gen8_dump_ppgtt;
1643
1644         return 0;
1645
1646 free_scratch:
1647         gen8_free_scratch(&ppgtt->base);
1648         return ret;
1649 }
1650
1651 static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m)
1652 {
1653         struct i915_address_space *vm = &ppgtt->base;
1654         struct i915_page_table *unused;
1655         gen6_pte_t scratch_pte;
1656         u32 pd_entry, pte, pde;
1657         u32 start = 0, length = ppgtt->base.total;
1658
1659         scratch_pte = vm->pte_encode(vm->scratch_page.daddr,
1660                                      I915_CACHE_LLC, 0);
1661
1662         gen6_for_each_pde(unused, &ppgtt->pd, start, length, pde) {
1663                 u32 expected;
1664                 gen6_pte_t *pt_vaddr;
1665                 const dma_addr_t pt_addr = px_dma(ppgtt->pd.page_table[pde]);
1666                 pd_entry = readl(ppgtt->pd_addr + pde);
1667                 expected = (GEN6_PDE_ADDR_ENCODE(pt_addr) | GEN6_PDE_VALID);
1668
1669                 if (pd_entry != expected)
1670                         seq_printf(m, "\tPDE #%d mismatch: Actual PDE: %x Expected PDE: %x\n",
1671                                    pde,
1672                                    pd_entry,
1673                                    expected);
1674                 seq_printf(m, "\tPDE: %x\n", pd_entry);
1675
1676                 pt_vaddr = kmap_atomic_px(ppgtt->pd.page_table[pde]);
1677
1678                 for (pte = 0; pte < GEN6_PTES; pte+=4) {
1679                         unsigned long va =
1680                                 (pde * PAGE_SIZE * GEN6_PTES) +
1681                                 (pte * PAGE_SIZE);
1682                         int i;
1683                         bool found = false;
1684                         for (i = 0; i < 4; i++)
1685                                 if (pt_vaddr[pte + i] != scratch_pte)
1686                                         found = true;
1687                         if (!found)
1688                                 continue;
1689
1690                         seq_printf(m, "\t\t0x%lx [%03d,%04d]: =", va, pde, pte);
1691                         for (i = 0; i < 4; i++) {
1692                                 if (pt_vaddr[pte + i] != scratch_pte)
1693                                         seq_printf(m, " %08x", pt_vaddr[pte + i]);
1694                                 else
1695                                         seq_puts(m, "  SCRATCH ");
1696                         }
1697                         seq_puts(m, "\n");
1698                 }
1699                 kunmap_atomic(pt_vaddr);
1700         }
1701 }
1702
1703 /* Write pde (index) from the page directory @pd to the page table @pt */
1704 static inline void gen6_write_pde(const struct i915_hw_ppgtt *ppgtt,
1705                                   const unsigned int pde,
1706                                   const struct i915_page_table *pt)
1707 {
1708         /* Caller needs to make sure the write completes if necessary */
1709         writel_relaxed(GEN6_PDE_ADDR_ENCODE(px_dma(pt)) | GEN6_PDE_VALID,
1710                        ppgtt->pd_addr + pde);
1711 }
1712
1713 /* Write all the page tables found in the ppgtt structure to incrementing page
1714  * directories. */
1715 static void gen6_write_page_range(struct i915_hw_ppgtt *ppgtt,
1716                                   u32 start, u32 length)
1717 {
1718         struct i915_page_table *pt;
1719         unsigned int pde;
1720
1721         gen6_for_each_pde(pt, &ppgtt->pd, start, length, pde)
1722                 gen6_write_pde(ppgtt, pde, pt);
1723
1724         mark_tlbs_dirty(ppgtt);
1725         wmb();
1726 }
1727
1728 static inline u32 get_pd_offset(struct i915_hw_ppgtt *ppgtt)
1729 {
1730         GEM_BUG_ON(ppgtt->pd.base.ggtt_offset & 0x3f);
1731         return ppgtt->pd.base.ggtt_offset << 10;
1732 }
1733
1734 static int hsw_mm_switch(struct i915_hw_ppgtt *ppgtt,
1735                          struct i915_request *rq)
1736 {
1737         struct intel_engine_cs *engine = rq->engine;
1738         u32 *cs;
1739
1740         /* NB: TLBs must be flushed and invalidated before a switch */
1741         cs = intel_ring_begin(rq, 6);
1742         if (IS_ERR(cs))
1743                 return PTR_ERR(cs);
1744
1745         *cs++ = MI_LOAD_REGISTER_IMM(2);
1746         *cs++ = i915_mmio_reg_offset(RING_PP_DIR_DCLV(engine));
1747         *cs++ = PP_DIR_DCLV_2G;
1748         *cs++ = i915_mmio_reg_offset(RING_PP_DIR_BASE(engine));
1749         *cs++ = get_pd_offset(ppgtt);
1750         *cs++ = MI_NOOP;
1751         intel_ring_advance(rq, cs);
1752
1753         return 0;
1754 }
1755
1756 static int gen7_mm_switch(struct i915_hw_ppgtt *ppgtt,
1757                           struct i915_request *rq)
1758 {
1759         struct intel_engine_cs *engine = rq->engine;
1760         u32 *cs;
1761
1762         /* NB: TLBs must be flushed and invalidated before a switch */
1763         cs = intel_ring_begin(rq, 6);
1764         if (IS_ERR(cs))
1765                 return PTR_ERR(cs);
1766
1767         *cs++ = MI_LOAD_REGISTER_IMM(2);
1768         *cs++ = i915_mmio_reg_offset(RING_PP_DIR_DCLV(engine));
1769         *cs++ = PP_DIR_DCLV_2G;
1770         *cs++ = i915_mmio_reg_offset(RING_PP_DIR_BASE(engine));
1771         *cs++ = get_pd_offset(ppgtt);
1772         *cs++ = MI_NOOP;
1773         intel_ring_advance(rq, cs);
1774
1775         return 0;
1776 }
1777
1778 static int gen6_mm_switch(struct i915_hw_ppgtt *ppgtt,
1779                           struct i915_request *rq)
1780 {
1781         struct intel_engine_cs *engine = rq->engine;
1782         struct drm_i915_private *dev_priv = rq->i915;
1783
1784         I915_WRITE(RING_PP_DIR_DCLV(engine), PP_DIR_DCLV_2G);
1785         I915_WRITE(RING_PP_DIR_BASE(engine), get_pd_offset(ppgtt));
1786         return 0;
1787 }
1788
1789 static void gen8_ppgtt_enable(struct drm_i915_private *dev_priv)
1790 {
1791         struct intel_engine_cs *engine;
1792         enum intel_engine_id id;
1793
1794         for_each_engine(engine, dev_priv, id) {
1795                 u32 four_level = USES_FULL_48BIT_PPGTT(dev_priv) ?
1796                                  GEN8_GFX_PPGTT_48B : 0;
1797                 I915_WRITE(RING_MODE_GEN7(engine),
1798                            _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE | four_level));
1799         }
1800 }
1801
1802 static void gen7_ppgtt_enable(struct drm_i915_private *dev_priv)
1803 {
1804         struct intel_engine_cs *engine;
1805         u32 ecochk, ecobits;
1806         enum intel_engine_id id;
1807
1808         ecobits = I915_READ(GAC_ECO_BITS);
1809         I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B);
1810
1811         ecochk = I915_READ(GAM_ECOCHK);
1812         if (IS_HASWELL(dev_priv)) {
1813                 ecochk |= ECOCHK_PPGTT_WB_HSW;
1814         } else {
1815                 ecochk |= ECOCHK_PPGTT_LLC_IVB;
1816                 ecochk &= ~ECOCHK_PPGTT_GFDT_IVB;
1817         }
1818         I915_WRITE(GAM_ECOCHK, ecochk);
1819
1820         for_each_engine(engine, dev_priv, id) {
1821                 /* GFX_MODE is per-ring on gen7+ */
1822                 I915_WRITE(RING_MODE_GEN7(engine),
1823                            _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
1824         }
1825 }
1826
1827 static void gen6_ppgtt_enable(struct drm_i915_private *dev_priv)
1828 {
1829         u32 ecochk, gab_ctl, ecobits;
1830
1831         ecobits = I915_READ(GAC_ECO_BITS);
1832         I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_SNB_BIT |
1833                    ECOBITS_PPGTT_CACHE64B);
1834
1835         gab_ctl = I915_READ(GAB_CTL);
1836         I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT);
1837
1838         ecochk = I915_READ(GAM_ECOCHK);
1839         I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT | ECOCHK_PPGTT_CACHE64B);
1840
1841         I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
1842 }
1843
1844 /* PPGTT support for Sandybdrige/Gen6 and later */
1845 static void gen6_ppgtt_clear_range(struct i915_address_space *vm,
1846                                    u64 start, u64 length)
1847 {
1848         struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
1849         unsigned int first_entry = start >> PAGE_SHIFT;
1850         unsigned int pde = first_entry / GEN6_PTES;
1851         unsigned int pte = first_entry % GEN6_PTES;
1852         unsigned int num_entries = length >> PAGE_SHIFT;
1853         gen6_pte_t scratch_pte =
1854                 vm->pte_encode(vm->scratch_page.daddr, I915_CACHE_LLC, 0);
1855
1856         while (num_entries) {
1857                 struct i915_page_table *pt = ppgtt->pd.page_table[pde++];
1858                 unsigned int end = min(pte + num_entries, GEN6_PTES);
1859                 gen6_pte_t *vaddr;
1860
1861                 num_entries -= end - pte;
1862
1863                 /* Note that the hw doesn't support removing PDE on the fly
1864                  * (they are cached inside the context with no means to
1865                  * invalidate the cache), so we can only reset the PTE
1866                  * entries back to scratch.
1867                  */
1868
1869                 vaddr = kmap_atomic_px(pt);
1870                 do {
1871                         vaddr[pte++] = scratch_pte;
1872                 } while (pte < end);
1873                 kunmap_atomic(vaddr);
1874
1875                 pte = 0;
1876         }
1877 }
1878
1879 static void gen6_ppgtt_insert_entries(struct i915_address_space *vm,
1880                                       struct i915_vma *vma,
1881                                       enum i915_cache_level cache_level,
1882                                       u32 flags)
1883 {
1884         struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
1885         unsigned first_entry = vma->node.start >> PAGE_SHIFT;
1886         unsigned act_pt = first_entry / GEN6_PTES;
1887         unsigned act_pte = first_entry % GEN6_PTES;
1888         const u32 pte_encode = vm->pte_encode(0, cache_level, flags);
1889         struct sgt_dma iter = sgt_dma(vma);
1890         gen6_pte_t *vaddr;
1891
1892         vaddr = kmap_atomic_px(ppgtt->pd.page_table[act_pt]);
1893         do {
1894                 vaddr[act_pte] = pte_encode | GEN6_PTE_ADDR_ENCODE(iter.dma);
1895
1896                 iter.dma += PAGE_SIZE;
1897                 if (iter.dma == iter.max) {
1898                         iter.sg = __sg_next(iter.sg);
1899                         if (!iter.sg)
1900                                 break;
1901
1902                         iter.dma = sg_dma_address(iter.sg);
1903                         iter.max = iter.dma + iter.sg->length;
1904                 }
1905
1906                 if (++act_pte == GEN6_PTES) {
1907                         kunmap_atomic(vaddr);
1908                         vaddr = kmap_atomic_px(ppgtt->pd.page_table[++act_pt]);
1909                         act_pte = 0;
1910                 }
1911         } while (1);
1912         kunmap_atomic(vaddr);
1913
1914         vma->page_sizes.gtt = I915_GTT_PAGE_SIZE;
1915 }
1916
1917 static int gen6_alloc_va_range(struct i915_address_space *vm,
1918                                u64 start, u64 length)
1919 {
1920         struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
1921         struct i915_page_table *pt;
1922         u64 from = start;
1923         unsigned int pde;
1924         bool flush = false;
1925
1926         gen6_for_each_pde(pt, &ppgtt->pd, start, length, pde) {
1927                 if (pt == vm->scratch_pt) {
1928                         pt = alloc_pt(vm);
1929                         if (IS_ERR(pt))
1930                                 goto unwind_out;
1931
1932                         gen6_initialize_pt(vm, pt);
1933                         ppgtt->pd.page_table[pde] = pt;
1934                         gen6_write_pde(ppgtt, pde, pt);
1935                         flush = true;
1936                 }
1937         }
1938
1939         if (flush) {
1940                 mark_tlbs_dirty(ppgtt);
1941                 wmb();
1942         }
1943
1944         return 0;
1945
1946 unwind_out:
1947         gen6_ppgtt_clear_range(vm, from, start);
1948         return -ENOMEM;
1949 }
1950
1951 static int gen6_init_scratch(struct i915_address_space *vm)
1952 {
1953         int ret;
1954
1955         ret = setup_scratch_page(vm, I915_GFP_DMA);
1956         if (ret)
1957                 return ret;
1958
1959         vm->scratch_pt = alloc_pt(vm);
1960         if (IS_ERR(vm->scratch_pt)) {
1961                 cleanup_scratch_page(vm);
1962                 return PTR_ERR(vm->scratch_pt);
1963         }
1964
1965         gen6_initialize_pt(vm, vm->scratch_pt);
1966
1967         return 0;
1968 }
1969
1970 static void gen6_free_scratch(struct i915_address_space *vm)
1971 {
1972         free_pt(vm, vm->scratch_pt);
1973         cleanup_scratch_page(vm);
1974 }
1975
1976 static void gen6_ppgtt_cleanup(struct i915_address_space *vm)
1977 {
1978         struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
1979         struct i915_page_directory *pd = &ppgtt->pd;
1980         struct i915_page_table *pt;
1981         u32 pde;
1982
1983         drm_mm_remove_node(&ppgtt->node);
1984
1985         gen6_for_all_pdes(pt, pd, pde)
1986                 if (pt != vm->scratch_pt)
1987                         free_pt(vm, pt);
1988
1989         gen6_free_scratch(vm);
1990 }
1991
1992 static int gen6_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt)
1993 {
1994         struct i915_address_space *vm = &ppgtt->base;
1995         struct drm_i915_private *dev_priv = ppgtt->base.i915;
1996         struct i915_ggtt *ggtt = &dev_priv->ggtt;
1997         int ret;
1998
1999         /* PPGTT PDEs reside in the GGTT and consists of 512 entries. The
2000          * allocator works in address space sizes, so it's multiplied by page
2001          * size. We allocate at the top of the GTT to avoid fragmentation.
2002          */
2003         BUG_ON(!drm_mm_initialized(&ggtt->base.mm));
2004
2005         ret = gen6_init_scratch(vm);
2006         if (ret)
2007                 return ret;
2008
2009         ret = i915_gem_gtt_insert(&ggtt->base, &ppgtt->node,
2010                                   GEN6_PD_SIZE, GEN6_PD_ALIGN,
2011                                   I915_COLOR_UNEVICTABLE,
2012                                   0, ggtt->base.total,
2013                                   PIN_HIGH);
2014         if (ret)
2015                 goto err_out;
2016
2017         if (ppgtt->node.start < ggtt->mappable_end)
2018                 DRM_DEBUG("Forced to use aperture for PDEs\n");
2019
2020         ppgtt->pd.base.ggtt_offset =
2021                 ppgtt->node.start / PAGE_SIZE * sizeof(gen6_pte_t);
2022
2023         ppgtt->pd_addr = (gen6_pte_t __iomem *)ggtt->gsm +
2024                 ppgtt->pd.base.ggtt_offset / sizeof(gen6_pte_t);
2025
2026         return 0;
2027
2028 err_out:
2029         gen6_free_scratch(vm);
2030         return ret;
2031 }
2032
2033 static int gen6_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt)
2034 {
2035         return gen6_ppgtt_allocate_page_directories(ppgtt);
2036 }
2037
2038 static void gen6_scratch_va_range(struct i915_hw_ppgtt *ppgtt,
2039                                   u64 start, u64 length)
2040 {
2041         struct i915_page_table *unused;
2042         u32 pde;
2043
2044         gen6_for_each_pde(unused, &ppgtt->pd, start, length, pde)
2045                 ppgtt->pd.page_table[pde] = ppgtt->base.scratch_pt;
2046 }
2047
2048 static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
2049 {
2050         struct drm_i915_private *dev_priv = ppgtt->base.i915;
2051         struct i915_ggtt *ggtt = &dev_priv->ggtt;
2052         int ret;
2053
2054         ppgtt->base.pte_encode = ggtt->base.pte_encode;
2055         if (intel_vgpu_active(dev_priv) || IS_GEN6(dev_priv))
2056                 ppgtt->switch_mm = gen6_mm_switch;
2057         else if (IS_HASWELL(dev_priv))
2058                 ppgtt->switch_mm = hsw_mm_switch;
2059         else if (IS_GEN7(dev_priv))
2060                 ppgtt->switch_mm = gen7_mm_switch;
2061         else
2062                 BUG();
2063
2064         ret = gen6_ppgtt_alloc(ppgtt);
2065         if (ret)
2066                 return ret;
2067
2068         ppgtt->base.total = I915_PDES * GEN6_PTES * PAGE_SIZE;
2069
2070         gen6_scratch_va_range(ppgtt, 0, ppgtt->base.total);
2071         gen6_write_page_range(ppgtt, 0, ppgtt->base.total);
2072
2073         ret = gen6_alloc_va_range(&ppgtt->base, 0, ppgtt->base.total);
2074         if (ret) {
2075                 gen6_ppgtt_cleanup(&ppgtt->base);
2076                 return ret;
2077         }
2078
2079         ppgtt->base.clear_range = gen6_ppgtt_clear_range;
2080         ppgtt->base.insert_entries = gen6_ppgtt_insert_entries;
2081         ppgtt->base.unbind_vma = ppgtt_unbind_vma;
2082         ppgtt->base.bind_vma = ppgtt_bind_vma;
2083         ppgtt->base.set_pages = ppgtt_set_pages;
2084         ppgtt->base.clear_pages = clear_pages;
2085         ppgtt->base.cleanup = gen6_ppgtt_cleanup;
2086         ppgtt->debug_dump = gen6_dump_ppgtt;
2087
2088         DRM_DEBUG_DRIVER("Allocated pde space (%lldM) at GTT entry: %llx\n",
2089                          ppgtt->node.size >> 20,
2090                          ppgtt->node.start / PAGE_SIZE);
2091
2092         DRM_DEBUG_DRIVER("Adding PPGTT at offset %x\n",
2093                          ppgtt->pd.base.ggtt_offset << 10);
2094
2095         return 0;
2096 }
2097
2098 static int __hw_ppgtt_init(struct i915_hw_ppgtt *ppgtt,
2099                            struct drm_i915_private *dev_priv)
2100 {
2101         ppgtt->base.i915 = dev_priv;
2102         ppgtt->base.dma = &dev_priv->drm.pdev->dev;
2103
2104         if (INTEL_GEN(dev_priv) < 8)
2105                 return gen6_ppgtt_init(ppgtt);
2106         else
2107                 return gen8_ppgtt_init(ppgtt);
2108 }
2109
2110 static void i915_address_space_init(struct i915_address_space *vm,
2111                                     struct drm_i915_private *dev_priv,
2112                                     const char *name)
2113 {
2114         i915_gem_timeline_init(dev_priv, &vm->timeline, name);
2115
2116         drm_mm_init(&vm->mm, 0, vm->total);
2117         vm->mm.head_node.color = I915_COLOR_UNEVICTABLE;
2118
2119         INIT_LIST_HEAD(&vm->active_list);
2120         INIT_LIST_HEAD(&vm->inactive_list);
2121         INIT_LIST_HEAD(&vm->unbound_list);
2122
2123         list_add_tail(&vm->global_link, &dev_priv->vm_list);
2124         pagevec_init(&vm->free_pages);
2125 }
2126
2127 static void i915_address_space_fini(struct i915_address_space *vm)
2128 {
2129         if (pagevec_count(&vm->free_pages))
2130                 vm_free_pages_release(vm, true);
2131
2132         i915_gem_timeline_fini(&vm->timeline);
2133         drm_mm_takedown(&vm->mm);
2134         list_del(&vm->global_link);
2135 }
2136
2137 static void gtt_write_workarounds(struct drm_i915_private *dev_priv)
2138 {
2139         /* This function is for gtt related workarounds. This function is
2140          * called on driver load and after a GPU reset, so you can place
2141          * workarounds here even if they get overwritten by GPU reset.
2142          */
2143         /* WaIncreaseDefaultTLBEntries:chv,bdw,skl,bxt,kbl,glk,cfl,cnl */
2144         if (IS_BROADWELL(dev_priv))
2145                 I915_WRITE(GEN8_L3_LRA_1_GPGPU, GEN8_L3_LRA_1_GPGPU_DEFAULT_VALUE_BDW);
2146         else if (IS_CHERRYVIEW(dev_priv))
2147                 I915_WRITE(GEN8_L3_LRA_1_GPGPU, GEN8_L3_LRA_1_GPGPU_DEFAULT_VALUE_CHV);
2148         else if (IS_GEN9_BC(dev_priv) || IS_GEN10(dev_priv))
2149                 I915_WRITE(GEN8_L3_LRA_1_GPGPU, GEN9_L3_LRA_1_GPGPU_DEFAULT_VALUE_SKL);
2150         else if (IS_GEN9_LP(dev_priv))
2151                 I915_WRITE(GEN8_L3_LRA_1_GPGPU, GEN9_L3_LRA_1_GPGPU_DEFAULT_VALUE_BXT);
2152
2153         /*
2154          * To support 64K PTEs we need to first enable the use of the
2155          * Intermediate-Page-Size(IPS) bit of the PDE field via some magical
2156          * mmio, otherwise the page-walker will simply ignore the IPS bit. This
2157          * shouldn't be needed after GEN10.
2158          *
2159          * 64K pages were first introduced from BDW+, although technically they
2160          * only *work* from gen9+. For pre-BDW we instead have the option for
2161          * 32K pages, but we don't currently have any support for it in our
2162          * driver.
2163          */
2164         if (HAS_PAGE_SIZES(dev_priv, I915_GTT_PAGE_SIZE_64K) &&
2165             INTEL_GEN(dev_priv) <= 10)
2166                 I915_WRITE(GEN8_GAMW_ECO_DEV_RW_IA,
2167                            I915_READ(GEN8_GAMW_ECO_DEV_RW_IA) |
2168                            GAMW_ECO_ENABLE_64K_IPS_FIELD);
2169 }
2170
2171 int i915_ppgtt_init_hw(struct drm_i915_private *dev_priv)
2172 {
2173         gtt_write_workarounds(dev_priv);
2174
2175         /* In the case of execlists, PPGTT is enabled by the context descriptor
2176          * and the PDPs are contained within the context itself.  We don't
2177          * need to do anything here. */
2178         if (HAS_LOGICAL_RING_CONTEXTS(dev_priv))
2179                 return 0;
2180
2181         if (!USES_PPGTT(dev_priv))
2182                 return 0;
2183
2184         if (IS_GEN6(dev_priv))
2185                 gen6_ppgtt_enable(dev_priv);
2186         else if (IS_GEN7(dev_priv))
2187                 gen7_ppgtt_enable(dev_priv);
2188         else if (INTEL_GEN(dev_priv) >= 8)
2189                 gen8_ppgtt_enable(dev_priv);
2190         else
2191                 MISSING_CASE(INTEL_GEN(dev_priv));
2192
2193         return 0;
2194 }
2195
2196 struct i915_hw_ppgtt *
2197 i915_ppgtt_create(struct drm_i915_private *dev_priv,
2198                   struct drm_i915_file_private *fpriv,
2199                   const char *name)
2200 {
2201         struct i915_hw_ppgtt *ppgtt;
2202         int ret;
2203
2204         ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
2205         if (!ppgtt)
2206                 return ERR_PTR(-ENOMEM);
2207
2208         ret = __hw_ppgtt_init(ppgtt, dev_priv);
2209         if (ret) {
2210                 kfree(ppgtt);
2211                 return ERR_PTR(ret);
2212         }
2213
2214         kref_init(&ppgtt->ref);
2215         i915_address_space_init(&ppgtt->base, dev_priv, name);
2216         ppgtt->base.file = fpriv;
2217
2218         trace_i915_ppgtt_create(&ppgtt->base);
2219
2220         return ppgtt;
2221 }
2222
2223 void i915_ppgtt_close(struct i915_address_space *vm)
2224 {
2225         struct list_head *phases[] = {
2226                 &vm->active_list,
2227                 &vm->inactive_list,
2228                 &vm->unbound_list,
2229                 NULL,
2230         }, **phase;
2231
2232         GEM_BUG_ON(vm->closed);
2233         vm->closed = true;
2234
2235         for (phase = phases; *phase; phase++) {
2236                 struct i915_vma *vma, *vn;
2237
2238                 list_for_each_entry_safe(vma, vn, *phase, vm_link)
2239                         if (!i915_vma_is_closed(vma))
2240                                 i915_vma_close(vma);
2241         }
2242 }
2243
2244 void i915_ppgtt_release(struct kref *kref)
2245 {
2246         struct i915_hw_ppgtt *ppgtt =
2247                 container_of(kref, struct i915_hw_ppgtt, ref);
2248
2249         trace_i915_ppgtt_release(&ppgtt->base);
2250
2251         /* vmas should already be unbound and destroyed */
2252         GEM_BUG_ON(!list_empty(&ppgtt->base.active_list));
2253         GEM_BUG_ON(!list_empty(&ppgtt->base.inactive_list));
2254         GEM_BUG_ON(!list_empty(&ppgtt->base.unbound_list));
2255
2256         ppgtt->base.cleanup(&ppgtt->base);
2257         i915_address_space_fini(&ppgtt->base);
2258         kfree(ppgtt);
2259 }
2260
2261 /* Certain Gen5 chipsets require require idling the GPU before
2262  * unmapping anything from the GTT when VT-d is enabled.
2263  */
2264 static bool needs_idle_maps(struct drm_i915_private *dev_priv)
2265 {
2266         /* Query intel_iommu to see if we need the workaround. Presumably that
2267          * was loaded first.
2268          */
2269         return IS_GEN5(dev_priv) && IS_MOBILE(dev_priv) && intel_vtd_active();
2270 }
2271
2272 static void gen6_check_and_clear_faults(struct drm_i915_private *dev_priv)
2273 {
2274         struct intel_engine_cs *engine;
2275         enum intel_engine_id id;
2276         u32 fault;
2277
2278         for_each_engine(engine, dev_priv, id) {
2279                 fault = I915_READ(RING_FAULT_REG(engine));
2280                 if (fault & RING_FAULT_VALID) {
2281                         DRM_DEBUG_DRIVER("Unexpected fault\n"
2282                                          "\tAddr: 0x%08lx\n"
2283                                          "\tAddress space: %s\n"
2284                                          "\tSource ID: %d\n"
2285                                          "\tType: %d\n",
2286                                          fault & PAGE_MASK,
2287                                          fault & RING_FAULT_GTTSEL_MASK ? "GGTT" : "PPGTT",
2288                                          RING_FAULT_SRCID(fault),
2289                                          RING_FAULT_FAULT_TYPE(fault));
2290                         I915_WRITE(RING_FAULT_REG(engine),
2291                                    fault & ~RING_FAULT_VALID);
2292                 }
2293         }
2294
2295         POSTING_READ(RING_FAULT_REG(dev_priv->engine[RCS]));
2296 }
2297
2298 static void gen8_check_and_clear_faults(struct drm_i915_private *dev_priv)
2299 {
2300         u32 fault = I915_READ(GEN8_RING_FAULT_REG);
2301
2302         if (fault & RING_FAULT_VALID) {
2303                 u32 fault_data0, fault_data1;
2304                 u64 fault_addr;
2305
2306                 fault_data0 = I915_READ(GEN8_FAULT_TLB_DATA0);
2307                 fault_data1 = I915_READ(GEN8_FAULT_TLB_DATA1);
2308                 fault_addr = ((u64)(fault_data1 & FAULT_VA_HIGH_BITS) << 44) |
2309                              ((u64)fault_data0 << 12);
2310
2311                 DRM_DEBUG_DRIVER("Unexpected fault\n"
2312                                  "\tAddr: 0x%08x_%08x\n"
2313                                  "\tAddress space: %s\n"
2314                                  "\tEngine ID: %d\n"
2315                                  "\tSource ID: %d\n"
2316                                  "\tType: %d\n",
2317                                  upper_32_bits(fault_addr),
2318                                  lower_32_bits(fault_addr),
2319                                  fault_data1 & FAULT_GTT_SEL ? "GGTT" : "PPGTT",
2320                                  GEN8_RING_FAULT_ENGINE_ID(fault),
2321                                  RING_FAULT_SRCID(fault),
2322                                  RING_FAULT_FAULT_TYPE(fault));
2323                 I915_WRITE(GEN8_RING_FAULT_REG,
2324                            fault & ~RING_FAULT_VALID);
2325         }
2326
2327         POSTING_READ(GEN8_RING_FAULT_REG);
2328 }
2329
2330 void i915_check_and_clear_faults(struct drm_i915_private *dev_priv)
2331 {
2332         /* From GEN8 onwards we only have one 'All Engine Fault Register' */
2333         if (INTEL_GEN(dev_priv) >= 8)
2334                 gen8_check_and_clear_faults(dev_priv);
2335         else if (INTEL_GEN(dev_priv) >= 6)
2336                 gen6_check_and_clear_faults(dev_priv);
2337         else
2338                 return;
2339 }
2340
2341 void i915_gem_suspend_gtt_mappings(struct drm_i915_private *dev_priv)
2342 {
2343         struct i915_ggtt *ggtt = &dev_priv->ggtt;
2344
2345         /* Don't bother messing with faults pre GEN6 as we have little
2346          * documentation supporting that it's a good idea.
2347          */
2348         if (INTEL_GEN(dev_priv) < 6)
2349                 return;
2350
2351         i915_check_and_clear_faults(dev_priv);
2352
2353         ggtt->base.clear_range(&ggtt->base, 0, ggtt->base.total);
2354
2355         i915_ggtt_invalidate(dev_priv);
2356 }
2357
2358 int i915_gem_gtt_prepare_pages(struct drm_i915_gem_object *obj,
2359                                struct sg_table *pages)
2360 {
2361         do {
2362                 if (dma_map_sg_attrs(&obj->base.dev->pdev->dev,
2363                                      pages->sgl, pages->nents,
2364                                      PCI_DMA_BIDIRECTIONAL,
2365                                      DMA_ATTR_NO_WARN))
2366                         return 0;
2367
2368                 /* If the DMA remap fails, one cause can be that we have
2369                  * too many objects pinned in a small remapping table,
2370                  * such as swiotlb. Incrementally purge all other objects and
2371                  * try again - if there are no more pages to remove from
2372                  * the DMA remapper, i915_gem_shrink will return 0.
2373                  */
2374                 GEM_BUG_ON(obj->mm.pages == pages);
2375         } while (i915_gem_shrink(to_i915(obj->base.dev),
2376                                  obj->base.size >> PAGE_SHIFT, NULL,
2377                                  I915_SHRINK_BOUND |
2378                                  I915_SHRINK_UNBOUND |
2379                                  I915_SHRINK_ACTIVE));
2380
2381         return -ENOSPC;
2382 }
2383
2384 static void gen8_set_pte(void __iomem *addr, gen8_pte_t pte)
2385 {
2386         writeq(pte, addr);
2387 }
2388
2389 static void gen8_ggtt_insert_page(struct i915_address_space *vm,
2390                                   dma_addr_t addr,
2391                                   u64 offset,
2392                                   enum i915_cache_level level,
2393                                   u32 unused)
2394 {
2395         struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
2396         gen8_pte_t __iomem *pte =
2397                 (gen8_pte_t __iomem *)ggtt->gsm + (offset >> PAGE_SHIFT);
2398
2399         gen8_set_pte(pte, gen8_pte_encode(addr, level));
2400
2401         ggtt->invalidate(vm->i915);
2402 }
2403
2404 static void gen8_ggtt_insert_entries(struct i915_address_space *vm,
2405                                      struct i915_vma *vma,
2406                                      enum i915_cache_level level,
2407                                      u32 unused)
2408 {
2409         struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
2410         struct sgt_iter sgt_iter;
2411         gen8_pte_t __iomem *gtt_entries;
2412         const gen8_pte_t pte_encode = gen8_pte_encode(0, level);
2413         dma_addr_t addr;
2414
2415         gtt_entries = (gen8_pte_t __iomem *)ggtt->gsm;
2416         gtt_entries += vma->node.start >> PAGE_SHIFT;
2417         for_each_sgt_dma(addr, sgt_iter, vma->pages)
2418                 gen8_set_pte(gtt_entries++, pte_encode | addr);
2419
2420         wmb();
2421
2422         /* This next bit makes the above posting read even more important. We
2423          * want to flush the TLBs only after we're certain all the PTE updates
2424          * have finished.
2425          */
2426         ggtt->invalidate(vm->i915);
2427 }
2428
2429 static void gen6_ggtt_insert_page(struct i915_address_space *vm,
2430                                   dma_addr_t addr,
2431                                   u64 offset,
2432                                   enum i915_cache_level level,
2433                                   u32 flags)
2434 {
2435         struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
2436         gen6_pte_t __iomem *pte =
2437                 (gen6_pte_t __iomem *)ggtt->gsm + (offset >> PAGE_SHIFT);
2438
2439         iowrite32(vm->pte_encode(addr, level, flags), pte);
2440
2441         ggtt->invalidate(vm->i915);
2442 }
2443
2444 /*
2445  * Binds an object into the global gtt with the specified cache level. The object
2446  * will be accessible to the GPU via commands whose operands reference offsets
2447  * within the global GTT as well as accessible by the GPU through the GMADR
2448  * mapped BAR (dev_priv->mm.gtt->gtt).
2449  */
2450 static void gen6_ggtt_insert_entries(struct i915_address_space *vm,
2451                                      struct i915_vma *vma,
2452                                      enum i915_cache_level level,
2453                                      u32 flags)
2454 {
2455         struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
2456         gen6_pte_t __iomem *entries = (gen6_pte_t __iomem *)ggtt->gsm;
2457         unsigned int i = vma->node.start >> PAGE_SHIFT;
2458         struct sgt_iter iter;
2459         dma_addr_t addr;
2460         for_each_sgt_dma(addr, iter, vma->pages)
2461                 iowrite32(vm->pte_encode(addr, level, flags), &entries[i++]);
2462         wmb();
2463
2464         /* This next bit makes the above posting read even more important. We
2465          * want to flush the TLBs only after we're certain all the PTE updates
2466          * have finished.
2467          */
2468         ggtt->invalidate(vm->i915);
2469 }
2470
2471 static void nop_clear_range(struct i915_address_space *vm,
2472                             u64 start, u64 length)
2473 {
2474 }
2475
2476 static void gen8_ggtt_clear_range(struct i915_address_space *vm,
2477                                   u64 start, u64 length)
2478 {
2479         struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
2480         unsigned first_entry = start >> PAGE_SHIFT;
2481         unsigned num_entries = length >> PAGE_SHIFT;
2482         const gen8_pte_t scratch_pte =
2483                 gen8_pte_encode(vm->scratch_page.daddr, I915_CACHE_LLC);
2484         gen8_pte_t __iomem *gtt_base =
2485                 (gen8_pte_t __iomem *)ggtt->gsm + first_entry;
2486         const int max_entries = ggtt_total_entries(ggtt) - first_entry;
2487         int i;
2488
2489         if (WARN(num_entries > max_entries,
2490                  "First entry = %d; Num entries = %d (max=%d)\n",
2491                  first_entry, num_entries, max_entries))
2492                 num_entries = max_entries;
2493
2494         for (i = 0; i < num_entries; i++)
2495                 gen8_set_pte(&gtt_base[i], scratch_pte);
2496 }
2497
2498 static void bxt_vtd_ggtt_wa(struct i915_address_space *vm)
2499 {
2500         struct drm_i915_private *dev_priv = vm->i915;
2501
2502         /*
2503          * Make sure the internal GAM fifo has been cleared of all GTT
2504          * writes before exiting stop_machine(). This guarantees that
2505          * any aperture accesses waiting to start in another process
2506          * cannot back up behind the GTT writes causing a hang.
2507          * The register can be any arbitrary GAM register.
2508          */
2509         POSTING_READ(GFX_FLSH_CNTL_GEN6);
2510 }
2511
2512 struct insert_page {
2513         struct i915_address_space *vm;
2514         dma_addr_t addr;
2515         u64 offset;
2516         enum i915_cache_level level;
2517 };
2518
2519 static int bxt_vtd_ggtt_insert_page__cb(void *_arg)
2520 {
2521         struct insert_page *arg = _arg;
2522
2523         gen8_ggtt_insert_page(arg->vm, arg->addr, arg->offset, arg->level, 0);
2524         bxt_vtd_ggtt_wa(arg->vm);
2525
2526         return 0;
2527 }
2528
2529 static void bxt_vtd_ggtt_insert_page__BKL(struct i915_address_space *vm,
2530                                           dma_addr_t addr,
2531                                           u64 offset,
2532                                           enum i915_cache_level level,
2533                                           u32 unused)
2534 {
2535         struct insert_page arg = { vm, addr, offset, level };
2536
2537         stop_machine(bxt_vtd_ggtt_insert_page__cb, &arg, NULL);
2538 }
2539
2540 struct insert_entries {
2541         struct i915_address_space *vm;
2542         struct i915_vma *vma;
2543         enum i915_cache_level level;
2544 };
2545
2546 static int bxt_vtd_ggtt_insert_entries__cb(void *_arg)
2547 {
2548         struct insert_entries *arg = _arg;
2549
2550         gen8_ggtt_insert_entries(arg->vm, arg->vma, arg->level, 0);
2551         bxt_vtd_ggtt_wa(arg->vm);
2552
2553         return 0;
2554 }
2555
2556 static void bxt_vtd_ggtt_insert_entries__BKL(struct i915_address_space *vm,
2557                                              struct i915_vma *vma,
2558                                              enum i915_cache_level level,
2559                                              u32 unused)
2560 {
2561         struct insert_entries arg = { vm, vma, level };
2562
2563         stop_machine(bxt_vtd_ggtt_insert_entries__cb, &arg, NULL);
2564 }
2565
2566 struct clear_range {
2567         struct i915_address_space *vm;
2568         u64 start;
2569         u64 length;
2570 };
2571
2572 static int bxt_vtd_ggtt_clear_range__cb(void *_arg)
2573 {
2574         struct clear_range *arg = _arg;
2575
2576         gen8_ggtt_clear_range(arg->vm, arg->start, arg->length);
2577         bxt_vtd_ggtt_wa(arg->vm);
2578
2579         return 0;
2580 }
2581
2582 static void bxt_vtd_ggtt_clear_range__BKL(struct i915_address_space *vm,
2583                                           u64 start,
2584                                           u64 length)
2585 {
2586         struct clear_range arg = { vm, start, length };
2587
2588         stop_machine(bxt_vtd_ggtt_clear_range__cb, &arg, NULL);
2589 }
2590
2591 static void gen6_ggtt_clear_range(struct i915_address_space *vm,
2592                                   u64 start, u64 length)
2593 {
2594         struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
2595         unsigned first_entry = start >> PAGE_SHIFT;
2596         unsigned num_entries = length >> PAGE_SHIFT;
2597         gen6_pte_t scratch_pte, __iomem *gtt_base =
2598                 (gen6_pte_t __iomem *)ggtt->gsm + first_entry;
2599         const int max_entries = ggtt_total_entries(ggtt) - first_entry;
2600         int i;
2601
2602         if (WARN(num_entries > max_entries,
2603                  "First entry = %d; Num entries = %d (max=%d)\n",
2604                  first_entry, num_entries, max_entries))
2605                 num_entries = max_entries;
2606
2607         scratch_pte = vm->pte_encode(vm->scratch_page.daddr,
2608                                      I915_CACHE_LLC, 0);
2609
2610         for (i = 0; i < num_entries; i++)
2611                 iowrite32(scratch_pte, &gtt_base[i]);
2612 }
2613
2614 static void i915_ggtt_insert_page(struct i915_address_space *vm,
2615                                   dma_addr_t addr,
2616                                   u64 offset,
2617                                   enum i915_cache_level cache_level,
2618                                   u32 unused)
2619 {
2620         unsigned int flags = (cache_level == I915_CACHE_NONE) ?
2621                 AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY;
2622
2623         intel_gtt_insert_page(addr, offset >> PAGE_SHIFT, flags);
2624 }
2625
2626 static void i915_ggtt_insert_entries(struct i915_address_space *vm,
2627                                      struct i915_vma *vma,
2628                                      enum i915_cache_level cache_level,
2629                                      u32 unused)
2630 {
2631         unsigned int flags = (cache_level == I915_CACHE_NONE) ?
2632                 AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY;
2633
2634         intel_gtt_insert_sg_entries(vma->pages, vma->node.start >> PAGE_SHIFT,
2635                                     flags);
2636 }
2637
2638 static void i915_ggtt_clear_range(struct i915_address_space *vm,
2639                                   u64 start, u64 length)
2640 {
2641         intel_gtt_clear_range(start >> PAGE_SHIFT, length >> PAGE_SHIFT);
2642 }
2643
2644 static int ggtt_bind_vma(struct i915_vma *vma,
2645                          enum i915_cache_level cache_level,
2646                          u32 flags)
2647 {
2648         struct drm_i915_private *i915 = vma->vm->i915;
2649         struct drm_i915_gem_object *obj = vma->obj;
2650         u32 pte_flags;
2651
2652         /* Currently applicable only to VLV */
2653         pte_flags = 0;
2654         if (obj->gt_ro)
2655                 pte_flags |= PTE_READ_ONLY;
2656
2657         intel_runtime_pm_get(i915);
2658         vma->vm->insert_entries(vma->vm, vma, cache_level, pte_flags);
2659         intel_runtime_pm_put(i915);
2660
2661         vma->page_sizes.gtt = I915_GTT_PAGE_SIZE;
2662
2663         /*
2664          * Without aliasing PPGTT there's no difference between
2665          * GLOBAL/LOCAL_BIND, it's all the same ptes. Hence unconditionally
2666          * upgrade to both bound if we bind either to avoid double-binding.
2667          */
2668         vma->flags |= I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND;
2669
2670         return 0;
2671 }
2672
2673 static void ggtt_unbind_vma(struct i915_vma *vma)
2674 {
2675         struct drm_i915_private *i915 = vma->vm->i915;
2676
2677         intel_runtime_pm_get(i915);
2678         vma->vm->clear_range(vma->vm, vma->node.start, vma->size);
2679         intel_runtime_pm_put(i915);
2680 }
2681
2682 static int aliasing_gtt_bind_vma(struct i915_vma *vma,
2683                                  enum i915_cache_level cache_level,
2684                                  u32 flags)
2685 {
2686         struct drm_i915_private *i915 = vma->vm->i915;
2687         u32 pte_flags;
2688         int ret;
2689
2690         /* Currently applicable only to VLV */
2691         pte_flags = 0;
2692         if (vma->obj->gt_ro)
2693                 pte_flags |= PTE_READ_ONLY;
2694
2695         if (flags & I915_VMA_LOCAL_BIND) {
2696                 struct i915_hw_ppgtt *appgtt = i915->mm.aliasing_ppgtt;
2697
2698                 if (!(vma->flags & I915_VMA_LOCAL_BIND) &&
2699                     appgtt->base.allocate_va_range) {
2700                         ret = appgtt->base.allocate_va_range(&appgtt->base,
2701                                                              vma->node.start,
2702                                                              vma->size);
2703                         if (ret)
2704                                 return ret;
2705                 }
2706
2707                 appgtt->base.insert_entries(&appgtt->base, vma, cache_level,
2708                                             pte_flags);
2709         }
2710
2711         if (flags & I915_VMA_GLOBAL_BIND) {
2712                 intel_runtime_pm_get(i915);
2713                 vma->vm->insert_entries(vma->vm, vma, cache_level, pte_flags);
2714                 intel_runtime_pm_put(i915);
2715         }
2716
2717         return 0;
2718 }
2719
2720 static void aliasing_gtt_unbind_vma(struct i915_vma *vma)
2721 {
2722         struct drm_i915_private *i915 = vma->vm->i915;
2723
2724         if (vma->flags & I915_VMA_GLOBAL_BIND) {
2725                 intel_runtime_pm_get(i915);
2726                 vma->vm->clear_range(vma->vm, vma->node.start, vma->size);
2727                 intel_runtime_pm_put(i915);
2728         }
2729
2730         if (vma->flags & I915_VMA_LOCAL_BIND) {
2731                 struct i915_address_space *vm = &i915->mm.aliasing_ppgtt->base;
2732
2733                 vm->clear_range(vm, vma->node.start, vma->size);
2734         }
2735 }
2736
2737 void i915_gem_gtt_finish_pages(struct drm_i915_gem_object *obj,
2738                                struct sg_table *pages)
2739 {
2740         struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
2741         struct device *kdev = &dev_priv->drm.pdev->dev;
2742         struct i915_ggtt *ggtt = &dev_priv->ggtt;
2743
2744         if (unlikely(ggtt->do_idle_maps)) {
2745                 if (i915_gem_wait_for_idle(dev_priv, 0)) {
2746                         DRM_ERROR("Failed to wait for idle; VT'd may hang.\n");
2747                         /* Wait a bit, in hopes it avoids the hang */
2748                         udelay(10);
2749                 }
2750         }
2751
2752         dma_unmap_sg(kdev, pages->sgl, pages->nents, PCI_DMA_BIDIRECTIONAL);
2753 }
2754
2755 static int ggtt_set_pages(struct i915_vma *vma)
2756 {
2757         int ret;
2758
2759         GEM_BUG_ON(vma->pages);
2760
2761         ret = i915_get_ggtt_vma_pages(vma);
2762         if (ret)
2763                 return ret;
2764
2765         vma->page_sizes = vma->obj->mm.page_sizes;
2766
2767         return 0;
2768 }
2769
2770 static void i915_gtt_color_adjust(const struct drm_mm_node *node,
2771                                   unsigned long color,
2772                                   u64 *start,
2773                                   u64 *end)
2774 {
2775         if (node->allocated && node->color != color)
2776                 *start += I915_GTT_PAGE_SIZE;
2777
2778         /* Also leave a space between the unallocated reserved node after the
2779          * GTT and any objects within the GTT, i.e. we use the color adjustment
2780          * to insert a guard page to prevent prefetches crossing over the
2781          * GTT boundary.
2782          */
2783         node = list_next_entry(node, node_list);
2784         if (node->color != color)
2785                 *end -= I915_GTT_PAGE_SIZE;
2786 }
2787
2788 int i915_gem_init_aliasing_ppgtt(struct drm_i915_private *i915)
2789 {
2790         struct i915_ggtt *ggtt = &i915->ggtt;
2791         struct i915_hw_ppgtt *ppgtt;
2792         int err;
2793
2794         ppgtt = i915_ppgtt_create(i915, ERR_PTR(-EPERM), "[alias]");
2795         if (IS_ERR(ppgtt))
2796                 return PTR_ERR(ppgtt);
2797
2798         if (WARN_ON(ppgtt->base.total < ggtt->base.total)) {
2799                 err = -ENODEV;
2800                 goto err_ppgtt;
2801         }
2802
2803         if (ppgtt->base.allocate_va_range) {
2804                 /* Note we only pre-allocate as far as the end of the global
2805                  * GTT. On 48b / 4-level page-tables, the difference is very,
2806                  * very significant! We have to preallocate as GVT/vgpu does
2807                  * not like the page directory disappearing.
2808                  */
2809                 err = ppgtt->base.allocate_va_range(&ppgtt->base,
2810                                                     0, ggtt->base.total);
2811                 if (err)
2812                         goto err_ppgtt;
2813         }
2814
2815         i915->mm.aliasing_ppgtt = ppgtt;
2816
2817         GEM_BUG_ON(ggtt->base.bind_vma != ggtt_bind_vma);
2818         ggtt->base.bind_vma = aliasing_gtt_bind_vma;
2819
2820         GEM_BUG_ON(ggtt->base.unbind_vma != ggtt_unbind_vma);
2821         ggtt->base.unbind_vma = aliasing_gtt_unbind_vma;
2822
2823         return 0;
2824
2825 err_ppgtt:
2826         i915_ppgtt_put(ppgtt);
2827         return err;
2828 }
2829
2830 void i915_gem_fini_aliasing_ppgtt(struct drm_i915_private *i915)
2831 {
2832         struct i915_ggtt *ggtt = &i915->ggtt;
2833         struct i915_hw_ppgtt *ppgtt;
2834
2835         ppgtt = fetch_and_zero(&i915->mm.aliasing_ppgtt);
2836         if (!ppgtt)
2837                 return;
2838
2839         i915_ppgtt_put(ppgtt);
2840
2841         ggtt->base.bind_vma = ggtt_bind_vma;
2842         ggtt->base.unbind_vma = ggtt_unbind_vma;
2843 }
2844
2845 int i915_gem_init_ggtt(struct drm_i915_private *dev_priv)
2846 {
2847         /* Let GEM Manage all of the aperture.
2848          *
2849          * However, leave one page at the end still bound to the scratch page.
2850          * There are a number of places where the hardware apparently prefetches
2851          * past the end of the object, and we've seen multiple hangs with the
2852          * GPU head pointer stuck in a batchbuffer bound at the last page of the
2853          * aperture.  One page should be enough to keep any prefetching inside
2854          * of the aperture.
2855          */
2856         struct i915_ggtt *ggtt = &dev_priv->ggtt;
2857         unsigned long hole_start, hole_end;
2858         struct drm_mm_node *entry;
2859         int ret;
2860
2861         ret = intel_vgt_balloon(dev_priv);
2862         if (ret)
2863                 return ret;
2864
2865         /* Reserve a mappable slot for our lockless error capture */
2866         ret = drm_mm_insert_node_in_range(&ggtt->base.mm, &ggtt->error_capture,
2867                                           PAGE_SIZE, 0, I915_COLOR_UNEVICTABLE,
2868                                           0, ggtt->mappable_end,
2869                                           DRM_MM_INSERT_LOW);
2870         if (ret)
2871                 return ret;
2872
2873         /* Clear any non-preallocated blocks */
2874         drm_mm_for_each_hole(entry, &ggtt->base.mm, hole_start, hole_end) {
2875                 DRM_DEBUG_KMS("clearing unused GTT space: [%lx, %lx]\n",
2876                               hole_start, hole_end);
2877                 ggtt->base.clear_range(&ggtt->base, hole_start,
2878                                        hole_end - hole_start);
2879         }
2880
2881         /* And finally clear the reserved guard page */
2882         ggtt->base.clear_range(&ggtt->base,
2883                                ggtt->base.total - PAGE_SIZE, PAGE_SIZE);
2884
2885         if (USES_PPGTT(dev_priv) && !USES_FULL_PPGTT(dev_priv)) {
2886                 ret = i915_gem_init_aliasing_ppgtt(dev_priv);
2887                 if (ret)
2888                         goto err;
2889         }
2890
2891         return 0;
2892
2893 err:
2894         drm_mm_remove_node(&ggtt->error_capture);
2895         return ret;
2896 }
2897
2898 /**
2899  * i915_ggtt_cleanup_hw - Clean up GGTT hardware initialization
2900  * @dev_priv: i915 device
2901  */
2902 void i915_ggtt_cleanup_hw(struct drm_i915_private *dev_priv)
2903 {
2904         struct i915_ggtt *ggtt = &dev_priv->ggtt;
2905         struct i915_vma *vma, *vn;
2906         struct pagevec *pvec;
2907
2908         ggtt->base.closed = true;
2909
2910         mutex_lock(&dev_priv->drm.struct_mutex);
2911         GEM_BUG_ON(!list_empty(&ggtt->base.active_list));
2912         list_for_each_entry_safe(vma, vn, &ggtt->base.inactive_list, vm_link)
2913                 WARN_ON(i915_vma_unbind(vma));
2914         mutex_unlock(&dev_priv->drm.struct_mutex);
2915
2916         i915_gem_cleanup_stolen(&dev_priv->drm);
2917
2918         mutex_lock(&dev_priv->drm.struct_mutex);
2919         i915_gem_fini_aliasing_ppgtt(dev_priv);
2920
2921         if (drm_mm_node_allocated(&ggtt->error_capture))
2922                 drm_mm_remove_node(&ggtt->error_capture);
2923
2924         if (drm_mm_initialized(&ggtt->base.mm)) {
2925                 intel_vgt_deballoon(dev_priv);
2926                 i915_address_space_fini(&ggtt->base);
2927         }
2928
2929         ggtt->base.cleanup(&ggtt->base);
2930
2931         pvec = &dev_priv->mm.wc_stash;
2932         if (pvec->nr) {
2933                 set_pages_array_wb(pvec->pages, pvec->nr);
2934                 __pagevec_release(pvec);
2935         }
2936
2937         mutex_unlock(&dev_priv->drm.struct_mutex);
2938
2939         arch_phys_wc_del(ggtt->mtrr);
2940         io_mapping_fini(&ggtt->iomap);
2941 }
2942
2943 static unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl)
2944 {
2945         snb_gmch_ctl >>= SNB_GMCH_GGMS_SHIFT;
2946         snb_gmch_ctl &= SNB_GMCH_GGMS_MASK;
2947         return snb_gmch_ctl << 20;
2948 }
2949
2950 static unsigned int gen8_get_total_gtt_size(u16 bdw_gmch_ctl)
2951 {
2952         bdw_gmch_ctl >>= BDW_GMCH_GGMS_SHIFT;
2953         bdw_gmch_ctl &= BDW_GMCH_GGMS_MASK;
2954         if (bdw_gmch_ctl)
2955                 bdw_gmch_ctl = 1 << bdw_gmch_ctl;
2956
2957 #ifdef CONFIG_X86_32
2958         /* Limit 32b platforms to a 2GB GGTT: 4 << 20 / pte size * PAGE_SIZE */
2959         if (bdw_gmch_ctl > 4)
2960                 bdw_gmch_ctl = 4;
2961 #endif
2962
2963         return bdw_gmch_ctl << 20;
2964 }
2965
2966 static unsigned int chv_get_total_gtt_size(u16 gmch_ctrl)
2967 {
2968         gmch_ctrl >>= SNB_GMCH_GGMS_SHIFT;
2969         gmch_ctrl &= SNB_GMCH_GGMS_MASK;
2970
2971         if (gmch_ctrl)
2972                 return 1 << (20 + gmch_ctrl);
2973
2974         return 0;
2975 }
2976
2977 static int ggtt_probe_common(struct i915_ggtt *ggtt, u64 size)
2978 {
2979         struct drm_i915_private *dev_priv = ggtt->base.i915;
2980         struct pci_dev *pdev = dev_priv->drm.pdev;
2981         phys_addr_t phys_addr;
2982         int ret;
2983
2984         /* For Modern GENs the PTEs and register space are split in the BAR */
2985         phys_addr = pci_resource_start(pdev, 0) + pci_resource_len(pdev, 0) / 2;
2986
2987         /*
2988          * On BXT+/CNL+ writes larger than 64 bit to the GTT pagetable range
2989          * will be dropped. For WC mappings in general we have 64 byte burst
2990          * writes when the WC buffer is flushed, so we can't use it, but have to
2991          * resort to an uncached mapping. The WC issue is easily caught by the
2992          * readback check when writing GTT PTE entries.
2993          */
2994         if (IS_GEN9_LP(dev_priv) || INTEL_GEN(dev_priv) >= 10)
2995                 ggtt->gsm = ioremap_nocache(phys_addr, size);
2996         else
2997                 ggtt->gsm = ioremap_wc(phys_addr, size);
2998         if (!ggtt->gsm) {
2999                 DRM_ERROR("Failed to map the ggtt page table\n");
3000                 return -ENOMEM;
3001         }
3002
3003         ret = setup_scratch_page(&ggtt->base, GFP_DMA32);
3004         if (ret) {
3005                 DRM_ERROR("Scratch setup failed\n");
3006                 /* iounmap will also get called at remove, but meh */
3007                 iounmap(ggtt->gsm);
3008                 return ret;
3009         }
3010
3011         return 0;
3012 }
3013
3014 static struct intel_ppat_entry *
3015 __alloc_ppat_entry(struct intel_ppat *ppat, unsigned int index, u8 value)
3016 {
3017         struct intel_ppat_entry *entry = &ppat->entries[index];
3018
3019         GEM_BUG_ON(index >= ppat->max_entries);
3020         GEM_BUG_ON(test_bit(index, ppat->used));
3021
3022         entry->ppat = ppat;
3023         entry->value = value;
3024         kref_init(&entry->ref);
3025         set_bit(index, ppat->used);
3026         set_bit(index, ppat->dirty);
3027
3028         return entry;
3029 }
3030
3031 static void __free_ppat_entry(struct intel_ppat_entry *entry)
3032 {
3033         struct intel_ppat *ppat = entry->ppat;
3034         unsigned int index = entry - ppat->entries;
3035
3036         GEM_BUG_ON(index >= ppat->max_entries);
3037         GEM_BUG_ON(!test_bit(index, ppat->used));
3038
3039         entry->value = ppat->clear_value;
3040         clear_bit(index, ppat->used);
3041         set_bit(index, ppat->dirty);
3042 }
3043
3044 /**
3045  * intel_ppat_get - get a usable PPAT entry
3046  * @i915: i915 device instance
3047  * @value: the PPAT value required by the caller
3048  *
3049  * The function tries to search if there is an existing PPAT entry which
3050  * matches with the required value. If perfectly matched, the existing PPAT
3051  * entry will be used. If only partially matched, it will try to check if
3052  * there is any available PPAT index. If yes, it will allocate a new PPAT
3053  * index for the required entry and update the HW. If not, the partially
3054  * matched entry will be used.
3055  */
3056 const struct intel_ppat_entry *
3057 intel_ppat_get(struct drm_i915_private *i915, u8 value)
3058 {
3059         struct intel_ppat *ppat = &i915->ppat;
3060         struct intel_ppat_entry *entry = NULL;
3061         unsigned int scanned, best_score;
3062         int i;
3063
3064         GEM_BUG_ON(!ppat->max_entries);
3065
3066         scanned = best_score = 0;
3067         for_each_set_bit(i, ppat->used, ppat->max_entries) {
3068                 unsigned int score;
3069
3070                 score = ppat->match(ppat->entries[i].value, value);
3071                 if (score > best_score) {
3072                         entry = &ppat->entries[i];
3073                         if (score == INTEL_PPAT_PERFECT_MATCH) {
3074                                 kref_get(&entry->ref);
3075                                 return entry;
3076                         }
3077                         best_score = score;
3078                 }
3079                 scanned++;
3080         }
3081
3082         if (scanned == ppat->max_entries) {
3083                 if (!entry)
3084                         return ERR_PTR(-ENOSPC);
3085
3086                 kref_get(&entry->ref);
3087                 return entry;
3088         }
3089
3090         i = find_first_zero_bit(ppat->used, ppat->max_entries);
3091         entry = __alloc_ppat_entry(ppat, i, value);
3092         ppat->update_hw(i915);
3093         return entry;
3094 }
3095
3096 static void release_ppat(struct kref *kref)
3097 {
3098         struct intel_ppat_entry *entry =
3099                 container_of(kref, struct intel_ppat_entry, ref);
3100         struct drm_i915_private *i915 = entry->ppat->i915;
3101
3102         __free_ppat_entry(entry);
3103         entry->ppat->update_hw(i915);
3104 }
3105
3106 /**
3107  * intel_ppat_put - put back the PPAT entry got from intel_ppat_get()
3108  * @entry: an intel PPAT entry
3109  *
3110  * Put back the PPAT entry got from intel_ppat_get(). If the PPAT index of the
3111  * entry is dynamically allocated, its reference count will be decreased. Once
3112  * the reference count becomes into zero, the PPAT index becomes free again.
3113  */
3114 void intel_ppat_put(const struct intel_ppat_entry *entry)
3115 {
3116         struct intel_ppat *ppat = entry->ppat;
3117         unsigned int index = entry - ppat->entries;
3118
3119         GEM_BUG_ON(!ppat->max_entries);
3120
3121         kref_put(&ppat->entries[index].ref, release_ppat);
3122 }
3123
3124 static void cnl_private_pat_update_hw(struct drm_i915_private *dev_priv)
3125 {
3126         struct intel_ppat *ppat = &dev_priv->ppat;
3127         int i;
3128
3129         for_each_set_bit(i, ppat->dirty, ppat->max_entries) {
3130                 I915_WRITE(GEN10_PAT_INDEX(i), ppat->entries[i].value);
3131                 clear_bit(i, ppat->dirty);
3132         }
3133 }
3134
3135 static void bdw_private_pat_update_hw(struct drm_i915_private *dev_priv)
3136 {
3137         struct intel_ppat *ppat = &dev_priv->ppat;
3138         u64 pat = 0;
3139         int i;
3140
3141         for (i = 0; i < ppat->max_entries; i++)
3142                 pat |= GEN8_PPAT(i, ppat->entries[i].value);
3143
3144         bitmap_clear(ppat->dirty, 0, ppat->max_entries);
3145
3146         I915_WRITE(GEN8_PRIVATE_PAT_LO, lower_32_bits(pat));
3147         I915_WRITE(GEN8_PRIVATE_PAT_HI, upper_32_bits(pat));
3148 }
3149
3150 static unsigned int bdw_private_pat_match(u8 src, u8 dst)
3151 {
3152         unsigned int score = 0;
3153         enum {
3154                 AGE_MATCH = BIT(0),
3155                 TC_MATCH = BIT(1),
3156                 CA_MATCH = BIT(2),
3157         };
3158
3159         /* Cache attribute has to be matched. */
3160         if (GEN8_PPAT_GET_CA(src) != GEN8_PPAT_GET_CA(dst))
3161                 return 0;
3162
3163         score |= CA_MATCH;
3164
3165         if (GEN8_PPAT_GET_TC(src) == GEN8_PPAT_GET_TC(dst))
3166                 score |= TC_MATCH;
3167
3168         if (GEN8_PPAT_GET_AGE(src) == GEN8_PPAT_GET_AGE(dst))
3169                 score |= AGE_MATCH;
3170
3171         if (score == (AGE_MATCH | TC_MATCH | CA_MATCH))
3172                 return INTEL_PPAT_PERFECT_MATCH;
3173
3174         return score;
3175 }
3176
3177 static unsigned int chv_private_pat_match(u8 src, u8 dst)
3178 {
3179         return (CHV_PPAT_GET_SNOOP(src) == CHV_PPAT_GET_SNOOP(dst)) ?
3180                 INTEL_PPAT_PERFECT_MATCH : 0;
3181 }
3182
3183 static void cnl_setup_private_ppat(struct intel_ppat *ppat)
3184 {
3185         ppat->max_entries = 8;
3186         ppat->update_hw = cnl_private_pat_update_hw;
3187         ppat->match = bdw_private_pat_match;
3188         ppat->clear_value = GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(3);
3189
3190         __alloc_ppat_entry(ppat, 0, GEN8_PPAT_WB | GEN8_PPAT_LLC);
3191         __alloc_ppat_entry(ppat, 1, GEN8_PPAT_WC | GEN8_PPAT_LLCELLC);
3192         __alloc_ppat_entry(ppat, 2, GEN8_PPAT_WT | GEN8_PPAT_LLCELLC);
3193         __alloc_ppat_entry(ppat, 3, GEN8_PPAT_UC);
3194         __alloc_ppat_entry(ppat, 4, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(0));
3195         __alloc_ppat_entry(ppat, 5, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(1));
3196         __alloc_ppat_entry(ppat, 6, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(2));
3197         __alloc_ppat_entry(ppat, 7, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(3));
3198 }
3199
3200 /* The GGTT and PPGTT need a private PPAT setup in order to handle cacheability
3201  * bits. When using advanced contexts each context stores its own PAT, but
3202  * writing this data shouldn't be harmful even in those cases. */
3203 static void bdw_setup_private_ppat(struct intel_ppat *ppat)
3204 {
3205         ppat->max_entries = 8;
3206         ppat->update_hw = bdw_private_pat_update_hw;
3207         ppat->match = bdw_private_pat_match;
3208         ppat->clear_value = GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(3);
3209
3210         if (!USES_PPGTT(ppat->i915)) {
3211                 /* Spec: "For GGTT, there is NO pat_sel[2:0] from the entry,
3212                  * so RTL will always use the value corresponding to
3213                  * pat_sel = 000".
3214                  * So let's disable cache for GGTT to avoid screen corruptions.
3215                  * MOCS still can be used though.
3216                  * - System agent ggtt writes (i.e. cpu gtt mmaps) already work
3217                  * before this patch, i.e. the same uncached + snooping access
3218                  * like on gen6/7 seems to be in effect.
3219                  * - So this just fixes blitter/render access. Again it looks
3220                  * like it's not just uncached access, but uncached + snooping.
3221                  * So we can still hold onto all our assumptions wrt cpu
3222                  * clflushing on LLC machines.
3223                  */
3224                 __alloc_ppat_entry(ppat, 0, GEN8_PPAT_UC);
3225                 return;
3226         }
3227
3228         __alloc_ppat_entry(ppat, 0, GEN8_PPAT_WB | GEN8_PPAT_LLC);      /* for normal objects, no eLLC */
3229         __alloc_ppat_entry(ppat, 1, GEN8_PPAT_WC | GEN8_PPAT_LLCELLC);  /* for something pointing to ptes? */
3230         __alloc_ppat_entry(ppat, 2, GEN8_PPAT_WT | GEN8_PPAT_LLCELLC);  /* for scanout with eLLC */
3231         __alloc_ppat_entry(ppat, 3, GEN8_PPAT_UC);                      /* Uncached objects, mostly for scanout */
3232         __alloc_ppat_entry(ppat, 4, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(0));
3233         __alloc_ppat_entry(ppat, 5, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(1));
3234         __alloc_ppat_entry(ppat, 6, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(2));
3235         __alloc_ppat_entry(ppat, 7, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(3));
3236 }
3237
3238 static void chv_setup_private_ppat(struct intel_ppat *ppat)
3239 {
3240         ppat->max_entries = 8;
3241         ppat->update_hw = bdw_private_pat_update_hw;
3242         ppat->match = chv_private_pat_match;
3243         ppat->clear_value = CHV_PPAT_SNOOP;
3244
3245         /*
3246          * Map WB on BDW to snooped on CHV.
3247          *
3248          * Only the snoop bit has meaning for CHV, the rest is
3249          * ignored.
3250          *
3251          * The hardware will never snoop for certain types of accesses:
3252          * - CPU GTT (GMADR->GGTT->no snoop->memory)
3253          * - PPGTT page tables
3254          * - some other special cycles
3255          *
3256          * As with BDW, we also need to consider the following for GT accesses:
3257          * "For GGTT, there is NO pat_sel[2:0] from the entry,
3258          * so RTL will always use the value corresponding to
3259          * pat_sel = 000".
3260          * Which means we must set the snoop bit in PAT entry 0
3261          * in order to keep the global status page working.
3262          */
3263
3264         __alloc_ppat_entry(ppat, 0, CHV_PPAT_SNOOP);
3265         __alloc_ppat_entry(ppat, 1, 0);
3266         __alloc_ppat_entry(ppat, 2, 0);
3267         __alloc_ppat_entry(ppat, 3, 0);
3268         __alloc_ppat_entry(ppat, 4, CHV_PPAT_SNOOP);
3269         __alloc_ppat_entry(ppat, 5, CHV_PPAT_SNOOP);
3270         __alloc_ppat_entry(ppat, 6, CHV_PPAT_SNOOP);
3271         __alloc_ppat_entry(ppat, 7, CHV_PPAT_SNOOP);
3272 }
3273
3274 static void gen6_gmch_remove(struct i915_address_space *vm)
3275 {
3276         struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
3277
3278         iounmap(ggtt->gsm);
3279         cleanup_scratch_page(vm);
3280 }
3281
3282 static void setup_private_pat(struct drm_i915_private *dev_priv)
3283 {
3284         struct intel_ppat *ppat = &dev_priv->ppat;
3285         int i;
3286
3287         ppat->i915 = dev_priv;
3288
3289         if (INTEL_GEN(dev_priv) >= 10)
3290                 cnl_setup_private_ppat(ppat);
3291         else if (IS_CHERRYVIEW(dev_priv) || IS_GEN9_LP(dev_priv))
3292                 chv_setup_private_ppat(ppat);
3293         else
3294                 bdw_setup_private_ppat(ppat);
3295
3296         GEM_BUG_ON(ppat->max_entries > INTEL_MAX_PPAT_ENTRIES);
3297
3298         for_each_clear_bit(i, ppat->used, ppat->max_entries) {
3299                 ppat->entries[i].value = ppat->clear_value;
3300                 ppat->entries[i].ppat = ppat;
3301                 set_bit(i, ppat->dirty);
3302         }
3303
3304         ppat->update_hw(dev_priv);
3305 }
3306
3307 static int gen8_gmch_probe(struct i915_ggtt *ggtt)
3308 {
3309         struct drm_i915_private *dev_priv = ggtt->base.i915;
3310         struct pci_dev *pdev = dev_priv->drm.pdev;
3311         unsigned int size;
3312         u16 snb_gmch_ctl;
3313         int err;
3314
3315         /* TODO: We're not aware of mappable constraints on gen8 yet */
3316         ggtt->gmadr =
3317                 (struct resource) DEFINE_RES_MEM(pci_resource_start(pdev, 2),
3318                                                  pci_resource_len(pdev, 2));
3319         ggtt->mappable_end = resource_size(&ggtt->gmadr);
3320
3321         err = pci_set_dma_mask(pdev, DMA_BIT_MASK(39));
3322         if (!err)
3323                 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(39));
3324         if (err)
3325                 DRM_ERROR("Can't set DMA mask/consistent mask (%d)\n", err);
3326
3327         pci_read_config_word(pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
3328
3329         if (INTEL_GEN(dev_priv) >= 9) {
3330                 size = gen8_get_total_gtt_size(snb_gmch_ctl);
3331         } else if (IS_CHERRYVIEW(dev_priv)) {
3332                 size = chv_get_total_gtt_size(snb_gmch_ctl);
3333         } else {
3334                 size = gen8_get_total_gtt_size(snb_gmch_ctl);
3335         }
3336
3337         ggtt->base.total = (size / sizeof(gen8_pte_t)) << PAGE_SHIFT;
3338         ggtt->base.cleanup = gen6_gmch_remove;
3339         ggtt->base.bind_vma = ggtt_bind_vma;
3340         ggtt->base.unbind_vma = ggtt_unbind_vma;
3341         ggtt->base.set_pages = ggtt_set_pages;
3342         ggtt->base.clear_pages = clear_pages;
3343         ggtt->base.insert_page = gen8_ggtt_insert_page;
3344         ggtt->base.clear_range = nop_clear_range;
3345         if (!USES_FULL_PPGTT(dev_priv) || intel_scanout_needs_vtd_wa(dev_priv))
3346                 ggtt->base.clear_range = gen8_ggtt_clear_range;
3347
3348         ggtt->base.insert_entries = gen8_ggtt_insert_entries;
3349
3350         /* Serialize GTT updates with aperture access on BXT if VT-d is on. */
3351         if (intel_ggtt_update_needs_vtd_wa(dev_priv)) {
3352                 ggtt->base.insert_entries = bxt_vtd_ggtt_insert_entries__BKL;
3353                 ggtt->base.insert_page    = bxt_vtd_ggtt_insert_page__BKL;
3354                 if (ggtt->base.clear_range != nop_clear_range)
3355                         ggtt->base.clear_range = bxt_vtd_ggtt_clear_range__BKL;
3356         }
3357
3358         ggtt->invalidate = gen6_ggtt_invalidate;
3359
3360         setup_private_pat(dev_priv);
3361
3362         return ggtt_probe_common(ggtt, size);
3363 }
3364
3365 static int gen6_gmch_probe(struct i915_ggtt *ggtt)
3366 {
3367         struct drm_i915_private *dev_priv = ggtt->base.i915;
3368         struct pci_dev *pdev = dev_priv->drm.pdev;
3369         unsigned int size;
3370         u16 snb_gmch_ctl;
3371         int err;
3372
3373         ggtt->gmadr =
3374                 (struct resource) DEFINE_RES_MEM(pci_resource_start(pdev, 2),
3375                                                  pci_resource_len(pdev, 2));
3376         ggtt->mappable_end = resource_size(&ggtt->gmadr);
3377
3378         /* 64/512MB is the current min/max we actually know of, but this is just
3379          * a coarse sanity check.
3380          */
3381         if (ggtt->mappable_end < (64<<20) || ggtt->mappable_end > (512<<20)) {
3382                 DRM_ERROR("Unknown GMADR size (%pa)\n", &ggtt->mappable_end);
3383                 return -ENXIO;
3384         }
3385
3386         err = pci_set_dma_mask(pdev, DMA_BIT_MASK(40));
3387         if (!err)
3388                 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(40));
3389         if (err)
3390                 DRM_ERROR("Can't set DMA mask/consistent mask (%d)\n", err);
3391         pci_read_config_word(pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
3392
3393         size = gen6_get_total_gtt_size(snb_gmch_ctl);
3394         ggtt->base.total = (size / sizeof(gen6_pte_t)) << PAGE_SHIFT;
3395
3396         ggtt->base.clear_range = gen6_ggtt_clear_range;
3397         ggtt->base.insert_page = gen6_ggtt_insert_page;
3398         ggtt->base.insert_entries = gen6_ggtt_insert_entries;
3399         ggtt->base.bind_vma = ggtt_bind_vma;
3400         ggtt->base.unbind_vma = ggtt_unbind_vma;
3401         ggtt->base.set_pages = ggtt_set_pages;
3402         ggtt->base.clear_pages = clear_pages;
3403         ggtt->base.cleanup = gen6_gmch_remove;
3404
3405         ggtt->invalidate = gen6_ggtt_invalidate;
3406
3407         if (HAS_EDRAM(dev_priv))
3408                 ggtt->base.pte_encode = iris_pte_encode;
3409         else if (IS_HASWELL(dev_priv))
3410                 ggtt->base.pte_encode = hsw_pte_encode;
3411         else if (IS_VALLEYVIEW(dev_priv))
3412                 ggtt->base.pte_encode = byt_pte_encode;
3413         else if (INTEL_GEN(dev_priv) >= 7)
3414                 ggtt->base.pte_encode = ivb_pte_encode;
3415         else
3416                 ggtt->base.pte_encode = snb_pte_encode;
3417
3418         return ggtt_probe_common(ggtt, size);
3419 }
3420
3421 static void i915_gmch_remove(struct i915_address_space *vm)
3422 {
3423         intel_gmch_remove();
3424 }
3425
3426 static int i915_gmch_probe(struct i915_ggtt *ggtt)
3427 {
3428         struct drm_i915_private *dev_priv = ggtt->base.i915;
3429         phys_addr_t gmadr_base;
3430         int ret;
3431
3432         ret = intel_gmch_probe(dev_priv->bridge_dev, dev_priv->drm.pdev, NULL);
3433         if (!ret) {
3434                 DRM_ERROR("failed to set up gmch\n");
3435                 return -EIO;
3436         }
3437
3438         intel_gtt_get(&ggtt->base.total,
3439                       &gmadr_base,
3440                       &ggtt->mappable_end);
3441
3442         ggtt->gmadr =
3443                 (struct resource) DEFINE_RES_MEM(gmadr_base,
3444                                                  ggtt->mappable_end);
3445
3446         ggtt->do_idle_maps = needs_idle_maps(dev_priv);
3447         ggtt->base.insert_page = i915_ggtt_insert_page;
3448         ggtt->base.insert_entries = i915_ggtt_insert_entries;
3449         ggtt->base.clear_range = i915_ggtt_clear_range;
3450         ggtt->base.bind_vma = ggtt_bind_vma;
3451         ggtt->base.unbind_vma = ggtt_unbind_vma;
3452         ggtt->base.set_pages = ggtt_set_pages;
3453         ggtt->base.clear_pages = clear_pages;
3454         ggtt->base.cleanup = i915_gmch_remove;
3455
3456         ggtt->invalidate = gmch_ggtt_invalidate;
3457
3458         if (unlikely(ggtt->do_idle_maps))
3459                 DRM_INFO("applying Ironlake quirks for intel_iommu\n");
3460
3461         return 0;
3462 }
3463
3464 /**
3465  * i915_ggtt_probe_hw - Probe GGTT hardware location
3466  * @dev_priv: i915 device
3467  */
3468 int i915_ggtt_probe_hw(struct drm_i915_private *dev_priv)
3469 {
3470         struct i915_ggtt *ggtt = &dev_priv->ggtt;
3471         int ret;
3472
3473         ggtt->base.i915 = dev_priv;
3474         ggtt->base.dma = &dev_priv->drm.pdev->dev;
3475
3476         if (INTEL_GEN(dev_priv) <= 5)
3477                 ret = i915_gmch_probe(ggtt);
3478         else if (INTEL_GEN(dev_priv) < 8)
3479                 ret = gen6_gmch_probe(ggtt);
3480         else
3481                 ret = gen8_gmch_probe(ggtt);
3482         if (ret)
3483                 return ret;
3484
3485         /* Trim the GGTT to fit the GuC mappable upper range (when enabled).
3486          * This is easier than doing range restriction on the fly, as we
3487          * currently don't have any bits spare to pass in this upper
3488          * restriction!
3489          */
3490         if (USES_GUC(dev_priv)) {
3491                 ggtt->base.total = min_t(u64, ggtt->base.total, GUC_GGTT_TOP);
3492                 ggtt->mappable_end = min_t(u64, ggtt->mappable_end, ggtt->base.total);
3493         }
3494
3495         if ((ggtt->base.total - 1) >> 32) {
3496                 DRM_ERROR("We never expected a Global GTT with more than 32bits"
3497                           " of address space! Found %lldM!\n",
3498                           ggtt->base.total >> 20);
3499                 ggtt->base.total = 1ULL << 32;
3500                 ggtt->mappable_end = min_t(u64, ggtt->mappable_end, ggtt->base.total);
3501         }
3502
3503         if (ggtt->mappable_end > ggtt->base.total) {
3504                 DRM_ERROR("mappable aperture extends past end of GGTT,"
3505                           " aperture=%pa, total=%llx\n",
3506                           &ggtt->mappable_end, ggtt->base.total);
3507                 ggtt->mappable_end = ggtt->base.total;
3508         }
3509
3510         /* GMADR is the PCI mmio aperture into the global GTT. */
3511         DRM_DEBUG_DRIVER("GGTT size = %lluM\n", ggtt->base.total >> 20);
3512         DRM_DEBUG_DRIVER("GMADR size = %lluM\n", (u64)ggtt->mappable_end >> 20);
3513         DRM_DEBUG_DRIVER("DSM size = %lluM\n",
3514                          (u64)resource_size(&intel_graphics_stolen_res) >> 20);
3515         if (intel_vtd_active())
3516                 DRM_INFO("VT-d active for gfx access\n");
3517
3518         return 0;
3519 }
3520
3521 /**
3522  * i915_ggtt_init_hw - Initialize GGTT hardware
3523  * @dev_priv: i915 device
3524  */
3525 int i915_ggtt_init_hw(struct drm_i915_private *dev_priv)
3526 {
3527         struct i915_ggtt *ggtt = &dev_priv->ggtt;
3528         int ret;
3529
3530         INIT_LIST_HEAD(&dev_priv->vm_list);
3531
3532         /* Note that we use page colouring to enforce a guard page at the
3533          * end of the address space. This is required as the CS may prefetch
3534          * beyond the end of the batch buffer, across the page boundary,
3535          * and beyond the end of the GTT if we do not provide a guard.
3536          */
3537         mutex_lock(&dev_priv->drm.struct_mutex);
3538         i915_address_space_init(&ggtt->base, dev_priv, "[global]");
3539         if (!HAS_LLC(dev_priv) && !USES_PPGTT(dev_priv))
3540                 ggtt->base.mm.color_adjust = i915_gtt_color_adjust;
3541         mutex_unlock(&dev_priv->drm.struct_mutex);
3542
3543         if (!io_mapping_init_wc(&dev_priv->ggtt.iomap,
3544                                 dev_priv->ggtt.gmadr.start,
3545                                 dev_priv->ggtt.mappable_end)) {
3546                 ret = -EIO;
3547                 goto out_gtt_cleanup;
3548         }
3549
3550         ggtt->mtrr = arch_phys_wc_add(ggtt->gmadr.start, ggtt->mappable_end);
3551
3552         /*
3553          * Initialise stolen early so that we may reserve preallocated
3554          * objects for the BIOS to KMS transition.
3555          */
3556         ret = i915_gem_init_stolen(dev_priv);
3557         if (ret)
3558                 goto out_gtt_cleanup;
3559
3560         return 0;
3561
3562 out_gtt_cleanup:
3563         ggtt->base.cleanup(&ggtt->base);
3564         return ret;
3565 }
3566
3567 int i915_ggtt_enable_hw(struct drm_i915_private *dev_priv)
3568 {
3569         if (INTEL_GEN(dev_priv) < 6 && !intel_enable_gtt())
3570                 return -EIO;
3571
3572         return 0;
3573 }
3574
3575 void i915_ggtt_enable_guc(struct drm_i915_private *i915)
3576 {
3577         GEM_BUG_ON(i915->ggtt.invalidate != gen6_ggtt_invalidate);
3578
3579         i915->ggtt.invalidate = guc_ggtt_invalidate;
3580
3581         i915_ggtt_invalidate(i915);
3582 }
3583
3584 void i915_ggtt_disable_guc(struct drm_i915_private *i915)
3585 {
3586         /* We should only be called after i915_ggtt_enable_guc() */
3587         GEM_BUG_ON(i915->ggtt.invalidate != guc_ggtt_invalidate);
3588
3589         i915->ggtt.invalidate = gen6_ggtt_invalidate;
3590
3591         i915_ggtt_invalidate(i915);
3592 }
3593
3594 void i915_gem_restore_gtt_mappings(struct drm_i915_private *dev_priv)
3595 {
3596         struct i915_ggtt *ggtt = &dev_priv->ggtt;
3597         struct drm_i915_gem_object *obj, *on;
3598
3599         i915_check_and_clear_faults(dev_priv);
3600
3601         /* First fill our portion of the GTT with scratch pages */
3602         ggtt->base.clear_range(&ggtt->base, 0, ggtt->base.total);
3603
3604         ggtt->base.closed = true; /* skip rewriting PTE on VMA unbind */
3605
3606         /* clflush objects bound into the GGTT and rebind them. */
3607         list_for_each_entry_safe(obj, on, &dev_priv->mm.bound_list, mm.link) {
3608                 bool ggtt_bound = false;
3609                 struct i915_vma *vma;
3610
3611                 for_each_ggtt_vma(vma, obj) {
3612                         if (!i915_vma_unbind(vma))
3613                                 continue;
3614
3615                         WARN_ON(i915_vma_bind(vma, obj->cache_level,
3616                                               PIN_UPDATE));
3617                         ggtt_bound = true;
3618                 }
3619
3620                 if (ggtt_bound)
3621                         WARN_ON(i915_gem_object_set_to_gtt_domain(obj, false));
3622         }
3623
3624         ggtt->base.closed = false;
3625
3626         if (INTEL_GEN(dev_priv) >= 8) {
3627                 struct intel_ppat *ppat = &dev_priv->ppat;
3628
3629                 bitmap_set(ppat->dirty, 0, ppat->max_entries);
3630                 dev_priv->ppat.update_hw(dev_priv);
3631                 return;
3632         }
3633
3634         if (USES_PPGTT(dev_priv)) {
3635                 struct i915_address_space *vm;
3636
3637                 list_for_each_entry(vm, &dev_priv->vm_list, global_link) {
3638                         struct i915_hw_ppgtt *ppgtt;
3639
3640                         if (i915_is_ggtt(vm))
3641                                 ppgtt = dev_priv->mm.aliasing_ppgtt;
3642                         else
3643                                 ppgtt = i915_vm_to_ppgtt(vm);
3644
3645                         gen6_write_page_range(ppgtt, 0, ppgtt->base.total);
3646                 }
3647         }
3648
3649         i915_ggtt_invalidate(dev_priv);
3650 }
3651
3652 static struct scatterlist *
3653 rotate_pages(const dma_addr_t *in, unsigned int offset,
3654              unsigned int width, unsigned int height,
3655              unsigned int stride,
3656              struct sg_table *st, struct scatterlist *sg)
3657 {
3658         unsigned int column, row;
3659         unsigned int src_idx;
3660
3661         for (column = 0; column < width; column++) {
3662                 src_idx = stride * (height - 1) + column;
3663                 for (row = 0; row < height; row++) {
3664                         st->nents++;
3665                         /* We don't need the pages, but need to initialize
3666                          * the entries so the sg list can be happily traversed.
3667                          * The only thing we need are DMA addresses.
3668                          */
3669                         sg_set_page(sg, NULL, PAGE_SIZE, 0);
3670                         sg_dma_address(sg) = in[offset + src_idx];
3671                         sg_dma_len(sg) = PAGE_SIZE;
3672                         sg = sg_next(sg);
3673                         src_idx -= stride;
3674                 }
3675         }
3676
3677         return sg;
3678 }
3679
3680 static noinline struct sg_table *
3681 intel_rotate_pages(struct intel_rotation_info *rot_info,
3682                    struct drm_i915_gem_object *obj)
3683 {
3684         const unsigned long n_pages = obj->base.size / PAGE_SIZE;
3685         unsigned int size = intel_rotation_info_size(rot_info);
3686         struct sgt_iter sgt_iter;
3687         dma_addr_t dma_addr;
3688         unsigned long i;
3689         dma_addr_t *page_addr_list;
3690         struct sg_table *st;
3691         struct scatterlist *sg;
3692         int ret = -ENOMEM;
3693
3694         /* Allocate a temporary list of source pages for random access. */
3695         page_addr_list = kvmalloc_array(n_pages,
3696                                         sizeof(dma_addr_t),
3697                                         GFP_KERNEL);
3698         if (!page_addr_list)
3699                 return ERR_PTR(ret);
3700
3701         /* Allocate target SG list. */
3702         st = kmalloc(sizeof(*st), GFP_KERNEL);
3703         if (!st)
3704                 goto err_st_alloc;
3705
3706         ret = sg_alloc_table(st, size, GFP_KERNEL);
3707         if (ret)
3708                 goto err_sg_alloc;
3709
3710         /* Populate source page list from the object. */
3711         i = 0;
3712         for_each_sgt_dma(dma_addr, sgt_iter, obj->mm.pages)
3713                 page_addr_list[i++] = dma_addr;
3714
3715         GEM_BUG_ON(i != n_pages);
3716         st->nents = 0;
3717         sg = st->sgl;
3718
3719         for (i = 0 ; i < ARRAY_SIZE(rot_info->plane); i++) {
3720                 sg = rotate_pages(page_addr_list, rot_info->plane[i].offset,
3721                                   rot_info->plane[i].width, rot_info->plane[i].height,
3722                                   rot_info->plane[i].stride, st, sg);
3723         }
3724
3725         kvfree(page_addr_list);
3726
3727         return st;
3728
3729 err_sg_alloc:
3730         kfree(st);
3731 err_st_alloc:
3732         kvfree(page_addr_list);
3733
3734         DRM_DEBUG_DRIVER("Failed to create rotated mapping for object size %zu! (%ux%u tiles, %u pages)\n",
3735                          obj->base.size, rot_info->plane[0].width, rot_info->plane[0].height, size);
3736
3737         return ERR_PTR(ret);
3738 }
3739
3740 static noinline struct sg_table *
3741 intel_partial_pages(const struct i915_ggtt_view *view,
3742                     struct drm_i915_gem_object *obj)
3743 {
3744         struct sg_table *st;
3745         struct scatterlist *sg, *iter;
3746         unsigned int count = view->partial.size;
3747         unsigned int offset;
3748         int ret = -ENOMEM;
3749
3750         st = kmalloc(sizeof(*st), GFP_KERNEL);
3751         if (!st)
3752                 goto err_st_alloc;
3753
3754         ret = sg_alloc_table(st, count, GFP_KERNEL);
3755         if (ret)
3756                 goto err_sg_alloc;
3757
3758         iter = i915_gem_object_get_sg(obj, view->partial.offset, &offset);
3759         GEM_BUG_ON(!iter);
3760
3761         sg = st->sgl;
3762         st->nents = 0;
3763         do {
3764                 unsigned int len;
3765
3766                 len = min(iter->length - (offset << PAGE_SHIFT),
3767                           count << PAGE_SHIFT);
3768                 sg_set_page(sg, NULL, len, 0);
3769                 sg_dma_address(sg) =
3770                         sg_dma_address(iter) + (offset << PAGE_SHIFT);
3771                 sg_dma_len(sg) = len;
3772
3773                 st->nents++;
3774                 count -= len >> PAGE_SHIFT;
3775                 if (count == 0) {
3776                         sg_mark_end(sg);
3777                         return st;
3778                 }
3779
3780                 sg = __sg_next(sg);
3781                 iter = __sg_next(iter);
3782                 offset = 0;
3783         } while (1);
3784
3785 err_sg_alloc:
3786         kfree(st);
3787 err_st_alloc:
3788         return ERR_PTR(ret);
3789 }
3790
3791 static int
3792 i915_get_ggtt_vma_pages(struct i915_vma *vma)
3793 {
3794         int ret;
3795
3796         /* The vma->pages are only valid within the lifespan of the borrowed
3797          * obj->mm.pages. When the obj->mm.pages sg_table is regenerated, so
3798          * must be the vma->pages. A simple rule is that vma->pages must only
3799          * be accessed when the obj->mm.pages are pinned.
3800          */
3801         GEM_BUG_ON(!i915_gem_object_has_pinned_pages(vma->obj));
3802
3803         switch (vma->ggtt_view.type) {
3804         default:
3805                 GEM_BUG_ON(vma->ggtt_view.type);
3806                 /* fall through */
3807         case I915_GGTT_VIEW_NORMAL:
3808                 vma->pages = vma->obj->mm.pages;
3809                 return 0;
3810
3811         case I915_GGTT_VIEW_ROTATED:
3812                 vma->pages =
3813                         intel_rotate_pages(&vma->ggtt_view.rotated, vma->obj);
3814                 break;
3815
3816         case I915_GGTT_VIEW_PARTIAL:
3817                 vma->pages = intel_partial_pages(&vma->ggtt_view, vma->obj);
3818                 break;
3819         }
3820
3821         ret = 0;
3822         if (unlikely(IS_ERR(vma->pages))) {
3823                 ret = PTR_ERR(vma->pages);
3824                 vma->pages = NULL;
3825                 DRM_ERROR("Failed to get pages for VMA view type %u (%d)!\n",
3826                           vma->ggtt_view.type, ret);
3827         }
3828         return ret;
3829 }
3830
3831 /**
3832  * i915_gem_gtt_reserve - reserve a node in an address_space (GTT)
3833  * @vm: the &struct i915_address_space
3834  * @node: the &struct drm_mm_node (typically i915_vma.mode)
3835  * @size: how much space to allocate inside the GTT,
3836  *        must be #I915_GTT_PAGE_SIZE aligned
3837  * @offset: where to insert inside the GTT,
3838  *          must be #I915_GTT_MIN_ALIGNMENT aligned, and the node
3839  *          (@offset + @size) must fit within the address space
3840  * @color: color to apply to node, if this node is not from a VMA,
3841  *         color must be #I915_COLOR_UNEVICTABLE
3842  * @flags: control search and eviction behaviour
3843  *
3844  * i915_gem_gtt_reserve() tries to insert the @node at the exact @offset inside
3845  * the address space (using @size and @color). If the @node does not fit, it
3846  * tries to evict any overlapping nodes from the GTT, including any
3847  * neighbouring nodes if the colors do not match (to ensure guard pages between
3848  * differing domains). See i915_gem_evict_for_node() for the gory details
3849  * on the eviction algorithm. #PIN_NONBLOCK may used to prevent waiting on
3850  * evicting active overlapping objects, and any overlapping node that is pinned
3851  * or marked as unevictable will also result in failure.
3852  *
3853  * Returns: 0 on success, -ENOSPC if no suitable hole is found, -EINTR if
3854  * asked to wait for eviction and interrupted.
3855  */
3856 int i915_gem_gtt_reserve(struct i915_address_space *vm,
3857                          struct drm_mm_node *node,
3858                          u64 size, u64 offset, unsigned long color,
3859                          unsigned int flags)
3860 {
3861         int err;
3862
3863         GEM_BUG_ON(!size);
3864         GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE));
3865         GEM_BUG_ON(!IS_ALIGNED(offset, I915_GTT_MIN_ALIGNMENT));
3866         GEM_BUG_ON(range_overflows(offset, size, vm->total));
3867         GEM_BUG_ON(vm == &vm->i915->mm.aliasing_ppgtt->base);
3868         GEM_BUG_ON(drm_mm_node_allocated(node));
3869
3870         node->size = size;
3871         node->start = offset;
3872         node->color = color;
3873
3874         err = drm_mm_reserve_node(&vm->mm, node);
3875         if (err != -ENOSPC)
3876                 return err;
3877
3878         if (flags & PIN_NOEVICT)
3879                 return -ENOSPC;
3880
3881         err = i915_gem_evict_for_node(vm, node, flags);
3882         if (err == 0)
3883                 err = drm_mm_reserve_node(&vm->mm, node);
3884
3885         return err;
3886 }
3887
3888 static u64 random_offset(u64 start, u64 end, u64 len, u64 align)
3889 {
3890         u64 range, addr;
3891
3892         GEM_BUG_ON(range_overflows(start, len, end));
3893         GEM_BUG_ON(round_up(start, align) > round_down(end - len, align));
3894
3895         range = round_down(end - len, align) - round_up(start, align);
3896         if (range) {
3897                 if (sizeof(unsigned long) == sizeof(u64)) {
3898                         addr = get_random_long();
3899                 } else {
3900                         addr = get_random_int();
3901                         if (range > U32_MAX) {
3902                                 addr <<= 32;
3903                                 addr |= get_random_int();
3904                         }
3905                 }
3906                 div64_u64_rem(addr, range, &addr);
3907                 start += addr;
3908         }
3909
3910         return round_up(start, align);
3911 }
3912
3913 /**
3914  * i915_gem_gtt_insert - insert a node into an address_space (GTT)
3915  * @vm: the &struct i915_address_space
3916  * @node: the &struct drm_mm_node (typically i915_vma.node)
3917  * @size: how much space to allocate inside the GTT,
3918  *        must be #I915_GTT_PAGE_SIZE aligned
3919  * @alignment: required alignment of starting offset, may be 0 but
3920  *             if specified, this must be a power-of-two and at least
3921  *             #I915_GTT_MIN_ALIGNMENT
3922  * @color: color to apply to node
3923  * @start: start of any range restriction inside GTT (0 for all),
3924  *         must be #I915_GTT_PAGE_SIZE aligned
3925  * @end: end of any range restriction inside GTT (U64_MAX for all),
3926  *       must be #I915_GTT_PAGE_SIZE aligned if not U64_MAX
3927  * @flags: control search and eviction behaviour
3928  *
3929  * i915_gem_gtt_insert() first searches for an available hole into which
3930  * is can insert the node. The hole address is aligned to @alignment and
3931  * its @size must then fit entirely within the [@start, @end] bounds. The
3932  * nodes on either side of the hole must match @color, or else a guard page
3933  * will be inserted between the two nodes (or the node evicted). If no
3934  * suitable hole is found, first a victim is randomly selected and tested
3935  * for eviction, otherwise then the LRU list of objects within the GTT
3936  * is scanned to find the first set of replacement nodes to create the hole.
3937  * Those old overlapping nodes are evicted from the GTT (and so must be
3938  * rebound before any future use). Any node that is currently pinned cannot
3939  * be evicted (see i915_vma_pin()). Similar if the node's VMA is currently
3940  * active and #PIN_NONBLOCK is specified, that node is also skipped when
3941  * searching for an eviction candidate. See i915_gem_evict_something() for
3942  * the gory details on the eviction algorithm.
3943  *
3944  * Returns: 0 on success, -ENOSPC if no suitable hole is found, -EINTR if
3945  * asked to wait for eviction and interrupted.
3946  */
3947 int i915_gem_gtt_insert(struct i915_address_space *vm,
3948                         struct drm_mm_node *node,
3949                         u64 size, u64 alignment, unsigned long color,
3950                         u64 start, u64 end, unsigned int flags)
3951 {
3952         enum drm_mm_insert_mode mode;
3953         u64 offset;
3954         int err;
3955
3956         lockdep_assert_held(&vm->i915->drm.struct_mutex);
3957         GEM_BUG_ON(!size);
3958         GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE));
3959         GEM_BUG_ON(alignment && !is_power_of_2(alignment));
3960         GEM_BUG_ON(alignment && !IS_ALIGNED(alignment, I915_GTT_MIN_ALIGNMENT));
3961         GEM_BUG_ON(start >= end);
3962         GEM_BUG_ON(start > 0  && !IS_ALIGNED(start, I915_GTT_PAGE_SIZE));
3963         GEM_BUG_ON(end < U64_MAX && !IS_ALIGNED(end, I915_GTT_PAGE_SIZE));
3964         GEM_BUG_ON(vm == &vm->i915->mm.aliasing_ppgtt->base);
3965         GEM_BUG_ON(drm_mm_node_allocated(node));
3966
3967         if (unlikely(range_overflows(start, size, end)))
3968                 return -ENOSPC;
3969
3970         if (unlikely(round_up(start, alignment) > round_down(end - size, alignment)))
3971                 return -ENOSPC;
3972
3973         mode = DRM_MM_INSERT_BEST;
3974         if (flags & PIN_HIGH)
3975                 mode = DRM_MM_INSERT_HIGH;
3976         if (flags & PIN_MAPPABLE)
3977                 mode = DRM_MM_INSERT_LOW;
3978
3979         /* We only allocate in PAGE_SIZE/GTT_PAGE_SIZE (4096) chunks,
3980          * so we know that we always have a minimum alignment of 4096.
3981          * The drm_mm range manager is optimised to return results
3982          * with zero alignment, so where possible use the optimal
3983          * path.
3984          */
3985         BUILD_BUG_ON(I915_GTT_MIN_ALIGNMENT > I915_GTT_PAGE_SIZE);
3986         if (alignment <= I915_GTT_MIN_ALIGNMENT)
3987                 alignment = 0;
3988
3989         err = drm_mm_insert_node_in_range(&vm->mm, node,
3990                                           size, alignment, color,
3991                                           start, end, mode);
3992         if (err != -ENOSPC)
3993                 return err;
3994
3995         if (flags & PIN_NOEVICT)
3996                 return -ENOSPC;
3997
3998         /* No free space, pick a slot at random.
3999          *
4000          * There is a pathological case here using a GTT shared between
4001          * mmap and GPU (i.e. ggtt/aliasing_ppgtt but not full-ppgtt):
4002          *
4003          *    |<-- 256 MiB aperture -->||<-- 1792 MiB unmappable -->|
4004          *         (64k objects)             (448k objects)
4005          *
4006          * Now imagine that the eviction LRU is ordered top-down (just because
4007          * pathology meets real life), and that we need to evict an object to
4008          * make room inside the aperture. The eviction scan then has to walk
4009          * the 448k list before it finds one within range. And now imagine that
4010          * it has to search for a new hole between every byte inside the memcpy,
4011          * for several simultaneous clients.
4012          *
4013          * On a full-ppgtt system, if we have run out of available space, there
4014          * will be lots and lots of objects in the eviction list! Again,
4015          * searching that LRU list may be slow if we are also applying any
4016          * range restrictions (e.g. restriction to low 4GiB) and so, for
4017          * simplicity and similarilty between different GTT, try the single
4018          * random replacement first.
4019          */
4020         offset = random_offset(start, end,
4021                                size, alignment ?: I915_GTT_MIN_ALIGNMENT);
4022         err = i915_gem_gtt_reserve(vm, node, size, offset, color, flags);
4023         if (err != -ENOSPC)
4024                 return err;
4025
4026         /* Randomly selected placement is pinned, do a search */
4027         err = i915_gem_evict_something(vm, size, alignment, color,
4028                                        start, end, flags);
4029         if (err)
4030                 return err;
4031
4032         return drm_mm_insert_node_in_range(&vm->mm, node,
4033                                            size, alignment, color,
4034                                            start, end, DRM_MM_INSERT_EVICT);
4035 }
4036
4037 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
4038 #include "selftests/mock_gtt.c"
4039 #include "selftests/i915_gem_gtt.c"
4040 #endif