drivers/gpu/drm/i915/gem/i915_gem_pages.c

   1 /*
   2  * SPDX-License-Identifier: MIT
   3  *
   4  * Copyright © 2014-2016 Intel Corporation
   5  */
   6
   7 #include "i915_drv.h"
   8 #include "i915_gem_object.h"
   9 #include "i915_scatterlist.h"
  10 #include "i915_gem_lmem.h"
  11 #include "i915_gem_mman.h"
  12
  13 void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj,
  14                                  struct sg_table *pages,
  15                                  unsigned int sg_page_sizes)
  16 {
  17         struct drm_i915_private *i915 = to_i915(obj->base.dev);
  18         unsigned long supported = INTEL_INFO(i915)->page_sizes;
  19         bool shrinkable;
  20         int i;
  21
  22         assert_object_held_shared(obj);
  23
  24         if (i915_gem_object_is_volatile(obj))
  25                 obj->mm.madv = I915_MADV_DONTNEED;
  26
  27         /* Make the pages coherent with the GPU (flushing any swapin). */
  28         if (obj->cache_dirty) {
  29                 obj->write_domain = 0;
  30                 if (i915_gem_object_has_struct_page(obj))
  31                         drm_clflush_sg(pages);
  32                 obj->cache_dirty = false;
  33         }
  34
  35         obj->mm.get_page.sg_pos = pages->sgl;
  36         obj->mm.get_page.sg_idx = 0;
  37         obj->mm.get_dma_page.sg_pos = pages->sgl;
  38         obj->mm.get_dma_page.sg_idx = 0;
  39
  40         obj->mm.pages = pages;
  41
  42         GEM_BUG_ON(!sg_page_sizes);
  43         obj->mm.page_sizes.phys = sg_page_sizes;
  44
  45         /*
  46          * Calculate the supported page-sizes which fit into the given
  47          * sg_page_sizes. This will give us the page-sizes which we may be able
  48          * to use opportunistically when later inserting into the GTT. For
  49          * example if phys=2G, then in theory we should be able to use 1G, 2M,
  50          * 64K or 4K pages, although in practice this will depend on a number of
  51          * other factors.
  52          */
  53         obj->mm.page_sizes.sg = 0;
  54         for_each_set_bit(i, &supported, ilog2(I915_GTT_MAX_PAGE_SIZE) + 1) {
  55                 if (obj->mm.page_sizes.phys & ~0u << i)
  56                         obj->mm.page_sizes.sg |= BIT(i);
  57         }
  58         GEM_BUG_ON(!HAS_PAGE_SIZES(i915, obj->mm.page_sizes.sg));
  59
  60         shrinkable = i915_gem_object_is_shrinkable(obj);
  61
  62         if (i915_gem_object_is_tiled(obj) &&
  63             i915->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
  64                 GEM_BUG_ON(i915_gem_object_has_tiling_quirk(obj));
  65                 i915_gem_object_set_tiling_quirk(obj);
  66                 shrinkable = false;
  67         }
  68
  69         if (shrinkable) {
  70                 struct list_head *list;
  71                 unsigned long flags;
  72
  73                 assert_object_held(obj);
  74                 spin_lock_irqsave(&i915->mm.obj_lock, flags);
  75
  76                 i915->mm.shrink_count++;
  77                 i915->mm.shrink_memory += obj->base.size;
  78
  79                 if (obj->mm.madv != I915_MADV_WILLNEED)
  80                         list = &i915->mm.purge_list;
  81                 else
  82                         list = &i915->mm.shrink_list;
  83                 list_add_tail(&obj->mm.link, list);
  84
  85                 atomic_set(&obj->mm.shrink_pin, 0);
  86                 spin_unlock_irqrestore(&i915->mm.obj_lock, flags);
  87         }
  88 }
  89
  90 int ____i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
  91 {
  92         struct drm_i915_private *i915 = to_i915(obj->base.dev);
  93         int err;
  94
  95         assert_object_held_shared(obj);
  96
  97         if (unlikely(obj->mm.madv != I915_MADV_WILLNEED)) {
  98                 drm_dbg(&i915->drm,
  99                         "Attempting to obtain a purgeable object\n");
 100                 return -EFAULT;
 101         }
 102
 103         err = obj->ops->get_pages(obj);
 104         GEM_BUG_ON(!err && !i915_gem_object_has_pages(obj));
 105
 106         return err;
 107 }
 108
 109 /* Ensure that the associated pages are gathered from the backing storage
 110  * and pinned into our object. i915_gem_object_pin_pages() may be called
 111  * multiple times before they are released by a single call to
 112  * i915_gem_object_unpin_pages() - once the pages are no longer referenced
 113  * either as a result of memory pressure (reaping pages under the shrinker)
 114  * or as the object is itself released.
 115  */
 116 int __i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
 117 {
 118         int err;
 119
 120         assert_object_held(obj);
 121
 122         assert_object_held_shared(obj);
 123
 124         if (unlikely(!i915_gem_object_has_pages(obj))) {
 125                 GEM_BUG_ON(i915_gem_object_has_pinned_pages(obj));
 126
 127                 err = ____i915_gem_object_get_pages(obj);
 128                 if (err)
 129                         return err;
 130
 131                 smp_mb__before_atomic();
 132         }
 133         atomic_inc(&obj->mm.pages_pin_count);
 134
 135         return 0;
 136 }
 137
 138 int i915_gem_object_pin_pages_unlocked(struct drm_i915_gem_object *obj)
 139 {
 140         struct i915_gem_ww_ctx ww;
 141         int err;
 142
 143         i915_gem_ww_ctx_init(&ww, true);
 144 retry:
 145         err = i915_gem_object_lock(obj, &ww);
 146         if (!err)
 147                 err = i915_gem_object_pin_pages(obj);
 148
 149         if (err == -EDEADLK) {
 150                 err = i915_gem_ww_ctx_backoff(&ww);
 151                 if (!err)
 152                         goto retry;
 153         }
 154         i915_gem_ww_ctx_fini(&ww);
 155         return err;
 156 }
 157
 158 /* Immediately discard the backing storage */
 159 void i915_gem_object_truncate(struct drm_i915_gem_object *obj)
 160 {
 161         drm_gem_free_mmap_offset(&obj->base);
 162         if (obj->ops->truncate)
 163                 obj->ops->truncate(obj);
 164 }
 165
 166 /* Try to discard unwanted pages */
 167 void i915_gem_object_writeback(struct drm_i915_gem_object *obj)
 168 {
 169         assert_object_held_shared(obj);
 170         GEM_BUG_ON(i915_gem_object_has_pages(obj));
 171
 172         if (obj->ops->writeback)
 173                 obj->ops->writeback(obj);
 174 }
 175
 176 static void __i915_gem_object_reset_page_iter(struct drm_i915_gem_object *obj)
 177 {
 178         struct radix_tree_iter iter;
 179         void __rcu **slot;
 180
 181         rcu_read_lock();
 182         radix_tree_for_each_slot(slot, &obj->mm.get_page.radix, &iter, 0)
 183                 radix_tree_delete(&obj->mm.get_page.radix, iter.index);
 184         radix_tree_for_each_slot(slot, &obj->mm.get_dma_page.radix, &iter, 0)
 185                 radix_tree_delete(&obj->mm.get_dma_page.radix, iter.index);
 186         rcu_read_unlock();
 187 }
 188
 189 static void unmap_object(struct drm_i915_gem_object *obj, void *ptr)
 190 {
 191         if (is_vmalloc_addr(ptr))
 192                 vunmap(ptr);
 193 }
 194
 195 struct sg_table *
 196 __i915_gem_object_unset_pages(struct drm_i915_gem_object *obj)
 197 {
 198         struct sg_table *pages;
 199
 200         assert_object_held_shared(obj);
 201
 202         pages = fetch_and_zero(&obj->mm.pages);
 203         if (IS_ERR_OR_NULL(pages))
 204                 return pages;
 205
 206         if (i915_gem_object_is_volatile(obj))
 207                 obj->mm.madv = I915_MADV_WILLNEED;
 208
 209         i915_gem_object_make_unshrinkable(obj);
 210
 211         if (obj->mm.mapping) {
 212                 unmap_object(obj, page_mask_bits(obj->mm.mapping));
 213                 obj->mm.mapping = NULL;
 214         }
 215
 216         __i915_gem_object_reset_page_iter(obj);
 217         obj->mm.page_sizes.phys = obj->mm.page_sizes.sg = 0;
 218
 219         return pages;
 220 }
 221
 222 int __i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
 223 {
 224         struct sg_table *pages;
 225
 226         if (i915_gem_object_has_pinned_pages(obj))
 227                 return -EBUSY;
 228
 229         /* May be called by shrinker from within get_pages() (on another bo) */
 230         assert_object_held_shared(obj);
 231
 232         i915_gem_object_release_mmap_offset(obj);
 233
 234         /*
 235          * ->put_pages might need to allocate memory for the bit17 swizzle
 236          * array, hence protect them from being reaped by removing them from gtt
 237          * lists early.
 238          */
 239         pages = __i915_gem_object_unset_pages(obj);
 240
 241         /*
 242          * XXX Temporary hijinx to avoid updating all backends to handle
 243          * NULL pages. In the future, when we have more asynchronous
 244          * get_pages backends we should be better able to handle the
 245          * cancellation of the async task in a more uniform manner.
 246          */
 247         if (!IS_ERR_OR_NULL(pages))
 248                 obj->ops->put_pages(obj, pages);
 249
 250         return 0;
 251 }
 252
 253 /* The 'mapping' part of i915_gem_object_pin_map() below */
 254 static void *i915_gem_object_map_page(struct drm_i915_gem_object *obj,
 255                                       enum i915_map_type type)
 256 {
 257         unsigned long n_pages = obj->base.size >> PAGE_SHIFT, i;
 258         struct page *stack[32], **pages = stack, *page;
 259         struct sgt_iter iter;
 260         pgprot_t pgprot;
 261         void *vaddr;
 262
 263         switch (type) {
 264         default:
 265                 MISSING_CASE(type);
 266                 fallthrough;    /* to use PAGE_KERNEL anyway */
 267         case I915_MAP_WB:
 268                 /*
 269                  * On 32b, highmem using a finite set of indirect PTE (i.e.
 270                  * vmap) to provide virtual mappings of the high pages.
 271                  * As these are finite, map_new_virtual() must wait for some
 272                  * other kmap() to finish when it runs out. If we map a large
 273                  * number of objects, there is no method for it to tell us
 274                  * to release the mappings, and we deadlock.
 275                  *
 276                  * However, if we make an explicit vmap of the page, that
 277                  * uses a larger vmalloc arena, and also has the ability
 278                  * to tell us to release unwanted mappings. Most importantly,
 279                  * it will fail and propagate an error instead of waiting
 280                  * forever.
 281                  *
 282                  * So if the page is beyond the 32b boundary, make an explicit
 283                  * vmap.
 284                  */
 285                 if (n_pages == 1 && !PageHighMem(sg_page(obj->mm.pages->sgl)))
 286                         return page_address(sg_page(obj->mm.pages->sgl));
 287                 pgprot = PAGE_KERNEL;
 288                 break;
 289         case I915_MAP_WC:
 290                 pgprot = pgprot_writecombine(PAGE_KERNEL_IO);
 291                 break;
 292         }
 293
 294         if (n_pages > ARRAY_SIZE(stack)) {
 295                 /* Too big for stack -- allocate temporary array instead */
 296                 pages = kvmalloc_array(n_pages, sizeof(*pages), GFP_KERNEL);
 297                 if (!pages)
 298                         return ERR_PTR(-ENOMEM);
 299         }
 300
 301         i = 0;
 302         for_each_sgt_page(page, iter, obj->mm.pages)
 303                 pages[i++] = page;
 304         vaddr = vmap(pages, n_pages, 0, pgprot);
 305         if (pages != stack)
 306                 kvfree(pages);
 307
 308         return vaddr ?: ERR_PTR(-ENOMEM);
 309 }
 310
 311 static void *i915_gem_object_map_pfn(struct drm_i915_gem_object *obj,
 312                                      enum i915_map_type type)
 313 {
 314         resource_size_t iomap = obj->mm.region->iomap.base -
 315                 obj->mm.region->region.start;
 316         unsigned long n_pfn = obj->base.size >> PAGE_SHIFT;
 317         unsigned long stack[32], *pfns = stack, i;
 318         struct sgt_iter iter;
 319         dma_addr_t addr;
 320         void *vaddr;
 321
 322         if (type != I915_MAP_WC)
 323                 return ERR_PTR(-ENODEV);
 324
 325         if (n_pfn > ARRAY_SIZE(stack)) {
 326                 /* Too big for stack -- allocate temporary array instead */
 327                 pfns = kvmalloc_array(n_pfn, sizeof(*pfns), GFP_KERNEL);
 328                 if (!pfns)
 329                         return ERR_PTR(-ENOMEM);
 330         }
 331
 332         i = 0;
 333         for_each_sgt_daddr(addr, iter, obj->mm.pages)
 334                 pfns[i++] = (iomap + addr) >> PAGE_SHIFT;
 335         vaddr = vmap_pfn(pfns, n_pfn, pgprot_writecombine(PAGE_KERNEL_IO));
 336         if (pfns != stack)
 337                 kvfree(pfns);
 338
 339         return vaddr ?: ERR_PTR(-ENOMEM);
 340 }
 341
 342 /* get, pin, and map the pages of the object into kernel space */
 343 void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj,
 344                               enum i915_map_type type)
 345 {
 346         enum i915_map_type has_type;
 347         bool pinned;
 348         void *ptr;
 349         int err;
 350
 351         if (!i915_gem_object_has_struct_page(obj) &&
 352             !i915_gem_object_type_has(obj, I915_GEM_OBJECT_HAS_IOMEM))
 353                 return ERR_PTR(-ENXIO);
 354
 355         assert_object_held(obj);
 356
 357         pinned = !(type & I915_MAP_OVERRIDE);
 358         type &= ~I915_MAP_OVERRIDE;
 359
 360         if (!atomic_inc_not_zero(&obj->mm.pages_pin_count)) {
 361                 if (unlikely(!i915_gem_object_has_pages(obj))) {
 362                         GEM_BUG_ON(i915_gem_object_has_pinned_pages(obj));
 363
 364                         err = ____i915_gem_object_get_pages(obj);
 365                         if (err)
 366                                 return ERR_PTR(err);
 367
 368                         smp_mb__before_atomic();
 369                 }
 370                 atomic_inc(&obj->mm.pages_pin_count);
 371                 pinned = false;
 372         }
 373         GEM_BUG_ON(!i915_gem_object_has_pages(obj));
 374
 375         ptr = page_unpack_bits(obj->mm.mapping, &has_type);
 376         if (ptr && has_type != type) {
 377                 if (pinned) {
 378                         ptr = ERR_PTR(-EBUSY);
 379                         goto err_unpin;
 380                 }
 381
 382                 unmap_object(obj, ptr);
 383
 384                 ptr = obj->mm.mapping = NULL;
 385         }
 386
 387         if (!ptr) {
 388                 if (GEM_WARN_ON(type == I915_MAP_WC &&
 389                                 !static_cpu_has(X86_FEATURE_PAT)))
 390                         ptr = ERR_PTR(-ENODEV);
 391                 else if (i915_gem_object_has_struct_page(obj))
 392                         ptr = i915_gem_object_map_page(obj, type);
 393                 else
 394                         ptr = i915_gem_object_map_pfn(obj, type);
 395                 if (IS_ERR(ptr))
 396                         goto err_unpin;
 397
 398                 obj->mm.mapping = page_pack_bits(ptr, type);
 399         }
 400
 401         return ptr;
 402
 403 err_unpin:
 404         atomic_dec(&obj->mm.pages_pin_count);
 405         return ptr;
 406 }
 407
 408 void *i915_gem_object_pin_map_unlocked(struct drm_i915_gem_object *obj,
 409                                        enum i915_map_type type)
 410 {
 411         void *ret;
 412
 413         i915_gem_object_lock(obj, NULL);
 414         ret = i915_gem_object_pin_map(obj, type);
 415         i915_gem_object_unlock(obj);
 416
 417         return ret;
 418 }
 419
 420 void __i915_gem_object_flush_map(struct drm_i915_gem_object *obj,
 421                                  unsigned long offset,
 422                                  unsigned long size)
 423 {
 424         enum i915_map_type has_type;
 425         void *ptr;
 426
 427         GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
 428         GEM_BUG_ON(range_overflows_t(typeof(obj->base.size),
 429                                      offset, size, obj->base.size));
 430
 431         wmb(); /* let all previous writes be visible to coherent partners */
 432         obj->mm.dirty = true;
 433
 434         if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE)
 435                 return;
 436
 437         ptr = page_unpack_bits(obj->mm.mapping, &has_type);
 438         if (has_type == I915_MAP_WC)
 439                 return;
 440
 441         drm_clflush_virt_range(ptr + offset, size);
 442         if (size == obj->base.size) {
 443                 obj->write_domain &= ~I915_GEM_DOMAIN_CPU;
 444                 obj->cache_dirty = false;
 445         }
 446 }
 447
 448 void __i915_gem_object_release_map(struct drm_i915_gem_object *obj)
 449 {
 450         GEM_BUG_ON(!obj->mm.mapping);
 451
 452         /*
 453          * We allow removing the mapping from underneath pinned pages!
 454          *
 455          * Furthermore, since this is an unsafe operation reserved only
 456          * for construction time manipulation, we ignore locking prudence.
 457          */
 458         unmap_object(obj, page_mask_bits(fetch_and_zero(&obj->mm.mapping)));
 459
 460         i915_gem_object_unpin_map(obj);
 461 }
 462
 463 struct scatterlist *
 464 __i915_gem_object_get_sg(struct drm_i915_gem_object *obj,
 465                          struct i915_gem_object_page_iter *iter,
 466                          unsigned int n,
 467                          unsigned int *offset,
 468                          bool allow_alloc)
 469 {
 470         const bool dma = iter == &obj->mm.get_dma_page;
 471         struct scatterlist *sg;
 472         unsigned int idx, count;
 473
 474         might_sleep();
 475         GEM_BUG_ON(n >= obj->base.size >> PAGE_SHIFT);
 476         GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
 477
 478         /* As we iterate forward through the sg, we record each entry in a
 479          * radixtree for quick repeated (backwards) lookups. If we have seen
 480          * this index previously, we will have an entry for it.
 481          *
 482          * Initial lookup is O(N), but this is amortized to O(1) for
 483          * sequential page access (where each new request is consecutive
 484          * to the previous one). Repeated lookups are O(lg(obj->base.size)),
 485          * i.e. O(1) with a large constant!
 486          */
 487         if (n < READ_ONCE(iter->sg_idx))
 488                 goto lookup;
 489
 490         if (!allow_alloc)
 491                 goto manual_lookup;
 492
 493         mutex_lock(&iter->lock);
 494
 495         /* We prefer to reuse the last sg so that repeated lookup of this
 496          * (or the subsequent) sg are fast - comparing against the last
 497          * sg is faster than going through the radixtree.
 498          */
 499
 500         sg = iter->sg_pos;
 501         idx = iter->sg_idx;
 502         count = dma ? __sg_dma_page_count(sg) : __sg_page_count(sg);
 503
 504         while (idx + count <= n) {
 505                 void *entry;
 506                 unsigned long i;
 507                 int ret;
 508
 509                 /* If we cannot allocate and insert this entry, or the
 510                  * individual pages from this range, cancel updating the
 511                  * sg_idx so that on this lookup we are forced to linearly
 512                  * scan onwards, but on future lookups we will try the
 513                  * insertion again (in which case we need to be careful of
 514                  * the error return reporting that we have already inserted
 515                  * this index).
 516                  */
 517                 ret = radix_tree_insert(&iter->radix, idx, sg);
 518                 if (ret && ret != -EEXIST)
 519                         goto scan;
 520
 521                 entry = xa_mk_value(idx);
 522                 for (i = 1; i < count; i++) {
 523                         ret = radix_tree_insert(&iter->radix, idx + i, entry);
 524                         if (ret && ret != -EEXIST)
 525                                 goto scan;
 526                 }
 527
 528                 idx += count;
 529                 sg = ____sg_next(sg);
 530                 count = dma ? __sg_dma_page_count(sg) : __sg_page_count(sg);
 531         }
 532
 533 scan:
 534         iter->sg_pos = sg;
 535         iter->sg_idx = idx;
 536
 537         mutex_unlock(&iter->lock);
 538
 539         if (unlikely(n < idx)) /* insertion completed by another thread */
 540                 goto lookup;
 541
 542         goto manual_walk;
 543
 544 manual_lookup:
 545         idx = 0;
 546         sg = obj->mm.pages->sgl;
 547         count = __sg_page_count(sg);
 548
 549 manual_walk:
 550         /*
 551          * In case we failed to insert the entry into the radixtree, we need
 552          * to look beyond the current sg.
 553          */
 554         while (idx + count <= n) {
 555                 idx += count;
 556                 sg = ____sg_next(sg);
 557                 count = dma ? __sg_dma_page_count(sg) : __sg_page_count(sg);
 558         }
 559
 560         *offset = n - idx;
 561         return sg;
 562
 563 lookup:
 564         rcu_read_lock();
 565
 566         sg = radix_tree_lookup(&iter->radix, n);
 567         GEM_BUG_ON(!sg);
 568
 569         /* If this index is in the middle of multi-page sg entry,
 570          * the radix tree will contain a value entry that points
 571          * to the start of that range. We will return the pointer to
 572          * the base page and the offset of this page within the
 573          * sg entry's range.
 574          */
 575         *offset = 0;
 576         if (unlikely(xa_is_value(sg))) {
 577                 unsigned long base = xa_to_value(sg);
 578
 579                 sg = radix_tree_lookup(&iter->radix, base);
 580                 GEM_BUG_ON(!sg);
 581
 582                 *offset = n - base;
 583         }
 584
 585         rcu_read_unlock();
 586
 587         return sg;
 588 }
 589
 590 struct page *
 591 i915_gem_object_get_page(struct drm_i915_gem_object *obj, unsigned int n)
 592 {
 593         struct scatterlist *sg;
 594         unsigned int offset;
 595
 596         GEM_BUG_ON(!i915_gem_object_has_struct_page(obj));
 597
 598         sg = i915_gem_object_get_sg(obj, n, &offset, true);
 599         return nth_page(sg_page(sg), offset);
 600 }
 601
 602 /* Like i915_gem_object_get_page(), but mark the returned page dirty */
 603 struct page *
 604 i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj,
 605                                unsigned int n)
 606 {
 607         struct page *page;
 608
 609         page = i915_gem_object_get_page(obj, n);
 610         if (!obj->mm.dirty)
 611                 set_page_dirty(page);
 612
 613         return page;
 614 }
 615
 616 dma_addr_t
 617 i915_gem_object_get_dma_address_len(struct drm_i915_gem_object *obj,
 618                                     unsigned long n,
 619                                     unsigned int *len)
 620 {
 621         struct scatterlist *sg;
 622         unsigned int offset;
 623
 624         sg = i915_gem_object_get_sg_dma(obj, n, &offset, true);
 625
 626         if (len)
 627                 *len = sg_dma_len(sg) - (offset << PAGE_SHIFT);
 628
 629         return sg_dma_address(sg) + (offset << PAGE_SHIFT);
 630 }
 631
 632 dma_addr_t
 633 i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj,
 634                                 unsigned long n)
 635 {
 636         return i915_gem_object_get_dma_address_len(obj, n, NULL);
 637 }