drivers/gpu/drm/i915/i915_vma.c

   1 /*
   2  * Copyright © 2016 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  */
  24
  25 #include <linux/sched/mm.h>
  26 #include <drm/drm_gem.h>
  27
  28 #include "display/intel_frontbuffer.h"
  29
  30 #include "gem/i915_gem_lmem.h"
  31 #include "gt/intel_engine.h"
  32 #include "gt/intel_engine_heartbeat.h"
  33 #include "gt/intel_gt.h"
  34 #include "gt/intel_gt_requests.h"
  35
  36 #include "i915_drv.h"
  37 #include "i915_sw_fence_work.h"
  38 #include "i915_trace.h"
  39 #include "i915_vma.h"
  40 #include "i915_vma_resource.h"
  41
  42 static inline void assert_vma_held_evict(const struct i915_vma *vma)
  43 {
  44         /*
  45          * We may be forced to unbind when the vm is dead, to clean it up.
  46          * This is the only exception to the requirement of the object lock
  47          * being held.
  48          */
  49         if (atomic_read(&vma->vm->open))
  50                 assert_object_held_shared(vma->obj);
  51 }
  52
  53 static struct kmem_cache *slab_vmas;
  54
  55 static struct i915_vma *i915_vma_alloc(void)
  56 {
  57         return kmem_cache_zalloc(slab_vmas, GFP_KERNEL);
  58 }
  59
  60 static void i915_vma_free(struct i915_vma *vma)
  61 {
  62         return kmem_cache_free(slab_vmas, vma);
  63 }
  64
  65 #if IS_ENABLED(CONFIG_DRM_I915_ERRLOG_GEM) && IS_ENABLED(CONFIG_DRM_DEBUG_MM)
  66
  67 #include <linux/stackdepot.h>
  68
  69 static void vma_print_allocator(struct i915_vma *vma, const char *reason)
  70 {
  71         char buf[512];
  72
  73         if (!vma->node.stack) {
  74                 DRM_DEBUG_DRIVER("vma.node [%08llx + %08llx] %s: unknown owner\n",
  75                                  vma->node.start, vma->node.size, reason);
  76                 return;
  77         }
  78
  79         stack_depot_snprint(vma->node.stack, buf, sizeof(buf), 0);
  80         DRM_DEBUG_DRIVER("vma.node [%08llx + %08llx] %s: inserted at %s\n",
  81                          vma->node.start, vma->node.size, reason, buf);
  82 }
  83
  84 #else
  85
  86 static void vma_print_allocator(struct i915_vma *vma, const char *reason)
  87 {
  88 }
  89
  90 #endif
  91
  92 static inline struct i915_vma *active_to_vma(struct i915_active *ref)
  93 {
  94         return container_of(ref, typeof(struct i915_vma), active);
  95 }
  96
  97 static int __i915_vma_active(struct i915_active *ref)
  98 {
  99         return i915_vma_tryget(active_to_vma(ref)) ? 0 : -ENOENT;
 100 }
 101
 102 static void __i915_vma_retire(struct i915_active *ref)
 103 {
 104         i915_vma_put(active_to_vma(ref));
 105 }
 106
 107 static struct i915_vma *
 108 vma_create(struct drm_i915_gem_object *obj,
 109            struct i915_address_space *vm,
 110            const struct i915_ggtt_view *view)
 111 {
 112         struct i915_vma *pos = ERR_PTR(-E2BIG);
 113         struct i915_vma *vma;
 114         struct rb_node *rb, **p;
 115
 116         /* The aliasing_ppgtt should never be used directly! */
 117         GEM_BUG_ON(vm == &vm->gt->ggtt->alias->vm);
 118
 119         vma = i915_vma_alloc();
 120         if (vma == NULL)
 121                 return ERR_PTR(-ENOMEM);
 122
 123         kref_init(&vma->ref);
 124         vma->vm = i915_vm_get(vm);
 125         vma->ops = &vm->vma_ops;
 126         vma->obj = obj;
 127         vma->size = obj->base.size;
 128         vma->display_alignment = I915_GTT_MIN_ALIGNMENT;
 129
 130         i915_active_init(&vma->active, __i915_vma_active, __i915_vma_retire, 0);
 131
 132         /* Declare ourselves safe for use inside shrinkers */
 133         if (IS_ENABLED(CONFIG_LOCKDEP)) {
 134                 fs_reclaim_acquire(GFP_KERNEL);
 135                 might_lock(&vma->active.mutex);
 136                 fs_reclaim_release(GFP_KERNEL);
 137         }
 138
 139         INIT_LIST_HEAD(&vma->closed_link);
 140
 141         if (view && view->type != I915_GGTT_VIEW_NORMAL) {
 142                 vma->ggtt_view = *view;
 143                 if (view->type == I915_GGTT_VIEW_PARTIAL) {
 144                         GEM_BUG_ON(range_overflows_t(u64,
 145                                                      view->partial.offset,
 146                                                      view->partial.size,
 147                                                      obj->base.size >> PAGE_SHIFT));
 148                         vma->size = view->partial.size;
 149                         vma->size <<= PAGE_SHIFT;
 150                         GEM_BUG_ON(vma->size > obj->base.size);
 151                 } else if (view->type == I915_GGTT_VIEW_ROTATED) {
 152                         vma->size = intel_rotation_info_size(&view->rotated);
 153                         vma->size <<= PAGE_SHIFT;
 154                 } else if (view->type == I915_GGTT_VIEW_REMAPPED) {
 155                         vma->size = intel_remapped_info_size(&view->remapped);
 156                         vma->size <<= PAGE_SHIFT;
 157                 }
 158         }
 159
 160         if (unlikely(vma->size > vm->total))
 161                 goto err_vma;
 162
 163         GEM_BUG_ON(!IS_ALIGNED(vma->size, I915_GTT_PAGE_SIZE));
 164
 165         spin_lock(&obj->vma.lock);
 166
 167         if (i915_is_ggtt(vm)) {
 168                 if (unlikely(overflows_type(vma->size, u32)))
 169                         goto err_unlock;
 170
 171                 vma->fence_size = i915_gem_fence_size(vm->i915, vma->size,
 172                                                       i915_gem_object_get_tiling(obj),
 173                                                       i915_gem_object_get_stride(obj));
 174                 if (unlikely(vma->fence_size < vma->size || /* overflow */
 175                              vma->fence_size > vm->total))
 176                         goto err_unlock;
 177
 178                 GEM_BUG_ON(!IS_ALIGNED(vma->fence_size, I915_GTT_MIN_ALIGNMENT));
 179
 180                 vma->fence_alignment = i915_gem_fence_alignment(vm->i915, vma->size,
 181                                                                 i915_gem_object_get_tiling(obj),
 182                                                                 i915_gem_object_get_stride(obj));
 183                 GEM_BUG_ON(!is_power_of_2(vma->fence_alignment));
 184
 185                 __set_bit(I915_VMA_GGTT_BIT, __i915_vma_flags(vma));
 186         }
 187
 188         rb = NULL;
 189         p = &obj->vma.tree.rb_node;
 190         while (*p) {
 191                 long cmp;
 192
 193                 rb = *p;
 194                 pos = rb_entry(rb, struct i915_vma, obj_node);
 195
 196                 /*
 197                  * If the view already exists in the tree, another thread
 198                  * already created a matching vma, so return the older instance
 199                  * and dispose of ours.
 200                  */
 201                 cmp = i915_vma_compare(pos, vm, view);
 202                 if (cmp < 0)
 203                         p = &rb->rb_right;
 204                 else if (cmp > 0)
 205                         p = &rb->rb_left;
 206                 else
 207                         goto err_unlock;
 208         }
 209         rb_link_node(&vma->obj_node, rb, p);
 210         rb_insert_color(&vma->obj_node, &obj->vma.tree);
 211
 212         if (i915_vma_is_ggtt(vma))
 213                 /*
 214                  * We put the GGTT vma at the start of the vma-list, followed
 215                  * by the ppGGTT vma. This allows us to break early when
 216                  * iterating over only the GGTT vma for an object, see
 217                  * for_each_ggtt_vma()
 218                  */
 219                 list_add(&vma->obj_link, &obj->vma.list);
 220         else
 221                 list_add_tail(&vma->obj_link, &obj->vma.list);
 222
 223         spin_unlock(&obj->vma.lock);
 224
 225         return vma;
 226
 227 err_unlock:
 228         spin_unlock(&obj->vma.lock);
 229 err_vma:
 230         i915_vm_put(vm);
 231         i915_vma_free(vma);
 232         return pos;
 233 }
 234
 235 static struct i915_vma *
 236 i915_vma_lookup(struct drm_i915_gem_object *obj,
 237            struct i915_address_space *vm,
 238            const struct i915_ggtt_view *view)
 239 {
 240         struct rb_node *rb;
 241
 242         rb = obj->vma.tree.rb_node;
 243         while (rb) {
 244                 struct i915_vma *vma = rb_entry(rb, struct i915_vma, obj_node);
 245                 long cmp;
 246
 247                 cmp = i915_vma_compare(vma, vm, view);
 248                 if (cmp == 0)
 249                         return vma;
 250
 251                 if (cmp < 0)
 252                         rb = rb->rb_right;
 253                 else
 254                         rb = rb->rb_left;
 255         }
 256
 257         return NULL;
 258 }
 259
 260 /**
 261  * i915_vma_instance - return the singleton instance of the VMA
 262  * @obj: parent &struct drm_i915_gem_object to be mapped
 263  * @vm: address space in which the mapping is located
 264  * @view: additional mapping requirements
 265  *
 266  * i915_vma_instance() looks up an existing VMA of the @obj in the @vm with
 267  * the same @view characteristics. If a match is not found, one is created.
 268  * Once created, the VMA is kept until either the object is freed, or the
 269  * address space is closed.
 270  *
 271  * Returns the vma, or an error pointer.
 272  */
 273 struct i915_vma *
 274 i915_vma_instance(struct drm_i915_gem_object *obj,
 275                   struct i915_address_space *vm,
 276                   const struct i915_ggtt_view *view)
 277 {
 278         struct i915_vma *vma;
 279
 280         GEM_BUG_ON(view && !i915_is_ggtt_or_dpt(vm));
 281         GEM_BUG_ON(!atomic_read(&vm->open));
 282
 283         spin_lock(&obj->vma.lock);
 284         vma = i915_vma_lookup(obj, vm, view);
 285         spin_unlock(&obj->vma.lock);
 286
 287         /* vma_create() will resolve the race if another creates the vma */
 288         if (unlikely(!vma))
 289                 vma = vma_create(obj, vm, view);
 290
 291         GEM_BUG_ON(!IS_ERR(vma) && i915_vma_compare(vma, vm, view));
 292         return vma;
 293 }
 294
 295 struct i915_vma_work {
 296         struct dma_fence_work base;
 297         struct i915_address_space *vm;
 298         struct i915_vm_pt_stash stash;
 299         struct i915_vma_resource *vma_res;
 300         struct drm_i915_gem_object *pinned;
 301         struct i915_sw_dma_fence_cb cb;
 302         enum i915_cache_level cache_level;
 303         unsigned int flags;
 304 };
 305
 306 static void __vma_bind(struct dma_fence_work *work)
 307 {
 308         struct i915_vma_work *vw = container_of(work, typeof(*vw), base);
 309         struct i915_vma_resource *vma_res = vw->vma_res;
 310
 311         vma_res->ops->bind_vma(vma_res->vm, &vw->stash,
 312                                vma_res, vw->cache_level, vw->flags);
 313
 314 }
 315
 316 static void __vma_release(struct dma_fence_work *work)
 317 {
 318         struct i915_vma_work *vw = container_of(work, typeof(*vw), base);
 319
 320         if (vw->pinned)
 321                 i915_gem_object_put(vw->pinned);
 322
 323         i915_vm_free_pt_stash(vw->vm, &vw->stash);
 324         i915_vm_put(vw->vm);
 325         if (vw->vma_res)
 326                 i915_vma_resource_put(vw->vma_res);
 327 }
 328
 329 static const struct dma_fence_work_ops bind_ops = {
 330         .name = "bind",
 331         .work = __vma_bind,
 332         .release = __vma_release,
 333 };
 334
 335 struct i915_vma_work *i915_vma_work(void)
 336 {
 337         struct i915_vma_work *vw;
 338
 339         vw = kzalloc(sizeof(*vw), GFP_KERNEL);
 340         if (!vw)
 341                 return NULL;
 342
 343         dma_fence_work_init(&vw->base, &bind_ops);
 344         vw->base.dma.error = -EAGAIN; /* disable the worker by default */
 345
 346         return vw;
 347 }
 348
 349 int i915_vma_wait_for_bind(struct i915_vma *vma)
 350 {
 351         int err = 0;
 352
 353         if (rcu_access_pointer(vma->active.excl.fence)) {
 354                 struct dma_fence *fence;
 355
 356                 rcu_read_lock();
 357                 fence = dma_fence_get_rcu_safe(&vma->active.excl.fence);
 358                 rcu_read_unlock();
 359                 if (fence) {
 360                         err = dma_fence_wait(fence, true);
 361                         dma_fence_put(fence);
 362                 }
 363         }
 364
 365         return err;
 366 }
 367
 368 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)
 369 static int i915_vma_verify_bind_complete(struct i915_vma *vma)
 370 {
 371         struct dma_fence *fence = i915_active_fence_get(&vma->active.excl);
 372         int err;
 373
 374         if (!fence)
 375                 return 0;
 376
 377         if (dma_fence_is_signaled(fence))
 378                 err = fence->error;
 379         else
 380                 err = -EBUSY;
 381
 382         dma_fence_put(fence);
 383
 384         return err;
 385 }
 386 #else
 387 #define i915_vma_verify_bind_complete(_vma) 0
 388 #endif
 389
 390 I915_SELFTEST_EXPORT void
 391 i915_vma_resource_init_from_vma(struct i915_vma_resource *vma_res,
 392                                 struct i915_vma *vma)
 393 {
 394         struct drm_i915_gem_object *obj = vma->obj;
 395
 396         i915_vma_resource_init(vma_res, vma->vm, vma->pages, &vma->page_sizes,
 397                                obj->mm.rsgt, i915_gem_object_is_readonly(obj),
 398                                i915_gem_object_is_lmem(obj), obj->mm.region,
 399                                vma->ops, vma->private, vma->node.start,
 400                                vma->node.size, vma->size);
 401 }
 402
 403 /**
 404  * i915_vma_bind - Sets up PTEs for an VMA in it's corresponding address space.
 405  * @vma: VMA to map
 406  * @cache_level: mapping cache level
 407  * @flags: flags like global or local mapping
 408  * @work: preallocated worker for allocating and binding the PTE
 409  * @vma_res: pointer to a preallocated vma resource. The resource is either
 410  * consumed or freed.
 411  *
 412  * DMA addresses are taken from the scatter-gather table of this object (or of
 413  * this VMA in case of non-default GGTT views) and PTE entries set up.
 414  * Note that DMA addresses are also the only part of the SG table we care about.
 415  */
 416 int i915_vma_bind(struct i915_vma *vma,
 417                   enum i915_cache_level cache_level,
 418                   u32 flags,
 419                   struct i915_vma_work *work,
 420                   struct i915_vma_resource *vma_res)
 421 {
 422         u32 bind_flags;
 423         u32 vma_flags;
 424         int ret;
 425
 426         lockdep_assert_held(&vma->vm->mutex);
 427         GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
 428         GEM_BUG_ON(vma->size > vma->node.size);
 429
 430         if (GEM_DEBUG_WARN_ON(range_overflows(vma->node.start,
 431                                               vma->node.size,
 432                                               vma->vm->total))) {
 433                 i915_vma_resource_free(vma_res);
 434                 return -ENODEV;
 435         }
 436
 437         if (GEM_DEBUG_WARN_ON(!flags)) {
 438                 i915_vma_resource_free(vma_res);
 439                 return -EINVAL;
 440         }
 441
 442         bind_flags = flags;
 443         bind_flags &= I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND;
 444
 445         vma_flags = atomic_read(&vma->flags);
 446         vma_flags &= I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND;
 447
 448         bind_flags &= ~vma_flags;
 449         if (bind_flags == 0) {
 450                 i915_vma_resource_free(vma_res);
 451                 return 0;
 452         }
 453
 454         GEM_BUG_ON(!atomic_read(&vma->pages_count));
 455
 456         /* Wait for or await async unbinds touching our range */
 457         if (work && bind_flags & vma->vm->bind_async_flags)
 458                 ret = i915_vma_resource_bind_dep_await(vma->vm,
 459                                                        &work->base.chain,
 460                                                        vma->node.start,
 461                                                        vma->node.size,
 462                                                        true,
 463                                                        GFP_NOWAIT |
 464                                                        __GFP_RETRY_MAYFAIL |
 465                                                        __GFP_NOWARN);
 466         else
 467                 ret = i915_vma_resource_bind_dep_sync(vma->vm, vma->node.start,
 468                                                       vma->node.size, true);
 469         if (ret) {
 470                 i915_vma_resource_free(vma_res);
 471                 return ret;
 472         }
 473
 474         if (vma->resource || !vma_res) {
 475                 /* Rebinding with an additional I915_VMA_*_BIND */
 476                 GEM_WARN_ON(!vma_flags);
 477                 i915_vma_resource_free(vma_res);
 478         } else {
 479                 i915_vma_resource_init_from_vma(vma_res, vma);
 480                 vma->resource = vma_res;
 481         }
 482         trace_i915_vma_bind(vma, bind_flags);
 483         if (work && bind_flags & vma->vm->bind_async_flags) {
 484                 struct dma_fence *prev;
 485
 486                 work->vma_res = i915_vma_resource_get(vma->resource);
 487                 work->cache_level = cache_level;
 488                 work->flags = bind_flags;
 489
 490                 /*
 491                  * Note we only want to chain up to the migration fence on
 492                  * the pages (not the object itself). As we don't track that,
 493                  * yet, we have to use the exclusive fence instead.
 494                  *
 495                  * Also note that we do not want to track the async vma as
 496                  * part of the obj->resv->excl_fence as it only affects
 497                  * execution and not content or object's backing store lifetime.
 498                  */
 499                 prev = i915_active_set_exclusive(&vma->active, &work->base.dma);
 500                 if (prev) {
 501                         __i915_sw_fence_await_dma_fence(&work->base.chain,
 502                                                         prev,
 503                                                         &work->cb);
 504                         dma_fence_put(prev);
 505                 }
 506
 507                 work->base.dma.error = 0; /* enable the queue_work() */
 508
 509                 /*
 510                  * If we don't have the refcounted pages list, keep a reference
 511                  * on the object to avoid waiting for the async bind to
 512                  * complete in the object destruction path.
 513                  */
 514                 if (!work->vma_res->bi.pages_rsgt)
 515                         work->pinned = i915_gem_object_get(vma->obj);
 516         } else {
 517                 if (vma->obj) {
 518                         ret = i915_gem_object_wait_moving_fence(vma->obj, true);
 519                         if (ret) {
 520                                 i915_vma_resource_free(vma->resource);
 521                                 vma->resource = NULL;
 522
 523                                 return ret;
 524                         }
 525                 }
 526                 vma->ops->bind_vma(vma->vm, NULL, vma->resource, cache_level,
 527                                    bind_flags);
 528         }
 529
 530         atomic_or(bind_flags, &vma->flags);
 531         return 0;
 532 }
 533
 534 void __iomem *i915_vma_pin_iomap(struct i915_vma *vma)
 535 {
 536         void __iomem *ptr;
 537         int err;
 538
 539         if (!i915_gem_object_is_lmem(vma->obj)) {
 540                 if (GEM_WARN_ON(!i915_vma_is_map_and_fenceable(vma))) {
 541                         err = -ENODEV;
 542                         goto err;
 543                 }
 544         }
 545
 546         GEM_BUG_ON(!i915_vma_is_ggtt(vma));
 547         GEM_BUG_ON(!i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND));
 548         GEM_BUG_ON(i915_vma_verify_bind_complete(vma));
 549
 550         ptr = READ_ONCE(vma->iomap);
 551         if (ptr == NULL) {
 552                 /*
 553                  * TODO: consider just using i915_gem_object_pin_map() for lmem
 554                  * instead, which already supports mapping non-contiguous chunks
 555                  * of pages, that way we can also drop the
 556                  * I915_BO_ALLOC_CONTIGUOUS when allocating the object.
 557                  */
 558                 if (i915_gem_object_is_lmem(vma->obj))
 559                         ptr = i915_gem_object_lmem_io_map(vma->obj, 0,
 560                                                           vma->obj->base.size);
 561                 else
 562                         ptr = io_mapping_map_wc(&i915_vm_to_ggtt(vma->vm)->iomap,
 563                                                 vma->node.start,
 564                                                 vma->node.size);
 565                 if (ptr == NULL) {
 566                         err = -ENOMEM;
 567                         goto err;
 568                 }
 569
 570                 if (unlikely(cmpxchg(&vma->iomap, NULL, ptr))) {
 571                         io_mapping_unmap(ptr);
 572                         ptr = vma->iomap;
 573                 }
 574         }
 575
 576         __i915_vma_pin(vma);
 577
 578         err = i915_vma_pin_fence(vma);
 579         if (err)
 580                 goto err_unpin;
 581
 582         i915_vma_set_ggtt_write(vma);
 583
 584         /* NB Access through the GTT requires the device to be awake. */
 585         return ptr;
 586
 587 err_unpin:
 588         __i915_vma_unpin(vma);
 589 err:
 590         return IO_ERR_PTR(err);
 591 }
 592
 593 void i915_vma_flush_writes(struct i915_vma *vma)
 594 {
 595         if (i915_vma_unset_ggtt_write(vma))
 596                 intel_gt_flush_ggtt_writes(vma->vm->gt);
 597 }
 598
 599 void i915_vma_unpin_iomap(struct i915_vma *vma)
 600 {
 601         GEM_BUG_ON(vma->iomap == NULL);
 602
 603         i915_vma_flush_writes(vma);
 604
 605         i915_vma_unpin_fence(vma);
 606         i915_vma_unpin(vma);
 607 }
 608
 609 void i915_vma_unpin_and_release(struct i915_vma **p_vma, unsigned int flags)
 610 {
 611         struct i915_vma *vma;
 612         struct drm_i915_gem_object *obj;
 613
 614         vma = fetch_and_zero(p_vma);
 615         if (!vma)
 616                 return;
 617
 618         obj = vma->obj;
 619         GEM_BUG_ON(!obj);
 620
 621         i915_vma_unpin(vma);
 622
 623         if (flags & I915_VMA_RELEASE_MAP)
 624                 i915_gem_object_unpin_map(obj);
 625
 626         i915_gem_object_put(obj);
 627 }
 628
 629 bool i915_vma_misplaced(const struct i915_vma *vma,
 630                         u64 size, u64 alignment, u64 flags)
 631 {
 632         if (!drm_mm_node_allocated(&vma->node))
 633                 return false;
 634
 635         if (test_bit(I915_VMA_ERROR_BIT, __i915_vma_flags(vma)))
 636                 return true;
 637
 638         if (vma->node.size < size)
 639                 return true;
 640
 641         GEM_BUG_ON(alignment && !is_power_of_2(alignment));
 642         if (alignment && !IS_ALIGNED(vma->node.start, alignment))
 643                 return true;
 644
 645         if (flags & PIN_MAPPABLE && !i915_vma_is_map_and_fenceable(vma))
 646                 return true;
 647
 648         if (flags & PIN_OFFSET_BIAS &&
 649             vma->node.start < (flags & PIN_OFFSET_MASK))
 650                 return true;
 651
 652         if (flags & PIN_OFFSET_FIXED &&
 653             vma->node.start != (flags & PIN_OFFSET_MASK))
 654                 return true;
 655
 656         return false;
 657 }
 658
 659 void __i915_vma_set_map_and_fenceable(struct i915_vma *vma)
 660 {
 661         bool mappable, fenceable;
 662
 663         GEM_BUG_ON(!i915_vma_is_ggtt(vma));
 664         GEM_BUG_ON(!vma->fence_size);
 665
 666         fenceable = (vma->node.size >= vma->fence_size &&
 667                      IS_ALIGNED(vma->node.start, vma->fence_alignment));
 668
 669         mappable = vma->node.start + vma->fence_size <= i915_vm_to_ggtt(vma->vm)->mappable_end;
 670
 671         if (mappable && fenceable)
 672                 set_bit(I915_VMA_CAN_FENCE_BIT, __i915_vma_flags(vma));
 673         else
 674                 clear_bit(I915_VMA_CAN_FENCE_BIT, __i915_vma_flags(vma));
 675 }
 676
 677 bool i915_gem_valid_gtt_space(struct i915_vma *vma, unsigned long color)
 678 {
 679         struct drm_mm_node *node = &vma->node;
 680         struct drm_mm_node *other;
 681
 682         /*
 683          * On some machines we have to be careful when putting differing types
 684          * of snoopable memory together to avoid the prefetcher crossing memory
 685          * domains and dying. During vm initialisation, we decide whether or not
 686          * these constraints apply and set the drm_mm.color_adjust
 687          * appropriately.
 688          */
 689         if (!i915_vm_has_cache_coloring(vma->vm))
 690                 return true;
 691
 692         /* Only valid to be called on an already inserted vma */
 693         GEM_BUG_ON(!drm_mm_node_allocated(node));
 694         GEM_BUG_ON(list_empty(&node->node_list));
 695
 696         other = list_prev_entry(node, node_list);
 697         if (i915_node_color_differs(other, color) &&
 698             !drm_mm_hole_follows(other))
 699                 return false;
 700
 701         other = list_next_entry(node, node_list);
 702         if (i915_node_color_differs(other, color) &&
 703             !drm_mm_hole_follows(node))
 704                 return false;
 705
 706         return true;
 707 }
 708
 709 /**
 710  * i915_vma_insert - finds a slot for the vma in its address space
 711  * @vma: the vma
 712  * @size: requested size in bytes (can be larger than the VMA)
 713  * @alignment: required alignment
 714  * @flags: mask of PIN_* flags to use
 715  *
 716  * First we try to allocate some free space that meets the requirements for
 717  * the VMA. Failiing that, if the flags permit, it will evict an old VMA,
 718  * preferrably the oldest idle entry to make room for the new VMA.
 719  *
 720  * Returns:
 721  * 0 on success, negative error code otherwise.
 722  */
 723 static int
 724 i915_vma_insert(struct i915_vma *vma, struct i915_gem_ww_ctx *ww,
 725                 u64 size, u64 alignment, u64 flags)
 726 {
 727         unsigned long color;
 728         u64 start, end;
 729         int ret;
 730
 731         GEM_BUG_ON(i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND));
 732         GEM_BUG_ON(drm_mm_node_allocated(&vma->node));
 733
 734         size = max(size, vma->size);
 735         alignment = max(alignment, vma->display_alignment);
 736         if (flags & PIN_MAPPABLE) {
 737                 size = max_t(typeof(size), size, vma->fence_size);
 738                 alignment = max_t(typeof(alignment),
 739                                   alignment, vma->fence_alignment);
 740         }
 741
 742         GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE));
 743         GEM_BUG_ON(!IS_ALIGNED(alignment, I915_GTT_MIN_ALIGNMENT));
 744         GEM_BUG_ON(!is_power_of_2(alignment));
 745
 746         start = flags & PIN_OFFSET_BIAS ? flags & PIN_OFFSET_MASK : 0;
 747         GEM_BUG_ON(!IS_ALIGNED(start, I915_GTT_PAGE_SIZE));
 748
 749         end = vma->vm->total;
 750         if (flags & PIN_MAPPABLE)
 751                 end = min_t(u64, end, i915_vm_to_ggtt(vma->vm)->mappable_end);
 752         if (flags & PIN_ZONE_4G)
 753                 end = min_t(u64, end, (1ULL << 32) - I915_GTT_PAGE_SIZE);
 754         GEM_BUG_ON(!IS_ALIGNED(end, I915_GTT_PAGE_SIZE));
 755
 756         /* If binding the object/GGTT view requires more space than the entire
 757          * aperture has, reject it early before evicting everything in a vain
 758          * attempt to find space.
 759          */
 760         if (size > end) {
 761                 DRM_DEBUG("Attempting to bind an object larger than the aperture: request=%llu > %s aperture=%llu\n",
 762                           size, flags & PIN_MAPPABLE ? "mappable" : "total",
 763                           end);
 764                 return -ENOSPC;
 765         }
 766
 767         color = 0;
 768         if (i915_vm_has_cache_coloring(vma->vm))
 769                 color = vma->obj->cache_level;
 770
 771         if (flags & PIN_OFFSET_FIXED) {
 772                 u64 offset = flags & PIN_OFFSET_MASK;
 773                 if (!IS_ALIGNED(offset, alignment) ||
 774                     range_overflows(offset, size, end))
 775                         return -EINVAL;
 776
 777                 ret = i915_gem_gtt_reserve(vma->vm, ww, &vma->node,
 778                                            size, offset, color,
 779                                            flags);
 780                 if (ret)
 781                         return ret;
 782         } else {
 783                 /*
 784                  * We only support huge gtt pages through the 48b PPGTT,
 785                  * however we also don't want to force any alignment for
 786                  * objects which need to be tightly packed into the low 32bits.
 787                  *
 788                  * Note that we assume that GGTT are limited to 4GiB for the
 789                  * forseeable future. See also i915_ggtt_offset().
 790                  */
 791                 if (upper_32_bits(end - 1) &&
 792                     vma->page_sizes.sg > I915_GTT_PAGE_SIZE) {
 793                         /*
 794                          * We can't mix 64K and 4K PTEs in the same page-table
 795                          * (2M block), and so to avoid the ugliness and
 796                          * complexity of coloring we opt for just aligning 64K
 797                          * objects to 2M.
 798                          */
 799                         u64 page_alignment =
 800                                 rounddown_pow_of_two(vma->page_sizes.sg |
 801                                                      I915_GTT_PAGE_SIZE_2M);
 802
 803                         /*
 804                          * Check we don't expand for the limited Global GTT
 805                          * (mappable aperture is even more precious!). This
 806                          * also checks that we exclude the aliasing-ppgtt.
 807                          */
 808                         GEM_BUG_ON(i915_vma_is_ggtt(vma));
 809
 810                         alignment = max(alignment, page_alignment);
 811
 812                         if (vma->page_sizes.sg & I915_GTT_PAGE_SIZE_64K)
 813                                 size = round_up(size, I915_GTT_PAGE_SIZE_2M);
 814                 }
 815
 816                 ret = i915_gem_gtt_insert(vma->vm, ww, &vma->node,
 817                                           size, alignment, color,
 818                                           start, end, flags);
 819                 if (ret)
 820                         return ret;
 821
 822                 GEM_BUG_ON(vma->node.start < start);
 823                 GEM_BUG_ON(vma->node.start + vma->node.size > end);
 824         }
 825         GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
 826         GEM_BUG_ON(!i915_gem_valid_gtt_space(vma, color));
 827
 828         list_add_tail(&vma->vm_link, &vma->vm->bound_list);
 829
 830         return 0;
 831 }
 832
 833 static void
 834 i915_vma_detach(struct i915_vma *vma)
 835 {
 836         GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
 837         GEM_BUG_ON(i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND));
 838
 839         /*
 840          * And finally now the object is completely decoupled from this
 841          * vma, we can drop its hold on the backing storage and allow
 842          * it to be reaped by the shrinker.
 843          */
 844         list_del(&vma->vm_link);
 845 }
 846
 847 static bool try_qad_pin(struct i915_vma *vma, unsigned int flags)
 848 {
 849         unsigned int bound;
 850
 851         bound = atomic_read(&vma->flags);
 852
 853         if (flags & PIN_VALIDATE) {
 854                 flags &= I915_VMA_BIND_MASK;
 855
 856                 return (flags & bound) == flags;
 857         }
 858
 859         /* with the lock mandatory for unbind, we don't race here */
 860         flags &= I915_VMA_BIND_MASK;
 861         do {
 862                 if (unlikely(flags & ~bound))
 863                         return false;
 864
 865                 if (unlikely(bound & (I915_VMA_OVERFLOW | I915_VMA_ERROR)))
 866                         return false;
 867
 868                 GEM_BUG_ON(((bound + 1) & I915_VMA_PIN_MASK) == 0);
 869         } while (!atomic_try_cmpxchg(&vma->flags, &bound, bound + 1));
 870
 871         return true;
 872 }
 873
 874 static struct scatterlist *
 875 rotate_pages(struct drm_i915_gem_object *obj, unsigned int offset,
 876              unsigned int width, unsigned int height,
 877              unsigned int src_stride, unsigned int dst_stride,
 878              struct sg_table *st, struct scatterlist *sg)
 879 {
 880         unsigned int column, row;
 881         unsigned int src_idx;
 882
 883         for (column = 0; column < width; column++) {
 884                 unsigned int left;
 885
 886                 src_idx = src_stride * (height - 1) + column + offset;
 887                 for (row = 0; row < height; row++) {
 888                         st->nents++;
 889                         /*
 890                          * We don't need the pages, but need to initialize
 891                          * the entries so the sg list can be happily traversed.
 892                          * The only thing we need are DMA addresses.
 893                          */
 894                         sg_set_page(sg, NULL, I915_GTT_PAGE_SIZE, 0);
 895                         sg_dma_address(sg) =
 896                                 i915_gem_object_get_dma_address(obj, src_idx);
 897                         sg_dma_len(sg) = I915_GTT_PAGE_SIZE;
 898                         sg = sg_next(sg);
 899                         src_idx -= src_stride;
 900                 }
 901
 902                 left = (dst_stride - height) * I915_GTT_PAGE_SIZE;
 903
 904                 if (!left)
 905                         continue;
 906
 907                 st->nents++;
 908
 909                 /*
 910                  * The DE ignores the PTEs for the padding tiles, the sg entry
 911                  * here is just a conenience to indicate how many padding PTEs
 912                  * to insert at this spot.
 913                  */
 914                 sg_set_page(sg, NULL, left, 0);
 915                 sg_dma_address(sg) = 0;
 916                 sg_dma_len(sg) = left;
 917                 sg = sg_next(sg);
 918         }
 919
 920         return sg;
 921 }
 922
 923 static noinline struct sg_table *
 924 intel_rotate_pages(struct intel_rotation_info *rot_info,
 925                    struct drm_i915_gem_object *obj)
 926 {
 927         unsigned int size = intel_rotation_info_size(rot_info);
 928         struct drm_i915_private *i915 = to_i915(obj->base.dev);
 929         struct sg_table *st;
 930         struct scatterlist *sg;
 931         int ret = -ENOMEM;
 932         int i;
 933
 934         /* Allocate target SG list. */
 935         st = kmalloc(sizeof(*st), GFP_KERNEL);
 936         if (!st)
 937                 goto err_st_alloc;
 938
 939         ret = sg_alloc_table(st, size, GFP_KERNEL);
 940         if (ret)
 941                 goto err_sg_alloc;
 942
 943         st->nents = 0;
 944         sg = st->sgl;
 945
 946         for (i = 0 ; i < ARRAY_SIZE(rot_info->plane); i++)
 947                 sg = rotate_pages(obj, rot_info->plane[i].offset,
 948                                   rot_info->plane[i].width, rot_info->plane[i].height,
 949                                   rot_info->plane[i].src_stride,
 950                                   rot_info->plane[i].dst_stride,
 951                                   st, sg);
 952
 953         return st;
 954
 955 err_sg_alloc:
 956         kfree(st);
 957 err_st_alloc:
 958
 959         drm_dbg(&i915->drm, "Failed to create rotated mapping for object size %zu! (%ux%u tiles, %u pages)\n",
 960                 obj->base.size, rot_info->plane[0].width,
 961                 rot_info->plane[0].height, size);
 962
 963         return ERR_PTR(ret);
 964 }
 965
 966 static struct scatterlist *
 967 remap_pages(struct drm_i915_gem_object *obj,
 968             unsigned int offset, unsigned int alignment_pad,
 969             unsigned int width, unsigned int height,
 970             unsigned int src_stride, unsigned int dst_stride,
 971             struct sg_table *st, struct scatterlist *sg)
 972 {
 973         unsigned int row;
 974
 975         if (!width || !height)
 976                 return sg;
 977
 978         if (alignment_pad) {
 979                 st->nents++;
 980
 981                 /*
 982                  * The DE ignores the PTEs for the padding tiles, the sg entry
 983                  * here is just a convenience to indicate how many padding PTEs
 984                  * to insert at this spot.
 985                  */
 986                 sg_set_page(sg, NULL, alignment_pad * 4096, 0);
 987                 sg_dma_address(sg) = 0;
 988                 sg_dma_len(sg) = alignment_pad * 4096;
 989                 sg = sg_next(sg);
 990         }
 991
 992         for (row = 0; row < height; row++) {
 993                 unsigned int left = width * I915_GTT_PAGE_SIZE;
 994
 995                 while (left) {
 996                         dma_addr_t addr;
 997                         unsigned int length;
 998
 999                         /*
1000                          * We don't need the pages, but need to initialize
1001                          * the entries so the sg list can be happily traversed.
1002                          * The only thing we need are DMA addresses.
1003                          */
1004
1005                         addr = i915_gem_object_get_dma_address_len(obj, offset, &length);
1006
1007                         length = min(left, length);
1008
1009                         st->nents++;
1010
1011                         sg_set_page(sg, NULL, length, 0);
1012                         sg_dma_address(sg) = addr;
1013                         sg_dma_len(sg) = length;
1014                         sg = sg_next(sg);
1015
1016                         offset += length / I915_GTT_PAGE_SIZE;
1017                         left -= length;
1018                 }
1019
1020                 offset += src_stride - width;
1021
1022                 left = (dst_stride - width) * I915_GTT_PAGE_SIZE;
1023
1024                 if (!left)
1025                         continue;
1026
1027                 st->nents++;
1028
1029                 /*
1030                  * The DE ignores the PTEs for the padding tiles, the sg entry
1031                  * here is just a conenience to indicate how many padding PTEs
1032                  * to insert at this spot.
1033                  */
1034                 sg_set_page(sg, NULL, left, 0);
1035                 sg_dma_address(sg) = 0;
1036                 sg_dma_len(sg) = left;
1037                 sg = sg_next(sg);
1038         }
1039
1040         return sg;
1041 }
1042
1043 static noinline struct sg_table *
1044 intel_remap_pages(struct intel_remapped_info *rem_info,
1045                   struct drm_i915_gem_object *obj)
1046 {
1047         unsigned int size = intel_remapped_info_size(rem_info);
1048         struct drm_i915_private *i915 = to_i915(obj->base.dev);
1049         struct sg_table *st;
1050         struct scatterlist *sg;
1051         unsigned int gtt_offset = 0;
1052         int ret = -ENOMEM;
1053         int i;
1054
1055         /* Allocate target SG list. */
1056         st = kmalloc(sizeof(*st), GFP_KERNEL);
1057         if (!st)
1058                 goto err_st_alloc;
1059
1060         ret = sg_alloc_table(st, size, GFP_KERNEL);
1061         if (ret)
1062                 goto err_sg_alloc;
1063
1064         st->nents = 0;
1065         sg = st->sgl;
1066
1067         for (i = 0 ; i < ARRAY_SIZE(rem_info->plane); i++) {
1068                 unsigned int alignment_pad = 0;
1069
1070                 if (rem_info->plane_alignment)
1071                         alignment_pad = ALIGN(gtt_offset, rem_info->plane_alignment) - gtt_offset;
1072
1073                 sg = remap_pages(obj,
1074                                  rem_info->plane[i].offset, alignment_pad,
1075                                  rem_info->plane[i].width, rem_info->plane[i].height,
1076                                  rem_info->plane[i].src_stride, rem_info->plane[i].dst_stride,
1077                                  st, sg);
1078
1079                 gtt_offset += alignment_pad +
1080                               rem_info->plane[i].dst_stride * rem_info->plane[i].height;
1081         }
1082
1083         i915_sg_trim(st);
1084
1085         return st;
1086
1087 err_sg_alloc:
1088         kfree(st);
1089 err_st_alloc:
1090
1091         drm_dbg(&i915->drm, "Failed to create remapped mapping for object size %zu! (%ux%u tiles, %u pages)\n",
1092                 obj->base.size, rem_info->plane[0].width,
1093                 rem_info->plane[0].height, size);
1094
1095         return ERR_PTR(ret);
1096 }
1097
1098 static noinline struct sg_table *
1099 intel_partial_pages(const struct i915_ggtt_view *view,
1100                     struct drm_i915_gem_object *obj)
1101 {
1102         struct sg_table *st;
1103         struct scatterlist *sg, *iter;
1104         unsigned int count = view->partial.size;
1105         unsigned int offset;
1106         int ret = -ENOMEM;
1107
1108         st = kmalloc(sizeof(*st), GFP_KERNEL);
1109         if (!st)
1110                 goto err_st_alloc;
1111
1112         ret = sg_alloc_table(st, count, GFP_KERNEL);
1113         if (ret)
1114                 goto err_sg_alloc;
1115
1116         iter = i915_gem_object_get_sg_dma(obj, view->partial.offset, &offset);
1117         GEM_BUG_ON(!iter);
1118
1119         sg = st->sgl;
1120         st->nents = 0;
1121         do {
1122                 unsigned int len;
1123
1124                 len = min(sg_dma_len(iter) - (offset << PAGE_SHIFT),
1125                           count << PAGE_SHIFT);
1126                 sg_set_page(sg, NULL, len, 0);
1127                 sg_dma_address(sg) =
1128                         sg_dma_address(iter) + (offset << PAGE_SHIFT);
1129                 sg_dma_len(sg) = len;
1130
1131                 st->nents++;
1132                 count -= len >> PAGE_SHIFT;
1133                 if (count == 0) {
1134                         sg_mark_end(sg);
1135                         i915_sg_trim(st); /* Drop any unused tail entries. */
1136
1137                         return st;
1138                 }
1139
1140                 sg = __sg_next(sg);
1141                 iter = __sg_next(iter);
1142                 offset = 0;
1143         } while (1);
1144
1145 err_sg_alloc:
1146         kfree(st);
1147 err_st_alloc:
1148         return ERR_PTR(ret);
1149 }
1150
1151 static int
1152 __i915_vma_get_pages(struct i915_vma *vma)
1153 {
1154         struct sg_table *pages;
1155
1156         /*
1157          * The vma->pages are only valid within the lifespan of the borrowed
1158          * obj->mm.pages. When the obj->mm.pages sg_table is regenerated, so
1159          * must be the vma->pages. A simple rule is that vma->pages must only
1160          * be accessed when the obj->mm.pages are pinned.
1161          */
1162         GEM_BUG_ON(!i915_gem_object_has_pinned_pages(vma->obj));
1163
1164         switch (vma->ggtt_view.type) {
1165         default:
1166                 GEM_BUG_ON(vma->ggtt_view.type);
1167                 fallthrough;
1168         case I915_GGTT_VIEW_NORMAL:
1169                 pages = vma->obj->mm.pages;
1170                 break;
1171
1172         case I915_GGTT_VIEW_ROTATED:
1173                 pages =
1174                         intel_rotate_pages(&vma->ggtt_view.rotated, vma->obj);
1175                 break;
1176
1177         case I915_GGTT_VIEW_REMAPPED:
1178                 pages =
1179                         intel_remap_pages(&vma->ggtt_view.remapped, vma->obj);
1180                 break;
1181
1182         case I915_GGTT_VIEW_PARTIAL:
1183                 pages = intel_partial_pages(&vma->ggtt_view, vma->obj);
1184                 break;
1185         }
1186
1187         if (IS_ERR(pages)) {
1188                 drm_err(&vma->vm->i915->drm,
1189                         "Failed to get pages for VMA view type %u (%ld)!\n",
1190                         vma->ggtt_view.type, PTR_ERR(pages));
1191                 return PTR_ERR(pages);
1192         }
1193
1194         vma->pages = pages;
1195
1196         return 0;
1197 }
1198
1199 I915_SELFTEST_EXPORT int i915_vma_get_pages(struct i915_vma *vma)
1200 {
1201         int err;
1202
1203         if (atomic_add_unless(&vma->pages_count, 1, 0))
1204                 return 0;
1205
1206         err = i915_gem_object_pin_pages(vma->obj);
1207         if (err)
1208                 return err;
1209
1210         err = __i915_vma_get_pages(vma);
1211         if (err)
1212                 goto err_unpin;
1213
1214         vma->page_sizes = vma->obj->mm.page_sizes;
1215         atomic_inc(&vma->pages_count);
1216
1217         return 0;
1218
1219 err_unpin:
1220         __i915_gem_object_unpin_pages(vma->obj);
1221
1222         return err;
1223 }
1224
1225 static void __vma_put_pages(struct i915_vma *vma, unsigned int count)
1226 {
1227         /* We allocate under vma_get_pages, so beware the shrinker */
1228         GEM_BUG_ON(atomic_read(&vma->pages_count) < count);
1229
1230         if (atomic_sub_return(count, &vma->pages_count) == 0) {
1231                 if (vma->pages != vma->obj->mm.pages) {
1232                         sg_free_table(vma->pages);
1233                         kfree(vma->pages);
1234                 }
1235                 vma->pages = NULL;
1236
1237                 i915_gem_object_unpin_pages(vma->obj);
1238         }
1239 }
1240
1241 I915_SELFTEST_EXPORT void i915_vma_put_pages(struct i915_vma *vma)
1242 {
1243         if (atomic_add_unless(&vma->pages_count, -1, 1))
1244                 return;
1245
1246         __vma_put_pages(vma, 1);
1247 }
1248
1249 static void vma_unbind_pages(struct i915_vma *vma)
1250 {
1251         unsigned int count;
1252
1253         lockdep_assert_held(&vma->vm->mutex);
1254
1255         /* The upper portion of pages_count is the number of bindings */
1256         count = atomic_read(&vma->pages_count);
1257         count >>= I915_VMA_PAGES_BIAS;
1258         GEM_BUG_ON(!count);
1259
1260         __vma_put_pages(vma, count | count << I915_VMA_PAGES_BIAS);
1261 }
1262
1263 int i915_vma_pin_ww(struct i915_vma *vma, struct i915_gem_ww_ctx *ww,
1264                     u64 size, u64 alignment, u64 flags)
1265 {
1266         struct i915_vma_work *work = NULL;
1267         struct dma_fence *moving = NULL;
1268         struct i915_vma_resource *vma_res = NULL;
1269         intel_wakeref_t wakeref = 0;
1270         unsigned int bound;
1271         int err;
1272
1273         assert_vma_held(vma);
1274         GEM_BUG_ON(!ww);
1275
1276         BUILD_BUG_ON(PIN_GLOBAL != I915_VMA_GLOBAL_BIND);
1277         BUILD_BUG_ON(PIN_USER != I915_VMA_LOCAL_BIND);
1278
1279         GEM_BUG_ON(!(flags & (PIN_USER | PIN_GLOBAL)));
1280
1281         /* First try and grab the pin without rebinding the vma */
1282         if (try_qad_pin(vma, flags))
1283                 return 0;
1284
1285         err = i915_vma_get_pages(vma);
1286         if (err)
1287                 return err;
1288
1289         if (flags & PIN_GLOBAL)
1290                 wakeref = intel_runtime_pm_get(&vma->vm->i915->runtime_pm);
1291
1292         moving = vma->obj ? i915_gem_object_get_moving_fence(vma->obj) : NULL;
1293         if (flags & vma->vm->bind_async_flags || moving) {
1294                 /* lock VM */
1295                 err = i915_vm_lock_objects(vma->vm, ww);
1296                 if (err)
1297                         goto err_rpm;
1298
1299                 work = i915_vma_work();
1300                 if (!work) {
1301                         err = -ENOMEM;
1302                         goto err_rpm;
1303                 }
1304
1305                 work->vm = i915_vm_get(vma->vm);
1306
1307                 dma_fence_work_chain(&work->base, moving);
1308
1309                 /* Allocate enough page directories to used PTE */
1310                 if (vma->vm->allocate_va_range) {
1311                         err = i915_vm_alloc_pt_stash(vma->vm,
1312                                                      &work->stash,
1313                                                      vma->size);
1314                         if (err)
1315                                 goto err_fence;
1316
1317                         err = i915_vm_map_pt_stash(vma->vm, &work->stash);
1318                         if (err)
1319                                 goto err_fence;
1320                 }
1321         }
1322
1323         vma_res = i915_vma_resource_alloc();
1324         if (IS_ERR(vma_res)) {
1325                 err = PTR_ERR(vma_res);
1326                 goto err_fence;
1327         }
1328
1329         /*
1330          * Differentiate between user/kernel vma inside the aliasing-ppgtt.
1331          *
1332          * We conflate the Global GTT with the user's vma when using the
1333          * aliasing-ppgtt, but it is still vitally important to try and
1334          * keep the use cases distinct. For example, userptr objects are
1335          * not allowed inside the Global GTT as that will cause lock
1336          * inversions when we have to evict them the mmu_notifier callbacks -
1337          * but they are allowed to be part of the user ppGTT which can never
1338          * be mapped. As such we try to give the distinct users of the same
1339          * mutex, distinct lockclasses [equivalent to how we keep i915_ggtt
1340          * and i915_ppgtt separate].
1341          *
1342          * NB this may cause us to mask real lock inversions -- while the
1343          * code is safe today, lockdep may not be able to spot future
1344          * transgressions.
1345          */
1346         err = mutex_lock_interruptible_nested(&vma->vm->mutex,
1347                                               !(flags & PIN_GLOBAL));
1348         if (err)
1349                 goto err_vma_res;
1350
1351         /* No more allocations allowed now we hold vm->mutex */
1352
1353         if (unlikely(i915_vma_is_closed(vma))) {
1354                 err = -ENOENT;
1355                 goto err_unlock;
1356         }
1357
1358         bound = atomic_read(&vma->flags);
1359         if (unlikely(bound & I915_VMA_ERROR)) {
1360                 err = -ENOMEM;
1361                 goto err_unlock;
1362         }
1363
1364         if (unlikely(!((bound + 1) & I915_VMA_PIN_MASK))) {
1365                 err = -EAGAIN; /* pins are meant to be fairly temporary */
1366                 goto err_unlock;
1367         }
1368
1369         if (unlikely(!(flags & ~bound & I915_VMA_BIND_MASK))) {
1370                 if (!(flags & PIN_VALIDATE))
1371                         __i915_vma_pin(vma);
1372                 goto err_unlock;
1373         }
1374
1375         err = i915_active_acquire(&vma->active);
1376         if (err)
1377                 goto err_unlock;
1378
1379         if (!(bound & I915_VMA_BIND_MASK)) {
1380                 err = i915_vma_insert(vma, ww, size, alignment, flags);
1381                 if (err)
1382                         goto err_active;
1383
1384                 if (i915_is_ggtt(vma->vm))
1385                         __i915_vma_set_map_and_fenceable(vma);
1386         }
1387
1388         GEM_BUG_ON(!vma->pages);
1389         err = i915_vma_bind(vma,
1390                             vma->obj->cache_level,
1391                             flags, work, vma_res);
1392         vma_res = NULL;
1393         if (err)
1394                 goto err_remove;
1395
1396         /* There should only be at most 2 active bindings (user, global) */
1397         GEM_BUG_ON(bound + I915_VMA_PAGES_ACTIVE < bound);
1398         atomic_add(I915_VMA_PAGES_ACTIVE, &vma->pages_count);
1399         list_move_tail(&vma->vm_link, &vma->vm->bound_list);
1400
1401         if (!(flags & PIN_VALIDATE)) {
1402                 __i915_vma_pin(vma);
1403                 GEM_BUG_ON(!i915_vma_is_pinned(vma));
1404         }
1405         GEM_BUG_ON(!i915_vma_is_bound(vma, flags));
1406         GEM_BUG_ON(i915_vma_misplaced(vma, size, alignment, flags));
1407
1408 err_remove:
1409         if (!i915_vma_is_bound(vma, I915_VMA_BIND_MASK)) {
1410                 i915_vma_detach(vma);
1411                 drm_mm_remove_node(&vma->node);
1412         }
1413 err_active:
1414         i915_active_release(&vma->active);
1415 err_unlock:
1416         mutex_unlock(&vma->vm->mutex);
1417 err_vma_res:
1418         i915_vma_resource_free(vma_res);
1419 err_fence:
1420         if (work)
1421                 dma_fence_work_commit_imm(&work->base);
1422 err_rpm:
1423         if (wakeref)
1424                 intel_runtime_pm_put(&vma->vm->i915->runtime_pm, wakeref);
1425
1426         if (moving)
1427                 dma_fence_put(moving);
1428
1429         i915_vma_put_pages(vma);
1430         return err;
1431 }
1432
1433 static void flush_idle_contexts(struct intel_gt *gt)
1434 {
1435         struct intel_engine_cs *engine;
1436         enum intel_engine_id id;
1437
1438         for_each_engine(engine, gt, id)
1439                 intel_engine_flush_barriers(engine);
1440
1441         intel_gt_wait_for_idle(gt, MAX_SCHEDULE_TIMEOUT);
1442 }
1443
1444 static int __i915_ggtt_pin(struct i915_vma *vma, struct i915_gem_ww_ctx *ww,
1445                            u32 align, unsigned int flags)
1446 {
1447         struct i915_address_space *vm = vma->vm;
1448         int err;
1449
1450         do {
1451                 err = i915_vma_pin_ww(vma, ww, 0, align, flags | PIN_GLOBAL);
1452
1453                 if (err != -ENOSPC) {
1454                         if (!err) {
1455                                 err = i915_vma_wait_for_bind(vma);
1456                                 if (err)
1457                                         i915_vma_unpin(vma);
1458                         }
1459                         return err;
1460                 }
1461
1462                 /* Unlike i915_vma_pin, we don't take no for an answer! */
1463                 flush_idle_contexts(vm->gt);
1464                 if (mutex_lock_interruptible(&vm->mutex) == 0) {
1465                         /*
1466                          * We pass NULL ww here, as we don't want to unbind
1467                          * locked objects when called from execbuf when pinning
1468                          * is removed. This would probably regress badly.
1469                          */
1470                         i915_gem_evict_vm(vm, NULL);
1471                         mutex_unlock(&vm->mutex);
1472                 }
1473         } while (1);
1474 }
1475
1476 int i915_ggtt_pin(struct i915_vma *vma, struct i915_gem_ww_ctx *ww,
1477                   u32 align, unsigned int flags)
1478 {
1479         struct i915_gem_ww_ctx _ww;
1480         int err;
1481
1482         GEM_BUG_ON(!i915_vma_is_ggtt(vma));
1483
1484         if (ww)
1485                 return __i915_ggtt_pin(vma, ww, align, flags);
1486
1487 #ifdef CONFIG_LOCKDEP
1488         WARN_ON(dma_resv_held(vma->obj->base.resv));
1489 #endif
1490
1491         for_i915_gem_ww(&_ww, err, true) {
1492                 err = i915_gem_object_lock(vma->obj, &_ww);
1493                 if (!err)
1494                         err = __i915_ggtt_pin(vma, &_ww, align, flags);
1495         }
1496
1497         return err;
1498 }
1499
1500 static void __vma_close(struct i915_vma *vma, struct intel_gt *gt)
1501 {
1502         /*
1503          * We defer actually closing, unbinding and destroying the VMA until
1504          * the next idle point, or if the object is freed in the meantime. By
1505          * postponing the unbind, we allow for it to be resurrected by the
1506          * client, avoiding the work required to rebind the VMA. This is
1507          * advantageous for DRI, where the client/server pass objects
1508          * between themselves, temporarily opening a local VMA to the
1509          * object, and then closing it again. The same object is then reused
1510          * on the next frame (or two, depending on the depth of the swap queue)
1511          * causing us to rebind the VMA once more. This ends up being a lot
1512          * of wasted work for the steady state.
1513          */
1514         GEM_BUG_ON(i915_vma_is_closed(vma));
1515         list_add(&vma->closed_link, &gt->closed_vma);
1516 }
1517
1518 void i915_vma_close(struct i915_vma *vma)
1519 {
1520         struct intel_gt *gt = vma->vm->gt;
1521         unsigned long flags;
1522
1523         if (i915_vma_is_ggtt(vma))
1524                 return;
1525
1526         GEM_BUG_ON(!atomic_read(&vma->open_count));
1527         if (atomic_dec_and_lock_irqsave(&vma->open_count,
1528                                         &gt->closed_lock,
1529                                         flags)) {
1530                 __vma_close(vma, gt);
1531                 spin_unlock_irqrestore(&gt->closed_lock, flags);
1532         }
1533 }
1534
1535 static void __i915_vma_remove_closed(struct i915_vma *vma)
1536 {
1537         struct intel_gt *gt = vma->vm->gt;
1538
1539         spin_lock_irq(&gt->closed_lock);
1540         list_del_init(&vma->closed_link);
1541         spin_unlock_irq(&gt->closed_lock);
1542 }
1543
1544 void i915_vma_reopen(struct i915_vma *vma)
1545 {
1546         if (i915_vma_is_closed(vma))
1547                 __i915_vma_remove_closed(vma);
1548 }
1549
1550 void i915_vma_release(struct kref *ref)
1551 {
1552         struct i915_vma *vma = container_of(ref, typeof(*vma), ref);
1553         struct drm_i915_gem_object *obj = vma->obj;
1554
1555         if (drm_mm_node_allocated(&vma->node)) {
1556                 mutex_lock(&vma->vm->mutex);
1557                 atomic_and(~I915_VMA_PIN_MASK, &vma->flags);
1558                 WARN_ON(__i915_vma_unbind(vma));
1559                 mutex_unlock(&vma->vm->mutex);
1560                 GEM_BUG_ON(drm_mm_node_allocated(&vma->node));
1561         }
1562         GEM_BUG_ON(i915_vma_is_active(vma));
1563
1564         spin_lock(&obj->vma.lock);
1565         list_del(&vma->obj_link);
1566         if (!RB_EMPTY_NODE(&vma->obj_node))
1567                 rb_erase(&vma->obj_node, &obj->vma.tree);
1568         spin_unlock(&obj->vma.lock);
1569
1570         __i915_vma_remove_closed(vma);
1571         i915_vm_put(vma->vm);
1572
1573         i915_active_fini(&vma->active);
1574         GEM_WARN_ON(vma->resource);
1575         i915_vma_free(vma);
1576 }
1577
1578 void i915_vma_parked(struct intel_gt *gt)
1579 {
1580         struct i915_vma *vma, *next;
1581         LIST_HEAD(closed);
1582
1583         spin_lock_irq(&gt->closed_lock);
1584         list_for_each_entry_safe(vma, next, &gt->closed_vma, closed_link) {
1585                 struct drm_i915_gem_object *obj = vma->obj;
1586                 struct i915_address_space *vm = vma->vm;
1587
1588                 /* XXX All to avoid keeping a reference on i915_vma itself */
1589
1590                 if (!kref_get_unless_zero(&obj->base.refcount))
1591                         continue;
1592
1593                 if (!i915_vm_tryopen(vm)) {
1594                         i915_gem_object_put(obj);
1595                         continue;
1596                 }
1597
1598                 list_move(&vma->closed_link, &closed);
1599         }
1600         spin_unlock_irq(&gt->closed_lock);
1601
1602         /* As the GT is held idle, no vma can be reopened as we destroy them */
1603         list_for_each_entry_safe(vma, next, &closed, closed_link) {
1604                 struct drm_i915_gem_object *obj = vma->obj;
1605                 struct i915_address_space *vm = vma->vm;
1606
1607                 if (i915_gem_object_trylock(obj, NULL)) {
1608                         INIT_LIST_HEAD(&vma->closed_link);
1609                         __i915_vma_put(vma);
1610                         i915_gem_object_unlock(obj);
1611                 } else {
1612                         /* back you go.. */
1613                         spin_lock_irq(&gt->closed_lock);
1614                         list_add(&vma->closed_link, &gt->closed_vma);
1615                         spin_unlock_irq(&gt->closed_lock);
1616                 }
1617
1618                 i915_gem_object_put(obj);
1619                 i915_vm_close(vm);
1620         }
1621 }
1622
1623 static void __i915_vma_iounmap(struct i915_vma *vma)
1624 {
1625         GEM_BUG_ON(i915_vma_is_pinned(vma));
1626
1627         if (vma->iomap == NULL)
1628                 return;
1629
1630         io_mapping_unmap(vma->iomap);
1631         vma->iomap = NULL;
1632 }
1633
1634 void i915_vma_revoke_mmap(struct i915_vma *vma)
1635 {
1636         struct drm_vma_offset_node *node;
1637         u64 vma_offset;
1638
1639         if (!i915_vma_has_userfault(vma))
1640                 return;
1641
1642         GEM_BUG_ON(!i915_vma_is_map_and_fenceable(vma));
1643         GEM_BUG_ON(!vma->obj->userfault_count);
1644
1645         node = &vma->mmo->vma_node;
1646         vma_offset = vma->ggtt_view.partial.offset << PAGE_SHIFT;
1647         unmap_mapping_range(vma->vm->i915->drm.anon_inode->i_mapping,
1648                             drm_vma_node_offset_addr(node) + vma_offset,
1649                             vma->size,
1650                             1);
1651
1652         i915_vma_unset_userfault(vma);
1653         if (!--vma->obj->userfault_count)
1654                 list_del(&vma->obj->userfault_link);
1655 }
1656
1657 static int
1658 __i915_request_await_bind(struct i915_request *rq, struct i915_vma *vma)
1659 {
1660         return __i915_request_await_exclusive(rq, &vma->active);
1661 }
1662
1663 static int __i915_vma_move_to_active(struct i915_vma *vma, struct i915_request *rq)
1664 {
1665         int err;
1666
1667         /* Wait for the vma to be bound before we start! */
1668         err = __i915_request_await_bind(rq, vma);
1669         if (err)
1670                 return err;
1671
1672         return i915_active_add_request(&vma->active, rq);
1673 }
1674
1675 int _i915_vma_move_to_active(struct i915_vma *vma,
1676                              struct i915_request *rq,
1677                              struct dma_fence *fence,
1678                              unsigned int flags)
1679 {
1680         struct drm_i915_gem_object *obj = vma->obj;
1681         int err;
1682
1683         assert_object_held(obj);
1684
1685         GEM_BUG_ON(!vma->pages);
1686
1687         err = __i915_vma_move_to_active(vma, rq);
1688         if (unlikely(err))
1689                 return err;
1690
1691         if (flags & EXEC_OBJECT_WRITE) {
1692                 struct intel_frontbuffer *front;
1693
1694                 front = __intel_frontbuffer_get(obj);
1695                 if (unlikely(front)) {
1696                         if (intel_frontbuffer_invalidate(front, ORIGIN_CS))
1697                                 i915_active_add_request(&front->write, rq);
1698                         intel_frontbuffer_put(front);
1699                 }
1700
1701                 if (fence) {
1702                         dma_resv_add_excl_fence(vma->obj->base.resv, fence);
1703                         obj->write_domain = I915_GEM_DOMAIN_RENDER;
1704                         obj->read_domains = 0;
1705                 }
1706         } else {
1707                 if (!(flags & __EXEC_OBJECT_NO_RESERVE)) {
1708                         err = dma_resv_reserve_shared(vma->obj->base.resv, 1);
1709                         if (unlikely(err))
1710                                 return err;
1711                 }
1712
1713                 if (fence) {
1714                         dma_resv_add_shared_fence(vma->obj->base.resv, fence);
1715                         obj->write_domain = 0;
1716                 }
1717         }
1718
1719         if (flags & EXEC_OBJECT_NEEDS_FENCE && vma->fence)
1720                 i915_active_add_request(&vma->fence->active, rq);
1721
1722         obj->read_domains |= I915_GEM_GPU_DOMAINS;
1723         obj->mm.dirty = true;
1724
1725         GEM_BUG_ON(!i915_vma_is_active(vma));
1726         return 0;
1727 }
1728
1729 struct dma_fence *__i915_vma_evict(struct i915_vma *vma, bool async)
1730 {
1731         struct i915_vma_resource *vma_res = vma->resource;
1732         struct dma_fence *unbind_fence;
1733
1734         GEM_BUG_ON(i915_vma_is_pinned(vma));
1735         assert_vma_held_evict(vma);
1736
1737         if (i915_vma_is_map_and_fenceable(vma)) {
1738                 /* Force a pagefault for domain tracking on next user access */
1739                 i915_vma_revoke_mmap(vma);
1740
1741                 /*
1742                  * Check that we have flushed all writes through the GGTT
1743                  * before the unbind, other due to non-strict nature of those
1744                  * indirect writes they may end up referencing the GGTT PTE
1745                  * after the unbind.
1746                  *
1747                  * Note that we may be concurrently poking at the GGTT_WRITE
1748                  * bit from set-domain, as we mark all GGTT vma associated
1749                  * with an object. We know this is for another vma, as we
1750                  * are currently unbinding this one -- so if this vma will be
1751                  * reused, it will be refaulted and have its dirty bit set
1752                  * before the next write.
1753                  */
1754                 i915_vma_flush_writes(vma);
1755
1756                 /* release the fence reg _after_ flushing */
1757                 i915_vma_revoke_fence(vma);
1758
1759                 __i915_vma_iounmap(vma);
1760                 clear_bit(I915_VMA_CAN_FENCE_BIT, __i915_vma_flags(vma));
1761         }
1762         GEM_BUG_ON(vma->fence);
1763         GEM_BUG_ON(i915_vma_has_userfault(vma));
1764
1765         /* Object backend must be async capable. */
1766         GEM_WARN_ON(async && !vma->resource->bi.pages_rsgt);
1767
1768         /* If vm is not open, unbind is a nop. */
1769         vma_res->needs_wakeref = i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND) &&
1770                 atomic_read(&vma->vm->open);
1771         trace_i915_vma_unbind(vma);
1772
1773         unbind_fence = i915_vma_resource_unbind(vma_res);
1774         vma->resource = NULL;
1775
1776         atomic_and(~(I915_VMA_BIND_MASK | I915_VMA_ERROR | I915_VMA_GGTT_WRITE),
1777                    &vma->flags);
1778
1779         i915_vma_detach(vma);
1780
1781         if (!async && unbind_fence) {
1782                 dma_fence_wait(unbind_fence, false);
1783                 dma_fence_put(unbind_fence);
1784                 unbind_fence = NULL;
1785         }
1786
1787         /*
1788          * Binding itself may not have completed until the unbind fence signals,
1789          * so don't drop the pages until that happens, unless the resource is
1790          * async_capable.
1791          */
1792
1793         vma_unbind_pages(vma);
1794         return unbind_fence;
1795 }
1796
1797 int __i915_vma_unbind(struct i915_vma *vma)
1798 {
1799         int ret;
1800
1801         lockdep_assert_held(&vma->vm->mutex);
1802         assert_vma_held_evict(vma);
1803
1804         if (!drm_mm_node_allocated(&vma->node))
1805                 return 0;
1806
1807         if (i915_vma_is_pinned(vma)) {
1808                 vma_print_allocator(vma, "is pinned");
1809                 return -EAGAIN;
1810         }
1811
1812         /*
1813          * After confirming that no one else is pinning this vma, wait for
1814          * any laggards who may have crept in during the wait (through
1815          * a residual pin skipping the vm->mutex) to complete.
1816          */
1817         ret = i915_vma_sync(vma);
1818         if (ret)
1819                 return ret;
1820
1821         GEM_BUG_ON(i915_vma_is_active(vma));
1822         __i915_vma_evict(vma, false);
1823
1824         drm_mm_remove_node(&vma->node); /* pairs with i915_vma_release() */
1825         return 0;
1826 }
1827
1828 static struct dma_fence *__i915_vma_unbind_async(struct i915_vma *vma)
1829 {
1830         struct dma_fence *fence;
1831
1832         lockdep_assert_held(&vma->vm->mutex);
1833
1834         if (!drm_mm_node_allocated(&vma->node))
1835                 return NULL;
1836
1837         if (i915_vma_is_pinned(vma) ||
1838             &vma->obj->mm.rsgt->table != vma->resource->bi.pages)
1839                 return ERR_PTR(-EAGAIN);
1840
1841         /*
1842          * We probably need to replace this with awaiting the fences of the
1843          * object's dma_resv when the vma active goes away. When doing that
1844          * we need to be careful to not add the vma_resource unbind fence
1845          * immediately to the object's dma_resv, because then unbinding
1846          * the next vma from the object, in case there are many, will
1847          * actually await the unbinding of the previous vmas, which is
1848          * undesirable.
1849          */
1850         if (i915_sw_fence_await_active(&vma->resource->chain, &vma->active,
1851                                        I915_ACTIVE_AWAIT_EXCL |
1852                                        I915_ACTIVE_AWAIT_ACTIVE) < 0) {
1853                 return ERR_PTR(-EBUSY);
1854         }
1855
1856         fence = __i915_vma_evict(vma, true);
1857
1858         drm_mm_remove_node(&vma->node); /* pairs with i915_vma_release() */
1859
1860         return fence;
1861 }
1862
1863 int i915_vma_unbind(struct i915_vma *vma)
1864 {
1865         struct i915_address_space *vm = vma->vm;
1866         intel_wakeref_t wakeref = 0;
1867         int err;
1868
1869         assert_object_held_shared(vma->obj);
1870
1871         /* Optimistic wait before taking the mutex */
1872         err = i915_vma_sync(vma);
1873         if (err)
1874                 return err;
1875
1876         if (!drm_mm_node_allocated(&vma->node))
1877                 return 0;
1878
1879         if (i915_vma_is_pinned(vma)) {
1880                 vma_print_allocator(vma, "is pinned");
1881                 return -EAGAIN;
1882         }
1883
1884         if (i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND))
1885                 /* XXX not always required: nop_clear_range */
1886                 wakeref = intel_runtime_pm_get(&vm->i915->runtime_pm);
1887
1888         err = mutex_lock_interruptible_nested(&vma->vm->mutex, !wakeref);
1889         if (err)
1890                 goto out_rpm;
1891
1892         err = __i915_vma_unbind(vma);
1893         mutex_unlock(&vm->mutex);
1894
1895 out_rpm:
1896         if (wakeref)
1897                 intel_runtime_pm_put(&vm->i915->runtime_pm, wakeref);
1898         return err;
1899 }
1900
1901 int i915_vma_unbind_async(struct i915_vma *vma, bool trylock_vm)
1902 {
1903         struct drm_i915_gem_object *obj = vma->obj;
1904         struct i915_address_space *vm = vma->vm;
1905         intel_wakeref_t wakeref = 0;
1906         struct dma_fence *fence;
1907         int err;
1908
1909         /*
1910          * We need the dma-resv lock since we add the
1911          * unbind fence to the dma-resv object.
1912          */
1913         assert_object_held(obj);
1914
1915         if (!drm_mm_node_allocated(&vma->node))
1916                 return 0;
1917
1918         if (i915_vma_is_pinned(vma)) {
1919                 vma_print_allocator(vma, "is pinned");
1920                 return -EAGAIN;
1921         }
1922
1923         if (!obj->mm.rsgt)
1924                 return -EBUSY;
1925
1926         err = dma_resv_reserve_shared(obj->base.resv, 1);
1927         if (err)
1928                 return -EBUSY;
1929
1930         /*
1931          * It would be great if we could grab this wakeref from the
1932          * async unbind work if needed, but we can't because it uses
1933          * kmalloc and it's in the dma-fence signalling critical path.
1934          */
1935         if (i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND))
1936                 wakeref = intel_runtime_pm_get(&vm->i915->runtime_pm);
1937
1938         if (trylock_vm && !mutex_trylock(&vm->mutex)) {
1939                 err = -EBUSY;
1940                 goto out_rpm;
1941         } else if (!trylock_vm) {
1942                 err = mutex_lock_interruptible_nested(&vm->mutex, !wakeref);
1943                 if (err)
1944                         goto out_rpm;
1945         }
1946
1947         fence = __i915_vma_unbind_async(vma);
1948         mutex_unlock(&vm->mutex);
1949         if (IS_ERR_OR_NULL(fence)) {
1950                 err = PTR_ERR_OR_ZERO(fence);
1951                 goto out_rpm;
1952         }
1953
1954         dma_resv_add_shared_fence(obj->base.resv, fence);
1955         dma_fence_put(fence);
1956
1957 out_rpm:
1958         if (wakeref)
1959                 intel_runtime_pm_put(&vm->i915->runtime_pm, wakeref);
1960         return err;
1961 }
1962
1963 int i915_vma_unbind_unlocked(struct i915_vma *vma)
1964 {
1965         int err;
1966
1967         i915_gem_object_lock(vma->obj, NULL);
1968         err = i915_vma_unbind(vma);
1969         i915_gem_object_unlock(vma->obj);
1970
1971         return err;
1972 }
1973
1974 struct i915_vma *i915_vma_make_unshrinkable(struct i915_vma *vma)
1975 {
1976         i915_gem_object_make_unshrinkable(vma->obj);
1977         return vma;
1978 }
1979
1980 void i915_vma_make_shrinkable(struct i915_vma *vma)
1981 {
1982         i915_gem_object_make_shrinkable(vma->obj);
1983 }
1984
1985 void i915_vma_make_purgeable(struct i915_vma *vma)
1986 {
1987         i915_gem_object_make_purgeable(vma->obj);
1988 }
1989
1990 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
1991 #include "selftests/i915_vma.c"
1992 #endif
1993
1994 void i915_vma_module_exit(void)
1995 {
1996         kmem_cache_destroy(slab_vmas);
1997 }
1998
1999 int __init i915_vma_module_init(void)
2000 {
2001         slab_vmas = KMEM_CACHE(i915_vma, SLAB_HWCACHE_ALIGN);
2002         if (!slab_vmas)
2003                 return -ENOMEM;
2004
2005         return 0;
2006 }