drivers/gpu/drm/ttm/ttm_bo.c

   1 /* SPDX-License-Identifier: GPL-2.0 OR MIT */
   2 /**************************************************************************
   3  *
   4  * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
   5  * All Rights Reserved.
   6  *
   7  * Permission is hereby granted, free of charge, to any person obtaining a
   8  * copy of this software and associated documentation files (the
   9  * "Software"), to deal in the Software without restriction, including
  10  * without limitation the rights to use, copy, modify, merge, publish,
  11  * distribute, sub license, and/or sell copies of the Software, and to
  12  * permit persons to whom the Software is furnished to do so, subject to
  13  * the following conditions:
  14  *
  15  * The above copyright notice and this permission notice (including the
  16  * next paragraph) shall be included in all copies or substantial portions
  17  * of the Software.
  18  *
  19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  20  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  21  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  22  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
  23  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  24  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  25  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  26  *
  27  **************************************************************************/
  28 /*
  29  * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
  30  */
  31
  32 #define pr_fmt(fmt) "[TTM] " fmt
  33
  34 #include <drm/ttm/ttm_bo_driver.h>
  35 #include <drm/ttm/ttm_placement.h>
  36 #include <linux/jiffies.h>
  37 #include <linux/slab.h>
  38 #include <linux/sched.h>
  39 #include <linux/mm.h>
  40 #include <linux/file.h>
  41 #include <linux/module.h>
  42 #include <linux/atomic.h>
  43 #include <linux/dma-resv.h>
  44
  45 #include "ttm_module.h"
  46
  47 static void ttm_bo_global_kobj_release(struct kobject *kobj);
  48
  49 /*
  50  * ttm_global_mutex - protecting the global BO state
  51  */
  52 DEFINE_MUTEX(ttm_global_mutex);
  53 unsigned ttm_bo_glob_use_count;
  54 struct ttm_bo_global ttm_bo_glob;
  55 EXPORT_SYMBOL(ttm_bo_glob);
  56
  57 static struct attribute ttm_bo_count = {
  58         .name = "bo_count",
  59         .mode = S_IRUGO
  60 };
  61
  62 /* default destructor */
  63 static void ttm_bo_default_destroy(struct ttm_buffer_object *bo)
  64 {
  65         kfree(bo);
  66 }
  67
  68 static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo,
  69                                         struct ttm_placement *placement)
  70 {
  71         struct drm_printer p = drm_debug_printer(TTM_PFX);
  72         struct ttm_resource_manager *man;
  73         int i, mem_type;
  74
  75         drm_printf(&p, "No space for %p (%lu pages, %zuK, %zuM)\n",
  76                    bo, bo->mem.num_pages, bo->base.size >> 10,
  77                    bo->base.size >> 20);
  78         for (i = 0; i < placement->num_placement; i++) {
  79                 mem_type = placement->placement[i].mem_type;
  80                 drm_printf(&p, "  placement[%d]=0x%08X (%d)\n",
  81                            i, placement->placement[i].flags, mem_type);
  82                 man = ttm_manager_type(bo->bdev, mem_type);
  83                 ttm_resource_manager_debug(man, &p);
  84         }
  85 }
  86
  87 static ssize_t ttm_bo_global_show(struct kobject *kobj,
  88                                   struct attribute *attr,
  89                                   char *buffer)
  90 {
  91         struct ttm_bo_global *glob =
  92                 container_of(kobj, struct ttm_bo_global, kobj);
  93
  94         return snprintf(buffer, PAGE_SIZE, "%d\n",
  95                                 atomic_read(&glob->bo_count));
  96 }
  97
  98 static struct attribute *ttm_bo_global_attrs[] = {
  99         &ttm_bo_count,
 100         NULL
 101 };
 102
 103 static const struct sysfs_ops ttm_bo_global_ops = {
 104         .show = &ttm_bo_global_show
 105 };
 106
 107 static struct kobj_type ttm_bo_glob_kobj_type  = {
 108         .release = &ttm_bo_global_kobj_release,
 109         .sysfs_ops = &ttm_bo_global_ops,
 110         .default_attrs = ttm_bo_global_attrs
 111 };
 112
 113 static void ttm_bo_del_from_lru(struct ttm_buffer_object *bo)
 114 {
 115         struct ttm_bo_device *bdev = bo->bdev;
 116
 117         list_del_init(&bo->swap);
 118         list_del_init(&bo->lru);
 119
 120         if (bdev->driver->del_from_lru_notify)
 121                 bdev->driver->del_from_lru_notify(bo);
 122 }
 123
 124 static void ttm_bo_bulk_move_set_pos(struct ttm_lru_bulk_move_pos *pos,
 125                                      struct ttm_buffer_object *bo)
 126 {
 127         if (!pos->first)
 128                 pos->first = bo;
 129         pos->last = bo;
 130 }
 131
 132 void ttm_bo_move_to_lru_tail(struct ttm_buffer_object *bo,
 133                              struct ttm_resource *mem,
 134                              struct ttm_lru_bulk_move *bulk)
 135 {
 136         struct ttm_bo_device *bdev = bo->bdev;
 137         struct ttm_resource_manager *man;
 138
 139         dma_resv_assert_held(bo->base.resv);
 140
 141         if (bo->pin_count) {
 142                 ttm_bo_del_from_lru(bo);
 143                 return;
 144         }
 145
 146         man = ttm_manager_type(bdev, mem->mem_type);
 147         list_move_tail(&bo->lru, &man->lru[bo->priority]);
 148         if (man->use_tt && bo->ttm &&
 149             !(bo->ttm->page_flags & (TTM_PAGE_FLAG_SG |
 150                                      TTM_PAGE_FLAG_SWAPPED))) {
 151                 struct list_head *swap;
 152
 153                 swap = &ttm_bo_glob.swap_lru[bo->priority];
 154                 list_move_tail(&bo->swap, swap);
 155         }
 156
 157         if (bdev->driver->del_from_lru_notify)
 158                 bdev->driver->del_from_lru_notify(bo);
 159
 160         if (bulk && !bo->pin_count) {
 161                 switch (bo->mem.mem_type) {
 162                 case TTM_PL_TT:
 163                         ttm_bo_bulk_move_set_pos(&bulk->tt[bo->priority], bo);
 164                         break;
 165
 166                 case TTM_PL_VRAM:
 167                         ttm_bo_bulk_move_set_pos(&bulk->vram[bo->priority], bo);
 168                         break;
 169                 }
 170                 if (bo->ttm && !(bo->ttm->page_flags &
 171                                  (TTM_PAGE_FLAG_SG | TTM_PAGE_FLAG_SWAPPED)))
 172                         ttm_bo_bulk_move_set_pos(&bulk->swap[bo->priority], bo);
 173         }
 174 }
 175 EXPORT_SYMBOL(ttm_bo_move_to_lru_tail);
 176
 177 void ttm_bo_bulk_move_lru_tail(struct ttm_lru_bulk_move *bulk)
 178 {
 179         unsigned i;
 180
 181         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
 182                 struct ttm_lru_bulk_move_pos *pos = &bulk->tt[i];
 183                 struct ttm_resource_manager *man;
 184
 185                 if (!pos->first)
 186                         continue;
 187
 188                 dma_resv_assert_held(pos->first->base.resv);
 189                 dma_resv_assert_held(pos->last->base.resv);
 190
 191                 man = ttm_manager_type(pos->first->bdev, TTM_PL_TT);
 192                 list_bulk_move_tail(&man->lru[i], &pos->first->lru,
 193                                     &pos->last->lru);
 194         }
 195
 196         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
 197                 struct ttm_lru_bulk_move_pos *pos = &bulk->vram[i];
 198                 struct ttm_resource_manager *man;
 199
 200                 if (!pos->first)
 201                         continue;
 202
 203                 dma_resv_assert_held(pos->first->base.resv);
 204                 dma_resv_assert_held(pos->last->base.resv);
 205
 206                 man = ttm_manager_type(pos->first->bdev, TTM_PL_VRAM);
 207                 list_bulk_move_tail(&man->lru[i], &pos->first->lru,
 208                                     &pos->last->lru);
 209         }
 210
 211         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
 212                 struct ttm_lru_bulk_move_pos *pos = &bulk->swap[i];
 213                 struct list_head *lru;
 214
 215                 if (!pos->first)
 216                         continue;
 217
 218                 dma_resv_assert_held(pos->first->base.resv);
 219                 dma_resv_assert_held(pos->last->base.resv);
 220
 221                 lru = &ttm_bo_glob.swap_lru[i];
 222                 list_bulk_move_tail(lru, &pos->first->swap, &pos->last->swap);
 223         }
 224 }
 225 EXPORT_SYMBOL(ttm_bo_bulk_move_lru_tail);
 226
 227 static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo,
 228                                   struct ttm_resource *mem, bool evict,
 229                                   struct ttm_operation_ctx *ctx,
 230                                   struct ttm_place *hop)
 231 {
 232         struct ttm_bo_device *bdev = bo->bdev;
 233         struct ttm_resource_manager *old_man = ttm_manager_type(bdev, bo->mem.mem_type);
 234         struct ttm_resource_manager *new_man = ttm_manager_type(bdev, mem->mem_type);
 235         int ret;
 236
 237         ttm_bo_unmap_virtual(bo);
 238
 239         /*
 240          * Create and bind a ttm if required.
 241          */
 242
 243         if (new_man->use_tt) {
 244                 /* Zero init the new TTM structure if the old location should
 245                  * have used one as well.
 246                  */
 247                 ret = ttm_tt_create(bo, old_man->use_tt);
 248                 if (ret)
 249                         goto out_err;
 250
 251                 if (mem->mem_type != TTM_PL_SYSTEM) {
 252                         ret = ttm_tt_populate(bo->bdev, bo->ttm, ctx);
 253                         if (ret)
 254                                 goto out_err;
 255                 }
 256         }
 257
 258         ret = bdev->driver->move(bo, evict, ctx, mem, hop);
 259         if (ret) {
 260                 if (ret == -EMULTIHOP)
 261                         return ret;
 262                 goto out_err;
 263         }
 264
 265         ctx->bytes_moved += bo->base.size;
 266         return 0;
 267
 268 out_err:
 269         new_man = ttm_manager_type(bdev, bo->mem.mem_type);
 270         if (!new_man->use_tt)
 271                 ttm_bo_tt_destroy(bo);
 272
 273         return ret;
 274 }
 275
 276 /*
 277  * Call bo::reserved.
 278  * Will release GPU memory type usage on destruction.
 279  * This is the place to put in driver specific hooks to release
 280  * driver private resources.
 281  * Will release the bo::reserved lock.
 282  */
 283
 284 static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo)
 285 {
 286         if (bo->bdev->driver->delete_mem_notify)
 287                 bo->bdev->driver->delete_mem_notify(bo);
 288
 289         ttm_bo_tt_destroy(bo);
 290         ttm_resource_free(bo, &bo->mem);
 291 }
 292
 293 static int ttm_bo_individualize_resv(struct ttm_buffer_object *bo)
 294 {
 295         int r;
 296
 297         if (bo->base.resv == &bo->base._resv)
 298                 return 0;
 299
 300         BUG_ON(!dma_resv_trylock(&bo->base._resv));
 301
 302         r = dma_resv_copy_fences(&bo->base._resv, bo->base.resv);
 303         dma_resv_unlock(&bo->base._resv);
 304         if (r)
 305                 return r;
 306
 307         if (bo->type != ttm_bo_type_sg) {
 308                 /* This works because the BO is about to be destroyed and nobody
 309                  * reference it any more. The only tricky case is the trylock on
 310                  * the resv object while holding the lru_lock.
 311                  */
 312                 spin_lock(&ttm_bo_glob.lru_lock);
 313                 bo->base.resv = &bo->base._resv;
 314                 spin_unlock(&ttm_bo_glob.lru_lock);
 315         }
 316
 317         return r;
 318 }
 319
 320 static void ttm_bo_flush_all_fences(struct ttm_buffer_object *bo)
 321 {
 322         struct dma_resv *resv = &bo->base._resv;
 323         struct dma_resv_list *fobj;
 324         struct dma_fence *fence;
 325         int i;
 326
 327         rcu_read_lock();
 328         fobj = rcu_dereference(resv->fence);
 329         fence = rcu_dereference(resv->fence_excl);
 330         if (fence && !fence->ops->signaled)
 331                 dma_fence_enable_sw_signaling(fence);
 332
 333         for (i = 0; fobj && i < fobj->shared_count; ++i) {
 334                 fence = rcu_dereference(fobj->shared[i]);
 335
 336                 if (!fence->ops->signaled)
 337                         dma_fence_enable_sw_signaling(fence);
 338         }
 339         rcu_read_unlock();
 340 }
 341
 342 /**
 343  * function ttm_bo_cleanup_refs
 344  * If bo idle, remove from lru lists, and unref.
 345  * If not idle, block if possible.
 346  *
 347  * Must be called with lru_lock and reservation held, this function
 348  * will drop the lru lock and optionally the reservation lock before returning.
 349  *
 350  * @bo:                    The buffer object to clean-up
 351  * @interruptible:         Any sleeps should occur interruptibly.
 352  * @no_wait_gpu:           Never wait for gpu. Return -EBUSY instead.
 353  * @unlock_resv:           Unlock the reservation lock as well.
 354  */
 355
 356 static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo,
 357                                bool interruptible, bool no_wait_gpu,
 358                                bool unlock_resv)
 359 {
 360         struct dma_resv *resv = &bo->base._resv;
 361         int ret;
 362
 363         if (dma_resv_test_signaled_rcu(resv, true))
 364                 ret = 0;
 365         else
 366                 ret = -EBUSY;
 367
 368         if (ret && !no_wait_gpu) {
 369                 long lret;
 370
 371                 if (unlock_resv)
 372                         dma_resv_unlock(bo->base.resv);
 373                 spin_unlock(&ttm_bo_glob.lru_lock);
 374
 375                 lret = dma_resv_wait_timeout_rcu(resv, true, interruptible,
 376                                                  30 * HZ);
 377
 378                 if (lret < 0)
 379                         return lret;
 380                 else if (lret == 0)
 381                         return -EBUSY;
 382
 383                 spin_lock(&ttm_bo_glob.lru_lock);
 384                 if (unlock_resv && !dma_resv_trylock(bo->base.resv)) {
 385                         /*
 386                          * We raced, and lost, someone else holds the reservation now,
 387                          * and is probably busy in ttm_bo_cleanup_memtype_use.
 388                          *
 389                          * Even if it's not the case, because we finished waiting any
 390                          * delayed destruction would succeed, so just return success
 391                          * here.
 392                          */
 393                         spin_unlock(&ttm_bo_glob.lru_lock);
 394                         return 0;
 395                 }
 396                 ret = 0;
 397         }
 398
 399         if (ret || unlikely(list_empty(&bo->ddestroy))) {
 400                 if (unlock_resv)
 401                         dma_resv_unlock(bo->base.resv);
 402                 spin_unlock(&ttm_bo_glob.lru_lock);
 403                 return ret;
 404         }
 405
 406         ttm_bo_del_from_lru(bo);
 407         list_del_init(&bo->ddestroy);
 408         spin_unlock(&ttm_bo_glob.lru_lock);
 409         ttm_bo_cleanup_memtype_use(bo);
 410
 411         if (unlock_resv)
 412                 dma_resv_unlock(bo->base.resv);
 413
 414         ttm_bo_put(bo);
 415
 416         return 0;
 417 }
 418
 419 /*
 420  * Traverse the delayed list, and call ttm_bo_cleanup_refs on all
 421  * encountered buffers.
 422  */
 423 static bool ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all)
 424 {
 425         struct ttm_bo_global *glob = &ttm_bo_glob;
 426         struct list_head removed;
 427         bool empty;
 428
 429         INIT_LIST_HEAD(&removed);
 430
 431         spin_lock(&glob->lru_lock);
 432         while (!list_empty(&bdev->ddestroy)) {
 433                 struct ttm_buffer_object *bo;
 434
 435                 bo = list_first_entry(&bdev->ddestroy, struct ttm_buffer_object,
 436                                       ddestroy);
 437                 list_move_tail(&bo->ddestroy, &removed);
 438                 if (!ttm_bo_get_unless_zero(bo))
 439                         continue;
 440
 441                 if (remove_all || bo->base.resv != &bo->base._resv) {
 442                         spin_unlock(&glob->lru_lock);
 443                         dma_resv_lock(bo->base.resv, NULL);
 444
 445                         spin_lock(&glob->lru_lock);
 446                         ttm_bo_cleanup_refs(bo, false, !remove_all, true);
 447
 448                 } else if (dma_resv_trylock(bo->base.resv)) {
 449                         ttm_bo_cleanup_refs(bo, false, !remove_all, true);
 450                 } else {
 451                         spin_unlock(&glob->lru_lock);
 452                 }
 453
 454                 ttm_bo_put(bo);
 455                 spin_lock(&glob->lru_lock);
 456         }
 457         list_splice_tail(&removed, &bdev->ddestroy);
 458         empty = list_empty(&bdev->ddestroy);
 459         spin_unlock(&glob->lru_lock);
 460
 461         return empty;
 462 }
 463
 464 static void ttm_bo_delayed_workqueue(struct work_struct *work)
 465 {
 466         struct ttm_bo_device *bdev =
 467             container_of(work, struct ttm_bo_device, wq.work);
 468
 469         if (!ttm_bo_delayed_delete(bdev, false))
 470                 schedule_delayed_work(&bdev->wq,
 471                                       ((HZ / 100) < 1) ? 1 : HZ / 100);
 472 }
 473
 474 static void ttm_bo_release(struct kref *kref)
 475 {
 476         struct ttm_buffer_object *bo =
 477             container_of(kref, struct ttm_buffer_object, kref);
 478         struct ttm_bo_device *bdev = bo->bdev;
 479         size_t acc_size = bo->acc_size;
 480         int ret;
 481
 482         if (!bo->deleted) {
 483                 ret = ttm_bo_individualize_resv(bo);
 484                 if (ret) {
 485                         /* Last resort, if we fail to allocate memory for the
 486                          * fences block for the BO to become idle
 487                          */
 488                         dma_resv_wait_timeout_rcu(bo->base.resv, true, false,
 489                                                   30 * HZ);
 490                 }
 491
 492                 if (bo->bdev->driver->release_notify)
 493                         bo->bdev->driver->release_notify(bo);
 494
 495                 drm_vma_offset_remove(bdev->vma_manager, &bo->base.vma_node);
 496                 ttm_mem_io_free(bdev, &bo->mem);
 497         }
 498
 499         if (!dma_resv_test_signaled_rcu(bo->base.resv, true) ||
 500             !dma_resv_trylock(bo->base.resv)) {
 501                 /* The BO is not idle, resurrect it for delayed destroy */
 502                 ttm_bo_flush_all_fences(bo);
 503                 bo->deleted = true;
 504
 505                 spin_lock(&ttm_bo_glob.lru_lock);
 506
 507                 /*
 508                  * Make pinned bos immediately available to
 509                  * shrinkers, now that they are queued for
 510                  * destruction.
 511                  */
 512                 if (WARN_ON(bo->pin_count)) {
 513                         bo->pin_count = 0;
 514                         ttm_bo_move_to_lru_tail(bo, &bo->mem, NULL);
 515                 }
 516
 517                 kref_init(&bo->kref);
 518                 list_add_tail(&bo->ddestroy, &bdev->ddestroy);
 519                 spin_unlock(&ttm_bo_glob.lru_lock);
 520
 521                 schedule_delayed_work(&bdev->wq,
 522                                       ((HZ / 100) < 1) ? 1 : HZ / 100);
 523                 return;
 524         }
 525
 526         spin_lock(&ttm_bo_glob.lru_lock);
 527         ttm_bo_del_from_lru(bo);
 528         list_del(&bo->ddestroy);
 529         spin_unlock(&ttm_bo_glob.lru_lock);
 530
 531         ttm_bo_cleanup_memtype_use(bo);
 532         dma_resv_unlock(bo->base.resv);
 533
 534         atomic_dec(&ttm_bo_glob.bo_count);
 535         dma_fence_put(bo->moving);
 536         if (!ttm_bo_uses_embedded_gem_object(bo))
 537                 dma_resv_fini(&bo->base._resv);
 538         bo->destroy(bo);
 539         ttm_mem_global_free(&ttm_mem_glob, acc_size);
 540 }
 541
 542 void ttm_bo_put(struct ttm_buffer_object *bo)
 543 {
 544         kref_put(&bo->kref, ttm_bo_release);
 545 }
 546 EXPORT_SYMBOL(ttm_bo_put);
 547
 548 int ttm_bo_lock_delayed_workqueue(struct ttm_bo_device *bdev)
 549 {
 550         return cancel_delayed_work_sync(&bdev->wq);
 551 }
 552 EXPORT_SYMBOL(ttm_bo_lock_delayed_workqueue);
 553
 554 void ttm_bo_unlock_delayed_workqueue(struct ttm_bo_device *bdev, int resched)
 555 {
 556         if (resched)
 557                 schedule_delayed_work(&bdev->wq,
 558                                       ((HZ / 100) < 1) ? 1 : HZ / 100);
 559 }
 560 EXPORT_SYMBOL(ttm_bo_unlock_delayed_workqueue);
 561
 562 static int ttm_bo_evict(struct ttm_buffer_object *bo,
 563                         struct ttm_operation_ctx *ctx)
 564 {
 565         struct ttm_bo_device *bdev = bo->bdev;
 566         struct ttm_resource evict_mem;
 567         struct ttm_placement placement;
 568         struct ttm_place hop;
 569         int ret = 0;
 570
 571         memset(&hop, 0, sizeof(hop));
 572
 573         dma_resv_assert_held(bo->base.resv);
 574
 575         placement.num_placement = 0;
 576         placement.num_busy_placement = 0;
 577         bdev->driver->evict_flags(bo, &placement);
 578
 579         if (!placement.num_placement && !placement.num_busy_placement) {
 580                 ttm_bo_wait(bo, false, false);
 581
 582                 ttm_bo_cleanup_memtype_use(bo);
 583                 return ttm_tt_create(bo, false);
 584         }
 585
 586         evict_mem = bo->mem;
 587         evict_mem.mm_node = NULL;
 588         evict_mem.bus.offset = 0;
 589         evict_mem.bus.addr = NULL;
 590
 591         ret = ttm_bo_mem_space(bo, &placement, &evict_mem, ctx);
 592         if (ret) {
 593                 if (ret != -ERESTARTSYS) {
 594                         pr_err("Failed to find memory space for buffer 0x%p eviction\n",
 595                                bo);
 596                         ttm_bo_mem_space_debug(bo, &placement);
 597                 }
 598                 goto out;
 599         }
 600
 601         ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, ctx, &hop);
 602         if (unlikely(ret)) {
 603                 WARN(ret == -EMULTIHOP, "Unexpected multihop in eviction - likely driver bug\n");
 604                 if (ret != -ERESTARTSYS)
 605                         pr_err("Buffer eviction failed\n");
 606                 ttm_resource_free(bo, &evict_mem);
 607         }
 608 out:
 609         return ret;
 610 }
 611
 612 bool ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
 613                               const struct ttm_place *place)
 614 {
 615         /* Don't evict this BO if it's outside of the
 616          * requested placement range
 617          */
 618         if (place->fpfn >= (bo->mem.start + bo->mem.num_pages) ||
 619             (place->lpfn && place->lpfn <= bo->mem.start))
 620                 return false;
 621
 622         return true;
 623 }
 624 EXPORT_SYMBOL(ttm_bo_eviction_valuable);
 625
 626 /*
 627  * Check the target bo is allowable to be evicted or swapout, including cases:
 628  *
 629  * a. if share same reservation object with ctx->resv, have assumption
 630  * reservation objects should already be locked, so not lock again and
 631  * return true directly when either the opreation allow_reserved_eviction
 632  * or the target bo already is in delayed free list;
 633  *
 634  * b. Otherwise, trylock it.
 635  */
 636 static bool ttm_bo_evict_swapout_allowable(struct ttm_buffer_object *bo,
 637                         struct ttm_operation_ctx *ctx, bool *locked, bool *busy)
 638 {
 639         bool ret = false;
 640
 641         if (bo->base.resv == ctx->resv) {
 642                 dma_resv_assert_held(bo->base.resv);
 643                 if (ctx->allow_res_evict)
 644                         ret = true;
 645                 *locked = false;
 646                 if (busy)
 647                         *busy = false;
 648         } else {
 649                 ret = dma_resv_trylock(bo->base.resv);
 650                 *locked = ret;
 651                 if (busy)
 652                         *busy = !ret;
 653         }
 654
 655         return ret;
 656 }
 657
 658 /**
 659  * ttm_mem_evict_wait_busy - wait for a busy BO to become available
 660  *
 661  * @busy_bo: BO which couldn't be locked with trylock
 662  * @ctx: operation context
 663  * @ticket: acquire ticket
 664  *
 665  * Try to lock a busy buffer object to avoid failing eviction.
 666  */
 667 static int ttm_mem_evict_wait_busy(struct ttm_buffer_object *busy_bo,
 668                                    struct ttm_operation_ctx *ctx,
 669                                    struct ww_acquire_ctx *ticket)
 670 {
 671         int r;
 672
 673         if (!busy_bo || !ticket)
 674                 return -EBUSY;
 675
 676         if (ctx->interruptible)
 677                 r = dma_resv_lock_interruptible(busy_bo->base.resv,
 678                                                           ticket);
 679         else
 680                 r = dma_resv_lock(busy_bo->base.resv, ticket);
 681
 682         /*
 683          * TODO: It would be better to keep the BO locked until allocation is at
 684          * least tried one more time, but that would mean a much larger rework
 685          * of TTM.
 686          */
 687         if (!r)
 688                 dma_resv_unlock(busy_bo->base.resv);
 689
 690         return r == -EDEADLK ? -EBUSY : r;
 691 }
 692
 693 int ttm_mem_evict_first(struct ttm_bo_device *bdev,
 694                         struct ttm_resource_manager *man,
 695                         const struct ttm_place *place,
 696                         struct ttm_operation_ctx *ctx,
 697                         struct ww_acquire_ctx *ticket)
 698 {
 699         struct ttm_buffer_object *bo = NULL, *busy_bo = NULL;
 700         bool locked = false;
 701         unsigned i;
 702         int ret;
 703
 704         spin_lock(&ttm_bo_glob.lru_lock);
 705         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
 706                 list_for_each_entry(bo, &man->lru[i], lru) {
 707                         bool busy;
 708
 709                         if (!ttm_bo_evict_swapout_allowable(bo, ctx, &locked,
 710                                                             &busy)) {
 711                                 if (busy && !busy_bo && ticket !=
 712                                     dma_resv_locking_ctx(bo->base.resv))
 713                                         busy_bo = bo;
 714                                 continue;
 715                         }
 716
 717                         if (place && !bdev->driver->eviction_valuable(bo,
 718                                                                       place)) {
 719                                 if (locked)
 720                                         dma_resv_unlock(bo->base.resv);
 721                                 continue;
 722                         }
 723                         if (!ttm_bo_get_unless_zero(bo)) {
 724                                 if (locked)
 725                                         dma_resv_unlock(bo->base.resv);
 726                                 continue;
 727                         }
 728                         break;
 729                 }
 730
 731                 /* If the inner loop terminated early, we have our candidate */
 732                 if (&bo->lru != &man->lru[i])
 733                         break;
 734
 735                 bo = NULL;
 736         }
 737
 738         if (!bo) {
 739                 if (busy_bo && !ttm_bo_get_unless_zero(busy_bo))
 740                         busy_bo = NULL;
 741                 spin_unlock(&ttm_bo_glob.lru_lock);
 742                 ret = ttm_mem_evict_wait_busy(busy_bo, ctx, ticket);
 743                 if (busy_bo)
 744                         ttm_bo_put(busy_bo);
 745                 return ret;
 746         }
 747
 748         if (bo->deleted) {
 749                 ret = ttm_bo_cleanup_refs(bo, ctx->interruptible,
 750                                           ctx->no_wait_gpu, locked);
 751                 ttm_bo_put(bo);
 752                 return ret;
 753         }
 754
 755         spin_unlock(&ttm_bo_glob.lru_lock);
 756
 757         ret = ttm_bo_evict(bo, ctx);
 758         if (locked)
 759                 ttm_bo_unreserve(bo);
 760
 761         ttm_bo_put(bo);
 762         return ret;
 763 }
 764
 765 /*
 766  * Add the last move fence to the BO and reserve a new shared slot.
 767  */
 768 static int ttm_bo_add_move_fence(struct ttm_buffer_object *bo,
 769                                  struct ttm_resource_manager *man,
 770                                  struct ttm_resource *mem,
 771                                  bool no_wait_gpu)
 772 {
 773         struct dma_fence *fence;
 774         int ret;
 775
 776         spin_lock(&man->move_lock);
 777         fence = dma_fence_get(man->move);
 778         spin_unlock(&man->move_lock);
 779
 780         if (!fence)
 781                 return 0;
 782
 783         if (no_wait_gpu) {
 784                 dma_fence_put(fence);
 785                 return -EBUSY;
 786         }
 787
 788         dma_resv_add_shared_fence(bo->base.resv, fence);
 789
 790         ret = dma_resv_reserve_shared(bo->base.resv, 1);
 791         if (unlikely(ret)) {
 792                 dma_fence_put(fence);
 793                 return ret;
 794         }
 795
 796         dma_fence_put(bo->moving);
 797         bo->moving = fence;
 798         return 0;
 799 }
 800
 801 /*
 802  * Repeatedly evict memory from the LRU for @mem_type until we create enough
 803  * space, or we've evicted everything and there isn't enough space.
 804  */
 805 static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo,
 806                                   const struct ttm_place *place,
 807                                   struct ttm_resource *mem,
 808                                   struct ttm_operation_ctx *ctx)
 809 {
 810         struct ttm_bo_device *bdev = bo->bdev;
 811         struct ttm_resource_manager *man = ttm_manager_type(bdev, mem->mem_type);
 812         struct ww_acquire_ctx *ticket;
 813         int ret;
 814
 815         ticket = dma_resv_locking_ctx(bo->base.resv);
 816         do {
 817                 ret = ttm_resource_alloc(bo, place, mem);
 818                 if (likely(!ret))
 819                         break;
 820                 if (unlikely(ret != -ENOSPC))
 821                         return ret;
 822                 ret = ttm_mem_evict_first(bdev, man, place, ctx,
 823                                           ticket);
 824                 if (unlikely(ret != 0))
 825                         return ret;
 826         } while (1);
 827
 828         return ttm_bo_add_move_fence(bo, man, mem, ctx->no_wait_gpu);
 829 }
 830
 831 /**
 832  * ttm_bo_mem_placement - check if placement is compatible
 833  * @bo: BO to find memory for
 834  * @place: where to search
 835  * @mem: the memory object to fill in
 836  *
 837  * Check if placement is compatible and fill in mem structure.
 838  * Returns -EBUSY if placement won't work or negative error code.
 839  * 0 when placement can be used.
 840  */
 841 static int ttm_bo_mem_placement(struct ttm_buffer_object *bo,
 842                                 const struct ttm_place *place,
 843                                 struct ttm_resource *mem)
 844 {
 845         struct ttm_bo_device *bdev = bo->bdev;
 846         struct ttm_resource_manager *man;
 847
 848         man = ttm_manager_type(bdev, place->mem_type);
 849         if (!man || !ttm_resource_manager_used(man))
 850                 return -EBUSY;
 851
 852         mem->mem_type = place->mem_type;
 853         mem->placement = place->flags;
 854
 855         spin_lock(&ttm_bo_glob.lru_lock);
 856         ttm_bo_move_to_lru_tail(bo, mem, NULL);
 857         spin_unlock(&ttm_bo_glob.lru_lock);
 858
 859         return 0;
 860 }
 861
 862 /*
 863  * Creates space for memory region @mem according to its type.
 864  *
 865  * This function first searches for free space in compatible memory types in
 866  * the priority order defined by the driver.  If free space isn't found, then
 867  * ttm_bo_mem_force_space is attempted in priority order to evict and find
 868  * space.
 869  */
 870 int ttm_bo_mem_space(struct ttm_buffer_object *bo,
 871                         struct ttm_placement *placement,
 872                         struct ttm_resource *mem,
 873                         struct ttm_operation_ctx *ctx)
 874 {
 875         struct ttm_bo_device *bdev = bo->bdev;
 876         bool type_found = false;
 877         int i, ret;
 878
 879         ret = dma_resv_reserve_shared(bo->base.resv, 1);
 880         if (unlikely(ret))
 881                 return ret;
 882
 883         for (i = 0; i < placement->num_placement; ++i) {
 884                 const struct ttm_place *place = &placement->placement[i];
 885                 struct ttm_resource_manager *man;
 886
 887                 ret = ttm_bo_mem_placement(bo, place, mem);
 888                 if (ret)
 889                         continue;
 890
 891                 type_found = true;
 892                 ret = ttm_resource_alloc(bo, place, mem);
 893                 if (ret == -ENOSPC)
 894                         continue;
 895                 if (unlikely(ret))
 896                         goto error;
 897
 898                 man = ttm_manager_type(bdev, mem->mem_type);
 899                 ret = ttm_bo_add_move_fence(bo, man, mem, ctx->no_wait_gpu);
 900                 if (unlikely(ret)) {
 901                         ttm_resource_free(bo, mem);
 902                         if (ret == -EBUSY)
 903                                 continue;
 904
 905                         goto error;
 906                 }
 907                 return 0;
 908         }
 909
 910         for (i = 0; i < placement->num_busy_placement; ++i) {
 911                 const struct ttm_place *place = &placement->busy_placement[i];
 912
 913                 ret = ttm_bo_mem_placement(bo, place, mem);
 914                 if (ret)
 915                         continue;
 916
 917                 type_found = true;
 918                 ret = ttm_bo_mem_force_space(bo, place, mem, ctx);
 919                 if (likely(!ret))
 920                         return 0;
 921
 922                 if (ret && ret != -EBUSY)
 923                         goto error;
 924         }
 925
 926         ret = -ENOMEM;
 927         if (!type_found) {
 928                 pr_err(TTM_PFX "No compatible memory type found\n");
 929                 ret = -EINVAL;
 930         }
 931
 932 error:
 933         if (bo->mem.mem_type == TTM_PL_SYSTEM && !bo->pin_count)
 934                 ttm_bo_move_to_lru_tail_unlocked(bo);
 935
 936         return ret;
 937 }
 938 EXPORT_SYMBOL(ttm_bo_mem_space);
 939
 940 static int ttm_bo_bounce_temp_buffer(struct ttm_buffer_object *bo,
 941                                      struct ttm_resource *mem,
 942                                      struct ttm_operation_ctx *ctx,
 943                                      struct ttm_place *hop)
 944 {
 945         struct ttm_placement hop_placement;
 946         int ret;
 947         struct ttm_resource hop_mem = *mem;
 948
 949         hop_mem.mm_node = NULL;
 950         hop_mem.mem_type = TTM_PL_SYSTEM;
 951         hop_mem.placement = 0;
 952
 953         hop_placement.num_placement = hop_placement.num_busy_placement = 1;
 954         hop_placement.placement = hop_placement.busy_placement = hop;
 955
 956         /* find space in the bounce domain */
 957         ret = ttm_bo_mem_space(bo, &hop_placement, &hop_mem, ctx);
 958         if (ret)
 959                 return ret;
 960         /* move to the bounce domain */
 961         ret = ttm_bo_handle_move_mem(bo, &hop_mem, false, ctx, NULL);
 962         if (ret) {
 963                 ttm_resource_free(bo, &hop_mem);
 964                 return ret;
 965         }
 966         return 0;
 967 }
 968
 969 static int ttm_bo_move_buffer(struct ttm_buffer_object *bo,
 970                               struct ttm_placement *placement,
 971                               struct ttm_operation_ctx *ctx)
 972 {
 973         int ret = 0;
 974         struct ttm_place hop;
 975         struct ttm_resource mem;
 976
 977         dma_resv_assert_held(bo->base.resv);
 978
 979         memset(&hop, 0, sizeof(hop));
 980
 981         mem.num_pages = PAGE_ALIGN(bo->base.size) >> PAGE_SHIFT;
 982         mem.page_alignment = bo->mem.page_alignment;
 983         mem.bus.offset = 0;
 984         mem.bus.addr = NULL;
 985         mem.mm_node = NULL;
 986
 987         /*
 988          * Determine where to move the buffer.
 989          *
 990          * If driver determines move is going to need
 991          * an extra step then it will return -EMULTIHOP
 992          * and the buffer will be moved to the temporary
 993          * stop and the driver will be called to make
 994          * the second hop.
 995          */
 996         ret = ttm_bo_mem_space(bo, placement, &mem, ctx);
 997         if (ret)
 998                 return ret;
 999 bounce:
1000         ret = ttm_bo_handle_move_mem(bo, &mem, false, ctx, &hop);
1001         if (ret == -EMULTIHOP) {
1002                 ret = ttm_bo_bounce_temp_buffer(bo, &mem, ctx, &hop);
1003                 if (ret)
1004                         goto out;
1005                 /* try and move to final place now. */
1006                 goto bounce;
1007         }
1008 out:
1009         if (ret)
1010                 ttm_resource_free(bo, &mem);
1011         return ret;
1012 }
1013
1014 static bool ttm_bo_places_compat(const struct ttm_place *places,
1015                                  unsigned num_placement,
1016                                  struct ttm_resource *mem,
1017                                  uint32_t *new_flags)
1018 {
1019         unsigned i;
1020
1021         for (i = 0; i < num_placement; i++) {
1022                 const struct ttm_place *heap = &places[i];
1023
1024                 if ((mem->start < heap->fpfn ||
1025                      (heap->lpfn != 0 && (mem->start + mem->num_pages) > heap->lpfn)))
1026                         continue;
1027
1028                 *new_flags = heap->flags;
1029                 if ((mem->mem_type == heap->mem_type) &&
1030                     (!(*new_flags & TTM_PL_FLAG_CONTIGUOUS) ||
1031                      (mem->placement & TTM_PL_FLAG_CONTIGUOUS)))
1032                         return true;
1033         }
1034         return false;
1035 }
1036
1037 bool ttm_bo_mem_compat(struct ttm_placement *placement,
1038                        struct ttm_resource *mem,
1039                        uint32_t *new_flags)
1040 {
1041         if (ttm_bo_places_compat(placement->placement, placement->num_placement,
1042                                  mem, new_flags))
1043                 return true;
1044
1045         if ((placement->busy_placement != placement->placement ||
1046              placement->num_busy_placement > placement->num_placement) &&
1047             ttm_bo_places_compat(placement->busy_placement,
1048                                  placement->num_busy_placement,
1049                                  mem, new_flags))
1050                 return true;
1051
1052         return false;
1053 }
1054 EXPORT_SYMBOL(ttm_bo_mem_compat);
1055
1056 int ttm_bo_validate(struct ttm_buffer_object *bo,
1057                     struct ttm_placement *placement,
1058                     struct ttm_operation_ctx *ctx)
1059 {
1060         int ret;
1061         uint32_t new_flags;
1062
1063         dma_resv_assert_held(bo->base.resv);
1064
1065         /*
1066          * Remove the backing store if no placement is given.
1067          */
1068         if (!placement->num_placement && !placement->num_busy_placement) {
1069                 ret = ttm_bo_pipeline_gutting(bo);
1070                 if (ret)
1071                         return ret;
1072
1073                 return ttm_tt_create(bo, false);
1074         }
1075
1076         /*
1077          * Check whether we need to move buffer.
1078          */
1079         if (!ttm_bo_mem_compat(placement, &bo->mem, &new_flags)) {
1080                 ret = ttm_bo_move_buffer(bo, placement, ctx);
1081                 if (ret)
1082                         return ret;
1083         }
1084         /*
1085          * We might need to add a TTM.
1086          */
1087         if (bo->mem.mem_type == TTM_PL_SYSTEM) {
1088                 ret = ttm_tt_create(bo, true);
1089                 if (ret)
1090                         return ret;
1091         }
1092         return 0;
1093 }
1094 EXPORT_SYMBOL(ttm_bo_validate);
1095
1096 int ttm_bo_init_reserved(struct ttm_bo_device *bdev,
1097                          struct ttm_buffer_object *bo,
1098                          size_t size,
1099                          enum ttm_bo_type type,
1100                          struct ttm_placement *placement,
1101                          uint32_t page_alignment,
1102                          struct ttm_operation_ctx *ctx,
1103                          size_t acc_size,
1104                          struct sg_table *sg,
1105                          struct dma_resv *resv,
1106                          void (*destroy) (struct ttm_buffer_object *))
1107 {
1108         struct ttm_mem_global *mem_glob = &ttm_mem_glob;
1109         bool locked;
1110         int ret = 0;
1111
1112         ret = ttm_mem_global_alloc(mem_glob, acc_size, ctx);
1113         if (ret) {
1114                 pr_err("Out of kernel memory\n");
1115                 if (destroy)
1116                         (*destroy)(bo);
1117                 else
1118                         kfree(bo);
1119                 return -ENOMEM;
1120         }
1121
1122         bo->destroy = destroy ? destroy : ttm_bo_default_destroy;
1123
1124         kref_init(&bo->kref);
1125         INIT_LIST_HEAD(&bo->lru);
1126         INIT_LIST_HEAD(&bo->ddestroy);
1127         INIT_LIST_HEAD(&bo->swap);
1128         bo->bdev = bdev;
1129         bo->type = type;
1130         bo->mem.mem_type = TTM_PL_SYSTEM;
1131         bo->mem.num_pages = PAGE_ALIGN(size) >> PAGE_SHIFT;
1132         bo->mem.mm_node = NULL;
1133         bo->mem.page_alignment = page_alignment;
1134         bo->mem.bus.offset = 0;
1135         bo->mem.bus.addr = NULL;
1136         bo->moving = NULL;
1137         bo->mem.placement = 0;
1138         bo->acc_size = acc_size;
1139         bo->pin_count = 0;
1140         bo->sg = sg;
1141         if (resv) {
1142                 bo->base.resv = resv;
1143                 dma_resv_assert_held(bo->base.resv);
1144         } else {
1145                 bo->base.resv = &bo->base._resv;
1146         }
1147         if (!ttm_bo_uses_embedded_gem_object(bo)) {
1148                 /*
1149                  * bo.base is not initialized, so we have to setup the
1150                  * struct elements we want use regardless.
1151                  */
1152                 bo->base.size = size;
1153                 dma_resv_init(&bo->base._resv);
1154                 drm_vma_node_reset(&bo->base.vma_node);
1155         }
1156         atomic_inc(&ttm_bo_glob.bo_count);
1157
1158         /*
1159          * For ttm_bo_type_device buffers, allocate
1160          * address space from the device.
1161          */
1162         if (bo->type == ttm_bo_type_device ||
1163             bo->type == ttm_bo_type_sg)
1164                 ret = drm_vma_offset_add(bdev->vma_manager, &bo->base.vma_node,
1165                                          bo->mem.num_pages);
1166
1167         /* passed reservation objects should already be locked,
1168          * since otherwise lockdep will be angered in radeon.
1169          */
1170         if (!resv) {
1171                 locked = dma_resv_trylock(bo->base.resv);
1172                 WARN_ON(!locked);
1173         }
1174
1175         if (likely(!ret))
1176                 ret = ttm_bo_validate(bo, placement, ctx);
1177
1178         if (unlikely(ret)) {
1179                 if (!resv)
1180                         ttm_bo_unreserve(bo);
1181
1182                 ttm_bo_put(bo);
1183                 return ret;
1184         }
1185
1186         ttm_bo_move_to_lru_tail_unlocked(bo);
1187
1188         return ret;
1189 }
1190 EXPORT_SYMBOL(ttm_bo_init_reserved);
1191
1192 int ttm_bo_init(struct ttm_bo_device *bdev,
1193                 struct ttm_buffer_object *bo,
1194                 size_t size,
1195                 enum ttm_bo_type type,
1196                 struct ttm_placement *placement,
1197                 uint32_t page_alignment,
1198                 bool interruptible,
1199                 size_t acc_size,
1200                 struct sg_table *sg,
1201                 struct dma_resv *resv,
1202                 void (*destroy) (struct ttm_buffer_object *))
1203 {
1204         struct ttm_operation_ctx ctx = { interruptible, false };
1205         int ret;
1206
1207         ret = ttm_bo_init_reserved(bdev, bo, size, type, placement,
1208                                    page_alignment, &ctx, acc_size,
1209                                    sg, resv, destroy);
1210         if (ret)
1211                 return ret;
1212
1213         if (!resv)
1214                 ttm_bo_unreserve(bo);
1215
1216         return 0;
1217 }
1218 EXPORT_SYMBOL(ttm_bo_init);
1219
1220 size_t ttm_bo_dma_acc_size(struct ttm_bo_device *bdev,
1221                            unsigned long bo_size,
1222                            unsigned struct_size)
1223 {
1224         unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT;
1225         size_t size = 0;
1226
1227         size += ttm_round_pot(struct_size);
1228         size += ttm_round_pot(npages * (2*sizeof(void *) + sizeof(dma_addr_t)));
1229         size += ttm_round_pot(sizeof(struct ttm_tt));
1230         return size;
1231 }
1232 EXPORT_SYMBOL(ttm_bo_dma_acc_size);
1233
1234 static void ttm_bo_global_kobj_release(struct kobject *kobj)
1235 {
1236         struct ttm_bo_global *glob =
1237                 container_of(kobj, struct ttm_bo_global, kobj);
1238
1239         __free_page(glob->dummy_read_page);
1240 }
1241
1242 static void ttm_bo_global_release(void)
1243 {
1244         struct ttm_bo_global *glob = &ttm_bo_glob;
1245
1246         mutex_lock(&ttm_global_mutex);
1247         if (--ttm_bo_glob_use_count > 0)
1248                 goto out;
1249
1250         kobject_del(&glob->kobj);
1251         kobject_put(&glob->kobj);
1252         ttm_mem_global_release(&ttm_mem_glob);
1253         memset(glob, 0, sizeof(*glob));
1254 out:
1255         mutex_unlock(&ttm_global_mutex);
1256 }
1257
1258 static int ttm_bo_global_init(void)
1259 {
1260         struct ttm_bo_global *glob = &ttm_bo_glob;
1261         int ret = 0;
1262         unsigned i;
1263
1264         mutex_lock(&ttm_global_mutex);
1265         if (++ttm_bo_glob_use_count > 1)
1266                 goto out;
1267
1268         ret = ttm_mem_global_init(&ttm_mem_glob);
1269         if (ret)
1270                 goto out;
1271
1272         spin_lock_init(&glob->lru_lock);
1273         glob->dummy_read_page = alloc_page(__GFP_ZERO | GFP_DMA32);
1274
1275         if (unlikely(glob->dummy_read_page == NULL)) {
1276                 ret = -ENOMEM;
1277                 goto out;
1278         }
1279
1280         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i)
1281                 INIT_LIST_HEAD(&glob->swap_lru[i]);
1282         INIT_LIST_HEAD(&glob->device_list);
1283         atomic_set(&glob->bo_count, 0);
1284
1285         ret = kobject_init_and_add(
1286                 &glob->kobj, &ttm_bo_glob_kobj_type, ttm_get_kobj(), "buffer_objects");
1287         if (unlikely(ret != 0))
1288                 kobject_put(&glob->kobj);
1289 out:
1290         mutex_unlock(&ttm_global_mutex);
1291         return ret;
1292 }
1293
1294 int ttm_bo_device_release(struct ttm_bo_device *bdev)
1295 {
1296         struct ttm_bo_global *glob = &ttm_bo_glob;
1297         int ret = 0;
1298         unsigned i;
1299         struct ttm_resource_manager *man;
1300
1301         man = ttm_manager_type(bdev, TTM_PL_SYSTEM);
1302         ttm_resource_manager_set_used(man, false);
1303         ttm_set_driver_manager(bdev, TTM_PL_SYSTEM, NULL);
1304
1305         mutex_lock(&ttm_global_mutex);
1306         list_del(&bdev->device_list);
1307         mutex_unlock(&ttm_global_mutex);
1308
1309         cancel_delayed_work_sync(&bdev->wq);
1310
1311         if (ttm_bo_delayed_delete(bdev, true))
1312                 pr_debug("Delayed destroy list was clean\n");
1313
1314         spin_lock(&glob->lru_lock);
1315         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i)
1316                 if (list_empty(&man->lru[0]))
1317                         pr_debug("Swap list %d was clean\n", i);
1318         spin_unlock(&glob->lru_lock);
1319
1320         ttm_pool_fini(&bdev->pool);
1321
1322         if (!ret)
1323                 ttm_bo_global_release();
1324
1325         return ret;
1326 }
1327 EXPORT_SYMBOL(ttm_bo_device_release);
1328
1329 static void ttm_bo_init_sysman(struct ttm_bo_device *bdev)
1330 {
1331         struct ttm_resource_manager *man = &bdev->sysman;
1332
1333         /*
1334          * Initialize the system memory buffer type.
1335          * Other types need to be driver / IOCTL initialized.
1336          */
1337         man->use_tt = true;
1338
1339         ttm_resource_manager_init(man, 0);
1340         ttm_set_driver_manager(bdev, TTM_PL_SYSTEM, man);
1341         ttm_resource_manager_set_used(man, true);
1342 }
1343
1344 int ttm_bo_device_init(struct ttm_bo_device *bdev,
1345                        struct ttm_bo_driver *driver,
1346                        struct device *dev,
1347                        struct address_space *mapping,
1348                        struct drm_vma_offset_manager *vma_manager,
1349                        bool use_dma_alloc, bool use_dma32)
1350 {
1351         struct ttm_bo_global *glob = &ttm_bo_glob;
1352         int ret;
1353
1354         if (WARN_ON(vma_manager == NULL))
1355                 return -EINVAL;
1356
1357         ret = ttm_bo_global_init();
1358         if (ret)
1359                 return ret;
1360
1361         bdev->driver = driver;
1362
1363         ttm_bo_init_sysman(bdev);
1364         ttm_pool_init(&bdev->pool, dev, use_dma_alloc, use_dma32);
1365
1366         bdev->vma_manager = vma_manager;
1367         INIT_DELAYED_WORK(&bdev->wq, ttm_bo_delayed_workqueue);
1368         INIT_LIST_HEAD(&bdev->ddestroy);
1369         bdev->dev_mapping = mapping;
1370         mutex_lock(&ttm_global_mutex);
1371         list_add_tail(&bdev->device_list, &glob->device_list);
1372         mutex_unlock(&ttm_global_mutex);
1373
1374         return 0;
1375 }
1376 EXPORT_SYMBOL(ttm_bo_device_init);
1377
1378 /*
1379  * buffer object vm functions.
1380  */
1381
1382 void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo)
1383 {
1384         struct ttm_bo_device *bdev = bo->bdev;
1385
1386         drm_vma_node_unmap(&bo->base.vma_node, bdev->dev_mapping);
1387         ttm_mem_io_free(bdev, &bo->mem);
1388 }
1389 EXPORT_SYMBOL(ttm_bo_unmap_virtual);
1390
1391 int ttm_bo_wait(struct ttm_buffer_object *bo,
1392                 bool interruptible, bool no_wait)
1393 {
1394         long timeout = 15 * HZ;
1395
1396         if (no_wait) {
1397                 if (dma_resv_test_signaled_rcu(bo->base.resv, true))
1398                         return 0;
1399                 else
1400                         return -EBUSY;
1401         }
1402
1403         timeout = dma_resv_wait_timeout_rcu(bo->base.resv, true,
1404                                                       interruptible, timeout);
1405         if (timeout < 0)
1406                 return timeout;
1407
1408         if (timeout == 0)
1409                 return -EBUSY;
1410
1411         dma_resv_add_excl_fence(bo->base.resv, NULL);
1412         return 0;
1413 }
1414 EXPORT_SYMBOL(ttm_bo_wait);
1415
1416 /*
1417  * A buffer object shrink method that tries to swap out the first
1418  * buffer object on the bo_global::swap_lru list.
1419  */
1420 int ttm_bo_swapout(struct ttm_operation_ctx *ctx)
1421 {
1422         struct ttm_bo_global *glob = &ttm_bo_glob;
1423         struct ttm_buffer_object *bo;
1424         int ret = -EBUSY;
1425         bool locked;
1426         unsigned i;
1427
1428         spin_lock(&glob->lru_lock);
1429         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
1430                 list_for_each_entry(bo, &glob->swap_lru[i], swap) {
1431                         if (!ttm_bo_evict_swapout_allowable(bo, ctx, &locked,
1432                                                             NULL))
1433                                 continue;
1434
1435                         if (!ttm_bo_get_unless_zero(bo)) {
1436                                 if (locked)
1437                                         dma_resv_unlock(bo->base.resv);
1438                                 continue;
1439                         }
1440
1441                         ret = 0;
1442                         break;
1443                 }
1444                 if (!ret)
1445                         break;
1446         }
1447
1448         if (ret) {
1449                 spin_unlock(&glob->lru_lock);
1450                 return ret;
1451         }
1452
1453         if (bo->deleted) {
1454                 ret = ttm_bo_cleanup_refs(bo, false, false, locked);
1455                 ttm_bo_put(bo);
1456                 return ret;
1457         }
1458
1459         ttm_bo_del_from_lru(bo);
1460         spin_unlock(&glob->lru_lock);
1461
1462         /**
1463          * Move to system cached
1464          */
1465
1466         if (bo->mem.mem_type != TTM_PL_SYSTEM) {
1467                 struct ttm_operation_ctx ctx = { false, false };
1468                 struct ttm_resource evict_mem;
1469                 struct ttm_place hop;
1470
1471                 memset(&hop, 0, sizeof(hop));
1472
1473                 evict_mem = bo->mem;
1474                 evict_mem.mm_node = NULL;
1475                 evict_mem.placement = 0;
1476                 evict_mem.mem_type = TTM_PL_SYSTEM;
1477
1478                 ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, &ctx, &hop);
1479                 if (unlikely(ret != 0)) {
1480                         WARN(ret == -EMULTIHOP, "Unexpected multihop in swaput - likely driver bug.\n");
1481                         goto out;
1482                 }
1483         }
1484
1485         /**
1486          * Make sure BO is idle.
1487          */
1488
1489         ret = ttm_bo_wait(bo, false, false);
1490         if (unlikely(ret != 0))
1491                 goto out;
1492
1493         ttm_bo_unmap_virtual(bo);
1494
1495         /**
1496          * Swap out. Buffer will be swapped in again as soon as
1497          * anyone tries to access a ttm page.
1498          */
1499
1500         if (bo->bdev->driver->swap_notify)
1501                 bo->bdev->driver->swap_notify(bo);
1502
1503         ret = ttm_tt_swapout(bo->bdev, bo->ttm);
1504 out:
1505
1506         /**
1507          *
1508          * Unreserve without putting on LRU to avoid swapping out an
1509          * already swapped buffer.
1510          */
1511         if (locked)
1512                 dma_resv_unlock(bo->base.resv);
1513         ttm_bo_put(bo);
1514         return ret;
1515 }
1516 EXPORT_SYMBOL(ttm_bo_swapout);
1517
1518 void ttm_bo_tt_destroy(struct ttm_buffer_object *bo)
1519 {
1520         if (bo->ttm == NULL)
1521                 return;
1522
1523         ttm_tt_destroy(bo->bdev, bo->ttm);
1524         bo->ttm = NULL;
1525 }
1526