drivers/gpu/drm/ttm/ttm_bo.c

   1 /* SPDX-License-Identifier: GPL-2.0 OR MIT */
   2 /**************************************************************************
   3  *
   4  * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
   5  * All Rights Reserved.
   6  *
   7  * Permission is hereby granted, free of charge, to any person obtaining a
   8  * copy of this software and associated documentation files (the
   9  * "Software"), to deal in the Software without restriction, including
  10  * without limitation the rights to use, copy, modify, merge, publish,
  11  * distribute, sub license, and/or sell copies of the Software, and to
  12  * permit persons to whom the Software is furnished to do so, subject to
  13  * the following conditions:
  14  *
  15  * The above copyright notice and this permission notice (including the
  16  * next paragraph) shall be included in all copies or substantial portions
  17  * of the Software.
  18  *
  19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  20  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  21  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  22  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
  23  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  24  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  25  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  26  *
  27  **************************************************************************/
  28 /*
  29  * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
  30  */
  31
  32 #define pr_fmt(fmt) "[TTM] " fmt
  33
  34 #include <drm/ttm/ttm_module.h>
  35 #include <drm/ttm/ttm_bo_driver.h>
  36 #include <drm/ttm/ttm_placement.h>
  37 #include <linux/jiffies.h>
  38 #include <linux/slab.h>
  39 #include <linux/sched.h>
  40 #include <linux/mm.h>
  41 #include <linux/file.h>
  42 #include <linux/module.h>
  43 #include <linux/atomic.h>
  44 #include <linux/dma-resv.h>
  45
  46 static void ttm_bo_global_kobj_release(struct kobject *kobj);
  47
  48 /**
  49  * ttm_global_mutex - protecting the global BO state
  50  */
  51 DEFINE_MUTEX(ttm_global_mutex);
  52 unsigned ttm_bo_glob_use_count;
  53 struct ttm_bo_global ttm_bo_glob;
  54 EXPORT_SYMBOL(ttm_bo_glob);
  55
  56 static struct attribute ttm_bo_count = {
  57         .name = "bo_count",
  58         .mode = S_IRUGO
  59 };
  60
  61 /* default destructor */
  62 static void ttm_bo_default_destroy(struct ttm_buffer_object *bo)
  63 {
  64         kfree(bo);
  65 }
  66
  67 static inline int ttm_mem_type_from_place(const struct ttm_place *place,
  68                                           uint32_t *mem_type)
  69 {
  70         int pos;
  71
  72         pos = ffs(place->flags & TTM_PL_MASK_MEM);
  73         if (unlikely(!pos))
  74                 return -EINVAL;
  75
  76         *mem_type = pos - 1;
  77         return 0;
  78 }
  79
  80 static void ttm_mem_type_debug(struct ttm_bo_device *bdev, struct drm_printer *p,
  81                                int mem_type)
  82 {
  83         struct ttm_mem_type_manager *man = &bdev->man[mem_type];
  84
  85         drm_printf(p, "    has_type: %d\n", man->has_type);
  86         drm_printf(p, "    use_type: %d\n", man->use_type);
  87         drm_printf(p, "    flags: 0x%08X\n", man->flags);
  88         drm_printf(p, "    gpu_offset: 0x%08llX\n", man->gpu_offset);
  89         drm_printf(p, "    size: %llu\n", man->size);
  90         drm_printf(p, "    available_caching: 0x%08X\n", man->available_caching);
  91         drm_printf(p, "    default_caching: 0x%08X\n", man->default_caching);
  92         if (mem_type != TTM_PL_SYSTEM)
  93                 (*man->func->debug)(man, p);
  94 }
  95
  96 static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo,
  97                                         struct ttm_placement *placement)
  98 {
  99         struct drm_printer p = drm_debug_printer(TTM_PFX);
 100         int i, ret, mem_type;
 101
 102         drm_printf(&p, "No space for %p (%lu pages, %luK, %luM)\n",
 103                    bo, bo->mem.num_pages, bo->mem.size >> 10,
 104                    bo->mem.size >> 20);
 105         for (i = 0; i < placement->num_placement; i++) {
 106                 ret = ttm_mem_type_from_place(&placement->placement[i],
 107                                                 &mem_type);
 108                 if (ret)
 109                         return;
 110                 drm_printf(&p, "  placement[%d]=0x%08X (%d)\n",
 111                            i, placement->placement[i].flags, mem_type);
 112                 ttm_mem_type_debug(bo->bdev, &p, mem_type);
 113         }
 114 }
 115
 116 static ssize_t ttm_bo_global_show(struct kobject *kobj,
 117                                   struct attribute *attr,
 118                                   char *buffer)
 119 {
 120         struct ttm_bo_global *glob =
 121                 container_of(kobj, struct ttm_bo_global, kobj);
 122
 123         return snprintf(buffer, PAGE_SIZE, "%d\n",
 124                                 atomic_read(&glob->bo_count));
 125 }
 126
 127 static struct attribute *ttm_bo_global_attrs[] = {
 128         &ttm_bo_count,
 129         NULL
 130 };
 131
 132 static const struct sysfs_ops ttm_bo_global_ops = {
 133         .show = &ttm_bo_global_show
 134 };
 135
 136 static struct kobj_type ttm_bo_glob_kobj_type  = {
 137         .release = &ttm_bo_global_kobj_release,
 138         .sysfs_ops = &ttm_bo_global_ops,
 139         .default_attrs = ttm_bo_global_attrs
 140 };
 141
 142
 143 static inline uint32_t ttm_bo_type_flags(unsigned type)
 144 {
 145         return 1 << (type);
 146 }
 147
 148 static void ttm_bo_add_mem_to_lru(struct ttm_buffer_object *bo,
 149                                   struct ttm_mem_reg *mem)
 150 {
 151         struct ttm_bo_device *bdev = bo->bdev;
 152         struct ttm_mem_type_manager *man;
 153
 154         if (!list_empty(&bo->lru))
 155                 return;
 156
 157         if (mem->placement & TTM_PL_FLAG_NO_EVICT)
 158                 return;
 159
 160         man = &bdev->man[mem->mem_type];
 161         list_add_tail(&bo->lru, &man->lru[bo->priority]);
 162
 163         if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED) && bo->ttm &&
 164             !(bo->ttm->page_flags & (TTM_PAGE_FLAG_SG |
 165                                      TTM_PAGE_FLAG_SWAPPED))) {
 166                 list_add_tail(&bo->swap, &ttm_bo_glob.swap_lru[bo->priority]);
 167         }
 168 }
 169
 170 static void ttm_bo_del_from_lru(struct ttm_buffer_object *bo)
 171 {
 172         struct ttm_bo_device *bdev = bo->bdev;
 173         bool notify = false;
 174
 175         if (!list_empty(&bo->swap)) {
 176                 list_del_init(&bo->swap);
 177                 notify = true;
 178         }
 179         if (!list_empty(&bo->lru)) {
 180                 list_del_init(&bo->lru);
 181                 notify = true;
 182         }
 183
 184         if (notify && bdev->driver->del_from_lru_notify)
 185                 bdev->driver->del_from_lru_notify(bo);
 186 }
 187
 188 static void ttm_bo_bulk_move_set_pos(struct ttm_lru_bulk_move_pos *pos,
 189                                      struct ttm_buffer_object *bo)
 190 {
 191         if (!pos->first)
 192                 pos->first = bo;
 193         pos->last = bo;
 194 }
 195
 196 void ttm_bo_move_to_lru_tail(struct ttm_buffer_object *bo,
 197                              struct ttm_lru_bulk_move *bulk)
 198 {
 199         dma_resv_assert_held(bo->base.resv);
 200
 201         ttm_bo_del_from_lru(bo);
 202         ttm_bo_add_mem_to_lru(bo, &bo->mem);
 203
 204         if (bulk && !(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) {
 205                 switch (bo->mem.mem_type) {
 206                 case TTM_PL_TT:
 207                         ttm_bo_bulk_move_set_pos(&bulk->tt[bo->priority], bo);
 208                         break;
 209
 210                 case TTM_PL_VRAM:
 211                         ttm_bo_bulk_move_set_pos(&bulk->vram[bo->priority], bo);
 212                         break;
 213                 }
 214                 if (bo->ttm && !(bo->ttm->page_flags &
 215                                  (TTM_PAGE_FLAG_SG | TTM_PAGE_FLAG_SWAPPED)))
 216                         ttm_bo_bulk_move_set_pos(&bulk->swap[bo->priority], bo);
 217         }
 218 }
 219 EXPORT_SYMBOL(ttm_bo_move_to_lru_tail);
 220
 221 void ttm_bo_bulk_move_lru_tail(struct ttm_lru_bulk_move *bulk)
 222 {
 223         unsigned i;
 224
 225         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
 226                 struct ttm_lru_bulk_move_pos *pos = &bulk->tt[i];
 227                 struct ttm_mem_type_manager *man;
 228
 229                 if (!pos->first)
 230                         continue;
 231
 232                 dma_resv_assert_held(pos->first->base.resv);
 233                 dma_resv_assert_held(pos->last->base.resv);
 234
 235                 man = &pos->first->bdev->man[TTM_PL_TT];
 236                 list_bulk_move_tail(&man->lru[i], &pos->first->lru,
 237                                     &pos->last->lru);
 238         }
 239
 240         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
 241                 struct ttm_lru_bulk_move_pos *pos = &bulk->vram[i];
 242                 struct ttm_mem_type_manager *man;
 243
 244                 if (!pos->first)
 245                         continue;
 246
 247                 dma_resv_assert_held(pos->first->base.resv);
 248                 dma_resv_assert_held(pos->last->base.resv);
 249
 250                 man = &pos->first->bdev->man[TTM_PL_VRAM];
 251                 list_bulk_move_tail(&man->lru[i], &pos->first->lru,
 252                                     &pos->last->lru);
 253         }
 254
 255         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
 256                 struct ttm_lru_bulk_move_pos *pos = &bulk->swap[i];
 257                 struct list_head *lru;
 258
 259                 if (!pos->first)
 260                         continue;
 261
 262                 dma_resv_assert_held(pos->first->base.resv);
 263                 dma_resv_assert_held(pos->last->base.resv);
 264
 265                 lru = &ttm_bo_glob.swap_lru[i];
 266                 list_bulk_move_tail(lru, &pos->first->swap, &pos->last->swap);
 267         }
 268 }
 269 EXPORT_SYMBOL(ttm_bo_bulk_move_lru_tail);
 270
 271 static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo,
 272                                   struct ttm_mem_reg *mem, bool evict,
 273                                   struct ttm_operation_ctx *ctx)
 274 {
 275         struct ttm_bo_device *bdev = bo->bdev;
 276         bool old_is_pci = ttm_mem_reg_is_pci(bdev, &bo->mem);
 277         bool new_is_pci = ttm_mem_reg_is_pci(bdev, mem);
 278         struct ttm_mem_type_manager *old_man = &bdev->man[bo->mem.mem_type];
 279         struct ttm_mem_type_manager *new_man = &bdev->man[mem->mem_type];
 280         int ret = 0;
 281
 282         if (old_is_pci || new_is_pci ||
 283             ((mem->placement & bo->mem.placement & TTM_PL_MASK_CACHING) == 0)) {
 284                 ret = ttm_mem_io_lock(old_man, true);
 285                 if (unlikely(ret != 0))
 286                         goto out_err;
 287                 ttm_bo_unmap_virtual_locked(bo);
 288                 ttm_mem_io_unlock(old_man);
 289         }
 290
 291         /*
 292          * Create and bind a ttm if required.
 293          */
 294
 295         if (!(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) {
 296                 if (bo->ttm == NULL) {
 297                         bool zero = !(old_man->flags & TTM_MEMTYPE_FLAG_FIXED);
 298                         ret = ttm_tt_create(bo, zero);
 299                         if (ret)
 300                                 goto out_err;
 301                 }
 302
 303                 ret = ttm_tt_set_placement_caching(bo->ttm, mem->placement);
 304                 if (ret)
 305                         goto out_err;
 306
 307                 if (mem->mem_type != TTM_PL_SYSTEM) {
 308                         ret = ttm_tt_bind(bo->ttm, mem, ctx);
 309                         if (ret)
 310                                 goto out_err;
 311                 }
 312
 313                 if (bo->mem.mem_type == TTM_PL_SYSTEM) {
 314                         if (bdev->driver->move_notify)
 315                                 bdev->driver->move_notify(bo, evict, mem);
 316                         bo->mem = *mem;
 317                         mem->mm_node = NULL;
 318                         goto moved;
 319                 }
 320         }
 321
 322         if (bdev->driver->move_notify)
 323                 bdev->driver->move_notify(bo, evict, mem);
 324
 325         if (!(old_man->flags & TTM_MEMTYPE_FLAG_FIXED) &&
 326             !(new_man->flags & TTM_MEMTYPE_FLAG_FIXED))
 327                 ret = ttm_bo_move_ttm(bo, ctx, mem);
 328         else if (bdev->driver->move)
 329                 ret = bdev->driver->move(bo, evict, ctx, mem);
 330         else
 331                 ret = ttm_bo_move_memcpy(bo, ctx, mem);
 332
 333         if (ret) {
 334                 if (bdev->driver->move_notify) {
 335                         swap(*mem, bo->mem);
 336                         bdev->driver->move_notify(bo, false, mem);
 337                         swap(*mem, bo->mem);
 338                 }
 339
 340                 goto out_err;
 341         }
 342
 343 moved:
 344         bo->evicted = false;
 345
 346         if (bo->mem.mm_node)
 347                 bo->offset = (bo->mem.start << PAGE_SHIFT) +
 348                     bdev->man[bo->mem.mem_type].gpu_offset;
 349         else
 350                 bo->offset = 0;
 351
 352         ctx->bytes_moved += bo->num_pages << PAGE_SHIFT;
 353         return 0;
 354
 355 out_err:
 356         new_man = &bdev->man[bo->mem.mem_type];
 357         if (new_man->flags & TTM_MEMTYPE_FLAG_FIXED) {
 358                 ttm_tt_destroy(bo->ttm);
 359                 bo->ttm = NULL;
 360         }
 361
 362         return ret;
 363 }
 364
 365 /**
 366  * Call bo::reserved.
 367  * Will release GPU memory type usage on destruction.
 368  * This is the place to put in driver specific hooks to release
 369  * driver private resources.
 370  * Will release the bo::reserved lock.
 371  */
 372
 373 static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo)
 374 {
 375         if (bo->bdev->driver->move_notify)
 376                 bo->bdev->driver->move_notify(bo, false, NULL);
 377
 378         ttm_tt_destroy(bo->ttm);
 379         bo->ttm = NULL;
 380         ttm_bo_mem_put(bo, &bo->mem);
 381 }
 382
 383 static int ttm_bo_individualize_resv(struct ttm_buffer_object *bo)
 384 {
 385         int r;
 386
 387         if (bo->base.resv == &bo->base._resv)
 388                 return 0;
 389
 390         BUG_ON(!dma_resv_trylock(&bo->base._resv));
 391
 392         r = dma_resv_copy_fences(&bo->base._resv, bo->base.resv);
 393         dma_resv_unlock(&bo->base._resv);
 394         if (r)
 395                 return r;
 396
 397         if (bo->type != ttm_bo_type_sg) {
 398                 /* This works because the BO is about to be destroyed and nobody
 399                  * reference it any more. The only tricky case is the trylock on
 400                  * the resv object while holding the lru_lock.
 401                  */
 402                 spin_lock(&ttm_bo_glob.lru_lock);
 403                 bo->base.resv = &bo->base._resv;
 404                 spin_unlock(&ttm_bo_glob.lru_lock);
 405         }
 406
 407         return r;
 408 }
 409
 410 static void ttm_bo_flush_all_fences(struct ttm_buffer_object *bo)
 411 {
 412         struct dma_resv *resv = &bo->base._resv;
 413         struct dma_resv_list *fobj;
 414         struct dma_fence *fence;
 415         int i;
 416
 417         rcu_read_lock();
 418         fobj = rcu_dereference(resv->fence);
 419         fence = rcu_dereference(resv->fence_excl);
 420         if (fence && !fence->ops->signaled)
 421                 dma_fence_enable_sw_signaling(fence);
 422
 423         for (i = 0; fobj && i < fobj->shared_count; ++i) {
 424                 fence = rcu_dereference(fobj->shared[i]);
 425
 426                 if (!fence->ops->signaled)
 427                         dma_fence_enable_sw_signaling(fence);
 428         }
 429         rcu_read_unlock();
 430 }
 431
 432 /**
 433  * function ttm_bo_cleanup_refs
 434  * If bo idle, remove from lru lists, and unref.
 435  * If not idle, block if possible.
 436  *
 437  * Must be called with lru_lock and reservation held, this function
 438  * will drop the lru lock and optionally the reservation lock before returning.
 439  *
 440  * @interruptible         Any sleeps should occur interruptibly.
 441  * @no_wait_gpu           Never wait for gpu. Return -EBUSY instead.
 442  * @unlock_resv           Unlock the reservation lock as well.
 443  */
 444
 445 static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo,
 446                                bool interruptible, bool no_wait_gpu,
 447                                bool unlock_resv)
 448 {
 449         struct dma_resv *resv = &bo->base._resv;
 450         int ret;
 451
 452         if (dma_resv_test_signaled_rcu(resv, true))
 453                 ret = 0;
 454         else
 455                 ret = -EBUSY;
 456
 457         if (ret && !no_wait_gpu) {
 458                 long lret;
 459
 460                 if (unlock_resv)
 461                         dma_resv_unlock(bo->base.resv);
 462                 spin_unlock(&ttm_bo_glob.lru_lock);
 463
 464                 lret = dma_resv_wait_timeout_rcu(resv, true, interruptible,
 465                                                  30 * HZ);
 466
 467                 if (lret < 0)
 468                         return lret;
 469                 else if (lret == 0)
 470                         return -EBUSY;
 471
 472                 spin_lock(&ttm_bo_glob.lru_lock);
 473                 if (unlock_resv && !dma_resv_trylock(bo->base.resv)) {
 474                         /*
 475                          * We raced, and lost, someone else holds the reservation now,
 476                          * and is probably busy in ttm_bo_cleanup_memtype_use.
 477                          *
 478                          * Even if it's not the case, because we finished waiting any
 479                          * delayed destruction would succeed, so just return success
 480                          * here.
 481                          */
 482                         spin_unlock(&ttm_bo_glob.lru_lock);
 483                         return 0;
 484                 }
 485                 ret = 0;
 486         }
 487
 488         if (ret || unlikely(list_empty(&bo->ddestroy))) {
 489                 if (unlock_resv)
 490                         dma_resv_unlock(bo->base.resv);
 491                 spin_unlock(&ttm_bo_glob.lru_lock);
 492                 return ret;
 493         }
 494
 495         ttm_bo_del_from_lru(bo);
 496         list_del_init(&bo->ddestroy);
 497         spin_unlock(&ttm_bo_glob.lru_lock);
 498         ttm_bo_cleanup_memtype_use(bo);
 499
 500         if (unlock_resv)
 501                 dma_resv_unlock(bo->base.resv);
 502
 503         ttm_bo_put(bo);
 504
 505         return 0;
 506 }
 507
 508 /**
 509  * Traverse the delayed list, and call ttm_bo_cleanup_refs on all
 510  * encountered buffers.
 511  */
 512 static bool ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all)
 513 {
 514         struct ttm_bo_global *glob = &ttm_bo_glob;
 515         struct list_head removed;
 516         bool empty;
 517
 518         INIT_LIST_HEAD(&removed);
 519
 520         spin_lock(&glob->lru_lock);
 521         while (!list_empty(&bdev->ddestroy)) {
 522                 struct ttm_buffer_object *bo;
 523
 524                 bo = list_first_entry(&bdev->ddestroy, struct ttm_buffer_object,
 525                                       ddestroy);
 526                 list_move_tail(&bo->ddestroy, &removed);
 527                 if (!ttm_bo_get_unless_zero(bo))
 528                         continue;
 529
 530                 if (remove_all || bo->base.resv != &bo->base._resv) {
 531                         spin_unlock(&glob->lru_lock);
 532                         dma_resv_lock(bo->base.resv, NULL);
 533
 534                         spin_lock(&glob->lru_lock);
 535                         ttm_bo_cleanup_refs(bo, false, !remove_all, true);
 536
 537                 } else if (dma_resv_trylock(bo->base.resv)) {
 538                         ttm_bo_cleanup_refs(bo, false, !remove_all, true);
 539                 } else {
 540                         spin_unlock(&glob->lru_lock);
 541                 }
 542
 543                 ttm_bo_put(bo);
 544                 spin_lock(&glob->lru_lock);
 545         }
 546         list_splice_tail(&removed, &bdev->ddestroy);
 547         empty = list_empty(&bdev->ddestroy);
 548         spin_unlock(&glob->lru_lock);
 549
 550         return empty;
 551 }
 552
 553 static void ttm_bo_delayed_workqueue(struct work_struct *work)
 554 {
 555         struct ttm_bo_device *bdev =
 556             container_of(work, struct ttm_bo_device, wq.work);
 557
 558         if (!ttm_bo_delayed_delete(bdev, false))
 559                 schedule_delayed_work(&bdev->wq,
 560                                       ((HZ / 100) < 1) ? 1 : HZ / 100);
 561 }
 562
 563 static void ttm_bo_release(struct kref *kref)
 564 {
 565         struct ttm_buffer_object *bo =
 566             container_of(kref, struct ttm_buffer_object, kref);
 567         struct ttm_bo_device *bdev = bo->bdev;
 568         struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type];
 569         size_t acc_size = bo->acc_size;
 570         int ret;
 571
 572         if (!bo->deleted) {
 573                 ret = ttm_bo_individualize_resv(bo);
 574                 if (ret) {
 575                         /* Last resort, if we fail to allocate memory for the
 576                          * fences block for the BO to become idle
 577                          */
 578                         dma_resv_wait_timeout_rcu(bo->base.resv, true, false,
 579                                                   30 * HZ);
 580                 }
 581
 582                 if (bo->bdev->driver->release_notify)
 583                         bo->bdev->driver->release_notify(bo);
 584
 585                 drm_vma_offset_remove(bdev->vma_manager, &bo->base.vma_node);
 586                 ttm_mem_io_lock(man, false);
 587                 ttm_mem_io_free_vm(bo);
 588                 ttm_mem_io_unlock(man);
 589         }
 590
 591         if (!dma_resv_test_signaled_rcu(bo->base.resv, true) ||
 592             !dma_resv_trylock(bo->base.resv)) {
 593                 /* The BO is not idle, resurrect it for delayed destroy */
 594                 ttm_bo_flush_all_fences(bo);
 595                 bo->deleted = true;
 596
 597                 spin_lock(&ttm_bo_glob.lru_lock);
 598
 599                 /*
 600                  * Make NO_EVICT bos immediately available to
 601                  * shrinkers, now that they are queued for
 602                  * destruction.
 603                  */
 604                 if (bo->mem.placement & TTM_PL_FLAG_NO_EVICT) {
 605                         bo->mem.placement &= ~TTM_PL_FLAG_NO_EVICT;
 606                         ttm_bo_del_from_lru(bo);
 607                         ttm_bo_add_mem_to_lru(bo, &bo->mem);
 608                 }
 609
 610                 kref_init(&bo->kref);
 611                 list_add_tail(&bo->ddestroy, &bdev->ddestroy);
 612                 spin_unlock(&ttm_bo_glob.lru_lock);
 613
 614                 schedule_delayed_work(&bdev->wq,
 615                                       ((HZ / 100) < 1) ? 1 : HZ / 100);
 616                 return;
 617         }
 618
 619         spin_lock(&ttm_bo_glob.lru_lock);
 620         ttm_bo_del_from_lru(bo);
 621         list_del(&bo->ddestroy);
 622         spin_unlock(&ttm_bo_glob.lru_lock);
 623
 624         ttm_bo_cleanup_memtype_use(bo);
 625         dma_resv_unlock(bo->base.resv);
 626
 627         BUG_ON(bo->mem.mm_node != NULL);
 628         atomic_dec(&ttm_bo_glob.bo_count);
 629         dma_fence_put(bo->moving);
 630         if (!ttm_bo_uses_embedded_gem_object(bo))
 631                 dma_resv_fini(&bo->base._resv);
 632         bo->destroy(bo);
 633         ttm_mem_global_free(&ttm_mem_glob, acc_size);
 634 }
 635
 636 void ttm_bo_put(struct ttm_buffer_object *bo)
 637 {
 638         kref_put(&bo->kref, ttm_bo_release);
 639 }
 640 EXPORT_SYMBOL(ttm_bo_put);
 641
 642 int ttm_bo_lock_delayed_workqueue(struct ttm_bo_device *bdev)
 643 {
 644         return cancel_delayed_work_sync(&bdev->wq);
 645 }
 646 EXPORT_SYMBOL(ttm_bo_lock_delayed_workqueue);
 647
 648 void ttm_bo_unlock_delayed_workqueue(struct ttm_bo_device *bdev, int resched)
 649 {
 650         if (resched)
 651                 schedule_delayed_work(&bdev->wq,
 652                                       ((HZ / 100) < 1) ? 1 : HZ / 100);
 653 }
 654 EXPORT_SYMBOL(ttm_bo_unlock_delayed_workqueue);
 655
 656 static int ttm_bo_evict(struct ttm_buffer_object *bo,
 657                         struct ttm_operation_ctx *ctx)
 658 {
 659         struct ttm_bo_device *bdev = bo->bdev;
 660         struct ttm_mem_reg evict_mem;
 661         struct ttm_placement placement;
 662         int ret = 0;
 663
 664         dma_resv_assert_held(bo->base.resv);
 665
 666         placement.num_placement = 0;
 667         placement.num_busy_placement = 0;
 668         bdev->driver->evict_flags(bo, &placement);
 669
 670         if (!placement.num_placement && !placement.num_busy_placement) {
 671                 ret = ttm_bo_pipeline_gutting(bo);
 672                 if (ret)
 673                         return ret;
 674
 675                 return ttm_tt_create(bo, false);
 676         }
 677
 678         evict_mem = bo->mem;
 679         evict_mem.mm_node = NULL;
 680         evict_mem.bus.io_reserved_vm = false;
 681         evict_mem.bus.io_reserved_count = 0;
 682
 683         ret = ttm_bo_mem_space(bo, &placement, &evict_mem, ctx);
 684         if (ret) {
 685                 if (ret != -ERESTARTSYS) {
 686                         pr_err("Failed to find memory space for buffer 0x%p eviction\n",
 687                                bo);
 688                         ttm_bo_mem_space_debug(bo, &placement);
 689                 }
 690                 goto out;
 691         }
 692
 693         ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, ctx);
 694         if (unlikely(ret)) {
 695                 if (ret != -ERESTARTSYS)
 696                         pr_err("Buffer eviction failed\n");
 697                 ttm_bo_mem_put(bo, &evict_mem);
 698                 goto out;
 699         }
 700         bo->evicted = true;
 701 out:
 702         return ret;
 703 }
 704
 705 bool ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
 706                               const struct ttm_place *place)
 707 {
 708         /* Don't evict this BO if it's outside of the
 709          * requested placement range
 710          */
 711         if (place->fpfn >= (bo->mem.start + bo->mem.size) ||
 712             (place->lpfn && place->lpfn <= bo->mem.start))
 713                 return false;
 714
 715         return true;
 716 }
 717 EXPORT_SYMBOL(ttm_bo_eviction_valuable);
 718
 719 /**
 720  * Check the target bo is allowable to be evicted or swapout, including cases:
 721  *
 722  * a. if share same reservation object with ctx->resv, have assumption
 723  * reservation objects should already be locked, so not lock again and
 724  * return true directly when either the opreation allow_reserved_eviction
 725  * or the target bo already is in delayed free list;
 726  *
 727  * b. Otherwise, trylock it.
 728  */
 729 static bool ttm_bo_evict_swapout_allowable(struct ttm_buffer_object *bo,
 730                         struct ttm_operation_ctx *ctx, bool *locked, bool *busy)
 731 {
 732         bool ret = false;
 733
 734         if (bo->base.resv == ctx->resv) {
 735                 dma_resv_assert_held(bo->base.resv);
 736                 if (ctx->flags & TTM_OPT_FLAG_ALLOW_RES_EVICT)
 737                         ret = true;
 738                 *locked = false;
 739                 if (busy)
 740                         *busy = false;
 741         } else {
 742                 ret = dma_resv_trylock(bo->base.resv);
 743                 *locked = ret;
 744                 if (busy)
 745                         *busy = !ret;
 746         }
 747
 748         return ret;
 749 }
 750
 751 /**
 752  * ttm_mem_evict_wait_busy - wait for a busy BO to become available
 753  *
 754  * @busy_bo: BO which couldn't be locked with trylock
 755  * @ctx: operation context
 756  * @ticket: acquire ticket
 757  *
 758  * Try to lock a busy buffer object to avoid failing eviction.
 759  */
 760 static int ttm_mem_evict_wait_busy(struct ttm_buffer_object *busy_bo,
 761                                    struct ttm_operation_ctx *ctx,
 762                                    struct ww_acquire_ctx *ticket)
 763 {
 764         int r;
 765
 766         if (!busy_bo || !ticket)
 767                 return -EBUSY;
 768
 769         if (ctx->interruptible)
 770                 r = dma_resv_lock_interruptible(busy_bo->base.resv,
 771                                                           ticket);
 772         else
 773                 r = dma_resv_lock(busy_bo->base.resv, ticket);
 774
 775         /*
 776          * TODO: It would be better to keep the BO locked until allocation is at
 777          * least tried one more time, but that would mean a much larger rework
 778          * of TTM.
 779          */
 780         if (!r)
 781                 dma_resv_unlock(busy_bo->base.resv);
 782
 783         return r == -EDEADLK ? -EBUSY : r;
 784 }
 785
 786 static int ttm_mem_evict_first(struct ttm_bo_device *bdev,
 787                                uint32_t mem_type,
 788                                const struct ttm_place *place,
 789                                struct ttm_operation_ctx *ctx,
 790                                struct ww_acquire_ctx *ticket)
 791 {
 792         struct ttm_buffer_object *bo = NULL, *busy_bo = NULL;
 793         struct ttm_mem_type_manager *man = &bdev->man[mem_type];
 794         bool locked = false;
 795         unsigned i;
 796         int ret;
 797
 798         spin_lock(&ttm_bo_glob.lru_lock);
 799         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
 800                 list_for_each_entry(bo, &man->lru[i], lru) {
 801                         bool busy;
 802
 803                         if (!ttm_bo_evict_swapout_allowable(bo, ctx, &locked,
 804                                                             &busy)) {
 805                                 if (busy && !busy_bo && ticket !=
 806                                     dma_resv_locking_ctx(bo->base.resv))
 807                                         busy_bo = bo;
 808                                 continue;
 809                         }
 810
 811                         if (place && !bdev->driver->eviction_valuable(bo,
 812                                                                       place)) {
 813                                 if (locked)
 814                                         dma_resv_unlock(bo->base.resv);
 815                                 continue;
 816                         }
 817                         if (!ttm_bo_get_unless_zero(bo)) {
 818                                 if (locked)
 819                                         dma_resv_unlock(bo->base.resv);
 820                                 continue;
 821                         }
 822                         break;
 823                 }
 824
 825                 /* If the inner loop terminated early, we have our candidate */
 826                 if (&bo->lru != &man->lru[i])
 827                         break;
 828
 829                 bo = NULL;
 830         }
 831
 832         if (!bo) {
 833                 if (busy_bo && !ttm_bo_get_unless_zero(busy_bo))
 834                         busy_bo = NULL;
 835                 spin_unlock(&ttm_bo_glob.lru_lock);
 836                 ret = ttm_mem_evict_wait_busy(busy_bo, ctx, ticket);
 837                 if (busy_bo)
 838                         ttm_bo_put(busy_bo);
 839                 return ret;
 840         }
 841
 842         if (bo->deleted) {
 843                 ret = ttm_bo_cleanup_refs(bo, ctx->interruptible,
 844                                           ctx->no_wait_gpu, locked);
 845                 ttm_bo_put(bo);
 846                 return ret;
 847         }
 848
 849         spin_unlock(&ttm_bo_glob.lru_lock);
 850
 851         ret = ttm_bo_evict(bo, ctx);
 852         if (locked)
 853                 ttm_bo_unreserve(bo);
 854
 855         ttm_bo_put(bo);
 856         return ret;
 857 }
 858
 859 void ttm_bo_mem_put(struct ttm_buffer_object *bo, struct ttm_mem_reg *mem)
 860 {
 861         struct ttm_mem_type_manager *man = &bo->bdev->man[mem->mem_type];
 862
 863         if (mem->mm_node)
 864                 (*man->func->put_node)(man, mem);
 865 }
 866 EXPORT_SYMBOL(ttm_bo_mem_put);
 867
 868 /**
 869  * Add the last move fence to the BO and reserve a new shared slot.
 870  */
 871 static int ttm_bo_add_move_fence(struct ttm_buffer_object *bo,
 872                                  struct ttm_mem_type_manager *man,
 873                                  struct ttm_mem_reg *mem,
 874                                  bool no_wait_gpu)
 875 {
 876         struct dma_fence *fence;
 877         int ret;
 878
 879         spin_lock(&man->move_lock);
 880         fence = dma_fence_get(man->move);
 881         spin_unlock(&man->move_lock);
 882
 883         if (!fence)
 884                 return 0;
 885
 886         if (no_wait_gpu) {
 887                 dma_fence_put(fence);
 888                 return -EBUSY;
 889         }
 890
 891         dma_resv_add_shared_fence(bo->base.resv, fence);
 892
 893         ret = dma_resv_reserve_shared(bo->base.resv, 1);
 894         if (unlikely(ret)) {
 895                 dma_fence_put(fence);
 896                 return ret;
 897         }
 898
 899         dma_fence_put(bo->moving);
 900         bo->moving = fence;
 901         return 0;
 902 }
 903
 904 /**
 905  * Repeatedly evict memory from the LRU for @mem_type until we create enough
 906  * space, or we've evicted everything and there isn't enough space.
 907  */
 908 static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo,
 909                                   const struct ttm_place *place,
 910                                   struct ttm_mem_reg *mem,
 911                                   struct ttm_operation_ctx *ctx)
 912 {
 913         struct ttm_bo_device *bdev = bo->bdev;
 914         struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
 915         struct ww_acquire_ctx *ticket;
 916         int ret;
 917
 918         ticket = dma_resv_locking_ctx(bo->base.resv);
 919         do {
 920                 ret = (*man->func->get_node)(man, bo, place, mem);
 921                 if (unlikely(ret != 0))
 922                         return ret;
 923                 if (mem->mm_node)
 924                         break;
 925                 ret = ttm_mem_evict_first(bdev, mem->mem_type, place, ctx,
 926                                           ticket);
 927                 if (unlikely(ret != 0))
 928                         return ret;
 929         } while (1);
 930
 931         return ttm_bo_add_move_fence(bo, man, mem, ctx->no_wait_gpu);
 932 }
 933
 934 static uint32_t ttm_bo_select_caching(struct ttm_mem_type_manager *man,
 935                                       uint32_t cur_placement,
 936                                       uint32_t proposed_placement)
 937 {
 938         uint32_t caching = proposed_placement & TTM_PL_MASK_CACHING;
 939         uint32_t result = proposed_placement & ~TTM_PL_MASK_CACHING;
 940
 941         /**
 942          * Keep current caching if possible.
 943          */
 944
 945         if ((cur_placement & caching) != 0)
 946                 result |= (cur_placement & caching);
 947         else if ((man->default_caching & caching) != 0)
 948                 result |= man->default_caching;
 949         else if ((TTM_PL_FLAG_CACHED & caching) != 0)
 950                 result |= TTM_PL_FLAG_CACHED;
 951         else if ((TTM_PL_FLAG_WC & caching) != 0)
 952                 result |= TTM_PL_FLAG_WC;
 953         else if ((TTM_PL_FLAG_UNCACHED & caching) != 0)
 954                 result |= TTM_PL_FLAG_UNCACHED;
 955
 956         return result;
 957 }
 958
 959 static bool ttm_bo_mt_compatible(struct ttm_mem_type_manager *man,
 960                                  uint32_t mem_type,
 961                                  const struct ttm_place *place,
 962                                  uint32_t *masked_placement)
 963 {
 964         uint32_t cur_flags = ttm_bo_type_flags(mem_type);
 965
 966         if ((cur_flags & place->flags & TTM_PL_MASK_MEM) == 0)
 967                 return false;
 968
 969         if ((place->flags & man->available_caching) == 0)
 970                 return false;
 971
 972         cur_flags |= (place->flags & man->available_caching);
 973
 974         *masked_placement = cur_flags;
 975         return true;
 976 }
 977
 978 /**
 979  * ttm_bo_mem_placement - check if placement is compatible
 980  * @bo: BO to find memory for
 981  * @place: where to search
 982  * @mem: the memory object to fill in
 983  * @ctx: operation context
 984  *
 985  * Check if placement is compatible and fill in mem structure.
 986  * Returns -EBUSY if placement won't work or negative error code.
 987  * 0 when placement can be used.
 988  */
 989 static int ttm_bo_mem_placement(struct ttm_buffer_object *bo,
 990                                 const struct ttm_place *place,
 991                                 struct ttm_mem_reg *mem,
 992                                 struct ttm_operation_ctx *ctx)
 993 {
 994         struct ttm_bo_device *bdev = bo->bdev;
 995         uint32_t mem_type = TTM_PL_SYSTEM;
 996         struct ttm_mem_type_manager *man;
 997         uint32_t cur_flags = 0;
 998         int ret;
 999
1000         ret = ttm_mem_type_from_place(place, &mem_type);
1001         if (ret)
1002                 return ret;
1003
1004         man = &bdev->man[mem_type];
1005         if (!man->has_type || !man->use_type)
1006                 return -EBUSY;
1007
1008         if (!ttm_bo_mt_compatible(man, mem_type, place, &cur_flags))
1009                 return -EBUSY;
1010
1011         cur_flags = ttm_bo_select_caching(man, bo->mem.placement, cur_flags);
1012         /*
1013          * Use the access and other non-mapping-related flag bits from
1014          * the memory placement flags to the current flags
1015          */
1016         ttm_flag_masked(&cur_flags, place->flags, ~TTM_PL_MASK_MEMTYPE);
1017
1018         mem->mem_type = mem_type;
1019         mem->placement = cur_flags;
1020
1021         spin_lock(&ttm_bo_glob.lru_lock);
1022         ttm_bo_del_from_lru(bo);
1023         ttm_bo_add_mem_to_lru(bo, mem);
1024         spin_unlock(&ttm_bo_glob.lru_lock);
1025
1026         return 0;
1027 }
1028
1029 /**
1030  * Creates space for memory region @mem according to its type.
1031  *
1032  * This function first searches for free space in compatible memory types in
1033  * the priority order defined by the driver.  If free space isn't found, then
1034  * ttm_bo_mem_force_space is attempted in priority order to evict and find
1035  * space.
1036  */
1037 int ttm_bo_mem_space(struct ttm_buffer_object *bo,
1038                         struct ttm_placement *placement,
1039                         struct ttm_mem_reg *mem,
1040                         struct ttm_operation_ctx *ctx)
1041 {
1042         struct ttm_bo_device *bdev = bo->bdev;
1043         bool type_found = false;
1044         int i, ret;
1045
1046         ret = dma_resv_reserve_shared(bo->base.resv, 1);
1047         if (unlikely(ret))
1048                 return ret;
1049
1050         mem->mm_node = NULL;
1051         for (i = 0; i < placement->num_placement; ++i) {
1052                 const struct ttm_place *place = &placement->placement[i];
1053                 struct ttm_mem_type_manager *man;
1054
1055                 ret = ttm_bo_mem_placement(bo, place, mem, ctx);
1056                 if (ret == -EBUSY)
1057                         continue;
1058                 if (ret)
1059                         goto error;
1060
1061                 type_found = true;
1062                 mem->mm_node = NULL;
1063                 if (mem->mem_type == TTM_PL_SYSTEM)
1064                         return 0;
1065
1066                 man = &bdev->man[mem->mem_type];
1067                 ret = (*man->func->get_node)(man, bo, place, mem);
1068                 if (unlikely(ret))
1069                         goto error;
1070
1071                 if (!mem->mm_node)
1072                         continue;
1073
1074                 ret = ttm_bo_add_move_fence(bo, man, mem, ctx->no_wait_gpu);
1075                 if (unlikely(ret)) {
1076                         (*man->func->put_node)(man, mem);
1077                         if (ret == -EBUSY)
1078                                 continue;
1079
1080                         goto error;
1081                 }
1082                 return 0;
1083         }
1084
1085         for (i = 0; i < placement->num_busy_placement; ++i) {
1086                 const struct ttm_place *place = &placement->busy_placement[i];
1087
1088                 ret = ttm_bo_mem_placement(bo, place, mem, ctx);
1089                 if (ret == -EBUSY)
1090                         continue;
1091                 if (ret)
1092                         goto error;
1093
1094                 type_found = true;
1095                 mem->mm_node = NULL;
1096                 if (mem->mem_type == TTM_PL_SYSTEM)
1097                         return 0;
1098
1099                 ret = ttm_bo_mem_force_space(bo, place, mem, ctx);
1100                 if (ret == 0 && mem->mm_node)
1101                         return 0;
1102
1103                 if (ret && ret != -EBUSY)
1104                         goto error;
1105         }
1106
1107         ret = -ENOMEM;
1108         if (!type_found) {
1109                 pr_err(TTM_PFX "No compatible memory type found\n");
1110                 ret = -EINVAL;
1111         }
1112
1113 error:
1114         if (bo->mem.mem_type == TTM_PL_SYSTEM && !list_empty(&bo->lru)) {
1115                 spin_lock(&ttm_bo_glob.lru_lock);
1116                 ttm_bo_move_to_lru_tail(bo, NULL);
1117                 spin_unlock(&ttm_bo_glob.lru_lock);
1118         }
1119
1120         return ret;
1121 }
1122 EXPORT_SYMBOL(ttm_bo_mem_space);
1123
1124 static int ttm_bo_move_buffer(struct ttm_buffer_object *bo,
1125                               struct ttm_placement *placement,
1126                               struct ttm_operation_ctx *ctx)
1127 {
1128         int ret = 0;
1129         struct ttm_mem_reg mem;
1130
1131         dma_resv_assert_held(bo->base.resv);
1132
1133         mem.num_pages = bo->num_pages;
1134         mem.size = mem.num_pages << PAGE_SHIFT;
1135         mem.page_alignment = bo->mem.page_alignment;
1136         mem.bus.io_reserved_vm = false;
1137         mem.bus.io_reserved_count = 0;
1138         /*
1139          * Determine where to move the buffer.
1140          */
1141         ret = ttm_bo_mem_space(bo, placement, &mem, ctx);
1142         if (ret)
1143                 goto out_unlock;
1144         ret = ttm_bo_handle_move_mem(bo, &mem, false, ctx);
1145 out_unlock:
1146         if (ret && mem.mm_node)
1147                 ttm_bo_mem_put(bo, &mem);
1148         return ret;
1149 }
1150
1151 static bool ttm_bo_places_compat(const struct ttm_place *places,
1152                                  unsigned num_placement,
1153                                  struct ttm_mem_reg *mem,
1154                                  uint32_t *new_flags)
1155 {
1156         unsigned i;
1157
1158         for (i = 0; i < num_placement; i++) {
1159                 const struct ttm_place *heap = &places[i];
1160
1161                 if (mem->mm_node && (mem->start < heap->fpfn ||
1162                      (heap->lpfn != 0 && (mem->start + mem->num_pages) > heap->lpfn)))
1163                         continue;
1164
1165                 *new_flags = heap->flags;
1166                 if ((*new_flags & mem->placement & TTM_PL_MASK_CACHING) &&
1167                     (*new_flags & mem->placement & TTM_PL_MASK_MEM) &&
1168                     (!(*new_flags & TTM_PL_FLAG_CONTIGUOUS) ||
1169                      (mem->placement & TTM_PL_FLAG_CONTIGUOUS)))
1170                         return true;
1171         }
1172         return false;
1173 }
1174
1175 bool ttm_bo_mem_compat(struct ttm_placement *placement,
1176                        struct ttm_mem_reg *mem,
1177                        uint32_t *new_flags)
1178 {
1179         if (ttm_bo_places_compat(placement->placement, placement->num_placement,
1180                                  mem, new_flags))
1181                 return true;
1182
1183         if ((placement->busy_placement != placement->placement ||
1184              placement->num_busy_placement > placement->num_placement) &&
1185             ttm_bo_places_compat(placement->busy_placement,
1186                                  placement->num_busy_placement,
1187                                  mem, new_flags))
1188                 return true;
1189
1190         return false;
1191 }
1192 EXPORT_SYMBOL(ttm_bo_mem_compat);
1193
1194 int ttm_bo_validate(struct ttm_buffer_object *bo,
1195                     struct ttm_placement *placement,
1196                     struct ttm_operation_ctx *ctx)
1197 {
1198         int ret;
1199         uint32_t new_flags;
1200
1201         dma_resv_assert_held(bo->base.resv);
1202
1203         /*
1204          * Remove the backing store if no placement is given.
1205          */
1206         if (!placement->num_placement && !placement->num_busy_placement) {
1207                 ret = ttm_bo_pipeline_gutting(bo);
1208                 if (ret)
1209                         return ret;
1210
1211                 return ttm_tt_create(bo, false);
1212         }
1213
1214         /*
1215          * Check whether we need to move buffer.
1216          */
1217         if (!ttm_bo_mem_compat(placement, &bo->mem, &new_flags)) {
1218                 ret = ttm_bo_move_buffer(bo, placement, ctx);
1219                 if (ret)
1220                         return ret;
1221         } else {
1222                 /*
1223                  * Use the access and other non-mapping-related flag bits from
1224                  * the compatible memory placement flags to the active flags
1225                  */
1226                 ttm_flag_masked(&bo->mem.placement, new_flags,
1227                                 ~TTM_PL_MASK_MEMTYPE);
1228         }
1229         /*
1230          * We might need to add a TTM.
1231          */
1232         if (bo->mem.mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) {
1233                 ret = ttm_tt_create(bo, true);
1234                 if (ret)
1235                         return ret;
1236         }
1237         return 0;
1238 }
1239 EXPORT_SYMBOL(ttm_bo_validate);
1240
1241 int ttm_bo_init_reserved(struct ttm_bo_device *bdev,
1242                          struct ttm_buffer_object *bo,
1243                          unsigned long size,
1244                          enum ttm_bo_type type,
1245                          struct ttm_placement *placement,
1246                          uint32_t page_alignment,
1247                          struct ttm_operation_ctx *ctx,
1248                          size_t acc_size,
1249                          struct sg_table *sg,
1250                          struct dma_resv *resv,
1251                          void (*destroy) (struct ttm_buffer_object *))
1252 {
1253         struct ttm_mem_global *mem_glob = &ttm_mem_glob;
1254         int ret = 0;
1255         unsigned long num_pages;
1256         bool locked;
1257
1258         ret = ttm_mem_global_alloc(mem_glob, acc_size, ctx);
1259         if (ret) {
1260                 pr_err("Out of kernel memory\n");
1261                 if (destroy)
1262                         (*destroy)(bo);
1263                 else
1264                         kfree(bo);
1265                 return -ENOMEM;
1266         }
1267
1268         num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
1269         if (num_pages == 0) {
1270                 pr_err("Illegal buffer object size\n");
1271                 if (destroy)
1272                         (*destroy)(bo);
1273                 else
1274                         kfree(bo);
1275                 ttm_mem_global_free(mem_glob, acc_size);
1276                 return -EINVAL;
1277         }
1278         bo->destroy = destroy ? destroy : ttm_bo_default_destroy;
1279
1280         kref_init(&bo->kref);
1281         INIT_LIST_HEAD(&bo->lru);
1282         INIT_LIST_HEAD(&bo->ddestroy);
1283         INIT_LIST_HEAD(&bo->swap);
1284         INIT_LIST_HEAD(&bo->io_reserve_lru);
1285         bo->bdev = bdev;
1286         bo->type = type;
1287         bo->num_pages = num_pages;
1288         bo->mem.size = num_pages << PAGE_SHIFT;
1289         bo->mem.mem_type = TTM_PL_SYSTEM;
1290         bo->mem.num_pages = bo->num_pages;
1291         bo->mem.mm_node = NULL;
1292         bo->mem.page_alignment = page_alignment;
1293         bo->mem.bus.io_reserved_vm = false;
1294         bo->mem.bus.io_reserved_count = 0;
1295         bo->moving = NULL;
1296         bo->mem.placement = (TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED);
1297         bo->acc_size = acc_size;
1298         bo->sg = sg;
1299         if (resv) {
1300                 bo->base.resv = resv;
1301                 dma_resv_assert_held(bo->base.resv);
1302         } else {
1303                 bo->base.resv = &bo->base._resv;
1304         }
1305         if (!ttm_bo_uses_embedded_gem_object(bo)) {
1306                 /*
1307                  * bo.gem is not initialized, so we have to setup the
1308                  * struct elements we want use regardless.
1309                  */
1310                 dma_resv_init(&bo->base._resv);
1311                 drm_vma_node_reset(&bo->base.vma_node);
1312         }
1313         atomic_inc(&ttm_bo_glob.bo_count);
1314
1315         /*
1316          * For ttm_bo_type_device buffers, allocate
1317          * address space from the device.
1318          */
1319         if (bo->type == ttm_bo_type_device ||
1320             bo->type == ttm_bo_type_sg)
1321                 ret = drm_vma_offset_add(bdev->vma_manager, &bo->base.vma_node,
1322                                          bo->mem.num_pages);
1323
1324         /* passed reservation objects should already be locked,
1325          * since otherwise lockdep will be angered in radeon.
1326          */
1327         if (!resv) {
1328                 locked = dma_resv_trylock(bo->base.resv);
1329                 WARN_ON(!locked);
1330         }
1331
1332         if (likely(!ret))
1333                 ret = ttm_bo_validate(bo, placement, ctx);
1334
1335         if (unlikely(ret)) {
1336                 if (!resv)
1337                         ttm_bo_unreserve(bo);
1338
1339                 ttm_bo_put(bo);
1340                 return ret;
1341         }
1342
1343         spin_lock(&ttm_bo_glob.lru_lock);
1344         ttm_bo_move_to_lru_tail(bo, NULL);
1345         spin_unlock(&ttm_bo_glob.lru_lock);
1346
1347         return ret;
1348 }
1349 EXPORT_SYMBOL(ttm_bo_init_reserved);
1350
1351 int ttm_bo_init(struct ttm_bo_device *bdev,
1352                 struct ttm_buffer_object *bo,
1353                 unsigned long size,
1354                 enum ttm_bo_type type,
1355                 struct ttm_placement *placement,
1356                 uint32_t page_alignment,
1357                 bool interruptible,
1358                 size_t acc_size,
1359                 struct sg_table *sg,
1360                 struct dma_resv *resv,
1361                 void (*destroy) (struct ttm_buffer_object *))
1362 {
1363         struct ttm_operation_ctx ctx = { interruptible, false };
1364         int ret;
1365
1366         ret = ttm_bo_init_reserved(bdev, bo, size, type, placement,
1367                                    page_alignment, &ctx, acc_size,
1368                                    sg, resv, destroy);
1369         if (ret)
1370                 return ret;
1371
1372         if (!resv)
1373                 ttm_bo_unreserve(bo);
1374
1375         return 0;
1376 }
1377 EXPORT_SYMBOL(ttm_bo_init);
1378
1379 size_t ttm_bo_acc_size(struct ttm_bo_device *bdev,
1380                        unsigned long bo_size,
1381                        unsigned struct_size)
1382 {
1383         unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT;
1384         size_t size = 0;
1385
1386         size += ttm_round_pot(struct_size);
1387         size += ttm_round_pot(npages * sizeof(void *));
1388         size += ttm_round_pot(sizeof(struct ttm_tt));
1389         return size;
1390 }
1391 EXPORT_SYMBOL(ttm_bo_acc_size);
1392
1393 size_t ttm_bo_dma_acc_size(struct ttm_bo_device *bdev,
1394                            unsigned long bo_size,
1395                            unsigned struct_size)
1396 {
1397         unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT;
1398         size_t size = 0;
1399
1400         size += ttm_round_pot(struct_size);
1401         size += ttm_round_pot(npages * (2*sizeof(void *) + sizeof(dma_addr_t)));
1402         size += ttm_round_pot(sizeof(struct ttm_dma_tt));
1403         return size;
1404 }
1405 EXPORT_SYMBOL(ttm_bo_dma_acc_size);
1406
1407 int ttm_bo_create(struct ttm_bo_device *bdev,
1408                         unsigned long size,
1409                         enum ttm_bo_type type,
1410                         struct ttm_placement *placement,
1411                         uint32_t page_alignment,
1412                         bool interruptible,
1413                         struct ttm_buffer_object **p_bo)
1414 {
1415         struct ttm_buffer_object *bo;
1416         size_t acc_size;
1417         int ret;
1418
1419         bo = kzalloc(sizeof(*bo), GFP_KERNEL);
1420         if (unlikely(bo == NULL))
1421                 return -ENOMEM;
1422
1423         acc_size = ttm_bo_acc_size(bdev, size, sizeof(struct ttm_buffer_object));
1424         ret = ttm_bo_init(bdev, bo, size, type, placement, page_alignment,
1425                           interruptible, acc_size,
1426                           NULL, NULL, NULL);
1427         if (likely(ret == 0))
1428                 *p_bo = bo;
1429
1430         return ret;
1431 }
1432 EXPORT_SYMBOL(ttm_bo_create);
1433
1434 static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev,
1435                                    unsigned mem_type)
1436 {
1437         struct ttm_operation_ctx ctx = {
1438                 .interruptible = false,
1439                 .no_wait_gpu = false,
1440                 .flags = TTM_OPT_FLAG_FORCE_ALLOC
1441         };
1442         struct ttm_mem_type_manager *man = &bdev->man[mem_type];
1443         struct ttm_bo_global *glob = &ttm_bo_glob;
1444         struct dma_fence *fence;
1445         int ret;
1446         unsigned i;
1447
1448         /*
1449          * Can't use standard list traversal since we're unlocking.
1450          */
1451
1452         spin_lock(&glob->lru_lock);
1453         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
1454                 while (!list_empty(&man->lru[i])) {
1455                         spin_unlock(&glob->lru_lock);
1456                         ret = ttm_mem_evict_first(bdev, mem_type, NULL, &ctx,
1457                                                   NULL);
1458                         if (ret)
1459                                 return ret;
1460                         spin_lock(&glob->lru_lock);
1461                 }
1462         }
1463         spin_unlock(&glob->lru_lock);
1464
1465         spin_lock(&man->move_lock);
1466         fence = dma_fence_get(man->move);
1467         spin_unlock(&man->move_lock);
1468
1469         if (fence) {
1470                 ret = dma_fence_wait(fence, false);
1471                 dma_fence_put(fence);
1472                 if (ret)
1473                         return ret;
1474         }
1475
1476         return 0;
1477 }
1478
1479 int ttm_bo_clean_mm(struct ttm_bo_device *bdev, unsigned mem_type)
1480 {
1481         struct ttm_mem_type_manager *man;
1482         int ret = -EINVAL;
1483
1484         if (mem_type >= TTM_NUM_MEM_TYPES) {
1485                 pr_err("Illegal memory type %d\n", mem_type);
1486                 return ret;
1487         }
1488         man = &bdev->man[mem_type];
1489
1490         if (!man->has_type) {
1491                 pr_err("Trying to take down uninitialized memory manager type %u\n",
1492                        mem_type);
1493                 return ret;
1494         }
1495
1496         man->use_type = false;
1497         man->has_type = false;
1498
1499         ret = 0;
1500         if (mem_type > 0) {
1501                 ret = ttm_bo_force_list_clean(bdev, mem_type);
1502                 if (ret) {
1503                         pr_err("Cleanup eviction failed\n");
1504                         return ret;
1505                 }
1506
1507                 ret = (*man->func->takedown)(man);
1508         }
1509
1510         dma_fence_put(man->move);
1511         man->move = NULL;
1512
1513         return ret;
1514 }
1515 EXPORT_SYMBOL(ttm_bo_clean_mm);
1516
1517 int ttm_bo_evict_mm(struct ttm_bo_device *bdev, unsigned mem_type)
1518 {
1519         struct ttm_mem_type_manager *man = &bdev->man[mem_type];
1520
1521         if (mem_type == 0 || mem_type >= TTM_NUM_MEM_TYPES) {
1522                 pr_err("Illegal memory manager memory type %u\n", mem_type);
1523                 return -EINVAL;
1524         }
1525
1526         if (!man->has_type) {
1527                 pr_err("Memory type %u has not been initialized\n", mem_type);
1528                 return 0;
1529         }
1530
1531         return ttm_bo_force_list_clean(bdev, mem_type);
1532 }
1533 EXPORT_SYMBOL(ttm_bo_evict_mm);
1534
1535 int ttm_bo_init_mm(struct ttm_bo_device *bdev, unsigned type,
1536                         unsigned long p_size)
1537 {
1538         int ret;
1539         struct ttm_mem_type_manager *man;
1540         unsigned i;
1541
1542         BUG_ON(type >= TTM_NUM_MEM_TYPES);
1543         man = &bdev->man[type];
1544         BUG_ON(man->has_type);
1545         man->io_reserve_fastpath = true;
1546         man->use_io_reserve_lru = false;
1547         mutex_init(&man->io_reserve_mutex);
1548         spin_lock_init(&man->move_lock);
1549         INIT_LIST_HEAD(&man->io_reserve_lru);
1550
1551         ret = bdev->driver->init_mem_type(bdev, type, man);
1552         if (ret)
1553                 return ret;
1554         man->bdev = bdev;
1555
1556         if (type != TTM_PL_SYSTEM) {
1557                 ret = (*man->func->init)(man, p_size);
1558                 if (ret)
1559                         return ret;
1560         }
1561         man->has_type = true;
1562         man->use_type = true;
1563         man->size = p_size;
1564
1565         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i)
1566                 INIT_LIST_HEAD(&man->lru[i]);
1567         man->move = NULL;
1568
1569         return 0;
1570 }
1571 EXPORT_SYMBOL(ttm_bo_init_mm);
1572
1573 static void ttm_bo_global_kobj_release(struct kobject *kobj)
1574 {
1575         struct ttm_bo_global *glob =
1576                 container_of(kobj, struct ttm_bo_global, kobj);
1577
1578         __free_page(glob->dummy_read_page);
1579 }
1580
1581 static void ttm_bo_global_release(void)
1582 {
1583         struct ttm_bo_global *glob = &ttm_bo_glob;
1584
1585         mutex_lock(&ttm_global_mutex);
1586         if (--ttm_bo_glob_use_count > 0)
1587                 goto out;
1588
1589         kobject_del(&glob->kobj);
1590         kobject_put(&glob->kobj);
1591         ttm_mem_global_release(&ttm_mem_glob);
1592         memset(glob, 0, sizeof(*glob));
1593 out:
1594         mutex_unlock(&ttm_global_mutex);
1595 }
1596
1597 static int ttm_bo_global_init(void)
1598 {
1599         struct ttm_bo_global *glob = &ttm_bo_glob;
1600         int ret = 0;
1601         unsigned i;
1602
1603         mutex_lock(&ttm_global_mutex);
1604         if (++ttm_bo_glob_use_count > 1)
1605                 goto out;
1606
1607         ret = ttm_mem_global_init(&ttm_mem_glob);
1608         if (ret)
1609                 goto out;
1610
1611         spin_lock_init(&glob->lru_lock);
1612         glob->dummy_read_page = alloc_page(__GFP_ZERO | GFP_DMA32);
1613
1614         if (unlikely(glob->dummy_read_page == NULL)) {
1615                 ret = -ENOMEM;
1616                 goto out;
1617         }
1618
1619         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i)
1620                 INIT_LIST_HEAD(&glob->swap_lru[i]);
1621         INIT_LIST_HEAD(&glob->device_list);
1622         atomic_set(&glob->bo_count, 0);
1623
1624         ret = kobject_init_and_add(
1625                 &glob->kobj, &ttm_bo_glob_kobj_type, ttm_get_kobj(), "buffer_objects");
1626         if (unlikely(ret != 0))
1627                 kobject_put(&glob->kobj);
1628 out:
1629         mutex_unlock(&ttm_global_mutex);
1630         return ret;
1631 }
1632
1633 int ttm_bo_device_release(struct ttm_bo_device *bdev)
1634 {
1635         struct ttm_bo_global *glob = &ttm_bo_glob;
1636         int ret = 0;
1637         unsigned i = TTM_NUM_MEM_TYPES;
1638         struct ttm_mem_type_manager *man;
1639
1640         while (i--) {
1641                 man = &bdev->man[i];
1642                 if (man->has_type) {
1643                         man->use_type = false;
1644                         if ((i != TTM_PL_SYSTEM) && ttm_bo_clean_mm(bdev, i)) {
1645                                 ret = -EBUSY;
1646                                 pr_err("DRM memory manager type %d is not clean\n",
1647                                        i);
1648                         }
1649                         man->has_type = false;
1650                 }
1651         }
1652
1653         mutex_lock(&ttm_global_mutex);
1654         list_del(&bdev->device_list);
1655         mutex_unlock(&ttm_global_mutex);
1656
1657         cancel_delayed_work_sync(&bdev->wq);
1658
1659         if (ttm_bo_delayed_delete(bdev, true))
1660                 pr_debug("Delayed destroy list was clean\n");
1661
1662         spin_lock(&glob->lru_lock);
1663         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i)
1664                 if (list_empty(&bdev->man[0].lru[0]))
1665                         pr_debug("Swap list %d was clean\n", i);
1666         spin_unlock(&glob->lru_lock);
1667
1668         if (!ret)
1669                 ttm_bo_global_release();
1670
1671         return ret;
1672 }
1673 EXPORT_SYMBOL(ttm_bo_device_release);
1674
1675 int ttm_bo_device_init(struct ttm_bo_device *bdev,
1676                        struct ttm_bo_driver *driver,
1677                        struct address_space *mapping,
1678                        struct drm_vma_offset_manager *vma_manager,
1679                        bool need_dma32)
1680 {
1681         struct ttm_bo_global *glob = &ttm_bo_glob;
1682         int ret;
1683
1684         if (WARN_ON(vma_manager == NULL))
1685                 return -EINVAL;
1686
1687         ret = ttm_bo_global_init();
1688         if (ret)
1689                 return ret;
1690
1691         bdev->driver = driver;
1692
1693         memset(bdev->man, 0, sizeof(bdev->man));
1694
1695         /*
1696          * Initialize the system memory buffer type.
1697          * Other types need to be driver / IOCTL initialized.
1698          */
1699         ret = ttm_bo_init_mm(bdev, TTM_PL_SYSTEM, 0);
1700         if (unlikely(ret != 0))
1701                 goto out_no_sys;
1702
1703         bdev->vma_manager = vma_manager;
1704         INIT_DELAYED_WORK(&bdev->wq, ttm_bo_delayed_workqueue);
1705         INIT_LIST_HEAD(&bdev->ddestroy);
1706         bdev->dev_mapping = mapping;
1707         bdev->need_dma32 = need_dma32;
1708         mutex_lock(&ttm_global_mutex);
1709         list_add_tail(&bdev->device_list, &glob->device_list);
1710         mutex_unlock(&ttm_global_mutex);
1711
1712         return 0;
1713 out_no_sys:
1714         ttm_bo_global_release();
1715         return ret;
1716 }
1717 EXPORT_SYMBOL(ttm_bo_device_init);
1718
1719 /*
1720  * buffer object vm functions.
1721  */
1722
1723 bool ttm_mem_reg_is_pci(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
1724 {
1725         struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
1726
1727         if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED)) {
1728                 if (mem->mem_type == TTM_PL_SYSTEM)
1729                         return false;
1730
1731                 if (man->flags & TTM_MEMTYPE_FLAG_CMA)
1732                         return false;
1733
1734                 if (mem->placement & TTM_PL_FLAG_CACHED)
1735                         return false;
1736         }
1737         return true;
1738 }
1739
1740 void ttm_bo_unmap_virtual_locked(struct ttm_buffer_object *bo)
1741 {
1742         struct ttm_bo_device *bdev = bo->bdev;
1743
1744         drm_vma_node_unmap(&bo->base.vma_node, bdev->dev_mapping);
1745         ttm_mem_io_free_vm(bo);
1746 }
1747
1748 void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo)
1749 {
1750         struct ttm_bo_device *bdev = bo->bdev;
1751         struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type];
1752
1753         ttm_mem_io_lock(man, false);
1754         ttm_bo_unmap_virtual_locked(bo);
1755         ttm_mem_io_unlock(man);
1756 }
1757
1758
1759 EXPORT_SYMBOL(ttm_bo_unmap_virtual);
1760
1761 int ttm_bo_wait(struct ttm_buffer_object *bo,
1762                 bool interruptible, bool no_wait)
1763 {
1764         long timeout = 15 * HZ;
1765
1766         if (no_wait) {
1767                 if (dma_resv_test_signaled_rcu(bo->base.resv, true))
1768                         return 0;
1769                 else
1770                         return -EBUSY;
1771         }
1772
1773         timeout = dma_resv_wait_timeout_rcu(bo->base.resv, true,
1774                                                       interruptible, timeout);
1775         if (timeout < 0)
1776                 return timeout;
1777
1778         if (timeout == 0)
1779                 return -EBUSY;
1780
1781         dma_resv_add_excl_fence(bo->base.resv, NULL);
1782         return 0;
1783 }
1784 EXPORT_SYMBOL(ttm_bo_wait);
1785
1786 /**
1787  * A buffer object shrink method that tries to swap out the first
1788  * buffer object on the bo_global::swap_lru list.
1789  */
1790 int ttm_bo_swapout(struct ttm_bo_global *glob, struct ttm_operation_ctx *ctx)
1791 {
1792         struct ttm_buffer_object *bo;
1793         int ret = -EBUSY;
1794         bool locked;
1795         unsigned i;
1796
1797         spin_lock(&glob->lru_lock);
1798         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
1799                 list_for_each_entry(bo, &glob->swap_lru[i], swap) {
1800                         if (!ttm_bo_evict_swapout_allowable(bo, ctx, &locked,
1801                                                             NULL))
1802                                 continue;
1803
1804                         if (!ttm_bo_get_unless_zero(bo)) {
1805                                 if (locked)
1806                                         dma_resv_unlock(bo->base.resv);
1807                                 continue;
1808                         }
1809
1810                         ret = 0;
1811                         break;
1812                 }
1813                 if (!ret)
1814                         break;
1815         }
1816
1817         if (ret) {
1818                 spin_unlock(&glob->lru_lock);
1819                 return ret;
1820         }
1821
1822         if (bo->deleted) {
1823                 ret = ttm_bo_cleanup_refs(bo, false, false, locked);
1824                 ttm_bo_put(bo);
1825                 return ret;
1826         }
1827
1828         ttm_bo_del_from_lru(bo);
1829         spin_unlock(&glob->lru_lock);
1830
1831         /**
1832          * Move to system cached
1833          */
1834
1835         if (bo->mem.mem_type != TTM_PL_SYSTEM ||
1836             bo->ttm->caching_state != tt_cached) {
1837                 struct ttm_operation_ctx ctx = { false, false };
1838                 struct ttm_mem_reg evict_mem;
1839
1840                 evict_mem = bo->mem;
1841                 evict_mem.mm_node = NULL;
1842                 evict_mem.placement = TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED;
1843                 evict_mem.mem_type = TTM_PL_SYSTEM;
1844
1845                 ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, &ctx);
1846                 if (unlikely(ret != 0))
1847                         goto out;
1848         }
1849
1850         /**
1851          * Make sure BO is idle.
1852          */
1853
1854         ret = ttm_bo_wait(bo, false, false);
1855         if (unlikely(ret != 0))
1856                 goto out;
1857
1858         ttm_bo_unmap_virtual(bo);
1859
1860         /**
1861          * Swap out. Buffer will be swapped in again as soon as
1862          * anyone tries to access a ttm page.
1863          */
1864
1865         if (bo->bdev->driver->swap_notify)
1866                 bo->bdev->driver->swap_notify(bo);
1867
1868         ret = ttm_tt_swapout(bo->ttm, bo->persistent_swap_storage);
1869 out:
1870
1871         /**
1872          *
1873          * Unreserve without putting on LRU to avoid swapping out an
1874          * already swapped buffer.
1875          */
1876         if (locked)
1877                 dma_resv_unlock(bo->base.resv);
1878         ttm_bo_put(bo);
1879         return ret;
1880 }
1881 EXPORT_SYMBOL(ttm_bo_swapout);
1882
1883 void ttm_bo_swapout_all(struct ttm_bo_device *bdev)
1884 {
1885         struct ttm_operation_ctx ctx = {
1886                 .interruptible = false,
1887                 .no_wait_gpu = false
1888         };
1889
1890         while (ttm_bo_swapout(&ttm_bo_glob, &ctx) == 0);
1891 }
1892 EXPORT_SYMBOL(ttm_bo_swapout_all);