Merge tag 'v5.3-rc3' into drm-next-5.4
[sfrench/cifs-2.6.git] / drivers / gpu / drm / ttm / ttm_bo.c
1 /* SPDX-License-Identifier: GPL-2.0 OR MIT */
2 /**************************************************************************
3  *
4  * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
5  * All Rights Reserved.
6  *
7  * Permission is hereby granted, free of charge, to any person obtaining a
8  * copy of this software and associated documentation files (the
9  * "Software"), to deal in the Software without restriction, including
10  * without limitation the rights to use, copy, modify, merge, publish,
11  * distribute, sub license, and/or sell copies of the Software, and to
12  * permit persons to whom the Software is furnished to do so, subject to
13  * the following conditions:
14  *
15  * The above copyright notice and this permission notice (including the
16  * next paragraph) shall be included in all copies or substantial portions
17  * of the Software.
18  *
19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
20  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
21  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
22  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
23  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
24  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
25  * USE OR OTHER DEALINGS IN THE SOFTWARE.
26  *
27  **************************************************************************/
28 /*
29  * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
30  */
31
32 #define pr_fmt(fmt) "[TTM] " fmt
33
34 #include <drm/ttm/ttm_module.h>
35 #include <drm/ttm/ttm_bo_driver.h>
36 #include <drm/ttm/ttm_placement.h>
37 #include <linux/jiffies.h>
38 #include <linux/slab.h>
39 #include <linux/sched.h>
40 #include <linux/mm.h>
41 #include <linux/file.h>
42 #include <linux/module.h>
43 #include <linux/atomic.h>
44 #include <linux/reservation.h>
45
46 static void ttm_bo_global_kobj_release(struct kobject *kobj);
47
48 /**
49  * ttm_global_mutex - protecting the global BO state
50  */
51 DEFINE_MUTEX(ttm_global_mutex);
52 unsigned ttm_bo_glob_use_count;
53 struct ttm_bo_global ttm_bo_glob;
54
55 static struct attribute ttm_bo_count = {
56         .name = "bo_count",
57         .mode = S_IRUGO
58 };
59
60 /* default destructor */
61 static void ttm_bo_default_destroy(struct ttm_buffer_object *bo)
62 {
63         kfree(bo);
64 }
65
66 static inline int ttm_mem_type_from_place(const struct ttm_place *place,
67                                           uint32_t *mem_type)
68 {
69         int pos;
70
71         pos = ffs(place->flags & TTM_PL_MASK_MEM);
72         if (unlikely(!pos))
73                 return -EINVAL;
74
75         *mem_type = pos - 1;
76         return 0;
77 }
78
79 static void ttm_mem_type_debug(struct ttm_bo_device *bdev, struct drm_printer *p,
80                                int mem_type)
81 {
82         struct ttm_mem_type_manager *man = &bdev->man[mem_type];
83
84         drm_printf(p, "    has_type: %d\n", man->has_type);
85         drm_printf(p, "    use_type: %d\n", man->use_type);
86         drm_printf(p, "    flags: 0x%08X\n", man->flags);
87         drm_printf(p, "    gpu_offset: 0x%08llX\n", man->gpu_offset);
88         drm_printf(p, "    size: %llu\n", man->size);
89         drm_printf(p, "    available_caching: 0x%08X\n", man->available_caching);
90         drm_printf(p, "    default_caching: 0x%08X\n", man->default_caching);
91         if (mem_type != TTM_PL_SYSTEM)
92                 (*man->func->debug)(man, p);
93 }
94
95 static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo,
96                                         struct ttm_placement *placement)
97 {
98         struct drm_printer p = drm_debug_printer(TTM_PFX);
99         int i, ret, mem_type;
100
101         drm_printf(&p, "No space for %p (%lu pages, %luK, %luM)\n",
102                    bo, bo->mem.num_pages, bo->mem.size >> 10,
103                    bo->mem.size >> 20);
104         for (i = 0; i < placement->num_placement; i++) {
105                 ret = ttm_mem_type_from_place(&placement->placement[i],
106                                                 &mem_type);
107                 if (ret)
108                         return;
109                 drm_printf(&p, "  placement[%d]=0x%08X (%d)\n",
110                            i, placement->placement[i].flags, mem_type);
111                 ttm_mem_type_debug(bo->bdev, &p, mem_type);
112         }
113 }
114
115 static ssize_t ttm_bo_global_show(struct kobject *kobj,
116                                   struct attribute *attr,
117                                   char *buffer)
118 {
119         struct ttm_bo_global *glob =
120                 container_of(kobj, struct ttm_bo_global, kobj);
121
122         return snprintf(buffer, PAGE_SIZE, "%d\n",
123                                 atomic_read(&glob->bo_count));
124 }
125
126 static struct attribute *ttm_bo_global_attrs[] = {
127         &ttm_bo_count,
128         NULL
129 };
130
131 static const struct sysfs_ops ttm_bo_global_ops = {
132         .show = &ttm_bo_global_show
133 };
134
135 static struct kobj_type ttm_bo_glob_kobj_type  = {
136         .release = &ttm_bo_global_kobj_release,
137         .sysfs_ops = &ttm_bo_global_ops,
138         .default_attrs = ttm_bo_global_attrs
139 };
140
141
142 static inline uint32_t ttm_bo_type_flags(unsigned type)
143 {
144         return 1 << (type);
145 }
146
147 static void ttm_bo_release_list(struct kref *list_kref)
148 {
149         struct ttm_buffer_object *bo =
150             container_of(list_kref, struct ttm_buffer_object, list_kref);
151         struct ttm_bo_device *bdev = bo->bdev;
152         size_t acc_size = bo->acc_size;
153
154         BUG_ON(kref_read(&bo->list_kref));
155         BUG_ON(kref_read(&bo->kref));
156         BUG_ON(atomic_read(&bo->cpu_writers));
157         BUG_ON(bo->mem.mm_node != NULL);
158         BUG_ON(!list_empty(&bo->lru));
159         BUG_ON(!list_empty(&bo->ddestroy));
160         ttm_tt_destroy(bo->ttm);
161         atomic_dec(&bo->bdev->glob->bo_count);
162         dma_fence_put(bo->moving);
163         reservation_object_fini(&bo->ttm_resv);
164         mutex_destroy(&bo->wu_mutex);
165         bo->destroy(bo);
166         ttm_mem_global_free(bdev->glob->mem_glob, acc_size);
167 }
168
169 static void ttm_bo_add_mem_to_lru(struct ttm_buffer_object *bo,
170                                   struct ttm_mem_reg *mem)
171 {
172         struct ttm_bo_device *bdev = bo->bdev;
173         struct ttm_mem_type_manager *man;
174
175         reservation_object_assert_held(bo->resv);
176
177         if (!list_empty(&bo->lru))
178                 return;
179
180         if (mem->placement & TTM_PL_FLAG_NO_EVICT)
181                 return;
182
183         man = &bdev->man[mem->mem_type];
184         list_add_tail(&bo->lru, &man->lru[bo->priority]);
185         kref_get(&bo->list_kref);
186
187         if (bo->ttm && !(bo->ttm->page_flags &
188                          (TTM_PAGE_FLAG_SG | TTM_PAGE_FLAG_SWAPPED))) {
189                 list_add_tail(&bo->swap, &bdev->glob->swap_lru[bo->priority]);
190                 kref_get(&bo->list_kref);
191         }
192 }
193
194 void ttm_bo_add_to_lru(struct ttm_buffer_object *bo)
195 {
196         ttm_bo_add_mem_to_lru(bo, &bo->mem);
197 }
198 EXPORT_SYMBOL(ttm_bo_add_to_lru);
199
200 static void ttm_bo_ref_bug(struct kref *list_kref)
201 {
202         BUG();
203 }
204
205 void ttm_bo_del_from_lru(struct ttm_buffer_object *bo)
206 {
207         struct ttm_bo_device *bdev = bo->bdev;
208         bool notify = false;
209
210         if (!list_empty(&bo->swap)) {
211                 list_del_init(&bo->swap);
212                 kref_put(&bo->list_kref, ttm_bo_ref_bug);
213                 notify = true;
214         }
215         if (!list_empty(&bo->lru)) {
216                 list_del_init(&bo->lru);
217                 kref_put(&bo->list_kref, ttm_bo_ref_bug);
218                 notify = true;
219         }
220
221         if (notify && bdev->driver->del_from_lru_notify)
222                 bdev->driver->del_from_lru_notify(bo);
223 }
224
225 void ttm_bo_del_sub_from_lru(struct ttm_buffer_object *bo)
226 {
227         struct ttm_bo_global *glob = bo->bdev->glob;
228
229         spin_lock(&glob->lru_lock);
230         ttm_bo_del_from_lru(bo);
231         spin_unlock(&glob->lru_lock);
232 }
233 EXPORT_SYMBOL(ttm_bo_del_sub_from_lru);
234
235 static void ttm_bo_bulk_move_set_pos(struct ttm_lru_bulk_move_pos *pos,
236                                      struct ttm_buffer_object *bo)
237 {
238         if (!pos->first)
239                 pos->first = bo;
240         pos->last = bo;
241 }
242
243 void ttm_bo_move_to_lru_tail(struct ttm_buffer_object *bo,
244                              struct ttm_lru_bulk_move *bulk)
245 {
246         reservation_object_assert_held(bo->resv);
247
248         ttm_bo_del_from_lru(bo);
249         ttm_bo_add_to_lru(bo);
250
251         if (bulk && !(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) {
252                 switch (bo->mem.mem_type) {
253                 case TTM_PL_TT:
254                         ttm_bo_bulk_move_set_pos(&bulk->tt[bo->priority], bo);
255                         break;
256
257                 case TTM_PL_VRAM:
258                         ttm_bo_bulk_move_set_pos(&bulk->vram[bo->priority], bo);
259                         break;
260                 }
261                 if (bo->ttm && !(bo->ttm->page_flags &
262                                  (TTM_PAGE_FLAG_SG | TTM_PAGE_FLAG_SWAPPED)))
263                         ttm_bo_bulk_move_set_pos(&bulk->swap[bo->priority], bo);
264         }
265 }
266 EXPORT_SYMBOL(ttm_bo_move_to_lru_tail);
267
268 void ttm_bo_bulk_move_lru_tail(struct ttm_lru_bulk_move *bulk)
269 {
270         unsigned i;
271
272         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
273                 struct ttm_lru_bulk_move_pos *pos = &bulk->tt[i];
274                 struct ttm_mem_type_manager *man;
275
276                 if (!pos->first)
277                         continue;
278
279                 reservation_object_assert_held(pos->first->resv);
280                 reservation_object_assert_held(pos->last->resv);
281
282                 man = &pos->first->bdev->man[TTM_PL_TT];
283                 list_bulk_move_tail(&man->lru[i], &pos->first->lru,
284                                     &pos->last->lru);
285         }
286
287         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
288                 struct ttm_lru_bulk_move_pos *pos = &bulk->vram[i];
289                 struct ttm_mem_type_manager *man;
290
291                 if (!pos->first)
292                         continue;
293
294                 reservation_object_assert_held(pos->first->resv);
295                 reservation_object_assert_held(pos->last->resv);
296
297                 man = &pos->first->bdev->man[TTM_PL_VRAM];
298                 list_bulk_move_tail(&man->lru[i], &pos->first->lru,
299                                     &pos->last->lru);
300         }
301
302         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
303                 struct ttm_lru_bulk_move_pos *pos = &bulk->swap[i];
304                 struct list_head *lru;
305
306                 if (!pos->first)
307                         continue;
308
309                 reservation_object_assert_held(pos->first->resv);
310                 reservation_object_assert_held(pos->last->resv);
311
312                 lru = &pos->first->bdev->glob->swap_lru[i];
313                 list_bulk_move_tail(lru, &pos->first->swap, &pos->last->swap);
314         }
315 }
316 EXPORT_SYMBOL(ttm_bo_bulk_move_lru_tail);
317
318 static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo,
319                                   struct ttm_mem_reg *mem, bool evict,
320                                   struct ttm_operation_ctx *ctx)
321 {
322         struct ttm_bo_device *bdev = bo->bdev;
323         bool old_is_pci = ttm_mem_reg_is_pci(bdev, &bo->mem);
324         bool new_is_pci = ttm_mem_reg_is_pci(bdev, mem);
325         struct ttm_mem_type_manager *old_man = &bdev->man[bo->mem.mem_type];
326         struct ttm_mem_type_manager *new_man = &bdev->man[mem->mem_type];
327         int ret = 0;
328
329         if (old_is_pci || new_is_pci ||
330             ((mem->placement & bo->mem.placement & TTM_PL_MASK_CACHING) == 0)) {
331                 ret = ttm_mem_io_lock(old_man, true);
332                 if (unlikely(ret != 0))
333                         goto out_err;
334                 ttm_bo_unmap_virtual_locked(bo);
335                 ttm_mem_io_unlock(old_man);
336         }
337
338         /*
339          * Create and bind a ttm if required.
340          */
341
342         if (!(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) {
343                 if (bo->ttm == NULL) {
344                         bool zero = !(old_man->flags & TTM_MEMTYPE_FLAG_FIXED);
345                         ret = ttm_tt_create(bo, zero);
346                         if (ret)
347                                 goto out_err;
348                 }
349
350                 ret = ttm_tt_set_placement_caching(bo->ttm, mem->placement);
351                 if (ret)
352                         goto out_err;
353
354                 if (mem->mem_type != TTM_PL_SYSTEM) {
355                         ret = ttm_tt_bind(bo->ttm, mem, ctx);
356                         if (ret)
357                                 goto out_err;
358                 }
359
360                 if (bo->mem.mem_type == TTM_PL_SYSTEM) {
361                         if (bdev->driver->move_notify)
362                                 bdev->driver->move_notify(bo, evict, mem);
363                         bo->mem = *mem;
364                         mem->mm_node = NULL;
365                         goto moved;
366                 }
367         }
368
369         if (bdev->driver->move_notify)
370                 bdev->driver->move_notify(bo, evict, mem);
371
372         if (!(old_man->flags & TTM_MEMTYPE_FLAG_FIXED) &&
373             !(new_man->flags & TTM_MEMTYPE_FLAG_FIXED))
374                 ret = ttm_bo_move_ttm(bo, ctx, mem);
375         else if (bdev->driver->move)
376                 ret = bdev->driver->move(bo, evict, ctx, mem);
377         else
378                 ret = ttm_bo_move_memcpy(bo, ctx, mem);
379
380         if (ret) {
381                 if (bdev->driver->move_notify) {
382                         swap(*mem, bo->mem);
383                         bdev->driver->move_notify(bo, false, mem);
384                         swap(*mem, bo->mem);
385                 }
386
387                 goto out_err;
388         }
389
390 moved:
391         if (bo->evicted) {
392                 if (bdev->driver->invalidate_caches) {
393                         ret = bdev->driver->invalidate_caches(bdev, bo->mem.placement);
394                         if (ret)
395                                 pr_err("Can not flush read caches\n");
396                 }
397                 bo->evicted = false;
398         }
399
400         if (bo->mem.mm_node)
401                 bo->offset = (bo->mem.start << PAGE_SHIFT) +
402                     bdev->man[bo->mem.mem_type].gpu_offset;
403         else
404                 bo->offset = 0;
405
406         ctx->bytes_moved += bo->num_pages << PAGE_SHIFT;
407         return 0;
408
409 out_err:
410         new_man = &bdev->man[bo->mem.mem_type];
411         if (new_man->flags & TTM_MEMTYPE_FLAG_FIXED) {
412                 ttm_tt_destroy(bo->ttm);
413                 bo->ttm = NULL;
414         }
415
416         return ret;
417 }
418
419 /**
420  * Call bo::reserved.
421  * Will release GPU memory type usage on destruction.
422  * This is the place to put in driver specific hooks to release
423  * driver private resources.
424  * Will release the bo::reserved lock.
425  */
426
427 static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo)
428 {
429         if (bo->bdev->driver->move_notify)
430                 bo->bdev->driver->move_notify(bo, false, NULL);
431
432         ttm_tt_destroy(bo->ttm);
433         bo->ttm = NULL;
434         ttm_bo_mem_put(bo, &bo->mem);
435 }
436
437 static int ttm_bo_individualize_resv(struct ttm_buffer_object *bo)
438 {
439         int r;
440
441         if (bo->resv == &bo->ttm_resv)
442                 return 0;
443
444         BUG_ON(!reservation_object_trylock(&bo->ttm_resv));
445
446         r = reservation_object_copy_fences(&bo->ttm_resv, bo->resv);
447         if (r)
448                 reservation_object_unlock(&bo->ttm_resv);
449
450         return r;
451 }
452
453 static void ttm_bo_flush_all_fences(struct ttm_buffer_object *bo)
454 {
455         struct reservation_object_list *fobj;
456         struct dma_fence *fence;
457         int i;
458
459         fobj = reservation_object_get_list(&bo->ttm_resv);
460         fence = reservation_object_get_excl(&bo->ttm_resv);
461         if (fence && !fence->ops->signaled)
462                 dma_fence_enable_sw_signaling(fence);
463
464         for (i = 0; fobj && i < fobj->shared_count; ++i) {
465                 fence = rcu_dereference_protected(fobj->shared[i],
466                                         reservation_object_held(bo->resv));
467
468                 if (!fence->ops->signaled)
469                         dma_fence_enable_sw_signaling(fence);
470         }
471 }
472
473 static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo)
474 {
475         struct ttm_bo_device *bdev = bo->bdev;
476         struct ttm_bo_global *glob = bdev->glob;
477         int ret;
478
479         ret = ttm_bo_individualize_resv(bo);
480         if (ret) {
481                 /* Last resort, if we fail to allocate memory for the
482                  * fences block for the BO to become idle
483                  */
484                 reservation_object_wait_timeout_rcu(bo->resv, true, false,
485                                                     30 * HZ);
486                 spin_lock(&glob->lru_lock);
487                 goto error;
488         }
489
490         spin_lock(&glob->lru_lock);
491         ret = reservation_object_trylock(bo->resv) ? 0 : -EBUSY;
492         if (!ret) {
493                 if (reservation_object_test_signaled_rcu(&bo->ttm_resv, true)) {
494                         ttm_bo_del_from_lru(bo);
495                         spin_unlock(&glob->lru_lock);
496                         if (bo->resv != &bo->ttm_resv)
497                                 reservation_object_unlock(&bo->ttm_resv);
498
499                         ttm_bo_cleanup_memtype_use(bo);
500                         reservation_object_unlock(bo->resv);
501                         return;
502                 }
503
504                 ttm_bo_flush_all_fences(bo);
505
506                 /*
507                  * Make NO_EVICT bos immediately available to
508                  * shrinkers, now that they are queued for
509                  * destruction.
510                  */
511                 if (bo->mem.placement & TTM_PL_FLAG_NO_EVICT) {
512                         bo->mem.placement &= ~TTM_PL_FLAG_NO_EVICT;
513                         ttm_bo_add_to_lru(bo);
514                 }
515
516                 reservation_object_unlock(bo->resv);
517         }
518         if (bo->resv != &bo->ttm_resv)
519                 reservation_object_unlock(&bo->ttm_resv);
520
521 error:
522         kref_get(&bo->list_kref);
523         list_add_tail(&bo->ddestroy, &bdev->ddestroy);
524         spin_unlock(&glob->lru_lock);
525
526         schedule_delayed_work(&bdev->wq,
527                               ((HZ / 100) < 1) ? 1 : HZ / 100);
528 }
529
530 /**
531  * function ttm_bo_cleanup_refs
532  * If bo idle, remove from delayed- and lru lists, and unref.
533  * If not idle, do nothing.
534  *
535  * Must be called with lru_lock and reservation held, this function
536  * will drop the lru lock and optionally the reservation lock before returning.
537  *
538  * @interruptible         Any sleeps should occur interruptibly.
539  * @no_wait_gpu           Never wait for gpu. Return -EBUSY instead.
540  * @unlock_resv           Unlock the reservation lock as well.
541  */
542
543 static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo,
544                                bool interruptible, bool no_wait_gpu,
545                                bool unlock_resv)
546 {
547         struct ttm_bo_global *glob = bo->bdev->glob;
548         struct reservation_object *resv;
549         int ret;
550
551         if (unlikely(list_empty(&bo->ddestroy)))
552                 resv = bo->resv;
553         else
554                 resv = &bo->ttm_resv;
555
556         if (reservation_object_test_signaled_rcu(resv, true))
557                 ret = 0;
558         else
559                 ret = -EBUSY;
560
561         if (ret && !no_wait_gpu) {
562                 long lret;
563
564                 if (unlock_resv)
565                         reservation_object_unlock(bo->resv);
566                 spin_unlock(&glob->lru_lock);
567
568                 lret = reservation_object_wait_timeout_rcu(resv, true,
569                                                            interruptible,
570                                                            30 * HZ);
571
572                 if (lret < 0)
573                         return lret;
574                 else if (lret == 0)
575                         return -EBUSY;
576
577                 spin_lock(&glob->lru_lock);
578                 if (unlock_resv && !reservation_object_trylock(bo->resv)) {
579                         /*
580                          * We raced, and lost, someone else holds the reservation now,
581                          * and is probably busy in ttm_bo_cleanup_memtype_use.
582                          *
583                          * Even if it's not the case, because we finished waiting any
584                          * delayed destruction would succeed, so just return success
585                          * here.
586                          */
587                         spin_unlock(&glob->lru_lock);
588                         return 0;
589                 }
590                 ret = 0;
591         }
592
593         if (ret || unlikely(list_empty(&bo->ddestroy))) {
594                 if (unlock_resv)
595                         reservation_object_unlock(bo->resv);
596                 spin_unlock(&glob->lru_lock);
597                 return ret;
598         }
599
600         ttm_bo_del_from_lru(bo);
601         list_del_init(&bo->ddestroy);
602         kref_put(&bo->list_kref, ttm_bo_ref_bug);
603
604         spin_unlock(&glob->lru_lock);
605         ttm_bo_cleanup_memtype_use(bo);
606
607         if (unlock_resv)
608                 reservation_object_unlock(bo->resv);
609
610         return 0;
611 }
612
613 /**
614  * Traverse the delayed list, and call ttm_bo_cleanup_refs on all
615  * encountered buffers.
616  */
617 static bool ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all)
618 {
619         struct ttm_bo_global *glob = bdev->glob;
620         struct list_head removed;
621         bool empty;
622
623         INIT_LIST_HEAD(&removed);
624
625         spin_lock(&glob->lru_lock);
626         while (!list_empty(&bdev->ddestroy)) {
627                 struct ttm_buffer_object *bo;
628
629                 bo = list_first_entry(&bdev->ddestroy, struct ttm_buffer_object,
630                                       ddestroy);
631                 kref_get(&bo->list_kref);
632                 list_move_tail(&bo->ddestroy, &removed);
633
634                 if (remove_all || bo->resv != &bo->ttm_resv) {
635                         spin_unlock(&glob->lru_lock);
636                         reservation_object_lock(bo->resv, NULL);
637
638                         spin_lock(&glob->lru_lock);
639                         ttm_bo_cleanup_refs(bo, false, !remove_all, true);
640
641                 } else if (reservation_object_trylock(bo->resv)) {
642                         ttm_bo_cleanup_refs(bo, false, !remove_all, true);
643                 } else {
644                         spin_unlock(&glob->lru_lock);
645                 }
646
647                 kref_put(&bo->list_kref, ttm_bo_release_list);
648                 spin_lock(&glob->lru_lock);
649         }
650         list_splice_tail(&removed, &bdev->ddestroy);
651         empty = list_empty(&bdev->ddestroy);
652         spin_unlock(&glob->lru_lock);
653
654         return empty;
655 }
656
657 static void ttm_bo_delayed_workqueue(struct work_struct *work)
658 {
659         struct ttm_bo_device *bdev =
660             container_of(work, struct ttm_bo_device, wq.work);
661
662         if (!ttm_bo_delayed_delete(bdev, false))
663                 schedule_delayed_work(&bdev->wq,
664                                       ((HZ / 100) < 1) ? 1 : HZ / 100);
665 }
666
667 static void ttm_bo_release(struct kref *kref)
668 {
669         struct ttm_buffer_object *bo =
670             container_of(kref, struct ttm_buffer_object, kref);
671         struct ttm_bo_device *bdev = bo->bdev;
672         struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type];
673
674         if (bo->bdev->driver->release_notify)
675                 bo->bdev->driver->release_notify(bo);
676
677         drm_vma_offset_remove(&bdev->vma_manager, &bo->vma_node);
678         ttm_mem_io_lock(man, false);
679         ttm_mem_io_free_vm(bo);
680         ttm_mem_io_unlock(man);
681         ttm_bo_cleanup_refs_or_queue(bo);
682         kref_put(&bo->list_kref, ttm_bo_release_list);
683 }
684
685 void ttm_bo_put(struct ttm_buffer_object *bo)
686 {
687         kref_put(&bo->kref, ttm_bo_release);
688 }
689 EXPORT_SYMBOL(ttm_bo_put);
690
691 int ttm_bo_lock_delayed_workqueue(struct ttm_bo_device *bdev)
692 {
693         return cancel_delayed_work_sync(&bdev->wq);
694 }
695 EXPORT_SYMBOL(ttm_bo_lock_delayed_workqueue);
696
697 void ttm_bo_unlock_delayed_workqueue(struct ttm_bo_device *bdev, int resched)
698 {
699         if (resched)
700                 schedule_delayed_work(&bdev->wq,
701                                       ((HZ / 100) < 1) ? 1 : HZ / 100);
702 }
703 EXPORT_SYMBOL(ttm_bo_unlock_delayed_workqueue);
704
705 static int ttm_bo_evict(struct ttm_buffer_object *bo,
706                         struct ttm_operation_ctx *ctx)
707 {
708         struct ttm_bo_device *bdev = bo->bdev;
709         struct ttm_mem_reg evict_mem;
710         struct ttm_placement placement;
711         int ret = 0;
712
713         reservation_object_assert_held(bo->resv);
714
715         placement.num_placement = 0;
716         placement.num_busy_placement = 0;
717         bdev->driver->evict_flags(bo, &placement);
718
719         if (!placement.num_placement && !placement.num_busy_placement) {
720                 ret = ttm_bo_pipeline_gutting(bo);
721                 if (ret)
722                         return ret;
723
724                 return ttm_tt_create(bo, false);
725         }
726
727         evict_mem = bo->mem;
728         evict_mem.mm_node = NULL;
729         evict_mem.bus.io_reserved_vm = false;
730         evict_mem.bus.io_reserved_count = 0;
731
732         ret = ttm_bo_mem_space(bo, &placement, &evict_mem, ctx);
733         if (ret) {
734                 if (ret != -ERESTARTSYS) {
735                         pr_err("Failed to find memory space for buffer 0x%p eviction\n",
736                                bo);
737                         ttm_bo_mem_space_debug(bo, &placement);
738                 }
739                 goto out;
740         }
741
742         ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, ctx);
743         if (unlikely(ret)) {
744                 if (ret != -ERESTARTSYS)
745                         pr_err("Buffer eviction failed\n");
746                 ttm_bo_mem_put(bo, &evict_mem);
747                 goto out;
748         }
749         bo->evicted = true;
750 out:
751         return ret;
752 }
753
754 bool ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
755                               const struct ttm_place *place)
756 {
757         /* Don't evict this BO if it's outside of the
758          * requested placement range
759          */
760         if (place->fpfn >= (bo->mem.start + bo->mem.size) ||
761             (place->lpfn && place->lpfn <= bo->mem.start))
762                 return false;
763
764         return true;
765 }
766 EXPORT_SYMBOL(ttm_bo_eviction_valuable);
767
768 /**
769  * Check the target bo is allowable to be evicted or swapout, including cases:
770  *
771  * a. if share same reservation object with ctx->resv, have assumption
772  * reservation objects should already be locked, so not lock again and
773  * return true directly when either the opreation allow_reserved_eviction
774  * or the target bo already is in delayed free list;
775  *
776  * b. Otherwise, trylock it.
777  */
778 static bool ttm_bo_evict_swapout_allowable(struct ttm_buffer_object *bo,
779                         struct ttm_operation_ctx *ctx, bool *locked, bool *busy)
780 {
781         bool ret = false;
782
783         if (bo->resv == ctx->resv) {
784                 reservation_object_assert_held(bo->resv);
785                 if (ctx->flags & TTM_OPT_FLAG_ALLOW_RES_EVICT
786                     || !list_empty(&bo->ddestroy))
787                         ret = true;
788                 *locked = false;
789                 if (busy)
790                         *busy = false;
791         } else {
792                 ret = reservation_object_trylock(bo->resv);
793                 *locked = ret;
794                 if (busy)
795                         *busy = !ret;
796         }
797
798         return ret;
799 }
800
801 /**
802  * ttm_mem_evict_wait_busy - wait for a busy BO to become available
803  *
804  * @busy_bo: BO which couldn't be locked with trylock
805  * @ctx: operation context
806  * @ticket: acquire ticket
807  *
808  * Try to lock a busy buffer object to avoid failing eviction.
809  */
810 static int ttm_mem_evict_wait_busy(struct ttm_buffer_object *busy_bo,
811                                    struct ttm_operation_ctx *ctx,
812                                    struct ww_acquire_ctx *ticket)
813 {
814         int r;
815
816         if (!busy_bo || !ticket)
817                 return -EBUSY;
818
819         if (ctx->interruptible)
820                 r = reservation_object_lock_interruptible(busy_bo->resv,
821                                                           ticket);
822         else
823                 r = reservation_object_lock(busy_bo->resv, ticket);
824
825         /*
826          * TODO: It would be better to keep the BO locked until allocation is at
827          * least tried one more time, but that would mean a much larger rework
828          * of TTM.
829          */
830         if (!r)
831                 reservation_object_unlock(busy_bo->resv);
832
833         return r == -EDEADLK ? -EBUSY : r;
834 }
835
836 static int ttm_mem_evict_first(struct ttm_bo_device *bdev,
837                                uint32_t mem_type,
838                                const struct ttm_place *place,
839                                struct ttm_operation_ctx *ctx,
840                                struct ww_acquire_ctx *ticket)
841 {
842         struct ttm_buffer_object *bo = NULL, *busy_bo = NULL;
843         struct ttm_bo_global *glob = bdev->glob;
844         struct ttm_mem_type_manager *man = &bdev->man[mem_type];
845         bool locked = false;
846         unsigned i;
847         int ret;
848
849         spin_lock(&glob->lru_lock);
850         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
851                 list_for_each_entry(bo, &man->lru[i], lru) {
852                         bool busy;
853
854                         if (!ttm_bo_evict_swapout_allowable(bo, ctx, &locked,
855                                                             &busy)) {
856                                 if (busy && !busy_bo &&
857                                     bo->resv->lock.ctx != ticket)
858                                         busy_bo = bo;
859                                 continue;
860                         }
861
862                         if (place && !bdev->driver->eviction_valuable(bo,
863                                                                       place)) {
864                                 if (locked)
865                                         reservation_object_unlock(bo->resv);
866                                 continue;
867                         }
868                         break;
869                 }
870
871                 /* If the inner loop terminated early, we have our candidate */
872                 if (&bo->lru != &man->lru[i])
873                         break;
874
875                 bo = NULL;
876         }
877
878         if (!bo) {
879                 if (busy_bo)
880                         ttm_bo_get(busy_bo);
881                 spin_unlock(&glob->lru_lock);
882                 ret = ttm_mem_evict_wait_busy(busy_bo, ctx, ticket);
883                 if (busy_bo)
884                         ttm_bo_put(busy_bo);
885                 return ret;
886         }
887
888         kref_get(&bo->list_kref);
889
890         if (!list_empty(&bo->ddestroy)) {
891                 ret = ttm_bo_cleanup_refs(bo, ctx->interruptible,
892                                           ctx->no_wait_gpu, locked);
893                 kref_put(&bo->list_kref, ttm_bo_release_list);
894                 return ret;
895         }
896
897         ttm_bo_del_from_lru(bo);
898         spin_unlock(&glob->lru_lock);
899
900         ret = ttm_bo_evict(bo, ctx);
901         if (locked) {
902                 ttm_bo_unreserve(bo);
903         } else {
904                 spin_lock(&glob->lru_lock);
905                 ttm_bo_add_to_lru(bo);
906                 spin_unlock(&glob->lru_lock);
907         }
908
909         kref_put(&bo->list_kref, ttm_bo_release_list);
910         return ret;
911 }
912
913 void ttm_bo_mem_put(struct ttm_buffer_object *bo, struct ttm_mem_reg *mem)
914 {
915         struct ttm_mem_type_manager *man = &bo->bdev->man[mem->mem_type];
916
917         if (mem->mm_node)
918                 (*man->func->put_node)(man, mem);
919 }
920 EXPORT_SYMBOL(ttm_bo_mem_put);
921
922 /**
923  * Add the last move fence to the BO and reserve a new shared slot.
924  */
925 static int ttm_bo_add_move_fence(struct ttm_buffer_object *bo,
926                                  struct ttm_mem_type_manager *man,
927                                  struct ttm_mem_reg *mem)
928 {
929         struct dma_fence *fence;
930         int ret;
931
932         spin_lock(&man->move_lock);
933         fence = dma_fence_get(man->move);
934         spin_unlock(&man->move_lock);
935
936         if (fence) {
937                 reservation_object_add_shared_fence(bo->resv, fence);
938
939                 ret = reservation_object_reserve_shared(bo->resv, 1);
940                 if (unlikely(ret)) {
941                         dma_fence_put(fence);
942                         return ret;
943                 }
944
945                 dma_fence_put(bo->moving);
946                 bo->moving = fence;
947         }
948
949         return 0;
950 }
951
952 /**
953  * Repeatedly evict memory from the LRU for @mem_type until we create enough
954  * space, or we've evicted everything and there isn't enough space.
955  */
956 static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo,
957                                   const struct ttm_place *place,
958                                   struct ttm_mem_reg *mem,
959                                   struct ttm_operation_ctx *ctx)
960 {
961         struct ttm_bo_device *bdev = bo->bdev;
962         struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
963         int ret;
964
965         do {
966                 ret = (*man->func->get_node)(man, bo, place, mem);
967                 if (unlikely(ret != 0))
968                         return ret;
969                 if (mem->mm_node)
970                         break;
971                 ret = ttm_mem_evict_first(bdev, mem->mem_type, place, ctx,
972                                           bo->resv->lock.ctx);
973                 if (unlikely(ret != 0))
974                         return ret;
975         } while (1);
976
977         return ttm_bo_add_move_fence(bo, man, mem);
978 }
979
980 static uint32_t ttm_bo_select_caching(struct ttm_mem_type_manager *man,
981                                       uint32_t cur_placement,
982                                       uint32_t proposed_placement)
983 {
984         uint32_t caching = proposed_placement & TTM_PL_MASK_CACHING;
985         uint32_t result = proposed_placement & ~TTM_PL_MASK_CACHING;
986
987         /**
988          * Keep current caching if possible.
989          */
990
991         if ((cur_placement & caching) != 0)
992                 result |= (cur_placement & caching);
993         else if ((man->default_caching & caching) != 0)
994                 result |= man->default_caching;
995         else if ((TTM_PL_FLAG_CACHED & caching) != 0)
996                 result |= TTM_PL_FLAG_CACHED;
997         else if ((TTM_PL_FLAG_WC & caching) != 0)
998                 result |= TTM_PL_FLAG_WC;
999         else if ((TTM_PL_FLAG_UNCACHED & caching) != 0)
1000                 result |= TTM_PL_FLAG_UNCACHED;
1001
1002         return result;
1003 }
1004
1005 static bool ttm_bo_mt_compatible(struct ttm_mem_type_manager *man,
1006                                  uint32_t mem_type,
1007                                  const struct ttm_place *place,
1008                                  uint32_t *masked_placement)
1009 {
1010         uint32_t cur_flags = ttm_bo_type_flags(mem_type);
1011
1012         if ((cur_flags & place->flags & TTM_PL_MASK_MEM) == 0)
1013                 return false;
1014
1015         if ((place->flags & man->available_caching) == 0)
1016                 return false;
1017
1018         cur_flags |= (place->flags & man->available_caching);
1019
1020         *masked_placement = cur_flags;
1021         return true;
1022 }
1023
1024 /**
1025  * ttm_bo_mem_placement - check if placement is compatible
1026  * @bo: BO to find memory for
1027  * @place: where to search
1028  * @mem: the memory object to fill in
1029  * @ctx: operation context
1030  *
1031  * Check if placement is compatible and fill in mem structure.
1032  * Returns -EBUSY if placement won't work or negative error code.
1033  * 0 when placement can be used.
1034  */
1035 static int ttm_bo_mem_placement(struct ttm_buffer_object *bo,
1036                                 const struct ttm_place *place,
1037                                 struct ttm_mem_reg *mem,
1038                                 struct ttm_operation_ctx *ctx)
1039 {
1040         struct ttm_bo_device *bdev = bo->bdev;
1041         uint32_t mem_type = TTM_PL_SYSTEM;
1042         struct ttm_mem_type_manager *man;
1043         uint32_t cur_flags = 0;
1044         int ret;
1045
1046         ret = ttm_mem_type_from_place(place, &mem_type);
1047         if (ret)
1048                 return ret;
1049
1050         man = &bdev->man[mem_type];
1051         if (!man->has_type || !man->use_type)
1052                 return -EBUSY;
1053
1054         if (!ttm_bo_mt_compatible(man, mem_type, place, &cur_flags))
1055                 return -EBUSY;
1056
1057         cur_flags = ttm_bo_select_caching(man, bo->mem.placement, cur_flags);
1058         /*
1059          * Use the access and other non-mapping-related flag bits from
1060          * the memory placement flags to the current flags
1061          */
1062         ttm_flag_masked(&cur_flags, place->flags, ~TTM_PL_MASK_MEMTYPE);
1063
1064         mem->mem_type = mem_type;
1065         mem->placement = cur_flags;
1066
1067         if (bo->mem.mem_type < mem_type && !list_empty(&bo->lru)) {
1068                 spin_lock(&bo->bdev->glob->lru_lock);
1069                 ttm_bo_del_from_lru(bo);
1070                 ttm_bo_add_mem_to_lru(bo, mem);
1071                 spin_unlock(&bo->bdev->glob->lru_lock);
1072         }
1073
1074         return 0;
1075 }
1076
1077 /**
1078  * Creates space for memory region @mem according to its type.
1079  *
1080  * This function first searches for free space in compatible memory types in
1081  * the priority order defined by the driver.  If free space isn't found, then
1082  * ttm_bo_mem_force_space is attempted in priority order to evict and find
1083  * space.
1084  */
1085 int ttm_bo_mem_space(struct ttm_buffer_object *bo,
1086                         struct ttm_placement *placement,
1087                         struct ttm_mem_reg *mem,
1088                         struct ttm_operation_ctx *ctx)
1089 {
1090         struct ttm_bo_device *bdev = bo->bdev;
1091         bool type_found = false;
1092         int i, ret;
1093
1094         ret = reservation_object_reserve_shared(bo->resv, 1);
1095         if (unlikely(ret))
1096                 return ret;
1097
1098         mem->mm_node = NULL;
1099         for (i = 0; i < placement->num_placement; ++i) {
1100                 const struct ttm_place *place = &placement->placement[i];
1101                 struct ttm_mem_type_manager *man;
1102
1103                 ret = ttm_bo_mem_placement(bo, place, mem, ctx);
1104                 if (ret == -EBUSY)
1105                         continue;
1106                 if (ret)
1107                         goto error;
1108
1109                 type_found = true;
1110                 mem->mm_node = NULL;
1111                 if (mem->mem_type == TTM_PL_SYSTEM)
1112                         return 0;
1113
1114                 man = &bdev->man[mem->mem_type];
1115                 ret = (*man->func->get_node)(man, bo, place, mem);
1116                 if (unlikely(ret))
1117                         goto error;
1118
1119                 if (mem->mm_node) {
1120                         ret = ttm_bo_add_move_fence(bo, man, mem);
1121                         if (unlikely(ret)) {
1122                                 (*man->func->put_node)(man, mem);
1123                                 goto error;
1124                         }
1125                         return 0;
1126                 }
1127         }
1128
1129         for (i = 0; i < placement->num_busy_placement; ++i) {
1130                 const struct ttm_place *place = &placement->busy_placement[i];
1131
1132                 ret = ttm_bo_mem_placement(bo, place, mem, ctx);
1133                 if (ret == -EBUSY)
1134                         continue;
1135                 if (ret)
1136                         goto error;
1137
1138                 type_found = true;
1139                 mem->mm_node = NULL;
1140                 if (mem->mem_type == TTM_PL_SYSTEM)
1141                         return 0;
1142
1143                 ret = ttm_bo_mem_force_space(bo, place, mem, ctx);
1144                 if (ret == 0 && mem->mm_node)
1145                         return 0;
1146
1147                 if (ret && ret != -EBUSY)
1148                         goto error;
1149         }
1150
1151         ret = -ENOMEM;
1152         if (!type_found) {
1153                 pr_err(TTM_PFX "No compatible memory type found\n");
1154                 ret = -EINVAL;
1155         }
1156
1157 error:
1158         if (bo->mem.mem_type == TTM_PL_SYSTEM && !list_empty(&bo->lru)) {
1159                 spin_lock(&bo->bdev->glob->lru_lock);
1160                 ttm_bo_move_to_lru_tail(bo, NULL);
1161                 spin_unlock(&bo->bdev->glob->lru_lock);
1162         }
1163
1164         return ret;
1165 }
1166 EXPORT_SYMBOL(ttm_bo_mem_space);
1167
1168 static int ttm_bo_move_buffer(struct ttm_buffer_object *bo,
1169                               struct ttm_placement *placement,
1170                               struct ttm_operation_ctx *ctx)
1171 {
1172         int ret = 0;
1173         struct ttm_mem_reg mem;
1174
1175         reservation_object_assert_held(bo->resv);
1176
1177         mem.num_pages = bo->num_pages;
1178         mem.size = mem.num_pages << PAGE_SHIFT;
1179         mem.page_alignment = bo->mem.page_alignment;
1180         mem.bus.io_reserved_vm = false;
1181         mem.bus.io_reserved_count = 0;
1182         /*
1183          * Determine where to move the buffer.
1184          */
1185         ret = ttm_bo_mem_space(bo, placement, &mem, ctx);
1186         if (ret)
1187                 goto out_unlock;
1188         ret = ttm_bo_handle_move_mem(bo, &mem, false, ctx);
1189 out_unlock:
1190         if (ret && mem.mm_node)
1191                 ttm_bo_mem_put(bo, &mem);
1192         return ret;
1193 }
1194
1195 static bool ttm_bo_places_compat(const struct ttm_place *places,
1196                                  unsigned num_placement,
1197                                  struct ttm_mem_reg *mem,
1198                                  uint32_t *new_flags)
1199 {
1200         unsigned i;
1201
1202         for (i = 0; i < num_placement; i++) {
1203                 const struct ttm_place *heap = &places[i];
1204
1205                 if (mem->mm_node && (mem->start < heap->fpfn ||
1206                      (heap->lpfn != 0 && (mem->start + mem->num_pages) > heap->lpfn)))
1207                         continue;
1208
1209                 *new_flags = heap->flags;
1210                 if ((*new_flags & mem->placement & TTM_PL_MASK_CACHING) &&
1211                     (*new_flags & mem->placement & TTM_PL_MASK_MEM) &&
1212                     (!(*new_flags & TTM_PL_FLAG_CONTIGUOUS) ||
1213                      (mem->placement & TTM_PL_FLAG_CONTIGUOUS)))
1214                         return true;
1215         }
1216         return false;
1217 }
1218
1219 bool ttm_bo_mem_compat(struct ttm_placement *placement,
1220                        struct ttm_mem_reg *mem,
1221                        uint32_t *new_flags)
1222 {
1223         if (ttm_bo_places_compat(placement->placement, placement->num_placement,
1224                                  mem, new_flags))
1225                 return true;
1226
1227         if ((placement->busy_placement != placement->placement ||
1228              placement->num_busy_placement > placement->num_placement) &&
1229             ttm_bo_places_compat(placement->busy_placement,
1230                                  placement->num_busy_placement,
1231                                  mem, new_flags))
1232                 return true;
1233
1234         return false;
1235 }
1236 EXPORT_SYMBOL(ttm_bo_mem_compat);
1237
1238 int ttm_bo_validate(struct ttm_buffer_object *bo,
1239                     struct ttm_placement *placement,
1240                     struct ttm_operation_ctx *ctx)
1241 {
1242         int ret;
1243         uint32_t new_flags;
1244
1245         reservation_object_assert_held(bo->resv);
1246         /*
1247          * Check whether we need to move buffer.
1248          */
1249         if (!ttm_bo_mem_compat(placement, &bo->mem, &new_flags)) {
1250                 ret = ttm_bo_move_buffer(bo, placement, ctx);
1251                 if (ret)
1252                         return ret;
1253         } else {
1254                 /*
1255                  * Use the access and other non-mapping-related flag bits from
1256                  * the compatible memory placement flags to the active flags
1257                  */
1258                 ttm_flag_masked(&bo->mem.placement, new_flags,
1259                                 ~TTM_PL_MASK_MEMTYPE);
1260         }
1261         /*
1262          * We might need to add a TTM.
1263          */
1264         if (bo->mem.mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) {
1265                 ret = ttm_tt_create(bo, true);
1266                 if (ret)
1267                         return ret;
1268         }
1269         return 0;
1270 }
1271 EXPORT_SYMBOL(ttm_bo_validate);
1272
1273 int ttm_bo_init_reserved(struct ttm_bo_device *bdev,
1274                          struct ttm_buffer_object *bo,
1275                          unsigned long size,
1276                          enum ttm_bo_type type,
1277                          struct ttm_placement *placement,
1278                          uint32_t page_alignment,
1279                          struct ttm_operation_ctx *ctx,
1280                          size_t acc_size,
1281                          struct sg_table *sg,
1282                          struct reservation_object *resv,
1283                          void (*destroy) (struct ttm_buffer_object *))
1284 {
1285         int ret = 0;
1286         unsigned long num_pages;
1287         struct ttm_mem_global *mem_glob = bdev->glob->mem_glob;
1288         bool locked;
1289
1290         ret = ttm_mem_global_alloc(mem_glob, acc_size, ctx);
1291         if (ret) {
1292                 pr_err("Out of kernel memory\n");
1293                 if (destroy)
1294                         (*destroy)(bo);
1295                 else
1296                         kfree(bo);
1297                 return -ENOMEM;
1298         }
1299
1300         num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
1301         if (num_pages == 0) {
1302                 pr_err("Illegal buffer object size\n");
1303                 if (destroy)
1304                         (*destroy)(bo);
1305                 else
1306                         kfree(bo);
1307                 ttm_mem_global_free(mem_glob, acc_size);
1308                 return -EINVAL;
1309         }
1310         bo->destroy = destroy ? destroy : ttm_bo_default_destroy;
1311
1312         kref_init(&bo->kref);
1313         kref_init(&bo->list_kref);
1314         atomic_set(&bo->cpu_writers, 0);
1315         INIT_LIST_HEAD(&bo->lru);
1316         INIT_LIST_HEAD(&bo->ddestroy);
1317         INIT_LIST_HEAD(&bo->swap);
1318         INIT_LIST_HEAD(&bo->io_reserve_lru);
1319         mutex_init(&bo->wu_mutex);
1320         bo->bdev = bdev;
1321         bo->type = type;
1322         bo->num_pages = num_pages;
1323         bo->mem.size = num_pages << PAGE_SHIFT;
1324         bo->mem.mem_type = TTM_PL_SYSTEM;
1325         bo->mem.num_pages = bo->num_pages;
1326         bo->mem.mm_node = NULL;
1327         bo->mem.page_alignment = page_alignment;
1328         bo->mem.bus.io_reserved_vm = false;
1329         bo->mem.bus.io_reserved_count = 0;
1330         bo->moving = NULL;
1331         bo->mem.placement = (TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED);
1332         bo->acc_size = acc_size;
1333         bo->sg = sg;
1334         if (resv) {
1335                 bo->resv = resv;
1336                 reservation_object_assert_held(bo->resv);
1337         } else {
1338                 bo->resv = &bo->ttm_resv;
1339         }
1340         reservation_object_init(&bo->ttm_resv);
1341         atomic_inc(&bo->bdev->glob->bo_count);
1342         drm_vma_node_reset(&bo->vma_node);
1343
1344         /*
1345          * For ttm_bo_type_device buffers, allocate
1346          * address space from the device.
1347          */
1348         if (bo->type == ttm_bo_type_device ||
1349             bo->type == ttm_bo_type_sg)
1350                 ret = drm_vma_offset_add(&bdev->vma_manager, &bo->vma_node,
1351                                          bo->mem.num_pages);
1352
1353         /* passed reservation objects should already be locked,
1354          * since otherwise lockdep will be angered in radeon.
1355          */
1356         if (!resv) {
1357                 locked = reservation_object_trylock(bo->resv);
1358                 WARN_ON(!locked);
1359         }
1360
1361         if (likely(!ret))
1362                 ret = ttm_bo_validate(bo, placement, ctx);
1363
1364         if (unlikely(ret)) {
1365                 if (!resv)
1366                         ttm_bo_unreserve(bo);
1367
1368                 ttm_bo_put(bo);
1369                 return ret;
1370         }
1371
1372         if (resv && !(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) {
1373                 spin_lock(&bdev->glob->lru_lock);
1374                 ttm_bo_add_to_lru(bo);
1375                 spin_unlock(&bdev->glob->lru_lock);
1376         }
1377
1378         return ret;
1379 }
1380 EXPORT_SYMBOL(ttm_bo_init_reserved);
1381
1382 int ttm_bo_init(struct ttm_bo_device *bdev,
1383                 struct ttm_buffer_object *bo,
1384                 unsigned long size,
1385                 enum ttm_bo_type type,
1386                 struct ttm_placement *placement,
1387                 uint32_t page_alignment,
1388                 bool interruptible,
1389                 size_t acc_size,
1390                 struct sg_table *sg,
1391                 struct reservation_object *resv,
1392                 void (*destroy) (struct ttm_buffer_object *))
1393 {
1394         struct ttm_operation_ctx ctx = { interruptible, false };
1395         int ret;
1396
1397         ret = ttm_bo_init_reserved(bdev, bo, size, type, placement,
1398                                    page_alignment, &ctx, acc_size,
1399                                    sg, resv, destroy);
1400         if (ret)
1401                 return ret;
1402
1403         if (!resv)
1404                 ttm_bo_unreserve(bo);
1405
1406         return 0;
1407 }
1408 EXPORT_SYMBOL(ttm_bo_init);
1409
1410 size_t ttm_bo_acc_size(struct ttm_bo_device *bdev,
1411                        unsigned long bo_size,
1412                        unsigned struct_size)
1413 {
1414         unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT;
1415         size_t size = 0;
1416
1417         size += ttm_round_pot(struct_size);
1418         size += ttm_round_pot(npages * sizeof(void *));
1419         size += ttm_round_pot(sizeof(struct ttm_tt));
1420         return size;
1421 }
1422 EXPORT_SYMBOL(ttm_bo_acc_size);
1423
1424 size_t ttm_bo_dma_acc_size(struct ttm_bo_device *bdev,
1425                            unsigned long bo_size,
1426                            unsigned struct_size)
1427 {
1428         unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT;
1429         size_t size = 0;
1430
1431         size += ttm_round_pot(struct_size);
1432         size += ttm_round_pot(npages * (2*sizeof(void *) + sizeof(dma_addr_t)));
1433         size += ttm_round_pot(sizeof(struct ttm_dma_tt));
1434         return size;
1435 }
1436 EXPORT_SYMBOL(ttm_bo_dma_acc_size);
1437
1438 int ttm_bo_create(struct ttm_bo_device *bdev,
1439                         unsigned long size,
1440                         enum ttm_bo_type type,
1441                         struct ttm_placement *placement,
1442                         uint32_t page_alignment,
1443                         bool interruptible,
1444                         struct ttm_buffer_object **p_bo)
1445 {
1446         struct ttm_buffer_object *bo;
1447         size_t acc_size;
1448         int ret;
1449
1450         bo = kzalloc(sizeof(*bo), GFP_KERNEL);
1451         if (unlikely(bo == NULL))
1452                 return -ENOMEM;
1453
1454         acc_size = ttm_bo_acc_size(bdev, size, sizeof(struct ttm_buffer_object));
1455         ret = ttm_bo_init(bdev, bo, size, type, placement, page_alignment,
1456                           interruptible, acc_size,
1457                           NULL, NULL, NULL);
1458         if (likely(ret == 0))
1459                 *p_bo = bo;
1460
1461         return ret;
1462 }
1463 EXPORT_SYMBOL(ttm_bo_create);
1464
1465 static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev,
1466                                    unsigned mem_type)
1467 {
1468         struct ttm_operation_ctx ctx = {
1469                 .interruptible = false,
1470                 .no_wait_gpu = false,
1471                 .flags = TTM_OPT_FLAG_FORCE_ALLOC
1472         };
1473         struct ttm_mem_type_manager *man = &bdev->man[mem_type];
1474         struct ttm_bo_global *glob = bdev->glob;
1475         struct dma_fence *fence;
1476         int ret;
1477         unsigned i;
1478
1479         /*
1480          * Can't use standard list traversal since we're unlocking.
1481          */
1482
1483         spin_lock(&glob->lru_lock);
1484         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
1485                 while (!list_empty(&man->lru[i])) {
1486                         spin_unlock(&glob->lru_lock);
1487                         ret = ttm_mem_evict_first(bdev, mem_type, NULL, &ctx,
1488                                                   NULL);
1489                         if (ret)
1490                                 return ret;
1491                         spin_lock(&glob->lru_lock);
1492                 }
1493         }
1494         spin_unlock(&glob->lru_lock);
1495
1496         spin_lock(&man->move_lock);
1497         fence = dma_fence_get(man->move);
1498         spin_unlock(&man->move_lock);
1499
1500         if (fence) {
1501                 ret = dma_fence_wait(fence, false);
1502                 dma_fence_put(fence);
1503                 if (ret)
1504                         return ret;
1505         }
1506
1507         return 0;
1508 }
1509
1510 int ttm_bo_clean_mm(struct ttm_bo_device *bdev, unsigned mem_type)
1511 {
1512         struct ttm_mem_type_manager *man;
1513         int ret = -EINVAL;
1514
1515         if (mem_type >= TTM_NUM_MEM_TYPES) {
1516                 pr_err("Illegal memory type %d\n", mem_type);
1517                 return ret;
1518         }
1519         man = &bdev->man[mem_type];
1520
1521         if (!man->has_type) {
1522                 pr_err("Trying to take down uninitialized memory manager type %u\n",
1523                        mem_type);
1524                 return ret;
1525         }
1526
1527         man->use_type = false;
1528         man->has_type = false;
1529
1530         ret = 0;
1531         if (mem_type > 0) {
1532                 ret = ttm_bo_force_list_clean(bdev, mem_type);
1533                 if (ret) {
1534                         pr_err("Cleanup eviction failed\n");
1535                         return ret;
1536                 }
1537
1538                 ret = (*man->func->takedown)(man);
1539         }
1540
1541         dma_fence_put(man->move);
1542         man->move = NULL;
1543
1544         return ret;
1545 }
1546 EXPORT_SYMBOL(ttm_bo_clean_mm);
1547
1548 int ttm_bo_evict_mm(struct ttm_bo_device *bdev, unsigned mem_type)
1549 {
1550         struct ttm_mem_type_manager *man = &bdev->man[mem_type];
1551
1552         if (mem_type == 0 || mem_type >= TTM_NUM_MEM_TYPES) {
1553                 pr_err("Illegal memory manager memory type %u\n", mem_type);
1554                 return -EINVAL;
1555         }
1556
1557         if (!man->has_type) {
1558                 pr_err("Memory type %u has not been initialized\n", mem_type);
1559                 return 0;
1560         }
1561
1562         return ttm_bo_force_list_clean(bdev, mem_type);
1563 }
1564 EXPORT_SYMBOL(ttm_bo_evict_mm);
1565
1566 int ttm_bo_init_mm(struct ttm_bo_device *bdev, unsigned type,
1567                         unsigned long p_size)
1568 {
1569         int ret;
1570         struct ttm_mem_type_manager *man;
1571         unsigned i;
1572
1573         BUG_ON(type >= TTM_NUM_MEM_TYPES);
1574         man = &bdev->man[type];
1575         BUG_ON(man->has_type);
1576         man->io_reserve_fastpath = true;
1577         man->use_io_reserve_lru = false;
1578         mutex_init(&man->io_reserve_mutex);
1579         spin_lock_init(&man->move_lock);
1580         INIT_LIST_HEAD(&man->io_reserve_lru);
1581
1582         ret = bdev->driver->init_mem_type(bdev, type, man);
1583         if (ret)
1584                 return ret;
1585         man->bdev = bdev;
1586
1587         if (type != TTM_PL_SYSTEM) {
1588                 ret = (*man->func->init)(man, p_size);
1589                 if (ret)
1590                         return ret;
1591         }
1592         man->has_type = true;
1593         man->use_type = true;
1594         man->size = p_size;
1595
1596         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i)
1597                 INIT_LIST_HEAD(&man->lru[i]);
1598         man->move = NULL;
1599
1600         return 0;
1601 }
1602 EXPORT_SYMBOL(ttm_bo_init_mm);
1603
1604 static void ttm_bo_global_kobj_release(struct kobject *kobj)
1605 {
1606         struct ttm_bo_global *glob =
1607                 container_of(kobj, struct ttm_bo_global, kobj);
1608
1609         __free_page(glob->dummy_read_page);
1610 }
1611
1612 static void ttm_bo_global_release(void)
1613 {
1614         struct ttm_bo_global *glob = &ttm_bo_glob;
1615
1616         mutex_lock(&ttm_global_mutex);
1617         if (--ttm_bo_glob_use_count > 0)
1618                 goto out;
1619
1620         kobject_del(&glob->kobj);
1621         kobject_put(&glob->kobj);
1622         ttm_mem_global_release(&ttm_mem_glob);
1623         memset(glob, 0, sizeof(*glob));
1624 out:
1625         mutex_unlock(&ttm_global_mutex);
1626 }
1627
1628 static int ttm_bo_global_init(void)
1629 {
1630         struct ttm_bo_global *glob = &ttm_bo_glob;
1631         int ret = 0;
1632         unsigned i;
1633
1634         mutex_lock(&ttm_global_mutex);
1635         if (++ttm_bo_glob_use_count > 1)
1636                 goto out;
1637
1638         ret = ttm_mem_global_init(&ttm_mem_glob);
1639         if (ret)
1640                 goto out;
1641
1642         spin_lock_init(&glob->lru_lock);
1643         glob->mem_glob = &ttm_mem_glob;
1644         glob->mem_glob->bo_glob = glob;
1645         glob->dummy_read_page = alloc_page(__GFP_ZERO | GFP_DMA32);
1646
1647         if (unlikely(glob->dummy_read_page == NULL)) {
1648                 ret = -ENOMEM;
1649                 goto out;
1650         }
1651
1652         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i)
1653                 INIT_LIST_HEAD(&glob->swap_lru[i]);
1654         INIT_LIST_HEAD(&glob->device_list);
1655         atomic_set(&glob->bo_count, 0);
1656
1657         ret = kobject_init_and_add(
1658                 &glob->kobj, &ttm_bo_glob_kobj_type, ttm_get_kobj(), "buffer_objects");
1659         if (unlikely(ret != 0))
1660                 kobject_put(&glob->kobj);
1661 out:
1662         mutex_unlock(&ttm_global_mutex);
1663         return ret;
1664 }
1665
1666 int ttm_bo_device_release(struct ttm_bo_device *bdev)
1667 {
1668         int ret = 0;
1669         unsigned i = TTM_NUM_MEM_TYPES;
1670         struct ttm_mem_type_manager *man;
1671         struct ttm_bo_global *glob = bdev->glob;
1672
1673         while (i--) {
1674                 man = &bdev->man[i];
1675                 if (man->has_type) {
1676                         man->use_type = false;
1677                         if ((i != TTM_PL_SYSTEM) && ttm_bo_clean_mm(bdev, i)) {
1678                                 ret = -EBUSY;
1679                                 pr_err("DRM memory manager type %d is not clean\n",
1680                                        i);
1681                         }
1682                         man->has_type = false;
1683                 }
1684         }
1685
1686         mutex_lock(&ttm_global_mutex);
1687         list_del(&bdev->device_list);
1688         mutex_unlock(&ttm_global_mutex);
1689
1690         cancel_delayed_work_sync(&bdev->wq);
1691
1692         if (ttm_bo_delayed_delete(bdev, true))
1693                 pr_debug("Delayed destroy list was clean\n");
1694
1695         spin_lock(&glob->lru_lock);
1696         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i)
1697                 if (list_empty(&bdev->man[0].lru[0]))
1698                         pr_debug("Swap list %d was clean\n", i);
1699         spin_unlock(&glob->lru_lock);
1700
1701         drm_vma_offset_manager_destroy(&bdev->vma_manager);
1702
1703         if (!ret)
1704                 ttm_bo_global_release();
1705
1706         return ret;
1707 }
1708 EXPORT_SYMBOL(ttm_bo_device_release);
1709
1710 int ttm_bo_device_init(struct ttm_bo_device *bdev,
1711                        struct ttm_bo_driver *driver,
1712                        struct address_space *mapping,
1713                        bool need_dma32)
1714 {
1715         struct ttm_bo_global *glob = &ttm_bo_glob;
1716         int ret;
1717
1718         ret = ttm_bo_global_init();
1719         if (ret)
1720                 return ret;
1721
1722         bdev->driver = driver;
1723
1724         memset(bdev->man, 0, sizeof(bdev->man));
1725
1726         /*
1727          * Initialize the system memory buffer type.
1728          * Other types need to be driver / IOCTL initialized.
1729          */
1730         ret = ttm_bo_init_mm(bdev, TTM_PL_SYSTEM, 0);
1731         if (unlikely(ret != 0))
1732                 goto out_no_sys;
1733
1734         drm_vma_offset_manager_init(&bdev->vma_manager,
1735                                     DRM_FILE_PAGE_OFFSET_START,
1736                                     DRM_FILE_PAGE_OFFSET_SIZE);
1737         INIT_DELAYED_WORK(&bdev->wq, ttm_bo_delayed_workqueue);
1738         INIT_LIST_HEAD(&bdev->ddestroy);
1739         bdev->dev_mapping = mapping;
1740         bdev->glob = glob;
1741         bdev->need_dma32 = need_dma32;
1742         mutex_lock(&ttm_global_mutex);
1743         list_add_tail(&bdev->device_list, &glob->device_list);
1744         mutex_unlock(&ttm_global_mutex);
1745
1746         return 0;
1747 out_no_sys:
1748         ttm_bo_global_release();
1749         return ret;
1750 }
1751 EXPORT_SYMBOL(ttm_bo_device_init);
1752
1753 /*
1754  * buffer object vm functions.
1755  */
1756
1757 bool ttm_mem_reg_is_pci(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
1758 {
1759         struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
1760
1761         if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED)) {
1762                 if (mem->mem_type == TTM_PL_SYSTEM)
1763                         return false;
1764
1765                 if (man->flags & TTM_MEMTYPE_FLAG_CMA)
1766                         return false;
1767
1768                 if (mem->placement & TTM_PL_FLAG_CACHED)
1769                         return false;
1770         }
1771         return true;
1772 }
1773
1774 void ttm_bo_unmap_virtual_locked(struct ttm_buffer_object *bo)
1775 {
1776         struct ttm_bo_device *bdev = bo->bdev;
1777
1778         drm_vma_node_unmap(&bo->vma_node, bdev->dev_mapping);
1779         ttm_mem_io_free_vm(bo);
1780 }
1781
1782 void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo)
1783 {
1784         struct ttm_bo_device *bdev = bo->bdev;
1785         struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type];
1786
1787         ttm_mem_io_lock(man, false);
1788         ttm_bo_unmap_virtual_locked(bo);
1789         ttm_mem_io_unlock(man);
1790 }
1791
1792
1793 EXPORT_SYMBOL(ttm_bo_unmap_virtual);
1794
1795 int ttm_bo_wait(struct ttm_buffer_object *bo,
1796                 bool interruptible, bool no_wait)
1797 {
1798         long timeout = 15 * HZ;
1799
1800         if (no_wait) {
1801                 if (reservation_object_test_signaled_rcu(bo->resv, true))
1802                         return 0;
1803                 else
1804                         return -EBUSY;
1805         }
1806
1807         timeout = reservation_object_wait_timeout_rcu(bo->resv, true,
1808                                                       interruptible, timeout);
1809         if (timeout < 0)
1810                 return timeout;
1811
1812         if (timeout == 0)
1813                 return -EBUSY;
1814
1815         reservation_object_add_excl_fence(bo->resv, NULL);
1816         return 0;
1817 }
1818 EXPORT_SYMBOL(ttm_bo_wait);
1819
1820 int ttm_bo_synccpu_write_grab(struct ttm_buffer_object *bo, bool no_wait)
1821 {
1822         int ret = 0;
1823
1824         /*
1825          * Using ttm_bo_reserve makes sure the lru lists are updated.
1826          */
1827
1828         ret = ttm_bo_reserve(bo, true, no_wait, NULL);
1829         if (unlikely(ret != 0))
1830                 return ret;
1831         ret = ttm_bo_wait(bo, true, no_wait);
1832         if (likely(ret == 0))
1833                 atomic_inc(&bo->cpu_writers);
1834         ttm_bo_unreserve(bo);
1835         return ret;
1836 }
1837 EXPORT_SYMBOL(ttm_bo_synccpu_write_grab);
1838
1839 void ttm_bo_synccpu_write_release(struct ttm_buffer_object *bo)
1840 {
1841         atomic_dec(&bo->cpu_writers);
1842 }
1843 EXPORT_SYMBOL(ttm_bo_synccpu_write_release);
1844
1845 /**
1846  * A buffer object shrink method that tries to swap out the first
1847  * buffer object on the bo_global::swap_lru list.
1848  */
1849 int ttm_bo_swapout(struct ttm_bo_global *glob, struct ttm_operation_ctx *ctx)
1850 {
1851         struct ttm_buffer_object *bo;
1852         int ret = -EBUSY;
1853         bool locked;
1854         unsigned i;
1855
1856         spin_lock(&glob->lru_lock);
1857         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
1858                 list_for_each_entry(bo, &glob->swap_lru[i], swap) {
1859                         if (ttm_bo_evict_swapout_allowable(bo, ctx, &locked,
1860                                                            NULL)) {
1861                                 ret = 0;
1862                                 break;
1863                         }
1864                 }
1865                 if (!ret)
1866                         break;
1867         }
1868
1869         if (ret) {
1870                 spin_unlock(&glob->lru_lock);
1871                 return ret;
1872         }
1873
1874         kref_get(&bo->list_kref);
1875
1876         if (!list_empty(&bo->ddestroy)) {
1877                 ret = ttm_bo_cleanup_refs(bo, false, false, locked);
1878                 kref_put(&bo->list_kref, ttm_bo_release_list);
1879                 return ret;
1880         }
1881
1882         ttm_bo_del_from_lru(bo);
1883         spin_unlock(&glob->lru_lock);
1884
1885         /**
1886          * Move to system cached
1887          */
1888
1889         if (bo->mem.mem_type != TTM_PL_SYSTEM ||
1890             bo->ttm->caching_state != tt_cached) {
1891                 struct ttm_operation_ctx ctx = { false, false };
1892                 struct ttm_mem_reg evict_mem;
1893
1894                 evict_mem = bo->mem;
1895                 evict_mem.mm_node = NULL;
1896                 evict_mem.placement = TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED;
1897                 evict_mem.mem_type = TTM_PL_SYSTEM;
1898
1899                 ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, &ctx);
1900                 if (unlikely(ret != 0))
1901                         goto out;
1902         }
1903
1904         /**
1905          * Make sure BO is idle.
1906          */
1907
1908         ret = ttm_bo_wait(bo, false, false);
1909         if (unlikely(ret != 0))
1910                 goto out;
1911
1912         ttm_bo_unmap_virtual(bo);
1913
1914         /**
1915          * Swap out. Buffer will be swapped in again as soon as
1916          * anyone tries to access a ttm page.
1917          */
1918
1919         if (bo->bdev->driver->swap_notify)
1920                 bo->bdev->driver->swap_notify(bo);
1921
1922         ret = ttm_tt_swapout(bo->ttm, bo->persistent_swap_storage);
1923 out:
1924
1925         /**
1926          *
1927          * Unreserve without putting on LRU to avoid swapping out an
1928          * already swapped buffer.
1929          */
1930         if (locked)
1931                 reservation_object_unlock(bo->resv);
1932         kref_put(&bo->list_kref, ttm_bo_release_list);
1933         return ret;
1934 }
1935 EXPORT_SYMBOL(ttm_bo_swapout);
1936
1937 void ttm_bo_swapout_all(struct ttm_bo_device *bdev)
1938 {
1939         struct ttm_operation_ctx ctx = {
1940                 .interruptible = false,
1941                 .no_wait_gpu = false
1942         };
1943
1944         while (ttm_bo_swapout(bdev->glob, &ctx) == 0)
1945                 ;
1946 }
1947 EXPORT_SYMBOL(ttm_bo_swapout_all);
1948
1949 /**
1950  * ttm_bo_wait_unreserved - interruptible wait for a buffer object to become
1951  * unreserved
1952  *
1953  * @bo: Pointer to buffer
1954  */
1955 int ttm_bo_wait_unreserved(struct ttm_buffer_object *bo)
1956 {
1957         int ret;
1958
1959         /*
1960          * In the absense of a wait_unlocked API,
1961          * Use the bo::wu_mutex to avoid triggering livelocks due to
1962          * concurrent use of this function. Note that this use of
1963          * bo::wu_mutex can go away if we change locking order to
1964          * mmap_sem -> bo::reserve.
1965          */
1966         ret = mutex_lock_interruptible(&bo->wu_mutex);
1967         if (unlikely(ret != 0))
1968                 return -ERESTARTSYS;
1969         if (!ww_mutex_is_locked(&bo->resv->lock))
1970                 goto out_unlock;
1971         ret = reservation_object_lock_interruptible(bo->resv, NULL);
1972         if (ret == -EINTR)
1973                 ret = -ERESTARTSYS;
1974         if (unlikely(ret != 0))
1975                 goto out_unlock;
1976         reservation_object_unlock(bo->resv);
1977
1978 out_unlock:
1979         mutex_unlock(&bo->wu_mutex);
1980         return ret;
1981 }