1 // SPDX-License-Identifier: MIT
3 * Copyright © 2021 Intel Corporation
6 #include <drm/ttm/ttm_placement.h>
7 #include <drm/ttm/ttm_tt.h>
10 #include "intel_memory_region.h"
11 #include "intel_region_ttm.h"
13 #include "gem/i915_gem_region.h"
14 #include "gem/i915_gem_ttm.h"
15 #include "gem/i915_gem_ttm_move.h"
16 #include "gem/i915_gem_ttm_pm.h"
19 * i915_ttm_backup_free - Free any backup attached to this object
20 * @obj: The object whose backup is to be freed.
22 void i915_ttm_backup_free(struct drm_i915_gem_object *obj)
24 if (obj->ttm.backup) {
25 i915_gem_object_put(obj->ttm.backup);
26 obj->ttm.backup = NULL;
31 * struct i915_gem_ttm_pm_apply - Apply-to-region subclass for restore
32 * @base: The i915_gem_apply_to_region we derive from.
33 * @allow_gpu: Whether using the gpu blitter is allowed.
34 * @backup_pinned: On backup, backup also pinned objects.
36 struct i915_gem_ttm_pm_apply {
37 struct i915_gem_apply_to_region base;
39 bool backup_pinned : 1;
42 static int i915_ttm_backup(struct i915_gem_apply_to_region *apply,
43 struct drm_i915_gem_object *obj)
45 struct i915_gem_ttm_pm_apply *pm_apply =
46 container_of(apply, typeof(*pm_apply), base);
47 struct ttm_buffer_object *bo = i915_gem_to_ttm(obj);
48 struct ttm_buffer_object *backup_bo;
49 struct drm_i915_private *i915 =
50 container_of(bo->bdev, typeof(*i915), bdev);
51 struct drm_i915_gem_object *backup;
52 struct ttm_operation_ctx ctx = {};
55 if (bo->resource->mem_type == I915_PL_SYSTEM || obj->ttm.backup)
58 if (pm_apply->allow_gpu && i915_gem_object_evictable(obj))
59 return ttm_bo_validate(bo, i915_ttm_sys_placement(), &ctx);
61 if (!pm_apply->backup_pinned ||
62 (pm_apply->allow_gpu && (obj->flags & I915_BO_ALLOC_PM_EARLY)))
65 if (obj->flags & I915_BO_ALLOC_PM_VOLATILE)
68 backup = i915_gem_object_create_shmem(i915, obj->base.size);
70 return PTR_ERR(backup);
72 err = i915_gem_object_lock(backup, apply->ww);
76 backup_bo = i915_gem_to_ttm(backup);
77 err = ttm_tt_populate(backup_bo->bdev, backup_bo->ttm, &ctx);
81 err = i915_gem_obj_copy_ttm(backup, obj, pm_apply->allow_gpu, false);
84 "Unable to copy from device to system memory, err:%pe\n",
88 ttm_bo_wait_ctx(backup_bo, &ctx);
90 obj->ttm.backup = backup;
94 i915_gem_ww_unlock_single(backup);
96 i915_gem_object_put(backup);
101 static int i915_ttm_recover(struct i915_gem_apply_to_region *apply,
102 struct drm_i915_gem_object *obj)
104 i915_ttm_backup_free(obj);
109 * i915_ttm_recover_region - Free the backup of all objects of a region
110 * @mr: The memory region
112 * Checks all objects of a region if there is backup attached and if so
113 * frees that backup. Typically this is called to recover after a partially
116 void i915_ttm_recover_region(struct intel_memory_region *mr)
118 static const struct i915_gem_apply_to_region_ops recover_ops = {
119 .process_obj = i915_ttm_recover,
121 struct i915_gem_apply_to_region apply = {.ops = &recover_ops};
124 ret = i915_gem_process_region(mr, &apply);
129 * i915_ttm_backup_region - Back up all objects of a region to smem.
130 * @mr: The memory region
131 * @allow_gpu: Whether to allow the gpu blitter for this backup.
132 * @backup_pinned: Backup also pinned objects.
134 * Loops over all objects of a region and either evicts them if they are
135 * evictable or backs them up using a backup object if they are pinned.
137 * Return: Zero on success. Negative error code on error.
139 int i915_ttm_backup_region(struct intel_memory_region *mr, u32 flags)
141 static const struct i915_gem_apply_to_region_ops backup_ops = {
142 .process_obj = i915_ttm_backup,
144 struct i915_gem_ttm_pm_apply pm_apply = {
145 .base = {.ops = &backup_ops},
146 .allow_gpu = flags & I915_TTM_BACKUP_ALLOW_GPU,
147 .backup_pinned = flags & I915_TTM_BACKUP_PINNED,
150 return i915_gem_process_region(mr, &pm_apply.base);
153 static int i915_ttm_restore(struct i915_gem_apply_to_region *apply,
154 struct drm_i915_gem_object *obj)
156 struct i915_gem_ttm_pm_apply *pm_apply =
157 container_of(apply, typeof(*pm_apply), base);
158 struct drm_i915_gem_object *backup = obj->ttm.backup;
159 struct ttm_buffer_object *backup_bo = i915_gem_to_ttm(backup);
160 struct ttm_operation_ctx ctx = {};
166 if (!pm_apply->allow_gpu && !(obj->flags & I915_BO_ALLOC_PM_EARLY))
169 err = i915_gem_object_lock(backup, apply->ww);
173 /* Content may have been swapped. */
174 err = ttm_tt_populate(backup_bo->bdev, backup_bo->ttm, &ctx);
176 err = i915_gem_obj_copy_ttm(obj, backup, pm_apply->allow_gpu,
179 ttm_bo_wait_ctx(backup_bo, &ctx);
181 obj->ttm.backup = NULL;
185 i915_gem_ww_unlock_single(backup);
188 i915_gem_object_put(backup);
194 * i915_ttm_restore_region - Restore backed-up objects of a region from smem.
195 * @mr: The memory region
196 * @allow_gpu: Whether to allow the gpu blitter to recover.
198 * Loops over all objects of a region and if they are backed-up, restores
201 * Return: Zero on success. Negative error code on error.
203 int i915_ttm_restore_region(struct intel_memory_region *mr, u32 flags)
205 static const struct i915_gem_apply_to_region_ops restore_ops = {
206 .process_obj = i915_ttm_restore,
208 struct i915_gem_ttm_pm_apply pm_apply = {
209 .base = {.ops = &restore_ops},
210 .allow_gpu = flags & I915_TTM_BACKUP_ALLOW_GPU,
213 return i915_gem_process_region(mr, &pm_apply.base);