1 // SPDX-License-Identifier: GPL-2.0-or-later
3 #include <drm/drm_gem_vram_helper.h>
4 #include <drm/drm_device.h>
5 #include <drm/drm_mode.h>
6 #include <drm/drm_prime.h>
7 #include <drm/drm_vram_mm_helper.h>
8 #include <drm/ttm/ttm_page_alloc.h>
10 static const struct drm_gem_object_funcs drm_gem_vram_object_funcs;
15 * This library provides a GEM buffer object that is backed by video RAM
16 * (VRAM). It can be used for framebuffer devices with dedicated memory.
20 * Buffer-objects helpers
23 static void drm_gem_vram_cleanup(struct drm_gem_vram_object *gbo)
25 /* We got here via ttm_bo_put(), which means that the
26 * TTM buffer object in 'bo' has already been cleaned
27 * up; only release the GEM object.
29 drm_gem_object_release(&gbo->bo.base);
32 static void drm_gem_vram_destroy(struct drm_gem_vram_object *gbo)
34 drm_gem_vram_cleanup(gbo);
38 static void ttm_buffer_object_destroy(struct ttm_buffer_object *bo)
40 struct drm_gem_vram_object *gbo = drm_gem_vram_of_bo(bo);
42 drm_gem_vram_destroy(gbo);
45 static void drm_gem_vram_placement(struct drm_gem_vram_object *gbo,
46 unsigned long pl_flag)
51 gbo->placement.placement = gbo->placements;
52 gbo->placement.busy_placement = gbo->placements;
54 if (pl_flag & TTM_PL_FLAG_VRAM)
55 gbo->placements[c++].flags = TTM_PL_FLAG_WC |
56 TTM_PL_FLAG_UNCACHED |
59 if (pl_flag & TTM_PL_FLAG_SYSTEM)
60 gbo->placements[c++].flags = TTM_PL_MASK_CACHING |
64 gbo->placements[c++].flags = TTM_PL_MASK_CACHING |
67 gbo->placement.num_placement = c;
68 gbo->placement.num_busy_placement = c;
70 for (i = 0; i < c; ++i) {
71 gbo->placements[i].fpfn = 0;
72 gbo->placements[i].lpfn = 0;
76 static int drm_gem_vram_init(struct drm_device *dev,
77 struct ttm_bo_device *bdev,
78 struct drm_gem_vram_object *gbo,
79 size_t size, unsigned long pg_align,
85 if (!gbo->bo.base.funcs)
86 gbo->bo.base.funcs = &drm_gem_vram_object_funcs;
88 ret = drm_gem_object_init(dev, &gbo->bo.base, size);
92 acc_size = ttm_bo_dma_acc_size(bdev, size, sizeof(*gbo));
95 drm_gem_vram_placement(gbo, TTM_PL_FLAG_VRAM | TTM_PL_FLAG_SYSTEM);
97 ret = ttm_bo_init(bdev, &gbo->bo, size, ttm_bo_type_device,
98 &gbo->placement, pg_align, interruptible, acc_size,
99 NULL, NULL, ttm_buffer_object_destroy);
101 goto err_drm_gem_object_release;
105 err_drm_gem_object_release:
106 drm_gem_object_release(&gbo->bo.base);
111 * drm_gem_vram_create() - Creates a VRAM-backed GEM object
112 * @dev: the DRM device
113 * @bdev: the TTM BO device backing the object
114 * @size: the buffer size in bytes
115 * @pg_align: the buffer's alignment in multiples of the page size
116 * @interruptible: sleep interruptible if waiting for memory
119 * A new instance of &struct drm_gem_vram_object on success, or
120 * an ERR_PTR()-encoded error code otherwise.
122 struct drm_gem_vram_object *drm_gem_vram_create(struct drm_device *dev,
123 struct ttm_bo_device *bdev,
125 unsigned long pg_align,
128 struct drm_gem_vram_object *gbo;
131 gbo = kzalloc(sizeof(*gbo), GFP_KERNEL);
133 return ERR_PTR(-ENOMEM);
135 ret = drm_gem_vram_init(dev, bdev, gbo, size, pg_align, interruptible);
145 EXPORT_SYMBOL(drm_gem_vram_create);
148 * drm_gem_vram_put() - Releases a reference to a VRAM-backed GEM object
149 * @gbo: the GEM VRAM object
151 * See ttm_bo_put() for more information.
153 void drm_gem_vram_put(struct drm_gem_vram_object *gbo)
155 ttm_bo_put(&gbo->bo);
157 EXPORT_SYMBOL(drm_gem_vram_put);
160 * drm_gem_vram_mmap_offset() - Returns a GEM VRAM object's mmap offset
161 * @gbo: the GEM VRAM object
163 * See drm_vma_node_offset_addr() for more information.
166 * The buffer object's offset for userspace mappings on success, or
167 * 0 if no offset is allocated.
169 u64 drm_gem_vram_mmap_offset(struct drm_gem_vram_object *gbo)
171 return drm_vma_node_offset_addr(&gbo->bo.base.vma_node);
173 EXPORT_SYMBOL(drm_gem_vram_mmap_offset);
176 * drm_gem_vram_offset() - \
177 Returns a GEM VRAM object's offset in video memory
178 * @gbo: the GEM VRAM object
180 * This function returns the buffer object's offset in the device's video
181 * memory. The buffer object has to be pinned to %TTM_PL_VRAM.
184 * The buffer object's offset in video memory on success, or
185 * a negative errno code otherwise.
187 s64 drm_gem_vram_offset(struct drm_gem_vram_object *gbo)
189 if (WARN_ON_ONCE(!gbo->pin_count))
191 return gbo->bo.offset;
193 EXPORT_SYMBOL(drm_gem_vram_offset);
196 * drm_gem_vram_pin() - Pins a GEM VRAM object in a region.
197 * @gbo: the GEM VRAM object
198 * @pl_flag: a bitmask of possible memory regions
200 * Pinning a buffer object ensures that it is not evicted from
201 * a memory region. A pinned buffer object has to be unpinned before
202 * it can be pinned to another region. If the pl_flag argument is 0,
203 * the buffer is pinned at its current location (video RAM or system
208 * a negative error code otherwise.
210 int drm_gem_vram_pin(struct drm_gem_vram_object *gbo, unsigned long pl_flag)
213 struct ttm_operation_ctx ctx = { false, false };
215 ret = ttm_bo_reserve(&gbo->bo, true, false, NULL);
223 drm_gem_vram_placement(gbo, pl_flag);
225 for (i = 0; i < gbo->placement.num_placement; ++i)
226 gbo->placements[i].flags |= TTM_PL_FLAG_NO_EVICT;
228 ret = ttm_bo_validate(&gbo->bo, &gbo->placement, &ctx);
230 goto err_ttm_bo_unreserve;
234 ttm_bo_unreserve(&gbo->bo);
238 err_ttm_bo_unreserve:
239 ttm_bo_unreserve(&gbo->bo);
242 EXPORT_SYMBOL(drm_gem_vram_pin);
245 * drm_gem_vram_unpin() - Unpins a GEM VRAM object
246 * @gbo: the GEM VRAM object
250 * a negative error code otherwise.
252 int drm_gem_vram_unpin(struct drm_gem_vram_object *gbo)
255 struct ttm_operation_ctx ctx = { false, false };
257 ret = ttm_bo_reserve(&gbo->bo, true, false, NULL);
261 if (WARN_ON_ONCE(!gbo->pin_count))
268 for (i = 0; i < gbo->placement.num_placement ; ++i)
269 gbo->placements[i].flags &= ~TTM_PL_FLAG_NO_EVICT;
271 ret = ttm_bo_validate(&gbo->bo, &gbo->placement, &ctx);
273 goto err_ttm_bo_unreserve;
276 ttm_bo_unreserve(&gbo->bo);
280 err_ttm_bo_unreserve:
281 ttm_bo_unreserve(&gbo->bo);
284 EXPORT_SYMBOL(drm_gem_vram_unpin);
287 * drm_gem_vram_kmap() - Maps a GEM VRAM object into kernel address space
288 * @gbo: the GEM VRAM object
289 * @map: establish a mapping if necessary
290 * @is_iomem: returns true if the mapped memory is I/O memory, or false \
291 otherwise; can be NULL
293 * This function maps the buffer object into the kernel's address space
294 * or returns the current mapping. If the parameter map is false, the
295 * function only queries the current mapping, but does not establish a
299 * The buffers virtual address if mapped, or
300 * NULL if not mapped, or
301 * an ERR_PTR()-encoded error code otherwise.
303 void *drm_gem_vram_kmap(struct drm_gem_vram_object *gbo, bool map,
307 struct ttm_bo_kmap_obj *kmap = &gbo->kmap;
309 if (kmap->virtual || !map)
312 ret = ttm_bo_kmap(&gbo->bo, 0, gbo->bo.num_pages, kmap);
318 return kmap->virtual;
319 if (!kmap->virtual) {
323 return ttm_kmap_obj_virtual(kmap, is_iomem);
325 EXPORT_SYMBOL(drm_gem_vram_kmap);
328 * drm_gem_vram_kunmap() - Unmaps a GEM VRAM object
329 * @gbo: the GEM VRAM object
331 void drm_gem_vram_kunmap(struct drm_gem_vram_object *gbo)
333 struct ttm_bo_kmap_obj *kmap = &gbo->kmap;
339 kmap->virtual = NULL;
341 EXPORT_SYMBOL(drm_gem_vram_kunmap);
344 * drm_gem_vram_fill_create_dumb() - \
345 Helper for implementing &struct drm_driver.dumb_create
346 * @file: the DRM file
347 * @dev: the DRM device
348 * @bdev: the TTM BO device managing the buffer object
349 * @pg_align: the buffer's alignment in multiples of the page size
350 * @interruptible: sleep interruptible if waiting for memory
351 * @args: the arguments as provided to \
352 &struct drm_driver.dumb_create
354 * This helper function fills &struct drm_mode_create_dumb, which is used
355 * by &struct drm_driver.dumb_create. Implementations of this interface
356 * should forwards their arguments to this helper, plus the driver-specific
361 * a negative error code otherwise.
363 int drm_gem_vram_fill_create_dumb(struct drm_file *file,
364 struct drm_device *dev,
365 struct ttm_bo_device *bdev,
366 unsigned long pg_align,
368 struct drm_mode_create_dumb *args)
371 struct drm_gem_vram_object *gbo;
375 pitch = args->width * ((args->bpp + 7) / 8);
376 size = pitch * args->height;
378 size = roundup(size, PAGE_SIZE);
382 gbo = drm_gem_vram_create(dev, bdev, size, pg_align, interruptible);
386 ret = drm_gem_handle_create(file, &gbo->bo.base, &handle);
388 goto err_drm_gem_object_put_unlocked;
390 drm_gem_object_put_unlocked(&gbo->bo.base);
394 args->handle = handle;
398 err_drm_gem_object_put_unlocked:
399 drm_gem_object_put_unlocked(&gbo->bo.base);
402 EXPORT_SYMBOL(drm_gem_vram_fill_create_dumb);
405 * Helpers for struct ttm_bo_driver
408 static bool drm_is_gem_vram(struct ttm_buffer_object *bo)
410 return (bo->destroy == ttm_buffer_object_destroy);
414 * drm_gem_vram_bo_driver_evict_flags() - \
415 Implements &struct ttm_bo_driver.evict_flags
416 * @bo: TTM buffer object. Refers to &struct drm_gem_vram_object.bo
417 * @pl: TTM placement information.
419 void drm_gem_vram_bo_driver_evict_flags(struct ttm_buffer_object *bo,
420 struct ttm_placement *pl)
422 struct drm_gem_vram_object *gbo;
424 /* TTM may pass BOs that are not GEM VRAM BOs. */
425 if (!drm_is_gem_vram(bo))
428 gbo = drm_gem_vram_of_bo(bo);
429 drm_gem_vram_placement(gbo, TTM_PL_FLAG_SYSTEM);
430 *pl = gbo->placement;
432 EXPORT_SYMBOL(drm_gem_vram_bo_driver_evict_flags);
435 * drm_gem_vram_bo_driver_verify_access() - \
436 Implements &struct ttm_bo_driver.verify_access
437 * @bo: TTM buffer object. Refers to &struct drm_gem_vram_object.bo
438 * @filp: File pointer.
442 * a negative errno code otherwise.
444 int drm_gem_vram_bo_driver_verify_access(struct ttm_buffer_object *bo,
447 struct drm_gem_vram_object *gbo = drm_gem_vram_of_bo(bo);
449 return drm_vma_node_verify_access(&gbo->bo.base.vma_node,
452 EXPORT_SYMBOL(drm_gem_vram_bo_driver_verify_access);
455 * drm_gem_vram_mm_funcs - Functions for &struct drm_vram_mm
457 * Most users of @struct drm_gem_vram_object will also use
458 * @struct drm_vram_mm. This instance of &struct drm_vram_mm_funcs
459 * can be used to connect both.
461 const struct drm_vram_mm_funcs drm_gem_vram_mm_funcs = {
462 .evict_flags = drm_gem_vram_bo_driver_evict_flags,
463 .verify_access = drm_gem_vram_bo_driver_verify_access
465 EXPORT_SYMBOL(drm_gem_vram_mm_funcs);
468 * Helpers for struct drm_gem_object_funcs
472 * drm_gem_vram_object_free() - \
473 Implements &struct drm_gem_object_funcs.free
474 * @gem: GEM object. Refers to &struct drm_gem_vram_object.gem
476 static void drm_gem_vram_object_free(struct drm_gem_object *gem)
478 struct drm_gem_vram_object *gbo = drm_gem_vram_of_gem(gem);
480 drm_gem_vram_put(gbo);
484 * Helpers for dump buffers
488 * drm_gem_vram_driver_create_dumb() - \
489 Implements &struct drm_driver.dumb_create
490 * @file: the DRM file
491 * @dev: the DRM device
492 * @args: the arguments as provided to \
493 &struct drm_driver.dumb_create
495 * This function requires the driver to use @drm_device.vram_mm for its
496 * instance of VRAM MM.
500 * a negative error code otherwise.
502 int drm_gem_vram_driver_dumb_create(struct drm_file *file,
503 struct drm_device *dev,
504 struct drm_mode_create_dumb *args)
506 if (WARN_ONCE(!dev->vram_mm, "VRAM MM not initialized"))
509 return drm_gem_vram_fill_create_dumb(file, dev, &dev->vram_mm->bdev, 0,
512 EXPORT_SYMBOL(drm_gem_vram_driver_dumb_create);
515 * drm_gem_vram_driver_dumb_mmap_offset() - \
516 Implements &struct drm_driver.dumb_mmap_offset
517 * @file: DRM file pointer.
519 * @handle: GEM handle
520 * @offset: Returns the mapping's memory offset on success
524 * a negative errno code otherwise.
526 int drm_gem_vram_driver_dumb_mmap_offset(struct drm_file *file,
527 struct drm_device *dev,
528 uint32_t handle, uint64_t *offset)
530 struct drm_gem_object *gem;
531 struct drm_gem_vram_object *gbo;
533 gem = drm_gem_object_lookup(file, handle);
537 gbo = drm_gem_vram_of_gem(gem);
538 *offset = drm_gem_vram_mmap_offset(gbo);
540 drm_gem_object_put_unlocked(gem);
544 EXPORT_SYMBOL(drm_gem_vram_driver_dumb_mmap_offset);
551 * drm_gem_vram_object_pin() - \
552 Implements &struct drm_gem_object_funcs.pin
553 * @gem: The GEM object to pin
557 * a negative errno code otherwise.
559 static int drm_gem_vram_object_pin(struct drm_gem_object *gem)
561 struct drm_gem_vram_object *gbo = drm_gem_vram_of_gem(gem);
563 /* Fbdev console emulation is the use case of these PRIME
564 * helpers. This may involve updating a hardware buffer from
565 * a shadow FB. We pin the buffer to it's current location
566 * (either video RAM or system memory) to prevent it from
567 * being relocated during the update operation. If you require
568 * the buffer to be pinned to VRAM, implement a callback that
569 * sets the flags accordingly.
571 return drm_gem_vram_pin(gbo, 0);
575 * drm_gem_vram_object_unpin() - \
576 Implements &struct drm_gem_object_funcs.unpin
577 * @gem: The GEM object to unpin
579 static void drm_gem_vram_object_unpin(struct drm_gem_object *gem)
581 struct drm_gem_vram_object *gbo = drm_gem_vram_of_gem(gem);
583 drm_gem_vram_unpin(gbo);
587 * drm_gem_vram_object_vmap() - \
588 Implements &struct drm_gem_object_funcs.vmap
589 * @gem: The GEM object to map
592 * The buffers virtual address on success, or
595 static void *drm_gem_vram_object_vmap(struct drm_gem_object *gem)
597 struct drm_gem_vram_object *gbo = drm_gem_vram_of_gem(gem);
601 ret = drm_gem_vram_pin(gbo, 0);
604 base = drm_gem_vram_kmap(gbo, true, NULL);
606 drm_gem_vram_unpin(gbo);
613 * drm_gem_vram_object_vunmap() - \
614 Implements &struct drm_gem_object_funcs.vunmap
615 * @gem: The GEM object to unmap
616 * @vaddr: The mapping's base address
618 static void drm_gem_vram_object_vunmap(struct drm_gem_object *gem,
621 struct drm_gem_vram_object *gbo = drm_gem_vram_of_gem(gem);
623 drm_gem_vram_kunmap(gbo);
624 drm_gem_vram_unpin(gbo);
631 static const struct drm_gem_object_funcs drm_gem_vram_object_funcs = {
632 .free = drm_gem_vram_object_free,
633 .pin = drm_gem_vram_object_pin,
634 .unpin = drm_gem_vram_object_unpin,
635 .vmap = drm_gem_vram_object_vmap,
636 .vunmap = drm_gem_vram_object_vunmap