1 // SPDX-License-Identifier: GPL-2.0
4 * Copyright 2016-2019 HabanaLabs, Ltd.
8 #include <uapi/misc/habanalabs.h>
9 #include "habanalabs.h"
12 #include <linux/slab.h>
14 static void cb_fini(struct hl_device *hdev, struct hl_cb *cb)
16 hdev->asic_funcs->dma_free_coherent(hdev, cb->size,
17 (void *) (uintptr_t) cb->kernel_address,
22 static void cb_do_release(struct hl_device *hdev, struct hl_cb *cb)
25 spin_lock(&hdev->cb_pool_lock);
26 list_add(&cb->pool_list, &hdev->cb_pool);
27 spin_unlock(&hdev->cb_pool_lock);
33 static void cb_release(struct kref *ref)
35 struct hl_device *hdev;
38 cb = container_of(ref, struct hl_cb, refcount);
41 hl_debugfs_remove_cb(cb);
43 cb_do_release(hdev, cb);
46 static struct hl_cb *hl_cb_alloc(struct hl_device *hdev, u32 cb_size,
53 * We use of GFP_ATOMIC here because this function can be called from
54 * the latency-sensitive code path for command submission. Due to H/W
55 * limitations in some of the ASICs, the kernel must copy the user CB
56 * that is designated for an external queue and actually enqueue
57 * the kernel's copy. Hence, we must never sleep in this code section
58 * and must use GFP_ATOMIC for all memory allocations.
60 if (ctx_id == HL_KERNEL_ASID_ID)
61 cb = kzalloc(sizeof(*cb), GFP_ATOMIC);
63 cb = kzalloc(sizeof(*cb), GFP_KERNEL);
68 if (ctx_id == HL_KERNEL_ASID_ID)
69 p = hdev->asic_funcs->dma_alloc_coherent(hdev, cb_size,
70 &cb->bus_address, GFP_ATOMIC);
72 p = hdev->asic_funcs->dma_alloc_coherent(hdev, cb_size,
74 GFP_USER | __GFP_ZERO);
77 "failed to allocate %d of dma memory for CB\n",
83 cb->kernel_address = (u64) (uintptr_t) p;
89 int hl_cb_create(struct hl_device *hdev, struct hl_cb_mgr *mgr,
90 u32 cb_size, u64 *handle, int ctx_id)
93 bool alloc_new_cb = true;
97 * Can't use generic function to check this because of special case
98 * where we create a CB as part of the reset process
100 if ((hdev->disabled) || ((atomic_read(&hdev->in_reset)) &&
101 (ctx_id != HL_KERNEL_ASID_ID))) {
102 dev_warn_ratelimited(hdev->dev,
103 "Device is disabled or in reset. Can't create new CBs\n");
108 if (cb_size > HL_MAX_CB_SIZE) {
110 "CB size %d must be less then %d\n",
111 cb_size, HL_MAX_CB_SIZE);
116 /* Minimum allocation must be PAGE SIZE */
117 if (cb_size < PAGE_SIZE)
120 if (ctx_id == HL_KERNEL_ASID_ID &&
121 cb_size <= hdev->asic_prop.cb_pool_cb_size) {
123 spin_lock(&hdev->cb_pool_lock);
124 if (!list_empty(&hdev->cb_pool)) {
125 cb = list_first_entry(&hdev->cb_pool, typeof(*cb),
127 list_del(&cb->pool_list);
128 spin_unlock(&hdev->cb_pool_lock);
129 alloc_new_cb = false;
131 spin_unlock(&hdev->cb_pool_lock);
132 dev_dbg(hdev->dev, "CB pool is empty\n");
137 cb = hl_cb_alloc(hdev, cb_size, ctx_id);
147 spin_lock(&mgr->cb_lock);
148 rc = idr_alloc(&mgr->cb_handles, cb, 1, 0, GFP_ATOMIC);
149 spin_unlock(&mgr->cb_lock);
152 dev_err(hdev->dev, "Failed to allocate IDR for a new CB\n");
158 kref_init(&cb->refcount);
159 spin_lock_init(&cb->lock);
162 * idr is 32-bit so we can safely OR it with a mask that is above
165 *handle = cb->id | HL_MMAP_CB_MASK;
166 *handle <<= PAGE_SHIFT;
168 hl_debugfs_add_cb(cb);
173 cb_do_release(hdev, cb);
180 int hl_cb_destroy(struct hl_device *hdev, struct hl_cb_mgr *mgr, u64 cb_handle)
187 * handle was given to user to do mmap, I need to shift it back to
188 * how the idr module gave it to me
190 cb_handle >>= PAGE_SHIFT;
191 handle = (u32) cb_handle;
193 spin_lock(&mgr->cb_lock);
195 cb = idr_find(&mgr->cb_handles, handle);
197 idr_remove(&mgr->cb_handles, handle);
198 spin_unlock(&mgr->cb_lock);
199 kref_put(&cb->refcount, cb_release);
201 spin_unlock(&mgr->cb_lock);
203 "CB destroy failed, no match to handle 0x%x\n", handle);
210 int hl_cb_ioctl(struct hl_fpriv *hpriv, void *data)
212 union hl_cb_args *args = data;
213 struct hl_device *hdev = hpriv->hdev;
217 switch (args->in.op) {
218 case HL_CB_OP_CREATE:
219 rc = hl_cb_create(hdev, &hpriv->cb_mgr, args->in.cb_size,
220 &handle, hpriv->ctx->asid);
221 memset(args, 0, sizeof(*args));
222 args->out.cb_handle = handle;
224 case HL_CB_OP_DESTROY:
225 rc = hl_cb_destroy(hdev, &hpriv->cb_mgr,
236 static void cb_vm_close(struct vm_area_struct *vma)
238 struct hl_cb *cb = (struct hl_cb *) vma->vm_private_data;
241 new_mmap_size = cb->mmap_size - (vma->vm_end - vma->vm_start);
243 if (new_mmap_size > 0) {
244 cb->mmap_size = new_mmap_size;
248 spin_lock(&cb->lock);
250 spin_unlock(&cb->lock);
253 vma->vm_private_data = NULL;
256 static const struct vm_operations_struct cb_vm_ops = {
260 int hl_cb_mmap(struct hl_fpriv *hpriv, struct vm_area_struct *vma)
262 struct hl_device *hdev = hpriv->hdev;
268 handle = vma->vm_pgoff;
270 /* reference was taken here */
271 cb = hl_cb_get(hdev, &hpriv->cb_mgr, handle);
274 "CB mmap failed, no match to handle %d\n", handle);
278 /* Validation check */
279 if ((vma->vm_end - vma->vm_start) != ALIGN(cb->size, PAGE_SIZE)) {
281 "CB mmap failed, mmap size 0x%lx != 0x%x cb size\n",
282 vma->vm_end - vma->vm_start, cb->size);
287 spin_lock(&cb->lock);
291 "CB mmap failed, CB already mmaped to user\n");
298 spin_unlock(&cb->lock);
300 vma->vm_ops = &cb_vm_ops;
303 * Note: We're transferring the cb reference to
304 * vma->vm_private_data here.
307 vma->vm_private_data = cb;
309 /* Calculate address for CB */
310 address = virt_to_phys((void *) (uintptr_t) cb->kernel_address);
312 rc = hdev->asic_funcs->cb_mmap(hdev, vma, cb->kernel_address,
316 spin_lock(&cb->lock);
321 cb->mmap_size = cb->size;
326 spin_unlock(&cb->lock);
332 struct hl_cb *hl_cb_get(struct hl_device *hdev, struct hl_cb_mgr *mgr,
337 spin_lock(&mgr->cb_lock);
338 cb = idr_find(&mgr->cb_handles, handle);
341 spin_unlock(&mgr->cb_lock);
343 "CB get failed, no match to handle %d\n", handle);
347 kref_get(&cb->refcount);
349 spin_unlock(&mgr->cb_lock);
355 void hl_cb_put(struct hl_cb *cb)
357 kref_put(&cb->refcount, cb_release);
360 void hl_cb_mgr_init(struct hl_cb_mgr *mgr)
362 spin_lock_init(&mgr->cb_lock);
363 idr_init(&mgr->cb_handles);
366 void hl_cb_mgr_fini(struct hl_device *hdev, struct hl_cb_mgr *mgr)
372 idp = &mgr->cb_handles;
374 idr_for_each_entry(idp, cb, id) {
375 if (kref_put(&cb->refcount, cb_release) != 1)
377 "CB %d for CTX ID %d is still alive\n",
381 idr_destroy(&mgr->cb_handles);
384 struct hl_cb *hl_cb_kernel_create(struct hl_device *hdev, u32 cb_size)
390 rc = hl_cb_create(hdev, &hdev->kernel_cb_mgr, cb_size, &cb_handle,
393 dev_err(hdev->dev, "Failed to allocate CB for KMD %d\n", rc);
397 cb_handle >>= PAGE_SHIFT;
398 cb = hl_cb_get(hdev, &hdev->kernel_cb_mgr, (u32) cb_handle);
399 /* hl_cb_get should never fail here so use kernel WARN */
400 WARN(!cb, "Kernel CB handle invalid 0x%x\n", (u32) cb_handle);
407 hl_cb_destroy(hdev, &hdev->kernel_cb_mgr, cb_handle << PAGE_SHIFT);
412 int hl_cb_pool_init(struct hl_device *hdev)
417 INIT_LIST_HEAD(&hdev->cb_pool);
418 spin_lock_init(&hdev->cb_pool_lock);
420 for (i = 0 ; i < hdev->asic_prop.cb_pool_cb_cnt ; i++) {
421 cb = hl_cb_alloc(hdev, hdev->asic_prop.cb_pool_cb_size,
425 list_add(&cb->pool_list, &hdev->cb_pool);
427 hl_cb_pool_fini(hdev);
435 int hl_cb_pool_fini(struct hl_device *hdev)
437 struct hl_cb *cb, *tmp;
439 list_for_each_entry_safe(cb, tmp, &hdev->cb_pool, pool_list) {
440 list_del(&cb->pool_list);