2 * Copyright 2014 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
23 #include <linux/mutex.h>
24 #include <linux/log2.h>
25 #include <linux/sched.h>
26 #include <linux/sched/mm.h>
27 #include <linux/slab.h>
28 #include <linux/amd-iommu.h>
29 #include <linux/notifier.h>
30 #include <linux/compat.h>
35 #include "kfd_dbgmgr.h"
38 * List of struct kfd_process (field kfd_process).
39 * Unique/indexed by mm_struct*
41 #define KFD_PROCESS_TABLE_SIZE 5 /* bits: 32 entries */
42 static DEFINE_HASHTABLE(kfd_processes_table, KFD_PROCESS_TABLE_SIZE);
43 static DEFINE_MUTEX(kfd_processes_mutex);
45 DEFINE_STATIC_SRCU(kfd_processes_srcu);
47 static struct workqueue_struct *kfd_process_wq;
49 struct kfd_process_release_work {
50 struct work_struct kfd_work;
51 struct kfd_process *p;
54 static struct kfd_process *find_process(const struct task_struct *thread);
55 static struct kfd_process *create_process(const struct task_struct *thread);
57 void kfd_process_create_wq(void)
60 kfd_process_wq = alloc_workqueue("kfd_process_wq", 0, 0);
63 void kfd_process_destroy_wq(void)
66 destroy_workqueue(kfd_process_wq);
67 kfd_process_wq = NULL;
71 struct kfd_process *kfd_create_process(const struct task_struct *thread)
73 struct kfd_process *process;
76 return ERR_PTR(-EINVAL);
78 /* Only the pthreads threading model is supported. */
79 if (thread->group_leader->mm != thread->mm)
80 return ERR_PTR(-EINVAL);
82 /* Take mmap_sem because we call __mmu_notifier_register inside */
83 down_write(&thread->mm->mmap_sem);
86 * take kfd processes mutex before starting of process creation
87 * so there won't be a case where two threads of the same process
88 * create two kfd_process structures
90 mutex_lock(&kfd_processes_mutex);
92 /* A prior open of /dev/kfd could have already created the process. */
93 process = find_process(thread);
95 pr_debug("Process already found\n");
98 process = create_process(thread);
100 mutex_unlock(&kfd_processes_mutex);
102 up_write(&thread->mm->mmap_sem);
107 struct kfd_process *kfd_get_process(const struct task_struct *thread)
109 struct kfd_process *process;
112 return ERR_PTR(-EINVAL);
114 /* Only the pthreads threading model is supported. */
115 if (thread->group_leader->mm != thread->mm)
116 return ERR_PTR(-EINVAL);
118 process = find_process(thread);
123 static struct kfd_process *find_process_by_mm(const struct mm_struct *mm)
125 struct kfd_process *process;
127 hash_for_each_possible_rcu(kfd_processes_table, process,
128 kfd_processes, (uintptr_t)mm)
129 if (process->mm == mm)
135 static struct kfd_process *find_process(const struct task_struct *thread)
137 struct kfd_process *p;
140 idx = srcu_read_lock(&kfd_processes_srcu);
141 p = find_process_by_mm(thread->mm);
142 srcu_read_unlock(&kfd_processes_srcu, idx);
147 static void kfd_process_wq_release(struct work_struct *work)
149 struct kfd_process_release_work *my_work;
150 struct kfd_process_device *pdd, *temp;
151 struct kfd_process *p;
153 my_work = (struct kfd_process_release_work *) work;
157 pr_debug("Releasing process (pasid %d) in workqueue\n",
160 mutex_lock(&p->mutex);
162 list_for_each_entry_safe(pdd, temp, &p->per_device_data,
164 pr_debug("Releasing pdd (topology id %d) for process (pasid %d) in workqueue\n",
165 pdd->dev->id, p->pasid);
167 if (pdd->bound == PDD_BOUND)
168 amd_iommu_unbind_pasid(pdd->dev->pdev, p->pasid);
170 list_del(&pdd->per_device_list);
174 kfd_event_free_process(p);
176 kfd_pasid_free(p->pasid);
177 kfd_free_process_doorbells(p);
179 mutex_unlock(&p->mutex);
181 mutex_destroy(&p->mutex);
188 static void kfd_process_destroy_delayed(struct rcu_head *rcu)
190 struct kfd_process_release_work *work;
191 struct kfd_process *p;
193 p = container_of(rcu, struct kfd_process, rcu);
197 work = kmalloc(sizeof(struct kfd_process_release_work), GFP_ATOMIC);
200 INIT_WORK((struct work_struct *) work, kfd_process_wq_release);
202 queue_work(kfd_process_wq, (struct work_struct *) work);
206 static void kfd_process_notifier_release(struct mmu_notifier *mn,
207 struct mm_struct *mm)
209 struct kfd_process *p;
210 struct kfd_process_device *pdd = NULL;
213 * The kfd_process structure can not be free because the
214 * mmu_notifier srcu is read locked
216 p = container_of(mn, struct kfd_process, mmu_notifier);
217 if (WARN_ON(p->mm != mm))
220 mutex_lock(&kfd_processes_mutex);
221 hash_del_rcu(&p->kfd_processes);
222 mutex_unlock(&kfd_processes_mutex);
223 synchronize_srcu(&kfd_processes_srcu);
225 mutex_lock(&p->mutex);
227 /* Iterate over all process device data structures and if the
228 * pdd is in debug mode, we should first force unregistration,
229 * then we will be able to destroy the queues
231 list_for_each_entry(pdd, &p->per_device_data, per_device_list) {
232 struct kfd_dev *dev = pdd->dev;
234 mutex_lock(kfd_get_dbgmgr_mutex());
235 if (dev && dev->dbgmgr && dev->dbgmgr->pasid == p->pasid) {
236 if (!kfd_dbgmgr_unregister(dev->dbgmgr, p)) {
237 kfd_dbgmgr_destroy(dev->dbgmgr);
241 mutex_unlock(kfd_get_dbgmgr_mutex());
244 kfd_process_dequeue_from_all_devices(p);
247 mutex_unlock(&p->mutex);
250 * Because we drop mm_count inside kfd_process_destroy_delayed
251 * and because the mmu_notifier_unregister function also drop
252 * mm_count we need to take an extra count here.
255 mmu_notifier_unregister_no_release(&p->mmu_notifier, p->mm);
256 mmu_notifier_call_srcu(&p->rcu, &kfd_process_destroy_delayed);
259 static const struct mmu_notifier_ops kfd_process_mmu_notifier_ops = {
260 .release = kfd_process_notifier_release,
263 static struct kfd_process *create_process(const struct task_struct *thread)
265 struct kfd_process *process;
268 process = kzalloc(sizeof(*process), GFP_KERNEL);
271 goto err_alloc_process;
273 process->pasid = kfd_pasid_alloc();
274 if (process->pasid == 0)
275 goto err_alloc_pasid;
277 if (kfd_alloc_process_doorbells(process) < 0)
278 goto err_alloc_doorbells;
280 mutex_init(&process->mutex);
282 process->mm = thread->mm;
284 /* register notifier */
285 process->mmu_notifier.ops = &kfd_process_mmu_notifier_ops;
286 err = __mmu_notifier_register(&process->mmu_notifier, process->mm);
288 goto err_mmu_notifier;
290 hash_add_rcu(kfd_processes_table, &process->kfd_processes,
291 (uintptr_t)process->mm);
293 process->lead_thread = thread->group_leader;
295 INIT_LIST_HEAD(&process->per_device_data);
297 kfd_event_init_process(process);
299 err = pqm_init(&process->pqm, process);
301 goto err_process_pqm_init;
303 /* init process apertures*/
304 process->is_32bit_user_mode = in_compat_syscall();
305 err = kfd_init_apertures(process);
307 goto err_init_apertures;
312 pqm_uninit(&process->pqm);
313 err_process_pqm_init:
314 hash_del_rcu(&process->kfd_processes);
316 mmu_notifier_unregister_no_release(&process->mmu_notifier, process->mm);
318 mutex_destroy(&process->mutex);
319 kfd_free_process_doorbells(process);
321 kfd_pasid_free(process->pasid);
328 struct kfd_process_device *kfd_get_process_device_data(struct kfd_dev *dev,
329 struct kfd_process *p)
331 struct kfd_process_device *pdd = NULL;
333 list_for_each_entry(pdd, &p->per_device_data, per_device_list)
340 struct kfd_process_device *kfd_create_process_device_data(struct kfd_dev *dev,
341 struct kfd_process *p)
343 struct kfd_process_device *pdd = NULL;
345 pdd = kzalloc(sizeof(*pdd), GFP_KERNEL);
348 INIT_LIST_HEAD(&pdd->qpd.queues_list);
349 INIT_LIST_HEAD(&pdd->qpd.priv_queue_list);
350 pdd->qpd.dqm = dev->dqm;
352 pdd->bound = PDD_UNBOUND;
353 pdd->already_dequeued = false;
354 list_add(&pdd->per_device_list, &p->per_device_data);
361 * Direct the IOMMU to bind the process (specifically the pasid->mm)
363 * Unbinding occurs when the process dies or the device is removed.
365 * Assumes that the process lock is held.
367 struct kfd_process_device *kfd_bind_process_to_device(struct kfd_dev *dev,
368 struct kfd_process *p)
370 struct kfd_process_device *pdd;
373 pdd = kfd_get_process_device_data(dev, p);
375 pr_err("Process device data doesn't exist\n");
376 return ERR_PTR(-ENOMEM);
379 if (pdd->bound == PDD_BOUND) {
381 } else if (unlikely(pdd->bound == PDD_BOUND_SUSPENDED)) {
382 pr_err("Binding PDD_BOUND_SUSPENDED pdd is unexpected!\n");
383 return ERR_PTR(-EINVAL);
386 err = amd_iommu_bind_pasid(dev->pdev, p->pasid, p->lead_thread);
390 pdd->bound = PDD_BOUND;
396 * Bind processes do the device that have been temporarily unbound
397 * (PDD_BOUND_SUSPENDED) in kfd_unbind_processes_from_device.
399 int kfd_bind_processes_to_device(struct kfd_dev *dev)
401 struct kfd_process_device *pdd;
402 struct kfd_process *p;
406 int idx = srcu_read_lock(&kfd_processes_srcu);
408 hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
409 mutex_lock(&p->mutex);
410 pdd = kfd_get_process_device_data(dev, p);
411 if (pdd->bound != PDD_BOUND_SUSPENDED) {
412 mutex_unlock(&p->mutex);
416 err = amd_iommu_bind_pasid(dev->pdev, p->pasid,
419 pr_err("Unexpected pasid %d binding failure\n",
421 mutex_unlock(&p->mutex);
425 pdd->bound = PDD_BOUND;
426 mutex_unlock(&p->mutex);
429 srcu_read_unlock(&kfd_processes_srcu, idx);
435 * Mark currently bound processes as PDD_BOUND_SUSPENDED. These
436 * processes will be restored to PDD_BOUND state in
437 * kfd_bind_processes_to_device.
439 void kfd_unbind_processes_from_device(struct kfd_dev *dev)
441 struct kfd_process_device *pdd;
442 struct kfd_process *p;
445 int idx = srcu_read_lock(&kfd_processes_srcu);
447 hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
448 mutex_lock(&p->mutex);
449 pdd = kfd_get_process_device_data(dev, p);
451 if (pdd->bound == PDD_BOUND)
452 pdd->bound = PDD_BOUND_SUSPENDED;
453 mutex_unlock(&p->mutex);
456 srcu_read_unlock(&kfd_processes_srcu, idx);
459 void kfd_process_iommu_unbind_callback(struct kfd_dev *dev, unsigned int pasid)
461 struct kfd_process *p;
462 struct kfd_process_device *pdd;
465 * Look for the process that matches the pasid. If there is no such
466 * process, we either released it in amdkfd's own notifier, or there
467 * is a bug. Unfortunately, there is no way to tell...
469 p = kfd_lookup_process_by_pasid(pasid);
473 pr_debug("Unbinding process %d from IOMMU\n", pasid);
475 mutex_lock(kfd_get_dbgmgr_mutex());
477 if (dev->dbgmgr && dev->dbgmgr->pasid == p->pasid) {
478 if (!kfd_dbgmgr_unregister(dev->dbgmgr, p)) {
479 kfd_dbgmgr_destroy(dev->dbgmgr);
484 mutex_unlock(kfd_get_dbgmgr_mutex());
486 pdd = kfd_get_process_device_data(dev, p);
488 /* For GPU relying on IOMMU, we need to dequeue here
489 * when PASID is still bound.
491 kfd_process_dequeue_from_device(pdd);
493 mutex_unlock(&p->mutex);
496 struct kfd_process_device *kfd_get_first_process_device_data(
497 struct kfd_process *p)
499 return list_first_entry(&p->per_device_data,
500 struct kfd_process_device,
504 struct kfd_process_device *kfd_get_next_process_device_data(
505 struct kfd_process *p,
506 struct kfd_process_device *pdd)
508 if (list_is_last(&pdd->per_device_list, &p->per_device_data))
510 return list_next_entry(pdd, per_device_list);
513 bool kfd_has_process_device_data(struct kfd_process *p)
515 return !(list_empty(&p->per_device_data));
518 /* This returns with process->mutex locked. */
519 struct kfd_process *kfd_lookup_process_by_pasid(unsigned int pasid)
521 struct kfd_process *p;
524 int idx = srcu_read_lock(&kfd_processes_srcu);
526 hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
527 if (p->pasid == pasid) {
528 mutex_lock(&p->mutex);
533 srcu_read_unlock(&kfd_processes_srcu, idx);