1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23#include <linux/mutex.h>
24#include <linux/log2.h>
25#include <linux/sched.h>
26#include <linux/sched/mm.h>
27#include <linux/sched/task.h>
28#include <linux/mmu_context.h>
29#include <linux/slab.h>
30#include <linux/amd-iommu.h>
31#include <linux/notifier.h>
32#include <linux/compat.h>
33#include <linux/mman.h>
34#include <linux/file.h>
35#include <linux/pm_runtime.h>
36#include "amdgpu_amdkfd.h"
37#include "amdgpu.h"
38
39struct mm_struct;
40
41#include "kfd_priv.h"
42#include "kfd_device_queue_manager.h"
43#include "kfd_dbgmgr.h"
44#include "kfd_iommu.h"
45#include "kfd_svm.h"
46
47
48
49
50
51DEFINE_HASHTABLE(kfd_processes_table, KFD_PROCESS_TABLE_SIZE);
52static DEFINE_MUTEX(kfd_processes_mutex);
53
54DEFINE_SRCU(kfd_processes_srcu);
55
56
57static struct workqueue_struct *kfd_process_wq;
58
59
60
61
62
63
64
65static struct workqueue_struct *kfd_restore_wq;
66
67static struct kfd_process *find_process(const struct task_struct *thread);
68static void kfd_process_ref_release(struct kref *ref);
69static struct kfd_process *create_process(const struct task_struct *thread);
70static int kfd_process_init_cwsr_apu(struct kfd_process *p, struct file *filep);
71
72static void evict_process_worker(struct work_struct *work);
73static void restore_process_worker(struct work_struct *work);
74
75static void kfd_process_device_destroy_cwsr_dgpu(struct kfd_process_device *pdd);
76
77struct kfd_procfs_tree {
78 struct kobject *kobj;
79};
80
81static struct kfd_procfs_tree procfs;
82
83
84
85
86struct kfd_sdma_activity_handler_workarea {
87 struct work_struct sdma_activity_work;
88 struct kfd_process_device *pdd;
89 uint64_t sdma_activity_counter;
90};
91
92struct temp_sdma_queue_list {
93 uint64_t __user *rptr;
94 uint64_t sdma_val;
95 unsigned int queue_id;
96 struct list_head list;
97};
98
99static void kfd_sdma_activity_worker(struct work_struct *work)
100{
101 struct kfd_sdma_activity_handler_workarea *workarea;
102 struct kfd_process_device *pdd;
103 uint64_t val;
104 struct mm_struct *mm;
105 struct queue *q;
106 struct qcm_process_device *qpd;
107 struct device_queue_manager *dqm;
108 int ret = 0;
109 struct temp_sdma_queue_list sdma_q_list;
110 struct temp_sdma_queue_list *sdma_q, *next;
111
112 workarea = container_of(work, struct kfd_sdma_activity_handler_workarea,
113 sdma_activity_work);
114
115 pdd = workarea->pdd;
116 if (!pdd)
117 return;
118 dqm = pdd->dev->dqm;
119 qpd = &pdd->qpd;
120 if (!dqm || !qpd)
121 return;
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144 INIT_LIST_HEAD(&sdma_q_list.list);
145
146
147
148
149 dqm_lock(dqm);
150
151 list_for_each_entry(q, &qpd->queues_list, list) {
152 if ((q->properties.type != KFD_QUEUE_TYPE_SDMA) &&
153 (q->properties.type != KFD_QUEUE_TYPE_SDMA_XGMI))
154 continue;
155
156 sdma_q = kzalloc(sizeof(struct temp_sdma_queue_list), GFP_KERNEL);
157 if (!sdma_q) {
158 dqm_unlock(dqm);
159 goto cleanup;
160 }
161
162 INIT_LIST_HEAD(&sdma_q->list);
163 sdma_q->rptr = (uint64_t __user *)q->properties.read_ptr;
164 sdma_q->queue_id = q->properties.queue_id;
165 list_add_tail(&sdma_q->list, &sdma_q_list.list);
166 }
167
168
169
170
171
172
173 if (list_empty(&sdma_q_list.list)) {
174 workarea->sdma_activity_counter = pdd->sdma_past_activity_counter;
175 dqm_unlock(dqm);
176 return;
177 }
178
179 dqm_unlock(dqm);
180
181
182
183
184 mm = get_task_mm(pdd->process->lead_thread);
185 if (!mm)
186 goto cleanup;
187
188 kthread_use_mm(mm);
189
190 list_for_each_entry(sdma_q, &sdma_q_list.list, list) {
191 val = 0;
192 ret = read_sdma_queue_counter(sdma_q->rptr, &val);
193 if (ret) {
194 pr_debug("Failed to read SDMA queue active counter for queue id: %d",
195 sdma_q->queue_id);
196 } else {
197 sdma_q->sdma_val = val;
198 workarea->sdma_activity_counter += val;
199 }
200 }
201
202 kthread_unuse_mm(mm);
203 mmput(mm);
204
205
206
207
208
209 dqm_lock(dqm);
210
211 workarea->sdma_activity_counter += pdd->sdma_past_activity_counter;
212
213 list_for_each_entry(q, &qpd->queues_list, list) {
214 if (list_empty(&sdma_q_list.list))
215 break;
216
217 if ((q->properties.type != KFD_QUEUE_TYPE_SDMA) &&
218 (q->properties.type != KFD_QUEUE_TYPE_SDMA_XGMI))
219 continue;
220
221 list_for_each_entry_safe(sdma_q, next, &sdma_q_list.list, list) {
222 if (((uint64_t __user *)q->properties.read_ptr == sdma_q->rptr) &&
223 (sdma_q->queue_id == q->properties.queue_id)) {
224 list_del(&sdma_q->list);
225 kfree(sdma_q);
226 break;
227 }
228 }
229 }
230
231 dqm_unlock(dqm);
232
233
234
235
236
237
238 list_for_each_entry_safe(sdma_q, next, &sdma_q_list.list, list) {
239 workarea->sdma_activity_counter -= sdma_q->sdma_val;
240 list_del(&sdma_q->list);
241 kfree(sdma_q);
242 }
243
244 return;
245
246cleanup:
247 list_for_each_entry_safe(sdma_q, next, &sdma_q_list.list, list) {
248 list_del(&sdma_q->list);
249 kfree(sdma_q);
250 }
251}
252
253
254
255
256
257
258
259
260
261
262
263
264
265static int kfd_get_cu_occupancy(struct attribute *attr, char *buffer)
266{
267 int cu_cnt;
268 int wave_cnt;
269 int max_waves_per_cu;
270 struct kfd_dev *dev = NULL;
271 struct kfd_process *proc = NULL;
272 struct kfd_process_device *pdd = NULL;
273
274 pdd = container_of(attr, struct kfd_process_device, attr_cu_occupancy);
275 dev = pdd->dev;
276 if (dev->kfd2kgd->get_cu_occupancy == NULL)
277 return -EINVAL;
278
279 cu_cnt = 0;
280 proc = pdd->process;
281 if (pdd->qpd.queue_count == 0) {
282 pr_debug("Gpu-Id: %d has no active queues for process %d\n",
283 dev->id, proc->pasid);
284 return snprintf(buffer, PAGE_SIZE, "%d\n", cu_cnt);
285 }
286
287
288 wave_cnt = 0;
289 max_waves_per_cu = 0;
290 dev->kfd2kgd->get_cu_occupancy(dev->adev, proc->pasid, &wave_cnt,
291 &max_waves_per_cu);
292
293
294 cu_cnt = (wave_cnt + (max_waves_per_cu - 1)) / max_waves_per_cu;
295 return snprintf(buffer, PAGE_SIZE, "%d\n", cu_cnt);
296}
297
298static ssize_t kfd_procfs_show(struct kobject *kobj, struct attribute *attr,
299 char *buffer)
300{
301 if (strcmp(attr->name, "pasid") == 0) {
302 struct kfd_process *p = container_of(attr, struct kfd_process,
303 attr_pasid);
304
305 return snprintf(buffer, PAGE_SIZE, "%d\n", p->pasid);
306 } else if (strncmp(attr->name, "vram_", 5) == 0) {
307 struct kfd_process_device *pdd = container_of(attr, struct kfd_process_device,
308 attr_vram);
309 return snprintf(buffer, PAGE_SIZE, "%llu\n", READ_ONCE(pdd->vram_usage));
310 } else if (strncmp(attr->name, "sdma_", 5) == 0) {
311 struct kfd_process_device *pdd = container_of(attr, struct kfd_process_device,
312 attr_sdma);
313 struct kfd_sdma_activity_handler_workarea sdma_activity_work_handler;
314
315 INIT_WORK(&sdma_activity_work_handler.sdma_activity_work,
316 kfd_sdma_activity_worker);
317
318 sdma_activity_work_handler.pdd = pdd;
319 sdma_activity_work_handler.sdma_activity_counter = 0;
320
321 schedule_work(&sdma_activity_work_handler.sdma_activity_work);
322
323 flush_work(&sdma_activity_work_handler.sdma_activity_work);
324
325 return snprintf(buffer, PAGE_SIZE, "%llu\n",
326 (sdma_activity_work_handler.sdma_activity_counter)/
327 SDMA_ACTIVITY_DIVISOR);
328 } else {
329 pr_err("Invalid attribute");
330 return -EINVAL;
331 }
332
333 return 0;
334}
335
336static void kfd_procfs_kobj_release(struct kobject *kobj)
337{
338 kfree(kobj);
339}
340
341static const struct sysfs_ops kfd_procfs_ops = {
342 .show = kfd_procfs_show,
343};
344
345static struct kobj_type procfs_type = {
346 .release = kfd_procfs_kobj_release,
347 .sysfs_ops = &kfd_procfs_ops,
348};
349
350void kfd_procfs_init(void)
351{
352 int ret = 0;
353
354 procfs.kobj = kfd_alloc_struct(procfs.kobj);
355 if (!procfs.kobj)
356 return;
357
358 ret = kobject_init_and_add(procfs.kobj, &procfs_type,
359 &kfd_device->kobj, "proc");
360 if (ret) {
361 pr_warn("Could not create procfs proc folder");
362
363 kfd_procfs_shutdown();
364 }
365}
366
367void kfd_procfs_shutdown(void)
368{
369 if (procfs.kobj) {
370 kobject_del(procfs.kobj);
371 kobject_put(procfs.kobj);
372 procfs.kobj = NULL;
373 }
374}
375
376static ssize_t kfd_procfs_queue_show(struct kobject *kobj,
377 struct attribute *attr, char *buffer)
378{
379 struct queue *q = container_of(kobj, struct queue, kobj);
380
381 if (!strcmp(attr->name, "size"))
382 return snprintf(buffer, PAGE_SIZE, "%llu",
383 q->properties.queue_size);
384 else if (!strcmp(attr->name, "type"))
385 return snprintf(buffer, PAGE_SIZE, "%d", q->properties.type);
386 else if (!strcmp(attr->name, "gpuid"))
387 return snprintf(buffer, PAGE_SIZE, "%u", q->device->id);
388 else
389 pr_err("Invalid attribute");
390
391 return 0;
392}
393
394static ssize_t kfd_procfs_stats_show(struct kobject *kobj,
395 struct attribute *attr, char *buffer)
396{
397 if (strcmp(attr->name, "evicted_ms") == 0) {
398 struct kfd_process_device *pdd = container_of(attr,
399 struct kfd_process_device,
400 attr_evict);
401 uint64_t evict_jiffies;
402
403 evict_jiffies = atomic64_read(&pdd->evict_duration_counter);
404
405 return snprintf(buffer,
406 PAGE_SIZE,
407 "%llu\n",
408 jiffies64_to_msecs(evict_jiffies));
409
410
411 } else if (strcmp(attr->name, "cu_occupancy") == 0) {
412 return kfd_get_cu_occupancy(attr, buffer);
413 } else {
414 pr_err("Invalid attribute");
415 }
416
417 return 0;
418}
419
420static ssize_t kfd_sysfs_counters_show(struct kobject *kobj,
421 struct attribute *attr, char *buf)
422{
423 struct kfd_process_device *pdd;
424
425 if (!strcmp(attr->name, "faults")) {
426 pdd = container_of(attr, struct kfd_process_device,
427 attr_faults);
428 return sysfs_emit(buf, "%llu\n", READ_ONCE(pdd->faults));
429 }
430 if (!strcmp(attr->name, "page_in")) {
431 pdd = container_of(attr, struct kfd_process_device,
432 attr_page_in);
433 return sysfs_emit(buf, "%llu\n", READ_ONCE(pdd->page_in));
434 }
435 if (!strcmp(attr->name, "page_out")) {
436 pdd = container_of(attr, struct kfd_process_device,
437 attr_page_out);
438 return sysfs_emit(buf, "%llu\n", READ_ONCE(pdd->page_out));
439 }
440 return 0;
441}
442
443static struct attribute attr_queue_size = {
444 .name = "size",
445 .mode = KFD_SYSFS_FILE_MODE
446};
447
448static struct attribute attr_queue_type = {
449 .name = "type",
450 .mode = KFD_SYSFS_FILE_MODE
451};
452
453static struct attribute attr_queue_gpuid = {
454 .name = "gpuid",
455 .mode = KFD_SYSFS_FILE_MODE
456};
457
458static struct attribute *procfs_queue_attrs[] = {
459 &attr_queue_size,
460 &attr_queue_type,
461 &attr_queue_gpuid,
462 NULL
463};
464ATTRIBUTE_GROUPS(procfs_queue);
465
466static const struct sysfs_ops procfs_queue_ops = {
467 .show = kfd_procfs_queue_show,
468};
469
470static struct kobj_type procfs_queue_type = {
471 .sysfs_ops = &procfs_queue_ops,
472 .default_groups = procfs_queue_groups,
473};
474
475static const struct sysfs_ops procfs_stats_ops = {
476 .show = kfd_procfs_stats_show,
477};
478
479static struct kobj_type procfs_stats_type = {
480 .sysfs_ops = &procfs_stats_ops,
481 .release = kfd_procfs_kobj_release,
482};
483
484static const struct sysfs_ops sysfs_counters_ops = {
485 .show = kfd_sysfs_counters_show,
486};
487
488static struct kobj_type sysfs_counters_type = {
489 .sysfs_ops = &sysfs_counters_ops,
490 .release = kfd_procfs_kobj_release,
491};
492
493int kfd_procfs_add_queue(struct queue *q)
494{
495 struct kfd_process *proc;
496 int ret;
497
498 if (!q || !q->process)
499 return -EINVAL;
500 proc = q->process;
501
502
503 if (!proc->kobj_queues)
504 return -EFAULT;
505 ret = kobject_init_and_add(&q->kobj, &procfs_queue_type,
506 proc->kobj_queues, "%u", q->properties.queue_id);
507 if (ret < 0) {
508 pr_warn("Creating proc/<pid>/queues/%u failed",
509 q->properties.queue_id);
510 kobject_put(&q->kobj);
511 return ret;
512 }
513
514 return 0;
515}
516
517static void kfd_sysfs_create_file(struct kobject *kobj, struct attribute *attr,
518 char *name)
519{
520 int ret;
521
522 if (!kobj || !attr || !name)
523 return;
524
525 attr->name = name;
526 attr->mode = KFD_SYSFS_FILE_MODE;
527 sysfs_attr_init(attr);
528
529 ret = sysfs_create_file(kobj, attr);
530 if (ret)
531 pr_warn("Create sysfs %s/%s failed %d", kobj->name, name, ret);
532}
533
534static void kfd_procfs_add_sysfs_stats(struct kfd_process *p)
535{
536 int ret;
537 int i;
538 char stats_dir_filename[MAX_SYSFS_FILENAME_LEN];
539
540 if (!p || !p->kobj)
541 return;
542
543
544
545
546
547
548
549 for (i = 0; i < p->n_pdds; i++) {
550 struct kfd_process_device *pdd = p->pdds[i];
551
552 snprintf(stats_dir_filename, MAX_SYSFS_FILENAME_LEN,
553 "stats_%u", pdd->dev->id);
554 pdd->kobj_stats = kfd_alloc_struct(pdd->kobj_stats);
555 if (!pdd->kobj_stats)
556 return;
557
558 ret = kobject_init_and_add(pdd->kobj_stats,
559 &procfs_stats_type,
560 p->kobj,
561 stats_dir_filename);
562
563 if (ret) {
564 pr_warn("Creating KFD proc/stats_%s folder failed",
565 stats_dir_filename);
566 kobject_put(pdd->kobj_stats);
567 pdd->kobj_stats = NULL;
568 return;
569 }
570
571 kfd_sysfs_create_file(pdd->kobj_stats, &pdd->attr_evict,
572 "evicted_ms");
573
574 if (pdd->dev->kfd2kgd->get_cu_occupancy)
575 kfd_sysfs_create_file(pdd->kobj_stats,
576 &pdd->attr_cu_occupancy,
577 "cu_occupancy");
578 }
579}
580
581static void kfd_procfs_add_sysfs_counters(struct kfd_process *p)
582{
583 int ret = 0;
584 int i;
585 char counters_dir_filename[MAX_SYSFS_FILENAME_LEN];
586
587 if (!p || !p->kobj)
588 return;
589
590
591
592
593
594
595
596
597 for_each_set_bit(i, p->svms.bitmap_supported, p->n_pdds) {
598 struct kfd_process_device *pdd = p->pdds[i];
599 struct kobject *kobj_counters;
600
601 snprintf(counters_dir_filename, MAX_SYSFS_FILENAME_LEN,
602 "counters_%u", pdd->dev->id);
603 kobj_counters = kfd_alloc_struct(kobj_counters);
604 if (!kobj_counters)
605 return;
606
607 ret = kobject_init_and_add(kobj_counters, &sysfs_counters_type,
608 p->kobj, counters_dir_filename);
609 if (ret) {
610 pr_warn("Creating KFD proc/%s folder failed",
611 counters_dir_filename);
612 kobject_put(kobj_counters);
613 return;
614 }
615
616 pdd->kobj_counters = kobj_counters;
617 kfd_sysfs_create_file(kobj_counters, &pdd->attr_faults,
618 "faults");
619 kfd_sysfs_create_file(kobj_counters, &pdd->attr_page_in,
620 "page_in");
621 kfd_sysfs_create_file(kobj_counters, &pdd->attr_page_out,
622 "page_out");
623 }
624}
625
626static void kfd_procfs_add_sysfs_files(struct kfd_process *p)
627{
628 int i;
629
630 if (!p || !p->kobj)
631 return;
632
633
634
635
636
637
638 for (i = 0; i < p->n_pdds; i++) {
639 struct kfd_process_device *pdd = p->pdds[i];
640
641 snprintf(pdd->vram_filename, MAX_SYSFS_FILENAME_LEN, "vram_%u",
642 pdd->dev->id);
643 kfd_sysfs_create_file(p->kobj, &pdd->attr_vram,
644 pdd->vram_filename);
645
646 snprintf(pdd->sdma_filename, MAX_SYSFS_FILENAME_LEN, "sdma_%u",
647 pdd->dev->id);
648 kfd_sysfs_create_file(p->kobj, &pdd->attr_sdma,
649 pdd->sdma_filename);
650 }
651}
652
653void kfd_procfs_del_queue(struct queue *q)
654{
655 if (!q)
656 return;
657
658 kobject_del(&q->kobj);
659 kobject_put(&q->kobj);
660}
661
662int kfd_process_create_wq(void)
663{
664 if (!kfd_process_wq)
665 kfd_process_wq = alloc_workqueue("kfd_process_wq", 0, 0);
666 if (!kfd_restore_wq)
667 kfd_restore_wq = alloc_ordered_workqueue("kfd_restore_wq", 0);
668
669 if (!kfd_process_wq || !kfd_restore_wq) {
670 kfd_process_destroy_wq();
671 return -ENOMEM;
672 }
673
674 return 0;
675}
676
677void kfd_process_destroy_wq(void)
678{
679 if (kfd_process_wq) {
680 destroy_workqueue(kfd_process_wq);
681 kfd_process_wq = NULL;
682 }
683 if (kfd_restore_wq) {
684 destroy_workqueue(kfd_restore_wq);
685 kfd_restore_wq = NULL;
686 }
687}
688
689static void kfd_process_free_gpuvm(struct kgd_mem *mem,
690 struct kfd_process_device *pdd, void *kptr)
691{
692 struct kfd_dev *dev = pdd->dev;
693
694 if (kptr) {
695 amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel(dev->adev, mem);
696 kptr = NULL;
697 }
698
699 amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(dev->adev, mem, pdd->drm_priv);
700 amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->adev, mem, pdd->drm_priv,
701 NULL);
702}
703
704
705
706
707
708
709
710static int kfd_process_alloc_gpuvm(struct kfd_process_device *pdd,
711 uint64_t gpu_va, uint32_t size,
712 uint32_t flags, struct kgd_mem **mem, void **kptr)
713{
714 struct kfd_dev *kdev = pdd->dev;
715 int err;
716
717 err = amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(kdev->adev, gpu_va, size,
718 pdd->drm_priv, mem, NULL, flags);
719 if (err)
720 goto err_alloc_mem;
721
722 err = amdgpu_amdkfd_gpuvm_map_memory_to_gpu(kdev->adev, *mem,
723 pdd->drm_priv, NULL);
724 if (err)
725 goto err_map_mem;
726
727 err = amdgpu_amdkfd_gpuvm_sync_memory(kdev->adev, *mem, true);
728 if (err) {
729 pr_debug("Sync memory failed, wait interrupted by user signal\n");
730 goto sync_memory_failed;
731 }
732
733 if (kptr) {
734 err = amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(kdev->adev,
735 (struct kgd_mem *)*mem, kptr, NULL);
736 if (err) {
737 pr_debug("Map GTT BO to kernel failed\n");
738 goto sync_memory_failed;
739 }
740 }
741
742 return err;
743
744sync_memory_failed:
745 amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(kdev->adev, *mem, pdd->drm_priv);
746
747err_map_mem:
748 amdgpu_amdkfd_gpuvm_free_memory_of_gpu(kdev->adev, *mem, pdd->drm_priv,
749 NULL);
750err_alloc_mem:
751 *mem = NULL;
752 *kptr = NULL;
753 return err;
754}
755
756
757
758
759
760
761
762static int kfd_process_device_reserve_ib_mem(struct kfd_process_device *pdd)
763{
764 struct qcm_process_device *qpd = &pdd->qpd;
765 uint32_t flags = KFD_IOC_ALLOC_MEM_FLAGS_GTT |
766 KFD_IOC_ALLOC_MEM_FLAGS_NO_SUBSTITUTE |
767 KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE |
768 KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE;
769 struct kgd_mem *mem;
770 void *kaddr;
771 int ret;
772
773 if (qpd->ib_kaddr || !qpd->ib_base)
774 return 0;
775
776
777 ret = kfd_process_alloc_gpuvm(pdd, qpd->ib_base, PAGE_SIZE, flags,
778 &mem, &kaddr);
779 if (ret)
780 return ret;
781
782 qpd->ib_mem = mem;
783 qpd->ib_kaddr = kaddr;
784
785 return 0;
786}
787
788static void kfd_process_device_destroy_ib_mem(struct kfd_process_device *pdd)
789{
790 struct qcm_process_device *qpd = &pdd->qpd;
791
792 if (!qpd->ib_kaddr || !qpd->ib_base)
793 return;
794
795 kfd_process_free_gpuvm(qpd->ib_mem, pdd, qpd->ib_kaddr);
796}
797
798struct kfd_process *kfd_create_process(struct file *filep)
799{
800 struct kfd_process *process;
801 struct task_struct *thread = current;
802 int ret;
803
804 if (!thread->mm)
805 return ERR_PTR(-EINVAL);
806
807
808 if (thread->group_leader->mm != thread->mm)
809 return ERR_PTR(-EINVAL);
810
811
812
813
814
815
816 mutex_lock(&kfd_processes_mutex);
817
818
819 process = find_process(thread);
820 if (process) {
821 pr_debug("Process already found\n");
822 } else {
823 process = create_process(thread);
824 if (IS_ERR(process))
825 goto out;
826
827 ret = kfd_process_init_cwsr_apu(process, filep);
828 if (ret)
829 goto out_destroy;
830
831 if (!procfs.kobj)
832 goto out;
833
834 process->kobj = kfd_alloc_struct(process->kobj);
835 if (!process->kobj) {
836 pr_warn("Creating procfs kobject failed");
837 goto out;
838 }
839 ret = kobject_init_and_add(process->kobj, &procfs_type,
840 procfs.kobj, "%d",
841 (int)process->lead_thread->pid);
842 if (ret) {
843 pr_warn("Creating procfs pid directory failed");
844 kobject_put(process->kobj);
845 goto out;
846 }
847
848 kfd_sysfs_create_file(process->kobj, &process->attr_pasid,
849 "pasid");
850
851 process->kobj_queues = kobject_create_and_add("queues",
852 process->kobj);
853 if (!process->kobj_queues)
854 pr_warn("Creating KFD proc/queues folder failed");
855
856 kfd_procfs_add_sysfs_stats(process);
857 kfd_procfs_add_sysfs_files(process);
858 kfd_procfs_add_sysfs_counters(process);
859 }
860out:
861 if (!IS_ERR(process))
862 kref_get(&process->ref);
863 mutex_unlock(&kfd_processes_mutex);
864
865 return process;
866
867out_destroy:
868 hash_del_rcu(&process->kfd_processes);
869 mutex_unlock(&kfd_processes_mutex);
870 synchronize_srcu(&kfd_processes_srcu);
871
872 mmu_notifier_put(&process->mmu_notifier);
873 return ERR_PTR(ret);
874}
875
876struct kfd_process *kfd_get_process(const struct task_struct *thread)
877{
878 struct kfd_process *process;
879
880 if (!thread->mm)
881 return ERR_PTR(-EINVAL);
882
883
884 if (thread->group_leader->mm != thread->mm)
885 return ERR_PTR(-EINVAL);
886
887 process = find_process(thread);
888 if (!process)
889 return ERR_PTR(-EINVAL);
890
891 return process;
892}
893
894static struct kfd_process *find_process_by_mm(const struct mm_struct *mm)
895{
896 struct kfd_process *process;
897
898 hash_for_each_possible_rcu(kfd_processes_table, process,
899 kfd_processes, (uintptr_t)mm)
900 if (process->mm == mm)
901 return process;
902
903 return NULL;
904}
905
906static struct kfd_process *find_process(const struct task_struct *thread)
907{
908 struct kfd_process *p;
909 int idx;
910
911 idx = srcu_read_lock(&kfd_processes_srcu);
912 p = find_process_by_mm(thread->mm);
913 srcu_read_unlock(&kfd_processes_srcu, idx);
914
915 return p;
916}
917
918void kfd_unref_process(struct kfd_process *p)
919{
920 kref_put(&p->ref, kfd_process_ref_release);
921}
922
923
924static void kfd_process_device_free_bos(struct kfd_process_device *pdd)
925{
926 struct kfd_process *p = pdd->process;
927 void *mem;
928 int id;
929 int i;
930
931
932
933
934
935 idr_for_each_entry(&pdd->alloc_idr, mem, id) {
936
937 for (i = 0; i < p->n_pdds; i++) {
938 struct kfd_process_device *peer_pdd = p->pdds[i];
939
940 if (!peer_pdd->drm_priv)
941 continue;
942 amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
943 peer_pdd->dev->adev, mem, peer_pdd->drm_priv);
944 }
945
946 amdgpu_amdkfd_gpuvm_free_memory_of_gpu(pdd->dev->adev, mem,
947 pdd->drm_priv, NULL);
948 kfd_process_device_remove_obj_handle(pdd, id);
949 }
950}
951
952
953
954
955
956static void kfd_process_kunmap_signal_bo(struct kfd_process *p)
957{
958 struct kfd_process_device *pdd;
959 struct kfd_dev *kdev;
960 void *mem;
961
962 kdev = kfd_device_by_id(GET_GPU_ID(p->signal_handle));
963 if (!kdev)
964 return;
965
966 mutex_lock(&p->mutex);
967
968 pdd = kfd_get_process_device_data(kdev, p);
969 if (!pdd)
970 goto out;
971
972 mem = kfd_process_device_translate_handle(
973 pdd, GET_IDR_HANDLE(p->signal_handle));
974 if (!mem)
975 goto out;
976
977 amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel(kdev->adev, mem);
978
979out:
980 mutex_unlock(&p->mutex);
981}
982
983static void kfd_process_free_outstanding_kfd_bos(struct kfd_process *p)
984{
985 int i;
986
987 for (i = 0; i < p->n_pdds; i++)
988 kfd_process_device_free_bos(p->pdds[i]);
989}
990
991static void kfd_process_destroy_pdds(struct kfd_process *p)
992{
993 int i;
994
995 for (i = 0; i < p->n_pdds; i++) {
996 struct kfd_process_device *pdd = p->pdds[i];
997
998 pr_debug("Releasing pdd (topology id %d) for process (pasid 0x%x)\n",
999 pdd->dev->id, p->pasid);
1000
1001 kfd_process_device_destroy_cwsr_dgpu(pdd);
1002 kfd_process_device_destroy_ib_mem(pdd);
1003
1004 if (pdd->drm_file) {
1005 amdgpu_amdkfd_gpuvm_release_process_vm(
1006 pdd->dev->adev, pdd->drm_priv);
1007 fput(pdd->drm_file);
1008 }
1009
1010 if (pdd->qpd.cwsr_kaddr && !pdd->qpd.cwsr_base)
1011 free_pages((unsigned long)pdd->qpd.cwsr_kaddr,
1012 get_order(KFD_CWSR_TBA_TMA_SIZE));
1013
1014 bitmap_free(pdd->qpd.doorbell_bitmap);
1015 idr_destroy(&pdd->alloc_idr);
1016
1017 kfd_free_process_doorbells(pdd->dev, pdd->doorbell_index);
1018
1019
1020
1021
1022
1023 if (pdd->runtime_inuse) {
1024 pm_runtime_mark_last_busy(pdd->dev->ddev->dev);
1025 pm_runtime_put_autosuspend(pdd->dev->ddev->dev);
1026 pdd->runtime_inuse = false;
1027 }
1028
1029 kfree(pdd);
1030 p->pdds[i] = NULL;
1031 }
1032 p->n_pdds = 0;
1033}
1034
1035static void kfd_process_remove_sysfs(struct kfd_process *p)
1036{
1037 struct kfd_process_device *pdd;
1038 int i;
1039
1040 if (!p->kobj)
1041 return;
1042
1043 sysfs_remove_file(p->kobj, &p->attr_pasid);
1044 kobject_del(p->kobj_queues);
1045 kobject_put(p->kobj_queues);
1046 p->kobj_queues = NULL;
1047
1048 for (i = 0; i < p->n_pdds; i++) {
1049 pdd = p->pdds[i];
1050
1051 sysfs_remove_file(p->kobj, &pdd->attr_vram);
1052 sysfs_remove_file(p->kobj, &pdd->attr_sdma);
1053
1054 sysfs_remove_file(pdd->kobj_stats, &pdd->attr_evict);
1055 if (pdd->dev->kfd2kgd->get_cu_occupancy)
1056 sysfs_remove_file(pdd->kobj_stats,
1057 &pdd->attr_cu_occupancy);
1058 kobject_del(pdd->kobj_stats);
1059 kobject_put(pdd->kobj_stats);
1060 pdd->kobj_stats = NULL;
1061 }
1062
1063 for_each_set_bit(i, p->svms.bitmap_supported, p->n_pdds) {
1064 pdd = p->pdds[i];
1065
1066 sysfs_remove_file(pdd->kobj_counters, &pdd->attr_faults);
1067 sysfs_remove_file(pdd->kobj_counters, &pdd->attr_page_in);
1068 sysfs_remove_file(pdd->kobj_counters, &pdd->attr_page_out);
1069 kobject_del(pdd->kobj_counters);
1070 kobject_put(pdd->kobj_counters);
1071 pdd->kobj_counters = NULL;
1072 }
1073
1074 kobject_del(p->kobj);
1075 kobject_put(p->kobj);
1076 p->kobj = NULL;
1077}
1078
1079
1080
1081
1082
1083
1084static void kfd_process_wq_release(struct work_struct *work)
1085{
1086 struct kfd_process *p = container_of(work, struct kfd_process,
1087 release_work);
1088
1089 kfd_process_remove_sysfs(p);
1090 kfd_iommu_unbind_process(p);
1091
1092 kfd_process_kunmap_signal_bo(p);
1093 kfd_process_free_outstanding_kfd_bos(p);
1094 svm_range_list_fini(p);
1095
1096 kfd_process_destroy_pdds(p);
1097 dma_fence_put(p->ef);
1098
1099 kfd_event_free_process(p);
1100
1101 kfd_pasid_free(p->pasid);
1102 mutex_destroy(&p->mutex);
1103
1104 put_task_struct(p->lead_thread);
1105
1106 kfree(p);
1107}
1108
1109static void kfd_process_ref_release(struct kref *ref)
1110{
1111 struct kfd_process *p = container_of(ref, struct kfd_process, ref);
1112
1113 INIT_WORK(&p->release_work, kfd_process_wq_release);
1114 queue_work(kfd_process_wq, &p->release_work);
1115}
1116
1117static struct mmu_notifier *kfd_process_alloc_notifier(struct mm_struct *mm)
1118{
1119 int idx = srcu_read_lock(&kfd_processes_srcu);
1120 struct kfd_process *p = find_process_by_mm(mm);
1121
1122 srcu_read_unlock(&kfd_processes_srcu, idx);
1123
1124 return p ? &p->mmu_notifier : ERR_PTR(-ESRCH);
1125}
1126
1127static void kfd_process_free_notifier(struct mmu_notifier *mn)
1128{
1129 kfd_unref_process(container_of(mn, struct kfd_process, mmu_notifier));
1130}
1131
1132static void kfd_process_notifier_release(struct mmu_notifier *mn,
1133 struct mm_struct *mm)
1134{
1135 struct kfd_process *p;
1136 int i;
1137
1138
1139
1140
1141
1142 p = container_of(mn, struct kfd_process, mmu_notifier);
1143 if (WARN_ON(p->mm != mm))
1144 return;
1145
1146 mutex_lock(&kfd_processes_mutex);
1147 hash_del_rcu(&p->kfd_processes);
1148 mutex_unlock(&kfd_processes_mutex);
1149 synchronize_srcu(&kfd_processes_srcu);
1150
1151 cancel_delayed_work_sync(&p->eviction_work);
1152 cancel_delayed_work_sync(&p->restore_work);
1153 cancel_delayed_work_sync(&p->svms.restore_work);
1154
1155 mutex_lock(&p->mutex);
1156
1157
1158
1159
1160
1161 for (i = 0; i < p->n_pdds; i++) {
1162 struct kfd_dev *dev = p->pdds[i]->dev;
1163
1164 mutex_lock(kfd_get_dbgmgr_mutex());
1165 if (dev && dev->dbgmgr && dev->dbgmgr->pasid == p->pasid) {
1166 if (!kfd_dbgmgr_unregister(dev->dbgmgr, p)) {
1167 kfd_dbgmgr_destroy(dev->dbgmgr);
1168 dev->dbgmgr = NULL;
1169 }
1170 }
1171 mutex_unlock(kfd_get_dbgmgr_mutex());
1172 }
1173
1174 kfd_process_dequeue_from_all_devices(p);
1175 pqm_uninit(&p->pqm);
1176
1177
1178 p->mm = NULL;
1179
1180
1181
1182
1183 dma_fence_signal(p->ef);
1184
1185 mutex_unlock(&p->mutex);
1186
1187 mmu_notifier_put(&p->mmu_notifier);
1188}
1189
1190static const struct mmu_notifier_ops kfd_process_mmu_notifier_ops = {
1191 .release = kfd_process_notifier_release,
1192 .alloc_notifier = kfd_process_alloc_notifier,
1193 .free_notifier = kfd_process_free_notifier,
1194};
1195
1196static int kfd_process_init_cwsr_apu(struct kfd_process *p, struct file *filep)
1197{
1198 unsigned long offset;
1199 int i;
1200
1201 for (i = 0; i < p->n_pdds; i++) {
1202 struct kfd_dev *dev = p->pdds[i]->dev;
1203 struct qcm_process_device *qpd = &p->pdds[i]->qpd;
1204
1205 if (!dev->cwsr_enabled || qpd->cwsr_kaddr || qpd->cwsr_base)
1206 continue;
1207
1208 offset = KFD_MMAP_TYPE_RESERVED_MEM | KFD_MMAP_GPU_ID(dev->id);
1209 qpd->tba_addr = (int64_t)vm_mmap(filep, 0,
1210 KFD_CWSR_TBA_TMA_SIZE, PROT_READ | PROT_EXEC,
1211 MAP_SHARED, offset);
1212
1213 if (IS_ERR_VALUE(qpd->tba_addr)) {
1214 int err = qpd->tba_addr;
1215
1216 pr_err("Failure to set tba address. error %d.\n", err);
1217 qpd->tba_addr = 0;
1218 qpd->cwsr_kaddr = NULL;
1219 return err;
1220 }
1221
1222 memcpy(qpd->cwsr_kaddr, dev->cwsr_isa, dev->cwsr_isa_size);
1223
1224 qpd->tma_addr = qpd->tba_addr + KFD_CWSR_TMA_OFFSET;
1225 pr_debug("set tba :0x%llx, tma:0x%llx, cwsr_kaddr:%p for pqm.\n",
1226 qpd->tba_addr, qpd->tma_addr, qpd->cwsr_kaddr);
1227 }
1228
1229 return 0;
1230}
1231
1232static int kfd_process_device_init_cwsr_dgpu(struct kfd_process_device *pdd)
1233{
1234 struct kfd_dev *dev = pdd->dev;
1235 struct qcm_process_device *qpd = &pdd->qpd;
1236 uint32_t flags = KFD_IOC_ALLOC_MEM_FLAGS_GTT
1237 | KFD_IOC_ALLOC_MEM_FLAGS_NO_SUBSTITUTE
1238 | KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE;
1239 struct kgd_mem *mem;
1240 void *kaddr;
1241 int ret;
1242
1243 if (!dev->cwsr_enabled || qpd->cwsr_kaddr || !qpd->cwsr_base)
1244 return 0;
1245
1246
1247 ret = kfd_process_alloc_gpuvm(pdd, qpd->cwsr_base,
1248 KFD_CWSR_TBA_TMA_SIZE, flags, &mem, &kaddr);
1249 if (ret)
1250 return ret;
1251
1252 qpd->cwsr_mem = mem;
1253 qpd->cwsr_kaddr = kaddr;
1254 qpd->tba_addr = qpd->cwsr_base;
1255
1256 memcpy(qpd->cwsr_kaddr, dev->cwsr_isa, dev->cwsr_isa_size);
1257
1258 qpd->tma_addr = qpd->tba_addr + KFD_CWSR_TMA_OFFSET;
1259 pr_debug("set tba :0x%llx, tma:0x%llx, cwsr_kaddr:%p for pqm.\n",
1260 qpd->tba_addr, qpd->tma_addr, qpd->cwsr_kaddr);
1261
1262 return 0;
1263}
1264
1265static void kfd_process_device_destroy_cwsr_dgpu(struct kfd_process_device *pdd)
1266{
1267 struct kfd_dev *dev = pdd->dev;
1268 struct qcm_process_device *qpd = &pdd->qpd;
1269
1270 if (!dev->cwsr_enabled || !qpd->cwsr_kaddr || !qpd->cwsr_base)
1271 return;
1272
1273 kfd_process_free_gpuvm(qpd->cwsr_mem, pdd, qpd->cwsr_kaddr);
1274}
1275
1276void kfd_process_set_trap_handler(struct qcm_process_device *qpd,
1277 uint64_t tba_addr,
1278 uint64_t tma_addr)
1279{
1280 if (qpd->cwsr_kaddr) {
1281
1282
1283
1284 uint64_t *tma =
1285 (uint64_t *)(qpd->cwsr_kaddr + KFD_CWSR_TMA_OFFSET);
1286 tma[0] = tba_addr;
1287 tma[1] = tma_addr;
1288 } else {
1289
1290 qpd->tba_addr = tba_addr;
1291 qpd->tma_addr = tma_addr;
1292 }
1293}
1294
1295bool kfd_process_xnack_mode(struct kfd_process *p, bool supported)
1296{
1297 int i;
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313 for (i = 0; i < p->n_pdds; i++) {
1314 struct kfd_dev *dev = p->pdds[i]->dev;
1315
1316
1317
1318
1319
1320 if (!KFD_IS_SOC15(dev))
1321 continue;
1322
1323
1324
1325
1326 if (supported && KFD_GC_VERSION(dev) == IP_VERSION(9, 4, 2))
1327 continue;
1328
1329
1330
1331
1332
1333
1334 if (KFD_GC_VERSION(dev) >= IP_VERSION(10, 1, 1))
1335 return false;
1336
1337 if (dev->noretry)
1338 return false;
1339 }
1340
1341 return true;
1342}
1343
1344
1345
1346
1347
1348static struct kfd_process *create_process(const struct task_struct *thread)
1349{
1350 struct kfd_process *process;
1351 struct mmu_notifier *mn;
1352 int err = -ENOMEM;
1353
1354 process = kzalloc(sizeof(*process), GFP_KERNEL);
1355 if (!process)
1356 goto err_alloc_process;
1357
1358 kref_init(&process->ref);
1359 mutex_init(&process->mutex);
1360 process->mm = thread->mm;
1361 process->lead_thread = thread->group_leader;
1362 process->n_pdds = 0;
1363 INIT_DELAYED_WORK(&process->eviction_work, evict_process_worker);
1364 INIT_DELAYED_WORK(&process->restore_work, restore_process_worker);
1365 process->last_restore_timestamp = get_jiffies_64();
1366 kfd_event_init_process(process);
1367 process->is_32bit_user_mode = in_compat_syscall();
1368
1369 process->pasid = kfd_pasid_alloc();
1370 if (process->pasid == 0)
1371 goto err_alloc_pasid;
1372
1373 err = pqm_init(&process->pqm, process);
1374 if (err != 0)
1375 goto err_process_pqm_init;
1376
1377
1378 err = kfd_init_apertures(process);
1379 if (err != 0)
1380 goto err_init_apertures;
1381
1382
1383 process->xnack_enabled = kfd_process_xnack_mode(process, false);
1384
1385 err = svm_range_list_init(process);
1386 if (err)
1387 goto err_init_svm_range_list;
1388
1389
1390 hash_add_rcu(kfd_processes_table, &process->kfd_processes,
1391 (uintptr_t)process->mm);
1392
1393
1394
1395
1396
1397
1398 mn = mmu_notifier_get(&kfd_process_mmu_notifier_ops, process->mm);
1399 if (IS_ERR(mn)) {
1400 err = PTR_ERR(mn);
1401 goto err_register_notifier;
1402 }
1403 BUG_ON(mn != &process->mmu_notifier);
1404
1405 get_task_struct(process->lead_thread);
1406
1407 return process;
1408
1409err_register_notifier:
1410 hash_del_rcu(&process->kfd_processes);
1411 svm_range_list_fini(process);
1412err_init_svm_range_list:
1413 kfd_process_free_outstanding_kfd_bos(process);
1414 kfd_process_destroy_pdds(process);
1415err_init_apertures:
1416 pqm_uninit(&process->pqm);
1417err_process_pqm_init:
1418 kfd_pasid_free(process->pasid);
1419err_alloc_pasid:
1420 mutex_destroy(&process->mutex);
1421 kfree(process);
1422err_alloc_process:
1423 return ERR_PTR(err);
1424}
1425
1426static int init_doorbell_bitmap(struct qcm_process_device *qpd,
1427 struct kfd_dev *dev)
1428{
1429 unsigned int i;
1430 int range_start = dev->shared_resources.non_cp_doorbells_start;
1431 int range_end = dev->shared_resources.non_cp_doorbells_end;
1432
1433 if (!KFD_IS_SOC15(dev))
1434 return 0;
1435
1436 qpd->doorbell_bitmap = bitmap_zalloc(KFD_MAX_NUM_OF_QUEUES_PER_PROCESS,
1437 GFP_KERNEL);
1438 if (!qpd->doorbell_bitmap)
1439 return -ENOMEM;
1440
1441
1442 pr_debug("reserved doorbell 0x%03x - 0x%03x\n", range_start, range_end);
1443 pr_debug("reserved doorbell 0x%03x - 0x%03x\n",
1444 range_start + KFD_QUEUE_DOORBELL_MIRROR_OFFSET,
1445 range_end + KFD_QUEUE_DOORBELL_MIRROR_OFFSET);
1446
1447 for (i = 0; i < KFD_MAX_NUM_OF_QUEUES_PER_PROCESS / 2; i++) {
1448 if (i >= range_start && i <= range_end) {
1449 __set_bit(i, qpd->doorbell_bitmap);
1450 __set_bit(i + KFD_QUEUE_DOORBELL_MIRROR_OFFSET,
1451 qpd->doorbell_bitmap);
1452 }
1453 }
1454
1455 return 0;
1456}
1457
1458struct kfd_process_device *kfd_get_process_device_data(struct kfd_dev *dev,
1459 struct kfd_process *p)
1460{
1461 int i;
1462
1463 for (i = 0; i < p->n_pdds; i++)
1464 if (p->pdds[i]->dev == dev)
1465 return p->pdds[i];
1466
1467 return NULL;
1468}
1469
1470struct kfd_process_device *kfd_create_process_device_data(struct kfd_dev *dev,
1471 struct kfd_process *p)
1472{
1473 struct kfd_process_device *pdd = NULL;
1474
1475 if (WARN_ON_ONCE(p->n_pdds >= MAX_GPU_INSTANCE))
1476 return NULL;
1477 pdd = kzalloc(sizeof(*pdd), GFP_KERNEL);
1478 if (!pdd)
1479 return NULL;
1480
1481 if (kfd_alloc_process_doorbells(dev, &pdd->doorbell_index) < 0) {
1482 pr_err("Failed to alloc doorbell for pdd\n");
1483 goto err_free_pdd;
1484 }
1485
1486 if (init_doorbell_bitmap(&pdd->qpd, dev)) {
1487 pr_err("Failed to init doorbell for process\n");
1488 goto err_free_pdd;
1489 }
1490
1491 pdd->dev = dev;
1492 INIT_LIST_HEAD(&pdd->qpd.queues_list);
1493 INIT_LIST_HEAD(&pdd->qpd.priv_queue_list);
1494 pdd->qpd.dqm = dev->dqm;
1495 pdd->qpd.pqm = &p->pqm;
1496 pdd->qpd.evicted = 0;
1497 pdd->qpd.mapped_gws_queue = false;
1498 pdd->process = p;
1499 pdd->bound = PDD_UNBOUND;
1500 pdd->already_dequeued = false;
1501 pdd->runtime_inuse = false;
1502 pdd->vram_usage = 0;
1503 pdd->sdma_past_activity_counter = 0;
1504 atomic64_set(&pdd->evict_duration_counter, 0);
1505 p->pdds[p->n_pdds++] = pdd;
1506
1507
1508 idr_init(&pdd->alloc_idr);
1509
1510 return pdd;
1511
1512err_free_pdd:
1513 kfree(pdd);
1514 return NULL;
1515}
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531int kfd_process_device_init_vm(struct kfd_process_device *pdd,
1532 struct file *drm_file)
1533{
1534 struct kfd_process *p;
1535 struct kfd_dev *dev;
1536 int ret;
1537
1538 if (!drm_file)
1539 return -EINVAL;
1540
1541 if (pdd->drm_priv)
1542 return -EBUSY;
1543
1544 p = pdd->process;
1545 dev = pdd->dev;
1546
1547 ret = amdgpu_amdkfd_gpuvm_acquire_process_vm(
1548 dev->adev, drm_file, p->pasid,
1549 &p->kgd_process_info, &p->ef);
1550 if (ret) {
1551 pr_err("Failed to create process VM object\n");
1552 return ret;
1553 }
1554 pdd->drm_priv = drm_file->private_data;
1555
1556 ret = kfd_process_device_reserve_ib_mem(pdd);
1557 if (ret)
1558 goto err_reserve_ib_mem;
1559 ret = kfd_process_device_init_cwsr_dgpu(pdd);
1560 if (ret)
1561 goto err_init_cwsr;
1562
1563 pdd->drm_file = drm_file;
1564
1565 return 0;
1566
1567err_init_cwsr:
1568err_reserve_ib_mem:
1569 kfd_process_device_free_bos(pdd);
1570 pdd->drm_priv = NULL;
1571
1572 return ret;
1573}
1574
1575
1576
1577
1578
1579
1580
1581
1582struct kfd_process_device *kfd_bind_process_to_device(struct kfd_dev *dev,
1583 struct kfd_process *p)
1584{
1585 struct kfd_process_device *pdd;
1586 int err;
1587
1588 pdd = kfd_get_process_device_data(dev, p);
1589 if (!pdd) {
1590 pr_err("Process device data doesn't exist\n");
1591 return ERR_PTR(-ENOMEM);
1592 }
1593
1594 if (!pdd->drm_priv)
1595 return ERR_PTR(-ENODEV);
1596
1597
1598
1599
1600
1601
1602 if (!pdd->runtime_inuse) {
1603 err = pm_runtime_get_sync(dev->ddev->dev);
1604 if (err < 0) {
1605 pm_runtime_put_autosuspend(dev->ddev->dev);
1606 return ERR_PTR(err);
1607 }
1608 }
1609
1610 err = kfd_iommu_bind_process_to_device(pdd);
1611 if (err)
1612 goto out;
1613
1614
1615
1616
1617
1618 pdd->runtime_inuse = true;
1619
1620 return pdd;
1621
1622out:
1623
1624 if (!pdd->runtime_inuse) {
1625 pm_runtime_mark_last_busy(dev->ddev->dev);
1626 pm_runtime_put_autosuspend(dev->ddev->dev);
1627 }
1628
1629 return ERR_PTR(err);
1630}
1631
1632
1633
1634
1635int kfd_process_device_create_obj_handle(struct kfd_process_device *pdd,
1636 void *mem)
1637{
1638 return idr_alloc(&pdd->alloc_idr, mem, 0, 0, GFP_KERNEL);
1639}
1640
1641
1642
1643
1644void *kfd_process_device_translate_handle(struct kfd_process_device *pdd,
1645 int handle)
1646{
1647 if (handle < 0)
1648 return NULL;
1649
1650 return idr_find(&pdd->alloc_idr, handle);
1651}
1652
1653
1654
1655
1656void kfd_process_device_remove_obj_handle(struct kfd_process_device *pdd,
1657 int handle)
1658{
1659 if (handle >= 0)
1660 idr_remove(&pdd->alloc_idr, handle);
1661}
1662
1663
1664struct kfd_process *kfd_lookup_process_by_pasid(u32 pasid)
1665{
1666 struct kfd_process *p, *ret_p = NULL;
1667 unsigned int temp;
1668
1669 int idx = srcu_read_lock(&kfd_processes_srcu);
1670
1671 hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
1672 if (p->pasid == pasid) {
1673 kref_get(&p->ref);
1674 ret_p = p;
1675 break;
1676 }
1677 }
1678
1679 srcu_read_unlock(&kfd_processes_srcu, idx);
1680
1681 return ret_p;
1682}
1683
1684
1685struct kfd_process *kfd_lookup_process_by_mm(const struct mm_struct *mm)
1686{
1687 struct kfd_process *p;
1688
1689 int idx = srcu_read_lock(&kfd_processes_srcu);
1690
1691 p = find_process_by_mm(mm);
1692 if (p)
1693 kref_get(&p->ref);
1694
1695 srcu_read_unlock(&kfd_processes_srcu, idx);
1696
1697 return p;
1698}
1699
1700
1701
1702
1703
1704
1705int kfd_process_evict_queues(struct kfd_process *p)
1706{
1707 int r = 0;
1708 int i;
1709 unsigned int n_evicted = 0;
1710
1711 for (i = 0; i < p->n_pdds; i++) {
1712 struct kfd_process_device *pdd = p->pdds[i];
1713
1714 r = pdd->dev->dqm->ops.evict_process_queues(pdd->dev->dqm,
1715 &pdd->qpd);
1716
1717
1718
1719
1720 if (r && r != -EIO) {
1721 pr_err("Failed to evict process queues\n");
1722 goto fail;
1723 }
1724 n_evicted++;
1725 }
1726
1727 return r;
1728
1729fail:
1730
1731
1732
1733 for (i = 0; i < p->n_pdds; i++) {
1734 struct kfd_process_device *pdd = p->pdds[i];
1735
1736 if (n_evicted == 0)
1737 break;
1738 if (pdd->dev->dqm->ops.restore_process_queues(pdd->dev->dqm,
1739 &pdd->qpd))
1740 pr_err("Failed to restore queues\n");
1741
1742 n_evicted--;
1743 }
1744
1745 return r;
1746}
1747
1748
1749int kfd_process_restore_queues(struct kfd_process *p)
1750{
1751 int r, ret = 0;
1752 int i;
1753
1754 for (i = 0; i < p->n_pdds; i++) {
1755 struct kfd_process_device *pdd = p->pdds[i];
1756
1757 r = pdd->dev->dqm->ops.restore_process_queues(pdd->dev->dqm,
1758 &pdd->qpd);
1759 if (r) {
1760 pr_err("Failed to restore process queues\n");
1761 if (!ret)
1762 ret = r;
1763 }
1764 }
1765
1766 return ret;
1767}
1768
1769int kfd_process_gpuidx_from_gpuid(struct kfd_process *p, uint32_t gpu_id)
1770{
1771 int i;
1772
1773 for (i = 0; i < p->n_pdds; i++)
1774 if (p->pdds[i] && gpu_id == p->pdds[i]->dev->id)
1775 return i;
1776 return -EINVAL;
1777}
1778
1779int
1780kfd_process_gpuid_from_adev(struct kfd_process *p, struct amdgpu_device *adev,
1781 uint32_t *gpuid, uint32_t *gpuidx)
1782{
1783 int i;
1784
1785 for (i = 0; i < p->n_pdds; i++)
1786 if (p->pdds[i] && p->pdds[i]->dev->adev == adev) {
1787 *gpuid = p->pdds[i]->dev->id;
1788 *gpuidx = i;
1789 return 0;
1790 }
1791 return -EINVAL;
1792}
1793
1794static void evict_process_worker(struct work_struct *work)
1795{
1796 int ret;
1797 struct kfd_process *p;
1798 struct delayed_work *dwork;
1799
1800 dwork = to_delayed_work(work);
1801
1802
1803
1804
1805 p = container_of(dwork, struct kfd_process, eviction_work);
1806 WARN_ONCE(p->last_eviction_seqno != p->ef->seqno,
1807 "Eviction fence mismatch\n");
1808
1809
1810
1811
1812
1813
1814
1815 flush_delayed_work(&p->restore_work);
1816
1817 pr_debug("Started evicting pasid 0x%x\n", p->pasid);
1818 ret = kfd_process_evict_queues(p);
1819 if (!ret) {
1820 dma_fence_signal(p->ef);
1821 dma_fence_put(p->ef);
1822 p->ef = NULL;
1823 queue_delayed_work(kfd_restore_wq, &p->restore_work,
1824 msecs_to_jiffies(PROCESS_RESTORE_TIME_MS));
1825
1826 pr_debug("Finished evicting pasid 0x%x\n", p->pasid);
1827 } else
1828 pr_err("Failed to evict queues of pasid 0x%x\n", p->pasid);
1829}
1830
1831static void restore_process_worker(struct work_struct *work)
1832{
1833 struct delayed_work *dwork;
1834 struct kfd_process *p;
1835 int ret = 0;
1836
1837 dwork = to_delayed_work(work);
1838
1839
1840
1841
1842 p = container_of(dwork, struct kfd_process, restore_work);
1843 pr_debug("Started restoring pasid 0x%x\n", p->pasid);
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855 p->last_restore_timestamp = get_jiffies_64();
1856 ret = amdgpu_amdkfd_gpuvm_restore_process_bos(p->kgd_process_info,
1857 &p->ef);
1858 if (ret) {
1859 pr_debug("Failed to restore BOs of pasid 0x%x, retry after %d ms\n",
1860 p->pasid, PROCESS_BACK_OFF_TIME_MS);
1861 ret = queue_delayed_work(kfd_restore_wq, &p->restore_work,
1862 msecs_to_jiffies(PROCESS_BACK_OFF_TIME_MS));
1863 WARN(!ret, "reschedule restore work failed\n");
1864 return;
1865 }
1866
1867 ret = kfd_process_restore_queues(p);
1868 if (!ret)
1869 pr_debug("Finished restoring pasid 0x%x\n", p->pasid);
1870 else
1871 pr_err("Failed to restore queues of pasid 0x%x\n", p->pasid);
1872}
1873
1874void kfd_suspend_all_processes(void)
1875{
1876 struct kfd_process *p;
1877 unsigned int temp;
1878 int idx = srcu_read_lock(&kfd_processes_srcu);
1879
1880 WARN(debug_evictions, "Evicting all processes");
1881 hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
1882 cancel_delayed_work_sync(&p->eviction_work);
1883 cancel_delayed_work_sync(&p->restore_work);
1884
1885 if (kfd_process_evict_queues(p))
1886 pr_err("Failed to suspend process 0x%x\n", p->pasid);
1887 dma_fence_signal(p->ef);
1888 dma_fence_put(p->ef);
1889 p->ef = NULL;
1890 }
1891 srcu_read_unlock(&kfd_processes_srcu, idx);
1892}
1893
1894int kfd_resume_all_processes(void)
1895{
1896 struct kfd_process *p;
1897 unsigned int temp;
1898 int ret = 0, idx = srcu_read_lock(&kfd_processes_srcu);
1899
1900 hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
1901 if (!queue_delayed_work(kfd_restore_wq, &p->restore_work, 0)) {
1902 pr_err("Restore process %d failed during resume\n",
1903 p->pasid);
1904 ret = -EFAULT;
1905 }
1906 }
1907 srcu_read_unlock(&kfd_processes_srcu, idx);
1908 return ret;
1909}
1910
1911int kfd_reserved_mem_mmap(struct kfd_dev *dev, struct kfd_process *process,
1912 struct vm_area_struct *vma)
1913{
1914 struct kfd_process_device *pdd;
1915 struct qcm_process_device *qpd;
1916
1917 if ((vma->vm_end - vma->vm_start) != KFD_CWSR_TBA_TMA_SIZE) {
1918 pr_err("Incorrect CWSR mapping size.\n");
1919 return -EINVAL;
1920 }
1921
1922 pdd = kfd_get_process_device_data(dev, process);
1923 if (!pdd)
1924 return -EINVAL;
1925 qpd = &pdd->qpd;
1926
1927 qpd->cwsr_kaddr = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
1928 get_order(KFD_CWSR_TBA_TMA_SIZE));
1929 if (!qpd->cwsr_kaddr) {
1930 pr_err("Error allocating per process CWSR buffer.\n");
1931 return -ENOMEM;
1932 }
1933
1934 vma->vm_flags |= VM_IO | VM_DONTCOPY | VM_DONTEXPAND
1935 | VM_NORESERVE | VM_DONTDUMP | VM_PFNMAP;
1936
1937 return remap_pfn_range(vma, vma->vm_start,
1938 PFN_DOWN(__pa(qpd->cwsr_kaddr)),
1939 KFD_CWSR_TBA_TMA_SIZE, vma->vm_page_prot);
1940}
1941
1942void kfd_flush_tlb(struct kfd_process_device *pdd, enum TLB_FLUSH_TYPE type)
1943{
1944 struct kfd_dev *dev = pdd->dev;
1945
1946 if (dev->dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) {
1947
1948
1949
1950 if (pdd->qpd.vmid)
1951 amdgpu_amdkfd_flush_gpu_tlb_vmid(dev->adev,
1952 pdd->qpd.vmid);
1953 } else {
1954 amdgpu_amdkfd_flush_gpu_tlb_pasid(dev->adev,
1955 pdd->process->pasid, type);
1956 }
1957}
1958
1959#if defined(CONFIG_DEBUG_FS)
1960
1961int kfd_debugfs_mqds_by_process(struct seq_file *m, void *data)
1962{
1963 struct kfd_process *p;
1964 unsigned int temp;
1965 int r = 0;
1966
1967 int idx = srcu_read_lock(&kfd_processes_srcu);
1968
1969 hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
1970 seq_printf(m, "Process %d PASID 0x%x:\n",
1971 p->lead_thread->tgid, p->pasid);
1972
1973 mutex_lock(&p->mutex);
1974 r = pqm_debugfs_mqds(m, &p->pqm);
1975 mutex_unlock(&p->mutex);
1976
1977 if (r)
1978 break;
1979 }
1980
1981 srcu_read_unlock(&kfd_processes_srcu, idx);
1982
1983 return r;
1984}
1985
1986#endif
1987
1988