1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23#include <linux/device.h>
24#include <linux/export.h>
25#include <linux/err.h>
26#include <linux/fs.h>
27#include <linux/file.h>
28#include <linux/sched.h>
29#include <linux/slab.h>
30#include <linux/uaccess.h>
31#include <linux/compat.h>
32#include <uapi/linux/kfd_ioctl.h>
33#include <linux/time.h>
34#include <linux/mm.h>
35#include <linux/mman.h>
36#include <linux/dma-buf.h>
37#include <asm/processor.h>
38#include "kfd_priv.h"
39#include "kfd_device_queue_manager.h"
40#include "kfd_dbgmgr.h"
41#include "amdgpu_amdkfd.h"
42#include "kfd_smi_events.h"
43
44static long kfd_ioctl(struct file *, unsigned int, unsigned long);
45static int kfd_open(struct inode *, struct file *);
46static int kfd_release(struct inode *, struct file *);
47static int kfd_mmap(struct file *, struct vm_area_struct *);
48
49static const char kfd_dev_name[] = "kfd";
50
51static const struct file_operations kfd_fops = {
52 .owner = THIS_MODULE,
53 .unlocked_ioctl = kfd_ioctl,
54 .compat_ioctl = compat_ptr_ioctl,
55 .open = kfd_open,
56 .release = kfd_release,
57 .mmap = kfd_mmap,
58};
59
60static int kfd_char_dev_major = -1;
61static struct class *kfd_class;
62struct device *kfd_device;
63
64int kfd_chardev_init(void)
65{
66 int err = 0;
67
68 kfd_char_dev_major = register_chrdev(0, kfd_dev_name, &kfd_fops);
69 err = kfd_char_dev_major;
70 if (err < 0)
71 goto err_register_chrdev;
72
73 kfd_class = class_create(THIS_MODULE, kfd_dev_name);
74 err = PTR_ERR(kfd_class);
75 if (IS_ERR(kfd_class))
76 goto err_class_create;
77
78 kfd_device = device_create(kfd_class, NULL,
79 MKDEV(kfd_char_dev_major, 0),
80 NULL, kfd_dev_name);
81 err = PTR_ERR(kfd_device);
82 if (IS_ERR(kfd_device))
83 goto err_device_create;
84
85 return 0;
86
87err_device_create:
88 class_destroy(kfd_class);
89err_class_create:
90 unregister_chrdev(kfd_char_dev_major, kfd_dev_name);
91err_register_chrdev:
92 return err;
93}
94
95void kfd_chardev_exit(void)
96{
97 device_destroy(kfd_class, MKDEV(kfd_char_dev_major, 0));
98 class_destroy(kfd_class);
99 unregister_chrdev(kfd_char_dev_major, kfd_dev_name);
100 kfd_device = NULL;
101}
102
103struct device *kfd_chardev(void)
104{
105 return kfd_device;
106}
107
108
109static int kfd_open(struct inode *inode, struct file *filep)
110{
111 struct kfd_process *process;
112 bool is_32bit_user_mode;
113
114 if (iminor(inode) != 0)
115 return -ENODEV;
116
117 is_32bit_user_mode = in_compat_syscall();
118
119 if (is_32bit_user_mode) {
120 dev_warn(kfd_device,
121 "Process %d (32-bit) failed to open /dev/kfd\n"
122 "32-bit processes are not supported by amdkfd\n",
123 current->pid);
124 return -EPERM;
125 }
126
127 process = kfd_create_process(filep);
128 if (IS_ERR(process))
129 return PTR_ERR(process);
130
131 if (kfd_is_locked()) {
132 dev_dbg(kfd_device, "kfd is locked!\n"
133 "process %d unreferenced", process->pasid);
134 kfd_unref_process(process);
135 return -EAGAIN;
136 }
137
138
139 filep->private_data = process;
140
141 dev_dbg(kfd_device, "process %d opened, compat mode (32 bit) - %d\n",
142 process->pasid, process->is_32bit_user_mode);
143
144 return 0;
145}
146
147static int kfd_release(struct inode *inode, struct file *filep)
148{
149 struct kfd_process *process = filep->private_data;
150
151 if (process)
152 kfd_unref_process(process);
153
154 return 0;
155}
156
157static int kfd_ioctl_get_version(struct file *filep, struct kfd_process *p,
158 void *data)
159{
160 struct kfd_ioctl_get_version_args *args = data;
161
162 args->major_version = KFD_IOCTL_MAJOR_VERSION;
163 args->minor_version = KFD_IOCTL_MINOR_VERSION;
164
165 return 0;
166}
167
168static int set_queue_properties_from_user(struct queue_properties *q_properties,
169 struct kfd_ioctl_create_queue_args *args)
170{
171 if (args->queue_percentage > KFD_MAX_QUEUE_PERCENTAGE) {
172 pr_err("Queue percentage must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n");
173 return -EINVAL;
174 }
175
176 if (args->queue_priority > KFD_MAX_QUEUE_PRIORITY) {
177 pr_err("Queue priority must be between 0 to KFD_MAX_QUEUE_PRIORITY\n");
178 return -EINVAL;
179 }
180
181 if ((args->ring_base_address) &&
182 (!access_ok((const void __user *) args->ring_base_address,
183 sizeof(uint64_t)))) {
184 pr_err("Can't access ring base address\n");
185 return -EFAULT;
186 }
187
188 if (!is_power_of_2(args->ring_size) && (args->ring_size != 0)) {
189 pr_err("Ring size must be a power of 2 or 0\n");
190 return -EINVAL;
191 }
192
193 if (!access_ok((const void __user *) args->read_pointer_address,
194 sizeof(uint32_t))) {
195 pr_err("Can't access read pointer\n");
196 return -EFAULT;
197 }
198
199 if (!access_ok((const void __user *) args->write_pointer_address,
200 sizeof(uint32_t))) {
201 pr_err("Can't access write pointer\n");
202 return -EFAULT;
203 }
204
205 if (args->eop_buffer_address &&
206 !access_ok((const void __user *) args->eop_buffer_address,
207 sizeof(uint32_t))) {
208 pr_debug("Can't access eop buffer");
209 return -EFAULT;
210 }
211
212 if (args->ctx_save_restore_address &&
213 !access_ok((const void __user *) args->ctx_save_restore_address,
214 sizeof(uint32_t))) {
215 pr_debug("Can't access ctx save restore buffer");
216 return -EFAULT;
217 }
218
219 q_properties->is_interop = false;
220 q_properties->is_gws = false;
221 q_properties->queue_percent = args->queue_percentage;
222 q_properties->priority = args->queue_priority;
223 q_properties->queue_address = args->ring_base_address;
224 q_properties->queue_size = args->ring_size;
225 q_properties->read_ptr = (uint32_t *) args->read_pointer_address;
226 q_properties->write_ptr = (uint32_t *) args->write_pointer_address;
227 q_properties->eop_ring_buffer_address = args->eop_buffer_address;
228 q_properties->eop_ring_buffer_size = args->eop_buffer_size;
229 q_properties->ctx_save_restore_area_address =
230 args->ctx_save_restore_address;
231 q_properties->ctx_save_restore_area_size = args->ctx_save_restore_size;
232 q_properties->ctl_stack_size = args->ctl_stack_size;
233 if (args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE ||
234 args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE_AQL)
235 q_properties->type = KFD_QUEUE_TYPE_COMPUTE;
236 else if (args->queue_type == KFD_IOC_QUEUE_TYPE_SDMA)
237 q_properties->type = KFD_QUEUE_TYPE_SDMA;
238 else if (args->queue_type == KFD_IOC_QUEUE_TYPE_SDMA_XGMI)
239 q_properties->type = KFD_QUEUE_TYPE_SDMA_XGMI;
240 else
241 return -ENOTSUPP;
242
243 if (args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE_AQL)
244 q_properties->format = KFD_QUEUE_FORMAT_AQL;
245 else
246 q_properties->format = KFD_QUEUE_FORMAT_PM4;
247
248 pr_debug("Queue Percentage: %d, %d\n",
249 q_properties->queue_percent, args->queue_percentage);
250
251 pr_debug("Queue Priority: %d, %d\n",
252 q_properties->priority, args->queue_priority);
253
254 pr_debug("Queue Address: 0x%llX, 0x%llX\n",
255 q_properties->queue_address, args->ring_base_address);
256
257 pr_debug("Queue Size: 0x%llX, %u\n",
258 q_properties->queue_size, args->ring_size);
259
260 pr_debug("Queue r/w Pointers: %px, %px\n",
261 q_properties->read_ptr,
262 q_properties->write_ptr);
263
264 pr_debug("Queue Format: %d\n", q_properties->format);
265
266 pr_debug("Queue EOP: 0x%llX\n", q_properties->eop_ring_buffer_address);
267
268 pr_debug("Queue CTX save area: 0x%llX\n",
269 q_properties->ctx_save_restore_area_address);
270
271 return 0;
272}
273
274static int kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p,
275 void *data)
276{
277 struct kfd_ioctl_create_queue_args *args = data;
278 struct kfd_dev *dev;
279 int err = 0;
280 unsigned int queue_id;
281 struct kfd_process_device *pdd;
282 struct queue_properties q_properties;
283 uint32_t doorbell_offset_in_process = 0;
284
285 memset(&q_properties, 0, sizeof(struct queue_properties));
286
287 pr_debug("Creating queue ioctl\n");
288
289 err = set_queue_properties_from_user(&q_properties, args);
290 if (err)
291 return err;
292
293 pr_debug("Looking for gpu id 0x%x\n", args->gpu_id);
294 dev = kfd_device_by_id(args->gpu_id);
295 if (!dev) {
296 pr_debug("Could not find gpu id 0x%x\n", args->gpu_id);
297 return -EINVAL;
298 }
299
300 mutex_lock(&p->mutex);
301
302 pdd = kfd_bind_process_to_device(dev, p);
303 if (IS_ERR(pdd)) {
304 err = -ESRCH;
305 goto err_bind_process;
306 }
307
308 pr_debug("Creating queue for PASID 0x%x on gpu 0x%x\n",
309 p->pasid,
310 dev->id);
311
312 err = pqm_create_queue(&p->pqm, dev, filep, &q_properties, &queue_id,
313 &doorbell_offset_in_process);
314 if (err != 0)
315 goto err_create_queue;
316
317 args->queue_id = queue_id;
318
319
320
321 args->doorbell_offset = KFD_MMAP_TYPE_DOORBELL;
322 args->doorbell_offset |= KFD_MMAP_GPU_ID(args->gpu_id);
323 if (KFD_IS_SOC15(dev->device_info->asic_family))
324
325
326
327 args->doorbell_offset |= doorbell_offset_in_process;
328
329 mutex_unlock(&p->mutex);
330
331 pr_debug("Queue id %d was created successfully\n", args->queue_id);
332
333 pr_debug("Ring buffer address == 0x%016llX\n",
334 args->ring_base_address);
335
336 pr_debug("Read ptr address == 0x%016llX\n",
337 args->read_pointer_address);
338
339 pr_debug("Write ptr address == 0x%016llX\n",
340 args->write_pointer_address);
341
342 return 0;
343
344err_create_queue:
345err_bind_process:
346 mutex_unlock(&p->mutex);
347 return err;
348}
349
350static int kfd_ioctl_destroy_queue(struct file *filp, struct kfd_process *p,
351 void *data)
352{
353 int retval;
354 struct kfd_ioctl_destroy_queue_args *args = data;
355
356 pr_debug("Destroying queue id %d for pasid 0x%x\n",
357 args->queue_id,
358 p->pasid);
359
360 mutex_lock(&p->mutex);
361
362 retval = pqm_destroy_queue(&p->pqm, args->queue_id);
363
364 mutex_unlock(&p->mutex);
365 return retval;
366}
367
368static int kfd_ioctl_update_queue(struct file *filp, struct kfd_process *p,
369 void *data)
370{
371 int retval;
372 struct kfd_ioctl_update_queue_args *args = data;
373 struct queue_properties properties;
374
375 if (args->queue_percentage > KFD_MAX_QUEUE_PERCENTAGE) {
376 pr_err("Queue percentage must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n");
377 return -EINVAL;
378 }
379
380 if (args->queue_priority > KFD_MAX_QUEUE_PRIORITY) {
381 pr_err("Queue priority must be between 0 to KFD_MAX_QUEUE_PRIORITY\n");
382 return -EINVAL;
383 }
384
385 if ((args->ring_base_address) &&
386 (!access_ok((const void __user *) args->ring_base_address,
387 sizeof(uint64_t)))) {
388 pr_err("Can't access ring base address\n");
389 return -EFAULT;
390 }
391
392 if (!is_power_of_2(args->ring_size) && (args->ring_size != 0)) {
393 pr_err("Ring size must be a power of 2 or 0\n");
394 return -EINVAL;
395 }
396
397 properties.queue_address = args->ring_base_address;
398 properties.queue_size = args->ring_size;
399 properties.queue_percent = args->queue_percentage;
400 properties.priority = args->queue_priority;
401
402 pr_debug("Updating queue id %d for pasid 0x%x\n",
403 args->queue_id, p->pasid);
404
405 mutex_lock(&p->mutex);
406
407 retval = pqm_update_queue(&p->pqm, args->queue_id, &properties);
408
409 mutex_unlock(&p->mutex);
410
411 return retval;
412}
413
414static int kfd_ioctl_set_cu_mask(struct file *filp, struct kfd_process *p,
415 void *data)
416{
417 int retval;
418 const int max_num_cus = 1024;
419 struct kfd_ioctl_set_cu_mask_args *args = data;
420 struct queue_properties properties;
421 uint32_t __user *cu_mask_ptr = (uint32_t __user *)args->cu_mask_ptr;
422 size_t cu_mask_size = sizeof(uint32_t) * (args->num_cu_mask / 32);
423
424 if ((args->num_cu_mask % 32) != 0) {
425 pr_debug("num_cu_mask 0x%x must be a multiple of 32",
426 args->num_cu_mask);
427 return -EINVAL;
428 }
429
430 properties.cu_mask_count = args->num_cu_mask;
431 if (properties.cu_mask_count == 0) {
432 pr_debug("CU mask cannot be 0");
433 return -EINVAL;
434 }
435
436
437
438
439
440 if (properties.cu_mask_count > max_num_cus) {
441 pr_debug("CU mask cannot be greater than 1024 bits");
442 properties.cu_mask_count = max_num_cus;
443 cu_mask_size = sizeof(uint32_t) * (max_num_cus/32);
444 }
445
446 properties.cu_mask = kzalloc(cu_mask_size, GFP_KERNEL);
447 if (!properties.cu_mask)
448 return -ENOMEM;
449
450 retval = copy_from_user(properties.cu_mask, cu_mask_ptr, cu_mask_size);
451 if (retval) {
452 pr_debug("Could not copy CU mask from userspace");
453 kfree(properties.cu_mask);
454 return -EFAULT;
455 }
456
457 mutex_lock(&p->mutex);
458
459 retval = pqm_set_cu_mask(&p->pqm, args->queue_id, &properties);
460
461 mutex_unlock(&p->mutex);
462
463 if (retval)
464 kfree(properties.cu_mask);
465
466 return retval;
467}
468
469static int kfd_ioctl_get_queue_wave_state(struct file *filep,
470 struct kfd_process *p, void *data)
471{
472 struct kfd_ioctl_get_queue_wave_state_args *args = data;
473 int r;
474
475 mutex_lock(&p->mutex);
476
477 r = pqm_get_wave_state(&p->pqm, args->queue_id,
478 (void __user *)args->ctl_stack_address,
479 &args->ctl_stack_used_size,
480 &args->save_area_used_size);
481
482 mutex_unlock(&p->mutex);
483
484 return r;
485}
486
487static int kfd_ioctl_set_memory_policy(struct file *filep,
488 struct kfd_process *p, void *data)
489{
490 struct kfd_ioctl_set_memory_policy_args *args = data;
491 struct kfd_dev *dev;
492 int err = 0;
493 struct kfd_process_device *pdd;
494 enum cache_policy default_policy, alternate_policy;
495
496 if (args->default_policy != KFD_IOC_CACHE_POLICY_COHERENT
497 && args->default_policy != KFD_IOC_CACHE_POLICY_NONCOHERENT) {
498 return -EINVAL;
499 }
500
501 if (args->alternate_policy != KFD_IOC_CACHE_POLICY_COHERENT
502 && args->alternate_policy != KFD_IOC_CACHE_POLICY_NONCOHERENT) {
503 return -EINVAL;
504 }
505
506 dev = kfd_device_by_id(args->gpu_id);
507 if (!dev)
508 return -EINVAL;
509
510 mutex_lock(&p->mutex);
511
512 pdd = kfd_bind_process_to_device(dev, p);
513 if (IS_ERR(pdd)) {
514 err = -ESRCH;
515 goto out;
516 }
517
518 default_policy = (args->default_policy == KFD_IOC_CACHE_POLICY_COHERENT)
519 ? cache_policy_coherent : cache_policy_noncoherent;
520
521 alternate_policy =
522 (args->alternate_policy == KFD_IOC_CACHE_POLICY_COHERENT)
523 ? cache_policy_coherent : cache_policy_noncoherent;
524
525 if (!dev->dqm->ops.set_cache_memory_policy(dev->dqm,
526 &pdd->qpd,
527 default_policy,
528 alternate_policy,
529 (void __user *)args->alternate_aperture_base,
530 args->alternate_aperture_size))
531 err = -EINVAL;
532
533out:
534 mutex_unlock(&p->mutex);
535
536 return err;
537}
538
539static int kfd_ioctl_set_trap_handler(struct file *filep,
540 struct kfd_process *p, void *data)
541{
542 struct kfd_ioctl_set_trap_handler_args *args = data;
543 struct kfd_dev *dev;
544 int err = 0;
545 struct kfd_process_device *pdd;
546
547 dev = kfd_device_by_id(args->gpu_id);
548 if (!dev)
549 return -EINVAL;
550
551 mutex_lock(&p->mutex);
552
553 pdd = kfd_bind_process_to_device(dev, p);
554 if (IS_ERR(pdd)) {
555 err = -ESRCH;
556 goto out;
557 }
558
559 if (dev->dqm->ops.set_trap_handler(dev->dqm,
560 &pdd->qpd,
561 args->tba_addr,
562 args->tma_addr))
563 err = -EINVAL;
564
565out:
566 mutex_unlock(&p->mutex);
567
568 return err;
569}
570
571static int kfd_ioctl_dbg_register(struct file *filep,
572 struct kfd_process *p, void *data)
573{
574 struct kfd_ioctl_dbg_register_args *args = data;
575 struct kfd_dev *dev;
576 struct kfd_dbgmgr *dbgmgr_ptr;
577 struct kfd_process_device *pdd;
578 bool create_ok;
579 long status = 0;
580
581 dev = kfd_device_by_id(args->gpu_id);
582 if (!dev)
583 return -EINVAL;
584
585 if (dev->device_info->asic_family == CHIP_CARRIZO) {
586 pr_debug("kfd_ioctl_dbg_register not supported on CZ\n");
587 return -EINVAL;
588 }
589
590 mutex_lock(&p->mutex);
591 mutex_lock(kfd_get_dbgmgr_mutex());
592
593
594
595
596
597 pdd = kfd_bind_process_to_device(dev, p);
598 if (IS_ERR(pdd)) {
599 status = PTR_ERR(pdd);
600 goto out;
601 }
602
603 if (!dev->dbgmgr) {
604
605 create_ok = kfd_dbgmgr_create(&dbgmgr_ptr, dev);
606 if (create_ok) {
607 status = kfd_dbgmgr_register(dbgmgr_ptr, p);
608 if (status != 0)
609 kfd_dbgmgr_destroy(dbgmgr_ptr);
610 else
611 dev->dbgmgr = dbgmgr_ptr;
612 }
613 } else {
614 pr_debug("debugger already registered\n");
615 status = -EINVAL;
616 }
617
618out:
619 mutex_unlock(kfd_get_dbgmgr_mutex());
620 mutex_unlock(&p->mutex);
621
622 return status;
623}
624
625static int kfd_ioctl_dbg_unregister(struct file *filep,
626 struct kfd_process *p, void *data)
627{
628 struct kfd_ioctl_dbg_unregister_args *args = data;
629 struct kfd_dev *dev;
630 long status;
631
632 dev = kfd_device_by_id(args->gpu_id);
633 if (!dev || !dev->dbgmgr)
634 return -EINVAL;
635
636 if (dev->device_info->asic_family == CHIP_CARRIZO) {
637 pr_debug("kfd_ioctl_dbg_unregister not supported on CZ\n");
638 return -EINVAL;
639 }
640
641 mutex_lock(kfd_get_dbgmgr_mutex());
642
643 status = kfd_dbgmgr_unregister(dev->dbgmgr, p);
644 if (!status) {
645 kfd_dbgmgr_destroy(dev->dbgmgr);
646 dev->dbgmgr = NULL;
647 }
648
649 mutex_unlock(kfd_get_dbgmgr_mutex());
650
651 return status;
652}
653
654
655
656
657
658
659
660
661
662
663static int kfd_ioctl_dbg_address_watch(struct file *filep,
664 struct kfd_process *p, void *data)
665{
666 struct kfd_ioctl_dbg_address_watch_args *args = data;
667 struct kfd_dev *dev;
668 struct dbg_address_watch_info aw_info;
669 unsigned char *args_buff;
670 long status;
671 void __user *cmd_from_user;
672 uint64_t watch_mask_value = 0;
673 unsigned int args_idx = 0;
674
675 memset((void *) &aw_info, 0, sizeof(struct dbg_address_watch_info));
676
677 dev = kfd_device_by_id(args->gpu_id);
678 if (!dev)
679 return -EINVAL;
680
681 if (dev->device_info->asic_family == CHIP_CARRIZO) {
682 pr_debug("kfd_ioctl_dbg_wave_control not supported on CZ\n");
683 return -EINVAL;
684 }
685
686 cmd_from_user = (void __user *) args->content_ptr;
687
688
689
690 if ((args->buf_size_in_bytes > MAX_ALLOWED_AW_BUFF_SIZE) ||
691 (args->buf_size_in_bytes <= sizeof(*args) + sizeof(int) * 2) ||
692 (cmd_from_user == NULL))
693 return -EINVAL;
694
695
696 args_buff = memdup_user(cmd_from_user,
697 args->buf_size_in_bytes - sizeof(*args));
698 if (IS_ERR(args_buff))
699 return PTR_ERR(args_buff);
700
701 aw_info.process = p;
702
703 aw_info.num_watch_points = *((uint32_t *)(&args_buff[args_idx]));
704 args_idx += sizeof(aw_info.num_watch_points);
705
706 aw_info.watch_mode = (enum HSA_DBG_WATCH_MODE *) &args_buff[args_idx];
707 args_idx += sizeof(enum HSA_DBG_WATCH_MODE) * aw_info.num_watch_points;
708
709
710
711
712
713 aw_info.watch_address = (uint64_t *) &args_buff[args_idx];
714
715
716 args_idx += sizeof(aw_info.watch_address) * aw_info.num_watch_points;
717
718 if (args_idx >= args->buf_size_in_bytes - sizeof(*args)) {
719 status = -EINVAL;
720 goto out;
721 }
722
723 watch_mask_value = (uint64_t) args_buff[args_idx];
724
725 if (watch_mask_value > 0) {
726
727
728
729
730
731 aw_info.watch_mask = (uint64_t *) &args_buff[args_idx];
732
733
734 args_idx += sizeof(aw_info.watch_mask) *
735 aw_info.num_watch_points;
736 } else {
737
738 aw_info.watch_mask = NULL;
739 args_idx += sizeof(aw_info.watch_mask);
740 }
741
742 if (args_idx >= args->buf_size_in_bytes - sizeof(args)) {
743 status = -EINVAL;
744 goto out;
745 }
746
747
748 aw_info.watch_event = NULL;
749
750 mutex_lock(kfd_get_dbgmgr_mutex());
751
752 status = kfd_dbgmgr_address_watch(dev->dbgmgr, &aw_info);
753
754 mutex_unlock(kfd_get_dbgmgr_mutex());
755
756out:
757 kfree(args_buff);
758
759 return status;
760}
761
762
763static int kfd_ioctl_dbg_wave_control(struct file *filep,
764 struct kfd_process *p, void *data)
765{
766 struct kfd_ioctl_dbg_wave_control_args *args = data;
767 struct kfd_dev *dev;
768 struct dbg_wave_control_info wac_info;
769 unsigned char *args_buff;
770 uint32_t computed_buff_size;
771 long status;
772 void __user *cmd_from_user;
773 unsigned int args_idx = 0;
774
775 memset((void *) &wac_info, 0, sizeof(struct dbg_wave_control_info));
776
777
778 computed_buff_size = sizeof(*args) +
779 sizeof(wac_info.mode) +
780 sizeof(wac_info.operand) +
781 sizeof(wac_info.dbgWave_msg.DbgWaveMsg) +
782 sizeof(wac_info.dbgWave_msg.MemoryVA) +
783 sizeof(wac_info.trapId);
784
785 dev = kfd_device_by_id(args->gpu_id);
786 if (!dev)
787 return -EINVAL;
788
789 if (dev->device_info->asic_family == CHIP_CARRIZO) {
790 pr_debug("kfd_ioctl_dbg_wave_control not supported on CZ\n");
791 return -EINVAL;
792 }
793
794
795 if (args->buf_size_in_bytes != computed_buff_size) {
796 pr_debug("size mismatch, computed : actual %u : %u\n",
797 args->buf_size_in_bytes, computed_buff_size);
798 return -EINVAL;
799 }
800
801 cmd_from_user = (void __user *) args->content_ptr;
802
803 if (cmd_from_user == NULL)
804 return -EINVAL;
805
806
807
808 args_buff = memdup_user(cmd_from_user,
809 args->buf_size_in_bytes - sizeof(*args));
810 if (IS_ERR(args_buff))
811 return PTR_ERR(args_buff);
812
813
814 wac_info.process = p;
815
816 wac_info.operand = *((enum HSA_DBG_WAVEOP *)(&args_buff[args_idx]));
817 args_idx += sizeof(wac_info.operand);
818
819 wac_info.mode = *((enum HSA_DBG_WAVEMODE *)(&args_buff[args_idx]));
820 args_idx += sizeof(wac_info.mode);
821
822 wac_info.trapId = *((uint32_t *)(&args_buff[args_idx]));
823 args_idx += sizeof(wac_info.trapId);
824
825 wac_info.dbgWave_msg.DbgWaveMsg.WaveMsgInfoGen2.Value =
826 *((uint32_t *)(&args_buff[args_idx]));
827 wac_info.dbgWave_msg.MemoryVA = NULL;
828
829 mutex_lock(kfd_get_dbgmgr_mutex());
830
831 pr_debug("Calling dbg manager process %p, operand %u, mode %u, trapId %u, message %u\n",
832 wac_info.process, wac_info.operand,
833 wac_info.mode, wac_info.trapId,
834 wac_info.dbgWave_msg.DbgWaveMsg.WaveMsgInfoGen2.Value);
835
836 status = kfd_dbgmgr_wave_control(dev->dbgmgr, &wac_info);
837
838 pr_debug("Returned status of dbg manager is %ld\n", status);
839
840 mutex_unlock(kfd_get_dbgmgr_mutex());
841
842 kfree(args_buff);
843
844 return status;
845}
846
847static int kfd_ioctl_get_clock_counters(struct file *filep,
848 struct kfd_process *p, void *data)
849{
850 struct kfd_ioctl_get_clock_counters_args *args = data;
851 struct kfd_dev *dev;
852
853 dev = kfd_device_by_id(args->gpu_id);
854 if (dev)
855
856 args->gpu_clock_counter = amdgpu_amdkfd_get_gpu_clock_counter(dev->kgd);
857 else
858
859 args->gpu_clock_counter = 0;
860
861
862 args->cpu_clock_counter = ktime_get_raw_ns();
863 args->system_clock_counter = ktime_get_boottime_ns();
864
865
866 args->system_clock_freq = 1000000000;
867
868 return 0;
869}
870
871
872static int kfd_ioctl_get_process_apertures(struct file *filp,
873 struct kfd_process *p, void *data)
874{
875 struct kfd_ioctl_get_process_apertures_args *args = data;
876 struct kfd_process_device_apertures *pAperture;
877 struct kfd_process_device *pdd;
878
879 dev_dbg(kfd_device, "get apertures for PASID 0x%x", p->pasid);
880
881 args->num_of_nodes = 0;
882
883 mutex_lock(&p->mutex);
884
885
886 if (kfd_has_process_device_data(p)) {
887
888 pdd = kfd_get_first_process_device_data(p);
889 do {
890 pAperture =
891 &args->process_apertures[args->num_of_nodes];
892 pAperture->gpu_id = pdd->dev->id;
893 pAperture->lds_base = pdd->lds_base;
894 pAperture->lds_limit = pdd->lds_limit;
895 pAperture->gpuvm_base = pdd->gpuvm_base;
896 pAperture->gpuvm_limit = pdd->gpuvm_limit;
897 pAperture->scratch_base = pdd->scratch_base;
898 pAperture->scratch_limit = pdd->scratch_limit;
899
900 dev_dbg(kfd_device,
901 "node id %u\n", args->num_of_nodes);
902 dev_dbg(kfd_device,
903 "gpu id %u\n", pdd->dev->id);
904 dev_dbg(kfd_device,
905 "lds_base %llX\n", pdd->lds_base);
906 dev_dbg(kfd_device,
907 "lds_limit %llX\n", pdd->lds_limit);
908 dev_dbg(kfd_device,
909 "gpuvm_base %llX\n", pdd->gpuvm_base);
910 dev_dbg(kfd_device,
911 "gpuvm_limit %llX\n", pdd->gpuvm_limit);
912 dev_dbg(kfd_device,
913 "scratch_base %llX\n", pdd->scratch_base);
914 dev_dbg(kfd_device,
915 "scratch_limit %llX\n", pdd->scratch_limit);
916
917 args->num_of_nodes++;
918
919 pdd = kfd_get_next_process_device_data(p, pdd);
920 } while (pdd && (args->num_of_nodes < NUM_OF_SUPPORTED_GPUS));
921 }
922
923 mutex_unlock(&p->mutex);
924
925 return 0;
926}
927
928static int kfd_ioctl_get_process_apertures_new(struct file *filp,
929 struct kfd_process *p, void *data)
930{
931 struct kfd_ioctl_get_process_apertures_new_args *args = data;
932 struct kfd_process_device_apertures *pa;
933 struct kfd_process_device *pdd;
934 uint32_t nodes = 0;
935 int ret;
936
937 dev_dbg(kfd_device, "get apertures for PASID 0x%x", p->pasid);
938
939 if (args->num_of_nodes == 0) {
940
941
942
943 mutex_lock(&p->mutex);
944
945 if (!kfd_has_process_device_data(p))
946 goto out_unlock;
947
948
949 pdd = kfd_get_first_process_device_data(p);
950 do {
951 args->num_of_nodes++;
952 pdd = kfd_get_next_process_device_data(p, pdd);
953 } while (pdd);
954
955 goto out_unlock;
956 }
957
958
959
960
961
962 pa = kzalloc((sizeof(struct kfd_process_device_apertures) *
963 args->num_of_nodes), GFP_KERNEL);
964 if (!pa)
965 return -ENOMEM;
966
967 mutex_lock(&p->mutex);
968
969 if (!kfd_has_process_device_data(p)) {
970 args->num_of_nodes = 0;
971 kfree(pa);
972 goto out_unlock;
973 }
974
975
976 pdd = kfd_get_first_process_device_data(p);
977 do {
978 pa[nodes].gpu_id = pdd->dev->id;
979 pa[nodes].lds_base = pdd->lds_base;
980 pa[nodes].lds_limit = pdd->lds_limit;
981 pa[nodes].gpuvm_base = pdd->gpuvm_base;
982 pa[nodes].gpuvm_limit = pdd->gpuvm_limit;
983 pa[nodes].scratch_base = pdd->scratch_base;
984 pa[nodes].scratch_limit = pdd->scratch_limit;
985
986 dev_dbg(kfd_device,
987 "gpu id %u\n", pdd->dev->id);
988 dev_dbg(kfd_device,
989 "lds_base %llX\n", pdd->lds_base);
990 dev_dbg(kfd_device,
991 "lds_limit %llX\n", pdd->lds_limit);
992 dev_dbg(kfd_device,
993 "gpuvm_base %llX\n", pdd->gpuvm_base);
994 dev_dbg(kfd_device,
995 "gpuvm_limit %llX\n", pdd->gpuvm_limit);
996 dev_dbg(kfd_device,
997 "scratch_base %llX\n", pdd->scratch_base);
998 dev_dbg(kfd_device,
999 "scratch_limit %llX\n", pdd->scratch_limit);
1000 nodes++;
1001
1002 pdd = kfd_get_next_process_device_data(p, pdd);
1003 } while (pdd && (nodes < args->num_of_nodes));
1004 mutex_unlock(&p->mutex);
1005
1006 args->num_of_nodes = nodes;
1007 ret = copy_to_user(
1008 (void __user *)args->kfd_process_device_apertures_ptr,
1009 pa,
1010 (nodes * sizeof(struct kfd_process_device_apertures)));
1011 kfree(pa);
1012 return ret ? -EFAULT : 0;
1013
1014out_unlock:
1015 mutex_unlock(&p->mutex);
1016 return 0;
1017}
1018
1019static int kfd_ioctl_create_event(struct file *filp, struct kfd_process *p,
1020 void *data)
1021{
1022 struct kfd_ioctl_create_event_args *args = data;
1023 int err;
1024
1025
1026
1027
1028
1029 if (args->event_page_offset) {
1030 struct kfd_dev *kfd;
1031 struct kfd_process_device *pdd;
1032 void *mem, *kern_addr;
1033 uint64_t size;
1034
1035 if (p->signal_page) {
1036 pr_err("Event page is already set\n");
1037 return -EINVAL;
1038 }
1039
1040 kfd = kfd_device_by_id(GET_GPU_ID(args->event_page_offset));
1041 if (!kfd) {
1042 pr_err("Getting device by id failed in %s\n", __func__);
1043 return -EINVAL;
1044 }
1045
1046 mutex_lock(&p->mutex);
1047 pdd = kfd_bind_process_to_device(kfd, p);
1048 if (IS_ERR(pdd)) {
1049 err = PTR_ERR(pdd);
1050 goto out_unlock;
1051 }
1052
1053 mem = kfd_process_device_translate_handle(pdd,
1054 GET_IDR_HANDLE(args->event_page_offset));
1055 if (!mem) {
1056 pr_err("Can't find BO, offset is 0x%llx\n",
1057 args->event_page_offset);
1058 err = -EINVAL;
1059 goto out_unlock;
1060 }
1061 mutex_unlock(&p->mutex);
1062
1063 err = amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(kfd->kgd,
1064 mem, &kern_addr, &size);
1065 if (err) {
1066 pr_err("Failed to map event page to kernel\n");
1067 return err;
1068 }
1069
1070 err = kfd_event_page_set(p, kern_addr, size);
1071 if (err) {
1072 pr_err("Failed to set event page\n");
1073 return err;
1074 }
1075 }
1076
1077 err = kfd_event_create(filp, p, args->event_type,
1078 args->auto_reset != 0, args->node_id,
1079 &args->event_id, &args->event_trigger_data,
1080 &args->event_page_offset,
1081 &args->event_slot_index);
1082
1083 return err;
1084
1085out_unlock:
1086 mutex_unlock(&p->mutex);
1087 return err;
1088}
1089
1090static int kfd_ioctl_destroy_event(struct file *filp, struct kfd_process *p,
1091 void *data)
1092{
1093 struct kfd_ioctl_destroy_event_args *args = data;
1094
1095 return kfd_event_destroy(p, args->event_id);
1096}
1097
1098static int kfd_ioctl_set_event(struct file *filp, struct kfd_process *p,
1099 void *data)
1100{
1101 struct kfd_ioctl_set_event_args *args = data;
1102
1103 return kfd_set_event(p, args->event_id);
1104}
1105
1106static int kfd_ioctl_reset_event(struct file *filp, struct kfd_process *p,
1107 void *data)
1108{
1109 struct kfd_ioctl_reset_event_args *args = data;
1110
1111 return kfd_reset_event(p, args->event_id);
1112}
1113
1114static int kfd_ioctl_wait_events(struct file *filp, struct kfd_process *p,
1115 void *data)
1116{
1117 struct kfd_ioctl_wait_events_args *args = data;
1118 int err;
1119
1120 err = kfd_wait_on_events(p, args->num_events,
1121 (void __user *)args->events_ptr,
1122 (args->wait_for_all != 0),
1123 args->timeout, &args->wait_result);
1124
1125 return err;
1126}
1127static int kfd_ioctl_set_scratch_backing_va(struct file *filep,
1128 struct kfd_process *p, void *data)
1129{
1130 struct kfd_ioctl_set_scratch_backing_va_args *args = data;
1131 struct kfd_process_device *pdd;
1132 struct kfd_dev *dev;
1133 long err;
1134
1135 dev = kfd_device_by_id(args->gpu_id);
1136 if (!dev)
1137 return -EINVAL;
1138
1139 mutex_lock(&p->mutex);
1140
1141 pdd = kfd_bind_process_to_device(dev, p);
1142 if (IS_ERR(pdd)) {
1143 err = PTR_ERR(pdd);
1144 goto bind_process_to_device_fail;
1145 }
1146
1147 pdd->qpd.sh_hidden_private_base = args->va_addr;
1148
1149 mutex_unlock(&p->mutex);
1150
1151 if (dev->dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS &&
1152 pdd->qpd.vmid != 0 && dev->kfd2kgd->set_scratch_backing_va)
1153 dev->kfd2kgd->set_scratch_backing_va(
1154 dev->kgd, args->va_addr, pdd->qpd.vmid);
1155
1156 return 0;
1157
1158bind_process_to_device_fail:
1159 mutex_unlock(&p->mutex);
1160 return err;
1161}
1162
1163static int kfd_ioctl_get_tile_config(struct file *filep,
1164 struct kfd_process *p, void *data)
1165{
1166 struct kfd_ioctl_get_tile_config_args *args = data;
1167 struct kfd_dev *dev;
1168 struct tile_config config;
1169 int err = 0;
1170
1171 dev = kfd_device_by_id(args->gpu_id);
1172 if (!dev)
1173 return -EINVAL;
1174
1175 amdgpu_amdkfd_get_tile_config(dev->kgd, &config);
1176
1177 args->gb_addr_config = config.gb_addr_config;
1178 args->num_banks = config.num_banks;
1179 args->num_ranks = config.num_ranks;
1180
1181 if (args->num_tile_configs > config.num_tile_configs)
1182 args->num_tile_configs = config.num_tile_configs;
1183 err = copy_to_user((void __user *)args->tile_config_ptr,
1184 config.tile_config_ptr,
1185 args->num_tile_configs * sizeof(uint32_t));
1186 if (err) {
1187 args->num_tile_configs = 0;
1188 return -EFAULT;
1189 }
1190
1191 if (args->num_macro_tile_configs > config.num_macro_tile_configs)
1192 args->num_macro_tile_configs =
1193 config.num_macro_tile_configs;
1194 err = copy_to_user((void __user *)args->macro_tile_config_ptr,
1195 config.macro_tile_config_ptr,
1196 args->num_macro_tile_configs * sizeof(uint32_t));
1197 if (err) {
1198 args->num_macro_tile_configs = 0;
1199 return -EFAULT;
1200 }
1201
1202 return 0;
1203}
1204
1205static int kfd_ioctl_acquire_vm(struct file *filep, struct kfd_process *p,
1206 void *data)
1207{
1208 struct kfd_ioctl_acquire_vm_args *args = data;
1209 struct kfd_process_device *pdd;
1210 struct kfd_dev *dev;
1211 struct file *drm_file;
1212 int ret;
1213
1214 dev = kfd_device_by_id(args->gpu_id);
1215 if (!dev)
1216 return -EINVAL;
1217
1218 drm_file = fget(args->drm_fd);
1219 if (!drm_file)
1220 return -EINVAL;
1221
1222 mutex_lock(&p->mutex);
1223
1224 pdd = kfd_get_process_device_data(dev, p);
1225 if (!pdd) {
1226 ret = -EINVAL;
1227 goto err_unlock;
1228 }
1229
1230 if (pdd->drm_file) {
1231 ret = pdd->drm_file == drm_file ? 0 : -EBUSY;
1232 goto err_unlock;
1233 }
1234
1235 ret = kfd_process_device_init_vm(pdd, drm_file);
1236 if (ret)
1237 goto err_unlock;
1238
1239 mutex_unlock(&p->mutex);
1240
1241 return 0;
1242
1243err_unlock:
1244 mutex_unlock(&p->mutex);
1245 fput(drm_file);
1246 return ret;
1247}
1248
1249bool kfd_dev_is_large_bar(struct kfd_dev *dev)
1250{
1251 struct kfd_local_mem_info mem_info;
1252
1253 if (debug_largebar) {
1254 pr_debug("Simulate large-bar allocation on non large-bar machine\n");
1255 return true;
1256 }
1257
1258 if (dev->use_iommu_v2)
1259 return false;
1260
1261 amdgpu_amdkfd_get_local_mem_info(dev->kgd, &mem_info);
1262 if (mem_info.local_mem_size_private == 0 &&
1263 mem_info.local_mem_size_public > 0)
1264 return true;
1265 return false;
1266}
1267
1268static int kfd_ioctl_alloc_memory_of_gpu(struct file *filep,
1269 struct kfd_process *p, void *data)
1270{
1271 struct kfd_ioctl_alloc_memory_of_gpu_args *args = data;
1272 struct kfd_process_device *pdd;
1273 void *mem;
1274 struct kfd_dev *dev;
1275 int idr_handle;
1276 long err;
1277 uint64_t offset = args->mmap_offset;
1278 uint32_t flags = args->flags;
1279
1280 if (args->size == 0)
1281 return -EINVAL;
1282
1283 dev = kfd_device_by_id(args->gpu_id);
1284 if (!dev)
1285 return -EINVAL;
1286
1287 if ((flags & KFD_IOC_ALLOC_MEM_FLAGS_PUBLIC) &&
1288 (flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) &&
1289 !kfd_dev_is_large_bar(dev)) {
1290 pr_err("Alloc host visible vram on small bar is not allowed\n");
1291 return -EINVAL;
1292 }
1293
1294 mutex_lock(&p->mutex);
1295
1296 pdd = kfd_bind_process_to_device(dev, p);
1297 if (IS_ERR(pdd)) {
1298 err = PTR_ERR(pdd);
1299 goto err_unlock;
1300 }
1301
1302 if (flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL) {
1303 if (args->size != kfd_doorbell_process_slice(dev)) {
1304 err = -EINVAL;
1305 goto err_unlock;
1306 }
1307 offset = kfd_get_process_doorbells(pdd);
1308 } else if (flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP) {
1309 if (args->size != PAGE_SIZE) {
1310 err = -EINVAL;
1311 goto err_unlock;
1312 }
1313 offset = amdgpu_amdkfd_get_mmio_remap_phys_addr(dev->kgd);
1314 if (!offset) {
1315 err = -ENOMEM;
1316 goto err_unlock;
1317 }
1318 }
1319
1320 err = amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
1321 dev->kgd, args->va_addr, args->size,
1322 pdd->vm, (struct kgd_mem **) &mem, &offset,
1323 flags);
1324
1325 if (err)
1326 goto err_unlock;
1327
1328 idr_handle = kfd_process_device_create_obj_handle(pdd, mem);
1329 if (idr_handle < 0) {
1330 err = -EFAULT;
1331 goto err_free;
1332 }
1333
1334
1335 if (flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM)
1336 WRITE_ONCE(pdd->vram_usage, pdd->vram_usage + args->size);
1337
1338 mutex_unlock(&p->mutex);
1339
1340 args->handle = MAKE_HANDLE(args->gpu_id, idr_handle);
1341 args->mmap_offset = offset;
1342
1343
1344
1345
1346 if (flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)
1347 args->mmap_offset = KFD_MMAP_TYPE_MMIO
1348 | KFD_MMAP_GPU_ID(args->gpu_id);
1349
1350 return 0;
1351
1352err_free:
1353 amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->kgd, (struct kgd_mem *)mem, NULL);
1354err_unlock:
1355 mutex_unlock(&p->mutex);
1356 return err;
1357}
1358
1359static int kfd_ioctl_free_memory_of_gpu(struct file *filep,
1360 struct kfd_process *p, void *data)
1361{
1362 struct kfd_ioctl_free_memory_of_gpu_args *args = data;
1363 struct kfd_process_device *pdd;
1364 void *mem;
1365 struct kfd_dev *dev;
1366 int ret;
1367 uint64_t size = 0;
1368
1369 dev = kfd_device_by_id(GET_GPU_ID(args->handle));
1370 if (!dev)
1371 return -EINVAL;
1372
1373 mutex_lock(&p->mutex);
1374
1375 pdd = kfd_get_process_device_data(dev, p);
1376 if (!pdd) {
1377 pr_err("Process device data doesn't exist\n");
1378 ret = -EINVAL;
1379 goto err_unlock;
1380 }
1381
1382 mem = kfd_process_device_translate_handle(
1383 pdd, GET_IDR_HANDLE(args->handle));
1384 if (!mem) {
1385 ret = -EINVAL;
1386 goto err_unlock;
1387 }
1388
1389 ret = amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->kgd,
1390 (struct kgd_mem *)mem, &size);
1391
1392
1393
1394
1395 if (!ret)
1396 kfd_process_device_remove_obj_handle(
1397 pdd, GET_IDR_HANDLE(args->handle));
1398
1399 WRITE_ONCE(pdd->vram_usage, pdd->vram_usage - size);
1400
1401err_unlock:
1402 mutex_unlock(&p->mutex);
1403 return ret;
1404}
1405
1406static int kfd_ioctl_map_memory_to_gpu(struct file *filep,
1407 struct kfd_process *p, void *data)
1408{
1409 struct kfd_ioctl_map_memory_to_gpu_args *args = data;
1410 struct kfd_process_device *pdd, *peer_pdd;
1411 void *mem;
1412 struct kfd_dev *dev, *peer;
1413 long err = 0;
1414 int i;
1415 uint32_t *devices_arr = NULL;
1416
1417 dev = kfd_device_by_id(GET_GPU_ID(args->handle));
1418 if (!dev)
1419 return -EINVAL;
1420
1421 if (!args->n_devices) {
1422 pr_debug("Device IDs array empty\n");
1423 return -EINVAL;
1424 }
1425 if (args->n_success > args->n_devices) {
1426 pr_debug("n_success exceeds n_devices\n");
1427 return -EINVAL;
1428 }
1429
1430 devices_arr = kmalloc_array(args->n_devices, sizeof(*devices_arr),
1431 GFP_KERNEL);
1432 if (!devices_arr)
1433 return -ENOMEM;
1434
1435 err = copy_from_user(devices_arr,
1436 (void __user *)args->device_ids_array_ptr,
1437 args->n_devices * sizeof(*devices_arr));
1438 if (err != 0) {
1439 err = -EFAULT;
1440 goto copy_from_user_failed;
1441 }
1442
1443 mutex_lock(&p->mutex);
1444
1445 pdd = kfd_bind_process_to_device(dev, p);
1446 if (IS_ERR(pdd)) {
1447 err = PTR_ERR(pdd);
1448 goto bind_process_to_device_failed;
1449 }
1450
1451 mem = kfd_process_device_translate_handle(pdd,
1452 GET_IDR_HANDLE(args->handle));
1453 if (!mem) {
1454 err = -ENOMEM;
1455 goto get_mem_obj_from_handle_failed;
1456 }
1457
1458 for (i = args->n_success; i < args->n_devices; i++) {
1459 peer = kfd_device_by_id(devices_arr[i]);
1460 if (!peer) {
1461 pr_debug("Getting device by id failed for 0x%x\n",
1462 devices_arr[i]);
1463 err = -EINVAL;
1464 goto get_mem_obj_from_handle_failed;
1465 }
1466
1467 peer_pdd = kfd_bind_process_to_device(peer, p);
1468 if (IS_ERR(peer_pdd)) {
1469 err = PTR_ERR(peer_pdd);
1470 goto get_mem_obj_from_handle_failed;
1471 }
1472 err = amdgpu_amdkfd_gpuvm_map_memory_to_gpu(
1473 peer->kgd, (struct kgd_mem *)mem, peer_pdd->vm);
1474 if (err) {
1475 pr_err("Failed to map to gpu %d/%d\n",
1476 i, args->n_devices);
1477 goto map_memory_to_gpu_failed;
1478 }
1479 args->n_success = i+1;
1480 }
1481
1482 mutex_unlock(&p->mutex);
1483
1484 err = amdgpu_amdkfd_gpuvm_sync_memory(dev->kgd, (struct kgd_mem *) mem, true);
1485 if (err) {
1486 pr_debug("Sync memory failed, wait interrupted by user signal\n");
1487 goto sync_memory_failed;
1488 }
1489
1490
1491 for (i = 0; i < args->n_devices; i++) {
1492 peer = kfd_device_by_id(devices_arr[i]);
1493 if (WARN_ON_ONCE(!peer))
1494 continue;
1495 peer_pdd = kfd_get_process_device_data(peer, p);
1496 if (WARN_ON_ONCE(!peer_pdd))
1497 continue;
1498 kfd_flush_tlb(peer_pdd);
1499 }
1500
1501 kfree(devices_arr);
1502
1503 return err;
1504
1505bind_process_to_device_failed:
1506get_mem_obj_from_handle_failed:
1507map_memory_to_gpu_failed:
1508 mutex_unlock(&p->mutex);
1509copy_from_user_failed:
1510sync_memory_failed:
1511 kfree(devices_arr);
1512
1513 return err;
1514}
1515
1516static int kfd_ioctl_unmap_memory_from_gpu(struct file *filep,
1517 struct kfd_process *p, void *data)
1518{
1519 struct kfd_ioctl_unmap_memory_from_gpu_args *args = data;
1520 struct kfd_process_device *pdd, *peer_pdd;
1521 void *mem;
1522 struct kfd_dev *dev, *peer;
1523 long err = 0;
1524 uint32_t *devices_arr = NULL, i;
1525
1526 dev = kfd_device_by_id(GET_GPU_ID(args->handle));
1527 if (!dev)
1528 return -EINVAL;
1529
1530 if (!args->n_devices) {
1531 pr_debug("Device IDs array empty\n");
1532 return -EINVAL;
1533 }
1534 if (args->n_success > args->n_devices) {
1535 pr_debug("n_success exceeds n_devices\n");
1536 return -EINVAL;
1537 }
1538
1539 devices_arr = kmalloc_array(args->n_devices, sizeof(*devices_arr),
1540 GFP_KERNEL);
1541 if (!devices_arr)
1542 return -ENOMEM;
1543
1544 err = copy_from_user(devices_arr,
1545 (void __user *)args->device_ids_array_ptr,
1546 args->n_devices * sizeof(*devices_arr));
1547 if (err != 0) {
1548 err = -EFAULT;
1549 goto copy_from_user_failed;
1550 }
1551
1552 mutex_lock(&p->mutex);
1553
1554 pdd = kfd_get_process_device_data(dev, p);
1555 if (!pdd) {
1556 err = -EINVAL;
1557 goto bind_process_to_device_failed;
1558 }
1559
1560 mem = kfd_process_device_translate_handle(pdd,
1561 GET_IDR_HANDLE(args->handle));
1562 if (!mem) {
1563 err = -ENOMEM;
1564 goto get_mem_obj_from_handle_failed;
1565 }
1566
1567 for (i = args->n_success; i < args->n_devices; i++) {
1568 peer = kfd_device_by_id(devices_arr[i]);
1569 if (!peer) {
1570 err = -EINVAL;
1571 goto get_mem_obj_from_handle_failed;
1572 }
1573
1574 peer_pdd = kfd_get_process_device_data(peer, p);
1575 if (!peer_pdd) {
1576 err = -ENODEV;
1577 goto get_mem_obj_from_handle_failed;
1578 }
1579 err = amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
1580 peer->kgd, (struct kgd_mem *)mem, peer_pdd->vm);
1581 if (err) {
1582 pr_err("Failed to unmap from gpu %d/%d\n",
1583 i, args->n_devices);
1584 goto unmap_memory_from_gpu_failed;
1585 }
1586 args->n_success = i+1;
1587 }
1588 kfree(devices_arr);
1589
1590 mutex_unlock(&p->mutex);
1591
1592 return 0;
1593
1594bind_process_to_device_failed:
1595get_mem_obj_from_handle_failed:
1596unmap_memory_from_gpu_failed:
1597 mutex_unlock(&p->mutex);
1598copy_from_user_failed:
1599 kfree(devices_arr);
1600 return err;
1601}
1602
1603static int kfd_ioctl_alloc_queue_gws(struct file *filep,
1604 struct kfd_process *p, void *data)
1605{
1606 int retval;
1607 struct kfd_ioctl_alloc_queue_gws_args *args = data;
1608 struct queue *q;
1609 struct kfd_dev *dev;
1610
1611 mutex_lock(&p->mutex);
1612 q = pqm_get_user_queue(&p->pqm, args->queue_id);
1613
1614 if (q) {
1615 dev = q->device;
1616 } else {
1617 retval = -EINVAL;
1618 goto out_unlock;
1619 }
1620
1621 if (!dev->gws) {
1622 retval = -ENODEV;
1623 goto out_unlock;
1624 }
1625
1626 if (dev->dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) {
1627 retval = -ENODEV;
1628 goto out_unlock;
1629 }
1630
1631 retval = pqm_set_gws(&p->pqm, args->queue_id, args->num_gws ? dev->gws : NULL);
1632 mutex_unlock(&p->mutex);
1633
1634 args->first_gws = 0;
1635 return retval;
1636
1637out_unlock:
1638 mutex_unlock(&p->mutex);
1639 return retval;
1640}
1641
1642static int kfd_ioctl_get_dmabuf_info(struct file *filep,
1643 struct kfd_process *p, void *data)
1644{
1645 struct kfd_ioctl_get_dmabuf_info_args *args = data;
1646 struct kfd_dev *dev = NULL;
1647 struct kgd_dev *dma_buf_kgd;
1648 void *metadata_buffer = NULL;
1649 uint32_t flags;
1650 unsigned int i;
1651 int r;
1652
1653
1654 for (i = 0; kfd_topology_enum_kfd_devices(i, &dev) == 0; i++)
1655 if (dev)
1656 break;
1657 if (!dev)
1658 return -EINVAL;
1659
1660 if (args->metadata_ptr) {
1661 metadata_buffer = kzalloc(args->metadata_size, GFP_KERNEL);
1662 if (!metadata_buffer)
1663 return -ENOMEM;
1664 }
1665
1666
1667 r = amdgpu_amdkfd_get_dmabuf_info(dev->kgd, args->dmabuf_fd,
1668 &dma_buf_kgd, &args->size,
1669 metadata_buffer, args->metadata_size,
1670 &args->metadata_size, &flags);
1671 if (r)
1672 goto exit;
1673
1674
1675 dev = kfd_device_by_kgd(dma_buf_kgd);
1676 if (!dev) {
1677 r = -EINVAL;
1678 goto exit;
1679 }
1680 args->gpu_id = dev->id;
1681 args->flags = flags;
1682
1683
1684 if (metadata_buffer) {
1685 r = copy_to_user((void __user *)args->metadata_ptr,
1686 metadata_buffer, args->metadata_size);
1687 if (r != 0)
1688 r = -EFAULT;
1689 }
1690
1691exit:
1692 kfree(metadata_buffer);
1693
1694 return r;
1695}
1696
1697static int kfd_ioctl_import_dmabuf(struct file *filep,
1698 struct kfd_process *p, void *data)
1699{
1700 struct kfd_ioctl_import_dmabuf_args *args = data;
1701 struct kfd_process_device *pdd;
1702 struct dma_buf *dmabuf;
1703 struct kfd_dev *dev;
1704 int idr_handle;
1705 uint64_t size;
1706 void *mem;
1707 int r;
1708
1709 dev = kfd_device_by_id(args->gpu_id);
1710 if (!dev)
1711 return -EINVAL;
1712
1713 dmabuf = dma_buf_get(args->dmabuf_fd);
1714 if (IS_ERR(dmabuf))
1715 return PTR_ERR(dmabuf);
1716
1717 mutex_lock(&p->mutex);
1718
1719 pdd = kfd_bind_process_to_device(dev, p);
1720 if (IS_ERR(pdd)) {
1721 r = PTR_ERR(pdd);
1722 goto err_unlock;
1723 }
1724
1725 r = amdgpu_amdkfd_gpuvm_import_dmabuf(dev->kgd, dmabuf,
1726 args->va_addr, pdd->vm,
1727 (struct kgd_mem **)&mem, &size,
1728 NULL);
1729 if (r)
1730 goto err_unlock;
1731
1732 idr_handle = kfd_process_device_create_obj_handle(pdd, mem);
1733 if (idr_handle < 0) {
1734 r = -EFAULT;
1735 goto err_free;
1736 }
1737
1738 mutex_unlock(&p->mutex);
1739 dma_buf_put(dmabuf);
1740
1741 args->handle = MAKE_HANDLE(args->gpu_id, idr_handle);
1742
1743 return 0;
1744
1745err_free:
1746 amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->kgd, (struct kgd_mem *)mem, NULL);
1747err_unlock:
1748 mutex_unlock(&p->mutex);
1749 dma_buf_put(dmabuf);
1750 return r;
1751}
1752
1753
1754static int kfd_ioctl_smi_events(struct file *filep,
1755 struct kfd_process *p, void *data)
1756{
1757 struct kfd_ioctl_smi_events_args *args = data;
1758 struct kfd_dev *dev;
1759
1760 dev = kfd_device_by_id(args->gpuid);
1761 if (!dev)
1762 return -EINVAL;
1763
1764 return kfd_smi_event_open(dev, &args->anon_fd);
1765}
1766
1767#define AMDKFD_IOCTL_DEF(ioctl, _func, _flags) \
1768 [_IOC_NR(ioctl)] = {.cmd = ioctl, .func = _func, .flags = _flags, \
1769 .cmd_drv = 0, .name = #ioctl}
1770
1771
1772static const struct amdkfd_ioctl_desc amdkfd_ioctls[] = {
1773 AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_VERSION,
1774 kfd_ioctl_get_version, 0),
1775
1776 AMDKFD_IOCTL_DEF(AMDKFD_IOC_CREATE_QUEUE,
1777 kfd_ioctl_create_queue, 0),
1778
1779 AMDKFD_IOCTL_DEF(AMDKFD_IOC_DESTROY_QUEUE,
1780 kfd_ioctl_destroy_queue, 0),
1781
1782 AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_MEMORY_POLICY,
1783 kfd_ioctl_set_memory_policy, 0),
1784
1785 AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_CLOCK_COUNTERS,
1786 kfd_ioctl_get_clock_counters, 0),
1787
1788 AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_PROCESS_APERTURES,
1789 kfd_ioctl_get_process_apertures, 0),
1790
1791 AMDKFD_IOCTL_DEF(AMDKFD_IOC_UPDATE_QUEUE,
1792 kfd_ioctl_update_queue, 0),
1793
1794 AMDKFD_IOCTL_DEF(AMDKFD_IOC_CREATE_EVENT,
1795 kfd_ioctl_create_event, 0),
1796
1797 AMDKFD_IOCTL_DEF(AMDKFD_IOC_DESTROY_EVENT,
1798 kfd_ioctl_destroy_event, 0),
1799
1800 AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_EVENT,
1801 kfd_ioctl_set_event, 0),
1802
1803 AMDKFD_IOCTL_DEF(AMDKFD_IOC_RESET_EVENT,
1804 kfd_ioctl_reset_event, 0),
1805
1806 AMDKFD_IOCTL_DEF(AMDKFD_IOC_WAIT_EVENTS,
1807 kfd_ioctl_wait_events, 0),
1808
1809 AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_REGISTER,
1810 kfd_ioctl_dbg_register, 0),
1811
1812 AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_UNREGISTER,
1813 kfd_ioctl_dbg_unregister, 0),
1814
1815 AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_ADDRESS_WATCH,
1816 kfd_ioctl_dbg_address_watch, 0),
1817
1818 AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_WAVE_CONTROL,
1819 kfd_ioctl_dbg_wave_control, 0),
1820
1821 AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_SCRATCH_BACKING_VA,
1822 kfd_ioctl_set_scratch_backing_va, 0),
1823
1824 AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_TILE_CONFIG,
1825 kfd_ioctl_get_tile_config, 0),
1826
1827 AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_TRAP_HANDLER,
1828 kfd_ioctl_set_trap_handler, 0),
1829
1830 AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_PROCESS_APERTURES_NEW,
1831 kfd_ioctl_get_process_apertures_new, 0),
1832
1833 AMDKFD_IOCTL_DEF(AMDKFD_IOC_ACQUIRE_VM,
1834 kfd_ioctl_acquire_vm, 0),
1835
1836 AMDKFD_IOCTL_DEF(AMDKFD_IOC_ALLOC_MEMORY_OF_GPU,
1837 kfd_ioctl_alloc_memory_of_gpu, 0),
1838
1839 AMDKFD_IOCTL_DEF(AMDKFD_IOC_FREE_MEMORY_OF_GPU,
1840 kfd_ioctl_free_memory_of_gpu, 0),
1841
1842 AMDKFD_IOCTL_DEF(AMDKFD_IOC_MAP_MEMORY_TO_GPU,
1843 kfd_ioctl_map_memory_to_gpu, 0),
1844
1845 AMDKFD_IOCTL_DEF(AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU,
1846 kfd_ioctl_unmap_memory_from_gpu, 0),
1847
1848 AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_CU_MASK,
1849 kfd_ioctl_set_cu_mask, 0),
1850
1851 AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_QUEUE_WAVE_STATE,
1852 kfd_ioctl_get_queue_wave_state, 0),
1853
1854 AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_DMABUF_INFO,
1855 kfd_ioctl_get_dmabuf_info, 0),
1856
1857 AMDKFD_IOCTL_DEF(AMDKFD_IOC_IMPORT_DMABUF,
1858 kfd_ioctl_import_dmabuf, 0),
1859
1860 AMDKFD_IOCTL_DEF(AMDKFD_IOC_ALLOC_QUEUE_GWS,
1861 kfd_ioctl_alloc_queue_gws, 0),
1862
1863 AMDKFD_IOCTL_DEF(AMDKFD_IOC_SMI_EVENTS,
1864 kfd_ioctl_smi_events, 0),
1865};
1866
1867#define AMDKFD_CORE_IOCTL_COUNT ARRAY_SIZE(amdkfd_ioctls)
1868
1869static long kfd_ioctl(struct file *filep, unsigned int cmd, unsigned long arg)
1870{
1871 struct kfd_process *process;
1872 amdkfd_ioctl_t *func;
1873 const struct amdkfd_ioctl_desc *ioctl = NULL;
1874 unsigned int nr = _IOC_NR(cmd);
1875 char stack_kdata[128];
1876 char *kdata = NULL;
1877 unsigned int usize, asize;
1878 int retcode = -EINVAL;
1879
1880 if (nr >= AMDKFD_CORE_IOCTL_COUNT)
1881 goto err_i1;
1882
1883 if ((nr >= AMDKFD_COMMAND_START) && (nr < AMDKFD_COMMAND_END)) {
1884 u32 amdkfd_size;
1885
1886 ioctl = &amdkfd_ioctls[nr];
1887
1888 amdkfd_size = _IOC_SIZE(ioctl->cmd);
1889 usize = asize = _IOC_SIZE(cmd);
1890 if (amdkfd_size > asize)
1891 asize = amdkfd_size;
1892
1893 cmd = ioctl->cmd;
1894 } else
1895 goto err_i1;
1896
1897 dev_dbg(kfd_device, "ioctl cmd 0x%x (#0x%x), arg 0x%lx\n", cmd, nr, arg);
1898
1899
1900
1901
1902
1903 process = filep->private_data;
1904 if (process->lead_thread != current->group_leader) {
1905 dev_dbg(kfd_device, "Using KFD FD in wrong process\n");
1906 retcode = -EBADF;
1907 goto err_i1;
1908 }
1909
1910
1911 func = ioctl->func;
1912
1913 if (unlikely(!func)) {
1914 dev_dbg(kfd_device, "no function\n");
1915 retcode = -EINVAL;
1916 goto err_i1;
1917 }
1918
1919 if (cmd & (IOC_IN | IOC_OUT)) {
1920 if (asize <= sizeof(stack_kdata)) {
1921 kdata = stack_kdata;
1922 } else {
1923 kdata = kmalloc(asize, GFP_KERNEL);
1924 if (!kdata) {
1925 retcode = -ENOMEM;
1926 goto err_i1;
1927 }
1928 }
1929 if (asize > usize)
1930 memset(kdata + usize, 0, asize - usize);
1931 }
1932
1933 if (cmd & IOC_IN) {
1934 if (copy_from_user(kdata, (void __user *)arg, usize) != 0) {
1935 retcode = -EFAULT;
1936 goto err_i1;
1937 }
1938 } else if (cmd & IOC_OUT) {
1939 memset(kdata, 0, usize);
1940 }
1941
1942 retcode = func(filep, process, kdata);
1943
1944 if (cmd & IOC_OUT)
1945 if (copy_to_user((void __user *)arg, kdata, usize) != 0)
1946 retcode = -EFAULT;
1947
1948err_i1:
1949 if (!ioctl)
1950 dev_dbg(kfd_device, "invalid ioctl: pid=%d, cmd=0x%02x, nr=0x%02x\n",
1951 task_pid_nr(current), cmd, nr);
1952
1953 if (kdata != stack_kdata)
1954 kfree(kdata);
1955
1956 if (retcode)
1957 dev_dbg(kfd_device, "ioctl cmd (#0x%x), arg 0x%lx, ret = %d\n",
1958 nr, arg, retcode);
1959
1960 return retcode;
1961}
1962
1963static int kfd_mmio_mmap(struct kfd_dev *dev, struct kfd_process *process,
1964 struct vm_area_struct *vma)
1965{
1966 phys_addr_t address;
1967 int ret;
1968
1969 if (vma->vm_end - vma->vm_start != PAGE_SIZE)
1970 return -EINVAL;
1971
1972 address = amdgpu_amdkfd_get_mmio_remap_phys_addr(dev->kgd);
1973
1974 vma->vm_flags |= VM_IO | VM_DONTCOPY | VM_DONTEXPAND | VM_NORESERVE |
1975 VM_DONTDUMP | VM_PFNMAP;
1976
1977 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
1978
1979 pr_debug("pasid 0x%x mapping mmio page\n"
1980 " target user address == 0x%08llX\n"
1981 " physical address == 0x%08llX\n"
1982 " vm_flags == 0x%04lX\n"
1983 " size == 0x%04lX\n",
1984 process->pasid, (unsigned long long) vma->vm_start,
1985 address, vma->vm_flags, PAGE_SIZE);
1986
1987 ret = io_remap_pfn_range(vma,
1988 vma->vm_start,
1989 address >> PAGE_SHIFT,
1990 PAGE_SIZE,
1991 vma->vm_page_prot);
1992 return ret;
1993}
1994
1995
1996static int kfd_mmap(struct file *filp, struct vm_area_struct *vma)
1997{
1998 struct kfd_process *process;
1999 struct kfd_dev *dev = NULL;
2000 unsigned long mmap_offset;
2001 unsigned int gpu_id;
2002
2003 process = kfd_get_process(current);
2004 if (IS_ERR(process))
2005 return PTR_ERR(process);
2006
2007 mmap_offset = vma->vm_pgoff << PAGE_SHIFT;
2008 gpu_id = KFD_MMAP_GET_GPU_ID(mmap_offset);
2009 if (gpu_id)
2010 dev = kfd_device_by_id(gpu_id);
2011
2012 switch (mmap_offset & KFD_MMAP_TYPE_MASK) {
2013 case KFD_MMAP_TYPE_DOORBELL:
2014 if (!dev)
2015 return -ENODEV;
2016 return kfd_doorbell_mmap(dev, process, vma);
2017
2018 case KFD_MMAP_TYPE_EVENTS:
2019 return kfd_event_mmap(process, vma);
2020
2021 case KFD_MMAP_TYPE_RESERVED_MEM:
2022 if (!dev)
2023 return -ENODEV;
2024 return kfd_reserved_mem_mmap(dev, process, vma);
2025 case KFD_MMAP_TYPE_MMIO:
2026 if (!dev)
2027 return -ENODEV;
2028 return kfd_mmio_mmap(dev, process, vma);
2029 }
2030
2031 return -EFAULT;
2032}
2033
