1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23#ifndef KFD_PRIV_H_INCLUDED
24#define KFD_PRIV_H_INCLUDED
25
26#include <linux/hashtable.h>
27#include <linux/mmu_notifier.h>
28#include <linux/mutex.h>
29#include <linux/types.h>
30#include <linux/atomic.h>
31#include <linux/workqueue.h>
32#include <linux/spinlock.h>
33#include <linux/kfd_ioctl.h>
34#include <linux/idr.h>
35#include <linux/kfifo.h>
36#include <linux/seq_file.h>
37#include <linux/kref.h>
38#include <linux/sysfs.h>
39#include <linux/device_cgroup.h>
40#include <drm/drm_file.h>
41#include <drm/drm_drv.h>
42#include <drm/drm_device.h>
43#include <drm/drm_ioctl.h>
44#include <kgd_kfd_interface.h>
45#include <linux/swap.h>
46
47#include "amd_shared.h"
48
49#define KFD_MAX_RING_ENTRY_SIZE 8
50
51#define KFD_SYSFS_FILE_MODE 0444
52
53
54#define KFD_GPU_ID_HASH_WIDTH 16
55
56
57
58
59
60
61
62
63
64#define KFD_MMAP_TYPE_SHIFT 62
65#define KFD_MMAP_TYPE_MASK (0x3ULL << KFD_MMAP_TYPE_SHIFT)
66#define KFD_MMAP_TYPE_DOORBELL (0x3ULL << KFD_MMAP_TYPE_SHIFT)
67#define KFD_MMAP_TYPE_EVENTS (0x2ULL << KFD_MMAP_TYPE_SHIFT)
68#define KFD_MMAP_TYPE_RESERVED_MEM (0x1ULL << KFD_MMAP_TYPE_SHIFT)
69#define KFD_MMAP_TYPE_MMIO (0x0ULL << KFD_MMAP_TYPE_SHIFT)
70
71#define KFD_MMAP_GPU_ID_SHIFT 46
72#define KFD_MMAP_GPU_ID_MASK (((1ULL << KFD_GPU_ID_HASH_WIDTH) - 1) \
73 << KFD_MMAP_GPU_ID_SHIFT)
74#define KFD_MMAP_GPU_ID(gpu_id) ((((uint64_t)gpu_id) << KFD_MMAP_GPU_ID_SHIFT)\
75 & KFD_MMAP_GPU_ID_MASK)
76#define KFD_MMAP_GET_GPU_ID(offset) ((offset & KFD_MMAP_GPU_ID_MASK) \
77 >> KFD_MMAP_GPU_ID_SHIFT)
78
79
80
81
82
83
84
85
86#define KFD_CIK_HIQ_PIPE 4
87#define KFD_CIK_HIQ_QUEUE 0
88
89
90#define kfd_alloc_struct(ptr_to_struct) \
91 ((typeof(ptr_to_struct)) kzalloc(sizeof(*ptr_to_struct), GFP_KERNEL))
92
93#define KFD_MAX_NUM_OF_PROCESSES 512
94#define KFD_MAX_NUM_OF_QUEUES_PER_PROCESS 1024
95
96
97
98
99
100
101
102#define KFD_CWSR_TBA_TMA_SIZE (PAGE_SIZE * 2)
103#define KFD_CWSR_TMA_OFFSET PAGE_SIZE
104
105#define KFD_MAX_NUM_OF_QUEUES_PER_DEVICE \
106 (KFD_MAX_NUM_OF_PROCESSES * \
107 KFD_MAX_NUM_OF_QUEUES_PER_PROCESS)
108
109#define KFD_KERNEL_QUEUE_SIZE 2048
110
111#define KFD_UNMAP_LATENCY_MS (4000)
112
113
114
115
116
117
118
119
120
121#define KFD_QUEUE_DOORBELL_MIRROR_OFFSET 512
122
123
124
125
126
127
128extern int max_num_of_queues_per_device;
129
130
131
132extern int sched_policy;
133
134
135
136
137
138extern int hws_max_conc_proc;
139
140extern int cwsr_enable;
141
142
143
144
145
146extern int send_sigterm;
147
148
149
150
151
152extern int debug_largebar;
153
154
155
156
157
158extern int ignore_crat;
159
160
161
162
163extern int amdgpu_noretry;
164
165
166
167
168extern int halt_if_hws_hang;
169
170
171
172
173extern bool hws_gws_support;
174
175
176
177
178extern int queue_preemption_timeout_ms;
179
180enum cache_policy {
181 cache_policy_coherent,
182 cache_policy_noncoherent
183};
184
185#define KFD_IS_SOC15(chip) ((chip) >= CHIP_VEGA10)
186
187struct kfd_event_interrupt_class {
188 bool (*interrupt_isr)(struct kfd_dev *dev,
189 const uint32_t *ih_ring_entry, uint32_t *patched_ihre,
190 bool *patched_flag);
191 void (*interrupt_wq)(struct kfd_dev *dev,
192 const uint32_t *ih_ring_entry);
193};
194
195struct kfd_device_info {
196 enum amd_asic_type asic_family;
197 const char *asic_name;
198 const struct kfd_event_interrupt_class *event_interrupt_class;
199 unsigned int max_pasid_bits;
200 unsigned int max_no_of_hqd;
201 unsigned int doorbell_size;
202 size_t ih_ring_entry_size;
203 uint8_t num_of_watch_points;
204 uint16_t mqd_size_aligned;
205 bool supports_cwsr;
206 bool needs_iommu_device;
207 bool needs_pci_atomics;
208 unsigned int num_sdma_engines;
209 unsigned int num_xgmi_sdma_engines;
210 unsigned int num_sdma_queues_per_engine;
211};
212
213struct kfd_mem_obj {
214 uint32_t range_start;
215 uint32_t range_end;
216 uint64_t gpu_addr;
217 uint32_t *cpu_ptr;
218 void *gtt_mem;
219};
220
221struct kfd_vmid_info {
222 uint32_t first_vmid_kfd;
223 uint32_t last_vmid_kfd;
224 uint32_t vmid_num_kfd;
225};
226
227struct kfd_dev {
228 struct kgd_dev *kgd;
229
230 const struct kfd_device_info *device_info;
231 struct pci_dev *pdev;
232 struct drm_device *ddev;
233
234 unsigned int id;
235
236 phys_addr_t doorbell_base;
237
238
239
240 size_t doorbell_base_dw_offset;
241
242
243
244
245 u32 __iomem *doorbell_kernel_ptr;
246
247
248
249 struct kgd2kfd_shared_resources shared_resources;
250 struct kfd_vmid_info vm_info;
251
252 const struct kfd2kgd_calls *kfd2kgd;
253 struct mutex doorbell_mutex;
254 DECLARE_BITMAP(doorbell_available_index,
255 KFD_MAX_NUM_OF_QUEUES_PER_PROCESS);
256
257 void *gtt_mem;
258 uint64_t gtt_start_gpu_addr;
259 void *gtt_start_cpu_ptr;
260 void *gtt_sa_bitmap;
261 struct mutex gtt_sa_lock;
262 unsigned int gtt_sa_chunk_size;
263 unsigned int gtt_sa_num_of_chunks;
264
265
266 struct kfifo ih_fifo;
267 struct workqueue_struct *ih_wq;
268 struct work_struct interrupt_work;
269 spinlock_t interrupt_lock;
270
271
272 struct device_queue_manager *dqm;
273
274 bool init_complete;
275
276
277
278
279 bool interrupts_active;
280
281
282 struct kfd_dbgmgr *dbgmgr;
283
284
285 uint16_t mec_fw_version;
286 uint16_t mec2_fw_version;
287 uint16_t sdma_fw_version;
288
289
290 unsigned int max_proc_per_quantum;
291
292
293 bool cwsr_enabled;
294 const void *cwsr_isa;
295 unsigned int cwsr_isa_size;
296
297
298 uint64_t hive_id;
299
300
301 uint64_t unique_id;
302
303 bool pci_atomic_requested;
304
305
306 atomic_t sram_ecc_flag;
307
308
309 atomic_t compute_profile;
310
311
312 void *gws;
313};
314
315enum kfd_mempool {
316 KFD_MEMPOOL_SYSTEM_CACHEABLE = 1,
317 KFD_MEMPOOL_SYSTEM_WRITECOMBINE = 2,
318 KFD_MEMPOOL_FRAMEBUFFER = 3,
319};
320
321
322int kfd_chardev_init(void);
323void kfd_chardev_exit(void);
324struct device *kfd_chardev(void);
325
326
327
328
329
330
331
332
333
334
335
336
337
338enum kfd_unmap_queues_filter {
339 KFD_UNMAP_QUEUES_FILTER_SINGLE_QUEUE,
340 KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES,
341 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES,
342 KFD_UNMAP_QUEUES_FILTER_BY_PASID
343};
344
345
346
347
348
349
350
351
352
353
354
355
356enum kfd_queue_type {
357 KFD_QUEUE_TYPE_COMPUTE,
358 KFD_QUEUE_TYPE_SDMA,
359 KFD_QUEUE_TYPE_HIQ,
360 KFD_QUEUE_TYPE_DIQ,
361 KFD_QUEUE_TYPE_SDMA_XGMI
362};
363
364enum kfd_queue_format {
365 KFD_QUEUE_FORMAT_PM4,
366 KFD_QUEUE_FORMAT_AQL
367};
368
369enum KFD_QUEUE_PRIORITY {
370 KFD_QUEUE_PRIORITY_MINIMUM = 0,
371 KFD_QUEUE_PRIORITY_MAXIMUM = 15
372};
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426struct queue_properties {
427 enum kfd_queue_type type;
428 enum kfd_queue_format format;
429 unsigned int queue_id;
430 uint64_t queue_address;
431 uint64_t queue_size;
432 uint32_t priority;
433 uint32_t queue_percent;
434 uint32_t *read_ptr;
435 uint32_t *write_ptr;
436 void __iomem *doorbell_ptr;
437 uint32_t doorbell_off;
438 bool is_interop;
439 bool is_evicted;
440 bool is_active;
441 bool is_gws;
442
443 unsigned int vmid;
444
445 uint32_t sdma_engine_id;
446 uint32_t sdma_queue_id;
447 uint32_t sdma_vm_addr;
448
449 uint64_t eop_ring_buffer_address;
450 uint32_t eop_ring_buffer_size;
451 uint64_t ctx_save_restore_area_address;
452 uint32_t ctx_save_restore_area_size;
453 uint32_t ctl_stack_size;
454 uint64_t tba_addr;
455 uint64_t tma_addr;
456
457 uint32_t cu_mask_count;
458 uint32_t *cu_mask;
459};
460
461#define QUEUE_IS_ACTIVE(q) ((q).queue_size > 0 && \
462 (q).queue_address != 0 && \
463 (q).queue_percent > 0 && \
464 !(q).is_evicted)
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499struct queue {
500 struct list_head list;
501 void *mqd;
502 struct kfd_mem_obj *mqd_mem_obj;
503 uint64_t gart_mqd_addr;
504 struct queue_properties properties;
505
506 uint32_t mec;
507 uint32_t pipe;
508 uint32_t queue;
509
510 unsigned int sdma_id;
511 unsigned int doorbell_id;
512
513 struct kfd_process *process;
514 struct kfd_dev *device;
515 void *gws;
516
517
518 struct kobject kobj;
519};
520
521
522
523
524enum KFD_MQD_TYPE {
525 KFD_MQD_TYPE_HIQ = 0,
526 KFD_MQD_TYPE_CP,
527 KFD_MQD_TYPE_SDMA,
528 KFD_MQD_TYPE_DIQ,
529 KFD_MQD_TYPE_MAX
530};
531
532enum KFD_PIPE_PRIORITY {
533 KFD_PIPE_PRIORITY_CS_LOW = 0,
534 KFD_PIPE_PRIORITY_CS_MEDIUM,
535 KFD_PIPE_PRIORITY_CS_HIGH
536};
537
538struct scheduling_resources {
539 unsigned int vmid_mask;
540 enum kfd_queue_type type;
541 uint64_t queue_mask;
542 uint64_t gws_mask;
543 uint32_t oac_mask;
544 uint32_t gds_heap_base;
545 uint32_t gds_heap_size;
546};
547
548struct process_queue_manager {
549
550 struct kfd_process *process;
551 struct list_head queues;
552 unsigned long *queue_slot_bitmap;
553};
554
555struct qcm_process_device {
556
557 struct device_queue_manager *dqm;
558 struct process_queue_manager *pqm;
559
560 struct list_head queues_list;
561 struct list_head priv_queue_list;
562
563 unsigned int queue_count;
564 unsigned int vmid;
565 bool is_debug;
566 unsigned int evicted;
567
568
569
570
571 bool reset_wavefronts;
572
573
574
575
576
577
578
579 bool mapped_gws_queue;
580
581
582
583
584 uint64_t gds_context_area;
585
586 uint64_t page_table_base;
587 uint32_t sh_mem_config;
588 uint32_t sh_mem_bases;
589 uint32_t sh_mem_ape1_base;
590 uint32_t sh_mem_ape1_limit;
591 uint32_t gds_size;
592 uint32_t num_gws;
593 uint32_t num_oac;
594 uint32_t sh_hidden_private_base;
595
596
597 void *cwsr_kaddr;
598 uint64_t cwsr_base;
599 uint64_t tba_addr;
600 uint64_t tma_addr;
601
602
603 uint64_t ib_base;
604 void *ib_kaddr;
605
606
607 unsigned long *doorbell_bitmap;
608};
609
610
611
612
613#define PROCESS_RESTORE_TIME_MS 100
614
615#define PROCESS_BACK_OFF_TIME_MS 100
616
617#define PROCESS_ACTIVE_TIME_MS 10
618
619
620
621
622#define MAKE_HANDLE(gpu_id, idr_handle) \
623 (((uint64_t)(gpu_id) << 32) + idr_handle)
624#define GET_GPU_ID(handle) (handle >> 32)
625#define GET_IDR_HANDLE(handle) (handle & 0xFFFFFFFF)
626
627enum kfd_pdd_bound {
628 PDD_UNBOUND = 0,
629 PDD_BOUND,
630 PDD_BOUND_SUSPENDED,
631};
632
633#define MAX_VRAM_FILENAME_LEN 11
634
635
636struct kfd_process_device {
637
638
639
640
641 struct list_head per_device_list;
642
643
644 struct kfd_dev *dev;
645
646
647 struct kfd_process *process;
648
649
650 struct qcm_process_device qpd;
651
652
653 uint64_t lds_base;
654 uint64_t lds_limit;
655 uint64_t gpuvm_base;
656 uint64_t gpuvm_limit;
657 uint64_t scratch_base;
658 uint64_t scratch_limit;
659
660
661 struct file *drm_file;
662 void *vm;
663
664
665 struct idr alloc_idr;
666
667
668
669
670
671
672 bool already_dequeued;
673 bool runtime_inuse;
674
675
676 enum kfd_pdd_bound bound;
677
678
679 uint64_t vram_usage;
680 struct attribute attr_vram;
681 char vram_filename[MAX_VRAM_FILENAME_LEN];
682};
683
684#define qpd_to_pdd(x) container_of(x, struct kfd_process_device, qpd)
685
686
687struct kfd_process {
688
689
690
691
692 struct hlist_node kfd_processes;
693
694
695
696
697
698
699 void *mm;
700
701 struct kref ref;
702 struct work_struct release_work;
703
704 struct mutex mutex;
705
706
707
708
709
710
711
712
713 struct task_struct *lead_thread;
714
715
716 struct mmu_notifier mmu_notifier;
717
718 uint16_t pasid;
719 unsigned int doorbell_index;
720
721
722
723
724
725 struct list_head per_device_data;
726
727 struct process_queue_manager pqm;
728
729
730 bool is_32bit_user_mode;
731
732
733 struct mutex event_mutex;
734
735 struct idr event_idr;
736
737 struct kfd_signal_page *signal_page;
738 size_t signal_mapped_size;
739 size_t signal_event_count;
740 bool signal_event_limit_reached;
741
742
743 void *kgd_process_info;
744
745
746
747
748 struct dma_fence *ef;
749
750
751 struct delayed_work eviction_work;
752 struct delayed_work restore_work;
753
754 unsigned int last_eviction_seqno;
755
756
757
758 unsigned long last_restore_timestamp;
759
760
761 struct kobject *kobj;
762 struct kobject *kobj_queues;
763 struct attribute attr_pasid;
764};
765
766#define KFD_PROCESS_TABLE_SIZE 5
767extern DECLARE_HASHTABLE(kfd_processes_table, KFD_PROCESS_TABLE_SIZE);
768extern struct srcu_struct kfd_processes_srcu;
769
770
771
772
773
774
775
776
777typedef int amdkfd_ioctl_t(struct file *filep, struct kfd_process *p,
778 void *data);
779
780struct amdkfd_ioctl_desc {
781 unsigned int cmd;
782 int flags;
783 amdkfd_ioctl_t *func;
784 unsigned int cmd_drv;
785 const char *name;
786};
787bool kfd_dev_is_large_bar(struct kfd_dev *dev);
788
789int kfd_process_create_wq(void);
790void kfd_process_destroy_wq(void);
791struct kfd_process *kfd_create_process(struct file *filep);
792struct kfd_process *kfd_get_process(const struct task_struct *);
793struct kfd_process *kfd_lookup_process_by_pasid(unsigned int pasid);
794struct kfd_process *kfd_lookup_process_by_mm(const struct mm_struct *mm);
795void kfd_unref_process(struct kfd_process *p);
796int kfd_process_evict_queues(struct kfd_process *p);
797int kfd_process_restore_queues(struct kfd_process *p);
798void kfd_suspend_all_processes(void);
799int kfd_resume_all_processes(void);
800
801int kfd_process_device_init_vm(struct kfd_process_device *pdd,
802 struct file *drm_file);
803struct kfd_process_device *kfd_bind_process_to_device(struct kfd_dev *dev,
804 struct kfd_process *p);
805struct kfd_process_device *kfd_get_process_device_data(struct kfd_dev *dev,
806 struct kfd_process *p);
807struct kfd_process_device *kfd_create_process_device_data(struct kfd_dev *dev,
808 struct kfd_process *p);
809
810int kfd_reserved_mem_mmap(struct kfd_dev *dev, struct kfd_process *process,
811 struct vm_area_struct *vma);
812
813
814int kfd_process_device_create_obj_handle(struct kfd_process_device *pdd,
815 void *mem);
816void *kfd_process_device_translate_handle(struct kfd_process_device *p,
817 int handle);
818void kfd_process_device_remove_obj_handle(struct kfd_process_device *pdd,
819 int handle);
820
821
822struct kfd_process_device *kfd_get_first_process_device_data(
823 struct kfd_process *p);
824struct kfd_process_device *kfd_get_next_process_device_data(
825 struct kfd_process *p,
826 struct kfd_process_device *pdd);
827bool kfd_has_process_device_data(struct kfd_process *p);
828
829
830int kfd_pasid_init(void);
831void kfd_pasid_exit(void);
832bool kfd_set_pasid_limit(unsigned int new_limit);
833unsigned int kfd_get_pasid_limit(void);
834unsigned int kfd_pasid_alloc(void);
835void kfd_pasid_free(unsigned int pasid);
836
837
838size_t kfd_doorbell_process_slice(struct kfd_dev *kfd);
839int kfd_doorbell_init(struct kfd_dev *kfd);
840void kfd_doorbell_fini(struct kfd_dev *kfd);
841int kfd_doorbell_mmap(struct kfd_dev *dev, struct kfd_process *process,
842 struct vm_area_struct *vma);
843void __iomem *kfd_get_kernel_doorbell(struct kfd_dev *kfd,
844 unsigned int *doorbell_off);
845void kfd_release_kernel_doorbell(struct kfd_dev *kfd, u32 __iomem *db_addr);
846u32 read_kernel_doorbell(u32 __iomem *db);
847void write_kernel_doorbell(void __iomem *db, u32 value);
848void write_kernel_doorbell64(void __iomem *db, u64 value);
849unsigned int kfd_get_doorbell_dw_offset_in_bar(struct kfd_dev *kfd,
850 struct kfd_process *process,
851 unsigned int doorbell_id);
852phys_addr_t kfd_get_process_doorbells(struct kfd_dev *dev,
853 struct kfd_process *process);
854int kfd_alloc_process_doorbells(struct kfd_process *process);
855void kfd_free_process_doorbells(struct kfd_process *process);
856
857
858
859int kfd_gtt_sa_allocate(struct kfd_dev *kfd, unsigned int size,
860 struct kfd_mem_obj **mem_obj);
861
862int kfd_gtt_sa_free(struct kfd_dev *kfd, struct kfd_mem_obj *mem_obj);
863
864extern struct device *kfd_device;
865
866
867void kfd_procfs_init(void);
868void kfd_procfs_shutdown(void);
869int kfd_procfs_add_queue(struct queue *q);
870void kfd_procfs_del_queue(struct queue *q);
871
872
873int kfd_topology_init(void);
874void kfd_topology_shutdown(void);
875int kfd_topology_add_device(struct kfd_dev *gpu);
876int kfd_topology_remove_device(struct kfd_dev *gpu);
877struct kfd_topology_device *kfd_topology_device_by_proximity_domain(
878 uint32_t proximity_domain);
879struct kfd_topology_device *kfd_topology_device_by_id(uint32_t gpu_id);
880struct kfd_dev *kfd_device_by_id(uint32_t gpu_id);
881struct kfd_dev *kfd_device_by_pci_dev(const struct pci_dev *pdev);
882struct kfd_dev *kfd_device_by_kgd(const struct kgd_dev *kgd);
883int kfd_topology_enum_kfd_devices(uint8_t idx, struct kfd_dev **kdev);
884int kfd_numa_node_to_apic_id(int numa_node_id);
885
886
887int kfd_interrupt_init(struct kfd_dev *dev);
888void kfd_interrupt_exit(struct kfd_dev *dev);
889bool enqueue_ih_ring_entry(struct kfd_dev *kfd, const void *ih_ring_entry);
890bool interrupt_is_wanted(struct kfd_dev *dev,
891 const uint32_t *ih_ring_entry,
892 uint32_t *patched_ihre, bool *flag);
893
894
895int kfd_init_apertures(struct kfd_process *process);
896
897
898int init_queue(struct queue **q, const struct queue_properties *properties);
899void uninit_queue(struct queue *q);
900void print_queue_properties(struct queue_properties *q);
901void print_queue(struct queue *q);
902
903struct mqd_manager *mqd_manager_init_cik(enum KFD_MQD_TYPE type,
904 struct kfd_dev *dev);
905struct mqd_manager *mqd_manager_init_cik_hawaii(enum KFD_MQD_TYPE type,
906 struct kfd_dev *dev);
907struct mqd_manager *mqd_manager_init_vi(enum KFD_MQD_TYPE type,
908 struct kfd_dev *dev);
909struct mqd_manager *mqd_manager_init_vi_tonga(enum KFD_MQD_TYPE type,
910 struct kfd_dev *dev);
911struct mqd_manager *mqd_manager_init_v9(enum KFD_MQD_TYPE type,
912 struct kfd_dev *dev);
913struct mqd_manager *mqd_manager_init_v10(enum KFD_MQD_TYPE type,
914 struct kfd_dev *dev);
915struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev);
916void device_queue_manager_uninit(struct device_queue_manager *dqm);
917struct kernel_queue *kernel_queue_init(struct kfd_dev *dev,
918 enum kfd_queue_type type);
919void kernel_queue_uninit(struct kernel_queue *kq, bool hanging);
920int kfd_process_vm_fault(struct device_queue_manager *dqm, unsigned int pasid);
921
922
923struct process_queue_node {
924 struct queue *q;
925 struct kernel_queue *kq;
926 struct list_head process_queue_list;
927};
928
929void kfd_process_dequeue_from_device(struct kfd_process_device *pdd);
930void kfd_process_dequeue_from_all_devices(struct kfd_process *p);
931int pqm_init(struct process_queue_manager *pqm, struct kfd_process *p);
932void pqm_uninit(struct process_queue_manager *pqm);
933int pqm_create_queue(struct process_queue_manager *pqm,
934 struct kfd_dev *dev,
935 struct file *f,
936 struct queue_properties *properties,
937 unsigned int *qid,
938 uint32_t *p_doorbell_offset_in_process);
939int pqm_destroy_queue(struct process_queue_manager *pqm, unsigned int qid);
940int pqm_update_queue(struct process_queue_manager *pqm, unsigned int qid,
941 struct queue_properties *p);
942int pqm_set_cu_mask(struct process_queue_manager *pqm, unsigned int qid,
943 struct queue_properties *p);
944int pqm_set_gws(struct process_queue_manager *pqm, unsigned int qid,
945 void *gws);
946struct kernel_queue *pqm_get_kernel_queue(struct process_queue_manager *pqm,
947 unsigned int qid);
948struct queue *pqm_get_user_queue(struct process_queue_manager *pqm,
949 unsigned int qid);
950int pqm_get_wave_state(struct process_queue_manager *pqm,
951 unsigned int qid,
952 void __user *ctl_stack,
953 u32 *ctl_stack_used_size,
954 u32 *save_area_used_size);
955
956int amdkfd_fence_wait_timeout(unsigned int *fence_addr,
957 unsigned int fence_value,
958 unsigned int timeout_ms);
959
960
961
962#define KFD_FENCE_COMPLETED (100)
963#define KFD_FENCE_INIT (10)
964
965struct packet_manager {
966 struct device_queue_manager *dqm;
967 struct kernel_queue *priv_queue;
968 struct mutex lock;
969 bool allocated;
970 struct kfd_mem_obj *ib_buffer_obj;
971 unsigned int ib_size_bytes;
972 bool is_over_subscription;
973
974 const struct packet_manager_funcs *pmf;
975};
976
977struct packet_manager_funcs {
978
979 int (*map_process)(struct packet_manager *pm, uint32_t *buffer,
980 struct qcm_process_device *qpd);
981 int (*runlist)(struct packet_manager *pm, uint32_t *buffer,
982 uint64_t ib, size_t ib_size_in_dwords, bool chain);
983 int (*set_resources)(struct packet_manager *pm, uint32_t *buffer,
984 struct scheduling_resources *res);
985 int (*map_queues)(struct packet_manager *pm, uint32_t *buffer,
986 struct queue *q, bool is_static);
987 int (*unmap_queues)(struct packet_manager *pm, uint32_t *buffer,
988 enum kfd_queue_type type,
989 enum kfd_unmap_queues_filter mode,
990 uint32_t filter_param, bool reset,
991 unsigned int sdma_engine);
992 int (*query_status)(struct packet_manager *pm, uint32_t *buffer,
993 uint64_t fence_address, uint32_t fence_value);
994 int (*release_mem)(uint64_t gpu_addr, uint32_t *buffer);
995
996
997 int map_process_size;
998 int runlist_size;
999 int set_resources_size;
1000 int map_queues_size;
1001 int unmap_queues_size;
1002 int query_status_size;
1003 int release_mem_size;
1004};
1005
1006extern const struct packet_manager_funcs kfd_vi_pm_funcs;
1007extern const struct packet_manager_funcs kfd_v9_pm_funcs;
1008
1009int pm_init(struct packet_manager *pm, struct device_queue_manager *dqm);
1010void pm_uninit(struct packet_manager *pm, bool hanging);
1011int pm_send_set_resources(struct packet_manager *pm,
1012 struct scheduling_resources *res);
1013int pm_send_runlist(struct packet_manager *pm, struct list_head *dqm_queues);
1014int pm_send_query_status(struct packet_manager *pm, uint64_t fence_address,
1015 uint32_t fence_value);
1016
1017int pm_send_unmap_queue(struct packet_manager *pm, enum kfd_queue_type type,
1018 enum kfd_unmap_queues_filter mode,
1019 uint32_t filter_param, bool reset,
1020 unsigned int sdma_engine);
1021
1022void pm_release_ib(struct packet_manager *pm);
1023
1024
1025unsigned int pm_build_pm4_header(unsigned int opcode, size_t packet_size);
1026
1027uint64_t kfd_get_number_elems(struct kfd_dev *kfd);
1028
1029
1030extern const struct kfd_event_interrupt_class event_interrupt_class_cik;
1031extern const struct kfd_event_interrupt_class event_interrupt_class_v9;
1032
1033extern const struct kfd_device_global_init_class device_global_init_class_cik;
1034
1035void kfd_event_init_process(struct kfd_process *p);
1036void kfd_event_free_process(struct kfd_process *p);
1037int kfd_event_mmap(struct kfd_process *process, struct vm_area_struct *vma);
1038int kfd_wait_on_events(struct kfd_process *p,
1039 uint32_t num_events, void __user *data,
1040 bool all, uint32_t user_timeout_ms,
1041 uint32_t *wait_result);
1042void kfd_signal_event_interrupt(unsigned int pasid, uint32_t partial_id,
1043 uint32_t valid_id_bits);
1044void kfd_signal_iommu_event(struct kfd_dev *dev,
1045 unsigned int pasid, unsigned long address,
1046 bool is_write_requested, bool is_execute_requested);
1047void kfd_signal_hw_exception_event(unsigned int pasid);
1048int kfd_set_event(struct kfd_process *p, uint32_t event_id);
1049int kfd_reset_event(struct kfd_process *p, uint32_t event_id);
1050int kfd_event_page_set(struct kfd_process *p, void *kernel_address,
1051 uint64_t size);
1052int kfd_event_create(struct file *devkfd, struct kfd_process *p,
1053 uint32_t event_type, bool auto_reset, uint32_t node_id,
1054 uint32_t *event_id, uint32_t *event_trigger_data,
1055 uint64_t *event_page_offset, uint32_t *event_slot_index);
1056int kfd_event_destroy(struct kfd_process *p, uint32_t event_id);
1057
1058void kfd_signal_vm_fault_event(struct kfd_dev *dev, unsigned int pasid,
1059 struct kfd_vm_fault_info *info);
1060
1061void kfd_signal_reset_event(struct kfd_dev *dev);
1062
1063void kfd_flush_tlb(struct kfd_process_device *pdd);
1064
1065int dbgdev_wave_reset_wavefronts(struct kfd_dev *dev, struct kfd_process *p);
1066
1067bool kfd_is_locked(void);
1068
1069
1070void kfd_inc_compute_active(struct kfd_dev *dev);
1071void kfd_dec_compute_active(struct kfd_dev *dev);
1072
1073
1074
1075static inline int kfd_devcgroup_check_permission(struct kfd_dev *kfd)
1076{
1077#if defined(CONFIG_CGROUP_DEVICE) || defined(CONFIG_CGROUP_BPF)
1078 struct drm_device *ddev = kfd->ddev;
1079
1080 return devcgroup_check_permission(DEVCG_DEV_CHAR, DRM_MAJOR,
1081 ddev->render->index,
1082 DEVCG_ACC_WRITE | DEVCG_ACC_READ);
1083#else
1084 return 0;
1085#endif
1086}
1087
1088
1089#if defined(CONFIG_DEBUG_FS)
1090
1091void kfd_debugfs_init(void);
1092void kfd_debugfs_fini(void);
1093int kfd_debugfs_mqds_by_process(struct seq_file *m, void *data);
1094int pqm_debugfs_mqds(struct seq_file *m, void *data);
1095int kfd_debugfs_hqds_by_device(struct seq_file *m, void *data);
1096int dqm_debugfs_hqds(struct seq_file *m, void *data);
1097int kfd_debugfs_rls_by_device(struct seq_file *m, void *data);
1098int pm_debugfs_runlist(struct seq_file *m, void *data);
1099
1100int kfd_debugfs_hang_hws(struct kfd_dev *dev);
1101int pm_debugfs_hang_hws(struct packet_manager *pm);
1102int dqm_debugfs_execute_queues(struct device_queue_manager *dqm);
1103
1104#else
1105
1106static inline void kfd_debugfs_init(void) {}
1107static inline void kfd_debugfs_fini(void) {}
1108
1109#endif
1110
1111#endif
1112