1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23#ifndef KFD_PRIV_H_INCLUDED
24#define KFD_PRIV_H_INCLUDED
25
26#include <linux/hashtable.h>
27#include <linux/mmu_notifier.h>
28#include <linux/mutex.h>
29#include <linux/types.h>
30#include <linux/atomic.h>
31#include <linux/workqueue.h>
32#include <linux/spinlock.h>
33#include <linux/kfd_ioctl.h>
34#include <linux/idr.h>
35#include <linux/kfifo.h>
36#include <linux/seq_file.h>
37#include <linux/kref.h>
38#include <linux/sysfs.h>
39#include <linux/device_cgroup.h>
40#include <drm/drm_file.h>
41#include <drm/drm_drv.h>
42#include <drm/drm_device.h>
43#include <drm/drm_ioctl.h>
44#include <kgd_kfd_interface.h>
45#include <linux/swap.h>
46
47#include "amd_shared.h"
48
49#define KFD_MAX_RING_ENTRY_SIZE 8
50
51#define KFD_SYSFS_FILE_MODE 0444
52
53
54#define KFD_GPU_ID_HASH_WIDTH 16
55
56
57
58
59
60
61
62
63
64#define KFD_MMAP_TYPE_SHIFT 62
65#define KFD_MMAP_TYPE_MASK (0x3ULL << KFD_MMAP_TYPE_SHIFT)
66#define KFD_MMAP_TYPE_DOORBELL (0x3ULL << KFD_MMAP_TYPE_SHIFT)
67#define KFD_MMAP_TYPE_EVENTS (0x2ULL << KFD_MMAP_TYPE_SHIFT)
68#define KFD_MMAP_TYPE_RESERVED_MEM (0x1ULL << KFD_MMAP_TYPE_SHIFT)
69#define KFD_MMAP_TYPE_MMIO (0x0ULL << KFD_MMAP_TYPE_SHIFT)
70
71#define KFD_MMAP_GPU_ID_SHIFT 46
72#define KFD_MMAP_GPU_ID_MASK (((1ULL << KFD_GPU_ID_HASH_WIDTH) - 1) \
73 << KFD_MMAP_GPU_ID_SHIFT)
74#define KFD_MMAP_GPU_ID(gpu_id) ((((uint64_t)gpu_id) << KFD_MMAP_GPU_ID_SHIFT)\
75 & KFD_MMAP_GPU_ID_MASK)
76#define KFD_MMAP_GET_GPU_ID(offset) ((offset & KFD_MMAP_GPU_ID_MASK) \
77 >> KFD_MMAP_GPU_ID_SHIFT)
78
79
80
81
82
83
84
85
86#define KFD_CIK_HIQ_PIPE 4
87#define KFD_CIK_HIQ_QUEUE 0
88
89
90#define kfd_alloc_struct(ptr_to_struct) \
91 ((typeof(ptr_to_struct)) kzalloc(sizeof(*ptr_to_struct), GFP_KERNEL))
92
93#define KFD_MAX_NUM_OF_PROCESSES 512
94#define KFD_MAX_NUM_OF_QUEUES_PER_PROCESS 1024
95
96
97
98
99
100
101
102#define KFD_CWSR_TBA_TMA_SIZE (PAGE_SIZE * 2)
103#define KFD_CWSR_TMA_OFFSET PAGE_SIZE
104
105#define KFD_MAX_NUM_OF_QUEUES_PER_DEVICE \
106 (KFD_MAX_NUM_OF_PROCESSES * \
107 KFD_MAX_NUM_OF_QUEUES_PER_PROCESS)
108
109#define KFD_KERNEL_QUEUE_SIZE 2048
110
111#define KFD_UNMAP_LATENCY_MS (4000)
112
113
114
115
116
117
118
119
120
121#define KFD_QUEUE_DOORBELL_MIRROR_OFFSET 512
122
123
124
125
126
127
128extern int max_num_of_queues_per_device;
129
130
131
132extern int sched_policy;
133
134
135
136
137
138extern int hws_max_conc_proc;
139
140extern int cwsr_enable;
141
142
143
144
145
146extern int send_sigterm;
147
148
149
150
151
152extern int debug_largebar;
153
154
155
156
157
158extern int ignore_crat;
159
160
161extern int amdgpu_noretry;
162
163
164extern int halt_if_hws_hang;
165
166
167extern bool hws_gws_support;
168
169
170extern int queue_preemption_timeout_ms;
171
172
173extern bool debug_evictions;
174
175enum cache_policy {
176 cache_policy_coherent,
177 cache_policy_noncoherent
178};
179
180#define KFD_IS_SOC15(chip) ((chip) >= CHIP_VEGA10)
181
182struct kfd_event_interrupt_class {
183 bool (*interrupt_isr)(struct kfd_dev *dev,
184 const uint32_t *ih_ring_entry, uint32_t *patched_ihre,
185 bool *patched_flag);
186 void (*interrupt_wq)(struct kfd_dev *dev,
187 const uint32_t *ih_ring_entry);
188};
189
190struct kfd_device_info {
191 enum amd_asic_type asic_family;
192 const char *asic_name;
193 const struct kfd_event_interrupt_class *event_interrupt_class;
194 unsigned int max_pasid_bits;
195 unsigned int max_no_of_hqd;
196 unsigned int doorbell_size;
197 size_t ih_ring_entry_size;
198 uint8_t num_of_watch_points;
199 uint16_t mqd_size_aligned;
200 bool supports_cwsr;
201 bool needs_iommu_device;
202 bool needs_pci_atomics;
203 unsigned int num_sdma_engines;
204 unsigned int num_xgmi_sdma_engines;
205 unsigned int num_sdma_queues_per_engine;
206};
207
208struct kfd_mem_obj {
209 uint32_t range_start;
210 uint32_t range_end;
211 uint64_t gpu_addr;
212 uint32_t *cpu_ptr;
213 void *gtt_mem;
214};
215
216struct kfd_vmid_info {
217 uint32_t first_vmid_kfd;
218 uint32_t last_vmid_kfd;
219 uint32_t vmid_num_kfd;
220};
221
222struct kfd_dev {
223 struct kgd_dev *kgd;
224
225 const struct kfd_device_info *device_info;
226 struct pci_dev *pdev;
227 struct drm_device *ddev;
228
229 unsigned int id;
230
231 phys_addr_t doorbell_base;
232
233
234
235 size_t doorbell_base_dw_offset;
236
237
238
239
240 u32 __iomem *doorbell_kernel_ptr;
241
242
243
244 struct kgd2kfd_shared_resources shared_resources;
245 struct kfd_vmid_info vm_info;
246
247 const struct kfd2kgd_calls *kfd2kgd;
248 struct mutex doorbell_mutex;
249 DECLARE_BITMAP(doorbell_available_index,
250 KFD_MAX_NUM_OF_QUEUES_PER_PROCESS);
251
252 void *gtt_mem;
253 uint64_t gtt_start_gpu_addr;
254 void *gtt_start_cpu_ptr;
255 void *gtt_sa_bitmap;
256 struct mutex gtt_sa_lock;
257 unsigned int gtt_sa_chunk_size;
258 unsigned int gtt_sa_num_of_chunks;
259
260
261 struct kfifo ih_fifo;
262 struct workqueue_struct *ih_wq;
263 struct work_struct interrupt_work;
264 spinlock_t interrupt_lock;
265
266
267 struct device_queue_manager *dqm;
268
269 bool init_complete;
270
271
272
273
274 bool interrupts_active;
275
276
277 struct kfd_dbgmgr *dbgmgr;
278
279
280 uint16_t mec_fw_version;
281 uint16_t mec2_fw_version;
282 uint16_t sdma_fw_version;
283
284
285 unsigned int max_proc_per_quantum;
286
287
288 bool cwsr_enabled;
289 const void *cwsr_isa;
290 unsigned int cwsr_isa_size;
291
292
293 uint64_t hive_id;
294
295
296 uint64_t unique_id;
297
298 bool pci_atomic_requested;
299
300
301 bool use_iommu_v2;
302
303
304 atomic_t sram_ecc_flag;
305
306
307 atomic_t compute_profile;
308
309
310 void *gws;
311
312
313 struct list_head smi_clients;
314 spinlock_t smi_lock;
315
316 uint32_t reset_seq_num;
317
318 struct ida doorbell_ida;
319 unsigned int max_doorbell_slices;
320
321 int noretry;
322};
323
324enum kfd_mempool {
325 KFD_MEMPOOL_SYSTEM_CACHEABLE = 1,
326 KFD_MEMPOOL_SYSTEM_WRITECOMBINE = 2,
327 KFD_MEMPOOL_FRAMEBUFFER = 3,
328};
329
330
331int kfd_chardev_init(void);
332void kfd_chardev_exit(void);
333struct device *kfd_chardev(void);
334
335
336
337
338
339
340
341
342
343
344
345
346
347enum kfd_unmap_queues_filter {
348 KFD_UNMAP_QUEUES_FILTER_SINGLE_QUEUE,
349 KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES,
350 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES,
351 KFD_UNMAP_QUEUES_FILTER_BY_PASID
352};
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367enum kfd_queue_type {
368 KFD_QUEUE_TYPE_COMPUTE,
369 KFD_QUEUE_TYPE_SDMA,
370 KFD_QUEUE_TYPE_HIQ,
371 KFD_QUEUE_TYPE_DIQ,
372 KFD_QUEUE_TYPE_SDMA_XGMI
373};
374
375enum kfd_queue_format {
376 KFD_QUEUE_FORMAT_PM4,
377 KFD_QUEUE_FORMAT_AQL
378};
379
380enum KFD_QUEUE_PRIORITY {
381 KFD_QUEUE_PRIORITY_MINIMUM = 0,
382 KFD_QUEUE_PRIORITY_MAXIMUM = 15
383};
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437struct queue_properties {
438 enum kfd_queue_type type;
439 enum kfd_queue_format format;
440 unsigned int queue_id;
441 uint64_t queue_address;
442 uint64_t queue_size;
443 uint32_t priority;
444 uint32_t queue_percent;
445 uint32_t *read_ptr;
446 uint32_t *write_ptr;
447 void __iomem *doorbell_ptr;
448 uint32_t doorbell_off;
449 bool is_interop;
450 bool is_evicted;
451 bool is_active;
452 bool is_gws;
453
454 unsigned int vmid;
455
456 uint32_t sdma_engine_id;
457 uint32_t sdma_queue_id;
458 uint32_t sdma_vm_addr;
459
460 uint64_t eop_ring_buffer_address;
461 uint32_t eop_ring_buffer_size;
462 uint64_t ctx_save_restore_area_address;
463 uint32_t ctx_save_restore_area_size;
464 uint32_t ctl_stack_size;
465 uint64_t tba_addr;
466 uint64_t tma_addr;
467
468 uint32_t cu_mask_count;
469 uint32_t *cu_mask;
470};
471
472#define QUEUE_IS_ACTIVE(q) ((q).queue_size > 0 && \
473 (q).queue_address != 0 && \
474 (q).queue_percent > 0 && \
475 !(q).is_evicted)
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510struct queue {
511 struct list_head list;
512 void *mqd;
513 struct kfd_mem_obj *mqd_mem_obj;
514 uint64_t gart_mqd_addr;
515 struct queue_properties properties;
516
517 uint32_t mec;
518 uint32_t pipe;
519 uint32_t queue;
520
521 unsigned int sdma_id;
522 unsigned int doorbell_id;
523
524 struct kfd_process *process;
525 struct kfd_dev *device;
526 void *gws;
527
528
529 struct kobject kobj;
530};
531
532enum KFD_MQD_TYPE {
533 KFD_MQD_TYPE_HIQ = 0,
534 KFD_MQD_TYPE_CP,
535 KFD_MQD_TYPE_SDMA,
536 KFD_MQD_TYPE_DIQ,
537 KFD_MQD_TYPE_MAX
538};
539
540enum KFD_PIPE_PRIORITY {
541 KFD_PIPE_PRIORITY_CS_LOW = 0,
542 KFD_PIPE_PRIORITY_CS_MEDIUM,
543 KFD_PIPE_PRIORITY_CS_HIGH
544};
545
546struct scheduling_resources {
547 unsigned int vmid_mask;
548 enum kfd_queue_type type;
549 uint64_t queue_mask;
550 uint64_t gws_mask;
551 uint32_t oac_mask;
552 uint32_t gds_heap_base;
553 uint32_t gds_heap_size;
554};
555
556struct process_queue_manager {
557
558 struct kfd_process *process;
559 struct list_head queues;
560 unsigned long *queue_slot_bitmap;
561};
562
563struct qcm_process_device {
564
565 struct device_queue_manager *dqm;
566 struct process_queue_manager *pqm;
567
568 struct list_head queues_list;
569 struct list_head priv_queue_list;
570
571 unsigned int queue_count;
572 unsigned int vmid;
573 bool is_debug;
574 unsigned int evicted;
575
576
577
578
579 bool reset_wavefronts;
580
581
582
583
584
585
586
587 bool mapped_gws_queue;
588
589
590 uint64_t gds_context_area;
591
592 uint64_t page_table_base;
593 uint32_t sh_mem_config;
594 uint32_t sh_mem_bases;
595 uint32_t sh_mem_ape1_base;
596 uint32_t sh_mem_ape1_limit;
597 uint32_t gds_size;
598 uint32_t num_gws;
599 uint32_t num_oac;
600 uint32_t sh_hidden_private_base;
601
602
603 void *cwsr_kaddr;
604 uint64_t cwsr_base;
605 uint64_t tba_addr;
606 uint64_t tma_addr;
607
608
609 uint64_t ib_base;
610 void *ib_kaddr;
611
612
613 unsigned long *doorbell_bitmap;
614};
615
616
617
618
619#define PROCESS_RESTORE_TIME_MS 100
620
621#define PROCESS_BACK_OFF_TIME_MS 100
622
623#define PROCESS_ACTIVE_TIME_MS 10
624
625
626
627
628#define MAKE_HANDLE(gpu_id, idr_handle) \
629 (((uint64_t)(gpu_id) << 32) + idr_handle)
630#define GET_GPU_ID(handle) (handle >> 32)
631#define GET_IDR_HANDLE(handle) (handle & 0xFFFFFFFF)
632
633enum kfd_pdd_bound {
634 PDD_UNBOUND = 0,
635 PDD_BOUND,
636 PDD_BOUND_SUSPENDED,
637};
638
639#define MAX_SYSFS_FILENAME_LEN 15
640
641
642
643
644
645
646#define SDMA_ACTIVITY_DIVISOR 100
647
648
649struct kfd_process_device {
650
651
652
653
654 struct list_head per_device_list;
655
656
657 struct kfd_dev *dev;
658
659
660 struct kfd_process *process;
661
662
663 struct qcm_process_device qpd;
664
665
666 uint64_t lds_base;
667 uint64_t lds_limit;
668 uint64_t gpuvm_base;
669 uint64_t gpuvm_limit;
670 uint64_t scratch_base;
671 uint64_t scratch_limit;
672
673
674 struct file *drm_file;
675 void *vm;
676
677
678 struct idr alloc_idr;
679
680
681
682
683
684
685 bool already_dequeued;
686 bool runtime_inuse;
687
688
689 enum kfd_pdd_bound bound;
690
691
692 uint64_t vram_usage;
693 struct attribute attr_vram;
694 char vram_filename[MAX_SYSFS_FILENAME_LEN];
695
696
697 uint64_t sdma_past_activity_counter;
698 struct attribute attr_sdma;
699 char sdma_filename[MAX_SYSFS_FILENAME_LEN];
700
701
702 uint64_t last_evict_timestamp;
703 atomic64_t evict_duration_counter;
704 struct attribute attr_evict;
705
706 struct kobject *kobj_stats;
707 unsigned int doorbell_index;
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732 struct attribute attr_cu_occupancy;
733};
734
735#define qpd_to_pdd(x) container_of(x, struct kfd_process_device, qpd)
736
737
738struct kfd_process {
739
740
741
742
743 struct hlist_node kfd_processes;
744
745
746
747
748
749
750 void *mm;
751
752 struct kref ref;
753 struct work_struct release_work;
754
755 struct mutex mutex;
756
757
758
759
760
761
762
763
764 struct task_struct *lead_thread;
765
766
767 struct mmu_notifier mmu_notifier;
768
769 u32 pasid;
770
771
772
773
774
775 struct list_head per_device_data;
776
777 struct process_queue_manager pqm;
778
779
780 bool is_32bit_user_mode;
781
782
783 struct mutex event_mutex;
784
785 struct idr event_idr;
786
787 struct kfd_signal_page *signal_page;
788 size_t signal_mapped_size;
789 size_t signal_event_count;
790 bool signal_event_limit_reached;
791
792
793 void *kgd_process_info;
794
795
796
797
798 struct dma_fence *ef;
799
800
801 struct delayed_work eviction_work;
802 struct delayed_work restore_work;
803
804 unsigned int last_eviction_seqno;
805
806
807
808 unsigned long last_restore_timestamp;
809
810
811 struct kobject *kobj;
812 struct kobject *kobj_queues;
813 struct attribute attr_pasid;
814};
815
816#define KFD_PROCESS_TABLE_SIZE 5
817extern DECLARE_HASHTABLE(kfd_processes_table, KFD_PROCESS_TABLE_SIZE);
818extern struct srcu_struct kfd_processes_srcu;
819
820
821
822
823
824
825
826
827
828
829typedef int amdkfd_ioctl_t(struct file *filep, struct kfd_process *p,
830 void *data);
831
832struct amdkfd_ioctl_desc {
833 unsigned int cmd;
834 int flags;
835 amdkfd_ioctl_t *func;
836 unsigned int cmd_drv;
837 const char *name;
838};
839bool kfd_dev_is_large_bar(struct kfd_dev *dev);
840
841int kfd_process_create_wq(void);
842void kfd_process_destroy_wq(void);
843struct kfd_process *kfd_create_process(struct file *filep);
844struct kfd_process *kfd_get_process(const struct task_struct *);
845struct kfd_process *kfd_lookup_process_by_pasid(u32 pasid);
846struct kfd_process *kfd_lookup_process_by_mm(const struct mm_struct *mm);
847void kfd_unref_process(struct kfd_process *p);
848int kfd_process_evict_queues(struct kfd_process *p);
849int kfd_process_restore_queues(struct kfd_process *p);
850void kfd_suspend_all_processes(void);
851int kfd_resume_all_processes(void);
852
853int kfd_process_device_init_vm(struct kfd_process_device *pdd,
854 struct file *drm_file);
855struct kfd_process_device *kfd_bind_process_to_device(struct kfd_dev *dev,
856 struct kfd_process *p);
857struct kfd_process_device *kfd_get_process_device_data(struct kfd_dev *dev,
858 struct kfd_process *p);
859struct kfd_process_device *kfd_create_process_device_data(struct kfd_dev *dev,
860 struct kfd_process *p);
861
862int kfd_reserved_mem_mmap(struct kfd_dev *dev, struct kfd_process *process,
863 struct vm_area_struct *vma);
864
865
866int kfd_process_device_create_obj_handle(struct kfd_process_device *pdd,
867 void *mem);
868void *kfd_process_device_translate_handle(struct kfd_process_device *p,
869 int handle);
870void kfd_process_device_remove_obj_handle(struct kfd_process_device *pdd,
871 int handle);
872
873
874struct kfd_process_device *kfd_get_first_process_device_data(
875 struct kfd_process *p);
876struct kfd_process_device *kfd_get_next_process_device_data(
877 struct kfd_process *p,
878 struct kfd_process_device *pdd);
879bool kfd_has_process_device_data(struct kfd_process *p);
880
881
882int kfd_pasid_init(void);
883void kfd_pasid_exit(void);
884bool kfd_set_pasid_limit(unsigned int new_limit);
885unsigned int kfd_get_pasid_limit(void);
886u32 kfd_pasid_alloc(void);
887void kfd_pasid_free(u32 pasid);
888
889
890size_t kfd_doorbell_process_slice(struct kfd_dev *kfd);
891int kfd_doorbell_init(struct kfd_dev *kfd);
892void kfd_doorbell_fini(struct kfd_dev *kfd);
893int kfd_doorbell_mmap(struct kfd_dev *dev, struct kfd_process *process,
894 struct vm_area_struct *vma);
895void __iomem *kfd_get_kernel_doorbell(struct kfd_dev *kfd,
896 unsigned int *doorbell_off);
897void kfd_release_kernel_doorbell(struct kfd_dev *kfd, u32 __iomem *db_addr);
898u32 read_kernel_doorbell(u32 __iomem *db);
899void write_kernel_doorbell(void __iomem *db, u32 value);
900void write_kernel_doorbell64(void __iomem *db, u64 value);
901unsigned int kfd_get_doorbell_dw_offset_in_bar(struct kfd_dev *kfd,
902 struct kfd_process_device *pdd,
903 unsigned int doorbell_id);
904phys_addr_t kfd_get_process_doorbells(struct kfd_process_device *pdd);
905int kfd_alloc_process_doorbells(struct kfd_dev *kfd,
906 unsigned int *doorbell_index);
907void kfd_free_process_doorbells(struct kfd_dev *kfd,
908 unsigned int doorbell_index);
909
910
911int kfd_gtt_sa_allocate(struct kfd_dev *kfd, unsigned int size,
912 struct kfd_mem_obj **mem_obj);
913
914int kfd_gtt_sa_free(struct kfd_dev *kfd, struct kfd_mem_obj *mem_obj);
915
916extern struct device *kfd_device;
917
918
919void kfd_procfs_init(void);
920void kfd_procfs_shutdown(void);
921int kfd_procfs_add_queue(struct queue *q);
922void kfd_procfs_del_queue(struct queue *q);
923
924
925int kfd_topology_init(void);
926void kfd_topology_shutdown(void);
927int kfd_topology_add_device(struct kfd_dev *gpu);
928int kfd_topology_remove_device(struct kfd_dev *gpu);
929struct kfd_topology_device *kfd_topology_device_by_proximity_domain(
930 uint32_t proximity_domain);
931struct kfd_topology_device *kfd_topology_device_by_id(uint32_t gpu_id);
932struct kfd_dev *kfd_device_by_id(uint32_t gpu_id);
933struct kfd_dev *kfd_device_by_pci_dev(const struct pci_dev *pdev);
934struct kfd_dev *kfd_device_by_kgd(const struct kgd_dev *kgd);
935int kfd_topology_enum_kfd_devices(uint8_t idx, struct kfd_dev **kdev);
936int kfd_numa_node_to_apic_id(int numa_node_id);
937void kfd_double_confirm_iommu_support(struct kfd_dev *gpu);
938
939
940int kfd_interrupt_init(struct kfd_dev *dev);
941void kfd_interrupt_exit(struct kfd_dev *dev);
942bool enqueue_ih_ring_entry(struct kfd_dev *kfd, const void *ih_ring_entry);
943bool interrupt_is_wanted(struct kfd_dev *dev,
944 const uint32_t *ih_ring_entry,
945 uint32_t *patched_ihre, bool *flag);
946
947
948int kfd_init_apertures(struct kfd_process *process);
949
950
951int init_queue(struct queue **q, const struct queue_properties *properties);
952void uninit_queue(struct queue *q);
953void print_queue_properties(struct queue_properties *q);
954void print_queue(struct queue *q);
955
956struct mqd_manager *mqd_manager_init_cik(enum KFD_MQD_TYPE type,
957 struct kfd_dev *dev);
958struct mqd_manager *mqd_manager_init_cik_hawaii(enum KFD_MQD_TYPE type,
959 struct kfd_dev *dev);
960struct mqd_manager *mqd_manager_init_vi(enum KFD_MQD_TYPE type,
961 struct kfd_dev *dev);
962struct mqd_manager *mqd_manager_init_vi_tonga(enum KFD_MQD_TYPE type,
963 struct kfd_dev *dev);
964struct mqd_manager *mqd_manager_init_v9(enum KFD_MQD_TYPE type,
965 struct kfd_dev *dev);
966struct mqd_manager *mqd_manager_init_v10(enum KFD_MQD_TYPE type,
967 struct kfd_dev *dev);
968struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev);
969void device_queue_manager_uninit(struct device_queue_manager *dqm);
970struct kernel_queue *kernel_queue_init(struct kfd_dev *dev,
971 enum kfd_queue_type type);
972void kernel_queue_uninit(struct kernel_queue *kq, bool hanging);
973int kfd_process_vm_fault(struct device_queue_manager *dqm, u32 pasid);
974
975
976struct process_queue_node {
977 struct queue *q;
978 struct kernel_queue *kq;
979 struct list_head process_queue_list;
980};
981
982void kfd_process_dequeue_from_device(struct kfd_process_device *pdd);
983void kfd_process_dequeue_from_all_devices(struct kfd_process *p);
984int pqm_init(struct process_queue_manager *pqm, struct kfd_process *p);
985void pqm_uninit(struct process_queue_manager *pqm);
986int pqm_create_queue(struct process_queue_manager *pqm,
987 struct kfd_dev *dev,
988 struct file *f,
989 struct queue_properties *properties,
990 unsigned int *qid,
991 uint32_t *p_doorbell_offset_in_process);
992int pqm_destroy_queue(struct process_queue_manager *pqm, unsigned int qid);
993int pqm_update_queue(struct process_queue_manager *pqm, unsigned int qid,
994 struct queue_properties *p);
995int pqm_set_cu_mask(struct process_queue_manager *pqm, unsigned int qid,
996 struct queue_properties *p);
997int pqm_set_gws(struct process_queue_manager *pqm, unsigned int qid,
998 void *gws);
999struct kernel_queue *pqm_get_kernel_queue(struct process_queue_manager *pqm,
1000 unsigned int qid);
1001struct queue *pqm_get_user_queue(struct process_queue_manager *pqm,
1002 unsigned int qid);
1003int pqm_get_wave_state(struct process_queue_manager *pqm,
1004 unsigned int qid,
1005 void __user *ctl_stack,
1006 u32 *ctl_stack_used_size,
1007 u32 *save_area_used_size);
1008
1009int amdkfd_fence_wait_timeout(unsigned int *fence_addr,
1010 unsigned int fence_value,
1011 unsigned int timeout_ms);
1012
1013
1014
1015#define KFD_FENCE_COMPLETED (100)
1016#define KFD_FENCE_INIT (10)
1017
1018struct packet_manager {
1019 struct device_queue_manager *dqm;
1020 struct kernel_queue *priv_queue;
1021 struct mutex lock;
1022 bool allocated;
1023 struct kfd_mem_obj *ib_buffer_obj;
1024 unsigned int ib_size_bytes;
1025 bool is_over_subscription;
1026
1027 const struct packet_manager_funcs *pmf;
1028};
1029
1030struct packet_manager_funcs {
1031
1032 int (*map_process)(struct packet_manager *pm, uint32_t *buffer,
1033 struct qcm_process_device *qpd);
1034 int (*runlist)(struct packet_manager *pm, uint32_t *buffer,
1035 uint64_t ib, size_t ib_size_in_dwords, bool chain);
1036 int (*set_resources)(struct packet_manager *pm, uint32_t *buffer,
1037 struct scheduling_resources *res);
1038 int (*map_queues)(struct packet_manager *pm, uint32_t *buffer,
1039 struct queue *q, bool is_static);
1040 int (*unmap_queues)(struct packet_manager *pm, uint32_t *buffer,
1041 enum kfd_queue_type type,
1042 enum kfd_unmap_queues_filter mode,
1043 uint32_t filter_param, bool reset,
1044 unsigned int sdma_engine);
1045 int (*query_status)(struct packet_manager *pm, uint32_t *buffer,
1046 uint64_t fence_address, uint32_t fence_value);
1047 int (*release_mem)(uint64_t gpu_addr, uint32_t *buffer);
1048
1049
1050 int map_process_size;
1051 int runlist_size;
1052 int set_resources_size;
1053 int map_queues_size;
1054 int unmap_queues_size;
1055 int query_status_size;
1056 int release_mem_size;
1057};
1058
1059extern const struct packet_manager_funcs kfd_vi_pm_funcs;
1060extern const struct packet_manager_funcs kfd_v9_pm_funcs;
1061
1062int pm_init(struct packet_manager *pm, struct device_queue_manager *dqm);
1063void pm_uninit(struct packet_manager *pm, bool hanging);
1064int pm_send_set_resources(struct packet_manager *pm,
1065 struct scheduling_resources *res);
1066int pm_send_runlist(struct packet_manager *pm, struct list_head *dqm_queues);
1067int pm_send_query_status(struct packet_manager *pm, uint64_t fence_address,
1068 uint32_t fence_value);
1069
1070int pm_send_unmap_queue(struct packet_manager *pm, enum kfd_queue_type type,
1071 enum kfd_unmap_queues_filter mode,
1072 uint32_t filter_param, bool reset,
1073 unsigned int sdma_engine);
1074
1075void pm_release_ib(struct packet_manager *pm);
1076
1077
1078unsigned int pm_build_pm4_header(unsigned int opcode, size_t packet_size);
1079
1080uint64_t kfd_get_number_elems(struct kfd_dev *kfd);
1081
1082
1083extern const struct kfd_event_interrupt_class event_interrupt_class_cik;
1084extern const struct kfd_event_interrupt_class event_interrupt_class_v9;
1085
1086extern const struct kfd_device_global_init_class device_global_init_class_cik;
1087
1088void kfd_event_init_process(struct kfd_process *p);
1089void kfd_event_free_process(struct kfd_process *p);
1090int kfd_event_mmap(struct kfd_process *process, struct vm_area_struct *vma);
1091int kfd_wait_on_events(struct kfd_process *p,
1092 uint32_t num_events, void __user *data,
1093 bool all, uint32_t user_timeout_ms,
1094 uint32_t *wait_result);
1095void kfd_signal_event_interrupt(u32 pasid, uint32_t partial_id,
1096 uint32_t valid_id_bits);
1097void kfd_signal_iommu_event(struct kfd_dev *dev,
1098 u32 pasid, unsigned long address,
1099 bool is_write_requested, bool is_execute_requested);
1100void kfd_signal_hw_exception_event(u32 pasid);
1101int kfd_set_event(struct kfd_process *p, uint32_t event_id);
1102int kfd_reset_event(struct kfd_process *p, uint32_t event_id);
1103int kfd_event_page_set(struct kfd_process *p, void *kernel_address,
1104 uint64_t size);
1105int kfd_event_create(struct file *devkfd, struct kfd_process *p,
1106 uint32_t event_type, bool auto_reset, uint32_t node_id,
1107 uint32_t *event_id, uint32_t *event_trigger_data,
1108 uint64_t *event_page_offset, uint32_t *event_slot_index);
1109int kfd_event_destroy(struct kfd_process *p, uint32_t event_id);
1110
1111void kfd_signal_vm_fault_event(struct kfd_dev *dev, u32 pasid,
1112 struct kfd_vm_fault_info *info);
1113
1114void kfd_signal_reset_event(struct kfd_dev *dev);
1115
1116void kfd_flush_tlb(struct kfd_process_device *pdd);
1117
1118int dbgdev_wave_reset_wavefronts(struct kfd_dev *dev, struct kfd_process *p);
1119
1120bool kfd_is_locked(void);
1121
1122
1123void kfd_inc_compute_active(struct kfd_dev *dev);
1124void kfd_dec_compute_active(struct kfd_dev *dev);
1125
1126
1127
1128static inline int kfd_devcgroup_check_permission(struct kfd_dev *kfd)
1129{
1130#if defined(CONFIG_CGROUP_DEVICE) || defined(CONFIG_CGROUP_BPF)
1131 struct drm_device *ddev = kfd->ddev;
1132
1133 return devcgroup_check_permission(DEVCG_DEV_CHAR, DRM_MAJOR,
1134 ddev->render->index,
1135 DEVCG_ACC_WRITE | DEVCG_ACC_READ);
1136#else
1137 return 0;
1138#endif
1139}
1140
1141
1142#if defined(CONFIG_DEBUG_FS)
1143
1144void kfd_debugfs_init(void);
1145void kfd_debugfs_fini(void);
1146int kfd_debugfs_mqds_by_process(struct seq_file *m, void *data);
1147int pqm_debugfs_mqds(struct seq_file *m, void *data);
1148int kfd_debugfs_hqds_by_device(struct seq_file *m, void *data);
1149int dqm_debugfs_hqds(struct seq_file *m, void *data);
1150int kfd_debugfs_rls_by_device(struct seq_file *m, void *data);
1151int pm_debugfs_runlist(struct seq_file *m, void *data);
1152
1153int kfd_debugfs_hang_hws(struct kfd_dev *dev);
1154int pm_debugfs_hang_hws(struct packet_manager *pm);
1155int dqm_debugfs_execute_queues(struct device_queue_manager *dqm);
1156
1157#else
1158
1159static inline void kfd_debugfs_init(void) {}
1160static inline void kfd_debugfs_fini(void) {}
1161
1162#endif
1163
1164#endif
1165