1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23#ifndef KFD_PRIV_H_INCLUDED
24#define KFD_PRIV_H_INCLUDED
25
26#include <linux/hashtable.h>
27#include <linux/mmu_notifier.h>
28#include <linux/mutex.h>
29#include <linux/types.h>
30#include <linux/atomic.h>
31#include <linux/workqueue.h>
32#include <linux/spinlock.h>
33#include <linux/kfd_ioctl.h>
34#include <linux/idr.h>
35#include <linux/kfifo.h>
36#include <linux/seq_file.h>
37#include <linux/kref.h>
38#include <linux/sysfs.h>
39#include <linux/device_cgroup.h>
40#include <drm/drm_file.h>
41#include <drm/drm_drv.h>
42#include <drm/drm_device.h>
43#include <drm/drm_ioctl.h>
44#include <kgd_kfd_interface.h>
45#include <linux/swap.h>
46
47#include "amd_shared.h"
48#include "amdgpu.h"
49
50#define KFD_MAX_RING_ENTRY_SIZE 8
51
52#define KFD_SYSFS_FILE_MODE 0444
53
54
55#define KFD_GPU_ID_HASH_WIDTH 16
56
57
58
59
60
61
62
63
64
65#define KFD_MMAP_TYPE_SHIFT 62
66#define KFD_MMAP_TYPE_MASK (0x3ULL << KFD_MMAP_TYPE_SHIFT)
67#define KFD_MMAP_TYPE_DOORBELL (0x3ULL << KFD_MMAP_TYPE_SHIFT)
68#define KFD_MMAP_TYPE_EVENTS (0x2ULL << KFD_MMAP_TYPE_SHIFT)
69#define KFD_MMAP_TYPE_RESERVED_MEM (0x1ULL << KFD_MMAP_TYPE_SHIFT)
70#define KFD_MMAP_TYPE_MMIO (0x0ULL << KFD_MMAP_TYPE_SHIFT)
71
72#define KFD_MMAP_GPU_ID_SHIFT 46
73#define KFD_MMAP_GPU_ID_MASK (((1ULL << KFD_GPU_ID_HASH_WIDTH) - 1) \
74 << KFD_MMAP_GPU_ID_SHIFT)
75#define KFD_MMAP_GPU_ID(gpu_id) ((((uint64_t)gpu_id) << KFD_MMAP_GPU_ID_SHIFT)\
76 & KFD_MMAP_GPU_ID_MASK)
77#define KFD_MMAP_GET_GPU_ID(offset) ((offset & KFD_MMAP_GPU_ID_MASK) \
78 >> KFD_MMAP_GPU_ID_SHIFT)
79
80
81
82
83
84
85
86
87#define KFD_CIK_HIQ_PIPE 4
88#define KFD_CIK_HIQ_QUEUE 0
89
90
91#define kfd_alloc_struct(ptr_to_struct) \
92 ((typeof(ptr_to_struct)) kzalloc(sizeof(*ptr_to_struct), GFP_KERNEL))
93
94#define KFD_MAX_NUM_OF_PROCESSES 512
95#define KFD_MAX_NUM_OF_QUEUES_PER_PROCESS 1024
96
97
98
99
100
101
102
103#define KFD_CWSR_TBA_TMA_SIZE (PAGE_SIZE * 2)
104#define KFD_CWSR_TMA_OFFSET PAGE_SIZE
105
106#define KFD_MAX_NUM_OF_QUEUES_PER_DEVICE \
107 (KFD_MAX_NUM_OF_PROCESSES * \
108 KFD_MAX_NUM_OF_QUEUES_PER_PROCESS)
109
110#define KFD_KERNEL_QUEUE_SIZE 2048
111
112#define KFD_UNMAP_LATENCY_MS (4000)
113
114
115
116
117
118
119
120
121
122#define KFD_QUEUE_DOORBELL_MIRROR_OFFSET 512
123
124
125
126
127
128
129extern int max_num_of_queues_per_device;
130
131
132
133extern int sched_policy;
134
135
136
137
138
139extern int hws_max_conc_proc;
140
141extern int cwsr_enable;
142
143
144
145
146
147extern int send_sigterm;
148
149
150
151
152
153extern int debug_largebar;
154
155
156
157
158
159extern int ignore_crat;
160
161
162extern int amdgpu_noretry;
163
164
165extern int halt_if_hws_hang;
166
167
168extern bool hws_gws_support;
169
170
171extern int queue_preemption_timeout_ms;
172
173
174
175
176extern int amdgpu_no_queue_eviction_on_vm_fault;
177
178
179extern bool debug_evictions;
180
181enum cache_policy {
182 cache_policy_coherent,
183 cache_policy_noncoherent
184};
185
186#define KFD_IS_SOC15(chip) ((chip) >= CHIP_VEGA10)
187
188struct kfd_event_interrupt_class {
189 bool (*interrupt_isr)(struct kfd_dev *dev,
190 const uint32_t *ih_ring_entry, uint32_t *patched_ihre,
191 bool *patched_flag);
192 void (*interrupt_wq)(struct kfd_dev *dev,
193 const uint32_t *ih_ring_entry);
194};
195
196struct kfd_device_info {
197 enum amd_asic_type asic_family;
198 const char *asic_name;
199 uint32_t gfx_target_version;
200 const struct kfd_event_interrupt_class *event_interrupt_class;
201 unsigned int max_pasid_bits;
202 unsigned int max_no_of_hqd;
203 unsigned int doorbell_size;
204 size_t ih_ring_entry_size;
205 uint8_t num_of_watch_points;
206 uint16_t mqd_size_aligned;
207 bool supports_cwsr;
208 bool needs_iommu_device;
209 bool needs_pci_atomics;
210 uint32_t no_atomic_fw_version;
211 unsigned int num_sdma_engines;
212 unsigned int num_xgmi_sdma_engines;
213 unsigned int num_sdma_queues_per_engine;
214};
215
216struct kfd_mem_obj {
217 uint32_t range_start;
218 uint32_t range_end;
219 uint64_t gpu_addr;
220 uint32_t *cpu_ptr;
221 void *gtt_mem;
222};
223
224struct kfd_vmid_info {
225 uint32_t first_vmid_kfd;
226 uint32_t last_vmid_kfd;
227 uint32_t vmid_num_kfd;
228};
229
230struct kfd_dev {
231 struct kgd_dev *kgd;
232
233 const struct kfd_device_info *device_info;
234 struct pci_dev *pdev;
235 struct drm_device *ddev;
236
237 unsigned int id;
238
239 phys_addr_t doorbell_base;
240
241
242
243 size_t doorbell_base_dw_offset;
244
245
246
247
248 u32 __iomem *doorbell_kernel_ptr;
249
250
251
252 struct kgd2kfd_shared_resources shared_resources;
253 struct kfd_vmid_info vm_info;
254
255 const struct kfd2kgd_calls *kfd2kgd;
256 struct mutex doorbell_mutex;
257 DECLARE_BITMAP(doorbell_available_index,
258 KFD_MAX_NUM_OF_QUEUES_PER_PROCESS);
259
260 void *gtt_mem;
261 uint64_t gtt_start_gpu_addr;
262 void *gtt_start_cpu_ptr;
263 void *gtt_sa_bitmap;
264 struct mutex gtt_sa_lock;
265 unsigned int gtt_sa_chunk_size;
266 unsigned int gtt_sa_num_of_chunks;
267
268
269 struct kfifo ih_fifo;
270 struct workqueue_struct *ih_wq;
271 struct work_struct interrupt_work;
272 spinlock_t interrupt_lock;
273
274
275 struct device_queue_manager *dqm;
276
277 bool init_complete;
278
279
280
281
282 bool interrupts_active;
283
284
285 struct kfd_dbgmgr *dbgmgr;
286
287
288 uint16_t mec_fw_version;
289 uint16_t mec2_fw_version;
290 uint16_t sdma_fw_version;
291
292
293 unsigned int max_proc_per_quantum;
294
295
296 bool cwsr_enabled;
297 const void *cwsr_isa;
298 unsigned int cwsr_isa_size;
299
300
301 uint64_t hive_id;
302
303 bool pci_atomic_requested;
304
305
306 bool use_iommu_v2;
307
308
309 atomic_t sram_ecc_flag;
310
311
312 atomic_t compute_profile;
313
314
315 void *gws;
316
317
318 struct list_head smi_clients;
319 spinlock_t smi_lock;
320
321 uint32_t reset_seq_num;
322
323 struct ida doorbell_ida;
324 unsigned int max_doorbell_slices;
325
326 int noretry;
327
328
329 struct dev_pagemap pgmap;
330};
331
332enum kfd_mempool {
333 KFD_MEMPOOL_SYSTEM_CACHEABLE = 1,
334 KFD_MEMPOOL_SYSTEM_WRITECOMBINE = 2,
335 KFD_MEMPOOL_FRAMEBUFFER = 3,
336};
337
338
339int kfd_chardev_init(void);
340void kfd_chardev_exit(void);
341struct device *kfd_chardev(void);
342
343
344
345
346
347
348
349
350
351
352
353
354
355enum kfd_unmap_queues_filter {
356 KFD_UNMAP_QUEUES_FILTER_SINGLE_QUEUE,
357 KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES,
358 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES,
359 KFD_UNMAP_QUEUES_FILTER_BY_PASID
360};
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375enum kfd_queue_type {
376 KFD_QUEUE_TYPE_COMPUTE,
377 KFD_QUEUE_TYPE_SDMA,
378 KFD_QUEUE_TYPE_HIQ,
379 KFD_QUEUE_TYPE_DIQ,
380 KFD_QUEUE_TYPE_SDMA_XGMI
381};
382
383enum kfd_queue_format {
384 KFD_QUEUE_FORMAT_PM4,
385 KFD_QUEUE_FORMAT_AQL
386};
387
388enum KFD_QUEUE_PRIORITY {
389 KFD_QUEUE_PRIORITY_MINIMUM = 0,
390 KFD_QUEUE_PRIORITY_MAXIMUM = 15
391};
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445struct queue_properties {
446 enum kfd_queue_type type;
447 enum kfd_queue_format format;
448 unsigned int queue_id;
449 uint64_t queue_address;
450 uint64_t queue_size;
451 uint32_t priority;
452 uint32_t queue_percent;
453 uint32_t *read_ptr;
454 uint32_t *write_ptr;
455 void __iomem *doorbell_ptr;
456 uint32_t doorbell_off;
457 bool is_interop;
458 bool is_evicted;
459 bool is_active;
460 bool is_gws;
461
462 unsigned int vmid;
463
464 uint32_t sdma_engine_id;
465 uint32_t sdma_queue_id;
466 uint32_t sdma_vm_addr;
467
468 uint64_t eop_ring_buffer_address;
469 uint32_t eop_ring_buffer_size;
470 uint64_t ctx_save_restore_area_address;
471 uint32_t ctx_save_restore_area_size;
472 uint32_t ctl_stack_size;
473 uint64_t tba_addr;
474 uint64_t tma_addr;
475
476 uint32_t cu_mask_count;
477 uint32_t *cu_mask;
478};
479
480#define QUEUE_IS_ACTIVE(q) ((q).queue_size > 0 && \
481 (q).queue_address != 0 && \
482 (q).queue_percent > 0 && \
483 !(q).is_evicted)
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518struct queue {
519 struct list_head list;
520 void *mqd;
521 struct kfd_mem_obj *mqd_mem_obj;
522 uint64_t gart_mqd_addr;
523 struct queue_properties properties;
524
525 uint32_t mec;
526 uint32_t pipe;
527 uint32_t queue;
528
529 unsigned int sdma_id;
530 unsigned int doorbell_id;
531
532 struct kfd_process *process;
533 struct kfd_dev *device;
534 void *gws;
535
536
537 struct kobject kobj;
538};
539
540enum KFD_MQD_TYPE {
541 KFD_MQD_TYPE_HIQ = 0,
542 KFD_MQD_TYPE_CP,
543 KFD_MQD_TYPE_SDMA,
544 KFD_MQD_TYPE_DIQ,
545 KFD_MQD_TYPE_MAX
546};
547
548enum KFD_PIPE_PRIORITY {
549 KFD_PIPE_PRIORITY_CS_LOW = 0,
550 KFD_PIPE_PRIORITY_CS_MEDIUM,
551 KFD_PIPE_PRIORITY_CS_HIGH
552};
553
554struct scheduling_resources {
555 unsigned int vmid_mask;
556 enum kfd_queue_type type;
557 uint64_t queue_mask;
558 uint64_t gws_mask;
559 uint32_t oac_mask;
560 uint32_t gds_heap_base;
561 uint32_t gds_heap_size;
562};
563
564struct process_queue_manager {
565
566 struct kfd_process *process;
567 struct list_head queues;
568 unsigned long *queue_slot_bitmap;
569};
570
571struct qcm_process_device {
572
573 struct device_queue_manager *dqm;
574 struct process_queue_manager *pqm;
575
576 struct list_head queues_list;
577 struct list_head priv_queue_list;
578
579 unsigned int queue_count;
580 unsigned int vmid;
581 bool is_debug;
582 unsigned int evicted;
583
584
585
586
587 bool reset_wavefronts;
588
589
590
591
592
593
594
595 bool mapped_gws_queue;
596
597
598 uint64_t gds_context_area;
599
600 uint64_t page_table_base;
601 uint32_t sh_mem_config;
602 uint32_t sh_mem_bases;
603 uint32_t sh_mem_ape1_base;
604 uint32_t sh_mem_ape1_limit;
605 uint32_t gds_size;
606 uint32_t num_gws;
607 uint32_t num_oac;
608 uint32_t sh_hidden_private_base;
609
610
611 void *cwsr_kaddr;
612 uint64_t cwsr_base;
613 uint64_t tba_addr;
614 uint64_t tma_addr;
615
616
617 uint64_t ib_base;
618 void *ib_kaddr;
619
620
621 unsigned long *doorbell_bitmap;
622};
623
624
625
626
627#define PROCESS_RESTORE_TIME_MS 100
628
629#define PROCESS_BACK_OFF_TIME_MS 100
630
631#define PROCESS_ACTIVE_TIME_MS 10
632
633
634
635
636#define MAKE_HANDLE(gpu_id, idr_handle) \
637 (((uint64_t)(gpu_id) << 32) + idr_handle)
638#define GET_GPU_ID(handle) (handle >> 32)
639#define GET_IDR_HANDLE(handle) (handle & 0xFFFFFFFF)
640
641enum kfd_pdd_bound {
642 PDD_UNBOUND = 0,
643 PDD_BOUND,
644 PDD_BOUND_SUSPENDED,
645};
646
647#define MAX_SYSFS_FILENAME_LEN 15
648
649
650
651
652
653
654#define SDMA_ACTIVITY_DIVISOR 100
655
656
657struct kfd_process_device {
658
659 struct kfd_dev *dev;
660
661
662 struct kfd_process *process;
663
664
665 struct qcm_process_device qpd;
666
667
668 uint64_t lds_base;
669 uint64_t lds_limit;
670 uint64_t gpuvm_base;
671 uint64_t gpuvm_limit;
672 uint64_t scratch_base;
673 uint64_t scratch_limit;
674
675
676 struct file *drm_file;
677 void *drm_priv;
678
679
680 struct idr alloc_idr;
681
682
683
684
685
686
687 bool already_dequeued;
688 bool runtime_inuse;
689
690
691 enum kfd_pdd_bound bound;
692
693
694 uint64_t vram_usage;
695 struct attribute attr_vram;
696 char vram_filename[MAX_SYSFS_FILENAME_LEN];
697
698
699 uint64_t sdma_past_activity_counter;
700 struct attribute attr_sdma;
701 char sdma_filename[MAX_SYSFS_FILENAME_LEN];
702
703
704 uint64_t last_evict_timestamp;
705 atomic64_t evict_duration_counter;
706 struct attribute attr_evict;
707
708 struct kobject *kobj_stats;
709 unsigned int doorbell_index;
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734 struct attribute attr_cu_occupancy;
735
736
737 struct kobject *kobj_counters;
738 struct attribute attr_faults;
739 struct attribute attr_page_in;
740 struct attribute attr_page_out;
741 uint64_t faults;
742 uint64_t page_in;
743 uint64_t page_out;
744};
745
746#define qpd_to_pdd(x) container_of(x, struct kfd_process_device, qpd)
747
748struct svm_range_list {
749 struct mutex lock;
750 struct rb_root_cached objects;
751 struct list_head list;
752 struct work_struct deferred_list_work;
753 struct list_head deferred_range_list;
754 spinlock_t deferred_list_lock;
755 atomic_t evicted_ranges;
756 struct delayed_work restore_work;
757 DECLARE_BITMAP(bitmap_supported, MAX_GPU_INSTANCE);
758};
759
760
761struct kfd_process {
762
763
764
765
766 struct hlist_node kfd_processes;
767
768
769
770
771
772
773 void *mm;
774
775 struct kref ref;
776 struct work_struct release_work;
777
778 struct mutex mutex;
779
780
781
782
783
784
785
786
787 struct task_struct *lead_thread;
788
789
790 struct mmu_notifier mmu_notifier;
791
792 u32 pasid;
793
794
795
796
797
798 struct kfd_process_device *pdds[MAX_GPU_INSTANCE];
799 uint32_t n_pdds;
800
801 struct process_queue_manager pqm;
802
803
804 bool is_32bit_user_mode;
805
806
807 struct mutex event_mutex;
808
809 struct idr event_idr;
810
811 struct kfd_signal_page *signal_page;
812 size_t signal_mapped_size;
813 size_t signal_event_count;
814 bool signal_event_limit_reached;
815
816
817 void *kgd_process_info;
818
819
820
821
822 struct dma_fence *ef;
823
824
825 struct delayed_work eviction_work;
826 struct delayed_work restore_work;
827
828 unsigned int last_eviction_seqno;
829
830
831
832 unsigned long last_restore_timestamp;
833
834
835 struct kobject *kobj;
836 struct kobject *kobj_queues;
837 struct attribute attr_pasid;
838
839
840 struct svm_range_list svms;
841
842 bool xnack_enabled;
843};
844
845#define KFD_PROCESS_TABLE_SIZE 5
846extern DECLARE_HASHTABLE(kfd_processes_table, KFD_PROCESS_TABLE_SIZE);
847extern struct srcu_struct kfd_processes_srcu;
848
849
850
851
852
853
854
855
856
857
858typedef int amdkfd_ioctl_t(struct file *filep, struct kfd_process *p,
859 void *data);
860
861struct amdkfd_ioctl_desc {
862 unsigned int cmd;
863 int flags;
864 amdkfd_ioctl_t *func;
865 unsigned int cmd_drv;
866 const char *name;
867};
868bool kfd_dev_is_large_bar(struct kfd_dev *dev);
869
870int kfd_process_create_wq(void);
871void kfd_process_destroy_wq(void);
872struct kfd_process *kfd_create_process(struct file *filep);
873struct kfd_process *kfd_get_process(const struct task_struct *);
874struct kfd_process *kfd_lookup_process_by_pasid(u32 pasid);
875struct kfd_process *kfd_lookup_process_by_mm(const struct mm_struct *mm);
876
877int kfd_process_gpuidx_from_gpuid(struct kfd_process *p, uint32_t gpu_id);
878int kfd_process_gpuid_from_kgd(struct kfd_process *p,
879 struct amdgpu_device *adev, uint32_t *gpuid,
880 uint32_t *gpuidx);
881static inline int kfd_process_gpuid_from_gpuidx(struct kfd_process *p,
882 uint32_t gpuidx, uint32_t *gpuid) {
883 return gpuidx < p->n_pdds ? p->pdds[gpuidx]->dev->id : -EINVAL;
884}
885static inline struct kfd_process_device *kfd_process_device_from_gpuidx(
886 struct kfd_process *p, uint32_t gpuidx) {
887 return gpuidx < p->n_pdds ? p->pdds[gpuidx] : NULL;
888}
889
890void kfd_unref_process(struct kfd_process *p);
891int kfd_process_evict_queues(struct kfd_process *p);
892int kfd_process_restore_queues(struct kfd_process *p);
893void kfd_suspend_all_processes(void);
894int kfd_resume_all_processes(void);
895
896int kfd_process_device_init_vm(struct kfd_process_device *pdd,
897 struct file *drm_file);
898struct kfd_process_device *kfd_bind_process_to_device(struct kfd_dev *dev,
899 struct kfd_process *p);
900struct kfd_process_device *kfd_get_process_device_data(struct kfd_dev *dev,
901 struct kfd_process *p);
902struct kfd_process_device *kfd_create_process_device_data(struct kfd_dev *dev,
903 struct kfd_process *p);
904
905bool kfd_process_xnack_mode(struct kfd_process *p, bool supported);
906
907int kfd_reserved_mem_mmap(struct kfd_dev *dev, struct kfd_process *process,
908 struct vm_area_struct *vma);
909
910
911int kfd_process_device_create_obj_handle(struct kfd_process_device *pdd,
912 void *mem);
913void *kfd_process_device_translate_handle(struct kfd_process_device *p,
914 int handle);
915void kfd_process_device_remove_obj_handle(struct kfd_process_device *pdd,
916 int handle);
917
918
919int kfd_pasid_init(void);
920void kfd_pasid_exit(void);
921bool kfd_set_pasid_limit(unsigned int new_limit);
922unsigned int kfd_get_pasid_limit(void);
923u32 kfd_pasid_alloc(void);
924void kfd_pasid_free(u32 pasid);
925
926
927size_t kfd_doorbell_process_slice(struct kfd_dev *kfd);
928int kfd_doorbell_init(struct kfd_dev *kfd);
929void kfd_doorbell_fini(struct kfd_dev *kfd);
930int kfd_doorbell_mmap(struct kfd_dev *dev, struct kfd_process *process,
931 struct vm_area_struct *vma);
932void __iomem *kfd_get_kernel_doorbell(struct kfd_dev *kfd,
933 unsigned int *doorbell_off);
934void kfd_release_kernel_doorbell(struct kfd_dev *kfd, u32 __iomem *db_addr);
935u32 read_kernel_doorbell(u32 __iomem *db);
936void write_kernel_doorbell(void __iomem *db, u32 value);
937void write_kernel_doorbell64(void __iomem *db, u64 value);
938unsigned int kfd_get_doorbell_dw_offset_in_bar(struct kfd_dev *kfd,
939 struct kfd_process_device *pdd,
940 unsigned int doorbell_id);
941phys_addr_t kfd_get_process_doorbells(struct kfd_process_device *pdd);
942int kfd_alloc_process_doorbells(struct kfd_dev *kfd,
943 unsigned int *doorbell_index);
944void kfd_free_process_doorbells(struct kfd_dev *kfd,
945 unsigned int doorbell_index);
946
947
948int kfd_gtt_sa_allocate(struct kfd_dev *kfd, unsigned int size,
949 struct kfd_mem_obj **mem_obj);
950
951int kfd_gtt_sa_free(struct kfd_dev *kfd, struct kfd_mem_obj *mem_obj);
952
953extern struct device *kfd_device;
954
955
956void kfd_procfs_init(void);
957void kfd_procfs_shutdown(void);
958int kfd_procfs_add_queue(struct queue *q);
959void kfd_procfs_del_queue(struct queue *q);
960
961
962int kfd_topology_init(void);
963void kfd_topology_shutdown(void);
964int kfd_topology_add_device(struct kfd_dev *gpu);
965int kfd_topology_remove_device(struct kfd_dev *gpu);
966struct kfd_topology_device *kfd_topology_device_by_proximity_domain(
967 uint32_t proximity_domain);
968struct kfd_topology_device *kfd_topology_device_by_id(uint32_t gpu_id);
969struct kfd_dev *kfd_device_by_id(uint32_t gpu_id);
970struct kfd_dev *kfd_device_by_pci_dev(const struct pci_dev *pdev);
971struct kfd_dev *kfd_device_by_kgd(const struct kgd_dev *kgd);
972int kfd_topology_enum_kfd_devices(uint8_t idx, struct kfd_dev **kdev);
973int kfd_numa_node_to_apic_id(int numa_node_id);
974void kfd_double_confirm_iommu_support(struct kfd_dev *gpu);
975
976
977int kfd_interrupt_init(struct kfd_dev *dev);
978void kfd_interrupt_exit(struct kfd_dev *dev);
979bool enqueue_ih_ring_entry(struct kfd_dev *kfd, const void *ih_ring_entry);
980bool interrupt_is_wanted(struct kfd_dev *dev,
981 const uint32_t *ih_ring_entry,
982 uint32_t *patched_ihre, bool *flag);
983
984
985int kfd_init_apertures(struct kfd_process *process);
986
987void kfd_process_set_trap_handler(struct qcm_process_device *qpd,
988 uint64_t tba_addr,
989 uint64_t tma_addr);
990
991
992int init_queue(struct queue **q, const struct queue_properties *properties);
993void uninit_queue(struct queue *q);
994void print_queue_properties(struct queue_properties *q);
995void print_queue(struct queue *q);
996
997struct mqd_manager *mqd_manager_init_cik(enum KFD_MQD_TYPE type,
998 struct kfd_dev *dev);
999struct mqd_manager *mqd_manager_init_cik_hawaii(enum KFD_MQD_TYPE type,
1000 struct kfd_dev *dev);
1001struct mqd_manager *mqd_manager_init_vi(enum KFD_MQD_TYPE type,
1002 struct kfd_dev *dev);
1003struct mqd_manager *mqd_manager_init_vi_tonga(enum KFD_MQD_TYPE type,
1004 struct kfd_dev *dev);
1005struct mqd_manager *mqd_manager_init_v9(enum KFD_MQD_TYPE type,
1006 struct kfd_dev *dev);
1007struct mqd_manager *mqd_manager_init_v10(enum KFD_MQD_TYPE type,
1008 struct kfd_dev *dev);
1009struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev);
1010void device_queue_manager_uninit(struct device_queue_manager *dqm);
1011struct kernel_queue *kernel_queue_init(struct kfd_dev *dev,
1012 enum kfd_queue_type type);
1013void kernel_queue_uninit(struct kernel_queue *kq, bool hanging);
1014int kfd_process_vm_fault(struct device_queue_manager *dqm, u32 pasid);
1015
1016
1017struct process_queue_node {
1018 struct queue *q;
1019 struct kernel_queue *kq;
1020 struct list_head process_queue_list;
1021};
1022
1023void kfd_process_dequeue_from_device(struct kfd_process_device *pdd);
1024void kfd_process_dequeue_from_all_devices(struct kfd_process *p);
1025int pqm_init(struct process_queue_manager *pqm, struct kfd_process *p);
1026void pqm_uninit(struct process_queue_manager *pqm);
1027int pqm_create_queue(struct process_queue_manager *pqm,
1028 struct kfd_dev *dev,
1029 struct file *f,
1030 struct queue_properties *properties,
1031 unsigned int *qid,
1032 uint32_t *p_doorbell_offset_in_process);
1033int pqm_destroy_queue(struct process_queue_manager *pqm, unsigned int qid);
1034int pqm_update_queue(struct process_queue_manager *pqm, unsigned int qid,
1035 struct queue_properties *p);
1036int pqm_set_cu_mask(struct process_queue_manager *pqm, unsigned int qid,
1037 struct queue_properties *p);
1038int pqm_set_gws(struct process_queue_manager *pqm, unsigned int qid,
1039 void *gws);
1040struct kernel_queue *pqm_get_kernel_queue(struct process_queue_manager *pqm,
1041 unsigned int qid);
1042struct queue *pqm_get_user_queue(struct process_queue_manager *pqm,
1043 unsigned int qid);
1044int pqm_get_wave_state(struct process_queue_manager *pqm,
1045 unsigned int qid,
1046 void __user *ctl_stack,
1047 u32 *ctl_stack_used_size,
1048 u32 *save_area_used_size);
1049
1050int amdkfd_fence_wait_timeout(uint64_t *fence_addr,
1051 uint64_t fence_value,
1052 unsigned int timeout_ms);
1053
1054
1055
1056#define KFD_FENCE_COMPLETED (100)
1057#define KFD_FENCE_INIT (10)
1058
1059struct packet_manager {
1060 struct device_queue_manager *dqm;
1061 struct kernel_queue *priv_queue;
1062 struct mutex lock;
1063 bool allocated;
1064 struct kfd_mem_obj *ib_buffer_obj;
1065 unsigned int ib_size_bytes;
1066 bool is_over_subscription;
1067
1068 const struct packet_manager_funcs *pmf;
1069};
1070
1071struct packet_manager_funcs {
1072
1073 int (*map_process)(struct packet_manager *pm, uint32_t *buffer,
1074 struct qcm_process_device *qpd);
1075 int (*runlist)(struct packet_manager *pm, uint32_t *buffer,
1076 uint64_t ib, size_t ib_size_in_dwords, bool chain);
1077 int (*set_resources)(struct packet_manager *pm, uint32_t *buffer,
1078 struct scheduling_resources *res);
1079 int (*map_queues)(struct packet_manager *pm, uint32_t *buffer,
1080 struct queue *q, bool is_static);
1081 int (*unmap_queues)(struct packet_manager *pm, uint32_t *buffer,
1082 enum kfd_queue_type type,
1083 enum kfd_unmap_queues_filter mode,
1084 uint32_t filter_param, bool reset,
1085 unsigned int sdma_engine);
1086 int (*query_status)(struct packet_manager *pm, uint32_t *buffer,
1087 uint64_t fence_address, uint64_t fence_value);
1088 int (*release_mem)(uint64_t gpu_addr, uint32_t *buffer);
1089
1090
1091 int map_process_size;
1092 int runlist_size;
1093 int set_resources_size;
1094 int map_queues_size;
1095 int unmap_queues_size;
1096 int query_status_size;
1097 int release_mem_size;
1098};
1099
1100extern const struct packet_manager_funcs kfd_vi_pm_funcs;
1101extern const struct packet_manager_funcs kfd_v9_pm_funcs;
1102extern const struct packet_manager_funcs kfd_aldebaran_pm_funcs;
1103
1104int pm_init(struct packet_manager *pm, struct device_queue_manager *dqm);
1105void pm_uninit(struct packet_manager *pm, bool hanging);
1106int pm_send_set_resources(struct packet_manager *pm,
1107 struct scheduling_resources *res);
1108int pm_send_runlist(struct packet_manager *pm, struct list_head *dqm_queues);
1109int pm_send_query_status(struct packet_manager *pm, uint64_t fence_address,
1110 uint64_t fence_value);
1111
1112int pm_send_unmap_queue(struct packet_manager *pm, enum kfd_queue_type type,
1113 enum kfd_unmap_queues_filter mode,
1114 uint32_t filter_param, bool reset,
1115 unsigned int sdma_engine);
1116
1117void pm_release_ib(struct packet_manager *pm);
1118
1119
1120unsigned int pm_build_pm4_header(unsigned int opcode, size_t packet_size);
1121
1122uint64_t kfd_get_number_elems(struct kfd_dev *kfd);
1123
1124
1125extern const struct kfd_event_interrupt_class event_interrupt_class_cik;
1126extern const struct kfd_event_interrupt_class event_interrupt_class_v9;
1127
1128extern const struct kfd_device_global_init_class device_global_init_class_cik;
1129
1130void kfd_event_init_process(struct kfd_process *p);
1131void kfd_event_free_process(struct kfd_process *p);
1132int kfd_event_mmap(struct kfd_process *process, struct vm_area_struct *vma);
1133int kfd_wait_on_events(struct kfd_process *p,
1134 uint32_t num_events, void __user *data,
1135 bool all, uint32_t user_timeout_ms,
1136 uint32_t *wait_result);
1137void kfd_signal_event_interrupt(u32 pasid, uint32_t partial_id,
1138 uint32_t valid_id_bits);
1139void kfd_signal_iommu_event(struct kfd_dev *dev,
1140 u32 pasid, unsigned long address,
1141 bool is_write_requested, bool is_execute_requested);
1142void kfd_signal_hw_exception_event(u32 pasid);
1143int kfd_set_event(struct kfd_process *p, uint32_t event_id);
1144int kfd_reset_event(struct kfd_process *p, uint32_t event_id);
1145int kfd_event_page_set(struct kfd_process *p, void *kernel_address,
1146 uint64_t size);
1147int kfd_event_create(struct file *devkfd, struct kfd_process *p,
1148 uint32_t event_type, bool auto_reset, uint32_t node_id,
1149 uint32_t *event_id, uint32_t *event_trigger_data,
1150 uint64_t *event_page_offset, uint32_t *event_slot_index);
1151int kfd_event_destroy(struct kfd_process *p, uint32_t event_id);
1152
1153void kfd_signal_vm_fault_event(struct kfd_dev *dev, u32 pasid,
1154 struct kfd_vm_fault_info *info);
1155
1156void kfd_signal_reset_event(struct kfd_dev *dev);
1157
1158void kfd_signal_poison_consumed_event(struct kfd_dev *dev, u32 pasid);
1159
1160void kfd_flush_tlb(struct kfd_process_device *pdd, enum TLB_FLUSH_TYPE type);
1161
1162int dbgdev_wave_reset_wavefronts(struct kfd_dev *dev, struct kfd_process *p);
1163
1164bool kfd_is_locked(void);
1165
1166
1167void kfd_inc_compute_active(struct kfd_dev *dev);
1168void kfd_dec_compute_active(struct kfd_dev *dev);
1169
1170
1171
1172static inline int kfd_devcgroup_check_permission(struct kfd_dev *kfd)
1173{
1174#if defined(CONFIG_CGROUP_DEVICE) || defined(CONFIG_CGROUP_BPF)
1175 struct drm_device *ddev = kfd->ddev;
1176
1177 return devcgroup_check_permission(DEVCG_DEV_CHAR, DRM_MAJOR,
1178 ddev->render->index,
1179 DEVCG_ACC_WRITE | DEVCG_ACC_READ);
1180#else
1181 return 0;
1182#endif
1183}
1184
1185
1186#if defined(CONFIG_DEBUG_FS)
1187
1188void kfd_debugfs_init(void);
1189void kfd_debugfs_fini(void);
1190int kfd_debugfs_mqds_by_process(struct seq_file *m, void *data);
1191int pqm_debugfs_mqds(struct seq_file *m, void *data);
1192int kfd_debugfs_hqds_by_device(struct seq_file *m, void *data);
1193int dqm_debugfs_hqds(struct seq_file *m, void *data);
1194int kfd_debugfs_rls_by_device(struct seq_file *m, void *data);
1195int pm_debugfs_runlist(struct seq_file *m, void *data);
1196
1197int kfd_debugfs_hang_hws(struct kfd_dev *dev);
1198int pm_debugfs_hang_hws(struct packet_manager *pm);
1199int dqm_debugfs_hang_hws(struct device_queue_manager *dqm);
1200
1201#else
1202
1203static inline void kfd_debugfs_init(void) {}
1204static inline void kfd_debugfs_fini(void) {}
1205
1206#endif
1207
1208#endif
1209