1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23#ifndef KFD_PRIV_H_INCLUDED
24#define KFD_PRIV_H_INCLUDED
25
26#include <linux/hashtable.h>
27#include <linux/mmu_notifier.h>
28#include <linux/mutex.h>
29#include <linux/types.h>
30#include <linux/atomic.h>
31#include <linux/workqueue.h>
32#include <linux/spinlock.h>
33#include <linux/kfd_ioctl.h>
34#include <kgd_kfd_interface.h>
35
36#define KFD_SYSFS_FILE_MODE 0444
37
38#define KFD_MMAP_DOORBELL_MASK 0x8000000000000
39#define KFD_MMAP_EVENTS_MASK 0x4000000000000
40
41
42
43
44
45
46
47
48#define KFD_CIK_HIQ_PIPE 4
49#define KFD_CIK_HIQ_QUEUE 0
50
51
52#define KFD_GPU_ID_HASH_WIDTH 16
53
54
55#define kfd_alloc_struct(ptr_to_struct) \
56 ((typeof(ptr_to_struct)) kzalloc(sizeof(*ptr_to_struct), GFP_KERNEL))
57
58#define KFD_MAX_NUM_OF_PROCESSES 512
59#define KFD_MAX_NUM_OF_QUEUES_PER_PROCESS 1024
60
61
62
63
64
65extern int max_num_of_queues_per_device;
66
67#define KFD_MAX_NUM_OF_QUEUES_PER_DEVICE_DEFAULT 4096
68#define KFD_MAX_NUM_OF_QUEUES_PER_DEVICE \
69 (KFD_MAX_NUM_OF_PROCESSES * \
70 KFD_MAX_NUM_OF_QUEUES_PER_PROCESS)
71
72#define KFD_KERNEL_QUEUE_SIZE 2048
73
74
75extern int sched_policy;
76
77
78
79
80
81extern int send_sigterm;
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104enum kfd_sched_policy {
105 KFD_SCHED_POLICY_HWS = 0,
106 KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION,
107 KFD_SCHED_POLICY_NO_HWS
108};
109
110enum cache_policy {
111 cache_policy_coherent,
112 cache_policy_noncoherent
113};
114
115enum asic_family_type {
116 CHIP_KAVERI = 0,
117 CHIP_CARRIZO
118};
119
120struct kfd_event_interrupt_class {
121 bool (*interrupt_isr)(struct kfd_dev *dev,
122 const uint32_t *ih_ring_entry);
123 void (*interrupt_wq)(struct kfd_dev *dev,
124 const uint32_t *ih_ring_entry);
125};
126
127struct kfd_device_info {
128 unsigned int asic_family;
129 const struct kfd_event_interrupt_class *event_interrupt_class;
130 unsigned int max_pasid_bits;
131 unsigned int max_no_of_hqd;
132 size_t ih_ring_entry_size;
133 uint8_t num_of_watch_points;
134 uint16_t mqd_size_aligned;
135};
136
137struct kfd_mem_obj {
138 uint32_t range_start;
139 uint32_t range_end;
140 uint64_t gpu_addr;
141 uint32_t *cpu_ptr;
142};
143
144struct kfd_dev {
145 struct kgd_dev *kgd;
146
147 const struct kfd_device_info *device_info;
148 struct pci_dev *pdev;
149
150 unsigned int id;
151
152 phys_addr_t doorbell_base;
153
154
155
156 size_t doorbell_id_offset;
157
158
159
160 size_t doorbell_process_limit;
161
162
163 u32 __iomem *doorbell_kernel_ptr;
164
165
166
167 struct kgd2kfd_shared_resources shared_resources;
168
169 const struct kfd2kgd_calls *kfd2kgd;
170 struct mutex doorbell_mutex;
171 DECLARE_BITMAP(doorbell_available_index,
172 KFD_MAX_NUM_OF_QUEUES_PER_PROCESS);
173
174 void *gtt_mem;
175 uint64_t gtt_start_gpu_addr;
176 void *gtt_start_cpu_ptr;
177 void *gtt_sa_bitmap;
178 struct mutex gtt_sa_lock;
179 unsigned int gtt_sa_chunk_size;
180 unsigned int gtt_sa_num_of_chunks;
181
182
183 void *interrupt_ring;
184 size_t interrupt_ring_size;
185 atomic_t interrupt_ring_rptr;
186 atomic_t interrupt_ring_wptr;
187 struct work_struct interrupt_work;
188 spinlock_t interrupt_lock;
189
190
191 struct device_queue_manager *dqm;
192
193 bool init_complete;
194
195
196
197
198 bool interrupts_active;
199
200
201 struct kfd_dbgmgr *dbgmgr;
202};
203
204
205void kgd2kfd_exit(void);
206struct kfd_dev *kgd2kfd_probe(struct kgd_dev *kgd,
207 struct pci_dev *pdev, const struct kfd2kgd_calls *f2g);
208bool kgd2kfd_device_init(struct kfd_dev *kfd,
209 const struct kgd2kfd_shared_resources *gpu_resources);
210void kgd2kfd_device_exit(struct kfd_dev *kfd);
211
212enum kfd_mempool {
213 KFD_MEMPOOL_SYSTEM_CACHEABLE = 1,
214 KFD_MEMPOOL_SYSTEM_WRITECOMBINE = 2,
215 KFD_MEMPOOL_FRAMEBUFFER = 3,
216};
217
218
219int kfd_chardev_init(void);
220void kfd_chardev_exit(void);
221struct device *kfd_chardev(void);
222
223
224
225
226
227
228
229
230
231
232
233
234
235enum kfd_preempt_type_filter {
236 KFD_PREEMPT_TYPE_FILTER_SINGLE_QUEUE,
237 KFD_PREEMPT_TYPE_FILTER_ALL_QUEUES,
238 KFD_PREEMPT_TYPE_FILTER_DYNAMIC_QUEUES,
239 KFD_PREEMPT_TYPE_FILTER_BY_PASID
240};
241
242enum kfd_preempt_type {
243 KFD_PREEMPT_TYPE_WAVEFRONT,
244 KFD_PREEMPT_TYPE_WAVEFRONT_RESET
245};
246
247
248
249
250
251
252
253
254
255
256
257
258enum kfd_queue_type {
259 KFD_QUEUE_TYPE_COMPUTE,
260 KFD_QUEUE_TYPE_SDMA,
261 KFD_QUEUE_TYPE_HIQ,
262 KFD_QUEUE_TYPE_DIQ
263};
264
265enum kfd_queue_format {
266 KFD_QUEUE_FORMAT_PM4,
267 KFD_QUEUE_FORMAT_AQL
268};
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314struct queue_properties {
315 enum kfd_queue_type type;
316 enum kfd_queue_format format;
317 unsigned int queue_id;
318 uint64_t queue_address;
319 uint64_t queue_size;
320 uint32_t priority;
321 uint32_t queue_percent;
322 uint32_t *read_ptr;
323 uint32_t *write_ptr;
324 uint32_t __iomem *doorbell_ptr;
325 uint32_t doorbell_off;
326 bool is_interop;
327 bool is_active;
328
329 unsigned int vmid;
330
331 uint32_t sdma_engine_id;
332 uint32_t sdma_queue_id;
333 uint32_t sdma_vm_addr;
334
335 uint64_t eop_ring_buffer_address;
336 uint32_t eop_ring_buffer_size;
337 uint64_t ctx_save_restore_area_address;
338 uint32_t ctx_save_restore_area_size;
339};
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370struct queue {
371 struct list_head list;
372 void *mqd;
373 struct kfd_mem_obj *mqd_mem_obj;
374 uint64_t gart_mqd_addr;
375 struct queue_properties properties;
376
377 uint32_t mec;
378 uint32_t pipe;
379 uint32_t queue;
380
381 unsigned int sdma_id;
382
383 struct kfd_process *process;
384 struct kfd_dev *device;
385};
386
387
388
389
390enum KFD_MQD_TYPE {
391 KFD_MQD_TYPE_COMPUTE = 0,
392 KFD_MQD_TYPE_HIQ,
393 KFD_MQD_TYPE_CP,
394 KFD_MQD_TYPE_SDMA,
395 KFD_MQD_TYPE_MAX
396};
397
398struct scheduling_resources {
399 unsigned int vmid_mask;
400 enum kfd_queue_type type;
401 uint64_t queue_mask;
402 uint64_t gws_mask;
403 uint32_t oac_mask;
404 uint32_t gds_heap_base;
405 uint32_t gds_heap_size;
406};
407
408struct process_queue_manager {
409
410 struct kfd_process *process;
411 unsigned int num_concurrent_processes;
412 struct list_head queues;
413 unsigned long *queue_slot_bitmap;
414};
415
416struct qcm_process_device {
417
418 struct device_queue_manager *dqm;
419 struct process_queue_manager *pqm;
420
421 struct list_head queues_list;
422 struct list_head priv_queue_list;
423
424 unsigned int queue_count;
425 unsigned int vmid;
426 bool is_debug;
427
428
429
430 uint64_t gds_context_area;
431 uint32_t sh_mem_config;
432 uint32_t sh_mem_bases;
433 uint32_t sh_mem_ape1_base;
434 uint32_t sh_mem_ape1_limit;
435 uint32_t page_table_base;
436 uint32_t gds_size;
437 uint32_t num_gws;
438 uint32_t num_oac;
439};
440
441
442struct kfd_process_device {
443
444
445
446
447 struct list_head per_device_list;
448
449
450 struct kfd_dev *dev;
451
452
453
454 struct qcm_process_device qpd;
455
456
457 uint64_t lds_base;
458 uint64_t lds_limit;
459 uint64_t gpuvm_base;
460 uint64_t gpuvm_limit;
461 uint64_t scratch_base;
462 uint64_t scratch_limit;
463
464
465 bool bound;
466
467
468
469
470 bool reset_wavefronts;
471};
472
473#define qpd_to_pdd(x) container_of(x, struct kfd_process_device, qpd)
474
475
476struct kfd_process {
477
478
479
480
481 struct hlist_node kfd_processes;
482
483 struct mm_struct *mm;
484
485 struct mutex mutex;
486
487
488
489
490
491
492 struct task_struct *lead_thread;
493
494
495 struct mmu_notifier mmu_notifier;
496
497
498 struct rcu_head rcu;
499
500 unsigned int pasid;
501
502
503
504
505
506 struct list_head per_device_data;
507
508 struct process_queue_manager pqm;
509
510
511 size_t queue_array_size;
512
513
514 struct kfd_queue **queues;
515
516
517 bool is_32bit_user_mode;
518
519
520 struct mutex event_mutex;
521
522 DECLARE_HASHTABLE(events, 4);
523 struct list_head signal_event_pages;
524
525 u32 next_nonsignal_event_id;
526 size_t signal_event_count;
527};
528
529
530
531
532
533
534
535
536typedef int amdkfd_ioctl_t(struct file *filep, struct kfd_process *p,
537 void *data);
538
539struct amdkfd_ioctl_desc {
540 unsigned int cmd;
541 int flags;
542 amdkfd_ioctl_t *func;
543 unsigned int cmd_drv;
544 const char *name;
545};
546
547void kfd_process_create_wq(void);
548void kfd_process_destroy_wq(void);
549struct kfd_process *kfd_create_process(const struct task_struct *);
550struct kfd_process *kfd_get_process(const struct task_struct *);
551struct kfd_process *kfd_lookup_process_by_pasid(unsigned int pasid);
552
553struct kfd_process_device *kfd_bind_process_to_device(struct kfd_dev *dev,
554 struct kfd_process *p);
555void kfd_unbind_process_from_device(struct kfd_dev *dev, unsigned int pasid);
556struct kfd_process_device *kfd_get_process_device_data(struct kfd_dev *dev,
557 struct kfd_process *p);
558struct kfd_process_device *kfd_create_process_device_data(struct kfd_dev *dev,
559 struct kfd_process *p);
560
561
562struct kfd_process_device *kfd_get_first_process_device_data(struct kfd_process *p);
563struct kfd_process_device *kfd_get_next_process_device_data(struct kfd_process *p,
564 struct kfd_process_device *pdd);
565bool kfd_has_process_device_data(struct kfd_process *p);
566
567
568int kfd_pasid_init(void);
569void kfd_pasid_exit(void);
570bool kfd_set_pasid_limit(unsigned int new_limit);
571unsigned int kfd_get_pasid_limit(void);
572unsigned int kfd_pasid_alloc(void);
573void kfd_pasid_free(unsigned int pasid);
574
575
576void kfd_doorbell_init(struct kfd_dev *kfd);
577int kfd_doorbell_mmap(struct kfd_process *process, struct vm_area_struct *vma);
578u32 __iomem *kfd_get_kernel_doorbell(struct kfd_dev *kfd,
579 unsigned int *doorbell_off);
580void kfd_release_kernel_doorbell(struct kfd_dev *kfd, u32 __iomem *db_addr);
581u32 read_kernel_doorbell(u32 __iomem *db);
582void write_kernel_doorbell(u32 __iomem *db, u32 value);
583unsigned int kfd_queue_id_to_doorbell(struct kfd_dev *kfd,
584 struct kfd_process *process,
585 unsigned int queue_id);
586
587
588
589int kfd_gtt_sa_allocate(struct kfd_dev *kfd, unsigned int size,
590 struct kfd_mem_obj **mem_obj);
591
592int kfd_gtt_sa_free(struct kfd_dev *kfd, struct kfd_mem_obj *mem_obj);
593
594extern struct device *kfd_device;
595
596
597int kfd_topology_init(void);
598void kfd_topology_shutdown(void);
599int kfd_topology_add_device(struct kfd_dev *gpu);
600int kfd_topology_remove_device(struct kfd_dev *gpu);
601struct kfd_dev *kfd_device_by_id(uint32_t gpu_id);
602struct kfd_dev *kfd_device_by_pci_dev(const struct pci_dev *pdev);
603struct kfd_dev *kfd_topology_enum_kfd_devices(uint8_t idx);
604
605
606int kfd_interrupt_init(struct kfd_dev *dev);
607void kfd_interrupt_exit(struct kfd_dev *dev);
608void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry);
609bool enqueue_ih_ring_entry(struct kfd_dev *kfd, const void *ih_ring_entry);
610bool interrupt_is_wanted(struct kfd_dev *dev, const uint32_t *ih_ring_entry);
611
612
613void kgd2kfd_suspend(struct kfd_dev *kfd);
614int kgd2kfd_resume(struct kfd_dev *kfd);
615
616
617int kfd_init_apertures(struct kfd_process *process);
618
619
620struct cik_sdma_rlc_registers *get_sdma_mqd(void *mqd);
621
622int init_queue(struct queue **q, const struct queue_properties *properties);
623void uninit_queue(struct queue *q);
624void print_queue_properties(struct queue_properties *q);
625void print_queue(struct queue *q);
626
627struct mqd_manager *mqd_manager_init(enum KFD_MQD_TYPE type,
628 struct kfd_dev *dev);
629struct mqd_manager *mqd_manager_init_cik(enum KFD_MQD_TYPE type,
630 struct kfd_dev *dev);
631struct mqd_manager *mqd_manager_init_vi(enum KFD_MQD_TYPE type,
632 struct kfd_dev *dev);
633struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev);
634void device_queue_manager_uninit(struct device_queue_manager *dqm);
635struct kernel_queue *kernel_queue_init(struct kfd_dev *dev,
636 enum kfd_queue_type type);
637void kernel_queue_uninit(struct kernel_queue *kq);
638
639
640struct process_queue_node {
641 struct queue *q;
642 struct kernel_queue *kq;
643 struct list_head process_queue_list;
644};
645
646int pqm_init(struct process_queue_manager *pqm, struct kfd_process *p);
647void pqm_uninit(struct process_queue_manager *pqm);
648int pqm_create_queue(struct process_queue_manager *pqm,
649 struct kfd_dev *dev,
650 struct file *f,
651 struct queue_properties *properties,
652 unsigned int flags,
653 enum kfd_queue_type type,
654 unsigned int *qid);
655int pqm_destroy_queue(struct process_queue_manager *pqm, unsigned int qid);
656int pqm_update_queue(struct process_queue_manager *pqm, unsigned int qid,
657 struct queue_properties *p);
658struct kernel_queue *pqm_get_kernel_queue(struct process_queue_manager *pqm,
659 unsigned int qid);
660
661int amdkfd_fence_wait_timeout(unsigned int *fence_addr,
662 unsigned int fence_value,
663 unsigned long timeout);
664
665
666
667#define KFD_HIQ_TIMEOUT (500)
668
669#define KFD_FENCE_COMPLETED (100)
670#define KFD_FENCE_INIT (10)
671#define KFD_UNMAP_LATENCY (150)
672
673struct packet_manager {
674 struct device_queue_manager *dqm;
675 struct kernel_queue *priv_queue;
676 struct mutex lock;
677 bool allocated;
678 struct kfd_mem_obj *ib_buffer_obj;
679};
680
681int pm_init(struct packet_manager *pm, struct device_queue_manager *dqm);
682void pm_uninit(struct packet_manager *pm);
683int pm_send_set_resources(struct packet_manager *pm,
684 struct scheduling_resources *res);
685int pm_send_runlist(struct packet_manager *pm, struct list_head *dqm_queues);
686int pm_send_query_status(struct packet_manager *pm, uint64_t fence_address,
687 uint32_t fence_value);
688
689int pm_send_unmap_queue(struct packet_manager *pm, enum kfd_queue_type type,
690 enum kfd_preempt_type_filter mode,
691 uint32_t filter_param, bool reset,
692 unsigned int sdma_engine);
693
694void pm_release_ib(struct packet_manager *pm);
695
696uint64_t kfd_get_number_elems(struct kfd_dev *kfd);
697phys_addr_t kfd_get_process_doorbells(struct kfd_dev *dev,
698 struct kfd_process *process);
699
700
701extern const struct kfd_event_interrupt_class event_interrupt_class_cik;
702extern const struct kfd_device_global_init_class device_global_init_class_cik;
703
704enum kfd_event_wait_result {
705 KFD_WAIT_COMPLETE,
706 KFD_WAIT_TIMEOUT,
707 KFD_WAIT_ERROR
708};
709
710void kfd_event_init_process(struct kfd_process *p);
711void kfd_event_free_process(struct kfd_process *p);
712int kfd_event_mmap(struct kfd_process *process, struct vm_area_struct *vma);
713int kfd_wait_on_events(struct kfd_process *p,
714 uint32_t num_events, void __user *data,
715 bool all, uint32_t user_timeout_ms,
716 enum kfd_event_wait_result *wait_result);
717void kfd_signal_event_interrupt(unsigned int pasid, uint32_t partial_id,
718 uint32_t valid_id_bits);
719void kfd_signal_iommu_event(struct kfd_dev *dev,
720 unsigned int pasid, unsigned long address,
721 bool is_write_requested, bool is_execute_requested);
722void kfd_signal_hw_exception_event(unsigned int pasid);
723int kfd_set_event(struct kfd_process *p, uint32_t event_id);
724int kfd_reset_event(struct kfd_process *p, uint32_t event_id);
725int kfd_event_create(struct file *devkfd, struct kfd_process *p,
726 uint32_t event_type, bool auto_reset, uint32_t node_id,
727 uint32_t *event_id, uint32_t *event_trigger_data,
728 uint64_t *event_page_offset, uint32_t *event_slot_index);
729int kfd_event_destroy(struct kfd_process *p, uint32_t event_id);
730
731int dbgdev_wave_reset_wavefronts(struct kfd_dev *dev, struct kfd_process *p);
732
733#endif
734