1
2
3
4
5
6
7
8#ifndef HABANALABSP_H_
9#define HABANALABSP_H_
10
11#include "include/armcp_if.h"
12#include "include/qman_if.h"
13
14#include <linux/cdev.h>
15#include <linux/iopoll.h>
16#include <linux/irqreturn.h>
17#include <linux/dma-fence.h>
18#include <linux/dma-direction.h>
19#include <linux/scatterlist.h>
20#include <linux/hashtable.h>
21
22#define HL_NAME "habanalabs"
23
24#define HL_MMAP_CB_MASK (0x8000000000000000ull >> PAGE_SHIFT)
25
26#define HL_PENDING_RESET_PER_SEC 5
27
28#define HL_DEVICE_TIMEOUT_USEC 1000000
29
30#define HL_HEARTBEAT_PER_USEC 5000000
31
32#define HL_PLL_LOW_JOB_FREQ_USEC 5000000
33
34#define HL_ARMCP_INFO_TIMEOUT_USEC 10000000
35#define HL_ARMCP_EEPROM_TIMEOUT_USEC 10000000
36
37#define HL_PCI_ELBI_TIMEOUT_MSEC 10
38
39#define HL_MAX_QUEUES 128
40
41#define HL_MAX_JOBS_PER_CS 64
42
43
44#define HL_MAX_PENDING_CS 64
45
46
47#define MEM_HASH_TABLE_BITS 7
48
49
50#define MMU_HASH_TABLE_BITS 7
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65struct pgt_info {
66 struct hlist_node node;
67 u64 phys_addr;
68 u64 shadow_addr;
69 struct hl_ctx *ctx;
70 int num_of_ptes;
71};
72
73struct hl_device;
74struct hl_fpriv;
75
76
77
78
79
80
81
82
83
84
85enum hl_queue_type {
86 QUEUE_TYPE_NA,
87 QUEUE_TYPE_EXT,
88 QUEUE_TYPE_INT,
89 QUEUE_TYPE_CPU
90};
91
92
93
94
95
96
97
98struct hw_queue_properties {
99 enum hl_queue_type type;
100 u8 kmd_only;
101};
102
103
104
105
106
107
108enum vm_type_t {
109 VM_TYPE_USERPTR,
110 VM_TYPE_PHYS_PACK
111};
112
113
114
115
116
117
118
119
120enum hl_device_hw_state {
121 HL_DEVICE_HW_STATE_CLEAN = 0,
122 HL_DEVICE_HW_STATE_DIRTY
123};
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174struct asic_fixed_properties {
175 struct hw_queue_properties hw_queues_props[HL_MAX_QUEUES];
176 struct armcp_info armcp_info;
177 char uboot_ver[VERSION_MAX_LEN];
178 char preboot_ver[VERSION_MAX_LEN];
179 u64 sram_base_address;
180 u64 sram_end_address;
181 u64 sram_user_base_address;
182 u64 dram_base_address;
183 u64 dram_end_address;
184 u64 dram_user_base_address;
185 u64 dram_size;
186 u64 dram_pci_bar_size;
187 u64 max_power_default;
188 u64 va_space_host_start_address;
189 u64 va_space_host_end_address;
190 u64 va_space_dram_start_address;
191 u64 va_space_dram_end_address;
192 u64 dram_size_for_default_page_mapping;
193 u64 pcie_dbi_base_address;
194 u64 pcie_aux_dbi_reg_addr;
195 u64 mmu_pgt_addr;
196 u64 mmu_dram_default_page_addr;
197 u32 mmu_pgt_size;
198 u32 mmu_pte_size;
199 u32 mmu_hop_table_size;
200 u32 mmu_hop0_tables_total_size;
201 u32 dram_page_size;
202 u32 cfg_size;
203 u32 sram_size;
204 u32 max_asid;
205 u32 num_of_events;
206 u32 psoc_pci_pll_nr;
207 u32 psoc_pci_pll_nf;
208 u32 psoc_pci_pll_od;
209 u32 psoc_pci_pll_div_factor;
210 u32 high_pll;
211 u32 cb_pool_cb_cnt;
212 u32 cb_pool_cb_size;
213 u8 completion_queues_count;
214 u8 tpc_enabled_mask;
215};
216
217
218
219
220
221
222
223
224struct hl_dma_fence {
225 struct dma_fence base_fence;
226 spinlock_t lock;
227 struct hl_device *hdev;
228 u64 cs_seq;
229};
230
231
232
233
234
235#define HL_MAX_CB_SIZE 0x200000
236
237
238
239
240
241
242struct hl_cb_mgr {
243 spinlock_t cb_lock;
244 struct idr cb_handles;
245};
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264struct hl_cb {
265 struct kref refcount;
266 struct hl_device *hdev;
267 spinlock_t lock;
268 struct list_head debugfs_list;
269 struct list_head pool_list;
270 u64 kernel_address;
271 dma_addr_t bus_address;
272 u32 mmap_size;
273 u32 size;
274 u32 id;
275 u32 cs_cnt;
276 u32 ctx_id;
277 u8 mmap;
278 u8 is_pool;
279};
280
281
282
283
284
285
286struct hl_cs_job;
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307#define HL_PAGE_SIZE 4096
308
309#define HL_QUEUE_LENGTH 256
310#define HL_QUEUE_SIZE_IN_BYTES (HL_QUEUE_LENGTH * HL_BD_SIZE)
311
312
313
314
315
316#define HL_CQ_LENGTH HL_QUEUE_LENGTH
317#define HL_CQ_SIZE_IN_BYTES (HL_CQ_LENGTH * HL_CQ_ENTRY_SIZE)
318
319
320#define HL_EQ_LENGTH 64
321#define HL_EQ_SIZE_IN_BYTES (HL_EQ_LENGTH * HL_EQ_ENTRY_SIZE)
322
323
324#define HL_CPU_ACCESSIBLE_MEM_SIZE SZ_2M
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339struct hl_hw_queue {
340 struct hl_cs_job **shadow_queue;
341 enum hl_queue_type queue_type;
342 u64 kernel_address;
343 dma_addr_t bus_address;
344 u32 pi;
345 u32 ci;
346 u32 hw_queue_id;
347 u16 int_queue_len;
348 u8 valid;
349};
350
351
352
353
354
355
356
357
358
359
360
361struct hl_cq {
362 struct hl_device *hdev;
363 u64 kernel_address;
364 dma_addr_t bus_address;
365 u32 hw_queue_id;
366 u32 ci;
367 u32 pi;
368 atomic_t free_slots_cnt;
369};
370
371
372
373
374
375
376
377
378struct hl_eq {
379 struct hl_device *hdev;
380 u64 kernel_address;
381 dma_addr_t bus_address;
382 u32 ci;
383};
384
385
386
387
388
389
390
391
392
393
394
395enum hl_asic_type {
396 ASIC_INVALID,
397 ASIC_GOYA
398};
399
400struct hl_cs_parser;
401
402
403
404
405
406
407
408enum hl_pm_mng_profile {
409 PM_AUTO = 1,
410 PM_MANUAL,
411 PM_LAST
412};
413
414
415
416
417
418
419
420enum hl_pll_frequency {
421 PLL_HIGH = 1,
422 PLL_LOW,
423 PLL_LAST
424};
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502struct hl_asic_funcs {
503 int (*early_init)(struct hl_device *hdev);
504 int (*early_fini)(struct hl_device *hdev);
505 int (*late_init)(struct hl_device *hdev);
506 void (*late_fini)(struct hl_device *hdev);
507 int (*sw_init)(struct hl_device *hdev);
508 int (*sw_fini)(struct hl_device *hdev);
509 int (*hw_init)(struct hl_device *hdev);
510 void (*hw_fini)(struct hl_device *hdev, bool hard_reset);
511 void (*halt_engines)(struct hl_device *hdev, bool hard_reset);
512 int (*suspend)(struct hl_device *hdev);
513 int (*resume)(struct hl_device *hdev);
514 int (*cb_mmap)(struct hl_device *hdev, struct vm_area_struct *vma,
515 u64 kaddress, phys_addr_t paddress, u32 size);
516 void (*ring_doorbell)(struct hl_device *hdev, u32 hw_queue_id, u32 pi);
517 void (*pqe_write)(struct hl_device *hdev, __le64 *pqe,
518 struct hl_bd *bd);
519 void* (*asic_dma_alloc_coherent)(struct hl_device *hdev, size_t size,
520 dma_addr_t *dma_handle, gfp_t flag);
521 void (*asic_dma_free_coherent)(struct hl_device *hdev, size_t size,
522 void *cpu_addr, dma_addr_t dma_handle);
523 void* (*get_int_queue_base)(struct hl_device *hdev, u32 queue_id,
524 dma_addr_t *dma_handle, u16 *queue_len);
525 int (*test_queues)(struct hl_device *hdev);
526 void* (*asic_dma_pool_zalloc)(struct hl_device *hdev, size_t size,
527 gfp_t mem_flags, dma_addr_t *dma_handle);
528 void (*asic_dma_pool_free)(struct hl_device *hdev, void *vaddr,
529 dma_addr_t dma_addr);
530 void* (*cpu_accessible_dma_pool_alloc)(struct hl_device *hdev,
531 size_t size, dma_addr_t *dma_handle);
532 void (*cpu_accessible_dma_pool_free)(struct hl_device *hdev,
533 size_t size, void *vaddr);
534 void (*hl_dma_unmap_sg)(struct hl_device *hdev,
535 struct scatterlist *sgl, int nents,
536 enum dma_data_direction dir);
537 int (*cs_parser)(struct hl_device *hdev, struct hl_cs_parser *parser);
538 int (*asic_dma_map_sg)(struct hl_device *hdev,
539 struct scatterlist *sgl, int nents,
540 enum dma_data_direction dir);
541 u32 (*get_dma_desc_list_size)(struct hl_device *hdev,
542 struct sg_table *sgt);
543 void (*add_end_of_cb_packets)(struct hl_device *hdev,
544 u64 kernel_address, u32 len,
545 u64 cq_addr, u32 cq_val, u32 msix_num);
546 void (*update_eq_ci)(struct hl_device *hdev, u32 val);
547 int (*context_switch)(struct hl_device *hdev, u32 asid);
548 void (*restore_phase_topology)(struct hl_device *hdev);
549 int (*debugfs_read32)(struct hl_device *hdev, u64 addr, u32 *val);
550 int (*debugfs_write32)(struct hl_device *hdev, u64 addr, u32 val);
551 void (*add_device_attr)(struct hl_device *hdev,
552 struct attribute_group *dev_attr_grp);
553 void (*handle_eqe)(struct hl_device *hdev,
554 struct hl_eq_entry *eq_entry);
555 void (*set_pll_profile)(struct hl_device *hdev,
556 enum hl_pll_frequency freq);
557 void* (*get_events_stat)(struct hl_device *hdev, u32 *size);
558 u64 (*read_pte)(struct hl_device *hdev, u64 addr);
559 void (*write_pte)(struct hl_device *hdev, u64 addr, u64 val);
560 void (*mmu_invalidate_cache)(struct hl_device *hdev, bool is_hard);
561 void (*mmu_invalidate_cache_range)(struct hl_device *hdev, bool is_hard,
562 u32 asid, u64 va, u64 size);
563 int (*send_heartbeat)(struct hl_device *hdev);
564 int (*debug_coresight)(struct hl_device *hdev, void *data);
565 bool (*is_device_idle)(struct hl_device *hdev, u32 *mask,
566 struct seq_file *s);
567 int (*soft_reset_late_init)(struct hl_device *hdev);
568 void (*hw_queues_lock)(struct hl_device *hdev);
569 void (*hw_queues_unlock)(struct hl_device *hdev);
570 u32 (*get_pci_id)(struct hl_device *hdev);
571 int (*get_eeprom_data)(struct hl_device *hdev, void *data,
572 size_t max_size);
573 int (*send_cpu_message)(struct hl_device *hdev, u32 *msg,
574 u16 len, u32 timeout, long *result);
575 enum hl_device_hw_state (*get_hw_state)(struct hl_device *hdev);
576 int (*pci_bars_map)(struct hl_device *hdev);
577 u64 (*set_dram_bar_base)(struct hl_device *hdev, u64 addr);
578 int (*init_iatu)(struct hl_device *hdev);
579 u32 (*rreg)(struct hl_device *hdev, u32 reg);
580 void (*wreg)(struct hl_device *hdev, u32 reg, u32 val);
581 void (*halt_coresight)(struct hl_device *hdev);
582};
583
584
585
586
587
588
589#define HL_KERNEL_ASID_ID 0
590
591
592
593
594
595
596
597
598struct hl_va_range {
599 struct mutex lock;
600 struct list_head list;
601 u64 start_addr;
602 u64 end_addr;
603};
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638struct hl_ctx {
639 DECLARE_HASHTABLE(mem_hash, MEM_HASH_TABLE_BITS);
640 DECLARE_HASHTABLE(mmu_phys_hash, MMU_HASH_TABLE_BITS);
641 DECLARE_HASHTABLE(mmu_shadow_hash, MMU_HASH_TABLE_BITS);
642 struct hl_fpriv *hpriv;
643 struct hl_device *hdev;
644 struct kref refcount;
645 struct dma_fence *cs_pending[HL_MAX_PENDING_CS];
646 struct hl_va_range host_va_range;
647 struct hl_va_range dram_va_range;
648 struct mutex mem_hash_lock;
649 struct mutex mmu_lock;
650 struct list_head debugfs_list;
651 u64 cs_sequence;
652 u64 *dram_default_hops;
653 spinlock_t cs_lock;
654 atomic64_t dram_phys_mem;
655 atomic_t thread_ctx_switch_token;
656 u32 thread_ctx_switch_wait_token;
657 u32 asid;
658};
659
660
661
662
663
664
665struct hl_ctx_mgr {
666 struct mutex ctx_lock;
667 struct idr ctx_handles;
668};
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688struct hl_userptr {
689 enum vm_type_t vm_type;
690 struct list_head job_node;
691 struct frame_vector *vec;
692 struct sg_table *sgt;
693 enum dma_data_direction dir;
694 struct list_head debugfs_list;
695 u64 addr;
696 u32 size;
697 u8 dma_mapped;
698};
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719struct hl_cs {
720 u8 jobs_in_queue_cnt[HL_MAX_QUEUES];
721 struct hl_ctx *ctx;
722 struct list_head job_list;
723 spinlock_t job_lock;
724 struct kref refcount;
725 struct dma_fence *fence;
726 struct delayed_work work_tdr;
727 struct list_head mirror_node;
728 struct list_head debugfs_list;
729 u64 sequence;
730 u8 submitted;
731 u8 completed;
732 u8 timedout;
733 u8 tdr_active;
734 u8 aborted;
735};
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754struct hl_cs_job {
755 struct list_head cs_node;
756 struct hl_cs *cs;
757 struct hl_cb *user_cb;
758 struct hl_cb *patched_cb;
759 struct work_struct finish_work;
760 struct list_head userptr_list;
761 struct list_head debugfs_list;
762 u32 id;
763 u32 hw_queue_id;
764 u32 user_cb_size;
765 u32 job_cb_size;
766 u8 ext_queue;
767};
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784struct hl_cs_parser {
785 struct hl_cb *user_cb;
786 struct hl_cb *patched_cb;
787 struct list_head *job_userptr_list;
788 u64 cs_sequence;
789 u32 ctx_id;
790 u32 hw_queue_id;
791 u32 user_cb_size;
792 u32 patched_cb_size;
793 u8 ext_queue;
794 u8 job_id;
795};
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810struct hl_vm_hash_node {
811 struct hlist_node node;
812 u64 vaddr;
813 void *ptr;
814};
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831struct hl_vm_phys_pg_pack {
832 enum vm_type_t vm_type;
833 u64 *pages;
834 u64 npages;
835 u64 total_size;
836 atomic_t mapping_cnt;
837 u32 asid;
838 u32 page_size;
839 u32 flags;
840 u32 handle;
841 u32 offset;
842 u8 contiguous;
843 u8 created_from_userptr;
844};
845
846
847
848
849
850
851
852
853struct hl_vm_va_block {
854 struct list_head node;
855 u64 start;
856 u64 end;
857 u64 size;
858};
859
860
861
862
863
864
865
866
867
868
869struct hl_vm {
870 struct gen_pool *dram_pg_pool;
871 struct kref dram_pg_pool_refcount;
872 spinlock_t idr_lock;
873 struct idr phys_pg_pack_handles;
874 u8 init_done;
875};
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891struct hl_debug_params {
892 void *input;
893 void *output;
894 u32 output_size;
895 u32 reg_idx;
896 u32 op;
897 bool enable;
898};
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916struct hl_fpriv {
917 struct hl_device *hdev;
918 struct file *filp;
919 struct pid *taskpid;
920 struct hl_ctx *ctx;
921 struct hl_ctx_mgr ctx_mgr;
922 struct hl_cb_mgr cb_mgr;
923 struct list_head debugfs_list;
924 struct kref refcount;
925 struct mutex restore_phase_mutex;
926};
927
928
929
930
931
932
933
934
935
936
937
938
939struct hl_info_list {
940 const char *name;
941 int (*show)(struct seq_file *s, void *data);
942 ssize_t (*write)(struct file *file, const char __user *buf,
943 size_t count, loff_t *f_pos);
944};
945
946
947
948
949
950
951
952struct hl_debugfs_entry {
953 struct dentry *dent;
954 const struct hl_info_list *info_ent;
955 struct hl_dbg_device_entry *dev_entry;
956};
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982struct hl_dbg_device_entry {
983 struct dentry *root;
984 struct hl_device *hdev;
985 struct hl_debugfs_entry *entry_arr;
986 struct list_head file_list;
987 struct mutex file_mutex;
988 struct list_head cb_list;
989 spinlock_t cb_spinlock;
990 struct list_head cs_list;
991 spinlock_t cs_spinlock;
992 struct list_head cs_job_list;
993 spinlock_t cs_job_spinlock;
994 struct list_head userptr_list;
995 spinlock_t userptr_spinlock;
996 struct list_head ctx_mem_hash_list;
997 spinlock_t ctx_mem_hash_spinlock;
998 u64 addr;
999 u64 mmu_addr;
1000 u32 mmu_asid;
1001 u8 i2c_bus;
1002 u8 i2c_addr;
1003 u8 i2c_reg;
1004};
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014#define HL_MAX_MINORS 256
1015
1016
1017
1018
1019
1020u32 hl_rreg(struct hl_device *hdev, u32 reg);
1021void hl_wreg(struct hl_device *hdev, u32 reg, u32 val);
1022
1023#define RREG32(reg) hdev->asic_funcs->rreg(hdev, (reg))
1024#define WREG32(reg, v) hdev->asic_funcs->wreg(hdev, (reg), (v))
1025#define DREG32(reg) pr_info("REGISTER: " #reg " : 0x%08X\n", \
1026 hdev->asic_funcs->rreg(hdev, (reg)))
1027
1028#define WREG32_P(reg, val, mask) \
1029 do { \
1030 u32 tmp_ = RREG32(reg); \
1031 tmp_ &= (mask); \
1032 tmp_ |= ((val) & ~(mask)); \
1033 WREG32(reg, tmp_); \
1034 } while (0)
1035#define WREG32_AND(reg, and) WREG32_P(reg, 0, and)
1036#define WREG32_OR(reg, or) WREG32_P(reg, or, ~(or))
1037
1038#define REG_FIELD_SHIFT(reg, field) reg##_##field##_SHIFT
1039#define REG_FIELD_MASK(reg, field) reg##_##field##_MASK
1040#define WREG32_FIELD(reg, field, val) \
1041 WREG32(mm##reg, (RREG32(mm##reg) & ~REG_FIELD_MASK(reg, field)) | \
1042 (val) << REG_FIELD_SHIFT(reg, field))
1043
1044#define hl_poll_timeout(hdev, addr, val, cond, sleep_us, timeout_us) \
1045({ \
1046 ktime_t __timeout; \
1047 \
1048 if (hdev->pdev) \
1049 __timeout = ktime_add_us(ktime_get(), timeout_us); \
1050 else \
1051 __timeout = ktime_add_us(ktime_get(), (timeout_us * 10)); \
1052 might_sleep_if(sleep_us); \
1053 for (;;) { \
1054 (val) = RREG32(addr); \
1055 if (cond) \
1056 break; \
1057 if (timeout_us && ktime_compare(ktime_get(), __timeout) > 0) { \
1058 (val) = RREG32(addr); \
1059 break; \
1060 } \
1061 if (sleep_us) \
1062 usleep_range((sleep_us >> 2) + 1, sleep_us); \
1063 } \
1064 (cond) ? 0 : -ETIMEDOUT; \
1065})
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079#define hl_poll_timeout_memory(hdev, addr, val, cond, sleep_us, timeout_us, \
1080 mem_written_by_device) \
1081({ \
1082 ktime_t __timeout; \
1083 \
1084 if (hdev->pdev) \
1085 __timeout = ktime_add_us(ktime_get(), timeout_us); \
1086 else \
1087 __timeout = ktime_add_us(ktime_get(), (timeout_us * 10)); \
1088 might_sleep_if(sleep_us); \
1089 for (;;) { \
1090 \
1091 mb(); \
1092 (val) = *((u32 *) (uintptr_t) (addr)); \
1093 if (mem_written_by_device) \
1094 (val) = le32_to_cpu(val); \
1095 if (cond) \
1096 break; \
1097 if (timeout_us && ktime_compare(ktime_get(), __timeout) > 0) { \
1098 (val) = *((u32 *) (uintptr_t) (addr)); \
1099 if (mem_written_by_device) \
1100 (val) = le32_to_cpu(val); \
1101 break; \
1102 } \
1103 if (sleep_us) \
1104 usleep_range((sleep_us >> 2) + 1, sleep_us); \
1105 } \
1106 (cond) ? 0 : -ETIMEDOUT; \
1107})
1108
1109#define hl_poll_timeout_device_memory(hdev, addr, val, cond, sleep_us, \
1110 timeout_us) \
1111({ \
1112 ktime_t __timeout; \
1113 \
1114 if (hdev->pdev) \
1115 __timeout = ktime_add_us(ktime_get(), timeout_us); \
1116 else \
1117 __timeout = ktime_add_us(ktime_get(), (timeout_us * 10)); \
1118 might_sleep_if(sleep_us); \
1119 for (;;) { \
1120 (val) = readl(addr); \
1121 if (cond) \
1122 break; \
1123 if (timeout_us && ktime_compare(ktime_get(), __timeout) > 0) { \
1124 (val) = readl(addr); \
1125 break; \
1126 } \
1127 if (sleep_us) \
1128 usleep_range((sleep_us >> 2) + 1, sleep_us); \
1129 } \
1130 (cond) ? 0 : -ETIMEDOUT; \
1131})
1132
1133struct hwmon_chip_info;
1134
1135
1136
1137
1138
1139
1140struct hl_device_reset_work {
1141 struct work_struct reset_work;
1142 struct hl_device *hdev;
1143};
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224struct hl_device {
1225 struct pci_dev *pdev;
1226 void __iomem *pcie_bar[6];
1227 void __iomem *rmmio;
1228 struct cdev cdev;
1229 struct device *dev;
1230 struct delayed_work work_freq;
1231 struct delayed_work work_heartbeat;
1232 char asic_name[16];
1233 enum hl_asic_type asic_type;
1234 struct hl_cq *completion_queue;
1235 struct workqueue_struct *cq_wq;
1236 struct workqueue_struct *eq_wq;
1237 struct hl_ctx *kernel_ctx;
1238 struct hl_hw_queue *kernel_queues;
1239 struct list_head hw_queues_mirror_list;
1240 spinlock_t hw_queues_mirror_lock;
1241 struct hl_cb_mgr kernel_cb_mgr;
1242 struct hl_eq event_queue;
1243 struct dma_pool *dma_pool;
1244 void *cpu_accessible_dma_mem;
1245 dma_addr_t cpu_accessible_dma_address;
1246 struct gen_pool *cpu_accessible_dma_pool;
1247 unsigned long *asid_bitmap;
1248 struct mutex asid_mutex;
1249
1250 struct mutex fd_open_cnt_lock;
1251 struct mutex send_cpu_message_lock;
1252 struct mutex debug_lock;
1253 struct asic_fixed_properties asic_prop;
1254 const struct hl_asic_funcs *asic_funcs;
1255 void *asic_specific;
1256 struct gen_pool *mmu_pgt_pool;
1257 struct hl_vm vm;
1258 struct mutex mmu_cache_lock;
1259 void *mmu_shadow_hop0;
1260 struct device *hwmon_dev;
1261 enum hl_pm_mng_profile pm_mng_profile;
1262 struct hwmon_chip_info *hl_chip_info;
1263
1264 struct hl_dbg_device_entry hl_debugfs;
1265
1266 struct list_head cb_pool;
1267 spinlock_t cb_pool_lock;
1268
1269
1270 struct hl_ctx *user_ctx;
1271
1272 atomic64_t dram_used_mem;
1273 u64 timeout_jiffies;
1274 u64 max_power;
1275 atomic_t in_reset;
1276 atomic_t curr_pll_profile;
1277 atomic_t fd_open_cnt;
1278 atomic_t cs_active_cnt;
1279 u32 major;
1280 u32 high_pll;
1281 u32 soft_reset_cnt;
1282 u32 hard_reset_cnt;
1283 u16 id;
1284 u8 disabled;
1285 u8 late_init_done;
1286 u8 hwmon_initialized;
1287 u8 hard_reset_pending;
1288 u8 heartbeat;
1289 u8 reset_on_lockup;
1290 u8 dram_supports_virtual_memory;
1291 u8 dram_default_page_mapping;
1292 u8 init_done;
1293 u8 device_cpu_disabled;
1294 u8 dma_mask;
1295 u8 in_debug;
1296
1297
1298 u8 mmu_enable;
1299 u8 cpu_enable;
1300 u8 reset_pcilink;
1301 u8 cpu_queues_enable;
1302 u8 fw_loading;
1303 u8 pldm;
1304};
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319typedef int hl_ioctl_t(struct hl_fpriv *hpriv, void *data);
1320
1321
1322
1323
1324
1325
1326struct hl_ioctl_desc {
1327 unsigned int cmd;
1328 hl_ioctl_t *func;
1329};
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345static inline bool hl_mem_area_inside_range(u64 address, u32 size,
1346 u64 range_start_address, u64 range_end_address)
1347{
1348 u64 end_address = address + size;
1349
1350 if ((address >= range_start_address) &&
1351 (end_address <= range_end_address) &&
1352 (end_address > address))
1353 return true;
1354
1355 return false;
1356}
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368static inline bool hl_mem_area_crosses_range(u64 address, u32 size,
1369 u64 range_start_address, u64 range_end_address)
1370{
1371 u64 end_address = address + size;
1372
1373 if ((address >= range_start_address) &&
1374 (address < range_end_address))
1375 return true;
1376
1377 if ((end_address >= range_start_address) &&
1378 (end_address < range_end_address))
1379 return true;
1380
1381 if ((address < range_start_address) &&
1382 (end_address >= range_end_address))
1383 return true;
1384
1385 return false;
1386}
1387
1388int hl_device_open(struct inode *inode, struct file *filp);
1389bool hl_device_disabled_or_in_reset(struct hl_device *hdev);
1390enum hl_device_status hl_device_status(struct hl_device *hdev);
1391int hl_device_set_debug_mode(struct hl_device *hdev, bool enable);
1392int create_hdev(struct hl_device **dev, struct pci_dev *pdev,
1393 enum hl_asic_type asic_type, int minor);
1394void destroy_hdev(struct hl_device *hdev);
1395int hl_hw_queues_create(struct hl_device *hdev);
1396void hl_hw_queues_destroy(struct hl_device *hdev);
1397int hl_hw_queue_send_cb_no_cmpl(struct hl_device *hdev, u32 hw_queue_id,
1398 u32 cb_size, u64 cb_ptr);
1399int hl_hw_queue_schedule_cs(struct hl_cs *cs);
1400u32 hl_hw_queue_add_ptr(u32 ptr, u16 val);
1401void hl_hw_queue_inc_ci_kernel(struct hl_device *hdev, u32 hw_queue_id);
1402void hl_int_hw_queue_update_ci(struct hl_cs *cs);
1403void hl_hw_queue_reset(struct hl_device *hdev, bool hard_reset);
1404
1405#define hl_queue_inc_ptr(p) hl_hw_queue_add_ptr(p, 1)
1406#define hl_pi_2_offset(pi) ((pi) & (HL_QUEUE_LENGTH - 1))
1407
1408int hl_cq_init(struct hl_device *hdev, struct hl_cq *q, u32 hw_queue_id);
1409void hl_cq_fini(struct hl_device *hdev, struct hl_cq *q);
1410int hl_eq_init(struct hl_device *hdev, struct hl_eq *q);
1411void hl_eq_fini(struct hl_device *hdev, struct hl_eq *q);
1412void hl_cq_reset(struct hl_device *hdev, struct hl_cq *q);
1413void hl_eq_reset(struct hl_device *hdev, struct hl_eq *q);
1414irqreturn_t hl_irq_handler_cq(int irq, void *arg);
1415irqreturn_t hl_irq_handler_eq(int irq, void *arg);
1416u32 hl_cq_inc_ptr(u32 ptr);
1417
1418int hl_asid_init(struct hl_device *hdev);
1419void hl_asid_fini(struct hl_device *hdev);
1420unsigned long hl_asid_alloc(struct hl_device *hdev);
1421void hl_asid_free(struct hl_device *hdev, unsigned long asid);
1422
1423int hl_ctx_create(struct hl_device *hdev, struct hl_fpriv *hpriv);
1424void hl_ctx_free(struct hl_device *hdev, struct hl_ctx *ctx);
1425int hl_ctx_init(struct hl_device *hdev, struct hl_ctx *ctx, bool is_kernel_ctx);
1426void hl_ctx_do_release(struct kref *ref);
1427void hl_ctx_get(struct hl_device *hdev, struct hl_ctx *ctx);
1428int hl_ctx_put(struct hl_ctx *ctx);
1429struct dma_fence *hl_ctx_get_fence(struct hl_ctx *ctx, u64 seq);
1430void hl_ctx_mgr_init(struct hl_ctx_mgr *mgr);
1431void hl_ctx_mgr_fini(struct hl_device *hdev, struct hl_ctx_mgr *mgr);
1432
1433int hl_device_init(struct hl_device *hdev, struct class *hclass);
1434void hl_device_fini(struct hl_device *hdev);
1435int hl_device_suspend(struct hl_device *hdev);
1436int hl_device_resume(struct hl_device *hdev);
1437int hl_device_reset(struct hl_device *hdev, bool hard_reset,
1438 bool from_hard_reset_thread);
1439void hl_hpriv_get(struct hl_fpriv *hpriv);
1440void hl_hpriv_put(struct hl_fpriv *hpriv);
1441int hl_device_set_frequency(struct hl_device *hdev, enum hl_pll_frequency freq);
1442
1443int hl_build_hwmon_channel_info(struct hl_device *hdev,
1444 struct armcp_sensor *sensors_arr);
1445
1446int hl_sysfs_init(struct hl_device *hdev);
1447void hl_sysfs_fini(struct hl_device *hdev);
1448
1449int hl_hwmon_init(struct hl_device *hdev);
1450void hl_hwmon_fini(struct hl_device *hdev);
1451
1452int hl_cb_create(struct hl_device *hdev, struct hl_cb_mgr *mgr, u32 cb_size,
1453 u64 *handle, int ctx_id);
1454int hl_cb_destroy(struct hl_device *hdev, struct hl_cb_mgr *mgr, u64 cb_handle);
1455int hl_cb_mmap(struct hl_fpriv *hpriv, struct vm_area_struct *vma);
1456struct hl_cb *hl_cb_get(struct hl_device *hdev, struct hl_cb_mgr *mgr,
1457 u32 handle);
1458void hl_cb_put(struct hl_cb *cb);
1459void hl_cb_mgr_init(struct hl_cb_mgr *mgr);
1460void hl_cb_mgr_fini(struct hl_device *hdev, struct hl_cb_mgr *mgr);
1461struct hl_cb *hl_cb_kernel_create(struct hl_device *hdev, u32 cb_size);
1462int hl_cb_pool_init(struct hl_device *hdev);
1463int hl_cb_pool_fini(struct hl_device *hdev);
1464
1465void hl_cs_rollback_all(struct hl_device *hdev);
1466struct hl_cs_job *hl_cs_allocate_job(struct hl_device *hdev, bool ext_queue);
1467
1468void goya_set_asic_funcs(struct hl_device *hdev);
1469
1470int hl_vm_ctx_init(struct hl_ctx *ctx);
1471void hl_vm_ctx_fini(struct hl_ctx *ctx);
1472
1473int hl_vm_init(struct hl_device *hdev);
1474void hl_vm_fini(struct hl_device *hdev);
1475
1476int hl_pin_host_memory(struct hl_device *hdev, u64 addr, u64 size,
1477 struct hl_userptr *userptr);
1478int hl_unpin_host_memory(struct hl_device *hdev, struct hl_userptr *userptr);
1479void hl_userptr_delete_list(struct hl_device *hdev,
1480 struct list_head *userptr_list);
1481bool hl_userptr_is_pinned(struct hl_device *hdev, u64 addr, u32 size,
1482 struct list_head *userptr_list,
1483 struct hl_userptr **userptr);
1484
1485int hl_mmu_init(struct hl_device *hdev);
1486void hl_mmu_fini(struct hl_device *hdev);
1487int hl_mmu_ctx_init(struct hl_ctx *ctx);
1488void hl_mmu_ctx_fini(struct hl_ctx *ctx);
1489int hl_mmu_map(struct hl_ctx *ctx, u64 virt_addr, u64 phys_addr, u32 page_size);
1490int hl_mmu_unmap(struct hl_ctx *ctx, u64 virt_addr, u32 page_size);
1491void hl_mmu_swap_out(struct hl_ctx *ctx);
1492void hl_mmu_swap_in(struct hl_ctx *ctx);
1493
1494int hl_fw_push_fw_to_device(struct hl_device *hdev, const char *fw_name,
1495 void __iomem *dst);
1496int hl_fw_send_pci_access_msg(struct hl_device *hdev, u32 opcode);
1497int hl_fw_send_cpu_message(struct hl_device *hdev, u32 hw_queue_id, u32 *msg,
1498 u16 len, u32 timeout, long *result);
1499int hl_fw_test_cpu_queue(struct hl_device *hdev);
1500void *hl_fw_cpu_accessible_dma_pool_alloc(struct hl_device *hdev, size_t size,
1501 dma_addr_t *dma_handle);
1502void hl_fw_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size,
1503 void *vaddr);
1504int hl_fw_send_heartbeat(struct hl_device *hdev);
1505int hl_fw_armcp_info_get(struct hl_device *hdev);
1506int hl_fw_get_eeprom_data(struct hl_device *hdev, void *data, size_t max_size);
1507
1508int hl_pci_bars_map(struct hl_device *hdev, const char * const name[3],
1509 bool is_wc[3]);
1510int hl_pci_iatu_write(struct hl_device *hdev, u32 addr, u32 data);
1511int hl_pci_set_dram_bar_base(struct hl_device *hdev, u8 inbound_region, u8 bar,
1512 u64 addr);
1513int hl_pci_init_iatu(struct hl_device *hdev, u64 sram_base_address,
1514 u64 dram_base_address, u64 host_phys_base_address,
1515 u64 host_phys_size);
1516int hl_pci_init(struct hl_device *hdev, u8 dma_mask);
1517void hl_pci_fini(struct hl_device *hdev);
1518int hl_pci_set_dma_mask(struct hl_device *hdev, u8 dma_mask);
1519
1520long hl_get_frequency(struct hl_device *hdev, u32 pll_index, bool curr);
1521void hl_set_frequency(struct hl_device *hdev, u32 pll_index, u64 freq);
1522long hl_get_temperature(struct hl_device *hdev, int sensor_index, u32 attr);
1523long hl_get_voltage(struct hl_device *hdev, int sensor_index, u32 attr);
1524long hl_get_current(struct hl_device *hdev, int sensor_index, u32 attr);
1525long hl_get_fan_speed(struct hl_device *hdev, int sensor_index, u32 attr);
1526long hl_get_pwm_info(struct hl_device *hdev, int sensor_index, u32 attr);
1527void hl_set_pwm_info(struct hl_device *hdev, int sensor_index, u32 attr,
1528 long value);
1529u64 hl_get_max_power(struct hl_device *hdev);
1530void hl_set_max_power(struct hl_device *hdev, u64 value);
1531
1532#ifdef CONFIG_DEBUG_FS
1533
1534void hl_debugfs_init(void);
1535void hl_debugfs_fini(void);
1536void hl_debugfs_add_device(struct hl_device *hdev);
1537void hl_debugfs_remove_device(struct hl_device *hdev);
1538void hl_debugfs_add_file(struct hl_fpriv *hpriv);
1539void hl_debugfs_remove_file(struct hl_fpriv *hpriv);
1540void hl_debugfs_add_cb(struct hl_cb *cb);
1541void hl_debugfs_remove_cb(struct hl_cb *cb);
1542void hl_debugfs_add_cs(struct hl_cs *cs);
1543void hl_debugfs_remove_cs(struct hl_cs *cs);
1544void hl_debugfs_add_job(struct hl_device *hdev, struct hl_cs_job *job);
1545void hl_debugfs_remove_job(struct hl_device *hdev, struct hl_cs_job *job);
1546void hl_debugfs_add_userptr(struct hl_device *hdev, struct hl_userptr *userptr);
1547void hl_debugfs_remove_userptr(struct hl_device *hdev,
1548 struct hl_userptr *userptr);
1549void hl_debugfs_add_ctx_mem_hash(struct hl_device *hdev, struct hl_ctx *ctx);
1550void hl_debugfs_remove_ctx_mem_hash(struct hl_device *hdev, struct hl_ctx *ctx);
1551
1552#else
1553
1554static inline void __init hl_debugfs_init(void)
1555{
1556}
1557
1558static inline void hl_debugfs_fini(void)
1559{
1560}
1561
1562static inline void hl_debugfs_add_device(struct hl_device *hdev)
1563{
1564}
1565
1566static inline void hl_debugfs_remove_device(struct hl_device *hdev)
1567{
1568}
1569
1570static inline void hl_debugfs_add_file(struct hl_fpriv *hpriv)
1571{
1572}
1573
1574static inline void hl_debugfs_remove_file(struct hl_fpriv *hpriv)
1575{
1576}
1577
1578static inline void hl_debugfs_add_cb(struct hl_cb *cb)
1579{
1580}
1581
1582static inline void hl_debugfs_remove_cb(struct hl_cb *cb)
1583{
1584}
1585
1586static inline void hl_debugfs_add_cs(struct hl_cs *cs)
1587{
1588}
1589
1590static inline void hl_debugfs_remove_cs(struct hl_cs *cs)
1591{
1592}
1593
1594static inline void hl_debugfs_add_job(struct hl_device *hdev,
1595 struct hl_cs_job *job)
1596{
1597}
1598
1599static inline void hl_debugfs_remove_job(struct hl_device *hdev,
1600 struct hl_cs_job *job)
1601{
1602}
1603
1604static inline void hl_debugfs_add_userptr(struct hl_device *hdev,
1605 struct hl_userptr *userptr)
1606{
1607}
1608
1609static inline void hl_debugfs_remove_userptr(struct hl_device *hdev,
1610 struct hl_userptr *userptr)
1611{
1612}
1613
1614static inline void hl_debugfs_add_ctx_mem_hash(struct hl_device *hdev,
1615 struct hl_ctx *ctx)
1616{
1617}
1618
1619static inline void hl_debugfs_remove_ctx_mem_hash(struct hl_device *hdev,
1620 struct hl_ctx *ctx)
1621{
1622}
1623
1624#endif
1625
1626
1627long hl_ioctl(struct file *filep, unsigned int cmd, unsigned long arg);
1628int hl_cb_ioctl(struct hl_fpriv *hpriv, void *data);
1629int hl_cs_ioctl(struct hl_fpriv *hpriv, void *data);
1630int hl_cs_wait_ioctl(struct hl_fpriv *hpriv, void *data);
1631int hl_mem_ioctl(struct hl_fpriv *hpriv, void *data);
1632
1633#endif
1634