1
2
3
4
5
6
7
8#ifndef HABANALABSP_H_
9#define HABANALABSP_H_
10
11#include "../include/common/cpucp_if.h"
12#include "../include/common/qman_if.h"
13#include "../include/hw_ip/mmu/mmu_general.h"
14#include <uapi/misc/habanalabs.h>
15
16#include <linux/cdev.h>
17#include <linux/iopoll.h>
18#include <linux/irqreturn.h>
19#include <linux/dma-direction.h>
20#include <linux/scatterlist.h>
21#include <linux/hashtable.h>
22#include <linux/debugfs.h>
23#include <linux/rwsem.h>
24#include <linux/bitfield.h>
25#include <linux/genalloc.h>
26#include <linux/sched/signal.h>
27#include <linux/io-64-nonatomic-lo-hi.h>
28#include <linux/coresight.h>
29
30#define HL_NAME "habanalabs"
31
32
33
34
35
36
37
38
39#define HL_MMAP_TYPE_SHIFT (61 - PAGE_SHIFT)
40#define HL_MMAP_TYPE_MASK (0x7ull << HL_MMAP_TYPE_SHIFT)
41#define HL_MMAP_TYPE_BLOCK (0x4ull << HL_MMAP_TYPE_SHIFT)
42#define HL_MMAP_TYPE_CB (0x2ull << HL_MMAP_TYPE_SHIFT)
43
44#define HL_MMAP_OFFSET_VALUE_MASK (0x1FFFFFFFFFFFull >> PAGE_SHIFT)
45#define HL_MMAP_OFFSET_VALUE_GET(off) (off & HL_MMAP_OFFSET_VALUE_MASK)
46
47#define HL_PENDING_RESET_PER_SEC 10
48#define HL_PENDING_RESET_MAX_TRIALS 60
49#define HL_PENDING_RESET_LONG_SEC 60
50
51#define HL_HARD_RESET_MAX_TIMEOUT 120
52#define HL_PLDM_HARD_RESET_MAX_TIMEOUT (HL_HARD_RESET_MAX_TIMEOUT * 3)
53
54#define HL_DEVICE_TIMEOUT_USEC 1000000
55
56#define HL_HEARTBEAT_PER_USEC 5000000
57
58#define HL_PLL_LOW_JOB_FREQ_USEC 5000000
59
60#define HL_CPUCP_INFO_TIMEOUT_USEC 10000000
61#define HL_CPUCP_EEPROM_TIMEOUT_USEC 10000000
62
63#define HL_PCI_ELBI_TIMEOUT_MSEC 10
64
65#define HL_SIM_MAX_TIMEOUT_US 10000000
66
67#define HL_COMMON_USER_INTERRUPT_ID 0xFFF
68
69#define HL_STATE_DUMP_HIST_LEN 5
70
71#define OBJ_NAMES_HASH_TABLE_BITS 7
72#define SYNC_TO_ENGINE_HASH_TABLE_BITS 7
73
74
75#define MEM_HASH_TABLE_BITS 7
76
77
78#define MMU_HASH_TABLE_BITS 7
79
80
81
82
83
84
85
86enum hl_mmu_page_table_location {
87 MMU_DR_PGT = 0,
88 MMU_HR_PGT,
89 MMU_NUM_PGT_LOCATIONS
90};
91
92
93
94
95
96#define HL_RSVD_SOBS 2
97#define HL_RSVD_MONS 1
98
99
100
101
102#define HL_COLLECTIVE_RSVD_MSTR_MONS 2
103
104#define HL_MAX_SOB_VAL (1 << 15)
105
106#define IS_POWER_OF_2(n) (n != 0 && ((n & (n - 1)) == 0))
107#define IS_MAX_PENDING_CS_VALID(n) (IS_POWER_OF_2(n) && (n > 1))
108
109#define HL_PCI_NUM_BARS 6
110
111#define HL_MAX_DCORES 4
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136#define HL_RESET_HARD (1 << 0)
137#define HL_RESET_FROM_RESET_THREAD (1 << 1)
138#define HL_RESET_HEARTBEAT (1 << 2)
139#define HL_RESET_TDR (1 << 3)
140#define HL_RESET_DEVICE_RELEASE (1 << 4)
141#define HL_RESET_FW (1 << 5)
142
143#define HL_MAX_SOBS_PER_MONITOR 8
144
145
146
147
148
149
150
151
152
153
154
155struct hl_gen_wait_properties {
156 void *data;
157 u32 q_idx;
158 u32 size;
159 u16 sob_base;
160 u16 sob_val;
161 u16 mon_id;
162 u8 sob_mask;
163};
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178struct pgt_info {
179 struct hlist_node node;
180 u64 phys_addr;
181 u64 shadow_addr;
182 struct hl_ctx *ctx;
183 int num_of_ptes;
184};
185
186struct hl_device;
187struct hl_fpriv;
188
189
190
191
192
193
194enum hl_pci_match_mode {
195 PCI_ADDRESS_MATCH_MODE,
196 PCI_BAR_MATCH_MODE
197};
198
199
200
201
202
203
204
205enum hl_fw_component {
206 FW_COMP_BOOT_FIT,
207 FW_COMP_PREBOOT,
208 FW_COMP_LINUX,
209};
210
211
212
213
214
215
216
217
218
219enum hl_fw_types {
220 FW_TYPE_LINUX = 0x1,
221 FW_TYPE_BOOT_CPU = 0x2,
222 FW_TYPE_PREBOOT_CPU = 0x4,
223 FW_TYPE_ALL_TYPES =
224 (FW_TYPE_LINUX | FW_TYPE_BOOT_CPU | FW_TYPE_PREBOOT_CPU)
225};
226
227
228
229
230
231
232
233
234
235
236
237
238enum hl_queue_type {
239 QUEUE_TYPE_NA,
240 QUEUE_TYPE_EXT,
241 QUEUE_TYPE_INT,
242 QUEUE_TYPE_CPU,
243 QUEUE_TYPE_HW
244};
245
246enum hl_cs_type {
247 CS_TYPE_DEFAULT,
248 CS_TYPE_SIGNAL,
249 CS_TYPE_WAIT,
250 CS_TYPE_COLLECTIVE_WAIT,
251 CS_RESERVE_SIGNALS,
252 CS_UNRESERVE_SIGNALS
253};
254
255
256
257
258
259
260
261
262
263struct hl_inbound_pci_region {
264 enum hl_pci_match_mode mode;
265 u64 addr;
266 u64 size;
267 u64 offset_in_bar;
268 u8 bar;
269};
270
271
272
273
274
275
276struct hl_outbound_pci_region {
277 u64 addr;
278 u64 size;
279};
280
281
282
283
284
285
286
287enum queue_cb_alloc_flags {
288 CB_ALLOC_KERNEL = 0x1,
289 CB_ALLOC_USER = 0x2
290};
291
292
293
294
295
296
297
298
299
300
301struct hl_hw_sob {
302 struct hl_device *hdev;
303 struct kref kref;
304 u32 sob_id;
305 u32 sob_addr;
306 u32 q_idx;
307 bool need_reset;
308};
309
310enum hl_collective_mode {
311 HL_COLLECTIVE_NOT_SUPPORTED = 0x0,
312 HL_COLLECTIVE_MASTER = 0x1,
313 HL_COLLECTIVE_SLAVE = 0x2
314};
315
316
317
318
319
320
321
322
323
324
325
326
327
328struct hw_queue_properties {
329 enum hl_queue_type type;
330 enum queue_cb_alloc_flags cb_alloc_flags;
331 enum hl_collective_mode collective_mode;
332 u8 driver_only;
333 u8 supports_sync_stream;
334};
335
336
337
338
339
340
341enum vm_type {
342 VM_TYPE_USERPTR = 0x1,
343 VM_TYPE_PHYS_PACK = 0x2
344};
345
346
347
348
349
350
351
352
353enum hl_device_hw_state {
354 HL_DEVICE_HW_STATE_CLEAN = 0,
355 HL_DEVICE_HW_STATE_DIRTY
356};
357
358#define HL_MMU_VA_ALIGNMENT_NOT_NEEDED 0
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381struct hl_mmu_properties {
382 u64 start_addr;
383 u64 end_addr;
384 u64 hop0_shift;
385 u64 hop1_shift;
386 u64 hop2_shift;
387 u64 hop3_shift;
388 u64 hop4_shift;
389 u64 hop5_shift;
390 u64 hop0_mask;
391 u64 hop1_mask;
392 u64 hop2_mask;
393 u64 hop3_mask;
394 u64 hop4_mask;
395 u64 hop5_mask;
396 u32 page_size;
397 u32 num_hops;
398 u8 host_resident;
399};
400
401
402
403
404
405
406struct hl_hints_range {
407 u64 start_addr;
408 u64 end_addr;
409};
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516struct asic_fixed_properties {
517 struct hw_queue_properties *hw_queues_props;
518 struct cpucp_info cpucp_info;
519 char uboot_ver[VERSION_MAX_LEN];
520 char preboot_ver[VERSION_MAX_LEN];
521 struct hl_mmu_properties dmmu;
522 struct hl_mmu_properties pmmu;
523 struct hl_mmu_properties pmmu_huge;
524 struct hl_hints_range hints_dram_reserved_va_range;
525 struct hl_hints_range hints_host_reserved_va_range;
526 struct hl_hints_range hints_host_hpage_reserved_va_range;
527 u64 sram_base_address;
528 u64 sram_end_address;
529 u64 sram_user_base_address;
530 u64 dram_base_address;
531 u64 dram_end_address;
532 u64 dram_user_base_address;
533 u64 dram_size;
534 u64 dram_pci_bar_size;
535 u64 max_power_default;
536 u64 dc_power_default;
537 u64 dram_size_for_default_page_mapping;
538 u64 pcie_dbi_base_address;
539 u64 pcie_aux_dbi_reg_addr;
540 u64 mmu_pgt_addr;
541 u64 mmu_dram_default_page_addr;
542 u64 cb_va_start_addr;
543 u64 cb_va_end_addr;
544 u64 dram_hints_align_mask;
545 u64 device_dma_offset_for_host_access;
546 u32 mmu_pgt_size;
547 u32 mmu_pte_size;
548 u32 mmu_hop_table_size;
549 u32 mmu_hop0_tables_total_size;
550 u32 dram_page_size;
551 u32 cfg_size;
552 u32 sram_size;
553 u32 max_asid;
554 u32 num_of_events;
555 u32 psoc_pci_pll_nr;
556 u32 psoc_pci_pll_nf;
557 u32 psoc_pci_pll_od;
558 u32 psoc_pci_pll_div_factor;
559 u32 psoc_timestamp_frequency;
560 u32 high_pll;
561 u32 cb_pool_cb_cnt;
562 u32 cb_pool_cb_size;
563 u32 max_pending_cs;
564 u32 max_queues;
565 u32 fw_preboot_cpu_boot_dev_sts0;
566 u32 fw_preboot_cpu_boot_dev_sts1;
567 u32 fw_bootfit_cpu_boot_dev_sts0;
568 u32 fw_bootfit_cpu_boot_dev_sts1;
569 u32 fw_app_cpu_boot_dev_sts0;
570 u32 fw_app_cpu_boot_dev_sts1;
571 u16 collective_first_sob;
572 u16 collective_first_mon;
573 u16 sync_stream_first_sob;
574 u16 sync_stream_first_mon;
575 u16 first_available_user_sob[HL_MAX_DCORES];
576 u16 first_available_user_mon[HL_MAX_DCORES];
577 u16 first_available_user_msix_interrupt;
578 u16 first_available_cq[HL_MAX_DCORES];
579 u16 user_interrupt_count;
580 u16 server_type;
581 u8 tpc_enabled_mask;
582 u8 completion_queues_count;
583 u8 fw_security_enabled;
584 u8 fw_cpu_boot_dev_sts0_valid;
585 u8 fw_cpu_boot_dev_sts1_valid;
586 u8 dram_supports_virtual_memory;
587 u8 hard_reset_done_by_fw;
588 u8 num_functional_hbms;
589 u8 hints_range_reservation;
590 u8 iatu_done_by_fw;
591 u8 dynamic_fw_load;
592 u8 gic_interrupts_enable;
593};
594
595
596
597
598
599
600
601
602
603
604
605struct hl_fence {
606 struct completion completion;
607 struct kref refcount;
608 u64 cs_sequence;
609 u32 stream_master_qid_map;
610 int error;
611 ktime_t timestamp;
612};
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628struct hl_cs_compl {
629 struct hl_fence base_fence;
630 spinlock_t lock;
631 struct hl_device *hdev;
632 struct hl_hw_sob *hw_sob;
633 struct hl_cs_encaps_sig_handle *encaps_sig_hdl;
634 u64 cs_seq;
635 enum hl_cs_type type;
636 u16 sob_val;
637 u16 sob_group;
638 bool encaps_signals;
639};
640
641
642
643
644
645
646
647
648
649
650struct hl_cb_mgr {
651 spinlock_t cb_lock;
652 struct idr cb_handles;
653};
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676struct hl_cb {
677 struct kref refcount;
678 struct hl_device *hdev;
679 struct hl_ctx *ctx;
680 spinlock_t lock;
681 struct list_head debugfs_list;
682 struct list_head pool_list;
683 struct list_head va_block_list;
684 u64 id;
685 void *kernel_address;
686 dma_addr_t bus_address;
687 u32 mmap_size;
688 u32 size;
689 atomic_t cs_cnt;
690 u8 mmap;
691 u8 is_pool;
692 u8 is_internal;
693 u8 is_mmu_mapped;
694};
695
696
697
698
699
700
701struct hl_cs;
702struct hl_cs_job;
703
704
705#define HL_QUEUE_LENGTH 4096
706#define HL_QUEUE_SIZE_IN_BYTES (HL_QUEUE_LENGTH * HL_BD_SIZE)
707
708#if (HL_MAX_JOBS_PER_CS > HL_QUEUE_LENGTH)
709#error "HL_QUEUE_LENGTH must be greater than HL_MAX_JOBS_PER_CS"
710#endif
711
712
713#define HL_CQ_LENGTH HL_QUEUE_LENGTH
714#define HL_CQ_SIZE_IN_BYTES (HL_CQ_LENGTH * HL_CQ_ENTRY_SIZE)
715
716
717#define HL_EQ_LENGTH 64
718#define HL_EQ_SIZE_IN_BYTES (HL_EQ_LENGTH * HL_EQ_ENTRY_SIZE)
719
720
721#define HL_CPU_ACCESSIBLE_MEM_SIZE SZ_2M
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739struct hl_sync_stream_properties {
740 struct hl_hw_sob hw_sob[HL_RSVD_SOBS];
741 u16 next_sob_val;
742 u16 base_sob_id;
743 u16 base_mon_id;
744 u16 collective_mstr_mon_id[HL_COLLECTIVE_RSVD_MSTR_MONS];
745 u16 collective_slave_mon_id;
746 u16 collective_sob_id;
747 u8 curr_sob_offset;
748};
749
750
751
752
753
754
755
756struct hl_encaps_signals_mgr {
757 spinlock_t lock;
758 struct idr handles;
759};
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779struct hl_hw_queue {
780 struct hl_cs_job **shadow_queue;
781 struct hl_sync_stream_properties sync_stream_prop;
782 enum hl_queue_type queue_type;
783 enum hl_collective_mode collective_mode;
784 void *kernel_address;
785 dma_addr_t bus_address;
786 u32 pi;
787 atomic_t ci;
788 u32 hw_queue_id;
789 u32 cq_id;
790 u32 msi_vec;
791 u16 int_queue_len;
792 u8 valid;
793 u8 supports_sync_stream;
794};
795
796
797
798
799
800
801
802
803
804
805
806
807struct hl_cq {
808 struct hl_device *hdev;
809 void *kernel_address;
810 dma_addr_t bus_address;
811 u32 cq_idx;
812 u32 hw_queue_id;
813 u32 ci;
814 u32 pi;
815 atomic_t free_slots_cnt;
816};
817
818
819
820
821
822
823
824
825struct hl_user_interrupt {
826 struct hl_device *hdev;
827 struct list_head wait_list_head;
828 spinlock_t wait_list_lock;
829 u32 interrupt_id;
830};
831
832
833
834
835
836
837
838struct hl_user_pending_interrupt {
839 struct list_head wait_list_node;
840 struct hl_fence fence;
841};
842
843
844
845
846
847
848
849
850
851
852
853
854
855struct hl_eq {
856 struct hl_device *hdev;
857 void *kernel_address;
858 dma_addr_t bus_address;
859 u32 ci;
860 u32 prev_eqe_index;
861 bool check_eqe_index;
862};
863
864
865
866
867
868
869
870
871
872
873
874
875
876enum hl_asic_type {
877 ASIC_INVALID,
878 ASIC_GOYA,
879 ASIC_GAUDI,
880 ASIC_GAUDI_SEC
881};
882
883struct hl_cs_parser;
884
885
886
887
888
889
890
891enum hl_pm_mng_profile {
892 PM_AUTO = 1,
893 PM_MANUAL,
894 PM_LAST
895};
896
897
898
899
900
901
902
903enum hl_pll_frequency {
904 PLL_HIGH = 1,
905 PLL_LOW,
906 PLL_LAST
907};
908
909#define PLL_REF_CLK 50
910
911enum div_select_defs {
912 DIV_SEL_REF_CLK = 0,
913 DIV_SEL_PLL_CLK = 1,
914 DIV_SEL_DIVIDED_REF = 2,
915 DIV_SEL_DIVIDED_PLL = 3,
916};
917
918enum pci_region {
919 PCI_REGION_CFG,
920 PCI_REGION_SRAM,
921 PCI_REGION_DRAM,
922 PCI_REGION_SP_SRAM,
923 PCI_REGION_NUMBER,
924};
925
926
927
928
929
930
931
932
933
934
935struct pci_mem_region {
936 u64 region_base;
937 u64 region_size;
938 u64 bar_size;
939 u64 offset_in_bar;
940 u8 bar_id;
941 u8 used;
942};
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960struct static_fw_load_mgr {
961 u64 preboot_version_max_off;
962 u64 boot_fit_version_max_off;
963 u32 kmd_msg_to_cpu_reg;
964 u32 cpu_cmd_status_to_host_reg;
965 u32 cpu_boot_status_reg;
966 u32 cpu_boot_dev_status0_reg;
967 u32 cpu_boot_dev_status1_reg;
968 u32 boot_err0_reg;
969 u32 boot_err1_reg;
970 u32 preboot_version_offset_reg;
971 u32 boot_fit_version_offset_reg;
972 u32 sram_offset_mask;
973 u32 cpu_reset_wait_msec;
974};
975
976
977
978
979
980
981
982struct fw_response {
983 u32 ram_offset;
984 u8 ram_type;
985 u8 status;
986};
987
988
989
990
991
992
993
994
995
996struct dynamic_fw_load_mgr {
997 struct fw_response response;
998 struct lkd_fw_comms_desc comm_desc;
999 struct pci_mem_region *image_region;
1000 size_t fw_image_size;
1001 u32 wait_for_bl_timeout;
1002};
1003
1004
1005
1006
1007
1008
1009
1010struct fw_image_props {
1011 char *image_name;
1012 u32 src_off;
1013 u32 copy_size;
1014};
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029struct fw_load_mgr {
1030 union {
1031 struct dynamic_fw_load_mgr dynamic_loader;
1032 struct static_fw_load_mgr static_loader;
1033 };
1034 struct fw_image_props boot_fit_img;
1035 struct fw_image_props linux_img;
1036 u32 cpu_timeout;
1037 u32 boot_fit_timeout;
1038 u8 skip_bmc;
1039 u8 sram_bar_id;
1040 u8 dram_bar_id;
1041 u8 linux_loaded;
1042};
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170struct hl_asic_funcs {
1171 int (*early_init)(struct hl_device *hdev);
1172 int (*early_fini)(struct hl_device *hdev);
1173 int (*late_init)(struct hl_device *hdev);
1174 void (*late_fini)(struct hl_device *hdev);
1175 int (*sw_init)(struct hl_device *hdev);
1176 int (*sw_fini)(struct hl_device *hdev);
1177 int (*hw_init)(struct hl_device *hdev);
1178 void (*hw_fini)(struct hl_device *hdev, bool hard_reset, bool fw_reset);
1179 void (*halt_engines)(struct hl_device *hdev, bool hard_reset, bool fw_reset);
1180 int (*suspend)(struct hl_device *hdev);
1181 int (*resume)(struct hl_device *hdev);
1182 int (*mmap)(struct hl_device *hdev, struct vm_area_struct *vma,
1183 void *cpu_addr, dma_addr_t dma_addr, size_t size);
1184 void (*ring_doorbell)(struct hl_device *hdev, u32 hw_queue_id, u32 pi);
1185 void (*pqe_write)(struct hl_device *hdev, __le64 *pqe,
1186 struct hl_bd *bd);
1187 void* (*asic_dma_alloc_coherent)(struct hl_device *hdev, size_t size,
1188 dma_addr_t *dma_handle, gfp_t flag);
1189 void (*asic_dma_free_coherent)(struct hl_device *hdev, size_t size,
1190 void *cpu_addr, dma_addr_t dma_handle);
1191 int (*scrub_device_mem)(struct hl_device *hdev, u64 addr, u64 size);
1192 void* (*get_int_queue_base)(struct hl_device *hdev, u32 queue_id,
1193 dma_addr_t *dma_handle, u16 *queue_len);
1194 int (*test_queues)(struct hl_device *hdev);
1195 void* (*asic_dma_pool_zalloc)(struct hl_device *hdev, size_t size,
1196 gfp_t mem_flags, dma_addr_t *dma_handle);
1197 void (*asic_dma_pool_free)(struct hl_device *hdev, void *vaddr,
1198 dma_addr_t dma_addr);
1199 void* (*cpu_accessible_dma_pool_alloc)(struct hl_device *hdev,
1200 size_t size, dma_addr_t *dma_handle);
1201 void (*cpu_accessible_dma_pool_free)(struct hl_device *hdev,
1202 size_t size, void *vaddr);
1203 void (*hl_dma_unmap_sg)(struct hl_device *hdev,
1204 struct scatterlist *sgl, int nents,
1205 enum dma_data_direction dir);
1206 int (*cs_parser)(struct hl_device *hdev, struct hl_cs_parser *parser);
1207 int (*asic_dma_map_sg)(struct hl_device *hdev,
1208 struct scatterlist *sgl, int nents,
1209 enum dma_data_direction dir);
1210 u32 (*get_dma_desc_list_size)(struct hl_device *hdev,
1211 struct sg_table *sgt);
1212 void (*add_end_of_cb_packets)(struct hl_device *hdev,
1213 void *kernel_address, u32 len,
1214 u64 cq_addr, u32 cq_val, u32 msix_num,
1215 bool eb);
1216 void (*update_eq_ci)(struct hl_device *hdev, u32 val);
1217 int (*context_switch)(struct hl_device *hdev, u32 asid);
1218 void (*restore_phase_topology)(struct hl_device *hdev);
1219 int (*debugfs_read32)(struct hl_device *hdev, u64 addr,
1220 bool user_address, u32 *val);
1221 int (*debugfs_write32)(struct hl_device *hdev, u64 addr,
1222 bool user_address, u32 val);
1223 int (*debugfs_read64)(struct hl_device *hdev, u64 addr,
1224 bool user_address, u64 *val);
1225 int (*debugfs_write64)(struct hl_device *hdev, u64 addr,
1226 bool user_address, u64 val);
1227 int (*debugfs_read_dma)(struct hl_device *hdev, u64 addr, u32 size,
1228 void *blob_addr);
1229 void (*add_device_attr)(struct hl_device *hdev,
1230 struct attribute_group *dev_attr_grp);
1231 void (*handle_eqe)(struct hl_device *hdev,
1232 struct hl_eq_entry *eq_entry);
1233 void (*set_pll_profile)(struct hl_device *hdev,
1234 enum hl_pll_frequency freq);
1235 void* (*get_events_stat)(struct hl_device *hdev, bool aggregate,
1236 u32 *size);
1237 u64 (*read_pte)(struct hl_device *hdev, u64 addr);
1238 void (*write_pte)(struct hl_device *hdev, u64 addr, u64 val);
1239 int (*mmu_invalidate_cache)(struct hl_device *hdev, bool is_hard,
1240 u32 flags);
1241 int (*mmu_invalidate_cache_range)(struct hl_device *hdev, bool is_hard,
1242 u32 flags, u32 asid, u64 va, u64 size);
1243 int (*send_heartbeat)(struct hl_device *hdev);
1244 void (*set_clock_gating)(struct hl_device *hdev);
1245 void (*disable_clock_gating)(struct hl_device *hdev);
1246 int (*debug_coresight)(struct hl_device *hdev, void *data);
1247 bool (*is_device_idle)(struct hl_device *hdev, u64 *mask_arr,
1248 u8 mask_len, struct seq_file *s);
1249 int (*soft_reset_late_init)(struct hl_device *hdev);
1250 void (*hw_queues_lock)(struct hl_device *hdev);
1251 void (*hw_queues_unlock)(struct hl_device *hdev);
1252 u32 (*get_pci_id)(struct hl_device *hdev);
1253 int (*get_eeprom_data)(struct hl_device *hdev, void *data,
1254 size_t max_size);
1255 int (*send_cpu_message)(struct hl_device *hdev, u32 *msg,
1256 u16 len, u32 timeout, u64 *result);
1257 int (*pci_bars_map)(struct hl_device *hdev);
1258 int (*init_iatu)(struct hl_device *hdev);
1259 u32 (*rreg)(struct hl_device *hdev, u32 reg);
1260 void (*wreg)(struct hl_device *hdev, u32 reg, u32 val);
1261 void (*halt_coresight)(struct hl_device *hdev);
1262 int (*ctx_init)(struct hl_ctx *ctx);
1263 void (*ctx_fini)(struct hl_ctx *ctx);
1264 int (*get_clk_rate)(struct hl_device *hdev, u32 *cur_clk, u32 *max_clk);
1265 u32 (*get_queue_id_for_cq)(struct hl_device *hdev, u32 cq_idx);
1266 int (*load_firmware_to_device)(struct hl_device *hdev);
1267 int (*load_boot_fit_to_device)(struct hl_device *hdev);
1268 u32 (*get_signal_cb_size)(struct hl_device *hdev);
1269 u32 (*get_wait_cb_size)(struct hl_device *hdev);
1270 u32 (*gen_signal_cb)(struct hl_device *hdev, void *data, u16 sob_id,
1271 u32 size, bool eb);
1272 u32 (*gen_wait_cb)(struct hl_device *hdev,
1273 struct hl_gen_wait_properties *prop);
1274 void (*reset_sob)(struct hl_device *hdev, void *data);
1275 void (*reset_sob_group)(struct hl_device *hdev, u16 sob_group);
1276 void (*set_dma_mask_from_fw)(struct hl_device *hdev);
1277 u64 (*get_device_time)(struct hl_device *hdev);
1278 int (*collective_wait_init_cs)(struct hl_cs *cs);
1279 int (*collective_wait_create_jobs)(struct hl_device *hdev,
1280 struct hl_ctx *ctx, struct hl_cs *cs,
1281 u32 wait_queue_id, u32 collective_engine_id,
1282 u32 encaps_signal_offset);
1283 u64 (*scramble_addr)(struct hl_device *hdev, u64 addr);
1284 u64 (*descramble_addr)(struct hl_device *hdev, u64 addr);
1285 void (*ack_protection_bits_errors)(struct hl_device *hdev);
1286 int (*get_hw_block_id)(struct hl_device *hdev, u64 block_addr,
1287 u32 *block_size, u32 *block_id);
1288 int (*hw_block_mmap)(struct hl_device *hdev, struct vm_area_struct *vma,
1289 u32 block_id, u32 block_size);
1290 void (*enable_events_from_fw)(struct hl_device *hdev);
1291 void (*get_msi_info)(__le32 *table);
1292 int (*map_pll_idx_to_fw_idx)(u32 pll_idx);
1293 void (*init_firmware_loader)(struct hl_device *hdev);
1294 void (*init_cpu_scrambler_dram)(struct hl_device *hdev);
1295 void (*state_dump_init)(struct hl_device *hdev);
1296 u32 (*get_sob_addr)(struct hl_device *hdev, u32 sob_id);
1297 void (*set_pci_memory_regions)(struct hl_device *hdev);
1298 u32* (*get_stream_master_qid_arr)(void);
1299};
1300
1301
1302
1303
1304
1305
1306#define HL_KERNEL_ASID_ID 0
1307
1308
1309
1310
1311
1312
1313
1314enum hl_va_range_type {
1315 HL_VA_RANGE_TYPE_HOST,
1316 HL_VA_RANGE_TYPE_HOST_HUGE,
1317 HL_VA_RANGE_TYPE_DRAM,
1318 HL_VA_RANGE_TYPE_MAX
1319};
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329struct hl_va_range {
1330 struct mutex lock;
1331 struct list_head list;
1332 u64 start_addr;
1333 u64 end_addr;
1334 u32 page_size;
1335};
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346struct hl_cs_counters_atomic {
1347 atomic64_t out_of_mem_drop_cnt;
1348 atomic64_t parsing_drop_cnt;
1349 atomic64_t queue_full_drop_cnt;
1350 atomic64_t device_in_reset_drop_cnt;
1351 atomic64_t max_cs_in_flight_drop_cnt;
1352 atomic64_t validation_drop_cnt;
1353};
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393struct hl_ctx {
1394 DECLARE_HASHTABLE(mem_hash, MEM_HASH_TABLE_BITS);
1395 DECLARE_HASHTABLE(mmu_shadow_hash, MMU_HASH_TABLE_BITS);
1396 struct hl_fpriv *hpriv;
1397 struct hl_device *hdev;
1398 struct kref refcount;
1399 struct hl_fence **cs_pending;
1400 struct hl_va_range *va_range[HL_VA_RANGE_TYPE_MAX];
1401 struct mutex mem_hash_lock;
1402 struct mutex mmu_lock;
1403 struct mutex hw_block_list_lock;
1404 struct list_head debugfs_list;
1405 struct list_head hw_block_mem_list;
1406 struct hl_cs_counters_atomic cs_counters;
1407 struct gen_pool *cb_va_pool;
1408 struct hl_encaps_signals_mgr sig_mgr;
1409 u64 cs_sequence;
1410 u64 *dram_default_hops;
1411 spinlock_t cs_lock;
1412 atomic64_t dram_phys_mem;
1413 atomic_t thread_ctx_switch_token;
1414 u32 thread_ctx_switch_wait_token;
1415 u32 asid;
1416 u32 handle;
1417};
1418
1419
1420
1421
1422
1423
1424struct hl_ctx_mgr {
1425 struct mutex ctx_lock;
1426 struct idr ctx_handles;
1427};
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449struct hl_userptr {
1450 enum vm_type vm_type;
1451 struct list_head job_node;
1452 struct page **pages;
1453 unsigned int npages;
1454 struct sg_table *sgt;
1455 enum dma_data_direction dir;
1456 struct list_head debugfs_list;
1457 pid_t pid;
1458 u64 addr;
1459 u64 size;
1460 u8 dma_mapped;
1461};
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501struct hl_cs {
1502 u16 *jobs_in_queue_cnt;
1503 struct hl_ctx *ctx;
1504 struct list_head job_list;
1505 spinlock_t job_lock;
1506 struct kref refcount;
1507 struct hl_fence *fence;
1508 struct hl_fence *signal_fence;
1509 struct work_struct finish_work;
1510 struct delayed_work work_tdr;
1511 struct list_head mirror_node;
1512 struct list_head staged_cs_node;
1513 struct list_head debugfs_list;
1514 struct hl_cs_encaps_sig_handle *encaps_sig_hdl;
1515 u64 sequence;
1516 u64 staged_sequence;
1517 u64 timeout_jiffies;
1518 u64 submission_time_jiffies;
1519 enum hl_cs_type type;
1520 u32 encaps_sig_hdl_id;
1521 u8 submitted;
1522 u8 completed;
1523 u8 timedout;
1524 u8 tdr_active;
1525 u8 aborted;
1526 u8 timestamp;
1527 u8 staged_last;
1528 u8 staged_first;
1529 u8 staged_cs;
1530 u8 skip_reset_on_timeout;
1531 u8 encaps_signals;
1532};
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563struct hl_cs_job {
1564 struct list_head cs_node;
1565 struct hl_cs *cs;
1566 struct hl_cb *user_cb;
1567 struct hl_cb *patched_cb;
1568 struct work_struct finish_work;
1569 struct list_head userptr_list;
1570 struct list_head debugfs_list;
1571 struct kref refcount;
1572 enum hl_queue_type queue_type;
1573 u32 id;
1574 u32 hw_queue_id;
1575 u32 user_cb_size;
1576 u32 job_cb_size;
1577 u32 encaps_sig_wait_offset;
1578 u8 is_kernel_allocated_cb;
1579 u8 contains_dma_pkt;
1580};
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607struct hl_cs_parser {
1608 struct hl_cb *user_cb;
1609 struct hl_cb *patched_cb;
1610 struct list_head *job_userptr_list;
1611 u64 cs_sequence;
1612 enum hl_queue_type queue_type;
1613 u32 ctx_id;
1614 u32 hw_queue_id;
1615 u32 user_cb_size;
1616 u32 patched_cb_size;
1617 u8 job_id;
1618 u8 is_kernel_allocated_cb;
1619 u8 contains_dma_pkt;
1620 u8 completion;
1621};
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635struct hl_vm_hash_node {
1636 struct hlist_node node;
1637 u64 vaddr;
1638 void *ptr;
1639};
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650struct hl_vm_hw_block_list_node {
1651 struct list_head node;
1652 struct hl_ctx *ctx;
1653 unsigned long vaddr;
1654 u32 size;
1655 u32 id;
1656};
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673struct hl_vm_phys_pg_pack {
1674 enum vm_type vm_type;
1675 u64 *pages;
1676 u64 npages;
1677 u64 total_size;
1678 atomic_t mapping_cnt;
1679 u32 asid;
1680 u32 page_size;
1681 u32 flags;
1682 u32 handle;
1683 u32 offset;
1684 u8 contiguous;
1685 u8 created_from_userptr;
1686};
1687
1688
1689
1690
1691
1692
1693
1694
1695struct hl_vm_va_block {
1696 struct list_head node;
1697 u64 start;
1698 u64 end;
1699 u64 size;
1700};
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711struct hl_vm {
1712 struct gen_pool *dram_pg_pool;
1713 struct kref dram_pg_pool_refcount;
1714 spinlock_t idr_lock;
1715 struct idr phys_pg_pack_handles;
1716 u8 init_done;
1717};
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733struct hl_debug_params {
1734 void *input;
1735 void *output;
1736 u32 output_size;
1737 u32 reg_idx;
1738 u32 op;
1739 bool enable;
1740};
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760struct hl_fpriv {
1761 struct hl_device *hdev;
1762 struct file *filp;
1763 struct pid *taskpid;
1764 struct hl_ctx *ctx;
1765 struct hl_ctx_mgr ctx_mgr;
1766 struct hl_cb_mgr cb_mgr;
1767 struct list_head debugfs_list;
1768 struct list_head dev_node;
1769 struct kref refcount;
1770 struct mutex restore_phase_mutex;
1771 u8 is_control;
1772};
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785struct hl_info_list {
1786 const char *name;
1787 int (*show)(struct seq_file *s, void *data);
1788 ssize_t (*write)(struct file *file, const char __user *buf,
1789 size_t count, loff_t *f_pos);
1790};
1791
1792
1793
1794
1795
1796
1797struct hl_debugfs_entry {
1798 const struct hl_info_list *info_ent;
1799 struct hl_dbg_device_entry *dev_entry;
1800};
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831struct hl_dbg_device_entry {
1832 struct dentry *root;
1833 struct hl_device *hdev;
1834 struct hl_debugfs_entry *entry_arr;
1835 struct list_head file_list;
1836 struct mutex file_mutex;
1837 struct list_head cb_list;
1838 spinlock_t cb_spinlock;
1839 struct list_head cs_list;
1840 spinlock_t cs_spinlock;
1841 struct list_head cs_job_list;
1842 spinlock_t cs_job_spinlock;
1843 struct list_head userptr_list;
1844 spinlock_t userptr_spinlock;
1845 struct list_head ctx_mem_hash_list;
1846 spinlock_t ctx_mem_hash_spinlock;
1847 struct debugfs_blob_wrapper blob_desc;
1848 char *state_dump[HL_STATE_DUMP_HIST_LEN];
1849 struct rw_semaphore state_dump_sem;
1850 u64 addr;
1851 u64 mmu_addr;
1852 u64 userptr_lookup;
1853 u32 mmu_asid;
1854 u32 state_dump_head;
1855 u8 i2c_bus;
1856 u8 i2c_addr;
1857 u8 i2c_reg;
1858};
1859
1860
1861
1862
1863
1864
1865
1866
1867struct hl_hw_obj_name_entry {
1868 struct hlist_node node;
1869 const char *name;
1870 u32 id;
1871};
1872
1873enum hl_state_dump_specs_props {
1874 SP_SYNC_OBJ_BASE_ADDR,
1875 SP_NEXT_SYNC_OBJ_ADDR,
1876 SP_SYNC_OBJ_AMOUNT,
1877 SP_MON_OBJ_WR_ADDR_LOW,
1878 SP_MON_OBJ_WR_ADDR_HIGH,
1879 SP_MON_OBJ_WR_DATA,
1880 SP_MON_OBJ_ARM_DATA,
1881 SP_MON_OBJ_STATUS,
1882 SP_MONITORS_AMOUNT,
1883 SP_TPC0_CMDQ,
1884 SP_TPC0_CFG_SO,
1885 SP_NEXT_TPC,
1886 SP_MME_CMDQ,
1887 SP_MME_CFG_SO,
1888 SP_NEXT_MME,
1889 SP_DMA_CMDQ,
1890 SP_DMA_CFG_SO,
1891 SP_DMA_QUEUES_OFFSET,
1892 SP_NUM_OF_MME_ENGINES,
1893 SP_SUB_MME_ENG_NUM,
1894 SP_NUM_OF_DMA_ENGINES,
1895 SP_NUM_OF_TPC_ENGINES,
1896 SP_ENGINE_NUM_OF_QUEUES,
1897 SP_ENGINE_NUM_OF_STREAMS,
1898 SP_ENGINE_NUM_OF_FENCES,
1899 SP_FENCE0_CNT_OFFSET,
1900 SP_FENCE0_RDATA_OFFSET,
1901 SP_CP_STS_OFFSET,
1902 SP_NUM_CORES,
1903
1904 SP_MAX
1905};
1906
1907enum hl_sync_engine_type {
1908 ENGINE_TPC,
1909 ENGINE_DMA,
1910 ENGINE_MME,
1911};
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922struct hl_mon_state_dump {
1923 u32 id;
1924 u32 wr_addr_low;
1925 u32 wr_addr_high;
1926 u32 wr_data;
1927 u32 arm_data;
1928 u32 status;
1929};
1930
1931
1932
1933
1934
1935
1936
1937struct hl_sync_to_engine_map_entry {
1938 struct hlist_node node;
1939 enum hl_sync_engine_type engine_type;
1940 u32 engine_id;
1941 u32 sync_id;
1942};
1943
1944
1945
1946
1947
1948
1949struct hl_sync_to_engine_map {
1950 DECLARE_HASHTABLE(tb, SYNC_TO_ENGINE_HASH_TABLE_BITS);
1951};
1952
1953
1954
1955
1956
1957
1958
1959
1960struct hl_state_dump_specs_funcs {
1961 int (*gen_sync_to_engine_map)(struct hl_device *hdev,
1962 struct hl_sync_to_engine_map *map);
1963 int (*print_single_monitor)(char **buf, size_t *size, size_t *offset,
1964 struct hl_device *hdev,
1965 struct hl_mon_state_dump *mon);
1966 int (*monitor_valid)(struct hl_mon_state_dump *mon);
1967 int (*print_fences_single_engine)(struct hl_device *hdev,
1968 u64 base_offset,
1969 u64 status_base_offset,
1970 enum hl_sync_engine_type engine_type,
1971 u32 engine_id, char **buf,
1972 size_t *size, size_t *offset);
1973};
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983struct hl_state_dump_specs {
1984 DECLARE_HASHTABLE(so_id_to_str_tb, OBJ_NAMES_HASH_TABLE_BITS);
1985 DECLARE_HASHTABLE(monitor_id_to_str_tb, OBJ_NAMES_HASH_TABLE_BITS);
1986 struct hl_state_dump_specs_funcs funcs;
1987 const char * const *sync_namager_names;
1988 s64 *props;
1989};
1990
1991
1992
1993
1994
1995
1996#define HL_STR_MAX 32
1997
1998#define HL_DEV_STS_MAX (HL_DEVICE_STATUS_LAST + 1)
1999
2000
2001
2002
2003#define HL_MAX_MINORS 256
2004
2005
2006
2007
2008
2009u32 hl_rreg(struct hl_device *hdev, u32 reg);
2010void hl_wreg(struct hl_device *hdev, u32 reg, u32 val);
2011
2012#define RREG32(reg) hdev->asic_funcs->rreg(hdev, (reg))
2013#define WREG32(reg, v) hdev->asic_funcs->wreg(hdev, (reg), (v))
2014#define DREG32(reg) pr_info("REGISTER: " #reg " : 0x%08X\n", \
2015 hdev->asic_funcs->rreg(hdev, (reg)))
2016
2017#define WREG32_P(reg, val, mask) \
2018 do { \
2019 u32 tmp_ = RREG32(reg); \
2020 tmp_ &= (mask); \
2021 tmp_ |= ((val) & ~(mask)); \
2022 WREG32(reg, tmp_); \
2023 } while (0)
2024#define WREG32_AND(reg, and) WREG32_P(reg, 0, and)
2025#define WREG32_OR(reg, or) WREG32_P(reg, or, ~(or))
2026
2027#define RMWREG32(reg, val, mask) \
2028 do { \
2029 u32 tmp_ = RREG32(reg); \
2030 tmp_ &= ~(mask); \
2031 tmp_ |= ((val) << __ffs(mask)); \
2032 WREG32(reg, tmp_); \
2033 } while (0)
2034
2035#define RREG32_MASK(reg, mask) ((RREG32(reg) & mask) >> __ffs(mask))
2036
2037#define REG_FIELD_SHIFT(reg, field) reg##_##field##_SHIFT
2038#define REG_FIELD_MASK(reg, field) reg##_##field##_MASK
2039#define WREG32_FIELD(reg, offset, field, val) \
2040 WREG32(mm##reg + offset, (RREG32(mm##reg + offset) & \
2041 ~REG_FIELD_MASK(reg, field)) | \
2042 (val) << REG_FIELD_SHIFT(reg, field))
2043
2044
2045
2046
2047#define hl_poll_timeout(hdev, addr, val, cond, sleep_us, timeout_us) \
2048({ \
2049 ktime_t __timeout; \
2050 if (hdev->pdev) \
2051 __timeout = ktime_add_us(ktime_get(), timeout_us); \
2052 else \
2053 __timeout = ktime_add_us(ktime_get(),\
2054 min((u64)(timeout_us * 10), \
2055 (u64) HL_SIM_MAX_TIMEOUT_US)); \
2056 might_sleep_if(sleep_us); \
2057 for (;;) { \
2058 (val) = RREG32(addr); \
2059 if (cond) \
2060 break; \
2061 if (timeout_us && ktime_compare(ktime_get(), __timeout) > 0) { \
2062 (val) = RREG32(addr); \
2063 break; \
2064 } \
2065 if (sleep_us) \
2066 usleep_range((sleep_us >> 2) + 1, sleep_us); \
2067 } \
2068 (cond) ? 0 : -ETIMEDOUT; \
2069})
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083#define hl_poll_timeout_memory(hdev, addr, val, cond, sleep_us, timeout_us, \
2084 mem_written_by_device) \
2085({ \
2086 ktime_t __timeout; \
2087 if (hdev->pdev) \
2088 __timeout = ktime_add_us(ktime_get(), timeout_us); \
2089 else \
2090 __timeout = ktime_add_us(ktime_get(),\
2091 min((u64)(timeout_us * 10), \
2092 (u64) HL_SIM_MAX_TIMEOUT_US)); \
2093 might_sleep_if(sleep_us); \
2094 for (;;) { \
2095 \
2096 mb(); \
2097 (val) = *((u32 *)(addr)); \
2098 if (mem_written_by_device) \
2099 (val) = le32_to_cpu(*(__le32 *) &(val)); \
2100 if (cond) \
2101 break; \
2102 if (timeout_us && ktime_compare(ktime_get(), __timeout) > 0) { \
2103 (val) = *((u32 *)(addr)); \
2104 if (mem_written_by_device) \
2105 (val) = le32_to_cpu(*(__le32 *) &(val)); \
2106 break; \
2107 } \
2108 if (sleep_us) \
2109 usleep_range((sleep_us >> 2) + 1, sleep_us); \
2110 } \
2111 (cond) ? 0 : -ETIMEDOUT; \
2112})
2113
2114#define hl_poll_timeout_device_memory(hdev, addr, val, cond, sleep_us, \
2115 timeout_us) \
2116({ \
2117 ktime_t __timeout; \
2118 if (hdev->pdev) \
2119 __timeout = ktime_add_us(ktime_get(), timeout_us); \
2120 else \
2121 __timeout = ktime_add_us(ktime_get(),\
2122 min((u64)(timeout_us * 10), \
2123 (u64) HL_SIM_MAX_TIMEOUT_US)); \
2124 might_sleep_if(sleep_us); \
2125 for (;;) { \
2126 (val) = readl(addr); \
2127 if (cond) \
2128 break; \
2129 if (timeout_us && ktime_compare(ktime_get(), __timeout) > 0) { \
2130 (val) = readl(addr); \
2131 break; \
2132 } \
2133 if (sleep_us) \
2134 usleep_range((sleep_us >> 2) + 1, sleep_us); \
2135 } \
2136 (cond) ? 0 : -ETIMEDOUT; \
2137})
2138
2139struct hwmon_chip_info;
2140
2141
2142
2143
2144
2145
2146
2147
2148struct hl_device_reset_work {
2149 struct workqueue_struct *wq;
2150 struct delayed_work reset_work;
2151 struct hl_device *hdev;
2152 bool fw_reset;
2153};
2154
2155
2156
2157
2158
2159
2160
2161
2162struct hr_mmu_hop_addrs {
2163 u64 virt_addr;
2164 u64 phys_addr;
2165 u64 shadow_addr;
2166};
2167
2168
2169
2170
2171
2172
2173
2174struct hl_mmu_hr_priv {
2175 struct gen_pool *mmu_pgt_pool;
2176 struct hr_mmu_hop_addrs *mmu_shadow_hop0;
2177};
2178
2179
2180
2181
2182
2183
2184
2185struct hl_mmu_dr_priv {
2186 struct gen_pool *mmu_pgt_pool;
2187 void *mmu_shadow_hop0;
2188};
2189
2190
2191
2192
2193
2194
2195struct hl_mmu_priv {
2196 struct hl_mmu_dr_priv dr;
2197 struct hl_mmu_hr_priv hr;
2198};
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208struct hl_mmu_per_hop_info {
2209 u64 hop_addr;
2210 u64 hop_pte_addr;
2211 u64 hop_pte_val;
2212};
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226struct hl_mmu_hop_info {
2227 u64 scrambled_vaddr;
2228 u64 unscrambled_paddr;
2229 struct hl_mmu_per_hop_info hop_info[MMU_ARCH_5_HOPS];
2230 u32 used_hops;
2231 enum hl_va_range_type range_type;
2232};
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249struct hl_mmu_funcs {
2250 int (*init)(struct hl_device *hdev);
2251 void (*fini)(struct hl_device *hdev);
2252 int (*ctx_init)(struct hl_ctx *ctx);
2253 void (*ctx_fini)(struct hl_ctx *ctx);
2254 int (*map)(struct hl_ctx *ctx,
2255 u64 virt_addr, u64 phys_addr, u32 page_size,
2256 bool is_dram_addr);
2257 int (*unmap)(struct hl_ctx *ctx,
2258 u64 virt_addr, bool is_dram_addr);
2259 void (*flush)(struct hl_ctx *ctx);
2260 void (*swap_out)(struct hl_ctx *ctx);
2261 void (*swap_in)(struct hl_ctx *ctx);
2262 int (*get_tlb_info)(struct hl_ctx *ctx,
2263 u64 virt_addr, struct hl_mmu_hop_info *hops);
2264};
2265
2266
2267
2268
2269
2270#define MULTI_CS_MAX_USER_CTX 2
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281struct multi_cs_completion {
2282 struct completion completion;
2283 spinlock_t lock;
2284 s64 timestamp;
2285 u32 stream_master_qid_map;
2286 u8 used;
2287};
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304struct multi_cs_data {
2305 struct hl_ctx *ctx;
2306 struct hl_fence **fence_arr;
2307 u64 *seq_arr;
2308 s64 timeout_us;
2309 s64 timestamp;
2310 long wait_status;
2311 u32 completion_bitmap;
2312 u32 stream_master_qid_map;
2313 u8 arr_len;
2314 u8 gone_cs;
2315 u8 update_ts;
2316};
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463struct hl_device {
2464 struct pci_dev *pdev;
2465 u64 pcie_bar_phys[HL_PCI_NUM_BARS];
2466 void __iomem *pcie_bar[HL_PCI_NUM_BARS];
2467 void __iomem *rmmio;
2468 struct cdev cdev;
2469 struct cdev cdev_ctrl;
2470 struct device *dev;
2471 struct device *dev_ctrl;
2472 struct delayed_work work_freq;
2473 struct delayed_work work_heartbeat;
2474 struct hl_device_reset_work device_reset_work;
2475 char asic_name[HL_STR_MAX];
2476 char status[HL_DEV_STS_MAX][HL_STR_MAX];
2477 enum hl_asic_type asic_type;
2478 struct hl_cq *completion_queue;
2479 struct hl_user_interrupt *user_interrupt;
2480 struct hl_user_interrupt common_user_interrupt;
2481 struct workqueue_struct **cq_wq;
2482 struct workqueue_struct *eq_wq;
2483 struct workqueue_struct *sob_reset_wq;
2484 struct hl_ctx *kernel_ctx;
2485 struct hl_hw_queue *kernel_queues;
2486 struct list_head cs_mirror_list;
2487 spinlock_t cs_mirror_lock;
2488 struct hl_cb_mgr kernel_cb_mgr;
2489 struct hl_eq event_queue;
2490 struct dma_pool *dma_pool;
2491 void *cpu_accessible_dma_mem;
2492 dma_addr_t cpu_accessible_dma_address;
2493 struct gen_pool *cpu_accessible_dma_pool;
2494 unsigned long *asid_bitmap;
2495 struct mutex asid_mutex;
2496 struct mutex send_cpu_message_lock;
2497 struct mutex debug_lock;
2498 struct asic_fixed_properties asic_prop;
2499 const struct hl_asic_funcs *asic_funcs;
2500 void *asic_specific;
2501 struct hl_vm vm;
2502 struct device *hwmon_dev;
2503 enum hl_pm_mng_profile pm_mng_profile;
2504 struct hwmon_chip_info *hl_chip_info;
2505
2506 struct hl_dbg_device_entry hl_debugfs;
2507
2508 struct list_head cb_pool;
2509 spinlock_t cb_pool_lock;
2510
2511 void *internal_cb_pool_virt_addr;
2512 dma_addr_t internal_cb_pool_dma_addr;
2513 struct gen_pool *internal_cb_pool;
2514 u64 internal_cb_va_base;
2515
2516 struct list_head fpriv_list;
2517 struct mutex fpriv_list_lock;
2518
2519 struct hl_ctx *compute_ctx;
2520
2521 struct hl_cs_counters_atomic aggregated_cs_counters;
2522
2523 struct hl_mmu_priv mmu_priv;
2524 struct hl_mmu_funcs mmu_func[MMU_NUM_PGT_LOCATIONS];
2525
2526 struct fw_load_mgr fw_loader;
2527
2528 struct pci_mem_region pci_mem_region[PCI_REGION_NUMBER];
2529
2530 struct hl_state_dump_specs state_dump_specs;
2531
2532 struct multi_cs_completion multi_cs_completion[
2533 MULTI_CS_MAX_USER_CTX];
2534 u32 *stream_master_qid_arr;
2535 atomic64_t dram_used_mem;
2536 u64 timeout_jiffies;
2537 u64 max_power;
2538 u64 clock_gating_mask;
2539 u64 boot_error_status_mask;
2540 u64 last_successful_open_jif;
2541 u64 last_open_session_duration_jif;
2542 u64 open_counter;
2543 atomic_t in_reset;
2544 enum hl_pll_frequency curr_pll_profile;
2545 enum cpucp_card_types card_type;
2546 u32 major;
2547 u32 high_pll;
2548 u32 soft_reset_cnt;
2549 u32 hard_reset_cnt;
2550 u32 clk_throttling_reason;
2551 u16 id;
2552 u16 id_control;
2553 u16 cpu_pci_msb_addr;
2554 u8 disabled;
2555 u8 late_init_done;
2556 u8 hwmon_initialized;
2557 u8 hard_reset_pending;
2558 u8 heartbeat;
2559 u8 reset_on_lockup;
2560 u8 dram_default_page_mapping;
2561 u8 memory_scrub;
2562 u8 pmmu_huge_range;
2563 u8 init_done;
2564 u8 device_cpu_disabled;
2565 u8 dma_mask;
2566 u8 in_debug;
2567 u8 power9_64bit_dma_enable;
2568 u8 cdev_sysfs_created;
2569 u8 stop_on_err;
2570 u8 supports_sync_stream;
2571 u8 sync_stream_queue_idx;
2572 u8 collective_mon_idx;
2573 u8 supports_coresight;
2574 u8 supports_soft_reset;
2575 u8 allow_external_soft_reset;
2576 u8 supports_cb_mapping;
2577 u8 needs_reset;
2578 u8 process_kill_trial_cnt;
2579 u8 device_fini_pending;
2580 u8 supports_staged_submission;
2581 u8 curr_reset_cause;
2582 u8 skip_reset_on_timeout;
2583 u8 device_cpu_is_halted;
2584 u8 supports_wait_for_multi_cs;
2585 u8 stream_master_qid_arr_size;
2586
2587
2588 u64 nic_ports_mask;
2589 u64 fw_components;
2590 u8 mmu_enable;
2591 u8 mmu_huge_page_opt;
2592 u8 reset_pcilink;
2593 u8 cpu_queues_enable;
2594 u8 pldm;
2595 u8 axi_drain;
2596 u8 sram_scrambler_enable;
2597 u8 dram_scrambler_enable;
2598 u8 hard_reset_on_fw_events;
2599 u8 bmc_enable;
2600 u8 rl_enable;
2601 u8 reset_on_preboot_fail;
2602 u8 reset_upon_device_release;
2603 u8 reset_if_device_not_idle;
2604};
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619struct hl_cs_encaps_sig_handle {
2620 struct kref refcount;
2621 struct hl_device *hdev;
2622 struct hl_hw_sob *hw_sob;
2623 u64 cs_seq;
2624 u32 id;
2625 u32 q_idx;
2626 u32 pre_sob_val;
2627 u32 count;
2628};
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642typedef int hl_ioctl_t(struct hl_fpriv *hpriv, void *data);
2643
2644
2645
2646
2647
2648
2649struct hl_ioctl_desc {
2650 unsigned int cmd;
2651 hl_ioctl_t *func;
2652};
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668static inline u32 hl_get_sg_info(struct scatterlist *sg, dma_addr_t *dma_addr)
2669{
2670 *dma_addr = sg_dma_address(sg);
2671
2672 return ((((*dma_addr) & (PAGE_SIZE - 1)) + sg_dma_len(sg)) +
2673 (PAGE_SIZE - 1)) >> PAGE_SHIFT;
2674}
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685static inline bool hl_mem_area_inside_range(u64 address, u64 size,
2686 u64 range_start_address, u64 range_end_address)
2687{
2688 u64 end_address = address + size;
2689
2690 if ((address >= range_start_address) &&
2691 (end_address <= range_end_address) &&
2692 (end_address > address))
2693 return true;
2694
2695 return false;
2696}
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708static inline bool hl_mem_area_crosses_range(u64 address, u32 size,
2709 u64 range_start_address, u64 range_end_address)
2710{
2711 u64 end_address = address + size;
2712
2713 if ((address >= range_start_address) &&
2714 (address < range_end_address))
2715 return true;
2716
2717 if ((end_address >= range_start_address) &&
2718 (end_address < range_end_address))
2719 return true;
2720
2721 if ((address < range_start_address) &&
2722 (end_address >= range_end_address))
2723 return true;
2724
2725 return false;
2726}
2727
2728int hl_device_open(struct inode *inode, struct file *filp);
2729int hl_device_open_ctrl(struct inode *inode, struct file *filp);
2730bool hl_device_operational(struct hl_device *hdev,
2731 enum hl_device_status *status);
2732enum hl_device_status hl_device_status(struct hl_device *hdev);
2733int hl_device_set_debug_mode(struct hl_device *hdev, bool enable);
2734int create_hdev(struct hl_device **dev, struct pci_dev *pdev,
2735 enum hl_asic_type asic_type, int minor);
2736void destroy_hdev(struct hl_device *hdev);
2737int hl_hw_queues_create(struct hl_device *hdev);
2738void hl_hw_queues_destroy(struct hl_device *hdev);
2739int hl_hw_queue_send_cb_no_cmpl(struct hl_device *hdev, u32 hw_queue_id,
2740 u32 cb_size, u64 cb_ptr);
2741void hl_hw_queue_submit_bd(struct hl_device *hdev, struct hl_hw_queue *q,
2742 u32 ctl, u32 len, u64 ptr);
2743int hl_hw_queue_schedule_cs(struct hl_cs *cs);
2744u32 hl_hw_queue_add_ptr(u32 ptr, u16 val);
2745void hl_hw_queue_inc_ci_kernel(struct hl_device *hdev, u32 hw_queue_id);
2746void hl_hw_queue_update_ci(struct hl_cs *cs);
2747void hl_hw_queue_reset(struct hl_device *hdev, bool hard_reset);
2748
2749#define hl_queue_inc_ptr(p) hl_hw_queue_add_ptr(p, 1)
2750#define hl_pi_2_offset(pi) ((pi) & (HL_QUEUE_LENGTH - 1))
2751
2752int hl_cq_init(struct hl_device *hdev, struct hl_cq *q, u32 hw_queue_id);
2753void hl_cq_fini(struct hl_device *hdev, struct hl_cq *q);
2754int hl_eq_init(struct hl_device *hdev, struct hl_eq *q);
2755void hl_eq_fini(struct hl_device *hdev, struct hl_eq *q);
2756void hl_cq_reset(struct hl_device *hdev, struct hl_cq *q);
2757void hl_eq_reset(struct hl_device *hdev, struct hl_eq *q);
2758irqreturn_t hl_irq_handler_cq(int irq, void *arg);
2759irqreturn_t hl_irq_handler_eq(int irq, void *arg);
2760irqreturn_t hl_irq_handler_user_cq(int irq, void *arg);
2761irqreturn_t hl_irq_handler_default(int irq, void *arg);
2762u32 hl_cq_inc_ptr(u32 ptr);
2763
2764int hl_asid_init(struct hl_device *hdev);
2765void hl_asid_fini(struct hl_device *hdev);
2766unsigned long hl_asid_alloc(struct hl_device *hdev);
2767void hl_asid_free(struct hl_device *hdev, unsigned long asid);
2768
2769int hl_ctx_create(struct hl_device *hdev, struct hl_fpriv *hpriv);
2770void hl_ctx_free(struct hl_device *hdev, struct hl_ctx *ctx);
2771int hl_ctx_init(struct hl_device *hdev, struct hl_ctx *ctx, bool is_kernel_ctx);
2772void hl_ctx_do_release(struct kref *ref);
2773void hl_ctx_get(struct hl_device *hdev, struct hl_ctx *ctx);
2774int hl_ctx_put(struct hl_ctx *ctx);
2775struct hl_fence *hl_ctx_get_fence(struct hl_ctx *ctx, u64 seq);
2776int hl_ctx_get_fences(struct hl_ctx *ctx, u64 *seq_arr,
2777 struct hl_fence **fence, u32 arr_len);
2778void hl_ctx_mgr_init(struct hl_ctx_mgr *mgr);
2779void hl_ctx_mgr_fini(struct hl_device *hdev, struct hl_ctx_mgr *mgr);
2780
2781int hl_device_init(struct hl_device *hdev, struct class *hclass);
2782void hl_device_fini(struct hl_device *hdev);
2783int hl_device_suspend(struct hl_device *hdev);
2784int hl_device_resume(struct hl_device *hdev);
2785int hl_device_reset(struct hl_device *hdev, u32 flags);
2786void hl_hpriv_get(struct hl_fpriv *hpriv);
2787int hl_hpriv_put(struct hl_fpriv *hpriv);
2788int hl_device_set_frequency(struct hl_device *hdev, enum hl_pll_frequency freq);
2789int hl_device_utilization(struct hl_device *hdev, u32 *utilization);
2790
2791int hl_build_hwmon_channel_info(struct hl_device *hdev,
2792 struct cpucp_sensor *sensors_arr);
2793
2794int hl_sysfs_init(struct hl_device *hdev);
2795void hl_sysfs_fini(struct hl_device *hdev);
2796
2797int hl_hwmon_init(struct hl_device *hdev);
2798void hl_hwmon_fini(struct hl_device *hdev);
2799
2800int hl_cb_create(struct hl_device *hdev, struct hl_cb_mgr *mgr,
2801 struct hl_ctx *ctx, u32 cb_size, bool internal_cb,
2802 bool map_cb, u64 *handle);
2803int hl_cb_destroy(struct hl_device *hdev, struct hl_cb_mgr *mgr, u64 cb_handle);
2804int hl_cb_mmap(struct hl_fpriv *hpriv, struct vm_area_struct *vma);
2805int hl_hw_block_mmap(struct hl_fpriv *hpriv, struct vm_area_struct *vma);
2806struct hl_cb *hl_cb_get(struct hl_device *hdev, struct hl_cb_mgr *mgr,
2807 u32 handle);
2808void hl_cb_put(struct hl_cb *cb);
2809void hl_cb_mgr_init(struct hl_cb_mgr *mgr);
2810void hl_cb_mgr_fini(struct hl_device *hdev, struct hl_cb_mgr *mgr);
2811struct hl_cb *hl_cb_kernel_create(struct hl_device *hdev, u32 cb_size,
2812 bool internal_cb);
2813int hl_cb_pool_init(struct hl_device *hdev);
2814int hl_cb_pool_fini(struct hl_device *hdev);
2815int hl_cb_va_pool_init(struct hl_ctx *ctx);
2816void hl_cb_va_pool_fini(struct hl_ctx *ctx);
2817
2818void hl_cs_rollback_all(struct hl_device *hdev);
2819struct hl_cs_job *hl_cs_allocate_job(struct hl_device *hdev,
2820 enum hl_queue_type queue_type, bool is_kernel_allocated_cb);
2821void hl_sob_reset_error(struct kref *ref);
2822int hl_gen_sob_mask(u16 sob_base, u8 sob_mask, u8 *mask);
2823void hl_fence_put(struct hl_fence *fence);
2824void hl_fences_put(struct hl_fence **fence, int len);
2825void hl_fence_get(struct hl_fence *fence);
2826void cs_get(struct hl_cs *cs);
2827bool cs_needs_completion(struct hl_cs *cs);
2828bool cs_needs_timeout(struct hl_cs *cs);
2829bool is_staged_cs_last_exists(struct hl_device *hdev, struct hl_cs *cs);
2830struct hl_cs *hl_staged_cs_find_first(struct hl_device *hdev, u64 cs_seq);
2831void hl_multi_cs_completion_init(struct hl_device *hdev);
2832
2833void goya_set_asic_funcs(struct hl_device *hdev);
2834void gaudi_set_asic_funcs(struct hl_device *hdev);
2835
2836int hl_vm_ctx_init(struct hl_ctx *ctx);
2837void hl_vm_ctx_fini(struct hl_ctx *ctx);
2838
2839int hl_vm_init(struct hl_device *hdev);
2840void hl_vm_fini(struct hl_device *hdev);
2841
2842void hl_hw_block_mem_init(struct hl_ctx *ctx);
2843void hl_hw_block_mem_fini(struct hl_ctx *ctx);
2844
2845u64 hl_reserve_va_block(struct hl_device *hdev, struct hl_ctx *ctx,
2846 enum hl_va_range_type type, u32 size, u32 alignment);
2847int hl_unreserve_va_block(struct hl_device *hdev, struct hl_ctx *ctx,
2848 u64 start_addr, u64 size);
2849int hl_pin_host_memory(struct hl_device *hdev, u64 addr, u64 size,
2850 struct hl_userptr *userptr);
2851void hl_unpin_host_memory(struct hl_device *hdev, struct hl_userptr *userptr);
2852void hl_userptr_delete_list(struct hl_device *hdev,
2853 struct list_head *userptr_list);
2854bool hl_userptr_is_pinned(struct hl_device *hdev, u64 addr, u32 size,
2855 struct list_head *userptr_list,
2856 struct hl_userptr **userptr);
2857
2858int hl_mmu_init(struct hl_device *hdev);
2859void hl_mmu_fini(struct hl_device *hdev);
2860int hl_mmu_ctx_init(struct hl_ctx *ctx);
2861void hl_mmu_ctx_fini(struct hl_ctx *ctx);
2862int hl_mmu_map_page(struct hl_ctx *ctx, u64 virt_addr, u64 phys_addr,
2863 u32 page_size, bool flush_pte);
2864int hl_mmu_unmap_page(struct hl_ctx *ctx, u64 virt_addr, u32 page_size,
2865 bool flush_pte);
2866int hl_mmu_map_contiguous(struct hl_ctx *ctx, u64 virt_addr,
2867 u64 phys_addr, u32 size);
2868int hl_mmu_unmap_contiguous(struct hl_ctx *ctx, u64 virt_addr, u32 size);
2869void hl_mmu_swap_out(struct hl_ctx *ctx);
2870void hl_mmu_swap_in(struct hl_ctx *ctx);
2871int hl_mmu_if_set_funcs(struct hl_device *hdev);
2872void hl_mmu_v1_set_funcs(struct hl_device *hdev, struct hl_mmu_funcs *mmu);
2873int hl_mmu_va_to_pa(struct hl_ctx *ctx, u64 virt_addr, u64 *phys_addr);
2874int hl_mmu_get_tlb_info(struct hl_ctx *ctx, u64 virt_addr,
2875 struct hl_mmu_hop_info *hops);
2876u64 hl_mmu_scramble_addr(struct hl_device *hdev, u64 addr);
2877u64 hl_mmu_descramble_addr(struct hl_device *hdev, u64 addr);
2878bool hl_is_dram_va(struct hl_device *hdev, u64 virt_addr);
2879
2880int hl_fw_load_fw_to_device(struct hl_device *hdev, const char *fw_name,
2881 void __iomem *dst, u32 src_offset, u32 size);
2882int hl_fw_send_pci_access_msg(struct hl_device *hdev, u32 opcode);
2883int hl_fw_send_cpu_message(struct hl_device *hdev, u32 hw_queue_id, u32 *msg,
2884 u16 len, u32 timeout, u64 *result);
2885int hl_fw_unmask_irq(struct hl_device *hdev, u16 event_type);
2886int hl_fw_unmask_irq_arr(struct hl_device *hdev, const u32 *irq_arr,
2887 size_t irq_arr_size);
2888int hl_fw_test_cpu_queue(struct hl_device *hdev);
2889void *hl_fw_cpu_accessible_dma_pool_alloc(struct hl_device *hdev, size_t size,
2890 dma_addr_t *dma_handle);
2891void hl_fw_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size,
2892 void *vaddr);
2893int hl_fw_send_heartbeat(struct hl_device *hdev);
2894int hl_fw_cpucp_info_get(struct hl_device *hdev,
2895 u32 sts_boot_dev_sts0_reg,
2896 u32 sts_boot_dev_sts1_reg, u32 boot_err0_reg,
2897 u32 boot_err1_reg);
2898int hl_fw_cpucp_handshake(struct hl_device *hdev,
2899 u32 sts_boot_dev_sts0_reg,
2900 u32 sts_boot_dev_sts1_reg, u32 boot_err0_reg,
2901 u32 boot_err1_reg);
2902int hl_fw_get_eeprom_data(struct hl_device *hdev, void *data, size_t max_size);
2903int hl_fw_cpucp_pci_counters_get(struct hl_device *hdev,
2904 struct hl_info_pci_counters *counters);
2905int hl_fw_cpucp_total_energy_get(struct hl_device *hdev,
2906 u64 *total_energy);
2907int get_used_pll_index(struct hl_device *hdev, u32 input_pll_index,
2908 enum pll_index *pll_index);
2909int hl_fw_cpucp_pll_info_get(struct hl_device *hdev, u32 pll_index,
2910 u16 *pll_freq_arr);
2911int hl_fw_cpucp_power_get(struct hl_device *hdev, u64 *power);
2912void hl_fw_ask_hard_reset_without_linux(struct hl_device *hdev);
2913void hl_fw_ask_halt_machine_without_linux(struct hl_device *hdev);
2914int hl_fw_init_cpu(struct hl_device *hdev);
2915int hl_fw_read_preboot_status(struct hl_device *hdev, u32 cpu_boot_status_reg,
2916 u32 sts_boot_dev_sts0_reg,
2917 u32 sts_boot_dev_sts1_reg, u32 boot_err0_reg,
2918 u32 boot_err1_reg, u32 timeout);
2919int hl_fw_dynamic_send_protocol_cmd(struct hl_device *hdev,
2920 struct fw_load_mgr *fw_loader,
2921 enum comms_cmd cmd, unsigned int size,
2922 bool wait_ok, u32 timeout);
2923int hl_pci_bars_map(struct hl_device *hdev, const char * const name[3],
2924 bool is_wc[3]);
2925int hl_pci_elbi_read(struct hl_device *hdev, u64 addr, u32 *data);
2926int hl_pci_iatu_write(struct hl_device *hdev, u32 addr, u32 data);
2927int hl_pci_set_inbound_region(struct hl_device *hdev, u8 region,
2928 struct hl_inbound_pci_region *pci_region);
2929int hl_pci_set_outbound_region(struct hl_device *hdev,
2930 struct hl_outbound_pci_region *pci_region);
2931enum pci_region hl_get_pci_memory_region(struct hl_device *hdev, u64 addr);
2932int hl_pci_init(struct hl_device *hdev);
2933void hl_pci_fini(struct hl_device *hdev);
2934
2935long hl_get_frequency(struct hl_device *hdev, u32 pll_index,
2936 bool curr);
2937void hl_set_frequency(struct hl_device *hdev, u32 pll_index,
2938 u64 freq);
2939int hl_get_temperature(struct hl_device *hdev,
2940 int sensor_index, u32 attr, long *value);
2941int hl_set_temperature(struct hl_device *hdev,
2942 int sensor_index, u32 attr, long value);
2943int hl_get_voltage(struct hl_device *hdev,
2944 int sensor_index, u32 attr, long *value);
2945int hl_get_current(struct hl_device *hdev,
2946 int sensor_index, u32 attr, long *value);
2947int hl_get_fan_speed(struct hl_device *hdev,
2948 int sensor_index, u32 attr, long *value);
2949int hl_get_pwm_info(struct hl_device *hdev,
2950 int sensor_index, u32 attr, long *value);
2951void hl_set_pwm_info(struct hl_device *hdev, int sensor_index, u32 attr,
2952 long value);
2953u64 hl_get_max_power(struct hl_device *hdev);
2954void hl_set_max_power(struct hl_device *hdev);
2955int hl_set_voltage(struct hl_device *hdev,
2956 int sensor_index, u32 attr, long value);
2957int hl_set_current(struct hl_device *hdev,
2958 int sensor_index, u32 attr, long value);
2959void hw_sob_get(struct hl_hw_sob *hw_sob);
2960void hw_sob_put(struct hl_hw_sob *hw_sob);
2961void hl_encaps_handle_do_release(struct kref *ref);
2962void hl_hw_queue_encaps_sig_set_sob_info(struct hl_device *hdev,
2963 struct hl_cs *cs, struct hl_cs_job *job,
2964 struct hl_cs_compl *cs_cmpl);
2965void hl_release_pending_user_interrupts(struct hl_device *hdev);
2966int hl_cs_signal_sob_wraparound_handler(struct hl_device *hdev, u32 q_idx,
2967 struct hl_hw_sob **hw_sob, u32 count, bool encaps_sig);
2968
2969int hl_state_dump(struct hl_device *hdev);
2970const char *hl_state_dump_get_sync_name(struct hl_device *hdev, u32 sync_id);
2971const char *hl_state_dump_get_monitor_name(struct hl_device *hdev,
2972 struct hl_mon_state_dump *mon);
2973void hl_state_dump_free_sync_to_engine_map(struct hl_sync_to_engine_map *map);
2974__printf(4, 5) int hl_snprintf_resize(char **buf, size_t *size, size_t *offset,
2975 const char *format, ...);
2976char *hl_format_as_binary(char *buf, size_t buf_len, u32 n);
2977const char *hl_sync_engine_to_string(enum hl_sync_engine_type engine_type);
2978
2979#ifdef CONFIG_DEBUG_FS
2980
2981void hl_debugfs_init(void);
2982void hl_debugfs_fini(void);
2983void hl_debugfs_add_device(struct hl_device *hdev);
2984void hl_debugfs_remove_device(struct hl_device *hdev);
2985void hl_debugfs_add_file(struct hl_fpriv *hpriv);
2986void hl_debugfs_remove_file(struct hl_fpriv *hpriv);
2987void hl_debugfs_add_cb(struct hl_cb *cb);
2988void hl_debugfs_remove_cb(struct hl_cb *cb);
2989void hl_debugfs_add_cs(struct hl_cs *cs);
2990void hl_debugfs_remove_cs(struct hl_cs *cs);
2991void hl_debugfs_add_job(struct hl_device *hdev, struct hl_cs_job *job);
2992void hl_debugfs_remove_job(struct hl_device *hdev, struct hl_cs_job *job);
2993void hl_debugfs_add_userptr(struct hl_device *hdev, struct hl_userptr *userptr);
2994void hl_debugfs_remove_userptr(struct hl_device *hdev,
2995 struct hl_userptr *userptr);
2996void hl_debugfs_add_ctx_mem_hash(struct hl_device *hdev, struct hl_ctx *ctx);
2997void hl_debugfs_remove_ctx_mem_hash(struct hl_device *hdev, struct hl_ctx *ctx);
2998void hl_debugfs_set_state_dump(struct hl_device *hdev, char *data,
2999 unsigned long length);
3000
3001#else
3002
3003static inline void __init hl_debugfs_init(void)
3004{
3005}
3006
3007static inline void hl_debugfs_fini(void)
3008{
3009}
3010
3011static inline void hl_debugfs_add_device(struct hl_device *hdev)
3012{
3013}
3014
3015static inline void hl_debugfs_remove_device(struct hl_device *hdev)
3016{
3017}
3018
3019static inline void hl_debugfs_add_file(struct hl_fpriv *hpriv)
3020{
3021}
3022
3023static inline void hl_debugfs_remove_file(struct hl_fpriv *hpriv)
3024{
3025}
3026
3027static inline void hl_debugfs_add_cb(struct hl_cb *cb)
3028{
3029}
3030
3031static inline void hl_debugfs_remove_cb(struct hl_cb *cb)
3032{
3033}
3034
3035static inline void hl_debugfs_add_cs(struct hl_cs *cs)
3036{
3037}
3038
3039static inline void hl_debugfs_remove_cs(struct hl_cs *cs)
3040{
3041}
3042
3043static inline void hl_debugfs_add_job(struct hl_device *hdev,
3044 struct hl_cs_job *job)
3045{
3046}
3047
3048static inline void hl_debugfs_remove_job(struct hl_device *hdev,
3049 struct hl_cs_job *job)
3050{
3051}
3052
3053static inline void hl_debugfs_add_userptr(struct hl_device *hdev,
3054 struct hl_userptr *userptr)
3055{
3056}
3057
3058static inline void hl_debugfs_remove_userptr(struct hl_device *hdev,
3059 struct hl_userptr *userptr)
3060{
3061}
3062
3063static inline void hl_debugfs_add_ctx_mem_hash(struct hl_device *hdev,
3064 struct hl_ctx *ctx)
3065{
3066}
3067
3068static inline void hl_debugfs_remove_ctx_mem_hash(struct hl_device *hdev,
3069 struct hl_ctx *ctx)
3070{
3071}
3072
3073static inline void hl_debugfs_set_state_dump(struct hl_device *hdev,
3074 char *data, unsigned long length)
3075{
3076}
3077
3078#endif
3079
3080
3081long hl_ioctl(struct file *filep, unsigned int cmd, unsigned long arg);
3082long hl_ioctl_control(struct file *filep, unsigned int cmd, unsigned long arg);
3083int hl_cb_ioctl(struct hl_fpriv *hpriv, void *data);
3084int hl_cs_ioctl(struct hl_fpriv *hpriv, void *data);
3085int hl_wait_ioctl(struct hl_fpriv *hpriv, void *data);
3086int hl_mem_ioctl(struct hl_fpriv *hpriv, void *data);
3087
3088#endif
3089