1
2
3
4
5
6
7
8
9#define pr_fmt(fmt) "habanalabs: " fmt
10
11#include "habanalabs.h"
12
13#include <linux/pci.h>
14#include <linux/module.h>
15
16#define HL_DRIVER_AUTHOR "HabanaLabs Kernel Driver Team"
17
18#define HL_DRIVER_DESC "Driver for HabanaLabs's AI Accelerators"
19
20MODULE_AUTHOR(HL_DRIVER_AUTHOR);
21MODULE_DESCRIPTION(HL_DRIVER_DESC);
22MODULE_LICENSE("GPL v2");
23
24static int hl_major;
25static struct class *hl_class;
26static DEFINE_IDR(hl_devs_idr);
27static DEFINE_MUTEX(hl_devs_idr_lock);
28
29static int timeout_locked = 5;
30static int reset_on_lockup = 1;
31
32module_param(timeout_locked, int, 0444);
33MODULE_PARM_DESC(timeout_locked,
34 "Device lockup timeout in seconds (0 = disabled, default 5s)");
35
36module_param(reset_on_lockup, int, 0444);
37MODULE_PARM_DESC(reset_on_lockup,
38 "Do device reset on lockup (0 = no, 1 = yes, default yes)");
39
40#define PCI_VENDOR_ID_HABANALABS 0x1da3
41
42#define PCI_IDS_GOYA 0x0001
43
44static const struct pci_device_id ids[] = {
45 { PCI_DEVICE(PCI_VENDOR_ID_HABANALABS, PCI_IDS_GOYA), },
46 { 0, }
47};
48MODULE_DEVICE_TABLE(pci, ids);
49
50
51
52
53
54
55
56
57
58static enum hl_asic_type get_asic_type(u16 device)
59{
60 enum hl_asic_type asic_type;
61
62 switch (device) {
63 case PCI_IDS_GOYA:
64 asic_type = ASIC_GOYA;
65 break;
66 default:
67 asic_type = ASIC_INVALID;
68 break;
69 }
70
71 return asic_type;
72}
73
74
75
76
77
78
79
80
81
82int hl_device_open(struct inode *inode, struct file *filp)
83{
84 struct hl_device *hdev;
85 struct hl_fpriv *hpriv;
86 int rc;
87
88 mutex_lock(&hl_devs_idr_lock);
89 hdev = idr_find(&hl_devs_idr, iminor(inode));
90 mutex_unlock(&hl_devs_idr_lock);
91
92 if (!hdev) {
93 pr_err("Couldn't find device %d:%d\n",
94 imajor(inode), iminor(inode));
95 return -ENXIO;
96 }
97
98 mutex_lock(&hdev->fd_open_cnt_lock);
99
100 if (hl_device_disabled_or_in_reset(hdev)) {
101 dev_err_ratelimited(hdev->dev,
102 "Can't open %s because it is disabled or in reset\n",
103 dev_name(hdev->dev));
104 mutex_unlock(&hdev->fd_open_cnt_lock);
105 return -EPERM;
106 }
107
108 if (hdev->in_debug) {
109 dev_err_ratelimited(hdev->dev,
110 "Can't open %s because it is being debugged by another user\n",
111 dev_name(hdev->dev));
112 mutex_unlock(&hdev->fd_open_cnt_lock);
113 return -EPERM;
114 }
115
116 if (atomic_read(&hdev->fd_open_cnt)) {
117 dev_info_ratelimited(hdev->dev,
118 "Can't open %s because another user is working on it\n",
119 dev_name(hdev->dev));
120 mutex_unlock(&hdev->fd_open_cnt_lock);
121 return -EBUSY;
122 }
123
124 atomic_inc(&hdev->fd_open_cnt);
125
126 mutex_unlock(&hdev->fd_open_cnt_lock);
127
128 hpriv = kzalloc(sizeof(*hpriv), GFP_KERNEL);
129 if (!hpriv) {
130 rc = -ENOMEM;
131 goto close_device;
132 }
133
134 hpriv->hdev = hdev;
135 filp->private_data = hpriv;
136 hpriv->filp = filp;
137 mutex_init(&hpriv->restore_phase_mutex);
138 kref_init(&hpriv->refcount);
139 nonseekable_open(inode, filp);
140
141 hl_cb_mgr_init(&hpriv->cb_mgr);
142 hl_ctx_mgr_init(&hpriv->ctx_mgr);
143
144 rc = hl_ctx_create(hdev, hpriv);
145 if (rc) {
146 dev_err(hdev->dev, "Failed to open FD (CTX fail)\n");
147 goto out_err;
148 }
149
150 hpriv->taskpid = find_get_pid(current->pid);
151
152
153
154
155
156
157 hl_device_set_frequency(hdev, PLL_HIGH);
158
159 hl_debugfs_add_file(hpriv);
160
161 return 0;
162
163out_err:
164 filp->private_data = NULL;
165 hl_ctx_mgr_fini(hpriv->hdev, &hpriv->ctx_mgr);
166 hl_cb_mgr_fini(hpriv->hdev, &hpriv->cb_mgr);
167 mutex_destroy(&hpriv->restore_phase_mutex);
168 kfree(hpriv);
169
170close_device:
171 atomic_dec(&hdev->fd_open_cnt);
172 return rc;
173}
174
175static void set_driver_behavior_per_device(struct hl_device *hdev)
176{
177 hdev->mmu_enable = 1;
178 hdev->cpu_enable = 1;
179 hdev->fw_loading = 1;
180 hdev->cpu_queues_enable = 1;
181 hdev->heartbeat = 1;
182
183 hdev->reset_pcilink = 0;
184}
185
186
187
188
189
190
191
192
193
194
195
196
197
198int create_hdev(struct hl_device **dev, struct pci_dev *pdev,
199 enum hl_asic_type asic_type, int minor)
200{
201 struct hl_device *hdev;
202 int rc;
203
204 *dev = NULL;
205
206 hdev = kzalloc(sizeof(*hdev), GFP_KERNEL);
207 if (!hdev)
208 return -ENOMEM;
209
210
211
212
213 if (pdev) {
214 hdev->asic_type = get_asic_type(pdev->device);
215 if (hdev->asic_type == ASIC_INVALID) {
216 dev_err(&pdev->dev, "Unsupported ASIC\n");
217 rc = -ENODEV;
218 goto free_hdev;
219 }
220 } else {
221 hdev->asic_type = asic_type;
222 }
223
224 hdev->major = hl_major;
225 hdev->reset_on_lockup = reset_on_lockup;
226 hdev->pldm = 0;
227
228 set_driver_behavior_per_device(hdev);
229
230 if (timeout_locked)
231 hdev->timeout_jiffies = msecs_to_jiffies(timeout_locked * 1000);
232 else
233 hdev->timeout_jiffies = MAX_SCHEDULE_TIMEOUT;
234
235 hdev->disabled = true;
236 hdev->pdev = pdev;
237
238
239 hdev->dma_mask = 32;
240
241 mutex_lock(&hl_devs_idr_lock);
242
243 if (minor == -1) {
244 rc = idr_alloc(&hl_devs_idr, hdev, 0, HL_MAX_MINORS,
245 GFP_KERNEL);
246 } else {
247 void *old_idr = idr_replace(&hl_devs_idr, hdev, minor);
248
249 if (IS_ERR_VALUE(old_idr)) {
250 rc = PTR_ERR(old_idr);
251 pr_err("Error %d when trying to replace minor %d\n",
252 rc, minor);
253 mutex_unlock(&hl_devs_idr_lock);
254 goto free_hdev;
255 }
256 rc = minor;
257 }
258
259 mutex_unlock(&hl_devs_idr_lock);
260
261 if (rc < 0) {
262 if (rc == -ENOSPC) {
263 pr_err("too many devices in the system\n");
264 rc = -EBUSY;
265 }
266 goto free_hdev;
267 }
268
269 hdev->id = rc;
270
271 *dev = hdev;
272
273 return 0;
274
275free_hdev:
276 kfree(hdev);
277 return rc;
278}
279
280
281
282
283
284
285
286void destroy_hdev(struct hl_device *hdev)
287{
288
289 mutex_lock(&hl_devs_idr_lock);
290 idr_remove(&hl_devs_idr, hdev->id);
291 mutex_unlock(&hl_devs_idr_lock);
292
293 kfree(hdev);
294}
295
296static int hl_pmops_suspend(struct device *dev)
297{
298 struct pci_dev *pdev = to_pci_dev(dev);
299 struct hl_device *hdev = pci_get_drvdata(pdev);
300
301 pr_debug("Going to suspend PCI device\n");
302
303 if (!hdev) {
304 pr_err("device pointer is NULL in suspend\n");
305 return 0;
306 }
307
308 return hl_device_suspend(hdev);
309}
310
311static int hl_pmops_resume(struct device *dev)
312{
313 struct pci_dev *pdev = to_pci_dev(dev);
314 struct hl_device *hdev = pci_get_drvdata(pdev);
315
316 pr_debug("Going to resume PCI device\n");
317
318 if (!hdev) {
319 pr_err("device pointer is NULL in resume\n");
320 return 0;
321 }
322
323 return hl_device_resume(hdev);
324}
325
326
327
328
329
330
331
332
333
334
335
336static int hl_pci_probe(struct pci_dev *pdev,
337 const struct pci_device_id *id)
338{
339 struct hl_device *hdev;
340 int rc;
341
342 dev_info(&pdev->dev, HL_NAME
343 " device found [%04x:%04x] (rev %x)\n",
344 (int)pdev->vendor, (int)pdev->device, (int)pdev->revision);
345
346 rc = create_hdev(&hdev, pdev, ASIC_INVALID, -1);
347 if (rc)
348 return rc;
349
350 pci_set_drvdata(pdev, hdev);
351
352 rc = hl_device_init(hdev, hl_class);
353 if (rc) {
354 dev_err(&pdev->dev, "Fatal error during habanalabs device init\n");
355 rc = -ENODEV;
356 goto disable_device;
357 }
358
359 return 0;
360
361disable_device:
362 pci_set_drvdata(pdev, NULL);
363 destroy_hdev(hdev);
364
365 return rc;
366}
367
368
369
370
371
372
373
374
375static void hl_pci_remove(struct pci_dev *pdev)
376{
377 struct hl_device *hdev;
378
379 hdev = pci_get_drvdata(pdev);
380 if (!hdev)
381 return;
382
383 hl_device_fini(hdev);
384 pci_set_drvdata(pdev, NULL);
385
386 destroy_hdev(hdev);
387}
388
389static const struct dev_pm_ops hl_pm_ops = {
390 .suspend = hl_pmops_suspend,
391 .resume = hl_pmops_resume,
392};
393
394static struct pci_driver hl_pci_driver = {
395 .name = HL_NAME,
396 .id_table = ids,
397 .probe = hl_pci_probe,
398 .remove = hl_pci_remove,
399 .driver.pm = &hl_pm_ops,
400};
401
402
403
404
405static int __init hl_init(void)
406{
407 int rc;
408 dev_t dev;
409
410 pr_info("loading driver\n");
411
412 rc = alloc_chrdev_region(&dev, 0, HL_MAX_MINORS, HL_NAME);
413 if (rc < 0) {
414 pr_err("unable to get major\n");
415 return rc;
416 }
417
418 hl_major = MAJOR(dev);
419
420 hl_class = class_create(THIS_MODULE, HL_NAME);
421 if (IS_ERR(hl_class)) {
422 pr_err("failed to allocate class\n");
423 rc = PTR_ERR(hl_class);
424 goto remove_major;
425 }
426
427 hl_debugfs_init();
428
429 rc = pci_register_driver(&hl_pci_driver);
430 if (rc) {
431 pr_err("failed to register pci device\n");
432 goto remove_debugfs;
433 }
434
435 pr_debug("driver loaded\n");
436
437 return 0;
438
439remove_debugfs:
440 hl_debugfs_fini();
441 class_destroy(hl_class);
442remove_major:
443 unregister_chrdev_region(MKDEV(hl_major, 0), HL_MAX_MINORS);
444 return rc;
445}
446
447
448
449
450static void __exit hl_exit(void)
451{
452 pci_unregister_driver(&hl_pci_driver);
453
454
455
456
457
458
459 hl_debugfs_fini();
460
461 class_destroy(hl_class);
462 unregister_chrdev_region(MKDEV(hl_major, 0), HL_MAX_MINORS);
463
464 idr_destroy(&hl_devs_idr);
465
466 pr_debug("driver removed\n");
467}
468
469module_init(hl_init);
470module_exit(hl_exit);
471