1
2
3
4
5
6
7
8#include <uapi/misc/habanalabs.h>
9#include "habanalabs.h"
10
11#include <linux/fs.h>
12#include <linux/uaccess.h>
13#include <linux/slab.h>
14
15static u32 hl_debug_struct_size[HL_DEBUG_OP_TIMESTAMP + 1] = {
16 [HL_DEBUG_OP_ETR] = sizeof(struct hl_debug_params_etr),
17 [HL_DEBUG_OP_ETF] = sizeof(struct hl_debug_params_etf),
18 [HL_DEBUG_OP_STM] = sizeof(struct hl_debug_params_stm),
19 [HL_DEBUG_OP_FUNNEL] = 0,
20 [HL_DEBUG_OP_BMON] = sizeof(struct hl_debug_params_bmon),
21 [HL_DEBUG_OP_SPMU] = sizeof(struct hl_debug_params_spmu),
22 [HL_DEBUG_OP_TIMESTAMP] = 0
23
24};
25
26static int device_status_info(struct hl_device *hdev, struct hl_info_args *args)
27{
28 struct hl_info_device_status dev_stat = {0};
29 u32 size = args->return_size;
30 void __user *out = (void __user *) (uintptr_t) args->return_pointer;
31
32 if ((!size) || (!out))
33 return -EINVAL;
34
35 dev_stat.status = hl_device_status(hdev);
36
37 return copy_to_user(out, &dev_stat,
38 min((size_t)size, sizeof(dev_stat))) ? -EFAULT : 0;
39}
40
41static int hw_ip_info(struct hl_device *hdev, struct hl_info_args *args)
42{
43 struct hl_info_hw_ip_info hw_ip = {0};
44 u32 size = args->return_size;
45 void __user *out = (void __user *) (uintptr_t) args->return_pointer;
46 struct asic_fixed_properties *prop = &hdev->asic_prop;
47 u64 sram_kmd_size, dram_kmd_size;
48
49 if ((!size) || (!out))
50 return -EINVAL;
51
52 sram_kmd_size = (prop->sram_user_base_address -
53 prop->sram_base_address);
54 dram_kmd_size = (prop->dram_user_base_address -
55 prop->dram_base_address);
56
57 hw_ip.device_id = hdev->asic_funcs->get_pci_id(hdev);
58 hw_ip.sram_base_address = prop->sram_user_base_address;
59 hw_ip.dram_base_address = prop->dram_user_base_address;
60 hw_ip.tpc_enabled_mask = prop->tpc_enabled_mask;
61 hw_ip.sram_size = prop->sram_size - sram_kmd_size;
62 hw_ip.dram_size = prop->dram_size - dram_kmd_size;
63 if (hw_ip.dram_size > PAGE_SIZE)
64 hw_ip.dram_enabled = 1;
65 hw_ip.num_of_events = prop->num_of_events;
66
67 memcpy(hw_ip.armcp_version, prop->armcp_info.armcp_version,
68 min(VERSION_MAX_LEN, HL_INFO_VERSION_MAX_LEN));
69
70 memcpy(hw_ip.card_name, prop->armcp_info.card_name,
71 min(CARD_NAME_MAX_LEN, HL_INFO_CARD_NAME_MAX_LEN));
72
73 hw_ip.armcp_cpld_version = le32_to_cpu(prop->armcp_info.cpld_version);
74 hw_ip.module_id = le32_to_cpu(prop->armcp_info.card_location);
75
76 hw_ip.psoc_pci_pll_nr = prop->psoc_pci_pll_nr;
77 hw_ip.psoc_pci_pll_nf = prop->psoc_pci_pll_nf;
78 hw_ip.psoc_pci_pll_od = prop->psoc_pci_pll_od;
79 hw_ip.psoc_pci_pll_div_factor = prop->psoc_pci_pll_div_factor;
80
81 return copy_to_user(out, &hw_ip,
82 min((size_t)size, sizeof(hw_ip))) ? -EFAULT : 0;
83}
84
85static int hw_events_info(struct hl_device *hdev, bool aggregate,
86 struct hl_info_args *args)
87{
88 u32 size, max_size = args->return_size;
89 void __user *out = (void __user *) (uintptr_t) args->return_pointer;
90 void *arr;
91
92 if ((!max_size) || (!out))
93 return -EINVAL;
94
95 arr = hdev->asic_funcs->get_events_stat(hdev, aggregate, &size);
96
97 return copy_to_user(out, arr, min(max_size, size)) ? -EFAULT : 0;
98}
99
100static int dram_usage_info(struct hl_fpriv *hpriv, struct hl_info_args *args)
101{
102 struct hl_device *hdev = hpriv->hdev;
103 struct hl_info_dram_usage dram_usage = {0};
104 u32 max_size = args->return_size;
105 void __user *out = (void __user *) (uintptr_t) args->return_pointer;
106 struct asic_fixed_properties *prop = &hdev->asic_prop;
107 u64 dram_kmd_size;
108
109 if ((!max_size) || (!out))
110 return -EINVAL;
111
112 dram_kmd_size = (prop->dram_user_base_address -
113 prop->dram_base_address);
114 dram_usage.dram_free_mem = (prop->dram_size - dram_kmd_size) -
115 atomic64_read(&hdev->dram_used_mem);
116 if (hpriv->ctx)
117 dram_usage.ctx_dram_mem =
118 atomic64_read(&hpriv->ctx->dram_phys_mem);
119
120 return copy_to_user(out, &dram_usage,
121 min((size_t) max_size, sizeof(dram_usage))) ? -EFAULT : 0;
122}
123
124static int hw_idle(struct hl_device *hdev, struct hl_info_args *args)
125{
126 struct hl_info_hw_idle hw_idle = {0};
127 u32 max_size = args->return_size;
128 void __user *out = (void __user *) (uintptr_t) args->return_pointer;
129
130 if ((!max_size) || (!out))
131 return -EINVAL;
132
133 hw_idle.is_idle = hdev->asic_funcs->is_device_idle(hdev,
134 &hw_idle.busy_engines_mask, NULL);
135
136 return copy_to_user(out, &hw_idle,
137 min((size_t) max_size, sizeof(hw_idle))) ? -EFAULT : 0;
138}
139
140static int debug_coresight(struct hl_device *hdev, struct hl_debug_args *args)
141{
142 struct hl_debug_params *params;
143 void *input = NULL, *output = NULL;
144 int rc;
145
146 params = kzalloc(sizeof(*params), GFP_KERNEL);
147 if (!params)
148 return -ENOMEM;
149
150 params->reg_idx = args->reg_idx;
151 params->enable = args->enable;
152 params->op = args->op;
153
154 if (args->input_ptr && args->input_size) {
155 input = kzalloc(hl_debug_struct_size[args->op], GFP_KERNEL);
156 if (!input) {
157 rc = -ENOMEM;
158 goto out;
159 }
160
161 if (copy_from_user(input, u64_to_user_ptr(args->input_ptr),
162 args->input_size)) {
163 rc = -EFAULT;
164 dev_err(hdev->dev, "failed to copy input debug data\n");
165 goto out;
166 }
167
168 params->input = input;
169 }
170
171 if (args->output_ptr && args->output_size) {
172 output = kzalloc(args->output_size, GFP_KERNEL);
173 if (!output) {
174 rc = -ENOMEM;
175 goto out;
176 }
177
178 params->output = output;
179 params->output_size = args->output_size;
180 }
181
182 rc = hdev->asic_funcs->debug_coresight(hdev, params);
183 if (rc) {
184 dev_err(hdev->dev,
185 "debug coresight operation failed %d\n", rc);
186 goto out;
187 }
188
189 if (output && copy_to_user((void __user *) (uintptr_t) args->output_ptr,
190 output, args->output_size)) {
191 dev_err(hdev->dev, "copy to user failed in debug ioctl\n");
192 rc = -EFAULT;
193 goto out;
194 }
195
196
197out:
198 kfree(params);
199 kfree(output);
200 kfree(input);
201
202 return rc;
203}
204
205static int device_utilization(struct hl_device *hdev, struct hl_info_args *args)
206{
207 struct hl_info_device_utilization device_util = {0};
208 u32 max_size = args->return_size;
209 void __user *out = (void __user *) (uintptr_t) args->return_pointer;
210
211 if ((!max_size) || (!out))
212 return -EINVAL;
213
214 if ((args->period_ms < 100) || (args->period_ms > 1000) ||
215 (args->period_ms % 100)) {
216 dev_err(hdev->dev,
217 "period %u must be between 100 - 1000 and must be divisible by 100\n",
218 args->period_ms);
219 return -EINVAL;
220 }
221
222 device_util.utilization = hl_device_utilization(hdev, args->period_ms);
223
224 return copy_to_user(out, &device_util,
225 min((size_t) max_size, sizeof(device_util))) ? -EFAULT : 0;
226}
227
228static int get_clk_rate(struct hl_device *hdev, struct hl_info_args *args)
229{
230 struct hl_info_clk_rate clk_rate = {0};
231 u32 max_size = args->return_size;
232 void __user *out = (void __user *) (uintptr_t) args->return_pointer;
233 int rc;
234
235 if ((!max_size) || (!out))
236 return -EINVAL;
237
238 rc = hdev->asic_funcs->get_clk_rate(hdev, &clk_rate.cur_clk_rate_mhz,
239 &clk_rate.max_clk_rate_mhz);
240 if (rc)
241 return rc;
242
243 return copy_to_user(out, &clk_rate,
244 min((size_t) max_size, sizeof(clk_rate))) ? -EFAULT : 0;
245}
246
247static int get_reset_count(struct hl_device *hdev, struct hl_info_args *args)
248{
249 struct hl_info_reset_count reset_count = {0};
250 u32 max_size = args->return_size;
251 void __user *out = (void __user *) (uintptr_t) args->return_pointer;
252
253 if ((!max_size) || (!out))
254 return -EINVAL;
255
256 reset_count.hard_reset_cnt = hdev->hard_reset_cnt;
257 reset_count.soft_reset_cnt = hdev->soft_reset_cnt;
258
259 return copy_to_user(out, &reset_count,
260 min((size_t) max_size, sizeof(reset_count))) ? -EFAULT : 0;
261}
262
263static int time_sync_info(struct hl_device *hdev, struct hl_info_args *args)
264{
265 struct hl_info_time_sync time_sync = {0};
266 u32 max_size = args->return_size;
267 void __user *out = (void __user *) (uintptr_t) args->return_pointer;
268
269 if ((!max_size) || (!out))
270 return -EINVAL;
271
272 time_sync.device_time = hdev->asic_funcs->get_device_time(hdev);
273 time_sync.host_time = ktime_get_raw_ns();
274
275 return copy_to_user(out, &time_sync,
276 min((size_t) max_size, sizeof(time_sync))) ? -EFAULT : 0;
277}
278
279static int cs_counters_info(struct hl_fpriv *hpriv, struct hl_info_args *args)
280{
281 struct hl_device *hdev = hpriv->hdev;
282 struct hl_info_cs_counters cs_counters = {0};
283 u32 max_size = args->return_size;
284 void __user *out = (void __user *) (uintptr_t) args->return_pointer;
285
286 if ((!max_size) || (!out))
287 return -EINVAL;
288
289 memcpy(&cs_counters.cs_counters, &hdev->aggregated_cs_counters,
290 sizeof(struct hl_cs_counters));
291
292 if (hpriv->ctx)
293 memcpy(&cs_counters.ctx_cs_counters, &hpriv->ctx->cs_counters,
294 sizeof(struct hl_cs_counters));
295
296 return copy_to_user(out, &cs_counters,
297 min((size_t) max_size, sizeof(cs_counters))) ? -EFAULT : 0;
298}
299
300static int _hl_info_ioctl(struct hl_fpriv *hpriv, void *data,
301 struct device *dev)
302{
303 struct hl_info_args *args = data;
304 struct hl_device *hdev = hpriv->hdev;
305 int rc;
306
307
308
309
310
311 switch (args->op) {
312 case HL_INFO_HW_IP_INFO:
313 return hw_ip_info(hdev, args);
314
315 case HL_INFO_DEVICE_STATUS:
316 return device_status_info(hdev, args);
317
318 case HL_INFO_RESET_COUNT:
319 return get_reset_count(hdev, args);
320
321 default:
322 break;
323 }
324
325 if (hl_device_disabled_or_in_reset(hdev)) {
326 dev_warn_ratelimited(dev,
327 "Device is %s. Can't execute INFO IOCTL\n",
328 atomic_read(&hdev->in_reset) ? "in_reset" : "disabled");
329 return -EBUSY;
330 }
331
332 switch (args->op) {
333 case HL_INFO_HW_EVENTS:
334 rc = hw_events_info(hdev, false, args);
335 break;
336
337 case HL_INFO_DRAM_USAGE:
338 rc = dram_usage_info(hpriv, args);
339 break;
340
341 case HL_INFO_HW_IDLE:
342 rc = hw_idle(hdev, args);
343 break;
344
345 case HL_INFO_DEVICE_UTILIZATION:
346 rc = device_utilization(hdev, args);
347 break;
348
349 case HL_INFO_HW_EVENTS_AGGREGATE:
350 rc = hw_events_info(hdev, true, args);
351 break;
352
353 case HL_INFO_CLK_RATE:
354 rc = get_clk_rate(hdev, args);
355 break;
356
357 case HL_INFO_TIME_SYNC:
358 return time_sync_info(hdev, args);
359
360 case HL_INFO_CS_COUNTERS:
361 return cs_counters_info(hpriv, args);
362
363 default:
364 dev_err(dev, "Invalid request %d\n", args->op);
365 rc = -ENOTTY;
366 break;
367 }
368
369 return rc;
370}
371
372static int hl_info_ioctl(struct hl_fpriv *hpriv, void *data)
373{
374 return _hl_info_ioctl(hpriv, data, hpriv->hdev->dev);
375}
376
377static int hl_info_ioctl_control(struct hl_fpriv *hpriv, void *data)
378{
379 return _hl_info_ioctl(hpriv, data, hpriv->hdev->dev_ctrl);
380}
381
382static int hl_debug_ioctl(struct hl_fpriv *hpriv, void *data)
383{
384 struct hl_debug_args *args = data;
385 struct hl_device *hdev = hpriv->hdev;
386 int rc = 0;
387
388 if (hl_device_disabled_or_in_reset(hdev)) {
389 dev_warn_ratelimited(hdev->dev,
390 "Device is %s. Can't execute DEBUG IOCTL\n",
391 atomic_read(&hdev->in_reset) ? "in_reset" : "disabled");
392 return -EBUSY;
393 }
394
395 switch (args->op) {
396 case HL_DEBUG_OP_ETR:
397 case HL_DEBUG_OP_ETF:
398 case HL_DEBUG_OP_STM:
399 case HL_DEBUG_OP_FUNNEL:
400 case HL_DEBUG_OP_BMON:
401 case HL_DEBUG_OP_SPMU:
402 case HL_DEBUG_OP_TIMESTAMP:
403 if (!hdev->in_debug) {
404 dev_err_ratelimited(hdev->dev,
405 "Rejecting debug configuration request because device not in debug mode\n");
406 return -EFAULT;
407 }
408 args->input_size =
409 min(args->input_size, hl_debug_struct_size[args->op]);
410 rc = debug_coresight(hdev, args);
411 break;
412 case HL_DEBUG_OP_SET_MODE:
413 rc = hl_device_set_debug_mode(hdev, (bool) args->enable);
414 break;
415 default:
416 dev_err(hdev->dev, "Invalid request %d\n", args->op);
417 rc = -ENOTTY;
418 break;
419 }
420
421 return rc;
422}
423
424#define HL_IOCTL_DEF(ioctl, _func) \
425 [_IOC_NR(ioctl)] = {.cmd = ioctl, .func = _func}
426
427static const struct hl_ioctl_desc hl_ioctls[] = {
428 HL_IOCTL_DEF(HL_IOCTL_INFO, hl_info_ioctl),
429 HL_IOCTL_DEF(HL_IOCTL_CB, hl_cb_ioctl),
430 HL_IOCTL_DEF(HL_IOCTL_CS, hl_cs_ioctl),
431 HL_IOCTL_DEF(HL_IOCTL_WAIT_CS, hl_cs_wait_ioctl),
432 HL_IOCTL_DEF(HL_IOCTL_MEMORY, hl_mem_ioctl),
433 HL_IOCTL_DEF(HL_IOCTL_DEBUG, hl_debug_ioctl)
434};
435
436static const struct hl_ioctl_desc hl_ioctls_control[] = {
437 HL_IOCTL_DEF(HL_IOCTL_INFO, hl_info_ioctl_control)
438};
439
440static long _hl_ioctl(struct file *filep, unsigned int cmd, unsigned long arg,
441 const struct hl_ioctl_desc *ioctl, struct device *dev)
442{
443 struct hl_fpriv *hpriv = filep->private_data;
444 struct hl_device *hdev = hpriv->hdev;
445 unsigned int nr = _IOC_NR(cmd);
446 char stack_kdata[128] = {0};
447 char *kdata = NULL;
448 unsigned int usize, asize;
449 hl_ioctl_t *func;
450 u32 hl_size;
451 int retcode;
452
453 if (hdev->hard_reset_pending) {
454 dev_crit_ratelimited(hdev->dev_ctrl,
455 "Device HARD reset pending! Please close FD\n");
456 return -ENODEV;
457 }
458
459
460 func = ioctl->func;
461
462 if (unlikely(!func)) {
463 dev_dbg(dev, "no function\n");
464 retcode = -ENOTTY;
465 goto out_err;
466 }
467
468 hl_size = _IOC_SIZE(ioctl->cmd);
469 usize = asize = _IOC_SIZE(cmd);
470 if (hl_size > asize)
471 asize = hl_size;
472
473 cmd = ioctl->cmd;
474
475 if (cmd & (IOC_IN | IOC_OUT)) {
476 if (asize <= sizeof(stack_kdata)) {
477 kdata = stack_kdata;
478 } else {
479 kdata = kzalloc(asize, GFP_KERNEL);
480 if (!kdata) {
481 retcode = -ENOMEM;
482 goto out_err;
483 }
484 }
485 }
486
487 if (cmd & IOC_IN) {
488 if (copy_from_user(kdata, (void __user *)arg, usize)) {
489 retcode = -EFAULT;
490 goto out_err;
491 }
492 } else if (cmd & IOC_OUT) {
493 memset(kdata, 0, usize);
494 }
495
496 retcode = func(hpriv, kdata);
497
498 if ((cmd & IOC_OUT) && copy_to_user((void __user *)arg, kdata, usize))
499 retcode = -EFAULT;
500
501out_err:
502 if (retcode)
503 dev_dbg(dev, "error in ioctl: pid=%d, cmd=0x%02x, nr=0x%02x\n",
504 task_pid_nr(current), cmd, nr);
505
506 if (kdata != stack_kdata)
507 kfree(kdata);
508
509 return retcode;
510}
511
512long hl_ioctl(struct file *filep, unsigned int cmd, unsigned long arg)
513{
514 struct hl_fpriv *hpriv = filep->private_data;
515 struct hl_device *hdev = hpriv->hdev;
516 const struct hl_ioctl_desc *ioctl = NULL;
517 unsigned int nr = _IOC_NR(cmd);
518
519 if ((nr >= HL_COMMAND_START) && (nr < HL_COMMAND_END)) {
520 ioctl = &hl_ioctls[nr];
521 } else {
522 dev_err(hdev->dev, "invalid ioctl: pid=%d, nr=0x%02x\n",
523 task_pid_nr(current), nr);
524 return -ENOTTY;
525 }
526
527 return _hl_ioctl(filep, cmd, arg, ioctl, hdev->dev);
528}
529
530long hl_ioctl_control(struct file *filep, unsigned int cmd, unsigned long arg)
531{
532 struct hl_fpriv *hpriv = filep->private_data;
533 struct hl_device *hdev = hpriv->hdev;
534 const struct hl_ioctl_desc *ioctl = NULL;
535 unsigned int nr = _IOC_NR(cmd);
536
537 if (nr == _IOC_NR(HL_IOCTL_INFO)) {
538 ioctl = &hl_ioctls_control[nr];
539 } else {
540 dev_err(hdev->dev_ctrl, "invalid ioctl: pid=%d, nr=0x%02x\n",
541 task_pid_nr(current), nr);
542 return -ENOTTY;
543 }
544
545 return _hl_ioctl(filep, cmd, arg, ioctl, hdev->dev_ctrl);
546}
547
