1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32#define CREATE_TRACE_POINTS
33#include "lib/eq.h"
34#include "fw_tracer.h"
35#include "fw_tracer_tracepoint.h"
36
37static int mlx5_query_mtrc_caps(struct mlx5_fw_tracer *tracer)
38{
39 u32 *string_db_base_address_out = tracer->str_db.base_address_out;
40 u32 *string_db_size_out = tracer->str_db.size_out;
41 struct mlx5_core_dev *dev = tracer->dev;
42 u32 out[MLX5_ST_SZ_DW(mtrc_cap)] = {0};
43 u32 in[MLX5_ST_SZ_DW(mtrc_cap)] = {0};
44 void *mtrc_cap_sp;
45 int err, i;
46
47 err = mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out),
48 MLX5_REG_MTRC_CAP, 0, 0);
49 if (err) {
50 mlx5_core_warn(dev, "FWTracer: Error reading tracer caps %d\n",
51 err);
52 return err;
53 }
54
55 if (!MLX5_GET(mtrc_cap, out, trace_to_memory)) {
56 mlx5_core_dbg(dev, "FWTracer: Device does not support logging traces to memory\n");
57 return -ENOTSUPP;
58 }
59
60 tracer->trc_ver = MLX5_GET(mtrc_cap, out, trc_ver);
61 tracer->str_db.first_string_trace =
62 MLX5_GET(mtrc_cap, out, first_string_trace);
63 tracer->str_db.num_string_trace =
64 MLX5_GET(mtrc_cap, out, num_string_trace);
65 tracer->str_db.num_string_db = MLX5_GET(mtrc_cap, out, num_string_db);
66 tracer->owner = !!MLX5_GET(mtrc_cap, out, trace_owner);
67
68 for (i = 0; i < tracer->str_db.num_string_db; i++) {
69 mtrc_cap_sp = MLX5_ADDR_OF(mtrc_cap, out, string_db_param[i]);
70 string_db_base_address_out[i] = MLX5_GET(mtrc_string_db_param,
71 mtrc_cap_sp,
72 string_db_base_address);
73 string_db_size_out[i] = MLX5_GET(mtrc_string_db_param,
74 mtrc_cap_sp, string_db_size);
75 }
76
77 return err;
78}
79
80static int mlx5_set_mtrc_caps_trace_owner(struct mlx5_fw_tracer *tracer,
81 u32 *out, u32 out_size,
82 u8 trace_owner)
83{
84 struct mlx5_core_dev *dev = tracer->dev;
85 u32 in[MLX5_ST_SZ_DW(mtrc_cap)] = {0};
86
87 MLX5_SET(mtrc_cap, in, trace_owner, trace_owner);
88
89 return mlx5_core_access_reg(dev, in, sizeof(in), out, out_size,
90 MLX5_REG_MTRC_CAP, 0, 1);
91}
92
93static int mlx5_fw_tracer_ownership_acquire(struct mlx5_fw_tracer *tracer)
94{
95 struct mlx5_core_dev *dev = tracer->dev;
96 u32 out[MLX5_ST_SZ_DW(mtrc_cap)] = {0};
97 int err;
98
99 err = mlx5_set_mtrc_caps_trace_owner(tracer, out, sizeof(out),
100 MLX5_FW_TRACER_ACQUIRE_OWNERSHIP);
101 if (err) {
102 mlx5_core_warn(dev, "FWTracer: Acquire tracer ownership failed %d\n",
103 err);
104 return err;
105 }
106
107 tracer->owner = !!MLX5_GET(mtrc_cap, out, trace_owner);
108
109 if (!tracer->owner)
110 return -EBUSY;
111
112 return 0;
113}
114
115static void mlx5_fw_tracer_ownership_release(struct mlx5_fw_tracer *tracer)
116{
117 u32 out[MLX5_ST_SZ_DW(mtrc_cap)] = {0};
118
119 mlx5_set_mtrc_caps_trace_owner(tracer, out, sizeof(out),
120 MLX5_FW_TRACER_RELEASE_OWNERSHIP);
121 tracer->owner = false;
122}
123
124static int mlx5_fw_tracer_create_log_buf(struct mlx5_fw_tracer *tracer)
125{
126 struct mlx5_core_dev *dev = tracer->dev;
127 struct device *ddev;
128 dma_addr_t dma;
129 void *buff;
130 gfp_t gfp;
131 int err;
132
133 tracer->buff.size = TRACE_BUFFER_SIZE_BYTE;
134
135 gfp = GFP_KERNEL | __GFP_ZERO;
136 buff = (void *)__get_free_pages(gfp,
137 get_order(tracer->buff.size));
138 if (!buff) {
139 err = -ENOMEM;
140 mlx5_core_warn(dev, "FWTracer: Failed to allocate pages, %d\n", err);
141 return err;
142 }
143 tracer->buff.log_buf = buff;
144
145 ddev = mlx5_core_dma_dev(dev);
146 dma = dma_map_single(ddev, buff, tracer->buff.size, DMA_FROM_DEVICE);
147 if (dma_mapping_error(ddev, dma)) {
148 mlx5_core_warn(dev, "FWTracer: Unable to map DMA: %d\n",
149 dma_mapping_error(ddev, dma));
150 err = -ENOMEM;
151 goto free_pages;
152 }
153 tracer->buff.dma = dma;
154
155 return 0;
156
157free_pages:
158 free_pages((unsigned long)tracer->buff.log_buf, get_order(tracer->buff.size));
159
160 return err;
161}
162
163static void mlx5_fw_tracer_destroy_log_buf(struct mlx5_fw_tracer *tracer)
164{
165 struct mlx5_core_dev *dev = tracer->dev;
166 struct device *ddev;
167
168 if (!tracer->buff.log_buf)
169 return;
170
171 ddev = mlx5_core_dma_dev(dev);
172 dma_unmap_single(ddev, tracer->buff.dma, tracer->buff.size, DMA_FROM_DEVICE);
173 free_pages((unsigned long)tracer->buff.log_buf, get_order(tracer->buff.size));
174}
175
176static int mlx5_fw_tracer_create_mkey(struct mlx5_fw_tracer *tracer)
177{
178 struct mlx5_core_dev *dev = tracer->dev;
179 int err, inlen, i;
180 __be64 *mtt;
181 void *mkc;
182 u32 *in;
183
184 inlen = MLX5_ST_SZ_BYTES(create_mkey_in) +
185 sizeof(*mtt) * round_up(TRACER_BUFFER_PAGE_NUM, 2);
186
187 in = kvzalloc(inlen, GFP_KERNEL);
188 if (!in)
189 return -ENOMEM;
190
191 MLX5_SET(create_mkey_in, in, translations_octword_actual_size,
192 DIV_ROUND_UP(TRACER_BUFFER_PAGE_NUM, 2));
193 mtt = (__be64 *)MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt);
194 for (i = 0 ; i < TRACER_BUFFER_PAGE_NUM ; i++)
195 mtt[i] = cpu_to_be64(tracer->buff.dma + i * PAGE_SIZE);
196
197 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
198 MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_MTT);
199 MLX5_SET(mkc, mkc, lr, 1);
200 MLX5_SET(mkc, mkc, lw, 1);
201 MLX5_SET(mkc, mkc, pd, tracer->buff.pdn);
202 MLX5_SET(mkc, mkc, bsf_octword_size, 0);
203 MLX5_SET(mkc, mkc, qpn, 0xffffff);
204 MLX5_SET(mkc, mkc, log_page_size, PAGE_SHIFT);
205 MLX5_SET(mkc, mkc, translations_octword_size,
206 DIV_ROUND_UP(TRACER_BUFFER_PAGE_NUM, 2));
207 MLX5_SET64(mkc, mkc, start_addr, tracer->buff.dma);
208 MLX5_SET64(mkc, mkc, len, tracer->buff.size);
209 err = mlx5_core_create_mkey(dev, &tracer->buff.mkey, in, inlen);
210 if (err)
211 mlx5_core_warn(dev, "FWTracer: Failed to create mkey, %d\n", err);
212
213 kvfree(in);
214
215 return err;
216}
217
218static void mlx5_fw_tracer_free_strings_db(struct mlx5_fw_tracer *tracer)
219{
220 u32 num_string_db = tracer->str_db.num_string_db;
221 int i;
222
223 for (i = 0; i < num_string_db; i++) {
224 kfree(tracer->str_db.buffer[i]);
225 tracer->str_db.buffer[i] = NULL;
226 }
227}
228
229static int mlx5_fw_tracer_allocate_strings_db(struct mlx5_fw_tracer *tracer)
230{
231 u32 *string_db_size_out = tracer->str_db.size_out;
232 u32 num_string_db = tracer->str_db.num_string_db;
233 int i;
234
235 for (i = 0; i < num_string_db; i++) {
236 tracer->str_db.buffer[i] = kzalloc(string_db_size_out[i], GFP_KERNEL);
237 if (!tracer->str_db.buffer[i])
238 goto free_strings_db;
239 }
240
241 return 0;
242
243free_strings_db:
244 mlx5_fw_tracer_free_strings_db(tracer);
245 return -ENOMEM;
246}
247
248static void
249mlx5_fw_tracer_init_saved_traces_array(struct mlx5_fw_tracer *tracer)
250{
251 tracer->st_arr.saved_traces_index = 0;
252 mutex_init(&tracer->st_arr.lock);
253}
254
255static void
256mlx5_fw_tracer_clean_saved_traces_array(struct mlx5_fw_tracer *tracer)
257{
258 mutex_destroy(&tracer->st_arr.lock);
259}
260
261static void mlx5_tracer_read_strings_db(struct work_struct *work)
262{
263 struct mlx5_fw_tracer *tracer = container_of(work, struct mlx5_fw_tracer,
264 read_fw_strings_work);
265 u32 num_of_reads, num_string_db = tracer->str_db.num_string_db;
266 struct mlx5_core_dev *dev = tracer->dev;
267 u32 in[MLX5_ST_SZ_DW(mtrc_cap)] = {0};
268 u32 leftovers, offset;
269 int err = 0, i, j;
270 u32 *out, outlen;
271 void *out_value;
272
273 outlen = MLX5_ST_SZ_BYTES(mtrc_stdb) + STRINGS_DB_READ_SIZE_BYTES;
274 out = kzalloc(outlen, GFP_KERNEL);
275 if (!out) {
276 err = -ENOMEM;
277 goto out;
278 }
279
280 for (i = 0; i < num_string_db; i++) {
281 offset = 0;
282 MLX5_SET(mtrc_stdb, in, string_db_index, i);
283 num_of_reads = tracer->str_db.size_out[i] /
284 STRINGS_DB_READ_SIZE_BYTES;
285 leftovers = (tracer->str_db.size_out[i] %
286 STRINGS_DB_READ_SIZE_BYTES) /
287 STRINGS_DB_LEFTOVER_SIZE_BYTES;
288
289 MLX5_SET(mtrc_stdb, in, read_size, STRINGS_DB_READ_SIZE_BYTES);
290 for (j = 0; j < num_of_reads; j++) {
291 MLX5_SET(mtrc_stdb, in, start_offset, offset);
292
293 err = mlx5_core_access_reg(dev, in, sizeof(in), out,
294 outlen, MLX5_REG_MTRC_STDB,
295 0, 1);
296 if (err) {
297 mlx5_core_dbg(dev, "FWTracer: Failed to read strings DB %d\n",
298 err);
299 goto out_free;
300 }
301
302 out_value = MLX5_ADDR_OF(mtrc_stdb, out, string_db_data);
303 memcpy(tracer->str_db.buffer[i] + offset, out_value,
304 STRINGS_DB_READ_SIZE_BYTES);
305 offset += STRINGS_DB_READ_SIZE_BYTES;
306 }
307
308
309 MLX5_SET(mtrc_stdb, in, read_size,
310 STRINGS_DB_LEFTOVER_SIZE_BYTES);
311 for (j = 0; j < leftovers; j++) {
312 MLX5_SET(mtrc_stdb, in, start_offset, offset);
313
314 err = mlx5_core_access_reg(dev, in, sizeof(in), out,
315 outlen, MLX5_REG_MTRC_STDB,
316 0, 1);
317 if (err) {
318 mlx5_core_dbg(dev, "FWTracer: Failed to read strings DB %d\n",
319 err);
320 goto out_free;
321 }
322
323 out_value = MLX5_ADDR_OF(mtrc_stdb, out, string_db_data);
324 memcpy(tracer->str_db.buffer[i] + offset, out_value,
325 STRINGS_DB_LEFTOVER_SIZE_BYTES);
326 offset += STRINGS_DB_LEFTOVER_SIZE_BYTES;
327 }
328 }
329
330 tracer->str_db.loaded = true;
331
332out_free:
333 kfree(out);
334out:
335 return;
336}
337
338static void mlx5_fw_tracer_arm(struct mlx5_core_dev *dev)
339{
340 u32 out[MLX5_ST_SZ_DW(mtrc_ctrl)] = {0};
341 u32 in[MLX5_ST_SZ_DW(mtrc_ctrl)] = {0};
342 int err;
343
344 MLX5_SET(mtrc_ctrl, in, arm_event, 1);
345
346 err = mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out),
347 MLX5_REG_MTRC_CTRL, 0, 1);
348 if (err)
349 mlx5_core_warn(dev, "FWTracer: Failed to arm tracer event %d\n", err);
350}
351
352static const char *VAL_PARM = "%llx";
353static const char *REPLACE_64_VAL_PARM = "%x%x";
354static const char *PARAM_CHAR = "%";
355
356static int mlx5_tracer_message_hash(u32 message_id)
357{
358 return jhash_1word(message_id, 0) & (MESSAGE_HASH_SIZE - 1);
359}
360
361static struct tracer_string_format *mlx5_tracer_message_insert(struct mlx5_fw_tracer *tracer,
362 struct tracer_event *tracer_event)
363{
364 struct hlist_head *head =
365 &tracer->hash[mlx5_tracer_message_hash(tracer_event->string_event.tmsn)];
366 struct tracer_string_format *cur_string;
367
368 cur_string = kzalloc(sizeof(*cur_string), GFP_KERNEL);
369 if (!cur_string)
370 return NULL;
371
372 hlist_add_head(&cur_string->hlist, head);
373
374 return cur_string;
375}
376
377static struct tracer_string_format *mlx5_tracer_get_string(struct mlx5_fw_tracer *tracer,
378 struct tracer_event *tracer_event)
379{
380 struct tracer_string_format *cur_string;
381 u32 str_ptr, offset;
382 int i;
383
384 str_ptr = tracer_event->string_event.string_param;
385
386 for (i = 0; i < tracer->str_db.num_string_db; i++) {
387 if (str_ptr > tracer->str_db.base_address_out[i] &&
388 str_ptr < tracer->str_db.base_address_out[i] +
389 tracer->str_db.size_out[i]) {
390 offset = str_ptr - tracer->str_db.base_address_out[i];
391
392 cur_string = mlx5_tracer_message_insert(tracer, tracer_event);
393 if (!cur_string)
394 return NULL;
395 cur_string->string = (char *)(tracer->str_db.buffer[i] +
396 offset);
397 return cur_string;
398 }
399 }
400
401 return NULL;
402}
403
404static void mlx5_tracer_clean_message(struct tracer_string_format *str_frmt)
405{
406 hlist_del(&str_frmt->hlist);
407 kfree(str_frmt);
408}
409
410static int mlx5_tracer_get_num_of_params(char *str)
411{
412 char *substr, *pstr = str;
413 int num_of_params = 0;
414
415
416 substr = strstr(pstr, VAL_PARM);
417 while (substr) {
418 memcpy(substr, REPLACE_64_VAL_PARM, 4);
419 pstr = substr;
420 substr = strstr(pstr, VAL_PARM);
421 }
422
423
424 substr = strstr(str, PARAM_CHAR);
425 while (substr) {
426 num_of_params += 1;
427 str = substr + 1;
428 substr = strstr(str, PARAM_CHAR);
429 }
430
431 return num_of_params;
432}
433
434static struct tracer_string_format *mlx5_tracer_message_find(struct hlist_head *head,
435 u8 event_id, u32 tmsn)
436{
437 struct tracer_string_format *message;
438
439 hlist_for_each_entry(message, head, hlist)
440 if (message->event_id == event_id && message->tmsn == tmsn)
441 return message;
442
443 return NULL;
444}
445
446static struct tracer_string_format *mlx5_tracer_message_get(struct mlx5_fw_tracer *tracer,
447 struct tracer_event *tracer_event)
448{
449 struct hlist_head *head =
450 &tracer->hash[mlx5_tracer_message_hash(tracer_event->string_event.tmsn)];
451
452 return mlx5_tracer_message_find(head, tracer_event->event_id, tracer_event->string_event.tmsn);
453}
454
455static void poll_trace(struct mlx5_fw_tracer *tracer,
456 struct tracer_event *tracer_event, u64 *trace)
457{
458 u32 timestamp_low, timestamp_mid, timestamp_high, urts;
459
460 tracer_event->event_id = MLX5_GET(tracer_event, trace, event_id);
461 tracer_event->lost_event = MLX5_GET(tracer_event, trace, lost);
462
463 switch (tracer_event->event_id) {
464 case TRACER_EVENT_TYPE_TIMESTAMP:
465 tracer_event->type = TRACER_EVENT_TYPE_TIMESTAMP;
466 urts = MLX5_GET(tracer_timestamp_event, trace, urts);
467 if (tracer->trc_ver == 0)
468 tracer_event->timestamp_event.unreliable = !!(urts >> 2);
469 else
470 tracer_event->timestamp_event.unreliable = !!(urts & 1);
471
472 timestamp_low = MLX5_GET(tracer_timestamp_event,
473 trace, timestamp7_0);
474 timestamp_mid = MLX5_GET(tracer_timestamp_event,
475 trace, timestamp39_8);
476 timestamp_high = MLX5_GET(tracer_timestamp_event,
477 trace, timestamp52_40);
478
479 tracer_event->timestamp_event.timestamp =
480 ((u64)timestamp_high << 40) |
481 ((u64)timestamp_mid << 8) |
482 (u64)timestamp_low;
483 break;
484 default:
485 if (tracer_event->event_id >= tracer->str_db.first_string_trace ||
486 tracer_event->event_id <= tracer->str_db.first_string_trace +
487 tracer->str_db.num_string_trace) {
488 tracer_event->type = TRACER_EVENT_TYPE_STRING;
489 tracer_event->string_event.timestamp =
490 MLX5_GET(tracer_string_event, trace, timestamp);
491 tracer_event->string_event.string_param =
492 MLX5_GET(tracer_string_event, trace, string_param);
493 tracer_event->string_event.tmsn =
494 MLX5_GET(tracer_string_event, trace, tmsn);
495 tracer_event->string_event.tdsn =
496 MLX5_GET(tracer_string_event, trace, tdsn);
497 } else {
498 tracer_event->type = TRACER_EVENT_TYPE_UNRECOGNIZED;
499 }
500 break;
501 }
502}
503
504static u64 get_block_timestamp(struct mlx5_fw_tracer *tracer, u64 *ts_event)
505{
506 struct tracer_event tracer_event;
507 u8 event_id;
508
509 event_id = MLX5_GET(tracer_event, ts_event, event_id);
510
511 if (event_id == TRACER_EVENT_TYPE_TIMESTAMP)
512 poll_trace(tracer, &tracer_event, ts_event);
513 else
514 tracer_event.timestamp_event.timestamp = 0;
515
516 return tracer_event.timestamp_event.timestamp;
517}
518
519static void mlx5_fw_tracer_clean_print_hash(struct mlx5_fw_tracer *tracer)
520{
521 struct tracer_string_format *str_frmt;
522 struct hlist_node *n;
523 int i;
524
525 for (i = 0; i < MESSAGE_HASH_SIZE; i++) {
526 hlist_for_each_entry_safe(str_frmt, n, &tracer->hash[i], hlist)
527 mlx5_tracer_clean_message(str_frmt);
528 }
529}
530
531static void mlx5_fw_tracer_clean_ready_list(struct mlx5_fw_tracer *tracer)
532{
533 struct tracer_string_format *str_frmt, *tmp_str;
534
535 list_for_each_entry_safe(str_frmt, tmp_str, &tracer->ready_strings_list,
536 list)
537 list_del(&str_frmt->list);
538}
539
540static void mlx5_fw_tracer_save_trace(struct mlx5_fw_tracer *tracer,
541 u64 timestamp, bool lost,
542 u8 event_id, char *msg)
543{
544 struct mlx5_fw_trace_data *trace_data;
545
546 mutex_lock(&tracer->st_arr.lock);
547 trace_data = &tracer->st_arr.straces[tracer->st_arr.saved_traces_index];
548 trace_data->timestamp = timestamp;
549 trace_data->lost = lost;
550 trace_data->event_id = event_id;
551 strscpy_pad(trace_data->msg, msg, TRACE_STR_MSG);
552
553 tracer->st_arr.saved_traces_index =
554 (tracer->st_arr.saved_traces_index + 1) & (SAVED_TRACES_NUM - 1);
555 mutex_unlock(&tracer->st_arr.lock);
556}
557
558static noinline
559void mlx5_tracer_print_trace(struct tracer_string_format *str_frmt,
560 struct mlx5_core_dev *dev,
561 u64 trace_timestamp)
562{
563 char tmp[512];
564
565 snprintf(tmp, sizeof(tmp), str_frmt->string,
566 str_frmt->params[0],
567 str_frmt->params[1],
568 str_frmt->params[2],
569 str_frmt->params[3],
570 str_frmt->params[4],
571 str_frmt->params[5],
572 str_frmt->params[6]);
573
574 trace_mlx5_fw(dev->tracer, trace_timestamp, str_frmt->lost,
575 str_frmt->event_id, tmp);
576
577 mlx5_fw_tracer_save_trace(dev->tracer, trace_timestamp,
578 str_frmt->lost, str_frmt->event_id, tmp);
579
580
581 mlx5_tracer_clean_message(str_frmt);
582}
583
584static int mlx5_tracer_handle_string_trace(struct mlx5_fw_tracer *tracer,
585 struct tracer_event *tracer_event)
586{
587 struct tracer_string_format *cur_string;
588
589 if (tracer_event->string_event.tdsn == 0) {
590 cur_string = mlx5_tracer_get_string(tracer, tracer_event);
591 if (!cur_string)
592 return -1;
593
594 cur_string->num_of_params = mlx5_tracer_get_num_of_params(cur_string->string);
595 cur_string->last_param_num = 0;
596 cur_string->event_id = tracer_event->event_id;
597 cur_string->tmsn = tracer_event->string_event.tmsn;
598 cur_string->timestamp = tracer_event->string_event.timestamp;
599 cur_string->lost = tracer_event->lost_event;
600 if (cur_string->num_of_params == 0)
601 list_add_tail(&cur_string->list, &tracer->ready_strings_list);
602 } else {
603 cur_string = mlx5_tracer_message_get(tracer, tracer_event);
604 if (!cur_string) {
605 pr_debug("%s Got string event for unknown string tdsm: %d\n",
606 __func__, tracer_event->string_event.tmsn);
607 return -1;
608 }
609 cur_string->last_param_num += 1;
610 if (cur_string->last_param_num > TRACER_MAX_PARAMS) {
611 pr_debug("%s Number of params exceeds the max (%d)\n",
612 __func__, TRACER_MAX_PARAMS);
613 list_add_tail(&cur_string->list, &tracer->ready_strings_list);
614 return 0;
615 }
616
617 cur_string->params[cur_string->last_param_num - 1] =
618 tracer_event->string_event.string_param;
619 if (cur_string->last_param_num == cur_string->num_of_params)
620 list_add_tail(&cur_string->list, &tracer->ready_strings_list);
621 }
622
623 return 0;
624}
625
626static void mlx5_tracer_handle_timestamp_trace(struct mlx5_fw_tracer *tracer,
627 struct tracer_event *tracer_event)
628{
629 struct tracer_timestamp_event timestamp_event =
630 tracer_event->timestamp_event;
631 struct tracer_string_format *str_frmt, *tmp_str;
632 struct mlx5_core_dev *dev = tracer->dev;
633 u64 trace_timestamp;
634
635 list_for_each_entry_safe(str_frmt, tmp_str, &tracer->ready_strings_list, list) {
636 list_del(&str_frmt->list);
637 if (str_frmt->timestamp < (timestamp_event.timestamp & MASK_6_0))
638 trace_timestamp = (timestamp_event.timestamp & MASK_52_7) |
639 (str_frmt->timestamp & MASK_6_0);
640 else
641 trace_timestamp = ((timestamp_event.timestamp & MASK_52_7) - 1) |
642 (str_frmt->timestamp & MASK_6_0);
643
644 mlx5_tracer_print_trace(str_frmt, dev, trace_timestamp);
645 }
646}
647
648static int mlx5_tracer_handle_trace(struct mlx5_fw_tracer *tracer,
649 struct tracer_event *tracer_event)
650{
651 if (tracer_event->type == TRACER_EVENT_TYPE_STRING) {
652 mlx5_tracer_handle_string_trace(tracer, tracer_event);
653 } else if (tracer_event->type == TRACER_EVENT_TYPE_TIMESTAMP) {
654 if (!tracer_event->timestamp_event.unreliable)
655 mlx5_tracer_handle_timestamp_trace(tracer, tracer_event);
656 } else {
657 pr_debug("%s Got unrecognised type %d for parsing, exiting..\n",
658 __func__, tracer_event->type);
659 }
660 return 0;
661}
662
663static void mlx5_fw_tracer_handle_traces(struct work_struct *work)
664{
665 struct mlx5_fw_tracer *tracer =
666 container_of(work, struct mlx5_fw_tracer, handle_traces_work);
667 u64 block_timestamp, last_block_timestamp, tmp_trace_block[TRACES_PER_BLOCK];
668 u32 block_count, start_offset, prev_start_offset, prev_consumer_index;
669 u32 trace_event_size = MLX5_ST_SZ_BYTES(tracer_event);
670 struct mlx5_core_dev *dev = tracer->dev;
671 struct tracer_event tracer_event;
672 int i;
673
674 mlx5_core_dbg(dev, "FWTracer: Handle Trace event, owner=(%d)\n", tracer->owner);
675 if (!tracer->owner)
676 return;
677
678 block_count = tracer->buff.size / TRACER_BLOCK_SIZE_BYTE;
679 start_offset = tracer->buff.consumer_index * TRACER_BLOCK_SIZE_BYTE;
680
681
682 memcpy(tmp_trace_block, tracer->buff.log_buf + start_offset,
683 TRACER_BLOCK_SIZE_BYTE);
684
685 block_timestamp =
686 get_block_timestamp(tracer, &tmp_trace_block[TRACES_PER_BLOCK - 1]);
687
688 while (block_timestamp > tracer->last_timestamp) {
689
690 if (!tracer->last_timestamp) {
691 u64 *ts_event;
692
693
694
695
696
697 prev_consumer_index =
698 (tracer->buff.consumer_index - 1) & (block_count - 1);
699 prev_start_offset = prev_consumer_index * TRACER_BLOCK_SIZE_BYTE;
700
701 ts_event = tracer->buff.log_buf + prev_start_offset +
702 (TRACES_PER_BLOCK - 1) * trace_event_size;
703 last_block_timestamp = get_block_timestamp(tracer, ts_event);
704
705
706
707
708
709 if (tracer->last_timestamp != last_block_timestamp) {
710 mlx5_core_warn(dev, "FWTracer: Events were lost\n");
711 tracer->last_timestamp = block_timestamp;
712 tracer->buff.consumer_index =
713 (tracer->buff.consumer_index + 1) & (block_count - 1);
714 break;
715 }
716 }
717
718
719 for (i = 0; i < TRACES_PER_BLOCK ; i++) {
720 poll_trace(tracer, &tracer_event, &tmp_trace_block[i]);
721 mlx5_tracer_handle_trace(tracer, &tracer_event);
722 }
723
724 tracer->buff.consumer_index =
725 (tracer->buff.consumer_index + 1) & (block_count - 1);
726
727 tracer->last_timestamp = block_timestamp;
728 start_offset = tracer->buff.consumer_index * TRACER_BLOCK_SIZE_BYTE;
729 memcpy(tmp_trace_block, tracer->buff.log_buf + start_offset,
730 TRACER_BLOCK_SIZE_BYTE);
731 block_timestamp = get_block_timestamp(tracer,
732 &tmp_trace_block[TRACES_PER_BLOCK - 1]);
733 }
734
735 mlx5_fw_tracer_arm(dev);
736}
737
738static int mlx5_fw_tracer_set_mtrc_conf(struct mlx5_fw_tracer *tracer)
739{
740 struct mlx5_core_dev *dev = tracer->dev;
741 u32 out[MLX5_ST_SZ_DW(mtrc_conf)] = {0};
742 u32 in[MLX5_ST_SZ_DW(mtrc_conf)] = {0};
743 int err;
744
745 MLX5_SET(mtrc_conf, in, trace_mode, TRACE_TO_MEMORY);
746 MLX5_SET(mtrc_conf, in, log_trace_buffer_size,
747 ilog2(TRACER_BUFFER_PAGE_NUM));
748 MLX5_SET(mtrc_conf, in, trace_mkey, tracer->buff.mkey.key);
749
750 err = mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out),
751 MLX5_REG_MTRC_CONF, 0, 1);
752 if (err)
753 mlx5_core_warn(dev, "FWTracer: Failed to set tracer configurations %d\n", err);
754
755 return err;
756}
757
758static int mlx5_fw_tracer_set_mtrc_ctrl(struct mlx5_fw_tracer *tracer, u8 status, u8 arm)
759{
760 struct mlx5_core_dev *dev = tracer->dev;
761 u32 out[MLX5_ST_SZ_DW(mtrc_ctrl)] = {0};
762 u32 in[MLX5_ST_SZ_DW(mtrc_ctrl)] = {0};
763 int err;
764
765 MLX5_SET(mtrc_ctrl, in, modify_field_select, TRACE_STATUS);
766 MLX5_SET(mtrc_ctrl, in, trace_status, status);
767 MLX5_SET(mtrc_ctrl, in, arm_event, arm);
768
769 err = mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out),
770 MLX5_REG_MTRC_CTRL, 0, 1);
771
772 if (!err && status)
773 tracer->last_timestamp = 0;
774
775 return err;
776}
777
778static int mlx5_fw_tracer_start(struct mlx5_fw_tracer *tracer)
779{
780 struct mlx5_core_dev *dev = tracer->dev;
781 int err;
782
783 err = mlx5_fw_tracer_ownership_acquire(tracer);
784 if (err) {
785 mlx5_core_dbg(dev, "FWTracer: Ownership was not granted %d\n", err);
786
787 return 0;
788 }
789
790 err = mlx5_fw_tracer_set_mtrc_conf(tracer);
791 if (err) {
792 mlx5_core_warn(dev, "FWTracer: Failed to set tracer configuration %d\n", err);
793 goto release_ownership;
794 }
795
796
797 err = mlx5_fw_tracer_set_mtrc_ctrl(tracer, 1, 1);
798 if (err) {
799 mlx5_core_warn(dev, "FWTracer: Failed to enable tracer %d\n", err);
800 goto release_ownership;
801 }
802
803 mlx5_core_dbg(dev, "FWTracer: Ownership granted and active\n");
804 return 0;
805
806release_ownership:
807 mlx5_fw_tracer_ownership_release(tracer);
808 return err;
809}
810
811static void mlx5_fw_tracer_ownership_change(struct work_struct *work)
812{
813 struct mlx5_fw_tracer *tracer =
814 container_of(work, struct mlx5_fw_tracer, ownership_change_work);
815
816 mlx5_core_dbg(tracer->dev, "FWTracer: ownership changed, current=(%d)\n", tracer->owner);
817 if (tracer->owner) {
818 tracer->owner = false;
819 tracer->buff.consumer_index = 0;
820 return;
821 }
822
823 mlx5_fw_tracer_start(tracer);
824}
825
826static int mlx5_fw_tracer_set_core_dump_reg(struct mlx5_core_dev *dev,
827 u32 *in, int size_in)
828{
829 u32 out[MLX5_ST_SZ_DW(core_dump_reg)] = {};
830
831 if (!MLX5_CAP_DEBUG(dev, core_dump_general) &&
832 !MLX5_CAP_DEBUG(dev, core_dump_qp))
833 return -EOPNOTSUPP;
834
835 return mlx5_core_access_reg(dev, in, size_in, out, sizeof(out),
836 MLX5_REG_CORE_DUMP, 0, 1);
837}
838
839int mlx5_fw_tracer_trigger_core_dump_general(struct mlx5_core_dev *dev)
840{
841 struct mlx5_fw_tracer *tracer = dev->tracer;
842 u32 in[MLX5_ST_SZ_DW(core_dump_reg)] = {};
843 int err;
844
845 if (!MLX5_CAP_DEBUG(dev, core_dump_general) || !tracer)
846 return -EOPNOTSUPP;
847 if (!tracer->owner)
848 return -EPERM;
849
850 MLX5_SET(core_dump_reg, in, core_dump_type, 0x0);
851
852 err = mlx5_fw_tracer_set_core_dump_reg(dev, in, sizeof(in));
853 if (err)
854 return err;
855 queue_work(tracer->work_queue, &tracer->handle_traces_work);
856 flush_workqueue(tracer->work_queue);
857 return 0;
858}
859
860static int
861mlx5_devlink_fmsg_fill_trace(struct devlink_fmsg *fmsg,
862 struct mlx5_fw_trace_data *trace_data)
863{
864 int err;
865
866 err = devlink_fmsg_obj_nest_start(fmsg);
867 if (err)
868 return err;
869
870 err = devlink_fmsg_u64_pair_put(fmsg, "timestamp", trace_data->timestamp);
871 if (err)
872 return err;
873
874 err = devlink_fmsg_bool_pair_put(fmsg, "lost", trace_data->lost);
875 if (err)
876 return err;
877
878 err = devlink_fmsg_u8_pair_put(fmsg, "event_id", trace_data->event_id);
879 if (err)
880 return err;
881
882 err = devlink_fmsg_string_pair_put(fmsg, "msg", trace_data->msg);
883 if (err)
884 return err;
885
886 err = devlink_fmsg_obj_nest_end(fmsg);
887 if (err)
888 return err;
889 return 0;
890}
891
892int mlx5_fw_tracer_get_saved_traces_objects(struct mlx5_fw_tracer *tracer,
893 struct devlink_fmsg *fmsg)
894{
895 struct mlx5_fw_trace_data *straces = tracer->st_arr.straces;
896 u32 index, start_index, end_index;
897 u32 saved_traces_index;
898 int err;
899
900 if (!straces[0].timestamp)
901 return -ENOMSG;
902
903 mutex_lock(&tracer->st_arr.lock);
904 saved_traces_index = tracer->st_arr.saved_traces_index;
905 if (straces[saved_traces_index].timestamp)
906 start_index = saved_traces_index;
907 else
908 start_index = 0;
909 end_index = (saved_traces_index - 1) & (SAVED_TRACES_NUM - 1);
910
911 err = devlink_fmsg_arr_pair_nest_start(fmsg, "dump fw traces");
912 if (err)
913 goto unlock;
914 index = start_index;
915 while (index != end_index) {
916 err = mlx5_devlink_fmsg_fill_trace(fmsg, &straces[index]);
917 if (err)
918 goto unlock;
919
920 index = (index + 1) & (SAVED_TRACES_NUM - 1);
921 }
922
923 err = devlink_fmsg_arr_pair_nest_end(fmsg);
924unlock:
925 mutex_unlock(&tracer->st_arr.lock);
926 return err;
927}
928
929
930struct mlx5_fw_tracer *mlx5_fw_tracer_create(struct mlx5_core_dev *dev)
931{
932 struct mlx5_fw_tracer *tracer = NULL;
933 int err;
934
935 if (!MLX5_CAP_MCAM_REG(dev, tracer_registers)) {
936 mlx5_core_dbg(dev, "FWTracer: Tracer capability not present\n");
937 return NULL;
938 }
939
940 tracer = kvzalloc(sizeof(*tracer), GFP_KERNEL);
941 if (!tracer)
942 return ERR_PTR(-ENOMEM);
943
944 tracer->work_queue = create_singlethread_workqueue("mlx5_fw_tracer");
945 if (!tracer->work_queue) {
946 err = -ENOMEM;
947 goto free_tracer;
948 }
949
950 tracer->dev = dev;
951
952 INIT_LIST_HEAD(&tracer->ready_strings_list);
953 INIT_WORK(&tracer->ownership_change_work, mlx5_fw_tracer_ownership_change);
954 INIT_WORK(&tracer->read_fw_strings_work, mlx5_tracer_read_strings_db);
955 INIT_WORK(&tracer->handle_traces_work, mlx5_fw_tracer_handle_traces);
956
957
958 err = mlx5_query_mtrc_caps(tracer);
959 if (err) {
960 mlx5_core_dbg(dev, "FWTracer: Failed to query capabilities %d\n", err);
961 goto destroy_workqueue;
962 }
963
964 err = mlx5_fw_tracer_create_log_buf(tracer);
965 if (err) {
966 mlx5_core_warn(dev, "FWTracer: Create log buffer failed %d\n", err);
967 goto destroy_workqueue;
968 }
969
970 err = mlx5_fw_tracer_allocate_strings_db(tracer);
971 if (err) {
972 mlx5_core_warn(dev, "FWTracer: Allocate strings database failed %d\n", err);
973 goto free_log_buf;
974 }
975
976 mlx5_fw_tracer_init_saved_traces_array(tracer);
977 mlx5_core_dbg(dev, "FWTracer: Tracer created\n");
978
979 return tracer;
980
981free_log_buf:
982 mlx5_fw_tracer_destroy_log_buf(tracer);
983destroy_workqueue:
984 tracer->dev = NULL;
985 destroy_workqueue(tracer->work_queue);
986free_tracer:
987 kvfree(tracer);
988 return ERR_PTR(err);
989}
990
991static int fw_tracer_event(struct notifier_block *nb, unsigned long action, void *data);
992
993
994int mlx5_fw_tracer_init(struct mlx5_fw_tracer *tracer)
995{
996 struct mlx5_core_dev *dev;
997 int err;
998
999 if (IS_ERR_OR_NULL(tracer))
1000 return 0;
1001
1002 dev = tracer->dev;
1003
1004 if (!tracer->str_db.loaded)
1005 queue_work(tracer->work_queue, &tracer->read_fw_strings_work);
1006
1007 err = mlx5_core_alloc_pd(dev, &tracer->buff.pdn);
1008 if (err) {
1009 mlx5_core_warn(dev, "FWTracer: Failed to allocate PD %d\n", err);
1010 return err;
1011 }
1012
1013 err = mlx5_fw_tracer_create_mkey(tracer);
1014 if (err) {
1015 mlx5_core_warn(dev, "FWTracer: Failed to create mkey %d\n", err);
1016 goto err_dealloc_pd;
1017 }
1018
1019 MLX5_NB_INIT(&tracer->nb, fw_tracer_event, DEVICE_TRACER);
1020 mlx5_eq_notifier_register(dev, &tracer->nb);
1021
1022 mlx5_fw_tracer_start(tracer);
1023
1024 return 0;
1025
1026err_dealloc_pd:
1027 mlx5_core_dealloc_pd(dev, tracer->buff.pdn);
1028 return err;
1029}
1030
1031
1032void mlx5_fw_tracer_cleanup(struct mlx5_fw_tracer *tracer)
1033{
1034 if (IS_ERR_OR_NULL(tracer))
1035 return;
1036
1037 mlx5_core_dbg(tracer->dev, "FWTracer: Cleanup, is owner ? (%d)\n",
1038 tracer->owner);
1039 mlx5_eq_notifier_unregister(tracer->dev, &tracer->nb);
1040 cancel_work_sync(&tracer->ownership_change_work);
1041 cancel_work_sync(&tracer->handle_traces_work);
1042
1043 if (tracer->owner)
1044 mlx5_fw_tracer_ownership_release(tracer);
1045
1046 mlx5_core_destroy_mkey(tracer->dev, &tracer->buff.mkey);
1047 mlx5_core_dealloc_pd(tracer->dev, tracer->buff.pdn);
1048}
1049
1050
1051void mlx5_fw_tracer_destroy(struct mlx5_fw_tracer *tracer)
1052{
1053 if (IS_ERR_OR_NULL(tracer))
1054 return;
1055
1056 mlx5_core_dbg(tracer->dev, "FWTracer: Destroy\n");
1057
1058 cancel_work_sync(&tracer->read_fw_strings_work);
1059 mlx5_fw_tracer_clean_ready_list(tracer);
1060 mlx5_fw_tracer_clean_print_hash(tracer);
1061 mlx5_fw_tracer_clean_saved_traces_array(tracer);
1062 mlx5_fw_tracer_free_strings_db(tracer);
1063 mlx5_fw_tracer_destroy_log_buf(tracer);
1064 flush_workqueue(tracer->work_queue);
1065 destroy_workqueue(tracer->work_queue);
1066 kvfree(tracer);
1067}
1068
1069static int mlx5_fw_tracer_recreate_strings_db(struct mlx5_fw_tracer *tracer)
1070{
1071 struct mlx5_core_dev *dev;
1072 int err;
1073
1074 cancel_work_sync(&tracer->read_fw_strings_work);
1075 mlx5_fw_tracer_clean_ready_list(tracer);
1076 mlx5_fw_tracer_clean_print_hash(tracer);
1077 mlx5_fw_tracer_clean_saved_traces_array(tracer);
1078 mlx5_fw_tracer_free_strings_db(tracer);
1079
1080 dev = tracer->dev;
1081 err = mlx5_query_mtrc_caps(tracer);
1082 if (err) {
1083 mlx5_core_dbg(dev, "FWTracer: Failed to query capabilities %d\n", err);
1084 return err;
1085 }
1086
1087 err = mlx5_fw_tracer_allocate_strings_db(tracer);
1088 if (err) {
1089 mlx5_core_warn(dev, "FWTracer: Allocate strings DB failed %d\n", err);
1090 return err;
1091 }
1092 mlx5_fw_tracer_init_saved_traces_array(tracer);
1093
1094 return 0;
1095}
1096
1097int mlx5_fw_tracer_reload(struct mlx5_fw_tracer *tracer)
1098{
1099 struct mlx5_core_dev *dev;
1100 int err;
1101
1102 if (IS_ERR_OR_NULL(tracer))
1103 return -EINVAL;
1104
1105 dev = tracer->dev;
1106 mlx5_fw_tracer_cleanup(tracer);
1107 err = mlx5_fw_tracer_recreate_strings_db(tracer);
1108 if (err) {
1109 mlx5_core_warn(dev, "Failed to recreate FW tracer strings DB\n");
1110 return err;
1111 }
1112 err = mlx5_fw_tracer_init(tracer);
1113 if (err) {
1114 mlx5_core_warn(dev, "Failed to re-initialize FW tracer\n");
1115 return err;
1116 }
1117
1118 return 0;
1119}
1120
1121static int fw_tracer_event(struct notifier_block *nb, unsigned long action, void *data)
1122{
1123 struct mlx5_fw_tracer *tracer = mlx5_nb_cof(nb, struct mlx5_fw_tracer, nb);
1124 struct mlx5_core_dev *dev = tracer->dev;
1125 struct mlx5_eqe *eqe = data;
1126
1127 switch (eqe->sub_type) {
1128 case MLX5_TRACER_SUBTYPE_OWNERSHIP_CHANGE:
1129 if (test_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state))
1130 queue_work(tracer->work_queue, &tracer->ownership_change_work);
1131 break;
1132 case MLX5_TRACER_SUBTYPE_TRACES_AVAILABLE:
1133 if (likely(tracer->str_db.loaded))
1134 queue_work(tracer->work_queue, &tracer->handle_traces_work);
1135 break;
1136 default:
1137 mlx5_core_dbg(dev, "FWTracer: Event with unrecognized subtype: sub_type %d\n",
1138 eqe->sub_type);
1139 }
1140
1141 return NOTIFY_OK;
1142}
1143
1144EXPORT_TRACEPOINT_SYMBOL(mlx5_fw);
1145
