1
2
3
4
5
6
7
8#include <linux/module.h>
9#include <linux/mutex.h>
10#include <linux/ftrace.h>
11#include <linux/sched/clock.h>
12#include <linux/sched/mm.h>
13
14#include "trace_output.h"
15
16
17#define EVENT_HASHSIZE 128
18
19DECLARE_RWSEM(trace_event_sem);
20
21static struct hlist_head event_hash[EVENT_HASHSIZE] __read_mostly;
22
23static int next_event_type = __TRACE_LAST_TYPE;
24
25enum print_line_t trace_print_bputs_msg_only(struct trace_iterator *iter)
26{
27 struct trace_seq *s = &iter->seq;
28 struct trace_entry *entry = iter->ent;
29 struct bputs_entry *field;
30
31 trace_assign_type(field, entry);
32
33 trace_seq_puts(s, field->str);
34
35 return trace_handle_return(s);
36}
37
38enum print_line_t trace_print_bprintk_msg_only(struct trace_iterator *iter)
39{
40 struct trace_seq *s = &iter->seq;
41 struct trace_entry *entry = iter->ent;
42 struct bprint_entry *field;
43
44 trace_assign_type(field, entry);
45
46 trace_seq_bprintf(s, field->fmt, field->buf);
47
48 return trace_handle_return(s);
49}
50
51enum print_line_t trace_print_printk_msg_only(struct trace_iterator *iter)
52{
53 struct trace_seq *s = &iter->seq;
54 struct trace_entry *entry = iter->ent;
55 struct print_entry *field;
56
57 trace_assign_type(field, entry);
58
59 trace_seq_puts(s, field->buf);
60
61 return trace_handle_return(s);
62}
63
64const char *
65trace_print_flags_seq(struct trace_seq *p, const char *delim,
66 unsigned long flags,
67 const struct trace_print_flags *flag_array)
68{
69 unsigned long mask;
70 const char *str;
71 const char *ret = trace_seq_buffer_ptr(p);
72 int i, first = 1;
73
74 for (i = 0; flag_array[i].name && flags; i++) {
75
76 mask = flag_array[i].mask;
77 if ((flags & mask) != mask)
78 continue;
79
80 str = flag_array[i].name;
81 flags &= ~mask;
82 if (!first && delim)
83 trace_seq_puts(p, delim);
84 else
85 first = 0;
86 trace_seq_puts(p, str);
87 }
88
89
90 if (flags) {
91 if (!first && delim)
92 trace_seq_puts(p, delim);
93 trace_seq_printf(p, "0x%lx", flags);
94 }
95
96 trace_seq_putc(p, 0);
97
98 return ret;
99}
100EXPORT_SYMBOL(trace_print_flags_seq);
101
102const char *
103trace_print_symbols_seq(struct trace_seq *p, unsigned long val,
104 const struct trace_print_flags *symbol_array)
105{
106 int i;
107 const char *ret = trace_seq_buffer_ptr(p);
108
109 for (i = 0; symbol_array[i].name; i++) {
110
111 if (val != symbol_array[i].mask)
112 continue;
113
114 trace_seq_puts(p, symbol_array[i].name);
115 break;
116 }
117
118 if (ret == (const char *)(trace_seq_buffer_ptr(p)))
119 trace_seq_printf(p, "0x%lx", val);
120
121 trace_seq_putc(p, 0);
122
123 return ret;
124}
125EXPORT_SYMBOL(trace_print_symbols_seq);
126
127#if BITS_PER_LONG == 32
128const char *
129trace_print_flags_seq_u64(struct trace_seq *p, const char *delim,
130 unsigned long long flags,
131 const struct trace_print_flags_u64 *flag_array)
132{
133 unsigned long long mask;
134 const char *str;
135 const char *ret = trace_seq_buffer_ptr(p);
136 int i, first = 1;
137
138 for (i = 0; flag_array[i].name && flags; i++) {
139
140 mask = flag_array[i].mask;
141 if ((flags & mask) != mask)
142 continue;
143
144 str = flag_array[i].name;
145 flags &= ~mask;
146 if (!first && delim)
147 trace_seq_puts(p, delim);
148 else
149 first = 0;
150 trace_seq_puts(p, str);
151 }
152
153
154 if (flags) {
155 if (!first && delim)
156 trace_seq_puts(p, delim);
157 trace_seq_printf(p, "0x%llx", flags);
158 }
159
160 trace_seq_putc(p, 0);
161
162 return ret;
163}
164EXPORT_SYMBOL(trace_print_flags_seq_u64);
165
166const char *
167trace_print_symbols_seq_u64(struct trace_seq *p, unsigned long long val,
168 const struct trace_print_flags_u64 *symbol_array)
169{
170 int i;
171 const char *ret = trace_seq_buffer_ptr(p);
172
173 for (i = 0; symbol_array[i].name; i++) {
174
175 if (val != symbol_array[i].mask)
176 continue;
177
178 trace_seq_puts(p, symbol_array[i].name);
179 break;
180 }
181
182 if (ret == (const char *)(trace_seq_buffer_ptr(p)))
183 trace_seq_printf(p, "0x%llx", val);
184
185 trace_seq_putc(p, 0);
186
187 return ret;
188}
189EXPORT_SYMBOL(trace_print_symbols_seq_u64);
190#endif
191
192const char *
193trace_print_bitmask_seq(struct trace_seq *p, void *bitmask_ptr,
194 unsigned int bitmask_size)
195{
196 const char *ret = trace_seq_buffer_ptr(p);
197
198 trace_seq_bitmask(p, bitmask_ptr, bitmask_size * 8);
199 trace_seq_putc(p, 0);
200
201 return ret;
202}
203EXPORT_SYMBOL_GPL(trace_print_bitmask_seq);
204
205
206
207
208
209
210
211
212
213
214
215
216const char *
217trace_print_hex_seq(struct trace_seq *p, const unsigned char *buf, int buf_len,
218 bool concatenate)
219{
220 int i;
221 const char *ret = trace_seq_buffer_ptr(p);
222 const char *fmt = concatenate ? "%*phN" : "%*ph";
223
224 for (i = 0; i < buf_len; i += 16)
225 trace_seq_printf(p, fmt, min(buf_len - i, 16), &buf[i]);
226 trace_seq_putc(p, 0);
227
228 return ret;
229}
230EXPORT_SYMBOL(trace_print_hex_seq);
231
232const char *
233trace_print_array_seq(struct trace_seq *p, const void *buf, int count,
234 size_t el_size)
235{
236 const char *ret = trace_seq_buffer_ptr(p);
237 const char *prefix = "";
238 void *ptr = (void *)buf;
239 size_t buf_len = count * el_size;
240
241 trace_seq_putc(p, '{');
242
243 while (ptr < buf + buf_len) {
244 switch (el_size) {
245 case 1:
246 trace_seq_printf(p, "%s0x%x", prefix,
247 *(u8 *)ptr);
248 break;
249 case 2:
250 trace_seq_printf(p, "%s0x%x", prefix,
251 *(u16 *)ptr);
252 break;
253 case 4:
254 trace_seq_printf(p, "%s0x%x", prefix,
255 *(u32 *)ptr);
256 break;
257 case 8:
258 trace_seq_printf(p, "%s0x%llx", prefix,
259 *(u64 *)ptr);
260 break;
261 default:
262 trace_seq_printf(p, "BAD SIZE:%zu 0x%x", el_size,
263 *(u8 *)ptr);
264 el_size = 1;
265 }
266 prefix = ",";
267 ptr += el_size;
268 }
269
270 trace_seq_putc(p, '}');
271 trace_seq_putc(p, 0);
272
273 return ret;
274}
275EXPORT_SYMBOL(trace_print_array_seq);
276
277const char *
278trace_print_hex_dump_seq(struct trace_seq *p, const char *prefix_str,
279 int prefix_type, int rowsize, int groupsize,
280 const void *buf, size_t len, bool ascii)
281{
282 const char *ret = trace_seq_buffer_ptr(p);
283
284 trace_seq_putc(p, '\n');
285 trace_seq_hex_dump(p, prefix_str, prefix_type,
286 rowsize, groupsize, buf, len, ascii);
287 trace_seq_putc(p, 0);
288 return ret;
289}
290EXPORT_SYMBOL(trace_print_hex_dump_seq);
291
292int trace_raw_output_prep(struct trace_iterator *iter,
293 struct trace_event *trace_event)
294{
295 struct trace_event_call *event;
296 struct trace_seq *s = &iter->seq;
297 struct trace_seq *p = &iter->tmp_seq;
298 struct trace_entry *entry;
299
300 event = container_of(trace_event, struct trace_event_call, event);
301 entry = iter->ent;
302
303 if (entry->type != event->event.type) {
304 WARN_ON_ONCE(1);
305 return TRACE_TYPE_UNHANDLED;
306 }
307
308 trace_seq_init(p);
309 trace_seq_printf(s, "%s: ", trace_event_name(event));
310
311 return trace_handle_return(s);
312}
313EXPORT_SYMBOL(trace_raw_output_prep);
314
315void trace_event_printf(struct trace_iterator *iter, const char *fmt, ...)
316{
317 va_list ap;
318
319 va_start(ap, fmt);
320 trace_check_vprintf(iter, trace_event_format(iter, fmt), ap);
321 va_end(ap);
322}
323EXPORT_SYMBOL(trace_event_printf);
324
325static int trace_output_raw(struct trace_iterator *iter, char *name,
326 char *fmt, va_list ap)
327{
328 struct trace_seq *s = &iter->seq;
329
330 trace_seq_printf(s, "%s: ", name);
331 trace_seq_vprintf(s, trace_event_format(iter, fmt), ap);
332
333 return trace_handle_return(s);
334}
335
336int trace_output_call(struct trace_iterator *iter, char *name, char *fmt, ...)
337{
338 va_list ap;
339 int ret;
340
341 va_start(ap, fmt);
342 ret = trace_output_raw(iter, name, fmt, ap);
343 va_end(ap);
344
345 return ret;
346}
347EXPORT_SYMBOL_GPL(trace_output_call);
348
349#ifdef CONFIG_KRETPROBES
350static inline const char *kretprobed(const char *name)
351{
352 static const char tramp_name[] = "kretprobe_trampoline";
353 int size = sizeof(tramp_name);
354
355 if (strncmp(tramp_name, name, size) == 0)
356 return "[unknown/kretprobe'd]";
357 return name;
358}
359#else
360static inline const char *kretprobed(const char *name)
361{
362 return name;
363}
364#endif
365
366void
367trace_seq_print_sym(struct trace_seq *s, unsigned long address, bool offset)
368{
369#ifdef CONFIG_KALLSYMS
370 char str[KSYM_SYMBOL_LEN];
371 const char *name;
372
373 if (offset)
374 sprint_symbol(str, address);
375 else
376 kallsyms_lookup(address, NULL, NULL, NULL, str);
377 name = kretprobed(str);
378
379 if (name && strlen(name)) {
380 trace_seq_puts(s, name);
381 return;
382 }
383#endif
384 trace_seq_printf(s, "0x%08lx", address);
385}
386
387#ifndef CONFIG_64BIT
388# define IP_FMT "%08lx"
389#else
390# define IP_FMT "%016lx"
391#endif
392
393static int seq_print_user_ip(struct trace_seq *s, struct mm_struct *mm,
394 unsigned long ip, unsigned long sym_flags)
395{
396 struct file *file = NULL;
397 unsigned long vmstart = 0;
398 int ret = 1;
399
400 if (s->full)
401 return 0;
402
403 if (mm) {
404 const struct vm_area_struct *vma;
405
406 mmap_read_lock(mm);
407 vma = find_vma(mm, ip);
408 if (vma) {
409 file = vma->vm_file;
410 vmstart = vma->vm_start;
411 }
412 if (file) {
413 ret = trace_seq_path(s, &file->f_path);
414 if (ret)
415 trace_seq_printf(s, "[+0x%lx]",
416 ip - vmstart);
417 }
418 mmap_read_unlock(mm);
419 }
420 if (ret && ((sym_flags & TRACE_ITER_SYM_ADDR) || !file))
421 trace_seq_printf(s, " <" IP_FMT ">", ip);
422 return !trace_seq_has_overflowed(s);
423}
424
425int
426seq_print_ip_sym(struct trace_seq *s, unsigned long ip, unsigned long sym_flags)
427{
428 if (!ip) {
429 trace_seq_putc(s, '0');
430 goto out;
431 }
432
433 trace_seq_print_sym(s, ip, sym_flags & TRACE_ITER_SYM_OFFSET);
434
435 if (sym_flags & TRACE_ITER_SYM_ADDR)
436 trace_seq_printf(s, " <" IP_FMT ">", ip);
437
438 out:
439 return !trace_seq_has_overflowed(s);
440}
441
442
443
444
445
446
447
448
449
450int trace_print_lat_fmt(struct trace_seq *s, struct trace_entry *entry)
451{
452 char hardsoft_irq;
453 char need_resched;
454 char irqs_off;
455 int hardirq;
456 int softirq;
457 int nmi;
458
459 nmi = entry->flags & TRACE_FLAG_NMI;
460 hardirq = entry->flags & TRACE_FLAG_HARDIRQ;
461 softirq = entry->flags & TRACE_FLAG_SOFTIRQ;
462
463 irqs_off =
464 (entry->flags & TRACE_FLAG_IRQS_OFF) ? 'd' :
465 (entry->flags & TRACE_FLAG_IRQS_NOSUPPORT) ? 'X' :
466 '.';
467
468 switch (entry->flags & (TRACE_FLAG_NEED_RESCHED |
469 TRACE_FLAG_PREEMPT_RESCHED)) {
470 case TRACE_FLAG_NEED_RESCHED | TRACE_FLAG_PREEMPT_RESCHED:
471 need_resched = 'N';
472 break;
473 case TRACE_FLAG_NEED_RESCHED:
474 need_resched = 'n';
475 break;
476 case TRACE_FLAG_PREEMPT_RESCHED:
477 need_resched = 'p';
478 break;
479 default:
480 need_resched = '.';
481 break;
482 }
483
484 hardsoft_irq =
485 (nmi && hardirq) ? 'Z' :
486 nmi ? 'z' :
487 (hardirq && softirq) ? 'H' :
488 hardirq ? 'h' :
489 softirq ? 's' :
490 '.' ;
491
492 trace_seq_printf(s, "%c%c%c",
493 irqs_off, need_resched, hardsoft_irq);
494
495 if (entry->preempt_count & 0xf)
496 trace_seq_printf(s, "%x", entry->preempt_count & 0xf);
497 else
498 trace_seq_putc(s, '.');
499
500 if (entry->preempt_count & 0xf0)
501 trace_seq_printf(s, "%x", entry->preempt_count >> 4);
502 else
503 trace_seq_putc(s, '.');
504
505 return !trace_seq_has_overflowed(s);
506}
507
508static int
509lat_print_generic(struct trace_seq *s, struct trace_entry *entry, int cpu)
510{
511 char comm[TASK_COMM_LEN];
512
513 trace_find_cmdline(entry->pid, comm);
514
515 trace_seq_printf(s, "%8.8s-%-7d %3d",
516 comm, entry->pid, cpu);
517
518 return trace_print_lat_fmt(s, entry);
519}
520
521#undef MARK
522#define MARK(v, s) {.val = v, .sym = s}
523
524static const struct trace_mark {
525 unsigned long long val;
526 char sym;
527} mark[] = {
528 MARK(1000000000ULL , '$'),
529 MARK(100000000ULL , '@'),
530 MARK(10000000ULL , '*'),
531 MARK(1000000ULL , '#'),
532 MARK(100000ULL , '!'),
533 MARK(10000ULL , '+'),
534};
535#undef MARK
536
537char trace_find_mark(unsigned long long d)
538{
539 int i;
540 int size = ARRAY_SIZE(mark);
541
542 for (i = 0; i < size; i++) {
543 if (d > mark[i].val)
544 break;
545 }
546
547 return (i == size) ? ' ' : mark[i].sym;
548}
549
550static int
551lat_print_timestamp(struct trace_iterator *iter, u64 next_ts)
552{
553 struct trace_array *tr = iter->tr;
554 unsigned long verbose = tr->trace_flags & TRACE_ITER_VERBOSE;
555 unsigned long in_ns = iter->iter_flags & TRACE_FILE_TIME_IN_NS;
556 unsigned long long abs_ts = iter->ts - iter->array_buffer->time_start;
557 unsigned long long rel_ts = next_ts - iter->ts;
558 struct trace_seq *s = &iter->seq;
559
560 if (in_ns) {
561 abs_ts = ns2usecs(abs_ts);
562 rel_ts = ns2usecs(rel_ts);
563 }
564
565 if (verbose && in_ns) {
566 unsigned long abs_usec = do_div(abs_ts, USEC_PER_MSEC);
567 unsigned long abs_msec = (unsigned long)abs_ts;
568 unsigned long rel_usec = do_div(rel_ts, USEC_PER_MSEC);
569 unsigned long rel_msec = (unsigned long)rel_ts;
570
571 trace_seq_printf(
572 s, "[%08llx] %ld.%03ldms (+%ld.%03ldms): ",
573 ns2usecs(iter->ts),
574 abs_msec, abs_usec,
575 rel_msec, rel_usec);
576
577 } else if (verbose && !in_ns) {
578 trace_seq_printf(
579 s, "[%016llx] %lld (+%lld): ",
580 iter->ts, abs_ts, rel_ts);
581
582 } else if (!verbose && in_ns) {
583 trace_seq_printf(
584 s, " %4lldus%c: ",
585 abs_ts,
586 trace_find_mark(rel_ts * NSEC_PER_USEC));
587
588 } else {
589 trace_seq_printf(s, " %4lld: ", abs_ts);
590 }
591
592 return !trace_seq_has_overflowed(s);
593}
594
595static void trace_print_time(struct trace_seq *s, struct trace_iterator *iter,
596 unsigned long long ts)
597{
598 unsigned long secs, usec_rem;
599 unsigned long long t;
600
601 if (iter->iter_flags & TRACE_FILE_TIME_IN_NS) {
602 t = ns2usecs(ts);
603 usec_rem = do_div(t, USEC_PER_SEC);
604 secs = (unsigned long)t;
605 trace_seq_printf(s, " %5lu.%06lu", secs, usec_rem);
606 } else
607 trace_seq_printf(s, " %12llu", ts);
608}
609
610int trace_print_context(struct trace_iterator *iter)
611{
612 struct trace_array *tr = iter->tr;
613 struct trace_seq *s = &iter->seq;
614 struct trace_entry *entry = iter->ent;
615 char comm[TASK_COMM_LEN];
616
617 trace_find_cmdline(entry->pid, comm);
618
619 trace_seq_printf(s, "%16s-%-7d ", comm, entry->pid);
620
621 if (tr->trace_flags & TRACE_ITER_RECORD_TGID) {
622 unsigned int tgid = trace_find_tgid(entry->pid);
623
624 if (!tgid)
625 trace_seq_printf(s, "(-------) ");
626 else
627 trace_seq_printf(s, "(%7d) ", tgid);
628 }
629
630 trace_seq_printf(s, "[%03d] ", iter->cpu);
631
632 if (tr->trace_flags & TRACE_ITER_IRQ_INFO)
633 trace_print_lat_fmt(s, entry);
634
635 trace_print_time(s, iter, iter->ts);
636 trace_seq_puts(s, ": ");
637
638 return !trace_seq_has_overflowed(s);
639}
640
641int trace_print_lat_context(struct trace_iterator *iter)
642{
643 struct trace_entry *entry, *next_entry;
644 struct trace_array *tr = iter->tr;
645 struct trace_seq *s = &iter->seq;
646 unsigned long verbose = (tr->trace_flags & TRACE_ITER_VERBOSE);
647 u64 next_ts;
648
649 next_entry = trace_find_next_entry(iter, NULL, &next_ts);
650 if (!next_entry)
651 next_ts = iter->ts;
652
653
654 entry = iter->ent;
655
656 if (verbose) {
657 char comm[TASK_COMM_LEN];
658
659 trace_find_cmdline(entry->pid, comm);
660
661 trace_seq_printf(
662 s, "%16s %7d %3d %d %08x %08lx ",
663 comm, entry->pid, iter->cpu, entry->flags,
664 entry->preempt_count & 0xf, iter->idx);
665 } else {
666 lat_print_generic(s, entry, iter->cpu);
667 }
668
669 lat_print_timestamp(iter, next_ts);
670
671 return !trace_seq_has_overflowed(s);
672}
673
674
675
676
677
678
679
680
681struct trace_event *ftrace_find_event(int type)
682{
683 struct trace_event *event;
684 unsigned key;
685
686 key = type & (EVENT_HASHSIZE - 1);
687
688 hlist_for_each_entry(event, &event_hash[key], node) {
689 if (event->type == type)
690 return event;
691 }
692
693 return NULL;
694}
695
696static LIST_HEAD(ftrace_event_list);
697
698static int trace_search_list(struct list_head **list)
699{
700 struct trace_event *e;
701 int next = __TRACE_LAST_TYPE;
702
703 if (list_empty(&ftrace_event_list)) {
704 *list = &ftrace_event_list;
705 return next;
706 }
707
708
709
710
711
712 list_for_each_entry(e, &ftrace_event_list, list) {
713 if (e->type != next)
714 break;
715 next++;
716 }
717
718
719 if (next > TRACE_EVENT_TYPE_MAX)
720 return 0;
721
722 *list = &e->list;
723 return next;
724}
725
726void trace_event_read_lock(void)
727{
728 down_read(&trace_event_sem);
729}
730
731void trace_event_read_unlock(void)
732{
733 up_read(&trace_event_sem);
734}
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751int register_trace_event(struct trace_event *event)
752{
753 unsigned key;
754 int ret = 0;
755
756 down_write(&trace_event_sem);
757
758 if (WARN_ON(!event))
759 goto out;
760
761 if (WARN_ON(!event->funcs))
762 goto out;
763
764 INIT_LIST_HEAD(&event->list);
765
766 if (!event->type) {
767 struct list_head *list = NULL;
768
769 if (next_event_type > TRACE_EVENT_TYPE_MAX) {
770
771 event->type = trace_search_list(&list);
772 if (!event->type)
773 goto out;
774
775 } else {
776
777 event->type = next_event_type++;
778 list = &ftrace_event_list;
779 }
780
781 if (WARN_ON(ftrace_find_event(event->type)))
782 goto out;
783
784 list_add_tail(&event->list, list);
785
786 } else if (event->type > __TRACE_LAST_TYPE) {
787 printk(KERN_WARNING "Need to add type to trace.h\n");
788 WARN_ON(1);
789 goto out;
790 } else {
791
792 if (ftrace_find_event(event->type))
793 goto out;
794 }
795
796 if (event->funcs->trace == NULL)
797 event->funcs->trace = trace_nop_print;
798 if (event->funcs->raw == NULL)
799 event->funcs->raw = trace_nop_print;
800 if (event->funcs->hex == NULL)
801 event->funcs->hex = trace_nop_print;
802 if (event->funcs->binary == NULL)
803 event->funcs->binary = trace_nop_print;
804
805 key = event->type & (EVENT_HASHSIZE - 1);
806
807 hlist_add_head(&event->node, &event_hash[key]);
808
809 ret = event->type;
810 out:
811 up_write(&trace_event_sem);
812
813 return ret;
814}
815EXPORT_SYMBOL_GPL(register_trace_event);
816
817
818
819
820int __unregister_trace_event(struct trace_event *event)
821{
822 hlist_del(&event->node);
823 list_del(&event->list);
824 return 0;
825}
826
827
828
829
830
831int unregister_trace_event(struct trace_event *event)
832{
833 down_write(&trace_event_sem);
834 __unregister_trace_event(event);
835 up_write(&trace_event_sem);
836
837 return 0;
838}
839EXPORT_SYMBOL_GPL(unregister_trace_event);
840
841
842
843
844
845enum print_line_t trace_nop_print(struct trace_iterator *iter, int flags,
846 struct trace_event *event)
847{
848 trace_seq_printf(&iter->seq, "type: %d\n", iter->ent->type);
849
850 return trace_handle_return(&iter->seq);
851}
852
853static void print_fn_trace(struct trace_seq *s, unsigned long ip,
854 unsigned long parent_ip, int flags)
855{
856 seq_print_ip_sym(s, ip, flags);
857
858 if ((flags & TRACE_ITER_PRINT_PARENT) && parent_ip) {
859 trace_seq_puts(s, " <-");
860 seq_print_ip_sym(s, parent_ip, flags);
861 }
862}
863
864
865static enum print_line_t trace_fn_trace(struct trace_iterator *iter, int flags,
866 struct trace_event *event)
867{
868 struct ftrace_entry *field;
869 struct trace_seq *s = &iter->seq;
870
871 trace_assign_type(field, iter->ent);
872
873 print_fn_trace(s, field->ip, field->parent_ip, flags);
874 trace_seq_putc(s, '\n');
875
876 return trace_handle_return(s);
877}
878
879static enum print_line_t trace_fn_raw(struct trace_iterator *iter, int flags,
880 struct trace_event *event)
881{
882 struct ftrace_entry *field;
883
884 trace_assign_type(field, iter->ent);
885
886 trace_seq_printf(&iter->seq, "%lx %lx\n",
887 field->ip,
888 field->parent_ip);
889
890 return trace_handle_return(&iter->seq);
891}
892
893static enum print_line_t trace_fn_hex(struct trace_iterator *iter, int flags,
894 struct trace_event *event)
895{
896 struct ftrace_entry *field;
897 struct trace_seq *s = &iter->seq;
898
899 trace_assign_type(field, iter->ent);
900
901 SEQ_PUT_HEX_FIELD(s, field->ip);
902 SEQ_PUT_HEX_FIELD(s, field->parent_ip);
903
904 return trace_handle_return(s);
905}
906
907static enum print_line_t trace_fn_bin(struct trace_iterator *iter, int flags,
908 struct trace_event *event)
909{
910 struct ftrace_entry *field;
911 struct trace_seq *s = &iter->seq;
912
913 trace_assign_type(field, iter->ent);
914
915 SEQ_PUT_FIELD(s, field->ip);
916 SEQ_PUT_FIELD(s, field->parent_ip);
917
918 return trace_handle_return(s);
919}
920
921static struct trace_event_functions trace_fn_funcs = {
922 .trace = trace_fn_trace,
923 .raw = trace_fn_raw,
924 .hex = trace_fn_hex,
925 .binary = trace_fn_bin,
926};
927
928static struct trace_event trace_fn_event = {
929 .type = TRACE_FN,
930 .funcs = &trace_fn_funcs,
931};
932
933
934static enum print_line_t trace_ctxwake_print(struct trace_iterator *iter,
935 char *delim)
936{
937 struct ctx_switch_entry *field;
938 char comm[TASK_COMM_LEN];
939 int S, T;
940
941
942 trace_assign_type(field, iter->ent);
943
944 T = task_index_to_char(field->next_state);
945 S = task_index_to_char(field->prev_state);
946 trace_find_cmdline(field->next_pid, comm);
947 trace_seq_printf(&iter->seq,
948 " %7d:%3d:%c %s [%03d] %7d:%3d:%c %s\n",
949 field->prev_pid,
950 field->prev_prio,
951 S, delim,
952 field->next_cpu,
953 field->next_pid,
954 field->next_prio,
955 T, comm);
956
957 return trace_handle_return(&iter->seq);
958}
959
960static enum print_line_t trace_ctx_print(struct trace_iterator *iter, int flags,
961 struct trace_event *event)
962{
963 return trace_ctxwake_print(iter, "==>");
964}
965
966static enum print_line_t trace_wake_print(struct trace_iterator *iter,
967 int flags, struct trace_event *event)
968{
969 return trace_ctxwake_print(iter, " +");
970}
971
972static int trace_ctxwake_raw(struct trace_iterator *iter, char S)
973{
974 struct ctx_switch_entry *field;
975 int T;
976
977 trace_assign_type(field, iter->ent);
978
979 if (!S)
980 S = task_index_to_char(field->prev_state);
981 T = task_index_to_char(field->next_state);
982 trace_seq_printf(&iter->seq, "%d %d %c %d %d %d %c\n",
983 field->prev_pid,
984 field->prev_prio,
985 S,
986 field->next_cpu,
987 field->next_pid,
988 field->next_prio,
989 T);
990
991 return trace_handle_return(&iter->seq);
992}
993
994static enum print_line_t trace_ctx_raw(struct trace_iterator *iter, int flags,
995 struct trace_event *event)
996{
997 return trace_ctxwake_raw(iter, 0);
998}
999
1000static enum print_line_t trace_wake_raw(struct trace_iterator *iter, int flags,
1001 struct trace_event *event)
1002{
1003 return trace_ctxwake_raw(iter, '+');
1004}
1005
1006
1007static int trace_ctxwake_hex(struct trace_iterator *iter, char S)
1008{
1009 struct ctx_switch_entry *field;
1010 struct trace_seq *s = &iter->seq;
1011 int T;
1012
1013 trace_assign_type(field, iter->ent);
1014
1015 if (!S)
1016 S = task_index_to_char(field->prev_state);
1017 T = task_index_to_char(field->next_state);
1018
1019 SEQ_PUT_HEX_FIELD(s, field->prev_pid);
1020 SEQ_PUT_HEX_FIELD(s, field->prev_prio);
1021 SEQ_PUT_HEX_FIELD(s, S);
1022 SEQ_PUT_HEX_FIELD(s, field->next_cpu);
1023 SEQ_PUT_HEX_FIELD(s, field->next_pid);
1024 SEQ_PUT_HEX_FIELD(s, field->next_prio);
1025 SEQ_PUT_HEX_FIELD(s, T);
1026
1027 return trace_handle_return(s);
1028}
1029
1030static enum print_line_t trace_ctx_hex(struct trace_iterator *iter, int flags,
1031 struct trace_event *event)
1032{
1033 return trace_ctxwake_hex(iter, 0);
1034}
1035
1036static enum print_line_t trace_wake_hex(struct trace_iterator *iter, int flags,
1037 struct trace_event *event)
1038{
1039 return trace_ctxwake_hex(iter, '+');
1040}
1041
1042static enum print_line_t trace_ctxwake_bin(struct trace_iterator *iter,
1043 int flags, struct trace_event *event)
1044{
1045 struct ctx_switch_entry *field;
1046 struct trace_seq *s = &iter->seq;
1047
1048 trace_assign_type(field, iter->ent);
1049
1050 SEQ_PUT_FIELD(s, field->prev_pid);
1051 SEQ_PUT_FIELD(s, field->prev_prio);
1052 SEQ_PUT_FIELD(s, field->prev_state);
1053 SEQ_PUT_FIELD(s, field->next_cpu);
1054 SEQ_PUT_FIELD(s, field->next_pid);
1055 SEQ_PUT_FIELD(s, field->next_prio);
1056 SEQ_PUT_FIELD(s, field->next_state);
1057
1058 return trace_handle_return(s);
1059}
1060
1061static struct trace_event_functions trace_ctx_funcs = {
1062 .trace = trace_ctx_print,
1063 .raw = trace_ctx_raw,
1064 .hex = trace_ctx_hex,
1065 .binary = trace_ctxwake_bin,
1066};
1067
1068static struct trace_event trace_ctx_event = {
1069 .type = TRACE_CTX,
1070 .funcs = &trace_ctx_funcs,
1071};
1072
1073static struct trace_event_functions trace_wake_funcs = {
1074 .trace = trace_wake_print,
1075 .raw = trace_wake_raw,
1076 .hex = trace_wake_hex,
1077 .binary = trace_ctxwake_bin,
1078};
1079
1080static struct trace_event trace_wake_event = {
1081 .type = TRACE_WAKE,
1082 .funcs = &trace_wake_funcs,
1083};
1084
1085
1086
1087static enum print_line_t trace_stack_print(struct trace_iterator *iter,
1088 int flags, struct trace_event *event)
1089{
1090 struct stack_entry *field;
1091 struct trace_seq *s = &iter->seq;
1092 unsigned long *p;
1093 unsigned long *end;
1094
1095 trace_assign_type(field, iter->ent);
1096 end = (unsigned long *)((long)iter->ent + iter->ent_size);
1097
1098 trace_seq_puts(s, "<stack trace>\n");
1099
1100 for (p = field->caller; p && p < end && *p != ULONG_MAX; p++) {
1101
1102 if (trace_seq_has_overflowed(s))
1103 break;
1104
1105 trace_seq_puts(s, " => ");
1106 seq_print_ip_sym(s, *p, flags);
1107 trace_seq_putc(s, '\n');
1108 }
1109
1110 return trace_handle_return(s);
1111}
1112
1113static struct trace_event_functions trace_stack_funcs = {
1114 .trace = trace_stack_print,
1115};
1116
1117static struct trace_event trace_stack_event = {
1118 .type = TRACE_STACK,
1119 .funcs = &trace_stack_funcs,
1120};
1121
1122
1123static enum print_line_t trace_user_stack_print(struct trace_iterator *iter,
1124 int flags, struct trace_event *event)
1125{
1126 struct trace_array *tr = iter->tr;
1127 struct userstack_entry *field;
1128 struct trace_seq *s = &iter->seq;
1129 struct mm_struct *mm = NULL;
1130 unsigned int i;
1131
1132 trace_assign_type(field, iter->ent);
1133
1134 trace_seq_puts(s, "<user stack trace>\n");
1135
1136 if (tr->trace_flags & TRACE_ITER_SYM_USEROBJ) {
1137 struct task_struct *task;
1138
1139
1140
1141
1142 rcu_read_lock();
1143 task = find_task_by_vpid(field->tgid);
1144 if (task)
1145 mm = get_task_mm(task);
1146 rcu_read_unlock();
1147 }
1148
1149 for (i = 0; i < FTRACE_STACK_ENTRIES; i++) {
1150 unsigned long ip = field->caller[i];
1151
1152 if (!ip || trace_seq_has_overflowed(s))
1153 break;
1154
1155 trace_seq_puts(s, " => ");
1156 seq_print_user_ip(s, mm, ip, flags);
1157 trace_seq_putc(s, '\n');
1158 }
1159
1160 if (mm)
1161 mmput(mm);
1162
1163 return trace_handle_return(s);
1164}
1165
1166static struct trace_event_functions trace_user_stack_funcs = {
1167 .trace = trace_user_stack_print,
1168};
1169
1170static struct trace_event trace_user_stack_event = {
1171 .type = TRACE_USER_STACK,
1172 .funcs = &trace_user_stack_funcs,
1173};
1174
1175
1176static enum print_line_t
1177trace_hwlat_print(struct trace_iterator *iter, int flags,
1178 struct trace_event *event)
1179{
1180 struct trace_entry *entry = iter->ent;
1181 struct trace_seq *s = &iter->seq;
1182 struct hwlat_entry *field;
1183
1184 trace_assign_type(field, entry);
1185
1186 trace_seq_printf(s, "#%-5u inner/outer(us): %4llu/%-5llu ts:%lld.%09ld count:%d",
1187 field->seqnum,
1188 field->duration,
1189 field->outer_duration,
1190 (long long)field->timestamp.tv_sec,
1191 field->timestamp.tv_nsec, field->count);
1192
1193 if (field->nmi_count) {
1194
1195
1196
1197
1198 if (!IS_ENABLED(CONFIG_GENERIC_SCHED_CLOCK))
1199 trace_seq_printf(s, " nmi-total:%llu",
1200 field->nmi_total_ts);
1201 trace_seq_printf(s, " nmi-count:%u",
1202 field->nmi_count);
1203 }
1204
1205 trace_seq_putc(s, '\n');
1206
1207 return trace_handle_return(s);
1208}
1209
1210static enum print_line_t
1211trace_hwlat_raw(struct trace_iterator *iter, int flags,
1212 struct trace_event *event)
1213{
1214 struct hwlat_entry *field;
1215 struct trace_seq *s = &iter->seq;
1216
1217 trace_assign_type(field, iter->ent);
1218
1219 trace_seq_printf(s, "%llu %lld %lld %09ld %u\n",
1220 field->duration,
1221 field->outer_duration,
1222 (long long)field->timestamp.tv_sec,
1223 field->timestamp.tv_nsec,
1224 field->seqnum);
1225
1226 return trace_handle_return(s);
1227}
1228
1229static struct trace_event_functions trace_hwlat_funcs = {
1230 .trace = trace_hwlat_print,
1231 .raw = trace_hwlat_raw,
1232};
1233
1234static struct trace_event trace_hwlat_event = {
1235 .type = TRACE_HWLAT,
1236 .funcs = &trace_hwlat_funcs,
1237};
1238
1239
1240static enum print_line_t
1241trace_osnoise_print(struct trace_iterator *iter, int flags,
1242 struct trace_event *event)
1243{
1244 struct trace_entry *entry = iter->ent;
1245 struct trace_seq *s = &iter->seq;
1246 struct osnoise_entry *field;
1247 u64 ratio, ratio_dec;
1248 u64 net_runtime;
1249
1250 trace_assign_type(field, entry);
1251
1252
1253
1254
1255 net_runtime = field->runtime - field->noise;
1256 ratio = net_runtime * 10000000;
1257 do_div(ratio, field->runtime);
1258 ratio_dec = do_div(ratio, 100000);
1259
1260 trace_seq_printf(s, "%llu %10llu %3llu.%05llu %7llu",
1261 field->runtime,
1262 field->noise,
1263 ratio, ratio_dec,
1264 field->max_sample);
1265
1266 trace_seq_printf(s, " %6u", field->hw_count);
1267 trace_seq_printf(s, " %6u", field->nmi_count);
1268 trace_seq_printf(s, " %6u", field->irq_count);
1269 trace_seq_printf(s, " %6u", field->softirq_count);
1270 trace_seq_printf(s, " %6u", field->thread_count);
1271
1272 trace_seq_putc(s, '\n');
1273
1274 return trace_handle_return(s);
1275}
1276
1277static enum print_line_t
1278trace_osnoise_raw(struct trace_iterator *iter, int flags,
1279 struct trace_event *event)
1280{
1281 struct osnoise_entry *field;
1282 struct trace_seq *s = &iter->seq;
1283
1284 trace_assign_type(field, iter->ent);
1285
1286 trace_seq_printf(s, "%lld %llu %llu %u %u %u %u %u\n",
1287 field->runtime,
1288 field->noise,
1289 field->max_sample,
1290 field->hw_count,
1291 field->nmi_count,
1292 field->irq_count,
1293 field->softirq_count,
1294 field->thread_count);
1295
1296 return trace_handle_return(s);
1297}
1298
1299static struct trace_event_functions trace_osnoise_funcs = {
1300 .trace = trace_osnoise_print,
1301 .raw = trace_osnoise_raw,
1302};
1303
1304static struct trace_event trace_osnoise_event = {
1305 .type = TRACE_OSNOISE,
1306 .funcs = &trace_osnoise_funcs,
1307};
1308
1309
1310static enum print_line_t
1311trace_timerlat_print(struct trace_iterator *iter, int flags,
1312 struct trace_event *event)
1313{
1314 struct trace_entry *entry = iter->ent;
1315 struct trace_seq *s = &iter->seq;
1316 struct timerlat_entry *field;
1317
1318 trace_assign_type(field, entry);
1319
1320 trace_seq_printf(s, "#%-5u context %6s timer_latency %9llu ns\n",
1321 field->seqnum,
1322 field->context ? "thread" : "irq",
1323 field->timer_latency);
1324
1325 return trace_handle_return(s);
1326}
1327
1328static enum print_line_t
1329trace_timerlat_raw(struct trace_iterator *iter, int flags,
1330 struct trace_event *event)
1331{
1332 struct timerlat_entry *field;
1333 struct trace_seq *s = &iter->seq;
1334
1335 trace_assign_type(field, iter->ent);
1336
1337 trace_seq_printf(s, "%u %d %llu\n",
1338 field->seqnum,
1339 field->context,
1340 field->timer_latency);
1341
1342 return trace_handle_return(s);
1343}
1344
1345static struct trace_event_functions trace_timerlat_funcs = {
1346 .trace = trace_timerlat_print,
1347 .raw = trace_timerlat_raw,
1348};
1349
1350static struct trace_event trace_timerlat_event = {
1351 .type = TRACE_TIMERLAT,
1352 .funcs = &trace_timerlat_funcs,
1353};
1354
1355
1356static enum print_line_t
1357trace_bputs_print(struct trace_iterator *iter, int flags,
1358 struct trace_event *event)
1359{
1360 struct trace_entry *entry = iter->ent;
1361 struct trace_seq *s = &iter->seq;
1362 struct bputs_entry *field;
1363
1364 trace_assign_type(field, entry);
1365
1366 seq_print_ip_sym(s, field->ip, flags);
1367 trace_seq_puts(s, ": ");
1368 trace_seq_puts(s, field->str);
1369
1370 return trace_handle_return(s);
1371}
1372
1373
1374static enum print_line_t
1375trace_bputs_raw(struct trace_iterator *iter, int flags,
1376 struct trace_event *event)
1377{
1378 struct bputs_entry *field;
1379 struct trace_seq *s = &iter->seq;
1380
1381 trace_assign_type(field, iter->ent);
1382
1383 trace_seq_printf(s, ": %lx : ", field->ip);
1384 trace_seq_puts(s, field->str);
1385
1386 return trace_handle_return(s);
1387}
1388
1389static struct trace_event_functions trace_bputs_funcs = {
1390 .trace = trace_bputs_print,
1391 .raw = trace_bputs_raw,
1392};
1393
1394static struct trace_event trace_bputs_event = {
1395 .type = TRACE_BPUTS,
1396 .funcs = &trace_bputs_funcs,
1397};
1398
1399
1400static enum print_line_t
1401trace_bprint_print(struct trace_iterator *iter, int flags,
1402 struct trace_event *event)
1403{
1404 struct trace_entry *entry = iter->ent;
1405 struct trace_seq *s = &iter->seq;
1406 struct bprint_entry *field;
1407
1408 trace_assign_type(field, entry);
1409
1410 seq_print_ip_sym(s, field->ip, flags);
1411 trace_seq_puts(s, ": ");
1412 trace_seq_bprintf(s, field->fmt, field->buf);
1413
1414 return trace_handle_return(s);
1415}
1416
1417
1418static enum print_line_t
1419trace_bprint_raw(struct trace_iterator *iter, int flags,
1420 struct trace_event *event)
1421{
1422 struct bprint_entry *field;
1423 struct trace_seq *s = &iter->seq;
1424
1425 trace_assign_type(field, iter->ent);
1426
1427 trace_seq_printf(s, ": %lx : ", field->ip);
1428 trace_seq_bprintf(s, field->fmt, field->buf);
1429
1430 return trace_handle_return(s);
1431}
1432
1433static struct trace_event_functions trace_bprint_funcs = {
1434 .trace = trace_bprint_print,
1435 .raw = trace_bprint_raw,
1436};
1437
1438static struct trace_event trace_bprint_event = {
1439 .type = TRACE_BPRINT,
1440 .funcs = &trace_bprint_funcs,
1441};
1442
1443
1444static enum print_line_t trace_print_print(struct trace_iterator *iter,
1445 int flags, struct trace_event *event)
1446{
1447 struct print_entry *field;
1448 struct trace_seq *s = &iter->seq;
1449
1450 trace_assign_type(field, iter->ent);
1451
1452 seq_print_ip_sym(s, field->ip, flags);
1453 trace_seq_printf(s, ": %s", field->buf);
1454
1455 return trace_handle_return(s);
1456}
1457
1458static enum print_line_t trace_print_raw(struct trace_iterator *iter, int flags,
1459 struct trace_event *event)
1460{
1461 struct print_entry *field;
1462
1463 trace_assign_type(field, iter->ent);
1464
1465 trace_seq_printf(&iter->seq, "# %lx %s", field->ip, field->buf);
1466
1467 return trace_handle_return(&iter->seq);
1468}
1469
1470static struct trace_event_functions trace_print_funcs = {
1471 .trace = trace_print_print,
1472 .raw = trace_print_raw,
1473};
1474
1475static struct trace_event trace_print_event = {
1476 .type = TRACE_PRINT,
1477 .funcs = &trace_print_funcs,
1478};
1479
1480static enum print_line_t trace_raw_data(struct trace_iterator *iter, int flags,
1481 struct trace_event *event)
1482{
1483 struct raw_data_entry *field;
1484 int i;
1485
1486 trace_assign_type(field, iter->ent);
1487
1488 trace_seq_printf(&iter->seq, "# %x buf:", field->id);
1489
1490 for (i = 0; i < iter->ent_size - offsetof(struct raw_data_entry, buf); i++)
1491 trace_seq_printf(&iter->seq, " %02x",
1492 (unsigned char)field->buf[i]);
1493
1494 trace_seq_putc(&iter->seq, '\n');
1495
1496 return trace_handle_return(&iter->seq);
1497}
1498
1499static struct trace_event_functions trace_raw_data_funcs = {
1500 .trace = trace_raw_data,
1501 .raw = trace_raw_data,
1502};
1503
1504static struct trace_event trace_raw_data_event = {
1505 .type = TRACE_RAW_DATA,
1506 .funcs = &trace_raw_data_funcs,
1507};
1508
1509static enum print_line_t
1510trace_func_repeats_raw(struct trace_iterator *iter, int flags,
1511 struct trace_event *event)
1512{
1513 struct func_repeats_entry *field;
1514 struct trace_seq *s = &iter->seq;
1515
1516 trace_assign_type(field, iter->ent);
1517
1518 trace_seq_printf(s, "%lu %lu %u %llu\n",
1519 field->ip,
1520 field->parent_ip,
1521 field->count,
1522 FUNC_REPEATS_GET_DELTA_TS(field));
1523
1524 return trace_handle_return(s);
1525}
1526
1527static enum print_line_t
1528trace_func_repeats_print(struct trace_iterator *iter, int flags,
1529 struct trace_event *event)
1530{
1531 struct func_repeats_entry *field;
1532 struct trace_seq *s = &iter->seq;
1533
1534 trace_assign_type(field, iter->ent);
1535
1536 print_fn_trace(s, field->ip, field->parent_ip, flags);
1537 trace_seq_printf(s, " (repeats: %u, last_ts:", field->count);
1538 trace_print_time(s, iter,
1539 iter->ts - FUNC_REPEATS_GET_DELTA_TS(field));
1540 trace_seq_puts(s, ")\n");
1541
1542 return trace_handle_return(s);
1543}
1544
1545static struct trace_event_functions trace_func_repeats_funcs = {
1546 .trace = trace_func_repeats_print,
1547 .raw = trace_func_repeats_raw,
1548};
1549
1550static struct trace_event trace_func_repeats_event = {
1551 .type = TRACE_FUNC_REPEATS,
1552 .funcs = &trace_func_repeats_funcs,
1553};
1554
1555static struct trace_event *events[] __initdata = {
1556 &trace_fn_event,
1557 &trace_ctx_event,
1558 &trace_wake_event,
1559 &trace_stack_event,
1560 &trace_user_stack_event,
1561 &trace_bputs_event,
1562 &trace_bprint_event,
1563 &trace_print_event,
1564 &trace_hwlat_event,
1565 &trace_osnoise_event,
1566 &trace_timerlat_event,
1567 &trace_raw_data_event,
1568 &trace_func_repeats_event,
1569 NULL
1570};
1571
1572__init static int init_events(void)
1573{
1574 struct trace_event *event;
1575 int i, ret;
1576
1577 for (i = 0; events[i]; i++) {
1578 event = events[i];
1579
1580 ret = register_trace_event(event);
1581 if (!ret) {
1582 printk(KERN_WARNING "event %d failed to register\n",
1583 event->type);
1584 WARN_ON_ONCE(1);
1585 }
1586 }
1587
1588 return 0;
1589}
1590early_initcall(init_events);
1591
