1
2
3
4
5
6
7
8
9
10
11
12
13
14#include <linux/ring_buffer.h>
15#include <generated/utsrelease.h>
16#include <linux/stacktrace.h>
17#include <linux/writeback.h>
18#include <linux/kallsyms.h>
19#include <linux/seq_file.h>
20#include <linux/notifier.h>
21#include <linux/irqflags.h>
22#include <linux/debugfs.h>
23#include <linux/tracefs.h>
24#include <linux/pagemap.h>
25#include <linux/hardirq.h>
26#include <linux/linkage.h>
27#include <linux/uaccess.h>
28#include <linux/kprobes.h>
29#include <linux/ftrace.h>
30#include <linux/module.h>
31#include <linux/percpu.h>
32#include <linux/splice.h>
33#include <linux/kdebug.h>
34#include <linux/string.h>
35#include <linux/mount.h>
36#include <linux/rwsem.h>
37#include <linux/slab.h>
38#include <linux/ctype.h>
39#include <linux/init.h>
40#include <linux/poll.h>
41#include <linux/nmi.h>
42#include <linux/fs.h>
43#include <linux/sched/rt.h>
44
45#include "trace.h"
46#include "trace_output.h"
47
48
49
50
51
52bool ring_buffer_expanded;
53
54
55
56
57
58
59
60
61static bool __read_mostly tracing_selftest_running;
62
63
64
65
66bool __read_mostly tracing_selftest_disabled;
67
68
69struct trace_iterator *tracepoint_print_iter;
70int tracepoint_printk;
71
72
73static struct tracer_opt dummy_tracer_opt[] = {
74 { }
75};
76
77static struct tracer_flags dummy_tracer_flags = {
78 .val = 0,
79 .opts = dummy_tracer_opt
80};
81
82static int
83dummy_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set)
84{
85 return 0;
86}
87
88
89
90
91
92
93static DEFINE_PER_CPU(bool, trace_cmdline_save);
94
95
96
97
98
99
100
101static int tracing_disabled = 1;
102
103DEFINE_PER_CPU(int, ftrace_cpu_disabled);
104
105cpumask_var_t __read_mostly tracing_buffer_mask;
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123enum ftrace_dump_mode ftrace_dump_on_oops;
124
125
126int __disable_trace_on_warning;
127
128#ifdef CONFIG_TRACE_ENUM_MAP_FILE
129
130struct trace_enum_map_head {
131 struct module *mod;
132 unsigned long length;
133};
134
135union trace_enum_map_item;
136
137struct trace_enum_map_tail {
138
139
140
141
142 union trace_enum_map_item *next;
143 const char *end;
144};
145
146static DEFINE_MUTEX(trace_enum_mutex);
147
148
149
150
151
152
153
154
155union trace_enum_map_item {
156 struct trace_enum_map map;
157 struct trace_enum_map_head head;
158 struct trace_enum_map_tail tail;
159};
160
161static union trace_enum_map_item *trace_enum_maps;
162#endif
163
164static int tracing_set_tracer(struct trace_array *tr, const char *buf);
165
166#define MAX_TRACER_SIZE 100
167static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
168static char *default_bootup_tracer;
169
170static bool allocate_snapshot;
171
172static int __init set_cmdline_ftrace(char *str)
173{
174 strlcpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
175 default_bootup_tracer = bootup_tracer_buf;
176
177 ring_buffer_expanded = true;
178 return 1;
179}
180__setup("ftrace=", set_cmdline_ftrace);
181
182static int __init set_ftrace_dump_on_oops(char *str)
183{
184 if (*str++ != '=' || !*str) {
185 ftrace_dump_on_oops = DUMP_ALL;
186 return 1;
187 }
188
189 if (!strcmp("orig_cpu", str)) {
190 ftrace_dump_on_oops = DUMP_ORIG;
191 return 1;
192 }
193
194 return 0;
195}
196__setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
197
198static int __init stop_trace_on_warning(char *str)
199{
200 if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
201 __disable_trace_on_warning = 1;
202 return 1;
203}
204__setup("traceoff_on_warning", stop_trace_on_warning);
205
206static int __init boot_alloc_snapshot(char *str)
207{
208 allocate_snapshot = true;
209
210 ring_buffer_expanded = true;
211 return 1;
212}
213__setup("alloc_snapshot", boot_alloc_snapshot);
214
215
216static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata;
217static char *trace_boot_options __initdata;
218
219static int __init set_trace_boot_options(char *str)
220{
221 strlcpy(trace_boot_options_buf, str, MAX_TRACER_SIZE);
222 trace_boot_options = trace_boot_options_buf;
223 return 0;
224}
225__setup("trace_options=", set_trace_boot_options);
226
227static char trace_boot_clock_buf[MAX_TRACER_SIZE] __initdata;
228static char *trace_boot_clock __initdata;
229
230static int __init set_trace_boot_clock(char *str)
231{
232 strlcpy(trace_boot_clock_buf, str, MAX_TRACER_SIZE);
233 trace_boot_clock = trace_boot_clock_buf;
234 return 0;
235}
236__setup("trace_clock=", set_trace_boot_clock);
237
238static int __init set_tracepoint_printk(char *str)
239{
240 if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
241 tracepoint_printk = 1;
242 return 1;
243}
244__setup("tp_printk", set_tracepoint_printk);
245
246unsigned long long ns2usecs(cycle_t nsec)
247{
248 nsec += 500;
249 do_div(nsec, 1000);
250 return nsec;
251}
252
253
254
255
256
257
258
259
260
261
262
263
264
265static struct trace_array global_trace;
266
267LIST_HEAD(ftrace_trace_arrays);
268
269int trace_array_get(struct trace_array *this_tr)
270{
271 struct trace_array *tr;
272 int ret = -ENODEV;
273
274 mutex_lock(&trace_types_lock);
275 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
276 if (tr == this_tr) {
277 tr->ref++;
278 ret = 0;
279 break;
280 }
281 }
282 mutex_unlock(&trace_types_lock);
283
284 return ret;
285}
286
287static void __trace_array_put(struct trace_array *this_tr)
288{
289 WARN_ON(!this_tr->ref);
290 this_tr->ref--;
291}
292
293void trace_array_put(struct trace_array *this_tr)
294{
295 mutex_lock(&trace_types_lock);
296 __trace_array_put(this_tr);
297 mutex_unlock(&trace_types_lock);
298}
299
300int filter_check_discard(struct ftrace_event_file *file, void *rec,
301 struct ring_buffer *buffer,
302 struct ring_buffer_event *event)
303{
304 if (unlikely(file->flags & FTRACE_EVENT_FL_FILTERED) &&
305 !filter_match_preds(file->filter, rec)) {
306 ring_buffer_discard_commit(buffer, event);
307 return 1;
308 }
309
310 return 0;
311}
312EXPORT_SYMBOL_GPL(filter_check_discard);
313
314int call_filter_check_discard(struct ftrace_event_call *call, void *rec,
315 struct ring_buffer *buffer,
316 struct ring_buffer_event *event)
317{
318 if (unlikely(call->flags & TRACE_EVENT_FL_FILTERED) &&
319 !filter_match_preds(call->filter, rec)) {
320 ring_buffer_discard_commit(buffer, event);
321 return 1;
322 }
323
324 return 0;
325}
326EXPORT_SYMBOL_GPL(call_filter_check_discard);
327
328static cycle_t buffer_ftrace_now(struct trace_buffer *buf, int cpu)
329{
330 u64 ts;
331
332
333 if (!buf->buffer)
334 return trace_clock_local();
335
336 ts = ring_buffer_time_stamp(buf->buffer, cpu);
337 ring_buffer_normalize_time_stamp(buf->buffer, cpu, &ts);
338
339 return ts;
340}
341
342cycle_t ftrace_now(int cpu)
343{
344 return buffer_ftrace_now(&global_trace.trace_buffer, cpu);
345}
346
347
348
349
350
351
352
353
354
355
356int tracing_is_enabled(void)
357{
358
359
360
361
362
363 smp_rmb();
364 return !global_trace.buffer_disabled;
365}
366
367
368
369
370
371
372
373
374
375
376
377#define TRACE_BUF_SIZE_DEFAULT 1441792UL
378
379static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
380
381
382static struct tracer *trace_types __read_mostly;
383
384
385
386
387DEFINE_MUTEX(trace_types_lock);
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411#ifdef CONFIG_SMP
412static DECLARE_RWSEM(all_cpu_access_lock);
413static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
414
415static inline void trace_access_lock(int cpu)
416{
417 if (cpu == RING_BUFFER_ALL_CPUS) {
418
419 down_write(&all_cpu_access_lock);
420 } else {
421
422
423
424 down_read(&all_cpu_access_lock);
425
426
427 mutex_lock(&per_cpu(cpu_access_lock, cpu));
428 }
429}
430
431static inline void trace_access_unlock(int cpu)
432{
433 if (cpu == RING_BUFFER_ALL_CPUS) {
434 up_write(&all_cpu_access_lock);
435 } else {
436 mutex_unlock(&per_cpu(cpu_access_lock, cpu));
437 up_read(&all_cpu_access_lock);
438 }
439}
440
441static inline void trace_access_lock_init(void)
442{
443 int cpu;
444
445 for_each_possible_cpu(cpu)
446 mutex_init(&per_cpu(cpu_access_lock, cpu));
447}
448
449#else
450
451static DEFINE_MUTEX(access_lock);
452
453static inline void trace_access_lock(int cpu)
454{
455 (void)cpu;
456 mutex_lock(&access_lock);
457}
458
459static inline void trace_access_unlock(int cpu)
460{
461 (void)cpu;
462 mutex_unlock(&access_lock);
463}
464
465static inline void trace_access_lock_init(void)
466{
467}
468
469#endif
470
471
472unsigned long trace_flags = TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK |
473 TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | TRACE_ITER_SLEEP_TIME |
474 TRACE_ITER_GRAPH_TIME | TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE |
475 TRACE_ITER_IRQ_INFO | TRACE_ITER_MARKERS | TRACE_ITER_FUNCTION;
476
477static void tracer_tracing_on(struct trace_array *tr)
478{
479 if (tr->trace_buffer.buffer)
480 ring_buffer_record_on(tr->trace_buffer.buffer);
481
482
483
484
485
486
487
488
489 tr->buffer_disabled = 0;
490
491 smp_wmb();
492}
493
494
495
496
497
498
499
500void tracing_on(void)
501{
502 tracer_tracing_on(&global_trace);
503}
504EXPORT_SYMBOL_GPL(tracing_on);
505
506
507
508
509
510
511
512int __trace_puts(unsigned long ip, const char *str, int size)
513{
514 struct ring_buffer_event *event;
515 struct ring_buffer *buffer;
516 struct print_entry *entry;
517 unsigned long irq_flags;
518 int alloc;
519 int pc;
520
521 if (!(trace_flags & TRACE_ITER_PRINTK))
522 return 0;
523
524 pc = preempt_count();
525
526 if (unlikely(tracing_selftest_running || tracing_disabled))
527 return 0;
528
529 alloc = sizeof(*entry) + size + 2;
530
531 local_save_flags(irq_flags);
532 buffer = global_trace.trace_buffer.buffer;
533 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, alloc,
534 irq_flags, pc);
535 if (!event)
536 return 0;
537
538 entry = ring_buffer_event_data(event);
539 entry->ip = ip;
540
541 memcpy(&entry->buf, str, size);
542
543
544 if (entry->buf[size - 1] != '\n') {
545 entry->buf[size] = '\n';
546 entry->buf[size + 1] = '\0';
547 } else
548 entry->buf[size] = '\0';
549
550 __buffer_unlock_commit(buffer, event);
551 ftrace_trace_stack(buffer, irq_flags, 4, pc);
552
553 return size;
554}
555EXPORT_SYMBOL_GPL(__trace_puts);
556
557
558
559
560
561
562int __trace_bputs(unsigned long ip, const char *str)
563{
564 struct ring_buffer_event *event;
565 struct ring_buffer *buffer;
566 struct bputs_entry *entry;
567 unsigned long irq_flags;
568 int size = sizeof(struct bputs_entry);
569 int pc;
570
571 if (!(trace_flags & TRACE_ITER_PRINTK))
572 return 0;
573
574 pc = preempt_count();
575
576 if (unlikely(tracing_selftest_running || tracing_disabled))
577 return 0;
578
579 local_save_flags(irq_flags);
580 buffer = global_trace.trace_buffer.buffer;
581 event = trace_buffer_lock_reserve(buffer, TRACE_BPUTS, size,
582 irq_flags, pc);
583 if (!event)
584 return 0;
585
586 entry = ring_buffer_event_data(event);
587 entry->ip = ip;
588 entry->str = str;
589
590 __buffer_unlock_commit(buffer, event);
591 ftrace_trace_stack(buffer, irq_flags, 4, pc);
592
593 return 1;
594}
595EXPORT_SYMBOL_GPL(__trace_bputs);
596
597#ifdef CONFIG_TRACER_SNAPSHOT
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612void tracing_snapshot(void)
613{
614 struct trace_array *tr = &global_trace;
615 struct tracer *tracer = tr->current_trace;
616 unsigned long flags;
617
618 if (in_nmi()) {
619 internal_trace_puts("*** SNAPSHOT CALLED FROM NMI CONTEXT ***\n");
620 internal_trace_puts("*** snapshot is being ignored ***\n");
621 return;
622 }
623
624 if (!tr->allocated_snapshot) {
625 internal_trace_puts("*** SNAPSHOT NOT ALLOCATED ***\n");
626 internal_trace_puts("*** stopping trace here! ***\n");
627 tracing_off();
628 return;
629 }
630
631
632 if (tracer->use_max_tr) {
633 internal_trace_puts("*** LATENCY TRACER ACTIVE ***\n");
634 internal_trace_puts("*** Can not use snapshot (sorry) ***\n");
635 return;
636 }
637
638 local_irq_save(flags);
639 update_max_tr(tr, current, smp_processor_id());
640 local_irq_restore(flags);
641}
642EXPORT_SYMBOL_GPL(tracing_snapshot);
643
644static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
645 struct trace_buffer *size_buf, int cpu_id);
646static void set_buffer_entries(struct trace_buffer *buf, unsigned long val);
647
648static int alloc_snapshot(struct trace_array *tr)
649{
650 int ret;
651
652 if (!tr->allocated_snapshot) {
653
654
655 ret = resize_buffer_duplicate_size(&tr->max_buffer,
656 &tr->trace_buffer, RING_BUFFER_ALL_CPUS);
657 if (ret < 0)
658 return ret;
659
660 tr->allocated_snapshot = true;
661 }
662
663 return 0;
664}
665
666static void free_snapshot(struct trace_array *tr)
667{
668
669
670
671
672
673 ring_buffer_resize(tr->max_buffer.buffer, 1, RING_BUFFER_ALL_CPUS);
674 set_buffer_entries(&tr->max_buffer, 1);
675 tracing_reset_online_cpus(&tr->max_buffer);
676 tr->allocated_snapshot = false;
677}
678
679
680
681
682
683
684
685
686
687
688
689int tracing_alloc_snapshot(void)
690{
691 struct trace_array *tr = &global_trace;
692 int ret;
693
694 ret = alloc_snapshot(tr);
695 WARN_ON(ret < 0);
696
697 return ret;
698}
699EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
700
701
702
703
704
705
706
707
708
709
710
711
712void tracing_snapshot_alloc(void)
713{
714 int ret;
715
716 ret = tracing_alloc_snapshot();
717 if (ret < 0)
718 return;
719
720 tracing_snapshot();
721}
722EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
723#else
724void tracing_snapshot(void)
725{
726 WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used");
727}
728EXPORT_SYMBOL_GPL(tracing_snapshot);
729int tracing_alloc_snapshot(void)
730{
731 WARN_ONCE(1, "Snapshot feature not enabled, but snapshot allocation used");
732 return -ENODEV;
733}
734EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
735void tracing_snapshot_alloc(void)
736{
737
738 tracing_snapshot();
739}
740EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
741#endif
742
743static void tracer_tracing_off(struct trace_array *tr)
744{
745 if (tr->trace_buffer.buffer)
746 ring_buffer_record_off(tr->trace_buffer.buffer);
747
748
749
750
751
752
753
754
755 tr->buffer_disabled = 1;
756
757 smp_wmb();
758}
759
760
761
762
763
764
765
766
767
768void tracing_off(void)
769{
770 tracer_tracing_off(&global_trace);
771}
772EXPORT_SYMBOL_GPL(tracing_off);
773
774void disable_trace_on_warning(void)
775{
776 if (__disable_trace_on_warning)
777 tracing_off();
778}
779
780
781
782
783
784
785
786static int tracer_tracing_is_on(struct trace_array *tr)
787{
788 if (tr->trace_buffer.buffer)
789 return ring_buffer_record_is_on(tr->trace_buffer.buffer);
790 return !tr->buffer_disabled;
791}
792
793
794
795
796int tracing_is_on(void)
797{
798 return tracer_tracing_is_on(&global_trace);
799}
800EXPORT_SYMBOL_GPL(tracing_is_on);
801
802static int __init set_buf_size(char *str)
803{
804 unsigned long buf_size;
805
806 if (!str)
807 return 0;
808 buf_size = memparse(str, &str);
809
810 if (buf_size == 0)
811 return 0;
812 trace_buf_size = buf_size;
813 return 1;
814}
815__setup("trace_buf_size=", set_buf_size);
816
817static int __init set_tracing_thresh(char *str)
818{
819 unsigned long threshold;
820 int ret;
821
822 if (!str)
823 return 0;
824 ret = kstrtoul(str, 0, &threshold);
825 if (ret < 0)
826 return 0;
827 tracing_thresh = threshold * 1000;
828 return 1;
829}
830__setup("tracing_thresh=", set_tracing_thresh);
831
832unsigned long nsecs_to_usecs(unsigned long nsecs)
833{
834 return nsecs / 1000;
835}
836
837
838static const char *trace_options[] = {
839 "print-parent",
840 "sym-offset",
841 "sym-addr",
842 "verbose",
843 "raw",
844 "hex",
845 "bin",
846 "block",
847 "stacktrace",
848 "trace_printk",
849 "ftrace_preempt",
850 "branch",
851 "annotate",
852 "userstacktrace",
853 "sym-userobj",
854 "printk-msg-only",
855 "context-info",
856 "latency-format",
857 "sleep-time",
858 "graph-time",
859 "record-cmd",
860 "overwrite",
861 "disable_on_free",
862 "irq-info",
863 "markers",
864 "function-trace",
865 NULL
866};
867
868static struct {
869 u64 (*func)(void);
870 const char *name;
871 int in_ns;
872} trace_clocks[] = {
873 { trace_clock_local, "local", 1 },
874 { trace_clock_global, "global", 1 },
875 { trace_clock_counter, "counter", 0 },
876 { trace_clock_jiffies, "uptime", 0 },
877 { trace_clock, "perf", 1 },
878 { ktime_get_mono_fast_ns, "mono", 1 },
879 ARCH_TRACE_CLOCKS
880};
881
882
883
884
885int trace_parser_get_init(struct trace_parser *parser, int size)
886{
887 memset(parser, 0, sizeof(*parser));
888
889 parser->buffer = kmalloc(size, GFP_KERNEL);
890 if (!parser->buffer)
891 return 1;
892
893 parser->size = size;
894 return 0;
895}
896
897
898
899
900void trace_parser_put(struct trace_parser *parser)
901{
902 kfree(parser->buffer);
903}
904
905
906
907
908
909
910
911
912
913
914
915
916int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
917 size_t cnt, loff_t *ppos)
918{
919 char ch;
920 size_t read = 0;
921 ssize_t ret;
922
923 if (!*ppos)
924 trace_parser_clear(parser);
925
926 ret = get_user(ch, ubuf++);
927 if (ret)
928 goto out;
929
930 read++;
931 cnt--;
932
933
934
935
936
937 if (!parser->cont) {
938
939 while (cnt && isspace(ch)) {
940 ret = get_user(ch, ubuf++);
941 if (ret)
942 goto out;
943 read++;
944 cnt--;
945 }
946
947
948 if (isspace(ch)) {
949 *ppos += read;
950 ret = read;
951 goto out;
952 }
953
954 parser->idx = 0;
955 }
956
957
958 while (cnt && !isspace(ch)) {
959 if (parser->idx < parser->size - 1)
960 parser->buffer[parser->idx++] = ch;
961 else {
962 ret = -EINVAL;
963 goto out;
964 }
965 ret = get_user(ch, ubuf++);
966 if (ret)
967 goto out;
968 read++;
969 cnt--;
970 }
971
972
973 if (isspace(ch)) {
974 parser->buffer[parser->idx] = 0;
975 parser->cont = false;
976 } else if (parser->idx < parser->size - 1) {
977 parser->cont = true;
978 parser->buffer[parser->idx++] = ch;
979 } else {
980 ret = -EINVAL;
981 goto out;
982 }
983
984 *ppos += read;
985 ret = read;
986
987out:
988 return ret;
989}
990
991
992static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
993{
994 int len;
995
996 if (trace_seq_used(s) <= s->seq.readpos)
997 return -EBUSY;
998
999 len = trace_seq_used(s) - s->seq.readpos;
1000 if (cnt > len)
1001 cnt = len;
1002 memcpy(buf, s->buffer + s->seq.readpos, cnt);
1003
1004 s->seq.readpos += cnt;
1005 return cnt;
1006}
1007
1008unsigned long __read_mostly tracing_thresh;
1009
1010#ifdef CONFIG_TRACER_MAX_TRACE
1011
1012
1013
1014
1015
1016static void
1017__update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
1018{
1019 struct trace_buffer *trace_buf = &tr->trace_buffer;
1020 struct trace_buffer *max_buf = &tr->max_buffer;
1021 struct trace_array_cpu *data = per_cpu_ptr(trace_buf->data, cpu);
1022 struct trace_array_cpu *max_data = per_cpu_ptr(max_buf->data, cpu);
1023
1024 max_buf->cpu = cpu;
1025 max_buf->time_start = data->preempt_timestamp;
1026
1027 max_data->saved_latency = tr->max_latency;
1028 max_data->critical_start = data->critical_start;
1029 max_data->critical_end = data->critical_end;
1030
1031 memcpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
1032 max_data->pid = tsk->pid;
1033
1034
1035
1036
1037 if (tsk == current)
1038 max_data->uid = current_uid();
1039 else
1040 max_data->uid = task_uid(tsk);
1041
1042 max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
1043 max_data->policy = tsk->policy;
1044 max_data->rt_priority = tsk->rt_priority;
1045
1046
1047 tracing_record_cmdline(tsk);
1048}
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059void
1060update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
1061{
1062 struct ring_buffer *buf;
1063
1064 if (tr->stop_count)
1065 return;
1066
1067 WARN_ON_ONCE(!irqs_disabled());
1068
1069 if (!tr->allocated_snapshot) {
1070
1071 WARN_ON_ONCE(tr->current_trace != &nop_trace);
1072 return;
1073 }
1074
1075 arch_spin_lock(&tr->max_lock);
1076
1077 buf = tr->trace_buffer.buffer;
1078 tr->trace_buffer.buffer = tr->max_buffer.buffer;
1079 tr->max_buffer.buffer = buf;
1080
1081 __update_max_tr(tr, tsk, cpu);
1082 arch_spin_unlock(&tr->max_lock);
1083}
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093void
1094update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
1095{
1096 int ret;
1097
1098 if (tr->stop_count)
1099 return;
1100
1101 WARN_ON_ONCE(!irqs_disabled());
1102 if (!tr->allocated_snapshot) {
1103
1104 WARN_ON_ONCE(tr->current_trace != &nop_trace);
1105 return;
1106 }
1107
1108 arch_spin_lock(&tr->max_lock);
1109
1110 ret = ring_buffer_swap_cpu(tr->max_buffer.buffer, tr->trace_buffer.buffer, cpu);
1111
1112 if (ret == -EBUSY) {
1113
1114
1115
1116
1117
1118
1119 trace_array_printk_buf(tr->max_buffer.buffer, _THIS_IP_,
1120 "Failed to swap buffers due to commit in progress\n");
1121 }
1122
1123 WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
1124
1125 __update_max_tr(tr, tsk, cpu);
1126 arch_spin_unlock(&tr->max_lock);
1127}
1128#endif
1129
1130static int wait_on_pipe(struct trace_iterator *iter, bool full)
1131{
1132
1133 if (trace_buffer_iter(iter, iter->cpu_file))
1134 return 0;
1135
1136 return ring_buffer_wait(iter->trace_buffer->buffer, iter->cpu_file,
1137 full);
1138}
1139
1140#ifdef CONFIG_FTRACE_STARTUP_TEST
1141static int run_tracer_selftest(struct tracer *type)
1142{
1143 struct trace_array *tr = &global_trace;
1144 struct tracer *saved_tracer = tr->current_trace;
1145 int ret;
1146
1147 if (!type->selftest || tracing_selftest_disabled)
1148 return 0;
1149
1150
1151
1152
1153
1154
1155
1156
1157 tracing_reset_online_cpus(&tr->trace_buffer);
1158
1159 tr->current_trace = type;
1160
1161#ifdef CONFIG_TRACER_MAX_TRACE
1162 if (type->use_max_tr) {
1163
1164 if (ring_buffer_expanded)
1165 ring_buffer_resize(tr->max_buffer.buffer, trace_buf_size,
1166 RING_BUFFER_ALL_CPUS);
1167 tr->allocated_snapshot = true;
1168 }
1169#endif
1170
1171
1172 pr_info("Testing tracer %s: ", type->name);
1173 ret = type->selftest(type, tr);
1174
1175 tr->current_trace = saved_tracer;
1176 if (ret) {
1177 printk(KERN_CONT "FAILED!\n");
1178
1179 WARN_ON(1);
1180 return -1;
1181 }
1182
1183 tracing_reset_online_cpus(&tr->trace_buffer);
1184
1185#ifdef CONFIG_TRACER_MAX_TRACE
1186 if (type->use_max_tr) {
1187 tr->allocated_snapshot = false;
1188
1189
1190 if (ring_buffer_expanded)
1191 ring_buffer_resize(tr->max_buffer.buffer, 1,
1192 RING_BUFFER_ALL_CPUS);
1193 }
1194#endif
1195
1196 printk(KERN_CONT "PASSED\n");
1197 return 0;
1198}
1199#else
1200static inline int run_tracer_selftest(struct tracer *type)
1201{
1202 return 0;
1203}
1204#endif
1205
1206
1207
1208
1209
1210
1211
1212int register_tracer(struct tracer *type)
1213{
1214 struct tracer *t;
1215 int ret = 0;
1216
1217 if (!type->name) {
1218 pr_info("Tracer must have a name\n");
1219 return -1;
1220 }
1221
1222 if (strlen(type->name) >= MAX_TRACER_SIZE) {
1223 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
1224 return -1;
1225 }
1226
1227 mutex_lock(&trace_types_lock);
1228
1229 tracing_selftest_running = true;
1230
1231 for (t = trace_types; t; t = t->next) {
1232 if (strcmp(type->name, t->name) == 0) {
1233
1234 pr_info("Tracer %s already registered\n",
1235 type->name);
1236 ret = -1;
1237 goto out;
1238 }
1239 }
1240
1241 if (!type->set_flag)
1242 type->set_flag = &dummy_set_flag;
1243 if (!type->flags)
1244 type->flags = &dummy_tracer_flags;
1245 else
1246 if (!type->flags->opts)
1247 type->flags->opts = dummy_tracer_opt;
1248
1249 ret = run_tracer_selftest(type);
1250 if (ret < 0)
1251 goto out;
1252
1253 type->next = trace_types;
1254 trace_types = type;
1255
1256 out:
1257 tracing_selftest_running = false;
1258 mutex_unlock(&trace_types_lock);
1259
1260 if (ret || !default_bootup_tracer)
1261 goto out_unlock;
1262
1263 if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
1264 goto out_unlock;
1265
1266 printk(KERN_INFO "Starting tracer '%s'\n", type->name);
1267
1268 tracing_set_tracer(&global_trace, type->name);
1269 default_bootup_tracer = NULL;
1270
1271 tracing_selftest_disabled = true;
1272#ifdef CONFIG_FTRACE_STARTUP_TEST
1273 printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
1274 type->name);
1275#endif
1276
1277 out_unlock:
1278 return ret;
1279}
1280
1281void tracing_reset(struct trace_buffer *buf, int cpu)
1282{
1283 struct ring_buffer *buffer = buf->buffer;
1284
1285 if (!buffer)
1286 return;
1287
1288 ring_buffer_record_disable(buffer);
1289
1290
1291 synchronize_sched();
1292 ring_buffer_reset_cpu(buffer, cpu);
1293
1294 ring_buffer_record_enable(buffer);
1295}
1296
1297void tracing_reset_online_cpus(struct trace_buffer *buf)
1298{
1299 struct ring_buffer *buffer = buf->buffer;
1300 int cpu;
1301
1302 if (!buffer)
1303 return;
1304
1305 ring_buffer_record_disable(buffer);
1306
1307
1308 synchronize_sched();
1309
1310 buf->time_start = buffer_ftrace_now(buf, buf->cpu);
1311
1312 for_each_online_cpu(cpu)
1313 ring_buffer_reset_cpu(buffer, cpu);
1314
1315 ring_buffer_record_enable(buffer);
1316}
1317
1318
1319void tracing_reset_all_online_cpus(void)
1320{
1321 struct trace_array *tr;
1322
1323 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
1324 tracing_reset_online_cpus(&tr->trace_buffer);
1325#ifdef CONFIG_TRACER_MAX_TRACE
1326 tracing_reset_online_cpus(&tr->max_buffer);
1327#endif
1328 }
1329}
1330
1331#define SAVED_CMDLINES_DEFAULT 128
1332#define NO_CMDLINE_MAP UINT_MAX
1333static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
1334struct saved_cmdlines_buffer {
1335 unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
1336 unsigned *map_cmdline_to_pid;
1337 unsigned cmdline_num;
1338 int cmdline_idx;
1339 char *saved_cmdlines;
1340};
1341static struct saved_cmdlines_buffer *savedcmd;
1342
1343
1344static atomic_t trace_record_cmdline_disabled __read_mostly;
1345
1346static inline char *get_saved_cmdlines(int idx)
1347{
1348 return &savedcmd->saved_cmdlines[idx * TASK_COMM_LEN];
1349}
1350
1351static inline void set_cmdline(int idx, const char *cmdline)
1352{
1353 memcpy(get_saved_cmdlines(idx), cmdline, TASK_COMM_LEN);
1354}
1355
1356static int allocate_cmdlines_buffer(unsigned int val,
1357 struct saved_cmdlines_buffer *s)
1358{
1359 s->map_cmdline_to_pid = kmalloc(val * sizeof(*s->map_cmdline_to_pid),
1360 GFP_KERNEL);
1361 if (!s->map_cmdline_to_pid)
1362 return -ENOMEM;
1363
1364 s->saved_cmdlines = kmalloc(val * TASK_COMM_LEN, GFP_KERNEL);
1365 if (!s->saved_cmdlines) {
1366 kfree(s->map_cmdline_to_pid);
1367 return -ENOMEM;
1368 }
1369
1370 s->cmdline_idx = 0;
1371 s->cmdline_num = val;
1372 memset(&s->map_pid_to_cmdline, NO_CMDLINE_MAP,
1373 sizeof(s->map_pid_to_cmdline));
1374 memset(s->map_cmdline_to_pid, NO_CMDLINE_MAP,
1375 val * sizeof(*s->map_cmdline_to_pid));
1376
1377 return 0;
1378}
1379
1380static int trace_create_savedcmd(void)
1381{
1382 int ret;
1383
1384 savedcmd = kmalloc(sizeof(*savedcmd), GFP_KERNEL);
1385 if (!savedcmd)
1386 return -ENOMEM;
1387
1388 ret = allocate_cmdlines_buffer(SAVED_CMDLINES_DEFAULT, savedcmd);
1389 if (ret < 0) {
1390 kfree(savedcmd);
1391 savedcmd = NULL;
1392 return -ENOMEM;
1393 }
1394
1395 return 0;
1396}
1397
1398int is_tracing_stopped(void)
1399{
1400 return global_trace.stop_count;
1401}
1402
1403
1404
1405
1406
1407
1408
1409void tracing_start(void)
1410{
1411 struct ring_buffer *buffer;
1412 unsigned long flags;
1413
1414 if (tracing_disabled)
1415 return;
1416
1417 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1418 if (--global_trace.stop_count) {
1419 if (global_trace.stop_count < 0) {
1420
1421 WARN_ON_ONCE(1);
1422 global_trace.stop_count = 0;
1423 }
1424 goto out;
1425 }
1426
1427
1428 arch_spin_lock(&global_trace.max_lock);
1429
1430 buffer = global_trace.trace_buffer.buffer;
1431 if (buffer)
1432 ring_buffer_record_enable(buffer);
1433
1434#ifdef CONFIG_TRACER_MAX_TRACE
1435 buffer = global_trace.max_buffer.buffer;
1436 if (buffer)
1437 ring_buffer_record_enable(buffer);
1438#endif
1439
1440 arch_spin_unlock(&global_trace.max_lock);
1441
1442 out:
1443 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1444}
1445
1446static void tracing_start_tr(struct trace_array *tr)
1447{
1448 struct ring_buffer *buffer;
1449 unsigned long flags;
1450
1451 if (tracing_disabled)
1452 return;
1453
1454
1455 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1456 return tracing_start();
1457
1458 raw_spin_lock_irqsave(&tr->start_lock, flags);
1459
1460 if (--tr->stop_count) {
1461 if (tr->stop_count < 0) {
1462
1463 WARN_ON_ONCE(1);
1464 tr->stop_count = 0;
1465 }
1466 goto out;
1467 }
1468
1469 buffer = tr->trace_buffer.buffer;
1470 if (buffer)
1471 ring_buffer_record_enable(buffer);
1472
1473 out:
1474 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
1475}
1476
1477
1478
1479
1480
1481
1482
1483void tracing_stop(void)
1484{
1485 struct ring_buffer *buffer;
1486 unsigned long flags;
1487
1488 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1489 if (global_trace.stop_count++)
1490 goto out;
1491
1492
1493 arch_spin_lock(&global_trace.max_lock);
1494
1495 buffer = global_trace.trace_buffer.buffer;
1496 if (buffer)
1497 ring_buffer_record_disable(buffer);
1498
1499#ifdef CONFIG_TRACER_MAX_TRACE
1500 buffer = global_trace.max_buffer.buffer;
1501 if (buffer)
1502 ring_buffer_record_disable(buffer);
1503#endif
1504
1505 arch_spin_unlock(&global_trace.max_lock);
1506
1507 out:
1508 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1509}
1510
1511static void tracing_stop_tr(struct trace_array *tr)
1512{
1513 struct ring_buffer *buffer;
1514 unsigned long flags;
1515
1516
1517 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1518 return tracing_stop();
1519
1520 raw_spin_lock_irqsave(&tr->start_lock, flags);
1521 if (tr->stop_count++)
1522 goto out;
1523
1524 buffer = tr->trace_buffer.buffer;
1525 if (buffer)
1526 ring_buffer_record_disable(buffer);
1527
1528 out:
1529 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
1530}
1531
1532void trace_stop_cmdline_recording(void);
1533
1534static int trace_save_cmdline(struct task_struct *tsk)
1535{
1536 unsigned pid, idx;
1537
1538 if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT))
1539 return 0;
1540
1541
1542
1543
1544
1545
1546
1547 if (!arch_spin_trylock(&trace_cmdline_lock))
1548 return 0;
1549
1550 idx = savedcmd->map_pid_to_cmdline[tsk->pid];
1551 if (idx == NO_CMDLINE_MAP) {
1552 idx = (savedcmd->cmdline_idx + 1) % savedcmd->cmdline_num;
1553
1554
1555
1556
1557
1558
1559
1560 pid = savedcmd->map_cmdline_to_pid[idx];
1561 if (pid != NO_CMDLINE_MAP)
1562 savedcmd->map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
1563
1564 savedcmd->map_cmdline_to_pid[idx] = tsk->pid;
1565 savedcmd->map_pid_to_cmdline[tsk->pid] = idx;
1566
1567 savedcmd->cmdline_idx = idx;
1568 }
1569
1570 set_cmdline(idx, tsk->comm);
1571
1572 arch_spin_unlock(&trace_cmdline_lock);
1573
1574 return 1;
1575}
1576
1577static void __trace_find_cmdline(int pid, char comm[])
1578{
1579 unsigned map;
1580
1581 if (!pid) {
1582 strcpy(comm, "<idle>");
1583 return;
1584 }
1585
1586 if (WARN_ON_ONCE(pid < 0)) {
1587 strcpy(comm, "<XXX>");
1588 return;
1589 }
1590
1591 if (pid > PID_MAX_DEFAULT) {
1592 strcpy(comm, "<...>");
1593 return;
1594 }
1595
1596 map = savedcmd->map_pid_to_cmdline[pid];
1597 if (map != NO_CMDLINE_MAP)
1598 strcpy(comm, get_saved_cmdlines(map));
1599 else
1600 strcpy(comm, "<...>");
1601}
1602
1603void trace_find_cmdline(int pid, char comm[])
1604{
1605 preempt_disable();
1606 arch_spin_lock(&trace_cmdline_lock);
1607
1608 __trace_find_cmdline(pid, comm);
1609
1610 arch_spin_unlock(&trace_cmdline_lock);
1611 preempt_enable();
1612}
1613
1614void tracing_record_cmdline(struct task_struct *tsk)
1615{
1616 if (atomic_read(&trace_record_cmdline_disabled) || !tracing_is_on())
1617 return;
1618
1619 if (!__this_cpu_read(trace_cmdline_save))
1620 return;
1621
1622 if (trace_save_cmdline(tsk))
1623 __this_cpu_write(trace_cmdline_save, false);
1624}
1625
1626void
1627tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
1628 int pc)
1629{
1630 struct task_struct *tsk = current;
1631
1632 entry->preempt_count = pc & 0xff;
1633 entry->pid = (tsk) ? tsk->pid : 0;
1634 entry->flags =
1635#ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
1636 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
1637#else
1638 TRACE_FLAG_IRQS_NOSUPPORT |
1639#endif
1640 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
1641 ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
1642 (tif_need_resched() ? TRACE_FLAG_NEED_RESCHED : 0) |
1643 (test_preempt_need_resched() ? TRACE_FLAG_PREEMPT_RESCHED : 0);
1644}
1645EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
1646
1647struct ring_buffer_event *
1648trace_buffer_lock_reserve(struct ring_buffer *buffer,
1649 int type,
1650 unsigned long len,
1651 unsigned long flags, int pc)
1652{
1653 struct ring_buffer_event *event;
1654
1655 event = ring_buffer_lock_reserve(buffer, len);
1656 if (event != NULL) {
1657 struct trace_entry *ent = ring_buffer_event_data(event);
1658
1659 tracing_generic_entry_update(ent, flags, pc);
1660 ent->type = type;
1661 }
1662
1663 return event;
1664}
1665
1666void
1667__buffer_unlock_commit(struct ring_buffer *buffer, struct ring_buffer_event *event)
1668{
1669 __this_cpu_write(trace_cmdline_save, true);
1670 ring_buffer_unlock_commit(buffer, event);
1671}
1672
1673static inline void
1674__trace_buffer_unlock_commit(struct ring_buffer *buffer,
1675 struct ring_buffer_event *event,
1676 unsigned long flags, int pc)
1677{
1678 __buffer_unlock_commit(buffer, event);
1679
1680 ftrace_trace_stack(buffer, flags, 6, pc);
1681 ftrace_trace_userstack(buffer, flags, pc);
1682}
1683
1684void trace_buffer_unlock_commit(struct ring_buffer *buffer,
1685 struct ring_buffer_event *event,
1686 unsigned long flags, int pc)
1687{
1688 __trace_buffer_unlock_commit(buffer, event, flags, pc);
1689}
1690EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit);
1691
1692static struct ring_buffer *temp_buffer;
1693
1694struct ring_buffer_event *
1695trace_event_buffer_lock_reserve(struct ring_buffer **current_rb,
1696 struct ftrace_event_file *ftrace_file,
1697 int type, unsigned long len,
1698 unsigned long flags, int pc)
1699{
1700 struct ring_buffer_event *entry;
1701
1702 *current_rb = ftrace_file->tr->trace_buffer.buffer;
1703 entry = trace_buffer_lock_reserve(*current_rb,
1704 type, len, flags, pc);
1705
1706
1707
1708
1709
1710
1711 if (!entry && ftrace_file->flags & FTRACE_EVENT_FL_TRIGGER_COND) {
1712 *current_rb = temp_buffer;
1713 entry = trace_buffer_lock_reserve(*current_rb,
1714 type, len, flags, pc);
1715 }
1716 return entry;
1717}
1718EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve);
1719
1720struct ring_buffer_event *
1721trace_current_buffer_lock_reserve(struct ring_buffer **current_rb,
1722 int type, unsigned long len,
1723 unsigned long flags, int pc)
1724{
1725 *current_rb = global_trace.trace_buffer.buffer;
1726 return trace_buffer_lock_reserve(*current_rb,
1727 type, len, flags, pc);
1728}
1729EXPORT_SYMBOL_GPL(trace_current_buffer_lock_reserve);
1730
1731void trace_current_buffer_unlock_commit(struct ring_buffer *buffer,
1732 struct ring_buffer_event *event,
1733 unsigned long flags, int pc)
1734{
1735 __trace_buffer_unlock_commit(buffer, event, flags, pc);
1736}
1737EXPORT_SYMBOL_GPL(trace_current_buffer_unlock_commit);
1738
1739void trace_buffer_unlock_commit_regs(struct ring_buffer *buffer,
1740 struct ring_buffer_event *event,
1741 unsigned long flags, int pc,
1742 struct pt_regs *regs)
1743{
1744 __buffer_unlock_commit(buffer, event);
1745
1746 ftrace_trace_stack_regs(buffer, flags, 0, pc, regs);
1747 ftrace_trace_userstack(buffer, flags, pc);
1748}
1749EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit_regs);
1750
1751void trace_current_buffer_discard_commit(struct ring_buffer *buffer,
1752 struct ring_buffer_event *event)
1753{
1754 ring_buffer_discard_commit(buffer, event);
1755}
1756EXPORT_SYMBOL_GPL(trace_current_buffer_discard_commit);
1757
1758void
1759trace_function(struct trace_array *tr,
1760 unsigned long ip, unsigned long parent_ip, unsigned long flags,
1761 int pc)
1762{
1763 struct ftrace_event_call *call = &event_function;
1764 struct ring_buffer *buffer = tr->trace_buffer.buffer;
1765 struct ring_buffer_event *event;
1766 struct ftrace_entry *entry;
1767
1768
1769 if (unlikely(__this_cpu_read(ftrace_cpu_disabled)))
1770 return;
1771
1772 event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
1773 flags, pc);
1774 if (!event)
1775 return;
1776 entry = ring_buffer_event_data(event);
1777 entry->ip = ip;
1778 entry->parent_ip = parent_ip;
1779
1780 if (!call_filter_check_discard(call, entry, buffer, event))
1781 __buffer_unlock_commit(buffer, event);
1782}
1783
1784#ifdef CONFIG_STACKTRACE
1785
1786#define FTRACE_STACK_MAX_ENTRIES (PAGE_SIZE / sizeof(unsigned long))
1787struct ftrace_stack {
1788 unsigned long calls[FTRACE_STACK_MAX_ENTRIES];
1789};
1790
1791static DEFINE_PER_CPU(struct ftrace_stack, ftrace_stack);
1792static DEFINE_PER_CPU(int, ftrace_stack_reserve);
1793
1794static void __ftrace_trace_stack(struct ring_buffer *buffer,
1795 unsigned long flags,
1796 int skip, int pc, struct pt_regs *regs)
1797{
1798 struct ftrace_event_call *call = &event_kernel_stack;
1799 struct ring_buffer_event *event;
1800 struct stack_entry *entry;
1801 struct stack_trace trace;
1802 int use_stack;
1803 int size = FTRACE_STACK_ENTRIES;
1804
1805 trace.nr_entries = 0;
1806 trace.skip = skip;
1807
1808
1809
1810
1811
1812
1813
1814 preempt_disable_notrace();
1815
1816 use_stack = __this_cpu_inc_return(ftrace_stack_reserve);
1817
1818
1819
1820
1821
1822
1823
1824 barrier();
1825 if (use_stack == 1) {
1826 trace.entries = this_cpu_ptr(ftrace_stack.calls);
1827 trace.max_entries = FTRACE_STACK_MAX_ENTRIES;
1828
1829 if (regs)
1830 save_stack_trace_regs(regs, &trace);
1831 else
1832 save_stack_trace(&trace);
1833
1834 if (trace.nr_entries > size)
1835 size = trace.nr_entries;
1836 } else
1837
1838 use_stack = 0;
1839
1840 size *= sizeof(unsigned long);
1841
1842 event = trace_buffer_lock_reserve(buffer, TRACE_STACK,
1843 sizeof(*entry) + size, flags, pc);
1844 if (!event)
1845 goto out;
1846 entry = ring_buffer_event_data(event);
1847
1848 memset(&entry->caller, 0, size);
1849
1850 if (use_stack)
1851 memcpy(&entry->caller, trace.entries,
1852 trace.nr_entries * sizeof(unsigned long));
1853 else {
1854 trace.max_entries = FTRACE_STACK_ENTRIES;
1855 trace.entries = entry->caller;
1856 if (regs)
1857 save_stack_trace_regs(regs, &trace);
1858 else
1859 save_stack_trace(&trace);
1860 }
1861
1862 entry->size = trace.nr_entries;
1863
1864 if (!call_filter_check_discard(call, entry, buffer, event))
1865 __buffer_unlock_commit(buffer, event);
1866
1867 out:
1868
1869 barrier();
1870 __this_cpu_dec(ftrace_stack_reserve);
1871 preempt_enable_notrace();
1872
1873}
1874
1875void ftrace_trace_stack_regs(struct ring_buffer *buffer, unsigned long flags,
1876 int skip, int pc, struct pt_regs *regs)
1877{
1878 if (!(trace_flags & TRACE_ITER_STACKTRACE))
1879 return;
1880
1881 __ftrace_trace_stack(buffer, flags, skip, pc, regs);
1882}
1883
1884void ftrace_trace_stack(struct ring_buffer *buffer, unsigned long flags,
1885 int skip, int pc)
1886{
1887 if (!(trace_flags & TRACE_ITER_STACKTRACE))
1888 return;
1889
1890 __ftrace_trace_stack(buffer, flags, skip, pc, NULL);
1891}
1892
1893void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
1894 int pc)
1895{
1896 __ftrace_trace_stack(tr->trace_buffer.buffer, flags, skip, pc, NULL);
1897}
1898
1899
1900
1901
1902
1903void trace_dump_stack(int skip)
1904{
1905 unsigned long flags;
1906
1907 if (tracing_disabled || tracing_selftest_running)
1908 return;
1909
1910 local_save_flags(flags);
1911
1912
1913
1914
1915
1916 skip += 3;
1917 __ftrace_trace_stack(global_trace.trace_buffer.buffer,
1918 flags, skip, preempt_count(), NULL);
1919}
1920
1921static DEFINE_PER_CPU(int, user_stack_count);
1922
1923void
1924ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
1925{
1926 struct ftrace_event_call *call = &event_user_stack;
1927 struct ring_buffer_event *event;
1928 struct userstack_entry *entry;
1929 struct stack_trace trace;
1930
1931 if (!(trace_flags & TRACE_ITER_USERSTACKTRACE))
1932 return;
1933
1934
1935
1936
1937
1938 if (unlikely(in_nmi()))
1939 return;
1940
1941
1942
1943
1944
1945 preempt_disable();
1946 if (__this_cpu_read(user_stack_count))
1947 goto out;
1948
1949 __this_cpu_inc(user_stack_count);
1950
1951 event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
1952 sizeof(*entry), flags, pc);
1953 if (!event)
1954 goto out_drop_count;
1955 entry = ring_buffer_event_data(event);
1956
1957 entry->tgid = current->tgid;
1958 memset(&entry->caller, 0, sizeof(entry->caller));
1959
1960 trace.nr_entries = 0;
1961 trace.max_entries = FTRACE_STACK_ENTRIES;
1962 trace.skip = 0;
1963 trace.entries = entry->caller;
1964
1965 save_stack_trace_user(&trace);
1966 if (!call_filter_check_discard(call, entry, buffer, event))
1967 __buffer_unlock_commit(buffer, event);
1968
1969 out_drop_count:
1970 __this_cpu_dec(user_stack_count);
1971 out:
1972 preempt_enable();
1973}
1974
1975#ifdef UNUSED
1976static void __trace_userstack(struct trace_array *tr, unsigned long flags)
1977{
1978 ftrace_trace_userstack(tr, flags, preempt_count());
1979}
1980#endif
1981
1982#endif
1983
1984
1985struct trace_buffer_struct {
1986 char buffer[TRACE_BUF_SIZE];
1987};
1988
1989static struct trace_buffer_struct *trace_percpu_buffer;
1990static struct trace_buffer_struct *trace_percpu_sirq_buffer;
1991static struct trace_buffer_struct *trace_percpu_irq_buffer;
1992static struct trace_buffer_struct *trace_percpu_nmi_buffer;
1993
1994
1995
1996
1997
1998
1999
2000
2001static char *get_trace_buf(void)
2002{
2003 struct trace_buffer_struct *percpu_buffer;
2004
2005
2006
2007
2008
2009 if (in_nmi())
2010 percpu_buffer = trace_percpu_nmi_buffer;
2011 else if (in_irq())
2012 percpu_buffer = trace_percpu_irq_buffer;
2013 else if (in_softirq())
2014 percpu_buffer = trace_percpu_sirq_buffer;
2015 else
2016 percpu_buffer = trace_percpu_buffer;
2017
2018 if (!percpu_buffer)
2019 return NULL;
2020
2021 return this_cpu_ptr(&percpu_buffer->buffer[0]);
2022}
2023
2024static int alloc_percpu_trace_buffer(void)
2025{
2026 struct trace_buffer_struct *buffers;
2027 struct trace_buffer_struct *sirq_buffers;
2028 struct trace_buffer_struct *irq_buffers;
2029 struct trace_buffer_struct *nmi_buffers;
2030
2031 buffers = alloc_percpu(struct trace_buffer_struct);
2032 if (!buffers)
2033 goto err_warn;
2034
2035 sirq_buffers = alloc_percpu(struct trace_buffer_struct);
2036 if (!sirq_buffers)
2037 goto err_sirq;
2038
2039 irq_buffers = alloc_percpu(struct trace_buffer_struct);
2040 if (!irq_buffers)
2041 goto err_irq;
2042
2043 nmi_buffers = alloc_percpu(struct trace_buffer_struct);
2044 if (!nmi_buffers)
2045 goto err_nmi;
2046
2047 trace_percpu_buffer = buffers;
2048 trace_percpu_sirq_buffer = sirq_buffers;
2049 trace_percpu_irq_buffer = irq_buffers;
2050 trace_percpu_nmi_buffer = nmi_buffers;
2051
2052 return 0;
2053
2054 err_nmi:
2055 free_percpu(irq_buffers);
2056 err_irq:
2057 free_percpu(sirq_buffers);
2058 err_sirq:
2059 free_percpu(buffers);
2060 err_warn:
2061 WARN(1, "Could not allocate percpu trace_printk buffer");
2062 return -ENOMEM;
2063}
2064
2065static int buffers_allocated;
2066
2067void trace_printk_init_buffers(void)
2068{
2069 if (buffers_allocated)
2070 return;
2071
2072 if (alloc_percpu_trace_buffer())
2073 return;
2074
2075
2076
2077 pr_warning("\n");
2078 pr_warning("**********************************************************\n");
2079 pr_warning("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
2080 pr_warning("** **\n");
2081 pr_warning("** trace_printk() being used. Allocating extra memory. **\n");
2082 pr_warning("** **\n");
2083 pr_warning("** This means that this is a DEBUG kernel and it is **\n");
2084 pr_warning("** unsafe for production use. **\n");
2085 pr_warning("** **\n");
2086 pr_warning("** If you see this message and you are not debugging **\n");
2087 pr_warning("** the kernel, report this immediately to your vendor! **\n");
2088 pr_warning("** **\n");
2089 pr_warning("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
2090 pr_warning("**********************************************************\n");
2091
2092
2093 tracing_update_buffers();
2094
2095 buffers_allocated = 1;
2096
2097
2098
2099
2100
2101
2102
2103 if (global_trace.trace_buffer.buffer)
2104 tracing_start_cmdline_record();
2105}
2106
2107void trace_printk_start_comm(void)
2108{
2109
2110 if (!buffers_allocated)
2111 return;
2112 tracing_start_cmdline_record();
2113}
2114
2115static void trace_printk_start_stop_comm(int enabled)
2116{
2117 if (!buffers_allocated)
2118 return;
2119
2120 if (enabled)
2121 tracing_start_cmdline_record();
2122 else
2123 tracing_stop_cmdline_record();
2124}
2125
2126
2127
2128
2129
2130int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
2131{
2132 struct ftrace_event_call *call = &event_bprint;
2133 struct ring_buffer_event *event;
2134 struct ring_buffer *buffer;
2135 struct trace_array *tr = &global_trace;
2136 struct bprint_entry *entry;
2137 unsigned long flags;
2138 char *tbuffer;
2139 int len = 0, size, pc;
2140
2141 if (unlikely(tracing_selftest_running || tracing_disabled))
2142 return 0;
2143
2144
2145 pause_graph_tracing();
2146
2147 pc = preempt_count();
2148 preempt_disable_notrace();
2149
2150 tbuffer = get_trace_buf();
2151 if (!tbuffer) {
2152 len = 0;
2153 goto out;
2154 }
2155
2156 len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args);
2157
2158 if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0)
2159 goto out;
2160
2161 local_save_flags(flags);
2162 size = sizeof(*entry) + sizeof(u32) * len;
2163 buffer = tr->trace_buffer.buffer;
2164 event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
2165 flags, pc);
2166 if (!event)
2167 goto out;
2168 entry = ring_buffer_event_data(event);
2169 entry->ip = ip;
2170 entry->fmt = fmt;
2171
2172 memcpy(entry->buf, tbuffer, sizeof(u32) * len);
2173 if (!call_filter_check_discard(call, entry, buffer, event)) {
2174 __buffer_unlock_commit(buffer, event);
2175 ftrace_trace_stack(buffer, flags, 6, pc);
2176 }
2177
2178out:
2179 preempt_enable_notrace();
2180 unpause_graph_tracing();
2181
2182 return len;
2183}
2184EXPORT_SYMBOL_GPL(trace_vbprintk);
2185
2186static int
2187__trace_array_vprintk(struct ring_buffer *buffer,
2188 unsigned long ip, const char *fmt, va_list args)
2189{
2190 struct ftrace_event_call *call = &event_print;
2191 struct ring_buffer_event *event;
2192 int len = 0, size, pc;
2193 struct print_entry *entry;
2194 unsigned long flags;
2195 char *tbuffer;
2196
2197 if (tracing_disabled || tracing_selftest_running)
2198 return 0;
2199
2200
2201 pause_graph_tracing();
2202
2203 pc = preempt_count();
2204 preempt_disable_notrace();
2205
2206
2207 tbuffer = get_trace_buf();
2208 if (!tbuffer) {
2209 len = 0;
2210 goto out;
2211 }
2212
2213 len = vscnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args);
2214
2215 local_save_flags(flags);
2216 size = sizeof(*entry) + len + 1;
2217 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
2218 flags, pc);
2219 if (!event)
2220 goto out;
2221 entry = ring_buffer_event_data(event);
2222 entry->ip = ip;
2223
2224 memcpy(&entry->buf, tbuffer, len + 1);
2225 if (!call_filter_check_discard(call, entry, buffer, event)) {
2226 __buffer_unlock_commit(buffer, event);
2227 ftrace_trace_stack(buffer, flags, 6, pc);
2228 }
2229 out:
2230 preempt_enable_notrace();
2231 unpause_graph_tracing();
2232
2233 return len;
2234}
2235
2236int trace_array_vprintk(struct trace_array *tr,
2237 unsigned long ip, const char *fmt, va_list args)
2238{
2239 return __trace_array_vprintk(tr->trace_buffer.buffer, ip, fmt, args);
2240}
2241
2242int trace_array_printk(struct trace_array *tr,
2243 unsigned long ip, const char *fmt, ...)
2244{
2245 int ret;
2246 va_list ap;
2247
2248 if (!(trace_flags & TRACE_ITER_PRINTK))
2249 return 0;
2250
2251 va_start(ap, fmt);
2252 ret = trace_array_vprintk(tr, ip, fmt, ap);
2253 va_end(ap);
2254 return ret;
2255}
2256
2257int trace_array_printk_buf(struct ring_buffer *buffer,
2258 unsigned long ip, const char *fmt, ...)
2259{
2260 int ret;
2261 va_list ap;
2262
2263 if (!(trace_flags & TRACE_ITER_PRINTK))
2264 return 0;
2265
2266 va_start(ap, fmt);
2267 ret = __trace_array_vprintk(buffer, ip, fmt, ap);
2268 va_end(ap);
2269 return ret;
2270}
2271
2272int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
2273{
2274 return trace_array_vprintk(&global_trace, ip, fmt, args);
2275}
2276EXPORT_SYMBOL_GPL(trace_vprintk);
2277
2278static void trace_iterator_increment(struct trace_iterator *iter)
2279{
2280 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu);
2281
2282 iter->idx++;
2283 if (buf_iter)
2284 ring_buffer_read(buf_iter, NULL);
2285}
2286
2287static struct trace_entry *
2288peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts,
2289 unsigned long *lost_events)
2290{
2291 struct ring_buffer_event *event;
2292 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu);
2293
2294 if (buf_iter)
2295 event = ring_buffer_iter_peek(buf_iter, ts);
2296 else
2297 event = ring_buffer_peek(iter->trace_buffer->buffer, cpu, ts,
2298 lost_events);
2299
2300 if (event) {
2301 iter->ent_size = ring_buffer_event_length(event);
2302 return ring_buffer_event_data(event);
2303 }
2304 iter->ent_size = 0;
2305 return NULL;
2306}
2307
2308static struct trace_entry *
2309__find_next_entry(struct trace_iterator *iter, int *ent_cpu,
2310 unsigned long *missing_events, u64 *ent_ts)
2311{
2312 struct ring_buffer *buffer = iter->trace_buffer->buffer;
2313 struct trace_entry *ent, *next = NULL;
2314 unsigned long lost_events = 0, next_lost = 0;
2315 int cpu_file = iter->cpu_file;
2316 u64 next_ts = 0, ts;
2317 int next_cpu = -1;
2318 int next_size = 0;
2319 int cpu;
2320
2321
2322
2323
2324
2325 if (cpu_file > RING_BUFFER_ALL_CPUS) {
2326 if (ring_buffer_empty_cpu(buffer, cpu_file))
2327 return NULL;
2328 ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
2329 if (ent_cpu)
2330 *ent_cpu = cpu_file;
2331
2332 return ent;
2333 }
2334
2335 for_each_tracing_cpu(cpu) {
2336
2337 if (ring_buffer_empty_cpu(buffer, cpu))
2338 continue;
2339
2340 ent = peek_next_entry(iter, cpu, &ts, &lost_events);
2341
2342
2343
2344
2345 if (ent && (!next || ts < next_ts)) {
2346 next = ent;
2347 next_cpu = cpu;
2348 next_ts = ts;
2349 next_lost = lost_events;
2350 next_size = iter->ent_size;
2351 }
2352 }
2353
2354 iter->ent_size = next_size;
2355
2356 if (ent_cpu)
2357 *ent_cpu = next_cpu;
2358
2359 if (ent_ts)
2360 *ent_ts = next_ts;
2361
2362 if (missing_events)
2363 *missing_events = next_lost;
2364
2365 return next;
2366}
2367
2368
2369struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
2370 int *ent_cpu, u64 *ent_ts)
2371{
2372 return __find_next_entry(iter, ent_cpu, NULL, ent_ts);
2373}
2374
2375
2376void *trace_find_next_entry_inc(struct trace_iterator *iter)
2377{
2378 iter->ent = __find_next_entry(iter, &iter->cpu,
2379 &iter->lost_events, &iter->ts);
2380
2381 if (iter->ent)
2382 trace_iterator_increment(iter);
2383
2384 return iter->ent ? iter : NULL;
2385}
2386
2387static void trace_consume(struct trace_iterator *iter)
2388{
2389 ring_buffer_consume(iter->trace_buffer->buffer, iter->cpu, &iter->ts,
2390 &iter->lost_events);
2391}
2392
2393static void *s_next(struct seq_file *m, void *v, loff_t *pos)
2394{
2395 struct trace_iterator *iter = m->private;
2396 int i = (int)*pos;
2397 void *ent;
2398
2399 WARN_ON_ONCE(iter->leftover);
2400
2401 (*pos)++;
2402
2403
2404 if (iter->idx > i)
2405 return NULL;
2406
2407 if (iter->idx < 0)
2408 ent = trace_find_next_entry_inc(iter);
2409 else
2410 ent = iter;
2411
2412 while (ent && iter->idx < i)
2413 ent = trace_find_next_entry_inc(iter);
2414
2415 iter->pos = *pos;
2416
2417 return ent;
2418}
2419
2420void tracing_iter_reset(struct trace_iterator *iter, int cpu)
2421{
2422 struct ring_buffer_event *event;
2423 struct ring_buffer_iter *buf_iter;
2424 unsigned long entries = 0;
2425 u64 ts;
2426
2427 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = 0;
2428
2429 buf_iter = trace_buffer_iter(iter, cpu);
2430 if (!buf_iter)
2431 return;
2432
2433 ring_buffer_iter_reset(buf_iter);
2434
2435
2436
2437
2438
2439
2440 while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
2441 if (ts >= iter->trace_buffer->time_start)
2442 break;
2443 entries++;
2444 ring_buffer_read(buf_iter, NULL);
2445 }
2446
2447 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = entries;
2448}
2449
2450
2451
2452
2453
2454static void *s_start(struct seq_file *m, loff_t *pos)
2455{
2456 struct trace_iterator *iter = m->private;
2457 struct trace_array *tr = iter->tr;
2458 int cpu_file = iter->cpu_file;
2459 void *p = NULL;
2460 loff_t l = 0;
2461 int cpu;
2462
2463
2464
2465
2466
2467
2468
2469 mutex_lock(&trace_types_lock);
2470 if (unlikely(tr->current_trace && iter->trace->name != tr->current_trace->name))
2471 *iter->trace = *tr->current_trace;
2472 mutex_unlock(&trace_types_lock);
2473
2474#ifdef CONFIG_TRACER_MAX_TRACE
2475 if (iter->snapshot && iter->trace->use_max_tr)
2476 return ERR_PTR(-EBUSY);
2477#endif
2478
2479 if (!iter->snapshot)
2480 atomic_inc(&trace_record_cmdline_disabled);
2481
2482 if (*pos != iter->pos) {
2483 iter->ent = NULL;
2484 iter->cpu = 0;
2485 iter->idx = -1;
2486
2487 if (cpu_file == RING_BUFFER_ALL_CPUS) {
2488 for_each_tracing_cpu(cpu)
2489 tracing_iter_reset(iter, cpu);
2490 } else
2491 tracing_iter_reset(iter, cpu_file);
2492
2493 iter->leftover = 0;
2494 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
2495 ;
2496
2497 } else {
2498
2499
2500
2501
2502 if (iter->leftover)
2503 p = iter;
2504 else {
2505 l = *pos - 1;
2506 p = s_next(m, p, &l);
2507 }
2508 }
2509
2510 trace_event_read_lock();
2511 trace_access_lock(cpu_file);
2512 return p;
2513}
2514
2515static void s_stop(struct seq_file *m, void *p)
2516{
2517 struct trace_iterator *iter = m->private;
2518
2519#ifdef CONFIG_TRACER_MAX_TRACE
2520 if (iter->snapshot && iter->trace->use_max_tr)
2521 return;
2522#endif
2523
2524 if (!iter->snapshot)
2525 atomic_dec(&trace_record_cmdline_disabled);
2526
2527 trace_access_unlock(iter->cpu_file);
2528 trace_event_read_unlock();
2529}
2530
2531static void
2532get_total_entries(struct trace_buffer *buf,
2533 unsigned long *total, unsigned long *entries)
2534{
2535 unsigned long count;
2536 int cpu;
2537
2538 *total = 0;
2539 *entries = 0;
2540
2541 for_each_tracing_cpu(cpu) {
2542 count = ring_buffer_entries_cpu(buf->buffer, cpu);
2543
2544
2545
2546
2547
2548 if (per_cpu_ptr(buf->data, cpu)->skipped_entries) {
2549 count -= per_cpu_ptr(buf->data, cpu)->skipped_entries;
2550
2551 *total += count;
2552 } else
2553 *total += count +
2554 ring_buffer_overrun_cpu(buf->buffer, cpu);
2555 *entries += count;
2556 }
2557}
2558
2559static void print_lat_help_header(struct seq_file *m)
2560{
2561 seq_puts(m, "# _------=> CPU# \n"
2562 "# / _-----=> irqs-off \n"
2563 "# | / _----=> need-resched \n"
2564 "# || / _---=> hardirq/softirq \n"
2565 "# ||| / _--=> preempt-depth \n"
2566 "# |||| / delay \n"
2567 "# cmd pid ||||| time | caller \n"
2568 "# \\ / ||||| \\ | / \n");
2569}
2570
2571static void print_event_info(struct trace_buffer *buf, struct seq_file *m)
2572{
2573 unsigned long total;
2574 unsigned long entries;
2575
2576 get_total_entries(buf, &total, &entries);
2577 seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu #P:%d\n",
2578 entries, total, num_online_cpus());
2579 seq_puts(m, "#\n");
2580}
2581
2582static void print_func_help_header(struct trace_buffer *buf, struct seq_file *m)
2583{
2584 print_event_info(buf, m);
2585 seq_puts(m, "# TASK-PID CPU# TIMESTAMP FUNCTION\n"
2586 "# | | | | |\n");
2587}
2588
2589static void print_func_help_header_irq(struct trace_buffer *buf, struct seq_file *m)
2590{
2591 print_event_info(buf, m);
2592 seq_puts(m, "# _-----=> irqs-off\n"
2593 "# / _----=> need-resched\n"
2594 "# | / _---=> hardirq/softirq\n"
2595 "# || / _--=> preempt-depth\n"
2596 "# ||| / delay\n"
2597 "# TASK-PID CPU# |||| TIMESTAMP FUNCTION\n"
2598 "# | | | |||| | |\n");
2599}
2600
2601void
2602print_trace_header(struct seq_file *m, struct trace_iterator *iter)
2603{
2604 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
2605 struct trace_buffer *buf = iter->trace_buffer;
2606 struct trace_array_cpu *data = per_cpu_ptr(buf->data, buf->cpu);
2607 struct tracer *type = iter->trace;
2608 unsigned long entries;
2609 unsigned long total;
2610 const char *name = "preemption";
2611
2612 name = type->name;
2613
2614 get_total_entries(buf, &total, &entries);
2615
2616 seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
2617 name, UTS_RELEASE);
2618 seq_puts(m, "# -----------------------------------"
2619 "---------------------------------\n");
2620 seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
2621 " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
2622 nsecs_to_usecs(data->saved_latency),
2623 entries,
2624 total,
2625 buf->cpu,
2626#if defined(CONFIG_PREEMPT_NONE)
2627 "server",
2628#elif defined(CONFIG_PREEMPT_VOLUNTARY)
2629 "desktop",
2630#elif defined(CONFIG_PREEMPT)
2631 "preempt",
2632#else
2633 "unknown",
2634#endif
2635
2636 0, 0, 0, 0);
2637#ifdef CONFIG_SMP
2638 seq_printf(m, " #P:%d)\n", num_online_cpus());
2639#else
2640 seq_puts(m, ")\n");
2641#endif
2642 seq_puts(m, "# -----------------\n");
2643 seq_printf(m, "# | task: %.16s-%d "
2644 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
2645 data->comm, data->pid,
2646 from_kuid_munged(seq_user_ns(m), data->uid), data->nice,
2647 data->policy, data->rt_priority);
2648 seq_puts(m, "# -----------------\n");
2649
2650 if (data->critical_start) {
2651 seq_puts(m, "# => started at: ");
2652 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
2653 trace_print_seq(m, &iter->seq);
2654 seq_puts(m, "\n# => ended at: ");
2655 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
2656 trace_print_seq(m, &iter->seq);
2657 seq_puts(m, "\n#\n");
2658 }
2659
2660 seq_puts(m, "#\n");
2661}
2662
2663static void test_cpu_buff_start(struct trace_iterator *iter)
2664{
2665 struct trace_seq *s = &iter->seq;
2666
2667 if (!(trace_flags & TRACE_ITER_ANNOTATE))
2668 return;
2669
2670 if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
2671 return;
2672
2673 if (cpumask_test_cpu(iter->cpu, iter->started))
2674 return;
2675
2676 if (per_cpu_ptr(iter->trace_buffer->data, iter->cpu)->skipped_entries)
2677 return;
2678
2679 cpumask_set_cpu(iter->cpu, iter->started);
2680
2681
2682 if (iter->idx > 1)
2683 trace_seq_printf(s, "##### CPU %u buffer started ####\n",
2684 iter->cpu);
2685}
2686
2687static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
2688{
2689 struct trace_seq *s = &iter->seq;
2690 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
2691 struct trace_entry *entry;
2692 struct trace_event *event;
2693
2694 entry = iter->ent;
2695
2696 test_cpu_buff_start(iter);
2697
2698 event = ftrace_find_event(entry->type);
2699
2700 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2701 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
2702 trace_print_lat_context(iter);
2703 else
2704 trace_print_context(iter);
2705 }
2706
2707 if (trace_seq_has_overflowed(s))
2708 return TRACE_TYPE_PARTIAL_LINE;
2709
2710 if (event)
2711 return event->funcs->trace(iter, sym_flags, event);
2712
2713 trace_seq_printf(s, "Unknown type %d\n", entry->type);
2714
2715 return trace_handle_return(s);
2716}
2717
2718static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
2719{
2720 struct trace_seq *s = &iter->seq;
2721 struct trace_entry *entry;
2722 struct trace_event *event;
2723
2724 entry = iter->ent;
2725
2726 if (trace_flags & TRACE_ITER_CONTEXT_INFO)
2727 trace_seq_printf(s, "%d %d %llu ",
2728 entry->pid, iter->cpu, iter->ts);
2729
2730 if (trace_seq_has_overflowed(s))
2731 return TRACE_TYPE_PARTIAL_LINE;
2732
2733 event = ftrace_find_event(entry->type);
2734 if (event)
2735 return event->funcs->raw(iter, 0, event);
2736
2737 trace_seq_printf(s, "%d ?\n", entry->type);
2738
2739 return trace_handle_return(s);
2740}
2741
2742static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
2743{
2744 struct trace_seq *s = &iter->seq;
2745 unsigned char newline = '\n';
2746 struct trace_entry *entry;
2747 struct trace_event *event;
2748
2749 entry = iter->ent;
2750
2751 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2752 SEQ_PUT_HEX_FIELD(s, entry->pid);
2753 SEQ_PUT_HEX_FIELD(s, iter->cpu);
2754 SEQ_PUT_HEX_FIELD(s, iter->ts);
2755 if (trace_seq_has_overflowed(s))
2756 return TRACE_TYPE_PARTIAL_LINE;
2757 }
2758
2759 event = ftrace_find_event(entry->type);
2760 if (event) {
2761 enum print_line_t ret = event->funcs->hex(iter, 0, event);
2762 if (ret != TRACE_TYPE_HANDLED)
2763 return ret;
2764 }
2765
2766 SEQ_PUT_FIELD(s, newline);
2767
2768 return trace_handle_return(s);
2769}
2770
2771static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
2772{
2773 struct trace_seq *s = &iter->seq;
2774 struct trace_entry *entry;
2775 struct trace_event *event;
2776
2777 entry = iter->ent;
2778
2779 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2780 SEQ_PUT_FIELD(s, entry->pid);
2781 SEQ_PUT_FIELD(s, iter->cpu);
2782 SEQ_PUT_FIELD(s, iter->ts);
2783 if (trace_seq_has_overflowed(s))
2784 return TRACE_TYPE_PARTIAL_LINE;
2785 }
2786
2787 event = ftrace_find_event(entry->type);
2788 return event ? event->funcs->binary(iter, 0, event) :
2789 TRACE_TYPE_HANDLED;
2790}
2791
2792int trace_empty(struct trace_iterator *iter)
2793{
2794 struct ring_buffer_iter *buf_iter;
2795 int cpu;
2796
2797
2798 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
2799 cpu = iter->cpu_file;
2800 buf_iter = trace_buffer_iter(iter, cpu);
2801 if (buf_iter) {
2802 if (!ring_buffer_iter_empty(buf_iter))
2803 return 0;
2804 } else {
2805 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
2806 return 0;
2807 }
2808 return 1;
2809 }
2810
2811 for_each_tracing_cpu(cpu) {
2812 buf_iter = trace_buffer_iter(iter, cpu);
2813 if (buf_iter) {
2814 if (!ring_buffer_iter_empty(buf_iter))
2815 return 0;
2816 } else {
2817 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
2818 return 0;
2819 }
2820 }
2821
2822 return 1;
2823}
2824
2825
2826enum print_line_t print_trace_line(struct trace_iterator *iter)
2827{
2828 enum print_line_t ret;
2829
2830 if (iter->lost_events) {
2831 trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
2832 iter->cpu, iter->lost_events);
2833 if (trace_seq_has_overflowed(&iter->seq))
2834 return TRACE_TYPE_PARTIAL_LINE;
2835 }
2836
2837 if (iter->trace && iter->trace->print_line) {
2838 ret = iter->trace->print_line(iter);
2839 if (ret != TRACE_TYPE_UNHANDLED)
2840 return ret;
2841 }
2842
2843 if (iter->ent->type == TRACE_BPUTS &&
2844 trace_flags & TRACE_ITER_PRINTK &&
2845 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2846 return trace_print_bputs_msg_only(iter);
2847
2848 if (iter->ent->type == TRACE_BPRINT &&
2849 trace_flags & TRACE_ITER_PRINTK &&
2850 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2851 return trace_print_bprintk_msg_only(iter);
2852
2853 if (iter->ent->type == TRACE_PRINT &&
2854 trace_flags & TRACE_ITER_PRINTK &&
2855 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2856 return trace_print_printk_msg_only(iter);
2857
2858 if (trace_flags & TRACE_ITER_BIN)
2859 return print_bin_fmt(iter);
2860
2861 if (trace_flags & TRACE_ITER_HEX)
2862 return print_hex_fmt(iter);
2863
2864 if (trace_flags & TRACE_ITER_RAW)
2865 return print_raw_fmt(iter);
2866
2867 return print_trace_fmt(iter);
2868}
2869
2870void trace_latency_header(struct seq_file *m)
2871{
2872 struct trace_iterator *iter = m->private;
2873
2874
2875 if (trace_empty(iter))
2876 return;
2877
2878 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
2879 print_trace_header(m, iter);
2880
2881 if (!(trace_flags & TRACE_ITER_VERBOSE))
2882 print_lat_help_header(m);
2883}
2884
2885void trace_default_header(struct seq_file *m)
2886{
2887 struct trace_iterator *iter = m->private;
2888
2889 if (!(trace_flags & TRACE_ITER_CONTEXT_INFO))
2890 return;
2891
2892 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2893
2894 if (trace_empty(iter))
2895 return;
2896 print_trace_header(m, iter);
2897 if (!(trace_flags & TRACE_ITER_VERBOSE))
2898 print_lat_help_header(m);
2899 } else {
2900 if (!(trace_flags & TRACE_ITER_VERBOSE)) {
2901 if (trace_flags & TRACE_ITER_IRQ_INFO)
2902 print_func_help_header_irq(iter->trace_buffer, m);
2903 else
2904 print_func_help_header(iter->trace_buffer, m);
2905 }
2906 }
2907}
2908
2909static void test_ftrace_alive(struct seq_file *m)
2910{
2911 if (!ftrace_is_dead())
2912 return;
2913 seq_puts(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n"
2914 "# MAY BE MISSING FUNCTION EVENTS\n");
2915}
2916
2917#ifdef CONFIG_TRACER_MAX_TRACE
2918static void show_snapshot_main_help(struct seq_file *m)
2919{
2920 seq_puts(m, "# echo 0 > snapshot : Clears and frees snapshot buffer\n"
2921 "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
2922 "# Takes a snapshot of the main buffer.\n"
2923 "# echo 2 > snapshot : Clears snapshot buffer (but does not allocate or free)\n"
2924 "# (Doesn't have to be '2' works with any number that\n"
2925 "# is not a '0' or '1')\n");
2926}
2927
2928static void show_snapshot_percpu_help(struct seq_file *m)
2929{
2930 seq_puts(m, "# echo 0 > snapshot : Invalid for per_cpu snapshot file.\n");
2931#ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
2932 seq_puts(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
2933 "# Takes a snapshot of the main buffer for this cpu.\n");
2934#else
2935 seq_puts(m, "# echo 1 > snapshot : Not supported with this kernel.\n"
2936 "# Must use main snapshot file to allocate.\n");
2937#endif
2938 seq_puts(m, "# echo 2 > snapshot : Clears this cpu's snapshot buffer (but does not allocate)\n"
2939 "# (Doesn't have to be '2' works with any number that\n"
2940 "# is not a '0' or '1')\n");
2941}
2942
2943static void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter)
2944{
2945 if (iter->tr->allocated_snapshot)
2946 seq_puts(m, "#\n# * Snapshot is allocated *\n#\n");
2947 else
2948 seq_puts(m, "#\n# * Snapshot is freed *\n#\n");
2949
2950 seq_puts(m, "# Snapshot commands:\n");
2951 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
2952 show_snapshot_main_help(m);
2953 else
2954 show_snapshot_percpu_help(m);
2955}
2956#else
2957
2958static inline void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) { }
2959#endif
2960
2961static int s_show(struct seq_file *m, void *v)
2962{
2963 struct trace_iterator *iter = v;
2964 int ret;
2965
2966 if (iter->ent == NULL) {
2967 if (iter->tr) {
2968 seq_printf(m, "# tracer: %s\n", iter->trace->name);
2969 seq_puts(m, "#\n");
2970 test_ftrace_alive(m);
2971 }
2972 if (iter->snapshot && trace_empty(iter))
2973 print_snapshot_help(m, iter);
2974 else if (iter->trace && iter->trace->print_header)
2975 iter->trace->print_header(m);
2976 else
2977 trace_default_header(m);
2978
2979 } else if (iter->leftover) {
2980
2981
2982
2983
2984 ret = trace_print_seq(m, &iter->seq);
2985
2986
2987 iter->leftover = ret;
2988
2989 } else {
2990 print_trace_line(iter);
2991 ret = trace_print_seq(m, &iter->seq);
2992
2993
2994
2995
2996
2997
2998
2999 iter->leftover = ret;
3000 }
3001
3002 return 0;
3003}
3004
3005
3006
3007
3008
3009static inline int tracing_get_cpu(struct inode *inode)
3010{
3011 if (inode->i_cdev)
3012 return (long)inode->i_cdev - 1;
3013 return RING_BUFFER_ALL_CPUS;
3014}
3015
3016static const struct seq_operations tracer_seq_ops = {
3017 .start = s_start,
3018 .next = s_next,
3019 .stop = s_stop,
3020 .show = s_show,
3021};
3022
3023static struct trace_iterator *
3024__tracing_open(struct inode *inode, struct file *file, bool snapshot)
3025{
3026 struct trace_array *tr = inode->i_private;
3027 struct trace_iterator *iter;
3028 int cpu;
3029
3030 if (tracing_disabled)
3031 return ERR_PTR(-ENODEV);
3032
3033 iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter));
3034 if (!iter)
3035 return ERR_PTR(-ENOMEM);
3036
3037 iter->buffer_iter = kzalloc(sizeof(*iter->buffer_iter) * num_possible_cpus(),
3038 GFP_KERNEL);
3039 if (!iter->buffer_iter)
3040 goto release;
3041
3042
3043
3044
3045
3046 mutex_lock(&trace_types_lock);
3047 iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
3048 if (!iter->trace)
3049 goto fail;
3050
3051 *iter->trace = *tr->current_trace;
3052
3053 if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
3054 goto fail;
3055
3056 iter->tr = tr;
3057
3058#ifdef CONFIG_TRACER_MAX_TRACE
3059
3060 if (tr->current_trace->print_max || snapshot)
3061 iter->trace_buffer = &tr->max_buffer;
3062 else
3063#endif
3064 iter->trace_buffer = &tr->trace_buffer;
3065 iter->snapshot = snapshot;
3066 iter->pos = -1;
3067 iter->cpu_file = tracing_get_cpu(inode);
3068 mutex_init(&iter->mutex);
3069
3070
3071 if (iter->trace && iter->trace->open)
3072 iter->trace->open(iter);
3073
3074
3075 if (ring_buffer_overruns(iter->trace_buffer->buffer))
3076 iter->iter_flags |= TRACE_FILE_ANNOTATE;
3077
3078
3079 if (trace_clocks[tr->clock_id].in_ns)
3080 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
3081
3082
3083 if (!iter->snapshot)
3084 tracing_stop_tr(tr);
3085
3086 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) {
3087 for_each_tracing_cpu(cpu) {
3088 iter->buffer_iter[cpu] =
3089 ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
3090 }
3091 ring_buffer_read_prepare_sync();
3092 for_each_tracing_cpu(cpu) {
3093 ring_buffer_read_start(iter->buffer_iter[cpu]);
3094 tracing_iter_reset(iter, cpu);
3095 }
3096 } else {
3097 cpu = iter->cpu_file;
3098 iter->buffer_iter[cpu] =
3099 ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
3100 ring_buffer_read_prepare_sync();
3101 ring_buffer_read_start(iter->buffer_iter[cpu]);
3102 tracing_iter_reset(iter, cpu);
3103 }
3104
3105 mutex_unlock(&trace_types_lock);
3106
3107 return iter;
3108
3109 fail:
3110 mutex_unlock(&trace_types_lock);
3111 kfree(iter->trace);
3112 kfree(iter->buffer_iter);
3113release:
3114 seq_release_private(inode, file);
3115 return ERR_PTR(-ENOMEM);
3116}
3117
3118int tracing_open_generic(struct inode *inode, struct file *filp)
3119{
3120 if (tracing_disabled)
3121 return -ENODEV;
3122
3123 filp->private_data = inode->i_private;
3124 return 0;
3125}
3126
3127bool tracing_is_disabled(void)
3128{
3129 return (tracing_disabled) ? true: false;
3130}
3131
3132
3133
3134
3135
3136static int tracing_open_generic_tr(struct inode *inode, struct file *filp)
3137{
3138 struct trace_array *tr = inode->i_private;
3139
3140 if (tracing_disabled)
3141 return -ENODEV;
3142
3143 if (trace_array_get(tr) < 0)
3144 return -ENODEV;
3145
3146 filp->private_data = inode->i_private;
3147
3148 return 0;
3149}
3150
3151static int tracing_release(struct inode *inode, struct file *file)
3152{
3153 struct trace_array *tr = inode->i_private;
3154 struct seq_file *m = file->private_data;
3155 struct trace_iterator *iter;
3156 int cpu;
3157
3158 if (!(file->f_mode & FMODE_READ)) {
3159 trace_array_put(tr);
3160 return 0;
3161 }
3162
3163
3164 iter = m->private;
3165 mutex_lock(&trace_types_lock);
3166
3167 for_each_tracing_cpu(cpu) {
3168 if (iter->buffer_iter[cpu])
3169 ring_buffer_read_finish(iter->buffer_iter[cpu]);
3170 }
3171
3172 if (iter->trace && iter->trace->close)
3173 iter->trace->close(iter);
3174
3175 if (!iter->snapshot)
3176
3177 tracing_start_tr(tr);
3178
3179 __trace_array_put(tr);
3180
3181 mutex_unlock(&trace_types_lock);
3182
3183 mutex_destroy(&iter->mutex);
3184 free_cpumask_var(iter->started);
3185 kfree(iter->trace);
3186 kfree(iter->buffer_iter);
3187 seq_release_private(inode, file);
3188
3189 return 0;
3190}
3191
3192static int tracing_release_generic_tr(struct inode *inode, struct file *file)
3193{
3194 struct trace_array *tr = inode->i_private;
3195
3196 trace_array_put(tr);
3197 return 0;
3198}
3199
3200static int tracing_single_release_tr(struct inode *inode, struct file *file)
3201{
3202 struct trace_array *tr = inode->i_private;
3203
3204 trace_array_put(tr);
3205
3206 return single_release(inode, file);
3207}
3208
3209static int tracing_open(struct inode *inode, struct file *file)
3210{
3211 struct trace_array *tr = inode->i_private;
3212 struct trace_iterator *iter;
3213 int ret = 0;
3214
3215 if (trace_array_get(tr) < 0)
3216 return -ENODEV;
3217
3218
3219 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
3220 int cpu = tracing_get_cpu(inode);
3221
3222 if (cpu == RING_BUFFER_ALL_CPUS)
3223 tracing_reset_online_cpus(&tr->trace_buffer);
3224 else
3225 tracing_reset(&tr->trace_buffer, cpu);
3226 }
3227
3228 if (file->f_mode & FMODE_READ) {
3229 iter = __tracing_open(inode, file, false);
3230 if (IS_ERR(iter))
3231 ret = PTR_ERR(iter);
3232 else if (trace_flags & TRACE_ITER_LATENCY_FMT)
3233 iter->iter_flags |= TRACE_FILE_LAT_FMT;
3234 }
3235
3236 if (ret < 0)
3237 trace_array_put(tr);
3238
3239 return ret;
3240}
3241
3242
3243
3244
3245
3246
3247static bool
3248trace_ok_for_array(struct tracer *t, struct trace_array *tr)
3249{
3250 return (tr->flags & TRACE_ARRAY_FL_GLOBAL) || t->allow_instances;
3251}
3252
3253
3254static struct tracer *
3255get_tracer_for_array(struct trace_array *tr, struct tracer *t)
3256{
3257 while (t && !trace_ok_for_array(t, tr))
3258 t = t->next;
3259
3260 return t;
3261}
3262
3263static void *
3264t_next(struct seq_file *m, void *v, loff_t *pos)
3265{
3266 struct trace_array *tr = m->private;
3267 struct tracer *t = v;
3268
3269 (*pos)++;
3270
3271 if (t)
3272 t = get_tracer_for_array(tr, t->next);
3273
3274 return t;
3275}
3276
3277static void *t_start(struct seq_file *m, loff_t *pos)
3278{
3279 struct trace_array *tr = m->private;
3280 struct tracer *t;
3281 loff_t l = 0;
3282
3283 mutex_lock(&trace_types_lock);
3284
3285 t = get_tracer_for_array(tr, trace_types);
3286 for (; t && l < *pos; t = t_next(m, t, &l))
3287 ;
3288
3289 return t;
3290}
3291
3292static void t_stop(struct seq_file *m, void *p)
3293{
3294 mutex_unlock(&trace_types_lock);
3295}
3296
3297static int t_show(struct seq_file *m, void *v)
3298{
3299 struct tracer *t = v;
3300
3301 if (!t)
3302 return 0;
3303
3304 seq_puts(m, t->name);
3305 if (t->next)
3306 seq_putc(m, ' ');
3307 else
3308 seq_putc(m, '\n');
3309
3310 return 0;
3311}
3312
3313static const struct seq_operations show_traces_seq_ops = {
3314 .start = t_start,
3315 .next = t_next,
3316 .stop = t_stop,
3317 .show = t_show,
3318};
3319
3320static int show_traces_open(struct inode *inode, struct file *file)
3321{
3322 struct trace_array *tr = inode->i_private;
3323 struct seq_file *m;
3324 int ret;
3325
3326 if (tracing_disabled)
3327 return -ENODEV;
3328
3329 ret = seq_open(file, &show_traces_seq_ops);
3330 if (ret)
3331 return ret;
3332
3333 m = file->private_data;
3334 m->private = tr;
3335
3336 return 0;
3337}
3338
3339static ssize_t
3340tracing_write_stub(struct file *filp, const char __user *ubuf,
3341 size_t count, loff_t *ppos)
3342{
3343 return count;
3344}
3345
3346loff_t tracing_lseek(struct file *file, loff_t offset, int whence)
3347{
3348 int ret;
3349
3350 if (file->f_mode & FMODE_READ)
3351 ret = seq_lseek(file, offset, whence);
3352 else
3353 file->f_pos = ret = 0;
3354
3355 return ret;
3356}
3357
3358static const struct file_operations tracing_fops = {
3359 .open = tracing_open,
3360 .read = seq_read,
3361 .write = tracing_write_stub,
3362 .llseek = tracing_lseek,
3363 .release = tracing_release,
3364};
3365
3366static const struct file_operations show_traces_fops = {
3367 .open = show_traces_open,
3368 .read = seq_read,
3369 .release = seq_release,
3370 .llseek = seq_lseek,
3371};
3372
3373
3374
3375
3376
3377static DEFINE_MUTEX(tracing_cpumask_update_lock);
3378
3379
3380
3381
3382
3383static char mask_str[NR_CPUS + 1];
3384
3385static ssize_t
3386tracing_cpumask_read(struct file *filp, char __user *ubuf,
3387 size_t count, loff_t *ppos)
3388{
3389 struct trace_array *tr = file_inode(filp)->i_private;
3390 int len;
3391
3392 mutex_lock(&tracing_cpumask_update_lock);
3393
3394 len = snprintf(mask_str, count, "%*pb\n",
3395 cpumask_pr_args(tr->tracing_cpumask));
3396 if (len >= count) {
3397 count = -EINVAL;
3398 goto out_err;
3399 }
3400 count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);
3401
3402out_err:
3403 mutex_unlock(&tracing_cpumask_update_lock);
3404
3405 return count;
3406}
3407
3408static ssize_t
3409tracing_cpumask_write(struct file *filp, const char __user *ubuf,
3410 size_t count, loff_t *ppos)
3411{
3412 struct trace_array *tr = file_inode(filp)->i_private;
3413 cpumask_var_t tracing_cpumask_new;
3414 int err, cpu;
3415
3416 if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
3417 return -ENOMEM;
3418
3419 err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
3420 if (err)
3421 goto err_unlock;
3422
3423 mutex_lock(&tracing_cpumask_update_lock);
3424
3425 local_irq_disable();
3426 arch_spin_lock(&tr->max_lock);
3427 for_each_tracing_cpu(cpu) {
3428
3429
3430
3431
3432 if (cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
3433 !cpumask_test_cpu(cpu, tracing_cpumask_new)) {
3434 atomic_inc(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
3435 ring_buffer_record_disable_cpu(tr->trace_buffer.buffer, cpu);
3436 }
3437 if (!cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
3438 cpumask_test_cpu(cpu, tracing_cpumask_new)) {
3439 atomic_dec(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
3440 ring_buffer_record_enable_cpu(tr->trace_buffer.buffer, cpu);
3441 }
3442 }
3443 arch_spin_unlock(&tr->max_lock);
3444 local_irq_enable();
3445
3446 cpumask_copy(tr->tracing_cpumask, tracing_cpumask_new);
3447
3448 mutex_unlock(&tracing_cpumask_update_lock);
3449 free_cpumask_var(tracing_cpumask_new);
3450
3451 return count;
3452
3453err_unlock:
3454 free_cpumask_var(tracing_cpumask_new);
3455
3456 return err;
3457}
3458
3459static const struct file_operations tracing_cpumask_fops = {
3460 .open = tracing_open_generic_tr,
3461 .read = tracing_cpumask_read,
3462 .write = tracing_cpumask_write,
3463 .release = tracing_release_generic_tr,
3464 .llseek = generic_file_llseek,
3465};
3466
3467static int tracing_trace_options_show(struct seq_file *m, void *v)
3468{
3469 struct tracer_opt *trace_opts;
3470 struct trace_array *tr = m->private;
3471 u32 tracer_flags;
3472 int i;
3473
3474 mutex_lock(&trace_types_lock);
3475 tracer_flags = tr->current_trace->flags->val;
3476 trace_opts = tr->current_trace->flags->opts;
3477
3478 for (i = 0; trace_options[i]; i++) {
3479 if (trace_flags & (1 << i))
3480 seq_printf(m, "%s\n", trace_options[i]);
3481 else
3482 seq_printf(m, "no%s\n", trace_options[i]);
3483 }
3484
3485 for (i = 0; trace_opts[i].name; i++) {
3486 if (tracer_flags & trace_opts[i].bit)
3487 seq_printf(m, "%s\n", trace_opts[i].name);
3488 else
3489 seq_printf(m, "no%s\n", trace_opts[i].name);
3490 }
3491 mutex_unlock(&trace_types_lock);
3492
3493 return 0;
3494}
3495
3496static int __set_tracer_option(struct trace_array *tr,
3497 struct tracer_flags *tracer_flags,
3498 struct tracer_opt *opts, int neg)
3499{
3500 struct tracer *trace = tr->current_trace;
3501 int ret;
3502
3503 ret = trace->set_flag(tr, tracer_flags->val, opts->bit, !neg);
3504 if (ret)
3505 return ret;
3506
3507 if (neg)
3508 tracer_flags->val &= ~opts->bit;
3509 else
3510 tracer_flags->val |= opts->bit;
3511 return 0;
3512}
3513
3514
3515static int set_tracer_option(struct trace_array *tr, char *cmp, int neg)
3516{
3517 struct tracer *trace = tr->current_trace;
3518 struct tracer_flags *tracer_flags = trace->flags;
3519 struct tracer_opt *opts = NULL;
3520 int i;
3521
3522 for (i = 0; tracer_flags->opts[i].name; i++) {
3523 opts = &tracer_flags->opts[i];
3524
3525 if (strcmp(cmp, opts->name) == 0)
3526 return __set_tracer_option(tr, trace->flags, opts, neg);
3527 }
3528
3529 return -EINVAL;
3530}
3531
3532
3533int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set)
3534{
3535 if (tracer->enabled && (mask & TRACE_ITER_OVERWRITE) && !set)
3536 return -1;
3537
3538 return 0;
3539}
3540
3541int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled)
3542{
3543
3544 if (!!(trace_flags & mask) == !!enabled)
3545 return 0;
3546
3547
3548 if (tr->current_trace->flag_changed)
3549 if (tr->current_trace->flag_changed(tr, mask, !!enabled))
3550 return -EINVAL;
3551
3552 if (enabled)
3553 trace_flags |= mask;
3554 else
3555 trace_flags &= ~mask;
3556
3557 if (mask == TRACE_ITER_RECORD_CMD)
3558 trace_event_enable_cmd_record(enabled);
3559
3560 if (mask == TRACE_ITER_OVERWRITE) {
3561 ring_buffer_change_overwrite(tr->trace_buffer.buffer, enabled);
3562#ifdef CONFIG_TRACER_MAX_TRACE
3563 ring_buffer_change_overwrite(tr->max_buffer.buffer, enabled);
3564#endif
3565 }
3566
3567 if (mask == TRACE_ITER_PRINTK)
3568 trace_printk_start_stop_comm(enabled);
3569
3570 return 0;
3571}
3572
3573static int trace_set_options(struct trace_array *tr, char *option)
3574{
3575 char *cmp;
3576 int neg = 0;
3577 int ret = -ENODEV;
3578 int i;
3579
3580 cmp = strstrip(option);
3581
3582 if (strncmp(cmp, "no", 2) == 0) {
3583 neg = 1;
3584 cmp += 2;
3585 }
3586
3587 mutex_lock(&trace_types_lock);
3588
3589 for (i = 0; trace_options[i]; i++) {
3590 if (strcmp(cmp, trace_options[i]) == 0) {
3591 ret = set_tracer_flag(tr, 1 << i, !neg);
3592 break;
3593 }
3594 }
3595
3596
3597 if (!trace_options[i])
3598 ret = set_tracer_option(tr, cmp, neg);
3599
3600 mutex_unlock(&trace_types_lock);
3601
3602 return ret;
3603}
3604
3605static ssize_t
3606tracing_trace_options_write(struct file *filp, const char __user *ubuf,
3607 size_t cnt, loff_t *ppos)
3608{
3609 struct seq_file *m = filp->private_data;
3610 struct trace_array *tr = m->private;
3611 char buf[64];
3612 int ret;
3613
3614 if (cnt >= sizeof(buf))
3615 return -EINVAL;
3616
3617 if (copy_from_user(&buf, ubuf, cnt))
3618 return -EFAULT;
3619
3620 buf[cnt] = 0;
3621
3622 ret = trace_set_options(tr, buf);
3623 if (ret < 0)
3624 return ret;
3625
3626 *ppos += cnt;
3627
3628 return cnt;
3629}
3630
3631static int tracing_trace_options_open(struct inode *inode, struct file *file)
3632{
3633 struct trace_array *tr = inode->i_private;
3634 int ret;
3635
3636 if (tracing_disabled)
3637 return -ENODEV;
3638
3639 if (trace_array_get(tr) < 0)
3640 return -ENODEV;
3641
3642 ret = single_open(file, tracing_trace_options_show, inode->i_private);
3643 if (ret < 0)
3644 trace_array_put(tr);
3645
3646 return ret;
3647}
3648
3649static const struct file_operations tracing_iter_fops = {
3650 .open = tracing_trace_options_open,
3651 .read = seq_read,
3652 .llseek = seq_lseek,
3653 .release = tracing_single_release_tr,
3654 .write = tracing_trace_options_write,
3655};
3656
3657static const char readme_msg[] =
3658 "tracing mini-HOWTO:\n\n"
3659 "# echo 0 > tracing_on : quick way to disable tracing\n"
3660 "# echo 1 > tracing_on : quick way to re-enable tracing\n\n"
3661 " Important files:\n"
3662 " trace\t\t\t- The static contents of the buffer\n"
3663 "\t\t\t To clear the buffer write into this file: echo > trace\n"
3664 " trace_pipe\t\t- A consuming read to see the contents of the buffer\n"
3665 " current_tracer\t- function and latency tracers\n"
3666 " available_tracers\t- list of configured tracers for current_tracer\n"
3667 " buffer_size_kb\t- view and modify size of per cpu buffer\n"
3668 " buffer_total_size_kb - view total size of all cpu buffers\n\n"
3669 " trace_clock\t\t-change the clock used to order events\n"
3670 " local: Per cpu clock but may not be synced across CPUs\n"
3671 " global: Synced across CPUs but slows tracing down.\n"
3672 " counter: Not a clock, but just an increment\n"
3673 " uptime: Jiffy counter from time of boot\n"
3674 " perf: Same clock that perf events use\n"
3675#ifdef CONFIG_X86_64
3676 " x86-tsc: TSC cycle counter\n"
3677#endif
3678 "\n trace_marker\t\t- Writes into this file writes into the kernel buffer\n"
3679 " tracing_cpumask\t- Limit which CPUs to trace\n"
3680 " instances\t\t- Make sub-buffers with: mkdir instances/foo\n"
3681 "\t\t\t Remove sub-buffer with rmdir\n"
3682 " trace_options\t\t- Set format or modify how tracing happens\n"
3683 "\t\t\t Disable an option by adding a suffix 'no' to the\n"
3684 "\t\t\t option name\n"
3685 " saved_cmdlines_size\t- echo command number in here to store comm-pid list\n"
3686#ifdef CONFIG_DYNAMIC_FTRACE
3687 "\n available_filter_functions - list of functions that can be filtered on\n"
3688 " set_ftrace_filter\t- echo function name in here to only trace these\n"
3689 "\t\t\t functions\n"
3690 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
3691 "\t modules: Can select a group via module\n"
3692 "\t Format: :mod:<module-name>\n"
3693 "\t example: echo :mod:ext3 > set_ftrace_filter\n"
3694 "\t triggers: a command to perform when function is hit\n"
3695 "\t Format: <function>:<trigger>[:count]\n"
3696 "\t trigger: traceon, traceoff\n"
3697 "\t\t enable_event:<system>:<event>\n"
3698 "\t\t disable_event:<system>:<event>\n"
3699#ifdef CONFIG_STACKTRACE
3700 "\t\t stacktrace\n"
3701#endif
3702#ifdef CONFIG_TRACER_SNAPSHOT
3703 "\t\t snapshot\n"
3704#endif
3705 "\t\t dump\n"
3706 "\t\t cpudump\n"
3707 "\t example: echo do_fault:traceoff > set_ftrace_filter\n"
3708 "\t echo do_trap:traceoff:3 > set_ftrace_filter\n"
3709 "\t The first one will disable tracing every time do_fault is hit\n"
3710 "\t The second will disable tracing at most 3 times when do_trap is hit\n"
3711 "\t The first time do trap is hit and it disables tracing, the\n"
3712 "\t counter will decrement to 2. If tracing is already disabled,\n"
3713 "\t the counter will not decrement. It only decrements when the\n"
3714 "\t trigger did work\n"
3715 "\t To remove trigger without count:\n"
3716 "\t echo '!<function>:<trigger> > set_ftrace_filter\n"
3717 "\t To remove trigger with a count:\n"
3718 "\t echo '!<function>:<trigger>:0 > set_ftrace_filter\n"
3719 " set_ftrace_notrace\t- echo function name in here to never trace.\n"
3720 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
3721 "\t modules: Can select a group via module command :mod:\n"
3722 "\t Does not accept triggers\n"
3723#endif
3724#ifdef CONFIG_FUNCTION_TRACER
3725 " set_ftrace_pid\t- Write pid(s) to only function trace those pids\n"
3726 "\t\t (function)\n"
3727#endif
3728#ifdef CONFIG_FUNCTION_GRAPH_TRACER
3729 " set_graph_function\t- Trace the nested calls of a function (function_graph)\n"
3730 " set_graph_notrace\t- Do not trace the nested calls of a function (function_graph)\n"
3731 " max_graph_depth\t- Trace a limited depth of nested calls (0 is unlimited)\n"
3732#endif
3733#ifdef CONFIG_TRACER_SNAPSHOT
3734 "\n snapshot\t\t- Like 'trace' but shows the content of the static\n"
3735 "\t\t\t snapshot buffer. Read the contents for more\n"
3736 "\t\t\t information\n"
3737#endif
3738#ifdef CONFIG_STACK_TRACER
3739 " stack_trace\t\t- Shows the max stack trace when active\n"
3740 " stack_max_size\t- Shows current max stack size that was traced\n"
3741 "\t\t\t Write into this file to reset the max size (trigger a\n"
3742 "\t\t\t new trace)\n"
3743#ifdef CONFIG_DYNAMIC_FTRACE
3744 " stack_trace_filter\t- Like set_ftrace_filter but limits what stack_trace\n"
3745 "\t\t\t traces\n"
3746#endif
3747#endif
3748 " events/\t\t- Directory containing all trace event subsystems:\n"
3749 " enable\t\t- Write 0/1 to enable/disable tracing of all events\n"
3750 " events/<system>/\t- Directory containing all trace events for <system>:\n"
3751 " enable\t\t- Write 0/1 to enable/disable tracing of all <system>\n"
3752 "\t\t\t events\n"
3753 " filter\t\t- If set, only events passing filter are traced\n"
3754 " events/<system>/<event>/\t- Directory containing control files for\n"
3755 "\t\t\t <event>:\n"
3756 " enable\t\t- Write 0/1 to enable/disable tracing of <event>\n"
3757 " filter\t\t- If set, only events passing filter are traced\n"
3758 " trigger\t\t- If set, a command to perform when event is hit\n"
3759 "\t Format: <trigger>[:count][if <filter>]\n"
3760 "\t trigger: traceon, traceoff\n"
3761 "\t enable_event:<system>:<event>\n"
3762 "\t disable_event:<system>:<event>\n"
3763#ifdef CONFIG_STACKTRACE
3764 "\t\t stacktrace\n"
3765#endif
3766#ifdef CONFIG_TRACER_SNAPSHOT
3767 "\t\t snapshot\n"
3768#endif
3769 "\t example: echo traceoff > events/block/block_unplug/trigger\n"
3770 "\t echo traceoff:3 > events/block/block_unplug/trigger\n"
3771 "\t echo 'enable_event:kmem:kmalloc:3 if nr_rq > 1' > \\\n"
3772 "\t events/block/block_unplug/trigger\n"
3773 "\t The first disables tracing every time block_unplug is hit.\n"
3774 "\t The second disables tracing the first 3 times block_unplug is hit.\n"
3775 "\t The third enables the kmalloc event the first 3 times block_unplug\n"
3776 "\t is hit and has value of greater than 1 for the 'nr_rq' event field.\n"
3777 "\t Like function triggers, the counter is only decremented if it\n"
3778 "\t enabled or disabled tracing.\n"
3779 "\t To remove a trigger without a count:\n"
3780 "\t echo '!<trigger> > <system>/<event>/trigger\n"
3781 "\t To remove a trigger with a count:\n"
3782 "\t echo '!<trigger>:0 > <system>/<event>/trigger\n"
3783 "\t Filters can be ignored when removing a trigger.\n"
3784;
3785
3786static ssize_t
3787tracing_readme_read(struct file *filp, char __user *ubuf,
3788 size_t cnt, loff_t *ppos)
3789{
3790 return simple_read_from_buffer(ubuf, cnt, ppos,
3791 readme_msg, strlen(readme_msg));
3792}
3793
3794static const struct file_operations tracing_readme_fops = {
3795 .open = tracing_open_generic,
3796 .read = tracing_readme_read,
3797 .llseek = generic_file_llseek,
3798};
3799
3800static void *saved_cmdlines_next(struct seq_file *m, void *v, loff_t *pos)
3801{
3802 unsigned int *ptr = v;
3803
3804 if (*pos || m->count)
3805 ptr++;
3806
3807 (*pos)++;
3808
3809 for (; ptr < &savedcmd->map_cmdline_to_pid[savedcmd->cmdline_num];
3810 ptr++) {
3811 if (*ptr == -1 || *ptr == NO_CMDLINE_MAP)
3812 continue;
3813
3814 return ptr;
3815 }
3816
3817 return NULL;
3818}
3819
3820static void *saved_cmdlines_start(struct seq_file *m, loff_t *pos)
3821{
3822 void *v;
3823 loff_t l = 0;
3824
3825 preempt_disable();
3826 arch_spin_lock(&trace_cmdline_lock);
3827
3828 v = &savedcmd->map_cmdline_to_pid[0];
3829 while (l <= *pos) {
3830 v = saved_cmdlines_next(m, v, &l);
3831 if (!v)
3832 return NULL;
3833 }
3834
3835 return v;
3836}
3837
3838static void saved_cmdlines_stop(struct seq_file *m, void *v)
3839{
3840 arch_spin_unlock(&trace_cmdline_lock);
3841 preempt_enable();
3842}
3843
3844static int saved_cmdlines_show(struct seq_file *m, void *v)
3845{
3846 char buf[TASK_COMM_LEN];
3847 unsigned int *pid = v;
3848
3849 __trace_find_cmdline(*pid, buf);
3850 seq_printf(m, "%d %s\n", *pid, buf);
3851 return 0;
3852}
3853
3854static const struct seq_operations tracing_saved_cmdlines_seq_ops = {
3855 .start = saved_cmdlines_start,
3856 .next = saved_cmdlines_next,
3857 .stop = saved_cmdlines_stop,
3858 .show = saved_cmdlines_show,
3859};
3860
3861static int tracing_saved_cmdlines_open(struct inode *inode, struct file *filp)
3862{
3863 if (tracing_disabled)
3864 return -ENODEV;
3865
3866 return seq_open(filp, &tracing_saved_cmdlines_seq_ops);
3867}
3868
3869static const struct file_operations tracing_saved_cmdlines_fops = {
3870 .open = tracing_saved_cmdlines_open,
3871 .read = seq_read,
3872 .llseek = seq_lseek,
3873 .release = seq_release,
3874};
3875
3876static ssize_t
3877tracing_saved_cmdlines_size_read(struct file *filp, char __user *ubuf,
3878 size_t cnt, loff_t *ppos)
3879{
3880 char buf[64];
3881 int r;
3882
3883 arch_spin_lock(&trace_cmdline_lock);
3884 r = scnprintf(buf, sizeof(buf), "%u\n", savedcmd->cmdline_num);
3885 arch_spin_unlock(&trace_cmdline_lock);
3886
3887 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3888}
3889
3890static void free_saved_cmdlines_buffer(struct saved_cmdlines_buffer *s)
3891{
3892 kfree(s->saved_cmdlines);
3893 kfree(s->map_cmdline_to_pid);
3894 kfree(s);
3895}
3896
3897static int tracing_resize_saved_cmdlines(unsigned int val)
3898{
3899 struct saved_cmdlines_buffer *s, *savedcmd_temp;
3900
3901 s = kmalloc(sizeof(*s), GFP_KERNEL);
3902 if (!s)
3903 return -ENOMEM;
3904
3905 if (allocate_cmdlines_buffer(val, s) < 0) {
3906 kfree(s);
3907 return -ENOMEM;
3908 }
3909
3910 arch_spin_lock(&trace_cmdline_lock);
3911 savedcmd_temp = savedcmd;
3912 savedcmd = s;
3913 arch_spin_unlock(&trace_cmdline_lock);
3914 free_saved_cmdlines_buffer(savedcmd_temp);
3915
3916 return 0;
3917}
3918
3919static ssize_t
3920tracing_saved_cmdlines_size_write(struct file *filp, const char __user *ubuf,
3921 size_t cnt, loff_t *ppos)
3922{
3923 unsigned long val;
3924 int ret;
3925
3926 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
3927 if (ret)
3928 return ret;
3929
3930
3931 if (!val || val > PID_MAX_DEFAULT)
3932 return -EINVAL;
3933
3934 ret = tracing_resize_saved_cmdlines((unsigned int)val);
3935 if (ret < 0)
3936 return ret;
3937
3938 *ppos += cnt;
3939
3940 return cnt;
3941}
3942
3943static const struct file_operations tracing_saved_cmdlines_size_fops = {
3944 .open = tracing_open_generic,
3945 .read = tracing_saved_cmdlines_size_read,
3946 .write = tracing_saved_cmdlines_size_write,
3947};
3948
3949#ifdef CONFIG_TRACE_ENUM_MAP_FILE
3950static union trace_enum_map_item *
3951update_enum_map(union trace_enum_map_item *ptr)
3952{
3953 if (!ptr->map.enum_string) {
3954 if (ptr->tail.next) {
3955 ptr = ptr->tail.next;
3956
3957 ptr++;
3958 } else
3959 return NULL;
3960 }
3961 return ptr;
3962}
3963
3964static void *enum_map_next(struct seq_file *m, void *v, loff_t *pos)
3965{
3966 union trace_enum_map_item *ptr = v;
3967
3968
3969
3970
3971
3972 ptr = update_enum_map(ptr);
3973 if (WARN_ON_ONCE(!ptr))
3974 return NULL;
3975
3976 ptr++;
3977
3978 (*pos)++;
3979
3980 ptr = update_enum_map(ptr);
3981
3982 return ptr;
3983}
3984
3985static void *enum_map_start(struct seq_file *m, loff_t *pos)
3986{
3987 union trace_enum_map_item *v;
3988 loff_t l = 0;
3989
3990 mutex_lock(&trace_enum_mutex);
3991
3992 v = trace_enum_maps;
3993 if (v)
3994 v++;
3995
3996 while (v && l < *pos) {
3997 v = enum_map_next(m, v, &l);
3998 }
3999
4000 return v;
4001}
4002
4003static void enum_map_stop(struct seq_file *m, void *v)
4004{
4005 mutex_unlock(&trace_enum_mutex);
4006}
4007
4008static int enum_map_show(struct seq_file *m, void *v)
4009{
4010 union trace_enum_map_item *ptr = v;
4011
4012 seq_printf(m, "%s %ld (%s)\n",
4013 ptr->map.enum_string, ptr->map.enum_value,
4014 ptr->map.system);
4015
4016 return 0;
4017}
4018
4019static const struct seq_operations tracing_enum_map_seq_ops = {
4020 .start = enum_map_start,
4021 .next = enum_map_next,
4022 .stop = enum_map_stop,
4023 .show = enum_map_show,
4024};
4025
4026static int tracing_enum_map_open(struct inode *inode, struct file *filp)
4027{
4028 if (tracing_disabled)
4029 return -ENODEV;
4030
4031 return seq_open(filp, &tracing_enum_map_seq_ops);
4032}
4033
4034static const struct file_operations tracing_enum_map_fops = {
4035 .open = tracing_enum_map_open,
4036 .read = seq_read,
4037 .llseek = seq_lseek,
4038 .release = seq_release,
4039};
4040
4041static inline union trace_enum_map_item *
4042trace_enum_jmp_to_tail(union trace_enum_map_item *ptr)
4043{
4044
4045 return ptr + ptr->head.length + 1;
4046}
4047
4048static void
4049trace_insert_enum_map_file(struct module *mod, struct trace_enum_map **start,
4050 int len)
4051{
4052 struct trace_enum_map **stop;
4053 struct trace_enum_map **map;
4054 union trace_enum_map_item *map_array;
4055 union trace_enum_map_item *ptr;
4056
4057 stop = start + len;
4058
4059
4060
4061
4062
4063
4064 map_array = kmalloc(sizeof(*map_array) * (len + 2), GFP_KERNEL);
4065 if (!map_array) {
4066 pr_warning("Unable to allocate trace enum mapping\n");
4067 return;
4068 }
4069
4070 mutex_lock(&trace_enum_mutex);
4071
4072 if (!trace_enum_maps)
4073 trace_enum_maps = map_array;
4074 else {
4075 ptr = trace_enum_maps;
4076 for (;;) {
4077 ptr = trace_enum_jmp_to_tail(ptr);
4078 if (!ptr->tail.next)
4079 break;
4080 ptr = ptr->tail.next;
4081
4082 }
4083 ptr->tail.next = map_array;
4084 }
4085 map_array->head.mod = mod;
4086 map_array->head.length = len;
4087 map_array++;
4088
4089 for (map = start; (unsigned long)map < (unsigned long)stop; map++) {
4090 map_array->map = **map;
4091 map_array++;
4092 }
4093 memset(map_array, 0, sizeof(*map_array));
4094
4095 mutex_unlock(&trace_enum_mutex);
4096}
4097
4098static void trace_create_enum_file(struct dentry *d_tracer)
4099{
4100 trace_create_file("enum_map", 0444, d_tracer,
4101 NULL, &tracing_enum_map_fops);
4102}
4103
4104#else
4105static inline void trace_create_enum_file(struct dentry *d_tracer) { }
4106static inline void trace_insert_enum_map_file(struct module *mod,
4107 struct trace_enum_map **start, int len) { }
4108#endif
4109
4110static void trace_insert_enum_map(struct module *mod,
4111 struct trace_enum_map **start, int len)
4112{
4113 struct trace_enum_map **map;
4114
4115 if (len <= 0)
4116 return;
4117
4118 map = start;
4119
4120 trace_event_enum_update(map, len);
4121
4122 trace_insert_enum_map_file(mod, start, len);
4123}
4124
4125static ssize_t
4126tracing_set_trace_read(struct file *filp, char __user *ubuf,
4127 size_t cnt, loff_t *ppos)
4128{
4129 struct trace_array *tr = filp->private_data;
4130 char buf[MAX_TRACER_SIZE+2];
4131 int r;
4132
4133 mutex_lock(&trace_types_lock);
4134 r = sprintf(buf, "%s\n", tr->current_trace->name);
4135 mutex_unlock(&trace_types_lock);
4136
4137 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4138}
4139
4140int tracer_init(struct tracer *t, struct trace_array *tr)
4141{
4142 tracing_reset_online_cpus(&tr->trace_buffer);
4143 return t->init(tr);
4144}
4145
4146static void set_buffer_entries(struct trace_buffer *buf, unsigned long val)
4147{
4148 int cpu;
4149
4150 for_each_tracing_cpu(cpu)
4151 per_cpu_ptr(buf->data, cpu)->entries = val;
4152}
4153
4154#ifdef CONFIG_TRACER_MAX_TRACE
4155
4156static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
4157 struct trace_buffer *size_buf, int cpu_id)
4158{
4159 int cpu, ret = 0;
4160
4161 if (cpu_id == RING_BUFFER_ALL_CPUS) {
4162 for_each_tracing_cpu(cpu) {
4163 ret = ring_buffer_resize(trace_buf->buffer,
4164 per_cpu_ptr(size_buf->data, cpu)->entries, cpu);
4165 if (ret < 0)
4166 break;
4167 per_cpu_ptr(trace_buf->data, cpu)->entries =
4168 per_cpu_ptr(size_buf->data, cpu)->entries;
4169 }
4170 } else {
4171 ret = ring_buffer_resize(trace_buf->buffer,
4172 per_cpu_ptr(size_buf->data, cpu_id)->entries, cpu_id);
4173 if (ret == 0)
4174 per_cpu_ptr(trace_buf->data, cpu_id)->entries =
4175 per_cpu_ptr(size_buf->data, cpu_id)->entries;
4176 }
4177
4178 return ret;
4179}
4180#endif
4181
4182static int __tracing_resize_ring_buffer(struct trace_array *tr,
4183 unsigned long size, int cpu)
4184{
4185 int ret;
4186
4187
4188
4189
4190
4191
4192 ring_buffer_expanded = true;
4193
4194
4195 if (!tr->trace_buffer.buffer)
4196 return 0;
4197
4198 ret = ring_buffer_resize(tr->trace_buffer.buffer, size, cpu);
4199 if (ret < 0)
4200 return ret;
4201
4202#ifdef CONFIG_TRACER_MAX_TRACE
4203 if (!(tr->flags & TRACE_ARRAY_FL_GLOBAL) ||
4204 !tr->current_trace->use_max_tr)
4205 goto out;
4206
4207 ret = ring_buffer_resize(tr->max_buffer.buffer, size, cpu);
4208 if (ret < 0) {
4209 int r = resize_buffer_duplicate_size(&tr->trace_buffer,
4210 &tr->trace_buffer, cpu);
4211 if (r < 0) {
4212
4213
4214
4215
4216
4217
4218
4219
4220
4221
4222
4223
4224
4225
4226 WARN_ON(1);
4227 tracing_disabled = 1;
4228 }
4229 return ret;
4230 }
4231
4232 if (cpu == RING_BUFFER_ALL_CPUS)
4233 set_buffer_entries(&tr->max_buffer, size);
4234 else
4235 per_cpu_ptr(tr->max_buffer.data, cpu)->entries = size;
4236
4237 out:
4238#endif
4239
4240 if (cpu == RING_BUFFER_ALL_CPUS)
4241 set_buffer_entries(&tr->trace_buffer, size);
4242 else
4243 per_cpu_ptr(tr->trace_buffer.data, cpu)->entries = size;
4244
4245 return ret;
4246}
4247
4248static ssize_t tracing_resize_ring_buffer(struct trace_array *tr,
4249 unsigned long size, int cpu_id)
4250{
4251 int ret = size;
4252
4253 mutex_lock(&trace_types_lock);
4254
4255 if (cpu_id != RING_BUFFER_ALL_CPUS) {
4256
4257 if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) {
4258 ret = -EINVAL;
4259 goto out;
4260 }
4261 }
4262
4263 ret = __tracing_resize_ring_buffer(tr, size, cpu_id);
4264 if (ret < 0)
4265 ret = -ENOMEM;
4266
4267out:
4268 mutex_unlock(&trace_types_lock);
4269
4270 return ret;
4271}
4272
4273
4274
4275
4276
4277
4278
4279
4280
4281
4282
4283
4284int tracing_update_buffers(void)
4285{
4286 int ret = 0;
4287
4288 mutex_lock(&trace_types_lock);
4289 if (!ring_buffer_expanded)
4290 ret = __tracing_resize_ring_buffer(&global_trace, trace_buf_size,
4291 RING_BUFFER_ALL_CPUS);
4292 mutex_unlock(&trace_types_lock);
4293
4294 return ret;
4295}
4296
4297struct trace_option_dentry;
4298
4299static struct trace_option_dentry *
4300create_trace_option_files(struct trace_array *tr, struct tracer *tracer);
4301
4302static void
4303destroy_trace_option_files(struct trace_option_dentry *topts);
4304
4305
4306
4307
4308
4309static void tracing_set_nop(struct trace_array *tr)
4310{
4311 if (tr->current_trace == &nop_trace)
4312 return;
4313
4314 tr->current_trace->enabled--;
4315
4316 if (tr->current_trace->reset)
4317 tr->current_trace->reset(tr);
4318
4319 tr->current_trace = &nop_trace;
4320}
4321
4322static void update_tracer_options(struct trace_array *tr, struct tracer *t)
4323{
4324 static struct trace_option_dentry *topts;
4325
4326
4327 if (!tr->dir)
4328 return;
4329
4330
4331 if (!(tr->flags & TRACE_ARRAY_FL_GLOBAL))
4332 return;
4333
4334 destroy_trace_option_files(topts);
4335 topts = create_trace_option_files(tr, t);
4336}
4337
4338static int tracing_set_tracer(struct trace_array *tr, const char *buf)
4339{
4340 struct tracer *t;
4341#ifdef CONFIG_TRACER_MAX_TRACE
4342 bool had_max_tr;
4343#endif
4344 int ret = 0;
4345
4346 mutex_lock(&trace_types_lock);
4347
4348 if (!ring_buffer_expanded) {
4349 ret = __tracing_resize_ring_buffer(tr, trace_buf_size,
4350 RING_BUFFER_ALL_CPUS);
4351 if (ret < 0)
4352 goto out;
4353 ret = 0;
4354 }
4355
4356 for (t = trace_types; t; t = t->next) {
4357 if (strcmp(t->name, buf) == 0)
4358 break;
4359 }
4360 if (!t) {
4361 ret = -EINVAL;
4362 goto out;
4363 }
4364 if (t == tr->current_trace)
4365 goto out;
4366
4367
4368 if (!trace_ok_for_array(t, tr)) {
4369 ret = -EINVAL;
4370 goto out;
4371 }
4372
4373
4374 if (tr->current_trace->ref) {
4375 ret = -EBUSY;
4376 goto out;
4377 }
4378
4379 trace_branch_disable();
4380
4381 tr->current_trace->enabled--;
4382
4383 if (tr->current_trace->reset)
4384 tr->current_trace->reset(tr);
4385
4386
4387 tr->current_trace = &nop_trace;
4388
4389#ifdef CONFIG_TRACER_MAX_TRACE
4390 had_max_tr = tr->allocated_snapshot;
4391
4392 if (had_max_tr && !t->use_max_tr) {
4393
4394
4395
4396
4397
4398
4399
4400 synchronize_sched();
4401 free_snapshot(tr);
4402 }
4403#endif
4404 update_tracer_options(tr, t);
4405
4406#ifdef CONFIG_TRACER_MAX_TRACE
4407 if (t->use_max_tr && !had_max_tr) {
4408 ret = alloc_snapshot(tr);
4409 if (ret < 0)
4410 goto out;
4411 }
4412#endif
4413
4414 if (t->init) {
4415 ret = tracer_init(t, tr);
4416 if (ret)
4417 goto out;
4418 }
4419
4420 tr->current_trace = t;
4421 tr->current_trace->enabled++;
4422 trace_branch_enable(tr);
4423 out:
4424 mutex_unlock(&trace_types_lock);
4425
4426 return ret;
4427}
4428
4429static ssize_t
4430tracing_set_trace_write(struct file *filp, const char __user *ubuf,
4431 size_t cnt, loff_t *ppos)
4432{
4433 struct trace_array *tr = filp->private_data;
4434 char buf[MAX_TRACER_SIZE+1];
4435 int i;
4436 size_t ret;
4437 int err;
4438
4439 ret = cnt;
4440
4441 if (cnt > MAX_TRACER_SIZE)
4442 cnt = MAX_TRACER_SIZE;
4443
4444 if (copy_from_user(&buf, ubuf, cnt))
4445 return -EFAULT;
4446
4447 buf[cnt] = 0;
4448
4449
4450 for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
4451 buf[i] = 0;
4452
4453 err = tracing_set_tracer(tr, buf);
4454 if (err)
4455 return err;
4456
4457 *ppos += ret;
4458
4459 return ret;
4460}
4461
4462static ssize_t
4463tracing_nsecs_read(unsigned long *ptr, char __user *ubuf,
4464 size_t cnt, loff_t *ppos)
4465{
4466 char buf[64];
4467 int r;
4468
4469 r = snprintf(buf, sizeof(buf), "%ld\n",
4470 *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
4471 if (r > sizeof(buf))
4472 r = sizeof(buf);
4473 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4474}
4475
4476static ssize_t
4477tracing_nsecs_write(unsigned long *ptr, const char __user *ubuf,
4478 size_t cnt, loff_t *ppos)
4479{
4480 unsigned long val;
4481 int ret;
4482
4483 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4484 if (ret)
4485 return ret;
4486
4487 *ptr = val * 1000;
4488
4489 return cnt;
4490}
4491
4492static ssize_t
4493tracing_thresh_read(struct file *filp, char __user *ubuf,
4494 size_t cnt, loff_t *ppos)
4495{
4496 return tracing_nsecs_read(&tracing_thresh, ubuf, cnt, ppos);
4497}
4498
4499static ssize_t
4500tracing_thresh_write(struct file *filp, const char __user *ubuf,
4501 size_t cnt, loff_t *ppos)
4502{
4503 struct trace_array *tr = filp->private_data;
4504 int ret;
4505
4506 mutex_lock(&trace_types_lock);
4507 ret = tracing_nsecs_write(&tracing_thresh, ubuf, cnt, ppos);
4508 if (ret < 0)
4509 goto out;
4510
4511 if (tr->current_trace->update_thresh) {
4512 ret = tr->current_trace->update_thresh(tr);
4513 if (ret < 0)
4514 goto out;
4515 }
4516
4517 ret = cnt;
4518out:
4519 mutex_unlock(&trace_types_lock);
4520
4521 return ret;
4522}
4523
4524static ssize_t
4525tracing_max_lat_read(struct file *filp, char __user *ubuf,
4526 size_t cnt, loff_t *ppos)
4527{
4528 return tracing_nsecs_read(filp->private_data, ubuf, cnt, ppos);
4529}
4530
4531static ssize_t
4532tracing_max_lat_write(struct file *filp, const char __user *ubuf,
4533 size_t cnt, loff_t *ppos)
4534{
4535 return tracing_nsecs_write(filp->private_data, ubuf, cnt, ppos);
4536}
4537
4538static int tracing_open_pipe(struct inode *inode, struct file *filp)
4539{
4540 struct trace_array *tr = inode->i_private;
4541 struct trace_iterator *iter;
4542 int ret = 0;
4543
4544 if (tracing_disabled)
4545 return -ENODEV;
4546
4547 if (trace_array_get(tr) < 0)
4548 return -ENODEV;
4549
4550 mutex_lock(&trace_types_lock);
4551
4552
4553 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
4554 if (!iter) {
4555 ret = -ENOMEM;
4556 __trace_array_put(tr);
4557 goto out;
4558 }
4559
4560 trace_seq_init(&iter->seq);
4561 iter->trace = tr->current_trace;
4562
4563 if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
4564 ret = -ENOMEM;
4565 goto fail;
4566 }
4567
4568
4569 cpumask_setall(iter->started);
4570
4571 if (trace_flags & TRACE_ITER_LATENCY_FMT)
4572 iter->iter_flags |= TRACE_FILE_LAT_FMT;
4573
4574
4575 if (trace_clocks[tr->clock_id].in_ns)
4576 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
4577
4578 iter->tr = tr;
4579 iter->trace_buffer = &tr->trace_buffer;
4580 iter->cpu_file = tracing_get_cpu(inode);
4581 mutex_init(&iter->mutex);
4582 filp->private_data = iter;
4583
4584 if (iter->trace->pipe_open)
4585 iter->trace->pipe_open(iter);
4586
4587 nonseekable_open(inode, filp);
4588
4589 tr->current_trace->ref++;
4590out:
4591 mutex_unlock(&trace_types_lock);
4592 return ret;
4593
4594fail:
4595 kfree(iter->trace);
4596 kfree(iter);
4597 __trace_array_put(tr);
4598 mutex_unlock(&trace_types_lock);
4599 return ret;
4600}
4601
4602static int tracing_release_pipe(struct inode *inode, struct file *file)
4603{
4604 struct trace_iterator *iter = file->private_data;
4605 struct trace_array *tr = inode->i_private;
4606
4607 mutex_lock(&trace_types_lock);
4608
4609 tr->current_trace->ref--;
4610
4611 if (iter->trace->pipe_close)
4612 iter->trace->pipe_close(iter);
4613
4614 mutex_unlock(&trace_types_lock);
4615
4616 free_cpumask_var(iter->started);
4617 mutex_destroy(&iter->mutex);
4618 kfree(iter);
4619
4620 trace_array_put(tr);
4621
4622 return 0;
4623}
4624
4625static unsigned int
4626trace_poll(struct trace_iterator *iter, struct file *filp, poll_table *poll_table)
4627{
4628
4629 if (trace_buffer_iter(iter, iter->cpu_file))
4630 return POLLIN | POLLRDNORM;
4631
4632 if (trace_flags & TRACE_ITER_BLOCK)
4633
4634
4635
4636 return POLLIN | POLLRDNORM;
4637 else
4638 return ring_buffer_poll_wait(iter->trace_buffer->buffer, iter->cpu_file,
4639 filp, poll_table);
4640}
4641
4642static unsigned int
4643tracing_poll_pipe(struct file *filp, poll_table *poll_table)
4644{
4645 struct trace_iterator *iter = filp->private_data;
4646
4647 return trace_poll(iter, filp, poll_table);
4648}
4649
4650
4651static int tracing_wait_pipe(struct file *filp)
4652{
4653 struct trace_iterator *iter = filp->private_data;
4654 int ret;
4655
4656 while (trace_empty(iter)) {
4657
4658 if ((filp->f_flags & O_NONBLOCK)) {
4659 return -EAGAIN;
4660 }
4661
4662
4663
4664
4665
4666
4667
4668
4669
4670
4671 if (!tracing_is_on() && iter->pos)
4672 break;
4673
4674 mutex_unlock(&iter->mutex);
4675
4676 ret = wait_on_pipe(iter, false);
4677
4678 mutex_lock(&iter->mutex);
4679
4680 if (ret)
4681 return ret;
4682 }
4683
4684 return 1;
4685}
4686
4687
4688
4689
4690static ssize_t
4691tracing_read_pipe(struct file *filp, char __user *ubuf,
4692 size_t cnt, loff_t *ppos)
4693{
4694 struct trace_iterator *iter = filp->private_data;
4695 ssize_t sret;
4696
4697
4698 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
4699 if (sret != -EBUSY)
4700 return sret;
4701
4702 trace_seq_init(&iter->seq);
4703
4704
4705
4706
4707
4708
4709 mutex_lock(&iter->mutex);
4710 if (iter->trace->read) {
4711 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
4712 if (sret)
4713 goto out;
4714 }
4715
4716waitagain:
4717 sret = tracing_wait_pipe(filp);
4718 if (sret <= 0)
4719 goto out;
4720
4721
4722 if (trace_empty(iter)) {
4723 sret = 0;
4724 goto out;
4725 }
4726
4727 if (cnt >= PAGE_SIZE)
4728 cnt = PAGE_SIZE - 1;
4729
4730
4731 memset(&iter->seq, 0,
4732 sizeof(struct trace_iterator) -
4733 offsetof(struct trace_iterator, seq));
4734 cpumask_clear(iter->started);
4735 iter->pos = -1;
4736
4737 trace_event_read_lock();
4738 trace_access_lock(iter->cpu_file);
4739 while (trace_find_next_entry_inc(iter) != NULL) {
4740 enum print_line_t ret;
4741 int save_len = iter->seq.seq.len;
4742
4743 ret = print_trace_line(iter);
4744 if (ret == TRACE_TYPE_PARTIAL_LINE) {
4745
4746 iter->seq.seq.len = save_len;
4747 break;
4748 }
4749 if (ret != TRACE_TYPE_NO_CONSUME)
4750 trace_consume(iter);
4751
4752 if (trace_seq_used(&iter->seq) >= cnt)
4753 break;
4754
4755
4756
4757
4758
4759
4760 WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
4761 iter->ent->type);
4762 }
4763 trace_access_unlock(iter->cpu_file);
4764 trace_event_read_unlock();
4765
4766
4767 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
4768 if (iter->seq.seq.readpos >= trace_seq_used(&iter->seq))
4769 trace_seq_init(&iter->seq);
4770
4771
4772
4773
4774
4775 if (sret == -EBUSY)
4776 goto waitagain;
4777
4778out:
4779 mutex_unlock(&iter->mutex);
4780
4781 return sret;
4782}
4783
4784static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
4785 unsigned int idx)
4786{
4787 __free_page(spd->pages[idx]);
4788}
4789
4790static const struct pipe_buf_operations tracing_pipe_buf_ops = {
4791 .can_merge = 0,
4792 .confirm = generic_pipe_buf_confirm,
4793 .release = generic_pipe_buf_release,
4794 .steal = generic_pipe_buf_steal,
4795 .get = generic_pipe_buf_get,
4796};
4797
4798static size_t
4799tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
4800{
4801 size_t count;
4802 int save_len;
4803 int ret;
4804
4805
4806 for (;;) {
4807 save_len = iter->seq.seq.len;
4808 ret = print_trace_line(iter);
4809
4810 if (trace_seq_has_overflowed(&iter->seq)) {
4811 iter->seq.seq.len = save_len;
4812 break;
4813 }
4814
4815
4816
4817
4818
4819
4820 if (ret == TRACE_TYPE_PARTIAL_LINE) {
4821 iter->seq.seq.len = save_len;
4822 break;
4823 }
4824
4825 count = trace_seq_used(&iter->seq) - save_len;
4826 if (rem < count) {
4827 rem = 0;
4828 iter->seq.seq.len = save_len;
4829 break;
4830 }
4831
4832 if (ret != TRACE_TYPE_NO_CONSUME)
4833 trace_consume(iter);
4834 rem -= count;
4835 if (!trace_find_next_entry_inc(iter)) {
4836 rem = 0;
4837 iter->ent = NULL;
4838 break;
4839 }
4840 }
4841
4842 return rem;
4843}
4844
4845static ssize_t tracing_splice_read_pipe(struct file *filp,
4846 loff_t *ppos,
4847 struct pipe_inode_info *pipe,
4848 size_t len,
4849 unsigned int flags)
4850{
4851 struct page *pages_def[PIPE_DEF_BUFFERS];
4852 struct partial_page partial_def[PIPE_DEF_BUFFERS];
4853 struct trace_iterator *iter = filp->private_data;
4854 struct splice_pipe_desc spd = {
4855 .pages = pages_def,
4856 .partial = partial_def,
4857 .nr_pages = 0,
4858 .nr_pages_max = PIPE_DEF_BUFFERS,
4859 .flags = flags,
4860 .ops = &tracing_pipe_buf_ops,
4861 .spd_release = tracing_spd_release_pipe,
4862 };
4863 ssize_t ret;
4864 size_t rem;
4865 unsigned int i;
4866
4867 if (splice_grow_spd(pipe, &spd))
4868 return -ENOMEM;
4869
4870 mutex_lock(&iter->mutex);
4871
4872 if (iter->trace->splice_read) {
4873 ret = iter->trace->splice_read(iter, filp,
4874 ppos, pipe, len, flags);
4875 if (ret)
4876 goto out_err;
4877 }
4878
4879 ret = tracing_wait_pipe(filp);
4880 if (ret <= 0)
4881 goto out_err;
4882
4883 if (!iter->ent && !trace_find_next_entry_inc(iter)) {
4884 ret = -EFAULT;
4885 goto out_err;
4886 }
4887
4888 trace_event_read_lock();
4889 trace_access_lock(iter->cpu_file);
4890
4891
4892 for (i = 0, rem = len; i < spd.nr_pages_max && rem; i++) {
4893 spd.pages[i] = alloc_page(GFP_KERNEL);
4894 if (!spd.pages[i])
4895 break;
4896
4897 rem = tracing_fill_pipe_page(rem, iter);
4898
4899
4900 ret = trace_seq_to_buffer(&iter->seq,
4901 page_address(spd.pages[i]),
4902 trace_seq_used(&iter->seq));
4903 if (ret < 0) {
4904 __free_page(spd.pages[i]);
4905 break;
4906 }
4907 spd.partial[i].offset = 0;
4908 spd.partial[i].len = trace_seq_used(&iter->seq);
4909
4910 trace_seq_init(&iter->seq);
4911 }
4912
4913 trace_access_unlock(iter->cpu_file);
4914 trace_event_read_unlock();
4915 mutex_unlock(&iter->mutex);
4916
4917 spd.nr_pages = i;
4918
4919 ret = splice_to_pipe(pipe, &spd);
4920out:
4921 splice_shrink_spd(&spd);
4922 return ret;
4923
4924out_err:
4925 mutex_unlock(&iter->mutex);
4926 goto out;
4927}
4928
4929static ssize_t
4930tracing_entries_read(struct file *filp, char __user *ubuf,
4931 size_t cnt, loff_t *ppos)
4932{
4933 struct inode *inode = file_inode(filp);
4934 struct trace_array *tr = inode->i_private;
4935 int cpu = tracing_get_cpu(inode);
4936 char buf[64];
4937 int r = 0;
4938 ssize_t ret;
4939
4940 mutex_lock(&trace_types_lock);
4941
4942 if (cpu == RING_BUFFER_ALL_CPUS) {
4943 int cpu, buf_size_same;
4944 unsigned long size;
4945
4946 size = 0;
4947 buf_size_same = 1;
4948
4949 for_each_tracing_cpu(cpu) {
4950
4951 if (size == 0)
4952 size = per_cpu_ptr(tr->trace_buffer.data, cpu)->entries;
4953 if (size != per_cpu_ptr(tr->trace_buffer.data, cpu)->entries) {
4954 buf_size_same = 0;
4955 break;
4956 }
4957 }
4958
4959 if (buf_size_same) {
4960 if (!ring_buffer_expanded)
4961 r = sprintf(buf, "%lu (expanded: %lu)\n",
4962 size >> 10,
4963 trace_buf_size >> 10);
4964 else
4965 r = sprintf(buf, "%lu\n", size >> 10);
4966 } else
4967 r = sprintf(buf, "X\n");
4968 } else
4969 r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10);
4970
4971 mutex_unlock(&trace_types_lock);
4972
4973 ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4974 return ret;
4975}
4976
4977static ssize_t
4978tracing_entries_write(struct file *filp, const char __user *ubuf,
4979 size_t cnt, loff_t *ppos)
4980{
4981 struct inode *inode = file_inode(filp);
4982 struct trace_array *tr = inode->i_private;
4983 unsigned long val;
4984 int ret;
4985
4986 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4987 if (ret)
4988 return ret;
4989
4990
4991 if (!val)
4992 return -EINVAL;
4993
4994
4995 val <<= 10;
4996 ret = tracing_resize_ring_buffer(tr, val, tracing_get_cpu(inode));
4997 if (ret < 0)
4998 return ret;
4999
5000 *ppos += cnt;
5001
5002 return cnt;
5003}
5004
5005static ssize_t
5006tracing_total_entries_read(struct file *filp, char __user *ubuf,
5007 size_t cnt, loff_t *ppos)
5008{
5009 struct trace_array *tr = filp->private_data;
5010 char buf[64];
5011 int r, cpu;
5012 unsigned long size = 0, expanded_size = 0;
5013
5014 mutex_lock(&trace_types_lock);
5015 for_each_tracing_cpu(cpu) {
5016 size += per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10;
5017 if (!ring_buffer_expanded)
5018 expanded_size += trace_buf_size >> 10;
5019 }
5020 if (ring_buffer_expanded)
5021 r = sprintf(buf, "%lu\n", size);
5022 else
5023 r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size);
5024 mutex_unlock(&trace_types_lock);
5025
5026 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5027}
5028
5029static ssize_t
5030tracing_free_buffer_write(struct file *filp, const char __user *ubuf,
5031 size_t cnt, loff_t *ppos)
5032{
5033
5034
5035
5036
5037
5038 *ppos += cnt;
5039
5040 return cnt;
5041}
5042
5043static int
5044tracing_free_buffer_release(struct inode *inode, struct file *filp)
5045{
5046 struct trace_array *tr = inode->i_private;
5047
5048
5049 if (trace_flags & TRACE_ITER_STOP_ON_FREE)
5050 tracer_tracing_off(tr);
5051
5052 tracing_resize_ring_buffer(tr, 0, RING_BUFFER_ALL_CPUS);
5053
5054 trace_array_put(tr);
5055
5056 return 0;
5057}
5058
5059static ssize_t
5060tracing_mark_write(struct file *filp, const char __user *ubuf,
5061 size_t cnt, loff_t *fpos)
5062{
5063 unsigned long addr = (unsigned long)ubuf;
5064 struct trace_array *tr = filp->private_data;
5065 struct ring_buffer_event *event;
5066 struct ring_buffer *buffer;
5067 struct print_entry *entry;
5068 unsigned long irq_flags;
5069 struct page *pages[2];
5070 void *map_page[2];
5071 int nr_pages = 1;
5072 ssize_t written;
5073 int offset;
5074 int size;
5075 int len;
5076 int ret;
5077 int i;
5078
5079 if (tracing_disabled)
5080 return -EINVAL;
5081
5082 if (!(trace_flags & TRACE_ITER_MARKERS))
5083 return -EINVAL;
5084
5085 if (cnt > TRACE_BUF_SIZE)
5086 cnt = TRACE_BUF_SIZE;
5087
5088
5089
5090
5091
5092
5093
5094
5095
5096
5097
5098
5099
5100
5101
5102 BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
5103
5104
5105 if ((addr & PAGE_MASK) != ((addr + cnt) & PAGE_MASK))
5106 nr_pages = 2;
5107
5108 offset = addr & (PAGE_SIZE - 1);
5109 addr &= PAGE_MASK;
5110
5111 ret = get_user_pages_fast(addr, nr_pages, 0, pages);
5112 if (ret < nr_pages) {
5113 while (--ret >= 0)
5114 put_page(pages[ret]);
5115 written = -EFAULT;
5116 goto out;
5117 }
5118
5119 for (i = 0; i < nr_pages; i++)
5120 map_page[i] = kmap_atomic(pages[i]);
5121
5122 local_save_flags(irq_flags);
5123 size = sizeof(*entry) + cnt + 2;
5124 buffer = tr->trace_buffer.buffer;
5125 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
5126 irq_flags, preempt_count());
5127 if (!event) {
5128
5129 written = -EBADF;
5130 goto out_unlock;
5131 }
5132
5133 entry = ring_buffer_event_data(event);
5134 entry->ip = _THIS_IP_;
5135
5136 if (nr_pages == 2) {
5137 len = PAGE_SIZE - offset;
5138 memcpy(&entry->buf, map_page[0] + offset, len);
5139 memcpy(&entry->buf[len], map_page[1], cnt - len);
5140 } else
5141 memcpy(&entry->buf, map_page[0] + offset, cnt);
5142
5143 if (entry->buf[cnt - 1] != '\n') {
5144 entry->buf[cnt] = '\n';
5145 entry->buf[cnt + 1] = '\0';
5146 } else
5147 entry->buf[cnt] = '\0';
5148
5149 __buffer_unlock_commit(buffer, event);
5150
5151 written = cnt;
5152
5153 *fpos += written;
5154
5155 out_unlock:
5156 for (i = nr_pages - 1; i >= 0; i--) {
5157 kunmap_atomic(map_page[i]);
5158 put_page(pages[i]);
5159 }
5160 out:
5161 return written;
5162}
5163
5164static int tracing_clock_show(struct seq_file *m, void *v)
5165{
5166 struct trace_array *tr = m->private;
5167 int i;
5168
5169 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
5170 seq_printf(m,
5171 "%s%s%s%s", i ? " " : "",
5172 i == tr->clock_id ? "[" : "", trace_clocks[i].name,
5173 i == tr->clock_id ? "]" : "");
5174 seq_putc(m, '\n');
5175
5176 return 0;
5177}
5178
5179static int tracing_set_clock(struct trace_array *tr, const char *clockstr)
5180{
5181 int i;
5182
5183 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
5184 if (strcmp(trace_clocks[i].name, clockstr) == 0)
5185 break;
5186 }
5187 if (i == ARRAY_SIZE(trace_clocks))
5188 return -EINVAL;
5189
5190 mutex_lock(&trace_types_lock);
5191
5192 tr->clock_id = i;
5193
5194 ring_buffer_set_clock(tr->trace_buffer.buffer, trace_clocks[i].func);
5195
5196
5197
5198
5199
5200 tracing_reset_online_cpus(&tr->trace_buffer);
5201
5202#ifdef CONFIG_TRACER_MAX_TRACE
5203 if (tr->flags & TRACE_ARRAY_FL_GLOBAL && tr->max_buffer.buffer)
5204 ring_buffer_set_clock(tr->max_buffer.buffer, trace_clocks[i].func);
5205 tracing_reset_online_cpus(&tr->max_buffer);
5206#endif
5207
5208 mutex_unlock(&trace_types_lock);
5209
5210 return 0;
5211}
5212
5213static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
5214 size_t cnt, loff_t *fpos)
5215{
5216 struct seq_file *m = filp->private_data;
5217 struct trace_array *tr = m->private;
5218 char buf[64];
5219 const char *clockstr;
5220 int ret;
5221
5222 if (cnt >= sizeof(buf))
5223 return -EINVAL;
5224
5225 if (copy_from_user(&buf, ubuf, cnt))
5226 return -EFAULT;
5227
5228 buf[cnt] = 0;
5229
5230 clockstr = strstrip(buf);
5231
5232 ret = tracing_set_clock(tr, clockstr);
5233 if (ret)
5234 return ret;
5235
5236 *fpos += cnt;
5237
5238 return cnt;
5239}
5240
5241static int tracing_clock_open(struct inode *inode, struct file *file)
5242{
5243 struct trace_array *tr = inode->i_private;
5244 int ret;
5245
5246 if (tracing_disabled)
5247 return -ENODEV;
5248
5249 if (trace_array_get(tr))
5250 return -ENODEV;
5251
5252 ret = single_open(file, tracing_clock_show, inode->i_private);
5253 if (ret < 0)
5254 trace_array_put(tr);
5255
5256 return ret;
5257}
5258
5259struct ftrace_buffer_info {
5260 struct trace_iterator iter;
5261 void *spare;
5262 unsigned int read;
5263};
5264
5265#ifdef CONFIG_TRACER_SNAPSHOT
5266static int tracing_snapshot_open(struct inode *inode, struct file *file)
5267{
5268 struct trace_array *tr = inode->i_private;
5269 struct trace_iterator *iter;
5270 struct seq_file *m;
5271 int ret = 0;
5272
5273 if (trace_array_get(tr) < 0)
5274 return -ENODEV;
5275
5276 if (file->f_mode & FMODE_READ) {
5277 iter = __tracing_open(inode, file, true);
5278 if (IS_ERR(iter))
5279 ret = PTR_ERR(iter);
5280 } else {
5281
5282 ret = -ENOMEM;
5283 m = kzalloc(sizeof(*m), GFP_KERNEL);
5284 if (!m)
5285 goto out;
5286 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
5287 if (!iter) {
5288 kfree(m);
5289 goto out;
5290 }
5291 ret = 0;
5292
5293 iter->tr = tr;
5294 iter->trace_buffer = &tr->max_buffer;
5295 iter->cpu_file = tracing_get_cpu(inode);
5296 m->private = iter;
5297 file->private_data = m;
5298 }
5299out:
5300 if (ret < 0)
5301 trace_array_put(tr);
5302
5303 return ret;
5304}
5305
5306static ssize_t
5307tracing_snapshot_write(struct file *filp, const char __user *ubuf, size_t cnt,
5308 loff_t *ppos)
5309{
5310 struct seq_file *m = filp->private_data;
5311 struct trace_iterator *iter = m->private;
5312 struct trace_array *tr = iter->tr;
5313 unsigned long val;
5314 int ret;
5315
5316 ret = tracing_update_buffers();
5317 if (ret < 0)
5318 return ret;
5319
5320 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5321 if (ret)
5322 return ret;
5323
5324 mutex_lock(&trace_types_lock);
5325
5326 if (tr->current_trace->use_max_tr) {
5327 ret = -EBUSY;
5328 goto out;
5329 }
5330
5331 switch (val) {
5332 case 0:
5333 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
5334 ret = -EINVAL;
5335 break;
5336 }
5337 if (tr->allocated_snapshot)
5338 free_snapshot(tr);
5339 break;
5340 case 1:
5341
5342#ifndef CONFIG_RING_BUFFER_ALLOW_SWAP
5343 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
5344 ret = -EINVAL;
5345 break;
5346 }
5347#endif
5348 if (!tr->allocated_snapshot) {
5349 ret = alloc_snapshot(tr);
5350 if (ret < 0)
5351 break;
5352 }
5353 local_irq_disable();
5354
5355 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
5356 update_max_tr(tr, current, smp_processor_id());
5357 else
5358 update_max_tr_single(tr, current, iter->cpu_file);
5359 local_irq_enable();
5360 break;
5361 default:
5362 if (tr->allocated_snapshot) {
5363 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
5364 tracing_reset_online_cpus(&tr->max_buffer);
5365 else
5366 tracing_reset(&tr->max_buffer, iter->cpu_file);
5367 }
5368 break;
5369 }
5370
5371 if (ret >= 0) {
5372 *ppos += cnt;
5373 ret = cnt;
5374 }
5375out:
5376 mutex_unlock(&trace_types_lock);
5377 return ret;
5378}
5379
5380static int tracing_snapshot_release(struct inode *inode, struct file *file)
5381{
5382 struct seq_file *m = file->private_data;
5383 int ret;
5384
5385 ret = tracing_release(inode, file);
5386
5387 if (file->f_mode & FMODE_READ)
5388 return ret;
5389
5390
5391 if (m)
5392 kfree(m->private);
5393 kfree(m);
5394
5395 return 0;
5396}
5397
5398static int tracing_buffers_open(struct inode *inode, struct file *filp);
5399static ssize_t tracing_buffers_read(struct file *filp, char __user *ubuf,
5400 size_t count, loff_t *ppos);
5401static int tracing_buffers_release(struct inode *inode, struct file *file);
5402static ssize_t tracing_buffers_splice_read(struct file *file, loff_t *ppos,
5403 struct pipe_inode_info *pipe, size_t len, unsigned int flags);
5404
5405static int snapshot_raw_open(struct inode *inode, struct file *filp)
5406{
5407 struct ftrace_buffer_info *info;
5408 int ret;
5409
5410 ret = tracing_buffers_open(inode, filp);
5411 if (ret < 0)
5412 return ret;
5413
5414 info = filp->private_data;
5415
5416 if (info->iter.trace->use_max_tr) {
5417 tracing_buffers_release(inode, filp);
5418 return -EBUSY;
5419 }
5420
5421 info->iter.snapshot = true;
5422 info->iter.trace_buffer = &info->iter.tr->max_buffer;
5423
5424 return ret;
5425}
5426
5427#endif
5428
5429
5430static const struct file_operations tracing_thresh_fops = {
5431 .open = tracing_open_generic,
5432 .read = tracing_thresh_read,
5433 .write = tracing_thresh_write,
5434 .llseek = generic_file_llseek,
5435};
5436
5437static const struct file_operations tracing_max_lat_fops = {
5438 .open = tracing_open_generic,
5439 .read = tracing_max_lat_read,
5440 .write = tracing_max_lat_write,
5441 .llseek = generic_file_llseek,
5442};
5443
5444static const struct file_operations set_tracer_fops = {
5445 .open = tracing_open_generic,
5446 .read = tracing_set_trace_read,
5447 .write = tracing_set_trace_write,
5448 .llseek = generic_file_llseek,
5449};
5450
5451static const struct file_operations tracing_pipe_fops = {
5452 .open = tracing_open_pipe,
5453 .poll = tracing_poll_pipe,
5454 .read = tracing_read_pipe,
5455 .splice_read = tracing_splice_read_pipe,
5456 .release = tracing_release_pipe,
5457 .llseek = no_llseek,
5458};
5459
5460static const struct file_operations tracing_entries_fops = {
5461 .open = tracing_open_generic_tr,
5462 .read = tracing_entries_read,
5463 .write = tracing_entries_write,
5464 .llseek = generic_file_llseek,
5465 .release = tracing_release_generic_tr,
5466};
5467
5468static const struct file_operations tracing_total_entries_fops = {
5469 .open = tracing_open_generic_tr,
5470 .read = tracing_total_entries_read,
5471 .llseek = generic_file_llseek,
5472 .release = tracing_release_generic_tr,
5473};
5474
5475static const struct file_operations tracing_free_buffer_fops = {
5476 .open = tracing_open_generic_tr,
5477 .write = tracing_free_buffer_write,
5478 .release = tracing_free_buffer_release,
5479};
5480
5481static const struct file_operations tracing_mark_fops = {
5482 .open = tracing_open_generic_tr,
5483 .write = tracing_mark_write,
5484 .llseek = generic_file_llseek,
5485 .release = tracing_release_generic_tr,
5486};
5487
5488static const struct file_operations trace_clock_fops = {
5489 .open = tracing_clock_open,
5490 .read = seq_read,
5491 .llseek = seq_lseek,
5492 .release = tracing_single_release_tr,
5493 .write = tracing_clock_write,
5494};
5495
5496#ifdef CONFIG_TRACER_SNAPSHOT
5497static const struct file_operations snapshot_fops = {
5498 .open = tracing_snapshot_open,
5499 .read = seq_read,
5500 .write = tracing_snapshot_write,
5501 .llseek = tracing_lseek,
5502 .release = tracing_snapshot_release,
5503};
5504
5505static const struct file_operations snapshot_raw_fops = {
5506 .open = snapshot_raw_open,
5507 .read = tracing_buffers_read,
5508 .release = tracing_buffers_release,
5509 .splice_read = tracing_buffers_splice_read,
5510 .llseek = no_llseek,
5511};
5512
5513#endif
5514
5515static int tracing_buffers_open(struct inode *inode, struct file *filp)
5516{
5517 struct trace_array *tr = inode->i_private;
5518 struct ftrace_buffer_info *info;
5519 int ret;
5520
5521 if (tracing_disabled)
5522 return -ENODEV;
5523
5524 if (trace_array_get(tr) < 0)
5525 return -ENODEV;
5526
5527 info = kzalloc(sizeof(*info), GFP_KERNEL);
5528 if (!info) {
5529 trace_array_put(tr);
5530 return -ENOMEM;
5531 }
5532
5533 mutex_lock(&trace_types_lock);
5534
5535 info->iter.tr = tr;
5536 info->iter.cpu_file = tracing_get_cpu(inode);
5537 info->iter.trace = tr->current_trace;
5538 info->iter.trace_buffer = &tr->trace_buffer;
5539 info->spare = NULL;
5540
5541 info->read = (unsigned int)-1;
5542
5543 filp->private_data = info;
5544
5545 tr->current_trace->ref++;
5546
5547 mutex_unlock(&trace_types_lock);
5548
5549 ret = nonseekable_open(inode, filp);
5550 if (ret < 0)
5551 trace_array_put(tr);
5552
5553 return ret;
5554}
5555
5556static unsigned int
5557tracing_buffers_poll(struct file *filp, poll_table *poll_table)
5558{
5559 struct ftrace_buffer_info *info = filp->private_data;
5560 struct trace_iterator *iter = &info->iter;
5561
5562 return trace_poll(iter, filp, poll_table);
5563}
5564
5565static ssize_t
5566tracing_buffers_read(struct file *filp, char __user *ubuf,
5567 size_t count, loff_t *ppos)
5568{
5569 struct ftrace_buffer_info *info = filp->private_data;
5570 struct trace_iterator *iter = &info->iter;
5571 ssize_t ret;
5572 ssize_t size;
5573
5574 if (!count)
5575 return 0;
5576
5577#ifdef CONFIG_TRACER_MAX_TRACE
5578 if (iter->snapshot && iter->tr->current_trace->use_max_tr)
5579 return -EBUSY;
5580#endif
5581
5582 if (!info->spare)
5583 info->spare = ring_buffer_alloc_read_page(iter->trace_buffer->buffer,
5584 iter->cpu_file);
5585 if (!info->spare)
5586 return -ENOMEM;
5587
5588
5589 if (info->read < PAGE_SIZE)
5590 goto read;
5591
5592 again:
5593 trace_access_lock(iter->cpu_file);
5594 ret = ring_buffer_read_page(iter->trace_buffer->buffer,
5595 &info->spare,
5596 count,
5597 iter->cpu_file, 0);
5598 trace_access_unlock(iter->cpu_file);
5599
5600 if (ret < 0) {
5601 if (trace_empty(iter)) {
5602 if ((filp->f_flags & O_NONBLOCK))
5603 return -EAGAIN;
5604
5605 ret = wait_on_pipe(iter, false);
5606 if (ret)
5607 return ret;
5608
5609 goto again;
5610 }
5611 return 0;
5612 }
5613
5614 info->read = 0;
5615 read:
5616 size = PAGE_SIZE - info->read;
5617 if (size > count)
5618 size = count;
5619
5620 ret = copy_to_user(ubuf, info->spare + info->read, size);
5621 if (ret == size)
5622 return -EFAULT;
5623
5624 size -= ret;
5625
5626 *ppos += size;
5627 info->read += size;
5628
5629 return size;
5630}
5631
5632static int tracing_buffers_release(struct inode *inode, struct file *file)
5633{
5634 struct ftrace_buffer_info *info = file->private_data;
5635 struct trace_iterator *iter = &info->iter;
5636
5637 mutex_lock(&trace_types_lock);
5638
5639 iter->tr->current_trace->ref--;
5640
5641 __trace_array_put(iter->tr);
5642
5643 if (info->spare)
5644 ring_buffer_free_read_page(iter->trace_buffer->buffer, info->spare);
5645 kfree(info);
5646
5647 mutex_unlock(&trace_types_lock);
5648
5649 return 0;
5650}
5651
5652struct buffer_ref {
5653 struct ring_buffer *buffer;
5654 void *page;
5655 int ref;
5656};
5657
5658static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
5659 struct pipe_buffer *buf)
5660{
5661 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
5662
5663 if (--ref->ref)
5664 return;
5665
5666 ring_buffer_free_read_page(ref->buffer, ref->page);
5667 kfree(ref);
5668 buf->private = 0;
5669}
5670
5671static void buffer_pipe_buf_get(struct pipe_inode_info *pipe,
5672 struct pipe_buffer *buf)
5673{
5674 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
5675
5676 ref->ref++;
5677}
5678
5679
5680static const struct pipe_buf_operations buffer_pipe_buf_ops = {
5681 .can_merge = 0,
5682 .confirm = generic_pipe_buf_confirm,
5683 .release = buffer_pipe_buf_release,
5684 .steal = generic_pipe_buf_steal,
5685 .get = buffer_pipe_buf_get,
5686};
5687
5688
5689
5690
5691
5692static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
5693{
5694 struct buffer_ref *ref =
5695 (struct buffer_ref *)spd->partial[i].private;
5696
5697 if (--ref->ref)
5698 return;
5699
5700 ring_buffer_free_read_page(ref->buffer, ref->page);
5701 kfree(ref);
5702 spd->partial[i].private = 0;
5703}
5704
5705static ssize_t
5706tracing_buffers_splice_read(struct file *file, loff_t *ppos,
5707 struct pipe_inode_info *pipe, size_t len,
5708 unsigned int flags)
5709{
5710 struct ftrace_buffer_info *info = file->private_data;
5711 struct trace_iterator *iter = &info->iter;
5712 struct partial_page partial_def[PIPE_DEF_BUFFERS];
5713 struct page *pages_def[PIPE_DEF_BUFFERS];
5714 struct splice_pipe_desc spd = {
5715 .pages = pages_def,
5716 .partial = partial_def,
5717 .nr_pages_max = PIPE_DEF_BUFFERS,
5718 .flags = flags,
5719 .ops = &buffer_pipe_buf_ops,
5720 .spd_release = buffer_spd_release,
5721 };
5722 struct buffer_ref *ref;
5723 int entries, size, i;
5724 ssize_t ret = 0;
5725
5726#ifdef CONFIG_TRACER_MAX_TRACE
5727 if (iter->snapshot && iter->tr->current_trace->use_max_tr)
5728 return -EBUSY;
5729#endif
5730
5731 if (splice_grow_spd(pipe, &spd))
5732 return -ENOMEM;
5733
5734 if (*ppos & (PAGE_SIZE - 1))
5735 return -EINVAL;
5736
5737 if (len & (PAGE_SIZE - 1)) {
5738 if (len < PAGE_SIZE)
5739 return -EINVAL;
5740 len &= PAGE_MASK;
5741 }
5742
5743 again:
5744 trace_access_lock(iter->cpu_file);
5745 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
5746
5747 for (i = 0; i < spd.nr_pages_max && len && entries; i++, len -= PAGE_SIZE) {
5748 struct page *page;
5749 int r;
5750
5751 ref = kzalloc(sizeof(*ref), GFP_KERNEL);
5752 if (!ref) {
5753 ret = -ENOMEM;
5754 break;
5755 }
5756
5757 ref->ref = 1;
5758 ref->buffer = iter->trace_buffer->buffer;
5759 ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file);
5760 if (!ref->page) {
5761 ret = -ENOMEM;
5762 kfree(ref);
5763 break;
5764 }
5765
5766 r = ring_buffer_read_page(ref->buffer, &ref->page,
5767 len, iter->cpu_file, 1);
5768 if (r < 0) {
5769 ring_buffer_free_read_page(ref->buffer, ref->page);
5770 kfree(ref);
5771 break;
5772 }
5773
5774
5775
5776
5777
5778 size = ring_buffer_page_len(ref->page);
5779 if (size < PAGE_SIZE)
5780 memset(ref->page + size, 0, PAGE_SIZE - size);
5781
5782 page = virt_to_page(ref->page);
5783
5784 spd.pages[i] = page;
5785 spd.partial[i].len = PAGE_SIZE;
5786 spd.partial[i].offset = 0;
5787 spd.partial[i].private = (unsigned long)ref;
5788 spd.nr_pages++;
5789 *ppos += PAGE_SIZE;
5790
5791 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
5792 }
5793
5794 trace_access_unlock(iter->cpu_file);
5795 spd.nr_pages = i;
5796
5797
5798 if (!spd.nr_pages) {
5799 if (ret)
5800 return ret;
5801
5802 if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK))
5803 return -EAGAIN;
5804
5805 ret = wait_on_pipe(iter, true);
5806 if (ret)
5807 return ret;
5808
5809 goto again;
5810 }
5811
5812 ret = splice_to_pipe(pipe, &spd);
5813 splice_shrink_spd(&spd);
5814
5815 return ret;
5816}
5817
5818static const struct file_operations tracing_buffers_fops = {
5819 .open = tracing_buffers_open,
5820 .read = tracing_buffers_read,
5821 .poll = tracing_buffers_poll,
5822 .release = tracing_buffers_release,
5823 .splice_read = tracing_buffers_splice_read,
5824 .llseek = no_llseek,
5825};
5826
5827static ssize_t
5828tracing_stats_read(struct file *filp, char __user *ubuf,
5829 size_t count, loff_t *ppos)
5830{
5831 struct inode *inode = file_inode(filp);
5832 struct trace_array *tr = inode->i_private;
5833 struct trace_buffer *trace_buf = &tr->trace_buffer;
5834 int cpu = tracing_get_cpu(inode);
5835 struct trace_seq *s;
5836 unsigned long cnt;
5837 unsigned long long t;
5838 unsigned long usec_rem;
5839
5840 s = kmalloc(sizeof(*s), GFP_KERNEL);
5841 if (!s)
5842 return -ENOMEM;
5843
5844 trace_seq_init(s);
5845
5846 cnt = ring_buffer_entries_cpu(trace_buf->buffer, cpu);
5847 trace_seq_printf(s, "entries: %ld\n", cnt);
5848
5849 cnt = ring_buffer_overrun_cpu(trace_buf->buffer, cpu);
5850 trace_seq_printf(s, "overrun: %ld\n", cnt);
5851
5852 cnt = ring_buffer_commit_overrun_cpu(trace_buf->buffer, cpu);
5853 trace_seq_printf(s, "commit overrun: %ld\n", cnt);
5854
5855 cnt = ring_buffer_bytes_cpu(trace_buf->buffer, cpu);
5856 trace_seq_printf(s, "bytes: %ld\n", cnt);
5857
5858 if (trace_clocks[tr->clock_id].in_ns) {
5859
5860 t = ns2usecs(ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
5861 usec_rem = do_div(t, USEC_PER_SEC);
5862 trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n",
5863 t, usec_rem);
5864
5865 t = ns2usecs(ring_buffer_time_stamp(trace_buf->buffer, cpu));
5866 usec_rem = do_div(t, USEC_PER_SEC);
5867 trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem);
5868 } else {
5869
5870 trace_seq_printf(s, "oldest event ts: %llu\n",
5871 ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
5872
5873 trace_seq_printf(s, "now ts: %llu\n",
5874 ring_buffer_time_stamp(trace_buf->buffer, cpu));
5875 }
5876
5877 cnt = ring_buffer_dropped_events_cpu(trace_buf->buffer, cpu);
5878 trace_seq_printf(s, "dropped events: %ld\n", cnt);
5879
5880 cnt = ring_buffer_read_events_cpu(trace_buf->buffer, cpu);
5881 trace_seq_printf(s, "read events: %ld\n", cnt);
5882
5883 count = simple_read_from_buffer(ubuf, count, ppos,
5884 s->buffer, trace_seq_used(s));
5885
5886 kfree(s);
5887
5888 return count;
5889}
5890
5891static const struct file_operations tracing_stats_fops = {
5892 .open = tracing_open_generic_tr,
5893 .read = tracing_stats_read,
5894 .llseek = generic_file_llseek,
5895 .release = tracing_release_generic_tr,
5896};
5897
5898#ifdef CONFIG_DYNAMIC_FTRACE
5899
5900int __weak ftrace_arch_read_dyn_info(char *buf, int size)
5901{
5902 return 0;
5903}
5904
5905static ssize_t
5906tracing_read_dyn_info(struct file *filp, char __user *ubuf,
5907 size_t cnt, loff_t *ppos)
5908{
5909 static char ftrace_dyn_info_buffer[1024];
5910 static DEFINE_MUTEX(dyn_info_mutex);
5911 unsigned long *p = filp->private_data;
5912 char *buf = ftrace_dyn_info_buffer;
5913 int size = ARRAY_SIZE(ftrace_dyn_info_buffer);
5914 int r;
5915
5916 mutex_lock(&dyn_info_mutex);
5917 r = sprintf(buf, "%ld ", *p);
5918
5919 r += ftrace_arch_read_dyn_info(buf+r, (size-1)-r);
5920 buf[r++] = '\n';
5921
5922 r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5923
5924 mutex_unlock(&dyn_info_mutex);
5925
5926 return r;
5927}
5928
5929static const struct file_operations tracing_dyn_info_fops = {
5930 .open = tracing_open_generic,
5931 .read = tracing_read_dyn_info,
5932 .llseek = generic_file_llseek,
5933};
5934#endif
5935
5936#if defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE)
5937static void
5938ftrace_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
5939{
5940 tracing_snapshot();
5941}
5942
5943static void
5944ftrace_count_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
5945{
5946 unsigned long *count = (long *)data;
5947
5948 if (!*count)
5949 return;
5950
5951 if (*count != -1)
5952 (*count)--;
5953
5954 tracing_snapshot();
5955}
5956
5957static int
5958ftrace_snapshot_print(struct seq_file *m, unsigned long ip,
5959 struct ftrace_probe_ops *ops, void *data)
5960{
5961 long count = (long)data;
5962
5963 seq_printf(m, "%ps:", (void *)ip);
5964
5965 seq_puts(m, "snapshot");
5966
5967 if (count == -1)
5968 seq_puts(m, ":unlimited\n");
5969 else
5970 seq_printf(m, ":count=%ld\n", count);
5971
5972 return 0;
5973}
5974
5975static struct ftrace_probe_ops snapshot_probe_ops = {
5976 .func = ftrace_snapshot,
5977 .print = ftrace_snapshot_print,
5978};
5979
5980static struct ftrace_probe_ops snapshot_count_probe_ops = {
5981 .func = ftrace_count_snapshot,
5982 .print = ftrace_snapshot_print,
5983};
5984
5985static int
5986ftrace_trace_snapshot_callback(struct ftrace_hash *hash,
5987 char *glob, char *cmd, char *param, int enable)
5988{
5989 struct ftrace_probe_ops *ops;
5990 void *count = (void *)-1;
5991 char *number;
5992 int ret;
5993
5994
5995 if (!enable)
5996 return -EINVAL;
5997
5998 ops = param ? &snapshot_count_probe_ops : &snapshot_probe_ops;
5999
6000 if (glob[0] == '!') {
6001 unregister_ftrace_function_probe_func(glob+1, ops);
6002 return 0;
6003 }
6004
6005 if (!param)
6006 goto out_reg;
6007
6008 number = strsep(¶m, ":");
6009
6010 if (!strlen(number))
6011 goto out_reg;
6012
6013
6014
6015
6016
6017 ret = kstrtoul(number, 0, (unsigned long *)&count);
6018 if (ret)
6019 return ret;
6020
6021 out_reg:
6022 ret = register_ftrace_function_probe(glob, ops, count);
6023
6024 if (ret >= 0)
6025 alloc_snapshot(&global_trace);
6026
6027 return ret < 0 ? ret : 0;
6028}
6029
6030static struct ftrace_func_command ftrace_snapshot_cmd = {
6031 .name = "snapshot",
6032 .func = ftrace_trace_snapshot_callback,
6033};
6034
6035static __init int register_snapshot_cmd(void)
6036{
6037 return register_ftrace_command(&ftrace_snapshot_cmd);
6038}
6039#else
6040static inline __init int register_snapshot_cmd(void) { return 0; }
6041#endif
6042
6043static struct dentry *tracing_get_dentry(struct trace_array *tr)
6044{
6045 if (WARN_ON(!tr->dir))
6046 return ERR_PTR(-ENODEV);
6047
6048
6049 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
6050 return NULL;
6051
6052
6053 return tr->dir;
6054}
6055
6056static struct dentry *tracing_dentry_percpu(struct trace_array *tr, int cpu)
6057{
6058 struct dentry *d_tracer;
6059
6060 if (tr->percpu_dir)
6061 return tr->percpu_dir;
6062
6063 d_tracer = tracing_get_dentry(tr);
6064 if (IS_ERR(d_tracer))
6065 return NULL;
6066
6067 tr->percpu_dir = tracefs_create_dir("per_cpu", d_tracer);
6068
6069 WARN_ONCE(!tr->percpu_dir,
6070 "Could not create tracefs directory 'per_cpu/%d'\n", cpu);
6071
6072 return tr->percpu_dir;
6073}
6074
6075static struct dentry *
6076trace_create_cpu_file(const char *name, umode_t mode, struct dentry *parent,
6077 void *data, long cpu, const struct file_operations *fops)
6078{
6079 struct dentry *ret = trace_create_file(name, mode, parent, data, fops);
6080
6081 if (ret)
6082 d_inode(ret)->i_cdev = (void *)(cpu + 1);
6083 return ret;
6084}
6085
6086static void
6087tracing_init_tracefs_percpu(struct trace_array *tr, long cpu)
6088{
6089 struct dentry *d_percpu = tracing_dentry_percpu(tr, cpu);
6090 struct dentry *d_cpu;
6091 char cpu_dir[30];
6092
6093 if (!d_percpu)
6094 return;
6095
6096 snprintf(cpu_dir, 30, "cpu%ld", cpu);
6097 d_cpu = tracefs_create_dir(cpu_dir, d_percpu);
6098 if (!d_cpu) {
6099 pr_warning("Could not create tracefs '%s' entry\n", cpu_dir);
6100 return;
6101 }
6102
6103
6104 trace_create_cpu_file("trace_pipe", 0444, d_cpu,
6105 tr, cpu, &tracing_pipe_fops);
6106
6107
6108 trace_create_cpu_file("trace", 0644, d_cpu,
6109 tr, cpu, &tracing_fops);
6110
6111 trace_create_cpu_file("trace_pipe_raw", 0444, d_cpu,
6112 tr, cpu, &tracing_buffers_fops);
6113
6114 trace_create_cpu_file("stats", 0444, d_cpu,
6115 tr, cpu, &tracing_stats_fops);
6116
6117 trace_create_cpu_file("buffer_size_kb", 0444, d_cpu,
6118 tr, cpu, &tracing_entries_fops);
6119
6120#ifdef CONFIG_TRACER_SNAPSHOT
6121 trace_create_cpu_file("snapshot", 0644, d_cpu,
6122 tr, cpu, &snapshot_fops);
6123
6124 trace_create_cpu_file("snapshot_raw", 0444, d_cpu,
6125 tr, cpu, &snapshot_raw_fops);
6126#endif
6127}
6128
6129#ifdef CONFIG_FTRACE_SELFTEST
6130
6131#include "trace_selftest.c"
6132#endif
6133
6134struct trace_option_dentry {
6135 struct tracer_opt *opt;
6136 struct tracer_flags *flags;
6137 struct trace_array *tr;
6138 struct dentry *entry;
6139};
6140
6141static ssize_t
6142trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
6143 loff_t *ppos)
6144{
6145 struct trace_option_dentry *topt = filp->private_data;
6146 char *buf;
6147
6148 if (topt->flags->val & topt->opt->bit)
6149 buf = "1\n";
6150 else
6151 buf = "0\n";
6152
6153 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
6154}
6155
6156static ssize_t
6157trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
6158 loff_t *ppos)
6159{
6160 struct trace_option_dentry *topt = filp->private_data;
6161 unsigned long val;
6162 int ret;
6163
6164 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6165 if (ret)
6166 return ret;
6167
6168 if (val != 0 && val != 1)
6169 return -EINVAL;
6170
6171 if (!!(topt->flags->val & topt->opt->bit) != val) {
6172 mutex_lock(&trace_types_lock);
6173 ret = __set_tracer_option(topt->tr, topt->flags,
6174 topt->opt, !val);
6175 mutex_unlock(&trace_types_lock);
6176 if (ret)
6177 return ret;
6178 }
6179
6180 *ppos += cnt;
6181
6182 return cnt;
6183}
6184
6185
6186static const struct file_operations trace_options_fops = {
6187 .open = tracing_open_generic,
6188 .read = trace_options_read,
6189 .write = trace_options_write,
6190 .llseek = generic_file_llseek,
6191};
6192
6193static ssize_t
6194trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
6195 loff_t *ppos)
6196{
6197 long index = (long)filp->private_data;
6198 char *buf;
6199
6200 if (trace_flags & (1 << index))
6201 buf = "1\n";
6202 else
6203 buf = "0\n";
6204
6205 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
6206}
6207
6208static ssize_t
6209trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
6210 loff_t *ppos)
6211{
6212 struct trace_array *tr = &global_trace;
6213 long index = (long)filp->private_data;
6214 unsigned long val;
6215 int ret;
6216
6217 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6218 if (ret)
6219 return ret;
6220
6221 if (val != 0 && val != 1)
6222 return -EINVAL;
6223
6224 mutex_lock(&trace_types_lock);
6225 ret = set_tracer_flag(tr, 1 << index, val);
6226 mutex_unlock(&trace_types_lock);
6227
6228 if (ret < 0)
6229 return ret;
6230
6231 *ppos += cnt;
6232
6233 return cnt;
6234}
6235
6236static const struct file_operations trace_options_core_fops = {
6237 .open = tracing_open_generic,
6238 .read = trace_options_core_read,
6239 .write = trace_options_core_write,
6240 .llseek = generic_file_llseek,
6241};
6242
6243struct dentry *trace_create_file(const char *name,
6244 umode_t mode,
6245 struct dentry *parent,
6246 void *data,
6247 const struct file_operations *fops)
6248{
6249 struct dentry *ret;
6250
6251 ret = tracefs_create_file(name, mode, parent, data, fops);
6252 if (!ret)
6253 pr_warning("Could not create tracefs '%s' entry\n", name);
6254
6255 return ret;
6256}
6257
6258
6259static struct dentry *trace_options_init_dentry(struct trace_array *tr)
6260{
6261 struct dentry *d_tracer;
6262
6263 if (tr->options)
6264 return tr->options;
6265
6266 d_tracer = tracing_get_dentry(tr);
6267 if (IS_ERR(d_tracer))
6268 return NULL;
6269
6270 tr->options = tracefs_create_dir("options", d_tracer);
6271 if (!tr->options) {
6272 pr_warning("Could not create tracefs directory 'options'\n");
6273 return NULL;
6274 }
6275
6276 return tr->options;
6277}
6278
6279static void
6280create_trace_option_file(struct trace_array *tr,
6281 struct trace_option_dentry *topt,
6282 struct tracer_flags *flags,
6283 struct tracer_opt *opt)
6284{
6285 struct dentry *t_options;
6286
6287 t_options = trace_options_init_dentry(tr);
6288 if (!t_options)
6289 return;
6290
6291 topt->flags = flags;
6292 topt->opt = opt;
6293 topt->tr = tr;
6294
6295 topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
6296 &trace_options_fops);
6297
6298}
6299
6300static struct trace_option_dentry *
6301create_trace_option_files(struct trace_array *tr, struct tracer *tracer)
6302{
6303 struct trace_option_dentry *topts;
6304 struct tracer_flags *flags;
6305 struct tracer_opt *opts;
6306 int cnt;
6307
6308 if (!tracer)
6309 return NULL;
6310
6311 flags = tracer->flags;
6312
6313 if (!flags || !flags->opts)
6314 return NULL;
6315
6316 opts = flags->opts;
6317
6318 for (cnt = 0; opts[cnt].name; cnt++)
6319 ;
6320
6321 topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
6322 if (!topts)
6323 return NULL;
6324
6325 for (cnt = 0; opts[cnt].name; cnt++)
6326 create_trace_option_file(tr, &topts[cnt], flags,
6327 &opts[cnt]);
6328
6329 return topts;
6330}
6331
6332static void
6333destroy_trace_option_files(struct trace_option_dentry *topts)
6334{
6335 int cnt;
6336
6337 if (!topts)
6338 return;
6339
6340 for (cnt = 0; topts[cnt].opt; cnt++)
6341 tracefs_remove(topts[cnt].entry);
6342
6343 kfree(topts);
6344}
6345
6346static struct dentry *
6347create_trace_option_core_file(struct trace_array *tr,
6348 const char *option, long index)
6349{
6350 struct dentry *t_options;
6351
6352 t_options = trace_options_init_dentry(tr);
6353 if (!t_options)
6354 return NULL;
6355
6356 return trace_create_file(option, 0644, t_options, (void *)index,
6357 &trace_options_core_fops);
6358}
6359
6360static __init void create_trace_options_dir(struct trace_array *tr)
6361{
6362 struct dentry *t_options;
6363 int i;
6364
6365 t_options = trace_options_init_dentry(tr);
6366 if (!t_options)
6367 return;
6368
6369 for (i = 0; trace_options[i]; i++)
6370 create_trace_option_core_file(tr, trace_options[i], i);
6371}
6372
6373static ssize_t
6374rb_simple_read(struct file *filp, char __user *ubuf,
6375 size_t cnt, loff_t *ppos)
6376{
6377 struct trace_array *tr = filp->private_data;
6378 char buf[64];
6379 int r;
6380
6381 r = tracer_tracing_is_on(tr);
6382 r = sprintf(buf, "%d\n", r);
6383
6384 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
6385}
6386
6387static ssize_t
6388rb_simple_write(struct file *filp, const char __user *ubuf,
6389 size_t cnt, loff_t *ppos)
6390{
6391 struct trace_array *tr = filp->private_data;
6392 struct ring_buffer *buffer = tr->trace_buffer.buffer;
6393 unsigned long val;
6394 int ret;
6395
6396 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6397 if (ret)
6398 return ret;
6399
6400 if (buffer) {
6401 mutex_lock(&trace_types_lock);
6402 if (val) {
6403 tracer_tracing_on(tr);
6404 if (tr->current_trace->start)
6405 tr->current_trace->start(tr);
6406 } else {
6407 tracer_tracing_off(tr);
6408 if (tr->current_trace->stop)
6409 tr->current_trace->stop(tr);
6410 }
6411 mutex_unlock(&trace_types_lock);
6412 }
6413
6414 (*ppos)++;
6415
6416 return cnt;
6417}
6418
6419static const struct file_operations rb_simple_fops = {
6420 .open = tracing_open_generic_tr,
6421 .read = rb_simple_read,
6422 .write = rb_simple_write,
6423 .release = tracing_release_generic_tr,
6424 .llseek = default_llseek,
6425};
6426
6427struct dentry *trace_instance_dir;
6428
6429static void
6430init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer);
6431
6432static int
6433allocate_trace_buffer(struct trace_array *tr, struct trace_buffer *buf, int size)
6434{
6435 enum ring_buffer_flags rb_flags;
6436
6437 rb_flags = trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;
6438
6439 buf->tr = tr;
6440
6441 buf->buffer = ring_buffer_alloc(size, rb_flags);
6442 if (!buf->buffer)
6443 return -ENOMEM;
6444
6445 buf->data = alloc_percpu(struct trace_array_cpu);
6446 if (!buf->data) {
6447 ring_buffer_free(buf->buffer);
6448 return -ENOMEM;
6449 }
6450
6451
6452 set_buffer_entries(&tr->trace_buffer,
6453 ring_buffer_size(tr->trace_buffer.buffer, 0));
6454
6455 return 0;
6456}
6457
6458static int allocate_trace_buffers(struct trace_array *tr, int size)
6459{
6460 int ret;
6461
6462 ret = allocate_trace_buffer(tr, &tr->trace_buffer, size);
6463 if (ret)
6464 return ret;
6465
6466#ifdef CONFIG_TRACER_MAX_TRACE
6467 ret = allocate_trace_buffer(tr, &tr->max_buffer,
6468 allocate_snapshot ? size : 1);
6469 if (WARN_ON(ret)) {
6470 ring_buffer_free(tr->trace_buffer.buffer);
6471 free_percpu(tr->trace_buffer.data);
6472 return -ENOMEM;
6473 }
6474 tr->allocated_snapshot = allocate_snapshot;
6475
6476
6477
6478
6479
6480 allocate_snapshot = false;
6481#endif
6482 return 0;
6483}
6484
6485static void free_trace_buffer(struct trace_buffer *buf)
6486{
6487 if (buf->buffer) {
6488 ring_buffer_free(buf->buffer);
6489 buf->buffer = NULL;
6490 free_percpu(buf->data);
6491 buf->data = NULL;
6492 }
6493}
6494
6495static void free_trace_buffers(struct trace_array *tr)
6496{
6497 if (!tr)
6498 return;
6499
6500 free_trace_buffer(&tr->trace_buffer);
6501
6502#ifdef CONFIG_TRACER_MAX_TRACE
6503 free_trace_buffer(&tr->max_buffer);
6504#endif
6505}
6506
6507static int instance_mkdir(const char *name)
6508{
6509 struct trace_array *tr;
6510 int ret;
6511
6512 mutex_lock(&trace_types_lock);
6513
6514 ret = -EEXIST;
6515 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
6516 if (tr->name && strcmp(tr->name, name) == 0)
6517 goto out_unlock;
6518 }
6519
6520 ret = -ENOMEM;
6521 tr = kzalloc(sizeof(*tr), GFP_KERNEL);
6522 if (!tr)
6523 goto out_unlock;
6524
6525 tr->name = kstrdup(name, GFP_KERNEL);
6526 if (!tr->name)
6527 goto out_free_tr;
6528
6529 if (!alloc_cpumask_var(&tr->tracing_cpumask, GFP_KERNEL))
6530 goto out_free_tr;
6531
6532 cpumask_copy(tr->tracing_cpumask, cpu_all_mask);
6533
6534 raw_spin_lock_init(&tr->start_lock);
6535
6536 tr->max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
6537
6538 tr->current_trace = &nop_trace;
6539
6540 INIT_LIST_HEAD(&tr->systems);
6541 INIT_LIST_HEAD(&tr->events);
6542
6543 if (allocate_trace_buffers(tr, trace_buf_size) < 0)
6544 goto out_free_tr;
6545
6546 tr->dir = tracefs_create_dir(name, trace_instance_dir);
6547 if (!tr->dir)
6548 goto out_free_tr;
6549
6550 ret = event_trace_add_tracer(tr->dir, tr);
6551 if (ret) {
6552 tracefs_remove_recursive(tr->dir);
6553 goto out_free_tr;
6554 }
6555
6556 init_tracer_tracefs(tr, tr->dir);
6557
6558 list_add(&tr->list, &ftrace_trace_arrays);
6559
6560 mutex_unlock(&trace_types_lock);
6561
6562 return 0;
6563
6564 out_free_tr:
6565 free_trace_buffers(tr);
6566 free_cpumask_var(tr->tracing_cpumask);
6567 kfree(tr->name);
6568 kfree(tr);
6569
6570 out_unlock:
6571 mutex_unlock(&trace_types_lock);
6572
6573 return ret;
6574
6575}
6576
6577static int instance_rmdir(const char *name)
6578{
6579 struct trace_array *tr;
6580 int found = 0;
6581 int ret;
6582
6583 mutex_lock(&trace_types_lock);
6584
6585 ret = -ENODEV;
6586 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
6587 if (tr->name && strcmp(tr->name, name) == 0) {
6588 found = 1;
6589 break;
6590 }
6591 }
6592 if (!found)
6593 goto out_unlock;
6594
6595 ret = -EBUSY;
6596 if (tr->ref || (tr->current_trace && tr->current_trace->ref))
6597 goto out_unlock;
6598
6599 list_del(&tr->list);
6600
6601 tracing_set_nop(tr);
6602 event_trace_del_tracer(tr);
6603 ftrace_destroy_function_files(tr);
6604 debugfs_remove_recursive(tr->dir);
6605 free_trace_buffers(tr);
6606
6607 kfree(tr->name);
6608 kfree(tr);
6609
6610 ret = 0;
6611
6612 out_unlock:
6613 mutex_unlock(&trace_types_lock);
6614
6615 return ret;
6616}
6617
6618static __init void create_trace_instances(struct dentry *d_tracer)
6619{
6620 trace_instance_dir = tracefs_create_instance_dir("instances", d_tracer,
6621 instance_mkdir,
6622 instance_rmdir);
6623 if (WARN_ON(!trace_instance_dir))
6624 return;
6625}
6626
6627static void
6628init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer)
6629{
6630 int cpu;
6631
6632 trace_create_file("available_tracers", 0444, d_tracer,
6633 tr, &show_traces_fops);
6634
6635 trace_create_file("current_tracer", 0644, d_tracer,
6636 tr, &set_tracer_fops);
6637
6638 trace_create_file("tracing_cpumask", 0644, d_tracer,
6639 tr, &tracing_cpumask_fops);
6640
6641 trace_create_file("trace_options", 0644, d_tracer,
6642 tr, &tracing_iter_fops);
6643
6644 trace_create_file("trace", 0644, d_tracer,
6645 tr, &tracing_fops);
6646
6647 trace_create_file("trace_pipe", 0444, d_tracer,
6648 tr, &tracing_pipe_fops);
6649
6650 trace_create_file("buffer_size_kb", 0644, d_tracer,
6651 tr, &tracing_entries_fops);
6652
6653 trace_create_file("buffer_total_size_kb", 0444, d_tracer,
6654 tr, &tracing_total_entries_fops);
6655
6656 trace_create_file("free_buffer", 0200, d_tracer,
6657 tr, &tracing_free_buffer_fops);
6658
6659 trace_create_file("trace_marker", 0220, d_tracer,
6660 tr, &tracing_mark_fops);
6661
6662 trace_create_file("trace_clock", 0644, d_tracer, tr,
6663 &trace_clock_fops);
6664
6665 trace_create_file("tracing_on", 0644, d_tracer,
6666 tr, &rb_simple_fops);
6667
6668#ifdef CONFIG_TRACER_MAX_TRACE
6669 trace_create_file("tracing_max_latency", 0644, d_tracer,
6670 &tr->max_latency, &tracing_max_lat_fops);
6671#endif
6672
6673 if (ftrace_create_function_files(tr, d_tracer))
6674 WARN(1, "Could not allocate function filter files");
6675
6676#ifdef CONFIG_TRACER_SNAPSHOT
6677 trace_create_file("snapshot", 0644, d_tracer,
6678 tr, &snapshot_fops);
6679#endif
6680
6681 for_each_tracing_cpu(cpu)
6682 tracing_init_tracefs_percpu(tr, cpu);
6683
6684}
6685
6686static struct vfsmount *trace_automount(void *ingore)
6687{
6688 struct vfsmount *mnt;
6689 struct file_system_type *type;
6690
6691
6692
6693
6694
6695
6696 type = get_fs_type("tracefs");
6697 if (!type)
6698 return NULL;
6699 mnt = vfs_kern_mount(type, 0, "tracefs", NULL);
6700 put_filesystem(type);
6701 if (IS_ERR(mnt))
6702 return NULL;
6703 mntget(mnt);
6704
6705 return mnt;
6706}
6707
6708
6709
6710
6711
6712
6713
6714
6715struct dentry *tracing_init_dentry(void)
6716{
6717 struct trace_array *tr = &global_trace;
6718
6719
6720 if (tr->dir)
6721 return NULL;
6722
6723 if (WARN_ON(!debugfs_initialized()))
6724 return ERR_PTR(-ENODEV);
6725
6726
6727
6728
6729
6730
6731
6732 tr->dir = debugfs_create_automount("tracing", NULL,
6733 trace_automount, NULL);
6734 if (!tr->dir) {
6735 pr_warn_once("Could not create debugfs directory 'tracing'\n");
6736 return ERR_PTR(-ENOMEM);
6737 }
6738
6739 return NULL;
6740}
6741
6742extern struct trace_enum_map *__start_ftrace_enum_maps[];
6743extern struct trace_enum_map *__stop_ftrace_enum_maps[];
6744
6745static void __init trace_enum_init(void)
6746{
6747 int len;
6748
6749 len = __stop_ftrace_enum_maps - __start_ftrace_enum_maps;
6750 trace_insert_enum_map(NULL, __start_ftrace_enum_maps, len);
6751}
6752
6753#ifdef CONFIG_MODULES
6754static void trace_module_add_enums(struct module *mod)
6755{
6756 if (!mod->num_trace_enums)
6757 return;
6758
6759
6760
6761
6762
6763 if (trace_module_has_bad_taint(mod))
6764 return;
6765
6766 trace_insert_enum_map(mod, mod->trace_enums, mod->num_trace_enums);
6767}
6768
6769#ifdef CONFIG_TRACE_ENUM_MAP_FILE
6770static void trace_module_remove_enums(struct module *mod)
6771{
6772 union trace_enum_map_item *map;
6773 union trace_enum_map_item **last = &trace_enum_maps;
6774
6775 if (!mod->num_trace_enums)
6776 return;
6777
6778 mutex_lock(&trace_enum_mutex);
6779
6780 map = trace_enum_maps;
6781
6782 while (map) {
6783 if (map->head.mod == mod)
6784 break;
6785 map = trace_enum_jmp_to_tail(map);
6786 last = &map->tail.next;
6787 map = map->tail.next;
6788 }
6789 if (!map)
6790 goto out;
6791
6792 *last = trace_enum_jmp_to_tail(map)->tail.next;
6793 kfree(map);
6794 out:
6795 mutex_unlock(&trace_enum_mutex);
6796}
6797#else
6798static inline void trace_module_remove_enums(struct module *mod) { }
6799#endif
6800
6801static int trace_module_notify(struct notifier_block *self,
6802 unsigned long val, void *data)
6803{
6804 struct module *mod = data;
6805
6806 switch (val) {
6807 case MODULE_STATE_COMING:
6808 trace_module_add_enums(mod);
6809 break;
6810 case MODULE_STATE_GOING:
6811 trace_module_remove_enums(mod);
6812 break;
6813 }
6814
6815 return 0;
6816}
6817
6818static struct notifier_block trace_module_nb = {
6819 .notifier_call = trace_module_notify,
6820 .priority = 0,
6821};
6822#endif
6823
6824static __init int tracer_init_tracefs(void)
6825{
6826 struct dentry *d_tracer;
6827
6828 trace_access_lock_init();
6829
6830 d_tracer = tracing_init_dentry();
6831 if (IS_ERR(d_tracer))
6832 return 0;
6833
6834 init_tracer_tracefs(&global_trace, d_tracer);
6835
6836 trace_create_file("tracing_thresh", 0644, d_tracer,
6837 &global_trace, &tracing_thresh_fops);
6838
6839 trace_create_file("README", 0444, d_tracer,
6840 NULL, &tracing_readme_fops);
6841
6842 trace_create_file("saved_cmdlines", 0444, d_tracer,
6843 NULL, &tracing_saved_cmdlines_fops);
6844
6845 trace_create_file("saved_cmdlines_size", 0644, d_tracer,
6846 NULL, &tracing_saved_cmdlines_size_fops);
6847
6848 trace_enum_init();
6849
6850 trace_create_enum_file(d_tracer);
6851
6852#ifdef CONFIG_MODULES
6853 register_module_notifier(&trace_module_nb);
6854#endif
6855
6856#ifdef CONFIG_DYNAMIC_FTRACE
6857 trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
6858 &ftrace_update_tot_cnt, &tracing_dyn_info_fops);
6859#endif
6860
6861 create_trace_instances(d_tracer);
6862
6863 create_trace_options_dir(&global_trace);
6864
6865
6866 if (global_trace.current_trace != &nop_trace)
6867 update_tracer_options(&global_trace, global_trace.current_trace);
6868
6869 return 0;
6870}
6871
6872static int trace_panic_handler(struct notifier_block *this,
6873 unsigned long event, void *unused)
6874{
6875 if (ftrace_dump_on_oops)
6876 ftrace_dump(ftrace_dump_on_oops);
6877 return NOTIFY_OK;
6878}
6879
6880static struct notifier_block trace_panic_notifier = {
6881 .notifier_call = trace_panic_handler,
6882 .next = NULL,
6883 .priority = 150
6884};
6885
6886static int trace_die_handler(struct notifier_block *self,
6887 unsigned long val,
6888 void *data)
6889{
6890 switch (val) {
6891 case DIE_OOPS:
6892 if (ftrace_dump_on_oops)
6893 ftrace_dump(ftrace_dump_on_oops);
6894 break;
6895 default:
6896 break;
6897 }
6898 return NOTIFY_OK;
6899}
6900
6901static struct notifier_block trace_die_notifier = {
6902 .notifier_call = trace_die_handler,
6903 .priority = 200
6904};
6905
6906
6907
6908
6909
6910#define TRACE_MAX_PRINT 1000
6911
6912
6913
6914
6915
6916
6917#define KERN_TRACE KERN_EMERG
6918
6919void
6920trace_printk_seq(struct trace_seq *s)
6921{
6922
6923 if (s->seq.len >= TRACE_MAX_PRINT)
6924 s->seq.len = TRACE_MAX_PRINT;
6925
6926
6927
6928
6929
6930
6931 if (WARN_ON_ONCE(s->seq.len >= s->seq.size))
6932 s->seq.len = s->seq.size - 1;
6933
6934
6935 s->buffer[s->seq.len] = 0;
6936
6937 printk(KERN_TRACE "%s", s->buffer);
6938
6939 trace_seq_init(s);
6940}
6941
6942void trace_init_global_iter(struct trace_iterator *iter)
6943{
6944 iter->tr = &global_trace;
6945 iter->trace = iter->tr->current_trace;
6946 iter->cpu_file = RING_BUFFER_ALL_CPUS;
6947 iter->trace_buffer = &global_trace.trace_buffer;
6948
6949 if (iter->trace && iter->trace->open)
6950 iter->trace->open(iter);
6951
6952
6953 if (ring_buffer_overruns(iter->trace_buffer->buffer))
6954 iter->iter_flags |= TRACE_FILE_ANNOTATE;
6955
6956
6957 if (trace_clocks[iter->tr->clock_id].in_ns)
6958 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
6959}
6960
6961void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
6962{
6963
6964 static struct trace_iterator iter;
6965 static atomic_t dump_running;
6966 unsigned int old_userobj;
6967 unsigned long flags;
6968 int cnt = 0, cpu;
6969
6970
6971 if (atomic_inc_return(&dump_running) != 1) {
6972 atomic_dec(&dump_running);
6973 return;
6974 }
6975
6976
6977
6978
6979
6980
6981
6982
6983
6984 tracing_off();
6985
6986 local_irq_save(flags);
6987
6988
6989 trace_init_global_iter(&iter);
6990
6991 for_each_tracing_cpu(cpu) {
6992 atomic_inc(&per_cpu_ptr(iter.tr->trace_buffer.data, cpu)->disabled);
6993 }
6994
6995 old_userobj = trace_flags & TRACE_ITER_SYM_USEROBJ;
6996
6997
6998 trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
6999
7000 switch (oops_dump_mode) {
7001 case DUMP_ALL:
7002 iter.cpu_file = RING_BUFFER_ALL_CPUS;
7003 break;
7004 case DUMP_ORIG:
7005 iter.cpu_file = raw_smp_processor_id();
7006 break;
7007 case DUMP_NONE:
7008 goto out_enable;
7009 default:
7010 printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
7011 iter.cpu_file = RING_BUFFER_ALL_CPUS;
7012 }
7013
7014 printk(KERN_TRACE "Dumping ftrace buffer:\n");
7015
7016
7017 if (ftrace_is_dead()) {
7018 printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
7019 printk("# MAY BE MISSING FUNCTION EVENTS\n");
7020 }
7021
7022
7023
7024
7025
7026
7027
7028
7029 while (!trace_empty(&iter)) {
7030
7031 if (!cnt)
7032 printk(KERN_TRACE "---------------------------------\n");
7033
7034 cnt++;
7035
7036
7037 memset(&iter.seq, 0,
7038 sizeof(struct trace_iterator) -
7039 offsetof(struct trace_iterator, seq));
7040 iter.iter_flags |= TRACE_FILE_LAT_FMT;
7041 iter.pos = -1;
7042
7043 if (trace_find_next_entry_inc(&iter) != NULL) {
7044 int ret;
7045
7046 ret = print_trace_line(&iter);
7047 if (ret != TRACE_TYPE_NO_CONSUME)
7048 trace_consume(&iter);
7049 }
7050 touch_nmi_watchdog();
7051
7052 trace_printk_seq(&iter.seq);
7053 }
7054
7055 if (!cnt)
7056 printk(KERN_TRACE " (ftrace buffer empty)\n");
7057 else
7058 printk(KERN_TRACE "---------------------------------\n");
7059
7060 out_enable:
7061 trace_flags |= old_userobj;
7062
7063 for_each_tracing_cpu(cpu) {
7064 atomic_dec(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled);
7065 }
7066 atomic_dec(&dump_running);
7067 local_irq_restore(flags);
7068}
7069EXPORT_SYMBOL_GPL(ftrace_dump);
7070
7071__init static int tracer_alloc_buffers(void)
7072{
7073 int ring_buf_size;
7074 int ret = -ENOMEM;
7075
7076 if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
7077 goto out;
7078
7079 if (!alloc_cpumask_var(&global_trace.tracing_cpumask, GFP_KERNEL))
7080 goto out_free_buffer_mask;
7081
7082
7083 if (__stop___trace_bprintk_fmt != __start___trace_bprintk_fmt)
7084
7085 trace_printk_init_buffers();
7086
7087
7088 if (ring_buffer_expanded)
7089 ring_buf_size = trace_buf_size;
7090 else
7091 ring_buf_size = 1;
7092
7093 cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
7094 cpumask_copy(global_trace.tracing_cpumask, cpu_all_mask);
7095
7096 raw_spin_lock_init(&global_trace.start_lock);
7097
7098
7099 temp_buffer = ring_buffer_alloc(PAGE_SIZE, RB_FL_OVERWRITE);
7100 if (!temp_buffer)
7101 goto out_free_cpumask;
7102
7103 if (trace_create_savedcmd() < 0)
7104 goto out_free_temp_buffer;
7105
7106
7107 if (allocate_trace_buffers(&global_trace, ring_buf_size) < 0) {
7108 printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
7109 WARN_ON(1);
7110 goto out_free_savedcmd;
7111 }
7112
7113 if (global_trace.buffer_disabled)
7114 tracing_off();
7115
7116 if (trace_boot_clock) {
7117 ret = tracing_set_clock(&global_trace, trace_boot_clock);
7118 if (ret < 0)
7119 pr_warning("Trace clock %s not defined, going back to default\n",
7120 trace_boot_clock);
7121 }
7122
7123
7124
7125
7126
7127
7128 global_trace.current_trace = &nop_trace;
7129
7130 global_trace.max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
7131
7132 ftrace_init_global_array_ops(&global_trace);
7133
7134 register_tracer(&nop_trace);
7135
7136
7137 tracing_disabled = 0;
7138
7139 atomic_notifier_chain_register(&panic_notifier_list,
7140 &trace_panic_notifier);
7141
7142 register_die_notifier(&trace_die_notifier);
7143
7144 global_trace.flags = TRACE_ARRAY_FL_GLOBAL;
7145
7146 INIT_LIST_HEAD(&global_trace.systems);
7147 INIT_LIST_HEAD(&global_trace.events);
7148 list_add(&global_trace.list, &ftrace_trace_arrays);
7149
7150 while (trace_boot_options) {
7151 char *option;
7152
7153 option = strsep(&trace_boot_options, ",");
7154 trace_set_options(&global_trace, option);
7155 }
7156
7157 register_snapshot_cmd();
7158
7159 return 0;
7160
7161out_free_savedcmd:
7162 free_saved_cmdlines_buffer(savedcmd);
7163out_free_temp_buffer:
7164 ring_buffer_free(temp_buffer);
7165out_free_cpumask:
7166 free_cpumask_var(global_trace.tracing_cpumask);
7167out_free_buffer_mask:
7168 free_cpumask_var(tracing_buffer_mask);
7169out:
7170 return ret;
7171}
7172
7173void __init trace_init(void)
7174{
7175 if (tracepoint_printk) {
7176 tracepoint_print_iter =
7177 kmalloc(sizeof(*tracepoint_print_iter), GFP_KERNEL);
7178 if (WARN_ON(!tracepoint_print_iter))
7179 tracepoint_printk = 0;
7180 }
7181 tracer_alloc_buffers();
7182 trace_event_init();
7183}
7184
7185__init static int clear_boot_tracer(void)
7186{
7187
7188
7189
7190
7191
7192
7193
7194 if (!default_bootup_tracer)
7195 return 0;
7196
7197 printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
7198 default_bootup_tracer);
7199 default_bootup_tracer = NULL;
7200
7201 return 0;
7202}
7203
7204fs_initcall(tracer_init_tracefs);
7205late_initcall(clear_boot_tracer);
7206