1
2
3
4
5
6
7
8
9
10
11
12
13
14#include <linux/ring_buffer.h>
15#include <generated/utsrelease.h>
16#include <linux/stacktrace.h>
17#include <linux/writeback.h>
18#include <linux/kallsyms.h>
19#include <linux/seq_file.h>
20#include <linux/notifier.h>
21#include <linux/irqflags.h>
22#include <linux/debugfs.h>
23#include <linux/pagemap.h>
24#include <linux/hardirq.h>
25#include <linux/linkage.h>
26#include <linux/uaccess.h>
27#include <linux/kprobes.h>
28#include <linux/ftrace.h>
29#include <linux/module.h>
30#include <linux/percpu.h>
31#include <linux/splice.h>
32#include <linux/kdebug.h>
33#include <linux/string.h>
34#include <linux/rwsem.h>
35#include <linux/slab.h>
36#include <linux/ctype.h>
37#include <linux/init.h>
38#include <linux/poll.h>
39#include <linux/nmi.h>
40#include <linux/fs.h>
41#include <linux/sched/rt.h>
42
43#include "trace.h"
44#include "trace_output.h"
45
46
47
48
49
50bool ring_buffer_expanded;
51
52
53
54
55
56
57
58
59static bool __read_mostly tracing_selftest_running;
60
61
62
63
64bool __read_mostly tracing_selftest_disabled;
65
66
67static struct tracer_opt dummy_tracer_opt[] = {
68 { }
69};
70
71static struct tracer_flags dummy_tracer_flags = {
72 .val = 0,
73 .opts = dummy_tracer_opt
74};
75
76static int dummy_set_flag(u32 old_flags, u32 bit, int set)
77{
78 return 0;
79}
80
81
82
83
84
85
86static DEFINE_PER_CPU(bool, trace_cmdline_save);
87
88
89
90
91
92
93
94static int tracing_disabled = 1;
95
96DEFINE_PER_CPU(int, ftrace_cpu_disabled);
97
98cpumask_var_t __read_mostly tracing_buffer_mask;
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116enum ftrace_dump_mode ftrace_dump_on_oops;
117
118static int tracing_set_tracer(const char *buf);
119
120#define MAX_TRACER_SIZE 100
121static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
122static char *default_bootup_tracer;
123
124static bool allocate_snapshot;
125
126static int __init set_cmdline_ftrace(char *str)
127{
128 strlcpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
129 default_bootup_tracer = bootup_tracer_buf;
130
131 ring_buffer_expanded = true;
132 return 1;
133}
134__setup("ftrace=", set_cmdline_ftrace);
135
136static int __init set_ftrace_dump_on_oops(char *str)
137{
138 if (*str++ != '=' || !*str) {
139 ftrace_dump_on_oops = DUMP_ALL;
140 return 1;
141 }
142
143 if (!strcmp("orig_cpu", str)) {
144 ftrace_dump_on_oops = DUMP_ORIG;
145 return 1;
146 }
147
148 return 0;
149}
150__setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
151
152static int __init boot_alloc_snapshot(char *str)
153{
154 allocate_snapshot = true;
155
156 ring_buffer_expanded = true;
157 return 1;
158}
159__setup("alloc_snapshot", boot_alloc_snapshot);
160
161
162static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata;
163static char *trace_boot_options __initdata;
164
165static int __init set_trace_boot_options(char *str)
166{
167 strlcpy(trace_boot_options_buf, str, MAX_TRACER_SIZE);
168 trace_boot_options = trace_boot_options_buf;
169 return 0;
170}
171__setup("trace_options=", set_trace_boot_options);
172
173unsigned long long ns2usecs(cycle_t nsec)
174{
175 nsec += 500;
176 do_div(nsec, 1000);
177 return nsec;
178}
179
180
181
182
183
184
185
186
187
188
189
190
191
192static struct trace_array global_trace;
193
194LIST_HEAD(ftrace_trace_arrays);
195
196int filter_current_check_discard(struct ring_buffer *buffer,
197 struct ftrace_event_call *call, void *rec,
198 struct ring_buffer_event *event)
199{
200 return filter_check_discard(call, rec, buffer, event);
201}
202EXPORT_SYMBOL_GPL(filter_current_check_discard);
203
204cycle_t ftrace_now(int cpu)
205{
206 u64 ts;
207
208
209 if (!global_trace.trace_buffer.buffer)
210 return trace_clock_local();
211
212 ts = ring_buffer_time_stamp(global_trace.trace_buffer.buffer, cpu);
213 ring_buffer_normalize_time_stamp(global_trace.trace_buffer.buffer, cpu, &ts);
214
215 return ts;
216}
217
218int tracing_is_enabled(void)
219{
220 return tracing_is_on();
221}
222
223
224
225
226
227
228
229
230
231
232
233#define TRACE_BUF_SIZE_DEFAULT 1441792UL
234
235static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
236
237
238static struct tracer *trace_types __read_mostly;
239
240
241
242
243static DEFINE_MUTEX(trace_types_lock);
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267#ifdef CONFIG_SMP
268static DECLARE_RWSEM(all_cpu_access_lock);
269static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
270
271static inline void trace_access_lock(int cpu)
272{
273 if (cpu == RING_BUFFER_ALL_CPUS) {
274
275 down_write(&all_cpu_access_lock);
276 } else {
277
278
279
280 down_read(&all_cpu_access_lock);
281
282
283 mutex_lock(&per_cpu(cpu_access_lock, cpu));
284 }
285}
286
287static inline void trace_access_unlock(int cpu)
288{
289 if (cpu == RING_BUFFER_ALL_CPUS) {
290 up_write(&all_cpu_access_lock);
291 } else {
292 mutex_unlock(&per_cpu(cpu_access_lock, cpu));
293 up_read(&all_cpu_access_lock);
294 }
295}
296
297static inline void trace_access_lock_init(void)
298{
299 int cpu;
300
301 for_each_possible_cpu(cpu)
302 mutex_init(&per_cpu(cpu_access_lock, cpu));
303}
304
305#else
306
307static DEFINE_MUTEX(access_lock);
308
309static inline void trace_access_lock(int cpu)
310{
311 (void)cpu;
312 mutex_lock(&access_lock);
313}
314
315static inline void trace_access_unlock(int cpu)
316{
317 (void)cpu;
318 mutex_unlock(&access_lock);
319}
320
321static inline void trace_access_lock_init(void)
322{
323}
324
325#endif
326
327
328unsigned long trace_flags = TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK |
329 TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | TRACE_ITER_SLEEP_TIME |
330 TRACE_ITER_GRAPH_TIME | TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE |
331 TRACE_ITER_IRQ_INFO | TRACE_ITER_MARKERS | TRACE_ITER_FUNCTION;
332
333
334
335
336
337
338
339void tracing_on(void)
340{
341 if (global_trace.trace_buffer.buffer)
342 ring_buffer_record_on(global_trace.trace_buffer.buffer);
343
344
345
346
347
348
349 global_trace.buffer_disabled = 0;
350}
351EXPORT_SYMBOL_GPL(tracing_on);
352
353
354
355
356
357
358
359int __trace_puts(unsigned long ip, const char *str, int size)
360{
361 struct ring_buffer_event *event;
362 struct ring_buffer *buffer;
363 struct print_entry *entry;
364 unsigned long irq_flags;
365 int alloc;
366
367 alloc = sizeof(*entry) + size + 2;
368
369 local_save_flags(irq_flags);
370 buffer = global_trace.trace_buffer.buffer;
371 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, alloc,
372 irq_flags, preempt_count());
373 if (!event)
374 return 0;
375
376 entry = ring_buffer_event_data(event);
377 entry->ip = ip;
378
379 memcpy(&entry->buf, str, size);
380
381
382 if (entry->buf[size - 1] != '\n') {
383 entry->buf[size] = '\n';
384 entry->buf[size + 1] = '\0';
385 } else
386 entry->buf[size] = '\0';
387
388 __buffer_unlock_commit(buffer, event);
389
390 return size;
391}
392EXPORT_SYMBOL_GPL(__trace_puts);
393
394
395
396
397
398
399int __trace_bputs(unsigned long ip, const char *str)
400{
401 struct ring_buffer_event *event;
402 struct ring_buffer *buffer;
403 struct bputs_entry *entry;
404 unsigned long irq_flags;
405 int size = sizeof(struct bputs_entry);
406
407 local_save_flags(irq_flags);
408 buffer = global_trace.trace_buffer.buffer;
409 event = trace_buffer_lock_reserve(buffer, TRACE_BPUTS, size,
410 irq_flags, preempt_count());
411 if (!event)
412 return 0;
413
414 entry = ring_buffer_event_data(event);
415 entry->ip = ip;
416 entry->str = str;
417
418 __buffer_unlock_commit(buffer, event);
419
420 return 1;
421}
422EXPORT_SYMBOL_GPL(__trace_bputs);
423
424#ifdef CONFIG_TRACER_SNAPSHOT
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439void tracing_snapshot(void)
440{
441 struct trace_array *tr = &global_trace;
442 struct tracer *tracer = tr->current_trace;
443 unsigned long flags;
444
445 if (in_nmi()) {
446 internal_trace_puts("*** SNAPSHOT CALLED FROM NMI CONTEXT ***\n");
447 internal_trace_puts("*** snapshot is being ignored ***\n");
448 return;
449 }
450
451 if (!tr->allocated_snapshot) {
452 internal_trace_puts("*** SNAPSHOT NOT ALLOCATED ***\n");
453 internal_trace_puts("*** stopping trace here! ***\n");
454 tracing_off();
455 return;
456 }
457
458
459 if (tracer->use_max_tr) {
460 internal_trace_puts("*** LATENCY TRACER ACTIVE ***\n");
461 internal_trace_puts("*** Can not use snapshot (sorry) ***\n");
462 return;
463 }
464
465 local_irq_save(flags);
466 update_max_tr(tr, current, smp_processor_id());
467 local_irq_restore(flags);
468}
469EXPORT_SYMBOL_GPL(tracing_snapshot);
470
471static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
472 struct trace_buffer *size_buf, int cpu_id);
473static void set_buffer_entries(struct trace_buffer *buf, unsigned long val);
474
475static int alloc_snapshot(struct trace_array *tr)
476{
477 int ret;
478
479 if (!tr->allocated_snapshot) {
480
481
482 ret = resize_buffer_duplicate_size(&tr->max_buffer,
483 &tr->trace_buffer, RING_BUFFER_ALL_CPUS);
484 if (ret < 0)
485 return ret;
486
487 tr->allocated_snapshot = true;
488 }
489
490 return 0;
491}
492
493void free_snapshot(struct trace_array *tr)
494{
495
496
497
498
499
500 ring_buffer_resize(tr->max_buffer.buffer, 1, RING_BUFFER_ALL_CPUS);
501 set_buffer_entries(&tr->max_buffer, 1);
502 tracing_reset_online_cpus(&tr->max_buffer);
503 tr->allocated_snapshot = false;
504}
505
506
507
508
509
510
511
512
513
514
515
516
517void tracing_snapshot_alloc(void)
518{
519 struct trace_array *tr = &global_trace;
520 int ret;
521
522 ret = alloc_snapshot(tr);
523 if (WARN_ON(ret < 0))
524 return;
525
526 tracing_snapshot();
527}
528EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
529#else
530void tracing_snapshot(void)
531{
532 WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used");
533}
534EXPORT_SYMBOL_GPL(tracing_snapshot);
535void tracing_snapshot_alloc(void)
536{
537
538 tracing_snapshot();
539}
540EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
541#endif
542
543
544
545
546
547
548
549
550
551void tracing_off(void)
552{
553 if (global_trace.trace_buffer.buffer)
554 ring_buffer_record_off(global_trace.trace_buffer.buffer);
555
556
557
558
559
560
561 global_trace.buffer_disabled = 1;
562}
563EXPORT_SYMBOL_GPL(tracing_off);
564
565
566
567
568int tracing_is_on(void)
569{
570 if (global_trace.trace_buffer.buffer)
571 return ring_buffer_record_is_on(global_trace.trace_buffer.buffer);
572 return !global_trace.buffer_disabled;
573}
574EXPORT_SYMBOL_GPL(tracing_is_on);
575
576static int __init set_buf_size(char *str)
577{
578 unsigned long buf_size;
579
580 if (!str)
581 return 0;
582 buf_size = memparse(str, &str);
583
584 if (buf_size == 0)
585 return 0;
586 trace_buf_size = buf_size;
587 return 1;
588}
589__setup("trace_buf_size=", set_buf_size);
590
591static int __init set_tracing_thresh(char *str)
592{
593 unsigned long threshold;
594 int ret;
595
596 if (!str)
597 return 0;
598 ret = kstrtoul(str, 0, &threshold);
599 if (ret < 0)
600 return 0;
601 tracing_thresh = threshold * 1000;
602 return 1;
603}
604__setup("tracing_thresh=", set_tracing_thresh);
605
606unsigned long nsecs_to_usecs(unsigned long nsecs)
607{
608 return nsecs / 1000;
609}
610
611
612static const char *trace_options[] = {
613 "print-parent",
614 "sym-offset",
615 "sym-addr",
616 "verbose",
617 "raw",
618 "hex",
619 "bin",
620 "block",
621 "stacktrace",
622 "trace_printk",
623 "ftrace_preempt",
624 "branch",
625 "annotate",
626 "userstacktrace",
627 "sym-userobj",
628 "printk-msg-only",
629 "context-info",
630 "latency-format",
631 "sleep-time",
632 "graph-time",
633 "record-cmd",
634 "overwrite",
635 "disable_on_free",
636 "irq-info",
637 "markers",
638 "function-trace",
639 NULL
640};
641
642static struct {
643 u64 (*func)(void);
644 const char *name;
645 int in_ns;
646} trace_clocks[] = {
647 { trace_clock_local, "local", 1 },
648 { trace_clock_global, "global", 1 },
649 { trace_clock_counter, "counter", 0 },
650 { trace_clock_jiffies, "uptime", 1 },
651 { trace_clock, "perf", 1 },
652 ARCH_TRACE_CLOCKS
653};
654
655
656
657
658int trace_parser_get_init(struct trace_parser *parser, int size)
659{
660 memset(parser, 0, sizeof(*parser));
661
662 parser->buffer = kmalloc(size, GFP_KERNEL);
663 if (!parser->buffer)
664 return 1;
665
666 parser->size = size;
667 return 0;
668}
669
670
671
672
673void trace_parser_put(struct trace_parser *parser)
674{
675 kfree(parser->buffer);
676}
677
678
679
680
681
682
683
684
685
686
687
688
689int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
690 size_t cnt, loff_t *ppos)
691{
692 char ch;
693 size_t read = 0;
694 ssize_t ret;
695
696 if (!*ppos)
697 trace_parser_clear(parser);
698
699 ret = get_user(ch, ubuf++);
700 if (ret)
701 goto out;
702
703 read++;
704 cnt--;
705
706
707
708
709
710 if (!parser->cont) {
711
712 while (cnt && isspace(ch)) {
713 ret = get_user(ch, ubuf++);
714 if (ret)
715 goto out;
716 read++;
717 cnt--;
718 }
719
720
721 if (isspace(ch)) {
722 *ppos += read;
723 ret = read;
724 goto out;
725 }
726
727 parser->idx = 0;
728 }
729
730
731 while (cnt && !isspace(ch)) {
732 if (parser->idx < parser->size - 1)
733 parser->buffer[parser->idx++] = ch;
734 else {
735 ret = -EINVAL;
736 goto out;
737 }
738 ret = get_user(ch, ubuf++);
739 if (ret)
740 goto out;
741 read++;
742 cnt--;
743 }
744
745
746 if (isspace(ch)) {
747 parser->buffer[parser->idx] = 0;
748 parser->cont = false;
749 } else {
750 parser->cont = true;
751 parser->buffer[parser->idx++] = ch;
752 }
753
754 *ppos += read;
755 ret = read;
756
757out:
758 return ret;
759}
760
761ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf, size_t cnt)
762{
763 int len;
764 int ret;
765
766 if (!cnt)
767 return 0;
768
769 if (s->len <= s->readpos)
770 return -EBUSY;
771
772 len = s->len - s->readpos;
773 if (cnt > len)
774 cnt = len;
775 ret = copy_to_user(ubuf, s->buffer + s->readpos, cnt);
776 if (ret == cnt)
777 return -EFAULT;
778
779 cnt -= ret;
780
781 s->readpos += cnt;
782 return cnt;
783}
784
785static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
786{
787 int len;
788
789 if (s->len <= s->readpos)
790 return -EBUSY;
791
792 len = s->len - s->readpos;
793 if (cnt > len)
794 cnt = len;
795 memcpy(buf, s->buffer + s->readpos, cnt);
796
797 s->readpos += cnt;
798 return cnt;
799}
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814static arch_spinlock_t ftrace_max_lock =
815 (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
816
817unsigned long __read_mostly tracing_thresh;
818
819#ifdef CONFIG_TRACER_MAX_TRACE
820unsigned long __read_mostly tracing_max_latency;
821
822
823
824
825
826
827static void
828__update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
829{
830 struct trace_buffer *trace_buf = &tr->trace_buffer;
831 struct trace_buffer *max_buf = &tr->max_buffer;
832 struct trace_array_cpu *data = per_cpu_ptr(trace_buf->data, cpu);
833 struct trace_array_cpu *max_data = per_cpu_ptr(max_buf->data, cpu);
834
835 max_buf->cpu = cpu;
836 max_buf->time_start = data->preempt_timestamp;
837
838 max_data->saved_latency = tracing_max_latency;
839 max_data->critical_start = data->critical_start;
840 max_data->critical_end = data->critical_end;
841
842 memcpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
843 max_data->pid = tsk->pid;
844
845
846
847
848 if (tsk == current)
849 max_data->uid = current_uid();
850 else
851 max_data->uid = task_uid(tsk);
852
853 max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
854 max_data->policy = tsk->policy;
855 max_data->rt_priority = tsk->rt_priority;
856
857
858 tracing_record_cmdline(tsk);
859}
860
861
862
863
864
865
866
867
868
869
870void
871update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
872{
873 struct ring_buffer *buf;
874
875 if (tr->stop_count)
876 return;
877
878 WARN_ON_ONCE(!irqs_disabled());
879
880 if (!tr->allocated_snapshot) {
881
882 WARN_ON_ONCE(tr->current_trace != &nop_trace);
883 return;
884 }
885
886 arch_spin_lock(&ftrace_max_lock);
887
888 buf = tr->trace_buffer.buffer;
889 tr->trace_buffer.buffer = tr->max_buffer.buffer;
890 tr->max_buffer.buffer = buf;
891
892 __update_max_tr(tr, tsk, cpu);
893 arch_spin_unlock(&ftrace_max_lock);
894}
895
896
897
898
899
900
901
902
903
904void
905update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
906{
907 int ret;
908
909 if (tr->stop_count)
910 return;
911
912 WARN_ON_ONCE(!irqs_disabled());
913 if (!tr->allocated_snapshot) {
914
915 WARN_ON_ONCE(tr->current_trace != &nop_trace);
916 return;
917 }
918
919 arch_spin_lock(&ftrace_max_lock);
920
921 ret = ring_buffer_swap_cpu(tr->max_buffer.buffer, tr->trace_buffer.buffer, cpu);
922
923 if (ret == -EBUSY) {
924
925
926
927
928
929
930 trace_array_printk_buf(tr->max_buffer.buffer, _THIS_IP_,
931 "Failed to swap buffers due to commit in progress\n");
932 }
933
934 WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
935
936 __update_max_tr(tr, tsk, cpu);
937 arch_spin_unlock(&ftrace_max_lock);
938}
939#endif
940
941static void default_wait_pipe(struct trace_iterator *iter)
942{
943
944 if (trace_buffer_iter(iter, iter->cpu_file))
945 return;
946
947 ring_buffer_wait(iter->trace_buffer->buffer, iter->cpu_file);
948}
949
950#ifdef CONFIG_FTRACE_STARTUP_TEST
951static int run_tracer_selftest(struct tracer *type)
952{
953 struct trace_array *tr = &global_trace;
954 struct tracer *saved_tracer = tr->current_trace;
955 int ret;
956
957 if (!type->selftest || tracing_selftest_disabled)
958 return 0;
959
960
961
962
963
964
965
966
967 tracing_reset_online_cpus(&tr->trace_buffer);
968
969 tr->current_trace = type;
970
971#ifdef CONFIG_TRACER_MAX_TRACE
972 if (type->use_max_tr) {
973
974 if (ring_buffer_expanded)
975 ring_buffer_resize(tr->max_buffer.buffer, trace_buf_size,
976 RING_BUFFER_ALL_CPUS);
977 tr->allocated_snapshot = true;
978 }
979#endif
980
981
982 pr_info("Testing tracer %s: ", type->name);
983 ret = type->selftest(type, tr);
984
985 tr->current_trace = saved_tracer;
986 if (ret) {
987 printk(KERN_CONT "FAILED!\n");
988
989 WARN_ON(1);
990 return -1;
991 }
992
993 tracing_reset_online_cpus(&tr->trace_buffer);
994
995#ifdef CONFIG_TRACER_MAX_TRACE
996 if (type->use_max_tr) {
997 tr->allocated_snapshot = false;
998
999
1000 if (ring_buffer_expanded)
1001 ring_buffer_resize(tr->max_buffer.buffer, 1,
1002 RING_BUFFER_ALL_CPUS);
1003 }
1004#endif
1005
1006 printk(KERN_CONT "PASSED\n");
1007 return 0;
1008}
1009#else
1010static inline int run_tracer_selftest(struct tracer *type)
1011{
1012 return 0;
1013}
1014#endif
1015
1016
1017
1018
1019
1020
1021
1022int register_tracer(struct tracer *type)
1023{
1024 struct tracer *t;
1025 int ret = 0;
1026
1027 if (!type->name) {
1028 pr_info("Tracer must have a name\n");
1029 return -1;
1030 }
1031
1032 if (strlen(type->name) >= MAX_TRACER_SIZE) {
1033 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
1034 return -1;
1035 }
1036
1037 mutex_lock(&trace_types_lock);
1038
1039 tracing_selftest_running = true;
1040
1041 for (t = trace_types; t; t = t->next) {
1042 if (strcmp(type->name, t->name) == 0) {
1043
1044 pr_info("Tracer %s already registered\n",
1045 type->name);
1046 ret = -1;
1047 goto out;
1048 }
1049 }
1050
1051 if (!type->set_flag)
1052 type->set_flag = &dummy_set_flag;
1053 if (!type->flags)
1054 type->flags = &dummy_tracer_flags;
1055 else
1056 if (!type->flags->opts)
1057 type->flags->opts = dummy_tracer_opt;
1058 if (!type->wait_pipe)
1059 type->wait_pipe = default_wait_pipe;
1060
1061 ret = run_tracer_selftest(type);
1062 if (ret < 0)
1063 goto out;
1064
1065 type->next = trace_types;
1066 trace_types = type;
1067
1068 out:
1069 tracing_selftest_running = false;
1070 mutex_unlock(&trace_types_lock);
1071
1072 if (ret || !default_bootup_tracer)
1073 goto out_unlock;
1074
1075 if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
1076 goto out_unlock;
1077
1078 printk(KERN_INFO "Starting tracer '%s'\n", type->name);
1079
1080 tracing_set_tracer(type->name);
1081 default_bootup_tracer = NULL;
1082
1083 tracing_selftest_disabled = true;
1084#ifdef CONFIG_FTRACE_STARTUP_TEST
1085 printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
1086 type->name);
1087#endif
1088
1089 out_unlock:
1090 return ret;
1091}
1092
1093void tracing_reset(struct trace_buffer *buf, int cpu)
1094{
1095 struct ring_buffer *buffer = buf->buffer;
1096
1097 if (!buffer)
1098 return;
1099
1100 ring_buffer_record_disable(buffer);
1101
1102
1103 synchronize_sched();
1104 ring_buffer_reset_cpu(buffer, cpu);
1105
1106 ring_buffer_record_enable(buffer);
1107}
1108
1109void tracing_reset_online_cpus(struct trace_buffer *buf)
1110{
1111 struct ring_buffer *buffer = buf->buffer;
1112 int cpu;
1113
1114 if (!buffer)
1115 return;
1116
1117 ring_buffer_record_disable(buffer);
1118
1119
1120 synchronize_sched();
1121
1122 buf->time_start = ftrace_now(buf->cpu);
1123
1124 for_each_online_cpu(cpu)
1125 ring_buffer_reset_cpu(buffer, cpu);
1126
1127 ring_buffer_record_enable(buffer);
1128}
1129
1130void tracing_reset_current(int cpu)
1131{
1132 tracing_reset(&global_trace.trace_buffer, cpu);
1133}
1134
1135void tracing_reset_all_online_cpus(void)
1136{
1137 struct trace_array *tr;
1138
1139 mutex_lock(&trace_types_lock);
1140 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
1141 tracing_reset_online_cpus(&tr->trace_buffer);
1142#ifdef CONFIG_TRACER_MAX_TRACE
1143 tracing_reset_online_cpus(&tr->max_buffer);
1144#endif
1145 }
1146 mutex_unlock(&trace_types_lock);
1147}
1148
1149#define SAVED_CMDLINES 128
1150#define NO_CMDLINE_MAP UINT_MAX
1151static unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
1152static unsigned map_cmdline_to_pid[SAVED_CMDLINES];
1153static char saved_cmdlines[SAVED_CMDLINES][TASK_COMM_LEN];
1154static int cmdline_idx;
1155static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
1156
1157
1158static atomic_t trace_record_cmdline_disabled __read_mostly;
1159
1160static void trace_init_cmdlines(void)
1161{
1162 memset(&map_pid_to_cmdline, NO_CMDLINE_MAP, sizeof(map_pid_to_cmdline));
1163 memset(&map_cmdline_to_pid, NO_CMDLINE_MAP, sizeof(map_cmdline_to_pid));
1164 cmdline_idx = 0;
1165}
1166
1167int is_tracing_stopped(void)
1168{
1169 return global_trace.stop_count;
1170}
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180void ftrace_off_permanent(void)
1181{
1182 tracing_disabled = 1;
1183 ftrace_stop();
1184 tracing_off_permanent();
1185}
1186
1187
1188
1189
1190
1191
1192
1193void tracing_start(void)
1194{
1195 struct ring_buffer *buffer;
1196 unsigned long flags;
1197
1198 if (tracing_disabled)
1199 return;
1200
1201 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1202 if (--global_trace.stop_count) {
1203 if (global_trace.stop_count < 0) {
1204
1205 WARN_ON_ONCE(1);
1206 global_trace.stop_count = 0;
1207 }
1208 goto out;
1209 }
1210
1211
1212 arch_spin_lock(&ftrace_max_lock);
1213
1214 buffer = global_trace.trace_buffer.buffer;
1215 if (buffer)
1216 ring_buffer_record_enable(buffer);
1217
1218#ifdef CONFIG_TRACER_MAX_TRACE
1219 buffer = global_trace.max_buffer.buffer;
1220 if (buffer)
1221 ring_buffer_record_enable(buffer);
1222#endif
1223
1224 arch_spin_unlock(&ftrace_max_lock);
1225
1226 ftrace_start();
1227 out:
1228 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1229}
1230
1231static void tracing_start_tr(struct trace_array *tr)
1232{
1233 struct ring_buffer *buffer;
1234 unsigned long flags;
1235
1236 if (tracing_disabled)
1237 return;
1238
1239
1240 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1241 return tracing_start();
1242
1243 raw_spin_lock_irqsave(&tr->start_lock, flags);
1244
1245 if (--tr->stop_count) {
1246 if (tr->stop_count < 0) {
1247
1248 WARN_ON_ONCE(1);
1249 tr->stop_count = 0;
1250 }
1251 goto out;
1252 }
1253
1254 buffer = tr->trace_buffer.buffer;
1255 if (buffer)
1256 ring_buffer_record_enable(buffer);
1257
1258 out:
1259 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
1260}
1261
1262
1263
1264
1265
1266
1267
1268void tracing_stop(void)
1269{
1270 struct ring_buffer *buffer;
1271 unsigned long flags;
1272
1273 ftrace_stop();
1274 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1275 if (global_trace.stop_count++)
1276 goto out;
1277
1278
1279 arch_spin_lock(&ftrace_max_lock);
1280
1281 buffer = global_trace.trace_buffer.buffer;
1282 if (buffer)
1283 ring_buffer_record_disable(buffer);
1284
1285#ifdef CONFIG_TRACER_MAX_TRACE
1286 buffer = global_trace.max_buffer.buffer;
1287 if (buffer)
1288 ring_buffer_record_disable(buffer);
1289#endif
1290
1291 arch_spin_unlock(&ftrace_max_lock);
1292
1293 out:
1294 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1295}
1296
1297static void tracing_stop_tr(struct trace_array *tr)
1298{
1299 struct ring_buffer *buffer;
1300 unsigned long flags;
1301
1302
1303 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1304 return tracing_stop();
1305
1306 raw_spin_lock_irqsave(&tr->start_lock, flags);
1307 if (tr->stop_count++)
1308 goto out;
1309
1310 buffer = tr->trace_buffer.buffer;
1311 if (buffer)
1312 ring_buffer_record_disable(buffer);
1313
1314 out:
1315 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
1316}
1317
1318void trace_stop_cmdline_recording(void);
1319
1320static void trace_save_cmdline(struct task_struct *tsk)
1321{
1322 unsigned pid, idx;
1323
1324 if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT))
1325 return;
1326
1327
1328
1329
1330
1331
1332
1333 if (!arch_spin_trylock(&trace_cmdline_lock))
1334 return;
1335
1336 idx = map_pid_to_cmdline[tsk->pid];
1337 if (idx == NO_CMDLINE_MAP) {
1338 idx = (cmdline_idx + 1) % SAVED_CMDLINES;
1339
1340
1341
1342
1343
1344
1345
1346 pid = map_cmdline_to_pid[idx];
1347 if (pid != NO_CMDLINE_MAP)
1348 map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
1349
1350 map_cmdline_to_pid[idx] = tsk->pid;
1351 map_pid_to_cmdline[tsk->pid] = idx;
1352
1353 cmdline_idx = idx;
1354 }
1355
1356 memcpy(&saved_cmdlines[idx], tsk->comm, TASK_COMM_LEN);
1357
1358 arch_spin_unlock(&trace_cmdline_lock);
1359}
1360
1361void trace_find_cmdline(int pid, char comm[])
1362{
1363 unsigned map;
1364
1365 if (!pid) {
1366 strcpy(comm, "<idle>");
1367 return;
1368 }
1369
1370 if (WARN_ON_ONCE(pid < 0)) {
1371 strcpy(comm, "<XXX>");
1372 return;
1373 }
1374
1375 if (pid > PID_MAX_DEFAULT) {
1376 strcpy(comm, "<...>");
1377 return;
1378 }
1379
1380 preempt_disable();
1381 arch_spin_lock(&trace_cmdline_lock);
1382 map = map_pid_to_cmdline[pid];
1383 if (map != NO_CMDLINE_MAP)
1384 strcpy(comm, saved_cmdlines[map]);
1385 else
1386 strcpy(comm, "<...>");
1387
1388 arch_spin_unlock(&trace_cmdline_lock);
1389 preempt_enable();
1390}
1391
1392void tracing_record_cmdline(struct task_struct *tsk)
1393{
1394 if (atomic_read(&trace_record_cmdline_disabled) || !tracing_is_on())
1395 return;
1396
1397 if (!__this_cpu_read(trace_cmdline_save))
1398 return;
1399
1400 __this_cpu_write(trace_cmdline_save, false);
1401
1402 trace_save_cmdline(tsk);
1403}
1404
1405void
1406tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
1407 int pc)
1408{
1409 struct task_struct *tsk = current;
1410
1411 entry->preempt_count = pc & 0xff;
1412 entry->pid = (tsk) ? tsk->pid : 0;
1413 entry->flags =
1414#ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
1415 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
1416#else
1417 TRACE_FLAG_IRQS_NOSUPPORT |
1418#endif
1419 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
1420 ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
1421 (need_resched() ? TRACE_FLAG_NEED_RESCHED : 0);
1422}
1423EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
1424
1425struct ring_buffer_event *
1426trace_buffer_lock_reserve(struct ring_buffer *buffer,
1427 int type,
1428 unsigned long len,
1429 unsigned long flags, int pc)
1430{
1431 struct ring_buffer_event *event;
1432
1433 event = ring_buffer_lock_reserve(buffer, len);
1434 if (event != NULL) {
1435 struct trace_entry *ent = ring_buffer_event_data(event);
1436
1437 tracing_generic_entry_update(ent, flags, pc);
1438 ent->type = type;
1439 }
1440
1441 return event;
1442}
1443
1444void
1445__buffer_unlock_commit(struct ring_buffer *buffer, struct ring_buffer_event *event)
1446{
1447 __this_cpu_write(trace_cmdline_save, true);
1448 ring_buffer_unlock_commit(buffer, event);
1449}
1450
1451static inline void
1452__trace_buffer_unlock_commit(struct ring_buffer *buffer,
1453 struct ring_buffer_event *event,
1454 unsigned long flags, int pc)
1455{
1456 __buffer_unlock_commit(buffer, event);
1457
1458 ftrace_trace_stack(buffer, flags, 6, pc);
1459 ftrace_trace_userstack(buffer, flags, pc);
1460}
1461
1462void trace_buffer_unlock_commit(struct ring_buffer *buffer,
1463 struct ring_buffer_event *event,
1464 unsigned long flags, int pc)
1465{
1466 __trace_buffer_unlock_commit(buffer, event, flags, pc);
1467}
1468EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit);
1469
1470struct ring_buffer_event *
1471trace_event_buffer_lock_reserve(struct ring_buffer **current_rb,
1472 struct ftrace_event_file *ftrace_file,
1473 int type, unsigned long len,
1474 unsigned long flags, int pc)
1475{
1476 *current_rb = ftrace_file->tr->trace_buffer.buffer;
1477 return trace_buffer_lock_reserve(*current_rb,
1478 type, len, flags, pc);
1479}
1480EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve);
1481
1482struct ring_buffer_event *
1483trace_current_buffer_lock_reserve(struct ring_buffer **current_rb,
1484 int type, unsigned long len,
1485 unsigned long flags, int pc)
1486{
1487 *current_rb = global_trace.trace_buffer.buffer;
1488 return trace_buffer_lock_reserve(*current_rb,
1489 type, len, flags, pc);
1490}
1491EXPORT_SYMBOL_GPL(trace_current_buffer_lock_reserve);
1492
1493void trace_current_buffer_unlock_commit(struct ring_buffer *buffer,
1494 struct ring_buffer_event *event,
1495 unsigned long flags, int pc)
1496{
1497 __trace_buffer_unlock_commit(buffer, event, flags, pc);
1498}
1499EXPORT_SYMBOL_GPL(trace_current_buffer_unlock_commit);
1500
1501void trace_buffer_unlock_commit_regs(struct ring_buffer *buffer,
1502 struct ring_buffer_event *event,
1503 unsigned long flags, int pc,
1504 struct pt_regs *regs)
1505{
1506 __buffer_unlock_commit(buffer, event);
1507
1508 ftrace_trace_stack_regs(buffer, flags, 0, pc, regs);
1509 ftrace_trace_userstack(buffer, flags, pc);
1510}
1511EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit_regs);
1512
1513void trace_current_buffer_discard_commit(struct ring_buffer *buffer,
1514 struct ring_buffer_event *event)
1515{
1516 ring_buffer_discard_commit(buffer, event);
1517}
1518EXPORT_SYMBOL_GPL(trace_current_buffer_discard_commit);
1519
1520void
1521trace_function(struct trace_array *tr,
1522 unsigned long ip, unsigned long parent_ip, unsigned long flags,
1523 int pc)
1524{
1525 struct ftrace_event_call *call = &event_function;
1526 struct ring_buffer *buffer = tr->trace_buffer.buffer;
1527 struct ring_buffer_event *event;
1528 struct ftrace_entry *entry;
1529
1530
1531 if (unlikely(__this_cpu_read(ftrace_cpu_disabled)))
1532 return;
1533
1534 event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
1535 flags, pc);
1536 if (!event)
1537 return;
1538 entry = ring_buffer_event_data(event);
1539 entry->ip = ip;
1540 entry->parent_ip = parent_ip;
1541
1542 if (!filter_check_discard(call, entry, buffer, event))
1543 __buffer_unlock_commit(buffer, event);
1544}
1545
1546void
1547ftrace(struct trace_array *tr, struct trace_array_cpu *data,
1548 unsigned long ip, unsigned long parent_ip, unsigned long flags,
1549 int pc)
1550{
1551 if (likely(!atomic_read(&data->disabled)))
1552 trace_function(tr, ip, parent_ip, flags, pc);
1553}
1554
1555#ifdef CONFIG_STACKTRACE
1556
1557#define FTRACE_STACK_MAX_ENTRIES (PAGE_SIZE / sizeof(unsigned long))
1558struct ftrace_stack {
1559 unsigned long calls[FTRACE_STACK_MAX_ENTRIES];
1560};
1561
1562static DEFINE_PER_CPU(struct ftrace_stack, ftrace_stack);
1563static DEFINE_PER_CPU(int, ftrace_stack_reserve);
1564
1565static void __ftrace_trace_stack(struct ring_buffer *buffer,
1566 unsigned long flags,
1567 int skip, int pc, struct pt_regs *regs)
1568{
1569 struct ftrace_event_call *call = &event_kernel_stack;
1570 struct ring_buffer_event *event;
1571 struct stack_entry *entry;
1572 struct stack_trace trace;
1573 int use_stack;
1574 int size = FTRACE_STACK_ENTRIES;
1575
1576 trace.nr_entries = 0;
1577 trace.skip = skip;
1578
1579
1580
1581
1582
1583
1584
1585 preempt_disable_notrace();
1586
1587 use_stack = __this_cpu_inc_return(ftrace_stack_reserve);
1588
1589
1590
1591
1592
1593
1594
1595 barrier();
1596 if (use_stack == 1) {
1597 trace.entries = &__get_cpu_var(ftrace_stack).calls[0];
1598 trace.max_entries = FTRACE_STACK_MAX_ENTRIES;
1599
1600 if (regs)
1601 save_stack_trace_regs(regs, &trace);
1602 else
1603 save_stack_trace(&trace);
1604
1605 if (trace.nr_entries > size)
1606 size = trace.nr_entries;
1607 } else
1608
1609 use_stack = 0;
1610
1611 size *= sizeof(unsigned long);
1612
1613 event = trace_buffer_lock_reserve(buffer, TRACE_STACK,
1614 sizeof(*entry) + size, flags, pc);
1615 if (!event)
1616 goto out;
1617 entry = ring_buffer_event_data(event);
1618
1619 memset(&entry->caller, 0, size);
1620
1621 if (use_stack)
1622 memcpy(&entry->caller, trace.entries,
1623 trace.nr_entries * sizeof(unsigned long));
1624 else {
1625 trace.max_entries = FTRACE_STACK_ENTRIES;
1626 trace.entries = entry->caller;
1627 if (regs)
1628 save_stack_trace_regs(regs, &trace);
1629 else
1630 save_stack_trace(&trace);
1631 }
1632
1633 entry->size = trace.nr_entries;
1634
1635 if (!filter_check_discard(call, entry, buffer, event))
1636 __buffer_unlock_commit(buffer, event);
1637
1638 out:
1639
1640 barrier();
1641 __this_cpu_dec(ftrace_stack_reserve);
1642 preempt_enable_notrace();
1643
1644}
1645
1646void ftrace_trace_stack_regs(struct ring_buffer *buffer, unsigned long flags,
1647 int skip, int pc, struct pt_regs *regs)
1648{
1649 if (!(trace_flags & TRACE_ITER_STACKTRACE))
1650 return;
1651
1652 __ftrace_trace_stack(buffer, flags, skip, pc, regs);
1653}
1654
1655void ftrace_trace_stack(struct ring_buffer *buffer, unsigned long flags,
1656 int skip, int pc)
1657{
1658 if (!(trace_flags & TRACE_ITER_STACKTRACE))
1659 return;
1660
1661 __ftrace_trace_stack(buffer, flags, skip, pc, NULL);
1662}
1663
1664void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
1665 int pc)
1666{
1667 __ftrace_trace_stack(tr->trace_buffer.buffer, flags, skip, pc, NULL);
1668}
1669
1670
1671
1672
1673
1674void trace_dump_stack(int skip)
1675{
1676 unsigned long flags;
1677
1678 if (tracing_disabled || tracing_selftest_running)
1679 return;
1680
1681 local_save_flags(flags);
1682
1683
1684
1685
1686
1687 skip += 3;
1688 __ftrace_trace_stack(global_trace.trace_buffer.buffer,
1689 flags, skip, preempt_count(), NULL);
1690}
1691
1692static DEFINE_PER_CPU(int, user_stack_count);
1693
1694void
1695ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
1696{
1697 struct ftrace_event_call *call = &event_user_stack;
1698 struct ring_buffer_event *event;
1699 struct userstack_entry *entry;
1700 struct stack_trace trace;
1701
1702 if (!(trace_flags & TRACE_ITER_USERSTACKTRACE))
1703 return;
1704
1705
1706
1707
1708
1709 if (unlikely(in_nmi()))
1710 return;
1711
1712
1713
1714
1715
1716 preempt_disable();
1717 if (__this_cpu_read(user_stack_count))
1718 goto out;
1719
1720 __this_cpu_inc(user_stack_count);
1721
1722 event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
1723 sizeof(*entry), flags, pc);
1724 if (!event)
1725 goto out_drop_count;
1726 entry = ring_buffer_event_data(event);
1727
1728 entry->tgid = current->tgid;
1729 memset(&entry->caller, 0, sizeof(entry->caller));
1730
1731 trace.nr_entries = 0;
1732 trace.max_entries = FTRACE_STACK_ENTRIES;
1733 trace.skip = 0;
1734 trace.entries = entry->caller;
1735
1736 save_stack_trace_user(&trace);
1737 if (!filter_check_discard(call, entry, buffer, event))
1738 __buffer_unlock_commit(buffer, event);
1739
1740 out_drop_count:
1741 __this_cpu_dec(user_stack_count);
1742 out:
1743 preempt_enable();
1744}
1745
1746#ifdef UNUSED
1747static void __trace_userstack(struct trace_array *tr, unsigned long flags)
1748{
1749 ftrace_trace_userstack(tr, flags, preempt_count());
1750}
1751#endif
1752
1753#endif
1754
1755
1756struct trace_buffer_struct {
1757 char buffer[TRACE_BUF_SIZE];
1758};
1759
1760static struct trace_buffer_struct *trace_percpu_buffer;
1761static struct trace_buffer_struct *trace_percpu_sirq_buffer;
1762static struct trace_buffer_struct *trace_percpu_irq_buffer;
1763static struct trace_buffer_struct *trace_percpu_nmi_buffer;
1764
1765
1766
1767
1768
1769
1770
1771
1772static char *get_trace_buf(void)
1773{
1774 struct trace_buffer_struct *percpu_buffer;
1775
1776
1777
1778
1779
1780 if (in_nmi())
1781 percpu_buffer = trace_percpu_nmi_buffer;
1782 else if (in_irq())
1783 percpu_buffer = trace_percpu_irq_buffer;
1784 else if (in_softirq())
1785 percpu_buffer = trace_percpu_sirq_buffer;
1786 else
1787 percpu_buffer = trace_percpu_buffer;
1788
1789 if (!percpu_buffer)
1790 return NULL;
1791
1792 return this_cpu_ptr(&percpu_buffer->buffer[0]);
1793}
1794
1795static int alloc_percpu_trace_buffer(void)
1796{
1797 struct trace_buffer_struct *buffers;
1798 struct trace_buffer_struct *sirq_buffers;
1799 struct trace_buffer_struct *irq_buffers;
1800 struct trace_buffer_struct *nmi_buffers;
1801
1802 buffers = alloc_percpu(struct trace_buffer_struct);
1803 if (!buffers)
1804 goto err_warn;
1805
1806 sirq_buffers = alloc_percpu(struct trace_buffer_struct);
1807 if (!sirq_buffers)
1808 goto err_sirq;
1809
1810 irq_buffers = alloc_percpu(struct trace_buffer_struct);
1811 if (!irq_buffers)
1812 goto err_irq;
1813
1814 nmi_buffers = alloc_percpu(struct trace_buffer_struct);
1815 if (!nmi_buffers)
1816 goto err_nmi;
1817
1818 trace_percpu_buffer = buffers;
1819 trace_percpu_sirq_buffer = sirq_buffers;
1820 trace_percpu_irq_buffer = irq_buffers;
1821 trace_percpu_nmi_buffer = nmi_buffers;
1822
1823 return 0;
1824
1825 err_nmi:
1826 free_percpu(irq_buffers);
1827 err_irq:
1828 free_percpu(sirq_buffers);
1829 err_sirq:
1830 free_percpu(buffers);
1831 err_warn:
1832 WARN(1, "Could not allocate percpu trace_printk buffer");
1833 return -ENOMEM;
1834}
1835
1836static int buffers_allocated;
1837
1838void trace_printk_init_buffers(void)
1839{
1840 if (buffers_allocated)
1841 return;
1842
1843 if (alloc_percpu_trace_buffer())
1844 return;
1845
1846 pr_info("ftrace: Allocated trace_printk buffers\n");
1847
1848
1849 tracing_update_buffers();
1850
1851 buffers_allocated = 1;
1852
1853
1854
1855
1856
1857
1858
1859 if (global_trace.trace_buffer.buffer)
1860 tracing_start_cmdline_record();
1861}
1862
1863void trace_printk_start_comm(void)
1864{
1865
1866 if (!buffers_allocated)
1867 return;
1868 tracing_start_cmdline_record();
1869}
1870
1871static void trace_printk_start_stop_comm(int enabled)
1872{
1873 if (!buffers_allocated)
1874 return;
1875
1876 if (enabled)
1877 tracing_start_cmdline_record();
1878 else
1879 tracing_stop_cmdline_record();
1880}
1881
1882
1883
1884
1885
1886int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
1887{
1888 struct ftrace_event_call *call = &event_bprint;
1889 struct ring_buffer_event *event;
1890 struct ring_buffer *buffer;
1891 struct trace_array *tr = &global_trace;
1892 struct bprint_entry *entry;
1893 unsigned long flags;
1894 char *tbuffer;
1895 int len = 0, size, pc;
1896
1897 if (unlikely(tracing_selftest_running || tracing_disabled))
1898 return 0;
1899
1900
1901 pause_graph_tracing();
1902
1903 pc = preempt_count();
1904 preempt_disable_notrace();
1905
1906 tbuffer = get_trace_buf();
1907 if (!tbuffer) {
1908 len = 0;
1909 goto out;
1910 }
1911
1912 len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args);
1913
1914 if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0)
1915 goto out;
1916
1917 local_save_flags(flags);
1918 size = sizeof(*entry) + sizeof(u32) * len;
1919 buffer = tr->trace_buffer.buffer;
1920 event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
1921 flags, pc);
1922 if (!event)
1923 goto out;
1924 entry = ring_buffer_event_data(event);
1925 entry->ip = ip;
1926 entry->fmt = fmt;
1927
1928 memcpy(entry->buf, tbuffer, sizeof(u32) * len);
1929 if (!filter_check_discard(call, entry, buffer, event)) {
1930 __buffer_unlock_commit(buffer, event);
1931 ftrace_trace_stack(buffer, flags, 6, pc);
1932 }
1933
1934out:
1935 preempt_enable_notrace();
1936 unpause_graph_tracing();
1937
1938 return len;
1939}
1940EXPORT_SYMBOL_GPL(trace_vbprintk);
1941
1942static int
1943__trace_array_vprintk(struct ring_buffer *buffer,
1944 unsigned long ip, const char *fmt, va_list args)
1945{
1946 struct ftrace_event_call *call = &event_print;
1947 struct ring_buffer_event *event;
1948 int len = 0, size, pc;
1949 struct print_entry *entry;
1950 unsigned long flags;
1951 char *tbuffer;
1952
1953 if (tracing_disabled || tracing_selftest_running)
1954 return 0;
1955
1956
1957 pause_graph_tracing();
1958
1959 pc = preempt_count();
1960 preempt_disable_notrace();
1961
1962
1963 tbuffer = get_trace_buf();
1964 if (!tbuffer) {
1965 len = 0;
1966 goto out;
1967 }
1968
1969 len = vsnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args);
1970 if (len > TRACE_BUF_SIZE)
1971 goto out;
1972
1973 local_save_flags(flags);
1974 size = sizeof(*entry) + len + 1;
1975 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
1976 flags, pc);
1977 if (!event)
1978 goto out;
1979 entry = ring_buffer_event_data(event);
1980 entry->ip = ip;
1981
1982 memcpy(&entry->buf, tbuffer, len);
1983 entry->buf[len] = '\0';
1984 if (!filter_check_discard(call, entry, buffer, event)) {
1985 __buffer_unlock_commit(buffer, event);
1986 ftrace_trace_stack(buffer, flags, 6, pc);
1987 }
1988 out:
1989 preempt_enable_notrace();
1990 unpause_graph_tracing();
1991
1992 return len;
1993}
1994
1995int trace_array_vprintk(struct trace_array *tr,
1996 unsigned long ip, const char *fmt, va_list args)
1997{
1998 return __trace_array_vprintk(tr->trace_buffer.buffer, ip, fmt, args);
1999}
2000
2001int trace_array_printk(struct trace_array *tr,
2002 unsigned long ip, const char *fmt, ...)
2003{
2004 int ret;
2005 va_list ap;
2006
2007 if (!(trace_flags & TRACE_ITER_PRINTK))
2008 return 0;
2009
2010 va_start(ap, fmt);
2011 ret = trace_array_vprintk(tr, ip, fmt, ap);
2012 va_end(ap);
2013 return ret;
2014}
2015
2016int trace_array_printk_buf(struct ring_buffer *buffer,
2017 unsigned long ip, const char *fmt, ...)
2018{
2019 int ret;
2020 va_list ap;
2021
2022 if (!(trace_flags & TRACE_ITER_PRINTK))
2023 return 0;
2024
2025 va_start(ap, fmt);
2026 ret = __trace_array_vprintk(buffer, ip, fmt, ap);
2027 va_end(ap);
2028 return ret;
2029}
2030
2031int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
2032{
2033 return trace_array_vprintk(&global_trace, ip, fmt, args);
2034}
2035EXPORT_SYMBOL_GPL(trace_vprintk);
2036
2037static void trace_iterator_increment(struct trace_iterator *iter)
2038{
2039 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu);
2040
2041 iter->idx++;
2042 if (buf_iter)
2043 ring_buffer_read(buf_iter, NULL);
2044}
2045
2046static struct trace_entry *
2047peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts,
2048 unsigned long *lost_events)
2049{
2050 struct ring_buffer_event *event;
2051 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu);
2052
2053 if (buf_iter)
2054 event = ring_buffer_iter_peek(buf_iter, ts);
2055 else
2056 event = ring_buffer_peek(iter->trace_buffer->buffer, cpu, ts,
2057 lost_events);
2058
2059 if (event) {
2060 iter->ent_size = ring_buffer_event_length(event);
2061 return ring_buffer_event_data(event);
2062 }
2063 iter->ent_size = 0;
2064 return NULL;
2065}
2066
2067static struct trace_entry *
2068__find_next_entry(struct trace_iterator *iter, int *ent_cpu,
2069 unsigned long *missing_events, u64 *ent_ts)
2070{
2071 struct ring_buffer *buffer = iter->trace_buffer->buffer;
2072 struct trace_entry *ent, *next = NULL;
2073 unsigned long lost_events = 0, next_lost = 0;
2074 int cpu_file = iter->cpu_file;
2075 u64 next_ts = 0, ts;
2076 int next_cpu = -1;
2077 int next_size = 0;
2078 int cpu;
2079
2080
2081
2082
2083
2084 if (cpu_file > RING_BUFFER_ALL_CPUS) {
2085 if (ring_buffer_empty_cpu(buffer, cpu_file))
2086 return NULL;
2087 ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
2088 if (ent_cpu)
2089 *ent_cpu = cpu_file;
2090
2091 return ent;
2092 }
2093
2094 for_each_tracing_cpu(cpu) {
2095
2096 if (ring_buffer_empty_cpu(buffer, cpu))
2097 continue;
2098
2099 ent = peek_next_entry(iter, cpu, &ts, &lost_events);
2100
2101
2102
2103
2104 if (ent && (!next || ts < next_ts)) {
2105 next = ent;
2106 next_cpu = cpu;
2107 next_ts = ts;
2108 next_lost = lost_events;
2109 next_size = iter->ent_size;
2110 }
2111 }
2112
2113 iter->ent_size = next_size;
2114
2115 if (ent_cpu)
2116 *ent_cpu = next_cpu;
2117
2118 if (ent_ts)
2119 *ent_ts = next_ts;
2120
2121 if (missing_events)
2122 *missing_events = next_lost;
2123
2124 return next;
2125}
2126
2127
2128struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
2129 int *ent_cpu, u64 *ent_ts)
2130{
2131 return __find_next_entry(iter, ent_cpu, NULL, ent_ts);
2132}
2133
2134
2135void *trace_find_next_entry_inc(struct trace_iterator *iter)
2136{
2137 iter->ent = __find_next_entry(iter, &iter->cpu,
2138 &iter->lost_events, &iter->ts);
2139
2140 if (iter->ent)
2141 trace_iterator_increment(iter);
2142
2143 return iter->ent ? iter : NULL;
2144}
2145
2146static void trace_consume(struct trace_iterator *iter)
2147{
2148 ring_buffer_consume(iter->trace_buffer->buffer, iter->cpu, &iter->ts,
2149 &iter->lost_events);
2150}
2151
2152static void *s_next(struct seq_file *m, void *v, loff_t *pos)
2153{
2154 struct trace_iterator *iter = m->private;
2155 int i = (int)*pos;
2156 void *ent;
2157
2158 WARN_ON_ONCE(iter->leftover);
2159
2160 (*pos)++;
2161
2162
2163 if (iter->idx > i)
2164 return NULL;
2165
2166 if (iter->idx < 0)
2167 ent = trace_find_next_entry_inc(iter);
2168 else
2169 ent = iter;
2170
2171 while (ent && iter->idx < i)
2172 ent = trace_find_next_entry_inc(iter);
2173
2174 iter->pos = *pos;
2175
2176 return ent;
2177}
2178
2179void tracing_iter_reset(struct trace_iterator *iter, int cpu)
2180{
2181 struct ring_buffer_event *event;
2182 struct ring_buffer_iter *buf_iter;
2183 unsigned long entries = 0;
2184 u64 ts;
2185
2186 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = 0;
2187
2188 buf_iter = trace_buffer_iter(iter, cpu);
2189 if (!buf_iter)
2190 return;
2191
2192 ring_buffer_iter_reset(buf_iter);
2193
2194
2195
2196
2197
2198
2199 while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
2200 if (ts >= iter->trace_buffer->time_start)
2201 break;
2202 entries++;
2203 ring_buffer_read(buf_iter, NULL);
2204 }
2205
2206 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = entries;
2207}
2208
2209
2210
2211
2212
2213static void *s_start(struct seq_file *m, loff_t *pos)
2214{
2215 struct trace_iterator *iter = m->private;
2216 struct trace_array *tr = iter->tr;
2217 int cpu_file = iter->cpu_file;
2218 void *p = NULL;
2219 loff_t l = 0;
2220 int cpu;
2221
2222
2223
2224
2225
2226
2227
2228 mutex_lock(&trace_types_lock);
2229 if (unlikely(tr->current_trace && iter->trace->name != tr->current_trace->name))
2230 *iter->trace = *tr->current_trace;
2231 mutex_unlock(&trace_types_lock);
2232
2233#ifdef CONFIG_TRACER_MAX_TRACE
2234 if (iter->snapshot && iter->trace->use_max_tr)
2235 return ERR_PTR(-EBUSY);
2236#endif
2237
2238 if (!iter->snapshot)
2239 atomic_inc(&trace_record_cmdline_disabled);
2240
2241 if (*pos != iter->pos) {
2242 iter->ent = NULL;
2243 iter->cpu = 0;
2244 iter->idx = -1;
2245
2246 if (cpu_file == RING_BUFFER_ALL_CPUS) {
2247 for_each_tracing_cpu(cpu)
2248 tracing_iter_reset(iter, cpu);
2249 } else
2250 tracing_iter_reset(iter, cpu_file);
2251
2252 iter->leftover = 0;
2253 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
2254 ;
2255
2256 } else {
2257
2258
2259
2260
2261 if (iter->leftover)
2262 p = iter;
2263 else {
2264 l = *pos - 1;
2265 p = s_next(m, p, &l);
2266 }
2267 }
2268
2269 trace_event_read_lock();
2270 trace_access_lock(cpu_file);
2271 return p;
2272}
2273
2274static void s_stop(struct seq_file *m, void *p)
2275{
2276 struct trace_iterator *iter = m->private;
2277
2278#ifdef CONFIG_TRACER_MAX_TRACE
2279 if (iter->snapshot && iter->trace->use_max_tr)
2280 return;
2281#endif
2282
2283 if (!iter->snapshot)
2284 atomic_dec(&trace_record_cmdline_disabled);
2285
2286 trace_access_unlock(iter->cpu_file);
2287 trace_event_read_unlock();
2288}
2289
2290static void
2291get_total_entries(struct trace_buffer *buf,
2292 unsigned long *total, unsigned long *entries)
2293{
2294 unsigned long count;
2295 int cpu;
2296
2297 *total = 0;
2298 *entries = 0;
2299
2300 for_each_tracing_cpu(cpu) {
2301 count = ring_buffer_entries_cpu(buf->buffer, cpu);
2302
2303
2304
2305
2306
2307 if (per_cpu_ptr(buf->data, cpu)->skipped_entries) {
2308 count -= per_cpu_ptr(buf->data, cpu)->skipped_entries;
2309
2310 *total += count;
2311 } else
2312 *total += count +
2313 ring_buffer_overrun_cpu(buf->buffer, cpu);
2314 *entries += count;
2315 }
2316}
2317
2318static void print_lat_help_header(struct seq_file *m)
2319{
2320 seq_puts(m, "# _------=> CPU# \n");
2321 seq_puts(m, "# / _-----=> irqs-off \n");
2322 seq_puts(m, "# | / _----=> need-resched \n");
2323 seq_puts(m, "# || / _---=> hardirq/softirq \n");
2324 seq_puts(m, "# ||| / _--=> preempt-depth \n");
2325 seq_puts(m, "# |||| / delay \n");
2326 seq_puts(m, "# cmd pid ||||| time | caller \n");
2327 seq_puts(m, "# \\ / ||||| \\ | / \n");
2328}
2329
2330static void print_event_info(struct trace_buffer *buf, struct seq_file *m)
2331{
2332 unsigned long total;
2333 unsigned long entries;
2334
2335 get_total_entries(buf, &total, &entries);
2336 seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu #P:%d\n",
2337 entries, total, num_online_cpus());
2338 seq_puts(m, "#\n");
2339}
2340
2341static void print_func_help_header(struct trace_buffer *buf, struct seq_file *m)
2342{
2343 print_event_info(buf, m);
2344 seq_puts(m, "# TASK-PID CPU# TIMESTAMP FUNCTION\n");
2345 seq_puts(m, "# | | | | |\n");
2346}
2347
2348static void print_func_help_header_irq(struct trace_buffer *buf, struct seq_file *m)
2349{
2350 print_event_info(buf, m);
2351 seq_puts(m, "# _-----=> irqs-off\n");
2352 seq_puts(m, "# / _----=> need-resched\n");
2353 seq_puts(m, "# | / _---=> hardirq/softirq\n");
2354 seq_puts(m, "# || / _--=> preempt-depth\n");
2355 seq_puts(m, "# ||| / delay\n");
2356 seq_puts(m, "# TASK-PID CPU# |||| TIMESTAMP FUNCTION\n");
2357 seq_puts(m, "# | | | |||| | |\n");
2358}
2359
2360void
2361print_trace_header(struct seq_file *m, struct trace_iterator *iter)
2362{
2363 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
2364 struct trace_buffer *buf = iter->trace_buffer;
2365 struct trace_array_cpu *data = per_cpu_ptr(buf->data, buf->cpu);
2366 struct tracer *type = iter->trace;
2367 unsigned long entries;
2368 unsigned long total;
2369 const char *name = "preemption";
2370
2371 name = type->name;
2372
2373 get_total_entries(buf, &total, &entries);
2374
2375 seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
2376 name, UTS_RELEASE);
2377 seq_puts(m, "# -----------------------------------"
2378 "---------------------------------\n");
2379 seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
2380 " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
2381 nsecs_to_usecs(data->saved_latency),
2382 entries,
2383 total,
2384 buf->cpu,
2385#if defined(CONFIG_PREEMPT_NONE)
2386 "server",
2387#elif defined(CONFIG_PREEMPT_VOLUNTARY)
2388 "desktop",
2389#elif defined(CONFIG_PREEMPT)
2390 "preempt",
2391#else
2392 "unknown",
2393#endif
2394
2395 0, 0, 0, 0);
2396#ifdef CONFIG_SMP
2397 seq_printf(m, " #P:%d)\n", num_online_cpus());
2398#else
2399 seq_puts(m, ")\n");
2400#endif
2401 seq_puts(m, "# -----------------\n");
2402 seq_printf(m, "# | task: %.16s-%d "
2403 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
2404 data->comm, data->pid,
2405 from_kuid_munged(seq_user_ns(m), data->uid), data->nice,
2406 data->policy, data->rt_priority);
2407 seq_puts(m, "# -----------------\n");
2408
2409 if (data->critical_start) {
2410 seq_puts(m, "# => started at: ");
2411 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
2412 trace_print_seq(m, &iter->seq);
2413 seq_puts(m, "\n# => ended at: ");
2414 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
2415 trace_print_seq(m, &iter->seq);
2416 seq_puts(m, "\n#\n");
2417 }
2418
2419 seq_puts(m, "#\n");
2420}
2421
2422static void test_cpu_buff_start(struct trace_iterator *iter)
2423{
2424 struct trace_seq *s = &iter->seq;
2425
2426 if (!(trace_flags & TRACE_ITER_ANNOTATE))
2427 return;
2428
2429 if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
2430 return;
2431
2432 if (cpumask_test_cpu(iter->cpu, iter->started))
2433 return;
2434
2435 if (per_cpu_ptr(iter->trace_buffer->data, iter->cpu)->skipped_entries)
2436 return;
2437
2438 cpumask_set_cpu(iter->cpu, iter->started);
2439
2440
2441 if (iter->idx > 1)
2442 trace_seq_printf(s, "##### CPU %u buffer started ####\n",
2443 iter->cpu);
2444}
2445
2446static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
2447{
2448 struct trace_seq *s = &iter->seq;
2449 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
2450 struct trace_entry *entry;
2451 struct trace_event *event;
2452
2453 entry = iter->ent;
2454
2455 test_cpu_buff_start(iter);
2456
2457 event = ftrace_find_event(entry->type);
2458
2459 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2460 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2461 if (!trace_print_lat_context(iter))
2462 goto partial;
2463 } else {
2464 if (!trace_print_context(iter))
2465 goto partial;
2466 }
2467 }
2468
2469 if (event)
2470 return event->funcs->trace(iter, sym_flags, event);
2471
2472 if (!trace_seq_printf(s, "Unknown type %d\n", entry->type))
2473 goto partial;
2474
2475 return TRACE_TYPE_HANDLED;
2476partial:
2477 return TRACE_TYPE_PARTIAL_LINE;
2478}
2479
2480static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
2481{
2482 struct trace_seq *s = &iter->seq;
2483 struct trace_entry *entry;
2484 struct trace_event *event;
2485
2486 entry = iter->ent;
2487
2488 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2489 if (!trace_seq_printf(s, "%d %d %llu ",
2490 entry->pid, iter->cpu, iter->ts))
2491 goto partial;
2492 }
2493
2494 event = ftrace_find_event(entry->type);
2495 if (event)
2496 return event->funcs->raw(iter, 0, event);
2497
2498 if (!trace_seq_printf(s, "%d ?\n", entry->type))
2499 goto partial;
2500
2501 return TRACE_TYPE_HANDLED;
2502partial:
2503 return TRACE_TYPE_PARTIAL_LINE;
2504}
2505
2506static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
2507{
2508 struct trace_seq *s = &iter->seq;
2509 unsigned char newline = '\n';
2510 struct trace_entry *entry;
2511 struct trace_event *event;
2512
2513 entry = iter->ent;
2514
2515 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2516 SEQ_PUT_HEX_FIELD_RET(s, entry->pid);
2517 SEQ_PUT_HEX_FIELD_RET(s, iter->cpu);
2518 SEQ_PUT_HEX_FIELD_RET(s, iter->ts);
2519 }
2520
2521 event = ftrace_find_event(entry->type);
2522 if (event) {
2523 enum print_line_t ret = event->funcs->hex(iter, 0, event);
2524 if (ret != TRACE_TYPE_HANDLED)
2525 return ret;
2526 }
2527
2528 SEQ_PUT_FIELD_RET(s, newline);
2529
2530 return TRACE_TYPE_HANDLED;
2531}
2532
2533static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
2534{
2535 struct trace_seq *s = &iter->seq;
2536 struct trace_entry *entry;
2537 struct trace_event *event;
2538
2539 entry = iter->ent;
2540
2541 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2542 SEQ_PUT_FIELD_RET(s, entry->pid);
2543 SEQ_PUT_FIELD_RET(s, iter->cpu);
2544 SEQ_PUT_FIELD_RET(s, iter->ts);
2545 }
2546
2547 event = ftrace_find_event(entry->type);
2548 return event ? event->funcs->binary(iter, 0, event) :
2549 TRACE_TYPE_HANDLED;
2550}
2551
2552int trace_empty(struct trace_iterator *iter)
2553{
2554 struct ring_buffer_iter *buf_iter;
2555 int cpu;
2556
2557
2558 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
2559 cpu = iter->cpu_file;
2560 buf_iter = trace_buffer_iter(iter, cpu);
2561 if (buf_iter) {
2562 if (!ring_buffer_iter_empty(buf_iter))
2563 return 0;
2564 } else {
2565 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
2566 return 0;
2567 }
2568 return 1;
2569 }
2570
2571 for_each_tracing_cpu(cpu) {
2572 buf_iter = trace_buffer_iter(iter, cpu);
2573 if (buf_iter) {
2574 if (!ring_buffer_iter_empty(buf_iter))
2575 return 0;
2576 } else {
2577 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
2578 return 0;
2579 }
2580 }
2581
2582 return 1;
2583}
2584
2585
2586enum print_line_t print_trace_line(struct trace_iterator *iter)
2587{
2588 enum print_line_t ret;
2589
2590 if (iter->lost_events &&
2591 !trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
2592 iter->cpu, iter->lost_events))
2593 return TRACE_TYPE_PARTIAL_LINE;
2594
2595 if (iter->trace && iter->trace->print_line) {
2596 ret = iter->trace->print_line(iter);
2597 if (ret != TRACE_TYPE_UNHANDLED)
2598 return ret;
2599 }
2600
2601 if (iter->ent->type == TRACE_BPUTS &&
2602 trace_flags & TRACE_ITER_PRINTK &&
2603 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2604 return trace_print_bputs_msg_only(iter);
2605
2606 if (iter->ent->type == TRACE_BPRINT &&
2607 trace_flags & TRACE_ITER_PRINTK &&
2608 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2609 return trace_print_bprintk_msg_only(iter);
2610
2611 if (iter->ent->type == TRACE_PRINT &&
2612 trace_flags & TRACE_ITER_PRINTK &&
2613 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2614 return trace_print_printk_msg_only(iter);
2615
2616 if (trace_flags & TRACE_ITER_BIN)
2617 return print_bin_fmt(iter);
2618
2619 if (trace_flags & TRACE_ITER_HEX)
2620 return print_hex_fmt(iter);
2621
2622 if (trace_flags & TRACE_ITER_RAW)
2623 return print_raw_fmt(iter);
2624
2625 return print_trace_fmt(iter);
2626}
2627
2628void trace_latency_header(struct seq_file *m)
2629{
2630 struct trace_iterator *iter = m->private;
2631
2632
2633 if (trace_empty(iter))
2634 return;
2635
2636 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
2637 print_trace_header(m, iter);
2638
2639 if (!(trace_flags & TRACE_ITER_VERBOSE))
2640 print_lat_help_header(m);
2641}
2642
2643void trace_default_header(struct seq_file *m)
2644{
2645 struct trace_iterator *iter = m->private;
2646
2647 if (!(trace_flags & TRACE_ITER_CONTEXT_INFO))
2648 return;
2649
2650 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2651
2652 if (trace_empty(iter))
2653 return;
2654 print_trace_header(m, iter);
2655 if (!(trace_flags & TRACE_ITER_VERBOSE))
2656 print_lat_help_header(m);
2657 } else {
2658 if (!(trace_flags & TRACE_ITER_VERBOSE)) {
2659 if (trace_flags & TRACE_ITER_IRQ_INFO)
2660 print_func_help_header_irq(iter->trace_buffer, m);
2661 else
2662 print_func_help_header(iter->trace_buffer, m);
2663 }
2664 }
2665}
2666
2667static void test_ftrace_alive(struct seq_file *m)
2668{
2669 if (!ftrace_is_dead())
2670 return;
2671 seq_printf(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n");
2672 seq_printf(m, "# MAY BE MISSING FUNCTION EVENTS\n");
2673}
2674
2675#ifdef CONFIG_TRACER_MAX_TRACE
2676static void show_snapshot_main_help(struct seq_file *m)
2677{
2678 seq_printf(m, "# echo 0 > snapshot : Clears and frees snapshot buffer\n");
2679 seq_printf(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n");
2680 seq_printf(m, "# Takes a snapshot of the main buffer.\n");
2681 seq_printf(m, "# echo 2 > snapshot : Clears snapshot buffer (but does not allocate)\n");
2682 seq_printf(m, "# (Doesn't have to be '2' works with any number that\n");
2683 seq_printf(m, "# is not a '0' or '1')\n");
2684}
2685
2686static void show_snapshot_percpu_help(struct seq_file *m)
2687{
2688 seq_printf(m, "# echo 0 > snapshot : Invalid for per_cpu snapshot file.\n");
2689#ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
2690 seq_printf(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n");
2691 seq_printf(m, "# Takes a snapshot of the main buffer for this cpu.\n");
2692#else
2693 seq_printf(m, "# echo 1 > snapshot : Not supported with this kernel.\n");
2694 seq_printf(m, "# Must use main snapshot file to allocate.\n");
2695#endif
2696 seq_printf(m, "# echo 2 > snapshot : Clears this cpu's snapshot buffer (but does not allocate)\n");
2697 seq_printf(m, "# (Doesn't have to be '2' works with any number that\n");
2698 seq_printf(m, "# is not a '0' or '1')\n");
2699}
2700
2701static void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter)
2702{
2703 if (iter->tr->allocated_snapshot)
2704 seq_printf(m, "#\n# * Snapshot is allocated *\n#\n");
2705 else
2706 seq_printf(m, "#\n# * Snapshot is freed *\n#\n");
2707
2708 seq_printf(m, "# Snapshot commands:\n");
2709 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
2710 show_snapshot_main_help(m);
2711 else
2712 show_snapshot_percpu_help(m);
2713}
2714#else
2715
2716static inline void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) { }
2717#endif
2718
2719static int s_show(struct seq_file *m, void *v)
2720{
2721 struct trace_iterator *iter = v;
2722 int ret;
2723
2724 if (iter->ent == NULL) {
2725 if (iter->tr) {
2726 seq_printf(m, "# tracer: %s\n", iter->trace->name);
2727 seq_puts(m, "#\n");
2728 test_ftrace_alive(m);
2729 }
2730 if (iter->snapshot && trace_empty(iter))
2731 print_snapshot_help(m, iter);
2732 else if (iter->trace && iter->trace->print_header)
2733 iter->trace->print_header(m);
2734 else
2735 trace_default_header(m);
2736
2737 } else if (iter->leftover) {
2738
2739
2740
2741
2742 ret = trace_print_seq(m, &iter->seq);
2743
2744
2745 iter->leftover = ret;
2746
2747 } else {
2748 print_trace_line(iter);
2749 ret = trace_print_seq(m, &iter->seq);
2750
2751
2752
2753
2754
2755
2756
2757 iter->leftover = ret;
2758 }
2759
2760 return 0;
2761}
2762
2763static const struct seq_operations tracer_seq_ops = {
2764 .start = s_start,
2765 .next = s_next,
2766 .stop = s_stop,
2767 .show = s_show,
2768};
2769
2770static struct trace_iterator *
2771__tracing_open(struct inode *inode, struct file *file, bool snapshot)
2772{
2773 struct trace_cpu *tc = inode->i_private;
2774 struct trace_array *tr = tc->tr;
2775 struct trace_iterator *iter;
2776 int cpu;
2777
2778 if (tracing_disabled)
2779 return ERR_PTR(-ENODEV);
2780
2781 iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter));
2782 if (!iter)
2783 return ERR_PTR(-ENOMEM);
2784
2785 iter->buffer_iter = kzalloc(sizeof(*iter->buffer_iter) * num_possible_cpus(),
2786 GFP_KERNEL);
2787 if (!iter->buffer_iter)
2788 goto release;
2789
2790
2791
2792
2793
2794 mutex_lock(&trace_types_lock);
2795 iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
2796 if (!iter->trace)
2797 goto fail;
2798
2799 *iter->trace = *tr->current_trace;
2800
2801 if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
2802 goto fail;
2803
2804 iter->tr = tr;
2805
2806#ifdef CONFIG_TRACER_MAX_TRACE
2807
2808 if (tr->current_trace->print_max || snapshot)
2809 iter->trace_buffer = &tr->max_buffer;
2810 else
2811#endif
2812 iter->trace_buffer = &tr->trace_buffer;
2813 iter->snapshot = snapshot;
2814 iter->pos = -1;
2815 mutex_init(&iter->mutex);
2816 iter->cpu_file = tc->cpu;
2817
2818
2819 if (iter->trace && iter->trace->open)
2820 iter->trace->open(iter);
2821
2822
2823 if (ring_buffer_overruns(iter->trace_buffer->buffer))
2824 iter->iter_flags |= TRACE_FILE_ANNOTATE;
2825
2826
2827 if (trace_clocks[tr->clock_id].in_ns)
2828 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
2829
2830
2831 if (!iter->snapshot)
2832 tracing_stop_tr(tr);
2833
2834 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) {
2835 for_each_tracing_cpu(cpu) {
2836 iter->buffer_iter[cpu] =
2837 ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
2838 }
2839 ring_buffer_read_prepare_sync();
2840 for_each_tracing_cpu(cpu) {
2841 ring_buffer_read_start(iter->buffer_iter[cpu]);
2842 tracing_iter_reset(iter, cpu);
2843 }
2844 } else {
2845 cpu = iter->cpu_file;
2846 iter->buffer_iter[cpu] =
2847 ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
2848 ring_buffer_read_prepare_sync();
2849 ring_buffer_read_start(iter->buffer_iter[cpu]);
2850 tracing_iter_reset(iter, cpu);
2851 }
2852
2853 tr->ref++;
2854
2855 mutex_unlock(&trace_types_lock);
2856
2857 return iter;
2858
2859 fail:
2860 mutex_unlock(&trace_types_lock);
2861 kfree(iter->trace);
2862 kfree(iter->buffer_iter);
2863release:
2864 seq_release_private(inode, file);
2865 return ERR_PTR(-ENOMEM);
2866}
2867
2868int tracing_open_generic(struct inode *inode, struct file *filp)
2869{
2870 if (tracing_disabled)
2871 return -ENODEV;
2872
2873 filp->private_data = inode->i_private;
2874 return 0;
2875}
2876
2877static int tracing_release(struct inode *inode, struct file *file)
2878{
2879 struct seq_file *m = file->private_data;
2880 struct trace_iterator *iter;
2881 struct trace_array *tr;
2882 int cpu;
2883
2884 if (!(file->f_mode & FMODE_READ))
2885 return 0;
2886
2887 iter = m->private;
2888 tr = iter->tr;
2889
2890 mutex_lock(&trace_types_lock);
2891
2892 WARN_ON(!tr->ref);
2893 tr->ref--;
2894
2895 for_each_tracing_cpu(cpu) {
2896 if (iter->buffer_iter[cpu])
2897 ring_buffer_read_finish(iter->buffer_iter[cpu]);
2898 }
2899
2900 if (iter->trace && iter->trace->close)
2901 iter->trace->close(iter);
2902
2903 if (!iter->snapshot)
2904
2905 tracing_start_tr(tr);
2906 mutex_unlock(&trace_types_lock);
2907
2908 mutex_destroy(&iter->mutex);
2909 free_cpumask_var(iter->started);
2910 kfree(iter->trace);
2911 kfree(iter->buffer_iter);
2912 seq_release_private(inode, file);
2913 return 0;
2914}
2915
2916static int tracing_open(struct inode *inode, struct file *file)
2917{
2918 struct trace_iterator *iter;
2919 int ret = 0;
2920
2921
2922 if ((file->f_mode & FMODE_WRITE) &&
2923 (file->f_flags & O_TRUNC)) {
2924 struct trace_cpu *tc = inode->i_private;
2925 struct trace_array *tr = tc->tr;
2926
2927 if (tc->cpu == RING_BUFFER_ALL_CPUS)
2928 tracing_reset_online_cpus(&tr->trace_buffer);
2929 else
2930 tracing_reset(&tr->trace_buffer, tc->cpu);
2931 }
2932
2933 if (file->f_mode & FMODE_READ) {
2934 iter = __tracing_open(inode, file, false);
2935 if (IS_ERR(iter))
2936 ret = PTR_ERR(iter);
2937 else if (trace_flags & TRACE_ITER_LATENCY_FMT)
2938 iter->iter_flags |= TRACE_FILE_LAT_FMT;
2939 }
2940 return ret;
2941}
2942
2943static void *
2944t_next(struct seq_file *m, void *v, loff_t *pos)
2945{
2946 struct tracer *t = v;
2947
2948 (*pos)++;
2949
2950 if (t)
2951 t = t->next;
2952
2953 return t;
2954}
2955
2956static void *t_start(struct seq_file *m, loff_t *pos)
2957{
2958 struct tracer *t;
2959 loff_t l = 0;
2960
2961 mutex_lock(&trace_types_lock);
2962 for (t = trace_types; t && l < *pos; t = t_next(m, t, &l))
2963 ;
2964
2965 return t;
2966}
2967
2968static void t_stop(struct seq_file *m, void *p)
2969{
2970 mutex_unlock(&trace_types_lock);
2971}
2972
2973static int t_show(struct seq_file *m, void *v)
2974{
2975 struct tracer *t = v;
2976
2977 if (!t)
2978 return 0;
2979
2980 seq_printf(m, "%s", t->name);
2981 if (t->next)
2982 seq_putc(m, ' ');
2983 else
2984 seq_putc(m, '\n');
2985
2986 return 0;
2987}
2988
2989static const struct seq_operations show_traces_seq_ops = {
2990 .start = t_start,
2991 .next = t_next,
2992 .stop = t_stop,
2993 .show = t_show,
2994};
2995
2996static int show_traces_open(struct inode *inode, struct file *file)
2997{
2998 if (tracing_disabled)
2999 return -ENODEV;
3000
3001 return seq_open(file, &show_traces_seq_ops);
3002}
3003
3004static ssize_t
3005tracing_write_stub(struct file *filp, const char __user *ubuf,
3006 size_t count, loff_t *ppos)
3007{
3008 return count;
3009}
3010
3011static loff_t tracing_seek(struct file *file, loff_t offset, int origin)
3012{
3013 if (file->f_mode & FMODE_READ)
3014 return seq_lseek(file, offset, origin);
3015 else
3016 return 0;
3017}
3018
3019static const struct file_operations tracing_fops = {
3020 .open = tracing_open,
3021 .read = seq_read,
3022 .write = tracing_write_stub,
3023 .llseek = tracing_seek,
3024 .release = tracing_release,
3025};
3026
3027static const struct file_operations show_traces_fops = {
3028 .open = show_traces_open,
3029 .read = seq_read,
3030 .release = seq_release,
3031 .llseek = seq_lseek,
3032};
3033
3034
3035
3036
3037static cpumask_var_t tracing_cpumask;
3038
3039
3040
3041
3042
3043static DEFINE_MUTEX(tracing_cpumask_update_lock);
3044
3045
3046
3047
3048
3049static char mask_str[NR_CPUS + 1];
3050
3051static ssize_t
3052tracing_cpumask_read(struct file *filp, char __user *ubuf,
3053 size_t count, loff_t *ppos)
3054{
3055 int len;
3056
3057 mutex_lock(&tracing_cpumask_update_lock);
3058
3059 len = cpumask_scnprintf(mask_str, count, tracing_cpumask);
3060 if (count - len < 2) {
3061 count = -EINVAL;
3062 goto out_err;
3063 }
3064 len += sprintf(mask_str + len, "\n");
3065 count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);
3066
3067out_err:
3068 mutex_unlock(&tracing_cpumask_update_lock);
3069
3070 return count;
3071}
3072
3073static ssize_t
3074tracing_cpumask_write(struct file *filp, const char __user *ubuf,
3075 size_t count, loff_t *ppos)
3076{
3077 struct trace_array *tr = filp->private_data;
3078 cpumask_var_t tracing_cpumask_new;
3079 int err, cpu;
3080
3081 if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
3082 return -ENOMEM;
3083
3084 err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
3085 if (err)
3086 goto err_unlock;
3087
3088 mutex_lock(&tracing_cpumask_update_lock);
3089
3090 local_irq_disable();
3091 arch_spin_lock(&ftrace_max_lock);
3092 for_each_tracing_cpu(cpu) {
3093
3094
3095
3096
3097 if (cpumask_test_cpu(cpu, tracing_cpumask) &&
3098 !cpumask_test_cpu(cpu, tracing_cpumask_new)) {
3099 atomic_inc(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
3100 ring_buffer_record_disable_cpu(tr->trace_buffer.buffer, cpu);
3101 }
3102 if (!cpumask_test_cpu(cpu, tracing_cpumask) &&
3103 cpumask_test_cpu(cpu, tracing_cpumask_new)) {
3104 atomic_dec(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
3105 ring_buffer_record_enable_cpu(tr->trace_buffer.buffer, cpu);
3106 }
3107 }
3108 arch_spin_unlock(&ftrace_max_lock);
3109 local_irq_enable();
3110
3111 cpumask_copy(tracing_cpumask, tracing_cpumask_new);
3112
3113 mutex_unlock(&tracing_cpumask_update_lock);
3114 free_cpumask_var(tracing_cpumask_new);
3115
3116 return count;
3117
3118err_unlock:
3119 free_cpumask_var(tracing_cpumask_new);
3120
3121 return err;
3122}
3123
3124static const struct file_operations tracing_cpumask_fops = {
3125 .open = tracing_open_generic,
3126 .read = tracing_cpumask_read,
3127 .write = tracing_cpumask_write,
3128 .llseek = generic_file_llseek,
3129};
3130
3131static int tracing_trace_options_show(struct seq_file *m, void *v)
3132{
3133 struct tracer_opt *trace_opts;
3134 struct trace_array *tr = m->private;
3135 u32 tracer_flags;
3136 int i;
3137
3138 mutex_lock(&trace_types_lock);
3139 tracer_flags = tr->current_trace->flags->val;
3140 trace_opts = tr->current_trace->flags->opts;
3141
3142 for (i = 0; trace_options[i]; i++) {
3143 if (trace_flags & (1 << i))
3144 seq_printf(m, "%s\n", trace_options[i]);
3145 else
3146 seq_printf(m, "no%s\n", trace_options[i]);
3147 }
3148
3149 for (i = 0; trace_opts[i].name; i++) {
3150 if (tracer_flags & trace_opts[i].bit)
3151 seq_printf(m, "%s\n", trace_opts[i].name);
3152 else
3153 seq_printf(m, "no%s\n", trace_opts[i].name);
3154 }
3155 mutex_unlock(&trace_types_lock);
3156
3157 return 0;
3158}
3159
3160static int __set_tracer_option(struct tracer *trace,
3161 struct tracer_flags *tracer_flags,
3162 struct tracer_opt *opts, int neg)
3163{
3164 int ret;
3165
3166 ret = trace->set_flag(tracer_flags->val, opts->bit, !neg);
3167 if (ret)
3168 return ret;
3169
3170 if (neg)
3171 tracer_flags->val &= ~opts->bit;
3172 else
3173 tracer_flags->val |= opts->bit;
3174 return 0;
3175}
3176
3177
3178static int set_tracer_option(struct tracer *trace, char *cmp, int neg)
3179{
3180 struct tracer_flags *tracer_flags = trace->flags;
3181 struct tracer_opt *opts = NULL;
3182 int i;
3183
3184 for (i = 0; tracer_flags->opts[i].name; i++) {
3185 opts = &tracer_flags->opts[i];
3186
3187 if (strcmp(cmp, opts->name) == 0)
3188 return __set_tracer_option(trace, trace->flags,
3189 opts, neg);
3190 }
3191
3192 return -EINVAL;
3193}
3194
3195
3196int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set)
3197{
3198 if (tracer->enabled && (mask & TRACE_ITER_OVERWRITE) && !set)
3199 return -1;
3200
3201 return 0;
3202}
3203
3204int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled)
3205{
3206
3207 if (!!(trace_flags & mask) == !!enabled)
3208 return 0;
3209
3210
3211 if (tr->current_trace->flag_changed)
3212 if (tr->current_trace->flag_changed(tr->current_trace, mask, !!enabled))
3213 return -EINVAL;
3214
3215 if (enabled)
3216 trace_flags |= mask;
3217 else
3218 trace_flags &= ~mask;
3219
3220 if (mask == TRACE_ITER_RECORD_CMD)
3221 trace_event_enable_cmd_record(enabled);
3222
3223 if (mask == TRACE_ITER_OVERWRITE) {
3224 ring_buffer_change_overwrite(tr->trace_buffer.buffer, enabled);
3225#ifdef CONFIG_TRACER_MAX_TRACE
3226 ring_buffer_change_overwrite(tr->max_buffer.buffer, enabled);
3227#endif
3228 }
3229
3230 if (mask == TRACE_ITER_PRINTK)
3231 trace_printk_start_stop_comm(enabled);
3232
3233 return 0;
3234}
3235
3236static int trace_set_options(struct trace_array *tr, char *option)
3237{
3238 char *cmp;
3239 int neg = 0;
3240 int ret = -ENODEV;
3241 int i;
3242
3243 cmp = strstrip(option);
3244
3245 if (strncmp(cmp, "no", 2) == 0) {
3246 neg = 1;
3247 cmp += 2;
3248 }
3249
3250 mutex_lock(&trace_types_lock);
3251
3252 for (i = 0; trace_options[i]; i++) {
3253 if (strcmp(cmp, trace_options[i]) == 0) {
3254 ret = set_tracer_flag(tr, 1 << i, !neg);
3255 break;
3256 }
3257 }
3258
3259
3260 if (!trace_options[i])
3261 ret = set_tracer_option(tr->current_trace, cmp, neg);
3262
3263 mutex_unlock(&trace_types_lock);
3264
3265 return ret;
3266}
3267
3268static ssize_t
3269tracing_trace_options_write(struct file *filp, const char __user *ubuf,
3270 size_t cnt, loff_t *ppos)
3271{
3272 struct seq_file *m = filp->private_data;
3273 struct trace_array *tr = m->private;
3274 char buf[64];
3275 int ret;
3276
3277 if (cnt >= sizeof(buf))
3278 return -EINVAL;
3279
3280 if (copy_from_user(&buf, ubuf, cnt))
3281 return -EFAULT;
3282
3283 buf[cnt] = 0;
3284
3285 ret = trace_set_options(tr, buf);
3286 if (ret < 0)
3287 return ret;
3288
3289 *ppos += cnt;
3290
3291 return cnt;
3292}
3293
3294static int tracing_trace_options_open(struct inode *inode, struct file *file)
3295{
3296 if (tracing_disabled)
3297 return -ENODEV;
3298
3299 return single_open(file, tracing_trace_options_show, inode->i_private);
3300}
3301
3302static const struct file_operations tracing_iter_fops = {
3303 .open = tracing_trace_options_open,
3304 .read = seq_read,
3305 .llseek = seq_lseek,
3306 .release = single_release,
3307 .write = tracing_trace_options_write,
3308};
3309
3310static const char readme_msg[] =
3311 "tracing mini-HOWTO:\n\n"
3312 "# echo 0 > tracing_on : quick way to disable tracing\n"
3313 "# echo 1 > tracing_on : quick way to re-enable tracing\n\n"
3314 " Important files:\n"
3315 " trace\t\t\t- The static contents of the buffer\n"
3316 "\t\t\t To clear the buffer write into this file: echo > trace\n"
3317 " trace_pipe\t\t- A consuming read to see the contents of the buffer\n"
3318 " current_tracer\t- function and latency tracers\n"
3319 " available_tracers\t- list of configured tracers for current_tracer\n"
3320 " buffer_size_kb\t- view and modify size of per cpu buffer\n"
3321 " buffer_total_size_kb - view total size of all cpu buffers\n\n"
3322 " trace_clock\t\t-change the clock used to order events\n"
3323 " local: Per cpu clock but may not be synced across CPUs\n"
3324 " global: Synced across CPUs but slows tracing down.\n"
3325 " counter: Not a clock, but just an increment\n"
3326 " uptime: Jiffy counter from time of boot\n"
3327 " perf: Same clock that perf events use\n"
3328#ifdef CONFIG_X86_64
3329 " x86-tsc: TSC cycle counter\n"
3330#endif
3331 "\n trace_marker\t\t- Writes into this file writes into the kernel buffer\n"
3332 " tracing_cpumask\t- Limit which CPUs to trace\n"
3333 " instances\t\t- Make sub-buffers with: mkdir instances/foo\n"
3334 "\t\t\t Remove sub-buffer with rmdir\n"
3335 " trace_options\t\t- Set format or modify how tracing happens\n"
3336 "\t\t\t Disable an option by adding a suffix 'no' to the option name\n"
3337#ifdef CONFIG_DYNAMIC_FTRACE
3338 "\n available_filter_functions - list of functions that can be filtered on\n"
3339 " set_ftrace_filter\t- echo function name in here to only trace these functions\n"
3340 " accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
3341 " modules: Can select a group via module\n"
3342 " Format: :mod:<module-name>\n"
3343 " example: echo :mod:ext3 > set_ftrace_filter\n"
3344 " triggers: a command to perform when function is hit\n"
3345 " Format: <function>:<trigger>[:count]\n"
3346 " trigger: traceon, traceoff\n"
3347 " enable_event:<system>:<event>\n"
3348 " disable_event:<system>:<event>\n"
3349#ifdef CONFIG_STACKTRACE
3350 " stacktrace\n"
3351#endif
3352#ifdef CONFIG_TRACER_SNAPSHOT
3353 " snapshot\n"
3354#endif
3355 " example: echo do_fault:traceoff > set_ftrace_filter\n"
3356 " echo do_trap:traceoff:3 > set_ftrace_filter\n"
3357 " The first one will disable tracing every time do_fault is hit\n"
3358 " The second will disable tracing at most 3 times when do_trap is hit\n"
3359 " The first time do trap is hit and it disables tracing, the counter\n"
3360 " will decrement to 2. If tracing is already disabled, the counter\n"
3361 " will not decrement. It only decrements when the trigger did work\n"
3362 " To remove trigger without count:\n"
3363 " echo '!<function>:<trigger> > set_ftrace_filter\n"
3364 " To remove trigger with a count:\n"
3365 " echo '!<function>:<trigger>:0 > set_ftrace_filter\n"
3366 " set_ftrace_notrace\t- echo function name in here to never trace.\n"
3367 " accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
3368 " modules: Can select a group via module command :mod:\n"
3369 " Does not accept triggers\n"
3370#endif
3371#ifdef CONFIG_FUNCTION_TRACER
3372 " set_ftrace_pid\t- Write pid(s) to only function trace those pids (function)\n"
3373#endif
3374#ifdef CONFIG_FUNCTION_GRAPH_TRACER
3375 " set_graph_function\t- Trace the nested calls of a function (function_graph)\n"
3376 " max_graph_depth\t- Trace a limited depth of nested calls (0 is unlimited)\n"
3377#endif
3378#ifdef CONFIG_TRACER_SNAPSHOT
3379 "\n snapshot\t\t- Like 'trace' but shows the content of the static snapshot buffer\n"
3380 "\t\t\t Read the contents for more information\n"
3381#endif
3382#ifdef CONFIG_STACKTRACE
3383 " stack_trace\t\t- Shows the max stack trace when active\n"
3384 " stack_max_size\t- Shows current max stack size that was traced\n"
3385 "\t\t\t Write into this file to reset the max size (trigger a new trace)\n"
3386#ifdef CONFIG_DYNAMIC_FTRACE
3387 " stack_trace_filter\t- Like set_ftrace_filter but limits what stack_trace traces\n"
3388#endif
3389#endif
3390;
3391
3392static ssize_t
3393tracing_readme_read(struct file *filp, char __user *ubuf,
3394 size_t cnt, loff_t *ppos)
3395{
3396 return simple_read_from_buffer(ubuf, cnt, ppos,
3397 readme_msg, strlen(readme_msg));
3398}
3399
3400static const struct file_operations tracing_readme_fops = {
3401 .open = tracing_open_generic,
3402 .read = tracing_readme_read,
3403 .llseek = generic_file_llseek,
3404};
3405
3406static ssize_t
3407tracing_saved_cmdlines_read(struct file *file, char __user *ubuf,
3408 size_t cnt, loff_t *ppos)
3409{
3410 char *buf_comm;
3411 char *file_buf;
3412 char *buf;
3413 int len = 0;
3414 int pid;
3415 int i;
3416
3417 file_buf = kmalloc(SAVED_CMDLINES*(16+TASK_COMM_LEN), GFP_KERNEL);
3418 if (!file_buf)
3419 return -ENOMEM;
3420
3421 buf_comm = kmalloc(TASK_COMM_LEN, GFP_KERNEL);
3422 if (!buf_comm) {
3423 kfree(file_buf);
3424 return -ENOMEM;
3425 }
3426
3427 buf = file_buf;
3428
3429 for (i = 0; i < SAVED_CMDLINES; i++) {
3430 int r;
3431
3432 pid = map_cmdline_to_pid[i];
3433 if (pid == -1 || pid == NO_CMDLINE_MAP)
3434 continue;
3435
3436 trace_find_cmdline(pid, buf_comm);
3437 r = sprintf(buf, "%d %s\n", pid, buf_comm);
3438 buf += r;
3439 len += r;
3440 }
3441
3442 len = simple_read_from_buffer(ubuf, cnt, ppos,
3443 file_buf, len);
3444
3445 kfree(file_buf);
3446 kfree(buf_comm);
3447
3448 return len;
3449}
3450
3451static const struct file_operations tracing_saved_cmdlines_fops = {
3452 .open = tracing_open_generic,
3453 .read = tracing_saved_cmdlines_read,
3454 .llseek = generic_file_llseek,
3455};
3456
3457static ssize_t
3458tracing_set_trace_read(struct file *filp, char __user *ubuf,
3459 size_t cnt, loff_t *ppos)
3460{
3461 struct trace_array *tr = filp->private_data;
3462 char buf[MAX_TRACER_SIZE+2];
3463 int r;
3464
3465 mutex_lock(&trace_types_lock);
3466 r = sprintf(buf, "%s\n", tr->current_trace->name);
3467 mutex_unlock(&trace_types_lock);
3468
3469 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3470}
3471
3472int tracer_init(struct tracer *t, struct trace_array *tr)
3473{
3474 tracing_reset_online_cpus(&tr->trace_buffer);
3475 return t->init(tr);
3476}
3477
3478static void set_buffer_entries(struct trace_buffer *buf, unsigned long val)
3479{
3480 int cpu;
3481
3482 for_each_tracing_cpu(cpu)
3483 per_cpu_ptr(buf->data, cpu)->entries = val;
3484}
3485
3486#ifdef CONFIG_TRACER_MAX_TRACE
3487
3488static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
3489 struct trace_buffer *size_buf, int cpu_id)
3490{
3491 int cpu, ret = 0;
3492
3493 if (cpu_id == RING_BUFFER_ALL_CPUS) {
3494 for_each_tracing_cpu(cpu) {
3495 ret = ring_buffer_resize(trace_buf->buffer,
3496 per_cpu_ptr(size_buf->data, cpu)->entries, cpu);
3497 if (ret < 0)
3498 break;
3499 per_cpu_ptr(trace_buf->data, cpu)->entries =
3500 per_cpu_ptr(size_buf->data, cpu)->entries;
3501 }
3502 } else {
3503 ret = ring_buffer_resize(trace_buf->buffer,
3504 per_cpu_ptr(size_buf->data, cpu_id)->entries, cpu_id);
3505 if (ret == 0)
3506 per_cpu_ptr(trace_buf->data, cpu_id)->entries =
3507 per_cpu_ptr(size_buf->data, cpu_id)->entries;
3508 }
3509
3510 return ret;
3511}
3512#endif
3513
3514static int __tracing_resize_ring_buffer(struct trace_array *tr,
3515 unsigned long size, int cpu)
3516{
3517 int ret;
3518
3519
3520
3521
3522
3523
3524 ring_buffer_expanded = true;
3525
3526
3527 if (!tr->trace_buffer.buffer)
3528 return 0;
3529
3530 ret = ring_buffer_resize(tr->trace_buffer.buffer, size, cpu);
3531 if (ret < 0)
3532 return ret;
3533
3534#ifdef CONFIG_TRACER_MAX_TRACE
3535 if (!(tr->flags & TRACE_ARRAY_FL_GLOBAL) ||
3536 !tr->current_trace->use_max_tr)
3537 goto out;
3538
3539 ret = ring_buffer_resize(tr->max_buffer.buffer, size, cpu);
3540 if (ret < 0) {
3541 int r = resize_buffer_duplicate_size(&tr->trace_buffer,
3542 &tr->trace_buffer, cpu);
3543 if (r < 0) {
3544
3545
3546
3547
3548
3549
3550
3551
3552
3553
3554
3555
3556
3557
3558 WARN_ON(1);
3559 tracing_disabled = 1;
3560 }
3561 return ret;
3562 }
3563
3564 if (cpu == RING_BUFFER_ALL_CPUS)
3565 set_buffer_entries(&tr->max_buffer, size);
3566 else
3567 per_cpu_ptr(tr->max_buffer.data, cpu)->entries = size;
3568
3569 out:
3570#endif
3571
3572 if (cpu == RING_BUFFER_ALL_CPUS)
3573 set_buffer_entries(&tr->trace_buffer, size);
3574 else
3575 per_cpu_ptr(tr->trace_buffer.data, cpu)->entries = size;
3576
3577 return ret;
3578}
3579
3580static ssize_t tracing_resize_ring_buffer(struct trace_array *tr,
3581 unsigned long size, int cpu_id)
3582{
3583 int ret = size;
3584
3585 mutex_lock(&trace_types_lock);
3586
3587 if (cpu_id != RING_BUFFER_ALL_CPUS) {
3588
3589 if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) {
3590 ret = -EINVAL;
3591 goto out;
3592 }
3593 }
3594
3595 ret = __tracing_resize_ring_buffer(tr, size, cpu_id);
3596 if (ret < 0)
3597 ret = -ENOMEM;
3598
3599out:
3600 mutex_unlock(&trace_types_lock);
3601
3602 return ret;
3603}
3604
3605
3606
3607
3608
3609
3610
3611
3612
3613
3614
3615
3616int tracing_update_buffers(void)
3617{
3618 int ret = 0;
3619
3620 mutex_lock(&trace_types_lock);
3621 if (!ring_buffer_expanded)
3622 ret = __tracing_resize_ring_buffer(&global_trace, trace_buf_size,
3623 RING_BUFFER_ALL_CPUS);
3624 mutex_unlock(&trace_types_lock);
3625
3626 return ret;
3627}
3628
3629struct trace_option_dentry;
3630
3631static struct trace_option_dentry *
3632create_trace_option_files(struct trace_array *tr, struct tracer *tracer);
3633
3634static void
3635destroy_trace_option_files(struct trace_option_dentry *topts);
3636
3637static int tracing_set_tracer(const char *buf)
3638{
3639 static struct trace_option_dentry *topts;
3640 struct trace_array *tr = &global_trace;
3641 struct tracer *t;
3642#ifdef CONFIG_TRACER_MAX_TRACE
3643 bool had_max_tr;
3644#endif
3645 int ret = 0;
3646
3647 mutex_lock(&trace_types_lock);
3648
3649 if (!ring_buffer_expanded) {
3650 ret = __tracing_resize_ring_buffer(tr, trace_buf_size,
3651 RING_BUFFER_ALL_CPUS);
3652 if (ret < 0)
3653 goto out;
3654 ret = 0;
3655 }
3656
3657 for (t = trace_types; t; t = t->next) {
3658 if (strcmp(t->name, buf) == 0)
3659 break;
3660 }
3661 if (!t) {
3662 ret = -EINVAL;
3663 goto out;
3664 }
3665 if (t == tr->current_trace)
3666 goto out;
3667
3668 trace_branch_disable();
3669
3670 tr->current_trace->enabled = false;
3671
3672 if (tr->current_trace->reset)
3673 tr->current_trace->reset(tr);
3674
3675
3676 tr->current_trace = &nop_trace;
3677
3678#ifdef CONFIG_TRACER_MAX_TRACE
3679 had_max_tr = tr->allocated_snapshot;
3680
3681 if (had_max_tr && !t->use_max_tr) {
3682
3683
3684
3685
3686
3687
3688
3689 synchronize_sched();
3690 free_snapshot(tr);
3691 }
3692#endif
3693 destroy_trace_option_files(topts);
3694
3695 topts = create_trace_option_files(tr, t);
3696
3697#ifdef CONFIG_TRACER_MAX_TRACE
3698 if (t->use_max_tr && !had_max_tr) {
3699 ret = alloc_snapshot(tr);
3700 if (ret < 0)
3701 goto out;
3702 }
3703#endif
3704
3705 if (t->init) {
3706 ret = tracer_init(t, tr);
3707 if (ret)
3708 goto out;
3709 }
3710
3711 tr->current_trace = t;
3712 tr->current_trace->enabled = true;
3713 trace_branch_enable(tr);
3714 out:
3715 mutex_unlock(&trace_types_lock);
3716
3717 return ret;
3718}
3719
3720static ssize_t
3721tracing_set_trace_write(struct file *filp, const char __user *ubuf,
3722 size_t cnt, loff_t *ppos)
3723{
3724 char buf[MAX_TRACER_SIZE+1];
3725 int i;
3726 size_t ret;
3727 int err;
3728
3729 ret = cnt;
3730
3731 if (cnt > MAX_TRACER_SIZE)
3732 cnt = MAX_TRACER_SIZE;
3733
3734 if (copy_from_user(&buf, ubuf, cnt))
3735 return -EFAULT;
3736
3737 buf[cnt] = 0;
3738
3739
3740 for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
3741 buf[i] = 0;
3742
3743 err = tracing_set_tracer(buf);
3744 if (err)
3745 return err;
3746
3747 *ppos += ret;
3748
3749 return ret;
3750}
3751
3752static ssize_t
3753tracing_max_lat_read(struct file *filp, char __user *ubuf,
3754 size_t cnt, loff_t *ppos)
3755{
3756 unsigned long *ptr = filp->private_data;
3757 char buf[64];
3758 int r;
3759
3760 r = snprintf(buf, sizeof(buf), "%ld\n",
3761 *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
3762 if (r > sizeof(buf))
3763 r = sizeof(buf);
3764 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3765}
3766
3767static ssize_t
3768tracing_max_lat_write(struct file *filp, const char __user *ubuf,
3769 size_t cnt, loff_t *ppos)
3770{
3771 unsigned long *ptr = filp->private_data;
3772 unsigned long val;
3773 int ret;
3774
3775 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
3776 if (ret)
3777 return ret;
3778
3779 *ptr = val * 1000;
3780
3781 return cnt;
3782}
3783
3784static int tracing_open_pipe(struct inode *inode, struct file *filp)
3785{
3786 struct trace_cpu *tc = inode->i_private;
3787 struct trace_array *tr = tc->tr;
3788 struct trace_iterator *iter;
3789 int ret = 0;
3790
3791 if (tracing_disabled)
3792 return -ENODEV;
3793
3794 mutex_lock(&trace_types_lock);
3795
3796
3797 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
3798 if (!iter) {
3799 ret = -ENOMEM;
3800 goto out;
3801 }
3802
3803
3804
3805
3806
3807 iter->trace = kmalloc(sizeof(*iter->trace), GFP_KERNEL);
3808 if (!iter->trace) {
3809 ret = -ENOMEM;
3810 goto fail;
3811 }
3812 *iter->trace = *tr->current_trace;
3813
3814 if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
3815 ret = -ENOMEM;
3816 goto fail;
3817 }
3818
3819
3820 cpumask_setall(iter->started);
3821
3822 if (trace_flags & TRACE_ITER_LATENCY_FMT)
3823 iter->iter_flags |= TRACE_FILE_LAT_FMT;
3824
3825
3826 if (trace_clocks[tr->clock_id].in_ns)
3827 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
3828
3829 iter->cpu_file = tc->cpu;
3830 iter->tr = tc->tr;
3831 iter->trace_buffer = &tc->tr->trace_buffer;
3832 mutex_init(&iter->mutex);
3833 filp->private_data = iter;
3834
3835 if (iter->trace->pipe_open)
3836 iter->trace->pipe_open(iter);
3837
3838 nonseekable_open(inode, filp);
3839out:
3840 mutex_unlock(&trace_types_lock);
3841 return ret;
3842
3843fail:
3844 kfree(iter->trace);
3845 kfree(iter);
3846 mutex_unlock(&trace_types_lock);
3847 return ret;
3848}
3849
3850static int tracing_release_pipe(struct inode *inode, struct file *file)
3851{
3852 struct trace_iterator *iter = file->private_data;
3853
3854 mutex_lock(&trace_types_lock);
3855
3856 if (iter->trace->pipe_close)
3857 iter->trace->pipe_close(iter);
3858
3859 mutex_unlock(&trace_types_lock);
3860
3861 free_cpumask_var(iter->started);
3862 mutex_destroy(&iter->mutex);
3863 kfree(iter->trace);
3864 kfree(iter);
3865
3866 return 0;
3867}
3868
3869static unsigned int
3870trace_poll(struct trace_iterator *iter, struct file *filp, poll_table *poll_table)
3871{
3872
3873 if (trace_buffer_iter(iter, iter->cpu_file))
3874 return POLLIN | POLLRDNORM;
3875
3876 if (trace_flags & TRACE_ITER_BLOCK)
3877
3878
3879
3880 return POLLIN | POLLRDNORM;
3881 else
3882 return ring_buffer_poll_wait(iter->trace_buffer->buffer, iter->cpu_file,
3883 filp, poll_table);
3884}
3885
3886static unsigned int
3887tracing_poll_pipe(struct file *filp, poll_table *poll_table)
3888{
3889 struct trace_iterator *iter = filp->private_data;
3890
3891 return trace_poll(iter, filp, poll_table);
3892}
3893
3894
3895
3896
3897
3898
3899
3900
3901
3902
3903
3904
3905
3906void poll_wait_pipe(struct trace_iterator *iter)
3907{
3908 set_current_state(TASK_INTERRUPTIBLE);
3909
3910 schedule_timeout(HZ / 10);
3911}
3912
3913
3914static int tracing_wait_pipe(struct file *filp)
3915{
3916 struct trace_iterator *iter = filp->private_data;
3917
3918 while (trace_empty(iter)) {
3919
3920 if ((filp->f_flags & O_NONBLOCK)) {
3921 return -EAGAIN;
3922 }
3923
3924 mutex_unlock(&iter->mutex);
3925
3926 iter->trace->wait_pipe(iter);
3927
3928 mutex_lock(&iter->mutex);
3929
3930 if (signal_pending(current))
3931 return -EINTR;
3932
3933
3934
3935
3936
3937
3938
3939
3940
3941
3942 if (!tracing_is_enabled() && iter->pos)
3943 break;
3944 }
3945
3946 return 1;
3947}
3948
3949
3950
3951
3952static ssize_t
3953tracing_read_pipe(struct file *filp, char __user *ubuf,
3954 size_t cnt, loff_t *ppos)
3955{
3956 struct trace_iterator *iter = filp->private_data;
3957 struct trace_array *tr = iter->tr;
3958 ssize_t sret;
3959
3960
3961 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
3962 if (sret != -EBUSY)
3963 return sret;
3964
3965 trace_seq_init(&iter->seq);
3966
3967
3968 mutex_lock(&trace_types_lock);
3969 if (unlikely(iter->trace->name != tr->current_trace->name))
3970 *iter->trace = *tr->current_trace;
3971 mutex_unlock(&trace_types_lock);
3972
3973
3974
3975
3976
3977
3978 mutex_lock(&iter->mutex);
3979 if (iter->trace->read) {
3980 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
3981 if (sret)
3982 goto out;
3983 }
3984
3985waitagain:
3986 sret = tracing_wait_pipe(filp);
3987 if (sret <= 0)
3988 goto out;
3989
3990
3991 if (trace_empty(iter)) {
3992 sret = 0;
3993 goto out;
3994 }
3995
3996 if (cnt >= PAGE_SIZE)
3997 cnt = PAGE_SIZE - 1;
3998
3999
4000 memset(&iter->seq, 0,
4001 sizeof(struct trace_iterator) -
4002 offsetof(struct trace_iterator, seq));
4003 iter->pos = -1;
4004
4005 trace_event_read_lock();
4006 trace_access_lock(iter->cpu_file);
4007 while (trace_find_next_entry_inc(iter) != NULL) {
4008 enum print_line_t ret;
4009 int len = iter->seq.len;
4010
4011 ret = print_trace_line(iter);
4012 if (ret == TRACE_TYPE_PARTIAL_LINE) {
4013
4014 iter->seq.len = len;
4015 break;
4016 }
4017 if (ret != TRACE_TYPE_NO_CONSUME)
4018 trace_consume(iter);
4019
4020 if (iter->seq.len >= cnt)
4021 break;
4022
4023
4024
4025
4026
4027
4028 WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
4029 iter->ent->type);
4030 }
4031 trace_access_unlock(iter->cpu_file);
4032 trace_event_read_unlock();
4033
4034
4035 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
4036 if (iter->seq.readpos >= iter->seq.len)
4037 trace_seq_init(&iter->seq);
4038
4039
4040
4041
4042
4043 if (sret == -EBUSY)
4044 goto waitagain;
4045
4046out:
4047 mutex_unlock(&iter->mutex);
4048
4049 return sret;
4050}
4051
4052static void tracing_pipe_buf_release(struct pipe_inode_info *pipe,
4053 struct pipe_buffer *buf)
4054{
4055 __free_page(buf->page);
4056}
4057
4058static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
4059 unsigned int idx)
4060{
4061 __free_page(spd->pages[idx]);
4062}
4063
4064static const struct pipe_buf_operations tracing_pipe_buf_ops = {
4065 .can_merge = 0,
4066 .map = generic_pipe_buf_map,
4067 .unmap = generic_pipe_buf_unmap,
4068 .confirm = generic_pipe_buf_confirm,
4069 .release = tracing_pipe_buf_release,
4070 .steal = generic_pipe_buf_steal,
4071 .get = generic_pipe_buf_get,
4072};
4073
4074static size_t
4075tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
4076{
4077 size_t count;
4078 int ret;
4079
4080
4081 for (;;) {
4082 count = iter->seq.len;
4083 ret = print_trace_line(iter);
4084 count = iter->seq.len - count;
4085 if (rem < count) {
4086 rem = 0;
4087 iter->seq.len -= count;
4088 break;
4089 }
4090 if (ret == TRACE_TYPE_PARTIAL_LINE) {
4091 iter->seq.len -= count;
4092 break;
4093 }
4094
4095 if (ret != TRACE_TYPE_NO_CONSUME)
4096 trace_consume(iter);
4097 rem -= count;
4098 if (!trace_find_next_entry_inc(iter)) {
4099 rem = 0;
4100 iter->ent = NULL;
4101 break;
4102 }
4103 }
4104
4105 return rem;
4106}
4107
4108static ssize_t tracing_splice_read_pipe(struct file *filp,
4109 loff_t *ppos,
4110 struct pipe_inode_info *pipe,
4111 size_t len,
4112 unsigned int flags)
4113{
4114 struct page *pages_def[PIPE_DEF_BUFFERS];
4115 struct partial_page partial_def[PIPE_DEF_BUFFERS];
4116 struct trace_iterator *iter = filp->private_data;
4117 struct splice_pipe_desc spd = {
4118 .pages = pages_def,
4119 .partial = partial_def,
4120 .nr_pages = 0,
4121 .nr_pages_max = PIPE_DEF_BUFFERS,
4122 .flags = flags,
4123 .ops = &tracing_pipe_buf_ops,
4124 .spd_release = tracing_spd_release_pipe,
4125 };
4126 struct trace_array *tr = iter->tr;
4127 ssize_t ret;
4128 size_t rem;
4129 unsigned int i;
4130
4131 if (splice_grow_spd(pipe, &spd))
4132 return -ENOMEM;
4133
4134
4135 mutex_lock(&trace_types_lock);
4136 if (unlikely(iter->trace->name != tr->current_trace->name))
4137 *iter->trace = *tr->current_trace;
4138 mutex_unlock(&trace_types_lock);
4139
4140 mutex_lock(&iter->mutex);
4141
4142 if (iter->trace->splice_read) {
4143 ret = iter->trace->splice_read(iter, filp,
4144 ppos, pipe, len, flags);
4145 if (ret)
4146 goto out_err;
4147 }
4148
4149 ret = tracing_wait_pipe(filp);
4150 if (ret <= 0)
4151 goto out_err;
4152
4153 if (!iter->ent && !trace_find_next_entry_inc(iter)) {
4154 ret = -EFAULT;
4155 goto out_err;
4156 }
4157
4158 trace_event_read_lock();
4159 trace_access_lock(iter->cpu_file);
4160
4161
4162 for (i = 0, rem = len; i < pipe->buffers && rem; i++) {
4163 spd.pages[i] = alloc_page(GFP_KERNEL);
4164 if (!spd.pages[i])
4165 break;
4166
4167 rem = tracing_fill_pipe_page(rem, iter);
4168
4169
4170 ret = trace_seq_to_buffer(&iter->seq,
4171 page_address(spd.pages[i]),
4172 iter->seq.len);
4173 if (ret < 0) {
4174 __free_page(spd.pages[i]);
4175 break;
4176 }
4177 spd.partial[i].offset = 0;
4178 spd.partial[i].len = iter->seq.len;
4179
4180 trace_seq_init(&iter->seq);
4181 }
4182
4183 trace_access_unlock(iter->cpu_file);
4184 trace_event_read_unlock();
4185 mutex_unlock(&iter->mutex);
4186
4187 spd.nr_pages = i;
4188
4189 ret = splice_to_pipe(pipe, &spd);
4190out:
4191 splice_shrink_spd(&spd);
4192 return ret;
4193
4194out_err:
4195 mutex_unlock(&iter->mutex);
4196 goto out;
4197}
4198
4199static ssize_t
4200tracing_entries_read(struct file *filp, char __user *ubuf,
4201 size_t cnt, loff_t *ppos)
4202{
4203 struct trace_cpu *tc = filp->private_data;
4204 struct trace_array *tr = tc->tr;
4205 char buf[64];
4206 int r = 0;
4207 ssize_t ret;
4208
4209 mutex_lock(&trace_types_lock);
4210
4211 if (tc->cpu == RING_BUFFER_ALL_CPUS) {
4212 int cpu, buf_size_same;
4213 unsigned long size;
4214
4215 size = 0;
4216 buf_size_same = 1;
4217
4218 for_each_tracing_cpu(cpu) {
4219
4220 if (size == 0)
4221 size = per_cpu_ptr(tr->trace_buffer.data, cpu)->entries;
4222 if (size != per_cpu_ptr(tr->trace_buffer.data, cpu)->entries) {
4223 buf_size_same = 0;
4224 break;
4225 }
4226 }
4227
4228 if (buf_size_same) {
4229 if (!ring_buffer_expanded)
4230 r = sprintf(buf, "%lu (expanded: %lu)\n",
4231 size >> 10,
4232 trace_buf_size >> 10);
4233 else
4234 r = sprintf(buf, "%lu\n", size >> 10);
4235 } else
4236 r = sprintf(buf, "X\n");
4237 } else
4238 r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->trace_buffer.data, tc->cpu)->entries >> 10);
4239
4240 mutex_unlock(&trace_types_lock);
4241
4242 ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4243 return ret;
4244}
4245
4246static ssize_t
4247tracing_entries_write(struct file *filp, const char __user *ubuf,
4248 size_t cnt, loff_t *ppos)
4249{
4250 struct trace_cpu *tc = filp->private_data;
4251 unsigned long val;
4252 int ret;
4253
4254 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4255 if (ret)
4256 return ret;
4257
4258
4259 if (!val)
4260 return -EINVAL;
4261
4262
4263 val <<= 10;
4264
4265 ret = tracing_resize_ring_buffer(tc->tr, val, tc->cpu);
4266 if (ret < 0)
4267 return ret;
4268
4269 *ppos += cnt;
4270
4271 return cnt;
4272}
4273
4274static ssize_t
4275tracing_total_entries_read(struct file *filp, char __user *ubuf,
4276 size_t cnt, loff_t *ppos)
4277{
4278 struct trace_array *tr = filp->private_data;
4279 char buf[64];
4280 int r, cpu;
4281 unsigned long size = 0, expanded_size = 0;
4282
4283 mutex_lock(&trace_types_lock);
4284 for_each_tracing_cpu(cpu) {
4285 size += per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10;
4286 if (!ring_buffer_expanded)
4287 expanded_size += trace_buf_size >> 10;
4288 }
4289 if (ring_buffer_expanded)
4290 r = sprintf(buf, "%lu\n", size);
4291 else
4292 r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size);
4293 mutex_unlock(&trace_types_lock);
4294
4295 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4296}
4297
4298static ssize_t
4299tracing_free_buffer_write(struct file *filp, const char __user *ubuf,
4300 size_t cnt, loff_t *ppos)
4301{
4302
4303
4304
4305
4306
4307 *ppos += cnt;
4308
4309 return cnt;
4310}
4311
4312static int
4313tracing_free_buffer_release(struct inode *inode, struct file *filp)
4314{
4315 struct trace_array *tr = inode->i_private;
4316
4317
4318 if (trace_flags & TRACE_ITER_STOP_ON_FREE)
4319 tracing_off();
4320
4321 tracing_resize_ring_buffer(tr, 0, RING_BUFFER_ALL_CPUS);
4322
4323 return 0;
4324}
4325
4326static ssize_t
4327tracing_mark_write(struct file *filp, const char __user *ubuf,
4328 size_t cnt, loff_t *fpos)
4329{
4330 unsigned long addr = (unsigned long)ubuf;
4331 struct ring_buffer_event *event;
4332 struct ring_buffer *buffer;
4333 struct print_entry *entry;
4334 unsigned long irq_flags;
4335 struct page *pages[2];
4336 void *map_page[2];
4337 int nr_pages = 1;
4338 ssize_t written;
4339 int offset;
4340 int size;
4341 int len;
4342 int ret;
4343 int i;
4344
4345 if (tracing_disabled)
4346 return -EINVAL;
4347
4348 if (!(trace_flags & TRACE_ITER_MARKERS))
4349 return -EINVAL;
4350
4351 if (cnt > TRACE_BUF_SIZE)
4352 cnt = TRACE_BUF_SIZE;
4353
4354
4355
4356
4357
4358
4359
4360
4361
4362
4363
4364
4365
4366
4367
4368 BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
4369
4370
4371 if ((addr & PAGE_MASK) != ((addr + cnt) & PAGE_MASK))
4372 nr_pages = 2;
4373
4374 offset = addr & (PAGE_SIZE - 1);
4375 addr &= PAGE_MASK;
4376
4377 ret = get_user_pages_fast(addr, nr_pages, 0, pages);
4378 if (ret < nr_pages) {
4379 while (--ret >= 0)
4380 put_page(pages[ret]);
4381 written = -EFAULT;
4382 goto out;
4383 }
4384
4385 for (i = 0; i < nr_pages; i++)
4386 map_page[i] = kmap_atomic(pages[i]);
4387
4388 local_save_flags(irq_flags);
4389 size = sizeof(*entry) + cnt + 2;
4390 buffer = global_trace.trace_buffer.buffer;
4391 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
4392 irq_flags, preempt_count());
4393 if (!event) {
4394
4395 written = -EBADF;
4396 goto out_unlock;
4397 }
4398
4399 entry = ring_buffer_event_data(event);
4400 entry->ip = _THIS_IP_;
4401
4402 if (nr_pages == 2) {
4403 len = PAGE_SIZE - offset;
4404 memcpy(&entry->buf, map_page[0] + offset, len);
4405 memcpy(&entry->buf[len], map_page[1], cnt - len);
4406 } else
4407 memcpy(&entry->buf, map_page[0] + offset, cnt);
4408
4409 if (entry->buf[cnt - 1] != '\n') {
4410 entry->buf[cnt] = '\n';
4411 entry->buf[cnt + 1] = '\0';
4412 } else
4413 entry->buf[cnt] = '\0';
4414
4415 __buffer_unlock_commit(buffer, event);
4416
4417 written = cnt;
4418
4419 *fpos += written;
4420
4421 out_unlock:
4422 for (i = 0; i < nr_pages; i++){
4423 kunmap_atomic(map_page[i]);
4424 put_page(pages[i]);
4425 }
4426 out:
4427 return written;
4428}
4429
4430static int tracing_clock_show(struct seq_file *m, void *v)
4431{
4432 struct trace_array *tr = m->private;
4433 int i;
4434
4435 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
4436 seq_printf(m,
4437 "%s%s%s%s", i ? " " : "",
4438 i == tr->clock_id ? "[" : "", trace_clocks[i].name,
4439 i == tr->clock_id ? "]" : "");
4440 seq_putc(m, '\n');
4441
4442 return 0;
4443}
4444
4445static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
4446 size_t cnt, loff_t *fpos)
4447{
4448 struct seq_file *m = filp->private_data;
4449 struct trace_array *tr = m->private;
4450 char buf[64];
4451 const char *clockstr;
4452 int i;
4453
4454 if (cnt >= sizeof(buf))
4455 return -EINVAL;
4456
4457 if (copy_from_user(&buf, ubuf, cnt))
4458 return -EFAULT;
4459
4460 buf[cnt] = 0;
4461
4462 clockstr = strstrip(buf);
4463
4464 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
4465 if (strcmp(trace_clocks[i].name, clockstr) == 0)
4466 break;
4467 }
4468 if (i == ARRAY_SIZE(trace_clocks))
4469 return -EINVAL;
4470
4471 mutex_lock(&trace_types_lock);
4472
4473 tr->clock_id = i;
4474
4475 ring_buffer_set_clock(tr->trace_buffer.buffer, trace_clocks[i].func);
4476
4477
4478
4479
4480
4481 tracing_reset_online_cpus(&global_trace.trace_buffer);
4482
4483#ifdef CONFIG_TRACER_MAX_TRACE
4484 if (tr->flags & TRACE_ARRAY_FL_GLOBAL && tr->max_buffer.buffer)
4485 ring_buffer_set_clock(tr->max_buffer.buffer, trace_clocks[i].func);
4486 tracing_reset_online_cpus(&global_trace.max_buffer);
4487#endif
4488
4489 mutex_unlock(&trace_types_lock);
4490
4491 *fpos += cnt;
4492
4493 return cnt;
4494}
4495
4496static int tracing_clock_open(struct inode *inode, struct file *file)
4497{
4498 if (tracing_disabled)
4499 return -ENODEV;
4500
4501 return single_open(file, tracing_clock_show, inode->i_private);
4502}
4503
4504struct ftrace_buffer_info {
4505 struct trace_iterator iter;
4506 void *spare;
4507 unsigned int read;
4508};
4509
4510#ifdef CONFIG_TRACER_SNAPSHOT
4511static int tracing_snapshot_open(struct inode *inode, struct file *file)
4512{
4513 struct trace_cpu *tc = inode->i_private;
4514 struct trace_iterator *iter;
4515 struct seq_file *m;
4516 int ret = 0;
4517
4518 if (file->f_mode & FMODE_READ) {
4519 iter = __tracing_open(inode, file, true);
4520 if (IS_ERR(iter))
4521 ret = PTR_ERR(iter);
4522 } else {
4523
4524 m = kzalloc(sizeof(*m), GFP_KERNEL);
4525 if (!m)
4526 return -ENOMEM;
4527 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
4528 if (!iter) {
4529 kfree(m);
4530 return -ENOMEM;
4531 }
4532 iter->tr = tc->tr;
4533 iter->trace_buffer = &tc->tr->max_buffer;
4534 iter->cpu_file = tc->cpu;
4535 m->private = iter;
4536 file->private_data = m;
4537 }
4538
4539 return ret;
4540}
4541
4542static ssize_t
4543tracing_snapshot_write(struct file *filp, const char __user *ubuf, size_t cnt,
4544 loff_t *ppos)
4545{
4546 struct seq_file *m = filp->private_data;
4547 struct trace_iterator *iter = m->private;
4548 struct trace_array *tr = iter->tr;
4549 unsigned long val;
4550 int ret;
4551
4552 ret = tracing_update_buffers();
4553 if (ret < 0)
4554 return ret;
4555
4556 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4557 if (ret)
4558 return ret;
4559
4560 mutex_lock(&trace_types_lock);
4561
4562 if (tr->current_trace->use_max_tr) {
4563 ret = -EBUSY;
4564 goto out;
4565 }
4566
4567 switch (val) {
4568 case 0:
4569 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
4570 ret = -EINVAL;
4571 break;
4572 }
4573 if (tr->allocated_snapshot)
4574 free_snapshot(tr);
4575 break;
4576 case 1:
4577
4578#ifndef CONFIG_RING_BUFFER_ALLOW_SWAP
4579 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
4580 ret = -EINVAL;
4581 break;
4582 }
4583#endif
4584 if (!tr->allocated_snapshot) {
4585 ret = alloc_snapshot(tr);
4586 if (ret < 0)
4587 break;
4588 }
4589 local_irq_disable();
4590
4591 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
4592 update_max_tr(tr, current, smp_processor_id());
4593 else
4594 update_max_tr_single(tr, current, iter->cpu_file);
4595 local_irq_enable();
4596 break;
4597 default:
4598 if (tr->allocated_snapshot) {
4599 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
4600 tracing_reset_online_cpus(&tr->max_buffer);
4601 else
4602 tracing_reset(&tr->max_buffer, iter->cpu_file);
4603 }
4604 break;
4605 }
4606
4607 if (ret >= 0) {
4608 *ppos += cnt;
4609 ret = cnt;
4610 }
4611out:
4612 mutex_unlock(&trace_types_lock);
4613 return ret;
4614}
4615
4616static int tracing_snapshot_release(struct inode *inode, struct file *file)
4617{
4618 struct seq_file *m = file->private_data;
4619
4620 if (file->f_mode & FMODE_READ)
4621 return tracing_release(inode, file);
4622
4623
4624 if (m)
4625 kfree(m->private);
4626 kfree(m);
4627
4628 return 0;
4629}
4630
4631static int tracing_buffers_open(struct inode *inode, struct file *filp);
4632static ssize_t tracing_buffers_read(struct file *filp, char __user *ubuf,
4633 size_t count, loff_t *ppos);
4634static int tracing_buffers_release(struct inode *inode, struct file *file);
4635static ssize_t tracing_buffers_splice_read(struct file *file, loff_t *ppos,
4636 struct pipe_inode_info *pipe, size_t len, unsigned int flags);
4637
4638static int snapshot_raw_open(struct inode *inode, struct file *filp)
4639{
4640 struct ftrace_buffer_info *info;
4641 int ret;
4642
4643 ret = tracing_buffers_open(inode, filp);
4644 if (ret < 0)
4645 return ret;
4646
4647 info = filp->private_data;
4648
4649 if (info->iter.trace->use_max_tr) {
4650 tracing_buffers_release(inode, filp);
4651 return -EBUSY;
4652 }
4653
4654 info->iter.snapshot = true;
4655 info->iter.trace_buffer = &info->iter.tr->max_buffer;
4656
4657 return ret;
4658}
4659
4660#endif
4661
4662
4663static const struct file_operations tracing_max_lat_fops = {
4664 .open = tracing_open_generic,
4665 .read = tracing_max_lat_read,
4666 .write = tracing_max_lat_write,
4667 .llseek = generic_file_llseek,
4668};
4669
4670static const struct file_operations set_tracer_fops = {
4671 .open = tracing_open_generic,
4672 .read = tracing_set_trace_read,
4673 .write = tracing_set_trace_write,
4674 .llseek = generic_file_llseek,
4675};
4676
4677static const struct file_operations tracing_pipe_fops = {
4678 .open = tracing_open_pipe,
4679 .poll = tracing_poll_pipe,
4680 .read = tracing_read_pipe,
4681 .splice_read = tracing_splice_read_pipe,
4682 .release = tracing_release_pipe,
4683 .llseek = no_llseek,
4684};
4685
4686static const struct file_operations tracing_entries_fops = {
4687 .open = tracing_open_generic,
4688 .read = tracing_entries_read,
4689 .write = tracing_entries_write,
4690 .llseek = generic_file_llseek,
4691};
4692
4693static const struct file_operations tracing_total_entries_fops = {
4694 .open = tracing_open_generic,
4695 .read = tracing_total_entries_read,
4696 .llseek = generic_file_llseek,
4697};
4698
4699static const struct file_operations tracing_free_buffer_fops = {
4700 .write = tracing_free_buffer_write,
4701 .release = tracing_free_buffer_release,
4702};
4703
4704static const struct file_operations tracing_mark_fops = {
4705 .open = tracing_open_generic,
4706 .write = tracing_mark_write,
4707 .llseek = generic_file_llseek,
4708};
4709
4710static const struct file_operations trace_clock_fops = {
4711 .open = tracing_clock_open,
4712 .read = seq_read,
4713 .llseek = seq_lseek,
4714 .release = single_release,
4715 .write = tracing_clock_write,
4716};
4717
4718#ifdef CONFIG_TRACER_SNAPSHOT
4719static const struct file_operations snapshot_fops = {
4720 .open = tracing_snapshot_open,
4721 .read = seq_read,
4722 .write = tracing_snapshot_write,
4723 .llseek = tracing_seek,
4724 .release = tracing_snapshot_release,
4725};
4726
4727static const struct file_operations snapshot_raw_fops = {
4728 .open = snapshot_raw_open,
4729 .read = tracing_buffers_read,
4730 .release = tracing_buffers_release,
4731 .splice_read = tracing_buffers_splice_read,
4732 .llseek = no_llseek,
4733};
4734
4735#endif
4736
4737static int tracing_buffers_open(struct inode *inode, struct file *filp)
4738{
4739 struct trace_cpu *tc = inode->i_private;
4740 struct trace_array *tr = tc->tr;
4741 struct ftrace_buffer_info *info;
4742
4743 if (tracing_disabled)
4744 return -ENODEV;
4745
4746 info = kzalloc(sizeof(*info), GFP_KERNEL);
4747 if (!info)
4748 return -ENOMEM;
4749
4750 mutex_lock(&trace_types_lock);
4751
4752 tr->ref++;
4753
4754 info->iter.tr = tr;
4755 info->iter.cpu_file = tc->cpu;
4756 info->iter.trace = tr->current_trace;
4757 info->iter.trace_buffer = &tr->trace_buffer;
4758 info->spare = NULL;
4759
4760 info->read = (unsigned int)-1;
4761
4762 filp->private_data = info;
4763
4764 mutex_unlock(&trace_types_lock);
4765
4766 return nonseekable_open(inode, filp);
4767}
4768
4769static unsigned int
4770tracing_buffers_poll(struct file *filp, poll_table *poll_table)
4771{
4772 struct ftrace_buffer_info *info = filp->private_data;
4773 struct trace_iterator *iter = &info->iter;
4774
4775 return trace_poll(iter, filp, poll_table);
4776}
4777
4778static ssize_t
4779tracing_buffers_read(struct file *filp, char __user *ubuf,
4780 size_t count, loff_t *ppos)
4781{
4782 struct ftrace_buffer_info *info = filp->private_data;
4783 struct trace_iterator *iter = &info->iter;
4784 ssize_t ret;
4785 ssize_t size;
4786
4787 if (!count)
4788 return 0;
4789
4790 mutex_lock(&trace_types_lock);
4791
4792#ifdef CONFIG_TRACER_MAX_TRACE
4793 if (iter->snapshot && iter->tr->current_trace->use_max_tr) {
4794 size = -EBUSY;
4795 goto out_unlock;
4796 }
4797#endif
4798
4799 if (!info->spare)
4800 info->spare = ring_buffer_alloc_read_page(iter->trace_buffer->buffer,
4801 iter->cpu_file);
4802 size = -ENOMEM;
4803 if (!info->spare)
4804 goto out_unlock;
4805
4806
4807 if (info->read < PAGE_SIZE)
4808 goto read;
4809
4810 again:
4811 trace_access_lock(iter->cpu_file);
4812 ret = ring_buffer_read_page(iter->trace_buffer->buffer,
4813 &info->spare,
4814 count,
4815 iter->cpu_file, 0);
4816 trace_access_unlock(iter->cpu_file);
4817
4818 if (ret < 0) {
4819 if (trace_empty(iter)) {
4820 if ((filp->f_flags & O_NONBLOCK)) {
4821 size = -EAGAIN;
4822 goto out_unlock;
4823 }
4824 mutex_unlock(&trace_types_lock);
4825 iter->trace->wait_pipe(iter);
4826 mutex_lock(&trace_types_lock);
4827 if (signal_pending(current)) {
4828 size = -EINTR;
4829 goto out_unlock;
4830 }
4831 goto again;
4832 }
4833 size = 0;
4834 goto out_unlock;
4835 }
4836
4837 info->read = 0;
4838 read:
4839 size = PAGE_SIZE - info->read;
4840 if (size > count)
4841 size = count;
4842
4843 ret = copy_to_user(ubuf, info->spare + info->read, size);
4844 if (ret == size) {
4845 size = -EFAULT;
4846 goto out_unlock;
4847 }
4848 size -= ret;
4849
4850 *ppos += size;
4851 info->read += size;
4852
4853 out_unlock:
4854 mutex_unlock(&trace_types_lock);
4855
4856 return size;
4857}
4858
4859static int tracing_buffers_release(struct inode *inode, struct file *file)
4860{
4861 struct ftrace_buffer_info *info = file->private_data;
4862 struct trace_iterator *iter = &info->iter;
4863
4864 mutex_lock(&trace_types_lock);
4865
4866 WARN_ON(!iter->tr->ref);
4867 iter->tr->ref--;
4868
4869 if (info->spare)
4870 ring_buffer_free_read_page(iter->trace_buffer->buffer, info->spare);
4871 kfree(info);
4872
4873 mutex_unlock(&trace_types_lock);
4874
4875 return 0;
4876}
4877
4878struct buffer_ref {
4879 struct ring_buffer *buffer;
4880 void *page;
4881 int ref;
4882};
4883
4884static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
4885 struct pipe_buffer *buf)
4886{
4887 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
4888
4889 if (--ref->ref)
4890 return;
4891
4892 ring_buffer_free_read_page(ref->buffer, ref->page);
4893 kfree(ref);
4894 buf->private = 0;
4895}
4896
4897static void buffer_pipe_buf_get(struct pipe_inode_info *pipe,
4898 struct pipe_buffer *buf)
4899{
4900 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
4901
4902 ref->ref++;
4903}
4904
4905
4906static const struct pipe_buf_operations buffer_pipe_buf_ops = {
4907 .can_merge = 0,
4908 .map = generic_pipe_buf_map,
4909 .unmap = generic_pipe_buf_unmap,
4910 .confirm = generic_pipe_buf_confirm,
4911 .release = buffer_pipe_buf_release,
4912 .steal = generic_pipe_buf_steal,
4913 .get = buffer_pipe_buf_get,
4914};
4915
4916
4917
4918
4919
4920static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
4921{
4922 struct buffer_ref *ref =
4923 (struct buffer_ref *)spd->partial[i].private;
4924
4925 if (--ref->ref)
4926 return;
4927
4928 ring_buffer_free_read_page(ref->buffer, ref->page);
4929 kfree(ref);
4930 spd->partial[i].private = 0;
4931}
4932
4933static ssize_t
4934tracing_buffers_splice_read(struct file *file, loff_t *ppos,
4935 struct pipe_inode_info *pipe, size_t len,
4936 unsigned int flags)
4937{
4938 struct ftrace_buffer_info *info = file->private_data;
4939 struct trace_iterator *iter = &info->iter;
4940 struct partial_page partial_def[PIPE_DEF_BUFFERS];
4941 struct page *pages_def[PIPE_DEF_BUFFERS];
4942 struct splice_pipe_desc spd = {
4943 .pages = pages_def,
4944 .partial = partial_def,
4945 .nr_pages_max = PIPE_DEF_BUFFERS,
4946 .flags = flags,
4947 .ops = &buffer_pipe_buf_ops,
4948 .spd_release = buffer_spd_release,
4949 };
4950 struct buffer_ref *ref;
4951 int entries, size, i;
4952 ssize_t ret;
4953
4954 mutex_lock(&trace_types_lock);
4955
4956#ifdef CONFIG_TRACER_MAX_TRACE
4957 if (iter->snapshot && iter->tr->current_trace->use_max_tr) {
4958 ret = -EBUSY;
4959 goto out;
4960 }
4961#endif
4962
4963 if (splice_grow_spd(pipe, &spd)) {
4964 ret = -ENOMEM;
4965 goto out;
4966 }
4967
4968 if (*ppos & (PAGE_SIZE - 1)) {
4969 ret = -EINVAL;
4970 goto out;
4971 }
4972
4973 if (len & (PAGE_SIZE - 1)) {
4974 if (len < PAGE_SIZE) {
4975 ret = -EINVAL;
4976 goto out;
4977 }
4978 len &= PAGE_MASK;
4979 }
4980
4981 again:
4982 trace_access_lock(iter->cpu_file);
4983 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
4984
4985 for (i = 0; i < pipe->buffers && len && entries; i++, len -= PAGE_SIZE) {
4986 struct page *page;
4987 int r;
4988
4989 ref = kzalloc(sizeof(*ref), GFP_KERNEL);
4990 if (!ref)
4991 break;
4992
4993 ref->ref = 1;
4994 ref->buffer = iter->trace_buffer->buffer;
4995 ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file);
4996 if (!ref->page) {
4997 kfree(ref);
4998 break;
4999 }
5000
5001 r = ring_buffer_read_page(ref->buffer, &ref->page,
5002 len, iter->cpu_file, 1);
5003 if (r < 0) {
5004 ring_buffer_free_read_page(ref->buffer, ref->page);
5005 kfree(ref);
5006 break;
5007 }
5008
5009
5010
5011
5012
5013 size = ring_buffer_page_len(ref->page);
5014 if (size < PAGE_SIZE)
5015 memset(ref->page + size, 0, PAGE_SIZE - size);
5016
5017 page = virt_to_page(ref->page);
5018
5019 spd.pages[i] = page;
5020 spd.partial[i].len = PAGE_SIZE;
5021 spd.partial[i].offset = 0;
5022 spd.partial[i].private = (unsigned long)ref;
5023 spd.nr_pages++;
5024 *ppos += PAGE_SIZE;
5025
5026 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
5027 }
5028
5029 trace_access_unlock(iter->cpu_file);
5030 spd.nr_pages = i;
5031
5032
5033 if (!spd.nr_pages) {
5034 if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK)) {
5035 ret = -EAGAIN;
5036 goto out;
5037 }
5038 mutex_unlock(&trace_types_lock);
5039 iter->trace->wait_pipe(iter);
5040 mutex_lock(&trace_types_lock);
5041 if (signal_pending(current)) {
5042 ret = -EINTR;
5043 goto out;
5044 }
5045 goto again;
5046 }
5047
5048 ret = splice_to_pipe(pipe, &spd);
5049 splice_shrink_spd(&spd);
5050out:
5051 mutex_unlock(&trace_types_lock);
5052
5053 return ret;
5054}
5055
5056static const struct file_operations tracing_buffers_fops = {
5057 .open = tracing_buffers_open,
5058 .read = tracing_buffers_read,
5059 .poll = tracing_buffers_poll,
5060 .release = tracing_buffers_release,
5061 .splice_read = tracing_buffers_splice_read,
5062 .llseek = no_llseek,
5063};
5064
5065static ssize_t
5066tracing_stats_read(struct file *filp, char __user *ubuf,
5067 size_t count, loff_t *ppos)
5068{
5069 struct trace_cpu *tc = filp->private_data;
5070 struct trace_array *tr = tc->tr;
5071 struct trace_buffer *trace_buf = &tr->trace_buffer;
5072 struct trace_seq *s;
5073 unsigned long cnt;
5074 unsigned long long t;
5075 unsigned long usec_rem;
5076 int cpu = tc->cpu;
5077
5078 s = kmalloc(sizeof(*s), GFP_KERNEL);
5079 if (!s)
5080 return -ENOMEM;
5081
5082 trace_seq_init(s);
5083
5084 cnt = ring_buffer_entries_cpu(trace_buf->buffer, cpu);
5085 trace_seq_printf(s, "entries: %ld\n", cnt);
5086
5087 cnt = ring_buffer_overrun_cpu(trace_buf->buffer, cpu);
5088 trace_seq_printf(s, "overrun: %ld\n", cnt);
5089
5090 cnt = ring_buffer_commit_overrun_cpu(trace_buf->buffer, cpu);
5091 trace_seq_printf(s, "commit overrun: %ld\n", cnt);
5092
5093 cnt = ring_buffer_bytes_cpu(trace_buf->buffer, cpu);
5094 trace_seq_printf(s, "bytes: %ld\n", cnt);
5095
5096 if (trace_clocks[tr->clock_id].in_ns) {
5097
5098 t = ns2usecs(ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
5099 usec_rem = do_div(t, USEC_PER_SEC);
5100 trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n",
5101 t, usec_rem);
5102
5103 t = ns2usecs(ring_buffer_time_stamp(trace_buf->buffer, cpu));
5104 usec_rem = do_div(t, USEC_PER_SEC);
5105 trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem);
5106 } else {
5107
5108 trace_seq_printf(s, "oldest event ts: %llu\n",
5109 ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
5110
5111 trace_seq_printf(s, "now ts: %llu\n",
5112 ring_buffer_time_stamp(trace_buf->buffer, cpu));
5113 }
5114
5115 cnt = ring_buffer_dropped_events_cpu(trace_buf->buffer, cpu);
5116 trace_seq_printf(s, "dropped events: %ld\n", cnt);
5117
5118 cnt = ring_buffer_read_events_cpu(trace_buf->buffer, cpu);
5119 trace_seq_printf(s, "read events: %ld\n", cnt);
5120
5121 count = simple_read_from_buffer(ubuf, count, ppos, s->buffer, s->len);
5122
5123 kfree(s);
5124
5125 return count;
5126}
5127
5128static const struct file_operations tracing_stats_fops = {
5129 .open = tracing_open_generic,
5130 .read = tracing_stats_read,
5131 .llseek = generic_file_llseek,
5132};
5133
5134#ifdef CONFIG_DYNAMIC_FTRACE
5135
5136int __weak ftrace_arch_read_dyn_info(char *buf, int size)
5137{
5138 return 0;
5139}
5140
5141static ssize_t
5142tracing_read_dyn_info(struct file *filp, char __user *ubuf,
5143 size_t cnt, loff_t *ppos)
5144{
5145 static char ftrace_dyn_info_buffer[1024];
5146 static DEFINE_MUTEX(dyn_info_mutex);
5147 unsigned long *p = filp->private_data;
5148 char *buf = ftrace_dyn_info_buffer;
5149 int size = ARRAY_SIZE(ftrace_dyn_info_buffer);
5150 int r;
5151
5152 mutex_lock(&dyn_info_mutex);
5153 r = sprintf(buf, "%ld ", *p);
5154
5155 r += ftrace_arch_read_dyn_info(buf+r, (size-1)-r);
5156 buf[r++] = '\n';
5157
5158 r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5159
5160 mutex_unlock(&dyn_info_mutex);
5161
5162 return r;
5163}
5164
5165static const struct file_operations tracing_dyn_info_fops = {
5166 .open = tracing_open_generic,
5167 .read = tracing_read_dyn_info,
5168 .llseek = generic_file_llseek,
5169};
5170#endif
5171
5172#if defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE)
5173static void
5174ftrace_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
5175{
5176 tracing_snapshot();
5177}
5178
5179static void
5180ftrace_count_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
5181{
5182 unsigned long *count = (long *)data;
5183
5184 if (!*count)
5185 return;
5186
5187 if (*count != -1)
5188 (*count)--;
5189
5190 tracing_snapshot();
5191}
5192
5193static int
5194ftrace_snapshot_print(struct seq_file *m, unsigned long ip,
5195 struct ftrace_probe_ops *ops, void *data)
5196{
5197 long count = (long)data;
5198
5199 seq_printf(m, "%ps:", (void *)ip);
5200
5201 seq_printf(m, "snapshot");
5202
5203 if (count == -1)
5204 seq_printf(m, ":unlimited\n");
5205 else
5206 seq_printf(m, ":count=%ld\n", count);
5207
5208 return 0;
5209}
5210
5211static struct ftrace_probe_ops snapshot_probe_ops = {
5212 .func = ftrace_snapshot,
5213 .print = ftrace_snapshot_print,
5214};
5215
5216static struct ftrace_probe_ops snapshot_count_probe_ops = {
5217 .func = ftrace_count_snapshot,
5218 .print = ftrace_snapshot_print,
5219};
5220
5221static int
5222ftrace_trace_snapshot_callback(struct ftrace_hash *hash,
5223 char *glob, char *cmd, char *param, int enable)
5224{
5225 struct ftrace_probe_ops *ops;
5226 void *count = (void *)-1;
5227 char *number;
5228 int ret;
5229
5230
5231 if (!enable)
5232 return -EINVAL;
5233
5234 ops = param ? &snapshot_count_probe_ops : &snapshot_probe_ops;
5235
5236 if (glob[0] == '!') {
5237 unregister_ftrace_function_probe_func(glob+1, ops);
5238 return 0;
5239 }
5240
5241 if (!param)
5242 goto out_reg;
5243
5244 number = strsep(¶m, ":");
5245
5246 if (!strlen(number))
5247 goto out_reg;
5248
5249
5250
5251
5252
5253 ret = kstrtoul(number, 0, (unsigned long *)&count);
5254 if (ret)
5255 return ret;
5256
5257 out_reg:
5258 ret = register_ftrace_function_probe(glob, ops, count);
5259
5260 if (ret >= 0)
5261 alloc_snapshot(&global_trace);
5262
5263 return ret < 0 ? ret : 0;
5264}
5265
5266static struct ftrace_func_command ftrace_snapshot_cmd = {
5267 .name = "snapshot",
5268 .func = ftrace_trace_snapshot_callback,
5269};
5270
5271static int register_snapshot_cmd(void)
5272{
5273 return register_ftrace_command(&ftrace_snapshot_cmd);
5274}
5275#else
5276static inline int register_snapshot_cmd(void) { return 0; }
5277#endif
5278
5279struct dentry *tracing_init_dentry_tr(struct trace_array *tr)
5280{
5281 if (tr->dir)
5282 return tr->dir;
5283
5284 if (!debugfs_initialized())
5285 return NULL;
5286
5287 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
5288 tr->dir = debugfs_create_dir("tracing", NULL);
5289
5290 if (!tr->dir)
5291 pr_warn_once("Could not create debugfs directory 'tracing'\n");
5292
5293 return tr->dir;
5294}
5295
5296struct dentry *tracing_init_dentry(void)
5297{
5298 return tracing_init_dentry_tr(&global_trace);
5299}
5300
5301static struct dentry *tracing_dentry_percpu(struct trace_array *tr, int cpu)
5302{
5303 struct dentry *d_tracer;
5304
5305 if (tr->percpu_dir)
5306 return tr->percpu_dir;
5307
5308 d_tracer = tracing_init_dentry_tr(tr);
5309 if (!d_tracer)
5310 return NULL;
5311
5312 tr->percpu_dir = debugfs_create_dir("per_cpu", d_tracer);
5313
5314 WARN_ONCE(!tr->percpu_dir,
5315 "Could not create debugfs directory 'per_cpu/%d'\n", cpu);
5316
5317 return tr->percpu_dir;
5318}
5319
5320static void
5321tracing_init_debugfs_percpu(struct trace_array *tr, long cpu)
5322{
5323 struct trace_array_cpu *data = per_cpu_ptr(tr->trace_buffer.data, cpu);
5324 struct dentry *d_percpu = tracing_dentry_percpu(tr, cpu);
5325 struct dentry *d_cpu;
5326 char cpu_dir[30];
5327
5328 if (!d_percpu)
5329 return;
5330
5331 snprintf(cpu_dir, 30, "cpu%ld", cpu);
5332 d_cpu = debugfs_create_dir(cpu_dir, d_percpu);
5333 if (!d_cpu) {
5334 pr_warning("Could not create debugfs '%s' entry\n", cpu_dir);
5335 return;
5336 }
5337
5338
5339 trace_create_file("trace_pipe", 0444, d_cpu,
5340 (void *)&data->trace_cpu, &tracing_pipe_fops);
5341
5342
5343 trace_create_file("trace", 0644, d_cpu,
5344 (void *)&data->trace_cpu, &tracing_fops);
5345
5346 trace_create_file("trace_pipe_raw", 0444, d_cpu,
5347 (void *)&data->trace_cpu, &tracing_buffers_fops);
5348
5349 trace_create_file("stats", 0444, d_cpu,
5350 (void *)&data->trace_cpu, &tracing_stats_fops);
5351
5352 trace_create_file("buffer_size_kb", 0444, d_cpu,
5353 (void *)&data->trace_cpu, &tracing_entries_fops);
5354
5355#ifdef CONFIG_TRACER_SNAPSHOT
5356 trace_create_file("snapshot", 0644, d_cpu,
5357 (void *)&data->trace_cpu, &snapshot_fops);
5358
5359 trace_create_file("snapshot_raw", 0444, d_cpu,
5360 (void *)&data->trace_cpu, &snapshot_raw_fops);
5361#endif
5362}
5363
5364#ifdef CONFIG_FTRACE_SELFTEST
5365
5366#include "trace_selftest.c"
5367#endif
5368
5369struct trace_option_dentry {
5370 struct tracer_opt *opt;
5371 struct tracer_flags *flags;
5372 struct trace_array *tr;
5373 struct dentry *entry;
5374};
5375
5376static ssize_t
5377trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
5378 loff_t *ppos)
5379{
5380 struct trace_option_dentry *topt = filp->private_data;
5381 char *buf;
5382
5383 if (topt->flags->val & topt->opt->bit)
5384 buf = "1\n";
5385 else
5386 buf = "0\n";
5387
5388 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
5389}
5390
5391static ssize_t
5392trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
5393 loff_t *ppos)
5394{
5395 struct trace_option_dentry *topt = filp->private_data;
5396 unsigned long val;
5397 int ret;
5398
5399 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5400 if (ret)
5401 return ret;
5402
5403 if (val != 0 && val != 1)
5404 return -EINVAL;
5405
5406 if (!!(topt->flags->val & topt->opt->bit) != val) {
5407 mutex_lock(&trace_types_lock);
5408 ret = __set_tracer_option(topt->tr->current_trace, topt->flags,
5409 topt->opt, !val);
5410 mutex_unlock(&trace_types_lock);
5411 if (ret)
5412 return ret;
5413 }
5414
5415 *ppos += cnt;
5416
5417 return cnt;
5418}
5419
5420
5421static const struct file_operations trace_options_fops = {
5422 .open = tracing_open_generic,
5423 .read = trace_options_read,
5424 .write = trace_options_write,
5425 .llseek = generic_file_llseek,
5426};
5427
5428static ssize_t
5429trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
5430 loff_t *ppos)
5431{
5432 long index = (long)filp->private_data;
5433 char *buf;
5434
5435 if (trace_flags & (1 << index))
5436 buf = "1\n";
5437 else
5438 buf = "0\n";
5439
5440 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
5441}
5442
5443static ssize_t
5444trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
5445 loff_t *ppos)
5446{
5447 struct trace_array *tr = &global_trace;
5448 long index = (long)filp->private_data;
5449 unsigned long val;
5450 int ret;
5451
5452 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5453 if (ret)
5454 return ret;
5455
5456 if (val != 0 && val != 1)
5457 return -EINVAL;
5458
5459 mutex_lock(&trace_types_lock);
5460 ret = set_tracer_flag(tr, 1 << index, val);
5461 mutex_unlock(&trace_types_lock);
5462
5463 if (ret < 0)
5464 return ret;
5465
5466 *ppos += cnt;
5467
5468 return cnt;
5469}
5470
5471static const struct file_operations trace_options_core_fops = {
5472 .open = tracing_open_generic,
5473 .read = trace_options_core_read,
5474 .write = trace_options_core_write,
5475 .llseek = generic_file_llseek,
5476};
5477
5478struct dentry *trace_create_file(const char *name,
5479 umode_t mode,
5480 struct dentry *parent,
5481 void *data,
5482 const struct file_operations *fops)
5483{
5484 struct dentry *ret;
5485
5486 ret = debugfs_create_file(name, mode, parent, data, fops);
5487 if (!ret)
5488 pr_warning("Could not create debugfs '%s' entry\n", name);
5489
5490 return ret;
5491}
5492
5493
5494static struct dentry *trace_options_init_dentry(struct trace_array *tr)
5495{
5496 struct dentry *d_tracer;
5497
5498 if (tr->options)
5499 return tr->options;
5500
5501 d_tracer = tracing_init_dentry_tr(tr);
5502 if (!d_tracer)
5503 return NULL;
5504
5505 tr->options = debugfs_create_dir("options", d_tracer);
5506 if (!tr->options) {
5507 pr_warning("Could not create debugfs directory 'options'\n");
5508 return NULL;
5509 }
5510
5511 return tr->options;
5512}
5513
5514static void
5515create_trace_option_file(struct trace_array *tr,
5516 struct trace_option_dentry *topt,
5517 struct tracer_flags *flags,
5518 struct tracer_opt *opt)
5519{
5520 struct dentry *t_options;
5521
5522 t_options = trace_options_init_dentry(tr);
5523 if (!t_options)
5524 return;
5525
5526 topt->flags = flags;
5527 topt->opt = opt;
5528 topt->tr = tr;
5529
5530 topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
5531 &trace_options_fops);
5532
5533}
5534
5535static struct trace_option_dentry *
5536create_trace_option_files(struct trace_array *tr, struct tracer *tracer)
5537{
5538 struct trace_option_dentry *topts;
5539 struct tracer_flags *flags;
5540 struct tracer_opt *opts;
5541 int cnt;
5542
5543 if (!tracer)
5544 return NULL;
5545
5546 flags = tracer->flags;
5547
5548 if (!flags || !flags->opts)
5549 return NULL;
5550
5551 opts = flags->opts;
5552
5553 for (cnt = 0; opts[cnt].name; cnt++)
5554 ;
5555
5556 topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
5557 if (!topts)
5558 return NULL;
5559
5560 for (cnt = 0; opts[cnt].name; cnt++)
5561 create_trace_option_file(tr, &topts[cnt], flags,
5562 &opts[cnt]);
5563
5564 return topts;
5565}
5566
5567static void
5568destroy_trace_option_files(struct trace_option_dentry *topts)
5569{
5570 int cnt;
5571
5572 if (!topts)
5573 return;
5574
5575 for (cnt = 0; topts[cnt].opt; cnt++) {
5576 if (topts[cnt].entry)
5577 debugfs_remove(topts[cnt].entry);
5578 }
5579
5580 kfree(topts);
5581}
5582
5583static struct dentry *
5584create_trace_option_core_file(struct trace_array *tr,
5585 const char *option, long index)
5586{
5587 struct dentry *t_options;
5588
5589 t_options = trace_options_init_dentry(tr);
5590 if (!t_options)
5591 return NULL;
5592
5593 return trace_create_file(option, 0644, t_options, (void *)index,
5594 &trace_options_core_fops);
5595}
5596
5597static __init void create_trace_options_dir(struct trace_array *tr)
5598{
5599 struct dentry *t_options;
5600 int i;
5601
5602 t_options = trace_options_init_dentry(tr);
5603 if (!t_options)
5604 return;
5605
5606 for (i = 0; trace_options[i]; i++)
5607 create_trace_option_core_file(tr, trace_options[i], i);
5608}
5609
5610static ssize_t
5611rb_simple_read(struct file *filp, char __user *ubuf,
5612 size_t cnt, loff_t *ppos)
5613{
5614 struct trace_array *tr = filp->private_data;
5615 struct ring_buffer *buffer = tr->trace_buffer.buffer;
5616 char buf[64];
5617 int r;
5618
5619 if (buffer)
5620 r = ring_buffer_record_is_on(buffer);
5621 else
5622 r = 0;
5623
5624 r = sprintf(buf, "%d\n", r);
5625
5626 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5627}
5628
5629static ssize_t
5630rb_simple_write(struct file *filp, const char __user *ubuf,
5631 size_t cnt, loff_t *ppos)
5632{
5633 struct trace_array *tr = filp->private_data;
5634 struct ring_buffer *buffer = tr->trace_buffer.buffer;
5635 unsigned long val;
5636 int ret;
5637
5638 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5639 if (ret)
5640 return ret;
5641
5642 if (buffer) {
5643 mutex_lock(&trace_types_lock);
5644 if (val) {
5645 ring_buffer_record_on(buffer);
5646 if (tr->current_trace->start)
5647 tr->current_trace->start(tr);
5648 } else {
5649 ring_buffer_record_off(buffer);
5650 if (tr->current_trace->stop)
5651 tr->current_trace->stop(tr);
5652 }
5653 mutex_unlock(&trace_types_lock);
5654 }
5655
5656 (*ppos)++;
5657
5658 return cnt;
5659}
5660
5661static const struct file_operations rb_simple_fops = {
5662 .open = tracing_open_generic,
5663 .read = rb_simple_read,
5664 .write = rb_simple_write,
5665 .llseek = default_llseek,
5666};
5667
5668struct dentry *trace_instance_dir;
5669
5670static void
5671init_tracer_debugfs(struct trace_array *tr, struct dentry *d_tracer);
5672
5673static void init_trace_buffers(struct trace_array *tr, struct trace_buffer *buf)
5674{
5675 int cpu;
5676
5677 for_each_tracing_cpu(cpu) {
5678 memset(per_cpu_ptr(buf->data, cpu), 0, sizeof(struct trace_array_cpu));
5679 per_cpu_ptr(buf->data, cpu)->trace_cpu.cpu = cpu;
5680 per_cpu_ptr(buf->data, cpu)->trace_cpu.tr = tr;
5681 }
5682}
5683
5684static int
5685allocate_trace_buffer(struct trace_array *tr, struct trace_buffer *buf, int size)
5686{
5687 enum ring_buffer_flags rb_flags;
5688
5689 rb_flags = trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;
5690
5691 buf->buffer = ring_buffer_alloc(size, rb_flags);
5692 if (!buf->buffer)
5693 return -ENOMEM;
5694
5695 buf->data = alloc_percpu(struct trace_array_cpu);
5696 if (!buf->data) {
5697 ring_buffer_free(buf->buffer);
5698 return -ENOMEM;
5699 }
5700
5701 init_trace_buffers(tr, buf);
5702
5703
5704 set_buffer_entries(&tr->trace_buffer,
5705 ring_buffer_size(tr->trace_buffer.buffer, 0));
5706
5707 return 0;
5708}
5709
5710static int allocate_trace_buffers(struct trace_array *tr, int size)
5711{
5712 int ret;
5713
5714 ret = allocate_trace_buffer(tr, &tr->trace_buffer, size);
5715 if (ret)
5716 return ret;
5717
5718#ifdef CONFIG_TRACER_MAX_TRACE
5719 ret = allocate_trace_buffer(tr, &tr->max_buffer,
5720 allocate_snapshot ? size : 1);
5721 if (WARN_ON(ret)) {
5722 ring_buffer_free(tr->trace_buffer.buffer);
5723 free_percpu(tr->trace_buffer.data);
5724 return -ENOMEM;
5725 }
5726 tr->allocated_snapshot = allocate_snapshot;
5727
5728
5729
5730
5731
5732 allocate_snapshot = false;
5733#endif
5734 return 0;
5735}
5736
5737static int new_instance_create(const char *name)
5738{
5739 struct trace_array *tr;
5740 int ret;
5741
5742 mutex_lock(&trace_types_lock);
5743
5744 ret = -EEXIST;
5745 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
5746 if (tr->name && strcmp(tr->name, name) == 0)
5747 goto out_unlock;
5748 }
5749
5750 ret = -ENOMEM;
5751 tr = kzalloc(sizeof(*tr), GFP_KERNEL);
5752 if (!tr)
5753 goto out_unlock;
5754
5755 tr->name = kstrdup(name, GFP_KERNEL);
5756 if (!tr->name)
5757 goto out_free_tr;
5758
5759 raw_spin_lock_init(&tr->start_lock);
5760
5761 tr->current_trace = &nop_trace;
5762
5763 INIT_LIST_HEAD(&tr->systems);
5764 INIT_LIST_HEAD(&tr->events);
5765
5766 if (allocate_trace_buffers(tr, trace_buf_size) < 0)
5767 goto out_free_tr;
5768
5769
5770 tr->trace_cpu.cpu = RING_BUFFER_ALL_CPUS;
5771 tr->trace_cpu.tr = tr;
5772
5773 tr->dir = debugfs_create_dir(name, trace_instance_dir);
5774 if (!tr->dir)
5775 goto out_free_tr;
5776
5777 ret = event_trace_add_tracer(tr->dir, tr);
5778 if (ret)
5779 goto out_free_tr;
5780
5781 init_tracer_debugfs(tr, tr->dir);
5782
5783 list_add(&tr->list, &ftrace_trace_arrays);
5784
5785 mutex_unlock(&trace_types_lock);
5786
5787 return 0;
5788
5789 out_free_tr:
5790 if (tr->trace_buffer.buffer)
5791 ring_buffer_free(tr->trace_buffer.buffer);
5792 kfree(tr->name);
5793 kfree(tr);
5794
5795 out_unlock:
5796 mutex_unlock(&trace_types_lock);
5797
5798 return ret;
5799
5800}
5801
5802static int instance_delete(const char *name)
5803{
5804 struct trace_array *tr;
5805 int found = 0;
5806 int ret;
5807
5808 mutex_lock(&trace_types_lock);
5809
5810 ret = -ENODEV;
5811 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
5812 if (tr->name && strcmp(tr->name, name) == 0) {
5813 found = 1;
5814 break;
5815 }
5816 }
5817 if (!found)
5818 goto out_unlock;
5819
5820 ret = -EBUSY;
5821 if (tr->ref)
5822 goto out_unlock;
5823
5824 list_del(&tr->list);
5825
5826 event_trace_del_tracer(tr);
5827 debugfs_remove_recursive(tr->dir);
5828 free_percpu(tr->trace_buffer.data);
5829 ring_buffer_free(tr->trace_buffer.buffer);
5830
5831 kfree(tr->name);
5832 kfree(tr);
5833
5834 ret = 0;
5835
5836 out_unlock:
5837 mutex_unlock(&trace_types_lock);
5838
5839 return ret;
5840}
5841
5842static int instance_mkdir (struct inode *inode, struct dentry *dentry, umode_t mode)
5843{
5844 struct dentry *parent;
5845 int ret;
5846
5847
5848 parent = hlist_entry(inode->i_dentry.first, struct dentry, d_alias);
5849 if (WARN_ON_ONCE(parent != trace_instance_dir))
5850 return -ENOENT;
5851
5852
5853
5854
5855
5856
5857
5858
5859
5860 mutex_unlock(&inode->i_mutex);
5861
5862 ret = new_instance_create(dentry->d_iname);
5863
5864 mutex_lock(&inode->i_mutex);
5865
5866 return ret;
5867}
5868
5869static int instance_rmdir(struct inode *inode, struct dentry *dentry)
5870{
5871 struct dentry *parent;
5872 int ret;
5873
5874
5875 parent = hlist_entry(inode->i_dentry.first, struct dentry, d_alias);
5876 if (WARN_ON_ONCE(parent != trace_instance_dir))
5877 return -ENOENT;
5878
5879
5880 mutex_unlock(&dentry->d_inode->i_mutex);
5881
5882
5883
5884
5885
5886
5887
5888
5889
5890 mutex_unlock(&inode->i_mutex);
5891
5892 ret = instance_delete(dentry->d_iname);
5893
5894 mutex_lock_nested(&inode->i_mutex, I_MUTEX_PARENT);
5895 mutex_lock(&dentry->d_inode->i_mutex);
5896
5897 return ret;
5898}
5899
5900static const struct inode_operations instance_dir_inode_operations = {
5901 .lookup = simple_lookup,
5902 .mkdir = instance_mkdir,
5903 .rmdir = instance_rmdir,
5904};
5905
5906static __init void create_trace_instances(struct dentry *d_tracer)
5907{
5908 trace_instance_dir = debugfs_create_dir("instances", d_tracer);
5909 if (WARN_ON(!trace_instance_dir))
5910 return;
5911
5912
5913 trace_instance_dir->d_inode->i_op = &instance_dir_inode_operations;
5914}
5915
5916static void
5917init_tracer_debugfs(struct trace_array *tr, struct dentry *d_tracer)
5918{
5919 int cpu;
5920
5921 trace_create_file("trace_options", 0644, d_tracer,
5922 tr, &tracing_iter_fops);
5923
5924 trace_create_file("trace", 0644, d_tracer,
5925 (void *)&tr->trace_cpu, &tracing_fops);
5926
5927 trace_create_file("trace_pipe", 0444, d_tracer,
5928 (void *)&tr->trace_cpu, &tracing_pipe_fops);
5929
5930 trace_create_file("buffer_size_kb", 0644, d_tracer,
5931 (void *)&tr->trace_cpu, &tracing_entries_fops);
5932
5933 trace_create_file("buffer_total_size_kb", 0444, d_tracer,
5934 tr, &tracing_total_entries_fops);
5935
5936 trace_create_file("free_buffer", 0644, d_tracer,
5937 tr, &tracing_free_buffer_fops);
5938
5939 trace_create_file("trace_marker", 0220, d_tracer,
5940 tr, &tracing_mark_fops);
5941
5942 trace_create_file("trace_clock", 0644, d_tracer, tr,
5943 &trace_clock_fops);
5944
5945 trace_create_file("tracing_on", 0644, d_tracer,
5946 tr, &rb_simple_fops);
5947
5948#ifdef CONFIG_TRACER_SNAPSHOT
5949 trace_create_file("snapshot", 0644, d_tracer,
5950 (void *)&tr->trace_cpu, &snapshot_fops);
5951#endif
5952
5953 for_each_tracing_cpu(cpu)
5954 tracing_init_debugfs_percpu(tr, cpu);
5955
5956}
5957
5958static __init int tracer_init_debugfs(void)
5959{
5960 struct dentry *d_tracer;
5961
5962 trace_access_lock_init();
5963
5964 d_tracer = tracing_init_dentry();
5965 if (!d_tracer)
5966 return 0;
5967
5968 init_tracer_debugfs(&global_trace, d_tracer);
5969
5970 trace_create_file("tracing_cpumask", 0644, d_tracer,
5971 &global_trace, &tracing_cpumask_fops);
5972
5973 trace_create_file("available_tracers", 0444, d_tracer,
5974 &global_trace, &show_traces_fops);
5975
5976 trace_create_file("current_tracer", 0644, d_tracer,
5977 &global_trace, &set_tracer_fops);
5978
5979#ifdef CONFIG_TRACER_MAX_TRACE
5980 trace_create_file("tracing_max_latency", 0644, d_tracer,
5981 &tracing_max_latency, &tracing_max_lat_fops);
5982#endif
5983
5984 trace_create_file("tracing_thresh", 0644, d_tracer,
5985 &tracing_thresh, &tracing_max_lat_fops);
5986
5987 trace_create_file("README", 0444, d_tracer,
5988 NULL, &tracing_readme_fops);
5989
5990 trace_create_file("saved_cmdlines", 0444, d_tracer,
5991 NULL, &tracing_saved_cmdlines_fops);
5992
5993#ifdef CONFIG_DYNAMIC_FTRACE
5994 trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
5995 &ftrace_update_tot_cnt, &tracing_dyn_info_fops);
5996#endif
5997
5998 create_trace_instances(d_tracer);
5999
6000 create_trace_options_dir(&global_trace);
6001
6002 return 0;
6003}
6004
6005static int trace_panic_handler(struct notifier_block *this,
6006 unsigned long event, void *unused)
6007{
6008 if (ftrace_dump_on_oops)
6009 ftrace_dump(ftrace_dump_on_oops);
6010 return NOTIFY_OK;
6011}
6012
6013static struct notifier_block trace_panic_notifier = {
6014 .notifier_call = trace_panic_handler,
6015 .next = NULL,
6016 .priority = 150
6017};
6018
6019static int trace_die_handler(struct notifier_block *self,
6020 unsigned long val,
6021 void *data)
6022{
6023 switch (val) {
6024 case DIE_OOPS:
6025 if (ftrace_dump_on_oops)
6026 ftrace_dump(ftrace_dump_on_oops);
6027 break;
6028 default:
6029 break;
6030 }
6031 return NOTIFY_OK;
6032}
6033
6034static struct notifier_block trace_die_notifier = {
6035 .notifier_call = trace_die_handler,
6036 .priority = 200
6037};
6038
6039
6040
6041
6042
6043#define TRACE_MAX_PRINT 1000
6044
6045
6046
6047
6048
6049
6050#define KERN_TRACE KERN_EMERG
6051
6052void
6053trace_printk_seq(struct trace_seq *s)
6054{
6055
6056 if (s->len >= TRACE_MAX_PRINT)
6057 s->len = TRACE_MAX_PRINT;
6058
6059
6060 s->buffer[s->len] = 0;
6061
6062 printk(KERN_TRACE "%s", s->buffer);
6063
6064 trace_seq_init(s);
6065}
6066
6067void trace_init_global_iter(struct trace_iterator *iter)
6068{
6069 iter->tr = &global_trace;
6070 iter->trace = iter->tr->current_trace;
6071 iter->cpu_file = RING_BUFFER_ALL_CPUS;
6072 iter->trace_buffer = &global_trace.trace_buffer;
6073}
6074
6075void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
6076{
6077
6078 static struct trace_iterator iter;
6079 static atomic_t dump_running;
6080 unsigned int old_userobj;
6081 unsigned long flags;
6082 int cnt = 0, cpu;
6083
6084
6085 if (atomic_inc_return(&dump_running) != 1) {
6086 atomic_dec(&dump_running);
6087 return;
6088 }
6089
6090
6091
6092
6093
6094
6095
6096
6097
6098 tracing_off();
6099
6100 local_irq_save(flags);
6101
6102
6103 trace_init_global_iter(&iter);
6104
6105 for_each_tracing_cpu(cpu) {
6106 atomic_inc(&per_cpu_ptr(iter.tr->trace_buffer.data, cpu)->disabled);
6107 }
6108
6109 old_userobj = trace_flags & TRACE_ITER_SYM_USEROBJ;
6110
6111
6112 trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
6113
6114 switch (oops_dump_mode) {
6115 case DUMP_ALL:
6116 iter.cpu_file = RING_BUFFER_ALL_CPUS;
6117 break;
6118 case DUMP_ORIG:
6119 iter.cpu_file = raw_smp_processor_id();
6120 break;
6121 case DUMP_NONE:
6122 goto out_enable;
6123 default:
6124 printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
6125 iter.cpu_file = RING_BUFFER_ALL_CPUS;
6126 }
6127
6128 printk(KERN_TRACE "Dumping ftrace buffer:\n");
6129
6130
6131 if (ftrace_is_dead()) {
6132 printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
6133 printk("# MAY BE MISSING FUNCTION EVENTS\n");
6134 }
6135
6136
6137
6138
6139
6140
6141
6142
6143 while (!trace_empty(&iter)) {
6144
6145 if (!cnt)
6146 printk(KERN_TRACE "---------------------------------\n");
6147
6148 cnt++;
6149
6150
6151 memset(&iter.seq, 0,
6152 sizeof(struct trace_iterator) -
6153 offsetof(struct trace_iterator, seq));
6154 iter.iter_flags |= TRACE_FILE_LAT_FMT;
6155 iter.pos = -1;
6156
6157 if (trace_find_next_entry_inc(&iter) != NULL) {
6158 int ret;
6159
6160 ret = print_trace_line(&iter);
6161 if (ret != TRACE_TYPE_NO_CONSUME)
6162 trace_consume(&iter);
6163 }
6164 touch_nmi_watchdog();
6165
6166 trace_printk_seq(&iter.seq);
6167 }
6168
6169 if (!cnt)
6170 printk(KERN_TRACE " (ftrace buffer empty)\n");
6171 else
6172 printk(KERN_TRACE "---------------------------------\n");
6173
6174 out_enable:
6175 trace_flags |= old_userobj;
6176
6177 for_each_tracing_cpu(cpu) {
6178 atomic_dec(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled);
6179 }
6180 atomic_dec(&dump_running);
6181 local_irq_restore(flags);
6182}
6183EXPORT_SYMBOL_GPL(ftrace_dump);
6184
6185__init static int tracer_alloc_buffers(void)
6186{
6187 int ring_buf_size;
6188 int ret = -ENOMEM;
6189
6190
6191 if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
6192 goto out;
6193
6194 if (!alloc_cpumask_var(&tracing_cpumask, GFP_KERNEL))
6195 goto out_free_buffer_mask;
6196
6197
6198 if (__stop___trace_bprintk_fmt != __start___trace_bprintk_fmt)
6199
6200 trace_printk_init_buffers();
6201
6202
6203 if (ring_buffer_expanded)
6204 ring_buf_size = trace_buf_size;
6205 else
6206 ring_buf_size = 1;
6207
6208 cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
6209 cpumask_copy(tracing_cpumask, cpu_all_mask);
6210
6211 raw_spin_lock_init(&global_trace.start_lock);
6212
6213
6214 if (allocate_trace_buffers(&global_trace, ring_buf_size) < 0) {
6215 printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
6216 WARN_ON(1);
6217 goto out_free_cpumask;
6218 }
6219
6220 if (global_trace.buffer_disabled)
6221 tracing_off();
6222
6223 trace_init_cmdlines();
6224
6225
6226
6227
6228
6229
6230 global_trace.current_trace = &nop_trace;
6231
6232 register_tracer(&nop_trace);
6233
6234
6235 tracing_disabled = 0;
6236
6237 atomic_notifier_chain_register(&panic_notifier_list,
6238 &trace_panic_notifier);
6239
6240 register_die_notifier(&trace_die_notifier);
6241
6242 global_trace.flags = TRACE_ARRAY_FL_GLOBAL;
6243
6244
6245 global_trace.trace_cpu.cpu = RING_BUFFER_ALL_CPUS;
6246 global_trace.trace_cpu.tr = &global_trace;
6247
6248 INIT_LIST_HEAD(&global_trace.systems);
6249 INIT_LIST_HEAD(&global_trace.events);
6250 list_add(&global_trace.list, &ftrace_trace_arrays);
6251
6252 while (trace_boot_options) {
6253 char *option;
6254
6255 option = strsep(&trace_boot_options, ",");
6256 trace_set_options(&global_trace, option);
6257 }
6258
6259 register_snapshot_cmd();
6260
6261 return 0;
6262
6263out_free_cpumask:
6264 free_percpu(global_trace.trace_buffer.data);
6265#ifdef CONFIG_TRACER_MAX_TRACE
6266 free_percpu(global_trace.max_buffer.data);
6267#endif
6268 free_cpumask_var(tracing_cpumask);
6269out_free_buffer_mask:
6270 free_cpumask_var(tracing_buffer_mask);
6271out:
6272 return ret;
6273}
6274
6275__init static int clear_boot_tracer(void)
6276{
6277
6278
6279
6280
6281
6282
6283
6284 if (!default_bootup_tracer)
6285 return 0;
6286
6287 printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
6288 default_bootup_tracer);
6289 default_bootup_tracer = NULL;
6290
6291 return 0;
6292}
6293
6294early_initcall(tracer_alloc_buffers);
6295fs_initcall(tracer_init_debugfs);
6296late_initcall(clear_boot_tracer);
6297