1
2
3#ifndef _LINUX_KERNEL_TRACE_H
4#define _LINUX_KERNEL_TRACE_H
5
6#include <linux/fs.h>
7#include <linux/atomic.h>
8#include <linux/sched.h>
9#include <linux/clocksource.h>
10#include <linux/ring_buffer.h>
11#include <linux/mmiotrace.h>
12#include <linux/tracepoint.h>
13#include <linux/ftrace.h>
14#include <linux/trace.h>
15#include <linux/hw_breakpoint.h>
16#include <linux/trace_seq.h>
17#include <linux/trace_events.h>
18#include <linux/compiler.h>
19#include <linux/glob.h>
20#include <linux/irq_work.h>
21#include <linux/workqueue.h>
22
23#ifdef CONFIG_FTRACE_SYSCALLS
24#include <asm/unistd.h>
25#include <asm/syscall.h>
26#endif
27
28enum trace_type {
29 __TRACE_FIRST_TYPE = 0,
30
31 TRACE_FN,
32 TRACE_CTX,
33 TRACE_WAKE,
34 TRACE_STACK,
35 TRACE_PRINT,
36 TRACE_BPRINT,
37 TRACE_MMIO_RW,
38 TRACE_MMIO_MAP,
39 TRACE_BRANCH,
40 TRACE_GRAPH_RET,
41 TRACE_GRAPH_ENT,
42 TRACE_USER_STACK,
43 TRACE_BLK,
44 TRACE_BPUTS,
45 TRACE_HWLAT,
46 TRACE_RAW_DATA,
47
48 __TRACE_LAST_TYPE,
49};
50
51
52#undef __field
53#define __field(type, item) type item;
54
55#undef __field_fn
56#define __field_fn(type, item) type item;
57
58#undef __field_struct
59#define __field_struct(type, item) __field(type, item)
60
61#undef __field_desc
62#define __field_desc(type, container, item)
63
64#undef __field_packed
65#define __field_packed(type, container, item)
66
67#undef __array
68#define __array(type, item, size) type item[size];
69
70#undef __array_desc
71#define __array_desc(type, container, item, size)
72
73#undef __dynamic_array
74#define __dynamic_array(type, item) type item[];
75
76#undef F_STRUCT
77#define F_STRUCT(args...) args
78
79#undef FTRACE_ENTRY
80#define FTRACE_ENTRY(name, struct_name, id, tstruct, print) \
81 struct struct_name { \
82 struct trace_entry ent; \
83 tstruct \
84 }
85
86#undef FTRACE_ENTRY_DUP
87#define FTRACE_ENTRY_DUP(name, name_struct, id, tstruct, printk)
88
89#undef FTRACE_ENTRY_REG
90#define FTRACE_ENTRY_REG(name, struct_name, id, tstruct, print, regfn) \
91 FTRACE_ENTRY(name, struct_name, id, PARAMS(tstruct), PARAMS(print))
92
93#undef FTRACE_ENTRY_PACKED
94#define FTRACE_ENTRY_PACKED(name, struct_name, id, tstruct, print) \
95 FTRACE_ENTRY(name, struct_name, id, PARAMS(tstruct), PARAMS(print)) __packed
96
97#include "trace_entries.h"
98
99
100#define MEM_FAIL(condition, fmt, ...) ({ \
101 static bool __section(.data.once) __warned; \
102 int __ret_warn_once = !!(condition); \
103 \
104 if (unlikely(__ret_warn_once && !__warned)) { \
105 __warned = true; \
106 pr_err("ERROR: " fmt, ##__VA_ARGS__); \
107 } \
108 unlikely(__ret_warn_once); \
109})
110
111
112
113
114
115struct syscall_trace_enter {
116 struct trace_entry ent;
117 int nr;
118 unsigned long args[];
119};
120
121struct syscall_trace_exit {
122 struct trace_entry ent;
123 int nr;
124 long ret;
125};
126
127struct kprobe_trace_entry_head {
128 struct trace_entry ent;
129 unsigned long ip;
130};
131
132struct kretprobe_trace_entry_head {
133 struct trace_entry ent;
134 unsigned long func;
135 unsigned long ret_ip;
136};
137
138
139
140
141
142
143
144
145
146
147enum trace_flag_type {
148 TRACE_FLAG_IRQS_OFF = 0x01,
149 TRACE_FLAG_IRQS_NOSUPPORT = 0x02,
150 TRACE_FLAG_NEED_RESCHED = 0x04,
151 TRACE_FLAG_HARDIRQ = 0x08,
152 TRACE_FLAG_SOFTIRQ = 0x10,
153 TRACE_FLAG_PREEMPT_RESCHED = 0x20,
154 TRACE_FLAG_NMI = 0x40,
155};
156
157#define TRACE_BUF_SIZE 1024
158
159struct trace_array;
160
161
162
163
164
165
166struct trace_array_cpu {
167 atomic_t disabled;
168 void *buffer_page;
169
170 unsigned long entries;
171 unsigned long saved_latency;
172 unsigned long critical_start;
173 unsigned long critical_end;
174 unsigned long critical_sequence;
175 unsigned long nice;
176 unsigned long policy;
177 unsigned long rt_priority;
178 unsigned long skipped_entries;
179 u64 preempt_timestamp;
180 pid_t pid;
181 kuid_t uid;
182 char comm[TASK_COMM_LEN];
183
184#ifdef CONFIG_FUNCTION_TRACER
185 int ftrace_ignore_pid;
186#endif
187 bool ignore_pid;
188};
189
190struct tracer;
191struct trace_option_dentry;
192
193struct array_buffer {
194 struct trace_array *tr;
195 struct trace_buffer *buffer;
196 struct trace_array_cpu __percpu *data;
197 u64 time_start;
198 int cpu;
199};
200
201#define TRACE_FLAGS_MAX_SIZE 32
202
203struct trace_options {
204 struct tracer *tracer;
205 struct trace_option_dentry *topts;
206};
207
208struct trace_pid_list {
209 int pid_max;
210 unsigned long *pids;
211};
212
213enum {
214 TRACE_PIDS = BIT(0),
215 TRACE_NO_PIDS = BIT(1),
216};
217
218static inline bool pid_type_enabled(int type, struct trace_pid_list *pid_list,
219 struct trace_pid_list *no_pid_list)
220{
221
222 return ((type & TRACE_PIDS) && pid_list) ||
223 ((type & TRACE_NO_PIDS) && no_pid_list);
224}
225
226static inline bool still_need_pid_events(int type, struct trace_pid_list *pid_list,
227 struct trace_pid_list *no_pid_list)
228{
229
230
231
232
233 return (!(type & TRACE_PIDS) && pid_list) ||
234 (!(type & TRACE_NO_PIDS) && no_pid_list);
235}
236
237typedef bool (*cond_update_fn_t)(struct trace_array *tr, void *cond_data);
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277struct cond_snapshot {
278 void *cond_data;
279 cond_update_fn_t update;
280};
281
282
283
284
285
286
287struct trace_array {
288 struct list_head list;
289 char *name;
290 struct array_buffer array_buffer;
291#ifdef CONFIG_TRACER_MAX_TRACE
292
293
294
295
296
297
298
299
300
301
302
303 struct array_buffer max_buffer;
304 bool allocated_snapshot;
305#endif
306#if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)
307 unsigned long max_latency;
308#ifdef CONFIG_FSNOTIFY
309 struct dentry *d_max_latency;
310 struct work_struct fsnotify_work;
311 struct irq_work fsnotify_irqwork;
312#endif
313#endif
314 struct trace_pid_list __rcu *filtered_pids;
315 struct trace_pid_list __rcu *filtered_no_pids;
316
317
318
319
320
321
322
323
324
325
326
327
328
329 arch_spinlock_t max_lock;
330 int buffer_disabled;
331#ifdef CONFIG_FTRACE_SYSCALLS
332 int sys_refcount_enter;
333 int sys_refcount_exit;
334 struct trace_event_file __rcu *enter_syscall_files[NR_syscalls];
335 struct trace_event_file __rcu *exit_syscall_files[NR_syscalls];
336#endif
337 int stop_count;
338 int clock_id;
339 int nr_topts;
340 bool clear_trace;
341 int buffer_percent;
342 unsigned int n_err_log_entries;
343 struct tracer *current_trace;
344 unsigned int trace_flags;
345 unsigned char trace_flags_index[TRACE_FLAGS_MAX_SIZE];
346 unsigned int flags;
347 raw_spinlock_t start_lock;
348 struct list_head err_log;
349 struct dentry *dir;
350 struct dentry *options;
351 struct dentry *percpu_dir;
352 struct dentry *event_dir;
353 struct trace_options *topts;
354 struct list_head systems;
355 struct list_head events;
356 struct trace_event_file *trace_marker_file;
357 cpumask_var_t tracing_cpumask;
358 int ref;
359 int trace_ref;
360#ifdef CONFIG_FUNCTION_TRACER
361 struct ftrace_ops *ops;
362 struct trace_pid_list __rcu *function_pids;
363 struct trace_pid_list __rcu *function_no_pids;
364#ifdef CONFIG_DYNAMIC_FTRACE
365
366 struct list_head func_probes;
367 struct list_head mod_trace;
368 struct list_head mod_notrace;
369#endif
370
371 int function_enabled;
372#endif
373 int time_stamp_abs_ref;
374 struct list_head hist_vars;
375#ifdef CONFIG_TRACER_SNAPSHOT
376 struct cond_snapshot *cond_snapshot;
377#endif
378};
379
380enum {
381 TRACE_ARRAY_FL_GLOBAL = (1 << 0)
382};
383
384extern struct list_head ftrace_trace_arrays;
385
386extern struct mutex trace_types_lock;
387
388extern int trace_array_get(struct trace_array *tr);
389extern int tracing_check_open_get_tr(struct trace_array *tr);
390extern struct trace_array *trace_array_find(const char *instance);
391extern struct trace_array *trace_array_find_get(const char *instance);
392
393extern int tracing_set_time_stamp_abs(struct trace_array *tr, bool abs);
394extern int tracing_set_clock(struct trace_array *tr, const char *clockstr);
395
396extern bool trace_clock_in_ns(struct trace_array *tr);
397
398
399
400
401
402static inline struct trace_array *top_trace_array(void)
403{
404 struct trace_array *tr;
405
406 if (list_empty(&ftrace_trace_arrays))
407 return NULL;
408
409 tr = list_entry(ftrace_trace_arrays.prev,
410 typeof(*tr), list);
411 WARN_ON(!(tr->flags & TRACE_ARRAY_FL_GLOBAL));
412 return tr;
413}
414
415#define FTRACE_CMP_TYPE(var, type) \
416 __builtin_types_compatible_p(typeof(var), type *)
417
418#undef IF_ASSIGN
419#define IF_ASSIGN(var, entry, etype, id) \
420 if (FTRACE_CMP_TYPE(var, etype)) { \
421 var = (typeof(var))(entry); \
422 WARN_ON(id != 0 && (entry)->type != id); \
423 break; \
424 }
425
426
427extern void __ftrace_bad_type(void);
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442#define trace_assign_type(var, ent) \
443 do { \
444 IF_ASSIGN(var, ent, struct ftrace_entry, TRACE_FN); \
445 IF_ASSIGN(var, ent, struct ctx_switch_entry, 0); \
446 IF_ASSIGN(var, ent, struct stack_entry, TRACE_STACK); \
447 IF_ASSIGN(var, ent, struct userstack_entry, TRACE_USER_STACK);\
448 IF_ASSIGN(var, ent, struct print_entry, TRACE_PRINT); \
449 IF_ASSIGN(var, ent, struct bprint_entry, TRACE_BPRINT); \
450 IF_ASSIGN(var, ent, struct bputs_entry, TRACE_BPUTS); \
451 IF_ASSIGN(var, ent, struct hwlat_entry, TRACE_HWLAT); \
452 IF_ASSIGN(var, ent, struct raw_data_entry, TRACE_RAW_DATA);\
453 IF_ASSIGN(var, ent, struct trace_mmiotrace_rw, \
454 TRACE_MMIO_RW); \
455 IF_ASSIGN(var, ent, struct trace_mmiotrace_map, \
456 TRACE_MMIO_MAP); \
457 IF_ASSIGN(var, ent, struct trace_branch, TRACE_BRANCH); \
458 IF_ASSIGN(var, ent, struct ftrace_graph_ent_entry, \
459 TRACE_GRAPH_ENT); \
460 IF_ASSIGN(var, ent, struct ftrace_graph_ret_entry, \
461 TRACE_GRAPH_RET); \
462 __ftrace_bad_type(); \
463 } while (0)
464
465
466
467
468
469
470struct tracer_opt {
471 const char *name;
472 u32 bit;
473};
474
475
476
477
478
479struct tracer_flags {
480 u32 val;
481 struct tracer_opt *opts;
482 struct tracer *trace;
483};
484
485
486#define TRACER_OPT(s, b) .name = #s, .bit = b
487
488
489struct trace_option_dentry {
490 struct tracer_opt *opt;
491 struct tracer_flags *flags;
492 struct trace_array *tr;
493 struct dentry *entry;
494};
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516struct tracer {
517 const char *name;
518 int (*init)(struct trace_array *tr);
519 void (*reset)(struct trace_array *tr);
520 void (*start)(struct trace_array *tr);
521 void (*stop)(struct trace_array *tr);
522 int (*update_thresh)(struct trace_array *tr);
523 void (*open)(struct trace_iterator *iter);
524 void (*pipe_open)(struct trace_iterator *iter);
525 void (*close)(struct trace_iterator *iter);
526 void (*pipe_close)(struct trace_iterator *iter);
527 ssize_t (*read)(struct trace_iterator *iter,
528 struct file *filp, char __user *ubuf,
529 size_t cnt, loff_t *ppos);
530 ssize_t (*splice_read)(struct trace_iterator *iter,
531 struct file *filp,
532 loff_t *ppos,
533 struct pipe_inode_info *pipe,
534 size_t len,
535 unsigned int flags);
536#ifdef CONFIG_FTRACE_STARTUP_TEST
537 int (*selftest)(struct tracer *trace,
538 struct trace_array *tr);
539#endif
540 void (*print_header)(struct seq_file *m);
541 enum print_line_t (*print_line)(struct trace_iterator *iter);
542
543 int (*set_flag)(struct trace_array *tr,
544 u32 old_flags, u32 bit, int set);
545
546 int (*flag_changed)(struct trace_array *tr,
547 u32 mask, int set);
548 struct tracer *next;
549 struct tracer_flags *flags;
550 int enabled;
551 bool print_max;
552 bool allow_instances;
553#ifdef CONFIG_TRACER_MAX_TRACE
554 bool use_max_tr;
555#endif
556
557 bool noboot;
558};
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588enum {
589
590 TRACE_FTRACE_BIT,
591 TRACE_FTRACE_NMI_BIT,
592 TRACE_FTRACE_IRQ_BIT,
593 TRACE_FTRACE_SIRQ_BIT,
594
595
596 TRACE_INTERNAL_BIT,
597 TRACE_INTERNAL_NMI_BIT,
598 TRACE_INTERNAL_IRQ_BIT,
599 TRACE_INTERNAL_SIRQ_BIT,
600
601 TRACE_BRANCH_BIT,
602
603
604
605
606
607
608
609 TRACE_IRQ_BIT,
610
611
612 TRACE_GRAPH_BIT,
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630 TRACE_GRAPH_DEPTH_START_BIT,
631 TRACE_GRAPH_DEPTH_END_BIT,
632
633
634
635
636
637
638 TRACE_GRAPH_NOTRACE_BIT,
639};
640
641#define trace_recursion_set(bit) do { (current)->trace_recursion |= (1<<(bit)); } while (0)
642#define trace_recursion_clear(bit) do { (current)->trace_recursion &= ~(1<<(bit)); } while (0)
643#define trace_recursion_test(bit) ((current)->trace_recursion & (1<<(bit)))
644
645#define trace_recursion_depth() \
646 (((current)->trace_recursion >> TRACE_GRAPH_DEPTH_START_BIT) & 3)
647#define trace_recursion_set_depth(depth) \
648 do { \
649 current->trace_recursion &= \
650 ~(3 << TRACE_GRAPH_DEPTH_START_BIT); \
651 current->trace_recursion |= \
652 ((depth) & 3) << TRACE_GRAPH_DEPTH_START_BIT; \
653 } while (0)
654
655#define TRACE_CONTEXT_BITS 4
656
657#define TRACE_FTRACE_START TRACE_FTRACE_BIT
658#define TRACE_FTRACE_MAX ((1 << (TRACE_FTRACE_START + TRACE_CONTEXT_BITS)) - 1)
659
660#define TRACE_LIST_START TRACE_INTERNAL_BIT
661#define TRACE_LIST_MAX ((1 << (TRACE_LIST_START + TRACE_CONTEXT_BITS)) - 1)
662
663#define TRACE_CONTEXT_MASK TRACE_LIST_MAX
664
665static __always_inline int trace_get_context_bit(void)
666{
667 int bit;
668
669 if (in_interrupt()) {
670 if (in_nmi())
671 bit = 0;
672
673 else if (in_irq())
674 bit = 1;
675 else
676 bit = 2;
677 } else
678 bit = 3;
679
680 return bit;
681}
682
683static __always_inline int trace_test_and_set_recursion(int start, int max)
684{
685 unsigned int val = current->trace_recursion;
686 int bit;
687
688
689 if ((val & TRACE_CONTEXT_MASK) > max)
690 return 0;
691
692 bit = trace_get_context_bit() + start;
693 if (unlikely(val & (1 << bit)))
694 return -1;
695
696 val |= 1 << bit;
697 current->trace_recursion = val;
698 barrier();
699
700 return bit;
701}
702
703static __always_inline void trace_clear_recursion(int bit)
704{
705 unsigned int val = current->trace_recursion;
706
707 if (!bit)
708 return;
709
710 bit = 1 << bit;
711 val &= ~bit;
712
713 barrier();
714 current->trace_recursion = val;
715}
716
717static inline struct ring_buffer_iter *
718trace_buffer_iter(struct trace_iterator *iter, int cpu)
719{
720 return iter->buffer_iter ? iter->buffer_iter[cpu] : NULL;
721}
722
723int tracer_init(struct tracer *t, struct trace_array *tr);
724int tracing_is_enabled(void);
725void tracing_reset_online_cpus(struct array_buffer *buf);
726void tracing_reset_current(int cpu);
727void tracing_reset_all_online_cpus(void);
728int tracing_open_generic(struct inode *inode, struct file *filp);
729int tracing_open_generic_tr(struct inode *inode, struct file *filp);
730bool tracing_is_disabled(void);
731bool tracer_tracing_is_on(struct trace_array *tr);
732void tracer_tracing_on(struct trace_array *tr);
733void tracer_tracing_off(struct trace_array *tr);
734struct dentry *trace_create_file(const char *name,
735 umode_t mode,
736 struct dentry *parent,
737 void *data,
738 const struct file_operations *fops);
739
740struct dentry *tracing_init_dentry(void);
741
742struct ring_buffer_event;
743
744struct ring_buffer_event *
745trace_buffer_lock_reserve(struct trace_buffer *buffer,
746 int type,
747 unsigned long len,
748 unsigned long flags,
749 int pc);
750
751struct trace_entry *tracing_get_trace_entry(struct trace_array *tr,
752 struct trace_array_cpu *data);
753
754struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
755 int *ent_cpu, u64 *ent_ts);
756
757void trace_buffer_unlock_commit_nostack(struct trace_buffer *buffer,
758 struct ring_buffer_event *event);
759
760int trace_empty(struct trace_iterator *iter);
761
762void *trace_find_next_entry_inc(struct trace_iterator *iter);
763
764void trace_init_global_iter(struct trace_iterator *iter);
765
766void tracing_iter_reset(struct trace_iterator *iter, int cpu);
767
768unsigned long trace_total_entries_cpu(struct trace_array *tr, int cpu);
769unsigned long trace_total_entries(struct trace_array *tr);
770
771void trace_function(struct trace_array *tr,
772 unsigned long ip,
773 unsigned long parent_ip,
774 unsigned long flags, int pc);
775void trace_graph_function(struct trace_array *tr,
776 unsigned long ip,
777 unsigned long parent_ip,
778 unsigned long flags, int pc);
779void trace_latency_header(struct seq_file *m);
780void trace_default_header(struct seq_file *m);
781void print_trace_header(struct seq_file *m, struct trace_iterator *iter);
782int trace_empty(struct trace_iterator *iter);
783
784void trace_graph_return(struct ftrace_graph_ret *trace);
785int trace_graph_entry(struct ftrace_graph_ent *trace);
786void set_graph_array(struct trace_array *tr);
787
788void tracing_start_cmdline_record(void);
789void tracing_stop_cmdline_record(void);
790void tracing_start_tgid_record(void);
791void tracing_stop_tgid_record(void);
792
793int register_tracer(struct tracer *type);
794int is_tracing_stopped(void);
795
796loff_t tracing_lseek(struct file *file, loff_t offset, int whence);
797
798extern cpumask_var_t __read_mostly tracing_buffer_mask;
799
800#define for_each_tracing_cpu(cpu) \
801 for_each_cpu(cpu, tracing_buffer_mask)
802
803extern unsigned long nsecs_to_usecs(unsigned long nsecs);
804
805extern unsigned long tracing_thresh;
806
807
808
809extern int pid_max;
810
811bool trace_find_filtered_pid(struct trace_pid_list *filtered_pids,
812 pid_t search_pid);
813bool trace_ignore_this_task(struct trace_pid_list *filtered_pids,
814 struct trace_pid_list *filtered_no_pids,
815 struct task_struct *task);
816void trace_filter_add_remove_task(struct trace_pid_list *pid_list,
817 struct task_struct *self,
818 struct task_struct *task);
819void *trace_pid_next(struct trace_pid_list *pid_list, void *v, loff_t *pos);
820void *trace_pid_start(struct trace_pid_list *pid_list, loff_t *pos);
821int trace_pid_show(struct seq_file *m, void *v);
822void trace_free_pid_list(struct trace_pid_list *pid_list);
823int trace_pid_write(struct trace_pid_list *filtered_pids,
824 struct trace_pid_list **new_pid_list,
825 const char __user *ubuf, size_t cnt);
826
827#ifdef CONFIG_TRACER_MAX_TRACE
828void update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu,
829 void *cond_data);
830void update_max_tr_single(struct trace_array *tr,
831 struct task_struct *tsk, int cpu);
832#endif
833
834#if (defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)) && \
835 defined(CONFIG_FSNOTIFY)
836
837void latency_fsnotify(struct trace_array *tr);
838
839#else
840
841static inline void latency_fsnotify(struct trace_array *tr) { }
842
843#endif
844
845#ifdef CONFIG_STACKTRACE
846void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
847 int pc);
848#else
849static inline void __trace_stack(struct trace_array *tr, unsigned long flags,
850 int skip, int pc)
851{
852}
853#endif
854
855extern u64 ftrace_now(int cpu);
856
857extern void trace_find_cmdline(int pid, char comm[]);
858extern int trace_find_tgid(int pid);
859extern void trace_event_follow_fork(struct trace_array *tr, bool enable);
860
861#ifdef CONFIG_DYNAMIC_FTRACE
862extern unsigned long ftrace_update_tot_cnt;
863extern unsigned long ftrace_number_of_pages;
864extern unsigned long ftrace_number_of_groups;
865void ftrace_init_trace_array(struct trace_array *tr);
866#else
867static inline void ftrace_init_trace_array(struct trace_array *tr) { }
868#endif
869#define DYN_FTRACE_TEST_NAME trace_selftest_dynamic_test_func
870extern int DYN_FTRACE_TEST_NAME(void);
871#define DYN_FTRACE_TEST_NAME2 trace_selftest_dynamic_test_func2
872extern int DYN_FTRACE_TEST_NAME2(void);
873
874extern bool ring_buffer_expanded;
875extern bool tracing_selftest_disabled;
876
877#ifdef CONFIG_FTRACE_STARTUP_TEST
878extern int trace_selftest_startup_function(struct tracer *trace,
879 struct trace_array *tr);
880extern int trace_selftest_startup_function_graph(struct tracer *trace,
881 struct trace_array *tr);
882extern int trace_selftest_startup_irqsoff(struct tracer *trace,
883 struct trace_array *tr);
884extern int trace_selftest_startup_preemptoff(struct tracer *trace,
885 struct trace_array *tr);
886extern int trace_selftest_startup_preemptirqsoff(struct tracer *trace,
887 struct trace_array *tr);
888extern int trace_selftest_startup_wakeup(struct tracer *trace,
889 struct trace_array *tr);
890extern int trace_selftest_startup_nop(struct tracer *trace,
891 struct trace_array *tr);
892extern int trace_selftest_startup_branch(struct tracer *trace,
893 struct trace_array *tr);
894
895
896
897
898
899#define __tracer_data __refdata
900#else
901
902#define __tracer_data __read_mostly
903#endif
904
905extern void *head_page(struct trace_array_cpu *data);
906extern unsigned long long ns2usecs(u64 nsec);
907extern int
908trace_vbprintk(unsigned long ip, const char *fmt, va_list args);
909extern int
910trace_vprintk(unsigned long ip, const char *fmt, va_list args);
911extern int
912trace_array_vprintk(struct trace_array *tr,
913 unsigned long ip, const char *fmt, va_list args);
914int trace_array_printk_buf(struct trace_buffer *buffer,
915 unsigned long ip, const char *fmt, ...);
916void trace_printk_seq(struct trace_seq *s);
917enum print_line_t print_trace_line(struct trace_iterator *iter);
918
919extern char trace_find_mark(unsigned long long duration);
920
921struct ftrace_hash;
922
923struct ftrace_mod_load {
924 struct list_head list;
925 char *func;
926 char *module;
927 int enable;
928};
929
930enum {
931 FTRACE_HASH_FL_MOD = (1 << 0),
932};
933
934struct ftrace_hash {
935 unsigned long size_bits;
936 struct hlist_head *buckets;
937 unsigned long count;
938 unsigned long flags;
939 struct rcu_head rcu;
940};
941
942struct ftrace_func_entry *
943ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip);
944
945static __always_inline bool ftrace_hash_empty(struct ftrace_hash *hash)
946{
947 return !hash || !(hash->count || (hash->flags & FTRACE_HASH_FL_MOD));
948}
949
950
951#ifdef CONFIG_FUNCTION_GRAPH_TRACER
952
953
954#define TRACE_GRAPH_PRINT_OVERRUN 0x1
955#define TRACE_GRAPH_PRINT_CPU 0x2
956#define TRACE_GRAPH_PRINT_OVERHEAD 0x4
957#define TRACE_GRAPH_PRINT_PROC 0x8
958#define TRACE_GRAPH_PRINT_DURATION 0x10
959#define TRACE_GRAPH_PRINT_ABS_TIME 0x20
960#define TRACE_GRAPH_PRINT_REL_TIME 0x40
961#define TRACE_GRAPH_PRINT_IRQS 0x80
962#define TRACE_GRAPH_PRINT_TAIL 0x100
963#define TRACE_GRAPH_SLEEP_TIME 0x200
964#define TRACE_GRAPH_GRAPH_TIME 0x400
965#define TRACE_GRAPH_PRINT_FILL_SHIFT 28
966#define TRACE_GRAPH_PRINT_FILL_MASK (0x3 << TRACE_GRAPH_PRINT_FILL_SHIFT)
967
968extern void ftrace_graph_sleep_time_control(bool enable);
969
970#ifdef CONFIG_FUNCTION_PROFILER
971extern void ftrace_graph_graph_time_control(bool enable);
972#else
973static inline void ftrace_graph_graph_time_control(bool enable) { }
974#endif
975
976extern enum print_line_t
977print_graph_function_flags(struct trace_iterator *iter, u32 flags);
978extern void print_graph_headers_flags(struct seq_file *s, u32 flags);
979extern void
980trace_print_graph_duration(unsigned long long duration, struct trace_seq *s);
981extern void graph_trace_open(struct trace_iterator *iter);
982extern void graph_trace_close(struct trace_iterator *iter);
983extern int __trace_graph_entry(struct trace_array *tr,
984 struct ftrace_graph_ent *trace,
985 unsigned long flags, int pc);
986extern void __trace_graph_return(struct trace_array *tr,
987 struct ftrace_graph_ret *trace,
988 unsigned long flags, int pc);
989
990#ifdef CONFIG_DYNAMIC_FTRACE
991extern struct ftrace_hash __rcu *ftrace_graph_hash;
992extern struct ftrace_hash __rcu *ftrace_graph_notrace_hash;
993
994static inline int ftrace_graph_addr(struct ftrace_graph_ent *trace)
995{
996 unsigned long addr = trace->func;
997 int ret = 0;
998 struct ftrace_hash *hash;
999
1000 preempt_disable_notrace();
1001
1002
1003
1004
1005
1006
1007
1008 hash = rcu_dereference_protected(ftrace_graph_hash, !preemptible());
1009
1010 if (ftrace_hash_empty(hash)) {
1011 ret = 1;
1012 goto out;
1013 }
1014
1015 if (ftrace_lookup_ip(hash, addr)) {
1016
1017
1018
1019
1020
1021 trace_recursion_set(TRACE_GRAPH_BIT);
1022 trace_recursion_set_depth(trace->depth);
1023
1024
1025
1026
1027
1028
1029 if (in_irq())
1030 trace_recursion_set(TRACE_IRQ_BIT);
1031 else
1032 trace_recursion_clear(TRACE_IRQ_BIT);
1033 ret = 1;
1034 }
1035
1036out:
1037 preempt_enable_notrace();
1038 return ret;
1039}
1040
1041static inline void ftrace_graph_addr_finish(struct ftrace_graph_ret *trace)
1042{
1043 if (trace_recursion_test(TRACE_GRAPH_BIT) &&
1044 trace->depth == trace_recursion_depth())
1045 trace_recursion_clear(TRACE_GRAPH_BIT);
1046}
1047
1048static inline int ftrace_graph_notrace_addr(unsigned long addr)
1049{
1050 int ret = 0;
1051 struct ftrace_hash *notrace_hash;
1052
1053 preempt_disable_notrace();
1054
1055
1056
1057
1058
1059
1060
1061 notrace_hash = rcu_dereference_protected(ftrace_graph_notrace_hash,
1062 !preemptible());
1063
1064 if (ftrace_lookup_ip(notrace_hash, addr))
1065 ret = 1;
1066
1067 preempt_enable_notrace();
1068 return ret;
1069}
1070#else
1071static inline int ftrace_graph_addr(struct ftrace_graph_ent *trace)
1072{
1073 return 1;
1074}
1075
1076static inline int ftrace_graph_notrace_addr(unsigned long addr)
1077{
1078 return 0;
1079}
1080static inline void ftrace_graph_addr_finish(struct ftrace_graph_ret *trace)
1081{ }
1082#endif
1083
1084extern unsigned int fgraph_max_depth;
1085
1086static inline bool ftrace_graph_ignore_func(struct ftrace_graph_ent *trace)
1087{
1088
1089 return !(trace_recursion_test(TRACE_GRAPH_BIT) ||
1090 ftrace_graph_addr(trace)) ||
1091 (trace->depth < 0) ||
1092 (fgraph_max_depth && trace->depth >= fgraph_max_depth);
1093}
1094
1095#else
1096static inline enum print_line_t
1097print_graph_function_flags(struct trace_iterator *iter, u32 flags)
1098{
1099 return TRACE_TYPE_UNHANDLED;
1100}
1101#endif
1102
1103extern struct list_head ftrace_pids;
1104
1105#ifdef CONFIG_FUNCTION_TRACER
1106
1107#define FTRACE_PID_IGNORE -1
1108#define FTRACE_PID_TRACE -2
1109
1110struct ftrace_func_command {
1111 struct list_head list;
1112 char *name;
1113 int (*func)(struct trace_array *tr,
1114 struct ftrace_hash *hash,
1115 char *func, char *cmd,
1116 char *params, int enable);
1117};
1118extern bool ftrace_filter_param __initdata;
1119static inline int ftrace_trace_task(struct trace_array *tr)
1120{
1121 return this_cpu_read(tr->array_buffer.data->ftrace_ignore_pid) !=
1122 FTRACE_PID_IGNORE;
1123}
1124extern int ftrace_is_dead(void);
1125int ftrace_create_function_files(struct trace_array *tr,
1126 struct dentry *parent);
1127void ftrace_destroy_function_files(struct trace_array *tr);
1128void ftrace_init_global_array_ops(struct trace_array *tr);
1129void ftrace_init_array_ops(struct trace_array *tr, ftrace_func_t func);
1130void ftrace_reset_array_ops(struct trace_array *tr);
1131void ftrace_init_tracefs(struct trace_array *tr, struct dentry *d_tracer);
1132void ftrace_init_tracefs_toplevel(struct trace_array *tr,
1133 struct dentry *d_tracer);
1134void ftrace_clear_pids(struct trace_array *tr);
1135int init_function_trace(void);
1136void ftrace_pid_follow_fork(struct trace_array *tr, bool enable);
1137#else
1138static inline int ftrace_trace_task(struct trace_array *tr)
1139{
1140 return 1;
1141}
1142static inline int ftrace_is_dead(void) { return 0; }
1143static inline int
1144ftrace_create_function_files(struct trace_array *tr,
1145 struct dentry *parent)
1146{
1147 return 0;
1148}
1149static inline void ftrace_destroy_function_files(struct trace_array *tr) { }
1150static inline __init void
1151ftrace_init_global_array_ops(struct trace_array *tr) { }
1152static inline void ftrace_reset_array_ops(struct trace_array *tr) { }
1153static inline void ftrace_init_tracefs(struct trace_array *tr, struct dentry *d) { }
1154static inline void ftrace_init_tracefs_toplevel(struct trace_array *tr, struct dentry *d) { }
1155static inline void ftrace_clear_pids(struct trace_array *tr) { }
1156static inline int init_function_trace(void) { return 0; }
1157static inline void ftrace_pid_follow_fork(struct trace_array *tr, bool enable) { }
1158
1159#define ftrace_init_array_ops(tr, func) do { } while (0)
1160#endif
1161
1162#if defined(CONFIG_FUNCTION_TRACER) && defined(CONFIG_DYNAMIC_FTRACE)
1163
1164struct ftrace_probe_ops {
1165 void (*func)(unsigned long ip,
1166 unsigned long parent_ip,
1167 struct trace_array *tr,
1168 struct ftrace_probe_ops *ops,
1169 void *data);
1170 int (*init)(struct ftrace_probe_ops *ops,
1171 struct trace_array *tr,
1172 unsigned long ip, void *init_data,
1173 void **data);
1174 void (*free)(struct ftrace_probe_ops *ops,
1175 struct trace_array *tr,
1176 unsigned long ip, void *data);
1177 int (*print)(struct seq_file *m,
1178 unsigned long ip,
1179 struct ftrace_probe_ops *ops,
1180 void *data);
1181};
1182
1183struct ftrace_func_mapper;
1184typedef int (*ftrace_mapper_func)(void *data);
1185
1186struct ftrace_func_mapper *allocate_ftrace_func_mapper(void);
1187void **ftrace_func_mapper_find_ip(struct ftrace_func_mapper *mapper,
1188 unsigned long ip);
1189int ftrace_func_mapper_add_ip(struct ftrace_func_mapper *mapper,
1190 unsigned long ip, void *data);
1191void *ftrace_func_mapper_remove_ip(struct ftrace_func_mapper *mapper,
1192 unsigned long ip);
1193void free_ftrace_func_mapper(struct ftrace_func_mapper *mapper,
1194 ftrace_mapper_func free_func);
1195
1196extern int
1197register_ftrace_function_probe(char *glob, struct trace_array *tr,
1198 struct ftrace_probe_ops *ops, void *data);
1199extern int
1200unregister_ftrace_function_probe_func(char *glob, struct trace_array *tr,
1201 struct ftrace_probe_ops *ops);
1202extern void clear_ftrace_function_probes(struct trace_array *tr);
1203
1204int register_ftrace_command(struct ftrace_func_command *cmd);
1205int unregister_ftrace_command(struct ftrace_func_command *cmd);
1206
1207void ftrace_create_filter_files(struct ftrace_ops *ops,
1208 struct dentry *parent);
1209void ftrace_destroy_filter_files(struct ftrace_ops *ops);
1210
1211extern int ftrace_set_filter(struct ftrace_ops *ops, unsigned char *buf,
1212 int len, int reset);
1213extern int ftrace_set_notrace(struct ftrace_ops *ops, unsigned char *buf,
1214 int len, int reset);
1215#else
1216struct ftrace_func_command;
1217
1218static inline __init int register_ftrace_command(struct ftrace_func_command *cmd)
1219{
1220 return -EINVAL;
1221}
1222static inline __init int unregister_ftrace_command(char *cmd_name)
1223{
1224 return -EINVAL;
1225}
1226static inline void clear_ftrace_function_probes(struct trace_array *tr)
1227{
1228}
1229
1230
1231
1232
1233
1234#define ftrace_create_filter_files(ops, parent) do { } while (0)
1235#define ftrace_destroy_filter_files(ops) do { } while (0)
1236#endif
1237
1238bool ftrace_event_is_function(struct trace_event_call *call);
1239
1240
1241
1242
1243
1244
1245
1246
1247struct trace_parser {
1248 bool cont;
1249 char *buffer;
1250 unsigned idx;
1251 unsigned size;
1252};
1253
1254static inline bool trace_parser_loaded(struct trace_parser *parser)
1255{
1256 return (parser->idx != 0);
1257}
1258
1259static inline bool trace_parser_cont(struct trace_parser *parser)
1260{
1261 return parser->cont;
1262}
1263
1264static inline void trace_parser_clear(struct trace_parser *parser)
1265{
1266 parser->cont = false;
1267 parser->idx = 0;
1268}
1269
1270extern int trace_parser_get_init(struct trace_parser *parser, int size);
1271extern void trace_parser_put(struct trace_parser *parser);
1272extern int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
1273 size_t cnt, loff_t *ppos);
1274
1275
1276
1277
1278#ifdef CONFIG_FUNCTION_GRAPH_TRACER
1279# define FGRAPH_FLAGS \
1280 C(DISPLAY_GRAPH, "display-graph"),
1281#else
1282# define FGRAPH_FLAGS
1283#endif
1284
1285#ifdef CONFIG_BRANCH_TRACER
1286# define BRANCH_FLAGS \
1287 C(BRANCH, "branch"),
1288#else
1289# define BRANCH_FLAGS
1290#endif
1291
1292#ifdef CONFIG_FUNCTION_TRACER
1293# define FUNCTION_FLAGS \
1294 C(FUNCTION, "function-trace"), \
1295 C(FUNC_FORK, "function-fork"),
1296# define FUNCTION_DEFAULT_FLAGS TRACE_ITER_FUNCTION
1297#else
1298# define FUNCTION_FLAGS
1299# define FUNCTION_DEFAULT_FLAGS 0UL
1300# define TRACE_ITER_FUNC_FORK 0UL
1301#endif
1302
1303#ifdef CONFIG_STACKTRACE
1304# define STACK_FLAGS \
1305 C(STACKTRACE, "stacktrace"),
1306#else
1307# define STACK_FLAGS
1308#endif
1309
1310
1311
1312
1313
1314
1315
1316
1317#define TRACE_FLAGS \
1318 C(PRINT_PARENT, "print-parent"), \
1319 C(SYM_OFFSET, "sym-offset"), \
1320 C(SYM_ADDR, "sym-addr"), \
1321 C(VERBOSE, "verbose"), \
1322 C(RAW, "raw"), \
1323 C(HEX, "hex"), \
1324 C(BIN, "bin"), \
1325 C(BLOCK, "block"), \
1326 C(PRINTK, "trace_printk"), \
1327 C(ANNOTATE, "annotate"), \
1328 C(USERSTACKTRACE, "userstacktrace"), \
1329 C(SYM_USEROBJ, "sym-userobj"), \
1330 C(PRINTK_MSGONLY, "printk-msg-only"), \
1331 C(CONTEXT_INFO, "context-info"), \
1332 C(LATENCY_FMT, "latency-format"), \
1333 C(RECORD_CMD, "record-cmd"), \
1334 C(RECORD_TGID, "record-tgid"), \
1335 C(OVERWRITE, "overwrite"), \
1336 C(STOP_ON_FREE, "disable_on_free"), \
1337 C(IRQ_INFO, "irq-info"), \
1338 C(MARKERS, "markers"), \
1339 C(EVENT_FORK, "event-fork"), \
1340 C(PAUSE_ON_TRACE, "pause-on-trace"), \
1341 FUNCTION_FLAGS \
1342 FGRAPH_FLAGS \
1343 STACK_FLAGS \
1344 BRANCH_FLAGS
1345
1346
1347
1348
1349
1350#undef C
1351#define C(a, b) TRACE_ITER_##a##_BIT
1352
1353enum trace_iterator_bits {
1354 TRACE_FLAGS
1355
1356 TRACE_ITER_LAST_BIT
1357};
1358
1359
1360
1361
1362
1363#undef C
1364#define C(a, b) TRACE_ITER_##a = (1 << TRACE_ITER_##a##_BIT)
1365
1366enum trace_iterator_flags { TRACE_FLAGS };
1367
1368
1369
1370
1371
1372#define TRACE_ITER_SYM_MASK \
1373 (TRACE_ITER_PRINT_PARENT|TRACE_ITER_SYM_OFFSET|TRACE_ITER_SYM_ADDR)
1374
1375extern struct tracer nop_trace;
1376
1377#ifdef CONFIG_BRANCH_TRACER
1378extern int enable_branch_tracing(struct trace_array *tr);
1379extern void disable_branch_tracing(void);
1380static inline int trace_branch_enable(struct trace_array *tr)
1381{
1382 if (tr->trace_flags & TRACE_ITER_BRANCH)
1383 return enable_branch_tracing(tr);
1384 return 0;
1385}
1386static inline void trace_branch_disable(void)
1387{
1388
1389 disable_branch_tracing();
1390}
1391#else
1392static inline int trace_branch_enable(struct trace_array *tr)
1393{
1394 return 0;
1395}
1396static inline void trace_branch_disable(void)
1397{
1398}
1399#endif
1400
1401
1402int tracing_update_buffers(void);
1403
1404struct ftrace_event_field {
1405 struct list_head link;
1406 const char *name;
1407 const char *type;
1408 int filter_type;
1409 int offset;
1410 int size;
1411 int is_signed;
1412};
1413
1414struct prog_entry;
1415
1416struct event_filter {
1417 struct prog_entry __rcu *prog;
1418 char *filter_string;
1419};
1420
1421struct event_subsystem {
1422 struct list_head list;
1423 const char *name;
1424 struct event_filter *filter;
1425 int ref_count;
1426};
1427
1428struct trace_subsystem_dir {
1429 struct list_head list;
1430 struct event_subsystem *subsystem;
1431 struct trace_array *tr;
1432 struct dentry *entry;
1433 int ref_count;
1434 int nr_events;
1435};
1436
1437extern int call_filter_check_discard(struct trace_event_call *call, void *rec,
1438 struct trace_buffer *buffer,
1439 struct ring_buffer_event *event);
1440
1441void trace_buffer_unlock_commit_regs(struct trace_array *tr,
1442 struct trace_buffer *buffer,
1443 struct ring_buffer_event *event,
1444 unsigned long flags, int pc,
1445 struct pt_regs *regs);
1446
1447static inline void trace_buffer_unlock_commit(struct trace_array *tr,
1448 struct trace_buffer *buffer,
1449 struct ring_buffer_event *event,
1450 unsigned long flags, int pc)
1451{
1452 trace_buffer_unlock_commit_regs(tr, buffer, event, flags, pc, NULL);
1453}
1454
1455DECLARE_PER_CPU(struct ring_buffer_event *, trace_buffered_event);
1456DECLARE_PER_CPU(int, trace_buffered_event_cnt);
1457void trace_buffered_event_disable(void);
1458void trace_buffered_event_enable(void);
1459
1460static inline void
1461__trace_event_discard_commit(struct trace_buffer *buffer,
1462 struct ring_buffer_event *event)
1463{
1464 if (this_cpu_read(trace_buffered_event) == event) {
1465
1466 this_cpu_dec(trace_buffered_event_cnt);
1467 return;
1468 }
1469 ring_buffer_discard_commit(buffer, event);
1470}
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485static inline bool
1486__event_trigger_test_discard(struct trace_event_file *file,
1487 struct trace_buffer *buffer,
1488 struct ring_buffer_event *event,
1489 void *entry,
1490 enum event_trigger_type *tt)
1491{
1492 unsigned long eflags = file->flags;
1493
1494 if (eflags & EVENT_FILE_FL_TRIGGER_COND)
1495 *tt = event_triggers_call(file, entry, event);
1496
1497 if (test_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &file->flags) ||
1498 (unlikely(file->flags & EVENT_FILE_FL_FILTERED) &&
1499 !filter_match_preds(file->filter, entry))) {
1500 __trace_event_discard_commit(buffer, event);
1501 return true;
1502 }
1503
1504 return false;
1505}
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520static inline void
1521event_trigger_unlock_commit(struct trace_event_file *file,
1522 struct trace_buffer *buffer,
1523 struct ring_buffer_event *event,
1524 void *entry, unsigned long irq_flags, int pc)
1525{
1526 enum event_trigger_type tt = ETT_NONE;
1527
1528 if (!__event_trigger_test_discard(file, buffer, event, entry, &tt))
1529 trace_buffer_unlock_commit(file->tr, buffer, event, irq_flags, pc);
1530
1531 if (tt)
1532 event_triggers_post_call(file, tt);
1533}
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551static inline void
1552event_trigger_unlock_commit_regs(struct trace_event_file *file,
1553 struct trace_buffer *buffer,
1554 struct ring_buffer_event *event,
1555 void *entry, unsigned long irq_flags, int pc,
1556 struct pt_regs *regs)
1557{
1558 enum event_trigger_type tt = ETT_NONE;
1559
1560 if (!__event_trigger_test_discard(file, buffer, event, entry, &tt))
1561 trace_buffer_unlock_commit_regs(file->tr, buffer, event,
1562 irq_flags, pc, regs);
1563
1564 if (tt)
1565 event_triggers_post_call(file, tt);
1566}
1567
1568#define FILTER_PRED_INVALID ((unsigned short)-1)
1569#define FILTER_PRED_IS_RIGHT (1 << 15)
1570#define FILTER_PRED_FOLD (1 << 15)
1571
1572
1573
1574
1575
1576
1577
1578
1579#define MAX_FILTER_PRED 16384
1580
1581struct filter_pred;
1582struct regex;
1583
1584typedef int (*filter_pred_fn_t) (struct filter_pred *pred, void *event);
1585
1586typedef int (*regex_match_func)(char *str, struct regex *r, int len);
1587
1588enum regex_type {
1589 MATCH_FULL = 0,
1590 MATCH_FRONT_ONLY,
1591 MATCH_MIDDLE_ONLY,
1592 MATCH_END_ONLY,
1593 MATCH_GLOB,
1594 MATCH_INDEX,
1595};
1596
1597struct regex {
1598 char pattern[MAX_FILTER_STR_VAL];
1599 int len;
1600 int field_len;
1601 regex_match_func match;
1602};
1603
1604struct filter_pred {
1605 filter_pred_fn_t fn;
1606 u64 val;
1607 struct regex regex;
1608 unsigned short *ops;
1609 struct ftrace_event_field *field;
1610 int offset;
1611 int not;
1612 int op;
1613};
1614
1615static inline bool is_string_field(struct ftrace_event_field *field)
1616{
1617 return field->filter_type == FILTER_DYN_STRING ||
1618 field->filter_type == FILTER_STATIC_STRING ||
1619 field->filter_type == FILTER_PTR_STRING ||
1620 field->filter_type == FILTER_COMM;
1621}
1622
1623static inline bool is_function_field(struct ftrace_event_field *field)
1624{
1625 return field->filter_type == FILTER_TRACE_FN;
1626}
1627
1628extern enum regex_type
1629filter_parse_regex(char *buff, int len, char **search, int *not);
1630extern void print_event_filter(struct trace_event_file *file,
1631 struct trace_seq *s);
1632extern int apply_event_filter(struct trace_event_file *file,
1633 char *filter_string);
1634extern int apply_subsystem_event_filter(struct trace_subsystem_dir *dir,
1635 char *filter_string);
1636extern void print_subsystem_event_filter(struct event_subsystem *system,
1637 struct trace_seq *s);
1638extern int filter_assign_type(const char *type);
1639extern int create_event_filter(struct trace_array *tr,
1640 struct trace_event_call *call,
1641 char *filter_str, bool set_str,
1642 struct event_filter **filterp);
1643extern void free_event_filter(struct event_filter *filter);
1644
1645struct ftrace_event_field *
1646trace_find_event_field(struct trace_event_call *call, char *name);
1647
1648extern void trace_event_enable_cmd_record(bool enable);
1649extern void trace_event_enable_tgid_record(bool enable);
1650
1651extern int event_trace_init(void);
1652extern int event_trace_add_tracer(struct dentry *parent, struct trace_array *tr);
1653extern int event_trace_del_tracer(struct trace_array *tr);
1654
1655extern struct trace_event_file *__find_event_file(struct trace_array *tr,
1656 const char *system,
1657 const char *event);
1658extern struct trace_event_file *find_event_file(struct trace_array *tr,
1659 const char *system,
1660 const char *event);
1661
1662static inline void *event_file_data(struct file *filp)
1663{
1664 return READ_ONCE(file_inode(filp)->i_private);
1665}
1666
1667extern struct mutex event_mutex;
1668extern struct list_head ftrace_events;
1669
1670extern const struct file_operations event_trigger_fops;
1671extern const struct file_operations event_hist_fops;
1672extern const struct file_operations event_hist_debug_fops;
1673extern const struct file_operations event_inject_fops;
1674
1675#ifdef CONFIG_HIST_TRIGGERS
1676extern int register_trigger_hist_cmd(void);
1677extern int register_trigger_hist_enable_disable_cmds(void);
1678#else
1679static inline int register_trigger_hist_cmd(void) { return 0; }
1680static inline int register_trigger_hist_enable_disable_cmds(void) { return 0; }
1681#endif
1682
1683extern int register_trigger_cmds(void);
1684extern void clear_event_triggers(struct trace_array *tr);
1685
1686struct event_trigger_data {
1687 unsigned long count;
1688 int ref;
1689 struct event_trigger_ops *ops;
1690 struct event_command *cmd_ops;
1691 struct event_filter __rcu *filter;
1692 char *filter_str;
1693 void *private_data;
1694 bool paused;
1695 bool paused_tmp;
1696 struct list_head list;
1697 char *name;
1698 struct list_head named_list;
1699 struct event_trigger_data *named_data;
1700};
1701
1702
1703#define ENABLE_EVENT_STR "enable_event"
1704#define DISABLE_EVENT_STR "disable_event"
1705#define ENABLE_HIST_STR "enable_hist"
1706#define DISABLE_HIST_STR "disable_hist"
1707
1708struct enable_trigger_data {
1709 struct trace_event_file *file;
1710 bool enable;
1711 bool hist;
1712};
1713
1714extern int event_enable_trigger_print(struct seq_file *m,
1715 struct event_trigger_ops *ops,
1716 struct event_trigger_data *data);
1717extern void event_enable_trigger_free(struct event_trigger_ops *ops,
1718 struct event_trigger_data *data);
1719extern int event_enable_trigger_func(struct event_command *cmd_ops,
1720 struct trace_event_file *file,
1721 char *glob, char *cmd, char *param);
1722extern int event_enable_register_trigger(char *glob,
1723 struct event_trigger_ops *ops,
1724 struct event_trigger_data *data,
1725 struct trace_event_file *file);
1726extern void event_enable_unregister_trigger(char *glob,
1727 struct event_trigger_ops *ops,
1728 struct event_trigger_data *test,
1729 struct trace_event_file *file);
1730extern void trigger_data_free(struct event_trigger_data *data);
1731extern int event_trigger_init(struct event_trigger_ops *ops,
1732 struct event_trigger_data *data);
1733extern int trace_event_trigger_enable_disable(struct trace_event_file *file,
1734 int trigger_enable);
1735extern void update_cond_flag(struct trace_event_file *file);
1736extern int set_trigger_filter(char *filter_str,
1737 struct event_trigger_data *trigger_data,
1738 struct trace_event_file *file);
1739extern struct event_trigger_data *find_named_trigger(const char *name);
1740extern bool is_named_trigger(struct event_trigger_data *test);
1741extern int save_named_trigger(const char *name,
1742 struct event_trigger_data *data);
1743extern void del_named_trigger(struct event_trigger_data *data);
1744extern void pause_named_trigger(struct event_trigger_data *data);
1745extern void unpause_named_trigger(struct event_trigger_data *data);
1746extern void set_named_trigger_data(struct event_trigger_data *data,
1747 struct event_trigger_data *named_data);
1748extern struct event_trigger_data *
1749get_named_trigger_data(struct event_trigger_data *data);
1750extern int register_event_command(struct event_command *cmd);
1751extern int unregister_event_command(struct event_command *cmd);
1752extern int register_trigger_hist_enable_disable_cmds(void);
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791struct event_trigger_ops {
1792 void (*func)(struct event_trigger_data *data,
1793 void *rec,
1794 struct ring_buffer_event *rbe);
1795 int (*init)(struct event_trigger_ops *ops,
1796 struct event_trigger_data *data);
1797 void (*free)(struct event_trigger_ops *ops,
1798 struct event_trigger_data *data);
1799 int (*print)(struct seq_file *m,
1800 struct event_trigger_ops *ops,
1801 struct event_trigger_data *data);
1802};
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879struct event_command {
1880 struct list_head list;
1881 char *name;
1882 enum event_trigger_type trigger_type;
1883 int flags;
1884 int (*func)(struct event_command *cmd_ops,
1885 struct trace_event_file *file,
1886 char *glob, char *cmd, char *params);
1887 int (*reg)(char *glob,
1888 struct event_trigger_ops *ops,
1889 struct event_trigger_data *data,
1890 struct trace_event_file *file);
1891 void (*unreg)(char *glob,
1892 struct event_trigger_ops *ops,
1893 struct event_trigger_data *data,
1894 struct trace_event_file *file);
1895 void (*unreg_all)(struct trace_event_file *file);
1896 int (*set_filter)(char *filter_str,
1897 struct event_trigger_data *data,
1898 struct trace_event_file *file);
1899 struct event_trigger_ops *(*get_trigger_ops)(char *cmd, char *param);
1900};
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930enum event_command_flags {
1931 EVENT_CMD_FL_POST_TRIGGER = 1,
1932 EVENT_CMD_FL_NEEDS_REC = 2,
1933};
1934
1935static inline bool event_command_post_trigger(struct event_command *cmd_ops)
1936{
1937 return cmd_ops->flags & EVENT_CMD_FL_POST_TRIGGER;
1938}
1939
1940static inline bool event_command_needs_rec(struct event_command *cmd_ops)
1941{
1942 return cmd_ops->flags & EVENT_CMD_FL_NEEDS_REC;
1943}
1944
1945extern int trace_event_enable_disable(struct trace_event_file *file,
1946 int enable, int soft_disable);
1947extern int tracing_alloc_snapshot(void);
1948extern void tracing_snapshot_cond(struct trace_array *tr, void *cond_data);
1949extern int tracing_snapshot_cond_enable(struct trace_array *tr, void *cond_data, cond_update_fn_t update);
1950
1951extern int tracing_snapshot_cond_disable(struct trace_array *tr);
1952extern void *tracing_cond_snapshot_data(struct trace_array *tr);
1953
1954extern const char *__start___trace_bprintk_fmt[];
1955extern const char *__stop___trace_bprintk_fmt[];
1956
1957extern const char *__start___tracepoint_str[];
1958extern const char *__stop___tracepoint_str[];
1959
1960void trace_printk_control(bool enabled);
1961void trace_printk_start_comm(void);
1962int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set);
1963int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled);
1964
1965
1966extern int trace_set_options(struct trace_array *tr, char *option);
1967extern int tracing_set_tracer(struct trace_array *tr, const char *buf);
1968extern ssize_t tracing_resize_ring_buffer(struct trace_array *tr,
1969 unsigned long size, int cpu_id);
1970extern int tracing_set_cpumask(struct trace_array *tr,
1971 cpumask_var_t tracing_cpumask_new);
1972
1973
1974#define MAX_EVENT_NAME_LEN 64
1975
1976extern int trace_run_command(const char *buf, int (*createfn)(int, char**));
1977extern ssize_t trace_parse_run_command(struct file *file,
1978 const char __user *buffer, size_t count, loff_t *ppos,
1979 int (*createfn)(int, char**));
1980
1981extern unsigned int err_pos(char *cmd, const char *str);
1982extern void tracing_log_err(struct trace_array *tr,
1983 const char *loc, const char *cmd,
1984 const char **errs, u8 type, u8 pos);
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995#define internal_trace_puts(str) __trace_puts(_THIS_IP_, str, strlen(str))
1996
1997#undef FTRACE_ENTRY
1998#define FTRACE_ENTRY(call, struct_name, id, tstruct, print) \
1999 extern struct trace_event_call \
2000 __aligned(4) event_##call;
2001#undef FTRACE_ENTRY_DUP
2002#define FTRACE_ENTRY_DUP(call, struct_name, id, tstruct, print) \
2003 FTRACE_ENTRY(call, struct_name, id, PARAMS(tstruct), PARAMS(print))
2004#undef FTRACE_ENTRY_PACKED
2005#define FTRACE_ENTRY_PACKED(call, struct_name, id, tstruct, print) \
2006 FTRACE_ENTRY(call, struct_name, id, PARAMS(tstruct), PARAMS(print))
2007
2008#include "trace_entries.h"
2009
2010#if defined(CONFIG_PERF_EVENTS) && defined(CONFIG_FUNCTION_TRACER)
2011int perf_ftrace_event_register(struct trace_event_call *call,
2012 enum trace_reg type, void *data);
2013#else
2014#define perf_ftrace_event_register NULL
2015#endif
2016
2017#ifdef CONFIG_FTRACE_SYSCALLS
2018void init_ftrace_syscalls(void);
2019const char *get_syscall_name(int syscall);
2020#else
2021static inline void init_ftrace_syscalls(void) { }
2022static inline const char *get_syscall_name(int syscall)
2023{
2024 return NULL;
2025}
2026#endif
2027
2028#ifdef CONFIG_EVENT_TRACING
2029void trace_event_init(void);
2030void trace_event_eval_update(struct trace_eval_map **map, int len);
2031
2032extern int ftrace_set_clr_event(struct trace_array *tr, char *buf, int set);
2033extern int trigger_process_regex(struct trace_event_file *file, char *buff);
2034#else
2035static inline void __init trace_event_init(void) { }
2036static inline void trace_event_eval_update(struct trace_eval_map **map, int len) { }
2037#endif
2038
2039#ifdef CONFIG_TRACER_SNAPSHOT
2040void tracing_snapshot_instance(struct trace_array *tr);
2041int tracing_alloc_snapshot_instance(struct trace_array *tr);
2042#else
2043static inline void tracing_snapshot_instance(struct trace_array *tr) { }
2044static inline int tracing_alloc_snapshot_instance(struct trace_array *tr)
2045{
2046 return 0;
2047}
2048#endif
2049
2050#ifdef CONFIG_PREEMPT_TRACER
2051void tracer_preempt_on(unsigned long a0, unsigned long a1);
2052void tracer_preempt_off(unsigned long a0, unsigned long a1);
2053#else
2054static inline void tracer_preempt_on(unsigned long a0, unsigned long a1) { }
2055static inline void tracer_preempt_off(unsigned long a0, unsigned long a1) { }
2056#endif
2057#ifdef CONFIG_IRQSOFF_TRACER
2058void tracer_hardirqs_on(unsigned long a0, unsigned long a1);
2059void tracer_hardirqs_off(unsigned long a0, unsigned long a1);
2060#else
2061static inline void tracer_hardirqs_on(unsigned long a0, unsigned long a1) { }
2062static inline void tracer_hardirqs_off(unsigned long a0, unsigned long a1) { }
2063#endif
2064
2065extern struct trace_iterator *tracepoint_print_iter;
2066
2067
2068
2069
2070
2071
2072static __always_inline void trace_iterator_reset(struct trace_iterator *iter)
2073{
2074 const size_t offset = offsetof(struct trace_iterator, seq);
2075
2076
2077
2078
2079
2080 memset((char *)iter + offset, 0, sizeof(struct trace_iterator) - offset);
2081
2082 iter->pos = -1;
2083}
2084
2085#endif
2086