1
2
3
4
5
6
7
8
9
10
11
12
13
14#ifndef _LINUX_PERF_EVENT_H
15#define _LINUX_PERF_EVENT_H
16
17#include <uapi/linux/perf_event.h>
18
19
20
21
22
23#ifdef CONFIG_PERF_EVENTS
24# include <asm/perf_event.h>
25# include <asm/local64.h>
26#endif
27
28struct perf_guest_info_callbacks {
29 int (*is_in_guest)(void);
30 int (*is_user_mode)(void);
31 unsigned long (*get_guest_ip)(void);
32};
33
34#ifdef CONFIG_HAVE_HW_BREAKPOINT
35#include <asm/hw_breakpoint.h>
36#endif
37
38#include <linux/list.h>
39#include <linux/mutex.h>
40#include <linux/rculist.h>
41#include <linux/rcupdate.h>
42#include <linux/spinlock.h>
43#include <linux/hrtimer.h>
44#include <linux/fs.h>
45#include <linux/pid_namespace.h>
46#include <linux/workqueue.h>
47#include <linux/ftrace.h>
48#include <linux/cpu.h>
49#include <linux/irq_work.h>
50#include <linux/static_key.h>
51#include <linux/atomic.h>
52#include <linux/sysfs.h>
53#include <linux/perf_regs.h>
54#include <asm/local.h>
55
56struct perf_callchain_entry {
57 __u64 nr;
58 __u64 ip[PERF_MAX_STACK_DEPTH];
59};
60
61struct perf_raw_record {
62 u32 size;
63 void *data;
64};
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80struct perf_branch_entry {
81 __u64 from;
82 __u64 to;
83 __u64 mispred:1,
84 predicted:1,
85 in_tx:1,
86 abort:1,
87 reserved:60;
88};
89
90
91
92
93
94
95
96
97
98
99struct perf_branch_stack {
100 __u64 nr;
101 struct perf_branch_entry entries[0];
102};
103
104struct perf_regs_user {
105 __u64 abi;
106 struct pt_regs *regs;
107};
108
109struct task_struct;
110
111
112
113
114struct hw_perf_event_extra {
115 u64 config;
116 unsigned int reg;
117 int alloc;
118 int idx;
119};
120
121struct event_constraint;
122
123
124
125
126struct hw_perf_event {
127#ifdef CONFIG_PERF_EVENTS
128 union {
129 struct {
130 u64 config;
131 u64 last_tag;
132 unsigned long config_base;
133 unsigned long event_base;
134 int event_base_rdpmc;
135 int idx;
136 int last_cpu;
137 int flags;
138
139 struct hw_perf_event_extra extra_reg;
140 struct hw_perf_event_extra branch_reg;
141
142 struct event_constraint *constraint;
143 };
144 struct {
145 struct hrtimer hrtimer;
146 };
147 struct {
148 struct task_struct *tp_target;
149
150 struct list_head tp_list;
151 };
152#ifdef CONFIG_HAVE_HW_BREAKPOINT
153 struct {
154
155
156
157
158
159 struct task_struct *bp_target;
160 struct arch_hw_breakpoint info;
161 struct list_head bp_list;
162 };
163#endif
164 };
165 int state;
166 local64_t prev_count;
167 u64 sample_period;
168 u64 last_period;
169 local64_t period_left;
170 u64 interrupts_seq;
171 u64 interrupts;
172
173 u64 freq_time_stamp;
174 u64 freq_count_stamp;
175#endif
176};
177
178
179
180
181#define PERF_HES_STOPPED 0x01
182#define PERF_HES_UPTODATE 0x02
183#define PERF_HES_ARCH 0x04
184
185struct perf_event;
186
187
188
189
190#define PERF_EVENT_TXN 0x1
191
192
193
194
195struct pmu {
196 struct list_head entry;
197
198 struct device *dev;
199 const struct attribute_group **attr_groups;
200 const char *name;
201 int type;
202
203 int * __percpu pmu_disable_count;
204 struct perf_cpu_context * __percpu pmu_cpu_context;
205 int task_ctx_nr;
206 int hrtimer_interval_ms;
207
208
209
210
211
212 void (*pmu_enable) (struct pmu *pmu);
213 void (*pmu_disable) (struct pmu *pmu);
214
215
216
217
218
219 int (*event_init) (struct perf_event *event);
220
221#define PERF_EF_START 0x01
222#define PERF_EF_RELOAD 0x02
223#define PERF_EF_UPDATE 0x04
224
225
226
227
228
229 int (*add) (struct perf_event *event, int flags);
230 void (*del) (struct perf_event *event, int flags);
231
232
233
234
235
236
237 void (*start) (struct perf_event *event, int flags);
238 void (*stop) (struct perf_event *event, int flags);
239
240
241
242
243 void (*read) (struct perf_event *event);
244
245
246
247
248
249
250
251
252
253 void (*start_txn) (struct pmu *pmu);
254
255
256
257
258
259
260 int (*commit_txn) (struct pmu *pmu);
261
262
263
264
265 void (*cancel_txn) (struct pmu *pmu);
266
267
268
269
270
271 int (*event_idx) (struct perf_event *event);
272
273
274
275
276 void (*flush_branch_stack) (void);
277};
278
279
280
281
282enum perf_event_active_state {
283 PERF_EVENT_STATE_ERROR = -2,
284 PERF_EVENT_STATE_OFF = -1,
285 PERF_EVENT_STATE_INACTIVE = 0,
286 PERF_EVENT_STATE_ACTIVE = 1,
287};
288
289struct file;
290struct perf_sample_data;
291
292typedef void (*perf_overflow_handler_t)(struct perf_event *,
293 struct perf_sample_data *,
294 struct pt_regs *regs);
295
296enum perf_group_flag {
297 PERF_GROUP_SOFTWARE = 0x1,
298};
299
300#define SWEVENT_HLIST_BITS 8
301#define SWEVENT_HLIST_SIZE (1 << SWEVENT_HLIST_BITS)
302
303struct swevent_hlist {
304 struct hlist_head heads[SWEVENT_HLIST_SIZE];
305 struct rcu_head rcu_head;
306};
307
308#define PERF_ATTACH_CONTEXT 0x01
309#define PERF_ATTACH_GROUP 0x02
310#define PERF_ATTACH_TASK 0x04
311
312struct perf_cgroup;
313struct ring_buffer;
314
315
316
317
318struct perf_event {
319#ifdef CONFIG_PERF_EVENTS
320 struct list_head group_entry;
321 struct list_head event_entry;
322 struct list_head sibling_list;
323 struct hlist_node hlist_entry;
324 int nr_siblings;
325 int group_flags;
326 struct perf_event *group_leader;
327 struct pmu *pmu;
328
329 enum perf_event_active_state state;
330 unsigned int attach_state;
331 local64_t count;
332 atomic64_t child_count;
333
334
335
336
337
338
339
340
341
342
343 u64 total_time_enabled;
344 u64 total_time_running;
345
346
347
348
349
350
351
352
353
354
355
356 u64 tstamp_enabled;
357 u64 tstamp_running;
358 u64 tstamp_stopped;
359
360
361
362
363
364
365
366
367
368 u64 shadow_ctx_time;
369
370 struct perf_event_attr attr;
371 u16 header_size;
372 u16 id_header_size;
373 u16 read_size;
374 struct hw_perf_event hw;
375
376 struct perf_event_context *ctx;
377 atomic_long_t refcount;
378
379
380
381
382
383 atomic64_t child_total_time_enabled;
384 atomic64_t child_total_time_running;
385
386
387
388
389 struct mutex child_mutex;
390 struct list_head child_list;
391 struct perf_event *parent;
392
393 int oncpu;
394 int cpu;
395
396 struct list_head owner_entry;
397 struct task_struct *owner;
398
399
400 struct mutex mmap_mutex;
401 atomic_t mmap_count;
402
403 struct ring_buffer *rb;
404 struct list_head rb_entry;
405
406
407 wait_queue_head_t waitq;
408 struct fasync_struct *fasync;
409
410
411 int pending_wakeup;
412 int pending_kill;
413 int pending_disable;
414 struct irq_work pending;
415
416 atomic_t event_limit;
417
418 void (*destroy)(struct perf_event *);
419 struct rcu_head rcu_head;
420
421 struct pid_namespace *ns;
422 u64 id;
423
424 perf_overflow_handler_t overflow_handler;
425 void *overflow_handler_context;
426
427#ifdef CONFIG_EVENT_TRACING
428 struct ftrace_event_call *tp_event;
429 struct event_filter *filter;
430#ifdef CONFIG_FUNCTION_TRACER
431 struct ftrace_ops ftrace_ops;
432#endif
433#endif
434
435#ifdef CONFIG_CGROUP_PERF
436 struct perf_cgroup *cgrp;
437 int cgrp_defer_enabled;
438#endif
439
440#endif
441};
442
443enum perf_event_context_type {
444 task_context,
445 cpu_context,
446};
447
448
449
450
451
452
453struct perf_event_context {
454 struct pmu *pmu;
455 enum perf_event_context_type type;
456
457
458
459
460 raw_spinlock_t lock;
461
462
463
464
465
466 struct mutex mutex;
467
468 struct list_head pinned_groups;
469 struct list_head flexible_groups;
470 struct list_head event_list;
471 int nr_events;
472 int nr_active;
473 int is_active;
474 int nr_stat;
475 int nr_freq;
476 int rotate_disable;
477 atomic_t refcount;
478 struct task_struct *task;
479
480
481
482
483 u64 time;
484 u64 timestamp;
485
486
487
488
489
490 struct perf_event_context *parent_ctx;
491 u64 parent_gen;
492 u64 generation;
493 int pin_count;
494 int nr_cgroups;
495 int nr_branch_stack;
496 struct rcu_head rcu_head;
497};
498
499
500
501
502
503#define PERF_NR_CONTEXTS 4
504
505
506
507
508struct perf_cpu_context {
509 struct perf_event_context ctx;
510 struct perf_event_context *task_ctx;
511 int active_oncpu;
512 int exclusive;
513 struct hrtimer hrtimer;
514 ktime_t hrtimer_interval;
515 struct list_head rotation_list;
516 struct pmu *unique_pmu;
517 struct perf_cgroup *cgrp;
518};
519
520struct perf_output_handle {
521 struct perf_event *event;
522 struct ring_buffer *rb;
523 unsigned long wakeup;
524 unsigned long size;
525 void *addr;
526 int page;
527};
528
529#ifdef CONFIG_PERF_EVENTS
530
531extern int perf_pmu_register(struct pmu *pmu, const char *name, int type);
532extern void perf_pmu_unregister(struct pmu *pmu);
533
534extern int perf_num_counters(void);
535extern const char *perf_pmu_name(void);
536extern void __perf_event_task_sched_in(struct task_struct *prev,
537 struct task_struct *task);
538extern void __perf_event_task_sched_out(struct task_struct *prev,
539 struct task_struct *next);
540extern int perf_event_init_task(struct task_struct *child);
541extern void perf_event_exit_task(struct task_struct *child);
542extern void perf_event_free_task(struct task_struct *task);
543extern void perf_event_delayed_put(struct task_struct *task);
544extern void perf_event_print_debug(void);
545extern void perf_pmu_disable(struct pmu *pmu);
546extern void perf_pmu_enable(struct pmu *pmu);
547extern int perf_event_task_disable(void);
548extern int perf_event_task_enable(void);
549extern int perf_event_refresh(struct perf_event *event, int refresh);
550extern void perf_event_update_userpage(struct perf_event *event);
551extern int perf_event_release_kernel(struct perf_event *event);
552extern struct perf_event *
553perf_event_create_kernel_counter(struct perf_event_attr *attr,
554 int cpu,
555 struct task_struct *task,
556 perf_overflow_handler_t callback,
557 void *context);
558extern void perf_pmu_migrate_context(struct pmu *pmu,
559 int src_cpu, int dst_cpu);
560extern u64 perf_event_read_value(struct perf_event *event,
561 u64 *enabled, u64 *running);
562
563
564struct perf_sample_data {
565 u64 type;
566
567 u64 ip;
568 struct {
569 u32 pid;
570 u32 tid;
571 } tid_entry;
572 u64 time;
573 u64 addr;
574 u64 id;
575 u64 stream_id;
576 struct {
577 u32 cpu;
578 u32 reserved;
579 } cpu_entry;
580 u64 period;
581 union perf_mem_data_src data_src;
582 struct perf_callchain_entry *callchain;
583 struct perf_raw_record *raw;
584 struct perf_branch_stack *br_stack;
585 struct perf_regs_user regs_user;
586 u64 stack_user_size;
587 u64 weight;
588};
589
590static inline void perf_sample_data_init(struct perf_sample_data *data,
591 u64 addr, u64 period)
592{
593
594 data->addr = addr;
595 data->raw = NULL;
596 data->br_stack = NULL;
597 data->period = period;
598 data->regs_user.abi = PERF_SAMPLE_REGS_ABI_NONE;
599 data->regs_user.regs = NULL;
600 data->stack_user_size = 0;
601 data->weight = 0;
602 data->data_src.val = 0;
603}
604
605extern void perf_output_sample(struct perf_output_handle *handle,
606 struct perf_event_header *header,
607 struct perf_sample_data *data,
608 struct perf_event *event);
609extern void perf_prepare_sample(struct perf_event_header *header,
610 struct perf_sample_data *data,
611 struct perf_event *event,
612 struct pt_regs *regs);
613
614extern int perf_event_overflow(struct perf_event *event,
615 struct perf_sample_data *data,
616 struct pt_regs *regs);
617
618static inline bool is_sampling_event(struct perf_event *event)
619{
620 return event->attr.sample_period != 0;
621}
622
623
624
625
626static inline int is_software_event(struct perf_event *event)
627{
628 return event->pmu->task_ctx_nr == perf_sw_context;
629}
630
631extern struct static_key perf_swevent_enabled[PERF_COUNT_SW_MAX];
632
633extern void __perf_sw_event(u32, u64, struct pt_regs *, u64);
634
635#ifndef perf_arch_fetch_caller_regs
636static inline void perf_arch_fetch_caller_regs(struct pt_regs *regs, unsigned long ip) { }
637#endif
638
639
640
641
642
643
644
645
646
647static inline void perf_fetch_caller_regs(struct pt_regs *regs)
648{
649 memset(regs, 0, sizeof(*regs));
650
651 perf_arch_fetch_caller_regs(regs, CALLER_ADDR0);
652}
653
654static __always_inline void
655perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr)
656{
657 struct pt_regs hot_regs;
658
659 if (static_key_false(&perf_swevent_enabled[event_id])) {
660 if (!regs) {
661 perf_fetch_caller_regs(&hot_regs);
662 regs = &hot_regs;
663 }
664 __perf_sw_event(event_id, nr, regs, addr);
665 }
666}
667
668extern struct static_key_deferred perf_sched_events;
669
670static inline void perf_event_task_sched_in(struct task_struct *prev,
671 struct task_struct *task)
672{
673 if (static_key_false(&perf_sched_events.key))
674 __perf_event_task_sched_in(prev, task);
675}
676
677static inline void perf_event_task_sched_out(struct task_struct *prev,
678 struct task_struct *next)
679{
680 perf_sw_event(PERF_COUNT_SW_CONTEXT_SWITCHES, 1, NULL, 0);
681
682 if (static_key_false(&perf_sched_events.key))
683 __perf_event_task_sched_out(prev, next);
684}
685
686extern void perf_event_mmap(struct vm_area_struct *vma);
687extern struct perf_guest_info_callbacks *perf_guest_cbs;
688extern int perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *callbacks);
689extern int perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *callbacks);
690
691extern void perf_event_comm(struct task_struct *tsk);
692extern void perf_event_fork(struct task_struct *tsk);
693
694
695DECLARE_PER_CPU(struct perf_callchain_entry, perf_callchain_entry);
696
697extern void perf_callchain_user(struct perf_callchain_entry *entry, struct pt_regs *regs);
698extern void perf_callchain_kernel(struct perf_callchain_entry *entry, struct pt_regs *regs);
699
700static inline void perf_callchain_store(struct perf_callchain_entry *entry, u64 ip)
701{
702 if (entry->nr < PERF_MAX_STACK_DEPTH)
703 entry->ip[entry->nr++] = ip;
704}
705
706extern int sysctl_perf_event_paranoid;
707extern int sysctl_perf_event_mlock;
708extern int sysctl_perf_event_sample_rate;
709extern int sysctl_perf_cpu_time_max_percent;
710
711extern void perf_sample_event_took(u64 sample_len_ns);
712
713extern int perf_proc_update_handler(struct ctl_table *table, int write,
714 void __user *buffer, size_t *lenp,
715 loff_t *ppos);
716extern int perf_cpu_time_max_percent_handler(struct ctl_table *table, int write,
717 void __user *buffer, size_t *lenp,
718 loff_t *ppos);
719
720
721static inline bool perf_paranoid_tracepoint_raw(void)
722{
723 return sysctl_perf_event_paranoid > -1;
724}
725
726static inline bool perf_paranoid_cpu(void)
727{
728 return sysctl_perf_event_paranoid > 0;
729}
730
731static inline bool perf_paranoid_kernel(void)
732{
733 return sysctl_perf_event_paranoid > 1;
734}
735
736extern void perf_event_init(void);
737extern void perf_tp_event(u64 addr, u64 count, void *record,
738 int entry_size, struct pt_regs *regs,
739 struct hlist_head *head, int rctx,
740 struct task_struct *task);
741extern void perf_bp_event(struct perf_event *event, void *data);
742
743#ifndef perf_misc_flags
744# define perf_misc_flags(regs) \
745 (user_mode(regs) ? PERF_RECORD_MISC_USER : PERF_RECORD_MISC_KERNEL)
746# define perf_instruction_pointer(regs) instruction_pointer(regs)
747#endif
748
749static inline bool has_branch_stack(struct perf_event *event)
750{
751 return event->attr.sample_type & PERF_SAMPLE_BRANCH_STACK;
752}
753
754extern int perf_output_begin(struct perf_output_handle *handle,
755 struct perf_event *event, unsigned int size);
756extern void perf_output_end(struct perf_output_handle *handle);
757extern unsigned int perf_output_copy(struct perf_output_handle *handle,
758 const void *buf, unsigned int len);
759extern unsigned int perf_output_skip(struct perf_output_handle *handle,
760 unsigned int len);
761extern int perf_swevent_get_recursion_context(void);
762extern void perf_swevent_put_recursion_context(int rctx);
763extern u64 perf_swevent_set_period(struct perf_event *event);
764extern void perf_event_enable(struct perf_event *event);
765extern void perf_event_disable(struct perf_event *event);
766extern int __perf_event_disable(void *info);
767extern void perf_event_task_tick(void);
768#else
769static inline void
770perf_event_task_sched_in(struct task_struct *prev,
771 struct task_struct *task) { }
772static inline void
773perf_event_task_sched_out(struct task_struct *prev,
774 struct task_struct *next) { }
775static inline int perf_event_init_task(struct task_struct *child) { return 0; }
776static inline void perf_event_exit_task(struct task_struct *child) { }
777static inline void perf_event_free_task(struct task_struct *task) { }
778static inline void perf_event_delayed_put(struct task_struct *task) { }
779static inline void perf_event_print_debug(void) { }
780static inline int perf_event_task_disable(void) { return -EINVAL; }
781static inline int perf_event_task_enable(void) { return -EINVAL; }
782static inline int perf_event_refresh(struct perf_event *event, int refresh)
783{
784 return -EINVAL;
785}
786
787static inline void
788perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr) { }
789static inline void
790perf_bp_event(struct perf_event *event, void *data) { }
791
792static inline int perf_register_guest_info_callbacks
793(struct perf_guest_info_callbacks *callbacks) { return 0; }
794static inline int perf_unregister_guest_info_callbacks
795(struct perf_guest_info_callbacks *callbacks) { return 0; }
796
797static inline void perf_event_mmap(struct vm_area_struct *vma) { }
798static inline void perf_event_comm(struct task_struct *tsk) { }
799static inline void perf_event_fork(struct task_struct *tsk) { }
800static inline void perf_event_init(void) { }
801static inline int perf_swevent_get_recursion_context(void) { return -1; }
802static inline void perf_swevent_put_recursion_context(int rctx) { }
803static inline u64 perf_swevent_set_period(struct perf_event *event) { return 0; }
804static inline void perf_event_enable(struct perf_event *event) { }
805static inline void perf_event_disable(struct perf_event *event) { }
806static inline int __perf_event_disable(void *info) { return -1; }
807static inline void perf_event_task_tick(void) { }
808#endif
809
810#if defined(CONFIG_PERF_EVENTS) && defined(CONFIG_NO_HZ_FULL)
811extern bool perf_event_can_stop_tick(void);
812#else
813static inline bool perf_event_can_stop_tick(void) { return true; }
814#endif
815
816#if defined(CONFIG_PERF_EVENTS) && defined(CONFIG_CPU_SUP_INTEL)
817extern void perf_restore_debug_store(void);
818#else
819static inline void perf_restore_debug_store(void) { }
820#endif
821
822#define perf_output_put(handle, x) perf_output_copy((handle), &(x), sizeof(x))
823
824
825
826
827#define perf_cpu_notifier(fn) \
828do { \
829 static struct notifier_block fn##_nb = \
830 { .notifier_call = fn, .priority = CPU_PRI_PERF }; \
831 unsigned long cpu = smp_processor_id(); \
832 unsigned long flags; \
833 fn(&fn##_nb, (unsigned long)CPU_UP_PREPARE, \
834 (void *)(unsigned long)cpu); \
835 local_irq_save(flags); \
836 fn(&fn##_nb, (unsigned long)CPU_STARTING, \
837 (void *)(unsigned long)cpu); \
838 local_irq_restore(flags); \
839 fn(&fn##_nb, (unsigned long)CPU_ONLINE, \
840 (void *)(unsigned long)cpu); \
841 register_cpu_notifier(&fn##_nb); \
842} while (0)
843
844
845struct perf_pmu_events_attr {
846 struct device_attribute attr;
847 u64 id;
848 const char *event_str;
849};
850
851#define PMU_EVENT_ATTR(_name, _var, _id, _show) \
852static struct perf_pmu_events_attr _var = { \
853 .attr = __ATTR(_name, 0444, _show, NULL), \
854 .id = _id, \
855};
856
857#define PMU_FORMAT_ATTR(_name, _format) \
858static ssize_t \
859_name##_show(struct device *dev, \
860 struct device_attribute *attr, \
861 char *page) \
862{ \
863 BUILD_BUG_ON(sizeof(_format) >= PAGE_SIZE); \
864 return sprintf(page, _format "\n"); \
865} \
866 \
867static struct device_attribute format_attr_##_name = __ATTR_RO(_name)
868
869#endif
870