1
2
3
4
5
6
7
8
9
10
11
12
13
14#ifndef _LINUX_PERF_EVENT_H
15#define _LINUX_PERF_EVENT_H
16
17#include <uapi/linux/perf_event.h>
18#include <uapi/linux/bpf_perf_event.h>
19
20
21
22
23
24#ifdef CONFIG_PERF_EVENTS
25# include <asm/perf_event.h>
26# include <asm/local64.h>
27#endif
28
29struct perf_guest_info_callbacks {
30 int (*is_in_guest)(void);
31 int (*is_user_mode)(void);
32 unsigned long (*get_guest_ip)(void);
33 void (*handle_intel_pt_intr)(void);
34};
35
36#ifdef CONFIG_HAVE_HW_BREAKPOINT
37#include <asm/hw_breakpoint.h>
38#endif
39
40#include <linux/list.h>
41#include <linux/mutex.h>
42#include <linux/rculist.h>
43#include <linux/rcupdate.h>
44#include <linux/spinlock.h>
45#include <linux/hrtimer.h>
46#include <linux/fs.h>
47#include <linux/pid_namespace.h>
48#include <linux/workqueue.h>
49#include <linux/ftrace.h>
50#include <linux/cpu.h>
51#include <linux/irq_work.h>
52#include <linux/static_key.h>
53#include <linux/jump_label_ratelimit.h>
54#include <linux/atomic.h>
55#include <linux/sysfs.h>
56#include <linux/perf_regs.h>
57#include <linux/cgroup.h>
58#include <linux/refcount.h>
59#include <linux/security.h>
60#include <linux/static_call.h>
61#include <asm/local.h>
62
63struct perf_callchain_entry {
64 __u64 nr;
65 __u64 ip[];
66};
67
68struct perf_callchain_entry_ctx {
69 struct perf_callchain_entry *entry;
70 u32 max_stack;
71 u32 nr;
72 short contexts;
73 bool contexts_maxed;
74};
75
76typedef unsigned long (*perf_copy_f)(void *dst, const void *src,
77 unsigned long off, unsigned long len);
78
79struct perf_raw_frag {
80 union {
81 struct perf_raw_frag *next;
82 unsigned long pad;
83 };
84 perf_copy_f copy;
85 void *data;
86 u32 size;
87} __packed;
88
89struct perf_raw_record {
90 struct perf_raw_frag frag;
91 u32 size;
92};
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114struct perf_branch_stack {
115 __u64 nr;
116 __u64 hw_idx;
117 struct perf_branch_entry entries[];
118};
119
120struct task_struct;
121
122
123
124
125struct hw_perf_event_extra {
126 u64 config;
127 unsigned int reg;
128 int alloc;
129 int idx;
130};
131
132
133
134
135struct hw_perf_event {
136#ifdef CONFIG_PERF_EVENTS
137 union {
138 struct {
139 u64 config;
140 u64 last_tag;
141 unsigned long config_base;
142 unsigned long event_base;
143 int event_base_rdpmc;
144 int idx;
145 int last_cpu;
146 int flags;
147
148 struct hw_perf_event_extra extra_reg;
149 struct hw_perf_event_extra branch_reg;
150 };
151 struct {
152 struct hrtimer hrtimer;
153 };
154 struct {
155
156 struct list_head tp_list;
157 };
158 struct {
159 u64 pwr_acc;
160 u64 ptsc;
161 };
162#ifdef CONFIG_HAVE_HW_BREAKPOINT
163 struct {
164
165
166
167
168
169 struct arch_hw_breakpoint info;
170 struct list_head bp_list;
171 };
172#endif
173 struct {
174 u8 iommu_bank;
175 u8 iommu_cntr;
176 u16 padding;
177 u64 conf;
178 u64 conf1;
179 };
180 };
181
182
183
184
185 struct task_struct *target;
186
187
188
189
190
191 void *addr_filters;
192
193
194 unsigned long addr_filters_gen;
195
196
197
198
199#define PERF_HES_STOPPED 0x01
200#define PERF_HES_UPTODATE 0x02
201#define PERF_HES_ARCH 0x04
202
203 int state;
204
205
206
207
208
209 local64_t prev_count;
210
211
212
213
214 u64 sample_period;
215
216 union {
217 struct {
218
219
220
221 u64 last_period;
222
223
224
225
226
227
228
229 local64_t period_left;
230 };
231 struct {
232 u64 saved_metric;
233 u64 saved_slots;
234 };
235 };
236
237
238
239
240
241 u64 interrupts_seq;
242 u64 interrupts;
243
244
245
246
247
248 u64 freq_time_stamp;
249 u64 freq_count_stamp;
250#endif
251};
252
253struct perf_event;
254
255
256
257
258#define PERF_PMU_TXN_ADD 0x1
259#define PERF_PMU_TXN_READ 0x2
260
261
262
263
264#define PERF_PMU_CAP_NO_INTERRUPT 0x0001
265#define PERF_PMU_CAP_NO_NMI 0x0002
266#define PERF_PMU_CAP_AUX_NO_SG 0x0004
267#define PERF_PMU_CAP_EXTENDED_REGS 0x0008
268#define PERF_PMU_CAP_EXCLUSIVE 0x0010
269#define PERF_PMU_CAP_ITRACE 0x0020
270#define PERF_PMU_CAP_HETEROGENEOUS_CPUS 0x0040
271#define PERF_PMU_CAP_NO_EXCLUDE 0x0080
272#define PERF_PMU_CAP_AUX_OUTPUT 0x0100
273#define PERF_PMU_CAP_EXTENDED_HW_TYPE 0x0200
274
275struct perf_output_handle;
276
277
278
279
280struct pmu {
281 struct list_head entry;
282
283 struct module *module;
284 struct device *dev;
285 const struct attribute_group **attr_groups;
286 const struct attribute_group **attr_update;
287 const char *name;
288 int type;
289
290
291
292
293 int capabilities;
294
295 int __percpu *pmu_disable_count;
296 struct perf_cpu_context __percpu *pmu_cpu_context;
297 atomic_t exclusive_cnt;
298 int task_ctx_nr;
299 int hrtimer_interval_ms;
300
301
302 unsigned int nr_addr_filters;
303
304
305
306
307
308 void (*pmu_enable) (struct pmu *pmu);
309 void (*pmu_disable) (struct pmu *pmu);
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327 int (*event_init) (struct perf_event *event);
328
329
330
331
332
333 void (*event_mapped) (struct perf_event *event, struct mm_struct *mm);
334 void (*event_unmapped) (struct perf_event *event, struct mm_struct *mm);
335
336
337
338
339
340#define PERF_EF_START 0x01
341#define PERF_EF_RELOAD 0x02
342#define PERF_EF_UPDATE 0x04
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362 int (*add) (struct perf_event *event, int flags);
363 void (*del) (struct perf_event *event, int flags);
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383 void (*start) (struct perf_event *event, int flags);
384 void (*stop) (struct perf_event *event, int flags);
385
386
387
388
389
390
391
392 void (*read) (struct perf_event *event);
393
394
395
396
397
398
399
400
401
402
403
404 void (*start_txn) (struct pmu *pmu, unsigned int txn_flags);
405
406
407
408
409
410
411
412
413 int (*commit_txn) (struct pmu *pmu);
414
415
416
417
418
419
420 void (*cancel_txn) (struct pmu *pmu);
421
422
423
424
425
426 int (*event_idx) (struct perf_event *event);
427
428
429
430
431 void (*sched_task) (struct perf_event_context *ctx,
432 bool sched_in);
433
434
435
436
437 struct kmem_cache *task_ctx_cache;
438
439
440
441
442
443
444
445 void (*swap_task_ctx) (struct perf_event_context *prev,
446 struct perf_event_context *next);
447
448
449
450
451
452 void *(*setup_aux) (struct perf_event *event, void **pages,
453 int nr_pages, bool overwrite);
454
455
456
457
458
459 void (*free_aux) (void *aux);
460
461
462
463
464
465
466
467
468
469
470 long (*snapshot_aux) (struct perf_event *event,
471 struct perf_output_handle *handle,
472 unsigned long size);
473
474
475
476
477
478
479
480
481
482 int (*addr_filters_validate) (struct list_head *filters);
483
484
485
486
487
488
489
490
491
492
493
494
495
496 void (*addr_filters_sync) (struct perf_event *event);
497
498
499
500
501
502
503
504
505
506 int (*aux_output_match) (struct perf_event *event);
507
508
509
510
511
512 int (*filter_match) (struct perf_event *event);
513
514
515
516
517 int (*check_period) (struct perf_event *event, u64 value);
518};
519
520enum perf_addr_filter_action_t {
521 PERF_ADDR_FILTER_ACTION_STOP = 0,
522 PERF_ADDR_FILTER_ACTION_START,
523 PERF_ADDR_FILTER_ACTION_FILTER,
524};
525
526
527
528
529
530
531
532
533
534
535
536struct perf_addr_filter {
537 struct list_head entry;
538 struct path path;
539 unsigned long offset;
540 unsigned long size;
541 enum perf_addr_filter_action_t action;
542};
543
544
545
546
547
548
549
550
551
552
553
554struct perf_addr_filters_head {
555 struct list_head list;
556 raw_spinlock_t lock;
557 unsigned int nr_file_filters;
558};
559
560struct perf_addr_filter_range {
561 unsigned long start;
562 unsigned long size;
563};
564
565
566
567
568enum perf_event_state {
569 PERF_EVENT_STATE_DEAD = -4,
570 PERF_EVENT_STATE_EXIT = -3,
571 PERF_EVENT_STATE_ERROR = -2,
572 PERF_EVENT_STATE_OFF = -1,
573 PERF_EVENT_STATE_INACTIVE = 0,
574 PERF_EVENT_STATE_ACTIVE = 1,
575};
576
577struct file;
578struct perf_sample_data;
579
580typedef void (*perf_overflow_handler_t)(struct perf_event *,
581 struct perf_sample_data *,
582 struct pt_regs *regs);
583
584
585
586
587
588
589
590
591
592
593
594#define PERF_EV_CAP_SOFTWARE BIT(0)
595#define PERF_EV_CAP_READ_ACTIVE_PKG BIT(1)
596#define PERF_EV_CAP_SIBLING BIT(2)
597
598#define SWEVENT_HLIST_BITS 8
599#define SWEVENT_HLIST_SIZE (1 << SWEVENT_HLIST_BITS)
600
601struct swevent_hlist {
602 struct hlist_head heads[SWEVENT_HLIST_SIZE];
603 struct rcu_head rcu_head;
604};
605
606#define PERF_ATTACH_CONTEXT 0x01
607#define PERF_ATTACH_GROUP 0x02
608#define PERF_ATTACH_TASK 0x04
609#define PERF_ATTACH_TASK_DATA 0x08
610#define PERF_ATTACH_ITRACE 0x10
611#define PERF_ATTACH_SCHED_CB 0x20
612#define PERF_ATTACH_CHILD 0x40
613
614struct perf_cgroup;
615struct perf_buffer;
616
617struct pmu_event_list {
618 raw_spinlock_t lock;
619 struct list_head list;
620};
621
622#define for_each_sibling_event(sibling, event) \
623 if ((event)->group_leader == (event)) \
624 list_for_each_entry((sibling), &(event)->sibling_list, sibling_list)
625
626
627
628
629struct perf_event {
630#ifdef CONFIG_PERF_EVENTS
631
632
633
634
635
636 struct list_head event_entry;
637
638
639
640
641
642 struct list_head sibling_list;
643 struct list_head active_list;
644
645
646
647 struct rb_node group_node;
648 u64 group_index;
649
650
651
652
653
654 struct list_head migrate_entry;
655
656 struct hlist_node hlist_entry;
657 struct list_head active_entry;
658 int nr_siblings;
659
660
661 int event_caps;
662
663 int group_caps;
664
665 struct perf_event *group_leader;
666 struct pmu *pmu;
667 void *pmu_private;
668
669 enum perf_event_state state;
670 unsigned int attach_state;
671 local64_t count;
672 atomic64_t child_count;
673
674
675
676
677
678
679
680 u64 total_time_enabled;
681 u64 total_time_running;
682 u64 tstamp;
683
684
685
686
687
688
689
690
691
692
693
694 u64 shadow_ctx_time;
695
696 struct perf_event_attr attr;
697 u16 header_size;
698 u16 id_header_size;
699 u16 read_size;
700 struct hw_perf_event hw;
701
702 struct perf_event_context *ctx;
703 atomic_long_t refcount;
704
705
706
707
708
709 atomic64_t child_total_time_enabled;
710 atomic64_t child_total_time_running;
711
712
713
714
715 struct mutex child_mutex;
716 struct list_head child_list;
717 struct perf_event *parent;
718
719 int oncpu;
720 int cpu;
721
722 struct list_head owner_entry;
723 struct task_struct *owner;
724
725
726 struct mutex mmap_mutex;
727 atomic_t mmap_count;
728
729 struct perf_buffer *rb;
730 struct list_head rb_entry;
731 unsigned long rcu_batches;
732 int rcu_pending;
733
734
735 wait_queue_head_t waitq;
736 struct fasync_struct *fasync;
737
738
739 int pending_wakeup;
740 int pending_kill;
741 int pending_disable;
742 unsigned long pending_addr;
743 struct irq_work pending;
744
745 atomic_t event_limit;
746
747
748 struct perf_addr_filters_head addr_filters;
749
750 struct perf_addr_filter_range *addr_filter_ranges;
751 unsigned long addr_filters_gen;
752
753
754 struct perf_event *aux_event;
755
756 void (*destroy)(struct perf_event *);
757 struct rcu_head rcu_head;
758
759 struct pid_namespace *ns;
760 u64 id;
761
762 u64 (*clock)(void);
763 perf_overflow_handler_t overflow_handler;
764 void *overflow_handler_context;
765#ifdef CONFIG_BPF_SYSCALL
766 perf_overflow_handler_t orig_overflow_handler;
767 struct bpf_prog *prog;
768 u64 bpf_cookie;
769#endif
770
771#ifdef CONFIG_EVENT_TRACING
772 struct trace_event_call *tp_event;
773 struct event_filter *filter;
774#ifdef CONFIG_FUNCTION_TRACER
775 struct ftrace_ops ftrace_ops;
776#endif
777#endif
778
779#ifdef CONFIG_CGROUP_PERF
780 struct perf_cgroup *cgrp;
781#endif
782
783#ifdef CONFIG_SECURITY
784 void *security;
785#endif
786 struct list_head sb_list;
787#endif
788};
789
790
791struct perf_event_groups {
792 struct rb_root tree;
793 u64 index;
794};
795
796
797
798
799
800
801struct perf_event_context {
802 struct pmu *pmu;
803
804
805
806
807 raw_spinlock_t lock;
808
809
810
811
812
813 struct mutex mutex;
814
815 struct list_head active_ctx_list;
816 struct perf_event_groups pinned_groups;
817 struct perf_event_groups flexible_groups;
818 struct list_head event_list;
819
820 struct list_head pinned_active;
821 struct list_head flexible_active;
822
823 int nr_events;
824 int nr_active;
825 int is_active;
826 int nr_stat;
827 int nr_freq;
828 int rotate_disable;
829
830
831
832
833 int rotate_necessary;
834 refcount_t refcount;
835 struct task_struct *task;
836
837
838
839
840 u64 time;
841 u64 timestamp;
842
843
844
845
846
847 struct perf_event_context *parent_ctx;
848 u64 parent_gen;
849 u64 generation;
850 int pin_count;
851#ifdef CONFIG_CGROUP_PERF
852 int nr_cgroups;
853#endif
854 void *task_ctx_data;
855 struct rcu_head rcu_head;
856};
857
858
859
860
861
862#define PERF_NR_CONTEXTS 4
863
864
865
866
867struct perf_cpu_context {
868 struct perf_event_context ctx;
869 struct perf_event_context *task_ctx;
870 int active_oncpu;
871 int exclusive;
872
873 raw_spinlock_t hrtimer_lock;
874 struct hrtimer hrtimer;
875 ktime_t hrtimer_interval;
876 unsigned int hrtimer_active;
877
878#ifdef CONFIG_CGROUP_PERF
879 struct perf_cgroup *cgrp;
880 struct list_head cgrp_cpuctx_entry;
881#endif
882
883 struct list_head sched_cb_entry;
884 int sched_cb_usage;
885
886 int online;
887
888
889
890
891 int heap_size;
892 struct perf_event **heap;
893 struct perf_event *heap_default[2];
894};
895
896struct perf_output_handle {
897 struct perf_event *event;
898 struct perf_buffer *rb;
899 unsigned long wakeup;
900 unsigned long size;
901 u64 aux_flags;
902 union {
903 void *addr;
904 unsigned long head;
905 };
906 int page;
907};
908
909struct bpf_perf_event_data_kern {
910 bpf_user_pt_regs_t *regs;
911 struct perf_sample_data *data;
912 struct perf_event *event;
913};
914
915#ifdef CONFIG_CGROUP_PERF
916
917
918
919
920
921struct perf_cgroup_info {
922 u64 time;
923 u64 timestamp;
924};
925
926struct perf_cgroup {
927 struct cgroup_subsys_state css;
928 struct perf_cgroup_info __percpu *info;
929};
930
931
932
933
934
935
936static inline struct perf_cgroup *
937perf_cgroup_from_task(struct task_struct *task, struct perf_event_context *ctx)
938{
939 return container_of(task_css_check(task, perf_event_cgrp_id,
940 ctx ? lockdep_is_held(&ctx->lock)
941 : true),
942 struct perf_cgroup, css);
943}
944#endif
945
946#ifdef CONFIG_PERF_EVENTS
947
948extern void *perf_aux_output_begin(struct perf_output_handle *handle,
949 struct perf_event *event);
950extern void perf_aux_output_end(struct perf_output_handle *handle,
951 unsigned long size);
952extern int perf_aux_output_skip(struct perf_output_handle *handle,
953 unsigned long size);
954extern void *perf_get_aux(struct perf_output_handle *handle);
955extern void perf_aux_output_flag(struct perf_output_handle *handle, u64 flags);
956extern void perf_event_itrace_started(struct perf_event *event);
957
958extern int perf_pmu_register(struct pmu *pmu, const char *name, int type);
959extern void perf_pmu_unregister(struct pmu *pmu);
960
961extern void __perf_event_task_sched_in(struct task_struct *prev,
962 struct task_struct *task);
963extern void __perf_event_task_sched_out(struct task_struct *prev,
964 struct task_struct *next);
965extern int perf_event_init_task(struct task_struct *child, u64 clone_flags);
966extern void perf_event_exit_task(struct task_struct *child);
967extern void perf_event_free_task(struct task_struct *task);
968extern void perf_event_delayed_put(struct task_struct *task);
969extern struct file *perf_event_get(unsigned int fd);
970extern const struct perf_event *perf_get_event(struct file *file);
971extern const struct perf_event_attr *perf_event_attrs(struct perf_event *event);
972extern void perf_event_print_debug(void);
973extern void perf_pmu_disable(struct pmu *pmu);
974extern void perf_pmu_enable(struct pmu *pmu);
975extern void perf_sched_cb_dec(struct pmu *pmu);
976extern void perf_sched_cb_inc(struct pmu *pmu);
977extern int perf_event_task_disable(void);
978extern int perf_event_task_enable(void);
979
980extern void perf_pmu_resched(struct pmu *pmu);
981
982extern int perf_event_refresh(struct perf_event *event, int refresh);
983extern void perf_event_update_userpage(struct perf_event *event);
984extern int perf_event_release_kernel(struct perf_event *event);
985extern struct perf_event *
986perf_event_create_kernel_counter(struct perf_event_attr *attr,
987 int cpu,
988 struct task_struct *task,
989 perf_overflow_handler_t callback,
990 void *context);
991extern void perf_pmu_migrate_context(struct pmu *pmu,
992 int src_cpu, int dst_cpu);
993int perf_event_read_local(struct perf_event *event, u64 *value,
994 u64 *enabled, u64 *running);
995extern u64 perf_event_read_value(struct perf_event *event,
996 u64 *enabled, u64 *running);
997
998
999struct perf_sample_data {
1000
1001
1002
1003
1004 u64 addr;
1005 struct perf_raw_record *raw;
1006 struct perf_branch_stack *br_stack;
1007 u64 period;
1008 union perf_sample_weight weight;
1009 u64 txn;
1010 union perf_mem_data_src data_src;
1011
1012
1013
1014
1015
1016 u64 type;
1017 u64 ip;
1018 struct {
1019 u32 pid;
1020 u32 tid;
1021 } tid_entry;
1022 u64 time;
1023 u64 id;
1024 u64 stream_id;
1025 struct {
1026 u32 cpu;
1027 u32 reserved;
1028 } cpu_entry;
1029 struct perf_callchain_entry *callchain;
1030 u64 aux_size;
1031
1032 struct perf_regs regs_user;
1033 struct perf_regs regs_intr;
1034 u64 stack_user_size;
1035
1036 u64 phys_addr;
1037 u64 cgroup;
1038 u64 data_page_size;
1039 u64 code_page_size;
1040} ____cacheline_aligned;
1041
1042
1043#define PERF_MEM_NA (PERF_MEM_S(OP, NA) |\
1044 PERF_MEM_S(LVL, NA) |\
1045 PERF_MEM_S(SNOOP, NA) |\
1046 PERF_MEM_S(LOCK, NA) |\
1047 PERF_MEM_S(TLB, NA))
1048
1049static inline void perf_sample_data_init(struct perf_sample_data *data,
1050 u64 addr, u64 period)
1051{
1052
1053 data->addr = addr;
1054 data->raw = NULL;
1055 data->br_stack = NULL;
1056 data->period = period;
1057 data->weight.full = 0;
1058 data->data_src.val = PERF_MEM_NA;
1059 data->txn = 0;
1060}
1061
1062extern void perf_output_sample(struct perf_output_handle *handle,
1063 struct perf_event_header *header,
1064 struct perf_sample_data *data,
1065 struct perf_event *event);
1066extern void perf_prepare_sample(struct perf_event_header *header,
1067 struct perf_sample_data *data,
1068 struct perf_event *event,
1069 struct pt_regs *regs);
1070
1071extern int perf_event_overflow(struct perf_event *event,
1072 struct perf_sample_data *data,
1073 struct pt_regs *regs);
1074
1075extern void perf_event_output_forward(struct perf_event *event,
1076 struct perf_sample_data *data,
1077 struct pt_regs *regs);
1078extern void perf_event_output_backward(struct perf_event *event,
1079 struct perf_sample_data *data,
1080 struct pt_regs *regs);
1081extern int perf_event_output(struct perf_event *event,
1082 struct perf_sample_data *data,
1083 struct pt_regs *regs);
1084
1085static inline bool
1086is_default_overflow_handler(struct perf_event *event)
1087{
1088 if (likely(event->overflow_handler == perf_event_output_forward))
1089 return true;
1090 if (unlikely(event->overflow_handler == perf_event_output_backward))
1091 return true;
1092 return false;
1093}
1094
1095extern void
1096perf_event_header__init_id(struct perf_event_header *header,
1097 struct perf_sample_data *data,
1098 struct perf_event *event);
1099extern void
1100perf_event__output_id_sample(struct perf_event *event,
1101 struct perf_output_handle *handle,
1102 struct perf_sample_data *sample);
1103
1104extern void
1105perf_log_lost_samples(struct perf_event *event, u64 lost);
1106
1107static inline bool event_has_any_exclude_flag(struct perf_event *event)
1108{
1109 struct perf_event_attr *attr = &event->attr;
1110
1111 return attr->exclude_idle || attr->exclude_user ||
1112 attr->exclude_kernel || attr->exclude_hv ||
1113 attr->exclude_guest || attr->exclude_host;
1114}
1115
1116static inline bool is_sampling_event(struct perf_event *event)
1117{
1118 return event->attr.sample_period != 0;
1119}
1120
1121
1122
1123
1124static inline int is_software_event(struct perf_event *event)
1125{
1126 return event->event_caps & PERF_EV_CAP_SOFTWARE;
1127}
1128
1129
1130
1131
1132static inline int in_software_context(struct perf_event *event)
1133{
1134 return event->ctx->pmu->task_ctx_nr == perf_sw_context;
1135}
1136
1137static inline int is_exclusive_pmu(struct pmu *pmu)
1138{
1139 return pmu->capabilities & PERF_PMU_CAP_EXCLUSIVE;
1140}
1141
1142extern struct static_key perf_swevent_enabled[PERF_COUNT_SW_MAX];
1143
1144extern void ___perf_sw_event(u32, u64, struct pt_regs *, u64);
1145extern void __perf_sw_event(u32, u64, struct pt_regs *, u64);
1146
1147#ifndef perf_arch_fetch_caller_regs
1148static inline void perf_arch_fetch_caller_regs(struct pt_regs *regs, unsigned long ip) { }
1149#endif
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165static inline void perf_fetch_caller_regs(struct pt_regs *regs)
1166{
1167 perf_arch_fetch_caller_regs(regs, CALLER_ADDR0);
1168}
1169
1170static __always_inline void
1171perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr)
1172{
1173 if (static_key_false(&perf_swevent_enabled[event_id]))
1174 __perf_sw_event(event_id, nr, regs, addr);
1175}
1176
1177DECLARE_PER_CPU(struct pt_regs, __perf_regs[4]);
1178
1179
1180
1181
1182
1183
1184static __always_inline void __perf_sw_event_sched(u32 event_id, u64 nr, u64 addr)
1185{
1186 struct pt_regs *regs = this_cpu_ptr(&__perf_regs[0]);
1187
1188 perf_fetch_caller_regs(regs);
1189 ___perf_sw_event(event_id, nr, regs, addr);
1190}
1191
1192extern struct static_key_false perf_sched_events;
1193
1194static __always_inline bool __perf_sw_enabled(int swevt)
1195{
1196 return static_key_false(&perf_swevent_enabled[swevt]);
1197}
1198
1199static inline void perf_event_task_migrate(struct task_struct *task)
1200{
1201 if (__perf_sw_enabled(PERF_COUNT_SW_CPU_MIGRATIONS))
1202 task->sched_migrated = 1;
1203}
1204
1205static inline void perf_event_task_sched_in(struct task_struct *prev,
1206 struct task_struct *task)
1207{
1208 if (static_branch_unlikely(&perf_sched_events))
1209 __perf_event_task_sched_in(prev, task);
1210
1211 if (__perf_sw_enabled(PERF_COUNT_SW_CPU_MIGRATIONS) &&
1212 task->sched_migrated) {
1213 __perf_sw_event_sched(PERF_COUNT_SW_CPU_MIGRATIONS, 1, 0);
1214 task->sched_migrated = 0;
1215 }
1216}
1217
1218static inline void perf_event_task_sched_out(struct task_struct *prev,
1219 struct task_struct *next)
1220{
1221 if (__perf_sw_enabled(PERF_COUNT_SW_CONTEXT_SWITCHES))
1222 __perf_sw_event_sched(PERF_COUNT_SW_CONTEXT_SWITCHES, 1, 0);
1223
1224#ifdef CONFIG_CGROUP_PERF
1225 if (__perf_sw_enabled(PERF_COUNT_SW_CGROUP_SWITCHES) &&
1226 perf_cgroup_from_task(prev, NULL) !=
1227 perf_cgroup_from_task(next, NULL))
1228 __perf_sw_event_sched(PERF_COUNT_SW_CGROUP_SWITCHES, 1, 0);
1229#endif
1230
1231 if (static_branch_unlikely(&perf_sched_events))
1232 __perf_event_task_sched_out(prev, next);
1233}
1234
1235extern void perf_event_mmap(struct vm_area_struct *vma);
1236
1237extern void perf_event_ksymbol(u16 ksym_type, u64 addr, u32 len,
1238 bool unregister, const char *sym);
1239extern void perf_event_bpf_event(struct bpf_prog *prog,
1240 enum perf_bpf_event_type type,
1241 u16 flags);
1242
1243extern struct perf_guest_info_callbacks *perf_guest_cbs;
1244extern int perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *callbacks);
1245extern int perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *callbacks);
1246
1247extern void perf_event_exec(void);
1248extern void perf_event_comm(struct task_struct *tsk, bool exec);
1249extern void perf_event_namespaces(struct task_struct *tsk);
1250extern void perf_event_fork(struct task_struct *tsk);
1251extern void perf_event_text_poke(const void *addr,
1252 const void *old_bytes, size_t old_len,
1253 const void *new_bytes, size_t new_len);
1254
1255
1256DECLARE_PER_CPU(struct perf_callchain_entry, perf_callchain_entry);
1257
1258extern void perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs);
1259extern void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs);
1260extern struct perf_callchain_entry *
1261get_perf_callchain(struct pt_regs *regs, u32 init_nr, bool kernel, bool user,
1262 u32 max_stack, bool crosstask, bool add_mark);
1263extern struct perf_callchain_entry *perf_callchain(struct perf_event *event, struct pt_regs *regs);
1264extern int get_callchain_buffers(int max_stack);
1265extern void put_callchain_buffers(void);
1266extern struct perf_callchain_entry *get_callchain_entry(int *rctx);
1267extern void put_callchain_entry(int rctx);
1268
1269extern int sysctl_perf_event_max_stack;
1270extern int sysctl_perf_event_max_contexts_per_stack;
1271
1272static inline int perf_callchain_store_context(struct perf_callchain_entry_ctx *ctx, u64 ip)
1273{
1274 if (ctx->contexts < sysctl_perf_event_max_contexts_per_stack) {
1275 struct perf_callchain_entry *entry = ctx->entry;
1276 entry->ip[entry->nr++] = ip;
1277 ++ctx->contexts;
1278 return 0;
1279 } else {
1280 ctx->contexts_maxed = true;
1281 return -1;
1282 }
1283}
1284
1285static inline int perf_callchain_store(struct perf_callchain_entry_ctx *ctx, u64 ip)
1286{
1287 if (ctx->nr < ctx->max_stack && !ctx->contexts_maxed) {
1288 struct perf_callchain_entry *entry = ctx->entry;
1289 entry->ip[entry->nr++] = ip;
1290 ++ctx->nr;
1291 return 0;
1292 } else {
1293 return -1;
1294 }
1295}
1296
1297extern int sysctl_perf_event_paranoid;
1298extern int sysctl_perf_event_mlock;
1299extern int sysctl_perf_event_sample_rate;
1300extern int sysctl_perf_cpu_time_max_percent;
1301
1302extern void perf_sample_event_took(u64 sample_len_ns);
1303
1304int perf_proc_update_handler(struct ctl_table *table, int write,
1305 void *buffer, size_t *lenp, loff_t *ppos);
1306int perf_cpu_time_max_percent_handler(struct ctl_table *table, int write,
1307 void *buffer, size_t *lenp, loff_t *ppos);
1308int perf_event_max_stack_handler(struct ctl_table *table, int write,
1309 void *buffer, size_t *lenp, loff_t *ppos);
1310
1311
1312#define PERF_SECURITY_OPEN 0
1313
1314
1315#define PERF_SECURITY_CPU 1
1316#define PERF_SECURITY_KERNEL 2
1317#define PERF_SECURITY_TRACEPOINT 3
1318
1319static inline int perf_is_paranoid(void)
1320{
1321 return sysctl_perf_event_paranoid > -1;
1322}
1323
1324static inline int perf_allow_kernel(struct perf_event_attr *attr)
1325{
1326 if (sysctl_perf_event_paranoid > 1 && !perfmon_capable())
1327 return -EACCES;
1328
1329 return security_perf_event_open(attr, PERF_SECURITY_KERNEL);
1330}
1331
1332static inline int perf_allow_cpu(struct perf_event_attr *attr)
1333{
1334 if (sysctl_perf_event_paranoid > 0 && !perfmon_capable())
1335 return -EACCES;
1336
1337 return security_perf_event_open(attr, PERF_SECURITY_CPU);
1338}
1339
1340static inline int perf_allow_tracepoint(struct perf_event_attr *attr)
1341{
1342 if (sysctl_perf_event_paranoid > -1 && !perfmon_capable())
1343 return -EPERM;
1344
1345 return security_perf_event_open(attr, PERF_SECURITY_TRACEPOINT);
1346}
1347
1348extern void perf_event_init(void);
1349extern void perf_tp_event(u16 event_type, u64 count, void *record,
1350 int entry_size, struct pt_regs *regs,
1351 struct hlist_head *head, int rctx,
1352 struct task_struct *task);
1353extern void perf_bp_event(struct perf_event *event, void *data);
1354
1355#ifndef perf_misc_flags
1356# define perf_misc_flags(regs) \
1357 (user_mode(regs) ? PERF_RECORD_MISC_USER : PERF_RECORD_MISC_KERNEL)
1358# define perf_instruction_pointer(regs) instruction_pointer(regs)
1359#endif
1360#ifndef perf_arch_bpf_user_pt_regs
1361# define perf_arch_bpf_user_pt_regs(regs) regs
1362#endif
1363
1364static inline bool has_branch_stack(struct perf_event *event)
1365{
1366 return event->attr.sample_type & PERF_SAMPLE_BRANCH_STACK;
1367}
1368
1369static inline bool needs_branch_stack(struct perf_event *event)
1370{
1371 return event->attr.branch_sample_type != 0;
1372}
1373
1374static inline bool has_aux(struct perf_event *event)
1375{
1376 return event->pmu->setup_aux;
1377}
1378
1379static inline bool is_write_backward(struct perf_event *event)
1380{
1381 return !!event->attr.write_backward;
1382}
1383
1384static inline bool has_addr_filter(struct perf_event *event)
1385{
1386 return event->pmu->nr_addr_filters;
1387}
1388
1389
1390
1391
1392static inline struct perf_addr_filters_head *
1393perf_event_addr_filters(struct perf_event *event)
1394{
1395 struct perf_addr_filters_head *ifh = &event->addr_filters;
1396
1397 if (event->parent)
1398 ifh = &event->parent->addr_filters;
1399
1400 return ifh;
1401}
1402
1403extern void perf_event_addr_filters_sync(struct perf_event *event);
1404extern void perf_report_aux_output_id(struct perf_event *event, u64 hw_id);
1405
1406extern int perf_output_begin(struct perf_output_handle *handle,
1407 struct perf_sample_data *data,
1408 struct perf_event *event, unsigned int size);
1409extern int perf_output_begin_forward(struct perf_output_handle *handle,
1410 struct perf_sample_data *data,
1411 struct perf_event *event,
1412 unsigned int size);
1413extern int perf_output_begin_backward(struct perf_output_handle *handle,
1414 struct perf_sample_data *data,
1415 struct perf_event *event,
1416 unsigned int size);
1417
1418extern void perf_output_end(struct perf_output_handle *handle);
1419extern unsigned int perf_output_copy(struct perf_output_handle *handle,
1420 const void *buf, unsigned int len);
1421extern unsigned int perf_output_skip(struct perf_output_handle *handle,
1422 unsigned int len);
1423extern long perf_output_copy_aux(struct perf_output_handle *aux_handle,
1424 struct perf_output_handle *handle,
1425 unsigned long from, unsigned long to);
1426extern int perf_swevent_get_recursion_context(void);
1427extern void perf_swevent_put_recursion_context(int rctx);
1428extern u64 perf_swevent_set_period(struct perf_event *event);
1429extern void perf_event_enable(struct perf_event *event);
1430extern void perf_event_disable(struct perf_event *event);
1431extern void perf_event_disable_local(struct perf_event *event);
1432extern void perf_event_disable_inatomic(struct perf_event *event);
1433extern void perf_event_task_tick(void);
1434extern int perf_event_account_interrupt(struct perf_event *event);
1435extern int perf_event_period(struct perf_event *event, u64 value);
1436extern u64 perf_event_pause(struct perf_event *event, bool reset);
1437#else
1438static inline void *
1439perf_aux_output_begin(struct perf_output_handle *handle,
1440 struct perf_event *event) { return NULL; }
1441static inline void
1442perf_aux_output_end(struct perf_output_handle *handle, unsigned long size)
1443 { }
1444static inline int
1445perf_aux_output_skip(struct perf_output_handle *handle,
1446 unsigned long size) { return -EINVAL; }
1447static inline void *
1448perf_get_aux(struct perf_output_handle *handle) { return NULL; }
1449static inline void
1450perf_event_task_migrate(struct task_struct *task) { }
1451static inline void
1452perf_event_task_sched_in(struct task_struct *prev,
1453 struct task_struct *task) { }
1454static inline void
1455perf_event_task_sched_out(struct task_struct *prev,
1456 struct task_struct *next) { }
1457static inline int perf_event_init_task(struct task_struct *child,
1458 u64 clone_flags) { return 0; }
1459static inline void perf_event_exit_task(struct task_struct *child) { }
1460static inline void perf_event_free_task(struct task_struct *task) { }
1461static inline void perf_event_delayed_put(struct task_struct *task) { }
1462static inline struct file *perf_event_get(unsigned int fd) { return ERR_PTR(-EINVAL); }
1463static inline const struct perf_event *perf_get_event(struct file *file)
1464{
1465 return ERR_PTR(-EINVAL);
1466}
1467static inline const struct perf_event_attr *perf_event_attrs(struct perf_event *event)
1468{
1469 return ERR_PTR(-EINVAL);
1470}
1471static inline int perf_event_read_local(struct perf_event *event, u64 *value,
1472 u64 *enabled, u64 *running)
1473{
1474 return -EINVAL;
1475}
1476static inline void perf_event_print_debug(void) { }
1477static inline int perf_event_task_disable(void) { return -EINVAL; }
1478static inline int perf_event_task_enable(void) { return -EINVAL; }
1479static inline int perf_event_refresh(struct perf_event *event, int refresh)
1480{
1481 return -EINVAL;
1482}
1483
1484static inline void
1485perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr) { }
1486static inline void
1487perf_bp_event(struct perf_event *event, void *data) { }
1488
1489static inline int perf_register_guest_info_callbacks
1490(struct perf_guest_info_callbacks *callbacks) { return 0; }
1491static inline int perf_unregister_guest_info_callbacks
1492(struct perf_guest_info_callbacks *callbacks) { return 0; }
1493
1494static inline void perf_event_mmap(struct vm_area_struct *vma) { }
1495
1496typedef int (perf_ksymbol_get_name_f)(char *name, int name_len, void *data);
1497static inline void perf_event_ksymbol(u16 ksym_type, u64 addr, u32 len,
1498 bool unregister, const char *sym) { }
1499static inline void perf_event_bpf_event(struct bpf_prog *prog,
1500 enum perf_bpf_event_type type,
1501 u16 flags) { }
1502static inline void perf_event_exec(void) { }
1503static inline void perf_event_comm(struct task_struct *tsk, bool exec) { }
1504static inline void perf_event_namespaces(struct task_struct *tsk) { }
1505static inline void perf_event_fork(struct task_struct *tsk) { }
1506static inline void perf_event_text_poke(const void *addr,
1507 const void *old_bytes,
1508 size_t old_len,
1509 const void *new_bytes,
1510 size_t new_len) { }
1511static inline void perf_event_init(void) { }
1512static inline int perf_swevent_get_recursion_context(void) { return -1; }
1513static inline void perf_swevent_put_recursion_context(int rctx) { }
1514static inline u64 perf_swevent_set_period(struct perf_event *event) { return 0; }
1515static inline void perf_event_enable(struct perf_event *event) { }
1516static inline void perf_event_disable(struct perf_event *event) { }
1517static inline int __perf_event_disable(void *info) { return -1; }
1518static inline void perf_event_task_tick(void) { }
1519static inline int perf_event_release_kernel(struct perf_event *event) { return 0; }
1520static inline int perf_event_period(struct perf_event *event, u64 value)
1521{
1522 return -EINVAL;
1523}
1524static inline u64 perf_event_pause(struct perf_event *event, bool reset)
1525{
1526 return 0;
1527}
1528#endif
1529
1530#if defined(CONFIG_PERF_EVENTS) && defined(CONFIG_CPU_SUP_INTEL)
1531extern void perf_restore_debug_store(void);
1532#else
1533static inline void perf_restore_debug_store(void) { }
1534#endif
1535
1536static __always_inline bool perf_raw_frag_last(const struct perf_raw_frag *frag)
1537{
1538 return frag->pad < sizeof(u64);
1539}
1540
1541#define perf_output_put(handle, x) perf_output_copy((handle), &(x), sizeof(x))
1542
1543struct perf_pmu_events_attr {
1544 struct device_attribute attr;
1545 u64 id;
1546 const char *event_str;
1547};
1548
1549struct perf_pmu_events_ht_attr {
1550 struct device_attribute attr;
1551 u64 id;
1552 const char *event_str_ht;
1553 const char *event_str_noht;
1554};
1555
1556struct perf_pmu_events_hybrid_attr {
1557 struct device_attribute attr;
1558 u64 id;
1559 const char *event_str;
1560 u64 pmu_type;
1561};
1562
1563struct perf_pmu_format_hybrid_attr {
1564 struct device_attribute attr;
1565 u64 pmu_type;
1566};
1567
1568ssize_t perf_event_sysfs_show(struct device *dev, struct device_attribute *attr,
1569 char *page);
1570
1571#define PMU_EVENT_ATTR(_name, _var, _id, _show) \
1572static struct perf_pmu_events_attr _var = { \
1573 .attr = __ATTR(_name, 0444, _show, NULL), \
1574 .id = _id, \
1575};
1576
1577#define PMU_EVENT_ATTR_STRING(_name, _var, _str) \
1578static struct perf_pmu_events_attr _var = { \
1579 .attr = __ATTR(_name, 0444, perf_event_sysfs_show, NULL), \
1580 .id = 0, \
1581 .event_str = _str, \
1582};
1583
1584#define PMU_EVENT_ATTR_ID(_name, _show, _id) \
1585 (&((struct perf_pmu_events_attr[]) { \
1586 { .attr = __ATTR(_name, 0444, _show, NULL), \
1587 .id = _id, } \
1588 })[0].attr.attr)
1589
1590#define PMU_FORMAT_ATTR(_name, _format) \
1591static ssize_t \
1592_name##_show(struct device *dev, \
1593 struct device_attribute *attr, \
1594 char *page) \
1595{ \
1596 BUILD_BUG_ON(sizeof(_format) >= PAGE_SIZE); \
1597 return sprintf(page, _format "\n"); \
1598} \
1599 \
1600static struct device_attribute format_attr_##_name = __ATTR_RO(_name)
1601
1602
1603#ifdef CONFIG_PERF_EVENTS
1604int perf_event_init_cpu(unsigned int cpu);
1605int perf_event_exit_cpu(unsigned int cpu);
1606#else
1607#define perf_event_init_cpu NULL
1608#define perf_event_exit_cpu NULL
1609#endif
1610
1611extern void __weak arch_perf_update_userpage(struct perf_event *event,
1612 struct perf_event_mmap_page *userpg,
1613 u64 now);
1614
1615#ifdef CONFIG_MMU
1616extern __weak u64 arch_perf_get_page_size(struct mm_struct *mm, unsigned long addr);
1617#endif
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637typedef int (perf_snapshot_branch_stack_t)(struct perf_branch_entry *entries,
1638 unsigned int cnt);
1639DECLARE_STATIC_CALL(perf_snapshot_branch_stack, perf_snapshot_branch_stack_t);
1640
1641#endif
1642