1
2
3
4
5
6
7
8
9
10
11
12
13
14#ifndef _LINUX_PERF_EVENT_H
15#define _LINUX_PERF_EVENT_H
16
17#include <uapi/linux/perf_event.h>
18#include <uapi/linux/bpf_perf_event.h>
19
20
21
22
23
24#ifdef CONFIG_PERF_EVENTS
25# include <asm/perf_event.h>
26# include <asm/local64.h>
27#endif
28
29struct perf_guest_info_callbacks {
30 int (*is_in_guest)(void);
31 int (*is_user_mode)(void);
32 unsigned long (*get_guest_ip)(void);
33 void (*handle_intel_pt_intr)(void);
34};
35
36#ifdef CONFIG_HAVE_HW_BREAKPOINT
37#include <asm/hw_breakpoint.h>
38#endif
39
40#include <linux/list.h>
41#include <linux/mutex.h>
42#include <linux/rculist.h>
43#include <linux/rcupdate.h>
44#include <linux/spinlock.h>
45#include <linux/hrtimer.h>
46#include <linux/fs.h>
47#include <linux/pid_namespace.h>
48#include <linux/workqueue.h>
49#include <linux/ftrace.h>
50#include <linux/cpu.h>
51#include <linux/irq_work.h>
52#include <linux/static_key.h>
53#include <linux/jump_label_ratelimit.h>
54#include <linux/atomic.h>
55#include <linux/sysfs.h>
56#include <linux/perf_regs.h>
57#include <linux/cgroup.h>
58#include <linux/refcount.h>
59#include <asm/local.h>
60
61struct perf_callchain_entry {
62 __u64 nr;
63 __u64 ip[0];
64};
65
66struct perf_callchain_entry_ctx {
67 struct perf_callchain_entry *entry;
68 u32 max_stack;
69 u32 nr;
70 short contexts;
71 bool contexts_maxed;
72};
73
74typedef unsigned long (*perf_copy_f)(void *dst, const void *src,
75 unsigned long off, unsigned long len);
76
77struct perf_raw_frag {
78 union {
79 struct perf_raw_frag *next;
80 unsigned long pad;
81 };
82 perf_copy_f copy;
83 void *data;
84 u32 size;
85} __packed;
86
87struct perf_raw_record {
88 struct perf_raw_frag frag;
89 u32 size;
90};
91
92
93
94
95
96
97
98
99
100
101struct perf_branch_stack {
102 __u64 nr;
103 struct perf_branch_entry entries[0];
104};
105
106struct task_struct;
107
108
109
110
111struct hw_perf_event_extra {
112 u64 config;
113 unsigned int reg;
114 int alloc;
115 int idx;
116};
117
118
119
120
121struct hw_perf_event {
122#ifdef CONFIG_PERF_EVENTS
123 union {
124 struct {
125 u64 config;
126 u64 last_tag;
127 unsigned long config_base;
128 unsigned long event_base;
129 int event_base_rdpmc;
130 int idx;
131 int last_cpu;
132 int flags;
133
134 struct hw_perf_event_extra extra_reg;
135 struct hw_perf_event_extra branch_reg;
136 };
137 struct {
138 struct hrtimer hrtimer;
139 };
140 struct {
141
142 struct list_head tp_list;
143 };
144 struct {
145 u64 pwr_acc;
146 u64 ptsc;
147 };
148#ifdef CONFIG_HAVE_HW_BREAKPOINT
149 struct {
150
151
152
153
154
155 struct arch_hw_breakpoint info;
156 struct list_head bp_list;
157 };
158#endif
159 struct {
160 u8 iommu_bank;
161 u8 iommu_cntr;
162 u16 padding;
163 u64 conf;
164 u64 conf1;
165 };
166 };
167
168
169
170
171 struct task_struct *target;
172
173
174
175
176
177 void *addr_filters;
178
179
180 unsigned long addr_filters_gen;
181
182
183
184
185#define PERF_HES_STOPPED 0x01
186#define PERF_HES_UPTODATE 0x02
187#define PERF_HES_ARCH 0x04
188
189 int state;
190
191
192
193
194
195 local64_t prev_count;
196
197
198
199
200 u64 sample_period;
201
202
203
204
205 u64 last_period;
206
207
208
209
210
211
212 local64_t period_left;
213
214
215
216
217
218 u64 interrupts_seq;
219 u64 interrupts;
220
221
222
223
224
225 u64 freq_time_stamp;
226 u64 freq_count_stamp;
227#endif
228};
229
230struct perf_event;
231
232
233
234
235#define PERF_PMU_TXN_ADD 0x1
236#define PERF_PMU_TXN_READ 0x2
237
238
239
240
241#define PERF_PMU_CAP_NO_INTERRUPT 0x01
242#define PERF_PMU_CAP_NO_NMI 0x02
243#define PERF_PMU_CAP_AUX_NO_SG 0x04
244#define PERF_PMU_CAP_EXTENDED_REGS 0x08
245#define PERF_PMU_CAP_EXCLUSIVE 0x10
246#define PERF_PMU_CAP_ITRACE 0x20
247#define PERF_PMU_CAP_HETEROGENEOUS_CPUS 0x40
248#define PERF_PMU_CAP_NO_EXCLUDE 0x80
249#define PERF_PMU_CAP_AUX_OUTPUT 0x100
250
251
252
253
254struct pmu {
255 struct list_head entry;
256
257 struct module *module;
258 struct device *dev;
259 const struct attribute_group **attr_groups;
260 RH_KABI_BROKEN_INSERT(const struct attribute_group **attr_update)
261 const char *name;
262 int type;
263
264
265
266
267 int capabilities;
268
269 int __percpu *pmu_disable_count;
270 struct perf_cpu_context __percpu *pmu_cpu_context;
271 atomic_t exclusive_cnt;
272 int task_ctx_nr;
273 int hrtimer_interval_ms;
274
275
276 unsigned int nr_addr_filters;
277
278
279
280
281
282 void (*pmu_enable) (struct pmu *pmu);
283 void (*pmu_disable) (struct pmu *pmu);
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301 int (*event_init) (struct perf_event *event);
302
303
304
305
306
307 void (*event_mapped) (struct perf_event *event, struct mm_struct *mm);
308 void (*event_unmapped) (struct perf_event *event, struct mm_struct *mm);
309
310
311
312
313
314#define PERF_EF_START 0x01
315#define PERF_EF_RELOAD 0x02
316#define PERF_EF_UPDATE 0x04
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336 int (*add) (struct perf_event *event, int flags);
337 void (*del) (struct perf_event *event, int flags);
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357 void (*start) (struct perf_event *event, int flags);
358 void (*stop) (struct perf_event *event, int flags);
359
360
361
362
363
364
365
366 void (*read) (struct perf_event *event);
367
368
369
370
371
372
373
374
375
376
377
378 void (*start_txn) (struct pmu *pmu, unsigned int txn_flags);
379
380
381
382
383
384
385
386
387 int (*commit_txn) (struct pmu *pmu);
388
389
390
391
392
393
394 void (*cancel_txn) (struct pmu *pmu);
395
396
397
398
399
400 int (*event_idx) (struct perf_event *event);
401
402
403
404
405 void (*sched_task) (struct perf_event_context *ctx,
406 bool sched_in);
407
408
409
410 size_t task_ctx_size;
411
412
413
414
415
416 RH_KABI_REPLACE(void *(*setup_aux) (int cpu, void **pages, int nr_pages, bool overwrite), void *(*setup_aux) (struct perf_event *event, void **pages, int nr_pages, bool overwrite))
417
418
419
420
421
422 void (*free_aux) (void *aux);
423
424
425
426
427
428
429
430
431
432 int (*addr_filters_validate) (struct list_head *filters);
433
434
435
436
437
438
439
440
441
442
443
444
445
446 void (*addr_filters_sync) (struct perf_event *event);
447
448
449
450
451
452
453
454
455
456 RH_KABI_BROKEN_INSERT(int (*aux_output_match) (struct perf_event *event))
457
458
459
460
461
462 int (*filter_match) (struct perf_event *event);
463
464
465
466
467 RH_KABI_BROKEN_INSERT(int (*check_period) (struct perf_event *event, u64 value))
468};
469
470enum perf_addr_filter_action_t {
471 PERF_ADDR_FILTER_ACTION_STOP = 0,
472 PERF_ADDR_FILTER_ACTION_START,
473 PERF_ADDR_FILTER_ACTION_FILTER,
474};
475
476
477
478
479
480
481
482
483
484
485
486struct perf_addr_filter {
487 struct list_head entry;
488 struct path path;
489 unsigned long offset;
490 unsigned long size;
491 enum perf_addr_filter_action_t action;
492};
493
494
495
496
497
498
499
500
501
502
503
504struct perf_addr_filters_head {
505 struct list_head list;
506 raw_spinlock_t lock;
507 unsigned int nr_file_filters;
508};
509
510#ifndef __GENKSYMS__
511struct perf_addr_filter_range {
512 unsigned long start;
513 unsigned long size;
514};
515#endif
516
517
518
519
520enum perf_event_state {
521 PERF_EVENT_STATE_DEAD = -4,
522 PERF_EVENT_STATE_EXIT = -3,
523 PERF_EVENT_STATE_ERROR = -2,
524 PERF_EVENT_STATE_OFF = -1,
525 PERF_EVENT_STATE_INACTIVE = 0,
526 PERF_EVENT_STATE_ACTIVE = 1,
527};
528
529struct file;
530struct perf_sample_data;
531
532typedef void (*perf_overflow_handler_t)(struct perf_event *,
533 struct perf_sample_data *,
534 struct pt_regs *regs);
535
536
537
538
539
540
541
542
543#define PERF_EV_CAP_SOFTWARE BIT(0)
544#define PERF_EV_CAP_READ_ACTIVE_PKG BIT(1)
545
546#define SWEVENT_HLIST_BITS 8
547#define SWEVENT_HLIST_SIZE (1 << SWEVENT_HLIST_BITS)
548
549struct swevent_hlist {
550 struct hlist_head heads[SWEVENT_HLIST_SIZE];
551 struct rcu_head rcu_head;
552};
553
554#define PERF_ATTACH_CONTEXT 0x01
555#define PERF_ATTACH_GROUP 0x02
556#define PERF_ATTACH_TASK 0x04
557#define PERF_ATTACH_TASK_DATA 0x08
558#define PERF_ATTACH_ITRACE 0x10
559
560struct perf_cgroup;
561struct ring_buffer;
562
563struct pmu_event_list {
564 raw_spinlock_t lock;
565 struct list_head list;
566};
567
568#define for_each_sibling_event(sibling, event) \
569 if ((event)->group_leader == (event)) \
570 list_for_each_entry((sibling), &(event)->sibling_list, sibling_list)
571
572
573
574
575struct perf_event {
576#ifdef CONFIG_PERF_EVENTS
577
578
579
580
581
582 struct list_head event_entry;
583
584
585
586
587
588 struct list_head sibling_list;
589 struct list_head active_list;
590
591
592
593 struct rb_node group_node;
594 u64 group_index;
595
596
597
598
599
600 struct list_head migrate_entry;
601
602 struct hlist_node hlist_entry;
603 struct list_head active_entry;
604 int nr_siblings;
605
606
607 int event_caps;
608
609 int group_caps;
610
611 struct perf_event *group_leader;
612 struct pmu *pmu;
613 void *pmu_private;
614
615 enum perf_event_state state;
616 unsigned int attach_state;
617 local64_t count;
618 atomic64_t child_count;
619
620
621
622
623
624
625
626 u64 total_time_enabled;
627 u64 total_time_running;
628 u64 tstamp;
629
630
631
632
633
634
635
636
637
638 u64 shadow_ctx_time;
639
640 struct perf_event_attr attr;
641 u16 header_size;
642 u16 id_header_size;
643 u16 read_size;
644 struct hw_perf_event hw;
645
646 struct perf_event_context *ctx;
647 atomic_long_t refcount;
648
649
650
651
652
653 atomic64_t child_total_time_enabled;
654 atomic64_t child_total_time_running;
655
656
657
658
659 struct mutex child_mutex;
660 struct list_head child_list;
661 struct perf_event *parent;
662
663 int oncpu;
664 int cpu;
665
666 struct list_head owner_entry;
667 struct task_struct *owner;
668
669
670 struct mutex mmap_mutex;
671 atomic_t mmap_count;
672
673 struct ring_buffer *rb;
674 struct list_head rb_entry;
675 unsigned long rcu_batches;
676 int rcu_pending;
677
678
679 wait_queue_head_t waitq;
680 struct fasync_struct *fasync;
681
682
683 int pending_wakeup;
684 int pending_kill;
685 int pending_disable;
686 struct irq_work pending;
687
688 atomic_t event_limit;
689
690
691 struct perf_addr_filters_head addr_filters;
692
693 RH_KABI_REPLACE(unsigned long *addr_filters_offs, struct perf_addr_filter_range *addr_filter_ranges)
694 unsigned long addr_filters_gen;
695
696
697 RH_KABI_BROKEN_INSERT(struct perf_event *aux_event)
698
699 void (*destroy)(struct perf_event *);
700 struct rcu_head rcu_head;
701
702 struct pid_namespace *ns;
703 u64 id;
704
705 u64 (*clock)(void);
706 perf_overflow_handler_t overflow_handler;
707 void *overflow_handler_context;
708#ifdef CONFIG_BPF_SYSCALL
709 perf_overflow_handler_t orig_overflow_handler;
710 struct bpf_prog *prog;
711#endif
712
713#ifdef CONFIG_EVENT_TRACING
714 struct trace_event_call *tp_event;
715 struct event_filter *filter;
716#ifdef CONFIG_FUNCTION_TRACER
717 struct ftrace_ops ftrace_ops;
718#endif
719#endif
720
721#ifdef CONFIG_CGROUP_PERF
722 struct perf_cgroup *cgrp;
723#endif
724
725 struct list_head sb_list;
726#endif
727};
728
729
730struct perf_event_groups {
731 struct rb_root tree;
732 u64 index;
733};
734
735
736
737
738
739
740struct perf_event_context {
741 struct pmu *pmu;
742
743
744
745
746 raw_spinlock_t lock;
747
748
749
750
751
752 struct mutex mutex;
753
754 struct list_head active_ctx_list;
755 struct perf_event_groups pinned_groups;
756 struct perf_event_groups flexible_groups;
757 struct list_head event_list;
758
759 struct list_head pinned_active;
760 struct list_head flexible_active;
761
762 int nr_events;
763 int nr_active;
764 int is_active;
765 int nr_stat;
766 int nr_freq;
767 int rotate_disable;
768
769
770
771
772 RH_KABI_BROKEN_INSERT(int rotate_necessary)
773 RH_KABI_REPLACE(atomic_t refcount, refcount_t refcount)
774 struct task_struct *task;
775
776
777
778
779 u64 time;
780 u64 timestamp;
781
782
783
784
785
786 struct perf_event_context *parent_ctx;
787 u64 parent_gen;
788 u64 generation;
789 int pin_count;
790#ifdef CONFIG_CGROUP_PERF
791 int nr_cgroups;
792#endif
793 void *task_ctx_data;
794 struct rcu_head rcu_head;
795};
796
797
798
799
800
801#define PERF_NR_CONTEXTS 4
802
803
804
805
806struct perf_cpu_context {
807 struct perf_event_context ctx;
808 struct perf_event_context *task_ctx;
809 int active_oncpu;
810 int exclusive;
811
812 raw_spinlock_t hrtimer_lock;
813 struct hrtimer hrtimer;
814 ktime_t hrtimer_interval;
815 unsigned int hrtimer_active;
816
817#ifdef CONFIG_CGROUP_PERF
818 struct perf_cgroup *cgrp;
819 struct list_head cgrp_cpuctx_entry;
820#endif
821
822 struct list_head sched_cb_entry;
823 int sched_cb_usage;
824
825 int online;
826};
827
828struct perf_output_handle {
829 struct perf_event *event;
830 struct ring_buffer *rb;
831 unsigned long wakeup;
832 unsigned long size;
833 u64 aux_flags;
834 union {
835 void *addr;
836 unsigned long head;
837 };
838 int page;
839};
840
841struct bpf_perf_event_data_kern {
842 bpf_user_pt_regs_t *regs;
843 struct perf_sample_data *data;
844 struct perf_event *event;
845};
846
847#ifdef CONFIG_CGROUP_PERF
848
849
850
851
852
853struct perf_cgroup_info {
854 u64 time;
855 u64 timestamp;
856};
857
858struct perf_cgroup {
859 struct cgroup_subsys_state css;
860 struct perf_cgroup_info __percpu *info;
861};
862
863
864
865
866
867
868static inline struct perf_cgroup *
869perf_cgroup_from_task(struct task_struct *task, struct perf_event_context *ctx)
870{
871 return container_of(task_css_check(task, perf_event_cgrp_id,
872 ctx ? lockdep_is_held(&ctx->lock)
873 : true),
874 struct perf_cgroup, css);
875}
876#endif
877
878#ifdef CONFIG_PERF_EVENTS
879
880extern void *perf_aux_output_begin(struct perf_output_handle *handle,
881 struct perf_event *event);
882extern void perf_aux_output_end(struct perf_output_handle *handle,
883 unsigned long size);
884extern int perf_aux_output_skip(struct perf_output_handle *handle,
885 unsigned long size);
886extern void *perf_get_aux(struct perf_output_handle *handle);
887extern void perf_aux_output_flag(struct perf_output_handle *handle, u64 flags);
888extern void perf_event_itrace_started(struct perf_event *event);
889
890extern int perf_pmu_register(struct pmu *pmu, const char *name, int type);
891extern void perf_pmu_unregister(struct pmu *pmu);
892
893extern int perf_num_counters(void);
894extern const char *perf_pmu_name(void);
895extern void __perf_event_task_sched_in(struct task_struct *prev,
896 struct task_struct *task);
897extern void __perf_event_task_sched_out(struct task_struct *prev,
898 struct task_struct *next);
899extern int perf_event_init_task(struct task_struct *child);
900extern void perf_event_exit_task(struct task_struct *child);
901extern void perf_event_free_task(struct task_struct *task);
902extern void perf_event_delayed_put(struct task_struct *task);
903extern struct file *perf_event_get(unsigned int fd);
904extern const struct perf_event *perf_get_event(struct file *file);
905extern const struct perf_event_attr *perf_event_attrs(struct perf_event *event);
906extern void perf_event_print_debug(void);
907extern void perf_pmu_disable(struct pmu *pmu);
908extern void perf_pmu_enable(struct pmu *pmu);
909extern void perf_sched_cb_dec(struct pmu *pmu);
910extern void perf_sched_cb_inc(struct pmu *pmu);
911extern int perf_event_task_disable(void);
912extern int perf_event_task_enable(void);
913
914extern void perf_pmu_resched(struct pmu *pmu);
915
916extern int perf_event_refresh(struct perf_event *event, int refresh);
917extern void perf_event_update_userpage(struct perf_event *event);
918extern int perf_event_release_kernel(struct perf_event *event);
919extern struct perf_event *
920perf_event_create_kernel_counter(struct perf_event_attr *attr,
921 int cpu,
922 struct task_struct *task,
923 perf_overflow_handler_t callback,
924 void *context);
925extern void perf_pmu_migrate_context(struct pmu *pmu,
926 int src_cpu, int dst_cpu);
927int perf_event_read_local(struct perf_event *event, u64 *value,
928 u64 *enabled, u64 *running);
929extern u64 perf_event_read_value(struct perf_event *event,
930 u64 *enabled, u64 *running);
931
932
933struct perf_sample_data {
934
935
936
937
938 u64 addr;
939 struct perf_raw_record *raw;
940 struct perf_branch_stack *br_stack;
941 u64 period;
942 u64 weight;
943 u64 txn;
944 union perf_mem_data_src data_src;
945
946
947
948
949
950 u64 type;
951 u64 ip;
952 struct {
953 u32 pid;
954 u32 tid;
955 } tid_entry;
956 u64 time;
957 u64 id;
958 u64 stream_id;
959 struct {
960 u32 cpu;
961 u32 reserved;
962 } cpu_entry;
963 struct perf_callchain_entry *callchain;
964
965
966
967
968
969 struct perf_regs regs_user;
970 struct pt_regs regs_user_copy;
971
972 struct perf_regs regs_intr;
973 u64 stack_user_size;
974
975 u64 phys_addr;
976} ____cacheline_aligned;
977
978
979#define PERF_MEM_NA (PERF_MEM_S(OP, NA) |\
980 PERF_MEM_S(LVL, NA) |\
981 PERF_MEM_S(SNOOP, NA) |\
982 PERF_MEM_S(LOCK, NA) |\
983 PERF_MEM_S(TLB, NA))
984
985static inline void perf_sample_data_init(struct perf_sample_data *data,
986 u64 addr, u64 period)
987{
988
989 data->addr = addr;
990 data->raw = NULL;
991 data->br_stack = NULL;
992 data->period = period;
993 data->weight = 0;
994 data->data_src.val = PERF_MEM_NA;
995 data->txn = 0;
996}
997
998extern void perf_output_sample(struct perf_output_handle *handle,
999 struct perf_event_header *header,
1000 struct perf_sample_data *data,
1001 struct perf_event *event);
1002extern void perf_prepare_sample(struct perf_event_header *header,
1003 struct perf_sample_data *data,
1004 struct perf_event *event,
1005 struct pt_regs *regs);
1006
1007extern int perf_event_overflow(struct perf_event *event,
1008 struct perf_sample_data *data,
1009 struct pt_regs *regs);
1010
1011extern void perf_event_output_forward(struct perf_event *event,
1012 struct perf_sample_data *data,
1013 struct pt_regs *regs);
1014extern void perf_event_output_backward(struct perf_event *event,
1015 struct perf_sample_data *data,
1016 struct pt_regs *regs);
1017extern int perf_event_output(struct perf_event *event,
1018 struct perf_sample_data *data,
1019 struct pt_regs *regs);
1020
1021static inline bool
1022is_default_overflow_handler(struct perf_event *event)
1023{
1024 if (likely(event->overflow_handler == perf_event_output_forward))
1025 return true;
1026 if (unlikely(event->overflow_handler == perf_event_output_backward))
1027 return true;
1028 return false;
1029}
1030
1031extern void
1032perf_event_header__init_id(struct perf_event_header *header,
1033 struct perf_sample_data *data,
1034 struct perf_event *event);
1035extern void
1036perf_event__output_id_sample(struct perf_event *event,
1037 struct perf_output_handle *handle,
1038 struct perf_sample_data *sample);
1039
1040extern void
1041perf_log_lost_samples(struct perf_event *event, u64 lost);
1042
1043static inline bool event_has_any_exclude_flag(struct perf_event *event)
1044{
1045 struct perf_event_attr *attr = &event->attr;
1046
1047 return attr->exclude_idle || attr->exclude_user ||
1048 attr->exclude_kernel || attr->exclude_hv ||
1049 attr->exclude_guest || attr->exclude_host;
1050}
1051
1052static inline bool is_sampling_event(struct perf_event *event)
1053{
1054 return event->attr.sample_period != 0;
1055}
1056
1057
1058
1059
1060static inline int is_software_event(struct perf_event *event)
1061{
1062 return event->event_caps & PERF_EV_CAP_SOFTWARE;
1063}
1064
1065
1066
1067
1068static inline int in_software_context(struct perf_event *event)
1069{
1070 return event->ctx->pmu->task_ctx_nr == perf_sw_context;
1071}
1072
1073static inline int is_exclusive_pmu(struct pmu *pmu)
1074{
1075 return pmu->capabilities & PERF_PMU_CAP_EXCLUSIVE;
1076}
1077
1078extern struct static_key perf_swevent_enabled[PERF_COUNT_SW_MAX];
1079
1080extern void ___perf_sw_event(u32, u64, struct pt_regs *, u64);
1081extern void __perf_sw_event(u32, u64, struct pt_regs *, u64);
1082
1083#ifndef perf_arch_fetch_caller_regs
1084static inline void perf_arch_fetch_caller_regs(struct pt_regs *regs, unsigned long ip) { }
1085#endif
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101static inline void perf_fetch_caller_regs(struct pt_regs *regs)
1102{
1103 perf_arch_fetch_caller_regs(regs, CALLER_ADDR0);
1104}
1105
1106static __always_inline void
1107perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr)
1108{
1109 if (static_key_false(&perf_swevent_enabled[event_id]))
1110 __perf_sw_event(event_id, nr, regs, addr);
1111}
1112
1113DECLARE_PER_CPU(struct pt_regs, __perf_regs[4]);
1114
1115
1116
1117
1118
1119
1120static __always_inline void
1121perf_sw_event_sched(u32 event_id, u64 nr, u64 addr)
1122{
1123 if (static_key_false(&perf_swevent_enabled[event_id])) {
1124 struct pt_regs *regs = this_cpu_ptr(&__perf_regs[0]);
1125
1126 perf_fetch_caller_regs(regs);
1127 ___perf_sw_event(event_id, nr, regs, addr);
1128 }
1129}
1130
1131extern struct static_key_false perf_sched_events;
1132
1133static __always_inline bool
1134perf_sw_migrate_enabled(void)
1135{
1136 if (static_key_false(&perf_swevent_enabled[PERF_COUNT_SW_CPU_MIGRATIONS]))
1137 return true;
1138 return false;
1139}
1140
1141static inline void perf_event_task_migrate(struct task_struct *task)
1142{
1143 if (perf_sw_migrate_enabled())
1144 task->sched_migrated = 1;
1145}
1146
1147static inline void perf_event_task_sched_in(struct task_struct *prev,
1148 struct task_struct *task)
1149{
1150 if (static_branch_unlikely(&perf_sched_events))
1151 __perf_event_task_sched_in(prev, task);
1152
1153 if (perf_sw_migrate_enabled() && task->sched_migrated) {
1154 struct pt_regs *regs = this_cpu_ptr(&__perf_regs[0]);
1155
1156 perf_fetch_caller_regs(regs);
1157 ___perf_sw_event(PERF_COUNT_SW_CPU_MIGRATIONS, 1, regs, 0);
1158 task->sched_migrated = 0;
1159 }
1160}
1161
1162static inline void perf_event_task_sched_out(struct task_struct *prev,
1163 struct task_struct *next)
1164{
1165 perf_sw_event_sched(PERF_COUNT_SW_CONTEXT_SWITCHES, 1, 0);
1166
1167 if (static_branch_unlikely(&perf_sched_events))
1168 __perf_event_task_sched_out(prev, next);
1169}
1170
1171extern void perf_event_mmap(struct vm_area_struct *vma);
1172
1173extern void perf_event_ksymbol(u16 ksym_type, u64 addr, u32 len,
1174 bool unregister, const char *sym);
1175extern void perf_event_bpf_event(struct bpf_prog *prog,
1176 enum perf_bpf_event_type type,
1177 u16 flags);
1178
1179extern struct perf_guest_info_callbacks *perf_guest_cbs;
1180extern int perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *callbacks);
1181extern int perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *callbacks);
1182
1183extern void perf_event_exec(void);
1184extern void perf_event_comm(struct task_struct *tsk, bool exec);
1185extern void perf_event_namespaces(struct task_struct *tsk);
1186extern void perf_event_fork(struct task_struct *tsk);
1187
1188
1189DECLARE_PER_CPU(struct perf_callchain_entry, perf_callchain_entry);
1190
1191extern void perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs);
1192extern void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs);
1193extern struct perf_callchain_entry *
1194get_perf_callchain(struct pt_regs *regs, u32 init_nr, bool kernel, bool user,
1195 u32 max_stack, bool crosstask, bool add_mark);
1196extern struct perf_callchain_entry *perf_callchain(struct perf_event *event, struct pt_regs *regs);
1197extern int get_callchain_buffers(int max_stack);
1198extern void put_callchain_buffers(void);
1199
1200extern int sysctl_perf_event_max_stack;
1201extern int sysctl_perf_event_max_contexts_per_stack;
1202
1203static inline int perf_callchain_store_context(struct perf_callchain_entry_ctx *ctx, u64 ip)
1204{
1205 if (ctx->contexts < sysctl_perf_event_max_contexts_per_stack) {
1206 struct perf_callchain_entry *entry = ctx->entry;
1207 entry->ip[entry->nr++] = ip;
1208 ++ctx->contexts;
1209 return 0;
1210 } else {
1211 ctx->contexts_maxed = true;
1212 return -1;
1213 }
1214}
1215
1216static inline int perf_callchain_store(struct perf_callchain_entry_ctx *ctx, u64 ip)
1217{
1218 if (ctx->nr < ctx->max_stack && !ctx->contexts_maxed) {
1219 struct perf_callchain_entry *entry = ctx->entry;
1220 entry->ip[entry->nr++] = ip;
1221 ++ctx->nr;
1222 return 0;
1223 } else {
1224 return -1;
1225 }
1226}
1227
1228extern int sysctl_perf_event_paranoid;
1229extern int sysctl_perf_event_mlock;
1230extern int sysctl_perf_event_sample_rate;
1231extern int sysctl_perf_cpu_time_max_percent;
1232
1233extern void perf_sample_event_took(u64 sample_len_ns);
1234
1235extern int perf_proc_update_handler(struct ctl_table *table, int write,
1236 void __user *buffer, size_t *lenp,
1237 loff_t *ppos);
1238extern int perf_cpu_time_max_percent_handler(struct ctl_table *table, int write,
1239 void __user *buffer, size_t *lenp,
1240 loff_t *ppos);
1241
1242int perf_event_max_stack_handler(struct ctl_table *table, int write,
1243 void __user *buffer, size_t *lenp, loff_t *ppos);
1244
1245static inline bool perf_paranoid_tracepoint_raw(void)
1246{
1247 return sysctl_perf_event_paranoid > -1;
1248}
1249
1250static inline bool perf_paranoid_cpu(void)
1251{
1252 return sysctl_perf_event_paranoid > 0;
1253}
1254
1255static inline bool perf_paranoid_kernel(void)
1256{
1257 return sysctl_perf_event_paranoid > 1;
1258}
1259
1260extern void perf_event_init(void);
1261extern void perf_tp_event(u16 event_type, u64 count, void *record,
1262 int entry_size, struct pt_regs *regs,
1263 struct hlist_head *head, int rctx,
1264 struct task_struct *task);
1265extern void perf_bp_event(struct perf_event *event, void *data);
1266
1267#ifndef perf_misc_flags
1268# define perf_misc_flags(regs) \
1269 (user_mode(regs) ? PERF_RECORD_MISC_USER : PERF_RECORD_MISC_KERNEL)
1270# define perf_instruction_pointer(regs) instruction_pointer(regs)
1271#endif
1272#ifndef perf_arch_bpf_user_pt_regs
1273# define perf_arch_bpf_user_pt_regs(regs) regs
1274#endif
1275
1276static inline bool has_branch_stack(struct perf_event *event)
1277{
1278 return event->attr.sample_type & PERF_SAMPLE_BRANCH_STACK;
1279}
1280
1281static inline bool needs_branch_stack(struct perf_event *event)
1282{
1283 return event->attr.branch_sample_type != 0;
1284}
1285
1286static inline bool has_aux(struct perf_event *event)
1287{
1288 return event->pmu->setup_aux;
1289}
1290
1291static inline bool is_write_backward(struct perf_event *event)
1292{
1293 return !!event->attr.write_backward;
1294}
1295
1296static inline bool has_addr_filter(struct perf_event *event)
1297{
1298 return event->pmu->nr_addr_filters;
1299}
1300
1301
1302
1303
1304static inline struct perf_addr_filters_head *
1305perf_event_addr_filters(struct perf_event *event)
1306{
1307 struct perf_addr_filters_head *ifh = &event->addr_filters;
1308
1309 if (event->parent)
1310 ifh = &event->parent->addr_filters;
1311
1312 return ifh;
1313}
1314
1315extern void perf_event_addr_filters_sync(struct perf_event *event);
1316
1317extern int perf_output_begin(struct perf_output_handle *handle,
1318 struct perf_event *event, unsigned int size);
1319extern int perf_output_begin_forward(struct perf_output_handle *handle,
1320 struct perf_event *event,
1321 unsigned int size);
1322extern int perf_output_begin_backward(struct perf_output_handle *handle,
1323 struct perf_event *event,
1324 unsigned int size);
1325
1326extern void perf_output_end(struct perf_output_handle *handle);
1327extern unsigned int perf_output_copy(struct perf_output_handle *handle,
1328 const void *buf, unsigned int len);
1329extern unsigned int perf_output_skip(struct perf_output_handle *handle,
1330 unsigned int len);
1331extern int perf_swevent_get_recursion_context(void);
1332extern void perf_swevent_put_recursion_context(int rctx);
1333extern u64 perf_swevent_set_period(struct perf_event *event);
1334extern void perf_event_enable(struct perf_event *event);
1335extern void perf_event_disable(struct perf_event *event);
1336extern void perf_event_disable_local(struct perf_event *event);
1337extern void perf_event_disable_inatomic(struct perf_event *event);
1338extern void perf_event_task_tick(void);
1339extern int perf_event_account_interrupt(struct perf_event *event);
1340#else
1341static inline void *
1342perf_aux_output_begin(struct perf_output_handle *handle,
1343 struct perf_event *event) { return NULL; }
1344static inline void
1345perf_aux_output_end(struct perf_output_handle *handle, unsigned long size)
1346 { }
1347static inline int
1348perf_aux_output_skip(struct perf_output_handle *handle,
1349 unsigned long size) { return -EINVAL; }
1350static inline void *
1351perf_get_aux(struct perf_output_handle *handle) { return NULL; }
1352static inline void
1353perf_event_task_migrate(struct task_struct *task) { }
1354static inline void
1355perf_event_task_sched_in(struct task_struct *prev,
1356 struct task_struct *task) { }
1357static inline void
1358perf_event_task_sched_out(struct task_struct *prev,
1359 struct task_struct *next) { }
1360static inline int perf_event_init_task(struct task_struct *child) { return 0; }
1361static inline void perf_event_exit_task(struct task_struct *child) { }
1362static inline void perf_event_free_task(struct task_struct *task) { }
1363static inline void perf_event_delayed_put(struct task_struct *task) { }
1364static inline struct file *perf_event_get(unsigned int fd) { return ERR_PTR(-EINVAL); }
1365static inline const struct perf_event *perf_get_event(struct file *file)
1366{
1367 return ERR_PTR(-EINVAL);
1368}
1369static inline const struct perf_event_attr *perf_event_attrs(struct perf_event *event)
1370{
1371 return ERR_PTR(-EINVAL);
1372}
1373static inline int perf_event_read_local(struct perf_event *event, u64 *value,
1374 u64 *enabled, u64 *running)
1375{
1376 return -EINVAL;
1377}
1378static inline void perf_event_print_debug(void) { }
1379static inline int perf_event_task_disable(void) { return -EINVAL; }
1380static inline int perf_event_task_enable(void) { return -EINVAL; }
1381static inline int perf_event_refresh(struct perf_event *event, int refresh)
1382{
1383 return -EINVAL;
1384}
1385
1386static inline void
1387perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr) { }
1388static inline void
1389perf_sw_event_sched(u32 event_id, u64 nr, u64 addr) { }
1390static inline void
1391perf_bp_event(struct perf_event *event, void *data) { }
1392
1393static inline int perf_register_guest_info_callbacks
1394(struct perf_guest_info_callbacks *callbacks) { return 0; }
1395static inline int perf_unregister_guest_info_callbacks
1396(struct perf_guest_info_callbacks *callbacks) { return 0; }
1397
1398static inline void perf_event_mmap(struct vm_area_struct *vma) { }
1399
1400typedef int (perf_ksymbol_get_name_f)(char *name, int name_len, void *data);
1401static inline void perf_event_ksymbol(u16 ksym_type, u64 addr, u32 len,
1402 bool unregister, const char *sym) { }
1403static inline void perf_event_bpf_event(struct bpf_prog *prog,
1404 enum perf_bpf_event_type type,
1405 u16 flags) { }
1406static inline void perf_event_exec(void) { }
1407static inline void perf_event_comm(struct task_struct *tsk, bool exec) { }
1408static inline void perf_event_namespaces(struct task_struct *tsk) { }
1409static inline void perf_event_fork(struct task_struct *tsk) { }
1410static inline void perf_event_init(void) { }
1411static inline int perf_swevent_get_recursion_context(void) { return -1; }
1412static inline void perf_swevent_put_recursion_context(int rctx) { }
1413static inline u64 perf_swevent_set_period(struct perf_event *event) { return 0; }
1414static inline void perf_event_enable(struct perf_event *event) { }
1415static inline void perf_event_disable(struct perf_event *event) { }
1416static inline int __perf_event_disable(void *info) { return -1; }
1417static inline void perf_event_task_tick(void) { }
1418static inline int perf_event_release_kernel(struct perf_event *event) { return 0; }
1419#endif
1420
1421#if defined(CONFIG_PERF_EVENTS) && defined(CONFIG_CPU_SUP_INTEL)
1422extern void perf_restore_debug_store(void);
1423#else
1424static inline void perf_restore_debug_store(void) { }
1425#endif
1426
1427static __always_inline bool perf_raw_frag_last(const struct perf_raw_frag *frag)
1428{
1429 return frag->pad < sizeof(u64);
1430}
1431
1432#define perf_output_put(handle, x) perf_output_copy((handle), &(x), sizeof(x))
1433
1434struct perf_pmu_events_attr {
1435 struct device_attribute attr;
1436 u64 id;
1437 const char *event_str;
1438};
1439
1440struct perf_pmu_events_ht_attr {
1441 struct device_attribute attr;
1442 u64 id;
1443 const char *event_str_ht;
1444 const char *event_str_noht;
1445};
1446
1447ssize_t perf_event_sysfs_show(struct device *dev, struct device_attribute *attr,
1448 char *page);
1449
1450#define PMU_EVENT_ATTR(_name, _var, _id, _show) \
1451static struct perf_pmu_events_attr _var = { \
1452 .attr = __ATTR(_name, 0444, _show, NULL), \
1453 .id = _id, \
1454};
1455
1456#define PMU_EVENT_ATTR_STRING(_name, _var, _str) \
1457static struct perf_pmu_events_attr _var = { \
1458 .attr = __ATTR(_name, 0444, perf_event_sysfs_show, NULL), \
1459 .id = 0, \
1460 .event_str = _str, \
1461};
1462
1463#define PMU_FORMAT_ATTR(_name, _format) \
1464static ssize_t \
1465_name##_show(struct device *dev, \
1466 struct device_attribute *attr, \
1467 char *page) \
1468{ \
1469 BUILD_BUG_ON(sizeof(_format) >= PAGE_SIZE); \
1470 return sprintf(page, _format "\n"); \
1471} \
1472 \
1473static struct device_attribute format_attr_##_name = __ATTR_RO(_name)
1474
1475
1476#ifdef CONFIG_PERF_EVENTS
1477int perf_event_init_cpu(unsigned int cpu);
1478int perf_event_exit_cpu(unsigned int cpu);
1479#else
1480#define perf_event_init_cpu NULL
1481#define perf_event_exit_cpu NULL
1482#endif
1483
1484#endif
1485