1
2
3
4
5
6
7
8
9
10
11
12
13
14#ifndef _LINUX_PERF_EVENT_H
15#define _LINUX_PERF_EVENT_H
16
17#include <uapi/linux/perf_event.h>
18#include <uapi/linux/bpf_perf_event.h>
19
20
21
22
23
24#ifdef CONFIG_PERF_EVENTS
25# include <asm/perf_event.h>
26# include <asm/local64.h>
27#endif
28
29struct perf_guest_info_callbacks {
30 int (*is_in_guest)(void);
31 int (*is_user_mode)(void);
32 unsigned long (*get_guest_ip)(void);
33 void (*handle_intel_pt_intr)(void);
34};
35
36#ifdef CONFIG_HAVE_HW_BREAKPOINT
37#include <asm/hw_breakpoint.h>
38#endif
39
40#include <linux/list.h>
41#include <linux/mutex.h>
42#include <linux/rculist.h>
43#include <linux/rcupdate.h>
44#include <linux/spinlock.h>
45#include <linux/hrtimer.h>
46#include <linux/fs.h>
47#include <linux/pid_namespace.h>
48#include <linux/workqueue.h>
49#include <linux/ftrace.h>
50#include <linux/cpu.h>
51#include <linux/irq_work.h>
52#include <linux/static_key.h>
53#include <linux/jump_label_ratelimit.h>
54#include <linux/atomic.h>
55#include <linux/sysfs.h>
56#include <linux/perf_regs.h>
57#include <linux/cgroup.h>
58#include <linux/refcount.h>
59#include <asm/local.h>
60
61struct perf_callchain_entry {
62 __u64 nr;
63 __u64 ip[0];
64};
65
66struct perf_callchain_entry_ctx {
67 struct perf_callchain_entry *entry;
68 u32 max_stack;
69 u32 nr;
70 short contexts;
71 bool contexts_maxed;
72};
73
74typedef unsigned long (*perf_copy_f)(void *dst, const void *src,
75 unsigned long off, unsigned long len);
76
77struct perf_raw_frag {
78 union {
79 struct perf_raw_frag *next;
80 unsigned long pad;
81 };
82 perf_copy_f copy;
83 void *data;
84 u32 size;
85} __packed;
86
87struct perf_raw_record {
88 struct perf_raw_frag frag;
89 u32 size;
90};
91
92
93
94
95
96
97
98
99
100
101struct perf_branch_stack {
102 __u64 nr;
103 struct perf_branch_entry entries[0];
104};
105
106struct task_struct;
107
108
109
110
111struct hw_perf_event_extra {
112 u64 config;
113 unsigned int reg;
114 int alloc;
115 int idx;
116};
117
118
119
120
121struct hw_perf_event {
122#ifdef CONFIG_PERF_EVENTS
123 union {
124 struct {
125 u64 config;
126 u64 last_tag;
127 unsigned long config_base;
128 unsigned long event_base;
129 int event_base_rdpmc;
130 int idx;
131 int last_cpu;
132 int flags;
133
134 struct hw_perf_event_extra extra_reg;
135 struct hw_perf_event_extra branch_reg;
136 };
137 struct {
138 struct hrtimer hrtimer;
139 };
140 struct {
141
142 struct list_head tp_list;
143 };
144 struct {
145 u64 pwr_acc;
146 u64 ptsc;
147 };
148#ifdef CONFIG_HAVE_HW_BREAKPOINT
149 struct {
150
151
152
153
154
155 struct arch_hw_breakpoint info;
156 struct list_head bp_list;
157 };
158#endif
159 struct {
160 u8 iommu_bank;
161 u8 iommu_cntr;
162 u16 padding;
163 u64 conf;
164 u64 conf1;
165 };
166 };
167
168
169
170
171 struct task_struct *target;
172
173
174
175
176
177 void *addr_filters;
178
179
180 unsigned long addr_filters_gen;
181
182
183
184
185#define PERF_HES_STOPPED 0x01
186#define PERF_HES_UPTODATE 0x02
187#define PERF_HES_ARCH 0x04
188
189 int state;
190
191
192
193
194
195 local64_t prev_count;
196
197
198
199
200 u64 sample_period;
201
202
203
204
205 u64 last_period;
206
207
208
209
210
211
212 local64_t period_left;
213
214
215
216
217
218 u64 interrupts_seq;
219 u64 interrupts;
220
221
222
223
224
225 u64 freq_time_stamp;
226 u64 freq_count_stamp;
227#endif
228};
229
230struct perf_event;
231
232
233
234
235#define PERF_PMU_TXN_ADD 0x1
236#define PERF_PMU_TXN_READ 0x2
237
238
239
240
241#define PERF_PMU_CAP_NO_INTERRUPT 0x01
242#define PERF_PMU_CAP_NO_NMI 0x02
243#define PERF_PMU_CAP_AUX_NO_SG 0x04
244#define PERF_PMU_CAP_EXTENDED_REGS 0x08
245#define PERF_PMU_CAP_EXCLUSIVE 0x10
246#define PERF_PMU_CAP_ITRACE 0x20
247#define PERF_PMU_CAP_HETEROGENEOUS_CPUS 0x40
248#define PERF_PMU_CAP_NO_EXCLUDE 0x80
249
250
251
252
253struct pmu {
254 struct list_head entry;
255
256 struct module *module;
257 struct device *dev;
258 const struct attribute_group **attr_groups;
259 const char *name;
260 int type;
261
262
263
264
265 int capabilities;
266
267 int __percpu *pmu_disable_count;
268 struct perf_cpu_context __percpu *pmu_cpu_context;
269 atomic_t exclusive_cnt;
270 int task_ctx_nr;
271 int hrtimer_interval_ms;
272
273
274 unsigned int nr_addr_filters;
275
276
277
278
279
280 void (*pmu_enable) (struct pmu *pmu);
281 void (*pmu_disable) (struct pmu *pmu);
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299 int (*event_init) (struct perf_event *event);
300
301
302
303
304
305 void (*event_mapped) (struct perf_event *event, struct mm_struct *mm);
306 void (*event_unmapped) (struct perf_event *event, struct mm_struct *mm);
307
308
309
310
311
312#define PERF_EF_START 0x01
313#define PERF_EF_RELOAD 0x02
314#define PERF_EF_UPDATE 0x04
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334 int (*add) (struct perf_event *event, int flags);
335 void (*del) (struct perf_event *event, int flags);
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355 void (*start) (struct perf_event *event, int flags);
356 void (*stop) (struct perf_event *event, int flags);
357
358
359
360
361
362
363
364 void (*read) (struct perf_event *event);
365
366
367
368
369
370
371
372
373
374
375
376 void (*start_txn) (struct pmu *pmu, unsigned int txn_flags);
377
378
379
380
381
382
383
384
385 int (*commit_txn) (struct pmu *pmu);
386
387
388
389
390
391
392 void (*cancel_txn) (struct pmu *pmu);
393
394
395
396
397
398 int (*event_idx) (struct perf_event *event);
399
400
401
402
403 void (*sched_task) (struct perf_event_context *ctx,
404 bool sched_in);
405
406
407
408 size_t task_ctx_size;
409
410
411
412
413
414 void *(*setup_aux) (struct perf_event *event, void **pages,
415 int nr_pages, bool overwrite);
416
417
418
419
420
421 void (*free_aux) (void *aux);
422
423
424
425
426
427
428
429
430
431 int (*addr_filters_validate) (struct list_head *filters);
432
433
434
435
436
437
438
439
440
441
442
443
444
445 void (*addr_filters_sync) (struct perf_event *event);
446
447
448
449
450
451 int (*filter_match) (struct perf_event *event);
452
453
454
455
456 int (*check_period) (struct perf_event *event, u64 value);
457};
458
459enum perf_addr_filter_action_t {
460 PERF_ADDR_FILTER_ACTION_STOP = 0,
461 PERF_ADDR_FILTER_ACTION_START,
462 PERF_ADDR_FILTER_ACTION_FILTER,
463};
464
465
466
467
468
469
470
471
472
473
474
475struct perf_addr_filter {
476 struct list_head entry;
477 struct path path;
478 unsigned long offset;
479 unsigned long size;
480 enum perf_addr_filter_action_t action;
481};
482
483
484
485
486
487
488
489
490
491
492
493struct perf_addr_filters_head {
494 struct list_head list;
495 raw_spinlock_t lock;
496 unsigned int nr_file_filters;
497};
498
499struct perf_addr_filter_range {
500 unsigned long start;
501 unsigned long size;
502};
503
504
505
506
507enum perf_event_state {
508 PERF_EVENT_STATE_DEAD = -4,
509 PERF_EVENT_STATE_EXIT = -3,
510 PERF_EVENT_STATE_ERROR = -2,
511 PERF_EVENT_STATE_OFF = -1,
512 PERF_EVENT_STATE_INACTIVE = 0,
513 PERF_EVENT_STATE_ACTIVE = 1,
514};
515
516struct file;
517struct perf_sample_data;
518
519typedef void (*perf_overflow_handler_t)(struct perf_event *,
520 struct perf_sample_data *,
521 struct pt_regs *regs);
522
523
524
525
526
527
528
529
530#define PERF_EV_CAP_SOFTWARE BIT(0)
531#define PERF_EV_CAP_READ_ACTIVE_PKG BIT(1)
532
533#define SWEVENT_HLIST_BITS 8
534#define SWEVENT_HLIST_SIZE (1 << SWEVENT_HLIST_BITS)
535
536struct swevent_hlist {
537 struct hlist_head heads[SWEVENT_HLIST_SIZE];
538 struct rcu_head rcu_head;
539};
540
541#define PERF_ATTACH_CONTEXT 0x01
542#define PERF_ATTACH_GROUP 0x02
543#define PERF_ATTACH_TASK 0x04
544#define PERF_ATTACH_TASK_DATA 0x08
545#define PERF_ATTACH_ITRACE 0x10
546
547struct perf_cgroup;
548struct ring_buffer;
549
550struct pmu_event_list {
551 raw_spinlock_t lock;
552 struct list_head list;
553};
554
555#define for_each_sibling_event(sibling, event) \
556 if ((event)->group_leader == (event)) \
557 list_for_each_entry((sibling), &(event)->sibling_list, sibling_list)
558
559
560
561
562struct perf_event {
563#ifdef CONFIG_PERF_EVENTS
564
565
566
567
568
569 struct list_head event_entry;
570
571
572
573
574
575 struct list_head sibling_list;
576 struct list_head active_list;
577
578
579
580 struct rb_node group_node;
581 u64 group_index;
582
583
584
585
586
587 struct list_head migrate_entry;
588
589 struct hlist_node hlist_entry;
590 struct list_head active_entry;
591 int nr_siblings;
592
593
594 int event_caps;
595
596 int group_caps;
597
598 struct perf_event *group_leader;
599 struct pmu *pmu;
600 void *pmu_private;
601
602 enum perf_event_state state;
603 unsigned int attach_state;
604 local64_t count;
605 atomic64_t child_count;
606
607
608
609
610
611
612
613 u64 total_time_enabled;
614 u64 total_time_running;
615 u64 tstamp;
616
617
618
619
620
621
622
623
624
625 u64 shadow_ctx_time;
626
627 struct perf_event_attr attr;
628 u16 header_size;
629 u16 id_header_size;
630 u16 read_size;
631 struct hw_perf_event hw;
632
633 struct perf_event_context *ctx;
634 atomic_long_t refcount;
635
636
637
638
639
640 atomic64_t child_total_time_enabled;
641 atomic64_t child_total_time_running;
642
643
644
645
646 struct mutex child_mutex;
647 struct list_head child_list;
648 struct perf_event *parent;
649
650 int oncpu;
651 int cpu;
652
653 struct list_head owner_entry;
654 struct task_struct *owner;
655
656
657 struct mutex mmap_mutex;
658 atomic_t mmap_count;
659
660 struct ring_buffer *rb;
661 struct list_head rb_entry;
662 unsigned long rcu_batches;
663 int rcu_pending;
664
665
666 wait_queue_head_t waitq;
667 struct fasync_struct *fasync;
668
669
670 int pending_wakeup;
671 int pending_kill;
672 int pending_disable;
673 struct irq_work pending;
674
675 atomic_t event_limit;
676
677
678 struct perf_addr_filters_head addr_filters;
679
680 struct perf_addr_filter_range *addr_filter_ranges;
681 unsigned long addr_filters_gen;
682
683 void (*destroy)(struct perf_event *);
684 struct rcu_head rcu_head;
685
686 struct pid_namespace *ns;
687 u64 id;
688
689 u64 (*clock)(void);
690 perf_overflow_handler_t overflow_handler;
691 void *overflow_handler_context;
692#ifdef CONFIG_BPF_SYSCALL
693 perf_overflow_handler_t orig_overflow_handler;
694 struct bpf_prog *prog;
695#endif
696
697#ifdef CONFIG_EVENT_TRACING
698 struct trace_event_call *tp_event;
699 struct event_filter *filter;
700#ifdef CONFIG_FUNCTION_TRACER
701 struct ftrace_ops ftrace_ops;
702#endif
703#endif
704
705#ifdef CONFIG_CGROUP_PERF
706 struct perf_cgroup *cgrp;
707#endif
708
709 struct list_head sb_list;
710#endif
711};
712
713
714struct perf_event_groups {
715 struct rb_root tree;
716 u64 index;
717};
718
719
720
721
722
723
724struct perf_event_context {
725 struct pmu *pmu;
726
727
728
729
730 raw_spinlock_t lock;
731
732
733
734
735
736 struct mutex mutex;
737
738 struct list_head active_ctx_list;
739 struct perf_event_groups pinned_groups;
740 struct perf_event_groups flexible_groups;
741 struct list_head event_list;
742
743 struct list_head pinned_active;
744 struct list_head flexible_active;
745
746 int nr_events;
747 int nr_active;
748 int is_active;
749 int nr_stat;
750 int nr_freq;
751 int rotate_disable;
752 refcount_t refcount;
753 struct task_struct *task;
754
755
756
757
758 u64 time;
759 u64 timestamp;
760
761
762
763
764
765 struct perf_event_context *parent_ctx;
766 u64 parent_gen;
767 u64 generation;
768 int pin_count;
769#ifdef CONFIG_CGROUP_PERF
770 int nr_cgroups;
771#endif
772 void *task_ctx_data;
773 struct rcu_head rcu_head;
774};
775
776
777
778
779
780#define PERF_NR_CONTEXTS 4
781
782
783
784
785struct perf_cpu_context {
786 struct perf_event_context ctx;
787 struct perf_event_context *task_ctx;
788 int active_oncpu;
789 int exclusive;
790
791 raw_spinlock_t hrtimer_lock;
792 struct hrtimer hrtimer;
793 ktime_t hrtimer_interval;
794 unsigned int hrtimer_active;
795
796#ifdef CONFIG_CGROUP_PERF
797 struct perf_cgroup *cgrp;
798 struct list_head cgrp_cpuctx_entry;
799#endif
800
801 struct list_head sched_cb_entry;
802 int sched_cb_usage;
803
804 int online;
805};
806
807struct perf_output_handle {
808 struct perf_event *event;
809 struct ring_buffer *rb;
810 unsigned long wakeup;
811 unsigned long size;
812 u64 aux_flags;
813 union {
814 void *addr;
815 unsigned long head;
816 };
817 int page;
818};
819
820struct bpf_perf_event_data_kern {
821 bpf_user_pt_regs_t *regs;
822 struct perf_sample_data *data;
823 struct perf_event *event;
824};
825
826#ifdef CONFIG_CGROUP_PERF
827
828
829
830
831
832struct perf_cgroup_info {
833 u64 time;
834 u64 timestamp;
835};
836
837struct perf_cgroup {
838 struct cgroup_subsys_state css;
839 struct perf_cgroup_info __percpu *info;
840};
841
842
843
844
845
846
847static inline struct perf_cgroup *
848perf_cgroup_from_task(struct task_struct *task, struct perf_event_context *ctx)
849{
850 return container_of(task_css_check(task, perf_event_cgrp_id,
851 ctx ? lockdep_is_held(&ctx->lock)
852 : true),
853 struct perf_cgroup, css);
854}
855#endif
856
857#ifdef CONFIG_PERF_EVENTS
858
859extern void *perf_aux_output_begin(struct perf_output_handle *handle,
860 struct perf_event *event);
861extern void perf_aux_output_end(struct perf_output_handle *handle,
862 unsigned long size);
863extern int perf_aux_output_skip(struct perf_output_handle *handle,
864 unsigned long size);
865extern void *perf_get_aux(struct perf_output_handle *handle);
866extern void perf_aux_output_flag(struct perf_output_handle *handle, u64 flags);
867extern void perf_event_itrace_started(struct perf_event *event);
868
869extern int perf_pmu_register(struct pmu *pmu, const char *name, int type);
870extern void perf_pmu_unregister(struct pmu *pmu);
871
872extern int perf_num_counters(void);
873extern const char *perf_pmu_name(void);
874extern void __perf_event_task_sched_in(struct task_struct *prev,
875 struct task_struct *task);
876extern void __perf_event_task_sched_out(struct task_struct *prev,
877 struct task_struct *next);
878extern int perf_event_init_task(struct task_struct *child);
879extern void perf_event_exit_task(struct task_struct *child);
880extern void perf_event_free_task(struct task_struct *task);
881extern void perf_event_delayed_put(struct task_struct *task);
882extern struct file *perf_event_get(unsigned int fd);
883extern const struct perf_event *perf_get_event(struct file *file);
884extern const struct perf_event_attr *perf_event_attrs(struct perf_event *event);
885extern void perf_event_print_debug(void);
886extern void perf_pmu_disable(struct pmu *pmu);
887extern void perf_pmu_enable(struct pmu *pmu);
888extern void perf_sched_cb_dec(struct pmu *pmu);
889extern void perf_sched_cb_inc(struct pmu *pmu);
890extern int perf_event_task_disable(void);
891extern int perf_event_task_enable(void);
892
893extern void perf_pmu_resched(struct pmu *pmu);
894
895extern int perf_event_refresh(struct perf_event *event, int refresh);
896extern void perf_event_update_userpage(struct perf_event *event);
897extern int perf_event_release_kernel(struct perf_event *event);
898extern struct perf_event *
899perf_event_create_kernel_counter(struct perf_event_attr *attr,
900 int cpu,
901 struct task_struct *task,
902 perf_overflow_handler_t callback,
903 void *context);
904extern void perf_pmu_migrate_context(struct pmu *pmu,
905 int src_cpu, int dst_cpu);
906int perf_event_read_local(struct perf_event *event, u64 *value,
907 u64 *enabled, u64 *running);
908extern u64 perf_event_read_value(struct perf_event *event,
909 u64 *enabled, u64 *running);
910
911
912struct perf_sample_data {
913
914
915
916
917 u64 addr;
918 struct perf_raw_record *raw;
919 struct perf_branch_stack *br_stack;
920 u64 period;
921 u64 weight;
922 u64 txn;
923 union perf_mem_data_src data_src;
924
925
926
927
928
929 u64 type;
930 u64 ip;
931 struct {
932 u32 pid;
933 u32 tid;
934 } tid_entry;
935 u64 time;
936 u64 id;
937 u64 stream_id;
938 struct {
939 u32 cpu;
940 u32 reserved;
941 } cpu_entry;
942 struct perf_callchain_entry *callchain;
943
944
945
946
947
948 struct perf_regs regs_user;
949 struct pt_regs regs_user_copy;
950
951 struct perf_regs regs_intr;
952 u64 stack_user_size;
953
954 u64 phys_addr;
955} ____cacheline_aligned;
956
957
958#define PERF_MEM_NA (PERF_MEM_S(OP, NA) |\
959 PERF_MEM_S(LVL, NA) |\
960 PERF_MEM_S(SNOOP, NA) |\
961 PERF_MEM_S(LOCK, NA) |\
962 PERF_MEM_S(TLB, NA))
963
964static inline void perf_sample_data_init(struct perf_sample_data *data,
965 u64 addr, u64 period)
966{
967
968 data->addr = addr;
969 data->raw = NULL;
970 data->br_stack = NULL;
971 data->period = period;
972 data->weight = 0;
973 data->data_src.val = PERF_MEM_NA;
974 data->txn = 0;
975}
976
977extern void perf_output_sample(struct perf_output_handle *handle,
978 struct perf_event_header *header,
979 struct perf_sample_data *data,
980 struct perf_event *event);
981extern void perf_prepare_sample(struct perf_event_header *header,
982 struct perf_sample_data *data,
983 struct perf_event *event,
984 struct pt_regs *regs);
985
986extern int perf_event_overflow(struct perf_event *event,
987 struct perf_sample_data *data,
988 struct pt_regs *regs);
989
990extern void perf_event_output_forward(struct perf_event *event,
991 struct perf_sample_data *data,
992 struct pt_regs *regs);
993extern void perf_event_output_backward(struct perf_event *event,
994 struct perf_sample_data *data,
995 struct pt_regs *regs);
996extern int perf_event_output(struct perf_event *event,
997 struct perf_sample_data *data,
998 struct pt_regs *regs);
999
1000static inline bool
1001is_default_overflow_handler(struct perf_event *event)
1002{
1003 if (likely(event->overflow_handler == perf_event_output_forward))
1004 return true;
1005 if (unlikely(event->overflow_handler == perf_event_output_backward))
1006 return true;
1007 return false;
1008}
1009
1010extern void
1011perf_event_header__init_id(struct perf_event_header *header,
1012 struct perf_sample_data *data,
1013 struct perf_event *event);
1014extern void
1015perf_event__output_id_sample(struct perf_event *event,
1016 struct perf_output_handle *handle,
1017 struct perf_sample_data *sample);
1018
1019extern void
1020perf_log_lost_samples(struct perf_event *event, u64 lost);
1021
1022static inline bool event_has_any_exclude_flag(struct perf_event *event)
1023{
1024 struct perf_event_attr *attr = &event->attr;
1025
1026 return attr->exclude_idle || attr->exclude_user ||
1027 attr->exclude_kernel || attr->exclude_hv ||
1028 attr->exclude_guest || attr->exclude_host;
1029}
1030
1031static inline bool is_sampling_event(struct perf_event *event)
1032{
1033 return event->attr.sample_period != 0;
1034}
1035
1036
1037
1038
1039static inline int is_software_event(struct perf_event *event)
1040{
1041 return event->event_caps & PERF_EV_CAP_SOFTWARE;
1042}
1043
1044
1045
1046
1047static inline int in_software_context(struct perf_event *event)
1048{
1049 return event->ctx->pmu->task_ctx_nr == perf_sw_context;
1050}
1051
1052extern struct static_key perf_swevent_enabled[PERF_COUNT_SW_MAX];
1053
1054extern void ___perf_sw_event(u32, u64, struct pt_regs *, u64);
1055extern void __perf_sw_event(u32, u64, struct pt_regs *, u64);
1056
1057#ifndef perf_arch_fetch_caller_regs
1058static inline void perf_arch_fetch_caller_regs(struct pt_regs *regs, unsigned long ip) { }
1059#endif
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075static inline void perf_fetch_caller_regs(struct pt_regs *regs)
1076{
1077 perf_arch_fetch_caller_regs(regs, CALLER_ADDR0);
1078}
1079
1080static __always_inline void
1081perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr)
1082{
1083 if (static_key_false(&perf_swevent_enabled[event_id]))
1084 __perf_sw_event(event_id, nr, regs, addr);
1085}
1086
1087DECLARE_PER_CPU(struct pt_regs, __perf_regs[4]);
1088
1089
1090
1091
1092
1093
1094static __always_inline void
1095perf_sw_event_sched(u32 event_id, u64 nr, u64 addr)
1096{
1097 if (static_key_false(&perf_swevent_enabled[event_id])) {
1098 struct pt_regs *regs = this_cpu_ptr(&__perf_regs[0]);
1099
1100 perf_fetch_caller_regs(regs);
1101 ___perf_sw_event(event_id, nr, regs, addr);
1102 }
1103}
1104
1105extern struct static_key_false perf_sched_events;
1106
1107static __always_inline bool
1108perf_sw_migrate_enabled(void)
1109{
1110 if (static_key_false(&perf_swevent_enabled[PERF_COUNT_SW_CPU_MIGRATIONS]))
1111 return true;
1112 return false;
1113}
1114
1115static inline void perf_event_task_migrate(struct task_struct *task)
1116{
1117 if (perf_sw_migrate_enabled())
1118 task->sched_migrated = 1;
1119}
1120
1121static inline void perf_event_task_sched_in(struct task_struct *prev,
1122 struct task_struct *task)
1123{
1124 if (static_branch_unlikely(&perf_sched_events))
1125 __perf_event_task_sched_in(prev, task);
1126
1127 if (perf_sw_migrate_enabled() && task->sched_migrated) {
1128 struct pt_regs *regs = this_cpu_ptr(&__perf_regs[0]);
1129
1130 perf_fetch_caller_regs(regs);
1131 ___perf_sw_event(PERF_COUNT_SW_CPU_MIGRATIONS, 1, regs, 0);
1132 task->sched_migrated = 0;
1133 }
1134}
1135
1136static inline void perf_event_task_sched_out(struct task_struct *prev,
1137 struct task_struct *next)
1138{
1139 perf_sw_event_sched(PERF_COUNT_SW_CONTEXT_SWITCHES, 1, 0);
1140
1141 if (static_branch_unlikely(&perf_sched_events))
1142 __perf_event_task_sched_out(prev, next);
1143}
1144
1145extern void perf_event_mmap(struct vm_area_struct *vma);
1146
1147extern void perf_event_ksymbol(u16 ksym_type, u64 addr, u32 len,
1148 bool unregister, const char *sym);
1149extern void perf_event_bpf_event(struct bpf_prog *prog,
1150 enum perf_bpf_event_type type,
1151 u16 flags);
1152
1153extern struct perf_guest_info_callbacks *perf_guest_cbs;
1154extern int perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *callbacks);
1155extern int perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *callbacks);
1156
1157extern void perf_event_exec(void);
1158extern void perf_event_comm(struct task_struct *tsk, bool exec);
1159extern void perf_event_namespaces(struct task_struct *tsk);
1160extern void perf_event_fork(struct task_struct *tsk);
1161
1162
1163DECLARE_PER_CPU(struct perf_callchain_entry, perf_callchain_entry);
1164
1165extern void perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs);
1166extern void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs);
1167extern struct perf_callchain_entry *
1168get_perf_callchain(struct pt_regs *regs, u32 init_nr, bool kernel, bool user,
1169 u32 max_stack, bool crosstask, bool add_mark);
1170extern struct perf_callchain_entry *perf_callchain(struct perf_event *event, struct pt_regs *regs);
1171extern int get_callchain_buffers(int max_stack);
1172extern void put_callchain_buffers(void);
1173
1174extern int sysctl_perf_event_max_stack;
1175extern int sysctl_perf_event_max_contexts_per_stack;
1176
1177static inline int perf_callchain_store_context(struct perf_callchain_entry_ctx *ctx, u64 ip)
1178{
1179 if (ctx->contexts < sysctl_perf_event_max_contexts_per_stack) {
1180 struct perf_callchain_entry *entry = ctx->entry;
1181 entry->ip[entry->nr++] = ip;
1182 ++ctx->contexts;
1183 return 0;
1184 } else {
1185 ctx->contexts_maxed = true;
1186 return -1;
1187 }
1188}
1189
1190static inline int perf_callchain_store(struct perf_callchain_entry_ctx *ctx, u64 ip)
1191{
1192 if (ctx->nr < ctx->max_stack && !ctx->contexts_maxed) {
1193 struct perf_callchain_entry *entry = ctx->entry;
1194 entry->ip[entry->nr++] = ip;
1195 ++ctx->nr;
1196 return 0;
1197 } else {
1198 return -1;
1199 }
1200}
1201
1202extern int sysctl_perf_event_paranoid;
1203extern int sysctl_perf_event_mlock;
1204extern int sysctl_perf_event_sample_rate;
1205extern int sysctl_perf_cpu_time_max_percent;
1206
1207extern void perf_sample_event_took(u64 sample_len_ns);
1208
1209extern int perf_proc_update_handler(struct ctl_table *table, int write,
1210 void __user *buffer, size_t *lenp,
1211 loff_t *ppos);
1212extern int perf_cpu_time_max_percent_handler(struct ctl_table *table, int write,
1213 void __user *buffer, size_t *lenp,
1214 loff_t *ppos);
1215
1216int perf_event_max_stack_handler(struct ctl_table *table, int write,
1217 void __user *buffer, size_t *lenp, loff_t *ppos);
1218
1219static inline bool perf_paranoid_tracepoint_raw(void)
1220{
1221 return sysctl_perf_event_paranoid > -1;
1222}
1223
1224static inline bool perf_paranoid_cpu(void)
1225{
1226 return sysctl_perf_event_paranoid > 0;
1227}
1228
1229static inline bool perf_paranoid_kernel(void)
1230{
1231 return sysctl_perf_event_paranoid > 1;
1232}
1233
1234extern void perf_event_init(void);
1235extern void perf_tp_event(u16 event_type, u64 count, void *record,
1236 int entry_size, struct pt_regs *regs,
1237 struct hlist_head *head, int rctx,
1238 struct task_struct *task);
1239extern void perf_bp_event(struct perf_event *event, void *data);
1240
1241#ifndef perf_misc_flags
1242# define perf_misc_flags(regs) \
1243 (user_mode(regs) ? PERF_RECORD_MISC_USER : PERF_RECORD_MISC_KERNEL)
1244# define perf_instruction_pointer(regs) instruction_pointer(regs)
1245#endif
1246#ifndef perf_arch_bpf_user_pt_regs
1247# define perf_arch_bpf_user_pt_regs(regs) regs
1248#endif
1249
1250static inline bool has_branch_stack(struct perf_event *event)
1251{
1252 return event->attr.sample_type & PERF_SAMPLE_BRANCH_STACK;
1253}
1254
1255static inline bool needs_branch_stack(struct perf_event *event)
1256{
1257 return event->attr.branch_sample_type != 0;
1258}
1259
1260static inline bool has_aux(struct perf_event *event)
1261{
1262 return event->pmu->setup_aux;
1263}
1264
1265static inline bool is_write_backward(struct perf_event *event)
1266{
1267 return !!event->attr.write_backward;
1268}
1269
1270static inline bool has_addr_filter(struct perf_event *event)
1271{
1272 return event->pmu->nr_addr_filters;
1273}
1274
1275
1276
1277
1278static inline struct perf_addr_filters_head *
1279perf_event_addr_filters(struct perf_event *event)
1280{
1281 struct perf_addr_filters_head *ifh = &event->addr_filters;
1282
1283 if (event->parent)
1284 ifh = &event->parent->addr_filters;
1285
1286 return ifh;
1287}
1288
1289extern void perf_event_addr_filters_sync(struct perf_event *event);
1290
1291extern int perf_output_begin(struct perf_output_handle *handle,
1292 struct perf_event *event, unsigned int size);
1293extern int perf_output_begin_forward(struct perf_output_handle *handle,
1294 struct perf_event *event,
1295 unsigned int size);
1296extern int perf_output_begin_backward(struct perf_output_handle *handle,
1297 struct perf_event *event,
1298 unsigned int size);
1299
1300extern void perf_output_end(struct perf_output_handle *handle);
1301extern unsigned int perf_output_copy(struct perf_output_handle *handle,
1302 const void *buf, unsigned int len);
1303extern unsigned int perf_output_skip(struct perf_output_handle *handle,
1304 unsigned int len);
1305extern int perf_swevent_get_recursion_context(void);
1306extern void perf_swevent_put_recursion_context(int rctx);
1307extern u64 perf_swevent_set_period(struct perf_event *event);
1308extern void perf_event_enable(struct perf_event *event);
1309extern void perf_event_disable(struct perf_event *event);
1310extern void perf_event_disable_local(struct perf_event *event);
1311extern void perf_event_disable_inatomic(struct perf_event *event);
1312extern void perf_event_task_tick(void);
1313extern int perf_event_account_interrupt(struct perf_event *event);
1314#else
1315static inline void *
1316perf_aux_output_begin(struct perf_output_handle *handle,
1317 struct perf_event *event) { return NULL; }
1318static inline void
1319perf_aux_output_end(struct perf_output_handle *handle, unsigned long size)
1320 { }
1321static inline int
1322perf_aux_output_skip(struct perf_output_handle *handle,
1323 unsigned long size) { return -EINVAL; }
1324static inline void *
1325perf_get_aux(struct perf_output_handle *handle) { return NULL; }
1326static inline void
1327perf_event_task_migrate(struct task_struct *task) { }
1328static inline void
1329perf_event_task_sched_in(struct task_struct *prev,
1330 struct task_struct *task) { }
1331static inline void
1332perf_event_task_sched_out(struct task_struct *prev,
1333 struct task_struct *next) { }
1334static inline int perf_event_init_task(struct task_struct *child) { return 0; }
1335static inline void perf_event_exit_task(struct task_struct *child) { }
1336static inline void perf_event_free_task(struct task_struct *task) { }
1337static inline void perf_event_delayed_put(struct task_struct *task) { }
1338static inline struct file *perf_event_get(unsigned int fd) { return ERR_PTR(-EINVAL); }
1339static inline const struct perf_event *perf_get_event(struct file *file)
1340{
1341 return ERR_PTR(-EINVAL);
1342}
1343static inline const struct perf_event_attr *perf_event_attrs(struct perf_event *event)
1344{
1345 return ERR_PTR(-EINVAL);
1346}
1347static inline int perf_event_read_local(struct perf_event *event, u64 *value,
1348 u64 *enabled, u64 *running)
1349{
1350 return -EINVAL;
1351}
1352static inline void perf_event_print_debug(void) { }
1353static inline int perf_event_task_disable(void) { return -EINVAL; }
1354static inline int perf_event_task_enable(void) { return -EINVAL; }
1355static inline int perf_event_refresh(struct perf_event *event, int refresh)
1356{
1357 return -EINVAL;
1358}
1359
1360static inline void
1361perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr) { }
1362static inline void
1363perf_sw_event_sched(u32 event_id, u64 nr, u64 addr) { }
1364static inline void
1365perf_bp_event(struct perf_event *event, void *data) { }
1366
1367static inline int perf_register_guest_info_callbacks
1368(struct perf_guest_info_callbacks *callbacks) { return 0; }
1369static inline int perf_unregister_guest_info_callbacks
1370(struct perf_guest_info_callbacks *callbacks) { return 0; }
1371
1372static inline void perf_event_mmap(struct vm_area_struct *vma) { }
1373
1374typedef int (perf_ksymbol_get_name_f)(char *name, int name_len, void *data);
1375static inline void perf_event_ksymbol(u16 ksym_type, u64 addr, u32 len,
1376 bool unregister, const char *sym) { }
1377static inline void perf_event_bpf_event(struct bpf_prog *prog,
1378 enum perf_bpf_event_type type,
1379 u16 flags) { }
1380static inline void perf_event_exec(void) { }
1381static inline void perf_event_comm(struct task_struct *tsk, bool exec) { }
1382static inline void perf_event_namespaces(struct task_struct *tsk) { }
1383static inline void perf_event_fork(struct task_struct *tsk) { }
1384static inline void perf_event_init(void) { }
1385static inline int perf_swevent_get_recursion_context(void) { return -1; }
1386static inline void perf_swevent_put_recursion_context(int rctx) { }
1387static inline u64 perf_swevent_set_period(struct perf_event *event) { return 0; }
1388static inline void perf_event_enable(struct perf_event *event) { }
1389static inline void perf_event_disable(struct perf_event *event) { }
1390static inline int __perf_event_disable(void *info) { return -1; }
1391static inline void perf_event_task_tick(void) { }
1392static inline int perf_event_release_kernel(struct perf_event *event) { return 0; }
1393#endif
1394
1395#if defined(CONFIG_PERF_EVENTS) && defined(CONFIG_CPU_SUP_INTEL)
1396extern void perf_restore_debug_store(void);
1397#else
1398static inline void perf_restore_debug_store(void) { }
1399#endif
1400
1401static __always_inline bool perf_raw_frag_last(const struct perf_raw_frag *frag)
1402{
1403 return frag->pad < sizeof(u64);
1404}
1405
1406#define perf_output_put(handle, x) perf_output_copy((handle), &(x), sizeof(x))
1407
1408struct perf_pmu_events_attr {
1409 struct device_attribute attr;
1410 u64 id;
1411 const char *event_str;
1412};
1413
1414struct perf_pmu_events_ht_attr {
1415 struct device_attribute attr;
1416 u64 id;
1417 const char *event_str_ht;
1418 const char *event_str_noht;
1419};
1420
1421ssize_t perf_event_sysfs_show(struct device *dev, struct device_attribute *attr,
1422 char *page);
1423
1424#define PMU_EVENT_ATTR(_name, _var, _id, _show) \
1425static struct perf_pmu_events_attr _var = { \
1426 .attr = __ATTR(_name, 0444, _show, NULL), \
1427 .id = _id, \
1428};
1429
1430#define PMU_EVENT_ATTR_STRING(_name, _var, _str) \
1431static struct perf_pmu_events_attr _var = { \
1432 .attr = __ATTR(_name, 0444, perf_event_sysfs_show, NULL), \
1433 .id = 0, \
1434 .event_str = _str, \
1435};
1436
1437#define PMU_FORMAT_ATTR(_name, _format) \
1438static ssize_t \
1439_name##_show(struct device *dev, \
1440 struct device_attribute *attr, \
1441 char *page) \
1442{ \
1443 BUILD_BUG_ON(sizeof(_format) >= PAGE_SIZE); \
1444 return sprintf(page, _format "\n"); \
1445} \
1446 \
1447static struct device_attribute format_attr_##_name = __ATTR_RO(_name)
1448
1449
1450#ifdef CONFIG_PERF_EVENTS
1451int perf_event_init_cpu(unsigned int cpu);
1452int perf_event_exit_cpu(unsigned int cpu);
1453#else
1454#define perf_event_init_cpu NULL
1455#define perf_event_exit_cpu NULL
1456#endif
1457
1458#endif
1459