1
2
3
4
5
6
7#ifndef __PERF_AUXTRACE_H
8#define __PERF_AUXTRACE_H
9
10#include <sys/types.h>
11#include <errno.h>
12#include <stdbool.h>
13#include <stddef.h>
14#include <stdio.h>
15#include <linux/list.h>
16#include <linux/perf_event.h>
17#include <linux/types.h>
18#include <asm/bitsperlong.h>
19#include <asm/barrier.h>
20
21union perf_event;
22struct perf_session;
23struct evlist;
24struct evsel;
25struct perf_tool;
26struct mmap;
27struct perf_sample;
28struct option;
29struct record_opts;
30struct perf_record_auxtrace_error;
31struct perf_record_auxtrace_info;
32struct events_stats;
33struct perf_pmu;
34
35enum auxtrace_error_type {
36 PERF_AUXTRACE_ERROR_ITRACE = 1,
37 PERF_AUXTRACE_ERROR_MAX
38};
39
40
41#define PERF_AUXTRACE_RECORD_ALIGNMENT 8
42
43enum auxtrace_type {
44 PERF_AUXTRACE_UNKNOWN,
45 PERF_AUXTRACE_INTEL_PT,
46 PERF_AUXTRACE_INTEL_BTS,
47 PERF_AUXTRACE_CS_ETM,
48 PERF_AUXTRACE_ARM_SPE,
49 PERF_AUXTRACE_S390_CPUMSF,
50};
51
52enum itrace_period_type {
53 PERF_ITRACE_PERIOD_INSTRUCTIONS,
54 PERF_ITRACE_PERIOD_TICKS,
55 PERF_ITRACE_PERIOD_NANOSECS,
56};
57
58#define AUXTRACE_ERR_FLG_OVERFLOW (1 << ('o' - 'a'))
59#define AUXTRACE_ERR_FLG_DATA_LOST (1 << ('l' - 'a'))
60
61#define AUXTRACE_LOG_FLG_ALL_PERF_EVTS (1 << ('a' - 'a'))
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110struct itrace_synth_opts {
111 bool set;
112 bool default_no_sample;
113 bool inject;
114 bool instructions;
115 bool branches;
116 bool transactions;
117 bool ptwrites;
118 bool pwr_events;
119 bool other_events;
120 bool errors;
121 bool dont_decode;
122 bool log;
123 bool calls;
124 bool returns;
125 bool callchain;
126 bool add_callchain;
127 bool thread_stack;
128 bool last_branch;
129 bool add_last_branch;
130 bool flc;
131 bool llc;
132 bool tlb;
133 bool remote_access;
134 bool mem;
135 bool timeless_decoding;
136 bool vm_time_correlation;
137 bool vm_tm_corr_dry_run;
138 char *vm_tm_corr_args;
139 unsigned int callchain_sz;
140 unsigned int last_branch_sz;
141 unsigned long long period;
142 enum itrace_period_type period_type;
143 unsigned long initial_skip;
144 unsigned long *cpu_bitmap;
145 struct perf_time_interval *ptime_range;
146 int range_num;
147 unsigned int error_plus_flags;
148 unsigned int error_minus_flags;
149 unsigned int log_plus_flags;
150 unsigned int log_minus_flags;
151 unsigned int quick;
152};
153
154
155
156
157
158
159
160struct auxtrace_index_entry {
161 u64 file_offset;
162 u64 sz;
163};
164
165#define PERF_AUXTRACE_INDEX_ENTRY_COUNT 256
166
167
168
169
170
171
172
173
174struct auxtrace_index {
175 struct list_head list;
176 size_t nr;
177 struct auxtrace_index_entry entries[PERF_AUXTRACE_INDEX_ENTRY_COUNT];
178};
179
180
181
182
183
184
185
186
187
188
189
190
191struct auxtrace {
192 int (*process_event)(struct perf_session *session,
193 union perf_event *event,
194 struct perf_sample *sample,
195 struct perf_tool *tool);
196 int (*process_auxtrace_event)(struct perf_session *session,
197 union perf_event *event,
198 struct perf_tool *tool);
199 int (*queue_data)(struct perf_session *session,
200 struct perf_sample *sample, union perf_event *event,
201 u64 data_offset);
202 void (*dump_auxtrace_sample)(struct perf_session *session,
203 struct perf_sample *sample);
204 int (*flush_events)(struct perf_session *session,
205 struct perf_tool *tool);
206 void (*free_events)(struct perf_session *session);
207 void (*free)(struct perf_session *session);
208 bool (*evsel_is_auxtrace)(struct perf_session *session,
209 struct evsel *evsel);
210};
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235struct auxtrace_buffer {
236 struct list_head list;
237 size_t size;
238 pid_t pid;
239 pid_t tid;
240 int cpu;
241 void *data;
242 off_t data_offset;
243 void *mmap_addr;
244 size_t mmap_size;
245 bool data_needs_freeing;
246 bool consecutive;
247 u64 offset;
248 u64 reference;
249 u64 buffer_nr;
250 size_t use_size;
251 void *use_data;
252};
253
254
255
256
257
258
259
260
261
262struct auxtrace_queue {
263 struct list_head head;
264 pid_t tid;
265 int cpu;
266 bool set;
267 void *priv;
268};
269
270
271
272
273
274
275
276
277
278struct auxtrace_queues {
279 struct auxtrace_queue *queue_array;
280 unsigned int nr_queues;
281 bool new_data;
282 bool populated;
283 u64 next_buffer_nr;
284};
285
286
287
288
289
290
291
292struct auxtrace_heap_item {
293 unsigned int queue_nr;
294 u64 ordinal;
295};
296
297
298
299
300
301
302
303struct auxtrace_heap {
304 struct auxtrace_heap_item *heap_array;
305 unsigned int heap_cnt;
306 unsigned int heap_sz;
307};
308
309
310
311
312
313
314
315
316
317
318
319
320
321struct auxtrace_mmap {
322 void *base;
323 void *userpg;
324 size_t mask;
325 size_t len;
326 u64 prev;
327 int idx;
328 pid_t tid;
329 int cpu;
330};
331
332
333
334
335
336
337
338
339
340
341
342
343struct auxtrace_mmap_params {
344 size_t mask;
345 off_t offset;
346 size_t len;
347 int prot;
348 int idx;
349 pid_t tid;
350 int cpu;
351};
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370struct auxtrace_record {
371 int (*recording_options)(struct auxtrace_record *itr,
372 struct evlist *evlist,
373 struct record_opts *opts);
374 size_t (*info_priv_size)(struct auxtrace_record *itr,
375 struct evlist *evlist);
376 int (*info_fill)(struct auxtrace_record *itr,
377 struct perf_session *session,
378 struct perf_record_auxtrace_info *auxtrace_info,
379 size_t priv_size);
380 void (*free)(struct auxtrace_record *itr);
381 int (*snapshot_start)(struct auxtrace_record *itr);
382 int (*snapshot_finish)(struct auxtrace_record *itr);
383 int (*find_snapshot)(struct auxtrace_record *itr, int idx,
384 struct auxtrace_mmap *mm, unsigned char *data,
385 u64 *head, u64 *old);
386 int (*parse_snapshot_options)(struct auxtrace_record *itr,
387 struct record_opts *opts,
388 const char *str);
389 u64 (*reference)(struct auxtrace_record *itr);
390 int (*read_finish)(struct auxtrace_record *itr, int idx);
391 unsigned int alignment;
392 unsigned int default_aux_sample_size;
393 struct perf_pmu *pmu;
394 struct evlist *evlist;
395};
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414struct addr_filter {
415 struct list_head list;
416 bool range;
417 bool start;
418 const char *action;
419 const char *sym_from;
420 const char *sym_to;
421 int sym_from_idx;
422 int sym_to_idx;
423 u64 addr;
424 u64 size;
425 const char *filename;
426 char *str;
427};
428
429
430
431
432
433
434struct addr_filters {
435 struct list_head head;
436 int cnt;
437};
438
439struct auxtrace_cache;
440
441#ifdef HAVE_AUXTRACE_SUPPORT
442
443
444
445
446
447
448
449static inline u64 auxtrace_mmap__read_snapshot_head(struct auxtrace_mmap *mm)
450{
451 struct perf_event_mmap_page *pc = mm->userpg;
452 u64 head = READ_ONCE(pc->aux_head);
453
454
455 smp_rmb();
456 return head;
457}
458
459static inline u64 auxtrace_mmap__read_head(struct auxtrace_mmap *mm)
460{
461 struct perf_event_mmap_page *pc = mm->userpg;
462#if BITS_PER_LONG == 64 || !defined(HAVE_SYNC_COMPARE_AND_SWAP_SUPPORT)
463 u64 head = READ_ONCE(pc->aux_head);
464#else
465 u64 head = __sync_val_compare_and_swap(&pc->aux_head, 0, 0);
466#endif
467
468
469 smp_rmb();
470 return head;
471}
472
473static inline void auxtrace_mmap__write_tail(struct auxtrace_mmap *mm, u64 tail)
474{
475 struct perf_event_mmap_page *pc = mm->userpg;
476#if BITS_PER_LONG != 64 && defined(HAVE_SYNC_COMPARE_AND_SWAP_SUPPORT)
477 u64 old_tail;
478#endif
479
480
481 smp_mb();
482#if BITS_PER_LONG == 64 || !defined(HAVE_SYNC_COMPARE_AND_SWAP_SUPPORT)
483 pc->aux_tail = tail;
484#else
485 do {
486 old_tail = __sync_val_compare_and_swap(&pc->aux_tail, 0, 0);
487 } while (!__sync_bool_compare_and_swap(&pc->aux_tail, old_tail, tail));
488#endif
489}
490
491int auxtrace_mmap__mmap(struct auxtrace_mmap *mm,
492 struct auxtrace_mmap_params *mp,
493 void *userpg, int fd);
494void auxtrace_mmap__munmap(struct auxtrace_mmap *mm);
495void auxtrace_mmap_params__init(struct auxtrace_mmap_params *mp,
496 off_t auxtrace_offset,
497 unsigned int auxtrace_pages,
498 bool auxtrace_overwrite);
499void auxtrace_mmap_params__set_idx(struct auxtrace_mmap_params *mp,
500 struct evlist *evlist, int idx,
501 bool per_cpu);
502
503typedef int (*process_auxtrace_t)(struct perf_tool *tool,
504 struct mmap *map,
505 union perf_event *event, void *data1,
506 size_t len1, void *data2, size_t len2);
507
508int auxtrace_mmap__read(struct mmap *map, struct auxtrace_record *itr,
509 struct perf_tool *tool, process_auxtrace_t fn);
510
511int auxtrace_mmap__read_snapshot(struct mmap *map,
512 struct auxtrace_record *itr,
513 struct perf_tool *tool, process_auxtrace_t fn,
514 size_t snapshot_size);
515
516int auxtrace_queues__init(struct auxtrace_queues *queues);
517int auxtrace_queues__add_event(struct auxtrace_queues *queues,
518 struct perf_session *session,
519 union perf_event *event, off_t data_offset,
520 struct auxtrace_buffer **buffer_ptr);
521struct auxtrace_queue *
522auxtrace_queues__sample_queue(struct auxtrace_queues *queues,
523 struct perf_sample *sample,
524 struct perf_session *session);
525int auxtrace_queues__add_sample(struct auxtrace_queues *queues,
526 struct perf_session *session,
527 struct perf_sample *sample, u64 data_offset,
528 u64 reference);
529void auxtrace_queues__free(struct auxtrace_queues *queues);
530int auxtrace_queues__process_index(struct auxtrace_queues *queues,
531 struct perf_session *session);
532int auxtrace_queue_data(struct perf_session *session, bool samples,
533 bool events);
534struct auxtrace_buffer *auxtrace_buffer__next(struct auxtrace_queue *queue,
535 struct auxtrace_buffer *buffer);
536void *auxtrace_buffer__get_data_rw(struct auxtrace_buffer *buffer, int fd, bool rw);
537static inline void *auxtrace_buffer__get_data(struct auxtrace_buffer *buffer, int fd)
538{
539 return auxtrace_buffer__get_data_rw(buffer, fd, false);
540}
541void auxtrace_buffer__put_data(struct auxtrace_buffer *buffer);
542void auxtrace_buffer__drop_data(struct auxtrace_buffer *buffer);
543void auxtrace_buffer__free(struct auxtrace_buffer *buffer);
544
545int auxtrace_heap__add(struct auxtrace_heap *heap, unsigned int queue_nr,
546 u64 ordinal);
547void auxtrace_heap__pop(struct auxtrace_heap *heap);
548void auxtrace_heap__free(struct auxtrace_heap *heap);
549
550struct auxtrace_cache_entry {
551 struct hlist_node hash;
552 u32 key;
553};
554
555struct auxtrace_cache *auxtrace_cache__new(unsigned int bits, size_t entry_size,
556 unsigned int limit_percent);
557void auxtrace_cache__free(struct auxtrace_cache *auxtrace_cache);
558void *auxtrace_cache__alloc_entry(struct auxtrace_cache *c);
559void auxtrace_cache__free_entry(struct auxtrace_cache *c, void *entry);
560int auxtrace_cache__add(struct auxtrace_cache *c, u32 key,
561 struct auxtrace_cache_entry *entry);
562void auxtrace_cache__remove(struct auxtrace_cache *c, u32 key);
563void *auxtrace_cache__lookup(struct auxtrace_cache *c, u32 key);
564
565struct auxtrace_record *auxtrace_record__init(struct evlist *evlist,
566 int *err);
567
568int auxtrace_parse_snapshot_options(struct auxtrace_record *itr,
569 struct record_opts *opts,
570 const char *str);
571int auxtrace_parse_sample_options(struct auxtrace_record *itr,
572 struct evlist *evlist,
573 struct record_opts *opts, const char *str);
574void auxtrace_regroup_aux_output(struct evlist *evlist);
575int auxtrace_record__options(struct auxtrace_record *itr,
576 struct evlist *evlist,
577 struct record_opts *opts);
578size_t auxtrace_record__info_priv_size(struct auxtrace_record *itr,
579 struct evlist *evlist);
580int auxtrace_record__info_fill(struct auxtrace_record *itr,
581 struct perf_session *session,
582 struct perf_record_auxtrace_info *auxtrace_info,
583 size_t priv_size);
584void auxtrace_record__free(struct auxtrace_record *itr);
585int auxtrace_record__snapshot_start(struct auxtrace_record *itr);
586int auxtrace_record__snapshot_finish(struct auxtrace_record *itr, bool on_exit);
587int auxtrace_record__find_snapshot(struct auxtrace_record *itr, int idx,
588 struct auxtrace_mmap *mm,
589 unsigned char *data, u64 *head, u64 *old);
590u64 auxtrace_record__reference(struct auxtrace_record *itr);
591int auxtrace_record__read_finish(struct auxtrace_record *itr, int idx);
592
593int auxtrace_index__auxtrace_event(struct list_head *head, union perf_event *event,
594 off_t file_offset);
595int auxtrace_index__write(int fd, struct list_head *head);
596int auxtrace_index__process(int fd, u64 size, struct perf_session *session,
597 bool needs_swap);
598void auxtrace_index__free(struct list_head *head);
599
600void auxtrace_synth_error(struct perf_record_auxtrace_error *auxtrace_error, int type,
601 int code, int cpu, pid_t pid, pid_t tid, u64 ip,
602 const char *msg, u64 timestamp);
603
604int perf_event__process_auxtrace_info(struct perf_session *session,
605 union perf_event *event);
606s64 perf_event__process_auxtrace(struct perf_session *session,
607 union perf_event *event);
608int perf_event__process_auxtrace_error(struct perf_session *session,
609 union perf_event *event);
610int itrace_do_parse_synth_opts(struct itrace_synth_opts *synth_opts,
611 const char *str, int unset);
612int itrace_parse_synth_opts(const struct option *opt, const char *str,
613 int unset);
614void itrace_synth_opts__set_default(struct itrace_synth_opts *synth_opts,
615 bool no_sample);
616
617size_t perf_event__fprintf_auxtrace_error(union perf_event *event, FILE *fp);
618void perf_session__auxtrace_error_inc(struct perf_session *session,
619 union perf_event *event);
620void events_stats__auxtrace_error_warn(const struct events_stats *stats);
621
622void addr_filters__init(struct addr_filters *filts);
623void addr_filters__exit(struct addr_filters *filts);
624int addr_filters__parse_bare_filter(struct addr_filters *filts,
625 const char *filter);
626int auxtrace_parse_filters(struct evlist *evlist);
627
628int auxtrace__process_event(struct perf_session *session, union perf_event *event,
629 struct perf_sample *sample, struct perf_tool *tool);
630void auxtrace__dump_auxtrace_sample(struct perf_session *session,
631 struct perf_sample *sample);
632int auxtrace__flush_events(struct perf_session *session, struct perf_tool *tool);
633void auxtrace__free_events(struct perf_session *session);
634void auxtrace__free(struct perf_session *session);
635bool auxtrace__evsel_is_auxtrace(struct perf_session *session,
636 struct evsel *evsel);
637
638#define ITRACE_HELP \
639" i[period]: synthesize instructions events\n" \
640" b: synthesize branches events (branch misses for Arm SPE)\n" \
641" c: synthesize branches events (calls only)\n" \
642" r: synthesize branches events (returns only)\n" \
643" x: synthesize transactions events\n" \
644" w: synthesize ptwrite events\n" \
645" p: synthesize power events\n" \
646" o: synthesize other events recorded due to the use\n" \
647" of aux-output (refer to perf record)\n" \
648" e[flags]: synthesize error events\n" \
649" each flag must be preceded by + or -\n" \
650" error flags are: o (overflow)\n" \
651" l (data lost)\n" \
652" d[flags]: create a debug log\n" \
653" each flag must be preceded by + or -\n" \
654" log flags are: a (all perf events)\n" \
655" f: synthesize first level cache events\n" \
656" m: synthesize last level cache events\n" \
657" t: synthesize TLB events\n" \
658" a: synthesize remote access events\n" \
659" g[len]: synthesize a call chain (use with i or x)\n" \
660" G[len]: synthesize a call chain on existing event records\n" \
661" l[len]: synthesize last branch entries (use with i or x)\n" \
662" L[len]: synthesize last branch entries on existing event records\n" \
663" sNUMBER: skip initial number of events\n" \
664" q: quicker (less detailed) decoding\n" \
665" PERIOD[ns|us|ms|i|t]: specify period to sample stream\n" \
666" concatenate multiple options. Default is ibxwpe or cewp\n"
667
668static inline
669void itrace_synth_opts__set_time_range(struct itrace_synth_opts *opts,
670 struct perf_time_interval *ptime_range,
671 int range_num)
672{
673 opts->ptime_range = ptime_range;
674 opts->range_num = range_num;
675}
676
677static inline
678void itrace_synth_opts__clear_time_range(struct itrace_synth_opts *opts)
679{
680 opts->ptime_range = NULL;
681 opts->range_num = 0;
682}
683
684#else
685#include "debug.h"
686
687static inline struct auxtrace_record *
688auxtrace_record__init(struct evlist *evlist __maybe_unused,
689 int *err)
690{
691 *err = 0;
692 return NULL;
693}
694
695static inline
696void auxtrace_record__free(struct auxtrace_record *itr __maybe_unused)
697{
698}
699
700static inline
701int auxtrace_record__options(struct auxtrace_record *itr __maybe_unused,
702 struct evlist *evlist __maybe_unused,
703 struct record_opts *opts __maybe_unused)
704{
705 return 0;
706}
707
708static inline
709int perf_event__process_auxtrace_info(struct perf_session *session __maybe_unused,
710 union perf_event *event __maybe_unused)
711{
712 return 0;
713}
714
715static inline
716s64 perf_event__process_auxtrace(struct perf_session *session __maybe_unused,
717 union perf_event *event __maybe_unused)
718{
719 return 0;
720}
721
722static inline
723int perf_event__process_auxtrace_error(struct perf_session *session __maybe_unused,
724 union perf_event *event __maybe_unused)
725{
726 return 0;
727}
728
729static inline
730void perf_session__auxtrace_error_inc(struct perf_session *session
731 __maybe_unused,
732 union perf_event *event
733 __maybe_unused)
734{
735}
736
737static inline
738void events_stats__auxtrace_error_warn(const struct events_stats *stats
739 __maybe_unused)
740{
741}
742
743static inline
744int itrace_do_parse_synth_opts(struct itrace_synth_opts *synth_opts __maybe_unused,
745 const char *str __maybe_unused, int unset __maybe_unused)
746{
747 pr_err("AUX area tracing not supported\n");
748 return -EINVAL;
749}
750
751static inline
752int itrace_parse_synth_opts(const struct option *opt __maybe_unused,
753 const char *str __maybe_unused,
754 int unset __maybe_unused)
755{
756 pr_err("AUX area tracing not supported\n");
757 return -EINVAL;
758}
759
760static inline
761int auxtrace_parse_snapshot_options(struct auxtrace_record *itr __maybe_unused,
762 struct record_opts *opts __maybe_unused,
763 const char *str)
764{
765 if (!str)
766 return 0;
767 pr_err("AUX area tracing not supported\n");
768 return -EINVAL;
769}
770
771static inline
772int auxtrace_parse_sample_options(struct auxtrace_record *itr __maybe_unused,
773 struct evlist *evlist __maybe_unused,
774 struct record_opts *opts __maybe_unused,
775 const char *str)
776{
777 if (!str)
778 return 0;
779 pr_err("AUX area tracing not supported\n");
780 return -EINVAL;
781}
782
783static inline
784void auxtrace_regroup_aux_output(struct evlist *evlist __maybe_unused)
785{
786}
787
788static inline
789int auxtrace__process_event(struct perf_session *session __maybe_unused,
790 union perf_event *event __maybe_unused,
791 struct perf_sample *sample __maybe_unused,
792 struct perf_tool *tool __maybe_unused)
793{
794 return 0;
795}
796
797static inline
798void auxtrace__dump_auxtrace_sample(struct perf_session *session __maybe_unused,
799 struct perf_sample *sample __maybe_unused)
800{
801}
802
803static inline
804int auxtrace__flush_events(struct perf_session *session __maybe_unused,
805 struct perf_tool *tool __maybe_unused)
806{
807 return 0;
808}
809
810static inline
811void auxtrace__free_events(struct perf_session *session __maybe_unused)
812{
813}
814
815static inline
816void auxtrace_cache__free(struct auxtrace_cache *auxtrace_cache __maybe_unused)
817{
818}
819
820static inline
821void auxtrace__free(struct perf_session *session __maybe_unused)
822{
823}
824
825static inline
826int auxtrace_index__write(int fd __maybe_unused,
827 struct list_head *head __maybe_unused)
828{
829 return -EINVAL;
830}
831
832static inline
833int auxtrace_index__process(int fd __maybe_unused,
834 u64 size __maybe_unused,
835 struct perf_session *session __maybe_unused,
836 bool needs_swap __maybe_unused)
837{
838 return -EINVAL;
839}
840
841static inline
842void auxtrace_index__free(struct list_head *head __maybe_unused)
843{
844}
845
846static inline
847bool auxtrace__evsel_is_auxtrace(struct perf_session *session __maybe_unused,
848 struct evsel *evsel __maybe_unused)
849{
850 return false;
851}
852
853static inline
854int auxtrace_parse_filters(struct evlist *evlist __maybe_unused)
855{
856 return 0;
857}
858
859int auxtrace_mmap__mmap(struct auxtrace_mmap *mm,
860 struct auxtrace_mmap_params *mp,
861 void *userpg, int fd);
862void auxtrace_mmap__munmap(struct auxtrace_mmap *mm);
863void auxtrace_mmap_params__init(struct auxtrace_mmap_params *mp,
864 off_t auxtrace_offset,
865 unsigned int auxtrace_pages,
866 bool auxtrace_overwrite);
867void auxtrace_mmap_params__set_idx(struct auxtrace_mmap_params *mp,
868 struct evlist *evlist, int idx,
869 bool per_cpu);
870
871#define ITRACE_HELP ""
872
873static inline
874void itrace_synth_opts__set_time_range(struct itrace_synth_opts *opts
875 __maybe_unused,
876 struct perf_time_interval *ptime_range
877 __maybe_unused,
878 int range_num __maybe_unused)
879{
880}
881
882static inline
883void itrace_synth_opts__clear_time_range(struct itrace_synth_opts *opts
884 __maybe_unused)
885{
886}
887
888#endif
889
890#endif
891