1
2
3
4
5
6
7
8
9#include <byteswap.h>
10#include <errno.h>
11#include <inttypes.h>
12#include <linux/bitops.h>
13#include <api/fs/fs.h>
14#include <api/fs/tracing_path.h>
15#include <traceevent/event-parse.h>
16#include <linux/hw_breakpoint.h>
17#include <linux/perf_event.h>
18#include <linux/compiler.h>
19#include <linux/err.h>
20#include <linux/zalloc.h>
21#include <sys/ioctl.h>
22#include <sys/resource.h>
23#include <sys/types.h>
24#include <dirent.h>
25#include <stdlib.h>
26#include <perf/evsel.h>
27#include "asm/bug.h"
28#include "bpf_counter.h"
29#include "callchain.h"
30#include "cgroup.h"
31#include "counts.h"
32#include "event.h"
33#include "evsel.h"
34#include "util/env.h"
35#include "util/evsel_config.h"
36#include "util/evsel_fprintf.h"
37#include "evlist.h"
38#include <perf/cpumap.h>
39#include "thread_map.h"
40#include "target.h"
41#include "perf_regs.h"
42#include "record.h"
43#include "debug.h"
44#include "trace-event.h"
45#include "stat.h"
46#include "string2.h"
47#include "memswap.h"
48#include "util.h"
49#include "hashmap.h"
50#include "pmu-hybrid.h"
51#include "../perf-sys.h"
52#include "util/parse-branch-options.h"
53#include <internal/xyarray.h>
54#include <internal/lib.h>
55
56#include <linux/ctype.h>
57
58struct perf_missing_features perf_missing_features;
59
60static clockid_t clockid;
61
62static int evsel__no_extra_init(struct evsel *evsel __maybe_unused)
63{
64 return 0;
65}
66
67void __weak test_attr__ready(void) { }
68
69static void evsel__no_extra_fini(struct evsel *evsel __maybe_unused)
70{
71}
72
73static struct {
74 size_t size;
75 int (*init)(struct evsel *evsel);
76 void (*fini)(struct evsel *evsel);
77} perf_evsel__object = {
78 .size = sizeof(struct evsel),
79 .init = evsel__no_extra_init,
80 .fini = evsel__no_extra_fini,
81};
82
83int evsel__object_config(size_t object_size, int (*init)(struct evsel *evsel),
84 void (*fini)(struct evsel *evsel))
85{
86
87 if (object_size == 0)
88 goto set_methods;
89
90 if (perf_evsel__object.size > object_size)
91 return -EINVAL;
92
93 perf_evsel__object.size = object_size;
94
95set_methods:
96 if (init != NULL)
97 perf_evsel__object.init = init;
98
99 if (fini != NULL)
100 perf_evsel__object.fini = fini;
101
102 return 0;
103}
104
105#define FD(e, x, y) (*(int *)xyarray__entry(e->core.fd, x, y))
106
107int __evsel__sample_size(u64 sample_type)
108{
109 u64 mask = sample_type & PERF_SAMPLE_MASK;
110 int size = 0;
111 int i;
112
113 for (i = 0; i < 64; i++) {
114 if (mask & (1ULL << i))
115 size++;
116 }
117
118 size *= sizeof(u64);
119
120 return size;
121}
122
123
124
125
126
127
128
129
130
131static int __perf_evsel__calc_id_pos(u64 sample_type)
132{
133 int idx = 0;
134
135 if (sample_type & PERF_SAMPLE_IDENTIFIER)
136 return 0;
137
138 if (!(sample_type & PERF_SAMPLE_ID))
139 return -1;
140
141 if (sample_type & PERF_SAMPLE_IP)
142 idx += 1;
143
144 if (sample_type & PERF_SAMPLE_TID)
145 idx += 1;
146
147 if (sample_type & PERF_SAMPLE_TIME)
148 idx += 1;
149
150 if (sample_type & PERF_SAMPLE_ADDR)
151 idx += 1;
152
153 return idx;
154}
155
156
157
158
159
160
161
162
163
164static int __perf_evsel__calc_is_pos(u64 sample_type)
165{
166 int idx = 1;
167
168 if (sample_type & PERF_SAMPLE_IDENTIFIER)
169 return 1;
170
171 if (!(sample_type & PERF_SAMPLE_ID))
172 return -1;
173
174 if (sample_type & PERF_SAMPLE_CPU)
175 idx += 1;
176
177 if (sample_type & PERF_SAMPLE_STREAM_ID)
178 idx += 1;
179
180 return idx;
181}
182
183void evsel__calc_id_pos(struct evsel *evsel)
184{
185 evsel->id_pos = __perf_evsel__calc_id_pos(evsel->core.attr.sample_type);
186 evsel->is_pos = __perf_evsel__calc_is_pos(evsel->core.attr.sample_type);
187}
188
189void __evsel__set_sample_bit(struct evsel *evsel,
190 enum perf_event_sample_format bit)
191{
192 if (!(evsel->core.attr.sample_type & bit)) {
193 evsel->core.attr.sample_type |= bit;
194 evsel->sample_size += sizeof(u64);
195 evsel__calc_id_pos(evsel);
196 }
197}
198
199void __evsel__reset_sample_bit(struct evsel *evsel,
200 enum perf_event_sample_format bit)
201{
202 if (evsel->core.attr.sample_type & bit) {
203 evsel->core.attr.sample_type &= ~bit;
204 evsel->sample_size -= sizeof(u64);
205 evsel__calc_id_pos(evsel);
206 }
207}
208
209void evsel__set_sample_id(struct evsel *evsel,
210 bool can_sample_identifier)
211{
212 if (can_sample_identifier) {
213 evsel__reset_sample_bit(evsel, ID);
214 evsel__set_sample_bit(evsel, IDENTIFIER);
215 } else {
216 evsel__set_sample_bit(evsel, ID);
217 }
218 evsel->core.attr.read_format |= PERF_FORMAT_ID;
219}
220
221
222
223
224
225
226
227
228
229bool evsel__is_function_event(struct evsel *evsel)
230{
231#define FUNCTION_EVENT "ftrace:function"
232
233 return evsel->name &&
234 !strncmp(FUNCTION_EVENT, evsel->name, sizeof(FUNCTION_EVENT));
235
236#undef FUNCTION_EVENT
237}
238
239void evsel__init(struct evsel *evsel,
240 struct perf_event_attr *attr, int idx)
241{
242 perf_evsel__init(&evsel->core, attr, idx);
243 evsel->tracking = !idx;
244 evsel->unit = "";
245 evsel->scale = 1.0;
246 evsel->max_events = ULONG_MAX;
247 evsel->evlist = NULL;
248 evsel->bpf_obj = NULL;
249 evsel->bpf_fd = -1;
250 INIT_LIST_HEAD(&evsel->config_terms);
251 INIT_LIST_HEAD(&evsel->bpf_counter_list);
252 perf_evsel__object.init(evsel);
253 evsel->sample_size = __evsel__sample_size(attr->sample_type);
254 evsel__calc_id_pos(evsel);
255 evsel->cmdline_group_boundary = false;
256 evsel->metric_expr = NULL;
257 evsel->metric_name = NULL;
258 evsel->metric_events = NULL;
259 evsel->per_pkg_mask = NULL;
260 evsel->collect_stat = false;
261 evsel->pmu_name = NULL;
262}
263
264struct evsel *evsel__new_idx(struct perf_event_attr *attr, int idx)
265{
266 struct evsel *evsel = zalloc(perf_evsel__object.size);
267
268 if (!evsel)
269 return NULL;
270 evsel__init(evsel, attr, idx);
271
272 if (evsel__is_bpf_output(evsel)) {
273 evsel->core.attr.sample_type |= (PERF_SAMPLE_RAW | PERF_SAMPLE_TIME |
274 PERF_SAMPLE_CPU | PERF_SAMPLE_PERIOD),
275 evsel->core.attr.sample_period = 1;
276 }
277
278 if (evsel__is_clock(evsel)) {
279
280
281
282
283 static const char *unit = "msec";
284
285 evsel->unit = unit;
286 evsel->scale = 1e-6;
287 }
288
289 return evsel;
290}
291
292static bool perf_event_can_profile_kernel(void)
293{
294 return perf_event_paranoid_check(1);
295}
296
297struct evsel *evsel__new_cycles(bool precise, __u32 type, __u64 config)
298{
299 struct perf_event_attr attr = {
300 .type = type,
301 .config = config,
302 .exclude_kernel = !perf_event_can_profile_kernel(),
303 };
304 struct evsel *evsel;
305
306 event_attr_init(&attr);
307
308 if (!precise)
309 goto new_event;
310
311
312
313
314
315new_event:
316 evsel = evsel__new(&attr);
317 if (evsel == NULL)
318 goto out;
319
320 evsel->precise_max = true;
321
322
323 if (asprintf(&evsel->name, "cycles%s%s%.*s",
324 (attr.precise_ip || attr.exclude_kernel) ? ":" : "",
325 attr.exclude_kernel ? "u" : "",
326 attr.precise_ip ? attr.precise_ip + 1 : 0, "ppp") < 0)
327 goto error_free;
328out:
329 return evsel;
330error_free:
331 evsel__delete(evsel);
332 evsel = NULL;
333 goto out;
334}
335
336int copy_config_terms(struct list_head *dst, struct list_head *src)
337{
338 struct evsel_config_term *pos, *tmp;
339
340 list_for_each_entry(pos, src, list) {
341 tmp = malloc(sizeof(*tmp));
342 if (tmp == NULL)
343 return -ENOMEM;
344
345 *tmp = *pos;
346 if (tmp->free_str) {
347 tmp->val.str = strdup(pos->val.str);
348 if (tmp->val.str == NULL) {
349 free(tmp);
350 return -ENOMEM;
351 }
352 }
353 list_add_tail(&tmp->list, dst);
354 }
355 return 0;
356}
357
358static int evsel__copy_config_terms(struct evsel *dst, struct evsel *src)
359{
360 return copy_config_terms(&dst->config_terms, &src->config_terms);
361}
362
363
364
365
366
367
368
369
370struct evsel *evsel__clone(struct evsel *orig)
371{
372 struct evsel *evsel;
373
374 BUG_ON(orig->core.fd);
375 BUG_ON(orig->counts);
376 BUG_ON(orig->priv);
377 BUG_ON(orig->per_pkg_mask);
378
379
380 if (orig->bpf_obj)
381 return NULL;
382
383 evsel = evsel__new(&orig->core.attr);
384 if (evsel == NULL)
385 return NULL;
386
387 evsel->core.cpus = perf_cpu_map__get(orig->core.cpus);
388 evsel->core.own_cpus = perf_cpu_map__get(orig->core.own_cpus);
389 evsel->core.threads = perf_thread_map__get(orig->core.threads);
390 evsel->core.nr_members = orig->core.nr_members;
391 evsel->core.system_wide = orig->core.system_wide;
392
393 if (orig->name) {
394 evsel->name = strdup(orig->name);
395 if (evsel->name == NULL)
396 goto out_err;
397 }
398 if (orig->group_name) {
399 evsel->group_name = strdup(orig->group_name);
400 if (evsel->group_name == NULL)
401 goto out_err;
402 }
403 if (orig->pmu_name) {
404 evsel->pmu_name = strdup(orig->pmu_name);
405 if (evsel->pmu_name == NULL)
406 goto out_err;
407 }
408 if (orig->filter) {
409 evsel->filter = strdup(orig->filter);
410 if (evsel->filter == NULL)
411 goto out_err;
412 }
413 evsel->cgrp = cgroup__get(orig->cgrp);
414 evsel->tp_format = orig->tp_format;
415 evsel->handler = orig->handler;
416 evsel->core.leader = orig->core.leader;
417
418 evsel->max_events = orig->max_events;
419 evsel->tool_event = orig->tool_event;
420 evsel->unit = orig->unit;
421 evsel->scale = orig->scale;
422 evsel->snapshot = orig->snapshot;
423 evsel->per_pkg = orig->per_pkg;
424 evsel->percore = orig->percore;
425 evsel->precise_max = orig->precise_max;
426 evsel->use_uncore_alias = orig->use_uncore_alias;
427 evsel->is_libpfm_event = orig->is_libpfm_event;
428
429 evsel->exclude_GH = orig->exclude_GH;
430 evsel->sample_read = orig->sample_read;
431 evsel->auto_merge_stats = orig->auto_merge_stats;
432 evsel->collect_stat = orig->collect_stat;
433 evsel->weak_group = orig->weak_group;
434 evsel->use_config_name = orig->use_config_name;
435
436 if (evsel__copy_config_terms(evsel, orig) < 0)
437 goto out_err;
438
439 return evsel;
440
441out_err:
442 evsel__delete(evsel);
443 return NULL;
444}
445
446
447
448
449struct evsel *evsel__newtp_idx(const char *sys, const char *name, int idx)
450{
451 struct evsel *evsel = zalloc(perf_evsel__object.size);
452 int err = -ENOMEM;
453
454 if (evsel == NULL) {
455 goto out_err;
456 } else {
457 struct perf_event_attr attr = {
458 .type = PERF_TYPE_TRACEPOINT,
459 .sample_type = (PERF_SAMPLE_RAW | PERF_SAMPLE_TIME |
460 PERF_SAMPLE_CPU | PERF_SAMPLE_PERIOD),
461 };
462
463 if (asprintf(&evsel->name, "%s:%s", sys, name) < 0)
464 goto out_free;
465
466 evsel->tp_format = trace_event__tp_format(sys, name);
467 if (IS_ERR(evsel->tp_format)) {
468 err = PTR_ERR(evsel->tp_format);
469 goto out_free;
470 }
471
472 event_attr_init(&attr);
473 attr.config = evsel->tp_format->id;
474 attr.sample_period = 1;
475 evsel__init(evsel, &attr, idx);
476 }
477
478 return evsel;
479
480out_free:
481 zfree(&evsel->name);
482 free(evsel);
483out_err:
484 return ERR_PTR(err);
485}
486
487const char *evsel__hw_names[PERF_COUNT_HW_MAX] = {
488 "cycles",
489 "instructions",
490 "cache-references",
491 "cache-misses",
492 "branches",
493 "branch-misses",
494 "bus-cycles",
495 "stalled-cycles-frontend",
496 "stalled-cycles-backend",
497 "ref-cycles",
498};
499
500char *evsel__bpf_counter_events;
501
502bool evsel__match_bpf_counter_events(const char *name)
503{
504 int name_len;
505 bool match;
506 char *ptr;
507
508 if (!evsel__bpf_counter_events)
509 return false;
510
511 ptr = strstr(evsel__bpf_counter_events, name);
512 name_len = strlen(name);
513
514
515 match = (ptr != NULL) &&
516 ((ptr == evsel__bpf_counter_events) || (*(ptr - 1) == ',')) &&
517 ((*(ptr + name_len) == ',') || (*(ptr + name_len) == '\0'));
518
519 return match;
520}
521
522static const char *__evsel__hw_name(u64 config)
523{
524 if (config < PERF_COUNT_HW_MAX && evsel__hw_names[config])
525 return evsel__hw_names[config];
526
527 return "unknown-hardware";
528}
529
530static int evsel__add_modifiers(struct evsel *evsel, char *bf, size_t size)
531{
532 int colon = 0, r = 0;
533 struct perf_event_attr *attr = &evsel->core.attr;
534 bool exclude_guest_default = false;
535
536#define MOD_PRINT(context, mod) do { \
537 if (!attr->exclude_##context) { \
538 if (!colon) colon = ++r; \
539 r += scnprintf(bf + r, size - r, "%c", mod); \
540 } } while(0)
541
542 if (attr->exclude_kernel || attr->exclude_user || attr->exclude_hv) {
543 MOD_PRINT(kernel, 'k');
544 MOD_PRINT(user, 'u');
545 MOD_PRINT(hv, 'h');
546 exclude_guest_default = true;
547 }
548
549 if (attr->precise_ip) {
550 if (!colon)
551 colon = ++r;
552 r += scnprintf(bf + r, size - r, "%.*s", attr->precise_ip, "ppp");
553 exclude_guest_default = true;
554 }
555
556 if (attr->exclude_host || attr->exclude_guest == exclude_guest_default) {
557 MOD_PRINT(host, 'H');
558 MOD_PRINT(guest, 'G');
559 }
560#undef MOD_PRINT
561 if (colon)
562 bf[colon - 1] = ':';
563 return r;
564}
565
566static int evsel__hw_name(struct evsel *evsel, char *bf, size_t size)
567{
568 int r = scnprintf(bf, size, "%s", __evsel__hw_name(evsel->core.attr.config));
569 return r + evsel__add_modifiers(evsel, bf + r, size - r);
570}
571
572const char *evsel__sw_names[PERF_COUNT_SW_MAX] = {
573 "cpu-clock",
574 "task-clock",
575 "page-faults",
576 "context-switches",
577 "cpu-migrations",
578 "minor-faults",
579 "major-faults",
580 "alignment-faults",
581 "emulation-faults",
582 "dummy",
583};
584
585static const char *__evsel__sw_name(u64 config)
586{
587 if (config < PERF_COUNT_SW_MAX && evsel__sw_names[config])
588 return evsel__sw_names[config];
589 return "unknown-software";
590}
591
592static int evsel__sw_name(struct evsel *evsel, char *bf, size_t size)
593{
594 int r = scnprintf(bf, size, "%s", __evsel__sw_name(evsel->core.attr.config));
595 return r + evsel__add_modifiers(evsel, bf + r, size - r);
596}
597
598static int __evsel__bp_name(char *bf, size_t size, u64 addr, u64 type)
599{
600 int r;
601
602 r = scnprintf(bf, size, "mem:0x%" PRIx64 ":", addr);
603
604 if (type & HW_BREAKPOINT_R)
605 r += scnprintf(bf + r, size - r, "r");
606
607 if (type & HW_BREAKPOINT_W)
608 r += scnprintf(bf + r, size - r, "w");
609
610 if (type & HW_BREAKPOINT_X)
611 r += scnprintf(bf + r, size - r, "x");
612
613 return r;
614}
615
616static int evsel__bp_name(struct evsel *evsel, char *bf, size_t size)
617{
618 struct perf_event_attr *attr = &evsel->core.attr;
619 int r = __evsel__bp_name(bf, size, attr->bp_addr, attr->bp_type);
620 return r + evsel__add_modifiers(evsel, bf + r, size - r);
621}
622
623const char *evsel__hw_cache[PERF_COUNT_HW_CACHE_MAX][EVSEL__MAX_ALIASES] = {
624 { "L1-dcache", "l1-d", "l1d", "L1-data", },
625 { "L1-icache", "l1-i", "l1i", "L1-instruction", },
626 { "LLC", "L2", },
627 { "dTLB", "d-tlb", "Data-TLB", },
628 { "iTLB", "i-tlb", "Instruction-TLB", },
629 { "branch", "branches", "bpu", "btb", "bpc", },
630 { "node", },
631};
632
633const char *evsel__hw_cache_op[PERF_COUNT_HW_CACHE_OP_MAX][EVSEL__MAX_ALIASES] = {
634 { "load", "loads", "read", },
635 { "store", "stores", "write", },
636 { "prefetch", "prefetches", "speculative-read", "speculative-load", },
637};
638
639const char *evsel__hw_cache_result[PERF_COUNT_HW_CACHE_RESULT_MAX][EVSEL__MAX_ALIASES] = {
640 { "refs", "Reference", "ops", "access", },
641 { "misses", "miss", },
642};
643
644#define C(x) PERF_COUNT_HW_CACHE_##x
645#define CACHE_READ (1 << C(OP_READ))
646#define CACHE_WRITE (1 << C(OP_WRITE))
647#define CACHE_PREFETCH (1 << C(OP_PREFETCH))
648#define COP(x) (1 << x)
649
650
651
652
653
654
655static unsigned long evsel__hw_cache_stat[C(MAX)] = {
656 [C(L1D)] = (CACHE_READ | CACHE_WRITE | CACHE_PREFETCH),
657 [C(L1I)] = (CACHE_READ | CACHE_PREFETCH),
658 [C(LL)] = (CACHE_READ | CACHE_WRITE | CACHE_PREFETCH),
659 [C(DTLB)] = (CACHE_READ | CACHE_WRITE | CACHE_PREFETCH),
660 [C(ITLB)] = (CACHE_READ),
661 [C(BPU)] = (CACHE_READ),
662 [C(NODE)] = (CACHE_READ | CACHE_WRITE | CACHE_PREFETCH),
663};
664
665bool evsel__is_cache_op_valid(u8 type, u8 op)
666{
667 if (evsel__hw_cache_stat[type] & COP(op))
668 return true;
669 else
670 return false;
671}
672
673int __evsel__hw_cache_type_op_res_name(u8 type, u8 op, u8 result, char *bf, size_t size)
674{
675 if (result) {
676 return scnprintf(bf, size, "%s-%s-%s", evsel__hw_cache[type][0],
677 evsel__hw_cache_op[op][0],
678 evsel__hw_cache_result[result][0]);
679 }
680
681 return scnprintf(bf, size, "%s-%s", evsel__hw_cache[type][0],
682 evsel__hw_cache_op[op][1]);
683}
684
685static int __evsel__hw_cache_name(u64 config, char *bf, size_t size)
686{
687 u8 op, result, type = (config >> 0) & 0xff;
688 const char *err = "unknown-ext-hardware-cache-type";
689
690 if (type >= PERF_COUNT_HW_CACHE_MAX)
691 goto out_err;
692
693 op = (config >> 8) & 0xff;
694 err = "unknown-ext-hardware-cache-op";
695 if (op >= PERF_COUNT_HW_CACHE_OP_MAX)
696 goto out_err;
697
698 result = (config >> 16) & 0xff;
699 err = "unknown-ext-hardware-cache-result";
700 if (result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
701 goto out_err;
702
703 err = "invalid-cache";
704 if (!evsel__is_cache_op_valid(type, op))
705 goto out_err;
706
707 return __evsel__hw_cache_type_op_res_name(type, op, result, bf, size);
708out_err:
709 return scnprintf(bf, size, "%s", err);
710}
711
712static int evsel__hw_cache_name(struct evsel *evsel, char *bf, size_t size)
713{
714 int ret = __evsel__hw_cache_name(evsel->core.attr.config, bf, size);
715 return ret + evsel__add_modifiers(evsel, bf + ret, size - ret);
716}
717
718static int evsel__raw_name(struct evsel *evsel, char *bf, size_t size)
719{
720 int ret = scnprintf(bf, size, "raw 0x%" PRIx64, evsel->core.attr.config);
721 return ret + evsel__add_modifiers(evsel, bf + ret, size - ret);
722}
723
724static int evsel__tool_name(char *bf, size_t size)
725{
726 int ret = scnprintf(bf, size, "duration_time");
727 return ret;
728}
729
730const char *evsel__name(struct evsel *evsel)
731{
732 char bf[128];
733
734 if (!evsel)
735 goto out_unknown;
736
737 if (evsel->name)
738 return evsel->name;
739
740 switch (evsel->core.attr.type) {
741 case PERF_TYPE_RAW:
742 evsel__raw_name(evsel, bf, sizeof(bf));
743 break;
744
745 case PERF_TYPE_HARDWARE:
746 evsel__hw_name(evsel, bf, sizeof(bf));
747 break;
748
749 case PERF_TYPE_HW_CACHE:
750 evsel__hw_cache_name(evsel, bf, sizeof(bf));
751 break;
752
753 case PERF_TYPE_SOFTWARE:
754 if (evsel->tool_event)
755 evsel__tool_name(bf, sizeof(bf));
756 else
757 evsel__sw_name(evsel, bf, sizeof(bf));
758 break;
759
760 case PERF_TYPE_TRACEPOINT:
761 scnprintf(bf, sizeof(bf), "%s", "unknown tracepoint");
762 break;
763
764 case PERF_TYPE_BREAKPOINT:
765 evsel__bp_name(evsel, bf, sizeof(bf));
766 break;
767
768 default:
769 scnprintf(bf, sizeof(bf), "unknown attr type: %d",
770 evsel->core.attr.type);
771 break;
772 }
773
774 evsel->name = strdup(bf);
775
776 if (evsel->name)
777 return evsel->name;
778out_unknown:
779 return "unknown";
780}
781
782const char *evsel__group_name(struct evsel *evsel)
783{
784 return evsel->group_name ?: "anon group";
785}
786
787
788
789
790
791
792
793
794
795
796
797int evsel__group_desc(struct evsel *evsel, char *buf, size_t size)
798{
799 int ret = 0;
800 struct evsel *pos;
801 const char *group_name = evsel__group_name(evsel);
802
803 if (!evsel->forced_leader)
804 ret = scnprintf(buf, size, "%s { ", group_name);
805
806 ret += scnprintf(buf + ret, size - ret, "%s", evsel__name(evsel));
807
808 for_each_group_member(pos, evsel)
809 ret += scnprintf(buf + ret, size - ret, ", %s", evsel__name(pos));
810
811 if (!evsel->forced_leader)
812 ret += scnprintf(buf + ret, size - ret, " }");
813
814 return ret;
815}
816
817static void __evsel__config_callchain(struct evsel *evsel, struct record_opts *opts,
818 struct callchain_param *param)
819{
820 bool function = evsel__is_function_event(evsel);
821 struct perf_event_attr *attr = &evsel->core.attr;
822
823 evsel__set_sample_bit(evsel, CALLCHAIN);
824
825 attr->sample_max_stack = param->max_stack;
826
827 if (opts->kernel_callchains)
828 attr->exclude_callchain_user = 1;
829 if (opts->user_callchains)
830 attr->exclude_callchain_kernel = 1;
831 if (param->record_mode == CALLCHAIN_LBR) {
832 if (!opts->branch_stack) {
833 if (attr->exclude_user) {
834 pr_warning("LBR callstack option is only available "
835 "to get user callchain information. "
836 "Falling back to framepointers.\n");
837 } else {
838 evsel__set_sample_bit(evsel, BRANCH_STACK);
839 attr->branch_sample_type = PERF_SAMPLE_BRANCH_USER |
840 PERF_SAMPLE_BRANCH_CALL_STACK |
841 PERF_SAMPLE_BRANCH_NO_CYCLES |
842 PERF_SAMPLE_BRANCH_NO_FLAGS |
843 PERF_SAMPLE_BRANCH_HW_INDEX;
844 }
845 } else
846 pr_warning("Cannot use LBR callstack with branch stack. "
847 "Falling back to framepointers.\n");
848 }
849
850 if (param->record_mode == CALLCHAIN_DWARF) {
851 if (!function) {
852 evsel__set_sample_bit(evsel, REGS_USER);
853 evsel__set_sample_bit(evsel, STACK_USER);
854 if (opts->sample_user_regs && DWARF_MINIMAL_REGS != PERF_REGS_MASK) {
855 attr->sample_regs_user |= DWARF_MINIMAL_REGS;
856 pr_warning("WARNING: The use of --call-graph=dwarf may require all the user registers, "
857 "specifying a subset with --user-regs may render DWARF unwinding unreliable, "
858 "so the minimal registers set (IP, SP) is explicitly forced.\n");
859 } else {
860 attr->sample_regs_user |= PERF_REGS_MASK;
861 }
862 attr->sample_stack_user = param->dump_size;
863 attr->exclude_callchain_user = 1;
864 } else {
865 pr_info("Cannot use DWARF unwind for function trace event,"
866 " falling back to framepointers.\n");
867 }
868 }
869
870 if (function) {
871 pr_info("Disabling user space callchains for function trace event.\n");
872 attr->exclude_callchain_user = 1;
873 }
874}
875
876void evsel__config_callchain(struct evsel *evsel, struct record_opts *opts,
877 struct callchain_param *param)
878{
879 if (param->enabled)
880 return __evsel__config_callchain(evsel, opts, param);
881}
882
883static void evsel__reset_callgraph(struct evsel *evsel, struct callchain_param *param)
884{
885 struct perf_event_attr *attr = &evsel->core.attr;
886
887 evsel__reset_sample_bit(evsel, CALLCHAIN);
888 if (param->record_mode == CALLCHAIN_LBR) {
889 evsel__reset_sample_bit(evsel, BRANCH_STACK);
890 attr->branch_sample_type &= ~(PERF_SAMPLE_BRANCH_USER |
891 PERF_SAMPLE_BRANCH_CALL_STACK |
892 PERF_SAMPLE_BRANCH_HW_INDEX);
893 }
894 if (param->record_mode == CALLCHAIN_DWARF) {
895 evsel__reset_sample_bit(evsel, REGS_USER);
896 evsel__reset_sample_bit(evsel, STACK_USER);
897 }
898}
899
900static void evsel__apply_config_terms(struct evsel *evsel,
901 struct record_opts *opts, bool track)
902{
903 struct evsel_config_term *term;
904 struct list_head *config_terms = &evsel->config_terms;
905 struct perf_event_attr *attr = &evsel->core.attr;
906
907 struct callchain_param param = {
908 .record_mode = callchain_param.record_mode,
909 };
910 u32 dump_size = 0;
911 int max_stack = 0;
912 const char *callgraph_buf = NULL;
913
914 list_for_each_entry(term, config_terms, list) {
915 switch (term->type) {
916 case EVSEL__CONFIG_TERM_PERIOD:
917 if (!(term->weak && opts->user_interval != ULLONG_MAX)) {
918 attr->sample_period = term->val.period;
919 attr->freq = 0;
920 evsel__reset_sample_bit(evsel, PERIOD);
921 }
922 break;
923 case EVSEL__CONFIG_TERM_FREQ:
924 if (!(term->weak && opts->user_freq != UINT_MAX)) {
925 attr->sample_freq = term->val.freq;
926 attr->freq = 1;
927 evsel__set_sample_bit(evsel, PERIOD);
928 }
929 break;
930 case EVSEL__CONFIG_TERM_TIME:
931 if (term->val.time)
932 evsel__set_sample_bit(evsel, TIME);
933 else
934 evsel__reset_sample_bit(evsel, TIME);
935 break;
936 case EVSEL__CONFIG_TERM_CALLGRAPH:
937 callgraph_buf = term->val.str;
938 break;
939 case EVSEL__CONFIG_TERM_BRANCH:
940 if (term->val.str && strcmp(term->val.str, "no")) {
941 evsel__set_sample_bit(evsel, BRANCH_STACK);
942 parse_branch_str(term->val.str,
943 &attr->branch_sample_type);
944 } else
945 evsel__reset_sample_bit(evsel, BRANCH_STACK);
946 break;
947 case EVSEL__CONFIG_TERM_STACK_USER:
948 dump_size = term->val.stack_user;
949 break;
950 case EVSEL__CONFIG_TERM_MAX_STACK:
951 max_stack = term->val.max_stack;
952 break;
953 case EVSEL__CONFIG_TERM_MAX_EVENTS:
954 evsel->max_events = term->val.max_events;
955 break;
956 case EVSEL__CONFIG_TERM_INHERIT:
957
958
959
960
961
962
963 attr->inherit = term->val.inherit ? 1 : 0;
964 break;
965 case EVSEL__CONFIG_TERM_OVERWRITE:
966 attr->write_backward = term->val.overwrite ? 1 : 0;
967 break;
968 case EVSEL__CONFIG_TERM_DRV_CFG:
969 break;
970 case EVSEL__CONFIG_TERM_PERCORE:
971 break;
972 case EVSEL__CONFIG_TERM_AUX_OUTPUT:
973 attr->aux_output = term->val.aux_output ? 1 : 0;
974 break;
975 case EVSEL__CONFIG_TERM_AUX_SAMPLE_SIZE:
976
977 break;
978 case EVSEL__CONFIG_TERM_CFG_CHG:
979 break;
980 default:
981 break;
982 }
983 }
984
985
986 if ((callgraph_buf != NULL) || (dump_size > 0) || max_stack) {
987 bool sample_address = false;
988
989 if (max_stack) {
990 param.max_stack = max_stack;
991 if (callgraph_buf == NULL)
992 callgraph_buf = "fp";
993 }
994
995
996 if (callgraph_buf != NULL) {
997 if (!strcmp(callgraph_buf, "no")) {
998 param.enabled = false;
999 param.record_mode = CALLCHAIN_NONE;
1000 } else {
1001 param.enabled = true;
1002 if (parse_callchain_record(callgraph_buf, ¶m)) {
1003 pr_err("per-event callgraph setting for %s failed. "
1004 "Apply callgraph global setting for it\n",
1005 evsel->name);
1006 return;
1007 }
1008 if (param.record_mode == CALLCHAIN_DWARF)
1009 sample_address = true;
1010 }
1011 }
1012 if (dump_size > 0) {
1013 dump_size = round_up(dump_size, sizeof(u64));
1014 param.dump_size = dump_size;
1015 }
1016
1017
1018 if (callchain_param.enabled)
1019 evsel__reset_callgraph(evsel, &callchain_param);
1020
1021
1022 if (param.enabled) {
1023 if (sample_address) {
1024 evsel__set_sample_bit(evsel, ADDR);
1025 evsel__set_sample_bit(evsel, DATA_SRC);
1026 evsel->core.attr.mmap_data = track;
1027 }
1028 evsel__config_callchain(evsel, opts, ¶m);
1029 }
1030 }
1031}
1032
1033struct evsel_config_term *__evsel__get_config_term(struct evsel *evsel, enum evsel_term_type type)
1034{
1035 struct evsel_config_term *term, *found_term = NULL;
1036
1037 list_for_each_entry(term, &evsel->config_terms, list) {
1038 if (term->type == type)
1039 found_term = term;
1040 }
1041
1042 return found_term;
1043}
1044
1045void __weak arch_evsel__set_sample_weight(struct evsel *evsel)
1046{
1047 evsel__set_sample_bit(evsel, WEIGHT);
1048}
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078void evsel__config(struct evsel *evsel, struct record_opts *opts,
1079 struct callchain_param *callchain)
1080{
1081 struct evsel *leader = evsel__leader(evsel);
1082 struct perf_event_attr *attr = &evsel->core.attr;
1083 int track = evsel->tracking;
1084 bool per_cpu = opts->target.default_per_cpu && !opts->target.per_thread;
1085
1086 attr->sample_id_all = perf_missing_features.sample_id_all ? 0 : 1;
1087 attr->inherit = !opts->no_inherit;
1088 attr->write_backward = opts->overwrite ? 1 : 0;
1089
1090 evsel__set_sample_bit(evsel, IP);
1091 evsel__set_sample_bit(evsel, TID);
1092
1093 if (evsel->sample_read) {
1094 evsel__set_sample_bit(evsel, READ);
1095
1096
1097
1098
1099
1100 evsel__set_sample_id(evsel, false);
1101
1102
1103
1104
1105
1106 if (leader->core.nr_members > 1) {
1107 attr->read_format |= PERF_FORMAT_GROUP;
1108 attr->inherit = 0;
1109 }
1110 }
1111
1112
1113
1114
1115
1116 if (!attr->sample_period) {
1117 if (opts->freq) {
1118 attr->freq = 1;
1119 attr->sample_freq = opts->freq;
1120 } else {
1121 attr->sample_period = opts->default_interval;
1122 }
1123 }
1124
1125
1126
1127
1128 if (attr->freq)
1129 evsel__set_sample_bit(evsel, PERIOD);
1130
1131 if (opts->no_samples)
1132 attr->sample_freq = 0;
1133
1134 if (opts->inherit_stat) {
1135 evsel->core.attr.read_format |=
1136 PERF_FORMAT_TOTAL_TIME_ENABLED |
1137 PERF_FORMAT_TOTAL_TIME_RUNNING |
1138 PERF_FORMAT_ID;
1139 attr->inherit_stat = 1;
1140 }
1141
1142 if (opts->sample_address) {
1143 evsel__set_sample_bit(evsel, ADDR);
1144 attr->mmap_data = track;
1145 }
1146
1147
1148
1149
1150
1151
1152 if (evsel__is_function_event(evsel))
1153 evsel->core.attr.exclude_callchain_user = 1;
1154
1155 if (callchain && callchain->enabled && !evsel->no_aux_samples)
1156 evsel__config_callchain(evsel, opts, callchain);
1157
1158 if (opts->sample_intr_regs && !evsel->no_aux_samples &&
1159 !evsel__is_dummy_event(evsel)) {
1160 attr->sample_regs_intr = opts->sample_intr_regs;
1161 evsel__set_sample_bit(evsel, REGS_INTR);
1162 }
1163
1164 if (opts->sample_user_regs && !evsel->no_aux_samples &&
1165 !evsel__is_dummy_event(evsel)) {
1166 attr->sample_regs_user |= opts->sample_user_regs;
1167 evsel__set_sample_bit(evsel, REGS_USER);
1168 }
1169
1170 if (target__has_cpu(&opts->target) || opts->sample_cpu)
1171 evsel__set_sample_bit(evsel, CPU);
1172
1173
1174
1175
1176 if (opts->sample_time &&
1177 (!perf_missing_features.sample_id_all &&
1178 (!opts->no_inherit || target__has_cpu(&opts->target) || per_cpu ||
1179 opts->sample_time_set)))
1180 evsel__set_sample_bit(evsel, TIME);
1181
1182 if (opts->raw_samples && !evsel->no_aux_samples) {
1183 evsel__set_sample_bit(evsel, TIME);
1184 evsel__set_sample_bit(evsel, RAW);
1185 evsel__set_sample_bit(evsel, CPU);
1186 }
1187
1188 if (opts->sample_address)
1189 evsel__set_sample_bit(evsel, DATA_SRC);
1190
1191 if (opts->sample_phys_addr)
1192 evsel__set_sample_bit(evsel, PHYS_ADDR);
1193
1194 if (opts->no_buffering) {
1195 attr->watermark = 0;
1196 attr->wakeup_events = 1;
1197 }
1198 if (opts->branch_stack && !evsel->no_aux_samples) {
1199 evsel__set_sample_bit(evsel, BRANCH_STACK);
1200 attr->branch_sample_type = opts->branch_stack;
1201 }
1202
1203 if (opts->sample_weight)
1204 arch_evsel__set_sample_weight(evsel);
1205
1206 attr->task = track;
1207 attr->mmap = track;
1208 attr->mmap2 = track && !perf_missing_features.mmap2;
1209 attr->comm = track;
1210 attr->build_id = track && opts->build_id;
1211
1212
1213
1214
1215
1216 if (!opts->text_poke)
1217 attr->ksymbol = track && !perf_missing_features.ksymbol;
1218 attr->bpf_event = track && !opts->no_bpf_event && !perf_missing_features.bpf;
1219
1220 if (opts->record_namespaces)
1221 attr->namespaces = track;
1222
1223 if (opts->record_cgroup) {
1224 attr->cgroup = track && !perf_missing_features.cgroup;
1225 evsel__set_sample_bit(evsel, CGROUP);
1226 }
1227
1228 if (opts->sample_data_page_size)
1229 evsel__set_sample_bit(evsel, DATA_PAGE_SIZE);
1230
1231 if (opts->sample_code_page_size)
1232 evsel__set_sample_bit(evsel, CODE_PAGE_SIZE);
1233
1234 if (opts->record_switch_events)
1235 attr->context_switch = track;
1236
1237 if (opts->sample_transaction)
1238 evsel__set_sample_bit(evsel, TRANSACTION);
1239
1240 if (opts->running_time) {
1241 evsel->core.attr.read_format |=
1242 PERF_FORMAT_TOTAL_TIME_ENABLED |
1243 PERF_FORMAT_TOTAL_TIME_RUNNING;
1244 }
1245
1246
1247
1248
1249
1250
1251
1252 if (evsel__is_group_leader(evsel))
1253 attr->disabled = 1;
1254
1255
1256
1257
1258
1259 if (target__none(&opts->target) && evsel__is_group_leader(evsel) &&
1260 !opts->initial_delay)
1261 attr->enable_on_exec = 1;
1262
1263 if (evsel->immediate) {
1264 attr->disabled = 0;
1265 attr->enable_on_exec = 0;
1266 }
1267
1268 clockid = opts->clockid;
1269 if (opts->use_clockid) {
1270 attr->use_clockid = 1;
1271 attr->clockid = opts->clockid;
1272 }
1273
1274 if (evsel->precise_max)
1275 attr->precise_ip = 3;
1276
1277 if (opts->all_user) {
1278 attr->exclude_kernel = 1;
1279 attr->exclude_user = 0;
1280 }
1281
1282 if (opts->all_kernel) {
1283 attr->exclude_kernel = 0;
1284 attr->exclude_user = 1;
1285 }
1286
1287 if (evsel->core.own_cpus || evsel->unit)
1288 evsel->core.attr.read_format |= PERF_FORMAT_ID;
1289
1290
1291
1292
1293
1294 evsel__apply_config_terms(evsel, opts, track);
1295
1296 evsel->ignore_missing_thread = opts->ignore_missing_thread;
1297
1298
1299 if (opts->period_set) {
1300 if (opts->period)
1301 evsel__set_sample_bit(evsel, PERIOD);
1302 else
1303 evsel__reset_sample_bit(evsel, PERIOD);
1304 }
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314 if (evsel__is_dummy_event(evsel))
1315 evsel__reset_sample_bit(evsel, BRANCH_STACK);
1316}
1317
1318int evsel__set_filter(struct evsel *evsel, const char *filter)
1319{
1320 char *new_filter = strdup(filter);
1321
1322 if (new_filter != NULL) {
1323 free(evsel->filter);
1324 evsel->filter = new_filter;
1325 return 0;
1326 }
1327
1328 return -1;
1329}
1330
1331static int evsel__append_filter(struct evsel *evsel, const char *fmt, const char *filter)
1332{
1333 char *new_filter;
1334
1335 if (evsel->filter == NULL)
1336 return evsel__set_filter(evsel, filter);
1337
1338 if (asprintf(&new_filter, fmt, evsel->filter, filter) > 0) {
1339 free(evsel->filter);
1340 evsel->filter = new_filter;
1341 return 0;
1342 }
1343
1344 return -1;
1345}
1346
1347int evsel__append_tp_filter(struct evsel *evsel, const char *filter)
1348{
1349 return evsel__append_filter(evsel, "(%s) && (%s)", filter);
1350}
1351
1352int evsel__append_addr_filter(struct evsel *evsel, const char *filter)
1353{
1354 return evsel__append_filter(evsel, "%s,%s", filter);
1355}
1356
1357
1358int evsel__enable_cpu(struct evsel *evsel, int cpu)
1359{
1360 return perf_evsel__enable_cpu(&evsel->core, cpu);
1361}
1362
1363int evsel__enable(struct evsel *evsel)
1364{
1365 int err = perf_evsel__enable(&evsel->core);
1366
1367 if (!err)
1368 evsel->disabled = false;
1369 return err;
1370}
1371
1372
1373int evsel__disable_cpu(struct evsel *evsel, int cpu)
1374{
1375 return perf_evsel__disable_cpu(&evsel->core, cpu);
1376}
1377
1378int evsel__disable(struct evsel *evsel)
1379{
1380 int err = perf_evsel__disable(&evsel->core);
1381
1382
1383
1384
1385
1386
1387 if (!err)
1388 evsel->disabled = true;
1389
1390 return err;
1391}
1392
1393void free_config_terms(struct list_head *config_terms)
1394{
1395 struct evsel_config_term *term, *h;
1396
1397 list_for_each_entry_safe(term, h, config_terms, list) {
1398 list_del_init(&term->list);
1399 if (term->free_str)
1400 zfree(&term->val.str);
1401 free(term);
1402 }
1403}
1404
1405static void evsel__free_config_terms(struct evsel *evsel)
1406{
1407 free_config_terms(&evsel->config_terms);
1408}
1409
1410void evsel__exit(struct evsel *evsel)
1411{
1412 assert(list_empty(&evsel->core.node));
1413 assert(evsel->evlist == NULL);
1414 bpf_counter__destroy(evsel);
1415 evsel__free_counts(evsel);
1416 perf_evsel__free_fd(&evsel->core);
1417 perf_evsel__free_id(&evsel->core);
1418 evsel__free_config_terms(evsel);
1419 cgroup__put(evsel->cgrp);
1420 perf_cpu_map__put(evsel->core.cpus);
1421 perf_cpu_map__put(evsel->core.own_cpus);
1422 perf_thread_map__put(evsel->core.threads);
1423 zfree(&evsel->group_name);
1424 zfree(&evsel->name);
1425 zfree(&evsel->pmu_name);
1426 evsel__zero_per_pkg(evsel);
1427 hashmap__free(evsel->per_pkg_mask);
1428 evsel->per_pkg_mask = NULL;
1429 zfree(&evsel->metric_events);
1430 perf_evsel__object.fini(evsel);
1431}
1432
1433void evsel__delete(struct evsel *evsel)
1434{
1435 evsel__exit(evsel);
1436 free(evsel);
1437}
1438
1439void evsel__compute_deltas(struct evsel *evsel, int cpu, int thread,
1440 struct perf_counts_values *count)
1441{
1442 struct perf_counts_values tmp;
1443
1444 if (!evsel->prev_raw_counts)
1445 return;
1446
1447 if (cpu == -1) {
1448 tmp = evsel->prev_raw_counts->aggr;
1449 evsel->prev_raw_counts->aggr = *count;
1450 } else {
1451 tmp = *perf_counts(evsel->prev_raw_counts, cpu, thread);
1452 *perf_counts(evsel->prev_raw_counts, cpu, thread) = *count;
1453 }
1454
1455 count->val = count->val - tmp.val;
1456 count->ena = count->ena - tmp.ena;
1457 count->run = count->run - tmp.run;
1458}
1459
1460void perf_counts_values__scale(struct perf_counts_values *count,
1461 bool scale, s8 *pscaled)
1462{
1463 s8 scaled = 0;
1464
1465 if (scale) {
1466 if (count->run == 0) {
1467 scaled = -1;
1468 count->val = 0;
1469 } else if (count->run < count->ena) {
1470 scaled = 1;
1471 count->val = (u64)((double) count->val * count->ena / count->run);
1472 }
1473 }
1474
1475 if (pscaled)
1476 *pscaled = scaled;
1477}
1478
1479static int evsel__read_one(struct evsel *evsel, int cpu, int thread)
1480{
1481 struct perf_counts_values *count = perf_counts(evsel->counts, cpu, thread);
1482
1483 return perf_evsel__read(&evsel->core, cpu, thread, count);
1484}
1485
1486static void evsel__set_count(struct evsel *counter, int cpu, int thread, u64 val, u64 ena, u64 run)
1487{
1488 struct perf_counts_values *count;
1489
1490 count = perf_counts(counter->counts, cpu, thread);
1491
1492 count->val = val;
1493 count->ena = ena;
1494 count->run = run;
1495
1496 perf_counts__set_loaded(counter->counts, cpu, thread, true);
1497}
1498
1499static int evsel__process_group_data(struct evsel *leader, int cpu, int thread, u64 *data)
1500{
1501 u64 read_format = leader->core.attr.read_format;
1502 struct sample_read_value *v;
1503 u64 nr, ena = 0, run = 0, i;
1504
1505 nr = *data++;
1506
1507 if (nr != (u64) leader->core.nr_members)
1508 return -EINVAL;
1509
1510 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
1511 ena = *data++;
1512
1513 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
1514 run = *data++;
1515
1516 v = (struct sample_read_value *) data;
1517
1518 evsel__set_count(leader, cpu, thread, v[0].value, ena, run);
1519
1520 for (i = 1; i < nr; i++) {
1521 struct evsel *counter;
1522
1523 counter = evlist__id2evsel(leader->evlist, v[i].id);
1524 if (!counter)
1525 return -EINVAL;
1526
1527 evsel__set_count(counter, cpu, thread, v[i].value, ena, run);
1528 }
1529
1530 return 0;
1531}
1532
1533static int evsel__read_group(struct evsel *leader, int cpu, int thread)
1534{
1535 struct perf_stat_evsel *ps = leader->stats;
1536 u64 read_format = leader->core.attr.read_format;
1537 int size = perf_evsel__read_size(&leader->core);
1538 u64 *data = ps->group_data;
1539
1540 if (!(read_format & PERF_FORMAT_ID))
1541 return -EINVAL;
1542
1543 if (!evsel__is_group_leader(leader))
1544 return -EINVAL;
1545
1546 if (!data) {
1547 data = zalloc(size);
1548 if (!data)
1549 return -ENOMEM;
1550
1551 ps->group_data = data;
1552 }
1553
1554 if (FD(leader, cpu, thread) < 0)
1555 return -EINVAL;
1556
1557 if (readn(FD(leader, cpu, thread), data, size) <= 0)
1558 return -errno;
1559
1560 return evsel__process_group_data(leader, cpu, thread, data);
1561}
1562
1563int evsel__read_counter(struct evsel *evsel, int cpu, int thread)
1564{
1565 u64 read_format = evsel->core.attr.read_format;
1566
1567 if (read_format & PERF_FORMAT_GROUP)
1568 return evsel__read_group(evsel, cpu, thread);
1569
1570 return evsel__read_one(evsel, cpu, thread);
1571}
1572
1573int __evsel__read_on_cpu(struct evsel *evsel, int cpu, int thread, bool scale)
1574{
1575 struct perf_counts_values count;
1576 size_t nv = scale ? 3 : 1;
1577
1578 if (FD(evsel, cpu, thread) < 0)
1579 return -EINVAL;
1580
1581 if (evsel->counts == NULL && evsel__alloc_counts(evsel, cpu + 1, thread + 1) < 0)
1582 return -ENOMEM;
1583
1584 if (readn(FD(evsel, cpu, thread), &count, nv * sizeof(u64)) <= 0)
1585 return -errno;
1586
1587 evsel__compute_deltas(evsel, cpu, thread, &count);
1588 perf_counts_values__scale(&count, scale, NULL);
1589 *perf_counts(evsel->counts, cpu, thread) = count;
1590 return 0;
1591}
1592
1593static int evsel__match_other_cpu(struct evsel *evsel, struct evsel *other,
1594 int cpu)
1595{
1596 int cpuid;
1597
1598 cpuid = perf_cpu_map__cpu(evsel->core.cpus, cpu);
1599 return perf_cpu_map__idx(other->core.cpus, cpuid);
1600}
1601
1602static int evsel__hybrid_group_cpu(struct evsel *evsel, int cpu)
1603{
1604 struct evsel *leader = evsel__leader(evsel);
1605
1606 if ((evsel__is_hybrid(evsel) && !evsel__is_hybrid(leader)) ||
1607 (!evsel__is_hybrid(evsel) && evsel__is_hybrid(leader))) {
1608 return evsel__match_other_cpu(evsel, leader, cpu);
1609 }
1610
1611 return cpu;
1612}
1613
1614static int get_group_fd(struct evsel *evsel, int cpu, int thread)
1615{
1616 struct evsel *leader = evsel__leader(evsel);
1617 int fd;
1618
1619 if (evsel__is_group_leader(evsel))
1620 return -1;
1621
1622
1623
1624
1625
1626 BUG_ON(!leader->core.fd);
1627
1628 cpu = evsel__hybrid_group_cpu(evsel, cpu);
1629 if (cpu == -1)
1630 return -1;
1631
1632 fd = FD(leader, cpu, thread);
1633 BUG_ON(fd == -1);
1634
1635 return fd;
1636}
1637
1638static void evsel__remove_fd(struct evsel *pos, int nr_cpus, int nr_threads, int thread_idx)
1639{
1640 for (int cpu = 0; cpu < nr_cpus; cpu++)
1641 for (int thread = thread_idx; thread < nr_threads - 1; thread++)
1642 FD(pos, cpu, thread) = FD(pos, cpu, thread + 1);
1643}
1644
1645static int update_fds(struct evsel *evsel,
1646 int nr_cpus, int cpu_idx,
1647 int nr_threads, int thread_idx)
1648{
1649 struct evsel *pos;
1650
1651 if (cpu_idx >= nr_cpus || thread_idx >= nr_threads)
1652 return -EINVAL;
1653
1654 evlist__for_each_entry(evsel->evlist, pos) {
1655 nr_cpus = pos != evsel ? nr_cpus : cpu_idx;
1656
1657 evsel__remove_fd(pos, nr_cpus, nr_threads, thread_idx);
1658
1659
1660
1661
1662
1663 if (pos == evsel)
1664 break;
1665 }
1666 return 0;
1667}
1668
1669bool evsel__ignore_missing_thread(struct evsel *evsel,
1670 int nr_cpus, int cpu,
1671 struct perf_thread_map *threads,
1672 int thread, int err)
1673{
1674 pid_t ignore_pid = perf_thread_map__pid(threads, thread);
1675
1676 if (!evsel->ignore_missing_thread)
1677 return false;
1678
1679
1680 if (evsel->core.system_wide)
1681 return false;
1682
1683
1684 if (err != -ESRCH)
1685 return false;
1686
1687
1688 if (threads->nr == 1)
1689 return false;
1690
1691
1692
1693
1694
1695 if (update_fds(evsel, nr_cpus, cpu, threads->nr, thread))
1696 return false;
1697
1698 if (thread_map__remove(threads, thread))
1699 return false;
1700
1701 pr_warning("WARNING: Ignored open failure for pid %d\n",
1702 ignore_pid);
1703 return true;
1704}
1705
1706static int __open_attr__fprintf(FILE *fp, const char *name, const char *val,
1707 void *priv __maybe_unused)
1708{
1709 return fprintf(fp, " %-32s %s\n", name, val);
1710}
1711
1712static void display_attr(struct perf_event_attr *attr)
1713{
1714 if (verbose >= 2 || debug_peo_args) {
1715 fprintf(stderr, "%.60s\n", graph_dotted_line);
1716 fprintf(stderr, "perf_event_attr:\n");
1717 perf_event_attr__fprintf(stderr, attr, __open_attr__fprintf, NULL);
1718 fprintf(stderr, "%.60s\n", graph_dotted_line);
1719 }
1720}
1721
1722bool evsel__precise_ip_fallback(struct evsel *evsel)
1723{
1724
1725 if (!evsel->precise_max)
1726 return false;
1727
1728
1729
1730
1731
1732 if (!evsel->core.attr.precise_ip) {
1733 evsel->core.attr.precise_ip = evsel->precise_ip_original;
1734 return false;
1735 }
1736
1737 if (!evsel->precise_ip_original)
1738 evsel->precise_ip_original = evsel->core.attr.precise_ip;
1739
1740 evsel->core.attr.precise_ip--;
1741 pr_debug2_peo("decreasing precise_ip by one (%d)\n", evsel->core.attr.precise_ip);
1742 display_attr(&evsel->core.attr);
1743 return true;
1744}
1745
1746static struct perf_cpu_map *empty_cpu_map;
1747static struct perf_thread_map *empty_thread_map;
1748
1749static int __evsel__prepare_open(struct evsel *evsel, struct perf_cpu_map *cpus,
1750 struct perf_thread_map *threads)
1751{
1752 int nthreads;
1753
1754 if ((perf_missing_features.write_backward && evsel->core.attr.write_backward) ||
1755 (perf_missing_features.aux_output && evsel->core.attr.aux_output))
1756 return -EINVAL;
1757
1758 if (cpus == NULL) {
1759 if (empty_cpu_map == NULL) {
1760 empty_cpu_map = perf_cpu_map__dummy_new();
1761 if (empty_cpu_map == NULL)
1762 return -ENOMEM;
1763 }
1764
1765 cpus = empty_cpu_map;
1766 }
1767
1768 if (threads == NULL) {
1769 if (empty_thread_map == NULL) {
1770 empty_thread_map = thread_map__new_by_tid(-1);
1771 if (empty_thread_map == NULL)
1772 return -ENOMEM;
1773 }
1774
1775 threads = empty_thread_map;
1776 }
1777
1778 if (evsel->core.system_wide)
1779 nthreads = 1;
1780 else
1781 nthreads = threads->nr;
1782
1783 if (evsel->core.fd == NULL &&
1784 perf_evsel__alloc_fd(&evsel->core, cpus->nr, nthreads) < 0)
1785 return -ENOMEM;
1786
1787 evsel->open_flags = PERF_FLAG_FD_CLOEXEC;
1788 if (evsel->cgrp)
1789 evsel->open_flags |= PERF_FLAG_PID_CGROUP;
1790
1791 return 0;
1792}
1793
1794static void evsel__disable_missing_features(struct evsel *evsel)
1795{
1796 if (perf_missing_features.weight_struct) {
1797 evsel__set_sample_bit(evsel, WEIGHT);
1798 evsel__reset_sample_bit(evsel, WEIGHT_STRUCT);
1799 }
1800 if (perf_missing_features.clockid_wrong)
1801 evsel->core.attr.clockid = CLOCK_MONOTONIC;
1802 if (perf_missing_features.clockid) {
1803 evsel->core.attr.use_clockid = 0;
1804 evsel->core.attr.clockid = 0;
1805 }
1806 if (perf_missing_features.cloexec)
1807 evsel->open_flags &= ~(unsigned long)PERF_FLAG_FD_CLOEXEC;
1808 if (perf_missing_features.mmap2)
1809 evsel->core.attr.mmap2 = 0;
1810 if (perf_missing_features.exclude_guest)
1811 evsel->core.attr.exclude_guest = evsel->core.attr.exclude_host = 0;
1812 if (perf_missing_features.lbr_flags)
1813 evsel->core.attr.branch_sample_type &= ~(PERF_SAMPLE_BRANCH_NO_FLAGS |
1814 PERF_SAMPLE_BRANCH_NO_CYCLES);
1815 if (perf_missing_features.group_read && evsel->core.attr.inherit)
1816 evsel->core.attr.read_format &= ~(PERF_FORMAT_GROUP|PERF_FORMAT_ID);
1817 if (perf_missing_features.ksymbol)
1818 evsel->core.attr.ksymbol = 0;
1819 if (perf_missing_features.bpf)
1820 evsel->core.attr.bpf_event = 0;
1821 if (perf_missing_features.branch_hw_idx)
1822 evsel->core.attr.branch_sample_type &= ~PERF_SAMPLE_BRANCH_HW_INDEX;
1823 if (perf_missing_features.sample_id_all)
1824 evsel->core.attr.sample_id_all = 0;
1825}
1826
1827int evsel__prepare_open(struct evsel *evsel, struct perf_cpu_map *cpus,
1828 struct perf_thread_map *threads)
1829{
1830 int err;
1831
1832 err = __evsel__prepare_open(evsel, cpus, threads);
1833 if (err)
1834 return err;
1835
1836 evsel__disable_missing_features(evsel);
1837
1838 return err;
1839}
1840
1841bool evsel__detect_missing_features(struct evsel *evsel)
1842{
1843
1844
1845
1846
1847 if (!perf_missing_features.weight_struct &&
1848 (evsel->core.attr.sample_type & PERF_SAMPLE_WEIGHT_STRUCT)) {
1849 perf_missing_features.weight_struct = true;
1850 pr_debug2("switching off weight struct support\n");
1851 return true;
1852 } else if (!perf_missing_features.code_page_size &&
1853 (evsel->core.attr.sample_type & PERF_SAMPLE_CODE_PAGE_SIZE)) {
1854 perf_missing_features.code_page_size = true;
1855 pr_debug2_peo("Kernel has no PERF_SAMPLE_CODE_PAGE_SIZE support, bailing out\n");
1856 return false;
1857 } else if (!perf_missing_features.data_page_size &&
1858 (evsel->core.attr.sample_type & PERF_SAMPLE_DATA_PAGE_SIZE)) {
1859 perf_missing_features.data_page_size = true;
1860 pr_debug2_peo("Kernel has no PERF_SAMPLE_DATA_PAGE_SIZE support, bailing out\n");
1861 return false;
1862 } else if (!perf_missing_features.cgroup && evsel->core.attr.cgroup) {
1863 perf_missing_features.cgroup = true;
1864 pr_debug2_peo("Kernel has no cgroup sampling support, bailing out\n");
1865 return false;
1866 } else if (!perf_missing_features.branch_hw_idx &&
1867 (evsel->core.attr.branch_sample_type & PERF_SAMPLE_BRANCH_HW_INDEX)) {
1868 perf_missing_features.branch_hw_idx = true;
1869 pr_debug2("switching off branch HW index support\n");
1870 return true;
1871 } else if (!perf_missing_features.aux_output && evsel->core.attr.aux_output) {
1872 perf_missing_features.aux_output = true;
1873 pr_debug2_peo("Kernel has no attr.aux_output support, bailing out\n");
1874 return false;
1875 } else if (!perf_missing_features.bpf && evsel->core.attr.bpf_event) {
1876 perf_missing_features.bpf = true;
1877 pr_debug2_peo("switching off bpf_event\n");
1878 return true;
1879 } else if (!perf_missing_features.ksymbol && evsel->core.attr.ksymbol) {
1880 perf_missing_features.ksymbol = true;
1881 pr_debug2_peo("switching off ksymbol\n");
1882 return true;
1883 } else if (!perf_missing_features.write_backward && evsel->core.attr.write_backward) {
1884 perf_missing_features.write_backward = true;
1885 pr_debug2_peo("switching off write_backward\n");
1886 return false;
1887 } else if (!perf_missing_features.clockid_wrong && evsel->core.attr.use_clockid) {
1888 perf_missing_features.clockid_wrong = true;
1889 pr_debug2_peo("switching off clockid\n");
1890 return true;
1891 } else if (!perf_missing_features.clockid && evsel->core.attr.use_clockid) {
1892 perf_missing_features.clockid = true;
1893 pr_debug2_peo("switching off use_clockid\n");
1894 return true;
1895 } else if (!perf_missing_features.cloexec && (evsel->open_flags & PERF_FLAG_FD_CLOEXEC)) {
1896 perf_missing_features.cloexec = true;
1897 pr_debug2_peo("switching off cloexec flag\n");
1898 return true;
1899 } else if (!perf_missing_features.mmap2 && evsel->core.attr.mmap2) {
1900 perf_missing_features.mmap2 = true;
1901 pr_debug2_peo("switching off mmap2\n");
1902 return true;
1903 } else if (!perf_missing_features.exclude_guest &&
1904 (evsel->core.attr.exclude_guest || evsel->core.attr.exclude_host)) {
1905 perf_missing_features.exclude_guest = true;
1906 pr_debug2_peo("switching off exclude_guest, exclude_host\n");
1907 return true;
1908 } else if (!perf_missing_features.sample_id_all) {
1909 perf_missing_features.sample_id_all = true;
1910 pr_debug2_peo("switching off sample_id_all\n");
1911 return true;
1912 } else if (!perf_missing_features.lbr_flags &&
1913 (evsel->core.attr.branch_sample_type &
1914 (PERF_SAMPLE_BRANCH_NO_CYCLES |
1915 PERF_SAMPLE_BRANCH_NO_FLAGS))) {
1916 perf_missing_features.lbr_flags = true;
1917 pr_debug2_peo("switching off branch sample type no (cycles/flags)\n");
1918 return true;
1919 } else if (!perf_missing_features.group_read &&
1920 evsel->core.attr.inherit &&
1921 (evsel->core.attr.read_format & PERF_FORMAT_GROUP) &&
1922 evsel__is_group_leader(evsel)) {
1923 perf_missing_features.group_read = true;
1924 pr_debug2_peo("switching off group read\n");
1925 return true;
1926 } else {
1927 return false;
1928 }
1929}
1930
1931bool evsel__increase_rlimit(enum rlimit_action *set_rlimit)
1932{
1933 int old_errno;
1934 struct rlimit l;
1935
1936 if (*set_rlimit < INCREASED_MAX) {
1937 old_errno = errno;
1938
1939 if (getrlimit(RLIMIT_NOFILE, &l) == 0) {
1940 if (*set_rlimit == NO_CHANGE) {
1941 l.rlim_cur = l.rlim_max;
1942 } else {
1943 l.rlim_cur = l.rlim_max + 1000;
1944 l.rlim_max = l.rlim_cur;
1945 }
1946 if (setrlimit(RLIMIT_NOFILE, &l) == 0) {
1947 (*set_rlimit) += 1;
1948 errno = old_errno;
1949 return true;
1950 }
1951 }
1952 errno = old_errno;
1953 }
1954
1955 return false;
1956}
1957
1958static int evsel__open_cpu(struct evsel *evsel, struct perf_cpu_map *cpus,
1959 struct perf_thread_map *threads,
1960 int start_cpu, int end_cpu)
1961{
1962 int cpu, thread, nthreads;
1963 int pid = -1, err, old_errno;
1964 enum rlimit_action set_rlimit = NO_CHANGE;
1965
1966 err = __evsel__prepare_open(evsel, cpus, threads);
1967 if (err)
1968 return err;
1969
1970 if (cpus == NULL)
1971 cpus = empty_cpu_map;
1972
1973 if (threads == NULL)
1974 threads = empty_thread_map;
1975
1976 if (evsel->core.system_wide)
1977 nthreads = 1;
1978 else
1979 nthreads = threads->nr;
1980
1981 if (evsel->cgrp)
1982 pid = evsel->cgrp->fd;
1983
1984fallback_missing_features:
1985 evsel__disable_missing_features(evsel);
1986
1987 display_attr(&evsel->core.attr);
1988
1989 for (cpu = start_cpu; cpu < end_cpu; cpu++) {
1990
1991 for (thread = 0; thread < nthreads; thread++) {
1992 int fd, group_fd;
1993retry_open:
1994 if (thread >= nthreads)
1995 break;
1996
1997 if (!evsel->cgrp && !evsel->core.system_wide)
1998 pid = perf_thread_map__pid(threads, thread);
1999
2000 group_fd = get_group_fd(evsel, cpu, thread);
2001
2002 test_attr__ready();
2003
2004 pr_debug2_peo("sys_perf_event_open: pid %d cpu %d group_fd %d flags %#lx",
2005 pid, cpus->map[cpu], group_fd, evsel->open_flags);
2006
2007 fd = sys_perf_event_open(&evsel->core.attr, pid, cpus->map[cpu],
2008 group_fd, evsel->open_flags);
2009
2010 FD(evsel, cpu, thread) = fd;
2011
2012 if (fd < 0) {
2013 err = -errno;
2014
2015 pr_debug2_peo("\nsys_perf_event_open failed, error %d\n",
2016 err);
2017 goto try_fallback;
2018 }
2019
2020 bpf_counter__install_pe(evsel, cpu, fd);
2021
2022 if (unlikely(test_attr__enabled)) {
2023 test_attr__open(&evsel->core.attr, pid, cpus->map[cpu],
2024 fd, group_fd, evsel->open_flags);
2025 }
2026
2027 pr_debug2_peo(" = %d\n", fd);
2028
2029 if (evsel->bpf_fd >= 0) {
2030 int evt_fd = fd;
2031 int bpf_fd = evsel->bpf_fd;
2032
2033 err = ioctl(evt_fd,
2034 PERF_EVENT_IOC_SET_BPF,
2035 bpf_fd);
2036 if (err && errno != EEXIST) {
2037 pr_err("failed to attach bpf fd %d: %s\n",
2038 bpf_fd, strerror(errno));
2039 err = -EINVAL;
2040 goto out_close;
2041 }
2042 }
2043
2044 set_rlimit = NO_CHANGE;
2045
2046
2047
2048
2049
2050 if (perf_missing_features.clockid ||
2051 perf_missing_features.clockid_wrong) {
2052 err = -EINVAL;
2053 goto out_close;
2054 }
2055 }
2056 }
2057
2058 return 0;
2059
2060try_fallback:
2061 if (evsel__precise_ip_fallback(evsel))
2062 goto retry_open;
2063
2064 if (evsel__ignore_missing_thread(evsel, cpus->nr, cpu, threads, thread, err)) {
2065
2066 nthreads--;
2067
2068
2069 err = 0;
2070 goto retry_open;
2071 }
2072
2073
2074
2075
2076 if (err == -EMFILE && evsel__increase_rlimit(&set_rlimit))
2077 goto retry_open;
2078
2079 if (err != -EINVAL || cpu > 0 || thread > 0)
2080 goto out_close;
2081
2082 if (evsel__detect_missing_features(evsel))
2083 goto fallback_missing_features;
2084out_close:
2085 if (err)
2086 threads->err_thread = thread;
2087
2088 old_errno = errno;
2089 do {
2090 while (--thread >= 0) {
2091 if (FD(evsel, cpu, thread) >= 0)
2092 close(FD(evsel, cpu, thread));
2093 FD(evsel, cpu, thread) = -1;
2094 }
2095 thread = nthreads;
2096 } while (--cpu >= 0);
2097 errno = old_errno;
2098 return err;
2099}
2100
2101int evsel__open(struct evsel *evsel, struct perf_cpu_map *cpus,
2102 struct perf_thread_map *threads)
2103{
2104 return evsel__open_cpu(evsel, cpus, threads, 0, cpus ? cpus->nr : 1);
2105}
2106
2107void evsel__close(struct evsel *evsel)
2108{
2109 perf_evsel__close(&evsel->core);
2110 perf_evsel__free_id(&evsel->core);
2111}
2112
2113int evsel__open_per_cpu(struct evsel *evsel, struct perf_cpu_map *cpus, int cpu)
2114{
2115 if (cpu == -1)
2116 return evsel__open_cpu(evsel, cpus, NULL, 0,
2117 cpus ? cpus->nr : 1);
2118
2119 return evsel__open_cpu(evsel, cpus, NULL, cpu, cpu + 1);
2120}
2121
2122int evsel__open_per_thread(struct evsel *evsel, struct perf_thread_map *threads)
2123{
2124 return evsel__open(evsel, NULL, threads);
2125}
2126
2127static int perf_evsel__parse_id_sample(const struct evsel *evsel,
2128 const union perf_event *event,
2129 struct perf_sample *sample)
2130{
2131 u64 type = evsel->core.attr.sample_type;
2132 const __u64 *array = event->sample.array;
2133 bool swapped = evsel->needs_swap;
2134 union u64_swap u;
2135
2136 array += ((event->header.size -
2137 sizeof(event->header)) / sizeof(u64)) - 1;
2138
2139 if (type & PERF_SAMPLE_IDENTIFIER) {
2140 sample->id = *array;
2141 array--;
2142 }
2143
2144 if (type & PERF_SAMPLE_CPU) {
2145 u.val64 = *array;
2146 if (swapped) {
2147
2148 u.val64 = bswap_64(u.val64);
2149 u.val32[0] = bswap_32(u.val32[0]);
2150 }
2151
2152 sample->cpu = u.val32[0];
2153 array--;
2154 }
2155
2156 if (type & PERF_SAMPLE_STREAM_ID) {
2157 sample->stream_id = *array;
2158 array--;
2159 }
2160
2161 if (type & PERF_SAMPLE_ID) {
2162 sample->id = *array;
2163 array--;
2164 }
2165
2166 if (type & PERF_SAMPLE_TIME) {
2167 sample->time = *array;
2168 array--;
2169 }
2170
2171 if (type & PERF_SAMPLE_TID) {
2172 u.val64 = *array;
2173 if (swapped) {
2174
2175 u.val64 = bswap_64(u.val64);
2176 u.val32[0] = bswap_32(u.val32[0]);
2177 u.val32[1] = bswap_32(u.val32[1]);
2178 }
2179
2180 sample->pid = u.val32[0];
2181 sample->tid = u.val32[1];
2182 array--;
2183 }
2184
2185 return 0;
2186}
2187
2188static inline bool overflow(const void *endp, u16 max_size, const void *offset,
2189 u64 size)
2190{
2191 return size > max_size || offset + size > endp;
2192}
2193
2194#define OVERFLOW_CHECK(offset, size, max_size) \
2195 do { \
2196 if (overflow(endp, (max_size), (offset), (size))) \
2197 return -EFAULT; \
2198 } while (0)
2199
2200#define OVERFLOW_CHECK_u64(offset) \
2201 OVERFLOW_CHECK(offset, sizeof(u64), sizeof(u64))
2202
2203static int
2204perf_event__check_size(union perf_event *event, unsigned int sample_size)
2205{
2206
2207
2208
2209
2210
2211 if (sample_size + sizeof(event->header) > event->header.size)
2212 return -EFAULT;
2213
2214 return 0;
2215}
2216
2217void __weak arch_perf_parse_sample_weight(struct perf_sample *data,
2218 const __u64 *array,
2219 u64 type __maybe_unused)
2220{
2221 data->weight = *array;
2222}
2223
2224int evsel__parse_sample(struct evsel *evsel, union perf_event *event,
2225 struct perf_sample *data)
2226{
2227 u64 type = evsel->core.attr.sample_type;
2228 bool swapped = evsel->needs_swap;
2229 const __u64 *array;
2230 u16 max_size = event->header.size;
2231 const void *endp = (void *)event + max_size;
2232 u64 sz;
2233
2234
2235
2236
2237
2238 union u64_swap u;
2239
2240 memset(data, 0, sizeof(*data));
2241 data->cpu = data->pid = data->tid = -1;
2242 data->stream_id = data->id = data->time = -1ULL;
2243 data->period = evsel->core.attr.sample_period;
2244 data->cpumode = event->header.misc & PERF_RECORD_MISC_CPUMODE_MASK;
2245 data->misc = event->header.misc;
2246 data->id = -1ULL;
2247 data->data_src = PERF_MEM_DATA_SRC_NONE;
2248
2249 if (event->header.type != PERF_RECORD_SAMPLE) {
2250 if (!evsel->core.attr.sample_id_all)
2251 return 0;
2252 return perf_evsel__parse_id_sample(evsel, event, data);
2253 }
2254
2255 array = event->sample.array;
2256
2257 if (perf_event__check_size(event, evsel->sample_size))
2258 return -EFAULT;
2259
2260 if (type & PERF_SAMPLE_IDENTIFIER) {
2261 data->id = *array;
2262 array++;
2263 }
2264
2265 if (type & PERF_SAMPLE_IP) {
2266 data->ip = *array;
2267 array++;
2268 }
2269
2270 if (type & PERF_SAMPLE_TID) {
2271 u.val64 = *array;
2272 if (swapped) {
2273
2274 u.val64 = bswap_64(u.val64);
2275 u.val32[0] = bswap_32(u.val32[0]);
2276 u.val32[1] = bswap_32(u.val32[1]);
2277 }
2278
2279 data->pid = u.val32[0];
2280 data->tid = u.val32[1];
2281 array++;
2282 }
2283
2284 if (type & PERF_SAMPLE_TIME) {
2285 data->time = *array;
2286 array++;
2287 }
2288
2289 if (type & PERF_SAMPLE_ADDR) {
2290 data->addr = *array;
2291 array++;
2292 }
2293
2294 if (type & PERF_SAMPLE_ID) {
2295 data->id = *array;
2296 array++;
2297 }
2298
2299 if (type & PERF_SAMPLE_STREAM_ID) {
2300 data->stream_id = *array;
2301 array++;
2302 }
2303
2304 if (type & PERF_SAMPLE_CPU) {
2305
2306 u.val64 = *array;
2307 if (swapped) {
2308
2309 u.val64 = bswap_64(u.val64);
2310 u.val32[0] = bswap_32(u.val32[0]);
2311 }
2312
2313 data->cpu = u.val32[0];
2314 array++;
2315 }
2316
2317 if (type & PERF_SAMPLE_PERIOD) {
2318 data->period = *array;
2319 array++;
2320 }
2321
2322 if (type & PERF_SAMPLE_READ) {
2323 u64 read_format = evsel->core.attr.read_format;
2324
2325 OVERFLOW_CHECK_u64(array);
2326 if (read_format & PERF_FORMAT_GROUP)
2327 data->read.group.nr = *array;
2328 else
2329 data->read.one.value = *array;
2330
2331 array++;
2332
2333 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
2334 OVERFLOW_CHECK_u64(array);
2335 data->read.time_enabled = *array;
2336 array++;
2337 }
2338
2339 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
2340 OVERFLOW_CHECK_u64(array);
2341 data->read.time_running = *array;
2342 array++;
2343 }
2344
2345
2346 if (read_format & PERF_FORMAT_GROUP) {
2347 const u64 max_group_nr = UINT64_MAX /
2348 sizeof(struct sample_read_value);
2349
2350 if (data->read.group.nr > max_group_nr)
2351 return -EFAULT;
2352 sz = data->read.group.nr *
2353 sizeof(struct sample_read_value);
2354 OVERFLOW_CHECK(array, sz, max_size);
2355 data->read.group.values =
2356 (struct sample_read_value *)array;
2357 array = (void *)array + sz;
2358 } else {
2359 OVERFLOW_CHECK_u64(array);
2360 data->read.one.id = *array;
2361 array++;
2362 }
2363 }
2364
2365 if (type & PERF_SAMPLE_CALLCHAIN) {
2366 const u64 max_callchain_nr = UINT64_MAX / sizeof(u64);
2367
2368 OVERFLOW_CHECK_u64(array);
2369 data->callchain = (struct ip_callchain *)array++;
2370 if (data->callchain->nr > max_callchain_nr)
2371 return -EFAULT;
2372 sz = data->callchain->nr * sizeof(u64);
2373 OVERFLOW_CHECK(array, sz, max_size);
2374 array = (void *)array + sz;
2375 }
2376
2377 if (type & PERF_SAMPLE_RAW) {
2378 OVERFLOW_CHECK_u64(array);
2379 u.val64 = *array;
2380
2381
2382
2383
2384
2385
2386
2387 if (swapped) {
2388 u.val64 = bswap_64(u.val64);
2389 u.val32[0] = bswap_32(u.val32[0]);
2390 u.val32[1] = bswap_32(u.val32[1]);
2391 }
2392 data->raw_size = u.val32[0];
2393
2394
2395
2396
2397
2398 if (swapped)
2399 mem_bswap_64((void *) array, data->raw_size);
2400
2401 array = (void *)array + sizeof(u32);
2402
2403 OVERFLOW_CHECK(array, data->raw_size, max_size);
2404 data->raw_data = (void *)array;
2405 array = (void *)array + data->raw_size;
2406 }
2407
2408 if (type & PERF_SAMPLE_BRANCH_STACK) {
2409 const u64 max_branch_nr = UINT64_MAX /
2410 sizeof(struct branch_entry);
2411
2412 OVERFLOW_CHECK_u64(array);
2413 data->branch_stack = (struct branch_stack *)array++;
2414
2415 if (data->branch_stack->nr > max_branch_nr)
2416 return -EFAULT;
2417
2418 sz = data->branch_stack->nr * sizeof(struct branch_entry);
2419 if (evsel__has_branch_hw_idx(evsel))
2420 sz += sizeof(u64);
2421 else
2422 data->no_hw_idx = true;
2423 OVERFLOW_CHECK(array, sz, max_size);
2424 array = (void *)array + sz;
2425 }
2426
2427 if (type & PERF_SAMPLE_REGS_USER) {
2428 OVERFLOW_CHECK_u64(array);
2429 data->user_regs.abi = *array;
2430 array++;
2431
2432 if (data->user_regs.abi) {
2433 u64 mask = evsel->core.attr.sample_regs_user;
2434
2435 sz = hweight64(mask) * sizeof(u64);
2436 OVERFLOW_CHECK(array, sz, max_size);
2437 data->user_regs.mask = mask;
2438 data->user_regs.regs = (u64 *)array;
2439 array = (void *)array + sz;
2440 }
2441 }
2442
2443 if (type & PERF_SAMPLE_STACK_USER) {
2444 OVERFLOW_CHECK_u64(array);
2445 sz = *array++;
2446
2447 data->user_stack.offset = ((char *)(array - 1)
2448 - (char *) event);
2449
2450 if (!sz) {
2451 data->user_stack.size = 0;
2452 } else {
2453 OVERFLOW_CHECK(array, sz, max_size);
2454 data->user_stack.data = (char *)array;
2455 array = (void *)array + sz;
2456 OVERFLOW_CHECK_u64(array);
2457 data->user_stack.size = *array++;
2458 if (WARN_ONCE(data->user_stack.size > sz,
2459 "user stack dump failure\n"))
2460 return -EFAULT;
2461 }
2462 }
2463
2464 if (type & PERF_SAMPLE_WEIGHT_TYPE) {
2465 OVERFLOW_CHECK_u64(array);
2466 arch_perf_parse_sample_weight(data, array, type);
2467 array++;
2468 }
2469
2470 if (type & PERF_SAMPLE_DATA_SRC) {
2471 OVERFLOW_CHECK_u64(array);
2472 data->data_src = *array;
2473 array++;
2474 }
2475
2476 if (type & PERF_SAMPLE_TRANSACTION) {
2477 OVERFLOW_CHECK_u64(array);
2478 data->transaction = *array;
2479 array++;
2480 }
2481
2482 data->intr_regs.abi = PERF_SAMPLE_REGS_ABI_NONE;
2483 if (type & PERF_SAMPLE_REGS_INTR) {
2484 OVERFLOW_CHECK_u64(array);
2485 data->intr_regs.abi = *array;
2486 array++;
2487
2488 if (data->intr_regs.abi != PERF_SAMPLE_REGS_ABI_NONE) {
2489 u64 mask = evsel->core.attr.sample_regs_intr;
2490
2491 sz = hweight64(mask) * sizeof(u64);
2492 OVERFLOW_CHECK(array, sz, max_size);
2493 data->intr_regs.mask = mask;
2494 data->intr_regs.regs = (u64 *)array;
2495 array = (void *)array + sz;
2496 }
2497 }
2498
2499 data->phys_addr = 0;
2500 if (type & PERF_SAMPLE_PHYS_ADDR) {
2501 data->phys_addr = *array;
2502 array++;
2503 }
2504
2505 data->cgroup = 0;
2506 if (type & PERF_SAMPLE_CGROUP) {
2507 data->cgroup = *array;
2508 array++;
2509 }
2510
2511 data->data_page_size = 0;
2512 if (type & PERF_SAMPLE_DATA_PAGE_SIZE) {
2513 data->data_page_size = *array;
2514 array++;
2515 }
2516
2517 data->code_page_size = 0;
2518 if (type & PERF_SAMPLE_CODE_PAGE_SIZE) {
2519 data->code_page_size = *array;
2520 array++;
2521 }
2522
2523 if (type & PERF_SAMPLE_AUX) {
2524 OVERFLOW_CHECK_u64(array);
2525 sz = *array++;
2526
2527 OVERFLOW_CHECK(array, sz, max_size);
2528
2529 if (swapped)
2530 mem_bswap_64((char *)array, sz);
2531 data->aux_sample.size = sz;
2532 data->aux_sample.data = (char *)array;
2533 array = (void *)array + sz;
2534 }
2535
2536 return 0;
2537}
2538
2539int evsel__parse_sample_timestamp(struct evsel *evsel, union perf_event *event,
2540 u64 *timestamp)
2541{
2542 u64 type = evsel->core.attr.sample_type;
2543 const __u64 *array;
2544
2545 if (!(type & PERF_SAMPLE_TIME))
2546 return -1;
2547
2548 if (event->header.type != PERF_RECORD_SAMPLE) {
2549 struct perf_sample data = {
2550 .time = -1ULL,
2551 };
2552
2553 if (!evsel->core.attr.sample_id_all)
2554 return -1;
2555 if (perf_evsel__parse_id_sample(evsel, event, &data))
2556 return -1;
2557
2558 *timestamp = data.time;
2559 return 0;
2560 }
2561
2562 array = event->sample.array;
2563
2564 if (perf_event__check_size(event, evsel->sample_size))
2565 return -EFAULT;
2566
2567 if (type & PERF_SAMPLE_IDENTIFIER)
2568 array++;
2569
2570 if (type & PERF_SAMPLE_IP)
2571 array++;
2572
2573 if (type & PERF_SAMPLE_TID)
2574 array++;
2575
2576 if (type & PERF_SAMPLE_TIME)
2577 *timestamp = *array;
2578
2579 return 0;
2580}
2581
2582struct tep_format_field *evsel__field(struct evsel *evsel, const char *name)
2583{
2584 return tep_find_field(evsel->tp_format, name);
2585}
2586
2587void *evsel__rawptr(struct evsel *evsel, struct perf_sample *sample, const char *name)
2588{
2589 struct tep_format_field *field = evsel__field(evsel, name);
2590 int offset;
2591
2592 if (!field)
2593 return NULL;
2594
2595 offset = field->offset;
2596
2597 if (field->flags & TEP_FIELD_IS_DYNAMIC) {
2598 offset = *(int *)(sample->raw_data + field->offset);
2599 offset &= 0xffff;
2600 }
2601
2602 return sample->raw_data + offset;
2603}
2604
2605u64 format_field__intval(struct tep_format_field *field, struct perf_sample *sample,
2606 bool needs_swap)
2607{
2608 u64 value;
2609 void *ptr = sample->raw_data + field->offset;
2610
2611 switch (field->size) {
2612 case 1:
2613 return *(u8 *)ptr;
2614 case 2:
2615 value = *(u16 *)ptr;
2616 break;
2617 case 4:
2618 value = *(u32 *)ptr;
2619 break;
2620 case 8:
2621 memcpy(&value, ptr, sizeof(u64));
2622 break;
2623 default:
2624 return 0;
2625 }
2626
2627 if (!needs_swap)
2628 return value;
2629
2630 switch (field->size) {
2631 case 2:
2632 return bswap_16(value);
2633 case 4:
2634 return bswap_32(value);
2635 case 8:
2636 return bswap_64(value);
2637 default:
2638 return 0;
2639 }
2640
2641 return 0;
2642}
2643
2644u64 evsel__intval(struct evsel *evsel, struct perf_sample *sample, const char *name)
2645{
2646 struct tep_format_field *field = evsel__field(evsel, name);
2647
2648 if (!field)
2649 return 0;
2650
2651 return field ? format_field__intval(field, sample, evsel->needs_swap) : 0;
2652}
2653
2654bool evsel__fallback(struct evsel *evsel, int err, char *msg, size_t msgsize)
2655{
2656 int paranoid;
2657
2658 if ((err == ENOENT || err == ENXIO || err == ENODEV) &&
2659 evsel->core.attr.type == PERF_TYPE_HARDWARE &&
2660 evsel->core.attr.config == PERF_COUNT_HW_CPU_CYCLES) {
2661
2662
2663
2664
2665
2666
2667
2668
2669 scnprintf(msg, msgsize, "%s",
2670"The cycles event is not supported, trying to fall back to cpu-clock-ticks");
2671
2672 evsel->core.attr.type = PERF_TYPE_SOFTWARE;
2673 evsel->core.attr.config = PERF_COUNT_SW_CPU_CLOCK;
2674
2675 zfree(&evsel->name);
2676 return true;
2677 } else if (err == EACCES && !evsel->core.attr.exclude_kernel &&
2678 (paranoid = perf_event_paranoid()) > 1) {
2679 const char *name = evsel__name(evsel);
2680 char *new_name;
2681 const char *sep = ":";
2682
2683
2684 if (evsel->core.attr.exclude_user)
2685 return false;
2686
2687
2688 if (strchr(name, '/') ||
2689 (strchr(name, ':') && !evsel->is_libpfm_event))
2690 sep = "";
2691
2692 if (asprintf(&new_name, "%s%su", name, sep) < 0)
2693 return false;
2694
2695 if (evsel->name)
2696 free(evsel->name);
2697 evsel->name = new_name;
2698 scnprintf(msg, msgsize, "kernel.perf_event_paranoid=%d, trying "
2699 "to fall back to excluding kernel and hypervisor "
2700 " samples", paranoid);
2701 evsel->core.attr.exclude_kernel = 1;
2702 evsel->core.attr.exclude_hv = 1;
2703
2704 return true;
2705 }
2706
2707 return false;
2708}
2709
2710static bool find_process(const char *name)
2711{
2712 size_t len = strlen(name);
2713 DIR *dir;
2714 struct dirent *d;
2715 int ret = -1;
2716
2717 dir = opendir(procfs__mountpoint());
2718 if (!dir)
2719 return false;
2720
2721
2722 while (ret && (d = readdir(dir)) != NULL) {
2723 char path[PATH_MAX];
2724 char *data;
2725 size_t size;
2726
2727 if ((d->d_type != DT_DIR) ||
2728 !strcmp(".", d->d_name) ||
2729 !strcmp("..", d->d_name))
2730 continue;
2731
2732 scnprintf(path, sizeof(path), "%s/%s/comm",
2733 procfs__mountpoint(), d->d_name);
2734
2735 if (filename__read_str(path, &data, &size))
2736 continue;
2737
2738 ret = strncmp(name, data, len);
2739 free(data);
2740 }
2741
2742 closedir(dir);
2743 return ret ? false : true;
2744}
2745
2746int evsel__open_strerror(struct evsel *evsel, struct target *target,
2747 int err, char *msg, size_t size)
2748{
2749 char sbuf[STRERR_BUFSIZE];
2750 int printed = 0, enforced = 0;
2751
2752 switch (err) {
2753 case EPERM:
2754 case EACCES:
2755 printed += scnprintf(msg + printed, size - printed,
2756 "Access to performance monitoring and observability operations is limited.\n");
2757
2758 if (!sysfs__read_int("fs/selinux/enforce", &enforced)) {
2759 if (enforced) {
2760 printed += scnprintf(msg + printed, size - printed,
2761 "Enforced MAC policy settings (SELinux) can limit access to performance\n"
2762 "monitoring and observability operations. Inspect system audit records for\n"
2763 "more perf_event access control information and adjusting the policy.\n");
2764 }
2765 }
2766
2767 if (err == EPERM)
2768 printed += scnprintf(msg, size,
2769 "No permission to enable %s event.\n\n", evsel__name(evsel));
2770
2771 return scnprintf(msg + printed, size - printed,
2772 "Consider adjusting /proc/sys/kernel/perf_event_paranoid setting to open\n"
2773 "access to performance monitoring and observability operations for processes\n"
2774 "without CAP_PERFMON, CAP_SYS_PTRACE or CAP_SYS_ADMIN Linux capability.\n"
2775 "More information can be found at 'Perf events and tool security' document:\n"
2776 "https://www.kernel.org/doc/html/latest/admin-guide/perf-security.html\n"
2777 "perf_event_paranoid setting is %d:\n"
2778 " -1: Allow use of (almost) all events by all users\n"
2779 " Ignore mlock limit after perf_event_mlock_kb without CAP_IPC_LOCK\n"
2780 ">= 0: Disallow raw and ftrace function tracepoint access\n"
2781 ">= 1: Disallow CPU event access\n"
2782 ">= 2: Disallow kernel profiling\n"
2783 "To make the adjusted perf_event_paranoid setting permanent preserve it\n"
2784 "in /etc/sysctl.conf (e.g. kernel.perf_event_paranoid = <setting>)",
2785 perf_event_paranoid());
2786 case ENOENT:
2787 return scnprintf(msg, size, "The %s event is not supported.", evsel__name(evsel));
2788 case EMFILE:
2789 return scnprintf(msg, size, "%s",
2790 "Too many events are opened.\n"
2791 "Probably the maximum number of open file descriptors has been reached.\n"
2792 "Hint: Try again after reducing the number of events.\n"
2793 "Hint: Try increasing the limit with 'ulimit -n <limit>'");
2794 case ENOMEM:
2795 if (evsel__has_callchain(evsel) &&
2796 access("/proc/sys/kernel/perf_event_max_stack", F_OK) == 0)
2797 return scnprintf(msg, size,
2798 "Not enough memory to setup event with callchain.\n"
2799 "Hint: Try tweaking /proc/sys/kernel/perf_event_max_stack\n"
2800 "Hint: Current value: %d", sysctl__max_stack());
2801 break;
2802 case ENODEV:
2803 if (target->cpu_list)
2804 return scnprintf(msg, size, "%s",
2805 "No such device - did you specify an out-of-range profile CPU?");
2806 break;
2807 case EOPNOTSUPP:
2808 if (evsel->core.attr.aux_output)
2809 return scnprintf(msg, size,
2810 "%s: PMU Hardware doesn't support 'aux_output' feature",
2811 evsel__name(evsel));
2812 if (evsel->core.attr.sample_period != 0)
2813 return scnprintf(msg, size,
2814 "%s: PMU Hardware doesn't support sampling/overflow-interrupts. Try 'perf stat'",
2815 evsel__name(evsel));
2816 if (evsel->core.attr.precise_ip)
2817 return scnprintf(msg, size, "%s",
2818 "\'precise\' request may not be supported. Try removing 'p' modifier.");
2819#if defined(__i386__) || defined(__x86_64__)
2820 if (evsel->core.attr.type == PERF_TYPE_HARDWARE)
2821 return scnprintf(msg, size, "%s",
2822 "No hardware sampling interrupt available.\n");
2823#endif
2824 break;
2825 case EBUSY:
2826 if (find_process("oprofiled"))
2827 return scnprintf(msg, size,
2828 "The PMU counters are busy/taken by another profiler.\n"
2829 "We found oprofile daemon running, please stop it and try again.");
2830 break;
2831 case EINVAL:
2832 if (evsel->core.attr.sample_type & PERF_SAMPLE_CODE_PAGE_SIZE && perf_missing_features.code_page_size)
2833 return scnprintf(msg, size, "Asking for the code page size isn't supported by this kernel.");
2834 if (evsel->core.attr.sample_type & PERF_SAMPLE_DATA_PAGE_SIZE && perf_missing_features.data_page_size)
2835 return scnprintf(msg, size, "Asking for the data page size isn't supported by this kernel.");
2836 if (evsel->core.attr.write_backward && perf_missing_features.write_backward)
2837 return scnprintf(msg, size, "Reading from overwrite event is not supported by this kernel.");
2838 if (perf_missing_features.clockid)
2839 return scnprintf(msg, size, "clockid feature not supported.");
2840 if (perf_missing_features.clockid_wrong)
2841 return scnprintf(msg, size, "wrong clockid (%d).", clockid);
2842 if (perf_missing_features.aux_output)
2843 return scnprintf(msg, size, "The 'aux_output' feature is not supported, update the kernel.");
2844 break;
2845 case ENODATA:
2846 return scnprintf(msg, size, "Cannot collect data source with the load latency event alone. "
2847 "Please add an auxiliary event in front of the load latency event.");
2848 default:
2849 break;
2850 }
2851
2852 return scnprintf(msg, size,
2853 "The sys_perf_event_open() syscall returned with %d (%s) for event (%s).\n"
2854 "/bin/dmesg | grep -i perf may provide additional information.\n",
2855 err, str_error_r(err, sbuf, sizeof(sbuf)), evsel__name(evsel));
2856}
2857
2858struct perf_env *evsel__env(struct evsel *evsel)
2859{
2860 if (evsel && evsel->evlist)
2861 return evsel->evlist->env;
2862 return &perf_env;
2863}
2864
2865static int store_evsel_ids(struct evsel *evsel, struct evlist *evlist)
2866{
2867 int cpu, thread;
2868
2869 for (cpu = 0; cpu < xyarray__max_x(evsel->core.fd); cpu++) {
2870 for (thread = 0; thread < xyarray__max_y(evsel->core.fd);
2871 thread++) {
2872 int fd = FD(evsel, cpu, thread);
2873
2874 if (perf_evlist__id_add_fd(&evlist->core, &evsel->core,
2875 cpu, thread, fd) < 0)
2876 return -1;
2877 }
2878 }
2879
2880 return 0;
2881}
2882
2883int evsel__store_ids(struct evsel *evsel, struct evlist *evlist)
2884{
2885 struct perf_cpu_map *cpus = evsel->core.cpus;
2886 struct perf_thread_map *threads = evsel->core.threads;
2887
2888 if (perf_evsel__alloc_id(&evsel->core, cpus->nr, threads->nr))
2889 return -ENOMEM;
2890
2891 return store_evsel_ids(evsel, evlist);
2892}
2893
2894void evsel__zero_per_pkg(struct evsel *evsel)
2895{
2896 struct hashmap_entry *cur;
2897 size_t bkt;
2898
2899 if (evsel->per_pkg_mask) {
2900 hashmap__for_each_entry(evsel->per_pkg_mask, cur, bkt)
2901 free((char *)cur->key);
2902
2903 hashmap__clear(evsel->per_pkg_mask);
2904 }
2905}
2906
2907bool evsel__is_hybrid(struct evsel *evsel)
2908{
2909 return evsel->pmu_name && perf_pmu__is_hybrid(evsel->pmu_name);
2910}
2911
2912struct evsel *evsel__leader(struct evsel *evsel)
2913{
2914 return container_of(evsel->core.leader, struct evsel, core);
2915}
2916
2917bool evsel__has_leader(struct evsel *evsel, struct evsel *leader)
2918{
2919 return evsel->core.leader == &leader->core;
2920}
2921
2922bool evsel__is_leader(struct evsel *evsel)
2923{
2924 return evsel__has_leader(evsel, evsel);
2925}
2926
2927void evsel__set_leader(struct evsel *evsel, struct evsel *leader)
2928{
2929 evsel->core.leader = &leader->core;
2930}
2931