1
2
3
4
5
6
7
8
9#include <byteswap.h>
10#include <errno.h>
11#include <inttypes.h>
12#include <linux/bitops.h>
13#include <api/fs/fs.h>
14#include <api/fs/tracing_path.h>
15#include <traceevent/event-parse.h>
16#include <linux/hw_breakpoint.h>
17#include <linux/perf_event.h>
18#include <linux/compiler.h>
19#include <linux/err.h>
20#include <linux/zalloc.h>
21#include <sys/ioctl.h>
22#include <sys/resource.h>
23#include <sys/types.h>
24#include <dirent.h>
25#include <stdlib.h>
26#include <perf/evsel.h>
27#include "asm/bug.h"
28#include "bpf_counter.h"
29#include "callchain.h"
30#include "cgroup.h"
31#include "counts.h"
32#include "event.h"
33#include "evsel.h"
34#include "util/env.h"
35#include "util/evsel_config.h"
36#include "util/evsel_fprintf.h"
37#include "evlist.h"
38#include <perf/cpumap.h>
39#include "thread_map.h"
40#include "target.h"
41#include "perf_regs.h"
42#include "record.h"
43#include "debug.h"
44#include "trace-event.h"
45#include "stat.h"
46#include "string2.h"
47#include "memswap.h"
48#include "util.h"
49#include "hashmap.h"
50#include "pmu-hybrid.h"
51#include "../perf-sys.h"
52#include "util/parse-branch-options.h"
53#include <internal/xyarray.h>
54#include <internal/lib.h>
55
56#include <linux/ctype.h>
57
58struct perf_missing_features perf_missing_features;
59
60static clockid_t clockid;
61
62static int evsel__no_extra_init(struct evsel *evsel __maybe_unused)
63{
64 return 0;
65}
66
67void __weak test_attr__ready(void) { }
68
69static void evsel__no_extra_fini(struct evsel *evsel __maybe_unused)
70{
71}
72
73static struct {
74 size_t size;
75 int (*init)(struct evsel *evsel);
76 void (*fini)(struct evsel *evsel);
77} perf_evsel__object = {
78 .size = sizeof(struct evsel),
79 .init = evsel__no_extra_init,
80 .fini = evsel__no_extra_fini,
81};
82
83int evsel__object_config(size_t object_size, int (*init)(struct evsel *evsel),
84 void (*fini)(struct evsel *evsel))
85{
86
87 if (object_size == 0)
88 goto set_methods;
89
90 if (perf_evsel__object.size > object_size)
91 return -EINVAL;
92
93 perf_evsel__object.size = object_size;
94
95set_methods:
96 if (init != NULL)
97 perf_evsel__object.init = init;
98
99 if (fini != NULL)
100 perf_evsel__object.fini = fini;
101
102 return 0;
103}
104
105#define FD(e, x, y) (*(int *)xyarray__entry(e->core.fd, x, y))
106
107int __evsel__sample_size(u64 sample_type)
108{
109 u64 mask = sample_type & PERF_SAMPLE_MASK;
110 int size = 0;
111 int i;
112
113 for (i = 0; i < 64; i++) {
114 if (mask & (1ULL << i))
115 size++;
116 }
117
118 size *= sizeof(u64);
119
120 return size;
121}
122
123
124
125
126
127
128
129
130
131static int __perf_evsel__calc_id_pos(u64 sample_type)
132{
133 int idx = 0;
134
135 if (sample_type & PERF_SAMPLE_IDENTIFIER)
136 return 0;
137
138 if (!(sample_type & PERF_SAMPLE_ID))
139 return -1;
140
141 if (sample_type & PERF_SAMPLE_IP)
142 idx += 1;
143
144 if (sample_type & PERF_SAMPLE_TID)
145 idx += 1;
146
147 if (sample_type & PERF_SAMPLE_TIME)
148 idx += 1;
149
150 if (sample_type & PERF_SAMPLE_ADDR)
151 idx += 1;
152
153 return idx;
154}
155
156
157
158
159
160
161
162
163
164static int __perf_evsel__calc_is_pos(u64 sample_type)
165{
166 int idx = 1;
167
168 if (sample_type & PERF_SAMPLE_IDENTIFIER)
169 return 1;
170
171 if (!(sample_type & PERF_SAMPLE_ID))
172 return -1;
173
174 if (sample_type & PERF_SAMPLE_CPU)
175 idx += 1;
176
177 if (sample_type & PERF_SAMPLE_STREAM_ID)
178 idx += 1;
179
180 return idx;
181}
182
183void evsel__calc_id_pos(struct evsel *evsel)
184{
185 evsel->id_pos = __perf_evsel__calc_id_pos(evsel->core.attr.sample_type);
186 evsel->is_pos = __perf_evsel__calc_is_pos(evsel->core.attr.sample_type);
187}
188
189void __evsel__set_sample_bit(struct evsel *evsel,
190 enum perf_event_sample_format bit)
191{
192 if (!(evsel->core.attr.sample_type & bit)) {
193 evsel->core.attr.sample_type |= bit;
194 evsel->sample_size += sizeof(u64);
195 evsel__calc_id_pos(evsel);
196 }
197}
198
199void __evsel__reset_sample_bit(struct evsel *evsel,
200 enum perf_event_sample_format bit)
201{
202 if (evsel->core.attr.sample_type & bit) {
203 evsel->core.attr.sample_type &= ~bit;
204 evsel->sample_size -= sizeof(u64);
205 evsel__calc_id_pos(evsel);
206 }
207}
208
209void evsel__set_sample_id(struct evsel *evsel,
210 bool can_sample_identifier)
211{
212 if (can_sample_identifier) {
213 evsel__reset_sample_bit(evsel, ID);
214 evsel__set_sample_bit(evsel, IDENTIFIER);
215 } else {
216 evsel__set_sample_bit(evsel, ID);
217 }
218 evsel->core.attr.read_format |= PERF_FORMAT_ID;
219}
220
221
222
223
224
225
226
227
228
229bool evsel__is_function_event(struct evsel *evsel)
230{
231#define FUNCTION_EVENT "ftrace:function"
232
233 return evsel->name &&
234 !strncmp(FUNCTION_EVENT, evsel->name, sizeof(FUNCTION_EVENT));
235
236#undef FUNCTION_EVENT
237}
238
239void evsel__init(struct evsel *evsel,
240 struct perf_event_attr *attr, int idx)
241{
242 perf_evsel__init(&evsel->core, attr, idx);
243 evsel->tracking = !idx;
244 evsel->unit = "";
245 evsel->scale = 1.0;
246 evsel->max_events = ULONG_MAX;
247 evsel->evlist = NULL;
248 evsel->bpf_obj = NULL;
249 evsel->bpf_fd = -1;
250 INIT_LIST_HEAD(&evsel->config_terms);
251 INIT_LIST_HEAD(&evsel->bpf_counter_list);
252 perf_evsel__object.init(evsel);
253 evsel->sample_size = __evsel__sample_size(attr->sample_type);
254 evsel__calc_id_pos(evsel);
255 evsel->cmdline_group_boundary = false;
256 evsel->metric_expr = NULL;
257 evsel->metric_name = NULL;
258 evsel->metric_events = NULL;
259 evsel->per_pkg_mask = NULL;
260 evsel->collect_stat = false;
261 evsel->pmu_name = NULL;
262}
263
264struct evsel *evsel__new_idx(struct perf_event_attr *attr, int idx)
265{
266 struct evsel *evsel = zalloc(perf_evsel__object.size);
267
268 if (!evsel)
269 return NULL;
270 evsel__init(evsel, attr, idx);
271
272 if (evsel__is_bpf_output(evsel)) {
273 evsel->core.attr.sample_type |= (PERF_SAMPLE_RAW | PERF_SAMPLE_TIME |
274 PERF_SAMPLE_CPU | PERF_SAMPLE_PERIOD),
275 evsel->core.attr.sample_period = 1;
276 }
277
278 if (evsel__is_clock(evsel)) {
279
280
281
282
283 static const char *unit = "msec";
284
285 evsel->unit = unit;
286 evsel->scale = 1e-6;
287 }
288
289 return evsel;
290}
291
292static bool perf_event_can_profile_kernel(void)
293{
294 return perf_event_paranoid_check(1);
295}
296
297struct evsel *evsel__new_cycles(bool precise, __u32 type, __u64 config)
298{
299 struct perf_event_attr attr = {
300 .type = type,
301 .config = config,
302 .exclude_kernel = !perf_event_can_profile_kernel(),
303 };
304 struct evsel *evsel;
305
306 event_attr_init(&attr);
307
308 if (!precise)
309 goto new_event;
310
311
312
313
314
315new_event:
316 evsel = evsel__new(&attr);
317 if (evsel == NULL)
318 goto out;
319
320 evsel->precise_max = true;
321
322
323 if (asprintf(&evsel->name, "cycles%s%s%.*s",
324 (attr.precise_ip || attr.exclude_kernel) ? ":" : "",
325 attr.exclude_kernel ? "u" : "",
326 attr.precise_ip ? attr.precise_ip + 1 : 0, "ppp") < 0)
327 goto error_free;
328out:
329 return evsel;
330error_free:
331 evsel__delete(evsel);
332 evsel = NULL;
333 goto out;
334}
335
336static int evsel__copy_config_terms(struct evsel *dst, struct evsel *src)
337{
338 struct evsel_config_term *pos, *tmp;
339
340 list_for_each_entry(pos, &src->config_terms, list) {
341 tmp = malloc(sizeof(*tmp));
342 if (tmp == NULL)
343 return -ENOMEM;
344
345 *tmp = *pos;
346 if (tmp->free_str) {
347 tmp->val.str = strdup(pos->val.str);
348 if (tmp->val.str == NULL) {
349 free(tmp);
350 return -ENOMEM;
351 }
352 }
353 list_add_tail(&tmp->list, &dst->config_terms);
354 }
355 return 0;
356}
357
358
359
360
361
362
363
364
365struct evsel *evsel__clone(struct evsel *orig)
366{
367 struct evsel *evsel;
368
369 BUG_ON(orig->core.fd);
370 BUG_ON(orig->counts);
371 BUG_ON(orig->priv);
372 BUG_ON(orig->per_pkg_mask);
373
374
375 if (orig->bpf_obj)
376 return NULL;
377
378 evsel = evsel__new(&orig->core.attr);
379 if (evsel == NULL)
380 return NULL;
381
382 evsel->core.cpus = perf_cpu_map__get(orig->core.cpus);
383 evsel->core.own_cpus = perf_cpu_map__get(orig->core.own_cpus);
384 evsel->core.threads = perf_thread_map__get(orig->core.threads);
385 evsel->core.nr_members = orig->core.nr_members;
386 evsel->core.system_wide = orig->core.system_wide;
387
388 if (orig->name) {
389 evsel->name = strdup(orig->name);
390 if (evsel->name == NULL)
391 goto out_err;
392 }
393 if (orig->group_name) {
394 evsel->group_name = strdup(orig->group_name);
395 if (evsel->group_name == NULL)
396 goto out_err;
397 }
398 if (orig->pmu_name) {
399 evsel->pmu_name = strdup(orig->pmu_name);
400 if (evsel->pmu_name == NULL)
401 goto out_err;
402 }
403 if (orig->filter) {
404 evsel->filter = strdup(orig->filter);
405 if (evsel->filter == NULL)
406 goto out_err;
407 }
408 evsel->cgrp = cgroup__get(orig->cgrp);
409 evsel->tp_format = orig->tp_format;
410 evsel->handler = orig->handler;
411 evsel->core.leader = orig->core.leader;
412
413 evsel->max_events = orig->max_events;
414 evsel->tool_event = orig->tool_event;
415 evsel->unit = orig->unit;
416 evsel->scale = orig->scale;
417 evsel->snapshot = orig->snapshot;
418 evsel->per_pkg = orig->per_pkg;
419 evsel->percore = orig->percore;
420 evsel->precise_max = orig->precise_max;
421 evsel->use_uncore_alias = orig->use_uncore_alias;
422 evsel->is_libpfm_event = orig->is_libpfm_event;
423
424 evsel->exclude_GH = orig->exclude_GH;
425 evsel->sample_read = orig->sample_read;
426 evsel->auto_merge_stats = orig->auto_merge_stats;
427 evsel->collect_stat = orig->collect_stat;
428 evsel->weak_group = orig->weak_group;
429 evsel->use_config_name = orig->use_config_name;
430
431 if (evsel__copy_config_terms(evsel, orig) < 0)
432 goto out_err;
433
434 return evsel;
435
436out_err:
437 evsel__delete(evsel);
438 return NULL;
439}
440
441
442
443
444struct evsel *evsel__newtp_idx(const char *sys, const char *name, int idx)
445{
446 struct evsel *evsel = zalloc(perf_evsel__object.size);
447 int err = -ENOMEM;
448
449 if (evsel == NULL) {
450 goto out_err;
451 } else {
452 struct perf_event_attr attr = {
453 .type = PERF_TYPE_TRACEPOINT,
454 .sample_type = (PERF_SAMPLE_RAW | PERF_SAMPLE_TIME |
455 PERF_SAMPLE_CPU | PERF_SAMPLE_PERIOD),
456 };
457
458 if (asprintf(&evsel->name, "%s:%s", sys, name) < 0)
459 goto out_free;
460
461 evsel->tp_format = trace_event__tp_format(sys, name);
462 if (IS_ERR(evsel->tp_format)) {
463 err = PTR_ERR(evsel->tp_format);
464 goto out_free;
465 }
466
467 event_attr_init(&attr);
468 attr.config = evsel->tp_format->id;
469 attr.sample_period = 1;
470 evsel__init(evsel, &attr, idx);
471 }
472
473 return evsel;
474
475out_free:
476 zfree(&evsel->name);
477 free(evsel);
478out_err:
479 return ERR_PTR(err);
480}
481
482const char *evsel__hw_names[PERF_COUNT_HW_MAX] = {
483 "cycles",
484 "instructions",
485 "cache-references",
486 "cache-misses",
487 "branches",
488 "branch-misses",
489 "bus-cycles",
490 "stalled-cycles-frontend",
491 "stalled-cycles-backend",
492 "ref-cycles",
493};
494
495char *evsel__bpf_counter_events;
496
497bool evsel__match_bpf_counter_events(const char *name)
498{
499 int name_len;
500 bool match;
501 char *ptr;
502
503 if (!evsel__bpf_counter_events)
504 return false;
505
506 ptr = strstr(evsel__bpf_counter_events, name);
507 name_len = strlen(name);
508
509
510 match = (ptr != NULL) &&
511 ((ptr == evsel__bpf_counter_events) || (*(ptr - 1) == ',')) &&
512 ((*(ptr + name_len) == ',') || (*(ptr + name_len) == '\0'));
513
514 return match;
515}
516
517static const char *__evsel__hw_name(u64 config)
518{
519 if (config < PERF_COUNT_HW_MAX && evsel__hw_names[config])
520 return evsel__hw_names[config];
521
522 return "unknown-hardware";
523}
524
525static int evsel__add_modifiers(struct evsel *evsel, char *bf, size_t size)
526{
527 int colon = 0, r = 0;
528 struct perf_event_attr *attr = &evsel->core.attr;
529 bool exclude_guest_default = false;
530
531#define MOD_PRINT(context, mod) do { \
532 if (!attr->exclude_##context) { \
533 if (!colon) colon = ++r; \
534 r += scnprintf(bf + r, size - r, "%c", mod); \
535 } } while(0)
536
537 if (attr->exclude_kernel || attr->exclude_user || attr->exclude_hv) {
538 MOD_PRINT(kernel, 'k');
539 MOD_PRINT(user, 'u');
540 MOD_PRINT(hv, 'h');
541 exclude_guest_default = true;
542 }
543
544 if (attr->precise_ip) {
545 if (!colon)
546 colon = ++r;
547 r += scnprintf(bf + r, size - r, "%.*s", attr->precise_ip, "ppp");
548 exclude_guest_default = true;
549 }
550
551 if (attr->exclude_host || attr->exclude_guest == exclude_guest_default) {
552 MOD_PRINT(host, 'H');
553 MOD_PRINT(guest, 'G');
554 }
555#undef MOD_PRINT
556 if (colon)
557 bf[colon - 1] = ':';
558 return r;
559}
560
561static int evsel__hw_name(struct evsel *evsel, char *bf, size_t size)
562{
563 int r = scnprintf(bf, size, "%s", __evsel__hw_name(evsel->core.attr.config));
564 return r + evsel__add_modifiers(evsel, bf + r, size - r);
565}
566
567const char *evsel__sw_names[PERF_COUNT_SW_MAX] = {
568 "cpu-clock",
569 "task-clock",
570 "page-faults",
571 "context-switches",
572 "cpu-migrations",
573 "minor-faults",
574 "major-faults",
575 "alignment-faults",
576 "emulation-faults",
577 "dummy",
578};
579
580static const char *__evsel__sw_name(u64 config)
581{
582 if (config < PERF_COUNT_SW_MAX && evsel__sw_names[config])
583 return evsel__sw_names[config];
584 return "unknown-software";
585}
586
587static int evsel__sw_name(struct evsel *evsel, char *bf, size_t size)
588{
589 int r = scnprintf(bf, size, "%s", __evsel__sw_name(evsel->core.attr.config));
590 return r + evsel__add_modifiers(evsel, bf + r, size - r);
591}
592
593static int __evsel__bp_name(char *bf, size_t size, u64 addr, u64 type)
594{
595 int r;
596
597 r = scnprintf(bf, size, "mem:0x%" PRIx64 ":", addr);
598
599 if (type & HW_BREAKPOINT_R)
600 r += scnprintf(bf + r, size - r, "r");
601
602 if (type & HW_BREAKPOINT_W)
603 r += scnprintf(bf + r, size - r, "w");
604
605 if (type & HW_BREAKPOINT_X)
606 r += scnprintf(bf + r, size - r, "x");
607
608 return r;
609}
610
611static int evsel__bp_name(struct evsel *evsel, char *bf, size_t size)
612{
613 struct perf_event_attr *attr = &evsel->core.attr;
614 int r = __evsel__bp_name(bf, size, attr->bp_addr, attr->bp_type);
615 return r + evsel__add_modifiers(evsel, bf + r, size - r);
616}
617
618const char *evsel__hw_cache[PERF_COUNT_HW_CACHE_MAX][EVSEL__MAX_ALIASES] = {
619 { "L1-dcache", "l1-d", "l1d", "L1-data", },
620 { "L1-icache", "l1-i", "l1i", "L1-instruction", },
621 { "LLC", "L2", },
622 { "dTLB", "d-tlb", "Data-TLB", },
623 { "iTLB", "i-tlb", "Instruction-TLB", },
624 { "branch", "branches", "bpu", "btb", "bpc", },
625 { "node", },
626};
627
628const char *evsel__hw_cache_op[PERF_COUNT_HW_CACHE_OP_MAX][EVSEL__MAX_ALIASES] = {
629 { "load", "loads", "read", },
630 { "store", "stores", "write", },
631 { "prefetch", "prefetches", "speculative-read", "speculative-load", },
632};
633
634const char *evsel__hw_cache_result[PERF_COUNT_HW_CACHE_RESULT_MAX][EVSEL__MAX_ALIASES] = {
635 { "refs", "Reference", "ops", "access", },
636 { "misses", "miss", },
637};
638
639#define C(x) PERF_COUNT_HW_CACHE_##x
640#define CACHE_READ (1 << C(OP_READ))
641#define CACHE_WRITE (1 << C(OP_WRITE))
642#define CACHE_PREFETCH (1 << C(OP_PREFETCH))
643#define COP(x) (1 << x)
644
645
646
647
648
649
650static unsigned long evsel__hw_cache_stat[C(MAX)] = {
651 [C(L1D)] = (CACHE_READ | CACHE_WRITE | CACHE_PREFETCH),
652 [C(L1I)] = (CACHE_READ | CACHE_PREFETCH),
653 [C(LL)] = (CACHE_READ | CACHE_WRITE | CACHE_PREFETCH),
654 [C(DTLB)] = (CACHE_READ | CACHE_WRITE | CACHE_PREFETCH),
655 [C(ITLB)] = (CACHE_READ),
656 [C(BPU)] = (CACHE_READ),
657 [C(NODE)] = (CACHE_READ | CACHE_WRITE | CACHE_PREFETCH),
658};
659
660bool evsel__is_cache_op_valid(u8 type, u8 op)
661{
662 if (evsel__hw_cache_stat[type] & COP(op))
663 return true;
664 else
665 return false;
666}
667
668int __evsel__hw_cache_type_op_res_name(u8 type, u8 op, u8 result, char *bf, size_t size)
669{
670 if (result) {
671 return scnprintf(bf, size, "%s-%s-%s", evsel__hw_cache[type][0],
672 evsel__hw_cache_op[op][0],
673 evsel__hw_cache_result[result][0]);
674 }
675
676 return scnprintf(bf, size, "%s-%s", evsel__hw_cache[type][0],
677 evsel__hw_cache_op[op][1]);
678}
679
680static int __evsel__hw_cache_name(u64 config, char *bf, size_t size)
681{
682 u8 op, result, type = (config >> 0) & 0xff;
683 const char *err = "unknown-ext-hardware-cache-type";
684
685 if (type >= PERF_COUNT_HW_CACHE_MAX)
686 goto out_err;
687
688 op = (config >> 8) & 0xff;
689 err = "unknown-ext-hardware-cache-op";
690 if (op >= PERF_COUNT_HW_CACHE_OP_MAX)
691 goto out_err;
692
693 result = (config >> 16) & 0xff;
694 err = "unknown-ext-hardware-cache-result";
695 if (result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
696 goto out_err;
697
698 err = "invalid-cache";
699 if (!evsel__is_cache_op_valid(type, op))
700 goto out_err;
701
702 return __evsel__hw_cache_type_op_res_name(type, op, result, bf, size);
703out_err:
704 return scnprintf(bf, size, "%s", err);
705}
706
707static int evsel__hw_cache_name(struct evsel *evsel, char *bf, size_t size)
708{
709 int ret = __evsel__hw_cache_name(evsel->core.attr.config, bf, size);
710 return ret + evsel__add_modifiers(evsel, bf + ret, size - ret);
711}
712
713static int evsel__raw_name(struct evsel *evsel, char *bf, size_t size)
714{
715 int ret = scnprintf(bf, size, "raw 0x%" PRIx64, evsel->core.attr.config);
716 return ret + evsel__add_modifiers(evsel, bf + ret, size - ret);
717}
718
719static int evsel__tool_name(char *bf, size_t size)
720{
721 int ret = scnprintf(bf, size, "duration_time");
722 return ret;
723}
724
725const char *evsel__name(struct evsel *evsel)
726{
727 char bf[128];
728
729 if (!evsel)
730 goto out_unknown;
731
732 if (evsel->name)
733 return evsel->name;
734
735 switch (evsel->core.attr.type) {
736 case PERF_TYPE_RAW:
737 evsel__raw_name(evsel, bf, sizeof(bf));
738 break;
739
740 case PERF_TYPE_HARDWARE:
741 evsel__hw_name(evsel, bf, sizeof(bf));
742 break;
743
744 case PERF_TYPE_HW_CACHE:
745 evsel__hw_cache_name(evsel, bf, sizeof(bf));
746 break;
747
748 case PERF_TYPE_SOFTWARE:
749 if (evsel->tool_event)
750 evsel__tool_name(bf, sizeof(bf));
751 else
752 evsel__sw_name(evsel, bf, sizeof(bf));
753 break;
754
755 case PERF_TYPE_TRACEPOINT:
756 scnprintf(bf, sizeof(bf), "%s", "unknown tracepoint");
757 break;
758
759 case PERF_TYPE_BREAKPOINT:
760 evsel__bp_name(evsel, bf, sizeof(bf));
761 break;
762
763 default:
764 scnprintf(bf, sizeof(bf), "unknown attr type: %d",
765 evsel->core.attr.type);
766 break;
767 }
768
769 evsel->name = strdup(bf);
770
771 if (evsel->name)
772 return evsel->name;
773out_unknown:
774 return "unknown";
775}
776
777const char *evsel__group_name(struct evsel *evsel)
778{
779 return evsel->group_name ?: "anon group";
780}
781
782
783
784
785
786
787
788
789
790
791
792int evsel__group_desc(struct evsel *evsel, char *buf, size_t size)
793{
794 int ret = 0;
795 struct evsel *pos;
796 const char *group_name = evsel__group_name(evsel);
797
798 if (!evsel->forced_leader)
799 ret = scnprintf(buf, size, "%s { ", group_name);
800
801 ret += scnprintf(buf + ret, size - ret, "%s", evsel__name(evsel));
802
803 for_each_group_member(pos, evsel)
804 ret += scnprintf(buf + ret, size - ret, ", %s", evsel__name(pos));
805
806 if (!evsel->forced_leader)
807 ret += scnprintf(buf + ret, size - ret, " }");
808
809 return ret;
810}
811
812static void __evsel__config_callchain(struct evsel *evsel, struct record_opts *opts,
813 struct callchain_param *param)
814{
815 bool function = evsel__is_function_event(evsel);
816 struct perf_event_attr *attr = &evsel->core.attr;
817
818 evsel__set_sample_bit(evsel, CALLCHAIN);
819
820 attr->sample_max_stack = param->max_stack;
821
822 if (opts->kernel_callchains)
823 attr->exclude_callchain_user = 1;
824 if (opts->user_callchains)
825 attr->exclude_callchain_kernel = 1;
826 if (param->record_mode == CALLCHAIN_LBR) {
827 if (!opts->branch_stack) {
828 if (attr->exclude_user) {
829 pr_warning("LBR callstack option is only available "
830 "to get user callchain information. "
831 "Falling back to framepointers.\n");
832 } else {
833 evsel__set_sample_bit(evsel, BRANCH_STACK);
834 attr->branch_sample_type = PERF_SAMPLE_BRANCH_USER |
835 PERF_SAMPLE_BRANCH_CALL_STACK |
836 PERF_SAMPLE_BRANCH_NO_CYCLES |
837 PERF_SAMPLE_BRANCH_NO_FLAGS |
838 PERF_SAMPLE_BRANCH_HW_INDEX;
839 }
840 } else
841 pr_warning("Cannot use LBR callstack with branch stack. "
842 "Falling back to framepointers.\n");
843 }
844
845 if (param->record_mode == CALLCHAIN_DWARF) {
846 if (!function) {
847 evsel__set_sample_bit(evsel, REGS_USER);
848 evsel__set_sample_bit(evsel, STACK_USER);
849 if (opts->sample_user_regs && DWARF_MINIMAL_REGS != PERF_REGS_MASK) {
850 attr->sample_regs_user |= DWARF_MINIMAL_REGS;
851 pr_warning("WARNING: The use of --call-graph=dwarf may require all the user registers, "
852 "specifying a subset with --user-regs may render DWARF unwinding unreliable, "
853 "so the minimal registers set (IP, SP) is explicitly forced.\n");
854 } else {
855 attr->sample_regs_user |= PERF_REGS_MASK;
856 }
857 attr->sample_stack_user = param->dump_size;
858 attr->exclude_callchain_user = 1;
859 } else {
860 pr_info("Cannot use DWARF unwind for function trace event,"
861 " falling back to framepointers.\n");
862 }
863 }
864
865 if (function) {
866 pr_info("Disabling user space callchains for function trace event.\n");
867 attr->exclude_callchain_user = 1;
868 }
869}
870
871void evsel__config_callchain(struct evsel *evsel, struct record_opts *opts,
872 struct callchain_param *param)
873{
874 if (param->enabled)
875 return __evsel__config_callchain(evsel, opts, param);
876}
877
878static void evsel__reset_callgraph(struct evsel *evsel, struct callchain_param *param)
879{
880 struct perf_event_attr *attr = &evsel->core.attr;
881
882 evsel__reset_sample_bit(evsel, CALLCHAIN);
883 if (param->record_mode == CALLCHAIN_LBR) {
884 evsel__reset_sample_bit(evsel, BRANCH_STACK);
885 attr->branch_sample_type &= ~(PERF_SAMPLE_BRANCH_USER |
886 PERF_SAMPLE_BRANCH_CALL_STACK |
887 PERF_SAMPLE_BRANCH_HW_INDEX);
888 }
889 if (param->record_mode == CALLCHAIN_DWARF) {
890 evsel__reset_sample_bit(evsel, REGS_USER);
891 evsel__reset_sample_bit(evsel, STACK_USER);
892 }
893}
894
895static void evsel__apply_config_terms(struct evsel *evsel,
896 struct record_opts *opts, bool track)
897{
898 struct evsel_config_term *term;
899 struct list_head *config_terms = &evsel->config_terms;
900 struct perf_event_attr *attr = &evsel->core.attr;
901
902 struct callchain_param param = {
903 .record_mode = callchain_param.record_mode,
904 };
905 u32 dump_size = 0;
906 int max_stack = 0;
907 const char *callgraph_buf = NULL;
908
909 list_for_each_entry(term, config_terms, list) {
910 switch (term->type) {
911 case EVSEL__CONFIG_TERM_PERIOD:
912 if (!(term->weak && opts->user_interval != ULLONG_MAX)) {
913 attr->sample_period = term->val.period;
914 attr->freq = 0;
915 evsel__reset_sample_bit(evsel, PERIOD);
916 }
917 break;
918 case EVSEL__CONFIG_TERM_FREQ:
919 if (!(term->weak && opts->user_freq != UINT_MAX)) {
920 attr->sample_freq = term->val.freq;
921 attr->freq = 1;
922 evsel__set_sample_bit(evsel, PERIOD);
923 }
924 break;
925 case EVSEL__CONFIG_TERM_TIME:
926 if (term->val.time)
927 evsel__set_sample_bit(evsel, TIME);
928 else
929 evsel__reset_sample_bit(evsel, TIME);
930 break;
931 case EVSEL__CONFIG_TERM_CALLGRAPH:
932 callgraph_buf = term->val.str;
933 break;
934 case EVSEL__CONFIG_TERM_BRANCH:
935 if (term->val.str && strcmp(term->val.str, "no")) {
936 evsel__set_sample_bit(evsel, BRANCH_STACK);
937 parse_branch_str(term->val.str,
938 &attr->branch_sample_type);
939 } else
940 evsel__reset_sample_bit(evsel, BRANCH_STACK);
941 break;
942 case EVSEL__CONFIG_TERM_STACK_USER:
943 dump_size = term->val.stack_user;
944 break;
945 case EVSEL__CONFIG_TERM_MAX_STACK:
946 max_stack = term->val.max_stack;
947 break;
948 case EVSEL__CONFIG_TERM_MAX_EVENTS:
949 evsel->max_events = term->val.max_events;
950 break;
951 case EVSEL__CONFIG_TERM_INHERIT:
952
953
954
955
956
957
958 attr->inherit = term->val.inherit ? 1 : 0;
959 break;
960 case EVSEL__CONFIG_TERM_OVERWRITE:
961 attr->write_backward = term->val.overwrite ? 1 : 0;
962 break;
963 case EVSEL__CONFIG_TERM_DRV_CFG:
964 break;
965 case EVSEL__CONFIG_TERM_PERCORE:
966 break;
967 case EVSEL__CONFIG_TERM_AUX_OUTPUT:
968 attr->aux_output = term->val.aux_output ? 1 : 0;
969 break;
970 case EVSEL__CONFIG_TERM_AUX_SAMPLE_SIZE:
971
972 break;
973 case EVSEL__CONFIG_TERM_CFG_CHG:
974 break;
975 default:
976 break;
977 }
978 }
979
980
981 if ((callgraph_buf != NULL) || (dump_size > 0) || max_stack) {
982 bool sample_address = false;
983
984 if (max_stack) {
985 param.max_stack = max_stack;
986 if (callgraph_buf == NULL)
987 callgraph_buf = "fp";
988 }
989
990
991 if (callgraph_buf != NULL) {
992 if (!strcmp(callgraph_buf, "no")) {
993 param.enabled = false;
994 param.record_mode = CALLCHAIN_NONE;
995 } else {
996 param.enabled = true;
997 if (parse_callchain_record(callgraph_buf, ¶m)) {
998 pr_err("per-event callgraph setting for %s failed. "
999 "Apply callgraph global setting for it\n",
1000 evsel->name);
1001 return;
1002 }
1003 if (param.record_mode == CALLCHAIN_DWARF)
1004 sample_address = true;
1005 }
1006 }
1007 if (dump_size > 0) {
1008 dump_size = round_up(dump_size, sizeof(u64));
1009 param.dump_size = dump_size;
1010 }
1011
1012
1013 if (callchain_param.enabled)
1014 evsel__reset_callgraph(evsel, &callchain_param);
1015
1016
1017 if (param.enabled) {
1018 if (sample_address) {
1019 evsel__set_sample_bit(evsel, ADDR);
1020 evsel__set_sample_bit(evsel, DATA_SRC);
1021 evsel->core.attr.mmap_data = track;
1022 }
1023 evsel__config_callchain(evsel, opts, ¶m);
1024 }
1025 }
1026}
1027
1028struct evsel_config_term *__evsel__get_config_term(struct evsel *evsel, enum evsel_term_type type)
1029{
1030 struct evsel_config_term *term, *found_term = NULL;
1031
1032 list_for_each_entry(term, &evsel->config_terms, list) {
1033 if (term->type == type)
1034 found_term = term;
1035 }
1036
1037 return found_term;
1038}
1039
1040void __weak arch_evsel__set_sample_weight(struct evsel *evsel)
1041{
1042 evsel__set_sample_bit(evsel, WEIGHT);
1043}
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073void evsel__config(struct evsel *evsel, struct record_opts *opts,
1074 struct callchain_param *callchain)
1075{
1076 struct evsel *leader = evsel__leader(evsel);
1077 struct perf_event_attr *attr = &evsel->core.attr;
1078 int track = evsel->tracking;
1079 bool per_cpu = opts->target.default_per_cpu && !opts->target.per_thread;
1080
1081 attr->sample_id_all = perf_missing_features.sample_id_all ? 0 : 1;
1082 attr->inherit = !opts->no_inherit;
1083 attr->write_backward = opts->overwrite ? 1 : 0;
1084
1085 evsel__set_sample_bit(evsel, IP);
1086 evsel__set_sample_bit(evsel, TID);
1087
1088 if (evsel->sample_read) {
1089 evsel__set_sample_bit(evsel, READ);
1090
1091
1092
1093
1094
1095 evsel__set_sample_id(evsel, false);
1096
1097
1098
1099
1100
1101 if (leader->core.nr_members > 1) {
1102 attr->read_format |= PERF_FORMAT_GROUP;
1103 attr->inherit = 0;
1104 }
1105 }
1106
1107
1108
1109
1110
1111 if (!attr->sample_period) {
1112 if (opts->freq) {
1113 attr->freq = 1;
1114 attr->sample_freq = opts->freq;
1115 } else {
1116 attr->sample_period = opts->default_interval;
1117 }
1118 }
1119
1120
1121
1122
1123 if (attr->freq)
1124 evsel__set_sample_bit(evsel, PERIOD);
1125
1126 if (opts->no_samples)
1127 attr->sample_freq = 0;
1128
1129 if (opts->inherit_stat) {
1130 evsel->core.attr.read_format |=
1131 PERF_FORMAT_TOTAL_TIME_ENABLED |
1132 PERF_FORMAT_TOTAL_TIME_RUNNING |
1133 PERF_FORMAT_ID;
1134 attr->inherit_stat = 1;
1135 }
1136
1137 if (opts->sample_address) {
1138 evsel__set_sample_bit(evsel, ADDR);
1139 attr->mmap_data = track;
1140 }
1141
1142
1143
1144
1145
1146
1147 if (evsel__is_function_event(evsel))
1148 evsel->core.attr.exclude_callchain_user = 1;
1149
1150 if (callchain && callchain->enabled && !evsel->no_aux_samples)
1151 evsel__config_callchain(evsel, opts, callchain);
1152
1153 if (opts->sample_intr_regs && !evsel->no_aux_samples &&
1154 !evsel__is_dummy_event(evsel)) {
1155 attr->sample_regs_intr = opts->sample_intr_regs;
1156 evsel__set_sample_bit(evsel, REGS_INTR);
1157 }
1158
1159 if (opts->sample_user_regs && !evsel->no_aux_samples &&
1160 !evsel__is_dummy_event(evsel)) {
1161 attr->sample_regs_user |= opts->sample_user_regs;
1162 evsel__set_sample_bit(evsel, REGS_USER);
1163 }
1164
1165 if (target__has_cpu(&opts->target) || opts->sample_cpu)
1166 evsel__set_sample_bit(evsel, CPU);
1167
1168
1169
1170
1171 if (opts->sample_time &&
1172 (!perf_missing_features.sample_id_all &&
1173 (!opts->no_inherit || target__has_cpu(&opts->target) || per_cpu ||
1174 opts->sample_time_set)))
1175 evsel__set_sample_bit(evsel, TIME);
1176
1177 if (opts->raw_samples && !evsel->no_aux_samples) {
1178 evsel__set_sample_bit(evsel, TIME);
1179 evsel__set_sample_bit(evsel, RAW);
1180 evsel__set_sample_bit(evsel, CPU);
1181 }
1182
1183 if (opts->sample_address)
1184 evsel__set_sample_bit(evsel, DATA_SRC);
1185
1186 if (opts->sample_phys_addr)
1187 evsel__set_sample_bit(evsel, PHYS_ADDR);
1188
1189 if (opts->no_buffering) {
1190 attr->watermark = 0;
1191 attr->wakeup_events = 1;
1192 }
1193 if (opts->branch_stack && !evsel->no_aux_samples) {
1194 evsel__set_sample_bit(evsel, BRANCH_STACK);
1195 attr->branch_sample_type = opts->branch_stack;
1196 }
1197
1198 if (opts->sample_weight)
1199 arch_evsel__set_sample_weight(evsel);
1200
1201 attr->task = track;
1202 attr->mmap = track;
1203 attr->mmap2 = track && !perf_missing_features.mmap2;
1204 attr->comm = track;
1205 attr->build_id = track && opts->build_id;
1206
1207
1208
1209
1210
1211 if (!opts->text_poke)
1212 attr->ksymbol = track && !perf_missing_features.ksymbol;
1213 attr->bpf_event = track && !opts->no_bpf_event && !perf_missing_features.bpf;
1214
1215 if (opts->record_namespaces)
1216 attr->namespaces = track;
1217
1218 if (opts->record_cgroup) {
1219 attr->cgroup = track && !perf_missing_features.cgroup;
1220 evsel__set_sample_bit(evsel, CGROUP);
1221 }
1222
1223 if (opts->sample_data_page_size)
1224 evsel__set_sample_bit(evsel, DATA_PAGE_SIZE);
1225
1226 if (opts->sample_code_page_size)
1227 evsel__set_sample_bit(evsel, CODE_PAGE_SIZE);
1228
1229 if (opts->record_switch_events)
1230 attr->context_switch = track;
1231
1232 if (opts->sample_transaction)
1233 evsel__set_sample_bit(evsel, TRANSACTION);
1234
1235 if (opts->running_time) {
1236 evsel->core.attr.read_format |=
1237 PERF_FORMAT_TOTAL_TIME_ENABLED |
1238 PERF_FORMAT_TOTAL_TIME_RUNNING;
1239 }
1240
1241
1242
1243
1244
1245
1246
1247 if (evsel__is_group_leader(evsel))
1248 attr->disabled = 1;
1249
1250
1251
1252
1253
1254 if (target__none(&opts->target) && evsel__is_group_leader(evsel) &&
1255 !opts->initial_delay)
1256 attr->enable_on_exec = 1;
1257
1258 if (evsel->immediate) {
1259 attr->disabled = 0;
1260 attr->enable_on_exec = 0;
1261 }
1262
1263 clockid = opts->clockid;
1264 if (opts->use_clockid) {
1265 attr->use_clockid = 1;
1266 attr->clockid = opts->clockid;
1267 }
1268
1269 if (evsel->precise_max)
1270 attr->precise_ip = 3;
1271
1272 if (opts->all_user) {
1273 attr->exclude_kernel = 1;
1274 attr->exclude_user = 0;
1275 }
1276
1277 if (opts->all_kernel) {
1278 attr->exclude_kernel = 0;
1279 attr->exclude_user = 1;
1280 }
1281
1282 if (evsel->core.own_cpus || evsel->unit)
1283 evsel->core.attr.read_format |= PERF_FORMAT_ID;
1284
1285
1286
1287
1288
1289 evsel__apply_config_terms(evsel, opts, track);
1290
1291 evsel->ignore_missing_thread = opts->ignore_missing_thread;
1292
1293
1294 if (opts->period_set) {
1295 if (opts->period)
1296 evsel__set_sample_bit(evsel, PERIOD);
1297 else
1298 evsel__reset_sample_bit(evsel, PERIOD);
1299 }
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309 if (evsel__is_dummy_event(evsel))
1310 evsel__reset_sample_bit(evsel, BRANCH_STACK);
1311}
1312
1313int evsel__set_filter(struct evsel *evsel, const char *filter)
1314{
1315 char *new_filter = strdup(filter);
1316
1317 if (new_filter != NULL) {
1318 free(evsel->filter);
1319 evsel->filter = new_filter;
1320 return 0;
1321 }
1322
1323 return -1;
1324}
1325
1326static int evsel__append_filter(struct evsel *evsel, const char *fmt, const char *filter)
1327{
1328 char *new_filter;
1329
1330 if (evsel->filter == NULL)
1331 return evsel__set_filter(evsel, filter);
1332
1333 if (asprintf(&new_filter, fmt, evsel->filter, filter) > 0) {
1334 free(evsel->filter);
1335 evsel->filter = new_filter;
1336 return 0;
1337 }
1338
1339 return -1;
1340}
1341
1342int evsel__append_tp_filter(struct evsel *evsel, const char *filter)
1343{
1344 return evsel__append_filter(evsel, "(%s) && (%s)", filter);
1345}
1346
1347int evsel__append_addr_filter(struct evsel *evsel, const char *filter)
1348{
1349 return evsel__append_filter(evsel, "%s,%s", filter);
1350}
1351
1352
1353int evsel__enable_cpu(struct evsel *evsel, int cpu)
1354{
1355 return perf_evsel__enable_cpu(&evsel->core, cpu);
1356}
1357
1358int evsel__enable(struct evsel *evsel)
1359{
1360 int err = perf_evsel__enable(&evsel->core);
1361
1362 if (!err)
1363 evsel->disabled = false;
1364 return err;
1365}
1366
1367
1368int evsel__disable_cpu(struct evsel *evsel, int cpu)
1369{
1370 return perf_evsel__disable_cpu(&evsel->core, cpu);
1371}
1372
1373int evsel__disable(struct evsel *evsel)
1374{
1375 int err = perf_evsel__disable(&evsel->core);
1376
1377
1378
1379
1380
1381
1382 if (!err)
1383 evsel->disabled = true;
1384
1385 return err;
1386}
1387
1388static void evsel__free_config_terms(struct evsel *evsel)
1389{
1390 struct evsel_config_term *term, *h;
1391
1392 list_for_each_entry_safe(term, h, &evsel->config_terms, list) {
1393 list_del_init(&term->list);
1394 if (term->free_str)
1395 zfree(&term->val.str);
1396 free(term);
1397 }
1398}
1399
1400void evsel__exit(struct evsel *evsel)
1401{
1402 assert(list_empty(&evsel->core.node));
1403 assert(evsel->evlist == NULL);
1404 bpf_counter__destroy(evsel);
1405 evsel__free_counts(evsel);
1406 perf_evsel__free_fd(&evsel->core);
1407 perf_evsel__free_id(&evsel->core);
1408 evsel__free_config_terms(evsel);
1409 cgroup__put(evsel->cgrp);
1410 perf_cpu_map__put(evsel->core.cpus);
1411 perf_cpu_map__put(evsel->core.own_cpus);
1412 perf_thread_map__put(evsel->core.threads);
1413 zfree(&evsel->group_name);
1414 zfree(&evsel->name);
1415 zfree(&evsel->pmu_name);
1416 evsel__zero_per_pkg(evsel);
1417 hashmap__free(evsel->per_pkg_mask);
1418 evsel->per_pkg_mask = NULL;
1419 zfree(&evsel->metric_events);
1420 perf_evsel__object.fini(evsel);
1421}
1422
1423void evsel__delete(struct evsel *evsel)
1424{
1425 evsel__exit(evsel);
1426 free(evsel);
1427}
1428
1429void evsel__compute_deltas(struct evsel *evsel, int cpu, int thread,
1430 struct perf_counts_values *count)
1431{
1432 struct perf_counts_values tmp;
1433
1434 if (!evsel->prev_raw_counts)
1435 return;
1436
1437 if (cpu == -1) {
1438 tmp = evsel->prev_raw_counts->aggr;
1439 evsel->prev_raw_counts->aggr = *count;
1440 } else {
1441 tmp = *perf_counts(evsel->prev_raw_counts, cpu, thread);
1442 *perf_counts(evsel->prev_raw_counts, cpu, thread) = *count;
1443 }
1444
1445 count->val = count->val - tmp.val;
1446 count->ena = count->ena - tmp.ena;
1447 count->run = count->run - tmp.run;
1448}
1449
1450void perf_counts_values__scale(struct perf_counts_values *count,
1451 bool scale, s8 *pscaled)
1452{
1453 s8 scaled = 0;
1454
1455 if (scale) {
1456 if (count->run == 0) {
1457 scaled = -1;
1458 count->val = 0;
1459 } else if (count->run < count->ena) {
1460 scaled = 1;
1461 count->val = (u64)((double) count->val * count->ena / count->run);
1462 }
1463 }
1464
1465 if (pscaled)
1466 *pscaled = scaled;
1467}
1468
1469static int evsel__read_one(struct evsel *evsel, int cpu, int thread)
1470{
1471 struct perf_counts_values *count = perf_counts(evsel->counts, cpu, thread);
1472
1473 return perf_evsel__read(&evsel->core, cpu, thread, count);
1474}
1475
1476static void evsel__set_count(struct evsel *counter, int cpu, int thread, u64 val, u64 ena, u64 run)
1477{
1478 struct perf_counts_values *count;
1479
1480 count = perf_counts(counter->counts, cpu, thread);
1481
1482 count->val = val;
1483 count->ena = ena;
1484 count->run = run;
1485
1486 perf_counts__set_loaded(counter->counts, cpu, thread, true);
1487}
1488
1489static int evsel__process_group_data(struct evsel *leader, int cpu, int thread, u64 *data)
1490{
1491 u64 read_format = leader->core.attr.read_format;
1492 struct sample_read_value *v;
1493 u64 nr, ena = 0, run = 0, i;
1494
1495 nr = *data++;
1496
1497 if (nr != (u64) leader->core.nr_members)
1498 return -EINVAL;
1499
1500 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
1501 ena = *data++;
1502
1503 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
1504 run = *data++;
1505
1506 v = (struct sample_read_value *) data;
1507
1508 evsel__set_count(leader, cpu, thread, v[0].value, ena, run);
1509
1510 for (i = 1; i < nr; i++) {
1511 struct evsel *counter;
1512
1513 counter = evlist__id2evsel(leader->evlist, v[i].id);
1514 if (!counter)
1515 return -EINVAL;
1516
1517 evsel__set_count(counter, cpu, thread, v[i].value, ena, run);
1518 }
1519
1520 return 0;
1521}
1522
1523static int evsel__read_group(struct evsel *leader, int cpu, int thread)
1524{
1525 struct perf_stat_evsel *ps = leader->stats;
1526 u64 read_format = leader->core.attr.read_format;
1527 int size = perf_evsel__read_size(&leader->core);
1528 u64 *data = ps->group_data;
1529
1530 if (!(read_format & PERF_FORMAT_ID))
1531 return -EINVAL;
1532
1533 if (!evsel__is_group_leader(leader))
1534 return -EINVAL;
1535
1536 if (!data) {
1537 data = zalloc(size);
1538 if (!data)
1539 return -ENOMEM;
1540
1541 ps->group_data = data;
1542 }
1543
1544 if (FD(leader, cpu, thread) < 0)
1545 return -EINVAL;
1546
1547 if (readn(FD(leader, cpu, thread), data, size) <= 0)
1548 return -errno;
1549
1550 return evsel__process_group_data(leader, cpu, thread, data);
1551}
1552
1553int evsel__read_counter(struct evsel *evsel, int cpu, int thread)
1554{
1555 u64 read_format = evsel->core.attr.read_format;
1556
1557 if (read_format & PERF_FORMAT_GROUP)
1558 return evsel__read_group(evsel, cpu, thread);
1559
1560 return evsel__read_one(evsel, cpu, thread);
1561}
1562
1563int __evsel__read_on_cpu(struct evsel *evsel, int cpu, int thread, bool scale)
1564{
1565 struct perf_counts_values count;
1566 size_t nv = scale ? 3 : 1;
1567
1568 if (FD(evsel, cpu, thread) < 0)
1569 return -EINVAL;
1570
1571 if (evsel->counts == NULL && evsel__alloc_counts(evsel, cpu + 1, thread + 1) < 0)
1572 return -ENOMEM;
1573
1574 if (readn(FD(evsel, cpu, thread), &count, nv * sizeof(u64)) <= 0)
1575 return -errno;
1576
1577 evsel__compute_deltas(evsel, cpu, thread, &count);
1578 perf_counts_values__scale(&count, scale, NULL);
1579 *perf_counts(evsel->counts, cpu, thread) = count;
1580 return 0;
1581}
1582
1583static int evsel__match_other_cpu(struct evsel *evsel, struct evsel *other,
1584 int cpu)
1585{
1586 int cpuid;
1587
1588 cpuid = perf_cpu_map__cpu(evsel->core.cpus, cpu);
1589 return perf_cpu_map__idx(other->core.cpus, cpuid);
1590}
1591
1592static int evsel__hybrid_group_cpu(struct evsel *evsel, int cpu)
1593{
1594 struct evsel *leader = evsel__leader(evsel);
1595
1596 if ((evsel__is_hybrid(evsel) && !evsel__is_hybrid(leader)) ||
1597 (!evsel__is_hybrid(evsel) && evsel__is_hybrid(leader))) {
1598 return evsel__match_other_cpu(evsel, leader, cpu);
1599 }
1600
1601 return cpu;
1602}
1603
1604static int get_group_fd(struct evsel *evsel, int cpu, int thread)
1605{
1606 struct evsel *leader = evsel__leader(evsel);
1607 int fd;
1608
1609 if (evsel__is_group_leader(evsel))
1610 return -1;
1611
1612
1613
1614
1615
1616 BUG_ON(!leader->core.fd);
1617
1618 cpu = evsel__hybrid_group_cpu(evsel, cpu);
1619 if (cpu == -1)
1620 return -1;
1621
1622 fd = FD(leader, cpu, thread);
1623 BUG_ON(fd == -1);
1624
1625 return fd;
1626}
1627
1628static void evsel__remove_fd(struct evsel *pos, int nr_cpus, int nr_threads, int thread_idx)
1629{
1630 for (int cpu = 0; cpu < nr_cpus; cpu++)
1631 for (int thread = thread_idx; thread < nr_threads - 1; thread++)
1632 FD(pos, cpu, thread) = FD(pos, cpu, thread + 1);
1633}
1634
1635static int update_fds(struct evsel *evsel,
1636 int nr_cpus, int cpu_idx,
1637 int nr_threads, int thread_idx)
1638{
1639 struct evsel *pos;
1640
1641 if (cpu_idx >= nr_cpus || thread_idx >= nr_threads)
1642 return -EINVAL;
1643
1644 evlist__for_each_entry(evsel->evlist, pos) {
1645 nr_cpus = pos != evsel ? nr_cpus : cpu_idx;
1646
1647 evsel__remove_fd(pos, nr_cpus, nr_threads, thread_idx);
1648
1649
1650
1651
1652
1653 if (pos == evsel)
1654 break;
1655 }
1656 return 0;
1657}
1658
1659static bool ignore_missing_thread(struct evsel *evsel,
1660 int nr_cpus, int cpu,
1661 struct perf_thread_map *threads,
1662 int thread, int err)
1663{
1664 pid_t ignore_pid = perf_thread_map__pid(threads, thread);
1665
1666 if (!evsel->ignore_missing_thread)
1667 return false;
1668
1669
1670 if (evsel->core.system_wide)
1671 return false;
1672
1673
1674 if (err != -ESRCH)
1675 return false;
1676
1677
1678 if (threads->nr == 1)
1679 return false;
1680
1681
1682
1683
1684
1685 if (update_fds(evsel, nr_cpus, cpu, threads->nr, thread))
1686 return false;
1687
1688 if (thread_map__remove(threads, thread))
1689 return false;
1690
1691 pr_warning("WARNING: Ignored open failure for pid %d\n",
1692 ignore_pid);
1693 return true;
1694}
1695
1696static int __open_attr__fprintf(FILE *fp, const char *name, const char *val,
1697 void *priv __maybe_unused)
1698{
1699 return fprintf(fp, " %-32s %s\n", name, val);
1700}
1701
1702static void display_attr(struct perf_event_attr *attr)
1703{
1704 if (verbose >= 2 || debug_peo_args) {
1705 fprintf(stderr, "%.60s\n", graph_dotted_line);
1706 fprintf(stderr, "perf_event_attr:\n");
1707 perf_event_attr__fprintf(stderr, attr, __open_attr__fprintf, NULL);
1708 fprintf(stderr, "%.60s\n", graph_dotted_line);
1709 }
1710}
1711
1712static int perf_event_open(struct evsel *evsel,
1713 pid_t pid, int cpu, int group_fd,
1714 unsigned long flags)
1715{
1716 int precise_ip = evsel->core.attr.precise_ip;
1717 int fd;
1718
1719 while (1) {
1720 pr_debug2_peo("sys_perf_event_open: pid %d cpu %d group_fd %d flags %#lx",
1721 pid, cpu, group_fd, flags);
1722
1723 fd = sys_perf_event_open(&evsel->core.attr, pid, cpu, group_fd, flags);
1724 if (fd >= 0)
1725 break;
1726
1727
1728 if (!evsel->precise_max)
1729 break;
1730
1731
1732
1733
1734
1735 if (!evsel->core.attr.precise_ip) {
1736 evsel->core.attr.precise_ip = precise_ip;
1737 break;
1738 }
1739
1740 pr_debug2_peo("\nsys_perf_event_open failed, error %d\n", -ENOTSUP);
1741 evsel->core.attr.precise_ip--;
1742 pr_debug2_peo("decreasing precise_ip by one (%d)\n", evsel->core.attr.precise_ip);
1743 display_attr(&evsel->core.attr);
1744 }
1745
1746 return fd;
1747}
1748
1749static int evsel__open_cpu(struct evsel *evsel, struct perf_cpu_map *cpus,
1750 struct perf_thread_map *threads,
1751 int start_cpu, int end_cpu)
1752{
1753 int cpu, thread, nthreads;
1754 unsigned long flags = PERF_FLAG_FD_CLOEXEC;
1755 int pid = -1, err, old_errno;
1756 enum { NO_CHANGE, SET_TO_MAX, INCREASED_MAX } set_rlimit = NO_CHANGE;
1757
1758 if ((perf_missing_features.write_backward && evsel->core.attr.write_backward) ||
1759 (perf_missing_features.aux_output && evsel->core.attr.aux_output))
1760 return -EINVAL;
1761
1762 if (cpus == NULL) {
1763 static struct perf_cpu_map *empty_cpu_map;
1764
1765 if (empty_cpu_map == NULL) {
1766 empty_cpu_map = perf_cpu_map__dummy_new();
1767 if (empty_cpu_map == NULL)
1768 return -ENOMEM;
1769 }
1770
1771 cpus = empty_cpu_map;
1772 }
1773
1774 if (threads == NULL) {
1775 static struct perf_thread_map *empty_thread_map;
1776
1777 if (empty_thread_map == NULL) {
1778 empty_thread_map = thread_map__new_by_tid(-1);
1779 if (empty_thread_map == NULL)
1780 return -ENOMEM;
1781 }
1782
1783 threads = empty_thread_map;
1784 }
1785
1786 if (evsel->core.system_wide)
1787 nthreads = 1;
1788 else
1789 nthreads = threads->nr;
1790
1791 if (evsel->core.fd == NULL &&
1792 perf_evsel__alloc_fd(&evsel->core, cpus->nr, nthreads) < 0)
1793 return -ENOMEM;
1794
1795 if (evsel->cgrp) {
1796 flags |= PERF_FLAG_PID_CGROUP;
1797 pid = evsel->cgrp->fd;
1798 }
1799
1800fallback_missing_features:
1801 if (perf_missing_features.weight_struct) {
1802 evsel__set_sample_bit(evsel, WEIGHT);
1803 evsel__reset_sample_bit(evsel, WEIGHT_STRUCT);
1804 }
1805 if (perf_missing_features.clockid_wrong)
1806 evsel->core.attr.clockid = CLOCK_MONOTONIC;
1807 if (perf_missing_features.clockid) {
1808 evsel->core.attr.use_clockid = 0;
1809 evsel->core.attr.clockid = 0;
1810 }
1811 if (perf_missing_features.cloexec)
1812 flags &= ~(unsigned long)PERF_FLAG_FD_CLOEXEC;
1813 if (perf_missing_features.mmap2)
1814 evsel->core.attr.mmap2 = 0;
1815 if (perf_missing_features.exclude_guest)
1816 evsel->core.attr.exclude_guest = evsel->core.attr.exclude_host = 0;
1817 if (perf_missing_features.lbr_flags)
1818 evsel->core.attr.branch_sample_type &= ~(PERF_SAMPLE_BRANCH_NO_FLAGS |
1819 PERF_SAMPLE_BRANCH_NO_CYCLES);
1820 if (perf_missing_features.group_read && evsel->core.attr.inherit)
1821 evsel->core.attr.read_format &= ~(PERF_FORMAT_GROUP|PERF_FORMAT_ID);
1822 if (perf_missing_features.ksymbol)
1823 evsel->core.attr.ksymbol = 0;
1824 if (perf_missing_features.bpf)
1825 evsel->core.attr.bpf_event = 0;
1826 if (perf_missing_features.branch_hw_idx)
1827 evsel->core.attr.branch_sample_type &= ~PERF_SAMPLE_BRANCH_HW_INDEX;
1828retry_sample_id:
1829 if (perf_missing_features.sample_id_all)
1830 evsel->core.attr.sample_id_all = 0;
1831
1832 display_attr(&evsel->core.attr);
1833
1834 for (cpu = start_cpu; cpu < end_cpu; cpu++) {
1835
1836 for (thread = 0; thread < nthreads; thread++) {
1837 int fd, group_fd;
1838
1839 if (!evsel->cgrp && !evsel->core.system_wide)
1840 pid = perf_thread_map__pid(threads, thread);
1841
1842 group_fd = get_group_fd(evsel, cpu, thread);
1843retry_open:
1844 test_attr__ready();
1845
1846 fd = perf_event_open(evsel, pid, cpus->map[cpu],
1847 group_fd, flags);
1848
1849 FD(evsel, cpu, thread) = fd;
1850
1851 bpf_counter__install_pe(evsel, cpu, fd);
1852
1853 if (unlikely(test_attr__enabled)) {
1854 test_attr__open(&evsel->core.attr, pid, cpus->map[cpu],
1855 fd, group_fd, flags);
1856 }
1857
1858 if (fd < 0) {
1859 err = -errno;
1860
1861 if (ignore_missing_thread(evsel, cpus->nr, cpu, threads, thread, err)) {
1862
1863
1864
1865
1866
1867 nthreads--;
1868 thread--;
1869
1870
1871 err = 0;
1872 continue;
1873 }
1874
1875 pr_debug2_peo("\nsys_perf_event_open failed, error %d\n",
1876 err);
1877 goto try_fallback;
1878 }
1879
1880 pr_debug2_peo(" = %d\n", fd);
1881
1882 if (evsel->bpf_fd >= 0) {
1883 int evt_fd = fd;
1884 int bpf_fd = evsel->bpf_fd;
1885
1886 err = ioctl(evt_fd,
1887 PERF_EVENT_IOC_SET_BPF,
1888 bpf_fd);
1889 if (err && errno != EEXIST) {
1890 pr_err("failed to attach bpf fd %d: %s\n",
1891 bpf_fd, strerror(errno));
1892 err = -EINVAL;
1893 goto out_close;
1894 }
1895 }
1896
1897 set_rlimit = NO_CHANGE;
1898
1899
1900
1901
1902
1903 if (perf_missing_features.clockid ||
1904 perf_missing_features.clockid_wrong) {
1905 err = -EINVAL;
1906 goto out_close;
1907 }
1908 }
1909 }
1910
1911 return 0;
1912
1913try_fallback:
1914
1915
1916
1917
1918 if (err == -EMFILE && set_rlimit < INCREASED_MAX) {
1919 struct rlimit l;
1920
1921 old_errno = errno;
1922 if (getrlimit(RLIMIT_NOFILE, &l) == 0) {
1923 if (set_rlimit == NO_CHANGE)
1924 l.rlim_cur = l.rlim_max;
1925 else {
1926 l.rlim_cur = l.rlim_max + 1000;
1927 l.rlim_max = l.rlim_cur;
1928 }
1929 if (setrlimit(RLIMIT_NOFILE, &l) == 0) {
1930 set_rlimit++;
1931 errno = old_errno;
1932 goto retry_open;
1933 }
1934 }
1935 errno = old_errno;
1936 }
1937
1938 if (err != -EINVAL || cpu > 0 || thread > 0)
1939 goto out_close;
1940
1941
1942
1943
1944
1945 if (!perf_missing_features.weight_struct &&
1946 (evsel->core.attr.sample_type & PERF_SAMPLE_WEIGHT_STRUCT)) {
1947 perf_missing_features.weight_struct = true;
1948 pr_debug2("switching off weight struct support\n");
1949 goto fallback_missing_features;
1950 } else if (!perf_missing_features.code_page_size &&
1951 (evsel->core.attr.sample_type & PERF_SAMPLE_CODE_PAGE_SIZE)) {
1952 perf_missing_features.code_page_size = true;
1953 pr_debug2_peo("Kernel has no PERF_SAMPLE_CODE_PAGE_SIZE support, bailing out\n");
1954 goto out_close;
1955 } else if (!perf_missing_features.data_page_size &&
1956 (evsel->core.attr.sample_type & PERF_SAMPLE_DATA_PAGE_SIZE)) {
1957 perf_missing_features.data_page_size = true;
1958 pr_debug2_peo("Kernel has no PERF_SAMPLE_DATA_PAGE_SIZE support, bailing out\n");
1959 goto out_close;
1960 } else if (!perf_missing_features.cgroup && evsel->core.attr.cgroup) {
1961 perf_missing_features.cgroup = true;
1962 pr_debug2_peo("Kernel has no cgroup sampling support, bailing out\n");
1963 goto out_close;
1964 } else if (!perf_missing_features.branch_hw_idx &&
1965 (evsel->core.attr.branch_sample_type & PERF_SAMPLE_BRANCH_HW_INDEX)) {
1966 perf_missing_features.branch_hw_idx = true;
1967 pr_debug2("switching off branch HW index support\n");
1968 goto fallback_missing_features;
1969 } else if (!perf_missing_features.aux_output && evsel->core.attr.aux_output) {
1970 perf_missing_features.aux_output = true;
1971 pr_debug2_peo("Kernel has no attr.aux_output support, bailing out\n");
1972 goto out_close;
1973 } else if (!perf_missing_features.bpf && evsel->core.attr.bpf_event) {
1974 perf_missing_features.bpf = true;
1975 pr_debug2_peo("switching off bpf_event\n");
1976 goto fallback_missing_features;
1977 } else if (!perf_missing_features.ksymbol && evsel->core.attr.ksymbol) {
1978 perf_missing_features.ksymbol = true;
1979 pr_debug2_peo("switching off ksymbol\n");
1980 goto fallback_missing_features;
1981 } else if (!perf_missing_features.write_backward && evsel->core.attr.write_backward) {
1982 perf_missing_features.write_backward = true;
1983 pr_debug2_peo("switching off write_backward\n");
1984 goto out_close;
1985 } else if (!perf_missing_features.clockid_wrong && evsel->core.attr.use_clockid) {
1986 perf_missing_features.clockid_wrong = true;
1987 pr_debug2_peo("switching off clockid\n");
1988 goto fallback_missing_features;
1989 } else if (!perf_missing_features.clockid && evsel->core.attr.use_clockid) {
1990 perf_missing_features.clockid = true;
1991 pr_debug2_peo("switching off use_clockid\n");
1992 goto fallback_missing_features;
1993 } else if (!perf_missing_features.cloexec && (flags & PERF_FLAG_FD_CLOEXEC)) {
1994 perf_missing_features.cloexec = true;
1995 pr_debug2_peo("switching off cloexec flag\n");
1996 goto fallback_missing_features;
1997 } else if (!perf_missing_features.mmap2 && evsel->core.attr.mmap2) {
1998 perf_missing_features.mmap2 = true;
1999 pr_debug2_peo("switching off mmap2\n");
2000 goto fallback_missing_features;
2001 } else if (!perf_missing_features.exclude_guest &&
2002 (evsel->core.attr.exclude_guest || evsel->core.attr.exclude_host)) {
2003 perf_missing_features.exclude_guest = true;
2004 pr_debug2_peo("switching off exclude_guest, exclude_host\n");
2005 goto fallback_missing_features;
2006 } else if (!perf_missing_features.sample_id_all) {
2007 perf_missing_features.sample_id_all = true;
2008 pr_debug2_peo("switching off sample_id_all\n");
2009 goto retry_sample_id;
2010 } else if (!perf_missing_features.lbr_flags &&
2011 (evsel->core.attr.branch_sample_type &
2012 (PERF_SAMPLE_BRANCH_NO_CYCLES |
2013 PERF_SAMPLE_BRANCH_NO_FLAGS))) {
2014 perf_missing_features.lbr_flags = true;
2015 pr_debug2_peo("switching off branch sample type no (cycles/flags)\n");
2016 goto fallback_missing_features;
2017 } else if (!perf_missing_features.group_read &&
2018 evsel->core.attr.inherit &&
2019 (evsel->core.attr.read_format & PERF_FORMAT_GROUP) &&
2020 evsel__is_group_leader(evsel)) {
2021 perf_missing_features.group_read = true;
2022 pr_debug2_peo("switching off group read\n");
2023 goto fallback_missing_features;
2024 }
2025out_close:
2026 if (err)
2027 threads->err_thread = thread;
2028
2029 old_errno = errno;
2030 do {
2031 while (--thread >= 0) {
2032 if (FD(evsel, cpu, thread) >= 0)
2033 close(FD(evsel, cpu, thread));
2034 FD(evsel, cpu, thread) = -1;
2035 }
2036 thread = nthreads;
2037 } while (--cpu >= 0);
2038 errno = old_errno;
2039 return err;
2040}
2041
2042int evsel__open(struct evsel *evsel, struct perf_cpu_map *cpus,
2043 struct perf_thread_map *threads)
2044{
2045 return evsel__open_cpu(evsel, cpus, threads, 0, cpus ? cpus->nr : 1);
2046}
2047
2048void evsel__close(struct evsel *evsel)
2049{
2050 perf_evsel__close(&evsel->core);
2051 perf_evsel__free_id(&evsel->core);
2052}
2053
2054int evsel__open_per_cpu(struct evsel *evsel, struct perf_cpu_map *cpus, int cpu)
2055{
2056 if (cpu == -1)
2057 return evsel__open_cpu(evsel, cpus, NULL, 0,
2058 cpus ? cpus->nr : 1);
2059
2060 return evsel__open_cpu(evsel, cpus, NULL, cpu, cpu + 1);
2061}
2062
2063int evsel__open_per_thread(struct evsel *evsel, struct perf_thread_map *threads)
2064{
2065 return evsel__open(evsel, NULL, threads);
2066}
2067
2068static int perf_evsel__parse_id_sample(const struct evsel *evsel,
2069 const union perf_event *event,
2070 struct perf_sample *sample)
2071{
2072 u64 type = evsel->core.attr.sample_type;
2073 const __u64 *array = event->sample.array;
2074 bool swapped = evsel->needs_swap;
2075 union u64_swap u;
2076
2077 array += ((event->header.size -
2078 sizeof(event->header)) / sizeof(u64)) - 1;
2079
2080 if (type & PERF_SAMPLE_IDENTIFIER) {
2081 sample->id = *array;
2082 array--;
2083 }
2084
2085 if (type & PERF_SAMPLE_CPU) {
2086 u.val64 = *array;
2087 if (swapped) {
2088
2089 u.val64 = bswap_64(u.val64);
2090 u.val32[0] = bswap_32(u.val32[0]);
2091 }
2092
2093 sample->cpu = u.val32[0];
2094 array--;
2095 }
2096
2097 if (type & PERF_SAMPLE_STREAM_ID) {
2098 sample->stream_id = *array;
2099 array--;
2100 }
2101
2102 if (type & PERF_SAMPLE_ID) {
2103 sample->id = *array;
2104 array--;
2105 }
2106
2107 if (type & PERF_SAMPLE_TIME) {
2108 sample->time = *array;
2109 array--;
2110 }
2111
2112 if (type & PERF_SAMPLE_TID) {
2113 u.val64 = *array;
2114 if (swapped) {
2115
2116 u.val64 = bswap_64(u.val64);
2117 u.val32[0] = bswap_32(u.val32[0]);
2118 u.val32[1] = bswap_32(u.val32[1]);
2119 }
2120
2121 sample->pid = u.val32[0];
2122 sample->tid = u.val32[1];
2123 array--;
2124 }
2125
2126 return 0;
2127}
2128
2129static inline bool overflow(const void *endp, u16 max_size, const void *offset,
2130 u64 size)
2131{
2132 return size > max_size || offset + size > endp;
2133}
2134
2135#define OVERFLOW_CHECK(offset, size, max_size) \
2136 do { \
2137 if (overflow(endp, (max_size), (offset), (size))) \
2138 return -EFAULT; \
2139 } while (0)
2140
2141#define OVERFLOW_CHECK_u64(offset) \
2142 OVERFLOW_CHECK(offset, sizeof(u64), sizeof(u64))
2143
2144static int
2145perf_event__check_size(union perf_event *event, unsigned int sample_size)
2146{
2147
2148
2149
2150
2151
2152 if (sample_size + sizeof(event->header) > event->header.size)
2153 return -EFAULT;
2154
2155 return 0;
2156}
2157
2158void __weak arch_perf_parse_sample_weight(struct perf_sample *data,
2159 const __u64 *array,
2160 u64 type __maybe_unused)
2161{
2162 data->weight = *array;
2163}
2164
2165int evsel__parse_sample(struct evsel *evsel, union perf_event *event,
2166 struct perf_sample *data)
2167{
2168 u64 type = evsel->core.attr.sample_type;
2169 bool swapped = evsel->needs_swap;
2170 const __u64 *array;
2171 u16 max_size = event->header.size;
2172 const void *endp = (void *)event + max_size;
2173 u64 sz;
2174
2175
2176
2177
2178
2179 union u64_swap u;
2180
2181 memset(data, 0, sizeof(*data));
2182 data->cpu = data->pid = data->tid = -1;
2183 data->stream_id = data->id = data->time = -1ULL;
2184 data->period = evsel->core.attr.sample_period;
2185 data->cpumode = event->header.misc & PERF_RECORD_MISC_CPUMODE_MASK;
2186 data->misc = event->header.misc;
2187 data->id = -1ULL;
2188 data->data_src = PERF_MEM_DATA_SRC_NONE;
2189
2190 if (event->header.type != PERF_RECORD_SAMPLE) {
2191 if (!evsel->core.attr.sample_id_all)
2192 return 0;
2193 return perf_evsel__parse_id_sample(evsel, event, data);
2194 }
2195
2196 array = event->sample.array;
2197
2198 if (perf_event__check_size(event, evsel->sample_size))
2199 return -EFAULT;
2200
2201 if (type & PERF_SAMPLE_IDENTIFIER) {
2202 data->id = *array;
2203 array++;
2204 }
2205
2206 if (type & PERF_SAMPLE_IP) {
2207 data->ip = *array;
2208 array++;
2209 }
2210
2211 if (type & PERF_SAMPLE_TID) {
2212 u.val64 = *array;
2213 if (swapped) {
2214
2215 u.val64 = bswap_64(u.val64);
2216 u.val32[0] = bswap_32(u.val32[0]);
2217 u.val32[1] = bswap_32(u.val32[1]);
2218 }
2219
2220 data->pid = u.val32[0];
2221 data->tid = u.val32[1];
2222 array++;
2223 }
2224
2225 if (type & PERF_SAMPLE_TIME) {
2226 data->time = *array;
2227 array++;
2228 }
2229
2230 if (type & PERF_SAMPLE_ADDR) {
2231 data->addr = *array;
2232 array++;
2233 }
2234
2235 if (type & PERF_SAMPLE_ID) {
2236 data->id = *array;
2237 array++;
2238 }
2239
2240 if (type & PERF_SAMPLE_STREAM_ID) {
2241 data->stream_id = *array;
2242 array++;
2243 }
2244
2245 if (type & PERF_SAMPLE_CPU) {
2246
2247 u.val64 = *array;
2248 if (swapped) {
2249
2250 u.val64 = bswap_64(u.val64);
2251 u.val32[0] = bswap_32(u.val32[0]);
2252 }
2253
2254 data->cpu = u.val32[0];
2255 array++;
2256 }
2257
2258 if (type & PERF_SAMPLE_PERIOD) {
2259 data->period = *array;
2260 array++;
2261 }
2262
2263 if (type & PERF_SAMPLE_READ) {
2264 u64 read_format = evsel->core.attr.read_format;
2265
2266 OVERFLOW_CHECK_u64(array);
2267 if (read_format & PERF_FORMAT_GROUP)
2268 data->read.group.nr = *array;
2269 else
2270 data->read.one.value = *array;
2271
2272 array++;
2273
2274 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
2275 OVERFLOW_CHECK_u64(array);
2276 data->read.time_enabled = *array;
2277 array++;
2278 }
2279
2280 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
2281 OVERFLOW_CHECK_u64(array);
2282 data->read.time_running = *array;
2283 array++;
2284 }
2285
2286
2287 if (read_format & PERF_FORMAT_GROUP) {
2288 const u64 max_group_nr = UINT64_MAX /
2289 sizeof(struct sample_read_value);
2290
2291 if (data->read.group.nr > max_group_nr)
2292 return -EFAULT;
2293 sz = data->read.group.nr *
2294 sizeof(struct sample_read_value);
2295 OVERFLOW_CHECK(array, sz, max_size);
2296 data->read.group.values =
2297 (struct sample_read_value *)array;
2298 array = (void *)array + sz;
2299 } else {
2300 OVERFLOW_CHECK_u64(array);
2301 data->read.one.id = *array;
2302 array++;
2303 }
2304 }
2305
2306 if (type & PERF_SAMPLE_CALLCHAIN) {
2307 const u64 max_callchain_nr = UINT64_MAX / sizeof(u64);
2308
2309 OVERFLOW_CHECK_u64(array);
2310 data->callchain = (struct ip_callchain *)array++;
2311 if (data->callchain->nr > max_callchain_nr)
2312 return -EFAULT;
2313 sz = data->callchain->nr * sizeof(u64);
2314 OVERFLOW_CHECK(array, sz, max_size);
2315 array = (void *)array + sz;
2316 }
2317
2318 if (type & PERF_SAMPLE_RAW) {
2319 OVERFLOW_CHECK_u64(array);
2320 u.val64 = *array;
2321
2322
2323
2324
2325
2326
2327
2328 if (swapped) {
2329 u.val64 = bswap_64(u.val64);
2330 u.val32[0] = bswap_32(u.val32[0]);
2331 u.val32[1] = bswap_32(u.val32[1]);
2332 }
2333 data->raw_size = u.val32[0];
2334
2335
2336
2337
2338
2339 if (swapped)
2340 mem_bswap_64((void *) array, data->raw_size);
2341
2342 array = (void *)array + sizeof(u32);
2343
2344 OVERFLOW_CHECK(array, data->raw_size, max_size);
2345 data->raw_data = (void *)array;
2346 array = (void *)array + data->raw_size;
2347 }
2348
2349 if (type & PERF_SAMPLE_BRANCH_STACK) {
2350 const u64 max_branch_nr = UINT64_MAX /
2351 sizeof(struct branch_entry);
2352
2353 OVERFLOW_CHECK_u64(array);
2354 data->branch_stack = (struct branch_stack *)array++;
2355
2356 if (data->branch_stack->nr > max_branch_nr)
2357 return -EFAULT;
2358
2359 sz = data->branch_stack->nr * sizeof(struct branch_entry);
2360 if (evsel__has_branch_hw_idx(evsel))
2361 sz += sizeof(u64);
2362 else
2363 data->no_hw_idx = true;
2364 OVERFLOW_CHECK(array, sz, max_size);
2365 array = (void *)array + sz;
2366 }
2367
2368 if (type & PERF_SAMPLE_REGS_USER) {
2369 OVERFLOW_CHECK_u64(array);
2370 data->user_regs.abi = *array;
2371 array++;
2372
2373 if (data->user_regs.abi) {
2374 u64 mask = evsel->core.attr.sample_regs_user;
2375
2376 sz = hweight64(mask) * sizeof(u64);
2377 OVERFLOW_CHECK(array, sz, max_size);
2378 data->user_regs.mask = mask;
2379 data->user_regs.regs = (u64 *)array;
2380 array = (void *)array + sz;
2381 }
2382 }
2383
2384 if (type & PERF_SAMPLE_STACK_USER) {
2385 OVERFLOW_CHECK_u64(array);
2386 sz = *array++;
2387
2388 data->user_stack.offset = ((char *)(array - 1)
2389 - (char *) event);
2390
2391 if (!sz) {
2392 data->user_stack.size = 0;
2393 } else {
2394 OVERFLOW_CHECK(array, sz, max_size);
2395 data->user_stack.data = (char *)array;
2396 array = (void *)array + sz;
2397 OVERFLOW_CHECK_u64(array);
2398 data->user_stack.size = *array++;
2399 if (WARN_ONCE(data->user_stack.size > sz,
2400 "user stack dump failure\n"))
2401 return -EFAULT;
2402 }
2403 }
2404
2405 if (type & PERF_SAMPLE_WEIGHT_TYPE) {
2406 OVERFLOW_CHECK_u64(array);
2407 arch_perf_parse_sample_weight(data, array, type);
2408 array++;
2409 }
2410
2411 if (type & PERF_SAMPLE_DATA_SRC) {
2412 OVERFLOW_CHECK_u64(array);
2413 data->data_src = *array;
2414 array++;
2415 }
2416
2417 if (type & PERF_SAMPLE_TRANSACTION) {
2418 OVERFLOW_CHECK_u64(array);
2419 data->transaction = *array;
2420 array++;
2421 }
2422
2423 data->intr_regs.abi = PERF_SAMPLE_REGS_ABI_NONE;
2424 if (type & PERF_SAMPLE_REGS_INTR) {
2425 OVERFLOW_CHECK_u64(array);
2426 data->intr_regs.abi = *array;
2427 array++;
2428
2429 if (data->intr_regs.abi != PERF_SAMPLE_REGS_ABI_NONE) {
2430 u64 mask = evsel->core.attr.sample_regs_intr;
2431
2432 sz = hweight64(mask) * sizeof(u64);
2433 OVERFLOW_CHECK(array, sz, max_size);
2434 data->intr_regs.mask = mask;
2435 data->intr_regs.regs = (u64 *)array;
2436 array = (void *)array + sz;
2437 }
2438 }
2439
2440 data->phys_addr = 0;
2441 if (type & PERF_SAMPLE_PHYS_ADDR) {
2442 data->phys_addr = *array;
2443 array++;
2444 }
2445
2446 data->cgroup = 0;
2447 if (type & PERF_SAMPLE_CGROUP) {
2448 data->cgroup = *array;
2449 array++;
2450 }
2451
2452 data->data_page_size = 0;
2453 if (type & PERF_SAMPLE_DATA_PAGE_SIZE) {
2454 data->data_page_size = *array;
2455 array++;
2456 }
2457
2458 data->code_page_size = 0;
2459 if (type & PERF_SAMPLE_CODE_PAGE_SIZE) {
2460 data->code_page_size = *array;
2461 array++;
2462 }
2463
2464 if (type & PERF_SAMPLE_AUX) {
2465 OVERFLOW_CHECK_u64(array);
2466 sz = *array++;
2467
2468 OVERFLOW_CHECK(array, sz, max_size);
2469
2470 if (swapped)
2471 mem_bswap_64((char *)array, sz);
2472 data->aux_sample.size = sz;
2473 data->aux_sample.data = (char *)array;
2474 array = (void *)array + sz;
2475 }
2476
2477 return 0;
2478}
2479
2480int evsel__parse_sample_timestamp(struct evsel *evsel, union perf_event *event,
2481 u64 *timestamp)
2482{
2483 u64 type = evsel->core.attr.sample_type;
2484 const __u64 *array;
2485
2486 if (!(type & PERF_SAMPLE_TIME))
2487 return -1;
2488
2489 if (event->header.type != PERF_RECORD_SAMPLE) {
2490 struct perf_sample data = {
2491 .time = -1ULL,
2492 };
2493
2494 if (!evsel->core.attr.sample_id_all)
2495 return -1;
2496 if (perf_evsel__parse_id_sample(evsel, event, &data))
2497 return -1;
2498
2499 *timestamp = data.time;
2500 return 0;
2501 }
2502
2503 array = event->sample.array;
2504
2505 if (perf_event__check_size(event, evsel->sample_size))
2506 return -EFAULT;
2507
2508 if (type & PERF_SAMPLE_IDENTIFIER)
2509 array++;
2510
2511 if (type & PERF_SAMPLE_IP)
2512 array++;
2513
2514 if (type & PERF_SAMPLE_TID)
2515 array++;
2516
2517 if (type & PERF_SAMPLE_TIME)
2518 *timestamp = *array;
2519
2520 return 0;
2521}
2522
2523struct tep_format_field *evsel__field(struct evsel *evsel, const char *name)
2524{
2525 return tep_find_field(evsel->tp_format, name);
2526}
2527
2528void *evsel__rawptr(struct evsel *evsel, struct perf_sample *sample, const char *name)
2529{
2530 struct tep_format_field *field = evsel__field(evsel, name);
2531 int offset;
2532
2533 if (!field)
2534 return NULL;
2535
2536 offset = field->offset;
2537
2538 if (field->flags & TEP_FIELD_IS_DYNAMIC) {
2539 offset = *(int *)(sample->raw_data + field->offset);
2540 offset &= 0xffff;
2541 }
2542
2543 return sample->raw_data + offset;
2544}
2545
2546u64 format_field__intval(struct tep_format_field *field, struct perf_sample *sample,
2547 bool needs_swap)
2548{
2549 u64 value;
2550 void *ptr = sample->raw_data + field->offset;
2551
2552 switch (field->size) {
2553 case 1:
2554 return *(u8 *)ptr;
2555 case 2:
2556 value = *(u16 *)ptr;
2557 break;
2558 case 4:
2559 value = *(u32 *)ptr;
2560 break;
2561 case 8:
2562 memcpy(&value, ptr, sizeof(u64));
2563 break;
2564 default:
2565 return 0;
2566 }
2567
2568 if (!needs_swap)
2569 return value;
2570
2571 switch (field->size) {
2572 case 2:
2573 return bswap_16(value);
2574 case 4:
2575 return bswap_32(value);
2576 case 8:
2577 return bswap_64(value);
2578 default:
2579 return 0;
2580 }
2581
2582 return 0;
2583}
2584
2585u64 evsel__intval(struct evsel *evsel, struct perf_sample *sample, const char *name)
2586{
2587 struct tep_format_field *field = evsel__field(evsel, name);
2588
2589 if (!field)
2590 return 0;
2591
2592 return field ? format_field__intval(field, sample, evsel->needs_swap) : 0;
2593}
2594
2595bool evsel__fallback(struct evsel *evsel, int err, char *msg, size_t msgsize)
2596{
2597 int paranoid;
2598
2599 if ((err == ENOENT || err == ENXIO || err == ENODEV) &&
2600 evsel->core.attr.type == PERF_TYPE_HARDWARE &&
2601 evsel->core.attr.config == PERF_COUNT_HW_CPU_CYCLES) {
2602
2603
2604
2605
2606
2607
2608
2609
2610 scnprintf(msg, msgsize, "%s",
2611"The cycles event is not supported, trying to fall back to cpu-clock-ticks");
2612
2613 evsel->core.attr.type = PERF_TYPE_SOFTWARE;
2614 evsel->core.attr.config = PERF_COUNT_SW_CPU_CLOCK;
2615
2616 zfree(&evsel->name);
2617 return true;
2618 } else if (err == EACCES && !evsel->core.attr.exclude_kernel &&
2619 (paranoid = perf_event_paranoid()) > 1) {
2620 const char *name = evsel__name(evsel);
2621 char *new_name;
2622 const char *sep = ":";
2623
2624
2625 if (evsel->core.attr.exclude_user)
2626 return false;
2627
2628
2629 if (strchr(name, '/') ||
2630 (strchr(name, ':') && !evsel->is_libpfm_event))
2631 sep = "";
2632
2633 if (asprintf(&new_name, "%s%su", name, sep) < 0)
2634 return false;
2635
2636 if (evsel->name)
2637 free(evsel->name);
2638 evsel->name = new_name;
2639 scnprintf(msg, msgsize, "kernel.perf_event_paranoid=%d, trying "
2640 "to fall back to excluding kernel and hypervisor "
2641 " samples", paranoid);
2642 evsel->core.attr.exclude_kernel = 1;
2643 evsel->core.attr.exclude_hv = 1;
2644
2645 return true;
2646 }
2647
2648 return false;
2649}
2650
2651static bool find_process(const char *name)
2652{
2653 size_t len = strlen(name);
2654 DIR *dir;
2655 struct dirent *d;
2656 int ret = -1;
2657
2658 dir = opendir(procfs__mountpoint());
2659 if (!dir)
2660 return false;
2661
2662
2663 while (ret && (d = readdir(dir)) != NULL) {
2664 char path[PATH_MAX];
2665 char *data;
2666 size_t size;
2667
2668 if ((d->d_type != DT_DIR) ||
2669 !strcmp(".", d->d_name) ||
2670 !strcmp("..", d->d_name))
2671 continue;
2672
2673 scnprintf(path, sizeof(path), "%s/%s/comm",
2674 procfs__mountpoint(), d->d_name);
2675
2676 if (filename__read_str(path, &data, &size))
2677 continue;
2678
2679 ret = strncmp(name, data, len);
2680 free(data);
2681 }
2682
2683 closedir(dir);
2684 return ret ? false : true;
2685}
2686
2687int evsel__open_strerror(struct evsel *evsel, struct target *target,
2688 int err, char *msg, size_t size)
2689{
2690 char sbuf[STRERR_BUFSIZE];
2691 int printed = 0, enforced = 0;
2692
2693 switch (err) {
2694 case EPERM:
2695 case EACCES:
2696 printed += scnprintf(msg + printed, size - printed,
2697 "Access to performance monitoring and observability operations is limited.\n");
2698
2699 if (!sysfs__read_int("fs/selinux/enforce", &enforced)) {
2700 if (enforced) {
2701 printed += scnprintf(msg + printed, size - printed,
2702 "Enforced MAC policy settings (SELinux) can limit access to performance\n"
2703 "monitoring and observability operations. Inspect system audit records for\n"
2704 "more perf_event access control information and adjusting the policy.\n");
2705 }
2706 }
2707
2708 if (err == EPERM)
2709 printed += scnprintf(msg, size,
2710 "No permission to enable %s event.\n\n", evsel__name(evsel));
2711
2712 return scnprintf(msg + printed, size - printed,
2713 "Consider adjusting /proc/sys/kernel/perf_event_paranoid setting to open\n"
2714 "access to performance monitoring and observability operations for processes\n"
2715 "without CAP_PERFMON, CAP_SYS_PTRACE or CAP_SYS_ADMIN Linux capability.\n"
2716 "More information can be found at 'Perf events and tool security' document:\n"
2717 "https://www.kernel.org/doc/html/latest/admin-guide/perf-security.html\n"
2718 "perf_event_paranoid setting is %d:\n"
2719 " -1: Allow use of (almost) all events by all users\n"
2720 " Ignore mlock limit after perf_event_mlock_kb without CAP_IPC_LOCK\n"
2721 ">= 0: Disallow raw and ftrace function tracepoint access\n"
2722 ">= 1: Disallow CPU event access\n"
2723 ">= 2: Disallow kernel profiling\n"
2724 "To make the adjusted perf_event_paranoid setting permanent preserve it\n"
2725 "in /etc/sysctl.conf (e.g. kernel.perf_event_paranoid = <setting>)",
2726 perf_event_paranoid());
2727 case ENOENT:
2728 return scnprintf(msg, size, "The %s event is not supported.", evsel__name(evsel));
2729 case EMFILE:
2730 return scnprintf(msg, size, "%s",
2731 "Too many events are opened.\n"
2732 "Probably the maximum number of open file descriptors has been reached.\n"
2733 "Hint: Try again after reducing the number of events.\n"
2734 "Hint: Try increasing the limit with 'ulimit -n <limit>'");
2735 case ENOMEM:
2736 if (evsel__has_callchain(evsel) &&
2737 access("/proc/sys/kernel/perf_event_max_stack", F_OK) == 0)
2738 return scnprintf(msg, size,
2739 "Not enough memory to setup event with callchain.\n"
2740 "Hint: Try tweaking /proc/sys/kernel/perf_event_max_stack\n"
2741 "Hint: Current value: %d", sysctl__max_stack());
2742 break;
2743 case ENODEV:
2744 if (target->cpu_list)
2745 return scnprintf(msg, size, "%s",
2746 "No such device - did you specify an out-of-range profile CPU?");
2747 break;
2748 case EOPNOTSUPP:
2749 if (evsel->core.attr.aux_output)
2750 return scnprintf(msg, size,
2751 "%s: PMU Hardware doesn't support 'aux_output' feature",
2752 evsel__name(evsel));
2753 if (evsel->core.attr.sample_period != 0)
2754 return scnprintf(msg, size,
2755 "%s: PMU Hardware doesn't support sampling/overflow-interrupts. Try 'perf stat'",
2756 evsel__name(evsel));
2757 if (evsel->core.attr.precise_ip)
2758 return scnprintf(msg, size, "%s",
2759 "\'precise\' request may not be supported. Try removing 'p' modifier.");
2760#if defined(__i386__) || defined(__x86_64__)
2761 if (evsel->core.attr.type == PERF_TYPE_HARDWARE)
2762 return scnprintf(msg, size, "%s",
2763 "No hardware sampling interrupt available.\n");
2764#endif
2765 break;
2766 case EBUSY:
2767 if (find_process("oprofiled"))
2768 return scnprintf(msg, size,
2769 "The PMU counters are busy/taken by another profiler.\n"
2770 "We found oprofile daemon running, please stop it and try again.");
2771 break;
2772 case EINVAL:
2773 if (evsel->core.attr.sample_type & PERF_SAMPLE_CODE_PAGE_SIZE && perf_missing_features.code_page_size)
2774 return scnprintf(msg, size, "Asking for the code page size isn't supported by this kernel.");
2775 if (evsel->core.attr.sample_type & PERF_SAMPLE_DATA_PAGE_SIZE && perf_missing_features.data_page_size)
2776 return scnprintf(msg, size, "Asking for the data page size isn't supported by this kernel.");
2777 if (evsel->core.attr.write_backward && perf_missing_features.write_backward)
2778 return scnprintf(msg, size, "Reading from overwrite event is not supported by this kernel.");
2779 if (perf_missing_features.clockid)
2780 return scnprintf(msg, size, "clockid feature not supported.");
2781 if (perf_missing_features.clockid_wrong)
2782 return scnprintf(msg, size, "wrong clockid (%d).", clockid);
2783 if (perf_missing_features.aux_output)
2784 return scnprintf(msg, size, "The 'aux_output' feature is not supported, update the kernel.");
2785 break;
2786 case ENODATA:
2787 return scnprintf(msg, size, "Cannot collect data source with the load latency event alone. "
2788 "Please add an auxiliary event in front of the load latency event.");
2789 default:
2790 break;
2791 }
2792
2793 return scnprintf(msg, size,
2794 "The sys_perf_event_open() syscall returned with %d (%s) for event (%s).\n"
2795 "/bin/dmesg | grep -i perf may provide additional information.\n",
2796 err, str_error_r(err, sbuf, sizeof(sbuf)), evsel__name(evsel));
2797}
2798
2799struct perf_env *evsel__env(struct evsel *evsel)
2800{
2801 if (evsel && evsel->evlist)
2802 return evsel->evlist->env;
2803 return &perf_env;
2804}
2805
2806static int store_evsel_ids(struct evsel *evsel, struct evlist *evlist)
2807{
2808 int cpu, thread;
2809
2810 for (cpu = 0; cpu < xyarray__max_x(evsel->core.fd); cpu++) {
2811 for (thread = 0; thread < xyarray__max_y(evsel->core.fd);
2812 thread++) {
2813 int fd = FD(evsel, cpu, thread);
2814
2815 if (perf_evlist__id_add_fd(&evlist->core, &evsel->core,
2816 cpu, thread, fd) < 0)
2817 return -1;
2818 }
2819 }
2820
2821 return 0;
2822}
2823
2824int evsel__store_ids(struct evsel *evsel, struct evlist *evlist)
2825{
2826 struct perf_cpu_map *cpus = evsel->core.cpus;
2827 struct perf_thread_map *threads = evsel->core.threads;
2828
2829 if (perf_evsel__alloc_id(&evsel->core, cpus->nr, threads->nr))
2830 return -ENOMEM;
2831
2832 return store_evsel_ids(evsel, evlist);
2833}
2834
2835void evsel__zero_per_pkg(struct evsel *evsel)
2836{
2837 struct hashmap_entry *cur;
2838 size_t bkt;
2839
2840 if (evsel->per_pkg_mask) {
2841 hashmap__for_each_entry(evsel->per_pkg_mask, cur, bkt)
2842 free((char *)cur->key);
2843
2844 hashmap__clear(evsel->per_pkg_mask);
2845 }
2846}
2847
2848bool evsel__is_hybrid(struct evsel *evsel)
2849{
2850 return evsel->pmu_name && perf_pmu__is_hybrid(evsel->pmu_name);
2851}
2852
2853struct evsel *evsel__leader(struct evsel *evsel)
2854{
2855 return container_of(evsel->core.leader, struct evsel, core);
2856}
2857
2858bool evsel__has_leader(struct evsel *evsel, struct evsel *leader)
2859{
2860 return evsel->core.leader == &leader->core;
2861}
2862
2863bool evsel__is_leader(struct evsel *evsel)
2864{
2865 return evsel__has_leader(evsel, evsel);
2866}
2867
2868void evsel__set_leader(struct evsel *evsel, struct evsel *leader)
2869{
2870 evsel->core.leader = &leader->core;
2871}
2872