1
2
3
4
5
6
7
8
9
10#include <byteswap.h>
11#include <errno.h>
12#include <inttypes.h>
13#include <linux/bitops.h>
14#include <api/fs/fs.h>
15#include <api/fs/tracing_path.h>
16#include <traceevent/event-parse.h>
17#include <linux/hw_breakpoint.h>
18#include <linux/perf_event.h>
19#include <linux/compiler.h>
20#include <linux/err.h>
21#include <linux/zalloc.h>
22#include <sys/ioctl.h>
23#include <sys/resource.h>
24#include <sys/types.h>
25#include <dirent.h>
26#include <stdlib.h>
27#include <perf/evsel.h>
28#include "asm/bug.h"
29#include "bpf_counter.h"
30#include "callchain.h"
31#include "cgroup.h"
32#include "counts.h"
33#include "event.h"
34#include "evsel.h"
35#include "util/env.h"
36#include "util/evsel_config.h"
37#include "util/evsel_fprintf.h"
38#include "evlist.h"
39#include <perf/cpumap.h>
40#include "thread_map.h"
41#include "target.h"
42#include "perf_regs.h"
43#include "record.h"
44#include "debug.h"
45#include "trace-event.h"
46#include "stat.h"
47#include "string2.h"
48#include "memswap.h"
49#include "util.h"
50#include "hashmap.h"
51#include "pmu-hybrid.h"
52#include "../perf-sys.h"
53#include "util/parse-branch-options.h"
54#include <internal/xyarray.h>
55#include <internal/lib.h>
56
57#include <linux/ctype.h>
58
59struct perf_missing_features perf_missing_features;
60
61static clockid_t clockid;
62
63static int evsel__no_extra_init(struct evsel *evsel __maybe_unused)
64{
65 return 0;
66}
67
68void __weak test_attr__ready(void) { }
69
70static void evsel__no_extra_fini(struct evsel *evsel __maybe_unused)
71{
72}
73
74static struct {
75 size_t size;
76 int (*init)(struct evsel *evsel);
77 void (*fini)(struct evsel *evsel);
78} perf_evsel__object = {
79 .size = sizeof(struct evsel),
80 .init = evsel__no_extra_init,
81 .fini = evsel__no_extra_fini,
82};
83
84int evsel__object_config(size_t object_size, int (*init)(struct evsel *evsel),
85 void (*fini)(struct evsel *evsel))
86{
87
88 if (object_size == 0)
89 goto set_methods;
90
91 if (perf_evsel__object.size > object_size)
92 return -EINVAL;
93
94 perf_evsel__object.size = object_size;
95
96set_methods:
97 if (init != NULL)
98 perf_evsel__object.init = init;
99
100 if (fini != NULL)
101 perf_evsel__object.fini = fini;
102
103 return 0;
104}
105
106#define FD(e, x, y) (*(int *)xyarray__entry(e->core.fd, x, y))
107
108int __evsel__sample_size(u64 sample_type)
109{
110 u64 mask = sample_type & PERF_SAMPLE_MASK;
111 int size = 0;
112 int i;
113
114 for (i = 0; i < 64; i++) {
115 if (mask & (1ULL << i))
116 size++;
117 }
118
119 size *= sizeof(u64);
120
121 return size;
122}
123
124
125
126
127
128
129
130
131
132static int __perf_evsel__calc_id_pos(u64 sample_type)
133{
134 int idx = 0;
135
136 if (sample_type & PERF_SAMPLE_IDENTIFIER)
137 return 0;
138
139 if (!(sample_type & PERF_SAMPLE_ID))
140 return -1;
141
142 if (sample_type & PERF_SAMPLE_IP)
143 idx += 1;
144
145 if (sample_type & PERF_SAMPLE_TID)
146 idx += 1;
147
148 if (sample_type & PERF_SAMPLE_TIME)
149 idx += 1;
150
151 if (sample_type & PERF_SAMPLE_ADDR)
152 idx += 1;
153
154 return idx;
155}
156
157
158
159
160
161
162
163
164
165static int __perf_evsel__calc_is_pos(u64 sample_type)
166{
167 int idx = 1;
168
169 if (sample_type & PERF_SAMPLE_IDENTIFIER)
170 return 1;
171
172 if (!(sample_type & PERF_SAMPLE_ID))
173 return -1;
174
175 if (sample_type & PERF_SAMPLE_CPU)
176 idx += 1;
177
178 if (sample_type & PERF_SAMPLE_STREAM_ID)
179 idx += 1;
180
181 return idx;
182}
183
184void evsel__calc_id_pos(struct evsel *evsel)
185{
186 evsel->id_pos = __perf_evsel__calc_id_pos(evsel->core.attr.sample_type);
187 evsel->is_pos = __perf_evsel__calc_is_pos(evsel->core.attr.sample_type);
188}
189
190void __evsel__set_sample_bit(struct evsel *evsel,
191 enum perf_event_sample_format bit)
192{
193 if (!(evsel->core.attr.sample_type & bit)) {
194 evsel->core.attr.sample_type |= bit;
195 evsel->sample_size += sizeof(u64);
196 evsel__calc_id_pos(evsel);
197 }
198}
199
200void __evsel__reset_sample_bit(struct evsel *evsel,
201 enum perf_event_sample_format bit)
202{
203 if (evsel->core.attr.sample_type & bit) {
204 evsel->core.attr.sample_type &= ~bit;
205 evsel->sample_size -= sizeof(u64);
206 evsel__calc_id_pos(evsel);
207 }
208}
209
210void evsel__set_sample_id(struct evsel *evsel,
211 bool can_sample_identifier)
212{
213 if (can_sample_identifier) {
214 evsel__reset_sample_bit(evsel, ID);
215 evsel__set_sample_bit(evsel, IDENTIFIER);
216 } else {
217 evsel__set_sample_bit(evsel, ID);
218 }
219 evsel->core.attr.read_format |= PERF_FORMAT_ID;
220}
221
222
223
224
225
226
227
228
229
230bool evsel__is_function_event(struct evsel *evsel)
231{
232#define FUNCTION_EVENT "ftrace:function"
233
234 return evsel->name &&
235 !strncmp(FUNCTION_EVENT, evsel->name, sizeof(FUNCTION_EVENT));
236
237#undef FUNCTION_EVENT
238}
239
240void evsel__init(struct evsel *evsel,
241 struct perf_event_attr *attr, int idx)
242{
243 perf_evsel__init(&evsel->core, attr, idx);
244 evsel->tracking = !idx;
245 evsel->unit = "";
246 evsel->scale = 1.0;
247 evsel->max_events = ULONG_MAX;
248 evsel->evlist = NULL;
249 evsel->bpf_obj = NULL;
250 evsel->bpf_fd = -1;
251 INIT_LIST_HEAD(&evsel->config_terms);
252 INIT_LIST_HEAD(&evsel->bpf_counter_list);
253 perf_evsel__object.init(evsel);
254 evsel->sample_size = __evsel__sample_size(attr->sample_type);
255 evsel__calc_id_pos(evsel);
256 evsel->cmdline_group_boundary = false;
257 evsel->metric_expr = NULL;
258 evsel->metric_name = NULL;
259 evsel->metric_events = NULL;
260 evsel->per_pkg_mask = NULL;
261 evsel->collect_stat = false;
262 evsel->pmu_name = NULL;
263}
264
265struct evsel *evsel__new_idx(struct perf_event_attr *attr, int idx)
266{
267 struct evsel *evsel = zalloc(perf_evsel__object.size);
268
269 if (!evsel)
270 return NULL;
271 evsel__init(evsel, attr, idx);
272
273 if (evsel__is_bpf_output(evsel)) {
274 evsel->core.attr.sample_type |= (PERF_SAMPLE_RAW | PERF_SAMPLE_TIME |
275 PERF_SAMPLE_CPU | PERF_SAMPLE_PERIOD),
276 evsel->core.attr.sample_period = 1;
277 }
278
279 if (evsel__is_clock(evsel)) {
280
281
282
283
284 static const char *unit = "msec";
285
286 evsel->unit = unit;
287 evsel->scale = 1e-6;
288 }
289
290 return evsel;
291}
292
293static bool perf_event_can_profile_kernel(void)
294{
295 return perf_event_paranoid_check(1);
296}
297
298struct evsel *evsel__new_cycles(bool precise, __u32 type, __u64 config)
299{
300 struct perf_event_attr attr = {
301 .type = type,
302 .config = config,
303 .exclude_kernel = !perf_event_can_profile_kernel(),
304 };
305 struct evsel *evsel;
306
307 event_attr_init(&attr);
308
309 if (!precise)
310 goto new_event;
311
312
313
314
315
316new_event:
317 evsel = evsel__new(&attr);
318 if (evsel == NULL)
319 goto out;
320
321 evsel->precise_max = true;
322
323
324 if (asprintf(&evsel->name, "cycles%s%s%.*s",
325 (attr.precise_ip || attr.exclude_kernel) ? ":" : "",
326 attr.exclude_kernel ? "u" : "",
327 attr.precise_ip ? attr.precise_ip + 1 : 0, "ppp") < 0)
328 goto error_free;
329out:
330 return evsel;
331error_free:
332 evsel__delete(evsel);
333 evsel = NULL;
334 goto out;
335}
336
337static int evsel__copy_config_terms(struct evsel *dst, struct evsel *src)
338{
339 struct evsel_config_term *pos, *tmp;
340
341 list_for_each_entry(pos, &src->config_terms, list) {
342 tmp = malloc(sizeof(*tmp));
343 if (tmp == NULL)
344 return -ENOMEM;
345
346 *tmp = *pos;
347 if (tmp->free_str) {
348 tmp->val.str = strdup(pos->val.str);
349 if (tmp->val.str == NULL) {
350 free(tmp);
351 return -ENOMEM;
352 }
353 }
354 list_add_tail(&tmp->list, &dst->config_terms);
355 }
356 return 0;
357}
358
359
360
361
362
363
364
365
366struct evsel *evsel__clone(struct evsel *orig)
367{
368 struct evsel *evsel;
369
370 BUG_ON(orig->core.fd);
371 BUG_ON(orig->counts);
372 BUG_ON(orig->priv);
373 BUG_ON(orig->per_pkg_mask);
374
375
376 if (orig->bpf_obj)
377 return NULL;
378
379 evsel = evsel__new(&orig->core.attr);
380 if (evsel == NULL)
381 return NULL;
382
383 evsel->core.cpus = perf_cpu_map__get(orig->core.cpus);
384 evsel->core.own_cpus = perf_cpu_map__get(orig->core.own_cpus);
385 evsel->core.threads = perf_thread_map__get(orig->core.threads);
386 evsel->core.nr_members = orig->core.nr_members;
387 evsel->core.system_wide = orig->core.system_wide;
388
389 if (orig->name) {
390 evsel->name = strdup(orig->name);
391 if (evsel->name == NULL)
392 goto out_err;
393 }
394 if (orig->group_name) {
395 evsel->group_name = strdup(orig->group_name);
396 if (evsel->group_name == NULL)
397 goto out_err;
398 }
399 if (orig->pmu_name) {
400 evsel->pmu_name = strdup(orig->pmu_name);
401 if (evsel->pmu_name == NULL)
402 goto out_err;
403 }
404 if (orig->filter) {
405 evsel->filter = strdup(orig->filter);
406 if (evsel->filter == NULL)
407 goto out_err;
408 }
409 evsel->cgrp = cgroup__get(orig->cgrp);
410 evsel->tp_format = orig->tp_format;
411 evsel->handler = orig->handler;
412 evsel->core.leader = orig->core.leader;
413
414 evsel->max_events = orig->max_events;
415 evsel->tool_event = orig->tool_event;
416 evsel->unit = orig->unit;
417 evsel->scale = orig->scale;
418 evsel->snapshot = orig->snapshot;
419 evsel->per_pkg = orig->per_pkg;
420 evsel->percore = orig->percore;
421 evsel->precise_max = orig->precise_max;
422 evsel->use_uncore_alias = orig->use_uncore_alias;
423 evsel->is_libpfm_event = orig->is_libpfm_event;
424
425 evsel->exclude_GH = orig->exclude_GH;
426 evsel->sample_read = orig->sample_read;
427 evsel->auto_merge_stats = orig->auto_merge_stats;
428 evsel->collect_stat = orig->collect_stat;
429 evsel->weak_group = orig->weak_group;
430 evsel->use_config_name = orig->use_config_name;
431
432 if (evsel__copy_config_terms(evsel, orig) < 0)
433 goto out_err;
434
435 return evsel;
436
437out_err:
438 evsel__delete(evsel);
439 return NULL;
440}
441
442
443
444
445struct evsel *evsel__newtp_idx(const char *sys, const char *name, int idx)
446{
447 struct evsel *evsel = zalloc(perf_evsel__object.size);
448 int err = -ENOMEM;
449
450 if (evsel == NULL) {
451 goto out_err;
452 } else {
453 struct perf_event_attr attr = {
454 .type = PERF_TYPE_TRACEPOINT,
455 .sample_type = (PERF_SAMPLE_RAW | PERF_SAMPLE_TIME |
456 PERF_SAMPLE_CPU | PERF_SAMPLE_PERIOD),
457 };
458
459 if (asprintf(&evsel->name, "%s:%s", sys, name) < 0)
460 goto out_free;
461
462 evsel->tp_format = trace_event__tp_format(sys, name);
463 if (IS_ERR(evsel->tp_format)) {
464 err = PTR_ERR(evsel->tp_format);
465 goto out_free;
466 }
467
468 event_attr_init(&attr);
469 attr.config = evsel->tp_format->id;
470 attr.sample_period = 1;
471 evsel__init(evsel, &attr, idx);
472 }
473
474 return evsel;
475
476out_free:
477 zfree(&evsel->name);
478 free(evsel);
479out_err:
480 return ERR_PTR(err);
481}
482
483const char *evsel__hw_names[PERF_COUNT_HW_MAX] = {
484 "cycles",
485 "instructions",
486 "cache-references",
487 "cache-misses",
488 "branches",
489 "branch-misses",
490 "bus-cycles",
491 "stalled-cycles-frontend",
492 "stalled-cycles-backend",
493 "ref-cycles",
494};
495
496char *evsel__bpf_counter_events;
497
498bool evsel__match_bpf_counter_events(const char *name)
499{
500 int name_len;
501 bool match;
502 char *ptr;
503
504 if (!evsel__bpf_counter_events)
505 return false;
506
507 ptr = strstr(evsel__bpf_counter_events, name);
508 name_len = strlen(name);
509
510
511 match = (ptr != NULL) &&
512 ((ptr == evsel__bpf_counter_events) || (*(ptr - 1) == ',')) &&
513 ((*(ptr + name_len) == ',') || (*(ptr + name_len) == '\0'));
514
515 return match;
516}
517
518static const char *__evsel__hw_name(u64 config)
519{
520 if (config < PERF_COUNT_HW_MAX && evsel__hw_names[config])
521 return evsel__hw_names[config];
522
523 return "unknown-hardware";
524}
525
526static int evsel__add_modifiers(struct evsel *evsel, char *bf, size_t size)
527{
528 int colon = 0, r = 0;
529 struct perf_event_attr *attr = &evsel->core.attr;
530 bool exclude_guest_default = false;
531
532#define MOD_PRINT(context, mod) do { \
533 if (!attr->exclude_##context) { \
534 if (!colon) colon = ++r; \
535 r += scnprintf(bf + r, size - r, "%c", mod); \
536 } } while(0)
537
538 if (attr->exclude_kernel || attr->exclude_user || attr->exclude_hv) {
539 MOD_PRINT(kernel, 'k');
540 MOD_PRINT(user, 'u');
541 MOD_PRINT(hv, 'h');
542 exclude_guest_default = true;
543 }
544
545 if (attr->precise_ip) {
546 if (!colon)
547 colon = ++r;
548 r += scnprintf(bf + r, size - r, "%.*s", attr->precise_ip, "ppp");
549 exclude_guest_default = true;
550 }
551
552 if (attr->exclude_host || attr->exclude_guest == exclude_guest_default) {
553 MOD_PRINT(host, 'H');
554 MOD_PRINT(guest, 'G');
555 }
556#undef MOD_PRINT
557 if (colon)
558 bf[colon - 1] = ':';
559 return r;
560}
561
562static int evsel__hw_name(struct evsel *evsel, char *bf, size_t size)
563{
564 int r = scnprintf(bf, size, "%s", __evsel__hw_name(evsel->core.attr.config));
565 return r + evsel__add_modifiers(evsel, bf + r, size - r);
566}
567
568const char *evsel__sw_names[PERF_COUNT_SW_MAX] = {
569 "cpu-clock",
570 "task-clock",
571 "page-faults",
572 "context-switches",
573 "cpu-migrations",
574 "minor-faults",
575 "major-faults",
576 "alignment-faults",
577 "emulation-faults",
578 "dummy",
579};
580
581static const char *__evsel__sw_name(u64 config)
582{
583 if (config < PERF_COUNT_SW_MAX && evsel__sw_names[config])
584 return evsel__sw_names[config];
585 return "unknown-software";
586}
587
588static int evsel__sw_name(struct evsel *evsel, char *bf, size_t size)
589{
590 int r = scnprintf(bf, size, "%s", __evsel__sw_name(evsel->core.attr.config));
591 return r + evsel__add_modifiers(evsel, bf + r, size - r);
592}
593
594static int __evsel__bp_name(char *bf, size_t size, u64 addr, u64 type)
595{
596 int r;
597
598 r = scnprintf(bf, size, "mem:0x%" PRIx64 ":", addr);
599
600 if (type & HW_BREAKPOINT_R)
601 r += scnprintf(bf + r, size - r, "r");
602
603 if (type & HW_BREAKPOINT_W)
604 r += scnprintf(bf + r, size - r, "w");
605
606 if (type & HW_BREAKPOINT_X)
607 r += scnprintf(bf + r, size - r, "x");
608
609 return r;
610}
611
612static int evsel__bp_name(struct evsel *evsel, char *bf, size_t size)
613{
614 struct perf_event_attr *attr = &evsel->core.attr;
615 int r = __evsel__bp_name(bf, size, attr->bp_addr, attr->bp_type);
616 return r + evsel__add_modifiers(evsel, bf + r, size - r);
617}
618
619const char *evsel__hw_cache[PERF_COUNT_HW_CACHE_MAX][EVSEL__MAX_ALIASES] = {
620 { "L1-dcache", "l1-d", "l1d", "L1-data", },
621 { "L1-icache", "l1-i", "l1i", "L1-instruction", },
622 { "LLC", "L2", },
623 { "dTLB", "d-tlb", "Data-TLB", },
624 { "iTLB", "i-tlb", "Instruction-TLB", },
625 { "branch", "branches", "bpu", "btb", "bpc", },
626 { "node", },
627};
628
629const char *evsel__hw_cache_op[PERF_COUNT_HW_CACHE_OP_MAX][EVSEL__MAX_ALIASES] = {
630 { "load", "loads", "read", },
631 { "store", "stores", "write", },
632 { "prefetch", "prefetches", "speculative-read", "speculative-load", },
633};
634
635const char *evsel__hw_cache_result[PERF_COUNT_HW_CACHE_RESULT_MAX][EVSEL__MAX_ALIASES] = {
636 { "refs", "Reference", "ops", "access", },
637 { "misses", "miss", },
638};
639
640#define C(x) PERF_COUNT_HW_CACHE_##x
641#define CACHE_READ (1 << C(OP_READ))
642#define CACHE_WRITE (1 << C(OP_WRITE))
643#define CACHE_PREFETCH (1 << C(OP_PREFETCH))
644#define COP(x) (1 << x)
645
646
647
648
649
650
651static unsigned long evsel__hw_cache_stat[C(MAX)] = {
652 [C(L1D)] = (CACHE_READ | CACHE_WRITE | CACHE_PREFETCH),
653 [C(L1I)] = (CACHE_READ | CACHE_PREFETCH),
654 [C(LL)] = (CACHE_READ | CACHE_WRITE | CACHE_PREFETCH),
655 [C(DTLB)] = (CACHE_READ | CACHE_WRITE | CACHE_PREFETCH),
656 [C(ITLB)] = (CACHE_READ),
657 [C(BPU)] = (CACHE_READ),
658 [C(NODE)] = (CACHE_READ | CACHE_WRITE | CACHE_PREFETCH),
659};
660
661bool evsel__is_cache_op_valid(u8 type, u8 op)
662{
663 if (evsel__hw_cache_stat[type] & COP(op))
664 return true;
665 else
666 return false;
667}
668
669int __evsel__hw_cache_type_op_res_name(u8 type, u8 op, u8 result, char *bf, size_t size)
670{
671 if (result) {
672 return scnprintf(bf, size, "%s-%s-%s", evsel__hw_cache[type][0],
673 evsel__hw_cache_op[op][0],
674 evsel__hw_cache_result[result][0]);
675 }
676
677 return scnprintf(bf, size, "%s-%s", evsel__hw_cache[type][0],
678 evsel__hw_cache_op[op][1]);
679}
680
681static int __evsel__hw_cache_name(u64 config, char *bf, size_t size)
682{
683 u8 op, result, type = (config >> 0) & 0xff;
684 const char *err = "unknown-ext-hardware-cache-type";
685
686 if (type >= PERF_COUNT_HW_CACHE_MAX)
687 goto out_err;
688
689 op = (config >> 8) & 0xff;
690 err = "unknown-ext-hardware-cache-op";
691 if (op >= PERF_COUNT_HW_CACHE_OP_MAX)
692 goto out_err;
693
694 result = (config >> 16) & 0xff;
695 err = "unknown-ext-hardware-cache-result";
696 if (result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
697 goto out_err;
698
699 err = "invalid-cache";
700 if (!evsel__is_cache_op_valid(type, op))
701 goto out_err;
702
703 return __evsel__hw_cache_type_op_res_name(type, op, result, bf, size);
704out_err:
705 return scnprintf(bf, size, "%s", err);
706}
707
708static int evsel__hw_cache_name(struct evsel *evsel, char *bf, size_t size)
709{
710 int ret = __evsel__hw_cache_name(evsel->core.attr.config, bf, size);
711 return ret + evsel__add_modifiers(evsel, bf + ret, size - ret);
712}
713
714static int evsel__raw_name(struct evsel *evsel, char *bf, size_t size)
715{
716 int ret = scnprintf(bf, size, "raw 0x%" PRIx64, evsel->core.attr.config);
717 return ret + evsel__add_modifiers(evsel, bf + ret, size - ret);
718}
719
720static int evsel__tool_name(char *bf, size_t size)
721{
722 int ret = scnprintf(bf, size, "duration_time");
723 return ret;
724}
725
726const char *evsel__name(struct evsel *evsel)
727{
728 char bf[128];
729
730 if (!evsel)
731 goto out_unknown;
732
733 if (evsel->name)
734 return evsel->name;
735
736 switch (evsel->core.attr.type) {
737 case PERF_TYPE_RAW:
738 evsel__raw_name(evsel, bf, sizeof(bf));
739 break;
740
741 case PERF_TYPE_HARDWARE:
742 evsel__hw_name(evsel, bf, sizeof(bf));
743 break;
744
745 case PERF_TYPE_HW_CACHE:
746 evsel__hw_cache_name(evsel, bf, sizeof(bf));
747 break;
748
749 case PERF_TYPE_SOFTWARE:
750 if (evsel->tool_event)
751 evsel__tool_name(bf, sizeof(bf));
752 else
753 evsel__sw_name(evsel, bf, sizeof(bf));
754 break;
755
756 case PERF_TYPE_TRACEPOINT:
757 scnprintf(bf, sizeof(bf), "%s", "unknown tracepoint");
758 break;
759
760 case PERF_TYPE_BREAKPOINT:
761 evsel__bp_name(evsel, bf, sizeof(bf));
762 break;
763
764 default:
765 scnprintf(bf, sizeof(bf), "unknown attr type: %d",
766 evsel->core.attr.type);
767 break;
768 }
769
770 evsel->name = strdup(bf);
771
772 if (evsel->name)
773 return evsel->name;
774out_unknown:
775 return "unknown";
776}
777
778const char *evsel__group_name(struct evsel *evsel)
779{
780 return evsel->group_name ?: "anon group";
781}
782
783
784
785
786
787
788
789
790
791
792
793int evsel__group_desc(struct evsel *evsel, char *buf, size_t size)
794{
795 int ret = 0;
796 struct evsel *pos;
797 const char *group_name = evsel__group_name(evsel);
798
799 if (!evsel->forced_leader)
800 ret = scnprintf(buf, size, "%s { ", group_name);
801
802 ret += scnprintf(buf + ret, size - ret, "%s", evsel__name(evsel));
803
804 for_each_group_member(pos, evsel)
805 ret += scnprintf(buf + ret, size - ret, ", %s", evsel__name(pos));
806
807 if (!evsel->forced_leader)
808 ret += scnprintf(buf + ret, size - ret, " }");
809
810 return ret;
811}
812
813static void __evsel__config_callchain(struct evsel *evsel, struct record_opts *opts,
814 struct callchain_param *param)
815{
816 bool function = evsel__is_function_event(evsel);
817 struct perf_event_attr *attr = &evsel->core.attr;
818
819 evsel__set_sample_bit(evsel, CALLCHAIN);
820
821 attr->sample_max_stack = param->max_stack;
822
823 if (opts->kernel_callchains)
824 attr->exclude_callchain_user = 1;
825 if (opts->user_callchains)
826 attr->exclude_callchain_kernel = 1;
827 if (param->record_mode == CALLCHAIN_LBR) {
828 if (!opts->branch_stack) {
829 if (attr->exclude_user) {
830 pr_warning("LBR callstack option is only available "
831 "to get user callchain information. "
832 "Falling back to framepointers.\n");
833 } else {
834 evsel__set_sample_bit(evsel, BRANCH_STACK);
835 attr->branch_sample_type = PERF_SAMPLE_BRANCH_USER |
836 PERF_SAMPLE_BRANCH_CALL_STACK |
837 PERF_SAMPLE_BRANCH_NO_CYCLES |
838 PERF_SAMPLE_BRANCH_NO_FLAGS |
839 PERF_SAMPLE_BRANCH_HW_INDEX;
840 }
841 } else
842 pr_warning("Cannot use LBR callstack with branch stack. "
843 "Falling back to framepointers.\n");
844 }
845
846 if (param->record_mode == CALLCHAIN_DWARF) {
847 if (!function) {
848 evsel__set_sample_bit(evsel, REGS_USER);
849 evsel__set_sample_bit(evsel, STACK_USER);
850 if (opts->sample_user_regs && DWARF_MINIMAL_REGS != PERF_REGS_MASK) {
851 attr->sample_regs_user |= DWARF_MINIMAL_REGS;
852 pr_warning("WARNING: The use of --call-graph=dwarf may require all the user registers, "
853 "specifying a subset with --user-regs may render DWARF unwinding unreliable, "
854 "so the minimal registers set (IP, SP) is explicitly forced.\n");
855 } else {
856 attr->sample_regs_user |= PERF_REGS_MASK;
857 }
858 attr->sample_stack_user = param->dump_size;
859 attr->exclude_callchain_user = 1;
860 } else {
861 pr_info("Cannot use DWARF unwind for function trace event,"
862 " falling back to framepointers.\n");
863 }
864 }
865
866 if (function) {
867 pr_info("Disabling user space callchains for function trace event.\n");
868 attr->exclude_callchain_user = 1;
869 }
870}
871
872void evsel__config_callchain(struct evsel *evsel, struct record_opts *opts,
873 struct callchain_param *param)
874{
875 if (param->enabled)
876 return __evsel__config_callchain(evsel, opts, param);
877}
878
879static void evsel__reset_callgraph(struct evsel *evsel, struct callchain_param *param)
880{
881 struct perf_event_attr *attr = &evsel->core.attr;
882
883 evsel__reset_sample_bit(evsel, CALLCHAIN);
884 if (param->record_mode == CALLCHAIN_LBR) {
885 evsel__reset_sample_bit(evsel, BRANCH_STACK);
886 attr->branch_sample_type &= ~(PERF_SAMPLE_BRANCH_USER |
887 PERF_SAMPLE_BRANCH_CALL_STACK |
888 PERF_SAMPLE_BRANCH_HW_INDEX);
889 }
890 if (param->record_mode == CALLCHAIN_DWARF) {
891 evsel__reset_sample_bit(evsel, REGS_USER);
892 evsel__reset_sample_bit(evsel, STACK_USER);
893 }
894}
895
896static void evsel__apply_config_terms(struct evsel *evsel,
897 struct record_opts *opts, bool track)
898{
899 struct evsel_config_term *term;
900 struct list_head *config_terms = &evsel->config_terms;
901 struct perf_event_attr *attr = &evsel->core.attr;
902
903 struct callchain_param param = {
904 .record_mode = callchain_param.record_mode,
905 };
906 u32 dump_size = 0;
907 int max_stack = 0;
908 const char *callgraph_buf = NULL;
909
910 list_for_each_entry(term, config_terms, list) {
911 switch (term->type) {
912 case EVSEL__CONFIG_TERM_PERIOD:
913 if (!(term->weak && opts->user_interval != ULLONG_MAX)) {
914 attr->sample_period = term->val.period;
915 attr->freq = 0;
916 evsel__reset_sample_bit(evsel, PERIOD);
917 }
918 break;
919 case EVSEL__CONFIG_TERM_FREQ:
920 if (!(term->weak && opts->user_freq != UINT_MAX)) {
921 attr->sample_freq = term->val.freq;
922 attr->freq = 1;
923 evsel__set_sample_bit(evsel, PERIOD);
924 }
925 break;
926 case EVSEL__CONFIG_TERM_TIME:
927 if (term->val.time)
928 evsel__set_sample_bit(evsel, TIME);
929 else
930 evsel__reset_sample_bit(evsel, TIME);
931 break;
932 case EVSEL__CONFIG_TERM_CALLGRAPH:
933 callgraph_buf = term->val.str;
934 break;
935 case EVSEL__CONFIG_TERM_BRANCH:
936 if (term->val.str && strcmp(term->val.str, "no")) {
937 evsel__set_sample_bit(evsel, BRANCH_STACK);
938 parse_branch_str(term->val.str,
939 &attr->branch_sample_type);
940 } else
941 evsel__reset_sample_bit(evsel, BRANCH_STACK);
942 break;
943 case EVSEL__CONFIG_TERM_STACK_USER:
944 dump_size = term->val.stack_user;
945 break;
946 case EVSEL__CONFIG_TERM_MAX_STACK:
947 max_stack = term->val.max_stack;
948 break;
949 case EVSEL__CONFIG_TERM_MAX_EVENTS:
950 evsel->max_events = term->val.max_events;
951 break;
952 case EVSEL__CONFIG_TERM_INHERIT:
953
954
955
956
957
958
959 attr->inherit = term->val.inherit ? 1 : 0;
960 break;
961 case EVSEL__CONFIG_TERM_OVERWRITE:
962 attr->write_backward = term->val.overwrite ? 1 : 0;
963 break;
964 case EVSEL__CONFIG_TERM_DRV_CFG:
965 break;
966 case EVSEL__CONFIG_TERM_PERCORE:
967 break;
968 case EVSEL__CONFIG_TERM_AUX_OUTPUT:
969 attr->aux_output = term->val.aux_output ? 1 : 0;
970 break;
971 case EVSEL__CONFIG_TERM_AUX_SAMPLE_SIZE:
972
973 break;
974 case EVSEL__CONFIG_TERM_CFG_CHG:
975 break;
976 default:
977 break;
978 }
979 }
980
981
982 if ((callgraph_buf != NULL) || (dump_size > 0) || max_stack) {
983 bool sample_address = false;
984
985 if (max_stack) {
986 param.max_stack = max_stack;
987 if (callgraph_buf == NULL)
988 callgraph_buf = "fp";
989 }
990
991
992 if (callgraph_buf != NULL) {
993 if (!strcmp(callgraph_buf, "no")) {
994 param.enabled = false;
995 param.record_mode = CALLCHAIN_NONE;
996 } else {
997 param.enabled = true;
998 if (parse_callchain_record(callgraph_buf, ¶m)) {
999 pr_err("per-event callgraph setting for %s failed. "
1000 "Apply callgraph global setting for it\n",
1001 evsel->name);
1002 return;
1003 }
1004 if (param.record_mode == CALLCHAIN_DWARF)
1005 sample_address = true;
1006 }
1007 }
1008 if (dump_size > 0) {
1009 dump_size = round_up(dump_size, sizeof(u64));
1010 param.dump_size = dump_size;
1011 }
1012
1013
1014 if (callchain_param.enabled)
1015 evsel__reset_callgraph(evsel, &callchain_param);
1016
1017
1018 if (param.enabled) {
1019 if (sample_address) {
1020 evsel__set_sample_bit(evsel, ADDR);
1021 evsel__set_sample_bit(evsel, DATA_SRC);
1022 evsel->core.attr.mmap_data = track;
1023 }
1024 evsel__config_callchain(evsel, opts, ¶m);
1025 }
1026 }
1027}
1028
1029struct evsel_config_term *__evsel__get_config_term(struct evsel *evsel, enum evsel_term_type type)
1030{
1031 struct evsel_config_term *term, *found_term = NULL;
1032
1033 list_for_each_entry(term, &evsel->config_terms, list) {
1034 if (term->type == type)
1035 found_term = term;
1036 }
1037
1038 return found_term;
1039}
1040
1041void __weak arch_evsel__set_sample_weight(struct evsel *evsel)
1042{
1043 evsel__set_sample_bit(evsel, WEIGHT);
1044}
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074void evsel__config(struct evsel *evsel, struct record_opts *opts,
1075 struct callchain_param *callchain)
1076{
1077 struct evsel *leader = evsel__leader(evsel);
1078 struct perf_event_attr *attr = &evsel->core.attr;
1079 int track = evsel->tracking;
1080 bool per_cpu = opts->target.default_per_cpu && !opts->target.per_thread;
1081
1082 attr->sample_id_all = perf_missing_features.sample_id_all ? 0 : 1;
1083 attr->inherit = !opts->no_inherit;
1084 attr->write_backward = opts->overwrite ? 1 : 0;
1085
1086 evsel__set_sample_bit(evsel, IP);
1087 evsel__set_sample_bit(evsel, TID);
1088
1089 if (evsel->sample_read) {
1090 evsel__set_sample_bit(evsel, READ);
1091
1092
1093
1094
1095
1096 evsel__set_sample_id(evsel, false);
1097
1098
1099
1100
1101
1102 if (leader->core.nr_members > 1) {
1103 attr->read_format |= PERF_FORMAT_GROUP;
1104 attr->inherit = 0;
1105 }
1106 }
1107
1108
1109
1110
1111
1112 if (!attr->sample_period) {
1113 if (opts->freq) {
1114 attr->freq = 1;
1115 attr->sample_freq = opts->freq;
1116 } else {
1117 attr->sample_period = opts->default_interval;
1118 }
1119 }
1120
1121
1122
1123
1124 if (attr->freq)
1125 evsel__set_sample_bit(evsel, PERIOD);
1126
1127 if (opts->no_samples)
1128 attr->sample_freq = 0;
1129
1130 if (opts->inherit_stat) {
1131 evsel->core.attr.read_format |=
1132 PERF_FORMAT_TOTAL_TIME_ENABLED |
1133 PERF_FORMAT_TOTAL_TIME_RUNNING |
1134 PERF_FORMAT_ID;
1135 attr->inherit_stat = 1;
1136 }
1137
1138 if (opts->sample_address) {
1139 evsel__set_sample_bit(evsel, ADDR);
1140 attr->mmap_data = track;
1141 }
1142
1143
1144
1145
1146
1147
1148 if (evsel__is_function_event(evsel))
1149 evsel->core.attr.exclude_callchain_user = 1;
1150
1151 if (callchain && callchain->enabled && !evsel->no_aux_samples)
1152 evsel__config_callchain(evsel, opts, callchain);
1153
1154 if (opts->sample_intr_regs && !evsel->no_aux_samples &&
1155 !evsel__is_dummy_event(evsel)) {
1156 attr->sample_regs_intr = opts->sample_intr_regs;
1157 evsel__set_sample_bit(evsel, REGS_INTR);
1158 }
1159
1160 if (opts->sample_user_regs && !evsel->no_aux_samples &&
1161 !evsel__is_dummy_event(evsel)) {
1162 attr->sample_regs_user |= opts->sample_user_regs;
1163 evsel__set_sample_bit(evsel, REGS_USER);
1164 }
1165
1166 if (target__has_cpu(&opts->target) || opts->sample_cpu)
1167 evsel__set_sample_bit(evsel, CPU);
1168
1169
1170
1171
1172 if (opts->sample_time &&
1173 (!perf_missing_features.sample_id_all &&
1174 (!opts->no_inherit || target__has_cpu(&opts->target) || per_cpu ||
1175 opts->sample_time_set)))
1176 evsel__set_sample_bit(evsel, TIME);
1177
1178 if (opts->raw_samples && !evsel->no_aux_samples) {
1179 evsel__set_sample_bit(evsel, TIME);
1180 evsel__set_sample_bit(evsel, RAW);
1181 evsel__set_sample_bit(evsel, CPU);
1182 }
1183
1184 if (opts->sample_address)
1185 evsel__set_sample_bit(evsel, DATA_SRC);
1186
1187 if (opts->sample_phys_addr)
1188 evsel__set_sample_bit(evsel, PHYS_ADDR);
1189
1190 if (opts->no_buffering) {
1191 attr->watermark = 0;
1192 attr->wakeup_events = 1;
1193 }
1194 if (opts->branch_stack && !evsel->no_aux_samples) {
1195 evsel__set_sample_bit(evsel, BRANCH_STACK);
1196 attr->branch_sample_type = opts->branch_stack;
1197 }
1198
1199 if (opts->sample_weight)
1200 arch_evsel__set_sample_weight(evsel);
1201
1202 attr->task = track;
1203 attr->mmap = track;
1204 attr->mmap2 = track && !perf_missing_features.mmap2;
1205 attr->comm = track;
1206 attr->build_id = track && opts->build_id;
1207
1208
1209
1210
1211
1212 if (!opts->text_poke)
1213 attr->ksymbol = track && !perf_missing_features.ksymbol;
1214 attr->bpf_event = track && !opts->no_bpf_event && !perf_missing_features.bpf;
1215
1216 if (opts->record_namespaces)
1217 attr->namespaces = track;
1218
1219 if (opts->record_cgroup) {
1220 attr->cgroup = track && !perf_missing_features.cgroup;
1221 evsel__set_sample_bit(evsel, CGROUP);
1222 }
1223
1224 if (opts->sample_data_page_size)
1225 evsel__set_sample_bit(evsel, DATA_PAGE_SIZE);
1226
1227 if (opts->sample_code_page_size)
1228 evsel__set_sample_bit(evsel, CODE_PAGE_SIZE);
1229
1230 if (opts->record_switch_events)
1231 attr->context_switch = track;
1232
1233 if (opts->sample_transaction)
1234 evsel__set_sample_bit(evsel, TRANSACTION);
1235
1236 if (opts->running_time) {
1237 evsel->core.attr.read_format |=
1238 PERF_FORMAT_TOTAL_TIME_ENABLED |
1239 PERF_FORMAT_TOTAL_TIME_RUNNING;
1240 }
1241
1242
1243
1244
1245
1246
1247
1248 if (evsel__is_group_leader(evsel))
1249 attr->disabled = 1;
1250
1251
1252
1253
1254
1255 if (target__none(&opts->target) && evsel__is_group_leader(evsel) &&
1256 !opts->initial_delay)
1257 attr->enable_on_exec = 1;
1258
1259 if (evsel->immediate) {
1260 attr->disabled = 0;
1261 attr->enable_on_exec = 0;
1262 }
1263
1264 clockid = opts->clockid;
1265 if (opts->use_clockid) {
1266 attr->use_clockid = 1;
1267 attr->clockid = opts->clockid;
1268 }
1269
1270 if (evsel->precise_max)
1271 attr->precise_ip = 3;
1272
1273 if (opts->all_user) {
1274 attr->exclude_kernel = 1;
1275 attr->exclude_user = 0;
1276 }
1277
1278 if (opts->all_kernel) {
1279 attr->exclude_kernel = 0;
1280 attr->exclude_user = 1;
1281 }
1282
1283 if (evsel->core.own_cpus || evsel->unit)
1284 evsel->core.attr.read_format |= PERF_FORMAT_ID;
1285
1286
1287
1288
1289
1290 evsel__apply_config_terms(evsel, opts, track);
1291
1292 evsel->ignore_missing_thread = opts->ignore_missing_thread;
1293
1294
1295 if (opts->period_set) {
1296 if (opts->period)
1297 evsel__set_sample_bit(evsel, PERIOD);
1298 else
1299 evsel__reset_sample_bit(evsel, PERIOD);
1300 }
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310 if (evsel__is_dummy_event(evsel))
1311 evsel__reset_sample_bit(evsel, BRANCH_STACK);
1312}
1313
1314int evsel__set_filter(struct evsel *evsel, const char *filter)
1315{
1316 char *new_filter = strdup(filter);
1317
1318 if (new_filter != NULL) {
1319 free(evsel->filter);
1320 evsel->filter = new_filter;
1321 return 0;
1322 }
1323
1324 return -1;
1325}
1326
1327static int evsel__append_filter(struct evsel *evsel, const char *fmt, const char *filter)
1328{
1329 char *new_filter;
1330
1331 if (evsel->filter == NULL)
1332 return evsel__set_filter(evsel, filter);
1333
1334 if (asprintf(&new_filter, fmt, evsel->filter, filter) > 0) {
1335 free(evsel->filter);
1336 evsel->filter = new_filter;
1337 return 0;
1338 }
1339
1340 return -1;
1341}
1342
1343int evsel__append_tp_filter(struct evsel *evsel, const char *filter)
1344{
1345 return evsel__append_filter(evsel, "(%s) && (%s)", filter);
1346}
1347
1348int evsel__append_addr_filter(struct evsel *evsel, const char *filter)
1349{
1350 return evsel__append_filter(evsel, "%s,%s", filter);
1351}
1352
1353
1354int evsel__enable_cpu(struct evsel *evsel, int cpu)
1355{
1356 return perf_evsel__enable_cpu(&evsel->core, cpu);
1357}
1358
1359int evsel__enable(struct evsel *evsel)
1360{
1361 int err = perf_evsel__enable(&evsel->core);
1362
1363 if (!err)
1364 evsel->disabled = false;
1365 return err;
1366}
1367
1368
1369int evsel__disable_cpu(struct evsel *evsel, int cpu)
1370{
1371 return perf_evsel__disable_cpu(&evsel->core, cpu);
1372}
1373
1374int evsel__disable(struct evsel *evsel)
1375{
1376 int err = perf_evsel__disable(&evsel->core);
1377
1378
1379
1380
1381
1382
1383 if (!err)
1384 evsel->disabled = true;
1385
1386 return err;
1387}
1388
1389static void evsel__free_config_terms(struct evsel *evsel)
1390{
1391 struct evsel_config_term *term, *h;
1392
1393 list_for_each_entry_safe(term, h, &evsel->config_terms, list) {
1394 list_del_init(&term->list);
1395 if (term->free_str)
1396 zfree(&term->val.str);
1397 free(term);
1398 }
1399}
1400
1401void evsel__exit(struct evsel *evsel)
1402{
1403 assert(list_empty(&evsel->core.node));
1404 assert(evsel->evlist == NULL);
1405 bpf_counter__destroy(evsel);
1406 evsel__free_counts(evsel);
1407 perf_evsel__free_fd(&evsel->core);
1408 perf_evsel__free_id(&evsel->core);
1409 evsel__free_config_terms(evsel);
1410 cgroup__put(evsel->cgrp);
1411 perf_cpu_map__put(evsel->core.cpus);
1412 perf_cpu_map__put(evsel->core.own_cpus);
1413 perf_thread_map__put(evsel->core.threads);
1414 zfree(&evsel->group_name);
1415 zfree(&evsel->name);
1416 zfree(&evsel->pmu_name);
1417 evsel__zero_per_pkg(evsel);
1418 hashmap__free(evsel->per_pkg_mask);
1419 evsel->per_pkg_mask = NULL;
1420 zfree(&evsel->metric_events);
1421 perf_evsel__object.fini(evsel);
1422}
1423
1424void evsel__delete(struct evsel *evsel)
1425{
1426 evsel__exit(evsel);
1427 free(evsel);
1428}
1429
1430void evsel__compute_deltas(struct evsel *evsel, int cpu, int thread,
1431 struct perf_counts_values *count)
1432{
1433 struct perf_counts_values tmp;
1434
1435 if (!evsel->prev_raw_counts)
1436 return;
1437
1438 if (cpu == -1) {
1439 tmp = evsel->prev_raw_counts->aggr;
1440 evsel->prev_raw_counts->aggr = *count;
1441 } else {
1442 tmp = *perf_counts(evsel->prev_raw_counts, cpu, thread);
1443 *perf_counts(evsel->prev_raw_counts, cpu, thread) = *count;
1444 }
1445
1446 count->val = count->val - tmp.val;
1447 count->ena = count->ena - tmp.ena;
1448 count->run = count->run - tmp.run;
1449}
1450
1451void perf_counts_values__scale(struct perf_counts_values *count,
1452 bool scale, s8 *pscaled)
1453{
1454 s8 scaled = 0;
1455
1456 if (scale) {
1457 if (count->run == 0) {
1458 scaled = -1;
1459 count->val = 0;
1460 } else if (count->run < count->ena) {
1461 scaled = 1;
1462 count->val = (u64)((double) count->val * count->ena / count->run);
1463 }
1464 }
1465
1466 if (pscaled)
1467 *pscaled = scaled;
1468}
1469
1470static int evsel__read_one(struct evsel *evsel, int cpu, int thread)
1471{
1472 struct perf_counts_values *count = perf_counts(evsel->counts, cpu, thread);
1473
1474 return perf_evsel__read(&evsel->core, cpu, thread, count);
1475}
1476
1477static void evsel__set_count(struct evsel *counter, int cpu, int thread, u64 val, u64 ena, u64 run)
1478{
1479 struct perf_counts_values *count;
1480
1481 count = perf_counts(counter->counts, cpu, thread);
1482
1483 count->val = val;
1484 count->ena = ena;
1485 count->run = run;
1486
1487 perf_counts__set_loaded(counter->counts, cpu, thread, true);
1488}
1489
1490static int evsel__process_group_data(struct evsel *leader, int cpu, int thread, u64 *data)
1491{
1492 u64 read_format = leader->core.attr.read_format;
1493 struct sample_read_value *v;
1494 u64 nr, ena = 0, run = 0, i;
1495
1496 nr = *data++;
1497
1498 if (nr != (u64) leader->core.nr_members)
1499 return -EINVAL;
1500
1501 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
1502 ena = *data++;
1503
1504 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
1505 run = *data++;
1506
1507 v = (struct sample_read_value *) data;
1508
1509 evsel__set_count(leader, cpu, thread, v[0].value, ena, run);
1510
1511 for (i = 1; i < nr; i++) {
1512 struct evsel *counter;
1513
1514 counter = evlist__id2evsel(leader->evlist, v[i].id);
1515 if (!counter)
1516 return -EINVAL;
1517
1518 evsel__set_count(counter, cpu, thread, v[i].value, ena, run);
1519 }
1520
1521 return 0;
1522}
1523
1524static int evsel__read_group(struct evsel *leader, int cpu, int thread)
1525{
1526 struct perf_stat_evsel *ps = leader->stats;
1527 u64 read_format = leader->core.attr.read_format;
1528 int size = perf_evsel__read_size(&leader->core);
1529 u64 *data = ps->group_data;
1530
1531 if (!(read_format & PERF_FORMAT_ID))
1532 return -EINVAL;
1533
1534 if (!evsel__is_group_leader(leader))
1535 return -EINVAL;
1536
1537 if (!data) {
1538 data = zalloc(size);
1539 if (!data)
1540 return -ENOMEM;
1541
1542 ps->group_data = data;
1543 }
1544
1545 if (FD(leader, cpu, thread) < 0)
1546 return -EINVAL;
1547
1548 if (readn(FD(leader, cpu, thread), data, size) <= 0)
1549 return -errno;
1550
1551 return evsel__process_group_data(leader, cpu, thread, data);
1552}
1553
1554int evsel__read_counter(struct evsel *evsel, int cpu, int thread)
1555{
1556 u64 read_format = evsel->core.attr.read_format;
1557
1558 if (read_format & PERF_FORMAT_GROUP)
1559 return evsel__read_group(evsel, cpu, thread);
1560
1561 return evsel__read_one(evsel, cpu, thread);
1562}
1563
1564int __evsel__read_on_cpu(struct evsel *evsel, int cpu, int thread, bool scale)
1565{
1566 struct perf_counts_values count;
1567 size_t nv = scale ? 3 : 1;
1568
1569 if (FD(evsel, cpu, thread) < 0)
1570 return -EINVAL;
1571
1572 if (evsel->counts == NULL && evsel__alloc_counts(evsel, cpu + 1, thread + 1) < 0)
1573 return -ENOMEM;
1574
1575 if (readn(FD(evsel, cpu, thread), &count, nv * sizeof(u64)) <= 0)
1576 return -errno;
1577
1578 evsel__compute_deltas(evsel, cpu, thread, &count);
1579 perf_counts_values__scale(&count, scale, NULL);
1580 *perf_counts(evsel->counts, cpu, thread) = count;
1581 return 0;
1582}
1583
1584static int evsel__match_other_cpu(struct evsel *evsel, struct evsel *other,
1585 int cpu)
1586{
1587 int cpuid;
1588
1589 cpuid = perf_cpu_map__cpu(evsel->core.cpus, cpu);
1590 return perf_cpu_map__idx(other->core.cpus, cpuid);
1591}
1592
1593static int evsel__hybrid_group_cpu(struct evsel *evsel, int cpu)
1594{
1595 struct evsel *leader = evsel__leader(evsel);
1596
1597 if ((evsel__is_hybrid(evsel) && !evsel__is_hybrid(leader)) ||
1598 (!evsel__is_hybrid(evsel) && evsel__is_hybrid(leader))) {
1599 return evsel__match_other_cpu(evsel, leader, cpu);
1600 }
1601
1602 return cpu;
1603}
1604
1605static int get_group_fd(struct evsel *evsel, int cpu, int thread)
1606{
1607 struct evsel *leader = evsel__leader(evsel);
1608 int fd;
1609
1610 if (evsel__is_group_leader(evsel))
1611 return -1;
1612
1613
1614
1615
1616
1617 BUG_ON(!leader->core.fd);
1618
1619 cpu = evsel__hybrid_group_cpu(evsel, cpu);
1620 if (cpu == -1)
1621 return -1;
1622
1623 fd = FD(leader, cpu, thread);
1624 BUG_ON(fd == -1);
1625
1626 return fd;
1627}
1628
1629static void evsel__remove_fd(struct evsel *pos, int nr_cpus, int nr_threads, int thread_idx)
1630{
1631 for (int cpu = 0; cpu < nr_cpus; cpu++)
1632 for (int thread = thread_idx; thread < nr_threads - 1; thread++)
1633 FD(pos, cpu, thread) = FD(pos, cpu, thread + 1);
1634}
1635
1636static int update_fds(struct evsel *evsel,
1637 int nr_cpus, int cpu_idx,
1638 int nr_threads, int thread_idx)
1639{
1640 struct evsel *pos;
1641
1642 if (cpu_idx >= nr_cpus || thread_idx >= nr_threads)
1643 return -EINVAL;
1644
1645 evlist__for_each_entry(evsel->evlist, pos) {
1646 nr_cpus = pos != evsel ? nr_cpus : cpu_idx;
1647
1648 evsel__remove_fd(pos, nr_cpus, nr_threads, thread_idx);
1649
1650
1651
1652
1653
1654 if (pos == evsel)
1655 break;
1656 }
1657 return 0;
1658}
1659
1660static bool ignore_missing_thread(struct evsel *evsel,
1661 int nr_cpus, int cpu,
1662 struct perf_thread_map *threads,
1663 int thread, int err)
1664{
1665 pid_t ignore_pid = perf_thread_map__pid(threads, thread);
1666
1667 if (!evsel->ignore_missing_thread)
1668 return false;
1669
1670
1671 if (evsel->core.system_wide)
1672 return false;
1673
1674
1675 if (err != -ESRCH)
1676 return false;
1677
1678
1679 if (threads->nr == 1)
1680 return false;
1681
1682
1683
1684
1685
1686 if (update_fds(evsel, nr_cpus, cpu, threads->nr, thread))
1687 return false;
1688
1689 if (thread_map__remove(threads, thread))
1690 return false;
1691
1692 pr_warning("WARNING: Ignored open failure for pid %d\n",
1693 ignore_pid);
1694 return true;
1695}
1696
1697static int __open_attr__fprintf(FILE *fp, const char *name, const char *val,
1698 void *priv __maybe_unused)
1699{
1700 return fprintf(fp, " %-32s %s\n", name, val);
1701}
1702
1703static void display_attr(struct perf_event_attr *attr)
1704{
1705 if (verbose >= 2 || debug_peo_args) {
1706 fprintf(stderr, "%.60s\n", graph_dotted_line);
1707 fprintf(stderr, "perf_event_attr:\n");
1708 perf_event_attr__fprintf(stderr, attr, __open_attr__fprintf, NULL);
1709 fprintf(stderr, "%.60s\n", graph_dotted_line);
1710 }
1711}
1712
1713static int perf_event_open(struct evsel *evsel,
1714 pid_t pid, int cpu, int group_fd,
1715 unsigned long flags)
1716{
1717 int precise_ip = evsel->core.attr.precise_ip;
1718 int fd;
1719
1720 while (1) {
1721 pr_debug2_peo("sys_perf_event_open: pid %d cpu %d group_fd %d flags %#lx",
1722 pid, cpu, group_fd, flags);
1723
1724 fd = sys_perf_event_open(&evsel->core.attr, pid, cpu, group_fd, flags);
1725 if (fd >= 0)
1726 break;
1727
1728
1729 if (!evsel->precise_max)
1730 break;
1731
1732
1733
1734
1735
1736 if (!evsel->core.attr.precise_ip) {
1737 evsel->core.attr.precise_ip = precise_ip;
1738 break;
1739 }
1740
1741 pr_debug2_peo("\nsys_perf_event_open failed, error %d\n", -ENOTSUP);
1742 evsel->core.attr.precise_ip--;
1743 pr_debug2_peo("decreasing precise_ip by one (%d)\n", evsel->core.attr.precise_ip);
1744 display_attr(&evsel->core.attr);
1745 }
1746
1747 return fd;
1748}
1749
1750static int evsel__open_cpu(struct evsel *evsel, struct perf_cpu_map *cpus,
1751 struct perf_thread_map *threads,
1752 int start_cpu, int end_cpu)
1753{
1754 int cpu, thread, nthreads;
1755 unsigned long flags = PERF_FLAG_FD_CLOEXEC;
1756 int pid = -1, err, old_errno;
1757 enum { NO_CHANGE, SET_TO_MAX, INCREASED_MAX } set_rlimit = NO_CHANGE;
1758
1759 if ((perf_missing_features.write_backward && evsel->core.attr.write_backward) ||
1760 (perf_missing_features.aux_output && evsel->core.attr.aux_output))
1761 return -EINVAL;
1762
1763 if (cpus == NULL) {
1764 static struct perf_cpu_map *empty_cpu_map;
1765
1766 if (empty_cpu_map == NULL) {
1767 empty_cpu_map = perf_cpu_map__dummy_new();
1768 if (empty_cpu_map == NULL)
1769 return -ENOMEM;
1770 }
1771
1772 cpus = empty_cpu_map;
1773 }
1774
1775 if (threads == NULL) {
1776 static struct perf_thread_map *empty_thread_map;
1777
1778 if (empty_thread_map == NULL) {
1779 empty_thread_map = thread_map__new_by_tid(-1);
1780 if (empty_thread_map == NULL)
1781 return -ENOMEM;
1782 }
1783
1784 threads = empty_thread_map;
1785 }
1786
1787 if (evsel->core.system_wide)
1788 nthreads = 1;
1789 else
1790 nthreads = threads->nr;
1791
1792 if (evsel->core.fd == NULL &&
1793 perf_evsel__alloc_fd(&evsel->core, cpus->nr, nthreads) < 0)
1794 return -ENOMEM;
1795
1796 if (evsel->cgrp) {
1797 flags |= PERF_FLAG_PID_CGROUP;
1798 pid = evsel->cgrp->fd;
1799 }
1800
1801fallback_missing_features:
1802 if (perf_missing_features.weight_struct) {
1803 evsel__set_sample_bit(evsel, WEIGHT);
1804 evsel__reset_sample_bit(evsel, WEIGHT_STRUCT);
1805 }
1806 if (perf_missing_features.clockid_wrong)
1807 evsel->core.attr.clockid = CLOCK_MONOTONIC;
1808 if (perf_missing_features.clockid) {
1809 evsel->core.attr.use_clockid = 0;
1810 evsel->core.attr.clockid = 0;
1811 }
1812 if (perf_missing_features.cloexec)
1813 flags &= ~(unsigned long)PERF_FLAG_FD_CLOEXEC;
1814 if (perf_missing_features.mmap2)
1815 evsel->core.attr.mmap2 = 0;
1816 if (perf_missing_features.exclude_guest)
1817 evsel->core.attr.exclude_guest = evsel->core.attr.exclude_host = 0;
1818 if (perf_missing_features.lbr_flags)
1819 evsel->core.attr.branch_sample_type &= ~(PERF_SAMPLE_BRANCH_NO_FLAGS |
1820 PERF_SAMPLE_BRANCH_NO_CYCLES);
1821 if (perf_missing_features.group_read && evsel->core.attr.inherit)
1822 evsel->core.attr.read_format &= ~(PERF_FORMAT_GROUP|PERF_FORMAT_ID);
1823 if (perf_missing_features.ksymbol)
1824 evsel->core.attr.ksymbol = 0;
1825 if (perf_missing_features.bpf)
1826 evsel->core.attr.bpf_event = 0;
1827 if (perf_missing_features.branch_hw_idx)
1828 evsel->core.attr.branch_sample_type &= ~PERF_SAMPLE_BRANCH_HW_INDEX;
1829retry_sample_id:
1830 if (perf_missing_features.sample_id_all)
1831 evsel->core.attr.sample_id_all = 0;
1832
1833 display_attr(&evsel->core.attr);
1834
1835 for (cpu = start_cpu; cpu < end_cpu; cpu++) {
1836
1837 for (thread = 0; thread < nthreads; thread++) {
1838 int fd, group_fd;
1839
1840 if (!evsel->cgrp && !evsel->core.system_wide)
1841 pid = perf_thread_map__pid(threads, thread);
1842
1843 group_fd = get_group_fd(evsel, cpu, thread);
1844retry_open:
1845 test_attr__ready();
1846
1847 fd = perf_event_open(evsel, pid, cpus->map[cpu],
1848 group_fd, flags);
1849
1850 FD(evsel, cpu, thread) = fd;
1851
1852 bpf_counter__install_pe(evsel, cpu, fd);
1853
1854 if (unlikely(test_attr__enabled)) {
1855 test_attr__open(&evsel->core.attr, pid, cpus->map[cpu],
1856 fd, group_fd, flags);
1857 }
1858
1859 if (fd < 0) {
1860 err = -errno;
1861
1862 if (ignore_missing_thread(evsel, cpus->nr, cpu, threads, thread, err)) {
1863
1864
1865
1866
1867
1868 nthreads--;
1869 thread--;
1870
1871
1872 err = 0;
1873 continue;
1874 }
1875
1876 pr_debug2_peo("\nsys_perf_event_open failed, error %d\n",
1877 err);
1878 goto try_fallback;
1879 }
1880
1881 pr_debug2_peo(" = %d\n", fd);
1882
1883 if (evsel->bpf_fd >= 0) {
1884 int evt_fd = fd;
1885 int bpf_fd = evsel->bpf_fd;
1886
1887 err = ioctl(evt_fd,
1888 PERF_EVENT_IOC_SET_BPF,
1889 bpf_fd);
1890 if (err && errno != EEXIST) {
1891 pr_err("failed to attach bpf fd %d: %s\n",
1892 bpf_fd, strerror(errno));
1893 err = -EINVAL;
1894 goto out_close;
1895 }
1896 }
1897
1898 set_rlimit = NO_CHANGE;
1899
1900
1901
1902
1903
1904 if (perf_missing_features.clockid ||
1905 perf_missing_features.clockid_wrong) {
1906 err = -EINVAL;
1907 goto out_close;
1908 }
1909 }
1910 }
1911
1912 return 0;
1913
1914try_fallback:
1915
1916
1917
1918
1919 if (err == -EMFILE && set_rlimit < INCREASED_MAX) {
1920 struct rlimit l;
1921
1922 old_errno = errno;
1923 if (getrlimit(RLIMIT_NOFILE, &l) == 0) {
1924 if (set_rlimit == NO_CHANGE)
1925 l.rlim_cur = l.rlim_max;
1926 else {
1927 l.rlim_cur = l.rlim_max + 1000;
1928 l.rlim_max = l.rlim_cur;
1929 }
1930 if (setrlimit(RLIMIT_NOFILE, &l) == 0) {
1931 set_rlimit++;
1932 errno = old_errno;
1933 goto retry_open;
1934 }
1935 }
1936 errno = old_errno;
1937 }
1938
1939 if (err != -EINVAL || cpu > 0 || thread > 0)
1940 goto out_close;
1941
1942
1943
1944
1945
1946 if (!perf_missing_features.weight_struct &&
1947 (evsel->core.attr.sample_type & PERF_SAMPLE_WEIGHT_STRUCT)) {
1948 perf_missing_features.weight_struct = true;
1949 pr_debug2("switching off weight struct support\n");
1950 goto fallback_missing_features;
1951 } else if (!perf_missing_features.code_page_size &&
1952 (evsel->core.attr.sample_type & PERF_SAMPLE_CODE_PAGE_SIZE)) {
1953 perf_missing_features.code_page_size = true;
1954 pr_debug2_peo("Kernel has no PERF_SAMPLE_CODE_PAGE_SIZE support, bailing out\n");
1955 goto out_close;
1956 } else if (!perf_missing_features.data_page_size &&
1957 (evsel->core.attr.sample_type & PERF_SAMPLE_DATA_PAGE_SIZE)) {
1958 perf_missing_features.data_page_size = true;
1959 pr_debug2_peo("Kernel has no PERF_SAMPLE_DATA_PAGE_SIZE support, bailing out\n");
1960 goto out_close;
1961 } else if (!perf_missing_features.cgroup && evsel->core.attr.cgroup) {
1962 perf_missing_features.cgroup = true;
1963 pr_debug2_peo("Kernel has no cgroup sampling support, bailing out\n");
1964 goto out_close;
1965 } else if (!perf_missing_features.branch_hw_idx &&
1966 (evsel->core.attr.branch_sample_type & PERF_SAMPLE_BRANCH_HW_INDEX)) {
1967 perf_missing_features.branch_hw_idx = true;
1968 pr_debug2("switching off branch HW index support\n");
1969 goto fallback_missing_features;
1970 } else if (!perf_missing_features.aux_output && evsel->core.attr.aux_output) {
1971 perf_missing_features.aux_output = true;
1972 pr_debug2_peo("Kernel has no attr.aux_output support, bailing out\n");
1973 goto out_close;
1974 } else if (!perf_missing_features.bpf && evsel->core.attr.bpf_event) {
1975 perf_missing_features.bpf = true;
1976 pr_debug2_peo("switching off bpf_event\n");
1977 goto fallback_missing_features;
1978 } else if (!perf_missing_features.ksymbol && evsel->core.attr.ksymbol) {
1979 perf_missing_features.ksymbol = true;
1980 pr_debug2_peo("switching off ksymbol\n");
1981 goto fallback_missing_features;
1982 } else if (!perf_missing_features.write_backward && evsel->core.attr.write_backward) {
1983 perf_missing_features.write_backward = true;
1984 pr_debug2_peo("switching off write_backward\n");
1985 goto out_close;
1986 } else if (!perf_missing_features.clockid_wrong && evsel->core.attr.use_clockid) {
1987 perf_missing_features.clockid_wrong = true;
1988 pr_debug2_peo("switching off clockid\n");
1989 goto fallback_missing_features;
1990 } else if (!perf_missing_features.clockid && evsel->core.attr.use_clockid) {
1991 perf_missing_features.clockid = true;
1992 pr_debug2_peo("switching off use_clockid\n");
1993 goto fallback_missing_features;
1994 } else if (!perf_missing_features.cloexec && (flags & PERF_FLAG_FD_CLOEXEC)) {
1995 perf_missing_features.cloexec = true;
1996 pr_debug2_peo("switching off cloexec flag\n");
1997 goto fallback_missing_features;
1998 } else if (!perf_missing_features.mmap2 && evsel->core.attr.mmap2) {
1999 perf_missing_features.mmap2 = true;
2000 pr_debug2_peo("switching off mmap2\n");
2001 goto fallback_missing_features;
2002 } else if (!perf_missing_features.exclude_guest &&
2003 (evsel->core.attr.exclude_guest || evsel->core.attr.exclude_host)) {
2004 perf_missing_features.exclude_guest = true;
2005 pr_debug2_peo("switching off exclude_guest, exclude_host\n");
2006 goto fallback_missing_features;
2007 } else if (!perf_missing_features.sample_id_all) {
2008 perf_missing_features.sample_id_all = true;
2009 pr_debug2_peo("switching off sample_id_all\n");
2010 goto retry_sample_id;
2011 } else if (!perf_missing_features.lbr_flags &&
2012 (evsel->core.attr.branch_sample_type &
2013 (PERF_SAMPLE_BRANCH_NO_CYCLES |
2014 PERF_SAMPLE_BRANCH_NO_FLAGS))) {
2015 perf_missing_features.lbr_flags = true;
2016 pr_debug2_peo("switching off branch sample type no (cycles/flags)\n");
2017 goto fallback_missing_features;
2018 } else if (!perf_missing_features.group_read &&
2019 evsel->core.attr.inherit &&
2020 (evsel->core.attr.read_format & PERF_FORMAT_GROUP) &&
2021 evsel__is_group_leader(evsel)) {
2022 perf_missing_features.group_read = true;
2023 pr_debug2_peo("switching off group read\n");
2024 goto fallback_missing_features;
2025 }
2026out_close:
2027 if (err)
2028 threads->err_thread = thread;
2029
2030 old_errno = errno;
2031 do {
2032 while (--thread >= 0) {
2033 if (FD(evsel, cpu, thread) >= 0)
2034 close(FD(evsel, cpu, thread));
2035 FD(evsel, cpu, thread) = -1;
2036 }
2037 thread = nthreads;
2038 } while (--cpu >= 0);
2039 errno = old_errno;
2040 return err;
2041}
2042
2043int evsel__open(struct evsel *evsel, struct perf_cpu_map *cpus,
2044 struct perf_thread_map *threads)
2045{
2046 return evsel__open_cpu(evsel, cpus, threads, 0, cpus ? cpus->nr : 1);
2047}
2048
2049void evsel__close(struct evsel *evsel)
2050{
2051 perf_evsel__close(&evsel->core);
2052 perf_evsel__free_id(&evsel->core);
2053}
2054
2055int evsel__open_per_cpu(struct evsel *evsel, struct perf_cpu_map *cpus, int cpu)
2056{
2057 if (cpu == -1)
2058 return evsel__open_cpu(evsel, cpus, NULL, 0,
2059 cpus ? cpus->nr : 1);
2060
2061 return evsel__open_cpu(evsel, cpus, NULL, cpu, cpu + 1);
2062}
2063
2064int evsel__open_per_thread(struct evsel *evsel, struct perf_thread_map *threads)
2065{
2066 return evsel__open(evsel, NULL, threads);
2067}
2068
2069static int perf_evsel__parse_id_sample(const struct evsel *evsel,
2070 const union perf_event *event,
2071 struct perf_sample *sample)
2072{
2073 u64 type = evsel->core.attr.sample_type;
2074 const __u64 *array = event->sample.array;
2075 bool swapped = evsel->needs_swap;
2076 union u64_swap u;
2077
2078 array += ((event->header.size -
2079 sizeof(event->header)) / sizeof(u64)) - 1;
2080
2081 if (type & PERF_SAMPLE_IDENTIFIER) {
2082 sample->id = *array;
2083 array--;
2084 }
2085
2086 if (type & PERF_SAMPLE_CPU) {
2087 u.val64 = *array;
2088 if (swapped) {
2089
2090 u.val64 = bswap_64(u.val64);
2091 u.val32[0] = bswap_32(u.val32[0]);
2092 }
2093
2094 sample->cpu = u.val32[0];
2095 array--;
2096 }
2097
2098 if (type & PERF_SAMPLE_STREAM_ID) {
2099 sample->stream_id = *array;
2100 array--;
2101 }
2102
2103 if (type & PERF_SAMPLE_ID) {
2104 sample->id = *array;
2105 array--;
2106 }
2107
2108 if (type & PERF_SAMPLE_TIME) {
2109 sample->time = *array;
2110 array--;
2111 }
2112
2113 if (type & PERF_SAMPLE_TID) {
2114 u.val64 = *array;
2115 if (swapped) {
2116
2117 u.val64 = bswap_64(u.val64);
2118 u.val32[0] = bswap_32(u.val32[0]);
2119 u.val32[1] = bswap_32(u.val32[1]);
2120 }
2121
2122 sample->pid = u.val32[0];
2123 sample->tid = u.val32[1];
2124 array--;
2125 }
2126
2127 return 0;
2128}
2129
2130static inline bool overflow(const void *endp, u16 max_size, const void *offset,
2131 u64 size)
2132{
2133 return size > max_size || offset + size > endp;
2134}
2135
2136#define OVERFLOW_CHECK(offset, size, max_size) \
2137 do { \
2138 if (overflow(endp, (max_size), (offset), (size))) \
2139 return -EFAULT; \
2140 } while (0)
2141
2142#define OVERFLOW_CHECK_u64(offset) \
2143 OVERFLOW_CHECK(offset, sizeof(u64), sizeof(u64))
2144
2145static int
2146perf_event__check_size(union perf_event *event, unsigned int sample_size)
2147{
2148
2149
2150
2151
2152
2153 if (sample_size + sizeof(event->header) > event->header.size)
2154 return -EFAULT;
2155
2156 return 0;
2157}
2158
2159void __weak arch_perf_parse_sample_weight(struct perf_sample *data,
2160 const __u64 *array,
2161 u64 type __maybe_unused)
2162{
2163 data->weight = *array;
2164}
2165
2166int evsel__parse_sample(struct evsel *evsel, union perf_event *event,
2167 struct perf_sample *data)
2168{
2169 u64 type = evsel->core.attr.sample_type;
2170 bool swapped = evsel->needs_swap;
2171 const __u64 *array;
2172 u16 max_size = event->header.size;
2173 const void *endp = (void *)event + max_size;
2174 u64 sz;
2175
2176
2177
2178
2179
2180 union u64_swap u;
2181
2182 memset(data, 0, sizeof(*data));
2183 data->cpu = data->pid = data->tid = -1;
2184 data->stream_id = data->id = data->time = -1ULL;
2185 data->period = evsel->core.attr.sample_period;
2186 data->cpumode = event->header.misc & PERF_RECORD_MISC_CPUMODE_MASK;
2187 data->misc = event->header.misc;
2188 data->id = -1ULL;
2189 data->data_src = PERF_MEM_DATA_SRC_NONE;
2190
2191 if (event->header.type != PERF_RECORD_SAMPLE) {
2192 if (!evsel->core.attr.sample_id_all)
2193 return 0;
2194 return perf_evsel__parse_id_sample(evsel, event, data);
2195 }
2196
2197 array = event->sample.array;
2198
2199 if (perf_event__check_size(event, evsel->sample_size))
2200 return -EFAULT;
2201
2202 if (type & PERF_SAMPLE_IDENTIFIER) {
2203 data->id = *array;
2204 array++;
2205 }
2206
2207 if (type & PERF_SAMPLE_IP) {
2208 data->ip = *array;
2209 array++;
2210 }
2211
2212 if (type & PERF_SAMPLE_TID) {
2213 u.val64 = *array;
2214 if (swapped) {
2215
2216 u.val64 = bswap_64(u.val64);
2217 u.val32[0] = bswap_32(u.val32[0]);
2218 u.val32[1] = bswap_32(u.val32[1]);
2219 }
2220
2221 data->pid = u.val32[0];
2222 data->tid = u.val32[1];
2223 array++;
2224 }
2225
2226 if (type & PERF_SAMPLE_TIME) {
2227 data->time = *array;
2228 array++;
2229 }
2230
2231 if (type & PERF_SAMPLE_ADDR) {
2232 data->addr = *array;
2233 array++;
2234 }
2235
2236 if (type & PERF_SAMPLE_ID) {
2237 data->id = *array;
2238 array++;
2239 }
2240
2241 if (type & PERF_SAMPLE_STREAM_ID) {
2242 data->stream_id = *array;
2243 array++;
2244 }
2245
2246 if (type & PERF_SAMPLE_CPU) {
2247
2248 u.val64 = *array;
2249 if (swapped) {
2250
2251 u.val64 = bswap_64(u.val64);
2252 u.val32[0] = bswap_32(u.val32[0]);
2253 }
2254
2255 data->cpu = u.val32[0];
2256 array++;
2257 }
2258
2259 if (type & PERF_SAMPLE_PERIOD) {
2260 data->period = *array;
2261 array++;
2262 }
2263
2264 if (type & PERF_SAMPLE_READ) {
2265 u64 read_format = evsel->core.attr.read_format;
2266
2267 OVERFLOW_CHECK_u64(array);
2268 if (read_format & PERF_FORMAT_GROUP)
2269 data->read.group.nr = *array;
2270 else
2271 data->read.one.value = *array;
2272
2273 array++;
2274
2275 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
2276 OVERFLOW_CHECK_u64(array);
2277 data->read.time_enabled = *array;
2278 array++;
2279 }
2280
2281 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
2282 OVERFLOW_CHECK_u64(array);
2283 data->read.time_running = *array;
2284 array++;
2285 }
2286
2287
2288 if (read_format & PERF_FORMAT_GROUP) {
2289 const u64 max_group_nr = UINT64_MAX /
2290 sizeof(struct sample_read_value);
2291
2292 if (data->read.group.nr > max_group_nr)
2293 return -EFAULT;
2294 sz = data->read.group.nr *
2295 sizeof(struct sample_read_value);
2296 OVERFLOW_CHECK(array, sz, max_size);
2297 data->read.group.values =
2298 (struct sample_read_value *)array;
2299 array = (void *)array + sz;
2300 } else {
2301 OVERFLOW_CHECK_u64(array);
2302 data->read.one.id = *array;
2303 array++;
2304 }
2305 }
2306
2307 if (type & PERF_SAMPLE_CALLCHAIN) {
2308 const u64 max_callchain_nr = UINT64_MAX / sizeof(u64);
2309
2310 OVERFLOW_CHECK_u64(array);
2311 data->callchain = (struct ip_callchain *)array++;
2312 if (data->callchain->nr > max_callchain_nr)
2313 return -EFAULT;
2314 sz = data->callchain->nr * sizeof(u64);
2315 OVERFLOW_CHECK(array, sz, max_size);
2316 array = (void *)array + sz;
2317 }
2318
2319 if (type & PERF_SAMPLE_RAW) {
2320 OVERFLOW_CHECK_u64(array);
2321 u.val64 = *array;
2322
2323
2324
2325
2326
2327
2328
2329 if (swapped) {
2330 u.val64 = bswap_64(u.val64);
2331 u.val32[0] = bswap_32(u.val32[0]);
2332 u.val32[1] = bswap_32(u.val32[1]);
2333 }
2334 data->raw_size = u.val32[0];
2335
2336
2337
2338
2339
2340 if (swapped)
2341 mem_bswap_64((void *) array, data->raw_size);
2342
2343 array = (void *)array + sizeof(u32);
2344
2345 OVERFLOW_CHECK(array, data->raw_size, max_size);
2346 data->raw_data = (void *)array;
2347 array = (void *)array + data->raw_size;
2348 }
2349
2350 if (type & PERF_SAMPLE_BRANCH_STACK) {
2351 const u64 max_branch_nr = UINT64_MAX /
2352 sizeof(struct branch_entry);
2353
2354 OVERFLOW_CHECK_u64(array);
2355 data->branch_stack = (struct branch_stack *)array++;
2356
2357 if (data->branch_stack->nr > max_branch_nr)
2358 return -EFAULT;
2359
2360 sz = data->branch_stack->nr * sizeof(struct branch_entry);
2361 if (evsel__has_branch_hw_idx(evsel))
2362 sz += sizeof(u64);
2363 else
2364 data->no_hw_idx = true;
2365 OVERFLOW_CHECK(array, sz, max_size);
2366 array = (void *)array + sz;
2367 }
2368
2369 if (type & PERF_SAMPLE_REGS_USER) {
2370 OVERFLOW_CHECK_u64(array);
2371 data->user_regs.abi = *array;
2372 array++;
2373
2374 if (data->user_regs.abi) {
2375 u64 mask = evsel->core.attr.sample_regs_user;
2376
2377 sz = hweight64(mask) * sizeof(u64);
2378 OVERFLOW_CHECK(array, sz, max_size);
2379 data->user_regs.mask = mask;
2380 data->user_regs.regs = (u64 *)array;
2381 array = (void *)array + sz;
2382 }
2383 }
2384
2385 if (type & PERF_SAMPLE_STACK_USER) {
2386 OVERFLOW_CHECK_u64(array);
2387 sz = *array++;
2388
2389 data->user_stack.offset = ((char *)(array - 1)
2390 - (char *) event);
2391
2392 if (!sz) {
2393 data->user_stack.size = 0;
2394 } else {
2395 OVERFLOW_CHECK(array, sz, max_size);
2396 data->user_stack.data = (char *)array;
2397 array = (void *)array + sz;
2398 OVERFLOW_CHECK_u64(array);
2399 data->user_stack.size = *array++;
2400 if (WARN_ONCE(data->user_stack.size > sz,
2401 "user stack dump failure\n"))
2402 return -EFAULT;
2403 }
2404 }
2405
2406 if (type & PERF_SAMPLE_WEIGHT_TYPE) {
2407 OVERFLOW_CHECK_u64(array);
2408 arch_perf_parse_sample_weight(data, array, type);
2409 array++;
2410 }
2411
2412 if (type & PERF_SAMPLE_DATA_SRC) {
2413 OVERFLOW_CHECK_u64(array);
2414 data->data_src = *array;
2415 array++;
2416 }
2417
2418 if (type & PERF_SAMPLE_TRANSACTION) {
2419 OVERFLOW_CHECK_u64(array);
2420 data->transaction = *array;
2421 array++;
2422 }
2423
2424 data->intr_regs.abi = PERF_SAMPLE_REGS_ABI_NONE;
2425 if (type & PERF_SAMPLE_REGS_INTR) {
2426 OVERFLOW_CHECK_u64(array);
2427 data->intr_regs.abi = *array;
2428 array++;
2429
2430 if (data->intr_regs.abi != PERF_SAMPLE_REGS_ABI_NONE) {
2431 u64 mask = evsel->core.attr.sample_regs_intr;
2432
2433 sz = hweight64(mask) * sizeof(u64);
2434 OVERFLOW_CHECK(array, sz, max_size);
2435 data->intr_regs.mask = mask;
2436 data->intr_regs.regs = (u64 *)array;
2437 array = (void *)array + sz;
2438 }
2439 }
2440
2441 data->phys_addr = 0;
2442 if (type & PERF_SAMPLE_PHYS_ADDR) {
2443 data->phys_addr = *array;
2444 array++;
2445 }
2446
2447 data->cgroup = 0;
2448 if (type & PERF_SAMPLE_CGROUP) {
2449 data->cgroup = *array;
2450 array++;
2451 }
2452
2453 data->data_page_size = 0;
2454 if (type & PERF_SAMPLE_DATA_PAGE_SIZE) {
2455 data->data_page_size = *array;
2456 array++;
2457 }
2458
2459 data->code_page_size = 0;
2460 if (type & PERF_SAMPLE_CODE_PAGE_SIZE) {
2461 data->code_page_size = *array;
2462 array++;
2463 }
2464
2465 if (type & PERF_SAMPLE_AUX) {
2466 OVERFLOW_CHECK_u64(array);
2467 sz = *array++;
2468
2469 OVERFLOW_CHECK(array, sz, max_size);
2470
2471 if (swapped)
2472 mem_bswap_64((char *)array, sz);
2473 data->aux_sample.size = sz;
2474 data->aux_sample.data = (char *)array;
2475 array = (void *)array + sz;
2476 }
2477
2478 return 0;
2479}
2480
2481int evsel__parse_sample_timestamp(struct evsel *evsel, union perf_event *event,
2482 u64 *timestamp)
2483{
2484 u64 type = evsel->core.attr.sample_type;
2485 const __u64 *array;
2486
2487 if (!(type & PERF_SAMPLE_TIME))
2488 return -1;
2489
2490 if (event->header.type != PERF_RECORD_SAMPLE) {
2491 struct perf_sample data = {
2492 .time = -1ULL,
2493 };
2494
2495 if (!evsel->core.attr.sample_id_all)
2496 return -1;
2497 if (perf_evsel__parse_id_sample(evsel, event, &data))
2498 return -1;
2499
2500 *timestamp = data.time;
2501 return 0;
2502 }
2503
2504 array = event->sample.array;
2505
2506 if (perf_event__check_size(event, evsel->sample_size))
2507 return -EFAULT;
2508
2509 if (type & PERF_SAMPLE_IDENTIFIER)
2510 array++;
2511
2512 if (type & PERF_SAMPLE_IP)
2513 array++;
2514
2515 if (type & PERF_SAMPLE_TID)
2516 array++;
2517
2518 if (type & PERF_SAMPLE_TIME)
2519 *timestamp = *array;
2520
2521 return 0;
2522}
2523
2524struct tep_format_field *evsel__field(struct evsel *evsel, const char *name)
2525{
2526 return tep_find_field(evsel->tp_format, name);
2527}
2528
2529void *evsel__rawptr(struct evsel *evsel, struct perf_sample *sample, const char *name)
2530{
2531 struct tep_format_field *field = evsel__field(evsel, name);
2532 int offset;
2533
2534 if (!field)
2535 return NULL;
2536
2537 offset = field->offset;
2538
2539 if (field->flags & TEP_FIELD_IS_DYNAMIC) {
2540 offset = *(int *)(sample->raw_data + field->offset);
2541 offset &= 0xffff;
2542 }
2543
2544 return sample->raw_data + offset;
2545}
2546
2547u64 format_field__intval(struct tep_format_field *field, struct perf_sample *sample,
2548 bool needs_swap)
2549{
2550 u64 value;
2551 void *ptr = sample->raw_data + field->offset;
2552
2553 switch (field->size) {
2554 case 1:
2555 return *(u8 *)ptr;
2556 case 2:
2557 value = *(u16 *)ptr;
2558 break;
2559 case 4:
2560 value = *(u32 *)ptr;
2561 break;
2562 case 8:
2563 memcpy(&value, ptr, sizeof(u64));
2564 break;
2565 default:
2566 return 0;
2567 }
2568
2569 if (!needs_swap)
2570 return value;
2571
2572 switch (field->size) {
2573 case 2:
2574 return bswap_16(value);
2575 case 4:
2576 return bswap_32(value);
2577 case 8:
2578 return bswap_64(value);
2579 default:
2580 return 0;
2581 }
2582
2583 return 0;
2584}
2585
2586u64 evsel__intval(struct evsel *evsel, struct perf_sample *sample, const char *name)
2587{
2588 struct tep_format_field *field = evsel__field(evsel, name);
2589
2590 if (!field)
2591 return 0;
2592
2593 return field ? format_field__intval(field, sample, evsel->needs_swap) : 0;
2594}
2595
2596bool evsel__fallback(struct evsel *evsel, int err, char *msg, size_t msgsize)
2597{
2598 int paranoid;
2599
2600 if ((err == ENOENT || err == ENXIO || err == ENODEV) &&
2601 evsel->core.attr.type == PERF_TYPE_HARDWARE &&
2602 evsel->core.attr.config == PERF_COUNT_HW_CPU_CYCLES) {
2603
2604
2605
2606
2607
2608
2609
2610
2611 scnprintf(msg, msgsize, "%s",
2612"The cycles event is not supported, trying to fall back to cpu-clock-ticks");
2613
2614 evsel->core.attr.type = PERF_TYPE_SOFTWARE;
2615 evsel->core.attr.config = PERF_COUNT_SW_CPU_CLOCK;
2616
2617 zfree(&evsel->name);
2618 return true;
2619 } else if (err == EACCES && !evsel->core.attr.exclude_kernel &&
2620 (paranoid = perf_event_paranoid()) > 1) {
2621 const char *name = evsel__name(evsel);
2622 char *new_name;
2623 const char *sep = ":";
2624
2625
2626 if (evsel->core.attr.exclude_user)
2627 return false;
2628
2629
2630 if (strchr(name, '/') ||
2631 (strchr(name, ':') && !evsel->is_libpfm_event))
2632 sep = "";
2633
2634 if (asprintf(&new_name, "%s%su", name, sep) < 0)
2635 return false;
2636
2637 if (evsel->name)
2638 free(evsel->name);
2639 evsel->name = new_name;
2640 scnprintf(msg, msgsize, "kernel.perf_event_paranoid=%d, trying "
2641 "to fall back to excluding kernel and hypervisor "
2642 " samples", paranoid);
2643 evsel->core.attr.exclude_kernel = 1;
2644 evsel->core.attr.exclude_hv = 1;
2645
2646 return true;
2647 }
2648
2649 return false;
2650}
2651
2652static bool find_process(const char *name)
2653{
2654 size_t len = strlen(name);
2655 DIR *dir;
2656 struct dirent *d;
2657 int ret = -1;
2658
2659 dir = opendir(procfs__mountpoint());
2660 if (!dir)
2661 return false;
2662
2663
2664 while (ret && (d = readdir(dir)) != NULL) {
2665 char path[PATH_MAX];
2666 char *data;
2667 size_t size;
2668
2669 if ((d->d_type != DT_DIR) ||
2670 !strcmp(".", d->d_name) ||
2671 !strcmp("..", d->d_name))
2672 continue;
2673
2674 scnprintf(path, sizeof(path), "%s/%s/comm",
2675 procfs__mountpoint(), d->d_name);
2676
2677 if (filename__read_str(path, &data, &size))
2678 continue;
2679
2680 ret = strncmp(name, data, len);
2681 free(data);
2682 }
2683
2684 closedir(dir);
2685 return ret ? false : true;
2686}
2687
2688int evsel__open_strerror(struct evsel *evsel, struct target *target,
2689 int err, char *msg, size_t size)
2690{
2691 char sbuf[STRERR_BUFSIZE];
2692 int printed = 0, enforced = 0;
2693
2694 switch (err) {
2695 case EPERM:
2696 case EACCES:
2697 printed += scnprintf(msg + printed, size - printed,
2698 "Access to performance monitoring and observability operations is limited.\n");
2699
2700 if (!sysfs__read_int("fs/selinux/enforce", &enforced)) {
2701 if (enforced) {
2702 printed += scnprintf(msg + printed, size - printed,
2703 "Enforced MAC policy settings (SELinux) can limit access to performance\n"
2704 "monitoring and observability operations. Inspect system audit records for\n"
2705 "more perf_event access control information and adjusting the policy.\n");
2706 }
2707 }
2708
2709 if (err == EPERM)
2710 printed += scnprintf(msg, size,
2711 "No permission to enable %s event.\n\n", evsel__name(evsel));
2712
2713 return scnprintf(msg + printed, size - printed,
2714 "Consider adjusting /proc/sys/kernel/perf_event_paranoid setting to open\n"
2715 "access to performance monitoring and observability operations for processes\n"
2716 "without CAP_PERFMON, CAP_SYS_PTRACE or CAP_SYS_ADMIN Linux capability.\n"
2717 "More information can be found at 'Perf events and tool security' document:\n"
2718 "https://www.kernel.org/doc/html/latest/admin-guide/perf-security.html\n"
2719 "perf_event_paranoid setting is %d:\n"
2720 " -1: Allow use of (almost) all events by all users\n"
2721 " Ignore mlock limit after perf_event_mlock_kb without CAP_IPC_LOCK\n"
2722 ">= 0: Disallow raw and ftrace function tracepoint access\n"
2723 ">= 1: Disallow CPU event access\n"
2724 ">= 2: Disallow kernel profiling\n"
2725 "To make the adjusted perf_event_paranoid setting permanent preserve it\n"
2726 "in /etc/sysctl.conf (e.g. kernel.perf_event_paranoid = <setting>)",
2727 perf_event_paranoid());
2728 case ENOENT:
2729 return scnprintf(msg, size, "The %s event is not supported.", evsel__name(evsel));
2730 case EMFILE:
2731 return scnprintf(msg, size, "%s",
2732 "Too many events are opened.\n"
2733 "Probably the maximum number of open file descriptors has been reached.\n"
2734 "Hint: Try again after reducing the number of events.\n"
2735 "Hint: Try increasing the limit with 'ulimit -n <limit>'");
2736 case ENOMEM:
2737 if (evsel__has_callchain(evsel) &&
2738 access("/proc/sys/kernel/perf_event_max_stack", F_OK) == 0)
2739 return scnprintf(msg, size,
2740 "Not enough memory to setup event with callchain.\n"
2741 "Hint: Try tweaking /proc/sys/kernel/perf_event_max_stack\n"
2742 "Hint: Current value: %d", sysctl__max_stack());
2743 break;
2744 case ENODEV:
2745 if (target->cpu_list)
2746 return scnprintf(msg, size, "%s",
2747 "No such device - did you specify an out-of-range profile CPU?");
2748 break;
2749 case EOPNOTSUPP:
2750 if (evsel->core.attr.aux_output)
2751 return scnprintf(msg, size,
2752 "%s: PMU Hardware doesn't support 'aux_output' feature",
2753 evsel__name(evsel));
2754 if (evsel->core.attr.sample_period != 0)
2755 return scnprintf(msg, size,
2756 "%s: PMU Hardware doesn't support sampling/overflow-interrupts. Try 'perf stat'",
2757 evsel__name(evsel));
2758 if (evsel->core.attr.precise_ip)
2759 return scnprintf(msg, size, "%s",
2760 "\'precise\' request may not be supported. Try removing 'p' modifier.");
2761#if defined(__i386__) || defined(__x86_64__)
2762 if (evsel->core.attr.type == PERF_TYPE_HARDWARE)
2763 return scnprintf(msg, size, "%s",
2764 "No hardware sampling interrupt available.\n");
2765#endif
2766 break;
2767 case EBUSY:
2768 if (find_process("oprofiled"))
2769 return scnprintf(msg, size,
2770 "The PMU counters are busy/taken by another profiler.\n"
2771 "We found oprofile daemon running, please stop it and try again.");
2772 break;
2773 case EINVAL:
2774 if (evsel->core.attr.sample_type & PERF_SAMPLE_CODE_PAGE_SIZE && perf_missing_features.code_page_size)
2775 return scnprintf(msg, size, "Asking for the code page size isn't supported by this kernel.");
2776 if (evsel->core.attr.sample_type & PERF_SAMPLE_DATA_PAGE_SIZE && perf_missing_features.data_page_size)
2777 return scnprintf(msg, size, "Asking for the data page size isn't supported by this kernel.");
2778 if (evsel->core.attr.write_backward && perf_missing_features.write_backward)
2779 return scnprintf(msg, size, "Reading from overwrite event is not supported by this kernel.");
2780 if (perf_missing_features.clockid)
2781 return scnprintf(msg, size, "clockid feature not supported.");
2782 if (perf_missing_features.clockid_wrong)
2783 return scnprintf(msg, size, "wrong clockid (%d).", clockid);
2784 if (perf_missing_features.aux_output)
2785 return scnprintf(msg, size, "The 'aux_output' feature is not supported, update the kernel.");
2786 break;
2787 case ENODATA:
2788 return scnprintf(msg, size, "Cannot collect data source with the load latency event alone. "
2789 "Please add an auxiliary event in front of the load latency event.");
2790 default:
2791 break;
2792 }
2793
2794 return scnprintf(msg, size,
2795 "The sys_perf_event_open() syscall returned with %d (%s) for event (%s).\n"
2796 "/bin/dmesg | grep -i perf may provide additional information.\n",
2797 err, str_error_r(err, sbuf, sizeof(sbuf)), evsel__name(evsel));
2798}
2799
2800struct perf_env *evsel__env(struct evsel *evsel)
2801{
2802 if (evsel && evsel->evlist)
2803 return evsel->evlist->env;
2804 return &perf_env;
2805}
2806
2807static int store_evsel_ids(struct evsel *evsel, struct evlist *evlist)
2808{
2809 int cpu, thread;
2810
2811 for (cpu = 0; cpu < xyarray__max_x(evsel->core.fd); cpu++) {
2812 for (thread = 0; thread < xyarray__max_y(evsel->core.fd);
2813 thread++) {
2814 int fd = FD(evsel, cpu, thread);
2815
2816 if (perf_evlist__id_add_fd(&evlist->core, &evsel->core,
2817 cpu, thread, fd) < 0)
2818 return -1;
2819 }
2820 }
2821
2822 return 0;
2823}
2824
2825int evsel__store_ids(struct evsel *evsel, struct evlist *evlist)
2826{
2827 struct perf_cpu_map *cpus = evsel->core.cpus;
2828 struct perf_thread_map *threads = evsel->core.threads;
2829
2830 if (perf_evsel__alloc_id(&evsel->core, cpus->nr, threads->nr))
2831 return -ENOMEM;
2832
2833 return store_evsel_ids(evsel, evlist);
2834}
2835
2836void evsel__zero_per_pkg(struct evsel *evsel)
2837{
2838 struct hashmap_entry *cur;
2839 size_t bkt;
2840
2841 if (evsel->per_pkg_mask) {
2842 hashmap__for_each_entry(evsel->per_pkg_mask, cur, bkt)
2843 free((char *)cur->key);
2844
2845 hashmap__clear(evsel->per_pkg_mask);
2846 }
2847}
2848
2849bool evsel__is_hybrid(struct evsel *evsel)
2850{
2851 return evsel->pmu_name && perf_pmu__is_hybrid(evsel->pmu_name);
2852}
2853
2854struct evsel *evsel__leader(struct evsel *evsel)
2855{
2856 return container_of(evsel->core.leader, struct evsel, core);
2857}
2858
2859bool evsel__has_leader(struct evsel *evsel, struct evsel *leader)
2860{
2861 return evsel->core.leader == &leader->core;
2862}
2863
2864bool evsel__is_leader(struct evsel *evsel)
2865{
2866 return evsel__has_leader(evsel, evsel);
2867}
2868
2869void evsel__set_leader(struct evsel *evsel, struct evsel *leader)
2870{
2871 evsel->core.leader = &leader->core;
2872}
2873