1
2
3
4
5
6
7
8
9
10#include <byteswap.h>
11#include <errno.h>
12#include <inttypes.h>
13#include <linux/bitops.h>
14#include <api/fs/fs.h>
15#include <api/fs/tracing_path.h>
16#include <traceevent/event-parse.h>
17#include <linux/hw_breakpoint.h>
18#include <linux/perf_event.h>
19#include <linux/compiler.h>
20#include <linux/err.h>
21#include <sys/ioctl.h>
22#include <sys/resource.h>
23#include <sys/types.h>
24#include <dirent.h>
25#include "asm/bug.h"
26#include "callchain.h"
27#include "cgroup.h"
28#include "event.h"
29#include "evsel.h"
30#include "evlist.h"
31#include "util.h"
32#include "cpumap.h"
33#include "thread_map.h"
34#include "target.h"
35#include "perf_regs.h"
36#include "debug.h"
37#include "trace-event.h"
38#include "stat.h"
39#include "memswap.h"
40#include "util/parse-branch-options.h"
41
42#include "sane_ctype.h"
43
44struct perf_missing_features perf_missing_features;
45
46static clockid_t clockid;
47
48static int perf_evsel__no_extra_init(struct perf_evsel *evsel __maybe_unused)
49{
50 return 0;
51}
52
53void __weak test_attr__ready(void) { }
54
55static void perf_evsel__no_extra_fini(struct perf_evsel *evsel __maybe_unused)
56{
57}
58
59static struct {
60 size_t size;
61 int (*init)(struct perf_evsel *evsel);
62 void (*fini)(struct perf_evsel *evsel);
63} perf_evsel__object = {
64 .size = sizeof(struct perf_evsel),
65 .init = perf_evsel__no_extra_init,
66 .fini = perf_evsel__no_extra_fini,
67};
68
69int perf_evsel__object_config(size_t object_size,
70 int (*init)(struct perf_evsel *evsel),
71 void (*fini)(struct perf_evsel *evsel))
72{
73
74 if (object_size == 0)
75 goto set_methods;
76
77 if (perf_evsel__object.size > object_size)
78 return -EINVAL;
79
80 perf_evsel__object.size = object_size;
81
82set_methods:
83 if (init != NULL)
84 perf_evsel__object.init = init;
85
86 if (fini != NULL)
87 perf_evsel__object.fini = fini;
88
89 return 0;
90}
91
92#define FD(e, x, y) (*(int *)xyarray__entry(e->fd, x, y))
93
94int __perf_evsel__sample_size(u64 sample_type)
95{
96 u64 mask = sample_type & PERF_SAMPLE_MASK;
97 int size = 0;
98 int i;
99
100 for (i = 0; i < 64; i++) {
101 if (mask & (1ULL << i))
102 size++;
103 }
104
105 size *= sizeof(u64);
106
107 return size;
108}
109
110
111
112
113
114
115
116
117
118static int __perf_evsel__calc_id_pos(u64 sample_type)
119{
120 int idx = 0;
121
122 if (sample_type & PERF_SAMPLE_IDENTIFIER)
123 return 0;
124
125 if (!(sample_type & PERF_SAMPLE_ID))
126 return -1;
127
128 if (sample_type & PERF_SAMPLE_IP)
129 idx += 1;
130
131 if (sample_type & PERF_SAMPLE_TID)
132 idx += 1;
133
134 if (sample_type & PERF_SAMPLE_TIME)
135 idx += 1;
136
137 if (sample_type & PERF_SAMPLE_ADDR)
138 idx += 1;
139
140 return idx;
141}
142
143
144
145
146
147
148
149
150
151static int __perf_evsel__calc_is_pos(u64 sample_type)
152{
153 int idx = 1;
154
155 if (sample_type & PERF_SAMPLE_IDENTIFIER)
156 return 1;
157
158 if (!(sample_type & PERF_SAMPLE_ID))
159 return -1;
160
161 if (sample_type & PERF_SAMPLE_CPU)
162 idx += 1;
163
164 if (sample_type & PERF_SAMPLE_STREAM_ID)
165 idx += 1;
166
167 return idx;
168}
169
170void perf_evsel__calc_id_pos(struct perf_evsel *evsel)
171{
172 evsel->id_pos = __perf_evsel__calc_id_pos(evsel->attr.sample_type);
173 evsel->is_pos = __perf_evsel__calc_is_pos(evsel->attr.sample_type);
174}
175
176void __perf_evsel__set_sample_bit(struct perf_evsel *evsel,
177 enum perf_event_sample_format bit)
178{
179 if (!(evsel->attr.sample_type & bit)) {
180 evsel->attr.sample_type |= bit;
181 evsel->sample_size += sizeof(u64);
182 perf_evsel__calc_id_pos(evsel);
183 }
184}
185
186void __perf_evsel__reset_sample_bit(struct perf_evsel *evsel,
187 enum perf_event_sample_format bit)
188{
189 if (evsel->attr.sample_type & bit) {
190 evsel->attr.sample_type &= ~bit;
191 evsel->sample_size -= sizeof(u64);
192 perf_evsel__calc_id_pos(evsel);
193 }
194}
195
196void perf_evsel__set_sample_id(struct perf_evsel *evsel,
197 bool can_sample_identifier)
198{
199 if (can_sample_identifier) {
200 perf_evsel__reset_sample_bit(evsel, ID);
201 perf_evsel__set_sample_bit(evsel, IDENTIFIER);
202 } else {
203 perf_evsel__set_sample_bit(evsel, ID);
204 }
205 evsel->attr.read_format |= PERF_FORMAT_ID;
206}
207
208
209
210
211
212
213
214
215
216bool perf_evsel__is_function_event(struct perf_evsel *evsel)
217{
218#define FUNCTION_EVENT "ftrace:function"
219
220 return evsel->name &&
221 !strncmp(FUNCTION_EVENT, evsel->name, sizeof(FUNCTION_EVENT));
222
223#undef FUNCTION_EVENT
224}
225
226void perf_evsel__init(struct perf_evsel *evsel,
227 struct perf_event_attr *attr, int idx)
228{
229 evsel->idx = idx;
230 evsel->tracking = !idx;
231 evsel->attr = *attr;
232 evsel->leader = evsel;
233 evsel->unit = "";
234 evsel->scale = 1.0;
235 evsel->evlist = NULL;
236 evsel->bpf_fd = -1;
237 INIT_LIST_HEAD(&evsel->node);
238 INIT_LIST_HEAD(&evsel->config_terms);
239 perf_evsel__object.init(evsel);
240 evsel->sample_size = __perf_evsel__sample_size(attr->sample_type);
241 perf_evsel__calc_id_pos(evsel);
242 evsel->cmdline_group_boundary = false;
243 evsel->metric_expr = NULL;
244 evsel->metric_name = NULL;
245 evsel->metric_events = NULL;
246 evsel->collect_stat = false;
247 evsel->pmu_name = NULL;
248}
249
250struct perf_evsel *perf_evsel__new_idx(struct perf_event_attr *attr, int idx)
251{
252 struct perf_evsel *evsel = zalloc(perf_evsel__object.size);
253
254 if (evsel != NULL)
255 perf_evsel__init(evsel, attr, idx);
256
257 if (perf_evsel__is_bpf_output(evsel)) {
258 evsel->attr.sample_type |= (PERF_SAMPLE_RAW | PERF_SAMPLE_TIME |
259 PERF_SAMPLE_CPU | PERF_SAMPLE_PERIOD),
260 evsel->attr.sample_period = 1;
261 }
262
263 return evsel;
264}
265
266static bool perf_event_can_profile_kernel(void)
267{
268 return geteuid() == 0 || perf_event_paranoid() == -1;
269}
270
271struct perf_evsel *perf_evsel__new_cycles(bool precise)
272{
273 struct perf_event_attr attr = {
274 .type = PERF_TYPE_HARDWARE,
275 .config = PERF_COUNT_HW_CPU_CYCLES,
276 .exclude_kernel = !perf_event_can_profile_kernel(),
277 };
278 struct perf_evsel *evsel;
279
280 event_attr_init(&attr);
281
282 if (!precise)
283 goto new_event;
284
285
286
287
288
289
290 attr.sample_period = 1;
291
292 perf_event_attr__set_max_precise_ip(&attr);
293
294
295
296
297 attr.sample_period = 0;
298new_event:
299 evsel = perf_evsel__new(&attr);
300 if (evsel == NULL)
301 goto out;
302
303
304 if (asprintf(&evsel->name, "cycles%s%s%.*s",
305 (attr.precise_ip || attr.exclude_kernel) ? ":" : "",
306 attr.exclude_kernel ? "u" : "",
307 attr.precise_ip ? attr.precise_ip + 1 : 0, "ppp") < 0)
308 goto error_free;
309out:
310 return evsel;
311error_free:
312 perf_evsel__delete(evsel);
313 evsel = NULL;
314 goto out;
315}
316
317
318
319
320struct perf_evsel *perf_evsel__newtp_idx(const char *sys, const char *name, int idx)
321{
322 struct perf_evsel *evsel = zalloc(perf_evsel__object.size);
323 int err = -ENOMEM;
324
325 if (evsel == NULL) {
326 goto out_err;
327 } else {
328 struct perf_event_attr attr = {
329 .type = PERF_TYPE_TRACEPOINT,
330 .sample_type = (PERF_SAMPLE_RAW | PERF_SAMPLE_TIME |
331 PERF_SAMPLE_CPU | PERF_SAMPLE_PERIOD),
332 };
333
334 if (asprintf(&evsel->name, "%s:%s", sys, name) < 0)
335 goto out_free;
336
337 evsel->tp_format = trace_event__tp_format(sys, name);
338 if (IS_ERR(evsel->tp_format)) {
339 err = PTR_ERR(evsel->tp_format);
340 goto out_free;
341 }
342
343 event_attr_init(&attr);
344 attr.config = evsel->tp_format->id;
345 attr.sample_period = 1;
346 perf_evsel__init(evsel, &attr, idx);
347 }
348
349 return evsel;
350
351out_free:
352 zfree(&evsel->name);
353 free(evsel);
354out_err:
355 return ERR_PTR(err);
356}
357
358const char *perf_evsel__hw_names[PERF_COUNT_HW_MAX] = {
359 "cycles",
360 "instructions",
361 "cache-references",
362 "cache-misses",
363 "branches",
364 "branch-misses",
365 "bus-cycles",
366 "stalled-cycles-frontend",
367 "stalled-cycles-backend",
368 "ref-cycles",
369};
370
371static const char *__perf_evsel__hw_name(u64 config)
372{
373 if (config < PERF_COUNT_HW_MAX && perf_evsel__hw_names[config])
374 return perf_evsel__hw_names[config];
375
376 return "unknown-hardware";
377}
378
379static int perf_evsel__add_modifiers(struct perf_evsel *evsel, char *bf, size_t size)
380{
381 int colon = 0, r = 0;
382 struct perf_event_attr *attr = &evsel->attr;
383 bool exclude_guest_default = false;
384
385#define MOD_PRINT(context, mod) do { \
386 if (!attr->exclude_##context) { \
387 if (!colon) colon = ++r; \
388 r += scnprintf(bf + r, size - r, "%c", mod); \
389 } } while(0)
390
391 if (attr->exclude_kernel || attr->exclude_user || attr->exclude_hv) {
392 MOD_PRINT(kernel, 'k');
393 MOD_PRINT(user, 'u');
394 MOD_PRINT(hv, 'h');
395 exclude_guest_default = true;
396 }
397
398 if (attr->precise_ip) {
399 if (!colon)
400 colon = ++r;
401 r += scnprintf(bf + r, size - r, "%.*s", attr->precise_ip, "ppp");
402 exclude_guest_default = true;
403 }
404
405 if (attr->exclude_host || attr->exclude_guest == exclude_guest_default) {
406 MOD_PRINT(host, 'H');
407 MOD_PRINT(guest, 'G');
408 }
409#undef MOD_PRINT
410 if (colon)
411 bf[colon - 1] = ':';
412 return r;
413}
414
415static int perf_evsel__hw_name(struct perf_evsel *evsel, char *bf, size_t size)
416{
417 int r = scnprintf(bf, size, "%s", __perf_evsel__hw_name(evsel->attr.config));
418 return r + perf_evsel__add_modifiers(evsel, bf + r, size - r);
419}
420
421const char *perf_evsel__sw_names[PERF_COUNT_SW_MAX] = {
422 "cpu-clock",
423 "task-clock",
424 "page-faults",
425 "context-switches",
426 "cpu-migrations",
427 "minor-faults",
428 "major-faults",
429 "alignment-faults",
430 "emulation-faults",
431 "dummy",
432};
433
434static const char *__perf_evsel__sw_name(u64 config)
435{
436 if (config < PERF_COUNT_SW_MAX && perf_evsel__sw_names[config])
437 return perf_evsel__sw_names[config];
438 return "unknown-software";
439}
440
441static int perf_evsel__sw_name(struct perf_evsel *evsel, char *bf, size_t size)
442{
443 int r = scnprintf(bf, size, "%s", __perf_evsel__sw_name(evsel->attr.config));
444 return r + perf_evsel__add_modifiers(evsel, bf + r, size - r);
445}
446
447static int __perf_evsel__bp_name(char *bf, size_t size, u64 addr, u64 type)
448{
449 int r;
450
451 r = scnprintf(bf, size, "mem:0x%" PRIx64 ":", addr);
452
453 if (type & HW_BREAKPOINT_R)
454 r += scnprintf(bf + r, size - r, "r");
455
456 if (type & HW_BREAKPOINT_W)
457 r += scnprintf(bf + r, size - r, "w");
458
459 if (type & HW_BREAKPOINT_X)
460 r += scnprintf(bf + r, size - r, "x");
461
462 return r;
463}
464
465static int perf_evsel__bp_name(struct perf_evsel *evsel, char *bf, size_t size)
466{
467 struct perf_event_attr *attr = &evsel->attr;
468 int r = __perf_evsel__bp_name(bf, size, attr->bp_addr, attr->bp_type);
469 return r + perf_evsel__add_modifiers(evsel, bf + r, size - r);
470}
471
472const char *perf_evsel__hw_cache[PERF_COUNT_HW_CACHE_MAX]
473 [PERF_EVSEL__MAX_ALIASES] = {
474 { "L1-dcache", "l1-d", "l1d", "L1-data", },
475 { "L1-icache", "l1-i", "l1i", "L1-instruction", },
476 { "LLC", "L2", },
477 { "dTLB", "d-tlb", "Data-TLB", },
478 { "iTLB", "i-tlb", "Instruction-TLB", },
479 { "branch", "branches", "bpu", "btb", "bpc", },
480 { "node", },
481};
482
483const char *perf_evsel__hw_cache_op[PERF_COUNT_HW_CACHE_OP_MAX]
484 [PERF_EVSEL__MAX_ALIASES] = {
485 { "load", "loads", "read", },
486 { "store", "stores", "write", },
487 { "prefetch", "prefetches", "speculative-read", "speculative-load", },
488};
489
490const char *perf_evsel__hw_cache_result[PERF_COUNT_HW_CACHE_RESULT_MAX]
491 [PERF_EVSEL__MAX_ALIASES] = {
492 { "refs", "Reference", "ops", "access", },
493 { "misses", "miss", },
494};
495
496#define C(x) PERF_COUNT_HW_CACHE_##x
497#define CACHE_READ (1 << C(OP_READ))
498#define CACHE_WRITE (1 << C(OP_WRITE))
499#define CACHE_PREFETCH (1 << C(OP_PREFETCH))
500#define COP(x) (1 << x)
501
502
503
504
505
506
507static unsigned long perf_evsel__hw_cache_stat[C(MAX)] = {
508 [C(L1D)] = (CACHE_READ | CACHE_WRITE | CACHE_PREFETCH),
509 [C(L1I)] = (CACHE_READ | CACHE_PREFETCH),
510 [C(LL)] = (CACHE_READ | CACHE_WRITE | CACHE_PREFETCH),
511 [C(DTLB)] = (CACHE_READ | CACHE_WRITE | CACHE_PREFETCH),
512 [C(ITLB)] = (CACHE_READ),
513 [C(BPU)] = (CACHE_READ),
514 [C(NODE)] = (CACHE_READ | CACHE_WRITE | CACHE_PREFETCH),
515};
516
517bool perf_evsel__is_cache_op_valid(u8 type, u8 op)
518{
519 if (perf_evsel__hw_cache_stat[type] & COP(op))
520 return true;
521 else
522 return false;
523}
524
525int __perf_evsel__hw_cache_type_op_res_name(u8 type, u8 op, u8 result,
526 char *bf, size_t size)
527{
528 if (result) {
529 return scnprintf(bf, size, "%s-%s-%s", perf_evsel__hw_cache[type][0],
530 perf_evsel__hw_cache_op[op][0],
531 perf_evsel__hw_cache_result[result][0]);
532 }
533
534 return scnprintf(bf, size, "%s-%s", perf_evsel__hw_cache[type][0],
535 perf_evsel__hw_cache_op[op][1]);
536}
537
538static int __perf_evsel__hw_cache_name(u64 config, char *bf, size_t size)
539{
540 u8 op, result, type = (config >> 0) & 0xff;
541 const char *err = "unknown-ext-hardware-cache-type";
542
543 if (type >= PERF_COUNT_HW_CACHE_MAX)
544 goto out_err;
545
546 op = (config >> 8) & 0xff;
547 err = "unknown-ext-hardware-cache-op";
548 if (op >= PERF_COUNT_HW_CACHE_OP_MAX)
549 goto out_err;
550
551 result = (config >> 16) & 0xff;
552 err = "unknown-ext-hardware-cache-result";
553 if (result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
554 goto out_err;
555
556 err = "invalid-cache";
557 if (!perf_evsel__is_cache_op_valid(type, op))
558 goto out_err;
559
560 return __perf_evsel__hw_cache_type_op_res_name(type, op, result, bf, size);
561out_err:
562 return scnprintf(bf, size, "%s", err);
563}
564
565static int perf_evsel__hw_cache_name(struct perf_evsel *evsel, char *bf, size_t size)
566{
567 int ret = __perf_evsel__hw_cache_name(evsel->attr.config, bf, size);
568 return ret + perf_evsel__add_modifiers(evsel, bf + ret, size - ret);
569}
570
571static int perf_evsel__raw_name(struct perf_evsel *evsel, char *bf, size_t size)
572{
573 int ret = scnprintf(bf, size, "raw 0x%" PRIx64, evsel->attr.config);
574 return ret + perf_evsel__add_modifiers(evsel, bf + ret, size - ret);
575}
576
577const char *perf_evsel__name(struct perf_evsel *evsel)
578{
579 char bf[128];
580
581 if (evsel->name)
582 return evsel->name;
583
584 switch (evsel->attr.type) {
585 case PERF_TYPE_RAW:
586 perf_evsel__raw_name(evsel, bf, sizeof(bf));
587 break;
588
589 case PERF_TYPE_HARDWARE:
590 perf_evsel__hw_name(evsel, bf, sizeof(bf));
591 break;
592
593 case PERF_TYPE_HW_CACHE:
594 perf_evsel__hw_cache_name(evsel, bf, sizeof(bf));
595 break;
596
597 case PERF_TYPE_SOFTWARE:
598 perf_evsel__sw_name(evsel, bf, sizeof(bf));
599 break;
600
601 case PERF_TYPE_TRACEPOINT:
602 scnprintf(bf, sizeof(bf), "%s", "unknown tracepoint");
603 break;
604
605 case PERF_TYPE_BREAKPOINT:
606 perf_evsel__bp_name(evsel, bf, sizeof(bf));
607 break;
608
609 default:
610 scnprintf(bf, sizeof(bf), "unknown attr type: %d",
611 evsel->attr.type);
612 break;
613 }
614
615 evsel->name = strdup(bf);
616
617 return evsel->name ?: "unknown";
618}
619
620const char *perf_evsel__group_name(struct perf_evsel *evsel)
621{
622 return evsel->group_name ?: "anon group";
623}
624
625
626
627
628
629
630
631
632
633
634
635int perf_evsel__group_desc(struct perf_evsel *evsel, char *buf, size_t size)
636{
637 int ret = 0;
638 struct perf_evsel *pos;
639 const char *group_name = perf_evsel__group_name(evsel);
640
641 if (!evsel->forced_leader)
642 ret = scnprintf(buf, size, "%s { ", group_name);
643
644 ret += scnprintf(buf + ret, size - ret, "%s",
645 perf_evsel__name(evsel));
646
647 for_each_group_member(pos, evsel)
648 ret += scnprintf(buf + ret, size - ret, ", %s",
649 perf_evsel__name(pos));
650
651 if (!evsel->forced_leader)
652 ret += scnprintf(buf + ret, size - ret, " }");
653
654 return ret;
655}
656
657static void __perf_evsel__config_callchain(struct perf_evsel *evsel,
658 struct record_opts *opts,
659 struct callchain_param *param)
660{
661 bool function = perf_evsel__is_function_event(evsel);
662 struct perf_event_attr *attr = &evsel->attr;
663
664 perf_evsel__set_sample_bit(evsel, CALLCHAIN);
665
666 attr->sample_max_stack = param->max_stack;
667
668 if (param->record_mode == CALLCHAIN_LBR) {
669 if (!opts->branch_stack) {
670 if (attr->exclude_user) {
671 pr_warning("LBR callstack option is only available "
672 "to get user callchain information. "
673 "Falling back to framepointers.\n");
674 } else {
675 perf_evsel__set_sample_bit(evsel, BRANCH_STACK);
676 attr->branch_sample_type = PERF_SAMPLE_BRANCH_USER |
677 PERF_SAMPLE_BRANCH_CALL_STACK |
678 PERF_SAMPLE_BRANCH_NO_CYCLES |
679 PERF_SAMPLE_BRANCH_NO_FLAGS;
680 }
681 } else
682 pr_warning("Cannot use LBR callstack with branch stack. "
683 "Falling back to framepointers.\n");
684 }
685
686 if (param->record_mode == CALLCHAIN_DWARF) {
687 if (!function) {
688 perf_evsel__set_sample_bit(evsel, REGS_USER);
689 perf_evsel__set_sample_bit(evsel, STACK_USER);
690 attr->sample_regs_user |= PERF_REGS_MASK;
691 attr->sample_stack_user = param->dump_size;
692 attr->exclude_callchain_user = 1;
693 } else {
694 pr_info("Cannot use DWARF unwind for function trace event,"
695 " falling back to framepointers.\n");
696 }
697 }
698
699 if (function) {
700 pr_info("Disabling user space callchains for function trace event.\n");
701 attr->exclude_callchain_user = 1;
702 }
703}
704
705void perf_evsel__config_callchain(struct perf_evsel *evsel,
706 struct record_opts *opts,
707 struct callchain_param *param)
708{
709 if (param->enabled)
710 return __perf_evsel__config_callchain(evsel, opts, param);
711}
712
713static void
714perf_evsel__reset_callgraph(struct perf_evsel *evsel,
715 struct callchain_param *param)
716{
717 struct perf_event_attr *attr = &evsel->attr;
718
719 perf_evsel__reset_sample_bit(evsel, CALLCHAIN);
720 if (param->record_mode == CALLCHAIN_LBR) {
721 perf_evsel__reset_sample_bit(evsel, BRANCH_STACK);
722 attr->branch_sample_type &= ~(PERF_SAMPLE_BRANCH_USER |
723 PERF_SAMPLE_BRANCH_CALL_STACK);
724 }
725 if (param->record_mode == CALLCHAIN_DWARF) {
726 perf_evsel__reset_sample_bit(evsel, REGS_USER);
727 perf_evsel__reset_sample_bit(evsel, STACK_USER);
728 }
729}
730
731static void apply_config_terms(struct perf_evsel *evsel,
732 struct record_opts *opts, bool track)
733{
734 struct perf_evsel_config_term *term;
735 struct list_head *config_terms = &evsel->config_terms;
736 struct perf_event_attr *attr = &evsel->attr;
737
738 struct callchain_param param = {
739 .record_mode = callchain_param.record_mode,
740 };
741 u32 dump_size = 0;
742 int max_stack = 0;
743 const char *callgraph_buf = NULL;
744
745 list_for_each_entry(term, config_terms, list) {
746 switch (term->type) {
747 case PERF_EVSEL__CONFIG_TERM_PERIOD:
748 if (!(term->weak && opts->user_interval != ULLONG_MAX)) {
749 attr->sample_period = term->val.period;
750 attr->freq = 0;
751 perf_evsel__reset_sample_bit(evsel, PERIOD);
752 }
753 break;
754 case PERF_EVSEL__CONFIG_TERM_FREQ:
755 if (!(term->weak && opts->user_freq != UINT_MAX)) {
756 attr->sample_freq = term->val.freq;
757 attr->freq = 1;
758 perf_evsel__set_sample_bit(evsel, PERIOD);
759 }
760 break;
761 case PERF_EVSEL__CONFIG_TERM_TIME:
762 if (term->val.time)
763 perf_evsel__set_sample_bit(evsel, TIME);
764 else
765 perf_evsel__reset_sample_bit(evsel, TIME);
766 break;
767 case PERF_EVSEL__CONFIG_TERM_CALLGRAPH:
768 callgraph_buf = term->val.callgraph;
769 break;
770 case PERF_EVSEL__CONFIG_TERM_BRANCH:
771 if (term->val.branch && strcmp(term->val.branch, "no")) {
772 perf_evsel__set_sample_bit(evsel, BRANCH_STACK);
773 parse_branch_str(term->val.branch,
774 &attr->branch_sample_type);
775 } else
776 perf_evsel__reset_sample_bit(evsel, BRANCH_STACK);
777 break;
778 case PERF_EVSEL__CONFIG_TERM_STACK_USER:
779 dump_size = term->val.stack_user;
780 break;
781 case PERF_EVSEL__CONFIG_TERM_MAX_STACK:
782 max_stack = term->val.max_stack;
783 break;
784 case PERF_EVSEL__CONFIG_TERM_INHERIT:
785
786
787
788
789
790
791 attr->inherit = term->val.inherit ? 1 : 0;
792 break;
793 case PERF_EVSEL__CONFIG_TERM_OVERWRITE:
794 attr->write_backward = term->val.overwrite ? 1 : 0;
795 break;
796 case PERF_EVSEL__CONFIG_TERM_DRV_CFG:
797 break;
798 default:
799 break;
800 }
801 }
802
803
804 if ((callgraph_buf != NULL) || (dump_size > 0) || max_stack) {
805 bool sample_address = false;
806
807 if (max_stack) {
808 param.max_stack = max_stack;
809 if (callgraph_buf == NULL)
810 callgraph_buf = "fp";
811 }
812
813
814 if (callgraph_buf != NULL) {
815 if (!strcmp(callgraph_buf, "no")) {
816 param.enabled = false;
817 param.record_mode = CALLCHAIN_NONE;
818 } else {
819 param.enabled = true;
820 if (parse_callchain_record(callgraph_buf, ¶m)) {
821 pr_err("per-event callgraph setting for %s failed. "
822 "Apply callgraph global setting for it\n",
823 evsel->name);
824 return;
825 }
826 if (param.record_mode == CALLCHAIN_DWARF)
827 sample_address = true;
828 }
829 }
830 if (dump_size > 0) {
831 dump_size = round_up(dump_size, sizeof(u64));
832 param.dump_size = dump_size;
833 }
834
835
836 if (callchain_param.enabled)
837 perf_evsel__reset_callgraph(evsel, &callchain_param);
838
839
840 if (param.enabled) {
841 if (sample_address) {
842 perf_evsel__set_sample_bit(evsel, ADDR);
843 perf_evsel__set_sample_bit(evsel, DATA_SRC);
844 evsel->attr.mmap_data = track;
845 }
846 perf_evsel__config_callchain(evsel, opts, ¶m);
847 }
848 }
849}
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879void perf_evsel__config(struct perf_evsel *evsel, struct record_opts *opts,
880 struct callchain_param *callchain)
881{
882 struct perf_evsel *leader = evsel->leader;
883 struct perf_event_attr *attr = &evsel->attr;
884 int track = evsel->tracking;
885 bool per_cpu = opts->target.default_per_cpu && !opts->target.per_thread;
886
887 attr->sample_id_all = perf_missing_features.sample_id_all ? 0 : 1;
888 attr->inherit = !opts->no_inherit;
889 attr->write_backward = opts->overwrite ? 1 : 0;
890
891 perf_evsel__set_sample_bit(evsel, IP);
892 perf_evsel__set_sample_bit(evsel, TID);
893
894 if (evsel->sample_read) {
895 perf_evsel__set_sample_bit(evsel, READ);
896
897
898
899
900
901 perf_evsel__set_sample_id(evsel, false);
902
903
904
905
906
907 if (leader->nr_members > 1) {
908 attr->read_format |= PERF_FORMAT_GROUP;
909 attr->inherit = 0;
910 }
911 }
912
913
914
915
916
917 if (!attr->sample_period || (opts->user_freq != UINT_MAX ||
918 opts->user_interval != ULLONG_MAX)) {
919 if (opts->freq) {
920 perf_evsel__set_sample_bit(evsel, PERIOD);
921 attr->freq = 1;
922 attr->sample_freq = opts->freq;
923 } else {
924 attr->sample_period = opts->default_interval;
925 }
926 }
927
928
929
930
931
932 if ((leader != evsel) && leader->sample_read) {
933 attr->freq = 0;
934 attr->sample_freq = 0;
935 attr->sample_period = 0;
936 attr->write_backward = 0;
937 attr->sample_id_all = 0;
938 }
939
940 if (opts->no_samples)
941 attr->sample_freq = 0;
942
943 if (opts->inherit_stat) {
944 evsel->attr.read_format |=
945 PERF_FORMAT_TOTAL_TIME_ENABLED |
946 PERF_FORMAT_TOTAL_TIME_RUNNING |
947 PERF_FORMAT_ID;
948 attr->inherit_stat = 1;
949 }
950
951 if (opts->sample_address) {
952 perf_evsel__set_sample_bit(evsel, ADDR);
953 attr->mmap_data = track;
954 }
955
956
957
958
959
960
961 if (perf_evsel__is_function_event(evsel))
962 evsel->attr.exclude_callchain_user = 1;
963
964 if (callchain && callchain->enabled && !evsel->no_aux_samples)
965 perf_evsel__config_callchain(evsel, opts, callchain);
966
967 if (opts->sample_intr_regs) {
968 attr->sample_regs_intr = opts->sample_intr_regs;
969 perf_evsel__set_sample_bit(evsel, REGS_INTR);
970 }
971
972 if (opts->sample_user_regs) {
973 attr->sample_regs_user |= opts->sample_user_regs;
974 perf_evsel__set_sample_bit(evsel, REGS_USER);
975 }
976
977 if (target__has_cpu(&opts->target) || opts->sample_cpu)
978 perf_evsel__set_sample_bit(evsel, CPU);
979
980
981
982
983 if (opts->sample_time &&
984 (!perf_missing_features.sample_id_all &&
985 (!opts->no_inherit || target__has_cpu(&opts->target) || per_cpu ||
986 opts->sample_time_set)))
987 perf_evsel__set_sample_bit(evsel, TIME);
988
989 if (opts->raw_samples && !evsel->no_aux_samples) {
990 perf_evsel__set_sample_bit(evsel, TIME);
991 perf_evsel__set_sample_bit(evsel, RAW);
992 perf_evsel__set_sample_bit(evsel, CPU);
993 }
994
995 if (opts->sample_address)
996 perf_evsel__set_sample_bit(evsel, DATA_SRC);
997
998 if (opts->sample_phys_addr)
999 perf_evsel__set_sample_bit(evsel, PHYS_ADDR);
1000
1001 if (opts->no_buffering) {
1002 attr->watermark = 0;
1003 attr->wakeup_events = 1;
1004 }
1005 if (opts->branch_stack && !evsel->no_aux_samples) {
1006 perf_evsel__set_sample_bit(evsel, BRANCH_STACK);
1007 attr->branch_sample_type = opts->branch_stack;
1008 }
1009
1010 if (opts->sample_weight)
1011 perf_evsel__set_sample_bit(evsel, WEIGHT);
1012
1013 attr->task = track;
1014 attr->mmap = track;
1015 attr->mmap2 = track && !perf_missing_features.mmap2;
1016 attr->comm = track;
1017
1018 if (opts->record_namespaces)
1019 attr->namespaces = track;
1020
1021 if (opts->record_switch_events)
1022 attr->context_switch = track;
1023
1024 if (opts->sample_transaction)
1025 perf_evsel__set_sample_bit(evsel, TRANSACTION);
1026
1027 if (opts->running_time) {
1028 evsel->attr.read_format |=
1029 PERF_FORMAT_TOTAL_TIME_ENABLED |
1030 PERF_FORMAT_TOTAL_TIME_RUNNING;
1031 }
1032
1033
1034
1035
1036
1037
1038
1039 if (perf_evsel__is_group_leader(evsel))
1040 attr->disabled = 1;
1041
1042
1043
1044
1045
1046 if (target__none(&opts->target) && perf_evsel__is_group_leader(evsel) &&
1047 !opts->initial_delay)
1048 attr->enable_on_exec = 1;
1049
1050 if (evsel->immediate) {
1051 attr->disabled = 0;
1052 attr->enable_on_exec = 0;
1053 }
1054
1055 clockid = opts->clockid;
1056 if (opts->use_clockid) {
1057 attr->use_clockid = 1;
1058 attr->clockid = opts->clockid;
1059 }
1060
1061 if (evsel->precise_max)
1062 perf_event_attr__set_max_precise_ip(attr);
1063
1064 if (opts->all_user) {
1065 attr->exclude_kernel = 1;
1066 attr->exclude_user = 0;
1067 }
1068
1069 if (opts->all_kernel) {
1070 attr->exclude_kernel = 0;
1071 attr->exclude_user = 1;
1072 }
1073
1074
1075
1076
1077
1078 apply_config_terms(evsel, opts, track);
1079
1080 evsel->ignore_missing_thread = opts->ignore_missing_thread;
1081
1082
1083 if (opts->period_set) {
1084 if (opts->period)
1085 perf_evsel__set_sample_bit(evsel, PERIOD);
1086 else
1087 perf_evsel__reset_sample_bit(evsel, PERIOD);
1088 }
1089}
1090
1091static int perf_evsel__alloc_fd(struct perf_evsel *evsel, int ncpus, int nthreads)
1092{
1093 if (evsel->system_wide)
1094 nthreads = 1;
1095
1096 evsel->fd = xyarray__new(ncpus, nthreads, sizeof(int));
1097
1098 if (evsel->fd) {
1099 int cpu, thread;
1100 for (cpu = 0; cpu < ncpus; cpu++) {
1101 for (thread = 0; thread < nthreads; thread++) {
1102 FD(evsel, cpu, thread) = -1;
1103 }
1104 }
1105 }
1106
1107 return evsel->fd != NULL ? 0 : -ENOMEM;
1108}
1109
1110static int perf_evsel__run_ioctl(struct perf_evsel *evsel,
1111 int ioc, void *arg)
1112{
1113 int cpu, thread;
1114
1115 for (cpu = 0; cpu < xyarray__max_x(evsel->fd); cpu++) {
1116 for (thread = 0; thread < xyarray__max_y(evsel->fd); thread++) {
1117 int fd = FD(evsel, cpu, thread),
1118 err = ioctl(fd, ioc, arg);
1119
1120 if (err)
1121 return err;
1122 }
1123 }
1124
1125 return 0;
1126}
1127
1128int perf_evsel__apply_filter(struct perf_evsel *evsel, const char *filter)
1129{
1130 return perf_evsel__run_ioctl(evsel,
1131 PERF_EVENT_IOC_SET_FILTER,
1132 (void *)filter);
1133}
1134
1135int perf_evsel__set_filter(struct perf_evsel *evsel, const char *filter)
1136{
1137 char *new_filter = strdup(filter);
1138
1139 if (new_filter != NULL) {
1140 free(evsel->filter);
1141 evsel->filter = new_filter;
1142 return 0;
1143 }
1144
1145 return -1;
1146}
1147
1148static int perf_evsel__append_filter(struct perf_evsel *evsel,
1149 const char *fmt, const char *filter)
1150{
1151 char *new_filter;
1152
1153 if (evsel->filter == NULL)
1154 return perf_evsel__set_filter(evsel, filter);
1155
1156 if (asprintf(&new_filter, fmt, evsel->filter, filter) > 0) {
1157 free(evsel->filter);
1158 evsel->filter = new_filter;
1159 return 0;
1160 }
1161
1162 return -1;
1163}
1164
1165int perf_evsel__append_tp_filter(struct perf_evsel *evsel, const char *filter)
1166{
1167 return perf_evsel__append_filter(evsel, "(%s) && (%s)", filter);
1168}
1169
1170int perf_evsel__append_addr_filter(struct perf_evsel *evsel, const char *filter)
1171{
1172 return perf_evsel__append_filter(evsel, "%s,%s", filter);
1173}
1174
1175int perf_evsel__enable(struct perf_evsel *evsel)
1176{
1177 return perf_evsel__run_ioctl(evsel,
1178 PERF_EVENT_IOC_ENABLE,
1179 0);
1180}
1181
1182int perf_evsel__disable(struct perf_evsel *evsel)
1183{
1184 return perf_evsel__run_ioctl(evsel,
1185 PERF_EVENT_IOC_DISABLE,
1186 0);
1187}
1188
1189int perf_evsel__alloc_id(struct perf_evsel *evsel, int ncpus, int nthreads)
1190{
1191 if (ncpus == 0 || nthreads == 0)
1192 return 0;
1193
1194 if (evsel->system_wide)
1195 nthreads = 1;
1196
1197 evsel->sample_id = xyarray__new(ncpus, nthreads, sizeof(struct perf_sample_id));
1198 if (evsel->sample_id == NULL)
1199 return -ENOMEM;
1200
1201 evsel->id = zalloc(ncpus * nthreads * sizeof(u64));
1202 if (evsel->id == NULL) {
1203 xyarray__delete(evsel->sample_id);
1204 evsel->sample_id = NULL;
1205 return -ENOMEM;
1206 }
1207
1208 return 0;
1209}
1210
1211static void perf_evsel__free_fd(struct perf_evsel *evsel)
1212{
1213 xyarray__delete(evsel->fd);
1214 evsel->fd = NULL;
1215}
1216
1217static void perf_evsel__free_id(struct perf_evsel *evsel)
1218{
1219 xyarray__delete(evsel->sample_id);
1220 evsel->sample_id = NULL;
1221 zfree(&evsel->id);
1222}
1223
1224static void perf_evsel__free_config_terms(struct perf_evsel *evsel)
1225{
1226 struct perf_evsel_config_term *term, *h;
1227
1228 list_for_each_entry_safe(term, h, &evsel->config_terms, list) {
1229 list_del(&term->list);
1230 free(term);
1231 }
1232}
1233
1234void perf_evsel__close_fd(struct perf_evsel *evsel)
1235{
1236 int cpu, thread;
1237
1238 for (cpu = 0; cpu < xyarray__max_x(evsel->fd); cpu++)
1239 for (thread = 0; thread < xyarray__max_y(evsel->fd); ++thread) {
1240 close(FD(evsel, cpu, thread));
1241 FD(evsel, cpu, thread) = -1;
1242 }
1243}
1244
1245void perf_evsel__exit(struct perf_evsel *evsel)
1246{
1247 assert(list_empty(&evsel->node));
1248 assert(evsel->evlist == NULL);
1249 perf_evsel__free_fd(evsel);
1250 perf_evsel__free_id(evsel);
1251 perf_evsel__free_config_terms(evsel);
1252 cgroup__put(evsel->cgrp);
1253 cpu_map__put(evsel->cpus);
1254 cpu_map__put(evsel->own_cpus);
1255 thread_map__put(evsel->threads);
1256 zfree(&evsel->group_name);
1257 zfree(&evsel->name);
1258 perf_evsel__object.fini(evsel);
1259}
1260
1261void perf_evsel__delete(struct perf_evsel *evsel)
1262{
1263 perf_evsel__exit(evsel);
1264 free(evsel);
1265}
1266
1267void perf_evsel__compute_deltas(struct perf_evsel *evsel, int cpu, int thread,
1268 struct perf_counts_values *count)
1269{
1270 struct perf_counts_values tmp;
1271
1272 if (!evsel->prev_raw_counts)
1273 return;
1274
1275 if (cpu == -1) {
1276 tmp = evsel->prev_raw_counts->aggr;
1277 evsel->prev_raw_counts->aggr = *count;
1278 } else {
1279 tmp = *perf_counts(evsel->prev_raw_counts, cpu, thread);
1280 *perf_counts(evsel->prev_raw_counts, cpu, thread) = *count;
1281 }
1282
1283 count->val = count->val - tmp.val;
1284 count->ena = count->ena - tmp.ena;
1285 count->run = count->run - tmp.run;
1286}
1287
1288void perf_counts_values__scale(struct perf_counts_values *count,
1289 bool scale, s8 *pscaled)
1290{
1291 s8 scaled = 0;
1292
1293 if (scale) {
1294 if (count->run == 0) {
1295 scaled = -1;
1296 count->val = 0;
1297 } else if (count->run < count->ena) {
1298 scaled = 1;
1299 count->val = (u64)((double) count->val * count->ena / count->run + 0.5);
1300 }
1301 } else
1302 count->ena = count->run = 0;
1303
1304 if (pscaled)
1305 *pscaled = scaled;
1306}
1307
1308static int perf_evsel__read_size(struct perf_evsel *evsel)
1309{
1310 u64 read_format = evsel->attr.read_format;
1311 int entry = sizeof(u64);
1312 int size = 0;
1313 int nr = 1;
1314
1315 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
1316 size += sizeof(u64);
1317
1318 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
1319 size += sizeof(u64);
1320
1321 if (read_format & PERF_FORMAT_ID)
1322 entry += sizeof(u64);
1323
1324 if (read_format & PERF_FORMAT_GROUP) {
1325 nr = evsel->nr_members;
1326 size += sizeof(u64);
1327 }
1328
1329 size += entry * nr;
1330 return size;
1331}
1332
1333int perf_evsel__read(struct perf_evsel *evsel, int cpu, int thread,
1334 struct perf_counts_values *count)
1335{
1336 size_t size = perf_evsel__read_size(evsel);
1337
1338 memset(count, 0, sizeof(*count));
1339
1340 if (FD(evsel, cpu, thread) < 0)
1341 return -EINVAL;
1342
1343 if (readn(FD(evsel, cpu, thread), count->values, size) <= 0)
1344 return -errno;
1345
1346 return 0;
1347}
1348
1349static int
1350perf_evsel__read_one(struct perf_evsel *evsel, int cpu, int thread)
1351{
1352 struct perf_counts_values *count = perf_counts(evsel->counts, cpu, thread);
1353
1354 return perf_evsel__read(evsel, cpu, thread, count);
1355}
1356
1357static void
1358perf_evsel__set_count(struct perf_evsel *counter, int cpu, int thread,
1359 u64 val, u64 ena, u64 run)
1360{
1361 struct perf_counts_values *count;
1362
1363 count = perf_counts(counter->counts, cpu, thread);
1364
1365 count->val = val;
1366 count->ena = ena;
1367 count->run = run;
1368 count->loaded = true;
1369}
1370
1371static int
1372perf_evsel__process_group_data(struct perf_evsel *leader,
1373 int cpu, int thread, u64 *data)
1374{
1375 u64 read_format = leader->attr.read_format;
1376 struct sample_read_value *v;
1377 u64 nr, ena = 0, run = 0, i;
1378
1379 nr = *data++;
1380
1381 if (nr != (u64) leader->nr_members)
1382 return -EINVAL;
1383
1384 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
1385 ena = *data++;
1386
1387 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
1388 run = *data++;
1389
1390 v = (struct sample_read_value *) data;
1391
1392 perf_evsel__set_count(leader, cpu, thread,
1393 v[0].value, ena, run);
1394
1395 for (i = 1; i < nr; i++) {
1396 struct perf_evsel *counter;
1397
1398 counter = perf_evlist__id2evsel(leader->evlist, v[i].id);
1399 if (!counter)
1400 return -EINVAL;
1401
1402 perf_evsel__set_count(counter, cpu, thread,
1403 v[i].value, ena, run);
1404 }
1405
1406 return 0;
1407}
1408
1409static int
1410perf_evsel__read_group(struct perf_evsel *leader, int cpu, int thread)
1411{
1412 struct perf_stat_evsel *ps = leader->stats;
1413 u64 read_format = leader->attr.read_format;
1414 int size = perf_evsel__read_size(leader);
1415 u64 *data = ps->group_data;
1416
1417 if (!(read_format & PERF_FORMAT_ID))
1418 return -EINVAL;
1419
1420 if (!perf_evsel__is_group_leader(leader))
1421 return -EINVAL;
1422
1423 if (!data) {
1424 data = zalloc(size);
1425 if (!data)
1426 return -ENOMEM;
1427
1428 ps->group_data = data;
1429 }
1430
1431 if (FD(leader, cpu, thread) < 0)
1432 return -EINVAL;
1433
1434 if (readn(FD(leader, cpu, thread), data, size) <= 0)
1435 return -errno;
1436
1437 return perf_evsel__process_group_data(leader, cpu, thread, data);
1438}
1439
1440int perf_evsel__read_counter(struct perf_evsel *evsel, int cpu, int thread)
1441{
1442 u64 read_format = evsel->attr.read_format;
1443
1444 if (read_format & PERF_FORMAT_GROUP)
1445 return perf_evsel__read_group(evsel, cpu, thread);
1446 else
1447 return perf_evsel__read_one(evsel, cpu, thread);
1448}
1449
1450int __perf_evsel__read_on_cpu(struct perf_evsel *evsel,
1451 int cpu, int thread, bool scale)
1452{
1453 struct perf_counts_values count;
1454 size_t nv = scale ? 3 : 1;
1455
1456 if (FD(evsel, cpu, thread) < 0)
1457 return -EINVAL;
1458
1459 if (evsel->counts == NULL && perf_evsel__alloc_counts(evsel, cpu + 1, thread + 1) < 0)
1460 return -ENOMEM;
1461
1462 if (readn(FD(evsel, cpu, thread), &count, nv * sizeof(u64)) <= 0)
1463 return -errno;
1464
1465 perf_evsel__compute_deltas(evsel, cpu, thread, &count);
1466 perf_counts_values__scale(&count, scale, NULL);
1467 *perf_counts(evsel->counts, cpu, thread) = count;
1468 return 0;
1469}
1470
1471static int get_group_fd(struct perf_evsel *evsel, int cpu, int thread)
1472{
1473 struct perf_evsel *leader = evsel->leader;
1474 int fd;
1475
1476 if (perf_evsel__is_group_leader(evsel))
1477 return -1;
1478
1479
1480
1481
1482
1483 BUG_ON(!leader->fd);
1484
1485 fd = FD(leader, cpu, thread);
1486 BUG_ON(fd == -1);
1487
1488 return fd;
1489}
1490
1491struct bit_names {
1492 int bit;
1493 const char *name;
1494};
1495
1496static void __p_bits(char *buf, size_t size, u64 value, struct bit_names *bits)
1497{
1498 bool first_bit = true;
1499 int i = 0;
1500
1501 do {
1502 if (value & bits[i].bit) {
1503 buf += scnprintf(buf, size, "%s%s", first_bit ? "" : "|", bits[i].name);
1504 first_bit = false;
1505 }
1506 } while (bits[++i].name != NULL);
1507}
1508
1509static void __p_sample_type(char *buf, size_t size, u64 value)
1510{
1511#define bit_name(n) { PERF_SAMPLE_##n, #n }
1512 struct bit_names bits[] = {
1513 bit_name(IP), bit_name(TID), bit_name(TIME), bit_name(ADDR),
1514 bit_name(READ), bit_name(CALLCHAIN), bit_name(ID), bit_name(CPU),
1515 bit_name(PERIOD), bit_name(STREAM_ID), bit_name(RAW),
1516 bit_name(BRANCH_STACK), bit_name(REGS_USER), bit_name(STACK_USER),
1517 bit_name(IDENTIFIER), bit_name(REGS_INTR), bit_name(DATA_SRC),
1518 bit_name(WEIGHT), bit_name(PHYS_ADDR),
1519 { .name = NULL, }
1520 };
1521#undef bit_name
1522 __p_bits(buf, size, value, bits);
1523}
1524
1525static void __p_branch_sample_type(char *buf, size_t size, u64 value)
1526{
1527#define bit_name(n) { PERF_SAMPLE_BRANCH_##n, #n }
1528 struct bit_names bits[] = {
1529 bit_name(USER), bit_name(KERNEL), bit_name(HV), bit_name(ANY),
1530 bit_name(ANY_CALL), bit_name(ANY_RETURN), bit_name(IND_CALL),
1531 bit_name(ABORT_TX), bit_name(IN_TX), bit_name(NO_TX),
1532 bit_name(COND), bit_name(CALL_STACK), bit_name(IND_JUMP),
1533 bit_name(CALL), bit_name(NO_FLAGS), bit_name(NO_CYCLES),
1534 { .name = NULL, }
1535 };
1536#undef bit_name
1537 __p_bits(buf, size, value, bits);
1538}
1539
1540static void __p_read_format(char *buf, size_t size, u64 value)
1541{
1542#define bit_name(n) { PERF_FORMAT_##n, #n }
1543 struct bit_names bits[] = {
1544 bit_name(TOTAL_TIME_ENABLED), bit_name(TOTAL_TIME_RUNNING),
1545 bit_name(ID), bit_name(GROUP),
1546 { .name = NULL, }
1547 };
1548#undef bit_name
1549 __p_bits(buf, size, value, bits);
1550}
1551
1552#define BUF_SIZE 1024
1553
1554#define p_hex(val) snprintf(buf, BUF_SIZE, "%#"PRIx64, (uint64_t)(val))
1555#define p_unsigned(val) snprintf(buf, BUF_SIZE, "%"PRIu64, (uint64_t)(val))
1556#define p_signed(val) snprintf(buf, BUF_SIZE, "%"PRId64, (int64_t)(val))
1557#define p_sample_type(val) __p_sample_type(buf, BUF_SIZE, val)
1558#define p_branch_sample_type(val) __p_branch_sample_type(buf, BUF_SIZE, val)
1559#define p_read_format(val) __p_read_format(buf, BUF_SIZE, val)
1560
1561#define PRINT_ATTRn(_n, _f, _p) \
1562do { \
1563 if (attr->_f) { \
1564 _p(attr->_f); \
1565 ret += attr__fprintf(fp, _n, buf, priv);\
1566 } \
1567} while (0)
1568
1569#define PRINT_ATTRf(_f, _p) PRINT_ATTRn(#_f, _f, _p)
1570
1571int perf_event_attr__fprintf(FILE *fp, struct perf_event_attr *attr,
1572 attr__fprintf_f attr__fprintf, void *priv)
1573{
1574 char buf[BUF_SIZE];
1575 int ret = 0;
1576
1577 PRINT_ATTRf(type, p_unsigned);
1578 PRINT_ATTRf(size, p_unsigned);
1579 PRINT_ATTRf(config, p_hex);
1580 PRINT_ATTRn("{ sample_period, sample_freq }", sample_period, p_unsigned);
1581 PRINT_ATTRf(sample_type, p_sample_type);
1582 PRINT_ATTRf(read_format, p_read_format);
1583
1584 PRINT_ATTRf(disabled, p_unsigned);
1585 PRINT_ATTRf(inherit, p_unsigned);
1586 PRINT_ATTRf(pinned, p_unsigned);
1587 PRINT_ATTRf(exclusive, p_unsigned);
1588 PRINT_ATTRf(exclude_user, p_unsigned);
1589 PRINT_ATTRf(exclude_kernel, p_unsigned);
1590 PRINT_ATTRf(exclude_hv, p_unsigned);
1591 PRINT_ATTRf(exclude_idle, p_unsigned);
1592 PRINT_ATTRf(mmap, p_unsigned);
1593 PRINT_ATTRf(comm, p_unsigned);
1594 PRINT_ATTRf(freq, p_unsigned);
1595 PRINT_ATTRf(inherit_stat, p_unsigned);
1596 PRINT_ATTRf(enable_on_exec, p_unsigned);
1597 PRINT_ATTRf(task, p_unsigned);
1598 PRINT_ATTRf(watermark, p_unsigned);
1599 PRINT_ATTRf(precise_ip, p_unsigned);
1600 PRINT_ATTRf(mmap_data, p_unsigned);
1601 PRINT_ATTRf(sample_id_all, p_unsigned);
1602 PRINT_ATTRf(exclude_host, p_unsigned);
1603 PRINT_ATTRf(exclude_guest, p_unsigned);
1604 PRINT_ATTRf(exclude_callchain_kernel, p_unsigned);
1605 PRINT_ATTRf(exclude_callchain_user, p_unsigned);
1606 PRINT_ATTRf(mmap2, p_unsigned);
1607 PRINT_ATTRf(comm_exec, p_unsigned);
1608 PRINT_ATTRf(use_clockid, p_unsigned);
1609 PRINT_ATTRf(context_switch, p_unsigned);
1610 PRINT_ATTRf(write_backward, p_unsigned);
1611 PRINT_ATTRf(namespaces, p_unsigned);
1612
1613 PRINT_ATTRn("{ wakeup_events, wakeup_watermark }", wakeup_events, p_unsigned);
1614 PRINT_ATTRf(bp_type, p_unsigned);
1615 PRINT_ATTRn("{ bp_addr, config1 }", bp_addr, p_hex);
1616 PRINT_ATTRn("{ bp_len, config2 }", bp_len, p_hex);
1617 PRINT_ATTRf(branch_sample_type, p_branch_sample_type);
1618 PRINT_ATTRf(sample_regs_user, p_hex);
1619 PRINT_ATTRf(sample_stack_user, p_unsigned);
1620 PRINT_ATTRf(clockid, p_signed);
1621 PRINT_ATTRf(sample_regs_intr, p_hex);
1622 PRINT_ATTRf(aux_watermark, p_unsigned);
1623 PRINT_ATTRf(sample_max_stack, p_unsigned);
1624
1625 return ret;
1626}
1627
1628static int __open_attr__fprintf(FILE *fp, const char *name, const char *val,
1629 void *priv __maybe_unused)
1630{
1631 return fprintf(fp, " %-32s %s\n", name, val);
1632}
1633
1634static void perf_evsel__remove_fd(struct perf_evsel *pos,
1635 int nr_cpus, int nr_threads,
1636 int thread_idx)
1637{
1638 for (int cpu = 0; cpu < nr_cpus; cpu++)
1639 for (int thread = thread_idx; thread < nr_threads - 1; thread++)
1640 FD(pos, cpu, thread) = FD(pos, cpu, thread + 1);
1641}
1642
1643static int update_fds(struct perf_evsel *evsel,
1644 int nr_cpus, int cpu_idx,
1645 int nr_threads, int thread_idx)
1646{
1647 struct perf_evsel *pos;
1648
1649 if (cpu_idx >= nr_cpus || thread_idx >= nr_threads)
1650 return -EINVAL;
1651
1652 evlist__for_each_entry(evsel->evlist, pos) {
1653 nr_cpus = pos != evsel ? nr_cpus : cpu_idx;
1654
1655 perf_evsel__remove_fd(pos, nr_cpus, nr_threads, thread_idx);
1656
1657
1658
1659
1660
1661 if (pos == evsel)
1662 break;
1663 }
1664 return 0;
1665}
1666
1667static bool ignore_missing_thread(struct perf_evsel *evsel,
1668 int nr_cpus, int cpu,
1669 struct thread_map *threads,
1670 int thread, int err)
1671{
1672 pid_t ignore_pid = thread_map__pid(threads, thread);
1673
1674 if (!evsel->ignore_missing_thread)
1675 return false;
1676
1677
1678 if (evsel->system_wide)
1679 return false;
1680
1681
1682 if (err != -ESRCH)
1683 return false;
1684
1685
1686 if (threads->nr == 1)
1687 return false;
1688
1689
1690
1691
1692
1693 if (update_fds(evsel, nr_cpus, cpu, threads->nr, thread))
1694 return false;
1695
1696 if (thread_map__remove(threads, thread))
1697 return false;
1698
1699 pr_warning("WARNING: Ignored open failure for pid %d\n",
1700 ignore_pid);
1701 return true;
1702}
1703
1704int perf_evsel__open(struct perf_evsel *evsel, struct cpu_map *cpus,
1705 struct thread_map *threads)
1706{
1707 int cpu, thread, nthreads;
1708 unsigned long flags = PERF_FLAG_FD_CLOEXEC;
1709 int pid = -1, err;
1710 enum { NO_CHANGE, SET_TO_MAX, INCREASED_MAX } set_rlimit = NO_CHANGE;
1711
1712 if (perf_missing_features.write_backward && evsel->attr.write_backward)
1713 return -EINVAL;
1714
1715 if (cpus == NULL) {
1716 static struct cpu_map *empty_cpu_map;
1717
1718 if (empty_cpu_map == NULL) {
1719 empty_cpu_map = cpu_map__dummy_new();
1720 if (empty_cpu_map == NULL)
1721 return -ENOMEM;
1722 }
1723
1724 cpus = empty_cpu_map;
1725 }
1726
1727 if (threads == NULL) {
1728 static struct thread_map *empty_thread_map;
1729
1730 if (empty_thread_map == NULL) {
1731 empty_thread_map = thread_map__new_by_tid(-1);
1732 if (empty_thread_map == NULL)
1733 return -ENOMEM;
1734 }
1735
1736 threads = empty_thread_map;
1737 }
1738
1739 if (evsel->system_wide)
1740 nthreads = 1;
1741 else
1742 nthreads = threads->nr;
1743
1744 if (evsel->fd == NULL &&
1745 perf_evsel__alloc_fd(evsel, cpus->nr, nthreads) < 0)
1746 return -ENOMEM;
1747
1748 if (evsel->cgrp) {
1749 flags |= PERF_FLAG_PID_CGROUP;
1750 pid = evsel->cgrp->fd;
1751 }
1752
1753fallback_missing_features:
1754 if (perf_missing_features.clockid_wrong)
1755 evsel->attr.clockid = CLOCK_MONOTONIC;
1756 if (perf_missing_features.clockid) {
1757 evsel->attr.use_clockid = 0;
1758 evsel->attr.clockid = 0;
1759 }
1760 if (perf_missing_features.cloexec)
1761 flags &= ~(unsigned long)PERF_FLAG_FD_CLOEXEC;
1762 if (perf_missing_features.mmap2)
1763 evsel->attr.mmap2 = 0;
1764 if (perf_missing_features.exclude_guest)
1765 evsel->attr.exclude_guest = evsel->attr.exclude_host = 0;
1766 if (perf_missing_features.lbr_flags)
1767 evsel->attr.branch_sample_type &= ~(PERF_SAMPLE_BRANCH_NO_FLAGS |
1768 PERF_SAMPLE_BRANCH_NO_CYCLES);
1769 if (perf_missing_features.group_read && evsel->attr.inherit)
1770 evsel->attr.read_format &= ~(PERF_FORMAT_GROUP|PERF_FORMAT_ID);
1771retry_sample_id:
1772 if (perf_missing_features.sample_id_all)
1773 evsel->attr.sample_id_all = 0;
1774
1775 if (verbose >= 2) {
1776 fprintf(stderr, "%.60s\n", graph_dotted_line);
1777 fprintf(stderr, "perf_event_attr:\n");
1778 perf_event_attr__fprintf(stderr, &evsel->attr, __open_attr__fprintf, NULL);
1779 fprintf(stderr, "%.60s\n", graph_dotted_line);
1780 }
1781
1782 for (cpu = 0; cpu < cpus->nr; cpu++) {
1783
1784 for (thread = 0; thread < nthreads; thread++) {
1785 int fd, group_fd;
1786
1787 if (!evsel->cgrp && !evsel->system_wide)
1788 pid = thread_map__pid(threads, thread);
1789
1790 group_fd = get_group_fd(evsel, cpu, thread);
1791retry_open:
1792 pr_debug2("sys_perf_event_open: pid %d cpu %d group_fd %d flags %#lx",
1793 pid, cpus->map[cpu], group_fd, flags);
1794
1795 test_attr__ready();
1796
1797 fd = sys_perf_event_open(&evsel->attr, pid, cpus->map[cpu],
1798 group_fd, flags);
1799
1800 FD(evsel, cpu, thread) = fd;
1801
1802 if (fd < 0) {
1803 err = -errno;
1804
1805 if (ignore_missing_thread(evsel, cpus->nr, cpu, threads, thread, err)) {
1806
1807
1808
1809
1810
1811 nthreads--;
1812 thread--;
1813
1814
1815 err = 0;
1816 continue;
1817 }
1818
1819 pr_debug2("\nsys_perf_event_open failed, error %d\n",
1820 err);
1821 goto try_fallback;
1822 }
1823
1824 pr_debug2(" = %d\n", fd);
1825
1826 if (evsel->bpf_fd >= 0) {
1827 int evt_fd = fd;
1828 int bpf_fd = evsel->bpf_fd;
1829
1830 err = ioctl(evt_fd,
1831 PERF_EVENT_IOC_SET_BPF,
1832 bpf_fd);
1833 if (err && errno != EEXIST) {
1834 pr_err("failed to attach bpf fd %d: %s\n",
1835 bpf_fd, strerror(errno));
1836 err = -EINVAL;
1837 goto out_close;
1838 }
1839 }
1840
1841 set_rlimit = NO_CHANGE;
1842
1843
1844
1845
1846
1847
1848 if (perf_missing_features.clockid ||
1849 perf_missing_features.clockid_wrong) {
1850 err = -EINVAL;
1851 goto out_close;
1852 }
1853 }
1854 }
1855
1856 return 0;
1857
1858try_fallback:
1859
1860
1861
1862
1863 if (err == -EMFILE && set_rlimit < INCREASED_MAX) {
1864 struct rlimit l;
1865 int old_errno = errno;
1866
1867 if (getrlimit(RLIMIT_NOFILE, &l) == 0) {
1868 if (set_rlimit == NO_CHANGE)
1869 l.rlim_cur = l.rlim_max;
1870 else {
1871 l.rlim_cur = l.rlim_max + 1000;
1872 l.rlim_max = l.rlim_cur;
1873 }
1874 if (setrlimit(RLIMIT_NOFILE, &l) == 0) {
1875 set_rlimit++;
1876 errno = old_errno;
1877 goto retry_open;
1878 }
1879 }
1880 errno = old_errno;
1881 }
1882
1883 if (err != -EINVAL || cpu > 0 || thread > 0)
1884 goto out_close;
1885
1886
1887
1888
1889
1890 if (!perf_missing_features.write_backward && evsel->attr.write_backward) {
1891 perf_missing_features.write_backward = true;
1892 pr_debug2("switching off write_backward\n");
1893 goto out_close;
1894 } else if (!perf_missing_features.clockid_wrong && evsel->attr.use_clockid) {
1895 perf_missing_features.clockid_wrong = true;
1896 pr_debug2("switching off clockid\n");
1897 goto fallback_missing_features;
1898 } else if (!perf_missing_features.clockid && evsel->attr.use_clockid) {
1899 perf_missing_features.clockid = true;
1900 pr_debug2("switching off use_clockid\n");
1901 goto fallback_missing_features;
1902 } else if (!perf_missing_features.cloexec && (flags & PERF_FLAG_FD_CLOEXEC)) {
1903 perf_missing_features.cloexec = true;
1904 pr_debug2("switching off cloexec flag\n");
1905 goto fallback_missing_features;
1906 } else if (!perf_missing_features.mmap2 && evsel->attr.mmap2) {
1907 perf_missing_features.mmap2 = true;
1908 pr_debug2("switching off mmap2\n");
1909 goto fallback_missing_features;
1910 } else if (!perf_missing_features.exclude_guest &&
1911 (evsel->attr.exclude_guest || evsel->attr.exclude_host)) {
1912 perf_missing_features.exclude_guest = true;
1913 pr_debug2("switching off exclude_guest, exclude_host\n");
1914 goto fallback_missing_features;
1915 } else if (!perf_missing_features.sample_id_all) {
1916 perf_missing_features.sample_id_all = true;
1917 pr_debug2("switching off sample_id_all\n");
1918 goto retry_sample_id;
1919 } else if (!perf_missing_features.lbr_flags &&
1920 (evsel->attr.branch_sample_type &
1921 (PERF_SAMPLE_BRANCH_NO_CYCLES |
1922 PERF_SAMPLE_BRANCH_NO_FLAGS))) {
1923 perf_missing_features.lbr_flags = true;
1924 pr_debug2("switching off branch sample type no (cycles/flags)\n");
1925 goto fallback_missing_features;
1926 } else if (!perf_missing_features.group_read &&
1927 evsel->attr.inherit &&
1928 (evsel->attr.read_format & PERF_FORMAT_GROUP) &&
1929 perf_evsel__is_group_leader(evsel)) {
1930 perf_missing_features.group_read = true;
1931 pr_debug2("switching off group read\n");
1932 goto fallback_missing_features;
1933 }
1934out_close:
1935 if (err)
1936 threads->err_thread = thread;
1937
1938 do {
1939 while (--thread >= 0) {
1940 close(FD(evsel, cpu, thread));
1941 FD(evsel, cpu, thread) = -1;
1942 }
1943 thread = nthreads;
1944 } while (--cpu >= 0);
1945 return err;
1946}
1947
1948void perf_evsel__close(struct perf_evsel *evsel)
1949{
1950 if (evsel->fd == NULL)
1951 return;
1952
1953 perf_evsel__close_fd(evsel);
1954 perf_evsel__free_fd(evsel);
1955}
1956
1957int perf_evsel__open_per_cpu(struct perf_evsel *evsel,
1958 struct cpu_map *cpus)
1959{
1960 return perf_evsel__open(evsel, cpus, NULL);
1961}
1962
1963int perf_evsel__open_per_thread(struct perf_evsel *evsel,
1964 struct thread_map *threads)
1965{
1966 return perf_evsel__open(evsel, NULL, threads);
1967}
1968
1969static int perf_evsel__parse_id_sample(const struct perf_evsel *evsel,
1970 const union perf_event *event,
1971 struct perf_sample *sample)
1972{
1973 u64 type = evsel->attr.sample_type;
1974 const u64 *array = event->sample.array;
1975 bool swapped = evsel->needs_swap;
1976 union u64_swap u;
1977
1978 array += ((event->header.size -
1979 sizeof(event->header)) / sizeof(u64)) - 1;
1980
1981 if (type & PERF_SAMPLE_IDENTIFIER) {
1982 sample->id = *array;
1983 array--;
1984 }
1985
1986 if (type & PERF_SAMPLE_CPU) {
1987 u.val64 = *array;
1988 if (swapped) {
1989
1990 u.val64 = bswap_64(u.val64);
1991 u.val32[0] = bswap_32(u.val32[0]);
1992 }
1993
1994 sample->cpu = u.val32[0];
1995 array--;
1996 }
1997
1998 if (type & PERF_SAMPLE_STREAM_ID) {
1999 sample->stream_id = *array;
2000 array--;
2001 }
2002
2003 if (type & PERF_SAMPLE_ID) {
2004 sample->id = *array;
2005 array--;
2006 }
2007
2008 if (type & PERF_SAMPLE_TIME) {
2009 sample->time = *array;
2010 array--;
2011 }
2012
2013 if (type & PERF_SAMPLE_TID) {
2014 u.val64 = *array;
2015 if (swapped) {
2016
2017 u.val64 = bswap_64(u.val64);
2018 u.val32[0] = bswap_32(u.val32[0]);
2019 u.val32[1] = bswap_32(u.val32[1]);
2020 }
2021
2022 sample->pid = u.val32[0];
2023 sample->tid = u.val32[1];
2024 array--;
2025 }
2026
2027 return 0;
2028}
2029
2030static inline bool overflow(const void *endp, u16 max_size, const void *offset,
2031 u64 size)
2032{
2033 return size > max_size || offset + size > endp;
2034}
2035
2036#define OVERFLOW_CHECK(offset, size, max_size) \
2037 do { \
2038 if (overflow(endp, (max_size), (offset), (size))) \
2039 return -EFAULT; \
2040 } while (0)
2041
2042#define OVERFLOW_CHECK_u64(offset) \
2043 OVERFLOW_CHECK(offset, sizeof(u64), sizeof(u64))
2044
2045static int
2046perf_event__check_size(union perf_event *event, unsigned int sample_size)
2047{
2048
2049
2050
2051
2052
2053 if (sample_size + sizeof(event->header) > event->header.size)
2054 return -EFAULT;
2055
2056 return 0;
2057}
2058
2059int perf_evsel__parse_sample(struct perf_evsel *evsel, union perf_event *event,
2060 struct perf_sample *data)
2061{
2062 u64 type = evsel->attr.sample_type;
2063 bool swapped = evsel->needs_swap;
2064 const u64 *array;
2065 u16 max_size = event->header.size;
2066 const void *endp = (void *)event + max_size;
2067 u64 sz;
2068
2069
2070
2071
2072
2073 union u64_swap u;
2074
2075 memset(data, 0, sizeof(*data));
2076 data->cpu = data->pid = data->tid = -1;
2077 data->stream_id = data->id = data->time = -1ULL;
2078 data->period = evsel->attr.sample_period;
2079 data->cpumode = event->header.misc & PERF_RECORD_MISC_CPUMODE_MASK;
2080 data->misc = event->header.misc;
2081 data->id = -1ULL;
2082 data->data_src = PERF_MEM_DATA_SRC_NONE;
2083
2084 if (event->header.type != PERF_RECORD_SAMPLE) {
2085 if (!evsel->attr.sample_id_all)
2086 return 0;
2087 return perf_evsel__parse_id_sample(evsel, event, data);
2088 }
2089
2090 array = event->sample.array;
2091
2092 if (perf_event__check_size(event, evsel->sample_size))
2093 return -EFAULT;
2094
2095 if (type & PERF_SAMPLE_IDENTIFIER) {
2096 data->id = *array;
2097 array++;
2098 }
2099
2100 if (type & PERF_SAMPLE_IP) {
2101 data->ip = *array;
2102 array++;
2103 }
2104
2105 if (type & PERF_SAMPLE_TID) {
2106 u.val64 = *array;
2107 if (swapped) {
2108
2109 u.val64 = bswap_64(u.val64);
2110 u.val32[0] = bswap_32(u.val32[0]);
2111 u.val32[1] = bswap_32(u.val32[1]);
2112 }
2113
2114 data->pid = u.val32[0];
2115 data->tid = u.val32[1];
2116 array++;
2117 }
2118
2119 if (type & PERF_SAMPLE_TIME) {
2120 data->time = *array;
2121 array++;
2122 }
2123
2124 if (type & PERF_SAMPLE_ADDR) {
2125 data->addr = *array;
2126 array++;
2127 }
2128
2129 if (type & PERF_SAMPLE_ID) {
2130 data->id = *array;
2131 array++;
2132 }
2133
2134 if (type & PERF_SAMPLE_STREAM_ID) {
2135 data->stream_id = *array;
2136 array++;
2137 }
2138
2139 if (type & PERF_SAMPLE_CPU) {
2140
2141 u.val64 = *array;
2142 if (swapped) {
2143
2144 u.val64 = bswap_64(u.val64);
2145 u.val32[0] = bswap_32(u.val32[0]);
2146 }
2147
2148 data->cpu = u.val32[0];
2149 array++;
2150 }
2151
2152 if (type & PERF_SAMPLE_PERIOD) {
2153 data->period = *array;
2154 array++;
2155 }
2156
2157 if (type & PERF_SAMPLE_READ) {
2158 u64 read_format = evsel->attr.read_format;
2159
2160 OVERFLOW_CHECK_u64(array);
2161 if (read_format & PERF_FORMAT_GROUP)
2162 data->read.group.nr = *array;
2163 else
2164 data->read.one.value = *array;
2165
2166 array++;
2167
2168 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
2169 OVERFLOW_CHECK_u64(array);
2170 data->read.time_enabled = *array;
2171 array++;
2172 }
2173
2174 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
2175 OVERFLOW_CHECK_u64(array);
2176 data->read.time_running = *array;
2177 array++;
2178 }
2179
2180
2181 if (read_format & PERF_FORMAT_GROUP) {
2182 const u64 max_group_nr = UINT64_MAX /
2183 sizeof(struct sample_read_value);
2184
2185 if (data->read.group.nr > max_group_nr)
2186 return -EFAULT;
2187 sz = data->read.group.nr *
2188 sizeof(struct sample_read_value);
2189 OVERFLOW_CHECK(array, sz, max_size);
2190 data->read.group.values =
2191 (struct sample_read_value *)array;
2192 array = (void *)array + sz;
2193 } else {
2194 OVERFLOW_CHECK_u64(array);
2195 data->read.one.id = *array;
2196 array++;
2197 }
2198 }
2199
2200 if (type & PERF_SAMPLE_CALLCHAIN) {
2201 const u64 max_callchain_nr = UINT64_MAX / sizeof(u64);
2202
2203 OVERFLOW_CHECK_u64(array);
2204 data->callchain = (struct ip_callchain *)array++;
2205 if (data->callchain->nr > max_callchain_nr)
2206 return -EFAULT;
2207 sz = data->callchain->nr * sizeof(u64);
2208 OVERFLOW_CHECK(array, sz, max_size);
2209 array = (void *)array + sz;
2210 }
2211
2212 if (type & PERF_SAMPLE_RAW) {
2213 OVERFLOW_CHECK_u64(array);
2214 u.val64 = *array;
2215
2216
2217
2218
2219
2220
2221
2222 if (swapped) {
2223 u.val64 = bswap_64(u.val64);
2224 u.val32[0] = bswap_32(u.val32[0]);
2225 u.val32[1] = bswap_32(u.val32[1]);
2226 }
2227 data->raw_size = u.val32[0];
2228
2229
2230
2231
2232
2233 if (swapped)
2234 mem_bswap_64((void *) array, data->raw_size);
2235
2236 array = (void *)array + sizeof(u32);
2237
2238 OVERFLOW_CHECK(array, data->raw_size, max_size);
2239 data->raw_data = (void *)array;
2240 array = (void *)array + data->raw_size;
2241 }
2242
2243 if (type & PERF_SAMPLE_BRANCH_STACK) {
2244 const u64 max_branch_nr = UINT64_MAX /
2245 sizeof(struct branch_entry);
2246
2247 OVERFLOW_CHECK_u64(array);
2248 data->branch_stack = (struct branch_stack *)array++;
2249
2250 if (data->branch_stack->nr > max_branch_nr)
2251 return -EFAULT;
2252 sz = data->branch_stack->nr * sizeof(struct branch_entry);
2253 OVERFLOW_CHECK(array, sz, max_size);
2254 array = (void *)array + sz;
2255 }
2256
2257 if (type & PERF_SAMPLE_REGS_USER) {
2258 OVERFLOW_CHECK_u64(array);
2259 data->user_regs.abi = *array;
2260 array++;
2261
2262 if (data->user_regs.abi) {
2263 u64 mask = evsel->attr.sample_regs_user;
2264
2265 sz = hweight_long(mask) * sizeof(u64);
2266 OVERFLOW_CHECK(array, sz, max_size);
2267 data->user_regs.mask = mask;
2268 data->user_regs.regs = (u64 *)array;
2269 array = (void *)array + sz;
2270 }
2271 }
2272
2273 if (type & PERF_SAMPLE_STACK_USER) {
2274 OVERFLOW_CHECK_u64(array);
2275 sz = *array++;
2276
2277 data->user_stack.offset = ((char *)(array - 1)
2278 - (char *) event);
2279
2280 if (!sz) {
2281 data->user_stack.size = 0;
2282 } else {
2283 OVERFLOW_CHECK(array, sz, max_size);
2284 data->user_stack.data = (char *)array;
2285 array = (void *)array + sz;
2286 OVERFLOW_CHECK_u64(array);
2287 data->user_stack.size = *array++;
2288 if (WARN_ONCE(data->user_stack.size > sz,
2289 "user stack dump failure\n"))
2290 return -EFAULT;
2291 }
2292 }
2293
2294 if (type & PERF_SAMPLE_WEIGHT) {
2295 OVERFLOW_CHECK_u64(array);
2296 data->weight = *array;
2297 array++;
2298 }
2299
2300 if (type & PERF_SAMPLE_DATA_SRC) {
2301 OVERFLOW_CHECK_u64(array);
2302 data->data_src = *array;
2303 array++;
2304 }
2305
2306 if (type & PERF_SAMPLE_TRANSACTION) {
2307 OVERFLOW_CHECK_u64(array);
2308 data->transaction = *array;
2309 array++;
2310 }
2311
2312 data->intr_regs.abi = PERF_SAMPLE_REGS_ABI_NONE;
2313 if (type & PERF_SAMPLE_REGS_INTR) {
2314 OVERFLOW_CHECK_u64(array);
2315 data->intr_regs.abi = *array;
2316 array++;
2317
2318 if (data->intr_regs.abi != PERF_SAMPLE_REGS_ABI_NONE) {
2319 u64 mask = evsel->attr.sample_regs_intr;
2320
2321 sz = hweight_long(mask) * sizeof(u64);
2322 OVERFLOW_CHECK(array, sz, max_size);
2323 data->intr_regs.mask = mask;
2324 data->intr_regs.regs = (u64 *)array;
2325 array = (void *)array + sz;
2326 }
2327 }
2328
2329 data->phys_addr = 0;
2330 if (type & PERF_SAMPLE_PHYS_ADDR) {
2331 data->phys_addr = *array;
2332 array++;
2333 }
2334
2335 return 0;
2336}
2337
2338int perf_evsel__parse_sample_timestamp(struct perf_evsel *evsel,
2339 union perf_event *event,
2340 u64 *timestamp)
2341{
2342 u64 type = evsel->attr.sample_type;
2343 const u64 *array;
2344
2345 if (!(type & PERF_SAMPLE_TIME))
2346 return -1;
2347
2348 if (event->header.type != PERF_RECORD_SAMPLE) {
2349 struct perf_sample data = {
2350 .time = -1ULL,
2351 };
2352
2353 if (!evsel->attr.sample_id_all)
2354 return -1;
2355 if (perf_evsel__parse_id_sample(evsel, event, &data))
2356 return -1;
2357
2358 *timestamp = data.time;
2359 return 0;
2360 }
2361
2362 array = event->sample.array;
2363
2364 if (perf_event__check_size(event, evsel->sample_size))
2365 return -EFAULT;
2366
2367 if (type & PERF_SAMPLE_IDENTIFIER)
2368 array++;
2369
2370 if (type & PERF_SAMPLE_IP)
2371 array++;
2372
2373 if (type & PERF_SAMPLE_TID)
2374 array++;
2375
2376 if (type & PERF_SAMPLE_TIME)
2377 *timestamp = *array;
2378
2379 return 0;
2380}
2381
2382size_t perf_event__sample_event_size(const struct perf_sample *sample, u64 type,
2383 u64 read_format)
2384{
2385 size_t sz, result = sizeof(struct sample_event);
2386
2387 if (type & PERF_SAMPLE_IDENTIFIER)
2388 result += sizeof(u64);
2389
2390 if (type & PERF_SAMPLE_IP)
2391 result += sizeof(u64);
2392
2393 if (type & PERF_SAMPLE_TID)
2394 result += sizeof(u64);
2395
2396 if (type & PERF_SAMPLE_TIME)
2397 result += sizeof(u64);
2398
2399 if (type & PERF_SAMPLE_ADDR)
2400 result += sizeof(u64);
2401
2402 if (type & PERF_SAMPLE_ID)
2403 result += sizeof(u64);
2404
2405 if (type & PERF_SAMPLE_STREAM_ID)
2406 result += sizeof(u64);
2407
2408 if (type & PERF_SAMPLE_CPU)
2409 result += sizeof(u64);
2410
2411 if (type & PERF_SAMPLE_PERIOD)
2412 result += sizeof(u64);
2413
2414 if (type & PERF_SAMPLE_READ) {
2415 result += sizeof(u64);
2416 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
2417 result += sizeof(u64);
2418 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
2419 result += sizeof(u64);
2420
2421 if (read_format & PERF_FORMAT_GROUP) {
2422 sz = sample->read.group.nr *
2423 sizeof(struct sample_read_value);
2424 result += sz;
2425 } else {
2426 result += sizeof(u64);
2427 }
2428 }
2429
2430 if (type & PERF_SAMPLE_CALLCHAIN) {
2431 sz = (sample->callchain->nr + 1) * sizeof(u64);
2432 result += sz;
2433 }
2434
2435 if (type & PERF_SAMPLE_RAW) {
2436 result += sizeof(u32);
2437 result += sample->raw_size;
2438 }
2439
2440 if (type & PERF_SAMPLE_BRANCH_STACK) {
2441 sz = sample->branch_stack->nr * sizeof(struct branch_entry);
2442 sz += sizeof(u64);
2443 result += sz;
2444 }
2445
2446 if (type & PERF_SAMPLE_REGS_USER) {
2447 if (sample->user_regs.abi) {
2448 result += sizeof(u64);
2449 sz = hweight_long(sample->user_regs.mask) * sizeof(u64);
2450 result += sz;
2451 } else {
2452 result += sizeof(u64);
2453 }
2454 }
2455
2456 if (type & PERF_SAMPLE_STACK_USER) {
2457 sz = sample->user_stack.size;
2458 result += sizeof(u64);
2459 if (sz) {
2460 result += sz;
2461 result += sizeof(u64);
2462 }
2463 }
2464
2465 if (type & PERF_SAMPLE_WEIGHT)
2466 result += sizeof(u64);
2467
2468 if (type & PERF_SAMPLE_DATA_SRC)
2469 result += sizeof(u64);
2470
2471 if (type & PERF_SAMPLE_TRANSACTION)
2472 result += sizeof(u64);
2473
2474 if (type & PERF_SAMPLE_REGS_INTR) {
2475 if (sample->intr_regs.abi) {
2476 result += sizeof(u64);
2477 sz = hweight_long(sample->intr_regs.mask) * sizeof(u64);
2478 result += sz;
2479 } else {
2480 result += sizeof(u64);
2481 }
2482 }
2483
2484 if (type & PERF_SAMPLE_PHYS_ADDR)
2485 result += sizeof(u64);
2486
2487 return result;
2488}
2489
2490int perf_event__synthesize_sample(union perf_event *event, u64 type,
2491 u64 read_format,
2492 const struct perf_sample *sample)
2493{
2494 u64 *array;
2495 size_t sz;
2496
2497
2498
2499
2500 union u64_swap u;
2501
2502 array = event->sample.array;
2503
2504 if (type & PERF_SAMPLE_IDENTIFIER) {
2505 *array = sample->id;
2506 array++;
2507 }
2508
2509 if (type & PERF_SAMPLE_IP) {
2510 *array = sample->ip;
2511 array++;
2512 }
2513
2514 if (type & PERF_SAMPLE_TID) {
2515 u.val32[0] = sample->pid;
2516 u.val32[1] = sample->tid;
2517 *array = u.val64;
2518 array++;
2519 }
2520
2521 if (type & PERF_SAMPLE_TIME) {
2522 *array = sample->time;
2523 array++;
2524 }
2525
2526 if (type & PERF_SAMPLE_ADDR) {
2527 *array = sample->addr;
2528 array++;
2529 }
2530
2531 if (type & PERF_SAMPLE_ID) {
2532 *array = sample->id;
2533 array++;
2534 }
2535
2536 if (type & PERF_SAMPLE_STREAM_ID) {
2537 *array = sample->stream_id;
2538 array++;
2539 }
2540
2541 if (type & PERF_SAMPLE_CPU) {
2542 u.val32[0] = sample->cpu;
2543 u.val32[1] = 0;
2544 *array = u.val64;
2545 array++;
2546 }
2547
2548 if (type & PERF_SAMPLE_PERIOD) {
2549 *array = sample->period;
2550 array++;
2551 }
2552
2553 if (type & PERF_SAMPLE_READ) {
2554 if (read_format & PERF_FORMAT_GROUP)
2555 *array = sample->read.group.nr;
2556 else
2557 *array = sample->read.one.value;
2558 array++;
2559
2560 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
2561 *array = sample->read.time_enabled;
2562 array++;
2563 }
2564
2565 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
2566 *array = sample->read.time_running;
2567 array++;
2568 }
2569
2570
2571 if (read_format & PERF_FORMAT_GROUP) {
2572 sz = sample->read.group.nr *
2573 sizeof(struct sample_read_value);
2574 memcpy(array, sample->read.group.values, sz);
2575 array = (void *)array + sz;
2576 } else {
2577 *array = sample->read.one.id;
2578 array++;
2579 }
2580 }
2581
2582 if (type & PERF_SAMPLE_CALLCHAIN) {
2583 sz = (sample->callchain->nr + 1) * sizeof(u64);
2584 memcpy(array, sample->callchain, sz);
2585 array = (void *)array + sz;
2586 }
2587
2588 if (type & PERF_SAMPLE_RAW) {
2589 u.val32[0] = sample->raw_size;
2590 *array = u.val64;
2591 array = (void *)array + sizeof(u32);
2592
2593 memcpy(array, sample->raw_data, sample->raw_size);
2594 array = (void *)array + sample->raw_size;
2595 }
2596
2597 if (type & PERF_SAMPLE_BRANCH_STACK) {
2598 sz = sample->branch_stack->nr * sizeof(struct branch_entry);
2599 sz += sizeof(u64);
2600 memcpy(array, sample->branch_stack, sz);
2601 array = (void *)array + sz;
2602 }
2603
2604 if (type & PERF_SAMPLE_REGS_USER) {
2605 if (sample->user_regs.abi) {
2606 *array++ = sample->user_regs.abi;
2607 sz = hweight_long(sample->user_regs.mask) * sizeof(u64);
2608 memcpy(array, sample->user_regs.regs, sz);
2609 array = (void *)array + sz;
2610 } else {
2611 *array++ = 0;
2612 }
2613 }
2614
2615 if (type & PERF_SAMPLE_STACK_USER) {
2616 sz = sample->user_stack.size;
2617 *array++ = sz;
2618 if (sz) {
2619 memcpy(array, sample->user_stack.data, sz);
2620 array = (void *)array + sz;
2621 *array++ = sz;
2622 }
2623 }
2624
2625 if (type & PERF_SAMPLE_WEIGHT) {
2626 *array = sample->weight;
2627 array++;
2628 }
2629
2630 if (type & PERF_SAMPLE_DATA_SRC) {
2631 *array = sample->data_src;
2632 array++;
2633 }
2634
2635 if (type & PERF_SAMPLE_TRANSACTION) {
2636 *array = sample->transaction;
2637 array++;
2638 }
2639
2640 if (type & PERF_SAMPLE_REGS_INTR) {
2641 if (sample->intr_regs.abi) {
2642 *array++ = sample->intr_regs.abi;
2643 sz = hweight_long(sample->intr_regs.mask) * sizeof(u64);
2644 memcpy(array, sample->intr_regs.regs, sz);
2645 array = (void *)array + sz;
2646 } else {
2647 *array++ = 0;
2648 }
2649 }
2650
2651 if (type & PERF_SAMPLE_PHYS_ADDR) {
2652 *array = sample->phys_addr;
2653 array++;
2654 }
2655
2656 return 0;
2657}
2658
2659struct format_field *perf_evsel__field(struct perf_evsel *evsel, const char *name)
2660{
2661 return pevent_find_field(evsel->tp_format, name);
2662}
2663
2664void *perf_evsel__rawptr(struct perf_evsel *evsel, struct perf_sample *sample,
2665 const char *name)
2666{
2667 struct format_field *field = perf_evsel__field(evsel, name);
2668 int offset;
2669
2670 if (!field)
2671 return NULL;
2672
2673 offset = field->offset;
2674
2675 if (field->flags & FIELD_IS_DYNAMIC) {
2676 offset = *(int *)(sample->raw_data + field->offset);
2677 offset &= 0xffff;
2678 }
2679
2680 return sample->raw_data + offset;
2681}
2682
2683u64 format_field__intval(struct format_field *field, struct perf_sample *sample,
2684 bool needs_swap)
2685{
2686 u64 value;
2687 void *ptr = sample->raw_data + field->offset;
2688
2689 switch (field->size) {
2690 case 1:
2691 return *(u8 *)ptr;
2692 case 2:
2693 value = *(u16 *)ptr;
2694 break;
2695 case 4:
2696 value = *(u32 *)ptr;
2697 break;
2698 case 8:
2699 memcpy(&value, ptr, sizeof(u64));
2700 break;
2701 default:
2702 return 0;
2703 }
2704
2705 if (!needs_swap)
2706 return value;
2707
2708 switch (field->size) {
2709 case 2:
2710 return bswap_16(value);
2711 case 4:
2712 return bswap_32(value);
2713 case 8:
2714 return bswap_64(value);
2715 default:
2716 return 0;
2717 }
2718
2719 return 0;
2720}
2721
2722u64 perf_evsel__intval(struct perf_evsel *evsel, struct perf_sample *sample,
2723 const char *name)
2724{
2725 struct format_field *field = perf_evsel__field(evsel, name);
2726
2727 if (!field)
2728 return 0;
2729
2730 return field ? format_field__intval(field, sample, evsel->needs_swap) : 0;
2731}
2732
2733bool perf_evsel__fallback(struct perf_evsel *evsel, int err,
2734 char *msg, size_t msgsize)
2735{
2736 int paranoid;
2737
2738 if ((err == ENOENT || err == ENXIO || err == ENODEV) &&
2739 evsel->attr.type == PERF_TYPE_HARDWARE &&
2740 evsel->attr.config == PERF_COUNT_HW_CPU_CYCLES) {
2741
2742
2743
2744
2745
2746
2747
2748
2749 scnprintf(msg, msgsize, "%s",
2750"The cycles event is not supported, trying to fall back to cpu-clock-ticks");
2751
2752 evsel->attr.type = PERF_TYPE_SOFTWARE;
2753 evsel->attr.config = PERF_COUNT_SW_CPU_CLOCK;
2754
2755 zfree(&evsel->name);
2756 return true;
2757 } else if (err == EACCES && !evsel->attr.exclude_kernel &&
2758 (paranoid = perf_event_paranoid()) > 1) {
2759 const char *name = perf_evsel__name(evsel);
2760 char *new_name;
2761 const char *sep = ":";
2762
2763
2764 if (strchr(name, '/') ||
2765 strchr(name, ':'))
2766 sep = "";
2767
2768 if (asprintf(&new_name, "%s%su", name, sep) < 0)
2769 return false;
2770
2771 if (evsel->name)
2772 free(evsel->name);
2773 evsel->name = new_name;
2774 scnprintf(msg, msgsize,
2775"kernel.perf_event_paranoid=%d, trying to fall back to excluding kernel samples", paranoid);
2776 evsel->attr.exclude_kernel = 1;
2777
2778 return true;
2779 }
2780
2781 return false;
2782}
2783
2784static bool find_process(const char *name)
2785{
2786 size_t len = strlen(name);
2787 DIR *dir;
2788 struct dirent *d;
2789 int ret = -1;
2790
2791 dir = opendir(procfs__mountpoint());
2792 if (!dir)
2793 return false;
2794
2795
2796 while (ret && (d = readdir(dir)) != NULL) {
2797 char path[PATH_MAX];
2798 char *data;
2799 size_t size;
2800
2801 if ((d->d_type != DT_DIR) ||
2802 !strcmp(".", d->d_name) ||
2803 !strcmp("..", d->d_name))
2804 continue;
2805
2806 scnprintf(path, sizeof(path), "%s/%s/comm",
2807 procfs__mountpoint(), d->d_name);
2808
2809 if (filename__read_str(path, &data, &size))
2810 continue;
2811
2812 ret = strncmp(name, data, len);
2813 free(data);
2814 }
2815
2816 closedir(dir);
2817 return ret ? false : true;
2818}
2819
2820int perf_evsel__open_strerror(struct perf_evsel *evsel, struct target *target,
2821 int err, char *msg, size_t size)
2822{
2823 char sbuf[STRERR_BUFSIZE];
2824 int printed = 0;
2825
2826 switch (err) {
2827 case EPERM:
2828 case EACCES:
2829 if (err == EPERM)
2830 printed = scnprintf(msg, size,
2831 "No permission to enable %s event.\n\n",
2832 perf_evsel__name(evsel));
2833
2834 return scnprintf(msg + printed, size - printed,
2835 "You may not have permission to collect %sstats.\n\n"
2836 "Consider tweaking /proc/sys/kernel/perf_event_paranoid,\n"
2837 "which controls use of the performance events system by\n"
2838 "unprivileged users (without CAP_SYS_ADMIN).\n\n"
2839 "The current value is %d:\n\n"
2840 " -1: Allow use of (almost) all events by all users\n"
2841 " Ignore mlock limit after perf_event_mlock_kb without CAP_IPC_LOCK\n"
2842 ">= 0: Disallow ftrace function tracepoint by users without CAP_SYS_ADMIN\n"
2843 " Disallow raw tracepoint access by users without CAP_SYS_ADMIN\n"
2844 ">= 1: Disallow CPU event access by users without CAP_SYS_ADMIN\n"
2845 ">= 2: Disallow kernel profiling by users without CAP_SYS_ADMIN\n\n"
2846 "To make this setting permanent, edit /etc/sysctl.conf too, e.g.:\n\n"
2847 " kernel.perf_event_paranoid = -1\n" ,
2848 target->system_wide ? "system-wide " : "",
2849 perf_event_paranoid());
2850 case ENOENT:
2851 return scnprintf(msg, size, "The %s event is not supported.",
2852 perf_evsel__name(evsel));
2853 case EMFILE:
2854 return scnprintf(msg, size, "%s",
2855 "Too many events are opened.\n"
2856 "Probably the maximum number of open file descriptors has been reached.\n"
2857 "Hint: Try again after reducing the number of events.\n"
2858 "Hint: Try increasing the limit with 'ulimit -n <limit>'");
2859 case ENOMEM:
2860 if ((evsel->attr.sample_type & PERF_SAMPLE_CALLCHAIN) != 0 &&
2861 access("/proc/sys/kernel/perf_event_max_stack", F_OK) == 0)
2862 return scnprintf(msg, size,
2863 "Not enough memory to setup event with callchain.\n"
2864 "Hint: Try tweaking /proc/sys/kernel/perf_event_max_stack\n"
2865 "Hint: Current value: %d", sysctl_perf_event_max_stack);
2866 break;
2867 case ENODEV:
2868 if (target->cpu_list)
2869 return scnprintf(msg, size, "%s",
2870 "No such device - did you specify an out-of-range profile CPU?");
2871 break;
2872 case EOPNOTSUPP:
2873 if (evsel->attr.sample_period != 0)
2874 return scnprintf(msg, size,
2875 "%s: PMU Hardware doesn't support sampling/overflow-interrupts. Try 'perf stat'",
2876 perf_evsel__name(evsel));
2877 if (evsel->attr.precise_ip)
2878 return scnprintf(msg, size, "%s",
2879 "\'precise\' request may not be supported. Try removing 'p' modifier.");
2880#if defined(__i386__) || defined(__x86_64__)
2881 if (evsel->attr.type == PERF_TYPE_HARDWARE)
2882 return scnprintf(msg, size, "%s",
2883 "No hardware sampling interrupt available.\n");
2884#endif
2885 break;
2886 case EBUSY:
2887 if (find_process("oprofiled"))
2888 return scnprintf(msg, size,
2889 "The PMU counters are busy/taken by another profiler.\n"
2890 "We found oprofile daemon running, please stop it and try again.");
2891 break;
2892 case EINVAL:
2893 if (evsel->attr.write_backward && perf_missing_features.write_backward)
2894 return scnprintf(msg, size, "Reading from overwrite event is not supported by this kernel.");
2895 if (perf_missing_features.clockid)
2896 return scnprintf(msg, size, "clockid feature not supported.");
2897 if (perf_missing_features.clockid_wrong)
2898 return scnprintf(msg, size, "wrong clockid (%d).", clockid);
2899 break;
2900 default:
2901 break;
2902 }
2903
2904 return scnprintf(msg, size,
2905 "The sys_perf_event_open() syscall returned with %d (%s) for event (%s).\n"
2906 "/bin/dmesg | grep -i perf may provide additional information.\n",
2907 err, str_error_r(err, sbuf, sizeof(sbuf)),
2908 perf_evsel__name(evsel));
2909}
2910
2911struct perf_env *perf_evsel__env(struct perf_evsel *evsel)
2912{
2913 if (evsel && evsel->evlist)
2914 return evsel->evlist->env;
2915 return NULL;
2916}
2917