1
2
3
4
5
6
7
8
9
10#include <byteswap.h>
11#include <linux/bitops.h>
12#include <api/fs/debugfs.h>
13#include <traceevent/event-parse.h>
14#include <linux/hw_breakpoint.h>
15#include <linux/perf_event.h>
16#include <sys/resource.h>
17#include "asm/bug.h"
18#include "callchain.h"
19#include "cgroup.h"
20#include "evsel.h"
21#include "evlist.h"
22#include "util.h"
23#include "cpumap.h"
24#include "thread_map.h"
25#include "target.h"
26#include "perf_regs.h"
27#include "debug.h"
28#include "trace-event.h"
29#include "stat.h"
30
31static struct {
32 bool sample_id_all;
33 bool exclude_guest;
34 bool mmap2;
35 bool cloexec;
36 bool clockid;
37 bool clockid_wrong;
38} perf_missing_features;
39
40static clockid_t clockid;
41
42static int perf_evsel__no_extra_init(struct perf_evsel *evsel __maybe_unused)
43{
44 return 0;
45}
46
47static void perf_evsel__no_extra_fini(struct perf_evsel *evsel __maybe_unused)
48{
49}
50
51static struct {
52 size_t size;
53 int (*init)(struct perf_evsel *evsel);
54 void (*fini)(struct perf_evsel *evsel);
55} perf_evsel__object = {
56 .size = sizeof(struct perf_evsel),
57 .init = perf_evsel__no_extra_init,
58 .fini = perf_evsel__no_extra_fini,
59};
60
61int perf_evsel__object_config(size_t object_size,
62 int (*init)(struct perf_evsel *evsel),
63 void (*fini)(struct perf_evsel *evsel))
64{
65
66 if (object_size == 0)
67 goto set_methods;
68
69 if (perf_evsel__object.size > object_size)
70 return -EINVAL;
71
72 perf_evsel__object.size = object_size;
73
74set_methods:
75 if (init != NULL)
76 perf_evsel__object.init = init;
77
78 if (fini != NULL)
79 perf_evsel__object.fini = fini;
80
81 return 0;
82}
83
84#define FD(e, x, y) (*(int *)xyarray__entry(e->fd, x, y))
85
86int __perf_evsel__sample_size(u64 sample_type)
87{
88 u64 mask = sample_type & PERF_SAMPLE_MASK;
89 int size = 0;
90 int i;
91
92 for (i = 0; i < 64; i++) {
93 if (mask & (1ULL << i))
94 size++;
95 }
96
97 size *= sizeof(u64);
98
99 return size;
100}
101
102
103
104
105
106
107
108
109
110static int __perf_evsel__calc_id_pos(u64 sample_type)
111{
112 int idx = 0;
113
114 if (sample_type & PERF_SAMPLE_IDENTIFIER)
115 return 0;
116
117 if (!(sample_type & PERF_SAMPLE_ID))
118 return -1;
119
120 if (sample_type & PERF_SAMPLE_IP)
121 idx += 1;
122
123 if (sample_type & PERF_SAMPLE_TID)
124 idx += 1;
125
126 if (sample_type & PERF_SAMPLE_TIME)
127 idx += 1;
128
129 if (sample_type & PERF_SAMPLE_ADDR)
130 idx += 1;
131
132 return idx;
133}
134
135
136
137
138
139
140
141
142
143static int __perf_evsel__calc_is_pos(u64 sample_type)
144{
145 int idx = 1;
146
147 if (sample_type & PERF_SAMPLE_IDENTIFIER)
148 return 1;
149
150 if (!(sample_type & PERF_SAMPLE_ID))
151 return -1;
152
153 if (sample_type & PERF_SAMPLE_CPU)
154 idx += 1;
155
156 if (sample_type & PERF_SAMPLE_STREAM_ID)
157 idx += 1;
158
159 return idx;
160}
161
162void perf_evsel__calc_id_pos(struct perf_evsel *evsel)
163{
164 evsel->id_pos = __perf_evsel__calc_id_pos(evsel->attr.sample_type);
165 evsel->is_pos = __perf_evsel__calc_is_pos(evsel->attr.sample_type);
166}
167
168void __perf_evsel__set_sample_bit(struct perf_evsel *evsel,
169 enum perf_event_sample_format bit)
170{
171 if (!(evsel->attr.sample_type & bit)) {
172 evsel->attr.sample_type |= bit;
173 evsel->sample_size += sizeof(u64);
174 perf_evsel__calc_id_pos(evsel);
175 }
176}
177
178void __perf_evsel__reset_sample_bit(struct perf_evsel *evsel,
179 enum perf_event_sample_format bit)
180{
181 if (evsel->attr.sample_type & bit) {
182 evsel->attr.sample_type &= ~bit;
183 evsel->sample_size -= sizeof(u64);
184 perf_evsel__calc_id_pos(evsel);
185 }
186}
187
188void perf_evsel__set_sample_id(struct perf_evsel *evsel,
189 bool can_sample_identifier)
190{
191 if (can_sample_identifier) {
192 perf_evsel__reset_sample_bit(evsel, ID);
193 perf_evsel__set_sample_bit(evsel, IDENTIFIER);
194 } else {
195 perf_evsel__set_sample_bit(evsel, ID);
196 }
197 evsel->attr.read_format |= PERF_FORMAT_ID;
198}
199
200void perf_evsel__init(struct perf_evsel *evsel,
201 struct perf_event_attr *attr, int idx)
202{
203 evsel->idx = idx;
204 evsel->tracking = !idx;
205 evsel->attr = *attr;
206 evsel->leader = evsel;
207 evsel->unit = "";
208 evsel->scale = 1.0;
209 INIT_LIST_HEAD(&evsel->node);
210 perf_evsel__object.init(evsel);
211 evsel->sample_size = __perf_evsel__sample_size(attr->sample_type);
212 perf_evsel__calc_id_pos(evsel);
213}
214
215struct perf_evsel *perf_evsel__new_idx(struct perf_event_attr *attr, int idx)
216{
217 struct perf_evsel *evsel = zalloc(perf_evsel__object.size);
218
219 if (evsel != NULL)
220 perf_evsel__init(evsel, attr, idx);
221
222 return evsel;
223}
224
225struct perf_evsel *perf_evsel__newtp_idx(const char *sys, const char *name, int idx)
226{
227 struct perf_evsel *evsel = zalloc(perf_evsel__object.size);
228
229 if (evsel != NULL) {
230 struct perf_event_attr attr = {
231 .type = PERF_TYPE_TRACEPOINT,
232 .sample_type = (PERF_SAMPLE_RAW | PERF_SAMPLE_TIME |
233 PERF_SAMPLE_CPU | PERF_SAMPLE_PERIOD),
234 };
235
236 if (asprintf(&evsel->name, "%s:%s", sys, name) < 0)
237 goto out_free;
238
239 evsel->tp_format = trace_event__tp_format(sys, name);
240 if (evsel->tp_format == NULL)
241 goto out_free;
242
243 event_attr_init(&attr);
244 attr.config = evsel->tp_format->id;
245 attr.sample_period = 1;
246 perf_evsel__init(evsel, &attr, idx);
247 }
248
249 return evsel;
250
251out_free:
252 zfree(&evsel->name);
253 free(evsel);
254 return NULL;
255}
256
257const char *perf_evsel__hw_names[PERF_COUNT_HW_MAX] = {
258 "cycles",
259 "instructions",
260 "cache-references",
261 "cache-misses",
262 "branches",
263 "branch-misses",
264 "bus-cycles",
265 "stalled-cycles-frontend",
266 "stalled-cycles-backend",
267 "ref-cycles",
268};
269
270static const char *__perf_evsel__hw_name(u64 config)
271{
272 if (config < PERF_COUNT_HW_MAX && perf_evsel__hw_names[config])
273 return perf_evsel__hw_names[config];
274
275 return "unknown-hardware";
276}
277
278static int perf_evsel__add_modifiers(struct perf_evsel *evsel, char *bf, size_t size)
279{
280 int colon = 0, r = 0;
281 struct perf_event_attr *attr = &evsel->attr;
282 bool exclude_guest_default = false;
283
284#define MOD_PRINT(context, mod) do { \
285 if (!attr->exclude_##context) { \
286 if (!colon) colon = ++r; \
287 r += scnprintf(bf + r, size - r, "%c", mod); \
288 } } while(0)
289
290 if (attr->exclude_kernel || attr->exclude_user || attr->exclude_hv) {
291 MOD_PRINT(kernel, 'k');
292 MOD_PRINT(user, 'u');
293 MOD_PRINT(hv, 'h');
294 exclude_guest_default = true;
295 }
296
297 if (attr->precise_ip) {
298 if (!colon)
299 colon = ++r;
300 r += scnprintf(bf + r, size - r, "%.*s", attr->precise_ip, "ppp");
301 exclude_guest_default = true;
302 }
303
304 if (attr->exclude_host || attr->exclude_guest == exclude_guest_default) {
305 MOD_PRINT(host, 'H');
306 MOD_PRINT(guest, 'G');
307 }
308#undef MOD_PRINT
309 if (colon)
310 bf[colon - 1] = ':';
311 return r;
312}
313
314static int perf_evsel__hw_name(struct perf_evsel *evsel, char *bf, size_t size)
315{
316 int r = scnprintf(bf, size, "%s", __perf_evsel__hw_name(evsel->attr.config));
317 return r + perf_evsel__add_modifiers(evsel, bf + r, size - r);
318}
319
320const char *perf_evsel__sw_names[PERF_COUNT_SW_MAX] = {
321 "cpu-clock",
322 "task-clock",
323 "page-faults",
324 "context-switches",
325 "cpu-migrations",
326 "minor-faults",
327 "major-faults",
328 "alignment-faults",
329 "emulation-faults",
330 "dummy",
331};
332
333static const char *__perf_evsel__sw_name(u64 config)
334{
335 if (config < PERF_COUNT_SW_MAX && perf_evsel__sw_names[config])
336 return perf_evsel__sw_names[config];
337 return "unknown-software";
338}
339
340static int perf_evsel__sw_name(struct perf_evsel *evsel, char *bf, size_t size)
341{
342 int r = scnprintf(bf, size, "%s", __perf_evsel__sw_name(evsel->attr.config));
343 return r + perf_evsel__add_modifiers(evsel, bf + r, size - r);
344}
345
346static int __perf_evsel__bp_name(char *bf, size_t size, u64 addr, u64 type)
347{
348 int r;
349
350 r = scnprintf(bf, size, "mem:0x%" PRIx64 ":", addr);
351
352 if (type & HW_BREAKPOINT_R)
353 r += scnprintf(bf + r, size - r, "r");
354
355 if (type & HW_BREAKPOINT_W)
356 r += scnprintf(bf + r, size - r, "w");
357
358 if (type & HW_BREAKPOINT_X)
359 r += scnprintf(bf + r, size - r, "x");
360
361 return r;
362}
363
364static int perf_evsel__bp_name(struct perf_evsel *evsel, char *bf, size_t size)
365{
366 struct perf_event_attr *attr = &evsel->attr;
367 int r = __perf_evsel__bp_name(bf, size, attr->bp_addr, attr->bp_type);
368 return r + perf_evsel__add_modifiers(evsel, bf + r, size - r);
369}
370
371const char *perf_evsel__hw_cache[PERF_COUNT_HW_CACHE_MAX]
372 [PERF_EVSEL__MAX_ALIASES] = {
373 { "L1-dcache", "l1-d", "l1d", "L1-data", },
374 { "L1-icache", "l1-i", "l1i", "L1-instruction", },
375 { "LLC", "L2", },
376 { "dTLB", "d-tlb", "Data-TLB", },
377 { "iTLB", "i-tlb", "Instruction-TLB", },
378 { "branch", "branches", "bpu", "btb", "bpc", },
379 { "node", },
380};
381
382const char *perf_evsel__hw_cache_op[PERF_COUNT_HW_CACHE_OP_MAX]
383 [PERF_EVSEL__MAX_ALIASES] = {
384 { "load", "loads", "read", },
385 { "store", "stores", "write", },
386 { "prefetch", "prefetches", "speculative-read", "speculative-load", },
387};
388
389const char *perf_evsel__hw_cache_result[PERF_COUNT_HW_CACHE_RESULT_MAX]
390 [PERF_EVSEL__MAX_ALIASES] = {
391 { "refs", "Reference", "ops", "access", },
392 { "misses", "miss", },
393};
394
395#define C(x) PERF_COUNT_HW_CACHE_##x
396#define CACHE_READ (1 << C(OP_READ))
397#define CACHE_WRITE (1 << C(OP_WRITE))
398#define CACHE_PREFETCH (1 << C(OP_PREFETCH))
399#define COP(x) (1 << x)
400
401
402
403
404
405
406static unsigned long perf_evsel__hw_cache_stat[C(MAX)] = {
407 [C(L1D)] = (CACHE_READ | CACHE_WRITE | CACHE_PREFETCH),
408 [C(L1I)] = (CACHE_READ | CACHE_PREFETCH),
409 [C(LL)] = (CACHE_READ | CACHE_WRITE | CACHE_PREFETCH),
410 [C(DTLB)] = (CACHE_READ | CACHE_WRITE | CACHE_PREFETCH),
411 [C(ITLB)] = (CACHE_READ),
412 [C(BPU)] = (CACHE_READ),
413 [C(NODE)] = (CACHE_READ | CACHE_WRITE | CACHE_PREFETCH),
414};
415
416bool perf_evsel__is_cache_op_valid(u8 type, u8 op)
417{
418 if (perf_evsel__hw_cache_stat[type] & COP(op))
419 return true;
420 else
421 return false;
422}
423
424int __perf_evsel__hw_cache_type_op_res_name(u8 type, u8 op, u8 result,
425 char *bf, size_t size)
426{
427 if (result) {
428 return scnprintf(bf, size, "%s-%s-%s", perf_evsel__hw_cache[type][0],
429 perf_evsel__hw_cache_op[op][0],
430 perf_evsel__hw_cache_result[result][0]);
431 }
432
433 return scnprintf(bf, size, "%s-%s", perf_evsel__hw_cache[type][0],
434 perf_evsel__hw_cache_op[op][1]);
435}
436
437static int __perf_evsel__hw_cache_name(u64 config, char *bf, size_t size)
438{
439 u8 op, result, type = (config >> 0) & 0xff;
440 const char *err = "unknown-ext-hardware-cache-type";
441
442 if (type > PERF_COUNT_HW_CACHE_MAX)
443 goto out_err;
444
445 op = (config >> 8) & 0xff;
446 err = "unknown-ext-hardware-cache-op";
447 if (op > PERF_COUNT_HW_CACHE_OP_MAX)
448 goto out_err;
449
450 result = (config >> 16) & 0xff;
451 err = "unknown-ext-hardware-cache-result";
452 if (result > PERF_COUNT_HW_CACHE_RESULT_MAX)
453 goto out_err;
454
455 err = "invalid-cache";
456 if (!perf_evsel__is_cache_op_valid(type, op))
457 goto out_err;
458
459 return __perf_evsel__hw_cache_type_op_res_name(type, op, result, bf, size);
460out_err:
461 return scnprintf(bf, size, "%s", err);
462}
463
464static int perf_evsel__hw_cache_name(struct perf_evsel *evsel, char *bf, size_t size)
465{
466 int ret = __perf_evsel__hw_cache_name(evsel->attr.config, bf, size);
467 return ret + perf_evsel__add_modifiers(evsel, bf + ret, size - ret);
468}
469
470static int perf_evsel__raw_name(struct perf_evsel *evsel, char *bf, size_t size)
471{
472 int ret = scnprintf(bf, size, "raw 0x%" PRIx64, evsel->attr.config);
473 return ret + perf_evsel__add_modifiers(evsel, bf + ret, size - ret);
474}
475
476const char *perf_evsel__name(struct perf_evsel *evsel)
477{
478 char bf[128];
479
480 if (evsel->name)
481 return evsel->name;
482
483 switch (evsel->attr.type) {
484 case PERF_TYPE_RAW:
485 perf_evsel__raw_name(evsel, bf, sizeof(bf));
486 break;
487
488 case PERF_TYPE_HARDWARE:
489 perf_evsel__hw_name(evsel, bf, sizeof(bf));
490 break;
491
492 case PERF_TYPE_HW_CACHE:
493 perf_evsel__hw_cache_name(evsel, bf, sizeof(bf));
494 break;
495
496 case PERF_TYPE_SOFTWARE:
497 perf_evsel__sw_name(evsel, bf, sizeof(bf));
498 break;
499
500 case PERF_TYPE_TRACEPOINT:
501 scnprintf(bf, sizeof(bf), "%s", "unknown tracepoint");
502 break;
503
504 case PERF_TYPE_BREAKPOINT:
505 perf_evsel__bp_name(evsel, bf, sizeof(bf));
506 break;
507
508 default:
509 scnprintf(bf, sizeof(bf), "unknown attr type: %d",
510 evsel->attr.type);
511 break;
512 }
513
514 evsel->name = strdup(bf);
515
516 return evsel->name ?: "unknown";
517}
518
519const char *perf_evsel__group_name(struct perf_evsel *evsel)
520{
521 return evsel->group_name ?: "anon group";
522}
523
524int perf_evsel__group_desc(struct perf_evsel *evsel, char *buf, size_t size)
525{
526 int ret;
527 struct perf_evsel *pos;
528 const char *group_name = perf_evsel__group_name(evsel);
529
530 ret = scnprintf(buf, size, "%s", group_name);
531
532 ret += scnprintf(buf + ret, size - ret, " { %s",
533 perf_evsel__name(evsel));
534
535 for_each_group_member(pos, evsel)
536 ret += scnprintf(buf + ret, size - ret, ", %s",
537 perf_evsel__name(pos));
538
539 ret += scnprintf(buf + ret, size - ret, " }");
540
541 return ret;
542}
543
544static void
545perf_evsel__config_callgraph(struct perf_evsel *evsel,
546 struct record_opts *opts)
547{
548 bool function = perf_evsel__is_function_event(evsel);
549 struct perf_event_attr *attr = &evsel->attr;
550
551 perf_evsel__set_sample_bit(evsel, CALLCHAIN);
552
553 if (callchain_param.record_mode == CALLCHAIN_LBR) {
554 if (!opts->branch_stack) {
555 if (attr->exclude_user) {
556 pr_warning("LBR callstack option is only available "
557 "to get user callchain information. "
558 "Falling back to framepointers.\n");
559 } else {
560 perf_evsel__set_sample_bit(evsel, BRANCH_STACK);
561 attr->branch_sample_type = PERF_SAMPLE_BRANCH_USER |
562 PERF_SAMPLE_BRANCH_CALL_STACK;
563 }
564 } else
565 pr_warning("Cannot use LBR callstack with branch stack. "
566 "Falling back to framepointers.\n");
567 }
568
569 if (callchain_param.record_mode == CALLCHAIN_DWARF) {
570 if (!function) {
571 perf_evsel__set_sample_bit(evsel, REGS_USER);
572 perf_evsel__set_sample_bit(evsel, STACK_USER);
573 attr->sample_regs_user = PERF_REGS_MASK;
574 attr->sample_stack_user = callchain_param.dump_size;
575 attr->exclude_callchain_user = 1;
576 } else {
577 pr_info("Cannot use DWARF unwind for function trace event,"
578 " falling back to framepointers.\n");
579 }
580 }
581
582 if (function) {
583 pr_info("Disabling user space callchains for function trace event.\n");
584 attr->exclude_callchain_user = 1;
585 }
586}
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616void perf_evsel__config(struct perf_evsel *evsel, struct record_opts *opts)
617{
618 struct perf_evsel *leader = evsel->leader;
619 struct perf_event_attr *attr = &evsel->attr;
620 int track = evsel->tracking;
621 bool per_cpu = opts->target.default_per_cpu && !opts->target.per_thread;
622
623 attr->sample_id_all = perf_missing_features.sample_id_all ? 0 : 1;
624 attr->inherit = !opts->no_inherit;
625
626 perf_evsel__set_sample_bit(evsel, IP);
627 perf_evsel__set_sample_bit(evsel, TID);
628
629 if (evsel->sample_read) {
630 perf_evsel__set_sample_bit(evsel, READ);
631
632
633
634
635
636 perf_evsel__set_sample_id(evsel, false);
637
638
639
640
641
642 if (leader->nr_members > 1) {
643 attr->read_format |= PERF_FORMAT_GROUP;
644 attr->inherit = 0;
645 }
646 }
647
648
649
650
651
652 if (!attr->sample_period || (opts->user_freq != UINT_MAX ||
653 opts->user_interval != ULLONG_MAX)) {
654 if (opts->freq) {
655 perf_evsel__set_sample_bit(evsel, PERIOD);
656 attr->freq = 1;
657 attr->sample_freq = opts->freq;
658 } else {
659 attr->sample_period = opts->default_interval;
660 }
661 }
662
663
664
665
666
667 if ((leader != evsel) && leader->sample_read) {
668 attr->sample_freq = 0;
669 attr->sample_period = 0;
670 }
671
672 if (opts->no_samples)
673 attr->sample_freq = 0;
674
675 if (opts->inherit_stat)
676 attr->inherit_stat = 1;
677
678 if (opts->sample_address) {
679 perf_evsel__set_sample_bit(evsel, ADDR);
680 attr->mmap_data = track;
681 }
682
683
684
685
686
687
688 if (perf_evsel__is_function_event(evsel))
689 evsel->attr.exclude_callchain_user = 1;
690
691 if (callchain_param.enabled && !evsel->no_aux_samples)
692 perf_evsel__config_callgraph(evsel, opts);
693
694 if (opts->sample_intr_regs) {
695 attr->sample_regs_intr = PERF_REGS_MASK;
696 perf_evsel__set_sample_bit(evsel, REGS_INTR);
697 }
698
699 if (target__has_cpu(&opts->target))
700 perf_evsel__set_sample_bit(evsel, CPU);
701
702 if (opts->period)
703 perf_evsel__set_sample_bit(evsel, PERIOD);
704
705
706
707
708 if (opts->sample_time &&
709 (!perf_missing_features.sample_id_all &&
710 (!opts->no_inherit || target__has_cpu(&opts->target) || per_cpu)))
711 perf_evsel__set_sample_bit(evsel, TIME);
712
713 if (opts->raw_samples && !evsel->no_aux_samples) {
714 perf_evsel__set_sample_bit(evsel, TIME);
715 perf_evsel__set_sample_bit(evsel, RAW);
716 perf_evsel__set_sample_bit(evsel, CPU);
717 }
718
719 if (opts->sample_address)
720 perf_evsel__set_sample_bit(evsel, DATA_SRC);
721
722 if (opts->no_buffering) {
723 attr->watermark = 0;
724 attr->wakeup_events = 1;
725 }
726 if (opts->branch_stack && !evsel->no_aux_samples) {
727 perf_evsel__set_sample_bit(evsel, BRANCH_STACK);
728 attr->branch_sample_type = opts->branch_stack;
729 }
730
731 if (opts->sample_weight)
732 perf_evsel__set_sample_bit(evsel, WEIGHT);
733
734 attr->task = track;
735 attr->mmap = track;
736 attr->mmap2 = track && !perf_missing_features.mmap2;
737 attr->comm = track;
738
739 if (opts->sample_transaction)
740 perf_evsel__set_sample_bit(evsel, TRANSACTION);
741
742 if (opts->running_time) {
743 evsel->attr.read_format |=
744 PERF_FORMAT_TOTAL_TIME_ENABLED |
745 PERF_FORMAT_TOTAL_TIME_RUNNING;
746 }
747
748
749
750
751
752
753
754 if (perf_evsel__is_group_leader(evsel))
755 attr->disabled = 1;
756
757
758
759
760
761 if (target__none(&opts->target) && perf_evsel__is_group_leader(evsel) &&
762 !opts->initial_delay)
763 attr->enable_on_exec = 1;
764
765 if (evsel->immediate) {
766 attr->disabled = 0;
767 attr->enable_on_exec = 0;
768 }
769
770 clockid = opts->clockid;
771 if (opts->use_clockid) {
772 attr->use_clockid = 1;
773 attr->clockid = opts->clockid;
774 }
775}
776
777static int perf_evsel__alloc_fd(struct perf_evsel *evsel, int ncpus, int nthreads)
778{
779 int cpu, thread;
780
781 if (evsel->system_wide)
782 nthreads = 1;
783
784 evsel->fd = xyarray__new(ncpus, nthreads, sizeof(int));
785
786 if (evsel->fd) {
787 for (cpu = 0; cpu < ncpus; cpu++) {
788 for (thread = 0; thread < nthreads; thread++) {
789 FD(evsel, cpu, thread) = -1;
790 }
791 }
792 }
793
794 return evsel->fd != NULL ? 0 : -ENOMEM;
795}
796
797static int perf_evsel__run_ioctl(struct perf_evsel *evsel, int ncpus, int nthreads,
798 int ioc, void *arg)
799{
800 int cpu, thread;
801
802 if (evsel->system_wide)
803 nthreads = 1;
804
805 for (cpu = 0; cpu < ncpus; cpu++) {
806 for (thread = 0; thread < nthreads; thread++) {
807 int fd = FD(evsel, cpu, thread),
808 err = ioctl(fd, ioc, arg);
809
810 if (err)
811 return err;
812 }
813 }
814
815 return 0;
816}
817
818int perf_evsel__set_filter(struct perf_evsel *evsel, int ncpus, int nthreads,
819 const char *filter)
820{
821 return perf_evsel__run_ioctl(evsel, ncpus, nthreads,
822 PERF_EVENT_IOC_SET_FILTER,
823 (void *)filter);
824}
825
826int perf_evsel__enable(struct perf_evsel *evsel, int ncpus, int nthreads)
827{
828 return perf_evsel__run_ioctl(evsel, ncpus, nthreads,
829 PERF_EVENT_IOC_ENABLE,
830 0);
831}
832
833int perf_evsel__alloc_id(struct perf_evsel *evsel, int ncpus, int nthreads)
834{
835 if (ncpus == 0 || nthreads == 0)
836 return 0;
837
838 if (evsel->system_wide)
839 nthreads = 1;
840
841 evsel->sample_id = xyarray__new(ncpus, nthreads, sizeof(struct perf_sample_id));
842 if (evsel->sample_id == NULL)
843 return -ENOMEM;
844
845 evsel->id = zalloc(ncpus * nthreads * sizeof(u64));
846 if (evsel->id == NULL) {
847 xyarray__delete(evsel->sample_id);
848 evsel->sample_id = NULL;
849 return -ENOMEM;
850 }
851
852 return 0;
853}
854
855static void perf_evsel__free_fd(struct perf_evsel *evsel)
856{
857 xyarray__delete(evsel->fd);
858 evsel->fd = NULL;
859}
860
861static void perf_evsel__free_id(struct perf_evsel *evsel)
862{
863 xyarray__delete(evsel->sample_id);
864 evsel->sample_id = NULL;
865 zfree(&evsel->id);
866}
867
868void perf_evsel__close_fd(struct perf_evsel *evsel, int ncpus, int nthreads)
869{
870 int cpu, thread;
871
872 if (evsel->system_wide)
873 nthreads = 1;
874
875 for (cpu = 0; cpu < ncpus; cpu++)
876 for (thread = 0; thread < nthreads; ++thread) {
877 close(FD(evsel, cpu, thread));
878 FD(evsel, cpu, thread) = -1;
879 }
880}
881
882void perf_evsel__exit(struct perf_evsel *evsel)
883{
884 assert(list_empty(&evsel->node));
885 perf_evsel__free_fd(evsel);
886 perf_evsel__free_id(evsel);
887 close_cgroup(evsel->cgrp);
888 cpu_map__put(evsel->cpus);
889 thread_map__put(evsel->threads);
890 zfree(&evsel->group_name);
891 zfree(&evsel->name);
892 perf_evsel__object.fini(evsel);
893}
894
895void perf_evsel__delete(struct perf_evsel *evsel)
896{
897 perf_evsel__exit(evsel);
898 free(evsel);
899}
900
901void perf_evsel__compute_deltas(struct perf_evsel *evsel, int cpu, int thread,
902 struct perf_counts_values *count)
903{
904 struct perf_counts_values tmp;
905
906 if (!evsel->prev_raw_counts)
907 return;
908
909 if (cpu == -1) {
910 tmp = evsel->prev_raw_counts->aggr;
911 evsel->prev_raw_counts->aggr = *count;
912 } else {
913 tmp = *perf_counts(evsel->prev_raw_counts, cpu, thread);
914 *perf_counts(evsel->prev_raw_counts, cpu, thread) = *count;
915 }
916
917 count->val = count->val - tmp.val;
918 count->ena = count->ena - tmp.ena;
919 count->run = count->run - tmp.run;
920}
921
922void perf_counts_values__scale(struct perf_counts_values *count,
923 bool scale, s8 *pscaled)
924{
925 s8 scaled = 0;
926
927 if (scale) {
928 if (count->run == 0) {
929 scaled = -1;
930 count->val = 0;
931 } else if (count->run < count->ena) {
932 scaled = 1;
933 count->val = (u64)((double) count->val * count->ena / count->run + 0.5);
934 }
935 } else
936 count->ena = count->run = 0;
937
938 if (pscaled)
939 *pscaled = scaled;
940}
941
942int perf_evsel__read(struct perf_evsel *evsel, int cpu, int thread,
943 struct perf_counts_values *count)
944{
945 memset(count, 0, sizeof(*count));
946
947 if (FD(evsel, cpu, thread) < 0)
948 return -EINVAL;
949
950 if (readn(FD(evsel, cpu, thread), count, sizeof(*count)) < 0)
951 return -errno;
952
953 return 0;
954}
955
956int __perf_evsel__read_on_cpu(struct perf_evsel *evsel,
957 int cpu, int thread, bool scale)
958{
959 struct perf_counts_values count;
960 size_t nv = scale ? 3 : 1;
961
962 if (FD(evsel, cpu, thread) < 0)
963 return -EINVAL;
964
965 if (evsel->counts == NULL && perf_evsel__alloc_counts(evsel, cpu + 1, thread + 1) < 0)
966 return -ENOMEM;
967
968 if (readn(FD(evsel, cpu, thread), &count, nv * sizeof(u64)) < 0)
969 return -errno;
970
971 perf_evsel__compute_deltas(evsel, cpu, thread, &count);
972 perf_counts_values__scale(&count, scale, NULL);
973 *perf_counts(evsel->counts, cpu, thread) = count;
974 return 0;
975}
976
977static int get_group_fd(struct perf_evsel *evsel, int cpu, int thread)
978{
979 struct perf_evsel *leader = evsel->leader;
980 int fd;
981
982 if (perf_evsel__is_group_leader(evsel))
983 return -1;
984
985
986
987
988
989 BUG_ON(!leader->fd);
990
991 fd = FD(leader, cpu, thread);
992 BUG_ON(fd == -1);
993
994 return fd;
995}
996
997struct bit_names {
998 int bit;
999 const char *name;
1000};
1001
1002static void __p_bits(char *buf, size_t size, u64 value, struct bit_names *bits)
1003{
1004 bool first_bit = true;
1005 int i = 0;
1006
1007 do {
1008 if (value & bits[i].bit) {
1009 buf += scnprintf(buf, size, "%s%s", first_bit ? "" : "|", bits[i].name);
1010 first_bit = false;
1011 }
1012 } while (bits[++i].name != NULL);
1013}
1014
1015static void __p_sample_type(char *buf, size_t size, u64 value)
1016{
1017#define bit_name(n) { PERF_SAMPLE_##n, #n }
1018 struct bit_names bits[] = {
1019 bit_name(IP), bit_name(TID), bit_name(TIME), bit_name(ADDR),
1020 bit_name(READ), bit_name(CALLCHAIN), bit_name(ID), bit_name(CPU),
1021 bit_name(PERIOD), bit_name(STREAM_ID), bit_name(RAW),
1022 bit_name(BRANCH_STACK), bit_name(REGS_USER), bit_name(STACK_USER),
1023 bit_name(IDENTIFIER), bit_name(REGS_INTR),
1024 { .name = NULL, }
1025 };
1026#undef bit_name
1027 __p_bits(buf, size, value, bits);
1028}
1029
1030static void __p_read_format(char *buf, size_t size, u64 value)
1031{
1032#define bit_name(n) { PERF_FORMAT_##n, #n }
1033 struct bit_names bits[] = {
1034 bit_name(TOTAL_TIME_ENABLED), bit_name(TOTAL_TIME_RUNNING),
1035 bit_name(ID), bit_name(GROUP),
1036 { .name = NULL, }
1037 };
1038#undef bit_name
1039 __p_bits(buf, size, value, bits);
1040}
1041
1042#define BUF_SIZE 1024
1043
1044#define p_hex(val) snprintf(buf, BUF_SIZE, "%#"PRIx64, (uint64_t)(val))
1045#define p_unsigned(val) snprintf(buf, BUF_SIZE, "%"PRIu64, (uint64_t)(val))
1046#define p_signed(val) snprintf(buf, BUF_SIZE, "%"PRId64, (int64_t)(val))
1047#define p_sample_type(val) __p_sample_type(buf, BUF_SIZE, val)
1048#define p_read_format(val) __p_read_format(buf, BUF_SIZE, val)
1049
1050#define PRINT_ATTRn(_n, _f, _p) \
1051do { \
1052 if (attr->_f) { \
1053 _p(attr->_f); \
1054 ret += attr__fprintf(fp, _n, buf, priv);\
1055 } \
1056} while (0)
1057
1058#define PRINT_ATTRf(_f, _p) PRINT_ATTRn(#_f, _f, _p)
1059
1060int perf_event_attr__fprintf(FILE *fp, struct perf_event_attr *attr,
1061 attr__fprintf_f attr__fprintf, void *priv)
1062{
1063 char buf[BUF_SIZE];
1064 int ret = 0;
1065
1066 PRINT_ATTRf(type, p_unsigned);
1067 PRINT_ATTRf(size, p_unsigned);
1068 PRINT_ATTRf(config, p_hex);
1069 PRINT_ATTRn("{ sample_period, sample_freq }", sample_period, p_unsigned);
1070 PRINT_ATTRf(sample_type, p_sample_type);
1071 PRINT_ATTRf(read_format, p_read_format);
1072
1073 PRINT_ATTRf(disabled, p_unsigned);
1074 PRINT_ATTRf(inherit, p_unsigned);
1075 PRINT_ATTRf(pinned, p_unsigned);
1076 PRINT_ATTRf(exclusive, p_unsigned);
1077 PRINT_ATTRf(exclude_user, p_unsigned);
1078 PRINT_ATTRf(exclude_kernel, p_unsigned);
1079 PRINT_ATTRf(exclude_hv, p_unsigned);
1080 PRINT_ATTRf(exclude_idle, p_unsigned);
1081 PRINT_ATTRf(mmap, p_unsigned);
1082 PRINT_ATTRf(comm, p_unsigned);
1083 PRINT_ATTRf(freq, p_unsigned);
1084 PRINT_ATTRf(inherit_stat, p_unsigned);
1085 PRINT_ATTRf(enable_on_exec, p_unsigned);
1086 PRINT_ATTRf(task, p_unsigned);
1087 PRINT_ATTRf(watermark, p_unsigned);
1088 PRINT_ATTRf(precise_ip, p_unsigned);
1089 PRINT_ATTRf(mmap_data, p_unsigned);
1090 PRINT_ATTRf(sample_id_all, p_unsigned);
1091 PRINT_ATTRf(exclude_host, p_unsigned);
1092 PRINT_ATTRf(exclude_guest, p_unsigned);
1093 PRINT_ATTRf(exclude_callchain_kernel, p_unsigned);
1094 PRINT_ATTRf(exclude_callchain_user, p_unsigned);
1095 PRINT_ATTRf(mmap2, p_unsigned);
1096 PRINT_ATTRf(comm_exec, p_unsigned);
1097 PRINT_ATTRf(use_clockid, p_unsigned);
1098
1099 PRINT_ATTRn("{ wakeup_events, wakeup_watermark }", wakeup_events, p_unsigned);
1100 PRINT_ATTRf(bp_type, p_unsigned);
1101 PRINT_ATTRn("{ bp_addr, config1 }", bp_addr, p_hex);
1102 PRINT_ATTRn("{ bp_len, config2 }", bp_len, p_hex);
1103 PRINT_ATTRf(sample_regs_user, p_hex);
1104 PRINT_ATTRf(sample_stack_user, p_unsigned);
1105 PRINT_ATTRf(clockid, p_signed);
1106 PRINT_ATTRf(sample_regs_intr, p_hex);
1107 PRINT_ATTRf(aux_watermark, p_unsigned);
1108
1109 return ret;
1110}
1111
1112static int __open_attr__fprintf(FILE *fp, const char *name, const char *val,
1113 void *priv __attribute__((unused)))
1114{
1115 return fprintf(fp, " %-32s %s\n", name, val);
1116}
1117
1118static int __perf_evsel__open(struct perf_evsel *evsel, struct cpu_map *cpus,
1119 struct thread_map *threads)
1120{
1121 int cpu, thread, nthreads;
1122 unsigned long flags = PERF_FLAG_FD_CLOEXEC;
1123 int pid = -1, err;
1124 enum { NO_CHANGE, SET_TO_MAX, INCREASED_MAX } set_rlimit = NO_CHANGE;
1125
1126 if (evsel->system_wide)
1127 nthreads = 1;
1128 else
1129 nthreads = threads->nr;
1130
1131 if (evsel->fd == NULL &&
1132 perf_evsel__alloc_fd(evsel, cpus->nr, nthreads) < 0)
1133 return -ENOMEM;
1134
1135 if (evsel->cgrp) {
1136 flags |= PERF_FLAG_PID_CGROUP;
1137 pid = evsel->cgrp->fd;
1138 }
1139
1140fallback_missing_features:
1141 if (perf_missing_features.clockid_wrong)
1142 evsel->attr.clockid = CLOCK_MONOTONIC;
1143 if (perf_missing_features.clockid) {
1144 evsel->attr.use_clockid = 0;
1145 evsel->attr.clockid = 0;
1146 }
1147 if (perf_missing_features.cloexec)
1148 flags &= ~(unsigned long)PERF_FLAG_FD_CLOEXEC;
1149 if (perf_missing_features.mmap2)
1150 evsel->attr.mmap2 = 0;
1151 if (perf_missing_features.exclude_guest)
1152 evsel->attr.exclude_guest = evsel->attr.exclude_host = 0;
1153retry_sample_id:
1154 if (perf_missing_features.sample_id_all)
1155 evsel->attr.sample_id_all = 0;
1156
1157 if (verbose >= 2) {
1158 fprintf(stderr, "%.60s\n", graph_dotted_line);
1159 fprintf(stderr, "perf_event_attr:\n");
1160 perf_event_attr__fprintf(stderr, &evsel->attr, __open_attr__fprintf, NULL);
1161 fprintf(stderr, "%.60s\n", graph_dotted_line);
1162 }
1163
1164 for (cpu = 0; cpu < cpus->nr; cpu++) {
1165
1166 for (thread = 0; thread < nthreads; thread++) {
1167 int group_fd;
1168
1169 if (!evsel->cgrp && !evsel->system_wide)
1170 pid = thread_map__pid(threads, thread);
1171
1172 group_fd = get_group_fd(evsel, cpu, thread);
1173retry_open:
1174 pr_debug2("sys_perf_event_open: pid %d cpu %d group_fd %d flags %#lx\n",
1175 pid, cpus->map[cpu], group_fd, flags);
1176
1177 FD(evsel, cpu, thread) = sys_perf_event_open(&evsel->attr,
1178 pid,
1179 cpus->map[cpu],
1180 group_fd, flags);
1181 if (FD(evsel, cpu, thread) < 0) {
1182 err = -errno;
1183 pr_debug2("sys_perf_event_open failed, error %d\n",
1184 err);
1185 goto try_fallback;
1186 }
1187 set_rlimit = NO_CHANGE;
1188
1189
1190
1191
1192
1193
1194 if (perf_missing_features.clockid ||
1195 perf_missing_features.clockid_wrong) {
1196 err = -EINVAL;
1197 goto out_close;
1198 }
1199 }
1200 }
1201
1202 return 0;
1203
1204try_fallback:
1205
1206
1207
1208
1209 if (err == -EMFILE && set_rlimit < INCREASED_MAX) {
1210 struct rlimit l;
1211 int old_errno = errno;
1212
1213 if (getrlimit(RLIMIT_NOFILE, &l) == 0) {
1214 if (set_rlimit == NO_CHANGE)
1215 l.rlim_cur = l.rlim_max;
1216 else {
1217 l.rlim_cur = l.rlim_max + 1000;
1218 l.rlim_max = l.rlim_cur;
1219 }
1220 if (setrlimit(RLIMIT_NOFILE, &l) == 0) {
1221 set_rlimit++;
1222 errno = old_errno;
1223 goto retry_open;
1224 }
1225 }
1226 errno = old_errno;
1227 }
1228
1229 if (err != -EINVAL || cpu > 0 || thread > 0)
1230 goto out_close;
1231
1232
1233
1234
1235
1236 if (!perf_missing_features.clockid_wrong && evsel->attr.use_clockid) {
1237 perf_missing_features.clockid_wrong = true;
1238 goto fallback_missing_features;
1239 } else if (!perf_missing_features.clockid && evsel->attr.use_clockid) {
1240 perf_missing_features.clockid = true;
1241 goto fallback_missing_features;
1242 } else if (!perf_missing_features.cloexec && (flags & PERF_FLAG_FD_CLOEXEC)) {
1243 perf_missing_features.cloexec = true;
1244 goto fallback_missing_features;
1245 } else if (!perf_missing_features.mmap2 && evsel->attr.mmap2) {
1246 perf_missing_features.mmap2 = true;
1247 goto fallback_missing_features;
1248 } else if (!perf_missing_features.exclude_guest &&
1249 (evsel->attr.exclude_guest || evsel->attr.exclude_host)) {
1250 perf_missing_features.exclude_guest = true;
1251 goto fallback_missing_features;
1252 } else if (!perf_missing_features.sample_id_all) {
1253 perf_missing_features.sample_id_all = true;
1254 goto retry_sample_id;
1255 }
1256
1257out_close:
1258 do {
1259 while (--thread >= 0) {
1260 close(FD(evsel, cpu, thread));
1261 FD(evsel, cpu, thread) = -1;
1262 }
1263 thread = nthreads;
1264 } while (--cpu >= 0);
1265 return err;
1266}
1267
1268void perf_evsel__close(struct perf_evsel *evsel, int ncpus, int nthreads)
1269{
1270 if (evsel->fd == NULL)
1271 return;
1272
1273 perf_evsel__close_fd(evsel, ncpus, nthreads);
1274 perf_evsel__free_fd(evsel);
1275}
1276
1277static struct {
1278 struct cpu_map map;
1279 int cpus[1];
1280} empty_cpu_map = {
1281 .map.nr = 1,
1282 .cpus = { -1, },
1283};
1284
1285static struct {
1286 struct thread_map map;
1287 int threads[1];
1288} empty_thread_map = {
1289 .map.nr = 1,
1290 .threads = { -1, },
1291};
1292
1293int perf_evsel__open(struct perf_evsel *evsel, struct cpu_map *cpus,
1294 struct thread_map *threads)
1295{
1296 if (cpus == NULL) {
1297
1298 cpus = &empty_cpu_map.map;
1299 }
1300
1301 if (threads == NULL)
1302 threads = &empty_thread_map.map;
1303
1304 return __perf_evsel__open(evsel, cpus, threads);
1305}
1306
1307int perf_evsel__open_per_cpu(struct perf_evsel *evsel,
1308 struct cpu_map *cpus)
1309{
1310 return __perf_evsel__open(evsel, cpus, &empty_thread_map.map);
1311}
1312
1313int perf_evsel__open_per_thread(struct perf_evsel *evsel,
1314 struct thread_map *threads)
1315{
1316 return __perf_evsel__open(evsel, &empty_cpu_map.map, threads);
1317}
1318
1319static int perf_evsel__parse_id_sample(const struct perf_evsel *evsel,
1320 const union perf_event *event,
1321 struct perf_sample *sample)
1322{
1323 u64 type = evsel->attr.sample_type;
1324 const u64 *array = event->sample.array;
1325 bool swapped = evsel->needs_swap;
1326 union u64_swap u;
1327
1328 array += ((event->header.size -
1329 sizeof(event->header)) / sizeof(u64)) - 1;
1330
1331 if (type & PERF_SAMPLE_IDENTIFIER) {
1332 sample->id = *array;
1333 array--;
1334 }
1335
1336 if (type & PERF_SAMPLE_CPU) {
1337 u.val64 = *array;
1338 if (swapped) {
1339
1340 u.val64 = bswap_64(u.val64);
1341 u.val32[0] = bswap_32(u.val32[0]);
1342 }
1343
1344 sample->cpu = u.val32[0];
1345 array--;
1346 }
1347
1348 if (type & PERF_SAMPLE_STREAM_ID) {
1349 sample->stream_id = *array;
1350 array--;
1351 }
1352
1353 if (type & PERF_SAMPLE_ID) {
1354 sample->id = *array;
1355 array--;
1356 }
1357
1358 if (type & PERF_SAMPLE_TIME) {
1359 sample->time = *array;
1360 array--;
1361 }
1362
1363 if (type & PERF_SAMPLE_TID) {
1364 u.val64 = *array;
1365 if (swapped) {
1366
1367 u.val64 = bswap_64(u.val64);
1368 u.val32[0] = bswap_32(u.val32[0]);
1369 u.val32[1] = bswap_32(u.val32[1]);
1370 }
1371
1372 sample->pid = u.val32[0];
1373 sample->tid = u.val32[1];
1374 array--;
1375 }
1376
1377 return 0;
1378}
1379
1380static inline bool overflow(const void *endp, u16 max_size, const void *offset,
1381 u64 size)
1382{
1383 return size > max_size || offset + size > endp;
1384}
1385
1386#define OVERFLOW_CHECK(offset, size, max_size) \
1387 do { \
1388 if (overflow(endp, (max_size), (offset), (size))) \
1389 return -EFAULT; \
1390 } while (0)
1391
1392#define OVERFLOW_CHECK_u64(offset) \
1393 OVERFLOW_CHECK(offset, sizeof(u64), sizeof(u64))
1394
1395int perf_evsel__parse_sample(struct perf_evsel *evsel, union perf_event *event,
1396 struct perf_sample *data)
1397{
1398 u64 type = evsel->attr.sample_type;
1399 bool swapped = evsel->needs_swap;
1400 const u64 *array;
1401 u16 max_size = event->header.size;
1402 const void *endp = (void *)event + max_size;
1403 u64 sz;
1404
1405
1406
1407
1408
1409 union u64_swap u;
1410
1411 memset(data, 0, sizeof(*data));
1412 data->cpu = data->pid = data->tid = -1;
1413 data->stream_id = data->id = data->time = -1ULL;
1414 data->period = evsel->attr.sample_period;
1415 data->weight = 0;
1416
1417 if (event->header.type != PERF_RECORD_SAMPLE) {
1418 if (!evsel->attr.sample_id_all)
1419 return 0;
1420 return perf_evsel__parse_id_sample(evsel, event, data);
1421 }
1422
1423 array = event->sample.array;
1424
1425
1426
1427
1428
1429
1430 if (evsel->sample_size + sizeof(event->header) > event->header.size)
1431 return -EFAULT;
1432
1433 data->id = -1ULL;
1434 if (type & PERF_SAMPLE_IDENTIFIER) {
1435 data->id = *array;
1436 array++;
1437 }
1438
1439 if (type & PERF_SAMPLE_IP) {
1440 data->ip = *array;
1441 array++;
1442 }
1443
1444 if (type & PERF_SAMPLE_TID) {
1445 u.val64 = *array;
1446 if (swapped) {
1447
1448 u.val64 = bswap_64(u.val64);
1449 u.val32[0] = bswap_32(u.val32[0]);
1450 u.val32[1] = bswap_32(u.val32[1]);
1451 }
1452
1453 data->pid = u.val32[0];
1454 data->tid = u.val32[1];
1455 array++;
1456 }
1457
1458 if (type & PERF_SAMPLE_TIME) {
1459 data->time = *array;
1460 array++;
1461 }
1462
1463 data->addr = 0;
1464 if (type & PERF_SAMPLE_ADDR) {
1465 data->addr = *array;
1466 array++;
1467 }
1468
1469 if (type & PERF_SAMPLE_ID) {
1470 data->id = *array;
1471 array++;
1472 }
1473
1474 if (type & PERF_SAMPLE_STREAM_ID) {
1475 data->stream_id = *array;
1476 array++;
1477 }
1478
1479 if (type & PERF_SAMPLE_CPU) {
1480
1481 u.val64 = *array;
1482 if (swapped) {
1483
1484 u.val64 = bswap_64(u.val64);
1485 u.val32[0] = bswap_32(u.val32[0]);
1486 }
1487
1488 data->cpu = u.val32[0];
1489 array++;
1490 }
1491
1492 if (type & PERF_SAMPLE_PERIOD) {
1493 data->period = *array;
1494 array++;
1495 }
1496
1497 if (type & PERF_SAMPLE_READ) {
1498 u64 read_format = evsel->attr.read_format;
1499
1500 OVERFLOW_CHECK_u64(array);
1501 if (read_format & PERF_FORMAT_GROUP)
1502 data->read.group.nr = *array;
1503 else
1504 data->read.one.value = *array;
1505
1506 array++;
1507
1508 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
1509 OVERFLOW_CHECK_u64(array);
1510 data->read.time_enabled = *array;
1511 array++;
1512 }
1513
1514 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
1515 OVERFLOW_CHECK_u64(array);
1516 data->read.time_running = *array;
1517 array++;
1518 }
1519
1520
1521 if (read_format & PERF_FORMAT_GROUP) {
1522 const u64 max_group_nr = UINT64_MAX /
1523 sizeof(struct sample_read_value);
1524
1525 if (data->read.group.nr > max_group_nr)
1526 return -EFAULT;
1527 sz = data->read.group.nr *
1528 sizeof(struct sample_read_value);
1529 OVERFLOW_CHECK(array, sz, max_size);
1530 data->read.group.values =
1531 (struct sample_read_value *)array;
1532 array = (void *)array + sz;
1533 } else {
1534 OVERFLOW_CHECK_u64(array);
1535 data->read.one.id = *array;
1536 array++;
1537 }
1538 }
1539
1540 if (type & PERF_SAMPLE_CALLCHAIN) {
1541 const u64 max_callchain_nr = UINT64_MAX / sizeof(u64);
1542
1543 OVERFLOW_CHECK_u64(array);
1544 data->callchain = (struct ip_callchain *)array++;
1545 if (data->callchain->nr > max_callchain_nr)
1546 return -EFAULT;
1547 sz = data->callchain->nr * sizeof(u64);
1548 OVERFLOW_CHECK(array, sz, max_size);
1549 array = (void *)array + sz;
1550 }
1551
1552 if (type & PERF_SAMPLE_RAW) {
1553 OVERFLOW_CHECK_u64(array);
1554 u.val64 = *array;
1555 if (WARN_ONCE(swapped,
1556 "Endianness of raw data not corrected!\n")) {
1557
1558 u.val64 = bswap_64(u.val64);
1559 u.val32[0] = bswap_32(u.val32[0]);
1560 u.val32[1] = bswap_32(u.val32[1]);
1561 }
1562 data->raw_size = u.val32[0];
1563 array = (void *)array + sizeof(u32);
1564
1565 OVERFLOW_CHECK(array, data->raw_size, max_size);
1566 data->raw_data = (void *)array;
1567 array = (void *)array + data->raw_size;
1568 }
1569
1570 if (type & PERF_SAMPLE_BRANCH_STACK) {
1571 const u64 max_branch_nr = UINT64_MAX /
1572 sizeof(struct branch_entry);
1573
1574 OVERFLOW_CHECK_u64(array);
1575 data->branch_stack = (struct branch_stack *)array++;
1576
1577 if (data->branch_stack->nr > max_branch_nr)
1578 return -EFAULT;
1579 sz = data->branch_stack->nr * sizeof(struct branch_entry);
1580 OVERFLOW_CHECK(array, sz, max_size);
1581 array = (void *)array + sz;
1582 }
1583
1584 if (type & PERF_SAMPLE_REGS_USER) {
1585 OVERFLOW_CHECK_u64(array);
1586 data->user_regs.abi = *array;
1587 array++;
1588
1589 if (data->user_regs.abi) {
1590 u64 mask = evsel->attr.sample_regs_user;
1591
1592 sz = hweight_long(mask) * sizeof(u64);
1593 OVERFLOW_CHECK(array, sz, max_size);
1594 data->user_regs.mask = mask;
1595 data->user_regs.regs = (u64 *)array;
1596 array = (void *)array + sz;
1597 }
1598 }
1599
1600 if (type & PERF_SAMPLE_STACK_USER) {
1601 OVERFLOW_CHECK_u64(array);
1602 sz = *array++;
1603
1604 data->user_stack.offset = ((char *)(array - 1)
1605 - (char *) event);
1606
1607 if (!sz) {
1608 data->user_stack.size = 0;
1609 } else {
1610 OVERFLOW_CHECK(array, sz, max_size);
1611 data->user_stack.data = (char *)array;
1612 array = (void *)array + sz;
1613 OVERFLOW_CHECK_u64(array);
1614 data->user_stack.size = *array++;
1615 if (WARN_ONCE(data->user_stack.size > sz,
1616 "user stack dump failure\n"))
1617 return -EFAULT;
1618 }
1619 }
1620
1621 data->weight = 0;
1622 if (type & PERF_SAMPLE_WEIGHT) {
1623 OVERFLOW_CHECK_u64(array);
1624 data->weight = *array;
1625 array++;
1626 }
1627
1628 data->data_src = PERF_MEM_DATA_SRC_NONE;
1629 if (type & PERF_SAMPLE_DATA_SRC) {
1630 OVERFLOW_CHECK_u64(array);
1631 data->data_src = *array;
1632 array++;
1633 }
1634
1635 data->transaction = 0;
1636 if (type & PERF_SAMPLE_TRANSACTION) {
1637 OVERFLOW_CHECK_u64(array);
1638 data->transaction = *array;
1639 array++;
1640 }
1641
1642 data->intr_regs.abi = PERF_SAMPLE_REGS_ABI_NONE;
1643 if (type & PERF_SAMPLE_REGS_INTR) {
1644 OVERFLOW_CHECK_u64(array);
1645 data->intr_regs.abi = *array;
1646 array++;
1647
1648 if (data->intr_regs.abi != PERF_SAMPLE_REGS_ABI_NONE) {
1649 u64 mask = evsel->attr.sample_regs_intr;
1650
1651 sz = hweight_long(mask) * sizeof(u64);
1652 OVERFLOW_CHECK(array, sz, max_size);
1653 data->intr_regs.mask = mask;
1654 data->intr_regs.regs = (u64 *)array;
1655 array = (void *)array + sz;
1656 }
1657 }
1658
1659 return 0;
1660}
1661
1662size_t perf_event__sample_event_size(const struct perf_sample *sample, u64 type,
1663 u64 read_format)
1664{
1665 size_t sz, result = sizeof(struct sample_event);
1666
1667 if (type & PERF_SAMPLE_IDENTIFIER)
1668 result += sizeof(u64);
1669
1670 if (type & PERF_SAMPLE_IP)
1671 result += sizeof(u64);
1672
1673 if (type & PERF_SAMPLE_TID)
1674 result += sizeof(u64);
1675
1676 if (type & PERF_SAMPLE_TIME)
1677 result += sizeof(u64);
1678
1679 if (type & PERF_SAMPLE_ADDR)
1680 result += sizeof(u64);
1681
1682 if (type & PERF_SAMPLE_ID)
1683 result += sizeof(u64);
1684
1685 if (type & PERF_SAMPLE_STREAM_ID)
1686 result += sizeof(u64);
1687
1688 if (type & PERF_SAMPLE_CPU)
1689 result += sizeof(u64);
1690
1691 if (type & PERF_SAMPLE_PERIOD)
1692 result += sizeof(u64);
1693
1694 if (type & PERF_SAMPLE_READ) {
1695 result += sizeof(u64);
1696 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
1697 result += sizeof(u64);
1698 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
1699 result += sizeof(u64);
1700
1701 if (read_format & PERF_FORMAT_GROUP) {
1702 sz = sample->read.group.nr *
1703 sizeof(struct sample_read_value);
1704 result += sz;
1705 } else {
1706 result += sizeof(u64);
1707 }
1708 }
1709
1710 if (type & PERF_SAMPLE_CALLCHAIN) {
1711 sz = (sample->callchain->nr + 1) * sizeof(u64);
1712 result += sz;
1713 }
1714
1715 if (type & PERF_SAMPLE_RAW) {
1716 result += sizeof(u32);
1717 result += sample->raw_size;
1718 }
1719
1720 if (type & PERF_SAMPLE_BRANCH_STACK) {
1721 sz = sample->branch_stack->nr * sizeof(struct branch_entry);
1722 sz += sizeof(u64);
1723 result += sz;
1724 }
1725
1726 if (type & PERF_SAMPLE_REGS_USER) {
1727 if (sample->user_regs.abi) {
1728 result += sizeof(u64);
1729 sz = hweight_long(sample->user_regs.mask) * sizeof(u64);
1730 result += sz;
1731 } else {
1732 result += sizeof(u64);
1733 }
1734 }
1735
1736 if (type & PERF_SAMPLE_STACK_USER) {
1737 sz = sample->user_stack.size;
1738 result += sizeof(u64);
1739 if (sz) {
1740 result += sz;
1741 result += sizeof(u64);
1742 }
1743 }
1744
1745 if (type & PERF_SAMPLE_WEIGHT)
1746 result += sizeof(u64);
1747
1748 if (type & PERF_SAMPLE_DATA_SRC)
1749 result += sizeof(u64);
1750
1751 if (type & PERF_SAMPLE_TRANSACTION)
1752 result += sizeof(u64);
1753
1754 if (type & PERF_SAMPLE_REGS_INTR) {
1755 if (sample->intr_regs.abi) {
1756 result += sizeof(u64);
1757 sz = hweight_long(sample->intr_regs.mask) * sizeof(u64);
1758 result += sz;
1759 } else {
1760 result += sizeof(u64);
1761 }
1762 }
1763
1764 return result;
1765}
1766
1767int perf_event__synthesize_sample(union perf_event *event, u64 type,
1768 u64 read_format,
1769 const struct perf_sample *sample,
1770 bool swapped)
1771{
1772 u64 *array;
1773 size_t sz;
1774
1775
1776
1777
1778 union u64_swap u;
1779
1780 array = event->sample.array;
1781
1782 if (type & PERF_SAMPLE_IDENTIFIER) {
1783 *array = sample->id;
1784 array++;
1785 }
1786
1787 if (type & PERF_SAMPLE_IP) {
1788 *array = sample->ip;
1789 array++;
1790 }
1791
1792 if (type & PERF_SAMPLE_TID) {
1793 u.val32[0] = sample->pid;
1794 u.val32[1] = sample->tid;
1795 if (swapped) {
1796
1797
1798
1799 u.val32[0] = bswap_32(u.val32[0]);
1800 u.val32[1] = bswap_32(u.val32[1]);
1801 u.val64 = bswap_64(u.val64);
1802 }
1803
1804 *array = u.val64;
1805 array++;
1806 }
1807
1808 if (type & PERF_SAMPLE_TIME) {
1809 *array = sample->time;
1810 array++;
1811 }
1812
1813 if (type & PERF_SAMPLE_ADDR) {
1814 *array = sample->addr;
1815 array++;
1816 }
1817
1818 if (type & PERF_SAMPLE_ID) {
1819 *array = sample->id;
1820 array++;
1821 }
1822
1823 if (type & PERF_SAMPLE_STREAM_ID) {
1824 *array = sample->stream_id;
1825 array++;
1826 }
1827
1828 if (type & PERF_SAMPLE_CPU) {
1829 u.val32[0] = sample->cpu;
1830 if (swapped) {
1831
1832
1833
1834 u.val32[0] = bswap_32(u.val32[0]);
1835 u.val64 = bswap_64(u.val64);
1836 }
1837 *array = u.val64;
1838 array++;
1839 }
1840
1841 if (type & PERF_SAMPLE_PERIOD) {
1842 *array = sample->period;
1843 array++;
1844 }
1845
1846 if (type & PERF_SAMPLE_READ) {
1847 if (read_format & PERF_FORMAT_GROUP)
1848 *array = sample->read.group.nr;
1849 else
1850 *array = sample->read.one.value;
1851 array++;
1852
1853 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
1854 *array = sample->read.time_enabled;
1855 array++;
1856 }
1857
1858 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
1859 *array = sample->read.time_running;
1860 array++;
1861 }
1862
1863
1864 if (read_format & PERF_FORMAT_GROUP) {
1865 sz = sample->read.group.nr *
1866 sizeof(struct sample_read_value);
1867 memcpy(array, sample->read.group.values, sz);
1868 array = (void *)array + sz;
1869 } else {
1870 *array = sample->read.one.id;
1871 array++;
1872 }
1873 }
1874
1875 if (type & PERF_SAMPLE_CALLCHAIN) {
1876 sz = (sample->callchain->nr + 1) * sizeof(u64);
1877 memcpy(array, sample->callchain, sz);
1878 array = (void *)array + sz;
1879 }
1880
1881 if (type & PERF_SAMPLE_RAW) {
1882 u.val32[0] = sample->raw_size;
1883 if (WARN_ONCE(swapped,
1884 "Endianness of raw data not corrected!\n")) {
1885
1886
1887
1888 u.val32[0] = bswap_32(u.val32[0]);
1889 u.val32[1] = bswap_32(u.val32[1]);
1890 u.val64 = bswap_64(u.val64);
1891 }
1892 *array = u.val64;
1893 array = (void *)array + sizeof(u32);
1894
1895 memcpy(array, sample->raw_data, sample->raw_size);
1896 array = (void *)array + sample->raw_size;
1897 }
1898
1899 if (type & PERF_SAMPLE_BRANCH_STACK) {
1900 sz = sample->branch_stack->nr * sizeof(struct branch_entry);
1901 sz += sizeof(u64);
1902 memcpy(array, sample->branch_stack, sz);
1903 array = (void *)array + sz;
1904 }
1905
1906 if (type & PERF_SAMPLE_REGS_USER) {
1907 if (sample->user_regs.abi) {
1908 *array++ = sample->user_regs.abi;
1909 sz = hweight_long(sample->user_regs.mask) * sizeof(u64);
1910 memcpy(array, sample->user_regs.regs, sz);
1911 array = (void *)array + sz;
1912 } else {
1913 *array++ = 0;
1914 }
1915 }
1916
1917 if (type & PERF_SAMPLE_STACK_USER) {
1918 sz = sample->user_stack.size;
1919 *array++ = sz;
1920 if (sz) {
1921 memcpy(array, sample->user_stack.data, sz);
1922 array = (void *)array + sz;
1923 *array++ = sz;
1924 }
1925 }
1926
1927 if (type & PERF_SAMPLE_WEIGHT) {
1928 *array = sample->weight;
1929 array++;
1930 }
1931
1932 if (type & PERF_SAMPLE_DATA_SRC) {
1933 *array = sample->data_src;
1934 array++;
1935 }
1936
1937 if (type & PERF_SAMPLE_TRANSACTION) {
1938 *array = sample->transaction;
1939 array++;
1940 }
1941
1942 if (type & PERF_SAMPLE_REGS_INTR) {
1943 if (sample->intr_regs.abi) {
1944 *array++ = sample->intr_regs.abi;
1945 sz = hweight_long(sample->intr_regs.mask) * sizeof(u64);
1946 memcpy(array, sample->intr_regs.regs, sz);
1947 array = (void *)array + sz;
1948 } else {
1949 *array++ = 0;
1950 }
1951 }
1952
1953 return 0;
1954}
1955
1956struct format_field *perf_evsel__field(struct perf_evsel *evsel, const char *name)
1957{
1958 return pevent_find_field(evsel->tp_format, name);
1959}
1960
1961void *perf_evsel__rawptr(struct perf_evsel *evsel, struct perf_sample *sample,
1962 const char *name)
1963{
1964 struct format_field *field = perf_evsel__field(evsel, name);
1965 int offset;
1966
1967 if (!field)
1968 return NULL;
1969
1970 offset = field->offset;
1971
1972 if (field->flags & FIELD_IS_DYNAMIC) {
1973 offset = *(int *)(sample->raw_data + field->offset);
1974 offset &= 0xffff;
1975 }
1976
1977 return sample->raw_data + offset;
1978}
1979
1980u64 perf_evsel__intval(struct perf_evsel *evsel, struct perf_sample *sample,
1981 const char *name)
1982{
1983 struct format_field *field = perf_evsel__field(evsel, name);
1984 void *ptr;
1985 u64 value;
1986
1987 if (!field)
1988 return 0;
1989
1990 ptr = sample->raw_data + field->offset;
1991
1992 switch (field->size) {
1993 case 1:
1994 return *(u8 *)ptr;
1995 case 2:
1996 value = *(u16 *)ptr;
1997 break;
1998 case 4:
1999 value = *(u32 *)ptr;
2000 break;
2001 case 8:
2002 memcpy(&value, ptr, sizeof(u64));
2003 break;
2004 default:
2005 return 0;
2006 }
2007
2008 if (!evsel->needs_swap)
2009 return value;
2010
2011 switch (field->size) {
2012 case 2:
2013 return bswap_16(value);
2014 case 4:
2015 return bswap_32(value);
2016 case 8:
2017 return bswap_64(value);
2018 default:
2019 return 0;
2020 }
2021
2022 return 0;
2023}
2024
2025static int comma_fprintf(FILE *fp, bool *first, const char *fmt, ...)
2026{
2027 va_list args;
2028 int ret = 0;
2029
2030 if (!*first) {
2031 ret += fprintf(fp, ",");
2032 } else {
2033 ret += fprintf(fp, ":");
2034 *first = false;
2035 }
2036
2037 va_start(args, fmt);
2038 ret += vfprintf(fp, fmt, args);
2039 va_end(args);
2040 return ret;
2041}
2042
2043static int __print_attr__fprintf(FILE *fp, const char *name, const char *val, void *priv)
2044{
2045 return comma_fprintf(fp, (bool *)priv, " %s: %s", name, val);
2046}
2047
2048int perf_evsel__fprintf(struct perf_evsel *evsel,
2049 struct perf_attr_details *details, FILE *fp)
2050{
2051 bool first = true;
2052 int printed = 0;
2053
2054 if (details->event_group) {
2055 struct perf_evsel *pos;
2056
2057 if (!perf_evsel__is_group_leader(evsel))
2058 return 0;
2059
2060 if (evsel->nr_members > 1)
2061 printed += fprintf(fp, "%s{", evsel->group_name ?: "");
2062
2063 printed += fprintf(fp, "%s", perf_evsel__name(evsel));
2064 for_each_group_member(pos, evsel)
2065 printed += fprintf(fp, ",%s", perf_evsel__name(pos));
2066
2067 if (evsel->nr_members > 1)
2068 printed += fprintf(fp, "}");
2069 goto out;
2070 }
2071
2072 printed += fprintf(fp, "%s", perf_evsel__name(evsel));
2073
2074 if (details->verbose) {
2075 printed += perf_event_attr__fprintf(fp, &evsel->attr,
2076 __print_attr__fprintf, &first);
2077 } else if (details->freq) {
2078 printed += comma_fprintf(fp, &first, " sample_freq=%" PRIu64,
2079 (u64)evsel->attr.sample_freq);
2080 }
2081out:
2082 fputc('\n', fp);
2083 return ++printed;
2084}
2085
2086bool perf_evsel__fallback(struct perf_evsel *evsel, int err,
2087 char *msg, size_t msgsize)
2088{
2089 if ((err == ENOENT || err == ENXIO || err == ENODEV) &&
2090 evsel->attr.type == PERF_TYPE_HARDWARE &&
2091 evsel->attr.config == PERF_COUNT_HW_CPU_CYCLES) {
2092
2093
2094
2095
2096
2097
2098
2099
2100 scnprintf(msg, msgsize, "%s",
2101"The cycles event is not supported, trying to fall back to cpu-clock-ticks");
2102
2103 evsel->attr.type = PERF_TYPE_SOFTWARE;
2104 evsel->attr.config = PERF_COUNT_SW_CPU_CLOCK;
2105
2106 zfree(&evsel->name);
2107 return true;
2108 }
2109
2110 return false;
2111}
2112
2113int perf_evsel__open_strerror(struct perf_evsel *evsel, struct target *target,
2114 int err, char *msg, size_t size)
2115{
2116 char sbuf[STRERR_BUFSIZE];
2117
2118 switch (err) {
2119 case EPERM:
2120 case EACCES:
2121 return scnprintf(msg, size,
2122 "You may not have permission to collect %sstats.\n"
2123 "Consider tweaking /proc/sys/kernel/perf_event_paranoid:\n"
2124 " -1 - Not paranoid at all\n"
2125 " 0 - Disallow raw tracepoint access for unpriv\n"
2126 " 1 - Disallow cpu events for unpriv\n"
2127 " 2 - Disallow kernel profiling for unpriv",
2128 target->system_wide ? "system-wide " : "");
2129 case ENOENT:
2130 return scnprintf(msg, size, "The %s event is not supported.",
2131 perf_evsel__name(evsel));
2132 case EMFILE:
2133 return scnprintf(msg, size, "%s",
2134 "Too many events are opened.\n"
2135 "Probably the maximum number of open file descriptors has been reached.\n"
2136 "Hint: Try again after reducing the number of events.\n"
2137 "Hint: Try increasing the limit with 'ulimit -n <limit>'");
2138 case ENODEV:
2139 if (target->cpu_list)
2140 return scnprintf(msg, size, "%s",
2141 "No such device - did you specify an out-of-range profile CPU?\n");
2142 break;
2143 case EOPNOTSUPP:
2144 if (evsel->attr.precise_ip)
2145 return scnprintf(msg, size, "%s",
2146 "\'precise\' request may not be supported. Try removing 'p' modifier.");
2147#if defined(__i386__) || defined(__x86_64__)
2148 if (evsel->attr.type == PERF_TYPE_HARDWARE)
2149 return scnprintf(msg, size, "%s",
2150 "No hardware sampling interrupt available.\n"
2151 "No APIC? If so then you can boot the kernel with the \"lapic\" boot parameter to force-enable it.");
2152#endif
2153 break;
2154 case EBUSY:
2155 if (find_process("oprofiled"))
2156 return scnprintf(msg, size,
2157 "The PMU counters are busy/taken by another profiler.\n"
2158 "We found oprofile daemon running, please stop it and try again.");
2159 break;
2160 case EINVAL:
2161 if (perf_missing_features.clockid)
2162 return scnprintf(msg, size, "clockid feature not supported.");
2163 if (perf_missing_features.clockid_wrong)
2164 return scnprintf(msg, size, "wrong clockid (%d).", clockid);
2165 break;
2166 default:
2167 break;
2168 }
2169
2170 return scnprintf(msg, size,
2171 "The sys_perf_event_open() syscall returned with %d (%s) for event (%s).\n"
2172 "/bin/dmesg may provide additional information.\n"
2173 "No CONFIG_PERF_EVENTS=y kernel support configured?\n",
2174 err, strerror_r(err, sbuf, sizeof(sbuf)),
2175 perf_evsel__name(evsel));
2176}
2177
