1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16#include <linux/kernel.h>
17#include <linux/types.h>
18#include <linux/bitops.h>
19#include <linux/log2.h>
20
21#include "../../util/cpumap.h"
22#include "../../util/evsel.h"
23#include "../../util/evlist.h"
24#include "../../util/session.h"
25#include "../../util/util.h"
26#include "../../util/pmu.h"
27#include "../../util/debug.h"
28#include "../../util/tsc.h"
29#include "../../util/auxtrace.h"
30#include "../../util/intel-bts.h"
31
32#define KiB(x) ((x) * 1024)
33#define MiB(x) ((x) * 1024 * 1024)
34#define KiB_MASK(x) (KiB(x) - 1)
35#define MiB_MASK(x) (MiB(x) - 1)
36
37#define INTEL_BTS_DFLT_SAMPLE_SIZE KiB(4)
38
39#define INTEL_BTS_MAX_SAMPLE_SIZE KiB(60)
40
41struct intel_bts_snapshot_ref {
42 void *ref_buf;
43 size_t ref_offset;
44 bool wrapped;
45};
46
47struct intel_bts_recording {
48 struct auxtrace_record itr;
49 struct perf_pmu *intel_bts_pmu;
50 struct perf_evlist *evlist;
51 bool snapshot_mode;
52 size_t snapshot_size;
53 int snapshot_ref_cnt;
54 struct intel_bts_snapshot_ref *snapshot_refs;
55};
56
57struct branch {
58 u64 from;
59 u64 to;
60 u64 misc;
61};
62
63static size_t
64intel_bts_info_priv_size(struct auxtrace_record *itr __maybe_unused,
65 struct perf_evlist *evlist __maybe_unused)
66{
67 return INTEL_BTS_AUXTRACE_PRIV_SIZE;
68}
69
70static int intel_bts_info_fill(struct auxtrace_record *itr,
71 struct perf_session *session,
72 struct auxtrace_info_event *auxtrace_info,
73 size_t priv_size)
74{
75 struct intel_bts_recording *btsr =
76 container_of(itr, struct intel_bts_recording, itr);
77 struct perf_pmu *intel_bts_pmu = btsr->intel_bts_pmu;
78 struct perf_event_mmap_page *pc;
79 struct perf_tsc_conversion tc = { .time_mult = 0, };
80 bool cap_user_time_zero = false;
81 int err;
82
83 if (priv_size != INTEL_BTS_AUXTRACE_PRIV_SIZE)
84 return -EINVAL;
85
86 if (!session->evlist->nr_mmaps)
87 return -EINVAL;
88
89 pc = session->evlist->mmap[0].base;
90 if (pc) {
91 err = perf_read_tsc_conversion(pc, &tc);
92 if (err) {
93 if (err != -EOPNOTSUPP)
94 return err;
95 } else {
96 cap_user_time_zero = tc.time_mult != 0;
97 }
98 if (!cap_user_time_zero)
99 ui__warning("Intel BTS: TSC not available\n");
100 }
101
102 auxtrace_info->type = PERF_AUXTRACE_INTEL_BTS;
103 auxtrace_info->priv[INTEL_BTS_PMU_TYPE] = intel_bts_pmu->type;
104 auxtrace_info->priv[INTEL_BTS_TIME_SHIFT] = tc.time_shift;
105 auxtrace_info->priv[INTEL_BTS_TIME_MULT] = tc.time_mult;
106 auxtrace_info->priv[INTEL_BTS_TIME_ZERO] = tc.time_zero;
107 auxtrace_info->priv[INTEL_BTS_CAP_USER_TIME_ZERO] = cap_user_time_zero;
108 auxtrace_info->priv[INTEL_BTS_SNAPSHOT_MODE] = btsr->snapshot_mode;
109
110 return 0;
111}
112
113static int intel_bts_recording_options(struct auxtrace_record *itr,
114 struct perf_evlist *evlist,
115 struct record_opts *opts)
116{
117 struct intel_bts_recording *btsr =
118 container_of(itr, struct intel_bts_recording, itr);
119 struct perf_pmu *intel_bts_pmu = btsr->intel_bts_pmu;
120 struct perf_evsel *evsel, *intel_bts_evsel = NULL;
121 const struct cpu_map *cpus = evlist->cpus;
122 bool privileged = geteuid() == 0 || perf_event_paranoid() < 0;
123
124 btsr->evlist = evlist;
125 btsr->snapshot_mode = opts->auxtrace_snapshot_mode;
126
127 evlist__for_each(evlist, evsel) {
128 if (evsel->attr.type == intel_bts_pmu->type) {
129 if (intel_bts_evsel) {
130 pr_err("There may be only one " INTEL_BTS_PMU_NAME " event\n");
131 return -EINVAL;
132 }
133 evsel->attr.freq = 0;
134 evsel->attr.sample_period = 1;
135 intel_bts_evsel = evsel;
136 opts->full_auxtrace = true;
137 }
138 }
139
140 if (opts->auxtrace_snapshot_mode && !opts->full_auxtrace) {
141 pr_err("Snapshot mode (-S option) requires " INTEL_BTS_PMU_NAME " PMU event (-e " INTEL_BTS_PMU_NAME ")\n");
142 return -EINVAL;
143 }
144
145 if (!opts->full_auxtrace)
146 return 0;
147
148 if (opts->full_auxtrace && !cpu_map__empty(cpus)) {
149 pr_err(INTEL_BTS_PMU_NAME " does not support per-cpu recording\n");
150 return -EINVAL;
151 }
152
153
154 if (opts->auxtrace_snapshot_mode) {
155 if (!opts->auxtrace_snapshot_size && !opts->auxtrace_mmap_pages) {
156 if (privileged) {
157 opts->auxtrace_mmap_pages = MiB(4) / page_size;
158 } else {
159 opts->auxtrace_mmap_pages = KiB(128) / page_size;
160 if (opts->mmap_pages == UINT_MAX)
161 opts->mmap_pages = KiB(256) / page_size;
162 }
163 } else if (!opts->auxtrace_mmap_pages && !privileged &&
164 opts->mmap_pages == UINT_MAX) {
165 opts->mmap_pages = KiB(256) / page_size;
166 }
167 if (!opts->auxtrace_snapshot_size)
168 opts->auxtrace_snapshot_size =
169 opts->auxtrace_mmap_pages * (size_t)page_size;
170 if (!opts->auxtrace_mmap_pages) {
171 size_t sz = opts->auxtrace_snapshot_size;
172
173 sz = round_up(sz, page_size) / page_size;
174 opts->auxtrace_mmap_pages = roundup_pow_of_two(sz);
175 }
176 if (opts->auxtrace_snapshot_size >
177 opts->auxtrace_mmap_pages * (size_t)page_size) {
178 pr_err("Snapshot size %zu must not be greater than AUX area tracing mmap size %zu\n",
179 opts->auxtrace_snapshot_size,
180 opts->auxtrace_mmap_pages * (size_t)page_size);
181 return -EINVAL;
182 }
183 if (!opts->auxtrace_snapshot_size || !opts->auxtrace_mmap_pages) {
184 pr_err("Failed to calculate default snapshot size and/or AUX area tracing mmap pages\n");
185 return -EINVAL;
186 }
187 pr_debug2("Intel BTS snapshot size: %zu\n",
188 opts->auxtrace_snapshot_size);
189 }
190
191
192 if (opts->full_auxtrace && !opts->auxtrace_mmap_pages) {
193 if (privileged) {
194 opts->auxtrace_mmap_pages = MiB(4) / page_size;
195 } else {
196 opts->auxtrace_mmap_pages = KiB(128) / page_size;
197 if (opts->mmap_pages == UINT_MAX)
198 opts->mmap_pages = KiB(256) / page_size;
199 }
200 }
201
202
203 if (opts->auxtrace_mmap_pages) {
204 size_t sz = opts->auxtrace_mmap_pages * (size_t)page_size;
205 size_t min_sz;
206
207 if (opts->auxtrace_snapshot_mode)
208 min_sz = KiB(4);
209 else
210 min_sz = KiB(8);
211
212 if (sz < min_sz || !is_power_of_2(sz)) {
213 pr_err("Invalid mmap size for Intel BTS: must be at least %zuKiB and a power of 2\n",
214 min_sz / 1024);
215 return -EINVAL;
216 }
217 }
218
219 if (intel_bts_evsel) {
220
221
222
223
224 perf_evlist__to_front(evlist, intel_bts_evsel);
225
226
227
228
229 if (!cpu_map__empty(cpus))
230 perf_evsel__set_sample_bit(intel_bts_evsel, CPU);
231 }
232
233
234 if (opts->full_auxtrace) {
235 struct perf_evsel *tracking_evsel;
236 int err;
237
238 err = parse_events(evlist, "dummy:u", NULL);
239 if (err)
240 return err;
241
242 tracking_evsel = perf_evlist__last(evlist);
243
244 perf_evlist__set_tracking_event(evlist, tracking_evsel);
245
246 tracking_evsel->attr.freq = 0;
247 tracking_evsel->attr.sample_period = 1;
248 }
249
250 return 0;
251}
252
253static int intel_bts_parse_snapshot_options(struct auxtrace_record *itr,
254 struct record_opts *opts,
255 const char *str)
256{
257 struct intel_bts_recording *btsr =
258 container_of(itr, struct intel_bts_recording, itr);
259 unsigned long long snapshot_size = 0;
260 char *endptr;
261
262 if (str) {
263 snapshot_size = strtoull(str, &endptr, 0);
264 if (*endptr || snapshot_size > SIZE_MAX)
265 return -1;
266 }
267
268 opts->auxtrace_snapshot_mode = true;
269 opts->auxtrace_snapshot_size = snapshot_size;
270
271 btsr->snapshot_size = snapshot_size;
272
273 return 0;
274}
275
276static u64 intel_bts_reference(struct auxtrace_record *itr __maybe_unused)
277{
278 return rdtsc();
279}
280
281static int intel_bts_alloc_snapshot_refs(struct intel_bts_recording *btsr,
282 int idx)
283{
284 const size_t sz = sizeof(struct intel_bts_snapshot_ref);
285 int cnt = btsr->snapshot_ref_cnt, new_cnt = cnt * 2;
286 struct intel_bts_snapshot_ref *refs;
287
288 if (!new_cnt)
289 new_cnt = 16;
290
291 while (new_cnt <= idx)
292 new_cnt *= 2;
293
294 refs = calloc(new_cnt, sz);
295 if (!refs)
296 return -ENOMEM;
297
298 memcpy(refs, btsr->snapshot_refs, cnt * sz);
299
300 btsr->snapshot_refs = refs;
301 btsr->snapshot_ref_cnt = new_cnt;
302
303 return 0;
304}
305
306static void intel_bts_free_snapshot_refs(struct intel_bts_recording *btsr)
307{
308 int i;
309
310 for (i = 0; i < btsr->snapshot_ref_cnt; i++)
311 zfree(&btsr->snapshot_refs[i].ref_buf);
312 zfree(&btsr->snapshot_refs);
313}
314
315static void intel_bts_recording_free(struct auxtrace_record *itr)
316{
317 struct intel_bts_recording *btsr =
318 container_of(itr, struct intel_bts_recording, itr);
319
320 intel_bts_free_snapshot_refs(btsr);
321 free(btsr);
322}
323
324static int intel_bts_snapshot_start(struct auxtrace_record *itr)
325{
326 struct intel_bts_recording *btsr =
327 container_of(itr, struct intel_bts_recording, itr);
328 struct perf_evsel *evsel;
329
330 evlist__for_each(btsr->evlist, evsel) {
331 if (evsel->attr.type == btsr->intel_bts_pmu->type)
332 return perf_evsel__disable(evsel);
333 }
334 return -EINVAL;
335}
336
337static int intel_bts_snapshot_finish(struct auxtrace_record *itr)
338{
339 struct intel_bts_recording *btsr =
340 container_of(itr, struct intel_bts_recording, itr);
341 struct perf_evsel *evsel;
342
343 evlist__for_each(btsr->evlist, evsel) {
344 if (evsel->attr.type == btsr->intel_bts_pmu->type)
345 return perf_evsel__enable(evsel);
346 }
347 return -EINVAL;
348}
349
350static bool intel_bts_first_wrap(u64 *data, size_t buf_size)
351{
352 int i, a, b;
353
354 b = buf_size >> 3;
355 a = b - 512;
356 if (a < 0)
357 a = 0;
358
359 for (i = a; i < b; i++) {
360 if (data[i])
361 return true;
362 }
363
364 return false;
365}
366
367static int intel_bts_find_snapshot(struct auxtrace_record *itr, int idx,
368 struct auxtrace_mmap *mm, unsigned char *data,
369 u64 *head, u64 *old)
370{
371 struct intel_bts_recording *btsr =
372 container_of(itr, struct intel_bts_recording, itr);
373 bool wrapped;
374 int err;
375
376 pr_debug3("%s: mmap index %d old head %zu new head %zu\n",
377 __func__, idx, (size_t)*old, (size_t)*head);
378
379 if (idx >= btsr->snapshot_ref_cnt) {
380 err = intel_bts_alloc_snapshot_refs(btsr, idx);
381 if (err)
382 goto out_err;
383 }
384
385 wrapped = btsr->snapshot_refs[idx].wrapped;
386 if (!wrapped && intel_bts_first_wrap((u64 *)data, mm->len)) {
387 btsr->snapshot_refs[idx].wrapped = true;
388 wrapped = true;
389 }
390
391
392
393
394
395
396
397 if (wrapped) {
398 *old = *head;
399 *head += mm->len;
400 } else {
401 if (mm->mask)
402 *old &= mm->mask;
403 else
404 *old %= mm->len;
405 if (*old > *head)
406 *head += mm->len;
407 }
408
409 pr_debug3("%s: wrap-around %sdetected, adjusted old head %zu adjusted new head %zu\n",
410 __func__, wrapped ? "" : "not ", (size_t)*old, (size_t)*head);
411
412 return 0;
413
414out_err:
415 pr_err("%s: failed, error %d\n", __func__, err);
416 return err;
417}
418
419static int intel_bts_read_finish(struct auxtrace_record *itr, int idx)
420{
421 struct intel_bts_recording *btsr =
422 container_of(itr, struct intel_bts_recording, itr);
423 struct perf_evsel *evsel;
424
425 evlist__for_each(btsr->evlist, evsel) {
426 if (evsel->attr.type == btsr->intel_bts_pmu->type)
427 return perf_evlist__enable_event_idx(btsr->evlist,
428 evsel, idx);
429 }
430 return -EINVAL;
431}
432
433struct auxtrace_record *intel_bts_recording_init(int *err)
434{
435 struct perf_pmu *intel_bts_pmu = perf_pmu__find(INTEL_BTS_PMU_NAME);
436 struct intel_bts_recording *btsr;
437
438 if (!intel_bts_pmu)
439 return NULL;
440
441 btsr = zalloc(sizeof(struct intel_bts_recording));
442 if (!btsr) {
443 *err = -ENOMEM;
444 return NULL;
445 }
446
447 btsr->intel_bts_pmu = intel_bts_pmu;
448 btsr->itr.recording_options = intel_bts_recording_options;
449 btsr->itr.info_priv_size = intel_bts_info_priv_size;
450 btsr->itr.info_fill = intel_bts_info_fill;
451 btsr->itr.free = intel_bts_recording_free;
452 btsr->itr.snapshot_start = intel_bts_snapshot_start;
453 btsr->itr.snapshot_finish = intel_bts_snapshot_finish;
454 btsr->itr.find_snapshot = intel_bts_find_snapshot;
455 btsr->itr.parse_snapshot_options = intel_bts_parse_snapshot_options;
456 btsr->itr.reference = intel_bts_reference;
457 btsr->itr.read_finish = intel_bts_read_finish;
458 btsr->itr.alignment = sizeof(struct branch);
459 return &btsr->itr;
460}
461
