1
2#include <linux/bitops.h>
3#include <linux/types.h>
4#include <linux/slab.h>
5
6#include <asm/cpu_entry_area.h>
7#include <asm/perf_event.h>
8#include <asm/tlbflush.h>
9#include <asm/insn.h>
10
11#include "../perf_event.h"
12
13
14DEFINE_PER_CPU_PAGE_ALIGNED(struct debug_store, cpu_debug_store);
15
16
17#define BTS_RECORD_SIZE 24
18
19#define PEBS_FIXUP_SIZE PAGE_SIZE
20
21
22
23
24
25
26
27
28
29
30
31
32union intel_x86_pebs_dse {
33 u64 val;
34 struct {
35 unsigned int ld_dse:4;
36 unsigned int ld_stlb_miss:1;
37 unsigned int ld_locked:1;
38 unsigned int ld_reserved:26;
39 };
40 struct {
41 unsigned int st_l1d_hit:1;
42 unsigned int st_reserved1:3;
43 unsigned int st_stlb_miss:1;
44 unsigned int st_locked:1;
45 unsigned int st_reserved2:26;
46 };
47};
48
49
50
51
52
53
54#define P(a, b) PERF_MEM_S(a, b)
55#define OP_LH (P(OP, LOAD) | P(LVL, HIT))
56#define LEVEL(x) P(LVLNUM, x)
57#define REM P(REMOTE, REMOTE)
58#define SNOOP_NONE_MISS (P(SNOOP, NONE) | P(SNOOP, MISS))
59
60
61static u64 pebs_data_source[] = {
62 P(OP, LOAD) | P(LVL, MISS) | LEVEL(L3) | P(SNOOP, NA),
63 OP_LH | P(LVL, L1) | LEVEL(L1) | P(SNOOP, NONE),
64 OP_LH | P(LVL, LFB) | LEVEL(LFB) | P(SNOOP, NONE),
65 OP_LH | P(LVL, L2) | LEVEL(L2) | P(SNOOP, NONE),
66 OP_LH | P(LVL, L3) | LEVEL(L3) | P(SNOOP, NONE),
67 OP_LH | P(LVL, L3) | LEVEL(L3) | P(SNOOP, MISS),
68 OP_LH | P(LVL, L3) | LEVEL(L3) | P(SNOOP, HIT),
69 OP_LH | P(LVL, L3) | LEVEL(L3) | P(SNOOP, HITM),
70 OP_LH | P(LVL, REM_CCE1) | REM | LEVEL(L3) | P(SNOOP, HIT),
71 OP_LH | P(LVL, REM_CCE1) | REM | LEVEL(L3) | P(SNOOP, HITM),
72 OP_LH | P(LVL, LOC_RAM) | LEVEL(RAM) | P(SNOOP, HIT),
73 OP_LH | P(LVL, REM_RAM1) | REM | LEVEL(L3) | P(SNOOP, HIT),
74 OP_LH | P(LVL, LOC_RAM) | LEVEL(RAM) | SNOOP_NONE_MISS,
75 OP_LH | P(LVL, REM_RAM1) | LEVEL(RAM) | REM | SNOOP_NONE_MISS,
76 OP_LH | P(LVL, IO) | LEVEL(NA) | P(SNOOP, NONE),
77 OP_LH | P(LVL, UNC) | LEVEL(NA) | P(SNOOP, NONE),
78};
79
80
81void __init intel_pmu_pebs_data_source_nhm(void)
82{
83 pebs_data_source[0x05] = OP_LH | P(LVL, L3) | LEVEL(L3) | P(SNOOP, HIT);
84 pebs_data_source[0x06] = OP_LH | P(LVL, L3) | LEVEL(L3) | P(SNOOP, HITM);
85 pebs_data_source[0x07] = OP_LH | P(LVL, L3) | LEVEL(L3) | P(SNOOP, HITM);
86}
87
88void __init intel_pmu_pebs_data_source_skl(bool pmem)
89{
90 u64 pmem_or_l4 = pmem ? LEVEL(PMEM) : LEVEL(L4);
91
92 pebs_data_source[0x08] = OP_LH | pmem_or_l4 | P(SNOOP, HIT);
93 pebs_data_source[0x09] = OP_LH | pmem_or_l4 | REM | P(SNOOP, HIT);
94 pebs_data_source[0x0b] = OP_LH | LEVEL(RAM) | REM | P(SNOOP, NONE);
95 pebs_data_source[0x0c] = OP_LH | LEVEL(ANY_CACHE) | REM | P(SNOOPX, FWD);
96 pebs_data_source[0x0d] = OP_LH | LEVEL(ANY_CACHE) | REM | P(SNOOP, HITM);
97}
98
99static u64 precise_store_data(u64 status)
100{
101 union intel_x86_pebs_dse dse;
102 u64 val = P(OP, STORE) | P(SNOOP, NA) | P(LVL, L1) | P(TLB, L2);
103
104 dse.val = status;
105
106
107
108
109
110
111
112
113 if (dse.st_stlb_miss)
114 val |= P(TLB, MISS);
115 else
116 val |= P(TLB, HIT);
117
118
119
120
121
122
123 if (dse.st_l1d_hit)
124 val |= P(LVL, HIT);
125 else
126 val |= P(LVL, MISS);
127
128
129
130
131 if (dse.st_locked)
132 val |= P(LOCK, LOCKED);
133
134 return val;
135}
136
137static u64 precise_datala_hsw(struct perf_event *event, u64 status)
138{
139 union perf_mem_data_src dse;
140
141 dse.val = PERF_MEM_NA;
142
143 if (event->hw.flags & PERF_X86_EVENT_PEBS_ST_HSW)
144 dse.mem_op = PERF_MEM_OP_STORE;
145 else if (event->hw.flags & PERF_X86_EVENT_PEBS_LD_HSW)
146 dse.mem_op = PERF_MEM_OP_LOAD;
147
148
149
150
151
152
153
154
155
156 if (event->hw.flags & PERF_X86_EVENT_PEBS_ST_HSW) {
157 if (status & 1)
158 dse.mem_lvl = PERF_MEM_LVL_L1 | PERF_MEM_LVL_HIT;
159 else
160 dse.mem_lvl = PERF_MEM_LVL_L1 | PERF_MEM_LVL_MISS;
161 }
162 return dse.val;
163}
164
165static u64 load_latency_data(u64 status)
166{
167 union intel_x86_pebs_dse dse;
168 u64 val;
169
170 dse.val = status;
171
172
173
174
175 val = pebs_data_source[dse.ld_dse];
176
177
178
179
180 if (x86_pmu.pebs_no_tlb) {
181 val |= P(TLB, NA) | P(LOCK, NA);
182 return val;
183 }
184
185
186
187
188
189 if (dse.ld_stlb_miss)
190 val |= P(TLB, MISS) | P(TLB, L2);
191 else
192 val |= P(TLB, HIT) | P(TLB, L1) | P(TLB, L2);
193
194
195
196
197 if (dse.ld_locked)
198 val |= P(LOCK, LOCKED);
199
200 return val;
201}
202
203struct pebs_record_core {
204 u64 flags, ip;
205 u64 ax, bx, cx, dx;
206 u64 si, di, bp, sp;
207 u64 r8, r9, r10, r11;
208 u64 r12, r13, r14, r15;
209};
210
211struct pebs_record_nhm {
212 u64 flags, ip;
213 u64 ax, bx, cx, dx;
214 u64 si, di, bp, sp;
215 u64 r8, r9, r10, r11;
216 u64 r12, r13, r14, r15;
217 u64 status, dla, dse, lat;
218};
219
220
221
222
223struct pebs_record_hsw {
224 u64 flags, ip;
225 u64 ax, bx, cx, dx;
226 u64 si, di, bp, sp;
227 u64 r8, r9, r10, r11;
228 u64 r12, r13, r14, r15;
229 u64 status, dla, dse, lat;
230 u64 real_ip, tsx_tuning;
231};
232
233union hsw_tsx_tuning {
234 struct {
235 u32 cycles_last_block : 32,
236 hle_abort : 1,
237 rtm_abort : 1,
238 instruction_abort : 1,
239 non_instruction_abort : 1,
240 retry : 1,
241 data_conflict : 1,
242 capacity_writes : 1,
243 capacity_reads : 1;
244 };
245 u64 value;
246};
247
248#define PEBS_HSW_TSX_FLAGS 0xff00000000ULL
249
250
251
252struct pebs_record_skl {
253 u64 flags, ip;
254 u64 ax, bx, cx, dx;
255 u64 si, di, bp, sp;
256 u64 r8, r9, r10, r11;
257 u64 r12, r13, r14, r15;
258 u64 status, dla, dse, lat;
259 u64 real_ip, tsx_tuning;
260 u64 tsc;
261};
262
263void init_debug_store_on_cpu(int cpu)
264{
265 struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
266
267 if (!ds)
268 return;
269
270 wrmsr_on_cpu(cpu, MSR_IA32_DS_AREA,
271 (u32)((u64)(unsigned long)ds),
272 (u32)((u64)(unsigned long)ds >> 32));
273}
274
275void fini_debug_store_on_cpu(int cpu)
276{
277 if (!per_cpu(cpu_hw_events, cpu).ds)
278 return;
279
280 wrmsr_on_cpu(cpu, MSR_IA32_DS_AREA, 0, 0);
281}
282
283static DEFINE_PER_CPU(void *, insn_buffer);
284
285static void ds_update_cea(void *cea, void *addr, size_t size, pgprot_t prot)
286{
287 unsigned long start = (unsigned long)cea;
288 phys_addr_t pa;
289 size_t msz = 0;
290
291 pa = virt_to_phys(addr);
292
293 preempt_disable();
294 for (; msz < size; msz += PAGE_SIZE, pa += PAGE_SIZE, cea += PAGE_SIZE)
295 cea_set_pte(cea, pa, prot);
296
297
298
299
300
301 flush_tlb_kernel_range(start, start + size);
302 preempt_enable();
303}
304
305static void ds_clear_cea(void *cea, size_t size)
306{
307 unsigned long start = (unsigned long)cea;
308 size_t msz = 0;
309
310 preempt_disable();
311 for (; msz < size; msz += PAGE_SIZE, cea += PAGE_SIZE)
312 cea_set_pte(cea, 0, PAGE_NONE);
313
314 flush_tlb_kernel_range(start, start + size);
315 preempt_enable();
316}
317
318static void *dsalloc_pages(size_t size, gfp_t flags, int cpu)
319{
320 unsigned int order = get_order(size);
321 int node = cpu_to_node(cpu);
322 struct page *page;
323
324 page = __alloc_pages_node(node, flags | __GFP_ZERO, order);
325 return page ? page_address(page) : NULL;
326}
327
328static void dsfree_pages(const void *buffer, size_t size)
329{
330 if (buffer)
331 free_pages((unsigned long)buffer, get_order(size));
332}
333
334static int alloc_pebs_buffer(int cpu)
335{
336 struct cpu_hw_events *hwev = per_cpu_ptr(&cpu_hw_events, cpu);
337 struct debug_store *ds = hwev->ds;
338 size_t bsiz = x86_pmu.pebs_buffer_size;
339 int max, node = cpu_to_node(cpu);
340 void *buffer, *ibuffer, *cea;
341
342 if (!x86_pmu.pebs)
343 return 0;
344
345 buffer = dsalloc_pages(bsiz, GFP_KERNEL, cpu);
346 if (unlikely(!buffer))
347 return -ENOMEM;
348
349
350
351
352
353 if (x86_pmu.intel_cap.pebs_format < 2) {
354 ibuffer = kzalloc_node(PEBS_FIXUP_SIZE, GFP_KERNEL, node);
355 if (!ibuffer) {
356 dsfree_pages(buffer, bsiz);
357 return -ENOMEM;
358 }
359 per_cpu(insn_buffer, cpu) = ibuffer;
360 }
361 hwev->ds_pebs_vaddr = buffer;
362
363 cea = &get_cpu_entry_area(cpu)->cpu_debug_buffers.pebs_buffer;
364 ds->pebs_buffer_base = (unsigned long) cea;
365 ds_update_cea(cea, buffer, bsiz, PAGE_KERNEL);
366 ds->pebs_index = ds->pebs_buffer_base;
367 max = x86_pmu.pebs_record_size * (bsiz / x86_pmu.pebs_record_size);
368 ds->pebs_absolute_maximum = ds->pebs_buffer_base + max;
369 return 0;
370}
371
372static void release_pebs_buffer(int cpu)
373{
374 struct cpu_hw_events *hwev = per_cpu_ptr(&cpu_hw_events, cpu);
375 void *cea;
376
377 if (!x86_pmu.pebs)
378 return;
379
380 kfree(per_cpu(insn_buffer, cpu));
381 per_cpu(insn_buffer, cpu) = NULL;
382
383
384 cea = &get_cpu_entry_area(cpu)->cpu_debug_buffers.pebs_buffer;
385 ds_clear_cea(cea, x86_pmu.pebs_buffer_size);
386 dsfree_pages(hwev->ds_pebs_vaddr, x86_pmu.pebs_buffer_size);
387 hwev->ds_pebs_vaddr = NULL;
388}
389
390static int alloc_bts_buffer(int cpu)
391{
392 struct cpu_hw_events *hwev = per_cpu_ptr(&cpu_hw_events, cpu);
393 struct debug_store *ds = hwev->ds;
394 void *buffer, *cea;
395 int max;
396
397 if (!x86_pmu.bts)
398 return 0;
399
400 buffer = dsalloc_pages(BTS_BUFFER_SIZE, GFP_KERNEL | __GFP_NOWARN, cpu);
401 if (unlikely(!buffer)) {
402 WARN_ONCE(1, "%s: BTS buffer allocation failure\n", __func__);
403 return -ENOMEM;
404 }
405 hwev->ds_bts_vaddr = buffer;
406
407 cea = &get_cpu_entry_area(cpu)->cpu_debug_buffers.bts_buffer;
408 ds->bts_buffer_base = (unsigned long) cea;
409 ds_update_cea(cea, buffer, BTS_BUFFER_SIZE, PAGE_KERNEL);
410 ds->bts_index = ds->bts_buffer_base;
411 max = BTS_RECORD_SIZE * (BTS_BUFFER_SIZE / BTS_RECORD_SIZE);
412 ds->bts_absolute_maximum = ds->bts_buffer_base + max;
413 ds->bts_interrupt_threshold = ds->bts_absolute_maximum - (max / 16);
414 return 0;
415}
416
417static void release_bts_buffer(int cpu)
418{
419 struct cpu_hw_events *hwev = per_cpu_ptr(&cpu_hw_events, cpu);
420 void *cea;
421
422 if (!x86_pmu.bts)
423 return;
424
425
426 cea = &get_cpu_entry_area(cpu)->cpu_debug_buffers.bts_buffer;
427 ds_clear_cea(cea, BTS_BUFFER_SIZE);
428 dsfree_pages(hwev->ds_bts_vaddr, BTS_BUFFER_SIZE);
429 hwev->ds_bts_vaddr = NULL;
430}
431
432static int alloc_ds_buffer(int cpu)
433{
434 struct debug_store *ds = &get_cpu_entry_area(cpu)->cpu_debug_store;
435
436 memset(ds, 0, sizeof(*ds));
437 per_cpu(cpu_hw_events, cpu).ds = ds;
438 return 0;
439}
440
441static void release_ds_buffer(int cpu)
442{
443 per_cpu(cpu_hw_events, cpu).ds = NULL;
444}
445
446void release_ds_buffers(void)
447{
448 int cpu;
449
450 if (!x86_pmu.bts && !x86_pmu.pebs)
451 return;
452
453 for_each_possible_cpu(cpu)
454 release_ds_buffer(cpu);
455
456 for_each_possible_cpu(cpu) {
457
458
459
460
461
462 fini_debug_store_on_cpu(cpu);
463 }
464
465 for_each_possible_cpu(cpu) {
466 release_pebs_buffer(cpu);
467 release_bts_buffer(cpu);
468 }
469}
470
471void reserve_ds_buffers(void)
472{
473 int bts_err = 0, pebs_err = 0;
474 int cpu;
475
476 x86_pmu.bts_active = 0;
477 x86_pmu.pebs_active = 0;
478
479 if (!x86_pmu.bts && !x86_pmu.pebs)
480 return;
481
482 if (!x86_pmu.bts)
483 bts_err = 1;
484
485 if (!x86_pmu.pebs)
486 pebs_err = 1;
487
488 for_each_possible_cpu(cpu) {
489 if (alloc_ds_buffer(cpu)) {
490 bts_err = 1;
491 pebs_err = 1;
492 }
493
494 if (!bts_err && alloc_bts_buffer(cpu))
495 bts_err = 1;
496
497 if (!pebs_err && alloc_pebs_buffer(cpu))
498 pebs_err = 1;
499
500 if (bts_err && pebs_err)
501 break;
502 }
503
504 if (bts_err) {
505 for_each_possible_cpu(cpu)
506 release_bts_buffer(cpu);
507 }
508
509 if (pebs_err) {
510 for_each_possible_cpu(cpu)
511 release_pebs_buffer(cpu);
512 }
513
514 if (bts_err && pebs_err) {
515 for_each_possible_cpu(cpu)
516 release_ds_buffer(cpu);
517 } else {
518 if (x86_pmu.bts && !bts_err)
519 x86_pmu.bts_active = 1;
520
521 if (x86_pmu.pebs && !pebs_err)
522 x86_pmu.pebs_active = 1;
523
524 for_each_possible_cpu(cpu) {
525
526
527
528
529 init_debug_store_on_cpu(cpu);
530 }
531 }
532}
533
534
535
536
537
538struct event_constraint bts_constraint =
539 EVENT_CONSTRAINT(0, 1ULL << INTEL_PMC_IDX_FIXED_BTS, 0);
540
541void intel_pmu_enable_bts(u64 config)
542{
543 unsigned long debugctlmsr;
544
545 debugctlmsr = get_debugctlmsr();
546
547 debugctlmsr |= DEBUGCTLMSR_TR;
548 debugctlmsr |= DEBUGCTLMSR_BTS;
549 if (config & ARCH_PERFMON_EVENTSEL_INT)
550 debugctlmsr |= DEBUGCTLMSR_BTINT;
551
552 if (!(config & ARCH_PERFMON_EVENTSEL_OS))
553 debugctlmsr |= DEBUGCTLMSR_BTS_OFF_OS;
554
555 if (!(config & ARCH_PERFMON_EVENTSEL_USR))
556 debugctlmsr |= DEBUGCTLMSR_BTS_OFF_USR;
557
558 update_debugctlmsr(debugctlmsr);
559}
560
561void intel_pmu_disable_bts(void)
562{
563 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
564 unsigned long debugctlmsr;
565
566 if (!cpuc->ds)
567 return;
568
569 debugctlmsr = get_debugctlmsr();
570
571 debugctlmsr &=
572 ~(DEBUGCTLMSR_TR | DEBUGCTLMSR_BTS | DEBUGCTLMSR_BTINT |
573 DEBUGCTLMSR_BTS_OFF_OS | DEBUGCTLMSR_BTS_OFF_USR);
574
575 update_debugctlmsr(debugctlmsr);
576}
577
578int intel_pmu_drain_bts_buffer(void)
579{
580 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
581 struct debug_store *ds = cpuc->ds;
582 struct bts_record {
583 u64 from;
584 u64 to;
585 u64 flags;
586 };
587 struct perf_event *event = cpuc->events[INTEL_PMC_IDX_FIXED_BTS];
588 struct bts_record *at, *base, *top;
589 struct perf_output_handle handle;
590 struct perf_event_header header;
591 struct perf_sample_data data;
592 unsigned long skip = 0;
593 struct pt_regs regs;
594
595 if (!event)
596 return 0;
597
598 if (!x86_pmu.bts_active)
599 return 0;
600
601 base = (struct bts_record *)(unsigned long)ds->bts_buffer_base;
602 top = (struct bts_record *)(unsigned long)ds->bts_index;
603
604 if (top <= base)
605 return 0;
606
607 memset(®s, 0, sizeof(regs));
608
609 ds->bts_index = ds->bts_buffer_base;
610
611 perf_sample_data_init(&data, 0, event->hw.last_period);
612
613
614
615
616
617
618
619
620
621
622
623 for (at = base; at < top; at++) {
624
625
626
627
628
629 if (event->attr.exclude_kernel &&
630 (kernel_ip(at->from) || kernel_ip(at->to)))
631 skip++;
632 }
633
634
635
636
637
638
639 rcu_read_lock();
640 perf_prepare_sample(&header, &data, event, ®s);
641
642 if (perf_output_begin(&handle, event, header.size *
643 (top - base - skip)))
644 goto unlock;
645
646 for (at = base; at < top; at++) {
647
648 if (event->attr.exclude_kernel &&
649 (kernel_ip(at->from) || kernel_ip(at->to)))
650 continue;
651
652 data.ip = at->from;
653 data.addr = at->to;
654
655 perf_output_sample(&handle, &header, &data, event);
656 }
657
658 perf_output_end(&handle);
659
660
661 event->hw.interrupts++;
662 event->pending_kill = POLL_IN;
663unlock:
664 rcu_read_unlock();
665 return 1;
666}
667
668static inline void intel_pmu_drain_pebs_buffer(void)
669{
670 struct pt_regs regs;
671
672 x86_pmu.drain_pebs(®s);
673}
674
675
676
677
678struct event_constraint intel_core2_pebs_event_constraints[] = {
679 INTEL_FLAGS_UEVENT_CONSTRAINT(0x00c0, 0x1),
680 INTEL_FLAGS_UEVENT_CONSTRAINT(0xfec1, 0x1),
681 INTEL_FLAGS_UEVENT_CONSTRAINT(0x00c5, 0x1),
682 INTEL_FLAGS_UEVENT_CONSTRAINT(0x1fc7, 0x1),
683 INTEL_FLAGS_EVENT_CONSTRAINT(0xcb, 0x1),
684
685 INTEL_FLAGS_EVENT_CONSTRAINT(0x108000c0, 0x01),
686 EVENT_CONSTRAINT_END
687};
688
689struct event_constraint intel_atom_pebs_event_constraints[] = {
690 INTEL_FLAGS_UEVENT_CONSTRAINT(0x00c0, 0x1),
691 INTEL_FLAGS_UEVENT_CONSTRAINT(0x00c5, 0x1),
692 INTEL_FLAGS_EVENT_CONSTRAINT(0xcb, 0x1),
693
694 INTEL_FLAGS_EVENT_CONSTRAINT(0x108000c0, 0x01),
695
696 INTEL_ALL_EVENT_CONSTRAINT(0, 0x1),
697 EVENT_CONSTRAINT_END
698};
699
700struct event_constraint intel_slm_pebs_event_constraints[] = {
701
702 INTEL_FLAGS_EVENT_CONSTRAINT(0x108000c0, 0x1),
703
704 INTEL_ALL_EVENT_CONSTRAINT(0, 0x1),
705 EVENT_CONSTRAINT_END
706};
707
708struct event_constraint intel_glm_pebs_event_constraints[] = {
709
710 INTEL_ALL_EVENT_CONSTRAINT(0, 0x1),
711 EVENT_CONSTRAINT_END
712};
713
714struct event_constraint intel_glp_pebs_event_constraints[] = {
715
716 INTEL_ALL_EVENT_CONSTRAINT(0, 0xf),
717 EVENT_CONSTRAINT_END
718};
719
720struct event_constraint intel_nehalem_pebs_event_constraints[] = {
721 INTEL_PLD_CONSTRAINT(0x100b, 0xf),
722 INTEL_FLAGS_EVENT_CONSTRAINT(0x0f, 0xf),
723 INTEL_FLAGS_UEVENT_CONSTRAINT(0x010c, 0xf),
724 INTEL_FLAGS_EVENT_CONSTRAINT(0xc0, 0xf),
725 INTEL_EVENT_CONSTRAINT(0xc2, 0xf),
726 INTEL_FLAGS_EVENT_CONSTRAINT(0xc4, 0xf),
727 INTEL_FLAGS_UEVENT_CONSTRAINT(0x02c5, 0xf),
728 INTEL_FLAGS_EVENT_CONSTRAINT(0xc7, 0xf),
729 INTEL_FLAGS_UEVENT_CONSTRAINT(0x20c8, 0xf),
730 INTEL_FLAGS_EVENT_CONSTRAINT(0xcb, 0xf),
731 INTEL_FLAGS_EVENT_CONSTRAINT(0xf7, 0xf),
732
733 INTEL_FLAGS_EVENT_CONSTRAINT(0x108000c0, 0x0f),
734 EVENT_CONSTRAINT_END
735};
736
737struct event_constraint intel_westmere_pebs_event_constraints[] = {
738 INTEL_PLD_CONSTRAINT(0x100b, 0xf),
739 INTEL_FLAGS_EVENT_CONSTRAINT(0x0f, 0xf),
740 INTEL_FLAGS_UEVENT_CONSTRAINT(0x010c, 0xf),
741 INTEL_FLAGS_EVENT_CONSTRAINT(0xc0, 0xf),
742 INTEL_EVENT_CONSTRAINT(0xc2, 0xf),
743 INTEL_FLAGS_EVENT_CONSTRAINT(0xc4, 0xf),
744 INTEL_FLAGS_EVENT_CONSTRAINT(0xc5, 0xf),
745 INTEL_FLAGS_EVENT_CONSTRAINT(0xc7, 0xf),
746 INTEL_FLAGS_UEVENT_CONSTRAINT(0x20c8, 0xf),
747 INTEL_FLAGS_EVENT_CONSTRAINT(0xcb, 0xf),
748 INTEL_FLAGS_EVENT_CONSTRAINT(0xf7, 0xf),
749
750 INTEL_FLAGS_EVENT_CONSTRAINT(0x108000c0, 0x0f),
751 EVENT_CONSTRAINT_END
752};
753
754struct event_constraint intel_snb_pebs_event_constraints[] = {
755 INTEL_FLAGS_UEVENT_CONSTRAINT(0x01c0, 0x2),
756 INTEL_PLD_CONSTRAINT(0x01cd, 0x8),
757 INTEL_PST_CONSTRAINT(0x02cd, 0x8),
758
759 INTEL_FLAGS_EVENT_CONSTRAINT(0x108001c2, 0xf),
760 INTEL_EXCLEVT_CONSTRAINT(0xd0, 0xf),
761 INTEL_EXCLEVT_CONSTRAINT(0xd1, 0xf),
762 INTEL_EXCLEVT_CONSTRAINT(0xd2, 0xf),
763 INTEL_EXCLEVT_CONSTRAINT(0xd3, 0xf),
764
765 INTEL_ALL_EVENT_CONSTRAINT(0, 0xf),
766 EVENT_CONSTRAINT_END
767};
768
769struct event_constraint intel_ivb_pebs_event_constraints[] = {
770 INTEL_FLAGS_UEVENT_CONSTRAINT(0x01c0, 0x2),
771 INTEL_PLD_CONSTRAINT(0x01cd, 0x8),
772 INTEL_PST_CONSTRAINT(0x02cd, 0x8),
773
774 INTEL_FLAGS_EVENT_CONSTRAINT(0x108001c2, 0xf),
775
776 INTEL_FLAGS_EVENT_CONSTRAINT(0x108001c0, 0x2),
777 INTEL_EXCLEVT_CONSTRAINT(0xd0, 0xf),
778 INTEL_EXCLEVT_CONSTRAINT(0xd1, 0xf),
779 INTEL_EXCLEVT_CONSTRAINT(0xd2, 0xf),
780 INTEL_EXCLEVT_CONSTRAINT(0xd3, 0xf),
781
782 INTEL_ALL_EVENT_CONSTRAINT(0, 0xf),
783 EVENT_CONSTRAINT_END
784};
785
786struct event_constraint intel_hsw_pebs_event_constraints[] = {
787 INTEL_FLAGS_UEVENT_CONSTRAINT(0x01c0, 0x2),
788 INTEL_PLD_CONSTRAINT(0x01cd, 0xf),
789
790 INTEL_FLAGS_EVENT_CONSTRAINT(0x108001c2, 0xf),
791
792 INTEL_FLAGS_EVENT_CONSTRAINT(0x108001c0, 0x2),
793 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_NA(0x01c2, 0xf),
794 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_XLD(0x11d0, 0xf),
795 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_XLD(0x21d0, 0xf),
796 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_XLD(0x41d0, 0xf),
797 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_XLD(0x81d0, 0xf),
798 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_XST(0x12d0, 0xf),
799 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_XST(0x42d0, 0xf),
800 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_XST(0x82d0, 0xf),
801 INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_XLD(0xd1, 0xf),
802 INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_XLD(0xd2, 0xf),
803 INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_XLD(0xd3, 0xf),
804
805 INTEL_ALL_EVENT_CONSTRAINT(0, 0xf),
806 EVENT_CONSTRAINT_END
807};
808
809struct event_constraint intel_bdw_pebs_event_constraints[] = {
810 INTEL_FLAGS_UEVENT_CONSTRAINT(0x01c0, 0x2),
811 INTEL_PLD_CONSTRAINT(0x01cd, 0xf),
812
813 INTEL_FLAGS_EVENT_CONSTRAINT(0x108001c2, 0xf),
814
815 INTEL_FLAGS_EVENT_CONSTRAINT(0x108001c0, 0x2),
816 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_NA(0x01c2, 0xf),
817 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x11d0, 0xf),
818 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x21d0, 0xf),
819 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x41d0, 0xf),
820 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x81d0, 0xf),
821 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x12d0, 0xf),
822 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x42d0, 0xf),
823 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x82d0, 0xf),
824 INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_LD(0xd1, 0xf),
825 INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_LD(0xd2, 0xf),
826 INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_LD(0xd3, 0xf),
827
828 INTEL_ALL_EVENT_CONSTRAINT(0, 0xf),
829 EVENT_CONSTRAINT_END
830};
831
832
833struct event_constraint intel_skl_pebs_event_constraints[] = {
834 INTEL_FLAGS_UEVENT_CONSTRAINT(0x1c0, 0x2),
835
836 INTEL_FLAGS_EVENT_CONSTRAINT(0x108001c0, 0x2),
837
838 INTEL_FLAGS_EVENT_CONSTRAINT(0x108000c0, 0x0f),
839 INTEL_PLD_CONSTRAINT(0x1cd, 0xf),
840 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x11d0, 0xf),
841 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x12d0, 0xf),
842 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x21d0, 0xf),
843 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x22d0, 0xf),
844 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x41d0, 0xf),
845 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x42d0, 0xf),
846 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x81d0, 0xf),
847 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x82d0, 0xf),
848 INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_LD(0xd1, 0xf),
849 INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_LD(0xd2, 0xf),
850 INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_LD(0xd3, 0xf),
851
852 INTEL_ALL_EVENT_CONSTRAINT(0, 0xf),
853 EVENT_CONSTRAINT_END
854};
855
856struct event_constraint *intel_pebs_constraints(struct perf_event *event)
857{
858 struct event_constraint *c;
859
860 if (!event->attr.precise_ip)
861 return NULL;
862
863 if (x86_pmu.pebs_constraints) {
864 for_each_event_constraint(c, x86_pmu.pebs_constraints) {
865 if ((event->hw.config & c->cmask) == c->code) {
866 event->hw.flags |= c->flags;
867 return c;
868 }
869 }
870 }
871
872 return &emptyconstraint;
873}
874
875
876
877
878
879
880static inline bool pebs_needs_sched_cb(struct cpu_hw_events *cpuc)
881{
882 return cpuc->n_pebs && (cpuc->n_pebs == cpuc->n_large_pebs);
883}
884
885void intel_pmu_pebs_sched_task(struct perf_event_context *ctx, bool sched_in)
886{
887 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
888
889 if (!sched_in && pebs_needs_sched_cb(cpuc))
890 intel_pmu_drain_pebs_buffer();
891}
892
893static inline void pebs_update_threshold(struct cpu_hw_events *cpuc)
894{
895 struct debug_store *ds = cpuc->ds;
896 u64 threshold;
897
898 if (cpuc->n_pebs == cpuc->n_large_pebs) {
899 threshold = ds->pebs_absolute_maximum -
900 x86_pmu.max_pebs_events * x86_pmu.pebs_record_size;
901 } else {
902 threshold = ds->pebs_buffer_base + x86_pmu.pebs_record_size;
903 }
904
905 ds->pebs_interrupt_threshold = threshold;
906}
907
908static void
909pebs_update_state(bool needed_cb, struct cpu_hw_events *cpuc, struct pmu *pmu)
910{
911
912
913
914
915
916 bool update = cpuc->n_pebs == 1;
917
918 if (needed_cb != pebs_needs_sched_cb(cpuc)) {
919 if (!needed_cb)
920 perf_sched_cb_inc(pmu);
921 else
922 perf_sched_cb_dec(pmu);
923
924 update = true;
925 }
926
927 if (update)
928 pebs_update_threshold(cpuc);
929}
930
931void intel_pmu_pebs_add(struct perf_event *event)
932{
933 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
934 struct hw_perf_event *hwc = &event->hw;
935 bool needed_cb = pebs_needs_sched_cb(cpuc);
936
937 cpuc->n_pebs++;
938 if (hwc->flags & PERF_X86_EVENT_LARGE_PEBS)
939 cpuc->n_large_pebs++;
940
941 pebs_update_state(needed_cb, cpuc, event->ctx->pmu);
942}
943
944void intel_pmu_pebs_enable(struct perf_event *event)
945{
946 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
947 struct hw_perf_event *hwc = &event->hw;
948 struct debug_store *ds = cpuc->ds;
949
950 hwc->config &= ~ARCH_PERFMON_EVENTSEL_INT;
951
952 cpuc->pebs_enabled |= 1ULL << hwc->idx;
953
954 if (event->hw.flags & PERF_X86_EVENT_PEBS_LDLAT)
955 cpuc->pebs_enabled |= 1ULL << (hwc->idx + 32);
956 else if (event->hw.flags & PERF_X86_EVENT_PEBS_ST)
957 cpuc->pebs_enabled |= 1ULL << 63;
958
959
960
961
962
963 if (hwc->flags & PERF_X86_EVENT_AUTO_RELOAD) {
964 ds->pebs_event_reset[hwc->idx] =
965 (u64)(-hwc->sample_period) & x86_pmu.cntval_mask;
966 } else {
967 ds->pebs_event_reset[hwc->idx] = 0;
968 }
969}
970
971void intel_pmu_pebs_del(struct perf_event *event)
972{
973 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
974 struct hw_perf_event *hwc = &event->hw;
975 bool needed_cb = pebs_needs_sched_cb(cpuc);
976
977 cpuc->n_pebs--;
978 if (hwc->flags & PERF_X86_EVENT_LARGE_PEBS)
979 cpuc->n_large_pebs--;
980
981 pebs_update_state(needed_cb, cpuc, event->ctx->pmu);
982}
983
984void intel_pmu_pebs_disable(struct perf_event *event)
985{
986 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
987 struct hw_perf_event *hwc = &event->hw;
988
989 if (cpuc->n_pebs == cpuc->n_large_pebs)
990 intel_pmu_drain_pebs_buffer();
991
992 cpuc->pebs_enabled &= ~(1ULL << hwc->idx);
993
994 if (event->hw.flags & PERF_X86_EVENT_PEBS_LDLAT)
995 cpuc->pebs_enabled &= ~(1ULL << (hwc->idx + 32));
996 else if (event->hw.flags & PERF_X86_EVENT_PEBS_ST)
997 cpuc->pebs_enabled &= ~(1ULL << 63);
998
999 if (cpuc->enabled)
1000 wrmsrl(MSR_IA32_PEBS_ENABLE, cpuc->pebs_enabled);
1001
1002 hwc->config |= ARCH_PERFMON_EVENTSEL_INT;
1003}
1004
1005void intel_pmu_pebs_enable_all(void)
1006{
1007 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1008
1009 if (cpuc->pebs_enabled)
1010 wrmsrl(MSR_IA32_PEBS_ENABLE, cpuc->pebs_enabled);
1011}
1012
1013void intel_pmu_pebs_disable_all(void)
1014{
1015 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1016
1017 if (cpuc->pebs_enabled)
1018 wrmsrl(MSR_IA32_PEBS_ENABLE, 0);
1019}
1020
1021static int intel_pmu_pebs_fixup_ip(struct pt_regs *regs)
1022{
1023 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1024 unsigned long from = cpuc->lbr_entries[0].from;
1025 unsigned long old_to, to = cpuc->lbr_entries[0].to;
1026 unsigned long ip = regs->ip;
1027 int is_64bit = 0;
1028 void *kaddr;
1029 int size;
1030
1031
1032
1033
1034 if (!x86_pmu.intel_cap.pebs_trap)
1035 return 1;
1036
1037
1038
1039
1040 if (!cpuc->lbr_stack.nr || !from || !to)
1041 return 0;
1042
1043
1044
1045
1046 if (kernel_ip(ip) != kernel_ip(to))
1047 return 0;
1048
1049
1050
1051
1052
1053 if ((ip - to) > PEBS_FIXUP_SIZE)
1054 return 0;
1055
1056
1057
1058
1059 if (ip == to) {
1060 set_linear_ip(regs, from);
1061 return 1;
1062 }
1063
1064 size = ip - to;
1065 if (!kernel_ip(ip)) {
1066 int bytes;
1067 u8 *buf = this_cpu_read(insn_buffer);
1068
1069
1070 bytes = copy_from_user_nmi(buf, (void __user *)to, size);
1071 if (bytes != 0)
1072 return 0;
1073
1074 kaddr = buf;
1075 } else {
1076 kaddr = (void *)to;
1077 }
1078
1079 do {
1080 struct insn insn;
1081
1082 old_to = to;
1083
1084#ifdef CONFIG_X86_64
1085 is_64bit = kernel_ip(to) || !test_thread_flag(TIF_IA32);
1086#endif
1087 insn_init(&insn, kaddr, size, is_64bit);
1088 insn_get_length(&insn);
1089
1090
1091
1092
1093
1094
1095 if (!insn.length)
1096 break;
1097
1098 to += insn.length;
1099 kaddr += insn.length;
1100 size -= insn.length;
1101 } while (to < ip);
1102
1103 if (to == ip) {
1104 set_linear_ip(regs, old_to);
1105 return 1;
1106 }
1107
1108
1109
1110
1111
1112 return 0;
1113}
1114
1115static inline u64 intel_hsw_weight(struct pebs_record_skl *pebs)
1116{
1117 if (pebs->tsx_tuning) {
1118 union hsw_tsx_tuning tsx = { .value = pebs->tsx_tuning };
1119 return tsx.cycles_last_block;
1120 }
1121 return 0;
1122}
1123
1124static inline u64 intel_hsw_transaction(struct pebs_record_skl *pebs)
1125{
1126 u64 txn = (pebs->tsx_tuning & PEBS_HSW_TSX_FLAGS) >> 32;
1127
1128
1129 if ((txn & PERF_TXN_TRANSACTION) && (pebs->ax & 1))
1130 txn |= ((pebs->ax >> 24) & 0xff) << PERF_TXN_ABORT_SHIFT;
1131 return txn;
1132}
1133
1134static void setup_pebs_sample_data(struct perf_event *event,
1135 struct pt_regs *iregs, void *__pebs,
1136 struct perf_sample_data *data,
1137 struct pt_regs *regs)
1138{
1139#define PERF_X86_EVENT_PEBS_HSW_PREC \
1140 (PERF_X86_EVENT_PEBS_ST_HSW | \
1141 PERF_X86_EVENT_PEBS_LD_HSW | \
1142 PERF_X86_EVENT_PEBS_NA_HSW)
1143
1144
1145
1146
1147 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1148 struct pebs_record_skl *pebs = __pebs;
1149 u64 sample_type;
1150 int fll, fst, dsrc;
1151 int fl = event->hw.flags;
1152
1153 if (pebs == NULL)
1154 return;
1155
1156 sample_type = event->attr.sample_type;
1157 dsrc = sample_type & PERF_SAMPLE_DATA_SRC;
1158
1159 fll = fl & PERF_X86_EVENT_PEBS_LDLAT;
1160 fst = fl & (PERF_X86_EVENT_PEBS_ST | PERF_X86_EVENT_PEBS_HSW_PREC);
1161
1162 perf_sample_data_init(data, 0, event->hw.last_period);
1163
1164 data->period = event->hw.last_period;
1165
1166
1167
1168
1169 if (fll && (sample_type & PERF_SAMPLE_WEIGHT))
1170 data->weight = pebs->lat;
1171
1172
1173
1174
1175 if (dsrc) {
1176 u64 val = PERF_MEM_NA;
1177 if (fll)
1178 val = load_latency_data(pebs->dse);
1179 else if (fst && (fl & PERF_X86_EVENT_PEBS_HSW_PREC))
1180 val = precise_datala_hsw(event, pebs->dse);
1181 else if (fst)
1182 val = precise_store_data(pebs->dse);
1183 data->data_src.val = val;
1184 }
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198 *regs = *iregs;
1199
1200
1201
1202
1203
1204
1205 regs->flags = pebs->flags & ~PERF_EFLAGS_EXACT;
1206
1207 if (sample_type & PERF_SAMPLE_REGS_INTR) {
1208 regs->ax = pebs->ax;
1209 regs->bx = pebs->bx;
1210 regs->cx = pebs->cx;
1211 regs->dx = pebs->dx;
1212 regs->si = pebs->si;
1213 regs->di = pebs->di;
1214
1215
1216
1217
1218
1219
1220 if (!(sample_type & PERF_SAMPLE_CALLCHAIN)) {
1221 regs->bp = pebs->bp;
1222 regs->sp = pebs->sp;
1223 }
1224
1225#ifndef CONFIG_X86_32
1226 regs->r8 = pebs->r8;
1227 regs->r9 = pebs->r9;
1228 regs->r10 = pebs->r10;
1229 regs->r11 = pebs->r11;
1230 regs->r12 = pebs->r12;
1231 regs->r13 = pebs->r13;
1232 regs->r14 = pebs->r14;
1233 regs->r15 = pebs->r15;
1234#endif
1235 }
1236
1237 if (event->attr.precise_ip > 1) {
1238
1239
1240
1241
1242
1243 if (x86_pmu.intel_cap.pebs_format >= 2) {
1244 set_linear_ip(regs, pebs->real_ip);
1245 regs->flags |= PERF_EFLAGS_EXACT;
1246 } else {
1247
1248 set_linear_ip(regs, pebs->ip);
1249
1250
1251
1252
1253
1254
1255 if (intel_pmu_pebs_fixup_ip(regs))
1256 regs->flags |= PERF_EFLAGS_EXACT;
1257 }
1258 } else {
1259
1260
1261
1262
1263 set_linear_ip(regs, pebs->ip);
1264 }
1265
1266
1267 if ((sample_type & (PERF_SAMPLE_ADDR | PERF_SAMPLE_PHYS_ADDR)) &&
1268 x86_pmu.intel_cap.pebs_format >= 1)
1269 data->addr = pebs->dla;
1270
1271 if (x86_pmu.intel_cap.pebs_format >= 2) {
1272
1273 if ((sample_type & PERF_SAMPLE_WEIGHT) && !fll)
1274 data->weight = intel_hsw_weight(pebs);
1275
1276 if (sample_type & PERF_SAMPLE_TRANSACTION)
1277 data->txn = intel_hsw_transaction(pebs);
1278 }
1279
1280
1281
1282
1283
1284
1285
1286 if (x86_pmu.intel_cap.pebs_format >= 3 &&
1287 event->attr.use_clockid == 0)
1288 data->time = native_sched_clock_from_tsc(pebs->tsc);
1289
1290 if (has_branch_stack(event))
1291 data->br_stack = &cpuc->lbr_stack;
1292}
1293
1294static inline void *
1295get_next_pebs_record_by_bit(void *base, void *top, int bit)
1296{
1297 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1298 void *at;
1299 u64 pebs_status;
1300
1301
1302
1303
1304
1305 if (x86_pmu.intel_cap.pebs_format < 1)
1306 return base;
1307
1308 if (base == NULL)
1309 return NULL;
1310
1311 for (at = base; at < top; at += x86_pmu.pebs_record_size) {
1312 struct pebs_record_nhm *p = at;
1313
1314 if (test_bit(bit, (unsigned long *)&p->status)) {
1315
1316 if (x86_pmu.intel_cap.pebs_format >= 3)
1317 return at;
1318
1319 if (p->status == (1 << bit))
1320 return at;
1321
1322
1323 pebs_status = p->status & cpuc->pebs_enabled;
1324 pebs_status &= PEBS_COUNTER_MASK;
1325 if (pebs_status == (1 << bit))
1326 return at;
1327 }
1328 }
1329 return NULL;
1330}
1331
1332void intel_pmu_auto_reload_read(struct perf_event *event)
1333{
1334 WARN_ON(!(event->hw.flags & PERF_X86_EVENT_AUTO_RELOAD));
1335
1336 perf_pmu_disable(event->pmu);
1337 intel_pmu_drain_pebs_buffer();
1338 perf_pmu_enable(event->pmu);
1339}
1340
1341
1342
1343
1344static int
1345intel_pmu_save_and_restart_reload(struct perf_event *event, int count)
1346{
1347 struct hw_perf_event *hwc = &event->hw;
1348 int shift = 64 - x86_pmu.cntval_bits;
1349 u64 period = hwc->sample_period;
1350 u64 prev_raw_count, new_raw_count;
1351 s64 new, old;
1352
1353 WARN_ON(!period);
1354
1355
1356
1357
1358 WARN_ON(this_cpu_read(cpu_hw_events.enabled));
1359
1360 prev_raw_count = local64_read(&hwc->prev_count);
1361 rdpmcl(hwc->event_base_rdpmc, new_raw_count);
1362 local64_set(&hwc->prev_count, new_raw_count);
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391 new = ((s64)(new_raw_count << shift) >> shift);
1392 old = ((s64)(prev_raw_count << shift) >> shift);
1393 local64_add(new - old + count * period, &event->count);
1394
1395 perf_event_update_userpage(event);
1396
1397 return 0;
1398}
1399
1400static void __intel_pmu_pebs_event(struct perf_event *event,
1401 struct pt_regs *iregs,
1402 void *base, void *top,
1403 int bit, int count)
1404{
1405 struct hw_perf_event *hwc = &event->hw;
1406 struct perf_sample_data data;
1407 struct pt_regs regs;
1408 void *at = get_next_pebs_record_by_bit(base, top, bit);
1409
1410 if (hwc->flags & PERF_X86_EVENT_AUTO_RELOAD) {
1411
1412
1413
1414
1415
1416
1417 intel_pmu_save_and_restart_reload(event, count);
1418 } else if (!intel_pmu_save_and_restart(event))
1419 return;
1420
1421 while (count > 1) {
1422 setup_pebs_sample_data(event, iregs, at, &data, ®s);
1423 perf_event_output(event, &data, ®s);
1424 at += x86_pmu.pebs_record_size;
1425 at = get_next_pebs_record_by_bit(at, top, bit);
1426 count--;
1427 }
1428
1429 setup_pebs_sample_data(event, iregs, at, &data, ®s);
1430
1431
1432
1433
1434
1435 if (perf_event_overflow(event, &data, ®s)) {
1436 x86_pmu_stop(event, 0);
1437 return;
1438 }
1439
1440}
1441
1442static void intel_pmu_drain_pebs_core(struct pt_regs *iregs)
1443{
1444 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1445 struct debug_store *ds = cpuc->ds;
1446 struct perf_event *event = cpuc->events[0];
1447 struct pebs_record_core *at, *top;
1448 int n;
1449
1450 if (!x86_pmu.pebs_active)
1451 return;
1452
1453 at = (struct pebs_record_core *)(unsigned long)ds->pebs_buffer_base;
1454 top = (struct pebs_record_core *)(unsigned long)ds->pebs_index;
1455
1456
1457
1458
1459 ds->pebs_index = ds->pebs_buffer_base;
1460
1461 if (!test_bit(0, cpuc->active_mask))
1462 return;
1463
1464 WARN_ON_ONCE(!event);
1465
1466 if (!event->attr.precise_ip)
1467 return;
1468
1469 n = top - at;
1470 if (n <= 0) {
1471 if (event->hw.flags & PERF_X86_EVENT_AUTO_RELOAD)
1472 intel_pmu_save_and_restart_reload(event, 0);
1473 return;
1474 }
1475
1476 __intel_pmu_pebs_event(event, iregs, at, top, 0, n);
1477}
1478
1479static void intel_pmu_drain_pebs_nhm(struct pt_regs *iregs)
1480{
1481 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1482 struct debug_store *ds = cpuc->ds;
1483 struct perf_event *event;
1484 void *base, *at, *top;
1485 short counts[MAX_PEBS_EVENTS] = {};
1486 short error[MAX_PEBS_EVENTS] = {};
1487 int bit, i;
1488
1489 if (!x86_pmu.pebs_active)
1490 return;
1491
1492 base = (struct pebs_record_nhm *)(unsigned long)ds->pebs_buffer_base;
1493 top = (struct pebs_record_nhm *)(unsigned long)ds->pebs_index;
1494
1495 ds->pebs_index = ds->pebs_buffer_base;
1496
1497 if (unlikely(base >= top)) {
1498
1499
1500
1501
1502
1503
1504
1505 for_each_set_bit(bit, (unsigned long *)&cpuc->pebs_enabled,
1506 x86_pmu.max_pebs_events) {
1507 event = cpuc->events[bit];
1508 if (event->hw.flags & PERF_X86_EVENT_AUTO_RELOAD)
1509 intel_pmu_save_and_restart_reload(event, 0);
1510 }
1511 return;
1512 }
1513
1514 for (at = base; at < top; at += x86_pmu.pebs_record_size) {
1515 struct pebs_record_nhm *p = at;
1516 u64 pebs_status;
1517
1518 pebs_status = p->status & cpuc->pebs_enabled;
1519 pebs_status &= (1ULL << x86_pmu.max_pebs_events) - 1;
1520
1521
1522 if (x86_pmu.intel_cap.pebs_format >= 3) {
1523 for_each_set_bit(bit, (unsigned long *)&pebs_status,
1524 x86_pmu.max_pebs_events)
1525 counts[bit]++;
1526
1527 continue;
1528 }
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538 if (!pebs_status && cpuc->pebs_enabled &&
1539 !(cpuc->pebs_enabled & (cpuc->pebs_enabled-1)))
1540 pebs_status = cpuc->pebs_enabled;
1541
1542 bit = find_first_bit((unsigned long *)&pebs_status,
1543 x86_pmu.max_pebs_events);
1544 if (bit >= x86_pmu.max_pebs_events)
1545 continue;
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562 if (p->status != (1ULL << bit)) {
1563 for_each_set_bit(i, (unsigned long *)&pebs_status,
1564 x86_pmu.max_pebs_events)
1565 error[i]++;
1566 continue;
1567 }
1568
1569 counts[bit]++;
1570 }
1571
1572 for (bit = 0; bit < x86_pmu.max_pebs_events; bit++) {
1573 if ((counts[bit] == 0) && (error[bit] == 0))
1574 continue;
1575
1576 event = cpuc->events[bit];
1577 if (WARN_ON_ONCE(!event))
1578 continue;
1579
1580 if (WARN_ON_ONCE(!event->attr.precise_ip))
1581 continue;
1582
1583
1584 if (error[bit]) {
1585 perf_log_lost_samples(event, error[bit]);
1586
1587 if (perf_event_account_interrupt(event))
1588 x86_pmu_stop(event, 0);
1589 }
1590
1591 if (counts[bit]) {
1592 __intel_pmu_pebs_event(event, iregs, base,
1593 top, bit, counts[bit]);
1594 }
1595 }
1596}
1597
1598
1599
1600
1601
1602void __init intel_ds_init(void)
1603{
1604
1605
1606
1607 if (!boot_cpu_has(X86_FEATURE_DTES64))
1608 return;
1609
1610 x86_pmu.bts = boot_cpu_has(X86_FEATURE_BTS);
1611 x86_pmu.pebs = boot_cpu_has(X86_FEATURE_PEBS);
1612 x86_pmu.pebs_buffer_size = PEBS_BUFFER_SIZE;
1613 if (x86_pmu.pebs) {
1614 char pebs_type = x86_pmu.intel_cap.pebs_trap ? '+' : '-';
1615 int format = x86_pmu.intel_cap.pebs_format;
1616
1617 switch (format) {
1618 case 0:
1619 pr_cont("PEBS fmt0%c, ", pebs_type);
1620 x86_pmu.pebs_record_size = sizeof(struct pebs_record_core);
1621
1622
1623
1624
1625
1626
1627
1628 x86_pmu.pebs_buffer_size = PAGE_SIZE;
1629 x86_pmu.drain_pebs = intel_pmu_drain_pebs_core;
1630 break;
1631
1632 case 1:
1633 pr_cont("PEBS fmt1%c, ", pebs_type);
1634 x86_pmu.pebs_record_size = sizeof(struct pebs_record_nhm);
1635 x86_pmu.drain_pebs = intel_pmu_drain_pebs_nhm;
1636 break;
1637
1638 case 2:
1639 pr_cont("PEBS fmt2%c, ", pebs_type);
1640 x86_pmu.pebs_record_size = sizeof(struct pebs_record_hsw);
1641 x86_pmu.drain_pebs = intel_pmu_drain_pebs_nhm;
1642 break;
1643
1644 case 3:
1645 pr_cont("PEBS fmt3%c, ", pebs_type);
1646 x86_pmu.pebs_record_size =
1647 sizeof(struct pebs_record_skl);
1648 x86_pmu.drain_pebs = intel_pmu_drain_pebs_nhm;
1649 x86_pmu.large_pebs_flags |= PERF_SAMPLE_TIME;
1650 break;
1651
1652 default:
1653 pr_cont("no PEBS fmt%d%c, ", format, pebs_type);
1654 x86_pmu.pebs = 0;
1655 }
1656 }
1657}
1658
1659void perf_restore_debug_store(void)
1660{
1661 struct debug_store *ds = __this_cpu_read(cpu_hw_events.ds);
1662
1663 if (!x86_pmu.bts && !x86_pmu.pebs)
1664 return;
1665
1666 wrmsrl(MSR_IA32_DS_AREA, (unsigned long)ds);
1667}
1668