1
2
3
4
5
6
7
8
9
10
11
12
13
14
15#include <linux/perf_event.h>
16
17#include <asm/intel_ds.h>
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35enum extra_reg_type {
36 EXTRA_REG_NONE = -1,
37
38 EXTRA_REG_RSP_0 = 0,
39 EXTRA_REG_RSP_1 = 1,
40 EXTRA_REG_LBR = 2,
41 EXTRA_REG_LDLAT = 3,
42 EXTRA_REG_FE = 4,
43
44 EXTRA_REG_MAX
45};
46
47struct event_constraint {
48 union {
49 unsigned long idxmsk[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
50 u64 idxmsk64;
51 };
52 u64 code;
53 u64 cmask;
54 int weight;
55 int overlap;
56 int flags;
57 unsigned int size;
58};
59
60static inline bool constraint_match(struct event_constraint *c, u64 ecode)
61{
62 return ((ecode & c->cmask) - c->code) <= (u64)c->size;
63}
64
65
66
67
68#define PERF_X86_EVENT_PEBS_LDLAT 0x0001
69#define PERF_X86_EVENT_PEBS_ST 0x0002
70#define PERF_X86_EVENT_PEBS_ST_HSW 0x0004
71#define PERF_X86_EVENT_PEBS_LD_HSW 0x0008
72#define PERF_X86_EVENT_PEBS_NA_HSW 0x0010
73#define PERF_X86_EVENT_EXCL 0x0020
74#define PERF_X86_EVENT_DYNAMIC 0x0040
75#define PERF_X86_EVENT_RDPMC_ALLOWED 0x0080
76#define PERF_X86_EVENT_EXCL_ACCT 0x0100
77#define PERF_X86_EVENT_AUTO_RELOAD 0x0200
78#define PERF_X86_EVENT_LARGE_PEBS 0x0400
79
80struct amd_nb {
81 int nb_id;
82 int refcnt;
83 struct perf_event *owners[X86_PMC_IDX_MAX];
84 struct event_constraint event_constraints[X86_PMC_IDX_MAX];
85};
86
87#define PEBS_COUNTER_MASK ((1ULL << MAX_PEBS_EVENTS) - 1)
88
89
90
91
92
93
94
95
96#define LARGE_PEBS_FLAGS \
97 (PERF_SAMPLE_IP | PERF_SAMPLE_TID | PERF_SAMPLE_ADDR | \
98 PERF_SAMPLE_ID | PERF_SAMPLE_CPU | PERF_SAMPLE_STREAM_ID | \
99 PERF_SAMPLE_DATA_SRC | PERF_SAMPLE_IDENTIFIER | \
100 PERF_SAMPLE_TRANSACTION | PERF_SAMPLE_PHYS_ADDR | \
101 PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER | \
102 PERF_SAMPLE_PERIOD)
103
104#define PEBS_GP_REGS \
105 ((1ULL << PERF_REG_X86_AX) | \
106 (1ULL << PERF_REG_X86_BX) | \
107 (1ULL << PERF_REG_X86_CX) | \
108 (1ULL << PERF_REG_X86_DX) | \
109 (1ULL << PERF_REG_X86_DI) | \
110 (1ULL << PERF_REG_X86_SI) | \
111 (1ULL << PERF_REG_X86_SP) | \
112 (1ULL << PERF_REG_X86_BP) | \
113 (1ULL << PERF_REG_X86_IP) | \
114 (1ULL << PERF_REG_X86_FLAGS) | \
115 (1ULL << PERF_REG_X86_R8) | \
116 (1ULL << PERF_REG_X86_R9) | \
117 (1ULL << PERF_REG_X86_R10) | \
118 (1ULL << PERF_REG_X86_R11) | \
119 (1ULL << PERF_REG_X86_R12) | \
120 (1ULL << PERF_REG_X86_R13) | \
121 (1ULL << PERF_REG_X86_R14) | \
122 (1ULL << PERF_REG_X86_R15))
123
124
125
126
127struct er_account {
128 raw_spinlock_t lock;
129 u64 config;
130 u64 reg;
131 atomic_t ref;
132};
133
134
135
136
137
138
139
140struct intel_shared_regs {
141 struct er_account regs[EXTRA_REG_MAX];
142 int refcnt;
143 unsigned core_id;
144};
145
146enum intel_excl_state_type {
147 INTEL_EXCL_UNUSED = 0,
148 INTEL_EXCL_SHARED = 1,
149 INTEL_EXCL_EXCLUSIVE = 2,
150};
151
152struct intel_excl_states {
153 enum intel_excl_state_type state[X86_PMC_IDX_MAX];
154 bool sched_started;
155};
156
157struct intel_excl_cntrs {
158 raw_spinlock_t lock;
159
160 struct intel_excl_states states[2];
161
162 union {
163 u16 has_exclusive[2];
164 u32 exclusive_present;
165 };
166
167 int refcnt;
168 unsigned core_id;
169};
170
171struct x86_perf_task_context;
172#define MAX_LBR_ENTRIES 32
173
174enum {
175 X86_PERF_KFREE_SHARED = 0,
176 X86_PERF_KFREE_EXCL = 1,
177 X86_PERF_KFREE_MAX
178};
179
180struct cpu_hw_events {
181
182
183
184 struct perf_event *events[X86_PMC_IDX_MAX];
185 unsigned long active_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
186 unsigned long running[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
187 int enabled;
188
189 int n_events;
190 int n_added;
191
192 int n_txn;
193
194 int assign[X86_PMC_IDX_MAX];
195 u64 tags[X86_PMC_IDX_MAX];
196
197 struct perf_event *event_list[X86_PMC_IDX_MAX];
198 struct event_constraint *event_constraint[X86_PMC_IDX_MAX];
199
200 int n_excl;
201
202 unsigned int txn_flags;
203 int is_fake;
204
205
206
207
208 struct debug_store *ds;
209 void *ds_pebs_vaddr;
210 void *ds_bts_vaddr;
211 u64 pebs_enabled;
212 int n_pebs;
213 int n_large_pebs;
214
215
216 u64 pebs_data_cfg;
217 u64 active_pebs_data_cfg;
218 int pebs_record_size;
219
220
221
222
223 int lbr_users;
224 int lbr_pebs_users;
225 struct perf_branch_stack lbr_stack;
226 struct perf_branch_entry lbr_entries[MAX_LBR_ENTRIES];
227 struct er_account *lbr_sel;
228 u64 br_sel;
229 struct x86_perf_task_context *last_task_ctx;
230 int last_log_id;
231
232
233
234
235 u64 intel_ctrl_guest_mask;
236 u64 intel_ctrl_host_mask;
237 struct perf_guest_switch_msr guest_switch_msrs[X86_PMC_IDX_MAX];
238
239
240
241
242 u64 intel_cp_status;
243
244
245
246
247
248 struct intel_shared_regs *shared_regs;
249
250
251
252 struct event_constraint *constraint_list;
253 struct intel_excl_cntrs *excl_cntrs;
254 int excl_thread_id;
255
256
257
258
259 u64 tfa_shadow;
260
261
262
263
264 struct amd_nb *amd_nb;
265
266 u64 perf_ctr_virt_mask;
267
268 void *kfree_on_online[X86_PERF_KFREE_MAX];
269};
270
271#define __EVENT_CONSTRAINT_RANGE(c, e, n, m, w, o, f) { \
272 { .idxmsk64 = (n) }, \
273 .code = (c), \
274 .size = (e) - (c), \
275 .cmask = (m), \
276 .weight = (w), \
277 .overlap = (o), \
278 .flags = f, \
279}
280
281#define __EVENT_CONSTRAINT(c, n, m, w, o, f) \
282 __EVENT_CONSTRAINT_RANGE(c, c, n, m, w, o, f)
283
284#define EVENT_CONSTRAINT(c, n, m) \
285 __EVENT_CONSTRAINT(c, n, m, HWEIGHT(n), 0, 0)
286
287
288
289
290
291#define EVENT_CONSTRAINT_RANGE(c, e, n, m) \
292 __EVENT_CONSTRAINT_RANGE(c, e, n, m, HWEIGHT(n), 0, 0)
293
294#define INTEL_EXCLEVT_CONSTRAINT(c, n) \
295 __EVENT_CONSTRAINT(c, n, ARCH_PERFMON_EVENTSEL_EVENT, HWEIGHT(n),\
296 0, PERF_X86_EVENT_EXCL)
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319#define EVENT_CONSTRAINT_OVERLAP(c, n, m) \
320 __EVENT_CONSTRAINT(c, n, m, HWEIGHT(n), 1, 0)
321
322
323
324
325#define INTEL_EVENT_CONSTRAINT(c, n) \
326 EVENT_CONSTRAINT(c, n, ARCH_PERFMON_EVENTSEL_EVENT)
327
328
329
330
331#define INTEL_EVENT_CONSTRAINT_RANGE(c, e, n) \
332 EVENT_CONSTRAINT_RANGE(c, e, n, ARCH_PERFMON_EVENTSEL_EVENT)
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347#define FIXED_EVENT_FLAGS (X86_RAW_EVENT_MASK|HSW_IN_TX|HSW_IN_TX_CHECKPOINTED)
348#define FIXED_EVENT_CONSTRAINT(c, n) \
349 EVENT_CONSTRAINT(c, (1ULL << (32+n)), FIXED_EVENT_FLAGS)
350
351
352
353
354#define INTEL_UEVENT_CONSTRAINT(c, n) \
355 EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK)
356
357
358#define INTEL_UBIT_EVENT_CONSTRAINT(c, n) \
359 EVENT_CONSTRAINT(c, n, ARCH_PERFMON_EVENTSEL_EVENT|(c))
360
361
362#define INTEL_FLAGS_UEVENT_CONSTRAINT(c, n) \
363 EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS)
364
365#define INTEL_EXCLUEVT_CONSTRAINT(c, n) \
366 __EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK, \
367 HWEIGHT(n), 0, PERF_X86_EVENT_EXCL)
368
369#define INTEL_PLD_CONSTRAINT(c, n) \
370 __EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
371 HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_LDLAT)
372
373#define INTEL_PST_CONSTRAINT(c, n) \
374 __EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
375 HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_ST)
376
377
378#define INTEL_FLAGS_EVENT_CONSTRAINT(c, n) \
379 EVENT_CONSTRAINT(c, n, ARCH_PERFMON_EVENTSEL_EVENT|X86_ALL_EVENT_FLAGS)
380
381#define INTEL_FLAGS_EVENT_CONSTRAINT_RANGE(c, e, n) \
382 EVENT_CONSTRAINT_RANGE(c, e, n, ARCH_PERFMON_EVENTSEL_EVENT|X86_ALL_EVENT_FLAGS)
383
384
385#define INTEL_ALL_EVENT_CONSTRAINT(code, n) \
386 EVENT_CONSTRAINT(code, n, X86_ALL_EVENT_FLAGS)
387
388
389#define INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_ST(code, n) \
390 __EVENT_CONSTRAINT(code, n, \
391 ARCH_PERFMON_EVENTSEL_EVENT|X86_ALL_EVENT_FLAGS, \
392 HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_ST_HSW)
393
394
395#define INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_LD(code, n) \
396 __EVENT_CONSTRAINT(code, n, \
397 ARCH_PERFMON_EVENTSEL_EVENT|X86_ALL_EVENT_FLAGS, \
398 HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_LD_HSW)
399
400#define INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_LD_RANGE(code, end, n) \
401 __EVENT_CONSTRAINT_RANGE(code, end, n, \
402 ARCH_PERFMON_EVENTSEL_EVENT|X86_ALL_EVENT_FLAGS, \
403 HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_LD_HSW)
404
405#define INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_XLD(code, n) \
406 __EVENT_CONSTRAINT(code, n, \
407 ARCH_PERFMON_EVENTSEL_EVENT|X86_ALL_EVENT_FLAGS, \
408 HWEIGHT(n), 0, \
409 PERF_X86_EVENT_PEBS_LD_HSW|PERF_X86_EVENT_EXCL)
410
411
412#define INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(code, n) \
413 __EVENT_CONSTRAINT(code, n, \
414 INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
415 HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_ST_HSW)
416
417#define INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_XST(code, n) \
418 __EVENT_CONSTRAINT(code, n, \
419 INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
420 HWEIGHT(n), 0, \
421 PERF_X86_EVENT_PEBS_ST_HSW|PERF_X86_EVENT_EXCL)
422
423
424#define INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(code, n) \
425 __EVENT_CONSTRAINT(code, n, \
426 INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
427 HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_LD_HSW)
428
429#define INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_XLD(code, n) \
430 __EVENT_CONSTRAINT(code, n, \
431 INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
432 HWEIGHT(n), 0, \
433 PERF_X86_EVENT_PEBS_LD_HSW|PERF_X86_EVENT_EXCL)
434
435
436#define INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_NA(code, n) \
437 __EVENT_CONSTRAINT(code, n, \
438 INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
439 HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_NA_HSW)
440
441
442
443
444
445
446
447
448
449#define EVENT_CONSTRAINT_END { .weight = -1 }
450
451
452
453
454#define for_each_event_constraint(e, c) \
455 for ((e) = (c); (e)->weight != -1; (e)++)
456
457
458
459
460
461
462
463
464
465
466
467struct extra_reg {
468 unsigned int event;
469 unsigned int msr;
470 u64 config_mask;
471 u64 valid_mask;
472 int idx;
473 bool extra_msr_access;
474};
475
476#define EVENT_EXTRA_REG(e, ms, m, vm, i) { \
477 .event = (e), \
478 .msr = (ms), \
479 .config_mask = (m), \
480 .valid_mask = (vm), \
481 .idx = EXTRA_REG_##i, \
482 .extra_msr_access = true, \
483 }
484
485#define INTEL_EVENT_EXTRA_REG(event, msr, vm, idx) \
486 EVENT_EXTRA_REG(event, msr, ARCH_PERFMON_EVENTSEL_EVENT, vm, idx)
487
488#define INTEL_UEVENT_EXTRA_REG(event, msr, vm, idx) \
489 EVENT_EXTRA_REG(event, msr, ARCH_PERFMON_EVENTSEL_EVENT | \
490 ARCH_PERFMON_EVENTSEL_UMASK, vm, idx)
491
492#define INTEL_UEVENT_PEBS_LDLAT_EXTRA_REG(c) \
493 INTEL_UEVENT_EXTRA_REG(c, \
494 MSR_PEBS_LD_LAT_THRESHOLD, \
495 0xffff, \
496 LDLAT)
497
498#define EVENT_EXTRA_END EVENT_EXTRA_REG(0, 0, 0, 0, RSP_0)
499
500union perf_capabilities {
501 struct {
502 u64 lbr_format:6;
503 u64 pebs_trap:1;
504 u64 pebs_arch_reg:1;
505 u64 pebs_format:4;
506 u64 smm_freeze:1;
507
508
509
510
511 u64 full_width_write:1;
512 u64 pebs_baseline:1;
513 };
514 u64 capabilities;
515};
516
517struct x86_pmu_quirk {
518 struct x86_pmu_quirk *next;
519 void (*func)(void);
520};
521
522union x86_pmu_config {
523 struct {
524 u64 event:8,
525 umask:8,
526 usr:1,
527 os:1,
528 edge:1,
529 pc:1,
530 interrupt:1,
531 __reserved1:1,
532 en:1,
533 inv:1,
534 cmask:8,
535 event2:4,
536 __reserved2:4,
537 go:1,
538 ho:1;
539 } bits;
540 u64 value;
541};
542
543#define X86_CONFIG(args...) ((union x86_pmu_config){.bits = {args}}).value
544
545enum {
546 x86_lbr_exclusive_lbr,
547 x86_lbr_exclusive_bts,
548 x86_lbr_exclusive_pt,
549 x86_lbr_exclusive_max,
550};
551
552
553
554
555struct x86_pmu {
556
557
558
559 const char *name;
560 int version;
561 int (*handle_irq)(struct pt_regs *);
562 void (*disable_all)(void);
563 void (*enable_all)(int added);
564 void (*enable)(struct perf_event *);
565 void (*disable)(struct perf_event *);
566 void (*add)(struct perf_event *);
567 void (*del)(struct perf_event *);
568 void (*read)(struct perf_event *event);
569 int (*hw_config)(struct perf_event *event);
570 int (*schedule_events)(struct cpu_hw_events *cpuc, int n, int *assign);
571 unsigned eventsel;
572 unsigned perfctr;
573 int (*addr_offset)(int index, bool eventsel);
574 int (*rdpmc_index)(int index);
575 u64 (*event_map)(int);
576 int max_events;
577 int num_counters;
578 int num_counters_fixed;
579 int cntval_bits;
580 u64 cntval_mask;
581 union {
582 unsigned long events_maskl;
583 unsigned long events_mask[BITS_TO_LONGS(ARCH_PERFMON_EVENTS_COUNT)];
584 };
585 int events_mask_len;
586 int apic;
587 u64 max_period;
588 struct event_constraint *
589 (*get_event_constraints)(struct cpu_hw_events *cpuc,
590 int idx,
591 struct perf_event *event);
592
593 void (*put_event_constraints)(struct cpu_hw_events *cpuc,
594 struct perf_event *event);
595
596 void (*start_scheduling)(struct cpu_hw_events *cpuc);
597
598 void (*commit_scheduling)(struct cpu_hw_events *cpuc, int idx, int cntr);
599
600 void (*stop_scheduling)(struct cpu_hw_events *cpuc);
601
602 struct event_constraint *event_constraints;
603 struct x86_pmu_quirk *quirks;
604 int perfctr_second_write;
605 u64 (*limit_period)(struct perf_event *event, u64 l);
606
607
608 unsigned int late_ack :1,
609 counter_freezing :1;
610
611
612
613 int attr_rdpmc_broken;
614 int attr_rdpmc;
615 struct attribute **format_attrs;
616
617 ssize_t (*events_sysfs_show)(char *page, u64 config);
618 const struct attribute_group **attr_update;
619
620 unsigned long attr_freeze_on_smi;
621
622
623
624
625 int (*cpu_prepare)(int cpu);
626 void (*cpu_starting)(int cpu);
627 void (*cpu_dying)(int cpu);
628 void (*cpu_dead)(int cpu);
629
630 void (*check_microcode)(void);
631 void (*sched_task)(struct perf_event_context *ctx,
632 bool sched_in);
633
634
635
636
637 u64 intel_ctrl;
638 union perf_capabilities intel_cap;
639
640
641
642
643 unsigned int bts :1,
644 bts_active :1,
645 pebs :1,
646 pebs_active :1,
647 pebs_broken :1,
648 pebs_prec_dist :1,
649 pebs_no_tlb :1,
650 pebs_no_isolation :1;
651 int pebs_record_size;
652 int pebs_buffer_size;
653 int max_pebs_events;
654 void (*drain_pebs)(struct pt_regs *regs);
655 struct event_constraint *pebs_constraints;
656 void (*pebs_aliases)(struct perf_event *event);
657 unsigned long large_pebs_flags;
658 u64 rtm_abort_event;
659
660
661
662
663 unsigned long lbr_tos, lbr_from, lbr_to;
664 int lbr_nr;
665 u64 lbr_sel_mask;
666 const int *lbr_sel_map;
667 bool lbr_double_abort;
668 bool lbr_pt_coexist;
669
670
671
672
673 atomic_t lbr_exclusive[x86_lbr_exclusive_max];
674
675
676
677
678 unsigned int amd_nb_constraints : 1;
679
680
681
682
683 struct extra_reg *extra_regs;
684 unsigned int flags;
685
686
687
688
689 struct perf_guest_switch_msr *(*guest_get_msrs)(int *nr);
690
691
692
693
694 int (*check_period) (struct perf_event *event, u64 period);
695};
696
697struct x86_perf_task_context {
698 u64 lbr_from[MAX_LBR_ENTRIES];
699 u64 lbr_to[MAX_LBR_ENTRIES];
700 u64 lbr_info[MAX_LBR_ENTRIES];
701 int tos;
702 int valid_lbrs;
703 int lbr_callstack_users;
704 int lbr_stack_state;
705 int log_id;
706};
707
708#define x86_add_quirk(func_) \
709do { \
710 static struct x86_pmu_quirk __quirk __initdata = { \
711 .func = func_, \
712 }; \
713 __quirk.next = x86_pmu.quirks; \
714 x86_pmu.quirks = &__quirk; \
715} while (0)
716
717
718
719
720#define PMU_FL_NO_HT_SHARING 0x1
721#define PMU_FL_HAS_RSP_1 0x2
722#define PMU_FL_EXCL_CNTRS 0x4
723#define PMU_FL_EXCL_ENABLED 0x8
724#define PMU_FL_PEBS_ALL 0x10
725#define PMU_FL_TFA 0x20
726
727#define EVENT_VAR(_id) event_attr_##_id
728#define EVENT_PTR(_id) &event_attr_##_id.attr.attr
729
730#define EVENT_ATTR(_name, _id) \
731static struct perf_pmu_events_attr EVENT_VAR(_id) = { \
732 .attr = __ATTR(_name, 0444, events_sysfs_show, NULL), \
733 .id = PERF_COUNT_HW_##_id, \
734 .event_str = NULL, \
735};
736
737#define EVENT_ATTR_STR(_name, v, str) \
738static struct perf_pmu_events_attr event_attr_##v = { \
739 .attr = __ATTR(_name, 0444, events_sysfs_show, NULL), \
740 .id = 0, \
741 .event_str = str, \
742};
743
744#define EVENT_ATTR_STR_HT(_name, v, noht, ht) \
745static struct perf_pmu_events_ht_attr event_attr_##v = { \
746 .attr = __ATTR(_name, 0444, events_ht_sysfs_show, NULL),\
747 .id = 0, \
748 .event_str_noht = noht, \
749 .event_str_ht = ht, \
750}
751
752struct pmu *x86_get_pmu(void);
753extern struct x86_pmu x86_pmu __read_mostly;
754
755static inline bool x86_pmu_has_lbr_callstack(void)
756{
757 return x86_pmu.lbr_sel_map &&
758 x86_pmu.lbr_sel_map[PERF_SAMPLE_BRANCH_CALL_STACK_SHIFT] > 0;
759}
760
761DECLARE_PER_CPU(struct cpu_hw_events, cpu_hw_events);
762
763int x86_perf_event_set_period(struct perf_event *event);
764
765
766
767
768
769
770
771
772
773#define C(x) PERF_COUNT_HW_CACHE_##x
774
775extern u64 __read_mostly hw_cache_event_ids
776 [PERF_COUNT_HW_CACHE_MAX]
777 [PERF_COUNT_HW_CACHE_OP_MAX]
778 [PERF_COUNT_HW_CACHE_RESULT_MAX];
779extern u64 __read_mostly hw_cache_extra_regs
780 [PERF_COUNT_HW_CACHE_MAX]
781 [PERF_COUNT_HW_CACHE_OP_MAX]
782 [PERF_COUNT_HW_CACHE_RESULT_MAX];
783
784u64 x86_perf_event_update(struct perf_event *event);
785
786static inline unsigned int x86_pmu_config_addr(int index)
787{
788 return x86_pmu.eventsel + (x86_pmu.addr_offset ?
789 x86_pmu.addr_offset(index, true) : index);
790}
791
792static inline unsigned int x86_pmu_event_addr(int index)
793{
794 return x86_pmu.perfctr + (x86_pmu.addr_offset ?
795 x86_pmu.addr_offset(index, false) : index);
796}
797
798static inline int x86_pmu_rdpmc_index(int index)
799{
800 return x86_pmu.rdpmc_index ? x86_pmu.rdpmc_index(index) : index;
801}
802
803int x86_add_exclusive(unsigned int what);
804
805void x86_del_exclusive(unsigned int what);
806
807int x86_reserve_hardware(void);
808
809void x86_release_hardware(void);
810
811int x86_pmu_max_precise(void);
812
813void hw_perf_lbr_event_destroy(struct perf_event *event);
814
815int x86_setup_perfctr(struct perf_event *event);
816
817int x86_pmu_hw_config(struct perf_event *event);
818
819void x86_pmu_disable_all(void);
820
821static inline void __x86_pmu_enable_event(struct hw_perf_event *hwc,
822 u64 enable_mask)
823{
824 u64 disable_mask = __this_cpu_read(cpu_hw_events.perf_ctr_virt_mask);
825
826 if (hwc->extra_reg.reg)
827 wrmsrl(hwc->extra_reg.reg, hwc->extra_reg.config);
828 wrmsrl(hwc->config_base, (hwc->config | enable_mask) & ~disable_mask);
829}
830
831void x86_pmu_enable_all(int added);
832
833int perf_assign_events(struct event_constraint **constraints, int n,
834 int wmin, int wmax, int gpmax, int *assign);
835int x86_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign);
836
837void x86_pmu_stop(struct perf_event *event, int flags);
838
839static inline void x86_pmu_disable_event(struct perf_event *event)
840{
841 struct hw_perf_event *hwc = &event->hw;
842
843 wrmsrl(hwc->config_base, hwc->config);
844}
845
846void x86_pmu_enable_event(struct perf_event *event);
847
848int x86_pmu_handle_irq(struct pt_regs *regs);
849
850extern struct event_constraint emptyconstraint;
851
852extern struct event_constraint unconstrained;
853
854static inline bool kernel_ip(unsigned long ip)
855{
856#ifdef CONFIG_X86_32
857 return ip > PAGE_OFFSET;
858#else
859 return (long)ip < 0;
860#endif
861}
862
863
864
865
866
867
868
869
870
871
872
873
874
875static inline void set_linear_ip(struct pt_regs *regs, unsigned long ip)
876{
877 regs->cs = kernel_ip(ip) ? __KERNEL_CS : __USER_CS;
878 if (regs->flags & X86_VM_MASK)
879 regs->flags ^= (PERF_EFLAGS_VM | X86_VM_MASK);
880 regs->ip = ip;
881}
882
883ssize_t x86_event_sysfs_show(char *page, u64 config, u64 event);
884ssize_t intel_event_sysfs_show(char *page, u64 config);
885
886ssize_t events_sysfs_show(struct device *dev, struct device_attribute *attr,
887 char *page);
888ssize_t events_ht_sysfs_show(struct device *dev, struct device_attribute *attr,
889 char *page);
890
891#ifdef CONFIG_CPU_SUP_AMD
892
893int amd_pmu_init(void);
894
895#else
896
897static inline int amd_pmu_init(void)
898{
899 return 0;
900}
901
902#endif
903
904#ifdef CONFIG_CPU_SUP_INTEL
905
906static inline bool intel_pmu_has_bts_period(struct perf_event *event, u64 period)
907{
908 struct hw_perf_event *hwc = &event->hw;
909 unsigned int hw_event, bts_event;
910
911 if (event->attr.freq)
912 return false;
913
914 hw_event = hwc->config & INTEL_ARCH_EVENT_MASK;
915 bts_event = x86_pmu.event_map(PERF_COUNT_HW_BRANCH_INSTRUCTIONS);
916
917 return hw_event == bts_event && period == 1;
918}
919
920static inline bool intel_pmu_has_bts(struct perf_event *event)
921{
922 struct hw_perf_event *hwc = &event->hw;
923
924 return intel_pmu_has_bts_period(event, hwc->sample_period);
925}
926
927int intel_pmu_save_and_restart(struct perf_event *event);
928
929struct event_constraint *
930x86_get_event_constraints(struct cpu_hw_events *cpuc, int idx,
931 struct perf_event *event);
932
933extern int intel_cpuc_prepare(struct cpu_hw_events *cpuc, int cpu);
934extern void intel_cpuc_finish(struct cpu_hw_events *cpuc);
935
936int intel_pmu_init(void);
937
938void init_debug_store_on_cpu(int cpu);
939
940void fini_debug_store_on_cpu(int cpu);
941
942void release_ds_buffers(void);
943
944void reserve_ds_buffers(void);
945
946extern struct event_constraint bts_constraint;
947
948void intel_pmu_enable_bts(u64 config);
949
950void intel_pmu_disable_bts(void);
951
952int intel_pmu_drain_bts_buffer(void);
953
954extern struct event_constraint intel_core2_pebs_event_constraints[];
955
956extern struct event_constraint intel_atom_pebs_event_constraints[];
957
958extern struct event_constraint intel_slm_pebs_event_constraints[];
959
960extern struct event_constraint intel_glm_pebs_event_constraints[];
961
962extern struct event_constraint intel_glp_pebs_event_constraints[];
963
964extern struct event_constraint intel_nehalem_pebs_event_constraints[];
965
966extern struct event_constraint intel_westmere_pebs_event_constraints[];
967
968extern struct event_constraint intel_snb_pebs_event_constraints[];
969
970extern struct event_constraint intel_ivb_pebs_event_constraints[];
971
972extern struct event_constraint intel_hsw_pebs_event_constraints[];
973
974extern struct event_constraint intel_bdw_pebs_event_constraints[];
975
976extern struct event_constraint intel_skl_pebs_event_constraints[];
977
978extern struct event_constraint intel_icl_pebs_event_constraints[];
979
980struct event_constraint *intel_pebs_constraints(struct perf_event *event);
981
982void intel_pmu_pebs_add(struct perf_event *event);
983
984void intel_pmu_pebs_del(struct perf_event *event);
985
986void intel_pmu_pebs_enable(struct perf_event *event);
987
988void intel_pmu_pebs_disable(struct perf_event *event);
989
990void intel_pmu_pebs_enable_all(void);
991
992void intel_pmu_pebs_disable_all(void);
993
994void intel_pmu_pebs_sched_task(struct perf_event_context *ctx, bool sched_in);
995
996void intel_pmu_auto_reload_read(struct perf_event *event);
997
998void intel_pmu_store_pebs_lbrs(struct pebs_lbr *lbr);
999
1000void intel_ds_init(void);
1001
1002void intel_pmu_lbr_sched_task(struct perf_event_context *ctx, bool sched_in);
1003
1004u64 lbr_from_signext_quirk_wr(u64 val);
1005
1006void intel_pmu_lbr_reset(void);
1007
1008void intel_pmu_lbr_add(struct perf_event *event);
1009
1010void intel_pmu_lbr_del(struct perf_event *event);
1011
1012void intel_pmu_lbr_enable_all(bool pmi);
1013
1014void intel_pmu_lbr_disable_all(void);
1015
1016void intel_pmu_lbr_read(void);
1017
1018void intel_pmu_lbr_init_core(void);
1019
1020void intel_pmu_lbr_init_nhm(void);
1021
1022void intel_pmu_lbr_init_atom(void);
1023
1024void intel_pmu_lbr_init_slm(void);
1025
1026void intel_pmu_lbr_init_snb(void);
1027
1028void intel_pmu_lbr_init_hsw(void);
1029
1030void intel_pmu_lbr_init_skl(void);
1031
1032void intel_pmu_lbr_init_knl(void);
1033
1034void intel_pmu_pebs_data_source_nhm(void);
1035
1036void intel_pmu_pebs_data_source_skl(bool pmem);
1037
1038int intel_pmu_setup_lbr_filter(struct perf_event *event);
1039
1040void intel_pt_interrupt(void);
1041
1042int intel_bts_interrupt(void);
1043
1044void intel_bts_enable_local(void);
1045
1046void intel_bts_disable_local(void);
1047
1048int p4_pmu_init(void);
1049
1050int p6_pmu_init(void);
1051
1052int knc_pmu_init(void);
1053
1054static inline int is_ht_workaround_enabled(void)
1055{
1056 return !!(x86_pmu.flags & PMU_FL_EXCL_ENABLED);
1057}
1058
1059#else
1060
1061static inline void reserve_ds_buffers(void)
1062{
1063}
1064
1065static inline void release_ds_buffers(void)
1066{
1067}
1068
1069static inline int intel_pmu_init(void)
1070{
1071 return 0;
1072}
1073
1074static inline int intel_cpuc_prepare(struct cpu_hw_events *cpuc, int cpu)
1075{
1076 return 0;
1077}
1078
1079static inline void intel_cpuc_finish(struct cpu_hw_events *cpuc)
1080{
1081}
1082
1083static inline int is_ht_workaround_enabled(void)
1084{
1085 return 0;
1086}
1087#endif
1088