1
2
3
4
5
6
7
8
9
10
11
12
13
14
15#include <linux/perf_event.h>
16#include <linux/capability.h>
17#include <linux/notifier.h>
18#include <linux/hardirq.h>
19#include <linux/kprobes.h>
20#include <linux/export.h>
21#include <linux/init.h>
22#include <linux/kdebug.h>
23#include <linux/sched/mm.h>
24#include <linux/sched/clock.h>
25#include <linux/uaccess.h>
26#include <linux/slab.h>
27#include <linux/cpu.h>
28#include <linux/bitops.h>
29#include <linux/device.h>
30#include <linux/nospec.h>
31
32#include <asm/apic.h>
33#include <asm/stacktrace.h>
34#include <asm/nmi.h>
35#include <asm/smp.h>
36#include <asm/alternative.h>
37#include <asm/mmu_context.h>
38#include <asm/tlbflush.h>
39#include <asm/timer.h>
40#include <asm/desc.h>
41#include <asm/ldt.h>
42#include <asm/unwind.h>
43
44#include "perf_event.h"
45
46struct x86_pmu x86_pmu __read_mostly;
47
48DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = {
49 .enabled = 1,
50};
51
52DEFINE_STATIC_KEY_FALSE(rdpmc_never_available_key);
53DEFINE_STATIC_KEY_FALSE(rdpmc_always_available_key);
54
55u64 __read_mostly hw_cache_event_ids
56 [PERF_COUNT_HW_CACHE_MAX]
57 [PERF_COUNT_HW_CACHE_OP_MAX]
58 [PERF_COUNT_HW_CACHE_RESULT_MAX];
59u64 __read_mostly hw_cache_extra_regs
60 [PERF_COUNT_HW_CACHE_MAX]
61 [PERF_COUNT_HW_CACHE_OP_MAX]
62 [PERF_COUNT_HW_CACHE_RESULT_MAX];
63
64
65
66
67
68
69u64 x86_perf_event_update(struct perf_event *event)
70{
71 struct hw_perf_event *hwc = &event->hw;
72 int shift = 64 - x86_pmu.cntval_bits;
73 u64 prev_raw_count, new_raw_count;
74 u64 delta;
75
76 if (unlikely(!hwc->event_base))
77 return 0;
78
79
80
81
82
83
84
85
86again:
87 prev_raw_count = local64_read(&hwc->prev_count);
88 rdpmcl(hwc->event_base_rdpmc, new_raw_count);
89
90 if (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
91 new_raw_count) != prev_raw_count)
92 goto again;
93
94
95
96
97
98
99
100
101
102 delta = (new_raw_count << shift) - (prev_raw_count << shift);
103 delta >>= shift;
104
105 local64_add(delta, &event->count);
106 local64_sub(delta, &hwc->period_left);
107
108 return new_raw_count;
109}
110
111
112
113
114static int x86_pmu_extra_regs(u64 config, struct perf_event *event)
115{
116 struct hw_perf_event_extra *reg;
117 struct extra_reg *er;
118
119 reg = &event->hw.extra_reg;
120
121 if (!x86_pmu.extra_regs)
122 return 0;
123
124 for (er = x86_pmu.extra_regs; er->msr; er++) {
125 if (er->event != (config & er->config_mask))
126 continue;
127 if (event->attr.config1 & ~er->valid_mask)
128 return -EINVAL;
129
130 if (!er->extra_msr_access)
131 return -ENXIO;
132
133 reg->idx = er->idx;
134 reg->config = event->attr.config1;
135 reg->reg = er->msr;
136 break;
137 }
138 return 0;
139}
140
141static atomic_t active_events;
142static atomic_t pmc_refcount;
143static DEFINE_MUTEX(pmc_reserve_mutex);
144
145#ifdef CONFIG_X86_LOCAL_APIC
146
147static bool reserve_pmc_hardware(void)
148{
149 int i;
150
151 for (i = 0; i < x86_pmu.num_counters; i++) {
152 if (!reserve_perfctr_nmi(x86_pmu_event_addr(i)))
153 goto perfctr_fail;
154 }
155
156 for (i = 0; i < x86_pmu.num_counters; i++) {
157 if (!reserve_evntsel_nmi(x86_pmu_config_addr(i)))
158 goto eventsel_fail;
159 }
160
161 return true;
162
163eventsel_fail:
164 for (i--; i >= 0; i--)
165 release_evntsel_nmi(x86_pmu_config_addr(i));
166
167 i = x86_pmu.num_counters;
168
169perfctr_fail:
170 for (i--; i >= 0; i--)
171 release_perfctr_nmi(x86_pmu_event_addr(i));
172
173 return false;
174}
175
176static void release_pmc_hardware(void)
177{
178 int i;
179
180 for (i = 0; i < x86_pmu.num_counters; i++) {
181 release_perfctr_nmi(x86_pmu_event_addr(i));
182 release_evntsel_nmi(x86_pmu_config_addr(i));
183 }
184}
185
186#else
187
188static bool reserve_pmc_hardware(void) { return true; }
189static void release_pmc_hardware(void) {}
190
191#endif
192
193static bool check_hw_exists(void)
194{
195 u64 val, val_fail = -1, val_new= ~0;
196 int i, reg, reg_fail = -1, ret = 0;
197 int bios_fail = 0;
198 int reg_safe = -1;
199
200
201
202
203
204 for (i = 0; i < x86_pmu.num_counters; i++) {
205 reg = x86_pmu_config_addr(i);
206 ret = rdmsrl_safe(reg, &val);
207 if (ret)
208 goto msr_fail;
209 if (val & ARCH_PERFMON_EVENTSEL_ENABLE) {
210 bios_fail = 1;
211 val_fail = val;
212 reg_fail = reg;
213 } else {
214 reg_safe = i;
215 }
216 }
217
218 if (x86_pmu.num_counters_fixed) {
219 reg = MSR_ARCH_PERFMON_FIXED_CTR_CTRL;
220 ret = rdmsrl_safe(reg, &val);
221 if (ret)
222 goto msr_fail;
223 for (i = 0; i < x86_pmu.num_counters_fixed; i++) {
224 if (val & (0x03 << i*4)) {
225 bios_fail = 1;
226 val_fail = val;
227 reg_fail = reg;
228 }
229 }
230 }
231
232
233
234
235
236
237
238 if (reg_safe == -1) {
239 reg = reg_safe;
240 goto msr_fail;
241 }
242
243
244
245
246
247
248 reg = x86_pmu_event_addr(reg_safe);
249 if (rdmsrl_safe(reg, &val))
250 goto msr_fail;
251 val ^= 0xffffUL;
252 ret = wrmsrl_safe(reg, val);
253 ret |= rdmsrl_safe(reg, &val_new);
254 if (ret || val != val_new)
255 goto msr_fail;
256
257
258
259
260 if (bios_fail) {
261 pr_cont("Broken BIOS detected, complain to your hardware vendor.\n");
262 pr_err(FW_BUG "the BIOS has corrupted hw-PMU resources (MSR %x is %Lx)\n",
263 reg_fail, val_fail);
264 }
265
266 return true;
267
268msr_fail:
269 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) {
270 pr_cont("PMU not available due to virtualization, using software events only.\n");
271 } else {
272 pr_cont("Broken PMU hardware detected, using software events only.\n");
273 pr_err("Failed to access perfctr msr (MSR %x is %Lx)\n",
274 reg, val_new);
275 }
276
277 return false;
278}
279
280static void hw_perf_event_destroy(struct perf_event *event)
281{
282 x86_release_hardware();
283 atomic_dec(&active_events);
284}
285
286void hw_perf_lbr_event_destroy(struct perf_event *event)
287{
288 hw_perf_event_destroy(event);
289
290
291 x86_del_exclusive(x86_lbr_exclusive_lbr);
292}
293
294static inline int x86_pmu_initialized(void)
295{
296 return x86_pmu.handle_irq != NULL;
297}
298
299static inline int
300set_ext_hw_attr(struct hw_perf_event *hwc, struct perf_event *event)
301{
302 struct perf_event_attr *attr = &event->attr;
303 unsigned int cache_type, cache_op, cache_result;
304 u64 config, val;
305
306 config = attr->config;
307
308 cache_type = (config >> 0) & 0xff;
309 if (cache_type >= PERF_COUNT_HW_CACHE_MAX)
310 return -EINVAL;
311 cache_type = array_index_nospec(cache_type, PERF_COUNT_HW_CACHE_MAX);
312
313 cache_op = (config >> 8) & 0xff;
314 if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX)
315 return -EINVAL;
316 cache_op = array_index_nospec(cache_op, PERF_COUNT_HW_CACHE_OP_MAX);
317
318 cache_result = (config >> 16) & 0xff;
319 if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
320 return -EINVAL;
321 cache_result = array_index_nospec(cache_result, PERF_COUNT_HW_CACHE_RESULT_MAX);
322
323 val = hw_cache_event_ids[cache_type][cache_op][cache_result];
324
325 if (val == 0)
326 return -ENOENT;
327
328 if (val == -1)
329 return -EINVAL;
330
331 hwc->config |= val;
332 attr->config1 = hw_cache_extra_regs[cache_type][cache_op][cache_result];
333 return x86_pmu_extra_regs(val, event);
334}
335
336int x86_reserve_hardware(void)
337{
338 int err = 0;
339
340 if (!atomic_inc_not_zero(&pmc_refcount)) {
341 mutex_lock(&pmc_reserve_mutex);
342 if (atomic_read(&pmc_refcount) == 0) {
343 if (!reserve_pmc_hardware())
344 err = -EBUSY;
345 else
346 reserve_ds_buffers();
347 }
348 if (!err)
349 atomic_inc(&pmc_refcount);
350 mutex_unlock(&pmc_reserve_mutex);
351 }
352
353 return err;
354}
355
356void x86_release_hardware(void)
357{
358 if (atomic_dec_and_mutex_lock(&pmc_refcount, &pmc_reserve_mutex)) {
359 release_pmc_hardware();
360 release_ds_buffers();
361 release_lbr_buffers();
362 mutex_unlock(&pmc_reserve_mutex);
363 }
364}
365
366
367
368
369
370int x86_add_exclusive(unsigned int what)
371{
372 int i;
373
374
375
376
377
378 if (x86_pmu.lbr_pt_coexist && what == x86_lbr_exclusive_pt)
379 goto out;
380
381 if (!atomic_inc_not_zero(&x86_pmu.lbr_exclusive[what])) {
382 mutex_lock(&pmc_reserve_mutex);
383 for (i = 0; i < ARRAY_SIZE(x86_pmu.lbr_exclusive); i++) {
384 if (i != what && atomic_read(&x86_pmu.lbr_exclusive[i]))
385 goto fail_unlock;
386 }
387 atomic_inc(&x86_pmu.lbr_exclusive[what]);
388 mutex_unlock(&pmc_reserve_mutex);
389 }
390
391out:
392 atomic_inc(&active_events);
393 return 0;
394
395fail_unlock:
396 mutex_unlock(&pmc_reserve_mutex);
397 return -EBUSY;
398}
399
400void x86_del_exclusive(unsigned int what)
401{
402 atomic_dec(&active_events);
403
404
405
406
407 if (x86_pmu.lbr_pt_coexist && what == x86_lbr_exclusive_pt)
408 return;
409
410 atomic_dec(&x86_pmu.lbr_exclusive[what]);
411}
412
413int x86_setup_perfctr(struct perf_event *event)
414{
415 struct perf_event_attr *attr = &event->attr;
416 struct hw_perf_event *hwc = &event->hw;
417 u64 config;
418
419 if (!is_sampling_event(event)) {
420 hwc->sample_period = x86_pmu.max_period;
421 hwc->last_period = hwc->sample_period;
422 local64_set(&hwc->period_left, hwc->sample_period);
423 }
424
425 if (attr->type == PERF_TYPE_RAW)
426 return x86_pmu_extra_regs(event->attr.config, event);
427
428 if (attr->type == PERF_TYPE_HW_CACHE)
429 return set_ext_hw_attr(hwc, event);
430
431 if (attr->config >= x86_pmu.max_events)
432 return -EINVAL;
433
434 attr->config = array_index_nospec((unsigned long)attr->config, x86_pmu.max_events);
435
436
437
438
439 config = x86_pmu.event_map(attr->config);
440
441 if (config == 0)
442 return -ENOENT;
443
444 if (config == -1LL)
445 return -EINVAL;
446
447 hwc->config |= config;
448
449 return 0;
450}
451
452
453
454
455
456
457
458static inline int precise_br_compat(struct perf_event *event)
459{
460 u64 m = event->attr.branch_sample_type;
461 u64 b = 0;
462
463
464 if (!(m & PERF_SAMPLE_BRANCH_ANY))
465 return 0;
466
467 m &= PERF_SAMPLE_BRANCH_KERNEL | PERF_SAMPLE_BRANCH_USER;
468
469 if (!event->attr.exclude_user)
470 b |= PERF_SAMPLE_BRANCH_USER;
471
472 if (!event->attr.exclude_kernel)
473 b |= PERF_SAMPLE_BRANCH_KERNEL;
474
475
476
477
478
479 return m == b;
480}
481
482int x86_pmu_max_precise(void)
483{
484 int precise = 0;
485
486
487 if (x86_pmu.pebs_active && !x86_pmu.pebs_broken) {
488 precise++;
489
490
491 if (x86_pmu.lbr_nr || x86_pmu.intel_cap.pebs_format >= 2)
492 precise++;
493
494 if (x86_pmu.pebs_prec_dist)
495 precise++;
496 }
497 return precise;
498}
499
500int x86_pmu_hw_config(struct perf_event *event)
501{
502 if (event->attr.precise_ip) {
503 int precise = x86_pmu_max_precise();
504
505 if (event->attr.precise_ip > precise)
506 return -EOPNOTSUPP;
507
508
509 if (!is_sampling_event(event))
510 return -EINVAL;
511 }
512
513
514
515
516 if (event->attr.precise_ip > 1 && x86_pmu.intel_cap.pebs_format < 2) {
517 u64 *br_type = &event->attr.branch_sample_type;
518
519 if (has_branch_stack(event)) {
520 if (!precise_br_compat(event))
521 return -EOPNOTSUPP;
522
523
524
525 } else {
526
527
528
529
530
531
532
533 *br_type = PERF_SAMPLE_BRANCH_ANY;
534
535 if (!event->attr.exclude_user)
536 *br_type |= PERF_SAMPLE_BRANCH_USER;
537
538 if (!event->attr.exclude_kernel)
539 *br_type |= PERF_SAMPLE_BRANCH_KERNEL;
540 }
541 }
542
543 if (event->attr.branch_sample_type & PERF_SAMPLE_BRANCH_CALL_STACK)
544 event->attach_state |= PERF_ATTACH_TASK_DATA;
545
546
547
548
549
550 event->hw.config = ARCH_PERFMON_EVENTSEL_INT;
551
552
553
554
555 if (!event->attr.exclude_user)
556 event->hw.config |= ARCH_PERFMON_EVENTSEL_USR;
557 if (!event->attr.exclude_kernel)
558 event->hw.config |= ARCH_PERFMON_EVENTSEL_OS;
559
560 if (event->attr.type == PERF_TYPE_RAW)
561 event->hw.config |= event->attr.config & X86_RAW_EVENT_MASK;
562
563 if (event->attr.sample_period && x86_pmu.limit_period) {
564 if (x86_pmu.limit_period(event, event->attr.sample_period) >
565 event->attr.sample_period)
566 return -EINVAL;
567 }
568
569
570 if (unlikely(event->attr.sample_regs_user & PERF_REG_EXTENDED_MASK))
571 return -EINVAL;
572
573
574
575
576 if (unlikely(event->attr.sample_regs_intr & PERF_REG_EXTENDED_MASK)) {
577 if (!(event->pmu->capabilities & PERF_PMU_CAP_EXTENDED_REGS))
578 return -EINVAL;
579
580 if (!event->attr.precise_ip)
581 return -EINVAL;
582 }
583
584 return x86_setup_perfctr(event);
585}
586
587
588
589
590static int __x86_pmu_event_init(struct perf_event *event)
591{
592 int err;
593
594 if (!x86_pmu_initialized())
595 return -ENODEV;
596
597 err = x86_reserve_hardware();
598 if (err)
599 return err;
600
601 atomic_inc(&active_events);
602 event->destroy = hw_perf_event_destroy;
603
604 event->hw.idx = -1;
605 event->hw.last_cpu = -1;
606 event->hw.last_tag = ~0ULL;
607
608
609 event->hw.extra_reg.idx = EXTRA_REG_NONE;
610 event->hw.branch_reg.idx = EXTRA_REG_NONE;
611
612 return x86_pmu.hw_config(event);
613}
614
615void x86_pmu_disable_all(void)
616{
617 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
618 int idx;
619
620 for (idx = 0; idx < x86_pmu.num_counters; idx++) {
621 struct hw_perf_event *hwc = &cpuc->events[idx]->hw;
622 u64 val;
623
624 if (!test_bit(idx, cpuc->active_mask))
625 continue;
626 rdmsrl(x86_pmu_config_addr(idx), val);
627 if (!(val & ARCH_PERFMON_EVENTSEL_ENABLE))
628 continue;
629 val &= ~ARCH_PERFMON_EVENTSEL_ENABLE;
630 wrmsrl(x86_pmu_config_addr(idx), val);
631 if (is_counter_pair(hwc))
632 wrmsrl(x86_pmu_config_addr(idx + 1), 0);
633 }
634}
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649static void x86_pmu_disable(struct pmu *pmu)
650{
651 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
652
653 if (!x86_pmu_initialized())
654 return;
655
656 if (!cpuc->enabled)
657 return;
658
659 cpuc->n_added = 0;
660 cpuc->enabled = 0;
661 barrier();
662
663 x86_pmu.disable_all();
664}
665
666void x86_pmu_enable_all(int added)
667{
668 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
669 int idx;
670
671 for (idx = 0; idx < x86_pmu.num_counters; idx++) {
672 struct hw_perf_event *hwc = &cpuc->events[idx]->hw;
673
674 if (!test_bit(idx, cpuc->active_mask))
675 continue;
676
677 __x86_pmu_enable_event(hwc, ARCH_PERFMON_EVENTSEL_ENABLE);
678 }
679}
680
681static struct pmu pmu;
682
683static inline int is_x86_event(struct perf_event *event)
684{
685 return event->pmu == &pmu;
686}
687
688struct pmu *x86_get_pmu(void)
689{
690 return &pmu;
691}
692
693
694
695
696
697
698
699struct sched_state {
700 int weight;
701 int event;
702 int counter;
703 int unassigned;
704 int nr_gp;
705 u64 used;
706};
707
708
709#define SCHED_STATES_MAX 2
710
711struct perf_sched {
712 int max_weight;
713 int max_events;
714 int max_gp;
715 int saved_states;
716 struct event_constraint **constraints;
717 struct sched_state state;
718 struct sched_state saved[SCHED_STATES_MAX];
719};
720
721
722
723
724static void perf_sched_init(struct perf_sched *sched, struct event_constraint **constraints,
725 int num, int wmin, int wmax, int gpmax)
726{
727 int idx;
728
729 memset(sched, 0, sizeof(*sched));
730 sched->max_events = num;
731 sched->max_weight = wmax;
732 sched->max_gp = gpmax;
733 sched->constraints = constraints;
734
735 for (idx = 0; idx < num; idx++) {
736 if (constraints[idx]->weight == wmin)
737 break;
738 }
739
740 sched->state.event = idx;
741 sched->state.weight = wmin;
742 sched->state.unassigned = num;
743}
744
745static void perf_sched_save_state(struct perf_sched *sched)
746{
747 if (WARN_ON_ONCE(sched->saved_states >= SCHED_STATES_MAX))
748 return;
749
750 sched->saved[sched->saved_states] = sched->state;
751 sched->saved_states++;
752}
753
754static bool perf_sched_restore_state(struct perf_sched *sched)
755{
756 if (!sched->saved_states)
757 return false;
758
759 sched->saved_states--;
760 sched->state = sched->saved[sched->saved_states];
761
762
763
764 sched->state.used &= ~BIT_ULL(sched->state.counter);
765
766
767 sched->state.counter++;
768
769 return true;
770}
771
772
773
774
775
776static bool __perf_sched_find_counter(struct perf_sched *sched)
777{
778 struct event_constraint *c;
779 int idx;
780
781 if (!sched->state.unassigned)
782 return false;
783
784 if (sched->state.event >= sched->max_events)
785 return false;
786
787 c = sched->constraints[sched->state.event];
788
789 if (c->idxmsk64 & (~0ULL << INTEL_PMC_IDX_FIXED)) {
790 idx = INTEL_PMC_IDX_FIXED;
791 for_each_set_bit_from(idx, c->idxmsk, X86_PMC_IDX_MAX) {
792 u64 mask = BIT_ULL(idx);
793
794 if (sched->state.used & mask)
795 continue;
796
797 sched->state.used |= mask;
798 goto done;
799 }
800 }
801
802
803 idx = sched->state.counter;
804 for_each_set_bit_from(idx, c->idxmsk, INTEL_PMC_IDX_FIXED) {
805 u64 mask = BIT_ULL(idx);
806
807 if (c->flags & PERF_X86_EVENT_PAIR)
808 mask |= mask << 1;
809
810 if (sched->state.used & mask)
811 continue;
812
813 if (sched->state.nr_gp++ >= sched->max_gp)
814 return false;
815
816 sched->state.used |= mask;
817 goto done;
818 }
819
820 return false;
821
822done:
823 sched->state.counter = idx;
824
825 if (c->overlap)
826 perf_sched_save_state(sched);
827
828 return true;
829}
830
831static bool perf_sched_find_counter(struct perf_sched *sched)
832{
833 while (!__perf_sched_find_counter(sched)) {
834 if (!perf_sched_restore_state(sched))
835 return false;
836 }
837
838 return true;
839}
840
841
842
843
844
845static bool perf_sched_next_event(struct perf_sched *sched)
846{
847 struct event_constraint *c;
848
849 if (!sched->state.unassigned || !--sched->state.unassigned)
850 return false;
851
852 do {
853
854 sched->state.event++;
855 if (sched->state.event >= sched->max_events) {
856
857 sched->state.event = 0;
858 sched->state.weight++;
859 if (sched->state.weight > sched->max_weight)
860 return false;
861 }
862 c = sched->constraints[sched->state.event];
863 } while (c->weight != sched->state.weight);
864
865 sched->state.counter = 0;
866
867 return true;
868}
869
870
871
872
873int perf_assign_events(struct event_constraint **constraints, int n,
874 int wmin, int wmax, int gpmax, int *assign)
875{
876 struct perf_sched sched;
877
878 perf_sched_init(&sched, constraints, n, wmin, wmax, gpmax);
879
880 do {
881 if (!perf_sched_find_counter(&sched))
882 break;
883 if (assign)
884 assign[sched.state.event] = sched.state.counter;
885 } while (perf_sched_next_event(&sched));
886
887 return sched.state.unassigned;
888}
889EXPORT_SYMBOL_GPL(perf_assign_events);
890
891int x86_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign)
892{
893 struct event_constraint *c;
894 struct perf_event *e;
895 int n0, i, wmin, wmax, unsched = 0;
896 struct hw_perf_event *hwc;
897 u64 used_mask = 0;
898
899
900
901
902
903
904
905
906 n0 = cpuc->n_events;
907 if (cpuc->txn_flags & PERF_PMU_TXN_ADD)
908 n0 -= cpuc->n_txn;
909
910 if (x86_pmu.start_scheduling)
911 x86_pmu.start_scheduling(cpuc);
912
913 for (i = 0, wmin = X86_PMC_IDX_MAX, wmax = 0; i < n; i++) {
914 c = cpuc->event_constraint[i];
915
916
917
918
919
920 WARN_ON_ONCE((c && i >= n0) || (!c && i < n0));
921
922
923
924
925
926
927 if (!c || (c->flags & PERF_X86_EVENT_DYNAMIC)) {
928 c = x86_pmu.get_event_constraints(cpuc, i, cpuc->event_list[i]);
929 cpuc->event_constraint[i] = c;
930 }
931
932 wmin = min(wmin, c->weight);
933 wmax = max(wmax, c->weight);
934 }
935
936
937
938
939 for (i = 0; i < n; i++) {
940 u64 mask;
941
942 hwc = &cpuc->event_list[i]->hw;
943 c = cpuc->event_constraint[i];
944
945
946 if (hwc->idx == -1)
947 break;
948
949
950 if (!test_bit(hwc->idx, c->idxmsk))
951 break;
952
953 mask = BIT_ULL(hwc->idx);
954 if (is_counter_pair(hwc))
955 mask |= mask << 1;
956
957
958 if (used_mask & mask)
959 break;
960
961 used_mask |= mask;
962
963 if (assign)
964 assign[i] = hwc->idx;
965 }
966
967
968 if (i != n) {
969 int gpmax = x86_pmu.num_counters;
970
971
972
973
974
975
976
977
978
979
980
981 if (is_ht_workaround_enabled() && !cpuc->is_fake &&
982 READ_ONCE(cpuc->excl_cntrs->exclusive_present))
983 gpmax /= 2;
984
985
986
987
988
989 if (x86_pmu.flags & PMU_FL_PAIR) {
990 gpmax = x86_pmu.num_counters - cpuc->n_pair;
991 WARN_ON(gpmax <= 0);
992 }
993
994 unsched = perf_assign_events(cpuc->event_constraint, n, wmin,
995 wmax, gpmax, assign);
996 }
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008 if (!unsched && assign) {
1009 for (i = 0; i < n; i++) {
1010 e = cpuc->event_list[i];
1011 if (x86_pmu.commit_scheduling)
1012 x86_pmu.commit_scheduling(cpuc, i, assign[i]);
1013 }
1014 } else {
1015 for (i = n0; i < n; i++) {
1016 e = cpuc->event_list[i];
1017
1018
1019
1020
1021 if (x86_pmu.put_event_constraints)
1022 x86_pmu.put_event_constraints(cpuc, e);
1023
1024 cpuc->event_constraint[i] = NULL;
1025 }
1026 }
1027
1028 if (x86_pmu.stop_scheduling)
1029 x86_pmu.stop_scheduling(cpuc);
1030
1031 return unsched ? -EINVAL : 0;
1032}
1033
1034
1035
1036
1037
1038static int collect_events(struct cpu_hw_events *cpuc, struct perf_event *leader, bool dogrp)
1039{
1040 struct perf_event *event;
1041 int n, max_count;
1042
1043 max_count = x86_pmu.num_counters + x86_pmu.num_counters_fixed;
1044
1045
1046 n = cpuc->n_events;
1047 if (!cpuc->n_events)
1048 cpuc->pebs_output = 0;
1049
1050 if (!cpuc->is_fake && leader->attr.precise_ip) {
1051
1052
1053
1054
1055
1056 if (is_pebs_pt(leader) && !leader->aux_event)
1057 return -EINVAL;
1058
1059
1060
1061
1062 if (cpuc->pebs_output &&
1063 cpuc->pebs_output != is_pebs_pt(leader) + 1)
1064 return -EINVAL;
1065
1066 cpuc->pebs_output = is_pebs_pt(leader) + 1;
1067 }
1068
1069 if (is_x86_event(leader)) {
1070 if (n >= max_count)
1071 return -EINVAL;
1072 cpuc->event_list[n] = leader;
1073 n++;
1074 if (is_counter_pair(&leader->hw))
1075 cpuc->n_pair++;
1076 }
1077 if (!dogrp)
1078 return n;
1079
1080 for_each_sibling_event(event, leader) {
1081 if (!is_x86_event(event) ||
1082 event->state <= PERF_EVENT_STATE_OFF)
1083 continue;
1084
1085 if (n >= max_count)
1086 return -EINVAL;
1087
1088 cpuc->event_list[n] = event;
1089 n++;
1090 if (is_counter_pair(&event->hw))
1091 cpuc->n_pair++;
1092 }
1093 return n;
1094}
1095
1096static inline void x86_assign_hw_event(struct perf_event *event,
1097 struct cpu_hw_events *cpuc, int i)
1098{
1099 struct hw_perf_event *hwc = &event->hw;
1100 int idx;
1101
1102 idx = hwc->idx = cpuc->assign[i];
1103 hwc->last_cpu = smp_processor_id();
1104 hwc->last_tag = ++cpuc->tags[i];
1105
1106 switch (hwc->idx) {
1107 case INTEL_PMC_IDX_FIXED_BTS:
1108 case INTEL_PMC_IDX_FIXED_VLBR:
1109 hwc->config_base = 0;
1110 hwc->event_base = 0;
1111 break;
1112
1113 case INTEL_PMC_IDX_FIXED ... INTEL_PMC_IDX_FIXED_BTS-1:
1114 hwc->config_base = MSR_ARCH_PERFMON_FIXED_CTR_CTRL;
1115 hwc->event_base = MSR_ARCH_PERFMON_FIXED_CTR0 +
1116 (idx - INTEL_PMC_IDX_FIXED);
1117 hwc->event_base_rdpmc = (idx - INTEL_PMC_IDX_FIXED) | 1<<30;
1118 break;
1119
1120 default:
1121 hwc->config_base = x86_pmu_config_addr(hwc->idx);
1122 hwc->event_base = x86_pmu_event_addr(hwc->idx);
1123 hwc->event_base_rdpmc = x86_pmu_rdpmc_index(hwc->idx);
1124 break;
1125 }
1126}
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142int x86_perf_rdpmc_index(struct perf_event *event)
1143{
1144 lockdep_assert_irqs_disabled();
1145
1146 return event->hw.event_base_rdpmc;
1147}
1148
1149static inline int match_prev_assignment(struct hw_perf_event *hwc,
1150 struct cpu_hw_events *cpuc,
1151 int i)
1152{
1153 return hwc->idx == cpuc->assign[i] &&
1154 hwc->last_cpu == smp_processor_id() &&
1155 hwc->last_tag == cpuc->tags[i];
1156}
1157
1158static void x86_pmu_start(struct perf_event *event, int flags);
1159
1160static void x86_pmu_enable(struct pmu *pmu)
1161{
1162 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1163 struct perf_event *event;
1164 struct hw_perf_event *hwc;
1165 int i, added = cpuc->n_added;
1166
1167 if (!x86_pmu_initialized())
1168 return;
1169
1170 if (cpuc->enabled)
1171 return;
1172
1173 if (cpuc->n_added) {
1174 int n_running = cpuc->n_events - cpuc->n_added;
1175
1176
1177
1178
1179
1180
1181 for (i = 0; i < n_running; i++) {
1182 event = cpuc->event_list[i];
1183 hwc = &event->hw;
1184
1185
1186
1187
1188
1189
1190
1191 if (hwc->idx == -1 ||
1192 match_prev_assignment(hwc, cpuc, i))
1193 continue;
1194
1195
1196
1197
1198
1199 if (hwc->state & PERF_HES_STOPPED)
1200 hwc->state |= PERF_HES_ARCH;
1201
1202 x86_pmu_stop(event, PERF_EF_UPDATE);
1203 }
1204
1205
1206
1207
1208 for (i = 0; i < cpuc->n_events; i++) {
1209 event = cpuc->event_list[i];
1210 hwc = &event->hw;
1211
1212 if (!match_prev_assignment(hwc, cpuc, i))
1213 x86_assign_hw_event(event, cpuc, i);
1214 else if (i < n_running)
1215 continue;
1216
1217 if (hwc->state & PERF_HES_ARCH)
1218 continue;
1219
1220 x86_pmu_start(event, PERF_EF_RELOAD);
1221 }
1222 cpuc->n_added = 0;
1223 perf_events_lapic_init();
1224 }
1225
1226 cpuc->enabled = 1;
1227 barrier();
1228
1229 x86_pmu.enable_all(added);
1230}
1231
1232static DEFINE_PER_CPU(u64 [X86_PMC_IDX_MAX], pmc_prev_left);
1233
1234
1235
1236
1237
1238int x86_perf_event_set_period(struct perf_event *event)
1239{
1240 struct hw_perf_event *hwc = &event->hw;
1241 s64 left = local64_read(&hwc->period_left);
1242 s64 period = hwc->sample_period;
1243 int ret = 0, idx = hwc->idx;
1244
1245 if (unlikely(!hwc->event_base))
1246 return 0;
1247
1248
1249
1250
1251 if (unlikely(left <= -period)) {
1252 left = period;
1253 local64_set(&hwc->period_left, left);
1254 hwc->last_period = period;
1255 ret = 1;
1256 }
1257
1258 if (unlikely(left <= 0)) {
1259 left += period;
1260 local64_set(&hwc->period_left, left);
1261 hwc->last_period = period;
1262 ret = 1;
1263 }
1264
1265
1266
1267 if (unlikely(left < 2))
1268 left = 2;
1269
1270 if (left > x86_pmu.max_period)
1271 left = x86_pmu.max_period;
1272
1273 if (x86_pmu.limit_period)
1274 left = x86_pmu.limit_period(event, left);
1275
1276 per_cpu(pmc_prev_left[idx], smp_processor_id()) = left;
1277
1278
1279
1280
1281
1282 local64_set(&hwc->prev_count, (u64)-left);
1283
1284 wrmsrl(hwc->event_base, (u64)(-left) & x86_pmu.cntval_mask);
1285
1286
1287
1288
1289
1290 if (is_counter_pair(hwc))
1291 wrmsrl(x86_pmu_event_addr(idx + 1), 0);
1292
1293
1294
1295
1296
1297
1298 if (x86_pmu.perfctr_second_write) {
1299 wrmsrl(hwc->event_base,
1300 (u64)(-left) & x86_pmu.cntval_mask);
1301 }
1302
1303 perf_event_update_userpage(event);
1304
1305 return ret;
1306}
1307
1308void x86_pmu_enable_event(struct perf_event *event)
1309{
1310 if (__this_cpu_read(cpu_hw_events.enabled))
1311 __x86_pmu_enable_event(&event->hw,
1312 ARCH_PERFMON_EVENTSEL_ENABLE);
1313}
1314
1315
1316
1317
1318
1319
1320
1321static int x86_pmu_add(struct perf_event *event, int flags)
1322{
1323 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1324 struct hw_perf_event *hwc;
1325 int assign[X86_PMC_IDX_MAX];
1326 int n, n0, ret;
1327
1328 hwc = &event->hw;
1329
1330 n0 = cpuc->n_events;
1331 ret = n = collect_events(cpuc, event, false);
1332 if (ret < 0)
1333 goto out;
1334
1335 hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
1336 if (!(flags & PERF_EF_START))
1337 hwc->state |= PERF_HES_ARCH;
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347 if (cpuc->txn_flags & PERF_PMU_TXN_ADD)
1348 goto done_collect;
1349
1350 ret = x86_pmu.schedule_events(cpuc, n, assign);
1351 if (ret)
1352 goto out;
1353
1354
1355
1356
1357 memcpy(cpuc->assign, assign, n*sizeof(int));
1358
1359done_collect:
1360
1361
1362
1363
1364 cpuc->n_events = n;
1365 cpuc->n_added += n - n0;
1366 cpuc->n_txn += n - n0;
1367
1368 if (x86_pmu.add) {
1369
1370
1371
1372
1373 x86_pmu.add(event);
1374 }
1375
1376 ret = 0;
1377out:
1378 return ret;
1379}
1380
1381static void x86_pmu_start(struct perf_event *event, int flags)
1382{
1383 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1384 int idx = event->hw.idx;
1385
1386 if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED)))
1387 return;
1388
1389 if (WARN_ON_ONCE(idx == -1))
1390 return;
1391
1392 if (flags & PERF_EF_RELOAD) {
1393 WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE));
1394 x86_perf_event_set_period(event);
1395 }
1396
1397 event->hw.state = 0;
1398
1399 cpuc->events[idx] = event;
1400 __set_bit(idx, cpuc->active_mask);
1401 __set_bit(idx, cpuc->running);
1402 x86_pmu.enable(event);
1403 perf_event_update_userpage(event);
1404}
1405
1406void perf_event_print_debug(void)
1407{
1408 u64 ctrl, status, overflow, pmc_ctrl, pmc_count, prev_left, fixed;
1409 u64 pebs, debugctl;
1410 struct cpu_hw_events *cpuc;
1411 unsigned long flags;
1412 int cpu, idx;
1413
1414 if (!x86_pmu.num_counters)
1415 return;
1416
1417 local_irq_save(flags);
1418
1419 cpu = smp_processor_id();
1420 cpuc = &per_cpu(cpu_hw_events, cpu);
1421
1422 if (x86_pmu.version >= 2) {
1423 rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl);
1424 rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
1425 rdmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, overflow);
1426 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR_CTRL, fixed);
1427
1428 pr_info("\n");
1429 pr_info("CPU#%d: ctrl: %016llx\n", cpu, ctrl);
1430 pr_info("CPU#%d: status: %016llx\n", cpu, status);
1431 pr_info("CPU#%d: overflow: %016llx\n", cpu, overflow);
1432 pr_info("CPU#%d: fixed: %016llx\n", cpu, fixed);
1433 if (x86_pmu.pebs_constraints) {
1434 rdmsrl(MSR_IA32_PEBS_ENABLE, pebs);
1435 pr_info("CPU#%d: pebs: %016llx\n", cpu, pebs);
1436 }
1437 if (x86_pmu.lbr_nr) {
1438 rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
1439 pr_info("CPU#%d: debugctl: %016llx\n", cpu, debugctl);
1440 }
1441 }
1442 pr_info("CPU#%d: active: %016llx\n", cpu, *(u64 *)cpuc->active_mask);
1443
1444 for (idx = 0; idx < x86_pmu.num_counters; idx++) {
1445 rdmsrl(x86_pmu_config_addr(idx), pmc_ctrl);
1446 rdmsrl(x86_pmu_event_addr(idx), pmc_count);
1447
1448 prev_left = per_cpu(pmc_prev_left[idx], cpu);
1449
1450 pr_info("CPU#%d: gen-PMC%d ctrl: %016llx\n",
1451 cpu, idx, pmc_ctrl);
1452 pr_info("CPU#%d: gen-PMC%d count: %016llx\n",
1453 cpu, idx, pmc_count);
1454 pr_info("CPU#%d: gen-PMC%d left: %016llx\n",
1455 cpu, idx, prev_left);
1456 }
1457 for (idx = 0; idx < x86_pmu.num_counters_fixed; idx++) {
1458 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, pmc_count);
1459
1460 pr_info("CPU#%d: fixed-PMC%d count: %016llx\n",
1461 cpu, idx, pmc_count);
1462 }
1463 local_irq_restore(flags);
1464}
1465
1466void x86_pmu_stop(struct perf_event *event, int flags)
1467{
1468 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1469 struct hw_perf_event *hwc = &event->hw;
1470
1471 if (test_bit(hwc->idx, cpuc->active_mask)) {
1472 x86_pmu.disable(event);
1473 __clear_bit(hwc->idx, cpuc->active_mask);
1474 cpuc->events[hwc->idx] = NULL;
1475 WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED);
1476 hwc->state |= PERF_HES_STOPPED;
1477 }
1478
1479 if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) {
1480
1481
1482
1483
1484 x86_perf_event_update(event);
1485 hwc->state |= PERF_HES_UPTODATE;
1486 }
1487}
1488
1489static void x86_pmu_del(struct perf_event *event, int flags)
1490{
1491 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1492 int i;
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502 if (cpuc->txn_flags & PERF_PMU_TXN_ADD)
1503 goto do_del;
1504
1505
1506
1507
1508 x86_pmu_stop(event, PERF_EF_UPDATE);
1509
1510 for (i = 0; i < cpuc->n_events; i++) {
1511 if (event == cpuc->event_list[i])
1512 break;
1513 }
1514
1515 if (WARN_ON_ONCE(i == cpuc->n_events))
1516 return;
1517
1518
1519 if (i >= cpuc->n_events - cpuc->n_added)
1520 --cpuc->n_added;
1521
1522 if (x86_pmu.put_event_constraints)
1523 x86_pmu.put_event_constraints(cpuc, event);
1524
1525
1526 while (++i < cpuc->n_events) {
1527 cpuc->event_list[i-1] = cpuc->event_list[i];
1528 cpuc->event_constraint[i-1] = cpuc->event_constraint[i];
1529 }
1530 cpuc->event_constraint[i-1] = NULL;
1531 --cpuc->n_events;
1532
1533 perf_event_update_userpage(event);
1534
1535do_del:
1536 if (x86_pmu.del) {
1537
1538
1539
1540
1541 x86_pmu.del(event);
1542 }
1543}
1544
1545int x86_pmu_handle_irq(struct pt_regs *regs)
1546{
1547 struct perf_sample_data data;
1548 struct cpu_hw_events *cpuc;
1549 struct perf_event *event;
1550 int idx, handled = 0;
1551 u64 val;
1552
1553 cpuc = this_cpu_ptr(&cpu_hw_events);
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563 apic_write(APIC_LVTPC, APIC_DM_NMI);
1564
1565 for (idx = 0; idx < x86_pmu.num_counters; idx++) {
1566 if (!test_bit(idx, cpuc->active_mask))
1567 continue;
1568
1569 event = cpuc->events[idx];
1570
1571 val = x86_perf_event_update(event);
1572 if (val & (1ULL << (x86_pmu.cntval_bits - 1)))
1573 continue;
1574
1575
1576
1577
1578 handled++;
1579 perf_sample_data_init(&data, 0, event->hw.last_period);
1580
1581 if (!x86_perf_event_set_period(event))
1582 continue;
1583
1584 if (perf_event_overflow(event, &data, regs))
1585 x86_pmu_stop(event, 0);
1586 }
1587
1588 if (handled)
1589 inc_irq_stat(apic_perf_irqs);
1590
1591 return handled;
1592}
1593
1594void perf_events_lapic_init(void)
1595{
1596 if (!x86_pmu.apic || !x86_pmu_initialized())
1597 return;
1598
1599
1600
1601
1602 apic_write(APIC_LVTPC, APIC_DM_NMI);
1603}
1604
1605static int
1606perf_event_nmi_handler(unsigned int cmd, struct pt_regs *regs)
1607{
1608 u64 start_clock;
1609 u64 finish_clock;
1610 int ret;
1611
1612
1613
1614
1615
1616 if (!atomic_read(&active_events))
1617 return NMI_DONE;
1618
1619 start_clock = sched_clock();
1620 ret = x86_pmu.handle_irq(regs);
1621 finish_clock = sched_clock();
1622
1623 perf_sample_event_took(finish_clock - start_clock);
1624
1625 return ret;
1626}
1627NOKPROBE_SYMBOL(perf_event_nmi_handler);
1628
1629struct event_constraint emptyconstraint;
1630struct event_constraint unconstrained;
1631
1632static int x86_pmu_prepare_cpu(unsigned int cpu)
1633{
1634 struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);
1635 int i;
1636
1637 for (i = 0 ; i < X86_PERF_KFREE_MAX; i++)
1638 cpuc->kfree_on_online[i] = NULL;
1639 if (x86_pmu.cpu_prepare)
1640 return x86_pmu.cpu_prepare(cpu);
1641 return 0;
1642}
1643
1644static int x86_pmu_dead_cpu(unsigned int cpu)
1645{
1646 if (x86_pmu.cpu_dead)
1647 x86_pmu.cpu_dead(cpu);
1648 return 0;
1649}
1650
1651static int x86_pmu_online_cpu(unsigned int cpu)
1652{
1653 struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);
1654 int i;
1655
1656 for (i = 0 ; i < X86_PERF_KFREE_MAX; i++) {
1657 kfree(cpuc->kfree_on_online[i]);
1658 cpuc->kfree_on_online[i] = NULL;
1659 }
1660 return 0;
1661}
1662
1663static int x86_pmu_starting_cpu(unsigned int cpu)
1664{
1665 if (x86_pmu.cpu_starting)
1666 x86_pmu.cpu_starting(cpu);
1667 return 0;
1668}
1669
1670static int x86_pmu_dying_cpu(unsigned int cpu)
1671{
1672 if (x86_pmu.cpu_dying)
1673 x86_pmu.cpu_dying(cpu);
1674 return 0;
1675}
1676
1677static void __init pmu_check_apic(void)
1678{
1679 if (boot_cpu_has(X86_FEATURE_APIC))
1680 return;
1681
1682 x86_pmu.apic = 0;
1683 pr_info("no APIC, boot with the \"lapic\" boot parameter to force-enable it.\n");
1684 pr_info("no hardware sampling interrupt available.\n");
1685
1686
1687
1688
1689
1690
1691
1692 pmu.capabilities |= PERF_PMU_CAP_NO_INTERRUPT;
1693
1694}
1695
1696static struct attribute_group x86_pmu_format_group __ro_after_init = {
1697 .name = "format",
1698 .attrs = NULL,
1699};
1700
1701ssize_t events_sysfs_show(struct device *dev, struct device_attribute *attr, char *page)
1702{
1703 struct perf_pmu_events_attr *pmu_attr =
1704 container_of(attr, struct perf_pmu_events_attr, attr);
1705 u64 config = 0;
1706
1707 if (pmu_attr->id < x86_pmu.max_events)
1708 config = x86_pmu.event_map(pmu_attr->id);
1709
1710
1711 if (pmu_attr->event_str)
1712 return sprintf(page, "%s", pmu_attr->event_str);
1713
1714 return x86_pmu.events_sysfs_show(page, config);
1715}
1716EXPORT_SYMBOL_GPL(events_sysfs_show);
1717
1718ssize_t events_ht_sysfs_show(struct device *dev, struct device_attribute *attr,
1719 char *page)
1720{
1721 struct perf_pmu_events_ht_attr *pmu_attr =
1722 container_of(attr, struct perf_pmu_events_ht_attr, attr);
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735 return sprintf(page, "%s",
1736 topology_max_smt_threads() > 1 ?
1737 pmu_attr->event_str_ht :
1738 pmu_attr->event_str_noht);
1739}
1740
1741EVENT_ATTR(cpu-cycles, CPU_CYCLES );
1742EVENT_ATTR(instructions, INSTRUCTIONS );
1743EVENT_ATTR(cache-references, CACHE_REFERENCES );
1744EVENT_ATTR(cache-misses, CACHE_MISSES );
1745EVENT_ATTR(branch-instructions, BRANCH_INSTRUCTIONS );
1746EVENT_ATTR(branch-misses, BRANCH_MISSES );
1747EVENT_ATTR(bus-cycles, BUS_CYCLES );
1748EVENT_ATTR(stalled-cycles-frontend, STALLED_CYCLES_FRONTEND );
1749EVENT_ATTR(stalled-cycles-backend, STALLED_CYCLES_BACKEND );
1750EVENT_ATTR(ref-cycles, REF_CPU_CYCLES );
1751
1752static struct attribute *empty_attrs;
1753
1754static struct attribute *events_attr[] = {
1755 EVENT_PTR(CPU_CYCLES),
1756 EVENT_PTR(INSTRUCTIONS),
1757 EVENT_PTR(CACHE_REFERENCES),
1758 EVENT_PTR(CACHE_MISSES),
1759 EVENT_PTR(BRANCH_INSTRUCTIONS),
1760 EVENT_PTR(BRANCH_MISSES),
1761 EVENT_PTR(BUS_CYCLES),
1762 EVENT_PTR(STALLED_CYCLES_FRONTEND),
1763 EVENT_PTR(STALLED_CYCLES_BACKEND),
1764 EVENT_PTR(REF_CPU_CYCLES),
1765 NULL,
1766};
1767
1768
1769
1770
1771
1772static umode_t
1773is_visible(struct kobject *kobj, struct attribute *attr, int idx)
1774{
1775 struct perf_pmu_events_attr *pmu_attr;
1776
1777 if (idx >= x86_pmu.max_events)
1778 return 0;
1779
1780 pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr.attr);
1781
1782 return pmu_attr->event_str || x86_pmu.event_map(idx) ? attr->mode : 0;
1783}
1784
1785static struct attribute_group x86_pmu_events_group __ro_after_init = {
1786 .name = "events",
1787 .attrs = events_attr,
1788 .is_visible = is_visible,
1789};
1790
1791ssize_t x86_event_sysfs_show(char *page, u64 config, u64 event)
1792{
1793 u64 umask = (config & ARCH_PERFMON_EVENTSEL_UMASK) >> 8;
1794 u64 cmask = (config & ARCH_PERFMON_EVENTSEL_CMASK) >> 24;
1795 bool edge = (config & ARCH_PERFMON_EVENTSEL_EDGE);
1796 bool pc = (config & ARCH_PERFMON_EVENTSEL_PIN_CONTROL);
1797 bool any = (config & ARCH_PERFMON_EVENTSEL_ANY);
1798 bool inv = (config & ARCH_PERFMON_EVENTSEL_INV);
1799 ssize_t ret;
1800
1801
1802
1803
1804
1805 ret = sprintf(page, "event=0x%02llx", event);
1806
1807 if (umask)
1808 ret += sprintf(page + ret, ",umask=0x%02llx", umask);
1809
1810 if (edge)
1811 ret += sprintf(page + ret, ",edge");
1812
1813 if (pc)
1814 ret += sprintf(page + ret, ",pc");
1815
1816 if (any)
1817 ret += sprintf(page + ret, ",any");
1818
1819 if (inv)
1820 ret += sprintf(page + ret, ",inv");
1821
1822 if (cmask)
1823 ret += sprintf(page + ret, ",cmask=0x%02llx", cmask);
1824
1825 ret += sprintf(page + ret, "\n");
1826
1827 return ret;
1828}
1829
1830static struct attribute_group x86_pmu_attr_group;
1831static struct attribute_group x86_pmu_caps_group;
1832
1833static int __init init_hw_perf_events(void)
1834{
1835 struct x86_pmu_quirk *quirk;
1836 int err;
1837
1838 pr_info("Performance Events: ");
1839
1840 switch (boot_cpu_data.x86_vendor) {
1841 case X86_VENDOR_INTEL:
1842 err = intel_pmu_init();
1843 break;
1844 case X86_VENDOR_AMD:
1845 err = amd_pmu_init();
1846 break;
1847 case X86_VENDOR_HYGON:
1848 err = amd_pmu_init();
1849 x86_pmu.name = "HYGON";
1850 break;
1851 case X86_VENDOR_ZHAOXIN:
1852 case X86_VENDOR_CENTAUR:
1853 err = zhaoxin_pmu_init();
1854 break;
1855 default:
1856 err = -ENOTSUPP;
1857 }
1858 if (err != 0) {
1859 pr_cont("no PMU driver, software events only.\n");
1860 return 0;
1861 }
1862
1863 pmu_check_apic();
1864
1865
1866 if (!check_hw_exists())
1867 return 0;
1868
1869 pr_cont("%s PMU driver.\n", x86_pmu.name);
1870
1871 x86_pmu.attr_rdpmc = 1;
1872
1873 for (quirk = x86_pmu.quirks; quirk; quirk = quirk->next)
1874 quirk->func();
1875
1876 if (!x86_pmu.intel_ctrl)
1877 x86_pmu.intel_ctrl = (1 << x86_pmu.num_counters) - 1;
1878
1879 perf_events_lapic_init();
1880 register_nmi_handler(NMI_LOCAL, perf_event_nmi_handler, 0, "PMI");
1881
1882 unconstrained = (struct event_constraint)
1883 __EVENT_CONSTRAINT(0, (1ULL << x86_pmu.num_counters) - 1,
1884 0, x86_pmu.num_counters, 0, 0);
1885
1886 x86_pmu_format_group.attrs = x86_pmu.format_attrs;
1887
1888 if (!x86_pmu.events_sysfs_show)
1889 x86_pmu_events_group.attrs = &empty_attrs;
1890
1891 pmu.attr_update = x86_pmu.attr_update;
1892
1893 pr_info("... version: %d\n", x86_pmu.version);
1894 pr_info("... bit width: %d\n", x86_pmu.cntval_bits);
1895 pr_info("... generic registers: %d\n", x86_pmu.num_counters);
1896 pr_info("... value mask: %016Lx\n", x86_pmu.cntval_mask);
1897 pr_info("... max period: %016Lx\n", x86_pmu.max_period);
1898 pr_info("... fixed-purpose events: %d\n", x86_pmu.num_counters_fixed);
1899 pr_info("... event mask: %016Lx\n", x86_pmu.intel_ctrl);
1900
1901
1902
1903
1904
1905 err = cpuhp_setup_state(CPUHP_PERF_X86_PREPARE, "perf/x86:prepare",
1906 x86_pmu_prepare_cpu, x86_pmu_dead_cpu);
1907 if (err)
1908 return err;
1909
1910 err = cpuhp_setup_state(CPUHP_AP_PERF_X86_STARTING,
1911 "perf/x86:starting", x86_pmu_starting_cpu,
1912 x86_pmu_dying_cpu);
1913 if (err)
1914 goto out;
1915
1916 err = cpuhp_setup_state(CPUHP_AP_PERF_X86_ONLINE, "perf/x86:online",
1917 x86_pmu_online_cpu, NULL);
1918 if (err)
1919 goto out1;
1920
1921 err = perf_pmu_register(&pmu, "cpu", PERF_TYPE_RAW);
1922 if (err)
1923 goto out2;
1924
1925 return 0;
1926
1927out2:
1928 cpuhp_remove_state(CPUHP_AP_PERF_X86_ONLINE);
1929out1:
1930 cpuhp_remove_state(CPUHP_AP_PERF_X86_STARTING);
1931out:
1932 cpuhp_remove_state(CPUHP_PERF_X86_PREPARE);
1933 return err;
1934}
1935early_initcall(init_hw_perf_events);
1936
1937static inline void x86_pmu_read(struct perf_event *event)
1938{
1939 if (x86_pmu.read)
1940 return x86_pmu.read(event);
1941 x86_perf_event_update(event);
1942}
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953static void x86_pmu_start_txn(struct pmu *pmu, unsigned int txn_flags)
1954{
1955 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1956
1957 WARN_ON_ONCE(cpuc->txn_flags);
1958
1959 cpuc->txn_flags = txn_flags;
1960 if (txn_flags & ~PERF_PMU_TXN_ADD)
1961 return;
1962
1963 perf_pmu_disable(pmu);
1964 __this_cpu_write(cpu_hw_events.n_txn, 0);
1965}
1966
1967
1968
1969
1970
1971
1972static void x86_pmu_cancel_txn(struct pmu *pmu)
1973{
1974 unsigned int txn_flags;
1975 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1976
1977 WARN_ON_ONCE(!cpuc->txn_flags);
1978
1979 txn_flags = cpuc->txn_flags;
1980 cpuc->txn_flags = 0;
1981 if (txn_flags & ~PERF_PMU_TXN_ADD)
1982 return;
1983
1984
1985
1986
1987
1988 __this_cpu_sub(cpu_hw_events.n_added, __this_cpu_read(cpu_hw_events.n_txn));
1989 __this_cpu_sub(cpu_hw_events.n_events, __this_cpu_read(cpu_hw_events.n_txn));
1990 perf_pmu_enable(pmu);
1991}
1992
1993
1994
1995
1996
1997
1998
1999
2000static int x86_pmu_commit_txn(struct pmu *pmu)
2001{
2002 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
2003 int assign[X86_PMC_IDX_MAX];
2004 int n, ret;
2005
2006 WARN_ON_ONCE(!cpuc->txn_flags);
2007
2008 if (cpuc->txn_flags & ~PERF_PMU_TXN_ADD) {
2009 cpuc->txn_flags = 0;
2010 return 0;
2011 }
2012
2013 n = cpuc->n_events;
2014
2015 if (!x86_pmu_initialized())
2016 return -EAGAIN;
2017
2018 ret = x86_pmu.schedule_events(cpuc, n, assign);
2019 if (ret)
2020 return ret;
2021
2022
2023
2024
2025
2026 memcpy(cpuc->assign, assign, n*sizeof(int));
2027
2028 cpuc->txn_flags = 0;
2029 perf_pmu_enable(pmu);
2030 return 0;
2031}
2032
2033
2034
2035
2036
2037
2038
2039
2040static void free_fake_cpuc(struct cpu_hw_events *cpuc)
2041{
2042 intel_cpuc_finish(cpuc);
2043 kfree(cpuc);
2044}
2045
2046static struct cpu_hw_events *allocate_fake_cpuc(void)
2047{
2048 struct cpu_hw_events *cpuc;
2049 int cpu = raw_smp_processor_id();
2050
2051 cpuc = kzalloc(sizeof(*cpuc), GFP_KERNEL);
2052 if (!cpuc)
2053 return ERR_PTR(-ENOMEM);
2054 cpuc->is_fake = 1;
2055
2056 if (intel_cpuc_prepare(cpuc, cpu))
2057 goto error;
2058
2059 return cpuc;
2060error:
2061 free_fake_cpuc(cpuc);
2062 return ERR_PTR(-ENOMEM);
2063}
2064
2065
2066
2067
2068static int validate_event(struct perf_event *event)
2069{
2070 struct cpu_hw_events *fake_cpuc;
2071 struct event_constraint *c;
2072 int ret = 0;
2073
2074 fake_cpuc = allocate_fake_cpuc();
2075 if (IS_ERR(fake_cpuc))
2076 return PTR_ERR(fake_cpuc);
2077
2078 c = x86_pmu.get_event_constraints(fake_cpuc, 0, event);
2079
2080 if (!c || !c->weight)
2081 ret = -EINVAL;
2082
2083 if (x86_pmu.put_event_constraints)
2084 x86_pmu.put_event_constraints(fake_cpuc, event);
2085
2086 free_fake_cpuc(fake_cpuc);
2087
2088 return ret;
2089}
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102static int validate_group(struct perf_event *event)
2103{
2104 struct perf_event *leader = event->group_leader;
2105 struct cpu_hw_events *fake_cpuc;
2106 int ret = -EINVAL, n;
2107
2108 fake_cpuc = allocate_fake_cpuc();
2109 if (IS_ERR(fake_cpuc))
2110 return PTR_ERR(fake_cpuc);
2111
2112
2113
2114
2115
2116
2117 n = collect_events(fake_cpuc, leader, true);
2118 if (n < 0)
2119 goto out;
2120
2121 fake_cpuc->n_events = n;
2122 n = collect_events(fake_cpuc, event, false);
2123 if (n < 0)
2124 goto out;
2125
2126 fake_cpuc->n_events = 0;
2127 ret = x86_pmu.schedule_events(fake_cpuc, n, NULL);
2128
2129out:
2130 free_fake_cpuc(fake_cpuc);
2131 return ret;
2132}
2133
2134static int x86_pmu_event_init(struct perf_event *event)
2135{
2136 struct pmu *tmp;
2137 int err;
2138
2139 switch (event->attr.type) {
2140 case PERF_TYPE_RAW:
2141 case PERF_TYPE_HARDWARE:
2142 case PERF_TYPE_HW_CACHE:
2143 break;
2144
2145 default:
2146 return -ENOENT;
2147 }
2148
2149 err = __x86_pmu_event_init(event);
2150 if (!err) {
2151
2152
2153
2154
2155
2156 tmp = event->pmu;
2157 event->pmu = &pmu;
2158
2159 if (event->group_leader != event)
2160 err = validate_group(event);
2161 else
2162 err = validate_event(event);
2163
2164 event->pmu = tmp;
2165 }
2166 if (err) {
2167 if (event->destroy)
2168 event->destroy(event);
2169 }
2170
2171 if (READ_ONCE(x86_pmu.attr_rdpmc) &&
2172 !(event->hw.flags & PERF_X86_EVENT_LARGE_PEBS))
2173 event->hw.flags |= PERF_X86_EVENT_RDPMC_ALLOWED;
2174
2175 return err;
2176}
2177
2178static void x86_pmu_event_mapped(struct perf_event *event, struct mm_struct *mm)
2179{
2180 if (!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED))
2181 return;
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193 mmap_assert_write_locked(mm);
2194
2195 if (atomic_inc_return(&mm->context.perf_rdpmc_allowed) == 1)
2196 on_each_cpu_mask(mm_cpumask(mm), cr4_update_pce, NULL, 1);
2197}
2198
2199static void x86_pmu_event_unmapped(struct perf_event *event, struct mm_struct *mm)
2200{
2201
2202 if (!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED))
2203 return;
2204
2205 if (atomic_dec_and_test(&mm->context.perf_rdpmc_allowed))
2206 on_each_cpu_mask(mm_cpumask(mm), cr4_update_pce, NULL, 1);
2207}
2208
2209static int x86_pmu_event_idx(struct perf_event *event)
2210{
2211 int idx = event->hw.idx;
2212
2213 if (!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED))
2214 return 0;
2215
2216 if (x86_pmu.num_counters_fixed && idx >= INTEL_PMC_IDX_FIXED) {
2217 idx -= INTEL_PMC_IDX_FIXED;
2218 idx |= 1 << 30;
2219 }
2220
2221 return idx + 1;
2222}
2223
2224static ssize_t get_attr_rdpmc(struct device *cdev,
2225 struct device_attribute *attr,
2226 char *buf)
2227{
2228 return snprintf(buf, 40, "%d\n", x86_pmu.attr_rdpmc);
2229}
2230
2231static ssize_t set_attr_rdpmc(struct device *cdev,
2232 struct device_attribute *attr,
2233 const char *buf, size_t count)
2234{
2235 unsigned long val;
2236 ssize_t ret;
2237
2238 ret = kstrtoul(buf, 0, &val);
2239 if (ret)
2240 return ret;
2241
2242 if (val > 2)
2243 return -EINVAL;
2244
2245 if (x86_pmu.attr_rdpmc_broken)
2246 return -ENOTSUPP;
2247
2248 if (val != x86_pmu.attr_rdpmc) {
2249
2250
2251
2252
2253
2254 if (val == 0)
2255 static_branch_inc(&rdpmc_never_available_key);
2256 else if (x86_pmu.attr_rdpmc == 0)
2257 static_branch_dec(&rdpmc_never_available_key);
2258
2259 if (val == 2)
2260 static_branch_inc(&rdpmc_always_available_key);
2261 else if (x86_pmu.attr_rdpmc == 2)
2262 static_branch_dec(&rdpmc_always_available_key);
2263
2264 on_each_cpu(cr4_update_pce, NULL, 1);
2265 x86_pmu.attr_rdpmc = val;
2266 }
2267
2268 return count;
2269}
2270
2271static DEVICE_ATTR(rdpmc, S_IRUSR | S_IWUSR, get_attr_rdpmc, set_attr_rdpmc);
2272
2273static struct attribute *x86_pmu_attrs[] = {
2274 &dev_attr_rdpmc.attr,
2275 NULL,
2276};
2277
2278static struct attribute_group x86_pmu_attr_group __ro_after_init = {
2279 .attrs = x86_pmu_attrs,
2280};
2281
2282static ssize_t max_precise_show(struct device *cdev,
2283 struct device_attribute *attr,
2284 char *buf)
2285{
2286 return snprintf(buf, PAGE_SIZE, "%d\n", x86_pmu_max_precise());
2287}
2288
2289static DEVICE_ATTR_RO(max_precise);
2290
2291static struct attribute *x86_pmu_caps_attrs[] = {
2292 &dev_attr_max_precise.attr,
2293 NULL
2294};
2295
2296static struct attribute_group x86_pmu_caps_group __ro_after_init = {
2297 .name = "caps",
2298 .attrs = x86_pmu_caps_attrs,
2299};
2300
2301static const struct attribute_group *x86_pmu_attr_groups[] = {
2302 &x86_pmu_attr_group,
2303 &x86_pmu_format_group,
2304 &x86_pmu_events_group,
2305 &x86_pmu_caps_group,
2306 NULL,
2307};
2308
2309static void x86_pmu_sched_task(struct perf_event_context *ctx, bool sched_in)
2310{
2311 if (x86_pmu.sched_task)
2312 x86_pmu.sched_task(ctx, sched_in);
2313}
2314
2315static void x86_pmu_swap_task_ctx(struct perf_event_context *prev,
2316 struct perf_event_context *next)
2317{
2318 if (x86_pmu.swap_task_ctx)
2319 x86_pmu.swap_task_ctx(prev, next);
2320}
2321
2322void perf_check_microcode(void)
2323{
2324 if (x86_pmu.check_microcode)
2325 x86_pmu.check_microcode();
2326}
2327
2328static int x86_pmu_check_period(struct perf_event *event, u64 value)
2329{
2330 if (x86_pmu.check_period && x86_pmu.check_period(event, value))
2331 return -EINVAL;
2332
2333 if (value && x86_pmu.limit_period) {
2334 if (x86_pmu.limit_period(event, value) > value)
2335 return -EINVAL;
2336 }
2337
2338 return 0;
2339}
2340
2341static int x86_pmu_aux_output_match(struct perf_event *event)
2342{
2343 if (!(pmu.capabilities & PERF_PMU_CAP_AUX_OUTPUT))
2344 return 0;
2345
2346 if (x86_pmu.aux_output_match)
2347 return x86_pmu.aux_output_match(event);
2348
2349 return 0;
2350}
2351
2352static struct pmu pmu = {
2353 .pmu_enable = x86_pmu_enable,
2354 .pmu_disable = x86_pmu_disable,
2355
2356 .attr_groups = x86_pmu_attr_groups,
2357
2358 .event_init = x86_pmu_event_init,
2359
2360 .event_mapped = x86_pmu_event_mapped,
2361 .event_unmapped = x86_pmu_event_unmapped,
2362
2363 .add = x86_pmu_add,
2364 .del = x86_pmu_del,
2365 .start = x86_pmu_start,
2366 .stop = x86_pmu_stop,
2367 .read = x86_pmu_read,
2368
2369 .start_txn = x86_pmu_start_txn,
2370 .cancel_txn = x86_pmu_cancel_txn,
2371 .commit_txn = x86_pmu_commit_txn,
2372
2373 .event_idx = x86_pmu_event_idx,
2374 .sched_task = x86_pmu_sched_task,
2375 .swap_task_ctx = x86_pmu_swap_task_ctx,
2376 .check_period = x86_pmu_check_period,
2377
2378 .aux_output_match = x86_pmu_aux_output_match,
2379};
2380
2381void arch_perf_update_userpage(struct perf_event *event,
2382 struct perf_event_mmap_page *userpg, u64 now)
2383{
2384 struct cyc2ns_data data;
2385 u64 offset;
2386
2387 userpg->cap_user_time = 0;
2388 userpg->cap_user_time_zero = 0;
2389 userpg->cap_user_rdpmc =
2390 !!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED);
2391 userpg->pmc_width = x86_pmu.cntval_bits;
2392
2393 if (!using_native_sched_clock() || !sched_clock_stable())
2394 return;
2395
2396 cyc2ns_read_begin(&data);
2397
2398 offset = data.cyc2ns_offset + __sched_clock_offset;
2399
2400
2401
2402
2403
2404 userpg->cap_user_time = 1;
2405 userpg->time_mult = data.cyc2ns_mul;
2406 userpg->time_shift = data.cyc2ns_shift;
2407 userpg->time_offset = offset - now;
2408
2409
2410
2411
2412
2413 if (!event->attr.use_clockid) {
2414 userpg->cap_user_time_zero = 1;
2415 userpg->time_zero = offset;
2416 }
2417
2418 cyc2ns_read_end();
2419}
2420
2421
2422
2423
2424
2425static bool perf_hw_regs(struct pt_regs *regs)
2426{
2427 return regs->flags & X86_EFLAGS_FIXED;
2428}
2429
2430void
2431perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs)
2432{
2433 struct unwind_state state;
2434 unsigned long addr;
2435
2436 if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
2437
2438 return;
2439 }
2440
2441 if (perf_callchain_store(entry, regs->ip))
2442 return;
2443
2444 if (perf_hw_regs(regs))
2445 unwind_start(&state, current, regs, NULL);
2446 else
2447 unwind_start(&state, current, NULL, (void *)regs->sp);
2448
2449 for (; !unwind_done(&state); unwind_next_frame(&state)) {
2450 addr = unwind_get_return_address(&state);
2451 if (!addr || perf_callchain_store(entry, addr))
2452 return;
2453 }
2454}
2455
2456static inline int
2457valid_user_frame(const void __user *fp, unsigned long size)
2458{
2459 return (__range_not_ok(fp, size, TASK_SIZE) == 0);
2460}
2461
2462static unsigned long get_segment_base(unsigned int segment)
2463{
2464 struct desc_struct *desc;
2465 unsigned int idx = segment >> 3;
2466
2467 if ((segment & SEGMENT_TI_MASK) == SEGMENT_LDT) {
2468#ifdef CONFIG_MODIFY_LDT_SYSCALL
2469 struct ldt_struct *ldt;
2470
2471
2472 ldt = READ_ONCE(current->active_mm->context.ldt);
2473 if (!ldt || idx >= ldt->nr_entries)
2474 return 0;
2475
2476 desc = &ldt->entries[idx];
2477#else
2478 return 0;
2479#endif
2480 } else {
2481 if (idx >= GDT_ENTRIES)
2482 return 0;
2483
2484 desc = raw_cpu_ptr(gdt_page.gdt) + idx;
2485 }
2486
2487 return get_desc_base(desc);
2488}
2489
2490#ifdef CONFIG_IA32_EMULATION
2491
2492#include <linux/compat.h>
2493
2494static inline int
2495perf_callchain_user32(struct pt_regs *regs, struct perf_callchain_entry_ctx *entry)
2496{
2497
2498 unsigned long ss_base, cs_base;
2499 struct stack_frame_ia32 frame;
2500 const struct stack_frame_ia32 __user *fp;
2501
2502 if (!test_thread_flag(TIF_IA32))
2503 return 0;
2504
2505 cs_base = get_segment_base(regs->cs);
2506 ss_base = get_segment_base(regs->ss);
2507
2508 fp = compat_ptr(ss_base + regs->bp);
2509 pagefault_disable();
2510 while (entry->nr < entry->max_stack) {
2511 if (!valid_user_frame(fp, sizeof(frame)))
2512 break;
2513
2514 if (__get_user(frame.next_frame, &fp->next_frame))
2515 break;
2516 if (__get_user(frame.return_address, &fp->return_address))
2517 break;
2518
2519 perf_callchain_store(entry, cs_base + frame.return_address);
2520 fp = compat_ptr(ss_base + frame.next_frame);
2521 }
2522 pagefault_enable();
2523 return 1;
2524}
2525#else
2526static inline int
2527perf_callchain_user32(struct pt_regs *regs, struct perf_callchain_entry_ctx *entry)
2528{
2529 return 0;
2530}
2531#endif
2532
2533void
2534perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs)
2535{
2536 struct stack_frame frame;
2537 const struct stack_frame __user *fp;
2538
2539 if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
2540
2541 return;
2542 }
2543
2544
2545
2546
2547 if (regs->flags & (X86_VM_MASK | PERF_EFLAGS_VM))
2548 return;
2549
2550 fp = (void __user *)regs->bp;
2551
2552 perf_callchain_store(entry, regs->ip);
2553
2554 if (!nmi_uaccess_okay())
2555 return;
2556
2557 if (perf_callchain_user32(regs, entry))
2558 return;
2559
2560 pagefault_disable();
2561 while (entry->nr < entry->max_stack) {
2562 if (!valid_user_frame(fp, sizeof(frame)))
2563 break;
2564
2565 if (__get_user(frame.next_frame, &fp->next_frame))
2566 break;
2567 if (__get_user(frame.return_address, &fp->return_address))
2568 break;
2569
2570 perf_callchain_store(entry, frame.return_address);
2571 fp = (void __user *)frame.next_frame;
2572 }
2573 pagefault_enable();
2574}
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589static unsigned long code_segment_base(struct pt_regs *regs)
2590{
2591
2592
2593
2594
2595
2596#ifdef CONFIG_X86_32
2597
2598
2599
2600
2601 if (regs->flags & X86_VM_MASK)
2602 return 0x10 * regs->cs;
2603
2604 if (user_mode(regs) && regs->cs != __USER_CS)
2605 return get_segment_base(regs->cs);
2606#else
2607 if (user_mode(regs) && !user_64bit_mode(regs) &&
2608 regs->cs != __USER32_CS)
2609 return get_segment_base(regs->cs);
2610#endif
2611 return 0;
2612}
2613
2614unsigned long perf_instruction_pointer(struct pt_regs *regs)
2615{
2616 if (perf_guest_cbs && perf_guest_cbs->is_in_guest())
2617 return perf_guest_cbs->get_guest_ip();
2618
2619 return regs->ip + code_segment_base(regs);
2620}
2621
2622unsigned long perf_misc_flags(struct pt_regs *regs)
2623{
2624 int misc = 0;
2625
2626 if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
2627 if (perf_guest_cbs->is_user_mode())
2628 misc |= PERF_RECORD_MISC_GUEST_USER;
2629 else
2630 misc |= PERF_RECORD_MISC_GUEST_KERNEL;
2631 } else {
2632 if (user_mode(regs))
2633 misc |= PERF_RECORD_MISC_USER;
2634 else
2635 misc |= PERF_RECORD_MISC_KERNEL;
2636 }
2637
2638 if (regs->flags & PERF_EFLAGS_EXACT)
2639 misc |= PERF_RECORD_MISC_EXACT_IP;
2640
2641 return misc;
2642}
2643
2644void perf_get_x86_pmu_capability(struct x86_pmu_capability *cap)
2645{
2646 cap->version = x86_pmu.version;
2647 cap->num_counters_gp = x86_pmu.num_counters;
2648 cap->num_counters_fixed = x86_pmu.num_counters_fixed;
2649 cap->bit_width_gp = x86_pmu.cntval_bits;
2650 cap->bit_width_fixed = x86_pmu.cntval_bits;
2651 cap->events_mask = (unsigned int)x86_pmu.events_maskl;
2652 cap->events_mask_len = x86_pmu.events_mask_len;
2653}
2654EXPORT_SYMBOL_GPL(perf_get_x86_pmu_capability);
2655