1
2
3
4
5
6
7
8
9
10
11#include <linux/kernel.h>
12#include <linux/sched.h>
13#include <linux/perf_event.h>
14#include <linux/percpu.h>
15#include <linux/hardirq.h>
16#include <linux/uaccess.h>
17#include <asm/reg.h>
18#include <asm/pmc.h>
19#include <asm/machdep.h>
20#include <asm/firmware.h>
21#include <asm/ptrace.h>
22#include <asm/code-patching.h>
23
24#define BHRB_MAX_ENTRIES 32
25#define BHRB_TARGET 0x0000000000000002
26#define BHRB_PREDICTION 0x0000000000000001
27#define BHRB_EA 0xFFFFFFFFFFFFFFFCUL
28
29struct cpu_hw_events {
30 int n_events;
31 int n_percpu;
32 int disabled;
33 int n_added;
34 int n_limited;
35 u8 pmcs_enabled;
36 struct perf_event *event[MAX_HWEVENTS];
37 u64 events[MAX_HWEVENTS];
38 unsigned int flags[MAX_HWEVENTS];
39 unsigned long mmcr[3];
40 struct perf_event *limited_counter[MAX_LIMITED_HWCOUNTERS];
41 u8 limited_hwidx[MAX_LIMITED_HWCOUNTERS];
42 u64 alternatives[MAX_HWEVENTS][MAX_EVENT_ALTERNATIVES];
43 unsigned long amasks[MAX_HWEVENTS][MAX_EVENT_ALTERNATIVES];
44 unsigned long avalues[MAX_HWEVENTS][MAX_EVENT_ALTERNATIVES];
45
46 unsigned int group_flag;
47 int n_txn_start;
48
49
50 u64 bhrb_filter;
51 int bhrb_users;
52 void *bhrb_context;
53 struct perf_branch_stack bhrb_stack;
54 struct perf_branch_entry bhrb_entries[BHRB_MAX_ENTRIES];
55};
56
57DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events);
58
59struct power_pmu *ppmu;
60
61
62
63
64
65
66
67
68static unsigned int freeze_events_kernel = MMCR0_FCS;
69
70
71
72
73
74#ifdef CONFIG_PPC32
75
76#define MMCR0_FCHV 0
77#define MMCR0_PMCjCE MMCR0_PMCnCE
78#define MMCR0_FC56 0
79#define MMCR0_PMAO 0
80#define MMCR0_EBE 0
81#define MMCR0_PMCC 0
82#define MMCR0_PMCC_U6 0
83
84#define SPRN_MMCRA SPRN_MMCR2
85#define MMCRA_SAMPLE_ENABLE 0
86
87static inline unsigned long perf_ip_adjust(struct pt_regs *regs)
88{
89 return 0;
90}
91static inline void perf_get_data_addr(struct pt_regs *regs, u64 *addrp) { }
92static inline u32 perf_get_misc_flags(struct pt_regs *regs)
93{
94 return 0;
95}
96static inline void perf_read_regs(struct pt_regs *regs)
97{
98 regs->result = 0;
99}
100static inline int perf_intr_is_nmi(struct pt_regs *regs)
101{
102 return 0;
103}
104
105static inline int siar_valid(struct pt_regs *regs)
106{
107 return 1;
108}
109
110static bool is_ebb_event(struct perf_event *event) { return false; }
111static int ebb_event_check(struct perf_event *event) { return 0; }
112static void ebb_event_add(struct perf_event *event) { }
113static void ebb_switch_out(unsigned long mmcr0) { }
114static unsigned long ebb_switch_in(bool ebb, unsigned long mmcr0)
115{
116 return mmcr0;
117}
118
119static inline void power_pmu_bhrb_enable(struct perf_event *event) {}
120static inline void power_pmu_bhrb_disable(struct perf_event *event) {}
121void power_pmu_flush_branch_stack(void) {}
122static inline void power_pmu_bhrb_read(struct cpu_hw_events *cpuhw) {}
123#endif
124
125static bool regs_use_siar(struct pt_regs *regs)
126{
127 return !!regs->result;
128}
129
130
131
132
133#ifdef CONFIG_PPC64
134
135static inline unsigned long perf_ip_adjust(struct pt_regs *regs)
136{
137 unsigned long mmcra = regs->dsisr;
138
139 if ((ppmu->flags & PPMU_HAS_SSLOT) && (mmcra & MMCRA_SAMPLE_ENABLE)) {
140 unsigned long slot = (mmcra & MMCRA_SLOT) >> MMCRA_SLOT_SHIFT;
141 if (slot > 1)
142 return 4 * (slot - 1);
143 }
144
145 return 0;
146}
147
148
149
150
151
152
153
154
155
156static inline void perf_get_data_addr(struct pt_regs *regs, u64 *addrp)
157{
158 unsigned long mmcra = regs->dsisr;
159 bool sdar_valid;
160
161 if (ppmu->flags & PPMU_HAS_SIER)
162 sdar_valid = regs->dar & SIER_SDAR_VALID;
163 else {
164 unsigned long sdsync;
165
166 if (ppmu->flags & PPMU_SIAR_VALID)
167 sdsync = POWER7P_MMCRA_SDAR_VALID;
168 else if (ppmu->flags & PPMU_ALT_SIPR)
169 sdsync = POWER6_MMCRA_SDSYNC;
170 else
171 sdsync = MMCRA_SDSYNC;
172
173 sdar_valid = mmcra & sdsync;
174 }
175
176 if (!(mmcra & MMCRA_SAMPLE_ENABLE) || sdar_valid)
177 *addrp = mfspr(SPRN_SDAR);
178}
179
180static bool regs_sihv(struct pt_regs *regs)
181{
182 unsigned long sihv = MMCRA_SIHV;
183
184 if (ppmu->flags & PPMU_HAS_SIER)
185 return !!(regs->dar & SIER_SIHV);
186
187 if (ppmu->flags & PPMU_ALT_SIPR)
188 sihv = POWER6_MMCRA_SIHV;
189
190 return !!(regs->dsisr & sihv);
191}
192
193static bool regs_sipr(struct pt_regs *regs)
194{
195 unsigned long sipr = MMCRA_SIPR;
196
197 if (ppmu->flags & PPMU_HAS_SIER)
198 return !!(regs->dar & SIER_SIPR);
199
200 if (ppmu->flags & PPMU_ALT_SIPR)
201 sipr = POWER6_MMCRA_SIPR;
202
203 return !!(regs->dsisr & sipr);
204}
205
206static inline u32 perf_flags_from_msr(struct pt_regs *regs)
207{
208 if (regs->msr & MSR_PR)
209 return PERF_RECORD_MISC_USER;
210 if ((regs->msr & MSR_HV) && freeze_events_kernel != MMCR0_FCHV)
211 return PERF_RECORD_MISC_HYPERVISOR;
212 return PERF_RECORD_MISC_KERNEL;
213}
214
215static inline u32 perf_get_misc_flags(struct pt_regs *regs)
216{
217 bool use_siar = regs_use_siar(regs);
218
219 if (!use_siar)
220 return perf_flags_from_msr(regs);
221
222
223
224
225
226
227
228 if (ppmu->flags & PPMU_NO_SIPR) {
229 unsigned long siar = mfspr(SPRN_SIAR);
230 if (siar >= PAGE_OFFSET)
231 return PERF_RECORD_MISC_KERNEL;
232 return PERF_RECORD_MISC_USER;
233 }
234
235
236 if (regs_sipr(regs))
237 return PERF_RECORD_MISC_USER;
238
239 if (regs_sihv(regs) && (freeze_events_kernel != MMCR0_FCHV))
240 return PERF_RECORD_MISC_HYPERVISOR;
241
242 return PERF_RECORD_MISC_KERNEL;
243}
244
245
246
247
248
249
250
251
252static inline void perf_read_regs(struct pt_regs *regs)
253{
254 unsigned long mmcra = mfspr(SPRN_MMCRA);
255 int marked = mmcra & MMCRA_SAMPLE_ENABLE;
256 int use_siar;
257
258 regs->dsisr = mmcra;
259
260 if (ppmu->flags & PPMU_HAS_SIER)
261 regs->dar = mfspr(SPRN_SIER);
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280 if (TRAP(regs) != 0xf00)
281 use_siar = 0;
282 else if (marked)
283 use_siar = 1;
284 else if ((ppmu->flags & PPMU_NO_CONT_SAMPLING))
285 use_siar = 0;
286 else if (!(ppmu->flags & PPMU_NO_SIPR) && regs_sipr(regs))
287 use_siar = 0;
288 else
289 use_siar = 1;
290
291 regs->result = use_siar;
292}
293
294
295
296
297
298static inline int perf_intr_is_nmi(struct pt_regs *regs)
299{
300 return !regs->softe;
301}
302
303
304
305
306
307
308
309
310static inline int siar_valid(struct pt_regs *regs)
311{
312 unsigned long mmcra = regs->dsisr;
313 int marked = mmcra & MMCRA_SAMPLE_ENABLE;
314
315 if (marked) {
316 if (ppmu->flags & PPMU_HAS_SIER)
317 return regs->dar & SIER_SIAR_VALID;
318
319 if (ppmu->flags & PPMU_SIAR_VALID)
320 return mmcra & POWER7P_MMCRA_SIAR_VALID;
321 }
322
323 return 1;
324}
325
326
327
328static void power_pmu_bhrb_reset(void)
329{
330 asm volatile(PPC_CLRBHRB);
331}
332
333static void power_pmu_bhrb_enable(struct perf_event *event)
334{
335 struct cpu_hw_events *cpuhw = &__get_cpu_var(cpu_hw_events);
336
337 if (!ppmu->bhrb_nr)
338 return;
339
340
341 if (event->ctx->task && cpuhw->bhrb_context != event->ctx) {
342 power_pmu_bhrb_reset();
343 cpuhw->bhrb_context = event->ctx;
344 }
345 cpuhw->bhrb_users++;
346}
347
348static void power_pmu_bhrb_disable(struct perf_event *event)
349{
350 struct cpu_hw_events *cpuhw = &__get_cpu_var(cpu_hw_events);
351
352 if (!ppmu->bhrb_nr)
353 return;
354
355 cpuhw->bhrb_users--;
356 WARN_ON_ONCE(cpuhw->bhrb_users < 0);
357
358 if (!cpuhw->disabled && !cpuhw->bhrb_users) {
359
360
361
362
363
364 cpuhw->bhrb_context = NULL;
365 }
366}
367
368
369
370
371void power_pmu_flush_branch_stack(void)
372{
373 if (ppmu->bhrb_nr)
374 power_pmu_bhrb_reset();
375}
376
377static __u64 power_pmu_bhrb_to(u64 addr)
378{
379 unsigned int instr;
380 int ret;
381 __u64 target;
382
383 if (is_kernel_addr(addr))
384 return branch_target((unsigned int *)addr);
385
386
387 pagefault_disable();
388 ret = __get_user_inatomic(instr, (unsigned int __user *)addr);
389 if (ret) {
390 pagefault_enable();
391 return 0;
392 }
393 pagefault_enable();
394
395 target = branch_target(&instr);
396 if ((!target) || (instr & BRANCH_ABSOLUTE))
397 return target;
398
399
400 return target - (unsigned long)&instr + addr;
401}
402
403
404void power_pmu_bhrb_read(struct cpu_hw_events *cpuhw)
405{
406 u64 val;
407 u64 addr;
408 int r_index, u_index, pred;
409
410 r_index = 0;
411 u_index = 0;
412 while (r_index < ppmu->bhrb_nr) {
413
414 val = read_bhrb(r_index++);
415 if (!val)
416
417 break;
418 else {
419 addr = val & BHRB_EA;
420 pred = val & BHRB_PREDICTION;
421
422 if (!addr)
423
424 continue;
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444 if (val & BHRB_TARGET) {
445
446
447
448 cpuhw->bhrb_entries[u_index].to = addr;
449 cpuhw->bhrb_entries[u_index].mispred = pred;
450 cpuhw->bhrb_entries[u_index].predicted = ~pred;
451
452
453 val = read_bhrb(r_index++);
454 addr = val & BHRB_EA;
455 if (val & BHRB_TARGET) {
456
457
458 r_index--;
459 addr = 0;
460 }
461 cpuhw->bhrb_entries[u_index].from = addr;
462 } else {
463
464
465 cpuhw->bhrb_entries[u_index].from = addr;
466 cpuhw->bhrb_entries[u_index].to =
467 power_pmu_bhrb_to(addr);
468 cpuhw->bhrb_entries[u_index].mispred = pred;
469 cpuhw->bhrb_entries[u_index].predicted = ~pred;
470 }
471 u_index++;
472
473 }
474 }
475 cpuhw->bhrb_stack.nr = u_index;
476 return;
477}
478
479static bool is_ebb_event(struct perf_event *event)
480{
481
482
483
484
485
486 return (ppmu->flags & PPMU_EBB) &&
487 ((event->attr.config >> PERF_EVENT_CONFIG_EBB_SHIFT) & 1);
488}
489
490static int ebb_event_check(struct perf_event *event)
491{
492 struct perf_event *leader = event->group_leader;
493
494
495 if (is_ebb_event(leader) != is_ebb_event(event))
496 return -EINVAL;
497
498 if (is_ebb_event(event)) {
499 if (!(event->attach_state & PERF_ATTACH_TASK))
500 return -EINVAL;
501
502 if (!leader->attr.pinned || !leader->attr.exclusive)
503 return -EINVAL;
504
505 if (event->attr.inherit || event->attr.sample_period ||
506 event->attr.enable_on_exec || event->attr.freq)
507 return -EINVAL;
508 }
509
510 return 0;
511}
512
513static void ebb_event_add(struct perf_event *event)
514{
515 if (!is_ebb_event(event) || current->thread.used_ebb)
516 return;
517
518
519
520
521
522
523
524 current->thread.used_ebb = 1;
525 current->thread.mmcr0 |= MMCR0_PMXE;
526}
527
528static void ebb_switch_out(unsigned long mmcr0)
529{
530 if (!(mmcr0 & MMCR0_EBE))
531 return;
532
533 current->thread.siar = mfspr(SPRN_SIAR);
534 current->thread.sier = mfspr(SPRN_SIER);
535 current->thread.sdar = mfspr(SPRN_SDAR);
536 current->thread.mmcr0 = mmcr0 & MMCR0_USER_MASK;
537 current->thread.mmcr2 = mfspr(SPRN_MMCR2) & MMCR2_USER_MASK;
538}
539
540static unsigned long ebb_switch_in(bool ebb, unsigned long mmcr0)
541{
542 if (!ebb)
543 goto out;
544
545
546 mmcr0 |= MMCR0_EBE | MMCR0_PMCC_U6;
547
548
549 mmcr0 |= current->thread.mmcr0;
550
551
552 if (!(current->thread.mmcr0 & MMCR0_PMXE))
553 mmcr0 &= ~MMCR0_PMXE;
554
555 mtspr(SPRN_SIAR, current->thread.siar);
556 mtspr(SPRN_SIER, current->thread.sier);
557 mtspr(SPRN_SDAR, current->thread.sdar);
558 mtspr(SPRN_MMCR2, current->thread.mmcr2);
559out:
560 return mmcr0;
561}
562#endif
563
564static void perf_event_interrupt(struct pt_regs *regs);
565
566void perf_event_print_debug(void)
567{
568}
569
570
571
572
573static unsigned long read_pmc(int idx)
574{
575 unsigned long val;
576
577 switch (idx) {
578 case 1:
579 val = mfspr(SPRN_PMC1);
580 break;
581 case 2:
582 val = mfspr(SPRN_PMC2);
583 break;
584 case 3:
585 val = mfspr(SPRN_PMC3);
586 break;
587 case 4:
588 val = mfspr(SPRN_PMC4);
589 break;
590 case 5:
591 val = mfspr(SPRN_PMC5);
592 break;
593 case 6:
594 val = mfspr(SPRN_PMC6);
595 break;
596#ifdef CONFIG_PPC64
597 case 7:
598 val = mfspr(SPRN_PMC7);
599 break;
600 case 8:
601 val = mfspr(SPRN_PMC8);
602 break;
603#endif
604 default:
605 printk(KERN_ERR "oops trying to read PMC%d\n", idx);
606 val = 0;
607 }
608 return val;
609}
610
611
612
613
614static void write_pmc(int idx, unsigned long val)
615{
616 switch (idx) {
617 case 1:
618 mtspr(SPRN_PMC1, val);
619 break;
620 case 2:
621 mtspr(SPRN_PMC2, val);
622 break;
623 case 3:
624 mtspr(SPRN_PMC3, val);
625 break;
626 case 4:
627 mtspr(SPRN_PMC4, val);
628 break;
629 case 5:
630 mtspr(SPRN_PMC5, val);
631 break;
632 case 6:
633 mtspr(SPRN_PMC6, val);
634 break;
635#ifdef CONFIG_PPC64
636 case 7:
637 mtspr(SPRN_PMC7, val);
638 break;
639 case 8:
640 mtspr(SPRN_PMC8, val);
641 break;
642#endif
643 default:
644 printk(KERN_ERR "oops trying to write PMC%d\n", idx);
645 }
646}
647
648
649
650
651
652
653
654static int power_check_constraints(struct cpu_hw_events *cpuhw,
655 u64 event_id[], unsigned int cflags[],
656 int n_ev)
657{
658 unsigned long mask, value, nv;
659 unsigned long smasks[MAX_HWEVENTS], svalues[MAX_HWEVENTS];
660 int n_alt[MAX_HWEVENTS], choice[MAX_HWEVENTS];
661 int i, j;
662 unsigned long addf = ppmu->add_fields;
663 unsigned long tadd = ppmu->test_adder;
664
665 if (n_ev > ppmu->n_counter)
666 return -1;
667
668
669 for (i = 0; i < n_ev; ++i) {
670 if ((cflags[i] & PPMU_LIMITED_PMC_REQD)
671 && !ppmu->limited_pmc_event(event_id[i])) {
672 ppmu->get_alternatives(event_id[i], cflags[i],
673 cpuhw->alternatives[i]);
674 event_id[i] = cpuhw->alternatives[i][0];
675 }
676 if (ppmu->get_constraint(event_id[i], &cpuhw->amasks[i][0],
677 &cpuhw->avalues[i][0]))
678 return -1;
679 }
680 value = mask = 0;
681 for (i = 0; i < n_ev; ++i) {
682 nv = (value | cpuhw->avalues[i][0]) +
683 (value & cpuhw->avalues[i][0] & addf);
684 if ((((nv + tadd) ^ value) & mask) != 0 ||
685 (((nv + tadd) ^ cpuhw->avalues[i][0]) &
686 cpuhw->amasks[i][0]) != 0)
687 break;
688 value = nv;
689 mask |= cpuhw->amasks[i][0];
690 }
691 if (i == n_ev)
692 return 0;
693
694
695 if (!ppmu->get_alternatives)
696 return -1;
697 for (i = 0; i < n_ev; ++i) {
698 choice[i] = 0;
699 n_alt[i] = ppmu->get_alternatives(event_id[i], cflags[i],
700 cpuhw->alternatives[i]);
701 for (j = 1; j < n_alt[i]; ++j)
702 ppmu->get_constraint(cpuhw->alternatives[i][j],
703 &cpuhw->amasks[i][j],
704 &cpuhw->avalues[i][j]);
705 }
706
707
708 i = 0;
709 j = -1;
710 value = mask = nv = 0;
711 while (i < n_ev) {
712 if (j >= 0) {
713
714 value = svalues[i];
715 mask = smasks[i];
716 j = choice[i];
717 }
718
719
720
721
722 while (++j < n_alt[i]) {
723 nv = (value | cpuhw->avalues[i][j]) +
724 (value & cpuhw->avalues[i][j] & addf);
725 if ((((nv + tadd) ^ value) & mask) == 0 &&
726 (((nv + tadd) ^ cpuhw->avalues[i][j])
727 & cpuhw->amasks[i][j]) == 0)
728 break;
729 }
730 if (j >= n_alt[i]) {
731
732
733
734
735
736 if (--i < 0)
737 return -1;
738 } else {
739
740
741
742
743
744
745 choice[i] = j;
746 svalues[i] = value;
747 smasks[i] = mask;
748 value = nv;
749 mask |= cpuhw->amasks[i][j];
750 ++i;
751 j = -1;
752 }
753 }
754
755
756 for (i = 0; i < n_ev; ++i)
757 event_id[i] = cpuhw->alternatives[i][choice[i]];
758 return 0;
759}
760
761
762
763
764
765
766static int check_excludes(struct perf_event **ctrs, unsigned int cflags[],
767 int n_prev, int n_new)
768{
769 int eu = 0, ek = 0, eh = 0;
770 int i, n, first;
771 struct perf_event *event;
772
773 n = n_prev + n_new;
774 if (n <= 1)
775 return 0;
776
777 first = 1;
778 for (i = 0; i < n; ++i) {
779 if (cflags[i] & PPMU_LIMITED_PMC_OK) {
780 cflags[i] &= ~PPMU_LIMITED_PMC_REQD;
781 continue;
782 }
783 event = ctrs[i];
784 if (first) {
785 eu = event->attr.exclude_user;
786 ek = event->attr.exclude_kernel;
787 eh = event->attr.exclude_hv;
788 first = 0;
789 } else if (event->attr.exclude_user != eu ||
790 event->attr.exclude_kernel != ek ||
791 event->attr.exclude_hv != eh) {
792 return -EAGAIN;
793 }
794 }
795
796 if (eu || ek || eh)
797 for (i = 0; i < n; ++i)
798 if (cflags[i] & PPMU_LIMITED_PMC_OK)
799 cflags[i] |= PPMU_LIMITED_PMC_REQD;
800
801 return 0;
802}
803
804static u64 check_and_compute_delta(u64 prev, u64 val)
805{
806 u64 delta = (val - prev) & 0xfffffffful;
807
808
809
810
811
812
813
814
815
816
817 if (prev > val && (prev - val) < 256)
818 delta = 0;
819
820 return delta;
821}
822
823static void power_pmu_read(struct perf_event *event)
824{
825 s64 val, delta, prev;
826
827 if (event->hw.state & PERF_HES_STOPPED)
828 return;
829
830 if (!event->hw.idx)
831 return;
832
833 if (is_ebb_event(event)) {
834 val = read_pmc(event->hw.idx);
835 local64_set(&event->hw.prev_count, val);
836 return;
837 }
838
839
840
841
842
843
844 do {
845 prev = local64_read(&event->hw.prev_count);
846 barrier();
847 val = read_pmc(event->hw.idx);
848 delta = check_and_compute_delta(prev, val);
849 if (!delta)
850 return;
851 } while (local64_cmpxchg(&event->hw.prev_count, prev, val) != prev);
852
853 local64_add(delta, &event->count);
854 local64_sub(delta, &event->hw.period_left);
855}
856
857
858
859
860
861
862static int is_limited_pmc(int pmcnum)
863{
864 return (ppmu->flags & PPMU_LIMITED_PMC5_6)
865 && (pmcnum == 5 || pmcnum == 6);
866}
867
868static void freeze_limited_counters(struct cpu_hw_events *cpuhw,
869 unsigned long pmc5, unsigned long pmc6)
870{
871 struct perf_event *event;
872 u64 val, prev, delta;
873 int i;
874
875 for (i = 0; i < cpuhw->n_limited; ++i) {
876 event = cpuhw->limited_counter[i];
877 if (!event->hw.idx)
878 continue;
879 val = (event->hw.idx == 5) ? pmc5 : pmc6;
880 prev = local64_read(&event->hw.prev_count);
881 event->hw.idx = 0;
882 delta = check_and_compute_delta(prev, val);
883 if (delta)
884 local64_add(delta, &event->count);
885 }
886}
887
888static void thaw_limited_counters(struct cpu_hw_events *cpuhw,
889 unsigned long pmc5, unsigned long pmc6)
890{
891 struct perf_event *event;
892 u64 val, prev;
893 int i;
894
895 for (i = 0; i < cpuhw->n_limited; ++i) {
896 event = cpuhw->limited_counter[i];
897 event->hw.idx = cpuhw->limited_hwidx[i];
898 val = (event->hw.idx == 5) ? pmc5 : pmc6;
899 prev = local64_read(&event->hw.prev_count);
900 if (check_and_compute_delta(prev, val))
901 local64_set(&event->hw.prev_count, val);
902 perf_event_update_userpage(event);
903 }
904}
905
906
907
908
909
910
911
912
913
914
915
916
917static void write_mmcr0(struct cpu_hw_events *cpuhw, unsigned long mmcr0)
918{
919 unsigned long pmc5, pmc6;
920
921 if (!cpuhw->n_limited) {
922 mtspr(SPRN_MMCR0, mmcr0);
923 return;
924 }
925
926
927
928
929
930
931
932
933 asm volatile("mtspr %3,%2; mfspr %0,%4; mfspr %1,%5"
934 : "=&r" (pmc5), "=&r" (pmc6)
935 : "r" (mmcr0 & ~(MMCR0_PMC1CE | MMCR0_PMCjCE)),
936 "i" (SPRN_MMCR0),
937 "i" (SPRN_PMC5), "i" (SPRN_PMC6));
938
939 if (mmcr0 & MMCR0_FC)
940 freeze_limited_counters(cpuhw, pmc5, pmc6);
941 else
942 thaw_limited_counters(cpuhw, pmc5, pmc6);
943
944
945
946
947
948 if (mmcr0 & (MMCR0_PMC1CE | MMCR0_PMCjCE))
949 mtspr(SPRN_MMCR0, mmcr0);
950}
951
952
953
954
955
956static void power_pmu_disable(struct pmu *pmu)
957{
958 struct cpu_hw_events *cpuhw;
959 unsigned long flags, mmcr0, val;
960
961 if (!ppmu)
962 return;
963 local_irq_save(flags);
964 cpuhw = &__get_cpu_var(cpu_hw_events);
965
966 if (!cpuhw->disabled) {
967
968
969
970 if (!cpuhw->pmcs_enabled) {
971 ppc_enable_pmcs();
972 cpuhw->pmcs_enabled = 1;
973 }
974
975
976
977
978 val = mmcr0 = mfspr(SPRN_MMCR0);
979 val |= MMCR0_FC;
980 val &= ~(MMCR0_EBE | MMCR0_PMCC | MMCR0_PMAO | MMCR0_FC56);
981
982
983
984
985
986
987 write_mmcr0(cpuhw, val);
988 mb();
989
990
991
992
993 if (cpuhw->mmcr[2] & MMCRA_SAMPLE_ENABLE) {
994 mtspr(SPRN_MMCRA,
995 cpuhw->mmcr[2] & ~MMCRA_SAMPLE_ENABLE);
996 mb();
997 }
998
999 cpuhw->disabled = 1;
1000 cpuhw->n_added = 0;
1001
1002 ebb_switch_out(mmcr0);
1003 }
1004
1005 local_irq_restore(flags);
1006}
1007
1008
1009
1010
1011
1012
1013static void power_pmu_enable(struct pmu *pmu)
1014{
1015 struct perf_event *event;
1016 struct cpu_hw_events *cpuhw;
1017 unsigned long flags;
1018 long i;
1019 unsigned long val, mmcr0;
1020 s64 left;
1021 unsigned int hwc_index[MAX_HWEVENTS];
1022 int n_lim;
1023 int idx;
1024 bool ebb;
1025
1026 if (!ppmu)
1027 return;
1028 local_irq_save(flags);
1029
1030 cpuhw = &__get_cpu_var(cpu_hw_events);
1031 if (!cpuhw->disabled)
1032 goto out;
1033
1034 if (cpuhw->n_events == 0) {
1035 ppc_set_pmu_inuse(0);
1036 goto out;
1037 }
1038
1039 cpuhw->disabled = 0;
1040
1041
1042
1043
1044
1045
1046 ebb = is_ebb_event(cpuhw->event[0]);
1047
1048
1049
1050
1051
1052
1053
1054 if (!cpuhw->n_added) {
1055 mtspr(SPRN_MMCRA, cpuhw->mmcr[2] & ~MMCRA_SAMPLE_ENABLE);
1056 mtspr(SPRN_MMCR1, cpuhw->mmcr[1]);
1057 goto out_enable;
1058 }
1059
1060
1061
1062
1063 if (ppmu->compute_mmcr(cpuhw->events, cpuhw->n_events, hwc_index,
1064 cpuhw->mmcr)) {
1065
1066 printk(KERN_ERR "oops compute_mmcr failed\n");
1067 goto out;
1068 }
1069
1070
1071
1072
1073
1074
1075
1076 event = cpuhw->event[0];
1077 if (event->attr.exclude_user)
1078 cpuhw->mmcr[0] |= MMCR0_FCP;
1079 if (event->attr.exclude_kernel)
1080 cpuhw->mmcr[0] |= freeze_events_kernel;
1081 if (event->attr.exclude_hv)
1082 cpuhw->mmcr[0] |= MMCR0_FCHV;
1083
1084
1085
1086
1087
1088
1089 ppc_set_pmu_inuse(1);
1090 mtspr(SPRN_MMCRA, cpuhw->mmcr[2] & ~MMCRA_SAMPLE_ENABLE);
1091 mtspr(SPRN_MMCR1, cpuhw->mmcr[1]);
1092 mtspr(SPRN_MMCR0, (cpuhw->mmcr[0] & ~(MMCR0_PMC1CE | MMCR0_PMCjCE))
1093 | MMCR0_FC);
1094
1095
1096
1097
1098
1099 for (i = 0; i < cpuhw->n_events; ++i) {
1100 event = cpuhw->event[i];
1101 if (event->hw.idx && event->hw.idx != hwc_index[i] + 1) {
1102 power_pmu_read(event);
1103 write_pmc(event->hw.idx, 0);
1104 event->hw.idx = 0;
1105 }
1106 }
1107
1108
1109
1110
1111 cpuhw->n_limited = n_lim = 0;
1112 for (i = 0; i < cpuhw->n_events; ++i) {
1113 event = cpuhw->event[i];
1114 if (event->hw.idx)
1115 continue;
1116 idx = hwc_index[i] + 1;
1117 if (is_limited_pmc(idx)) {
1118 cpuhw->limited_counter[n_lim] = event;
1119 cpuhw->limited_hwidx[n_lim] = idx;
1120 ++n_lim;
1121 continue;
1122 }
1123
1124 if (ebb)
1125 val = local64_read(&event->hw.prev_count);
1126 else {
1127 val = 0;
1128 if (event->hw.sample_period) {
1129 left = local64_read(&event->hw.period_left);
1130 if (left < 0x80000000L)
1131 val = 0x80000000L - left;
1132 }
1133 local64_set(&event->hw.prev_count, val);
1134 }
1135
1136 event->hw.idx = idx;
1137 if (event->hw.state & PERF_HES_STOPPED)
1138 val = 0;
1139 write_pmc(idx, val);
1140
1141 perf_event_update_userpage(event);
1142 }
1143 cpuhw->n_limited = n_lim;
1144 cpuhw->mmcr[0] |= MMCR0_PMXE | MMCR0_FCECE;
1145
1146 out_enable:
1147 mmcr0 = ebb_switch_in(ebb, cpuhw->mmcr[0]);
1148
1149 mb();
1150 write_mmcr0(cpuhw, mmcr0);
1151
1152
1153
1154
1155 if (cpuhw->mmcr[2] & MMCRA_SAMPLE_ENABLE) {
1156 mb();
1157 mtspr(SPRN_MMCRA, cpuhw->mmcr[2]);
1158 }
1159
1160 out:
1161 if (cpuhw->bhrb_users)
1162 ppmu->config_bhrb(cpuhw->bhrb_filter);
1163
1164 local_irq_restore(flags);
1165}
1166
1167static int collect_events(struct perf_event *group, int max_count,
1168 struct perf_event *ctrs[], u64 *events,
1169 unsigned int *flags)
1170{
1171 int n = 0;
1172 struct perf_event *event;
1173
1174 if (!is_software_event(group)) {
1175 if (n >= max_count)
1176 return -1;
1177 ctrs[n] = group;
1178 flags[n] = group->hw.event_base;
1179 events[n++] = group->hw.config;
1180 }
1181 list_for_each_entry(event, &group->sibling_list, group_entry) {
1182 if (!is_software_event(event) &&
1183 event->state != PERF_EVENT_STATE_OFF) {
1184 if (n >= max_count)
1185 return -1;
1186 ctrs[n] = event;
1187 flags[n] = event->hw.event_base;
1188 events[n++] = event->hw.config;
1189 }
1190 }
1191 return n;
1192}
1193
1194
1195
1196
1197
1198
1199
1200static int power_pmu_add(struct perf_event *event, int ef_flags)
1201{
1202 struct cpu_hw_events *cpuhw;
1203 unsigned long flags;
1204 int n0;
1205 int ret = -EAGAIN;
1206
1207 local_irq_save(flags);
1208 perf_pmu_disable(event->pmu);
1209
1210
1211
1212
1213
1214 cpuhw = &__get_cpu_var(cpu_hw_events);
1215 n0 = cpuhw->n_events;
1216 if (n0 >= ppmu->n_counter)
1217 goto out;
1218 cpuhw->event[n0] = event;
1219 cpuhw->events[n0] = event->hw.config;
1220 cpuhw->flags[n0] = event->hw.event_base;
1221
1222
1223
1224
1225
1226
1227
1228 if (!(ef_flags & PERF_EF_START))
1229 event->hw.state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
1230 else
1231 event->hw.state = 0;
1232
1233
1234
1235
1236
1237
1238 if (cpuhw->group_flag & PERF_EVENT_TXN)
1239 goto nocheck;
1240
1241 if (check_excludes(cpuhw->event, cpuhw->flags, n0, 1))
1242 goto out;
1243 if (power_check_constraints(cpuhw, cpuhw->events, cpuhw->flags, n0 + 1))
1244 goto out;
1245 event->hw.config = cpuhw->events[n0];
1246
1247nocheck:
1248 ebb_event_add(event);
1249
1250 ++cpuhw->n_events;
1251 ++cpuhw->n_added;
1252
1253 ret = 0;
1254 out:
1255 if (has_branch_stack(event)) {
1256 power_pmu_bhrb_enable(event);
1257 cpuhw->bhrb_filter = ppmu->bhrb_filter_map(
1258 event->attr.branch_sample_type);
1259 }
1260
1261 perf_pmu_enable(event->pmu);
1262 local_irq_restore(flags);
1263 return ret;
1264}
1265
1266
1267
1268
1269static void power_pmu_del(struct perf_event *event, int ef_flags)
1270{
1271 struct cpu_hw_events *cpuhw;
1272 long i;
1273 unsigned long flags;
1274
1275 local_irq_save(flags);
1276 perf_pmu_disable(event->pmu);
1277
1278 power_pmu_read(event);
1279
1280 cpuhw = &__get_cpu_var(cpu_hw_events);
1281 for (i = 0; i < cpuhw->n_events; ++i) {
1282 if (event == cpuhw->event[i]) {
1283 while (++i < cpuhw->n_events) {
1284 cpuhw->event[i-1] = cpuhw->event[i];
1285 cpuhw->events[i-1] = cpuhw->events[i];
1286 cpuhw->flags[i-1] = cpuhw->flags[i];
1287 }
1288 --cpuhw->n_events;
1289 ppmu->disable_pmc(event->hw.idx - 1, cpuhw->mmcr);
1290 if (event->hw.idx) {
1291 write_pmc(event->hw.idx, 0);
1292 event->hw.idx = 0;
1293 }
1294 perf_event_update_userpage(event);
1295 break;
1296 }
1297 }
1298 for (i = 0; i < cpuhw->n_limited; ++i)
1299 if (event == cpuhw->limited_counter[i])
1300 break;
1301 if (i < cpuhw->n_limited) {
1302 while (++i < cpuhw->n_limited) {
1303 cpuhw->limited_counter[i-1] = cpuhw->limited_counter[i];
1304 cpuhw->limited_hwidx[i-1] = cpuhw->limited_hwidx[i];
1305 }
1306 --cpuhw->n_limited;
1307 }
1308 if (cpuhw->n_events == 0) {
1309
1310 cpuhw->mmcr[0] &= ~(MMCR0_PMXE | MMCR0_FCECE);
1311 }
1312
1313 if (has_branch_stack(event))
1314 power_pmu_bhrb_disable(event);
1315
1316 perf_pmu_enable(event->pmu);
1317 local_irq_restore(flags);
1318}
1319
1320
1321
1322
1323
1324
1325static void power_pmu_start(struct perf_event *event, int ef_flags)
1326{
1327 unsigned long flags;
1328 s64 left;
1329 unsigned long val;
1330
1331 if (!event->hw.idx || !event->hw.sample_period)
1332 return;
1333
1334 if (!(event->hw.state & PERF_HES_STOPPED))
1335 return;
1336
1337 if (ef_flags & PERF_EF_RELOAD)
1338 WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE));
1339
1340 local_irq_save(flags);
1341 perf_pmu_disable(event->pmu);
1342
1343 event->hw.state = 0;
1344 left = local64_read(&event->hw.period_left);
1345
1346 val = 0;
1347 if (left < 0x80000000L)
1348 val = 0x80000000L - left;
1349
1350 write_pmc(event->hw.idx, val);
1351
1352 perf_event_update_userpage(event);
1353 perf_pmu_enable(event->pmu);
1354 local_irq_restore(flags);
1355}
1356
1357static void power_pmu_stop(struct perf_event *event, int ef_flags)
1358{
1359 unsigned long flags;
1360
1361 if (!event->hw.idx || !event->hw.sample_period)
1362 return;
1363
1364 if (event->hw.state & PERF_HES_STOPPED)
1365 return;
1366
1367 local_irq_save(flags);
1368 perf_pmu_disable(event->pmu);
1369
1370 power_pmu_read(event);
1371 event->hw.state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
1372 write_pmc(event->hw.idx, 0);
1373
1374 perf_event_update_userpage(event);
1375 perf_pmu_enable(event->pmu);
1376 local_irq_restore(flags);
1377}
1378
1379
1380
1381
1382
1383
1384void power_pmu_start_txn(struct pmu *pmu)
1385{
1386 struct cpu_hw_events *cpuhw = &__get_cpu_var(cpu_hw_events);
1387
1388 perf_pmu_disable(pmu);
1389 cpuhw->group_flag |= PERF_EVENT_TXN;
1390 cpuhw->n_txn_start = cpuhw->n_events;
1391}
1392
1393
1394
1395
1396
1397
1398void power_pmu_cancel_txn(struct pmu *pmu)
1399{
1400 struct cpu_hw_events *cpuhw = &__get_cpu_var(cpu_hw_events);
1401
1402 cpuhw->group_flag &= ~PERF_EVENT_TXN;
1403 perf_pmu_enable(pmu);
1404}
1405
1406
1407
1408
1409
1410
1411int power_pmu_commit_txn(struct pmu *pmu)
1412{
1413 struct cpu_hw_events *cpuhw;
1414 long i, n;
1415
1416 if (!ppmu)
1417 return -EAGAIN;
1418 cpuhw = &__get_cpu_var(cpu_hw_events);
1419 n = cpuhw->n_events;
1420 if (check_excludes(cpuhw->event, cpuhw->flags, 0, n))
1421 return -EAGAIN;
1422 i = power_check_constraints(cpuhw, cpuhw->events, cpuhw->flags, n);
1423 if (i < 0)
1424 return -EAGAIN;
1425
1426 for (i = cpuhw->n_txn_start; i < n; ++i)
1427 cpuhw->event[i]->hw.config = cpuhw->events[i];
1428
1429 cpuhw->group_flag &= ~PERF_EVENT_TXN;
1430 perf_pmu_enable(pmu);
1431 return 0;
1432}
1433
1434
1435
1436
1437
1438
1439
1440
1441static int can_go_on_limited_pmc(struct perf_event *event, u64 ev,
1442 unsigned int flags)
1443{
1444 int n;
1445 u64 alt[MAX_EVENT_ALTERNATIVES];
1446
1447 if (event->attr.exclude_user
1448 || event->attr.exclude_kernel
1449 || event->attr.exclude_hv
1450 || event->attr.sample_period)
1451 return 0;
1452
1453 if (ppmu->limited_pmc_event(ev))
1454 return 1;
1455
1456
1457
1458
1459
1460 if (!ppmu->get_alternatives)
1461 return 0;
1462
1463 flags |= PPMU_LIMITED_PMC_OK | PPMU_LIMITED_PMC_REQD;
1464 n = ppmu->get_alternatives(ev, flags, alt);
1465
1466 return n > 0;
1467}
1468
1469
1470
1471
1472
1473
1474static u64 normal_pmc_alternative(u64 ev, unsigned long flags)
1475{
1476 u64 alt[MAX_EVENT_ALTERNATIVES];
1477 int n;
1478
1479 flags &= ~(PPMU_LIMITED_PMC_OK | PPMU_LIMITED_PMC_REQD);
1480 n = ppmu->get_alternatives(ev, flags, alt);
1481 if (!n)
1482 return 0;
1483 return alt[0];
1484}
1485
1486
1487static atomic_t num_events;
1488
1489static DEFINE_MUTEX(pmc_reserve_mutex);
1490
1491
1492
1493
1494static void hw_perf_event_destroy(struct perf_event *event)
1495{
1496 if (!atomic_add_unless(&num_events, -1, 1)) {
1497 mutex_lock(&pmc_reserve_mutex);
1498 if (atomic_dec_return(&num_events) == 0)
1499 release_pmc_hardware();
1500 mutex_unlock(&pmc_reserve_mutex);
1501 }
1502}
1503
1504
1505
1506
1507static int hw_perf_cache_event(u64 config, u64 *eventp)
1508{
1509 unsigned long type, op, result;
1510 int ev;
1511
1512 if (!ppmu->cache_events)
1513 return -EINVAL;
1514
1515
1516 type = config & 0xff;
1517 op = (config >> 8) & 0xff;
1518 result = (config >> 16) & 0xff;
1519
1520 if (type >= PERF_COUNT_HW_CACHE_MAX ||
1521 op >= PERF_COUNT_HW_CACHE_OP_MAX ||
1522 result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
1523 return -EINVAL;
1524
1525 ev = (*ppmu->cache_events)[type][op][result];
1526 if (ev == 0)
1527 return -EOPNOTSUPP;
1528 if (ev == -1)
1529 return -EINVAL;
1530 *eventp = ev;
1531 return 0;
1532}
1533
1534static int power_pmu_event_init(struct perf_event *event)
1535{
1536 u64 ev;
1537 unsigned long flags;
1538 struct perf_event *ctrs[MAX_HWEVENTS];
1539 u64 events[MAX_HWEVENTS];
1540 unsigned int cflags[MAX_HWEVENTS];
1541 int n;
1542 int err;
1543 struct cpu_hw_events *cpuhw;
1544
1545 if (!ppmu)
1546 return -ENOENT;
1547
1548 if (has_branch_stack(event)) {
1549
1550 if (!(ppmu->flags & PPMU_BHRB))
1551 return -EOPNOTSUPP;
1552 }
1553
1554 switch (event->attr.type) {
1555 case PERF_TYPE_HARDWARE:
1556 ev = event->attr.config;
1557 if (ev >= ppmu->n_generic || ppmu->generic_events[ev] == 0)
1558 return -EOPNOTSUPP;
1559 ev = ppmu->generic_events[ev];
1560 break;
1561 case PERF_TYPE_HW_CACHE:
1562 err = hw_perf_cache_event(event->attr.config, &ev);
1563 if (err)
1564 return err;
1565 break;
1566 case PERF_TYPE_RAW:
1567 ev = event->attr.config;
1568 break;
1569 default:
1570 return -ENOENT;
1571 }
1572
1573 event->hw.config_base = ev;
1574 event->hw.idx = 0;
1575
1576
1577
1578
1579
1580
1581 if (!firmware_has_feature(FW_FEATURE_LPAR))
1582 event->attr.exclude_hv = 0;
1583
1584
1585
1586
1587
1588
1589
1590 flags = 0;
1591 if (event->attach_state & PERF_ATTACH_TASK)
1592 flags |= PPMU_ONLY_COUNT_RUN;
1593
1594
1595
1596
1597
1598 if (ppmu->flags & PPMU_LIMITED_PMC5_6) {
1599 if (can_go_on_limited_pmc(event, ev, flags)) {
1600 flags |= PPMU_LIMITED_PMC_OK;
1601 } else if (ppmu->limited_pmc_event(ev)) {
1602
1603
1604
1605
1606
1607 ev = normal_pmc_alternative(ev, flags);
1608 if (!ev)
1609 return -EINVAL;
1610 }
1611 }
1612
1613
1614 err = ebb_event_check(event);
1615 if (err)
1616 return err;
1617
1618
1619
1620
1621
1622
1623 n = 0;
1624 if (event->group_leader != event) {
1625 n = collect_events(event->group_leader, ppmu->n_counter - 1,
1626 ctrs, events, cflags);
1627 if (n < 0)
1628 return -EINVAL;
1629 }
1630 events[n] = ev;
1631 ctrs[n] = event;
1632 cflags[n] = flags;
1633 if (check_excludes(ctrs, cflags, n, 1))
1634 return -EINVAL;
1635
1636 cpuhw = &get_cpu_var(cpu_hw_events);
1637 err = power_check_constraints(cpuhw, events, cflags, n + 1);
1638
1639 if (has_branch_stack(event)) {
1640 cpuhw->bhrb_filter = ppmu->bhrb_filter_map(
1641 event->attr.branch_sample_type);
1642
1643 if(cpuhw->bhrb_filter == -1)
1644 return -EOPNOTSUPP;
1645 }
1646
1647 put_cpu_var(cpu_hw_events);
1648 if (err)
1649 return -EINVAL;
1650
1651 event->hw.config = events[n];
1652 event->hw.event_base = cflags[n];
1653 event->hw.last_period = event->hw.sample_period;
1654 local64_set(&event->hw.period_left, event->hw.last_period);
1655
1656
1657
1658
1659
1660 if (is_ebb_event(event))
1661 local64_set(&event->hw.prev_count, 0);
1662
1663
1664
1665
1666
1667
1668
1669 err = 0;
1670 if (!atomic_inc_not_zero(&num_events)) {
1671 mutex_lock(&pmc_reserve_mutex);
1672 if (atomic_read(&num_events) == 0 &&
1673 reserve_pmc_hardware(perf_event_interrupt))
1674 err = -EBUSY;
1675 else
1676 atomic_inc(&num_events);
1677 mutex_unlock(&pmc_reserve_mutex);
1678 }
1679 event->destroy = hw_perf_event_destroy;
1680
1681 return err;
1682}
1683
1684static int power_pmu_event_idx(struct perf_event *event)
1685{
1686 return event->hw.idx;
1687}
1688
1689ssize_t power_events_sysfs_show(struct device *dev,
1690 struct device_attribute *attr, char *page)
1691{
1692 struct perf_pmu_events_attr *pmu_attr;
1693
1694 pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr);
1695
1696 return sprintf(page, "event=0x%02llx\n", pmu_attr->id);
1697}
1698
1699struct pmu power_pmu = {
1700 .pmu_enable = power_pmu_enable,
1701 .pmu_disable = power_pmu_disable,
1702 .event_init = power_pmu_event_init,
1703 .add = power_pmu_add,
1704 .del = power_pmu_del,
1705 .start = power_pmu_start,
1706 .stop = power_pmu_stop,
1707 .read = power_pmu_read,
1708 .start_txn = power_pmu_start_txn,
1709 .cancel_txn = power_pmu_cancel_txn,
1710 .commit_txn = power_pmu_commit_txn,
1711 .event_idx = power_pmu_event_idx,
1712 .flush_branch_stack = power_pmu_flush_branch_stack,
1713};
1714
1715
1716
1717
1718
1719
1720static void record_and_restart(struct perf_event *event, unsigned long val,
1721 struct pt_regs *regs)
1722{
1723 u64 period = event->hw.sample_period;
1724 s64 prev, delta, left;
1725 int record = 0;
1726
1727 if (event->hw.state & PERF_HES_STOPPED) {
1728 write_pmc(event->hw.idx, 0);
1729 return;
1730 }
1731
1732
1733 prev = local64_read(&event->hw.prev_count);
1734 delta = check_and_compute_delta(prev, val);
1735 local64_add(delta, &event->count);
1736
1737
1738
1739
1740
1741 val = 0;
1742 left = local64_read(&event->hw.period_left) - delta;
1743 if (delta == 0)
1744 left++;
1745 if (period) {
1746 if (left <= 0) {
1747 left += period;
1748 if (left <= 0)
1749 left = period;
1750 record = siar_valid(regs);
1751 event->hw.last_period = event->hw.sample_period;
1752 }
1753 if (left < 0x80000000LL)
1754 val = 0x80000000LL - left;
1755 }
1756
1757 write_pmc(event->hw.idx, val);
1758 local64_set(&event->hw.prev_count, val);
1759 local64_set(&event->hw.period_left, left);
1760 perf_event_update_userpage(event);
1761
1762
1763
1764
1765 if (record) {
1766 struct perf_sample_data data;
1767
1768 perf_sample_data_init(&data, ~0ULL, event->hw.last_period);
1769
1770 if (event->attr.sample_type & PERF_SAMPLE_ADDR)
1771 perf_get_data_addr(regs, &data.addr);
1772
1773 if (event->attr.sample_type & PERF_SAMPLE_BRANCH_STACK) {
1774 struct cpu_hw_events *cpuhw;
1775 cpuhw = &__get_cpu_var(cpu_hw_events);
1776 power_pmu_bhrb_read(cpuhw);
1777 data.br_stack = &cpuhw->bhrb_stack;
1778 }
1779
1780 if (perf_event_overflow(event, &data, regs))
1781 power_pmu_stop(event, 0);
1782 }
1783}
1784
1785
1786
1787
1788
1789unsigned long perf_misc_flags(struct pt_regs *regs)
1790{
1791 u32 flags = perf_get_misc_flags(regs);
1792
1793 if (flags)
1794 return flags;
1795 return user_mode(regs) ? PERF_RECORD_MISC_USER :
1796 PERF_RECORD_MISC_KERNEL;
1797}
1798
1799
1800
1801
1802
1803unsigned long perf_instruction_pointer(struct pt_regs *regs)
1804{
1805 bool use_siar = regs_use_siar(regs);
1806
1807 if (use_siar && siar_valid(regs))
1808 return mfspr(SPRN_SIAR) + perf_ip_adjust(regs);
1809 else if (use_siar)
1810 return 0;
1811 else
1812 return regs->nip;
1813}
1814
1815static bool pmc_overflow_power7(unsigned long val)
1816{
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828 if ((0x80000000 - val) <= 256)
1829 return true;
1830
1831 return false;
1832}
1833
1834static bool pmc_overflow(unsigned long val)
1835{
1836 if ((int)val < 0)
1837 return true;
1838
1839 return false;
1840}
1841
1842
1843
1844
1845static void perf_event_interrupt(struct pt_regs *regs)
1846{
1847 int i, j;
1848 struct cpu_hw_events *cpuhw = &__get_cpu_var(cpu_hw_events);
1849 struct perf_event *event;
1850 unsigned long val[8];
1851 int found, active;
1852 int nmi;
1853
1854 if (cpuhw->n_limited)
1855 freeze_limited_counters(cpuhw, mfspr(SPRN_PMC5),
1856 mfspr(SPRN_PMC6));
1857
1858 perf_read_regs(regs);
1859
1860 nmi = perf_intr_is_nmi(regs);
1861 if (nmi)
1862 nmi_enter();
1863 else
1864 irq_enter();
1865
1866
1867 for (i = 0; i < ppmu->n_counter; ++i)
1868 val[i] = read_pmc(i + 1);
1869
1870
1871 found = 0;
1872 for (i = 0; i < ppmu->n_counter; ++i) {
1873 if (!pmc_overflow(val[i]))
1874 continue;
1875 if (is_limited_pmc(i + 1))
1876 continue;
1877
1878
1879
1880
1881
1882 found = 1;
1883 active = 0;
1884 for (j = 0; j < cpuhw->n_events; ++j) {
1885 event = cpuhw->event[j];
1886 if (event->hw.idx == (i + 1)) {
1887 active = 1;
1888 record_and_restart(event, val[i], regs);
1889 break;
1890 }
1891 }
1892 if (!active)
1893
1894 write_pmc(i + 1, 0);
1895 }
1896 if (!found && pvr_version_is(PVR_POWER7)) {
1897
1898 for (i = 0; i < cpuhw->n_events; ++i) {
1899 event = cpuhw->event[i];
1900 if (!event->hw.idx || is_limited_pmc(event->hw.idx))
1901 continue;
1902 if (pmc_overflow_power7(val[event->hw.idx - 1])) {
1903
1904 found = 1;
1905 record_and_restart(event,
1906 val[event->hw.idx - 1],
1907 regs);
1908 }
1909 }
1910 }
1911 if (!found && !nmi && printk_ratelimit())
1912 printk(KERN_WARNING "Can't find PMC that caused IRQ\n");
1913
1914
1915
1916
1917
1918
1919
1920
1921 write_mmcr0(cpuhw, cpuhw->mmcr[0]);
1922
1923 if (nmi)
1924 nmi_exit();
1925 else
1926 irq_exit();
1927}
1928
1929static void power_pmu_setup(int cpu)
1930{
1931 struct cpu_hw_events *cpuhw = &per_cpu(cpu_hw_events, cpu);
1932
1933 if (!ppmu)
1934 return;
1935 memset(cpuhw, 0, sizeof(*cpuhw));
1936 cpuhw->mmcr[0] = MMCR0_FC;
1937}
1938
1939static int
1940power_pmu_notifier(struct notifier_block *self, unsigned long action, void *hcpu)
1941{
1942 unsigned int cpu = (long)hcpu;
1943
1944 switch (action & ~CPU_TASKS_FROZEN) {
1945 case CPU_UP_PREPARE:
1946 power_pmu_setup(cpu);
1947 break;
1948
1949 default:
1950 break;
1951 }
1952
1953 return NOTIFY_OK;
1954}
1955
1956int register_power_pmu(struct power_pmu *pmu)
1957{
1958 if (ppmu)
1959 return -EBUSY;
1960
1961 ppmu = pmu;
1962 pr_info("%s performance monitor hardware support registered\n",
1963 pmu->name);
1964
1965 power_pmu.attr_groups = ppmu->attr_groups;
1966
1967#ifdef MSR_HV
1968
1969
1970
1971 if (mfmsr() & MSR_HV)
1972 freeze_events_kernel = MMCR0_FCHV;
1973#endif
1974
1975 perf_pmu_register(&power_pmu, "cpu", PERF_TYPE_RAW);
1976 perf_cpu_notifier(power_pmu_notifier);
1977
1978 return 0;
1979}
1980