1
2
3
4
5
6
7
8
9
10
11#include <linux/kernel.h>
12#include <linux/sched.h>
13#include <linux/perf_event.h>
14#include <linux/percpu.h>
15#include <linux/hardirq.h>
16#include <linux/uaccess.h>
17#include <asm/reg.h>
18#include <asm/pmc.h>
19#include <asm/machdep.h>
20#include <asm/firmware.h>
21#include <asm/ptrace.h>
22#include <asm/code-patching.h>
23
24#define BHRB_MAX_ENTRIES 32
25#define BHRB_TARGET 0x0000000000000002
26#define BHRB_PREDICTION 0x0000000000000001
27#define BHRB_EA 0xFFFFFFFFFFFFFFFCUL
28
29struct cpu_hw_events {
30 int n_events;
31 int n_percpu;
32 int disabled;
33 int n_added;
34 int n_limited;
35 u8 pmcs_enabled;
36 struct perf_event *event[MAX_HWEVENTS];
37 u64 events[MAX_HWEVENTS];
38 unsigned int flags[MAX_HWEVENTS];
39
40
41
42
43
44 unsigned long mmcr[4];
45 struct perf_event *limited_counter[MAX_LIMITED_HWCOUNTERS];
46 u8 limited_hwidx[MAX_LIMITED_HWCOUNTERS];
47 u64 alternatives[MAX_HWEVENTS][MAX_EVENT_ALTERNATIVES];
48 unsigned long amasks[MAX_HWEVENTS][MAX_EVENT_ALTERNATIVES];
49 unsigned long avalues[MAX_HWEVENTS][MAX_EVENT_ALTERNATIVES];
50
51 unsigned int group_flag;
52 int n_txn_start;
53
54
55 u64 bhrb_filter;
56 int bhrb_users;
57 void *bhrb_context;
58 struct perf_branch_stack bhrb_stack;
59 struct perf_branch_entry bhrb_entries[BHRB_MAX_ENTRIES];
60};
61
62static DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events);
63
64static struct power_pmu *ppmu;
65
66
67
68
69
70
71
72
73static unsigned int freeze_events_kernel = MMCR0_FCS;
74
75
76
77
78
79#ifdef CONFIG_PPC32
80
81#define MMCR0_FCHV 0
82#define MMCR0_PMCjCE MMCR0_PMCnCE
83#define MMCR0_FC56 0
84#define MMCR0_PMAO 0
85#define MMCR0_EBE 0
86#define MMCR0_BHRBA 0
87#define MMCR0_PMCC 0
88#define MMCR0_PMCC_U6 0
89
90#define SPRN_MMCRA SPRN_MMCR2
91#define MMCRA_SAMPLE_ENABLE 0
92
93static inline unsigned long perf_ip_adjust(struct pt_regs *regs)
94{
95 return 0;
96}
97static inline void perf_get_data_addr(struct pt_regs *regs, u64 *addrp) { }
98static inline u32 perf_get_misc_flags(struct pt_regs *regs)
99{
100 return 0;
101}
102static inline void perf_read_regs(struct pt_regs *regs)
103{
104 regs->result = 0;
105}
106static inline int perf_intr_is_nmi(struct pt_regs *regs)
107{
108 return 0;
109}
110
111static inline int siar_valid(struct pt_regs *regs)
112{
113 return 1;
114}
115
116static bool is_ebb_event(struct perf_event *event) { return false; }
117static int ebb_event_check(struct perf_event *event) { return 0; }
118static void ebb_event_add(struct perf_event *event) { }
119static void ebb_switch_out(unsigned long mmcr0) { }
120static unsigned long ebb_switch_in(bool ebb, struct cpu_hw_events *cpuhw)
121{
122 return cpuhw->mmcr[0];
123}
124
125static inline void power_pmu_bhrb_enable(struct perf_event *event) {}
126static inline void power_pmu_bhrb_disable(struct perf_event *event) {}
127static void power_pmu_flush_branch_stack(void) {}
128static inline void power_pmu_bhrb_read(struct cpu_hw_events *cpuhw) {}
129static void pmao_restore_workaround(bool ebb) { }
130#endif
131
132static bool regs_use_siar(struct pt_regs *regs)
133{
134 return !!regs->result;
135}
136
137
138
139
140#ifdef CONFIG_PPC64
141
142static inline unsigned long perf_ip_adjust(struct pt_regs *regs)
143{
144 unsigned long mmcra = regs->dsisr;
145
146 if ((ppmu->flags & PPMU_HAS_SSLOT) && (mmcra & MMCRA_SAMPLE_ENABLE)) {
147 unsigned long slot = (mmcra & MMCRA_SLOT) >> MMCRA_SLOT_SHIFT;
148 if (slot > 1)
149 return 4 * (slot - 1);
150 }
151
152 return 0;
153}
154
155
156
157
158
159
160
161
162
163static inline void perf_get_data_addr(struct pt_regs *regs, u64 *addrp)
164{
165 unsigned long mmcra = regs->dsisr;
166 bool sdar_valid;
167
168 if (ppmu->flags & PPMU_HAS_SIER)
169 sdar_valid = regs->dar & SIER_SDAR_VALID;
170 else {
171 unsigned long sdsync;
172
173 if (ppmu->flags & PPMU_SIAR_VALID)
174 sdsync = POWER7P_MMCRA_SDAR_VALID;
175 else if (ppmu->flags & PPMU_ALT_SIPR)
176 sdsync = POWER6_MMCRA_SDSYNC;
177 else
178 sdsync = MMCRA_SDSYNC;
179
180 sdar_valid = mmcra & sdsync;
181 }
182
183 if (!(mmcra & MMCRA_SAMPLE_ENABLE) || sdar_valid)
184 *addrp = mfspr(SPRN_SDAR);
185}
186
187static bool regs_sihv(struct pt_regs *regs)
188{
189 unsigned long sihv = MMCRA_SIHV;
190
191 if (ppmu->flags & PPMU_HAS_SIER)
192 return !!(regs->dar & SIER_SIHV);
193
194 if (ppmu->flags & PPMU_ALT_SIPR)
195 sihv = POWER6_MMCRA_SIHV;
196
197 return !!(regs->dsisr & sihv);
198}
199
200static bool regs_sipr(struct pt_regs *regs)
201{
202 unsigned long sipr = MMCRA_SIPR;
203
204 if (ppmu->flags & PPMU_HAS_SIER)
205 return !!(regs->dar & SIER_SIPR);
206
207 if (ppmu->flags & PPMU_ALT_SIPR)
208 sipr = POWER6_MMCRA_SIPR;
209
210 return !!(regs->dsisr & sipr);
211}
212
213static inline u32 perf_flags_from_msr(struct pt_regs *regs)
214{
215 if (regs->msr & MSR_PR)
216 return PERF_RECORD_MISC_USER;
217 if ((regs->msr & MSR_HV) && freeze_events_kernel != MMCR0_FCHV)
218 return PERF_RECORD_MISC_HYPERVISOR;
219 return PERF_RECORD_MISC_KERNEL;
220}
221
222static inline u32 perf_get_misc_flags(struct pt_regs *regs)
223{
224 bool use_siar = regs_use_siar(regs);
225
226 if (!use_siar)
227 return perf_flags_from_msr(regs);
228
229
230
231
232
233
234
235 if (ppmu->flags & PPMU_NO_SIPR) {
236 unsigned long siar = mfspr(SPRN_SIAR);
237 if (siar >= PAGE_OFFSET)
238 return PERF_RECORD_MISC_KERNEL;
239 return PERF_RECORD_MISC_USER;
240 }
241
242
243 if (regs_sipr(regs))
244 return PERF_RECORD_MISC_USER;
245
246 if (regs_sihv(regs) && (freeze_events_kernel != MMCR0_FCHV))
247 return PERF_RECORD_MISC_HYPERVISOR;
248
249 return PERF_RECORD_MISC_KERNEL;
250}
251
252
253
254
255
256
257
258
259static inline void perf_read_regs(struct pt_regs *regs)
260{
261 unsigned long mmcra = mfspr(SPRN_MMCRA);
262 int marked = mmcra & MMCRA_SAMPLE_ENABLE;
263 int use_siar;
264
265 regs->dsisr = mmcra;
266
267 if (ppmu->flags & PPMU_HAS_SIER)
268 regs->dar = mfspr(SPRN_SIER);
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287 if (TRAP(regs) != 0xf00)
288 use_siar = 0;
289 else if (marked)
290 use_siar = 1;
291 else if ((ppmu->flags & PPMU_NO_CONT_SAMPLING))
292 use_siar = 0;
293 else if (!(ppmu->flags & PPMU_NO_SIPR) && regs_sipr(regs))
294 use_siar = 0;
295 else
296 use_siar = 1;
297
298 regs->result = use_siar;
299}
300
301
302
303
304
305static inline int perf_intr_is_nmi(struct pt_regs *regs)
306{
307 return !regs->softe;
308}
309
310
311
312
313
314
315
316
317static inline int siar_valid(struct pt_regs *regs)
318{
319 unsigned long mmcra = regs->dsisr;
320 int marked = mmcra & MMCRA_SAMPLE_ENABLE;
321
322 if (marked) {
323 if (ppmu->flags & PPMU_HAS_SIER)
324 return regs->dar & SIER_SIAR_VALID;
325
326 if (ppmu->flags & PPMU_SIAR_VALID)
327 return mmcra & POWER7P_MMCRA_SIAR_VALID;
328 }
329
330 return 1;
331}
332
333
334
335static void power_pmu_bhrb_reset(void)
336{
337 asm volatile(PPC_CLRBHRB);
338}
339
340static void power_pmu_bhrb_enable(struct perf_event *event)
341{
342 struct cpu_hw_events *cpuhw = &__get_cpu_var(cpu_hw_events);
343
344 if (!ppmu->bhrb_nr)
345 return;
346
347
348 if (event->ctx->task && cpuhw->bhrb_context != event->ctx) {
349 power_pmu_bhrb_reset();
350 cpuhw->bhrb_context = event->ctx;
351 }
352 cpuhw->bhrb_users++;
353}
354
355static void power_pmu_bhrb_disable(struct perf_event *event)
356{
357 struct cpu_hw_events *cpuhw = &__get_cpu_var(cpu_hw_events);
358
359 if (!ppmu->bhrb_nr)
360 return;
361
362 cpuhw->bhrb_users--;
363 WARN_ON_ONCE(cpuhw->bhrb_users < 0);
364
365 if (!cpuhw->disabled && !cpuhw->bhrb_users) {
366
367
368
369
370
371 cpuhw->bhrb_context = NULL;
372 }
373}
374
375
376
377
378static void power_pmu_flush_branch_stack(void)
379{
380 if (ppmu->bhrb_nr)
381 power_pmu_bhrb_reset();
382}
383
384static __u64 power_pmu_bhrb_to(u64 addr)
385{
386 unsigned int instr;
387 int ret;
388 __u64 target;
389
390 if (is_kernel_addr(addr))
391 return branch_target((unsigned int *)addr);
392
393
394 pagefault_disable();
395 ret = __get_user_inatomic(instr, (unsigned int __user *)addr);
396 if (ret) {
397 pagefault_enable();
398 return 0;
399 }
400 pagefault_enable();
401
402 target = branch_target(&instr);
403 if ((!target) || (instr & BRANCH_ABSOLUTE))
404 return target;
405
406
407 return target - (unsigned long)&instr + addr;
408}
409
410
411static void power_pmu_bhrb_read(struct cpu_hw_events *cpuhw)
412{
413 u64 val;
414 u64 addr;
415 int r_index, u_index, pred;
416
417 r_index = 0;
418 u_index = 0;
419 while (r_index < ppmu->bhrb_nr) {
420
421 val = read_bhrb(r_index++);
422 if (!val)
423
424 break;
425 else {
426 addr = val & BHRB_EA;
427 pred = val & BHRB_PREDICTION;
428
429 if (!addr)
430
431 continue;
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451 if (val & BHRB_TARGET) {
452
453
454
455 cpuhw->bhrb_entries[u_index].to = addr;
456 cpuhw->bhrb_entries[u_index].mispred = pred;
457 cpuhw->bhrb_entries[u_index].predicted = ~pred;
458
459
460 val = read_bhrb(r_index++);
461 addr = val & BHRB_EA;
462 if (val & BHRB_TARGET) {
463
464
465 r_index--;
466 addr = 0;
467 }
468 cpuhw->bhrb_entries[u_index].from = addr;
469 } else {
470
471
472 cpuhw->bhrb_entries[u_index].from = addr;
473 cpuhw->bhrb_entries[u_index].to =
474 power_pmu_bhrb_to(addr);
475 cpuhw->bhrb_entries[u_index].mispred = pred;
476 cpuhw->bhrb_entries[u_index].predicted = ~pred;
477 }
478 u_index++;
479
480 }
481 }
482 cpuhw->bhrb_stack.nr = u_index;
483 return;
484}
485
486static bool is_ebb_event(struct perf_event *event)
487{
488
489
490
491
492
493 return (ppmu->flags & PPMU_ARCH_207S) &&
494 ((event->attr.config >> PERF_EVENT_CONFIG_EBB_SHIFT) & 1);
495}
496
497static int ebb_event_check(struct perf_event *event)
498{
499 struct perf_event *leader = event->group_leader;
500
501
502 if (is_ebb_event(leader) != is_ebb_event(event))
503 return -EINVAL;
504
505 if (is_ebb_event(event)) {
506 if (!(event->attach_state & PERF_ATTACH_TASK))
507 return -EINVAL;
508
509 if (!leader->attr.pinned || !leader->attr.exclusive)
510 return -EINVAL;
511
512 if (event->attr.freq ||
513 event->attr.inherit ||
514 event->attr.sample_type ||
515 event->attr.sample_period ||
516 event->attr.enable_on_exec)
517 return -EINVAL;
518 }
519
520 return 0;
521}
522
523static void ebb_event_add(struct perf_event *event)
524{
525 if (!is_ebb_event(event) || current->thread.used_ebb)
526 return;
527
528
529
530
531
532
533
534 current->thread.used_ebb = 1;
535 current->thread.mmcr0 |= MMCR0_PMXE;
536}
537
538static void ebb_switch_out(unsigned long mmcr0)
539{
540 if (!(mmcr0 & MMCR0_EBE))
541 return;
542
543 current->thread.siar = mfspr(SPRN_SIAR);
544 current->thread.sier = mfspr(SPRN_SIER);
545 current->thread.sdar = mfspr(SPRN_SDAR);
546 current->thread.mmcr0 = mmcr0 & MMCR0_USER_MASK;
547 current->thread.mmcr2 = mfspr(SPRN_MMCR2) & MMCR2_USER_MASK;
548}
549
550static unsigned long ebb_switch_in(bool ebb, struct cpu_hw_events *cpuhw)
551{
552 unsigned long mmcr0 = cpuhw->mmcr[0];
553
554 if (!ebb)
555 goto out;
556
557
558 mmcr0 |= MMCR0_EBE | MMCR0_BHRBA | MMCR0_PMCC_U6;
559
560
561
562
563
564
565 mmcr0 |= current->thread.mmcr0;
566
567
568
569
570
571
572 if (!(current->thread.mmcr0 & MMCR0_PMXE))
573 mmcr0 &= ~MMCR0_PMXE;
574
575 mtspr(SPRN_SIAR, current->thread.siar);
576 mtspr(SPRN_SIER, current->thread.sier);
577 mtspr(SPRN_SDAR, current->thread.sdar);
578
579
580
581
582
583
584
585
586 mtspr(SPRN_MMCR2, cpuhw->mmcr[3] | current->thread.mmcr2);
587out:
588 return mmcr0;
589}
590
591static void pmao_restore_workaround(bool ebb)
592{
593 unsigned pmcs[6];
594
595 if (!cpu_has_feature(CPU_FTR_PMAO_BUG))
596 return;
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631 if ((current->thread.mmcr0 & (MMCR0_PMAO | MMCR0_PMAO_SYNC)) != MMCR0_PMAO)
632 return;
633
634
635 if (ebb && !(current->thread.bescr & BESCR_GE))
636 return;
637
638
639
640
641
642 hard_irq_disable();
643
644
645
646
647
648
649 pmcs[0] = mfspr(SPRN_PMC1);
650 pmcs[1] = mfspr(SPRN_PMC2);
651 pmcs[2] = mfspr(SPRN_PMC3);
652 pmcs[3] = mfspr(SPRN_PMC4);
653 pmcs[4] = mfspr(SPRN_PMC5);
654 pmcs[5] = mfspr(SPRN_PMC6);
655
656
657 mtspr(SPRN_MMCR2, 0);
658
659
660 mtspr(SPRN_PMC6, 0x7FFFFFFE);
661
662
663 mtspr(SPRN_MMCR0, MMCR0_PMXE | MMCR0_PMCjCE | MMCR0_PMAO);
664
665
666 mtspr(SPRN_MMCR0, MMCR0_FC | MMCR0_PMAO);
667
668 mtspr(SPRN_PMC1, pmcs[0]);
669 mtspr(SPRN_PMC2, pmcs[1]);
670 mtspr(SPRN_PMC3, pmcs[2]);
671 mtspr(SPRN_PMC4, pmcs[3]);
672 mtspr(SPRN_PMC5, pmcs[4]);
673 mtspr(SPRN_PMC6, pmcs[5]);
674}
675#endif
676
677static void perf_event_interrupt(struct pt_regs *regs);
678
679
680
681
682static unsigned long read_pmc(int idx)
683{
684 unsigned long val;
685
686 switch (idx) {
687 case 1:
688 val = mfspr(SPRN_PMC1);
689 break;
690 case 2:
691 val = mfspr(SPRN_PMC2);
692 break;
693 case 3:
694 val = mfspr(SPRN_PMC3);
695 break;
696 case 4:
697 val = mfspr(SPRN_PMC4);
698 break;
699 case 5:
700 val = mfspr(SPRN_PMC5);
701 break;
702 case 6:
703 val = mfspr(SPRN_PMC6);
704 break;
705#ifdef CONFIG_PPC64
706 case 7:
707 val = mfspr(SPRN_PMC7);
708 break;
709 case 8:
710 val = mfspr(SPRN_PMC8);
711 break;
712#endif
713 default:
714 printk(KERN_ERR "oops trying to read PMC%d\n", idx);
715 val = 0;
716 }
717 return val;
718}
719
720
721
722
723static void write_pmc(int idx, unsigned long val)
724{
725 switch (idx) {
726 case 1:
727 mtspr(SPRN_PMC1, val);
728 break;
729 case 2:
730 mtspr(SPRN_PMC2, val);
731 break;
732 case 3:
733 mtspr(SPRN_PMC3, val);
734 break;
735 case 4:
736 mtspr(SPRN_PMC4, val);
737 break;
738 case 5:
739 mtspr(SPRN_PMC5, val);
740 break;
741 case 6:
742 mtspr(SPRN_PMC6, val);
743 break;
744#ifdef CONFIG_PPC64
745 case 7:
746 mtspr(SPRN_PMC7, val);
747 break;
748 case 8:
749 mtspr(SPRN_PMC8, val);
750 break;
751#endif
752 default:
753 printk(KERN_ERR "oops trying to write PMC%d\n", idx);
754 }
755}
756
757
758void perf_event_print_debug(void)
759{
760 unsigned long sdar, sier, flags;
761 u32 pmcs[MAX_HWEVENTS];
762 int i;
763
764 if (!ppmu->n_counter)
765 return;
766
767 local_irq_save(flags);
768
769 pr_info("CPU: %d PMU registers, ppmu = %s n_counters = %d",
770 smp_processor_id(), ppmu->name, ppmu->n_counter);
771
772 for (i = 0; i < ppmu->n_counter; i++)
773 pmcs[i] = read_pmc(i + 1);
774
775 for (; i < MAX_HWEVENTS; i++)
776 pmcs[i] = 0xdeadbeef;
777
778 pr_info("PMC1: %08x PMC2: %08x PMC3: %08x PMC4: %08x\n",
779 pmcs[0], pmcs[1], pmcs[2], pmcs[3]);
780
781 if (ppmu->n_counter > 4)
782 pr_info("PMC5: %08x PMC6: %08x PMC7: %08x PMC8: %08x\n",
783 pmcs[4], pmcs[5], pmcs[6], pmcs[7]);
784
785 pr_info("MMCR0: %016lx MMCR1: %016lx MMCRA: %016lx\n",
786 mfspr(SPRN_MMCR0), mfspr(SPRN_MMCR1), mfspr(SPRN_MMCRA));
787
788 sdar = sier = 0;
789#ifdef CONFIG_PPC64
790 sdar = mfspr(SPRN_SDAR);
791
792 if (ppmu->flags & PPMU_HAS_SIER)
793 sier = mfspr(SPRN_SIER);
794
795 if (ppmu->flags & PPMU_ARCH_207S) {
796 pr_info("MMCR2: %016lx EBBHR: %016lx\n",
797 mfspr(SPRN_MMCR2), mfspr(SPRN_EBBHR));
798 pr_info("EBBRR: %016lx BESCR: %016lx\n",
799 mfspr(SPRN_EBBRR), mfspr(SPRN_BESCR));
800 }
801#endif
802 pr_info("SIAR: %016lx SDAR: %016lx SIER: %016lx\n",
803 mfspr(SPRN_SIAR), sdar, sier);
804
805 local_irq_restore(flags);
806}
807
808
809
810
811
812
813
814static int power_check_constraints(struct cpu_hw_events *cpuhw,
815 u64 event_id[], unsigned int cflags[],
816 int n_ev)
817{
818 unsigned long mask, value, nv;
819 unsigned long smasks[MAX_HWEVENTS], svalues[MAX_HWEVENTS];
820 int n_alt[MAX_HWEVENTS], choice[MAX_HWEVENTS];
821 int i, j;
822 unsigned long addf = ppmu->add_fields;
823 unsigned long tadd = ppmu->test_adder;
824
825 if (n_ev > ppmu->n_counter)
826 return -1;
827
828
829 for (i = 0; i < n_ev; ++i) {
830 if ((cflags[i] & PPMU_LIMITED_PMC_REQD)
831 && !ppmu->limited_pmc_event(event_id[i])) {
832 ppmu->get_alternatives(event_id[i], cflags[i],
833 cpuhw->alternatives[i]);
834 event_id[i] = cpuhw->alternatives[i][0];
835 }
836 if (ppmu->get_constraint(event_id[i], &cpuhw->amasks[i][0],
837 &cpuhw->avalues[i][0]))
838 return -1;
839 }
840 value = mask = 0;
841 for (i = 0; i < n_ev; ++i) {
842 nv = (value | cpuhw->avalues[i][0]) +
843 (value & cpuhw->avalues[i][0] & addf);
844 if ((((nv + tadd) ^ value) & mask) != 0 ||
845 (((nv + tadd) ^ cpuhw->avalues[i][0]) &
846 cpuhw->amasks[i][0]) != 0)
847 break;
848 value = nv;
849 mask |= cpuhw->amasks[i][0];
850 }
851 if (i == n_ev)
852 return 0;
853
854
855 if (!ppmu->get_alternatives)
856 return -1;
857 for (i = 0; i < n_ev; ++i) {
858 choice[i] = 0;
859 n_alt[i] = ppmu->get_alternatives(event_id[i], cflags[i],
860 cpuhw->alternatives[i]);
861 for (j = 1; j < n_alt[i]; ++j)
862 ppmu->get_constraint(cpuhw->alternatives[i][j],
863 &cpuhw->amasks[i][j],
864 &cpuhw->avalues[i][j]);
865 }
866
867
868 i = 0;
869 j = -1;
870 value = mask = nv = 0;
871 while (i < n_ev) {
872 if (j >= 0) {
873
874 value = svalues[i];
875 mask = smasks[i];
876 j = choice[i];
877 }
878
879
880
881
882 while (++j < n_alt[i]) {
883 nv = (value | cpuhw->avalues[i][j]) +
884 (value & cpuhw->avalues[i][j] & addf);
885 if ((((nv + tadd) ^ value) & mask) == 0 &&
886 (((nv + tadd) ^ cpuhw->avalues[i][j])
887 & cpuhw->amasks[i][j]) == 0)
888 break;
889 }
890 if (j >= n_alt[i]) {
891
892
893
894
895
896 if (--i < 0)
897 return -1;
898 } else {
899
900
901
902
903
904
905 choice[i] = j;
906 svalues[i] = value;
907 smasks[i] = mask;
908 value = nv;
909 mask |= cpuhw->amasks[i][j];
910 ++i;
911 j = -1;
912 }
913 }
914
915
916 for (i = 0; i < n_ev; ++i)
917 event_id[i] = cpuhw->alternatives[i][choice[i]];
918 return 0;
919}
920
921
922
923
924
925
926static int check_excludes(struct perf_event **ctrs, unsigned int cflags[],
927 int n_prev, int n_new)
928{
929 int eu = 0, ek = 0, eh = 0;
930 int i, n, first;
931 struct perf_event *event;
932
933
934
935
936
937
938 if (ppmu->flags & PPMU_ARCH_207S)
939 return 0;
940
941 n = n_prev + n_new;
942 if (n <= 1)
943 return 0;
944
945 first = 1;
946 for (i = 0; i < n; ++i) {
947 if (cflags[i] & PPMU_LIMITED_PMC_OK) {
948 cflags[i] &= ~PPMU_LIMITED_PMC_REQD;
949 continue;
950 }
951 event = ctrs[i];
952 if (first) {
953 eu = event->attr.exclude_user;
954 ek = event->attr.exclude_kernel;
955 eh = event->attr.exclude_hv;
956 first = 0;
957 } else if (event->attr.exclude_user != eu ||
958 event->attr.exclude_kernel != ek ||
959 event->attr.exclude_hv != eh) {
960 return -EAGAIN;
961 }
962 }
963
964 if (eu || ek || eh)
965 for (i = 0; i < n; ++i)
966 if (cflags[i] & PPMU_LIMITED_PMC_OK)
967 cflags[i] |= PPMU_LIMITED_PMC_REQD;
968
969 return 0;
970}
971
972static u64 check_and_compute_delta(u64 prev, u64 val)
973{
974 u64 delta = (val - prev) & 0xfffffffful;
975
976
977
978
979
980
981
982
983
984
985 if (prev > val && (prev - val) < 256)
986 delta = 0;
987
988 return delta;
989}
990
991static void power_pmu_read(struct perf_event *event)
992{
993 s64 val, delta, prev;
994
995 if (event->hw.state & PERF_HES_STOPPED)
996 return;
997
998 if (!event->hw.idx)
999 return;
1000
1001 if (is_ebb_event(event)) {
1002 val = read_pmc(event->hw.idx);
1003 local64_set(&event->hw.prev_count, val);
1004 return;
1005 }
1006
1007
1008
1009
1010
1011
1012 do {
1013 prev = local64_read(&event->hw.prev_count);
1014 barrier();
1015 val = read_pmc(event->hw.idx);
1016 delta = check_and_compute_delta(prev, val);
1017 if (!delta)
1018 return;
1019 } while (local64_cmpxchg(&event->hw.prev_count, prev, val) != prev);
1020
1021 local64_add(delta, &event->count);
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032 do {
1033 prev = local64_read(&event->hw.period_left);
1034 val = prev - delta;
1035 if (val < 1)
1036 val = 1;
1037 } while (local64_cmpxchg(&event->hw.period_left, prev, val) != prev);
1038}
1039
1040
1041
1042
1043
1044
1045static int is_limited_pmc(int pmcnum)
1046{
1047 return (ppmu->flags & PPMU_LIMITED_PMC5_6)
1048 && (pmcnum == 5 || pmcnum == 6);
1049}
1050
1051static void freeze_limited_counters(struct cpu_hw_events *cpuhw,
1052 unsigned long pmc5, unsigned long pmc6)
1053{
1054 struct perf_event *event;
1055 u64 val, prev, delta;
1056 int i;
1057
1058 for (i = 0; i < cpuhw->n_limited; ++i) {
1059 event = cpuhw->limited_counter[i];
1060 if (!event->hw.idx)
1061 continue;
1062 val = (event->hw.idx == 5) ? pmc5 : pmc6;
1063 prev = local64_read(&event->hw.prev_count);
1064 event->hw.idx = 0;
1065 delta = check_and_compute_delta(prev, val);
1066 if (delta)
1067 local64_add(delta, &event->count);
1068 }
1069}
1070
1071static void thaw_limited_counters(struct cpu_hw_events *cpuhw,
1072 unsigned long pmc5, unsigned long pmc6)
1073{
1074 struct perf_event *event;
1075 u64 val, prev;
1076 int i;
1077
1078 for (i = 0; i < cpuhw->n_limited; ++i) {
1079 event = cpuhw->limited_counter[i];
1080 event->hw.idx = cpuhw->limited_hwidx[i];
1081 val = (event->hw.idx == 5) ? pmc5 : pmc6;
1082 prev = local64_read(&event->hw.prev_count);
1083 if (check_and_compute_delta(prev, val))
1084 local64_set(&event->hw.prev_count, val);
1085 perf_event_update_userpage(event);
1086 }
1087}
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100static void write_mmcr0(struct cpu_hw_events *cpuhw, unsigned long mmcr0)
1101{
1102 unsigned long pmc5, pmc6;
1103
1104 if (!cpuhw->n_limited) {
1105 mtspr(SPRN_MMCR0, mmcr0);
1106 return;
1107 }
1108
1109
1110
1111
1112
1113
1114
1115
1116 asm volatile("mtspr %3,%2; mfspr %0,%4; mfspr %1,%5"
1117 : "=&r" (pmc5), "=&r" (pmc6)
1118 : "r" (mmcr0 & ~(MMCR0_PMC1CE | MMCR0_PMCjCE)),
1119 "i" (SPRN_MMCR0),
1120 "i" (SPRN_PMC5), "i" (SPRN_PMC6));
1121
1122 if (mmcr0 & MMCR0_FC)
1123 freeze_limited_counters(cpuhw, pmc5, pmc6);
1124 else
1125 thaw_limited_counters(cpuhw, pmc5, pmc6);
1126
1127
1128
1129
1130
1131 if (mmcr0 & (MMCR0_PMC1CE | MMCR0_PMCjCE))
1132 mtspr(SPRN_MMCR0, mmcr0);
1133}
1134
1135
1136
1137
1138
1139static void power_pmu_disable(struct pmu *pmu)
1140{
1141 struct cpu_hw_events *cpuhw;
1142 unsigned long flags, mmcr0, val;
1143
1144 if (!ppmu)
1145 return;
1146 local_irq_save(flags);
1147 cpuhw = &__get_cpu_var(cpu_hw_events);
1148
1149 if (!cpuhw->disabled) {
1150
1151
1152
1153 if (!cpuhw->pmcs_enabled) {
1154 ppc_enable_pmcs();
1155 cpuhw->pmcs_enabled = 1;
1156 }
1157
1158
1159
1160
1161 val = mmcr0 = mfspr(SPRN_MMCR0);
1162 val |= MMCR0_FC;
1163 val &= ~(MMCR0_EBE | MMCR0_BHRBA | MMCR0_PMCC | MMCR0_PMAO |
1164 MMCR0_FC56);
1165
1166
1167
1168
1169
1170
1171 write_mmcr0(cpuhw, val);
1172 mb();
1173
1174
1175
1176
1177 if (cpuhw->mmcr[2] & MMCRA_SAMPLE_ENABLE) {
1178 mtspr(SPRN_MMCRA,
1179 cpuhw->mmcr[2] & ~MMCRA_SAMPLE_ENABLE);
1180 mb();
1181 }
1182
1183 cpuhw->disabled = 1;
1184 cpuhw->n_added = 0;
1185
1186 ebb_switch_out(mmcr0);
1187 }
1188
1189 local_irq_restore(flags);
1190}
1191
1192
1193
1194
1195
1196
1197static void power_pmu_enable(struct pmu *pmu)
1198{
1199 struct perf_event *event;
1200 struct cpu_hw_events *cpuhw;
1201 unsigned long flags;
1202 long i;
1203 unsigned long val, mmcr0;
1204 s64 left;
1205 unsigned int hwc_index[MAX_HWEVENTS];
1206 int n_lim;
1207 int idx;
1208 bool ebb;
1209
1210 if (!ppmu)
1211 return;
1212 local_irq_save(flags);
1213
1214 cpuhw = &__get_cpu_var(cpu_hw_events);
1215 if (!cpuhw->disabled)
1216 goto out;
1217
1218 if (cpuhw->n_events == 0) {
1219 ppc_set_pmu_inuse(0);
1220 goto out;
1221 }
1222
1223 cpuhw->disabled = 0;
1224
1225
1226
1227
1228
1229
1230 ebb = is_ebb_event(cpuhw->event[0]);
1231
1232
1233
1234
1235
1236
1237
1238 if (!cpuhw->n_added) {
1239 mtspr(SPRN_MMCRA, cpuhw->mmcr[2] & ~MMCRA_SAMPLE_ENABLE);
1240 mtspr(SPRN_MMCR1, cpuhw->mmcr[1]);
1241 goto out_enable;
1242 }
1243
1244
1245
1246
1247 memset(cpuhw->mmcr, 0, sizeof(cpuhw->mmcr));
1248
1249 if (ppmu->compute_mmcr(cpuhw->events, cpuhw->n_events, hwc_index,
1250 cpuhw->mmcr, cpuhw->event)) {
1251
1252 printk(KERN_ERR "oops compute_mmcr failed\n");
1253 goto out;
1254 }
1255
1256 if (!(ppmu->flags & PPMU_ARCH_207S)) {
1257
1258
1259
1260
1261
1262 event = cpuhw->event[0];
1263 if (event->attr.exclude_user)
1264 cpuhw->mmcr[0] |= MMCR0_FCP;
1265 if (event->attr.exclude_kernel)
1266 cpuhw->mmcr[0] |= freeze_events_kernel;
1267 if (event->attr.exclude_hv)
1268 cpuhw->mmcr[0] |= MMCR0_FCHV;
1269 }
1270
1271
1272
1273
1274
1275
1276 ppc_set_pmu_inuse(1);
1277 mtspr(SPRN_MMCRA, cpuhw->mmcr[2] & ~MMCRA_SAMPLE_ENABLE);
1278 mtspr(SPRN_MMCR1, cpuhw->mmcr[1]);
1279 mtspr(SPRN_MMCR0, (cpuhw->mmcr[0] & ~(MMCR0_PMC1CE | MMCR0_PMCjCE))
1280 | MMCR0_FC);
1281 if (ppmu->flags & PPMU_ARCH_207S)
1282 mtspr(SPRN_MMCR2, cpuhw->mmcr[3]);
1283
1284
1285
1286
1287
1288 for (i = 0; i < cpuhw->n_events; ++i) {
1289 event = cpuhw->event[i];
1290 if (event->hw.idx && event->hw.idx != hwc_index[i] + 1) {
1291 power_pmu_read(event);
1292 write_pmc(event->hw.idx, 0);
1293 event->hw.idx = 0;
1294 }
1295 }
1296
1297
1298
1299
1300 cpuhw->n_limited = n_lim = 0;
1301 for (i = 0; i < cpuhw->n_events; ++i) {
1302 event = cpuhw->event[i];
1303 if (event->hw.idx)
1304 continue;
1305 idx = hwc_index[i] + 1;
1306 if (is_limited_pmc(idx)) {
1307 cpuhw->limited_counter[n_lim] = event;
1308 cpuhw->limited_hwidx[n_lim] = idx;
1309 ++n_lim;
1310 continue;
1311 }
1312
1313 if (ebb)
1314 val = local64_read(&event->hw.prev_count);
1315 else {
1316 val = 0;
1317 if (event->hw.sample_period) {
1318 left = local64_read(&event->hw.period_left);
1319 if (left < 0x80000000L)
1320 val = 0x80000000L - left;
1321 }
1322 local64_set(&event->hw.prev_count, val);
1323 }
1324
1325 event->hw.idx = idx;
1326 if (event->hw.state & PERF_HES_STOPPED)
1327 val = 0;
1328 write_pmc(idx, val);
1329
1330 perf_event_update_userpage(event);
1331 }
1332 cpuhw->n_limited = n_lim;
1333 cpuhw->mmcr[0] |= MMCR0_PMXE | MMCR0_FCECE;
1334
1335 out_enable:
1336 pmao_restore_workaround(ebb);
1337
1338 mmcr0 = ebb_switch_in(ebb, cpuhw);
1339
1340 mb();
1341 if (cpuhw->bhrb_users)
1342 ppmu->config_bhrb(cpuhw->bhrb_filter);
1343
1344 write_mmcr0(cpuhw, mmcr0);
1345
1346
1347
1348
1349 if (cpuhw->mmcr[2] & MMCRA_SAMPLE_ENABLE) {
1350 mb();
1351 mtspr(SPRN_MMCRA, cpuhw->mmcr[2]);
1352 }
1353
1354 out:
1355
1356 local_irq_restore(flags);
1357}
1358
1359static int collect_events(struct perf_event *group, int max_count,
1360 struct perf_event *ctrs[], u64 *events,
1361 unsigned int *flags)
1362{
1363 int n = 0;
1364 struct perf_event *event;
1365
1366 if (!is_software_event(group)) {
1367 if (n >= max_count)
1368 return -1;
1369 ctrs[n] = group;
1370 flags[n] = group->hw.event_base;
1371 events[n++] = group->hw.config;
1372 }
1373 list_for_each_entry(event, &group->sibling_list, group_entry) {
1374 if (!is_software_event(event) &&
1375 event->state != PERF_EVENT_STATE_OFF) {
1376 if (n >= max_count)
1377 return -1;
1378 ctrs[n] = event;
1379 flags[n] = event->hw.event_base;
1380 events[n++] = event->hw.config;
1381 }
1382 }
1383 return n;
1384}
1385
1386
1387
1388
1389
1390
1391
1392static int power_pmu_add(struct perf_event *event, int ef_flags)
1393{
1394 struct cpu_hw_events *cpuhw;
1395 unsigned long flags;
1396 int n0;
1397 int ret = -EAGAIN;
1398
1399 local_irq_save(flags);
1400 perf_pmu_disable(event->pmu);
1401
1402
1403
1404
1405
1406 cpuhw = &__get_cpu_var(cpu_hw_events);
1407 n0 = cpuhw->n_events;
1408 if (n0 >= ppmu->n_counter)
1409 goto out;
1410 cpuhw->event[n0] = event;
1411 cpuhw->events[n0] = event->hw.config;
1412 cpuhw->flags[n0] = event->hw.event_base;
1413
1414
1415
1416
1417
1418
1419
1420 if (!(ef_flags & PERF_EF_START))
1421 event->hw.state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
1422 else
1423 event->hw.state = 0;
1424
1425
1426
1427
1428
1429
1430 if (cpuhw->group_flag & PERF_EVENT_TXN)
1431 goto nocheck;
1432
1433 if (check_excludes(cpuhw->event, cpuhw->flags, n0, 1))
1434 goto out;
1435 if (power_check_constraints(cpuhw, cpuhw->events, cpuhw->flags, n0 + 1))
1436 goto out;
1437 event->hw.config = cpuhw->events[n0];
1438
1439nocheck:
1440 ebb_event_add(event);
1441
1442 ++cpuhw->n_events;
1443 ++cpuhw->n_added;
1444
1445 ret = 0;
1446 out:
1447 if (has_branch_stack(event)) {
1448 power_pmu_bhrb_enable(event);
1449 cpuhw->bhrb_filter = ppmu->bhrb_filter_map(
1450 event->attr.branch_sample_type);
1451 }
1452
1453 perf_pmu_enable(event->pmu);
1454 local_irq_restore(flags);
1455 return ret;
1456}
1457
1458
1459
1460
1461static void power_pmu_del(struct perf_event *event, int ef_flags)
1462{
1463 struct cpu_hw_events *cpuhw;
1464 long i;
1465 unsigned long flags;
1466
1467 local_irq_save(flags);
1468 perf_pmu_disable(event->pmu);
1469
1470 power_pmu_read(event);
1471
1472 cpuhw = &__get_cpu_var(cpu_hw_events);
1473 for (i = 0; i < cpuhw->n_events; ++i) {
1474 if (event == cpuhw->event[i]) {
1475 while (++i < cpuhw->n_events) {
1476 cpuhw->event[i-1] = cpuhw->event[i];
1477 cpuhw->events[i-1] = cpuhw->events[i];
1478 cpuhw->flags[i-1] = cpuhw->flags[i];
1479 }
1480 --cpuhw->n_events;
1481 ppmu->disable_pmc(event->hw.idx - 1, cpuhw->mmcr);
1482 if (event->hw.idx) {
1483 write_pmc(event->hw.idx, 0);
1484 event->hw.idx = 0;
1485 }
1486 perf_event_update_userpage(event);
1487 break;
1488 }
1489 }
1490 for (i = 0; i < cpuhw->n_limited; ++i)
1491 if (event == cpuhw->limited_counter[i])
1492 break;
1493 if (i < cpuhw->n_limited) {
1494 while (++i < cpuhw->n_limited) {
1495 cpuhw->limited_counter[i-1] = cpuhw->limited_counter[i];
1496 cpuhw->limited_hwidx[i-1] = cpuhw->limited_hwidx[i];
1497 }
1498 --cpuhw->n_limited;
1499 }
1500 if (cpuhw->n_events == 0) {
1501
1502 cpuhw->mmcr[0] &= ~(MMCR0_PMXE | MMCR0_FCECE);
1503 }
1504
1505 if (has_branch_stack(event))
1506 power_pmu_bhrb_disable(event);
1507
1508 perf_pmu_enable(event->pmu);
1509 local_irq_restore(flags);
1510}
1511
1512
1513
1514
1515
1516
1517static void power_pmu_start(struct perf_event *event, int ef_flags)
1518{
1519 unsigned long flags;
1520 s64 left;
1521 unsigned long val;
1522
1523 if (!event->hw.idx || !event->hw.sample_period)
1524 return;
1525
1526 if (!(event->hw.state & PERF_HES_STOPPED))
1527 return;
1528
1529 if (ef_flags & PERF_EF_RELOAD)
1530 WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE));
1531
1532 local_irq_save(flags);
1533 perf_pmu_disable(event->pmu);
1534
1535 event->hw.state = 0;
1536 left = local64_read(&event->hw.period_left);
1537
1538 val = 0;
1539 if (left < 0x80000000L)
1540 val = 0x80000000L - left;
1541
1542 write_pmc(event->hw.idx, val);
1543
1544 perf_event_update_userpage(event);
1545 perf_pmu_enable(event->pmu);
1546 local_irq_restore(flags);
1547}
1548
1549static void power_pmu_stop(struct perf_event *event, int ef_flags)
1550{
1551 unsigned long flags;
1552
1553 if (!event->hw.idx || !event->hw.sample_period)
1554 return;
1555
1556 if (event->hw.state & PERF_HES_STOPPED)
1557 return;
1558
1559 local_irq_save(flags);
1560 perf_pmu_disable(event->pmu);
1561
1562 power_pmu_read(event);
1563 event->hw.state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
1564 write_pmc(event->hw.idx, 0);
1565
1566 perf_event_update_userpage(event);
1567 perf_pmu_enable(event->pmu);
1568 local_irq_restore(flags);
1569}
1570
1571
1572
1573
1574
1575
1576static void power_pmu_start_txn(struct pmu *pmu)
1577{
1578 struct cpu_hw_events *cpuhw = &__get_cpu_var(cpu_hw_events);
1579
1580 perf_pmu_disable(pmu);
1581 cpuhw->group_flag |= PERF_EVENT_TXN;
1582 cpuhw->n_txn_start = cpuhw->n_events;
1583}
1584
1585
1586
1587
1588
1589
1590static void power_pmu_cancel_txn(struct pmu *pmu)
1591{
1592 struct cpu_hw_events *cpuhw = &__get_cpu_var(cpu_hw_events);
1593
1594 cpuhw->group_flag &= ~PERF_EVENT_TXN;
1595 perf_pmu_enable(pmu);
1596}
1597
1598
1599
1600
1601
1602
1603static int power_pmu_commit_txn(struct pmu *pmu)
1604{
1605 struct cpu_hw_events *cpuhw;
1606 long i, n;
1607
1608 if (!ppmu)
1609 return -EAGAIN;
1610 cpuhw = &__get_cpu_var(cpu_hw_events);
1611 n = cpuhw->n_events;
1612 if (check_excludes(cpuhw->event, cpuhw->flags, 0, n))
1613 return -EAGAIN;
1614 i = power_check_constraints(cpuhw, cpuhw->events, cpuhw->flags, n);
1615 if (i < 0)
1616 return -EAGAIN;
1617
1618 for (i = cpuhw->n_txn_start; i < n; ++i)
1619 cpuhw->event[i]->hw.config = cpuhw->events[i];
1620
1621 cpuhw->group_flag &= ~PERF_EVENT_TXN;
1622 perf_pmu_enable(pmu);
1623 return 0;
1624}
1625
1626
1627
1628
1629
1630
1631
1632
1633static int can_go_on_limited_pmc(struct perf_event *event, u64 ev,
1634 unsigned int flags)
1635{
1636 int n;
1637 u64 alt[MAX_EVENT_ALTERNATIVES];
1638
1639 if (event->attr.exclude_user
1640 || event->attr.exclude_kernel
1641 || event->attr.exclude_hv
1642 || event->attr.sample_period)
1643 return 0;
1644
1645 if (ppmu->limited_pmc_event(ev))
1646 return 1;
1647
1648
1649
1650
1651
1652 if (!ppmu->get_alternatives)
1653 return 0;
1654
1655 flags |= PPMU_LIMITED_PMC_OK | PPMU_LIMITED_PMC_REQD;
1656 n = ppmu->get_alternatives(ev, flags, alt);
1657
1658 return n > 0;
1659}
1660
1661
1662
1663
1664
1665
1666static u64 normal_pmc_alternative(u64 ev, unsigned long flags)
1667{
1668 u64 alt[MAX_EVENT_ALTERNATIVES];
1669 int n;
1670
1671 flags &= ~(PPMU_LIMITED_PMC_OK | PPMU_LIMITED_PMC_REQD);
1672 n = ppmu->get_alternatives(ev, flags, alt);
1673 if (!n)
1674 return 0;
1675 return alt[0];
1676}
1677
1678
1679static atomic_t num_events;
1680
1681static DEFINE_MUTEX(pmc_reserve_mutex);
1682
1683
1684
1685
1686static void hw_perf_event_destroy(struct perf_event *event)
1687{
1688 if (!atomic_add_unless(&num_events, -1, 1)) {
1689 mutex_lock(&pmc_reserve_mutex);
1690 if (atomic_dec_return(&num_events) == 0)
1691 release_pmc_hardware();
1692 mutex_unlock(&pmc_reserve_mutex);
1693 }
1694}
1695
1696
1697
1698
1699static int hw_perf_cache_event(u64 config, u64 *eventp)
1700{
1701 unsigned long type, op, result;
1702 int ev;
1703
1704 if (!ppmu->cache_events)
1705 return -EINVAL;
1706
1707
1708 type = config & 0xff;
1709 op = (config >> 8) & 0xff;
1710 result = (config >> 16) & 0xff;
1711
1712 if (type >= PERF_COUNT_HW_CACHE_MAX ||
1713 op >= PERF_COUNT_HW_CACHE_OP_MAX ||
1714 result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
1715 return -EINVAL;
1716
1717 ev = (*ppmu->cache_events)[type][op][result];
1718 if (ev == 0)
1719 return -EOPNOTSUPP;
1720 if (ev == -1)
1721 return -EINVAL;
1722 *eventp = ev;
1723 return 0;
1724}
1725
1726static int power_pmu_event_init(struct perf_event *event)
1727{
1728 u64 ev;
1729 unsigned long flags;
1730 struct perf_event *ctrs[MAX_HWEVENTS];
1731 u64 events[MAX_HWEVENTS];
1732 unsigned int cflags[MAX_HWEVENTS];
1733 int n;
1734 int err;
1735 struct cpu_hw_events *cpuhw;
1736
1737 if (!ppmu)
1738 return -ENOENT;
1739
1740 if (has_branch_stack(event)) {
1741
1742 if (!(ppmu->flags & PPMU_ARCH_207S))
1743 return -EOPNOTSUPP;
1744 }
1745
1746 switch (event->attr.type) {
1747 case PERF_TYPE_HARDWARE:
1748 ev = event->attr.config;
1749 if (ev >= ppmu->n_generic || ppmu->generic_events[ev] == 0)
1750 return -EOPNOTSUPP;
1751 ev = ppmu->generic_events[ev];
1752 break;
1753 case PERF_TYPE_HW_CACHE:
1754 err = hw_perf_cache_event(event->attr.config, &ev);
1755 if (err)
1756 return err;
1757 break;
1758 case PERF_TYPE_RAW:
1759 ev = event->attr.config;
1760 break;
1761 default:
1762 return -ENOENT;
1763 }
1764
1765 event->hw.config_base = ev;
1766 event->hw.idx = 0;
1767
1768
1769
1770
1771
1772
1773 if (!firmware_has_feature(FW_FEATURE_LPAR))
1774 event->attr.exclude_hv = 0;
1775
1776
1777
1778
1779
1780
1781
1782 flags = 0;
1783 if (event->attach_state & PERF_ATTACH_TASK)
1784 flags |= PPMU_ONLY_COUNT_RUN;
1785
1786
1787
1788
1789
1790 if (ppmu->flags & PPMU_LIMITED_PMC5_6) {
1791 if (can_go_on_limited_pmc(event, ev, flags)) {
1792 flags |= PPMU_LIMITED_PMC_OK;
1793 } else if (ppmu->limited_pmc_event(ev)) {
1794
1795
1796
1797
1798
1799 ev = normal_pmc_alternative(ev, flags);
1800 if (!ev)
1801 return -EINVAL;
1802 }
1803 }
1804
1805
1806 err = ebb_event_check(event);
1807 if (err)
1808 return err;
1809
1810
1811
1812
1813
1814
1815 n = 0;
1816 if (event->group_leader != event) {
1817 n = collect_events(event->group_leader, ppmu->n_counter - 1,
1818 ctrs, events, cflags);
1819 if (n < 0)
1820 return -EINVAL;
1821 }
1822 events[n] = ev;
1823 ctrs[n] = event;
1824 cflags[n] = flags;
1825 if (check_excludes(ctrs, cflags, n, 1))
1826 return -EINVAL;
1827
1828 cpuhw = &get_cpu_var(cpu_hw_events);
1829 err = power_check_constraints(cpuhw, events, cflags, n + 1);
1830
1831 if (has_branch_stack(event)) {
1832 cpuhw->bhrb_filter = ppmu->bhrb_filter_map(
1833 event->attr.branch_sample_type);
1834
1835 if(cpuhw->bhrb_filter == -1)
1836 return -EOPNOTSUPP;
1837 }
1838
1839 put_cpu_var(cpu_hw_events);
1840 if (err)
1841 return -EINVAL;
1842
1843 event->hw.config = events[n];
1844 event->hw.event_base = cflags[n];
1845 event->hw.last_period = event->hw.sample_period;
1846 local64_set(&event->hw.period_left, event->hw.last_period);
1847
1848
1849
1850
1851
1852 if (is_ebb_event(event))
1853 local64_set(&event->hw.prev_count, 0);
1854
1855
1856
1857
1858
1859
1860
1861 err = 0;
1862 if (!atomic_inc_not_zero(&num_events)) {
1863 mutex_lock(&pmc_reserve_mutex);
1864 if (atomic_read(&num_events) == 0 &&
1865 reserve_pmc_hardware(perf_event_interrupt))
1866 err = -EBUSY;
1867 else
1868 atomic_inc(&num_events);
1869 mutex_unlock(&pmc_reserve_mutex);
1870 }
1871 event->destroy = hw_perf_event_destroy;
1872
1873 return err;
1874}
1875
1876static int power_pmu_event_idx(struct perf_event *event)
1877{
1878 return event->hw.idx;
1879}
1880
1881ssize_t power_events_sysfs_show(struct device *dev,
1882 struct device_attribute *attr, char *page)
1883{
1884 struct perf_pmu_events_attr *pmu_attr;
1885
1886 pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr);
1887
1888 return sprintf(page, "event=0x%02llx\n", pmu_attr->id);
1889}
1890
1891static struct pmu power_pmu = {
1892 .pmu_enable = power_pmu_enable,
1893 .pmu_disable = power_pmu_disable,
1894 .event_init = power_pmu_event_init,
1895 .add = power_pmu_add,
1896 .del = power_pmu_del,
1897 .start = power_pmu_start,
1898 .stop = power_pmu_stop,
1899 .read = power_pmu_read,
1900 .start_txn = power_pmu_start_txn,
1901 .cancel_txn = power_pmu_cancel_txn,
1902 .commit_txn = power_pmu_commit_txn,
1903 .event_idx = power_pmu_event_idx,
1904 .flush_branch_stack = power_pmu_flush_branch_stack,
1905};
1906
1907
1908
1909
1910
1911
1912static void record_and_restart(struct perf_event *event, unsigned long val,
1913 struct pt_regs *regs)
1914{
1915 u64 period = event->hw.sample_period;
1916 s64 prev, delta, left;
1917 int record = 0;
1918
1919 if (event->hw.state & PERF_HES_STOPPED) {
1920 write_pmc(event->hw.idx, 0);
1921 return;
1922 }
1923
1924
1925 prev = local64_read(&event->hw.prev_count);
1926 delta = check_and_compute_delta(prev, val);
1927 local64_add(delta, &event->count);
1928
1929
1930
1931
1932
1933 val = 0;
1934 left = local64_read(&event->hw.period_left) - delta;
1935 if (delta == 0)
1936 left++;
1937 if (period) {
1938 if (left <= 0) {
1939 left += period;
1940 if (left <= 0)
1941 left = period;
1942 record = siar_valid(regs);
1943 event->hw.last_period = event->hw.sample_period;
1944 }
1945 if (left < 0x80000000LL)
1946 val = 0x80000000LL - left;
1947 }
1948
1949 write_pmc(event->hw.idx, val);
1950 local64_set(&event->hw.prev_count, val);
1951 local64_set(&event->hw.period_left, left);
1952 perf_event_update_userpage(event);
1953
1954
1955
1956
1957 if (record) {
1958 struct perf_sample_data data;
1959
1960 perf_sample_data_init(&data, ~0ULL, event->hw.last_period);
1961
1962 if (event->attr.sample_type & PERF_SAMPLE_ADDR)
1963 perf_get_data_addr(regs, &data.addr);
1964
1965 if (event->attr.sample_type & PERF_SAMPLE_BRANCH_STACK) {
1966 struct cpu_hw_events *cpuhw;
1967 cpuhw = &__get_cpu_var(cpu_hw_events);
1968 power_pmu_bhrb_read(cpuhw);
1969 data.br_stack = &cpuhw->bhrb_stack;
1970 }
1971
1972 if (perf_event_overflow(event, &data, regs))
1973 power_pmu_stop(event, 0);
1974 }
1975}
1976
1977
1978
1979
1980
1981unsigned long perf_misc_flags(struct pt_regs *regs)
1982{
1983 u32 flags = perf_get_misc_flags(regs);
1984
1985 if (flags)
1986 return flags;
1987 return user_mode(regs) ? PERF_RECORD_MISC_USER :
1988 PERF_RECORD_MISC_KERNEL;
1989}
1990
1991
1992
1993
1994
1995unsigned long perf_instruction_pointer(struct pt_regs *regs)
1996{
1997 bool use_siar = regs_use_siar(regs);
1998
1999 if (use_siar && siar_valid(regs))
2000 return mfspr(SPRN_SIAR) + perf_ip_adjust(regs);
2001 else if (use_siar)
2002 return 0;
2003 else
2004 return regs->nip;
2005}
2006
2007static bool pmc_overflow_power7(unsigned long val)
2008{
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020 if ((0x80000000 - val) <= 256)
2021 return true;
2022
2023 return false;
2024}
2025
2026static bool pmc_overflow(unsigned long val)
2027{
2028 if ((int)val < 0)
2029 return true;
2030
2031 return false;
2032}
2033
2034
2035
2036
2037static void perf_event_interrupt(struct pt_regs *regs)
2038{
2039 int i, j;
2040 struct cpu_hw_events *cpuhw = &__get_cpu_var(cpu_hw_events);
2041 struct perf_event *event;
2042 unsigned long val[8];
2043 int found, active;
2044 int nmi;
2045
2046 if (cpuhw->n_limited)
2047 freeze_limited_counters(cpuhw, mfspr(SPRN_PMC5),
2048 mfspr(SPRN_PMC6));
2049
2050 perf_read_regs(regs);
2051
2052 nmi = perf_intr_is_nmi(regs);
2053 if (nmi)
2054 nmi_enter();
2055 else
2056 irq_enter();
2057
2058
2059 for (i = 0; i < ppmu->n_counter; ++i)
2060 val[i] = read_pmc(i + 1);
2061
2062
2063 found = 0;
2064 for (i = 0; i < ppmu->n_counter; ++i) {
2065 if (!pmc_overflow(val[i]))
2066 continue;
2067 if (is_limited_pmc(i + 1))
2068 continue;
2069
2070
2071
2072
2073
2074 found = 1;
2075 active = 0;
2076 for (j = 0; j < cpuhw->n_events; ++j) {
2077 event = cpuhw->event[j];
2078 if (event->hw.idx == (i + 1)) {
2079 active = 1;
2080 record_and_restart(event, val[i], regs);
2081 break;
2082 }
2083 }
2084 if (!active)
2085
2086 write_pmc(i + 1, 0);
2087 }
2088 if (!found && pvr_version_is(PVR_POWER7)) {
2089
2090 for (i = 0; i < cpuhw->n_events; ++i) {
2091 event = cpuhw->event[i];
2092 if (!event->hw.idx || is_limited_pmc(event->hw.idx))
2093 continue;
2094 if (pmc_overflow_power7(val[event->hw.idx - 1])) {
2095
2096 found = 1;
2097 record_and_restart(event,
2098 val[event->hw.idx - 1],
2099 regs);
2100 }
2101 }
2102 }
2103 if (!found && !nmi && printk_ratelimit())
2104 printk(KERN_WARNING "Can't find PMC that caused IRQ\n");
2105
2106
2107
2108
2109
2110
2111
2112
2113 write_mmcr0(cpuhw, cpuhw->mmcr[0]);
2114
2115 if (nmi)
2116 nmi_exit();
2117 else
2118 irq_exit();
2119}
2120
2121static void power_pmu_setup(int cpu)
2122{
2123 struct cpu_hw_events *cpuhw = &per_cpu(cpu_hw_events, cpu);
2124
2125 if (!ppmu)
2126 return;
2127 memset(cpuhw, 0, sizeof(*cpuhw));
2128 cpuhw->mmcr[0] = MMCR0_FC;
2129}
2130
2131static int
2132power_pmu_notifier(struct notifier_block *self, unsigned long action, void *hcpu)
2133{
2134 unsigned int cpu = (long)hcpu;
2135
2136 switch (action & ~CPU_TASKS_FROZEN) {
2137 case CPU_UP_PREPARE:
2138 power_pmu_setup(cpu);
2139 break;
2140
2141 default:
2142 break;
2143 }
2144
2145 return NOTIFY_OK;
2146}
2147
2148int register_power_pmu(struct power_pmu *pmu)
2149{
2150 if (ppmu)
2151 return -EBUSY;
2152
2153 ppmu = pmu;
2154 pr_info("%s performance monitor hardware support registered\n",
2155 pmu->name);
2156
2157 power_pmu.attr_groups = ppmu->attr_groups;
2158
2159#ifdef MSR_HV
2160
2161
2162
2163 if (mfmsr() & MSR_HV)
2164 freeze_events_kernel = MMCR0_FCHV;
2165#endif
2166
2167 perf_pmu_register(&power_pmu, "cpu", PERF_TYPE_RAW);
2168 perf_cpu_notifier(power_pmu_notifier);
2169
2170 return 0;
2171}
2172