1
2
3
4
5
6
7
8
9
10
11#include <linux/kernel.h>
12#include <linux/sched.h>
13#include <linux/perf_event.h>
14#include <linux/percpu.h>
15#include <linux/hardirq.h>
16#include <linux/uaccess.h>
17#include <asm/reg.h>
18#include <asm/pmc.h>
19#include <asm/machdep.h>
20#include <asm/firmware.h>
21#include <asm/ptrace.h>
22#include <asm/code-patching.h>
23
24#define BHRB_MAX_ENTRIES 32
25#define BHRB_TARGET 0x0000000000000002
26#define BHRB_PREDICTION 0x0000000000000001
27#define BHRB_EA 0xFFFFFFFFFFFFFFFCUL
28
29struct cpu_hw_events {
30 int n_events;
31 int n_percpu;
32 int disabled;
33 int n_added;
34 int n_limited;
35 u8 pmcs_enabled;
36 struct perf_event *event[MAX_HWEVENTS];
37 u64 events[MAX_HWEVENTS];
38 unsigned int flags[MAX_HWEVENTS];
39
40
41
42
43
44 unsigned long mmcr[4];
45 struct perf_event *limited_counter[MAX_LIMITED_HWCOUNTERS];
46 u8 limited_hwidx[MAX_LIMITED_HWCOUNTERS];
47 u64 alternatives[MAX_HWEVENTS][MAX_EVENT_ALTERNATIVES];
48 unsigned long amasks[MAX_HWEVENTS][MAX_EVENT_ALTERNATIVES];
49 unsigned long avalues[MAX_HWEVENTS][MAX_EVENT_ALTERNATIVES];
50
51 unsigned int txn_flags;
52 int n_txn_start;
53
54
55 u64 bhrb_filter;
56 unsigned int bhrb_users;
57 void *bhrb_context;
58 struct perf_branch_stack bhrb_stack;
59 struct perf_branch_entry bhrb_entries[BHRB_MAX_ENTRIES];
60 u64 ic_init;
61};
62
63static DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events);
64
65static struct power_pmu *ppmu;
66
67
68
69
70
71
72
73
74static unsigned int freeze_events_kernel = MMCR0_FCS;
75
76
77
78
79
80#ifdef CONFIG_PPC32
81
82#define MMCR0_FCHV 0
83#define MMCR0_PMCjCE MMCR0_PMCnCE
84#define MMCR0_FC56 0
85#define MMCR0_PMAO 0
86#define MMCR0_EBE 0
87#define MMCR0_BHRBA 0
88#define MMCR0_PMCC 0
89#define MMCR0_PMCC_U6 0
90
91#define SPRN_MMCRA SPRN_MMCR2
92#define MMCRA_SAMPLE_ENABLE 0
93
94static inline unsigned long perf_ip_adjust(struct pt_regs *regs)
95{
96 return 0;
97}
98static inline void perf_get_data_addr(struct pt_regs *regs, u64 *addrp) { }
99static inline u32 perf_get_misc_flags(struct pt_regs *regs)
100{
101 return 0;
102}
103static inline void perf_read_regs(struct pt_regs *regs)
104{
105 regs->result = 0;
106}
107static inline int perf_intr_is_nmi(struct pt_regs *regs)
108{
109 return 0;
110}
111
112static inline int siar_valid(struct pt_regs *regs)
113{
114 return 1;
115}
116
117static bool is_ebb_event(struct perf_event *event) { return false; }
118static int ebb_event_check(struct perf_event *event) { return 0; }
119static void ebb_event_add(struct perf_event *event) { }
120static void ebb_switch_out(unsigned long mmcr0) { }
121static unsigned long ebb_switch_in(bool ebb, struct cpu_hw_events *cpuhw)
122{
123 return cpuhw->mmcr[0];
124}
125
126static inline void power_pmu_bhrb_enable(struct perf_event *event) {}
127static inline void power_pmu_bhrb_disable(struct perf_event *event) {}
128static void power_pmu_sched_task(struct perf_event_context *ctx, bool sched_in) {}
129static inline void power_pmu_bhrb_read(struct cpu_hw_events *cpuhw) {}
130static void pmao_restore_workaround(bool ebb) { }
131static bool use_ic(u64 event)
132{
133 return false;
134}
135#endif
136
137static bool regs_use_siar(struct pt_regs *regs)
138{
139
140
141
142
143
144
145
146
147
148 return ((TRAP(regs) == 0xf00) && regs->result);
149}
150
151
152
153
154#ifdef CONFIG_PPC64
155
156static inline unsigned long perf_ip_adjust(struct pt_regs *regs)
157{
158 unsigned long mmcra = regs->dsisr;
159
160 if ((ppmu->flags & PPMU_HAS_SSLOT) && (mmcra & MMCRA_SAMPLE_ENABLE)) {
161 unsigned long slot = (mmcra & MMCRA_SLOT) >> MMCRA_SLOT_SHIFT;
162 if (slot > 1)
163 return 4 * (slot - 1);
164 }
165
166 return 0;
167}
168
169
170
171
172
173
174
175
176
177static inline void perf_get_data_addr(struct pt_regs *regs, u64 *addrp)
178{
179 unsigned long mmcra = regs->dsisr;
180 bool sdar_valid;
181
182 if (ppmu->flags & PPMU_HAS_SIER)
183 sdar_valid = regs->dar & SIER_SDAR_VALID;
184 else {
185 unsigned long sdsync;
186
187 if (ppmu->flags & PPMU_SIAR_VALID)
188 sdsync = POWER7P_MMCRA_SDAR_VALID;
189 else if (ppmu->flags & PPMU_ALT_SIPR)
190 sdsync = POWER6_MMCRA_SDSYNC;
191 else if (ppmu->flags & PPMU_NO_SIAR)
192 sdsync = MMCRA_SAMPLE_ENABLE;
193 else
194 sdsync = MMCRA_SDSYNC;
195
196 sdar_valid = mmcra & sdsync;
197 }
198
199 if (!(mmcra & MMCRA_SAMPLE_ENABLE) || sdar_valid)
200 *addrp = mfspr(SPRN_SDAR);
201
202 if (perf_paranoid_kernel() && !capable(CAP_SYS_ADMIN) &&
203 is_kernel_addr(mfspr(SPRN_SDAR)))
204 *addrp = 0;
205}
206
207static bool regs_sihv(struct pt_regs *regs)
208{
209 unsigned long sihv = MMCRA_SIHV;
210
211 if (ppmu->flags & PPMU_HAS_SIER)
212 return !!(regs->dar & SIER_SIHV);
213
214 if (ppmu->flags & PPMU_ALT_SIPR)
215 sihv = POWER6_MMCRA_SIHV;
216
217 return !!(regs->dsisr & sihv);
218}
219
220static bool regs_sipr(struct pt_regs *regs)
221{
222 unsigned long sipr = MMCRA_SIPR;
223
224 if (ppmu->flags & PPMU_HAS_SIER)
225 return !!(regs->dar & SIER_SIPR);
226
227 if (ppmu->flags & PPMU_ALT_SIPR)
228 sipr = POWER6_MMCRA_SIPR;
229
230 return !!(regs->dsisr & sipr);
231}
232
233static inline u32 perf_flags_from_msr(struct pt_regs *regs)
234{
235 if (regs->msr & MSR_PR)
236 return PERF_RECORD_MISC_USER;
237 if ((regs->msr & MSR_HV) && freeze_events_kernel != MMCR0_FCHV)
238 return PERF_RECORD_MISC_HYPERVISOR;
239 return PERF_RECORD_MISC_KERNEL;
240}
241
242static inline u32 perf_get_misc_flags(struct pt_regs *regs)
243{
244 bool use_siar = regs_use_siar(regs);
245
246 if (!use_siar)
247 return perf_flags_from_msr(regs);
248
249
250
251
252
253
254
255 if (ppmu->flags & PPMU_NO_SIPR) {
256 unsigned long siar = mfspr(SPRN_SIAR);
257 if (is_kernel_addr(siar))
258 return PERF_RECORD_MISC_KERNEL;
259 return PERF_RECORD_MISC_USER;
260 }
261
262
263 if (regs_sipr(regs))
264 return PERF_RECORD_MISC_USER;
265
266 if (regs_sihv(regs) && (freeze_events_kernel != MMCR0_FCHV))
267 return PERF_RECORD_MISC_HYPERVISOR;
268
269 return PERF_RECORD_MISC_KERNEL;
270}
271
272
273
274
275
276
277
278
279static inline void perf_read_regs(struct pt_regs *regs)
280{
281 unsigned long mmcra = mfspr(SPRN_MMCRA);
282 int marked = mmcra & MMCRA_SAMPLE_ENABLE;
283 int use_siar;
284
285 regs->dsisr = mmcra;
286
287 if (ppmu->flags & PPMU_HAS_SIER)
288 regs->dar = mfspr(SPRN_SIER);
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307 if (TRAP(regs) != 0xf00)
308 use_siar = 0;
309 else if ((ppmu->flags & PPMU_NO_SIAR))
310 use_siar = 0;
311 else if (marked)
312 use_siar = 1;
313 else if ((ppmu->flags & PPMU_NO_CONT_SAMPLING))
314 use_siar = 0;
315 else if (!(ppmu->flags & PPMU_NO_SIPR) && regs_sipr(regs))
316 use_siar = 0;
317 else
318 use_siar = 1;
319
320 regs->result = use_siar;
321}
322
323
324
325
326
327static inline int perf_intr_is_nmi(struct pt_regs *regs)
328{
329 return (regs->softe & IRQS_DISABLED);
330}
331
332
333
334
335
336
337
338
339static inline int siar_valid(struct pt_regs *regs)
340{
341 unsigned long mmcra = regs->dsisr;
342 int marked = mmcra & MMCRA_SAMPLE_ENABLE;
343
344 if (marked) {
345 if (ppmu->flags & PPMU_HAS_SIER)
346 return regs->dar & SIER_SIAR_VALID;
347
348 if (ppmu->flags & PPMU_SIAR_VALID)
349 return mmcra & POWER7P_MMCRA_SIAR_VALID;
350 }
351
352 return 1;
353}
354
355
356
357static void power_pmu_bhrb_reset(void)
358{
359 asm volatile(PPC_CLRBHRB);
360}
361
362static void power_pmu_bhrb_enable(struct perf_event *event)
363{
364 struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events);
365
366 if (!ppmu->bhrb_nr)
367 return;
368
369
370 if (event->ctx->task && cpuhw->bhrb_context != event->ctx) {
371 power_pmu_bhrb_reset();
372 cpuhw->bhrb_context = event->ctx;
373 }
374 cpuhw->bhrb_users++;
375 perf_sched_cb_inc(event->ctx->pmu);
376}
377
378static void power_pmu_bhrb_disable(struct perf_event *event)
379{
380 struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events);
381
382 if (!ppmu->bhrb_nr)
383 return;
384
385 WARN_ON_ONCE(!cpuhw->bhrb_users);
386 cpuhw->bhrb_users--;
387 perf_sched_cb_dec(event->ctx->pmu);
388
389 if (!cpuhw->disabled && !cpuhw->bhrb_users) {
390
391
392
393
394
395 cpuhw->bhrb_context = NULL;
396 }
397}
398
399
400
401
402static void power_pmu_sched_task(struct perf_event_context *ctx, bool sched_in)
403{
404 if (!ppmu->bhrb_nr)
405 return;
406
407 if (sched_in)
408 power_pmu_bhrb_reset();
409}
410
411static __u64 power_pmu_bhrb_to(u64 addr)
412{
413 unsigned int instr;
414 int ret;
415 __u64 target;
416
417 if (is_kernel_addr(addr)) {
418 if (probe_kernel_read(&instr, (void *)addr, sizeof(instr)))
419 return 0;
420
421 return branch_target(&instr);
422 }
423
424
425 pagefault_disable();
426 ret = __get_user_inatomic(instr, (unsigned int __user *)addr);
427 if (ret) {
428 pagefault_enable();
429 return 0;
430 }
431 pagefault_enable();
432
433 target = branch_target(&instr);
434 if ((!target) || (instr & BRANCH_ABSOLUTE))
435 return target;
436
437
438 return target - (unsigned long)&instr + addr;
439}
440
441
442static void power_pmu_bhrb_read(struct cpu_hw_events *cpuhw)
443{
444 u64 val;
445 u64 addr;
446 int r_index, u_index, pred;
447
448 r_index = 0;
449 u_index = 0;
450 while (r_index < ppmu->bhrb_nr) {
451
452 val = read_bhrb(r_index++);
453 if (!val)
454
455 break;
456 else {
457 addr = val & BHRB_EA;
458 pred = val & BHRB_PREDICTION;
459
460 if (!addr)
461
462 continue;
463
464
465
466
467
468
469
470 if (perf_paranoid_kernel() && !capable(CAP_SYS_ADMIN) &&
471 is_kernel_addr(addr))
472 continue;
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492 if (val & BHRB_TARGET) {
493
494
495
496 cpuhw->bhrb_entries[u_index].to = addr;
497 cpuhw->bhrb_entries[u_index].mispred = pred;
498 cpuhw->bhrb_entries[u_index].predicted = ~pred;
499
500
501 val = read_bhrb(r_index++);
502 addr = val & BHRB_EA;
503 if (val & BHRB_TARGET) {
504
505
506 r_index--;
507 addr = 0;
508 }
509 cpuhw->bhrb_entries[u_index].from = addr;
510 } else {
511
512
513 cpuhw->bhrb_entries[u_index].from = addr;
514 cpuhw->bhrb_entries[u_index].to =
515 power_pmu_bhrb_to(addr);
516 cpuhw->bhrb_entries[u_index].mispred = pred;
517 cpuhw->bhrb_entries[u_index].predicted = ~pred;
518 }
519 u_index++;
520
521 }
522 }
523 cpuhw->bhrb_stack.nr = u_index;
524 return;
525}
526
527static bool is_ebb_event(struct perf_event *event)
528{
529
530
531
532
533
534 return (ppmu->flags & PPMU_ARCH_207S) &&
535 ((event->attr.config >> PERF_EVENT_CONFIG_EBB_SHIFT) & 1);
536}
537
538static int ebb_event_check(struct perf_event *event)
539{
540 struct perf_event *leader = event->group_leader;
541
542
543 if (is_ebb_event(leader) != is_ebb_event(event))
544 return -EINVAL;
545
546 if (is_ebb_event(event)) {
547 if (!(event->attach_state & PERF_ATTACH_TASK))
548 return -EINVAL;
549
550 if (!leader->attr.pinned || !leader->attr.exclusive)
551 return -EINVAL;
552
553 if (event->attr.freq ||
554 event->attr.inherit ||
555 event->attr.sample_type ||
556 event->attr.sample_period ||
557 event->attr.enable_on_exec)
558 return -EINVAL;
559 }
560
561 return 0;
562}
563
564static void ebb_event_add(struct perf_event *event)
565{
566 if (!is_ebb_event(event) || current->thread.used_ebb)
567 return;
568
569
570
571
572
573
574
575 current->thread.used_ebb = 1;
576 current->thread.mmcr0 |= MMCR0_PMXE;
577}
578
579static void ebb_switch_out(unsigned long mmcr0)
580{
581 if (!(mmcr0 & MMCR0_EBE))
582 return;
583
584 current->thread.siar = mfspr(SPRN_SIAR);
585 current->thread.sier = mfspr(SPRN_SIER);
586 current->thread.sdar = mfspr(SPRN_SDAR);
587 current->thread.mmcr0 = mmcr0 & MMCR0_USER_MASK;
588 current->thread.mmcr2 = mfspr(SPRN_MMCR2) & MMCR2_USER_MASK;
589}
590
591static unsigned long ebb_switch_in(bool ebb, struct cpu_hw_events *cpuhw)
592{
593 unsigned long mmcr0 = cpuhw->mmcr[0];
594
595 if (!ebb)
596 goto out;
597
598
599 mmcr0 |= MMCR0_EBE | MMCR0_BHRBA | MMCR0_PMCC_U6;
600
601
602
603
604
605
606 mmcr0 |= current->thread.mmcr0;
607
608
609
610
611
612
613 if (!(current->thread.mmcr0 & MMCR0_PMXE))
614 mmcr0 &= ~MMCR0_PMXE;
615
616 mtspr(SPRN_SIAR, current->thread.siar);
617 mtspr(SPRN_SIER, current->thread.sier);
618 mtspr(SPRN_SDAR, current->thread.sdar);
619
620
621
622
623
624
625
626
627 mtspr(SPRN_MMCR2, cpuhw->mmcr[3] | current->thread.mmcr2);
628out:
629 return mmcr0;
630}
631
632static void pmao_restore_workaround(bool ebb)
633{
634 unsigned pmcs[6];
635
636 if (!cpu_has_feature(CPU_FTR_PMAO_BUG))
637 return;
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672 if ((current->thread.mmcr0 & (MMCR0_PMAO | MMCR0_PMAO_SYNC)) != MMCR0_PMAO)
673 return;
674
675
676 if (ebb && !(current->thread.bescr & BESCR_GE))
677 return;
678
679
680
681
682
683 hard_irq_disable();
684
685
686
687
688
689
690 pmcs[0] = mfspr(SPRN_PMC1);
691 pmcs[1] = mfspr(SPRN_PMC2);
692 pmcs[2] = mfspr(SPRN_PMC3);
693 pmcs[3] = mfspr(SPRN_PMC4);
694 pmcs[4] = mfspr(SPRN_PMC5);
695 pmcs[5] = mfspr(SPRN_PMC6);
696
697
698 mtspr(SPRN_MMCR2, 0);
699
700
701 mtspr(SPRN_PMC6, 0x7FFFFFFE);
702
703
704 mtspr(SPRN_MMCR0, MMCR0_PMXE | MMCR0_PMCjCE | MMCR0_PMAO);
705
706
707 mtspr(SPRN_MMCR0, MMCR0_FC | MMCR0_PMAO);
708
709 mtspr(SPRN_PMC1, pmcs[0]);
710 mtspr(SPRN_PMC2, pmcs[1]);
711 mtspr(SPRN_PMC3, pmcs[2]);
712 mtspr(SPRN_PMC4, pmcs[3]);
713 mtspr(SPRN_PMC5, pmcs[4]);
714 mtspr(SPRN_PMC6, pmcs[5]);
715}
716
717static bool use_ic(u64 event)
718{
719 if (cpu_has_feature(CPU_FTR_POWER9_DD1) &&
720 (event == 0x200f2 || event == 0x300f2))
721 return true;
722
723 return false;
724}
725#endif
726
727static void perf_event_interrupt(struct pt_regs *regs);
728
729
730
731
732static unsigned long read_pmc(int idx)
733{
734 unsigned long val;
735
736 switch (idx) {
737 case 1:
738 val = mfspr(SPRN_PMC1);
739 break;
740 case 2:
741 val = mfspr(SPRN_PMC2);
742 break;
743 case 3:
744 val = mfspr(SPRN_PMC3);
745 break;
746 case 4:
747 val = mfspr(SPRN_PMC4);
748 break;
749 case 5:
750 val = mfspr(SPRN_PMC5);
751 break;
752 case 6:
753 val = mfspr(SPRN_PMC6);
754 break;
755#ifdef CONFIG_PPC64
756 case 7:
757 val = mfspr(SPRN_PMC7);
758 break;
759 case 8:
760 val = mfspr(SPRN_PMC8);
761 break;
762#endif
763 default:
764 printk(KERN_ERR "oops trying to read PMC%d\n", idx);
765 val = 0;
766 }
767 return val;
768}
769
770
771
772
773static void write_pmc(int idx, unsigned long val)
774{
775 switch (idx) {
776 case 1:
777 mtspr(SPRN_PMC1, val);
778 break;
779 case 2:
780 mtspr(SPRN_PMC2, val);
781 break;
782 case 3:
783 mtspr(SPRN_PMC3, val);
784 break;
785 case 4:
786 mtspr(SPRN_PMC4, val);
787 break;
788 case 5:
789 mtspr(SPRN_PMC5, val);
790 break;
791 case 6:
792 mtspr(SPRN_PMC6, val);
793 break;
794#ifdef CONFIG_PPC64
795 case 7:
796 mtspr(SPRN_PMC7, val);
797 break;
798 case 8:
799 mtspr(SPRN_PMC8, val);
800 break;
801#endif
802 default:
803 printk(KERN_ERR "oops trying to write PMC%d\n", idx);
804 }
805}
806
807
808void perf_event_print_debug(void)
809{
810 unsigned long sdar, sier, flags;
811 u32 pmcs[MAX_HWEVENTS];
812 int i;
813
814 if (!ppmu) {
815 pr_info("Performance monitor hardware not registered.\n");
816 return;
817 }
818
819 if (!ppmu->n_counter)
820 return;
821
822 local_irq_save(flags);
823
824 pr_info("CPU: %d PMU registers, ppmu = %s n_counters = %d",
825 smp_processor_id(), ppmu->name, ppmu->n_counter);
826
827 for (i = 0; i < ppmu->n_counter; i++)
828 pmcs[i] = read_pmc(i + 1);
829
830 for (; i < MAX_HWEVENTS; i++)
831 pmcs[i] = 0xdeadbeef;
832
833 pr_info("PMC1: %08x PMC2: %08x PMC3: %08x PMC4: %08x\n",
834 pmcs[0], pmcs[1], pmcs[2], pmcs[3]);
835
836 if (ppmu->n_counter > 4)
837 pr_info("PMC5: %08x PMC6: %08x PMC7: %08x PMC8: %08x\n",
838 pmcs[4], pmcs[5], pmcs[6], pmcs[7]);
839
840 pr_info("MMCR0: %016lx MMCR1: %016lx MMCRA: %016lx\n",
841 mfspr(SPRN_MMCR0), mfspr(SPRN_MMCR1), mfspr(SPRN_MMCRA));
842
843 sdar = sier = 0;
844#ifdef CONFIG_PPC64
845 sdar = mfspr(SPRN_SDAR);
846
847 if (ppmu->flags & PPMU_HAS_SIER)
848 sier = mfspr(SPRN_SIER);
849
850 if (ppmu->flags & PPMU_ARCH_207S) {
851 pr_info("MMCR2: %016lx EBBHR: %016lx\n",
852 mfspr(SPRN_MMCR2), mfspr(SPRN_EBBHR));
853 pr_info("EBBRR: %016lx BESCR: %016lx\n",
854 mfspr(SPRN_EBBRR), mfspr(SPRN_BESCR));
855 }
856#endif
857 pr_info("SIAR: %016lx SDAR: %016lx SIER: %016lx\n",
858 mfspr(SPRN_SIAR), sdar, sier);
859
860 local_irq_restore(flags);
861}
862
863
864
865
866
867
868
869static int power_check_constraints(struct cpu_hw_events *cpuhw,
870 u64 event_id[], unsigned int cflags[],
871 int n_ev)
872{
873 unsigned long mask, value, nv;
874 unsigned long smasks[MAX_HWEVENTS], svalues[MAX_HWEVENTS];
875 int n_alt[MAX_HWEVENTS], choice[MAX_HWEVENTS];
876 int i, j;
877 unsigned long addf = ppmu->add_fields;
878 unsigned long tadd = ppmu->test_adder;
879
880 if (n_ev > ppmu->n_counter)
881 return -1;
882
883
884 for (i = 0; i < n_ev; ++i) {
885 if ((cflags[i] & PPMU_LIMITED_PMC_REQD)
886 && !ppmu->limited_pmc_event(event_id[i])) {
887 ppmu->get_alternatives(event_id[i], cflags[i],
888 cpuhw->alternatives[i]);
889 event_id[i] = cpuhw->alternatives[i][0];
890 }
891 if (ppmu->get_constraint(event_id[i], &cpuhw->amasks[i][0],
892 &cpuhw->avalues[i][0]))
893 return -1;
894 }
895 value = mask = 0;
896 for (i = 0; i < n_ev; ++i) {
897 nv = (value | cpuhw->avalues[i][0]) +
898 (value & cpuhw->avalues[i][0] & addf);
899 if ((((nv + tadd) ^ value) & mask) != 0 ||
900 (((nv + tadd) ^ cpuhw->avalues[i][0]) &
901 cpuhw->amasks[i][0]) != 0)
902 break;
903 value = nv;
904 mask |= cpuhw->amasks[i][0];
905 }
906 if (i == n_ev)
907 return 0;
908
909
910 if (!ppmu->get_alternatives)
911 return -1;
912 for (i = 0; i < n_ev; ++i) {
913 choice[i] = 0;
914 n_alt[i] = ppmu->get_alternatives(event_id[i], cflags[i],
915 cpuhw->alternatives[i]);
916 for (j = 1; j < n_alt[i]; ++j)
917 ppmu->get_constraint(cpuhw->alternatives[i][j],
918 &cpuhw->amasks[i][j],
919 &cpuhw->avalues[i][j]);
920 }
921
922
923 i = 0;
924 j = -1;
925 value = mask = nv = 0;
926 while (i < n_ev) {
927 if (j >= 0) {
928
929 value = svalues[i];
930 mask = smasks[i];
931 j = choice[i];
932 }
933
934
935
936
937 while (++j < n_alt[i]) {
938 nv = (value | cpuhw->avalues[i][j]) +
939 (value & cpuhw->avalues[i][j] & addf);
940 if ((((nv + tadd) ^ value) & mask) == 0 &&
941 (((nv + tadd) ^ cpuhw->avalues[i][j])
942 & cpuhw->amasks[i][j]) == 0)
943 break;
944 }
945 if (j >= n_alt[i]) {
946
947
948
949
950
951 if (--i < 0)
952 return -1;
953 } else {
954
955
956
957
958
959
960 choice[i] = j;
961 svalues[i] = value;
962 smasks[i] = mask;
963 value = nv;
964 mask |= cpuhw->amasks[i][j];
965 ++i;
966 j = -1;
967 }
968 }
969
970
971 for (i = 0; i < n_ev; ++i)
972 event_id[i] = cpuhw->alternatives[i][choice[i]];
973 return 0;
974}
975
976
977
978
979
980
981static int check_excludes(struct perf_event **ctrs, unsigned int cflags[],
982 int n_prev, int n_new)
983{
984 int eu = 0, ek = 0, eh = 0;
985 int i, n, first;
986 struct perf_event *event;
987
988
989
990
991
992
993 if (ppmu->flags & PPMU_ARCH_207S)
994 return 0;
995
996 n = n_prev + n_new;
997 if (n <= 1)
998 return 0;
999
1000 first = 1;
1001 for (i = 0; i < n; ++i) {
1002 if (cflags[i] & PPMU_LIMITED_PMC_OK) {
1003 cflags[i] &= ~PPMU_LIMITED_PMC_REQD;
1004 continue;
1005 }
1006 event = ctrs[i];
1007 if (first) {
1008 eu = event->attr.exclude_user;
1009 ek = event->attr.exclude_kernel;
1010 eh = event->attr.exclude_hv;
1011 first = 0;
1012 } else if (event->attr.exclude_user != eu ||
1013 event->attr.exclude_kernel != ek ||
1014 event->attr.exclude_hv != eh) {
1015 return -EAGAIN;
1016 }
1017 }
1018
1019 if (eu || ek || eh)
1020 for (i = 0; i < n; ++i)
1021 if (cflags[i] & PPMU_LIMITED_PMC_OK)
1022 cflags[i] |= PPMU_LIMITED_PMC_REQD;
1023
1024 return 0;
1025}
1026
1027static u64 check_and_compute_delta(u64 prev, u64 val)
1028{
1029 u64 delta = (val - prev) & 0xfffffffful;
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040 if (prev > val && (prev - val) < 256)
1041 delta = 0;
1042
1043 return delta;
1044}
1045
1046static void power_pmu_read(struct perf_event *event)
1047{
1048 s64 val, delta, prev;
1049 struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events);
1050
1051 if (event->hw.state & PERF_HES_STOPPED)
1052 return;
1053
1054 if (!event->hw.idx)
1055 return;
1056
1057 if (is_ebb_event(event)) {
1058 val = read_pmc(event->hw.idx);
1059 if (use_ic(event->attr.config)) {
1060 val = mfspr(SPRN_IC);
1061 if (val > cpuhw->ic_init)
1062 val = val - cpuhw->ic_init;
1063 else
1064 val = val + (0 - cpuhw->ic_init);
1065 }
1066 local64_set(&event->hw.prev_count, val);
1067 return;
1068 }
1069
1070
1071
1072
1073
1074
1075 do {
1076 prev = local64_read(&event->hw.prev_count);
1077 barrier();
1078 val = read_pmc(event->hw.idx);
1079 if (use_ic(event->attr.config)) {
1080 val = mfspr(SPRN_IC);
1081 if (val > cpuhw->ic_init)
1082 val = val - cpuhw->ic_init;
1083 else
1084 val = val + (0 - cpuhw->ic_init);
1085 }
1086 delta = check_and_compute_delta(prev, val);
1087 if (!delta)
1088 return;
1089 } while (local64_cmpxchg(&event->hw.prev_count, prev, val) != prev);
1090
1091 local64_add(delta, &event->count);
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102 do {
1103 prev = local64_read(&event->hw.period_left);
1104 val = prev - delta;
1105 if (val < 1)
1106 val = 1;
1107 } while (local64_cmpxchg(&event->hw.period_left, prev, val) != prev);
1108}
1109
1110
1111
1112
1113
1114
1115static int is_limited_pmc(int pmcnum)
1116{
1117 return (ppmu->flags & PPMU_LIMITED_PMC5_6)
1118 && (pmcnum == 5 || pmcnum == 6);
1119}
1120
1121static void freeze_limited_counters(struct cpu_hw_events *cpuhw,
1122 unsigned long pmc5, unsigned long pmc6)
1123{
1124 struct perf_event *event;
1125 u64 val, prev, delta;
1126 int i;
1127
1128 for (i = 0; i < cpuhw->n_limited; ++i) {
1129 event = cpuhw->limited_counter[i];
1130 if (!event->hw.idx)
1131 continue;
1132 val = (event->hw.idx == 5) ? pmc5 : pmc6;
1133 prev = local64_read(&event->hw.prev_count);
1134 event->hw.idx = 0;
1135 delta = check_and_compute_delta(prev, val);
1136 if (delta)
1137 local64_add(delta, &event->count);
1138 }
1139}
1140
1141static void thaw_limited_counters(struct cpu_hw_events *cpuhw,
1142 unsigned long pmc5, unsigned long pmc6)
1143{
1144 struct perf_event *event;
1145 u64 val, prev;
1146 int i;
1147
1148 for (i = 0; i < cpuhw->n_limited; ++i) {
1149 event = cpuhw->limited_counter[i];
1150 event->hw.idx = cpuhw->limited_hwidx[i];
1151 val = (event->hw.idx == 5) ? pmc5 : pmc6;
1152 prev = local64_read(&event->hw.prev_count);
1153 if (check_and_compute_delta(prev, val))
1154 local64_set(&event->hw.prev_count, val);
1155 perf_event_update_userpage(event);
1156 }
1157}
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170static void write_mmcr0(struct cpu_hw_events *cpuhw, unsigned long mmcr0)
1171{
1172 unsigned long pmc5, pmc6;
1173
1174 if (!cpuhw->n_limited) {
1175 mtspr(SPRN_MMCR0, mmcr0);
1176 return;
1177 }
1178
1179
1180
1181
1182
1183
1184
1185
1186 asm volatile("mtspr %3,%2; mfspr %0,%4; mfspr %1,%5"
1187 : "=&r" (pmc5), "=&r" (pmc6)
1188 : "r" (mmcr0 & ~(MMCR0_PMC1CE | MMCR0_PMCjCE)),
1189 "i" (SPRN_MMCR0),
1190 "i" (SPRN_PMC5), "i" (SPRN_PMC6));
1191
1192 if (mmcr0 & MMCR0_FC)
1193 freeze_limited_counters(cpuhw, pmc5, pmc6);
1194 else
1195 thaw_limited_counters(cpuhw, pmc5, pmc6);
1196
1197
1198
1199
1200
1201 if (mmcr0 & (MMCR0_PMC1CE | MMCR0_PMCjCE))
1202 mtspr(SPRN_MMCR0, mmcr0);
1203}
1204
1205
1206
1207
1208
1209static void power_pmu_disable(struct pmu *pmu)
1210{
1211 struct cpu_hw_events *cpuhw;
1212 unsigned long flags, mmcr0, val;
1213
1214 if (!ppmu)
1215 return;
1216 local_irq_save(flags);
1217 cpuhw = this_cpu_ptr(&cpu_hw_events);
1218
1219 if (!cpuhw->disabled) {
1220
1221
1222
1223 if (!cpuhw->pmcs_enabled) {
1224 ppc_enable_pmcs();
1225 cpuhw->pmcs_enabled = 1;
1226 }
1227
1228
1229
1230
1231 val = mmcr0 = mfspr(SPRN_MMCR0);
1232 val |= MMCR0_FC;
1233 val &= ~(MMCR0_EBE | MMCR0_BHRBA | MMCR0_PMCC | MMCR0_PMAO |
1234 MMCR0_FC56);
1235
1236
1237
1238
1239
1240
1241 write_mmcr0(cpuhw, val);
1242 mb();
1243 isync();
1244
1245
1246
1247
1248 if (cpuhw->mmcr[2] & MMCRA_SAMPLE_ENABLE) {
1249 mtspr(SPRN_MMCRA,
1250 cpuhw->mmcr[2] & ~MMCRA_SAMPLE_ENABLE);
1251 mb();
1252 isync();
1253 }
1254
1255 cpuhw->disabled = 1;
1256 cpuhw->n_added = 0;
1257
1258 ebb_switch_out(mmcr0);
1259
1260#ifdef CONFIG_PPC64
1261
1262
1263
1264
1265
1266
1267 if (ppmu->flags & PPMU_ARCH_207S) {
1268 mtspr(SPRN_SDAR, 0);
1269 mtspr(SPRN_SIAR, 0);
1270 }
1271#endif
1272 }
1273
1274 local_irq_restore(flags);
1275}
1276
1277
1278
1279
1280
1281
1282static void power_pmu_enable(struct pmu *pmu)
1283{
1284 struct perf_event *event;
1285 struct cpu_hw_events *cpuhw;
1286 unsigned long flags;
1287 long i;
1288 unsigned long val, mmcr0;
1289 s64 left;
1290 unsigned int hwc_index[MAX_HWEVENTS];
1291 int n_lim;
1292 int idx;
1293 bool ebb;
1294
1295 if (!ppmu)
1296 return;
1297 local_irq_save(flags);
1298
1299 cpuhw = this_cpu_ptr(&cpu_hw_events);
1300 if (!cpuhw->disabled)
1301 goto out;
1302
1303 if (cpuhw->n_events == 0) {
1304 ppc_set_pmu_inuse(0);
1305 goto out;
1306 }
1307
1308 cpuhw->disabled = 0;
1309
1310
1311
1312
1313
1314
1315 ebb = is_ebb_event(cpuhw->event[0]);
1316
1317
1318
1319
1320
1321
1322
1323 if (!cpuhw->n_added) {
1324 mtspr(SPRN_MMCRA, cpuhw->mmcr[2] & ~MMCRA_SAMPLE_ENABLE);
1325 mtspr(SPRN_MMCR1, cpuhw->mmcr[1]);
1326 goto out_enable;
1327 }
1328
1329
1330
1331
1332 memset(cpuhw->mmcr, 0, sizeof(cpuhw->mmcr));
1333
1334 if (ppmu->compute_mmcr(cpuhw->events, cpuhw->n_events, hwc_index,
1335 cpuhw->mmcr, cpuhw->event)) {
1336
1337 printk(KERN_ERR "oops compute_mmcr failed\n");
1338 goto out;
1339 }
1340
1341 if (!(ppmu->flags & PPMU_ARCH_207S)) {
1342
1343
1344
1345
1346
1347 event = cpuhw->event[0];
1348 if (event->attr.exclude_user)
1349 cpuhw->mmcr[0] |= MMCR0_FCP;
1350 if (event->attr.exclude_kernel)
1351 cpuhw->mmcr[0] |= freeze_events_kernel;
1352 if (event->attr.exclude_hv)
1353 cpuhw->mmcr[0] |= MMCR0_FCHV;
1354 }
1355
1356
1357
1358
1359
1360
1361 ppc_set_pmu_inuse(1);
1362 mtspr(SPRN_MMCRA, cpuhw->mmcr[2] & ~MMCRA_SAMPLE_ENABLE);
1363 mtspr(SPRN_MMCR1, cpuhw->mmcr[1]);
1364 mtspr(SPRN_MMCR0, (cpuhw->mmcr[0] & ~(MMCR0_PMC1CE | MMCR0_PMCjCE))
1365 | MMCR0_FC);
1366 if (ppmu->flags & PPMU_ARCH_207S)
1367 mtspr(SPRN_MMCR2, cpuhw->mmcr[3]);
1368
1369
1370
1371
1372
1373 for (i = 0; i < cpuhw->n_events; ++i) {
1374 event = cpuhw->event[i];
1375 if (event->hw.idx && event->hw.idx != hwc_index[i] + 1) {
1376 power_pmu_read(event);
1377 write_pmc(event->hw.idx, 0);
1378 event->hw.idx = 0;
1379 }
1380 }
1381
1382
1383
1384
1385 cpuhw->n_limited = n_lim = 0;
1386 for (i = 0; i < cpuhw->n_events; ++i) {
1387 event = cpuhw->event[i];
1388 if (event->hw.idx)
1389 continue;
1390 idx = hwc_index[i] + 1;
1391 if (is_limited_pmc(idx)) {
1392 cpuhw->limited_counter[n_lim] = event;
1393 cpuhw->limited_hwidx[n_lim] = idx;
1394 ++n_lim;
1395 continue;
1396 }
1397
1398 if (ebb)
1399 val = local64_read(&event->hw.prev_count);
1400 else {
1401 val = 0;
1402 if (event->hw.sample_period) {
1403 left = local64_read(&event->hw.period_left);
1404 if (left < 0x80000000L)
1405 val = 0x80000000L - left;
1406 }
1407 local64_set(&event->hw.prev_count, val);
1408 }
1409
1410 event->hw.idx = idx;
1411 if (event->hw.state & PERF_HES_STOPPED)
1412 val = 0;
1413 write_pmc(idx, val);
1414
1415 perf_event_update_userpage(event);
1416 }
1417 cpuhw->n_limited = n_lim;
1418 cpuhw->mmcr[0] |= MMCR0_PMXE | MMCR0_FCECE;
1419
1420 out_enable:
1421 pmao_restore_workaround(ebb);
1422
1423 mmcr0 = ebb_switch_in(ebb, cpuhw);
1424
1425 mb();
1426 if (cpuhw->bhrb_users)
1427 ppmu->config_bhrb(cpuhw->bhrb_filter);
1428
1429 write_mmcr0(cpuhw, mmcr0);
1430
1431
1432
1433
1434 if (cpuhw->mmcr[2] & MMCRA_SAMPLE_ENABLE) {
1435 mb();
1436 mtspr(SPRN_MMCRA, cpuhw->mmcr[2]);
1437 }
1438
1439 out:
1440
1441 local_irq_restore(flags);
1442}
1443
1444static int collect_events(struct perf_event *group, int max_count,
1445 struct perf_event *ctrs[], u64 *events,
1446 unsigned int *flags)
1447{
1448 int n = 0;
1449 struct perf_event *event;
1450
1451 if (group->pmu->task_ctx_nr == perf_hw_context) {
1452 if (n >= max_count)
1453 return -1;
1454 ctrs[n] = group;
1455 flags[n] = group->hw.event_base;
1456 events[n++] = group->hw.config;
1457 }
1458 for_each_sibling_event(event, group) {
1459 if (event->pmu->task_ctx_nr == perf_hw_context &&
1460 event->state != PERF_EVENT_STATE_OFF) {
1461 if (n >= max_count)
1462 return -1;
1463 ctrs[n] = event;
1464 flags[n] = event->hw.event_base;
1465 events[n++] = event->hw.config;
1466 }
1467 }
1468 return n;
1469}
1470
1471
1472
1473
1474
1475
1476
1477static int power_pmu_add(struct perf_event *event, int ef_flags)
1478{
1479 struct cpu_hw_events *cpuhw;
1480 unsigned long flags;
1481 int n0;
1482 int ret = -EAGAIN;
1483
1484 local_irq_save(flags);
1485 perf_pmu_disable(event->pmu);
1486
1487
1488
1489
1490
1491 cpuhw = this_cpu_ptr(&cpu_hw_events);
1492 n0 = cpuhw->n_events;
1493 if (n0 >= ppmu->n_counter)
1494 goto out;
1495 cpuhw->event[n0] = event;
1496 cpuhw->events[n0] = event->hw.config;
1497 cpuhw->flags[n0] = event->hw.event_base;
1498
1499
1500
1501
1502
1503
1504
1505 if (!(ef_flags & PERF_EF_START))
1506 event->hw.state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
1507 else
1508 event->hw.state = 0;
1509
1510
1511
1512
1513
1514
1515 if (cpuhw->txn_flags & PERF_PMU_TXN_ADD)
1516 goto nocheck;
1517
1518 if (check_excludes(cpuhw->event, cpuhw->flags, n0, 1))
1519 goto out;
1520 if (power_check_constraints(cpuhw, cpuhw->events, cpuhw->flags, n0 + 1))
1521 goto out;
1522 event->hw.config = cpuhw->events[n0];
1523
1524nocheck:
1525 ebb_event_add(event);
1526
1527 ++cpuhw->n_events;
1528 ++cpuhw->n_added;
1529
1530 ret = 0;
1531 out:
1532 if (has_branch_stack(event)) {
1533 power_pmu_bhrb_enable(event);
1534 cpuhw->bhrb_filter = ppmu->bhrb_filter_map(
1535 event->attr.branch_sample_type);
1536 }
1537
1538
1539
1540
1541
1542 if (use_ic(event->attr.config))
1543 cpuhw->ic_init = mfspr(SPRN_IC);
1544
1545 perf_pmu_enable(event->pmu);
1546 local_irq_restore(flags);
1547 return ret;
1548}
1549
1550
1551
1552
1553static void power_pmu_del(struct perf_event *event, int ef_flags)
1554{
1555 struct cpu_hw_events *cpuhw;
1556 long i;
1557 unsigned long flags;
1558
1559 local_irq_save(flags);
1560 perf_pmu_disable(event->pmu);
1561
1562 power_pmu_read(event);
1563
1564 cpuhw = this_cpu_ptr(&cpu_hw_events);
1565 for (i = 0; i < cpuhw->n_events; ++i) {
1566 if (event == cpuhw->event[i]) {
1567 while (++i < cpuhw->n_events) {
1568 cpuhw->event[i-1] = cpuhw->event[i];
1569 cpuhw->events[i-1] = cpuhw->events[i];
1570 cpuhw->flags[i-1] = cpuhw->flags[i];
1571 }
1572 --cpuhw->n_events;
1573 ppmu->disable_pmc(event->hw.idx - 1, cpuhw->mmcr);
1574 if (event->hw.idx) {
1575 write_pmc(event->hw.idx, 0);
1576 event->hw.idx = 0;
1577 }
1578 perf_event_update_userpage(event);
1579 break;
1580 }
1581 }
1582 for (i = 0; i < cpuhw->n_limited; ++i)
1583 if (event == cpuhw->limited_counter[i])
1584 break;
1585 if (i < cpuhw->n_limited) {
1586 while (++i < cpuhw->n_limited) {
1587 cpuhw->limited_counter[i-1] = cpuhw->limited_counter[i];
1588 cpuhw->limited_hwidx[i-1] = cpuhw->limited_hwidx[i];
1589 }
1590 --cpuhw->n_limited;
1591 }
1592 if (cpuhw->n_events == 0) {
1593
1594 cpuhw->mmcr[0] &= ~(MMCR0_PMXE | MMCR0_FCECE);
1595 }
1596
1597 if (has_branch_stack(event))
1598 power_pmu_bhrb_disable(event);
1599
1600 perf_pmu_enable(event->pmu);
1601 local_irq_restore(flags);
1602}
1603
1604
1605
1606
1607
1608
1609static void power_pmu_start(struct perf_event *event, int ef_flags)
1610{
1611 unsigned long flags;
1612 s64 left;
1613 unsigned long val;
1614
1615 if (!event->hw.idx || !event->hw.sample_period)
1616 return;
1617
1618 if (!(event->hw.state & PERF_HES_STOPPED))
1619 return;
1620
1621 if (ef_flags & PERF_EF_RELOAD)
1622 WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE));
1623
1624 local_irq_save(flags);
1625 perf_pmu_disable(event->pmu);
1626
1627 event->hw.state = 0;
1628 left = local64_read(&event->hw.period_left);
1629
1630 val = 0;
1631 if (left < 0x80000000L)
1632 val = 0x80000000L - left;
1633
1634 write_pmc(event->hw.idx, val);
1635
1636 perf_event_update_userpage(event);
1637 perf_pmu_enable(event->pmu);
1638 local_irq_restore(flags);
1639}
1640
1641static void power_pmu_stop(struct perf_event *event, int ef_flags)
1642{
1643 unsigned long flags;
1644
1645 if (!event->hw.idx || !event->hw.sample_period)
1646 return;
1647
1648 if (event->hw.state & PERF_HES_STOPPED)
1649 return;
1650
1651 local_irq_save(flags);
1652 perf_pmu_disable(event->pmu);
1653
1654 power_pmu_read(event);
1655 event->hw.state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
1656 write_pmc(event->hw.idx, 0);
1657
1658 perf_event_update_userpage(event);
1659 perf_pmu_enable(event->pmu);
1660 local_irq_restore(flags);
1661}
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672static void power_pmu_start_txn(struct pmu *pmu, unsigned int txn_flags)
1673{
1674 struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events);
1675
1676 WARN_ON_ONCE(cpuhw->txn_flags);
1677
1678 cpuhw->txn_flags = txn_flags;
1679 if (txn_flags & ~PERF_PMU_TXN_ADD)
1680 return;
1681
1682 perf_pmu_disable(pmu);
1683 cpuhw->n_txn_start = cpuhw->n_events;
1684}
1685
1686
1687
1688
1689
1690
1691static void power_pmu_cancel_txn(struct pmu *pmu)
1692{
1693 struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events);
1694 unsigned int txn_flags;
1695
1696 WARN_ON_ONCE(!cpuhw->txn_flags);
1697
1698 txn_flags = cpuhw->txn_flags;
1699 cpuhw->txn_flags = 0;
1700 if (txn_flags & ~PERF_PMU_TXN_ADD)
1701 return;
1702
1703 perf_pmu_enable(pmu);
1704}
1705
1706
1707
1708
1709
1710
1711static int power_pmu_commit_txn(struct pmu *pmu)
1712{
1713 struct cpu_hw_events *cpuhw;
1714 long i, n;
1715
1716 if (!ppmu)
1717 return -EAGAIN;
1718
1719 cpuhw = this_cpu_ptr(&cpu_hw_events);
1720 WARN_ON_ONCE(!cpuhw->txn_flags);
1721
1722 if (cpuhw->txn_flags & ~PERF_PMU_TXN_ADD) {
1723 cpuhw->txn_flags = 0;
1724 return 0;
1725 }
1726
1727 n = cpuhw->n_events;
1728 if (check_excludes(cpuhw->event, cpuhw->flags, 0, n))
1729 return -EAGAIN;
1730 i = power_check_constraints(cpuhw, cpuhw->events, cpuhw->flags, n);
1731 if (i < 0)
1732 return -EAGAIN;
1733
1734 for (i = cpuhw->n_txn_start; i < n; ++i)
1735 cpuhw->event[i]->hw.config = cpuhw->events[i];
1736
1737 cpuhw->txn_flags = 0;
1738 perf_pmu_enable(pmu);
1739 return 0;
1740}
1741
1742
1743
1744
1745
1746
1747
1748
1749static int can_go_on_limited_pmc(struct perf_event *event, u64 ev,
1750 unsigned int flags)
1751{
1752 int n;
1753 u64 alt[MAX_EVENT_ALTERNATIVES];
1754
1755 if (event->attr.exclude_user
1756 || event->attr.exclude_kernel
1757 || event->attr.exclude_hv
1758 || event->attr.sample_period)
1759 return 0;
1760
1761 if (ppmu->limited_pmc_event(ev))
1762 return 1;
1763
1764
1765
1766
1767
1768 if (!ppmu->get_alternatives)
1769 return 0;
1770
1771 flags |= PPMU_LIMITED_PMC_OK | PPMU_LIMITED_PMC_REQD;
1772 n = ppmu->get_alternatives(ev, flags, alt);
1773
1774 return n > 0;
1775}
1776
1777
1778
1779
1780
1781
1782static u64 normal_pmc_alternative(u64 ev, unsigned long flags)
1783{
1784 u64 alt[MAX_EVENT_ALTERNATIVES];
1785 int n;
1786
1787 flags &= ~(PPMU_LIMITED_PMC_OK | PPMU_LIMITED_PMC_REQD);
1788 n = ppmu->get_alternatives(ev, flags, alt);
1789 if (!n)
1790 return 0;
1791 return alt[0];
1792}
1793
1794
1795static atomic_t num_events;
1796
1797static DEFINE_MUTEX(pmc_reserve_mutex);
1798
1799
1800
1801
1802static void hw_perf_event_destroy(struct perf_event *event)
1803{
1804 if (!atomic_add_unless(&num_events, -1, 1)) {
1805 mutex_lock(&pmc_reserve_mutex);
1806 if (atomic_dec_return(&num_events) == 0)
1807 release_pmc_hardware();
1808 mutex_unlock(&pmc_reserve_mutex);
1809 }
1810}
1811
1812
1813
1814
1815static int hw_perf_cache_event(u64 config, u64 *eventp)
1816{
1817 unsigned long type, op, result;
1818 int ev;
1819
1820 if (!ppmu->cache_events)
1821 return -EINVAL;
1822
1823
1824 type = config & 0xff;
1825 op = (config >> 8) & 0xff;
1826 result = (config >> 16) & 0xff;
1827
1828 if (type >= PERF_COUNT_HW_CACHE_MAX ||
1829 op >= PERF_COUNT_HW_CACHE_OP_MAX ||
1830 result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
1831 return -EINVAL;
1832
1833 ev = (*ppmu->cache_events)[type][op][result];
1834 if (ev == 0)
1835 return -EOPNOTSUPP;
1836 if (ev == -1)
1837 return -EINVAL;
1838 *eventp = ev;
1839 return 0;
1840}
1841
1842static bool is_event_blacklisted(u64 ev)
1843{
1844 int i;
1845
1846 for (i=0; i < ppmu->n_blacklist_ev; i++) {
1847 if (ppmu->blacklist_ev[i] == ev)
1848 return true;
1849 }
1850
1851 return false;
1852}
1853
1854static int power_pmu_event_init(struct perf_event *event)
1855{
1856 u64 ev;
1857 unsigned long flags;
1858 struct perf_event *ctrs[MAX_HWEVENTS];
1859 u64 events[MAX_HWEVENTS];
1860 unsigned int cflags[MAX_HWEVENTS];
1861 int n;
1862 int err;
1863 struct cpu_hw_events *cpuhw;
1864
1865 if (!ppmu)
1866 return -ENOENT;
1867
1868 if (has_branch_stack(event)) {
1869
1870 if (!(ppmu->flags & PPMU_ARCH_207S))
1871 return -EOPNOTSUPP;
1872 }
1873
1874 switch (event->attr.type) {
1875 case PERF_TYPE_HARDWARE:
1876 ev = event->attr.config;
1877 if (ev >= ppmu->n_generic || ppmu->generic_events[ev] == 0)
1878 return -EOPNOTSUPP;
1879
1880 if (ppmu->blacklist_ev && is_event_blacklisted(ev))
1881 return -EINVAL;
1882 ev = ppmu->generic_events[ev];
1883 break;
1884 case PERF_TYPE_HW_CACHE:
1885 err = hw_perf_cache_event(event->attr.config, &ev);
1886 if (err)
1887 return err;
1888
1889 if (ppmu->blacklist_ev && is_event_blacklisted(ev))
1890 return -EINVAL;
1891 break;
1892 case PERF_TYPE_RAW:
1893 ev = event->attr.config;
1894
1895 if (ppmu->blacklist_ev && is_event_blacklisted(ev))
1896 return -EINVAL;
1897 break;
1898 default:
1899 return -ENOENT;
1900 }
1901
1902 event->hw.config_base = ev;
1903 event->hw.idx = 0;
1904
1905
1906
1907
1908
1909
1910 if (!firmware_has_feature(FW_FEATURE_LPAR))
1911 event->attr.exclude_hv = 0;
1912
1913
1914
1915
1916
1917
1918
1919 flags = 0;
1920 if (event->attach_state & PERF_ATTACH_TASK)
1921 flags |= PPMU_ONLY_COUNT_RUN;
1922
1923
1924
1925
1926
1927 if (ppmu->flags & PPMU_LIMITED_PMC5_6) {
1928 if (can_go_on_limited_pmc(event, ev, flags)) {
1929 flags |= PPMU_LIMITED_PMC_OK;
1930 } else if (ppmu->limited_pmc_event(ev)) {
1931
1932
1933
1934
1935
1936 ev = normal_pmc_alternative(ev, flags);
1937 if (!ev)
1938 return -EINVAL;
1939 }
1940 }
1941
1942
1943 err = ebb_event_check(event);
1944 if (err)
1945 return err;
1946
1947
1948
1949
1950
1951
1952 n = 0;
1953 if (event->group_leader != event) {
1954 n = collect_events(event->group_leader, ppmu->n_counter - 1,
1955 ctrs, events, cflags);
1956 if (n < 0)
1957 return -EINVAL;
1958 }
1959 events[n] = ev;
1960 ctrs[n] = event;
1961 cflags[n] = flags;
1962 if (check_excludes(ctrs, cflags, n, 1))
1963 return -EINVAL;
1964
1965 cpuhw = &get_cpu_var(cpu_hw_events);
1966 err = power_check_constraints(cpuhw, events, cflags, n + 1);
1967
1968 if (has_branch_stack(event)) {
1969 cpuhw->bhrb_filter = ppmu->bhrb_filter_map(
1970 event->attr.branch_sample_type);
1971
1972 if (cpuhw->bhrb_filter == -1) {
1973 put_cpu_var(cpu_hw_events);
1974 return -EOPNOTSUPP;
1975 }
1976 }
1977
1978 put_cpu_var(cpu_hw_events);
1979 if (err)
1980 return -EINVAL;
1981
1982 event->hw.config = events[n];
1983 event->hw.event_base = cflags[n];
1984 event->hw.last_period = event->hw.sample_period;
1985 local64_set(&event->hw.period_left, event->hw.last_period);
1986
1987
1988
1989
1990
1991 if (is_ebb_event(event))
1992 local64_set(&event->hw.prev_count, 0);
1993
1994
1995
1996
1997
1998
1999
2000 err = 0;
2001 if (!atomic_inc_not_zero(&num_events)) {
2002 mutex_lock(&pmc_reserve_mutex);
2003 if (atomic_read(&num_events) == 0 &&
2004 reserve_pmc_hardware(perf_event_interrupt))
2005 err = -EBUSY;
2006 else
2007 atomic_inc(&num_events);
2008 mutex_unlock(&pmc_reserve_mutex);
2009 }
2010 event->destroy = hw_perf_event_destroy;
2011
2012 return err;
2013}
2014
2015static int power_pmu_event_idx(struct perf_event *event)
2016{
2017 return event->hw.idx;
2018}
2019
2020ssize_t power_events_sysfs_show(struct device *dev,
2021 struct device_attribute *attr, char *page)
2022{
2023 struct perf_pmu_events_attr *pmu_attr;
2024
2025 pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr);
2026
2027 return sprintf(page, "event=0x%02llx\n", pmu_attr->id);
2028}
2029
2030static struct pmu power_pmu = {
2031 .pmu_enable = power_pmu_enable,
2032 .pmu_disable = power_pmu_disable,
2033 .event_init = power_pmu_event_init,
2034 .add = power_pmu_add,
2035 .del = power_pmu_del,
2036 .start = power_pmu_start,
2037 .stop = power_pmu_stop,
2038 .read = power_pmu_read,
2039 .start_txn = power_pmu_start_txn,
2040 .cancel_txn = power_pmu_cancel_txn,
2041 .commit_txn = power_pmu_commit_txn,
2042 .event_idx = power_pmu_event_idx,
2043 .sched_task = power_pmu_sched_task,
2044};
2045
2046
2047
2048
2049
2050
2051static void record_and_restart(struct perf_event *event, unsigned long val,
2052 struct pt_regs *regs)
2053{
2054 u64 period = event->hw.sample_period;
2055 s64 prev, delta, left;
2056 int record = 0;
2057
2058 if (event->hw.state & PERF_HES_STOPPED) {
2059 write_pmc(event->hw.idx, 0);
2060 return;
2061 }
2062
2063
2064 prev = local64_read(&event->hw.prev_count);
2065 delta = check_and_compute_delta(prev, val);
2066 local64_add(delta, &event->count);
2067
2068
2069
2070
2071
2072 val = 0;
2073 left = local64_read(&event->hw.period_left) - delta;
2074 if (delta == 0)
2075 left++;
2076 if (period) {
2077 if (left <= 0) {
2078 left += period;
2079 if (left <= 0)
2080 left = period;
2081 record = siar_valid(regs);
2082 event->hw.last_period = event->hw.sample_period;
2083 }
2084 if (left < 0x80000000LL)
2085 val = 0x80000000LL - left;
2086 }
2087
2088 write_pmc(event->hw.idx, val);
2089 local64_set(&event->hw.prev_count, val);
2090 local64_set(&event->hw.period_left, left);
2091 perf_event_update_userpage(event);
2092
2093
2094
2095
2096 if (record) {
2097 struct perf_sample_data data;
2098
2099 perf_sample_data_init(&data, ~0ULL, event->hw.last_period);
2100
2101 if (event->attr.sample_type &
2102 (PERF_SAMPLE_ADDR | PERF_SAMPLE_PHYS_ADDR))
2103 perf_get_data_addr(regs, &data.addr);
2104
2105 if (event->attr.sample_type & PERF_SAMPLE_BRANCH_STACK) {
2106 struct cpu_hw_events *cpuhw;
2107 cpuhw = this_cpu_ptr(&cpu_hw_events);
2108 power_pmu_bhrb_read(cpuhw);
2109 data.br_stack = &cpuhw->bhrb_stack;
2110 }
2111
2112 if (event->attr.sample_type & PERF_SAMPLE_DATA_SRC &&
2113 ppmu->get_mem_data_src)
2114 ppmu->get_mem_data_src(&data.data_src, ppmu->flags, regs);
2115
2116 if (event->attr.sample_type & PERF_SAMPLE_WEIGHT &&
2117 ppmu->get_mem_weight)
2118 ppmu->get_mem_weight(&data.weight);
2119
2120 if (perf_event_overflow(event, &data, regs))
2121 power_pmu_stop(event, 0);
2122 }
2123}
2124
2125
2126
2127
2128
2129unsigned long perf_misc_flags(struct pt_regs *regs)
2130{
2131 u32 flags = perf_get_misc_flags(regs);
2132
2133 if (flags)
2134 return flags;
2135 return user_mode(regs) ? PERF_RECORD_MISC_USER :
2136 PERF_RECORD_MISC_KERNEL;
2137}
2138
2139
2140
2141
2142
2143unsigned long perf_instruction_pointer(struct pt_regs *regs)
2144{
2145 bool use_siar = regs_use_siar(regs);
2146
2147 if (use_siar && siar_valid(regs))
2148 return mfspr(SPRN_SIAR) + perf_ip_adjust(regs);
2149 else if (use_siar)
2150 return 0;
2151 else
2152 return regs->nip;
2153}
2154
2155static bool pmc_overflow_power7(unsigned long val)
2156{
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168 if ((0x80000000 - val) <= 256)
2169 return true;
2170
2171 return false;
2172}
2173
2174static bool pmc_overflow(unsigned long val)
2175{
2176 if ((int)val < 0)
2177 return true;
2178
2179 return false;
2180}
2181
2182
2183
2184
2185static void perf_event_interrupt(struct pt_regs *regs)
2186{
2187 int i, j;
2188 struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events);
2189 struct perf_event *event;
2190 unsigned long val[8];
2191 int found, active;
2192 int nmi;
2193
2194 if (cpuhw->n_limited)
2195 freeze_limited_counters(cpuhw, mfspr(SPRN_PMC5),
2196 mfspr(SPRN_PMC6));
2197
2198 perf_read_regs(regs);
2199
2200 nmi = perf_intr_is_nmi(regs);
2201 if (nmi)
2202 nmi_enter();
2203 else
2204 irq_enter();
2205
2206
2207 for (i = 0; i < ppmu->n_counter; ++i)
2208 val[i] = read_pmc(i + 1);
2209
2210
2211 found = 0;
2212 for (i = 0; i < ppmu->n_counter; ++i) {
2213 if (!pmc_overflow(val[i]))
2214 continue;
2215 if (is_limited_pmc(i + 1))
2216 continue;
2217
2218
2219
2220
2221
2222 found = 1;
2223 active = 0;
2224 for (j = 0; j < cpuhw->n_events; ++j) {
2225 event = cpuhw->event[j];
2226 if (event->hw.idx == (i + 1)) {
2227 active = 1;
2228 record_and_restart(event, val[i], regs);
2229 break;
2230 }
2231 }
2232 if (!active)
2233
2234 write_pmc(i + 1, 0);
2235 }
2236 if (!found && pvr_version_is(PVR_POWER7)) {
2237
2238 for (i = 0; i < cpuhw->n_events; ++i) {
2239 event = cpuhw->event[i];
2240 if (!event->hw.idx || is_limited_pmc(event->hw.idx))
2241 continue;
2242 if (pmc_overflow_power7(val[event->hw.idx - 1])) {
2243
2244 found = 1;
2245 record_and_restart(event,
2246 val[event->hw.idx - 1],
2247 regs);
2248 }
2249 }
2250 }
2251 if (!found && !nmi && printk_ratelimit())
2252 printk(KERN_WARNING "Can't find PMC that caused IRQ\n");
2253
2254
2255
2256
2257
2258
2259
2260
2261 write_mmcr0(cpuhw, cpuhw->mmcr[0]);
2262
2263 if (nmi)
2264 nmi_exit();
2265 else
2266 irq_exit();
2267}
2268
2269static int power_pmu_prepare_cpu(unsigned int cpu)
2270{
2271 struct cpu_hw_events *cpuhw = &per_cpu(cpu_hw_events, cpu);
2272
2273 if (ppmu) {
2274 memset(cpuhw, 0, sizeof(*cpuhw));
2275 cpuhw->mmcr[0] = MMCR0_FC;
2276 }
2277 return 0;
2278}
2279
2280int register_power_pmu(struct power_pmu *pmu)
2281{
2282 if (ppmu)
2283 return -EBUSY;
2284
2285 ppmu = pmu;
2286 pr_info("%s performance monitor hardware support registered\n",
2287 pmu->name);
2288
2289 power_pmu.attr_groups = ppmu->attr_groups;
2290
2291#ifdef MSR_HV
2292
2293
2294
2295 if (mfmsr() & MSR_HV)
2296 freeze_events_kernel = MMCR0_FCHV;
2297#endif
2298
2299 perf_pmu_register(&power_pmu, "cpu", PERF_TYPE_RAW);
2300 cpuhp_setup_state(CPUHP_PERF_POWER, "perf/powerpc:prepare",
2301 power_pmu_prepare_cpu, NULL);
2302 return 0;
2303}
2304