1
2
3
4
5
6
7
8
9
10
11
12#include <linux/perf_event.h>
13#include <linux/kprobes.h>
14#include <linux/kernel.h>
15#include <linux/kdebug.h>
16#include <linux/mutex.h>
17#include <linux/init.h>
18
19#include <asm/hwrpb.h>
20#include <linux/atomic.h>
21#include <asm/irq.h>
22#include <asm/irq_regs.h>
23#include <asm/pal.h>
24#include <asm/wrperfmon.h>
25#include <asm/hw_irq.h>
26
27
28
29#define MAX_HWEVENTS 3
30#define PMC_NO_INDEX -1
31
32
33struct cpu_hw_events {
34 int enabled;
35
36 int n_events;
37
38 int n_added;
39
40 struct perf_event *event[MAX_HWEVENTS];
41
42 unsigned long evtype[MAX_HWEVENTS];
43
44
45
46 int current_idx[MAX_HWEVENTS];
47
48 unsigned long config;
49
50 unsigned long idx_mask;
51};
52DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events);
53
54
55
56
57
58
59
60struct alpha_pmu_t {
61
62 const int *event_map;
63
64 int max_events;
65
66 int num_pmcs;
67
68
69
70
71 int pmc_count_shift[MAX_HWEVENTS];
72
73
74
75
76 unsigned long pmc_count_mask[MAX_HWEVENTS];
77
78 unsigned long pmc_max_period[MAX_HWEVENTS];
79
80
81
82
83 long pmc_left[3];
84
85 int (*check_constraints)(struct perf_event **, unsigned long *, int);
86
87 int (*raw_event_valid)(u64 config);
88};
89
90
91
92
93
94static const struct alpha_pmu_t *alpha_pmu;
95
96
97#define HW_OP_UNSUPPORTED -1
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112enum ev67_pmc_event_type {
113 EV67_CYCLES = 1,
114 EV67_INSTRUCTIONS,
115 EV67_BCACHEMISS,
116 EV67_MBOXREPLAY,
117 EV67_LAST_ET
118};
119#define EV67_NUM_EVENT_TYPES (EV67_LAST_ET-EV67_CYCLES)
120
121
122
123static const int ev67_perfmon_event_map[] = {
124 [PERF_COUNT_HW_CPU_CYCLES] = EV67_CYCLES,
125 [PERF_COUNT_HW_INSTRUCTIONS] = EV67_INSTRUCTIONS,
126 [PERF_COUNT_HW_CACHE_REFERENCES] = HW_OP_UNSUPPORTED,
127 [PERF_COUNT_HW_CACHE_MISSES] = EV67_BCACHEMISS,
128};
129
130struct ev67_mapping_t {
131 int config;
132 int idx;
133};
134
135
136
137
138
139static const struct ev67_mapping_t ev67_mapping[] = {
140 {EV67_PCTR_INSTR_CYCLES, 1},
141 {EV67_PCTR_INSTR_CYCLES, 0},
142 {EV67_PCTR_INSTR_BCACHEMISS, 1},
143 {EV67_PCTR_CYCLES_MBOX, 1}
144};
145
146
147
148
149
150
151static int ev67_check_constraints(struct perf_event **event,
152 unsigned long *evtype, int n_ev)
153{
154 int idx0;
155 unsigned long config;
156
157 idx0 = ev67_mapping[evtype[0]-1].idx;
158 config = ev67_mapping[evtype[0]-1].config;
159 if (n_ev == 1)
160 goto success;
161
162 BUG_ON(n_ev != 2);
163
164 if (evtype[0] == EV67_MBOXREPLAY || evtype[1] == EV67_MBOXREPLAY) {
165
166 idx0 = (evtype[0] == EV67_MBOXREPLAY) ? 1 : 0;
167
168 if (evtype[idx0] == EV67_CYCLES) {
169 config = EV67_PCTR_CYCLES_MBOX;
170 goto success;
171 }
172 }
173
174 if (evtype[0] == EV67_BCACHEMISS || evtype[1] == EV67_BCACHEMISS) {
175
176 idx0 = (evtype[0] == EV67_BCACHEMISS) ? 1 : 0;
177
178 if (evtype[idx0] == EV67_INSTRUCTIONS) {
179 config = EV67_PCTR_INSTR_BCACHEMISS;
180 goto success;
181 }
182 }
183
184 if (evtype[0] == EV67_INSTRUCTIONS || evtype[1] == EV67_INSTRUCTIONS) {
185
186 idx0 = (evtype[0] == EV67_INSTRUCTIONS) ? 0 : 1;
187
188 if (evtype[idx0^1] == EV67_CYCLES) {
189 config = EV67_PCTR_INSTR_CYCLES;
190 goto success;
191 }
192 }
193
194
195 return -1;
196
197success:
198 event[0]->hw.idx = idx0;
199 event[0]->hw.config_base = config;
200 if (n_ev == 2) {
201 event[1]->hw.idx = idx0 ^ 1;
202 event[1]->hw.config_base = config;
203 }
204 return 0;
205}
206
207
208static int ev67_raw_event_valid(u64 config)
209{
210 return config >= EV67_CYCLES && config < EV67_LAST_ET;
211};
212
213
214static const struct alpha_pmu_t ev67_pmu = {
215 .event_map = ev67_perfmon_event_map,
216 .max_events = ARRAY_SIZE(ev67_perfmon_event_map),
217 .num_pmcs = 2,
218 .pmc_count_shift = {EV67_PCTR_0_COUNT_SHIFT, EV67_PCTR_1_COUNT_SHIFT, 0},
219 .pmc_count_mask = {EV67_PCTR_0_COUNT_MASK, EV67_PCTR_1_COUNT_MASK, 0},
220 .pmc_max_period = {(1UL<<20) - 1, (1UL<<20) - 1, 0},
221 .pmc_left = {16, 4, 0},
222 .check_constraints = ev67_check_constraints,
223 .raw_event_valid = ev67_raw_event_valid,
224};
225
226
227
228
229
230
231
232static inline void alpha_write_pmc(int idx, unsigned long val)
233{
234 val &= alpha_pmu->pmc_count_mask[idx];
235 val <<= alpha_pmu->pmc_count_shift[idx];
236 val |= (1<<idx);
237 wrperfmon(PERFMON_CMD_WRITE, val);
238}
239
240static inline unsigned long alpha_read_pmc(int idx)
241{
242 unsigned long val;
243
244 val = wrperfmon(PERFMON_CMD_READ, 0);
245 val >>= alpha_pmu->pmc_count_shift[idx];
246 val &= alpha_pmu->pmc_count_mask[idx];
247 return val;
248}
249
250
251static int alpha_perf_event_set_period(struct perf_event *event,
252 struct hw_perf_event *hwc, int idx)
253{
254 long left = local64_read(&hwc->period_left);
255 long period = hwc->sample_period;
256 int ret = 0;
257
258 if (unlikely(left <= -period)) {
259 left = period;
260 local64_set(&hwc->period_left, left);
261 hwc->last_period = period;
262 ret = 1;
263 }
264
265 if (unlikely(left <= 0)) {
266 left += period;
267 local64_set(&hwc->period_left, left);
268 hwc->last_period = period;
269 ret = 1;
270 }
271
272
273
274
275
276 if (unlikely(left < alpha_pmu->pmc_left[idx]))
277 left = alpha_pmu->pmc_left[idx];
278
279 if (left > (long)alpha_pmu->pmc_max_period[idx])
280 left = alpha_pmu->pmc_max_period[idx];
281
282 local64_set(&hwc->prev_count, (unsigned long)(-left));
283
284 alpha_write_pmc(idx, (unsigned long)(-left));
285
286 perf_event_update_userpage(event);
287
288 return ret;
289}
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306static unsigned long alpha_perf_event_update(struct perf_event *event,
307 struct hw_perf_event *hwc, int idx, long ovf)
308{
309 long prev_raw_count, new_raw_count;
310 long delta;
311
312again:
313 prev_raw_count = local64_read(&hwc->prev_count);
314 new_raw_count = alpha_read_pmc(idx);
315
316 if (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
317 new_raw_count) != prev_raw_count)
318 goto again;
319
320 delta = (new_raw_count - (prev_raw_count & alpha_pmu->pmc_count_mask[idx])) + ovf;
321
322
323
324
325 if (unlikely(delta < 0)) {
326 delta += alpha_pmu->pmc_max_period[idx] + 1;
327 }
328
329 local64_add(delta, &event->count);
330 local64_sub(delta, &hwc->period_left);
331
332 return new_raw_count;
333}
334
335
336
337
338
339static int collect_events(struct perf_event *group, int max_count,
340 struct perf_event *event[], unsigned long *evtype,
341 int *current_idx)
342{
343 struct perf_event *pe;
344 int n = 0;
345
346 if (!is_software_event(group)) {
347 if (n >= max_count)
348 return -1;
349 event[n] = group;
350 evtype[n] = group->hw.event_base;
351 current_idx[n++] = PMC_NO_INDEX;
352 }
353 list_for_each_entry(pe, &group->sibling_list, group_entry) {
354 if (!is_software_event(pe) && pe->state != PERF_EVENT_STATE_OFF) {
355 if (n >= max_count)
356 return -1;
357 event[n] = pe;
358 evtype[n] = pe->hw.event_base;
359 current_idx[n++] = PMC_NO_INDEX;
360 }
361 }
362 return n;
363}
364
365
366
367
368
369
370static int alpha_check_constraints(struct perf_event **events,
371 unsigned long *evtypes, int n_ev)
372{
373
374
375 if (n_ev == 0)
376 return 0;
377
378 if (n_ev > alpha_pmu->num_pmcs)
379 return -1;
380
381 return alpha_pmu->check_constraints(events, evtypes, n_ev);
382}
383
384
385
386
387
388
389
390static void maybe_change_configuration(struct cpu_hw_events *cpuc)
391{
392 int j;
393
394 if (cpuc->n_added == 0)
395 return;
396
397
398 for (j = 0; j < cpuc->n_events; j++) {
399 struct perf_event *pe = cpuc->event[j];
400
401 if (cpuc->current_idx[j] != PMC_NO_INDEX &&
402 cpuc->current_idx[j] != pe->hw.idx) {
403 alpha_perf_event_update(pe, &pe->hw, cpuc->current_idx[j], 0);
404 cpuc->current_idx[j] = PMC_NO_INDEX;
405 }
406 }
407
408
409 cpuc->idx_mask = 0;
410 for (j = 0; j < cpuc->n_events; j++) {
411 struct perf_event *pe = cpuc->event[j];
412 struct hw_perf_event *hwc = &pe->hw;
413 int idx = hwc->idx;
414
415 if (cpuc->current_idx[j] == PMC_NO_INDEX) {
416 alpha_perf_event_set_period(pe, hwc, idx);
417 cpuc->current_idx[j] = idx;
418 }
419
420 if (!(hwc->state & PERF_HES_STOPPED))
421 cpuc->idx_mask |= (1<<cpuc->current_idx[j]);
422 }
423 cpuc->config = cpuc->event[0]->hw.config_base;
424}
425
426
427
428
429
430
431
432static int alpha_pmu_add(struct perf_event *event, int flags)
433{
434 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
435 struct hw_perf_event *hwc = &event->hw;
436 int n0;
437 int ret;
438 unsigned long irq_flags;
439
440
441
442
443
444
445
446
447
448 perf_pmu_disable(event->pmu);
449 local_irq_save(irq_flags);
450
451
452 ret = -EAGAIN;
453
454
455 n0 = cpuc->n_events;
456 if (n0 < alpha_pmu->num_pmcs) {
457 cpuc->event[n0] = event;
458 cpuc->evtype[n0] = event->hw.event_base;
459 cpuc->current_idx[n0] = PMC_NO_INDEX;
460
461 if (!alpha_check_constraints(cpuc->event, cpuc->evtype, n0+1)) {
462 cpuc->n_events++;
463 cpuc->n_added++;
464 ret = 0;
465 }
466 }
467
468 hwc->state = PERF_HES_UPTODATE;
469 if (!(flags & PERF_EF_START))
470 hwc->state |= PERF_HES_STOPPED;
471
472 local_irq_restore(irq_flags);
473 perf_pmu_enable(event->pmu);
474
475 return ret;
476}
477
478
479
480
481
482
483
484static void alpha_pmu_del(struct perf_event *event, int flags)
485{
486 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
487 struct hw_perf_event *hwc = &event->hw;
488 unsigned long irq_flags;
489 int j;
490
491 perf_pmu_disable(event->pmu);
492 local_irq_save(irq_flags);
493
494 for (j = 0; j < cpuc->n_events; j++) {
495 if (event == cpuc->event[j]) {
496 int idx = cpuc->current_idx[j];
497
498
499
500
501 while (++j < cpuc->n_events) {
502 cpuc->event[j - 1] = cpuc->event[j];
503 cpuc->evtype[j - 1] = cpuc->evtype[j];
504 cpuc->current_idx[j - 1] =
505 cpuc->current_idx[j];
506 }
507
508
509 alpha_perf_event_update(event, hwc, idx, 0);
510 perf_event_update_userpage(event);
511
512 cpuc->idx_mask &= ~(1UL<<idx);
513 cpuc->n_events--;
514 break;
515 }
516 }
517
518 local_irq_restore(irq_flags);
519 perf_pmu_enable(event->pmu);
520}
521
522
523static void alpha_pmu_read(struct perf_event *event)
524{
525 struct hw_perf_event *hwc = &event->hw;
526
527 alpha_perf_event_update(event, hwc, hwc->idx, 0);
528}
529
530
531static void alpha_pmu_stop(struct perf_event *event, int flags)
532{
533 struct hw_perf_event *hwc = &event->hw;
534 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
535
536 if (!(hwc->state & PERF_HES_STOPPED)) {
537 cpuc->idx_mask &= ~(1UL<<hwc->idx);
538 hwc->state |= PERF_HES_STOPPED;
539 }
540
541 if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) {
542 alpha_perf_event_update(event, hwc, hwc->idx, 0);
543 hwc->state |= PERF_HES_UPTODATE;
544 }
545
546 if (cpuc->enabled)
547 wrperfmon(PERFMON_CMD_DISABLE, (1UL<<hwc->idx));
548}
549
550
551static void alpha_pmu_start(struct perf_event *event, int flags)
552{
553 struct hw_perf_event *hwc = &event->hw;
554 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
555
556 if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED)))
557 return;
558
559 if (flags & PERF_EF_RELOAD) {
560 WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE));
561 alpha_perf_event_set_period(event, hwc, hwc->idx);
562 }
563
564 hwc->state = 0;
565
566 cpuc->idx_mask |= 1UL<<hwc->idx;
567 if (cpuc->enabled)
568 wrperfmon(PERFMON_CMD_ENABLE, (1UL<<hwc->idx));
569}
570
571
572
573
574
575
576
577
578
579static int supported_cpu(void)
580{
581 struct percpu_struct *cpu;
582 unsigned long cputype;
583
584
585 cpu = (struct percpu_struct *)((char *)hwrpb + hwrpb->processor_offset);
586 cputype = cpu->type & 0xffffffff;
587
588 return (cputype >= EV67_CPU) && (cputype <= EV69_CPU);
589}
590
591
592
593static void hw_perf_event_destroy(struct perf_event *event)
594{
595
596 return;
597}
598
599
600
601static int __hw_perf_event_init(struct perf_event *event)
602{
603 struct perf_event_attr *attr = &event->attr;
604 struct hw_perf_event *hwc = &event->hw;
605 struct perf_event *evts[MAX_HWEVENTS];
606 unsigned long evtypes[MAX_HWEVENTS];
607 int idx_rubbish_bin[MAX_HWEVENTS];
608 int ev;
609 int n;
610
611
612
613
614 if (attr->type == PERF_TYPE_HARDWARE) {
615 if (attr->config >= alpha_pmu->max_events)
616 return -EINVAL;
617 ev = alpha_pmu->event_map[attr->config];
618 } else if (attr->type == PERF_TYPE_HW_CACHE) {
619 return -EOPNOTSUPP;
620 } else if (attr->type == PERF_TYPE_RAW) {
621 if (!alpha_pmu->raw_event_valid(attr->config))
622 return -EINVAL;
623 ev = attr->config;
624 } else {
625 return -EOPNOTSUPP;
626 }
627
628 if (ev < 0) {
629 return ev;
630 }
631
632
633 if (attr->exclude_kernel || attr->exclude_user
634 || attr->exclude_hv || attr->exclude_idle) {
635 return -EPERM;
636 }
637
638
639
640
641
642
643
644
645
646
647 hwc->event_base = ev;
648
649
650
651
652
653 n = 0;
654 if (event->group_leader != event) {
655 n = collect_events(event->group_leader,
656 alpha_pmu->num_pmcs - 1,
657 evts, evtypes, idx_rubbish_bin);
658 if (n < 0)
659 return -EINVAL;
660 }
661 evtypes[n] = hwc->event_base;
662 evts[n] = event;
663
664 if (alpha_check_constraints(evts, evtypes, n + 1))
665 return -EINVAL;
666
667
668 hwc->config_base = 0;
669 hwc->idx = PMC_NO_INDEX;
670
671 event->destroy = hw_perf_event_destroy;
672
673
674
675
676
677
678
679
680
681
682
683 if (!hwc->sample_period) {
684 hwc->sample_period = alpha_pmu->pmc_max_period[0];
685 hwc->last_period = hwc->sample_period;
686 local64_set(&hwc->period_left, hwc->sample_period);
687 }
688
689 return 0;
690}
691
692
693
694
695static int alpha_pmu_event_init(struct perf_event *event)
696{
697 int err;
698
699
700 if (has_branch_stack(event))
701 return -EOPNOTSUPP;
702
703 switch (event->attr.type) {
704 case PERF_TYPE_RAW:
705 case PERF_TYPE_HARDWARE:
706 case PERF_TYPE_HW_CACHE:
707 break;
708
709 default:
710 return -ENOENT;
711 }
712
713 if (!alpha_pmu)
714 return -ENODEV;
715
716
717 err = __hw_perf_event_init(event);
718
719 return err;
720}
721
722
723
724
725static void alpha_pmu_enable(struct pmu *pmu)
726{
727 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
728
729 if (cpuc->enabled)
730 return;
731
732 cpuc->enabled = 1;
733 barrier();
734
735 if (cpuc->n_events > 0) {
736
737 maybe_change_configuration(cpuc);
738
739
740 wrperfmon(PERFMON_CMD_LOGGING_OPTIONS, EV67_PCTR_MODE_AGGREGATE);
741 wrperfmon(PERFMON_CMD_DESIRED_EVENTS, cpuc->config);
742 wrperfmon(PERFMON_CMD_ENABLE, cpuc->idx_mask);
743 }
744}
745
746
747
748
749
750
751static void alpha_pmu_disable(struct pmu *pmu)
752{
753 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
754
755 if (!cpuc->enabled)
756 return;
757
758 cpuc->enabled = 0;
759 cpuc->n_added = 0;
760
761 wrperfmon(PERFMON_CMD_DISABLE, cpuc->idx_mask);
762}
763
764static struct pmu pmu = {
765 .pmu_enable = alpha_pmu_enable,
766 .pmu_disable = alpha_pmu_disable,
767 .event_init = alpha_pmu_event_init,
768 .add = alpha_pmu_add,
769 .del = alpha_pmu_del,
770 .start = alpha_pmu_start,
771 .stop = alpha_pmu_stop,
772 .read = alpha_pmu_read,
773};
774
775
776
777
778
779
780void perf_event_print_debug(void)
781{
782 unsigned long flags;
783 unsigned long pcr;
784 int pcr0, pcr1;
785 int cpu;
786
787 if (!supported_cpu())
788 return;
789
790 local_irq_save(flags);
791
792 cpu = smp_processor_id();
793
794 pcr = wrperfmon(PERFMON_CMD_READ, 0);
795 pcr0 = (pcr >> alpha_pmu->pmc_count_shift[0]) & alpha_pmu->pmc_count_mask[0];
796 pcr1 = (pcr >> alpha_pmu->pmc_count_shift[1]) & alpha_pmu->pmc_count_mask[1];
797
798 pr_info("CPU#%d: PCTR0[%06x] PCTR1[%06x]\n", cpu, pcr0, pcr1);
799
800 local_irq_restore(flags);
801}
802
803
804
805
806
807
808static void alpha_perf_event_irq_handler(unsigned long la_ptr,
809 struct pt_regs *regs)
810{
811 struct cpu_hw_events *cpuc;
812 struct perf_sample_data data;
813 struct perf_event *event;
814 struct hw_perf_event *hwc;
815 int idx, j;
816
817 __this_cpu_inc(irq_pmi_count);
818 cpuc = this_cpu_ptr(&cpu_hw_events);
819
820
821
822
823
824
825 wrperfmon(PERFMON_CMD_DISABLE, cpuc->idx_mask);
826
827
828 if (unlikely(la_ptr >= alpha_pmu->num_pmcs)) {
829
830 irq_err_count++;
831 pr_warning("PMI: silly index %ld\n", la_ptr);
832 wrperfmon(PERFMON_CMD_ENABLE, cpuc->idx_mask);
833 return;
834 }
835
836 idx = la_ptr;
837
838 for (j = 0; j < cpuc->n_events; j++) {
839 if (cpuc->current_idx[j] == idx)
840 break;
841 }
842
843 if (unlikely(j == cpuc->n_events)) {
844
845 wrperfmon(PERFMON_CMD_ENABLE, cpuc->idx_mask);
846 return;
847 }
848
849 event = cpuc->event[j];
850
851 if (unlikely(!event)) {
852
853 irq_err_count++;
854 pr_warning("PMI: No event at index %d!\n", idx);
855 wrperfmon(PERFMON_CMD_ENABLE, cpuc->idx_mask);
856 return;
857 }
858
859 hwc = &event->hw;
860 alpha_perf_event_update(event, hwc, idx, alpha_pmu->pmc_max_period[idx]+1);
861 perf_sample_data_init(&data, 0, hwc->last_period);
862
863 if (alpha_perf_event_set_period(event, hwc, idx)) {
864 if (perf_event_overflow(event, &data, regs)) {
865
866
867
868 alpha_pmu_stop(event, 0);
869 }
870 }
871 wrperfmon(PERFMON_CMD_ENABLE, cpuc->idx_mask);
872
873 return;
874}
875
876
877
878
879
880
881int __init init_hw_perf_events(void)
882{
883 pr_info("Performance events: ");
884
885 if (!supported_cpu()) {
886 pr_cont("No support for your CPU.\n");
887 return 0;
888 }
889
890 pr_cont("Supported CPU type!\n");
891
892
893
894 perf_irq = alpha_perf_event_irq_handler;
895
896
897 alpha_pmu = &ev67_pmu;
898
899 perf_pmu_register(&pmu, "cpu", PERF_TYPE_RAW);
900
901 return 0;
902}
903early_initcall(init_hw_perf_events);
904