1
2
3
4
5
6
7
8
9
10
11
12#include <linux/kernel.h>
13#include <linux/sched.h>
14#include <linux/perf_event.h>
15#include <linux/percpu.h>
16#include <linux/hardirq.h>
17#include <asm/reg_fsl_emb.h>
18#include <asm/pmc.h>
19#include <asm/machdep.h>
20#include <asm/firmware.h>
21#include <asm/ptrace.h>
22
23struct cpu_hw_events {
24 int n_events;
25 int disabled;
26 u8 pmcs_enabled;
27 struct perf_event *event[MAX_HWEVENTS];
28};
29static DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events);
30
31static struct fsl_emb_pmu *ppmu;
32
33
34static atomic_t num_events;
35
36static DEFINE_MUTEX(pmc_reserve_mutex);
37
38
39
40
41
42static inline int perf_intr_is_nmi(struct pt_regs *regs)
43{
44#ifdef __powerpc64__
45 return !regs->softe;
46#else
47 return 0;
48#endif
49}
50
51static void perf_event_interrupt(struct pt_regs *regs);
52
53
54
55
56static unsigned long read_pmc(int idx)
57{
58 unsigned long val;
59
60 switch (idx) {
61 case 0:
62 val = mfpmr(PMRN_PMC0);
63 break;
64 case 1:
65 val = mfpmr(PMRN_PMC1);
66 break;
67 case 2:
68 val = mfpmr(PMRN_PMC2);
69 break;
70 case 3:
71 val = mfpmr(PMRN_PMC3);
72 break;
73 case 4:
74 val = mfpmr(PMRN_PMC4);
75 break;
76 case 5:
77 val = mfpmr(PMRN_PMC5);
78 break;
79 default:
80 printk(KERN_ERR "oops trying to read PMC%d\n", idx);
81 val = 0;
82 }
83 return val;
84}
85
86
87
88
89static void write_pmc(int idx, unsigned long val)
90{
91 switch (idx) {
92 case 0:
93 mtpmr(PMRN_PMC0, val);
94 break;
95 case 1:
96 mtpmr(PMRN_PMC1, val);
97 break;
98 case 2:
99 mtpmr(PMRN_PMC2, val);
100 break;
101 case 3:
102 mtpmr(PMRN_PMC3, val);
103 break;
104 case 4:
105 mtpmr(PMRN_PMC4, val);
106 break;
107 case 5:
108 mtpmr(PMRN_PMC5, val);
109 break;
110 default:
111 printk(KERN_ERR "oops trying to write PMC%d\n", idx);
112 }
113
114 isync();
115}
116
117
118
119
120static void write_pmlca(int idx, unsigned long val)
121{
122 switch (idx) {
123 case 0:
124 mtpmr(PMRN_PMLCA0, val);
125 break;
126 case 1:
127 mtpmr(PMRN_PMLCA1, val);
128 break;
129 case 2:
130 mtpmr(PMRN_PMLCA2, val);
131 break;
132 case 3:
133 mtpmr(PMRN_PMLCA3, val);
134 break;
135 case 4:
136 mtpmr(PMRN_PMLCA4, val);
137 break;
138 case 5:
139 mtpmr(PMRN_PMLCA5, val);
140 break;
141 default:
142 printk(KERN_ERR "oops trying to write PMLCA%d\n", idx);
143 }
144
145 isync();
146}
147
148
149
150
151static void write_pmlcb(int idx, unsigned long val)
152{
153 switch (idx) {
154 case 0:
155 mtpmr(PMRN_PMLCB0, val);
156 break;
157 case 1:
158 mtpmr(PMRN_PMLCB1, val);
159 break;
160 case 2:
161 mtpmr(PMRN_PMLCB2, val);
162 break;
163 case 3:
164 mtpmr(PMRN_PMLCB3, val);
165 break;
166 case 4:
167 mtpmr(PMRN_PMLCB4, val);
168 break;
169 case 5:
170 mtpmr(PMRN_PMLCB5, val);
171 break;
172 default:
173 printk(KERN_ERR "oops trying to write PMLCB%d\n", idx);
174 }
175
176 isync();
177}
178
179static void fsl_emb_pmu_read(struct perf_event *event)
180{
181 s64 val, delta, prev;
182
183 if (event->hw.state & PERF_HES_STOPPED)
184 return;
185
186
187
188
189
190
191 do {
192 prev = local64_read(&event->hw.prev_count);
193 barrier();
194 val = read_pmc(event->hw.idx);
195 } while (local64_cmpxchg(&event->hw.prev_count, prev, val) != prev);
196
197
198 delta = (val - prev) & 0xfffffffful;
199 local64_add(delta, &event->count);
200 local64_sub(delta, &event->hw.period_left);
201}
202
203
204
205
206
207static void fsl_emb_pmu_disable(struct pmu *pmu)
208{
209 struct cpu_hw_events *cpuhw;
210 unsigned long flags;
211
212 local_irq_save(flags);
213 cpuhw = &__get_cpu_var(cpu_hw_events);
214
215 if (!cpuhw->disabled) {
216 cpuhw->disabled = 1;
217
218
219
220
221 if (!cpuhw->pmcs_enabled) {
222 ppc_enable_pmcs();
223 cpuhw->pmcs_enabled = 1;
224 }
225
226 if (atomic_read(&num_events)) {
227
228
229
230
231
232
233
234 mtpmr(PMRN_PMGC0, PMGC0_FAC);
235 isync();
236 }
237 }
238 local_irq_restore(flags);
239}
240
241
242
243
244
245
246static void fsl_emb_pmu_enable(struct pmu *pmu)
247{
248 struct cpu_hw_events *cpuhw;
249 unsigned long flags;
250
251 local_irq_save(flags);
252 cpuhw = &__get_cpu_var(cpu_hw_events);
253 if (!cpuhw->disabled)
254 goto out;
255
256 cpuhw->disabled = 0;
257 ppc_set_pmu_inuse(cpuhw->n_events != 0);
258
259 if (cpuhw->n_events > 0) {
260 mtpmr(PMRN_PMGC0, PMGC0_PMIE | PMGC0_FCECE);
261 isync();
262 }
263
264 out:
265 local_irq_restore(flags);
266}
267
268static int collect_events(struct perf_event *group, int max_count,
269 struct perf_event *ctrs[])
270{
271 int n = 0;
272 struct perf_event *event;
273
274 if (!is_software_event(group)) {
275 if (n >= max_count)
276 return -1;
277 ctrs[n] = group;
278 n++;
279 }
280 list_for_each_entry(event, &group->sibling_list, group_entry) {
281 if (!is_software_event(event) &&
282 event->state != PERF_EVENT_STATE_OFF) {
283 if (n >= max_count)
284 return -1;
285 ctrs[n] = event;
286 n++;
287 }
288 }
289 return n;
290}
291
292
293static int fsl_emb_pmu_add(struct perf_event *event, int flags)
294{
295 struct cpu_hw_events *cpuhw;
296 int ret = -EAGAIN;
297 int num_counters = ppmu->n_counter;
298 u64 val;
299 int i;
300
301 perf_pmu_disable(event->pmu);
302 cpuhw = &get_cpu_var(cpu_hw_events);
303
304 if (event->hw.config & FSL_EMB_EVENT_RESTRICTED)
305 num_counters = ppmu->n_restricted;
306
307
308
309
310
311 for (i = num_counters - 1; i >= 0; i--) {
312 if (cpuhw->event[i])
313 continue;
314
315 break;
316 }
317
318 if (i < 0)
319 goto out;
320
321 event->hw.idx = i;
322 cpuhw->event[i] = event;
323 ++cpuhw->n_events;
324
325 val = 0;
326 if (event->hw.sample_period) {
327 s64 left = local64_read(&event->hw.period_left);
328 if (left < 0x80000000L)
329 val = 0x80000000L - left;
330 }
331 local64_set(&event->hw.prev_count, val);
332
333 if (!(flags & PERF_EF_START)) {
334 event->hw.state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
335 val = 0;
336 }
337
338 write_pmc(i, val);
339 perf_event_update_userpage(event);
340
341 write_pmlcb(i, event->hw.config >> 32);
342 write_pmlca(i, event->hw.config_base);
343
344 ret = 0;
345 out:
346 put_cpu_var(cpu_hw_events);
347 perf_pmu_enable(event->pmu);
348 return ret;
349}
350
351
352static void fsl_emb_pmu_del(struct perf_event *event, int flags)
353{
354 struct cpu_hw_events *cpuhw;
355 int i = event->hw.idx;
356
357 perf_pmu_disable(event->pmu);
358 if (i < 0)
359 goto out;
360
361 fsl_emb_pmu_read(event);
362
363 cpuhw = &get_cpu_var(cpu_hw_events);
364
365 WARN_ON(event != cpuhw->event[event->hw.idx]);
366
367 write_pmlca(i, 0);
368 write_pmlcb(i, 0);
369 write_pmc(i, 0);
370
371 cpuhw->event[i] = NULL;
372 event->hw.idx = -1;
373
374
375
376
377
378
379
380
381
382 cpuhw->n_events--;
383
384 out:
385 perf_pmu_enable(event->pmu);
386 put_cpu_var(cpu_hw_events);
387}
388
389static void fsl_emb_pmu_start(struct perf_event *event, int ef_flags)
390{
391 unsigned long flags;
392 s64 left;
393
394 if (event->hw.idx < 0 || !event->hw.sample_period)
395 return;
396
397 if (!(event->hw.state & PERF_HES_STOPPED))
398 return;
399
400 if (ef_flags & PERF_EF_RELOAD)
401 WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE));
402
403 local_irq_save(flags);
404 perf_pmu_disable(event->pmu);
405
406 event->hw.state = 0;
407 left = local64_read(&event->hw.period_left);
408 write_pmc(event->hw.idx, left);
409
410 perf_event_update_userpage(event);
411 perf_pmu_enable(event->pmu);
412 local_irq_restore(flags);
413}
414
415static void fsl_emb_pmu_stop(struct perf_event *event, int ef_flags)
416{
417 unsigned long flags;
418
419 if (event->hw.idx < 0 || !event->hw.sample_period)
420 return;
421
422 if (event->hw.state & PERF_HES_STOPPED)
423 return;
424
425 local_irq_save(flags);
426 perf_pmu_disable(event->pmu);
427
428 fsl_emb_pmu_read(event);
429 event->hw.state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
430 write_pmc(event->hw.idx, 0);
431
432 perf_event_update_userpage(event);
433 perf_pmu_enable(event->pmu);
434 local_irq_restore(flags);
435}
436
437
438
439
440static void hw_perf_event_destroy(struct perf_event *event)
441{
442 if (!atomic_add_unless(&num_events, -1, 1)) {
443 mutex_lock(&pmc_reserve_mutex);
444 if (atomic_dec_return(&num_events) == 0)
445 release_pmc_hardware();
446 mutex_unlock(&pmc_reserve_mutex);
447 }
448}
449
450
451
452
453static int hw_perf_cache_event(u64 config, u64 *eventp)
454{
455 unsigned long type, op, result;
456 int ev;
457
458 if (!ppmu->cache_events)
459 return -EINVAL;
460
461
462 type = config & 0xff;
463 op = (config >> 8) & 0xff;
464 result = (config >> 16) & 0xff;
465
466 if (type >= PERF_COUNT_HW_CACHE_MAX ||
467 op >= PERF_COUNT_HW_CACHE_OP_MAX ||
468 result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
469 return -EINVAL;
470
471 ev = (*ppmu->cache_events)[type][op][result];
472 if (ev == 0)
473 return -EOPNOTSUPP;
474 if (ev == -1)
475 return -EINVAL;
476 *eventp = ev;
477 return 0;
478}
479
480static int fsl_emb_pmu_event_init(struct perf_event *event)
481{
482 u64 ev;
483 struct perf_event *events[MAX_HWEVENTS];
484 int n;
485 int err;
486 int num_restricted;
487 int i;
488
489 if (ppmu->n_counter > MAX_HWEVENTS) {
490 WARN(1, "No. of perf counters (%d) is higher than max array size(%d)\n",
491 ppmu->n_counter, MAX_HWEVENTS);
492 ppmu->n_counter = MAX_HWEVENTS;
493 }
494
495 switch (event->attr.type) {
496 case PERF_TYPE_HARDWARE:
497 ev = event->attr.config;
498 if (ev >= ppmu->n_generic || ppmu->generic_events[ev] == 0)
499 return -EOPNOTSUPP;
500 ev = ppmu->generic_events[ev];
501 break;
502
503 case PERF_TYPE_HW_CACHE:
504 err = hw_perf_cache_event(event->attr.config, &ev);
505 if (err)
506 return err;
507 break;
508
509 case PERF_TYPE_RAW:
510 ev = event->attr.config;
511 break;
512
513 default:
514 return -ENOENT;
515 }
516
517 event->hw.config = ppmu->xlate_event(ev);
518 if (!(event->hw.config & FSL_EMB_EVENT_VALID))
519 return -EINVAL;
520
521
522
523
524
525
526 n = 0;
527 if (event->group_leader != event) {
528 n = collect_events(event->group_leader,
529 ppmu->n_counter - 1, events);
530 if (n < 0)
531 return -EINVAL;
532 }
533
534 if (event->hw.config & FSL_EMB_EVENT_RESTRICTED) {
535 num_restricted = 0;
536 for (i = 0; i < n; i++) {
537 if (events[i]->hw.config & FSL_EMB_EVENT_RESTRICTED)
538 num_restricted++;
539 }
540
541 if (num_restricted >= ppmu->n_restricted)
542 return -EINVAL;
543 }
544
545 event->hw.idx = -1;
546
547 event->hw.config_base = PMLCA_CE | PMLCA_FCM1 |
548 (u32)((ev << 16) & PMLCA_EVENT_MASK);
549
550 if (event->attr.exclude_user)
551 event->hw.config_base |= PMLCA_FCU;
552 if (event->attr.exclude_kernel)
553 event->hw.config_base |= PMLCA_FCS;
554 if (event->attr.exclude_idle)
555 return -ENOTSUPP;
556
557 event->hw.last_period = event->hw.sample_period;
558 local64_set(&event->hw.period_left, event->hw.last_period);
559
560
561
562
563
564
565
566 err = 0;
567 if (!atomic_inc_not_zero(&num_events)) {
568 mutex_lock(&pmc_reserve_mutex);
569 if (atomic_read(&num_events) == 0 &&
570 reserve_pmc_hardware(perf_event_interrupt))
571 err = -EBUSY;
572 else
573 atomic_inc(&num_events);
574 mutex_unlock(&pmc_reserve_mutex);
575
576 mtpmr(PMRN_PMGC0, PMGC0_FAC);
577 isync();
578 }
579 event->destroy = hw_perf_event_destroy;
580
581 return err;
582}
583
584static struct pmu fsl_emb_pmu = {
585 .pmu_enable = fsl_emb_pmu_enable,
586 .pmu_disable = fsl_emb_pmu_disable,
587 .event_init = fsl_emb_pmu_event_init,
588 .add = fsl_emb_pmu_add,
589 .del = fsl_emb_pmu_del,
590 .start = fsl_emb_pmu_start,
591 .stop = fsl_emb_pmu_stop,
592 .read = fsl_emb_pmu_read,
593};
594
595
596
597
598
599
600static void record_and_restart(struct perf_event *event, unsigned long val,
601 struct pt_regs *regs)
602{
603 u64 period = event->hw.sample_period;
604 s64 prev, delta, left;
605 int record = 0;
606
607 if (event->hw.state & PERF_HES_STOPPED) {
608 write_pmc(event->hw.idx, 0);
609 return;
610 }
611
612
613 prev = local64_read(&event->hw.prev_count);
614 delta = (val - prev) & 0xfffffffful;
615 local64_add(delta, &event->count);
616
617
618
619
620
621 val = 0;
622 left = local64_read(&event->hw.period_left) - delta;
623 if (period) {
624 if (left <= 0) {
625 left += period;
626 if (left <= 0)
627 left = period;
628 record = 1;
629 event->hw.last_period = event->hw.sample_period;
630 }
631 if (left < 0x80000000LL)
632 val = 0x80000000LL - left;
633 }
634
635 write_pmc(event->hw.idx, val);
636 local64_set(&event->hw.prev_count, val);
637 local64_set(&event->hw.period_left, left);
638 perf_event_update_userpage(event);
639
640
641
642
643 if (record) {
644 struct perf_sample_data data;
645
646 perf_sample_data_init(&data, 0, event->hw.last_period);
647
648 if (perf_event_overflow(event, &data, regs))
649 fsl_emb_pmu_stop(event, 0);
650 }
651}
652
653static void perf_event_interrupt(struct pt_regs *regs)
654{
655 int i;
656 struct cpu_hw_events *cpuhw = &__get_cpu_var(cpu_hw_events);
657 struct perf_event *event;
658 unsigned long val;
659 int found = 0;
660 int nmi;
661
662 nmi = perf_intr_is_nmi(regs);
663 if (nmi)
664 nmi_enter();
665 else
666 irq_enter();
667
668 for (i = 0; i < ppmu->n_counter; ++i) {
669 event = cpuhw->event[i];
670
671 val = read_pmc(i);
672 if ((int)val < 0) {
673 if (event) {
674
675 found = 1;
676 record_and_restart(event, val, regs);
677 } else {
678
679
680
681
682 write_pmc(i, 0);
683 }
684 }
685 }
686
687
688 mtmsr(mfmsr() | MSR_PMM);
689 mtpmr(PMRN_PMGC0, PMGC0_PMIE | PMGC0_FCECE);
690 isync();
691
692 if (nmi)
693 nmi_exit();
694 else
695 irq_exit();
696}
697
698void hw_perf_event_setup(int cpu)
699{
700 struct cpu_hw_events *cpuhw = &per_cpu(cpu_hw_events, cpu);
701
702 memset(cpuhw, 0, sizeof(*cpuhw));
703}
704
705int register_fsl_emb_pmu(struct fsl_emb_pmu *pmu)
706{
707 if (ppmu)
708 return -EBUSY;
709
710 ppmu = pmu;
711 pr_info("%s performance monitor hardware support registered\n",
712 pmu->name);
713
714 perf_pmu_register(&fsl_emb_pmu, "cpu", PERF_TYPE_RAW);
715
716 return 0;
717}
718