1
2
3
4
5
6
7#include <linux/kernel.h>
8#include <linux/perf_event.h>
9#include <linux/string.h>
10#include <asm/reg.h>
11#include <asm/cputable.h>
12
13#include "internal.h"
14
15
16
17
18#define PM_PMC_SH 20
19#define PM_PMC_MSK 0xf
20#define PM_PMC_MSKS (PM_PMC_MSK << PM_PMC_SH)
21#define PM_UNIT_SH 16
22#define PM_UNIT_MSK 0xf
23#define PM_BYTE_SH 12
24#define PM_BYTE_MSK 7
25#define PM_GRS_SH 8
26#define PM_GRS_MSK 7
27#define PM_BUSEVENT_MSK 0x80
28#define PM_PMCSEL_MSK 0x7f
29
30
31#define PM_FPU 0
32#define PM_ISU0 1
33#define PM_IFU 2
34#define PM_ISU1 3
35#define PM_IDU 4
36#define PM_ISU0_ALT 6
37#define PM_GRS 7
38#define PM_LSU0 8
39#define PM_LSU1 0xc
40#define PM_LASTUNIT 0xc
41
42
43
44
45#define MMCR1_TTM0SEL_SH 62
46#define MMCR1_TTM1SEL_SH 60
47#define MMCR1_TTM2SEL_SH 58
48#define MMCR1_TTM3SEL_SH 56
49#define MMCR1_TTMSEL_MSK 3
50#define MMCR1_TD_CP_DBG0SEL_SH 54
51#define MMCR1_TD_CP_DBG1SEL_SH 52
52#define MMCR1_TD_CP_DBG2SEL_SH 50
53#define MMCR1_TD_CP_DBG3SEL_SH 48
54#define MMCR1_GRS_L2SEL_SH 46
55#define MMCR1_GRS_L2SEL_MSK 3
56#define MMCR1_GRS_L3SEL_SH 44
57#define MMCR1_GRS_L3SEL_MSK 3
58#define MMCR1_GRS_MCSEL_SH 41
59#define MMCR1_GRS_MCSEL_MSK 7
60#define MMCR1_GRS_FABSEL_SH 39
61#define MMCR1_GRS_FABSEL_MSK 3
62#define MMCR1_PMC1_ADDER_SEL_SH 35
63#define MMCR1_PMC2_ADDER_SEL_SH 34
64#define MMCR1_PMC3_ADDER_SEL_SH 33
65#define MMCR1_PMC4_ADDER_SEL_SH 32
66#define MMCR1_PMC1SEL_SH 25
67#define MMCR1_PMC2SEL_SH 17
68#define MMCR1_PMC3SEL_SH 9
69#define MMCR1_PMC4SEL_SH 1
70#define MMCR1_PMCSEL_SH(n) (MMCR1_PMC1SEL_SH - (n) * 8)
71#define MMCR1_PMCSEL_MSK 0x7f
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118static const int grsel_shift[8] = {
119 MMCR1_GRS_L2SEL_SH, MMCR1_GRS_L2SEL_SH, MMCR1_GRS_L2SEL_SH,
120 MMCR1_GRS_L3SEL_SH, MMCR1_GRS_L3SEL_SH, MMCR1_GRS_L3SEL_SH,
121 MMCR1_GRS_MCSEL_SH, MMCR1_GRS_FABSEL_SH
122};
123
124
125static unsigned long unit_cons[PM_LASTUNIT+1][2] = {
126 [PM_FPU] = { 0x3200000000ul, 0x0100000000ul },
127 [PM_ISU0] = { 0x0200000000ul, 0x0080000000ul },
128 [PM_ISU1] = { 0x3200000000ul, 0x3100000000ul },
129 [PM_IFU] = { 0x3200000000ul, 0x2100000000ul },
130 [PM_IDU] = { 0x0e00000000ul, 0x0040000000ul },
131 [PM_GRS] = { 0x0e00000000ul, 0x0c40000000ul },
132};
133
134static int power5p_get_constraint(u64 event, unsigned long *maskp,
135 unsigned long *valp, u64 event_config1 __maybe_unused)
136{
137 int pmc, byte, unit, sh;
138 int bit, fmask;
139 unsigned long mask = 0, value = 0;
140
141 pmc = (event >> PM_PMC_SH) & PM_PMC_MSK;
142 if (pmc) {
143 if (pmc > 6)
144 return -1;
145 sh = (pmc - 1) * 2;
146 mask |= 2 << sh;
147 value |= 1 << sh;
148 if (pmc >= 5 && !(event == 0x500009 || event == 0x600005))
149 return -1;
150 }
151 if (event & PM_BUSEVENT_MSK) {
152 unit = (event >> PM_UNIT_SH) & PM_UNIT_MSK;
153 if (unit > PM_LASTUNIT)
154 return -1;
155 if (unit == PM_ISU0_ALT)
156 unit = PM_ISU0;
157 mask |= unit_cons[unit][0];
158 value |= unit_cons[unit][1];
159 byte = (event >> PM_BYTE_SH) & PM_BYTE_MSK;
160 if (byte >= 4) {
161 if (unit != PM_LSU1)
162 return -1;
163
164 ++unit;
165 byte &= 3;
166 }
167 if (unit == PM_GRS) {
168 bit = event & 7;
169 fmask = (bit == 6)? 7: 3;
170 sh = grsel_shift[bit];
171 mask |= (unsigned long)fmask << sh;
172 value |= (unsigned long)((event >> PM_GRS_SH) & fmask)
173 << sh;
174 }
175
176 mask |= 0xfUL << (24 - 4 * byte);
177 value |= (unsigned long)unit << (24 - 4 * byte);
178 }
179 if (pmc < 5) {
180
181 mask |= 0x8000000000000ul;
182 value |= 0x1000000000000ul;
183 }
184 *maskp = mask;
185 *valp = value;
186 return 0;
187}
188
189static int power5p_limited_pmc_event(u64 event)
190{
191 int pmc = (event >> PM_PMC_SH) & PM_PMC_MSK;
192
193 return pmc == 5 || pmc == 6;
194}
195
196#define MAX_ALT 3
197
198static const unsigned int event_alternatives[][MAX_ALT] = {
199 { 0x100c0, 0x40001f },
200 { 0x120e4, 0x400002 },
201 { 0x230e2, 0x323087 },
202 { 0x230e3, 0x223087, 0x3230a0 },
203 { 0x410c7, 0x441084 },
204 { 0x800c4, 0xc20e0 },
205 { 0xc50c6, 0xc60e0 },
206 { 0x100005, 0x600005 },
207 { 0x100009, 0x200009 },
208 { 0x200015, 0x300015 },
209 { 0x300009, 0x400009 },
210};
211
212
213
214
215
216static int find_alternative(unsigned int event)
217{
218 int i, j;
219
220 for (i = 0; i < ARRAY_SIZE(event_alternatives); ++i) {
221 if (event < event_alternatives[i][0])
222 break;
223 for (j = 0; j < MAX_ALT && event_alternatives[i][j]; ++j)
224 if (event == event_alternatives[i][j])
225 return i;
226 }
227 return -1;
228}
229
230static const unsigned char bytedecode_alternatives[4][4] = {
231 { 0x21, 0x23, 0x25, 0x27 },
232 { 0x07, 0x17, 0x0e, 0x1e },
233 { 0x20, 0x22, 0x24, 0x26 },
234 { 0x07, 0x17, 0x0e, 0x1e }
235};
236
237
238
239
240
241
242
243static s64 find_alternative_bdecode(u64 event)
244{
245 int pmc, altpmc, pp, j;
246
247 pmc = (event >> PM_PMC_SH) & PM_PMC_MSK;
248 if (pmc == 0 || pmc > 4)
249 return -1;
250 altpmc = 5 - pmc;
251 pp = event & PM_PMCSEL_MSK;
252 for (j = 0; j < 4; ++j) {
253 if (bytedecode_alternatives[pmc - 1][j] == pp) {
254 return (event & ~(PM_PMC_MSKS | PM_PMCSEL_MSK)) |
255 (altpmc << PM_PMC_SH) |
256 bytedecode_alternatives[altpmc - 1][j];
257 }
258 }
259
260
261 if (pmc == 1 && (pp == 0x0d || pp == 0x0e))
262 return event + (2 << PM_PMC_SH) + (0x2e - 0x0d);
263 if (pmc == 3 && (pp == 0x2e || pp == 0x2f))
264 return event - (2 << PM_PMC_SH) - (0x2e - 0x0d);
265
266
267 if (pp == 0x10 || pp == 0x28)
268 return ((event ^ (0x10 ^ 0x28)) & ~PM_PMC_MSKS) |
269 (altpmc << PM_PMC_SH);
270
271 return -1;
272}
273
274static int power5p_get_alternatives(u64 event, unsigned int flags, u64 alt[])
275{
276 int i, j, nalt = 1;
277 int nlim;
278 s64 ae;
279
280 alt[0] = event;
281 nalt = 1;
282 nlim = power5p_limited_pmc_event(event);
283 i = find_alternative(event);
284 if (i >= 0) {
285 for (j = 0; j < MAX_ALT; ++j) {
286 ae = event_alternatives[i][j];
287 if (ae && ae != event)
288 alt[nalt++] = ae;
289 nlim += power5p_limited_pmc_event(ae);
290 }
291 } else {
292 ae = find_alternative_bdecode(event);
293 if (ae > 0)
294 alt[nalt++] = ae;
295 }
296
297 if (flags & PPMU_ONLY_COUNT_RUN) {
298
299
300
301
302
303
304
305
306
307
308 j = nalt;
309 for (i = 0; i < nalt; ++i) {
310 switch (alt[i]) {
311 case 0xf:
312 alt[j++] = 0x600005;
313 ++nlim;
314 break;
315 case 0x600005:
316 alt[j++] = 0xf;
317 break;
318 case 0x100009:
319 alt[j++] = 0x500009;
320 ++nlim;
321 break;
322 case 0x500009:
323 alt[j++] = 0x100009;
324 alt[j++] = 0x200009;
325 break;
326 }
327 }
328 nalt = j;
329 }
330
331 if (!(flags & PPMU_LIMITED_PMC_OK) && nlim) {
332
333 j = 0;
334 for (i = 0; i < nalt; ++i) {
335 if (!power5p_limited_pmc_event(alt[i])) {
336 alt[j] = alt[i];
337 ++j;
338 }
339 }
340 nalt = j;
341 } else if ((flags & PPMU_LIMITED_PMC_REQD) && nlim < nalt) {
342
343 j = 0;
344 for (i = 0; i < nalt; ++i) {
345 if (power5p_limited_pmc_event(alt[i])) {
346 alt[j] = alt[i];
347 ++j;
348 }
349 }
350 nalt = j;
351 }
352
353 return nalt;
354}
355
356
357
358
359
360
361
362static unsigned char direct_event_is_marked[0x28] = {
363 0,
364 0x1f,
365 0x2,
366 0xe,
367 0,
368 0x1c,
369 0x80,
370 0x80,
371 0, 0, 0,
372 0x18,
373 0,
374 0x80,
375 0x80,
376 0,
377 0,
378 0x14,
379 0,
380 0x10,
381 0x1f,
382 0x2,
383 0x80,
384 0x80,
385 0, 0, 0, 0, 0,
386 0x80,
387 0x80,
388 0,
389 0x80,
390 0x80,
391 0x80,
392 0x80,
393 0x80,
394 0x80,
395 0x80,
396 0x80,
397};
398
399
400
401
402
403static int power5p_marked_instr_event(u64 event)
404{
405 int pmc, psel;
406 int bit, byte, unit;
407 u32 mask;
408
409 pmc = (event >> PM_PMC_SH) & PM_PMC_MSK;
410 psel = event & PM_PMCSEL_MSK;
411 if (pmc >= 5)
412 return 0;
413
414 bit = -1;
415 if (psel < sizeof(direct_event_is_marked)) {
416 if (direct_event_is_marked[psel] & (1 << pmc))
417 return 1;
418 if (direct_event_is_marked[psel] & 0x80)
419 bit = 4;
420 else if (psel == 0x08)
421 bit = pmc - 1;
422 else if (psel == 0x10)
423 bit = 4 - pmc;
424 else if (psel == 0x1b && (pmc == 1 || pmc == 3))
425 bit = 4;
426 } else if ((psel & 0x48) == 0x40) {
427 bit = psel & 7;
428 } else if (psel == 0x28) {
429 bit = pmc - 1;
430 } else if (pmc == 3 && (psel == 0x2e || psel == 0x2f)) {
431 bit = 4;
432 }
433
434 if (!(event & PM_BUSEVENT_MSK) || bit == -1)
435 return 0;
436
437 byte = (event >> PM_BYTE_SH) & PM_BYTE_MSK;
438 unit = (event >> PM_UNIT_SH) & PM_UNIT_MSK;
439 if (unit == PM_LSU0) {
440
441 mask = 0x5dff00;
442 } else if (unit == PM_LSU1 && byte >= 4) {
443 byte -= 4;
444
445 mask = 0x5f11c000;
446 } else
447 return 0;
448
449 return (mask >> (byte * 8 + bit)) & 1;
450}
451
452static int power5p_compute_mmcr(u64 event[], int n_ev,
453 unsigned int hwc[], struct mmcr_regs *mmcr,
454 struct perf_event *pevents[],
455 u32 flags __maybe_unused)
456{
457 unsigned long mmcr1 = 0;
458 unsigned long mmcra = 0;
459 unsigned int pmc, unit, byte, psel;
460 unsigned int ttm;
461 int i, isbus, bit, grsel;
462 unsigned int pmc_inuse = 0;
463 unsigned char busbyte[4];
464 unsigned char unituse[16];
465 int ttmuse;
466
467 if (n_ev > 6)
468 return -1;
469
470
471 memset(busbyte, 0, sizeof(busbyte));
472 memset(unituse, 0, sizeof(unituse));
473 for (i = 0; i < n_ev; ++i) {
474 pmc = (event[i] >> PM_PMC_SH) & PM_PMC_MSK;
475 if (pmc) {
476 if (pmc > 6)
477 return -1;
478 if (pmc_inuse & (1 << (pmc - 1)))
479 return -1;
480 pmc_inuse |= 1 << (pmc - 1);
481 }
482 if (event[i] & PM_BUSEVENT_MSK) {
483 unit = (event[i] >> PM_UNIT_SH) & PM_UNIT_MSK;
484 byte = (event[i] >> PM_BYTE_SH) & PM_BYTE_MSK;
485 if (unit > PM_LASTUNIT)
486 return -1;
487 if (unit == PM_ISU0_ALT)
488 unit = PM_ISU0;
489 if (byte >= 4) {
490 if (unit != PM_LSU1)
491 return -1;
492 ++unit;
493 byte &= 3;
494 }
495 if (busbyte[byte] && busbyte[byte] != unit)
496 return -1;
497 busbyte[byte] = unit;
498 unituse[unit] = 1;
499 }
500 }
501
502
503
504
505
506
507
508 if (unituse[PM_ISU0] &
509 (unituse[PM_FPU] | unituse[PM_IFU] | unituse[PM_ISU1])) {
510 unituse[PM_ISU0_ALT] = 1;
511 unituse[PM_ISU0] = 0;
512 }
513
514 ttmuse = 0;
515 for (i = PM_FPU; i <= PM_ISU1; ++i) {
516 if (!unituse[i])
517 continue;
518 if (ttmuse++)
519 return -1;
520 mmcr1 |= (unsigned long)i << MMCR1_TTM0SEL_SH;
521 }
522 ttmuse = 0;
523 for (; i <= PM_GRS; ++i) {
524 if (!unituse[i])
525 continue;
526 if (ttmuse++)
527 return -1;
528 mmcr1 |= (unsigned long)(i & 3) << MMCR1_TTM1SEL_SH;
529 }
530 if (ttmuse > 1)
531 return -1;
532
533
534 for (byte = 0; byte < 4; ++byte) {
535 unit = busbyte[byte];
536 if (!unit)
537 continue;
538 if (unit == PM_ISU0 && unituse[PM_ISU0_ALT]) {
539
540 unit = PM_ISU0_ALT;
541 } else if (unit == PM_LSU1 + 1) {
542
543 mmcr1 |= 1ul << (MMCR1_TTM3SEL_SH + 3 - byte);
544 }
545 ttm = unit >> 2;
546 mmcr1 |= (unsigned long)ttm
547 << (MMCR1_TD_CP_DBG0SEL_SH - 2 * byte);
548 }
549
550
551 for (i = 0; i < n_ev; ++i) {
552 pmc = (event[i] >> PM_PMC_SH) & PM_PMC_MSK;
553 unit = (event[i] >> PM_UNIT_SH) & PM_UNIT_MSK;
554 byte = (event[i] >> PM_BYTE_SH) & PM_BYTE_MSK;
555 psel = event[i] & PM_PMCSEL_MSK;
556 isbus = event[i] & PM_BUSEVENT_MSK;
557 if (!pmc) {
558
559 for (pmc = 0; pmc < 4; ++pmc) {
560 if (!(pmc_inuse & (1 << pmc)))
561 break;
562 }
563 if (pmc >= 4)
564 return -1;
565 pmc_inuse |= 1 << pmc;
566 } else if (pmc <= 4) {
567
568 --pmc;
569 if (isbus && (byte & 2) &&
570 (psel == 8 || psel == 0x10 || psel == 0x28))
571
572 mmcr1 |= 1ul << (MMCR1_PMC1_ADDER_SEL_SH - pmc);
573 } else {
574
575 --pmc;
576 }
577 if (isbus && unit == PM_GRS) {
578 bit = psel & 7;
579 grsel = (event[i] >> PM_GRS_SH) & PM_GRS_MSK;
580 mmcr1 |= (unsigned long)grsel << grsel_shift[bit];
581 }
582 if (power5p_marked_instr_event(event[i]))
583 mmcra |= MMCRA_SAMPLE_ENABLE;
584 if ((psel & 0x58) == 0x40 && (byte & 1) != ((pmc >> 1) & 1))
585
586 psel |= 0x10;
587 if (pmc <= 3)
588 mmcr1 |= psel << MMCR1_PMCSEL_SH(pmc);
589 hwc[i] = pmc;
590 }
591
592
593 mmcr->mmcr0 = 0;
594 if (pmc_inuse & 1)
595 mmcr->mmcr0 = MMCR0_PMC1CE;
596 if (pmc_inuse & 0x3e)
597 mmcr->mmcr0 |= MMCR0_PMCjCE;
598 mmcr->mmcr1 = mmcr1;
599 mmcr->mmcra = mmcra;
600 return 0;
601}
602
603static void power5p_disable_pmc(unsigned int pmc, struct mmcr_regs *mmcr)
604{
605 if (pmc <= 3)
606 mmcr->mmcr1 &= ~(0x7fUL << MMCR1_PMCSEL_SH(pmc));
607}
608
609static int power5p_generic_events[] = {
610 [PERF_COUNT_HW_CPU_CYCLES] = 0xf,
611 [PERF_COUNT_HW_INSTRUCTIONS] = 0x100009,
612 [PERF_COUNT_HW_CACHE_REFERENCES] = 0x1c10a8,
613 [PERF_COUNT_HW_CACHE_MISSES] = 0x3c1088,
614 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = 0x230e4,
615 [PERF_COUNT_HW_BRANCH_MISSES] = 0x230e5,
616};
617
618#define C(x) PERF_COUNT_HW_CACHE_##x
619
620
621
622
623
624
625static u64 power5p_cache_events[C(MAX)][C(OP_MAX)][C(RESULT_MAX)] = {
626 [C(L1D)] = {
627 [C(OP_READ)] = { 0x1c10a8, 0x3c1088 },
628 [C(OP_WRITE)] = { 0x2c10a8, 0xc10c3 },
629 [C(OP_PREFETCH)] = { 0xc70e7, -1 },
630 },
631 [C(L1I)] = {
632 [C(OP_READ)] = { 0, 0 },
633 [C(OP_WRITE)] = { -1, -1 },
634 [C(OP_PREFETCH)] = { 0, 0 },
635 },
636 [C(LL)] = {
637 [C(OP_READ)] = { 0, 0 },
638 [C(OP_WRITE)] = { 0, 0 },
639 [C(OP_PREFETCH)] = { 0xc50c3, 0 },
640 },
641 [C(DTLB)] = {
642 [C(OP_READ)] = { 0xc20e4, 0x800c4 },
643 [C(OP_WRITE)] = { -1, -1 },
644 [C(OP_PREFETCH)] = { -1, -1 },
645 },
646 [C(ITLB)] = {
647 [C(OP_READ)] = { 0, 0x800c0 },
648 [C(OP_WRITE)] = { -1, -1 },
649 [C(OP_PREFETCH)] = { -1, -1 },
650 },
651 [C(BPU)] = {
652 [C(OP_READ)] = { 0x230e4, 0x230e5 },
653 [C(OP_WRITE)] = { -1, -1 },
654 [C(OP_PREFETCH)] = { -1, -1 },
655 },
656 [C(NODE)] = {
657 [C(OP_READ)] = { -1, -1 },
658 [C(OP_WRITE)] = { -1, -1 },
659 [C(OP_PREFETCH)] = { -1, -1 },
660 },
661};
662
663static struct power_pmu power5p_pmu = {
664 .name = "POWER5+/++",
665 .n_counter = 6,
666 .max_alternatives = MAX_ALT,
667 .add_fields = 0x7000000000055ul,
668 .test_adder = 0x3000040000000ul,
669 .compute_mmcr = power5p_compute_mmcr,
670 .get_constraint = power5p_get_constraint,
671 .get_alternatives = power5p_get_alternatives,
672 .disable_pmc = power5p_disable_pmc,
673 .limited_pmc_event = power5p_limited_pmc_event,
674 .flags = PPMU_LIMITED_PMC5_6 | PPMU_HAS_SSLOT,
675 .n_generic = ARRAY_SIZE(power5p_generic_events),
676 .generic_events = power5p_generic_events,
677 .cache_events = &power5p_cache_events,
678};
679
680int init_power5p_pmu(void)
681{
682 if (!cur_cpu_spec->oprofile_cpu_type ||
683 (strcmp(cur_cpu_spec->oprofile_cpu_type, "ppc64/power5+")
684 && strcmp(cur_cpu_spec->oprofile_cpu_type, "ppc64/power5++")))
685 return -ENODEV;
686
687 return register_power_pmu(&power5p_pmu);
688}
689