1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19#ifdef CONFIG_CPU_V7
20
21#include <asm/cp15.h>
22#include <asm/cputype.h>
23#include <asm/irq_regs.h>
24#include <asm/vfp.h>
25#include "../vfp/vfpinstr.h"
26
27#include <linux/of.h>
28#include <linux/perf/arm_pmu.h>
29#include <linux/platform_device.h>
30
31
32
33
34
35
36
37
38#define ARMV7_PERFCTR_PMNC_SW_INCR 0x00
39#define ARMV7_PERFCTR_L1_ICACHE_REFILL 0x01
40#define ARMV7_PERFCTR_ITLB_REFILL 0x02
41#define ARMV7_PERFCTR_L1_DCACHE_REFILL 0x03
42#define ARMV7_PERFCTR_L1_DCACHE_ACCESS 0x04
43#define ARMV7_PERFCTR_DTLB_REFILL 0x05
44#define ARMV7_PERFCTR_MEM_READ 0x06
45#define ARMV7_PERFCTR_MEM_WRITE 0x07
46#define ARMV7_PERFCTR_INSTR_EXECUTED 0x08
47#define ARMV7_PERFCTR_EXC_TAKEN 0x09
48#define ARMV7_PERFCTR_EXC_EXECUTED 0x0A
49#define ARMV7_PERFCTR_CID_WRITE 0x0B
50
51
52
53
54
55
56
57
58#define ARMV7_PERFCTR_PC_WRITE 0x0C
59#define ARMV7_PERFCTR_PC_IMM_BRANCH 0x0D
60#define ARMV7_PERFCTR_PC_PROC_RETURN 0x0E
61#define ARMV7_PERFCTR_MEM_UNALIGNED_ACCESS 0x0F
62#define ARMV7_PERFCTR_PC_BRANCH_MIS_PRED 0x10
63#define ARMV7_PERFCTR_CLOCK_CYCLES 0x11
64#define ARMV7_PERFCTR_PC_BRANCH_PRED 0x12
65
66
67#define ARMV7_PERFCTR_MEM_ACCESS 0x13
68#define ARMV7_PERFCTR_L1_ICACHE_ACCESS 0x14
69#define ARMV7_PERFCTR_L1_DCACHE_WB 0x15
70#define ARMV7_PERFCTR_L2_CACHE_ACCESS 0x16
71#define ARMV7_PERFCTR_L2_CACHE_REFILL 0x17
72#define ARMV7_PERFCTR_L2_CACHE_WB 0x18
73#define ARMV7_PERFCTR_BUS_ACCESS 0x19
74#define ARMV7_PERFCTR_MEM_ERROR 0x1A
75#define ARMV7_PERFCTR_INSTR_SPEC 0x1B
76#define ARMV7_PERFCTR_TTBR_WRITE 0x1C
77#define ARMV7_PERFCTR_BUS_CYCLES 0x1D
78
79#define ARMV7_PERFCTR_CPU_CYCLES 0xFF
80
81
82#define ARMV7_A8_PERFCTR_L2_CACHE_ACCESS 0x43
83#define ARMV7_A8_PERFCTR_L2_CACHE_REFILL 0x44
84#define ARMV7_A8_PERFCTR_L1_ICACHE_ACCESS 0x50
85#define ARMV7_A8_PERFCTR_STALL_ISIDE 0x56
86
87
88#define ARMV7_A9_PERFCTR_INSTR_CORE_RENAME 0x68
89#define ARMV7_A9_PERFCTR_STALL_ICACHE 0x60
90#define ARMV7_A9_PERFCTR_STALL_DISPATCH 0x66
91
92
93#define ARMV7_A5_PERFCTR_PREFETCH_LINEFILL 0xc2
94#define ARMV7_A5_PERFCTR_PREFETCH_LINEFILL_DROP 0xc3
95
96
97#define ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_READ 0x40
98#define ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_WRITE 0x41
99#define ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_READ 0x42
100#define ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_WRITE 0x43
101
102#define ARMV7_A15_PERFCTR_DTLB_REFILL_L1_READ 0x4C
103#define ARMV7_A15_PERFCTR_DTLB_REFILL_L1_WRITE 0x4D
104
105#define ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_READ 0x50
106#define ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_WRITE 0x51
107#define ARMV7_A15_PERFCTR_L2_CACHE_REFILL_READ 0x52
108#define ARMV7_A15_PERFCTR_L2_CACHE_REFILL_WRITE 0x53
109
110#define ARMV7_A15_PERFCTR_PC_WRITE_SPEC 0x76
111
112
113#define ARMV7_A12_PERFCTR_L1_DCACHE_ACCESS_READ 0x40
114#define ARMV7_A12_PERFCTR_L1_DCACHE_ACCESS_WRITE 0x41
115
116#define ARMV7_A12_PERFCTR_L2_CACHE_ACCESS_READ 0x50
117#define ARMV7_A12_PERFCTR_L2_CACHE_ACCESS_WRITE 0x51
118
119#define ARMV7_A12_PERFCTR_PC_WRITE_SPEC 0x76
120
121#define ARMV7_A12_PERFCTR_PF_TLB_REFILL 0xe7
122
123
124#define KRAIT_PMRESR0_GROUP0 0xcc
125#define KRAIT_PMRESR1_GROUP0 0xd0
126#define KRAIT_PMRESR2_GROUP0 0xd4
127#define KRAIT_VPMRESR0_GROUP0 0xd8
128
129#define KRAIT_PERFCTR_L1_ICACHE_ACCESS 0x10011
130#define KRAIT_PERFCTR_L1_ICACHE_MISS 0x10010
131
132#define KRAIT_PERFCTR_L1_ITLB_ACCESS 0x12222
133#define KRAIT_PERFCTR_L1_DTLB_ACCESS 0x12210
134
135
136#define SCORPION_LPM0_GROUP0 0x4c
137#define SCORPION_LPM1_GROUP0 0x50
138#define SCORPION_LPM2_GROUP0 0x54
139#define SCORPION_L2LPM_GROUP0 0x58
140#define SCORPION_VLPM_GROUP0 0x5c
141
142#define SCORPION_ICACHE_ACCESS 0x10053
143#define SCORPION_ICACHE_MISS 0x10052
144
145#define SCORPION_DTLB_ACCESS 0x12013
146#define SCORPION_DTLB_MISS 0x12012
147
148#define SCORPION_ITLB_MISS 0x12021
149
150
151
152
153
154
155
156
157static const unsigned armv7_a8_perf_map[PERF_COUNT_HW_MAX] = {
158 PERF_MAP_ALL_UNSUPPORTED,
159 [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES,
160 [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED,
161 [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
162 [PERF_COUNT_HW_CACHE_MISSES] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
163 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE,
164 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
165 [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = ARMV7_A8_PERFCTR_STALL_ISIDE,
166};
167
168static const unsigned armv7_a8_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
169 [PERF_COUNT_HW_CACHE_OP_MAX]
170 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
171 PERF_CACHE_MAP_ALL_UNSUPPORTED,
172
173
174
175
176
177
178 [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
179 [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
180 [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
181 [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
182
183 [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_A8_PERFCTR_L1_ICACHE_ACCESS,
184 [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL,
185
186 [C(LL)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_A8_PERFCTR_L2_CACHE_ACCESS,
187 [C(LL)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_A8_PERFCTR_L2_CACHE_REFILL,
188 [C(LL)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_A8_PERFCTR_L2_CACHE_ACCESS,
189 [C(LL)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_A8_PERFCTR_L2_CACHE_REFILL,
190
191 [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL,
192 [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL,
193
194 [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,
195 [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,
196
197 [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
198 [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
199 [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
200 [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
201};
202
203
204
205
206static const unsigned armv7_a9_perf_map[PERF_COUNT_HW_MAX] = {
207 PERF_MAP_ALL_UNSUPPORTED,
208 [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES,
209 [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_A9_PERFCTR_INSTR_CORE_RENAME,
210 [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
211 [PERF_COUNT_HW_CACHE_MISSES] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
212 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE,
213 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
214 [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = ARMV7_A9_PERFCTR_STALL_ICACHE,
215 [PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = ARMV7_A9_PERFCTR_STALL_DISPATCH,
216};
217
218static const unsigned armv7_a9_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
219 [PERF_COUNT_HW_CACHE_OP_MAX]
220 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
221 PERF_CACHE_MAP_ALL_UNSUPPORTED,
222
223
224
225
226
227
228 [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
229 [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
230 [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
231 [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
232
233 [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL,
234
235 [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL,
236 [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL,
237
238 [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,
239 [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,
240
241 [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
242 [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
243 [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
244 [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
245};
246
247
248
249
250static const unsigned armv7_a5_perf_map[PERF_COUNT_HW_MAX] = {
251 PERF_MAP_ALL_UNSUPPORTED,
252 [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES,
253 [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED,
254 [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
255 [PERF_COUNT_HW_CACHE_MISSES] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
256 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE,
257 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
258};
259
260static const unsigned armv7_a5_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
261 [PERF_COUNT_HW_CACHE_OP_MAX]
262 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
263 PERF_CACHE_MAP_ALL_UNSUPPORTED,
264
265 [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
266 [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
267 [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
268 [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
269 [C(L1D)][C(OP_PREFETCH)][C(RESULT_ACCESS)] = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL,
270 [C(L1D)][C(OP_PREFETCH)][C(RESULT_MISS)] = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL_DROP,
271
272 [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_ICACHE_ACCESS,
273 [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL,
274
275
276
277
278 [C(L1I)][C(OP_PREFETCH)][C(RESULT_ACCESS)] = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL,
279 [C(L1I)][C(OP_PREFETCH)][C(RESULT_MISS)] = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL_DROP,
280
281 [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL,
282 [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL,
283
284 [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,
285 [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,
286
287 [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
288 [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
289 [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
290 [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
291};
292
293
294
295
296static const unsigned armv7_a15_perf_map[PERF_COUNT_HW_MAX] = {
297 PERF_MAP_ALL_UNSUPPORTED,
298 [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES,
299 [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED,
300 [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
301 [PERF_COUNT_HW_CACHE_MISSES] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
302 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_A15_PERFCTR_PC_WRITE_SPEC,
303 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
304 [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_BUS_CYCLES,
305};
306
307static const unsigned armv7_a15_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
308 [PERF_COUNT_HW_CACHE_OP_MAX]
309 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
310 PERF_CACHE_MAP_ALL_UNSUPPORTED,
311
312 [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_READ,
313 [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_READ,
314 [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_WRITE,
315 [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_WRITE,
316
317
318
319
320
321
322 [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_ICACHE_ACCESS,
323 [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL,
324
325 [C(LL)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_READ,
326 [C(LL)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_A15_PERFCTR_L2_CACHE_REFILL_READ,
327 [C(LL)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_WRITE,
328 [C(LL)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_A15_PERFCTR_L2_CACHE_REFILL_WRITE,
329
330 [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_A15_PERFCTR_DTLB_REFILL_L1_READ,
331 [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_A15_PERFCTR_DTLB_REFILL_L1_WRITE,
332
333 [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,
334 [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,
335
336 [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
337 [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
338 [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
339 [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
340};
341
342
343
344
345static const unsigned armv7_a7_perf_map[PERF_COUNT_HW_MAX] = {
346 PERF_MAP_ALL_UNSUPPORTED,
347 [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES,
348 [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED,
349 [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
350 [PERF_COUNT_HW_CACHE_MISSES] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
351 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE,
352 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
353 [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_BUS_CYCLES,
354};
355
356static const unsigned armv7_a7_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
357 [PERF_COUNT_HW_CACHE_OP_MAX]
358 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
359 PERF_CACHE_MAP_ALL_UNSUPPORTED,
360
361
362
363
364
365
366 [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
367 [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
368 [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
369 [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
370
371 [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_ICACHE_ACCESS,
372 [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL,
373
374 [C(LL)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L2_CACHE_ACCESS,
375 [C(LL)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L2_CACHE_REFILL,
376 [C(LL)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L2_CACHE_ACCESS,
377 [C(LL)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L2_CACHE_REFILL,
378
379 [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL,
380 [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL,
381
382 [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,
383 [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,
384
385 [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
386 [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
387 [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
388 [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
389};
390
391
392
393
394static const unsigned armv7_a12_perf_map[PERF_COUNT_HW_MAX] = {
395 PERF_MAP_ALL_UNSUPPORTED,
396 [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES,
397 [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED,
398 [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
399 [PERF_COUNT_HW_CACHE_MISSES] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
400 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_A12_PERFCTR_PC_WRITE_SPEC,
401 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
402 [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_BUS_CYCLES,
403};
404
405static const unsigned armv7_a12_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
406 [PERF_COUNT_HW_CACHE_OP_MAX]
407 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
408 PERF_CACHE_MAP_ALL_UNSUPPORTED,
409
410 [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_A12_PERFCTR_L1_DCACHE_ACCESS_READ,
411 [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
412 [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_A12_PERFCTR_L1_DCACHE_ACCESS_WRITE,
413 [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
414
415
416
417
418
419
420 [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_ICACHE_ACCESS,
421 [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL,
422
423 [C(LL)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_A12_PERFCTR_L2_CACHE_ACCESS_READ,
424 [C(LL)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L2_CACHE_REFILL,
425 [C(LL)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_A12_PERFCTR_L2_CACHE_ACCESS_WRITE,
426 [C(LL)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L2_CACHE_REFILL,
427
428 [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL,
429 [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL,
430 [C(DTLB)][C(OP_PREFETCH)][C(RESULT_MISS)] = ARMV7_A12_PERFCTR_PF_TLB_REFILL,
431
432 [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,
433 [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,
434
435 [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
436 [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
437 [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
438 [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
439};
440
441
442
443
444static const unsigned krait_perf_map[PERF_COUNT_HW_MAX] = {
445 PERF_MAP_ALL_UNSUPPORTED,
446 [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES,
447 [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED,
448 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE,
449 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
450 [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_CLOCK_CYCLES,
451};
452
453static const unsigned krait_perf_map_no_branch[PERF_COUNT_HW_MAX] = {
454 PERF_MAP_ALL_UNSUPPORTED,
455 [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES,
456 [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED,
457 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
458 [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_CLOCK_CYCLES,
459};
460
461static const unsigned krait_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
462 [PERF_COUNT_HW_CACHE_OP_MAX]
463 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
464 PERF_CACHE_MAP_ALL_UNSUPPORTED,
465
466
467
468
469
470
471 [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
472 [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
473 [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
474 [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
475
476 [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = KRAIT_PERFCTR_L1_ICACHE_ACCESS,
477 [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = KRAIT_PERFCTR_L1_ICACHE_MISS,
478
479 [C(DTLB)][C(OP_READ)][C(RESULT_ACCESS)] = KRAIT_PERFCTR_L1_DTLB_ACCESS,
480 [C(DTLB)][C(OP_WRITE)][C(RESULT_ACCESS)] = KRAIT_PERFCTR_L1_DTLB_ACCESS,
481
482 [C(ITLB)][C(OP_READ)][C(RESULT_ACCESS)] = KRAIT_PERFCTR_L1_ITLB_ACCESS,
483 [C(ITLB)][C(OP_WRITE)][C(RESULT_ACCESS)] = KRAIT_PERFCTR_L1_ITLB_ACCESS,
484
485 [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
486 [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
487 [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
488 [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
489};
490
491
492
493
494static const unsigned scorpion_perf_map[PERF_COUNT_HW_MAX] = {
495 PERF_MAP_ALL_UNSUPPORTED,
496 [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES,
497 [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED,
498 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE,
499 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
500 [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_CLOCK_CYCLES,
501};
502
503static const unsigned scorpion_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
504 [PERF_COUNT_HW_CACHE_OP_MAX]
505 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
506 PERF_CACHE_MAP_ALL_UNSUPPORTED,
507
508
509
510
511
512 [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
513 [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
514 [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
515 [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
516 [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = SCORPION_ICACHE_ACCESS,
517 [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = SCORPION_ICACHE_MISS,
518
519
520
521
522 [C(DTLB)][C(OP_READ)][C(RESULT_ACCESS)] = SCORPION_DTLB_ACCESS,
523 [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = SCORPION_DTLB_MISS,
524 [C(DTLB)][C(OP_WRITE)][C(RESULT_ACCESS)] = SCORPION_DTLB_ACCESS,
525 [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = SCORPION_DTLB_MISS,
526 [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = SCORPION_ITLB_MISS,
527 [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = SCORPION_ITLB_MISS,
528 [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
529 [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
530 [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
531 [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
532};
533
534PMU_FORMAT_ATTR(event, "config:0-7");
535
536static struct attribute *armv7_pmu_format_attrs[] = {
537 &format_attr_event.attr,
538 NULL,
539};
540
541static struct attribute_group armv7_pmu_format_attr_group = {
542 .name = "format",
543 .attrs = armv7_pmu_format_attrs,
544};
545
546#define ARMV7_EVENT_ATTR_RESOLVE(m) #m
547#define ARMV7_EVENT_ATTR(name, config) \
548 PMU_EVENT_ATTR_STRING(name, armv7_event_attr_##name, \
549 "event=" ARMV7_EVENT_ATTR_RESOLVE(config))
550
551ARMV7_EVENT_ATTR(sw_incr, ARMV7_PERFCTR_PMNC_SW_INCR);
552ARMV7_EVENT_ATTR(l1i_cache_refill, ARMV7_PERFCTR_L1_ICACHE_REFILL);
553ARMV7_EVENT_ATTR(l1i_tlb_refill, ARMV7_PERFCTR_ITLB_REFILL);
554ARMV7_EVENT_ATTR(l1d_cache_refill, ARMV7_PERFCTR_L1_DCACHE_REFILL);
555ARMV7_EVENT_ATTR(l1d_cache, ARMV7_PERFCTR_L1_DCACHE_ACCESS);
556ARMV7_EVENT_ATTR(l1d_tlb_refill, ARMV7_PERFCTR_DTLB_REFILL);
557ARMV7_EVENT_ATTR(ld_retired, ARMV7_PERFCTR_MEM_READ);
558ARMV7_EVENT_ATTR(st_retired, ARMV7_PERFCTR_MEM_WRITE);
559ARMV7_EVENT_ATTR(inst_retired, ARMV7_PERFCTR_INSTR_EXECUTED);
560ARMV7_EVENT_ATTR(exc_taken, ARMV7_PERFCTR_EXC_TAKEN);
561ARMV7_EVENT_ATTR(exc_return, ARMV7_PERFCTR_EXC_EXECUTED);
562ARMV7_EVENT_ATTR(cid_write_retired, ARMV7_PERFCTR_CID_WRITE);
563ARMV7_EVENT_ATTR(pc_write_retired, ARMV7_PERFCTR_PC_WRITE);
564ARMV7_EVENT_ATTR(br_immed_retired, ARMV7_PERFCTR_PC_IMM_BRANCH);
565ARMV7_EVENT_ATTR(br_return_retired, ARMV7_PERFCTR_PC_PROC_RETURN);
566ARMV7_EVENT_ATTR(unaligned_ldst_retired, ARMV7_PERFCTR_MEM_UNALIGNED_ACCESS);
567ARMV7_EVENT_ATTR(br_mis_pred, ARMV7_PERFCTR_PC_BRANCH_MIS_PRED);
568ARMV7_EVENT_ATTR(cpu_cycles, ARMV7_PERFCTR_CLOCK_CYCLES);
569ARMV7_EVENT_ATTR(br_pred, ARMV7_PERFCTR_PC_BRANCH_PRED);
570
571static struct attribute *armv7_pmuv1_event_attrs[] = {
572 &armv7_event_attr_sw_incr.attr.attr,
573 &armv7_event_attr_l1i_cache_refill.attr.attr,
574 &armv7_event_attr_l1i_tlb_refill.attr.attr,
575 &armv7_event_attr_l1d_cache_refill.attr.attr,
576 &armv7_event_attr_l1d_cache.attr.attr,
577 &armv7_event_attr_l1d_tlb_refill.attr.attr,
578 &armv7_event_attr_ld_retired.attr.attr,
579 &armv7_event_attr_st_retired.attr.attr,
580 &armv7_event_attr_inst_retired.attr.attr,
581 &armv7_event_attr_exc_taken.attr.attr,
582 &armv7_event_attr_exc_return.attr.attr,
583 &armv7_event_attr_cid_write_retired.attr.attr,
584 &armv7_event_attr_pc_write_retired.attr.attr,
585 &armv7_event_attr_br_immed_retired.attr.attr,
586 &armv7_event_attr_br_return_retired.attr.attr,
587 &armv7_event_attr_unaligned_ldst_retired.attr.attr,
588 &armv7_event_attr_br_mis_pred.attr.attr,
589 &armv7_event_attr_cpu_cycles.attr.attr,
590 &armv7_event_attr_br_pred.attr.attr,
591 NULL,
592};
593
594static struct attribute_group armv7_pmuv1_events_attr_group = {
595 .name = "events",
596 .attrs = armv7_pmuv1_event_attrs,
597};
598
599static const struct attribute_group *armv7_pmuv1_attr_groups[] = {
600 &armv7_pmuv1_events_attr_group,
601 &armv7_pmu_format_attr_group,
602 NULL,
603};
604
605ARMV7_EVENT_ATTR(mem_access, ARMV7_PERFCTR_MEM_ACCESS);
606ARMV7_EVENT_ATTR(l1i_cache, ARMV7_PERFCTR_L1_ICACHE_ACCESS);
607ARMV7_EVENT_ATTR(l1d_cache_wb, ARMV7_PERFCTR_L1_DCACHE_WB);
608ARMV7_EVENT_ATTR(l2d_cache, ARMV7_PERFCTR_L2_CACHE_ACCESS);
609ARMV7_EVENT_ATTR(l2d_cache_refill, ARMV7_PERFCTR_L2_CACHE_REFILL);
610ARMV7_EVENT_ATTR(l2d_cache_wb, ARMV7_PERFCTR_L2_CACHE_WB);
611ARMV7_EVENT_ATTR(bus_access, ARMV7_PERFCTR_BUS_ACCESS);
612ARMV7_EVENT_ATTR(memory_error, ARMV7_PERFCTR_MEM_ERROR);
613ARMV7_EVENT_ATTR(inst_spec, ARMV7_PERFCTR_INSTR_SPEC);
614ARMV7_EVENT_ATTR(ttbr_write_retired, ARMV7_PERFCTR_TTBR_WRITE);
615ARMV7_EVENT_ATTR(bus_cycles, ARMV7_PERFCTR_BUS_CYCLES);
616
617static struct attribute *armv7_pmuv2_event_attrs[] = {
618 &armv7_event_attr_sw_incr.attr.attr,
619 &armv7_event_attr_l1i_cache_refill.attr.attr,
620 &armv7_event_attr_l1i_tlb_refill.attr.attr,
621 &armv7_event_attr_l1d_cache_refill.attr.attr,
622 &armv7_event_attr_l1d_cache.attr.attr,
623 &armv7_event_attr_l1d_tlb_refill.attr.attr,
624 &armv7_event_attr_ld_retired.attr.attr,
625 &armv7_event_attr_st_retired.attr.attr,
626 &armv7_event_attr_inst_retired.attr.attr,
627 &armv7_event_attr_exc_taken.attr.attr,
628 &armv7_event_attr_exc_return.attr.attr,
629 &armv7_event_attr_cid_write_retired.attr.attr,
630 &armv7_event_attr_pc_write_retired.attr.attr,
631 &armv7_event_attr_br_immed_retired.attr.attr,
632 &armv7_event_attr_br_return_retired.attr.attr,
633 &armv7_event_attr_unaligned_ldst_retired.attr.attr,
634 &armv7_event_attr_br_mis_pred.attr.attr,
635 &armv7_event_attr_cpu_cycles.attr.attr,
636 &armv7_event_attr_br_pred.attr.attr,
637 &armv7_event_attr_mem_access.attr.attr,
638 &armv7_event_attr_l1i_cache.attr.attr,
639 &armv7_event_attr_l1d_cache_wb.attr.attr,
640 &armv7_event_attr_l2d_cache.attr.attr,
641 &armv7_event_attr_l2d_cache_refill.attr.attr,
642 &armv7_event_attr_l2d_cache_wb.attr.attr,
643 &armv7_event_attr_bus_access.attr.attr,
644 &armv7_event_attr_memory_error.attr.attr,
645 &armv7_event_attr_inst_spec.attr.attr,
646 &armv7_event_attr_ttbr_write_retired.attr.attr,
647 &armv7_event_attr_bus_cycles.attr.attr,
648 NULL,
649};
650
651static struct attribute_group armv7_pmuv2_events_attr_group = {
652 .name = "events",
653 .attrs = armv7_pmuv2_event_attrs,
654};
655
656static const struct attribute_group *armv7_pmuv2_attr_groups[] = {
657 &armv7_pmuv2_events_attr_group,
658 &armv7_pmu_format_attr_group,
659 NULL,
660};
661
662
663
664
665#define ARMV7_IDX_CYCLE_COUNTER 0
666#define ARMV7_IDX_COUNTER0 1
667#define ARMV7_IDX_COUNTER_LAST(cpu_pmu) \
668 (ARMV7_IDX_CYCLE_COUNTER + cpu_pmu->num_events - 1)
669
670#define ARMV7_MAX_COUNTERS 32
671#define ARMV7_COUNTER_MASK (ARMV7_MAX_COUNTERS - 1)
672
673
674
675
676
677
678
679
680#define ARMV7_IDX_TO_COUNTER(x) \
681 (((x) - ARMV7_IDX_COUNTER0) & ARMV7_COUNTER_MASK)
682
683
684
685
686#define ARMV7_PMNC_E (1 << 0)
687#define ARMV7_PMNC_P (1 << 1)
688#define ARMV7_PMNC_C (1 << 2)
689#define ARMV7_PMNC_D (1 << 3)
690#define ARMV7_PMNC_X (1 << 4)
691#define ARMV7_PMNC_DP (1 << 5)
692#define ARMV7_PMNC_N_SHIFT 11
693#define ARMV7_PMNC_N_MASK 0x1f
694#define ARMV7_PMNC_MASK 0x3f
695
696
697
698
699#define ARMV7_FLAG_MASK 0xffffffff
700#define ARMV7_OVERFLOWED_MASK ARMV7_FLAG_MASK
701
702
703
704
705#define ARMV7_EVTYPE_MASK 0xc80000ff
706#define ARMV7_EVTYPE_EVENT 0xff
707
708
709
710
711#define ARMV7_EXCLUDE_PL1 (1 << 31)
712#define ARMV7_EXCLUDE_USER (1 << 30)
713#define ARMV7_INCLUDE_HYP (1 << 27)
714
715
716
717
718#define ARMV7_SDER_SUNIDEN BIT(1)
719
720static inline u32 armv7_pmnc_read(void)
721{
722 u32 val;
723 asm volatile("mrc p15, 0, %0, c9, c12, 0" : "=r"(val));
724 return val;
725}
726
727static inline void armv7_pmnc_write(u32 val)
728{
729 val &= ARMV7_PMNC_MASK;
730 isb();
731 asm volatile("mcr p15, 0, %0, c9, c12, 0" : : "r"(val));
732}
733
734static inline int armv7_pmnc_has_overflowed(u32 pmnc)
735{
736 return pmnc & ARMV7_OVERFLOWED_MASK;
737}
738
739static inline int armv7_pmnc_counter_valid(struct arm_pmu *cpu_pmu, int idx)
740{
741 return idx >= ARMV7_IDX_CYCLE_COUNTER &&
742 idx <= ARMV7_IDX_COUNTER_LAST(cpu_pmu);
743}
744
745static inline int armv7_pmnc_counter_has_overflowed(u32 pmnc, int idx)
746{
747 return pmnc & BIT(ARMV7_IDX_TO_COUNTER(idx));
748}
749
750static inline void armv7_pmnc_select_counter(int idx)
751{
752 u32 counter = ARMV7_IDX_TO_COUNTER(idx);
753 asm volatile("mcr p15, 0, %0, c9, c12, 5" : : "r" (counter));
754 isb();
755}
756
757static inline u32 armv7pmu_read_counter(struct perf_event *event)
758{
759 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
760 struct hw_perf_event *hwc = &event->hw;
761 int idx = hwc->idx;
762 u32 value = 0;
763
764 if (!armv7_pmnc_counter_valid(cpu_pmu, idx)) {
765 pr_err("CPU%u reading wrong counter %d\n",
766 smp_processor_id(), idx);
767 } else if (idx == ARMV7_IDX_CYCLE_COUNTER) {
768 asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r" (value));
769 } else {
770 armv7_pmnc_select_counter(idx);
771 asm volatile("mrc p15, 0, %0, c9, c13, 2" : "=r" (value));
772 }
773
774 return value;
775}
776
777static inline void armv7pmu_write_counter(struct perf_event *event, u32 value)
778{
779 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
780 struct hw_perf_event *hwc = &event->hw;
781 int idx = hwc->idx;
782
783 if (!armv7_pmnc_counter_valid(cpu_pmu, idx)) {
784 pr_err("CPU%u writing wrong counter %d\n",
785 smp_processor_id(), idx);
786 } else if (idx == ARMV7_IDX_CYCLE_COUNTER) {
787 asm volatile("mcr p15, 0, %0, c9, c13, 0" : : "r" (value));
788 } else {
789 armv7_pmnc_select_counter(idx);
790 asm volatile("mcr p15, 0, %0, c9, c13, 2" : : "r" (value));
791 }
792}
793
794static inline void armv7_pmnc_write_evtsel(int idx, u32 val)
795{
796 armv7_pmnc_select_counter(idx);
797 val &= ARMV7_EVTYPE_MASK;
798 asm volatile("mcr p15, 0, %0, c9, c13, 1" : : "r" (val));
799}
800
801static inline void armv7_pmnc_enable_counter(int idx)
802{
803 u32 counter = ARMV7_IDX_TO_COUNTER(idx);
804 asm volatile("mcr p15, 0, %0, c9, c12, 1" : : "r" (BIT(counter)));
805}
806
807static inline void armv7_pmnc_disable_counter(int idx)
808{
809 u32 counter = ARMV7_IDX_TO_COUNTER(idx);
810 asm volatile("mcr p15, 0, %0, c9, c12, 2" : : "r" (BIT(counter)));
811}
812
813static inline void armv7_pmnc_enable_intens(int idx)
814{
815 u32 counter = ARMV7_IDX_TO_COUNTER(idx);
816 asm volatile("mcr p15, 0, %0, c9, c14, 1" : : "r" (BIT(counter)));
817}
818
819static inline void armv7_pmnc_disable_intens(int idx)
820{
821 u32 counter = ARMV7_IDX_TO_COUNTER(idx);
822 asm volatile("mcr p15, 0, %0, c9, c14, 2" : : "r" (BIT(counter)));
823 isb();
824
825 asm volatile("mcr p15, 0, %0, c9, c12, 3" : : "r" (BIT(counter)));
826 isb();
827}
828
829static inline u32 armv7_pmnc_getreset_flags(void)
830{
831 u32 val;
832
833
834 asm volatile("mrc p15, 0, %0, c9, c12, 3" : "=r" (val));
835
836
837 val &= ARMV7_FLAG_MASK;
838 asm volatile("mcr p15, 0, %0, c9, c12, 3" : : "r" (val));
839
840 return val;
841}
842
843#ifdef DEBUG
844static void armv7_pmnc_dump_regs(struct arm_pmu *cpu_pmu)
845{
846 u32 val;
847 unsigned int cnt;
848
849 pr_info("PMNC registers dump:\n");
850
851 asm volatile("mrc p15, 0, %0, c9, c12, 0" : "=r" (val));
852 pr_info("PMNC =0x%08x\n", val);
853
854 asm volatile("mrc p15, 0, %0, c9, c12, 1" : "=r" (val));
855 pr_info("CNTENS=0x%08x\n", val);
856
857 asm volatile("mrc p15, 0, %0, c9, c14, 1" : "=r" (val));
858 pr_info("INTENS=0x%08x\n", val);
859
860 asm volatile("mrc p15, 0, %0, c9, c12, 3" : "=r" (val));
861 pr_info("FLAGS =0x%08x\n", val);
862
863 asm volatile("mrc p15, 0, %0, c9, c12, 5" : "=r" (val));
864 pr_info("SELECT=0x%08x\n", val);
865
866 asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r" (val));
867 pr_info("CCNT =0x%08x\n", val);
868
869 for (cnt = ARMV7_IDX_COUNTER0;
870 cnt <= ARMV7_IDX_COUNTER_LAST(cpu_pmu); cnt++) {
871 armv7_pmnc_select_counter(cnt);
872 asm volatile("mrc p15, 0, %0, c9, c13, 2" : "=r" (val));
873 pr_info("CNT[%d] count =0x%08x\n",
874 ARMV7_IDX_TO_COUNTER(cnt), val);
875 asm volatile("mrc p15, 0, %0, c9, c13, 1" : "=r" (val));
876 pr_info("CNT[%d] evtsel=0x%08x\n",
877 ARMV7_IDX_TO_COUNTER(cnt), val);
878 }
879}
880#endif
881
882static void armv7pmu_enable_event(struct perf_event *event)
883{
884 unsigned long flags;
885 struct hw_perf_event *hwc = &event->hw;
886 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
887 struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
888 int idx = hwc->idx;
889
890 if (!armv7_pmnc_counter_valid(cpu_pmu, idx)) {
891 pr_err("CPU%u enabling wrong PMNC counter IRQ enable %d\n",
892 smp_processor_id(), idx);
893 return;
894 }
895
896
897
898
899
900 raw_spin_lock_irqsave(&events->pmu_lock, flags);
901
902
903
904
905 armv7_pmnc_disable_counter(idx);
906
907
908
909
910
911
912 if (cpu_pmu->set_event_filter || idx != ARMV7_IDX_CYCLE_COUNTER)
913 armv7_pmnc_write_evtsel(idx, hwc->config_base);
914
915
916
917
918 armv7_pmnc_enable_intens(idx);
919
920
921
922
923 armv7_pmnc_enable_counter(idx);
924
925 raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
926}
927
928static void armv7pmu_disable_event(struct perf_event *event)
929{
930 unsigned long flags;
931 struct hw_perf_event *hwc = &event->hw;
932 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
933 struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
934 int idx = hwc->idx;
935
936 if (!armv7_pmnc_counter_valid(cpu_pmu, idx)) {
937 pr_err("CPU%u disabling wrong PMNC counter IRQ enable %d\n",
938 smp_processor_id(), idx);
939 return;
940 }
941
942
943
944
945 raw_spin_lock_irqsave(&events->pmu_lock, flags);
946
947
948
949
950 armv7_pmnc_disable_counter(idx);
951
952
953
954
955 armv7_pmnc_disable_intens(idx);
956
957 raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
958}
959
960static irqreturn_t armv7pmu_handle_irq(int irq_num, void *dev)
961{
962 u32 pmnc;
963 struct perf_sample_data data;
964 struct arm_pmu *cpu_pmu = (struct arm_pmu *)dev;
965 struct pmu_hw_events *cpuc = this_cpu_ptr(cpu_pmu->hw_events);
966 struct pt_regs *regs;
967 int idx;
968
969
970
971
972 pmnc = armv7_pmnc_getreset_flags();
973
974
975
976
977 if (!armv7_pmnc_has_overflowed(pmnc))
978 return IRQ_NONE;
979
980
981
982
983 regs = get_irq_regs();
984
985 for (idx = 0; idx < cpu_pmu->num_events; ++idx) {
986 struct perf_event *event = cpuc->events[idx];
987 struct hw_perf_event *hwc;
988
989
990 if (!event)
991 continue;
992
993
994
995
996
997 if (!armv7_pmnc_counter_has_overflowed(pmnc, idx))
998 continue;
999
1000 hwc = &event->hw;
1001 armpmu_event_update(event);
1002 perf_sample_data_init(&data, 0, hwc->last_period);
1003 if (!armpmu_event_set_period(event))
1004 continue;
1005
1006 if (perf_event_overflow(event, &data, regs))
1007 cpu_pmu->disable(event);
1008 }
1009
1010
1011
1012
1013
1014
1015
1016
1017 irq_work_run();
1018
1019 return IRQ_HANDLED;
1020}
1021
1022static void armv7pmu_start(struct arm_pmu *cpu_pmu)
1023{
1024 unsigned long flags;
1025 struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
1026
1027 raw_spin_lock_irqsave(&events->pmu_lock, flags);
1028
1029 armv7_pmnc_write(armv7_pmnc_read() | ARMV7_PMNC_E);
1030 raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
1031}
1032
1033static void armv7pmu_stop(struct arm_pmu *cpu_pmu)
1034{
1035 unsigned long flags;
1036 struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
1037
1038 raw_spin_lock_irqsave(&events->pmu_lock, flags);
1039
1040 armv7_pmnc_write(armv7_pmnc_read() & ~ARMV7_PMNC_E);
1041 raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
1042}
1043
1044static int armv7pmu_get_event_idx(struct pmu_hw_events *cpuc,
1045 struct perf_event *event)
1046{
1047 int idx;
1048 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
1049 struct hw_perf_event *hwc = &event->hw;
1050 unsigned long evtype = hwc->config_base & ARMV7_EVTYPE_EVENT;
1051
1052
1053 if (evtype == ARMV7_PERFCTR_CPU_CYCLES) {
1054 if (test_and_set_bit(ARMV7_IDX_CYCLE_COUNTER, cpuc->used_mask))
1055 return -EAGAIN;
1056
1057 return ARMV7_IDX_CYCLE_COUNTER;
1058 }
1059
1060
1061
1062
1063
1064 for (idx = ARMV7_IDX_COUNTER0; idx < cpu_pmu->num_events; ++idx) {
1065 if (!test_and_set_bit(idx, cpuc->used_mask))
1066 return idx;
1067 }
1068
1069
1070 return -EAGAIN;
1071}
1072
1073
1074
1075
1076static int armv7pmu_set_event_filter(struct hw_perf_event *event,
1077 struct perf_event_attr *attr)
1078{
1079 unsigned long config_base = 0;
1080
1081 if (attr->exclude_idle)
1082 return -EPERM;
1083 if (attr->exclude_user)
1084 config_base |= ARMV7_EXCLUDE_USER;
1085 if (attr->exclude_kernel)
1086 config_base |= ARMV7_EXCLUDE_PL1;
1087 if (!attr->exclude_hv)
1088 config_base |= ARMV7_INCLUDE_HYP;
1089
1090
1091
1092
1093
1094 event->config_base = config_base;
1095
1096 return 0;
1097}
1098
1099static void armv7pmu_reset(void *info)
1100{
1101 struct arm_pmu *cpu_pmu = (struct arm_pmu *)info;
1102 u32 idx, nb_cnt = cpu_pmu->num_events, val;
1103
1104 if (cpu_pmu->secure_access) {
1105 asm volatile("mrc p15, 0, %0, c1, c1, 1" : "=r" (val));
1106 val |= ARMV7_SDER_SUNIDEN;
1107 asm volatile("mcr p15, 0, %0, c1, c1, 1" : : "r" (val));
1108 }
1109
1110
1111 for (idx = ARMV7_IDX_CYCLE_COUNTER; idx < nb_cnt; ++idx) {
1112 armv7_pmnc_disable_counter(idx);
1113 armv7_pmnc_disable_intens(idx);
1114 }
1115
1116
1117 armv7_pmnc_write(ARMV7_PMNC_P | ARMV7_PMNC_C);
1118}
1119
1120static int armv7_a8_map_event(struct perf_event *event)
1121{
1122 return armpmu_map_event(event, &armv7_a8_perf_map,
1123 &armv7_a8_perf_cache_map, 0xFF);
1124}
1125
1126static int armv7_a9_map_event(struct perf_event *event)
1127{
1128 return armpmu_map_event(event, &armv7_a9_perf_map,
1129 &armv7_a9_perf_cache_map, 0xFF);
1130}
1131
1132static int armv7_a5_map_event(struct perf_event *event)
1133{
1134 return armpmu_map_event(event, &armv7_a5_perf_map,
1135 &armv7_a5_perf_cache_map, 0xFF);
1136}
1137
1138static int armv7_a15_map_event(struct perf_event *event)
1139{
1140 return armpmu_map_event(event, &armv7_a15_perf_map,
1141 &armv7_a15_perf_cache_map, 0xFF);
1142}
1143
1144static int armv7_a7_map_event(struct perf_event *event)
1145{
1146 return armpmu_map_event(event, &armv7_a7_perf_map,
1147 &armv7_a7_perf_cache_map, 0xFF);
1148}
1149
1150static int armv7_a12_map_event(struct perf_event *event)
1151{
1152 return armpmu_map_event(event, &armv7_a12_perf_map,
1153 &armv7_a12_perf_cache_map, 0xFF);
1154}
1155
1156static int krait_map_event(struct perf_event *event)
1157{
1158 return armpmu_map_event(event, &krait_perf_map,
1159 &krait_perf_cache_map, 0xFFFFF);
1160}
1161
1162static int krait_map_event_no_branch(struct perf_event *event)
1163{
1164 return armpmu_map_event(event, &krait_perf_map_no_branch,
1165 &krait_perf_cache_map, 0xFFFFF);
1166}
1167
1168static int scorpion_map_event(struct perf_event *event)
1169{
1170 return armpmu_map_event(event, &scorpion_perf_map,
1171 &scorpion_perf_cache_map, 0xFFFFF);
1172}
1173
1174static void armv7pmu_init(struct arm_pmu *cpu_pmu)
1175{
1176 cpu_pmu->handle_irq = armv7pmu_handle_irq;
1177 cpu_pmu->enable = armv7pmu_enable_event;
1178 cpu_pmu->disable = armv7pmu_disable_event;
1179 cpu_pmu->read_counter = armv7pmu_read_counter;
1180 cpu_pmu->write_counter = armv7pmu_write_counter;
1181 cpu_pmu->get_event_idx = armv7pmu_get_event_idx;
1182 cpu_pmu->start = armv7pmu_start;
1183 cpu_pmu->stop = armv7pmu_stop;
1184 cpu_pmu->reset = armv7pmu_reset;
1185 cpu_pmu->max_period = (1LLU << 32) - 1;
1186};
1187
1188static void armv7_read_num_pmnc_events(void *info)
1189{
1190 int *nb_cnt = info;
1191
1192
1193 *nb_cnt = (armv7_pmnc_read() >> ARMV7_PMNC_N_SHIFT) & ARMV7_PMNC_N_MASK;
1194
1195
1196 *nb_cnt += 1;
1197}
1198
1199static int armv7_probe_num_events(struct arm_pmu *arm_pmu)
1200{
1201 return smp_call_function_any(&arm_pmu->supported_cpus,
1202 armv7_read_num_pmnc_events,
1203 &arm_pmu->num_events, 1);
1204}
1205
1206static int armv7_a8_pmu_init(struct arm_pmu *cpu_pmu)
1207{
1208 armv7pmu_init(cpu_pmu);
1209 cpu_pmu->name = "armv7_cortex_a8";
1210 cpu_pmu->map_event = armv7_a8_map_event;
1211 cpu_pmu->pmu.attr_groups = armv7_pmuv1_attr_groups;
1212 return armv7_probe_num_events(cpu_pmu);
1213}
1214
1215static int armv7_a9_pmu_init(struct arm_pmu *cpu_pmu)
1216{
1217 armv7pmu_init(cpu_pmu);
1218 cpu_pmu->name = "armv7_cortex_a9";
1219 cpu_pmu->map_event = armv7_a9_map_event;
1220 cpu_pmu->pmu.attr_groups = armv7_pmuv1_attr_groups;
1221 return armv7_probe_num_events(cpu_pmu);
1222}
1223
1224static int armv7_a5_pmu_init(struct arm_pmu *cpu_pmu)
1225{
1226 armv7pmu_init(cpu_pmu);
1227 cpu_pmu->name = "armv7_cortex_a5";
1228 cpu_pmu->map_event = armv7_a5_map_event;
1229 cpu_pmu->pmu.attr_groups = armv7_pmuv1_attr_groups;
1230 return armv7_probe_num_events(cpu_pmu);
1231}
1232
1233static int armv7_a15_pmu_init(struct arm_pmu *cpu_pmu)
1234{
1235 armv7pmu_init(cpu_pmu);
1236 cpu_pmu->name = "armv7_cortex_a15";
1237 cpu_pmu->map_event = armv7_a15_map_event;
1238 cpu_pmu->set_event_filter = armv7pmu_set_event_filter;
1239 cpu_pmu->pmu.attr_groups = armv7_pmuv2_attr_groups;
1240 return armv7_probe_num_events(cpu_pmu);
1241}
1242
1243static int armv7_a7_pmu_init(struct arm_pmu *cpu_pmu)
1244{
1245 armv7pmu_init(cpu_pmu);
1246 cpu_pmu->name = "armv7_cortex_a7";
1247 cpu_pmu->map_event = armv7_a7_map_event;
1248 cpu_pmu->set_event_filter = armv7pmu_set_event_filter;
1249 cpu_pmu->pmu.attr_groups = armv7_pmuv2_attr_groups;
1250 return armv7_probe_num_events(cpu_pmu);
1251}
1252
1253static int armv7_a12_pmu_init(struct arm_pmu *cpu_pmu)
1254{
1255 armv7pmu_init(cpu_pmu);
1256 cpu_pmu->name = "armv7_cortex_a12";
1257 cpu_pmu->map_event = armv7_a12_map_event;
1258 cpu_pmu->set_event_filter = armv7pmu_set_event_filter;
1259 cpu_pmu->pmu.attr_groups = armv7_pmuv2_attr_groups;
1260 return armv7_probe_num_events(cpu_pmu);
1261}
1262
1263static int armv7_a17_pmu_init(struct arm_pmu *cpu_pmu)
1264{
1265 int ret = armv7_a12_pmu_init(cpu_pmu);
1266 cpu_pmu->name = "armv7_cortex_a17";
1267 cpu_pmu->pmu.attr_groups = armv7_pmuv2_attr_groups;
1268 return ret;
1269}
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304#define KRAIT_EVENT (1 << 16)
1305#define VENUM_EVENT (2 << 16)
1306#define KRAIT_EVENT_MASK (KRAIT_EVENT | VENUM_EVENT)
1307#define PMRESRn_EN BIT(31)
1308
1309#define EVENT_REGION(event) (((event) >> 12) & 0xf)
1310#define EVENT_GROUP(event) ((event) & 0xf)
1311#define EVENT_CODE(event) (((event) >> 4) & 0xff)
1312#define EVENT_VENUM(event) (!!(event & VENUM_EVENT))
1313#define EVENT_CPU(event) (!!(event & KRAIT_EVENT))
1314
1315static u32 krait_read_pmresrn(int n)
1316{
1317 u32 val;
1318
1319 switch (n) {
1320 case 0:
1321 asm volatile("mrc p15, 1, %0, c9, c15, 0" : "=r" (val));
1322 break;
1323 case 1:
1324 asm volatile("mrc p15, 1, %0, c9, c15, 1" : "=r" (val));
1325 break;
1326 case 2:
1327 asm volatile("mrc p15, 1, %0, c9, c15, 2" : "=r" (val));
1328 break;
1329 default:
1330 BUG();
1331 }
1332
1333 return val;
1334}
1335
1336static void krait_write_pmresrn(int n, u32 val)
1337{
1338 switch (n) {
1339 case 0:
1340 asm volatile("mcr p15, 1, %0, c9, c15, 0" : : "r" (val));
1341 break;
1342 case 1:
1343 asm volatile("mcr p15, 1, %0, c9, c15, 1" : : "r" (val));
1344 break;
1345 case 2:
1346 asm volatile("mcr p15, 1, %0, c9, c15, 2" : : "r" (val));
1347 break;
1348 default:
1349 BUG();
1350 }
1351}
1352
1353static u32 venum_read_pmresr(void)
1354{
1355 u32 val;
1356 asm volatile("mrc p10, 7, %0, c11, c0, 0" : "=r" (val));
1357 return val;
1358}
1359
1360static void venum_write_pmresr(u32 val)
1361{
1362 asm volatile("mcr p10, 7, %0, c11, c0, 0" : : "r" (val));
1363}
1364
1365static void venum_pre_pmresr(u32 *venum_orig_val, u32 *fp_orig_val)
1366{
1367 u32 venum_new_val;
1368 u32 fp_new_val;
1369
1370 BUG_ON(preemptible());
1371
1372 *venum_orig_val = get_copro_access();
1373 venum_new_val = *venum_orig_val | CPACC_SVC(10) | CPACC_SVC(11);
1374 set_copro_access(venum_new_val);
1375
1376
1377 *fp_orig_val = fmrx(FPEXC);
1378 fp_new_val = *fp_orig_val | FPEXC_EN;
1379 fmxr(FPEXC, fp_new_val);
1380}
1381
1382static void venum_post_pmresr(u32 venum_orig_val, u32 fp_orig_val)
1383{
1384 BUG_ON(preemptible());
1385
1386 fmxr(FPEXC, fp_orig_val);
1387 isb();
1388
1389 set_copro_access(venum_orig_val);
1390}
1391
1392static u32 krait_get_pmresrn_event(unsigned int region)
1393{
1394 static const u32 pmresrn_table[] = { KRAIT_PMRESR0_GROUP0,
1395 KRAIT_PMRESR1_GROUP0,
1396 KRAIT_PMRESR2_GROUP0 };
1397 return pmresrn_table[region];
1398}
1399
1400static void krait_evt_setup(int idx, u32 config_base)
1401{
1402 u32 val;
1403 u32 mask;
1404 u32 vval, fval;
1405 unsigned int region = EVENT_REGION(config_base);
1406 unsigned int group = EVENT_GROUP(config_base);
1407 unsigned int code = EVENT_CODE(config_base);
1408 unsigned int group_shift;
1409 bool venum_event = EVENT_VENUM(config_base);
1410
1411 group_shift = group * 8;
1412 mask = 0xff << group_shift;
1413
1414
1415 if (venum_event)
1416 val = KRAIT_VPMRESR0_GROUP0;
1417 else
1418 val = krait_get_pmresrn_event(region);
1419 val += group;
1420
1421 val |= config_base & (ARMV7_EXCLUDE_USER | ARMV7_EXCLUDE_PL1);
1422 armv7_pmnc_write_evtsel(idx, val);
1423
1424 if (venum_event) {
1425 venum_pre_pmresr(&vval, &fval);
1426 val = venum_read_pmresr();
1427 val &= ~mask;
1428 val |= code << group_shift;
1429 val |= PMRESRn_EN;
1430 venum_write_pmresr(val);
1431 venum_post_pmresr(vval, fval);
1432 } else {
1433 val = krait_read_pmresrn(region);
1434 val &= ~mask;
1435 val |= code << group_shift;
1436 val |= PMRESRn_EN;
1437 krait_write_pmresrn(region, val);
1438 }
1439}
1440
1441static u32 clear_pmresrn_group(u32 val, int group)
1442{
1443 u32 mask;
1444 int group_shift;
1445
1446 group_shift = group * 8;
1447 mask = 0xff << group_shift;
1448 val &= ~mask;
1449
1450
1451 if (val & ~PMRESRn_EN)
1452 return val |= PMRESRn_EN;
1453
1454 return 0;
1455}
1456
1457static void krait_clearpmu(u32 config_base)
1458{
1459 u32 val;
1460 u32 vval, fval;
1461 unsigned int region = EVENT_REGION(config_base);
1462 unsigned int group = EVENT_GROUP(config_base);
1463 bool venum_event = EVENT_VENUM(config_base);
1464
1465 if (venum_event) {
1466 venum_pre_pmresr(&vval, &fval);
1467 val = venum_read_pmresr();
1468 val = clear_pmresrn_group(val, group);
1469 venum_write_pmresr(val);
1470 venum_post_pmresr(vval, fval);
1471 } else {
1472 val = krait_read_pmresrn(region);
1473 val = clear_pmresrn_group(val, group);
1474 krait_write_pmresrn(region, val);
1475 }
1476}
1477
1478static void krait_pmu_disable_event(struct perf_event *event)
1479{
1480 unsigned long flags;
1481 struct hw_perf_event *hwc = &event->hw;
1482 int idx = hwc->idx;
1483 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
1484 struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
1485
1486
1487 raw_spin_lock_irqsave(&events->pmu_lock, flags);
1488
1489
1490 armv7_pmnc_disable_counter(idx);
1491
1492
1493
1494
1495 if (hwc->config_base & KRAIT_EVENT_MASK)
1496 krait_clearpmu(hwc->config_base);
1497
1498
1499 armv7_pmnc_disable_intens(idx);
1500
1501 raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
1502}
1503
1504static void krait_pmu_enable_event(struct perf_event *event)
1505{
1506 unsigned long flags;
1507 struct hw_perf_event *hwc = &event->hw;
1508 int idx = hwc->idx;
1509 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
1510 struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
1511
1512
1513
1514
1515
1516 raw_spin_lock_irqsave(&events->pmu_lock, flags);
1517
1518
1519 armv7_pmnc_disable_counter(idx);
1520
1521
1522
1523
1524
1525
1526 if (hwc->config_base & KRAIT_EVENT_MASK)
1527 krait_evt_setup(idx, hwc->config_base);
1528 else
1529 armv7_pmnc_write_evtsel(idx, hwc->config_base);
1530
1531
1532 armv7_pmnc_enable_intens(idx);
1533
1534
1535 armv7_pmnc_enable_counter(idx);
1536
1537 raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
1538}
1539
1540static void krait_pmu_reset(void *info)
1541{
1542 u32 vval, fval;
1543 struct arm_pmu *cpu_pmu = info;
1544 u32 idx, nb_cnt = cpu_pmu->num_events;
1545
1546 armv7pmu_reset(info);
1547
1548
1549 krait_write_pmresrn(0, 0);
1550 krait_write_pmresrn(1, 0);
1551 krait_write_pmresrn(2, 0);
1552
1553 venum_pre_pmresr(&vval, &fval);
1554 venum_write_pmresr(0);
1555 venum_post_pmresr(vval, fval);
1556
1557
1558 for (idx = ARMV7_IDX_CYCLE_COUNTER; idx < nb_cnt; ++idx) {
1559 armv7_pmnc_select_counter(idx);
1560 asm volatile("mcr p15, 0, %0, c9, c15, 0" : : "r" (0));
1561 }
1562
1563}
1564
1565static int krait_event_to_bit(struct perf_event *event, unsigned int region,
1566 unsigned int group)
1567{
1568 int bit;
1569 struct hw_perf_event *hwc = &event->hw;
1570 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
1571
1572 if (hwc->config_base & VENUM_EVENT)
1573 bit = KRAIT_VPMRESR0_GROUP0;
1574 else
1575 bit = krait_get_pmresrn_event(region);
1576 bit -= krait_get_pmresrn_event(0);
1577 bit += group;
1578
1579
1580
1581
1582 bit += ARMV7_IDX_COUNTER_LAST(cpu_pmu) + 1;
1583
1584 return bit;
1585}
1586
1587
1588
1589
1590
1591static int krait_pmu_get_event_idx(struct pmu_hw_events *cpuc,
1592 struct perf_event *event)
1593{
1594 int idx;
1595 int bit = -1;
1596 struct hw_perf_event *hwc = &event->hw;
1597 unsigned int region = EVENT_REGION(hwc->config_base);
1598 unsigned int code = EVENT_CODE(hwc->config_base);
1599 unsigned int group = EVENT_GROUP(hwc->config_base);
1600 bool venum_event = EVENT_VENUM(hwc->config_base);
1601 bool krait_event = EVENT_CPU(hwc->config_base);
1602
1603 if (venum_event || krait_event) {
1604
1605 if (group > 3 || region > 2)
1606 return -EINVAL;
1607 if (venum_event && (code & 0xe0))
1608 return -EINVAL;
1609
1610 bit = krait_event_to_bit(event, region, group);
1611 if (test_and_set_bit(bit, cpuc->used_mask))
1612 return -EAGAIN;
1613 }
1614
1615 idx = armv7pmu_get_event_idx(cpuc, event);
1616 if (idx < 0 && bit >= 0)
1617 clear_bit(bit, cpuc->used_mask);
1618
1619 return idx;
1620}
1621
1622static void krait_pmu_clear_event_idx(struct pmu_hw_events *cpuc,
1623 struct perf_event *event)
1624{
1625 int bit;
1626 struct hw_perf_event *hwc = &event->hw;
1627 unsigned int region = EVENT_REGION(hwc->config_base);
1628 unsigned int group = EVENT_GROUP(hwc->config_base);
1629 bool venum_event = EVENT_VENUM(hwc->config_base);
1630 bool krait_event = EVENT_CPU(hwc->config_base);
1631
1632 if (venum_event || krait_event) {
1633 bit = krait_event_to_bit(event, region, group);
1634 clear_bit(bit, cpuc->used_mask);
1635 }
1636}
1637
1638static int krait_pmu_init(struct arm_pmu *cpu_pmu)
1639{
1640 armv7pmu_init(cpu_pmu);
1641 cpu_pmu->name = "armv7_krait";
1642
1643 if (of_property_read_bool(cpu_pmu->plat_device->dev.of_node,
1644 "qcom,no-pc-write"))
1645 cpu_pmu->map_event = krait_map_event_no_branch;
1646 else
1647 cpu_pmu->map_event = krait_map_event;
1648 cpu_pmu->set_event_filter = armv7pmu_set_event_filter;
1649 cpu_pmu->reset = krait_pmu_reset;
1650 cpu_pmu->enable = krait_pmu_enable_event;
1651 cpu_pmu->disable = krait_pmu_disable_event;
1652 cpu_pmu->get_event_idx = krait_pmu_get_event_idx;
1653 cpu_pmu->clear_event_idx = krait_pmu_clear_event_idx;
1654 return armv7_probe_num_events(cpu_pmu);
1655}
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693static u32 scorpion_read_pmresrn(int n)
1694{
1695 u32 val;
1696
1697 switch (n) {
1698 case 0:
1699 asm volatile("mrc p15, 0, %0, c15, c0, 0" : "=r" (val));
1700 break;
1701 case 1:
1702 asm volatile("mrc p15, 1, %0, c15, c0, 0" : "=r" (val));
1703 break;
1704 case 2:
1705 asm volatile("mrc p15, 2, %0, c15, c0, 0" : "=r" (val));
1706 break;
1707 case 3:
1708 asm volatile("mrc p15, 3, %0, c15, c2, 0" : "=r" (val));
1709 break;
1710 default:
1711 BUG();
1712 }
1713
1714 return val;
1715}
1716
1717static void scorpion_write_pmresrn(int n, u32 val)
1718{
1719 switch (n) {
1720 case 0:
1721 asm volatile("mcr p15, 0, %0, c15, c0, 0" : : "r" (val));
1722 break;
1723 case 1:
1724 asm volatile("mcr p15, 1, %0, c15, c0, 0" : : "r" (val));
1725 break;
1726 case 2:
1727 asm volatile("mcr p15, 2, %0, c15, c0, 0" : : "r" (val));
1728 break;
1729 case 3:
1730 asm volatile("mcr p15, 3, %0, c15, c2, 0" : : "r" (val));
1731 break;
1732 default:
1733 BUG();
1734 }
1735}
1736
1737static u32 scorpion_get_pmresrn_event(unsigned int region)
1738{
1739 static const u32 pmresrn_table[] = { SCORPION_LPM0_GROUP0,
1740 SCORPION_LPM1_GROUP0,
1741 SCORPION_LPM2_GROUP0,
1742 SCORPION_L2LPM_GROUP0 };
1743 return pmresrn_table[region];
1744}
1745
1746static void scorpion_evt_setup(int idx, u32 config_base)
1747{
1748 u32 val;
1749 u32 mask;
1750 u32 vval, fval;
1751 unsigned int region = EVENT_REGION(config_base);
1752 unsigned int group = EVENT_GROUP(config_base);
1753 unsigned int code = EVENT_CODE(config_base);
1754 unsigned int group_shift;
1755 bool venum_event = EVENT_VENUM(config_base);
1756
1757 group_shift = group * 8;
1758 mask = 0xff << group_shift;
1759
1760
1761 if (venum_event)
1762 val = SCORPION_VLPM_GROUP0;
1763 else
1764 val = scorpion_get_pmresrn_event(region);
1765 val += group;
1766
1767 val |= config_base & (ARMV7_EXCLUDE_USER | ARMV7_EXCLUDE_PL1);
1768 armv7_pmnc_write_evtsel(idx, val);
1769
1770 asm volatile("mcr p15, 0, %0, c9, c15, 0" : : "r" (0));
1771
1772 if (venum_event) {
1773 venum_pre_pmresr(&vval, &fval);
1774 val = venum_read_pmresr();
1775 val &= ~mask;
1776 val |= code << group_shift;
1777 val |= PMRESRn_EN;
1778 venum_write_pmresr(val);
1779 venum_post_pmresr(vval, fval);
1780 } else {
1781 val = scorpion_read_pmresrn(region);
1782 val &= ~mask;
1783 val |= code << group_shift;
1784 val |= PMRESRn_EN;
1785 scorpion_write_pmresrn(region, val);
1786 }
1787}
1788
1789static void scorpion_clearpmu(u32 config_base)
1790{
1791 u32 val;
1792 u32 vval, fval;
1793 unsigned int region = EVENT_REGION(config_base);
1794 unsigned int group = EVENT_GROUP(config_base);
1795 bool venum_event = EVENT_VENUM(config_base);
1796
1797 if (venum_event) {
1798 venum_pre_pmresr(&vval, &fval);
1799 val = venum_read_pmresr();
1800 val = clear_pmresrn_group(val, group);
1801 venum_write_pmresr(val);
1802 venum_post_pmresr(vval, fval);
1803 } else {
1804 val = scorpion_read_pmresrn(region);
1805 val = clear_pmresrn_group(val, group);
1806 scorpion_write_pmresrn(region, val);
1807 }
1808}
1809
1810static void scorpion_pmu_disable_event(struct perf_event *event)
1811{
1812 unsigned long flags;
1813 struct hw_perf_event *hwc = &event->hw;
1814 int idx = hwc->idx;
1815 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
1816 struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
1817
1818
1819 raw_spin_lock_irqsave(&events->pmu_lock, flags);
1820
1821
1822 armv7_pmnc_disable_counter(idx);
1823
1824
1825
1826
1827 if (hwc->config_base & KRAIT_EVENT_MASK)
1828 scorpion_clearpmu(hwc->config_base);
1829
1830
1831 armv7_pmnc_disable_intens(idx);
1832
1833 raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
1834}
1835
1836static void scorpion_pmu_enable_event(struct perf_event *event)
1837{
1838 unsigned long flags;
1839 struct hw_perf_event *hwc = &event->hw;
1840 int idx = hwc->idx;
1841 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
1842 struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
1843
1844
1845
1846
1847
1848 raw_spin_lock_irqsave(&events->pmu_lock, flags);
1849
1850
1851 armv7_pmnc_disable_counter(idx);
1852
1853
1854
1855
1856
1857
1858 if (hwc->config_base & KRAIT_EVENT_MASK)
1859 scorpion_evt_setup(idx, hwc->config_base);
1860 else if (idx != ARMV7_IDX_CYCLE_COUNTER)
1861 armv7_pmnc_write_evtsel(idx, hwc->config_base);
1862
1863
1864 armv7_pmnc_enable_intens(idx);
1865
1866
1867 armv7_pmnc_enable_counter(idx);
1868
1869 raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
1870}
1871
1872static void scorpion_pmu_reset(void *info)
1873{
1874 u32 vval, fval;
1875 struct arm_pmu *cpu_pmu = info;
1876 u32 idx, nb_cnt = cpu_pmu->num_events;
1877
1878 armv7pmu_reset(info);
1879
1880
1881 scorpion_write_pmresrn(0, 0);
1882 scorpion_write_pmresrn(1, 0);
1883 scorpion_write_pmresrn(2, 0);
1884 scorpion_write_pmresrn(3, 0);
1885
1886 venum_pre_pmresr(&vval, &fval);
1887 venum_write_pmresr(0);
1888 venum_post_pmresr(vval, fval);
1889
1890
1891 for (idx = ARMV7_IDX_CYCLE_COUNTER; idx < nb_cnt; ++idx) {
1892 armv7_pmnc_select_counter(idx);
1893 asm volatile("mcr p15, 0, %0, c9, c15, 0" : : "r" (0));
1894 }
1895}
1896
1897static int scorpion_event_to_bit(struct perf_event *event, unsigned int region,
1898 unsigned int group)
1899{
1900 int bit;
1901 struct hw_perf_event *hwc = &event->hw;
1902 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
1903
1904 if (hwc->config_base & VENUM_EVENT)
1905 bit = SCORPION_VLPM_GROUP0;
1906 else
1907 bit = scorpion_get_pmresrn_event(region);
1908 bit -= scorpion_get_pmresrn_event(0);
1909 bit += group;
1910
1911
1912
1913
1914 bit += ARMV7_IDX_COUNTER_LAST(cpu_pmu) + 1;
1915
1916 return bit;
1917}
1918
1919
1920
1921
1922
1923static int scorpion_pmu_get_event_idx(struct pmu_hw_events *cpuc,
1924 struct perf_event *event)
1925{
1926 int idx;
1927 int bit = -1;
1928 struct hw_perf_event *hwc = &event->hw;
1929 unsigned int region = EVENT_REGION(hwc->config_base);
1930 unsigned int group = EVENT_GROUP(hwc->config_base);
1931 bool venum_event = EVENT_VENUM(hwc->config_base);
1932 bool scorpion_event = EVENT_CPU(hwc->config_base);
1933
1934 if (venum_event || scorpion_event) {
1935
1936 if (group > 3 || region > 3)
1937 return -EINVAL;
1938
1939 bit = scorpion_event_to_bit(event, region, group);
1940 if (test_and_set_bit(bit, cpuc->used_mask))
1941 return -EAGAIN;
1942 }
1943
1944 idx = armv7pmu_get_event_idx(cpuc, event);
1945 if (idx < 0 && bit >= 0)
1946 clear_bit(bit, cpuc->used_mask);
1947
1948 return idx;
1949}
1950
1951static void scorpion_pmu_clear_event_idx(struct pmu_hw_events *cpuc,
1952 struct perf_event *event)
1953{
1954 int bit;
1955 struct hw_perf_event *hwc = &event->hw;
1956 unsigned int region = EVENT_REGION(hwc->config_base);
1957 unsigned int group = EVENT_GROUP(hwc->config_base);
1958 bool venum_event = EVENT_VENUM(hwc->config_base);
1959 bool scorpion_event = EVENT_CPU(hwc->config_base);
1960
1961 if (venum_event || scorpion_event) {
1962 bit = scorpion_event_to_bit(event, region, group);
1963 clear_bit(bit, cpuc->used_mask);
1964 }
1965}
1966
1967static int scorpion_pmu_init(struct arm_pmu *cpu_pmu)
1968{
1969 armv7pmu_init(cpu_pmu);
1970 cpu_pmu->name = "armv7_scorpion";
1971 cpu_pmu->map_event = scorpion_map_event;
1972 cpu_pmu->reset = scorpion_pmu_reset;
1973 cpu_pmu->enable = scorpion_pmu_enable_event;
1974 cpu_pmu->disable = scorpion_pmu_disable_event;
1975 cpu_pmu->get_event_idx = scorpion_pmu_get_event_idx;
1976 cpu_pmu->clear_event_idx = scorpion_pmu_clear_event_idx;
1977 return armv7_probe_num_events(cpu_pmu);
1978}
1979
1980static int scorpion_mp_pmu_init(struct arm_pmu *cpu_pmu)
1981{
1982 armv7pmu_init(cpu_pmu);
1983 cpu_pmu->name = "armv7_scorpion_mp";
1984 cpu_pmu->map_event = scorpion_map_event;
1985 cpu_pmu->reset = scorpion_pmu_reset;
1986 cpu_pmu->enable = scorpion_pmu_enable_event;
1987 cpu_pmu->disable = scorpion_pmu_disable_event;
1988 cpu_pmu->get_event_idx = scorpion_pmu_get_event_idx;
1989 cpu_pmu->clear_event_idx = scorpion_pmu_clear_event_idx;
1990 return armv7_probe_num_events(cpu_pmu);
1991}
1992
1993static const struct of_device_id armv7_pmu_of_device_ids[] = {
1994 {.compatible = "arm,cortex-a17-pmu", .data = armv7_a17_pmu_init},
1995 {.compatible = "arm,cortex-a15-pmu", .data = armv7_a15_pmu_init},
1996 {.compatible = "arm,cortex-a12-pmu", .data = armv7_a12_pmu_init},
1997 {.compatible = "arm,cortex-a9-pmu", .data = armv7_a9_pmu_init},
1998 {.compatible = "arm,cortex-a8-pmu", .data = armv7_a8_pmu_init},
1999 {.compatible = "arm,cortex-a7-pmu", .data = armv7_a7_pmu_init},
2000 {.compatible = "arm,cortex-a5-pmu", .data = armv7_a5_pmu_init},
2001 {.compatible = "qcom,krait-pmu", .data = krait_pmu_init},
2002 {.compatible = "qcom,scorpion-pmu", .data = scorpion_pmu_init},
2003 {.compatible = "qcom,scorpion-mp-pmu", .data = scorpion_mp_pmu_init},
2004 {},
2005};
2006
2007static const struct pmu_probe_info armv7_pmu_probe_table[] = {
2008 ARM_PMU_PROBE(ARM_CPU_PART_CORTEX_A8, armv7_a8_pmu_init),
2009 ARM_PMU_PROBE(ARM_CPU_PART_CORTEX_A9, armv7_a9_pmu_init),
2010 { }
2011};
2012
2013
2014static int armv7_pmu_device_probe(struct platform_device *pdev)
2015{
2016 return arm_pmu_device_probe(pdev, armv7_pmu_of_device_ids,
2017 armv7_pmu_probe_table);
2018}
2019
2020static struct platform_driver armv7_pmu_driver = {
2021 .driver = {
2022 .name = "armv7-pmu",
2023 .of_match_table = armv7_pmu_of_device_ids,
2024 },
2025 .probe = armv7_pmu_device_probe,
2026};
2027
2028static int __init register_armv7_pmu_driver(void)
2029{
2030 return platform_driver_register(&armv7_pmu_driver);
2031}
2032device_initcall(register_armv7_pmu_driver);
2033#endif
2034