1
2
3
4
5
6
7
8
9
10
11#include <linux/init.h>
12#include <linux/kernel.h>
13#include <linux/device.h>
14#include <linux/smp.h>
15#include <linux/cpu.h>
16#include <linux/clockchips.h>
17#include <linux/interrupt.h>
18#include <linux/of_irq.h>
19#include <linux/io.h>
20
21#include <asm/arch_timer.h>
22#include <asm/virt.h>
23
24#include <clocksource/arm_arch_timer.h>
25
26static u32 arch_timer_rate;
27
28enum ppi_nr {
29 PHYS_SECURE_PPI,
30 PHYS_NONSECURE_PPI,
31 VIRT_PPI,
32 HYP_PPI,
33 MAX_TIMER_PPI
34};
35
36static int arch_timer_ppi[MAX_TIMER_PPI];
37
38static struct clock_event_device __percpu *arch_timer_evt;
39
40static bool arch_timer_use_virtual = true;
41
42
43
44
45
46static inline irqreturn_t timer_handler(const int access,
47 struct clock_event_device *evt)
48{
49 unsigned long ctrl;
50 ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL);
51 if (ctrl & ARCH_TIMER_CTRL_IT_STAT) {
52 ctrl |= ARCH_TIMER_CTRL_IT_MASK;
53 arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl);
54 evt->event_handler(evt);
55 return IRQ_HANDLED;
56 }
57
58 return IRQ_NONE;
59}
60
61static irqreturn_t arch_timer_handler_virt(int irq, void *dev_id)
62{
63 struct clock_event_device *evt = dev_id;
64
65 return timer_handler(ARCH_TIMER_VIRT_ACCESS, evt);
66}
67
68static irqreturn_t arch_timer_handler_phys(int irq, void *dev_id)
69{
70 struct clock_event_device *evt = dev_id;
71
72 return timer_handler(ARCH_TIMER_PHYS_ACCESS, evt);
73}
74
75static inline void timer_set_mode(const int access, int mode)
76{
77 unsigned long ctrl;
78 switch (mode) {
79 case CLOCK_EVT_MODE_UNUSED:
80 case CLOCK_EVT_MODE_SHUTDOWN:
81 ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL);
82 ctrl &= ~ARCH_TIMER_CTRL_ENABLE;
83 arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl);
84 break;
85 default:
86 break;
87 }
88}
89
90static void arch_timer_set_mode_virt(enum clock_event_mode mode,
91 struct clock_event_device *clk)
92{
93 timer_set_mode(ARCH_TIMER_VIRT_ACCESS, mode);
94}
95
96static void arch_timer_set_mode_phys(enum clock_event_mode mode,
97 struct clock_event_device *clk)
98{
99 timer_set_mode(ARCH_TIMER_PHYS_ACCESS, mode);
100}
101
102static inline void set_next_event(const int access, unsigned long evt)
103{
104 unsigned long ctrl;
105 ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL);
106 ctrl |= ARCH_TIMER_CTRL_ENABLE;
107 ctrl &= ~ARCH_TIMER_CTRL_IT_MASK;
108 arch_timer_reg_write(access, ARCH_TIMER_REG_TVAL, evt);
109 arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl);
110}
111
112static int arch_timer_set_next_event_virt(unsigned long evt,
113 struct clock_event_device *unused)
114{
115 set_next_event(ARCH_TIMER_VIRT_ACCESS, evt);
116 return 0;
117}
118
119static int arch_timer_set_next_event_phys(unsigned long evt,
120 struct clock_event_device *unused)
121{
122 set_next_event(ARCH_TIMER_PHYS_ACCESS, evt);
123 return 0;
124}
125
126static int arch_timer_setup(struct clock_event_device *clk)
127{
128 clk->features = CLOCK_EVT_FEAT_ONESHOT | CLOCK_EVT_FEAT_C3STOP;
129 clk->name = "arch_sys_timer";
130 clk->rating = 450;
131 if (arch_timer_use_virtual) {
132 clk->irq = arch_timer_ppi[VIRT_PPI];
133 clk->set_mode = arch_timer_set_mode_virt;
134 clk->set_next_event = arch_timer_set_next_event_virt;
135 } else {
136 clk->irq = arch_timer_ppi[PHYS_SECURE_PPI];
137 clk->set_mode = arch_timer_set_mode_phys;
138 clk->set_next_event = arch_timer_set_next_event_phys;
139 }
140
141 clk->cpumask = cpumask_of(smp_processor_id());
142
143 clk->set_mode(CLOCK_EVT_MODE_SHUTDOWN, NULL);
144
145 clockevents_config_and_register(clk, arch_timer_rate,
146 0xf, 0x7fffffff);
147
148 if (arch_timer_use_virtual)
149 enable_percpu_irq(arch_timer_ppi[VIRT_PPI], 0);
150 else {
151 enable_percpu_irq(arch_timer_ppi[PHYS_SECURE_PPI], 0);
152 if (arch_timer_ppi[PHYS_NONSECURE_PPI])
153 enable_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI], 0);
154 }
155
156 arch_counter_set_user_access();
157
158 return 0;
159}
160
161static int arch_timer_available(void)
162{
163 u32 freq;
164
165 if (arch_timer_rate == 0) {
166 freq = arch_timer_get_cntfrq();
167
168
169 if (freq == 0) {
170 pr_warn("Architected timer frequency not available\n");
171 return -EINVAL;
172 }
173
174 arch_timer_rate = freq;
175 }
176
177 pr_info_once("Architected local timer running at %lu.%02luMHz (%s).\n",
178 (unsigned long)arch_timer_rate / 1000000,
179 (unsigned long)(arch_timer_rate / 10000) % 100,
180 arch_timer_use_virtual ? "virt" : "phys");
181 return 0;
182}
183
184u32 arch_timer_get_rate(void)
185{
186 return arch_timer_rate;
187}
188
189u64 arch_timer_read_counter(void)
190{
191 return arch_counter_get_cntvct();
192}
193
194static cycle_t arch_counter_read(struct clocksource *cs)
195{
196 return arch_counter_get_cntvct();
197}
198
199static cycle_t arch_counter_read_cc(const struct cyclecounter *cc)
200{
201 return arch_counter_get_cntvct();
202}
203
204static struct clocksource clocksource_counter = {
205 .name = "arch_sys_counter",
206 .rating = 400,
207 .read = arch_counter_read,
208 .mask = CLOCKSOURCE_MASK(56),
209 .flags = CLOCK_SOURCE_IS_CONTINUOUS,
210};
211
212static struct cyclecounter cyclecounter = {
213 .read = arch_counter_read_cc,
214 .mask = CLOCKSOURCE_MASK(56),
215};
216
217static struct timecounter timecounter;
218
219struct timecounter *arch_timer_get_timecounter(void)
220{
221 return &timecounter;
222}
223
224static void arch_timer_stop(struct clock_event_device *clk)
225{
226 pr_debug("arch_timer_teardown disable IRQ%d cpu #%d\n",
227 clk->irq, smp_processor_id());
228
229 if (arch_timer_use_virtual)
230 disable_percpu_irq(arch_timer_ppi[VIRT_PPI]);
231 else {
232 disable_percpu_irq(arch_timer_ppi[PHYS_SECURE_PPI]);
233 if (arch_timer_ppi[PHYS_NONSECURE_PPI])
234 disable_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI]);
235 }
236
237 clk->set_mode(CLOCK_EVT_MODE_UNUSED, clk);
238}
239
240static int arch_timer_cpu_notify(struct notifier_block *self,
241 unsigned long action, void *hcpu)
242{
243
244
245
246
247 switch (action & ~CPU_TASKS_FROZEN) {
248 case CPU_STARTING:
249 arch_timer_setup(this_cpu_ptr(arch_timer_evt));
250 break;
251 case CPU_DYING:
252 arch_timer_stop(this_cpu_ptr(arch_timer_evt));
253 break;
254 }
255
256 return NOTIFY_OK;
257}
258
259static struct notifier_block arch_timer_cpu_nb = {
260 .notifier_call = arch_timer_cpu_notify,
261};
262
263static int __init arch_timer_register(void)
264{
265 int err;
266 int ppi;
267
268 err = arch_timer_available();
269 if (err)
270 goto out;
271
272 arch_timer_evt = alloc_percpu(struct clock_event_device);
273 if (!arch_timer_evt) {
274 err = -ENOMEM;
275 goto out;
276 }
277
278 clocksource_register_hz(&clocksource_counter, arch_timer_rate);
279 cyclecounter.mult = clocksource_counter.mult;
280 cyclecounter.shift = clocksource_counter.shift;
281 timecounter_init(&timecounter, &cyclecounter,
282 arch_counter_get_cntvct());
283
284 if (arch_timer_use_virtual) {
285 ppi = arch_timer_ppi[VIRT_PPI];
286 err = request_percpu_irq(ppi, arch_timer_handler_virt,
287 "arch_timer", arch_timer_evt);
288 } else {
289 ppi = arch_timer_ppi[PHYS_SECURE_PPI];
290 err = request_percpu_irq(ppi, arch_timer_handler_phys,
291 "arch_timer", arch_timer_evt);
292 if (!err && arch_timer_ppi[PHYS_NONSECURE_PPI]) {
293 ppi = arch_timer_ppi[PHYS_NONSECURE_PPI];
294 err = request_percpu_irq(ppi, arch_timer_handler_phys,
295 "arch_timer", arch_timer_evt);
296 if (err)
297 free_percpu_irq(arch_timer_ppi[PHYS_SECURE_PPI],
298 arch_timer_evt);
299 }
300 }
301
302 if (err) {
303 pr_err("arch_timer: can't register interrupt %d (%d)\n",
304 ppi, err);
305 goto out_free;
306 }
307
308 err = register_cpu_notifier(&arch_timer_cpu_nb);
309 if (err)
310 goto out_free_irq;
311
312
313 arch_timer_setup(this_cpu_ptr(arch_timer_evt));
314
315 return 0;
316
317out_free_irq:
318 if (arch_timer_use_virtual)
319 free_percpu_irq(arch_timer_ppi[VIRT_PPI], arch_timer_evt);
320 else {
321 free_percpu_irq(arch_timer_ppi[PHYS_SECURE_PPI],
322 arch_timer_evt);
323 if (arch_timer_ppi[PHYS_NONSECURE_PPI])
324 free_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI],
325 arch_timer_evt);
326 }
327
328out_free:
329 free_percpu(arch_timer_evt);
330out:
331 return err;
332}
333
334static void __init arch_timer_init(struct device_node *np)
335{
336 u32 freq;
337 int i;
338
339 if (arch_timer_get_rate()) {
340 pr_warn("arch_timer: multiple nodes in dt, skipping\n");
341 return;
342 }
343
344
345 if (!of_property_read_u32(np, "clock-frequency", &freq))
346 arch_timer_rate = freq;
347
348 for (i = PHYS_SECURE_PPI; i < MAX_TIMER_PPI; i++)
349 arch_timer_ppi[i] = irq_of_parse_and_map(np, i);
350
351 of_node_put(np);
352
353
354
355
356
357
358
359
360
361 if (is_hyp_mode_available() || !arch_timer_ppi[VIRT_PPI]) {
362 arch_timer_use_virtual = false;
363
364 if (!arch_timer_ppi[PHYS_SECURE_PPI] ||
365 !arch_timer_ppi[PHYS_NONSECURE_PPI]) {
366 pr_warn("arch_timer: No interrupt available, giving up\n");
367 return;
368 }
369 }
370
371 arch_timer_register();
372 arch_timer_arch_init();
373}
374CLOCKSOURCE_OF_DECLARE(armv7_arch_timer, "arm,armv7-timer", arch_timer_init);
375CLOCKSOURCE_OF_DECLARE(armv8_arch_timer, "arm,armv8-timer", arch_timer_init);
376
