1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19#include <linux/cpu.h>
20#include <linux/of_irq.h>
21#include <linux/kvm.h>
22#include <linux/kvm_host.h>
23#include <linux/interrupt.h>
24
25#include <clocksource/arm_arch_timer.h>
26#include <asm/arch_timer.h>
27
28#include <kvm/arm_vgic.h>
29#include <kvm/arm_arch_timer.h>
30
31#include "trace.h"
32
33static struct timecounter *timecounter;
34static struct workqueue_struct *wqueue;
35static unsigned int host_vtimer_irq;
36
37void kvm_timer_vcpu_put(struct kvm_vcpu *vcpu)
38{
39 vcpu->arch.timer_cpu.active_cleared_last = false;
40}
41
42static cycle_t kvm_phys_timer_read(void)
43{
44 return timecounter->cc->read(timecounter->cc);
45}
46
47static bool timer_is_armed(struct arch_timer_cpu *timer)
48{
49 return timer->armed;
50}
51
52
53static void timer_arm(struct arch_timer_cpu *timer, u64 ns)
54{
55 timer->armed = true;
56 hrtimer_start(&timer->timer, ktime_add_ns(ktime_get(), ns),
57 HRTIMER_MODE_ABS);
58}
59
60static void timer_disarm(struct arch_timer_cpu *timer)
61{
62 if (timer_is_armed(timer)) {
63 hrtimer_cancel(&timer->timer);
64 cancel_work_sync(&timer->expired);
65 timer->armed = false;
66 }
67}
68
69static irqreturn_t kvm_arch_timer_handler(int irq, void *dev_id)
70{
71 struct kvm_vcpu *vcpu = *(struct kvm_vcpu **)dev_id;
72
73
74
75
76
77
78
79 pr_warn("Unexpected interrupt %d on vcpu %p\n", irq, vcpu);
80 return IRQ_HANDLED;
81}
82
83
84
85
86
87static void kvm_timer_inject_irq_work(struct work_struct *work)
88{
89 struct kvm_vcpu *vcpu;
90
91 vcpu = container_of(work, struct kvm_vcpu, arch.timer_cpu.expired);
92 vcpu->arch.timer_cpu.armed = false;
93
94 WARN_ON(!kvm_timer_should_fire(vcpu));
95
96
97
98
99
100 kvm_vcpu_kick(vcpu);
101}
102
103static u64 kvm_timer_compute_delta(struct kvm_vcpu *vcpu)
104{
105 cycle_t cval, now;
106
107 cval = vcpu->arch.timer_cpu.cntv_cval;
108 now = kvm_phys_timer_read() - vcpu->kvm->arch.timer.cntvoff;
109
110 if (now < cval) {
111 u64 ns;
112
113 ns = cyclecounter_cyc2ns(timecounter->cc,
114 cval - now,
115 timecounter->mask,
116 &timecounter->frac);
117 return ns;
118 }
119
120 return 0;
121}
122
123static enum hrtimer_restart kvm_timer_expire(struct hrtimer *hrt)
124{
125 struct arch_timer_cpu *timer;
126 struct kvm_vcpu *vcpu;
127 u64 ns;
128
129 timer = container_of(hrt, struct arch_timer_cpu, timer);
130 vcpu = container_of(timer, struct kvm_vcpu, arch.timer_cpu);
131
132
133
134
135
136
137 ns = kvm_timer_compute_delta(vcpu);
138 if (unlikely(ns)) {
139 hrtimer_forward_now(hrt, ns_to_ktime(ns));
140 return HRTIMER_RESTART;
141 }
142
143 queue_work(wqueue, &timer->expired);
144 return HRTIMER_NORESTART;
145}
146
147static bool kvm_timer_irq_can_fire(struct kvm_vcpu *vcpu)
148{
149 struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
150
151 return !(timer->cntv_ctl & ARCH_TIMER_CTRL_IT_MASK) &&
152 (timer->cntv_ctl & ARCH_TIMER_CTRL_ENABLE);
153}
154
155bool kvm_timer_should_fire(struct kvm_vcpu *vcpu)
156{
157 struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
158 cycle_t cval, now;
159
160 if (!kvm_timer_irq_can_fire(vcpu))
161 return false;
162
163 cval = timer->cntv_cval;
164 now = kvm_phys_timer_read() - vcpu->kvm->arch.timer.cntvoff;
165
166 return cval <= now;
167}
168
169static void kvm_timer_update_irq(struct kvm_vcpu *vcpu, bool new_level)
170{
171 int ret;
172 struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
173
174 BUG_ON(!vgic_initialized(vcpu->kvm));
175
176 timer->active_cleared_last = false;
177 timer->irq.level = new_level;
178 trace_kvm_timer_update_irq(vcpu->vcpu_id, timer->map->virt_irq,
179 timer->irq.level);
180 ret = kvm_vgic_inject_mapped_irq(vcpu->kvm, vcpu->vcpu_id,
181 timer->map,
182 timer->irq.level);
183 WARN_ON(ret);
184}
185
186
187
188
189
190static int kvm_timer_update_state(struct kvm_vcpu *vcpu)
191{
192 struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
193
194
195
196
197
198
199
200 if (!vgic_initialized(vcpu->kvm))
201 return -ENODEV;
202
203 if (kvm_timer_should_fire(vcpu) != timer->irq.level)
204 kvm_timer_update_irq(vcpu, !timer->irq.level);
205
206 return 0;
207}
208
209
210
211
212
213
214void kvm_timer_schedule(struct kvm_vcpu *vcpu)
215{
216 struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
217
218 BUG_ON(timer_is_armed(timer));
219
220
221
222
223
224
225 if (kvm_timer_should_fire(vcpu))
226 return;
227
228
229
230
231
232 if (!kvm_timer_irq_can_fire(vcpu))
233 return;
234
235
236 timer_arm(timer, kvm_timer_compute_delta(vcpu));
237}
238
239void kvm_timer_unschedule(struct kvm_vcpu *vcpu)
240{
241 struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
242 timer_disarm(timer);
243}
244
245
246
247
248
249
250
251
252void kvm_timer_flush_hwstate(struct kvm_vcpu *vcpu)
253{
254 struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
255 bool phys_active;
256 int ret;
257
258 if (kvm_timer_update_state(vcpu))
259 return;
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278 if (timer->irq.level || kvm_vgic_map_is_active(vcpu, timer->map))
279 phys_active = true;
280 else
281 phys_active = false;
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303 if (timer->active_cleared_last && !phys_active)
304 return;
305
306 ret = irq_set_irqchip_state(timer->map->irq,
307 IRQCHIP_STATE_ACTIVE,
308 phys_active);
309 WARN_ON(ret);
310
311 timer->active_cleared_last = !phys_active;
312}
313
314
315
316
317
318
319
320
321void kvm_timer_sync_hwstate(struct kvm_vcpu *vcpu)
322{
323 struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
324
325 BUG_ON(timer_is_armed(timer));
326
327
328
329
330
331 kvm_timer_update_state(vcpu);
332}
333
334int kvm_timer_vcpu_reset(struct kvm_vcpu *vcpu,
335 const struct kvm_irq_level *irq)
336{
337 struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
338 struct irq_phys_map *map;
339
340
341
342
343
344
345
346 timer->irq.irq = irq->irq;
347
348
349
350
351
352
353
354 timer->cntv_ctl = 0;
355 kvm_timer_update_state(vcpu);
356
357
358
359
360
361 map = kvm_vgic_map_phys_irq(vcpu, irq->irq, host_vtimer_irq);
362 if (WARN_ON(IS_ERR(map)))
363 return PTR_ERR(map);
364
365 timer->map = map;
366 return 0;
367}
368
369void kvm_timer_vcpu_init(struct kvm_vcpu *vcpu)
370{
371 struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
372
373 INIT_WORK(&timer->expired, kvm_timer_inject_irq_work);
374 hrtimer_init(&timer->timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
375 timer->timer.function = kvm_timer_expire;
376}
377
378static void kvm_timer_init_interrupt(void *info)
379{
380 enable_percpu_irq(host_vtimer_irq, 0);
381}
382
383int kvm_arm_timer_set_reg(struct kvm_vcpu *vcpu, u64 regid, u64 value)
384{
385 struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
386
387 switch (regid) {
388 case KVM_REG_ARM_TIMER_CTL:
389 timer->cntv_ctl = value;
390 break;
391 case KVM_REG_ARM_TIMER_CNT:
392 vcpu->kvm->arch.timer.cntvoff = kvm_phys_timer_read() - value;
393 break;
394 case KVM_REG_ARM_TIMER_CVAL:
395 timer->cntv_cval = value;
396 break;
397 default:
398 return -1;
399 }
400
401 kvm_timer_update_state(vcpu);
402 return 0;
403}
404
405u64 kvm_arm_timer_get_reg(struct kvm_vcpu *vcpu, u64 regid)
406{
407 struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
408
409 switch (regid) {
410 case KVM_REG_ARM_TIMER_CTL:
411 return timer->cntv_ctl;
412 case KVM_REG_ARM_TIMER_CNT:
413 return kvm_phys_timer_read() - vcpu->kvm->arch.timer.cntvoff;
414 case KVM_REG_ARM_TIMER_CVAL:
415 return timer->cntv_cval;
416 }
417 return (u64)-1;
418}
419
420static int kvm_timer_cpu_notify(struct notifier_block *self,
421 unsigned long action, void *cpu)
422{
423 switch (action) {
424 case CPU_STARTING:
425 case CPU_STARTING_FROZEN:
426 kvm_timer_init_interrupt(NULL);
427 break;
428 case CPU_DYING:
429 case CPU_DYING_FROZEN:
430 disable_percpu_irq(host_vtimer_irq);
431 break;
432 }
433
434 return NOTIFY_OK;
435}
436
437static struct notifier_block kvm_timer_cpu_nb = {
438 .notifier_call = kvm_timer_cpu_notify,
439};
440
441static const struct of_device_id arch_timer_of_match[] = {
442 { .compatible = "arm,armv7-timer", },
443 { .compatible = "arm,armv8-timer", },
444 {},
445};
446
447int kvm_timer_hyp_init(void)
448{
449 struct device_node *np;
450 unsigned int ppi;
451 int err;
452
453 timecounter = arch_timer_get_timecounter();
454 if (!timecounter)
455 return -ENODEV;
456
457 np = of_find_matching_node(NULL, arch_timer_of_match);
458 if (!np) {
459 kvm_err("kvm_arch_timer: can't find DT node\n");
460 return -ENODEV;
461 }
462
463 ppi = irq_of_parse_and_map(np, 2);
464 if (!ppi) {
465 kvm_err("kvm_arch_timer: no virtual timer interrupt\n");
466 err = -EINVAL;
467 goto out;
468 }
469
470 err = request_percpu_irq(ppi, kvm_arch_timer_handler,
471 "kvm guest timer", kvm_get_running_vcpus());
472 if (err) {
473 kvm_err("kvm_arch_timer: can't request interrupt %d (%d)\n",
474 ppi, err);
475 goto out;
476 }
477
478 host_vtimer_irq = ppi;
479
480 err = __register_cpu_notifier(&kvm_timer_cpu_nb);
481 if (err) {
482 kvm_err("Cannot register timer CPU notifier\n");
483 goto out_free;
484 }
485
486 wqueue = create_singlethread_workqueue("kvm_arch_timer");
487 if (!wqueue) {
488 err = -ENOMEM;
489 goto out_free;
490 }
491
492 kvm_info("%s IRQ%d\n", np->name, ppi);
493 on_each_cpu(kvm_timer_init_interrupt, NULL, 1);
494
495 goto out;
496out_free:
497 free_percpu_irq(ppi, kvm_get_running_vcpus());
498out:
499 of_node_put(np);
500 return err;
501}
502
503void kvm_timer_vcpu_terminate(struct kvm_vcpu *vcpu)
504{
505 struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
506
507 timer_disarm(timer);
508 if (timer->map)
509 kvm_vgic_unmap_phys_irq(vcpu, timer->map);
510}
511
512void kvm_timer_enable(struct kvm *kvm)
513{
514 if (kvm->arch.timer.enabled)
515 return;
516
517
518
519
520
521
522
523
524
525 if (timecounter && wqueue)
526 kvm->arch.timer.enabled = 1;
527}
528
529void kvm_timer_init(struct kvm *kvm)
530{
531 kvm->arch.timer.cntvoff = kvm_phys_timer_read();
532}
533