1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19#include <linux/cpu.h>
20#include <linux/kvm.h>
21#include <linux/kvm_host.h>
22#include <linux/interrupt.h>
23#include <linux/irq.h>
24#include <linux/uaccess.h>
25
26#include <clocksource/arm_arch_timer.h>
27#include <asm/arch_timer.h>
28#include <asm/kvm_hyp.h>
29
30#include <kvm/arm_vgic.h>
31#include <kvm/arm_arch_timer.h>
32
33#include "trace.h"
34
35static struct timecounter *timecounter;
36static unsigned int host_vtimer_irq;
37static u32 host_vtimer_irq_flags;
38
39static const struct kvm_irq_level default_ptimer_irq = {
40 .irq = 30,
41 .level = 1,
42};
43
44static const struct kvm_irq_level default_vtimer_irq = {
45 .irq = 27,
46 .level = 1,
47};
48
49void kvm_timer_vcpu_put(struct kvm_vcpu *vcpu)
50{
51 vcpu_vtimer(vcpu)->active_cleared_last = false;
52}
53
54u64 kvm_phys_timer_read(void)
55{
56 return timecounter->cc->read(timecounter->cc);
57}
58
59static bool timer_is_armed(struct arch_timer_cpu *timer)
60{
61 return timer->armed;
62}
63
64
65static void timer_arm(struct arch_timer_cpu *timer, u64 ns)
66{
67 timer->armed = true;
68 hrtimer_start(&timer->timer, ktime_add_ns(ktime_get(), ns),
69 HRTIMER_MODE_ABS);
70}
71
72static void timer_disarm(struct arch_timer_cpu *timer)
73{
74 if (timer_is_armed(timer)) {
75 hrtimer_cancel(&timer->timer);
76 cancel_work_sync(&timer->expired);
77 timer->armed = false;
78 }
79}
80
81static irqreturn_t kvm_arch_timer_handler(int irq, void *dev_id)
82{
83 struct kvm_vcpu *vcpu = *(struct kvm_vcpu **)dev_id;
84
85
86
87
88
89
90
91 pr_warn("Unexpected interrupt %d on vcpu %p\n", irq, vcpu);
92 return IRQ_HANDLED;
93}
94
95
96
97
98
99static void kvm_timer_inject_irq_work(struct work_struct *work)
100{
101 struct kvm_vcpu *vcpu;
102
103 vcpu = container_of(work, struct kvm_vcpu, arch.timer_cpu.expired);
104
105
106
107
108
109 kvm_vcpu_wake_up(vcpu);
110}
111
112static u64 kvm_timer_compute_delta(struct arch_timer_context *timer_ctx)
113{
114 u64 cval, now;
115
116 cval = timer_ctx->cnt_cval;
117 now = kvm_phys_timer_read() - timer_ctx->cntvoff;
118
119 if (now < cval) {
120 u64 ns;
121
122 ns = cyclecounter_cyc2ns(timecounter->cc,
123 cval - now,
124 timecounter->mask,
125 &timecounter->frac);
126 return ns;
127 }
128
129 return 0;
130}
131
132static bool kvm_timer_irq_can_fire(struct arch_timer_context *timer_ctx)
133{
134 return !(timer_ctx->cnt_ctl & ARCH_TIMER_CTRL_IT_MASK) &&
135 (timer_ctx->cnt_ctl & ARCH_TIMER_CTRL_ENABLE);
136}
137
138
139
140
141
142static u64 kvm_timer_earliest_exp(struct kvm_vcpu *vcpu)
143{
144 u64 min_virt = ULLONG_MAX, min_phys = ULLONG_MAX;
145 struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
146 struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
147
148 if (kvm_timer_irq_can_fire(vtimer))
149 min_virt = kvm_timer_compute_delta(vtimer);
150
151 if (kvm_timer_irq_can_fire(ptimer))
152 min_phys = kvm_timer_compute_delta(ptimer);
153
154
155 if ((min_virt == ULLONG_MAX) && (min_phys == ULLONG_MAX))
156 return 0;
157
158 return min(min_virt, min_phys);
159}
160
161static enum hrtimer_restart kvm_timer_expire(struct hrtimer *hrt)
162{
163 struct arch_timer_cpu *timer;
164 struct kvm_vcpu *vcpu;
165 u64 ns;
166
167 timer = container_of(hrt, struct arch_timer_cpu, timer);
168 vcpu = container_of(timer, struct kvm_vcpu, arch.timer_cpu);
169
170
171
172
173
174
175 ns = kvm_timer_earliest_exp(vcpu);
176 if (unlikely(ns)) {
177 hrtimer_forward_now(hrt, ns_to_ktime(ns));
178 return HRTIMER_RESTART;
179 }
180
181 schedule_work(&timer->expired);
182 return HRTIMER_NORESTART;
183}
184
185bool kvm_timer_should_fire(struct arch_timer_context *timer_ctx)
186{
187 u64 cval, now;
188
189 if (!kvm_timer_irq_can_fire(timer_ctx))
190 return false;
191
192 cval = timer_ctx->cnt_cval;
193 now = kvm_phys_timer_read() - timer_ctx->cntvoff;
194
195 return cval <= now;
196}
197
198
199
200
201void kvm_timer_update_run(struct kvm_vcpu *vcpu)
202{
203 struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
204 struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
205 struct kvm_sync_regs *regs = &vcpu->run->s.regs;
206
207
208 regs->device_irq_level &= ~(KVM_ARM_DEV_EL1_VTIMER |
209 KVM_ARM_DEV_EL1_PTIMER);
210 if (vtimer->irq.level)
211 regs->device_irq_level |= KVM_ARM_DEV_EL1_VTIMER;
212 if (ptimer->irq.level)
213 regs->device_irq_level |= KVM_ARM_DEV_EL1_PTIMER;
214}
215
216static void kvm_timer_update_irq(struct kvm_vcpu *vcpu, bool new_level,
217 struct arch_timer_context *timer_ctx)
218{
219 int ret;
220
221 timer_ctx->active_cleared_last = false;
222 timer_ctx->irq.level = new_level;
223 trace_kvm_timer_update_irq(vcpu->vcpu_id, timer_ctx->irq.irq,
224 timer_ctx->irq.level);
225
226 if (likely(irqchip_in_kernel(vcpu->kvm))) {
227 ret = kvm_vgic_inject_irq(vcpu->kvm, vcpu->vcpu_id,
228 timer_ctx->irq.irq,
229 timer_ctx->irq.level,
230 timer_ctx);
231 WARN_ON(ret);
232 }
233}
234
235
236
237
238
239static void kvm_timer_update_state(struct kvm_vcpu *vcpu)
240{
241 struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
242 struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
243 struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
244
245
246
247
248
249
250
251 if (unlikely(!timer->enabled))
252 return;
253
254 if (kvm_timer_should_fire(vtimer) != vtimer->irq.level)
255 kvm_timer_update_irq(vcpu, !vtimer->irq.level, vtimer);
256
257 if (kvm_timer_should_fire(ptimer) != ptimer->irq.level)
258 kvm_timer_update_irq(vcpu, !ptimer->irq.level, ptimer);
259}
260
261
262static void kvm_timer_emulate(struct kvm_vcpu *vcpu,
263 struct arch_timer_context *timer_ctx)
264{
265 struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
266
267 if (kvm_timer_should_fire(timer_ctx))
268 return;
269
270 if (!kvm_timer_irq_can_fire(timer_ctx))
271 return;
272
273
274 timer_arm(timer, kvm_timer_compute_delta(timer_ctx));
275}
276
277
278
279
280
281
282void kvm_timer_schedule(struct kvm_vcpu *vcpu)
283{
284 struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
285 struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
286 struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
287
288 BUG_ON(timer_is_armed(timer));
289
290
291
292
293
294
295 if (kvm_timer_should_fire(vtimer) || kvm_timer_should_fire(ptimer))
296 return;
297
298
299
300
301
302 if (!kvm_timer_irq_can_fire(vtimer) && !kvm_timer_irq_can_fire(ptimer))
303 return;
304
305
306
307
308
309 timer_arm(timer, kvm_timer_earliest_exp(vcpu));
310}
311
312void kvm_timer_unschedule(struct kvm_vcpu *vcpu)
313{
314 struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
315 timer_disarm(timer);
316}
317
318static void kvm_timer_flush_hwstate_vgic(struct kvm_vcpu *vcpu)
319{
320 struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
321 bool phys_active;
322 int ret;
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341 phys_active = vtimer->irq.level ||
342 kvm_vgic_map_is_active(vcpu, vtimer->irq.irq);
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364 if (vtimer->active_cleared_last && !phys_active)
365 return;
366
367 ret = irq_set_irqchip_state(host_vtimer_irq,
368 IRQCHIP_STATE_ACTIVE,
369 phys_active);
370 WARN_ON(ret);
371
372 vtimer->active_cleared_last = !phys_active;
373}
374
375bool kvm_timer_should_notify_user(struct kvm_vcpu *vcpu)
376{
377 struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
378 struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
379 struct kvm_sync_regs *sregs = &vcpu->run->s.regs;
380 bool vlevel, plevel;
381
382 if (likely(irqchip_in_kernel(vcpu->kvm)))
383 return false;
384
385 vlevel = sregs->device_irq_level & KVM_ARM_DEV_EL1_VTIMER;
386 plevel = sregs->device_irq_level & KVM_ARM_DEV_EL1_PTIMER;
387
388 return vtimer->irq.level != vlevel ||
389 ptimer->irq.level != plevel;
390}
391
392static void kvm_timer_flush_hwstate_user(struct kvm_vcpu *vcpu)
393{
394 struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
395
396
397
398
399
400
401
402
403 if (vtimer->irq.level)
404 disable_percpu_irq(host_vtimer_irq);
405 else
406 enable_percpu_irq(host_vtimer_irq, 0);
407}
408
409
410
411
412
413
414
415
416
417
418void kvm_timer_flush_hwstate(struct kvm_vcpu *vcpu)
419{
420 struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
421
422 if (unlikely(!timer->enabled))
423 return;
424
425 kvm_timer_update_state(vcpu);
426
427
428 kvm_timer_emulate(vcpu, vcpu_ptimer(vcpu));
429
430 if (unlikely(!irqchip_in_kernel(vcpu->kvm)))
431 kvm_timer_flush_hwstate_user(vcpu);
432 else
433 kvm_timer_flush_hwstate_vgic(vcpu);
434}
435
436
437
438
439
440
441
442
443void kvm_timer_sync_hwstate(struct kvm_vcpu *vcpu)
444{
445 struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
446
447
448
449
450
451 timer_disarm(timer);
452
453
454
455
456
457 kvm_timer_update_state(vcpu);
458}
459
460int kvm_timer_vcpu_reset(struct kvm_vcpu *vcpu)
461{
462 struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
463 struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
464
465
466
467
468
469
470
471 vtimer->cnt_ctl = 0;
472 ptimer->cnt_ctl = 0;
473 kvm_timer_update_state(vcpu);
474
475 return 0;
476}
477
478
479static void update_vtimer_cntvoff(struct kvm_vcpu *vcpu, u64 cntvoff)
480{
481 int i;
482 struct kvm *kvm = vcpu->kvm;
483 struct kvm_vcpu *tmp;
484
485 mutex_lock(&kvm->lock);
486 kvm_for_each_vcpu(i, tmp, kvm)
487 vcpu_vtimer(tmp)->cntvoff = cntvoff;
488
489
490
491
492
493 vcpu_vtimer(vcpu)->cntvoff = cntvoff;
494 mutex_unlock(&kvm->lock);
495}
496
497void kvm_timer_vcpu_init(struct kvm_vcpu *vcpu)
498{
499 struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
500 struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
501 struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
502
503
504 update_vtimer_cntvoff(vcpu, kvm_phys_timer_read());
505 vcpu_ptimer(vcpu)->cntvoff = 0;
506
507 INIT_WORK(&timer->expired, kvm_timer_inject_irq_work);
508 hrtimer_init(&timer->timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
509 timer->timer.function = kvm_timer_expire;
510
511 vtimer->irq.irq = default_vtimer_irq.irq;
512 ptimer->irq.irq = default_ptimer_irq.irq;
513}
514
515static void kvm_timer_init_interrupt(void *info)
516{
517 enable_percpu_irq(host_vtimer_irq, host_vtimer_irq_flags);
518}
519
520int kvm_arm_timer_set_reg(struct kvm_vcpu *vcpu, u64 regid, u64 value)
521{
522 struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
523
524 switch (regid) {
525 case KVM_REG_ARM_TIMER_CTL:
526 vtimer->cnt_ctl = value;
527 break;
528 case KVM_REG_ARM_TIMER_CNT:
529 update_vtimer_cntvoff(vcpu, kvm_phys_timer_read() - value);
530 break;
531 case KVM_REG_ARM_TIMER_CVAL:
532 vtimer->cnt_cval = value;
533 break;
534 default:
535 return -1;
536 }
537
538 kvm_timer_update_state(vcpu);
539 return 0;
540}
541
542u64 kvm_arm_timer_get_reg(struct kvm_vcpu *vcpu, u64 regid)
543{
544 struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
545
546 switch (regid) {
547 case KVM_REG_ARM_TIMER_CTL:
548 return vtimer->cnt_ctl;
549 case KVM_REG_ARM_TIMER_CNT:
550 return kvm_phys_timer_read() - vtimer->cntvoff;
551 case KVM_REG_ARM_TIMER_CVAL:
552 return vtimer->cnt_cval;
553 }
554 return (u64)-1;
555}
556
557static int kvm_timer_starting_cpu(unsigned int cpu)
558{
559 kvm_timer_init_interrupt(NULL);
560 return 0;
561}
562
563static int kvm_timer_dying_cpu(unsigned int cpu)
564{
565 disable_percpu_irq(host_vtimer_irq);
566 return 0;
567}
568
569int kvm_timer_hyp_init(void)
570{
571 struct arch_timer_kvm_info *info;
572 int err;
573
574 info = arch_timer_get_kvm_info();
575 timecounter = &info->timecounter;
576
577 if (!timecounter->cc) {
578 kvm_err("kvm_arch_timer: uninitialized timecounter\n");
579 return -ENODEV;
580 }
581
582 if (info->virtual_irq <= 0) {
583 kvm_err("kvm_arch_timer: invalid virtual timer IRQ: %d\n",
584 info->virtual_irq);
585 return -ENODEV;
586 }
587 host_vtimer_irq = info->virtual_irq;
588
589 host_vtimer_irq_flags = irq_get_trigger_type(host_vtimer_irq);
590 if (host_vtimer_irq_flags != IRQF_TRIGGER_HIGH &&
591 host_vtimer_irq_flags != IRQF_TRIGGER_LOW) {
592 kvm_err("Invalid trigger for IRQ%d, assuming level low\n",
593 host_vtimer_irq);
594 host_vtimer_irq_flags = IRQF_TRIGGER_LOW;
595 }
596
597 err = request_percpu_irq(host_vtimer_irq, kvm_arch_timer_handler,
598 "kvm guest timer", kvm_get_running_vcpus());
599 if (err) {
600 kvm_err("kvm_arch_timer: can't request interrupt %d (%d)\n",
601 host_vtimer_irq, err);
602 return err;
603 }
604
605 kvm_info("virtual timer IRQ%d\n", host_vtimer_irq);
606
607 cpuhp_setup_state(CPUHP_AP_KVM_ARM_TIMER_STARTING,
608 "kvm/arm/timer:starting", kvm_timer_starting_cpu,
609 kvm_timer_dying_cpu);
610 return err;
611}
612
613void kvm_timer_vcpu_terminate(struct kvm_vcpu *vcpu)
614{
615 struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
616 struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
617
618 timer_disarm(timer);
619 kvm_vgic_unmap_phys_irq(vcpu, vtimer->irq.irq);
620}
621
622static bool timer_irqs_are_valid(struct kvm_vcpu *vcpu)
623{
624 int vtimer_irq, ptimer_irq;
625 int i, ret;
626
627 vtimer_irq = vcpu_vtimer(vcpu)->irq.irq;
628 ret = kvm_vgic_set_owner(vcpu, vtimer_irq, vcpu_vtimer(vcpu));
629 if (ret)
630 return false;
631
632 ptimer_irq = vcpu_ptimer(vcpu)->irq.irq;
633 ret = kvm_vgic_set_owner(vcpu, ptimer_irq, vcpu_ptimer(vcpu));
634 if (ret)
635 return false;
636
637 kvm_for_each_vcpu(i, vcpu, vcpu->kvm) {
638 if (vcpu_vtimer(vcpu)->irq.irq != vtimer_irq ||
639 vcpu_ptimer(vcpu)->irq.irq != ptimer_irq)
640 return false;
641 }
642
643 return true;
644}
645
646int kvm_timer_enable(struct kvm_vcpu *vcpu)
647{
648 struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
649 struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
650 struct irq_desc *desc;
651 struct irq_data *data;
652 int phys_irq;
653 int ret;
654
655 if (timer->enabled)
656 return 0;
657
658
659 if (!irqchip_in_kernel(vcpu->kvm))
660 goto no_vgic;
661
662 if (!vgic_initialized(vcpu->kvm))
663 return -ENODEV;
664
665 if (!timer_irqs_are_valid(vcpu)) {
666 kvm_debug("incorrectly configured timer irqs\n");
667 return -EINVAL;
668 }
669
670
671
672
673 desc = irq_to_desc(host_vtimer_irq);
674 if (!desc) {
675 kvm_err("%s: no interrupt descriptor\n", __func__);
676 return -EINVAL;
677 }
678
679 data = irq_desc_get_irq_data(desc);
680 while (data->parent_data)
681 data = data->parent_data;
682
683 phys_irq = data->hwirq;
684
685
686
687
688
689 ret = kvm_vgic_map_phys_irq(vcpu, vtimer->irq.irq, phys_irq);
690 if (ret)
691 return ret;
692
693no_vgic:
694 timer->enabled = 1;
695 return 0;
696}
697
698
699
700
701
702
703
704void kvm_timer_init_vhe(void)
705{
706
707 u32 cnthctl_shift = 10;
708 u64 val;
709
710
711
712
713
714 val = read_sysreg(cnthctl_el2);
715 val &= ~(CNTHCTL_EL1PCEN << cnthctl_shift);
716 val |= (CNTHCTL_EL1PCTEN << cnthctl_shift);
717 write_sysreg(val, cnthctl_el2);
718}
719
720static void set_timer_irqs(struct kvm *kvm, int vtimer_irq, int ptimer_irq)
721{
722 struct kvm_vcpu *vcpu;
723 int i;
724
725 kvm_for_each_vcpu(i, vcpu, kvm) {
726 vcpu_vtimer(vcpu)->irq.irq = vtimer_irq;
727 vcpu_ptimer(vcpu)->irq.irq = ptimer_irq;
728 }
729}
730
731int kvm_arm_timer_set_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr)
732{
733 int __user *uaddr = (int __user *)(long)attr->addr;
734 struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
735 struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
736 int irq;
737
738 if (!irqchip_in_kernel(vcpu->kvm))
739 return -EINVAL;
740
741 if (get_user(irq, uaddr))
742 return -EFAULT;
743
744 if (!(irq_is_ppi(irq)))
745 return -EINVAL;
746
747 if (vcpu->arch.timer_cpu.enabled)
748 return -EBUSY;
749
750 switch (attr->attr) {
751 case KVM_ARM_VCPU_TIMER_IRQ_VTIMER:
752 set_timer_irqs(vcpu->kvm, irq, ptimer->irq.irq);
753 break;
754 case KVM_ARM_VCPU_TIMER_IRQ_PTIMER:
755 set_timer_irqs(vcpu->kvm, vtimer->irq.irq, irq);
756 break;
757 default:
758 return -ENXIO;
759 }
760
761 return 0;
762}
763
764int kvm_arm_timer_get_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr)
765{
766 int __user *uaddr = (int __user *)(long)attr->addr;
767 struct arch_timer_context *timer;
768 int irq;
769
770 switch (attr->attr) {
771 case KVM_ARM_VCPU_TIMER_IRQ_VTIMER:
772 timer = vcpu_vtimer(vcpu);
773 break;
774 case KVM_ARM_VCPU_TIMER_IRQ_PTIMER:
775 timer = vcpu_ptimer(vcpu);
776 break;
777 default:
778 return -ENXIO;
779 }
780
781 irq = timer->irq.irq;
782 return put_user(irq, uaddr);
783}
784
785int kvm_arm_timer_has_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr)
786{
787 switch (attr->attr) {
788 case KVM_ARM_VCPU_TIMER_IRQ_VTIMER:
789 case KVM_ARM_VCPU_TIMER_IRQ_PTIMER:
790 return 0;
791 }
792
793 return -ENXIO;
794}
795