1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19#include <linux/cpu.h>
20#include <linux/kvm.h>
21#include <linux/kvm_host.h>
22#include <linux/interrupt.h>
23#include <linux/irq.h>
24#include <linux/uaccess.h>
25
26#include <clocksource/arm_arch_timer.h>
27#include <asm/arch_timer.h>
28#include <asm/kvm_emulate.h>
29#include <asm/kvm_hyp.h>
30
31#include <kvm/arm_vgic.h>
32#include <kvm/arm_arch_timer.h>
33
34#include "trace.h"
35
36static struct timecounter *timecounter;
37static unsigned int host_vtimer_irq;
38static unsigned int host_ptimer_irq;
39static u32 host_vtimer_irq_flags;
40static u32 host_ptimer_irq_flags;
41
42static DEFINE_STATIC_KEY_FALSE(has_gic_active_state);
43
44static const struct kvm_irq_level default_ptimer_irq = {
45 .irq = 30,
46 .level = 1,
47};
48
49static const struct kvm_irq_level default_vtimer_irq = {
50 .irq = 27,
51 .level = 1,
52};
53
54static bool kvm_timer_irq_can_fire(struct arch_timer_context *timer_ctx);
55static void kvm_timer_update_irq(struct kvm_vcpu *vcpu, bool new_level,
56 struct arch_timer_context *timer_ctx);
57static bool kvm_timer_should_fire(struct arch_timer_context *timer_ctx);
58static void kvm_arm_timer_write(struct kvm_vcpu *vcpu,
59 struct arch_timer_context *timer,
60 enum kvm_arch_timer_regs treg,
61 u64 val);
62static u64 kvm_arm_timer_read(struct kvm_vcpu *vcpu,
63 struct arch_timer_context *timer,
64 enum kvm_arch_timer_regs treg);
65
66u64 kvm_phys_timer_read(void)
67{
68 return timecounter->cc->read(timecounter->cc);
69}
70
71static void get_timer_map(struct kvm_vcpu *vcpu, struct timer_map *map)
72{
73 if (has_vhe()) {
74 map->direct_vtimer = vcpu_vtimer(vcpu);
75 map->direct_ptimer = vcpu_ptimer(vcpu);
76 map->emul_ptimer = NULL;
77 } else {
78 map->direct_vtimer = vcpu_vtimer(vcpu);
79 map->direct_ptimer = NULL;
80 map->emul_ptimer = vcpu_ptimer(vcpu);
81 }
82
83 trace_kvm_get_timer_map(vcpu->vcpu_id, map);
84}
85
86static inline bool userspace_irqchip(struct kvm *kvm)
87{
88 return static_branch_unlikely(&userspace_irqchip_in_use) &&
89 unlikely(!irqchip_in_kernel(kvm));
90}
91
92static void soft_timer_start(struct hrtimer *hrt, u64 ns)
93{
94 hrtimer_start(hrt, ktime_add_ns(ktime_get(), ns),
95 HRTIMER_MODE_ABS);
96}
97
98static void soft_timer_cancel(struct hrtimer *hrt)
99{
100 hrtimer_cancel(hrt);
101}
102
103static irqreturn_t kvm_arch_timer_handler(int irq, void *dev_id)
104{
105 struct kvm_vcpu *vcpu = *(struct kvm_vcpu **)dev_id;
106 struct arch_timer_context *ctx;
107 struct timer_map map;
108
109
110
111
112
113
114
115 if (!vcpu)
116 return IRQ_HANDLED;
117
118 get_timer_map(vcpu, &map);
119
120 if (irq == host_vtimer_irq)
121 ctx = map.direct_vtimer;
122 else
123 ctx = map.direct_ptimer;
124
125 if (kvm_timer_should_fire(ctx))
126 kvm_timer_update_irq(vcpu, true, ctx);
127
128 if (userspace_irqchip(vcpu->kvm) &&
129 !static_branch_unlikely(&has_gic_active_state))
130 disable_percpu_irq(host_vtimer_irq);
131
132 return IRQ_HANDLED;
133}
134
135static u64 kvm_timer_compute_delta(struct arch_timer_context *timer_ctx)
136{
137 u64 cval, now;
138
139 cval = timer_ctx->cnt_cval;
140 now = kvm_phys_timer_read() - timer_ctx->cntvoff;
141
142 if (now < cval) {
143 u64 ns;
144
145 ns = cyclecounter_cyc2ns(timecounter->cc,
146 cval - now,
147 timecounter->mask,
148 &timecounter->frac);
149 return ns;
150 }
151
152 return 0;
153}
154
155static bool kvm_timer_irq_can_fire(struct arch_timer_context *timer_ctx)
156{
157 WARN_ON(timer_ctx && timer_ctx->loaded);
158 return timer_ctx &&
159 !(timer_ctx->cnt_ctl & ARCH_TIMER_CTRL_IT_MASK) &&
160 (timer_ctx->cnt_ctl & ARCH_TIMER_CTRL_ENABLE);
161}
162
163
164
165
166
167static u64 kvm_timer_earliest_exp(struct kvm_vcpu *vcpu)
168{
169 u64 min_delta = ULLONG_MAX;
170 int i;
171
172 for (i = 0; i < NR_KVM_TIMERS; i++) {
173 struct arch_timer_context *ctx = &vcpu->arch.timer_cpu.timers[i];
174
175 WARN(ctx->loaded, "timer %d loaded\n", i);
176 if (kvm_timer_irq_can_fire(ctx))
177 min_delta = min(min_delta, kvm_timer_compute_delta(ctx));
178 }
179
180
181 if (min_delta == ULLONG_MAX)
182 return 0;
183
184 return min_delta;
185}
186
187static enum hrtimer_restart kvm_bg_timer_expire(struct hrtimer *hrt)
188{
189 struct arch_timer_cpu *timer;
190 struct kvm_vcpu *vcpu;
191 u64 ns;
192
193 timer = container_of(hrt, struct arch_timer_cpu, bg_timer);
194 vcpu = container_of(timer, struct kvm_vcpu, arch.timer_cpu);
195
196
197
198
199
200
201 ns = kvm_timer_earliest_exp(vcpu);
202 if (unlikely(ns)) {
203 hrtimer_forward_now(hrt, ns_to_ktime(ns));
204 return HRTIMER_RESTART;
205 }
206
207 kvm_vcpu_wake_up(vcpu);
208 return HRTIMER_NORESTART;
209}
210
211static enum hrtimer_restart kvm_hrtimer_expire(struct hrtimer *hrt)
212{
213 struct arch_timer_context *ctx;
214 struct kvm_vcpu *vcpu;
215 u64 ns;
216
217 ctx = container_of(hrt, struct arch_timer_context, hrtimer);
218 vcpu = ctx->vcpu;
219
220 trace_kvm_timer_hrtimer_expire(ctx);
221
222
223
224
225
226
227 ns = kvm_timer_compute_delta(ctx);
228 if (unlikely(ns)) {
229 hrtimer_forward_now(hrt, ns_to_ktime(ns));
230 return HRTIMER_RESTART;
231 }
232
233 kvm_timer_update_irq(vcpu, true, ctx);
234 return HRTIMER_NORESTART;
235}
236
237static bool kvm_timer_should_fire(struct arch_timer_context *timer_ctx)
238{
239 enum kvm_arch_timers index;
240 u64 cval, now;
241
242 if (!timer_ctx)
243 return false;
244
245 index = arch_timer_ctx_index(timer_ctx);
246
247 if (timer_ctx->loaded) {
248 u32 cnt_ctl = 0;
249
250 switch (index) {
251 case TIMER_VTIMER:
252 cnt_ctl = read_sysreg_el0(cntv_ctl);
253 break;
254 case TIMER_PTIMER:
255 cnt_ctl = read_sysreg_el0(cntp_ctl);
256 break;
257 case NR_KVM_TIMERS:
258
259 cnt_ctl = 0;
260 break;
261 }
262
263 return (cnt_ctl & ARCH_TIMER_CTRL_ENABLE) &&
264 (cnt_ctl & ARCH_TIMER_CTRL_IT_STAT) &&
265 !(cnt_ctl & ARCH_TIMER_CTRL_IT_MASK);
266 }
267
268 if (!kvm_timer_irq_can_fire(timer_ctx))
269 return false;
270
271 cval = timer_ctx->cnt_cval;
272 now = kvm_phys_timer_read() - timer_ctx->cntvoff;
273
274 return cval <= now;
275}
276
277bool kvm_timer_is_pending(struct kvm_vcpu *vcpu)
278{
279 struct timer_map map;
280
281 get_timer_map(vcpu, &map);
282
283 return kvm_timer_should_fire(map.direct_vtimer) ||
284 kvm_timer_should_fire(map.direct_ptimer) ||
285 kvm_timer_should_fire(map.emul_ptimer);
286}
287
288
289
290
291void kvm_timer_update_run(struct kvm_vcpu *vcpu)
292{
293 struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
294 struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
295 struct kvm_sync_regs *regs = &vcpu->run->s.regs;
296
297
298 regs->device_irq_level &= ~(KVM_ARM_DEV_EL1_VTIMER |
299 KVM_ARM_DEV_EL1_PTIMER);
300 if (kvm_timer_should_fire(vtimer))
301 regs->device_irq_level |= KVM_ARM_DEV_EL1_VTIMER;
302 if (kvm_timer_should_fire(ptimer))
303 regs->device_irq_level |= KVM_ARM_DEV_EL1_PTIMER;
304}
305
306static void kvm_timer_update_irq(struct kvm_vcpu *vcpu, bool new_level,
307 struct arch_timer_context *timer_ctx)
308{
309 int ret;
310
311 timer_ctx->irq.level = new_level;
312 trace_kvm_timer_update_irq(vcpu->vcpu_id, timer_ctx->irq.irq,
313 timer_ctx->irq.level);
314
315 if (!userspace_irqchip(vcpu->kvm)) {
316 ret = kvm_vgic_inject_irq(vcpu->kvm, vcpu->vcpu_id,
317 timer_ctx->irq.irq,
318 timer_ctx->irq.level,
319 timer_ctx);
320 WARN_ON(ret);
321 }
322}
323
324static void timer_emulate(struct arch_timer_context *ctx)
325{
326 bool should_fire = kvm_timer_should_fire(ctx);
327
328 trace_kvm_timer_emulate(ctx, should_fire);
329
330 if (should_fire) {
331 kvm_timer_update_irq(ctx->vcpu, true, ctx);
332 return;
333 }
334
335
336
337
338
339
340 if (!kvm_timer_irq_can_fire(ctx)) {
341 soft_timer_cancel(&ctx->hrtimer);
342 return;
343 }
344
345 soft_timer_start(&ctx->hrtimer, kvm_timer_compute_delta(ctx));
346}
347
348static void timer_save_state(struct arch_timer_context *ctx)
349{
350 struct arch_timer_cpu *timer = vcpu_timer(ctx->vcpu);
351 enum kvm_arch_timers index = arch_timer_ctx_index(ctx);
352 unsigned long flags;
353
354 if (!timer->enabled)
355 return;
356
357 local_irq_save(flags);
358
359 if (!ctx->loaded)
360 goto out;
361
362 switch (index) {
363 case TIMER_VTIMER:
364 ctx->cnt_ctl = read_sysreg_el0(cntv_ctl);
365 ctx->cnt_cval = read_sysreg_el0(cntv_cval);
366
367
368 write_sysreg_el0(0, cntv_ctl);
369 isb();
370
371 break;
372 case TIMER_PTIMER:
373 ctx->cnt_ctl = read_sysreg_el0(cntp_ctl);
374 ctx->cnt_cval = read_sysreg_el0(cntp_cval);
375
376
377 write_sysreg_el0(0, cntp_ctl);
378 isb();
379
380 break;
381 case NR_KVM_TIMERS:
382 BUG();
383 }
384
385 trace_kvm_timer_save_state(ctx);
386
387 ctx->loaded = false;
388out:
389 local_irq_restore(flags);
390}
391
392
393
394
395
396
397static void kvm_timer_blocking(struct kvm_vcpu *vcpu)
398{
399 struct arch_timer_cpu *timer = vcpu_timer(vcpu);
400 struct timer_map map;
401
402 get_timer_map(vcpu, &map);
403
404
405
406
407
408 if (!kvm_timer_irq_can_fire(map.direct_vtimer) &&
409 !kvm_timer_irq_can_fire(map.direct_ptimer) &&
410 !kvm_timer_irq_can_fire(map.emul_ptimer))
411 return;
412
413
414
415
416
417 soft_timer_start(&timer->bg_timer, kvm_timer_earliest_exp(vcpu));
418}
419
420static void kvm_timer_unblocking(struct kvm_vcpu *vcpu)
421{
422 struct arch_timer_cpu *timer = vcpu_timer(vcpu);
423
424 soft_timer_cancel(&timer->bg_timer);
425}
426
427static void timer_restore_state(struct arch_timer_context *ctx)
428{
429 struct arch_timer_cpu *timer = vcpu_timer(ctx->vcpu);
430 enum kvm_arch_timers index = arch_timer_ctx_index(ctx);
431 unsigned long flags;
432
433 if (!timer->enabled)
434 return;
435
436 local_irq_save(flags);
437
438 if (ctx->loaded)
439 goto out;
440
441 switch (index) {
442 case TIMER_VTIMER:
443 write_sysreg_el0(ctx->cnt_cval, cntv_cval);
444 isb();
445 write_sysreg_el0(ctx->cnt_ctl, cntv_ctl);
446 break;
447 case TIMER_PTIMER:
448 write_sysreg_el0(ctx->cnt_cval, cntp_cval);
449 isb();
450 write_sysreg_el0(ctx->cnt_ctl, cntp_ctl);
451 break;
452 case NR_KVM_TIMERS:
453 BUG();
454 }
455
456 trace_kvm_timer_restore_state(ctx);
457
458 ctx->loaded = true;
459out:
460 local_irq_restore(flags);
461}
462
463static void set_cntvoff(u64 cntvoff)
464{
465 u32 low = lower_32_bits(cntvoff);
466 u32 high = upper_32_bits(cntvoff);
467
468
469
470
471
472
473
474
475 kvm_call_hyp(__kvm_timer_set_cntvoff, low, high);
476}
477
478static inline void set_timer_irq_phys_active(struct arch_timer_context *ctx, bool active)
479{
480 int r;
481 r = irq_set_irqchip_state(ctx->host_timer_irq, IRQCHIP_STATE_ACTIVE, active);
482 WARN_ON(r);
483}
484
485static void kvm_timer_vcpu_load_gic(struct arch_timer_context *ctx)
486{
487 struct kvm_vcpu *vcpu = ctx->vcpu;
488 bool phys_active = false;
489
490
491
492
493
494
495
496 kvm_timer_update_irq(ctx->vcpu, kvm_timer_should_fire(ctx), ctx);
497
498 if (irqchip_in_kernel(vcpu->kvm))
499 phys_active = kvm_vgic_map_is_active(vcpu, ctx->irq.irq);
500
501 phys_active |= ctx->irq.level;
502
503 set_timer_irq_phys_active(ctx, phys_active);
504}
505
506static void kvm_timer_vcpu_load_nogic(struct kvm_vcpu *vcpu)
507{
508 struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
509
510
511
512
513
514
515
516 kvm_timer_update_irq(vcpu, kvm_timer_should_fire(vtimer), vtimer);
517
518
519
520
521
522
523
524
525
526
527
528 if (vtimer->irq.level)
529 disable_percpu_irq(host_vtimer_irq);
530 else
531 enable_percpu_irq(host_vtimer_irq, host_vtimer_irq_flags);
532}
533
534void kvm_timer_vcpu_load(struct kvm_vcpu *vcpu)
535{
536 struct arch_timer_cpu *timer = vcpu_timer(vcpu);
537 struct timer_map map;
538
539 if (unlikely(!timer->enabled))
540 return;
541
542 get_timer_map(vcpu, &map);
543
544 if (static_branch_likely(&has_gic_active_state)) {
545 kvm_timer_vcpu_load_gic(map.direct_vtimer);
546 if (map.direct_ptimer)
547 kvm_timer_vcpu_load_gic(map.direct_ptimer);
548 } else {
549 kvm_timer_vcpu_load_nogic(vcpu);
550 }
551
552 set_cntvoff(map.direct_vtimer->cntvoff);
553
554 kvm_timer_unblocking(vcpu);
555
556 timer_restore_state(map.direct_vtimer);
557 if (map.direct_ptimer)
558 timer_restore_state(map.direct_ptimer);
559
560 if (map.emul_ptimer)
561 timer_emulate(map.emul_ptimer);
562}
563
564bool kvm_timer_should_notify_user(struct kvm_vcpu *vcpu)
565{
566 struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
567 struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
568 struct kvm_sync_regs *sregs = &vcpu->run->s.regs;
569 bool vlevel, plevel;
570
571 if (likely(irqchip_in_kernel(vcpu->kvm)))
572 return false;
573
574 vlevel = sregs->device_irq_level & KVM_ARM_DEV_EL1_VTIMER;
575 plevel = sregs->device_irq_level & KVM_ARM_DEV_EL1_PTIMER;
576
577 return kvm_timer_should_fire(vtimer) != vlevel ||
578 kvm_timer_should_fire(ptimer) != plevel;
579}
580
581void kvm_timer_vcpu_put(struct kvm_vcpu *vcpu)
582{
583 struct arch_timer_cpu *timer = vcpu_timer(vcpu);
584 struct timer_map map;
585
586 if (unlikely(!timer->enabled))
587 return;
588
589 get_timer_map(vcpu, &map);
590
591 timer_save_state(map.direct_vtimer);
592 if (map.direct_ptimer)
593 timer_save_state(map.direct_ptimer);
594
595
596
597
598
599
600
601
602
603
604 if (map.emul_ptimer)
605 soft_timer_cancel(&map.emul_ptimer->hrtimer);
606
607 if (swait_active(kvm_arch_vcpu_wq(vcpu)))
608 kvm_timer_blocking(vcpu);
609
610
611
612
613
614
615
616
617 set_cntvoff(0);
618}
619
620
621
622
623
624
625static void unmask_vtimer_irq_user(struct kvm_vcpu *vcpu)
626{
627 struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
628
629 if (!kvm_timer_should_fire(vtimer)) {
630 kvm_timer_update_irq(vcpu, false, vtimer);
631 if (static_branch_likely(&has_gic_active_state))
632 set_timer_irq_phys_active(vtimer, false);
633 else
634 enable_percpu_irq(host_vtimer_irq, host_vtimer_irq_flags);
635 }
636}
637
638void kvm_timer_sync_hwstate(struct kvm_vcpu *vcpu)
639{
640 struct arch_timer_cpu *timer = vcpu_timer(vcpu);
641
642 if (unlikely(!timer->enabled))
643 return;
644
645 if (unlikely(!irqchip_in_kernel(vcpu->kvm)))
646 unmask_vtimer_irq_user(vcpu);
647}
648
649int kvm_timer_vcpu_reset(struct kvm_vcpu *vcpu)
650{
651 struct arch_timer_cpu *timer = vcpu_timer(vcpu);
652 struct timer_map map;
653
654 get_timer_map(vcpu, &map);
655
656
657
658
659
660
661
662 vcpu_vtimer(vcpu)->cnt_ctl = 0;
663 vcpu_ptimer(vcpu)->cnt_ctl = 0;
664
665 if (timer->enabled) {
666 kvm_timer_update_irq(vcpu, false, vcpu_vtimer(vcpu));
667 kvm_timer_update_irq(vcpu, false, vcpu_ptimer(vcpu));
668
669 if (irqchip_in_kernel(vcpu->kvm)) {
670 kvm_vgic_reset_mapped_irq(vcpu, map.direct_vtimer->irq.irq);
671 if (map.direct_ptimer)
672 kvm_vgic_reset_mapped_irq(vcpu, map.direct_ptimer->irq.irq);
673 }
674 }
675
676 if (map.emul_ptimer)
677 soft_timer_cancel(&map.emul_ptimer->hrtimer);
678
679 return 0;
680}
681
682
683static void update_vtimer_cntvoff(struct kvm_vcpu *vcpu, u64 cntvoff)
684{
685 int i;
686 struct kvm *kvm = vcpu->kvm;
687 struct kvm_vcpu *tmp;
688
689 mutex_lock(&kvm->lock);
690 kvm_for_each_vcpu(i, tmp, kvm)
691 vcpu_vtimer(tmp)->cntvoff = cntvoff;
692
693
694
695
696
697 vcpu_vtimer(vcpu)->cntvoff = cntvoff;
698 mutex_unlock(&kvm->lock);
699}
700
701void kvm_timer_vcpu_init(struct kvm_vcpu *vcpu)
702{
703 struct arch_timer_cpu *timer = vcpu_timer(vcpu);
704 struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
705 struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
706
707
708 update_vtimer_cntvoff(vcpu, kvm_phys_timer_read());
709 ptimer->cntvoff = 0;
710
711 hrtimer_init(&timer->bg_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
712 timer->bg_timer.function = kvm_bg_timer_expire;
713
714 hrtimer_init(&vtimer->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
715 hrtimer_init(&ptimer->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
716 vtimer->hrtimer.function = kvm_hrtimer_expire;
717 ptimer->hrtimer.function = kvm_hrtimer_expire;
718
719 vtimer->irq.irq = default_vtimer_irq.irq;
720 ptimer->irq.irq = default_ptimer_irq.irq;
721
722 vtimer->host_timer_irq = host_vtimer_irq;
723 ptimer->host_timer_irq = host_ptimer_irq;
724
725 vtimer->host_timer_irq_flags = host_vtimer_irq_flags;
726 ptimer->host_timer_irq_flags = host_ptimer_irq_flags;
727
728 vtimer->vcpu = vcpu;
729 ptimer->vcpu = vcpu;
730}
731
732static void kvm_timer_init_interrupt(void *info)
733{
734 enable_percpu_irq(host_vtimer_irq, host_vtimer_irq_flags);
735 enable_percpu_irq(host_ptimer_irq, host_ptimer_irq_flags);
736}
737
738int kvm_arm_timer_set_reg(struct kvm_vcpu *vcpu, u64 regid, u64 value)
739{
740 struct arch_timer_context *timer;
741
742 switch (regid) {
743 case KVM_REG_ARM_TIMER_CTL:
744 timer = vcpu_vtimer(vcpu);
745 kvm_arm_timer_write(vcpu, timer, TIMER_REG_CTL, value);
746 break;
747 case KVM_REG_ARM_TIMER_CNT:
748 timer = vcpu_vtimer(vcpu);
749 update_vtimer_cntvoff(vcpu, kvm_phys_timer_read() - value);
750 break;
751 case KVM_REG_ARM_TIMER_CVAL:
752 timer = vcpu_vtimer(vcpu);
753 kvm_arm_timer_write(vcpu, timer, TIMER_REG_CVAL, value);
754 break;
755 case KVM_REG_ARM_PTIMER_CTL:
756 timer = vcpu_ptimer(vcpu);
757 kvm_arm_timer_write(vcpu, timer, TIMER_REG_CTL, value);
758 break;
759 case KVM_REG_ARM_PTIMER_CVAL:
760 timer = vcpu_ptimer(vcpu);
761 kvm_arm_timer_write(vcpu, timer, TIMER_REG_CVAL, value);
762 break;
763
764 default:
765 return -1;
766 }
767
768 return 0;
769}
770
771static u64 read_timer_ctl(struct arch_timer_context *timer)
772{
773
774
775
776
777
778
779 if (!kvm_timer_compute_delta(timer))
780 return timer->cnt_ctl | ARCH_TIMER_CTRL_IT_STAT;
781 else
782 return timer->cnt_ctl;
783}
784
785u64 kvm_arm_timer_get_reg(struct kvm_vcpu *vcpu, u64 regid)
786{
787 switch (regid) {
788 case KVM_REG_ARM_TIMER_CTL:
789 return kvm_arm_timer_read(vcpu,
790 vcpu_vtimer(vcpu), TIMER_REG_CTL);
791 case KVM_REG_ARM_TIMER_CNT:
792 return kvm_arm_timer_read(vcpu,
793 vcpu_vtimer(vcpu), TIMER_REG_CNT);
794 case KVM_REG_ARM_TIMER_CVAL:
795 return kvm_arm_timer_read(vcpu,
796 vcpu_vtimer(vcpu), TIMER_REG_CVAL);
797 case KVM_REG_ARM_PTIMER_CTL:
798 return kvm_arm_timer_read(vcpu,
799 vcpu_ptimer(vcpu), TIMER_REG_CTL);
800 case KVM_REG_ARM_PTIMER_CNT:
801 return kvm_arm_timer_read(vcpu,
802 vcpu_vtimer(vcpu), TIMER_REG_CNT);
803 case KVM_REG_ARM_PTIMER_CVAL:
804 return kvm_arm_timer_read(vcpu,
805 vcpu_ptimer(vcpu), TIMER_REG_CVAL);
806 }
807 return (u64)-1;
808}
809
810static u64 kvm_arm_timer_read(struct kvm_vcpu *vcpu,
811 struct arch_timer_context *timer,
812 enum kvm_arch_timer_regs treg)
813{
814 u64 val;
815
816 switch (treg) {
817 case TIMER_REG_TVAL:
818 val = timer->cnt_cval - kvm_phys_timer_read() + timer->cntvoff;
819 break;
820
821 case TIMER_REG_CTL:
822 val = read_timer_ctl(timer);
823 break;
824
825 case TIMER_REG_CVAL:
826 val = timer->cnt_cval;
827 break;
828
829 case TIMER_REG_CNT:
830 val = kvm_phys_timer_read() - timer->cntvoff;
831 break;
832
833 default:
834 BUG();
835 }
836
837 return val;
838}
839
840u64 kvm_arm_timer_read_sysreg(struct kvm_vcpu *vcpu,
841 enum kvm_arch_timers tmr,
842 enum kvm_arch_timer_regs treg)
843{
844 u64 val;
845
846 preempt_disable();
847 kvm_timer_vcpu_put(vcpu);
848
849 val = kvm_arm_timer_read(vcpu, vcpu_get_timer(vcpu, tmr), treg);
850
851 kvm_timer_vcpu_load(vcpu);
852 preempt_enable();
853
854 return val;
855}
856
857static void kvm_arm_timer_write(struct kvm_vcpu *vcpu,
858 struct arch_timer_context *timer,
859 enum kvm_arch_timer_regs treg,
860 u64 val)
861{
862 switch (treg) {
863 case TIMER_REG_TVAL:
864 timer->cnt_cval = kvm_phys_timer_read() - timer->cntvoff + val;
865 break;
866
867 case TIMER_REG_CTL:
868 timer->cnt_ctl = val & ~ARCH_TIMER_CTRL_IT_STAT;
869 break;
870
871 case TIMER_REG_CVAL:
872 timer->cnt_cval = val;
873 break;
874
875 default:
876 BUG();
877 }
878}
879
880void kvm_arm_timer_write_sysreg(struct kvm_vcpu *vcpu,
881 enum kvm_arch_timers tmr,
882 enum kvm_arch_timer_regs treg,
883 u64 val)
884{
885 preempt_disable();
886 kvm_timer_vcpu_put(vcpu);
887
888 kvm_arm_timer_write(vcpu, vcpu_get_timer(vcpu, tmr), treg, val);
889
890 kvm_timer_vcpu_load(vcpu);
891 preempt_enable();
892}
893
894static int kvm_timer_starting_cpu(unsigned int cpu)
895{
896 kvm_timer_init_interrupt(NULL);
897 return 0;
898}
899
900static int kvm_timer_dying_cpu(unsigned int cpu)
901{
902 disable_percpu_irq(host_vtimer_irq);
903 return 0;
904}
905
906int kvm_timer_hyp_init(bool has_gic)
907{
908 struct arch_timer_kvm_info *info;
909 int err;
910
911 info = arch_timer_get_kvm_info();
912 timecounter = &info->timecounter;
913
914 if (!timecounter->cc) {
915 kvm_err("kvm_arch_timer: uninitialized timecounter\n");
916 return -ENODEV;
917 }
918
919
920
921 if (info->virtual_irq <= 0) {
922 kvm_err("kvm_arch_timer: invalid virtual timer IRQ: %d\n",
923 info->virtual_irq);
924 return -ENODEV;
925 }
926 host_vtimer_irq = info->virtual_irq;
927
928 host_vtimer_irq_flags = irq_get_trigger_type(host_vtimer_irq);
929 if (host_vtimer_irq_flags != IRQF_TRIGGER_HIGH &&
930 host_vtimer_irq_flags != IRQF_TRIGGER_LOW) {
931 kvm_err("Invalid trigger for vtimer IRQ%d, assuming level low\n",
932 host_vtimer_irq);
933 host_vtimer_irq_flags = IRQF_TRIGGER_LOW;
934 }
935
936 err = request_percpu_irq(host_vtimer_irq, kvm_arch_timer_handler,
937 "kvm guest vtimer", kvm_get_running_vcpus());
938 if (err) {
939 kvm_err("kvm_arch_timer: can't request vtimer interrupt %d (%d)\n",
940 host_vtimer_irq, err);
941 return err;
942 }
943
944 if (has_gic) {
945 err = irq_set_vcpu_affinity(host_vtimer_irq,
946 kvm_get_running_vcpus());
947 if (err) {
948 kvm_err("kvm_arch_timer: error setting vcpu affinity\n");
949 goto out_free_irq;
950 }
951
952 static_branch_enable(&has_gic_active_state);
953 }
954
955 kvm_debug("virtual timer IRQ%d\n", host_vtimer_irq);
956
957
958
959 if (info->physical_irq > 0) {
960 host_ptimer_irq = info->physical_irq;
961 host_ptimer_irq_flags = irq_get_trigger_type(host_ptimer_irq);
962 if (host_ptimer_irq_flags != IRQF_TRIGGER_HIGH &&
963 host_ptimer_irq_flags != IRQF_TRIGGER_LOW) {
964 kvm_err("Invalid trigger for ptimer IRQ%d, assuming level low\n",
965 host_ptimer_irq);
966 host_ptimer_irq_flags = IRQF_TRIGGER_LOW;
967 }
968
969 err = request_percpu_irq(host_ptimer_irq, kvm_arch_timer_handler,
970 "kvm guest ptimer", kvm_get_running_vcpus());
971 if (err) {
972 kvm_err("kvm_arch_timer: can't request ptimer interrupt %d (%d)\n",
973 host_ptimer_irq, err);
974 return err;
975 }
976
977 if (has_gic) {
978 err = irq_set_vcpu_affinity(host_ptimer_irq,
979 kvm_get_running_vcpus());
980 if (err) {
981 kvm_err("kvm_arch_timer: error setting vcpu affinity\n");
982 goto out_free_irq;
983 }
984 }
985
986 kvm_debug("physical timer IRQ%d\n", host_ptimer_irq);
987 } else if (has_vhe()) {
988 kvm_err("kvm_arch_timer: invalid physical timer IRQ: %d\n",
989 info->physical_irq);
990 err = -ENODEV;
991 goto out_free_irq;
992 }
993
994 cpuhp_setup_state(CPUHP_AP_KVM_ARM_TIMER_STARTING,
995 "kvm/arm/timer:starting", kvm_timer_starting_cpu,
996 kvm_timer_dying_cpu);
997 return 0;
998out_free_irq:
999 free_percpu_irq(host_vtimer_irq, kvm_get_running_vcpus());
1000 return err;
1001}
1002
1003void kvm_timer_vcpu_terminate(struct kvm_vcpu *vcpu)
1004{
1005 struct arch_timer_cpu *timer = vcpu_timer(vcpu);
1006
1007 soft_timer_cancel(&timer->bg_timer);
1008}
1009
1010static bool timer_irqs_are_valid(struct kvm_vcpu *vcpu)
1011{
1012 int vtimer_irq, ptimer_irq;
1013 int i, ret;
1014
1015 vtimer_irq = vcpu_vtimer(vcpu)->irq.irq;
1016 ret = kvm_vgic_set_owner(vcpu, vtimer_irq, vcpu_vtimer(vcpu));
1017 if (ret)
1018 return false;
1019
1020 ptimer_irq = vcpu_ptimer(vcpu)->irq.irq;
1021 ret = kvm_vgic_set_owner(vcpu, ptimer_irq, vcpu_ptimer(vcpu));
1022 if (ret)
1023 return false;
1024
1025 kvm_for_each_vcpu(i, vcpu, vcpu->kvm) {
1026 if (vcpu_vtimer(vcpu)->irq.irq != vtimer_irq ||
1027 vcpu_ptimer(vcpu)->irq.irq != ptimer_irq)
1028 return false;
1029 }
1030
1031 return true;
1032}
1033
1034bool kvm_arch_timer_get_input_level(int vintid)
1035{
1036 struct kvm_vcpu *vcpu = kvm_arm_get_running_vcpu();
1037 struct arch_timer_context *timer;
1038
1039 if (vintid == vcpu_vtimer(vcpu)->irq.irq)
1040 timer = vcpu_vtimer(vcpu);
1041 else if (vintid == vcpu_ptimer(vcpu)->irq.irq)
1042 timer = vcpu_ptimer(vcpu);
1043 else
1044 BUG();
1045
1046 return kvm_timer_should_fire(timer);
1047}
1048
1049int kvm_timer_enable(struct kvm_vcpu *vcpu)
1050{
1051 struct arch_timer_cpu *timer = vcpu_timer(vcpu);
1052 struct timer_map map;
1053 int ret;
1054
1055 if (timer->enabled)
1056 return 0;
1057
1058
1059 if (!irqchip_in_kernel(vcpu->kvm))
1060 goto no_vgic;
1061
1062 if (!vgic_initialized(vcpu->kvm))
1063 return -ENODEV;
1064
1065 if (!timer_irqs_are_valid(vcpu)) {
1066 kvm_debug("incorrectly configured timer irqs\n");
1067 return -EINVAL;
1068 }
1069
1070 get_timer_map(vcpu, &map);
1071
1072 ret = kvm_vgic_map_phys_irq(vcpu,
1073 map.direct_vtimer->host_timer_irq,
1074 map.direct_vtimer->irq.irq,
1075 kvm_arch_timer_get_input_level);
1076 if (ret)
1077 return ret;
1078
1079 if (map.direct_ptimer) {
1080 ret = kvm_vgic_map_phys_irq(vcpu,
1081 map.direct_ptimer->host_timer_irq,
1082 map.direct_ptimer->irq.irq,
1083 kvm_arch_timer_get_input_level);
1084 }
1085
1086 if (ret)
1087 return ret;
1088
1089no_vgic:
1090 timer->enabled = 1;
1091 return 0;
1092}
1093
1094
1095
1096
1097
1098
1099
1100void kvm_timer_init_vhe(void)
1101{
1102
1103 u32 cnthctl_shift = 10;
1104 u64 val;
1105
1106
1107
1108
1109
1110 val = read_sysreg(cnthctl_el2);
1111 val |= (CNTHCTL_EL1PCEN << cnthctl_shift);
1112 val |= (CNTHCTL_EL1PCTEN << cnthctl_shift);
1113 write_sysreg(val, cnthctl_el2);
1114}
1115
1116static void set_timer_irqs(struct kvm *kvm, int vtimer_irq, int ptimer_irq)
1117{
1118 struct kvm_vcpu *vcpu;
1119 int i;
1120
1121 kvm_for_each_vcpu(i, vcpu, kvm) {
1122 vcpu_vtimer(vcpu)->irq.irq = vtimer_irq;
1123 vcpu_ptimer(vcpu)->irq.irq = ptimer_irq;
1124 }
1125}
1126
1127int kvm_arm_timer_set_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr)
1128{
1129 int __user *uaddr = (int __user *)(long)attr->addr;
1130 struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
1131 struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
1132 int irq;
1133
1134 if (!irqchip_in_kernel(vcpu->kvm))
1135 return -EINVAL;
1136
1137 if (get_user(irq, uaddr))
1138 return -EFAULT;
1139
1140 if (!(irq_is_ppi(irq)))
1141 return -EINVAL;
1142
1143 if (vcpu->arch.timer_cpu.enabled)
1144 return -EBUSY;
1145
1146 switch (attr->attr) {
1147 case KVM_ARM_VCPU_TIMER_IRQ_VTIMER:
1148 set_timer_irqs(vcpu->kvm, irq, ptimer->irq.irq);
1149 break;
1150 case KVM_ARM_VCPU_TIMER_IRQ_PTIMER:
1151 set_timer_irqs(vcpu->kvm, vtimer->irq.irq, irq);
1152 break;
1153 default:
1154 return -ENXIO;
1155 }
1156
1157 return 0;
1158}
1159
1160int kvm_arm_timer_get_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr)
1161{
1162 int __user *uaddr = (int __user *)(long)attr->addr;
1163 struct arch_timer_context *timer;
1164 int irq;
1165
1166 switch (attr->attr) {
1167 case KVM_ARM_VCPU_TIMER_IRQ_VTIMER:
1168 timer = vcpu_vtimer(vcpu);
1169 break;
1170 case KVM_ARM_VCPU_TIMER_IRQ_PTIMER:
1171 timer = vcpu_ptimer(vcpu);
1172 break;
1173 default:
1174 return -ENXIO;
1175 }
1176
1177 irq = timer->irq.irq;
1178 return put_user(irq, uaddr);
1179}
1180
1181int kvm_arm_timer_has_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr)
1182{
1183 switch (attr->attr) {
1184 case KVM_ARM_VCPU_TIMER_IRQ_VTIMER:
1185 case KVM_ARM_VCPU_TIMER_IRQ_PTIMER:
1186 return 0;
1187 }
1188
1189 return -ENXIO;
1190}
1191