1
2
3
4
5
6#include <linux/interrupt.h>
7#include <linux/irq.h>
8#include <linux/kvm.h>
9#include <linux/kvm_host.h>
10#include <linux/list_sort.h>
11#include <linux/nospec.h>
12
13#include <asm/kvm_hyp.h>
14
15#include "vgic.h"
16
17#define CREATE_TRACE_POINTS
18#include "trace.h"
19
20struct vgic_global kvm_vgic_global_state __ro_after_init = {
21 .gicv3_cpuif = STATIC_KEY_FALSE_INIT,
22};
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58static struct vgic_irq *vgic_get_lpi(struct kvm *kvm, u32 intid)
59{
60 struct vgic_dist *dist = &kvm->arch.vgic;
61 struct vgic_irq *irq = NULL;
62 unsigned long flags;
63
64 raw_spin_lock_irqsave(&dist->lpi_list_lock, flags);
65
66 list_for_each_entry(irq, &dist->lpi_list_head, lpi_list) {
67 if (irq->intid != intid)
68 continue;
69
70
71
72
73
74 vgic_get_irq_kref(irq);
75 goto out_unlock;
76 }
77 irq = NULL;
78
79out_unlock:
80 raw_spin_unlock_irqrestore(&dist->lpi_list_lock, flags);
81
82 return irq;
83}
84
85
86
87
88
89
90struct vgic_irq *vgic_get_irq(struct kvm *kvm, struct kvm_vcpu *vcpu,
91 u32 intid)
92{
93
94 if (intid <= VGIC_MAX_PRIVATE) {
95 intid = array_index_nospec(intid, VGIC_MAX_PRIVATE + 1);
96 return &vcpu->arch.vgic_cpu.private_irqs[intid];
97 }
98
99
100 if (intid < (kvm->arch.vgic.nr_spis + VGIC_NR_PRIVATE_IRQS)) {
101 intid = array_index_nospec(intid, kvm->arch.vgic.nr_spis + VGIC_NR_PRIVATE_IRQS);
102 return &kvm->arch.vgic.spis[intid - VGIC_NR_PRIVATE_IRQS];
103 }
104
105
106 if (intid >= VGIC_MIN_LPI)
107 return vgic_get_lpi(kvm, intid);
108
109 return NULL;
110}
111
112
113
114
115
116
117static void vgic_irq_release(struct kref *ref)
118{
119}
120
121
122
123
124void __vgic_put_lpi_locked(struct kvm *kvm, struct vgic_irq *irq)
125{
126 struct vgic_dist *dist = &kvm->arch.vgic;
127
128 if (!kref_put(&irq->refcount, vgic_irq_release))
129 return;
130
131 list_del(&irq->lpi_list);
132 dist->lpi_list_count--;
133
134 kfree(irq);
135}
136
137void vgic_put_irq(struct kvm *kvm, struct vgic_irq *irq)
138{
139 struct vgic_dist *dist = &kvm->arch.vgic;
140 unsigned long flags;
141
142 if (irq->intid < VGIC_MIN_LPI)
143 return;
144
145 raw_spin_lock_irqsave(&dist->lpi_list_lock, flags);
146 __vgic_put_lpi_locked(kvm, irq);
147 raw_spin_unlock_irqrestore(&dist->lpi_list_lock, flags);
148}
149
150void vgic_flush_pending_lpis(struct kvm_vcpu *vcpu)
151{
152 struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
153 struct vgic_irq *irq, *tmp;
154 unsigned long flags;
155
156 raw_spin_lock_irqsave(&vgic_cpu->ap_list_lock, flags);
157
158 list_for_each_entry_safe(irq, tmp, &vgic_cpu->ap_list_head, ap_list) {
159 if (irq->intid >= VGIC_MIN_LPI) {
160 raw_spin_lock(&irq->irq_lock);
161 list_del(&irq->ap_list);
162 irq->vcpu = NULL;
163 raw_spin_unlock(&irq->irq_lock);
164 vgic_put_irq(vcpu->kvm, irq);
165 }
166 }
167
168 raw_spin_unlock_irqrestore(&vgic_cpu->ap_list_lock, flags);
169}
170
171void vgic_irq_set_phys_pending(struct vgic_irq *irq, bool pending)
172{
173 WARN_ON(irq_set_irqchip_state(irq->host_irq,
174 IRQCHIP_STATE_PENDING,
175 pending));
176}
177
178bool vgic_get_phys_line_level(struct vgic_irq *irq)
179{
180 bool line_level;
181
182 BUG_ON(!irq->hw);
183
184 if (irq->ops && irq->ops->get_input_level)
185 return irq->ops->get_input_level(irq->intid);
186
187 WARN_ON(irq_get_irqchip_state(irq->host_irq,
188 IRQCHIP_STATE_PENDING,
189 &line_level));
190 return line_level;
191}
192
193
194void vgic_irq_set_phys_active(struct vgic_irq *irq, bool active)
195{
196
197 BUG_ON(!irq->hw);
198 WARN_ON(irq_set_irqchip_state(irq->host_irq,
199 IRQCHIP_STATE_ACTIVE,
200 active));
201}
202
203
204
205
206
207
208
209
210
211
212
213
214static struct kvm_vcpu *vgic_target_oracle(struct vgic_irq *irq)
215{
216 lockdep_assert_held(&irq->irq_lock);
217
218
219 if (irq->active)
220 return irq->vcpu ? : irq->target_vcpu;
221
222
223
224
225
226
227
228 if (irq->enabled && irq_is_pending(irq)) {
229 if (unlikely(irq->target_vcpu &&
230 !irq->target_vcpu->kvm->arch.vgic.enabled))
231 return NULL;
232
233 return irq->target_vcpu;
234 }
235
236
237
238
239 return NULL;
240}
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257static int vgic_irq_cmp(void *priv, const struct list_head *a,
258 const struct list_head *b)
259{
260 struct vgic_irq *irqa = container_of(a, struct vgic_irq, ap_list);
261 struct vgic_irq *irqb = container_of(b, struct vgic_irq, ap_list);
262 bool penda, pendb;
263 int ret;
264
265
266
267
268
269 if (unlikely(irqa == irqb))
270 return 0;
271
272 raw_spin_lock(&irqa->irq_lock);
273 raw_spin_lock_nested(&irqb->irq_lock, SINGLE_DEPTH_NESTING);
274
275 if (irqa->active || irqb->active) {
276 ret = (int)irqb->active - (int)irqa->active;
277 goto out;
278 }
279
280 penda = irqa->enabled && irq_is_pending(irqa);
281 pendb = irqb->enabled && irq_is_pending(irqb);
282
283 if (!penda || !pendb) {
284 ret = (int)pendb - (int)penda;
285 goto out;
286 }
287
288
289 ret = irqa->priority - irqb->priority;
290out:
291 raw_spin_unlock(&irqb->irq_lock);
292 raw_spin_unlock(&irqa->irq_lock);
293 return ret;
294}
295
296
297static void vgic_sort_ap_list(struct kvm_vcpu *vcpu)
298{
299 struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
300
301 lockdep_assert_held(&vgic_cpu->ap_list_lock);
302
303 list_sort(NULL, &vgic_cpu->ap_list_head, vgic_irq_cmp);
304}
305
306
307
308
309
310
311static bool vgic_validate_injection(struct vgic_irq *irq, bool level, void *owner)
312{
313 if (irq->owner != owner)
314 return false;
315
316 switch (irq->config) {
317 case VGIC_CONFIG_LEVEL:
318 return irq->line_level != level;
319 case VGIC_CONFIG_EDGE:
320 return level;
321 }
322
323 return false;
324}
325
326
327
328
329
330
331
332
333
334bool vgic_queue_irq_unlock(struct kvm *kvm, struct vgic_irq *irq,
335 unsigned long flags)
336{
337 struct kvm_vcpu *vcpu;
338
339 lockdep_assert_held(&irq->irq_lock);
340
341retry:
342 vcpu = vgic_target_oracle(irq);
343 if (irq->vcpu || !vcpu) {
344
345
346
347
348
349
350
351
352
353 raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
354
355
356
357
358
359
360
361
362
363
364 if (vcpu) {
365 kvm_make_request(KVM_REQ_IRQ_PENDING, vcpu);
366 kvm_vcpu_kick(vcpu);
367 }
368 return false;
369 }
370
371
372
373
374
375 raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
376
377
378
379 raw_spin_lock_irqsave(&vcpu->arch.vgic_cpu.ap_list_lock, flags);
380 raw_spin_lock(&irq->irq_lock);
381
382
383
384
385
386
387
388
389
390
391
392
393
394 if (unlikely(irq->vcpu || vcpu != vgic_target_oracle(irq))) {
395 raw_spin_unlock(&irq->irq_lock);
396 raw_spin_unlock_irqrestore(&vcpu->arch.vgic_cpu.ap_list_lock,
397 flags);
398
399 raw_spin_lock_irqsave(&irq->irq_lock, flags);
400 goto retry;
401 }
402
403
404
405
406
407 vgic_get_irq_kref(irq);
408 list_add_tail(&irq->ap_list, &vcpu->arch.vgic_cpu.ap_list_head);
409 irq->vcpu = vcpu;
410
411 raw_spin_unlock(&irq->irq_lock);
412 raw_spin_unlock_irqrestore(&vcpu->arch.vgic_cpu.ap_list_lock, flags);
413
414 kvm_make_request(KVM_REQ_IRQ_PENDING, vcpu);
415 kvm_vcpu_kick(vcpu);
416
417 return true;
418}
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437int kvm_vgic_inject_irq(struct kvm *kvm, int cpuid, unsigned int intid,
438 bool level, void *owner)
439{
440 struct kvm_vcpu *vcpu;
441 struct vgic_irq *irq;
442 unsigned long flags;
443 int ret;
444
445 trace_vgic_update_irq_pending(cpuid, intid, level);
446
447 ret = vgic_lazy_init(kvm);
448 if (ret)
449 return ret;
450
451 vcpu = kvm_get_vcpu(kvm, cpuid);
452 if (!vcpu && intid < VGIC_NR_PRIVATE_IRQS)
453 return -EINVAL;
454
455 irq = vgic_get_irq(kvm, vcpu, intid);
456 if (!irq)
457 return -EINVAL;
458
459 raw_spin_lock_irqsave(&irq->irq_lock, flags);
460
461 if (!vgic_validate_injection(irq, level, owner)) {
462
463 raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
464 vgic_put_irq(kvm, irq);
465 return 0;
466 }
467
468 if (irq->config == VGIC_CONFIG_LEVEL)
469 irq->line_level = level;
470 else
471 irq->pending_latch = true;
472
473 vgic_queue_irq_unlock(kvm, irq, flags);
474 vgic_put_irq(kvm, irq);
475
476 return 0;
477}
478
479
480static int kvm_vgic_map_irq(struct kvm_vcpu *vcpu, struct vgic_irq *irq,
481 unsigned int host_irq,
482 struct irq_ops *ops)
483{
484 struct irq_desc *desc;
485 struct irq_data *data;
486
487
488
489
490 desc = irq_to_desc(host_irq);
491 if (!desc) {
492 kvm_err("%s: no interrupt descriptor\n", __func__);
493 return -EINVAL;
494 }
495 data = irq_desc_get_irq_data(desc);
496 while (data->parent_data)
497 data = data->parent_data;
498
499 irq->hw = true;
500 irq->host_irq = host_irq;
501 irq->hwintid = data->hwirq;
502 irq->ops = ops;
503 return 0;
504}
505
506
507static inline void kvm_vgic_unmap_irq(struct vgic_irq *irq)
508{
509 irq->hw = false;
510 irq->hwintid = 0;
511 irq->ops = NULL;
512}
513
514int kvm_vgic_map_phys_irq(struct kvm_vcpu *vcpu, unsigned int host_irq,
515 u32 vintid, struct irq_ops *ops)
516{
517 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, vintid);
518 unsigned long flags;
519 int ret;
520
521 BUG_ON(!irq);
522
523 raw_spin_lock_irqsave(&irq->irq_lock, flags);
524 ret = kvm_vgic_map_irq(vcpu, irq, host_irq, ops);
525 raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
526 vgic_put_irq(vcpu->kvm, irq);
527
528 return ret;
529}
530
531
532
533
534
535
536
537
538
539
540void kvm_vgic_reset_mapped_irq(struct kvm_vcpu *vcpu, u32 vintid)
541{
542 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, vintid);
543 unsigned long flags;
544
545 if (!irq->hw)
546 goto out;
547
548 raw_spin_lock_irqsave(&irq->irq_lock, flags);
549 irq->active = false;
550 irq->pending_latch = false;
551 irq->line_level = false;
552 raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
553out:
554 vgic_put_irq(vcpu->kvm, irq);
555}
556
557int kvm_vgic_unmap_phys_irq(struct kvm_vcpu *vcpu, unsigned int vintid)
558{
559 struct vgic_irq *irq;
560 unsigned long flags;
561
562 if (!vgic_initialized(vcpu->kvm))
563 return -EAGAIN;
564
565 irq = vgic_get_irq(vcpu->kvm, vcpu, vintid);
566 BUG_ON(!irq);
567
568 raw_spin_lock_irqsave(&irq->irq_lock, flags);
569 kvm_vgic_unmap_irq(irq);
570 raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
571 vgic_put_irq(vcpu->kvm, irq);
572
573 return 0;
574}
575
576
577
578
579
580
581
582
583
584
585
586int kvm_vgic_set_owner(struct kvm_vcpu *vcpu, unsigned int intid, void *owner)
587{
588 struct vgic_irq *irq;
589 unsigned long flags;
590 int ret = 0;
591
592 if (!vgic_initialized(vcpu->kvm))
593 return -EAGAIN;
594
595
596 if (!irq_is_ppi(intid) && !vgic_valid_spi(vcpu->kvm, intid))
597 return -EINVAL;
598
599 irq = vgic_get_irq(vcpu->kvm, vcpu, intid);
600 raw_spin_lock_irqsave(&irq->irq_lock, flags);
601 if (irq->owner && irq->owner != owner)
602 ret = -EEXIST;
603 else
604 irq->owner = owner;
605 raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
606
607 return ret;
608}
609
610
611
612
613
614
615
616
617
618static void vgic_prune_ap_list(struct kvm_vcpu *vcpu)
619{
620 struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
621 struct vgic_irq *irq, *tmp;
622
623 DEBUG_SPINLOCK_BUG_ON(!irqs_disabled());
624
625retry:
626 raw_spin_lock(&vgic_cpu->ap_list_lock);
627
628 list_for_each_entry_safe(irq, tmp, &vgic_cpu->ap_list_head, ap_list) {
629 struct kvm_vcpu *target_vcpu, *vcpuA, *vcpuB;
630 bool target_vcpu_needs_kick = false;
631
632 raw_spin_lock(&irq->irq_lock);
633
634 BUG_ON(vcpu != irq->vcpu);
635
636 target_vcpu = vgic_target_oracle(irq);
637
638 if (!target_vcpu) {
639
640
641
642
643 list_del(&irq->ap_list);
644 irq->vcpu = NULL;
645 raw_spin_unlock(&irq->irq_lock);
646
647
648
649
650
651
652
653
654 vgic_put_irq(vcpu->kvm, irq);
655 continue;
656 }
657
658 if (target_vcpu == vcpu) {
659
660 raw_spin_unlock(&irq->irq_lock);
661 continue;
662 }
663
664
665
666 raw_spin_unlock(&irq->irq_lock);
667 raw_spin_unlock(&vgic_cpu->ap_list_lock);
668
669
670
671
672
673 if (vcpu->vcpu_id < target_vcpu->vcpu_id) {
674 vcpuA = vcpu;
675 vcpuB = target_vcpu;
676 } else {
677 vcpuA = target_vcpu;
678 vcpuB = vcpu;
679 }
680
681 raw_spin_lock(&vcpuA->arch.vgic_cpu.ap_list_lock);
682 raw_spin_lock_nested(&vcpuB->arch.vgic_cpu.ap_list_lock,
683 SINGLE_DEPTH_NESTING);
684 raw_spin_lock(&irq->irq_lock);
685
686
687
688
689
690
691
692
693
694
695 if (target_vcpu == vgic_target_oracle(irq)) {
696 struct vgic_cpu *new_cpu = &target_vcpu->arch.vgic_cpu;
697
698 list_del(&irq->ap_list);
699 irq->vcpu = target_vcpu;
700 list_add_tail(&irq->ap_list, &new_cpu->ap_list_head);
701 target_vcpu_needs_kick = true;
702 }
703
704 raw_spin_unlock(&irq->irq_lock);
705 raw_spin_unlock(&vcpuB->arch.vgic_cpu.ap_list_lock);
706 raw_spin_unlock(&vcpuA->arch.vgic_cpu.ap_list_lock);
707
708 if (target_vcpu_needs_kick) {
709 kvm_make_request(KVM_REQ_IRQ_PENDING, target_vcpu);
710 kvm_vcpu_kick(target_vcpu);
711 }
712
713 goto retry;
714 }
715
716 raw_spin_unlock(&vgic_cpu->ap_list_lock);
717}
718
719static inline void vgic_fold_lr_state(struct kvm_vcpu *vcpu)
720{
721 if (kvm_vgic_global_state.type == VGIC_V2)
722 vgic_v2_fold_lr_state(vcpu);
723 else
724 vgic_v3_fold_lr_state(vcpu);
725}
726
727
728static inline void vgic_populate_lr(struct kvm_vcpu *vcpu,
729 struct vgic_irq *irq, int lr)
730{
731 lockdep_assert_held(&irq->irq_lock);
732
733 if (kvm_vgic_global_state.type == VGIC_V2)
734 vgic_v2_populate_lr(vcpu, irq, lr);
735 else
736 vgic_v3_populate_lr(vcpu, irq, lr);
737}
738
739static inline void vgic_clear_lr(struct kvm_vcpu *vcpu, int lr)
740{
741 if (kvm_vgic_global_state.type == VGIC_V2)
742 vgic_v2_clear_lr(vcpu, lr);
743 else
744 vgic_v3_clear_lr(vcpu, lr);
745}
746
747static inline void vgic_set_underflow(struct kvm_vcpu *vcpu)
748{
749 if (kvm_vgic_global_state.type == VGIC_V2)
750 vgic_v2_set_underflow(vcpu);
751 else
752 vgic_v3_set_underflow(vcpu);
753}
754
755
756static int compute_ap_list_depth(struct kvm_vcpu *vcpu,
757 bool *multi_sgi)
758{
759 struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
760 struct vgic_irq *irq;
761 int count = 0;
762
763 *multi_sgi = false;
764
765 lockdep_assert_held(&vgic_cpu->ap_list_lock);
766
767 list_for_each_entry(irq, &vgic_cpu->ap_list_head, ap_list) {
768 int w;
769
770 raw_spin_lock(&irq->irq_lock);
771
772 w = vgic_irq_get_lr_count(irq);
773 raw_spin_unlock(&irq->irq_lock);
774
775 count += w;
776 *multi_sgi |= (w > 1);
777 }
778 return count;
779}
780
781
782static void vgic_flush_lr_state(struct kvm_vcpu *vcpu)
783{
784 struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
785 struct vgic_irq *irq;
786 int count;
787 bool multi_sgi;
788 u8 prio = 0xff;
789 int i = 0;
790
791 lockdep_assert_held(&vgic_cpu->ap_list_lock);
792
793 count = compute_ap_list_depth(vcpu, &multi_sgi);
794 if (count > kvm_vgic_global_state.nr_lr || multi_sgi)
795 vgic_sort_ap_list(vcpu);
796
797 count = 0;
798
799 list_for_each_entry(irq, &vgic_cpu->ap_list_head, ap_list) {
800 raw_spin_lock(&irq->irq_lock);
801
802
803
804
805
806
807
808
809 if (multi_sgi && irq->priority > prio) {
810 _raw_spin_unlock(&irq->irq_lock);
811 break;
812 }
813
814 if (likely(vgic_target_oracle(irq) == vcpu)) {
815 vgic_populate_lr(vcpu, irq, count++);
816
817 if (irq->source)
818 prio = irq->priority;
819 }
820
821 raw_spin_unlock(&irq->irq_lock);
822
823 if (count == kvm_vgic_global_state.nr_lr) {
824 if (!list_is_last(&irq->ap_list,
825 &vgic_cpu->ap_list_head))
826 vgic_set_underflow(vcpu);
827 break;
828 }
829 }
830
831
832 for (i = count ; i < kvm_vgic_global_state.nr_lr; i++)
833 vgic_clear_lr(vcpu, i);
834
835 if (!static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif))
836 vcpu->arch.vgic_cpu.vgic_v2.used_lrs = count;
837 else
838 vcpu->arch.vgic_cpu.vgic_v3.used_lrs = count;
839}
840
841static inline bool can_access_vgic_from_kernel(void)
842{
843
844
845
846
847
848 return !static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif) || has_vhe();
849}
850
851static inline void vgic_save_state(struct kvm_vcpu *vcpu)
852{
853 if (!static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif))
854 vgic_v2_save_state(vcpu);
855 else
856 __vgic_v3_save_state(&vcpu->arch.vgic_cpu.vgic_v3);
857}
858
859
860void kvm_vgic_sync_hwstate(struct kvm_vcpu *vcpu)
861{
862 int used_lrs;
863
864
865 if (list_empty(&vcpu->arch.vgic_cpu.ap_list_head))
866 return;
867
868 if (can_access_vgic_from_kernel())
869 vgic_save_state(vcpu);
870
871 if (!static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif))
872 used_lrs = vcpu->arch.vgic_cpu.vgic_v2.used_lrs;
873 else
874 used_lrs = vcpu->arch.vgic_cpu.vgic_v3.used_lrs;
875
876 if (used_lrs)
877 vgic_fold_lr_state(vcpu);
878 vgic_prune_ap_list(vcpu);
879}
880
881static inline void vgic_restore_state(struct kvm_vcpu *vcpu)
882{
883 if (!static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif))
884 vgic_v2_restore_state(vcpu);
885 else
886 __vgic_v3_restore_state(&vcpu->arch.vgic_cpu.vgic_v3);
887}
888
889
890void kvm_vgic_flush_hwstate(struct kvm_vcpu *vcpu)
891{
892
893
894
895
896
897
898
899
900
901
902
903
904 if (list_empty(&vcpu->arch.vgic_cpu.ap_list_head) &&
905 !vgic_supports_direct_msis(vcpu->kvm))
906 return;
907
908 DEBUG_SPINLOCK_BUG_ON(!irqs_disabled());
909
910 if (!list_empty(&vcpu->arch.vgic_cpu.ap_list_head)) {
911 raw_spin_lock(&vcpu->arch.vgic_cpu.ap_list_lock);
912 vgic_flush_lr_state(vcpu);
913 raw_spin_unlock(&vcpu->arch.vgic_cpu.ap_list_lock);
914 }
915
916 if (can_access_vgic_from_kernel())
917 vgic_restore_state(vcpu);
918
919 if (vgic_supports_direct_msis(vcpu->kvm))
920 vgic_v4_commit(vcpu);
921}
922
923void kvm_vgic_load(struct kvm_vcpu *vcpu)
924{
925 if (unlikely(!vgic_initialized(vcpu->kvm)))
926 return;
927
928 if (kvm_vgic_global_state.type == VGIC_V2)
929 vgic_v2_load(vcpu);
930 else
931 vgic_v3_load(vcpu);
932}
933
934void kvm_vgic_put(struct kvm_vcpu *vcpu)
935{
936 if (unlikely(!vgic_initialized(vcpu->kvm)))
937 return;
938
939 if (kvm_vgic_global_state.type == VGIC_V2)
940 vgic_v2_put(vcpu);
941 else
942 vgic_v3_put(vcpu);
943}
944
945void kvm_vgic_vmcr_sync(struct kvm_vcpu *vcpu)
946{
947 if (unlikely(!irqchip_in_kernel(vcpu->kvm)))
948 return;
949
950 if (kvm_vgic_global_state.type == VGIC_V2)
951 vgic_v2_vmcr_sync(vcpu);
952 else
953 vgic_v3_vmcr_sync(vcpu);
954}
955
956int kvm_vgic_vcpu_pending_irq(struct kvm_vcpu *vcpu)
957{
958 struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
959 struct vgic_irq *irq;
960 bool pending = false;
961 unsigned long flags;
962 struct vgic_vmcr vmcr;
963
964 if (!vcpu->kvm->arch.vgic.enabled)
965 return false;
966
967 if (vcpu->arch.vgic_cpu.vgic_v3.its_vpe.pending_last)
968 return true;
969
970 vgic_get_vmcr(vcpu, &vmcr);
971
972 raw_spin_lock_irqsave(&vgic_cpu->ap_list_lock, flags);
973
974 list_for_each_entry(irq, &vgic_cpu->ap_list_head, ap_list) {
975 raw_spin_lock(&irq->irq_lock);
976 pending = irq_is_pending(irq) && irq->enabled &&
977 !irq->active &&
978 irq->priority < vmcr.pmr;
979 raw_spin_unlock(&irq->irq_lock);
980
981 if (pending)
982 break;
983 }
984
985 raw_spin_unlock_irqrestore(&vgic_cpu->ap_list_lock, flags);
986
987 return pending;
988}
989
990void vgic_kick_vcpus(struct kvm *kvm)
991{
992 struct kvm_vcpu *vcpu;
993 int c;
994
995
996
997
998
999 kvm_for_each_vcpu(c, vcpu, kvm) {
1000 if (kvm_vgic_vcpu_pending_irq(vcpu)) {
1001 kvm_make_request(KVM_REQ_IRQ_PENDING, vcpu);
1002 kvm_vcpu_kick(vcpu);
1003 }
1004 }
1005}
1006
1007bool kvm_vgic_map_is_active(struct kvm_vcpu *vcpu, unsigned int vintid)
1008{
1009 struct vgic_irq *irq;
1010 bool map_is_active;
1011 unsigned long flags;
1012
1013 if (!vgic_initialized(vcpu->kvm))
1014 return false;
1015
1016 irq = vgic_get_irq(vcpu->kvm, vcpu, vintid);
1017 raw_spin_lock_irqsave(&irq->irq_lock, flags);
1018 map_is_active = irq->hw && irq->active;
1019 raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
1020 vgic_put_irq(vcpu->kvm, irq);
1021
1022 return map_is_active;
1023}
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045void vgic_irq_handle_resampling(struct vgic_irq *irq,
1046 bool lr_deactivated, bool lr_pending)
1047{
1048 if (vgic_irq_is_mapped_level(irq)) {
1049 bool resample = false;
1050
1051 if (unlikely(vgic_irq_needs_resampling(irq))) {
1052 resample = !(irq->active || irq->pending_latch);
1053 } else if (lr_pending || (lr_deactivated && irq->line_level)) {
1054 irq->line_level = vgic_get_phys_line_level(irq);
1055 resample = !irq->line_level;
1056 }
1057
1058 if (resample)
1059 vgic_irq_set_phys_active(irq, false);
1060 }
1061}
1062