1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17#include <linux/uaccess.h>
18#include <linux/interrupt.h>
19#include <linux/cpu.h>
20#include <linux/kvm_host.h>
21#include <kvm/arm_vgic.h>
22#include <asm/kvm_mmu.h>
23#include "vgic.h"
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62void kvm_vgic_early_init(struct kvm *kvm)
63{
64 struct vgic_dist *dist = &kvm->arch.vgic;
65
66 INIT_LIST_HEAD(&dist->lpi_list_head);
67 spin_lock_init(&dist->lpi_list_lock);
68}
69
70
71
72
73
74
75
76
77void kvm_vgic_vcpu_early_init(struct kvm_vcpu *vcpu)
78{
79 struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
80 int i;
81
82 INIT_LIST_HEAD(&vgic_cpu->ap_list_head);
83 spin_lock_init(&vgic_cpu->ap_list_lock);
84
85
86
87
88
89 for (i = 0; i < VGIC_NR_PRIVATE_IRQS; i++) {
90 struct vgic_irq *irq = &vgic_cpu->private_irqs[i];
91
92 INIT_LIST_HEAD(&irq->ap_list);
93 spin_lock_init(&irq->irq_lock);
94 irq->intid = i;
95 irq->vcpu = NULL;
96 irq->target_vcpu = vcpu;
97 irq->targets = 1U << vcpu->vcpu_id;
98 kref_init(&irq->refcount);
99 if (vgic_irq_is_sgi(i)) {
100
101 irq->enabled = 1;
102 irq->config = VGIC_CONFIG_EDGE;
103 } else {
104
105 irq->config = VGIC_CONFIG_LEVEL;
106 }
107 }
108}
109
110
111
112
113
114
115
116
117
118
119
120int kvm_vgic_create(struct kvm *kvm, u32 type)
121{
122 int i, vcpu_lock_idx = -1, ret;
123 struct kvm_vcpu *vcpu;
124
125 if (irqchip_in_kernel(kvm))
126 return -EEXIST;
127
128
129
130
131
132
133
134 if (type == KVM_DEV_TYPE_ARM_VGIC_V2 &&
135 !kvm_vgic_global_state.can_emulate_gicv2)
136 return -ENODEV;
137
138
139
140
141
142
143 ret = -EBUSY;
144 kvm_for_each_vcpu(i, vcpu, kvm) {
145 if (!mutex_trylock(&vcpu->mutex))
146 goto out_unlock;
147 vcpu_lock_idx = i;
148 }
149
150 kvm_for_each_vcpu(i, vcpu, kvm) {
151 if (vcpu->arch.has_run_once)
152 goto out_unlock;
153 }
154 ret = 0;
155
156 if (type == KVM_DEV_TYPE_ARM_VGIC_V2)
157 kvm->arch.max_vcpus = VGIC_V2_MAX_CPUS;
158 else
159 kvm->arch.max_vcpus = VGIC_V3_MAX_CPUS;
160
161 if (atomic_read(&kvm->online_vcpus) > kvm->arch.max_vcpus) {
162 ret = -E2BIG;
163 goto out_unlock;
164 }
165
166 kvm->arch.vgic.in_kernel = true;
167 kvm->arch.vgic.vgic_model = type;
168
169
170
171
172
173 kvm->arch.vgic.vctrl_base = kvm_vgic_global_state.vctrl_base;
174
175 kvm->arch.vgic.vgic_dist_base = VGIC_ADDR_UNDEF;
176 kvm->arch.vgic.vgic_cpu_base = VGIC_ADDR_UNDEF;
177 kvm->arch.vgic.vgic_redist_base = VGIC_ADDR_UNDEF;
178
179out_unlock:
180 for (; vcpu_lock_idx >= 0; vcpu_lock_idx--) {
181 vcpu = kvm_get_vcpu(kvm, vcpu_lock_idx);
182 mutex_unlock(&vcpu->mutex);
183 }
184 return ret;
185}
186
187
188
189
190
191
192
193
194static int kvm_vgic_dist_init(struct kvm *kvm, unsigned int nr_spis)
195{
196 struct vgic_dist *dist = &kvm->arch.vgic;
197 struct kvm_vcpu *vcpu0 = kvm_get_vcpu(kvm, 0);
198 int i;
199
200 dist->spis = kcalloc(nr_spis, sizeof(struct vgic_irq), GFP_KERNEL);
201 if (!dist->spis)
202 return -ENOMEM;
203
204
205
206
207
208
209
210
211
212 for (i = 0; i < nr_spis; i++) {
213 struct vgic_irq *irq = &dist->spis[i];
214
215 irq->intid = i + VGIC_NR_PRIVATE_IRQS;
216 INIT_LIST_HEAD(&irq->ap_list);
217 spin_lock_init(&irq->irq_lock);
218 irq->vcpu = NULL;
219 irq->target_vcpu = vcpu0;
220 kref_init(&irq->refcount);
221 if (dist->vgic_model == KVM_DEV_TYPE_ARM_VGIC_V2)
222 irq->targets = 0;
223 else
224 irq->mpidr = 0;
225 }
226 return 0;
227}
228
229
230
231
232
233int kvm_vgic_vcpu_init(struct kvm_vcpu *vcpu)
234{
235 int ret = 0;
236 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
237
238 if (!irqchip_in_kernel(vcpu->kvm))
239 return 0;
240
241
242
243
244
245 if (dist->vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3) {
246 mutex_lock(&vcpu->kvm->lock);
247 ret = vgic_register_redist_iodev(vcpu);
248 mutex_unlock(&vcpu->kvm->lock);
249 }
250 return ret;
251}
252
253static void kvm_vgic_vcpu_enable(struct kvm_vcpu *vcpu)
254{
255 if (kvm_vgic_global_state.type == VGIC_V2)
256 vgic_v2_enable(vcpu);
257 else
258 vgic_v3_enable(vcpu);
259}
260
261
262
263
264
265
266
267
268
269
270
271int vgic_init(struct kvm *kvm)
272{
273 struct vgic_dist *dist = &kvm->arch.vgic;
274 struct kvm_vcpu *vcpu;
275 int ret = 0, i;
276
277 if (vgic_initialized(kvm))
278 return 0;
279
280
281 if (!dist->nr_spis)
282 dist->nr_spis = VGIC_NR_IRQS_LEGACY - VGIC_NR_PRIVATE_IRQS;
283
284 ret = kvm_vgic_dist_init(kvm, dist->nr_spis);
285 if (ret)
286 goto out;
287
288 if (vgic_has_its(kvm)) {
289 ret = vgic_v4_init(kvm);
290 if (ret)
291 goto out;
292 }
293
294 kvm_for_each_vcpu(i, vcpu, kvm)
295 kvm_vgic_vcpu_enable(vcpu);
296
297 ret = kvm_vgic_setup_default_irq_routing(kvm);
298 if (ret)
299 goto out;
300
301 vgic_debug_init(kvm);
302
303 dist->initialized = true;
304
305
306
307
308
309
310
311 preempt_disable();
312 vcpu = kvm_arm_get_running_vcpu();
313 if (vcpu)
314 kvm_vgic_load(vcpu);
315 preempt_enable();
316out:
317 return ret;
318}
319
320static void kvm_vgic_dist_destroy(struct kvm *kvm)
321{
322 struct vgic_dist *dist = &kvm->arch.vgic;
323
324 dist->ready = false;
325 dist->initialized = false;
326
327 kfree(dist->spis);
328 dist->nr_spis = 0;
329
330 if (vgic_supports_direct_msis(kvm))
331 vgic_v4_teardown(kvm);
332}
333
334void kvm_vgic_vcpu_destroy(struct kvm_vcpu *vcpu)
335{
336 struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
337
338 INIT_LIST_HEAD(&vgic_cpu->ap_list_head);
339}
340
341
342static void __kvm_vgic_destroy(struct kvm *kvm)
343{
344 struct kvm_vcpu *vcpu;
345 int i;
346
347 vgic_debug_destroy(kvm);
348
349 kvm_vgic_dist_destroy(kvm);
350
351 kvm_for_each_vcpu(i, vcpu, kvm)
352 kvm_vgic_vcpu_destroy(vcpu);
353}
354
355void kvm_vgic_destroy(struct kvm *kvm)
356{
357 mutex_lock(&kvm->lock);
358 __kvm_vgic_destroy(kvm);
359 mutex_unlock(&kvm->lock);
360}
361
362
363
364
365
366
367
368int vgic_lazy_init(struct kvm *kvm)
369{
370 int ret = 0;
371
372 if (unlikely(!vgic_initialized(kvm))) {
373
374
375
376
377
378
379 if (kvm->arch.vgic.vgic_model != KVM_DEV_TYPE_ARM_VGIC_V2)
380 return -EBUSY;
381
382 mutex_lock(&kvm->lock);
383 ret = vgic_init(kvm);
384 mutex_unlock(&kvm->lock);
385 }
386
387 return ret;
388}
389
390
391
392
393
394
395
396
397
398
399
400int kvm_vgic_map_resources(struct kvm *kvm)
401{
402 struct vgic_dist *dist = &kvm->arch.vgic;
403 int ret = 0;
404
405 mutex_lock(&kvm->lock);
406 if (!irqchip_in_kernel(kvm))
407 goto out;
408
409 if (dist->vgic_model == KVM_DEV_TYPE_ARM_VGIC_V2)
410 ret = vgic_v2_map_resources(kvm);
411 else
412 ret = vgic_v3_map_resources(kvm);
413
414 if (ret)
415 __kvm_vgic_destroy(kvm);
416
417out:
418 mutex_unlock(&kvm->lock);
419 return ret;
420}
421
422
423
424static int vgic_init_cpu_starting(unsigned int cpu)
425{
426 enable_percpu_irq(kvm_vgic_global_state.maint_irq, 0);
427 return 0;
428}
429
430
431static int vgic_init_cpu_dying(unsigned int cpu)
432{
433 disable_percpu_irq(kvm_vgic_global_state.maint_irq);
434 return 0;
435}
436
437static irqreturn_t vgic_maintenance_handler(int irq, void *data)
438{
439
440
441
442
443
444
445 return IRQ_HANDLED;
446}
447
448
449
450
451
452
453void kvm_vgic_init_cpu_hardware(void)
454{
455 BUG_ON(preemptible());
456
457
458
459
460
461 if (kvm_vgic_global_state.type == VGIC_V2)
462 vgic_v2_init_lrs();
463 else
464 kvm_call_hyp(__vgic_v3_init_lrs);
465}
466
467
468
469
470
471
472
473int kvm_vgic_hyp_init(void)
474{
475 const struct gic_kvm_info *gic_kvm_info;
476 int ret;
477
478 gic_kvm_info = gic_get_kvm_info();
479 if (!gic_kvm_info)
480 return -ENODEV;
481
482 if (!gic_kvm_info->maint_irq) {
483 kvm_err("No vgic maintenance irq\n");
484 return -ENXIO;
485 }
486
487 switch (gic_kvm_info->type) {
488 case GIC_V2:
489 ret = vgic_v2_probe(gic_kvm_info);
490 break;
491 case GIC_V3:
492 ret = vgic_v3_probe(gic_kvm_info);
493 if (!ret) {
494 static_branch_enable(&kvm_vgic_global_state.gicv3_cpuif);
495 kvm_info("GIC system register CPU interface enabled\n");
496 }
497 break;
498 default:
499 ret = -ENODEV;
500 };
501
502 if (ret)
503 return ret;
504
505 kvm_vgic_global_state.maint_irq = gic_kvm_info->maint_irq;
506 ret = request_percpu_irq(kvm_vgic_global_state.maint_irq,
507 vgic_maintenance_handler,
508 "vgic", kvm_get_running_vcpus());
509 if (ret) {
510 kvm_err("Cannot register interrupt %d\n",
511 kvm_vgic_global_state.maint_irq);
512 return ret;
513 }
514
515 ret = cpuhp_setup_state(CPUHP_AP_KVM_ARM_VGIC_INIT_STARTING,
516 "kvm/arm/vgic:starting",
517 vgic_init_cpu_starting, vgic_init_cpu_dying);
518 if (ret) {
519 kvm_err("Cannot register vgic CPU notifier\n");
520 goto out_free_irq;
521 }
522
523 kvm_info("vgic interrupt IRQ%d\n", kvm_vgic_global_state.maint_irq);
524 return 0;
525
526out_free_irq:
527 free_percpu_irq(kvm_vgic_global_state.maint_irq,
528 kvm_get_running_vcpus());
529 return ret;
530}
531