1
2
3
4
5
6
7
8
9
10
11
12
13
14#include <linux/bitops.h>
15#include <linux/bsearch.h>
16#include <linux/kvm.h>
17#include <linux/kvm_host.h>
18#include <kvm/iodev.h>
19#include <kvm/arm_vgic.h>
20
21#include "vgic.h"
22#include "vgic-mmio.h"
23
24unsigned long vgic_mmio_read_raz(struct kvm_vcpu *vcpu,
25 gpa_t addr, unsigned int len)
26{
27 return 0;
28}
29
30unsigned long vgic_mmio_read_rao(struct kvm_vcpu *vcpu,
31 gpa_t addr, unsigned int len)
32{
33 return -1UL;
34}
35
36void vgic_mmio_write_wi(struct kvm_vcpu *vcpu, gpa_t addr,
37 unsigned int len, unsigned long val)
38{
39
40}
41
42
43
44
45
46unsigned long vgic_mmio_read_enable(struct kvm_vcpu *vcpu,
47 gpa_t addr, unsigned int len)
48{
49 u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
50 u32 value = 0;
51 int i;
52
53
54 for (i = 0; i < len * 8; i++) {
55 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
56
57 if (irq->enabled)
58 value |= (1U << i);
59
60 vgic_put_irq(vcpu->kvm, irq);
61 }
62
63 return value;
64}
65
66void vgic_mmio_write_senable(struct kvm_vcpu *vcpu,
67 gpa_t addr, unsigned int len,
68 unsigned long val)
69{
70 u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
71 int i;
72
73 for_each_set_bit(i, &val, len * 8) {
74 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
75
76 spin_lock(&irq->irq_lock);
77 irq->enabled = true;
78 vgic_queue_irq_unlock(vcpu->kvm, irq);
79
80 vgic_put_irq(vcpu->kvm, irq);
81 }
82}
83
84void vgic_mmio_write_cenable(struct kvm_vcpu *vcpu,
85 gpa_t addr, unsigned int len,
86 unsigned long val)
87{
88 u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
89 int i;
90
91 for_each_set_bit(i, &val, len * 8) {
92 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
93
94 spin_lock(&irq->irq_lock);
95
96 irq->enabled = false;
97
98 spin_unlock(&irq->irq_lock);
99 vgic_put_irq(vcpu->kvm, irq);
100 }
101}
102
103unsigned long vgic_mmio_read_pending(struct kvm_vcpu *vcpu,
104 gpa_t addr, unsigned int len)
105{
106 u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
107 u32 value = 0;
108 int i;
109
110
111 for (i = 0; i < len * 8; i++) {
112 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
113
114 if (irq->pending)
115 value |= (1U << i);
116
117 vgic_put_irq(vcpu->kvm, irq);
118 }
119
120 return value;
121}
122
123void vgic_mmio_write_spending(struct kvm_vcpu *vcpu,
124 gpa_t addr, unsigned int len,
125 unsigned long val)
126{
127 u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
128 int i;
129
130 for_each_set_bit(i, &val, len * 8) {
131 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
132
133 spin_lock(&irq->irq_lock);
134 irq->pending = true;
135 if (irq->config == VGIC_CONFIG_LEVEL)
136 irq->soft_pending = true;
137
138 vgic_queue_irq_unlock(vcpu->kvm, irq);
139 vgic_put_irq(vcpu->kvm, irq);
140 }
141}
142
143void vgic_mmio_write_cpending(struct kvm_vcpu *vcpu,
144 gpa_t addr, unsigned int len,
145 unsigned long val)
146{
147 u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
148 int i;
149
150 for_each_set_bit(i, &val, len * 8) {
151 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
152
153 spin_lock(&irq->irq_lock);
154
155 if (irq->config == VGIC_CONFIG_LEVEL) {
156 irq->soft_pending = false;
157 irq->pending = irq->line_level;
158 } else {
159 irq->pending = false;
160 }
161
162 spin_unlock(&irq->irq_lock);
163 vgic_put_irq(vcpu->kvm, irq);
164 }
165}
166
167unsigned long vgic_mmio_read_active(struct kvm_vcpu *vcpu,
168 gpa_t addr, unsigned int len)
169{
170 u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
171 u32 value = 0;
172 int i;
173
174
175 for (i = 0; i < len * 8; i++) {
176 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
177
178 if (irq->active)
179 value |= (1U << i);
180
181 vgic_put_irq(vcpu->kvm, irq);
182 }
183
184 return value;
185}
186
187static void vgic_mmio_change_active(struct kvm_vcpu *vcpu, struct vgic_irq *irq,
188 bool new_active_state)
189{
190 spin_lock(&irq->irq_lock);
191
192
193
194
195
196
197
198
199
200
201
202
203
204 while (irq->vcpu &&
205 irq->vcpu->cpu != -1)
206 cond_resched_lock(&irq->irq_lock);
207
208 irq->active = new_active_state;
209 if (new_active_state)
210 vgic_queue_irq_unlock(vcpu->kvm, irq);
211 else
212 spin_unlock(&irq->irq_lock);
213}
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228static void vgic_change_active_prepare(struct kvm_vcpu *vcpu, u32 intid)
229{
230 if (intid < VGIC_NR_PRIVATE_IRQS)
231 kvm_arm_halt_vcpu(vcpu);
232 else
233 kvm_arm_halt_guest(vcpu->kvm);
234}
235
236
237static void vgic_change_active_finish(struct kvm_vcpu *vcpu, u32 intid)
238{
239 if (intid < VGIC_NR_PRIVATE_IRQS)
240 kvm_arm_resume_vcpu(vcpu);
241 else
242 kvm_arm_resume_guest(vcpu->kvm);
243}
244
245void vgic_mmio_write_cactive(struct kvm_vcpu *vcpu,
246 gpa_t addr, unsigned int len,
247 unsigned long val)
248{
249 u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
250 int i;
251
252 vgic_change_active_prepare(vcpu, intid);
253 for_each_set_bit(i, &val, len * 8) {
254 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
255 vgic_mmio_change_active(vcpu, irq, false);
256 vgic_put_irq(vcpu->kvm, irq);
257 }
258 vgic_change_active_finish(vcpu, intid);
259}
260
261void vgic_mmio_write_sactive(struct kvm_vcpu *vcpu,
262 gpa_t addr, unsigned int len,
263 unsigned long val)
264{
265 u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
266 int i;
267
268 vgic_change_active_prepare(vcpu, intid);
269 for_each_set_bit(i, &val, len * 8) {
270 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
271 vgic_mmio_change_active(vcpu, irq, true);
272 vgic_put_irq(vcpu->kvm, irq);
273 }
274 vgic_change_active_finish(vcpu, intid);
275}
276
277unsigned long vgic_mmio_read_priority(struct kvm_vcpu *vcpu,
278 gpa_t addr, unsigned int len)
279{
280 u32 intid = VGIC_ADDR_TO_INTID(addr, 8);
281 int i;
282 u64 val = 0;
283
284 for (i = 0; i < len; i++) {
285 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
286
287 val |= (u64)irq->priority << (i * 8);
288
289 vgic_put_irq(vcpu->kvm, irq);
290 }
291
292 return val;
293}
294
295
296
297
298
299
300
301
302void vgic_mmio_write_priority(struct kvm_vcpu *vcpu,
303 gpa_t addr, unsigned int len,
304 unsigned long val)
305{
306 u32 intid = VGIC_ADDR_TO_INTID(addr, 8);
307 int i;
308
309 for (i = 0; i < len; i++) {
310 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
311
312 spin_lock(&irq->irq_lock);
313
314 irq->priority = (val >> (i * 8)) & GENMASK(7, 8 - VGIC_PRI_BITS);
315 spin_unlock(&irq->irq_lock);
316
317 vgic_put_irq(vcpu->kvm, irq);
318 }
319}
320
321unsigned long vgic_mmio_read_config(struct kvm_vcpu *vcpu,
322 gpa_t addr, unsigned int len)
323{
324 u32 intid = VGIC_ADDR_TO_INTID(addr, 2);
325 u32 value = 0;
326 int i;
327
328 for (i = 0; i < len * 4; i++) {
329 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
330
331 if (irq->config == VGIC_CONFIG_EDGE)
332 value |= (2U << (i * 2));
333
334 vgic_put_irq(vcpu->kvm, irq);
335 }
336
337 return value;
338}
339
340void vgic_mmio_write_config(struct kvm_vcpu *vcpu,
341 gpa_t addr, unsigned int len,
342 unsigned long val)
343{
344 u32 intid = VGIC_ADDR_TO_INTID(addr, 2);
345 int i;
346
347 for (i = 0; i < len * 4; i++) {
348 struct vgic_irq *irq;
349
350
351
352
353
354
355
356 if (intid + i < VGIC_NR_PRIVATE_IRQS)
357 continue;
358
359 irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
360 spin_lock(&irq->irq_lock);
361
362 if (test_bit(i * 2 + 1, &val)) {
363 irq->config = VGIC_CONFIG_EDGE;
364 } else {
365 irq->config = VGIC_CONFIG_LEVEL;
366 irq->pending = irq->line_level | irq->soft_pending;
367 }
368
369 spin_unlock(&irq->irq_lock);
370 vgic_put_irq(vcpu->kvm, irq);
371 }
372}
373
374static int match_region(const void *key, const void *elt)
375{
376 const unsigned int offset = (unsigned long)key;
377 const struct vgic_register_region *region = elt;
378
379 if (offset < region->reg_offset)
380 return -1;
381
382 if (offset >= region->reg_offset + region->len)
383 return 1;
384
385 return 0;
386}
387
388
389static const struct vgic_register_region *
390vgic_find_mmio_region(const struct vgic_register_region *region, int nr_regions,
391 unsigned int offset)
392{
393 return bsearch((void *)(uintptr_t)offset, region, nr_regions,
394 sizeof(region[0]), match_region);
395}
396
397
398
399
400
401
402
403
404
405
406unsigned long vgic_data_mmio_bus_to_host(const void *val, unsigned int len)
407{
408 unsigned long data = kvm_mmio_read_buf(val, len);
409
410 switch (len) {
411 case 1:
412 return data;
413 case 2:
414 return le16_to_cpu(data);
415 case 4:
416 return le32_to_cpu(data);
417 default:
418 return le64_to_cpu(data);
419 }
420}
421
422
423
424
425
426
427
428
429
430
431void vgic_data_host_to_mmio_bus(void *buf, unsigned int len,
432 unsigned long data)
433{
434 switch (len) {
435 case 1:
436 break;
437 case 2:
438 data = cpu_to_le16(data);
439 break;
440 case 4:
441 data = cpu_to_le32(data);
442 break;
443 default:
444 data = cpu_to_le64(data);
445 }
446
447 kvm_mmio_write_buf(buf, len, data);
448}
449
450static
451struct vgic_io_device *kvm_to_vgic_iodev(const struct kvm_io_device *dev)
452{
453 return container_of(dev, struct vgic_io_device, dev);
454}
455
456static bool check_region(const struct kvm *kvm,
457 const struct vgic_register_region *region,
458 gpa_t addr, int len)
459{
460 int flags, nr_irqs = kvm->arch.vgic.nr_spis + VGIC_NR_PRIVATE_IRQS;
461
462 switch (len) {
463 case sizeof(u8):
464 flags = VGIC_ACCESS_8bit;
465 break;
466 case sizeof(u32):
467 flags = VGIC_ACCESS_32bit;
468 break;
469 case sizeof(u64):
470 flags = VGIC_ACCESS_64bit;
471 break;
472 default:
473 return false;
474 }
475
476 if ((region->access_flags & flags) && IS_ALIGNED(addr, len)) {
477 if (!region->bits_per_irq)
478 return true;
479
480
481 return VGIC_ADDR_TO_INTID(addr, region->bits_per_irq) < nr_irqs;
482 }
483
484 return false;
485}
486
487static int dispatch_mmio_read(struct kvm_vcpu *vcpu, struct kvm_io_device *dev,
488 gpa_t addr, int len, void *val)
489{
490 struct vgic_io_device *iodev = kvm_to_vgic_iodev(dev);
491 const struct vgic_register_region *region;
492 unsigned long data = 0;
493
494 region = vgic_find_mmio_region(iodev->regions, iodev->nr_regions,
495 addr - iodev->base_addr);
496 if (!region || !check_region(vcpu->kvm, region, addr, len)) {
497 memset(val, 0, len);
498 return 0;
499 }
500
501 switch (iodev->iodev_type) {
502 case IODEV_CPUIF:
503 data = region->read(vcpu, addr, len);
504 break;
505 case IODEV_DIST:
506 data = region->read(vcpu, addr, len);
507 break;
508 case IODEV_REDIST:
509 data = region->read(iodev->redist_vcpu, addr, len);
510 break;
511 case IODEV_ITS:
512 data = region->its_read(vcpu->kvm, iodev->its, addr, len);
513 break;
514 }
515
516 vgic_data_host_to_mmio_bus(val, len, data);
517 return 0;
518}
519
520static int dispatch_mmio_write(struct kvm_vcpu *vcpu, struct kvm_io_device *dev,
521 gpa_t addr, int len, const void *val)
522{
523 struct vgic_io_device *iodev = kvm_to_vgic_iodev(dev);
524 const struct vgic_register_region *region;
525 unsigned long data = vgic_data_mmio_bus_to_host(val, len);
526
527 region = vgic_find_mmio_region(iodev->regions, iodev->nr_regions,
528 addr - iodev->base_addr);
529 if (!region || !check_region(vcpu->kvm, region, addr, len))
530 return 0;
531
532 switch (iodev->iodev_type) {
533 case IODEV_CPUIF:
534 region->write(vcpu, addr, len, data);
535 break;
536 case IODEV_DIST:
537 region->write(vcpu, addr, len, data);
538 break;
539 case IODEV_REDIST:
540 region->write(iodev->redist_vcpu, addr, len, data);
541 break;
542 case IODEV_ITS:
543 region->its_write(vcpu->kvm, iodev->its, addr, len, data);
544 break;
545 }
546
547 return 0;
548}
549
550struct kvm_io_device_ops kvm_io_gic_ops = {
551 .read = dispatch_mmio_read,
552 .write = dispatch_mmio_write,
553};
554
555int vgic_register_dist_iodev(struct kvm *kvm, gpa_t dist_base_address,
556 enum vgic_type type)
557{
558 struct vgic_io_device *io_device = &kvm->arch.vgic.dist_iodev;
559 int ret = 0;
560 unsigned int len;
561
562 switch (type) {
563 case VGIC_V2:
564 len = vgic_v2_init_dist_iodev(io_device);
565 break;
566 case VGIC_V3:
567 len = vgic_v3_init_dist_iodev(io_device);
568 break;
569 default:
570 BUG_ON(1);
571 }
572
573 io_device->base_addr = dist_base_address;
574 io_device->iodev_type = IODEV_DIST;
575 io_device->redist_vcpu = NULL;
576
577 mutex_lock(&kvm->slots_lock);
578 ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, dist_base_address,
579 len, &io_device->dev);
580 mutex_unlock(&kvm->slots_lock);
581
582 return ret;
583}
584