1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20#include <linux/cpu.h>
21#include <linux/kvm.h>
22#include <linux/kvm_host.h>
23#include <linux/interrupt.h>
24#include <linux/list.h>
25#include <linux/uaccess.h>
26#include <linux/list_sort.h>
27
28#include <linux/irqchip/arm-gic-v3.h>
29
30#include <asm/kvm_emulate.h>
31#include <asm/kvm_arm.h>
32#include <asm/kvm_mmu.h>
33
34#include "vgic.h"
35#include "vgic-mmio.h"
36
37static int vgic_its_save_tables_v0(struct vgic_its *its);
38static int vgic_its_restore_tables_v0(struct vgic_its *its);
39static int vgic_its_commit_v0(struct vgic_its *its);
40static int update_lpi_config(struct kvm *kvm, struct vgic_irq *irq,
41 struct kvm_vcpu *filter_vcpu, bool needs_inv);
42
43
44
45
46
47
48
49
50static struct vgic_irq *vgic_add_lpi(struct kvm *kvm, u32 intid,
51 struct kvm_vcpu *vcpu)
52{
53 struct vgic_dist *dist = &kvm->arch.vgic;
54 struct vgic_irq *irq = vgic_get_irq(kvm, NULL, intid), *oldirq;
55 int ret;
56
57
58 if (irq)
59 return irq;
60
61 irq = kzalloc(sizeof(struct vgic_irq), GFP_KERNEL);
62 if (!irq)
63 return ERR_PTR(-ENOMEM);
64
65 INIT_LIST_HEAD(&irq->lpi_list);
66 INIT_LIST_HEAD(&irq->ap_list);
67 spin_lock_init(&irq->irq_lock);
68
69 irq->config = VGIC_CONFIG_EDGE;
70 kref_init(&irq->refcount);
71 irq->intid = intid;
72 irq->target_vcpu = vcpu;
73
74 spin_lock(&dist->lpi_list_lock);
75
76
77
78
79
80 list_for_each_entry(oldirq, &dist->lpi_list_head, lpi_list) {
81 if (oldirq->intid != intid)
82 continue;
83
84
85 kfree(irq);
86 irq = oldirq;
87
88
89
90
91
92
93 vgic_get_irq_kref(irq);
94
95 goto out_unlock;
96 }
97
98 list_add_tail(&irq->lpi_list, &dist->lpi_list_head);
99 dist->lpi_list_count++;
100
101out_unlock:
102 spin_unlock(&dist->lpi_list_lock);
103
104
105
106
107
108
109 ret = update_lpi_config(kvm, irq, NULL, false);
110 if (ret)
111 return ERR_PTR(ret);
112
113 ret = vgic_v3_lpi_sync_pending_status(kvm, irq);
114 if (ret)
115 return ERR_PTR(ret);
116
117 return irq;
118}
119
120struct its_device {
121 struct list_head dev_list;
122
123
124 struct list_head itt_head;
125 u32 num_eventid_bits;
126 gpa_t itt_addr;
127 u32 device_id;
128};
129
130#define COLLECTION_NOT_MAPPED ((u32)~0)
131
132struct its_collection {
133 struct list_head coll_list;
134
135 u32 collection_id;
136 u32 target_addr;
137};
138
139#define its_is_collection_mapped(coll) ((coll) && \
140 ((coll)->target_addr != COLLECTION_NOT_MAPPED))
141
142struct its_ite {
143 struct list_head ite_list;
144
145 struct vgic_irq *irq;
146 struct its_collection *collection;
147 u32 event_id;
148};
149
150
151
152
153
154
155
156
157
158
159
160
161struct vgic_its_abi {
162 int cte_esz;
163 int dte_esz;
164 int ite_esz;
165 int (*save_tables)(struct vgic_its *its);
166 int (*restore_tables)(struct vgic_its *its);
167 int (*commit)(struct vgic_its *its);
168};
169
170static const struct vgic_its_abi its_table_abi_versions[] = {
171 [0] = {.cte_esz = 8, .dte_esz = 8, .ite_esz = 8,
172 .save_tables = vgic_its_save_tables_v0,
173 .restore_tables = vgic_its_restore_tables_v0,
174 .commit = vgic_its_commit_v0,
175 },
176};
177
178#define NR_ITS_ABIS ARRAY_SIZE(its_table_abi_versions)
179
180inline const struct vgic_its_abi *vgic_its_get_abi(struct vgic_its *its)
181{
182 return &its_table_abi_versions[its->abi_rev];
183}
184
185int vgic_its_set_abi(struct vgic_its *its, int rev)
186{
187 const struct vgic_its_abi *abi;
188
189 its->abi_rev = rev;
190 abi = vgic_its_get_abi(its);
191 return abi->commit(its);
192}
193
194
195
196
197
198static struct its_device *find_its_device(struct vgic_its *its, u32 device_id)
199{
200 struct its_device *device;
201
202 list_for_each_entry(device, &its->device_list, dev_list)
203 if (device_id == device->device_id)
204 return device;
205
206 return NULL;
207}
208
209
210
211
212
213
214static struct its_ite *find_ite(struct vgic_its *its, u32 device_id,
215 u32 event_id)
216{
217 struct its_device *device;
218 struct its_ite *ite;
219
220 device = find_its_device(its, device_id);
221 if (device == NULL)
222 return NULL;
223
224 list_for_each_entry(ite, &device->itt_head, ite_list)
225 if (ite->event_id == event_id)
226 return ite;
227
228 return NULL;
229}
230
231
232#define for_each_lpi_its(dev, ite, its) \
233 list_for_each_entry(dev, &(its)->device_list, dev_list) \
234 list_for_each_entry(ite, &(dev)->itt_head, ite_list)
235
236
237
238
239
240#define BASER_ADDRESS(x) ((x) & GENMASK_ULL(47, 16))
241#define CBASER_ADDRESS(x) ((x) & GENMASK_ULL(47, 12))
242
243#define GIC_LPI_OFFSET 8192
244
245#define VITS_TYPER_IDBITS 16
246#define VITS_TYPER_DEVBITS 16
247#define VITS_DTE_MAX_DEVID_OFFSET (BIT(14) - 1)
248#define VITS_ITE_MAX_EVENTID_OFFSET (BIT(16) - 1)
249
250
251
252
253
254static struct its_collection *find_collection(struct vgic_its *its, int coll_id)
255{
256 struct its_collection *collection;
257
258 list_for_each_entry(collection, &its->collection_list, coll_list) {
259 if (coll_id == collection->collection_id)
260 return collection;
261 }
262
263 return NULL;
264}
265
266#define LPI_PROP_ENABLE_BIT(p) ((p) & LPI_PROP_ENABLED)
267#define LPI_PROP_PRIORITY(p) ((p) & 0xfc)
268
269
270
271
272
273
274
275static int update_lpi_config(struct kvm *kvm, struct vgic_irq *irq,
276 struct kvm_vcpu *filter_vcpu, bool needs_inv)
277{
278 u64 propbase = GICR_PROPBASER_ADDRESS(kvm->arch.vgic.propbaser);
279 u8 prop;
280 int ret;
281 unsigned long flags;
282
283 ret = kvm_read_guest(kvm, propbase + irq->intid - GIC_LPI_OFFSET,
284 &prop, 1);
285
286 if (ret)
287 return ret;
288
289 spin_lock_irqsave(&irq->irq_lock, flags);
290
291 if (!filter_vcpu || filter_vcpu == irq->target_vcpu) {
292 irq->priority = LPI_PROP_PRIORITY(prop);
293 irq->enabled = LPI_PROP_ENABLE_BIT(prop);
294
295 if (!irq->hw) {
296 vgic_queue_irq_unlock(kvm, irq, flags);
297 return 0;
298 }
299 }
300
301 spin_unlock_irqrestore(&irq->irq_lock, flags);
302
303 if (irq->hw)
304 return its_prop_update_vlpi(irq->host_irq, prop, needs_inv);
305
306 return 0;
307}
308
309
310
311
312
313
314static int vgic_copy_lpi_list(struct kvm_vcpu *vcpu, u32 **intid_ptr)
315{
316 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
317 struct vgic_irq *irq;
318 u32 *intids;
319 int irq_count = dist->lpi_list_count, i = 0;
320
321
322
323
324
325
326
327
328 intids = kmalloc_array(irq_count, sizeof(intids[0]), GFP_KERNEL);
329 if (!intids)
330 return -ENOMEM;
331
332 spin_lock(&dist->lpi_list_lock);
333 list_for_each_entry(irq, &dist->lpi_list_head, lpi_list) {
334
335 if (irq->target_vcpu != vcpu)
336 continue;
337 intids[i++] = irq->intid;
338 }
339 spin_unlock(&dist->lpi_list_lock);
340
341 *intid_ptr = intids;
342 return i;
343}
344
345static int update_affinity(struct vgic_irq *irq, struct kvm_vcpu *vcpu)
346{
347 int ret = 0;
348
349 spin_lock(&irq->irq_lock);
350 irq->target_vcpu = vcpu;
351 spin_unlock(&irq->irq_lock);
352
353 if (irq->hw) {
354 struct its_vlpi_map map;
355
356 ret = its_get_vlpi(irq->host_irq, &map);
357 if (ret)
358 return ret;
359
360 map.vpe = &vcpu->arch.vgic_cpu.vgic_v3.its_vpe;
361
362 ret = its_map_vlpi(irq->host_irq, &map);
363 }
364
365 return ret;
366}
367
368
369
370
371
372
373
374static void update_affinity_ite(struct kvm *kvm, struct its_ite *ite)
375{
376 struct kvm_vcpu *vcpu;
377
378 if (!its_is_collection_mapped(ite->collection))
379 return;
380
381 vcpu = kvm_get_vcpu(kvm, ite->collection->target_addr);
382 update_affinity(ite->irq, vcpu);
383}
384
385
386
387
388
389static void update_affinity_collection(struct kvm *kvm, struct vgic_its *its,
390 struct its_collection *coll)
391{
392 struct its_device *device;
393 struct its_ite *ite;
394
395 for_each_lpi_its(device, ite, its) {
396 if (!ite->collection || coll != ite->collection)
397 continue;
398
399 update_affinity_ite(kvm, ite);
400 }
401}
402
403static u32 max_lpis_propbaser(u64 propbaser)
404{
405 int nr_idbits = (propbaser & 0x1f) + 1;
406
407 return 1U << min(nr_idbits, INTERRUPT_ID_BITS_ITS);
408}
409
410
411
412
413
414
415static int its_sync_lpi_pending_table(struct kvm_vcpu *vcpu)
416{
417 gpa_t pendbase = GICR_PENDBASER_ADDRESS(vcpu->arch.vgic_cpu.pendbaser);
418 struct vgic_irq *irq;
419 int last_byte_offset = -1;
420 int ret = 0;
421 u32 *intids;
422 int nr_irqs, i;
423 unsigned long flags;
424 u8 pendmask;
425
426 nr_irqs = vgic_copy_lpi_list(vcpu, &intids);
427 if (nr_irqs < 0)
428 return nr_irqs;
429
430 for (i = 0; i < nr_irqs; i++) {
431 int byte_offset, bit_nr;
432
433 byte_offset = intids[i] / BITS_PER_BYTE;
434 bit_nr = intids[i] % BITS_PER_BYTE;
435
436
437
438
439
440 if (byte_offset != last_byte_offset) {
441 ret = kvm_read_guest(vcpu->kvm, pendbase + byte_offset,
442 &pendmask, 1);
443 if (ret) {
444 kfree(intids);
445 return ret;
446 }
447 last_byte_offset = byte_offset;
448 }
449
450 irq = vgic_get_irq(vcpu->kvm, NULL, intids[i]);
451 spin_lock_irqsave(&irq->irq_lock, flags);
452 irq->pending_latch = pendmask & (1U << bit_nr);
453 vgic_queue_irq_unlock(vcpu->kvm, irq, flags);
454 vgic_put_irq(vcpu->kvm, irq);
455 }
456
457 kfree(intids);
458
459 return ret;
460}
461
462static unsigned long vgic_mmio_read_its_typer(struct kvm *kvm,
463 struct vgic_its *its,
464 gpa_t addr, unsigned int len)
465{
466 const struct vgic_its_abi *abi = vgic_its_get_abi(its);
467 u64 reg = GITS_TYPER_PLPIS;
468
469
470
471
472
473
474
475
476
477 reg |= GIC_ENCODE_SZ(VITS_TYPER_DEVBITS, 5) << GITS_TYPER_DEVBITS_SHIFT;
478 reg |= GIC_ENCODE_SZ(VITS_TYPER_IDBITS, 5) << GITS_TYPER_IDBITS_SHIFT;
479 reg |= GIC_ENCODE_SZ(abi->ite_esz, 4) << GITS_TYPER_ITT_ENTRY_SIZE_SHIFT;
480
481 return extract_bytes(reg, addr & 7, len);
482}
483
484static unsigned long vgic_mmio_read_its_iidr(struct kvm *kvm,
485 struct vgic_its *its,
486 gpa_t addr, unsigned int len)
487{
488 u32 val;
489
490 val = (its->abi_rev << GITS_IIDR_REV_SHIFT) & GITS_IIDR_REV_MASK;
491 val |= (PRODUCT_ID_KVM << GITS_IIDR_PRODUCTID_SHIFT) | IMPLEMENTER_ARM;
492 return val;
493}
494
495static int vgic_mmio_uaccess_write_its_iidr(struct kvm *kvm,
496 struct vgic_its *its,
497 gpa_t addr, unsigned int len,
498 unsigned long val)
499{
500 u32 rev = GITS_IIDR_REV(val);
501
502 if (rev >= NR_ITS_ABIS)
503 return -EINVAL;
504 return vgic_its_set_abi(its, rev);
505}
506
507static unsigned long vgic_mmio_read_its_idregs(struct kvm *kvm,
508 struct vgic_its *its,
509 gpa_t addr, unsigned int len)
510{
511 switch (addr & 0xffff) {
512 case GITS_PIDR0:
513 return 0x92;
514 case GITS_PIDR1:
515 return 0xb4;
516 case GITS_PIDR2:
517 return GIC_PIDR2_ARCH_GICv3 | 0x0b;
518 case GITS_PIDR4:
519 return 0x40;
520
521 case GITS_CIDR0:
522 return 0x0d;
523 case GITS_CIDR1:
524 return 0xf0;
525 case GITS_CIDR2:
526 return 0x05;
527 case GITS_CIDR3:
528 return 0xb1;
529 }
530
531 return 0;
532}
533
534int vgic_its_resolve_lpi(struct kvm *kvm, struct vgic_its *its,
535 u32 devid, u32 eventid, struct vgic_irq **irq)
536{
537 struct kvm_vcpu *vcpu;
538 struct its_ite *ite;
539
540 if (!its->enabled)
541 return -EBUSY;
542
543 ite = find_ite(its, devid, eventid);
544 if (!ite || !its_is_collection_mapped(ite->collection))
545 return E_ITS_INT_UNMAPPED_INTERRUPT;
546
547 vcpu = kvm_get_vcpu(kvm, ite->collection->target_addr);
548 if (!vcpu)
549 return E_ITS_INT_UNMAPPED_INTERRUPT;
550
551 if (!vcpu->arch.vgic_cpu.lpis_enabled)
552 return -EBUSY;
553
554 *irq = ite->irq;
555 return 0;
556}
557
558struct vgic_its *vgic_msi_to_its(struct kvm *kvm, struct kvm_msi *msi)
559{
560 u64 address;
561 struct kvm_io_device *kvm_io_dev;
562 struct vgic_io_device *iodev;
563
564 if (!vgic_has_its(kvm))
565 return ERR_PTR(-ENODEV);
566
567 if (!(msi->flags & KVM_MSI_VALID_DEVID))
568 return ERR_PTR(-EINVAL);
569
570 address = (u64)msi->address_hi << 32 | msi->address_lo;
571
572 kvm_io_dev = kvm_io_bus_get_dev(kvm, KVM_MMIO_BUS, address);
573 if (!kvm_io_dev)
574 return ERR_PTR(-EINVAL);
575
576 if (kvm_io_dev->ops != &kvm_io_gic_ops)
577 return ERR_PTR(-EINVAL);
578
579 iodev = container_of(kvm_io_dev, struct vgic_io_device, dev);
580 if (iodev->iodev_type != IODEV_ITS)
581 return ERR_PTR(-EINVAL);
582
583 return iodev->its;
584}
585
586
587
588
589
590
591
592
593static int vgic_its_trigger_msi(struct kvm *kvm, struct vgic_its *its,
594 u32 devid, u32 eventid)
595{
596 struct vgic_irq *irq = NULL;
597 unsigned long flags;
598 int err;
599
600 err = vgic_its_resolve_lpi(kvm, its, devid, eventid, &irq);
601 if (err)
602 return err;
603
604 if (irq->hw)
605 return irq_set_irqchip_state(irq->host_irq,
606 IRQCHIP_STATE_PENDING, true);
607
608 spin_lock_irqsave(&irq->irq_lock, flags);
609 irq->pending_latch = true;
610 vgic_queue_irq_unlock(kvm, irq, flags);
611
612 return 0;
613}
614
615
616
617
618
619
620
621int vgic_its_inject_msi(struct kvm *kvm, struct kvm_msi *msi)
622{
623 struct vgic_its *its;
624 int ret;
625
626 its = vgic_msi_to_its(kvm, msi);
627 if (IS_ERR(its))
628 return PTR_ERR(its);
629
630 mutex_lock(&its->its_lock);
631 ret = vgic_its_trigger_msi(kvm, its, msi->devid, msi->data);
632 mutex_unlock(&its->its_lock);
633
634 if (ret < 0)
635 return ret;
636
637
638
639
640
641
642 if (ret)
643 return 0;
644 else
645 return 1;
646}
647
648
649static void its_free_ite(struct kvm *kvm, struct its_ite *ite)
650{
651 list_del(&ite->ite_list);
652
653
654 if (ite->irq) {
655 if (ite->irq->hw)
656 WARN_ON(its_unmap_vlpi(ite->irq->host_irq));
657
658 vgic_put_irq(kvm, ite->irq);
659 }
660
661 kfree(ite);
662}
663
664static u64 its_cmd_mask_field(u64 *its_cmd, int word, int shift, int size)
665{
666 return (le64_to_cpu(its_cmd[word]) >> shift) & (BIT_ULL(size) - 1);
667}
668
669#define its_cmd_get_command(cmd) its_cmd_mask_field(cmd, 0, 0, 8)
670#define its_cmd_get_deviceid(cmd) its_cmd_mask_field(cmd, 0, 32, 32)
671#define its_cmd_get_size(cmd) (its_cmd_mask_field(cmd, 1, 0, 5) + 1)
672#define its_cmd_get_id(cmd) its_cmd_mask_field(cmd, 1, 0, 32)
673#define its_cmd_get_physical_id(cmd) its_cmd_mask_field(cmd, 1, 32, 32)
674#define its_cmd_get_collection(cmd) its_cmd_mask_field(cmd, 2, 0, 16)
675#define its_cmd_get_ittaddr(cmd) (its_cmd_mask_field(cmd, 2, 8, 44) << 8)
676#define its_cmd_get_target_addr(cmd) its_cmd_mask_field(cmd, 2, 16, 32)
677#define its_cmd_get_validbit(cmd) its_cmd_mask_field(cmd, 2, 63, 1)
678
679
680
681
682
683static int vgic_its_cmd_handle_discard(struct kvm *kvm, struct vgic_its *its,
684 u64 *its_cmd)
685{
686 u32 device_id = its_cmd_get_deviceid(its_cmd);
687 u32 event_id = its_cmd_get_id(its_cmd);
688 struct its_ite *ite;
689
690
691 ite = find_ite(its, device_id, event_id);
692 if (ite && ite->collection) {
693
694
695
696
697
698 its_free_ite(kvm, ite);
699 return 0;
700 }
701
702 return E_ITS_DISCARD_UNMAPPED_INTERRUPT;
703}
704
705
706
707
708
709static int vgic_its_cmd_handle_movi(struct kvm *kvm, struct vgic_its *its,
710 u64 *its_cmd)
711{
712 u32 device_id = its_cmd_get_deviceid(its_cmd);
713 u32 event_id = its_cmd_get_id(its_cmd);
714 u32 coll_id = its_cmd_get_collection(its_cmd);
715 struct kvm_vcpu *vcpu;
716 struct its_ite *ite;
717 struct its_collection *collection;
718
719 ite = find_ite(its, device_id, event_id);
720 if (!ite)
721 return E_ITS_MOVI_UNMAPPED_INTERRUPT;
722
723 if (!its_is_collection_mapped(ite->collection))
724 return E_ITS_MOVI_UNMAPPED_COLLECTION;
725
726 collection = find_collection(its, coll_id);
727 if (!its_is_collection_mapped(collection))
728 return E_ITS_MOVI_UNMAPPED_COLLECTION;
729
730 ite->collection = collection;
731 vcpu = kvm_get_vcpu(kvm, collection->target_addr);
732
733 return update_affinity(ite->irq, vcpu);
734}
735
736
737
738
739
740
741
742
743static bool vgic_its_check_id(struct vgic_its *its, u64 baser, u32 id,
744 gpa_t *eaddr)
745{
746 int l1_tbl_size = GITS_BASER_NR_PAGES(baser) * SZ_64K;
747 u64 indirect_ptr, type = GITS_BASER_TYPE(baser);
748 int esz = GITS_BASER_ENTRY_SIZE(baser);
749 int index;
750 gfn_t gfn;
751
752 switch (type) {
753 case GITS_BASER_TYPE_DEVICE:
754 if (id >= BIT_ULL(VITS_TYPER_DEVBITS))
755 return false;
756 break;
757 case GITS_BASER_TYPE_COLLECTION:
758
759 if (id >= BIT_ULL(16))
760 return false;
761 break;
762 default:
763 return false;
764 }
765
766 if (!(baser & GITS_BASER_INDIRECT)) {
767 phys_addr_t addr;
768
769 if (id >= (l1_tbl_size / esz))
770 return false;
771
772 addr = BASER_ADDRESS(baser) + id * esz;
773 gfn = addr >> PAGE_SHIFT;
774
775 if (eaddr)
776 *eaddr = addr;
777 return kvm_is_visible_gfn(its->dev->kvm, gfn);
778 }
779
780
781 index = id / (SZ_64K / esz);
782 if (index >= (l1_tbl_size / sizeof(u64)))
783 return false;
784
785
786 if (kvm_read_guest(its->dev->kvm,
787 BASER_ADDRESS(baser) + index * sizeof(indirect_ptr),
788 &indirect_ptr, sizeof(indirect_ptr)))
789 return false;
790
791 indirect_ptr = le64_to_cpu(indirect_ptr);
792
793
794 if (!(indirect_ptr & BIT_ULL(63)))
795 return false;
796
797
798
799
800
801
802 indirect_ptr &= GENMASK_ULL(51, 16);
803
804
805 index = id % (SZ_64K / esz);
806 indirect_ptr += index * esz;
807 gfn = indirect_ptr >> PAGE_SHIFT;
808
809 if (eaddr)
810 *eaddr = indirect_ptr;
811 return kvm_is_visible_gfn(its->dev->kvm, gfn);
812}
813
814static int vgic_its_alloc_collection(struct vgic_its *its,
815 struct its_collection **colp,
816 u32 coll_id)
817{
818 struct its_collection *collection;
819
820 if (!vgic_its_check_id(its, its->baser_coll_table, coll_id, NULL))
821 return E_ITS_MAPC_COLLECTION_OOR;
822
823 collection = kzalloc(sizeof(*collection), GFP_KERNEL);
824 if (!collection)
825 return -ENOMEM;
826
827 collection->collection_id = coll_id;
828 collection->target_addr = COLLECTION_NOT_MAPPED;
829
830 list_add_tail(&collection->coll_list, &its->collection_list);
831 *colp = collection;
832
833 return 0;
834}
835
836static void vgic_its_free_collection(struct vgic_its *its, u32 coll_id)
837{
838 struct its_collection *collection;
839 struct its_device *device;
840 struct its_ite *ite;
841
842
843
844
845
846
847 collection = find_collection(its, coll_id);
848 if (!collection)
849 return;
850
851 for_each_lpi_its(device, ite, its)
852 if (ite->collection &&
853 ite->collection->collection_id == coll_id)
854 ite->collection = NULL;
855
856 list_del(&collection->coll_list);
857 kfree(collection);
858}
859
860
861static struct its_ite *vgic_its_alloc_ite(struct its_device *device,
862 struct its_collection *collection,
863 u32 event_id)
864{
865 struct its_ite *ite;
866
867 ite = kzalloc(sizeof(*ite), GFP_KERNEL);
868 if (!ite)
869 return ERR_PTR(-ENOMEM);
870
871 ite->event_id = event_id;
872 ite->collection = collection;
873
874 list_add_tail(&ite->ite_list, &device->itt_head);
875 return ite;
876}
877
878
879
880
881
882static int vgic_its_cmd_handle_mapi(struct kvm *kvm, struct vgic_its *its,
883 u64 *its_cmd)
884{
885 u32 device_id = its_cmd_get_deviceid(its_cmd);
886 u32 event_id = its_cmd_get_id(its_cmd);
887 u32 coll_id = its_cmd_get_collection(its_cmd);
888 struct its_ite *ite;
889 struct kvm_vcpu *vcpu = NULL;
890 struct its_device *device;
891 struct its_collection *collection, *new_coll = NULL;
892 struct vgic_irq *irq;
893 int lpi_nr;
894
895 device = find_its_device(its, device_id);
896 if (!device)
897 return E_ITS_MAPTI_UNMAPPED_DEVICE;
898
899 if (event_id >= BIT_ULL(device->num_eventid_bits))
900 return E_ITS_MAPTI_ID_OOR;
901
902 if (its_cmd_get_command(its_cmd) == GITS_CMD_MAPTI)
903 lpi_nr = its_cmd_get_physical_id(its_cmd);
904 else
905 lpi_nr = event_id;
906 if (lpi_nr < GIC_LPI_OFFSET ||
907 lpi_nr >= max_lpis_propbaser(kvm->arch.vgic.propbaser))
908 return E_ITS_MAPTI_PHYSICALID_OOR;
909
910
911 if (find_ite(its, device_id, event_id))
912 return 0;
913
914 collection = find_collection(its, coll_id);
915 if (!collection) {
916 int ret = vgic_its_alloc_collection(its, &collection, coll_id);
917 if (ret)
918 return ret;
919 new_coll = collection;
920 }
921
922 ite = vgic_its_alloc_ite(device, collection, event_id);
923 if (IS_ERR(ite)) {
924 if (new_coll)
925 vgic_its_free_collection(its, coll_id);
926 return PTR_ERR(ite);
927 }
928
929 if (its_is_collection_mapped(collection))
930 vcpu = kvm_get_vcpu(kvm, collection->target_addr);
931
932 irq = vgic_add_lpi(kvm, lpi_nr, vcpu);
933 if (IS_ERR(irq)) {
934 if (new_coll)
935 vgic_its_free_collection(its, coll_id);
936 its_free_ite(kvm, ite);
937 return PTR_ERR(irq);
938 }
939 ite->irq = irq;
940
941 return 0;
942}
943
944
945static void vgic_its_free_device(struct kvm *kvm, struct its_device *device)
946{
947 struct its_ite *ite, *temp;
948
949
950
951
952
953
954 list_for_each_entry_safe(ite, temp, &device->itt_head, ite_list)
955 its_free_ite(kvm, ite);
956
957 list_del(&device->dev_list);
958 kfree(device);
959}
960
961
962static void vgic_its_free_device_list(struct kvm *kvm, struct vgic_its *its)
963{
964 struct its_device *cur, *temp;
965
966 list_for_each_entry_safe(cur, temp, &its->device_list, dev_list)
967 vgic_its_free_device(kvm, cur);
968}
969
970
971static void vgic_its_free_collection_list(struct kvm *kvm, struct vgic_its *its)
972{
973 struct its_collection *cur, *temp;
974
975 list_for_each_entry_safe(cur, temp, &its->collection_list, coll_list)
976 vgic_its_free_collection(its, cur->collection_id);
977}
978
979
980static struct its_device *vgic_its_alloc_device(struct vgic_its *its,
981 u32 device_id, gpa_t itt_addr,
982 u8 num_eventid_bits)
983{
984 struct its_device *device;
985
986 device = kzalloc(sizeof(*device), GFP_KERNEL);
987 if (!device)
988 return ERR_PTR(-ENOMEM);
989
990 device->device_id = device_id;
991 device->itt_addr = itt_addr;
992 device->num_eventid_bits = num_eventid_bits;
993 INIT_LIST_HEAD(&device->itt_head);
994
995 list_add_tail(&device->dev_list, &its->device_list);
996 return device;
997}
998
999
1000
1001
1002
1003static int vgic_its_cmd_handle_mapd(struct kvm *kvm, struct vgic_its *its,
1004 u64 *its_cmd)
1005{
1006 u32 device_id = its_cmd_get_deviceid(its_cmd);
1007 bool valid = its_cmd_get_validbit(its_cmd);
1008 u8 num_eventid_bits = its_cmd_get_size(its_cmd);
1009 gpa_t itt_addr = its_cmd_get_ittaddr(its_cmd);
1010 struct its_device *device;
1011
1012 if (!vgic_its_check_id(its, its->baser_device_table, device_id, NULL))
1013 return E_ITS_MAPD_DEVICE_OOR;
1014
1015 if (valid && num_eventid_bits > VITS_TYPER_IDBITS)
1016 return E_ITS_MAPD_ITTSIZE_OOR;
1017
1018 device = find_its_device(its, device_id);
1019
1020
1021
1022
1023
1024
1025 if (device)
1026 vgic_its_free_device(kvm, device);
1027
1028
1029
1030
1031
1032 if (!valid)
1033 return 0;
1034
1035 device = vgic_its_alloc_device(its, device_id, itt_addr,
1036 num_eventid_bits);
1037
1038 return PTR_ERR_OR_ZERO(device);
1039}
1040
1041
1042
1043
1044
1045static int vgic_its_cmd_handle_mapc(struct kvm *kvm, struct vgic_its *its,
1046 u64 *its_cmd)
1047{
1048 u16 coll_id;
1049 u32 target_addr;
1050 struct its_collection *collection;
1051 bool valid;
1052
1053 valid = its_cmd_get_validbit(its_cmd);
1054 coll_id = its_cmd_get_collection(its_cmd);
1055 target_addr = its_cmd_get_target_addr(its_cmd);
1056
1057 if (target_addr >= atomic_read(&kvm->online_vcpus))
1058 return E_ITS_MAPC_PROCNUM_OOR;
1059
1060 if (!valid) {
1061 vgic_its_free_collection(its, coll_id);
1062 } else {
1063 collection = find_collection(its, coll_id);
1064
1065 if (!collection) {
1066 int ret;
1067
1068 ret = vgic_its_alloc_collection(its, &collection,
1069 coll_id);
1070 if (ret)
1071 return ret;
1072 collection->target_addr = target_addr;
1073 } else {
1074 collection->target_addr = target_addr;
1075 update_affinity_collection(kvm, its, collection);
1076 }
1077 }
1078
1079 return 0;
1080}
1081
1082
1083
1084
1085
1086static int vgic_its_cmd_handle_clear(struct kvm *kvm, struct vgic_its *its,
1087 u64 *its_cmd)
1088{
1089 u32 device_id = its_cmd_get_deviceid(its_cmd);
1090 u32 event_id = its_cmd_get_id(its_cmd);
1091 struct its_ite *ite;
1092
1093
1094 ite = find_ite(its, device_id, event_id);
1095 if (!ite)
1096 return E_ITS_CLEAR_UNMAPPED_INTERRUPT;
1097
1098 ite->irq->pending_latch = false;
1099
1100 if (ite->irq->hw)
1101 return irq_set_irqchip_state(ite->irq->host_irq,
1102 IRQCHIP_STATE_PENDING, false);
1103
1104 return 0;
1105}
1106
1107
1108
1109
1110
1111static int vgic_its_cmd_handle_inv(struct kvm *kvm, struct vgic_its *its,
1112 u64 *its_cmd)
1113{
1114 u32 device_id = its_cmd_get_deviceid(its_cmd);
1115 u32 event_id = its_cmd_get_id(its_cmd);
1116 struct its_ite *ite;
1117
1118
1119 ite = find_ite(its, device_id, event_id);
1120 if (!ite)
1121 return E_ITS_INV_UNMAPPED_INTERRUPT;
1122
1123 return update_lpi_config(kvm, ite->irq, NULL, true);
1124}
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134static int vgic_its_cmd_handle_invall(struct kvm *kvm, struct vgic_its *its,
1135 u64 *its_cmd)
1136{
1137 u32 coll_id = its_cmd_get_collection(its_cmd);
1138 struct its_collection *collection;
1139 struct kvm_vcpu *vcpu;
1140 struct vgic_irq *irq;
1141 u32 *intids;
1142 int irq_count, i;
1143
1144 collection = find_collection(its, coll_id);
1145 if (!its_is_collection_mapped(collection))
1146 return E_ITS_INVALL_UNMAPPED_COLLECTION;
1147
1148 vcpu = kvm_get_vcpu(kvm, collection->target_addr);
1149
1150 irq_count = vgic_copy_lpi_list(vcpu, &intids);
1151 if (irq_count < 0)
1152 return irq_count;
1153
1154 for (i = 0; i < irq_count; i++) {
1155 irq = vgic_get_irq(kvm, NULL, intids[i]);
1156 if (!irq)
1157 continue;
1158 update_lpi_config(kvm, irq, vcpu, false);
1159 vgic_put_irq(kvm, irq);
1160 }
1161
1162 kfree(intids);
1163
1164 if (vcpu->arch.vgic_cpu.vgic_v3.its_vpe.its_vm)
1165 its_invall_vpe(&vcpu->arch.vgic_cpu.vgic_v3.its_vpe);
1166
1167 return 0;
1168}
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178static int vgic_its_cmd_handle_movall(struct kvm *kvm, struct vgic_its *its,
1179 u64 *its_cmd)
1180{
1181 u32 target1_addr = its_cmd_get_target_addr(its_cmd);
1182 u32 target2_addr = its_cmd_mask_field(its_cmd, 3, 16, 32);
1183 struct kvm_vcpu *vcpu1, *vcpu2;
1184 struct vgic_irq *irq;
1185 u32 *intids;
1186 int irq_count, i;
1187
1188 if (target1_addr >= atomic_read(&kvm->online_vcpus) ||
1189 target2_addr >= atomic_read(&kvm->online_vcpus))
1190 return E_ITS_MOVALL_PROCNUM_OOR;
1191
1192 if (target1_addr == target2_addr)
1193 return 0;
1194
1195 vcpu1 = kvm_get_vcpu(kvm, target1_addr);
1196 vcpu2 = kvm_get_vcpu(kvm, target2_addr);
1197
1198 irq_count = vgic_copy_lpi_list(vcpu1, &intids);
1199 if (irq_count < 0)
1200 return irq_count;
1201
1202 for (i = 0; i < irq_count; i++) {
1203 irq = vgic_get_irq(kvm, NULL, intids[i]);
1204
1205 update_affinity(irq, vcpu2);
1206
1207 vgic_put_irq(kvm, irq);
1208 }
1209
1210 kfree(intids);
1211 return 0;
1212}
1213
1214
1215
1216
1217
1218static int vgic_its_cmd_handle_int(struct kvm *kvm, struct vgic_its *its,
1219 u64 *its_cmd)
1220{
1221 u32 msi_data = its_cmd_get_id(its_cmd);
1222 u64 msi_devid = its_cmd_get_deviceid(its_cmd);
1223
1224 return vgic_its_trigger_msi(kvm, its, msi_devid, msi_data);
1225}
1226
1227
1228
1229
1230
1231static int vgic_its_handle_command(struct kvm *kvm, struct vgic_its *its,
1232 u64 *its_cmd)
1233{
1234 int ret = -ENODEV;
1235
1236 mutex_lock(&its->its_lock);
1237 switch (its_cmd_get_command(its_cmd)) {
1238 case GITS_CMD_MAPD:
1239 ret = vgic_its_cmd_handle_mapd(kvm, its, its_cmd);
1240 break;
1241 case GITS_CMD_MAPC:
1242 ret = vgic_its_cmd_handle_mapc(kvm, its, its_cmd);
1243 break;
1244 case GITS_CMD_MAPI:
1245 ret = vgic_its_cmd_handle_mapi(kvm, its, its_cmd);
1246 break;
1247 case GITS_CMD_MAPTI:
1248 ret = vgic_its_cmd_handle_mapi(kvm, its, its_cmd);
1249 break;
1250 case GITS_CMD_MOVI:
1251 ret = vgic_its_cmd_handle_movi(kvm, its, its_cmd);
1252 break;
1253 case GITS_CMD_DISCARD:
1254 ret = vgic_its_cmd_handle_discard(kvm, its, its_cmd);
1255 break;
1256 case GITS_CMD_CLEAR:
1257 ret = vgic_its_cmd_handle_clear(kvm, its, its_cmd);
1258 break;
1259 case GITS_CMD_MOVALL:
1260 ret = vgic_its_cmd_handle_movall(kvm, its, its_cmd);
1261 break;
1262 case GITS_CMD_INT:
1263 ret = vgic_its_cmd_handle_int(kvm, its, its_cmd);
1264 break;
1265 case GITS_CMD_INV:
1266 ret = vgic_its_cmd_handle_inv(kvm, its, its_cmd);
1267 break;
1268 case GITS_CMD_INVALL:
1269 ret = vgic_its_cmd_handle_invall(kvm, its, its_cmd);
1270 break;
1271 case GITS_CMD_SYNC:
1272
1273 ret = 0;
1274 break;
1275 }
1276 mutex_unlock(&its->its_lock);
1277
1278 return ret;
1279}
1280
1281static u64 vgic_sanitise_its_baser(u64 reg)
1282{
1283 reg = vgic_sanitise_field(reg, GITS_BASER_SHAREABILITY_MASK,
1284 GITS_BASER_SHAREABILITY_SHIFT,
1285 vgic_sanitise_shareability);
1286 reg = vgic_sanitise_field(reg, GITS_BASER_INNER_CACHEABILITY_MASK,
1287 GITS_BASER_INNER_CACHEABILITY_SHIFT,
1288 vgic_sanitise_inner_cacheability);
1289 reg = vgic_sanitise_field(reg, GITS_BASER_OUTER_CACHEABILITY_MASK,
1290 GITS_BASER_OUTER_CACHEABILITY_SHIFT,
1291 vgic_sanitise_outer_cacheability);
1292
1293
1294 reg &= ~GENMASK_ULL(15, 12);
1295
1296
1297 reg = (reg & ~GITS_BASER_PAGE_SIZE_MASK) | GITS_BASER_PAGE_SIZE_64K;
1298
1299 return reg;
1300}
1301
1302static u64 vgic_sanitise_its_cbaser(u64 reg)
1303{
1304 reg = vgic_sanitise_field(reg, GITS_CBASER_SHAREABILITY_MASK,
1305 GITS_CBASER_SHAREABILITY_SHIFT,
1306 vgic_sanitise_shareability);
1307 reg = vgic_sanitise_field(reg, GITS_CBASER_INNER_CACHEABILITY_MASK,
1308 GITS_CBASER_INNER_CACHEABILITY_SHIFT,
1309 vgic_sanitise_inner_cacheability);
1310 reg = vgic_sanitise_field(reg, GITS_CBASER_OUTER_CACHEABILITY_MASK,
1311 GITS_CBASER_OUTER_CACHEABILITY_SHIFT,
1312 vgic_sanitise_outer_cacheability);
1313
1314
1315
1316
1317
1318 reg &= ~(GENMASK_ULL(51, 48) | GENMASK_ULL(15, 12));
1319
1320 return reg;
1321}
1322
1323static unsigned long vgic_mmio_read_its_cbaser(struct kvm *kvm,
1324 struct vgic_its *its,
1325 gpa_t addr, unsigned int len)
1326{
1327 return extract_bytes(its->cbaser, addr & 7, len);
1328}
1329
1330static void vgic_mmio_write_its_cbaser(struct kvm *kvm, struct vgic_its *its,
1331 gpa_t addr, unsigned int len,
1332 unsigned long val)
1333{
1334
1335 if (its->enabled)
1336 return;
1337
1338 mutex_lock(&its->cmd_lock);
1339 its->cbaser = update_64bit_reg(its->cbaser, addr & 7, len, val);
1340 its->cbaser = vgic_sanitise_its_cbaser(its->cbaser);
1341 its->creadr = 0;
1342
1343
1344
1345
1346 its->cwriter = its->creadr;
1347 mutex_unlock(&its->cmd_lock);
1348}
1349
1350#define ITS_CMD_BUFFER_SIZE(baser) ((((baser) & 0xff) + 1) << 12)
1351#define ITS_CMD_SIZE 32
1352#define ITS_CMD_OFFSET(reg) ((reg) & GENMASK(19, 5))
1353
1354
1355static void vgic_its_process_commands(struct kvm *kvm, struct vgic_its *its)
1356{
1357 gpa_t cbaser;
1358 u64 cmd_buf[4];
1359
1360
1361 if (!its->enabled)
1362 return;
1363
1364 cbaser = CBASER_ADDRESS(its->cbaser);
1365
1366 while (its->cwriter != its->creadr) {
1367 int ret = kvm_read_guest(kvm, cbaser + its->creadr,
1368 cmd_buf, ITS_CMD_SIZE);
1369
1370
1371
1372
1373
1374
1375
1376 if (!ret)
1377 vgic_its_handle_command(kvm, its, cmd_buf);
1378
1379 its->creadr += ITS_CMD_SIZE;
1380 if (its->creadr == ITS_CMD_BUFFER_SIZE(its->cbaser))
1381 its->creadr = 0;
1382 }
1383}
1384
1385
1386
1387
1388
1389
1390
1391static void vgic_mmio_write_its_cwriter(struct kvm *kvm, struct vgic_its *its,
1392 gpa_t addr, unsigned int len,
1393 unsigned long val)
1394{
1395 u64 reg;
1396
1397 if (!its)
1398 return;
1399
1400 mutex_lock(&its->cmd_lock);
1401
1402 reg = update_64bit_reg(its->cwriter, addr & 7, len, val);
1403 reg = ITS_CMD_OFFSET(reg);
1404 if (reg >= ITS_CMD_BUFFER_SIZE(its->cbaser)) {
1405 mutex_unlock(&its->cmd_lock);
1406 return;
1407 }
1408 its->cwriter = reg;
1409
1410 vgic_its_process_commands(kvm, its);
1411
1412 mutex_unlock(&its->cmd_lock);
1413}
1414
1415static unsigned long vgic_mmio_read_its_cwriter(struct kvm *kvm,
1416 struct vgic_its *its,
1417 gpa_t addr, unsigned int len)
1418{
1419 return extract_bytes(its->cwriter, addr & 0x7, len);
1420}
1421
1422static unsigned long vgic_mmio_read_its_creadr(struct kvm *kvm,
1423 struct vgic_its *its,
1424 gpa_t addr, unsigned int len)
1425{
1426 return extract_bytes(its->creadr, addr & 0x7, len);
1427}
1428
1429static int vgic_mmio_uaccess_write_its_creadr(struct kvm *kvm,
1430 struct vgic_its *its,
1431 gpa_t addr, unsigned int len,
1432 unsigned long val)
1433{
1434 u32 cmd_offset;
1435 int ret = 0;
1436
1437 mutex_lock(&its->cmd_lock);
1438
1439 if (its->enabled) {
1440 ret = -EBUSY;
1441 goto out;
1442 }
1443
1444 cmd_offset = ITS_CMD_OFFSET(val);
1445 if (cmd_offset >= ITS_CMD_BUFFER_SIZE(its->cbaser)) {
1446 ret = -EINVAL;
1447 goto out;
1448 }
1449
1450 its->creadr = cmd_offset;
1451out:
1452 mutex_unlock(&its->cmd_lock);
1453 return ret;
1454}
1455
1456#define BASER_INDEX(addr) (((addr) / sizeof(u64)) & 0x7)
1457static unsigned long vgic_mmio_read_its_baser(struct kvm *kvm,
1458 struct vgic_its *its,
1459 gpa_t addr, unsigned int len)
1460{
1461 u64 reg;
1462
1463 switch (BASER_INDEX(addr)) {
1464 case 0:
1465 reg = its->baser_device_table;
1466 break;
1467 case 1:
1468 reg = its->baser_coll_table;
1469 break;
1470 default:
1471 reg = 0;
1472 break;
1473 }
1474
1475 return extract_bytes(reg, addr & 7, len);
1476}
1477
1478#define GITS_BASER_RO_MASK (GENMASK_ULL(52, 48) | GENMASK_ULL(58, 56))
1479static void vgic_mmio_write_its_baser(struct kvm *kvm,
1480 struct vgic_its *its,
1481 gpa_t addr, unsigned int len,
1482 unsigned long val)
1483{
1484 const struct vgic_its_abi *abi = vgic_its_get_abi(its);
1485 u64 entry_size, table_type;
1486 u64 reg, *regptr, clearbits = 0;
1487
1488
1489 if (its->enabled)
1490 return;
1491
1492 switch (BASER_INDEX(addr)) {
1493 case 0:
1494 regptr = &its->baser_device_table;
1495 entry_size = abi->dte_esz;
1496 table_type = GITS_BASER_TYPE_DEVICE;
1497 break;
1498 case 1:
1499 regptr = &its->baser_coll_table;
1500 entry_size = abi->cte_esz;
1501 table_type = GITS_BASER_TYPE_COLLECTION;
1502 clearbits = GITS_BASER_INDIRECT;
1503 break;
1504 default:
1505 return;
1506 }
1507
1508 reg = update_64bit_reg(*regptr, addr & 7, len, val);
1509 reg &= ~GITS_BASER_RO_MASK;
1510 reg &= ~clearbits;
1511
1512 reg |= (entry_size - 1) << GITS_BASER_ENTRY_SIZE_SHIFT;
1513 reg |= table_type << GITS_BASER_TYPE_SHIFT;
1514 reg = vgic_sanitise_its_baser(reg);
1515
1516 *regptr = reg;
1517
1518 if (!(reg & GITS_BASER_VALID)) {
1519
1520 mutex_lock(&its->its_lock);
1521 switch (table_type) {
1522 case GITS_BASER_TYPE_DEVICE:
1523 vgic_its_free_device_list(kvm, its);
1524 break;
1525 case GITS_BASER_TYPE_COLLECTION:
1526 vgic_its_free_collection_list(kvm, its);
1527 break;
1528 }
1529 mutex_unlock(&its->its_lock);
1530 }
1531}
1532
1533static unsigned long vgic_mmio_read_its_ctlr(struct kvm *vcpu,
1534 struct vgic_its *its,
1535 gpa_t addr, unsigned int len)
1536{
1537 u32 reg = 0;
1538
1539 mutex_lock(&its->cmd_lock);
1540 if (its->creadr == its->cwriter)
1541 reg |= GITS_CTLR_QUIESCENT;
1542 if (its->enabled)
1543 reg |= GITS_CTLR_ENABLE;
1544 mutex_unlock(&its->cmd_lock);
1545
1546 return reg;
1547}
1548
1549static void vgic_mmio_write_its_ctlr(struct kvm *kvm, struct vgic_its *its,
1550 gpa_t addr, unsigned int len,
1551 unsigned long val)
1552{
1553 mutex_lock(&its->cmd_lock);
1554
1555
1556
1557
1558
1559 if (!its->enabled && (val & GITS_CTLR_ENABLE) &&
1560 (!(its->baser_device_table & GITS_BASER_VALID) ||
1561 !(its->baser_coll_table & GITS_BASER_VALID) ||
1562 !(its->cbaser & GITS_CBASER_VALID)))
1563 goto out;
1564
1565 its->enabled = !!(val & GITS_CTLR_ENABLE);
1566
1567
1568
1569
1570
1571 vgic_its_process_commands(kvm, its);
1572
1573out:
1574 mutex_unlock(&its->cmd_lock);
1575}
1576
1577#define REGISTER_ITS_DESC(off, rd, wr, length, acc) \
1578{ \
1579 .reg_offset = off, \
1580 .len = length, \
1581 .access_flags = acc, \
1582 .its_read = rd, \
1583 .its_write = wr, \
1584}
1585
1586#define REGISTER_ITS_DESC_UACCESS(off, rd, wr, uwr, length, acc)\
1587{ \
1588 .reg_offset = off, \
1589 .len = length, \
1590 .access_flags = acc, \
1591 .its_read = rd, \
1592 .its_write = wr, \
1593 .uaccess_its_write = uwr, \
1594}
1595
1596static void its_mmio_write_wi(struct kvm *kvm, struct vgic_its *its,
1597 gpa_t addr, unsigned int len, unsigned long val)
1598{
1599
1600}
1601
1602static struct vgic_register_region its_registers[] = {
1603 REGISTER_ITS_DESC(GITS_CTLR,
1604 vgic_mmio_read_its_ctlr, vgic_mmio_write_its_ctlr, 4,
1605 VGIC_ACCESS_32bit),
1606 REGISTER_ITS_DESC_UACCESS(GITS_IIDR,
1607 vgic_mmio_read_its_iidr, its_mmio_write_wi,
1608 vgic_mmio_uaccess_write_its_iidr, 4,
1609 VGIC_ACCESS_32bit),
1610 REGISTER_ITS_DESC(GITS_TYPER,
1611 vgic_mmio_read_its_typer, its_mmio_write_wi, 8,
1612 VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
1613 REGISTER_ITS_DESC(GITS_CBASER,
1614 vgic_mmio_read_its_cbaser, vgic_mmio_write_its_cbaser, 8,
1615 VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
1616 REGISTER_ITS_DESC(GITS_CWRITER,
1617 vgic_mmio_read_its_cwriter, vgic_mmio_write_its_cwriter, 8,
1618 VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
1619 REGISTER_ITS_DESC_UACCESS(GITS_CREADR,
1620 vgic_mmio_read_its_creadr, its_mmio_write_wi,
1621 vgic_mmio_uaccess_write_its_creadr, 8,
1622 VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
1623 REGISTER_ITS_DESC(GITS_BASER,
1624 vgic_mmio_read_its_baser, vgic_mmio_write_its_baser, 0x40,
1625 VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
1626 REGISTER_ITS_DESC(GITS_IDREGS_BASE,
1627 vgic_mmio_read_its_idregs, its_mmio_write_wi, 0x30,
1628 VGIC_ACCESS_32bit),
1629};
1630
1631
1632void vgic_enable_lpis(struct kvm_vcpu *vcpu)
1633{
1634 if (!(vcpu->arch.vgic_cpu.pendbaser & GICR_PENDBASER_PTZ))
1635 its_sync_lpi_pending_table(vcpu);
1636}
1637
1638static int vgic_register_its_iodev(struct kvm *kvm, struct vgic_its *its,
1639 u64 addr)
1640{
1641 struct vgic_io_device *iodev = &its->iodev;
1642 int ret;
1643
1644 mutex_lock(&kvm->slots_lock);
1645 if (!IS_VGIC_ADDR_UNDEF(its->vgic_its_base)) {
1646 ret = -EBUSY;
1647 goto out;
1648 }
1649
1650 its->vgic_its_base = addr;
1651 iodev->regions = its_registers;
1652 iodev->nr_regions = ARRAY_SIZE(its_registers);
1653 kvm_iodevice_init(&iodev->dev, &kvm_io_gic_ops);
1654
1655 iodev->base_addr = its->vgic_its_base;
1656 iodev->iodev_type = IODEV_ITS;
1657 iodev->its = its;
1658 ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, iodev->base_addr,
1659 KVM_VGIC_V3_ITS_SIZE, &iodev->dev);
1660out:
1661 mutex_unlock(&kvm->slots_lock);
1662
1663 return ret;
1664}
1665
1666#define INITIAL_BASER_VALUE \
1667 (GIC_BASER_CACHEABILITY(GITS_BASER, INNER, RaWb) | \
1668 GIC_BASER_CACHEABILITY(GITS_BASER, OUTER, SameAsInner) | \
1669 GIC_BASER_SHAREABILITY(GITS_BASER, InnerShareable) | \
1670 GITS_BASER_PAGE_SIZE_64K)
1671
1672#define INITIAL_PROPBASER_VALUE \
1673 (GIC_BASER_CACHEABILITY(GICR_PROPBASER, INNER, RaWb) | \
1674 GIC_BASER_CACHEABILITY(GICR_PROPBASER, OUTER, SameAsInner) | \
1675 GIC_BASER_SHAREABILITY(GICR_PROPBASER, InnerShareable))
1676
1677static int vgic_its_create(struct kvm_device *dev, u32 type)
1678{
1679 struct vgic_its *its;
1680
1681 if (type != KVM_DEV_TYPE_ARM_VGIC_ITS)
1682 return -ENODEV;
1683
1684 its = kzalloc(sizeof(struct vgic_its), GFP_KERNEL);
1685 if (!its)
1686 return -ENOMEM;
1687
1688 if (vgic_initialized(dev->kvm)) {
1689 int ret = vgic_v4_init(dev->kvm);
1690 if (ret < 0) {
1691 kfree(its);
1692 return ret;
1693 }
1694 }
1695
1696 mutex_init(&its->its_lock);
1697 mutex_init(&its->cmd_lock);
1698
1699 its->vgic_its_base = VGIC_ADDR_UNDEF;
1700
1701 INIT_LIST_HEAD(&its->device_list);
1702 INIT_LIST_HEAD(&its->collection_list);
1703
1704 dev->kvm->arch.vgic.msis_require_devid = true;
1705 dev->kvm->arch.vgic.has_its = true;
1706 its->enabled = false;
1707 its->dev = dev;
1708
1709 its->baser_device_table = INITIAL_BASER_VALUE |
1710 ((u64)GITS_BASER_TYPE_DEVICE << GITS_BASER_TYPE_SHIFT);
1711 its->baser_coll_table = INITIAL_BASER_VALUE |
1712 ((u64)GITS_BASER_TYPE_COLLECTION << GITS_BASER_TYPE_SHIFT);
1713 dev->kvm->arch.vgic.propbaser = INITIAL_PROPBASER_VALUE;
1714
1715 dev->private = its;
1716
1717 return vgic_its_set_abi(its, NR_ITS_ABIS - 1);
1718}
1719
1720static void vgic_its_destroy(struct kvm_device *kvm_dev)
1721{
1722 struct kvm *kvm = kvm_dev->kvm;
1723 struct vgic_its *its = kvm_dev->private;
1724
1725 mutex_lock(&its->its_lock);
1726
1727 vgic_its_free_device_list(kvm, its);
1728 vgic_its_free_collection_list(kvm, its);
1729
1730 mutex_unlock(&its->its_lock);
1731 kfree(its);
1732}
1733
1734int vgic_its_has_attr_regs(struct kvm_device *dev,
1735 struct kvm_device_attr *attr)
1736{
1737 const struct vgic_register_region *region;
1738 gpa_t offset = attr->attr;
1739 int align;
1740
1741 align = (offset < GITS_TYPER) || (offset >= GITS_PIDR4) ? 0x3 : 0x7;
1742
1743 if (offset & align)
1744 return -EINVAL;
1745
1746 region = vgic_find_mmio_region(its_registers,
1747 ARRAY_SIZE(its_registers),
1748 offset);
1749 if (!region)
1750 return -ENXIO;
1751
1752 return 0;
1753}
1754
1755int vgic_its_attr_regs_access(struct kvm_device *dev,
1756 struct kvm_device_attr *attr,
1757 u64 *reg, bool is_write)
1758{
1759 const struct vgic_register_region *region;
1760 struct vgic_its *its;
1761 gpa_t addr, offset;
1762 unsigned int len;
1763 int align, ret = 0;
1764
1765 its = dev->private;
1766 offset = attr->attr;
1767
1768
1769
1770
1771
1772
1773
1774
1775 if ((offset < GITS_TYPER) || (offset >= GITS_PIDR4))
1776 align = 0x3;
1777 else
1778 align = 0x7;
1779
1780 if (offset & align)
1781 return -EINVAL;
1782
1783 mutex_lock(&dev->kvm->lock);
1784
1785 if (IS_VGIC_ADDR_UNDEF(its->vgic_its_base)) {
1786 ret = -ENXIO;
1787 goto out;
1788 }
1789
1790 region = vgic_find_mmio_region(its_registers,
1791 ARRAY_SIZE(its_registers),
1792 offset);
1793 if (!region) {
1794 ret = -ENXIO;
1795 goto out;
1796 }
1797
1798 if (!lock_all_vcpus(dev->kvm)) {
1799 ret = -EBUSY;
1800 goto out;
1801 }
1802
1803 addr = its->vgic_its_base + offset;
1804
1805 len = region->access_flags & VGIC_ACCESS_64bit ? 8 : 4;
1806
1807 if (is_write) {
1808 if (region->uaccess_its_write)
1809 ret = region->uaccess_its_write(dev->kvm, its, addr,
1810 len, *reg);
1811 else
1812 region->its_write(dev->kvm, its, addr, len, *reg);
1813 } else {
1814 *reg = region->its_read(dev->kvm, its, addr, len);
1815 }
1816 unlock_all_vcpus(dev->kvm);
1817out:
1818 mutex_unlock(&dev->kvm->lock);
1819 return ret;
1820}
1821
1822static u32 compute_next_devid_offset(struct list_head *h,
1823 struct its_device *dev)
1824{
1825 struct its_device *next;
1826 u32 next_offset;
1827
1828 if (list_is_last(&dev->dev_list, h))
1829 return 0;
1830 next = list_next_entry(dev, dev_list);
1831 next_offset = next->device_id - dev->device_id;
1832
1833 return min_t(u32, next_offset, VITS_DTE_MAX_DEVID_OFFSET);
1834}
1835
1836static u32 compute_next_eventid_offset(struct list_head *h, struct its_ite *ite)
1837{
1838 struct its_ite *next;
1839 u32 next_offset;
1840
1841 if (list_is_last(&ite->ite_list, h))
1842 return 0;
1843 next = list_next_entry(ite, ite_list);
1844 next_offset = next->event_id - ite->event_id;
1845
1846 return min_t(u32, next_offset, VITS_ITE_MAX_EVENTID_OFFSET);
1847}
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859typedef int (*entry_fn_t)(struct vgic_its *its, u32 id, void *entry,
1860 void *opaque);
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877static int scan_its_table(struct vgic_its *its, gpa_t base, int size, int esz,
1878 int start_id, entry_fn_t fn, void *opaque)
1879{
1880 struct kvm *kvm = its->dev->kvm;
1881 unsigned long len = size;
1882 int id = start_id;
1883 gpa_t gpa = base;
1884 char entry[esz];
1885 int ret;
1886
1887 memset(entry, 0, esz);
1888
1889 while (len > 0) {
1890 int next_offset;
1891 size_t byte_offset;
1892
1893 ret = kvm_read_guest(kvm, gpa, entry, esz);
1894 if (ret)
1895 return ret;
1896
1897 next_offset = fn(its, id, entry, opaque);
1898 if (next_offset <= 0)
1899 return next_offset;
1900
1901 byte_offset = next_offset * esz;
1902 id += next_offset;
1903 gpa += byte_offset;
1904 len -= byte_offset;
1905 }
1906 return 1;
1907}
1908
1909
1910
1911
1912static int vgic_its_save_ite(struct vgic_its *its, struct its_device *dev,
1913 struct its_ite *ite, gpa_t gpa, int ite_esz)
1914{
1915 struct kvm *kvm = its->dev->kvm;
1916 u32 next_offset;
1917 u64 val;
1918
1919 next_offset = compute_next_eventid_offset(&dev->itt_head, ite);
1920 val = ((u64)next_offset << KVM_ITS_ITE_NEXT_SHIFT) |
1921 ((u64)ite->irq->intid << KVM_ITS_ITE_PINTID_SHIFT) |
1922 ite->collection->collection_id;
1923 val = cpu_to_le64(val);
1924 return kvm_write_guest(kvm, gpa, &val, ite_esz);
1925}
1926
1927
1928
1929
1930
1931
1932
1933static int vgic_its_restore_ite(struct vgic_its *its, u32 event_id,
1934 void *ptr, void *opaque)
1935{
1936 struct its_device *dev = (struct its_device *)opaque;
1937 struct its_collection *collection;
1938 struct kvm *kvm = its->dev->kvm;
1939 struct kvm_vcpu *vcpu = NULL;
1940 u64 val;
1941 u64 *p = (u64 *)ptr;
1942 struct vgic_irq *irq;
1943 u32 coll_id, lpi_id;
1944 struct its_ite *ite;
1945 u32 offset;
1946
1947 val = *p;
1948
1949 val = le64_to_cpu(val);
1950
1951 coll_id = val & KVM_ITS_ITE_ICID_MASK;
1952 lpi_id = (val & KVM_ITS_ITE_PINTID_MASK) >> KVM_ITS_ITE_PINTID_SHIFT;
1953
1954 if (!lpi_id)
1955 return 1;
1956
1957 if (lpi_id < VGIC_MIN_LPI)
1958 return -EINVAL;
1959
1960 offset = val >> KVM_ITS_ITE_NEXT_SHIFT;
1961 if (event_id + offset >= BIT_ULL(dev->num_eventid_bits))
1962 return -EINVAL;
1963
1964 collection = find_collection(its, coll_id);
1965 if (!collection)
1966 return -EINVAL;
1967
1968 ite = vgic_its_alloc_ite(dev, collection, event_id);
1969 if (IS_ERR(ite))
1970 return PTR_ERR(ite);
1971
1972 if (its_is_collection_mapped(collection))
1973 vcpu = kvm_get_vcpu(kvm, collection->target_addr);
1974
1975 irq = vgic_add_lpi(kvm, lpi_id, vcpu);
1976 if (IS_ERR(irq))
1977 return PTR_ERR(irq);
1978 ite->irq = irq;
1979
1980 return offset;
1981}
1982
1983static int vgic_its_ite_cmp(void *priv, struct list_head *a,
1984 struct list_head *b)
1985{
1986 struct its_ite *itea = container_of(a, struct its_ite, ite_list);
1987 struct its_ite *iteb = container_of(b, struct its_ite, ite_list);
1988
1989 if (itea->event_id < iteb->event_id)
1990 return -1;
1991 else
1992 return 1;
1993}
1994
1995static int vgic_its_save_itt(struct vgic_its *its, struct its_device *device)
1996{
1997 const struct vgic_its_abi *abi = vgic_its_get_abi(its);
1998 gpa_t base = device->itt_addr;
1999 struct its_ite *ite;
2000 int ret;
2001 int ite_esz = abi->ite_esz;
2002
2003 list_sort(NULL, &device->itt_head, vgic_its_ite_cmp);
2004
2005 list_for_each_entry(ite, &device->itt_head, ite_list) {
2006 gpa_t gpa = base + ite->event_id * ite_esz;
2007
2008
2009
2010
2011
2012
2013
2014 if (ite->irq->hw)
2015 return -EACCES;
2016
2017 ret = vgic_its_save_ite(its, device, ite, gpa, ite_esz);
2018 if (ret)
2019 return ret;
2020 }
2021 return 0;
2022}
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032static int vgic_its_restore_itt(struct vgic_its *its, struct its_device *dev)
2033{
2034 const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2035 gpa_t base = dev->itt_addr;
2036 int ret;
2037 int ite_esz = abi->ite_esz;
2038 size_t max_size = BIT_ULL(dev->num_eventid_bits) * ite_esz;
2039
2040 ret = scan_its_table(its, base, max_size, ite_esz, 0,
2041 vgic_its_restore_ite, dev);
2042
2043
2044 if (ret > 0)
2045 ret = 0;
2046
2047 return ret;
2048}
2049
2050
2051
2052
2053
2054
2055
2056
2057static int vgic_its_save_dte(struct vgic_its *its, struct its_device *dev,
2058 gpa_t ptr, int dte_esz)
2059{
2060 struct kvm *kvm = its->dev->kvm;
2061 u64 val, itt_addr_field;
2062 u32 next_offset;
2063
2064 itt_addr_field = dev->itt_addr >> 8;
2065 next_offset = compute_next_devid_offset(&its->device_list, dev);
2066 val = (1ULL << KVM_ITS_DTE_VALID_SHIFT |
2067 ((u64)next_offset << KVM_ITS_DTE_NEXT_SHIFT) |
2068 (itt_addr_field << KVM_ITS_DTE_ITTADDR_SHIFT) |
2069 (dev->num_eventid_bits - 1));
2070 val = cpu_to_le64(val);
2071 return kvm_write_guest(kvm, ptr, &val, dte_esz);
2072}
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085static int vgic_its_restore_dte(struct vgic_its *its, u32 id,
2086 void *ptr, void *opaque)
2087{
2088 struct its_device *dev;
2089 gpa_t itt_addr;
2090 u8 num_eventid_bits;
2091 u64 entry = *(u64 *)ptr;
2092 bool valid;
2093 u32 offset;
2094 int ret;
2095
2096 entry = le64_to_cpu(entry);
2097
2098 valid = entry >> KVM_ITS_DTE_VALID_SHIFT;
2099 num_eventid_bits = (entry & KVM_ITS_DTE_SIZE_MASK) + 1;
2100 itt_addr = ((entry & KVM_ITS_DTE_ITTADDR_MASK)
2101 >> KVM_ITS_DTE_ITTADDR_SHIFT) << 8;
2102
2103 if (!valid)
2104 return 1;
2105
2106
2107 offset = (entry & KVM_ITS_DTE_NEXT_MASK) >> KVM_ITS_DTE_NEXT_SHIFT;
2108
2109 dev = vgic_its_alloc_device(its, id, itt_addr, num_eventid_bits);
2110 if (IS_ERR(dev))
2111 return PTR_ERR(dev);
2112
2113 ret = vgic_its_restore_itt(its, dev);
2114 if (ret) {
2115 vgic_its_free_device(its->dev->kvm, dev);
2116 return ret;
2117 }
2118
2119 return offset;
2120}
2121
2122static int vgic_its_device_cmp(void *priv, struct list_head *a,
2123 struct list_head *b)
2124{
2125 struct its_device *deva = container_of(a, struct its_device, dev_list);
2126 struct its_device *devb = container_of(b, struct its_device, dev_list);
2127
2128 if (deva->device_id < devb->device_id)
2129 return -1;
2130 else
2131 return 1;
2132}
2133
2134
2135
2136
2137
2138
2139
2140
2141static int vgic_its_save_device_tables(struct vgic_its *its)
2142{
2143 const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2144 u64 baser = its->baser_device_table;
2145 struct its_device *dev;
2146 int dte_esz = abi->dte_esz;
2147
2148 if (!(baser & GITS_BASER_VALID))
2149 return 0;
2150
2151 list_sort(NULL, &its->device_list, vgic_its_device_cmp);
2152
2153 list_for_each_entry(dev, &its->device_list, dev_list) {
2154 int ret;
2155 gpa_t eaddr;
2156
2157 if (!vgic_its_check_id(its, baser,
2158 dev->device_id, &eaddr))
2159 return -EINVAL;
2160
2161 ret = vgic_its_save_itt(its, dev);
2162 if (ret)
2163 return ret;
2164
2165 ret = vgic_its_save_dte(its, dev, eaddr, dte_esz);
2166 if (ret)
2167 return ret;
2168 }
2169 return 0;
2170}
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184static int handle_l1_dte(struct vgic_its *its, u32 id, void *addr,
2185 void *opaque)
2186{
2187 const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2188 int l2_start_id = id * (SZ_64K / abi->dte_esz);
2189 u64 entry = *(u64 *)addr;
2190 int dte_esz = abi->dte_esz;
2191 gpa_t gpa;
2192 int ret;
2193
2194 entry = le64_to_cpu(entry);
2195
2196 if (!(entry & KVM_ITS_L1E_VALID_MASK))
2197 return 1;
2198
2199 gpa = entry & KVM_ITS_L1E_ADDR_MASK;
2200
2201 ret = scan_its_table(its, gpa, SZ_64K, dte_esz,
2202 l2_start_id, vgic_its_restore_dte, NULL);
2203
2204 return ret;
2205}
2206
2207
2208
2209
2210
2211static int vgic_its_restore_device_tables(struct vgic_its *its)
2212{
2213 const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2214 u64 baser = its->baser_device_table;
2215 int l1_esz, ret;
2216 int l1_tbl_size = GITS_BASER_NR_PAGES(baser) * SZ_64K;
2217 gpa_t l1_gpa;
2218
2219 if (!(baser & GITS_BASER_VALID))
2220 return 0;
2221
2222 l1_gpa = BASER_ADDRESS(baser);
2223
2224 if (baser & GITS_BASER_INDIRECT) {
2225 l1_esz = GITS_LVL1_ENTRY_SIZE;
2226 ret = scan_its_table(its, l1_gpa, l1_tbl_size, l1_esz, 0,
2227 handle_l1_dte, NULL);
2228 } else {
2229 l1_esz = abi->dte_esz;
2230 ret = scan_its_table(its, l1_gpa, l1_tbl_size, l1_esz, 0,
2231 vgic_its_restore_dte, NULL);
2232 }
2233
2234
2235 if (ret > 0)
2236 ret = 0;
2237
2238 return ret;
2239}
2240
2241static int vgic_its_save_cte(struct vgic_its *its,
2242 struct its_collection *collection,
2243 gpa_t gpa, int esz)
2244{
2245 u64 val;
2246
2247 val = (1ULL << KVM_ITS_CTE_VALID_SHIFT |
2248 ((u64)collection->target_addr << KVM_ITS_CTE_RDBASE_SHIFT) |
2249 collection->collection_id);
2250 val = cpu_to_le64(val);
2251 return kvm_write_guest(its->dev->kvm, gpa, &val, esz);
2252}
2253
2254static int vgic_its_restore_cte(struct vgic_its *its, gpa_t gpa, int esz)
2255{
2256 struct its_collection *collection;
2257 struct kvm *kvm = its->dev->kvm;
2258 u32 target_addr, coll_id;
2259 u64 val;
2260 int ret;
2261
2262 BUG_ON(esz > sizeof(val));
2263 ret = kvm_read_guest(kvm, gpa, &val, esz);
2264 if (ret)
2265 return ret;
2266 val = le64_to_cpu(val);
2267 if (!(val & KVM_ITS_CTE_VALID_MASK))
2268 return 0;
2269
2270 target_addr = (u32)(val >> KVM_ITS_CTE_RDBASE_SHIFT);
2271 coll_id = val & KVM_ITS_CTE_ICID_MASK;
2272
2273 if (target_addr >= atomic_read(&kvm->online_vcpus))
2274 return -EINVAL;
2275
2276 collection = find_collection(its, coll_id);
2277 if (collection)
2278 return -EEXIST;
2279 ret = vgic_its_alloc_collection(its, &collection, coll_id);
2280 if (ret)
2281 return ret;
2282 collection->target_addr = target_addr;
2283 return 1;
2284}
2285
2286
2287
2288
2289
2290static int vgic_its_save_collection_table(struct vgic_its *its)
2291{
2292 const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2293 u64 baser = its->baser_coll_table;
2294 gpa_t gpa = BASER_ADDRESS(baser);
2295 struct its_collection *collection;
2296 u64 val;
2297 size_t max_size, filled = 0;
2298 int ret, cte_esz = abi->cte_esz;
2299
2300 if (!(baser & GITS_BASER_VALID))
2301 return 0;
2302
2303 max_size = GITS_BASER_NR_PAGES(baser) * SZ_64K;
2304
2305 list_for_each_entry(collection, &its->collection_list, coll_list) {
2306 ret = vgic_its_save_cte(its, collection, gpa, cte_esz);
2307 if (ret)
2308 return ret;
2309 gpa += cte_esz;
2310 filled += cte_esz;
2311 }
2312
2313 if (filled == max_size)
2314 return 0;
2315
2316
2317
2318
2319
2320 val = 0;
2321 BUG_ON(cte_esz > sizeof(val));
2322 ret = kvm_write_guest(its->dev->kvm, gpa, &val, cte_esz);
2323 return ret;
2324}
2325
2326
2327
2328
2329
2330
2331static int vgic_its_restore_collection_table(struct vgic_its *its)
2332{
2333 const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2334 u64 baser = its->baser_coll_table;
2335 int cte_esz = abi->cte_esz;
2336 size_t max_size, read = 0;
2337 gpa_t gpa;
2338 int ret;
2339
2340 if (!(baser & GITS_BASER_VALID))
2341 return 0;
2342
2343 gpa = BASER_ADDRESS(baser);
2344
2345 max_size = GITS_BASER_NR_PAGES(baser) * SZ_64K;
2346
2347 while (read < max_size) {
2348 ret = vgic_its_restore_cte(its, gpa, cte_esz);
2349 if (ret <= 0)
2350 break;
2351 gpa += cte_esz;
2352 read += cte_esz;
2353 }
2354
2355 if (ret > 0)
2356 return 0;
2357
2358 return ret;
2359}
2360
2361
2362
2363
2364
2365static int vgic_its_save_tables_v0(struct vgic_its *its)
2366{
2367 int ret;
2368
2369 ret = vgic_its_save_device_tables(its);
2370 if (ret)
2371 return ret;
2372
2373 return vgic_its_save_collection_table(its);
2374}
2375
2376
2377
2378
2379
2380
2381static int vgic_its_restore_tables_v0(struct vgic_its *its)
2382{
2383 int ret;
2384
2385 ret = vgic_its_restore_collection_table(its);
2386 if (ret)
2387 return ret;
2388
2389 return vgic_its_restore_device_tables(its);
2390}
2391
2392static int vgic_its_commit_v0(struct vgic_its *its)
2393{
2394 const struct vgic_its_abi *abi;
2395
2396 abi = vgic_its_get_abi(its);
2397 its->baser_coll_table &= ~GITS_BASER_ENTRY_SIZE_MASK;
2398 its->baser_device_table &= ~GITS_BASER_ENTRY_SIZE_MASK;
2399
2400 its->baser_coll_table |= (GIC_ENCODE_SZ(abi->cte_esz, 5)
2401 << GITS_BASER_ENTRY_SIZE_SHIFT);
2402
2403 its->baser_device_table |= (GIC_ENCODE_SZ(abi->dte_esz, 5)
2404 << GITS_BASER_ENTRY_SIZE_SHIFT);
2405 return 0;
2406}
2407
2408static void vgic_its_reset(struct kvm *kvm, struct vgic_its *its)
2409{
2410
2411 its->baser_coll_table &= ~GITS_BASER_VALID;
2412 its->baser_device_table &= ~GITS_BASER_VALID;
2413 its->cbaser = 0;
2414 its->creadr = 0;
2415 its->cwriter = 0;
2416 its->enabled = 0;
2417 vgic_its_free_device_list(kvm, its);
2418 vgic_its_free_collection_list(kvm, its);
2419}
2420
2421static int vgic_its_has_attr(struct kvm_device *dev,
2422 struct kvm_device_attr *attr)
2423{
2424 switch (attr->group) {
2425 case KVM_DEV_ARM_VGIC_GRP_ADDR:
2426 switch (attr->attr) {
2427 case KVM_VGIC_ITS_ADDR_TYPE:
2428 return 0;
2429 }
2430 break;
2431 case KVM_DEV_ARM_VGIC_GRP_CTRL:
2432 switch (attr->attr) {
2433 case KVM_DEV_ARM_VGIC_CTRL_INIT:
2434 return 0;
2435 case KVM_DEV_ARM_ITS_CTRL_RESET:
2436 return 0;
2437 case KVM_DEV_ARM_ITS_SAVE_TABLES:
2438 return 0;
2439 case KVM_DEV_ARM_ITS_RESTORE_TABLES:
2440 return 0;
2441 }
2442 break;
2443 case KVM_DEV_ARM_VGIC_GRP_ITS_REGS:
2444 return vgic_its_has_attr_regs(dev, attr);
2445 }
2446 return -ENXIO;
2447}
2448
2449static int vgic_its_ctrl(struct kvm *kvm, struct vgic_its *its, u64 attr)
2450{
2451 const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2452 int ret = 0;
2453
2454 if (attr == KVM_DEV_ARM_VGIC_CTRL_INIT)
2455 return 0;
2456
2457 mutex_lock(&kvm->lock);
2458 mutex_lock(&its->its_lock);
2459
2460 if (!lock_all_vcpus(kvm)) {
2461 mutex_unlock(&its->its_lock);
2462 mutex_unlock(&kvm->lock);
2463 return -EBUSY;
2464 }
2465
2466 switch (attr) {
2467 case KVM_DEV_ARM_ITS_CTRL_RESET:
2468 vgic_its_reset(kvm, its);
2469 break;
2470 case KVM_DEV_ARM_ITS_SAVE_TABLES:
2471 ret = abi->save_tables(its);
2472 break;
2473 case KVM_DEV_ARM_ITS_RESTORE_TABLES:
2474 ret = abi->restore_tables(its);
2475 break;
2476 }
2477
2478 unlock_all_vcpus(kvm);
2479 mutex_unlock(&its->its_lock);
2480 mutex_unlock(&kvm->lock);
2481 return ret;
2482}
2483
2484static int vgic_its_set_attr(struct kvm_device *dev,
2485 struct kvm_device_attr *attr)
2486{
2487 struct vgic_its *its = dev->private;
2488 int ret;
2489
2490 switch (attr->group) {
2491 case KVM_DEV_ARM_VGIC_GRP_ADDR: {
2492 u64 __user *uaddr = (u64 __user *)(long)attr->addr;
2493 unsigned long type = (unsigned long)attr->attr;
2494 u64 addr;
2495
2496 if (type != KVM_VGIC_ITS_ADDR_TYPE)
2497 return -ENODEV;
2498
2499 if (copy_from_user(&addr, uaddr, sizeof(addr)))
2500 return -EFAULT;
2501
2502 ret = vgic_check_ioaddr(dev->kvm, &its->vgic_its_base,
2503 addr, SZ_64K);
2504 if (ret)
2505 return ret;
2506
2507 return vgic_register_its_iodev(dev->kvm, its, addr);
2508 }
2509 case KVM_DEV_ARM_VGIC_GRP_CTRL:
2510 return vgic_its_ctrl(dev->kvm, its, attr->attr);
2511 case KVM_DEV_ARM_VGIC_GRP_ITS_REGS: {
2512 u64 __user *uaddr = (u64 __user *)(long)attr->addr;
2513 u64 reg;
2514
2515 if (get_user(reg, uaddr))
2516 return -EFAULT;
2517
2518 return vgic_its_attr_regs_access(dev, attr, ®, true);
2519 }
2520 }
2521 return -ENXIO;
2522}
2523
2524static int vgic_its_get_attr(struct kvm_device *dev,
2525 struct kvm_device_attr *attr)
2526{
2527 switch (attr->group) {
2528 case KVM_DEV_ARM_VGIC_GRP_ADDR: {
2529 struct vgic_its *its = dev->private;
2530 u64 addr = its->vgic_its_base;
2531 u64 __user *uaddr = (u64 __user *)(long)attr->addr;
2532 unsigned long type = (unsigned long)attr->attr;
2533
2534 if (type != KVM_VGIC_ITS_ADDR_TYPE)
2535 return -ENODEV;
2536
2537 if (copy_to_user(uaddr, &addr, sizeof(addr)))
2538 return -EFAULT;
2539 break;
2540 }
2541 case KVM_DEV_ARM_VGIC_GRP_ITS_REGS: {
2542 u64 __user *uaddr = (u64 __user *)(long)attr->addr;
2543 u64 reg;
2544 int ret;
2545
2546 ret = vgic_its_attr_regs_access(dev, attr, ®, false);
2547 if (ret)
2548 return ret;
2549 return put_user(reg, uaddr);
2550 }
2551 default:
2552 return -ENXIO;
2553 }
2554
2555 return 0;
2556}
2557
2558static struct kvm_device_ops kvm_arm_vgic_its_ops = {
2559 .name = "kvm-arm-vgic-its",
2560 .create = vgic_its_create,
2561 .destroy = vgic_its_destroy,
2562 .set_attr = vgic_its_set_attr,
2563 .get_attr = vgic_its_get_attr,
2564 .has_attr = vgic_its_has_attr,
2565};
2566
2567int kvm_vgic_register_its_device(void)
2568{
2569 return kvm_register_device_ops(&kvm_arm_vgic_its_ops,
2570 KVM_DEV_TYPE_ARM_VGIC_ITS);
2571}
2572