1
2
3
4
5
6
7
8
9#include <linux/cpu.h>
10#include <linux/kvm.h>
11#include <linux/kvm_host.h>
12#include <linux/interrupt.h>
13#include <linux/list.h>
14#include <linux/uaccess.h>
15#include <linux/list_sort.h>
16
17#include <linux/irqchip/arm-gic-v3.h>
18
19#include <asm/kvm_emulate.h>
20#include <asm/kvm_arm.h>
21#include <asm/kvm_mmu.h>
22
23#include "vgic.h"
24#include "vgic-mmio.h"
25
26static int vgic_its_save_tables_v0(struct vgic_its *its);
27static int vgic_its_restore_tables_v0(struct vgic_its *its);
28static int vgic_its_commit_v0(struct vgic_its *its);
29static int update_lpi_config(struct kvm *kvm, struct vgic_irq *irq,
30 struct kvm_vcpu *filter_vcpu, bool needs_inv);
31
32
33
34
35
36
37
38
39static struct vgic_irq *vgic_add_lpi(struct kvm *kvm, u32 intid,
40 struct kvm_vcpu *vcpu)
41{
42 struct vgic_dist *dist = &kvm->arch.vgic;
43 struct vgic_irq *irq = vgic_get_irq(kvm, NULL, intid), *oldirq;
44 unsigned long flags;
45 int ret;
46
47
48 if (irq)
49 return irq;
50
51 irq = kzalloc(sizeof(struct vgic_irq), GFP_KERNEL);
52 if (!irq)
53 return ERR_PTR(-ENOMEM);
54
55 INIT_LIST_HEAD(&irq->lpi_list);
56 INIT_LIST_HEAD(&irq->ap_list);
57 raw_spin_lock_init(&irq->irq_lock);
58
59 irq->config = VGIC_CONFIG_EDGE;
60 kref_init(&irq->refcount);
61 irq->intid = intid;
62 irq->target_vcpu = vcpu;
63 irq->group = 1;
64
65 raw_spin_lock_irqsave(&dist->lpi_list_lock, flags);
66
67
68
69
70
71 list_for_each_entry(oldirq, &dist->lpi_list_head, lpi_list) {
72 if (oldirq->intid != intid)
73 continue;
74
75
76 kfree(irq);
77 irq = oldirq;
78
79
80
81
82
83
84 vgic_get_irq_kref(irq);
85
86 goto out_unlock;
87 }
88
89 list_add_tail(&irq->lpi_list, &dist->lpi_list_head);
90 dist->lpi_list_count++;
91
92out_unlock:
93 raw_spin_unlock_irqrestore(&dist->lpi_list_lock, flags);
94
95
96
97
98
99
100 ret = update_lpi_config(kvm, irq, NULL, false);
101 if (ret)
102 return ERR_PTR(ret);
103
104 ret = vgic_v3_lpi_sync_pending_status(kvm, irq);
105 if (ret)
106 return ERR_PTR(ret);
107
108 return irq;
109}
110
111struct its_device {
112 struct list_head dev_list;
113
114
115 struct list_head itt_head;
116 u32 num_eventid_bits;
117 gpa_t itt_addr;
118 u32 device_id;
119};
120
121#define COLLECTION_NOT_MAPPED ((u32)~0)
122
123struct its_collection {
124 struct list_head coll_list;
125
126 u32 collection_id;
127 u32 target_addr;
128};
129
130#define its_is_collection_mapped(coll) ((coll) && \
131 ((coll)->target_addr != COLLECTION_NOT_MAPPED))
132
133struct its_ite {
134 struct list_head ite_list;
135
136 struct vgic_irq *irq;
137 struct its_collection *collection;
138 u32 event_id;
139};
140
141
142
143
144
145
146
147
148
149
150
151
152struct vgic_its_abi {
153 int cte_esz;
154 int dte_esz;
155 int ite_esz;
156 int (*save_tables)(struct vgic_its *its);
157 int (*restore_tables)(struct vgic_its *its);
158 int (*commit)(struct vgic_its *its);
159};
160
161#define ABI_0_ESZ 8
162#define ESZ_MAX ABI_0_ESZ
163
164static const struct vgic_its_abi its_table_abi_versions[] = {
165 [0] = {
166 .cte_esz = ABI_0_ESZ,
167 .dte_esz = ABI_0_ESZ,
168 .ite_esz = ABI_0_ESZ,
169 .save_tables = vgic_its_save_tables_v0,
170 .restore_tables = vgic_its_restore_tables_v0,
171 .commit = vgic_its_commit_v0,
172 },
173};
174
175#define NR_ITS_ABIS ARRAY_SIZE(its_table_abi_versions)
176
177inline const struct vgic_its_abi *vgic_its_get_abi(struct vgic_its *its)
178{
179 return &its_table_abi_versions[its->abi_rev];
180}
181
182static int vgic_its_set_abi(struct vgic_its *its, u32 rev)
183{
184 const struct vgic_its_abi *abi;
185
186 its->abi_rev = rev;
187 abi = vgic_its_get_abi(its);
188 return abi->commit(its);
189}
190
191
192
193
194
195static struct its_device *find_its_device(struct vgic_its *its, u32 device_id)
196{
197 struct its_device *device;
198
199 list_for_each_entry(device, &its->device_list, dev_list)
200 if (device_id == device->device_id)
201 return device;
202
203 return NULL;
204}
205
206
207
208
209
210
211static struct its_ite *find_ite(struct vgic_its *its, u32 device_id,
212 u32 event_id)
213{
214 struct its_device *device;
215 struct its_ite *ite;
216
217 device = find_its_device(its, device_id);
218 if (device == NULL)
219 return NULL;
220
221 list_for_each_entry(ite, &device->itt_head, ite_list)
222 if (ite->event_id == event_id)
223 return ite;
224
225 return NULL;
226}
227
228
229#define for_each_lpi_its(dev, ite, its) \
230 list_for_each_entry(dev, &(its)->device_list, dev_list) \
231 list_for_each_entry(ite, &(dev)->itt_head, ite_list)
232
233#define GIC_LPI_OFFSET 8192
234
235#define VITS_TYPER_IDBITS 16
236#define VITS_TYPER_DEVBITS 16
237#define VITS_DTE_MAX_DEVID_OFFSET (BIT(14) - 1)
238#define VITS_ITE_MAX_EVENTID_OFFSET (BIT(16) - 1)
239
240
241
242
243
244static struct its_collection *find_collection(struct vgic_its *its, int coll_id)
245{
246 struct its_collection *collection;
247
248 list_for_each_entry(collection, &its->collection_list, coll_list) {
249 if (coll_id == collection->collection_id)
250 return collection;
251 }
252
253 return NULL;
254}
255
256#define LPI_PROP_ENABLE_BIT(p) ((p) & LPI_PROP_ENABLED)
257#define LPI_PROP_PRIORITY(p) ((p) & 0xfc)
258
259
260
261
262
263
264
265static int update_lpi_config(struct kvm *kvm, struct vgic_irq *irq,
266 struct kvm_vcpu *filter_vcpu, bool needs_inv)
267{
268 u64 propbase = GICR_PROPBASER_ADDRESS(kvm->arch.vgic.propbaser);
269 u8 prop;
270 int ret;
271 unsigned long flags;
272
273 ret = kvm_read_guest_lock(kvm, propbase + irq->intid - GIC_LPI_OFFSET,
274 &prop, 1);
275
276 if (ret)
277 return ret;
278
279 raw_spin_lock_irqsave(&irq->irq_lock, flags);
280
281 if (!filter_vcpu || filter_vcpu == irq->target_vcpu) {
282 irq->priority = LPI_PROP_PRIORITY(prop);
283 irq->enabled = LPI_PROP_ENABLE_BIT(prop);
284
285 if (!irq->hw) {
286 vgic_queue_irq_unlock(kvm, irq, flags);
287 return 0;
288 }
289 }
290
291 raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
292
293 if (irq->hw)
294 return its_prop_update_vlpi(irq->host_irq, prop, needs_inv);
295
296 return 0;
297}
298
299
300
301
302
303
304int vgic_copy_lpi_list(struct kvm *kvm, struct kvm_vcpu *vcpu, u32 **intid_ptr)
305{
306 struct vgic_dist *dist = &kvm->arch.vgic;
307 struct vgic_irq *irq;
308 unsigned long flags;
309 u32 *intids;
310 int irq_count, i = 0;
311
312
313
314
315
316
317
318
319 irq_count = READ_ONCE(dist->lpi_list_count);
320 intids = kmalloc_array(irq_count, sizeof(intids[0]), GFP_KERNEL);
321 if (!intids)
322 return -ENOMEM;
323
324 raw_spin_lock_irqsave(&dist->lpi_list_lock, flags);
325 list_for_each_entry(irq, &dist->lpi_list_head, lpi_list) {
326 if (i == irq_count)
327 break;
328
329 if (vcpu && irq->target_vcpu != vcpu)
330 continue;
331 intids[i++] = irq->intid;
332 }
333 raw_spin_unlock_irqrestore(&dist->lpi_list_lock, flags);
334
335 *intid_ptr = intids;
336 return i;
337}
338
339static int update_affinity(struct vgic_irq *irq, struct kvm_vcpu *vcpu)
340{
341 int ret = 0;
342 unsigned long flags;
343
344 raw_spin_lock_irqsave(&irq->irq_lock, flags);
345 irq->target_vcpu = vcpu;
346 raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
347
348 if (irq->hw) {
349 struct its_vlpi_map map;
350
351 ret = its_get_vlpi(irq->host_irq, &map);
352 if (ret)
353 return ret;
354
355 map.vpe = &vcpu->arch.vgic_cpu.vgic_v3.its_vpe;
356
357 ret = its_map_vlpi(irq->host_irq, &map);
358 }
359
360 return ret;
361}
362
363
364
365
366
367
368
369static void update_affinity_ite(struct kvm *kvm, struct its_ite *ite)
370{
371 struct kvm_vcpu *vcpu;
372
373 if (!its_is_collection_mapped(ite->collection))
374 return;
375
376 vcpu = kvm_get_vcpu(kvm, ite->collection->target_addr);
377 update_affinity(ite->irq, vcpu);
378}
379
380
381
382
383
384static void update_affinity_collection(struct kvm *kvm, struct vgic_its *its,
385 struct its_collection *coll)
386{
387 struct its_device *device;
388 struct its_ite *ite;
389
390 for_each_lpi_its(device, ite, its) {
391 if (!ite->collection || coll != ite->collection)
392 continue;
393
394 update_affinity_ite(kvm, ite);
395 }
396}
397
398static u32 max_lpis_propbaser(u64 propbaser)
399{
400 int nr_idbits = (propbaser & 0x1f) + 1;
401
402 return 1U << min(nr_idbits, INTERRUPT_ID_BITS_ITS);
403}
404
405
406
407
408
409
410static int its_sync_lpi_pending_table(struct kvm_vcpu *vcpu)
411{
412 gpa_t pendbase = GICR_PENDBASER_ADDRESS(vcpu->arch.vgic_cpu.pendbaser);
413 struct vgic_irq *irq;
414 int last_byte_offset = -1;
415 int ret = 0;
416 u32 *intids;
417 int nr_irqs, i;
418 unsigned long flags;
419 u8 pendmask;
420
421 nr_irqs = vgic_copy_lpi_list(vcpu->kvm, vcpu, &intids);
422 if (nr_irqs < 0)
423 return nr_irqs;
424
425 for (i = 0; i < nr_irqs; i++) {
426 int byte_offset, bit_nr;
427
428 byte_offset = intids[i] / BITS_PER_BYTE;
429 bit_nr = intids[i] % BITS_PER_BYTE;
430
431
432
433
434
435 if (byte_offset != last_byte_offset) {
436 ret = kvm_read_guest_lock(vcpu->kvm,
437 pendbase + byte_offset,
438 &pendmask, 1);
439 if (ret) {
440 kfree(intids);
441 return ret;
442 }
443 last_byte_offset = byte_offset;
444 }
445
446 irq = vgic_get_irq(vcpu->kvm, NULL, intids[i]);
447 raw_spin_lock_irqsave(&irq->irq_lock, flags);
448 irq->pending_latch = pendmask & (1U << bit_nr);
449 vgic_queue_irq_unlock(vcpu->kvm, irq, flags);
450 vgic_put_irq(vcpu->kvm, irq);
451 }
452
453 kfree(intids);
454
455 return ret;
456}
457
458static unsigned long vgic_mmio_read_its_typer(struct kvm *kvm,
459 struct vgic_its *its,
460 gpa_t addr, unsigned int len)
461{
462 const struct vgic_its_abi *abi = vgic_its_get_abi(its);
463 u64 reg = GITS_TYPER_PLPIS;
464
465
466
467
468
469
470
471
472
473 reg |= GIC_ENCODE_SZ(VITS_TYPER_DEVBITS, 5) << GITS_TYPER_DEVBITS_SHIFT;
474 reg |= GIC_ENCODE_SZ(VITS_TYPER_IDBITS, 5) << GITS_TYPER_IDBITS_SHIFT;
475 reg |= GIC_ENCODE_SZ(abi->ite_esz, 4) << GITS_TYPER_ITT_ENTRY_SIZE_SHIFT;
476
477 return extract_bytes(reg, addr & 7, len);
478}
479
480static unsigned long vgic_mmio_read_its_iidr(struct kvm *kvm,
481 struct vgic_its *its,
482 gpa_t addr, unsigned int len)
483{
484 u32 val;
485
486 val = (its->abi_rev << GITS_IIDR_REV_SHIFT) & GITS_IIDR_REV_MASK;
487 val |= (PRODUCT_ID_KVM << GITS_IIDR_PRODUCTID_SHIFT) | IMPLEMENTER_ARM;
488 return val;
489}
490
491static int vgic_mmio_uaccess_write_its_iidr(struct kvm *kvm,
492 struct vgic_its *its,
493 gpa_t addr, unsigned int len,
494 unsigned long val)
495{
496 u32 rev = GITS_IIDR_REV(val);
497
498 if (rev >= NR_ITS_ABIS)
499 return -EINVAL;
500 return vgic_its_set_abi(its, rev);
501}
502
503static unsigned long vgic_mmio_read_its_idregs(struct kvm *kvm,
504 struct vgic_its *its,
505 gpa_t addr, unsigned int len)
506{
507 switch (addr & 0xffff) {
508 case GITS_PIDR0:
509 return 0x92;
510 case GITS_PIDR1:
511 return 0xb4;
512 case GITS_PIDR2:
513 return GIC_PIDR2_ARCH_GICv3 | 0x0b;
514 case GITS_PIDR4:
515 return 0x40;
516
517 case GITS_CIDR0:
518 return 0x0d;
519 case GITS_CIDR1:
520 return 0xf0;
521 case GITS_CIDR2:
522 return 0x05;
523 case GITS_CIDR3:
524 return 0xb1;
525 }
526
527 return 0;
528}
529
530int vgic_its_resolve_lpi(struct kvm *kvm, struct vgic_its *its,
531 u32 devid, u32 eventid, struct vgic_irq **irq)
532{
533 struct kvm_vcpu *vcpu;
534 struct its_ite *ite;
535
536 if (!its->enabled)
537 return -EBUSY;
538
539 ite = find_ite(its, devid, eventid);
540 if (!ite || !its_is_collection_mapped(ite->collection))
541 return E_ITS_INT_UNMAPPED_INTERRUPT;
542
543 vcpu = kvm_get_vcpu(kvm, ite->collection->target_addr);
544 if (!vcpu)
545 return E_ITS_INT_UNMAPPED_INTERRUPT;
546
547 if (!vcpu->arch.vgic_cpu.lpis_enabled)
548 return -EBUSY;
549
550 *irq = ite->irq;
551 return 0;
552}
553
554struct vgic_its *vgic_msi_to_its(struct kvm *kvm, struct kvm_msi *msi)
555{
556 u64 address;
557 struct kvm_io_device *kvm_io_dev;
558 struct vgic_io_device *iodev;
559
560 if (!vgic_has_its(kvm))
561 return ERR_PTR(-ENODEV);
562
563 if (!(msi->flags & KVM_MSI_VALID_DEVID))
564 return ERR_PTR(-EINVAL);
565
566 address = (u64)msi->address_hi << 32 | msi->address_lo;
567
568 kvm_io_dev = kvm_io_bus_get_dev(kvm, KVM_MMIO_BUS, address);
569 if (!kvm_io_dev)
570 return ERR_PTR(-EINVAL);
571
572 if (kvm_io_dev->ops != &kvm_io_gic_ops)
573 return ERR_PTR(-EINVAL);
574
575 iodev = container_of(kvm_io_dev, struct vgic_io_device, dev);
576 if (iodev->iodev_type != IODEV_ITS)
577 return ERR_PTR(-EINVAL);
578
579 return iodev->its;
580}
581
582
583
584
585
586
587
588
589static int vgic_its_trigger_msi(struct kvm *kvm, struct vgic_its *its,
590 u32 devid, u32 eventid)
591{
592 struct vgic_irq *irq = NULL;
593 unsigned long flags;
594 int err;
595
596 err = vgic_its_resolve_lpi(kvm, its, devid, eventid, &irq);
597 if (err)
598 return err;
599
600 if (irq->hw)
601 return irq_set_irqchip_state(irq->host_irq,
602 IRQCHIP_STATE_PENDING, true);
603
604 raw_spin_lock_irqsave(&irq->irq_lock, flags);
605 irq->pending_latch = true;
606 vgic_queue_irq_unlock(kvm, irq, flags);
607
608 return 0;
609}
610
611
612
613
614
615
616
617int vgic_its_inject_msi(struct kvm *kvm, struct kvm_msi *msi)
618{
619 struct vgic_its *its;
620 int ret;
621
622 its = vgic_msi_to_its(kvm, msi);
623 if (IS_ERR(its))
624 return PTR_ERR(its);
625
626 mutex_lock(&its->its_lock);
627 ret = vgic_its_trigger_msi(kvm, its, msi->devid, msi->data);
628 mutex_unlock(&its->its_lock);
629
630 if (ret < 0)
631 return ret;
632
633
634
635
636
637
638 if (ret)
639 return 0;
640 else
641 return 1;
642}
643
644
645static void its_free_ite(struct kvm *kvm, struct its_ite *ite)
646{
647 list_del(&ite->ite_list);
648
649
650 if (ite->irq) {
651 if (ite->irq->hw)
652 WARN_ON(its_unmap_vlpi(ite->irq->host_irq));
653
654 vgic_put_irq(kvm, ite->irq);
655 }
656
657 kfree(ite);
658}
659
660static u64 its_cmd_mask_field(u64 *its_cmd, int word, int shift, int size)
661{
662 return (le64_to_cpu(its_cmd[word]) >> shift) & (BIT_ULL(size) - 1);
663}
664
665#define its_cmd_get_command(cmd) its_cmd_mask_field(cmd, 0, 0, 8)
666#define its_cmd_get_deviceid(cmd) its_cmd_mask_field(cmd, 0, 32, 32)
667#define its_cmd_get_size(cmd) (its_cmd_mask_field(cmd, 1, 0, 5) + 1)
668#define its_cmd_get_id(cmd) its_cmd_mask_field(cmd, 1, 0, 32)
669#define its_cmd_get_physical_id(cmd) its_cmd_mask_field(cmd, 1, 32, 32)
670#define its_cmd_get_collection(cmd) its_cmd_mask_field(cmd, 2, 0, 16)
671#define its_cmd_get_ittaddr(cmd) (its_cmd_mask_field(cmd, 2, 8, 44) << 8)
672#define its_cmd_get_target_addr(cmd) its_cmd_mask_field(cmd, 2, 16, 32)
673#define its_cmd_get_validbit(cmd) its_cmd_mask_field(cmd, 2, 63, 1)
674
675
676
677
678
679static int vgic_its_cmd_handle_discard(struct kvm *kvm, struct vgic_its *its,
680 u64 *its_cmd)
681{
682 u32 device_id = its_cmd_get_deviceid(its_cmd);
683 u32 event_id = its_cmd_get_id(its_cmd);
684 struct its_ite *ite;
685
686
687 ite = find_ite(its, device_id, event_id);
688 if (ite && ite->collection) {
689
690
691
692
693
694 its_free_ite(kvm, ite);
695 return 0;
696 }
697
698 return E_ITS_DISCARD_UNMAPPED_INTERRUPT;
699}
700
701
702
703
704
705static int vgic_its_cmd_handle_movi(struct kvm *kvm, struct vgic_its *its,
706 u64 *its_cmd)
707{
708 u32 device_id = its_cmd_get_deviceid(its_cmd);
709 u32 event_id = its_cmd_get_id(its_cmd);
710 u32 coll_id = its_cmd_get_collection(its_cmd);
711 struct kvm_vcpu *vcpu;
712 struct its_ite *ite;
713 struct its_collection *collection;
714
715 ite = find_ite(its, device_id, event_id);
716 if (!ite)
717 return E_ITS_MOVI_UNMAPPED_INTERRUPT;
718
719 if (!its_is_collection_mapped(ite->collection))
720 return E_ITS_MOVI_UNMAPPED_COLLECTION;
721
722 collection = find_collection(its, coll_id);
723 if (!its_is_collection_mapped(collection))
724 return E_ITS_MOVI_UNMAPPED_COLLECTION;
725
726 ite->collection = collection;
727 vcpu = kvm_get_vcpu(kvm, collection->target_addr);
728
729 return update_affinity(ite->irq, vcpu);
730}
731
732
733
734
735
736
737
738
739static bool vgic_its_check_id(struct vgic_its *its, u64 baser, u32 id,
740 gpa_t *eaddr)
741{
742 int l1_tbl_size = GITS_BASER_NR_PAGES(baser) * SZ_64K;
743 u64 indirect_ptr, type = GITS_BASER_TYPE(baser);
744 phys_addr_t base = GITS_BASER_ADDR_48_to_52(baser);
745 int esz = GITS_BASER_ENTRY_SIZE(baser);
746 int index, idx;
747 gfn_t gfn;
748 bool ret;
749
750 switch (type) {
751 case GITS_BASER_TYPE_DEVICE:
752 if (id >= BIT_ULL(VITS_TYPER_DEVBITS))
753 return false;
754 break;
755 case GITS_BASER_TYPE_COLLECTION:
756
757 if (id >= BIT_ULL(16))
758 return false;
759 break;
760 default:
761 return false;
762 }
763
764 if (!(baser & GITS_BASER_INDIRECT)) {
765 phys_addr_t addr;
766
767 if (id >= (l1_tbl_size / esz))
768 return false;
769
770 addr = base + id * esz;
771 gfn = addr >> PAGE_SHIFT;
772
773 if (eaddr)
774 *eaddr = addr;
775
776 goto out;
777 }
778
779
780 index = id / (SZ_64K / esz);
781 if (index >= (l1_tbl_size / sizeof(u64)))
782 return false;
783
784
785 if (kvm_read_guest_lock(its->dev->kvm,
786 base + index * sizeof(indirect_ptr),
787 &indirect_ptr, sizeof(indirect_ptr)))
788 return false;
789
790 indirect_ptr = le64_to_cpu(indirect_ptr);
791
792
793 if (!(indirect_ptr & BIT_ULL(63)))
794 return false;
795
796
797 indirect_ptr &= GENMASK_ULL(51, 16);
798
799
800 index = id % (SZ_64K / esz);
801 indirect_ptr += index * esz;
802 gfn = indirect_ptr >> PAGE_SHIFT;
803
804 if (eaddr)
805 *eaddr = indirect_ptr;
806
807out:
808 idx = srcu_read_lock(&its->dev->kvm->srcu);
809 ret = kvm_is_visible_gfn(its->dev->kvm, gfn);
810 srcu_read_unlock(&its->dev->kvm->srcu, idx);
811 return ret;
812}
813
814static int vgic_its_alloc_collection(struct vgic_its *its,
815 struct its_collection **colp,
816 u32 coll_id)
817{
818 struct its_collection *collection;
819
820 if (!vgic_its_check_id(its, its->baser_coll_table, coll_id, NULL))
821 return E_ITS_MAPC_COLLECTION_OOR;
822
823 collection = kzalloc(sizeof(*collection), GFP_KERNEL);
824 if (!collection)
825 return -ENOMEM;
826
827 collection->collection_id = coll_id;
828 collection->target_addr = COLLECTION_NOT_MAPPED;
829
830 list_add_tail(&collection->coll_list, &its->collection_list);
831 *colp = collection;
832
833 return 0;
834}
835
836static void vgic_its_free_collection(struct vgic_its *its, u32 coll_id)
837{
838 struct its_collection *collection;
839 struct its_device *device;
840 struct its_ite *ite;
841
842
843
844
845
846
847 collection = find_collection(its, coll_id);
848 if (!collection)
849 return;
850
851 for_each_lpi_its(device, ite, its)
852 if (ite->collection &&
853 ite->collection->collection_id == coll_id)
854 ite->collection = NULL;
855
856 list_del(&collection->coll_list);
857 kfree(collection);
858}
859
860
861static struct its_ite *vgic_its_alloc_ite(struct its_device *device,
862 struct its_collection *collection,
863 u32 event_id)
864{
865 struct its_ite *ite;
866
867 ite = kzalloc(sizeof(*ite), GFP_KERNEL);
868 if (!ite)
869 return ERR_PTR(-ENOMEM);
870
871 ite->event_id = event_id;
872 ite->collection = collection;
873
874 list_add_tail(&ite->ite_list, &device->itt_head);
875 return ite;
876}
877
878
879
880
881
882static int vgic_its_cmd_handle_mapi(struct kvm *kvm, struct vgic_its *its,
883 u64 *its_cmd)
884{
885 u32 device_id = its_cmd_get_deviceid(its_cmd);
886 u32 event_id = its_cmd_get_id(its_cmd);
887 u32 coll_id = its_cmd_get_collection(its_cmd);
888 struct its_ite *ite;
889 struct kvm_vcpu *vcpu = NULL;
890 struct its_device *device;
891 struct its_collection *collection, *new_coll = NULL;
892 struct vgic_irq *irq;
893 int lpi_nr;
894
895 device = find_its_device(its, device_id);
896 if (!device)
897 return E_ITS_MAPTI_UNMAPPED_DEVICE;
898
899 if (event_id >= BIT_ULL(device->num_eventid_bits))
900 return E_ITS_MAPTI_ID_OOR;
901
902 if (its_cmd_get_command(its_cmd) == GITS_CMD_MAPTI)
903 lpi_nr = its_cmd_get_physical_id(its_cmd);
904 else
905 lpi_nr = event_id;
906 if (lpi_nr < GIC_LPI_OFFSET ||
907 lpi_nr >= max_lpis_propbaser(kvm->arch.vgic.propbaser))
908 return E_ITS_MAPTI_PHYSICALID_OOR;
909
910
911 if (find_ite(its, device_id, event_id))
912 return 0;
913
914 collection = find_collection(its, coll_id);
915 if (!collection) {
916 int ret = vgic_its_alloc_collection(its, &collection, coll_id);
917 if (ret)
918 return ret;
919 new_coll = collection;
920 }
921
922 ite = vgic_its_alloc_ite(device, collection, event_id);
923 if (IS_ERR(ite)) {
924 if (new_coll)
925 vgic_its_free_collection(its, coll_id);
926 return PTR_ERR(ite);
927 }
928
929 if (its_is_collection_mapped(collection))
930 vcpu = kvm_get_vcpu(kvm, collection->target_addr);
931
932 irq = vgic_add_lpi(kvm, lpi_nr, vcpu);
933 if (IS_ERR(irq)) {
934 if (new_coll)
935 vgic_its_free_collection(its, coll_id);
936 its_free_ite(kvm, ite);
937 return PTR_ERR(irq);
938 }
939 ite->irq = irq;
940
941 return 0;
942}
943
944
945static void vgic_its_free_device(struct kvm *kvm, struct its_device *device)
946{
947 struct its_ite *ite, *temp;
948
949
950
951
952
953
954 list_for_each_entry_safe(ite, temp, &device->itt_head, ite_list)
955 its_free_ite(kvm, ite);
956
957 list_del(&device->dev_list);
958 kfree(device);
959}
960
961
962static void vgic_its_free_device_list(struct kvm *kvm, struct vgic_its *its)
963{
964 struct its_device *cur, *temp;
965
966 list_for_each_entry_safe(cur, temp, &its->device_list, dev_list)
967 vgic_its_free_device(kvm, cur);
968}
969
970
971static void vgic_its_free_collection_list(struct kvm *kvm, struct vgic_its *its)
972{
973 struct its_collection *cur, *temp;
974
975 list_for_each_entry_safe(cur, temp, &its->collection_list, coll_list)
976 vgic_its_free_collection(its, cur->collection_id);
977}
978
979
980static struct its_device *vgic_its_alloc_device(struct vgic_its *its,
981 u32 device_id, gpa_t itt_addr,
982 u8 num_eventid_bits)
983{
984 struct its_device *device;
985
986 device = kzalloc(sizeof(*device), GFP_KERNEL);
987 if (!device)
988 return ERR_PTR(-ENOMEM);
989
990 device->device_id = device_id;
991 device->itt_addr = itt_addr;
992 device->num_eventid_bits = num_eventid_bits;
993 INIT_LIST_HEAD(&device->itt_head);
994
995 list_add_tail(&device->dev_list, &its->device_list);
996 return device;
997}
998
999
1000
1001
1002
1003static int vgic_its_cmd_handle_mapd(struct kvm *kvm, struct vgic_its *its,
1004 u64 *its_cmd)
1005{
1006 u32 device_id = its_cmd_get_deviceid(its_cmd);
1007 bool valid = its_cmd_get_validbit(its_cmd);
1008 u8 num_eventid_bits = its_cmd_get_size(its_cmd);
1009 gpa_t itt_addr = its_cmd_get_ittaddr(its_cmd);
1010 struct its_device *device;
1011
1012 if (!vgic_its_check_id(its, its->baser_device_table, device_id, NULL))
1013 return E_ITS_MAPD_DEVICE_OOR;
1014
1015 if (valid && num_eventid_bits > VITS_TYPER_IDBITS)
1016 return E_ITS_MAPD_ITTSIZE_OOR;
1017
1018 device = find_its_device(its, device_id);
1019
1020
1021
1022
1023
1024
1025 if (device)
1026 vgic_its_free_device(kvm, device);
1027
1028
1029
1030
1031
1032 if (!valid)
1033 return 0;
1034
1035 device = vgic_its_alloc_device(its, device_id, itt_addr,
1036 num_eventid_bits);
1037
1038 return PTR_ERR_OR_ZERO(device);
1039}
1040
1041
1042
1043
1044
1045static int vgic_its_cmd_handle_mapc(struct kvm *kvm, struct vgic_its *its,
1046 u64 *its_cmd)
1047{
1048 u16 coll_id;
1049 u32 target_addr;
1050 struct its_collection *collection;
1051 bool valid;
1052
1053 valid = its_cmd_get_validbit(its_cmd);
1054 coll_id = its_cmd_get_collection(its_cmd);
1055 target_addr = its_cmd_get_target_addr(its_cmd);
1056
1057 if (target_addr >= atomic_read(&kvm->online_vcpus))
1058 return E_ITS_MAPC_PROCNUM_OOR;
1059
1060 if (!valid) {
1061 vgic_its_free_collection(its, coll_id);
1062 } else {
1063 collection = find_collection(its, coll_id);
1064
1065 if (!collection) {
1066 int ret;
1067
1068 ret = vgic_its_alloc_collection(its, &collection,
1069 coll_id);
1070 if (ret)
1071 return ret;
1072 collection->target_addr = target_addr;
1073 } else {
1074 collection->target_addr = target_addr;
1075 update_affinity_collection(kvm, its, collection);
1076 }
1077 }
1078
1079 return 0;
1080}
1081
1082
1083
1084
1085
1086static int vgic_its_cmd_handle_clear(struct kvm *kvm, struct vgic_its *its,
1087 u64 *its_cmd)
1088{
1089 u32 device_id = its_cmd_get_deviceid(its_cmd);
1090 u32 event_id = its_cmd_get_id(its_cmd);
1091 struct its_ite *ite;
1092
1093
1094 ite = find_ite(its, device_id, event_id);
1095 if (!ite)
1096 return E_ITS_CLEAR_UNMAPPED_INTERRUPT;
1097
1098 ite->irq->pending_latch = false;
1099
1100 if (ite->irq->hw)
1101 return irq_set_irqchip_state(ite->irq->host_irq,
1102 IRQCHIP_STATE_PENDING, false);
1103
1104 return 0;
1105}
1106
1107
1108
1109
1110
1111static int vgic_its_cmd_handle_inv(struct kvm *kvm, struct vgic_its *its,
1112 u64 *its_cmd)
1113{
1114 u32 device_id = its_cmd_get_deviceid(its_cmd);
1115 u32 event_id = its_cmd_get_id(its_cmd);
1116 struct its_ite *ite;
1117
1118
1119 ite = find_ite(its, device_id, event_id);
1120 if (!ite)
1121 return E_ITS_INV_UNMAPPED_INTERRUPT;
1122
1123 return update_lpi_config(kvm, ite->irq, NULL, true);
1124}
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134static int vgic_its_cmd_handle_invall(struct kvm *kvm, struct vgic_its *its,
1135 u64 *its_cmd)
1136{
1137 u32 coll_id = its_cmd_get_collection(its_cmd);
1138 struct its_collection *collection;
1139 struct kvm_vcpu *vcpu;
1140 struct vgic_irq *irq;
1141 u32 *intids;
1142 int irq_count, i;
1143
1144 collection = find_collection(its, coll_id);
1145 if (!its_is_collection_mapped(collection))
1146 return E_ITS_INVALL_UNMAPPED_COLLECTION;
1147
1148 vcpu = kvm_get_vcpu(kvm, collection->target_addr);
1149
1150 irq_count = vgic_copy_lpi_list(kvm, vcpu, &intids);
1151 if (irq_count < 0)
1152 return irq_count;
1153
1154 for (i = 0; i < irq_count; i++) {
1155 irq = vgic_get_irq(kvm, NULL, intids[i]);
1156 if (!irq)
1157 continue;
1158 update_lpi_config(kvm, irq, vcpu, false);
1159 vgic_put_irq(kvm, irq);
1160 }
1161
1162 kfree(intids);
1163
1164 if (vcpu->arch.vgic_cpu.vgic_v3.its_vpe.its_vm)
1165 its_invall_vpe(&vcpu->arch.vgic_cpu.vgic_v3.its_vpe);
1166
1167 return 0;
1168}
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178static int vgic_its_cmd_handle_movall(struct kvm *kvm, struct vgic_its *its,
1179 u64 *its_cmd)
1180{
1181 u32 target1_addr = its_cmd_get_target_addr(its_cmd);
1182 u32 target2_addr = its_cmd_mask_field(its_cmd, 3, 16, 32);
1183 struct kvm_vcpu *vcpu1, *vcpu2;
1184 struct vgic_irq *irq;
1185 u32 *intids;
1186 int irq_count, i;
1187
1188 if (target1_addr >= atomic_read(&kvm->online_vcpus) ||
1189 target2_addr >= atomic_read(&kvm->online_vcpus))
1190 return E_ITS_MOVALL_PROCNUM_OOR;
1191
1192 if (target1_addr == target2_addr)
1193 return 0;
1194
1195 vcpu1 = kvm_get_vcpu(kvm, target1_addr);
1196 vcpu2 = kvm_get_vcpu(kvm, target2_addr);
1197
1198 irq_count = vgic_copy_lpi_list(kvm, vcpu1, &intids);
1199 if (irq_count < 0)
1200 return irq_count;
1201
1202 for (i = 0; i < irq_count; i++) {
1203 irq = vgic_get_irq(kvm, NULL, intids[i]);
1204
1205 update_affinity(irq, vcpu2);
1206
1207 vgic_put_irq(kvm, irq);
1208 }
1209
1210 kfree(intids);
1211 return 0;
1212}
1213
1214
1215
1216
1217
1218static int vgic_its_cmd_handle_int(struct kvm *kvm, struct vgic_its *its,
1219 u64 *its_cmd)
1220{
1221 u32 msi_data = its_cmd_get_id(its_cmd);
1222 u64 msi_devid = its_cmd_get_deviceid(its_cmd);
1223
1224 return vgic_its_trigger_msi(kvm, its, msi_devid, msi_data);
1225}
1226
1227
1228
1229
1230
1231static int vgic_its_handle_command(struct kvm *kvm, struct vgic_its *its,
1232 u64 *its_cmd)
1233{
1234 int ret = -ENODEV;
1235
1236 mutex_lock(&its->its_lock);
1237 switch (its_cmd_get_command(its_cmd)) {
1238 case GITS_CMD_MAPD:
1239 ret = vgic_its_cmd_handle_mapd(kvm, its, its_cmd);
1240 break;
1241 case GITS_CMD_MAPC:
1242 ret = vgic_its_cmd_handle_mapc(kvm, its, its_cmd);
1243 break;
1244 case GITS_CMD_MAPI:
1245 ret = vgic_its_cmd_handle_mapi(kvm, its, its_cmd);
1246 break;
1247 case GITS_CMD_MAPTI:
1248 ret = vgic_its_cmd_handle_mapi(kvm, its, its_cmd);
1249 break;
1250 case GITS_CMD_MOVI:
1251 ret = vgic_its_cmd_handle_movi(kvm, its, its_cmd);
1252 break;
1253 case GITS_CMD_DISCARD:
1254 ret = vgic_its_cmd_handle_discard(kvm, its, its_cmd);
1255 break;
1256 case GITS_CMD_CLEAR:
1257 ret = vgic_its_cmd_handle_clear(kvm, its, its_cmd);
1258 break;
1259 case GITS_CMD_MOVALL:
1260 ret = vgic_its_cmd_handle_movall(kvm, its, its_cmd);
1261 break;
1262 case GITS_CMD_INT:
1263 ret = vgic_its_cmd_handle_int(kvm, its, its_cmd);
1264 break;
1265 case GITS_CMD_INV:
1266 ret = vgic_its_cmd_handle_inv(kvm, its, its_cmd);
1267 break;
1268 case GITS_CMD_INVALL:
1269 ret = vgic_its_cmd_handle_invall(kvm, its, its_cmd);
1270 break;
1271 case GITS_CMD_SYNC:
1272
1273 ret = 0;
1274 break;
1275 }
1276 mutex_unlock(&its->its_lock);
1277
1278 return ret;
1279}
1280
1281static u64 vgic_sanitise_its_baser(u64 reg)
1282{
1283 reg = vgic_sanitise_field(reg, GITS_BASER_SHAREABILITY_MASK,
1284 GITS_BASER_SHAREABILITY_SHIFT,
1285 vgic_sanitise_shareability);
1286 reg = vgic_sanitise_field(reg, GITS_BASER_INNER_CACHEABILITY_MASK,
1287 GITS_BASER_INNER_CACHEABILITY_SHIFT,
1288 vgic_sanitise_inner_cacheability);
1289 reg = vgic_sanitise_field(reg, GITS_BASER_OUTER_CACHEABILITY_MASK,
1290 GITS_BASER_OUTER_CACHEABILITY_SHIFT,
1291 vgic_sanitise_outer_cacheability);
1292
1293
1294 reg = (reg & ~GITS_BASER_PAGE_SIZE_MASK) | GITS_BASER_PAGE_SIZE_64K;
1295
1296 return reg;
1297}
1298
1299static u64 vgic_sanitise_its_cbaser(u64 reg)
1300{
1301 reg = vgic_sanitise_field(reg, GITS_CBASER_SHAREABILITY_MASK,
1302 GITS_CBASER_SHAREABILITY_SHIFT,
1303 vgic_sanitise_shareability);
1304 reg = vgic_sanitise_field(reg, GITS_CBASER_INNER_CACHEABILITY_MASK,
1305 GITS_CBASER_INNER_CACHEABILITY_SHIFT,
1306 vgic_sanitise_inner_cacheability);
1307 reg = vgic_sanitise_field(reg, GITS_CBASER_OUTER_CACHEABILITY_MASK,
1308 GITS_CBASER_OUTER_CACHEABILITY_SHIFT,
1309 vgic_sanitise_outer_cacheability);
1310
1311
1312 reg &= ~GENMASK_ULL(15, 12);
1313
1314 return reg;
1315}
1316
1317static unsigned long vgic_mmio_read_its_cbaser(struct kvm *kvm,
1318 struct vgic_its *its,
1319 gpa_t addr, unsigned int len)
1320{
1321 return extract_bytes(its->cbaser, addr & 7, len);
1322}
1323
1324static void vgic_mmio_write_its_cbaser(struct kvm *kvm, struct vgic_its *its,
1325 gpa_t addr, unsigned int len,
1326 unsigned long val)
1327{
1328
1329 if (its->enabled)
1330 return;
1331
1332 mutex_lock(&its->cmd_lock);
1333 its->cbaser = update_64bit_reg(its->cbaser, addr & 7, len, val);
1334 its->cbaser = vgic_sanitise_its_cbaser(its->cbaser);
1335 its->creadr = 0;
1336
1337
1338
1339
1340 its->cwriter = its->creadr;
1341 mutex_unlock(&its->cmd_lock);
1342}
1343
1344#define ITS_CMD_BUFFER_SIZE(baser) ((((baser) & 0xff) + 1) << 12)
1345#define ITS_CMD_SIZE 32
1346#define ITS_CMD_OFFSET(reg) ((reg) & GENMASK(19, 5))
1347
1348
1349static void vgic_its_process_commands(struct kvm *kvm, struct vgic_its *its)
1350{
1351 gpa_t cbaser;
1352 u64 cmd_buf[4];
1353
1354
1355 if (!its->enabled)
1356 return;
1357
1358 cbaser = GITS_CBASER_ADDRESS(its->cbaser);
1359
1360 while (its->cwriter != its->creadr) {
1361 int ret = kvm_read_guest_lock(kvm, cbaser + its->creadr,
1362 cmd_buf, ITS_CMD_SIZE);
1363
1364
1365
1366
1367
1368
1369
1370 if (!ret)
1371 vgic_its_handle_command(kvm, its, cmd_buf);
1372
1373 its->creadr += ITS_CMD_SIZE;
1374 if (its->creadr == ITS_CMD_BUFFER_SIZE(its->cbaser))
1375 its->creadr = 0;
1376 }
1377}
1378
1379
1380
1381
1382
1383
1384
1385static void vgic_mmio_write_its_cwriter(struct kvm *kvm, struct vgic_its *its,
1386 gpa_t addr, unsigned int len,
1387 unsigned long val)
1388{
1389 u64 reg;
1390
1391 if (!its)
1392 return;
1393
1394 mutex_lock(&its->cmd_lock);
1395
1396 reg = update_64bit_reg(its->cwriter, addr & 7, len, val);
1397 reg = ITS_CMD_OFFSET(reg);
1398 if (reg >= ITS_CMD_BUFFER_SIZE(its->cbaser)) {
1399 mutex_unlock(&its->cmd_lock);
1400 return;
1401 }
1402 its->cwriter = reg;
1403
1404 vgic_its_process_commands(kvm, its);
1405
1406 mutex_unlock(&its->cmd_lock);
1407}
1408
1409static unsigned long vgic_mmio_read_its_cwriter(struct kvm *kvm,
1410 struct vgic_its *its,
1411 gpa_t addr, unsigned int len)
1412{
1413 return extract_bytes(its->cwriter, addr & 0x7, len);
1414}
1415
1416static unsigned long vgic_mmio_read_its_creadr(struct kvm *kvm,
1417 struct vgic_its *its,
1418 gpa_t addr, unsigned int len)
1419{
1420 return extract_bytes(its->creadr, addr & 0x7, len);
1421}
1422
1423static int vgic_mmio_uaccess_write_its_creadr(struct kvm *kvm,
1424 struct vgic_its *its,
1425 gpa_t addr, unsigned int len,
1426 unsigned long val)
1427{
1428 u32 cmd_offset;
1429 int ret = 0;
1430
1431 mutex_lock(&its->cmd_lock);
1432
1433 if (its->enabled) {
1434 ret = -EBUSY;
1435 goto out;
1436 }
1437
1438 cmd_offset = ITS_CMD_OFFSET(val);
1439 if (cmd_offset >= ITS_CMD_BUFFER_SIZE(its->cbaser)) {
1440 ret = -EINVAL;
1441 goto out;
1442 }
1443
1444 its->creadr = cmd_offset;
1445out:
1446 mutex_unlock(&its->cmd_lock);
1447 return ret;
1448}
1449
1450#define BASER_INDEX(addr) (((addr) / sizeof(u64)) & 0x7)
1451static unsigned long vgic_mmio_read_its_baser(struct kvm *kvm,
1452 struct vgic_its *its,
1453 gpa_t addr, unsigned int len)
1454{
1455 u64 reg;
1456
1457 switch (BASER_INDEX(addr)) {
1458 case 0:
1459 reg = its->baser_device_table;
1460 break;
1461 case 1:
1462 reg = its->baser_coll_table;
1463 break;
1464 default:
1465 reg = 0;
1466 break;
1467 }
1468
1469 return extract_bytes(reg, addr & 7, len);
1470}
1471
1472#define GITS_BASER_RO_MASK (GENMASK_ULL(52, 48) | GENMASK_ULL(58, 56))
1473static void vgic_mmio_write_its_baser(struct kvm *kvm,
1474 struct vgic_its *its,
1475 gpa_t addr, unsigned int len,
1476 unsigned long val)
1477{
1478 const struct vgic_its_abi *abi = vgic_its_get_abi(its);
1479 u64 entry_size, table_type;
1480 u64 reg, *regptr, clearbits = 0;
1481
1482
1483 if (its->enabled)
1484 return;
1485
1486 switch (BASER_INDEX(addr)) {
1487 case 0:
1488 regptr = &its->baser_device_table;
1489 entry_size = abi->dte_esz;
1490 table_type = GITS_BASER_TYPE_DEVICE;
1491 break;
1492 case 1:
1493 regptr = &its->baser_coll_table;
1494 entry_size = abi->cte_esz;
1495 table_type = GITS_BASER_TYPE_COLLECTION;
1496 clearbits = GITS_BASER_INDIRECT;
1497 break;
1498 default:
1499 return;
1500 }
1501
1502 reg = update_64bit_reg(*regptr, addr & 7, len, val);
1503 reg &= ~GITS_BASER_RO_MASK;
1504 reg &= ~clearbits;
1505
1506 reg |= (entry_size - 1) << GITS_BASER_ENTRY_SIZE_SHIFT;
1507 reg |= table_type << GITS_BASER_TYPE_SHIFT;
1508 reg = vgic_sanitise_its_baser(reg);
1509
1510 *regptr = reg;
1511
1512 if (!(reg & GITS_BASER_VALID)) {
1513
1514 mutex_lock(&its->its_lock);
1515 switch (table_type) {
1516 case GITS_BASER_TYPE_DEVICE:
1517 vgic_its_free_device_list(kvm, its);
1518 break;
1519 case GITS_BASER_TYPE_COLLECTION:
1520 vgic_its_free_collection_list(kvm, its);
1521 break;
1522 }
1523 mutex_unlock(&its->its_lock);
1524 }
1525}
1526
1527static unsigned long vgic_mmio_read_its_ctlr(struct kvm *vcpu,
1528 struct vgic_its *its,
1529 gpa_t addr, unsigned int len)
1530{
1531 u32 reg = 0;
1532
1533 mutex_lock(&its->cmd_lock);
1534 if (its->creadr == its->cwriter)
1535 reg |= GITS_CTLR_QUIESCENT;
1536 if (its->enabled)
1537 reg |= GITS_CTLR_ENABLE;
1538 mutex_unlock(&its->cmd_lock);
1539
1540 return reg;
1541}
1542
1543static void vgic_mmio_write_its_ctlr(struct kvm *kvm, struct vgic_its *its,
1544 gpa_t addr, unsigned int len,
1545 unsigned long val)
1546{
1547 mutex_lock(&its->cmd_lock);
1548
1549
1550
1551
1552
1553 if (!its->enabled && (val & GITS_CTLR_ENABLE) &&
1554 (!(its->baser_device_table & GITS_BASER_VALID) ||
1555 !(its->baser_coll_table & GITS_BASER_VALID) ||
1556 !(its->cbaser & GITS_CBASER_VALID)))
1557 goto out;
1558
1559 its->enabled = !!(val & GITS_CTLR_ENABLE);
1560
1561
1562
1563
1564
1565 vgic_its_process_commands(kvm, its);
1566
1567out:
1568 mutex_unlock(&its->cmd_lock);
1569}
1570
1571#define REGISTER_ITS_DESC(off, rd, wr, length, acc) \
1572{ \
1573 .reg_offset = off, \
1574 .len = length, \
1575 .access_flags = acc, \
1576 .its_read = rd, \
1577 .its_write = wr, \
1578}
1579
1580#define REGISTER_ITS_DESC_UACCESS(off, rd, wr, uwr, length, acc)\
1581{ \
1582 .reg_offset = off, \
1583 .len = length, \
1584 .access_flags = acc, \
1585 .its_read = rd, \
1586 .its_write = wr, \
1587 .uaccess_its_write = uwr, \
1588}
1589
1590static void its_mmio_write_wi(struct kvm *kvm, struct vgic_its *its,
1591 gpa_t addr, unsigned int len, unsigned long val)
1592{
1593
1594}
1595
1596static struct vgic_register_region its_registers[] = {
1597 REGISTER_ITS_DESC(GITS_CTLR,
1598 vgic_mmio_read_its_ctlr, vgic_mmio_write_its_ctlr, 4,
1599 VGIC_ACCESS_32bit),
1600 REGISTER_ITS_DESC_UACCESS(GITS_IIDR,
1601 vgic_mmio_read_its_iidr, its_mmio_write_wi,
1602 vgic_mmio_uaccess_write_its_iidr, 4,
1603 VGIC_ACCESS_32bit),
1604 REGISTER_ITS_DESC(GITS_TYPER,
1605 vgic_mmio_read_its_typer, its_mmio_write_wi, 8,
1606 VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
1607 REGISTER_ITS_DESC(GITS_CBASER,
1608 vgic_mmio_read_its_cbaser, vgic_mmio_write_its_cbaser, 8,
1609 VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
1610 REGISTER_ITS_DESC(GITS_CWRITER,
1611 vgic_mmio_read_its_cwriter, vgic_mmio_write_its_cwriter, 8,
1612 VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
1613 REGISTER_ITS_DESC_UACCESS(GITS_CREADR,
1614 vgic_mmio_read_its_creadr, its_mmio_write_wi,
1615 vgic_mmio_uaccess_write_its_creadr, 8,
1616 VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
1617 REGISTER_ITS_DESC(GITS_BASER,
1618 vgic_mmio_read_its_baser, vgic_mmio_write_its_baser, 0x40,
1619 VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
1620 REGISTER_ITS_DESC(GITS_IDREGS_BASE,
1621 vgic_mmio_read_its_idregs, its_mmio_write_wi, 0x30,
1622 VGIC_ACCESS_32bit),
1623};
1624
1625
1626void vgic_enable_lpis(struct kvm_vcpu *vcpu)
1627{
1628 if (!(vcpu->arch.vgic_cpu.pendbaser & GICR_PENDBASER_PTZ))
1629 its_sync_lpi_pending_table(vcpu);
1630}
1631
1632static int vgic_register_its_iodev(struct kvm *kvm, struct vgic_its *its,
1633 u64 addr)
1634{
1635 struct vgic_io_device *iodev = &its->iodev;
1636 int ret;
1637
1638 mutex_lock(&kvm->slots_lock);
1639 if (!IS_VGIC_ADDR_UNDEF(its->vgic_its_base)) {
1640 ret = -EBUSY;
1641 goto out;
1642 }
1643
1644 its->vgic_its_base = addr;
1645 iodev->regions = its_registers;
1646 iodev->nr_regions = ARRAY_SIZE(its_registers);
1647 kvm_iodevice_init(&iodev->dev, &kvm_io_gic_ops);
1648
1649 iodev->base_addr = its->vgic_its_base;
1650 iodev->iodev_type = IODEV_ITS;
1651 iodev->its = its;
1652 ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, iodev->base_addr,
1653 KVM_VGIC_V3_ITS_SIZE, &iodev->dev);
1654out:
1655 mutex_unlock(&kvm->slots_lock);
1656
1657 return ret;
1658}
1659
1660#define INITIAL_BASER_VALUE \
1661 (GIC_BASER_CACHEABILITY(GITS_BASER, INNER, RaWb) | \
1662 GIC_BASER_CACHEABILITY(GITS_BASER, OUTER, SameAsInner) | \
1663 GIC_BASER_SHAREABILITY(GITS_BASER, InnerShareable) | \
1664 GITS_BASER_PAGE_SIZE_64K)
1665
1666#define INITIAL_PROPBASER_VALUE \
1667 (GIC_BASER_CACHEABILITY(GICR_PROPBASER, INNER, RaWb) | \
1668 GIC_BASER_CACHEABILITY(GICR_PROPBASER, OUTER, SameAsInner) | \
1669 GIC_BASER_SHAREABILITY(GICR_PROPBASER, InnerShareable))
1670
1671static int vgic_its_create(struct kvm_device *dev, u32 type)
1672{
1673 struct vgic_its *its;
1674
1675 if (type != KVM_DEV_TYPE_ARM_VGIC_ITS)
1676 return -ENODEV;
1677
1678 its = kzalloc(sizeof(struct vgic_its), GFP_KERNEL);
1679 if (!its)
1680 return -ENOMEM;
1681
1682 if (vgic_initialized(dev->kvm)) {
1683 int ret = vgic_v4_init(dev->kvm);
1684 if (ret < 0) {
1685 kfree(its);
1686 return ret;
1687 }
1688 }
1689
1690 mutex_init(&its->its_lock);
1691 mutex_init(&its->cmd_lock);
1692
1693 its->vgic_its_base = VGIC_ADDR_UNDEF;
1694
1695 INIT_LIST_HEAD(&its->device_list);
1696 INIT_LIST_HEAD(&its->collection_list);
1697
1698 dev->kvm->arch.vgic.msis_require_devid = true;
1699 dev->kvm->arch.vgic.has_its = true;
1700 its->enabled = false;
1701 its->dev = dev;
1702
1703 its->baser_device_table = INITIAL_BASER_VALUE |
1704 ((u64)GITS_BASER_TYPE_DEVICE << GITS_BASER_TYPE_SHIFT);
1705 its->baser_coll_table = INITIAL_BASER_VALUE |
1706 ((u64)GITS_BASER_TYPE_COLLECTION << GITS_BASER_TYPE_SHIFT);
1707 dev->kvm->arch.vgic.propbaser = INITIAL_PROPBASER_VALUE;
1708
1709 dev->private = its;
1710
1711 return vgic_its_set_abi(its, NR_ITS_ABIS - 1);
1712}
1713
1714static void vgic_its_destroy(struct kvm_device *kvm_dev)
1715{
1716 struct kvm *kvm = kvm_dev->kvm;
1717 struct vgic_its *its = kvm_dev->private;
1718
1719 mutex_lock(&its->its_lock);
1720
1721 vgic_its_free_device_list(kvm, its);
1722 vgic_its_free_collection_list(kvm, its);
1723
1724 mutex_unlock(&its->its_lock);
1725 kfree(its);
1726 kfree(kvm_dev);
1727}
1728
1729static int vgic_its_has_attr_regs(struct kvm_device *dev,
1730 struct kvm_device_attr *attr)
1731{
1732 const struct vgic_register_region *region;
1733 gpa_t offset = attr->attr;
1734 int align;
1735
1736 align = (offset < GITS_TYPER) || (offset >= GITS_PIDR4) ? 0x3 : 0x7;
1737
1738 if (offset & align)
1739 return -EINVAL;
1740
1741 region = vgic_find_mmio_region(its_registers,
1742 ARRAY_SIZE(its_registers),
1743 offset);
1744 if (!region)
1745 return -ENXIO;
1746
1747 return 0;
1748}
1749
1750static int vgic_its_attr_regs_access(struct kvm_device *dev,
1751 struct kvm_device_attr *attr,
1752 u64 *reg, bool is_write)
1753{
1754 const struct vgic_register_region *region;
1755 struct vgic_its *its;
1756 gpa_t addr, offset;
1757 unsigned int len;
1758 int align, ret = 0;
1759
1760 its = dev->private;
1761 offset = attr->attr;
1762
1763
1764
1765
1766
1767
1768
1769
1770 if ((offset < GITS_TYPER) || (offset >= GITS_PIDR4))
1771 align = 0x3;
1772 else
1773 align = 0x7;
1774
1775 if (offset & align)
1776 return -EINVAL;
1777
1778 mutex_lock(&dev->kvm->lock);
1779
1780 if (IS_VGIC_ADDR_UNDEF(its->vgic_its_base)) {
1781 ret = -ENXIO;
1782 goto out;
1783 }
1784
1785 region = vgic_find_mmio_region(its_registers,
1786 ARRAY_SIZE(its_registers),
1787 offset);
1788 if (!region) {
1789 ret = -ENXIO;
1790 goto out;
1791 }
1792
1793 if (!lock_all_vcpus(dev->kvm)) {
1794 ret = -EBUSY;
1795 goto out;
1796 }
1797
1798 addr = its->vgic_its_base + offset;
1799
1800 len = region->access_flags & VGIC_ACCESS_64bit ? 8 : 4;
1801
1802 if (is_write) {
1803 if (region->uaccess_its_write)
1804 ret = region->uaccess_its_write(dev->kvm, its, addr,
1805 len, *reg);
1806 else
1807 region->its_write(dev->kvm, its, addr, len, *reg);
1808 } else {
1809 *reg = region->its_read(dev->kvm, its, addr, len);
1810 }
1811 unlock_all_vcpus(dev->kvm);
1812out:
1813 mutex_unlock(&dev->kvm->lock);
1814 return ret;
1815}
1816
1817static u32 compute_next_devid_offset(struct list_head *h,
1818 struct its_device *dev)
1819{
1820 struct its_device *next;
1821 u32 next_offset;
1822
1823 if (list_is_last(&dev->dev_list, h))
1824 return 0;
1825 next = list_next_entry(dev, dev_list);
1826 next_offset = next->device_id - dev->device_id;
1827
1828 return min_t(u32, next_offset, VITS_DTE_MAX_DEVID_OFFSET);
1829}
1830
1831static u32 compute_next_eventid_offset(struct list_head *h, struct its_ite *ite)
1832{
1833 struct its_ite *next;
1834 u32 next_offset;
1835
1836 if (list_is_last(&ite->ite_list, h))
1837 return 0;
1838 next = list_next_entry(ite, ite_list);
1839 next_offset = next->event_id - ite->event_id;
1840
1841 return min_t(u32, next_offset, VITS_ITE_MAX_EVENTID_OFFSET);
1842}
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854typedef int (*entry_fn_t)(struct vgic_its *its, u32 id, void *entry,
1855 void *opaque);
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872static int scan_its_table(struct vgic_its *its, gpa_t base, int size, u32 esz,
1873 int start_id, entry_fn_t fn, void *opaque)
1874{
1875 struct kvm *kvm = its->dev->kvm;
1876 unsigned long len = size;
1877 int id = start_id;
1878 gpa_t gpa = base;
1879 char entry[ESZ_MAX];
1880 int ret;
1881
1882 memset(entry, 0, esz);
1883
1884 while (len > 0) {
1885 int next_offset;
1886 size_t byte_offset;
1887
1888 ret = kvm_read_guest_lock(kvm, gpa, entry, esz);
1889 if (ret)
1890 return ret;
1891
1892 next_offset = fn(its, id, entry, opaque);
1893 if (next_offset <= 0)
1894 return next_offset;
1895
1896 byte_offset = next_offset * esz;
1897 id += next_offset;
1898 gpa += byte_offset;
1899 len -= byte_offset;
1900 }
1901 return 1;
1902}
1903
1904
1905
1906
1907static int vgic_its_save_ite(struct vgic_its *its, struct its_device *dev,
1908 struct its_ite *ite, gpa_t gpa, int ite_esz)
1909{
1910 struct kvm *kvm = its->dev->kvm;
1911 u32 next_offset;
1912 u64 val;
1913
1914 next_offset = compute_next_eventid_offset(&dev->itt_head, ite);
1915 val = ((u64)next_offset << KVM_ITS_ITE_NEXT_SHIFT) |
1916 ((u64)ite->irq->intid << KVM_ITS_ITE_PINTID_SHIFT) |
1917 ite->collection->collection_id;
1918 val = cpu_to_le64(val);
1919 return kvm_write_guest_lock(kvm, gpa, &val, ite_esz);
1920}
1921
1922
1923
1924
1925
1926
1927
1928static int vgic_its_restore_ite(struct vgic_its *its, u32 event_id,
1929 void *ptr, void *opaque)
1930{
1931 struct its_device *dev = (struct its_device *)opaque;
1932 struct its_collection *collection;
1933 struct kvm *kvm = its->dev->kvm;
1934 struct kvm_vcpu *vcpu = NULL;
1935 u64 val;
1936 u64 *p = (u64 *)ptr;
1937 struct vgic_irq *irq;
1938 u32 coll_id, lpi_id;
1939 struct its_ite *ite;
1940 u32 offset;
1941
1942 val = *p;
1943
1944 val = le64_to_cpu(val);
1945
1946 coll_id = val & KVM_ITS_ITE_ICID_MASK;
1947 lpi_id = (val & KVM_ITS_ITE_PINTID_MASK) >> KVM_ITS_ITE_PINTID_SHIFT;
1948
1949 if (!lpi_id)
1950 return 1;
1951
1952 if (lpi_id < VGIC_MIN_LPI)
1953 return -EINVAL;
1954
1955 offset = val >> KVM_ITS_ITE_NEXT_SHIFT;
1956 if (event_id + offset >= BIT_ULL(dev->num_eventid_bits))
1957 return -EINVAL;
1958
1959 collection = find_collection(its, coll_id);
1960 if (!collection)
1961 return -EINVAL;
1962
1963 ite = vgic_its_alloc_ite(dev, collection, event_id);
1964 if (IS_ERR(ite))
1965 return PTR_ERR(ite);
1966
1967 if (its_is_collection_mapped(collection))
1968 vcpu = kvm_get_vcpu(kvm, collection->target_addr);
1969
1970 irq = vgic_add_lpi(kvm, lpi_id, vcpu);
1971 if (IS_ERR(irq))
1972 return PTR_ERR(irq);
1973 ite->irq = irq;
1974
1975 return offset;
1976}
1977
1978static int vgic_its_ite_cmp(void *priv, struct list_head *a,
1979 struct list_head *b)
1980{
1981 struct its_ite *itea = container_of(a, struct its_ite, ite_list);
1982 struct its_ite *iteb = container_of(b, struct its_ite, ite_list);
1983
1984 if (itea->event_id < iteb->event_id)
1985 return -1;
1986 else
1987 return 1;
1988}
1989
1990static int vgic_its_save_itt(struct vgic_its *its, struct its_device *device)
1991{
1992 const struct vgic_its_abi *abi = vgic_its_get_abi(its);
1993 gpa_t base = device->itt_addr;
1994 struct its_ite *ite;
1995 int ret;
1996 int ite_esz = abi->ite_esz;
1997
1998 list_sort(NULL, &device->itt_head, vgic_its_ite_cmp);
1999
2000 list_for_each_entry(ite, &device->itt_head, ite_list) {
2001 gpa_t gpa = base + ite->event_id * ite_esz;
2002
2003
2004
2005
2006
2007
2008
2009 if (ite->irq->hw)
2010 return -EACCES;
2011
2012 ret = vgic_its_save_ite(its, device, ite, gpa, ite_esz);
2013 if (ret)
2014 return ret;
2015 }
2016 return 0;
2017}
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027static int vgic_its_restore_itt(struct vgic_its *its, struct its_device *dev)
2028{
2029 const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2030 gpa_t base = dev->itt_addr;
2031 int ret;
2032 int ite_esz = abi->ite_esz;
2033 size_t max_size = BIT_ULL(dev->num_eventid_bits) * ite_esz;
2034
2035 ret = scan_its_table(its, base, max_size, ite_esz, 0,
2036 vgic_its_restore_ite, dev);
2037
2038
2039 if (ret > 0)
2040 ret = 0;
2041
2042 return ret;
2043}
2044
2045
2046
2047
2048
2049
2050
2051
2052static int vgic_its_save_dte(struct vgic_its *its, struct its_device *dev,
2053 gpa_t ptr, int dte_esz)
2054{
2055 struct kvm *kvm = its->dev->kvm;
2056 u64 val, itt_addr_field;
2057 u32 next_offset;
2058
2059 itt_addr_field = dev->itt_addr >> 8;
2060 next_offset = compute_next_devid_offset(&its->device_list, dev);
2061 val = (1ULL << KVM_ITS_DTE_VALID_SHIFT |
2062 ((u64)next_offset << KVM_ITS_DTE_NEXT_SHIFT) |
2063 (itt_addr_field << KVM_ITS_DTE_ITTADDR_SHIFT) |
2064 (dev->num_eventid_bits - 1));
2065 val = cpu_to_le64(val);
2066 return kvm_write_guest_lock(kvm, ptr, &val, dte_esz);
2067}
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080static int vgic_its_restore_dte(struct vgic_its *its, u32 id,
2081 void *ptr, void *opaque)
2082{
2083 struct its_device *dev;
2084 gpa_t itt_addr;
2085 u8 num_eventid_bits;
2086 u64 entry = *(u64 *)ptr;
2087 bool valid;
2088 u32 offset;
2089 int ret;
2090
2091 entry = le64_to_cpu(entry);
2092
2093 valid = entry >> KVM_ITS_DTE_VALID_SHIFT;
2094 num_eventid_bits = (entry & KVM_ITS_DTE_SIZE_MASK) + 1;
2095 itt_addr = ((entry & KVM_ITS_DTE_ITTADDR_MASK)
2096 >> KVM_ITS_DTE_ITTADDR_SHIFT) << 8;
2097
2098 if (!valid)
2099 return 1;
2100
2101
2102 offset = (entry & KVM_ITS_DTE_NEXT_MASK) >> KVM_ITS_DTE_NEXT_SHIFT;
2103
2104 dev = vgic_its_alloc_device(its, id, itt_addr, num_eventid_bits);
2105 if (IS_ERR(dev))
2106 return PTR_ERR(dev);
2107
2108 ret = vgic_its_restore_itt(its, dev);
2109 if (ret) {
2110 vgic_its_free_device(its->dev->kvm, dev);
2111 return ret;
2112 }
2113
2114 return offset;
2115}
2116
2117static int vgic_its_device_cmp(void *priv, struct list_head *a,
2118 struct list_head *b)
2119{
2120 struct its_device *deva = container_of(a, struct its_device, dev_list);
2121 struct its_device *devb = container_of(b, struct its_device, dev_list);
2122
2123 if (deva->device_id < devb->device_id)
2124 return -1;
2125 else
2126 return 1;
2127}
2128
2129
2130
2131
2132
2133
2134
2135
2136static int vgic_its_save_device_tables(struct vgic_its *its)
2137{
2138 const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2139 u64 baser = its->baser_device_table;
2140 struct its_device *dev;
2141 int dte_esz = abi->dte_esz;
2142
2143 if (!(baser & GITS_BASER_VALID))
2144 return 0;
2145
2146 list_sort(NULL, &its->device_list, vgic_its_device_cmp);
2147
2148 list_for_each_entry(dev, &its->device_list, dev_list) {
2149 int ret;
2150 gpa_t eaddr;
2151
2152 if (!vgic_its_check_id(its, baser,
2153 dev->device_id, &eaddr))
2154 return -EINVAL;
2155
2156 ret = vgic_its_save_itt(its, dev);
2157 if (ret)
2158 return ret;
2159
2160 ret = vgic_its_save_dte(its, dev, eaddr, dte_esz);
2161 if (ret)
2162 return ret;
2163 }
2164 return 0;
2165}
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179static int handle_l1_dte(struct vgic_its *its, u32 id, void *addr,
2180 void *opaque)
2181{
2182 const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2183 int l2_start_id = id * (SZ_64K / abi->dte_esz);
2184 u64 entry = *(u64 *)addr;
2185 int dte_esz = abi->dte_esz;
2186 gpa_t gpa;
2187 int ret;
2188
2189 entry = le64_to_cpu(entry);
2190
2191 if (!(entry & KVM_ITS_L1E_VALID_MASK))
2192 return 1;
2193
2194 gpa = entry & KVM_ITS_L1E_ADDR_MASK;
2195
2196 ret = scan_its_table(its, gpa, SZ_64K, dte_esz,
2197 l2_start_id, vgic_its_restore_dte, NULL);
2198
2199 return ret;
2200}
2201
2202
2203
2204
2205
2206static int vgic_its_restore_device_tables(struct vgic_its *its)
2207{
2208 const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2209 u64 baser = its->baser_device_table;
2210 int l1_esz, ret;
2211 int l1_tbl_size = GITS_BASER_NR_PAGES(baser) * SZ_64K;
2212 gpa_t l1_gpa;
2213
2214 if (!(baser & GITS_BASER_VALID))
2215 return 0;
2216
2217 l1_gpa = GITS_BASER_ADDR_48_to_52(baser);
2218
2219 if (baser & GITS_BASER_INDIRECT) {
2220 l1_esz = GITS_LVL1_ENTRY_SIZE;
2221 ret = scan_its_table(its, l1_gpa, l1_tbl_size, l1_esz, 0,
2222 handle_l1_dte, NULL);
2223 } else {
2224 l1_esz = abi->dte_esz;
2225 ret = scan_its_table(its, l1_gpa, l1_tbl_size, l1_esz, 0,
2226 vgic_its_restore_dte, NULL);
2227 }
2228
2229
2230 if (ret > 0)
2231 ret = 0;
2232
2233 return ret;
2234}
2235
2236static int vgic_its_save_cte(struct vgic_its *its,
2237 struct its_collection *collection,
2238 gpa_t gpa, int esz)
2239{
2240 u64 val;
2241
2242 val = (1ULL << KVM_ITS_CTE_VALID_SHIFT |
2243 ((u64)collection->target_addr << KVM_ITS_CTE_RDBASE_SHIFT) |
2244 collection->collection_id);
2245 val = cpu_to_le64(val);
2246 return kvm_write_guest_lock(its->dev->kvm, gpa, &val, esz);
2247}
2248
2249static int vgic_its_restore_cte(struct vgic_its *its, gpa_t gpa, int esz)
2250{
2251 struct its_collection *collection;
2252 struct kvm *kvm = its->dev->kvm;
2253 u32 target_addr, coll_id;
2254 u64 val;
2255 int ret;
2256
2257 BUG_ON(esz > sizeof(val));
2258 ret = kvm_read_guest_lock(kvm, gpa, &val, esz);
2259 if (ret)
2260 return ret;
2261 val = le64_to_cpu(val);
2262 if (!(val & KVM_ITS_CTE_VALID_MASK))
2263 return 0;
2264
2265 target_addr = (u32)(val >> KVM_ITS_CTE_RDBASE_SHIFT);
2266 coll_id = val & KVM_ITS_CTE_ICID_MASK;
2267
2268 if (target_addr >= atomic_read(&kvm->online_vcpus))
2269 return -EINVAL;
2270
2271 collection = find_collection(its, coll_id);
2272 if (collection)
2273 return -EEXIST;
2274 ret = vgic_its_alloc_collection(its, &collection, coll_id);
2275 if (ret)
2276 return ret;
2277 collection->target_addr = target_addr;
2278 return 1;
2279}
2280
2281
2282
2283
2284
2285static int vgic_its_save_collection_table(struct vgic_its *its)
2286{
2287 const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2288 u64 baser = its->baser_coll_table;
2289 gpa_t gpa = GITS_BASER_ADDR_48_to_52(baser);
2290 struct its_collection *collection;
2291 u64 val;
2292 size_t max_size, filled = 0;
2293 int ret, cte_esz = abi->cte_esz;
2294
2295 if (!(baser & GITS_BASER_VALID))
2296 return 0;
2297
2298 max_size = GITS_BASER_NR_PAGES(baser) * SZ_64K;
2299
2300 list_for_each_entry(collection, &its->collection_list, coll_list) {
2301 ret = vgic_its_save_cte(its, collection, gpa, cte_esz);
2302 if (ret)
2303 return ret;
2304 gpa += cte_esz;
2305 filled += cte_esz;
2306 }
2307
2308 if (filled == max_size)
2309 return 0;
2310
2311
2312
2313
2314
2315 val = 0;
2316 BUG_ON(cte_esz > sizeof(val));
2317 ret = kvm_write_guest_lock(its->dev->kvm, gpa, &val, cte_esz);
2318 return ret;
2319}
2320
2321
2322
2323
2324
2325
2326static int vgic_its_restore_collection_table(struct vgic_its *its)
2327{
2328 const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2329 u64 baser = its->baser_coll_table;
2330 int cte_esz = abi->cte_esz;
2331 size_t max_size, read = 0;
2332 gpa_t gpa;
2333 int ret;
2334
2335 if (!(baser & GITS_BASER_VALID))
2336 return 0;
2337
2338 gpa = GITS_BASER_ADDR_48_to_52(baser);
2339
2340 max_size = GITS_BASER_NR_PAGES(baser) * SZ_64K;
2341
2342 while (read < max_size) {
2343 ret = vgic_its_restore_cte(its, gpa, cte_esz);
2344 if (ret <= 0)
2345 break;
2346 gpa += cte_esz;
2347 read += cte_esz;
2348 }
2349
2350 if (ret > 0)
2351 return 0;
2352
2353 return ret;
2354}
2355
2356
2357
2358
2359
2360static int vgic_its_save_tables_v0(struct vgic_its *its)
2361{
2362 int ret;
2363
2364 ret = vgic_its_save_device_tables(its);
2365 if (ret)
2366 return ret;
2367
2368 return vgic_its_save_collection_table(its);
2369}
2370
2371
2372
2373
2374
2375
2376static int vgic_its_restore_tables_v0(struct vgic_its *its)
2377{
2378 int ret;
2379
2380 ret = vgic_its_restore_collection_table(its);
2381 if (ret)
2382 return ret;
2383
2384 return vgic_its_restore_device_tables(its);
2385}
2386
2387static int vgic_its_commit_v0(struct vgic_its *its)
2388{
2389 const struct vgic_its_abi *abi;
2390
2391 abi = vgic_its_get_abi(its);
2392 its->baser_coll_table &= ~GITS_BASER_ENTRY_SIZE_MASK;
2393 its->baser_device_table &= ~GITS_BASER_ENTRY_SIZE_MASK;
2394
2395 its->baser_coll_table |= (GIC_ENCODE_SZ(abi->cte_esz, 5)
2396 << GITS_BASER_ENTRY_SIZE_SHIFT);
2397
2398 its->baser_device_table |= (GIC_ENCODE_SZ(abi->dte_esz, 5)
2399 << GITS_BASER_ENTRY_SIZE_SHIFT);
2400 return 0;
2401}
2402
2403static void vgic_its_reset(struct kvm *kvm, struct vgic_its *its)
2404{
2405
2406 its->baser_coll_table &= ~GITS_BASER_VALID;
2407 its->baser_device_table &= ~GITS_BASER_VALID;
2408 its->cbaser = 0;
2409 its->creadr = 0;
2410 its->cwriter = 0;
2411 its->enabled = 0;
2412 vgic_its_free_device_list(kvm, its);
2413 vgic_its_free_collection_list(kvm, its);
2414}
2415
2416static int vgic_its_has_attr(struct kvm_device *dev,
2417 struct kvm_device_attr *attr)
2418{
2419 switch (attr->group) {
2420 case KVM_DEV_ARM_VGIC_GRP_ADDR:
2421 switch (attr->attr) {
2422 case KVM_VGIC_ITS_ADDR_TYPE:
2423 return 0;
2424 }
2425 break;
2426 case KVM_DEV_ARM_VGIC_GRP_CTRL:
2427 switch (attr->attr) {
2428 case KVM_DEV_ARM_VGIC_CTRL_INIT:
2429 return 0;
2430 case KVM_DEV_ARM_ITS_CTRL_RESET:
2431 return 0;
2432 case KVM_DEV_ARM_ITS_SAVE_TABLES:
2433 return 0;
2434 case KVM_DEV_ARM_ITS_RESTORE_TABLES:
2435 return 0;
2436 }
2437 break;
2438 case KVM_DEV_ARM_VGIC_GRP_ITS_REGS:
2439 return vgic_its_has_attr_regs(dev, attr);
2440 }
2441 return -ENXIO;
2442}
2443
2444static int vgic_its_ctrl(struct kvm *kvm, struct vgic_its *its, u64 attr)
2445{
2446 const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2447 int ret = 0;
2448
2449 if (attr == KVM_DEV_ARM_VGIC_CTRL_INIT)
2450 return 0;
2451
2452 mutex_lock(&kvm->lock);
2453 mutex_lock(&its->its_lock);
2454
2455 if (!lock_all_vcpus(kvm)) {
2456 mutex_unlock(&its->its_lock);
2457 mutex_unlock(&kvm->lock);
2458 return -EBUSY;
2459 }
2460
2461 switch (attr) {
2462 case KVM_DEV_ARM_ITS_CTRL_RESET:
2463 vgic_its_reset(kvm, its);
2464 break;
2465 case KVM_DEV_ARM_ITS_SAVE_TABLES:
2466 ret = abi->save_tables(its);
2467 break;
2468 case KVM_DEV_ARM_ITS_RESTORE_TABLES:
2469 ret = abi->restore_tables(its);
2470 break;
2471 }
2472
2473 unlock_all_vcpus(kvm);
2474 mutex_unlock(&its->its_lock);
2475 mutex_unlock(&kvm->lock);
2476 return ret;
2477}
2478
2479static int vgic_its_set_attr(struct kvm_device *dev,
2480 struct kvm_device_attr *attr)
2481{
2482 struct vgic_its *its = dev->private;
2483 int ret;
2484
2485 switch (attr->group) {
2486 case KVM_DEV_ARM_VGIC_GRP_ADDR: {
2487 u64 __user *uaddr = (u64 __user *)(long)attr->addr;
2488 unsigned long type = (unsigned long)attr->attr;
2489 u64 addr;
2490
2491 if (type != KVM_VGIC_ITS_ADDR_TYPE)
2492 return -ENODEV;
2493
2494 if (copy_from_user(&addr, uaddr, sizeof(addr)))
2495 return -EFAULT;
2496
2497 ret = vgic_check_ioaddr(dev->kvm, &its->vgic_its_base,
2498 addr, SZ_64K);
2499 if (ret)
2500 return ret;
2501
2502 return vgic_register_its_iodev(dev->kvm, its, addr);
2503 }
2504 case KVM_DEV_ARM_VGIC_GRP_CTRL:
2505 return vgic_its_ctrl(dev->kvm, its, attr->attr);
2506 case KVM_DEV_ARM_VGIC_GRP_ITS_REGS: {
2507 u64 __user *uaddr = (u64 __user *)(long)attr->addr;
2508 u64 reg;
2509
2510 if (get_user(reg, uaddr))
2511 return -EFAULT;
2512
2513 return vgic_its_attr_regs_access(dev, attr, ®, true);
2514 }
2515 }
2516 return -ENXIO;
2517}
2518
2519static int vgic_its_get_attr(struct kvm_device *dev,
2520 struct kvm_device_attr *attr)
2521{
2522 switch (attr->group) {
2523 case KVM_DEV_ARM_VGIC_GRP_ADDR: {
2524 struct vgic_its *its = dev->private;
2525 u64 addr = its->vgic_its_base;
2526 u64 __user *uaddr = (u64 __user *)(long)attr->addr;
2527 unsigned long type = (unsigned long)attr->attr;
2528
2529 if (type != KVM_VGIC_ITS_ADDR_TYPE)
2530 return -ENODEV;
2531
2532 if (copy_to_user(uaddr, &addr, sizeof(addr)))
2533 return -EFAULT;
2534 break;
2535 }
2536 case KVM_DEV_ARM_VGIC_GRP_ITS_REGS: {
2537 u64 __user *uaddr = (u64 __user *)(long)attr->addr;
2538 u64 reg;
2539 int ret;
2540
2541 ret = vgic_its_attr_regs_access(dev, attr, ®, false);
2542 if (ret)
2543 return ret;
2544 return put_user(reg, uaddr);
2545 }
2546 default:
2547 return -ENXIO;
2548 }
2549
2550 return 0;
2551}
2552
2553static struct kvm_device_ops kvm_arm_vgic_its_ops = {
2554 .name = "kvm-arm-vgic-its",
2555 .create = vgic_its_create,
2556 .destroy = vgic_its_destroy,
2557 .set_attr = vgic_its_set_attr,
2558 .get_attr = vgic_its_get_attr,
2559 .has_attr = vgic_its_has_attr,
2560};
2561
2562int kvm_vgic_register_its_device(void)
2563{
2564 return kvm_register_device_ops(&kvm_arm_vgic_its_ops,
2565 KVM_DEV_TYPE_ARM_VGIC_ITS);
2566}
2567