1
2
3
4
5
6
7
8
9
10
11
12#include <linux/bitfield.h>
13#include <linux/bsearch.h>
14#include <linux/kvm_host.h>
15#include <linux/mm.h>
16#include <linux/printk.h>
17#include <linux/uaccess.h>
18
19#include <asm/cacheflush.h>
20#include <asm/cputype.h>
21#include <asm/debug-monitors.h>
22#include <asm/esr.h>
23#include <asm/kvm_arm.h>
24#include <asm/kvm_emulate.h>
25#include <asm/kvm_hyp.h>
26#include <asm/kvm_mmu.h>
27#include <asm/perf_event.h>
28#include <asm/sysreg.h>
29
30#include <trace/events/kvm.h>
31
32#include "sys_regs.h"
33
34#include "trace.h"
35
36
37
38
39
40
41
42
43
44
45
46
47#define reg_to_encoding(x) \
48 sys_reg((u32)(x)->Op0, (u32)(x)->Op1, \
49 (u32)(x)->CRn, (u32)(x)->CRm, (u32)(x)->Op2)
50
51static bool read_from_write_only(struct kvm_vcpu *vcpu,
52 struct sys_reg_params *params,
53 const struct sys_reg_desc *r)
54{
55 WARN_ONCE(1, "Unexpected sys_reg read to write-only register\n");
56 print_sys_reg_instr(params);
57 kvm_inject_undefined(vcpu);
58 return false;
59}
60
61static bool write_to_read_only(struct kvm_vcpu *vcpu,
62 struct sys_reg_params *params,
63 const struct sys_reg_desc *r)
64{
65 WARN_ONCE(1, "Unexpected sys_reg write to read-only register\n");
66 print_sys_reg_instr(params);
67 kvm_inject_undefined(vcpu);
68 return false;
69}
70
71u64 vcpu_read_sys_reg(const struct kvm_vcpu *vcpu, int reg)
72{
73 u64 val = 0x8badf00d8badf00d;
74
75 if (vcpu->arch.sysregs_loaded_on_cpu &&
76 __vcpu_read_sys_reg_from_cpu(reg, &val))
77 return val;
78
79 return __vcpu_sys_reg(vcpu, reg);
80}
81
82void vcpu_write_sys_reg(struct kvm_vcpu *vcpu, u64 val, int reg)
83{
84 if (vcpu->arch.sysregs_loaded_on_cpu &&
85 __vcpu_write_sys_reg_to_cpu(val, reg))
86 return;
87
88 __vcpu_sys_reg(vcpu, reg) = val;
89}
90
91
92static u32 cache_levels;
93
94
95#define CSSELR_MAX 14
96
97
98static u32 get_ccsidr(u32 csselr)
99{
100 u32 ccsidr;
101
102
103 local_irq_disable();
104 write_sysreg(csselr, csselr_el1);
105 isb();
106 ccsidr = read_sysreg(ccsidr_el1);
107 local_irq_enable();
108
109 return ccsidr;
110}
111
112
113
114
115static bool access_dcsw(struct kvm_vcpu *vcpu,
116 struct sys_reg_params *p,
117 const struct sys_reg_desc *r)
118{
119 if (!p->is_write)
120 return read_from_write_only(vcpu, p, r);
121
122
123
124
125
126
127
128
129 if (!cpus_have_const_cap(ARM64_HAS_STAGE2_FWB))
130 kvm_set_way_flush(vcpu);
131
132 return true;
133}
134
135static void get_access_mask(const struct sys_reg_desc *r, u64 *mask, u64 *shift)
136{
137 switch (r->aarch32_map) {
138 case AA32_LO:
139 *mask = GENMASK_ULL(31, 0);
140 *shift = 0;
141 break;
142 case AA32_HI:
143 *mask = GENMASK_ULL(63, 32);
144 *shift = 32;
145 break;
146 default:
147 *mask = GENMASK_ULL(63, 0);
148 *shift = 0;
149 break;
150 }
151}
152
153
154
155
156
157
158static bool access_vm_reg(struct kvm_vcpu *vcpu,
159 struct sys_reg_params *p,
160 const struct sys_reg_desc *r)
161{
162 bool was_enabled = vcpu_has_cache_enabled(vcpu);
163 u64 val, mask, shift;
164
165 BUG_ON(!p->is_write);
166
167 get_access_mask(r, &mask, &shift);
168
169 if (~mask) {
170 val = vcpu_read_sys_reg(vcpu, r->reg);
171 val &= ~mask;
172 } else {
173 val = 0;
174 }
175
176 val |= (p->regval & (mask >> shift)) << shift;
177 vcpu_write_sys_reg(vcpu, val, r->reg);
178
179 kvm_toggle_cache(vcpu, was_enabled);
180 return true;
181}
182
183static bool access_actlr(struct kvm_vcpu *vcpu,
184 struct sys_reg_params *p,
185 const struct sys_reg_desc *r)
186{
187 u64 mask, shift;
188
189 if (p->is_write)
190 return ignore_write(vcpu, p);
191
192 get_access_mask(r, &mask, &shift);
193 p->regval = (vcpu_read_sys_reg(vcpu, r->reg) & mask) >> shift;
194
195 return true;
196}
197
198
199
200
201
202
203
204static bool access_gic_sgi(struct kvm_vcpu *vcpu,
205 struct sys_reg_params *p,
206 const struct sys_reg_desc *r)
207{
208 bool g1;
209
210 if (!p->is_write)
211 return read_from_write_only(vcpu, p, r);
212
213
214
215
216
217
218
219
220 if (p->Op0 == 0) {
221 switch (p->Op1) {
222 default:
223 case 0:
224 g1 = true;
225 break;
226 case 1:
227 case 2:
228 g1 = false;
229 break;
230 }
231 } else {
232 switch (p->Op2) {
233 default:
234 case 5:
235 g1 = true;
236 break;
237 case 6:
238 case 7:
239 g1 = false;
240 break;
241 }
242 }
243
244 vgic_v3_dispatch_sgi(vcpu, p->regval, g1);
245
246 return true;
247}
248
249static bool access_gic_sre(struct kvm_vcpu *vcpu,
250 struct sys_reg_params *p,
251 const struct sys_reg_desc *r)
252{
253 if (p->is_write)
254 return ignore_write(vcpu, p);
255
256 p->regval = vcpu->arch.vgic_cpu.vgic_v3.vgic_sre;
257 return true;
258}
259
260static bool trap_raz_wi(struct kvm_vcpu *vcpu,
261 struct sys_reg_params *p,
262 const struct sys_reg_desc *r)
263{
264 if (p->is_write)
265 return ignore_write(vcpu, p);
266 else
267 return read_zero(vcpu, p);
268}
269
270
271
272
273
274
275
276static bool trap_loregion(struct kvm_vcpu *vcpu,
277 struct sys_reg_params *p,
278 const struct sys_reg_desc *r)
279{
280 u64 val = read_sanitised_ftr_reg(SYS_ID_AA64MMFR1_EL1);
281 u32 sr = reg_to_encoding(r);
282
283 if (!(val & (0xfUL << ID_AA64MMFR1_LOR_SHIFT))) {
284 kvm_inject_undefined(vcpu);
285 return false;
286 }
287
288 if (p->is_write && sr == SYS_LORID_EL1)
289 return write_to_read_only(vcpu, p, r);
290
291 return trap_raz_wi(vcpu, p, r);
292}
293
294static bool trap_oslsr_el1(struct kvm_vcpu *vcpu,
295 struct sys_reg_params *p,
296 const struct sys_reg_desc *r)
297{
298 if (p->is_write) {
299 return ignore_write(vcpu, p);
300 } else {
301 p->regval = (1 << 3);
302 return true;
303 }
304}
305
306static bool trap_dbgauthstatus_el1(struct kvm_vcpu *vcpu,
307 struct sys_reg_params *p,
308 const struct sys_reg_desc *r)
309{
310 if (p->is_write) {
311 return ignore_write(vcpu, p);
312 } else {
313 p->regval = read_sysreg(dbgauthstatus_el1);
314 return true;
315 }
316}
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345static bool trap_debug_regs(struct kvm_vcpu *vcpu,
346 struct sys_reg_params *p,
347 const struct sys_reg_desc *r)
348{
349 if (p->is_write) {
350 vcpu_write_sys_reg(vcpu, p->regval, r->reg);
351 vcpu->arch.flags |= KVM_ARM64_DEBUG_DIRTY;
352 } else {
353 p->regval = vcpu_read_sys_reg(vcpu, r->reg);
354 }
355
356 trace_trap_reg(__func__, r->reg, p->is_write, p->regval);
357
358 return true;
359}
360
361
362
363
364
365
366
367
368
369
370static void reg_to_dbg(struct kvm_vcpu *vcpu,
371 struct sys_reg_params *p,
372 const struct sys_reg_desc *rd,
373 u64 *dbg_reg)
374{
375 u64 mask, shift, val;
376
377 get_access_mask(rd, &mask, &shift);
378
379 val = *dbg_reg;
380 val &= ~mask;
381 val |= (p->regval & (mask >> shift)) << shift;
382 *dbg_reg = val;
383
384 vcpu->arch.flags |= KVM_ARM64_DEBUG_DIRTY;
385}
386
387static void dbg_to_reg(struct kvm_vcpu *vcpu,
388 struct sys_reg_params *p,
389 const struct sys_reg_desc *rd,
390 u64 *dbg_reg)
391{
392 u64 mask, shift;
393
394 get_access_mask(rd, &mask, &shift);
395 p->regval = (*dbg_reg & mask) >> shift;
396}
397
398static bool trap_bvr(struct kvm_vcpu *vcpu,
399 struct sys_reg_params *p,
400 const struct sys_reg_desc *rd)
401{
402 u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_bvr[rd->CRm];
403
404 if (p->is_write)
405 reg_to_dbg(vcpu, p, rd, dbg_reg);
406 else
407 dbg_to_reg(vcpu, p, rd, dbg_reg);
408
409 trace_trap_reg(__func__, rd->CRm, p->is_write, *dbg_reg);
410
411 return true;
412}
413
414static int set_bvr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
415 const struct kvm_one_reg *reg, void __user *uaddr)
416{
417 __u64 *r = &vcpu->arch.vcpu_debug_state.dbg_bvr[rd->CRm];
418
419 if (copy_from_user(r, uaddr, KVM_REG_SIZE(reg->id)) != 0)
420 return -EFAULT;
421 return 0;
422}
423
424static int get_bvr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
425 const struct kvm_one_reg *reg, void __user *uaddr)
426{
427 __u64 *r = &vcpu->arch.vcpu_debug_state.dbg_bvr[rd->CRm];
428
429 if (copy_to_user(uaddr, r, KVM_REG_SIZE(reg->id)) != 0)
430 return -EFAULT;
431 return 0;
432}
433
434static void reset_bvr(struct kvm_vcpu *vcpu,
435 const struct sys_reg_desc *rd)
436{
437 vcpu->arch.vcpu_debug_state.dbg_bvr[rd->CRm] = rd->val;
438}
439
440static bool trap_bcr(struct kvm_vcpu *vcpu,
441 struct sys_reg_params *p,
442 const struct sys_reg_desc *rd)
443{
444 u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_bcr[rd->CRm];
445
446 if (p->is_write)
447 reg_to_dbg(vcpu, p, rd, dbg_reg);
448 else
449 dbg_to_reg(vcpu, p, rd, dbg_reg);
450
451 trace_trap_reg(__func__, rd->CRm, p->is_write, *dbg_reg);
452
453 return true;
454}
455
456static int set_bcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
457 const struct kvm_one_reg *reg, void __user *uaddr)
458{
459 __u64 *r = &vcpu->arch.vcpu_debug_state.dbg_bcr[rd->CRm];
460
461 if (copy_from_user(r, uaddr, KVM_REG_SIZE(reg->id)) != 0)
462 return -EFAULT;
463
464 return 0;
465}
466
467static int get_bcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
468 const struct kvm_one_reg *reg, void __user *uaddr)
469{
470 __u64 *r = &vcpu->arch.vcpu_debug_state.dbg_bcr[rd->CRm];
471
472 if (copy_to_user(uaddr, r, KVM_REG_SIZE(reg->id)) != 0)
473 return -EFAULT;
474 return 0;
475}
476
477static void reset_bcr(struct kvm_vcpu *vcpu,
478 const struct sys_reg_desc *rd)
479{
480 vcpu->arch.vcpu_debug_state.dbg_bcr[rd->CRm] = rd->val;
481}
482
483static bool trap_wvr(struct kvm_vcpu *vcpu,
484 struct sys_reg_params *p,
485 const struct sys_reg_desc *rd)
486{
487 u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_wvr[rd->CRm];
488
489 if (p->is_write)
490 reg_to_dbg(vcpu, p, rd, dbg_reg);
491 else
492 dbg_to_reg(vcpu, p, rd, dbg_reg);
493
494 trace_trap_reg(__func__, rd->CRm, p->is_write,
495 vcpu->arch.vcpu_debug_state.dbg_wvr[rd->CRm]);
496
497 return true;
498}
499
500static int set_wvr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
501 const struct kvm_one_reg *reg, void __user *uaddr)
502{
503 __u64 *r = &vcpu->arch.vcpu_debug_state.dbg_wvr[rd->CRm];
504
505 if (copy_from_user(r, uaddr, KVM_REG_SIZE(reg->id)) != 0)
506 return -EFAULT;
507 return 0;
508}
509
510static int get_wvr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
511 const struct kvm_one_reg *reg, void __user *uaddr)
512{
513 __u64 *r = &vcpu->arch.vcpu_debug_state.dbg_wvr[rd->CRm];
514
515 if (copy_to_user(uaddr, r, KVM_REG_SIZE(reg->id)) != 0)
516 return -EFAULT;
517 return 0;
518}
519
520static void reset_wvr(struct kvm_vcpu *vcpu,
521 const struct sys_reg_desc *rd)
522{
523 vcpu->arch.vcpu_debug_state.dbg_wvr[rd->CRm] = rd->val;
524}
525
526static bool trap_wcr(struct kvm_vcpu *vcpu,
527 struct sys_reg_params *p,
528 const struct sys_reg_desc *rd)
529{
530 u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_wcr[rd->CRm];
531
532 if (p->is_write)
533 reg_to_dbg(vcpu, p, rd, dbg_reg);
534 else
535 dbg_to_reg(vcpu, p, rd, dbg_reg);
536
537 trace_trap_reg(__func__, rd->CRm, p->is_write, *dbg_reg);
538
539 return true;
540}
541
542static int set_wcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
543 const struct kvm_one_reg *reg, void __user *uaddr)
544{
545 __u64 *r = &vcpu->arch.vcpu_debug_state.dbg_wcr[rd->CRm];
546
547 if (copy_from_user(r, uaddr, KVM_REG_SIZE(reg->id)) != 0)
548 return -EFAULT;
549 return 0;
550}
551
552static int get_wcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
553 const struct kvm_one_reg *reg, void __user *uaddr)
554{
555 __u64 *r = &vcpu->arch.vcpu_debug_state.dbg_wcr[rd->CRm];
556
557 if (copy_to_user(uaddr, r, KVM_REG_SIZE(reg->id)) != 0)
558 return -EFAULT;
559 return 0;
560}
561
562static void reset_wcr(struct kvm_vcpu *vcpu,
563 const struct sys_reg_desc *rd)
564{
565 vcpu->arch.vcpu_debug_state.dbg_wcr[rd->CRm] = rd->val;
566}
567
568static void reset_amair_el1(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
569{
570 u64 amair = read_sysreg(amair_el1);
571 vcpu_write_sys_reg(vcpu, amair, AMAIR_EL1);
572}
573
574static void reset_actlr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
575{
576 u64 actlr = read_sysreg(actlr_el1);
577 vcpu_write_sys_reg(vcpu, actlr, ACTLR_EL1);
578}
579
580static void reset_mpidr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
581{
582 u64 mpidr;
583
584
585
586
587
588
589
590
591 mpidr = (vcpu->vcpu_id & 0x0f) << MPIDR_LEVEL_SHIFT(0);
592 mpidr |= ((vcpu->vcpu_id >> 4) & 0xff) << MPIDR_LEVEL_SHIFT(1);
593 mpidr |= ((vcpu->vcpu_id >> 12) & 0xff) << MPIDR_LEVEL_SHIFT(2);
594 vcpu_write_sys_reg(vcpu, (1ULL << 31) | mpidr, MPIDR_EL1);
595}
596
597static unsigned int pmu_visibility(const struct kvm_vcpu *vcpu,
598 const struct sys_reg_desc *r)
599{
600 if (kvm_vcpu_has_pmu(vcpu))
601 return 0;
602
603 return REG_HIDDEN;
604}
605
606static void reset_pmcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
607{
608 u64 pmcr, val;
609
610
611 if (!kvm_arm_support_pmu_v3())
612 return;
613
614 pmcr = read_sysreg(pmcr_el0);
615
616
617
618
619 val = ((pmcr & ~ARMV8_PMU_PMCR_MASK)
620 | (ARMV8_PMU_PMCR_MASK & 0xdecafbad)) & (~ARMV8_PMU_PMCR_E);
621 if (!system_supports_32bit_el0())
622 val |= ARMV8_PMU_PMCR_LC;
623 __vcpu_sys_reg(vcpu, r->reg) = val;
624}
625
626static bool check_pmu_access_disabled(struct kvm_vcpu *vcpu, u64 flags)
627{
628 u64 reg = __vcpu_sys_reg(vcpu, PMUSERENR_EL0);
629 bool enabled = (reg & flags) || vcpu_mode_priv(vcpu);
630
631 if (!enabled)
632 kvm_inject_undefined(vcpu);
633
634 return !enabled;
635}
636
637static bool pmu_access_el0_disabled(struct kvm_vcpu *vcpu)
638{
639 return check_pmu_access_disabled(vcpu, ARMV8_PMU_USERENR_EN);
640}
641
642static bool pmu_write_swinc_el0_disabled(struct kvm_vcpu *vcpu)
643{
644 return check_pmu_access_disabled(vcpu, ARMV8_PMU_USERENR_SW | ARMV8_PMU_USERENR_EN);
645}
646
647static bool pmu_access_cycle_counter_el0_disabled(struct kvm_vcpu *vcpu)
648{
649 return check_pmu_access_disabled(vcpu, ARMV8_PMU_USERENR_CR | ARMV8_PMU_USERENR_EN);
650}
651
652static bool pmu_access_event_counter_el0_disabled(struct kvm_vcpu *vcpu)
653{
654 return check_pmu_access_disabled(vcpu, ARMV8_PMU_USERENR_ER | ARMV8_PMU_USERENR_EN);
655}
656
657static bool access_pmcr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
658 const struct sys_reg_desc *r)
659{
660 u64 val;
661
662 if (pmu_access_el0_disabled(vcpu))
663 return false;
664
665 if (p->is_write) {
666
667 val = __vcpu_sys_reg(vcpu, PMCR_EL0);
668 val &= ~ARMV8_PMU_PMCR_MASK;
669 val |= p->regval & ARMV8_PMU_PMCR_MASK;
670 if (!system_supports_32bit_el0())
671 val |= ARMV8_PMU_PMCR_LC;
672 __vcpu_sys_reg(vcpu, PMCR_EL0) = val;
673 kvm_pmu_handle_pmcr(vcpu, val);
674 kvm_vcpu_pmu_restore_guest(vcpu);
675 } else {
676
677 val = __vcpu_sys_reg(vcpu, PMCR_EL0)
678 & ~(ARMV8_PMU_PMCR_P | ARMV8_PMU_PMCR_C);
679 p->regval = val;
680 }
681
682 return true;
683}
684
685static bool access_pmselr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
686 const struct sys_reg_desc *r)
687{
688 if (pmu_access_event_counter_el0_disabled(vcpu))
689 return false;
690
691 if (p->is_write)
692 __vcpu_sys_reg(vcpu, PMSELR_EL0) = p->regval;
693 else
694
695 p->regval = __vcpu_sys_reg(vcpu, PMSELR_EL0)
696 & ARMV8_PMU_COUNTER_MASK;
697
698 return true;
699}
700
701static bool access_pmceid(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
702 const struct sys_reg_desc *r)
703{
704 u64 pmceid, mask, shift;
705
706 BUG_ON(p->is_write);
707
708 if (pmu_access_el0_disabled(vcpu))
709 return false;
710
711 get_access_mask(r, &mask, &shift);
712
713 pmceid = kvm_pmu_get_pmceid(vcpu, (p->Op2 & 1));
714 pmceid &= mask;
715 pmceid >>= shift;
716
717 p->regval = pmceid;
718
719 return true;
720}
721
722static bool pmu_counter_idx_valid(struct kvm_vcpu *vcpu, u64 idx)
723{
724 u64 pmcr, val;
725
726 pmcr = __vcpu_sys_reg(vcpu, PMCR_EL0);
727 val = (pmcr >> ARMV8_PMU_PMCR_N_SHIFT) & ARMV8_PMU_PMCR_N_MASK;
728 if (idx >= val && idx != ARMV8_PMU_CYCLE_IDX) {
729 kvm_inject_undefined(vcpu);
730 return false;
731 }
732
733 return true;
734}
735
736static bool access_pmu_evcntr(struct kvm_vcpu *vcpu,
737 struct sys_reg_params *p,
738 const struct sys_reg_desc *r)
739{
740 u64 idx = ~0UL;
741
742 if (r->CRn == 9 && r->CRm == 13) {
743 if (r->Op2 == 2) {
744
745 if (pmu_access_event_counter_el0_disabled(vcpu))
746 return false;
747
748 idx = __vcpu_sys_reg(vcpu, PMSELR_EL0)
749 & ARMV8_PMU_COUNTER_MASK;
750 } else if (r->Op2 == 0) {
751
752 if (pmu_access_cycle_counter_el0_disabled(vcpu))
753 return false;
754
755 idx = ARMV8_PMU_CYCLE_IDX;
756 }
757 } else if (r->CRn == 0 && r->CRm == 9) {
758
759 if (pmu_access_event_counter_el0_disabled(vcpu))
760 return false;
761
762 idx = ARMV8_PMU_CYCLE_IDX;
763 } else if (r->CRn == 14 && (r->CRm & 12) == 8) {
764
765 if (pmu_access_event_counter_el0_disabled(vcpu))
766 return false;
767
768 idx = ((r->CRm & 3) << 3) | (r->Op2 & 7);
769 }
770
771
772 WARN_ON(idx == ~0UL);
773
774 if (!pmu_counter_idx_valid(vcpu, idx))
775 return false;
776
777 if (p->is_write) {
778 if (pmu_access_el0_disabled(vcpu))
779 return false;
780
781 kvm_pmu_set_counter_value(vcpu, idx, p->regval);
782 } else {
783 p->regval = kvm_pmu_get_counter_value(vcpu, idx);
784 }
785
786 return true;
787}
788
789static bool access_pmu_evtyper(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
790 const struct sys_reg_desc *r)
791{
792 u64 idx, reg;
793
794 if (pmu_access_el0_disabled(vcpu))
795 return false;
796
797 if (r->CRn == 9 && r->CRm == 13 && r->Op2 == 1) {
798
799 idx = __vcpu_sys_reg(vcpu, PMSELR_EL0) & ARMV8_PMU_COUNTER_MASK;
800 reg = PMEVTYPER0_EL0 + idx;
801 } else if (r->CRn == 14 && (r->CRm & 12) == 12) {
802 idx = ((r->CRm & 3) << 3) | (r->Op2 & 7);
803 if (idx == ARMV8_PMU_CYCLE_IDX)
804 reg = PMCCFILTR_EL0;
805 else
806
807 reg = PMEVTYPER0_EL0 + idx;
808 } else {
809 BUG();
810 }
811
812 if (!pmu_counter_idx_valid(vcpu, idx))
813 return false;
814
815 if (p->is_write) {
816 kvm_pmu_set_counter_event_type(vcpu, p->regval, idx);
817 __vcpu_sys_reg(vcpu, reg) = p->regval & ARMV8_PMU_EVTYPE_MASK;
818 kvm_vcpu_pmu_restore_guest(vcpu);
819 } else {
820 p->regval = __vcpu_sys_reg(vcpu, reg) & ARMV8_PMU_EVTYPE_MASK;
821 }
822
823 return true;
824}
825
826static bool access_pmcnten(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
827 const struct sys_reg_desc *r)
828{
829 u64 val, mask;
830
831 if (pmu_access_el0_disabled(vcpu))
832 return false;
833
834 mask = kvm_pmu_valid_counter_mask(vcpu);
835 if (p->is_write) {
836 val = p->regval & mask;
837 if (r->Op2 & 0x1) {
838
839 __vcpu_sys_reg(vcpu, PMCNTENSET_EL0) |= val;
840 kvm_pmu_enable_counter_mask(vcpu, val);
841 kvm_vcpu_pmu_restore_guest(vcpu);
842 } else {
843
844 __vcpu_sys_reg(vcpu, PMCNTENSET_EL0) &= ~val;
845 kvm_pmu_disable_counter_mask(vcpu, val);
846 }
847 } else {
848 p->regval = __vcpu_sys_reg(vcpu, PMCNTENSET_EL0) & mask;
849 }
850
851 return true;
852}
853
854static bool access_pminten(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
855 const struct sys_reg_desc *r)
856{
857 u64 mask = kvm_pmu_valid_counter_mask(vcpu);
858
859 if (check_pmu_access_disabled(vcpu, 0))
860 return false;
861
862 if (p->is_write) {
863 u64 val = p->regval & mask;
864
865 if (r->Op2 & 0x1)
866
867 __vcpu_sys_reg(vcpu, PMINTENSET_EL1) |= val;
868 else
869
870 __vcpu_sys_reg(vcpu, PMINTENSET_EL1) &= ~val;
871 } else {
872 p->regval = __vcpu_sys_reg(vcpu, PMINTENSET_EL1) & mask;
873 }
874
875 return true;
876}
877
878static bool access_pmovs(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
879 const struct sys_reg_desc *r)
880{
881 u64 mask = kvm_pmu_valid_counter_mask(vcpu);
882
883 if (pmu_access_el0_disabled(vcpu))
884 return false;
885
886 if (p->is_write) {
887 if (r->CRm & 0x2)
888
889 __vcpu_sys_reg(vcpu, PMOVSSET_EL0) |= (p->regval & mask);
890 else
891
892 __vcpu_sys_reg(vcpu, PMOVSSET_EL0) &= ~(p->regval & mask);
893 } else {
894 p->regval = __vcpu_sys_reg(vcpu, PMOVSSET_EL0) & mask;
895 }
896
897 return true;
898}
899
900static bool access_pmswinc(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
901 const struct sys_reg_desc *r)
902{
903 u64 mask;
904
905 if (!p->is_write)
906 return read_from_write_only(vcpu, p, r);
907
908 if (pmu_write_swinc_el0_disabled(vcpu))
909 return false;
910
911 mask = kvm_pmu_valid_counter_mask(vcpu);
912 kvm_pmu_software_increment(vcpu, p->regval & mask);
913 return true;
914}
915
916static bool access_pmuserenr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
917 const struct sys_reg_desc *r)
918{
919 if (p->is_write) {
920 if (!vcpu_mode_priv(vcpu)) {
921 kvm_inject_undefined(vcpu);
922 return false;
923 }
924
925 __vcpu_sys_reg(vcpu, PMUSERENR_EL0) =
926 p->regval & ARMV8_PMU_USERENR_MASK;
927 } else {
928 p->regval = __vcpu_sys_reg(vcpu, PMUSERENR_EL0)
929 & ARMV8_PMU_USERENR_MASK;
930 }
931
932 return true;
933}
934
935
936#define DBG_BCR_BVR_WCR_WVR_EL1(n) \
937 { SYS_DESC(SYS_DBGBVRn_EL1(n)), \
938 trap_bvr, reset_bvr, 0, 0, get_bvr, set_bvr }, \
939 { SYS_DESC(SYS_DBGBCRn_EL1(n)), \
940 trap_bcr, reset_bcr, 0, 0, get_bcr, set_bcr }, \
941 { SYS_DESC(SYS_DBGWVRn_EL1(n)), \
942 trap_wvr, reset_wvr, 0, 0, get_wvr, set_wvr }, \
943 { SYS_DESC(SYS_DBGWCRn_EL1(n)), \
944 trap_wcr, reset_wcr, 0, 0, get_wcr, set_wcr }
945
946#define PMU_SYS_REG(r) \
947 SYS_DESC(r), .reset = reset_unknown, .visibility = pmu_visibility
948
949
950#define PMU_PMEVCNTR_EL0(n) \
951 { PMU_SYS_REG(SYS_PMEVCNTRn_EL0(n)), \
952 .access = access_pmu_evcntr, .reg = (PMEVCNTR0_EL0 + n), }
953
954
955#define PMU_PMEVTYPER_EL0(n) \
956 { PMU_SYS_REG(SYS_PMEVTYPERn_EL0(n)), \
957 .access = access_pmu_evtyper, .reg = (PMEVTYPER0_EL0 + n), }
958
959static bool undef_access(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
960 const struct sys_reg_desc *r)
961{
962 kvm_inject_undefined(vcpu);
963
964 return false;
965}
966
967
968#define AMU_AMEVCNTR0_EL0(n) { SYS_DESC(SYS_AMEVCNTR0_EL0(n)), undef_access }
969#define AMU_AMEVTYPER0_EL0(n) { SYS_DESC(SYS_AMEVTYPER0_EL0(n)), undef_access }
970#define AMU_AMEVCNTR1_EL0(n) { SYS_DESC(SYS_AMEVCNTR1_EL0(n)), undef_access }
971#define AMU_AMEVTYPER1_EL0(n) { SYS_DESC(SYS_AMEVTYPER1_EL0(n)), undef_access }
972
973static unsigned int ptrauth_visibility(const struct kvm_vcpu *vcpu,
974 const struct sys_reg_desc *rd)
975{
976 return vcpu_has_ptrauth(vcpu) ? 0 : REG_HIDDEN;
977}
978
979
980
981
982
983
984
985#define __PTRAUTH_KEY(k) \
986 { SYS_DESC(SYS_## k), undef_access, reset_unknown, k, \
987 .visibility = ptrauth_visibility}
988
989#define PTRAUTH_KEY(k) \
990 __PTRAUTH_KEY(k ## KEYLO_EL1), \
991 __PTRAUTH_KEY(k ## KEYHI_EL1)
992
993static bool access_arch_timer(struct kvm_vcpu *vcpu,
994 struct sys_reg_params *p,
995 const struct sys_reg_desc *r)
996{
997 enum kvm_arch_timers tmr;
998 enum kvm_arch_timer_regs treg;
999 u64 reg = reg_to_encoding(r);
1000
1001 switch (reg) {
1002 case SYS_CNTP_TVAL_EL0:
1003 case SYS_AARCH32_CNTP_TVAL:
1004 tmr = TIMER_PTIMER;
1005 treg = TIMER_REG_TVAL;
1006 break;
1007 case SYS_CNTP_CTL_EL0:
1008 case SYS_AARCH32_CNTP_CTL:
1009 tmr = TIMER_PTIMER;
1010 treg = TIMER_REG_CTL;
1011 break;
1012 case SYS_CNTP_CVAL_EL0:
1013 case SYS_AARCH32_CNTP_CVAL:
1014 tmr = TIMER_PTIMER;
1015 treg = TIMER_REG_CVAL;
1016 break;
1017 default:
1018 BUG();
1019 }
1020
1021 if (p->is_write)
1022 kvm_arm_timer_write_sysreg(vcpu, tmr, treg, p->regval);
1023 else
1024 p->regval = kvm_arm_timer_read_sysreg(vcpu, tmr, treg);
1025
1026 return true;
1027}
1028
1029#define FEATURE(x) (GENMASK_ULL(x##_SHIFT + 3, x##_SHIFT))
1030
1031
1032static u64 read_id_reg(const struct kvm_vcpu *vcpu,
1033 struct sys_reg_desc const *r, bool raz)
1034{
1035 u32 id = reg_to_encoding(r);
1036 u64 val = raz ? 0 : read_sanitised_ftr_reg(id);
1037
1038 switch (id) {
1039 case SYS_ID_AA64PFR0_EL1:
1040 if (!vcpu_has_sve(vcpu))
1041 val &= ~FEATURE(ID_AA64PFR0_SVE);
1042 val &= ~FEATURE(ID_AA64PFR0_AMU);
1043 val &= ~FEATURE(ID_AA64PFR0_CSV2);
1044 val |= FIELD_PREP(FEATURE(ID_AA64PFR0_CSV2), (u64)vcpu->kvm->arch.pfr0_csv2);
1045 val &= ~FEATURE(ID_AA64PFR0_CSV3);
1046 val |= FIELD_PREP(FEATURE(ID_AA64PFR0_CSV3), (u64)vcpu->kvm->arch.pfr0_csv3);
1047 break;
1048 case SYS_ID_AA64PFR1_EL1:
1049 val &= ~FEATURE(ID_AA64PFR1_MTE);
1050 break;
1051 case SYS_ID_AA64ISAR1_EL1:
1052 if (!vcpu_has_ptrauth(vcpu))
1053 val &= ~(FEATURE(ID_AA64ISAR1_APA) |
1054 FEATURE(ID_AA64ISAR1_API) |
1055 FEATURE(ID_AA64ISAR1_GPA) |
1056 FEATURE(ID_AA64ISAR1_GPI));
1057 break;
1058 case SYS_ID_AA64DFR0_EL1:
1059
1060 val &= ~FEATURE(ID_AA64DFR0_DEBUGVER);
1061 val |= FIELD_PREP(FEATURE(ID_AA64DFR0_DEBUGVER), 6);
1062
1063 val = cpuid_feature_cap_perfmon_field(val,
1064 ID_AA64DFR0_PMUVER_SHIFT,
1065 kvm_vcpu_has_pmu(vcpu) ? ID_AA64DFR0_PMUVER_8_4 : 0);
1066
1067 val &= ~FEATURE(ID_AA64DFR0_PMSVER);
1068 break;
1069 case SYS_ID_DFR0_EL1:
1070
1071 val = cpuid_feature_cap_perfmon_field(val,
1072 ID_DFR0_PERFMON_SHIFT,
1073 kvm_vcpu_has_pmu(vcpu) ? ID_DFR0_PERFMON_8_4 : 0);
1074 break;
1075 }
1076
1077 return val;
1078}
1079
1080static unsigned int id_visibility(const struct kvm_vcpu *vcpu,
1081 const struct sys_reg_desc *r)
1082{
1083 u32 id = reg_to_encoding(r);
1084
1085 switch (id) {
1086 case SYS_ID_AA64ZFR0_EL1:
1087 if (!vcpu_has_sve(vcpu))
1088 return REG_RAZ;
1089 break;
1090 }
1091
1092 return 0;
1093}
1094
1095
1096
1097static bool __access_id_reg(struct kvm_vcpu *vcpu,
1098 struct sys_reg_params *p,
1099 const struct sys_reg_desc *r,
1100 bool raz)
1101{
1102 if (p->is_write)
1103 return write_to_read_only(vcpu, p, r);
1104
1105 p->regval = read_id_reg(vcpu, r, raz);
1106 return true;
1107}
1108
1109static bool access_id_reg(struct kvm_vcpu *vcpu,
1110 struct sys_reg_params *p,
1111 const struct sys_reg_desc *r)
1112{
1113 bool raz = sysreg_visible_as_raz(vcpu, r);
1114
1115 return __access_id_reg(vcpu, p, r, raz);
1116}
1117
1118static bool access_raz_id_reg(struct kvm_vcpu *vcpu,
1119 struct sys_reg_params *p,
1120 const struct sys_reg_desc *r)
1121{
1122 return __access_id_reg(vcpu, p, r, true);
1123}
1124
1125static int reg_from_user(u64 *val, const void __user *uaddr, u64 id);
1126static int reg_to_user(void __user *uaddr, const u64 *val, u64 id);
1127static u64 sys_reg_to_index(const struct sys_reg_desc *reg);
1128
1129
1130static unsigned int sve_visibility(const struct kvm_vcpu *vcpu,
1131 const struct sys_reg_desc *rd)
1132{
1133 if (vcpu_has_sve(vcpu))
1134 return 0;
1135
1136 return REG_HIDDEN;
1137}
1138
1139static int set_id_aa64pfr0_el1(struct kvm_vcpu *vcpu,
1140 const struct sys_reg_desc *rd,
1141 const struct kvm_one_reg *reg, void __user *uaddr)
1142{
1143 const u64 id = sys_reg_to_index(rd);
1144 u8 csv2, csv3;
1145 int err;
1146 u64 val;
1147
1148 err = reg_from_user(&val, uaddr, id);
1149 if (err)
1150 return err;
1151
1152
1153
1154
1155
1156
1157 csv2 = cpuid_feature_extract_unsigned_field(val, ID_AA64PFR0_CSV2_SHIFT);
1158 if (csv2 > 1 ||
1159 (csv2 && arm64_get_spectre_v2_state() != SPECTRE_UNAFFECTED))
1160 return -EINVAL;
1161
1162
1163 csv3 = cpuid_feature_extract_unsigned_field(val, ID_AA64PFR0_CSV3_SHIFT);
1164 if (csv3 > 1 ||
1165 (csv3 && arm64_get_meltdown_state() != SPECTRE_UNAFFECTED))
1166 return -EINVAL;
1167
1168
1169 val ^= read_id_reg(vcpu, rd, false);
1170 val &= ~((0xFUL << ID_AA64PFR0_CSV2_SHIFT) |
1171 (0xFUL << ID_AA64PFR0_CSV3_SHIFT));
1172 if (val)
1173 return -EINVAL;
1174
1175 vcpu->kvm->arch.pfr0_csv2 = csv2;
1176 vcpu->kvm->arch.pfr0_csv3 = csv3 ;
1177
1178 return 0;
1179}
1180
1181
1182
1183
1184
1185
1186
1187
1188static int __get_id_reg(const struct kvm_vcpu *vcpu,
1189 const struct sys_reg_desc *rd, void __user *uaddr,
1190 bool raz)
1191{
1192 const u64 id = sys_reg_to_index(rd);
1193 const u64 val = read_id_reg(vcpu, rd, raz);
1194
1195 return reg_to_user(uaddr, &val, id);
1196}
1197
1198static int __set_id_reg(const struct kvm_vcpu *vcpu,
1199 const struct sys_reg_desc *rd, void __user *uaddr,
1200 bool raz)
1201{
1202 const u64 id = sys_reg_to_index(rd);
1203 int err;
1204 u64 val;
1205
1206 err = reg_from_user(&val, uaddr, id);
1207 if (err)
1208 return err;
1209
1210
1211 if (val != read_id_reg(vcpu, rd, raz))
1212 return -EINVAL;
1213
1214 return 0;
1215}
1216
1217static int get_id_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
1218 const struct kvm_one_reg *reg, void __user *uaddr)
1219{
1220 bool raz = sysreg_visible_as_raz(vcpu, rd);
1221
1222 return __get_id_reg(vcpu, rd, uaddr, raz);
1223}
1224
1225static int set_id_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
1226 const struct kvm_one_reg *reg, void __user *uaddr)
1227{
1228 bool raz = sysreg_visible_as_raz(vcpu, rd);
1229
1230 return __set_id_reg(vcpu, rd, uaddr, raz);
1231}
1232
1233static int get_raz_id_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
1234 const struct kvm_one_reg *reg, void __user *uaddr)
1235{
1236 return __get_id_reg(vcpu, rd, uaddr, true);
1237}
1238
1239static int set_raz_id_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
1240 const struct kvm_one_reg *reg, void __user *uaddr)
1241{
1242 return __set_id_reg(vcpu, rd, uaddr, true);
1243}
1244
1245static bool access_ctr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
1246 const struct sys_reg_desc *r)
1247{
1248 if (p->is_write)
1249 return write_to_read_only(vcpu, p, r);
1250
1251 p->regval = read_sanitised_ftr_reg(SYS_CTR_EL0);
1252 return true;
1253}
1254
1255static bool access_clidr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
1256 const struct sys_reg_desc *r)
1257{
1258 if (p->is_write)
1259 return write_to_read_only(vcpu, p, r);
1260
1261 p->regval = read_sysreg(clidr_el1);
1262 return true;
1263}
1264
1265static bool access_csselr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
1266 const struct sys_reg_desc *r)
1267{
1268 int reg = r->reg;
1269
1270 if (p->is_write)
1271 vcpu_write_sys_reg(vcpu, p->regval, reg);
1272 else
1273 p->regval = vcpu_read_sys_reg(vcpu, reg);
1274 return true;
1275}
1276
1277static bool access_ccsidr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
1278 const struct sys_reg_desc *r)
1279{
1280 u32 csselr;
1281
1282 if (p->is_write)
1283 return write_to_read_only(vcpu, p, r);
1284
1285 csselr = vcpu_read_sys_reg(vcpu, CSSELR_EL1);
1286 p->regval = get_ccsidr(csselr);
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300 if (!(csselr & 1))
1301 p->regval &= ~GENMASK(27, 3);
1302 return true;
1303}
1304
1305
1306#define ID_SANITISED(name) { \
1307 SYS_DESC(SYS_##name), \
1308 .access = access_id_reg, \
1309 .get_user = get_id_reg, \
1310 .set_user = set_id_reg, \
1311 .visibility = id_visibility, \
1312}
1313
1314
1315
1316
1317
1318
1319#define ID_UNALLOCATED(crm, op2) { \
1320 Op0(3), Op1(0), CRn(0), CRm(crm), Op2(op2), \
1321 .access = access_raz_id_reg, \
1322 .get_user = get_raz_id_reg, \
1323 .set_user = set_raz_id_reg, \
1324}
1325
1326
1327
1328
1329
1330
1331#define ID_HIDDEN(name) { \
1332 SYS_DESC(SYS_##name), \
1333 .access = access_raz_id_reg, \
1334 .get_user = get_raz_id_reg, \
1335 .set_user = set_raz_id_reg, \
1336}
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349static const struct sys_reg_desc sys_reg_descs[] = {
1350 { SYS_DESC(SYS_DC_ISW), access_dcsw },
1351 { SYS_DESC(SYS_DC_CSW), access_dcsw },
1352 { SYS_DESC(SYS_DC_CISW), access_dcsw },
1353
1354 DBG_BCR_BVR_WCR_WVR_EL1(0),
1355 DBG_BCR_BVR_WCR_WVR_EL1(1),
1356 { SYS_DESC(SYS_MDCCINT_EL1), trap_debug_regs, reset_val, MDCCINT_EL1, 0 },
1357 { SYS_DESC(SYS_MDSCR_EL1), trap_debug_regs, reset_val, MDSCR_EL1, 0 },
1358 DBG_BCR_BVR_WCR_WVR_EL1(2),
1359 DBG_BCR_BVR_WCR_WVR_EL1(3),
1360 DBG_BCR_BVR_WCR_WVR_EL1(4),
1361 DBG_BCR_BVR_WCR_WVR_EL1(5),
1362 DBG_BCR_BVR_WCR_WVR_EL1(6),
1363 DBG_BCR_BVR_WCR_WVR_EL1(7),
1364 DBG_BCR_BVR_WCR_WVR_EL1(8),
1365 DBG_BCR_BVR_WCR_WVR_EL1(9),
1366 DBG_BCR_BVR_WCR_WVR_EL1(10),
1367 DBG_BCR_BVR_WCR_WVR_EL1(11),
1368 DBG_BCR_BVR_WCR_WVR_EL1(12),
1369 DBG_BCR_BVR_WCR_WVR_EL1(13),
1370 DBG_BCR_BVR_WCR_WVR_EL1(14),
1371 DBG_BCR_BVR_WCR_WVR_EL1(15),
1372
1373 { SYS_DESC(SYS_MDRAR_EL1), trap_raz_wi },
1374 { SYS_DESC(SYS_OSLAR_EL1), trap_raz_wi },
1375 { SYS_DESC(SYS_OSLSR_EL1), trap_oslsr_el1 },
1376 { SYS_DESC(SYS_OSDLR_EL1), trap_raz_wi },
1377 { SYS_DESC(SYS_DBGPRCR_EL1), trap_raz_wi },
1378 { SYS_DESC(SYS_DBGCLAIMSET_EL1), trap_raz_wi },
1379 { SYS_DESC(SYS_DBGCLAIMCLR_EL1), trap_raz_wi },
1380 { SYS_DESC(SYS_DBGAUTHSTATUS_EL1), trap_dbgauthstatus_el1 },
1381
1382 { SYS_DESC(SYS_MDCCSR_EL0), trap_raz_wi },
1383 { SYS_DESC(SYS_DBGDTR_EL0), trap_raz_wi },
1384
1385 { SYS_DESC(SYS_DBGDTRTX_EL0), trap_raz_wi },
1386
1387 { SYS_DESC(SYS_DBGVCR32_EL2), NULL, reset_val, DBGVCR32_EL2, 0 },
1388
1389 { SYS_DESC(SYS_MPIDR_EL1), NULL, reset_mpidr, MPIDR_EL1 },
1390
1391
1392
1393
1394
1395
1396
1397
1398 ID_SANITISED(ID_PFR0_EL1),
1399 ID_SANITISED(ID_PFR1_EL1),
1400 ID_SANITISED(ID_DFR0_EL1),
1401 ID_HIDDEN(ID_AFR0_EL1),
1402 ID_SANITISED(ID_MMFR0_EL1),
1403 ID_SANITISED(ID_MMFR1_EL1),
1404 ID_SANITISED(ID_MMFR2_EL1),
1405 ID_SANITISED(ID_MMFR3_EL1),
1406
1407
1408 ID_SANITISED(ID_ISAR0_EL1),
1409 ID_SANITISED(ID_ISAR1_EL1),
1410 ID_SANITISED(ID_ISAR2_EL1),
1411 ID_SANITISED(ID_ISAR3_EL1),
1412 ID_SANITISED(ID_ISAR4_EL1),
1413 ID_SANITISED(ID_ISAR5_EL1),
1414 ID_SANITISED(ID_MMFR4_EL1),
1415 ID_SANITISED(ID_ISAR6_EL1),
1416
1417
1418 ID_SANITISED(MVFR0_EL1),
1419 ID_SANITISED(MVFR1_EL1),
1420 ID_SANITISED(MVFR2_EL1),
1421 ID_UNALLOCATED(3,3),
1422 ID_SANITISED(ID_PFR2_EL1),
1423 ID_HIDDEN(ID_DFR1_EL1),
1424 ID_SANITISED(ID_MMFR5_EL1),
1425 ID_UNALLOCATED(3,7),
1426
1427
1428
1429 { SYS_DESC(SYS_ID_AA64PFR0_EL1), .access = access_id_reg,
1430 .get_user = get_id_reg, .set_user = set_id_aa64pfr0_el1, },
1431 ID_SANITISED(ID_AA64PFR1_EL1),
1432 ID_UNALLOCATED(4,2),
1433 ID_UNALLOCATED(4,3),
1434 ID_SANITISED(ID_AA64ZFR0_EL1),
1435 ID_UNALLOCATED(4,5),
1436 ID_UNALLOCATED(4,6),
1437 ID_UNALLOCATED(4,7),
1438
1439
1440 ID_SANITISED(ID_AA64DFR0_EL1),
1441 ID_SANITISED(ID_AA64DFR1_EL1),
1442 ID_UNALLOCATED(5,2),
1443 ID_UNALLOCATED(5,3),
1444 ID_HIDDEN(ID_AA64AFR0_EL1),
1445 ID_HIDDEN(ID_AA64AFR1_EL1),
1446 ID_UNALLOCATED(5,6),
1447 ID_UNALLOCATED(5,7),
1448
1449
1450 ID_SANITISED(ID_AA64ISAR0_EL1),
1451 ID_SANITISED(ID_AA64ISAR1_EL1),
1452 ID_UNALLOCATED(6,2),
1453 ID_UNALLOCATED(6,3),
1454 ID_UNALLOCATED(6,4),
1455 ID_UNALLOCATED(6,5),
1456 ID_UNALLOCATED(6,6),
1457 ID_UNALLOCATED(6,7),
1458
1459
1460 ID_SANITISED(ID_AA64MMFR0_EL1),
1461 ID_SANITISED(ID_AA64MMFR1_EL1),
1462 ID_SANITISED(ID_AA64MMFR2_EL1),
1463 ID_UNALLOCATED(7,3),
1464 ID_UNALLOCATED(7,4),
1465 ID_UNALLOCATED(7,5),
1466 ID_UNALLOCATED(7,6),
1467 ID_UNALLOCATED(7,7),
1468
1469 { SYS_DESC(SYS_SCTLR_EL1), access_vm_reg, reset_val, SCTLR_EL1, 0x00C50078 },
1470 { SYS_DESC(SYS_ACTLR_EL1), access_actlr, reset_actlr, ACTLR_EL1 },
1471 { SYS_DESC(SYS_CPACR_EL1), NULL, reset_val, CPACR_EL1, 0 },
1472
1473 { SYS_DESC(SYS_RGSR_EL1), undef_access },
1474 { SYS_DESC(SYS_GCR_EL1), undef_access },
1475
1476 { SYS_DESC(SYS_ZCR_EL1), NULL, reset_val, ZCR_EL1, 0, .visibility = sve_visibility },
1477 { SYS_DESC(SYS_TRFCR_EL1), undef_access },
1478 { SYS_DESC(SYS_TTBR0_EL1), access_vm_reg, reset_unknown, TTBR0_EL1 },
1479 { SYS_DESC(SYS_TTBR1_EL1), access_vm_reg, reset_unknown, TTBR1_EL1 },
1480 { SYS_DESC(SYS_TCR_EL1), access_vm_reg, reset_val, TCR_EL1, 0 },
1481
1482 PTRAUTH_KEY(APIA),
1483 PTRAUTH_KEY(APIB),
1484 PTRAUTH_KEY(APDA),
1485 PTRAUTH_KEY(APDB),
1486 PTRAUTH_KEY(APGA),
1487
1488 { SYS_DESC(SYS_AFSR0_EL1), access_vm_reg, reset_unknown, AFSR0_EL1 },
1489 { SYS_DESC(SYS_AFSR1_EL1), access_vm_reg, reset_unknown, AFSR1_EL1 },
1490 { SYS_DESC(SYS_ESR_EL1), access_vm_reg, reset_unknown, ESR_EL1 },
1491
1492 { SYS_DESC(SYS_ERRIDR_EL1), trap_raz_wi },
1493 { SYS_DESC(SYS_ERRSELR_EL1), trap_raz_wi },
1494 { SYS_DESC(SYS_ERXFR_EL1), trap_raz_wi },
1495 { SYS_DESC(SYS_ERXCTLR_EL1), trap_raz_wi },
1496 { SYS_DESC(SYS_ERXSTATUS_EL1), trap_raz_wi },
1497 { SYS_DESC(SYS_ERXADDR_EL1), trap_raz_wi },
1498 { SYS_DESC(SYS_ERXMISC0_EL1), trap_raz_wi },
1499 { SYS_DESC(SYS_ERXMISC1_EL1), trap_raz_wi },
1500
1501 { SYS_DESC(SYS_TFSR_EL1), undef_access },
1502 { SYS_DESC(SYS_TFSRE0_EL1), undef_access },
1503
1504 { SYS_DESC(SYS_FAR_EL1), access_vm_reg, reset_unknown, FAR_EL1 },
1505 { SYS_DESC(SYS_PAR_EL1), NULL, reset_unknown, PAR_EL1 },
1506
1507 { SYS_DESC(SYS_PMSCR_EL1), undef_access },
1508 { SYS_DESC(SYS_PMSNEVFR_EL1), undef_access },
1509 { SYS_DESC(SYS_PMSICR_EL1), undef_access },
1510 { SYS_DESC(SYS_PMSIRR_EL1), undef_access },
1511 { SYS_DESC(SYS_PMSFCR_EL1), undef_access },
1512 { SYS_DESC(SYS_PMSEVFR_EL1), undef_access },
1513 { SYS_DESC(SYS_PMSLATFR_EL1), undef_access },
1514 { SYS_DESC(SYS_PMSIDR_EL1), undef_access },
1515 { SYS_DESC(SYS_PMBLIMITR_EL1), undef_access },
1516 { SYS_DESC(SYS_PMBPTR_EL1), undef_access },
1517 { SYS_DESC(SYS_PMBSR_EL1), undef_access },
1518
1519
1520 { PMU_SYS_REG(SYS_PMINTENSET_EL1),
1521 .access = access_pminten, .reg = PMINTENSET_EL1 },
1522 { PMU_SYS_REG(SYS_PMINTENCLR_EL1),
1523 .access = access_pminten, .reg = PMINTENSET_EL1 },
1524 { SYS_DESC(SYS_PMMIR_EL1), trap_raz_wi },
1525
1526 { SYS_DESC(SYS_MAIR_EL1), access_vm_reg, reset_unknown, MAIR_EL1 },
1527 { SYS_DESC(SYS_AMAIR_EL1), access_vm_reg, reset_amair_el1, AMAIR_EL1 },
1528
1529 { SYS_DESC(SYS_LORSA_EL1), trap_loregion },
1530 { SYS_DESC(SYS_LOREA_EL1), trap_loregion },
1531 { SYS_DESC(SYS_LORN_EL1), trap_loregion },
1532 { SYS_DESC(SYS_LORC_EL1), trap_loregion },
1533 { SYS_DESC(SYS_LORID_EL1), trap_loregion },
1534
1535 { SYS_DESC(SYS_VBAR_EL1), NULL, reset_val, VBAR_EL1, 0 },
1536 { SYS_DESC(SYS_DISR_EL1), NULL, reset_val, DISR_EL1, 0 },
1537
1538 { SYS_DESC(SYS_ICC_IAR0_EL1), write_to_read_only },
1539 { SYS_DESC(SYS_ICC_EOIR0_EL1), read_from_write_only },
1540 { SYS_DESC(SYS_ICC_HPPIR0_EL1), write_to_read_only },
1541 { SYS_DESC(SYS_ICC_DIR_EL1), read_from_write_only },
1542 { SYS_DESC(SYS_ICC_RPR_EL1), write_to_read_only },
1543 { SYS_DESC(SYS_ICC_SGI1R_EL1), access_gic_sgi },
1544 { SYS_DESC(SYS_ICC_ASGI1R_EL1), access_gic_sgi },
1545 { SYS_DESC(SYS_ICC_SGI0R_EL1), access_gic_sgi },
1546 { SYS_DESC(SYS_ICC_IAR1_EL1), write_to_read_only },
1547 { SYS_DESC(SYS_ICC_EOIR1_EL1), read_from_write_only },
1548 { SYS_DESC(SYS_ICC_HPPIR1_EL1), write_to_read_only },
1549 { SYS_DESC(SYS_ICC_SRE_EL1), access_gic_sre },
1550
1551 { SYS_DESC(SYS_CONTEXTIDR_EL1), access_vm_reg, reset_val, CONTEXTIDR_EL1, 0 },
1552 { SYS_DESC(SYS_TPIDR_EL1), NULL, reset_unknown, TPIDR_EL1 },
1553
1554 { SYS_DESC(SYS_SCXTNUM_EL1), undef_access },
1555
1556 { SYS_DESC(SYS_CNTKCTL_EL1), NULL, reset_val, CNTKCTL_EL1, 0},
1557
1558 { SYS_DESC(SYS_CCSIDR_EL1), access_ccsidr },
1559 { SYS_DESC(SYS_CLIDR_EL1), access_clidr },
1560 { SYS_DESC(SYS_CSSELR_EL1), access_csselr, reset_unknown, CSSELR_EL1 },
1561 { SYS_DESC(SYS_CTR_EL0), access_ctr },
1562
1563 { PMU_SYS_REG(SYS_PMCR_EL0), .access = access_pmcr,
1564 .reset = reset_pmcr, .reg = PMCR_EL0 },
1565 { PMU_SYS_REG(SYS_PMCNTENSET_EL0),
1566 .access = access_pmcnten, .reg = PMCNTENSET_EL0 },
1567 { PMU_SYS_REG(SYS_PMCNTENCLR_EL0),
1568 .access = access_pmcnten, .reg = PMCNTENSET_EL0 },
1569 { PMU_SYS_REG(SYS_PMOVSCLR_EL0),
1570 .access = access_pmovs, .reg = PMOVSSET_EL0 },
1571 { PMU_SYS_REG(SYS_PMSWINC_EL0),
1572 .access = access_pmswinc, .reg = PMSWINC_EL0 },
1573 { PMU_SYS_REG(SYS_PMSELR_EL0),
1574 .access = access_pmselr, .reg = PMSELR_EL0 },
1575 { PMU_SYS_REG(SYS_PMCEID0_EL0),
1576 .access = access_pmceid, .reset = NULL },
1577 { PMU_SYS_REG(SYS_PMCEID1_EL0),
1578 .access = access_pmceid, .reset = NULL },
1579 { PMU_SYS_REG(SYS_PMCCNTR_EL0),
1580 .access = access_pmu_evcntr, .reg = PMCCNTR_EL0 },
1581 { PMU_SYS_REG(SYS_PMXEVTYPER_EL0),
1582 .access = access_pmu_evtyper, .reset = NULL },
1583 { PMU_SYS_REG(SYS_PMXEVCNTR_EL0),
1584 .access = access_pmu_evcntr, .reset = NULL },
1585
1586
1587
1588
1589 { PMU_SYS_REG(SYS_PMUSERENR_EL0), .access = access_pmuserenr,
1590 .reset = reset_val, .reg = PMUSERENR_EL0, .val = 0 },
1591 { PMU_SYS_REG(SYS_PMOVSSET_EL0),
1592 .access = access_pmovs, .reg = PMOVSSET_EL0 },
1593
1594 { SYS_DESC(SYS_TPIDR_EL0), NULL, reset_unknown, TPIDR_EL0 },
1595 { SYS_DESC(SYS_TPIDRRO_EL0), NULL, reset_unknown, TPIDRRO_EL0 },
1596
1597 { SYS_DESC(SYS_SCXTNUM_EL0), undef_access },
1598
1599 { SYS_DESC(SYS_AMCR_EL0), undef_access },
1600 { SYS_DESC(SYS_AMCFGR_EL0), undef_access },
1601 { SYS_DESC(SYS_AMCGCR_EL0), undef_access },
1602 { SYS_DESC(SYS_AMUSERENR_EL0), undef_access },
1603 { SYS_DESC(SYS_AMCNTENCLR0_EL0), undef_access },
1604 { SYS_DESC(SYS_AMCNTENSET0_EL0), undef_access },
1605 { SYS_DESC(SYS_AMCNTENCLR1_EL0), undef_access },
1606 { SYS_DESC(SYS_AMCNTENSET1_EL0), undef_access },
1607 AMU_AMEVCNTR0_EL0(0),
1608 AMU_AMEVCNTR0_EL0(1),
1609 AMU_AMEVCNTR0_EL0(2),
1610 AMU_AMEVCNTR0_EL0(3),
1611 AMU_AMEVCNTR0_EL0(4),
1612 AMU_AMEVCNTR0_EL0(5),
1613 AMU_AMEVCNTR0_EL0(6),
1614 AMU_AMEVCNTR0_EL0(7),
1615 AMU_AMEVCNTR0_EL0(8),
1616 AMU_AMEVCNTR0_EL0(9),
1617 AMU_AMEVCNTR0_EL0(10),
1618 AMU_AMEVCNTR0_EL0(11),
1619 AMU_AMEVCNTR0_EL0(12),
1620 AMU_AMEVCNTR0_EL0(13),
1621 AMU_AMEVCNTR0_EL0(14),
1622 AMU_AMEVCNTR0_EL0(15),
1623 AMU_AMEVTYPER0_EL0(0),
1624 AMU_AMEVTYPER0_EL0(1),
1625 AMU_AMEVTYPER0_EL0(2),
1626 AMU_AMEVTYPER0_EL0(3),
1627 AMU_AMEVTYPER0_EL0(4),
1628 AMU_AMEVTYPER0_EL0(5),
1629 AMU_AMEVTYPER0_EL0(6),
1630 AMU_AMEVTYPER0_EL0(7),
1631 AMU_AMEVTYPER0_EL0(8),
1632 AMU_AMEVTYPER0_EL0(9),
1633 AMU_AMEVTYPER0_EL0(10),
1634 AMU_AMEVTYPER0_EL0(11),
1635 AMU_AMEVTYPER0_EL0(12),
1636 AMU_AMEVTYPER0_EL0(13),
1637 AMU_AMEVTYPER0_EL0(14),
1638 AMU_AMEVTYPER0_EL0(15),
1639 AMU_AMEVCNTR1_EL0(0),
1640 AMU_AMEVCNTR1_EL0(1),
1641 AMU_AMEVCNTR1_EL0(2),
1642 AMU_AMEVCNTR1_EL0(3),
1643 AMU_AMEVCNTR1_EL0(4),
1644 AMU_AMEVCNTR1_EL0(5),
1645 AMU_AMEVCNTR1_EL0(6),
1646 AMU_AMEVCNTR1_EL0(7),
1647 AMU_AMEVCNTR1_EL0(8),
1648 AMU_AMEVCNTR1_EL0(9),
1649 AMU_AMEVCNTR1_EL0(10),
1650 AMU_AMEVCNTR1_EL0(11),
1651 AMU_AMEVCNTR1_EL0(12),
1652 AMU_AMEVCNTR1_EL0(13),
1653 AMU_AMEVCNTR1_EL0(14),
1654 AMU_AMEVCNTR1_EL0(15),
1655 AMU_AMEVTYPER1_EL0(0),
1656 AMU_AMEVTYPER1_EL0(1),
1657 AMU_AMEVTYPER1_EL0(2),
1658 AMU_AMEVTYPER1_EL0(3),
1659 AMU_AMEVTYPER1_EL0(4),
1660 AMU_AMEVTYPER1_EL0(5),
1661 AMU_AMEVTYPER1_EL0(6),
1662 AMU_AMEVTYPER1_EL0(7),
1663 AMU_AMEVTYPER1_EL0(8),
1664 AMU_AMEVTYPER1_EL0(9),
1665 AMU_AMEVTYPER1_EL0(10),
1666 AMU_AMEVTYPER1_EL0(11),
1667 AMU_AMEVTYPER1_EL0(12),
1668 AMU_AMEVTYPER1_EL0(13),
1669 AMU_AMEVTYPER1_EL0(14),
1670 AMU_AMEVTYPER1_EL0(15),
1671
1672 { SYS_DESC(SYS_CNTP_TVAL_EL0), access_arch_timer },
1673 { SYS_DESC(SYS_CNTP_CTL_EL0), access_arch_timer },
1674 { SYS_DESC(SYS_CNTP_CVAL_EL0), access_arch_timer },
1675
1676
1677 PMU_PMEVCNTR_EL0(0),
1678 PMU_PMEVCNTR_EL0(1),
1679 PMU_PMEVCNTR_EL0(2),
1680 PMU_PMEVCNTR_EL0(3),
1681 PMU_PMEVCNTR_EL0(4),
1682 PMU_PMEVCNTR_EL0(5),
1683 PMU_PMEVCNTR_EL0(6),
1684 PMU_PMEVCNTR_EL0(7),
1685 PMU_PMEVCNTR_EL0(8),
1686 PMU_PMEVCNTR_EL0(9),
1687 PMU_PMEVCNTR_EL0(10),
1688 PMU_PMEVCNTR_EL0(11),
1689 PMU_PMEVCNTR_EL0(12),
1690 PMU_PMEVCNTR_EL0(13),
1691 PMU_PMEVCNTR_EL0(14),
1692 PMU_PMEVCNTR_EL0(15),
1693 PMU_PMEVCNTR_EL0(16),
1694 PMU_PMEVCNTR_EL0(17),
1695 PMU_PMEVCNTR_EL0(18),
1696 PMU_PMEVCNTR_EL0(19),
1697 PMU_PMEVCNTR_EL0(20),
1698 PMU_PMEVCNTR_EL0(21),
1699 PMU_PMEVCNTR_EL0(22),
1700 PMU_PMEVCNTR_EL0(23),
1701 PMU_PMEVCNTR_EL0(24),
1702 PMU_PMEVCNTR_EL0(25),
1703 PMU_PMEVCNTR_EL0(26),
1704 PMU_PMEVCNTR_EL0(27),
1705 PMU_PMEVCNTR_EL0(28),
1706 PMU_PMEVCNTR_EL0(29),
1707 PMU_PMEVCNTR_EL0(30),
1708
1709 PMU_PMEVTYPER_EL0(0),
1710 PMU_PMEVTYPER_EL0(1),
1711 PMU_PMEVTYPER_EL0(2),
1712 PMU_PMEVTYPER_EL0(3),
1713 PMU_PMEVTYPER_EL0(4),
1714 PMU_PMEVTYPER_EL0(5),
1715 PMU_PMEVTYPER_EL0(6),
1716 PMU_PMEVTYPER_EL0(7),
1717 PMU_PMEVTYPER_EL0(8),
1718 PMU_PMEVTYPER_EL0(9),
1719 PMU_PMEVTYPER_EL0(10),
1720 PMU_PMEVTYPER_EL0(11),
1721 PMU_PMEVTYPER_EL0(12),
1722 PMU_PMEVTYPER_EL0(13),
1723 PMU_PMEVTYPER_EL0(14),
1724 PMU_PMEVTYPER_EL0(15),
1725 PMU_PMEVTYPER_EL0(16),
1726 PMU_PMEVTYPER_EL0(17),
1727 PMU_PMEVTYPER_EL0(18),
1728 PMU_PMEVTYPER_EL0(19),
1729 PMU_PMEVTYPER_EL0(20),
1730 PMU_PMEVTYPER_EL0(21),
1731 PMU_PMEVTYPER_EL0(22),
1732 PMU_PMEVTYPER_EL0(23),
1733 PMU_PMEVTYPER_EL0(24),
1734 PMU_PMEVTYPER_EL0(25),
1735 PMU_PMEVTYPER_EL0(26),
1736 PMU_PMEVTYPER_EL0(27),
1737 PMU_PMEVTYPER_EL0(28),
1738 PMU_PMEVTYPER_EL0(29),
1739 PMU_PMEVTYPER_EL0(30),
1740
1741
1742
1743
1744 { PMU_SYS_REG(SYS_PMCCFILTR_EL0), .access = access_pmu_evtyper,
1745 .reset = reset_val, .reg = PMCCFILTR_EL0, .val = 0 },
1746
1747 { SYS_DESC(SYS_DACR32_EL2), NULL, reset_unknown, DACR32_EL2 },
1748 { SYS_DESC(SYS_IFSR32_EL2), NULL, reset_unknown, IFSR32_EL2 },
1749 { SYS_DESC(SYS_FPEXC32_EL2), NULL, reset_val, FPEXC32_EL2, 0x700 },
1750};
1751
1752static bool trap_dbgdidr(struct kvm_vcpu *vcpu,
1753 struct sys_reg_params *p,
1754 const struct sys_reg_desc *r)
1755{
1756 if (p->is_write) {
1757 return ignore_write(vcpu, p);
1758 } else {
1759 u64 dfr = read_sanitised_ftr_reg(SYS_ID_AA64DFR0_EL1);
1760 u64 pfr = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1);
1761 u32 el3 = !!cpuid_feature_extract_unsigned_field(pfr, ID_AA64PFR0_EL3_SHIFT);
1762
1763 p->regval = ((((dfr >> ID_AA64DFR0_WRPS_SHIFT) & 0xf) << 28) |
1764 (((dfr >> ID_AA64DFR0_BRPS_SHIFT) & 0xf) << 24) |
1765 (((dfr >> ID_AA64DFR0_CTX_CMPS_SHIFT) & 0xf) << 20)
1766 | (6 << 16) | (1 << 15) | (el3 << 14) | (el3 << 12));
1767 return true;
1768 }
1769}
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780#define DBG_BCR_BVR_WCR_WVR(n) \
1781 \
1782 { AA32(LO), Op1( 0), CRn( 0), CRm((n)), Op2( 4), trap_bvr, NULL, n }, \
1783 \
1784 { Op1( 0), CRn( 0), CRm((n)), Op2( 5), trap_bcr, NULL, n }, \
1785 \
1786 { Op1( 0), CRn( 0), CRm((n)), Op2( 6), trap_wvr, NULL, n }, \
1787 \
1788 { Op1( 0), CRn( 0), CRm((n)), Op2( 7), trap_wcr, NULL, n }
1789
1790#define DBGBXVR(n) \
1791 { AA32(HI), Op1( 0), CRn( 1), CRm((n)), Op2( 1), trap_bvr, NULL, n }
1792
1793
1794
1795
1796
1797
1798static const struct sys_reg_desc cp14_regs[] = {
1799
1800 { Op1( 0), CRn( 0), CRm( 0), Op2( 0), trap_dbgdidr },
1801
1802 { Op1( 0), CRn( 0), CRm( 0), Op2( 2), trap_raz_wi },
1803
1804 DBG_BCR_BVR_WCR_WVR(0),
1805
1806 { Op1( 0), CRn( 0), CRm( 1), Op2( 0), trap_raz_wi },
1807 DBG_BCR_BVR_WCR_WVR(1),
1808
1809 { Op1( 0), CRn( 0), CRm( 2), Op2( 0), trap_debug_regs, NULL, MDCCINT_EL1 },
1810
1811 { Op1( 0), CRn( 0), CRm( 2), Op2( 2), trap_debug_regs, NULL, MDSCR_EL1 },
1812 DBG_BCR_BVR_WCR_WVR(2),
1813
1814 { Op1( 0), CRn( 0), CRm( 3), Op2( 0), trap_raz_wi },
1815
1816 { Op1( 0), CRn( 0), CRm( 3), Op2( 2), trap_raz_wi },
1817 DBG_BCR_BVR_WCR_WVR(3),
1818 DBG_BCR_BVR_WCR_WVR(4),
1819 DBG_BCR_BVR_WCR_WVR(5),
1820
1821 { Op1( 0), CRn( 0), CRm( 6), Op2( 0), trap_raz_wi },
1822
1823 { Op1( 0), CRn( 0), CRm( 6), Op2( 2), trap_raz_wi },
1824 DBG_BCR_BVR_WCR_WVR(6),
1825
1826 { Op1( 0), CRn( 0), CRm( 7), Op2( 0), trap_debug_regs, NULL, DBGVCR32_EL2 },
1827 DBG_BCR_BVR_WCR_WVR(7),
1828 DBG_BCR_BVR_WCR_WVR(8),
1829 DBG_BCR_BVR_WCR_WVR(9),
1830 DBG_BCR_BVR_WCR_WVR(10),
1831 DBG_BCR_BVR_WCR_WVR(11),
1832 DBG_BCR_BVR_WCR_WVR(12),
1833 DBG_BCR_BVR_WCR_WVR(13),
1834 DBG_BCR_BVR_WCR_WVR(14),
1835 DBG_BCR_BVR_WCR_WVR(15),
1836
1837
1838 { Op1( 0), CRn( 1), CRm( 0), Op2( 0), trap_raz_wi },
1839
1840 DBGBXVR(0),
1841
1842 { Op1( 0), CRn( 1), CRm( 0), Op2( 4), trap_raz_wi },
1843 DBGBXVR(1),
1844
1845 { Op1( 0), CRn( 1), CRm( 1), Op2( 4), trap_oslsr_el1 },
1846 DBGBXVR(2),
1847 DBGBXVR(3),
1848
1849 { Op1( 0), CRn( 1), CRm( 3), Op2( 4), trap_raz_wi },
1850 DBGBXVR(4),
1851
1852 { Op1( 0), CRn( 1), CRm( 4), Op2( 4), trap_raz_wi },
1853 DBGBXVR(5),
1854 DBGBXVR(6),
1855 DBGBXVR(7),
1856 DBGBXVR(8),
1857 DBGBXVR(9),
1858 DBGBXVR(10),
1859 DBGBXVR(11),
1860 DBGBXVR(12),
1861 DBGBXVR(13),
1862 DBGBXVR(14),
1863 DBGBXVR(15),
1864
1865
1866 { Op1( 0), CRn( 2), CRm( 0), Op2( 0), trap_raz_wi },
1867
1868
1869 { Op1( 0), CRn( 7), CRm( 0), Op2( 7), trap_raz_wi },
1870
1871 { Op1( 0), CRn( 7), CRm( 1), Op2( 7), trap_raz_wi },
1872
1873 { Op1( 0), CRn( 7), CRm( 2), Op2( 7), trap_raz_wi },
1874
1875 { Op1( 0), CRn( 7), CRm( 8), Op2( 6), trap_raz_wi },
1876
1877 { Op1( 0), CRn( 7), CRm( 9), Op2( 6), trap_raz_wi },
1878
1879 { Op1( 0), CRn( 7), CRm(14), Op2( 6), trap_dbgauthstatus_el1 },
1880};
1881
1882
1883static const struct sys_reg_desc cp14_64_regs[] = {
1884
1885 { Op1( 0), CRm( 1), .access = trap_raz_wi },
1886
1887
1888 { Op1( 0), CRm( 2), .access = trap_raz_wi },
1889};
1890
1891
1892#define PMU_PMEVCNTR(n) \
1893 \
1894 { Op1(0), CRn(0b1110), \
1895 CRm((0b1000 | (((n) >> 3) & 0x3))), Op2(((n) & 0x7)), \
1896 access_pmu_evcntr }
1897
1898
1899#define PMU_PMEVTYPER(n) \
1900 \
1901 { Op1(0), CRn(0b1110), \
1902 CRm((0b1100 | (((n) >> 3) & 0x3))), Op2(((n) & 0x7)), \
1903 access_pmu_evtyper }
1904
1905
1906
1907
1908
1909
1910static const struct sys_reg_desc cp15_regs[] = {
1911 { Op1( 0), CRn( 0), CRm( 0), Op2( 1), access_ctr },
1912 { Op1( 0), CRn( 1), CRm( 0), Op2( 0), access_vm_reg, NULL, SCTLR_EL1 },
1913
1914 { AA32(LO), Op1( 0), CRn( 1), CRm( 0), Op2( 1), access_actlr, NULL, ACTLR_EL1 },
1915
1916 { AA32(HI), Op1( 0), CRn( 1), CRm( 0), Op2( 3), access_actlr, NULL, ACTLR_EL1 },
1917 { Op1( 0), CRn( 2), CRm( 0), Op2( 0), access_vm_reg, NULL, TTBR0_EL1 },
1918 { Op1( 0), CRn( 2), CRm( 0), Op2( 1), access_vm_reg, NULL, TTBR1_EL1 },
1919
1920 { AA32(LO), Op1( 0), CRn( 2), CRm( 0), Op2( 2), access_vm_reg, NULL, TCR_EL1 },
1921
1922 { AA32(HI), Op1( 0), CRn( 2), CRm( 0), Op2( 3), access_vm_reg, NULL, TCR_EL1 },
1923 { Op1( 0), CRn( 3), CRm( 0), Op2( 0), access_vm_reg, NULL, DACR32_EL2 },
1924
1925 { Op1( 0), CRn( 5), CRm( 0), Op2( 0), access_vm_reg, NULL, ESR_EL1 },
1926 { Op1( 0), CRn( 5), CRm( 0), Op2( 1), access_vm_reg, NULL, IFSR32_EL2 },
1927
1928 { Op1( 0), CRn( 5), CRm( 1), Op2( 0), access_vm_reg, NULL, AFSR0_EL1 },
1929
1930 { Op1( 0), CRn( 5), CRm( 1), Op2( 1), access_vm_reg, NULL, AFSR1_EL1 },
1931
1932 { AA32(LO), Op1( 0), CRn( 6), CRm( 0), Op2( 0), access_vm_reg, NULL, FAR_EL1 },
1933
1934 { AA32(HI), Op1( 0), CRn( 6), CRm( 0), Op2( 2), access_vm_reg, NULL, FAR_EL1 },
1935
1936
1937
1938
1939 { Op1( 0), CRn( 7), CRm( 6), Op2( 2), access_dcsw },
1940 { Op1( 0), CRn( 7), CRm(10), Op2( 2), access_dcsw },
1941 { Op1( 0), CRn( 7), CRm(14), Op2( 2), access_dcsw },
1942
1943
1944 { Op1( 0), CRn( 9), CRm(12), Op2( 0), access_pmcr },
1945 { Op1( 0), CRn( 9), CRm(12), Op2( 1), access_pmcnten },
1946 { Op1( 0), CRn( 9), CRm(12), Op2( 2), access_pmcnten },
1947 { Op1( 0), CRn( 9), CRm(12), Op2( 3), access_pmovs },
1948 { Op1( 0), CRn( 9), CRm(12), Op2( 4), access_pmswinc },
1949 { Op1( 0), CRn( 9), CRm(12), Op2( 5), access_pmselr },
1950 { AA32(LO), Op1( 0), CRn( 9), CRm(12), Op2( 6), access_pmceid },
1951 { AA32(LO), Op1( 0), CRn( 9), CRm(12), Op2( 7), access_pmceid },
1952 { Op1( 0), CRn( 9), CRm(13), Op2( 0), access_pmu_evcntr },
1953 { Op1( 0), CRn( 9), CRm(13), Op2( 1), access_pmu_evtyper },
1954 { Op1( 0), CRn( 9), CRm(13), Op2( 2), access_pmu_evcntr },
1955 { Op1( 0), CRn( 9), CRm(14), Op2( 0), access_pmuserenr },
1956 { Op1( 0), CRn( 9), CRm(14), Op2( 1), access_pminten },
1957 { Op1( 0), CRn( 9), CRm(14), Op2( 2), access_pminten },
1958 { Op1( 0), CRn( 9), CRm(14), Op2( 3), access_pmovs },
1959 { AA32(HI), Op1( 0), CRn( 9), CRm(14), Op2( 4), access_pmceid },
1960 { AA32(HI), Op1( 0), CRn( 9), CRm(14), Op2( 5), access_pmceid },
1961
1962 { Op1( 0), CRn( 9), CRm(14), Op2( 6), trap_raz_wi },
1963
1964
1965 { AA32(LO), Op1( 0), CRn(10), CRm( 2), Op2( 0), access_vm_reg, NULL, MAIR_EL1 },
1966
1967 { AA32(HI), Op1( 0), CRn(10), CRm( 2), Op2( 1), access_vm_reg, NULL, MAIR_EL1 },
1968
1969 { AA32(LO), Op1( 0), CRn(10), CRm( 3), Op2( 0), access_vm_reg, NULL, AMAIR_EL1 },
1970
1971 { AA32(HI), Op1( 0), CRn(10), CRm( 3), Op2( 1), access_vm_reg, NULL, AMAIR_EL1 },
1972
1973
1974 { Op1( 0), CRn(12), CRm(12), Op2( 5), access_gic_sre },
1975
1976 { Op1( 0), CRn(13), CRm( 0), Op2( 1), access_vm_reg, NULL, CONTEXTIDR_EL1 },
1977
1978
1979 { SYS_DESC(SYS_AARCH32_CNTP_TVAL), access_arch_timer },
1980 { SYS_DESC(SYS_AARCH32_CNTP_CTL), access_arch_timer },
1981
1982
1983 PMU_PMEVCNTR(0),
1984 PMU_PMEVCNTR(1),
1985 PMU_PMEVCNTR(2),
1986 PMU_PMEVCNTR(3),
1987 PMU_PMEVCNTR(4),
1988 PMU_PMEVCNTR(5),
1989 PMU_PMEVCNTR(6),
1990 PMU_PMEVCNTR(7),
1991 PMU_PMEVCNTR(8),
1992 PMU_PMEVCNTR(9),
1993 PMU_PMEVCNTR(10),
1994 PMU_PMEVCNTR(11),
1995 PMU_PMEVCNTR(12),
1996 PMU_PMEVCNTR(13),
1997 PMU_PMEVCNTR(14),
1998 PMU_PMEVCNTR(15),
1999 PMU_PMEVCNTR(16),
2000 PMU_PMEVCNTR(17),
2001 PMU_PMEVCNTR(18),
2002 PMU_PMEVCNTR(19),
2003 PMU_PMEVCNTR(20),
2004 PMU_PMEVCNTR(21),
2005 PMU_PMEVCNTR(22),
2006 PMU_PMEVCNTR(23),
2007 PMU_PMEVCNTR(24),
2008 PMU_PMEVCNTR(25),
2009 PMU_PMEVCNTR(26),
2010 PMU_PMEVCNTR(27),
2011 PMU_PMEVCNTR(28),
2012 PMU_PMEVCNTR(29),
2013 PMU_PMEVCNTR(30),
2014
2015 PMU_PMEVTYPER(0),
2016 PMU_PMEVTYPER(1),
2017 PMU_PMEVTYPER(2),
2018 PMU_PMEVTYPER(3),
2019 PMU_PMEVTYPER(4),
2020 PMU_PMEVTYPER(5),
2021 PMU_PMEVTYPER(6),
2022 PMU_PMEVTYPER(7),
2023 PMU_PMEVTYPER(8),
2024 PMU_PMEVTYPER(9),
2025 PMU_PMEVTYPER(10),
2026 PMU_PMEVTYPER(11),
2027 PMU_PMEVTYPER(12),
2028 PMU_PMEVTYPER(13),
2029 PMU_PMEVTYPER(14),
2030 PMU_PMEVTYPER(15),
2031 PMU_PMEVTYPER(16),
2032 PMU_PMEVTYPER(17),
2033 PMU_PMEVTYPER(18),
2034 PMU_PMEVTYPER(19),
2035 PMU_PMEVTYPER(20),
2036 PMU_PMEVTYPER(21),
2037 PMU_PMEVTYPER(22),
2038 PMU_PMEVTYPER(23),
2039 PMU_PMEVTYPER(24),
2040 PMU_PMEVTYPER(25),
2041 PMU_PMEVTYPER(26),
2042 PMU_PMEVTYPER(27),
2043 PMU_PMEVTYPER(28),
2044 PMU_PMEVTYPER(29),
2045 PMU_PMEVTYPER(30),
2046
2047 { Op1(0), CRn(14), CRm(15), Op2(7), access_pmu_evtyper },
2048
2049 { Op1(1), CRn( 0), CRm( 0), Op2(0), access_ccsidr },
2050 { Op1(1), CRn( 0), CRm( 0), Op2(1), access_clidr },
2051 { Op1(2), CRn( 0), CRm( 0), Op2(0), access_csselr, NULL, CSSELR_EL1 },
2052};
2053
2054static const struct sys_reg_desc cp15_64_regs[] = {
2055 { Op1( 0), CRn( 0), CRm( 2), Op2( 0), access_vm_reg, NULL, TTBR0_EL1 },
2056 { Op1( 0), CRn( 0), CRm( 9), Op2( 0), access_pmu_evcntr },
2057 { Op1( 0), CRn( 0), CRm(12), Op2( 0), access_gic_sgi },
2058 { Op1( 1), CRn( 0), CRm( 2), Op2( 0), access_vm_reg, NULL, TTBR1_EL1 },
2059 { Op1( 1), CRn( 0), CRm(12), Op2( 0), access_gic_sgi },
2060 { Op1( 2), CRn( 0), CRm(12), Op2( 0), access_gic_sgi },
2061 { SYS_DESC(SYS_AARCH32_CNTP_CVAL), access_arch_timer },
2062};
2063
2064static int check_sysreg_table(const struct sys_reg_desc *table, unsigned int n,
2065 bool is_32)
2066{
2067 unsigned int i;
2068
2069 for (i = 0; i < n; i++) {
2070 if (!is_32 && table[i].reg && !table[i].reset) {
2071 kvm_err("sys_reg table %p entry %d has lacks reset\n",
2072 table, i);
2073 return 1;
2074 }
2075
2076 if (i && cmp_sys_reg(&table[i-1], &table[i]) >= 0) {
2077 kvm_err("sys_reg table %p out of order (%d)\n", table, i - 1);
2078 return 1;
2079 }
2080 }
2081
2082 return 0;
2083}
2084
2085static int match_sys_reg(const void *key, const void *elt)
2086{
2087 const unsigned long pval = (unsigned long)key;
2088 const struct sys_reg_desc *r = elt;
2089
2090 return pval - reg_to_encoding(r);
2091}
2092
2093static const struct sys_reg_desc *find_reg(const struct sys_reg_params *params,
2094 const struct sys_reg_desc table[],
2095 unsigned int num)
2096{
2097 unsigned long pval = reg_to_encoding(params);
2098
2099 return bsearch((void *)pval, table, num, sizeof(table[0]), match_sys_reg);
2100}
2101
2102int kvm_handle_cp14_load_store(struct kvm_vcpu *vcpu)
2103{
2104 kvm_inject_undefined(vcpu);
2105 return 1;
2106}
2107
2108static void perform_access(struct kvm_vcpu *vcpu,
2109 struct sys_reg_params *params,
2110 const struct sys_reg_desc *r)
2111{
2112 trace_kvm_sys_access(*vcpu_pc(vcpu), params, r);
2113
2114
2115 if (sysreg_hidden(vcpu, r)) {
2116 kvm_inject_undefined(vcpu);
2117 return;
2118 }
2119
2120
2121
2122
2123
2124
2125 BUG_ON(!r->access);
2126
2127
2128 if (likely(r->access(vcpu, params, r)))
2129 kvm_incr_pc(vcpu);
2130}
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142static int emulate_cp(struct kvm_vcpu *vcpu,
2143 struct sys_reg_params *params,
2144 const struct sys_reg_desc *table,
2145 size_t num)
2146{
2147 const struct sys_reg_desc *r;
2148
2149 if (!table)
2150 return -1;
2151
2152 r = find_reg(params, table, num);
2153
2154 if (r) {
2155 perform_access(vcpu, params, r);
2156 return 0;
2157 }
2158
2159
2160 return -1;
2161}
2162
2163static void unhandled_cp_access(struct kvm_vcpu *vcpu,
2164 struct sys_reg_params *params)
2165{
2166 u8 esr_ec = kvm_vcpu_trap_get_class(vcpu);
2167 int cp = -1;
2168
2169 switch (esr_ec) {
2170 case ESR_ELx_EC_CP15_32:
2171 case ESR_ELx_EC_CP15_64:
2172 cp = 15;
2173 break;
2174 case ESR_ELx_EC_CP14_MR:
2175 case ESR_ELx_EC_CP14_64:
2176 cp = 14;
2177 break;
2178 default:
2179 WARN_ON(1);
2180 }
2181
2182 print_sys_reg_msg(params,
2183 "Unsupported guest CP%d access at: %08lx [%08lx]\n",
2184 cp, *vcpu_pc(vcpu), *vcpu_cpsr(vcpu));
2185 kvm_inject_undefined(vcpu);
2186}
2187
2188
2189
2190
2191
2192
2193static int kvm_handle_cp_64(struct kvm_vcpu *vcpu,
2194 const struct sys_reg_desc *global,
2195 size_t nr_global)
2196{
2197 struct sys_reg_params params;
2198 u32 esr = kvm_vcpu_get_esr(vcpu);
2199 int Rt = kvm_vcpu_sys_get_rt(vcpu);
2200 int Rt2 = (esr >> 10) & 0x1f;
2201
2202 params.CRm = (esr >> 1) & 0xf;
2203 params.is_write = ((esr & 1) == 0);
2204
2205 params.Op0 = 0;
2206 params.Op1 = (esr >> 16) & 0xf;
2207 params.Op2 = 0;
2208 params.CRn = 0;
2209
2210
2211
2212
2213
2214 if (params.is_write) {
2215 params.regval = vcpu_get_reg(vcpu, Rt) & 0xffffffff;
2216 params.regval |= vcpu_get_reg(vcpu, Rt2) << 32;
2217 }
2218
2219
2220
2221
2222
2223
2224 if (!emulate_cp(vcpu, ¶ms, global, nr_global)) {
2225
2226 if (!params.is_write) {
2227 vcpu_set_reg(vcpu, Rt, lower_32_bits(params.regval));
2228 vcpu_set_reg(vcpu, Rt2, upper_32_bits(params.regval));
2229 }
2230
2231 return 1;
2232 }
2233
2234 unhandled_cp_access(vcpu, ¶ms);
2235 return 1;
2236}
2237
2238
2239
2240
2241
2242
2243static int kvm_handle_cp_32(struct kvm_vcpu *vcpu,
2244 const struct sys_reg_desc *global,
2245 size_t nr_global)
2246{
2247 struct sys_reg_params params;
2248 u32 esr = kvm_vcpu_get_esr(vcpu);
2249 int Rt = kvm_vcpu_sys_get_rt(vcpu);
2250
2251 params.CRm = (esr >> 1) & 0xf;
2252 params.regval = vcpu_get_reg(vcpu, Rt);
2253 params.is_write = ((esr & 1) == 0);
2254 params.CRn = (esr >> 10) & 0xf;
2255 params.Op0 = 0;
2256 params.Op1 = (esr >> 14) & 0x7;
2257 params.Op2 = (esr >> 17) & 0x7;
2258
2259 if (!emulate_cp(vcpu, ¶ms, global, nr_global)) {
2260 if (!params.is_write)
2261 vcpu_set_reg(vcpu, Rt, params.regval);
2262 return 1;
2263 }
2264
2265 unhandled_cp_access(vcpu, ¶ms);
2266 return 1;
2267}
2268
2269int kvm_handle_cp15_64(struct kvm_vcpu *vcpu)
2270{
2271 return kvm_handle_cp_64(vcpu, cp15_64_regs, ARRAY_SIZE(cp15_64_regs));
2272}
2273
2274int kvm_handle_cp15_32(struct kvm_vcpu *vcpu)
2275{
2276 return kvm_handle_cp_32(vcpu, cp15_regs, ARRAY_SIZE(cp15_regs));
2277}
2278
2279int kvm_handle_cp14_64(struct kvm_vcpu *vcpu)
2280{
2281 return kvm_handle_cp_64(vcpu, cp14_64_regs, ARRAY_SIZE(cp14_64_regs));
2282}
2283
2284int kvm_handle_cp14_32(struct kvm_vcpu *vcpu)
2285{
2286 return kvm_handle_cp_32(vcpu, cp14_regs, ARRAY_SIZE(cp14_regs));
2287}
2288
2289static bool is_imp_def_sys_reg(struct sys_reg_params *params)
2290{
2291
2292 return params->Op0 == 3 && (params->CRn & 0b1011) == 0b1011;
2293}
2294
2295static int emulate_sys_reg(struct kvm_vcpu *vcpu,
2296 struct sys_reg_params *params)
2297{
2298 const struct sys_reg_desc *r;
2299
2300 r = find_reg(params, sys_reg_descs, ARRAY_SIZE(sys_reg_descs));
2301
2302 if (likely(r)) {
2303 perform_access(vcpu, params, r);
2304 } else if (is_imp_def_sys_reg(params)) {
2305 kvm_inject_undefined(vcpu);
2306 } else {
2307 print_sys_reg_msg(params,
2308 "Unsupported guest sys_reg access at: %lx [%08lx]\n",
2309 *vcpu_pc(vcpu), *vcpu_cpsr(vcpu));
2310 kvm_inject_undefined(vcpu);
2311 }
2312 return 1;
2313}
2314
2315
2316
2317
2318
2319
2320
2321
2322void kvm_reset_sys_regs(struct kvm_vcpu *vcpu)
2323{
2324 unsigned long i;
2325
2326 for (i = 0; i < ARRAY_SIZE(sys_reg_descs); i++)
2327 if (sys_reg_descs[i].reset)
2328 sys_reg_descs[i].reset(vcpu, &sys_reg_descs[i]);
2329}
2330
2331
2332
2333
2334
2335int kvm_handle_sys_reg(struct kvm_vcpu *vcpu)
2336{
2337 struct sys_reg_params params;
2338 unsigned long esr = kvm_vcpu_get_esr(vcpu);
2339 int Rt = kvm_vcpu_sys_get_rt(vcpu);
2340 int ret;
2341
2342 trace_kvm_handle_sys_reg(esr);
2343
2344 params.Op0 = (esr >> 20) & 3;
2345 params.Op1 = (esr >> 14) & 0x7;
2346 params.CRn = (esr >> 10) & 0xf;
2347 params.CRm = (esr >> 1) & 0xf;
2348 params.Op2 = (esr >> 17) & 0x7;
2349 params.regval = vcpu_get_reg(vcpu, Rt);
2350 params.is_write = !(esr & 1);
2351
2352 ret = emulate_sys_reg(vcpu, ¶ms);
2353
2354 if (!params.is_write)
2355 vcpu_set_reg(vcpu, Rt, params.regval);
2356 return ret;
2357}
2358
2359
2360
2361
2362
2363static bool index_to_params(u64 id, struct sys_reg_params *params)
2364{
2365 switch (id & KVM_REG_SIZE_MASK) {
2366 case KVM_REG_SIZE_U64:
2367
2368 if (id & ~(KVM_REG_ARCH_MASK | KVM_REG_SIZE_MASK
2369 | KVM_REG_ARM_COPROC_MASK
2370 | KVM_REG_ARM64_SYSREG_OP0_MASK
2371 | KVM_REG_ARM64_SYSREG_OP1_MASK
2372 | KVM_REG_ARM64_SYSREG_CRN_MASK
2373 | KVM_REG_ARM64_SYSREG_CRM_MASK
2374 | KVM_REG_ARM64_SYSREG_OP2_MASK))
2375 return false;
2376 params->Op0 = ((id & KVM_REG_ARM64_SYSREG_OP0_MASK)
2377 >> KVM_REG_ARM64_SYSREG_OP0_SHIFT);
2378 params->Op1 = ((id & KVM_REG_ARM64_SYSREG_OP1_MASK)
2379 >> KVM_REG_ARM64_SYSREG_OP1_SHIFT);
2380 params->CRn = ((id & KVM_REG_ARM64_SYSREG_CRN_MASK)
2381 >> KVM_REG_ARM64_SYSREG_CRN_SHIFT);
2382 params->CRm = ((id & KVM_REG_ARM64_SYSREG_CRM_MASK)
2383 >> KVM_REG_ARM64_SYSREG_CRM_SHIFT);
2384 params->Op2 = ((id & KVM_REG_ARM64_SYSREG_OP2_MASK)
2385 >> KVM_REG_ARM64_SYSREG_OP2_SHIFT);
2386 return true;
2387 default:
2388 return false;
2389 }
2390}
2391
2392const struct sys_reg_desc *find_reg_by_id(u64 id,
2393 struct sys_reg_params *params,
2394 const struct sys_reg_desc table[],
2395 unsigned int num)
2396{
2397 if (!index_to_params(id, params))
2398 return NULL;
2399
2400 return find_reg(params, table, num);
2401}
2402
2403
2404static const struct sys_reg_desc *index_to_sys_reg_desc(struct kvm_vcpu *vcpu,
2405 u64 id)
2406{
2407 const struct sys_reg_desc *r;
2408 struct sys_reg_params params;
2409
2410
2411 if ((id & KVM_REG_ARM_COPROC_MASK) != KVM_REG_ARM64_SYSREG)
2412 return NULL;
2413
2414 if (!index_to_params(id, ¶ms))
2415 return NULL;
2416
2417 r = find_reg(¶ms, sys_reg_descs, ARRAY_SIZE(sys_reg_descs));
2418
2419
2420 if (r && !(r->reg || r->get_user))
2421 r = NULL;
2422
2423 return r;
2424}
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434#define FUNCTION_INVARIANT(reg) \
2435 static void get_##reg(struct kvm_vcpu *v, \
2436 const struct sys_reg_desc *r) \
2437 { \
2438 ((struct sys_reg_desc *)r)->val = read_sysreg(reg); \
2439 }
2440
2441FUNCTION_INVARIANT(midr_el1)
2442FUNCTION_INVARIANT(revidr_el1)
2443FUNCTION_INVARIANT(clidr_el1)
2444FUNCTION_INVARIANT(aidr_el1)
2445
2446static void get_ctr_el0(struct kvm_vcpu *v, const struct sys_reg_desc *r)
2447{
2448 ((struct sys_reg_desc *)r)->val = read_sanitised_ftr_reg(SYS_CTR_EL0);
2449}
2450
2451
2452static struct sys_reg_desc invariant_sys_regs[] = {
2453 { SYS_DESC(SYS_MIDR_EL1), NULL, get_midr_el1 },
2454 { SYS_DESC(SYS_REVIDR_EL1), NULL, get_revidr_el1 },
2455 { SYS_DESC(SYS_CLIDR_EL1), NULL, get_clidr_el1 },
2456 { SYS_DESC(SYS_AIDR_EL1), NULL, get_aidr_el1 },
2457 { SYS_DESC(SYS_CTR_EL0), NULL, get_ctr_el0 },
2458};
2459
2460static int reg_from_user(u64 *val, const void __user *uaddr, u64 id)
2461{
2462 if (copy_from_user(val, uaddr, KVM_REG_SIZE(id)) != 0)
2463 return -EFAULT;
2464 return 0;
2465}
2466
2467static int reg_to_user(void __user *uaddr, const u64 *val, u64 id)
2468{
2469 if (copy_to_user(uaddr, val, KVM_REG_SIZE(id)) != 0)
2470 return -EFAULT;
2471 return 0;
2472}
2473
2474static int get_invariant_sys_reg(u64 id, void __user *uaddr)
2475{
2476 struct sys_reg_params params;
2477 const struct sys_reg_desc *r;
2478
2479 r = find_reg_by_id(id, ¶ms, invariant_sys_regs,
2480 ARRAY_SIZE(invariant_sys_regs));
2481 if (!r)
2482 return -ENOENT;
2483
2484 return reg_to_user(uaddr, &r->val, id);
2485}
2486
2487static int set_invariant_sys_reg(u64 id, void __user *uaddr)
2488{
2489 struct sys_reg_params params;
2490 const struct sys_reg_desc *r;
2491 int err;
2492 u64 val = 0;
2493
2494 r = find_reg_by_id(id, ¶ms, invariant_sys_regs,
2495 ARRAY_SIZE(invariant_sys_regs));
2496 if (!r)
2497 return -ENOENT;
2498
2499 err = reg_from_user(&val, uaddr, id);
2500 if (err)
2501 return err;
2502
2503
2504 if (r->val != val)
2505 return -EINVAL;
2506
2507 return 0;
2508}
2509
2510static bool is_valid_cache(u32 val)
2511{
2512 u32 level, ctype;
2513
2514 if (val >= CSSELR_MAX)
2515 return false;
2516
2517
2518 level = (val >> 1);
2519 ctype = (cache_levels >> (level * 3)) & 7;
2520
2521 switch (ctype) {
2522 case 0:
2523 return false;
2524 case 1:
2525 return (val & 1);
2526 case 2:
2527 case 4:
2528 return !(val & 1);
2529 case 3:
2530 return true;
2531 default:
2532 return false;
2533 }
2534}
2535
2536static int demux_c15_get(u64 id, void __user *uaddr)
2537{
2538 u32 val;
2539 u32 __user *uval = uaddr;
2540
2541
2542 if (id & ~(KVM_REG_ARCH_MASK|KVM_REG_SIZE_MASK|KVM_REG_ARM_COPROC_MASK
2543 | ((1 << KVM_REG_ARM_COPROC_SHIFT)-1)))
2544 return -ENOENT;
2545
2546 switch (id & KVM_REG_ARM_DEMUX_ID_MASK) {
2547 case KVM_REG_ARM_DEMUX_ID_CCSIDR:
2548 if (KVM_REG_SIZE(id) != 4)
2549 return -ENOENT;
2550 val = (id & KVM_REG_ARM_DEMUX_VAL_MASK)
2551 >> KVM_REG_ARM_DEMUX_VAL_SHIFT;
2552 if (!is_valid_cache(val))
2553 return -ENOENT;
2554
2555 return put_user(get_ccsidr(val), uval);
2556 default:
2557 return -ENOENT;
2558 }
2559}
2560
2561static int demux_c15_set(u64 id, void __user *uaddr)
2562{
2563 u32 val, newval;
2564 u32 __user *uval = uaddr;
2565
2566
2567 if (id & ~(KVM_REG_ARCH_MASK|KVM_REG_SIZE_MASK|KVM_REG_ARM_COPROC_MASK
2568 | ((1 << KVM_REG_ARM_COPROC_SHIFT)-1)))
2569 return -ENOENT;
2570
2571 switch (id & KVM_REG_ARM_DEMUX_ID_MASK) {
2572 case KVM_REG_ARM_DEMUX_ID_CCSIDR:
2573 if (KVM_REG_SIZE(id) != 4)
2574 return -ENOENT;
2575 val = (id & KVM_REG_ARM_DEMUX_VAL_MASK)
2576 >> KVM_REG_ARM_DEMUX_VAL_SHIFT;
2577 if (!is_valid_cache(val))
2578 return -ENOENT;
2579
2580 if (get_user(newval, uval))
2581 return -EFAULT;
2582
2583
2584 if (newval != get_ccsidr(val))
2585 return -EINVAL;
2586 return 0;
2587 default:
2588 return -ENOENT;
2589 }
2590}
2591
2592int kvm_arm_sys_reg_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
2593{
2594 const struct sys_reg_desc *r;
2595 void __user *uaddr = (void __user *)(unsigned long)reg->addr;
2596
2597 if ((reg->id & KVM_REG_ARM_COPROC_MASK) == KVM_REG_ARM_DEMUX)
2598 return demux_c15_get(reg->id, uaddr);
2599
2600 if (KVM_REG_SIZE(reg->id) != sizeof(__u64))
2601 return -ENOENT;
2602
2603 r = index_to_sys_reg_desc(vcpu, reg->id);
2604 if (!r)
2605 return get_invariant_sys_reg(reg->id, uaddr);
2606
2607
2608 if (sysreg_hidden(vcpu, r))
2609 return -ENOENT;
2610
2611 if (r->get_user)
2612 return (r->get_user)(vcpu, r, reg, uaddr);
2613
2614 return reg_to_user(uaddr, &__vcpu_sys_reg(vcpu, r->reg), reg->id);
2615}
2616
2617int kvm_arm_sys_reg_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
2618{
2619 const struct sys_reg_desc *r;
2620 void __user *uaddr = (void __user *)(unsigned long)reg->addr;
2621
2622 if ((reg->id & KVM_REG_ARM_COPROC_MASK) == KVM_REG_ARM_DEMUX)
2623 return demux_c15_set(reg->id, uaddr);
2624
2625 if (KVM_REG_SIZE(reg->id) != sizeof(__u64))
2626 return -ENOENT;
2627
2628 r = index_to_sys_reg_desc(vcpu, reg->id);
2629 if (!r)
2630 return set_invariant_sys_reg(reg->id, uaddr);
2631
2632
2633 if (sysreg_hidden(vcpu, r))
2634 return -ENOENT;
2635
2636 if (r->set_user)
2637 return (r->set_user)(vcpu, r, reg, uaddr);
2638
2639 return reg_from_user(&__vcpu_sys_reg(vcpu, r->reg), uaddr, reg->id);
2640}
2641
2642static unsigned int num_demux_regs(void)
2643{
2644 unsigned int i, count = 0;
2645
2646 for (i = 0; i < CSSELR_MAX; i++)
2647 if (is_valid_cache(i))
2648 count++;
2649
2650 return count;
2651}
2652
2653static int write_demux_regids(u64 __user *uindices)
2654{
2655 u64 val = KVM_REG_ARM64 | KVM_REG_SIZE_U32 | KVM_REG_ARM_DEMUX;
2656 unsigned int i;
2657
2658 val |= KVM_REG_ARM_DEMUX_ID_CCSIDR;
2659 for (i = 0; i < CSSELR_MAX; i++) {
2660 if (!is_valid_cache(i))
2661 continue;
2662 if (put_user(val | i, uindices))
2663 return -EFAULT;
2664 uindices++;
2665 }
2666 return 0;
2667}
2668
2669static u64 sys_reg_to_index(const struct sys_reg_desc *reg)
2670{
2671 return (KVM_REG_ARM64 | KVM_REG_SIZE_U64 |
2672 KVM_REG_ARM64_SYSREG |
2673 (reg->Op0 << KVM_REG_ARM64_SYSREG_OP0_SHIFT) |
2674 (reg->Op1 << KVM_REG_ARM64_SYSREG_OP1_SHIFT) |
2675 (reg->CRn << KVM_REG_ARM64_SYSREG_CRN_SHIFT) |
2676 (reg->CRm << KVM_REG_ARM64_SYSREG_CRM_SHIFT) |
2677 (reg->Op2 << KVM_REG_ARM64_SYSREG_OP2_SHIFT));
2678}
2679
2680static bool copy_reg_to_user(const struct sys_reg_desc *reg, u64 __user **uind)
2681{
2682 if (!*uind)
2683 return true;
2684
2685 if (put_user(sys_reg_to_index(reg), *uind))
2686 return false;
2687
2688 (*uind)++;
2689 return true;
2690}
2691
2692static int walk_one_sys_reg(const struct kvm_vcpu *vcpu,
2693 const struct sys_reg_desc *rd,
2694 u64 __user **uind,
2695 unsigned int *total)
2696{
2697
2698
2699
2700
2701 if (!(rd->reg || rd->get_user))
2702 return 0;
2703
2704 if (sysreg_hidden(vcpu, rd))
2705 return 0;
2706
2707 if (!copy_reg_to_user(rd, uind))
2708 return -EFAULT;
2709
2710 (*total)++;
2711 return 0;
2712}
2713
2714
2715static int walk_sys_regs(struct kvm_vcpu *vcpu, u64 __user *uind)
2716{
2717 const struct sys_reg_desc *i2, *end2;
2718 unsigned int total = 0;
2719 int err;
2720
2721 i2 = sys_reg_descs;
2722 end2 = sys_reg_descs + ARRAY_SIZE(sys_reg_descs);
2723
2724 while (i2 != end2) {
2725 err = walk_one_sys_reg(vcpu, i2++, &uind, &total);
2726 if (err)
2727 return err;
2728 }
2729 return total;
2730}
2731
2732unsigned long kvm_arm_num_sys_reg_descs(struct kvm_vcpu *vcpu)
2733{
2734 return ARRAY_SIZE(invariant_sys_regs)
2735 + num_demux_regs()
2736 + walk_sys_regs(vcpu, (u64 __user *)NULL);
2737}
2738
2739int kvm_arm_copy_sys_reg_indices(struct kvm_vcpu *vcpu, u64 __user *uindices)
2740{
2741 unsigned int i;
2742 int err;
2743
2744
2745 for (i = 0; i < ARRAY_SIZE(invariant_sys_regs); i++) {
2746 if (put_user(sys_reg_to_index(&invariant_sys_regs[i]), uindices))
2747 return -EFAULT;
2748 uindices++;
2749 }
2750
2751 err = walk_sys_regs(vcpu, uindices);
2752 if (err < 0)
2753 return err;
2754 uindices += err;
2755
2756 return write_demux_regids(uindices);
2757}
2758
2759void kvm_sys_reg_table_init(void)
2760{
2761 unsigned int i;
2762 struct sys_reg_desc clidr;
2763
2764
2765 BUG_ON(check_sysreg_table(sys_reg_descs, ARRAY_SIZE(sys_reg_descs), false));
2766 BUG_ON(check_sysreg_table(cp14_regs, ARRAY_SIZE(cp14_regs), true));
2767 BUG_ON(check_sysreg_table(cp14_64_regs, ARRAY_SIZE(cp14_64_regs), true));
2768 BUG_ON(check_sysreg_table(cp15_regs, ARRAY_SIZE(cp15_regs), true));
2769 BUG_ON(check_sysreg_table(cp15_64_regs, ARRAY_SIZE(cp15_64_regs), true));
2770 BUG_ON(check_sysreg_table(invariant_sys_regs, ARRAY_SIZE(invariant_sys_regs), false));
2771
2772
2773 for (i = 0; i < ARRAY_SIZE(invariant_sys_regs); i++)
2774 invariant_sys_regs[i].reset(NULL, &invariant_sys_regs[i]);
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786 get_clidr_el1(NULL, &clidr);
2787 cache_levels = clidr.val;
2788 for (i = 0; i < 7; i++)
2789 if (((cache_levels >> (i*3)) & 7) == 0)
2790 break;
2791
2792 cache_levels &= (1 << (i*3))-1;
2793}
2794