1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21#include <linux/errno.h>
22#include <linux/err.h>
23#include <linux/kvm_host.h>
24#include <linux/vmalloc.h>
25#include <linux/hrtimer.h>
26#include <linux/fs.h>
27#include <linux/slab.h>
28#include <linux/file.h>
29#include <linux/module.h>
30#include <asm/cputable.h>
31#include <asm/uaccess.h>
32#include <asm/kvm_ppc.h>
33#include <asm/tlbflush.h>
34#include <asm/cputhreads.h>
35#include <asm/irqflags.h>
36#include <asm/iommu.h>
37#include "timing.h"
38#include "irq.h"
39#include "../mm/mmu_decl.h"
40
41#define CREATE_TRACE_POINTS
42#include "trace.h"
43
44struct kvmppc_ops *kvmppc_hv_ops;
45EXPORT_SYMBOL_GPL(kvmppc_hv_ops);
46struct kvmppc_ops *kvmppc_pr_ops;
47EXPORT_SYMBOL_GPL(kvmppc_pr_ops);
48
49
50int kvm_arch_vcpu_runnable(struct kvm_vcpu *v)
51{
52 return !!(v->arch.pending_exceptions) ||
53 v->requests;
54}
55
56int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
57{
58 return 1;
59}
60
61
62
63
64
65
66
67
68
69
70int kvmppc_prepare_to_enter(struct kvm_vcpu *vcpu)
71{
72 int r;
73
74 WARN_ON(irqs_disabled());
75 hard_irq_disable();
76
77 while (true) {
78 if (need_resched()) {
79 local_irq_enable();
80 cond_resched();
81 hard_irq_disable();
82 continue;
83 }
84
85 if (signal_pending(current)) {
86 kvmppc_account_exit(vcpu, SIGNAL_EXITS);
87 vcpu->run->exit_reason = KVM_EXIT_INTR;
88 r = -EINTR;
89 break;
90 }
91
92 vcpu->mode = IN_GUEST_MODE;
93
94
95
96
97
98
99
100
101
102
103 smp_mb();
104
105 if (vcpu->requests) {
106
107 local_irq_enable();
108 trace_kvm_check_requests(vcpu);
109 r = kvmppc_core_check_requests(vcpu);
110 hard_irq_disable();
111 if (r > 0)
112 continue;
113 break;
114 }
115
116 if (kvmppc_core_prepare_to_enter(vcpu)) {
117
118
119 continue;
120 }
121
122 guest_enter_irqoff();
123 return 1;
124 }
125
126
127 local_irq_enable();
128 return r;
129}
130EXPORT_SYMBOL_GPL(kvmppc_prepare_to_enter);
131
132#if defined(CONFIG_PPC_BOOK3S_64) && defined(CONFIG_KVM_BOOK3S_PR_POSSIBLE)
133static void kvmppc_swab_shared(struct kvm_vcpu *vcpu)
134{
135 struct kvm_vcpu_arch_shared *shared = vcpu->arch.shared;
136 int i;
137
138 shared->sprg0 = swab64(shared->sprg0);
139 shared->sprg1 = swab64(shared->sprg1);
140 shared->sprg2 = swab64(shared->sprg2);
141 shared->sprg3 = swab64(shared->sprg3);
142 shared->srr0 = swab64(shared->srr0);
143 shared->srr1 = swab64(shared->srr1);
144 shared->dar = swab64(shared->dar);
145 shared->msr = swab64(shared->msr);
146 shared->dsisr = swab32(shared->dsisr);
147 shared->int_pending = swab32(shared->int_pending);
148 for (i = 0; i < ARRAY_SIZE(shared->sr); i++)
149 shared->sr[i] = swab32(shared->sr[i]);
150}
151#endif
152
153int kvmppc_kvm_pv(struct kvm_vcpu *vcpu)
154{
155 int nr = kvmppc_get_gpr(vcpu, 11);
156 int r;
157 unsigned long __maybe_unused param1 = kvmppc_get_gpr(vcpu, 3);
158 unsigned long __maybe_unused param2 = kvmppc_get_gpr(vcpu, 4);
159 unsigned long __maybe_unused param3 = kvmppc_get_gpr(vcpu, 5);
160 unsigned long __maybe_unused param4 = kvmppc_get_gpr(vcpu, 6);
161 unsigned long r2 = 0;
162
163 if (!(kvmppc_get_msr(vcpu) & MSR_SF)) {
164
165 param1 &= 0xffffffff;
166 param2 &= 0xffffffff;
167 param3 &= 0xffffffff;
168 param4 &= 0xffffffff;
169 }
170
171 switch (nr) {
172 case KVM_HCALL_TOKEN(KVM_HC_PPC_MAP_MAGIC_PAGE):
173 {
174#if defined(CONFIG_PPC_BOOK3S_64) && defined(CONFIG_KVM_BOOK3S_PR_POSSIBLE)
175
176 int shared_big_endian = true;
177 if (vcpu->arch.intr_msr & MSR_LE)
178 shared_big_endian = false;
179 if (shared_big_endian != vcpu->arch.shared_big_endian)
180 kvmppc_swab_shared(vcpu);
181 vcpu->arch.shared_big_endian = shared_big_endian;
182#endif
183
184 if (!(param2 & MAGIC_PAGE_FLAG_NOT_MAPPED_NX)) {
185
186
187
188
189
190 vcpu->arch.disable_kernel_nx = true;
191 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
192 }
193
194 vcpu->arch.magic_page_pa = param1 & ~0xfffULL;
195 vcpu->arch.magic_page_ea = param2 & ~0xfffULL;
196
197#ifdef CONFIG_PPC_64K_PAGES
198
199
200
201
202 if ((vcpu->arch.magic_page_pa & 0xf000) !=
203 ((ulong)vcpu->arch.shared & 0xf000)) {
204 void *old_shared = vcpu->arch.shared;
205 ulong shared = (ulong)vcpu->arch.shared;
206 void *new_shared;
207
208 shared &= PAGE_MASK;
209 shared |= vcpu->arch.magic_page_pa & 0xf000;
210 new_shared = (void*)shared;
211 memcpy(new_shared, old_shared, 0x1000);
212 vcpu->arch.shared = new_shared;
213 }
214#endif
215
216 r2 = KVM_MAGIC_FEAT_SR | KVM_MAGIC_FEAT_MAS0_TO_SPRG7;
217
218 r = EV_SUCCESS;
219 break;
220 }
221 case KVM_HCALL_TOKEN(KVM_HC_FEATURES):
222 r = EV_SUCCESS;
223#if defined(CONFIG_PPC_BOOK3S) || defined(CONFIG_KVM_E500V2)
224 r2 |= (1 << KVM_FEATURE_MAGIC_PAGE);
225#endif
226
227
228 break;
229 case EV_HCALL_TOKEN(EV_IDLE):
230 r = EV_SUCCESS;
231 kvm_vcpu_block(vcpu);
232 clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
233 break;
234 default:
235 r = EV_UNIMPLEMENTED;
236 break;
237 }
238
239 kvmppc_set_gpr(vcpu, 4, r2);
240
241 return r;
242}
243EXPORT_SYMBOL_GPL(kvmppc_kvm_pv);
244
245int kvmppc_sanity_check(struct kvm_vcpu *vcpu)
246{
247 int r = false;
248
249
250 if (!vcpu->arch.pvr)
251 goto out;
252
253
254 if ((vcpu->arch.cpu_type != KVM_CPU_3S_64) && vcpu->arch.papr_enabled)
255 goto out;
256
257
258 if (!vcpu->arch.papr_enabled && is_kvmppc_hv_enabled(vcpu->kvm))
259 goto out;
260
261#ifdef CONFIG_KVM_BOOKE_HV
262 if (!cpu_has_feature(CPU_FTR_EMB_HV))
263 goto out;
264#endif
265
266 r = true;
267
268out:
269 vcpu->arch.sane = r;
270 return r ? 0 : -EINVAL;
271}
272EXPORT_SYMBOL_GPL(kvmppc_sanity_check);
273
274int kvmppc_emulate_mmio(struct kvm_run *run, struct kvm_vcpu *vcpu)
275{
276 enum emulation_result er;
277 int r;
278
279 er = kvmppc_emulate_loadstore(vcpu);
280 switch (er) {
281 case EMULATE_DONE:
282
283
284 r = RESUME_GUEST_NV;
285 break;
286 case EMULATE_AGAIN:
287 r = RESUME_GUEST;
288 break;
289 case EMULATE_DO_MMIO:
290 run->exit_reason = KVM_EXIT_MMIO;
291
292
293
294
295 r = RESUME_HOST_NV;
296 break;
297 case EMULATE_FAIL:
298 {
299 u32 last_inst;
300
301 kvmppc_get_last_inst(vcpu, INST_GENERIC, &last_inst);
302
303 pr_emerg("%s: emulation failed (%08x)\n", __func__, last_inst);
304 r = RESUME_HOST;
305 break;
306 }
307 default:
308 WARN_ON(1);
309 r = RESUME_GUEST;
310 }
311
312 return r;
313}
314EXPORT_SYMBOL_GPL(kvmppc_emulate_mmio);
315
316int kvmppc_st(struct kvm_vcpu *vcpu, ulong *eaddr, int size, void *ptr,
317 bool data)
318{
319 ulong mp_pa = vcpu->arch.magic_page_pa & KVM_PAM & PAGE_MASK;
320 struct kvmppc_pte pte;
321 int r;
322
323 vcpu->stat.st++;
324
325 r = kvmppc_xlate(vcpu, *eaddr, data ? XLATE_DATA : XLATE_INST,
326 XLATE_WRITE, &pte);
327 if (r < 0)
328 return r;
329
330 *eaddr = pte.raddr;
331
332 if (!pte.may_write)
333 return -EPERM;
334
335
336 if (kvmppc_supports_magic_page(vcpu) && mp_pa &&
337 ((pte.raddr & KVM_PAM & PAGE_MASK) == mp_pa) &&
338 !(kvmppc_get_msr(vcpu) & MSR_PR)) {
339 void *magic = vcpu->arch.shared;
340 magic += pte.eaddr & 0xfff;
341 memcpy(magic, ptr, size);
342 return EMULATE_DONE;
343 }
344
345 if (kvm_write_guest(vcpu->kvm, pte.raddr, ptr, size))
346 return EMULATE_DO_MMIO;
347
348 return EMULATE_DONE;
349}
350EXPORT_SYMBOL_GPL(kvmppc_st);
351
352int kvmppc_ld(struct kvm_vcpu *vcpu, ulong *eaddr, int size, void *ptr,
353 bool data)
354{
355 ulong mp_pa = vcpu->arch.magic_page_pa & KVM_PAM & PAGE_MASK;
356 struct kvmppc_pte pte;
357 int rc;
358
359 vcpu->stat.ld++;
360
361 rc = kvmppc_xlate(vcpu, *eaddr, data ? XLATE_DATA : XLATE_INST,
362 XLATE_READ, &pte);
363 if (rc)
364 return rc;
365
366 *eaddr = pte.raddr;
367
368 if (!pte.may_read)
369 return -EPERM;
370
371 if (!data && !pte.may_execute)
372 return -ENOEXEC;
373
374
375 if (kvmppc_supports_magic_page(vcpu) && mp_pa &&
376 ((pte.raddr & KVM_PAM & PAGE_MASK) == mp_pa) &&
377 !(kvmppc_get_msr(vcpu) & MSR_PR)) {
378 void *magic = vcpu->arch.shared;
379 magic += pte.eaddr & 0xfff;
380 memcpy(ptr, magic, size);
381 return EMULATE_DONE;
382 }
383
384 if (kvm_read_guest(vcpu->kvm, pte.raddr, ptr, size))
385 return EMULATE_DO_MMIO;
386
387 return EMULATE_DONE;
388}
389EXPORT_SYMBOL_GPL(kvmppc_ld);
390
391int kvm_arch_hardware_enable(void)
392{
393 return 0;
394}
395
396int kvm_arch_hardware_setup(void)
397{
398 return 0;
399}
400
401void kvm_arch_check_processor_compat(void *rtn)
402{
403 *(int *)rtn = kvmppc_core_check_processor_compat();
404}
405
406int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
407{
408 struct kvmppc_ops *kvm_ops = NULL;
409
410
411
412 if (type == 0) {
413 if (kvmppc_hv_ops)
414 kvm_ops = kvmppc_hv_ops;
415 else
416 kvm_ops = kvmppc_pr_ops;
417 if (!kvm_ops)
418 goto err_out;
419 } else if (type == KVM_VM_PPC_HV) {
420 if (!kvmppc_hv_ops)
421 goto err_out;
422 kvm_ops = kvmppc_hv_ops;
423 } else if (type == KVM_VM_PPC_PR) {
424 if (!kvmppc_pr_ops)
425 goto err_out;
426 kvm_ops = kvmppc_pr_ops;
427 } else
428 goto err_out;
429
430 if (kvm_ops->owner && !try_module_get(kvm_ops->owner))
431 return -ENOENT;
432
433 kvm->arch.kvm_ops = kvm_ops;
434 return kvmppc_core_init_vm(kvm);
435err_out:
436 return -EINVAL;
437}
438
439void kvm_arch_destroy_vm(struct kvm *kvm)
440{
441 unsigned int i;
442 struct kvm_vcpu *vcpu;
443
444#ifdef CONFIG_KVM_XICS
445
446
447
448
449
450 if (is_kvmppc_hv_enabled(kvm))
451 kick_all_cpus_sync();
452#endif
453
454 kvm_for_each_vcpu(i, vcpu, kvm)
455 kvm_arch_vcpu_free(vcpu);
456
457 mutex_lock(&kvm->lock);
458 for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
459 kvm->vcpus[i] = NULL;
460
461 atomic_set(&kvm->online_vcpus, 0);
462
463 kvmppc_core_destroy_vm(kvm);
464
465 mutex_unlock(&kvm->lock);
466
467
468 module_put(kvm->arch.kvm_ops->owner);
469}
470
471int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
472{
473 int r;
474
475 int hv_enabled = kvmppc_hv_ops ? 1 : 0;
476
477 if (kvm) {
478
479
480
481
482 hv_enabled = is_kvmppc_hv_enabled(kvm);
483 }
484
485 switch (ext) {
486#ifdef CONFIG_BOOKE
487 case KVM_CAP_PPC_BOOKE_SREGS:
488 case KVM_CAP_PPC_BOOKE_WATCHDOG:
489 case KVM_CAP_PPC_EPR:
490#else
491 case KVM_CAP_PPC_SEGSTATE:
492 case KVM_CAP_PPC_HIOR:
493 case KVM_CAP_PPC_PAPR:
494#endif
495 case KVM_CAP_PPC_UNSET_IRQ:
496 case KVM_CAP_PPC_IRQ_LEVEL:
497 case KVM_CAP_ENABLE_CAP:
498 case KVM_CAP_ENABLE_CAP_VM:
499 case KVM_CAP_ONE_REG:
500 case KVM_CAP_IOEVENTFD:
501 case KVM_CAP_DEVICE_CTRL:
502 r = 1;
503 break;
504 case KVM_CAP_PPC_PAIRED_SINGLES:
505 case KVM_CAP_PPC_OSI:
506 case KVM_CAP_PPC_GET_PVINFO:
507#if defined(CONFIG_KVM_E500V2) || defined(CONFIG_KVM_E500MC)
508 case KVM_CAP_SW_TLB:
509#endif
510
511 r = !hv_enabled;
512 break;
513#ifdef CONFIG_KVM_MMIO
514 case KVM_CAP_COALESCED_MMIO:
515 r = KVM_COALESCED_MMIO_PAGE_OFFSET;
516 break;
517#endif
518#ifdef CONFIG_KVM_MPIC
519 case KVM_CAP_IRQ_MPIC:
520 r = 1;
521 break;
522#endif
523
524#ifdef CONFIG_PPC_BOOK3S_64
525 case KVM_CAP_SPAPR_TCE:
526 case KVM_CAP_SPAPR_TCE_64:
527 case KVM_CAP_PPC_ALLOC_HTAB:
528 case KVM_CAP_PPC_RTAS:
529 case KVM_CAP_PPC_FIXUP_HCALL:
530 case KVM_CAP_PPC_ENABLE_HCALL:
531#ifdef CONFIG_KVM_XICS
532 case KVM_CAP_IRQ_XICS:
533#endif
534 r = 1;
535 break;
536#endif
537#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
538 case KVM_CAP_PPC_SMT:
539 if (hv_enabled)
540 r = threads_per_subcore;
541 else
542 r = 0;
543 break;
544 case KVM_CAP_PPC_RMA:
545 r = 0;
546 break;
547 case KVM_CAP_PPC_HWRNG:
548 r = kvmppc_hwrng_present();
549 break;
550#endif
551 case KVM_CAP_SYNC_MMU:
552#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
553 r = hv_enabled;
554#elif defined(KVM_ARCH_WANT_MMU_NOTIFIER)
555 r = 1;
556#else
557 r = 0;
558#endif
559 break;
560#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
561 case KVM_CAP_PPC_HTAB_FD:
562 r = hv_enabled;
563 break;
564#endif
565 case KVM_CAP_NR_VCPUS:
566
567
568
569
570
571
572 if (hv_enabled)
573 r = num_present_cpus();
574 else
575 r = num_online_cpus();
576 break;
577 case KVM_CAP_NR_MEMSLOTS:
578 r = KVM_USER_MEM_SLOTS;
579 break;
580 case KVM_CAP_MAX_VCPUS:
581 r = KVM_MAX_VCPUS;
582 break;
583#ifdef CONFIG_PPC_BOOK3S_64
584 case KVM_CAP_PPC_GET_SMMU_INFO:
585 r = 1;
586 break;
587 case KVM_CAP_SPAPR_MULTITCE:
588 r = 1;
589 break;
590#endif
591 case KVM_CAP_PPC_HTM:
592 r = cpu_has_feature(CPU_FTR_TM_COMP) &&
593 is_kvmppc_hv_enabled(kvm);
594 break;
595 default:
596 r = 0;
597 break;
598 }
599 return r;
600
601}
602
603long kvm_arch_dev_ioctl(struct file *filp,
604 unsigned int ioctl, unsigned long arg)
605{
606 return -EINVAL;
607}
608
609void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free,
610 struct kvm_memory_slot *dont)
611{
612 kvmppc_core_free_memslot(kvm, free, dont);
613}
614
615int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
616 unsigned long npages)
617{
618 return kvmppc_core_create_memslot(kvm, slot, npages);
619}
620
621int kvm_arch_prepare_memory_region(struct kvm *kvm,
622 struct kvm_memory_slot *memslot,
623 const struct kvm_userspace_memory_region *mem,
624 enum kvm_mr_change change)
625{
626 return kvmppc_core_prepare_memory_region(kvm, memslot, mem);
627}
628
629void kvm_arch_commit_memory_region(struct kvm *kvm,
630 const struct kvm_userspace_memory_region *mem,
631 const struct kvm_memory_slot *old,
632 const struct kvm_memory_slot *new,
633 enum kvm_mr_change change)
634{
635 kvmppc_core_commit_memory_region(kvm, mem, old, new);
636}
637
638void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
639 struct kvm_memory_slot *slot)
640{
641 kvmppc_core_flush_memslot(kvm, slot);
642}
643
644struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm, unsigned int id)
645{
646 struct kvm_vcpu *vcpu;
647 vcpu = kvmppc_core_vcpu_create(kvm, id);
648 if (!IS_ERR(vcpu)) {
649 vcpu->arch.wqp = &vcpu->wq;
650 kvmppc_create_vcpu_debugfs(vcpu, id);
651 }
652 return vcpu;
653}
654
655void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
656{
657}
658
659void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu)
660{
661
662 hrtimer_cancel(&vcpu->arch.dec_timer);
663
664 kvmppc_remove_vcpu_debugfs(vcpu);
665
666 switch (vcpu->arch.irq_type) {
667 case KVMPPC_IRQ_MPIC:
668 kvmppc_mpic_disconnect_vcpu(vcpu->arch.mpic, vcpu);
669 break;
670 case KVMPPC_IRQ_XICS:
671 kvmppc_xics_free_icp(vcpu);
672 break;
673 }
674
675 kvmppc_core_vcpu_free(vcpu);
676}
677
678void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
679{
680 kvm_arch_vcpu_free(vcpu);
681}
682
683int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu)
684{
685 return kvmppc_core_pending_dec(vcpu);
686}
687
688static enum hrtimer_restart kvmppc_decrementer_wakeup(struct hrtimer *timer)
689{
690 struct kvm_vcpu *vcpu;
691
692 vcpu = container_of(timer, struct kvm_vcpu, arch.dec_timer);
693 kvmppc_decrementer_func(vcpu);
694
695 return HRTIMER_NORESTART;
696}
697
698int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
699{
700 int ret;
701
702 hrtimer_init(&vcpu->arch.dec_timer, CLOCK_REALTIME, HRTIMER_MODE_ABS);
703 vcpu->arch.dec_timer.function = kvmppc_decrementer_wakeup;
704 vcpu->arch.dec_expires = ~(u64)0;
705
706#ifdef CONFIG_KVM_EXIT_TIMING
707 mutex_init(&vcpu->arch.exit_timing_lock);
708#endif
709 ret = kvmppc_subarch_vcpu_init(vcpu);
710 return ret;
711}
712
713void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu)
714{
715 kvmppc_mmu_destroy(vcpu);
716 kvmppc_subarch_vcpu_uninit(vcpu);
717}
718
719void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
720{
721#ifdef CONFIG_BOOKE
722
723
724
725
726
727
728
729 mtspr(SPRN_VRSAVE, vcpu->arch.vrsave);
730#endif
731 kvmppc_core_vcpu_load(vcpu, cpu);
732}
733
734void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
735{
736 kvmppc_core_vcpu_put(vcpu);
737#ifdef CONFIG_BOOKE
738 vcpu->arch.vrsave = mfspr(SPRN_VRSAVE);
739#endif
740}
741
742static void kvmppc_complete_mmio_load(struct kvm_vcpu *vcpu,
743 struct kvm_run *run)
744{
745 u64 uninitialized_var(gpr);
746
747 if (run->mmio.len > sizeof(gpr)) {
748 printk(KERN_ERR "bad MMIO length: %d\n", run->mmio.len);
749 return;
750 }
751
752 if (!vcpu->arch.mmio_host_swabbed) {
753 switch (run->mmio.len) {
754 case 8: gpr = *(u64 *)run->mmio.data; break;
755 case 4: gpr = *(u32 *)run->mmio.data; break;
756 case 2: gpr = *(u16 *)run->mmio.data; break;
757 case 1: gpr = *(u8 *)run->mmio.data; break;
758 }
759 } else {
760 switch (run->mmio.len) {
761 case 8: gpr = swab64(*(u64 *)run->mmio.data); break;
762 case 4: gpr = swab32(*(u32 *)run->mmio.data); break;
763 case 2: gpr = swab16(*(u16 *)run->mmio.data); break;
764 case 1: gpr = *(u8 *)run->mmio.data; break;
765 }
766 }
767
768 if (vcpu->arch.mmio_sign_extend) {
769 switch (run->mmio.len) {
770#ifdef CONFIG_PPC64
771 case 4:
772 gpr = (s64)(s32)gpr;
773 break;
774#endif
775 case 2:
776 gpr = (s64)(s16)gpr;
777 break;
778 case 1:
779 gpr = (s64)(s8)gpr;
780 break;
781 }
782 }
783
784 kvmppc_set_gpr(vcpu, vcpu->arch.io_gpr, gpr);
785
786 switch (vcpu->arch.io_gpr & KVM_MMIO_REG_EXT_MASK) {
787 case KVM_MMIO_REG_GPR:
788 kvmppc_set_gpr(vcpu, vcpu->arch.io_gpr, gpr);
789 break;
790 case KVM_MMIO_REG_FPR:
791 VCPU_FPR(vcpu, vcpu->arch.io_gpr & KVM_MMIO_REG_MASK) = gpr;
792 break;
793#ifdef CONFIG_PPC_BOOK3S
794 case KVM_MMIO_REG_QPR:
795 vcpu->arch.qpr[vcpu->arch.io_gpr & KVM_MMIO_REG_MASK] = gpr;
796 break;
797 case KVM_MMIO_REG_FQPR:
798 VCPU_FPR(vcpu, vcpu->arch.io_gpr & KVM_MMIO_REG_MASK) = gpr;
799 vcpu->arch.qpr[vcpu->arch.io_gpr & KVM_MMIO_REG_MASK] = gpr;
800 break;
801#endif
802 default:
803 BUG();
804 }
805}
806
807static int __kvmppc_handle_load(struct kvm_run *run, struct kvm_vcpu *vcpu,
808 unsigned int rt, unsigned int bytes,
809 int is_default_endian, int sign_extend)
810{
811 int idx, ret;
812 bool host_swabbed;
813
814
815 if (kvmppc_need_byteswap(vcpu)) {
816 host_swabbed = is_default_endian;
817 } else {
818 host_swabbed = !is_default_endian;
819 }
820
821 if (bytes > sizeof(run->mmio.data)) {
822 printk(KERN_ERR "%s: bad MMIO length: %d\n", __func__,
823 run->mmio.len);
824 }
825
826 run->mmio.phys_addr = vcpu->arch.paddr_accessed;
827 run->mmio.len = bytes;
828 run->mmio.is_write = 0;
829
830 vcpu->arch.io_gpr = rt;
831 vcpu->arch.mmio_host_swabbed = host_swabbed;
832 vcpu->mmio_needed = 1;
833 vcpu->mmio_is_write = 0;
834 vcpu->arch.mmio_sign_extend = sign_extend;
835
836 idx = srcu_read_lock(&vcpu->kvm->srcu);
837
838 ret = kvm_io_bus_read(vcpu, KVM_MMIO_BUS, run->mmio.phys_addr,
839 bytes, &run->mmio.data);
840
841 srcu_read_unlock(&vcpu->kvm->srcu, idx);
842
843 if (!ret) {
844 kvmppc_complete_mmio_load(vcpu, run);
845 vcpu->mmio_needed = 0;
846 return EMULATE_DONE;
847 }
848
849 return EMULATE_DO_MMIO;
850}
851
852int kvmppc_handle_load(struct kvm_run *run, struct kvm_vcpu *vcpu,
853 unsigned int rt, unsigned int bytes,
854 int is_default_endian)
855{
856 return __kvmppc_handle_load(run, vcpu, rt, bytes, is_default_endian, 0);
857}
858EXPORT_SYMBOL_GPL(kvmppc_handle_load);
859
860
861int kvmppc_handle_loads(struct kvm_run *run, struct kvm_vcpu *vcpu,
862 unsigned int rt, unsigned int bytes,
863 int is_default_endian)
864{
865 return __kvmppc_handle_load(run, vcpu, rt, bytes, is_default_endian, 1);
866}
867
868int kvmppc_handle_store(struct kvm_run *run, struct kvm_vcpu *vcpu,
869 u64 val, unsigned int bytes, int is_default_endian)
870{
871 void *data = run->mmio.data;
872 int idx, ret;
873 bool host_swabbed;
874
875
876 if (kvmppc_need_byteswap(vcpu)) {
877 host_swabbed = is_default_endian;
878 } else {
879 host_swabbed = !is_default_endian;
880 }
881
882 if (bytes > sizeof(run->mmio.data)) {
883 printk(KERN_ERR "%s: bad MMIO length: %d\n", __func__,
884 run->mmio.len);
885 }
886
887 run->mmio.phys_addr = vcpu->arch.paddr_accessed;
888 run->mmio.len = bytes;
889 run->mmio.is_write = 1;
890 vcpu->mmio_needed = 1;
891 vcpu->mmio_is_write = 1;
892
893
894 if (!host_swabbed) {
895 switch (bytes) {
896 case 8: *(u64 *)data = val; break;
897 case 4: *(u32 *)data = val; break;
898 case 2: *(u16 *)data = val; break;
899 case 1: *(u8 *)data = val; break;
900 }
901 } else {
902 switch (bytes) {
903 case 8: *(u64 *)data = swab64(val); break;
904 case 4: *(u32 *)data = swab32(val); break;
905 case 2: *(u16 *)data = swab16(val); break;
906 case 1: *(u8 *)data = val; break;
907 }
908 }
909
910 idx = srcu_read_lock(&vcpu->kvm->srcu);
911
912 ret = kvm_io_bus_write(vcpu, KVM_MMIO_BUS, run->mmio.phys_addr,
913 bytes, &run->mmio.data);
914
915 srcu_read_unlock(&vcpu->kvm->srcu, idx);
916
917 if (!ret) {
918 vcpu->mmio_needed = 0;
919 return EMULATE_DONE;
920 }
921
922 return EMULATE_DO_MMIO;
923}
924EXPORT_SYMBOL_GPL(kvmppc_handle_store);
925
926int kvm_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu, struct kvm_one_reg *reg)
927{
928 int r = 0;
929 union kvmppc_one_reg val;
930 int size;
931
932 size = one_reg_size(reg->id);
933 if (size > sizeof(val))
934 return -EINVAL;
935
936 r = kvmppc_get_one_reg(vcpu, reg->id, &val);
937 if (r == -EINVAL) {
938 r = 0;
939 switch (reg->id) {
940#ifdef CONFIG_ALTIVEC
941 case KVM_REG_PPC_VR0 ... KVM_REG_PPC_VR31:
942 if (!cpu_has_feature(CPU_FTR_ALTIVEC)) {
943 r = -ENXIO;
944 break;
945 }
946 val.vval = vcpu->arch.vr.vr[reg->id - KVM_REG_PPC_VR0];
947 break;
948 case KVM_REG_PPC_VSCR:
949 if (!cpu_has_feature(CPU_FTR_ALTIVEC)) {
950 r = -ENXIO;
951 break;
952 }
953 val = get_reg_val(reg->id, vcpu->arch.vr.vscr.u[3]);
954 break;
955 case KVM_REG_PPC_VRSAVE:
956 val = get_reg_val(reg->id, vcpu->arch.vrsave);
957 break;
958#endif
959 default:
960 r = -EINVAL;
961 break;
962 }
963 }
964
965 if (r)
966 return r;
967
968 if (copy_to_user((char __user *)(unsigned long)reg->addr, &val, size))
969 r = -EFAULT;
970
971 return r;
972}
973
974int kvm_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu, struct kvm_one_reg *reg)
975{
976 int r;
977 union kvmppc_one_reg val;
978 int size;
979
980 size = one_reg_size(reg->id);
981 if (size > sizeof(val))
982 return -EINVAL;
983
984 if (copy_from_user(&val, (char __user *)(unsigned long)reg->addr, size))
985 return -EFAULT;
986
987 r = kvmppc_set_one_reg(vcpu, reg->id, &val);
988 if (r == -EINVAL) {
989 r = 0;
990 switch (reg->id) {
991#ifdef CONFIG_ALTIVEC
992 case KVM_REG_PPC_VR0 ... KVM_REG_PPC_VR31:
993 if (!cpu_has_feature(CPU_FTR_ALTIVEC)) {
994 r = -ENXIO;
995 break;
996 }
997 vcpu->arch.vr.vr[reg->id - KVM_REG_PPC_VR0] = val.vval;
998 break;
999 case KVM_REG_PPC_VSCR:
1000 if (!cpu_has_feature(CPU_FTR_ALTIVEC)) {
1001 r = -ENXIO;
1002 break;
1003 }
1004 vcpu->arch.vr.vscr.u[3] = set_reg_val(reg->id, val);
1005 break;
1006 case KVM_REG_PPC_VRSAVE:
1007 if (!cpu_has_feature(CPU_FTR_ALTIVEC)) {
1008 r = -ENXIO;
1009 break;
1010 }
1011 vcpu->arch.vrsave = set_reg_val(reg->id, val);
1012 break;
1013#endif
1014 default:
1015 r = -EINVAL;
1016 break;
1017 }
1018 }
1019
1020 return r;
1021}
1022
1023int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run)
1024{
1025 int r;
1026 sigset_t sigsaved;
1027
1028 if (vcpu->sigset_active)
1029 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
1030
1031 if (vcpu->mmio_needed) {
1032 if (!vcpu->mmio_is_write)
1033 kvmppc_complete_mmio_load(vcpu, run);
1034 vcpu->mmio_needed = 0;
1035 } else if (vcpu->arch.osi_needed) {
1036 u64 *gprs = run->osi.gprs;
1037 int i;
1038
1039 for (i = 0; i < 32; i++)
1040 kvmppc_set_gpr(vcpu, i, gprs[i]);
1041 vcpu->arch.osi_needed = 0;
1042 } else if (vcpu->arch.hcall_needed) {
1043 int i;
1044
1045 kvmppc_set_gpr(vcpu, 3, run->papr_hcall.ret);
1046 for (i = 0; i < 9; ++i)
1047 kvmppc_set_gpr(vcpu, 4 + i, run->papr_hcall.args[i]);
1048 vcpu->arch.hcall_needed = 0;
1049#ifdef CONFIG_BOOKE
1050 } else if (vcpu->arch.epr_needed) {
1051 kvmppc_set_epr(vcpu, run->epr.epr);
1052 vcpu->arch.epr_needed = 0;
1053#endif
1054 }
1055
1056 r = kvmppc_vcpu_run(run, vcpu);
1057
1058 if (vcpu->sigset_active)
1059 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
1060
1061 return r;
1062}
1063
1064int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu, struct kvm_interrupt *irq)
1065{
1066 if (irq->irq == KVM_INTERRUPT_UNSET) {
1067 kvmppc_core_dequeue_external(vcpu);
1068 return 0;
1069 }
1070
1071 kvmppc_core_queue_external(vcpu, irq);
1072
1073 kvm_vcpu_kick(vcpu);
1074
1075 return 0;
1076}
1077
1078static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
1079 struct kvm_enable_cap *cap)
1080{
1081 int r;
1082
1083 if (cap->flags)
1084 return -EINVAL;
1085
1086 switch (cap->cap) {
1087 case KVM_CAP_PPC_OSI:
1088 r = 0;
1089 vcpu->arch.osi_enabled = true;
1090 break;
1091 case KVM_CAP_PPC_PAPR:
1092 r = 0;
1093 vcpu->arch.papr_enabled = true;
1094 break;
1095 case KVM_CAP_PPC_EPR:
1096 r = 0;
1097 if (cap->args[0])
1098 vcpu->arch.epr_flags |= KVMPPC_EPR_USER;
1099 else
1100 vcpu->arch.epr_flags &= ~KVMPPC_EPR_USER;
1101 break;
1102#ifdef CONFIG_BOOKE
1103 case KVM_CAP_PPC_BOOKE_WATCHDOG:
1104 r = 0;
1105 vcpu->arch.watchdog_enabled = true;
1106 break;
1107#endif
1108#if defined(CONFIG_KVM_E500V2) || defined(CONFIG_KVM_E500MC)
1109 case KVM_CAP_SW_TLB: {
1110 struct kvm_config_tlb cfg;
1111 void __user *user_ptr = (void __user *)(uintptr_t)cap->args[0];
1112
1113 r = -EFAULT;
1114 if (copy_from_user(&cfg, user_ptr, sizeof(cfg)))
1115 break;
1116
1117 r = kvm_vcpu_ioctl_config_tlb(vcpu, &cfg);
1118 break;
1119 }
1120#endif
1121#ifdef CONFIG_KVM_MPIC
1122 case KVM_CAP_IRQ_MPIC: {
1123 struct fd f;
1124 struct kvm_device *dev;
1125
1126 r = -EBADF;
1127 f = fdget(cap->args[0]);
1128 if (!f.file)
1129 break;
1130
1131 r = -EPERM;
1132 dev = kvm_device_from_filp(f.file);
1133 if (dev)
1134 r = kvmppc_mpic_connect_vcpu(dev, vcpu, cap->args[1]);
1135
1136 fdput(f);
1137 break;
1138 }
1139#endif
1140#ifdef CONFIG_KVM_XICS
1141 case KVM_CAP_IRQ_XICS: {
1142 struct fd f;
1143 struct kvm_device *dev;
1144
1145 r = -EBADF;
1146 f = fdget(cap->args[0]);
1147 if (!f.file)
1148 break;
1149
1150 r = -EPERM;
1151 dev = kvm_device_from_filp(f.file);
1152 if (dev)
1153 r = kvmppc_xics_connect_vcpu(dev, vcpu, cap->args[1]);
1154
1155 fdput(f);
1156 break;
1157 }
1158#endif
1159 default:
1160 r = -EINVAL;
1161 break;
1162 }
1163
1164 if (!r)
1165 r = kvmppc_sanity_check(vcpu);
1166
1167 return r;
1168}
1169
1170int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
1171 struct kvm_mp_state *mp_state)
1172{
1173 return -EINVAL;
1174}
1175
1176int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
1177 struct kvm_mp_state *mp_state)
1178{
1179 return -EINVAL;
1180}
1181
1182long kvm_arch_vcpu_ioctl(struct file *filp,
1183 unsigned int ioctl, unsigned long arg)
1184{
1185 struct kvm_vcpu *vcpu = filp->private_data;
1186 void __user *argp = (void __user *)arg;
1187 long r;
1188
1189 switch (ioctl) {
1190 case KVM_INTERRUPT: {
1191 struct kvm_interrupt irq;
1192 r = -EFAULT;
1193 if (copy_from_user(&irq, argp, sizeof(irq)))
1194 goto out;
1195 r = kvm_vcpu_ioctl_interrupt(vcpu, &irq);
1196 goto out;
1197 }
1198
1199 case KVM_ENABLE_CAP:
1200 {
1201 struct kvm_enable_cap cap;
1202 r = -EFAULT;
1203 if (copy_from_user(&cap, argp, sizeof(cap)))
1204 goto out;
1205 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
1206 break;
1207 }
1208
1209 case KVM_SET_ONE_REG:
1210 case KVM_GET_ONE_REG:
1211 {
1212 struct kvm_one_reg reg;
1213 r = -EFAULT;
1214 if (copy_from_user(®, argp, sizeof(reg)))
1215 goto out;
1216 if (ioctl == KVM_SET_ONE_REG)
1217 r = kvm_vcpu_ioctl_set_one_reg(vcpu, ®);
1218 else
1219 r = kvm_vcpu_ioctl_get_one_reg(vcpu, ®);
1220 break;
1221 }
1222
1223#if defined(CONFIG_KVM_E500V2) || defined(CONFIG_KVM_E500MC)
1224 case KVM_DIRTY_TLB: {
1225 struct kvm_dirty_tlb dirty;
1226 r = -EFAULT;
1227 if (copy_from_user(&dirty, argp, sizeof(dirty)))
1228 goto out;
1229 r = kvm_vcpu_ioctl_dirty_tlb(vcpu, &dirty);
1230 break;
1231 }
1232#endif
1233 default:
1234 r = -EINVAL;
1235 }
1236
1237out:
1238 return r;
1239}
1240
1241int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
1242{
1243 return VM_FAULT_SIGBUS;
1244}
1245
1246static int kvm_vm_ioctl_get_pvinfo(struct kvm_ppc_pvinfo *pvinfo)
1247{
1248 u32 inst_nop = 0x60000000;
1249#ifdef CONFIG_KVM_BOOKE_HV
1250 u32 inst_sc1 = 0x44000022;
1251 pvinfo->hcall[0] = cpu_to_be32(inst_sc1);
1252 pvinfo->hcall[1] = cpu_to_be32(inst_nop);
1253 pvinfo->hcall[2] = cpu_to_be32(inst_nop);
1254 pvinfo->hcall[3] = cpu_to_be32(inst_nop);
1255#else
1256 u32 inst_lis = 0x3c000000;
1257 u32 inst_ori = 0x60000000;
1258 u32 inst_sc = 0x44000002;
1259 u32 inst_imm_mask = 0xffff;
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270 pvinfo->hcall[0] = cpu_to_be32(inst_lis | ((KVM_SC_MAGIC_R0 >> 16) & inst_imm_mask));
1271 pvinfo->hcall[1] = cpu_to_be32(inst_ori | (KVM_SC_MAGIC_R0 & inst_imm_mask));
1272 pvinfo->hcall[2] = cpu_to_be32(inst_sc);
1273 pvinfo->hcall[3] = cpu_to_be32(inst_nop);
1274#endif
1275
1276 pvinfo->flags = KVM_PPC_PVINFO_FLAGS_EV_IDLE;
1277
1278 return 0;
1279}
1280
1281int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_event,
1282 bool line_status)
1283{
1284 if (!irqchip_in_kernel(kvm))
1285 return -ENXIO;
1286
1287 irq_event->status = kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID,
1288 irq_event->irq, irq_event->level,
1289 line_status);
1290 return 0;
1291}
1292
1293
1294static int kvm_vm_ioctl_enable_cap(struct kvm *kvm,
1295 struct kvm_enable_cap *cap)
1296{
1297 int r;
1298
1299 if (cap->flags)
1300 return -EINVAL;
1301
1302 switch (cap->cap) {
1303#ifdef CONFIG_KVM_BOOK3S_64_HANDLER
1304 case KVM_CAP_PPC_ENABLE_HCALL: {
1305 unsigned long hcall = cap->args[0];
1306
1307 r = -EINVAL;
1308 if (hcall > MAX_HCALL_OPCODE || (hcall & 3) ||
1309 cap->args[1] > 1)
1310 break;
1311 if (!kvmppc_book3s_hcall_implemented(kvm, hcall))
1312 break;
1313 if (cap->args[1])
1314 set_bit(hcall / 4, kvm->arch.enabled_hcalls);
1315 else
1316 clear_bit(hcall / 4, kvm->arch.enabled_hcalls);
1317 r = 0;
1318 break;
1319 }
1320#endif
1321 default:
1322 r = -EINVAL;
1323 break;
1324 }
1325
1326 return r;
1327}
1328
1329long kvm_arch_vm_ioctl(struct file *filp,
1330 unsigned int ioctl, unsigned long arg)
1331{
1332 struct kvm *kvm __maybe_unused = filp->private_data;
1333 void __user *argp = (void __user *)arg;
1334 long r;
1335
1336 switch (ioctl) {
1337 case KVM_PPC_GET_PVINFO: {
1338 struct kvm_ppc_pvinfo pvinfo;
1339 memset(&pvinfo, 0, sizeof(pvinfo));
1340 r = kvm_vm_ioctl_get_pvinfo(&pvinfo);
1341 if (copy_to_user(argp, &pvinfo, sizeof(pvinfo))) {
1342 r = -EFAULT;
1343 goto out;
1344 }
1345
1346 break;
1347 }
1348 case KVM_ENABLE_CAP:
1349 {
1350 struct kvm_enable_cap cap;
1351 r = -EFAULT;
1352 if (copy_from_user(&cap, argp, sizeof(cap)))
1353 goto out;
1354 r = kvm_vm_ioctl_enable_cap(kvm, &cap);
1355 break;
1356 }
1357#ifdef CONFIG_PPC_BOOK3S_64
1358 case KVM_CREATE_SPAPR_TCE_64: {
1359 struct kvm_create_spapr_tce_64 create_tce_64;
1360
1361 r = -EFAULT;
1362 if (copy_from_user(&create_tce_64, argp, sizeof(create_tce_64)))
1363 goto out;
1364 if (create_tce_64.flags) {
1365 r = -EINVAL;
1366 goto out;
1367 }
1368 r = kvm_vm_ioctl_create_spapr_tce(kvm, &create_tce_64);
1369 goto out;
1370 }
1371 case KVM_CREATE_SPAPR_TCE: {
1372 struct kvm_create_spapr_tce create_tce;
1373 struct kvm_create_spapr_tce_64 create_tce_64;
1374
1375 r = -EFAULT;
1376 if (copy_from_user(&create_tce, argp, sizeof(create_tce)))
1377 goto out;
1378
1379 create_tce_64.liobn = create_tce.liobn;
1380 create_tce_64.page_shift = IOMMU_PAGE_SHIFT_4K;
1381 create_tce_64.offset = 0;
1382 create_tce_64.size = create_tce.window_size >>
1383 IOMMU_PAGE_SHIFT_4K;
1384 create_tce_64.flags = 0;
1385 r = kvm_vm_ioctl_create_spapr_tce(kvm, &create_tce_64);
1386 goto out;
1387 }
1388 case KVM_PPC_GET_SMMU_INFO: {
1389 struct kvm_ppc_smmu_info info;
1390 struct kvm *kvm = filp->private_data;
1391
1392 memset(&info, 0, sizeof(info));
1393 r = kvm->arch.kvm_ops->get_smmu_info(kvm, &info);
1394 if (r >= 0 && copy_to_user(argp, &info, sizeof(info)))
1395 r = -EFAULT;
1396 break;
1397 }
1398 case KVM_PPC_RTAS_DEFINE_TOKEN: {
1399 struct kvm *kvm = filp->private_data;
1400
1401 r = kvm_vm_ioctl_rtas_define_token(kvm, argp);
1402 break;
1403 }
1404 default: {
1405 struct kvm *kvm = filp->private_data;
1406 r = kvm->arch.kvm_ops->arch_vm_ioctl(filp, ioctl, arg);
1407 }
1408#else
1409 default:
1410 r = -ENOTTY;
1411#endif
1412 }
1413out:
1414 return r;
1415}
1416
1417static unsigned long lpid_inuse[BITS_TO_LONGS(KVMPPC_NR_LPIDS)];
1418static unsigned long nr_lpids;
1419
1420long kvmppc_alloc_lpid(void)
1421{
1422 long lpid;
1423
1424 do {
1425 lpid = find_first_zero_bit(lpid_inuse, KVMPPC_NR_LPIDS);
1426 if (lpid >= nr_lpids) {
1427 pr_err("%s: No LPIDs free\n", __func__);
1428 return -ENOMEM;
1429 }
1430 } while (test_and_set_bit(lpid, lpid_inuse));
1431
1432 return lpid;
1433}
1434EXPORT_SYMBOL_GPL(kvmppc_alloc_lpid);
1435
1436void kvmppc_claim_lpid(long lpid)
1437{
1438 set_bit(lpid, lpid_inuse);
1439}
1440EXPORT_SYMBOL_GPL(kvmppc_claim_lpid);
1441
1442void kvmppc_free_lpid(long lpid)
1443{
1444 clear_bit(lpid, lpid_inuse);
1445}
1446EXPORT_SYMBOL_GPL(kvmppc_free_lpid);
1447
1448void kvmppc_init_lpid(unsigned long nr_lpids_param)
1449{
1450 nr_lpids = min_t(unsigned long, KVMPPC_NR_LPIDS, nr_lpids_param);
1451 memset(lpid_inuse, 0, sizeof(lpid_inuse));
1452}
1453EXPORT_SYMBOL_GPL(kvmppc_init_lpid);
1454
1455int kvm_arch_init(void *opaque)
1456{
1457 return 0;
1458}
1459
1460EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_ppc_instr);
1461
