1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18#include <linux/kvm_host.h>
19#include <linux/kernel.h>
20#include <linux/err.h>
21#include <linux/slab.h>
22#include <linux/preempt.h>
23#include <linux/sched/signal.h>
24#include <linux/sched/stat.h>
25#include <linux/delay.h>
26#include <linux/export.h>
27#include <linux/fs.h>
28#include <linux/anon_inodes.h>
29#include <linux/cpu.h>
30#include <linux/cpumask.h>
31#include <linux/spinlock.h>
32#include <linux/page-flags.h>
33#include <linux/srcu.h>
34#include <linux/miscdevice.h>
35#include <linux/debugfs.h>
36#include <linux/gfp.h>
37#include <linux/vmalloc.h>
38#include <linux/highmem.h>
39#include <linux/hugetlb.h>
40#include <linux/kvm_irqfd.h>
41#include <linux/irqbypass.h>
42#include <linux/module.h>
43#include <linux/compiler.h>
44#include <linux/of.h>
45
46#include <asm/ftrace.h>
47#include <asm/reg.h>
48#include <asm/ppc-opcode.h>
49#include <asm/asm-prototypes.h>
50#include <asm/archrandom.h>
51#include <asm/debug.h>
52#include <asm/disassemble.h>
53#include <asm/cputable.h>
54#include <asm/cacheflush.h>
55#include <linux/uaccess.h>
56#include <asm/io.h>
57#include <asm/kvm_ppc.h>
58#include <asm/kvm_book3s.h>
59#include <asm/mmu_context.h>
60#include <asm/lppaca.h>
61#include <asm/processor.h>
62#include <asm/cputhreads.h>
63#include <asm/page.h>
64#include <asm/hvcall.h>
65#include <asm/switch_to.h>
66#include <asm/smp.h>
67#include <asm/dbell.h>
68#include <asm/hmi.h>
69#include <asm/pnv-pci.h>
70#include <asm/mmu.h>
71#include <asm/opal.h>
72#include <asm/xics.h>
73#include <asm/xive.h>
74#include <asm/hw_breakpoint.h>
75
76#include "book3s.h"
77
78#define CREATE_TRACE_POINTS
79#include "trace_hv.h"
80
81
82
83
84
85
86#define RESUME_PAGE_FAULT (RESUME_GUEST | RESUME_FLAG_ARCH1)
87
88#define RESUME_PASSTHROUGH (RESUME_GUEST | RESUME_FLAG_ARCH2)
89
90
91#define TB_NIL (~(u64)0)
92
93static DECLARE_BITMAP(default_enabled_hcalls, MAX_HCALL_OPCODE/4 + 1);
94
95static int dynamic_mt_modes = 6;
96module_param(dynamic_mt_modes, int, 0644);
97MODULE_PARM_DESC(dynamic_mt_modes, "Set of allowed dynamic micro-threading modes: 0 (= none), 2, 4, or 6 (= 2 or 4)");
98static int target_smt_mode;
99module_param(target_smt_mode, int, 0644);
100MODULE_PARM_DESC(target_smt_mode, "Target threads per core (0 = max)");
101
102static bool indep_threads_mode = true;
103module_param(indep_threads_mode, bool, S_IRUGO | S_IWUSR);
104MODULE_PARM_DESC(indep_threads_mode, "Independent-threads mode (only on POWER9)");
105
106static bool one_vm_per_core;
107module_param(one_vm_per_core, bool, S_IRUGO | S_IWUSR);
108MODULE_PARM_DESC(one_vm_per_core, "Only run vCPUs from the same VM on a core (requires indep_threads_mode=N)");
109
110#ifdef CONFIG_KVM_XICS
111static struct kernel_param_ops module_param_ops = {
112 .set = param_set_int,
113 .get = param_get_int,
114};
115
116module_param_cb(kvm_irq_bypass, &module_param_ops, &kvm_irq_bypass, 0644);
117MODULE_PARM_DESC(kvm_irq_bypass, "Bypass passthrough interrupt optimization");
118
119module_param_cb(h_ipi_redirect, &module_param_ops, &h_ipi_redirect, 0644);
120MODULE_PARM_DESC(h_ipi_redirect, "Redirect H_IPI wakeup to a free host core");
121#endif
122
123
124static bool nested = true;
125module_param(nested, bool, S_IRUGO | S_IWUSR);
126MODULE_PARM_DESC(nested, "Enable nested virtualization (only on POWER9)");
127
128static inline bool nesting_enabled(struct kvm *kvm)
129{
130 return kvm->arch.nested_enable && kvm_is_radix(kvm);
131}
132
133
134static bool no_mixing_hpt_and_radix;
135
136static void kvmppc_end_cede(struct kvm_vcpu *vcpu);
137static int kvmppc_hv_setup_htab_rma(struct kvm_vcpu *vcpu);
138
139
140
141
142
143
144#define RWMR_RPA_P8_1THREAD 0x164520C62609AECAUL
145#define RWMR_RPA_P8_2THREAD 0x7FFF2908450D8DA9UL
146#define RWMR_RPA_P8_3THREAD 0x164520C62609AECAUL
147#define RWMR_RPA_P8_4THREAD 0x199A421245058DA9UL
148#define RWMR_RPA_P8_5THREAD 0x164520C62609AECAUL
149#define RWMR_RPA_P8_6THREAD 0x164520C62609AECAUL
150#define RWMR_RPA_P8_7THREAD 0x164520C62609AECAUL
151#define RWMR_RPA_P8_8THREAD 0x164520C62609AECAUL
152
153static unsigned long p8_rwmr_values[MAX_SMT_THREADS + 1] = {
154 RWMR_RPA_P8_1THREAD,
155 RWMR_RPA_P8_1THREAD,
156 RWMR_RPA_P8_2THREAD,
157 RWMR_RPA_P8_3THREAD,
158 RWMR_RPA_P8_4THREAD,
159 RWMR_RPA_P8_5THREAD,
160 RWMR_RPA_P8_6THREAD,
161 RWMR_RPA_P8_7THREAD,
162 RWMR_RPA_P8_8THREAD,
163};
164
165static inline struct kvm_vcpu *next_runnable_thread(struct kvmppc_vcore *vc,
166 int *ip)
167{
168 int i = *ip;
169 struct kvm_vcpu *vcpu;
170
171 while (++i < MAX_SMT_THREADS) {
172 vcpu = READ_ONCE(vc->runnable_threads[i]);
173 if (vcpu) {
174 *ip = i;
175 return vcpu;
176 }
177 }
178 return NULL;
179}
180
181
182#define for_each_runnable_thread(i, vcpu, vc) \
183 for (i = -1; (vcpu = next_runnable_thread(vc, &i)); )
184
185static bool kvmppc_ipi_thread(int cpu)
186{
187 unsigned long msg = PPC_DBELL_TYPE(PPC_DBELL_SERVER);
188
189
190 if (kvmhv_on_pseries())
191 return false;
192
193
194 if (cpu_has_feature(CPU_FTR_ARCH_300)) {
195 msg |= get_hard_smp_processor_id(cpu);
196 smp_mb();
197 __asm__ __volatile__ (PPC_MSGSND(%0) : : "r" (msg));
198 return true;
199 }
200
201
202 if (cpu_has_feature(CPU_FTR_ARCH_207S)) {
203 preempt_disable();
204 if (cpu_first_thread_sibling(cpu) ==
205 cpu_first_thread_sibling(smp_processor_id())) {
206 msg |= cpu_thread_in_core(cpu);
207 smp_mb();
208 __asm__ __volatile__ (PPC_MSGSND(%0) : : "r" (msg));
209 preempt_enable();
210 return true;
211 }
212 preempt_enable();
213 }
214
215#if defined(CONFIG_PPC_ICP_NATIVE) && defined(CONFIG_SMP)
216 if (cpu >= 0 && cpu < nr_cpu_ids) {
217 if (paca_ptrs[cpu]->kvm_hstate.xics_phys) {
218 xics_wake_cpu(cpu);
219 return true;
220 }
221 opal_int_set_mfrr(get_hard_smp_processor_id(cpu), IPI_PRIORITY);
222 return true;
223 }
224#endif
225
226 return false;
227}
228
229static void kvmppc_fast_vcpu_kick_hv(struct kvm_vcpu *vcpu)
230{
231 int cpu;
232 struct swait_queue_head *wqp;
233
234 wqp = kvm_arch_vcpu_wq(vcpu);
235 if (swq_has_sleeper(wqp)) {
236 swake_up_one(wqp);
237 ++vcpu->stat.halt_wakeup;
238 }
239
240 cpu = READ_ONCE(vcpu->arch.thread_cpu);
241 if (cpu >= 0 && kvmppc_ipi_thread(cpu))
242 return;
243
244
245 cpu = vcpu->cpu;
246 if (cpu >= 0 && cpu < nr_cpu_ids && cpu_online(cpu))
247 smp_send_reschedule(cpu);
248}
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283static void kvmppc_core_start_stolen(struct kvmppc_vcore *vc)
284{
285 unsigned long flags;
286
287 spin_lock_irqsave(&vc->stoltb_lock, flags);
288 vc->preempt_tb = mftb();
289 spin_unlock_irqrestore(&vc->stoltb_lock, flags);
290}
291
292static void kvmppc_core_end_stolen(struct kvmppc_vcore *vc)
293{
294 unsigned long flags;
295
296 spin_lock_irqsave(&vc->stoltb_lock, flags);
297 if (vc->preempt_tb != TB_NIL) {
298 vc->stolen_tb += mftb() - vc->preempt_tb;
299 vc->preempt_tb = TB_NIL;
300 }
301 spin_unlock_irqrestore(&vc->stoltb_lock, flags);
302}
303
304static void kvmppc_core_vcpu_load_hv(struct kvm_vcpu *vcpu, int cpu)
305{
306 struct kvmppc_vcore *vc = vcpu->arch.vcore;
307 unsigned long flags;
308
309
310
311
312
313
314
315 if (vc->runner == vcpu && vc->vcore_state >= VCORE_SLEEPING)
316 kvmppc_core_end_stolen(vc);
317
318 spin_lock_irqsave(&vcpu->arch.tbacct_lock, flags);
319 if (vcpu->arch.state == KVMPPC_VCPU_BUSY_IN_HOST &&
320 vcpu->arch.busy_preempt != TB_NIL) {
321 vcpu->arch.busy_stolen += mftb() - vcpu->arch.busy_preempt;
322 vcpu->arch.busy_preempt = TB_NIL;
323 }
324 spin_unlock_irqrestore(&vcpu->arch.tbacct_lock, flags);
325}
326
327static void kvmppc_core_vcpu_put_hv(struct kvm_vcpu *vcpu)
328{
329 struct kvmppc_vcore *vc = vcpu->arch.vcore;
330 unsigned long flags;
331
332 if (vc->runner == vcpu && vc->vcore_state >= VCORE_SLEEPING)
333 kvmppc_core_start_stolen(vc);
334
335 spin_lock_irqsave(&vcpu->arch.tbacct_lock, flags);
336 if (vcpu->arch.state == KVMPPC_VCPU_BUSY_IN_HOST)
337 vcpu->arch.busy_preempt = mftb();
338 spin_unlock_irqrestore(&vcpu->arch.tbacct_lock, flags);
339}
340
341static void kvmppc_set_msr_hv(struct kvm_vcpu *vcpu, u64 msr)
342{
343
344
345
346
347 if ((msr & MSR_TS_MASK) == MSR_TS_MASK)
348 msr &= ~MSR_TS_MASK;
349 vcpu->arch.shregs.msr = msr;
350 kvmppc_end_cede(vcpu);
351}
352
353static void kvmppc_set_pvr_hv(struct kvm_vcpu *vcpu, u32 pvr)
354{
355 vcpu->arch.pvr = pvr;
356}
357
358
359#define PCR_ARCH_300 (PCR_ARCH_207 << 1)
360
361static int kvmppc_set_arch_compat(struct kvm_vcpu *vcpu, u32 arch_compat)
362{
363 unsigned long host_pcr_bit = 0, guest_pcr_bit = 0;
364 struct kvmppc_vcore *vc = vcpu->arch.vcore;
365
366
367 if (cpu_has_feature(CPU_FTR_ARCH_300))
368 host_pcr_bit = PCR_ARCH_300;
369 else if (cpu_has_feature(CPU_FTR_ARCH_207S))
370 host_pcr_bit = PCR_ARCH_207;
371 else if (cpu_has_feature(CPU_FTR_ARCH_206))
372 host_pcr_bit = PCR_ARCH_206;
373 else
374 host_pcr_bit = PCR_ARCH_205;
375
376
377 guest_pcr_bit = host_pcr_bit;
378 if (arch_compat) {
379 switch (arch_compat) {
380 case PVR_ARCH_205:
381 guest_pcr_bit = PCR_ARCH_205;
382 break;
383 case PVR_ARCH_206:
384 case PVR_ARCH_206p:
385 guest_pcr_bit = PCR_ARCH_206;
386 break;
387 case PVR_ARCH_207:
388 guest_pcr_bit = PCR_ARCH_207;
389 break;
390 case PVR_ARCH_300:
391 guest_pcr_bit = PCR_ARCH_300;
392 break;
393 default:
394 return -EINVAL;
395 }
396 }
397
398
399 if (guest_pcr_bit > host_pcr_bit)
400 return -EINVAL;
401
402 spin_lock(&vc->lock);
403 vc->arch_compat = arch_compat;
404
405 vc->pcr = host_pcr_bit - guest_pcr_bit;
406 spin_unlock(&vc->lock);
407
408 return 0;
409}
410
411static void kvmppc_dump_regs(struct kvm_vcpu *vcpu)
412{
413 int r;
414
415 pr_err("vcpu %p (%d):\n", vcpu, vcpu->vcpu_id);
416 pr_err("pc = %.16lx msr = %.16llx trap = %x\n",
417 vcpu->arch.regs.nip, vcpu->arch.shregs.msr, vcpu->arch.trap);
418 for (r = 0; r < 16; ++r)
419 pr_err("r%2d = %.16lx r%d = %.16lx\n",
420 r, kvmppc_get_gpr(vcpu, r),
421 r+16, kvmppc_get_gpr(vcpu, r+16));
422 pr_err("ctr = %.16lx lr = %.16lx\n",
423 vcpu->arch.regs.ctr, vcpu->arch.regs.link);
424 pr_err("srr0 = %.16llx srr1 = %.16llx\n",
425 vcpu->arch.shregs.srr0, vcpu->arch.shregs.srr1);
426 pr_err("sprg0 = %.16llx sprg1 = %.16llx\n",
427 vcpu->arch.shregs.sprg0, vcpu->arch.shregs.sprg1);
428 pr_err("sprg2 = %.16llx sprg3 = %.16llx\n",
429 vcpu->arch.shregs.sprg2, vcpu->arch.shregs.sprg3);
430 pr_err("cr = %.8lx xer = %.16lx dsisr = %.8x\n",
431 vcpu->arch.regs.ccr, vcpu->arch.regs.xer, vcpu->arch.shregs.dsisr);
432 pr_err("dar = %.16llx\n", vcpu->arch.shregs.dar);
433 pr_err("fault dar = %.16lx dsisr = %.8x\n",
434 vcpu->arch.fault_dar, vcpu->arch.fault_dsisr);
435 pr_err("SLB (%d entries):\n", vcpu->arch.slb_max);
436 for (r = 0; r < vcpu->arch.slb_max; ++r)
437 pr_err(" ESID = %.16llx VSID = %.16llx\n",
438 vcpu->arch.slb[r].orige, vcpu->arch.slb[r].origv);
439 pr_err("lpcr = %.16lx sdr1 = %.16lx last_inst = %.8x\n",
440 vcpu->arch.vcore->lpcr, vcpu->kvm->arch.sdr1,
441 vcpu->arch.last_inst);
442}
443
444static struct kvm_vcpu *kvmppc_find_vcpu(struct kvm *kvm, int id)
445{
446 return kvm_get_vcpu_by_id(kvm, id);
447}
448
449static void init_vpa(struct kvm_vcpu *vcpu, struct lppaca *vpa)
450{
451 vpa->__old_status |= LPPACA_OLD_SHARED_PROC;
452 vpa->yield_count = cpu_to_be32(1);
453}
454
455static int set_vpa(struct kvm_vcpu *vcpu, struct kvmppc_vpa *v,
456 unsigned long addr, unsigned long len)
457{
458
459 if (addr & (L1_CACHE_BYTES - 1))
460 return -EINVAL;
461 spin_lock(&vcpu->arch.vpa_update_lock);
462 if (v->next_gpa != addr || v->len != len) {
463 v->next_gpa = addr;
464 v->len = addr ? len : 0;
465 v->update_pending = 1;
466 }
467 spin_unlock(&vcpu->arch.vpa_update_lock);
468 return 0;
469}
470
471
472struct reg_vpa {
473 u32 dummy;
474 union {
475 __be16 hword;
476 __be32 word;
477 } length;
478};
479
480static int vpa_is_registered(struct kvmppc_vpa *vpap)
481{
482 if (vpap->update_pending)
483 return vpap->next_gpa != 0;
484 return vpap->pinned_addr != NULL;
485}
486
487static unsigned long do_h_register_vpa(struct kvm_vcpu *vcpu,
488 unsigned long flags,
489 unsigned long vcpuid, unsigned long vpa)
490{
491 struct kvm *kvm = vcpu->kvm;
492 unsigned long len, nb;
493 void *va;
494 struct kvm_vcpu *tvcpu;
495 int err;
496 int subfunc;
497 struct kvmppc_vpa *vpap;
498
499 tvcpu = kvmppc_find_vcpu(kvm, vcpuid);
500 if (!tvcpu)
501 return H_PARAMETER;
502
503 subfunc = (flags >> H_VPA_FUNC_SHIFT) & H_VPA_FUNC_MASK;
504 if (subfunc == H_VPA_REG_VPA || subfunc == H_VPA_REG_DTL ||
505 subfunc == H_VPA_REG_SLB) {
506
507 if ((vpa & (L1_CACHE_BYTES - 1)) || !vpa)
508 return H_PARAMETER;
509
510
511 va = kvmppc_pin_guest_page(kvm, vpa, &nb);
512 if (va == NULL)
513 return H_PARAMETER;
514 if (subfunc == H_VPA_REG_VPA)
515 len = be16_to_cpu(((struct reg_vpa *)va)->length.hword);
516 else
517 len = be32_to_cpu(((struct reg_vpa *)va)->length.word);
518 kvmppc_unpin_guest_page(kvm, va, vpa, false);
519
520
521 if (len > nb || len < sizeof(struct reg_vpa))
522 return H_PARAMETER;
523 } else {
524 vpa = 0;
525 len = 0;
526 }
527
528 err = H_PARAMETER;
529 vpap = NULL;
530 spin_lock(&tvcpu->arch.vpa_update_lock);
531
532 switch (subfunc) {
533 case H_VPA_REG_VPA:
534
535
536
537
538
539
540 BUILD_BUG_ON(sizeof(struct lppaca) != 640);
541 if (len < sizeof(struct lppaca))
542 break;
543 vpap = &tvcpu->arch.vpa;
544 err = 0;
545 break;
546
547 case H_VPA_REG_DTL:
548 if (len < sizeof(struct dtl_entry))
549 break;
550 len -= len % sizeof(struct dtl_entry);
551
552
553 err = H_RESOURCE;
554 if (!vpa_is_registered(&tvcpu->arch.vpa))
555 break;
556
557 vpap = &tvcpu->arch.dtl;
558 err = 0;
559 break;
560
561 case H_VPA_REG_SLB:
562
563 err = H_RESOURCE;
564 if (!vpa_is_registered(&tvcpu->arch.vpa))
565 break;
566
567 vpap = &tvcpu->arch.slb_shadow;
568 err = 0;
569 break;
570
571 case H_VPA_DEREG_VPA:
572
573 err = H_RESOURCE;
574 if (vpa_is_registered(&tvcpu->arch.dtl) ||
575 vpa_is_registered(&tvcpu->arch.slb_shadow))
576 break;
577
578 vpap = &tvcpu->arch.vpa;
579 err = 0;
580 break;
581
582 case H_VPA_DEREG_DTL:
583 vpap = &tvcpu->arch.dtl;
584 err = 0;
585 break;
586
587 case H_VPA_DEREG_SLB:
588 vpap = &tvcpu->arch.slb_shadow;
589 err = 0;
590 break;
591 }
592
593 if (vpap) {
594 vpap->next_gpa = vpa;
595 vpap->len = len;
596 vpap->update_pending = 1;
597 }
598
599 spin_unlock(&tvcpu->arch.vpa_update_lock);
600
601 return err;
602}
603
604static void kvmppc_update_vpa(struct kvm_vcpu *vcpu, struct kvmppc_vpa *vpap)
605{
606 struct kvm *kvm = vcpu->kvm;
607 void *va;
608 unsigned long nb;
609 unsigned long gpa;
610
611
612
613
614
615
616
617
618
619 for (;;) {
620 gpa = vpap->next_gpa;
621 spin_unlock(&vcpu->arch.vpa_update_lock);
622 va = NULL;
623 nb = 0;
624 if (gpa)
625 va = kvmppc_pin_guest_page(kvm, gpa, &nb);
626 spin_lock(&vcpu->arch.vpa_update_lock);
627 if (gpa == vpap->next_gpa)
628 break;
629
630 if (va)
631 kvmppc_unpin_guest_page(kvm, va, gpa, false);
632 }
633
634 vpap->update_pending = 0;
635 if (va && nb < vpap->len) {
636
637
638
639
640
641 kvmppc_unpin_guest_page(kvm, va, gpa, false);
642 va = NULL;
643 }
644 if (vpap->pinned_addr)
645 kvmppc_unpin_guest_page(kvm, vpap->pinned_addr, vpap->gpa,
646 vpap->dirty);
647 vpap->gpa = gpa;
648 vpap->pinned_addr = va;
649 vpap->dirty = false;
650 if (va)
651 vpap->pinned_end = va + vpap->len;
652}
653
654static void kvmppc_update_vpas(struct kvm_vcpu *vcpu)
655{
656 if (!(vcpu->arch.vpa.update_pending ||
657 vcpu->arch.slb_shadow.update_pending ||
658 vcpu->arch.dtl.update_pending))
659 return;
660
661 spin_lock(&vcpu->arch.vpa_update_lock);
662 if (vcpu->arch.vpa.update_pending) {
663 kvmppc_update_vpa(vcpu, &vcpu->arch.vpa);
664 if (vcpu->arch.vpa.pinned_addr)
665 init_vpa(vcpu, vcpu->arch.vpa.pinned_addr);
666 }
667 if (vcpu->arch.dtl.update_pending) {
668 kvmppc_update_vpa(vcpu, &vcpu->arch.dtl);
669 vcpu->arch.dtl_ptr = vcpu->arch.dtl.pinned_addr;
670 vcpu->arch.dtl_index = 0;
671 }
672 if (vcpu->arch.slb_shadow.update_pending)
673 kvmppc_update_vpa(vcpu, &vcpu->arch.slb_shadow);
674 spin_unlock(&vcpu->arch.vpa_update_lock);
675}
676
677
678
679
680
681static u64 vcore_stolen_time(struct kvmppc_vcore *vc, u64 now)
682{
683 u64 p;
684 unsigned long flags;
685
686 spin_lock_irqsave(&vc->stoltb_lock, flags);
687 p = vc->stolen_tb;
688 if (vc->vcore_state != VCORE_INACTIVE &&
689 vc->preempt_tb != TB_NIL)
690 p += now - vc->preempt_tb;
691 spin_unlock_irqrestore(&vc->stoltb_lock, flags);
692 return p;
693}
694
695static void kvmppc_create_dtl_entry(struct kvm_vcpu *vcpu,
696 struct kvmppc_vcore *vc)
697{
698 struct dtl_entry *dt;
699 struct lppaca *vpa;
700 unsigned long stolen;
701 unsigned long core_stolen;
702 u64 now;
703 unsigned long flags;
704
705 dt = vcpu->arch.dtl_ptr;
706 vpa = vcpu->arch.vpa.pinned_addr;
707 now = mftb();
708 core_stolen = vcore_stolen_time(vc, now);
709 stolen = core_stolen - vcpu->arch.stolen_logged;
710 vcpu->arch.stolen_logged = core_stolen;
711 spin_lock_irqsave(&vcpu->arch.tbacct_lock, flags);
712 stolen += vcpu->arch.busy_stolen;
713 vcpu->arch.busy_stolen = 0;
714 spin_unlock_irqrestore(&vcpu->arch.tbacct_lock, flags);
715 if (!dt || !vpa)
716 return;
717 memset(dt, 0, sizeof(struct dtl_entry));
718 dt->dispatch_reason = 7;
719 dt->processor_id = cpu_to_be16(vc->pcpu + vcpu->arch.ptid);
720 dt->timebase = cpu_to_be64(now + vc->tb_offset);
721 dt->enqueue_to_dispatch_time = cpu_to_be32(stolen);
722 dt->srr0 = cpu_to_be64(kvmppc_get_pc(vcpu));
723 dt->srr1 = cpu_to_be64(vcpu->arch.shregs.msr);
724 ++dt;
725 if (dt == vcpu->arch.dtl.pinned_end)
726 dt = vcpu->arch.dtl.pinned_addr;
727 vcpu->arch.dtl_ptr = dt;
728
729 smp_wmb();
730 vpa->dtl_idx = cpu_to_be64(++vcpu->arch.dtl_index);
731 vcpu->arch.dtl.dirty = true;
732}
733
734
735static bool kvmppc_doorbell_pending(struct kvm_vcpu *vcpu)
736{
737 int thr;
738 struct kvmppc_vcore *vc;
739
740 if (vcpu->arch.doorbell_request)
741 return true;
742
743
744
745
746
747 smp_rmb();
748 vc = vcpu->arch.vcore;
749 thr = vcpu->vcpu_id - vc->first_vcpuid;
750 return !!(vc->dpdes & (1 << thr));
751}
752
753static bool kvmppc_power8_compatible(struct kvm_vcpu *vcpu)
754{
755 if (vcpu->arch.vcore->arch_compat >= PVR_ARCH_207)
756 return true;
757 if ((!vcpu->arch.vcore->arch_compat) &&
758 cpu_has_feature(CPU_FTR_ARCH_207S))
759 return true;
760 return false;
761}
762
763static int kvmppc_h_set_mode(struct kvm_vcpu *vcpu, unsigned long mflags,
764 unsigned long resource, unsigned long value1,
765 unsigned long value2)
766{
767 switch (resource) {
768 case H_SET_MODE_RESOURCE_SET_CIABR:
769 if (!kvmppc_power8_compatible(vcpu))
770 return H_P2;
771 if (value2)
772 return H_P4;
773 if (mflags)
774 return H_UNSUPPORTED_FLAG_START;
775
776 if ((value1 & CIABR_PRIV) == CIABR_PRIV_HYPER)
777 return H_P3;
778 vcpu->arch.ciabr = value1;
779 return H_SUCCESS;
780 case H_SET_MODE_RESOURCE_SET_DAWR:
781 if (!kvmppc_power8_compatible(vcpu))
782 return H_P2;
783 if (!ppc_breakpoint_available())
784 return H_P2;
785 if (mflags)
786 return H_UNSUPPORTED_FLAG_START;
787 if (value2 & DABRX_HYP)
788 return H_P4;
789 vcpu->arch.dawr = value1;
790 vcpu->arch.dawrx = value2;
791 return H_SUCCESS;
792 default:
793 return H_TOO_HARD;
794 }
795}
796
797
798static int kvmppc_copy_guest(struct kvm *kvm, gpa_t to, gpa_t from,
799 unsigned long len)
800{
801 struct kvm_memory_slot *to_memslot = NULL;
802 struct kvm_memory_slot *from_memslot = NULL;
803 unsigned long to_addr, from_addr;
804 int r;
805
806
807 from_memslot = gfn_to_memslot(kvm, from >> PAGE_SHIFT);
808 if (!from_memslot)
809 return -EFAULT;
810 if ((from + len) >= ((from_memslot->base_gfn + from_memslot->npages)
811 << PAGE_SHIFT))
812 return -EINVAL;
813 from_addr = gfn_to_hva_memslot(from_memslot, from >> PAGE_SHIFT);
814 if (kvm_is_error_hva(from_addr))
815 return -EFAULT;
816 from_addr |= (from & (PAGE_SIZE - 1));
817
818
819 to_memslot = gfn_to_memslot(kvm, to >> PAGE_SHIFT);
820 if (!to_memslot)
821 return -EFAULT;
822 if ((to + len) >= ((to_memslot->base_gfn + to_memslot->npages)
823 << PAGE_SHIFT))
824 return -EINVAL;
825 to_addr = gfn_to_hva_memslot(to_memslot, to >> PAGE_SHIFT);
826 if (kvm_is_error_hva(to_addr))
827 return -EFAULT;
828 to_addr |= (to & (PAGE_SIZE - 1));
829
830
831 r = raw_copy_in_user((void __user *)to_addr, (void __user *)from_addr,
832 len);
833 if (r)
834 return -EFAULT;
835 mark_page_dirty(kvm, to >> PAGE_SHIFT);
836 return 0;
837}
838
839static long kvmppc_h_page_init(struct kvm_vcpu *vcpu, unsigned long flags,
840 unsigned long dest, unsigned long src)
841{
842 u64 pg_sz = SZ_4K;
843 u64 pg_mask = SZ_4K - 1;
844 int ret;
845
846
847 if (flags & ~(H_ICACHE_INVALIDATE | H_ICACHE_SYNCHRONIZE |
848 H_ZERO_PAGE | H_COPY_PAGE | H_PAGE_SET_LOANED))
849 return H_PARAMETER;
850
851
852 if ((dest & pg_mask) || ((flags & H_COPY_PAGE) && (src & pg_mask)))
853 return H_PARAMETER;
854
855
856 if (flags & H_COPY_PAGE) {
857 ret = kvmppc_copy_guest(vcpu->kvm, dest, src, pg_sz);
858 if (ret < 0)
859 return H_PARAMETER;
860 } else if (flags & H_ZERO_PAGE) {
861 ret = kvm_clear_guest(vcpu->kvm, dest, pg_sz);
862 if (ret < 0)
863 return H_PARAMETER;
864 }
865
866
867
868 return H_SUCCESS;
869}
870
871static int kvm_arch_vcpu_yield_to(struct kvm_vcpu *target)
872{
873 struct kvmppc_vcore *vcore = target->arch.vcore;
874
875
876
877
878
879
880
881
882
883 spin_lock(&vcore->lock);
884 if (target->arch.state == KVMPPC_VCPU_RUNNABLE &&
885 vcore->vcore_state != VCORE_INACTIVE &&
886 vcore->runner)
887 target = vcore->runner;
888 spin_unlock(&vcore->lock);
889
890 return kvm_vcpu_yield_to(target);
891}
892
893static int kvmppc_get_yield_count(struct kvm_vcpu *vcpu)
894{
895 int yield_count = 0;
896 struct lppaca *lppaca;
897
898 spin_lock(&vcpu->arch.vpa_update_lock);
899 lppaca = (struct lppaca *)vcpu->arch.vpa.pinned_addr;
900 if (lppaca)
901 yield_count = be32_to_cpu(lppaca->yield_count);
902 spin_unlock(&vcpu->arch.vpa_update_lock);
903 return yield_count;
904}
905
906int kvmppc_pseries_do_hcall(struct kvm_vcpu *vcpu)
907{
908 unsigned long req = kvmppc_get_gpr(vcpu, 3);
909 unsigned long target, ret = H_SUCCESS;
910 int yield_count;
911 struct kvm_vcpu *tvcpu;
912 int idx, rc;
913
914 if (req <= MAX_HCALL_OPCODE &&
915 !test_bit(req/4, vcpu->kvm->arch.enabled_hcalls))
916 return RESUME_HOST;
917
918 switch (req) {
919 case H_CEDE:
920 break;
921 case H_PROD:
922 target = kvmppc_get_gpr(vcpu, 4);
923 tvcpu = kvmppc_find_vcpu(vcpu->kvm, target);
924 if (!tvcpu) {
925 ret = H_PARAMETER;
926 break;
927 }
928 tvcpu->arch.prodded = 1;
929 smp_mb();
930 if (tvcpu->arch.ceded)
931 kvmppc_fast_vcpu_kick_hv(tvcpu);
932 break;
933 case H_CONFER:
934 target = kvmppc_get_gpr(vcpu, 4);
935 if (target == -1)
936 break;
937 tvcpu = kvmppc_find_vcpu(vcpu->kvm, target);
938 if (!tvcpu) {
939 ret = H_PARAMETER;
940 break;
941 }
942 yield_count = kvmppc_get_gpr(vcpu, 5);
943 if (kvmppc_get_yield_count(tvcpu) != yield_count)
944 break;
945 kvm_arch_vcpu_yield_to(tvcpu);
946 break;
947 case H_REGISTER_VPA:
948 ret = do_h_register_vpa(vcpu, kvmppc_get_gpr(vcpu, 4),
949 kvmppc_get_gpr(vcpu, 5),
950 kvmppc_get_gpr(vcpu, 6));
951 break;
952 case H_RTAS:
953 if (list_empty(&vcpu->kvm->arch.rtas_tokens))
954 return RESUME_HOST;
955
956 idx = srcu_read_lock(&vcpu->kvm->srcu);
957 rc = kvmppc_rtas_hcall(vcpu);
958 srcu_read_unlock(&vcpu->kvm->srcu, idx);
959
960 if (rc == -ENOENT)
961 return RESUME_HOST;
962 else if (rc == 0)
963 break;
964
965
966 return rc;
967 case H_LOGICAL_CI_LOAD:
968 ret = kvmppc_h_logical_ci_load(vcpu);
969 if (ret == H_TOO_HARD)
970 return RESUME_HOST;
971 break;
972 case H_LOGICAL_CI_STORE:
973 ret = kvmppc_h_logical_ci_store(vcpu);
974 if (ret == H_TOO_HARD)
975 return RESUME_HOST;
976 break;
977 case H_SET_MODE:
978 ret = kvmppc_h_set_mode(vcpu, kvmppc_get_gpr(vcpu, 4),
979 kvmppc_get_gpr(vcpu, 5),
980 kvmppc_get_gpr(vcpu, 6),
981 kvmppc_get_gpr(vcpu, 7));
982 if (ret == H_TOO_HARD)
983 return RESUME_HOST;
984 break;
985 case H_XIRR:
986 case H_CPPR:
987 case H_EOI:
988 case H_IPI:
989 case H_IPOLL:
990 case H_XIRR_X:
991 if (kvmppc_xics_enabled(vcpu)) {
992 if (xics_on_xive()) {
993 ret = H_NOT_AVAILABLE;
994 return RESUME_GUEST;
995 }
996 ret = kvmppc_xics_hcall(vcpu, req);
997 break;
998 }
999 return RESUME_HOST;
1000 case H_SET_DABR:
1001 ret = kvmppc_h_set_dabr(vcpu, kvmppc_get_gpr(vcpu, 4));
1002 break;
1003 case H_SET_XDABR:
1004 ret = kvmppc_h_set_xdabr(vcpu, kvmppc_get_gpr(vcpu, 4),
1005 kvmppc_get_gpr(vcpu, 5));
1006 break;
1007#ifdef CONFIG_SPAPR_TCE_IOMMU
1008 case H_GET_TCE:
1009 ret = kvmppc_h_get_tce(vcpu, kvmppc_get_gpr(vcpu, 4),
1010 kvmppc_get_gpr(vcpu, 5));
1011 if (ret == H_TOO_HARD)
1012 return RESUME_HOST;
1013 break;
1014 case H_PUT_TCE:
1015 ret = kvmppc_h_put_tce(vcpu, kvmppc_get_gpr(vcpu, 4),
1016 kvmppc_get_gpr(vcpu, 5),
1017 kvmppc_get_gpr(vcpu, 6));
1018 if (ret == H_TOO_HARD)
1019 return RESUME_HOST;
1020 break;
1021 case H_PUT_TCE_INDIRECT:
1022 ret = kvmppc_h_put_tce_indirect(vcpu, kvmppc_get_gpr(vcpu, 4),
1023 kvmppc_get_gpr(vcpu, 5),
1024 kvmppc_get_gpr(vcpu, 6),
1025 kvmppc_get_gpr(vcpu, 7));
1026 if (ret == H_TOO_HARD)
1027 return RESUME_HOST;
1028 break;
1029 case H_STUFF_TCE:
1030 ret = kvmppc_h_stuff_tce(vcpu, kvmppc_get_gpr(vcpu, 4),
1031 kvmppc_get_gpr(vcpu, 5),
1032 kvmppc_get_gpr(vcpu, 6),
1033 kvmppc_get_gpr(vcpu, 7));
1034 if (ret == H_TOO_HARD)
1035 return RESUME_HOST;
1036 break;
1037#endif
1038 case H_RANDOM:
1039 if (!powernv_get_random_long(&vcpu->arch.regs.gpr[4]))
1040 ret = H_HARDWARE;
1041 break;
1042
1043 case H_SET_PARTITION_TABLE:
1044 ret = H_FUNCTION;
1045 if (nesting_enabled(vcpu->kvm))
1046 ret = kvmhv_set_partition_table(vcpu);
1047 break;
1048 case H_ENTER_NESTED:
1049 ret = H_FUNCTION;
1050 if (!nesting_enabled(vcpu->kvm))
1051 break;
1052 ret = kvmhv_enter_nested_guest(vcpu);
1053 if (ret == H_INTERRUPT) {
1054 kvmppc_set_gpr(vcpu, 3, 0);
1055 vcpu->arch.hcall_needed = 0;
1056 return -EINTR;
1057 } else if (ret == H_TOO_HARD) {
1058 kvmppc_set_gpr(vcpu, 3, 0);
1059 vcpu->arch.hcall_needed = 0;
1060 return RESUME_HOST;
1061 }
1062 break;
1063 case H_TLB_INVALIDATE:
1064 ret = H_FUNCTION;
1065 if (nesting_enabled(vcpu->kvm))
1066 ret = kvmhv_do_nested_tlbie(vcpu);
1067 break;
1068 case H_COPY_TOFROM_GUEST:
1069 ret = H_FUNCTION;
1070 if (nesting_enabled(vcpu->kvm))
1071 ret = kvmhv_copy_tofrom_guest_nested(vcpu);
1072 break;
1073 case H_PAGE_INIT:
1074 ret = kvmppc_h_page_init(vcpu, kvmppc_get_gpr(vcpu, 4),
1075 kvmppc_get_gpr(vcpu, 5),
1076 kvmppc_get_gpr(vcpu, 6));
1077 break;
1078 default:
1079 return RESUME_HOST;
1080 }
1081 kvmppc_set_gpr(vcpu, 3, ret);
1082 vcpu->arch.hcall_needed = 0;
1083 return RESUME_GUEST;
1084}
1085
1086
1087
1088
1089
1090
1091
1092static void kvmppc_nested_cede(struct kvm_vcpu *vcpu)
1093{
1094 vcpu->arch.shregs.msr |= MSR_EE;
1095 vcpu->arch.ceded = 1;
1096 smp_mb();
1097 if (vcpu->arch.prodded) {
1098 vcpu->arch.prodded = 0;
1099 smp_mb();
1100 vcpu->arch.ceded = 0;
1101 }
1102}
1103
1104static int kvmppc_hcall_impl_hv(unsigned long cmd)
1105{
1106 switch (cmd) {
1107 case H_CEDE:
1108 case H_PROD:
1109 case H_CONFER:
1110 case H_REGISTER_VPA:
1111 case H_SET_MODE:
1112 case H_LOGICAL_CI_LOAD:
1113 case H_LOGICAL_CI_STORE:
1114#ifdef CONFIG_KVM_XICS
1115 case H_XIRR:
1116 case H_CPPR:
1117 case H_EOI:
1118 case H_IPI:
1119 case H_IPOLL:
1120 case H_XIRR_X:
1121#endif
1122 case H_PAGE_INIT:
1123 return 1;
1124 }
1125
1126
1127 return kvmppc_hcall_impl_hv_realmode(cmd);
1128}
1129
1130static int kvmppc_emulate_debug_inst(struct kvm_run *run,
1131 struct kvm_vcpu *vcpu)
1132{
1133 u32 last_inst;
1134
1135 if (kvmppc_get_last_inst(vcpu, INST_GENERIC, &last_inst) !=
1136 EMULATE_DONE) {
1137
1138
1139
1140
1141 return RESUME_GUEST;
1142 }
1143
1144 if (last_inst == KVMPPC_INST_SW_BREAKPOINT) {
1145 run->exit_reason = KVM_EXIT_DEBUG;
1146 run->debug.arch.address = kvmppc_get_pc(vcpu);
1147 return RESUME_HOST;
1148 } else {
1149 kvmppc_core_queue_program(vcpu, SRR1_PROGILL);
1150 return RESUME_GUEST;
1151 }
1152}
1153
1154static void do_nothing(void *x)
1155{
1156}
1157
1158static unsigned long kvmppc_read_dpdes(struct kvm_vcpu *vcpu)
1159{
1160 int thr, cpu, pcpu, nthreads;
1161 struct kvm_vcpu *v;
1162 unsigned long dpdes;
1163
1164 nthreads = vcpu->kvm->arch.emul_smt_mode;
1165 dpdes = 0;
1166 cpu = vcpu->vcpu_id & ~(nthreads - 1);
1167 for (thr = 0; thr < nthreads; ++thr, ++cpu) {
1168 v = kvmppc_find_vcpu(vcpu->kvm, cpu);
1169 if (!v)
1170 continue;
1171
1172
1173
1174
1175
1176 pcpu = READ_ONCE(v->cpu);
1177 if (pcpu >= 0)
1178 smp_call_function_single(pcpu, do_nothing, NULL, 1);
1179 if (kvmppc_doorbell_pending(v))
1180 dpdes |= 1 << thr;
1181 }
1182 return dpdes;
1183}
1184
1185
1186
1187
1188
1189
1190
1191static int kvmppc_emulate_doorbell_instr(struct kvm_vcpu *vcpu)
1192{
1193 u32 inst, rb, thr;
1194 unsigned long arg;
1195 struct kvm *kvm = vcpu->kvm;
1196 struct kvm_vcpu *tvcpu;
1197
1198 if (kvmppc_get_last_inst(vcpu, INST_GENERIC, &inst) != EMULATE_DONE)
1199 return RESUME_GUEST;
1200 if (get_op(inst) != 31)
1201 return EMULATE_FAIL;
1202 rb = get_rb(inst);
1203 thr = vcpu->vcpu_id & (kvm->arch.emul_smt_mode - 1);
1204 switch (get_xop(inst)) {
1205 case OP_31_XOP_MSGSNDP:
1206 arg = kvmppc_get_gpr(vcpu, rb);
1207 if (((arg >> 27) & 0xf) != PPC_DBELL_SERVER)
1208 break;
1209 arg &= 0x3f;
1210 if (arg >= kvm->arch.emul_smt_mode)
1211 break;
1212 tvcpu = kvmppc_find_vcpu(kvm, vcpu->vcpu_id - thr + arg);
1213 if (!tvcpu)
1214 break;
1215 if (!tvcpu->arch.doorbell_request) {
1216 tvcpu->arch.doorbell_request = 1;
1217 kvmppc_fast_vcpu_kick_hv(tvcpu);
1218 }
1219 break;
1220 case OP_31_XOP_MSGCLRP:
1221 arg = kvmppc_get_gpr(vcpu, rb);
1222 if (((arg >> 27) & 0xf) != PPC_DBELL_SERVER)
1223 break;
1224 vcpu->arch.vcore->dpdes = 0;
1225 vcpu->arch.doorbell_request = 0;
1226 break;
1227 case OP_31_XOP_MFSPR:
1228 switch (get_sprn(inst)) {
1229 case SPRN_TIR:
1230 arg = thr;
1231 break;
1232 case SPRN_DPDES:
1233 arg = kvmppc_read_dpdes(vcpu);
1234 break;
1235 default:
1236 return EMULATE_FAIL;
1237 }
1238 kvmppc_set_gpr(vcpu, get_rt(inst), arg);
1239 break;
1240 default:
1241 return EMULATE_FAIL;
1242 }
1243 kvmppc_set_pc(vcpu, kvmppc_get_pc(vcpu) + 4);
1244 return RESUME_GUEST;
1245}
1246
1247static int kvmppc_handle_exit_hv(struct kvm_run *run, struct kvm_vcpu *vcpu,
1248 struct task_struct *tsk)
1249{
1250 int r = RESUME_HOST;
1251
1252 vcpu->stat.sum_exits++;
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262 if (vcpu->arch.shregs.msr & MSR_HV) {
1263 printk(KERN_EMERG "KVM trap in HV mode!\n");
1264 printk(KERN_EMERG "trap=0x%x | pc=0x%lx | msr=0x%llx\n",
1265 vcpu->arch.trap, kvmppc_get_pc(vcpu),
1266 vcpu->arch.shregs.msr);
1267 kvmppc_dump_regs(vcpu);
1268 run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
1269 run->hw.hardware_exit_reason = vcpu->arch.trap;
1270 return RESUME_HOST;
1271 }
1272 run->exit_reason = KVM_EXIT_UNKNOWN;
1273 run->ready_for_interrupt_injection = 1;
1274 switch (vcpu->arch.trap) {
1275
1276 case BOOK3S_INTERRUPT_HV_DECREMENTER:
1277 vcpu->stat.dec_exits++;
1278 r = RESUME_GUEST;
1279 break;
1280 case BOOK3S_INTERRUPT_EXTERNAL:
1281 case BOOK3S_INTERRUPT_H_DOORBELL:
1282 case BOOK3S_INTERRUPT_H_VIRT:
1283 vcpu->stat.ext_intr_exits++;
1284 r = RESUME_GUEST;
1285 break;
1286
1287 case BOOK3S_INTERRUPT_HMI:
1288 case BOOK3S_INTERRUPT_PERFMON:
1289 case BOOK3S_INTERRUPT_SYSTEM_RESET:
1290 r = RESUME_GUEST;
1291 break;
1292 case BOOK3S_INTERRUPT_MACHINE_CHECK:
1293
1294 machine_check_print_event_info(&vcpu->arch.mce_evt, false, true);
1295
1296
1297
1298
1299
1300
1301
1302 if (!vcpu->kvm->arch.fwnmi_enabled) {
1303 ulong flags = vcpu->arch.shregs.msr & 0x083c0000;
1304 kvmppc_core_queue_machine_check(vcpu, flags);
1305 r = RESUME_GUEST;
1306 break;
1307 }
1308
1309
1310 run->exit_reason = KVM_EXIT_NMI;
1311 run->hw.hardware_exit_reason = vcpu->arch.trap;
1312
1313 run->flags &= ~KVM_RUN_PPC_NMI_DISP_MASK;
1314
1315 if (vcpu->arch.mce_evt.disposition == MCE_DISPOSITION_RECOVERED)
1316 run->flags |= KVM_RUN_PPC_NMI_DISP_FULLY_RECOV;
1317 else
1318 run->flags |= KVM_RUN_PPC_NMI_DISP_NOT_RECOV;
1319
1320 r = RESUME_HOST;
1321 break;
1322 case BOOK3S_INTERRUPT_PROGRAM:
1323 {
1324 ulong flags;
1325
1326
1327
1328
1329
1330
1331 flags = vcpu->arch.shregs.msr & 0x1f0000ull;
1332 kvmppc_core_queue_program(vcpu, flags);
1333 r = RESUME_GUEST;
1334 break;
1335 }
1336 case BOOK3S_INTERRUPT_SYSCALL:
1337 {
1338
1339 int i;
1340
1341
1342
1343
1344
1345 run->papr_hcall.nr = kvmppc_get_gpr(vcpu, 3);
1346 for (i = 0; i < 9; ++i)
1347 run->papr_hcall.args[i] = kvmppc_get_gpr(vcpu, 4 + i);
1348 run->exit_reason = KVM_EXIT_PAPR_HCALL;
1349 vcpu->arch.hcall_needed = 1;
1350 r = RESUME_HOST;
1351 break;
1352 }
1353
1354
1355
1356
1357
1358
1359
1360 case BOOK3S_INTERRUPT_H_DATA_STORAGE:
1361 r = RESUME_PAGE_FAULT;
1362 break;
1363 case BOOK3S_INTERRUPT_H_INST_STORAGE:
1364 vcpu->arch.fault_dar = kvmppc_get_pc(vcpu);
1365 vcpu->arch.fault_dsisr = vcpu->arch.shregs.msr &
1366 DSISR_SRR1_MATCH_64S;
1367 if (vcpu->arch.shregs.msr & HSRR1_HISI_WRITE)
1368 vcpu->arch.fault_dsisr |= DSISR_ISSTORE;
1369 r = RESUME_PAGE_FAULT;
1370 break;
1371
1372
1373
1374
1375
1376
1377
1378 case BOOK3S_INTERRUPT_H_EMUL_ASSIST:
1379 if (vcpu->arch.emul_inst != KVM_INST_FETCH_FAILED)
1380 vcpu->arch.last_inst = kvmppc_need_byteswap(vcpu) ?
1381 swab32(vcpu->arch.emul_inst) :
1382 vcpu->arch.emul_inst;
1383 if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP) {
1384 r = kvmppc_emulate_debug_inst(run, vcpu);
1385 } else {
1386 kvmppc_core_queue_program(vcpu, SRR1_PROGILL);
1387 r = RESUME_GUEST;
1388 }
1389 break;
1390
1391
1392
1393
1394
1395
1396
1397 case BOOK3S_INTERRUPT_H_FAC_UNAVAIL:
1398 r = EMULATE_FAIL;
1399 if (((vcpu->arch.hfscr >> 56) == FSCR_MSGP_LG) &&
1400 cpu_has_feature(CPU_FTR_ARCH_300))
1401 r = kvmppc_emulate_doorbell_instr(vcpu);
1402 if (r == EMULATE_FAIL) {
1403 kvmppc_core_queue_program(vcpu, SRR1_PROGILL);
1404 r = RESUME_GUEST;
1405 }
1406 break;
1407
1408#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1409 case BOOK3S_INTERRUPT_HV_SOFTPATCH:
1410
1411
1412
1413
1414
1415
1416 r = kvmhv_p9_tm_emulation(vcpu);
1417 break;
1418#endif
1419
1420 case BOOK3S_INTERRUPT_HV_RM_HARD:
1421 r = RESUME_PASSTHROUGH;
1422 break;
1423 default:
1424 kvmppc_dump_regs(vcpu);
1425 printk(KERN_EMERG "trap=0x%x | pc=0x%lx | msr=0x%llx\n",
1426 vcpu->arch.trap, kvmppc_get_pc(vcpu),
1427 vcpu->arch.shregs.msr);
1428 run->hw.hardware_exit_reason = vcpu->arch.trap;
1429 r = RESUME_HOST;
1430 break;
1431 }
1432
1433 return r;
1434}
1435
1436static int kvmppc_handle_nested_exit(struct kvm_run *run, struct kvm_vcpu *vcpu)
1437{
1438 int r;
1439 int srcu_idx;
1440
1441 vcpu->stat.sum_exits++;
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451 if (vcpu->arch.shregs.msr & MSR_HV) {
1452 pr_emerg("KVM trap in HV mode while nested!\n");
1453 pr_emerg("trap=0x%x | pc=0x%lx | msr=0x%llx\n",
1454 vcpu->arch.trap, kvmppc_get_pc(vcpu),
1455 vcpu->arch.shregs.msr);
1456 kvmppc_dump_regs(vcpu);
1457 return RESUME_HOST;
1458 }
1459 switch (vcpu->arch.trap) {
1460
1461 case BOOK3S_INTERRUPT_HV_DECREMENTER:
1462 vcpu->stat.dec_exits++;
1463 r = RESUME_GUEST;
1464 break;
1465 case BOOK3S_INTERRUPT_EXTERNAL:
1466 vcpu->stat.ext_intr_exits++;
1467 r = RESUME_HOST;
1468 break;
1469 case BOOK3S_INTERRUPT_H_DOORBELL:
1470 case BOOK3S_INTERRUPT_H_VIRT:
1471 vcpu->stat.ext_intr_exits++;
1472 r = RESUME_GUEST;
1473 break;
1474
1475 case BOOK3S_INTERRUPT_HMI:
1476 case BOOK3S_INTERRUPT_PERFMON:
1477 case BOOK3S_INTERRUPT_SYSTEM_RESET:
1478 r = RESUME_GUEST;
1479 break;
1480 case BOOK3S_INTERRUPT_MACHINE_CHECK:
1481
1482 r = RESUME_HOST;
1483
1484 machine_check_print_event_info(&vcpu->arch.mce_evt, false, true);
1485 break;
1486
1487
1488
1489
1490
1491
1492 case BOOK3S_INTERRUPT_H_DATA_STORAGE:
1493 srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
1494 r = kvmhv_nested_page_fault(run, vcpu);
1495 srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
1496 break;
1497 case BOOK3S_INTERRUPT_H_INST_STORAGE:
1498 vcpu->arch.fault_dar = kvmppc_get_pc(vcpu);
1499 vcpu->arch.fault_dsisr = kvmppc_get_msr(vcpu) &
1500 DSISR_SRR1_MATCH_64S;
1501 if (vcpu->arch.shregs.msr & HSRR1_HISI_WRITE)
1502 vcpu->arch.fault_dsisr |= DSISR_ISSTORE;
1503 srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
1504 r = kvmhv_nested_page_fault(run, vcpu);
1505 srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
1506 break;
1507
1508#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1509 case BOOK3S_INTERRUPT_HV_SOFTPATCH:
1510
1511
1512
1513
1514
1515
1516 r = kvmhv_p9_tm_emulation(vcpu);
1517 break;
1518#endif
1519
1520 case BOOK3S_INTERRUPT_HV_RM_HARD:
1521 vcpu->arch.trap = 0;
1522 r = RESUME_GUEST;
1523 if (!xics_on_xive())
1524 kvmppc_xics_rm_complete(vcpu, 0);
1525 break;
1526 default:
1527 r = RESUME_HOST;
1528 break;
1529 }
1530
1531 return r;
1532}
1533
1534static int kvm_arch_vcpu_ioctl_get_sregs_hv(struct kvm_vcpu *vcpu,
1535 struct kvm_sregs *sregs)
1536{
1537 int i;
1538
1539 memset(sregs, 0, sizeof(struct kvm_sregs));
1540 sregs->pvr = vcpu->arch.pvr;
1541 for (i = 0; i < vcpu->arch.slb_max; i++) {
1542 sregs->u.s.ppc64.slb[i].slbe = vcpu->arch.slb[i].orige;
1543 sregs->u.s.ppc64.slb[i].slbv = vcpu->arch.slb[i].origv;
1544 }
1545
1546 return 0;
1547}
1548
1549static int kvm_arch_vcpu_ioctl_set_sregs_hv(struct kvm_vcpu *vcpu,
1550 struct kvm_sregs *sregs)
1551{
1552 int i, j;
1553
1554
1555 if (sregs->pvr != vcpu->arch.pvr)
1556 return -EINVAL;
1557
1558 j = 0;
1559 for (i = 0; i < vcpu->arch.slb_nr; i++) {
1560 if (sregs->u.s.ppc64.slb[i].slbe & SLB_ESID_V) {
1561 vcpu->arch.slb[j].orige = sregs->u.s.ppc64.slb[i].slbe;
1562 vcpu->arch.slb[j].origv = sregs->u.s.ppc64.slb[i].slbv;
1563 ++j;
1564 }
1565 }
1566 vcpu->arch.slb_max = j;
1567
1568 return 0;
1569}
1570
1571static void kvmppc_set_lpcr(struct kvm_vcpu *vcpu, u64 new_lpcr,
1572 bool preserve_top32)
1573{
1574 struct kvm *kvm = vcpu->kvm;
1575 struct kvmppc_vcore *vc = vcpu->arch.vcore;
1576 u64 mask;
1577
1578 spin_lock(&vc->lock);
1579
1580
1581
1582
1583 if ((new_lpcr & LPCR_ILE) != (vc->lpcr & LPCR_ILE)) {
1584 struct kvm_vcpu *vcpu;
1585 int i;
1586
1587 kvm_for_each_vcpu(i, vcpu, kvm) {
1588 if (vcpu->arch.vcore != vc)
1589 continue;
1590 if (new_lpcr & LPCR_ILE)
1591 vcpu->arch.intr_msr |= MSR_LE;
1592 else
1593 vcpu->arch.intr_msr &= ~MSR_LE;
1594 }
1595 }
1596
1597
1598
1599
1600
1601
1602 mask = LPCR_DPFD | LPCR_ILE | LPCR_TC;
1603 if (cpu_has_feature(CPU_FTR_ARCH_207S))
1604 mask |= LPCR_AIL;
1605
1606
1607
1608
1609 if (cpu_has_feature(CPU_FTR_ARCH_300))
1610 mask |= LPCR_LD;
1611
1612
1613 if (preserve_top32)
1614 mask &= 0xFFFFFFFF;
1615 vc->lpcr = (vc->lpcr & ~mask) | (new_lpcr & mask);
1616 spin_unlock(&vc->lock);
1617}
1618
1619static int kvmppc_get_one_reg_hv(struct kvm_vcpu *vcpu, u64 id,
1620 union kvmppc_one_reg *val)
1621{
1622 int r = 0;
1623 long int i;
1624
1625 switch (id) {
1626 case KVM_REG_PPC_DEBUG_INST:
1627 *val = get_reg_val(id, KVMPPC_INST_SW_BREAKPOINT);
1628 break;
1629 case KVM_REG_PPC_HIOR:
1630 *val = get_reg_val(id, 0);
1631 break;
1632 case KVM_REG_PPC_DABR:
1633 *val = get_reg_val(id, vcpu->arch.dabr);
1634 break;
1635 case KVM_REG_PPC_DABRX:
1636 *val = get_reg_val(id, vcpu->arch.dabrx);
1637 break;
1638 case KVM_REG_PPC_DSCR:
1639 *val = get_reg_val(id, vcpu->arch.dscr);
1640 break;
1641 case KVM_REG_PPC_PURR:
1642 *val = get_reg_val(id, vcpu->arch.purr);
1643 break;
1644 case KVM_REG_PPC_SPURR:
1645 *val = get_reg_val(id, vcpu->arch.spurr);
1646 break;
1647 case KVM_REG_PPC_AMR:
1648 *val = get_reg_val(id, vcpu->arch.amr);
1649 break;
1650 case KVM_REG_PPC_UAMOR:
1651 *val = get_reg_val(id, vcpu->arch.uamor);
1652 break;
1653 case KVM_REG_PPC_MMCR0 ... KVM_REG_PPC_MMCRS:
1654 i = id - KVM_REG_PPC_MMCR0;
1655 *val = get_reg_val(id, vcpu->arch.mmcr[i]);
1656 break;
1657 case KVM_REG_PPC_PMC1 ... KVM_REG_PPC_PMC8:
1658 i = id - KVM_REG_PPC_PMC1;
1659 *val = get_reg_val(id, vcpu->arch.pmc[i]);
1660 break;
1661 case KVM_REG_PPC_SPMC1 ... KVM_REG_PPC_SPMC2:
1662 i = id - KVM_REG_PPC_SPMC1;
1663 *val = get_reg_val(id, vcpu->arch.spmc[i]);
1664 break;
1665 case KVM_REG_PPC_SIAR:
1666 *val = get_reg_val(id, vcpu->arch.siar);
1667 break;
1668 case KVM_REG_PPC_SDAR:
1669 *val = get_reg_val(id, vcpu->arch.sdar);
1670 break;
1671 case KVM_REG_PPC_SIER:
1672 *val = get_reg_val(id, vcpu->arch.sier);
1673 break;
1674 case KVM_REG_PPC_IAMR:
1675 *val = get_reg_val(id, vcpu->arch.iamr);
1676 break;
1677 case KVM_REG_PPC_PSPB:
1678 *val = get_reg_val(id, vcpu->arch.pspb);
1679 break;
1680 case KVM_REG_PPC_DPDES:
1681 *val = get_reg_val(id, vcpu->arch.vcore->dpdes);
1682 break;
1683 case KVM_REG_PPC_VTB:
1684 *val = get_reg_val(id, vcpu->arch.vcore->vtb);
1685 break;
1686 case KVM_REG_PPC_DAWR:
1687 *val = get_reg_val(id, vcpu->arch.dawr);
1688 break;
1689 case KVM_REG_PPC_DAWRX:
1690 *val = get_reg_val(id, vcpu->arch.dawrx);
1691 break;
1692 case KVM_REG_PPC_CIABR:
1693 *val = get_reg_val(id, vcpu->arch.ciabr);
1694 break;
1695 case KVM_REG_PPC_CSIGR:
1696 *val = get_reg_val(id, vcpu->arch.csigr);
1697 break;
1698 case KVM_REG_PPC_TACR:
1699 *val = get_reg_val(id, vcpu->arch.tacr);
1700 break;
1701 case KVM_REG_PPC_TCSCR:
1702 *val = get_reg_val(id, vcpu->arch.tcscr);
1703 break;
1704 case KVM_REG_PPC_PID:
1705 *val = get_reg_val(id, vcpu->arch.pid);
1706 break;
1707 case KVM_REG_PPC_ACOP:
1708 *val = get_reg_val(id, vcpu->arch.acop);
1709 break;
1710 case KVM_REG_PPC_WORT:
1711 *val = get_reg_val(id, vcpu->arch.wort);
1712 break;
1713 case KVM_REG_PPC_TIDR:
1714 *val = get_reg_val(id, vcpu->arch.tid);
1715 break;
1716 case KVM_REG_PPC_PSSCR:
1717 *val = get_reg_val(id, vcpu->arch.psscr);
1718 break;
1719 case KVM_REG_PPC_VPA_ADDR:
1720 spin_lock(&vcpu->arch.vpa_update_lock);
1721 *val = get_reg_val(id, vcpu->arch.vpa.next_gpa);
1722 spin_unlock(&vcpu->arch.vpa_update_lock);
1723 break;
1724 case KVM_REG_PPC_VPA_SLB:
1725 spin_lock(&vcpu->arch.vpa_update_lock);
1726 val->vpaval.addr = vcpu->arch.slb_shadow.next_gpa;
1727 val->vpaval.length = vcpu->arch.slb_shadow.len;
1728 spin_unlock(&vcpu->arch.vpa_update_lock);
1729 break;
1730 case KVM_REG_PPC_VPA_DTL:
1731 spin_lock(&vcpu->arch.vpa_update_lock);
1732 val->vpaval.addr = vcpu->arch.dtl.next_gpa;
1733 val->vpaval.length = vcpu->arch.dtl.len;
1734 spin_unlock(&vcpu->arch.vpa_update_lock);
1735 break;
1736 case KVM_REG_PPC_TB_OFFSET:
1737 *val = get_reg_val(id, vcpu->arch.vcore->tb_offset);
1738 break;
1739 case KVM_REG_PPC_LPCR:
1740 case KVM_REG_PPC_LPCR_64:
1741 *val = get_reg_val(id, vcpu->arch.vcore->lpcr);
1742 break;
1743 case KVM_REG_PPC_PPR:
1744 *val = get_reg_val(id, vcpu->arch.ppr);
1745 break;
1746#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1747 case KVM_REG_PPC_TFHAR:
1748 *val = get_reg_val(id, vcpu->arch.tfhar);
1749 break;
1750 case KVM_REG_PPC_TFIAR:
1751 *val = get_reg_val(id, vcpu->arch.tfiar);
1752 break;
1753 case KVM_REG_PPC_TEXASR:
1754 *val = get_reg_val(id, vcpu->arch.texasr);
1755 break;
1756 case KVM_REG_PPC_TM_GPR0 ... KVM_REG_PPC_TM_GPR31:
1757 i = id - KVM_REG_PPC_TM_GPR0;
1758 *val = get_reg_val(id, vcpu->arch.gpr_tm[i]);
1759 break;
1760 case KVM_REG_PPC_TM_VSR0 ... KVM_REG_PPC_TM_VSR63:
1761 {
1762 int j;
1763 i = id - KVM_REG_PPC_TM_VSR0;
1764 if (i < 32)
1765 for (j = 0; j < TS_FPRWIDTH; j++)
1766 val->vsxval[j] = vcpu->arch.fp_tm.fpr[i][j];
1767 else {
1768 if (cpu_has_feature(CPU_FTR_ALTIVEC))
1769 val->vval = vcpu->arch.vr_tm.vr[i-32];
1770 else
1771 r = -ENXIO;
1772 }
1773 break;
1774 }
1775 case KVM_REG_PPC_TM_CR:
1776 *val = get_reg_val(id, vcpu->arch.cr_tm);
1777 break;
1778 case KVM_REG_PPC_TM_XER:
1779 *val = get_reg_val(id, vcpu->arch.xer_tm);
1780 break;
1781 case KVM_REG_PPC_TM_LR:
1782 *val = get_reg_val(id, vcpu->arch.lr_tm);
1783 break;
1784 case KVM_REG_PPC_TM_CTR:
1785 *val = get_reg_val(id, vcpu->arch.ctr_tm);
1786 break;
1787 case KVM_REG_PPC_TM_FPSCR:
1788 *val = get_reg_val(id, vcpu->arch.fp_tm.fpscr);
1789 break;
1790 case KVM_REG_PPC_TM_AMR:
1791 *val = get_reg_val(id, vcpu->arch.amr_tm);
1792 break;
1793 case KVM_REG_PPC_TM_PPR:
1794 *val = get_reg_val(id, vcpu->arch.ppr_tm);
1795 break;
1796 case KVM_REG_PPC_TM_VRSAVE:
1797 *val = get_reg_val(id, vcpu->arch.vrsave_tm);
1798 break;
1799 case KVM_REG_PPC_TM_VSCR:
1800 if (cpu_has_feature(CPU_FTR_ALTIVEC))
1801 *val = get_reg_val(id, vcpu->arch.vr_tm.vscr.u[3]);
1802 else
1803 r = -ENXIO;
1804 break;
1805 case KVM_REG_PPC_TM_DSCR:
1806 *val = get_reg_val(id, vcpu->arch.dscr_tm);
1807 break;
1808 case KVM_REG_PPC_TM_TAR:
1809 *val = get_reg_val(id, vcpu->arch.tar_tm);
1810 break;
1811#endif
1812 case KVM_REG_PPC_ARCH_COMPAT:
1813 *val = get_reg_val(id, vcpu->arch.vcore->arch_compat);
1814 break;
1815 case KVM_REG_PPC_DEC_EXPIRY:
1816 *val = get_reg_val(id, vcpu->arch.dec_expires +
1817 vcpu->arch.vcore->tb_offset);
1818 break;
1819 case KVM_REG_PPC_ONLINE:
1820 *val = get_reg_val(id, vcpu->arch.online);
1821 break;
1822 case KVM_REG_PPC_PTCR:
1823 *val = get_reg_val(id, vcpu->kvm->arch.l1_ptcr);
1824 break;
1825 default:
1826 r = -EINVAL;
1827 break;
1828 }
1829
1830 return r;
1831}
1832
1833static int kvmppc_set_one_reg_hv(struct kvm_vcpu *vcpu, u64 id,
1834 union kvmppc_one_reg *val)
1835{
1836 int r = 0;
1837 long int i;
1838 unsigned long addr, len;
1839
1840 switch (id) {
1841 case KVM_REG_PPC_HIOR:
1842
1843 if (set_reg_val(id, *val))
1844 r = -EINVAL;
1845 break;
1846 case KVM_REG_PPC_DABR:
1847 vcpu->arch.dabr = set_reg_val(id, *val);
1848 break;
1849 case KVM_REG_PPC_DABRX:
1850 vcpu->arch.dabrx = set_reg_val(id, *val) & ~DABRX_HYP;
1851 break;
1852 case KVM_REG_PPC_DSCR:
1853 vcpu->arch.dscr = set_reg_val(id, *val);
1854 break;
1855 case KVM_REG_PPC_PURR:
1856 vcpu->arch.purr = set_reg_val(id, *val);
1857 break;
1858 case KVM_REG_PPC_SPURR:
1859 vcpu->arch.spurr = set_reg_val(id, *val);
1860 break;
1861 case KVM_REG_PPC_AMR:
1862 vcpu->arch.amr = set_reg_val(id, *val);
1863 break;
1864 case KVM_REG_PPC_UAMOR:
1865 vcpu->arch.uamor = set_reg_val(id, *val);
1866 break;
1867 case KVM_REG_PPC_MMCR0 ... KVM_REG_PPC_MMCRS:
1868 i = id - KVM_REG_PPC_MMCR0;
1869 vcpu->arch.mmcr[i] = set_reg_val(id, *val);
1870 break;
1871 case KVM_REG_PPC_PMC1 ... KVM_REG_PPC_PMC8:
1872 i = id - KVM_REG_PPC_PMC1;
1873 vcpu->arch.pmc[i] = set_reg_val(id, *val);
1874 break;
1875 case KVM_REG_PPC_SPMC1 ... KVM_REG_PPC_SPMC2:
1876 i = id - KVM_REG_PPC_SPMC1;
1877 vcpu->arch.spmc[i] = set_reg_val(id, *val);
1878 break;
1879 case KVM_REG_PPC_SIAR:
1880 vcpu->arch.siar = set_reg_val(id, *val);
1881 break;
1882 case KVM_REG_PPC_SDAR:
1883 vcpu->arch.sdar = set_reg_val(id, *val);
1884 break;
1885 case KVM_REG_PPC_SIER:
1886 vcpu->arch.sier = set_reg_val(id, *val);
1887 break;
1888 case KVM_REG_PPC_IAMR:
1889 vcpu->arch.iamr = set_reg_val(id, *val);
1890 break;
1891 case KVM_REG_PPC_PSPB:
1892 vcpu->arch.pspb = set_reg_val(id, *val);
1893 break;
1894 case KVM_REG_PPC_DPDES:
1895 vcpu->arch.vcore->dpdes = set_reg_val(id, *val);
1896 break;
1897 case KVM_REG_PPC_VTB:
1898 vcpu->arch.vcore->vtb = set_reg_val(id, *val);
1899 break;
1900 case KVM_REG_PPC_DAWR:
1901 vcpu->arch.dawr = set_reg_val(id, *val);
1902 break;
1903 case KVM_REG_PPC_DAWRX:
1904 vcpu->arch.dawrx = set_reg_val(id, *val) & ~DAWRX_HYP;
1905 break;
1906 case KVM_REG_PPC_CIABR:
1907 vcpu->arch.ciabr = set_reg_val(id, *val);
1908
1909 if ((vcpu->arch.ciabr & CIABR_PRIV) == CIABR_PRIV_HYPER)
1910 vcpu->arch.ciabr &= ~CIABR_PRIV;
1911 break;
1912 case KVM_REG_PPC_CSIGR:
1913 vcpu->arch.csigr = set_reg_val(id, *val);
1914 break;
1915 case KVM_REG_PPC_TACR:
1916 vcpu->arch.tacr = set_reg_val(id, *val);
1917 break;
1918 case KVM_REG_PPC_TCSCR:
1919 vcpu->arch.tcscr = set_reg_val(id, *val);
1920 break;
1921 case KVM_REG_PPC_PID:
1922 vcpu->arch.pid = set_reg_val(id, *val);
1923 break;
1924 case KVM_REG_PPC_ACOP:
1925 vcpu->arch.acop = set_reg_val(id, *val);
1926 break;
1927 case KVM_REG_PPC_WORT:
1928 vcpu->arch.wort = set_reg_val(id, *val);
1929 break;
1930 case KVM_REG_PPC_TIDR:
1931 vcpu->arch.tid = set_reg_val(id, *val);
1932 break;
1933 case KVM_REG_PPC_PSSCR:
1934 vcpu->arch.psscr = set_reg_val(id, *val) & PSSCR_GUEST_VIS;
1935 break;
1936 case KVM_REG_PPC_VPA_ADDR:
1937 addr = set_reg_val(id, *val);
1938 r = -EINVAL;
1939 if (!addr && (vcpu->arch.slb_shadow.next_gpa ||
1940 vcpu->arch.dtl.next_gpa))
1941 break;
1942 r = set_vpa(vcpu, &vcpu->arch.vpa, addr, sizeof(struct lppaca));
1943 break;
1944 case KVM_REG_PPC_VPA_SLB:
1945 addr = val->vpaval.addr;
1946 len = val->vpaval.length;
1947 r = -EINVAL;
1948 if (addr && !vcpu->arch.vpa.next_gpa)
1949 break;
1950 r = set_vpa(vcpu, &vcpu->arch.slb_shadow, addr, len);
1951 break;
1952 case KVM_REG_PPC_VPA_DTL:
1953 addr = val->vpaval.addr;
1954 len = val->vpaval.length;
1955 r = -EINVAL;
1956 if (addr && (len < sizeof(struct dtl_entry) ||
1957 !vcpu->arch.vpa.next_gpa))
1958 break;
1959 len -= len % sizeof(struct dtl_entry);
1960 r = set_vpa(vcpu, &vcpu->arch.dtl, addr, len);
1961 break;
1962 case KVM_REG_PPC_TB_OFFSET:
1963
1964 vcpu->arch.vcore->tb_offset =
1965 ALIGN(set_reg_val(id, *val), 1UL << 24);
1966 break;
1967 case KVM_REG_PPC_LPCR:
1968 kvmppc_set_lpcr(vcpu, set_reg_val(id, *val), true);
1969 break;
1970 case KVM_REG_PPC_LPCR_64:
1971 kvmppc_set_lpcr(vcpu, set_reg_val(id, *val), false);
1972 break;
1973 case KVM_REG_PPC_PPR:
1974 vcpu->arch.ppr = set_reg_val(id, *val);
1975 break;
1976#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1977 case KVM_REG_PPC_TFHAR:
1978 vcpu->arch.tfhar = set_reg_val(id, *val);
1979 break;
1980 case KVM_REG_PPC_TFIAR:
1981 vcpu->arch.tfiar = set_reg_val(id, *val);
1982 break;
1983 case KVM_REG_PPC_TEXASR:
1984 vcpu->arch.texasr = set_reg_val(id, *val);
1985 break;
1986 case KVM_REG_PPC_TM_GPR0 ... KVM_REG_PPC_TM_GPR31:
1987 i = id - KVM_REG_PPC_TM_GPR0;
1988 vcpu->arch.gpr_tm[i] = set_reg_val(id, *val);
1989 break;
1990 case KVM_REG_PPC_TM_VSR0 ... KVM_REG_PPC_TM_VSR63:
1991 {
1992 int j;
1993 i = id - KVM_REG_PPC_TM_VSR0;
1994 if (i < 32)
1995 for (j = 0; j < TS_FPRWIDTH; j++)
1996 vcpu->arch.fp_tm.fpr[i][j] = val->vsxval[j];
1997 else
1998 if (cpu_has_feature(CPU_FTR_ALTIVEC))
1999 vcpu->arch.vr_tm.vr[i-32] = val->vval;
2000 else
2001 r = -ENXIO;
2002 break;
2003 }
2004 case KVM_REG_PPC_TM_CR:
2005 vcpu->arch.cr_tm = set_reg_val(id, *val);
2006 break;
2007 case KVM_REG_PPC_TM_XER:
2008 vcpu->arch.xer_tm = set_reg_val(id, *val);
2009 break;
2010 case KVM_REG_PPC_TM_LR:
2011 vcpu->arch.lr_tm = set_reg_val(id, *val);
2012 break;
2013 case KVM_REG_PPC_TM_CTR:
2014 vcpu->arch.ctr_tm = set_reg_val(id, *val);
2015 break;
2016 case KVM_REG_PPC_TM_FPSCR:
2017 vcpu->arch.fp_tm.fpscr = set_reg_val(id, *val);
2018 break;
2019 case KVM_REG_PPC_TM_AMR:
2020 vcpu->arch.amr_tm = set_reg_val(id, *val);
2021 break;
2022 case KVM_REG_PPC_TM_PPR:
2023 vcpu->arch.ppr_tm = set_reg_val(id, *val);
2024 break;
2025 case KVM_REG_PPC_TM_VRSAVE:
2026 vcpu->arch.vrsave_tm = set_reg_val(id, *val);
2027 break;
2028 case KVM_REG_PPC_TM_VSCR:
2029 if (cpu_has_feature(CPU_FTR_ALTIVEC))
2030 vcpu->arch.vr.vscr.u[3] = set_reg_val(id, *val);
2031 else
2032 r = - ENXIO;
2033 break;
2034 case KVM_REG_PPC_TM_DSCR:
2035 vcpu->arch.dscr_tm = set_reg_val(id, *val);
2036 break;
2037 case KVM_REG_PPC_TM_TAR:
2038 vcpu->arch.tar_tm = set_reg_val(id, *val);
2039 break;
2040#endif
2041 case KVM_REG_PPC_ARCH_COMPAT:
2042 r = kvmppc_set_arch_compat(vcpu, set_reg_val(id, *val));
2043 break;
2044 case KVM_REG_PPC_DEC_EXPIRY:
2045 vcpu->arch.dec_expires = set_reg_val(id, *val) -
2046 vcpu->arch.vcore->tb_offset;
2047 break;
2048 case KVM_REG_PPC_ONLINE:
2049 i = set_reg_val(id, *val);
2050 if (i && !vcpu->arch.online)
2051 atomic_inc(&vcpu->arch.vcore->online_count);
2052 else if (!i && vcpu->arch.online)
2053 atomic_dec(&vcpu->arch.vcore->online_count);
2054 vcpu->arch.online = i;
2055 break;
2056 case KVM_REG_PPC_PTCR:
2057 vcpu->kvm->arch.l1_ptcr = set_reg_val(id, *val);
2058 break;
2059 default:
2060 r = -EINVAL;
2061 break;
2062 }
2063
2064 return r;
2065}
2066
2067
2068
2069
2070
2071
2072
2073
2074static int threads_per_vcore(struct kvm *kvm)
2075{
2076 if (kvm->arch.threads_indep)
2077 return 1;
2078 return threads_per_subcore;
2079}
2080
2081static struct kvmppc_vcore *kvmppc_vcore_create(struct kvm *kvm, int id)
2082{
2083 struct kvmppc_vcore *vcore;
2084
2085 vcore = kzalloc(sizeof(struct kvmppc_vcore), GFP_KERNEL);
2086
2087 if (vcore == NULL)
2088 return NULL;
2089
2090 spin_lock_init(&vcore->lock);
2091 spin_lock_init(&vcore->stoltb_lock);
2092 init_swait_queue_head(&vcore->wq);
2093 vcore->preempt_tb = TB_NIL;
2094 vcore->lpcr = kvm->arch.lpcr;
2095 vcore->first_vcpuid = id;
2096 vcore->kvm = kvm;
2097 INIT_LIST_HEAD(&vcore->preempt_list);
2098
2099 return vcore;
2100}
2101
2102#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
2103static struct debugfs_timings_element {
2104 const char *name;
2105 size_t offset;
2106} timings[] = {
2107 {"rm_entry", offsetof(struct kvm_vcpu, arch.rm_entry)},
2108 {"rm_intr", offsetof(struct kvm_vcpu, arch.rm_intr)},
2109 {"rm_exit", offsetof(struct kvm_vcpu, arch.rm_exit)},
2110 {"guest", offsetof(struct kvm_vcpu, arch.guest_time)},
2111 {"cede", offsetof(struct kvm_vcpu, arch.cede_time)},
2112};
2113
2114#define N_TIMINGS (ARRAY_SIZE(timings))
2115
2116struct debugfs_timings_state {
2117 struct kvm_vcpu *vcpu;
2118 unsigned int buflen;
2119 char buf[N_TIMINGS * 100];
2120};
2121
2122static int debugfs_timings_open(struct inode *inode, struct file *file)
2123{
2124 struct kvm_vcpu *vcpu = inode->i_private;
2125 struct debugfs_timings_state *p;
2126
2127 p = kzalloc(sizeof(*p), GFP_KERNEL);
2128 if (!p)
2129 return -ENOMEM;
2130
2131 kvm_get_kvm(vcpu->kvm);
2132 p->vcpu = vcpu;
2133 file->private_data = p;
2134
2135 return nonseekable_open(inode, file);
2136}
2137
2138static int debugfs_timings_release(struct inode *inode, struct file *file)
2139{
2140 struct debugfs_timings_state *p = file->private_data;
2141
2142 kvm_put_kvm(p->vcpu->kvm);
2143 kfree(p);
2144 return 0;
2145}
2146
2147static ssize_t debugfs_timings_read(struct file *file, char __user *buf,
2148 size_t len, loff_t *ppos)
2149{
2150 struct debugfs_timings_state *p = file->private_data;
2151 struct kvm_vcpu *vcpu = p->vcpu;
2152 char *s, *buf_end;
2153 struct kvmhv_tb_accumulator tb;
2154 u64 count;
2155 loff_t pos;
2156 ssize_t n;
2157 int i, loops;
2158 bool ok;
2159
2160 if (!p->buflen) {
2161 s = p->buf;
2162 buf_end = s + sizeof(p->buf);
2163 for (i = 0; i < N_TIMINGS; ++i) {
2164 struct kvmhv_tb_accumulator *acc;
2165
2166 acc = (struct kvmhv_tb_accumulator *)
2167 ((unsigned long)vcpu + timings[i].offset);
2168 ok = false;
2169 for (loops = 0; loops < 1000; ++loops) {
2170 count = acc->seqcount;
2171 if (!(count & 1)) {
2172 smp_rmb();
2173 tb = *acc;
2174 smp_rmb();
2175 if (count == acc->seqcount) {
2176 ok = true;
2177 break;
2178 }
2179 }
2180 udelay(1);
2181 }
2182 if (!ok)
2183 snprintf(s, buf_end - s, "%s: stuck\n",
2184 timings[i].name);
2185 else
2186 snprintf(s, buf_end - s,
2187 "%s: %llu %llu %llu %llu\n",
2188 timings[i].name, count / 2,
2189 tb_to_ns(tb.tb_total),
2190 tb_to_ns(tb.tb_min),
2191 tb_to_ns(tb.tb_max));
2192 s += strlen(s);
2193 }
2194 p->buflen = s - p->buf;
2195 }
2196
2197 pos = *ppos;
2198 if (pos >= p->buflen)
2199 return 0;
2200 if (len > p->buflen - pos)
2201 len = p->buflen - pos;
2202 n = copy_to_user(buf, p->buf + pos, len);
2203 if (n) {
2204 if (n == len)
2205 return -EFAULT;
2206 len -= n;
2207 }
2208 *ppos = pos + len;
2209 return len;
2210}
2211
2212static ssize_t debugfs_timings_write(struct file *file, const char __user *buf,
2213 size_t len, loff_t *ppos)
2214{
2215 return -EACCES;
2216}
2217
2218static const struct file_operations debugfs_timings_ops = {
2219 .owner = THIS_MODULE,
2220 .open = debugfs_timings_open,
2221 .release = debugfs_timings_release,
2222 .read = debugfs_timings_read,
2223 .write = debugfs_timings_write,
2224 .llseek = generic_file_llseek,
2225};
2226
2227
2228static void debugfs_vcpu_init(struct kvm_vcpu *vcpu, unsigned int id)
2229{
2230 char buf[16];
2231 struct kvm *kvm = vcpu->kvm;
2232
2233 snprintf(buf, sizeof(buf), "vcpu%u", id);
2234 if (IS_ERR_OR_NULL(kvm->arch.debugfs_dir))
2235 return;
2236 vcpu->arch.debugfs_dir = debugfs_create_dir(buf, kvm->arch.debugfs_dir);
2237 if (IS_ERR_OR_NULL(vcpu->arch.debugfs_dir))
2238 return;
2239 vcpu->arch.debugfs_timings =
2240 debugfs_create_file("timings", 0444, vcpu->arch.debugfs_dir,
2241 vcpu, &debugfs_timings_ops);
2242}
2243
2244#else
2245static void debugfs_vcpu_init(struct kvm_vcpu *vcpu, unsigned int id)
2246{
2247}
2248#endif
2249
2250static struct kvm_vcpu *kvmppc_core_vcpu_create_hv(struct kvm *kvm,
2251 unsigned int id)
2252{
2253 struct kvm_vcpu *vcpu;
2254 int err;
2255 int core;
2256 struct kvmppc_vcore *vcore;
2257
2258 err = -ENOMEM;
2259 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
2260 if (!vcpu)
2261 goto out;
2262
2263 err = kvm_vcpu_init(vcpu, kvm, id);
2264 if (err)
2265 goto free_vcpu;
2266
2267 vcpu->arch.shared = &vcpu->arch.shregs;
2268#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
2269
2270
2271
2272
2273#ifdef __BIG_ENDIAN__
2274 vcpu->arch.shared_big_endian = true;
2275#else
2276 vcpu->arch.shared_big_endian = false;
2277#endif
2278#endif
2279 vcpu->arch.mmcr[0] = MMCR0_FC;
2280 vcpu->arch.ctrl = CTRL_RUNLATCH;
2281
2282 kvmppc_set_pvr_hv(vcpu, mfspr(SPRN_PVR));
2283 spin_lock_init(&vcpu->arch.vpa_update_lock);
2284 spin_lock_init(&vcpu->arch.tbacct_lock);
2285 vcpu->arch.busy_preempt = TB_NIL;
2286 vcpu->arch.intr_msr = MSR_SF | MSR_ME;
2287
2288
2289
2290
2291
2292
2293
2294
2295 vcpu->arch.hfscr = HFSCR_TAR | HFSCR_EBB | HFSCR_PM | HFSCR_BHRB |
2296 HFSCR_DSCR | HFSCR_VECVSX | HFSCR_FP;
2297 if (cpu_has_feature(CPU_FTR_HVMODE)) {
2298 vcpu->arch.hfscr &= mfspr(SPRN_HFSCR);
2299 if (cpu_has_feature(CPU_FTR_P9_TM_HV_ASSIST))
2300 vcpu->arch.hfscr |= HFSCR_TM;
2301 }
2302 if (cpu_has_feature(CPU_FTR_TM_COMP))
2303 vcpu->arch.hfscr |= HFSCR_TM;
2304
2305 kvmppc_mmu_book3s_hv_init(vcpu);
2306
2307 vcpu->arch.state = KVMPPC_VCPU_NOTREADY;
2308
2309 init_waitqueue_head(&vcpu->arch.cpu_run);
2310
2311 mutex_lock(&kvm->lock);
2312 vcore = NULL;
2313 err = -EINVAL;
2314 if (cpu_has_feature(CPU_FTR_ARCH_300)) {
2315 if (id >= (KVM_MAX_VCPUS * kvm->arch.emul_smt_mode)) {
2316 pr_devel("KVM: VCPU ID too high\n");
2317 core = KVM_MAX_VCORES;
2318 } else {
2319 BUG_ON(kvm->arch.smt_mode != 1);
2320 core = kvmppc_pack_vcpu_id(kvm, id);
2321 }
2322 } else {
2323 core = id / kvm->arch.smt_mode;
2324 }
2325 if (core < KVM_MAX_VCORES) {
2326 vcore = kvm->arch.vcores[core];
2327 if (vcore && cpu_has_feature(CPU_FTR_ARCH_300)) {
2328 pr_devel("KVM: collision on id %u", id);
2329 vcore = NULL;
2330 } else if (!vcore) {
2331
2332
2333
2334
2335 err = -ENOMEM;
2336 vcore = kvmppc_vcore_create(kvm,
2337 id & ~(kvm->arch.smt_mode - 1));
2338 mutex_lock(&kvm->arch.mmu_setup_lock);
2339 kvm->arch.vcores[core] = vcore;
2340 kvm->arch.online_vcores++;
2341 mutex_unlock(&kvm->arch.mmu_setup_lock);
2342 }
2343 }
2344 mutex_unlock(&kvm->lock);
2345
2346 if (!vcore)
2347 goto free_vcpu;
2348
2349 spin_lock(&vcore->lock);
2350 ++vcore->num_threads;
2351 spin_unlock(&vcore->lock);
2352 vcpu->arch.vcore = vcore;
2353 vcpu->arch.ptid = vcpu->vcpu_id - vcore->first_vcpuid;
2354 vcpu->arch.thread_cpu = -1;
2355 vcpu->arch.prev_cpu = -1;
2356
2357 vcpu->arch.cpu_type = KVM_CPU_3S_64;
2358 kvmppc_sanity_check(vcpu);
2359
2360 debugfs_vcpu_init(vcpu, id);
2361
2362 return vcpu;
2363
2364free_vcpu:
2365 kmem_cache_free(kvm_vcpu_cache, vcpu);
2366out:
2367 return ERR_PTR(err);
2368}
2369
2370static int kvmhv_set_smt_mode(struct kvm *kvm, unsigned long smt_mode,
2371 unsigned long flags)
2372{
2373 int err;
2374 int esmt = 0;
2375
2376 if (flags)
2377 return -EINVAL;
2378 if (smt_mode > MAX_SMT_THREADS || !is_power_of_2(smt_mode))
2379 return -EINVAL;
2380 if (!cpu_has_feature(CPU_FTR_ARCH_300)) {
2381
2382
2383
2384
2385 if (smt_mode > threads_per_subcore)
2386 return -EINVAL;
2387 } else {
2388
2389
2390
2391
2392 esmt = smt_mode;
2393 smt_mode = 1;
2394 }
2395 mutex_lock(&kvm->lock);
2396 err = -EBUSY;
2397 if (!kvm->arch.online_vcores) {
2398 kvm->arch.smt_mode = smt_mode;
2399 kvm->arch.emul_smt_mode = esmt;
2400 err = 0;
2401 }
2402 mutex_unlock(&kvm->lock);
2403
2404 return err;
2405}
2406
2407static void unpin_vpa(struct kvm *kvm, struct kvmppc_vpa *vpa)
2408{
2409 if (vpa->pinned_addr)
2410 kvmppc_unpin_guest_page(kvm, vpa->pinned_addr, vpa->gpa,
2411 vpa->dirty);
2412}
2413
2414static void kvmppc_core_vcpu_free_hv(struct kvm_vcpu *vcpu)
2415{
2416 spin_lock(&vcpu->arch.vpa_update_lock);
2417 unpin_vpa(vcpu->kvm, &vcpu->arch.dtl);
2418 unpin_vpa(vcpu->kvm, &vcpu->arch.slb_shadow);
2419 unpin_vpa(vcpu->kvm, &vcpu->arch.vpa);
2420 spin_unlock(&vcpu->arch.vpa_update_lock);
2421 kvm_vcpu_uninit(vcpu);
2422 kmem_cache_free(kvm_vcpu_cache, vcpu);
2423}
2424
2425static int kvmppc_core_check_requests_hv(struct kvm_vcpu *vcpu)
2426{
2427
2428 return 1;
2429}
2430
2431static void kvmppc_set_timer(struct kvm_vcpu *vcpu)
2432{
2433 unsigned long dec_nsec, now;
2434
2435 now = get_tb();
2436 if (now > vcpu->arch.dec_expires) {
2437
2438 kvmppc_core_queue_dec(vcpu);
2439 kvmppc_core_prepare_to_enter(vcpu);
2440 return;
2441 }
2442 dec_nsec = tb_to_ns(vcpu->arch.dec_expires - now);
2443 hrtimer_start(&vcpu->arch.dec_timer, dec_nsec, HRTIMER_MODE_REL);
2444 vcpu->arch.timer_running = 1;
2445}
2446
2447static void kvmppc_end_cede(struct kvm_vcpu *vcpu)
2448{
2449 vcpu->arch.ceded = 0;
2450 if (vcpu->arch.timer_running) {
2451 hrtimer_try_to_cancel(&vcpu->arch.dec_timer);
2452 vcpu->arch.timer_running = 0;
2453 }
2454}
2455
2456extern int __kvmppc_vcore_entry(void);
2457
2458static void kvmppc_remove_runnable(struct kvmppc_vcore *vc,
2459 struct kvm_vcpu *vcpu)
2460{
2461 u64 now;
2462
2463 if (vcpu->arch.state != KVMPPC_VCPU_RUNNABLE)
2464 return;
2465 spin_lock_irq(&vcpu->arch.tbacct_lock);
2466 now = mftb();
2467 vcpu->arch.busy_stolen += vcore_stolen_time(vc, now) -
2468 vcpu->arch.stolen_logged;
2469 vcpu->arch.busy_preempt = now;
2470 vcpu->arch.state = KVMPPC_VCPU_BUSY_IN_HOST;
2471 spin_unlock_irq(&vcpu->arch.tbacct_lock);
2472 --vc->n_runnable;
2473 WRITE_ONCE(vc->runnable_threads[vcpu->arch.ptid], NULL);
2474}
2475
2476static int kvmppc_grab_hwthread(int cpu)
2477{
2478 struct paca_struct *tpaca;
2479 long timeout = 10000;
2480
2481 tpaca = paca_ptrs[cpu];
2482
2483
2484 tpaca->kvm_hstate.kvm_vcpu = NULL;
2485 tpaca->kvm_hstate.kvm_vcore = NULL;
2486 tpaca->kvm_hstate.napping = 0;
2487 smp_wmb();
2488 tpaca->kvm_hstate.hwthread_req = 1;
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499 smp_mb();
2500 while (tpaca->kvm_hstate.hwthread_state == KVM_HWTHREAD_IN_KERNEL) {
2501 if (--timeout <= 0) {
2502 pr_err("KVM: couldn't grab cpu %d\n", cpu);
2503 return -EBUSY;
2504 }
2505 udelay(1);
2506 }
2507 return 0;
2508}
2509
2510static void kvmppc_release_hwthread(int cpu)
2511{
2512 struct paca_struct *tpaca;
2513
2514 tpaca = paca_ptrs[cpu];
2515 tpaca->kvm_hstate.hwthread_req = 0;
2516 tpaca->kvm_hstate.kvm_vcpu = NULL;
2517 tpaca->kvm_hstate.kvm_vcore = NULL;
2518 tpaca->kvm_hstate.kvm_split_mode = NULL;
2519}
2520
2521static void radix_flush_cpu(struct kvm *kvm, int cpu, struct kvm_vcpu *vcpu)
2522{
2523 struct kvm_nested_guest *nested = vcpu->arch.nested;
2524 cpumask_t *cpu_in_guest;
2525 int i;
2526
2527 cpu = cpu_first_thread_sibling(cpu);
2528 if (nested) {
2529 cpumask_set_cpu(cpu, &nested->need_tlb_flush);
2530 cpu_in_guest = &nested->cpu_in_guest;
2531 } else {
2532 cpumask_set_cpu(cpu, &kvm->arch.need_tlb_flush);
2533 cpu_in_guest = &kvm->arch.cpu_in_guest;
2534 }
2535
2536
2537
2538
2539
2540 smp_mb();
2541 for (i = 0; i < threads_per_core; ++i)
2542 if (cpumask_test_cpu(cpu + i, cpu_in_guest))
2543 smp_call_function_single(cpu + i, do_nothing, NULL, 1);
2544}
2545
2546static void kvmppc_prepare_radix_vcpu(struct kvm_vcpu *vcpu, int pcpu)
2547{
2548 struct kvm_nested_guest *nested = vcpu->arch.nested;
2549 struct kvm *kvm = vcpu->kvm;
2550 int prev_cpu;
2551
2552 if (!cpu_has_feature(CPU_FTR_HVMODE))
2553 return;
2554
2555 if (nested)
2556 prev_cpu = nested->prev_cpu[vcpu->arch.nested_vcpu_id];
2557 else
2558 prev_cpu = vcpu->arch.prev_cpu;
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572 if (prev_cpu != pcpu) {
2573 if (prev_cpu >= 0 &&
2574 cpu_first_thread_sibling(prev_cpu) !=
2575 cpu_first_thread_sibling(pcpu))
2576 radix_flush_cpu(kvm, prev_cpu, vcpu);
2577 if (nested)
2578 nested->prev_cpu[vcpu->arch.nested_vcpu_id] = pcpu;
2579 else
2580 vcpu->arch.prev_cpu = pcpu;
2581 }
2582}
2583
2584static void kvmppc_start_thread(struct kvm_vcpu *vcpu, struct kvmppc_vcore *vc)
2585{
2586 int cpu;
2587 struct paca_struct *tpaca;
2588 struct kvm *kvm = vc->kvm;
2589
2590 cpu = vc->pcpu;
2591 if (vcpu) {
2592 if (vcpu->arch.timer_running) {
2593 hrtimer_try_to_cancel(&vcpu->arch.dec_timer);
2594 vcpu->arch.timer_running = 0;
2595 }
2596 cpu += vcpu->arch.ptid;
2597 vcpu->cpu = vc->pcpu;
2598 vcpu->arch.thread_cpu = cpu;
2599 cpumask_set_cpu(cpu, &kvm->arch.cpu_in_guest);
2600 }
2601 tpaca = paca_ptrs[cpu];
2602 tpaca->kvm_hstate.kvm_vcpu = vcpu;
2603 tpaca->kvm_hstate.ptid = cpu - vc->pcpu;
2604 tpaca->kvm_hstate.fake_suspend = 0;
2605
2606 smp_wmb();
2607 tpaca->kvm_hstate.kvm_vcore = vc;
2608 if (cpu != smp_processor_id())
2609 kvmppc_ipi_thread(cpu);
2610}
2611
2612static void kvmppc_wait_for_nap(int n_threads)
2613{
2614 int cpu = smp_processor_id();
2615 int i, loops;
2616
2617 if (n_threads <= 1)
2618 return;
2619 for (loops = 0; loops < 1000000; ++loops) {
2620
2621
2622
2623
2624
2625
2626 for (i = 1; i < n_threads; ++i)
2627 if (paca_ptrs[cpu + i]->kvm_hstate.kvm_vcore)
2628 break;
2629 if (i == n_threads) {
2630 HMT_medium();
2631 return;
2632 }
2633 HMT_low();
2634 }
2635 HMT_medium();
2636 for (i = 1; i < n_threads; ++i)
2637 if (paca_ptrs[cpu + i]->kvm_hstate.kvm_vcore)
2638 pr_err("KVM: CPU %d seems to be stuck\n", cpu + i);
2639}
2640
2641
2642
2643
2644
2645
2646static int on_primary_thread(void)
2647{
2648 int cpu = smp_processor_id();
2649 int thr;
2650
2651
2652 if (cpu_thread_in_subcore(cpu))
2653 return 0;
2654
2655 thr = 0;
2656 while (++thr < threads_per_subcore)
2657 if (cpu_online(cpu + thr))
2658 return 0;
2659
2660
2661 for (thr = 1; thr < threads_per_subcore; ++thr) {
2662 if (kvmppc_grab_hwthread(cpu + thr)) {
2663
2664 do {
2665 kvmppc_release_hwthread(cpu + thr);
2666 } while (--thr > 0);
2667 return 0;
2668 }
2669 }
2670 return 1;
2671}
2672
2673
2674
2675
2676
2677
2678struct preempted_vcore_list {
2679 struct list_head list;
2680 spinlock_t lock;
2681};
2682
2683static DEFINE_PER_CPU(struct preempted_vcore_list, preempted_vcores);
2684
2685static void init_vcore_lists(void)
2686{
2687 int cpu;
2688
2689 for_each_possible_cpu(cpu) {
2690 struct preempted_vcore_list *lp = &per_cpu(preempted_vcores, cpu);
2691 spin_lock_init(&lp->lock);
2692 INIT_LIST_HEAD(&lp->list);
2693 }
2694}
2695
2696static void kvmppc_vcore_preempt(struct kvmppc_vcore *vc)
2697{
2698 struct preempted_vcore_list *lp = this_cpu_ptr(&preempted_vcores);
2699
2700 vc->vcore_state = VCORE_PREEMPT;
2701 vc->pcpu = smp_processor_id();
2702 if (vc->num_threads < threads_per_vcore(vc->kvm)) {
2703 spin_lock(&lp->lock);
2704 list_add_tail(&vc->preempt_list, &lp->list);
2705 spin_unlock(&lp->lock);
2706 }
2707
2708
2709 kvmppc_core_start_stolen(vc);
2710}
2711
2712static void kvmppc_vcore_end_preempt(struct kvmppc_vcore *vc)
2713{
2714 struct preempted_vcore_list *lp;
2715
2716 kvmppc_core_end_stolen(vc);
2717 if (!list_empty(&vc->preempt_list)) {
2718 lp = &per_cpu(preempted_vcores, vc->pcpu);
2719 spin_lock(&lp->lock);
2720 list_del_init(&vc->preempt_list);
2721 spin_unlock(&lp->lock);
2722 }
2723 vc->vcore_state = VCORE_INACTIVE;
2724}
2725
2726
2727
2728
2729
2730struct core_info {
2731 int n_subcores;
2732 int max_subcore_threads;
2733 int total_threads;
2734 int subcore_threads[MAX_SUBCORES];
2735 struct kvmppc_vcore *vc[MAX_SUBCORES];
2736};
2737
2738
2739
2740
2741
2742static int subcore_thread_map[MAX_SUBCORES] = { 0, 4, 2, 6 };
2743
2744static void init_core_info(struct core_info *cip, struct kvmppc_vcore *vc)
2745{
2746 memset(cip, 0, sizeof(*cip));
2747 cip->n_subcores = 1;
2748 cip->max_subcore_threads = vc->num_threads;
2749 cip->total_threads = vc->num_threads;
2750 cip->subcore_threads[0] = vc->num_threads;
2751 cip->vc[0] = vc;
2752}
2753
2754static bool subcore_config_ok(int n_subcores, int n_threads)
2755{
2756
2757
2758
2759
2760 if (cpu_has_feature(CPU_FTR_ARCH_300))
2761 return n_subcores <= 4 && n_threads == 1;
2762
2763
2764 if (n_subcores > 1 && threads_per_subcore < MAX_SMT_THREADS)
2765 return false;
2766 if (n_subcores > MAX_SUBCORES)
2767 return false;
2768 if (n_subcores > 1) {
2769 if (!(dynamic_mt_modes & 2))
2770 n_subcores = 4;
2771 if (n_subcores > 2 && !(dynamic_mt_modes & 4))
2772 return false;
2773 }
2774
2775 return n_subcores * roundup_pow_of_two(n_threads) <= MAX_SMT_THREADS;
2776}
2777
2778static void init_vcore_to_run(struct kvmppc_vcore *vc)
2779{
2780 vc->entry_exit_map = 0;
2781 vc->in_guest = 0;
2782 vc->napping_threads = 0;
2783 vc->conferring_threads = 0;
2784 vc->tb_offset_applied = 0;
2785}
2786
2787static bool can_dynamic_split(struct kvmppc_vcore *vc, struct core_info *cip)
2788{
2789 int n_threads = vc->num_threads;
2790 int sub;
2791
2792 if (!cpu_has_feature(CPU_FTR_ARCH_207S))
2793 return false;
2794
2795
2796 if (one_vm_per_core && vc->kvm != cip->vc[0]->kvm)
2797 return false;
2798
2799
2800 if (no_mixing_hpt_and_radix &&
2801 kvm_is_radix(vc->kvm) != kvm_is_radix(cip->vc[0]->kvm))
2802 return false;
2803
2804 if (n_threads < cip->max_subcore_threads)
2805 n_threads = cip->max_subcore_threads;
2806 if (!subcore_config_ok(cip->n_subcores + 1, n_threads))
2807 return false;
2808 cip->max_subcore_threads = n_threads;
2809
2810 sub = cip->n_subcores;
2811 ++cip->n_subcores;
2812 cip->total_threads += vc->num_threads;
2813 cip->subcore_threads[sub] = vc->num_threads;
2814 cip->vc[sub] = vc;
2815 init_vcore_to_run(vc);
2816 list_del_init(&vc->preempt_list);
2817
2818 return true;
2819}
2820
2821
2822
2823
2824
2825static bool can_piggyback(struct kvmppc_vcore *pvc, struct core_info *cip,
2826 int target_threads)
2827{
2828 if (cip->total_threads + pvc->num_threads > target_threads)
2829 return false;
2830
2831 return can_dynamic_split(pvc, cip);
2832}
2833
2834static void prepare_threads(struct kvmppc_vcore *vc)
2835{
2836 int i;
2837 struct kvm_vcpu *vcpu;
2838
2839 for_each_runnable_thread(i, vcpu, vc) {
2840 if (signal_pending(vcpu->arch.run_task))
2841 vcpu->arch.ret = -EINTR;
2842 else if (vcpu->arch.vpa.update_pending ||
2843 vcpu->arch.slb_shadow.update_pending ||
2844 vcpu->arch.dtl.update_pending)
2845 vcpu->arch.ret = RESUME_GUEST;
2846 else
2847 continue;
2848 kvmppc_remove_runnable(vc, vcpu);
2849 wake_up(&vcpu->arch.cpu_run);
2850 }
2851}
2852
2853static void collect_piggybacks(struct core_info *cip, int target_threads)
2854{
2855 struct preempted_vcore_list *lp = this_cpu_ptr(&preempted_vcores);
2856 struct kvmppc_vcore *pvc, *vcnext;
2857
2858 spin_lock(&lp->lock);
2859 list_for_each_entry_safe(pvc, vcnext, &lp->list, preempt_list) {
2860 if (!spin_trylock(&pvc->lock))
2861 continue;
2862 prepare_threads(pvc);
2863 if (!pvc->n_runnable) {
2864 list_del_init(&pvc->preempt_list);
2865 if (pvc->runner == NULL) {
2866 pvc->vcore_state = VCORE_INACTIVE;
2867 kvmppc_core_end_stolen(pvc);
2868 }
2869 spin_unlock(&pvc->lock);
2870 continue;
2871 }
2872 if (!can_piggyback(pvc, cip, target_threads)) {
2873 spin_unlock(&pvc->lock);
2874 continue;
2875 }
2876 kvmppc_core_end_stolen(pvc);
2877 pvc->vcore_state = VCORE_PIGGYBACK;
2878 if (cip->total_threads >= target_threads)
2879 break;
2880 }
2881 spin_unlock(&lp->lock);
2882}
2883
2884static bool recheck_signals(struct core_info *cip)
2885{
2886 int sub, i;
2887 struct kvm_vcpu *vcpu;
2888
2889 for (sub = 0; sub < cip->n_subcores; ++sub)
2890 for_each_runnable_thread(i, vcpu, cip->vc[sub])
2891 if (signal_pending(vcpu->arch.run_task))
2892 return true;
2893 return false;
2894}
2895
2896static void post_guest_process(struct kvmppc_vcore *vc, bool is_master)
2897{
2898 int still_running = 0, i;
2899 u64 now;
2900 long ret;
2901 struct kvm_vcpu *vcpu;
2902
2903 spin_lock(&vc->lock);
2904 now = get_tb();
2905 for_each_runnable_thread(i, vcpu, vc) {
2906
2907
2908
2909
2910
2911
2912
2913 spin_unlock(&vc->lock);
2914
2915 if (now < vcpu->arch.dec_expires &&
2916 kvmppc_core_pending_dec(vcpu))
2917 kvmppc_core_dequeue_dec(vcpu);
2918
2919 trace_kvm_guest_exit(vcpu);
2920
2921 ret = RESUME_GUEST;
2922 if (vcpu->arch.trap)
2923 ret = kvmppc_handle_exit_hv(vcpu->arch.kvm_run, vcpu,
2924 vcpu->arch.run_task);
2925
2926 vcpu->arch.ret = ret;
2927 vcpu->arch.trap = 0;
2928
2929 spin_lock(&vc->lock);
2930 if (is_kvmppc_resume_guest(vcpu->arch.ret)) {
2931 if (vcpu->arch.pending_exceptions)
2932 kvmppc_core_prepare_to_enter(vcpu);
2933 if (vcpu->arch.ceded)
2934 kvmppc_set_timer(vcpu);
2935 else
2936 ++still_running;
2937 } else {
2938 kvmppc_remove_runnable(vc, vcpu);
2939 wake_up(&vcpu->arch.cpu_run);
2940 }
2941 }
2942 if (!is_master) {
2943 if (still_running > 0) {
2944 kvmppc_vcore_preempt(vc);
2945 } else if (vc->runner) {
2946 vc->vcore_state = VCORE_PREEMPT;
2947 kvmppc_core_start_stolen(vc);
2948 } else {
2949 vc->vcore_state = VCORE_INACTIVE;
2950 }
2951 if (vc->n_runnable > 0 && vc->runner == NULL) {
2952
2953 i = -1;
2954 vcpu = next_runnable_thread(vc, &i);
2955 wake_up(&vcpu->arch.cpu_run);
2956 }
2957 }
2958 spin_unlock(&vc->lock);
2959}
2960
2961
2962
2963
2964
2965
2966static inline int kvmppc_clear_host_core(unsigned int cpu)
2967{
2968 int core;
2969
2970 if (!kvmppc_host_rm_ops_hv || cpu_thread_in_core(cpu))
2971 return 0;
2972
2973
2974
2975
2976
2977 core = cpu >> threads_shift;
2978 kvmppc_host_rm_ops_hv->rm_core[core].rm_state.in_host = 0;
2979 return 0;
2980}
2981
2982
2983
2984
2985
2986
2987static inline int kvmppc_set_host_core(unsigned int cpu)
2988{
2989 int core;
2990
2991 if (!kvmppc_host_rm_ops_hv || cpu_thread_in_core(cpu))
2992 return 0;
2993
2994
2995
2996
2997
2998 core = cpu >> threads_shift;
2999 kvmppc_host_rm_ops_hv->rm_core[core].rm_state.in_host = 1;
3000 return 0;
3001}
3002
3003static void set_irq_happened(int trap)
3004{
3005 switch (trap) {
3006 case BOOK3S_INTERRUPT_EXTERNAL:
3007 local_paca->irq_happened |= PACA_IRQ_EE;
3008 break;
3009 case BOOK3S_INTERRUPT_H_DOORBELL:
3010 local_paca->irq_happened |= PACA_IRQ_DBELL;
3011 break;
3012 case BOOK3S_INTERRUPT_HMI:
3013 local_paca->irq_happened |= PACA_IRQ_HMI;
3014 break;
3015 case BOOK3S_INTERRUPT_SYSTEM_RESET:
3016 replay_system_reset();
3017 break;
3018 }
3019}
3020
3021
3022
3023
3024
3025static noinline void kvmppc_run_core(struct kvmppc_vcore *vc)
3026{
3027 struct kvm_vcpu *vcpu;
3028 int i;
3029 int srcu_idx;
3030 struct core_info core_info;
3031 struct kvmppc_vcore *pvc;
3032 struct kvm_split_mode split_info, *sip;
3033 int split, subcore_size, active;
3034 int sub;
3035 bool thr0_done;
3036 unsigned long cmd_bit, stat_bit;
3037 int pcpu, thr;
3038 int target_threads;
3039 int controlled_threads;
3040 int trap;
3041 bool is_power8;
3042 bool hpt_on_radix;
3043
3044
3045
3046
3047
3048 prepare_threads(vc);
3049
3050
3051 if (vc->runner->arch.state != KVMPPC_VCPU_RUNNABLE)
3052 return;
3053
3054
3055
3056
3057 init_vcore_to_run(vc);
3058 vc->preempt_tb = TB_NIL;
3059
3060
3061
3062
3063
3064
3065 controlled_threads = threads_per_vcore(vc->kvm);
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075 hpt_on_radix = no_mixing_hpt_and_radix && radix_enabled() &&
3076 !kvm_is_radix(vc->kvm);
3077 if (((controlled_threads > 1) &&
3078 ((vc->num_threads > threads_per_subcore) || !on_primary_thread())) ||
3079 (hpt_on_radix && vc->kvm->arch.threads_indep)) {
3080 for_each_runnable_thread(i, vcpu, vc) {
3081 vcpu->arch.ret = -EBUSY;
3082 kvmppc_remove_runnable(vc, vcpu);
3083 wake_up(&vcpu->arch.cpu_run);
3084 }
3085 goto out;
3086 }
3087
3088
3089
3090
3091
3092 init_core_info(&core_info, vc);
3093 pcpu = smp_processor_id();
3094 target_threads = controlled_threads;
3095 if (target_smt_mode && target_smt_mode < target_threads)
3096 target_threads = target_smt_mode;
3097 if (vc->num_threads < target_threads)
3098 collect_piggybacks(&core_info, target_threads);
3099
3100
3101
3102
3103
3104
3105 pcpu = smp_processor_id();
3106 if (kvm_is_radix(vc->kvm)) {
3107 for (sub = 0; sub < core_info.n_subcores; ++sub)
3108 for_each_runnable_thread(i, vcpu, core_info.vc[sub])
3109 kvmppc_prepare_radix_vcpu(vcpu, pcpu);
3110 }
3111
3112
3113
3114
3115
3116
3117
3118
3119 local_irq_disable();
3120 hard_irq_disable();
3121 if (lazy_irq_pending() || need_resched() ||
3122 recheck_signals(&core_info) || !vc->kvm->arch.mmu_ready) {
3123 local_irq_enable();
3124 vc->vcore_state = VCORE_INACTIVE;
3125
3126 for (sub = 1; sub < core_info.n_subcores; ++sub) {
3127 pvc = core_info.vc[sub];
3128
3129 kvmppc_vcore_preempt(pvc);
3130 spin_unlock(&pvc->lock);
3131 }
3132 for (i = 0; i < controlled_threads; ++i)
3133 kvmppc_release_hwthread(pcpu + i);
3134 return;
3135 }
3136
3137 kvmppc_clear_host_core(pcpu);
3138
3139
3140 subcore_size = threads_per_subcore;
3141 cmd_bit = stat_bit = 0;
3142 split = core_info.n_subcores;
3143 sip = NULL;
3144 is_power8 = cpu_has_feature(CPU_FTR_ARCH_207S)
3145 && !cpu_has_feature(CPU_FTR_ARCH_300);
3146
3147 if (split > 1 || hpt_on_radix) {
3148 sip = &split_info;
3149 memset(&split_info, 0, sizeof(split_info));
3150 for (sub = 0; sub < core_info.n_subcores; ++sub)
3151 split_info.vc[sub] = core_info.vc[sub];
3152
3153 if (is_power8) {
3154 if (split == 2 && (dynamic_mt_modes & 2)) {
3155 cmd_bit = HID0_POWER8_1TO2LPAR;
3156 stat_bit = HID0_POWER8_2LPARMODE;
3157 } else {
3158 split = 4;
3159 cmd_bit = HID0_POWER8_1TO4LPAR;
3160 stat_bit = HID0_POWER8_4LPARMODE;
3161 }
3162 subcore_size = MAX_SMT_THREADS / split;
3163 split_info.rpr = mfspr(SPRN_RPR);
3164 split_info.pmmar = mfspr(SPRN_PMMAR);
3165 split_info.ldbar = mfspr(SPRN_LDBAR);
3166 split_info.subcore_size = subcore_size;
3167 } else {
3168 split_info.subcore_size = 1;
3169 if (hpt_on_radix) {
3170
3171 split_info.lpcr_req = vc->lpcr;
3172 split_info.lpidr_req = vc->kvm->arch.lpid;
3173 split_info.host_lpcr = vc->kvm->arch.host_lpcr;
3174 split_info.do_set = 1;
3175 }
3176 }
3177
3178
3179 smp_wmb();
3180 }
3181
3182 for (thr = 0; thr < controlled_threads; ++thr) {
3183 struct paca_struct *paca = paca_ptrs[pcpu + thr];
3184
3185 paca->kvm_hstate.tid = thr;
3186 paca->kvm_hstate.napping = 0;
3187 paca->kvm_hstate.kvm_split_mode = sip;
3188 }
3189
3190
3191 if (cmd_bit) {
3192 unsigned long hid0 = mfspr(SPRN_HID0);
3193
3194 hid0 |= cmd_bit | HID0_POWER8_DYNLPARDIS;
3195 mb();
3196 mtspr(SPRN_HID0, hid0);
3197 isync();
3198 for (;;) {
3199 hid0 = mfspr(SPRN_HID0);
3200 if (hid0 & stat_bit)
3201 break;
3202 cpu_relax();
3203 }
3204 }
3205
3206
3207
3208
3209
3210
3211 if (is_power8) {
3212 unsigned long rwmr_val = RWMR_RPA_P8_8THREAD;
3213 int n_online = atomic_read(&vc->online_count);
3214
3215
3216
3217
3218
3219 if (split == 1 && threads_per_subcore == MAX_SMT_THREADS &&
3220 n_online >= 1 && n_online <= MAX_SMT_THREADS)
3221 rwmr_val = p8_rwmr_values[n_online];
3222 mtspr(SPRN_RWMR, rwmr_val);
3223 }
3224
3225
3226 active = 0;
3227 for (sub = 0; sub < core_info.n_subcores; ++sub) {
3228 thr = is_power8 ? subcore_thread_map[sub] : sub;
3229 thr0_done = false;
3230 active |= 1 << thr;
3231 pvc = core_info.vc[sub];
3232 pvc->pcpu = pcpu + thr;
3233 for_each_runnable_thread(i, vcpu, pvc) {
3234 kvmppc_start_thread(vcpu, pvc);
3235 kvmppc_create_dtl_entry(vcpu, pvc);
3236 trace_kvm_guest_enter(vcpu);
3237 if (!vcpu->arch.ptid)
3238 thr0_done = true;
3239 active |= 1 << (thr + vcpu->arch.ptid);
3240 }
3241
3242
3243
3244
3245 if (!thr0_done)
3246 kvmppc_start_thread(NULL, pvc);
3247 }
3248
3249
3250
3251
3252
3253 smp_mb();
3254
3255
3256
3257
3258
3259
3260
3261
3262 if (cmd_bit || hpt_on_radix) {
3263 split_info.do_nap = 1;
3264 for (thr = 1; thr < threads_per_subcore; ++thr)
3265 if (!(active & (1 << thr)))
3266 kvmppc_ipi_thread(pcpu + thr);
3267 }
3268
3269 vc->vcore_state = VCORE_RUNNING;
3270 preempt_disable();
3271
3272 trace_kvmppc_run_core(vc, 0);
3273
3274 for (sub = 0; sub < core_info.n_subcores; ++sub)
3275 spin_unlock(&core_info.vc[sub]->lock);
3276
3277 guest_enter_irqoff();
3278
3279 srcu_idx = srcu_read_lock(&vc->kvm->srcu);
3280
3281 this_cpu_disable_ftrace();
3282
3283
3284
3285
3286
3287 trace_hardirqs_on();
3288
3289 trap = __kvmppc_vcore_entry();
3290
3291 trace_hardirqs_off();
3292
3293 this_cpu_enable_ftrace();
3294
3295 srcu_read_unlock(&vc->kvm->srcu, srcu_idx);
3296
3297 set_irq_happened(trap);
3298
3299 spin_lock(&vc->lock);
3300
3301 vc->vcore_state = VCORE_EXITING;
3302
3303
3304 kvmppc_wait_for_nap(controlled_threads);
3305
3306
3307 if (cmd_bit) {
3308 unsigned long hid0 = mfspr(SPRN_HID0);
3309 unsigned long loops = 0;
3310
3311 hid0 &= ~HID0_POWER8_DYNLPARDIS;
3312 stat_bit = HID0_POWER8_2LPARMODE | HID0_POWER8_4LPARMODE;
3313 mb();
3314 mtspr(SPRN_HID0, hid0);
3315 isync();
3316 for (;;) {
3317 hid0 = mfspr(SPRN_HID0);
3318 if (!(hid0 & stat_bit))
3319 break;
3320 cpu_relax();
3321 ++loops;
3322 }
3323 } else if (hpt_on_radix) {
3324
3325 for (thr = 1; thr < controlled_threads; ++thr) {
3326 struct paca_struct *paca = paca_ptrs[pcpu + thr];
3327
3328 while (paca->kvm_hstate.kvm_split_mode) {
3329 HMT_low();
3330 barrier();
3331 }
3332 HMT_medium();
3333 }
3334 }
3335 split_info.do_nap = 0;
3336
3337 kvmppc_set_host_core(pcpu);
3338
3339 local_irq_enable();
3340 guest_exit();
3341
3342
3343 for (i = 0; i < controlled_threads; ++i) {
3344 kvmppc_release_hwthread(pcpu + i);
3345 if (sip && sip->napped[i])
3346 kvmppc_ipi_thread(pcpu + i);
3347 cpumask_clear_cpu(pcpu + i, &vc->kvm->arch.cpu_in_guest);
3348 }
3349
3350 spin_unlock(&vc->lock);
3351
3352
3353 smp_mb();
3354
3355 preempt_enable();
3356
3357 for (sub = 0; sub < core_info.n_subcores; ++sub) {
3358 pvc = core_info.vc[sub];
3359 post_guest_process(pvc, pvc == vc);
3360 }
3361
3362 spin_lock(&vc->lock);
3363
3364 out:
3365 vc->vcore_state = VCORE_INACTIVE;
3366 trace_kvmppc_run_core(vc, 1);
3367}
3368
3369
3370
3371
3372static int kvmhv_load_hv_regs_and_go(struct kvm_vcpu *vcpu, u64 time_limit,
3373 unsigned long lpcr)
3374{
3375 struct kvmppc_vcore *vc = vcpu->arch.vcore;
3376 s64 hdec;
3377 u64 tb, purr, spurr;
3378 int trap;
3379 unsigned long host_hfscr = mfspr(SPRN_HFSCR);
3380 unsigned long host_ciabr = mfspr(SPRN_CIABR);
3381 unsigned long host_dawr = mfspr(SPRN_DAWR);
3382 unsigned long host_dawrx = mfspr(SPRN_DAWRX);
3383 unsigned long host_psscr = mfspr(SPRN_PSSCR);
3384 unsigned long host_pidr = mfspr(SPRN_PID);
3385
3386 hdec = time_limit - mftb();
3387 if (hdec < 0)
3388 return BOOK3S_INTERRUPT_HV_DECREMENTER;
3389 mtspr(SPRN_HDEC, hdec);
3390
3391 if (vc->tb_offset) {
3392 u64 new_tb = mftb() + vc->tb_offset;
3393 mtspr(SPRN_TBU40, new_tb);
3394 tb = mftb();
3395 if ((tb & 0xffffff) < (new_tb & 0xffffff))
3396 mtspr(SPRN_TBU40, new_tb + 0x1000000);
3397 vc->tb_offset_applied = vc->tb_offset;
3398 }
3399
3400 if (vc->pcr)
3401 mtspr(SPRN_PCR, vc->pcr);
3402 mtspr(SPRN_DPDES, vc->dpdes);
3403 mtspr(SPRN_VTB, vc->vtb);
3404
3405 local_paca->kvm_hstate.host_purr = mfspr(SPRN_PURR);
3406 local_paca->kvm_hstate.host_spurr = mfspr(SPRN_SPURR);
3407 mtspr(SPRN_PURR, vcpu->arch.purr);
3408 mtspr(SPRN_SPURR, vcpu->arch.spurr);
3409
3410 if (dawr_enabled()) {
3411 mtspr(SPRN_DAWR, vcpu->arch.dawr);
3412 mtspr(SPRN_DAWRX, vcpu->arch.dawrx);
3413 }
3414 mtspr(SPRN_CIABR, vcpu->arch.ciabr);
3415 mtspr(SPRN_IC, vcpu->arch.ic);
3416 mtspr(SPRN_PID, vcpu->arch.pid);
3417
3418 mtspr(SPRN_PSSCR, vcpu->arch.psscr | PSSCR_EC |
3419 (local_paca->kvm_hstate.fake_suspend << PSSCR_FAKE_SUSPEND_LG));
3420
3421 mtspr(SPRN_HFSCR, vcpu->arch.hfscr);
3422
3423 mtspr(SPRN_SPRG0, vcpu->arch.shregs.sprg0);
3424 mtspr(SPRN_SPRG1, vcpu->arch.shregs.sprg1);
3425 mtspr(SPRN_SPRG2, vcpu->arch.shregs.sprg2);
3426 mtspr(SPRN_SPRG3, vcpu->arch.shregs.sprg3);
3427
3428 mtspr(SPRN_AMOR, ~0UL);
3429
3430 mtspr(SPRN_LPCR, lpcr);
3431 isync();
3432
3433 kvmppc_xive_push_vcpu(vcpu);
3434
3435 mtspr(SPRN_SRR0, vcpu->arch.shregs.srr0);
3436 mtspr(SPRN_SRR1, vcpu->arch.shregs.srr1);
3437
3438 trap = __kvmhv_vcpu_entry_p9(vcpu);
3439
3440
3441 purr = mfspr(SPRN_PURR);
3442 spurr = mfspr(SPRN_SPURR);
3443 mtspr(SPRN_PURR, local_paca->kvm_hstate.host_purr +
3444 purr - vcpu->arch.purr);
3445 mtspr(SPRN_SPURR, local_paca->kvm_hstate.host_spurr +
3446 spurr - vcpu->arch.spurr);
3447 vcpu->arch.purr = purr;
3448 vcpu->arch.spurr = spurr;
3449
3450 vcpu->arch.ic = mfspr(SPRN_IC);
3451 vcpu->arch.pid = mfspr(SPRN_PID);
3452 vcpu->arch.psscr = mfspr(SPRN_PSSCR) & PSSCR_GUEST_VIS;
3453
3454 vcpu->arch.shregs.sprg0 = mfspr(SPRN_SPRG0);
3455 vcpu->arch.shregs.sprg1 = mfspr(SPRN_SPRG1);
3456 vcpu->arch.shregs.sprg2 = mfspr(SPRN_SPRG2);
3457 vcpu->arch.shregs.sprg3 = mfspr(SPRN_SPRG3);
3458
3459
3460 mtspr(SPRN_PSSCR, host_psscr |
3461 (local_paca->kvm_hstate.fake_suspend << PSSCR_FAKE_SUSPEND_LG));
3462 mtspr(SPRN_HFSCR, host_hfscr);
3463 mtspr(SPRN_CIABR, host_ciabr);
3464 mtspr(SPRN_DAWR, host_dawr);
3465 mtspr(SPRN_DAWRX, host_dawrx);
3466 mtspr(SPRN_PID, host_pidr);
3467
3468
3469
3470
3471
3472 asm volatile("eieio; tlbsync; ptesync");
3473
3474 mtspr(SPRN_LPID, vcpu->kvm->arch.host_lpid);
3475 isync();
3476
3477 vc->dpdes = mfspr(SPRN_DPDES);
3478 vc->vtb = mfspr(SPRN_VTB);
3479 mtspr(SPRN_DPDES, 0);
3480 if (vc->pcr)
3481 mtspr(SPRN_PCR, 0);
3482
3483 if (vc->tb_offset_applied) {
3484 u64 new_tb = mftb() - vc->tb_offset_applied;
3485 mtspr(SPRN_TBU40, new_tb);
3486 tb = mftb();
3487 if ((tb & 0xffffff) < (new_tb & 0xffffff))
3488 mtspr(SPRN_TBU40, new_tb + 0x1000000);
3489 vc->tb_offset_applied = 0;
3490 }
3491
3492 mtspr(SPRN_HDEC, 0x7fffffff);
3493 mtspr(SPRN_LPCR, vcpu->kvm->arch.host_lpcr);
3494
3495 return trap;
3496}
3497
3498
3499
3500
3501
3502int kvmhv_p9_guest_entry(struct kvm_vcpu *vcpu, u64 time_limit,
3503 unsigned long lpcr)
3504{
3505 struct kvmppc_vcore *vc = vcpu->arch.vcore;
3506 unsigned long host_dscr = mfspr(SPRN_DSCR);
3507 unsigned long host_tidr = mfspr(SPRN_TIDR);
3508 unsigned long host_iamr = mfspr(SPRN_IAMR);
3509 unsigned long host_amr = mfspr(SPRN_AMR);
3510 s64 dec;
3511 u64 tb;
3512 int trap, save_pmu;
3513
3514 dec = mfspr(SPRN_DEC);
3515 tb = mftb();
3516 if (dec < 512)
3517 return BOOK3S_INTERRUPT_HV_DECREMENTER;
3518 local_paca->kvm_hstate.dec_expires = dec + tb;
3519 if (local_paca->kvm_hstate.dec_expires < time_limit)
3520 time_limit = local_paca->kvm_hstate.dec_expires;
3521
3522 vcpu->arch.ceded = 0;
3523
3524 kvmhv_save_host_pmu();
3525
3526 kvmppc_subcore_enter_guest();
3527
3528 vc->entry_exit_map = 1;
3529 vc->in_guest = 1;
3530
3531 if (vcpu->arch.vpa.pinned_addr) {
3532 struct lppaca *lp = vcpu->arch.vpa.pinned_addr;
3533 u32 yield_count = be32_to_cpu(lp->yield_count) + 1;
3534 lp->yield_count = cpu_to_be32(yield_count);
3535 vcpu->arch.vpa.dirty = 1;
3536 }
3537
3538 if (cpu_has_feature(CPU_FTR_TM) ||
3539 cpu_has_feature(CPU_FTR_P9_TM_HV_ASSIST))
3540 kvmppc_restore_tm_hv(vcpu, vcpu->arch.shregs.msr, true);
3541
3542 kvmhv_load_guest_pmu(vcpu);
3543
3544 msr_check_and_set(MSR_FP | MSR_VEC | MSR_VSX);
3545 load_fp_state(&vcpu->arch.fp);
3546#ifdef CONFIG_ALTIVEC
3547 load_vr_state(&vcpu->arch.vr);
3548#endif
3549 mtspr(SPRN_VRSAVE, vcpu->arch.vrsave);
3550
3551 mtspr(SPRN_DSCR, vcpu->arch.dscr);
3552 mtspr(SPRN_IAMR, vcpu->arch.iamr);
3553 mtspr(SPRN_PSPB, vcpu->arch.pspb);
3554 mtspr(SPRN_FSCR, vcpu->arch.fscr);
3555 mtspr(SPRN_TAR, vcpu->arch.tar);
3556 mtspr(SPRN_EBBHR, vcpu->arch.ebbhr);
3557 mtspr(SPRN_EBBRR, vcpu->arch.ebbrr);
3558 mtspr(SPRN_BESCR, vcpu->arch.bescr);
3559 mtspr(SPRN_WORT, vcpu->arch.wort);
3560 mtspr(SPRN_TIDR, vcpu->arch.tid);
3561 mtspr(SPRN_DAR, vcpu->arch.shregs.dar);
3562 mtspr(SPRN_DSISR, vcpu->arch.shregs.dsisr);
3563 mtspr(SPRN_AMR, vcpu->arch.amr);
3564 mtspr(SPRN_UAMOR, vcpu->arch.uamor);
3565
3566 if (!(vcpu->arch.ctrl & 1))
3567 mtspr(SPRN_CTRLT, mfspr(SPRN_CTRLF) & ~1);
3568
3569 mtspr(SPRN_DEC, vcpu->arch.dec_expires - mftb());
3570
3571 if (kvmhv_on_pseries()) {
3572
3573
3574
3575
3576
3577
3578 unsigned long host_psscr;
3579
3580 struct hv_guest_state hvregs;
3581
3582 host_psscr = mfspr(SPRN_PSSCR_PR);
3583 mtspr(SPRN_PSSCR_PR, vcpu->arch.psscr);
3584 kvmhv_save_hv_regs(vcpu, &hvregs);
3585 hvregs.lpcr = lpcr;
3586 vcpu->arch.regs.msr = vcpu->arch.shregs.msr;
3587 hvregs.version = HV_GUEST_STATE_VERSION;
3588 if (vcpu->arch.nested) {
3589 hvregs.lpid = vcpu->arch.nested->shadow_lpid;
3590 hvregs.vcpu_token = vcpu->arch.nested_vcpu_id;
3591 } else {
3592 hvregs.lpid = vcpu->kvm->arch.lpid;
3593 hvregs.vcpu_token = vcpu->vcpu_id;
3594 }
3595 hvregs.hdec_expiry = time_limit;
3596 trap = plpar_hcall_norets(H_ENTER_NESTED, __pa(&hvregs),
3597 __pa(&vcpu->arch.regs));
3598 kvmhv_restore_hv_return_state(vcpu, &hvregs);
3599 vcpu->arch.shregs.msr = vcpu->arch.regs.msr;
3600 vcpu->arch.shregs.dar = mfspr(SPRN_DAR);
3601 vcpu->arch.shregs.dsisr = mfspr(SPRN_DSISR);
3602 vcpu->arch.psscr = mfspr(SPRN_PSSCR_PR);
3603 mtspr(SPRN_PSSCR_PR, host_psscr);
3604
3605
3606 if (trap == BOOK3S_INTERRUPT_SYSCALL && !vcpu->arch.nested &&
3607 kvmppc_get_gpr(vcpu, 3) == H_CEDE) {
3608 kvmppc_nested_cede(vcpu);
3609 trap = 0;
3610 }
3611 } else {
3612 trap = kvmhv_load_hv_regs_and_go(vcpu, time_limit, lpcr);
3613 }
3614
3615 vcpu->arch.slb_max = 0;
3616 dec = mfspr(SPRN_DEC);
3617 if (!(lpcr & LPCR_LD))
3618 dec = (s32) dec;
3619 tb = mftb();
3620 vcpu->arch.dec_expires = dec + tb;
3621 vcpu->cpu = -1;
3622 vcpu->arch.thread_cpu = -1;
3623 vcpu->arch.ctrl = mfspr(SPRN_CTRLF);
3624
3625 vcpu->arch.iamr = mfspr(SPRN_IAMR);
3626 vcpu->arch.pspb = mfspr(SPRN_PSPB);
3627 vcpu->arch.fscr = mfspr(SPRN_FSCR);
3628 vcpu->arch.tar = mfspr(SPRN_TAR);
3629 vcpu->arch.ebbhr = mfspr(SPRN_EBBHR);
3630 vcpu->arch.ebbrr = mfspr(SPRN_EBBRR);
3631 vcpu->arch.bescr = mfspr(SPRN_BESCR);
3632 vcpu->arch.wort = mfspr(SPRN_WORT);
3633 vcpu->arch.tid = mfspr(SPRN_TIDR);
3634 vcpu->arch.amr = mfspr(SPRN_AMR);
3635 vcpu->arch.uamor = mfspr(SPRN_UAMOR);
3636 vcpu->arch.dscr = mfspr(SPRN_DSCR);
3637
3638 mtspr(SPRN_PSPB, 0);
3639 mtspr(SPRN_WORT, 0);
3640 mtspr(SPRN_UAMOR, 0);
3641 mtspr(SPRN_DSCR, host_dscr);
3642 mtspr(SPRN_TIDR, host_tidr);
3643 mtspr(SPRN_IAMR, host_iamr);
3644 mtspr(SPRN_PSPB, 0);
3645
3646 if (host_amr != vcpu->arch.amr)
3647 mtspr(SPRN_AMR, host_amr);
3648
3649 msr_check_and_set(MSR_FP | MSR_VEC | MSR_VSX);
3650 store_fp_state(&vcpu->arch.fp);
3651#ifdef CONFIG_ALTIVEC
3652 store_vr_state(&vcpu->arch.vr);
3653#endif
3654 vcpu->arch.vrsave = mfspr(SPRN_VRSAVE);
3655
3656 if (cpu_has_feature(CPU_FTR_TM) ||
3657 cpu_has_feature(CPU_FTR_P9_TM_HV_ASSIST))
3658 kvmppc_save_tm_hv(vcpu, vcpu->arch.shregs.msr, true);
3659
3660 save_pmu = 1;
3661 if (vcpu->arch.vpa.pinned_addr) {
3662 struct lppaca *lp = vcpu->arch.vpa.pinned_addr;
3663 u32 yield_count = be32_to_cpu(lp->yield_count) + 1;
3664 lp->yield_count = cpu_to_be32(yield_count);
3665 vcpu->arch.vpa.dirty = 1;
3666 save_pmu = lp->pmcregs_in_use;
3667 }
3668
3669 save_pmu |= nesting_enabled(vcpu->kvm);
3670
3671 kvmhv_save_guest_pmu(vcpu, save_pmu);
3672
3673 vc->entry_exit_map = 0x101;
3674 vc->in_guest = 0;
3675
3676 mtspr(SPRN_DEC, local_paca->kvm_hstate.dec_expires - mftb());
3677 mtspr(SPRN_SPRG_VDSO_WRITE, local_paca->sprg_vdso);
3678
3679 kvmhv_load_host_pmu();
3680
3681 kvmppc_subcore_exit_guest();
3682
3683 return trap;
3684}
3685
3686
3687
3688
3689
3690static void kvmppc_wait_for_exec(struct kvmppc_vcore *vc,
3691 struct kvm_vcpu *vcpu, int wait_state)
3692{
3693 DEFINE_WAIT(wait);
3694
3695 prepare_to_wait(&vcpu->arch.cpu_run, &wait, wait_state);
3696 if (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE) {
3697 spin_unlock(&vc->lock);
3698 schedule();
3699 spin_lock(&vc->lock);
3700 }
3701 finish_wait(&vcpu->arch.cpu_run, &wait);
3702}
3703
3704static void grow_halt_poll_ns(struct kvmppc_vcore *vc)
3705{
3706 if (!halt_poll_ns_grow)
3707 return;
3708
3709 vc->halt_poll_ns *= halt_poll_ns_grow;
3710 if (vc->halt_poll_ns < halt_poll_ns_grow_start)
3711 vc->halt_poll_ns = halt_poll_ns_grow_start;
3712}
3713
3714static void shrink_halt_poll_ns(struct kvmppc_vcore *vc)
3715{
3716 if (halt_poll_ns_shrink == 0)
3717 vc->halt_poll_ns = 0;
3718 else
3719 vc->halt_poll_ns /= halt_poll_ns_shrink;
3720}
3721
3722#ifdef CONFIG_KVM_XICS
3723static inline bool xive_interrupt_pending(struct kvm_vcpu *vcpu)
3724{
3725 if (!xics_on_xive())
3726 return false;
3727 return vcpu->arch.irq_pending || vcpu->arch.xive_saved_state.pipr <
3728 vcpu->arch.xive_saved_state.cppr;
3729}
3730#else
3731static inline bool xive_interrupt_pending(struct kvm_vcpu *vcpu)
3732{
3733 return false;
3734}
3735#endif
3736
3737static bool kvmppc_vcpu_woken(struct kvm_vcpu *vcpu)
3738{
3739 if (vcpu->arch.pending_exceptions || vcpu->arch.prodded ||
3740 kvmppc_doorbell_pending(vcpu) || xive_interrupt_pending(vcpu))
3741 return true;
3742
3743 return false;
3744}
3745
3746
3747
3748
3749
3750static int kvmppc_vcore_check_block(struct kvmppc_vcore *vc)
3751{
3752 struct kvm_vcpu *vcpu;
3753 int i;
3754
3755 for_each_runnable_thread(i, vcpu, vc) {
3756 if (!vcpu->arch.ceded || kvmppc_vcpu_woken(vcpu))
3757 return 1;
3758 }
3759
3760 return 0;
3761}
3762
3763
3764
3765
3766
3767static void kvmppc_vcore_blocked(struct kvmppc_vcore *vc)
3768{
3769 ktime_t cur, start_poll, start_wait;
3770 int do_sleep = 1;
3771 u64 block_ns;
3772 DECLARE_SWAITQUEUE(wait);
3773
3774
3775 cur = start_poll = ktime_get();
3776 if (vc->halt_poll_ns) {
3777 ktime_t stop = ktime_add_ns(start_poll, vc->halt_poll_ns);
3778 ++vc->runner->stat.halt_attempted_poll;
3779
3780 vc->vcore_state = VCORE_POLLING;
3781 spin_unlock(&vc->lock);
3782
3783 do {
3784 if (kvmppc_vcore_check_block(vc)) {
3785 do_sleep = 0;
3786 break;
3787 }
3788 cur = ktime_get();
3789 } while (single_task_running() && ktime_before(cur, stop));
3790
3791 spin_lock(&vc->lock);
3792 vc->vcore_state = VCORE_INACTIVE;
3793
3794 if (!do_sleep) {
3795 ++vc->runner->stat.halt_successful_poll;
3796 goto out;
3797 }
3798 }
3799
3800 prepare_to_swait_exclusive(&vc->wq, &wait, TASK_INTERRUPTIBLE);
3801
3802 if (kvmppc_vcore_check_block(vc)) {
3803 finish_swait(&vc->wq, &wait);
3804 do_sleep = 0;
3805
3806 if (vc->halt_poll_ns)
3807 ++vc->runner->stat.halt_successful_poll;
3808 goto out;
3809 }
3810
3811 start_wait = ktime_get();
3812
3813 vc->vcore_state = VCORE_SLEEPING;
3814 trace_kvmppc_vcore_blocked(vc, 0);
3815 spin_unlock(&vc->lock);
3816 schedule();
3817 finish_swait(&vc->wq, &wait);
3818 spin_lock(&vc->lock);
3819 vc->vcore_state = VCORE_INACTIVE;
3820 trace_kvmppc_vcore_blocked(vc, 1);
3821 ++vc->runner->stat.halt_successful_wait;
3822
3823 cur = ktime_get();
3824
3825out:
3826 block_ns = ktime_to_ns(cur) - ktime_to_ns(start_poll);
3827
3828
3829 if (do_sleep) {
3830 vc->runner->stat.halt_wait_ns +=
3831 ktime_to_ns(cur) - ktime_to_ns(start_wait);
3832
3833 if (vc->halt_poll_ns)
3834 vc->runner->stat.halt_poll_fail_ns +=
3835 ktime_to_ns(start_wait) -
3836 ktime_to_ns(start_poll);
3837 } else {
3838
3839 if (vc->halt_poll_ns)
3840 vc->runner->stat.halt_poll_success_ns +=
3841 ktime_to_ns(cur) -
3842 ktime_to_ns(start_poll);
3843 }
3844
3845
3846 if (halt_poll_ns) {
3847 if (block_ns <= vc->halt_poll_ns)
3848 ;
3849
3850 else if (vc->halt_poll_ns && block_ns > halt_poll_ns)
3851 shrink_halt_poll_ns(vc);
3852
3853 else if (vc->halt_poll_ns < halt_poll_ns &&
3854 block_ns < halt_poll_ns)
3855 grow_halt_poll_ns(vc);
3856 if (vc->halt_poll_ns > halt_poll_ns)
3857 vc->halt_poll_ns = halt_poll_ns;
3858 } else
3859 vc->halt_poll_ns = 0;
3860
3861 trace_kvmppc_vcore_wakeup(do_sleep, block_ns);
3862}
3863
3864
3865
3866
3867
3868
3869static int kvmhv_setup_mmu(struct kvm_vcpu *vcpu)
3870{
3871 int r = 0;
3872 struct kvm *kvm = vcpu->kvm;
3873
3874 mutex_lock(&kvm->arch.mmu_setup_lock);
3875 if (!kvm->arch.mmu_ready) {
3876 if (!kvm_is_radix(kvm))
3877 r = kvmppc_hv_setup_htab_rma(vcpu);
3878 if (!r) {
3879 if (cpu_has_feature(CPU_FTR_ARCH_300))
3880 kvmppc_setup_partition_table(kvm);
3881 kvm->arch.mmu_ready = 1;
3882 }
3883 }
3884 mutex_unlock(&kvm->arch.mmu_setup_lock);
3885 return r;
3886}
3887
3888static int kvmppc_run_vcpu(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu)
3889{
3890 int n_ceded, i, r;
3891 struct kvmppc_vcore *vc;
3892 struct kvm_vcpu *v;
3893
3894 trace_kvmppc_run_vcpu_enter(vcpu);
3895
3896 kvm_run->exit_reason = 0;
3897 vcpu->arch.ret = RESUME_GUEST;
3898 vcpu->arch.trap = 0;
3899 kvmppc_update_vpas(vcpu);
3900
3901
3902
3903
3904 vc = vcpu->arch.vcore;
3905 spin_lock(&vc->lock);
3906 vcpu->arch.ceded = 0;
3907 vcpu->arch.run_task = current;
3908 vcpu->arch.kvm_run = kvm_run;
3909 vcpu->arch.stolen_logged = vcore_stolen_time(vc, mftb());
3910 vcpu->arch.state = KVMPPC_VCPU_RUNNABLE;
3911 vcpu->arch.busy_preempt = TB_NIL;
3912 WRITE_ONCE(vc->runnable_threads[vcpu->arch.ptid], vcpu);
3913 ++vc->n_runnable;
3914
3915
3916
3917
3918
3919
3920 if (!signal_pending(current)) {
3921 if ((vc->vcore_state == VCORE_PIGGYBACK ||
3922 vc->vcore_state == VCORE_RUNNING) &&
3923 !VCORE_IS_EXITING(vc)) {
3924 kvmppc_create_dtl_entry(vcpu, vc);
3925 kvmppc_start_thread(vcpu, vc);
3926 trace_kvm_guest_enter(vcpu);
3927 } else if (vc->vcore_state == VCORE_SLEEPING) {
3928 swake_up_one(&vc->wq);
3929 }
3930
3931 }
3932
3933 while (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE &&
3934 !signal_pending(current)) {
3935
3936 if (!vcpu->kvm->arch.mmu_ready) {
3937 spin_unlock(&vc->lock);
3938 r = kvmhv_setup_mmu(vcpu);
3939 spin_lock(&vc->lock);
3940 if (r) {
3941 kvm_run->exit_reason = KVM_EXIT_FAIL_ENTRY;
3942 kvm_run->fail_entry.
3943 hardware_entry_failure_reason = 0;
3944 vcpu->arch.ret = r;
3945 break;
3946 }
3947 }
3948
3949 if (vc->vcore_state == VCORE_PREEMPT && vc->runner == NULL)
3950 kvmppc_vcore_end_preempt(vc);
3951
3952 if (vc->vcore_state != VCORE_INACTIVE) {
3953 kvmppc_wait_for_exec(vc, vcpu, TASK_INTERRUPTIBLE);
3954 continue;
3955 }
3956 for_each_runnable_thread(i, v, vc) {
3957 kvmppc_core_prepare_to_enter(v);
3958 if (signal_pending(v->arch.run_task)) {
3959 kvmppc_remove_runnable(vc, v);
3960 v->stat.signal_exits++;
3961 v->arch.kvm_run->exit_reason = KVM_EXIT_INTR;
3962 v->arch.ret = -EINTR;
3963 wake_up(&v->arch.cpu_run);
3964 }
3965 }
3966 if (!vc->n_runnable || vcpu->arch.state != KVMPPC_VCPU_RUNNABLE)
3967 break;
3968 n_ceded = 0;
3969 for_each_runnable_thread(i, v, vc) {
3970 if (!kvmppc_vcpu_woken(v))
3971 n_ceded += v->arch.ceded;
3972 else
3973 v->arch.ceded = 0;
3974 }
3975 vc->runner = vcpu;
3976 if (n_ceded == vc->n_runnable) {
3977 kvmppc_vcore_blocked(vc);
3978 } else if (need_resched()) {
3979 kvmppc_vcore_preempt(vc);
3980
3981 cond_resched_lock(&vc->lock);
3982 if (vc->vcore_state == VCORE_PREEMPT)
3983 kvmppc_vcore_end_preempt(vc);
3984 } else {
3985 kvmppc_run_core(vc);
3986 }
3987 vc->runner = NULL;
3988 }
3989
3990 while (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE &&
3991 (vc->vcore_state == VCORE_RUNNING ||
3992 vc->vcore_state == VCORE_EXITING ||
3993 vc->vcore_state == VCORE_PIGGYBACK))
3994 kvmppc_wait_for_exec(vc, vcpu, TASK_UNINTERRUPTIBLE);
3995
3996 if (vc->vcore_state == VCORE_PREEMPT && vc->runner == NULL)
3997 kvmppc_vcore_end_preempt(vc);
3998
3999 if (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE) {
4000 kvmppc_remove_runnable(vc, vcpu);
4001 vcpu->stat.signal_exits++;
4002 kvm_run->exit_reason = KVM_EXIT_INTR;
4003 vcpu->arch.ret = -EINTR;
4004 }
4005
4006 if (vc->n_runnable && vc->vcore_state == VCORE_INACTIVE) {
4007
4008 i = -1;
4009 v = next_runnable_thread(vc, &i);
4010 wake_up(&v->arch.cpu_run);
4011 }
4012
4013 trace_kvmppc_run_vcpu_exit(vcpu, kvm_run);
4014 spin_unlock(&vc->lock);
4015 return vcpu->arch.ret;
4016}
4017
4018int kvmhv_run_single_vcpu(struct kvm_run *kvm_run,
4019 struct kvm_vcpu *vcpu, u64 time_limit,
4020 unsigned long lpcr)
4021{
4022 int trap, r, pcpu;
4023 int srcu_idx, lpid;
4024 struct kvmppc_vcore *vc;
4025 struct kvm *kvm = vcpu->kvm;
4026 struct kvm_nested_guest *nested = vcpu->arch.nested;
4027
4028 trace_kvmppc_run_vcpu_enter(vcpu);
4029
4030 kvm_run->exit_reason = 0;
4031 vcpu->arch.ret = RESUME_GUEST;
4032 vcpu->arch.trap = 0;
4033
4034 vc = vcpu->arch.vcore;
4035 vcpu->arch.ceded = 0;
4036 vcpu->arch.run_task = current;
4037 vcpu->arch.kvm_run = kvm_run;
4038 vcpu->arch.stolen_logged = vcore_stolen_time(vc, mftb());
4039 vcpu->arch.state = KVMPPC_VCPU_RUNNABLE;
4040 vcpu->arch.busy_preempt = TB_NIL;
4041 vcpu->arch.last_inst = KVM_INST_FETCH_FAILED;
4042 vc->runnable_threads[0] = vcpu;
4043 vc->n_runnable = 1;
4044 vc->runner = vcpu;
4045
4046
4047 if (!kvm->arch.mmu_ready)
4048 kvmhv_setup_mmu(vcpu);
4049
4050 if (need_resched())
4051 cond_resched();
4052
4053 kvmppc_update_vpas(vcpu);
4054
4055 init_vcore_to_run(vc);
4056 vc->preempt_tb = TB_NIL;
4057
4058 preempt_disable();
4059 pcpu = smp_processor_id();
4060 vc->pcpu = pcpu;
4061 kvmppc_prepare_radix_vcpu(vcpu, pcpu);
4062
4063 local_irq_disable();
4064 hard_irq_disable();
4065 if (signal_pending(current))
4066 goto sigpend;
4067 if (lazy_irq_pending() || need_resched() || !kvm->arch.mmu_ready)
4068 goto out;
4069
4070 if (!nested) {
4071 kvmppc_core_prepare_to_enter(vcpu);
4072 if (vcpu->arch.doorbell_request) {
4073 vc->dpdes = 1;
4074 smp_wmb();
4075 vcpu->arch.doorbell_request = 0;
4076 }
4077 if (test_bit(BOOK3S_IRQPRIO_EXTERNAL,
4078 &vcpu->arch.pending_exceptions))
4079 lpcr |= LPCR_MER;
4080 } else if (vcpu->arch.pending_exceptions ||
4081 vcpu->arch.doorbell_request ||
4082 xive_interrupt_pending(vcpu)) {
4083 vcpu->arch.ret = RESUME_HOST;
4084 goto out;
4085 }
4086
4087 kvmppc_clear_host_core(pcpu);
4088
4089 local_paca->kvm_hstate.tid = 0;
4090 local_paca->kvm_hstate.napping = 0;
4091 local_paca->kvm_hstate.kvm_split_mode = NULL;
4092 kvmppc_start_thread(vcpu, vc);
4093 kvmppc_create_dtl_entry(vcpu, vc);
4094 trace_kvm_guest_enter(vcpu);
4095
4096 vc->vcore_state = VCORE_RUNNING;
4097 trace_kvmppc_run_core(vc, 0);
4098
4099 if (cpu_has_feature(CPU_FTR_HVMODE)) {
4100 lpid = nested ? nested->shadow_lpid : kvm->arch.lpid;
4101 mtspr(SPRN_LPID, lpid);
4102 isync();
4103 kvmppc_check_need_tlb_flush(kvm, pcpu, nested);
4104 }
4105
4106 guest_enter_irqoff();
4107
4108 srcu_idx = srcu_read_lock(&kvm->srcu);
4109
4110 this_cpu_disable_ftrace();
4111
4112
4113 trace_hardirqs_on();
4114
4115 trap = kvmhv_p9_guest_entry(vcpu, time_limit, lpcr);
4116 vcpu->arch.trap = trap;
4117
4118 trace_hardirqs_off();
4119
4120 this_cpu_enable_ftrace();
4121
4122 srcu_read_unlock(&kvm->srcu, srcu_idx);
4123
4124 if (cpu_has_feature(CPU_FTR_HVMODE)) {
4125 mtspr(SPRN_LPID, kvm->arch.host_lpid);
4126 isync();
4127 }
4128
4129 set_irq_happened(trap);
4130
4131 kvmppc_set_host_core(pcpu);
4132
4133 local_irq_enable();
4134 guest_exit();
4135
4136 cpumask_clear_cpu(pcpu, &kvm->arch.cpu_in_guest);
4137
4138 preempt_enable();
4139
4140
4141
4142
4143
4144
4145 if (kvmppc_core_pending_dec(vcpu) &&
4146 ((get_tb() < vcpu->arch.dec_expires) ||
4147 (trap == BOOK3S_INTERRUPT_SYSCALL &&
4148 kvmppc_get_gpr(vcpu, 3) == H_ENTER_NESTED)))
4149 kvmppc_core_dequeue_dec(vcpu);
4150
4151 trace_kvm_guest_exit(vcpu);
4152 r = RESUME_GUEST;
4153 if (trap) {
4154 if (!nested)
4155 r = kvmppc_handle_exit_hv(kvm_run, vcpu, current);
4156 else
4157 r = kvmppc_handle_nested_exit(kvm_run, vcpu);
4158 }
4159 vcpu->arch.ret = r;
4160
4161 if (is_kvmppc_resume_guest(r) && vcpu->arch.ceded &&
4162 !kvmppc_vcpu_woken(vcpu)) {
4163 kvmppc_set_timer(vcpu);
4164 while (vcpu->arch.ceded && !kvmppc_vcpu_woken(vcpu)) {
4165 if (signal_pending(current)) {
4166 vcpu->stat.signal_exits++;
4167 kvm_run->exit_reason = KVM_EXIT_INTR;
4168 vcpu->arch.ret = -EINTR;
4169 break;
4170 }
4171 spin_lock(&vc->lock);
4172 kvmppc_vcore_blocked(vc);
4173 spin_unlock(&vc->lock);
4174 }
4175 }
4176 vcpu->arch.ceded = 0;
4177
4178 vc->vcore_state = VCORE_INACTIVE;
4179 trace_kvmppc_run_core(vc, 1);
4180
4181 done:
4182 kvmppc_remove_runnable(vc, vcpu);
4183 trace_kvmppc_run_vcpu_exit(vcpu, kvm_run);
4184
4185 return vcpu->arch.ret;
4186
4187 sigpend:
4188 vcpu->stat.signal_exits++;
4189 kvm_run->exit_reason = KVM_EXIT_INTR;
4190 vcpu->arch.ret = -EINTR;
4191 out:
4192 local_irq_enable();
4193 preempt_enable();
4194 goto done;
4195}
4196
4197static int kvmppc_vcpu_run_hv(struct kvm_run *run, struct kvm_vcpu *vcpu)
4198{
4199 int r;
4200 int srcu_idx;
4201 unsigned long ebb_regs[3] = {};
4202 unsigned long user_tar = 0;
4203 unsigned int user_vrsave;
4204 struct kvm *kvm;
4205
4206 if (!vcpu->arch.sane) {
4207 run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
4208 return -EINVAL;
4209 }
4210
4211
4212
4213
4214
4215
4216
4217#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
4218 if (cpu_has_feature(CPU_FTR_TM) && current->thread.regs &&
4219 (current->thread.regs->msr & MSR_TM)) {
4220 if (MSR_TM_ACTIVE(current->thread.regs->msr)) {
4221 run->exit_reason = KVM_EXIT_FAIL_ENTRY;
4222 run->fail_entry.hardware_entry_failure_reason = 0;
4223 return -EINVAL;
4224 }
4225
4226 mtmsr(mfmsr() | MSR_TM);
4227 current->thread.tm_tfhar = mfspr(SPRN_TFHAR);
4228 current->thread.tm_tfiar = mfspr(SPRN_TFIAR);
4229 current->thread.tm_texasr = mfspr(SPRN_TEXASR);
4230 current->thread.regs->msr &= ~MSR_TM;
4231 }
4232#endif
4233
4234
4235
4236
4237
4238 if (!vcpu->arch.online) {
4239 atomic_inc(&vcpu->arch.vcore->online_count);
4240 vcpu->arch.online = 1;
4241 }
4242
4243 kvmppc_core_prepare_to_enter(vcpu);
4244
4245
4246 if (signal_pending(current)) {
4247 run->exit_reason = KVM_EXIT_INTR;
4248 return -EINTR;
4249 }
4250
4251 kvm = vcpu->kvm;
4252 atomic_inc(&kvm->arch.vcpus_running);
4253
4254 smp_mb();
4255
4256 flush_all_to_thread(current);
4257
4258
4259 if (cpu_has_feature(CPU_FTR_ARCH_207S)) {
4260 ebb_regs[0] = mfspr(SPRN_EBBHR);
4261 ebb_regs[1] = mfspr(SPRN_EBBRR);
4262 ebb_regs[2] = mfspr(SPRN_BESCR);
4263 user_tar = mfspr(SPRN_TAR);
4264 }
4265 user_vrsave = mfspr(SPRN_VRSAVE);
4266
4267 vcpu->arch.wqp = &vcpu->arch.vcore->wq;
4268 vcpu->arch.pgdir = current->mm->pgd;
4269 vcpu->arch.state = KVMPPC_VCPU_BUSY_IN_HOST;
4270
4271 do {
4272
4273
4274
4275
4276
4277
4278
4279
4280 if (kvm->arch.threads_indep && kvm_is_radix(kvm) &&
4281 !no_mixing_hpt_and_radix)
4282 r = kvmhv_run_single_vcpu(run, vcpu, ~(u64)0,
4283 vcpu->arch.vcore->lpcr);
4284 else
4285 r = kvmppc_run_vcpu(run, vcpu);
4286
4287 if (run->exit_reason == KVM_EXIT_PAPR_HCALL &&
4288 !(vcpu->arch.shregs.msr & MSR_PR)) {
4289 trace_kvm_hcall_enter(vcpu);
4290 r = kvmppc_pseries_do_hcall(vcpu);
4291 trace_kvm_hcall_exit(vcpu, r);
4292 kvmppc_core_prepare_to_enter(vcpu);
4293 } else if (r == RESUME_PAGE_FAULT) {
4294 srcu_idx = srcu_read_lock(&kvm->srcu);
4295 r = kvmppc_book3s_hv_page_fault(run, vcpu,
4296 vcpu->arch.fault_dar, vcpu->arch.fault_dsisr);
4297 srcu_read_unlock(&kvm->srcu, srcu_idx);
4298 } else if (r == RESUME_PASSTHROUGH) {
4299 if (WARN_ON(xics_on_xive()))
4300 r = H_SUCCESS;
4301 else
4302 r = kvmppc_xics_rm_complete(vcpu, 0);
4303 }
4304 } while (is_kvmppc_resume_guest(r));
4305
4306
4307 if (cpu_has_feature(CPU_FTR_ARCH_207S)) {
4308 mtspr(SPRN_EBBHR, ebb_regs[0]);
4309 mtspr(SPRN_EBBRR, ebb_regs[1]);
4310 mtspr(SPRN_BESCR, ebb_regs[2]);
4311 mtspr(SPRN_TAR, user_tar);
4312 mtspr(SPRN_FSCR, current->thread.fscr);
4313 }
4314 mtspr(SPRN_VRSAVE, user_vrsave);
4315
4316 vcpu->arch.state = KVMPPC_VCPU_NOTREADY;
4317 atomic_dec(&kvm->arch.vcpus_running);
4318 return r;
4319}
4320
4321static void kvmppc_add_seg_page_size(struct kvm_ppc_one_seg_page_size **sps,
4322 int shift, int sllp)
4323{
4324 (*sps)->page_shift = shift;
4325 (*sps)->slb_enc = sllp;
4326 (*sps)->enc[0].page_shift = shift;
4327 (*sps)->enc[0].pte_enc = kvmppc_pgsize_lp_encoding(shift, shift);
4328
4329
4330
4331 if (shift != 24) {
4332 int penc = kvmppc_pgsize_lp_encoding(shift, 24);
4333 if (penc != -1) {
4334 (*sps)->enc[1].page_shift = 24;
4335 (*sps)->enc[1].pte_enc = penc;
4336 }
4337 }
4338 (*sps)++;
4339}
4340
4341static int kvm_vm_ioctl_get_smmu_info_hv(struct kvm *kvm,
4342 struct kvm_ppc_smmu_info *info)
4343{
4344 struct kvm_ppc_one_seg_page_size *sps;
4345
4346
4347
4348
4349
4350
4351 info->data_keys = 32;
4352 info->instr_keys = cpu_has_feature(CPU_FTR_ARCH_207S) ? 32 : 0;
4353
4354
4355 info->flags = KVM_PPC_PAGE_SIZES_REAL | KVM_PPC_1T_SEGMENTS;
4356 info->slb_size = 32;
4357
4358
4359 sps = &info->sps[0];
4360 kvmppc_add_seg_page_size(&sps, 12, 0);
4361 kvmppc_add_seg_page_size(&sps, 16, SLB_VSID_L | SLB_VSID_LP_01);
4362 kvmppc_add_seg_page_size(&sps, 24, SLB_VSID_L);
4363
4364
4365 if (kvmhv_on_pseries())
4366 info->flags |= KVM_PPC_NO_HASH;
4367
4368 return 0;
4369}
4370
4371
4372
4373
4374static int kvm_vm_ioctl_get_dirty_log_hv(struct kvm *kvm,
4375 struct kvm_dirty_log *log)
4376{
4377 struct kvm_memslots *slots;
4378 struct kvm_memory_slot *memslot;
4379 int i, r;
4380 unsigned long n;
4381 unsigned long *buf, *p;
4382 struct kvm_vcpu *vcpu;
4383
4384 mutex_lock(&kvm->slots_lock);
4385
4386 r = -EINVAL;
4387 if (log->slot >= KVM_USER_MEM_SLOTS)
4388 goto out;
4389
4390 slots = kvm_memslots(kvm);
4391 memslot = id_to_memslot(slots, log->slot);
4392 r = -ENOENT;
4393 if (!memslot->dirty_bitmap)
4394 goto out;
4395
4396
4397
4398
4399
4400 n = kvm_dirty_bitmap_bytes(memslot);
4401 buf = memslot->dirty_bitmap + n / sizeof(long);
4402 memset(buf, 0, n);
4403
4404 if (kvm_is_radix(kvm))
4405 r = kvmppc_hv_get_dirty_log_radix(kvm, memslot, buf);
4406 else
4407 r = kvmppc_hv_get_dirty_log_hpt(kvm, memslot, buf);
4408 if (r)
4409 goto out;
4410
4411
4412
4413
4414
4415
4416
4417 p = memslot->dirty_bitmap;
4418 for (i = 0; i < n / sizeof(long); ++i)
4419 buf[i] |= xchg(&p[i], 0);
4420
4421
4422
4423 kvm_for_each_vcpu(i, vcpu, kvm) {
4424 spin_lock(&vcpu->arch.vpa_update_lock);
4425 kvmppc_harvest_vpa_dirty(&vcpu->arch.vpa, memslot, buf);
4426 kvmppc_harvest_vpa_dirty(&vcpu->arch.dtl, memslot, buf);
4427 spin_unlock(&vcpu->arch.vpa_update_lock);
4428 }
4429
4430 r = -EFAULT;
4431 if (copy_to_user(log->dirty_bitmap, buf, n))
4432 goto out;
4433
4434 r = 0;
4435out:
4436 mutex_unlock(&kvm->slots_lock);
4437 return r;
4438}
4439
4440static void kvmppc_core_free_memslot_hv(struct kvm_memory_slot *free,
4441 struct kvm_memory_slot *dont)
4442{
4443 if (!dont || free->arch.rmap != dont->arch.rmap) {
4444 vfree(free->arch.rmap);
4445 free->arch.rmap = NULL;
4446 }
4447}
4448
4449static int kvmppc_core_create_memslot_hv(struct kvm_memory_slot *slot,
4450 unsigned long npages)
4451{
4452 slot->arch.rmap = vzalloc(array_size(npages, sizeof(*slot->arch.rmap)));
4453 if (!slot->arch.rmap)
4454 return -ENOMEM;
4455
4456 return 0;
4457}
4458
4459static int kvmppc_core_prepare_memory_region_hv(struct kvm *kvm,
4460 struct kvm_memory_slot *memslot,
4461 const struct kvm_userspace_memory_region *mem)
4462{
4463 return 0;
4464}
4465
4466static void kvmppc_core_commit_memory_region_hv(struct kvm *kvm,
4467 const struct kvm_userspace_memory_region *mem,
4468 const struct kvm_memory_slot *old,
4469 const struct kvm_memory_slot *new,
4470 enum kvm_mr_change change)
4471{
4472 unsigned long npages = mem->memory_size >> PAGE_SHIFT;
4473
4474
4475
4476
4477
4478
4479
4480 if (npages)
4481 atomic64_inc(&kvm->arch.mmio_update);
4482
4483
4484
4485
4486
4487
4488
4489
4490
4491
4492
4493
4494
4495
4496 if (change == KVM_MR_FLAGS_ONLY && kvm_is_radix(kvm) &&
4497 ((new->flags ^ old->flags) & KVM_MEM_LOG_DIRTY_PAGES))
4498 kvmppc_radix_flush_memslot(kvm, old);
4499}
4500
4501
4502
4503
4504
4505
4506void kvmppc_update_lpcr(struct kvm *kvm, unsigned long lpcr, unsigned long mask)
4507{
4508 long int i;
4509 u32 cores_done = 0;
4510
4511 if ((kvm->arch.lpcr & mask) == lpcr)
4512 return;
4513
4514 kvm->arch.lpcr = (kvm->arch.lpcr & ~mask) | lpcr;
4515
4516 for (i = 0; i < KVM_MAX_VCORES; ++i) {
4517 struct kvmppc_vcore *vc = kvm->arch.vcores[i];
4518 if (!vc)
4519 continue;
4520 spin_lock(&vc->lock);
4521 vc->lpcr = (vc->lpcr & ~mask) | lpcr;
4522 spin_unlock(&vc->lock);
4523 if (++cores_done >= kvm->arch.online_vcores)
4524 break;
4525 }
4526}
4527
4528static void kvmppc_mmu_destroy_hv(struct kvm_vcpu *vcpu)
4529{
4530 return;
4531}
4532
4533void kvmppc_setup_partition_table(struct kvm *kvm)
4534{
4535 unsigned long dw0, dw1;
4536
4537 if (!kvm_is_radix(kvm)) {
4538
4539 dw0 = ((kvm->arch.vrma_slb_v & SLB_VSID_L) >> 1) |
4540 ((kvm->arch.vrma_slb_v & SLB_VSID_LP) << 1);
4541
4542 dw0 |= kvm->arch.sdr1;
4543
4544
4545 dw1 = kvm->arch.process_table;
4546 } else {
4547 dw0 = PATB_HR | radix__get_tree_size() |
4548 __pa(kvm->arch.pgtable) | RADIX_PGD_INDEX_SIZE;
4549 dw1 = PATB_GR | kvm->arch.process_table;
4550 }
4551 kvmhv_set_ptbl_entry(kvm->arch.lpid, dw0, dw1);
4552}
4553
4554
4555
4556
4557
4558static int kvmppc_hv_setup_htab_rma(struct kvm_vcpu *vcpu)
4559{
4560 int err = 0;
4561 struct kvm *kvm = vcpu->kvm;
4562 unsigned long hva;
4563 struct kvm_memory_slot *memslot;
4564 struct vm_area_struct *vma;
4565 unsigned long lpcr = 0, senc;
4566 unsigned long psize, porder;
4567 int srcu_idx;
4568
4569
4570 if (!kvm->arch.hpt.virt) {
4571 int order = KVM_DEFAULT_HPT_ORDER;
4572 struct kvm_hpt_info info;
4573
4574 err = kvmppc_allocate_hpt(&info, order);
4575
4576
4577
4578 while ((err == -ENOMEM) && --order >= PPC_MIN_HPT_ORDER)
4579 err = kvmppc_allocate_hpt(&info, order);
4580
4581 if (err < 0) {
4582 pr_err("KVM: Couldn't alloc HPT\n");
4583 goto out;
4584 }
4585
4586 kvmppc_set_hpt(kvm, &info);
4587 }
4588
4589
4590 srcu_idx = srcu_read_lock(&kvm->srcu);
4591 memslot = gfn_to_memslot(kvm, 0);
4592
4593
4594 err = -EINVAL;
4595 if (!memslot || (memslot->flags & KVM_MEMSLOT_INVALID))
4596 goto out_srcu;
4597
4598
4599 hva = memslot->userspace_addr;
4600 down_read(¤t->mm->mmap_sem);
4601 vma = find_vma(current->mm, hva);
4602 if (!vma || vma->vm_start > hva || (vma->vm_flags & VM_IO))
4603 goto up_out;
4604
4605 psize = vma_kernel_pagesize(vma);
4606
4607 up_read(¤t->mm->mmap_sem);
4608
4609
4610 if (psize >= 0x1000000)
4611 psize = 0x1000000;
4612 else if (psize >= 0x10000)
4613 psize = 0x10000;
4614 else
4615 psize = 0x1000;
4616 porder = __ilog2(psize);
4617
4618 senc = slb_pgsize_encoding(psize);
4619 kvm->arch.vrma_slb_v = senc | SLB_VSID_B_1T |
4620 (VRMA_VSID << SLB_VSID_SHIFT_1T);
4621
4622 kvmppc_map_vrma(vcpu, memslot, porder);
4623
4624
4625 if (!cpu_has_feature(CPU_FTR_ARCH_300)) {
4626
4627 lpcr = senc << (LPCR_VRMASD_SH - 4);
4628 kvmppc_update_lpcr(kvm, lpcr, LPCR_VRMASD);
4629 }
4630
4631
4632 smp_wmb();
4633 err = 0;
4634 out_srcu:
4635 srcu_read_unlock(&kvm->srcu, srcu_idx);
4636 out:
4637 return err;
4638
4639 up_out:
4640 up_read(¤t->mm->mmap_sem);
4641 goto out_srcu;
4642}
4643
4644
4645
4646
4647
4648int kvmppc_switch_mmu_to_hpt(struct kvm *kvm)
4649{
4650 if (nesting_enabled(kvm))
4651 kvmhv_release_all_nested(kvm);
4652 kvmppc_rmap_reset(kvm);
4653 kvm->arch.process_table = 0;
4654
4655 spin_lock(&kvm->mmu_lock);
4656 kvm->arch.radix = 0;
4657 spin_unlock(&kvm->mmu_lock);
4658 kvmppc_free_radix(kvm);
4659 kvmppc_update_lpcr(kvm, LPCR_VPM1,
4660 LPCR_VPM1 | LPCR_UPRT | LPCR_GTSE | LPCR_HR);
4661 return 0;
4662}
4663
4664
4665
4666
4667
4668int kvmppc_switch_mmu_to_radix(struct kvm *kvm)
4669{
4670 int err;
4671
4672 err = kvmppc_init_vm_radix(kvm);
4673 if (err)
4674 return err;
4675 kvmppc_rmap_reset(kvm);
4676
4677 spin_lock(&kvm->mmu_lock);
4678 kvm->arch.radix = 1;
4679 spin_unlock(&kvm->mmu_lock);
4680 kvmppc_free_hpt(&kvm->arch.hpt);
4681 kvmppc_update_lpcr(kvm, LPCR_UPRT | LPCR_GTSE | LPCR_HR,
4682 LPCR_VPM1 | LPCR_UPRT | LPCR_GTSE | LPCR_HR);
4683 return 0;
4684}
4685
4686#ifdef CONFIG_KVM_XICS
4687
4688
4689
4690
4691
4692
4693
4694
4695
4696
4697void kvmppc_alloc_host_rm_ops(void)
4698{
4699 struct kvmppc_host_rm_ops *ops;
4700 unsigned long l_ops;
4701 int cpu, core;
4702 int size;
4703
4704
4705 if (kvmppc_host_rm_ops_hv != NULL)
4706 return;
4707
4708 ops = kzalloc(sizeof(struct kvmppc_host_rm_ops), GFP_KERNEL);
4709 if (!ops)
4710 return;
4711
4712 size = cpu_nr_cores() * sizeof(struct kvmppc_host_rm_core);
4713 ops->rm_core = kzalloc(size, GFP_KERNEL);
4714
4715 if (!ops->rm_core) {
4716 kfree(ops);
4717 return;
4718 }
4719
4720 cpus_read_lock();
4721
4722 for (cpu = 0; cpu < nr_cpu_ids; cpu += threads_per_core) {
4723 if (!cpu_online(cpu))
4724 continue;
4725
4726 core = cpu >> threads_shift;
4727 ops->rm_core[core].rm_state.in_host = 1;
4728 }
4729
4730 ops->vcpu_kick = kvmppc_fast_vcpu_kick_hv;
4731
4732
4733
4734
4735
4736
4737
4738 smp_wmb();
4739 l_ops = (unsigned long) ops;
4740
4741 if (cmpxchg64((unsigned long *)&kvmppc_host_rm_ops_hv, 0, l_ops)) {
4742 cpus_read_unlock();
4743 kfree(ops->rm_core);
4744 kfree(ops);
4745 return;
4746 }
4747
4748 cpuhp_setup_state_nocalls_cpuslocked(CPUHP_KVM_PPC_BOOK3S_PREPARE,
4749 "ppc/kvm_book3s:prepare",
4750 kvmppc_set_host_core,
4751 kvmppc_clear_host_core);
4752 cpus_read_unlock();
4753}
4754
4755void kvmppc_free_host_rm_ops(void)
4756{
4757 if (kvmppc_host_rm_ops_hv) {
4758 cpuhp_remove_state_nocalls(CPUHP_KVM_PPC_BOOK3S_PREPARE);
4759 kfree(kvmppc_host_rm_ops_hv->rm_core);
4760 kfree(kvmppc_host_rm_ops_hv);
4761 kvmppc_host_rm_ops_hv = NULL;
4762 }
4763}
4764#endif
4765
4766static int kvmppc_core_init_vm_hv(struct kvm *kvm)
4767{
4768 unsigned long lpcr, lpid;
4769 char buf[32];
4770 int ret;
4771
4772 mutex_init(&kvm->arch.mmu_setup_lock);
4773
4774
4775
4776 lpid = kvmppc_alloc_lpid();
4777 if ((long)lpid < 0)
4778 return -ENOMEM;
4779 kvm->arch.lpid = lpid;
4780
4781 kvmppc_alloc_host_rm_ops();
4782
4783 kvmhv_vm_nested_init(kvm);
4784
4785
4786
4787
4788
4789
4790
4791
4792 if (!cpu_has_feature(CPU_FTR_ARCH_300))
4793 cpumask_setall(&kvm->arch.need_tlb_flush);
4794
4795
4796 memcpy(kvm->arch.enabled_hcalls, default_enabled_hcalls,
4797 sizeof(kvm->arch.enabled_hcalls));
4798
4799 if (!cpu_has_feature(CPU_FTR_ARCH_300))
4800 kvm->arch.host_sdr1 = mfspr(SPRN_SDR1);
4801
4802
4803 if (cpu_has_feature(CPU_FTR_HVMODE)) {
4804 kvm->arch.host_lpid = mfspr(SPRN_LPID);
4805 kvm->arch.host_lpcr = lpcr = mfspr(SPRN_LPCR);
4806 lpcr &= LPCR_PECE | LPCR_LPES;
4807 } else {
4808 lpcr = 0;
4809 }
4810 lpcr |= (4UL << LPCR_DPFD_SH) | LPCR_HDICE |
4811 LPCR_VPM0 | LPCR_VPM1;
4812 kvm->arch.vrma_slb_v = SLB_VSID_B_1T |
4813 (VRMA_VSID << SLB_VSID_SHIFT_1T);
4814
4815 if (cpu_has_feature(CPU_FTR_ARCH_207S))
4816 lpcr |= LPCR_ONL;
4817
4818
4819
4820
4821
4822
4823
4824 if (cpu_has_feature(CPU_FTR_ARCH_300)) {
4825 lpcr &= ~LPCR_VPM0;
4826 lpcr |= LPCR_HVICE | LPCR_HEIC;
4827
4828
4829
4830
4831
4832 if (xics_on_xive())
4833 lpcr |= LPCR_LPES;
4834 }
4835
4836
4837
4838
4839 if (radix_enabled()) {
4840 kvm->arch.radix = 1;
4841 kvm->arch.mmu_ready = 1;
4842 lpcr &= ~LPCR_VPM1;
4843 lpcr |= LPCR_UPRT | LPCR_GTSE | LPCR_HR;
4844 ret = kvmppc_init_vm_radix(kvm);
4845 if (ret) {
4846 kvmppc_free_lpid(kvm->arch.lpid);
4847 return ret;
4848 }
4849 kvmppc_setup_partition_table(kvm);
4850 }
4851
4852 kvm->arch.lpcr = lpcr;
4853
4854
4855 kvm->arch.resize_hpt = NULL;
4856
4857
4858
4859
4860
4861 if (radix_enabled())
4862 kvm->arch.tlb_sets = POWER9_TLB_SETS_RADIX;
4863 else if (cpu_has_feature(CPU_FTR_ARCH_300))
4864 kvm->arch.tlb_sets = POWER9_TLB_SETS_HASH;
4865 else if (cpu_has_feature(CPU_FTR_ARCH_207S))
4866 kvm->arch.tlb_sets = POWER8_TLB_SETS;
4867 else
4868 kvm->arch.tlb_sets = POWER7_TLB_SETS;
4869
4870
4871
4872
4873
4874
4875
4876 if (cpu_has_feature(CPU_FTR_ARCH_300)) {
4877 if (!indep_threads_mode && !cpu_has_feature(CPU_FTR_HVMODE)) {
4878 pr_warn("KVM: Ignoring indep_threads_mode=N in nested hypervisor\n");
4879 kvm->arch.threads_indep = true;
4880 } else {
4881 kvm->arch.threads_indep = indep_threads_mode;
4882 }
4883 }
4884 if (!kvm->arch.threads_indep)
4885 kvm_hv_vm_activated();
4886
4887
4888
4889
4890
4891
4892
4893
4894 if (!cpu_has_feature(CPU_FTR_ARCH_300))
4895 kvm->arch.smt_mode = threads_per_subcore;
4896 else
4897 kvm->arch.smt_mode = 1;
4898 kvm->arch.emul_smt_mode = 1;
4899
4900
4901
4902
4903 snprintf(buf, sizeof(buf), "vm%d", current->pid);
4904 kvm->arch.debugfs_dir = debugfs_create_dir(buf, kvm_debugfs_dir);
4905 kvmppc_mmu_debugfs_init(kvm);
4906 if (radix_enabled())
4907 kvmhv_radix_debugfs_init(kvm);
4908
4909 return 0;
4910}
4911
4912static void kvmppc_free_vcores(struct kvm *kvm)
4913{
4914 long int i;
4915
4916 for (i = 0; i < KVM_MAX_VCORES; ++i)
4917 kfree(kvm->arch.vcores[i]);
4918 kvm->arch.online_vcores = 0;
4919}
4920
4921static void kvmppc_core_destroy_vm_hv(struct kvm *kvm)
4922{
4923 debugfs_remove_recursive(kvm->arch.debugfs_dir);
4924
4925 if (!kvm->arch.threads_indep)
4926 kvm_hv_vm_deactivated();
4927
4928 kvmppc_free_vcores(kvm);
4929
4930
4931 if (kvm_is_radix(kvm))
4932 kvmppc_free_radix(kvm);
4933 else
4934 kvmppc_free_hpt(&kvm->arch.hpt);
4935
4936
4937 if (cpu_has_feature(CPU_FTR_ARCH_300)) {
4938 if (nesting_enabled(kvm))
4939 kvmhv_release_all_nested(kvm);
4940 kvm->arch.process_table = 0;
4941 kvmhv_set_ptbl_entry(kvm->arch.lpid, 0, 0);
4942 }
4943 kvmppc_free_lpid(kvm->arch.lpid);
4944
4945 kvmppc_free_pimap(kvm);
4946}
4947
4948
4949static int kvmppc_core_emulate_op_hv(struct kvm_run *run, struct kvm_vcpu *vcpu,
4950 unsigned int inst, int *advance)
4951{
4952 return EMULATE_FAIL;
4953}
4954
4955static int kvmppc_core_emulate_mtspr_hv(struct kvm_vcpu *vcpu, int sprn,
4956 ulong spr_val)
4957{
4958 return EMULATE_FAIL;
4959}
4960
4961static int kvmppc_core_emulate_mfspr_hv(struct kvm_vcpu *vcpu, int sprn,
4962 ulong *spr_val)
4963{
4964 return EMULATE_FAIL;
4965}
4966
4967static int kvmppc_core_check_processor_compat_hv(void)
4968{
4969 if (cpu_has_feature(CPU_FTR_HVMODE) &&
4970 cpu_has_feature(CPU_FTR_ARCH_206))
4971 return 0;
4972
4973
4974 if (cpu_has_feature(CPU_FTR_ARCH_300) && radix_enabled())
4975 return 0;
4976
4977 return -EIO;
4978}
4979
4980#ifdef CONFIG_KVM_XICS
4981
4982void kvmppc_free_pimap(struct kvm *kvm)
4983{
4984 kfree(kvm->arch.pimap);
4985}
4986
4987static struct kvmppc_passthru_irqmap *kvmppc_alloc_pimap(void)
4988{
4989 return kzalloc(sizeof(struct kvmppc_passthru_irqmap), GFP_KERNEL);
4990}
4991
4992static int kvmppc_set_passthru_irq(struct kvm *kvm, int host_irq, int guest_gsi)
4993{
4994 struct irq_desc *desc;
4995 struct kvmppc_irq_map *irq_map;
4996 struct kvmppc_passthru_irqmap *pimap;
4997 struct irq_chip *chip;
4998 int i, rc = 0;
4999
5000 if (!kvm_irq_bypass)
5001 return 1;
5002
5003 desc = irq_to_desc(host_irq);
5004 if (!desc)
5005 return -EIO;
5006
5007 mutex_lock(&kvm->lock);
5008
5009 pimap = kvm->arch.pimap;
5010 if (pimap == NULL) {
5011
5012 pimap = kvmppc_alloc_pimap();
5013 if (pimap == NULL) {
5014 mutex_unlock(&kvm->lock);
5015 return -ENOMEM;
5016 }
5017 kvm->arch.pimap = pimap;
5018 }
5019
5020
5021
5022
5023
5024
5025 chip = irq_data_get_irq_chip(&desc->irq_data);
5026 if (!chip || !(is_pnv_opal_msi(chip) || is_xive_irq(chip))) {
5027 pr_warn("kvmppc_set_passthru_irq_hv: Could not assign IRQ map for (%d,%d)\n",
5028 host_irq, guest_gsi);
5029 mutex_unlock(&kvm->lock);
5030 return -ENOENT;
5031 }
5032
5033
5034
5035
5036
5037
5038 for (i = 0; i < pimap->n_mapped; i++) {
5039 if (guest_gsi == pimap->mapped[i].v_hwirq) {
5040 if (pimap->mapped[i].r_hwirq) {
5041 mutex_unlock(&kvm->lock);
5042 return -EINVAL;
5043 }
5044 break;
5045 }
5046 }
5047
5048 if (i == KVMPPC_PIRQ_MAPPED) {
5049 mutex_unlock(&kvm->lock);
5050 return -EAGAIN;
5051 }
5052
5053 irq_map = &pimap->mapped[i];
5054
5055 irq_map->v_hwirq = guest_gsi;
5056 irq_map->desc = desc;
5057
5058
5059
5060
5061
5062 smp_wmb();
5063 irq_map->r_hwirq = desc->irq_data.hwirq;
5064
5065 if (i == pimap->n_mapped)
5066 pimap->n_mapped++;
5067
5068 if (xics_on_xive())
5069 rc = kvmppc_xive_set_mapped(kvm, guest_gsi, desc);
5070 else
5071 kvmppc_xics_set_mapped(kvm, guest_gsi, desc->irq_data.hwirq);
5072 if (rc)
5073 irq_map->r_hwirq = 0;
5074
5075 mutex_unlock(&kvm->lock);
5076
5077 return 0;
5078}
5079
5080static int kvmppc_clr_passthru_irq(struct kvm *kvm, int host_irq, int guest_gsi)
5081{
5082 struct irq_desc *desc;
5083 struct kvmppc_passthru_irqmap *pimap;
5084 int i, rc = 0;
5085
5086 if (!kvm_irq_bypass)
5087 return 0;
5088
5089 desc = irq_to_desc(host_irq);
5090 if (!desc)
5091 return -EIO;
5092
5093 mutex_lock(&kvm->lock);
5094 if (!kvm->arch.pimap)
5095 goto unlock;
5096
5097 pimap = kvm->arch.pimap;
5098
5099 for (i = 0; i < pimap->n_mapped; i++) {
5100 if (guest_gsi == pimap->mapped[i].v_hwirq)
5101 break;
5102 }
5103
5104 if (i == pimap->n_mapped) {
5105 mutex_unlock(&kvm->lock);
5106 return -ENODEV;
5107 }
5108
5109 if (xics_on_xive())
5110 rc = kvmppc_xive_clr_mapped(kvm, guest_gsi, pimap->mapped[i].desc);
5111 else
5112 kvmppc_xics_clr_mapped(kvm, guest_gsi, pimap->mapped[i].r_hwirq);
5113
5114
5115 pimap->mapped[i].r_hwirq = 0;
5116
5117
5118
5119
5120
5121 unlock:
5122 mutex_unlock(&kvm->lock);
5123 return rc;
5124}
5125
5126static int kvmppc_irq_bypass_add_producer_hv(struct irq_bypass_consumer *cons,
5127 struct irq_bypass_producer *prod)
5128{
5129 int ret = 0;
5130 struct kvm_kernel_irqfd *irqfd =
5131 container_of(cons, struct kvm_kernel_irqfd, consumer);
5132
5133 irqfd->producer = prod;
5134
5135 ret = kvmppc_set_passthru_irq(irqfd->kvm, prod->irq, irqfd->gsi);
5136 if (ret)
5137 pr_info("kvmppc_set_passthru_irq (irq %d, gsi %d) fails: %d\n",
5138 prod->irq, irqfd->gsi, ret);
5139
5140 return ret;
5141}
5142
5143static void kvmppc_irq_bypass_del_producer_hv(struct irq_bypass_consumer *cons,
5144 struct irq_bypass_producer *prod)
5145{
5146 int ret;
5147 struct kvm_kernel_irqfd *irqfd =
5148 container_of(cons, struct kvm_kernel_irqfd, consumer);
5149
5150 irqfd->producer = NULL;
5151
5152
5153
5154
5155
5156
5157 ret = kvmppc_clr_passthru_irq(irqfd->kvm, prod->irq, irqfd->gsi);
5158 if (ret)
5159 pr_warn("kvmppc_clr_passthru_irq (irq %d, gsi %d) fails: %d\n",
5160 prod->irq, irqfd->gsi, ret);
5161}
5162#endif
5163
5164static long kvm_arch_vm_ioctl_hv(struct file *filp,
5165 unsigned int ioctl, unsigned long arg)
5166{
5167 struct kvm *kvm __maybe_unused = filp->private_data;
5168 void __user *argp = (void __user *)arg;
5169 long r;
5170
5171 switch (ioctl) {
5172
5173 case KVM_PPC_ALLOCATE_HTAB: {
5174 u32 htab_order;
5175
5176 r = -EFAULT;
5177 if (get_user(htab_order, (u32 __user *)argp))
5178 break;
5179 r = kvmppc_alloc_reset_hpt(kvm, htab_order);
5180 if (r)
5181 break;
5182 r = 0;
5183 break;
5184 }
5185
5186 case KVM_PPC_GET_HTAB_FD: {
5187 struct kvm_get_htab_fd ghf;
5188
5189 r = -EFAULT;
5190 if (copy_from_user(&ghf, argp, sizeof(ghf)))
5191 break;
5192 r = kvm_vm_ioctl_get_htab_fd(kvm, &ghf);
5193 break;
5194 }
5195
5196 case KVM_PPC_RESIZE_HPT_PREPARE: {
5197 struct kvm_ppc_resize_hpt rhpt;
5198
5199 r = -EFAULT;
5200 if (copy_from_user(&rhpt, argp, sizeof(rhpt)))
5201 break;
5202
5203 r = kvm_vm_ioctl_resize_hpt_prepare(kvm, &rhpt);
5204 break;
5205 }
5206
5207 case KVM_PPC_RESIZE_HPT_COMMIT: {
5208 struct kvm_ppc_resize_hpt rhpt;
5209
5210 r = -EFAULT;
5211 if (copy_from_user(&rhpt, argp, sizeof(rhpt)))
5212 break;
5213
5214 r = kvm_vm_ioctl_resize_hpt_commit(kvm, &rhpt);
5215 break;
5216 }
5217
5218 default:
5219 r = -ENOTTY;
5220 }
5221
5222 return r;
5223}
5224
5225
5226
5227
5228
5229
5230
5231static unsigned int default_hcall_list[] = {
5232 H_REMOVE,
5233 H_ENTER,
5234 H_READ,
5235 H_PROTECT,
5236 H_BULK_REMOVE,
5237 H_GET_TCE,
5238 H_PUT_TCE,
5239 H_SET_DABR,
5240 H_SET_XDABR,
5241 H_CEDE,
5242 H_PROD,
5243 H_CONFER,
5244 H_REGISTER_VPA,
5245#ifdef CONFIG_KVM_XICS
5246 H_EOI,
5247 H_CPPR,
5248 H_IPI,
5249 H_IPOLL,
5250 H_XIRR,
5251 H_XIRR_X,
5252#endif
5253 0
5254};
5255
5256static void init_default_hcalls(void)
5257{
5258 int i;
5259 unsigned int hcall;
5260
5261 for (i = 0; default_hcall_list[i]; ++i) {
5262 hcall = default_hcall_list[i];
5263 WARN_ON(!kvmppc_hcall_impl_hv(hcall));
5264 __set_bit(hcall / 4, default_enabled_hcalls);
5265 }
5266}
5267
5268static int kvmhv_configure_mmu(struct kvm *kvm, struct kvm_ppc_mmuv3_cfg *cfg)
5269{
5270 unsigned long lpcr;
5271 int radix;
5272 int err;
5273
5274
5275 if (!cpu_has_feature(CPU_FTR_ARCH_300))
5276 return -ENODEV;
5277
5278
5279 if (cfg->flags & ~(KVM_PPC_MMUV3_RADIX | KVM_PPC_MMUV3_GTSE))
5280 return -EINVAL;
5281
5282
5283 radix = !!(cfg->flags & KVM_PPC_MMUV3_RADIX);
5284 if (!!(cfg->process_table & PATB_GR) != radix)
5285 return -EINVAL;
5286
5287
5288 if ((cfg->process_table & PRTS_MASK) > 24)
5289 return -EINVAL;
5290
5291
5292 if (radix && !radix_enabled())
5293 return -EINVAL;
5294
5295
5296 if (kvmhv_on_pseries() && !radix)
5297 return -EINVAL;
5298
5299 mutex_lock(&kvm->arch.mmu_setup_lock);
5300 if (radix != kvm_is_radix(kvm)) {
5301 if (kvm->arch.mmu_ready) {
5302 kvm->arch.mmu_ready = 0;
5303
5304 smp_mb();
5305 if (atomic_read(&kvm->arch.vcpus_running)) {
5306 kvm->arch.mmu_ready = 1;
5307 err = -EBUSY;
5308 goto out_unlock;
5309 }
5310 }
5311 if (radix)
5312 err = kvmppc_switch_mmu_to_radix(kvm);
5313 else
5314 err = kvmppc_switch_mmu_to_hpt(kvm);
5315 if (err)
5316 goto out_unlock;
5317 }
5318
5319 kvm->arch.process_table = cfg->process_table;
5320 kvmppc_setup_partition_table(kvm);
5321
5322 lpcr = (cfg->flags & KVM_PPC_MMUV3_GTSE) ? LPCR_GTSE : 0;
5323 kvmppc_update_lpcr(kvm, lpcr, LPCR_GTSE);
5324 err = 0;
5325
5326 out_unlock:
5327 mutex_unlock(&kvm->arch.mmu_setup_lock);
5328 return err;
5329}
5330
5331static int kvmhv_enable_nested(struct kvm *kvm)
5332{
5333 if (!nested)
5334 return -EPERM;
5335 if (!cpu_has_feature(CPU_FTR_ARCH_300) || no_mixing_hpt_and_radix)
5336 return -ENODEV;
5337
5338
5339 if (kvm)
5340 kvm->arch.nested_enable = true;
5341 return 0;
5342}
5343
5344static int kvmhv_load_from_eaddr(struct kvm_vcpu *vcpu, ulong *eaddr, void *ptr,
5345 int size)
5346{
5347 int rc = -EINVAL;
5348
5349 if (kvmhv_vcpu_is_radix(vcpu)) {
5350 rc = kvmhv_copy_from_guest_radix(vcpu, *eaddr, ptr, size);
5351
5352 if (rc > 0)
5353 rc = -EINVAL;
5354 }
5355
5356
5357 if (rc && vcpu->arch.nested)
5358 rc = -EAGAIN;
5359
5360 return rc;
5361}
5362
5363static int kvmhv_store_to_eaddr(struct kvm_vcpu *vcpu, ulong *eaddr, void *ptr,
5364 int size)
5365{
5366 int rc = -EINVAL;
5367
5368 if (kvmhv_vcpu_is_radix(vcpu)) {
5369 rc = kvmhv_copy_to_guest_radix(vcpu, *eaddr, ptr, size);
5370
5371 if (rc > 0)
5372 rc = -EINVAL;
5373 }
5374
5375
5376 if (rc && vcpu->arch.nested)
5377 rc = -EAGAIN;
5378
5379 return rc;
5380}
5381
5382static struct kvmppc_ops kvm_ops_hv = {
5383 .get_sregs = kvm_arch_vcpu_ioctl_get_sregs_hv,
5384 .set_sregs = kvm_arch_vcpu_ioctl_set_sregs_hv,
5385 .get_one_reg = kvmppc_get_one_reg_hv,
5386 .set_one_reg = kvmppc_set_one_reg_hv,
5387 .vcpu_load = kvmppc_core_vcpu_load_hv,
5388 .vcpu_put = kvmppc_core_vcpu_put_hv,
5389 .set_msr = kvmppc_set_msr_hv,
5390 .vcpu_run = kvmppc_vcpu_run_hv,
5391 .vcpu_create = kvmppc_core_vcpu_create_hv,
5392 .vcpu_free = kvmppc_core_vcpu_free_hv,
5393 .check_requests = kvmppc_core_check_requests_hv,
5394 .get_dirty_log = kvm_vm_ioctl_get_dirty_log_hv,
5395 .flush_memslot = kvmppc_core_flush_memslot_hv,
5396 .prepare_memory_region = kvmppc_core_prepare_memory_region_hv,
5397 .commit_memory_region = kvmppc_core_commit_memory_region_hv,
5398 .unmap_hva_range = kvm_unmap_hva_range_hv,
5399 .age_hva = kvm_age_hva_hv,
5400 .test_age_hva = kvm_test_age_hva_hv,
5401 .set_spte_hva = kvm_set_spte_hva_hv,
5402 .mmu_destroy = kvmppc_mmu_destroy_hv,
5403 .free_memslot = kvmppc_core_free_memslot_hv,
5404 .create_memslot = kvmppc_core_create_memslot_hv,
5405 .init_vm = kvmppc_core_init_vm_hv,
5406 .destroy_vm = kvmppc_core_destroy_vm_hv,
5407 .get_smmu_info = kvm_vm_ioctl_get_smmu_info_hv,
5408 .emulate_op = kvmppc_core_emulate_op_hv,
5409 .emulate_mtspr = kvmppc_core_emulate_mtspr_hv,
5410 .emulate_mfspr = kvmppc_core_emulate_mfspr_hv,
5411 .fast_vcpu_kick = kvmppc_fast_vcpu_kick_hv,
5412 .arch_vm_ioctl = kvm_arch_vm_ioctl_hv,
5413 .hcall_implemented = kvmppc_hcall_impl_hv,
5414#ifdef CONFIG_KVM_XICS
5415 .irq_bypass_add_producer = kvmppc_irq_bypass_add_producer_hv,
5416 .irq_bypass_del_producer = kvmppc_irq_bypass_del_producer_hv,
5417#endif
5418 .configure_mmu = kvmhv_configure_mmu,
5419 .get_rmmu_info = kvmhv_get_rmmu_info,
5420 .set_smt_mode = kvmhv_set_smt_mode,
5421 .enable_nested = kvmhv_enable_nested,
5422 .load_from_eaddr = kvmhv_load_from_eaddr,
5423 .store_to_eaddr = kvmhv_store_to_eaddr,
5424};
5425
5426static int kvm_init_subcore_bitmap(void)
5427{
5428 int i, j;
5429 int nr_cores = cpu_nr_cores();
5430 struct sibling_subcore_state *sibling_subcore_state;
5431
5432 for (i = 0; i < nr_cores; i++) {
5433 int first_cpu = i * threads_per_core;
5434 int node = cpu_to_node(first_cpu);
5435
5436
5437 if (paca_ptrs[first_cpu]->sibling_subcore_state)
5438 continue;
5439
5440 sibling_subcore_state =
5441 kzalloc_node(sizeof(struct sibling_subcore_state),
5442 GFP_KERNEL, node);
5443 if (!sibling_subcore_state)
5444 return -ENOMEM;
5445
5446
5447 for (j = 0; j < threads_per_core; j++) {
5448 int cpu = first_cpu + j;
5449
5450 paca_ptrs[cpu]->sibling_subcore_state =
5451 sibling_subcore_state;
5452 }
5453 }
5454 return 0;
5455}
5456
5457static int kvmppc_radix_possible(void)
5458{
5459 return cpu_has_feature(CPU_FTR_ARCH_300) && radix_enabled();
5460}
5461
5462static int kvmppc_book3s_init_hv(void)
5463{
5464 int r;
5465
5466
5467
5468 r = kvmppc_core_check_processor_compat_hv();
5469 if (r < 0)
5470 return -ENODEV;
5471
5472 r = kvmhv_nested_init();
5473 if (r)
5474 return r;
5475
5476 r = kvm_init_subcore_bitmap();
5477 if (r)
5478 return r;
5479
5480
5481
5482
5483
5484
5485#ifdef CONFIG_SMP
5486 if (!xics_on_xive() && !kvmhv_on_pseries() &&
5487 !local_paca->kvm_hstate.xics_phys) {
5488 struct device_node *np;
5489
5490 np = of_find_compatible_node(NULL, NULL, "ibm,opal-intc");
5491 if (!np) {
5492 pr_err("KVM-HV: Cannot determine method for accessing XICS\n");
5493 return -ENODEV;
5494 }
5495
5496 of_node_put(np);
5497 }
5498#endif
5499
5500 kvm_ops_hv.owner = THIS_MODULE;
5501 kvmppc_hv_ops = &kvm_ops_hv;
5502
5503 init_default_hcalls();
5504
5505 init_vcore_lists();
5506
5507 r = kvmppc_mmu_hv_init();
5508 if (r)
5509 return r;
5510
5511 if (kvmppc_radix_possible())
5512 r = kvmppc_radix_init();
5513
5514
5515
5516
5517
5518 if (cpu_has_feature(CPU_FTR_ARCH_300)) {
5519 unsigned int pvr = mfspr(SPRN_PVR);
5520 if ((pvr >> 16) == PVR_POWER9 &&
5521 (((pvr & 0xe000) == 0 && (pvr & 0xfff) < 0x202) ||
5522 ((pvr & 0xe000) == 0x2000 && (pvr & 0xfff) < 0x101)))
5523 no_mixing_hpt_and_radix = true;
5524 }
5525
5526 return r;
5527}
5528
5529static void kvmppc_book3s_exit_hv(void)
5530{
5531 kvmppc_free_host_rm_ops();
5532 if (kvmppc_radix_possible())
5533 kvmppc_radix_exit();
5534 kvmppc_hv_ops = NULL;
5535 kvmhv_nested_exit();
5536}
5537
5538module_init(kvmppc_book3s_init_hv);
5539module_exit(kvmppc_book3s_exit_hv);
5540MODULE_LICENSE("GPL");
5541MODULE_ALIAS_MISCDEV(KVM_MINOR);
5542MODULE_ALIAS("devname:kvm");
5543