1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18#include <linux/kvm_host.h>
19#include <linux/kernel.h>
20#include <linux/err.h>
21#include <linux/slab.h>
22#include <linux/preempt.h>
23#include <linux/sched/signal.h>
24#include <linux/sched/stat.h>
25#include <linux/delay.h>
26#include <linux/export.h>
27#include <linux/fs.h>
28#include <linux/anon_inodes.h>
29#include <linux/cpu.h>
30#include <linux/cpumask.h>
31#include <linux/spinlock.h>
32#include <linux/page-flags.h>
33#include <linux/srcu.h>
34#include <linux/miscdevice.h>
35#include <linux/debugfs.h>
36#include <linux/gfp.h>
37#include <linux/vmalloc.h>
38#include <linux/highmem.h>
39#include <linux/hugetlb.h>
40#include <linux/kvm_irqfd.h>
41#include <linux/irqbypass.h>
42#include <linux/module.h>
43#include <linux/compiler.h>
44#include <linux/of.h>
45
46#include <asm/ftrace.h>
47#include <asm/reg.h>
48#include <asm/ppc-opcode.h>
49#include <asm/asm-prototypes.h>
50#include <asm/archrandom.h>
51#include <asm/debug.h>
52#include <asm/disassemble.h>
53#include <asm/cputable.h>
54#include <asm/cacheflush.h>
55#include <linux/uaccess.h>
56#include <asm/interrupt.h>
57#include <asm/io.h>
58#include <asm/kvm_ppc.h>
59#include <asm/kvm_book3s.h>
60#include <asm/mmu_context.h>
61#include <asm/lppaca.h>
62#include <asm/pmc.h>
63#include <asm/processor.h>
64#include <asm/cputhreads.h>
65#include <asm/page.h>
66#include <asm/hvcall.h>
67#include <asm/switch_to.h>
68#include <asm/smp.h>
69#include <asm/dbell.h>
70#include <asm/hmi.h>
71#include <asm/pnv-pci.h>
72#include <asm/mmu.h>
73#include <asm/opal.h>
74#include <asm/xics.h>
75#include <asm/xive.h>
76#include <asm/hw_breakpoint.h>
77#include <asm/kvm_book3s_uvmem.h>
78#include <asm/ultravisor.h>
79#include <asm/dtl.h>
80#include <asm/plpar_wrappers.h>
81
82#include "book3s.h"
83
84#define CREATE_TRACE_POINTS
85#include "trace_hv.h"
86
87
88
89
90
91
92#define RESUME_PAGE_FAULT (RESUME_GUEST | RESUME_FLAG_ARCH1)
93
94#define RESUME_PASSTHROUGH (RESUME_GUEST | RESUME_FLAG_ARCH2)
95
96
97#define TB_NIL (~(u64)0)
98
99static DECLARE_BITMAP(default_enabled_hcalls, MAX_HCALL_OPCODE/4 + 1);
100
101static int dynamic_mt_modes = 6;
102module_param(dynamic_mt_modes, int, 0644);
103MODULE_PARM_DESC(dynamic_mt_modes, "Set of allowed dynamic micro-threading modes: 0 (= none), 2, 4, or 6 (= 2 or 4)");
104static int target_smt_mode;
105module_param(target_smt_mode, int, 0644);
106MODULE_PARM_DESC(target_smt_mode, "Target threads per core (0 = max)");
107
108static bool one_vm_per_core;
109module_param(one_vm_per_core, bool, S_IRUGO | S_IWUSR);
110MODULE_PARM_DESC(one_vm_per_core, "Only run vCPUs from the same VM on a core (requires POWER8 or older)");
111
112#ifdef CONFIG_KVM_XICS
113static const struct kernel_param_ops module_param_ops = {
114 .set = param_set_int,
115 .get = param_get_int,
116};
117
118module_param_cb(kvm_irq_bypass, &module_param_ops, &kvm_irq_bypass, 0644);
119MODULE_PARM_DESC(kvm_irq_bypass, "Bypass passthrough interrupt optimization");
120
121module_param_cb(h_ipi_redirect, &module_param_ops, &h_ipi_redirect, 0644);
122MODULE_PARM_DESC(h_ipi_redirect, "Redirect H_IPI wakeup to a free host core");
123#endif
124
125
126static bool nested = true;
127module_param(nested, bool, S_IRUGO | S_IWUSR);
128MODULE_PARM_DESC(nested, "Enable nested virtualization (only on POWER9)");
129
130static inline bool nesting_enabled(struct kvm *kvm)
131{
132 return kvm->arch.nested_enable && kvm_is_radix(kvm);
133}
134
135static int kvmppc_hv_setup_htab_rma(struct kvm_vcpu *vcpu);
136
137
138
139
140
141
142#define RWMR_RPA_P8_1THREAD 0x164520C62609AECAUL
143#define RWMR_RPA_P8_2THREAD 0x7FFF2908450D8DA9UL
144#define RWMR_RPA_P8_3THREAD 0x164520C62609AECAUL
145#define RWMR_RPA_P8_4THREAD 0x199A421245058DA9UL
146#define RWMR_RPA_P8_5THREAD 0x164520C62609AECAUL
147#define RWMR_RPA_P8_6THREAD 0x164520C62609AECAUL
148#define RWMR_RPA_P8_7THREAD 0x164520C62609AECAUL
149#define RWMR_RPA_P8_8THREAD 0x164520C62609AECAUL
150
151static unsigned long p8_rwmr_values[MAX_SMT_THREADS + 1] = {
152 RWMR_RPA_P8_1THREAD,
153 RWMR_RPA_P8_1THREAD,
154 RWMR_RPA_P8_2THREAD,
155 RWMR_RPA_P8_3THREAD,
156 RWMR_RPA_P8_4THREAD,
157 RWMR_RPA_P8_5THREAD,
158 RWMR_RPA_P8_6THREAD,
159 RWMR_RPA_P8_7THREAD,
160 RWMR_RPA_P8_8THREAD,
161};
162
163static inline struct kvm_vcpu *next_runnable_thread(struct kvmppc_vcore *vc,
164 int *ip)
165{
166 int i = *ip;
167 struct kvm_vcpu *vcpu;
168
169 while (++i < MAX_SMT_THREADS) {
170 vcpu = READ_ONCE(vc->runnable_threads[i]);
171 if (vcpu) {
172 *ip = i;
173 return vcpu;
174 }
175 }
176 return NULL;
177}
178
179
180#define for_each_runnable_thread(i, vcpu, vc) \
181 for (i = -1; (vcpu = next_runnable_thread(vc, &i)); )
182
183static bool kvmppc_ipi_thread(int cpu)
184{
185 unsigned long msg = PPC_DBELL_TYPE(PPC_DBELL_SERVER);
186
187
188 if (kvmhv_on_pseries())
189 return false;
190
191
192 if (cpu_has_feature(CPU_FTR_ARCH_300)) {
193 msg |= get_hard_smp_processor_id(cpu);
194 smp_mb();
195 __asm__ __volatile__ (PPC_MSGSND(%0) : : "r" (msg));
196 return true;
197 }
198
199
200 if (cpu_has_feature(CPU_FTR_ARCH_207S)) {
201 preempt_disable();
202 if (cpu_first_thread_sibling(cpu) ==
203 cpu_first_thread_sibling(smp_processor_id())) {
204 msg |= cpu_thread_in_core(cpu);
205 smp_mb();
206 __asm__ __volatile__ (PPC_MSGSND(%0) : : "r" (msg));
207 preempt_enable();
208 return true;
209 }
210 preempt_enable();
211 }
212
213#if defined(CONFIG_PPC_ICP_NATIVE) && defined(CONFIG_SMP)
214 if (cpu >= 0 && cpu < nr_cpu_ids) {
215 if (paca_ptrs[cpu]->kvm_hstate.xics_phys) {
216 xics_wake_cpu(cpu);
217 return true;
218 }
219 opal_int_set_mfrr(get_hard_smp_processor_id(cpu), IPI_PRIORITY);
220 return true;
221 }
222#endif
223
224 return false;
225}
226
227static void kvmppc_fast_vcpu_kick_hv(struct kvm_vcpu *vcpu)
228{
229 int cpu;
230 struct rcuwait *waitp;
231
232 waitp = kvm_arch_vcpu_get_wait(vcpu);
233 if (rcuwait_wake_up(waitp))
234 ++vcpu->stat.generic.halt_wakeup;
235
236 cpu = READ_ONCE(vcpu->arch.thread_cpu);
237 if (cpu >= 0 && kvmppc_ipi_thread(cpu))
238 return;
239
240
241 cpu = vcpu->cpu;
242 if (cpu >= 0 && cpu < nr_cpu_ids && cpu_online(cpu))
243 smp_send_reschedule(cpu);
244}
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279static void kvmppc_core_start_stolen(struct kvmppc_vcore *vc)
280{
281 unsigned long flags;
282
283 spin_lock_irqsave(&vc->stoltb_lock, flags);
284 vc->preempt_tb = mftb();
285 spin_unlock_irqrestore(&vc->stoltb_lock, flags);
286}
287
288static void kvmppc_core_end_stolen(struct kvmppc_vcore *vc)
289{
290 unsigned long flags;
291
292 spin_lock_irqsave(&vc->stoltb_lock, flags);
293 if (vc->preempt_tb != TB_NIL) {
294 vc->stolen_tb += mftb() - vc->preempt_tb;
295 vc->preempt_tb = TB_NIL;
296 }
297 spin_unlock_irqrestore(&vc->stoltb_lock, flags);
298}
299
300static void kvmppc_core_vcpu_load_hv(struct kvm_vcpu *vcpu, int cpu)
301{
302 struct kvmppc_vcore *vc = vcpu->arch.vcore;
303 unsigned long flags;
304
305
306
307
308
309
310
311 if (vc->runner == vcpu && vc->vcore_state >= VCORE_SLEEPING)
312 kvmppc_core_end_stolen(vc);
313
314 spin_lock_irqsave(&vcpu->arch.tbacct_lock, flags);
315 if (vcpu->arch.state == KVMPPC_VCPU_BUSY_IN_HOST &&
316 vcpu->arch.busy_preempt != TB_NIL) {
317 vcpu->arch.busy_stolen += mftb() - vcpu->arch.busy_preempt;
318 vcpu->arch.busy_preempt = TB_NIL;
319 }
320 spin_unlock_irqrestore(&vcpu->arch.tbacct_lock, flags);
321}
322
323static void kvmppc_core_vcpu_put_hv(struct kvm_vcpu *vcpu)
324{
325 struct kvmppc_vcore *vc = vcpu->arch.vcore;
326 unsigned long flags;
327
328 if (vc->runner == vcpu && vc->vcore_state >= VCORE_SLEEPING)
329 kvmppc_core_start_stolen(vc);
330
331 spin_lock_irqsave(&vcpu->arch.tbacct_lock, flags);
332 if (vcpu->arch.state == KVMPPC_VCPU_BUSY_IN_HOST)
333 vcpu->arch.busy_preempt = mftb();
334 spin_unlock_irqrestore(&vcpu->arch.tbacct_lock, flags);
335}
336
337static void kvmppc_set_pvr_hv(struct kvm_vcpu *vcpu, u32 pvr)
338{
339 vcpu->arch.pvr = pvr;
340}
341
342
343#define PCR_ARCH_31 (PCR_ARCH_300 << 1)
344
345static int kvmppc_set_arch_compat(struct kvm_vcpu *vcpu, u32 arch_compat)
346{
347 unsigned long host_pcr_bit = 0, guest_pcr_bit = 0;
348 struct kvmppc_vcore *vc = vcpu->arch.vcore;
349
350
351 if (cpu_has_feature(CPU_FTR_ARCH_31))
352 host_pcr_bit = PCR_ARCH_31;
353 else if (cpu_has_feature(CPU_FTR_ARCH_300))
354 host_pcr_bit = PCR_ARCH_300;
355 else if (cpu_has_feature(CPU_FTR_ARCH_207S))
356 host_pcr_bit = PCR_ARCH_207;
357 else if (cpu_has_feature(CPU_FTR_ARCH_206))
358 host_pcr_bit = PCR_ARCH_206;
359 else
360 host_pcr_bit = PCR_ARCH_205;
361
362
363 guest_pcr_bit = host_pcr_bit;
364 if (arch_compat) {
365 switch (arch_compat) {
366 case PVR_ARCH_205:
367 guest_pcr_bit = PCR_ARCH_205;
368 break;
369 case PVR_ARCH_206:
370 case PVR_ARCH_206p:
371 guest_pcr_bit = PCR_ARCH_206;
372 break;
373 case PVR_ARCH_207:
374 guest_pcr_bit = PCR_ARCH_207;
375 break;
376 case PVR_ARCH_300:
377 guest_pcr_bit = PCR_ARCH_300;
378 break;
379 case PVR_ARCH_31:
380 guest_pcr_bit = PCR_ARCH_31;
381 break;
382 default:
383 return -EINVAL;
384 }
385 }
386
387
388 if (guest_pcr_bit > host_pcr_bit)
389 return -EINVAL;
390
391 spin_lock(&vc->lock);
392 vc->arch_compat = arch_compat;
393
394
395
396
397 vc->pcr = (host_pcr_bit - guest_pcr_bit) | PCR_MASK;
398 spin_unlock(&vc->lock);
399
400 return 0;
401}
402
403static void kvmppc_dump_regs(struct kvm_vcpu *vcpu)
404{
405 int r;
406
407 pr_err("vcpu %p (%d):\n", vcpu, vcpu->vcpu_id);
408 pr_err("pc = %.16lx msr = %.16llx trap = %x\n",
409 vcpu->arch.regs.nip, vcpu->arch.shregs.msr, vcpu->arch.trap);
410 for (r = 0; r < 16; ++r)
411 pr_err("r%2d = %.16lx r%d = %.16lx\n",
412 r, kvmppc_get_gpr(vcpu, r),
413 r+16, kvmppc_get_gpr(vcpu, r+16));
414 pr_err("ctr = %.16lx lr = %.16lx\n",
415 vcpu->arch.regs.ctr, vcpu->arch.regs.link);
416 pr_err("srr0 = %.16llx srr1 = %.16llx\n",
417 vcpu->arch.shregs.srr0, vcpu->arch.shregs.srr1);
418 pr_err("sprg0 = %.16llx sprg1 = %.16llx\n",
419 vcpu->arch.shregs.sprg0, vcpu->arch.shregs.sprg1);
420 pr_err("sprg2 = %.16llx sprg3 = %.16llx\n",
421 vcpu->arch.shregs.sprg2, vcpu->arch.shregs.sprg3);
422 pr_err("cr = %.8lx xer = %.16lx dsisr = %.8x\n",
423 vcpu->arch.regs.ccr, vcpu->arch.regs.xer, vcpu->arch.shregs.dsisr);
424 pr_err("dar = %.16llx\n", vcpu->arch.shregs.dar);
425 pr_err("fault dar = %.16lx dsisr = %.8x\n",
426 vcpu->arch.fault_dar, vcpu->arch.fault_dsisr);
427 pr_err("SLB (%d entries):\n", vcpu->arch.slb_max);
428 for (r = 0; r < vcpu->arch.slb_max; ++r)
429 pr_err(" ESID = %.16llx VSID = %.16llx\n",
430 vcpu->arch.slb[r].orige, vcpu->arch.slb[r].origv);
431 pr_err("lpcr = %.16lx sdr1 = %.16lx last_inst = %.8x\n",
432 vcpu->arch.vcore->lpcr, vcpu->kvm->arch.sdr1,
433 vcpu->arch.last_inst);
434}
435
436static struct kvm_vcpu *kvmppc_find_vcpu(struct kvm *kvm, int id)
437{
438 return kvm_get_vcpu_by_id(kvm, id);
439}
440
441static void init_vpa(struct kvm_vcpu *vcpu, struct lppaca *vpa)
442{
443 vpa->__old_status |= LPPACA_OLD_SHARED_PROC;
444 vpa->yield_count = cpu_to_be32(1);
445}
446
447static int set_vpa(struct kvm_vcpu *vcpu, struct kvmppc_vpa *v,
448 unsigned long addr, unsigned long len)
449{
450
451 if (addr & (L1_CACHE_BYTES - 1))
452 return -EINVAL;
453 spin_lock(&vcpu->arch.vpa_update_lock);
454 if (v->next_gpa != addr || v->len != len) {
455 v->next_gpa = addr;
456 v->len = addr ? len : 0;
457 v->update_pending = 1;
458 }
459 spin_unlock(&vcpu->arch.vpa_update_lock);
460 return 0;
461}
462
463
464struct reg_vpa {
465 u32 dummy;
466 union {
467 __be16 hword;
468 __be32 word;
469 } length;
470};
471
472static int vpa_is_registered(struct kvmppc_vpa *vpap)
473{
474 if (vpap->update_pending)
475 return vpap->next_gpa != 0;
476 return vpap->pinned_addr != NULL;
477}
478
479static unsigned long do_h_register_vpa(struct kvm_vcpu *vcpu,
480 unsigned long flags,
481 unsigned long vcpuid, unsigned long vpa)
482{
483 struct kvm *kvm = vcpu->kvm;
484 unsigned long len, nb;
485 void *va;
486 struct kvm_vcpu *tvcpu;
487 int err;
488 int subfunc;
489 struct kvmppc_vpa *vpap;
490
491 tvcpu = kvmppc_find_vcpu(kvm, vcpuid);
492 if (!tvcpu)
493 return H_PARAMETER;
494
495 subfunc = (flags >> H_VPA_FUNC_SHIFT) & H_VPA_FUNC_MASK;
496 if (subfunc == H_VPA_REG_VPA || subfunc == H_VPA_REG_DTL ||
497 subfunc == H_VPA_REG_SLB) {
498
499 if ((vpa & (L1_CACHE_BYTES - 1)) || !vpa)
500 return H_PARAMETER;
501
502
503 va = kvmppc_pin_guest_page(kvm, vpa, &nb);
504 if (va == NULL)
505 return H_PARAMETER;
506 if (subfunc == H_VPA_REG_VPA)
507 len = be16_to_cpu(((struct reg_vpa *)va)->length.hword);
508 else
509 len = be32_to_cpu(((struct reg_vpa *)va)->length.word);
510 kvmppc_unpin_guest_page(kvm, va, vpa, false);
511
512
513 if (len > nb || len < sizeof(struct reg_vpa))
514 return H_PARAMETER;
515 } else {
516 vpa = 0;
517 len = 0;
518 }
519
520 err = H_PARAMETER;
521 vpap = NULL;
522 spin_lock(&tvcpu->arch.vpa_update_lock);
523
524 switch (subfunc) {
525 case H_VPA_REG_VPA:
526
527
528
529
530
531
532 BUILD_BUG_ON(sizeof(struct lppaca) != 640);
533 if (len < sizeof(struct lppaca))
534 break;
535 vpap = &tvcpu->arch.vpa;
536 err = 0;
537 break;
538
539 case H_VPA_REG_DTL:
540 if (len < sizeof(struct dtl_entry))
541 break;
542 len -= len % sizeof(struct dtl_entry);
543
544
545 err = H_RESOURCE;
546 if (!vpa_is_registered(&tvcpu->arch.vpa))
547 break;
548
549 vpap = &tvcpu->arch.dtl;
550 err = 0;
551 break;
552
553 case H_VPA_REG_SLB:
554
555 err = H_RESOURCE;
556 if (!vpa_is_registered(&tvcpu->arch.vpa))
557 break;
558
559 vpap = &tvcpu->arch.slb_shadow;
560 err = 0;
561 break;
562
563 case H_VPA_DEREG_VPA:
564
565 err = H_RESOURCE;
566 if (vpa_is_registered(&tvcpu->arch.dtl) ||
567 vpa_is_registered(&tvcpu->arch.slb_shadow))
568 break;
569
570 vpap = &tvcpu->arch.vpa;
571 err = 0;
572 break;
573
574 case H_VPA_DEREG_DTL:
575 vpap = &tvcpu->arch.dtl;
576 err = 0;
577 break;
578
579 case H_VPA_DEREG_SLB:
580 vpap = &tvcpu->arch.slb_shadow;
581 err = 0;
582 break;
583 }
584
585 if (vpap) {
586 vpap->next_gpa = vpa;
587 vpap->len = len;
588 vpap->update_pending = 1;
589 }
590
591 spin_unlock(&tvcpu->arch.vpa_update_lock);
592
593 return err;
594}
595
596static void kvmppc_update_vpa(struct kvm_vcpu *vcpu, struct kvmppc_vpa *vpap)
597{
598 struct kvm *kvm = vcpu->kvm;
599 void *va;
600 unsigned long nb;
601 unsigned long gpa;
602
603
604
605
606
607
608
609
610
611 for (;;) {
612 gpa = vpap->next_gpa;
613 spin_unlock(&vcpu->arch.vpa_update_lock);
614 va = NULL;
615 nb = 0;
616 if (gpa)
617 va = kvmppc_pin_guest_page(kvm, gpa, &nb);
618 spin_lock(&vcpu->arch.vpa_update_lock);
619 if (gpa == vpap->next_gpa)
620 break;
621
622 if (va)
623 kvmppc_unpin_guest_page(kvm, va, gpa, false);
624 }
625
626 vpap->update_pending = 0;
627 if (va && nb < vpap->len) {
628
629
630
631
632
633 kvmppc_unpin_guest_page(kvm, va, gpa, false);
634 va = NULL;
635 }
636 if (vpap->pinned_addr)
637 kvmppc_unpin_guest_page(kvm, vpap->pinned_addr, vpap->gpa,
638 vpap->dirty);
639 vpap->gpa = gpa;
640 vpap->pinned_addr = va;
641 vpap->dirty = false;
642 if (va)
643 vpap->pinned_end = va + vpap->len;
644}
645
646static void kvmppc_update_vpas(struct kvm_vcpu *vcpu)
647{
648 if (!(vcpu->arch.vpa.update_pending ||
649 vcpu->arch.slb_shadow.update_pending ||
650 vcpu->arch.dtl.update_pending))
651 return;
652
653 spin_lock(&vcpu->arch.vpa_update_lock);
654 if (vcpu->arch.vpa.update_pending) {
655 kvmppc_update_vpa(vcpu, &vcpu->arch.vpa);
656 if (vcpu->arch.vpa.pinned_addr)
657 init_vpa(vcpu, vcpu->arch.vpa.pinned_addr);
658 }
659 if (vcpu->arch.dtl.update_pending) {
660 kvmppc_update_vpa(vcpu, &vcpu->arch.dtl);
661 vcpu->arch.dtl_ptr = vcpu->arch.dtl.pinned_addr;
662 vcpu->arch.dtl_index = 0;
663 }
664 if (vcpu->arch.slb_shadow.update_pending)
665 kvmppc_update_vpa(vcpu, &vcpu->arch.slb_shadow);
666 spin_unlock(&vcpu->arch.vpa_update_lock);
667}
668
669
670
671
672
673static u64 vcore_stolen_time(struct kvmppc_vcore *vc, u64 now)
674{
675 u64 p;
676 unsigned long flags;
677
678 spin_lock_irqsave(&vc->stoltb_lock, flags);
679 p = vc->stolen_tb;
680 if (vc->vcore_state != VCORE_INACTIVE &&
681 vc->preempt_tb != TB_NIL)
682 p += now - vc->preempt_tb;
683 spin_unlock_irqrestore(&vc->stoltb_lock, flags);
684 return p;
685}
686
687static void kvmppc_create_dtl_entry(struct kvm_vcpu *vcpu,
688 struct kvmppc_vcore *vc)
689{
690 struct dtl_entry *dt;
691 struct lppaca *vpa;
692 unsigned long stolen;
693 unsigned long core_stolen;
694 u64 now;
695 unsigned long flags;
696
697 dt = vcpu->arch.dtl_ptr;
698 vpa = vcpu->arch.vpa.pinned_addr;
699 now = mftb();
700 core_stolen = vcore_stolen_time(vc, now);
701 stolen = core_stolen - vcpu->arch.stolen_logged;
702 vcpu->arch.stolen_logged = core_stolen;
703 spin_lock_irqsave(&vcpu->arch.tbacct_lock, flags);
704 stolen += vcpu->arch.busy_stolen;
705 vcpu->arch.busy_stolen = 0;
706 spin_unlock_irqrestore(&vcpu->arch.tbacct_lock, flags);
707 if (!dt || !vpa)
708 return;
709 memset(dt, 0, sizeof(struct dtl_entry));
710 dt->dispatch_reason = 7;
711 dt->processor_id = cpu_to_be16(vc->pcpu + vcpu->arch.ptid);
712 dt->timebase = cpu_to_be64(now + vc->tb_offset);
713 dt->enqueue_to_dispatch_time = cpu_to_be32(stolen);
714 dt->srr0 = cpu_to_be64(kvmppc_get_pc(vcpu));
715 dt->srr1 = cpu_to_be64(vcpu->arch.shregs.msr);
716 ++dt;
717 if (dt == vcpu->arch.dtl.pinned_end)
718 dt = vcpu->arch.dtl.pinned_addr;
719 vcpu->arch.dtl_ptr = dt;
720
721 smp_wmb();
722 vpa->dtl_idx = cpu_to_be64(++vcpu->arch.dtl_index);
723 vcpu->arch.dtl.dirty = true;
724}
725
726
727static bool kvmppc_doorbell_pending(struct kvm_vcpu *vcpu)
728{
729 int thr;
730 struct kvmppc_vcore *vc;
731
732 if (vcpu->arch.doorbell_request)
733 return true;
734
735
736
737
738
739 smp_rmb();
740 vc = vcpu->arch.vcore;
741 thr = vcpu->vcpu_id - vc->first_vcpuid;
742 return !!(vc->dpdes & (1 << thr));
743}
744
745static bool kvmppc_power8_compatible(struct kvm_vcpu *vcpu)
746{
747 if (vcpu->arch.vcore->arch_compat >= PVR_ARCH_207)
748 return true;
749 if ((!vcpu->arch.vcore->arch_compat) &&
750 cpu_has_feature(CPU_FTR_ARCH_207S))
751 return true;
752 return false;
753}
754
755static int kvmppc_h_set_mode(struct kvm_vcpu *vcpu, unsigned long mflags,
756 unsigned long resource, unsigned long value1,
757 unsigned long value2)
758{
759 switch (resource) {
760 case H_SET_MODE_RESOURCE_SET_CIABR:
761 if (!kvmppc_power8_compatible(vcpu))
762 return H_P2;
763 if (value2)
764 return H_P4;
765 if (mflags)
766 return H_UNSUPPORTED_FLAG_START;
767
768 if ((value1 & CIABR_PRIV) == CIABR_PRIV_HYPER)
769 return H_P3;
770 vcpu->arch.ciabr = value1;
771 return H_SUCCESS;
772 case H_SET_MODE_RESOURCE_SET_DAWR0:
773 if (!kvmppc_power8_compatible(vcpu))
774 return H_P2;
775 if (!ppc_breakpoint_available())
776 return H_P2;
777 if (mflags)
778 return H_UNSUPPORTED_FLAG_START;
779 if (value2 & DABRX_HYP)
780 return H_P4;
781 vcpu->arch.dawr0 = value1;
782 vcpu->arch.dawrx0 = value2;
783 return H_SUCCESS;
784 case H_SET_MODE_RESOURCE_SET_DAWR1:
785 if (!kvmppc_power8_compatible(vcpu))
786 return H_P2;
787 if (!ppc_breakpoint_available())
788 return H_P2;
789 if (!cpu_has_feature(CPU_FTR_DAWR1))
790 return H_P2;
791 if (!vcpu->kvm->arch.dawr1_enabled)
792 return H_FUNCTION;
793 if (mflags)
794 return H_UNSUPPORTED_FLAG_START;
795 if (value2 & DABRX_HYP)
796 return H_P4;
797 vcpu->arch.dawr1 = value1;
798 vcpu->arch.dawrx1 = value2;
799 return H_SUCCESS;
800 case H_SET_MODE_RESOURCE_ADDR_TRANS_MODE:
801
802
803
804
805 if (cpu_has_feature(CPU_FTR_P9_RADIX_PREFETCH_BUG) &&
806 kvmhv_vcpu_is_radix(vcpu) && mflags == 3)
807 return H_UNSUPPORTED_FLAG_START;
808 return H_TOO_HARD;
809 default:
810 return H_TOO_HARD;
811 }
812}
813
814
815static int kvmppc_copy_guest(struct kvm *kvm, gpa_t to, gpa_t from,
816 unsigned long len)
817{
818 struct kvm_memory_slot *to_memslot = NULL;
819 struct kvm_memory_slot *from_memslot = NULL;
820 unsigned long to_addr, from_addr;
821 int r;
822
823
824 from_memslot = gfn_to_memslot(kvm, from >> PAGE_SHIFT);
825 if (!from_memslot)
826 return -EFAULT;
827 if ((from + len) >= ((from_memslot->base_gfn + from_memslot->npages)
828 << PAGE_SHIFT))
829 return -EINVAL;
830 from_addr = gfn_to_hva_memslot(from_memslot, from >> PAGE_SHIFT);
831 if (kvm_is_error_hva(from_addr))
832 return -EFAULT;
833 from_addr |= (from & (PAGE_SIZE - 1));
834
835
836 to_memslot = gfn_to_memslot(kvm, to >> PAGE_SHIFT);
837 if (!to_memslot)
838 return -EFAULT;
839 if ((to + len) >= ((to_memslot->base_gfn + to_memslot->npages)
840 << PAGE_SHIFT))
841 return -EINVAL;
842 to_addr = gfn_to_hva_memslot(to_memslot, to >> PAGE_SHIFT);
843 if (kvm_is_error_hva(to_addr))
844 return -EFAULT;
845 to_addr |= (to & (PAGE_SIZE - 1));
846
847
848 r = raw_copy_in_user((void __user *)to_addr, (void __user *)from_addr,
849 len);
850 if (r)
851 return -EFAULT;
852 mark_page_dirty(kvm, to >> PAGE_SHIFT);
853 return 0;
854}
855
856static long kvmppc_h_page_init(struct kvm_vcpu *vcpu, unsigned long flags,
857 unsigned long dest, unsigned long src)
858{
859 u64 pg_sz = SZ_4K;
860 u64 pg_mask = SZ_4K - 1;
861 int ret;
862
863
864 if (flags & ~(H_ICACHE_INVALIDATE | H_ICACHE_SYNCHRONIZE |
865 H_ZERO_PAGE | H_COPY_PAGE | H_PAGE_SET_LOANED))
866 return H_PARAMETER;
867
868
869 if ((dest & pg_mask) || ((flags & H_COPY_PAGE) && (src & pg_mask)))
870 return H_PARAMETER;
871
872
873 if (flags & H_COPY_PAGE) {
874 ret = kvmppc_copy_guest(vcpu->kvm, dest, src, pg_sz);
875 if (ret < 0)
876 return H_PARAMETER;
877 } else if (flags & H_ZERO_PAGE) {
878 ret = kvm_clear_guest(vcpu->kvm, dest, pg_sz);
879 if (ret < 0)
880 return H_PARAMETER;
881 }
882
883
884
885 return H_SUCCESS;
886}
887
888static int kvm_arch_vcpu_yield_to(struct kvm_vcpu *target)
889{
890 struct kvmppc_vcore *vcore = target->arch.vcore;
891
892
893
894
895
896
897
898
899
900
901
902
903
904 spin_lock(&vcore->lock);
905 if (target->arch.state == KVMPPC_VCPU_RUNNABLE &&
906 vcore->vcore_state != VCORE_INACTIVE &&
907 vcore->runner)
908 target = vcore->runner;
909 spin_unlock(&vcore->lock);
910
911 return kvm_vcpu_yield_to(target);
912}
913
914static int kvmppc_get_yield_count(struct kvm_vcpu *vcpu)
915{
916 int yield_count = 0;
917 struct lppaca *lppaca;
918
919 spin_lock(&vcpu->arch.vpa_update_lock);
920 lppaca = (struct lppaca *)vcpu->arch.vpa.pinned_addr;
921 if (lppaca)
922 yield_count = be32_to_cpu(lppaca->yield_count);
923 spin_unlock(&vcpu->arch.vpa_update_lock);
924 return yield_count;
925}
926
927
928
929
930
931
932static int kvmppc_nested_h_rpt_invalidate(struct kvm_vcpu *vcpu)
933{
934 unsigned long type = kvmppc_get_gpr(vcpu, 6);
935 unsigned long pid, pg_sizes, start, end;
936
937
938
939
940 if (type & H_RPTI_TYPE_NESTED)
941 return RESUME_HOST;
942
943 pid = kvmppc_get_gpr(vcpu, 4);
944 pg_sizes = kvmppc_get_gpr(vcpu, 7);
945 start = kvmppc_get_gpr(vcpu, 8);
946 end = kvmppc_get_gpr(vcpu, 9);
947
948 do_h_rpt_invalidate_prt(pid, vcpu->arch.nested->shadow_lpid,
949 type, pg_sizes, start, end);
950
951 kvmppc_set_gpr(vcpu, 3, H_SUCCESS);
952 return RESUME_GUEST;
953}
954
955static long kvmppc_h_rpt_invalidate(struct kvm_vcpu *vcpu,
956 unsigned long id, unsigned long target,
957 unsigned long type, unsigned long pg_sizes,
958 unsigned long start, unsigned long end)
959{
960 if (!kvm_is_radix(vcpu->kvm))
961 return H_UNSUPPORTED;
962
963 if (end < start)
964 return H_P5;
965
966
967
968
969 if (type & H_RPTI_TYPE_NESTED) {
970 if (!nesting_enabled(vcpu->kvm))
971 return H_FUNCTION;
972
973
974 if (target != H_RPTI_TARGET_CMMU)
975 return H_P2;
976
977 return do_h_rpt_invalidate_pat(vcpu, id, type, pg_sizes,
978 start, end);
979 }
980
981
982
983
984 do_h_rpt_invalidate_prt(id, vcpu->kvm->arch.lpid,
985 type, pg_sizes, start, end);
986 return H_SUCCESS;
987}
988
989int kvmppc_pseries_do_hcall(struct kvm_vcpu *vcpu)
990{
991 struct kvm *kvm = vcpu->kvm;
992 unsigned long req = kvmppc_get_gpr(vcpu, 3);
993 unsigned long target, ret = H_SUCCESS;
994 int yield_count;
995 struct kvm_vcpu *tvcpu;
996 int idx, rc;
997
998 if (req <= MAX_HCALL_OPCODE &&
999 !test_bit(req/4, vcpu->kvm->arch.enabled_hcalls))
1000 return RESUME_HOST;
1001
1002 switch (req) {
1003 case H_REMOVE:
1004 ret = kvmppc_h_remove(vcpu, kvmppc_get_gpr(vcpu, 4),
1005 kvmppc_get_gpr(vcpu, 5),
1006 kvmppc_get_gpr(vcpu, 6));
1007 if (ret == H_TOO_HARD)
1008 return RESUME_HOST;
1009 break;
1010 case H_ENTER:
1011 ret = kvmppc_h_enter(vcpu, kvmppc_get_gpr(vcpu, 4),
1012 kvmppc_get_gpr(vcpu, 5),
1013 kvmppc_get_gpr(vcpu, 6),
1014 kvmppc_get_gpr(vcpu, 7));
1015 if (ret == H_TOO_HARD)
1016 return RESUME_HOST;
1017 break;
1018 case H_READ:
1019 ret = kvmppc_h_read(vcpu, kvmppc_get_gpr(vcpu, 4),
1020 kvmppc_get_gpr(vcpu, 5));
1021 if (ret == H_TOO_HARD)
1022 return RESUME_HOST;
1023 break;
1024 case H_CLEAR_MOD:
1025 ret = kvmppc_h_clear_mod(vcpu, kvmppc_get_gpr(vcpu, 4),
1026 kvmppc_get_gpr(vcpu, 5));
1027 if (ret == H_TOO_HARD)
1028 return RESUME_HOST;
1029 break;
1030 case H_CLEAR_REF:
1031 ret = kvmppc_h_clear_ref(vcpu, kvmppc_get_gpr(vcpu, 4),
1032 kvmppc_get_gpr(vcpu, 5));
1033 if (ret == H_TOO_HARD)
1034 return RESUME_HOST;
1035 break;
1036 case H_PROTECT:
1037 ret = kvmppc_h_protect(vcpu, kvmppc_get_gpr(vcpu, 4),
1038 kvmppc_get_gpr(vcpu, 5),
1039 kvmppc_get_gpr(vcpu, 6));
1040 if (ret == H_TOO_HARD)
1041 return RESUME_HOST;
1042 break;
1043 case H_BULK_REMOVE:
1044 ret = kvmppc_h_bulk_remove(vcpu);
1045 if (ret == H_TOO_HARD)
1046 return RESUME_HOST;
1047 break;
1048
1049 case H_CEDE:
1050 break;
1051 case H_PROD:
1052 target = kvmppc_get_gpr(vcpu, 4);
1053 tvcpu = kvmppc_find_vcpu(kvm, target);
1054 if (!tvcpu) {
1055 ret = H_PARAMETER;
1056 break;
1057 }
1058 tvcpu->arch.prodded = 1;
1059 smp_mb();
1060 if (tvcpu->arch.ceded)
1061 kvmppc_fast_vcpu_kick_hv(tvcpu);
1062 break;
1063 case H_CONFER:
1064 target = kvmppc_get_gpr(vcpu, 4);
1065 if (target == -1)
1066 break;
1067 tvcpu = kvmppc_find_vcpu(kvm, target);
1068 if (!tvcpu) {
1069 ret = H_PARAMETER;
1070 break;
1071 }
1072 yield_count = kvmppc_get_gpr(vcpu, 5);
1073 if (kvmppc_get_yield_count(tvcpu) != yield_count)
1074 break;
1075 kvm_arch_vcpu_yield_to(tvcpu);
1076 break;
1077 case H_REGISTER_VPA:
1078 ret = do_h_register_vpa(vcpu, kvmppc_get_gpr(vcpu, 4),
1079 kvmppc_get_gpr(vcpu, 5),
1080 kvmppc_get_gpr(vcpu, 6));
1081 break;
1082 case H_RTAS:
1083 if (list_empty(&kvm->arch.rtas_tokens))
1084 return RESUME_HOST;
1085
1086 idx = srcu_read_lock(&kvm->srcu);
1087 rc = kvmppc_rtas_hcall(vcpu);
1088 srcu_read_unlock(&kvm->srcu, idx);
1089
1090 if (rc == -ENOENT)
1091 return RESUME_HOST;
1092 else if (rc == 0)
1093 break;
1094
1095
1096 return rc;
1097 case H_LOGICAL_CI_LOAD:
1098 ret = kvmppc_h_logical_ci_load(vcpu);
1099 if (ret == H_TOO_HARD)
1100 return RESUME_HOST;
1101 break;
1102 case H_LOGICAL_CI_STORE:
1103 ret = kvmppc_h_logical_ci_store(vcpu);
1104 if (ret == H_TOO_HARD)
1105 return RESUME_HOST;
1106 break;
1107 case H_SET_MODE:
1108 ret = kvmppc_h_set_mode(vcpu, kvmppc_get_gpr(vcpu, 4),
1109 kvmppc_get_gpr(vcpu, 5),
1110 kvmppc_get_gpr(vcpu, 6),
1111 kvmppc_get_gpr(vcpu, 7));
1112 if (ret == H_TOO_HARD)
1113 return RESUME_HOST;
1114 break;
1115 case H_XIRR:
1116 case H_CPPR:
1117 case H_EOI:
1118 case H_IPI:
1119 case H_IPOLL:
1120 case H_XIRR_X:
1121 if (kvmppc_xics_enabled(vcpu)) {
1122 if (xics_on_xive()) {
1123 ret = H_NOT_AVAILABLE;
1124 return RESUME_GUEST;
1125 }
1126 ret = kvmppc_xics_hcall(vcpu, req);
1127 break;
1128 }
1129 return RESUME_HOST;
1130 case H_SET_DABR:
1131 ret = kvmppc_h_set_dabr(vcpu, kvmppc_get_gpr(vcpu, 4));
1132 break;
1133 case H_SET_XDABR:
1134 ret = kvmppc_h_set_xdabr(vcpu, kvmppc_get_gpr(vcpu, 4),
1135 kvmppc_get_gpr(vcpu, 5));
1136 break;
1137#ifdef CONFIG_SPAPR_TCE_IOMMU
1138 case H_GET_TCE:
1139 ret = kvmppc_h_get_tce(vcpu, kvmppc_get_gpr(vcpu, 4),
1140 kvmppc_get_gpr(vcpu, 5));
1141 if (ret == H_TOO_HARD)
1142 return RESUME_HOST;
1143 break;
1144 case H_PUT_TCE:
1145 ret = kvmppc_h_put_tce(vcpu, kvmppc_get_gpr(vcpu, 4),
1146 kvmppc_get_gpr(vcpu, 5),
1147 kvmppc_get_gpr(vcpu, 6));
1148 if (ret == H_TOO_HARD)
1149 return RESUME_HOST;
1150 break;
1151 case H_PUT_TCE_INDIRECT:
1152 ret = kvmppc_h_put_tce_indirect(vcpu, kvmppc_get_gpr(vcpu, 4),
1153 kvmppc_get_gpr(vcpu, 5),
1154 kvmppc_get_gpr(vcpu, 6),
1155 kvmppc_get_gpr(vcpu, 7));
1156 if (ret == H_TOO_HARD)
1157 return RESUME_HOST;
1158 break;
1159 case H_STUFF_TCE:
1160 ret = kvmppc_h_stuff_tce(vcpu, kvmppc_get_gpr(vcpu, 4),
1161 kvmppc_get_gpr(vcpu, 5),
1162 kvmppc_get_gpr(vcpu, 6),
1163 kvmppc_get_gpr(vcpu, 7));
1164 if (ret == H_TOO_HARD)
1165 return RESUME_HOST;
1166 break;
1167#endif
1168 case H_RANDOM:
1169 if (!arch_get_random_seed_long(&vcpu->arch.regs.gpr[4]))
1170 ret = H_HARDWARE;
1171 break;
1172 case H_RPT_INVALIDATE:
1173 ret = kvmppc_h_rpt_invalidate(vcpu, kvmppc_get_gpr(vcpu, 4),
1174 kvmppc_get_gpr(vcpu, 5),
1175 kvmppc_get_gpr(vcpu, 6),
1176 kvmppc_get_gpr(vcpu, 7),
1177 kvmppc_get_gpr(vcpu, 8),
1178 kvmppc_get_gpr(vcpu, 9));
1179 break;
1180
1181 case H_SET_PARTITION_TABLE:
1182 ret = H_FUNCTION;
1183 if (nesting_enabled(kvm))
1184 ret = kvmhv_set_partition_table(vcpu);
1185 break;
1186 case H_ENTER_NESTED:
1187 ret = H_FUNCTION;
1188 if (!nesting_enabled(kvm))
1189 break;
1190 ret = kvmhv_enter_nested_guest(vcpu);
1191 if (ret == H_INTERRUPT) {
1192 kvmppc_set_gpr(vcpu, 3, 0);
1193 vcpu->arch.hcall_needed = 0;
1194 return -EINTR;
1195 } else if (ret == H_TOO_HARD) {
1196 kvmppc_set_gpr(vcpu, 3, 0);
1197 vcpu->arch.hcall_needed = 0;
1198 return RESUME_HOST;
1199 }
1200 break;
1201 case H_TLB_INVALIDATE:
1202 ret = H_FUNCTION;
1203 if (nesting_enabled(kvm))
1204 ret = kvmhv_do_nested_tlbie(vcpu);
1205 break;
1206 case H_COPY_TOFROM_GUEST:
1207 ret = H_FUNCTION;
1208 if (nesting_enabled(kvm))
1209 ret = kvmhv_copy_tofrom_guest_nested(vcpu);
1210 break;
1211 case H_PAGE_INIT:
1212 ret = kvmppc_h_page_init(vcpu, kvmppc_get_gpr(vcpu, 4),
1213 kvmppc_get_gpr(vcpu, 5),
1214 kvmppc_get_gpr(vcpu, 6));
1215 break;
1216 case H_SVM_PAGE_IN:
1217 ret = H_UNSUPPORTED;
1218 if (kvmppc_get_srr1(vcpu) & MSR_S)
1219 ret = kvmppc_h_svm_page_in(kvm,
1220 kvmppc_get_gpr(vcpu, 4),
1221 kvmppc_get_gpr(vcpu, 5),
1222 kvmppc_get_gpr(vcpu, 6));
1223 break;
1224 case H_SVM_PAGE_OUT:
1225 ret = H_UNSUPPORTED;
1226 if (kvmppc_get_srr1(vcpu) & MSR_S)
1227 ret = kvmppc_h_svm_page_out(kvm,
1228 kvmppc_get_gpr(vcpu, 4),
1229 kvmppc_get_gpr(vcpu, 5),
1230 kvmppc_get_gpr(vcpu, 6));
1231 break;
1232 case H_SVM_INIT_START:
1233 ret = H_UNSUPPORTED;
1234 if (kvmppc_get_srr1(vcpu) & MSR_S)
1235 ret = kvmppc_h_svm_init_start(kvm);
1236 break;
1237 case H_SVM_INIT_DONE:
1238 ret = H_UNSUPPORTED;
1239 if (kvmppc_get_srr1(vcpu) & MSR_S)
1240 ret = kvmppc_h_svm_init_done(kvm);
1241 break;
1242 case H_SVM_INIT_ABORT:
1243
1244
1245
1246
1247
1248
1249
1250 ret = kvmppc_h_svm_init_abort(kvm);
1251 break;
1252
1253 default:
1254 return RESUME_HOST;
1255 }
1256 WARN_ON_ONCE(ret == H_TOO_HARD);
1257 kvmppc_set_gpr(vcpu, 3, ret);
1258 vcpu->arch.hcall_needed = 0;
1259 return RESUME_GUEST;
1260}
1261
1262
1263
1264
1265
1266
1267
1268
1269static void kvmppc_cede(struct kvm_vcpu *vcpu)
1270{
1271 vcpu->arch.shregs.msr |= MSR_EE;
1272 vcpu->arch.ceded = 1;
1273 smp_mb();
1274 if (vcpu->arch.prodded) {
1275 vcpu->arch.prodded = 0;
1276 smp_mb();
1277 vcpu->arch.ceded = 0;
1278 }
1279}
1280
1281static int kvmppc_hcall_impl_hv(unsigned long cmd)
1282{
1283 switch (cmd) {
1284 case H_CEDE:
1285 case H_PROD:
1286 case H_CONFER:
1287 case H_REGISTER_VPA:
1288 case H_SET_MODE:
1289 case H_LOGICAL_CI_LOAD:
1290 case H_LOGICAL_CI_STORE:
1291#ifdef CONFIG_KVM_XICS
1292 case H_XIRR:
1293 case H_CPPR:
1294 case H_EOI:
1295 case H_IPI:
1296 case H_IPOLL:
1297 case H_XIRR_X:
1298#endif
1299 case H_PAGE_INIT:
1300 case H_RPT_INVALIDATE:
1301 return 1;
1302 }
1303
1304
1305 return kvmppc_hcall_impl_hv_realmode(cmd);
1306}
1307
1308static int kvmppc_emulate_debug_inst(struct kvm_vcpu *vcpu)
1309{
1310 u32 last_inst;
1311
1312 if (kvmppc_get_last_inst(vcpu, INST_GENERIC, &last_inst) !=
1313 EMULATE_DONE) {
1314
1315
1316
1317
1318 return RESUME_GUEST;
1319 }
1320
1321 if (last_inst == KVMPPC_INST_SW_BREAKPOINT) {
1322 vcpu->run->exit_reason = KVM_EXIT_DEBUG;
1323 vcpu->run->debug.arch.address = kvmppc_get_pc(vcpu);
1324 return RESUME_HOST;
1325 } else {
1326 kvmppc_core_queue_program(vcpu, SRR1_PROGILL);
1327 return RESUME_GUEST;
1328 }
1329}
1330
1331static void do_nothing(void *x)
1332{
1333}
1334
1335static unsigned long kvmppc_read_dpdes(struct kvm_vcpu *vcpu)
1336{
1337 int thr, cpu, pcpu, nthreads;
1338 struct kvm_vcpu *v;
1339 unsigned long dpdes;
1340
1341 nthreads = vcpu->kvm->arch.emul_smt_mode;
1342 dpdes = 0;
1343 cpu = vcpu->vcpu_id & ~(nthreads - 1);
1344 for (thr = 0; thr < nthreads; ++thr, ++cpu) {
1345 v = kvmppc_find_vcpu(vcpu->kvm, cpu);
1346 if (!v)
1347 continue;
1348
1349
1350
1351
1352
1353 pcpu = READ_ONCE(v->cpu);
1354 if (pcpu >= 0)
1355 smp_call_function_single(pcpu, do_nothing, NULL, 1);
1356 if (kvmppc_doorbell_pending(v))
1357 dpdes |= 1 << thr;
1358 }
1359 return dpdes;
1360}
1361
1362
1363
1364
1365
1366
1367
1368static int kvmppc_emulate_doorbell_instr(struct kvm_vcpu *vcpu)
1369{
1370 u32 inst, rb, thr;
1371 unsigned long arg;
1372 struct kvm *kvm = vcpu->kvm;
1373 struct kvm_vcpu *tvcpu;
1374
1375 if (kvmppc_get_last_inst(vcpu, INST_GENERIC, &inst) != EMULATE_DONE)
1376 return RESUME_GUEST;
1377 if (get_op(inst) != 31)
1378 return EMULATE_FAIL;
1379 rb = get_rb(inst);
1380 thr = vcpu->vcpu_id & (kvm->arch.emul_smt_mode - 1);
1381 switch (get_xop(inst)) {
1382 case OP_31_XOP_MSGSNDP:
1383 arg = kvmppc_get_gpr(vcpu, rb);
1384 if (((arg >> 27) & 0x1f) != PPC_DBELL_SERVER)
1385 break;
1386 arg &= 0x7f;
1387 if (arg >= kvm->arch.emul_smt_mode)
1388 break;
1389 tvcpu = kvmppc_find_vcpu(kvm, vcpu->vcpu_id - thr + arg);
1390 if (!tvcpu)
1391 break;
1392 if (!tvcpu->arch.doorbell_request) {
1393 tvcpu->arch.doorbell_request = 1;
1394 kvmppc_fast_vcpu_kick_hv(tvcpu);
1395 }
1396 break;
1397 case OP_31_XOP_MSGCLRP:
1398 arg = kvmppc_get_gpr(vcpu, rb);
1399 if (((arg >> 27) & 0x1f) != PPC_DBELL_SERVER)
1400 break;
1401 vcpu->arch.vcore->dpdes = 0;
1402 vcpu->arch.doorbell_request = 0;
1403 break;
1404 case OP_31_XOP_MFSPR:
1405 switch (get_sprn(inst)) {
1406 case SPRN_TIR:
1407 arg = thr;
1408 break;
1409 case SPRN_DPDES:
1410 arg = kvmppc_read_dpdes(vcpu);
1411 break;
1412 default:
1413 return EMULATE_FAIL;
1414 }
1415 kvmppc_set_gpr(vcpu, get_rt(inst), arg);
1416 break;
1417 default:
1418 return EMULATE_FAIL;
1419 }
1420 kvmppc_set_pc(vcpu, kvmppc_get_pc(vcpu) + 4);
1421 return RESUME_GUEST;
1422}
1423
1424static int kvmppc_handle_exit_hv(struct kvm_vcpu *vcpu,
1425 struct task_struct *tsk)
1426{
1427 struct kvm_run *run = vcpu->run;
1428 int r = RESUME_HOST;
1429
1430 vcpu->stat.sum_exits++;
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440 if (vcpu->arch.shregs.msr & MSR_HV) {
1441 printk(KERN_EMERG "KVM trap in HV mode!\n");
1442 printk(KERN_EMERG "trap=0x%x | pc=0x%lx | msr=0x%llx\n",
1443 vcpu->arch.trap, kvmppc_get_pc(vcpu),
1444 vcpu->arch.shregs.msr);
1445 kvmppc_dump_regs(vcpu);
1446 run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
1447 run->hw.hardware_exit_reason = vcpu->arch.trap;
1448 return RESUME_HOST;
1449 }
1450 run->exit_reason = KVM_EXIT_UNKNOWN;
1451 run->ready_for_interrupt_injection = 1;
1452 switch (vcpu->arch.trap) {
1453
1454 case BOOK3S_INTERRUPT_HV_DECREMENTER:
1455 vcpu->stat.dec_exits++;
1456 r = RESUME_GUEST;
1457 break;
1458 case BOOK3S_INTERRUPT_EXTERNAL:
1459 case BOOK3S_INTERRUPT_H_DOORBELL:
1460 case BOOK3S_INTERRUPT_H_VIRT:
1461 vcpu->stat.ext_intr_exits++;
1462 r = RESUME_GUEST;
1463 break;
1464
1465 case BOOK3S_INTERRUPT_HMI:
1466 case BOOK3S_INTERRUPT_PERFMON:
1467 case BOOK3S_INTERRUPT_SYSTEM_RESET:
1468 r = RESUME_GUEST;
1469 break;
1470 case BOOK3S_INTERRUPT_MACHINE_CHECK: {
1471 static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL,
1472 DEFAULT_RATELIMIT_BURST);
1473
1474
1475
1476
1477 if (__ratelimit(&rs))
1478 machine_check_print_event_info(&vcpu->arch.mce_evt,false, true);
1479
1480
1481
1482
1483
1484
1485
1486 if (!vcpu->kvm->arch.fwnmi_enabled) {
1487 ulong flags = vcpu->arch.shregs.msr & 0x083c0000;
1488 kvmppc_core_queue_machine_check(vcpu, flags);
1489 r = RESUME_GUEST;
1490 break;
1491 }
1492
1493
1494 run->exit_reason = KVM_EXIT_NMI;
1495 run->hw.hardware_exit_reason = vcpu->arch.trap;
1496
1497 run->flags &= ~KVM_RUN_PPC_NMI_DISP_MASK;
1498
1499 if (vcpu->arch.mce_evt.disposition == MCE_DISPOSITION_RECOVERED)
1500 run->flags |= KVM_RUN_PPC_NMI_DISP_FULLY_RECOV;
1501 else
1502 run->flags |= KVM_RUN_PPC_NMI_DISP_NOT_RECOV;
1503
1504 r = RESUME_HOST;
1505 break;
1506 }
1507 case BOOK3S_INTERRUPT_PROGRAM:
1508 {
1509 ulong flags;
1510
1511
1512
1513
1514
1515
1516 flags = vcpu->arch.shregs.msr & 0x1f0000ull;
1517 kvmppc_core_queue_program(vcpu, flags);
1518 r = RESUME_GUEST;
1519 break;
1520 }
1521 case BOOK3S_INTERRUPT_SYSCALL:
1522 {
1523 int i;
1524
1525 if (unlikely(vcpu->arch.shregs.msr & MSR_PR)) {
1526
1527
1528
1529
1530
1531 if (!kvmhv_vcpu_is_radix(vcpu)) {
1532
1533
1534
1535
1536
1537 kvmppc_core_queue_syscall(vcpu);
1538 } else {
1539
1540
1541
1542
1543
1544
1545 kvmppc_core_queue_program(vcpu, SRR1_PROGPRIV);
1546 }
1547 r = RESUME_GUEST;
1548 break;
1549 }
1550
1551
1552
1553
1554
1555
1556 run->papr_hcall.nr = kvmppc_get_gpr(vcpu, 3);
1557 for (i = 0; i < 9; ++i)
1558 run->papr_hcall.args[i] = kvmppc_get_gpr(vcpu, 4 + i);
1559 run->exit_reason = KVM_EXIT_PAPR_HCALL;
1560 vcpu->arch.hcall_needed = 1;
1561 r = RESUME_HOST;
1562 break;
1563 }
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574 case BOOK3S_INTERRUPT_H_DATA_STORAGE: {
1575 unsigned long vsid;
1576 long err;
1577
1578 if (vcpu->arch.fault_dsisr == HDSISR_CANARY) {
1579 r = RESUME_GUEST;
1580 break;
1581 }
1582
1583 if (kvm_is_radix(vcpu->kvm) || !cpu_has_feature(CPU_FTR_ARCH_300)) {
1584
1585
1586
1587
1588
1589
1590 r = RESUME_PAGE_FAULT;
1591 break;
1592 }
1593
1594 if (!(vcpu->arch.fault_dsisr & (DSISR_NOHPTE | DSISR_PROTFAULT))) {
1595 kvmppc_core_queue_data_storage(vcpu,
1596 vcpu->arch.fault_dar, vcpu->arch.fault_dsisr);
1597 r = RESUME_GUEST;
1598 break;
1599 }
1600
1601 if (!(vcpu->arch.shregs.msr & MSR_DR))
1602 vsid = vcpu->kvm->arch.vrma_slb_v;
1603 else
1604 vsid = vcpu->arch.fault_gpa;
1605
1606 err = kvmppc_hpte_hv_fault(vcpu, vcpu->arch.fault_dar,
1607 vsid, vcpu->arch.fault_dsisr, true);
1608 if (err == 0) {
1609 r = RESUME_GUEST;
1610 } else if (err == -1 || err == -2) {
1611 r = RESUME_PAGE_FAULT;
1612 } else {
1613 kvmppc_core_queue_data_storage(vcpu,
1614 vcpu->arch.fault_dar, err);
1615 r = RESUME_GUEST;
1616 }
1617 break;
1618 }
1619 case BOOK3S_INTERRUPT_H_INST_STORAGE: {
1620 unsigned long vsid;
1621 long err;
1622
1623 vcpu->arch.fault_dar = kvmppc_get_pc(vcpu);
1624 vcpu->arch.fault_dsisr = vcpu->arch.shregs.msr &
1625 DSISR_SRR1_MATCH_64S;
1626 if (kvm_is_radix(vcpu->kvm) || !cpu_has_feature(CPU_FTR_ARCH_300)) {
1627
1628
1629
1630
1631
1632
1633 if (vcpu->arch.shregs.msr & HSRR1_HISI_WRITE)
1634 vcpu->arch.fault_dsisr |= DSISR_ISSTORE;
1635 r = RESUME_PAGE_FAULT;
1636 break;
1637 }
1638
1639 if (!(vcpu->arch.fault_dsisr & SRR1_ISI_NOPT)) {
1640 kvmppc_core_queue_inst_storage(vcpu,
1641 vcpu->arch.fault_dsisr);
1642 r = RESUME_GUEST;
1643 break;
1644 }
1645
1646 if (!(vcpu->arch.shregs.msr & MSR_IR))
1647 vsid = vcpu->kvm->arch.vrma_slb_v;
1648 else
1649 vsid = vcpu->arch.fault_gpa;
1650
1651 err = kvmppc_hpte_hv_fault(vcpu, vcpu->arch.fault_dar,
1652 vsid, vcpu->arch.fault_dsisr, false);
1653 if (err == 0) {
1654 r = RESUME_GUEST;
1655 } else if (err == -1) {
1656 r = RESUME_PAGE_FAULT;
1657 } else {
1658 kvmppc_core_queue_inst_storage(vcpu, err);
1659 r = RESUME_GUEST;
1660 }
1661 break;
1662 }
1663
1664
1665
1666
1667
1668
1669
1670
1671 case BOOK3S_INTERRUPT_H_EMUL_ASSIST:
1672 if (vcpu->arch.emul_inst != KVM_INST_FETCH_FAILED)
1673 vcpu->arch.last_inst = kvmppc_need_byteswap(vcpu) ?
1674 swab32(vcpu->arch.emul_inst) :
1675 vcpu->arch.emul_inst;
1676 if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP) {
1677 r = kvmppc_emulate_debug_inst(vcpu);
1678 } else {
1679 kvmppc_core_queue_program(vcpu, SRR1_PROGILL);
1680 r = RESUME_GUEST;
1681 }
1682 break;
1683
1684#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1685 case BOOK3S_INTERRUPT_HV_SOFTPATCH:
1686
1687
1688
1689
1690
1691
1692 r = kvmhv_p9_tm_emulation(vcpu);
1693 if (r != -1)
1694 break;
1695 fallthrough;
1696#endif
1697
1698
1699
1700
1701
1702
1703
1704
1705 case BOOK3S_INTERRUPT_H_FAC_UNAVAIL:
1706 r = EMULATE_FAIL;
1707 if (((vcpu->arch.hfscr >> 56) == FSCR_MSGP_LG) &&
1708 cpu_has_feature(CPU_FTR_ARCH_300))
1709 r = kvmppc_emulate_doorbell_instr(vcpu);
1710 if (r == EMULATE_FAIL) {
1711 kvmppc_core_queue_program(vcpu, SRR1_PROGILL);
1712 r = RESUME_GUEST;
1713 }
1714 break;
1715
1716 case BOOK3S_INTERRUPT_HV_RM_HARD:
1717 r = RESUME_PASSTHROUGH;
1718 break;
1719 default:
1720 kvmppc_dump_regs(vcpu);
1721 printk(KERN_EMERG "trap=0x%x | pc=0x%lx | msr=0x%llx\n",
1722 vcpu->arch.trap, kvmppc_get_pc(vcpu),
1723 vcpu->arch.shregs.msr);
1724 run->hw.hardware_exit_reason = vcpu->arch.trap;
1725 r = RESUME_HOST;
1726 break;
1727 }
1728
1729 return r;
1730}
1731
1732static int kvmppc_handle_nested_exit(struct kvm_vcpu *vcpu)
1733{
1734 struct kvm_nested_guest *nested = vcpu->arch.nested;
1735 int r;
1736 int srcu_idx;
1737
1738 vcpu->stat.sum_exits++;
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748 if (vcpu->arch.shregs.msr & MSR_HV) {
1749 pr_emerg("KVM trap in HV mode while nested!\n");
1750 pr_emerg("trap=0x%x | pc=0x%lx | msr=0x%llx\n",
1751 vcpu->arch.trap, kvmppc_get_pc(vcpu),
1752 vcpu->arch.shregs.msr);
1753 kvmppc_dump_regs(vcpu);
1754 return RESUME_HOST;
1755 }
1756 switch (vcpu->arch.trap) {
1757
1758 case BOOK3S_INTERRUPT_HV_DECREMENTER:
1759 vcpu->stat.dec_exits++;
1760 r = RESUME_GUEST;
1761 break;
1762 case BOOK3S_INTERRUPT_EXTERNAL:
1763 vcpu->stat.ext_intr_exits++;
1764 r = RESUME_HOST;
1765 break;
1766 case BOOK3S_INTERRUPT_H_DOORBELL:
1767 case BOOK3S_INTERRUPT_H_VIRT:
1768 vcpu->stat.ext_intr_exits++;
1769 r = RESUME_GUEST;
1770 break;
1771
1772 case BOOK3S_INTERRUPT_HMI:
1773 case BOOK3S_INTERRUPT_PERFMON:
1774 case BOOK3S_INTERRUPT_SYSTEM_RESET:
1775 r = RESUME_GUEST;
1776 break;
1777 case BOOK3S_INTERRUPT_MACHINE_CHECK:
1778 {
1779 static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL,
1780 DEFAULT_RATELIMIT_BURST);
1781
1782 r = RESUME_HOST;
1783
1784 if (__ratelimit(&rs))
1785 machine_check_print_event_info(&vcpu->arch.mce_evt, false, true);
1786 break;
1787 }
1788
1789
1790
1791
1792
1793
1794 case BOOK3S_INTERRUPT_H_DATA_STORAGE:
1795 srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
1796 r = kvmhv_nested_page_fault(vcpu);
1797 srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
1798 break;
1799 case BOOK3S_INTERRUPT_H_INST_STORAGE:
1800 vcpu->arch.fault_dar = kvmppc_get_pc(vcpu);
1801 vcpu->arch.fault_dsisr = kvmppc_get_msr(vcpu) &
1802 DSISR_SRR1_MATCH_64S;
1803 if (vcpu->arch.shregs.msr & HSRR1_HISI_WRITE)
1804 vcpu->arch.fault_dsisr |= DSISR_ISSTORE;
1805 srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
1806 r = kvmhv_nested_page_fault(vcpu);
1807 srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
1808 break;
1809
1810#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1811 case BOOK3S_INTERRUPT_HV_SOFTPATCH:
1812
1813
1814
1815
1816
1817
1818 r = kvmhv_p9_tm_emulation(vcpu);
1819 if (r != -1)
1820 break;
1821 fallthrough;
1822#endif
1823
1824 case BOOK3S_INTERRUPT_H_FAC_UNAVAIL: {
1825 u64 cause = vcpu->arch.hfscr >> 56;
1826
1827
1828
1829
1830
1831
1832
1833 if (!(vcpu->arch.hfscr_permitted & (1UL << cause)) ||
1834 (nested->hfscr & (1UL << cause))) {
1835 vcpu->arch.trap = BOOK3S_INTERRUPT_H_EMUL_ASSIST;
1836
1837
1838
1839
1840
1841 r = kvmppc_get_last_inst(vcpu, INST_GENERIC,
1842 &vcpu->arch.emul_inst);
1843 if (r != EMULATE_DONE)
1844 r = RESUME_GUEST;
1845 else
1846 r = RESUME_HOST;
1847 } else {
1848 r = RESUME_HOST;
1849 }
1850
1851 break;
1852 }
1853
1854 case BOOK3S_INTERRUPT_HV_RM_HARD:
1855 vcpu->arch.trap = 0;
1856 r = RESUME_GUEST;
1857 if (!xics_on_xive())
1858 kvmppc_xics_rm_complete(vcpu, 0);
1859 break;
1860 case BOOK3S_INTERRUPT_SYSCALL:
1861 {
1862 unsigned long req = kvmppc_get_gpr(vcpu, 3);
1863
1864
1865
1866
1867
1868
1869 if (req == H_RPT_INVALIDATE) {
1870 r = kvmppc_nested_h_rpt_invalidate(vcpu);
1871 break;
1872 }
1873
1874 r = RESUME_HOST;
1875 break;
1876 }
1877 default:
1878 r = RESUME_HOST;
1879 break;
1880 }
1881
1882 return r;
1883}
1884
1885static int kvm_arch_vcpu_ioctl_get_sregs_hv(struct kvm_vcpu *vcpu,
1886 struct kvm_sregs *sregs)
1887{
1888 int i;
1889
1890 memset(sregs, 0, sizeof(struct kvm_sregs));
1891 sregs->pvr = vcpu->arch.pvr;
1892 for (i = 0; i < vcpu->arch.slb_max; i++) {
1893 sregs->u.s.ppc64.slb[i].slbe = vcpu->arch.slb[i].orige;
1894 sregs->u.s.ppc64.slb[i].slbv = vcpu->arch.slb[i].origv;
1895 }
1896
1897 return 0;
1898}
1899
1900static int kvm_arch_vcpu_ioctl_set_sregs_hv(struct kvm_vcpu *vcpu,
1901 struct kvm_sregs *sregs)
1902{
1903 int i, j;
1904
1905
1906 if (sregs->pvr != vcpu->arch.pvr)
1907 return -EINVAL;
1908
1909 j = 0;
1910 for (i = 0; i < vcpu->arch.slb_nr; i++) {
1911 if (sregs->u.s.ppc64.slb[i].slbe & SLB_ESID_V) {
1912 vcpu->arch.slb[j].orige = sregs->u.s.ppc64.slb[i].slbe;
1913 vcpu->arch.slb[j].origv = sregs->u.s.ppc64.slb[i].slbv;
1914 ++j;
1915 }
1916 }
1917 vcpu->arch.slb_max = j;
1918
1919 return 0;
1920}
1921
1922
1923
1924
1925
1926
1927unsigned long kvmppc_filter_lpcr_hv(struct kvm *kvm, unsigned long lpcr)
1928{
1929
1930 if (kvm_is_radix(kvm))
1931 lpcr &= ~LPCR_TC;
1932
1933
1934 if (!cpu_has_feature(CPU_FTR_ARCH_207S))
1935 lpcr &= ~LPCR_AIL;
1936 if ((lpcr & LPCR_AIL) != LPCR_AIL_3)
1937 lpcr &= ~LPCR_AIL;
1938
1939
1940
1941
1942
1943
1944 if (kvm_is_radix(kvm) && cpu_has_feature(CPU_FTR_P9_RADIX_PREFETCH_BUG))
1945 lpcr &= ~LPCR_AIL;
1946
1947
1948
1949
1950
1951 if (!cpu_has_feature(CPU_FTR_ARCH_300))
1952 lpcr &= ~LPCR_LD;
1953
1954 return lpcr;
1955}
1956
1957static void verify_lpcr(struct kvm *kvm, unsigned long lpcr)
1958{
1959 if (lpcr != kvmppc_filter_lpcr_hv(kvm, lpcr)) {
1960 WARN_ONCE(1, "lpcr 0x%lx differs from filtered 0x%lx\n",
1961 lpcr, kvmppc_filter_lpcr_hv(kvm, lpcr));
1962 }
1963}
1964
1965static void kvmppc_set_lpcr(struct kvm_vcpu *vcpu, u64 new_lpcr,
1966 bool preserve_top32)
1967{
1968 struct kvm *kvm = vcpu->kvm;
1969 struct kvmppc_vcore *vc = vcpu->arch.vcore;
1970 u64 mask;
1971
1972 spin_lock(&vc->lock);
1973
1974
1975
1976
1977
1978
1979
1980
1981 mask = LPCR_DPFD | LPCR_ILE | LPCR_TC | LPCR_AIL | LPCR_LD;
1982
1983
1984 if (preserve_top32)
1985 mask &= 0xFFFFFFFF;
1986
1987 new_lpcr = kvmppc_filter_lpcr_hv(kvm,
1988 (vc->lpcr & ~mask) | (new_lpcr & mask));
1989
1990
1991
1992
1993
1994 if ((new_lpcr & LPCR_ILE) != (vc->lpcr & LPCR_ILE)) {
1995 struct kvm_vcpu *vcpu;
1996 int i;
1997
1998 kvm_for_each_vcpu(i, vcpu, kvm) {
1999 if (vcpu->arch.vcore != vc)
2000 continue;
2001 if (new_lpcr & LPCR_ILE)
2002 vcpu->arch.intr_msr |= MSR_LE;
2003 else
2004 vcpu->arch.intr_msr &= ~MSR_LE;
2005 }
2006 }
2007
2008 vc->lpcr = new_lpcr;
2009
2010 spin_unlock(&vc->lock);
2011}
2012
2013static int kvmppc_get_one_reg_hv(struct kvm_vcpu *vcpu, u64 id,
2014 union kvmppc_one_reg *val)
2015{
2016 int r = 0;
2017 long int i;
2018
2019 switch (id) {
2020 case KVM_REG_PPC_DEBUG_INST:
2021 *val = get_reg_val(id, KVMPPC_INST_SW_BREAKPOINT);
2022 break;
2023 case KVM_REG_PPC_HIOR:
2024 *val = get_reg_val(id, 0);
2025 break;
2026 case KVM_REG_PPC_DABR:
2027 *val = get_reg_val(id, vcpu->arch.dabr);
2028 break;
2029 case KVM_REG_PPC_DABRX:
2030 *val = get_reg_val(id, vcpu->arch.dabrx);
2031 break;
2032 case KVM_REG_PPC_DSCR:
2033 *val = get_reg_val(id, vcpu->arch.dscr);
2034 break;
2035 case KVM_REG_PPC_PURR:
2036 *val = get_reg_val(id, vcpu->arch.purr);
2037 break;
2038 case KVM_REG_PPC_SPURR:
2039 *val = get_reg_val(id, vcpu->arch.spurr);
2040 break;
2041 case KVM_REG_PPC_AMR:
2042 *val = get_reg_val(id, vcpu->arch.amr);
2043 break;
2044 case KVM_REG_PPC_UAMOR:
2045 *val = get_reg_val(id, vcpu->arch.uamor);
2046 break;
2047 case KVM_REG_PPC_MMCR0 ... KVM_REG_PPC_MMCR1:
2048 i = id - KVM_REG_PPC_MMCR0;
2049 *val = get_reg_val(id, vcpu->arch.mmcr[i]);
2050 break;
2051 case KVM_REG_PPC_MMCR2:
2052 *val = get_reg_val(id, vcpu->arch.mmcr[2]);
2053 break;
2054 case KVM_REG_PPC_MMCRA:
2055 *val = get_reg_val(id, vcpu->arch.mmcra);
2056 break;
2057 case KVM_REG_PPC_MMCRS:
2058 *val = get_reg_val(id, vcpu->arch.mmcrs);
2059 break;
2060 case KVM_REG_PPC_MMCR3:
2061 *val = get_reg_val(id, vcpu->arch.mmcr[3]);
2062 break;
2063 case KVM_REG_PPC_PMC1 ... KVM_REG_PPC_PMC8:
2064 i = id - KVM_REG_PPC_PMC1;
2065 *val = get_reg_val(id, vcpu->arch.pmc[i]);
2066 break;
2067 case KVM_REG_PPC_SPMC1 ... KVM_REG_PPC_SPMC2:
2068 i = id - KVM_REG_PPC_SPMC1;
2069 *val = get_reg_val(id, vcpu->arch.spmc[i]);
2070 break;
2071 case KVM_REG_PPC_SIAR:
2072 *val = get_reg_val(id, vcpu->arch.siar);
2073 break;
2074 case KVM_REG_PPC_SDAR:
2075 *val = get_reg_val(id, vcpu->arch.sdar);
2076 break;
2077 case KVM_REG_PPC_SIER:
2078 *val = get_reg_val(id, vcpu->arch.sier[0]);
2079 break;
2080 case KVM_REG_PPC_SIER2:
2081 *val = get_reg_val(id, vcpu->arch.sier[1]);
2082 break;
2083 case KVM_REG_PPC_SIER3:
2084 *val = get_reg_val(id, vcpu->arch.sier[2]);
2085 break;
2086 case KVM_REG_PPC_IAMR:
2087 *val = get_reg_val(id, vcpu->arch.iamr);
2088 break;
2089 case KVM_REG_PPC_PSPB:
2090 *val = get_reg_val(id, vcpu->arch.pspb);
2091 break;
2092 case KVM_REG_PPC_DPDES:
2093
2094
2095
2096
2097
2098
2099 *val = get_reg_val(id, vcpu->arch.vcore->dpdes |
2100 vcpu->arch.doorbell_request);
2101 break;
2102 case KVM_REG_PPC_VTB:
2103 *val = get_reg_val(id, vcpu->arch.vcore->vtb);
2104 break;
2105 case KVM_REG_PPC_DAWR:
2106 *val = get_reg_val(id, vcpu->arch.dawr0);
2107 break;
2108 case KVM_REG_PPC_DAWRX:
2109 *val = get_reg_val(id, vcpu->arch.dawrx0);
2110 break;
2111 case KVM_REG_PPC_DAWR1:
2112 *val = get_reg_val(id, vcpu->arch.dawr1);
2113 break;
2114 case KVM_REG_PPC_DAWRX1:
2115 *val = get_reg_val(id, vcpu->arch.dawrx1);
2116 break;
2117 case KVM_REG_PPC_CIABR:
2118 *val = get_reg_val(id, vcpu->arch.ciabr);
2119 break;
2120 case KVM_REG_PPC_CSIGR:
2121 *val = get_reg_val(id, vcpu->arch.csigr);
2122 break;
2123 case KVM_REG_PPC_TACR:
2124 *val = get_reg_val(id, vcpu->arch.tacr);
2125 break;
2126 case KVM_REG_PPC_TCSCR:
2127 *val = get_reg_val(id, vcpu->arch.tcscr);
2128 break;
2129 case KVM_REG_PPC_PID:
2130 *val = get_reg_val(id, vcpu->arch.pid);
2131 break;
2132 case KVM_REG_PPC_ACOP:
2133 *val = get_reg_val(id, vcpu->arch.acop);
2134 break;
2135 case KVM_REG_PPC_WORT:
2136 *val = get_reg_val(id, vcpu->arch.wort);
2137 break;
2138 case KVM_REG_PPC_TIDR:
2139 *val = get_reg_val(id, vcpu->arch.tid);
2140 break;
2141 case KVM_REG_PPC_PSSCR:
2142 *val = get_reg_val(id, vcpu->arch.psscr);
2143 break;
2144 case KVM_REG_PPC_VPA_ADDR:
2145 spin_lock(&vcpu->arch.vpa_update_lock);
2146 *val = get_reg_val(id, vcpu->arch.vpa.next_gpa);
2147 spin_unlock(&vcpu->arch.vpa_update_lock);
2148 break;
2149 case KVM_REG_PPC_VPA_SLB:
2150 spin_lock(&vcpu->arch.vpa_update_lock);
2151 val->vpaval.addr = vcpu->arch.slb_shadow.next_gpa;
2152 val->vpaval.length = vcpu->arch.slb_shadow.len;
2153 spin_unlock(&vcpu->arch.vpa_update_lock);
2154 break;
2155 case KVM_REG_PPC_VPA_DTL:
2156 spin_lock(&vcpu->arch.vpa_update_lock);
2157 val->vpaval.addr = vcpu->arch.dtl.next_gpa;
2158 val->vpaval.length = vcpu->arch.dtl.len;
2159 spin_unlock(&vcpu->arch.vpa_update_lock);
2160 break;
2161 case KVM_REG_PPC_TB_OFFSET:
2162 *val = get_reg_val(id, vcpu->arch.vcore->tb_offset);
2163 break;
2164 case KVM_REG_PPC_LPCR:
2165 case KVM_REG_PPC_LPCR_64:
2166 *val = get_reg_val(id, vcpu->arch.vcore->lpcr);
2167 break;
2168 case KVM_REG_PPC_PPR:
2169 *val = get_reg_val(id, vcpu->arch.ppr);
2170 break;
2171#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
2172 case KVM_REG_PPC_TFHAR:
2173 *val = get_reg_val(id, vcpu->arch.tfhar);
2174 break;
2175 case KVM_REG_PPC_TFIAR:
2176 *val = get_reg_val(id, vcpu->arch.tfiar);
2177 break;
2178 case KVM_REG_PPC_TEXASR:
2179 *val = get_reg_val(id, vcpu->arch.texasr);
2180 break;
2181 case KVM_REG_PPC_TM_GPR0 ... KVM_REG_PPC_TM_GPR31:
2182 i = id - KVM_REG_PPC_TM_GPR0;
2183 *val = get_reg_val(id, vcpu->arch.gpr_tm[i]);
2184 break;
2185 case KVM_REG_PPC_TM_VSR0 ... KVM_REG_PPC_TM_VSR63:
2186 {
2187 int j;
2188 i = id - KVM_REG_PPC_TM_VSR0;
2189 if (i < 32)
2190 for (j = 0; j < TS_FPRWIDTH; j++)
2191 val->vsxval[j] = vcpu->arch.fp_tm.fpr[i][j];
2192 else {
2193 if (cpu_has_feature(CPU_FTR_ALTIVEC))
2194 val->vval = vcpu->arch.vr_tm.vr[i-32];
2195 else
2196 r = -ENXIO;
2197 }
2198 break;
2199 }
2200 case KVM_REG_PPC_TM_CR:
2201 *val = get_reg_val(id, vcpu->arch.cr_tm);
2202 break;
2203 case KVM_REG_PPC_TM_XER:
2204 *val = get_reg_val(id, vcpu->arch.xer_tm);
2205 break;
2206 case KVM_REG_PPC_TM_LR:
2207 *val = get_reg_val(id, vcpu->arch.lr_tm);
2208 break;
2209 case KVM_REG_PPC_TM_CTR:
2210 *val = get_reg_val(id, vcpu->arch.ctr_tm);
2211 break;
2212 case KVM_REG_PPC_TM_FPSCR:
2213 *val = get_reg_val(id, vcpu->arch.fp_tm.fpscr);
2214 break;
2215 case KVM_REG_PPC_TM_AMR:
2216 *val = get_reg_val(id, vcpu->arch.amr_tm);
2217 break;
2218 case KVM_REG_PPC_TM_PPR:
2219 *val = get_reg_val(id, vcpu->arch.ppr_tm);
2220 break;
2221 case KVM_REG_PPC_TM_VRSAVE:
2222 *val = get_reg_val(id, vcpu->arch.vrsave_tm);
2223 break;
2224 case KVM_REG_PPC_TM_VSCR:
2225 if (cpu_has_feature(CPU_FTR_ALTIVEC))
2226 *val = get_reg_val(id, vcpu->arch.vr_tm.vscr.u[3]);
2227 else
2228 r = -ENXIO;
2229 break;
2230 case KVM_REG_PPC_TM_DSCR:
2231 *val = get_reg_val(id, vcpu->arch.dscr_tm);
2232 break;
2233 case KVM_REG_PPC_TM_TAR:
2234 *val = get_reg_val(id, vcpu->arch.tar_tm);
2235 break;
2236#endif
2237 case KVM_REG_PPC_ARCH_COMPAT:
2238 *val = get_reg_val(id, vcpu->arch.vcore->arch_compat);
2239 break;
2240 case KVM_REG_PPC_DEC_EXPIRY:
2241 *val = get_reg_val(id, vcpu->arch.dec_expires +
2242 vcpu->arch.vcore->tb_offset);
2243 break;
2244 case KVM_REG_PPC_ONLINE:
2245 *val = get_reg_val(id, vcpu->arch.online);
2246 break;
2247 case KVM_REG_PPC_PTCR:
2248 *val = get_reg_val(id, vcpu->kvm->arch.l1_ptcr);
2249 break;
2250 default:
2251 r = -EINVAL;
2252 break;
2253 }
2254
2255 return r;
2256}
2257
2258static int kvmppc_set_one_reg_hv(struct kvm_vcpu *vcpu, u64 id,
2259 union kvmppc_one_reg *val)
2260{
2261 int r = 0;
2262 long int i;
2263 unsigned long addr, len;
2264
2265 switch (id) {
2266 case KVM_REG_PPC_HIOR:
2267
2268 if (set_reg_val(id, *val))
2269 r = -EINVAL;
2270 break;
2271 case KVM_REG_PPC_DABR:
2272 vcpu->arch.dabr = set_reg_val(id, *val);
2273 break;
2274 case KVM_REG_PPC_DABRX:
2275 vcpu->arch.dabrx = set_reg_val(id, *val) & ~DABRX_HYP;
2276 break;
2277 case KVM_REG_PPC_DSCR:
2278 vcpu->arch.dscr = set_reg_val(id, *val);
2279 break;
2280 case KVM_REG_PPC_PURR:
2281 vcpu->arch.purr = set_reg_val(id, *val);
2282 break;
2283 case KVM_REG_PPC_SPURR:
2284 vcpu->arch.spurr = set_reg_val(id, *val);
2285 break;
2286 case KVM_REG_PPC_AMR:
2287 vcpu->arch.amr = set_reg_val(id, *val);
2288 break;
2289 case KVM_REG_PPC_UAMOR:
2290 vcpu->arch.uamor = set_reg_val(id, *val);
2291 break;
2292 case KVM_REG_PPC_MMCR0 ... KVM_REG_PPC_MMCR1:
2293 i = id - KVM_REG_PPC_MMCR0;
2294 vcpu->arch.mmcr[i] = set_reg_val(id, *val);
2295 break;
2296 case KVM_REG_PPC_MMCR2:
2297 vcpu->arch.mmcr[2] = set_reg_val(id, *val);
2298 break;
2299 case KVM_REG_PPC_MMCRA:
2300 vcpu->arch.mmcra = set_reg_val(id, *val);
2301 break;
2302 case KVM_REG_PPC_MMCRS:
2303 vcpu->arch.mmcrs = set_reg_val(id, *val);
2304 break;
2305 case KVM_REG_PPC_MMCR3:
2306 *val = get_reg_val(id, vcpu->arch.mmcr[3]);
2307 break;
2308 case KVM_REG_PPC_PMC1 ... KVM_REG_PPC_PMC8:
2309 i = id - KVM_REG_PPC_PMC1;
2310 vcpu->arch.pmc[i] = set_reg_val(id, *val);
2311 break;
2312 case KVM_REG_PPC_SPMC1 ... KVM_REG_PPC_SPMC2:
2313 i = id - KVM_REG_PPC_SPMC1;
2314 vcpu->arch.spmc[i] = set_reg_val(id, *val);
2315 break;
2316 case KVM_REG_PPC_SIAR:
2317 vcpu->arch.siar = set_reg_val(id, *val);
2318 break;
2319 case KVM_REG_PPC_SDAR:
2320 vcpu->arch.sdar = set_reg_val(id, *val);
2321 break;
2322 case KVM_REG_PPC_SIER:
2323 vcpu->arch.sier[0] = set_reg_val(id, *val);
2324 break;
2325 case KVM_REG_PPC_SIER2:
2326 vcpu->arch.sier[1] = set_reg_val(id, *val);
2327 break;
2328 case KVM_REG_PPC_SIER3:
2329 vcpu->arch.sier[2] = set_reg_val(id, *val);
2330 break;
2331 case KVM_REG_PPC_IAMR:
2332 vcpu->arch.iamr = set_reg_val(id, *val);
2333 break;
2334 case KVM_REG_PPC_PSPB:
2335 vcpu->arch.pspb = set_reg_val(id, *val);
2336 break;
2337 case KVM_REG_PPC_DPDES:
2338 vcpu->arch.vcore->dpdes = set_reg_val(id, *val);
2339 break;
2340 case KVM_REG_PPC_VTB:
2341 vcpu->arch.vcore->vtb = set_reg_val(id, *val);
2342 break;
2343 case KVM_REG_PPC_DAWR:
2344 vcpu->arch.dawr0 = set_reg_val(id, *val);
2345 break;
2346 case KVM_REG_PPC_DAWRX:
2347 vcpu->arch.dawrx0 = set_reg_val(id, *val) & ~DAWRX_HYP;
2348 break;
2349 case KVM_REG_PPC_DAWR1:
2350 vcpu->arch.dawr1 = set_reg_val(id, *val);
2351 break;
2352 case KVM_REG_PPC_DAWRX1:
2353 vcpu->arch.dawrx1 = set_reg_val(id, *val) & ~DAWRX_HYP;
2354 break;
2355 case KVM_REG_PPC_CIABR:
2356 vcpu->arch.ciabr = set_reg_val(id, *val);
2357
2358 if ((vcpu->arch.ciabr & CIABR_PRIV) == CIABR_PRIV_HYPER)
2359 vcpu->arch.ciabr &= ~CIABR_PRIV;
2360 break;
2361 case KVM_REG_PPC_CSIGR:
2362 vcpu->arch.csigr = set_reg_val(id, *val);
2363 break;
2364 case KVM_REG_PPC_TACR:
2365 vcpu->arch.tacr = set_reg_val(id, *val);
2366 break;
2367 case KVM_REG_PPC_TCSCR:
2368 vcpu->arch.tcscr = set_reg_val(id, *val);
2369 break;
2370 case KVM_REG_PPC_PID:
2371 vcpu->arch.pid = set_reg_val(id, *val);
2372 break;
2373 case KVM_REG_PPC_ACOP:
2374 vcpu->arch.acop = set_reg_val(id, *val);
2375 break;
2376 case KVM_REG_PPC_WORT:
2377 vcpu->arch.wort = set_reg_val(id, *val);
2378 break;
2379 case KVM_REG_PPC_TIDR:
2380 vcpu->arch.tid = set_reg_val(id, *val);
2381 break;
2382 case KVM_REG_PPC_PSSCR:
2383 vcpu->arch.psscr = set_reg_val(id, *val) & PSSCR_GUEST_VIS;
2384 break;
2385 case KVM_REG_PPC_VPA_ADDR:
2386 addr = set_reg_val(id, *val);
2387 r = -EINVAL;
2388 if (!addr && (vcpu->arch.slb_shadow.next_gpa ||
2389 vcpu->arch.dtl.next_gpa))
2390 break;
2391 r = set_vpa(vcpu, &vcpu->arch.vpa, addr, sizeof(struct lppaca));
2392 break;
2393 case KVM_REG_PPC_VPA_SLB:
2394 addr = val->vpaval.addr;
2395 len = val->vpaval.length;
2396 r = -EINVAL;
2397 if (addr && !vcpu->arch.vpa.next_gpa)
2398 break;
2399 r = set_vpa(vcpu, &vcpu->arch.slb_shadow, addr, len);
2400 break;
2401 case KVM_REG_PPC_VPA_DTL:
2402 addr = val->vpaval.addr;
2403 len = val->vpaval.length;
2404 r = -EINVAL;
2405 if (addr && (len < sizeof(struct dtl_entry) ||
2406 !vcpu->arch.vpa.next_gpa))
2407 break;
2408 len -= len % sizeof(struct dtl_entry);
2409 r = set_vpa(vcpu, &vcpu->arch.dtl, addr, len);
2410 break;
2411 case KVM_REG_PPC_TB_OFFSET:
2412
2413 vcpu->arch.vcore->tb_offset =
2414 ALIGN(set_reg_val(id, *val), 1UL << 24);
2415 break;
2416 case KVM_REG_PPC_LPCR:
2417 kvmppc_set_lpcr(vcpu, set_reg_val(id, *val), true);
2418 break;
2419 case KVM_REG_PPC_LPCR_64:
2420 kvmppc_set_lpcr(vcpu, set_reg_val(id, *val), false);
2421 break;
2422 case KVM_REG_PPC_PPR:
2423 vcpu->arch.ppr = set_reg_val(id, *val);
2424 break;
2425#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
2426 case KVM_REG_PPC_TFHAR:
2427 vcpu->arch.tfhar = set_reg_val(id, *val);
2428 break;
2429 case KVM_REG_PPC_TFIAR:
2430 vcpu->arch.tfiar = set_reg_val(id, *val);
2431 break;
2432 case KVM_REG_PPC_TEXASR:
2433 vcpu->arch.texasr = set_reg_val(id, *val);
2434 break;
2435 case KVM_REG_PPC_TM_GPR0 ... KVM_REG_PPC_TM_GPR31:
2436 i = id - KVM_REG_PPC_TM_GPR0;
2437 vcpu->arch.gpr_tm[i] = set_reg_val(id, *val);
2438 break;
2439 case KVM_REG_PPC_TM_VSR0 ... KVM_REG_PPC_TM_VSR63:
2440 {
2441 int j;
2442 i = id - KVM_REG_PPC_TM_VSR0;
2443 if (i < 32)
2444 for (j = 0; j < TS_FPRWIDTH; j++)
2445 vcpu->arch.fp_tm.fpr[i][j] = val->vsxval[j];
2446 else
2447 if (cpu_has_feature(CPU_FTR_ALTIVEC))
2448 vcpu->arch.vr_tm.vr[i-32] = val->vval;
2449 else
2450 r = -ENXIO;
2451 break;
2452 }
2453 case KVM_REG_PPC_TM_CR:
2454 vcpu->arch.cr_tm = set_reg_val(id, *val);
2455 break;
2456 case KVM_REG_PPC_TM_XER:
2457 vcpu->arch.xer_tm = set_reg_val(id, *val);
2458 break;
2459 case KVM_REG_PPC_TM_LR:
2460 vcpu->arch.lr_tm = set_reg_val(id, *val);
2461 break;
2462 case KVM_REG_PPC_TM_CTR:
2463 vcpu->arch.ctr_tm = set_reg_val(id, *val);
2464 break;
2465 case KVM_REG_PPC_TM_FPSCR:
2466 vcpu->arch.fp_tm.fpscr = set_reg_val(id, *val);
2467 break;
2468 case KVM_REG_PPC_TM_AMR:
2469 vcpu->arch.amr_tm = set_reg_val(id, *val);
2470 break;
2471 case KVM_REG_PPC_TM_PPR:
2472 vcpu->arch.ppr_tm = set_reg_val(id, *val);
2473 break;
2474 case KVM_REG_PPC_TM_VRSAVE:
2475 vcpu->arch.vrsave_tm = set_reg_val(id, *val);
2476 break;
2477 case KVM_REG_PPC_TM_VSCR:
2478 if (cpu_has_feature(CPU_FTR_ALTIVEC))
2479 vcpu->arch.vr.vscr.u[3] = set_reg_val(id, *val);
2480 else
2481 r = - ENXIO;
2482 break;
2483 case KVM_REG_PPC_TM_DSCR:
2484 vcpu->arch.dscr_tm = set_reg_val(id, *val);
2485 break;
2486 case KVM_REG_PPC_TM_TAR:
2487 vcpu->arch.tar_tm = set_reg_val(id, *val);
2488 break;
2489#endif
2490 case KVM_REG_PPC_ARCH_COMPAT:
2491 r = kvmppc_set_arch_compat(vcpu, set_reg_val(id, *val));
2492 break;
2493 case KVM_REG_PPC_DEC_EXPIRY:
2494 vcpu->arch.dec_expires = set_reg_val(id, *val) -
2495 vcpu->arch.vcore->tb_offset;
2496 break;
2497 case KVM_REG_PPC_ONLINE:
2498 i = set_reg_val(id, *val);
2499 if (i && !vcpu->arch.online)
2500 atomic_inc(&vcpu->arch.vcore->online_count);
2501 else if (!i && vcpu->arch.online)
2502 atomic_dec(&vcpu->arch.vcore->online_count);
2503 vcpu->arch.online = i;
2504 break;
2505 case KVM_REG_PPC_PTCR:
2506 vcpu->kvm->arch.l1_ptcr = set_reg_val(id, *val);
2507 break;
2508 default:
2509 r = -EINVAL;
2510 break;
2511 }
2512
2513 return r;
2514}
2515
2516
2517
2518
2519
2520
2521
2522
2523static int threads_per_vcore(struct kvm *kvm)
2524{
2525 if (cpu_has_feature(CPU_FTR_ARCH_300))
2526 return 1;
2527 return threads_per_subcore;
2528}
2529
2530static struct kvmppc_vcore *kvmppc_vcore_create(struct kvm *kvm, int id)
2531{
2532 struct kvmppc_vcore *vcore;
2533
2534 vcore = kzalloc(sizeof(struct kvmppc_vcore), GFP_KERNEL);
2535
2536 if (vcore == NULL)
2537 return NULL;
2538
2539 spin_lock_init(&vcore->lock);
2540 spin_lock_init(&vcore->stoltb_lock);
2541 rcuwait_init(&vcore->wait);
2542 vcore->preempt_tb = TB_NIL;
2543 vcore->lpcr = kvm->arch.lpcr;
2544 vcore->first_vcpuid = id;
2545 vcore->kvm = kvm;
2546 INIT_LIST_HEAD(&vcore->preempt_list);
2547
2548 return vcore;
2549}
2550
2551#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
2552static struct debugfs_timings_element {
2553 const char *name;
2554 size_t offset;
2555} timings[] = {
2556 {"rm_entry", offsetof(struct kvm_vcpu, arch.rm_entry)},
2557 {"rm_intr", offsetof(struct kvm_vcpu, arch.rm_intr)},
2558 {"rm_exit", offsetof(struct kvm_vcpu, arch.rm_exit)},
2559 {"guest", offsetof(struct kvm_vcpu, arch.guest_time)},
2560 {"cede", offsetof(struct kvm_vcpu, arch.cede_time)},
2561};
2562
2563#define N_TIMINGS (ARRAY_SIZE(timings))
2564
2565struct debugfs_timings_state {
2566 struct kvm_vcpu *vcpu;
2567 unsigned int buflen;
2568 char buf[N_TIMINGS * 100];
2569};
2570
2571static int debugfs_timings_open(struct inode *inode, struct file *file)
2572{
2573 struct kvm_vcpu *vcpu = inode->i_private;
2574 struct debugfs_timings_state *p;
2575
2576 p = kzalloc(sizeof(*p), GFP_KERNEL);
2577 if (!p)
2578 return -ENOMEM;
2579
2580 kvm_get_kvm(vcpu->kvm);
2581 p->vcpu = vcpu;
2582 file->private_data = p;
2583
2584 return nonseekable_open(inode, file);
2585}
2586
2587static int debugfs_timings_release(struct inode *inode, struct file *file)
2588{
2589 struct debugfs_timings_state *p = file->private_data;
2590
2591 kvm_put_kvm(p->vcpu->kvm);
2592 kfree(p);
2593 return 0;
2594}
2595
2596static ssize_t debugfs_timings_read(struct file *file, char __user *buf,
2597 size_t len, loff_t *ppos)
2598{
2599 struct debugfs_timings_state *p = file->private_data;
2600 struct kvm_vcpu *vcpu = p->vcpu;
2601 char *s, *buf_end;
2602 struct kvmhv_tb_accumulator tb;
2603 u64 count;
2604 loff_t pos;
2605 ssize_t n;
2606 int i, loops;
2607 bool ok;
2608
2609 if (!p->buflen) {
2610 s = p->buf;
2611 buf_end = s + sizeof(p->buf);
2612 for (i = 0; i < N_TIMINGS; ++i) {
2613 struct kvmhv_tb_accumulator *acc;
2614
2615 acc = (struct kvmhv_tb_accumulator *)
2616 ((unsigned long)vcpu + timings[i].offset);
2617 ok = false;
2618 for (loops = 0; loops < 1000; ++loops) {
2619 count = acc->seqcount;
2620 if (!(count & 1)) {
2621 smp_rmb();
2622 tb = *acc;
2623 smp_rmb();
2624 if (count == acc->seqcount) {
2625 ok = true;
2626 break;
2627 }
2628 }
2629 udelay(1);
2630 }
2631 if (!ok)
2632 snprintf(s, buf_end - s, "%s: stuck\n",
2633 timings[i].name);
2634 else
2635 snprintf(s, buf_end - s,
2636 "%s: %llu %llu %llu %llu\n",
2637 timings[i].name, count / 2,
2638 tb_to_ns(tb.tb_total),
2639 tb_to_ns(tb.tb_min),
2640 tb_to_ns(tb.tb_max));
2641 s += strlen(s);
2642 }
2643 p->buflen = s - p->buf;
2644 }
2645
2646 pos = *ppos;
2647 if (pos >= p->buflen)
2648 return 0;
2649 if (len > p->buflen - pos)
2650 len = p->buflen - pos;
2651 n = copy_to_user(buf, p->buf + pos, len);
2652 if (n) {
2653 if (n == len)
2654 return -EFAULT;
2655 len -= n;
2656 }
2657 *ppos = pos + len;
2658 return len;
2659}
2660
2661static ssize_t debugfs_timings_write(struct file *file, const char __user *buf,
2662 size_t len, loff_t *ppos)
2663{
2664 return -EACCES;
2665}
2666
2667static const struct file_operations debugfs_timings_ops = {
2668 .owner = THIS_MODULE,
2669 .open = debugfs_timings_open,
2670 .release = debugfs_timings_release,
2671 .read = debugfs_timings_read,
2672 .write = debugfs_timings_write,
2673 .llseek = generic_file_llseek,
2674};
2675
2676
2677static void debugfs_vcpu_init(struct kvm_vcpu *vcpu, unsigned int id)
2678{
2679 char buf[16];
2680 struct kvm *kvm = vcpu->kvm;
2681
2682 snprintf(buf, sizeof(buf), "vcpu%u", id);
2683 vcpu->arch.debugfs_dir = debugfs_create_dir(buf, kvm->arch.debugfs_dir);
2684 debugfs_create_file("timings", 0444, vcpu->arch.debugfs_dir, vcpu,
2685 &debugfs_timings_ops);
2686}
2687
2688#else
2689static void debugfs_vcpu_init(struct kvm_vcpu *vcpu, unsigned int id)
2690{
2691}
2692#endif
2693
2694static int kvmppc_core_vcpu_create_hv(struct kvm_vcpu *vcpu)
2695{
2696 int err;
2697 int core;
2698 struct kvmppc_vcore *vcore;
2699 struct kvm *kvm;
2700 unsigned int id;
2701
2702 kvm = vcpu->kvm;
2703 id = vcpu->vcpu_id;
2704
2705 vcpu->arch.shared = &vcpu->arch.shregs;
2706#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
2707
2708
2709
2710
2711#ifdef __BIG_ENDIAN__
2712 vcpu->arch.shared_big_endian = true;
2713#else
2714 vcpu->arch.shared_big_endian = false;
2715#endif
2716#endif
2717 vcpu->arch.mmcr[0] = MMCR0_FC;
2718 vcpu->arch.ctrl = CTRL_RUNLATCH;
2719
2720 kvmppc_set_pvr_hv(vcpu, mfspr(SPRN_PVR));
2721 spin_lock_init(&vcpu->arch.vpa_update_lock);
2722 spin_lock_init(&vcpu->arch.tbacct_lock);
2723 vcpu->arch.busy_preempt = TB_NIL;
2724 vcpu->arch.shregs.msr = MSR_ME;
2725 vcpu->arch.intr_msr = MSR_SF | MSR_ME;
2726
2727
2728
2729
2730
2731
2732
2733
2734 vcpu->arch.hfscr = HFSCR_TAR | HFSCR_EBB | HFSCR_PM | HFSCR_BHRB |
2735 HFSCR_DSCR | HFSCR_VECVSX | HFSCR_FP | HFSCR_PREFIX;
2736 if (cpu_has_feature(CPU_FTR_HVMODE)) {
2737 vcpu->arch.hfscr &= mfspr(SPRN_HFSCR);
2738#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
2739 if (cpu_has_feature(CPU_FTR_P9_TM_HV_ASSIST))
2740 vcpu->arch.hfscr |= HFSCR_TM;
2741#endif
2742 }
2743 if (cpu_has_feature(CPU_FTR_TM_COMP))
2744 vcpu->arch.hfscr |= HFSCR_TM;
2745
2746 vcpu->arch.hfscr_permitted = vcpu->arch.hfscr;
2747
2748 kvmppc_mmu_book3s_hv_init(vcpu);
2749
2750 vcpu->arch.state = KVMPPC_VCPU_NOTREADY;
2751
2752 init_waitqueue_head(&vcpu->arch.cpu_run);
2753
2754 mutex_lock(&kvm->lock);
2755 vcore = NULL;
2756 err = -EINVAL;
2757 if (cpu_has_feature(CPU_FTR_ARCH_300)) {
2758 if (id >= (KVM_MAX_VCPUS * kvm->arch.emul_smt_mode)) {
2759 pr_devel("KVM: VCPU ID too high\n");
2760 core = KVM_MAX_VCORES;
2761 } else {
2762 BUG_ON(kvm->arch.smt_mode != 1);
2763 core = kvmppc_pack_vcpu_id(kvm, id);
2764 }
2765 } else {
2766 core = id / kvm->arch.smt_mode;
2767 }
2768 if (core < KVM_MAX_VCORES) {
2769 vcore = kvm->arch.vcores[core];
2770 if (vcore && cpu_has_feature(CPU_FTR_ARCH_300)) {
2771 pr_devel("KVM: collision on id %u", id);
2772 vcore = NULL;
2773 } else if (!vcore) {
2774
2775
2776
2777
2778 err = -ENOMEM;
2779 vcore = kvmppc_vcore_create(kvm,
2780 id & ~(kvm->arch.smt_mode - 1));
2781 mutex_lock(&kvm->arch.mmu_setup_lock);
2782 kvm->arch.vcores[core] = vcore;
2783 kvm->arch.online_vcores++;
2784 mutex_unlock(&kvm->arch.mmu_setup_lock);
2785 }
2786 }
2787 mutex_unlock(&kvm->lock);
2788
2789 if (!vcore)
2790 return err;
2791
2792 spin_lock(&vcore->lock);
2793 ++vcore->num_threads;
2794 spin_unlock(&vcore->lock);
2795 vcpu->arch.vcore = vcore;
2796 vcpu->arch.ptid = vcpu->vcpu_id - vcore->first_vcpuid;
2797 vcpu->arch.thread_cpu = -1;
2798 vcpu->arch.prev_cpu = -1;
2799
2800 vcpu->arch.cpu_type = KVM_CPU_3S_64;
2801 kvmppc_sanity_check(vcpu);
2802
2803 debugfs_vcpu_init(vcpu, id);
2804
2805 return 0;
2806}
2807
2808static int kvmhv_set_smt_mode(struct kvm *kvm, unsigned long smt_mode,
2809 unsigned long flags)
2810{
2811 int err;
2812 int esmt = 0;
2813
2814 if (flags)
2815 return -EINVAL;
2816 if (smt_mode > MAX_SMT_THREADS || !is_power_of_2(smt_mode))
2817 return -EINVAL;
2818 if (!cpu_has_feature(CPU_FTR_ARCH_300)) {
2819
2820
2821
2822
2823 if (smt_mode > threads_per_subcore)
2824 return -EINVAL;
2825 } else {
2826
2827
2828
2829
2830 esmt = smt_mode;
2831 smt_mode = 1;
2832 }
2833 mutex_lock(&kvm->lock);
2834 err = -EBUSY;
2835 if (!kvm->arch.online_vcores) {
2836 kvm->arch.smt_mode = smt_mode;
2837 kvm->arch.emul_smt_mode = esmt;
2838 err = 0;
2839 }
2840 mutex_unlock(&kvm->lock);
2841
2842 return err;
2843}
2844
2845static void unpin_vpa(struct kvm *kvm, struct kvmppc_vpa *vpa)
2846{
2847 if (vpa->pinned_addr)
2848 kvmppc_unpin_guest_page(kvm, vpa->pinned_addr, vpa->gpa,
2849 vpa->dirty);
2850}
2851
2852static void kvmppc_core_vcpu_free_hv(struct kvm_vcpu *vcpu)
2853{
2854 spin_lock(&vcpu->arch.vpa_update_lock);
2855 unpin_vpa(vcpu->kvm, &vcpu->arch.dtl);
2856 unpin_vpa(vcpu->kvm, &vcpu->arch.slb_shadow);
2857 unpin_vpa(vcpu->kvm, &vcpu->arch.vpa);
2858 spin_unlock(&vcpu->arch.vpa_update_lock);
2859}
2860
2861static int kvmppc_core_check_requests_hv(struct kvm_vcpu *vcpu)
2862{
2863
2864 return 1;
2865}
2866
2867static void kvmppc_set_timer(struct kvm_vcpu *vcpu)
2868{
2869 unsigned long dec_nsec, now;
2870
2871 now = get_tb();
2872 if (now > vcpu->arch.dec_expires) {
2873
2874 kvmppc_core_queue_dec(vcpu);
2875 kvmppc_core_prepare_to_enter(vcpu);
2876 return;
2877 }
2878 dec_nsec = tb_to_ns(vcpu->arch.dec_expires - now);
2879 hrtimer_start(&vcpu->arch.dec_timer, dec_nsec, HRTIMER_MODE_REL);
2880 vcpu->arch.timer_running = 1;
2881}
2882
2883extern int __kvmppc_vcore_entry(void);
2884
2885static void kvmppc_remove_runnable(struct kvmppc_vcore *vc,
2886 struct kvm_vcpu *vcpu)
2887{
2888 u64 now;
2889
2890 if (vcpu->arch.state != KVMPPC_VCPU_RUNNABLE)
2891 return;
2892 spin_lock_irq(&vcpu->arch.tbacct_lock);
2893 now = mftb();
2894 vcpu->arch.busy_stolen += vcore_stolen_time(vc, now) -
2895 vcpu->arch.stolen_logged;
2896 vcpu->arch.busy_preempt = now;
2897 vcpu->arch.state = KVMPPC_VCPU_BUSY_IN_HOST;
2898 spin_unlock_irq(&vcpu->arch.tbacct_lock);
2899 --vc->n_runnable;
2900 WRITE_ONCE(vc->runnable_threads[vcpu->arch.ptid], NULL);
2901}
2902
2903static int kvmppc_grab_hwthread(int cpu)
2904{
2905 struct paca_struct *tpaca;
2906 long timeout = 10000;
2907
2908 tpaca = paca_ptrs[cpu];
2909
2910
2911 tpaca->kvm_hstate.kvm_vcpu = NULL;
2912 tpaca->kvm_hstate.kvm_vcore = NULL;
2913 tpaca->kvm_hstate.napping = 0;
2914 smp_wmb();
2915 tpaca->kvm_hstate.hwthread_req = 1;
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926 smp_mb();
2927 while (tpaca->kvm_hstate.hwthread_state == KVM_HWTHREAD_IN_KERNEL) {
2928 if (--timeout <= 0) {
2929 pr_err("KVM: couldn't grab cpu %d\n", cpu);
2930 return -EBUSY;
2931 }
2932 udelay(1);
2933 }
2934 return 0;
2935}
2936
2937static void kvmppc_release_hwthread(int cpu)
2938{
2939 struct paca_struct *tpaca;
2940
2941 tpaca = paca_ptrs[cpu];
2942 tpaca->kvm_hstate.hwthread_req = 0;
2943 tpaca->kvm_hstate.kvm_vcpu = NULL;
2944 tpaca->kvm_hstate.kvm_vcore = NULL;
2945 tpaca->kvm_hstate.kvm_split_mode = NULL;
2946}
2947
2948static void radix_flush_cpu(struct kvm *kvm, int cpu, struct kvm_vcpu *vcpu)
2949{
2950 struct kvm_nested_guest *nested = vcpu->arch.nested;
2951 cpumask_t *cpu_in_guest;
2952 int i;
2953
2954 cpu = cpu_first_tlb_thread_sibling(cpu);
2955 if (nested) {
2956 cpumask_set_cpu(cpu, &nested->need_tlb_flush);
2957 cpu_in_guest = &nested->cpu_in_guest;
2958 } else {
2959 cpumask_set_cpu(cpu, &kvm->arch.need_tlb_flush);
2960 cpu_in_guest = &kvm->arch.cpu_in_guest;
2961 }
2962
2963
2964
2965
2966
2967 smp_mb();
2968 for (i = cpu; i <= cpu_last_tlb_thread_sibling(cpu);
2969 i += cpu_tlb_thread_sibling_step())
2970 if (cpumask_test_cpu(i, cpu_in_guest))
2971 smp_call_function_single(i, do_nothing, NULL, 1);
2972}
2973
2974static void kvmppc_prepare_radix_vcpu(struct kvm_vcpu *vcpu, int pcpu)
2975{
2976 struct kvm_nested_guest *nested = vcpu->arch.nested;
2977 struct kvm *kvm = vcpu->kvm;
2978 int prev_cpu;
2979
2980 if (!cpu_has_feature(CPU_FTR_HVMODE))
2981 return;
2982
2983 if (nested)
2984 prev_cpu = nested->prev_cpu[vcpu->arch.nested_vcpu_id];
2985 else
2986 prev_cpu = vcpu->arch.prev_cpu;
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000 if (prev_cpu != pcpu) {
3001 if (prev_cpu >= 0 &&
3002 cpu_first_tlb_thread_sibling(prev_cpu) !=
3003 cpu_first_tlb_thread_sibling(pcpu))
3004 radix_flush_cpu(kvm, prev_cpu, vcpu);
3005 if (nested)
3006 nested->prev_cpu[vcpu->arch.nested_vcpu_id] = pcpu;
3007 else
3008 vcpu->arch.prev_cpu = pcpu;
3009 }
3010}
3011
3012static void kvmppc_start_thread(struct kvm_vcpu *vcpu, struct kvmppc_vcore *vc)
3013{
3014 int cpu;
3015 struct paca_struct *tpaca;
3016 struct kvm *kvm = vc->kvm;
3017
3018 cpu = vc->pcpu;
3019 if (vcpu) {
3020 if (vcpu->arch.timer_running) {
3021 hrtimer_try_to_cancel(&vcpu->arch.dec_timer);
3022 vcpu->arch.timer_running = 0;
3023 }
3024 cpu += vcpu->arch.ptid;
3025 vcpu->cpu = vc->pcpu;
3026 vcpu->arch.thread_cpu = cpu;
3027 cpumask_set_cpu(cpu, &kvm->arch.cpu_in_guest);
3028 }
3029 tpaca = paca_ptrs[cpu];
3030 tpaca->kvm_hstate.kvm_vcpu = vcpu;
3031 tpaca->kvm_hstate.ptid = cpu - vc->pcpu;
3032 tpaca->kvm_hstate.fake_suspend = 0;
3033
3034 smp_wmb();
3035 tpaca->kvm_hstate.kvm_vcore = vc;
3036 if (cpu != smp_processor_id())
3037 kvmppc_ipi_thread(cpu);
3038}
3039
3040static void kvmppc_wait_for_nap(int n_threads)
3041{
3042 int cpu = smp_processor_id();
3043 int i, loops;
3044
3045 if (n_threads <= 1)
3046 return;
3047 for (loops = 0; loops < 1000000; ++loops) {
3048
3049
3050
3051
3052
3053
3054 for (i = 1; i < n_threads; ++i)
3055 if (paca_ptrs[cpu + i]->kvm_hstate.kvm_vcore)
3056 break;
3057 if (i == n_threads) {
3058 HMT_medium();
3059 return;
3060 }
3061 HMT_low();
3062 }
3063 HMT_medium();
3064 for (i = 1; i < n_threads; ++i)
3065 if (paca_ptrs[cpu + i]->kvm_hstate.kvm_vcore)
3066 pr_err("KVM: CPU %d seems to be stuck\n", cpu + i);
3067}
3068
3069
3070
3071
3072
3073
3074static int on_primary_thread(void)
3075{
3076 int cpu = smp_processor_id();
3077 int thr;
3078
3079
3080 if (cpu_thread_in_subcore(cpu))
3081 return 0;
3082
3083 thr = 0;
3084 while (++thr < threads_per_subcore)
3085 if (cpu_online(cpu + thr))
3086 return 0;
3087
3088
3089 for (thr = 1; thr < threads_per_subcore; ++thr) {
3090 if (kvmppc_grab_hwthread(cpu + thr)) {
3091
3092 do {
3093 kvmppc_release_hwthread(cpu + thr);
3094 } while (--thr > 0);
3095 return 0;
3096 }
3097 }
3098 return 1;
3099}
3100
3101
3102
3103
3104
3105
3106struct preempted_vcore_list {
3107 struct list_head list;
3108 spinlock_t lock;
3109};
3110
3111static DEFINE_PER_CPU(struct preempted_vcore_list, preempted_vcores);
3112
3113static void init_vcore_lists(void)
3114{
3115 int cpu;
3116
3117 for_each_possible_cpu(cpu) {
3118 struct preempted_vcore_list *lp = &per_cpu(preempted_vcores, cpu);
3119 spin_lock_init(&lp->lock);
3120 INIT_LIST_HEAD(&lp->list);
3121 }
3122}
3123
3124static void kvmppc_vcore_preempt(struct kvmppc_vcore *vc)
3125{
3126 struct preempted_vcore_list *lp = this_cpu_ptr(&preempted_vcores);
3127
3128 vc->vcore_state = VCORE_PREEMPT;
3129 vc->pcpu = smp_processor_id();
3130 if (vc->num_threads < threads_per_vcore(vc->kvm)) {
3131 spin_lock(&lp->lock);
3132 list_add_tail(&vc->preempt_list, &lp->list);
3133 spin_unlock(&lp->lock);
3134 }
3135
3136
3137 kvmppc_core_start_stolen(vc);
3138}
3139
3140static void kvmppc_vcore_end_preempt(struct kvmppc_vcore *vc)
3141{
3142 struct preempted_vcore_list *lp;
3143
3144 kvmppc_core_end_stolen(vc);
3145 if (!list_empty(&vc->preempt_list)) {
3146 lp = &per_cpu(preempted_vcores, vc->pcpu);
3147 spin_lock(&lp->lock);
3148 list_del_init(&vc->preempt_list);
3149 spin_unlock(&lp->lock);
3150 }
3151 vc->vcore_state = VCORE_INACTIVE;
3152}
3153
3154
3155
3156
3157
3158struct core_info {
3159 int n_subcores;
3160 int max_subcore_threads;
3161 int total_threads;
3162 int subcore_threads[MAX_SUBCORES];
3163 struct kvmppc_vcore *vc[MAX_SUBCORES];
3164};
3165
3166
3167
3168
3169
3170static int subcore_thread_map[MAX_SUBCORES] = { 0, 4, 2, 6 };
3171
3172static void init_core_info(struct core_info *cip, struct kvmppc_vcore *vc)
3173{
3174 memset(cip, 0, sizeof(*cip));
3175 cip->n_subcores = 1;
3176 cip->max_subcore_threads = vc->num_threads;
3177 cip->total_threads = vc->num_threads;
3178 cip->subcore_threads[0] = vc->num_threads;
3179 cip->vc[0] = vc;
3180}
3181
3182static bool subcore_config_ok(int n_subcores, int n_threads)
3183{
3184
3185
3186
3187
3188 if (cpu_has_feature(CPU_FTR_ARCH_300))
3189 return n_subcores <= 4 && n_threads == 1;
3190
3191
3192 if (n_subcores > 1 && threads_per_subcore < MAX_SMT_THREADS)
3193 return false;
3194 if (n_subcores > MAX_SUBCORES)
3195 return false;
3196 if (n_subcores > 1) {
3197 if (!(dynamic_mt_modes & 2))
3198 n_subcores = 4;
3199 if (n_subcores > 2 && !(dynamic_mt_modes & 4))
3200 return false;
3201 }
3202
3203 return n_subcores * roundup_pow_of_two(n_threads) <= MAX_SMT_THREADS;
3204}
3205
3206static void init_vcore_to_run(struct kvmppc_vcore *vc)
3207{
3208 vc->entry_exit_map = 0;
3209 vc->in_guest = 0;
3210 vc->napping_threads = 0;
3211 vc->conferring_threads = 0;
3212 vc->tb_offset_applied = 0;
3213}
3214
3215static bool can_dynamic_split(struct kvmppc_vcore *vc, struct core_info *cip)
3216{
3217 int n_threads = vc->num_threads;
3218 int sub;
3219
3220 if (!cpu_has_feature(CPU_FTR_ARCH_207S))
3221 return false;
3222
3223
3224 if (one_vm_per_core && vc->kvm != cip->vc[0]->kvm)
3225 return false;
3226
3227 if (n_threads < cip->max_subcore_threads)
3228 n_threads = cip->max_subcore_threads;
3229 if (!subcore_config_ok(cip->n_subcores + 1, n_threads))
3230 return false;
3231 cip->max_subcore_threads = n_threads;
3232
3233 sub = cip->n_subcores;
3234 ++cip->n_subcores;
3235 cip->total_threads += vc->num_threads;
3236 cip->subcore_threads[sub] = vc->num_threads;
3237 cip->vc[sub] = vc;
3238 init_vcore_to_run(vc);
3239 list_del_init(&vc->preempt_list);
3240
3241 return true;
3242}
3243
3244
3245
3246
3247
3248static bool can_piggyback(struct kvmppc_vcore *pvc, struct core_info *cip,
3249 int target_threads)
3250{
3251 if (cip->total_threads + pvc->num_threads > target_threads)
3252 return false;
3253
3254 return can_dynamic_split(pvc, cip);
3255}
3256
3257static void prepare_threads(struct kvmppc_vcore *vc)
3258{
3259 int i;
3260 struct kvm_vcpu *vcpu;
3261
3262 for_each_runnable_thread(i, vcpu, vc) {
3263 if (signal_pending(vcpu->arch.run_task))
3264 vcpu->arch.ret = -EINTR;
3265 else if (vcpu->arch.vpa.update_pending ||
3266 vcpu->arch.slb_shadow.update_pending ||
3267 vcpu->arch.dtl.update_pending)
3268 vcpu->arch.ret = RESUME_GUEST;
3269 else
3270 continue;
3271 kvmppc_remove_runnable(vc, vcpu);
3272 wake_up(&vcpu->arch.cpu_run);
3273 }
3274}
3275
3276static void collect_piggybacks(struct core_info *cip, int target_threads)
3277{
3278 struct preempted_vcore_list *lp = this_cpu_ptr(&preempted_vcores);
3279 struct kvmppc_vcore *pvc, *vcnext;
3280
3281 spin_lock(&lp->lock);
3282 list_for_each_entry_safe(pvc, vcnext, &lp->list, preempt_list) {
3283 if (!spin_trylock(&pvc->lock))
3284 continue;
3285 prepare_threads(pvc);
3286 if (!pvc->n_runnable || !pvc->kvm->arch.mmu_ready) {
3287 list_del_init(&pvc->preempt_list);
3288 if (pvc->runner == NULL) {
3289 pvc->vcore_state = VCORE_INACTIVE;
3290 kvmppc_core_end_stolen(pvc);
3291 }
3292 spin_unlock(&pvc->lock);
3293 continue;
3294 }
3295 if (!can_piggyback(pvc, cip, target_threads)) {
3296 spin_unlock(&pvc->lock);
3297 continue;
3298 }
3299 kvmppc_core_end_stolen(pvc);
3300 pvc->vcore_state = VCORE_PIGGYBACK;
3301 if (cip->total_threads >= target_threads)
3302 break;
3303 }
3304 spin_unlock(&lp->lock);
3305}
3306
3307static bool recheck_signals_and_mmu(struct core_info *cip)
3308{
3309 int sub, i;
3310 struct kvm_vcpu *vcpu;
3311 struct kvmppc_vcore *vc;
3312
3313 for (sub = 0; sub < cip->n_subcores; ++sub) {
3314 vc = cip->vc[sub];
3315 if (!vc->kvm->arch.mmu_ready)
3316 return true;
3317 for_each_runnable_thread(i, vcpu, vc)
3318 if (signal_pending(vcpu->arch.run_task))
3319 return true;
3320 }
3321 return false;
3322}
3323
3324static void post_guest_process(struct kvmppc_vcore *vc, bool is_master)
3325{
3326 int still_running = 0, i;
3327 u64 now;
3328 long ret;
3329 struct kvm_vcpu *vcpu;
3330
3331 spin_lock(&vc->lock);
3332 now = get_tb();
3333 for_each_runnable_thread(i, vcpu, vc) {
3334
3335
3336
3337
3338
3339
3340
3341 spin_unlock(&vc->lock);
3342
3343 if (now < vcpu->arch.dec_expires &&
3344 kvmppc_core_pending_dec(vcpu))
3345 kvmppc_core_dequeue_dec(vcpu);
3346
3347 trace_kvm_guest_exit(vcpu);
3348
3349 ret = RESUME_GUEST;
3350 if (vcpu->arch.trap)
3351 ret = kvmppc_handle_exit_hv(vcpu,
3352 vcpu->arch.run_task);
3353
3354 vcpu->arch.ret = ret;
3355 vcpu->arch.trap = 0;
3356
3357 spin_lock(&vc->lock);
3358 if (is_kvmppc_resume_guest(vcpu->arch.ret)) {
3359 if (vcpu->arch.pending_exceptions)
3360 kvmppc_core_prepare_to_enter(vcpu);
3361 if (vcpu->arch.ceded)
3362 kvmppc_set_timer(vcpu);
3363 else
3364 ++still_running;
3365 } else {
3366 kvmppc_remove_runnable(vc, vcpu);
3367 wake_up(&vcpu->arch.cpu_run);
3368 }
3369 }
3370 if (!is_master) {
3371 if (still_running > 0) {
3372 kvmppc_vcore_preempt(vc);
3373 } else if (vc->runner) {
3374 vc->vcore_state = VCORE_PREEMPT;
3375 kvmppc_core_start_stolen(vc);
3376 } else {
3377 vc->vcore_state = VCORE_INACTIVE;
3378 }
3379 if (vc->n_runnable > 0 && vc->runner == NULL) {
3380
3381 i = -1;
3382 vcpu = next_runnable_thread(vc, &i);
3383 wake_up(&vcpu->arch.cpu_run);
3384 }
3385 }
3386 spin_unlock(&vc->lock);
3387}
3388
3389
3390
3391
3392
3393
3394static inline int kvmppc_clear_host_core(unsigned int cpu)
3395{
3396 int core;
3397
3398 if (!kvmppc_host_rm_ops_hv || cpu_thread_in_core(cpu))
3399 return 0;
3400
3401
3402
3403
3404
3405 core = cpu >> threads_shift;
3406 kvmppc_host_rm_ops_hv->rm_core[core].rm_state.in_host = 0;
3407 return 0;
3408}
3409
3410
3411
3412
3413
3414
3415static inline int kvmppc_set_host_core(unsigned int cpu)
3416{
3417 int core;
3418
3419 if (!kvmppc_host_rm_ops_hv || cpu_thread_in_core(cpu))
3420 return 0;
3421
3422
3423
3424
3425
3426 core = cpu >> threads_shift;
3427 kvmppc_host_rm_ops_hv->rm_core[core].rm_state.in_host = 1;
3428 return 0;
3429}
3430
3431static void set_irq_happened(int trap)
3432{
3433 switch (trap) {
3434 case BOOK3S_INTERRUPT_EXTERNAL:
3435 local_paca->irq_happened |= PACA_IRQ_EE;
3436 break;
3437 case BOOK3S_INTERRUPT_H_DOORBELL:
3438 local_paca->irq_happened |= PACA_IRQ_DBELL;
3439 break;
3440 case BOOK3S_INTERRUPT_HMI:
3441 local_paca->irq_happened |= PACA_IRQ_HMI;
3442 break;
3443 case BOOK3S_INTERRUPT_SYSTEM_RESET:
3444 replay_system_reset();
3445 break;
3446 }
3447}
3448
3449
3450
3451
3452
3453static noinline void kvmppc_run_core(struct kvmppc_vcore *vc)
3454{
3455 struct kvm_vcpu *vcpu;
3456 int i;
3457 int srcu_idx;
3458 struct core_info core_info;
3459 struct kvmppc_vcore *pvc;
3460 struct kvm_split_mode split_info, *sip;
3461 int split, subcore_size, active;
3462 int sub;
3463 bool thr0_done;
3464 unsigned long cmd_bit, stat_bit;
3465 int pcpu, thr;
3466 int target_threads;
3467 int controlled_threads;
3468 int trap;
3469 bool is_power8;
3470
3471 if (WARN_ON_ONCE(cpu_has_feature(CPU_FTR_ARCH_300)))
3472 return;
3473
3474
3475
3476
3477
3478 prepare_threads(vc);
3479
3480
3481 if (vc->runner->arch.state != KVMPPC_VCPU_RUNNABLE)
3482 return;
3483
3484
3485
3486
3487 init_vcore_to_run(vc);
3488 vc->preempt_tb = TB_NIL;
3489
3490
3491
3492
3493
3494
3495 controlled_threads = threads_per_vcore(vc->kvm);
3496
3497
3498
3499
3500
3501
3502 if ((controlled_threads > 1) &&
3503 ((vc->num_threads > threads_per_subcore) || !on_primary_thread())) {
3504 for_each_runnable_thread(i, vcpu, vc) {
3505 vcpu->arch.ret = -EBUSY;
3506 kvmppc_remove_runnable(vc, vcpu);
3507 wake_up(&vcpu->arch.cpu_run);
3508 }
3509 goto out;
3510 }
3511
3512
3513
3514
3515
3516 init_core_info(&core_info, vc);
3517 pcpu = smp_processor_id();
3518 target_threads = controlled_threads;
3519 if (target_smt_mode && target_smt_mode < target_threads)
3520 target_threads = target_smt_mode;
3521 if (vc->num_threads < target_threads)
3522 collect_piggybacks(&core_info, target_threads);
3523
3524
3525
3526
3527
3528
3529
3530
3531 local_irq_disable();
3532 hard_irq_disable();
3533 if (lazy_irq_pending() || need_resched() ||
3534 recheck_signals_and_mmu(&core_info)) {
3535 local_irq_enable();
3536 vc->vcore_state = VCORE_INACTIVE;
3537
3538 for (sub = 1; sub < core_info.n_subcores; ++sub) {
3539 pvc = core_info.vc[sub];
3540
3541 kvmppc_vcore_preempt(pvc);
3542 spin_unlock(&pvc->lock);
3543 }
3544 for (i = 0; i < controlled_threads; ++i)
3545 kvmppc_release_hwthread(pcpu + i);
3546 return;
3547 }
3548
3549 kvmppc_clear_host_core(pcpu);
3550
3551
3552 subcore_size = threads_per_subcore;
3553 cmd_bit = stat_bit = 0;
3554 split = core_info.n_subcores;
3555 sip = NULL;
3556 is_power8 = cpu_has_feature(CPU_FTR_ARCH_207S);
3557
3558 if (split > 1) {
3559 sip = &split_info;
3560 memset(&split_info, 0, sizeof(split_info));
3561 for (sub = 0; sub < core_info.n_subcores; ++sub)
3562 split_info.vc[sub] = core_info.vc[sub];
3563
3564 if (is_power8) {
3565 if (split == 2 && (dynamic_mt_modes & 2)) {
3566 cmd_bit = HID0_POWER8_1TO2LPAR;
3567 stat_bit = HID0_POWER8_2LPARMODE;
3568 } else {
3569 split = 4;
3570 cmd_bit = HID0_POWER8_1TO4LPAR;
3571 stat_bit = HID0_POWER8_4LPARMODE;
3572 }
3573 subcore_size = MAX_SMT_THREADS / split;
3574 split_info.rpr = mfspr(SPRN_RPR);
3575 split_info.pmmar = mfspr(SPRN_PMMAR);
3576 split_info.ldbar = mfspr(SPRN_LDBAR);
3577 split_info.subcore_size = subcore_size;
3578 } else {
3579 split_info.subcore_size = 1;
3580 }
3581
3582
3583 smp_wmb();
3584 }
3585
3586 for (thr = 0; thr < controlled_threads; ++thr) {
3587 struct paca_struct *paca = paca_ptrs[pcpu + thr];
3588
3589 paca->kvm_hstate.napping = 0;
3590 paca->kvm_hstate.kvm_split_mode = sip;
3591 }
3592
3593
3594 if (cmd_bit) {
3595 unsigned long hid0 = mfspr(SPRN_HID0);
3596
3597 hid0 |= cmd_bit | HID0_POWER8_DYNLPARDIS;
3598 mb();
3599 mtspr(SPRN_HID0, hid0);
3600 isync();
3601 for (;;) {
3602 hid0 = mfspr(SPRN_HID0);
3603 if (hid0 & stat_bit)
3604 break;
3605 cpu_relax();
3606 }
3607 }
3608
3609
3610
3611
3612
3613
3614 if (is_power8) {
3615 unsigned long rwmr_val = RWMR_RPA_P8_8THREAD;
3616 int n_online = atomic_read(&vc->online_count);
3617
3618
3619
3620
3621
3622 if (split == 1 && threads_per_subcore == MAX_SMT_THREADS &&
3623 n_online >= 1 && n_online <= MAX_SMT_THREADS)
3624 rwmr_val = p8_rwmr_values[n_online];
3625 mtspr(SPRN_RWMR, rwmr_val);
3626 }
3627
3628
3629 active = 0;
3630 for (sub = 0; sub < core_info.n_subcores; ++sub) {
3631 thr = is_power8 ? subcore_thread_map[sub] : sub;
3632 thr0_done = false;
3633 active |= 1 << thr;
3634 pvc = core_info.vc[sub];
3635 pvc->pcpu = pcpu + thr;
3636 for_each_runnable_thread(i, vcpu, pvc) {
3637 kvmppc_start_thread(vcpu, pvc);
3638 kvmppc_create_dtl_entry(vcpu, pvc);
3639 trace_kvm_guest_enter(vcpu);
3640 if (!vcpu->arch.ptid)
3641 thr0_done = true;
3642 active |= 1 << (thr + vcpu->arch.ptid);
3643 }
3644
3645
3646
3647
3648 if (!thr0_done)
3649 kvmppc_start_thread(NULL, pvc);
3650 }
3651
3652
3653
3654
3655
3656 smp_mb();
3657
3658
3659
3660
3661
3662
3663 if (cmd_bit) {
3664 split_info.do_nap = 1;
3665 for (thr = 1; thr < threads_per_subcore; ++thr)
3666 if (!(active & (1 << thr)))
3667 kvmppc_ipi_thread(pcpu + thr);
3668 }
3669
3670 vc->vcore_state = VCORE_RUNNING;
3671 preempt_disable();
3672
3673 trace_kvmppc_run_core(vc, 0);
3674
3675 for (sub = 0; sub < core_info.n_subcores; ++sub)
3676 spin_unlock(&core_info.vc[sub]->lock);
3677
3678 guest_enter_irqoff();
3679
3680 srcu_idx = srcu_read_lock(&vc->kvm->srcu);
3681
3682 this_cpu_disable_ftrace();
3683
3684
3685
3686
3687
3688 trace_hardirqs_on();
3689
3690 trap = __kvmppc_vcore_entry();
3691
3692 trace_hardirqs_off();
3693
3694 this_cpu_enable_ftrace();
3695
3696 srcu_read_unlock(&vc->kvm->srcu, srcu_idx);
3697
3698 set_irq_happened(trap);
3699
3700 spin_lock(&vc->lock);
3701
3702 vc->vcore_state = VCORE_EXITING;
3703
3704
3705 kvmppc_wait_for_nap(controlled_threads);
3706
3707
3708 if (cmd_bit) {
3709 unsigned long hid0 = mfspr(SPRN_HID0);
3710 unsigned long loops = 0;
3711
3712 hid0 &= ~HID0_POWER8_DYNLPARDIS;
3713 stat_bit = HID0_POWER8_2LPARMODE | HID0_POWER8_4LPARMODE;
3714 mb();
3715 mtspr(SPRN_HID0, hid0);
3716 isync();
3717 for (;;) {
3718 hid0 = mfspr(SPRN_HID0);
3719 if (!(hid0 & stat_bit))
3720 break;
3721 cpu_relax();
3722 ++loops;
3723 }
3724 split_info.do_nap = 0;
3725 }
3726
3727 kvmppc_set_host_core(pcpu);
3728
3729 guest_exit_irqoff();
3730
3731 local_irq_enable();
3732
3733
3734 for (i = 0; i < controlled_threads; ++i) {
3735 kvmppc_release_hwthread(pcpu + i);
3736 if (sip && sip->napped[i])
3737 kvmppc_ipi_thread(pcpu + i);
3738 cpumask_clear_cpu(pcpu + i, &vc->kvm->arch.cpu_in_guest);
3739 }
3740
3741 spin_unlock(&vc->lock);
3742
3743
3744 smp_mb();
3745
3746 preempt_enable();
3747
3748 for (sub = 0; sub < core_info.n_subcores; ++sub) {
3749 pvc = core_info.vc[sub];
3750 post_guest_process(pvc, pvc == vc);
3751 }
3752
3753 spin_lock(&vc->lock);
3754
3755 out:
3756 vc->vcore_state = VCORE_INACTIVE;
3757 trace_kvmppc_run_core(vc, 1);
3758}
3759
3760static void load_spr_state(struct kvm_vcpu *vcpu)
3761{
3762 mtspr(SPRN_DSCR, vcpu->arch.dscr);
3763 mtspr(SPRN_IAMR, vcpu->arch.iamr);
3764 mtspr(SPRN_PSPB, vcpu->arch.pspb);
3765 mtspr(SPRN_FSCR, vcpu->arch.fscr);
3766 mtspr(SPRN_TAR, vcpu->arch.tar);
3767 mtspr(SPRN_EBBHR, vcpu->arch.ebbhr);
3768 mtspr(SPRN_EBBRR, vcpu->arch.ebbrr);
3769 mtspr(SPRN_BESCR, vcpu->arch.bescr);
3770 mtspr(SPRN_TIDR, vcpu->arch.tid);
3771 mtspr(SPRN_AMR, vcpu->arch.amr);
3772 mtspr(SPRN_UAMOR, vcpu->arch.uamor);
3773
3774
3775
3776
3777
3778
3779
3780
3781 if (!(vcpu->arch.ctrl & 1))
3782 mtspr(SPRN_CTRLT, mfspr(SPRN_CTRLF) & ~1);
3783}
3784
3785static void store_spr_state(struct kvm_vcpu *vcpu)
3786{
3787 vcpu->arch.ctrl = mfspr(SPRN_CTRLF);
3788
3789 vcpu->arch.iamr = mfspr(SPRN_IAMR);
3790 vcpu->arch.pspb = mfspr(SPRN_PSPB);
3791 vcpu->arch.fscr = mfspr(SPRN_FSCR);
3792 vcpu->arch.tar = mfspr(SPRN_TAR);
3793 vcpu->arch.ebbhr = mfspr(SPRN_EBBHR);
3794 vcpu->arch.ebbrr = mfspr(SPRN_EBBRR);
3795 vcpu->arch.bescr = mfspr(SPRN_BESCR);
3796 vcpu->arch.tid = mfspr(SPRN_TIDR);
3797 vcpu->arch.amr = mfspr(SPRN_AMR);
3798 vcpu->arch.uamor = mfspr(SPRN_UAMOR);
3799 vcpu->arch.dscr = mfspr(SPRN_DSCR);
3800}
3801
3802
3803
3804
3805struct p9_host_os_sprs {
3806 unsigned long dscr;
3807 unsigned long tidr;
3808 unsigned long iamr;
3809 unsigned long amr;
3810 unsigned long fscr;
3811};
3812
3813static void save_p9_host_os_sprs(struct p9_host_os_sprs *host_os_sprs)
3814{
3815 host_os_sprs->dscr = mfspr(SPRN_DSCR);
3816 host_os_sprs->tidr = mfspr(SPRN_TIDR);
3817 host_os_sprs->iamr = mfspr(SPRN_IAMR);
3818 host_os_sprs->amr = mfspr(SPRN_AMR);
3819 host_os_sprs->fscr = mfspr(SPRN_FSCR);
3820}
3821
3822
3823static void restore_p9_host_os_sprs(struct kvm_vcpu *vcpu,
3824 struct p9_host_os_sprs *host_os_sprs)
3825{
3826 mtspr(SPRN_PSPB, 0);
3827 mtspr(SPRN_UAMOR, 0);
3828
3829 mtspr(SPRN_DSCR, host_os_sprs->dscr);
3830 mtspr(SPRN_TIDR, host_os_sprs->tidr);
3831 mtspr(SPRN_IAMR, host_os_sprs->iamr);
3832
3833 if (host_os_sprs->amr != vcpu->arch.amr)
3834 mtspr(SPRN_AMR, host_os_sprs->amr);
3835
3836 if (host_os_sprs->fscr != vcpu->arch.fscr)
3837 mtspr(SPRN_FSCR, host_os_sprs->fscr);
3838
3839
3840 if (!(vcpu->arch.ctrl & 1))
3841 mtspr(SPRN_CTRLT, 1);
3842}
3843
3844static inline bool hcall_is_xics(unsigned long req)
3845{
3846 return req == H_EOI || req == H_CPPR || req == H_IPI ||
3847 req == H_IPOLL || req == H_XIRR || req == H_XIRR_X;
3848}
3849
3850
3851
3852
3853static int kvmhv_p9_guest_entry(struct kvm_vcpu *vcpu, u64 time_limit,
3854 unsigned long lpcr)
3855{
3856 struct kvmppc_vcore *vc = vcpu->arch.vcore;
3857 struct p9_host_os_sprs host_os_sprs;
3858 s64 dec;
3859 u64 tb;
3860 int trap, save_pmu;
3861
3862 WARN_ON_ONCE(vcpu->arch.ceded);
3863
3864 dec = mfspr(SPRN_DEC);
3865 tb = mftb();
3866 if (dec < 0)
3867 return BOOK3S_INTERRUPT_HV_DECREMENTER;
3868 local_paca->kvm_hstate.dec_expires = dec + tb;
3869 if (local_paca->kvm_hstate.dec_expires < time_limit)
3870 time_limit = local_paca->kvm_hstate.dec_expires;
3871
3872 save_p9_host_os_sprs(&host_os_sprs);
3873
3874 kvmhv_save_host_pmu();
3875
3876 kvmppc_subcore_enter_guest();
3877
3878 vc->entry_exit_map = 1;
3879 vc->in_guest = 1;
3880
3881 if (vcpu->arch.vpa.pinned_addr) {
3882 struct lppaca *lp = vcpu->arch.vpa.pinned_addr;
3883 u32 yield_count = be32_to_cpu(lp->yield_count) + 1;
3884 lp->yield_count = cpu_to_be32(yield_count);
3885 vcpu->arch.vpa.dirty = 1;
3886 }
3887
3888 if (cpu_has_feature(CPU_FTR_TM) ||
3889 cpu_has_feature(CPU_FTR_P9_TM_HV_ASSIST))
3890 kvmppc_restore_tm_hv(vcpu, vcpu->arch.shregs.msr, true);
3891
3892#ifdef CONFIG_PPC_PSERIES
3893 if (kvmhv_on_pseries()) {
3894 barrier();
3895 if (vcpu->arch.vpa.pinned_addr) {
3896 struct lppaca *lp = vcpu->arch.vpa.pinned_addr;
3897 get_lppaca()->pmcregs_in_use = lp->pmcregs_in_use;
3898 } else {
3899 get_lppaca()->pmcregs_in_use = 1;
3900 }
3901 barrier();
3902 }
3903#endif
3904 kvmhv_load_guest_pmu(vcpu);
3905
3906 msr_check_and_set(MSR_FP | MSR_VEC | MSR_VSX);
3907 load_fp_state(&vcpu->arch.fp);
3908#ifdef CONFIG_ALTIVEC
3909 load_vr_state(&vcpu->arch.vr);
3910#endif
3911 mtspr(SPRN_VRSAVE, vcpu->arch.vrsave);
3912
3913 load_spr_state(vcpu);
3914
3915
3916
3917
3918
3919
3920
3921
3922
3923
3924
3925
3926
3927 mtspr(SPRN_DEC, vcpu->arch.dec_expires - mftb());
3928
3929 if (kvmhv_on_pseries()) {
3930
3931
3932
3933
3934
3935
3936 unsigned long host_psscr;
3937
3938 struct hv_guest_state hvregs;
3939
3940 host_psscr = mfspr(SPRN_PSSCR_PR);
3941 mtspr(SPRN_PSSCR_PR, vcpu->arch.psscr);
3942 kvmhv_save_hv_regs(vcpu, &hvregs);
3943 hvregs.lpcr = lpcr;
3944 vcpu->arch.regs.msr = vcpu->arch.shregs.msr;
3945 hvregs.version = HV_GUEST_STATE_VERSION;
3946 if (vcpu->arch.nested) {
3947 hvregs.lpid = vcpu->arch.nested->shadow_lpid;
3948 hvregs.vcpu_token = vcpu->arch.nested_vcpu_id;
3949 } else {
3950 hvregs.lpid = vcpu->kvm->arch.lpid;
3951 hvregs.vcpu_token = vcpu->vcpu_id;
3952 }
3953 hvregs.hdec_expiry = time_limit;
3954 mtspr(SPRN_DAR, vcpu->arch.shregs.dar);
3955 mtspr(SPRN_DSISR, vcpu->arch.shregs.dsisr);
3956 trap = plpar_hcall_norets(H_ENTER_NESTED, __pa(&hvregs),
3957 __pa(&vcpu->arch.regs));
3958 kvmhv_restore_hv_return_state(vcpu, &hvregs);
3959 vcpu->arch.shregs.msr = vcpu->arch.regs.msr;
3960 vcpu->arch.shregs.dar = mfspr(SPRN_DAR);
3961 vcpu->arch.shregs.dsisr = mfspr(SPRN_DSISR);
3962 vcpu->arch.psscr = mfspr(SPRN_PSSCR_PR);
3963 mtspr(SPRN_PSSCR_PR, host_psscr);
3964
3965
3966 if (trap == BOOK3S_INTERRUPT_SYSCALL && !vcpu->arch.nested &&
3967 kvmppc_get_gpr(vcpu, 3) == H_CEDE) {
3968 kvmppc_cede(vcpu);
3969 kvmppc_set_gpr(vcpu, 3, 0);
3970 trap = 0;
3971 }
3972 } else {
3973 kvmppc_xive_push_vcpu(vcpu);
3974 trap = kvmhv_vcpu_entry_p9(vcpu, time_limit, lpcr);
3975 if (trap == BOOK3S_INTERRUPT_SYSCALL && !vcpu->arch.nested &&
3976 !(vcpu->arch.shregs.msr & MSR_PR)) {
3977 unsigned long req = kvmppc_get_gpr(vcpu, 3);
3978
3979
3980 if (req == H_CEDE) {
3981 kvmppc_cede(vcpu);
3982 kvmppc_xive_rearm_escalation(vcpu);
3983 kvmppc_set_gpr(vcpu, 3, 0);
3984 trap = 0;
3985
3986
3987 } else if (hcall_is_xics(req)) {
3988 int ret;
3989
3990 ret = kvmppc_xive_xics_hcall(vcpu, req);
3991 if (ret != H_TOO_HARD) {
3992 kvmppc_set_gpr(vcpu, 3, ret);
3993 trap = 0;
3994 }
3995 }
3996 }
3997 kvmppc_xive_pull_vcpu(vcpu);
3998
3999 if (kvm_is_radix(vcpu->kvm))
4000 vcpu->arch.slb_max = 0;
4001 }
4002
4003 dec = mfspr(SPRN_DEC);
4004 if (!(lpcr & LPCR_LD))
4005 dec = (s32) dec;
4006 tb = mftb();
4007 vcpu->arch.dec_expires = dec + tb;
4008 vcpu->cpu = -1;
4009 vcpu->arch.thread_cpu = -1;
4010
4011 store_spr_state(vcpu);
4012
4013 restore_p9_host_os_sprs(vcpu, &host_os_sprs);
4014
4015 msr_check_and_set(MSR_FP | MSR_VEC | MSR_VSX);
4016 store_fp_state(&vcpu->arch.fp);
4017#ifdef CONFIG_ALTIVEC
4018 store_vr_state(&vcpu->arch.vr);
4019#endif
4020 vcpu->arch.vrsave = mfspr(SPRN_VRSAVE);
4021
4022 if (cpu_has_feature(CPU_FTR_TM) ||
4023 cpu_has_feature(CPU_FTR_P9_TM_HV_ASSIST))
4024 kvmppc_save_tm_hv(vcpu, vcpu->arch.shregs.msr, true);
4025
4026 save_pmu = 1;
4027 if (vcpu->arch.vpa.pinned_addr) {
4028 struct lppaca *lp = vcpu->arch.vpa.pinned_addr;
4029 u32 yield_count = be32_to_cpu(lp->yield_count) + 1;
4030 lp->yield_count = cpu_to_be32(yield_count);
4031 vcpu->arch.vpa.dirty = 1;
4032 save_pmu = lp->pmcregs_in_use;
4033 }
4034
4035 save_pmu |= nesting_enabled(vcpu->kvm);
4036
4037 kvmhv_save_guest_pmu(vcpu, save_pmu);
4038#ifdef CONFIG_PPC_PSERIES
4039 if (kvmhv_on_pseries()) {
4040 barrier();
4041 get_lppaca()->pmcregs_in_use = ppc_get_pmu_inuse();
4042 barrier();
4043 }
4044#endif
4045
4046 vc->entry_exit_map = 0x101;
4047 vc->in_guest = 0;
4048
4049 mtspr(SPRN_DEC, local_paca->kvm_hstate.dec_expires - mftb());
4050
4051 if (test_irq_work_pending())
4052 set_dec(1);
4053 mtspr(SPRN_SPRG_VDSO_WRITE, local_paca->sprg_vdso);
4054
4055 kvmhv_load_host_pmu();
4056
4057 kvmppc_subcore_exit_guest();
4058
4059 return trap;
4060}
4061
4062
4063
4064
4065
4066static void kvmppc_wait_for_exec(struct kvmppc_vcore *vc,
4067 struct kvm_vcpu *vcpu, int wait_state)
4068{
4069 DEFINE_WAIT(wait);
4070
4071 prepare_to_wait(&vcpu->arch.cpu_run, &wait, wait_state);
4072 if (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE) {
4073 spin_unlock(&vc->lock);
4074 schedule();
4075 spin_lock(&vc->lock);
4076 }
4077 finish_wait(&vcpu->arch.cpu_run, &wait);
4078}
4079
4080static void grow_halt_poll_ns(struct kvmppc_vcore *vc)
4081{
4082 if (!halt_poll_ns_grow)
4083 return;
4084
4085 vc->halt_poll_ns *= halt_poll_ns_grow;
4086 if (vc->halt_poll_ns < halt_poll_ns_grow_start)
4087 vc->halt_poll_ns = halt_poll_ns_grow_start;
4088}
4089
4090static void shrink_halt_poll_ns(struct kvmppc_vcore *vc)
4091{
4092 if (halt_poll_ns_shrink == 0)
4093 vc->halt_poll_ns = 0;
4094 else
4095 vc->halt_poll_ns /= halt_poll_ns_shrink;
4096}
4097
4098#ifdef CONFIG_KVM_XICS
4099static inline bool xive_interrupt_pending(struct kvm_vcpu *vcpu)
4100{
4101 if (!xics_on_xive())
4102 return false;
4103 return vcpu->arch.irq_pending || vcpu->arch.xive_saved_state.pipr <
4104 vcpu->arch.xive_saved_state.cppr;
4105}
4106#else
4107static inline bool xive_interrupt_pending(struct kvm_vcpu *vcpu)
4108{
4109 return false;
4110}
4111#endif
4112
4113static bool kvmppc_vcpu_woken(struct kvm_vcpu *vcpu)
4114{
4115 if (vcpu->arch.pending_exceptions || vcpu->arch.prodded ||
4116 kvmppc_doorbell_pending(vcpu) || xive_interrupt_pending(vcpu))
4117 return true;
4118
4119 return false;
4120}
4121
4122
4123
4124
4125
4126static int kvmppc_vcore_check_block(struct kvmppc_vcore *vc)
4127{
4128 struct kvm_vcpu *vcpu;
4129 int i;
4130
4131 for_each_runnable_thread(i, vcpu, vc) {
4132 if (!vcpu->arch.ceded || kvmppc_vcpu_woken(vcpu))
4133 return 1;
4134 }
4135
4136 return 0;
4137}
4138
4139
4140
4141
4142
4143static void kvmppc_vcore_blocked(struct kvmppc_vcore *vc)
4144{
4145 ktime_t cur, start_poll, start_wait;
4146 int do_sleep = 1;
4147 u64 block_ns;
4148
4149
4150 cur = start_poll = ktime_get();
4151 if (vc->halt_poll_ns) {
4152 ktime_t stop = ktime_add_ns(start_poll, vc->halt_poll_ns);
4153 ++vc->runner->stat.generic.halt_attempted_poll;
4154
4155 vc->vcore_state = VCORE_POLLING;
4156 spin_unlock(&vc->lock);
4157
4158 do {
4159 if (kvmppc_vcore_check_block(vc)) {
4160 do_sleep = 0;
4161 break;
4162 }
4163 cur = ktime_get();
4164 } while (kvm_vcpu_can_poll(cur, stop));
4165
4166 spin_lock(&vc->lock);
4167 vc->vcore_state = VCORE_INACTIVE;
4168
4169 if (!do_sleep) {
4170 ++vc->runner->stat.generic.halt_successful_poll;
4171 goto out;
4172 }
4173 }
4174
4175 prepare_to_rcuwait(&vc->wait);
4176 set_current_state(TASK_INTERRUPTIBLE);
4177 if (kvmppc_vcore_check_block(vc)) {
4178 finish_rcuwait(&vc->wait);
4179 do_sleep = 0;
4180
4181 if (vc->halt_poll_ns)
4182 ++vc->runner->stat.generic.halt_successful_poll;
4183 goto out;
4184 }
4185
4186 start_wait = ktime_get();
4187
4188 vc->vcore_state = VCORE_SLEEPING;
4189 trace_kvmppc_vcore_blocked(vc, 0);
4190 spin_unlock(&vc->lock);
4191 schedule();
4192 finish_rcuwait(&vc->wait);
4193 spin_lock(&vc->lock);
4194 vc->vcore_state = VCORE_INACTIVE;
4195 trace_kvmppc_vcore_blocked(vc, 1);
4196 ++vc->runner->stat.halt_successful_wait;
4197
4198 cur = ktime_get();
4199
4200out:
4201 block_ns = ktime_to_ns(cur) - ktime_to_ns(start_poll);
4202
4203
4204 if (do_sleep) {
4205 vc->runner->stat.generic.halt_wait_ns +=
4206 ktime_to_ns(cur) - ktime_to_ns(start_wait);
4207 KVM_STATS_LOG_HIST_UPDATE(
4208 vc->runner->stat.generic.halt_wait_hist,
4209 ktime_to_ns(cur) - ktime_to_ns(start_wait));
4210
4211 if (vc->halt_poll_ns) {
4212 vc->runner->stat.generic.halt_poll_fail_ns +=
4213 ktime_to_ns(start_wait) -
4214 ktime_to_ns(start_poll);
4215 KVM_STATS_LOG_HIST_UPDATE(
4216 vc->runner->stat.generic.halt_poll_fail_hist,
4217 ktime_to_ns(start_wait) -
4218 ktime_to_ns(start_poll));
4219 }
4220 } else {
4221
4222 if (vc->halt_poll_ns) {
4223 vc->runner->stat.generic.halt_poll_success_ns +=
4224 ktime_to_ns(cur) -
4225 ktime_to_ns(start_poll);
4226 KVM_STATS_LOG_HIST_UPDATE(
4227 vc->runner->stat.generic.halt_poll_success_hist,
4228 ktime_to_ns(cur) - ktime_to_ns(start_poll));
4229 }
4230 }
4231
4232
4233 if (halt_poll_ns) {
4234 if (block_ns <= vc->halt_poll_ns)
4235 ;
4236
4237 else if (vc->halt_poll_ns && block_ns > halt_poll_ns)
4238 shrink_halt_poll_ns(vc);
4239
4240 else if (vc->halt_poll_ns < halt_poll_ns &&
4241 block_ns < halt_poll_ns)
4242 grow_halt_poll_ns(vc);
4243 if (vc->halt_poll_ns > halt_poll_ns)
4244 vc->halt_poll_ns = halt_poll_ns;
4245 } else
4246 vc->halt_poll_ns = 0;
4247
4248 trace_kvmppc_vcore_wakeup(do_sleep, block_ns);
4249}
4250
4251
4252
4253
4254
4255static int kvmhv_setup_mmu(struct kvm_vcpu *vcpu)
4256{
4257 int r = 0;
4258 struct kvm *kvm = vcpu->kvm;
4259
4260 mutex_lock(&kvm->arch.mmu_setup_lock);
4261 if (!kvm->arch.mmu_ready) {
4262 if (!kvm_is_radix(kvm))
4263 r = kvmppc_hv_setup_htab_rma(vcpu);
4264 if (!r) {
4265 if (cpu_has_feature(CPU_FTR_ARCH_300))
4266 kvmppc_setup_partition_table(kvm);
4267 kvm->arch.mmu_ready = 1;
4268 }
4269 }
4270 mutex_unlock(&kvm->arch.mmu_setup_lock);
4271 return r;
4272}
4273
4274static int kvmppc_run_vcpu(struct kvm_vcpu *vcpu)
4275{
4276 struct kvm_run *run = vcpu->run;
4277 int n_ceded, i, r;
4278 struct kvmppc_vcore *vc;
4279 struct kvm_vcpu *v;
4280
4281 trace_kvmppc_run_vcpu_enter(vcpu);
4282
4283 run->exit_reason = 0;
4284 vcpu->arch.ret = RESUME_GUEST;
4285 vcpu->arch.trap = 0;
4286 kvmppc_update_vpas(vcpu);
4287
4288
4289
4290
4291 vc = vcpu->arch.vcore;
4292 spin_lock(&vc->lock);
4293 vcpu->arch.ceded = 0;
4294 vcpu->arch.run_task = current;
4295 vcpu->arch.stolen_logged = vcore_stolen_time(vc, mftb());
4296 vcpu->arch.state = KVMPPC_VCPU_RUNNABLE;
4297 vcpu->arch.busy_preempt = TB_NIL;
4298 WRITE_ONCE(vc->runnable_threads[vcpu->arch.ptid], vcpu);
4299 ++vc->n_runnable;
4300
4301
4302
4303
4304
4305
4306 if (!signal_pending(current)) {
4307 if ((vc->vcore_state == VCORE_PIGGYBACK ||
4308 vc->vcore_state == VCORE_RUNNING) &&
4309 !VCORE_IS_EXITING(vc)) {
4310 kvmppc_create_dtl_entry(vcpu, vc);
4311 kvmppc_start_thread(vcpu, vc);
4312 trace_kvm_guest_enter(vcpu);
4313 } else if (vc->vcore_state == VCORE_SLEEPING) {
4314 rcuwait_wake_up(&vc->wait);
4315 }
4316
4317 }
4318
4319 while (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE &&
4320 !signal_pending(current)) {
4321
4322 if (!vcpu->kvm->arch.mmu_ready) {
4323 spin_unlock(&vc->lock);
4324 r = kvmhv_setup_mmu(vcpu);
4325 spin_lock(&vc->lock);
4326 if (r) {
4327 run->exit_reason = KVM_EXIT_FAIL_ENTRY;
4328 run->fail_entry.
4329 hardware_entry_failure_reason = 0;
4330 vcpu->arch.ret = r;
4331 break;
4332 }
4333 }
4334
4335 if (vc->vcore_state == VCORE_PREEMPT && vc->runner == NULL)
4336 kvmppc_vcore_end_preempt(vc);
4337
4338 if (vc->vcore_state != VCORE_INACTIVE) {
4339 kvmppc_wait_for_exec(vc, vcpu, TASK_INTERRUPTIBLE);
4340 continue;
4341 }
4342 for_each_runnable_thread(i, v, vc) {
4343 kvmppc_core_prepare_to_enter(v);
4344 if (signal_pending(v->arch.run_task)) {
4345 kvmppc_remove_runnable(vc, v);
4346 v->stat.signal_exits++;
4347 v->run->exit_reason = KVM_EXIT_INTR;
4348 v->arch.ret = -EINTR;
4349 wake_up(&v->arch.cpu_run);
4350 }
4351 }
4352 if (!vc->n_runnable || vcpu->arch.state != KVMPPC_VCPU_RUNNABLE)
4353 break;
4354 n_ceded = 0;
4355 for_each_runnable_thread(i, v, vc) {
4356 if (!kvmppc_vcpu_woken(v))
4357 n_ceded += v->arch.ceded;
4358 else
4359 v->arch.ceded = 0;
4360 }
4361 vc->runner = vcpu;
4362 if (n_ceded == vc->n_runnable) {
4363 kvmppc_vcore_blocked(vc);
4364 } else if (need_resched()) {
4365 kvmppc_vcore_preempt(vc);
4366
4367 cond_resched_lock(&vc->lock);
4368 if (vc->vcore_state == VCORE_PREEMPT)
4369 kvmppc_vcore_end_preempt(vc);
4370 } else {
4371 kvmppc_run_core(vc);
4372 }
4373 vc->runner = NULL;
4374 }
4375
4376 while (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE &&
4377 (vc->vcore_state == VCORE_RUNNING ||
4378 vc->vcore_state == VCORE_EXITING ||
4379 vc->vcore_state == VCORE_PIGGYBACK))
4380 kvmppc_wait_for_exec(vc, vcpu, TASK_UNINTERRUPTIBLE);
4381
4382 if (vc->vcore_state == VCORE_PREEMPT && vc->runner == NULL)
4383 kvmppc_vcore_end_preempt(vc);
4384
4385 if (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE) {
4386 kvmppc_remove_runnable(vc, vcpu);
4387 vcpu->stat.signal_exits++;
4388 run->exit_reason = KVM_EXIT_INTR;
4389 vcpu->arch.ret = -EINTR;
4390 }
4391
4392 if (vc->n_runnable && vc->vcore_state == VCORE_INACTIVE) {
4393
4394 i = -1;
4395 v = next_runnable_thread(vc, &i);
4396 wake_up(&v->arch.cpu_run);
4397 }
4398
4399 trace_kvmppc_run_vcpu_exit(vcpu);
4400 spin_unlock(&vc->lock);
4401 return vcpu->arch.ret;
4402}
4403
4404int kvmhv_run_single_vcpu(struct kvm_vcpu *vcpu, u64 time_limit,
4405 unsigned long lpcr)
4406{
4407 struct kvm_run *run = vcpu->run;
4408 int trap, r, pcpu;
4409 int srcu_idx;
4410 struct kvmppc_vcore *vc;
4411 struct kvm *kvm = vcpu->kvm;
4412 struct kvm_nested_guest *nested = vcpu->arch.nested;
4413
4414 trace_kvmppc_run_vcpu_enter(vcpu);
4415
4416 run->exit_reason = 0;
4417 vcpu->arch.ret = RESUME_GUEST;
4418 vcpu->arch.trap = 0;
4419
4420 vc = vcpu->arch.vcore;
4421 vcpu->arch.ceded = 0;
4422 vcpu->arch.run_task = current;
4423 vcpu->arch.stolen_logged = vcore_stolen_time(vc, mftb());
4424 vcpu->arch.state = KVMPPC_VCPU_RUNNABLE;
4425 vcpu->arch.busy_preempt = TB_NIL;
4426 vcpu->arch.last_inst = KVM_INST_FETCH_FAILED;
4427 vc->runnable_threads[0] = vcpu;
4428 vc->n_runnable = 1;
4429 vc->runner = vcpu;
4430
4431
4432 if (!kvm->arch.mmu_ready) {
4433 r = kvmhv_setup_mmu(vcpu);
4434 if (r) {
4435 run->exit_reason = KVM_EXIT_FAIL_ENTRY;
4436 run->fail_entry.hardware_entry_failure_reason = 0;
4437 vcpu->arch.ret = r;
4438 return r;
4439 }
4440 }
4441
4442 if (need_resched())
4443 cond_resched();
4444
4445 kvmppc_update_vpas(vcpu);
4446
4447 init_vcore_to_run(vc);
4448 vc->preempt_tb = TB_NIL;
4449
4450 preempt_disable();
4451 pcpu = smp_processor_id();
4452 vc->pcpu = pcpu;
4453 if (kvm_is_radix(kvm))
4454 kvmppc_prepare_radix_vcpu(vcpu, pcpu);
4455
4456 local_irq_disable();
4457 hard_irq_disable();
4458 if (signal_pending(current))
4459 goto sigpend;
4460 if (lazy_irq_pending() || need_resched() || !kvm->arch.mmu_ready)
4461 goto out;
4462
4463 if (!nested) {
4464 kvmppc_core_prepare_to_enter(vcpu);
4465 if (vcpu->arch.doorbell_request) {
4466 vc->dpdes = 1;
4467 smp_wmb();
4468 vcpu->arch.doorbell_request = 0;
4469 }
4470 if (test_bit(BOOK3S_IRQPRIO_EXTERNAL,
4471 &vcpu->arch.pending_exceptions))
4472 lpcr |= LPCR_MER;
4473 } else if (vcpu->arch.pending_exceptions ||
4474 vcpu->arch.doorbell_request ||
4475 xive_interrupt_pending(vcpu)) {
4476 vcpu->arch.ret = RESUME_HOST;
4477 goto out;
4478 }
4479
4480 kvmppc_clear_host_core(pcpu);
4481
4482 local_paca->kvm_hstate.napping = 0;
4483 local_paca->kvm_hstate.kvm_split_mode = NULL;
4484 kvmppc_start_thread(vcpu, vc);
4485 kvmppc_create_dtl_entry(vcpu, vc);
4486 trace_kvm_guest_enter(vcpu);
4487
4488 vc->vcore_state = VCORE_RUNNING;
4489 trace_kvmppc_run_core(vc, 0);
4490
4491 guest_enter_irqoff();
4492
4493 srcu_idx = srcu_read_lock(&kvm->srcu);
4494
4495 this_cpu_disable_ftrace();
4496
4497
4498 trace_hardirqs_on();
4499
4500 trap = kvmhv_p9_guest_entry(vcpu, time_limit, lpcr);
4501 vcpu->arch.trap = trap;
4502
4503 trace_hardirqs_off();
4504
4505 this_cpu_enable_ftrace();
4506
4507 srcu_read_unlock(&kvm->srcu, srcu_idx);
4508
4509 set_irq_happened(trap);
4510
4511 kvmppc_set_host_core(pcpu);
4512
4513 guest_exit_irqoff();
4514
4515 local_irq_enable();
4516
4517 cpumask_clear_cpu(pcpu, &kvm->arch.cpu_in_guest);
4518
4519 preempt_enable();
4520
4521
4522
4523
4524
4525
4526 if (kvmppc_core_pending_dec(vcpu) &&
4527 ((get_tb() < vcpu->arch.dec_expires) ||
4528 (trap == BOOK3S_INTERRUPT_SYSCALL &&
4529 kvmppc_get_gpr(vcpu, 3) == H_ENTER_NESTED)))
4530 kvmppc_core_dequeue_dec(vcpu);
4531
4532 trace_kvm_guest_exit(vcpu);
4533 r = RESUME_GUEST;
4534 if (trap) {
4535 if (!nested)
4536 r = kvmppc_handle_exit_hv(vcpu, current);
4537 else
4538 r = kvmppc_handle_nested_exit(vcpu);
4539 }
4540 vcpu->arch.ret = r;
4541
4542 if (is_kvmppc_resume_guest(r) && vcpu->arch.ceded &&
4543 !kvmppc_vcpu_woken(vcpu)) {
4544 kvmppc_set_timer(vcpu);
4545 while (vcpu->arch.ceded && !kvmppc_vcpu_woken(vcpu)) {
4546 if (signal_pending(current)) {
4547 vcpu->stat.signal_exits++;
4548 run->exit_reason = KVM_EXIT_INTR;
4549 vcpu->arch.ret = -EINTR;
4550 break;
4551 }
4552 spin_lock(&vc->lock);
4553 kvmppc_vcore_blocked(vc);
4554 spin_unlock(&vc->lock);
4555 }
4556 }
4557 vcpu->arch.ceded = 0;
4558
4559 vc->vcore_state = VCORE_INACTIVE;
4560 trace_kvmppc_run_core(vc, 1);
4561
4562 done:
4563 kvmppc_remove_runnable(vc, vcpu);
4564 trace_kvmppc_run_vcpu_exit(vcpu);
4565
4566 return vcpu->arch.ret;
4567
4568 sigpend:
4569 vcpu->stat.signal_exits++;
4570 run->exit_reason = KVM_EXIT_INTR;
4571 vcpu->arch.ret = -EINTR;
4572 out:
4573 local_irq_enable();
4574 preempt_enable();
4575 goto done;
4576}
4577
4578static int kvmppc_vcpu_run_hv(struct kvm_vcpu *vcpu)
4579{
4580 struct kvm_run *run = vcpu->run;
4581 int r;
4582 int srcu_idx;
4583 unsigned long ebb_regs[3] = {};
4584 unsigned long user_tar = 0;
4585 unsigned int user_vrsave;
4586 struct kvm *kvm;
4587
4588 if (!vcpu->arch.sane) {
4589 run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
4590 return -EINVAL;
4591 }
4592
4593
4594
4595
4596
4597
4598
4599#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
4600 if (cpu_has_feature(CPU_FTR_TM) && current->thread.regs &&
4601 (current->thread.regs->msr & MSR_TM)) {
4602 if (MSR_TM_ACTIVE(current->thread.regs->msr)) {
4603 run->exit_reason = KVM_EXIT_FAIL_ENTRY;
4604 run->fail_entry.hardware_entry_failure_reason = 0;
4605 return -EINVAL;
4606 }
4607
4608 mtmsr(mfmsr() | MSR_TM);
4609 current->thread.tm_tfhar = mfspr(SPRN_TFHAR);
4610 current->thread.tm_tfiar = mfspr(SPRN_TFIAR);
4611 current->thread.tm_texasr = mfspr(SPRN_TEXASR);
4612 current->thread.regs->msr &= ~MSR_TM;
4613 }
4614#endif
4615
4616
4617
4618
4619
4620 if (!vcpu->arch.online) {
4621 atomic_inc(&vcpu->arch.vcore->online_count);
4622 vcpu->arch.online = 1;
4623 }
4624
4625 kvmppc_core_prepare_to_enter(vcpu);
4626
4627
4628 if (signal_pending(current)) {
4629 run->exit_reason = KVM_EXIT_INTR;
4630 return -EINTR;
4631 }
4632
4633 kvm = vcpu->kvm;
4634 atomic_inc(&kvm->arch.vcpus_running);
4635
4636 smp_mb();
4637
4638 flush_all_to_thread(current);
4639
4640
4641 if (cpu_has_feature(CPU_FTR_ARCH_207S)) {
4642 ebb_regs[0] = mfspr(SPRN_EBBHR);
4643 ebb_regs[1] = mfspr(SPRN_EBBRR);
4644 ebb_regs[2] = mfspr(SPRN_BESCR);
4645 user_tar = mfspr(SPRN_TAR);
4646 }
4647 user_vrsave = mfspr(SPRN_VRSAVE);
4648
4649 vcpu->arch.waitp = &vcpu->arch.vcore->wait;
4650 vcpu->arch.pgdir = kvm->mm->pgd;
4651 vcpu->arch.state = KVMPPC_VCPU_BUSY_IN_HOST;
4652
4653 do {
4654 if (cpu_has_feature(CPU_FTR_ARCH_300))
4655 r = kvmhv_run_single_vcpu(vcpu, ~(u64)0,
4656 vcpu->arch.vcore->lpcr);
4657 else
4658 r = kvmppc_run_vcpu(vcpu);
4659
4660 if (run->exit_reason == KVM_EXIT_PAPR_HCALL) {
4661 if (WARN_ON_ONCE(vcpu->arch.shregs.msr & MSR_PR)) {
4662
4663
4664
4665
4666
4667
4668 r = RESUME_GUEST;
4669 continue;
4670 }
4671 trace_kvm_hcall_enter(vcpu);
4672 r = kvmppc_pseries_do_hcall(vcpu);
4673 trace_kvm_hcall_exit(vcpu, r);
4674 kvmppc_core_prepare_to_enter(vcpu);
4675 } else if (r == RESUME_PAGE_FAULT) {
4676 srcu_idx = srcu_read_lock(&kvm->srcu);
4677 r = kvmppc_book3s_hv_page_fault(vcpu,
4678 vcpu->arch.fault_dar, vcpu->arch.fault_dsisr);
4679 srcu_read_unlock(&kvm->srcu, srcu_idx);
4680 } else if (r == RESUME_PASSTHROUGH) {
4681 if (WARN_ON(xics_on_xive()))
4682 r = H_SUCCESS;
4683 else
4684 r = kvmppc_xics_rm_complete(vcpu, 0);
4685 }
4686 } while (is_kvmppc_resume_guest(r));
4687
4688
4689 if (cpu_has_feature(CPU_FTR_ARCH_207S)) {
4690 mtspr(SPRN_EBBHR, ebb_regs[0]);
4691 mtspr(SPRN_EBBRR, ebb_regs[1]);
4692 mtspr(SPRN_BESCR, ebb_regs[2]);
4693 mtspr(SPRN_TAR, user_tar);
4694 }
4695 mtspr(SPRN_VRSAVE, user_vrsave);
4696
4697 vcpu->arch.state = KVMPPC_VCPU_NOTREADY;
4698 atomic_dec(&kvm->arch.vcpus_running);
4699
4700 srr_regs_clobbered();
4701
4702 return r;
4703}
4704
4705static void kvmppc_add_seg_page_size(struct kvm_ppc_one_seg_page_size **sps,
4706 int shift, int sllp)
4707{
4708 (*sps)->page_shift = shift;
4709 (*sps)->slb_enc = sllp;
4710 (*sps)->enc[0].page_shift = shift;
4711 (*sps)->enc[0].pte_enc = kvmppc_pgsize_lp_encoding(shift, shift);
4712
4713
4714
4715 if (shift != 24) {
4716 int penc = kvmppc_pgsize_lp_encoding(shift, 24);
4717 if (penc != -1) {
4718 (*sps)->enc[1].page_shift = 24;
4719 (*sps)->enc[1].pte_enc = penc;
4720 }
4721 }
4722 (*sps)++;
4723}
4724
4725static int kvm_vm_ioctl_get_smmu_info_hv(struct kvm *kvm,
4726 struct kvm_ppc_smmu_info *info)
4727{
4728 struct kvm_ppc_one_seg_page_size *sps;
4729
4730
4731
4732
4733
4734
4735 info->data_keys = 32;
4736 info->instr_keys = cpu_has_feature(CPU_FTR_ARCH_207S) ? 32 : 0;
4737
4738
4739 info->flags = KVM_PPC_PAGE_SIZES_REAL | KVM_PPC_1T_SEGMENTS;
4740 info->slb_size = 32;
4741
4742
4743 sps = &info->sps[0];
4744 kvmppc_add_seg_page_size(&sps, 12, 0);
4745 kvmppc_add_seg_page_size(&sps, 16, SLB_VSID_L | SLB_VSID_LP_01);
4746 kvmppc_add_seg_page_size(&sps, 24, SLB_VSID_L);
4747
4748
4749 if (kvmhv_on_pseries())
4750 info->flags |= KVM_PPC_NO_HASH;
4751
4752 return 0;
4753}
4754
4755
4756
4757
4758static int kvm_vm_ioctl_get_dirty_log_hv(struct kvm *kvm,
4759 struct kvm_dirty_log *log)
4760{
4761 struct kvm_memslots *slots;
4762 struct kvm_memory_slot *memslot;
4763 int i, r;
4764 unsigned long n;
4765 unsigned long *buf, *p;
4766 struct kvm_vcpu *vcpu;
4767
4768 mutex_lock(&kvm->slots_lock);
4769
4770 r = -EINVAL;
4771 if (log->slot >= KVM_USER_MEM_SLOTS)
4772 goto out;
4773
4774 slots = kvm_memslots(kvm);
4775 memslot = id_to_memslot(slots, log->slot);
4776 r = -ENOENT;
4777 if (!memslot || !memslot->dirty_bitmap)
4778 goto out;
4779
4780
4781
4782
4783
4784 n = kvm_dirty_bitmap_bytes(memslot);
4785 buf = memslot->dirty_bitmap + n / sizeof(long);
4786 memset(buf, 0, n);
4787
4788 if (kvm_is_radix(kvm))
4789 r = kvmppc_hv_get_dirty_log_radix(kvm, memslot, buf);
4790 else
4791 r = kvmppc_hv_get_dirty_log_hpt(kvm, memslot, buf);
4792 if (r)
4793 goto out;
4794
4795
4796
4797
4798
4799
4800
4801 p = memslot->dirty_bitmap;
4802 for (i = 0; i < n / sizeof(long); ++i)
4803 buf[i] |= xchg(&p[i], 0);
4804
4805
4806
4807 kvm_for_each_vcpu(i, vcpu, kvm) {
4808 spin_lock(&vcpu->arch.vpa_update_lock);
4809 kvmppc_harvest_vpa_dirty(&vcpu->arch.vpa, memslot, buf);
4810 kvmppc_harvest_vpa_dirty(&vcpu->arch.dtl, memslot, buf);
4811 spin_unlock(&vcpu->arch.vpa_update_lock);
4812 }
4813
4814 r = -EFAULT;
4815 if (copy_to_user(log->dirty_bitmap, buf, n))
4816 goto out;
4817
4818 r = 0;
4819out:
4820 mutex_unlock(&kvm->slots_lock);
4821 return r;
4822}
4823
4824static void kvmppc_core_free_memslot_hv(struct kvm_memory_slot *slot)
4825{
4826 vfree(slot->arch.rmap);
4827 slot->arch.rmap = NULL;
4828}
4829
4830static int kvmppc_core_prepare_memory_region_hv(struct kvm *kvm,
4831 struct kvm_memory_slot *slot,
4832 const struct kvm_userspace_memory_region *mem,
4833 enum kvm_mr_change change)
4834{
4835 unsigned long npages = mem->memory_size >> PAGE_SHIFT;
4836
4837 if (change == KVM_MR_CREATE) {
4838 slot->arch.rmap = vzalloc(array_size(npages,
4839 sizeof(*slot->arch.rmap)));
4840 if (!slot->arch.rmap)
4841 return -ENOMEM;
4842 }
4843
4844 return 0;
4845}
4846
4847static void kvmppc_core_commit_memory_region_hv(struct kvm *kvm,
4848 const struct kvm_userspace_memory_region *mem,
4849 const struct kvm_memory_slot *old,
4850 const struct kvm_memory_slot *new,
4851 enum kvm_mr_change change)
4852{
4853 unsigned long npages = mem->memory_size >> PAGE_SHIFT;
4854
4855
4856
4857
4858
4859
4860
4861 if (npages)
4862 atomic64_inc(&kvm->arch.mmio_update);
4863
4864
4865
4866
4867
4868
4869
4870
4871
4872
4873
4874
4875
4876
4877 if (change == KVM_MR_FLAGS_ONLY && kvm_is_radix(kvm) &&
4878 ((new->flags ^ old->flags) & KVM_MEM_LOG_DIRTY_PAGES))
4879 kvmppc_radix_flush_memslot(kvm, old);
4880
4881
4882
4883 if (!kvm->arch.secure_guest)
4884 return;
4885
4886 switch (change) {
4887 case KVM_MR_CREATE:
4888
4889
4890
4891
4892 kvmppc_uvmem_memslot_create(kvm, new);
4893 break;
4894 case KVM_MR_DELETE:
4895 kvmppc_uvmem_memslot_delete(kvm, old);
4896 break;
4897 default:
4898
4899 break;
4900 }
4901}
4902
4903
4904
4905
4906
4907
4908void kvmppc_update_lpcr(struct kvm *kvm, unsigned long lpcr, unsigned long mask)
4909{
4910 long int i;
4911 u32 cores_done = 0;
4912
4913 if ((kvm->arch.lpcr & mask) == lpcr)
4914 return;
4915
4916 kvm->arch.lpcr = (kvm->arch.lpcr & ~mask) | lpcr;
4917
4918 for (i = 0; i < KVM_MAX_VCORES; ++i) {
4919 struct kvmppc_vcore *vc = kvm->arch.vcores[i];
4920 if (!vc)
4921 continue;
4922
4923 spin_lock(&vc->lock);
4924 vc->lpcr = (vc->lpcr & ~mask) | lpcr;
4925 verify_lpcr(kvm, vc->lpcr);
4926 spin_unlock(&vc->lock);
4927 if (++cores_done >= kvm->arch.online_vcores)
4928 break;
4929 }
4930}
4931
4932void kvmppc_setup_partition_table(struct kvm *kvm)
4933{
4934 unsigned long dw0, dw1;
4935
4936 if (!kvm_is_radix(kvm)) {
4937
4938 dw0 = ((kvm->arch.vrma_slb_v & SLB_VSID_L) >> 1) |
4939 ((kvm->arch.vrma_slb_v & SLB_VSID_LP) << 1);
4940
4941 dw0 |= kvm->arch.sdr1;
4942
4943
4944 dw1 = kvm->arch.process_table;
4945 } else {
4946 dw0 = PATB_HR | radix__get_tree_size() |
4947 __pa(kvm->arch.pgtable) | RADIX_PGD_INDEX_SIZE;
4948 dw1 = PATB_GR | kvm->arch.process_table;
4949 }
4950 kvmhv_set_ptbl_entry(kvm->arch.lpid, dw0, dw1);
4951}
4952
4953
4954
4955
4956
4957static int kvmppc_hv_setup_htab_rma(struct kvm_vcpu *vcpu)
4958{
4959 int err = 0;
4960 struct kvm *kvm = vcpu->kvm;
4961 unsigned long hva;
4962 struct kvm_memory_slot *memslot;
4963 struct vm_area_struct *vma;
4964 unsigned long lpcr = 0, senc;
4965 unsigned long psize, porder;
4966 int srcu_idx;
4967
4968
4969 if (!kvm->arch.hpt.virt) {
4970 int order = KVM_DEFAULT_HPT_ORDER;
4971 struct kvm_hpt_info info;
4972
4973 err = kvmppc_allocate_hpt(&info, order);
4974
4975
4976
4977 while ((err == -ENOMEM) && --order >= PPC_MIN_HPT_ORDER)
4978 err = kvmppc_allocate_hpt(&info, order);
4979
4980 if (err < 0) {
4981 pr_err("KVM: Couldn't alloc HPT\n");
4982 goto out;
4983 }
4984
4985 kvmppc_set_hpt(kvm, &info);
4986 }
4987
4988
4989 srcu_idx = srcu_read_lock(&kvm->srcu);
4990 memslot = gfn_to_memslot(kvm, 0);
4991
4992
4993 err = -EINVAL;
4994 if (!memslot || (memslot->flags & KVM_MEMSLOT_INVALID))
4995 goto out_srcu;
4996
4997
4998 hva = memslot->userspace_addr;
4999 mmap_read_lock(kvm->mm);
5000 vma = vma_lookup(kvm->mm, hva);
5001 if (!vma || (vma->vm_flags & VM_IO))
5002 goto up_out;
5003
5004 psize = vma_kernel_pagesize(vma);
5005
5006 mmap_read_unlock(kvm->mm);
5007
5008
5009 if (psize >= 0x1000000)
5010 psize = 0x1000000;
5011 else if (psize >= 0x10000)
5012 psize = 0x10000;
5013 else
5014 psize = 0x1000;
5015 porder = __ilog2(psize);
5016
5017 senc = slb_pgsize_encoding(psize);
5018 kvm->arch.vrma_slb_v = senc | SLB_VSID_B_1T |
5019 (VRMA_VSID << SLB_VSID_SHIFT_1T);
5020
5021 kvmppc_map_vrma(vcpu, memslot, porder);
5022
5023
5024 if (!cpu_has_feature(CPU_FTR_ARCH_300)) {
5025
5026 lpcr = senc << (LPCR_VRMASD_SH - 4);
5027 kvmppc_update_lpcr(kvm, lpcr, LPCR_VRMASD);
5028 }
5029
5030
5031 smp_wmb();
5032 err = 0;
5033 out_srcu:
5034 srcu_read_unlock(&kvm->srcu, srcu_idx);
5035 out:
5036 return err;
5037
5038 up_out:
5039 mmap_read_unlock(kvm->mm);
5040 goto out_srcu;
5041}
5042
5043
5044
5045
5046
5047int kvmppc_switch_mmu_to_hpt(struct kvm *kvm)
5048{
5049 if (nesting_enabled(kvm))
5050 kvmhv_release_all_nested(kvm);
5051 kvmppc_rmap_reset(kvm);
5052 kvm->arch.process_table = 0;
5053
5054 spin_lock(&kvm->mmu_lock);
5055 kvm->arch.radix = 0;
5056 spin_unlock(&kvm->mmu_lock);
5057 kvmppc_free_radix(kvm);
5058 kvmppc_update_lpcr(kvm, LPCR_VPM1,
5059 LPCR_VPM1 | LPCR_UPRT | LPCR_GTSE | LPCR_HR);
5060 return 0;
5061}
5062
5063
5064
5065
5066
5067int kvmppc_switch_mmu_to_radix(struct kvm *kvm)
5068{
5069 int err;
5070
5071 err = kvmppc_init_vm_radix(kvm);
5072 if (err)
5073 return err;
5074 kvmppc_rmap_reset(kvm);
5075
5076 spin_lock(&kvm->mmu_lock);
5077 kvm->arch.radix = 1;
5078 spin_unlock(&kvm->mmu_lock);
5079 kvmppc_free_hpt(&kvm->arch.hpt);
5080 kvmppc_update_lpcr(kvm, LPCR_UPRT | LPCR_GTSE | LPCR_HR,
5081 LPCR_VPM1 | LPCR_UPRT | LPCR_GTSE | LPCR_HR);
5082 return 0;
5083}
5084
5085#ifdef CONFIG_KVM_XICS
5086
5087
5088
5089
5090
5091
5092
5093
5094
5095
5096void kvmppc_alloc_host_rm_ops(void)
5097{
5098 struct kvmppc_host_rm_ops *ops;
5099 unsigned long l_ops;
5100 int cpu, core;
5101 int size;
5102
5103
5104 if (kvmppc_host_rm_ops_hv != NULL)
5105 return;
5106
5107 ops = kzalloc(sizeof(struct kvmppc_host_rm_ops), GFP_KERNEL);
5108 if (!ops)
5109 return;
5110
5111 size = cpu_nr_cores() * sizeof(struct kvmppc_host_rm_core);
5112 ops->rm_core = kzalloc(size, GFP_KERNEL);
5113
5114 if (!ops->rm_core) {
5115 kfree(ops);
5116 return;
5117 }
5118
5119 cpus_read_lock();
5120
5121 for (cpu = 0; cpu < nr_cpu_ids; cpu += threads_per_core) {
5122 if (!cpu_online(cpu))
5123 continue;
5124
5125 core = cpu >> threads_shift;
5126 ops->rm_core[core].rm_state.in_host = 1;
5127 }
5128
5129 ops->vcpu_kick = kvmppc_fast_vcpu_kick_hv;
5130
5131
5132
5133
5134
5135
5136
5137 smp_wmb();
5138 l_ops = (unsigned long) ops;
5139
5140 if (cmpxchg64((unsigned long *)&kvmppc_host_rm_ops_hv, 0, l_ops)) {
5141 cpus_read_unlock();
5142 kfree(ops->rm_core);
5143 kfree(ops);
5144 return;
5145 }
5146
5147 cpuhp_setup_state_nocalls_cpuslocked(CPUHP_KVM_PPC_BOOK3S_PREPARE,
5148 "ppc/kvm_book3s:prepare",
5149 kvmppc_set_host_core,
5150 kvmppc_clear_host_core);
5151 cpus_read_unlock();
5152}
5153
5154void kvmppc_free_host_rm_ops(void)
5155{
5156 if (kvmppc_host_rm_ops_hv) {
5157 cpuhp_remove_state_nocalls(CPUHP_KVM_PPC_BOOK3S_PREPARE);
5158 kfree(kvmppc_host_rm_ops_hv->rm_core);
5159 kfree(kvmppc_host_rm_ops_hv);
5160 kvmppc_host_rm_ops_hv = NULL;
5161 }
5162}
5163#endif
5164
5165static int kvmppc_core_init_vm_hv(struct kvm *kvm)
5166{
5167 unsigned long lpcr, lpid;
5168 char buf[32];
5169 int ret;
5170
5171 mutex_init(&kvm->arch.uvmem_lock);
5172 INIT_LIST_HEAD(&kvm->arch.uvmem_pfns);
5173 mutex_init(&kvm->arch.mmu_setup_lock);
5174
5175
5176
5177 lpid = kvmppc_alloc_lpid();
5178 if ((long)lpid < 0)
5179 return -ENOMEM;
5180 kvm->arch.lpid = lpid;
5181
5182 kvmppc_alloc_host_rm_ops();
5183
5184 kvmhv_vm_nested_init(kvm);
5185
5186
5187
5188
5189
5190
5191
5192
5193 if (!cpu_has_feature(CPU_FTR_ARCH_300))
5194 cpumask_setall(&kvm->arch.need_tlb_flush);
5195
5196
5197 memcpy(kvm->arch.enabled_hcalls, default_enabled_hcalls,
5198 sizeof(kvm->arch.enabled_hcalls));
5199
5200 if (!cpu_has_feature(CPU_FTR_ARCH_300))
5201 kvm->arch.host_sdr1 = mfspr(SPRN_SDR1);
5202
5203
5204 if (cpu_has_feature(CPU_FTR_HVMODE)) {
5205 kvm->arch.host_lpid = mfspr(SPRN_LPID);
5206 kvm->arch.host_lpcr = lpcr = mfspr(SPRN_LPCR);
5207 lpcr &= LPCR_PECE | LPCR_LPES;
5208 } else {
5209 lpcr = 0;
5210 }
5211 lpcr |= (4UL << LPCR_DPFD_SH) | LPCR_HDICE |
5212 LPCR_VPM0 | LPCR_VPM1;
5213 kvm->arch.vrma_slb_v = SLB_VSID_B_1T |
5214 (VRMA_VSID << SLB_VSID_SHIFT_1T);
5215
5216 if (cpu_has_feature(CPU_FTR_ARCH_207S))
5217 lpcr |= LPCR_ONL;
5218
5219
5220
5221
5222
5223
5224
5225 if (cpu_has_feature(CPU_FTR_ARCH_300)) {
5226 lpcr &= ~LPCR_VPM0;
5227 lpcr |= LPCR_HVICE | LPCR_HEIC;
5228
5229
5230
5231
5232
5233 if (xics_on_xive())
5234 lpcr |= LPCR_LPES;
5235 }
5236
5237
5238
5239
5240 if (radix_enabled()) {
5241 kvm->arch.radix = 1;
5242 kvm->arch.mmu_ready = 1;
5243 lpcr &= ~LPCR_VPM1;
5244 lpcr |= LPCR_UPRT | LPCR_GTSE | LPCR_HR;
5245 ret = kvmppc_init_vm_radix(kvm);
5246 if (ret) {
5247 kvmppc_free_lpid(kvm->arch.lpid);
5248 return ret;
5249 }
5250 kvmppc_setup_partition_table(kvm);
5251 }
5252
5253 verify_lpcr(kvm, lpcr);
5254 kvm->arch.lpcr = lpcr;
5255
5256
5257 kvm->arch.resize_hpt = NULL;
5258
5259
5260
5261
5262
5263 if (cpu_has_feature(CPU_FTR_ARCH_31)) {
5264
5265
5266
5267 kvm->arch.tlb_sets = 1;
5268 } else if (radix_enabled())
5269 kvm->arch.tlb_sets = POWER9_TLB_SETS_RADIX;
5270 else if (cpu_has_feature(CPU_FTR_ARCH_300))
5271 kvm->arch.tlb_sets = POWER9_TLB_SETS_HASH;
5272 else if (cpu_has_feature(CPU_FTR_ARCH_207S))
5273 kvm->arch.tlb_sets = POWER8_TLB_SETS;
5274 else
5275 kvm->arch.tlb_sets = POWER7_TLB_SETS;
5276
5277
5278
5279
5280
5281 if (!cpu_has_feature(CPU_FTR_ARCH_300))
5282 kvm_hv_vm_activated();
5283
5284
5285
5286
5287
5288
5289
5290
5291 if (!cpu_has_feature(CPU_FTR_ARCH_300))
5292 kvm->arch.smt_mode = threads_per_subcore;
5293 else
5294 kvm->arch.smt_mode = 1;
5295 kvm->arch.emul_smt_mode = 1;
5296
5297
5298
5299
5300 snprintf(buf, sizeof(buf), "vm%d", current->pid);
5301 kvm->arch.debugfs_dir = debugfs_create_dir(buf, kvm_debugfs_dir);
5302 kvmppc_mmu_debugfs_init(kvm);
5303 if (radix_enabled())
5304 kvmhv_radix_debugfs_init(kvm);
5305
5306 return 0;
5307}
5308
5309static void kvmppc_free_vcores(struct kvm *kvm)
5310{
5311 long int i;
5312
5313 for (i = 0; i < KVM_MAX_VCORES; ++i)
5314 kfree(kvm->arch.vcores[i]);
5315 kvm->arch.online_vcores = 0;
5316}
5317
5318static void kvmppc_core_destroy_vm_hv(struct kvm *kvm)
5319{
5320 debugfs_remove_recursive(kvm->arch.debugfs_dir);
5321
5322 if (!cpu_has_feature(CPU_FTR_ARCH_300))
5323 kvm_hv_vm_deactivated();
5324
5325 kvmppc_free_vcores(kvm);
5326
5327
5328 if (kvm_is_radix(kvm))
5329 kvmppc_free_radix(kvm);
5330 else
5331 kvmppc_free_hpt(&kvm->arch.hpt);
5332
5333
5334 if (cpu_has_feature(CPU_FTR_ARCH_300)) {
5335 if (nesting_enabled(kvm))
5336 kvmhv_release_all_nested(kvm);
5337 kvm->arch.process_table = 0;
5338 if (kvm->arch.secure_guest)
5339 uv_svm_terminate(kvm->arch.lpid);
5340 kvmhv_set_ptbl_entry(kvm->arch.lpid, 0, 0);
5341 }
5342
5343 kvmppc_free_lpid(kvm->arch.lpid);
5344
5345 kvmppc_free_pimap(kvm);
5346}
5347
5348
5349static int kvmppc_core_emulate_op_hv(struct kvm_vcpu *vcpu,
5350 unsigned int inst, int *advance)
5351{
5352 return EMULATE_FAIL;
5353}
5354
5355static int kvmppc_core_emulate_mtspr_hv(struct kvm_vcpu *vcpu, int sprn,
5356 ulong spr_val)
5357{
5358 return EMULATE_FAIL;
5359}
5360
5361static int kvmppc_core_emulate_mfspr_hv(struct kvm_vcpu *vcpu, int sprn,
5362 ulong *spr_val)
5363{
5364 return EMULATE_FAIL;
5365}
5366
5367static int kvmppc_core_check_processor_compat_hv(void)
5368{
5369 if (cpu_has_feature(CPU_FTR_HVMODE) &&
5370 cpu_has_feature(CPU_FTR_ARCH_206))
5371 return 0;
5372
5373
5374 if (cpu_has_feature(CPU_FTR_ARCH_300) && radix_enabled())
5375 return 0;
5376
5377 return -EIO;
5378}
5379
5380#ifdef CONFIG_KVM_XICS
5381
5382void kvmppc_free_pimap(struct kvm *kvm)
5383{
5384 kfree(kvm->arch.pimap);
5385}
5386
5387static struct kvmppc_passthru_irqmap *kvmppc_alloc_pimap(void)
5388{
5389 return kzalloc(sizeof(struct kvmppc_passthru_irqmap), GFP_KERNEL);
5390}
5391
5392static int kvmppc_set_passthru_irq(struct kvm *kvm, int host_irq, int guest_gsi)
5393{
5394 struct irq_desc *desc;
5395 struct kvmppc_irq_map *irq_map;
5396 struct kvmppc_passthru_irqmap *pimap;
5397 struct irq_chip *chip;
5398 int i, rc = 0;
5399 struct irq_data *host_data;
5400
5401 if (!kvm_irq_bypass)
5402 return 1;
5403
5404 desc = irq_to_desc(host_irq);
5405 if (!desc)
5406 return -EIO;
5407
5408 mutex_lock(&kvm->lock);
5409
5410 pimap = kvm->arch.pimap;
5411 if (pimap == NULL) {
5412
5413 pimap = kvmppc_alloc_pimap();
5414 if (pimap == NULL) {
5415 mutex_unlock(&kvm->lock);
5416 return -ENOMEM;
5417 }
5418 kvm->arch.pimap = pimap;
5419 }
5420
5421
5422
5423
5424
5425
5426 chip = irq_data_get_irq_chip(&desc->irq_data);
5427 if (!chip || !is_pnv_opal_msi(chip)) {
5428 pr_warn("kvmppc_set_passthru_irq_hv: Could not assign IRQ map for (%d,%d)\n",
5429 host_irq, guest_gsi);
5430 mutex_unlock(&kvm->lock);
5431 return -ENOENT;
5432 }
5433
5434
5435
5436
5437
5438
5439 for (i = 0; i < pimap->n_mapped; i++) {
5440 if (guest_gsi == pimap->mapped[i].v_hwirq) {
5441 if (pimap->mapped[i].r_hwirq) {
5442 mutex_unlock(&kvm->lock);
5443 return -EINVAL;
5444 }
5445 break;
5446 }
5447 }
5448
5449 if (i == KVMPPC_PIRQ_MAPPED) {
5450 mutex_unlock(&kvm->lock);
5451 return -EAGAIN;
5452 }
5453
5454 irq_map = &pimap->mapped[i];
5455
5456 irq_map->v_hwirq = guest_gsi;
5457 irq_map->desc = desc;
5458
5459
5460
5461
5462
5463 smp_wmb();
5464
5465
5466
5467
5468
5469
5470 host_data = irq_domain_get_irq_data(irq_get_default_host(), host_irq);
5471 irq_map->r_hwirq = (unsigned int)irqd_to_hwirq(host_data);
5472
5473 if (i == pimap->n_mapped)
5474 pimap->n_mapped++;
5475
5476 if (xics_on_xive())
5477 rc = kvmppc_xive_set_mapped(kvm, guest_gsi, host_irq);
5478 else
5479 kvmppc_xics_set_mapped(kvm, guest_gsi, irq_map->r_hwirq);
5480 if (rc)
5481 irq_map->r_hwirq = 0;
5482
5483 mutex_unlock(&kvm->lock);
5484
5485 return 0;
5486}
5487
5488static int kvmppc_clr_passthru_irq(struct kvm *kvm, int host_irq, int guest_gsi)
5489{
5490 struct irq_desc *desc;
5491 struct kvmppc_passthru_irqmap *pimap;
5492 int i, rc = 0;
5493
5494 if (!kvm_irq_bypass)
5495 return 0;
5496
5497 desc = irq_to_desc(host_irq);
5498 if (!desc)
5499 return -EIO;
5500
5501 mutex_lock(&kvm->lock);
5502 if (!kvm->arch.pimap)
5503 goto unlock;
5504
5505 pimap = kvm->arch.pimap;
5506
5507 for (i = 0; i < pimap->n_mapped; i++) {
5508 if (guest_gsi == pimap->mapped[i].v_hwirq)
5509 break;
5510 }
5511
5512 if (i == pimap->n_mapped) {
5513 mutex_unlock(&kvm->lock);
5514 return -ENODEV;
5515 }
5516
5517 if (xics_on_xive())
5518 rc = kvmppc_xive_clr_mapped(kvm, guest_gsi, host_irq);
5519 else
5520 kvmppc_xics_clr_mapped(kvm, guest_gsi, pimap->mapped[i].r_hwirq);
5521
5522
5523 pimap->mapped[i].r_hwirq = 0;
5524
5525
5526
5527
5528
5529 unlock:
5530 mutex_unlock(&kvm->lock);
5531 return rc;
5532}
5533
5534static int kvmppc_irq_bypass_add_producer_hv(struct irq_bypass_consumer *cons,
5535 struct irq_bypass_producer *prod)
5536{
5537 int ret = 0;
5538 struct kvm_kernel_irqfd *irqfd =
5539 container_of(cons, struct kvm_kernel_irqfd, consumer);
5540
5541 irqfd->producer = prod;
5542
5543 ret = kvmppc_set_passthru_irq(irqfd->kvm, prod->irq, irqfd->gsi);
5544 if (ret)
5545 pr_info("kvmppc_set_passthru_irq (irq %d, gsi %d) fails: %d\n",
5546 prod->irq, irqfd->gsi, ret);
5547
5548 return ret;
5549}
5550
5551static void kvmppc_irq_bypass_del_producer_hv(struct irq_bypass_consumer *cons,
5552 struct irq_bypass_producer *prod)
5553{
5554 int ret;
5555 struct kvm_kernel_irqfd *irqfd =
5556 container_of(cons, struct kvm_kernel_irqfd, consumer);
5557
5558 irqfd->producer = NULL;
5559
5560
5561
5562
5563
5564
5565 ret = kvmppc_clr_passthru_irq(irqfd->kvm, prod->irq, irqfd->gsi);
5566 if (ret)
5567 pr_warn("kvmppc_clr_passthru_irq (irq %d, gsi %d) fails: %d\n",
5568 prod->irq, irqfd->gsi, ret);
5569}
5570#endif
5571
5572static long kvm_arch_vm_ioctl_hv(struct file *filp,
5573 unsigned int ioctl, unsigned long arg)
5574{
5575 struct kvm *kvm __maybe_unused = filp->private_data;
5576 void __user *argp = (void __user *)arg;
5577 long r;
5578
5579 switch (ioctl) {
5580
5581 case KVM_PPC_ALLOCATE_HTAB: {
5582 u32 htab_order;
5583
5584
5585 if (kvmhv_on_pseries()) {
5586 r = -EOPNOTSUPP;
5587 break;
5588 }
5589
5590 r = -EFAULT;
5591 if (get_user(htab_order, (u32 __user *)argp))
5592 break;
5593 r = kvmppc_alloc_reset_hpt(kvm, htab_order);
5594 if (r)
5595 break;
5596 r = 0;
5597 break;
5598 }
5599
5600 case KVM_PPC_GET_HTAB_FD: {
5601 struct kvm_get_htab_fd ghf;
5602
5603 r = -EFAULT;
5604 if (copy_from_user(&ghf, argp, sizeof(ghf)))
5605 break;
5606 r = kvm_vm_ioctl_get_htab_fd(kvm, &ghf);
5607 break;
5608 }
5609
5610 case KVM_PPC_RESIZE_HPT_PREPARE: {
5611 struct kvm_ppc_resize_hpt rhpt;
5612
5613 r = -EFAULT;
5614 if (copy_from_user(&rhpt, argp, sizeof(rhpt)))
5615 break;
5616
5617 r = kvm_vm_ioctl_resize_hpt_prepare(kvm, &rhpt);
5618 break;
5619 }
5620
5621 case KVM_PPC_RESIZE_HPT_COMMIT: {
5622 struct kvm_ppc_resize_hpt rhpt;
5623
5624 r = -EFAULT;
5625 if (copy_from_user(&rhpt, argp, sizeof(rhpt)))
5626 break;
5627
5628 r = kvm_vm_ioctl_resize_hpt_commit(kvm, &rhpt);
5629 break;
5630 }
5631
5632 default:
5633 r = -ENOTTY;
5634 }
5635
5636 return r;
5637}
5638
5639
5640
5641
5642
5643
5644
5645static unsigned int default_hcall_list[] = {
5646 H_REMOVE,
5647 H_ENTER,
5648 H_READ,
5649 H_PROTECT,
5650 H_BULK_REMOVE,
5651#ifdef CONFIG_SPAPR_TCE_IOMMU
5652 H_GET_TCE,
5653 H_PUT_TCE,
5654#endif
5655 H_SET_DABR,
5656 H_SET_XDABR,
5657 H_CEDE,
5658 H_PROD,
5659 H_CONFER,
5660 H_REGISTER_VPA,
5661#ifdef CONFIG_KVM_XICS
5662 H_EOI,
5663 H_CPPR,
5664 H_IPI,
5665 H_IPOLL,
5666 H_XIRR,
5667 H_XIRR_X,
5668#endif
5669 0
5670};
5671
5672static void init_default_hcalls(void)
5673{
5674 int i;
5675 unsigned int hcall;
5676
5677 for (i = 0; default_hcall_list[i]; ++i) {
5678 hcall = default_hcall_list[i];
5679 WARN_ON(!kvmppc_hcall_impl_hv(hcall));
5680 __set_bit(hcall / 4, default_enabled_hcalls);
5681 }
5682}
5683
5684static int kvmhv_configure_mmu(struct kvm *kvm, struct kvm_ppc_mmuv3_cfg *cfg)
5685{
5686 unsigned long lpcr;
5687 int radix;
5688 int err;
5689
5690
5691 if (!cpu_has_feature(CPU_FTR_ARCH_300))
5692 return -ENODEV;
5693
5694
5695 if (cfg->flags & ~(KVM_PPC_MMUV3_RADIX | KVM_PPC_MMUV3_GTSE))
5696 return -EINVAL;
5697
5698
5699 radix = !!(cfg->flags & KVM_PPC_MMUV3_RADIX);
5700 if (!!(cfg->process_table & PATB_GR) != radix)
5701 return -EINVAL;
5702
5703
5704 if ((cfg->process_table & PRTS_MASK) > 24)
5705 return -EINVAL;
5706
5707
5708 if (radix && !radix_enabled())
5709 return -EINVAL;
5710
5711
5712 if (kvmhv_on_pseries() && !radix)
5713 return -EINVAL;
5714
5715 mutex_lock(&kvm->arch.mmu_setup_lock);
5716 if (radix != kvm_is_radix(kvm)) {
5717 if (kvm->arch.mmu_ready) {
5718 kvm->arch.mmu_ready = 0;
5719
5720 smp_mb();
5721 if (atomic_read(&kvm->arch.vcpus_running)) {
5722 kvm->arch.mmu_ready = 1;
5723 err = -EBUSY;
5724 goto out_unlock;
5725 }
5726 }
5727 if (radix)
5728 err = kvmppc_switch_mmu_to_radix(kvm);
5729 else
5730 err = kvmppc_switch_mmu_to_hpt(kvm);
5731 if (err)
5732 goto out_unlock;
5733 }
5734
5735 kvm->arch.process_table = cfg->process_table;
5736 kvmppc_setup_partition_table(kvm);
5737
5738 lpcr = (cfg->flags & KVM_PPC_MMUV3_GTSE) ? LPCR_GTSE : 0;
5739 kvmppc_update_lpcr(kvm, lpcr, LPCR_GTSE);
5740 err = 0;
5741
5742 out_unlock:
5743 mutex_unlock(&kvm->arch.mmu_setup_lock);
5744 return err;
5745}
5746
5747static int kvmhv_enable_nested(struct kvm *kvm)
5748{
5749 if (!nested)
5750 return -EPERM;
5751 if (!cpu_has_feature(CPU_FTR_ARCH_300))
5752 return -ENODEV;
5753 if (!radix_enabled())
5754 return -ENODEV;
5755
5756
5757 if (kvm)
5758 kvm->arch.nested_enable = true;
5759 return 0;
5760}
5761
5762static int kvmhv_load_from_eaddr(struct kvm_vcpu *vcpu, ulong *eaddr, void *ptr,
5763 int size)
5764{
5765 int rc = -EINVAL;
5766
5767 if (kvmhv_vcpu_is_radix(vcpu)) {
5768 rc = kvmhv_copy_from_guest_radix(vcpu, *eaddr, ptr, size);
5769
5770 if (rc > 0)
5771 rc = -EINVAL;
5772 }
5773
5774
5775 if (rc && vcpu->arch.nested)
5776 rc = -EAGAIN;
5777
5778 return rc;
5779}
5780
5781static int kvmhv_store_to_eaddr(struct kvm_vcpu *vcpu, ulong *eaddr, void *ptr,
5782 int size)
5783{
5784 int rc = -EINVAL;
5785
5786 if (kvmhv_vcpu_is_radix(vcpu)) {
5787 rc = kvmhv_copy_to_guest_radix(vcpu, *eaddr, ptr, size);
5788
5789 if (rc > 0)
5790 rc = -EINVAL;
5791 }
5792
5793
5794 if (rc && vcpu->arch.nested)
5795 rc = -EAGAIN;
5796
5797 return rc;
5798}
5799
5800static void unpin_vpa_reset(struct kvm *kvm, struct kvmppc_vpa *vpa)
5801{
5802 unpin_vpa(kvm, vpa);
5803 vpa->gpa = 0;
5804 vpa->pinned_addr = NULL;
5805 vpa->dirty = false;
5806 vpa->update_pending = 0;
5807}
5808
5809
5810
5811
5812
5813
5814
5815static int kvmhv_enable_svm(struct kvm *kvm)
5816{
5817 if (!kvmppc_uvmem_available())
5818 return -EINVAL;
5819 if (kvm)
5820 kvm->arch.svm_enabled = 1;
5821 return 0;
5822}
5823
5824
5825
5826
5827
5828
5829
5830
5831
5832static int kvmhv_svm_off(struct kvm *kvm)
5833{
5834 struct kvm_vcpu *vcpu;
5835 int mmu_was_ready;
5836 int srcu_idx;
5837 int ret = 0;
5838 int i;
5839
5840 if (!(kvm->arch.secure_guest & KVMPPC_SECURE_INIT_START))
5841 return ret;
5842
5843 mutex_lock(&kvm->arch.mmu_setup_lock);
5844 mmu_was_ready = kvm->arch.mmu_ready;
5845 if (kvm->arch.mmu_ready) {
5846 kvm->arch.mmu_ready = 0;
5847
5848 smp_mb();
5849 if (atomic_read(&kvm->arch.vcpus_running)) {
5850 kvm->arch.mmu_ready = 1;
5851 ret = -EBUSY;
5852 goto out;
5853 }
5854 }
5855
5856 srcu_idx = srcu_read_lock(&kvm->srcu);
5857 for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
5858 struct kvm_memory_slot *memslot;
5859 struct kvm_memslots *slots = __kvm_memslots(kvm, i);
5860
5861 if (!slots)
5862 continue;
5863
5864 kvm_for_each_memslot(memslot, slots) {
5865 kvmppc_uvmem_drop_pages(memslot, kvm, true);
5866 uv_unregister_mem_slot(kvm->arch.lpid, memslot->id);
5867 }
5868 }
5869 srcu_read_unlock(&kvm->srcu, srcu_idx);
5870
5871 ret = uv_svm_terminate(kvm->arch.lpid);
5872 if (ret != U_SUCCESS) {
5873 ret = -EINVAL;
5874 goto out;
5875 }
5876
5877
5878
5879
5880
5881
5882
5883
5884
5885
5886
5887 kvm_for_each_vcpu(i, vcpu, kvm) {
5888 spin_lock(&vcpu->arch.vpa_update_lock);
5889 unpin_vpa_reset(kvm, &vcpu->arch.dtl);
5890 unpin_vpa_reset(kvm, &vcpu->arch.slb_shadow);
5891 unpin_vpa_reset(kvm, &vcpu->arch.vpa);
5892 spin_unlock(&vcpu->arch.vpa_update_lock);
5893 }
5894
5895 kvmppc_setup_partition_table(kvm);
5896 kvm->arch.secure_guest = 0;
5897 kvm->arch.mmu_ready = mmu_was_ready;
5898out:
5899 mutex_unlock(&kvm->arch.mmu_setup_lock);
5900 return ret;
5901}
5902
5903static int kvmhv_enable_dawr1(struct kvm *kvm)
5904{
5905 if (!cpu_has_feature(CPU_FTR_DAWR1))
5906 return -ENODEV;
5907
5908
5909 if (kvm)
5910 kvm->arch.dawr1_enabled = true;
5911 return 0;
5912}
5913
5914static bool kvmppc_hash_v3_possible(void)
5915{
5916 if (!cpu_has_feature(CPU_FTR_ARCH_300))
5917 return false;
5918
5919 if (!cpu_has_feature(CPU_FTR_HVMODE))
5920 return false;
5921
5922
5923
5924
5925
5926 if (radix_enabled()) {
5927 unsigned int pvr = mfspr(SPRN_PVR);
5928 if ((pvr >> 16) == PVR_POWER9 &&
5929 (((pvr & 0xe000) == 0 && (pvr & 0xfff) < 0x202) ||
5930 ((pvr & 0xe000) == 0x2000 && (pvr & 0xfff) < 0x101)))
5931 return false;
5932 }
5933
5934 return true;
5935}
5936
5937static struct kvmppc_ops kvm_ops_hv = {
5938 .get_sregs = kvm_arch_vcpu_ioctl_get_sregs_hv,
5939 .set_sregs = kvm_arch_vcpu_ioctl_set_sregs_hv,
5940 .get_one_reg = kvmppc_get_one_reg_hv,
5941 .set_one_reg = kvmppc_set_one_reg_hv,
5942 .vcpu_load = kvmppc_core_vcpu_load_hv,
5943 .vcpu_put = kvmppc_core_vcpu_put_hv,
5944 .inject_interrupt = kvmppc_inject_interrupt_hv,
5945 .set_msr = kvmppc_set_msr_hv,
5946 .vcpu_run = kvmppc_vcpu_run_hv,
5947 .vcpu_create = kvmppc_core_vcpu_create_hv,
5948 .vcpu_free = kvmppc_core_vcpu_free_hv,
5949 .check_requests = kvmppc_core_check_requests_hv,
5950 .get_dirty_log = kvm_vm_ioctl_get_dirty_log_hv,
5951 .flush_memslot = kvmppc_core_flush_memslot_hv,
5952 .prepare_memory_region = kvmppc_core_prepare_memory_region_hv,
5953 .commit_memory_region = kvmppc_core_commit_memory_region_hv,
5954 .unmap_gfn_range = kvm_unmap_gfn_range_hv,
5955 .age_gfn = kvm_age_gfn_hv,
5956 .test_age_gfn = kvm_test_age_gfn_hv,
5957 .set_spte_gfn = kvm_set_spte_gfn_hv,
5958 .free_memslot = kvmppc_core_free_memslot_hv,
5959 .init_vm = kvmppc_core_init_vm_hv,
5960 .destroy_vm = kvmppc_core_destroy_vm_hv,
5961 .get_smmu_info = kvm_vm_ioctl_get_smmu_info_hv,
5962 .emulate_op = kvmppc_core_emulate_op_hv,
5963 .emulate_mtspr = kvmppc_core_emulate_mtspr_hv,
5964 .emulate_mfspr = kvmppc_core_emulate_mfspr_hv,
5965 .fast_vcpu_kick = kvmppc_fast_vcpu_kick_hv,
5966 .arch_vm_ioctl = kvm_arch_vm_ioctl_hv,
5967 .hcall_implemented = kvmppc_hcall_impl_hv,
5968#ifdef CONFIG_KVM_XICS
5969 .irq_bypass_add_producer = kvmppc_irq_bypass_add_producer_hv,
5970 .irq_bypass_del_producer = kvmppc_irq_bypass_del_producer_hv,
5971#endif
5972 .configure_mmu = kvmhv_configure_mmu,
5973 .get_rmmu_info = kvmhv_get_rmmu_info,
5974 .set_smt_mode = kvmhv_set_smt_mode,
5975 .enable_nested = kvmhv_enable_nested,
5976 .load_from_eaddr = kvmhv_load_from_eaddr,
5977 .store_to_eaddr = kvmhv_store_to_eaddr,
5978 .enable_svm = kvmhv_enable_svm,
5979 .svm_off = kvmhv_svm_off,
5980 .enable_dawr1 = kvmhv_enable_dawr1,
5981 .hash_v3_possible = kvmppc_hash_v3_possible,
5982};
5983
5984static int kvm_init_subcore_bitmap(void)
5985{
5986 int i, j;
5987 int nr_cores = cpu_nr_cores();
5988 struct sibling_subcore_state *sibling_subcore_state;
5989
5990 for (i = 0; i < nr_cores; i++) {
5991 int first_cpu = i * threads_per_core;
5992 int node = cpu_to_node(first_cpu);
5993
5994
5995 if (paca_ptrs[first_cpu]->sibling_subcore_state)
5996 continue;
5997
5998 sibling_subcore_state =
5999 kzalloc_node(sizeof(struct sibling_subcore_state),
6000 GFP_KERNEL, node);
6001 if (!sibling_subcore_state)
6002 return -ENOMEM;
6003
6004
6005 for (j = 0; j < threads_per_core; j++) {
6006 int cpu = first_cpu + j;
6007
6008 paca_ptrs[cpu]->sibling_subcore_state =
6009 sibling_subcore_state;
6010 }
6011 }
6012 return 0;
6013}
6014
6015static int kvmppc_radix_possible(void)
6016{
6017 return cpu_has_feature(CPU_FTR_ARCH_300) && radix_enabled();
6018}
6019
6020static int kvmppc_book3s_init_hv(void)
6021{
6022 int r;
6023
6024 if (!tlbie_capable) {
6025 pr_err("KVM-HV: Host does not support TLBIE\n");
6026 return -ENODEV;
6027 }
6028
6029
6030
6031
6032 r = kvmppc_core_check_processor_compat_hv();
6033 if (r < 0)
6034 return -ENODEV;
6035
6036 r = kvmhv_nested_init();
6037 if (r)
6038 return r;
6039
6040 r = kvm_init_subcore_bitmap();
6041 if (r)
6042 return r;
6043
6044
6045
6046
6047
6048
6049#ifdef CONFIG_SMP
6050 if (!xics_on_xive() && !kvmhv_on_pseries() &&
6051 !local_paca->kvm_hstate.xics_phys) {
6052 struct device_node *np;
6053
6054 np = of_find_compatible_node(NULL, NULL, "ibm,opal-intc");
6055 if (!np) {
6056 pr_err("KVM-HV: Cannot determine method for accessing XICS\n");
6057 return -ENODEV;
6058 }
6059
6060 of_node_put(np);
6061 }
6062#endif
6063
6064 kvm_ops_hv.owner = THIS_MODULE;
6065 kvmppc_hv_ops = &kvm_ops_hv;
6066
6067 init_default_hcalls();
6068
6069 init_vcore_lists();
6070
6071 r = kvmppc_mmu_hv_init();
6072 if (r)
6073 return r;
6074
6075 if (kvmppc_radix_possible())
6076 r = kvmppc_radix_init();
6077
6078 r = kvmppc_uvmem_init();
6079 if (r < 0)
6080 pr_err("KVM-HV: kvmppc_uvmem_init failed %d\n", r);
6081
6082 return r;
6083}
6084
6085static void kvmppc_book3s_exit_hv(void)
6086{
6087 kvmppc_uvmem_free();
6088 kvmppc_free_host_rm_ops();
6089 if (kvmppc_radix_possible())
6090 kvmppc_radix_exit();
6091 kvmppc_hv_ops = NULL;
6092 kvmhv_nested_exit();
6093}
6094
6095module_init(kvmppc_book3s_init_hv);
6096module_exit(kvmppc_book3s_exit_hv);
6097MODULE_LICENSE("GPL");
6098MODULE_ALIAS_MISCDEV(KVM_MINOR);
6099MODULE_ALIAS("devname:kvm");
6100