1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21#include <linux/kvm_host.h>
22#include <linux/kernel.h>
23#include <linux/err.h>
24#include <linux/slab.h>
25#include <linux/preempt.h>
26#include <linux/sched/signal.h>
27#include <linux/sched/stat.h>
28#include <linux/delay.h>
29#include <linux/export.h>
30#include <linux/fs.h>
31#include <linux/anon_inodes.h>
32#include <linux/cpu.h>
33#include <linux/cpumask.h>
34#include <linux/spinlock.h>
35#include <linux/page-flags.h>
36#include <linux/srcu.h>
37#include <linux/miscdevice.h>
38#include <linux/debugfs.h>
39#include <linux/gfp.h>
40#include <linux/vmalloc.h>
41#include <linux/highmem.h>
42#include <linux/hugetlb.h>
43#include <linux/kvm_irqfd.h>
44#include <linux/irqbypass.h>
45#include <linux/module.h>
46#include <linux/compiler.h>
47#include <linux/of.h>
48
49#include <asm/ftrace.h>
50#include <asm/reg.h>
51#include <asm/ppc-opcode.h>
52#include <asm/asm-prototypes.h>
53#include <asm/debug.h>
54#include <asm/disassemble.h>
55#include <asm/cputable.h>
56#include <asm/cacheflush.h>
57#include <linux/uaccess.h>
58#include <asm/io.h>
59#include <asm/kvm_ppc.h>
60#include <asm/kvm_book3s.h>
61#include <asm/mmu_context.h>
62#include <asm/lppaca.h>
63#include <asm/processor.h>
64#include <asm/cputhreads.h>
65#include <asm/page.h>
66#include <asm/hvcall.h>
67#include <asm/switch_to.h>
68#include <asm/smp.h>
69#include <asm/dbell.h>
70#include <asm/hmi.h>
71#include <asm/pnv-pci.h>
72#include <asm/mmu.h>
73#include <asm/opal.h>
74#include <asm/xics.h>
75#include <asm/xive.h>
76
77#include "book3s.h"
78
79#define CREATE_TRACE_POINTS
80#include "trace_hv.h"
81
82
83
84
85
86
87#define RESUME_PAGE_FAULT (RESUME_GUEST | RESUME_FLAG_ARCH1)
88
89#define RESUME_PASSTHROUGH (RESUME_GUEST | RESUME_FLAG_ARCH2)
90
91
92#define TB_NIL (~(u64)0)
93
94static DECLARE_BITMAP(default_enabled_hcalls, MAX_HCALL_OPCODE/4 + 1);
95
96static int dynamic_mt_modes = 6;
97module_param(dynamic_mt_modes, int, 0644);
98MODULE_PARM_DESC(dynamic_mt_modes, "Set of allowed dynamic micro-threading modes: 0 (= none), 2, 4, or 6 (= 2 or 4)");
99static int target_smt_mode;
100module_param(target_smt_mode, int, 0644);
101MODULE_PARM_DESC(target_smt_mode, "Target threads per core (0 = max)");
102
103static bool indep_threads_mode = true;
104module_param(indep_threads_mode, bool, S_IRUGO | S_IWUSR);
105MODULE_PARM_DESC(indep_threads_mode, "Independent-threads mode (only on POWER9)");
106
107#ifdef CONFIG_KVM_XICS
108static struct kernel_param_ops module_param_ops = {
109 .set = param_set_int,
110 .get = param_get_int,
111};
112
113module_param_cb(kvm_irq_bypass, &module_param_ops, &kvm_irq_bypass, 0644);
114MODULE_PARM_DESC(kvm_irq_bypass, "Bypass passthrough interrupt optimization");
115
116module_param_cb(h_ipi_redirect, &module_param_ops, &h_ipi_redirect, 0644);
117MODULE_PARM_DESC(h_ipi_redirect, "Redirect H_IPI wakeup to a free host core");
118#endif
119
120
121static bool no_mixing_hpt_and_radix;
122
123static void kvmppc_end_cede(struct kvm_vcpu *vcpu);
124static int kvmppc_hv_setup_htab_rma(struct kvm_vcpu *vcpu);
125
126
127
128
129
130
131#define RWMR_RPA_P8_1THREAD 0x164520C62609AECAUL
132#define RWMR_RPA_P8_2THREAD 0x7FFF2908450D8DA9UL
133#define RWMR_RPA_P8_3THREAD 0x164520C62609AECAUL
134#define RWMR_RPA_P8_4THREAD 0x199A421245058DA9UL
135#define RWMR_RPA_P8_5THREAD 0x164520C62609AECAUL
136#define RWMR_RPA_P8_6THREAD 0x164520C62609AECAUL
137#define RWMR_RPA_P8_7THREAD 0x164520C62609AECAUL
138#define RWMR_RPA_P8_8THREAD 0x164520C62609AECAUL
139
140static unsigned long p8_rwmr_values[MAX_SMT_THREADS + 1] = {
141 RWMR_RPA_P8_1THREAD,
142 RWMR_RPA_P8_1THREAD,
143 RWMR_RPA_P8_2THREAD,
144 RWMR_RPA_P8_3THREAD,
145 RWMR_RPA_P8_4THREAD,
146 RWMR_RPA_P8_5THREAD,
147 RWMR_RPA_P8_6THREAD,
148 RWMR_RPA_P8_7THREAD,
149 RWMR_RPA_P8_8THREAD,
150};
151
152static inline struct kvm_vcpu *next_runnable_thread(struct kvmppc_vcore *vc,
153 int *ip)
154{
155 int i = *ip;
156 struct kvm_vcpu *vcpu;
157
158 while (++i < MAX_SMT_THREADS) {
159 vcpu = READ_ONCE(vc->runnable_threads[i]);
160 if (vcpu) {
161 *ip = i;
162 return vcpu;
163 }
164 }
165 return NULL;
166}
167
168
169#define for_each_runnable_thread(i, vcpu, vc) \
170 for (i = -1; (vcpu = next_runnable_thread(vc, &i)); )
171
172static bool kvmppc_ipi_thread(int cpu)
173{
174 unsigned long msg = PPC_DBELL_TYPE(PPC_DBELL_SERVER);
175
176
177 if (cpu_has_feature(CPU_FTR_ARCH_300)) {
178 msg |= get_hard_smp_processor_id(cpu);
179 smp_mb();
180 __asm__ __volatile__ (PPC_MSGSND(%0) : : "r" (msg));
181 return true;
182 }
183
184
185 if (cpu_has_feature(CPU_FTR_ARCH_207S)) {
186 preempt_disable();
187 if (cpu_first_thread_sibling(cpu) ==
188 cpu_first_thread_sibling(smp_processor_id())) {
189 msg |= cpu_thread_in_core(cpu);
190 smp_mb();
191 __asm__ __volatile__ (PPC_MSGSND(%0) : : "r" (msg));
192 preempt_enable();
193 return true;
194 }
195 preempt_enable();
196 }
197
198#if defined(CONFIG_PPC_ICP_NATIVE) && defined(CONFIG_SMP)
199 if (cpu >= 0 && cpu < nr_cpu_ids) {
200 if (paca_ptrs[cpu]->kvm_hstate.xics_phys) {
201 xics_wake_cpu(cpu);
202 return true;
203 }
204 opal_int_set_mfrr(get_hard_smp_processor_id(cpu), IPI_PRIORITY);
205 return true;
206 }
207#endif
208
209 return false;
210}
211
212static void kvmppc_fast_vcpu_kick_hv(struct kvm_vcpu *vcpu)
213{
214 int cpu;
215 struct swait_queue_head *wqp;
216
217 wqp = kvm_arch_vcpu_wq(vcpu);
218 if (swq_has_sleeper(wqp)) {
219 swake_up_one(wqp);
220 ++vcpu->stat.halt_wakeup;
221 }
222
223 cpu = READ_ONCE(vcpu->arch.thread_cpu);
224 if (cpu >= 0 && kvmppc_ipi_thread(cpu))
225 return;
226
227
228 cpu = vcpu->cpu;
229 if (cpu >= 0 && cpu < nr_cpu_ids && cpu_online(cpu))
230 smp_send_reschedule(cpu);
231}
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266static void kvmppc_core_start_stolen(struct kvmppc_vcore *vc)
267{
268 unsigned long flags;
269
270 spin_lock_irqsave(&vc->stoltb_lock, flags);
271 vc->preempt_tb = mftb();
272 spin_unlock_irqrestore(&vc->stoltb_lock, flags);
273}
274
275static void kvmppc_core_end_stolen(struct kvmppc_vcore *vc)
276{
277 unsigned long flags;
278
279 spin_lock_irqsave(&vc->stoltb_lock, flags);
280 if (vc->preempt_tb != TB_NIL) {
281 vc->stolen_tb += mftb() - vc->preempt_tb;
282 vc->preempt_tb = TB_NIL;
283 }
284 spin_unlock_irqrestore(&vc->stoltb_lock, flags);
285}
286
287static void kvmppc_core_vcpu_load_hv(struct kvm_vcpu *vcpu, int cpu)
288{
289 struct kvmppc_vcore *vc = vcpu->arch.vcore;
290 unsigned long flags;
291
292
293
294
295
296
297
298 if (vc->runner == vcpu && vc->vcore_state >= VCORE_SLEEPING)
299 kvmppc_core_end_stolen(vc);
300
301 spin_lock_irqsave(&vcpu->arch.tbacct_lock, flags);
302 if (vcpu->arch.state == KVMPPC_VCPU_BUSY_IN_HOST &&
303 vcpu->arch.busy_preempt != TB_NIL) {
304 vcpu->arch.busy_stolen += mftb() - vcpu->arch.busy_preempt;
305 vcpu->arch.busy_preempt = TB_NIL;
306 }
307 spin_unlock_irqrestore(&vcpu->arch.tbacct_lock, flags);
308}
309
310static void kvmppc_core_vcpu_put_hv(struct kvm_vcpu *vcpu)
311{
312 struct kvmppc_vcore *vc = vcpu->arch.vcore;
313 unsigned long flags;
314
315 if (vc->runner == vcpu && vc->vcore_state >= VCORE_SLEEPING)
316 kvmppc_core_start_stolen(vc);
317
318 spin_lock_irqsave(&vcpu->arch.tbacct_lock, flags);
319 if (vcpu->arch.state == KVMPPC_VCPU_BUSY_IN_HOST)
320 vcpu->arch.busy_preempt = mftb();
321 spin_unlock_irqrestore(&vcpu->arch.tbacct_lock, flags);
322}
323
324static void kvmppc_set_msr_hv(struct kvm_vcpu *vcpu, u64 msr)
325{
326
327
328
329
330 if ((msr & MSR_TS_MASK) == MSR_TS_MASK)
331 msr &= ~MSR_TS_MASK;
332 vcpu->arch.shregs.msr = msr;
333 kvmppc_end_cede(vcpu);
334}
335
336static void kvmppc_set_pvr_hv(struct kvm_vcpu *vcpu, u32 pvr)
337{
338 vcpu->arch.pvr = pvr;
339}
340
341
342#define PCR_ARCH_300 (PCR_ARCH_207 << 1)
343
344static int kvmppc_set_arch_compat(struct kvm_vcpu *vcpu, u32 arch_compat)
345{
346 unsigned long host_pcr_bit = 0, guest_pcr_bit = 0;
347 struct kvmppc_vcore *vc = vcpu->arch.vcore;
348
349
350 if (cpu_has_feature(CPU_FTR_ARCH_300))
351 host_pcr_bit = PCR_ARCH_300;
352 else if (cpu_has_feature(CPU_FTR_ARCH_207S))
353 host_pcr_bit = PCR_ARCH_207;
354 else if (cpu_has_feature(CPU_FTR_ARCH_206))
355 host_pcr_bit = PCR_ARCH_206;
356 else
357 host_pcr_bit = PCR_ARCH_205;
358
359
360 guest_pcr_bit = host_pcr_bit;
361 if (arch_compat) {
362 switch (arch_compat) {
363 case PVR_ARCH_205:
364 guest_pcr_bit = PCR_ARCH_205;
365 break;
366 case PVR_ARCH_206:
367 case PVR_ARCH_206p:
368 guest_pcr_bit = PCR_ARCH_206;
369 break;
370 case PVR_ARCH_207:
371 guest_pcr_bit = PCR_ARCH_207;
372 break;
373 case PVR_ARCH_300:
374 guest_pcr_bit = PCR_ARCH_300;
375 break;
376 default:
377 return -EINVAL;
378 }
379 }
380
381
382 if (guest_pcr_bit > host_pcr_bit)
383 return -EINVAL;
384
385 spin_lock(&vc->lock);
386 vc->arch_compat = arch_compat;
387
388 vc->pcr = host_pcr_bit - guest_pcr_bit;
389 spin_unlock(&vc->lock);
390
391 return 0;
392}
393
394static void kvmppc_dump_regs(struct kvm_vcpu *vcpu)
395{
396 int r;
397
398 pr_err("vcpu %p (%d):\n", vcpu, vcpu->vcpu_id);
399 pr_err("pc = %.16lx msr = %.16llx trap = %x\n",
400 vcpu->arch.regs.nip, vcpu->arch.shregs.msr, vcpu->arch.trap);
401 for (r = 0; r < 16; ++r)
402 pr_err("r%2d = %.16lx r%d = %.16lx\n",
403 r, kvmppc_get_gpr(vcpu, r),
404 r+16, kvmppc_get_gpr(vcpu, r+16));
405 pr_err("ctr = %.16lx lr = %.16lx\n",
406 vcpu->arch.regs.ctr, vcpu->arch.regs.link);
407 pr_err("srr0 = %.16llx srr1 = %.16llx\n",
408 vcpu->arch.shregs.srr0, vcpu->arch.shregs.srr1);
409 pr_err("sprg0 = %.16llx sprg1 = %.16llx\n",
410 vcpu->arch.shregs.sprg0, vcpu->arch.shregs.sprg1);
411 pr_err("sprg2 = %.16llx sprg3 = %.16llx\n",
412 vcpu->arch.shregs.sprg2, vcpu->arch.shregs.sprg3);
413 pr_err("cr = %.8x xer = %.16lx dsisr = %.8x\n",
414 vcpu->arch.cr, vcpu->arch.regs.xer, vcpu->arch.shregs.dsisr);
415 pr_err("dar = %.16llx\n", vcpu->arch.shregs.dar);
416 pr_err("fault dar = %.16lx dsisr = %.8x\n",
417 vcpu->arch.fault_dar, vcpu->arch.fault_dsisr);
418 pr_err("SLB (%d entries):\n", vcpu->arch.slb_max);
419 for (r = 0; r < vcpu->arch.slb_max; ++r)
420 pr_err(" ESID = %.16llx VSID = %.16llx\n",
421 vcpu->arch.slb[r].orige, vcpu->arch.slb[r].origv);
422 pr_err("lpcr = %.16lx sdr1 = %.16lx last_inst = %.8x\n",
423 vcpu->arch.vcore->lpcr, vcpu->kvm->arch.sdr1,
424 vcpu->arch.last_inst);
425}
426
427static struct kvm_vcpu *kvmppc_find_vcpu(struct kvm *kvm, int id)
428{
429 struct kvm_vcpu *ret;
430
431 mutex_lock(&kvm->lock);
432 ret = kvm_get_vcpu_by_id(kvm, id);
433 mutex_unlock(&kvm->lock);
434 return ret;
435}
436
437static void init_vpa(struct kvm_vcpu *vcpu, struct lppaca *vpa)
438{
439 vpa->__old_status |= LPPACA_OLD_SHARED_PROC;
440 vpa->yield_count = cpu_to_be32(1);
441}
442
443static int set_vpa(struct kvm_vcpu *vcpu, struct kvmppc_vpa *v,
444 unsigned long addr, unsigned long len)
445{
446
447 if (addr & (L1_CACHE_BYTES - 1))
448 return -EINVAL;
449 spin_lock(&vcpu->arch.vpa_update_lock);
450 if (v->next_gpa != addr || v->len != len) {
451 v->next_gpa = addr;
452 v->len = addr ? len : 0;
453 v->update_pending = 1;
454 }
455 spin_unlock(&vcpu->arch.vpa_update_lock);
456 return 0;
457}
458
459
460struct reg_vpa {
461 u32 dummy;
462 union {
463 __be16 hword;
464 __be32 word;
465 } length;
466};
467
468static int vpa_is_registered(struct kvmppc_vpa *vpap)
469{
470 if (vpap->update_pending)
471 return vpap->next_gpa != 0;
472 return vpap->pinned_addr != NULL;
473}
474
475static unsigned long do_h_register_vpa(struct kvm_vcpu *vcpu,
476 unsigned long flags,
477 unsigned long vcpuid, unsigned long vpa)
478{
479 struct kvm *kvm = vcpu->kvm;
480 unsigned long len, nb;
481 void *va;
482 struct kvm_vcpu *tvcpu;
483 int err;
484 int subfunc;
485 struct kvmppc_vpa *vpap;
486
487 tvcpu = kvmppc_find_vcpu(kvm, vcpuid);
488 if (!tvcpu)
489 return H_PARAMETER;
490
491 subfunc = (flags >> H_VPA_FUNC_SHIFT) & H_VPA_FUNC_MASK;
492 if (subfunc == H_VPA_REG_VPA || subfunc == H_VPA_REG_DTL ||
493 subfunc == H_VPA_REG_SLB) {
494
495 if ((vpa & (L1_CACHE_BYTES - 1)) || !vpa)
496 return H_PARAMETER;
497
498
499 va = kvmppc_pin_guest_page(kvm, vpa, &nb);
500 if (va == NULL)
501 return H_PARAMETER;
502 if (subfunc == H_VPA_REG_VPA)
503 len = be16_to_cpu(((struct reg_vpa *)va)->length.hword);
504 else
505 len = be32_to_cpu(((struct reg_vpa *)va)->length.word);
506 kvmppc_unpin_guest_page(kvm, va, vpa, false);
507
508
509 if (len > nb || len < sizeof(struct reg_vpa))
510 return H_PARAMETER;
511 } else {
512 vpa = 0;
513 len = 0;
514 }
515
516 err = H_PARAMETER;
517 vpap = NULL;
518 spin_lock(&tvcpu->arch.vpa_update_lock);
519
520 switch (subfunc) {
521 case H_VPA_REG_VPA:
522
523
524
525
526
527
528 BUILD_BUG_ON(sizeof(struct lppaca) != 640);
529 if (len < sizeof(struct lppaca))
530 break;
531 vpap = &tvcpu->arch.vpa;
532 err = 0;
533 break;
534
535 case H_VPA_REG_DTL:
536 if (len < sizeof(struct dtl_entry))
537 break;
538 len -= len % sizeof(struct dtl_entry);
539
540
541 err = H_RESOURCE;
542 if (!vpa_is_registered(&tvcpu->arch.vpa))
543 break;
544
545 vpap = &tvcpu->arch.dtl;
546 err = 0;
547 break;
548
549 case H_VPA_REG_SLB:
550
551 err = H_RESOURCE;
552 if (!vpa_is_registered(&tvcpu->arch.vpa))
553 break;
554
555 vpap = &tvcpu->arch.slb_shadow;
556 err = 0;
557 break;
558
559 case H_VPA_DEREG_VPA:
560
561 err = H_RESOURCE;
562 if (vpa_is_registered(&tvcpu->arch.dtl) ||
563 vpa_is_registered(&tvcpu->arch.slb_shadow))
564 break;
565
566 vpap = &tvcpu->arch.vpa;
567 err = 0;
568 break;
569
570 case H_VPA_DEREG_DTL:
571 vpap = &tvcpu->arch.dtl;
572 err = 0;
573 break;
574
575 case H_VPA_DEREG_SLB:
576 vpap = &tvcpu->arch.slb_shadow;
577 err = 0;
578 break;
579 }
580
581 if (vpap) {
582 vpap->next_gpa = vpa;
583 vpap->len = len;
584 vpap->update_pending = 1;
585 }
586
587 spin_unlock(&tvcpu->arch.vpa_update_lock);
588
589 return err;
590}
591
592static void kvmppc_update_vpa(struct kvm_vcpu *vcpu, struct kvmppc_vpa *vpap)
593{
594 struct kvm *kvm = vcpu->kvm;
595 void *va;
596 unsigned long nb;
597 unsigned long gpa;
598
599
600
601
602
603
604
605
606
607 for (;;) {
608 gpa = vpap->next_gpa;
609 spin_unlock(&vcpu->arch.vpa_update_lock);
610 va = NULL;
611 nb = 0;
612 if (gpa)
613 va = kvmppc_pin_guest_page(kvm, gpa, &nb);
614 spin_lock(&vcpu->arch.vpa_update_lock);
615 if (gpa == vpap->next_gpa)
616 break;
617
618 if (va)
619 kvmppc_unpin_guest_page(kvm, va, gpa, false);
620 }
621
622 vpap->update_pending = 0;
623 if (va && nb < vpap->len) {
624
625
626
627
628
629 kvmppc_unpin_guest_page(kvm, va, gpa, false);
630 va = NULL;
631 }
632 if (vpap->pinned_addr)
633 kvmppc_unpin_guest_page(kvm, vpap->pinned_addr, vpap->gpa,
634 vpap->dirty);
635 vpap->gpa = gpa;
636 vpap->pinned_addr = va;
637 vpap->dirty = false;
638 if (va)
639 vpap->pinned_end = va + vpap->len;
640}
641
642static void kvmppc_update_vpas(struct kvm_vcpu *vcpu)
643{
644 if (!(vcpu->arch.vpa.update_pending ||
645 vcpu->arch.slb_shadow.update_pending ||
646 vcpu->arch.dtl.update_pending))
647 return;
648
649 spin_lock(&vcpu->arch.vpa_update_lock);
650 if (vcpu->arch.vpa.update_pending) {
651 kvmppc_update_vpa(vcpu, &vcpu->arch.vpa);
652 if (vcpu->arch.vpa.pinned_addr)
653 init_vpa(vcpu, vcpu->arch.vpa.pinned_addr);
654 }
655 if (vcpu->arch.dtl.update_pending) {
656 kvmppc_update_vpa(vcpu, &vcpu->arch.dtl);
657 vcpu->arch.dtl_ptr = vcpu->arch.dtl.pinned_addr;
658 vcpu->arch.dtl_index = 0;
659 }
660 if (vcpu->arch.slb_shadow.update_pending)
661 kvmppc_update_vpa(vcpu, &vcpu->arch.slb_shadow);
662 spin_unlock(&vcpu->arch.vpa_update_lock);
663}
664
665
666
667
668
669static u64 vcore_stolen_time(struct kvmppc_vcore *vc, u64 now)
670{
671 u64 p;
672 unsigned long flags;
673
674 spin_lock_irqsave(&vc->stoltb_lock, flags);
675 p = vc->stolen_tb;
676 if (vc->vcore_state != VCORE_INACTIVE &&
677 vc->preempt_tb != TB_NIL)
678 p += now - vc->preempt_tb;
679 spin_unlock_irqrestore(&vc->stoltb_lock, flags);
680 return p;
681}
682
683static void kvmppc_create_dtl_entry(struct kvm_vcpu *vcpu,
684 struct kvmppc_vcore *vc)
685{
686 struct dtl_entry *dt;
687 struct lppaca *vpa;
688 unsigned long stolen;
689 unsigned long core_stolen;
690 u64 now;
691 unsigned long flags;
692
693 dt = vcpu->arch.dtl_ptr;
694 vpa = vcpu->arch.vpa.pinned_addr;
695 now = mftb();
696 core_stolen = vcore_stolen_time(vc, now);
697 stolen = core_stolen - vcpu->arch.stolen_logged;
698 vcpu->arch.stolen_logged = core_stolen;
699 spin_lock_irqsave(&vcpu->arch.tbacct_lock, flags);
700 stolen += vcpu->arch.busy_stolen;
701 vcpu->arch.busy_stolen = 0;
702 spin_unlock_irqrestore(&vcpu->arch.tbacct_lock, flags);
703 if (!dt || !vpa)
704 return;
705 memset(dt, 0, sizeof(struct dtl_entry));
706 dt->dispatch_reason = 7;
707 dt->processor_id = cpu_to_be16(vc->pcpu + vcpu->arch.ptid);
708 dt->timebase = cpu_to_be64(now + vc->tb_offset);
709 dt->enqueue_to_dispatch_time = cpu_to_be32(stolen);
710 dt->srr0 = cpu_to_be64(kvmppc_get_pc(vcpu));
711 dt->srr1 = cpu_to_be64(vcpu->arch.shregs.msr);
712 ++dt;
713 if (dt == vcpu->arch.dtl.pinned_end)
714 dt = vcpu->arch.dtl.pinned_addr;
715 vcpu->arch.dtl_ptr = dt;
716
717 smp_wmb();
718 vpa->dtl_idx = cpu_to_be64(++vcpu->arch.dtl_index);
719 vcpu->arch.dtl.dirty = true;
720}
721
722
723static bool kvmppc_doorbell_pending(struct kvm_vcpu *vcpu)
724{
725 int thr;
726 struct kvmppc_vcore *vc;
727
728 if (vcpu->arch.doorbell_request)
729 return true;
730
731
732
733
734
735
736 smp_rmb();
737 vc = vcpu->arch.vcore;
738 thr = vcpu->vcpu_id - vc->first_vcpuid;
739 return !!(vc->dpdes & (1 << thr));
740}
741
742static bool kvmppc_power8_compatible(struct kvm_vcpu *vcpu)
743{
744 if (vcpu->arch.vcore->arch_compat >= PVR_ARCH_207)
745 return true;
746 if ((!vcpu->arch.vcore->arch_compat) &&
747 cpu_has_feature(CPU_FTR_ARCH_207S))
748 return true;
749 return false;
750}
751
752static int kvmppc_h_set_mode(struct kvm_vcpu *vcpu, unsigned long mflags,
753 unsigned long resource, unsigned long value1,
754 unsigned long value2)
755{
756 switch (resource) {
757 case H_SET_MODE_RESOURCE_SET_CIABR:
758 if (!kvmppc_power8_compatible(vcpu))
759 return H_P2;
760 if (value2)
761 return H_P4;
762 if (mflags)
763 return H_UNSUPPORTED_FLAG_START;
764
765 if ((value1 & CIABR_PRIV) == CIABR_PRIV_HYPER)
766 return H_P3;
767 vcpu->arch.ciabr = value1;
768 return H_SUCCESS;
769 case H_SET_MODE_RESOURCE_SET_DAWR:
770 if (!kvmppc_power8_compatible(vcpu))
771 return H_P2;
772 if (!ppc_breakpoint_available())
773 return H_P2;
774 if (mflags)
775 return H_UNSUPPORTED_FLAG_START;
776 if (value2 & DABRX_HYP)
777 return H_P4;
778 vcpu->arch.dawr = value1;
779 vcpu->arch.dawrx = value2;
780 return H_SUCCESS;
781 default:
782 return H_TOO_HARD;
783 }
784}
785
786static int kvm_arch_vcpu_yield_to(struct kvm_vcpu *target)
787{
788 struct kvmppc_vcore *vcore = target->arch.vcore;
789
790
791
792
793
794
795
796
797
798 spin_lock(&vcore->lock);
799 if (target->arch.state == KVMPPC_VCPU_RUNNABLE &&
800 vcore->vcore_state != VCORE_INACTIVE &&
801 vcore->runner)
802 target = vcore->runner;
803 spin_unlock(&vcore->lock);
804
805 return kvm_vcpu_yield_to(target);
806}
807
808static int kvmppc_get_yield_count(struct kvm_vcpu *vcpu)
809{
810 int yield_count = 0;
811 struct lppaca *lppaca;
812
813 spin_lock(&vcpu->arch.vpa_update_lock);
814 lppaca = (struct lppaca *)vcpu->arch.vpa.pinned_addr;
815 if (lppaca)
816 yield_count = be32_to_cpu(lppaca->yield_count);
817 spin_unlock(&vcpu->arch.vpa_update_lock);
818 return yield_count;
819}
820
821int kvmppc_pseries_do_hcall(struct kvm_vcpu *vcpu)
822{
823 unsigned long req = kvmppc_get_gpr(vcpu, 3);
824 unsigned long target, ret = H_SUCCESS;
825 int yield_count;
826 struct kvm_vcpu *tvcpu;
827 int idx, rc;
828
829 if (req <= MAX_HCALL_OPCODE &&
830 !test_bit(req/4, vcpu->kvm->arch.enabled_hcalls))
831 return RESUME_HOST;
832
833 switch (req) {
834 case H_CEDE:
835 break;
836 case H_PROD:
837 target = kvmppc_get_gpr(vcpu, 4);
838 tvcpu = kvmppc_find_vcpu(vcpu->kvm, target);
839 if (!tvcpu) {
840 ret = H_PARAMETER;
841 break;
842 }
843 tvcpu->arch.prodded = 1;
844 smp_mb();
845 if (tvcpu->arch.ceded)
846 kvmppc_fast_vcpu_kick_hv(tvcpu);
847 break;
848 case H_CONFER:
849 target = kvmppc_get_gpr(vcpu, 4);
850 if (target == -1)
851 break;
852 tvcpu = kvmppc_find_vcpu(vcpu->kvm, target);
853 if (!tvcpu) {
854 ret = H_PARAMETER;
855 break;
856 }
857 yield_count = kvmppc_get_gpr(vcpu, 5);
858 if (kvmppc_get_yield_count(tvcpu) != yield_count)
859 break;
860 kvm_arch_vcpu_yield_to(tvcpu);
861 break;
862 case H_REGISTER_VPA:
863 ret = do_h_register_vpa(vcpu, kvmppc_get_gpr(vcpu, 4),
864 kvmppc_get_gpr(vcpu, 5),
865 kvmppc_get_gpr(vcpu, 6));
866 break;
867 case H_RTAS:
868 if (list_empty(&vcpu->kvm->arch.rtas_tokens))
869 return RESUME_HOST;
870
871 idx = srcu_read_lock(&vcpu->kvm->srcu);
872 rc = kvmppc_rtas_hcall(vcpu);
873 srcu_read_unlock(&vcpu->kvm->srcu, idx);
874
875 if (rc == -ENOENT)
876 return RESUME_HOST;
877 else if (rc == 0)
878 break;
879
880
881 return rc;
882 case H_LOGICAL_CI_LOAD:
883 ret = kvmppc_h_logical_ci_load(vcpu);
884 if (ret == H_TOO_HARD)
885 return RESUME_HOST;
886 break;
887 case H_LOGICAL_CI_STORE:
888 ret = kvmppc_h_logical_ci_store(vcpu);
889 if (ret == H_TOO_HARD)
890 return RESUME_HOST;
891 break;
892 case H_SET_MODE:
893 ret = kvmppc_h_set_mode(vcpu, kvmppc_get_gpr(vcpu, 4),
894 kvmppc_get_gpr(vcpu, 5),
895 kvmppc_get_gpr(vcpu, 6),
896 kvmppc_get_gpr(vcpu, 7));
897 if (ret == H_TOO_HARD)
898 return RESUME_HOST;
899 break;
900 case H_XIRR:
901 case H_CPPR:
902 case H_EOI:
903 case H_IPI:
904 case H_IPOLL:
905 case H_XIRR_X:
906 if (kvmppc_xics_enabled(vcpu)) {
907 if (xive_enabled()) {
908 ret = H_NOT_AVAILABLE;
909 return RESUME_GUEST;
910 }
911 ret = kvmppc_xics_hcall(vcpu, req);
912 break;
913 }
914 return RESUME_HOST;
915 case H_PUT_TCE:
916 ret = kvmppc_h_put_tce(vcpu, kvmppc_get_gpr(vcpu, 4),
917 kvmppc_get_gpr(vcpu, 5),
918 kvmppc_get_gpr(vcpu, 6));
919 if (ret == H_TOO_HARD)
920 return RESUME_HOST;
921 break;
922 case H_PUT_TCE_INDIRECT:
923 ret = kvmppc_h_put_tce_indirect(vcpu, kvmppc_get_gpr(vcpu, 4),
924 kvmppc_get_gpr(vcpu, 5),
925 kvmppc_get_gpr(vcpu, 6),
926 kvmppc_get_gpr(vcpu, 7));
927 if (ret == H_TOO_HARD)
928 return RESUME_HOST;
929 break;
930 case H_STUFF_TCE:
931 ret = kvmppc_h_stuff_tce(vcpu, kvmppc_get_gpr(vcpu, 4),
932 kvmppc_get_gpr(vcpu, 5),
933 kvmppc_get_gpr(vcpu, 6),
934 kvmppc_get_gpr(vcpu, 7));
935 if (ret == H_TOO_HARD)
936 return RESUME_HOST;
937 break;
938 default:
939 return RESUME_HOST;
940 }
941 kvmppc_set_gpr(vcpu, 3, ret);
942 vcpu->arch.hcall_needed = 0;
943 return RESUME_GUEST;
944}
945
946static int kvmppc_hcall_impl_hv(unsigned long cmd)
947{
948 switch (cmd) {
949 case H_CEDE:
950 case H_PROD:
951 case H_CONFER:
952 case H_REGISTER_VPA:
953 case H_SET_MODE:
954 case H_LOGICAL_CI_LOAD:
955 case H_LOGICAL_CI_STORE:
956#ifdef CONFIG_KVM_XICS
957 case H_XIRR:
958 case H_CPPR:
959 case H_EOI:
960 case H_IPI:
961 case H_IPOLL:
962 case H_XIRR_X:
963#endif
964 return 1;
965 }
966
967
968 return kvmppc_hcall_impl_hv_realmode(cmd);
969}
970
971static int kvmppc_emulate_debug_inst(struct kvm_run *run,
972 struct kvm_vcpu *vcpu)
973{
974 u32 last_inst;
975
976 if (kvmppc_get_last_inst(vcpu, INST_GENERIC, &last_inst) !=
977 EMULATE_DONE) {
978
979
980
981
982 return RESUME_GUEST;
983 }
984
985 if (last_inst == KVMPPC_INST_SW_BREAKPOINT) {
986 run->exit_reason = KVM_EXIT_DEBUG;
987 run->debug.arch.address = kvmppc_get_pc(vcpu);
988 return RESUME_HOST;
989 } else {
990 kvmppc_core_queue_program(vcpu, SRR1_PROGILL);
991 return RESUME_GUEST;
992 }
993}
994
995static void do_nothing(void *x)
996{
997}
998
999static unsigned long kvmppc_read_dpdes(struct kvm_vcpu *vcpu)
1000{
1001 int thr, cpu, pcpu, nthreads;
1002 struct kvm_vcpu *v;
1003 unsigned long dpdes;
1004
1005 nthreads = vcpu->kvm->arch.emul_smt_mode;
1006 dpdes = 0;
1007 cpu = vcpu->vcpu_id & ~(nthreads - 1);
1008 for (thr = 0; thr < nthreads; ++thr, ++cpu) {
1009 v = kvmppc_find_vcpu(vcpu->kvm, cpu);
1010 if (!v)
1011 continue;
1012
1013
1014
1015
1016
1017 pcpu = READ_ONCE(v->cpu);
1018 if (pcpu >= 0)
1019 smp_call_function_single(pcpu, do_nothing, NULL, 1);
1020 if (kvmppc_doorbell_pending(v))
1021 dpdes |= 1 << thr;
1022 }
1023 return dpdes;
1024}
1025
1026
1027
1028
1029
1030
1031
1032static int kvmppc_emulate_doorbell_instr(struct kvm_vcpu *vcpu)
1033{
1034 u32 inst, rb, thr;
1035 unsigned long arg;
1036 struct kvm *kvm = vcpu->kvm;
1037 struct kvm_vcpu *tvcpu;
1038
1039 if (kvmppc_get_last_inst(vcpu, INST_GENERIC, &inst) != EMULATE_DONE)
1040 return RESUME_GUEST;
1041 if (get_op(inst) != 31)
1042 return EMULATE_FAIL;
1043 rb = get_rb(inst);
1044 thr = vcpu->vcpu_id & (kvm->arch.emul_smt_mode - 1);
1045 switch (get_xop(inst)) {
1046 case OP_31_XOP_MSGSNDP:
1047 arg = kvmppc_get_gpr(vcpu, rb);
1048 if (((arg >> 27) & 0xf) != PPC_DBELL_SERVER)
1049 break;
1050 arg &= 0x3f;
1051 if (arg >= kvm->arch.emul_smt_mode)
1052 break;
1053 tvcpu = kvmppc_find_vcpu(kvm, vcpu->vcpu_id - thr + arg);
1054 if (!tvcpu)
1055 break;
1056 if (!tvcpu->arch.doorbell_request) {
1057 tvcpu->arch.doorbell_request = 1;
1058 kvmppc_fast_vcpu_kick_hv(tvcpu);
1059 }
1060 break;
1061 case OP_31_XOP_MSGCLRP:
1062 arg = kvmppc_get_gpr(vcpu, rb);
1063 if (((arg >> 27) & 0xf) != PPC_DBELL_SERVER)
1064 break;
1065 vcpu->arch.vcore->dpdes = 0;
1066 vcpu->arch.doorbell_request = 0;
1067 break;
1068 case OP_31_XOP_MFSPR:
1069 switch (get_sprn(inst)) {
1070 case SPRN_TIR:
1071 arg = thr;
1072 break;
1073 case SPRN_DPDES:
1074 arg = kvmppc_read_dpdes(vcpu);
1075 break;
1076 default:
1077 return EMULATE_FAIL;
1078 }
1079 kvmppc_set_gpr(vcpu, get_rt(inst), arg);
1080 break;
1081 default:
1082 return EMULATE_FAIL;
1083 }
1084 kvmppc_set_pc(vcpu, kvmppc_get_pc(vcpu) + 4);
1085 return RESUME_GUEST;
1086}
1087
1088
1089static int kvmppc_handle_exit_hv(struct kvm_run *run, struct kvm_vcpu *vcpu,
1090 struct task_struct *tsk)
1091{
1092 int r = RESUME_HOST;
1093
1094 vcpu->stat.sum_exits++;
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104 if (vcpu->arch.shregs.msr & MSR_HV) {
1105 printk(KERN_EMERG "KVM trap in HV mode!\n");
1106 printk(KERN_EMERG "trap=0x%x | pc=0x%lx | msr=0x%llx\n",
1107 vcpu->arch.trap, kvmppc_get_pc(vcpu),
1108 vcpu->arch.shregs.msr);
1109 kvmppc_dump_regs(vcpu);
1110 run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
1111 run->hw.hardware_exit_reason = vcpu->arch.trap;
1112 return RESUME_HOST;
1113 }
1114 run->exit_reason = KVM_EXIT_UNKNOWN;
1115 run->ready_for_interrupt_injection = 1;
1116 switch (vcpu->arch.trap) {
1117
1118 case BOOK3S_INTERRUPT_HV_DECREMENTER:
1119 vcpu->stat.dec_exits++;
1120 r = RESUME_GUEST;
1121 break;
1122 case BOOK3S_INTERRUPT_EXTERNAL:
1123 case BOOK3S_INTERRUPT_H_DOORBELL:
1124 case BOOK3S_INTERRUPT_H_VIRT:
1125 vcpu->stat.ext_intr_exits++;
1126 r = RESUME_GUEST;
1127 break;
1128
1129 case BOOK3S_INTERRUPT_HMI:
1130 case BOOK3S_INTERRUPT_PERFMON:
1131 case BOOK3S_INTERRUPT_SYSTEM_RESET:
1132 r = RESUME_GUEST;
1133 break;
1134 case BOOK3S_INTERRUPT_MACHINE_CHECK:
1135
1136 run->exit_reason = KVM_EXIT_NMI;
1137 run->hw.hardware_exit_reason = vcpu->arch.trap;
1138
1139 run->flags &= ~KVM_RUN_PPC_NMI_DISP_MASK;
1140
1141 if (vcpu->arch.mce_evt.disposition == MCE_DISPOSITION_RECOVERED)
1142 run->flags |= KVM_RUN_PPC_NMI_DISP_FULLY_RECOV;
1143 else
1144 run->flags |= KVM_RUN_PPC_NMI_DISP_NOT_RECOV;
1145
1146 r = RESUME_HOST;
1147
1148 machine_check_print_event_info(&vcpu->arch.mce_evt, false);
1149 break;
1150 case BOOK3S_INTERRUPT_PROGRAM:
1151 {
1152 ulong flags;
1153
1154
1155
1156
1157
1158
1159 flags = vcpu->arch.shregs.msr & 0x1f0000ull;
1160 kvmppc_core_queue_program(vcpu, flags);
1161 r = RESUME_GUEST;
1162 break;
1163 }
1164 case BOOK3S_INTERRUPT_SYSCALL:
1165 {
1166
1167 int i;
1168
1169
1170
1171
1172
1173 run->papr_hcall.nr = kvmppc_get_gpr(vcpu, 3);
1174 for (i = 0; i < 9; ++i)
1175 run->papr_hcall.args[i] = kvmppc_get_gpr(vcpu, 4 + i);
1176 run->exit_reason = KVM_EXIT_PAPR_HCALL;
1177 vcpu->arch.hcall_needed = 1;
1178 r = RESUME_HOST;
1179 break;
1180 }
1181
1182
1183
1184
1185
1186
1187
1188 case BOOK3S_INTERRUPT_H_DATA_STORAGE:
1189 r = RESUME_PAGE_FAULT;
1190 break;
1191 case BOOK3S_INTERRUPT_H_INST_STORAGE:
1192 vcpu->arch.fault_dar = kvmppc_get_pc(vcpu);
1193 vcpu->arch.fault_dsisr = 0;
1194 r = RESUME_PAGE_FAULT;
1195 break;
1196
1197
1198
1199
1200
1201
1202
1203 case BOOK3S_INTERRUPT_H_EMUL_ASSIST:
1204 if (vcpu->arch.emul_inst != KVM_INST_FETCH_FAILED)
1205 vcpu->arch.last_inst = kvmppc_need_byteswap(vcpu) ?
1206 swab32(vcpu->arch.emul_inst) :
1207 vcpu->arch.emul_inst;
1208 if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP) {
1209
1210 spin_unlock(&vcpu->arch.vcore->lock);
1211 r = kvmppc_emulate_debug_inst(run, vcpu);
1212 spin_lock(&vcpu->arch.vcore->lock);
1213 } else {
1214 kvmppc_core_queue_program(vcpu, SRR1_PROGILL);
1215 r = RESUME_GUEST;
1216 }
1217 break;
1218
1219
1220
1221
1222
1223
1224
1225 case BOOK3S_INTERRUPT_H_FAC_UNAVAIL:
1226 r = EMULATE_FAIL;
1227 if (((vcpu->arch.hfscr >> 56) == FSCR_MSGP_LG) &&
1228 cpu_has_feature(CPU_FTR_ARCH_300)) {
1229
1230 spin_unlock(&vcpu->arch.vcore->lock);
1231 r = kvmppc_emulate_doorbell_instr(vcpu);
1232 spin_lock(&vcpu->arch.vcore->lock);
1233 }
1234 if (r == EMULATE_FAIL) {
1235 kvmppc_core_queue_program(vcpu, SRR1_PROGILL);
1236 r = RESUME_GUEST;
1237 }
1238 break;
1239
1240#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1241 case BOOK3S_INTERRUPT_HV_SOFTPATCH:
1242
1243
1244
1245
1246
1247
1248 r = kvmhv_p9_tm_emulation(vcpu);
1249 break;
1250#endif
1251
1252 case BOOK3S_INTERRUPT_HV_RM_HARD:
1253 r = RESUME_PASSTHROUGH;
1254 break;
1255 default:
1256 kvmppc_dump_regs(vcpu);
1257 printk(KERN_EMERG "trap=0x%x | pc=0x%lx | msr=0x%llx\n",
1258 vcpu->arch.trap, kvmppc_get_pc(vcpu),
1259 vcpu->arch.shregs.msr);
1260 run->hw.hardware_exit_reason = vcpu->arch.trap;
1261 r = RESUME_HOST;
1262 break;
1263 }
1264
1265 return r;
1266}
1267
1268static int kvm_arch_vcpu_ioctl_get_sregs_hv(struct kvm_vcpu *vcpu,
1269 struct kvm_sregs *sregs)
1270{
1271 int i;
1272
1273 memset(sregs, 0, sizeof(struct kvm_sregs));
1274 sregs->pvr = vcpu->arch.pvr;
1275 for (i = 0; i < vcpu->arch.slb_max; i++) {
1276 sregs->u.s.ppc64.slb[i].slbe = vcpu->arch.slb[i].orige;
1277 sregs->u.s.ppc64.slb[i].slbv = vcpu->arch.slb[i].origv;
1278 }
1279
1280 return 0;
1281}
1282
1283static int kvm_arch_vcpu_ioctl_set_sregs_hv(struct kvm_vcpu *vcpu,
1284 struct kvm_sregs *sregs)
1285{
1286 int i, j;
1287
1288
1289 if (sregs->pvr != vcpu->arch.pvr)
1290 return -EINVAL;
1291
1292 j = 0;
1293 for (i = 0; i < vcpu->arch.slb_nr; i++) {
1294 if (sregs->u.s.ppc64.slb[i].slbe & SLB_ESID_V) {
1295 vcpu->arch.slb[j].orige = sregs->u.s.ppc64.slb[i].slbe;
1296 vcpu->arch.slb[j].origv = sregs->u.s.ppc64.slb[i].slbv;
1297 ++j;
1298 }
1299 }
1300 vcpu->arch.slb_max = j;
1301
1302 return 0;
1303}
1304
1305static void kvmppc_set_lpcr(struct kvm_vcpu *vcpu, u64 new_lpcr,
1306 bool preserve_top32)
1307{
1308 struct kvm *kvm = vcpu->kvm;
1309 struct kvmppc_vcore *vc = vcpu->arch.vcore;
1310 u64 mask;
1311
1312 mutex_lock(&kvm->lock);
1313 spin_lock(&vc->lock);
1314
1315
1316
1317
1318 if ((new_lpcr & LPCR_ILE) != (vc->lpcr & LPCR_ILE)) {
1319 struct kvm_vcpu *vcpu;
1320 int i;
1321
1322 kvm_for_each_vcpu(i, vcpu, kvm) {
1323 if (vcpu->arch.vcore != vc)
1324 continue;
1325 if (new_lpcr & LPCR_ILE)
1326 vcpu->arch.intr_msr |= MSR_LE;
1327 else
1328 vcpu->arch.intr_msr &= ~MSR_LE;
1329 }
1330 }
1331
1332
1333
1334
1335
1336
1337 mask = LPCR_DPFD | LPCR_ILE | LPCR_TC;
1338 if (cpu_has_feature(CPU_FTR_ARCH_207S))
1339 mask |= LPCR_AIL;
1340
1341
1342
1343
1344 if (cpu_has_feature(CPU_FTR_ARCH_300))
1345 mask |= LPCR_LD;
1346
1347
1348 if (preserve_top32)
1349 mask &= 0xFFFFFFFF;
1350 vc->lpcr = (vc->lpcr & ~mask) | (new_lpcr & mask);
1351 spin_unlock(&vc->lock);
1352 mutex_unlock(&kvm->lock);
1353}
1354
1355static int kvmppc_get_one_reg_hv(struct kvm_vcpu *vcpu, u64 id,
1356 union kvmppc_one_reg *val)
1357{
1358 int r = 0;
1359 long int i;
1360
1361 switch (id) {
1362 case KVM_REG_PPC_DEBUG_INST:
1363 *val = get_reg_val(id, KVMPPC_INST_SW_BREAKPOINT);
1364 break;
1365 case KVM_REG_PPC_HIOR:
1366 *val = get_reg_val(id, 0);
1367 break;
1368 case KVM_REG_PPC_DABR:
1369 *val = get_reg_val(id, vcpu->arch.dabr);
1370 break;
1371 case KVM_REG_PPC_DABRX:
1372 *val = get_reg_val(id, vcpu->arch.dabrx);
1373 break;
1374 case KVM_REG_PPC_DSCR:
1375 *val = get_reg_val(id, vcpu->arch.dscr);
1376 break;
1377 case KVM_REG_PPC_PURR:
1378 *val = get_reg_val(id, vcpu->arch.purr);
1379 break;
1380 case KVM_REG_PPC_SPURR:
1381 *val = get_reg_val(id, vcpu->arch.spurr);
1382 break;
1383 case KVM_REG_PPC_AMR:
1384 *val = get_reg_val(id, vcpu->arch.amr);
1385 break;
1386 case KVM_REG_PPC_UAMOR:
1387 *val = get_reg_val(id, vcpu->arch.uamor);
1388 break;
1389 case KVM_REG_PPC_MMCR0 ... KVM_REG_PPC_MMCRS:
1390 i = id - KVM_REG_PPC_MMCR0;
1391 *val = get_reg_val(id, vcpu->arch.mmcr[i]);
1392 break;
1393 case KVM_REG_PPC_PMC1 ... KVM_REG_PPC_PMC8:
1394 i = id - KVM_REG_PPC_PMC1;
1395 *val = get_reg_val(id, vcpu->arch.pmc[i]);
1396 break;
1397 case KVM_REG_PPC_SPMC1 ... KVM_REG_PPC_SPMC2:
1398 i = id - KVM_REG_PPC_SPMC1;
1399 *val = get_reg_val(id, vcpu->arch.spmc[i]);
1400 break;
1401 case KVM_REG_PPC_SIAR:
1402 *val = get_reg_val(id, vcpu->arch.siar);
1403 break;
1404 case KVM_REG_PPC_SDAR:
1405 *val = get_reg_val(id, vcpu->arch.sdar);
1406 break;
1407 case KVM_REG_PPC_SIER:
1408 *val = get_reg_val(id, vcpu->arch.sier);
1409 break;
1410 case KVM_REG_PPC_IAMR:
1411 *val = get_reg_val(id, vcpu->arch.iamr);
1412 break;
1413 case KVM_REG_PPC_PSPB:
1414 *val = get_reg_val(id, vcpu->arch.pspb);
1415 break;
1416 case KVM_REG_PPC_DPDES:
1417 *val = get_reg_val(id, vcpu->arch.vcore->dpdes);
1418 break;
1419 case KVM_REG_PPC_VTB:
1420 *val = get_reg_val(id, vcpu->arch.vcore->vtb);
1421 break;
1422 case KVM_REG_PPC_DAWR:
1423 *val = get_reg_val(id, vcpu->arch.dawr);
1424 break;
1425 case KVM_REG_PPC_DAWRX:
1426 *val = get_reg_val(id, vcpu->arch.dawrx);
1427 break;
1428 case KVM_REG_PPC_CIABR:
1429 *val = get_reg_val(id, vcpu->arch.ciabr);
1430 break;
1431 case KVM_REG_PPC_CSIGR:
1432 *val = get_reg_val(id, vcpu->arch.csigr);
1433 break;
1434 case KVM_REG_PPC_TACR:
1435 *val = get_reg_val(id, vcpu->arch.tacr);
1436 break;
1437 case KVM_REG_PPC_TCSCR:
1438 *val = get_reg_val(id, vcpu->arch.tcscr);
1439 break;
1440 case KVM_REG_PPC_PID:
1441 *val = get_reg_val(id, vcpu->arch.pid);
1442 break;
1443 case KVM_REG_PPC_ACOP:
1444 *val = get_reg_val(id, vcpu->arch.acop);
1445 break;
1446 case KVM_REG_PPC_WORT:
1447 *val = get_reg_val(id, vcpu->arch.wort);
1448 break;
1449 case KVM_REG_PPC_TIDR:
1450 *val = get_reg_val(id, vcpu->arch.tid);
1451 break;
1452 case KVM_REG_PPC_PSSCR:
1453 *val = get_reg_val(id, vcpu->arch.psscr);
1454 break;
1455 case KVM_REG_PPC_VPA_ADDR:
1456 spin_lock(&vcpu->arch.vpa_update_lock);
1457 *val = get_reg_val(id, vcpu->arch.vpa.next_gpa);
1458 spin_unlock(&vcpu->arch.vpa_update_lock);
1459 break;
1460 case KVM_REG_PPC_VPA_SLB:
1461 spin_lock(&vcpu->arch.vpa_update_lock);
1462 val->vpaval.addr = vcpu->arch.slb_shadow.next_gpa;
1463 val->vpaval.length = vcpu->arch.slb_shadow.len;
1464 spin_unlock(&vcpu->arch.vpa_update_lock);
1465 break;
1466 case KVM_REG_PPC_VPA_DTL:
1467 spin_lock(&vcpu->arch.vpa_update_lock);
1468 val->vpaval.addr = vcpu->arch.dtl.next_gpa;
1469 val->vpaval.length = vcpu->arch.dtl.len;
1470 spin_unlock(&vcpu->arch.vpa_update_lock);
1471 break;
1472 case KVM_REG_PPC_TB_OFFSET:
1473 *val = get_reg_val(id, vcpu->arch.vcore->tb_offset);
1474 break;
1475 case KVM_REG_PPC_LPCR:
1476 case KVM_REG_PPC_LPCR_64:
1477 *val = get_reg_val(id, vcpu->arch.vcore->lpcr);
1478 break;
1479 case KVM_REG_PPC_PPR:
1480 *val = get_reg_val(id, vcpu->arch.ppr);
1481 break;
1482#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1483 case KVM_REG_PPC_TFHAR:
1484 *val = get_reg_val(id, vcpu->arch.tfhar);
1485 break;
1486 case KVM_REG_PPC_TFIAR:
1487 *val = get_reg_val(id, vcpu->arch.tfiar);
1488 break;
1489 case KVM_REG_PPC_TEXASR:
1490 *val = get_reg_val(id, vcpu->arch.texasr);
1491 break;
1492 case KVM_REG_PPC_TM_GPR0 ... KVM_REG_PPC_TM_GPR31:
1493 i = id - KVM_REG_PPC_TM_GPR0;
1494 *val = get_reg_val(id, vcpu->arch.gpr_tm[i]);
1495 break;
1496 case KVM_REG_PPC_TM_VSR0 ... KVM_REG_PPC_TM_VSR63:
1497 {
1498 int j;
1499 i = id - KVM_REG_PPC_TM_VSR0;
1500 if (i < 32)
1501 for (j = 0; j < TS_FPRWIDTH; j++)
1502 val->vsxval[j] = vcpu->arch.fp_tm.fpr[i][j];
1503 else {
1504 if (cpu_has_feature(CPU_FTR_ALTIVEC))
1505 val->vval = vcpu->arch.vr_tm.vr[i-32];
1506 else
1507 r = -ENXIO;
1508 }
1509 break;
1510 }
1511 case KVM_REG_PPC_TM_CR:
1512 *val = get_reg_val(id, vcpu->arch.cr_tm);
1513 break;
1514 case KVM_REG_PPC_TM_XER:
1515 *val = get_reg_val(id, vcpu->arch.xer_tm);
1516 break;
1517 case KVM_REG_PPC_TM_LR:
1518 *val = get_reg_val(id, vcpu->arch.lr_tm);
1519 break;
1520 case KVM_REG_PPC_TM_CTR:
1521 *val = get_reg_val(id, vcpu->arch.ctr_tm);
1522 break;
1523 case KVM_REG_PPC_TM_FPSCR:
1524 *val = get_reg_val(id, vcpu->arch.fp_tm.fpscr);
1525 break;
1526 case KVM_REG_PPC_TM_AMR:
1527 *val = get_reg_val(id, vcpu->arch.amr_tm);
1528 break;
1529 case KVM_REG_PPC_TM_PPR:
1530 *val = get_reg_val(id, vcpu->arch.ppr_tm);
1531 break;
1532 case KVM_REG_PPC_TM_VRSAVE:
1533 *val = get_reg_val(id, vcpu->arch.vrsave_tm);
1534 break;
1535 case KVM_REG_PPC_TM_VSCR:
1536 if (cpu_has_feature(CPU_FTR_ALTIVEC))
1537 *val = get_reg_val(id, vcpu->arch.vr_tm.vscr.u[3]);
1538 else
1539 r = -ENXIO;
1540 break;
1541 case KVM_REG_PPC_TM_DSCR:
1542 *val = get_reg_val(id, vcpu->arch.dscr_tm);
1543 break;
1544 case KVM_REG_PPC_TM_TAR:
1545 *val = get_reg_val(id, vcpu->arch.tar_tm);
1546 break;
1547#endif
1548 case KVM_REG_PPC_ARCH_COMPAT:
1549 *val = get_reg_val(id, vcpu->arch.vcore->arch_compat);
1550 break;
1551 case KVM_REG_PPC_DEC_EXPIRY:
1552 *val = get_reg_val(id, vcpu->arch.dec_expires +
1553 vcpu->arch.vcore->tb_offset);
1554 break;
1555 case KVM_REG_PPC_ONLINE:
1556 *val = get_reg_val(id, vcpu->arch.online);
1557 break;
1558 default:
1559 r = -EINVAL;
1560 break;
1561 }
1562
1563 return r;
1564}
1565
1566static int kvmppc_set_one_reg_hv(struct kvm_vcpu *vcpu, u64 id,
1567 union kvmppc_one_reg *val)
1568{
1569 int r = 0;
1570 long int i;
1571 unsigned long addr, len;
1572
1573 switch (id) {
1574 case KVM_REG_PPC_HIOR:
1575
1576 if (set_reg_val(id, *val))
1577 r = -EINVAL;
1578 break;
1579 case KVM_REG_PPC_DABR:
1580 vcpu->arch.dabr = set_reg_val(id, *val);
1581 break;
1582 case KVM_REG_PPC_DABRX:
1583 vcpu->arch.dabrx = set_reg_val(id, *val) & ~DABRX_HYP;
1584 break;
1585 case KVM_REG_PPC_DSCR:
1586 vcpu->arch.dscr = set_reg_val(id, *val);
1587 break;
1588 case KVM_REG_PPC_PURR:
1589 vcpu->arch.purr = set_reg_val(id, *val);
1590 break;
1591 case KVM_REG_PPC_SPURR:
1592 vcpu->arch.spurr = set_reg_val(id, *val);
1593 break;
1594 case KVM_REG_PPC_AMR:
1595 vcpu->arch.amr = set_reg_val(id, *val);
1596 break;
1597 case KVM_REG_PPC_UAMOR:
1598 vcpu->arch.uamor = set_reg_val(id, *val);
1599 break;
1600 case KVM_REG_PPC_MMCR0 ... KVM_REG_PPC_MMCRS:
1601 i = id - KVM_REG_PPC_MMCR0;
1602 vcpu->arch.mmcr[i] = set_reg_val(id, *val);
1603 break;
1604 case KVM_REG_PPC_PMC1 ... KVM_REG_PPC_PMC8:
1605 i = id - KVM_REG_PPC_PMC1;
1606 vcpu->arch.pmc[i] = set_reg_val(id, *val);
1607 break;
1608 case KVM_REG_PPC_SPMC1 ... KVM_REG_PPC_SPMC2:
1609 i = id - KVM_REG_PPC_SPMC1;
1610 vcpu->arch.spmc[i] = set_reg_val(id, *val);
1611 break;
1612 case KVM_REG_PPC_SIAR:
1613 vcpu->arch.siar = set_reg_val(id, *val);
1614 break;
1615 case KVM_REG_PPC_SDAR:
1616 vcpu->arch.sdar = set_reg_val(id, *val);
1617 break;
1618 case KVM_REG_PPC_SIER:
1619 vcpu->arch.sier = set_reg_val(id, *val);
1620 break;
1621 case KVM_REG_PPC_IAMR:
1622 vcpu->arch.iamr = set_reg_val(id, *val);
1623 break;
1624 case KVM_REG_PPC_PSPB:
1625 vcpu->arch.pspb = set_reg_val(id, *val);
1626 break;
1627 case KVM_REG_PPC_DPDES:
1628 vcpu->arch.vcore->dpdes = set_reg_val(id, *val);
1629 break;
1630 case KVM_REG_PPC_VTB:
1631 vcpu->arch.vcore->vtb = set_reg_val(id, *val);
1632 break;
1633 case KVM_REG_PPC_DAWR:
1634 vcpu->arch.dawr = set_reg_val(id, *val);
1635 break;
1636 case KVM_REG_PPC_DAWRX:
1637 vcpu->arch.dawrx = set_reg_val(id, *val) & ~DAWRX_HYP;
1638 break;
1639 case KVM_REG_PPC_CIABR:
1640 vcpu->arch.ciabr = set_reg_val(id, *val);
1641
1642 if ((vcpu->arch.ciabr & CIABR_PRIV) == CIABR_PRIV_HYPER)
1643 vcpu->arch.ciabr &= ~CIABR_PRIV;
1644 break;
1645 case KVM_REG_PPC_CSIGR:
1646 vcpu->arch.csigr = set_reg_val(id, *val);
1647 break;
1648 case KVM_REG_PPC_TACR:
1649 vcpu->arch.tacr = set_reg_val(id, *val);
1650 break;
1651 case KVM_REG_PPC_TCSCR:
1652 vcpu->arch.tcscr = set_reg_val(id, *val);
1653 break;
1654 case KVM_REG_PPC_PID:
1655 vcpu->arch.pid = set_reg_val(id, *val);
1656 break;
1657 case KVM_REG_PPC_ACOP:
1658 vcpu->arch.acop = set_reg_val(id, *val);
1659 break;
1660 case KVM_REG_PPC_WORT:
1661 vcpu->arch.wort = set_reg_val(id, *val);
1662 break;
1663 case KVM_REG_PPC_TIDR:
1664 vcpu->arch.tid = set_reg_val(id, *val);
1665 break;
1666 case KVM_REG_PPC_PSSCR:
1667 vcpu->arch.psscr = set_reg_val(id, *val) & PSSCR_GUEST_VIS;
1668 break;
1669 case KVM_REG_PPC_VPA_ADDR:
1670 addr = set_reg_val(id, *val);
1671 r = -EINVAL;
1672 if (!addr && (vcpu->arch.slb_shadow.next_gpa ||
1673 vcpu->arch.dtl.next_gpa))
1674 break;
1675 r = set_vpa(vcpu, &vcpu->arch.vpa, addr, sizeof(struct lppaca));
1676 break;
1677 case KVM_REG_PPC_VPA_SLB:
1678 addr = val->vpaval.addr;
1679 len = val->vpaval.length;
1680 r = -EINVAL;
1681 if (addr && !vcpu->arch.vpa.next_gpa)
1682 break;
1683 r = set_vpa(vcpu, &vcpu->arch.slb_shadow, addr, len);
1684 break;
1685 case KVM_REG_PPC_VPA_DTL:
1686 addr = val->vpaval.addr;
1687 len = val->vpaval.length;
1688 r = -EINVAL;
1689 if (addr && (len < sizeof(struct dtl_entry) ||
1690 !vcpu->arch.vpa.next_gpa))
1691 break;
1692 len -= len % sizeof(struct dtl_entry);
1693 r = set_vpa(vcpu, &vcpu->arch.dtl, addr, len);
1694 break;
1695 case KVM_REG_PPC_TB_OFFSET:
1696
1697 vcpu->arch.vcore->tb_offset =
1698 ALIGN(set_reg_val(id, *val), 1UL << 24);
1699 break;
1700 case KVM_REG_PPC_LPCR:
1701 kvmppc_set_lpcr(vcpu, set_reg_val(id, *val), true);
1702 break;
1703 case KVM_REG_PPC_LPCR_64:
1704 kvmppc_set_lpcr(vcpu, set_reg_val(id, *val), false);
1705 break;
1706 case KVM_REG_PPC_PPR:
1707 vcpu->arch.ppr = set_reg_val(id, *val);
1708 break;
1709#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1710 case KVM_REG_PPC_TFHAR:
1711 vcpu->arch.tfhar = set_reg_val(id, *val);
1712 break;
1713 case KVM_REG_PPC_TFIAR:
1714 vcpu->arch.tfiar = set_reg_val(id, *val);
1715 break;
1716 case KVM_REG_PPC_TEXASR:
1717 vcpu->arch.texasr = set_reg_val(id, *val);
1718 break;
1719 case KVM_REG_PPC_TM_GPR0 ... KVM_REG_PPC_TM_GPR31:
1720 i = id - KVM_REG_PPC_TM_GPR0;
1721 vcpu->arch.gpr_tm[i] = set_reg_val(id, *val);
1722 break;
1723 case KVM_REG_PPC_TM_VSR0 ... KVM_REG_PPC_TM_VSR63:
1724 {
1725 int j;
1726 i = id - KVM_REG_PPC_TM_VSR0;
1727 if (i < 32)
1728 for (j = 0; j < TS_FPRWIDTH; j++)
1729 vcpu->arch.fp_tm.fpr[i][j] = val->vsxval[j];
1730 else
1731 if (cpu_has_feature(CPU_FTR_ALTIVEC))
1732 vcpu->arch.vr_tm.vr[i-32] = val->vval;
1733 else
1734 r = -ENXIO;
1735 break;
1736 }
1737 case KVM_REG_PPC_TM_CR:
1738 vcpu->arch.cr_tm = set_reg_val(id, *val);
1739 break;
1740 case KVM_REG_PPC_TM_XER:
1741 vcpu->arch.xer_tm = set_reg_val(id, *val);
1742 break;
1743 case KVM_REG_PPC_TM_LR:
1744 vcpu->arch.lr_tm = set_reg_val(id, *val);
1745 break;
1746 case KVM_REG_PPC_TM_CTR:
1747 vcpu->arch.ctr_tm = set_reg_val(id, *val);
1748 break;
1749 case KVM_REG_PPC_TM_FPSCR:
1750 vcpu->arch.fp_tm.fpscr = set_reg_val(id, *val);
1751 break;
1752 case KVM_REG_PPC_TM_AMR:
1753 vcpu->arch.amr_tm = set_reg_val(id, *val);
1754 break;
1755 case KVM_REG_PPC_TM_PPR:
1756 vcpu->arch.ppr_tm = set_reg_val(id, *val);
1757 break;
1758 case KVM_REG_PPC_TM_VRSAVE:
1759 vcpu->arch.vrsave_tm = set_reg_val(id, *val);
1760 break;
1761 case KVM_REG_PPC_TM_VSCR:
1762 if (cpu_has_feature(CPU_FTR_ALTIVEC))
1763 vcpu->arch.vr.vscr.u[3] = set_reg_val(id, *val);
1764 else
1765 r = - ENXIO;
1766 break;
1767 case KVM_REG_PPC_TM_DSCR:
1768 vcpu->arch.dscr_tm = set_reg_val(id, *val);
1769 break;
1770 case KVM_REG_PPC_TM_TAR:
1771 vcpu->arch.tar_tm = set_reg_val(id, *val);
1772 break;
1773#endif
1774 case KVM_REG_PPC_ARCH_COMPAT:
1775 r = kvmppc_set_arch_compat(vcpu, set_reg_val(id, *val));
1776 break;
1777 case KVM_REG_PPC_DEC_EXPIRY:
1778 vcpu->arch.dec_expires = set_reg_val(id, *val) -
1779 vcpu->arch.vcore->tb_offset;
1780 break;
1781 case KVM_REG_PPC_ONLINE:
1782 i = set_reg_val(id, *val);
1783 if (i && !vcpu->arch.online)
1784 atomic_inc(&vcpu->arch.vcore->online_count);
1785 else if (!i && vcpu->arch.online)
1786 atomic_dec(&vcpu->arch.vcore->online_count);
1787 vcpu->arch.online = i;
1788 break;
1789 default:
1790 r = -EINVAL;
1791 break;
1792 }
1793
1794 return r;
1795}
1796
1797
1798
1799
1800
1801
1802
1803
1804static int threads_per_vcore(struct kvm *kvm)
1805{
1806 if (kvm->arch.threads_indep)
1807 return 1;
1808 return threads_per_subcore;
1809}
1810
1811static struct kvmppc_vcore *kvmppc_vcore_create(struct kvm *kvm, int id)
1812{
1813 struct kvmppc_vcore *vcore;
1814
1815 vcore = kzalloc(sizeof(struct kvmppc_vcore), GFP_KERNEL);
1816
1817 if (vcore == NULL)
1818 return NULL;
1819
1820 spin_lock_init(&vcore->lock);
1821 spin_lock_init(&vcore->stoltb_lock);
1822 init_swait_queue_head(&vcore->wq);
1823 vcore->preempt_tb = TB_NIL;
1824 vcore->lpcr = kvm->arch.lpcr;
1825 vcore->first_vcpuid = id;
1826 vcore->kvm = kvm;
1827 INIT_LIST_HEAD(&vcore->preempt_list);
1828
1829 return vcore;
1830}
1831
1832#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
1833static struct debugfs_timings_element {
1834 const char *name;
1835 size_t offset;
1836} timings[] = {
1837 {"rm_entry", offsetof(struct kvm_vcpu, arch.rm_entry)},
1838 {"rm_intr", offsetof(struct kvm_vcpu, arch.rm_intr)},
1839 {"rm_exit", offsetof(struct kvm_vcpu, arch.rm_exit)},
1840 {"guest", offsetof(struct kvm_vcpu, arch.guest_time)},
1841 {"cede", offsetof(struct kvm_vcpu, arch.cede_time)},
1842};
1843
1844#define N_TIMINGS (ARRAY_SIZE(timings))
1845
1846struct debugfs_timings_state {
1847 struct kvm_vcpu *vcpu;
1848 unsigned int buflen;
1849 char buf[N_TIMINGS * 100];
1850};
1851
1852static int debugfs_timings_open(struct inode *inode, struct file *file)
1853{
1854 struct kvm_vcpu *vcpu = inode->i_private;
1855 struct debugfs_timings_state *p;
1856
1857 p = kzalloc(sizeof(*p), GFP_KERNEL);
1858 if (!p)
1859 return -ENOMEM;
1860
1861 kvm_get_kvm(vcpu->kvm);
1862 p->vcpu = vcpu;
1863 file->private_data = p;
1864
1865 return nonseekable_open(inode, file);
1866}
1867
1868static int debugfs_timings_release(struct inode *inode, struct file *file)
1869{
1870 struct debugfs_timings_state *p = file->private_data;
1871
1872 kvm_put_kvm(p->vcpu->kvm);
1873 kfree(p);
1874 return 0;
1875}
1876
1877static ssize_t debugfs_timings_read(struct file *file, char __user *buf,
1878 size_t len, loff_t *ppos)
1879{
1880 struct debugfs_timings_state *p = file->private_data;
1881 struct kvm_vcpu *vcpu = p->vcpu;
1882 char *s, *buf_end;
1883 struct kvmhv_tb_accumulator tb;
1884 u64 count;
1885 loff_t pos;
1886 ssize_t n;
1887 int i, loops;
1888 bool ok;
1889
1890 if (!p->buflen) {
1891 s = p->buf;
1892 buf_end = s + sizeof(p->buf);
1893 for (i = 0; i < N_TIMINGS; ++i) {
1894 struct kvmhv_tb_accumulator *acc;
1895
1896 acc = (struct kvmhv_tb_accumulator *)
1897 ((unsigned long)vcpu + timings[i].offset);
1898 ok = false;
1899 for (loops = 0; loops < 1000; ++loops) {
1900 count = acc->seqcount;
1901 if (!(count & 1)) {
1902 smp_rmb();
1903 tb = *acc;
1904 smp_rmb();
1905 if (count == acc->seqcount) {
1906 ok = true;
1907 break;
1908 }
1909 }
1910 udelay(1);
1911 }
1912 if (!ok)
1913 snprintf(s, buf_end - s, "%s: stuck\n",
1914 timings[i].name);
1915 else
1916 snprintf(s, buf_end - s,
1917 "%s: %llu %llu %llu %llu\n",
1918 timings[i].name, count / 2,
1919 tb_to_ns(tb.tb_total),
1920 tb_to_ns(tb.tb_min),
1921 tb_to_ns(tb.tb_max));
1922 s += strlen(s);
1923 }
1924 p->buflen = s - p->buf;
1925 }
1926
1927 pos = *ppos;
1928 if (pos >= p->buflen)
1929 return 0;
1930 if (len > p->buflen - pos)
1931 len = p->buflen - pos;
1932 n = copy_to_user(buf, p->buf + pos, len);
1933 if (n) {
1934 if (n == len)
1935 return -EFAULT;
1936 len -= n;
1937 }
1938 *ppos = pos + len;
1939 return len;
1940}
1941
1942static ssize_t debugfs_timings_write(struct file *file, const char __user *buf,
1943 size_t len, loff_t *ppos)
1944{
1945 return -EACCES;
1946}
1947
1948static const struct file_operations debugfs_timings_ops = {
1949 .owner = THIS_MODULE,
1950 .open = debugfs_timings_open,
1951 .release = debugfs_timings_release,
1952 .read = debugfs_timings_read,
1953 .write = debugfs_timings_write,
1954 .llseek = generic_file_llseek,
1955};
1956
1957
1958static void debugfs_vcpu_init(struct kvm_vcpu *vcpu, unsigned int id)
1959{
1960 char buf[16];
1961 struct kvm *kvm = vcpu->kvm;
1962
1963 snprintf(buf, sizeof(buf), "vcpu%u", id);
1964 if (IS_ERR_OR_NULL(kvm->arch.debugfs_dir))
1965 return;
1966 vcpu->arch.debugfs_dir = debugfs_create_dir(buf, kvm->arch.debugfs_dir);
1967 if (IS_ERR_OR_NULL(vcpu->arch.debugfs_dir))
1968 return;
1969 vcpu->arch.debugfs_timings =
1970 debugfs_create_file("timings", 0444, vcpu->arch.debugfs_dir,
1971 vcpu, &debugfs_timings_ops);
1972}
1973
1974#else
1975static void debugfs_vcpu_init(struct kvm_vcpu *vcpu, unsigned int id)
1976{
1977}
1978#endif
1979
1980static struct kvm_vcpu *kvmppc_core_vcpu_create_hv(struct kvm *kvm,
1981 unsigned int id)
1982{
1983 struct kvm_vcpu *vcpu;
1984 int err;
1985 int core;
1986 struct kvmppc_vcore *vcore;
1987
1988 err = -ENOMEM;
1989 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
1990 if (!vcpu)
1991 goto out;
1992
1993 err = kvm_vcpu_init(vcpu, kvm, id);
1994 if (err)
1995 goto free_vcpu;
1996
1997 vcpu->arch.shared = &vcpu->arch.shregs;
1998#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
1999
2000
2001
2002
2003#ifdef __BIG_ENDIAN__
2004 vcpu->arch.shared_big_endian = true;
2005#else
2006 vcpu->arch.shared_big_endian = false;
2007#endif
2008#endif
2009 vcpu->arch.mmcr[0] = MMCR0_FC;
2010 vcpu->arch.ctrl = CTRL_RUNLATCH;
2011
2012 kvmppc_set_pvr_hv(vcpu, mfspr(SPRN_PVR));
2013 spin_lock_init(&vcpu->arch.vpa_update_lock);
2014 spin_lock_init(&vcpu->arch.tbacct_lock);
2015 vcpu->arch.busy_preempt = TB_NIL;
2016 vcpu->arch.intr_msr = MSR_SF | MSR_ME;
2017
2018
2019
2020
2021
2022
2023
2024 vcpu->arch.hfscr = mfspr(SPRN_HFSCR);
2025 if (cpu_has_feature(CPU_FTR_P9_TM_HV_ASSIST))
2026 vcpu->arch.hfscr |= HFSCR_TM;
2027 else if (!cpu_has_feature(CPU_FTR_TM_COMP))
2028 vcpu->arch.hfscr &= ~HFSCR_TM;
2029 if (cpu_has_feature(CPU_FTR_ARCH_300))
2030 vcpu->arch.hfscr &= ~HFSCR_MSGP;
2031
2032 kvmppc_mmu_book3s_hv_init(vcpu);
2033
2034 vcpu->arch.state = KVMPPC_VCPU_NOTREADY;
2035
2036 init_waitqueue_head(&vcpu->arch.cpu_run);
2037
2038 mutex_lock(&kvm->lock);
2039 vcore = NULL;
2040 err = -EINVAL;
2041 if (cpu_has_feature(CPU_FTR_ARCH_300)) {
2042 if (id >= (KVM_MAX_VCPUS * kvm->arch.emul_smt_mode)) {
2043 pr_devel("KVM: VCPU ID too high\n");
2044 core = KVM_MAX_VCORES;
2045 } else {
2046 BUG_ON(kvm->arch.smt_mode != 1);
2047 core = kvmppc_pack_vcpu_id(kvm, id);
2048 }
2049 } else {
2050 core = id / kvm->arch.smt_mode;
2051 }
2052 if (core < KVM_MAX_VCORES) {
2053 vcore = kvm->arch.vcores[core];
2054 if (vcore && cpu_has_feature(CPU_FTR_ARCH_300)) {
2055 pr_devel("KVM: collision on id %u", id);
2056 vcore = NULL;
2057 } else if (!vcore) {
2058 err = -ENOMEM;
2059 vcore = kvmppc_vcore_create(kvm,
2060 id & ~(kvm->arch.smt_mode - 1));
2061 kvm->arch.vcores[core] = vcore;
2062 kvm->arch.online_vcores++;
2063 }
2064 }
2065 mutex_unlock(&kvm->lock);
2066
2067 if (!vcore)
2068 goto free_vcpu;
2069
2070 spin_lock(&vcore->lock);
2071 ++vcore->num_threads;
2072 spin_unlock(&vcore->lock);
2073 vcpu->arch.vcore = vcore;
2074 vcpu->arch.ptid = vcpu->vcpu_id - vcore->first_vcpuid;
2075 vcpu->arch.thread_cpu = -1;
2076 vcpu->arch.prev_cpu = -1;
2077
2078 vcpu->arch.cpu_type = KVM_CPU_3S_64;
2079 kvmppc_sanity_check(vcpu);
2080
2081 debugfs_vcpu_init(vcpu, id);
2082
2083 return vcpu;
2084
2085free_vcpu:
2086 kmem_cache_free(kvm_vcpu_cache, vcpu);
2087out:
2088 return ERR_PTR(err);
2089}
2090
2091static int kvmhv_set_smt_mode(struct kvm *kvm, unsigned long smt_mode,
2092 unsigned long flags)
2093{
2094 int err;
2095 int esmt = 0;
2096
2097 if (flags)
2098 return -EINVAL;
2099 if (smt_mode > MAX_SMT_THREADS || !is_power_of_2(smt_mode))
2100 return -EINVAL;
2101 if (!cpu_has_feature(CPU_FTR_ARCH_300)) {
2102
2103
2104
2105
2106 if (smt_mode > threads_per_subcore)
2107 return -EINVAL;
2108 } else {
2109
2110
2111
2112
2113 esmt = smt_mode;
2114 smt_mode = 1;
2115 }
2116 mutex_lock(&kvm->lock);
2117 err = -EBUSY;
2118 if (!kvm->arch.online_vcores) {
2119 kvm->arch.smt_mode = smt_mode;
2120 kvm->arch.emul_smt_mode = esmt;
2121 err = 0;
2122 }
2123 mutex_unlock(&kvm->lock);
2124
2125 return err;
2126}
2127
2128static void unpin_vpa(struct kvm *kvm, struct kvmppc_vpa *vpa)
2129{
2130 if (vpa->pinned_addr)
2131 kvmppc_unpin_guest_page(kvm, vpa->pinned_addr, vpa->gpa,
2132 vpa->dirty);
2133}
2134
2135static void kvmppc_core_vcpu_free_hv(struct kvm_vcpu *vcpu)
2136{
2137 spin_lock(&vcpu->arch.vpa_update_lock);
2138 unpin_vpa(vcpu->kvm, &vcpu->arch.dtl);
2139 unpin_vpa(vcpu->kvm, &vcpu->arch.slb_shadow);
2140 unpin_vpa(vcpu->kvm, &vcpu->arch.vpa);
2141 spin_unlock(&vcpu->arch.vpa_update_lock);
2142 kvm_vcpu_uninit(vcpu);
2143 kmem_cache_free(kvm_vcpu_cache, vcpu);
2144}
2145
2146static int kvmppc_core_check_requests_hv(struct kvm_vcpu *vcpu)
2147{
2148
2149 return 1;
2150}
2151
2152static void kvmppc_set_timer(struct kvm_vcpu *vcpu)
2153{
2154 unsigned long dec_nsec, now;
2155
2156 now = get_tb();
2157 if (now > vcpu->arch.dec_expires) {
2158
2159 kvmppc_core_queue_dec(vcpu);
2160 kvmppc_core_prepare_to_enter(vcpu);
2161 return;
2162 }
2163 dec_nsec = (vcpu->arch.dec_expires - now) * NSEC_PER_SEC
2164 / tb_ticks_per_sec;
2165 hrtimer_start(&vcpu->arch.dec_timer, dec_nsec, HRTIMER_MODE_REL);
2166 vcpu->arch.timer_running = 1;
2167}
2168
2169static void kvmppc_end_cede(struct kvm_vcpu *vcpu)
2170{
2171 vcpu->arch.ceded = 0;
2172 if (vcpu->arch.timer_running) {
2173 hrtimer_try_to_cancel(&vcpu->arch.dec_timer);
2174 vcpu->arch.timer_running = 0;
2175 }
2176}
2177
2178extern int __kvmppc_vcore_entry(void);
2179
2180static void kvmppc_remove_runnable(struct kvmppc_vcore *vc,
2181 struct kvm_vcpu *vcpu)
2182{
2183 u64 now;
2184
2185 if (vcpu->arch.state != KVMPPC_VCPU_RUNNABLE)
2186 return;
2187 spin_lock_irq(&vcpu->arch.tbacct_lock);
2188 now = mftb();
2189 vcpu->arch.busy_stolen += vcore_stolen_time(vc, now) -
2190 vcpu->arch.stolen_logged;
2191 vcpu->arch.busy_preempt = now;
2192 vcpu->arch.state = KVMPPC_VCPU_BUSY_IN_HOST;
2193 spin_unlock_irq(&vcpu->arch.tbacct_lock);
2194 --vc->n_runnable;
2195 WRITE_ONCE(vc->runnable_threads[vcpu->arch.ptid], NULL);
2196}
2197
2198static int kvmppc_grab_hwthread(int cpu)
2199{
2200 struct paca_struct *tpaca;
2201 long timeout = 10000;
2202
2203 tpaca = paca_ptrs[cpu];
2204
2205
2206 tpaca->kvm_hstate.kvm_vcpu = NULL;
2207 tpaca->kvm_hstate.kvm_vcore = NULL;
2208 tpaca->kvm_hstate.napping = 0;
2209 smp_wmb();
2210 tpaca->kvm_hstate.hwthread_req = 1;
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221 smp_mb();
2222 while (tpaca->kvm_hstate.hwthread_state == KVM_HWTHREAD_IN_KERNEL) {
2223 if (--timeout <= 0) {
2224 pr_err("KVM: couldn't grab cpu %d\n", cpu);
2225 return -EBUSY;
2226 }
2227 udelay(1);
2228 }
2229 return 0;
2230}
2231
2232static void kvmppc_release_hwthread(int cpu)
2233{
2234 struct paca_struct *tpaca;
2235
2236 tpaca = paca_ptrs[cpu];
2237 tpaca->kvm_hstate.hwthread_req = 0;
2238 tpaca->kvm_hstate.kvm_vcpu = NULL;
2239 tpaca->kvm_hstate.kvm_vcore = NULL;
2240 tpaca->kvm_hstate.kvm_split_mode = NULL;
2241}
2242
2243static void radix_flush_cpu(struct kvm *kvm, int cpu, struct kvm_vcpu *vcpu)
2244{
2245 int i;
2246
2247 cpu = cpu_first_thread_sibling(cpu);
2248 cpumask_set_cpu(cpu, &kvm->arch.need_tlb_flush);
2249
2250
2251
2252
2253
2254 smp_mb();
2255 for (i = 0; i < threads_per_core; ++i)
2256 if (cpumask_test_cpu(cpu + i, &kvm->arch.cpu_in_guest))
2257 smp_call_function_single(cpu + i, do_nothing, NULL, 1);
2258}
2259
2260static void kvmppc_prepare_radix_vcpu(struct kvm_vcpu *vcpu, int pcpu)
2261{
2262 struct kvm *kvm = vcpu->kvm;
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276 if (vcpu->arch.prev_cpu != pcpu) {
2277 if (vcpu->arch.prev_cpu >= 0 &&
2278 cpu_first_thread_sibling(vcpu->arch.prev_cpu) !=
2279 cpu_first_thread_sibling(pcpu))
2280 radix_flush_cpu(kvm, vcpu->arch.prev_cpu, vcpu);
2281 vcpu->arch.prev_cpu = pcpu;
2282 }
2283}
2284
2285static void kvmppc_start_thread(struct kvm_vcpu *vcpu, struct kvmppc_vcore *vc)
2286{
2287 int cpu;
2288 struct paca_struct *tpaca;
2289 struct kvm *kvm = vc->kvm;
2290
2291 cpu = vc->pcpu;
2292 if (vcpu) {
2293 if (vcpu->arch.timer_running) {
2294 hrtimer_try_to_cancel(&vcpu->arch.dec_timer);
2295 vcpu->arch.timer_running = 0;
2296 }
2297 cpu += vcpu->arch.ptid;
2298 vcpu->cpu = vc->pcpu;
2299 vcpu->arch.thread_cpu = cpu;
2300 cpumask_set_cpu(cpu, &kvm->arch.cpu_in_guest);
2301 }
2302 tpaca = paca_ptrs[cpu];
2303 tpaca->kvm_hstate.kvm_vcpu = vcpu;
2304 tpaca->kvm_hstate.ptid = cpu - vc->pcpu;
2305 tpaca->kvm_hstate.fake_suspend = 0;
2306
2307 smp_wmb();
2308 tpaca->kvm_hstate.kvm_vcore = vc;
2309 if (cpu != smp_processor_id())
2310 kvmppc_ipi_thread(cpu);
2311}
2312
2313static void kvmppc_wait_for_nap(int n_threads)
2314{
2315 int cpu = smp_processor_id();
2316 int i, loops;
2317
2318 if (n_threads <= 1)
2319 return;
2320 for (loops = 0; loops < 1000000; ++loops) {
2321
2322
2323
2324
2325
2326
2327 for (i = 1; i < n_threads; ++i)
2328 if (paca_ptrs[cpu + i]->kvm_hstate.kvm_vcore)
2329 break;
2330 if (i == n_threads) {
2331 HMT_medium();
2332 return;
2333 }
2334 HMT_low();
2335 }
2336 HMT_medium();
2337 for (i = 1; i < n_threads; ++i)
2338 if (paca_ptrs[cpu + i]->kvm_hstate.kvm_vcore)
2339 pr_err("KVM: CPU %d seems to be stuck\n", cpu + i);
2340}
2341
2342
2343
2344
2345
2346
2347static int on_primary_thread(void)
2348{
2349 int cpu = smp_processor_id();
2350 int thr;
2351
2352
2353 if (cpu_thread_in_subcore(cpu))
2354 return 0;
2355
2356 thr = 0;
2357 while (++thr < threads_per_subcore)
2358 if (cpu_online(cpu + thr))
2359 return 0;
2360
2361
2362 for (thr = 1; thr < threads_per_subcore; ++thr) {
2363 if (kvmppc_grab_hwthread(cpu + thr)) {
2364
2365 do {
2366 kvmppc_release_hwthread(cpu + thr);
2367 } while (--thr > 0);
2368 return 0;
2369 }
2370 }
2371 return 1;
2372}
2373
2374
2375
2376
2377
2378
2379struct preempted_vcore_list {
2380 struct list_head list;
2381 spinlock_t lock;
2382};
2383
2384static DEFINE_PER_CPU(struct preempted_vcore_list, preempted_vcores);
2385
2386static void init_vcore_lists(void)
2387{
2388 int cpu;
2389
2390 for_each_possible_cpu(cpu) {
2391 struct preempted_vcore_list *lp = &per_cpu(preempted_vcores, cpu);
2392 spin_lock_init(&lp->lock);
2393 INIT_LIST_HEAD(&lp->list);
2394 }
2395}
2396
2397static void kvmppc_vcore_preempt(struct kvmppc_vcore *vc)
2398{
2399 struct preempted_vcore_list *lp = this_cpu_ptr(&preempted_vcores);
2400
2401 vc->vcore_state = VCORE_PREEMPT;
2402 vc->pcpu = smp_processor_id();
2403 if (vc->num_threads < threads_per_vcore(vc->kvm)) {
2404 spin_lock(&lp->lock);
2405 list_add_tail(&vc->preempt_list, &lp->list);
2406 spin_unlock(&lp->lock);
2407 }
2408
2409
2410 kvmppc_core_start_stolen(vc);
2411}
2412
2413static void kvmppc_vcore_end_preempt(struct kvmppc_vcore *vc)
2414{
2415 struct preempted_vcore_list *lp;
2416
2417 kvmppc_core_end_stolen(vc);
2418 if (!list_empty(&vc->preempt_list)) {
2419 lp = &per_cpu(preempted_vcores, vc->pcpu);
2420 spin_lock(&lp->lock);
2421 list_del_init(&vc->preempt_list);
2422 spin_unlock(&lp->lock);
2423 }
2424 vc->vcore_state = VCORE_INACTIVE;
2425}
2426
2427
2428
2429
2430
2431struct core_info {
2432 int n_subcores;
2433 int max_subcore_threads;
2434 int total_threads;
2435 int subcore_threads[MAX_SUBCORES];
2436 struct kvmppc_vcore *vc[MAX_SUBCORES];
2437};
2438
2439
2440
2441
2442
2443static int subcore_thread_map[MAX_SUBCORES] = { 0, 4, 2, 6 };
2444
2445static void init_core_info(struct core_info *cip, struct kvmppc_vcore *vc)
2446{
2447 memset(cip, 0, sizeof(*cip));
2448 cip->n_subcores = 1;
2449 cip->max_subcore_threads = vc->num_threads;
2450 cip->total_threads = vc->num_threads;
2451 cip->subcore_threads[0] = vc->num_threads;
2452 cip->vc[0] = vc;
2453}
2454
2455static bool subcore_config_ok(int n_subcores, int n_threads)
2456{
2457
2458
2459
2460
2461 if (cpu_has_feature(CPU_FTR_ARCH_300))
2462 return n_subcores <= 4 && n_threads == 1;
2463
2464
2465 if (n_subcores > 1 && threads_per_subcore < MAX_SMT_THREADS)
2466 return false;
2467 if (n_subcores > MAX_SUBCORES)
2468 return false;
2469 if (n_subcores > 1) {
2470 if (!(dynamic_mt_modes & 2))
2471 n_subcores = 4;
2472 if (n_subcores > 2 && !(dynamic_mt_modes & 4))
2473 return false;
2474 }
2475
2476 return n_subcores * roundup_pow_of_two(n_threads) <= MAX_SMT_THREADS;
2477}
2478
2479static void init_vcore_to_run(struct kvmppc_vcore *vc)
2480{
2481 vc->entry_exit_map = 0;
2482 vc->in_guest = 0;
2483 vc->napping_threads = 0;
2484 vc->conferring_threads = 0;
2485 vc->tb_offset_applied = 0;
2486}
2487
2488static bool can_dynamic_split(struct kvmppc_vcore *vc, struct core_info *cip)
2489{
2490 int n_threads = vc->num_threads;
2491 int sub;
2492
2493 if (!cpu_has_feature(CPU_FTR_ARCH_207S))
2494 return false;
2495
2496
2497 if (no_mixing_hpt_and_radix &&
2498 kvm_is_radix(vc->kvm) != kvm_is_radix(cip->vc[0]->kvm))
2499 return false;
2500
2501 if (n_threads < cip->max_subcore_threads)
2502 n_threads = cip->max_subcore_threads;
2503 if (!subcore_config_ok(cip->n_subcores + 1, n_threads))
2504 return false;
2505 cip->max_subcore_threads = n_threads;
2506
2507 sub = cip->n_subcores;
2508 ++cip->n_subcores;
2509 cip->total_threads += vc->num_threads;
2510 cip->subcore_threads[sub] = vc->num_threads;
2511 cip->vc[sub] = vc;
2512 init_vcore_to_run(vc);
2513 list_del_init(&vc->preempt_list);
2514
2515 return true;
2516}
2517
2518
2519
2520
2521
2522static bool can_piggyback(struct kvmppc_vcore *pvc, struct core_info *cip,
2523 int target_threads)
2524{
2525 if (cip->total_threads + pvc->num_threads > target_threads)
2526 return false;
2527
2528 return can_dynamic_split(pvc, cip);
2529}
2530
2531static void prepare_threads(struct kvmppc_vcore *vc)
2532{
2533 int i;
2534 struct kvm_vcpu *vcpu;
2535
2536 for_each_runnable_thread(i, vcpu, vc) {
2537 if (signal_pending(vcpu->arch.run_task))
2538 vcpu->arch.ret = -EINTR;
2539 else if (vcpu->arch.vpa.update_pending ||
2540 vcpu->arch.slb_shadow.update_pending ||
2541 vcpu->arch.dtl.update_pending)
2542 vcpu->arch.ret = RESUME_GUEST;
2543 else
2544 continue;
2545 kvmppc_remove_runnable(vc, vcpu);
2546 wake_up(&vcpu->arch.cpu_run);
2547 }
2548}
2549
2550static void collect_piggybacks(struct core_info *cip, int target_threads)
2551{
2552 struct preempted_vcore_list *lp = this_cpu_ptr(&preempted_vcores);
2553 struct kvmppc_vcore *pvc, *vcnext;
2554
2555 spin_lock(&lp->lock);
2556 list_for_each_entry_safe(pvc, vcnext, &lp->list, preempt_list) {
2557 if (!spin_trylock(&pvc->lock))
2558 continue;
2559 prepare_threads(pvc);
2560 if (!pvc->n_runnable) {
2561 list_del_init(&pvc->preempt_list);
2562 if (pvc->runner == NULL) {
2563 pvc->vcore_state = VCORE_INACTIVE;
2564 kvmppc_core_end_stolen(pvc);
2565 }
2566 spin_unlock(&pvc->lock);
2567 continue;
2568 }
2569 if (!can_piggyback(pvc, cip, target_threads)) {
2570 spin_unlock(&pvc->lock);
2571 continue;
2572 }
2573 kvmppc_core_end_stolen(pvc);
2574 pvc->vcore_state = VCORE_PIGGYBACK;
2575 if (cip->total_threads >= target_threads)
2576 break;
2577 }
2578 spin_unlock(&lp->lock);
2579}
2580
2581static bool recheck_signals(struct core_info *cip)
2582{
2583 int sub, i;
2584 struct kvm_vcpu *vcpu;
2585
2586 for (sub = 0; sub < cip->n_subcores; ++sub)
2587 for_each_runnable_thread(i, vcpu, cip->vc[sub])
2588 if (signal_pending(vcpu->arch.run_task))
2589 return true;
2590 return false;
2591}
2592
2593static void post_guest_process(struct kvmppc_vcore *vc, bool is_master)
2594{
2595 int still_running = 0, i;
2596 u64 now;
2597 long ret;
2598 struct kvm_vcpu *vcpu;
2599
2600 spin_lock(&vc->lock);
2601 now = get_tb();
2602 for_each_runnable_thread(i, vcpu, vc) {
2603
2604 if (now < vcpu->arch.dec_expires &&
2605 kvmppc_core_pending_dec(vcpu))
2606 kvmppc_core_dequeue_dec(vcpu);
2607
2608 trace_kvm_guest_exit(vcpu);
2609
2610 ret = RESUME_GUEST;
2611 if (vcpu->arch.trap)
2612 ret = kvmppc_handle_exit_hv(vcpu->arch.kvm_run, vcpu,
2613 vcpu->arch.run_task);
2614
2615 vcpu->arch.ret = ret;
2616 vcpu->arch.trap = 0;
2617
2618 if (is_kvmppc_resume_guest(vcpu->arch.ret)) {
2619 if (vcpu->arch.pending_exceptions)
2620 kvmppc_core_prepare_to_enter(vcpu);
2621 if (vcpu->arch.ceded)
2622 kvmppc_set_timer(vcpu);
2623 else
2624 ++still_running;
2625 } else {
2626 kvmppc_remove_runnable(vc, vcpu);
2627 wake_up(&vcpu->arch.cpu_run);
2628 }
2629 }
2630 if (!is_master) {
2631 if (still_running > 0) {
2632 kvmppc_vcore_preempt(vc);
2633 } else if (vc->runner) {
2634 vc->vcore_state = VCORE_PREEMPT;
2635 kvmppc_core_start_stolen(vc);
2636 } else {
2637 vc->vcore_state = VCORE_INACTIVE;
2638 }
2639 if (vc->n_runnable > 0 && vc->runner == NULL) {
2640
2641 i = -1;
2642 vcpu = next_runnable_thread(vc, &i);
2643 wake_up(&vcpu->arch.cpu_run);
2644 }
2645 }
2646 spin_unlock(&vc->lock);
2647}
2648
2649
2650
2651
2652
2653
2654static inline int kvmppc_clear_host_core(unsigned int cpu)
2655{
2656 int core;
2657
2658 if (!kvmppc_host_rm_ops_hv || cpu_thread_in_core(cpu))
2659 return 0;
2660
2661
2662
2663
2664
2665 core = cpu >> threads_shift;
2666 kvmppc_host_rm_ops_hv->rm_core[core].rm_state.in_host = 0;
2667 return 0;
2668}
2669
2670
2671
2672
2673
2674
2675static inline int kvmppc_set_host_core(unsigned int cpu)
2676{
2677 int core;
2678
2679 if (!kvmppc_host_rm_ops_hv || cpu_thread_in_core(cpu))
2680 return 0;
2681
2682
2683
2684
2685
2686 core = cpu >> threads_shift;
2687 kvmppc_host_rm_ops_hv->rm_core[core].rm_state.in_host = 1;
2688 return 0;
2689}
2690
2691static void set_irq_happened(int trap)
2692{
2693 switch (trap) {
2694 case BOOK3S_INTERRUPT_EXTERNAL:
2695 local_paca->irq_happened |= PACA_IRQ_EE;
2696 break;
2697 case BOOK3S_INTERRUPT_H_DOORBELL:
2698 local_paca->irq_happened |= PACA_IRQ_DBELL;
2699 break;
2700 case BOOK3S_INTERRUPT_HMI:
2701 local_paca->irq_happened |= PACA_IRQ_HMI;
2702 break;
2703 case BOOK3S_INTERRUPT_SYSTEM_RESET:
2704 replay_system_reset();
2705 break;
2706 }
2707}
2708
2709
2710
2711
2712
2713static noinline void kvmppc_run_core(struct kvmppc_vcore *vc)
2714{
2715 struct kvm_vcpu *vcpu;
2716 int i;
2717 int srcu_idx;
2718 struct core_info core_info;
2719 struct kvmppc_vcore *pvc;
2720 struct kvm_split_mode split_info, *sip;
2721 int split, subcore_size, active;
2722 int sub;
2723 bool thr0_done;
2724 unsigned long cmd_bit, stat_bit;
2725 int pcpu, thr;
2726 int target_threads;
2727 int controlled_threads;
2728 int trap;
2729 bool is_power8;
2730 bool hpt_on_radix;
2731
2732
2733
2734
2735
2736 prepare_threads(vc);
2737
2738
2739 if (vc->runner->arch.state != KVMPPC_VCPU_RUNNABLE)
2740 return;
2741
2742
2743
2744
2745 init_vcore_to_run(vc);
2746 vc->preempt_tb = TB_NIL;
2747
2748
2749
2750
2751
2752
2753 controlled_threads = threads_per_vcore(vc->kvm);
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763 hpt_on_radix = no_mixing_hpt_and_radix && radix_enabled() &&
2764 !kvm_is_radix(vc->kvm);
2765 if (((controlled_threads > 1) &&
2766 ((vc->num_threads > threads_per_subcore) || !on_primary_thread())) ||
2767 (hpt_on_radix && vc->kvm->arch.threads_indep)) {
2768 for_each_runnable_thread(i, vcpu, vc) {
2769 vcpu->arch.ret = -EBUSY;
2770 kvmppc_remove_runnable(vc, vcpu);
2771 wake_up(&vcpu->arch.cpu_run);
2772 }
2773 goto out;
2774 }
2775
2776
2777
2778
2779
2780 init_core_info(&core_info, vc);
2781 pcpu = smp_processor_id();
2782 target_threads = controlled_threads;
2783 if (target_smt_mode && target_smt_mode < target_threads)
2784 target_threads = target_smt_mode;
2785 if (vc->num_threads < target_threads)
2786 collect_piggybacks(&core_info, target_threads);
2787
2788
2789
2790
2791
2792
2793 pcpu = smp_processor_id();
2794 if (kvm_is_radix(vc->kvm)) {
2795 for (sub = 0; sub < core_info.n_subcores; ++sub)
2796 for_each_runnable_thread(i, vcpu, core_info.vc[sub])
2797 kvmppc_prepare_radix_vcpu(vcpu, pcpu);
2798 }
2799
2800
2801
2802
2803
2804
2805
2806
2807 local_irq_disable();
2808 hard_irq_disable();
2809 if (lazy_irq_pending() || need_resched() ||
2810 recheck_signals(&core_info) || !vc->kvm->arch.mmu_ready) {
2811 local_irq_enable();
2812 vc->vcore_state = VCORE_INACTIVE;
2813
2814 for (sub = 1; sub < core_info.n_subcores; ++sub) {
2815 pvc = core_info.vc[sub];
2816
2817 kvmppc_vcore_preempt(pvc);
2818 spin_unlock(&pvc->lock);
2819 }
2820 for (i = 0; i < controlled_threads; ++i)
2821 kvmppc_release_hwthread(pcpu + i);
2822 return;
2823 }
2824
2825 kvmppc_clear_host_core(pcpu);
2826
2827
2828 subcore_size = threads_per_subcore;
2829 cmd_bit = stat_bit = 0;
2830 split = core_info.n_subcores;
2831 sip = NULL;
2832 is_power8 = cpu_has_feature(CPU_FTR_ARCH_207S)
2833 && !cpu_has_feature(CPU_FTR_ARCH_300);
2834
2835 if (split > 1 || hpt_on_radix) {
2836 sip = &split_info;
2837 memset(&split_info, 0, sizeof(split_info));
2838 for (sub = 0; sub < core_info.n_subcores; ++sub)
2839 split_info.vc[sub] = core_info.vc[sub];
2840
2841 if (is_power8) {
2842 if (split == 2 && (dynamic_mt_modes & 2)) {
2843 cmd_bit = HID0_POWER8_1TO2LPAR;
2844 stat_bit = HID0_POWER8_2LPARMODE;
2845 } else {
2846 split = 4;
2847 cmd_bit = HID0_POWER8_1TO4LPAR;
2848 stat_bit = HID0_POWER8_4LPARMODE;
2849 }
2850 subcore_size = MAX_SMT_THREADS / split;
2851 split_info.rpr = mfspr(SPRN_RPR);
2852 split_info.pmmar = mfspr(SPRN_PMMAR);
2853 split_info.ldbar = mfspr(SPRN_LDBAR);
2854 split_info.subcore_size = subcore_size;
2855 } else {
2856 split_info.subcore_size = 1;
2857 if (hpt_on_radix) {
2858
2859 split_info.lpcr_req = vc->lpcr;
2860 split_info.lpidr_req = vc->kvm->arch.lpid;
2861 split_info.host_lpcr = vc->kvm->arch.host_lpcr;
2862 split_info.do_set = 1;
2863 }
2864 }
2865
2866
2867 smp_wmb();
2868 }
2869
2870 for (thr = 0; thr < controlled_threads; ++thr) {
2871 struct paca_struct *paca = paca_ptrs[pcpu + thr];
2872
2873 paca->kvm_hstate.tid = thr;
2874 paca->kvm_hstate.napping = 0;
2875 paca->kvm_hstate.kvm_split_mode = sip;
2876 }
2877
2878
2879 if (cmd_bit) {
2880 unsigned long hid0 = mfspr(SPRN_HID0);
2881
2882 hid0 |= cmd_bit | HID0_POWER8_DYNLPARDIS;
2883 mb();
2884 mtspr(SPRN_HID0, hid0);
2885 isync();
2886 for (;;) {
2887 hid0 = mfspr(SPRN_HID0);
2888 if (hid0 & stat_bit)
2889 break;
2890 cpu_relax();
2891 }
2892 }
2893
2894
2895
2896
2897
2898
2899 if (is_power8) {
2900 unsigned long rwmr_val = RWMR_RPA_P8_8THREAD;
2901 int n_online = atomic_read(&vc->online_count);
2902
2903
2904
2905
2906
2907 if (split == 1 && threads_per_subcore == MAX_SMT_THREADS &&
2908 n_online >= 1 && n_online <= MAX_SMT_THREADS)
2909 rwmr_val = p8_rwmr_values[n_online];
2910 mtspr(SPRN_RWMR, rwmr_val);
2911 }
2912
2913
2914 active = 0;
2915 for (sub = 0; sub < core_info.n_subcores; ++sub) {
2916 thr = is_power8 ? subcore_thread_map[sub] : sub;
2917 thr0_done = false;
2918 active |= 1 << thr;
2919 pvc = core_info.vc[sub];
2920 pvc->pcpu = pcpu + thr;
2921 for_each_runnable_thread(i, vcpu, pvc) {
2922 kvmppc_start_thread(vcpu, pvc);
2923 kvmppc_create_dtl_entry(vcpu, pvc);
2924 trace_kvm_guest_enter(vcpu);
2925 if (!vcpu->arch.ptid)
2926 thr0_done = true;
2927 active |= 1 << (thr + vcpu->arch.ptid);
2928 }
2929
2930
2931
2932
2933 if (!thr0_done)
2934 kvmppc_start_thread(NULL, pvc);
2935 }
2936
2937
2938
2939
2940
2941 smp_mb();
2942
2943
2944
2945
2946
2947
2948
2949
2950 if (cmd_bit || hpt_on_radix) {
2951 split_info.do_nap = 1;
2952 for (thr = 1; thr < threads_per_subcore; ++thr)
2953 if (!(active & (1 << thr)))
2954 kvmppc_ipi_thread(pcpu + thr);
2955 }
2956
2957 vc->vcore_state = VCORE_RUNNING;
2958 preempt_disable();
2959
2960 trace_kvmppc_run_core(vc, 0);
2961
2962 for (sub = 0; sub < core_info.n_subcores; ++sub)
2963 spin_unlock(&core_info.vc[sub]->lock);
2964
2965 if (kvm_is_radix(vc->kvm)) {
2966 int tmp = pcpu;
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978 mtspr(SPRN_LPID, vc->kvm->arch.lpid);
2979 isync();
2980
2981 if (cpu_has_feature(CPU_FTR_ARCH_300))
2982 tmp &= ~0x3UL;
2983
2984 if (cpumask_test_cpu(tmp, &vc->kvm->arch.need_tlb_flush)) {
2985 radix__local_flush_tlb_lpid_guest(vc->kvm->arch.lpid);
2986
2987 cpumask_clear_cpu(tmp, &vc->kvm->arch.need_tlb_flush);
2988 }
2989 }
2990
2991
2992
2993
2994
2995 trace_hardirqs_on();
2996
2997 guest_enter_irqoff();
2998
2999 srcu_idx = srcu_read_lock(&vc->kvm->srcu);
3000
3001 this_cpu_disable_ftrace();
3002
3003 trap = __kvmppc_vcore_entry();
3004
3005 this_cpu_enable_ftrace();
3006
3007 srcu_read_unlock(&vc->kvm->srcu, srcu_idx);
3008
3009 trace_hardirqs_off();
3010 set_irq_happened(trap);
3011
3012 spin_lock(&vc->lock);
3013
3014 vc->vcore_state = VCORE_EXITING;
3015
3016
3017 kvmppc_wait_for_nap(controlled_threads);
3018
3019
3020 if (cmd_bit) {
3021 unsigned long hid0 = mfspr(SPRN_HID0);
3022 unsigned long loops = 0;
3023
3024 hid0 &= ~HID0_POWER8_DYNLPARDIS;
3025 stat_bit = HID0_POWER8_2LPARMODE | HID0_POWER8_4LPARMODE;
3026 mb();
3027 mtspr(SPRN_HID0, hid0);
3028 isync();
3029 for (;;) {
3030 hid0 = mfspr(SPRN_HID0);
3031 if (!(hid0 & stat_bit))
3032 break;
3033 cpu_relax();
3034 ++loops;
3035 }
3036 } else if (hpt_on_radix) {
3037
3038 for (thr = 1; thr < controlled_threads; ++thr) {
3039 struct paca_struct *paca = paca_ptrs[pcpu + thr];
3040
3041 while (paca->kvm_hstate.kvm_split_mode) {
3042 HMT_low();
3043 barrier();
3044 }
3045 HMT_medium();
3046 }
3047 }
3048 split_info.do_nap = 0;
3049
3050 kvmppc_set_host_core(pcpu);
3051
3052 local_irq_enable();
3053 guest_exit();
3054
3055
3056 for (i = 0; i < controlled_threads; ++i) {
3057 kvmppc_release_hwthread(pcpu + i);
3058 if (sip && sip->napped[i])
3059 kvmppc_ipi_thread(pcpu + i);
3060 cpumask_clear_cpu(pcpu + i, &vc->kvm->arch.cpu_in_guest);
3061 }
3062
3063 spin_unlock(&vc->lock);
3064
3065
3066 smp_mb();
3067
3068 preempt_enable();
3069
3070 for (sub = 0; sub < core_info.n_subcores; ++sub) {
3071 pvc = core_info.vc[sub];
3072 post_guest_process(pvc, pvc == vc);
3073 }
3074
3075 spin_lock(&vc->lock);
3076
3077 out:
3078 vc->vcore_state = VCORE_INACTIVE;
3079 trace_kvmppc_run_core(vc, 1);
3080}
3081
3082
3083
3084
3085
3086static void kvmppc_wait_for_exec(struct kvmppc_vcore *vc,
3087 struct kvm_vcpu *vcpu, int wait_state)
3088{
3089 DEFINE_WAIT(wait);
3090
3091 prepare_to_wait(&vcpu->arch.cpu_run, &wait, wait_state);
3092 if (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE) {
3093 spin_unlock(&vc->lock);
3094 schedule();
3095 spin_lock(&vc->lock);
3096 }
3097 finish_wait(&vcpu->arch.cpu_run, &wait);
3098}
3099
3100static void grow_halt_poll_ns(struct kvmppc_vcore *vc)
3101{
3102
3103 if (vc->halt_poll_ns == 0 && halt_poll_ns_grow)
3104 vc->halt_poll_ns = 10000;
3105 else
3106 vc->halt_poll_ns *= halt_poll_ns_grow;
3107}
3108
3109static void shrink_halt_poll_ns(struct kvmppc_vcore *vc)
3110{
3111 if (halt_poll_ns_shrink == 0)
3112 vc->halt_poll_ns = 0;
3113 else
3114 vc->halt_poll_ns /= halt_poll_ns_shrink;
3115}
3116
3117#ifdef CONFIG_KVM_XICS
3118static inline bool xive_interrupt_pending(struct kvm_vcpu *vcpu)
3119{
3120 if (!xive_enabled())
3121 return false;
3122 return vcpu->arch.irq_pending || vcpu->arch.xive_saved_state.pipr <
3123 vcpu->arch.xive_saved_state.cppr;
3124}
3125#else
3126static inline bool xive_interrupt_pending(struct kvm_vcpu *vcpu)
3127{
3128 return false;
3129}
3130#endif
3131
3132static bool kvmppc_vcpu_woken(struct kvm_vcpu *vcpu)
3133{
3134 if (vcpu->arch.pending_exceptions || vcpu->arch.prodded ||
3135 kvmppc_doorbell_pending(vcpu) || xive_interrupt_pending(vcpu))
3136 return true;
3137
3138 return false;
3139}
3140
3141
3142
3143
3144
3145static int kvmppc_vcore_check_block(struct kvmppc_vcore *vc)
3146{
3147 struct kvm_vcpu *vcpu;
3148 int i;
3149
3150 for_each_runnable_thread(i, vcpu, vc) {
3151 if (!vcpu->arch.ceded || kvmppc_vcpu_woken(vcpu))
3152 return 1;
3153 }
3154
3155 return 0;
3156}
3157
3158
3159
3160
3161
3162static void kvmppc_vcore_blocked(struct kvmppc_vcore *vc)
3163{
3164 ktime_t cur, start_poll, start_wait;
3165 int do_sleep = 1;
3166 u64 block_ns;
3167 DECLARE_SWAITQUEUE(wait);
3168
3169
3170 cur = start_poll = ktime_get();
3171 if (vc->halt_poll_ns) {
3172 ktime_t stop = ktime_add_ns(start_poll, vc->halt_poll_ns);
3173 ++vc->runner->stat.halt_attempted_poll;
3174
3175 vc->vcore_state = VCORE_POLLING;
3176 spin_unlock(&vc->lock);
3177
3178 do {
3179 if (kvmppc_vcore_check_block(vc)) {
3180 do_sleep = 0;
3181 break;
3182 }
3183 cur = ktime_get();
3184 } while (single_task_running() && ktime_before(cur, stop));
3185
3186 spin_lock(&vc->lock);
3187 vc->vcore_state = VCORE_INACTIVE;
3188
3189 if (!do_sleep) {
3190 ++vc->runner->stat.halt_successful_poll;
3191 goto out;
3192 }
3193 }
3194
3195 prepare_to_swait_exclusive(&vc->wq, &wait, TASK_INTERRUPTIBLE);
3196
3197 if (kvmppc_vcore_check_block(vc)) {
3198 finish_swait(&vc->wq, &wait);
3199 do_sleep = 0;
3200
3201 if (vc->halt_poll_ns)
3202 ++vc->runner->stat.halt_successful_poll;
3203 goto out;
3204 }
3205
3206 start_wait = ktime_get();
3207
3208 vc->vcore_state = VCORE_SLEEPING;
3209 trace_kvmppc_vcore_blocked(vc, 0);
3210 spin_unlock(&vc->lock);
3211 schedule();
3212 finish_swait(&vc->wq, &wait);
3213 spin_lock(&vc->lock);
3214 vc->vcore_state = VCORE_INACTIVE;
3215 trace_kvmppc_vcore_blocked(vc, 1);
3216 ++vc->runner->stat.halt_successful_wait;
3217
3218 cur = ktime_get();
3219
3220out:
3221 block_ns = ktime_to_ns(cur) - ktime_to_ns(start_poll);
3222
3223
3224 if (do_sleep) {
3225 vc->runner->stat.halt_wait_ns +=
3226 ktime_to_ns(cur) - ktime_to_ns(start_wait);
3227
3228 if (vc->halt_poll_ns)
3229 vc->runner->stat.halt_poll_fail_ns +=
3230 ktime_to_ns(start_wait) -
3231 ktime_to_ns(start_poll);
3232 } else {
3233
3234 if (vc->halt_poll_ns)
3235 vc->runner->stat.halt_poll_success_ns +=
3236 ktime_to_ns(cur) -
3237 ktime_to_ns(start_poll);
3238 }
3239
3240
3241 if (halt_poll_ns) {
3242 if (block_ns <= vc->halt_poll_ns)
3243 ;
3244
3245 else if (vc->halt_poll_ns && block_ns > halt_poll_ns)
3246 shrink_halt_poll_ns(vc);
3247
3248 else if (vc->halt_poll_ns < halt_poll_ns &&
3249 block_ns < halt_poll_ns)
3250 grow_halt_poll_ns(vc);
3251 if (vc->halt_poll_ns > halt_poll_ns)
3252 vc->halt_poll_ns = halt_poll_ns;
3253 } else
3254 vc->halt_poll_ns = 0;
3255
3256 trace_kvmppc_vcore_wakeup(do_sleep, block_ns);
3257}
3258
3259static int kvmhv_setup_mmu(struct kvm_vcpu *vcpu)
3260{
3261 int r = 0;
3262 struct kvm *kvm = vcpu->kvm;
3263
3264 mutex_lock(&kvm->lock);
3265 if (!kvm->arch.mmu_ready) {
3266 if (!kvm_is_radix(kvm))
3267 r = kvmppc_hv_setup_htab_rma(vcpu);
3268 if (!r) {
3269 if (cpu_has_feature(CPU_FTR_ARCH_300))
3270 kvmppc_setup_partition_table(kvm);
3271 kvm->arch.mmu_ready = 1;
3272 }
3273 }
3274 mutex_unlock(&kvm->lock);
3275 return r;
3276}
3277
3278static int kvmppc_run_vcpu(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu)
3279{
3280 int n_ceded, i, r;
3281 struct kvmppc_vcore *vc;
3282 struct kvm_vcpu *v;
3283
3284 trace_kvmppc_run_vcpu_enter(vcpu);
3285
3286 kvm_run->exit_reason = 0;
3287 vcpu->arch.ret = RESUME_GUEST;
3288 vcpu->arch.trap = 0;
3289 kvmppc_update_vpas(vcpu);
3290
3291
3292
3293
3294 vc = vcpu->arch.vcore;
3295 spin_lock(&vc->lock);
3296 vcpu->arch.ceded = 0;
3297 vcpu->arch.run_task = current;
3298 vcpu->arch.kvm_run = kvm_run;
3299 vcpu->arch.stolen_logged = vcore_stolen_time(vc, mftb());
3300 vcpu->arch.state = KVMPPC_VCPU_RUNNABLE;
3301 vcpu->arch.busy_preempt = TB_NIL;
3302 WRITE_ONCE(vc->runnable_threads[vcpu->arch.ptid], vcpu);
3303 ++vc->n_runnable;
3304
3305
3306
3307
3308
3309
3310 if (!signal_pending(current)) {
3311 if ((vc->vcore_state == VCORE_PIGGYBACK ||
3312 vc->vcore_state == VCORE_RUNNING) &&
3313 !VCORE_IS_EXITING(vc)) {
3314 kvmppc_create_dtl_entry(vcpu, vc);
3315 kvmppc_start_thread(vcpu, vc);
3316 trace_kvm_guest_enter(vcpu);
3317 } else if (vc->vcore_state == VCORE_SLEEPING) {
3318 swake_up_one(&vc->wq);
3319 }
3320
3321 }
3322
3323 while (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE &&
3324 !signal_pending(current)) {
3325
3326 if (!vcpu->kvm->arch.mmu_ready) {
3327 spin_unlock(&vc->lock);
3328 r = kvmhv_setup_mmu(vcpu);
3329 spin_lock(&vc->lock);
3330 if (r) {
3331 kvm_run->exit_reason = KVM_EXIT_FAIL_ENTRY;
3332 kvm_run->fail_entry.
3333 hardware_entry_failure_reason = 0;
3334 vcpu->arch.ret = r;
3335 break;
3336 }
3337 }
3338
3339 if (vc->vcore_state == VCORE_PREEMPT && vc->runner == NULL)
3340 kvmppc_vcore_end_preempt(vc);
3341
3342 if (vc->vcore_state != VCORE_INACTIVE) {
3343 kvmppc_wait_for_exec(vc, vcpu, TASK_INTERRUPTIBLE);
3344 continue;
3345 }
3346 for_each_runnable_thread(i, v, vc) {
3347 kvmppc_core_prepare_to_enter(v);
3348 if (signal_pending(v->arch.run_task)) {
3349 kvmppc_remove_runnable(vc, v);
3350 v->stat.signal_exits++;
3351 v->arch.kvm_run->exit_reason = KVM_EXIT_INTR;
3352 v->arch.ret = -EINTR;
3353 wake_up(&v->arch.cpu_run);
3354 }
3355 }
3356 if (!vc->n_runnable || vcpu->arch.state != KVMPPC_VCPU_RUNNABLE)
3357 break;
3358 n_ceded = 0;
3359 for_each_runnable_thread(i, v, vc) {
3360 if (!kvmppc_vcpu_woken(v))
3361 n_ceded += v->arch.ceded;
3362 else
3363 v->arch.ceded = 0;
3364 }
3365 vc->runner = vcpu;
3366 if (n_ceded == vc->n_runnable) {
3367 kvmppc_vcore_blocked(vc);
3368 } else if (need_resched()) {
3369 kvmppc_vcore_preempt(vc);
3370
3371 cond_resched_lock(&vc->lock);
3372 if (vc->vcore_state == VCORE_PREEMPT)
3373 kvmppc_vcore_end_preempt(vc);
3374 } else {
3375 kvmppc_run_core(vc);
3376 }
3377 vc->runner = NULL;
3378 }
3379
3380 while (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE &&
3381 (vc->vcore_state == VCORE_RUNNING ||
3382 vc->vcore_state == VCORE_EXITING ||
3383 vc->vcore_state == VCORE_PIGGYBACK))
3384 kvmppc_wait_for_exec(vc, vcpu, TASK_UNINTERRUPTIBLE);
3385
3386 if (vc->vcore_state == VCORE_PREEMPT && vc->runner == NULL)
3387 kvmppc_vcore_end_preempt(vc);
3388
3389 if (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE) {
3390 kvmppc_remove_runnable(vc, vcpu);
3391 vcpu->stat.signal_exits++;
3392 kvm_run->exit_reason = KVM_EXIT_INTR;
3393 vcpu->arch.ret = -EINTR;
3394 }
3395
3396 if (vc->n_runnable && vc->vcore_state == VCORE_INACTIVE) {
3397
3398 i = -1;
3399 v = next_runnable_thread(vc, &i);
3400 wake_up(&v->arch.cpu_run);
3401 }
3402
3403 trace_kvmppc_run_vcpu_exit(vcpu, kvm_run);
3404 spin_unlock(&vc->lock);
3405 return vcpu->arch.ret;
3406}
3407
3408static int kvmppc_vcpu_run_hv(struct kvm_run *run, struct kvm_vcpu *vcpu)
3409{
3410 int r;
3411 int srcu_idx;
3412 unsigned long ebb_regs[3] = {};
3413 unsigned long user_tar = 0;
3414 unsigned int user_vrsave;
3415 struct kvm *kvm;
3416
3417 if (!vcpu->arch.sane) {
3418 run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
3419 return -EINVAL;
3420 }
3421
3422
3423
3424
3425
3426
3427
3428#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
3429 if (cpu_has_feature(CPU_FTR_TM) && current->thread.regs &&
3430 (current->thread.regs->msr & MSR_TM)) {
3431 if (MSR_TM_ACTIVE(current->thread.regs->msr)) {
3432 run->exit_reason = KVM_EXIT_FAIL_ENTRY;
3433 run->fail_entry.hardware_entry_failure_reason = 0;
3434 return -EINVAL;
3435 }
3436
3437 mtmsr(mfmsr() | MSR_TM);
3438 current->thread.tm_tfhar = mfspr(SPRN_TFHAR);
3439 current->thread.tm_tfiar = mfspr(SPRN_TFIAR);
3440 current->thread.tm_texasr = mfspr(SPRN_TEXASR);
3441 current->thread.regs->msr &= ~MSR_TM;
3442 }
3443#endif
3444
3445
3446
3447
3448
3449 if (!vcpu->arch.online) {
3450 atomic_inc(&vcpu->arch.vcore->online_count);
3451 vcpu->arch.online = 1;
3452 }
3453
3454 kvmppc_core_prepare_to_enter(vcpu);
3455
3456
3457 if (signal_pending(current)) {
3458 run->exit_reason = KVM_EXIT_INTR;
3459 return -EINTR;
3460 }
3461
3462 kvm = vcpu->kvm;
3463 atomic_inc(&kvm->arch.vcpus_running);
3464
3465 smp_mb();
3466
3467 flush_all_to_thread(current);
3468
3469
3470 if (cpu_has_feature(CPU_FTR_ARCH_207S)) {
3471 ebb_regs[0] = mfspr(SPRN_EBBHR);
3472 ebb_regs[1] = mfspr(SPRN_EBBRR);
3473 ebb_regs[2] = mfspr(SPRN_BESCR);
3474 user_tar = mfspr(SPRN_TAR);
3475 }
3476 user_vrsave = mfspr(SPRN_VRSAVE);
3477
3478 vcpu->arch.wqp = &vcpu->arch.vcore->wq;
3479 vcpu->arch.pgdir = current->mm->pgd;
3480 vcpu->arch.state = KVMPPC_VCPU_BUSY_IN_HOST;
3481
3482 do {
3483 r = kvmppc_run_vcpu(run, vcpu);
3484
3485 if (run->exit_reason == KVM_EXIT_PAPR_HCALL &&
3486 !(vcpu->arch.shregs.msr & MSR_PR)) {
3487 trace_kvm_hcall_enter(vcpu);
3488 r = kvmppc_pseries_do_hcall(vcpu);
3489 trace_kvm_hcall_exit(vcpu, r);
3490 kvmppc_core_prepare_to_enter(vcpu);
3491 } else if (r == RESUME_PAGE_FAULT) {
3492 srcu_idx = srcu_read_lock(&kvm->srcu);
3493 r = kvmppc_book3s_hv_page_fault(run, vcpu,
3494 vcpu->arch.fault_dar, vcpu->arch.fault_dsisr);
3495 srcu_read_unlock(&kvm->srcu, srcu_idx);
3496 } else if (r == RESUME_PASSTHROUGH) {
3497 if (WARN_ON(xive_enabled()))
3498 r = H_SUCCESS;
3499 else
3500 r = kvmppc_xics_rm_complete(vcpu, 0);
3501 }
3502 } while (is_kvmppc_resume_guest(r));
3503
3504
3505 if (cpu_has_feature(CPU_FTR_ARCH_207S)) {
3506 mtspr(SPRN_EBBHR, ebb_regs[0]);
3507 mtspr(SPRN_EBBRR, ebb_regs[1]);
3508 mtspr(SPRN_BESCR, ebb_regs[2]);
3509 mtspr(SPRN_TAR, user_tar);
3510 mtspr(SPRN_FSCR, current->thread.fscr);
3511 }
3512 mtspr(SPRN_VRSAVE, user_vrsave);
3513
3514 vcpu->arch.state = KVMPPC_VCPU_NOTREADY;
3515 atomic_dec(&kvm->arch.vcpus_running);
3516 return r;
3517}
3518
3519static void kvmppc_add_seg_page_size(struct kvm_ppc_one_seg_page_size **sps,
3520 int shift, int sllp)
3521{
3522 (*sps)->page_shift = shift;
3523 (*sps)->slb_enc = sllp;
3524 (*sps)->enc[0].page_shift = shift;
3525 (*sps)->enc[0].pte_enc = kvmppc_pgsize_lp_encoding(shift, shift);
3526
3527
3528
3529 if (shift != 24) {
3530 int penc = kvmppc_pgsize_lp_encoding(shift, 24);
3531 if (penc != -1) {
3532 (*sps)->enc[1].page_shift = 24;
3533 (*sps)->enc[1].pte_enc = penc;
3534 }
3535 }
3536 (*sps)++;
3537}
3538
3539static int kvm_vm_ioctl_get_smmu_info_hv(struct kvm *kvm,
3540 struct kvm_ppc_smmu_info *info)
3541{
3542 struct kvm_ppc_one_seg_page_size *sps;
3543
3544
3545
3546
3547
3548
3549 info->data_keys = 32;
3550 info->instr_keys = cpu_has_feature(CPU_FTR_ARCH_207S) ? 32 : 0;
3551
3552
3553 info->flags = KVM_PPC_PAGE_SIZES_REAL | KVM_PPC_1T_SEGMENTS;
3554 info->slb_size = 32;
3555
3556
3557 sps = &info->sps[0];
3558 kvmppc_add_seg_page_size(&sps, 12, 0);
3559 kvmppc_add_seg_page_size(&sps, 16, SLB_VSID_L | SLB_VSID_LP_01);
3560 kvmppc_add_seg_page_size(&sps, 24, SLB_VSID_L);
3561
3562 return 0;
3563}
3564
3565
3566
3567
3568static int kvm_vm_ioctl_get_dirty_log_hv(struct kvm *kvm,
3569 struct kvm_dirty_log *log)
3570{
3571 struct kvm_memslots *slots;
3572 struct kvm_memory_slot *memslot;
3573 int i, r;
3574 unsigned long n;
3575 unsigned long *buf, *p;
3576 struct kvm_vcpu *vcpu;
3577
3578 mutex_lock(&kvm->slots_lock);
3579
3580 r = -EINVAL;
3581 if (log->slot >= KVM_USER_MEM_SLOTS)
3582 goto out;
3583
3584 slots = kvm_memslots(kvm);
3585 memslot = id_to_memslot(slots, log->slot);
3586 r = -ENOENT;
3587 if (!memslot->dirty_bitmap)
3588 goto out;
3589
3590
3591
3592
3593
3594 n = kvm_dirty_bitmap_bytes(memslot);
3595 buf = memslot->dirty_bitmap + n / sizeof(long);
3596 memset(buf, 0, n);
3597
3598 if (kvm_is_radix(kvm))
3599 r = kvmppc_hv_get_dirty_log_radix(kvm, memslot, buf);
3600 else
3601 r = kvmppc_hv_get_dirty_log_hpt(kvm, memslot, buf);
3602 if (r)
3603 goto out;
3604
3605
3606
3607
3608
3609
3610
3611 p = memslot->dirty_bitmap;
3612 for (i = 0; i < n / sizeof(long); ++i)
3613 buf[i] |= xchg(&p[i], 0);
3614
3615
3616
3617 kvm_for_each_vcpu(i, vcpu, kvm) {
3618 spin_lock(&vcpu->arch.vpa_update_lock);
3619 kvmppc_harvest_vpa_dirty(&vcpu->arch.vpa, memslot, buf);
3620 kvmppc_harvest_vpa_dirty(&vcpu->arch.dtl, memslot, buf);
3621 spin_unlock(&vcpu->arch.vpa_update_lock);
3622 }
3623
3624 r = -EFAULT;
3625 if (copy_to_user(log->dirty_bitmap, buf, n))
3626 goto out;
3627
3628 r = 0;
3629out:
3630 mutex_unlock(&kvm->slots_lock);
3631 return r;
3632}
3633
3634static void kvmppc_core_free_memslot_hv(struct kvm_memory_slot *free,
3635 struct kvm_memory_slot *dont)
3636{
3637 if (!dont || free->arch.rmap != dont->arch.rmap) {
3638 vfree(free->arch.rmap);
3639 free->arch.rmap = NULL;
3640 }
3641}
3642
3643static int kvmppc_core_create_memslot_hv(struct kvm_memory_slot *slot,
3644 unsigned long npages)
3645{
3646 slot->arch.rmap = vzalloc(array_size(npages, sizeof(*slot->arch.rmap)));
3647 if (!slot->arch.rmap)
3648 return -ENOMEM;
3649
3650 return 0;
3651}
3652
3653static int kvmppc_core_prepare_memory_region_hv(struct kvm *kvm,
3654 struct kvm_memory_slot *memslot,
3655 const struct kvm_userspace_memory_region *mem)
3656{
3657 return 0;
3658}
3659
3660static void kvmppc_core_commit_memory_region_hv(struct kvm *kvm,
3661 const struct kvm_userspace_memory_region *mem,
3662 const struct kvm_memory_slot *old,
3663 const struct kvm_memory_slot *new)
3664{
3665 unsigned long npages = mem->memory_size >> PAGE_SHIFT;
3666
3667
3668
3669
3670
3671
3672
3673 if (npages)
3674 atomic64_inc(&kvm->arch.mmio_update);
3675}
3676
3677
3678
3679
3680
3681void kvmppc_update_lpcr(struct kvm *kvm, unsigned long lpcr, unsigned long mask)
3682{
3683 long int i;
3684 u32 cores_done = 0;
3685
3686 if ((kvm->arch.lpcr & mask) == lpcr)
3687 return;
3688
3689 kvm->arch.lpcr = (kvm->arch.lpcr & ~mask) | lpcr;
3690
3691 for (i = 0; i < KVM_MAX_VCORES; ++i) {
3692 struct kvmppc_vcore *vc = kvm->arch.vcores[i];
3693 if (!vc)
3694 continue;
3695 spin_lock(&vc->lock);
3696 vc->lpcr = (vc->lpcr & ~mask) | lpcr;
3697 spin_unlock(&vc->lock);
3698 if (++cores_done >= kvm->arch.online_vcores)
3699 break;
3700 }
3701}
3702
3703static void kvmppc_mmu_destroy_hv(struct kvm_vcpu *vcpu)
3704{
3705 return;
3706}
3707
3708void kvmppc_setup_partition_table(struct kvm *kvm)
3709{
3710 unsigned long dw0, dw1;
3711
3712 if (!kvm_is_radix(kvm)) {
3713
3714 dw0 = ((kvm->arch.vrma_slb_v & SLB_VSID_L) >> 1) |
3715 ((kvm->arch.vrma_slb_v & SLB_VSID_LP) << 1);
3716
3717 dw0 |= kvm->arch.sdr1;
3718
3719
3720 dw1 = kvm->arch.process_table;
3721 } else {
3722 dw0 = PATB_HR | radix__get_tree_size() |
3723 __pa(kvm->arch.pgtable) | RADIX_PGD_INDEX_SIZE;
3724 dw1 = PATB_GR | kvm->arch.process_table;
3725 }
3726
3727 mmu_partition_table_set_entry(kvm->arch.lpid, dw0, dw1);
3728}
3729
3730
3731
3732
3733
3734static int kvmppc_hv_setup_htab_rma(struct kvm_vcpu *vcpu)
3735{
3736 int err = 0;
3737 struct kvm *kvm = vcpu->kvm;
3738 unsigned long hva;
3739 struct kvm_memory_slot *memslot;
3740 struct vm_area_struct *vma;
3741 unsigned long lpcr = 0, senc;
3742 unsigned long psize, porder;
3743 int srcu_idx;
3744
3745
3746 if (!kvm->arch.hpt.virt) {
3747 int order = KVM_DEFAULT_HPT_ORDER;
3748 struct kvm_hpt_info info;
3749
3750 err = kvmppc_allocate_hpt(&info, order);
3751
3752
3753
3754 while ((err == -ENOMEM) && --order >= PPC_MIN_HPT_ORDER)
3755 err = kvmppc_allocate_hpt(&info, order);
3756
3757 if (err < 0) {
3758 pr_err("KVM: Couldn't alloc HPT\n");
3759 goto out;
3760 }
3761
3762 kvmppc_set_hpt(kvm, &info);
3763 }
3764
3765
3766 srcu_idx = srcu_read_lock(&kvm->srcu);
3767 memslot = gfn_to_memslot(kvm, 0);
3768
3769
3770 err = -EINVAL;
3771 if (!memslot || (memslot->flags & KVM_MEMSLOT_INVALID))
3772 goto out_srcu;
3773
3774
3775 hva = memslot->userspace_addr;
3776 down_read(¤t->mm->mmap_sem);
3777 vma = find_vma(current->mm, hva);
3778 if (!vma || vma->vm_start > hva || (vma->vm_flags & VM_IO))
3779 goto up_out;
3780
3781 psize = vma_kernel_pagesize(vma);
3782
3783 up_read(¤t->mm->mmap_sem);
3784
3785
3786 if (psize >= 0x1000000)
3787 psize = 0x1000000;
3788 else if (psize >= 0x10000)
3789 psize = 0x10000;
3790 else
3791 psize = 0x1000;
3792 porder = __ilog2(psize);
3793
3794 senc = slb_pgsize_encoding(psize);
3795 kvm->arch.vrma_slb_v = senc | SLB_VSID_B_1T |
3796 (VRMA_VSID << SLB_VSID_SHIFT_1T);
3797
3798 kvmppc_map_vrma(vcpu, memslot, porder);
3799
3800
3801 if (!cpu_has_feature(CPU_FTR_ARCH_300)) {
3802
3803 lpcr = senc << (LPCR_VRMASD_SH - 4);
3804 kvmppc_update_lpcr(kvm, lpcr, LPCR_VRMASD);
3805 }
3806
3807
3808 smp_wmb();
3809 err = 0;
3810 out_srcu:
3811 srcu_read_unlock(&kvm->srcu, srcu_idx);
3812 out:
3813 return err;
3814
3815 up_out:
3816 up_read(¤t->mm->mmap_sem);
3817 goto out_srcu;
3818}
3819
3820
3821int kvmppc_switch_mmu_to_hpt(struct kvm *kvm)
3822{
3823 kvmppc_free_radix(kvm);
3824 kvmppc_update_lpcr(kvm, LPCR_VPM1,
3825 LPCR_VPM1 | LPCR_UPRT | LPCR_GTSE | LPCR_HR);
3826 kvmppc_rmap_reset(kvm);
3827 kvm->arch.radix = 0;
3828 kvm->arch.process_table = 0;
3829 return 0;
3830}
3831
3832
3833int kvmppc_switch_mmu_to_radix(struct kvm *kvm)
3834{
3835 int err;
3836
3837 err = kvmppc_init_vm_radix(kvm);
3838 if (err)
3839 return err;
3840
3841 kvmppc_free_hpt(&kvm->arch.hpt);
3842 kvmppc_update_lpcr(kvm, LPCR_UPRT | LPCR_GTSE | LPCR_HR,
3843 LPCR_VPM1 | LPCR_UPRT | LPCR_GTSE | LPCR_HR);
3844 kvm->arch.radix = 1;
3845 return 0;
3846}
3847
3848#ifdef CONFIG_KVM_XICS
3849
3850
3851
3852
3853
3854
3855
3856
3857
3858
3859void kvmppc_alloc_host_rm_ops(void)
3860{
3861 struct kvmppc_host_rm_ops *ops;
3862 unsigned long l_ops;
3863 int cpu, core;
3864 int size;
3865
3866
3867 if (kvmppc_host_rm_ops_hv != NULL)
3868 return;
3869
3870 ops = kzalloc(sizeof(struct kvmppc_host_rm_ops), GFP_KERNEL);
3871 if (!ops)
3872 return;
3873
3874 size = cpu_nr_cores() * sizeof(struct kvmppc_host_rm_core);
3875 ops->rm_core = kzalloc(size, GFP_KERNEL);
3876
3877 if (!ops->rm_core) {
3878 kfree(ops);
3879 return;
3880 }
3881
3882 cpus_read_lock();
3883
3884 for (cpu = 0; cpu < nr_cpu_ids; cpu += threads_per_core) {
3885 if (!cpu_online(cpu))
3886 continue;
3887
3888 core = cpu >> threads_shift;
3889 ops->rm_core[core].rm_state.in_host = 1;
3890 }
3891
3892 ops->vcpu_kick = kvmppc_fast_vcpu_kick_hv;
3893
3894
3895
3896
3897
3898
3899
3900 smp_wmb();
3901 l_ops = (unsigned long) ops;
3902
3903 if (cmpxchg64((unsigned long *)&kvmppc_host_rm_ops_hv, 0, l_ops)) {
3904 cpus_read_unlock();
3905 kfree(ops->rm_core);
3906 kfree(ops);
3907 return;
3908 }
3909
3910 cpuhp_setup_state_nocalls_cpuslocked(CPUHP_KVM_PPC_BOOK3S_PREPARE,
3911 "ppc/kvm_book3s:prepare",
3912 kvmppc_set_host_core,
3913 kvmppc_clear_host_core);
3914 cpus_read_unlock();
3915}
3916
3917void kvmppc_free_host_rm_ops(void)
3918{
3919 if (kvmppc_host_rm_ops_hv) {
3920 cpuhp_remove_state_nocalls(CPUHP_KVM_PPC_BOOK3S_PREPARE);
3921 kfree(kvmppc_host_rm_ops_hv->rm_core);
3922 kfree(kvmppc_host_rm_ops_hv);
3923 kvmppc_host_rm_ops_hv = NULL;
3924 }
3925}
3926#endif
3927
3928static int kvmppc_core_init_vm_hv(struct kvm *kvm)
3929{
3930 unsigned long lpcr, lpid;
3931 char buf[32];
3932 int ret;
3933
3934
3935
3936 lpid = kvmppc_alloc_lpid();
3937 if ((long)lpid < 0)
3938 return -ENOMEM;
3939 kvm->arch.lpid = lpid;
3940
3941 kvmppc_alloc_host_rm_ops();
3942
3943
3944
3945
3946
3947
3948
3949
3950 if (!cpu_has_feature(CPU_FTR_ARCH_300))
3951 cpumask_setall(&kvm->arch.need_tlb_flush);
3952
3953
3954 memcpy(kvm->arch.enabled_hcalls, default_enabled_hcalls,
3955 sizeof(kvm->arch.enabled_hcalls));
3956
3957 if (!cpu_has_feature(CPU_FTR_ARCH_300))
3958 kvm->arch.host_sdr1 = mfspr(SPRN_SDR1);
3959
3960
3961 kvm->arch.host_lpid = mfspr(SPRN_LPID);
3962 kvm->arch.host_lpcr = lpcr = mfspr(SPRN_LPCR);
3963 lpcr &= LPCR_PECE | LPCR_LPES;
3964 lpcr |= (4UL << LPCR_DPFD_SH) | LPCR_HDICE |
3965 LPCR_VPM0 | LPCR_VPM1;
3966 kvm->arch.vrma_slb_v = SLB_VSID_B_1T |
3967 (VRMA_VSID << SLB_VSID_SHIFT_1T);
3968
3969 if (cpu_has_feature(CPU_FTR_ARCH_207S))
3970 lpcr |= LPCR_ONL;
3971
3972
3973
3974
3975
3976
3977
3978 if (cpu_has_feature(CPU_FTR_ARCH_300)) {
3979 lpcr &= ~LPCR_VPM0;
3980 lpcr |= LPCR_HVICE | LPCR_HEIC;
3981
3982
3983
3984
3985
3986 if (xive_enabled())
3987 lpcr |= LPCR_LPES;
3988 }
3989
3990
3991
3992
3993 if (radix_enabled()) {
3994 kvm->arch.radix = 1;
3995 kvm->arch.mmu_ready = 1;
3996 lpcr &= ~LPCR_VPM1;
3997 lpcr |= LPCR_UPRT | LPCR_GTSE | LPCR_HR;
3998 ret = kvmppc_init_vm_radix(kvm);
3999 if (ret) {
4000 kvmppc_free_lpid(kvm->arch.lpid);
4001 return ret;
4002 }
4003 kvmppc_setup_partition_table(kvm);
4004 }
4005
4006 kvm->arch.lpcr = lpcr;
4007
4008
4009 kvm->arch.resize_hpt = NULL;
4010
4011
4012
4013
4014
4015 if (radix_enabled())
4016 kvm->arch.tlb_sets = POWER9_TLB_SETS_RADIX;
4017 else if (cpu_has_feature(CPU_FTR_ARCH_300))
4018 kvm->arch.tlb_sets = POWER9_TLB_SETS_HASH;
4019 else if (cpu_has_feature(CPU_FTR_ARCH_207S))
4020 kvm->arch.tlb_sets = POWER8_TLB_SETS;
4021 else
4022 kvm->arch.tlb_sets = POWER7_TLB_SETS;
4023
4024
4025
4026
4027
4028
4029
4030 if (cpu_has_feature(CPU_FTR_ARCH_300))
4031 kvm->arch.threads_indep = indep_threads_mode;
4032 if (!kvm->arch.threads_indep)
4033 kvm_hv_vm_activated();
4034
4035
4036
4037
4038
4039
4040
4041
4042 if (!cpu_has_feature(CPU_FTR_ARCH_300))
4043 kvm->arch.smt_mode = threads_per_subcore;
4044 else
4045 kvm->arch.smt_mode = 1;
4046 kvm->arch.emul_smt_mode = 1;
4047
4048
4049
4050
4051 snprintf(buf, sizeof(buf), "vm%d", current->pid);
4052 kvm->arch.debugfs_dir = debugfs_create_dir(buf, kvm_debugfs_dir);
4053 kvmppc_mmu_debugfs_init(kvm);
4054
4055 return 0;
4056}
4057
4058static void kvmppc_free_vcores(struct kvm *kvm)
4059{
4060 long int i;
4061
4062 for (i = 0; i < KVM_MAX_VCORES; ++i)
4063 kfree(kvm->arch.vcores[i]);
4064 kvm->arch.online_vcores = 0;
4065}
4066
4067static void kvmppc_core_destroy_vm_hv(struct kvm *kvm)
4068{
4069 debugfs_remove_recursive(kvm->arch.debugfs_dir);
4070
4071 if (!kvm->arch.threads_indep)
4072 kvm_hv_vm_deactivated();
4073
4074 kvmppc_free_vcores(kvm);
4075
4076 kvmppc_free_lpid(kvm->arch.lpid);
4077
4078 if (kvm_is_radix(kvm))
4079 kvmppc_free_radix(kvm);
4080 else
4081 kvmppc_free_hpt(&kvm->arch.hpt);
4082
4083 kvmppc_free_pimap(kvm);
4084}
4085
4086
4087static int kvmppc_core_emulate_op_hv(struct kvm_run *run, struct kvm_vcpu *vcpu,
4088 unsigned int inst, int *advance)
4089{
4090 return EMULATE_FAIL;
4091}
4092
4093static int kvmppc_core_emulate_mtspr_hv(struct kvm_vcpu *vcpu, int sprn,
4094 ulong spr_val)
4095{
4096 return EMULATE_FAIL;
4097}
4098
4099static int kvmppc_core_emulate_mfspr_hv(struct kvm_vcpu *vcpu, int sprn,
4100 ulong *spr_val)
4101{
4102 return EMULATE_FAIL;
4103}
4104
4105static int kvmppc_core_check_processor_compat_hv(void)
4106{
4107 if (!cpu_has_feature(CPU_FTR_HVMODE) ||
4108 !cpu_has_feature(CPU_FTR_ARCH_206))
4109 return -EIO;
4110
4111 return 0;
4112}
4113
4114#ifdef CONFIG_KVM_XICS
4115
4116void kvmppc_free_pimap(struct kvm *kvm)
4117{
4118 kfree(kvm->arch.pimap);
4119}
4120
4121static struct kvmppc_passthru_irqmap *kvmppc_alloc_pimap(void)
4122{
4123 return kzalloc(sizeof(struct kvmppc_passthru_irqmap), GFP_KERNEL);
4124}
4125
4126static int kvmppc_set_passthru_irq(struct kvm *kvm, int host_irq, int guest_gsi)
4127{
4128 struct irq_desc *desc;
4129 struct kvmppc_irq_map *irq_map;
4130 struct kvmppc_passthru_irqmap *pimap;
4131 struct irq_chip *chip;
4132 int i, rc = 0;
4133
4134 if (!kvm_irq_bypass)
4135 return 1;
4136
4137 desc = irq_to_desc(host_irq);
4138 if (!desc)
4139 return -EIO;
4140
4141 mutex_lock(&kvm->lock);
4142
4143 pimap = kvm->arch.pimap;
4144 if (pimap == NULL) {
4145
4146 pimap = kvmppc_alloc_pimap();
4147 if (pimap == NULL) {
4148 mutex_unlock(&kvm->lock);
4149 return -ENOMEM;
4150 }
4151 kvm->arch.pimap = pimap;
4152 }
4153
4154
4155
4156
4157
4158
4159 chip = irq_data_get_irq_chip(&desc->irq_data);
4160 if (!chip || !(is_pnv_opal_msi(chip) || is_xive_irq(chip))) {
4161 pr_warn("kvmppc_set_passthru_irq_hv: Could not assign IRQ map for (%d,%d)\n",
4162 host_irq, guest_gsi);
4163 mutex_unlock(&kvm->lock);
4164 return -ENOENT;
4165 }
4166
4167
4168
4169
4170
4171
4172 for (i = 0; i < pimap->n_mapped; i++) {
4173 if (guest_gsi == pimap->mapped[i].v_hwirq) {
4174 if (pimap->mapped[i].r_hwirq) {
4175 mutex_unlock(&kvm->lock);
4176 return -EINVAL;
4177 }
4178 break;
4179 }
4180 }
4181
4182 if (i == KVMPPC_PIRQ_MAPPED) {
4183 mutex_unlock(&kvm->lock);
4184 return -EAGAIN;
4185 }
4186
4187 irq_map = &pimap->mapped[i];
4188
4189 irq_map->v_hwirq = guest_gsi;
4190 irq_map->desc = desc;
4191
4192
4193
4194
4195
4196 smp_wmb();
4197 irq_map->r_hwirq = desc->irq_data.hwirq;
4198
4199 if (i == pimap->n_mapped)
4200 pimap->n_mapped++;
4201
4202 if (xive_enabled())
4203 rc = kvmppc_xive_set_mapped(kvm, guest_gsi, desc);
4204 else
4205 kvmppc_xics_set_mapped(kvm, guest_gsi, desc->irq_data.hwirq);
4206 if (rc)
4207 irq_map->r_hwirq = 0;
4208
4209 mutex_unlock(&kvm->lock);
4210
4211 return 0;
4212}
4213
4214static int kvmppc_clr_passthru_irq(struct kvm *kvm, int host_irq, int guest_gsi)
4215{
4216 struct irq_desc *desc;
4217 struct kvmppc_passthru_irqmap *pimap;
4218 int i, rc = 0;
4219
4220 if (!kvm_irq_bypass)
4221 return 0;
4222
4223 desc = irq_to_desc(host_irq);
4224 if (!desc)
4225 return -EIO;
4226
4227 mutex_lock(&kvm->lock);
4228 if (!kvm->arch.pimap)
4229 goto unlock;
4230
4231 pimap = kvm->arch.pimap;
4232
4233 for (i = 0; i < pimap->n_mapped; i++) {
4234 if (guest_gsi == pimap->mapped[i].v_hwirq)
4235 break;
4236 }
4237
4238 if (i == pimap->n_mapped) {
4239 mutex_unlock(&kvm->lock);
4240 return -ENODEV;
4241 }
4242
4243 if (xive_enabled())
4244 rc = kvmppc_xive_clr_mapped(kvm, guest_gsi, pimap->mapped[i].desc);
4245 else
4246 kvmppc_xics_clr_mapped(kvm, guest_gsi, pimap->mapped[i].r_hwirq);
4247
4248
4249 pimap->mapped[i].r_hwirq = 0;
4250
4251
4252
4253
4254
4255 unlock:
4256 mutex_unlock(&kvm->lock);
4257 return rc;
4258}
4259
4260static int kvmppc_irq_bypass_add_producer_hv(struct irq_bypass_consumer *cons,
4261 struct irq_bypass_producer *prod)
4262{
4263 int ret = 0;
4264 struct kvm_kernel_irqfd *irqfd =
4265 container_of(cons, struct kvm_kernel_irqfd, consumer);
4266
4267 irqfd->producer = prod;
4268
4269 ret = kvmppc_set_passthru_irq(irqfd->kvm, prod->irq, irqfd->gsi);
4270 if (ret)
4271 pr_info("kvmppc_set_passthru_irq (irq %d, gsi %d) fails: %d\n",
4272 prod->irq, irqfd->gsi, ret);
4273
4274 return ret;
4275}
4276
4277static void kvmppc_irq_bypass_del_producer_hv(struct irq_bypass_consumer *cons,
4278 struct irq_bypass_producer *prod)
4279{
4280 int ret;
4281 struct kvm_kernel_irqfd *irqfd =
4282 container_of(cons, struct kvm_kernel_irqfd, consumer);
4283
4284 irqfd->producer = NULL;
4285
4286
4287
4288
4289
4290
4291 ret = kvmppc_clr_passthru_irq(irqfd->kvm, prod->irq, irqfd->gsi);
4292 if (ret)
4293 pr_warn("kvmppc_clr_passthru_irq (irq %d, gsi %d) fails: %d\n",
4294 prod->irq, irqfd->gsi, ret);
4295}
4296#endif
4297
4298static long kvm_arch_vm_ioctl_hv(struct file *filp,
4299 unsigned int ioctl, unsigned long arg)
4300{
4301 struct kvm *kvm __maybe_unused = filp->private_data;
4302 void __user *argp = (void __user *)arg;
4303 long r;
4304
4305 switch (ioctl) {
4306
4307 case KVM_PPC_ALLOCATE_HTAB: {
4308 u32 htab_order;
4309
4310 r = -EFAULT;
4311 if (get_user(htab_order, (u32 __user *)argp))
4312 break;
4313 r = kvmppc_alloc_reset_hpt(kvm, htab_order);
4314 if (r)
4315 break;
4316 r = 0;
4317 break;
4318 }
4319
4320 case KVM_PPC_GET_HTAB_FD: {
4321 struct kvm_get_htab_fd ghf;
4322
4323 r = -EFAULT;
4324 if (copy_from_user(&ghf, argp, sizeof(ghf)))
4325 break;
4326 r = kvm_vm_ioctl_get_htab_fd(kvm, &ghf);
4327 break;
4328 }
4329
4330 case KVM_PPC_RESIZE_HPT_PREPARE: {
4331 struct kvm_ppc_resize_hpt rhpt;
4332
4333 r = -EFAULT;
4334 if (copy_from_user(&rhpt, argp, sizeof(rhpt)))
4335 break;
4336
4337 r = kvm_vm_ioctl_resize_hpt_prepare(kvm, &rhpt);
4338 break;
4339 }
4340
4341 case KVM_PPC_RESIZE_HPT_COMMIT: {
4342 struct kvm_ppc_resize_hpt rhpt;
4343
4344 r = -EFAULT;
4345 if (copy_from_user(&rhpt, argp, sizeof(rhpt)))
4346 break;
4347
4348 r = kvm_vm_ioctl_resize_hpt_commit(kvm, &rhpt);
4349 break;
4350 }
4351
4352 default:
4353 r = -ENOTTY;
4354 }
4355
4356 return r;
4357}
4358
4359
4360
4361
4362
4363
4364
4365static unsigned int default_hcall_list[] = {
4366 H_REMOVE,
4367 H_ENTER,
4368 H_READ,
4369 H_PROTECT,
4370 H_BULK_REMOVE,
4371 H_GET_TCE,
4372 H_PUT_TCE,
4373 H_SET_DABR,
4374 H_SET_XDABR,
4375 H_CEDE,
4376 H_PROD,
4377 H_CONFER,
4378 H_REGISTER_VPA,
4379#ifdef CONFIG_KVM_XICS
4380 H_EOI,
4381 H_CPPR,
4382 H_IPI,
4383 H_IPOLL,
4384 H_XIRR,
4385 H_XIRR_X,
4386#endif
4387 0
4388};
4389
4390static void init_default_hcalls(void)
4391{
4392 int i;
4393 unsigned int hcall;
4394
4395 for (i = 0; default_hcall_list[i]; ++i) {
4396 hcall = default_hcall_list[i];
4397 WARN_ON(!kvmppc_hcall_impl_hv(hcall));
4398 __set_bit(hcall / 4, default_enabled_hcalls);
4399 }
4400}
4401
4402static int kvmhv_configure_mmu(struct kvm *kvm, struct kvm_ppc_mmuv3_cfg *cfg)
4403{
4404 unsigned long lpcr;
4405 int radix;
4406 int err;
4407
4408
4409 if (!cpu_has_feature(CPU_FTR_ARCH_300))
4410 return -ENODEV;
4411
4412
4413 if (cfg->flags & ~(KVM_PPC_MMUV3_RADIX | KVM_PPC_MMUV3_GTSE))
4414 return -EINVAL;
4415
4416
4417 radix = !!(cfg->flags & KVM_PPC_MMUV3_RADIX);
4418 if (!!(cfg->process_table & PATB_GR) != radix)
4419 return -EINVAL;
4420
4421
4422 if ((cfg->process_table & PRTS_MASK) > 24)
4423 return -EINVAL;
4424
4425
4426 if (radix && !radix_enabled())
4427 return -EINVAL;
4428
4429 mutex_lock(&kvm->lock);
4430 if (radix != kvm_is_radix(kvm)) {
4431 if (kvm->arch.mmu_ready) {
4432 kvm->arch.mmu_ready = 0;
4433
4434 smp_mb();
4435 if (atomic_read(&kvm->arch.vcpus_running)) {
4436 kvm->arch.mmu_ready = 1;
4437 err = -EBUSY;
4438 goto out_unlock;
4439 }
4440 }
4441 if (radix)
4442 err = kvmppc_switch_mmu_to_radix(kvm);
4443 else
4444 err = kvmppc_switch_mmu_to_hpt(kvm);
4445 if (err)
4446 goto out_unlock;
4447 }
4448
4449 kvm->arch.process_table = cfg->process_table;
4450 kvmppc_setup_partition_table(kvm);
4451
4452 lpcr = (cfg->flags & KVM_PPC_MMUV3_GTSE) ? LPCR_GTSE : 0;
4453 kvmppc_update_lpcr(kvm, lpcr, LPCR_GTSE);
4454 err = 0;
4455
4456 out_unlock:
4457 mutex_unlock(&kvm->lock);
4458 return err;
4459}
4460
4461static struct kvmppc_ops kvm_ops_hv = {
4462 .get_sregs = kvm_arch_vcpu_ioctl_get_sregs_hv,
4463 .set_sregs = kvm_arch_vcpu_ioctl_set_sregs_hv,
4464 .get_one_reg = kvmppc_get_one_reg_hv,
4465 .set_one_reg = kvmppc_set_one_reg_hv,
4466 .vcpu_load = kvmppc_core_vcpu_load_hv,
4467 .vcpu_put = kvmppc_core_vcpu_put_hv,
4468 .set_msr = kvmppc_set_msr_hv,
4469 .vcpu_run = kvmppc_vcpu_run_hv,
4470 .vcpu_create = kvmppc_core_vcpu_create_hv,
4471 .vcpu_free = kvmppc_core_vcpu_free_hv,
4472 .check_requests = kvmppc_core_check_requests_hv,
4473 .get_dirty_log = kvm_vm_ioctl_get_dirty_log_hv,
4474 .flush_memslot = kvmppc_core_flush_memslot_hv,
4475 .prepare_memory_region = kvmppc_core_prepare_memory_region_hv,
4476 .commit_memory_region = kvmppc_core_commit_memory_region_hv,
4477 .unmap_hva_range = kvm_unmap_hva_range_hv,
4478 .age_hva = kvm_age_hva_hv,
4479 .test_age_hva = kvm_test_age_hva_hv,
4480 .set_spte_hva = kvm_set_spte_hva_hv,
4481 .mmu_destroy = kvmppc_mmu_destroy_hv,
4482 .free_memslot = kvmppc_core_free_memslot_hv,
4483 .create_memslot = kvmppc_core_create_memslot_hv,
4484 .init_vm = kvmppc_core_init_vm_hv,
4485 .destroy_vm = kvmppc_core_destroy_vm_hv,
4486 .get_smmu_info = kvm_vm_ioctl_get_smmu_info_hv,
4487 .emulate_op = kvmppc_core_emulate_op_hv,
4488 .emulate_mtspr = kvmppc_core_emulate_mtspr_hv,
4489 .emulate_mfspr = kvmppc_core_emulate_mfspr_hv,
4490 .fast_vcpu_kick = kvmppc_fast_vcpu_kick_hv,
4491 .arch_vm_ioctl = kvm_arch_vm_ioctl_hv,
4492 .hcall_implemented = kvmppc_hcall_impl_hv,
4493#ifdef CONFIG_KVM_XICS
4494 .irq_bypass_add_producer = kvmppc_irq_bypass_add_producer_hv,
4495 .irq_bypass_del_producer = kvmppc_irq_bypass_del_producer_hv,
4496#endif
4497 .configure_mmu = kvmhv_configure_mmu,
4498 .get_rmmu_info = kvmhv_get_rmmu_info,
4499 .set_smt_mode = kvmhv_set_smt_mode,
4500};
4501
4502static int kvm_init_subcore_bitmap(void)
4503{
4504 int i, j;
4505 int nr_cores = cpu_nr_cores();
4506 struct sibling_subcore_state *sibling_subcore_state;
4507
4508 for (i = 0; i < nr_cores; i++) {
4509 int first_cpu = i * threads_per_core;
4510 int node = cpu_to_node(first_cpu);
4511
4512
4513 if (paca_ptrs[first_cpu]->sibling_subcore_state)
4514 continue;
4515
4516 sibling_subcore_state =
4517 kmalloc_node(sizeof(struct sibling_subcore_state),
4518 GFP_KERNEL, node);
4519 if (!sibling_subcore_state)
4520 return -ENOMEM;
4521
4522 memset(sibling_subcore_state, 0,
4523 sizeof(struct sibling_subcore_state));
4524
4525 for (j = 0; j < threads_per_core; j++) {
4526 int cpu = first_cpu + j;
4527
4528 paca_ptrs[cpu]->sibling_subcore_state =
4529 sibling_subcore_state;
4530 }
4531 }
4532 return 0;
4533}
4534
4535static int kvmppc_radix_possible(void)
4536{
4537 return cpu_has_feature(CPU_FTR_ARCH_300) && radix_enabled();
4538}
4539
4540static int kvmppc_book3s_init_hv(void)
4541{
4542 int r;
4543
4544
4545
4546 r = kvmppc_core_check_processor_compat_hv();
4547 if (r < 0)
4548 return -ENODEV;
4549
4550 r = kvm_init_subcore_bitmap();
4551 if (r)
4552 return r;
4553
4554
4555
4556
4557
4558
4559#ifdef CONFIG_SMP
4560 if (!xive_enabled() && !local_paca->kvm_hstate.xics_phys) {
4561 struct device_node *np;
4562
4563 np = of_find_compatible_node(NULL, NULL, "ibm,opal-intc");
4564 if (!np) {
4565 pr_err("KVM-HV: Cannot determine method for accessing XICS\n");
4566 return -ENODEV;
4567 }
4568
4569 of_node_put(np);
4570 }
4571#endif
4572
4573 kvm_ops_hv.owner = THIS_MODULE;
4574 kvmppc_hv_ops = &kvm_ops_hv;
4575
4576 init_default_hcalls();
4577
4578 init_vcore_lists();
4579
4580 r = kvmppc_mmu_hv_init();
4581 if (r)
4582 return r;
4583
4584 if (kvmppc_radix_possible())
4585 r = kvmppc_radix_init();
4586
4587
4588
4589
4590
4591 if (cpu_has_feature(CPU_FTR_ARCH_300)) {
4592 unsigned int pvr = mfspr(SPRN_PVR);
4593 if ((pvr >> 16) == PVR_POWER9 &&
4594 (((pvr & 0xe000) == 0 && (pvr & 0xfff) < 0x202) ||
4595 ((pvr & 0xe000) == 0x2000 && (pvr & 0xfff) < 0x101)))
4596 no_mixing_hpt_and_radix = true;
4597 }
4598
4599 return r;
4600}
4601
4602static void kvmppc_book3s_exit_hv(void)
4603{
4604 kvmppc_free_host_rm_ops();
4605 if (kvmppc_radix_possible())
4606 kvmppc_radix_exit();
4607 kvmppc_hv_ops = NULL;
4608}
4609
4610module_init(kvmppc_book3s_init_hv);
4611module_exit(kvmppc_book3s_exit_hv);
4612MODULE_LICENSE("GPL");
4613MODULE_ALIAS_MISCDEV(KVM_MINOR);
4614MODULE_ALIAS("devname:kvm");
4615