1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20#include <asm/ppc_asm.h>
21#include <asm/kvm_asm.h>
22#include <asm/reg.h>
23#include <asm/mmu.h>
24#include <asm/page.h>
25#include <asm/ptrace.h>
26#include <asm/hvcall.h>
27#include <asm/asm-offsets.h>
28#include <asm/exception-64s.h>
29#include <asm/kvm_book3s_asm.h>
30#include <asm/book3s/64/mmu-hash.h>
31#include <asm/tm.h>
32#include <asm/opal.h>
33#include <asm/xive-regs.h>
34#include <asm/thread_info.h>
35
36
37#define EXTEND_HDEC(reg) \
38BEGIN_FTR_SECTION; \
39 extsw reg, reg; \
40END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300)
41
42#define VCPU_GPRS_TM(reg) (((reg) * ULONG_SIZE) + VCPU_GPR_TM)
43
44
45#define NAPPING_CEDE 1
46#define NAPPING_NOVCPU 2
47
48
49#define SFS 160
50#define STACK_SLOT_TRAP (SFS-4)
51#define STACK_SLOT_TID (SFS-16)
52#define STACK_SLOT_PSSCR (SFS-24)
53#define STACK_SLOT_PID (SFS-32)
54#define STACK_SLOT_IAMR (SFS-40)
55#define STACK_SLOT_CIABR (SFS-48)
56#define STACK_SLOT_DAWR (SFS-56)
57#define STACK_SLOT_DAWRX (SFS-64)
58#define STACK_SLOT_HFSCR (SFS-72)
59
60
61
62
63
64
65
66
67
68_GLOBAL_TOC(kvmppc_hv_entry_trampoline)
69 mflr r0
70 std r0, PPC_LR_STKOFF(r1)
71 stdu r1, -112(r1)
72 mfmsr r10
73 std r10, HSTATE_HOST_MSR(r13)
74 LOAD_REG_ADDR(r5, kvmppc_call_hv_entry)
75 li r0,MSR_RI
76 andc r0,r10,r0
77 li r6,MSR_IR | MSR_DR
78 andc r6,r10,r6
79 mtmsrd r0,1
80 mtsrr0 r5
81 mtsrr1 r6
82 RFI_TO_KERNEL
83
84kvmppc_call_hv_entry:
85BEGIN_FTR_SECTION
86
87 ld r3, HSTATE_SPLIT_MODE(r13)
88 cmpdi r3, 0
89 beq 46f
90 lwz r4, KVM_SPLIT_DO_SET(r3)
91 cmpwi r4, 0
92 beq 46f
93 bl kvmhv_p9_set_lpcr
94 nop
9546:
96END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
97
98 ld r4, HSTATE_KVM_VCPU(r13)
99 bl kvmppc_hv_entry
100
101
102
103BEGIN_FTR_SECTION
104
105 ld r5,HSTATE_DABR(r13)
106 li r6,7
107 mtspr SPRN_DABR,r5
108 mtspr SPRN_DABRX,r6
109END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
110
111
112 ld r3,PACA_SPRG_VDSO(r13)
113 mtspr SPRN_SPRG_VDSO_WRITE,r3
114
115
116 lbz r4, PACA_PMCINUSE(r13)
117 cmpwi r4, 0
118 beq 23f
119BEGIN_FTR_SECTION
120 ld r3, HSTATE_MMCR0(r13)
121 andi. r4, r3, MMCR0_PMAO_SYNC | MMCR0_PMAO
122 cmpwi r4, MMCR0_PMAO
123 beql kvmppc_fix_pmao
124END_FTR_SECTION_IFSET(CPU_FTR_PMAO_BUG)
125 lwz r3, HSTATE_PMC1(r13)
126 lwz r4, HSTATE_PMC2(r13)
127 lwz r5, HSTATE_PMC3(r13)
128 lwz r6, HSTATE_PMC4(r13)
129 lwz r8, HSTATE_PMC5(r13)
130 lwz r9, HSTATE_PMC6(r13)
131 mtspr SPRN_PMC1, r3
132 mtspr SPRN_PMC2, r4
133 mtspr SPRN_PMC3, r5
134 mtspr SPRN_PMC4, r6
135 mtspr SPRN_PMC5, r8
136 mtspr SPRN_PMC6, r9
137 ld r3, HSTATE_MMCR0(r13)
138 ld r4, HSTATE_MMCR1(r13)
139 ld r5, HSTATE_MMCRA(r13)
140 ld r6, HSTATE_SIAR(r13)
141 ld r7, HSTATE_SDAR(r13)
142 mtspr SPRN_MMCR1, r4
143 mtspr SPRN_MMCRA, r5
144 mtspr SPRN_SIAR, r6
145 mtspr SPRN_SDAR, r7
146BEGIN_FTR_SECTION
147 ld r8, HSTATE_MMCR2(r13)
148 ld r9, HSTATE_SIER(r13)
149 mtspr SPRN_MMCR2, r8
150 mtspr SPRN_SIER, r9
151END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
152 mtspr SPRN_MMCR0, r3
153 isync
15423:
155
156
157
158
159
160 ld r3, HSTATE_DECEXP(r13)
161 mftb r4
162 subf r4, r4, r3
163 mtspr SPRN_DEC, r4
164
165
166 li r0, 0
167 stb r0, HSTATE_HWTHREAD_REQ(r13)
168
169
170
171
172
173
174
175
176
177
178 ld r8, 112+PPC_LR_STKOFF(r1)
179 addi r1, r1, 112
180 ld r7, HSTATE_HOST_MSR(r13)
181
182
183 mr r3, r12
184
185
186
187
188
189
190 mfmsr r0
191 andi. r0, r0, MSR_IR
192 bne .Lvirt_return
193
194
195 mfmsr r6
196 li r0, MSR_RI
197 andc r6, r6, r0
198 mtmsrd r6, 1
199 mtsrr0 r8
200 mtsrr1 r7
201 RFI_TO_KERNEL
202
203
204.Lvirt_return:
205 mtlr r8
206 blr
207
208kvmppc_primary_no_guest:
209
210
211
212
213 mfspr r3, SPRN_HDEC
214 mtspr SPRN_DEC, r3
215
216
217
218
219
220 ld r5, HSTATE_KVM_VCORE(r13)
22165: lbz r0, VCORE_IN_GUEST(r5)
222 cmpwi r0, 0
223 beq 65b
224
225 ld r8,VCORE_LPCR(r5)
226 mtspr SPRN_LPCR,r8
227 isync
228
229 ld r5, HSTATE_KVM_VCORE(r13)
230 lbz r7, HSTATE_PTID(r13)
231 li r0, 1
232 sld r0, r0, r7
233 addi r6, r5, VCORE_NAPPING_THREADS
2341: lwarx r3, 0, r6
235 or r3, r3, r0
236 stwcx. r3, 0, r6
237 bne 1b
238
239 isync
240 li r12, 0
241 lwz r7, VCORE_ENTRY_EXIT(r5)
242 cmpwi r7, 0x100
243 bge kvm_novcpu_exit
244 li r3, NAPPING_NOVCPU
245 stb r3, HSTATE_NAPPING(r13)
246
247 li r3, 0
248 b kvm_do_nap
249
250
251
252
253
254
255
256
257kvm_novcpu_wakeup:
258 ld r1, HSTATE_HOST_R1(r13)
259 ld r5, HSTATE_KVM_VCORE(r13)
260 li r0, 0
261 stb r0, HSTATE_NAPPING(r13)
262
263
264 bl kvmppc_check_wake_reason
265
266
267
268
269
270
271 ld r5, HSTATE_KVM_VCORE(r13)
272
273
274 lwz r0, VCORE_ENTRY_EXIT(r5)
275 cmpwi r0, 0x100
276 bge kvm_novcpu_exit
277
278
279 lbz r7, HSTATE_PTID(r13)
280 li r0, 1
281 sld r0, r0, r7
282 addi r6, r5, VCORE_NAPPING_THREADS
2834: lwarx r7, 0, r6
284 andc r7, r7, r0
285 stwcx. r7, 0, r6
286 bne 4b
287
288
289 cmpdi r3, 0
290 bge kvm_novcpu_exit
291
292
293 mfspr r0, SPRN_HDEC
294 EXTEND_HDEC(r0)
295 li r12, BOOK3S_INTERRUPT_HV_DECREMENTER
296 cmpdi r0, 0
297 blt kvm_novcpu_exit
298
299
300 ld r4, HSTATE_KVM_VCPU(r13)
301 cmpdi r4, 0
302 beq kvmppc_primary_no_guest
303
304#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
305 addi r3, r4, VCPU_TB_RMENTRY
306 bl kvmhv_start_timing
307#endif
308 b kvmppc_got_guest
309
310kvm_novcpu_exit:
311#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
312 ld r4, HSTATE_KVM_VCPU(r13)
313 cmpdi r4, 0
314 beq 13f
315 addi r3, r4, VCPU_TB_RMEXIT
316 bl kvmhv_accumulate_time
317#endif
31813: mr r3, r12
319 stw r12, STACK_SLOT_TRAP(r1)
320 bl kvmhv_commence_exit
321 nop
322 b kvmhv_switch_to_host
323
324
325
326
327
328
329
330 .globl kvm_start_guest
331kvm_start_guest:
332
333 mfspr r0, SPRN_CTRLF
334 ori r0, r0, 1
335 mtspr SPRN_CTRLT, r0
336
337
338
339
340
341 mtspr SPRN_SRR1,r3
342
343 ld r2,PACATOC(r13)
344
345 li r0,KVM_HWTHREAD_IN_KVM
346 stb r0,HSTATE_HWTHREAD_STATE(r13)
347
348
349 li r0,1
350 stb r0,PACA_NAPSTATELOST(r13)
351
352
353 lbz r0,HSTATE_NAPPING(r13)
354 cmpwi r0,NAPPING_CEDE
355 beq kvm_end_cede
356 cmpwi r0,NAPPING_NOVCPU
357 beq kvm_novcpu_wakeup
358
359 ld r1,PACAEMERGSP(r13)
360 subi r1,r1,STACK_FRAME_OVERHEAD
361
362
363
364
365
366
367
368
369
370 bl kvmppc_check_wake_reason
371
372
373
374
375
376 cmpdi r3, 0
377 bge kvm_no_guest
378
379
380 ld r5,HSTATE_KVM_VCORE(r13)
381 cmpdi r5,0
382
383 beq kvm_no_guest
384
385kvm_secondary_got_guest:
386
387
388 ld r6, PACA_DSCR_DEFAULT(r13)
389 std r6, HSTATE_DSCR(r13)
390
391
392 lbz r4, HSTATE_PTID(r13)
393 cmpwi r4, 0
394 bne 63f
395 LOAD_REG_ADDR(r6, decrementer_max)
396 ld r6, 0(r6)
397 mtspr SPRN_HDEC, r6
398
399 ld r6, HSTATE_SPLIT_MODE(r13)
400 cmpdi r6, 0
401 beq 63f
402BEGIN_FTR_SECTION
403 ld r0, KVM_SPLIT_RPR(r6)
404 mtspr SPRN_RPR, r0
405 ld r0, KVM_SPLIT_PMMAR(r6)
406 mtspr SPRN_PMMAR, r0
407 ld r0, KVM_SPLIT_LDBAR(r6)
408 mtspr SPRN_LDBAR, r0
409 isync
410FTR_SECTION_ELSE
411
412 lwz r4, KVM_SPLIT_DO_SET(r6)
413 cmpwi r4, 0
414 beq 1f
415 mr r3, r6
416 bl kvmhv_p9_set_lpcr
417 nop
4181:
419ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_300)
42063:
421
422 lwsync
423 ld r4, HSTATE_KVM_VCPU(r13)
424 bl kvmppc_hv_entry
425
426
427
428 li r0, 0
429 std r0, HSTATE_KVM_VCPU(r13)
430
431
432
433
434
435
436 lwsync
437 std r0, HSTATE_KVM_VCORE(r13)
438
439
440
441
442
443
444
445
446 cmpwi r12, BOOK3S_INTERRUPT_HMI
447 bne kvm_no_guest
448
449 li r3,0
450 bl hmi_exception_realmode
451
452
453
454
455
456
457kvm_no_guest:
458 lbz r3, HSTATE_HWTHREAD_REQ(r13)
459 cmpwi r3, 0
460 bne 53f
461 HMT_MEDIUM
462 li r0, KVM_HWTHREAD_IN_KERNEL
463 stb r0, HSTATE_HWTHREAD_STATE(r13)
464
465 sync
466 lbz r3, HSTATE_HWTHREAD_REQ(r13)
467 cmpwi r3, 0
468 bne 54f
469
470
471
472
473
474
475 li r3, LPCR_PECE0
476 mfspr r4, SPRN_LPCR
477 rlwimi r4, r3, 0, LPCR_PECE0 | LPCR_PECE1
478 mtspr SPRN_LPCR, r4
479 li r3, 0
480 mfspr r12,SPRN_SRR1
481 b pnv_wakeup_loss
482
48353: HMT_LOW
484 ld r5, HSTATE_KVM_VCORE(r13)
485 cmpdi r5, 0
486 bne 60f
487 ld r3, HSTATE_SPLIT_MODE(r13)
488 cmpdi r3, 0
489 beq kvm_no_guest
490 lwz r0, KVM_SPLIT_DO_SET(r3)
491 cmpwi r0, 0
492 bne kvmhv_do_set
493 lwz r0, KVM_SPLIT_DO_RESTORE(r3)
494 cmpwi r0, 0
495 bne kvmhv_do_restore
496 lbz r0, KVM_SPLIT_DO_NAP(r3)
497 cmpwi r0, 0
498 beq kvm_no_guest
499 HMT_MEDIUM
500 b kvm_unsplit_nap
50160: HMT_MEDIUM
502 b kvm_secondary_got_guest
503
50454: li r0, KVM_HWTHREAD_IN_KVM
505 stb r0, HSTATE_HWTHREAD_STATE(r13)
506 b kvm_no_guest
507
508kvmhv_do_set:
509
510 HMT_MEDIUM
511 bl kvmhv_p9_set_lpcr
512 nop
513 b kvm_no_guest
514
515kvmhv_do_restore:
516 HMT_MEDIUM
517 bl kvmhv_p9_restore_lpcr
518 nop
519 b kvm_no_guest
520
521
522
523
524
525kvm_unsplit_nap:
526
527
528
529
530
531
532
533
534
535
536
537 cmpwi r12, BOOK3S_INTERRUPT_HMI
538 bne 55f
539 li r3, 0
540 bl hmi_exception_realmode
54155:
542
543
544
545
546 sync
547 ld r0, HSTATE_KVM_VCORE(r13)
548 cmpdi r0, 0
549 bne kvm_no_guest
550
551BEGIN_FTR_SECTION
552 lis r6, (PPC_DBELL_SERVER << (63-36))@h
553 PPC_MSGCLR(6)
554END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
555
556 ld r3, HSTATE_SPLIT_MODE(r13)
557 li r0, 1
558 lbz r4, HSTATE_TID(r13)
559 addi r4, r4, KVM_SPLIT_NAPPED
560 stbx r0, r3, r4
561
562 sync
563 lbz r0, KVM_SPLIT_DO_NAP(r3)
564 cmpwi r0, 0
565 beq 57f
566 li r3, (LPCR_PECEDH | LPCR_PECE0) >> 4
567 mfspr r5, SPRN_LPCR
568 rlwimi r5, r3, 4, (LPCR_PECEDP | LPCR_PECEDH | LPCR_PECE0 | LPCR_PECE1)
569 b kvm_nap_sequence
570
57157: li r0, 0
572 stbx r0, r3, r4
573 b kvm_no_guest
574
575
576
577
578
579
580
581.global kvmppc_hv_entry
582kvmppc_hv_entry:
583
584
585
586
587
588
589
590
591
592
593
594 mflr r0
595 std r0, PPC_LR_STKOFF(r1)
596 stdu r1, -SFS(r1)
597
598
599 std r1, HSTATE_HOST_R1(r13)
600
601 li r6, KVM_GUEST_MODE_HOST_HV
602 stb r6, HSTATE_IN_GUEST(r13)
603
604#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
605
606 cmpdi r4, 0
607 beq 1f
608 addi r3, r4, VCPU_TB_RMENTRY
609 bl kvmhv_start_timing
6101:
611#endif
612
613
614 ld r5, HSTATE_KVM_VCORE(r13)
615 ld r9, VCORE_KVM(r5)
616 lbz r0, KVM_RADIX(r9)
617 cmpwi cr7, r0, 0
618
619
620
621
622
623
624
625 li r7, 1
626 lbz r6, HSTATE_PTID(r13)
627 sld r7, r7, r6
628 addi r8, r5, VCORE_ENTRY_EXIT
62921: lwarx r3, 0, r8
630 cmpwi r3, 0x100
631 bge secondary_too_late
632 or r3, r3, r7
633 stwcx. r3, 0, r8
634 bne 21b
635
636
637 cmpwi r6,0
638 bne 10f
639 lwz r7,KVM_LPID(r9)
640BEGIN_FTR_SECTION
641 ld r6,KVM_SDR1(r9)
642 li r0,LPID_RSVD
643 mtspr SPRN_LPID,r0
644 ptesync
645 mtspr SPRN_SDR1,r6
646END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300)
647 mtspr SPRN_LPID,r7
648 isync
649
650
651 lhz r6,PACAPACAINDEX(r13)
652BEGIN_FTR_SECTION
653
654
655
656
657
658
659 clrrdi r6,r6,2
660END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
661 clrldi r7,r6,64-6
662 srdi r6,r6,6
663 sldi r6,r6,3
664 add r6,r6,r9
665 addi r6,r6,KVM_NEED_FLUSH
666 li r8,1
667 sld r8,r8,r7
668 ld r7,0(r6)
669 and. r7,r7,r8
670 beq 22f
671
672 lwz r0,KVM_TLB_SETS(r9)
673 mtctr r0
674 li r7,0x800
675 ptesync
676 li r0,0
677 bne cr7, 29f
67828: tlbiel r7
679 addi r7,r7,0x1000
680 bdnz 28b
681 b 30f
68229: PPC_TLBIEL(7,0,2,1,1)
683 addi r7,r7,0x1000
684 bdnz 29b
68530: ptesync
68623: ldarx r7,0,r6
687 andc r7,r7,r8
688 stdcx. r7,0,r6
689 bne 23b
690
691
69222: ld r8,VCORE_TB_OFFSET(r5)
693 cmpdi r8,0
694 beq 37f
695 std r8, VCORE_TB_OFFSET_APPL(r5)
696 mftb r6
697 add r8,r8,r6
698 mtspr SPRN_TBU40,r8
699 mftb r7
700 clrldi r6,r6,40
701 clrldi r7,r7,40
702 cmpld r7,r6
703 bge 37f
704 addis r8,r8,0x100
705 mtspr SPRN_TBU40,r8
706
707
70837: ld r7, VCORE_PCR(r5)
709 cmpdi r7, 0
710 beq 38f
711 mtspr SPRN_PCR, r7
71238:
713
714BEGIN_FTR_SECTION
715
716 ld r8, VCORE_DPDES(r5)
717 ld r7, VCORE_VTB(r5)
718 mtspr SPRN_DPDES, r8
719 mtspr SPRN_VTB, r7
720END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
721
722
723 bl kvmppc_subcore_enter_guest
724 nop
725 ld r5, HSTATE_KVM_VCORE(r13)
726 ld r4, HSTATE_KVM_VCPU(r13)
727 li r0,1
728 stb r0,VCORE_IN_GUEST(r5)
729
730
73110: cmpdi r4, 0
732 beq kvmppc_primary_no_guest
733kvmppc_got_guest:
734
735 ld r3, VCPU_VPA(r4)
736 cmpdi r3, 0
737 beq 25f
738 li r6, LPPACA_YIELDCOUNT
739 LWZX_BE r5, r3, r6
740 addi r5, r5, 1
741 STWX_BE r5, r3, r6
742 li r6, 1
743 stb r6, VCPU_VPA_DIRTY(r4)
74425:
745
746
747 mfspr r5,SPRN_PURR
748 mfspr r6,SPRN_SPURR
749 std r5,HSTATE_PURR(r13)
750 std r6,HSTATE_SPURR(r13)
751 ld r7,VCPU_PURR(r4)
752 ld r8,VCPU_SPURR(r4)
753 mtspr SPRN_PURR,r7
754 mtspr SPRN_SPURR,r8
755
756
757BEGIN_FTR_SECTION
758 mfspr r5, SPRN_TIDR
759 mfspr r6, SPRN_PSSCR
760 mfspr r7, SPRN_PID
761 mfspr r8, SPRN_IAMR
762 std r5, STACK_SLOT_TID(r1)
763 std r6, STACK_SLOT_PSSCR(r1)
764 std r7, STACK_SLOT_PID(r1)
765 std r8, STACK_SLOT_IAMR(r1)
766 mfspr r5, SPRN_HFSCR
767 std r5, STACK_SLOT_HFSCR(r1)
768END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
769BEGIN_FTR_SECTION
770 mfspr r5, SPRN_CIABR
771 mfspr r6, SPRN_DAWR
772 mfspr r7, SPRN_DAWRX
773 std r5, STACK_SLOT_CIABR(r1)
774 std r6, STACK_SLOT_DAWR(r1)
775 std r7, STACK_SLOT_DAWRX(r1)
776END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
777
778BEGIN_FTR_SECTION
779
780
781 lwz r5,VCPU_DABRX(r4)
782 ld r6,VCPU_DABR(r4)
783 mtspr SPRN_DABRX,r5
784 mtspr SPRN_DABR,r6
785 isync
786END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
787
788#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
789
790
791
792
793BEGIN_FTR_SECTION
794 b 91f
795END_FTR_SECTION(CPU_FTR_TM | CPU_FTR_P9_TM_HV_ASSIST, 0)
796
797
798
799 bl kvmppc_restore_tm
80091:
801#endif
802
803
804
805 li r3, 1
806 sldi r3, r3, 31
807 mtspr SPRN_MMCR0, r3
808 isync
809BEGIN_FTR_SECTION
810 ld r3, VCPU_MMCR(r4)
811 andi. r5, r3, MMCR0_PMAO_SYNC | MMCR0_PMAO
812 cmpwi r5, MMCR0_PMAO
813 beql kvmppc_fix_pmao
814END_FTR_SECTION_IFSET(CPU_FTR_PMAO_BUG)
815 lwz r3, VCPU_PMC(r4)
816 lwz r5, VCPU_PMC + 4(r4)
817 lwz r6, VCPU_PMC + 8(r4)
818 lwz r7, VCPU_PMC + 12(r4)
819 lwz r8, VCPU_PMC + 16(r4)
820 lwz r9, VCPU_PMC + 20(r4)
821 mtspr SPRN_PMC1, r3
822 mtspr SPRN_PMC2, r5
823 mtspr SPRN_PMC3, r6
824 mtspr SPRN_PMC4, r7
825 mtspr SPRN_PMC5, r8
826 mtspr SPRN_PMC6, r9
827 ld r3, VCPU_MMCR(r4)
828 ld r5, VCPU_MMCR + 8(r4)
829 ld r6, VCPU_MMCR + 16(r4)
830 ld r7, VCPU_SIAR(r4)
831 ld r8, VCPU_SDAR(r4)
832 mtspr SPRN_MMCR1, r5
833 mtspr SPRN_MMCRA, r6
834 mtspr SPRN_SIAR, r7
835 mtspr SPRN_SDAR, r8
836BEGIN_FTR_SECTION
837 ld r5, VCPU_MMCR + 24(r4)
838 ld r6, VCPU_SIER(r4)
839 mtspr SPRN_MMCR2, r5
840 mtspr SPRN_SIER, r6
841BEGIN_FTR_SECTION_NESTED(96)
842 lwz r7, VCPU_PMC + 24(r4)
843 lwz r8, VCPU_PMC + 28(r4)
844 ld r9, VCPU_MMCR + 32(r4)
845 mtspr SPRN_SPMC1, r7
846 mtspr SPRN_SPMC2, r8
847 mtspr SPRN_MMCRS, r9
848END_FTR_SECTION_NESTED(CPU_FTR_ARCH_300, 0, 96)
849END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
850 mtspr SPRN_MMCR0, r3
851 isync
852
853
854 bl kvmppc_load_fp
855
856 ld r14, VCPU_GPR(R14)(r4)
857 ld r15, VCPU_GPR(R15)(r4)
858 ld r16, VCPU_GPR(R16)(r4)
859 ld r17, VCPU_GPR(R17)(r4)
860 ld r18, VCPU_GPR(R18)(r4)
861 ld r19, VCPU_GPR(R19)(r4)
862 ld r20, VCPU_GPR(R20)(r4)
863 ld r21, VCPU_GPR(R21)(r4)
864 ld r22, VCPU_GPR(R22)(r4)
865 ld r23, VCPU_GPR(R23)(r4)
866 ld r24, VCPU_GPR(R24)(r4)
867 ld r25, VCPU_GPR(R25)(r4)
868 ld r26, VCPU_GPR(R26)(r4)
869 ld r27, VCPU_GPR(R27)(r4)
870 ld r28, VCPU_GPR(R28)(r4)
871 ld r29, VCPU_GPR(R29)(r4)
872 ld r30, VCPU_GPR(R30)(r4)
873 ld r31, VCPU_GPR(R31)(r4)
874
875
876 ld r5, VCPU_DSCR(r4)
877 mtspr SPRN_DSCR, r5
878
879BEGIN_FTR_SECTION
880
881 b 8f
882END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
883
884 ld r5, VCPU_IAMR(r4)
885 lwz r6, VCPU_PSPB(r4)
886 ld r7, VCPU_FSCR(r4)
887 mtspr SPRN_IAMR, r5
888 mtspr SPRN_PSPB, r6
889 mtspr SPRN_FSCR, r7
890 ld r5, VCPU_DAWR(r4)
891 ld r6, VCPU_DAWRX(r4)
892 ld r7, VCPU_CIABR(r4)
893 ld r8, VCPU_TAR(r4)
894
895
896
897
898BEGIN_FTR_SECTION
899 mtspr SPRN_DAWR, r5
900 mtspr SPRN_DAWRX, r6
901END_FTR_SECTION_IFSET(CPU_FTR_DAWR)
902 mtspr SPRN_CIABR, r7
903 mtspr SPRN_TAR, r8
904 ld r5, VCPU_IC(r4)
905 ld r8, VCPU_EBBHR(r4)
906 mtspr SPRN_IC, r5
907 mtspr SPRN_EBBHR, r8
908 ld r5, VCPU_EBBRR(r4)
909 ld r6, VCPU_BESCR(r4)
910 lwz r7, VCPU_GUEST_PID(r4)
911 ld r8, VCPU_WORT(r4)
912 mtspr SPRN_EBBRR, r5
913 mtspr SPRN_BESCR, r6
914 mtspr SPRN_PID, r7
915 mtspr SPRN_WORT, r8
916BEGIN_FTR_SECTION
917 PPC_INVALIDATE_ERAT
918END_FTR_SECTION_IFSET(CPU_FTR_POWER9_DD1)
919BEGIN_FTR_SECTION
920
921 ld r5, VCPU_TCSCR(r4)
922 ld r6, VCPU_ACOP(r4)
923 ld r7, VCPU_CSIGR(r4)
924 ld r8, VCPU_TACR(r4)
925 mtspr SPRN_TCSCR, r5
926 mtspr SPRN_ACOP, r6
927 mtspr SPRN_CSIGR, r7
928 mtspr SPRN_TACR, r8
929 nop
930FTR_SECTION_ELSE
931
932 ld r5, VCPU_TID(r4)
933 ld r6, VCPU_PSSCR(r4)
934 lbz r8, HSTATE_FAKE_SUSPEND(r13)
935 oris r6, r6, PSSCR_EC@h
936 rldimi r6, r8, PSSCR_FAKE_SUSPEND_LG, 63 - PSSCR_FAKE_SUSPEND_LG
937 ld r7, VCPU_HFSCR(r4)
938 mtspr SPRN_TIDR, r5
939 mtspr SPRN_PSSCR, r6
940 mtspr SPRN_HFSCR, r7
941ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_300)
9428:
943
944 ld r5, VCPU_SPRG0(r4)
945 ld r6, VCPU_SPRG1(r4)
946 ld r7, VCPU_SPRG2(r4)
947 ld r8, VCPU_SPRG3(r4)
948 mtspr SPRN_SPRG0, r5
949 mtspr SPRN_SPRG1, r6
950 mtspr SPRN_SPRG2, r7
951 mtspr SPRN_SPRG3, r8
952
953
954 ld r5, VCPU_DAR(r4)
955 lwz r6, VCPU_DSISR(r4)
956 mtspr SPRN_DAR, r5
957 mtspr SPRN_DSISR, r6
958
959
960 ld r5,VCPU_AMR(r4)
961 ld r6,VCPU_UAMOR(r4)
962 li r7,-1
963 mtspr SPRN_AMR,r5
964 mtspr SPRN_UAMOR,r6
965 mtspr SPRN_AMOR,r7
966
967
968 lwz r5,VCPU_CTRL(r4)
969 andi. r5,r5,1
970 bne 4f
971 mfspr r6,SPRN_CTRLF
972 clrrdi r6,r6,1
973 mtspr SPRN_CTRLT,r6
9744:
975
976 ld r5, HSTATE_KVM_VCORE(r13)
977 lbz r6, HSTATE_PTID(r13)
978 cmpwi r6, 0
979 beq 21f
980 lbz r0, VCORE_IN_GUEST(r5)
981 cmpwi r0, 0
982 bne 21f
983 HMT_LOW
98420: lwz r3, VCORE_ENTRY_EXIT(r5)
985 cmpwi r3, 0x100
986 bge no_switch_exit
987 lbz r0, VCORE_IN_GUEST(r5)
988 cmpwi r0, 0
989 beq 20b
990 HMT_MEDIUM
99121:
992
993 ld r8,VCORE_LPCR(r5)
994 mtspr SPRN_LPCR,r8
995 isync
996
997
998
999
1000 ld r8,VCPU_DEC_EXPIRES(r4)
1001
1002 ld r5,HSTATE_KVM_VCORE(r13)
1003 ld r6,VCORE_TB_OFFSET_APPL(r5)
1004 add r8,r8,r6
1005 mftb r7
1006 subf r3,r7,r8
1007 mtspr SPRN_DEC,r3
1008
1009
1010 mfspr r3, SPRN_HDEC
1011 EXTEND_HDEC(r3)
1012 cmpdi r3, 512
1013 blt hdec_soon
1014
1015
1016 ld r6, VCPU_KVM(r4)
1017 lbz r0, KVM_RADIX(r6)
1018 cmpwi r0, 0
1019 bne 9f
1020 li r6, 0
1021 slbmte r6, r6
1022 slbia
1023 ptesync
1024
1025
1026 lwz r5,VCPU_SLB_MAX(r4)
1027 cmpwi r5,0
1028 beq 9f
1029 mtctr r5
1030 addi r6,r4,VCPU_SLB
10311: ld r8,VCPU_SLB_E(r6)
1032 ld r9,VCPU_SLB_V(r6)
1033 slbmte r9,r8
1034 addi r6,r6,VCPU_SLB_SIZE
1035 bdnz 1b
10369:
1037
1038#ifdef CONFIG_KVM_XICS
1039
1040 ld r10, HSTATE_XIVE_TIMA_PHYS(r13)
1041 cmpldi cr0, r10, 0
1042 beq no_xive
1043 ld r11, VCPU_XIVE_SAVED_STATE(r4)
1044 li r9, TM_QW1_OS
1045 eieio
1046 stdcix r11,r9,r10
1047 lwz r11, VCPU_XIVE_CAM_WORD(r4)
1048 li r9, TM_QW1_OS + TM_WORD2
1049 stwcix r11,r9,r10
1050 li r9, 1
1051 stb r9, VCPU_XIVE_PUSHED(r4)
1052 eieio
1053
1054
1055
1056
1057
1058
1059
1060
1061 li r0,0
1062 stb r0, VCPU_IRQ_PENDING(r4)
1063
1064
1065
1066
1067
1068 lbz r0, VCPU_XIVE_ESC_ON(r4)
1069 cmpwi r0,0
1070 beq 1f
1071 ld r10, VCPU_XIVE_ESC_RADDR(r4)
1072 li r9, XIVE_ESB_SET_PQ_01
1073 ldcix r0, r10, r9
1074 sync
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091 andi. r0, r0, XIVE_ESB_VAL_P
1092 bne- 1f
1093
1094
1095 li r0, 0
1096 stb r0, VCPU_XIVE_ESC_ON(r4)
10971:
1098no_xive:
1099#endif
1100
1101deliver_guest_interrupt:
1102 ld r6, VCPU_CTR(r4)
1103 ld r7, VCPU_XER(r4)
1104
1105 mtctr r6
1106 mtxer r7
1107
1108kvmppc_cede_reentry:
1109 ld r10, VCPU_PC(r4)
1110 ld r11, VCPU_MSR(r4)
1111 ld r6, VCPU_SRR0(r4)
1112 ld r7, VCPU_SRR1(r4)
1113 mtspr SPRN_SRR0, r6
1114 mtspr SPRN_SRR1, r7
1115
1116
1117 rldicl r11, r11, 63 - MSR_HV_LG, 1
1118 rotldi r11, r11, 1 + MSR_HV_LG
1119 ori r11, r11, MSR_ME
1120
1121
1122 ld r0, VCPU_PENDING_EXC(r4)
1123 rldicl r0, r0, 64 - BOOK3S_IRQPRIO_EXTERNAL_LEVEL, 63
1124 cmpdi cr1, r0, 0
1125 andi. r8, r11, MSR_EE
1126 mfspr r8, SPRN_LPCR
1127
1128 rldimi r8, r0, LPCR_MER_SH, 63 - LPCR_MER_SH
1129 mtspr SPRN_LPCR, r8
1130 isync
1131 beq 5f
1132 li r0, BOOK3S_INTERRUPT_EXTERNAL
1133 bne cr1, 12f
1134 mfspr r0, SPRN_DEC
1135BEGIN_FTR_SECTION
1136
1137 andis. r8, r8, LPCR_LD@h
1138 bne 15f
1139END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
1140 extsw r0, r0
114115: cmpdi r0, 0
1142 li r0, BOOK3S_INTERRUPT_DECREMENTER
1143 bge 5f
1144
114512: mtspr SPRN_SRR0, r10
1146 mr r10,r0
1147 mtspr SPRN_SRR1, r11
1148 mr r9, r4
1149 bl kvmppc_msr_interrupt
11505:
1151BEGIN_FTR_SECTION
1152 b fast_guest_return
1153END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300)
1154
1155 lbz r0, VCPU_DBELL_REQ(r4)
1156 cmpwi r0, 0
1157 beq fast_guest_return
1158 ld r5, HSTATE_KVM_VCORE(r13)
1159
1160 li r0, 1
1161 mtspr SPRN_DPDES, r0
1162 std r0, VCORE_DPDES(r5)
1163
1164 lwsync
1165
1166 li r0, 0
1167 stb r0, VCPU_DBELL_REQ(r4)
1168
1169
1170
1171
1172
1173
1174
1175
1176fast_guest_return:
1177 li r0,0
1178 stb r0,VCPU_CEDED(r4)
1179 mtspr SPRN_HSRR0,r10
1180 mtspr SPRN_HSRR1,r11
1181
1182
1183 li r9, KVM_GUEST_MODE_GUEST_HV
1184 stb r9, HSTATE_IN_GUEST(r13)
1185
1186#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
1187
1188 addi r3, r4, VCPU_TB_GUEST
1189 bl kvmhv_accumulate_time
1190#endif
1191
1192
1193
1194BEGIN_FTR_SECTION
1195 ld r5, VCPU_CFAR(r4)
1196 mtspr SPRN_CFAR, r5
1197END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
1198BEGIN_FTR_SECTION
1199 ld r0, VCPU_PPR(r4)
1200END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
1201
1202 ld r5, VCPU_LR(r4)
1203 lwz r6, VCPU_CR(r4)
1204 mtlr r5
1205 mtcr r6
1206
1207 ld r1, VCPU_GPR(R1)(r4)
1208 ld r2, VCPU_GPR(R2)(r4)
1209 ld r3, VCPU_GPR(R3)(r4)
1210 ld r5, VCPU_GPR(R5)(r4)
1211 ld r6, VCPU_GPR(R6)(r4)
1212 ld r7, VCPU_GPR(R7)(r4)
1213 ld r8, VCPU_GPR(R8)(r4)
1214 ld r9, VCPU_GPR(R9)(r4)
1215 ld r10, VCPU_GPR(R10)(r4)
1216 ld r11, VCPU_GPR(R11)(r4)
1217 ld r12, VCPU_GPR(R12)(r4)
1218 ld r13, VCPU_GPR(R13)(r4)
1219
1220BEGIN_FTR_SECTION
1221 mtspr SPRN_PPR, r0
1222END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
1223
1224
1225BEGIN_FTR_SECTION
1226 li r0, 0x7fff
1227 mtspr SPRN_HDSISR, r0
1228END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
1229
1230 ld r0, VCPU_GPR(R0)(r4)
1231 ld r4, VCPU_GPR(R4)(r4)
1232 HRFI_TO_GUEST
1233 b .
1234
1235secondary_too_late:
1236 li r12, 0
1237 stw r12, STACK_SLOT_TRAP(r1)
1238 cmpdi r4, 0
1239 beq 11f
1240 stw r12, VCPU_TRAP(r4)
1241#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
1242 addi r3, r4, VCPU_TB_RMEXIT
1243 bl kvmhv_accumulate_time
1244#endif
124511: b kvmhv_switch_to_host
1246
1247no_switch_exit:
1248 HMT_MEDIUM
1249 li r12, 0
1250 b 12f
1251hdec_soon:
1252 li r12, BOOK3S_INTERRUPT_HV_DECREMENTER
125312: stw r12, VCPU_TRAP(r4)
1254 mr r9, r4
1255#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
1256 addi r3, r4, VCPU_TB_RMEXIT
1257 bl kvmhv_accumulate_time
1258#endif
1259 b guest_bypass
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270 .globl kvmppc_interrupt_hv
1271kvmppc_interrupt_hv:
1272
1273
1274
1275
1276
1277
1278
1279
1280 std r9, HSTATE_SCRATCH2(r13)
1281 lbz r9, HSTATE_IN_GUEST(r13)
1282 cmpwi r9, KVM_GUEST_MODE_HOST_HV
1283 beq kvmppc_bad_host_intr
1284#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
1285 cmpwi r9, KVM_GUEST_MODE_GUEST
1286 ld r9, HSTATE_SCRATCH2(r13)
1287 beq kvmppc_interrupt_pr
1288#endif
1289
1290 li r9, KVM_GUEST_MODE_HOST_HV
1291 stb r9, HSTATE_IN_GUEST(r13)
1292
1293 ld r9, HSTATE_KVM_VCPU(r13)
1294
1295
1296
1297 std r0, VCPU_GPR(R0)(r9)
1298 std r1, VCPU_GPR(R1)(r9)
1299 std r2, VCPU_GPR(R2)(r9)
1300 std r3, VCPU_GPR(R3)(r9)
1301 std r4, VCPU_GPR(R4)(r9)
1302 std r5, VCPU_GPR(R5)(r9)
1303 std r6, VCPU_GPR(R6)(r9)
1304 std r7, VCPU_GPR(R7)(r9)
1305 std r8, VCPU_GPR(R8)(r9)
1306 ld r0, HSTATE_SCRATCH2(r13)
1307 std r0, VCPU_GPR(R9)(r9)
1308 std r10, VCPU_GPR(R10)(r9)
1309 std r11, VCPU_GPR(R11)(r9)
1310 ld r3, HSTATE_SCRATCH0(r13)
1311 std r3, VCPU_GPR(R12)(r9)
1312
1313 srdi r4, r12, 32
1314 stw r4, VCPU_CR(r9)
1315BEGIN_FTR_SECTION
1316 ld r3, HSTATE_CFAR(r13)
1317 std r3, VCPU_CFAR(r9)
1318END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
1319BEGIN_FTR_SECTION
1320 ld r4, HSTATE_PPR(r13)
1321 std r4, VCPU_PPR(r9)
1322END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
1323
1324
1325 ld r1, HSTATE_HOST_R1(r13)
1326 ld r2, PACATOC(r13)
1327
1328 mfspr r10, SPRN_SRR0
1329 mfspr r11, SPRN_SRR1
1330 std r10, VCPU_SRR0(r9)
1331 std r11, VCPU_SRR1(r9)
1332
1333 clrldi r12, r12, 32
1334 andi. r0, r12, 2
1335 beq 1f
1336 mfspr r10, SPRN_HSRR0
1337 mfspr r11, SPRN_HSRR1
1338 clrrdi r12, r12, 2
13391: std r10, VCPU_PC(r9)
1340 std r11, VCPU_MSR(r9)
1341
1342 GET_SCRATCH0(r3)
1343 mflr r4
1344 std r3, VCPU_GPR(R13)(r9)
1345 std r4, VCPU_LR(r9)
1346
1347 stw r12,VCPU_TRAP(r9)
1348
1349
1350
1351
1352
1353
1354
1355 li r0, MSR_RI
1356 mtmsrd r0, 1
1357
1358#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
1359 addi r3, r9, VCPU_TB_RMINTR
1360 mr r4, r9
1361 bl kvmhv_accumulate_time
1362 ld r5, VCPU_GPR(R5)(r9)
1363 ld r6, VCPU_GPR(R6)(r9)
1364 ld r7, VCPU_GPR(R7)(r9)
1365 ld r8, VCPU_GPR(R8)(r9)
1366#endif
1367
1368
1369
1370 li r3,KVM_INST_FETCH_FAILED
1371 stw r3,VCPU_LAST_INST(r9)
1372 cmpwi r12,BOOK3S_INTERRUPT_H_EMUL_ASSIST
1373 bne 11f
1374 mfspr r3,SPRN_HEIR
137511: stw r3,VCPU_HEIR(r9)
1376
1377
1378#ifdef CONFIG_RELOCATABLE
1379 ld r3, HSTATE_SCRATCH1(r13)
1380 mtctr r3
1381#else
1382 mfctr r3
1383#endif
1384 mfxer r4
1385 std r3, VCPU_CTR(r9)
1386 std r4, VCPU_XER(r9)
1387
1388#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1389
1390 cmpwi r12, BOOK3S_INTERRUPT_HV_SOFTPATCH
1391 beq kvmppc_tm_emul
1392#endif
1393
1394
1395 cmpwi r12, BOOK3S_INTERRUPT_H_DATA_STORAGE
1396 beq kvmppc_hdsi
1397 cmpwi r12, BOOK3S_INTERRUPT_H_INST_STORAGE
1398 beq kvmppc_hisi
1399
1400
1401 cmpwi r12,BOOK3S_INTERRUPT_HV_DECREMENTER
1402 bne 2f
1403 mfspr r3,SPRN_HDEC
1404 EXTEND_HDEC(r3)
1405 cmpdi r3,0
1406 mr r4,r9
1407 bge fast_guest_return
14082:
1409
1410 cmpwi r12,BOOK3S_INTERRUPT_SYSCALL
1411 beq hcall_try_real_mode
1412
1413
1414 cmpwi r12, BOOK3S_INTERRUPT_H_DOORBELL
1415 bne 3f
1416BEGIN_FTR_SECTION
1417 PPC_MSGSYNC
1418 lwsync
1419END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
1420 lbz r0, HSTATE_HOST_IPI(r13)
1421 cmpwi r0, 0
1422 beq 4f
1423 b guest_exit_cont
14243:
1425
1426 cmpwi r12, BOOK3S_INTERRUPT_H_FAC_UNAVAIL
1427 bne 14f
1428 mfspr r3, SPRN_HFSCR
1429 std r3, VCPU_HFSCR(r9)
1430 b guest_exit_cont
143114:
1432
1433 cmpwi r12, BOOK3S_INTERRUPT_EXTERNAL
1434 bne+ guest_exit_cont
1435
1436
1437
1438
1439 bl kvmppc_read_intr
1440
1441
1442
1443
1444
1445 ld r9, HSTATE_KVM_VCPU(r13)
1446 li r12, BOOK3S_INTERRUPT_EXTERNAL
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475 cmpdi r3, 1
1476 ble 1f
1477
1478
1479 li r12, BOOK3S_INTERRUPT_HV_RM_HARD
1480 stw r12, VCPU_TRAP(r9)
1481 b guest_exit_cont
1482
14831:
1484 cmpdi r3, 0
1485 bgt guest_exit_cont
1486
1487
14884: ld r5, HSTATE_KVM_VCORE(r13)
1489 lwz r0, VCORE_ENTRY_EXIT(r5)
1490 cmpwi r0, 0x100
1491 mr r4, r9
1492 blt deliver_guest_interrupt
1493
1494guest_exit_cont:
1495
1496 mfdar r6
1497 mfdsisr r7
1498 std r6, VCPU_DAR(r9)
1499 stw r7, VCPU_DSISR(r9)
1500
1501 cmpwi r12,BOOK3S_INTERRUPT_H_DATA_STORAGE
1502 beq mc_cont
1503 std r6, VCPU_FAULT_DAR(r9)
1504 stw r7, VCPU_FAULT_DSISR(r9)
1505
1506
1507 cmpwi r12, BOOK3S_INTERRUPT_MACHINE_CHECK
1508 beq machine_check_realmode
1509mc_cont:
1510#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
1511 addi r3, r9, VCPU_TB_RMEXIT
1512 mr r4, r9
1513 bl kvmhv_accumulate_time
1514#endif
1515#ifdef CONFIG_KVM_XICS
1516
1517 lbz r0, VCPU_XIVE_PUSHED(r9)
1518 cmpwi cr0, r0, 0
1519 beq 1f
1520 li r7, TM_SPC_PULL_OS_CTX
1521 li r6, TM_QW1_OS
1522 mfmsr r0
1523 andi. r0, r0, MSR_DR
1524 beq 2f
1525 ld r10, HSTATE_XIVE_TIMA_VIRT(r13)
1526 cmpldi cr0, r10, 0
1527 beq 1f
1528
1529 eieio
1530 lwzx r11, r7, r10
1531
1532 ldx r11, r6, r10
1533 b 3f
15342: ld r10, HSTATE_XIVE_TIMA_PHYS(r13)
1535 cmpldi cr0, r10, 0
1536 beq 1f
1537
1538 eieio
1539 lwzcix r11, r7, r10
1540
1541 ldcix r11, r6, r10
15423: std r11, VCPU_XIVE_SAVED_STATE(r9)
1543
1544 li r10, 0
1545 li r0, 0xff
1546 stb r10, VCPU_XIVE_PUSHED(r9)
1547 stb r10, (VCPU_XIVE_SAVED_STATE+3)(r9)
1548 stb r0, (VCPU_XIVE_SAVED_STATE+4)(r9)
1549 eieio
15501:
1551#endif
1552
1553
1554 ld r5, VCPU_KVM(r9)
1555 lbz r0, KVM_RADIX(r5)
1556 li r5, 0
1557 cmpwi r0, 0
1558 bne 3f
1559 lwz r0,VCPU_SLB_NR(r9)
1560 mtctr r0
1561 li r6,0
1562 addi r7,r9,VCPU_SLB
15631: slbmfee r8,r6
1564 andis. r0,r8,SLB_ESID_V@h
1565 beq 2f
1566 add r8,r8,r6
1567 slbmfev r3,r6
1568 std r8,VCPU_SLB_E(r7)
1569 std r3,VCPU_SLB_V(r7)
1570 addi r7,r7,VCPU_SLB_SIZE
1571 addi r5,r5,1
15722: addi r6,r6,1
1573 bdnz 1b
1574
1575 li r0,0
1576 slbmte r0,r0
1577 slbia
1578 ptesync
15793: stw r5,VCPU_SLB_MAX(r9)
1580
1581
1582BEGIN_MMU_FTR_SECTION
1583 b 0f
1584END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_RADIX)
1585 ld r8,PACA_SLBSHADOWPTR(r13)
1586
1587 .rept SLB_NUM_BOLTED
1588 li r3, SLBSHADOW_SAVEAREA
1589 LDX_BE r5, r8, r3
1590 addi r3, r3, 8
1591 LDX_BE r6, r8, r3
1592 andis. r7,r5,SLB_ESID_V@h
1593 beq 1f
1594 slbmte r6,r5
15951: addi r8,r8,16
1596 .endr
15970:
1598
1599guest_bypass:
1600 stw r12, STACK_SLOT_TRAP(r1)
1601
1602
1603
1604 ld r3, HSTATE_KVM_VCORE(r13)
1605 mfspr r5,SPRN_DEC
1606 mftb r6
1607
1608BEGIN_FTR_SECTION
1609 ld r4, VCORE_LPCR(r3)
1610 andis. r4, r4, LPCR_LD@h
1611 bne 16f
1612END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
1613 extsw r5,r5
161416: add r5,r5,r6
1615
1616 ld r4,VCORE_TB_OFFSET_APPL(r3)
1617 subf r5,r4,r5
1618 std r5,VCPU_DEC_EXPIRES(r9)
1619
1620
1621 mr r3, r12
1622 bl kvmhv_commence_exit
1623 nop
1624 ld r9, HSTATE_KVM_VCPU(r13)
1625
1626
1627 li r0, -1
1628 stw r0, VCPU_CPU(r9)
1629 stw r0, VCPU_THREAD_CPU(r9)
1630
1631
1632 mfspr r6,SPRN_CTRLF
1633 stw r6,VCPU_CTRL(r9)
1634 andi. r0,r6,1
1635 bne 4f
1636 ori r6,r6,1
1637 mtspr SPRN_CTRLT,r6
16384:
1639
1640
1641
1642 mfspr r5,SPRN_PURR
1643 mfspr r6,SPRN_SPURR
1644 ld r7,VCPU_PURR(r9)
1645 ld r8,VCPU_SPURR(r9)
1646 std r5,VCPU_PURR(r9)
1647 std r6,VCPU_SPURR(r9)
1648 subf r5,r7,r5
1649 subf r6,r8,r6
1650
1651
1652
1653
1654
1655 ld r3,HSTATE_PURR(r13)
1656 ld r4,HSTATE_SPURR(r13)
1657 add r3,r3,r5
1658 add r4,r4,r6
1659 mtspr SPRN_PURR,r3
1660 mtspr SPRN_SPURR,r4
1661
1662BEGIN_FTR_SECTION
1663 b 8f
1664END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
1665
1666 mfspr r5, SPRN_IAMR
1667 mfspr r6, SPRN_PSPB
1668 mfspr r7, SPRN_FSCR
1669 std r5, VCPU_IAMR(r9)
1670 stw r6, VCPU_PSPB(r9)
1671 std r7, VCPU_FSCR(r9)
1672 mfspr r5, SPRN_IC
1673 mfspr r7, SPRN_TAR
1674 std r5, VCPU_IC(r9)
1675 std r7, VCPU_TAR(r9)
1676 mfspr r8, SPRN_EBBHR
1677 std r8, VCPU_EBBHR(r9)
1678 mfspr r5, SPRN_EBBRR
1679 mfspr r6, SPRN_BESCR
1680 mfspr r7, SPRN_PID
1681 mfspr r8, SPRN_WORT
1682 std r5, VCPU_EBBRR(r9)
1683 std r6, VCPU_BESCR(r9)
1684 stw r7, VCPU_GUEST_PID(r9)
1685 std r8, VCPU_WORT(r9)
1686BEGIN_FTR_SECTION
1687 mfspr r5, SPRN_TCSCR
1688 mfspr r6, SPRN_ACOP
1689 mfspr r7, SPRN_CSIGR
1690 mfspr r8, SPRN_TACR
1691 std r5, VCPU_TCSCR(r9)
1692 std r6, VCPU_ACOP(r9)
1693 std r7, VCPU_CSIGR(r9)
1694 std r8, VCPU_TACR(r9)
1695FTR_SECTION_ELSE
1696 mfspr r5, SPRN_TIDR
1697 mfspr r6, SPRN_PSSCR
1698 std r5, VCPU_TID(r9)
1699 rldicl r6, r6, 4, 50
1700 rotldi r6, r6, 60
1701 std r6, VCPU_PSSCR(r9)
1702
1703 ld r7, STACK_SLOT_HFSCR(r1)
1704 mtspr SPRN_HFSCR, r7
1705ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_300)
1706
1707
1708
1709
1710 li r0, 0
1711 mtspr SPRN_PSPB, r0
1712 mtspr SPRN_WORT, r0
1713BEGIN_FTR_SECTION
1714 mtspr SPRN_IAMR, r0
1715 mtspr SPRN_TCSCR, r0
1716
1717 li r0, 1
1718 sldi r0, r0, 31
1719 mtspr SPRN_MMCRS, r0
1720END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300)
17218:
1722
1723
1724 mfspr r5,SPRN_AMR
1725 mfspr r6,SPRN_UAMOR
1726 std r5,VCPU_AMR(r9)
1727 std r6,VCPU_UAMOR(r9)
1728 li r6,0
1729 mtspr SPRN_AMR,r6
1730 mtspr SPRN_UAMOR, r6
1731
1732
1733 mfspr r8, SPRN_DSCR
1734 ld r7, HSTATE_DSCR(r13)
1735 std r8, VCPU_DSCR(r9)
1736 mtspr SPRN_DSCR, r7
1737
1738
1739 std r14, VCPU_GPR(R14)(r9)
1740 std r15, VCPU_GPR(R15)(r9)
1741 std r16, VCPU_GPR(R16)(r9)
1742 std r17, VCPU_GPR(R17)(r9)
1743 std r18, VCPU_GPR(R18)(r9)
1744 std r19, VCPU_GPR(R19)(r9)
1745 std r20, VCPU_GPR(R20)(r9)
1746 std r21, VCPU_GPR(R21)(r9)
1747 std r22, VCPU_GPR(R22)(r9)
1748 std r23, VCPU_GPR(R23)(r9)
1749 std r24, VCPU_GPR(R24)(r9)
1750 std r25, VCPU_GPR(R25)(r9)
1751 std r26, VCPU_GPR(R26)(r9)
1752 std r27, VCPU_GPR(R27)(r9)
1753 std r28, VCPU_GPR(R28)(r9)
1754 std r29, VCPU_GPR(R29)(r9)
1755 std r30, VCPU_GPR(R30)(r9)
1756 std r31, VCPU_GPR(R31)(r9)
1757
1758
1759 mfspr r3, SPRN_SPRG0
1760 mfspr r4, SPRN_SPRG1
1761 mfspr r5, SPRN_SPRG2
1762 mfspr r6, SPRN_SPRG3
1763 std r3, VCPU_SPRG0(r9)
1764 std r4, VCPU_SPRG1(r9)
1765 std r5, VCPU_SPRG2(r9)
1766 std r6, VCPU_SPRG3(r9)
1767
1768
1769 mr r3, r9
1770 bl kvmppc_save_fp
1771
1772#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1773
1774
1775
1776
1777BEGIN_FTR_SECTION
1778 b 91f
1779END_FTR_SECTION(CPU_FTR_TM | CPU_FTR_P9_TM_HV_ASSIST, 0)
1780
1781
1782
1783 bl kvmppc_save_tm
178491:
1785#endif
1786
1787
1788 ld r8, VCPU_VPA(r9)
1789 cmpdi r8, 0
1790 beq 25f
1791 li r4, LPPACA_YIELDCOUNT
1792 LWZX_BE r3, r8, r4
1793 addi r3, r3, 1
1794 STWX_BE r3, r8, r4
1795 li r3, 1
1796 stb r3, VCPU_VPA_DIRTY(r9)
179725:
1798
1799
1800BEGIN_FTR_SECTION
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818 li r3, -1
1819 clrrdi r3, r3, 10
1820 mfspr r10, SPRN_MMCR2
1821 mtspr SPRN_MMCR2, r3
1822 isync
1823END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
1824 li r3, 1
1825 sldi r3, r3, 31
1826 mfspr r4, SPRN_MMCR0
1827 mtspr SPRN_MMCR0, r3
1828 mfspr r6, SPRN_MMCRA
1829
1830 li r7, 0
1831 mtspr SPRN_MMCRA, r7
1832 isync
1833 beq 21f
1834 lbz r7, LPPACA_PMCINUSE(r8)
1835 cmpwi r7, 0
1836 bne 21f
1837 std r3, VCPU_MMCR(r9)
1838 b 22f
183921: mfspr r5, SPRN_MMCR1
1840 mfspr r7, SPRN_SIAR
1841 mfspr r8, SPRN_SDAR
1842 std r4, VCPU_MMCR(r9)
1843 std r5, VCPU_MMCR + 8(r9)
1844 std r6, VCPU_MMCR + 16(r9)
1845BEGIN_FTR_SECTION
1846 std r10, VCPU_MMCR + 24(r9)
1847END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
1848 std r7, VCPU_SIAR(r9)
1849 std r8, VCPU_SDAR(r9)
1850 mfspr r3, SPRN_PMC1
1851 mfspr r4, SPRN_PMC2
1852 mfspr r5, SPRN_PMC3
1853 mfspr r6, SPRN_PMC4
1854 mfspr r7, SPRN_PMC5
1855 mfspr r8, SPRN_PMC6
1856 stw r3, VCPU_PMC(r9)
1857 stw r4, VCPU_PMC + 4(r9)
1858 stw r5, VCPU_PMC + 8(r9)
1859 stw r6, VCPU_PMC + 12(r9)
1860 stw r7, VCPU_PMC + 16(r9)
1861 stw r8, VCPU_PMC + 20(r9)
1862BEGIN_FTR_SECTION
1863 mfspr r5, SPRN_SIER
1864 std r5, VCPU_SIER(r9)
1865BEGIN_FTR_SECTION_NESTED(96)
1866 mfspr r6, SPRN_SPMC1
1867 mfspr r7, SPRN_SPMC2
1868 mfspr r8, SPRN_MMCRS
1869 stw r6, VCPU_PMC + 24(r9)
1870 stw r7, VCPU_PMC + 28(r9)
1871 std r8, VCPU_MMCR + 32(r9)
1872 lis r4, 0x8000
1873 mtspr SPRN_MMCRS, r4
1874END_FTR_SECTION_NESTED(CPU_FTR_ARCH_300, 0, 96)
1875END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
187622:
1877
1878
1879BEGIN_FTR_SECTION
1880 ld r5, STACK_SLOT_CIABR(r1)
1881 ld r6, STACK_SLOT_DAWR(r1)
1882 ld r7, STACK_SLOT_DAWRX(r1)
1883 mtspr SPRN_CIABR, r5
1884
1885
1886
1887
1888 mtspr SPRN_DAWR, r6
1889 mtspr SPRN_DAWRX, r7
1890END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
1891BEGIN_FTR_SECTION
1892 ld r5, STACK_SLOT_TID(r1)
1893 ld r6, STACK_SLOT_PSSCR(r1)
1894 ld r7, STACK_SLOT_PID(r1)
1895 ld r8, STACK_SLOT_IAMR(r1)
1896 mtspr SPRN_TIDR, r5
1897 mtspr SPRN_PSSCR, r6
1898 mtspr SPRN_PID, r7
1899 mtspr SPRN_IAMR, r8
1900END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
1901
1902#ifdef CONFIG_PPC_RADIX_MMU
1903
1904
1905
1906 ld r5, VCPU_KVM(r9)
1907 lbz r0, KVM_RADIX(r5)
1908 cmpwi cr2, r0, 0
1909 beq cr2, 4f
1910
1911
1912
1913
1914
1915 eieio
1916 tlbsync
1917 ptesync
1918
1919
1920 LOAD_REG_ADDR(r4, mmu_base_pid)
1921 lwz r3, VCPU_GUEST_PID(r9)
1922 lwz r5, 0(r4)
1923 cmpw cr0,r3,r5
1924 blt 2f
1925
1926
1927
1928
1929
1930
1931
1932
1933 li r0,0
1934 mtspr SPRN_LPID,r0
1935 isync
1936
1937
1938 ld r6,VCPU_KVM(r9)
1939 lwz r0,KVM_TLB_SETS(r6)
1940 mtctr r0
1941 li r7,0x400
1942 ptesync
1943 sldi r0,r3,32
19441: PPC_TLBIEL(7,0,2,1,1)
1945 addi r7,r7,0x1000
1946 bdnz 1b
1947 ptesync
1948
19492:
1950BEGIN_FTR_SECTION
1951 PPC_INVALIDATE_ERAT
1952END_FTR_SECTION_IFSET(CPU_FTR_POWER9_DD1)
19534:
1954#endif
1955
1956
1957
1958
1959
1960
1961
1962kvmhv_switch_to_host:
1963
1964 ld r5,HSTATE_KVM_VCORE(r13)
1965 ld r4,VCORE_KVM(r5)
1966 lbz r3,HSTATE_PTID(r13)
1967 cmpwi r3,0
1968 beq 15f
1969 HMT_LOW
197013: lbz r3,VCORE_IN_GUEST(r5)
1971 cmpwi r3,0
1972 bne 13b
1973 HMT_MEDIUM
1974 b 16f
1975
1976
197715: lwz r3,VCORE_ENTRY_EXIT(r5)
1978 rlwinm r0,r3,32-8,0xff
1979 clrldi r3,r3,56
1980 cmpw r3,r0
1981 bne 15b
1982 isync
1983
1984
1985 lbz r6, VCORE_IN_GUEST(r5)
1986 cmpwi r6, 0
1987 beq 19f
1988
1989
1990 lwz r7,KVM_HOST_LPID(r4)
1991BEGIN_FTR_SECTION
1992 ld r6,KVM_HOST_SDR1(r4)
1993 li r8,LPID_RSVD
1994 mtspr SPRN_LPID,r8
1995 ptesync
1996 mtspr SPRN_SDR1,r6
1997END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300)
1998 mtspr SPRN_LPID,r7
1999 isync
2000
2001BEGIN_FTR_SECTION
2002
2003 mfspr r7, SPRN_DPDES
2004 mfspr r8, SPRN_VTB
2005 std r7, VCORE_DPDES(r5)
2006 std r8, VCORE_VTB(r5)
2007
2008 li r8, 0
2009 mtspr SPRN_DPDES, r8
2010END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
2011
2012
2013 lwz r12, STACK_SLOT_TRAP(r1)
2014 cmpwi r12, BOOK3S_INTERRUPT_HMI
2015 bne 27f
2016 bl kvmppc_realmode_hmi_handler
2017 nop
2018 cmpdi r3, 0
2019
2020
2021
2022
2023
2024
2025
2026
2027 beq 30f
2028
202927:
2030
2031 ld r8, VCORE_TB_OFFSET_APPL(r5)
2032 cmpdi r8,0
2033 beq 17f
2034 li r0, 0
2035 std r0, VCORE_TB_OFFSET_APPL(r5)
2036 mftb r6
2037 subf r8,r8,r6
2038 mtspr SPRN_TBU40,r8
2039 mftb r7
2040 clrldi r6,r6,40
2041 clrldi r7,r7,40
2042 cmpld r7,r6
2043 bge 17f
2044 addis r8,r8,0x100
2045 mtspr SPRN_TBU40,r8
2046
204717: bl kvmppc_subcore_exit_guest
2048 nop
204930: ld r5,HSTATE_KVM_VCORE(r13)
2050 ld r4,VCORE_KVM(r5)
2051
2052
2053 ld r0, VCORE_PCR(r5)
2054 cmpdi r0, 0
2055 beq 18f
2056 li r0, 0
2057 mtspr SPRN_PCR, r0
205818:
2059
2060 stb r0,VCORE_IN_GUEST(r5)
206119: lis r8,0x7fff
2062 mtspr SPRN_HDEC,r8
2063
206416:
2065BEGIN_FTR_SECTION
2066
2067 ld r3, HSTATE_SPLIT_MODE(r13)
2068 cmpdi r3, 0
2069 beq 47f
2070 lwz r8, KVM_SPLIT_DO_RESTORE(r3)
2071 cmpwi r8, 0
2072 beq 47f
2073 bl kvmhv_p9_restore_lpcr
2074 nop
2075 b 48f
207647:
2077END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
2078 ld r8,KVM_HOST_LPCR(r4)
2079 mtspr SPRN_LPCR,r8
2080 isync
208148:
2082#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
2083
2084 ld r4, HSTATE_KVM_VCPU(r13)
2085 cmpdi r4, 0
2086 li r3, 0
2087 beq 2f
2088 bl kvmhv_accumulate_time
20892:
2090#endif
2091
2092 li r0, KVM_GUEST_MODE_NONE
2093 stb r0, HSTATE_IN_GUEST(r13)
2094
2095 lwz r12, STACK_SLOT_TRAP(r1)
2096 ld r0, SFS+PPC_LR_STKOFF(r1)
2097 addi r1, r1, SFS
2098 mtlr r0
2099 blr
2100
2101#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
2102
2103
2104
2105
2106
2107kvmppc_tm_emul:
2108
2109 mfspr r3, SPRN_HEIR
2110 stw r3, VCPU_HEIR(r9)
2111
2112
2113
2114
2115
2116
2117 lbz r0, HSTATE_FAKE_SUSPEND(r13)
2118 cmpwi r0, 0
2119 bne guest_exit_cont
2120 rldicl r3, r11, 64 - MSR_TS_S_LG, 62
2121 cmpwi r3, 1
2122 bne guest_exit_cont
2123
2124
2125 mr r3, r9
2126 bl kvmhv_p9_tm_emulation_early
2127 nop
2128 ld r9, HSTATE_KVM_VCPU(r13)
2129 li r12, BOOK3S_INTERRUPT_HV_SOFTPATCH
2130 cmpwi r3, 0
2131 beq guest_exit_cont
2132 ld r10, VCPU_PC(r9)
2133 ld r11, VCPU_MSR(r9)
2134 b fast_interrupt_c_return
2135#endif
2136
2137
2138
2139
2140
2141
2142
2143
2144kvmppc_hdsi:
2145 ld r3, VCPU_KVM(r9)
2146 lbz r0, KVM_RADIX(r3)
2147 mfspr r4, SPRN_HDAR
2148 mfspr r6, SPRN_HDSISR
2149BEGIN_FTR_SECTION
2150
2151 cmpdi r6, 0x7fff
2152 beq 6f
2153END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
2154 cmpwi r0, 0
2155 bne .Lradix_hdsi
2156
2157 andis. r0, r6, (DSISR_NOHPTE | DSISR_PROTFAULT)@h
2158 beq 1f
2159 andi. r0, r11, MSR_DR
2160 beq 3f
2161BEGIN_FTR_SECTION
2162 mfspr r5, SPRN_ASDR
2163 b 4f
2164END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
2165 clrrdi r0, r4, 28
2166 PPC_SLBFEE_DOT(R5, R0)
2167 li r0, BOOK3S_INTERRUPT_DATA_SEGMENT
2168 bne 7f
21694: std r4, VCPU_FAULT_DAR(r9)
2170 stw r6, VCPU_FAULT_DSISR(r9)
2171
2172
2173 mr r3, r9
2174 li r7, 1
2175 bl kvmppc_hpte_hv_fault
2176 ld r9, HSTATE_KVM_VCPU(r13)
2177 ld r10, VCPU_PC(r9)
2178 ld r11, VCPU_MSR(r9)
2179 li r12, BOOK3S_INTERRUPT_H_DATA_STORAGE
2180 cmpdi r3, 0
2181 beq 6f
2182 cmpdi r3, -1
2183 beq guest_exit_cont
2184 cmpdi r3, -2
2185 beq 2f
2186
2187
2188 ld r4, VCPU_FAULT_DAR(r9)
2189 mr r6, r3
21901: li r0, BOOK3S_INTERRUPT_DATA_STORAGE
2191 mtspr SPRN_DSISR, r6
21927: mtspr SPRN_DAR, r4
2193 mtspr SPRN_SRR0, r10
2194 mtspr SPRN_SRR1, r11
2195 mr r10, r0
2196 bl kvmppc_msr_interrupt
2197fast_interrupt_c_return:
21986: ld r7, VCPU_CTR(r9)
2199 ld r8, VCPU_XER(r9)
2200 mtctr r7
2201 mtxer r8
2202 mr r4, r9
2203 b fast_guest_return
2204
22053: ld r5, VCPU_KVM(r9)
2206 ld r5, KVM_VRMA_SLB_V(r5)
2207 b 4b
2208
2209
22102: li r8, KVM_INST_FETCH_FAILED
2211
2212
2213
2214 li r0, KVM_GUEST_MODE_SKIP
2215 stb r0, HSTATE_IN_GUEST(r13)
2216
2217
2218 mfmsr r3
2219 ori r4, r3, MSR_DR
2220 mtmsrd r4
2221 lwz r8, 0(r10)
2222 mtmsrd r3
2223
2224
2225 stw r8, VCPU_LAST_INST(r9)
2226
2227
2228 li r0, KVM_GUEST_MODE_HOST_HV
2229 stb r0, HSTATE_IN_GUEST(r13)
2230 b guest_exit_cont
2231
2232.Lradix_hdsi:
2233 std r4, VCPU_FAULT_DAR(r9)
2234 stw r6, VCPU_FAULT_DSISR(r9)
2235.Lradix_hisi:
2236 mfspr r5, SPRN_ASDR
2237 std r5, VCPU_FAULT_GPA(r9)
2238 b guest_exit_cont
2239
2240
2241
2242
2243
2244kvmppc_hisi:
2245 ld r3, VCPU_KVM(r9)
2246 lbz r0, KVM_RADIX(r3)
2247 cmpwi r0, 0
2248 bne .Lradix_hisi
2249 andis. r0, r11, SRR1_ISI_NOPT@h
2250 beq 1f
2251 andi. r0, r11, MSR_IR
2252 beq 3f
2253BEGIN_FTR_SECTION
2254 mfspr r5, SPRN_ASDR
2255 b 4f
2256END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
2257 clrrdi r0, r10, 28
2258 PPC_SLBFEE_DOT(R5, R0)
2259 li r0, BOOK3S_INTERRUPT_INST_SEGMENT
2260 bne 7f
22614:
2262
2263 mr r3, r9
2264 mr r4, r10
2265 mr r6, r11
2266 li r7, 0
2267 bl kvmppc_hpte_hv_fault
2268 ld r9, HSTATE_KVM_VCPU(r13)
2269 ld r10, VCPU_PC(r9)
2270 ld r11, VCPU_MSR(r9)
2271 li r12, BOOK3S_INTERRUPT_H_INST_STORAGE
2272 cmpdi r3, 0
2273 beq fast_interrupt_c_return
2274 cmpdi r3, -1
2275 beq guest_exit_cont
2276
2277
2278 mr r11, r3
22791: li r0, BOOK3S_INTERRUPT_INST_STORAGE
22807: mtspr SPRN_SRR0, r10
2281 mtspr SPRN_SRR1, r11
2282 mr r10, r0
2283 bl kvmppc_msr_interrupt
2284 b fast_interrupt_c_return
2285
22863: ld r6, VCPU_KVM(r9)
2287 ld r5, KVM_VRMA_SLB_V(r6)
2288 b 4b
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299hcall_try_real_mode:
2300 ld r3,VCPU_GPR(R3)(r9)
2301 andi. r0,r11,MSR_PR
2302
2303 bne sc_1_fast_return
2304 clrrdi r3,r3,2
2305 cmpldi r3,hcall_real_table_end - hcall_real_table
2306 bge guest_exit_cont
2307
2308 ld r4, VCPU_KVM(r9)
2309 srdi r0, r3, 8
2310 sldi r0, r0, 3
2311 add r4, r4, r0
2312 ld r0, KVM_ENABLED_HCALLS(r4)
2313 rlwinm r4, r3, 32-2, 0x3f
2314 srd r0, r0, r4
2315 andi. r0, r0, 1
2316 beq guest_exit_cont
2317
2318 LOAD_REG_ADDR(r4, hcall_real_table)
2319 lwax r3,r3,r4
2320 cmpwi r3,0
2321 beq guest_exit_cont
2322 add r12,r3,r4
2323 mtctr r12
2324 mr r3,r9
2325 ld r4,VCPU_GPR(R4)(r9)
2326 bctrl
2327 cmpdi r3,H_TOO_HARD
2328 beq hcall_real_fallback
2329 ld r4,HSTATE_KVM_VCPU(r13)
2330 std r3,VCPU_GPR(R3)(r4)
2331 ld r10,VCPU_PC(r4)
2332 ld r11,VCPU_MSR(r4)
2333 b fast_guest_return
2334
2335sc_1_fast_return:
2336 mtspr SPRN_SRR0,r10
2337 mtspr SPRN_SRR1,r11
2338 li r10, BOOK3S_INTERRUPT_SYSCALL
2339 bl kvmppc_msr_interrupt
2340 mr r4,r9
2341 b fast_guest_return
2342
2343
2344
2345
2346hcall_real_fallback:
2347 li r12,BOOK3S_INTERRUPT_SYSCALL
2348 ld r9, HSTATE_KVM_VCPU(r13)
2349
2350 b guest_exit_cont
2351
2352 .globl hcall_real_table
2353hcall_real_table:
2354 .long 0
2355 .long DOTSYM(kvmppc_h_remove) - hcall_real_table
2356 .long DOTSYM(kvmppc_h_enter) - hcall_real_table
2357 .long DOTSYM(kvmppc_h_read) - hcall_real_table
2358 .long DOTSYM(kvmppc_h_clear_mod) - hcall_real_table
2359 .long DOTSYM(kvmppc_h_clear_ref) - hcall_real_table
2360 .long DOTSYM(kvmppc_h_protect) - hcall_real_table
2361 .long DOTSYM(kvmppc_h_get_tce) - hcall_real_table
2362 .long DOTSYM(kvmppc_rm_h_put_tce) - hcall_real_table
2363 .long 0
2364 .long DOTSYM(kvmppc_h_set_dabr) - hcall_real_table
2365 .long 0
2366 .long 0
2367 .long 0
2368 .long 0
2369 .long 0
2370 .long 0
2371 .long 0
2372 .long 0
2373 .long 0
2374 .long 0
2375 .long 0
2376 .long 0
2377 .long 0
2378 .long 0
2379#ifdef CONFIG_KVM_XICS
2380 .long DOTSYM(kvmppc_rm_h_eoi) - hcall_real_table
2381 .long DOTSYM(kvmppc_rm_h_cppr) - hcall_real_table
2382 .long DOTSYM(kvmppc_rm_h_ipi) - hcall_real_table
2383 .long DOTSYM(kvmppc_rm_h_ipoll) - hcall_real_table
2384 .long DOTSYM(kvmppc_rm_h_xirr) - hcall_real_table
2385#else
2386 .long 0
2387 .long 0
2388 .long 0
2389 .long 0
2390 .long 0
2391#endif
2392 .long 0
2393 .long 0
2394 .long 0
2395 .long 0
2396 .long 0
2397 .long 0
2398 .long 0
2399 .long 0
2400 .long 0
2401 .long 0
2402 .long 0
2403 .long 0
2404 .long 0
2405 .long 0
2406 .long 0
2407 .long 0
2408 .long 0
2409 .long 0
2410 .long 0
2411 .long 0
2412 .long 0
2413 .long 0
2414 .long 0
2415 .long 0
2416 .long 0
2417 .long 0
2418 .long DOTSYM(kvmppc_h_cede) - hcall_real_table
2419 .long DOTSYM(kvmppc_rm_h_confer) - hcall_real_table
2420 .long 0
2421 .long 0
2422 .long 0
2423 .long 0
2424 .long 0
2425 .long 0
2426 .long 0
2427 .long 0
2428 .long 0
2429 .long 0
2430 .long 0
2431 .long 0
2432 .long 0
2433 .long 0
2434 .long 0
2435 .long DOTSYM(kvmppc_h_bulk_remove) - hcall_real_table
2436 .long 0
2437 .long 0
2438 .long 0
2439 .long DOTSYM(kvmppc_h_set_xdabr) - hcall_real_table
2440 .long DOTSYM(kvmppc_rm_h_stuff_tce) - hcall_real_table
2441 .long DOTSYM(kvmppc_rm_h_put_tce_indirect) - hcall_real_table
2442 .long 0
2443 .long 0
2444 .long 0
2445 .long 0
2446 .long 0
2447 .long 0
2448 .long 0
2449 .long 0
2450 .long 0
2451 .long 0
2452 .long 0
2453 .long 0
2454 .long 0
2455 .long 0
2456 .long 0
2457 .long 0
2458 .long 0
2459 .long 0
2460 .long 0
2461 .long 0
2462 .long 0
2463 .long 0
2464 .long 0
2465 .long 0
2466 .long 0
2467 .long 0
2468 .long 0
2469 .long 0
2470 .long 0
2471 .long 0
2472 .long 0
2473 .long 0
2474 .long 0
2475 .long 0
2476 .long 0
2477 .long 0
2478 .long 0
2479 .long 0
2480 .long 0
2481 .long 0
2482 .long 0
2483 .long 0
2484 .long 0
2485 .long 0
2486 .long 0
2487 .long 0
2488 .long 0
2489 .long 0
2490 .long 0
2491 .long 0
2492 .long 0
2493 .long 0
2494 .long 0
2495 .long 0
2496 .long 0
2497 .long 0
2498 .long 0
2499 .long 0
2500 .long 0
2501 .long 0
2502 .long 0
2503 .long 0
2504 .long 0
2505 .long 0
2506 .long 0
2507 .long 0
2508 .long 0
2509 .long 0
2510 .long 0
2511 .long 0
2512 .long 0
2513 .long 0
2514 .long 0
2515 .long 0
2516 .long 0
2517 .long 0
2518 .long 0
2519 .long 0
2520 .long 0
2521 .long 0
2522 .long 0
2523 .long 0
2524 .long 0
2525 .long 0
2526 .long 0
2527 .long 0
2528 .long 0
2529 .long 0
2530 .long 0
2531 .long 0
2532 .long 0
2533 .long 0
2534 .long 0
2535 .long 0
2536 .long 0
2537 .long 0
2538 .long 0
2539 .long 0
2540 .long 0
2541 .long 0
2542 .long 0
2543 .long 0
2544 .long 0
2545 .long 0
2546 .long 0
2547 .long 0
2548 .long 0
2549 .long 0
2550 .long 0
2551 .long 0
2552 .long 0
2553#ifdef CONFIG_KVM_XICS
2554 .long DOTSYM(kvmppc_rm_h_xirr_x) - hcall_real_table
2555#else
2556 .long 0
2557#endif
2558 .long DOTSYM(kvmppc_h_random) - hcall_real_table
2559 .globl hcall_real_table_end
2560hcall_real_table_end:
2561
2562_GLOBAL(kvmppc_h_set_xdabr)
2563 andi. r0, r5, DABRX_USER | DABRX_KERNEL
2564 beq 6f
2565 li r0, DABRX_USER | DABRX_KERNEL | DABRX_BTI
2566 andc. r0, r5, r0
2567 beq 3f
25686: li r3, H_PARAMETER
2569 blr
2570
2571_GLOBAL(kvmppc_h_set_dabr)
2572 li r5, DABRX_USER | DABRX_KERNEL
25733:
2574BEGIN_FTR_SECTION
2575 b 2f
2576END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
2577 std r4,VCPU_DABR(r3)
2578 stw r5, VCPU_DABRX(r3)
2579 mtspr SPRN_DABRX, r5
2580
25811: mtspr SPRN_DABR,r4
2582 mfspr r5, SPRN_DABR
2583 cmpd r4, r5
2584 bne 1b
2585 isync
2586 li r3,0
2587 blr
2588
25892:
2590BEGIN_FTR_SECTION
2591
2592 li r3, H_HARDWARE
2593 blr
2594END_FTR_SECTION_IFCLR(CPU_FTR_DAWR)
2595
2596 rlwimi r5, r4, 5, DAWRX_DR | DAWRX_DW
2597 rlwimi r5, r4, 2, DAWRX_WT
2598 clrrdi r4, r4, 3
2599 std r4, VCPU_DAWR(r3)
2600 std r5, VCPU_DAWRX(r3)
2601 mtspr SPRN_DAWR, r4
2602 mtspr SPRN_DAWRX, r5
2603 li r3, 0
2604 blr
2605
2606_GLOBAL(kvmppc_h_cede)
2607 ori r11,r11,MSR_EE
2608 std r11,VCPU_MSR(r3)
2609 li r0,1
2610 stb r0,VCPU_CEDED(r3)
2611 sync
2612 lbz r5,VCPU_PRODDED(r3)
2613 cmpwi r5,0
2614 bne kvm_cede_prodded
2615 li r12,0
2616 stw r12,VCPU_TRAP(r3)
2617 li r0,H_SUCCESS
2618 std r0,VCPU_GPR(R3)(r3)
2619
2620
2621
2622
2623
2624
2625 ld r5,HSTATE_KVM_VCORE(r13)
2626 lbz r6,HSTATE_PTID(r13)
2627 lwz r8,VCORE_ENTRY_EXIT(r5)
2628 clrldi r8,r8,56
2629 li r0,1
2630 sld r0,r0,r6
2631 addi r6,r5,VCORE_NAPPING_THREADS
263231: lwarx r4,0,r6
2633 or r4,r4,r0
2634 cmpw r4,r8
2635 beq kvm_cede_exit
2636 stwcx. r4,0,r6
2637 bne 31b
2638
2639 isync
2640 li r0,NAPPING_CEDE
2641 stb r0,HSTATE_NAPPING(r13)
2642 lwz r7,VCORE_ENTRY_EXIT(r5)
2643 cmpwi r7,0x100
2644 bge 33f
2645
2646
2647
2648
2649
2650
2651
2652
2653 std r14, VCPU_GPR(R14)(r3)
2654 std r15, VCPU_GPR(R15)(r3)
2655 std r16, VCPU_GPR(R16)(r3)
2656 std r17, VCPU_GPR(R17)(r3)
2657 std r18, VCPU_GPR(R18)(r3)
2658 std r19, VCPU_GPR(R19)(r3)
2659 std r20, VCPU_GPR(R20)(r3)
2660 std r21, VCPU_GPR(R21)(r3)
2661 std r22, VCPU_GPR(R22)(r3)
2662 std r23, VCPU_GPR(R23)(r3)
2663 std r24, VCPU_GPR(R24)(r3)
2664 std r25, VCPU_GPR(R25)(r3)
2665 std r26, VCPU_GPR(R26)(r3)
2666 std r27, VCPU_GPR(R27)(r3)
2667 std r28, VCPU_GPR(R28)(r3)
2668 std r29, VCPU_GPR(R29)(r3)
2669 std r30, VCPU_GPR(R30)(r3)
2670 std r31, VCPU_GPR(R31)(r3)
2671
2672
2673 bl kvmppc_save_fp
2674
2675#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
2676
2677
2678
2679
2680BEGIN_FTR_SECTION
2681 b 91f
2682END_FTR_SECTION(CPU_FTR_TM | CPU_FTR_P9_TM_HV_ASSIST, 0)
2683
2684
2685
2686 ld r9, HSTATE_KVM_VCPU(r13)
2687 bl kvmppc_save_tm
268891:
2689#endif
2690
2691
2692
2693
2694
2695
2696 mfspr r3, SPRN_DEC
2697 mfspr r4, SPRN_HDEC
2698 mftb r5
2699BEGIN_FTR_SECTION
2700
2701 ld r6, HSTATE_KVM_VCORE(r13)
2702 ld r6, VCORE_LPCR(r6)
2703 andis. r6, r6, LPCR_LD@h
2704 bne 68f
2705END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
2706 extsw r3, r3
270768: EXTEND_HDEC(r4)
2708 cmpd r3, r4
2709 ble 67f
2710 mtspr SPRN_DEC, r4
271167:
2712
2713 add r3, r3, r5
2714 ld r4, HSTATE_KVM_VCPU(r13)
2715 ld r5, HSTATE_KVM_VCORE(r13)
2716 ld r6, VCORE_TB_OFFSET_APPL(r5)
2717 subf r3, r6, r3
2718 std r3, VCPU_DEC_EXPIRES(r4)
2719
2720#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
2721 ld r4, HSTATE_KVM_VCPU(r13)
2722 addi r3, r4, VCPU_TB_CEDE
2723 bl kvmhv_accumulate_time
2724#endif
2725
2726 lis r3, LPCR_PECEDP@h
2727
2728
2729
2730
2731
2732
2733
2734kvm_do_nap:
2735 mfspr r0, SPRN_CTRLF
2736 clrrdi r0, r0, 1
2737 mtspr SPRN_CTRLT, r0
2738
2739 li r0,1
2740 stb r0,HSTATE_HWTHREAD_REQ(r13)
2741 mfspr r5,SPRN_LPCR
2742 ori r5,r5,LPCR_PECE0 | LPCR_PECE1
2743BEGIN_FTR_SECTION
2744 ori r5, r5, LPCR_PECEDH
2745 rlwimi r5, r3, 0, LPCR_PECEDP
2746END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
2747
2748kvm_nap_sequence:
2749BEGIN_FTR_SECTION
2750
2751
2752
2753
2754
2755 lis r3, (PSSCR_EC | PSSCR_ESL)@h
2756 mtspr SPRN_PSSCR, r3
2757
2758 li r4, LPCR_PECE_HVEE@higher
2759 sldi r4, r4, 32
2760 or r5, r5, r4
2761END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
2762 mtspr SPRN_LPCR,r5
2763 isync
2764 li r0, 0
2765 std r0, HSTATE_SCRATCH0(r13)
2766 ptesync
2767 ld r0, HSTATE_SCRATCH0(r13)
27681: cmpd r0, r0
2769 bne 1b
2770BEGIN_FTR_SECTION
2771 nap
2772FTR_SECTION_ELSE
2773 PPC_STOP
2774ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_300)
2775 b .
2776
277733: mr r4, r3
2778 li r3, 0
2779 li r12, 0
2780 b 34f
2781
2782kvm_end_cede:
2783
2784 ld r4, HSTATE_KVM_VCPU(r13)
2785
2786
2787 ld r1, HSTATE_HOST_R1(r13)
2788
2789#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
2790 addi r3, r4, VCPU_TB_RMINTR
2791 bl kvmhv_accumulate_time
2792#endif
2793
2794#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
2795
2796
2797
2798
2799BEGIN_FTR_SECTION
2800 b 91f
2801END_FTR_SECTION(CPU_FTR_TM | CPU_FTR_P9_TM_HV_ASSIST, 0)
2802
2803
2804
2805 bl kvmppc_restore_tm
280691:
2807#endif
2808
2809
2810 bl kvmppc_load_fp
2811
2812
2813 ld r3, VCPU_DEC_EXPIRES(r4)
2814 ld r5, HSTATE_KVM_VCORE(r13)
2815 ld r6, VCORE_TB_OFFSET_APPL(r5)
2816 add r3, r3, r6
2817 mftb r7
2818 subf r3, r7, r3
2819 mtspr SPRN_DEC, r3
2820
2821
2822 ld r14, VCPU_GPR(R14)(r4)
2823 ld r15, VCPU_GPR(R15)(r4)
2824 ld r16, VCPU_GPR(R16)(r4)
2825 ld r17, VCPU_GPR(R17)(r4)
2826 ld r18, VCPU_GPR(R18)(r4)
2827 ld r19, VCPU_GPR(R19)(r4)
2828 ld r20, VCPU_GPR(R20)(r4)
2829 ld r21, VCPU_GPR(R21)(r4)
2830 ld r22, VCPU_GPR(R22)(r4)
2831 ld r23, VCPU_GPR(R23)(r4)
2832 ld r24, VCPU_GPR(R24)(r4)
2833 ld r25, VCPU_GPR(R25)(r4)
2834 ld r26, VCPU_GPR(R26)(r4)
2835 ld r27, VCPU_GPR(R27)(r4)
2836 ld r28, VCPU_GPR(R28)(r4)
2837 ld r29, VCPU_GPR(R29)(r4)
2838 ld r30, VCPU_GPR(R30)(r4)
2839 ld r31, VCPU_GPR(R31)(r4)
2840
2841
2842 bl kvmppc_check_wake_reason
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852 ld r4, HSTATE_KVM_VCPU(r13)
2853
2854
285534: ld r5,HSTATE_KVM_VCORE(r13)
2856 lbz r7,HSTATE_PTID(r13)
2857 li r0,1
2858 sld r0,r0,r7
2859 addi r6,r5,VCORE_NAPPING_THREADS
286032: lwarx r7,0,r6
2861 andc r7,r7,r0
2862 stwcx. r7,0,r6
2863 bne 32b
2864 li r0,0
2865 stb r0,HSTATE_NAPPING(r13)
2866
2867
2868 stw r12, VCPU_TRAP(r4)
2869 mr r9, r4
2870 cmpdi r3, 0
2871 bgt guest_exit_cont
2872
2873
2874 lwz r0,VCORE_ENTRY_EXIT(r5)
2875 cmpwi r0,0x100
2876 bge guest_exit_cont
2877
2878 b kvmppc_cede_reentry
2879
2880
2881kvm_cede_prodded:
2882 li r0,0
2883 stb r0,VCPU_PRODDED(r3)
2884 sync
2885 stb r0,VCPU_CEDED(r3)
2886 li r3,H_SUCCESS
2887 blr
2888
2889
2890kvm_cede_exit:
2891 ld r9, HSTATE_KVM_VCPU(r13)
2892#ifdef CONFIG_KVM_XICS
2893
2894 lbz r5, VCPU_XIVE_ESC_ON(r9)
2895 cmpwi r5, 0
2896 beq 1f
2897 li r0, 0
2898 stb r0, VCPU_CEDED(r9)
28991:
2900 li r5, XIVE_ESB_SET_PQ_00
2901 mfmsr r0
2902 andi. r0, r0, MSR_DR
2903 beq 1f
2904 ld r10, VCPU_XIVE_ESC_VADDR(r9)
2905 cmpdi r10, 0
2906 beq 3f
2907 ldx r0, r10, r5
2908 b 2f
29091: ld r10, VCPU_XIVE_ESC_RADDR(r9)
2910 cmpdi r10, 0
2911 beq 3f
2912 ldcix r0, r10, r5
29132: sync
2914 li r0, 1
2915 stb r0, VCPU_XIVE_ESC_ON(r9)
2916#endif
29173: b guest_exit_cont
2918
2919
2920machine_check_realmode:
2921 mr r3, r9
2922 bl kvmppc_realmode_machine_check
2923 nop
2924 ld r9, HSTATE_KVM_VCPU(r13)
2925 li r12, BOOK3S_INTERRUPT_MACHINE_CHECK
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943 ld r11, VCPU_MSR(r9)
2944 rldicl. r0, r11, 64-MSR_HV_LG, 63
2945 bne mc_cont
2946
2947 ld r10, VCPU_KVM(r9)
2948 lbz r10, KVM_FWNMI(r10)
2949 cmpdi r10, 1
2950 beq mc_cont
2951
2952
2953 andi. r10, r11, MSR_RI
2954 beq 1f
2955 ld r10, VCPU_PC(r9)
2956 cmpdi r3, 0
2957 bne 2f
2958
29591: li r10, BOOK3S_INTERRUPT_MACHINE_CHECK
2960 bl kvmppc_msr_interrupt
29612: b fast_interrupt_c_return
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978kvmppc_check_wake_reason:
2979 mfspr r6, SPRN_SRR1
2980BEGIN_FTR_SECTION
2981 rlwinm r6, r6, 45-31, 0xf
2982FTR_SECTION_ELSE
2983 rlwinm r6, r6, 45-31, 0xe
2984ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_207S)
2985 cmpwi r6, 8
2986 beq 7f
2987 li r3, 0
2988 li r12, 0
2989 cmpwi r6, 6
2990 beq 0f
2991BEGIN_FTR_SECTION
2992 cmpwi r6, 5
2993 beq 0f
2994 cmpwi r6, 3
2995 beq 3f
2996END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
2997 cmpwi r6, 0xa
2998 beq 4f
2999 li r3, 1
30000: blr
3001
3002
30033: li r12, BOOK3S_INTERRUPT_H_DOORBELL
3004
3005
3006
3007
3008
3009 lis r6, (PPC_DBELL_SERVER << (63-36))@h
3010 PPC_MSGCLR(6)
3011
3012 li r3, 1
3013BEGIN_FTR_SECTION
3014 PPC_MSGSYNC
3015 lwsync
3016END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
3017 lbz r0, HSTATE_HOST_IPI(r13)
3018 cmpwi r0, 0
3019 bnelr
3020
3021 li r3, -1
3022 blr
3023
3024
30254: li r12, BOOK3S_INTERRUPT_HMI
3026 li r3, 1
3027 blr
3028
3029
30307: mflr r0
3031 std r0, PPC_LR_STKOFF(r1)
3032 stdu r1, -PPC_MIN_STKFRM(r1)
3033 bl kvmppc_read_intr
3034 nop
3035 li r12, BOOK3S_INTERRUPT_EXTERNAL
3036 cmpdi r3, 1
3037 ble 1f
3038
3039
3040
3041
3042
3043
3044
3045 li r12, BOOK3S_INTERRUPT_HV_RM_HARD
30461:
3047 ld r0, PPC_MIN_STKFRM+PPC_LR_STKOFF(r1)
3048 addi r1, r1, PPC_MIN_STKFRM
3049 mtlr r0
3050 blr
3051
3052
3053
3054
3055
3056
3057
3058kvmppc_save_fp:
3059 mflr r30
3060 mr r31,r3
3061 mfmsr r5
3062 ori r8,r5,MSR_FP
3063#ifdef CONFIG_ALTIVEC
3064BEGIN_FTR_SECTION
3065 oris r8,r8,MSR_VEC@h
3066END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
3067#endif
3068#ifdef CONFIG_VSX
3069BEGIN_FTR_SECTION
3070 oris r8,r8,MSR_VSX@h
3071END_FTR_SECTION_IFSET(CPU_FTR_VSX)
3072#endif
3073 mtmsrd r8
3074 addi r3,r3,VCPU_FPRS
3075 bl store_fp_state
3076#ifdef CONFIG_ALTIVEC
3077BEGIN_FTR_SECTION
3078 addi r3,r31,VCPU_VRS
3079 bl store_vr_state
3080END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
3081#endif
3082 mfspr r6,SPRN_VRSAVE
3083 stw r6,VCPU_VRSAVE(r31)
3084 mtlr r30
3085 blr
3086
3087
3088
3089
3090
3091
3092
3093kvmppc_load_fp:
3094 mflr r30
3095 mr r31,r4
3096 mfmsr r9
3097 ori r8,r9,MSR_FP
3098#ifdef CONFIG_ALTIVEC
3099BEGIN_FTR_SECTION
3100 oris r8,r8,MSR_VEC@h
3101END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
3102#endif
3103#ifdef CONFIG_VSX
3104BEGIN_FTR_SECTION
3105 oris r8,r8,MSR_VSX@h
3106END_FTR_SECTION_IFSET(CPU_FTR_VSX)
3107#endif
3108 mtmsrd r8
3109 addi r3,r4,VCPU_FPRS
3110 bl load_fp_state
3111#ifdef CONFIG_ALTIVEC
3112BEGIN_FTR_SECTION
3113 addi r3,r31,VCPU_VRS
3114 bl load_vr_state
3115END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
3116#endif
3117 lwz r7,VCPU_VRSAVE(r31)
3118 mtspr SPRN_VRSAVE,r7
3119 mtlr r30
3120 mr r4,r31
3121 blr
3122
3123#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
3124
3125
3126
3127
3128
3129
3130kvmppc_save_tm:
3131 mflr r0
3132 std r0, PPC_LR_STKOFF(r1)
3133 stdu r1, -PPC_MIN_STKFRM(r1)
3134
3135
3136 mfmsr r8
3137 li r0, 1
3138 rldimi r8, r0, MSR_TM_LG, 63-MSR_TM_LG
3139 mtmsrd r8
3140
3141 ld r5, VCPU_MSR(r9)
3142 rldicl. r5, r5, 64 - MSR_TS_S_LG, 62
3143 beq 1f
3144
3145 std r1, HSTATE_HOST_R1(r13)
3146 li r3, TM_CAUSE_KVM_RESCHED
3147
3148BEGIN_FTR_SECTION
3149 lbz r0, HSTATE_FAKE_SUSPEND(r13)
3150 cmpwi r0, 0
3151 beq 3f
3152 rldicl. r8, r8, 64 - MSR_TS_S_LG, 62
3153 beq 4f
3154BEGIN_FTR_SECTION_NESTED(96)
3155 bl pnv_power9_force_smt4_catch
3156END_FTR_SECTION_NESTED(CPU_FTR_P9_TM_XER_SO_BUG, CPU_FTR_P9_TM_XER_SO_BUG, 96)
3157 nop
3158 b 6f
31593:
3160
3161 mfspr r6, SPRN_TEXASR
3162 std r6, VCPU_ORIG_TEXASR(r9)
31636:
3164END_FTR_SECTION_IFSET(CPU_FTR_P9_TM_HV_ASSIST)
3165
3166
3167 li r5, 0
3168 mtmsrd r5, 1
3169
3170
3171 TRECLAIM(R3)
3172
3173
3174 SET_SCRATCH0(r13)
3175 GET_PACA(r13)
3176 std r9, PACATMSCRATCH(r13)
3177
3178
3179BEGIN_FTR_SECTION
3180 lbz r9, HSTATE_FAKE_SUSPEND(r13)
3181 cmpwi r9, 0
3182 beq 2f
3183
3184
3185
3186
3187
3188
3189 ld r1, HSTATE_HOST_R1(r13)
3190 ld r2, PACATOC(r13)
3191
3192 li r5, MSR_RI
3193 mtmsrd r5, 1
3194 HMT_MEDIUM
3195 ld r6, HSTATE_DSCR(r13)
3196 mtspr SPRN_DSCR, r6
3197BEGIN_FTR_SECTION_NESTED(96)
3198 bl pnv_power9_force_smt4_release
3199END_FTR_SECTION_NESTED(CPU_FTR_P9_TM_XER_SO_BUG, CPU_FTR_P9_TM_XER_SO_BUG, 96)
3200 nop
3201
32024:
3203 mfspr r3, SPRN_PSSCR
3204
3205 li r0, PSSCR_FAKE_SUSPEND
3206 andc r3, r3, r0
3207 mtspr SPRN_PSSCR, r3
3208 ld r9, HSTATE_KVM_VCPU(r13)
3209
3210 b 11f
32112:
3212END_FTR_SECTION_IFSET(CPU_FTR_P9_TM_HV_ASSIST)
3213
3214 ld r9, HSTATE_KVM_VCPU(r13)
3215
3216
3217 std r29, VCPU_GPRS_TM(29)(r9)
3218 std r30, VCPU_GPRS_TM(30)(r9)
3219 std r31, VCPU_GPRS_TM(31)(r9)
3220
3221
3222 mfspr r31, SPRN_PPR
3223 HMT_MEDIUM
3224 mfspr r30, SPRN_DSCR
3225 ld r29, HSTATE_DSCR(r13)
3226 mtspr SPRN_DSCR, r29
3227
3228
3229 reg = 0
3230 .rept 29
3231 .if (reg != 9) && (reg != 13)
3232 std reg, VCPU_GPRS_TM(reg)(r9)
3233 .endif
3234 reg = reg + 1
3235 .endr
3236
3237 GET_SCRATCH0(r4)
3238 std r4, VCPU_GPRS_TM(13)(r9)
3239
3240 ld r4, PACATMSCRATCH(r13)
3241 std r4, VCPU_GPRS_TM(9)(r9)
3242
3243
3244 ld r1, HSTATE_HOST_R1(r13)
3245 ld r2, PACATOC(r13)
3246
3247
3248 li r5, MSR_RI
3249 mtmsrd r5, 1
3250
3251
3252 std r31, VCPU_PPR_TM(r9)
3253 std r30, VCPU_DSCR_TM(r9)
3254 mflr r5
3255 mfcr r6
3256 mfctr r7
3257 mfspr r8, SPRN_AMR
3258 mfspr r10, SPRN_TAR
3259 mfxer r11
3260 std r5, VCPU_LR_TM(r9)
3261 stw r6, VCPU_CR_TM(r9)
3262 std r7, VCPU_CTR_TM(r9)
3263 std r8, VCPU_AMR_TM(r9)
3264 std r10, VCPU_TAR_TM(r9)
3265 std r11, VCPU_XER_TM(r9)
3266
3267
3268 lwz r12, VCPU_TRAP(r9)
3269
3270
3271 addi r3, r9, VCPU_FPRS_TM
3272 bl store_fp_state
3273 addi r3, r9, VCPU_VRS_TM
3274 bl store_vr_state
3275 mfspr r6, SPRN_VRSAVE
3276 stw r6, VCPU_VRSAVE_TM(r9)
32771:
3278
3279
3280
3281
3282
3283
3284 mfspr r7, SPRN_TEXASR
3285 std r7, VCPU_TEXASR(r9)
328611:
3287 mfspr r5, SPRN_TFHAR
3288 mfspr r6, SPRN_TFIAR
3289 std r5, VCPU_TFHAR(r9)
3290 std r6, VCPU_TFIAR(r9)
3291
3292 addi r1, r1, PPC_MIN_STKFRM
3293 ld r0, PPC_LR_STKOFF(r1)
3294 mtlr r0
3295 blr
3296
3297
3298
3299
3300
3301
3302
3303kvmppc_restore_tm:
3304 mflr r0
3305 std r0, PPC_LR_STKOFF(r1)
3306
3307
3308 mfmsr r5
3309 li r6, MSR_TM >> 32
3310 sldi r6, r6, 32
3311 or r5, r5, r6
3312 ori r5, r5, MSR_FP
3313 oris r5, r5, (MSR_VEC | MSR_VSX)@h
3314 mtmsrd r5
3315
3316
3317
3318
3319
3320 ld r5, VCPU_TFHAR(r4)
3321 ld r6, VCPU_TFIAR(r4)
3322 ld r7, VCPU_TEXASR(r4)
3323 mtspr SPRN_TFHAR, r5
3324 mtspr SPRN_TFIAR, r6
3325 mtspr SPRN_TEXASR, r7
3326
3327 li r0, 0
3328 stb r0, HSTATE_FAKE_SUSPEND(r13)
3329 ld r5, VCPU_MSR(r4)
3330 rldicl. r5, r5, 64 - MSR_TS_S_LG, 62
3331 beqlr
3332 std r1, HSTATE_HOST_R1(r13)
3333
3334
3335
3336
3337
3338
3339 oris r7, r7, (TEXASR_FS)@h
3340 mtspr SPRN_TEXASR, r7
3341
3342
3343
3344
3345
3346
3347BEGIN_FTR_SECTION
3348 b .Ldo_tm_fake_load
3349END_FTR_SECTION_IFSET(CPU_FTR_P9_TM_HV_ASSIST)
3350
3351
3352
3353
3354
3355
3356
3357 mr r31, r4
3358 addi r3, r31, VCPU_FPRS_TM
3359 bl load_fp_state
3360 addi r3, r31, VCPU_VRS_TM
3361 bl load_vr_state
3362 mr r4, r31
3363 lwz r7, VCPU_VRSAVE_TM(r4)
3364 mtspr SPRN_VRSAVE, r7
3365
3366 ld r5, VCPU_LR_TM(r4)
3367 lwz r6, VCPU_CR_TM(r4)
3368 ld r7, VCPU_CTR_TM(r4)
3369 ld r8, VCPU_AMR_TM(r4)
3370 ld r9, VCPU_TAR_TM(r4)
3371 ld r10, VCPU_XER_TM(r4)
3372 mtlr r5
3373 mtcr r6
3374 mtctr r7
3375 mtspr SPRN_AMR, r8
3376 mtspr SPRN_TAR, r9
3377 mtxer r10
3378
3379
3380
3381
3382
3383
3384 ld r29, VCPU_DSCR_TM(r4)
3385 ld r30, VCPU_PPR_TM(r4)
3386
3387 std r2, PACATMSCRATCH(r13)
3388
3389
3390 li r5, 0
3391 mtmsrd r5, 1
3392
3393
3394 reg = 0
3395 .rept 29
3396 ld reg, VCPU_GPRS_TM(reg)(r31)
3397 reg = reg + 1
3398 .endr
3399
3400 mtspr SPRN_DSCR, r29
3401 mtspr SPRN_PPR, r30
3402
3403
3404 ld 29, VCPU_GPRS_TM(29)(r31)
3405 ld 30, VCPU_GPRS_TM(30)(r31)
3406 ld 31, VCPU_GPRS_TM(31)(r31)
3407
3408
3409 TRECHKPT
3410
3411
3412 HMT_MEDIUM
3413 GET_PACA(r13)
3414 ld r29, HSTATE_DSCR(r13)
3415 mtspr SPRN_DSCR, r29
3416 ld r4, HSTATE_KVM_VCPU(r13)
3417 ld r1, HSTATE_HOST_R1(r13)
3418 ld r2, PACATMSCRATCH(r13)
3419
3420
3421 li r5, MSR_RI
3422 mtmsrd r5, 1
34239:
3424 ld r0, PPC_LR_STKOFF(r1)
3425 mtlr r0
3426 blr
3427
3428.Ldo_tm_fake_load:
3429 cmpwi r5, 1
3430 bgt 10f
3431 stb r5, HSTATE_FAKE_SUSPEND(r13)
3432 b 9b
343310: stdu r1, -PPC_MIN_STKFRM(r1)
3434
3435 mr r3, r4
3436 bl kvmhv_emulate_tm_rollback
3437 nop
3438 ld r4, HSTATE_KVM_VCPU(r13)
3439 addi r1, r1, PPC_MIN_STKFRM
3440 b 9b
3441#endif
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
3452
3453
3454
3455kvmppc_bad_host_intr:
3456
3457
3458
3459
3460 mr r9, r1
3461 std r1, PACAR1(r13)
3462 ld r1, PACAEMERGSP(r13)
3463 subi r1, r1, THREAD_SIZE/2 + INT_FRAME_SIZE
3464 std r9, 0(r1)
3465 std r0, GPR0(r1)
3466 std r9, GPR1(r1)
3467 std r2, GPR2(r1)
3468 SAVE_4GPRS(3, r1)
3469 SAVE_2GPRS(7, r1)
3470 srdi r0, r12, 32
3471 clrldi r12, r12, 32
3472 std r0, _CCR(r1)
3473 std r12, _TRAP(r1)
3474 andi. r0, r12, 2
3475 beq 1f
3476 mfspr r3, SPRN_HSRR0
3477 mfspr r4, SPRN_HSRR1
3478 mfspr r5, SPRN_HDAR
3479 mfspr r6, SPRN_HDSISR
3480 b 2f
34811: mfspr r3, SPRN_SRR0
3482 mfspr r4, SPRN_SRR1
3483 mfspr r5, SPRN_DAR
3484 mfspr r6, SPRN_DSISR
34852: std r3, _NIP(r1)
3486 std r4, _MSR(r1)
3487 std r5, _DAR(r1)
3488 std r6, _DSISR(r1)
3489 ld r9, HSTATE_SCRATCH2(r13)
3490 ld r12, HSTATE_SCRATCH0(r13)
3491 GET_SCRATCH0(r0)
3492 SAVE_4GPRS(9, r1)
3493 std r0, GPR13(r1)
3494 SAVE_NVGPRS(r1)
3495 ld r5, HSTATE_CFAR(r13)
3496 std r5, ORIG_GPR3(r1)
3497 mflr r3
3498#ifdef CONFIG_RELOCATABLE
3499 ld r4, HSTATE_SCRATCH1(r13)
3500#else
3501 mfctr r4
3502#endif
3503 mfxer r5
3504 lbz r6, PACAIRQSOFTMASK(r13)
3505 std r3, _LINK(r1)
3506 std r4, _CTR(r1)
3507 std r5, _XER(r1)
3508 std r6, SOFTE(r1)
3509 ld r2, PACATOC(r13)
3510 LOAD_REG_IMMEDIATE(3, 0x7265677368657265)
3511 std r3, STACK_FRAME_OVERHEAD-16(r1)
3512
3513
3514
3515
3516
3517
3518
3519
3520
3521BEGIN_FTR_SECTION
3522 b .
3523END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300)
3524 ld r9, HSTATE_KVM_VCPU(r13)
3525 ld r10, VCPU_KVM(r9)
3526
3527 li r0, 0
3528 mtspr SPRN_AMR, r0
3529 mtspr SPRN_IAMR, r0
3530 mtspr SPRN_CIABR, r0
3531 mtspr SPRN_DAWRX, r0
3532
3533
3534BEGIN_FTR_SECTION
3535 PPC_INVALIDATE_ERAT
3536END_FTR_SECTION_IFSET(CPU_FTR_POWER9_DD1)
3537
3538BEGIN_MMU_FTR_SECTION
3539 b 4f
3540END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_RADIX)
3541
3542 slbmte r0, r0
3543 slbia
3544 ptesync
3545 ld r8, PACA_SLBSHADOWPTR(r13)
3546 .rept SLB_NUM_BOLTED
3547 li r3, SLBSHADOW_SAVEAREA
3548 LDX_BE r5, r8, r3
3549 addi r3, r3, 8
3550 LDX_BE r6, r8, r3
3551 andis. r7, r5, SLB_ESID_V@h
3552 beq 3f
3553 slbmte r6, r5
35543: addi r8, r8, 16
3555 .endr
3556
35574: lwz r7, KVM_HOST_LPID(r10)
3558 mtspr SPRN_LPID, r7
3559 mtspr SPRN_PID, r0
3560 ld r8, KVM_HOST_LPCR(r10)
3561 mtspr SPRN_LPCR, r8
3562 isync
3563 li r0, KVM_GUEST_MODE_NONE
3564 stb r0, HSTATE_IN_GUEST(r13)
3565
3566
3567
3568
3569 bcl 20, 31, .+4
35705: mflr r3
3571 addi r3, r3, 9f - 5b
3572 ld r4, PACAKMSR(r13)
3573 mtspr SPRN_SRR0, r3
3574 mtspr SPRN_SRR1, r4
3575 RFI_TO_KERNEL
35769: addi r3, r1, STACK_FRAME_OVERHEAD
3577 bl kvmppc_bad_interrupt
3578 b 9b
3579
3580
3581
3582
3583
3584
3585
3586
3587kvmppc_msr_interrupt:
3588 rldicl r0, r11, 64 - MSR_TS_S_LG, 62
3589 cmpwi r0, 2
3590 ld r11, VCPU_INTR_MSR(r9)
3591 bne 1f
3592
3593 li r0, 1
35941: rldimi r11, r0, MSR_TS_S_LG, 63 - MSR_TS_T_LG
3595 blr
3596
3597
3598
3599
3600
3601
3602
3603kvmppc_fix_pmao:
3604 li r3, 0
3605 mtspr SPRN_MMCR2, r3
3606 lis r3, (MMCR0_PMXE | MMCR0_FCECE)@h
3607 ori r3, r3, MMCR0_PMCjCE | MMCR0_C56RUN
3608 mtspr SPRN_MMCR0, r3
3609 lis r3, 0x7fff
3610 ori r3, r3, 0xffff
3611 mtspr SPRN_PMC6, r3
3612 isync
3613 blr
3614
3615#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
3616
3617
3618
3619
3620kvmhv_start_timing:
3621 ld r5, HSTATE_KVM_VCORE(r13)
3622 ld r6, VCORE_TB_OFFSET_APPL(r5)
3623 mftb r5
3624 subf r5, r6, r5
3625 std r3, VCPU_CUR_ACTIVITY(r4)
3626 std r5, VCPU_ACTIVITY_START(r4)
3627 blr
3628
3629
3630
3631
3632
3633kvmhv_accumulate_time:
3634 ld r5, HSTATE_KVM_VCORE(r13)
3635 ld r8, VCORE_TB_OFFSET_APPL(r5)
3636 ld r5, VCPU_CUR_ACTIVITY(r4)
3637 ld r6, VCPU_ACTIVITY_START(r4)
3638 std r3, VCPU_CUR_ACTIVITY(r4)
3639 mftb r7
3640 subf r7, r8, r7
3641 std r7, VCPU_ACTIVITY_START(r4)
3642 cmpdi r5, 0
3643 beqlr
3644 subf r3, r6, r7
3645 ld r8, TAS_SEQCOUNT(r5)
3646 cmpdi r8, 0
3647 addi r8, r8, 1
3648 std r8, TAS_SEQCOUNT(r5)
3649 lwsync
3650 ld r7, TAS_TOTAL(r5)
3651 add r7, r7, r3
3652 std r7, TAS_TOTAL(r5)
3653 ld r6, TAS_MIN(r5)
3654 ld r7, TAS_MAX(r5)
3655 beq 3f
3656 cmpd r3, r6
3657 bge 1f
36583: std r3, TAS_MIN(r5)
36591: cmpd r3, r7
3660 ble 2f
3661 std r3, TAS_MAX(r5)
36622: lwsync
3663 addi r8, r8, 1
3664 std r8, TAS_SEQCOUNT(r5)
3665 blr
3666#endif
3667