1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16#include <asm/hw_irq.h>
17#include <asm/exception-64s.h>
18#include <asm/ptrace.h>
19#include <asm/cpuidle.h>
20#include <asm/head-64.h>
21#include <asm/feature-fixups.h>
22#include <asm/kup.h>
23
24
25#define EX_R9 0
26#define EX_R10 8
27#define EX_R11 16
28#define EX_R12 24
29#define EX_R13 32
30#define EX_DAR 40
31#define EX_DSISR 48
32#define EX_CCR 52
33#define EX_CFAR 56
34#define EX_PPR 64
35
36#define EX_CTR 72
37.if EX_SIZE != 10
38 .error "EX_SIZE is wrong"
39.endif
40#else
41.if EX_SIZE != 9
42 .error "EX_SIZE is wrong"
43.endif
44#endif
45
46
47
48
49
50
51
52
53#define LOAD_HANDLER(reg, label) \
54 ld reg,PACAKBASE(r13); \
55 ori reg,reg,FIXED_SYMBOL_ABS_ADDR(label)
56
57#define __LOAD_HANDLER(reg, label) \
58 ld reg,PACAKBASE(r13); \
59 ori reg,reg,(ABS_ADDR(label))@l
60
61
62
63
64
65#define __LOAD_FAR_HANDLER(reg, label) \
66 ld reg,PACAKBASE(r13); \
67 ori reg,reg,(ABS_ADDR(label))@l; \
68 addis reg,reg,(ABS_ADDR(label))@h
69
70
71#define EXC_HV 1
72#define EXC_STD 0
73
74
75
76
77
78
79
80#define SAVE_CTR(reg, area) mfctr reg ; std reg,area+EX_CTR(r13)
81#define GET_CTR(reg, area) ld reg,area+EX_CTR(r13)
82#define RESTORE_CTR(reg, area) ld reg,area+EX_CTR(r13) ; mtctr reg
83#else
84
85#define SAVE_CTR(reg, area)
86#define GET_CTR(reg, area) mfctr reg
87#define RESTORE_CTR(reg, area)
88#endif
89
90
91
92
93
94#define SAVE_PPR(area, ra) \
95BEGIN_FTR_SECTION_NESTED(940) \
96 ld ra,area+EX_PPR(r13); \
97 std ra,_PPR(r1); \
98END_FTR_SECTION_NESTED(CPU_FTR_HAS_PPR,CPU_FTR_HAS_PPR,940)
99
100#define RESTORE_PPR_PACA(area, ra) \
101BEGIN_FTR_SECTION_NESTED(941) \
102 ld ra,area+EX_PPR(r13); \
103 mtspr SPRN_PPR,ra; \
104END_FTR_SECTION_NESTED(CPU_FTR_HAS_PPR,CPU_FTR_HAS_PPR,941)
105
106
107
108
109#define OPT_GET_SPR(ra, spr, ftr) \
110BEGIN_FTR_SECTION_NESTED(943) \
111 mfspr ra,spr; \
112END_FTR_SECTION_NESTED(ftr,ftr,943)
113
114
115
116
117#define OPT_SET_SPR(ra, spr, ftr) \
118BEGIN_FTR_SECTION_NESTED(943) \
119 mtspr spr,ra; \
120END_FTR_SECTION_NESTED(ftr,ftr,943)
121
122
123
124
125#define OPT_SAVE_REG_TO_PACA(offset, ra, ftr) \
126BEGIN_FTR_SECTION_NESTED(943) \
127 std ra,offset(r13); \
128END_FTR_SECTION_NESTED(ftr,ftr,943)
129
130.macro EXCEPTION_PROLOG_0 area
131 SET_SCRATCH0(r13)
132 GET_PACA(r13)
133 std r9,\area\()+EX_R9(r13)
134 OPT_GET_SPR(r9, SPRN_PPR, CPU_FTR_HAS_PPR)
135 HMT_MEDIUM
136 std r10,\area\()+EX_R10(r13)
137 OPT_GET_SPR(r10, SPRN_CFAR, CPU_FTR_CFAR)
138.endm
139
140.macro EXCEPTION_PROLOG_1 hsrr, area, kvm, vec, dar, dsisr, bitmask
141 OPT_SAVE_REG_TO_PACA(\area\()+EX_PPR, r9, CPU_FTR_HAS_PPR)
142 OPT_SAVE_REG_TO_PACA(\area\()+EX_CFAR, r10, CPU_FTR_CFAR)
143 INTERRUPT_TO_KERNEL
144 SAVE_CTR(r10, \area\())
145 mfcr r9
146 .if \kvm
147 KVMTEST \hsrr \vec
148 .endif
149 .if \bitmask
150 lbz r10,PACAIRQSOFTMASK(r13)
151 andi. r10,r10,\bitmask
152
153 .if \vec == 0x500 || \vec == 0xea0
154 li r10,PACA_IRQ_EE
155 .elseif \vec == 0x900
156 li r10,PACA_IRQ_DEC
157 .elseif \vec == 0xa00 || \vec == 0xe80
158 li r10,PACA_IRQ_DBELL
159 .elseif \vec == 0xe60
160 li r10,PACA_IRQ_HMI
161 .elseif \vec == 0xf00
162 li r10,PACA_IRQ_PMI
163 .else
164 .abort "Bad maskable vector"
165 .endif
166
167 .if \hsrr
168 bne masked_Hinterrupt
169 .else
170 bne masked_interrupt
171 .endif
172 .endif
173
174 std r11,\area\()+EX_R11(r13)
175 std r12,\area\()+EX_R12(r13)
176
177
178
179
180
181
182 GET_SCRATCH0(r10)
183 std r10,\area\()+EX_R13(r13)
184 .if \dar
185 mfspr r10,SPRN_DAR
186 std r10,\area\()+EX_DAR(r13)
187 .endif
188 .if \dsisr
189 mfspr r10,SPRN_DSISR
190 stw r10,\area\()+EX_DSISR(r13)
191 .endif
192.endm
193
194.macro EXCEPTION_PROLOG_2_REAL label, hsrr, set_ri
195 ld r10,PACAKMSR(r13)
196 .if ! \set_ri
197 xori r10,r10,MSR_RI
198 .endif
199 .if \hsrr
200 mfspr r11,SPRN_HSRR0
201 mfspr r12,SPRN_HSRR1
202 mtspr SPRN_HSRR1,r10
203 .else
204 mfspr r11,SPRN_SRR0
205 mfspr r12,SPRN_SRR1
206 mtspr SPRN_SRR1,r10
207 .endif
208 LOAD_HANDLER(r10, \label\())
209 .if \hsrr
210 mtspr SPRN_HSRR0,r10
211 HRFI_TO_KERNEL
212 .else
213 mtspr SPRN_SRR0,r10
214 RFI_TO_KERNEL
215 .endif
216 b .
217.endm
218
219.macro EXCEPTION_PROLOG_2_VIRT label, hsrr
220#ifdef CONFIG_RELOCATABLE
221 .if \hsrr
222 mfspr r11,SPRN_HSRR0
223 .else
224 mfspr r11,SPRN_SRR0
225 .endif
226 LOAD_HANDLER(r12, \label\())
227 mtctr r12
228 .if \hsrr
229 mfspr r12,SPRN_HSRR1
230 .else
231 mfspr r12,SPRN_SRR1
232 .endif
233 li r10,MSR_RI
234 mtmsrd r10,1
235 bctr
236#else
237 .if \hsrr
238 mfspr r11,SPRN_HSRR0
239 mfspr r12,SPRN_HSRR1
240 .else
241 mfspr r11,SPRN_SRR0
242 mfspr r12,SPRN_SRR1
243 .endif
244 li r10,MSR_RI
245 mtmsrd r10,1
246 b \label
247#endif
248.endm
249
250
251
252
253
254
255
256
257
258#define BRANCH_TO_C000(reg, label) \
259 __LOAD_FAR_HANDLER(reg, label); \
260 mtctr reg; \
261 bctr
262
263#ifdef CONFIG_KVM_BOOK3S_64_HANDLER
264#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
265
266
267
268
269
270#define kvmppc_interrupt kvmppc_interrupt_hv
271#else
272#define kvmppc_interrupt kvmppc_interrupt_pr
273#endif
274
275.macro KVMTEST hsrr, n
276 lbz r10,HSTATE_IN_GUEST(r13)
277 cmpwi r10,0
278 .if \hsrr
279 bne do_kvm_H\n
280 .else
281 bne do_kvm_\n
282 .endif
283.endm
284
285.macro KVM_HANDLER area, hsrr, n, skip
286 .if \skip
287 cmpwi r10,KVM_GUEST_MODE_SKIP
288 beq 89f
289 .else
290BEGIN_FTR_SECTION_NESTED(947)
291 ld r10,\area+EX_CFAR(r13)
292 std r10,HSTATE_CFAR(r13)
293END_FTR_SECTION_NESTED(CPU_FTR_CFAR,CPU_FTR_CFAR,947)
294 .endif
295
296BEGIN_FTR_SECTION_NESTED(948)
297 ld r10,\area+EX_PPR(r13)
298 std r10,HSTATE_PPR(r13)
299END_FTR_SECTION_NESTED(CPU_FTR_HAS_PPR,CPU_FTR_HAS_PPR,948)
300 ld r10,\area+EX_R10(r13)
301 std r12,HSTATE_SCRATCH0(r13)
302 sldi r12,r9,32
303
304 .if \hsrr
305 ori r12,r12,(\n + 0x2)
306 .else
307 ori r12,r12,(\n)
308 .endif
309
310#ifdef CONFIG_RELOCATABLE
311
312
313
314
315
316 mfctr r9
317 std r9,HSTATE_SCRATCH1(r13)
318 __LOAD_FAR_HANDLER(r9, kvmppc_interrupt)
319 mtctr r9
320 ld r9,\area+EX_R9(r13)
321 bctr
322#else
323 ld r9,\area+EX_R9(r13)
324 b kvmppc_interrupt
325#endif
326
327
328 .if \skip
32989: mtocrf 0x80,r9
330 ld r9,\area+EX_R9(r13)
331 ld r10,\area+EX_R10(r13)
332 .if \hsrr
333 b kvmppc_skip_Hinterrupt
334 .else
335 b kvmppc_skip_interrupt
336 .endif
337 .endif
338.endm
339
340#else
341.macro KVMTEST hsrr, n
342.endm
343.macro KVM_HANDLER area, hsrr, n, skip
344.endm
345#endif
346
347#define EXCEPTION_PROLOG_COMMON_1() \
348 std r9,_CCR(r1); \
349 std r11,_NIP(r1); \
350 std r12,_MSR(r1); \
351 std r10,0(r1); \
352 std r0,GPR0(r1); \
353 std r10,GPR1(r1); \
354
355
356#define EXCEPTION_PROLOG_COMMON_2(area) \
357 ld r9,area+EX_R9(r13); \
358 ld r10,area+EX_R10(r13); \
359 std r9,GPR9(r1); \
360 std r10,GPR10(r1); \
361 ld r9,area+EX_R11(r13); \
362 ld r10,area+EX_R12(r13); \
363 ld r11,area+EX_R13(r13); \
364 std r9,GPR11(r1); \
365 std r10,GPR12(r1); \
366 std r11,GPR13(r1); \
367BEGIN_FTR_SECTION_NESTED(66); \
368 ld r10,area+EX_CFAR(r13); \
369 std r10,ORIG_GPR3(r1); \
370END_FTR_SECTION_NESTED(CPU_FTR_CFAR, CPU_FTR_CFAR, 66); \
371 GET_CTR(r10, area); \
372 std r10,_CTR(r1);
373
374#define EXCEPTION_PROLOG_COMMON_3(trap) \
375 std r2,GPR2(r1); \
376 SAVE_4GPRS(3, r1); \
377 SAVE_2GPRS(7, r1); \
378 mflr r9; \
379 ld r2,PACATOC(r13); \
380 std r9,_LINK(r1); \
381 lbz r10,PACAIRQSOFTMASK(r13); \
382 mfspr r11,SPRN_XER; \
383 std r10,SOFTE(r1); \
384 std r11,_XER(r1); \
385 li r9,(trap)+1; \
386 std r9,_TRAP(r1); \
387 li r10,0; \
388 ld r11,exception_marker@toc(r2); \
389 std r10,RESULT(r1); \
390 std r11,STACK_FRAME_OVERHEAD-16(r1);
391
392
393
394
395
396
397#define EXCEPTION_COMMON(area, trap) \
398 andi. r10,r12,MSR_PR; \
399 mr r10,r1; \
400 subi r1,r1,INT_FRAME_SIZE; \
401 beq- 1f; \
402 ld r1,PACAKSAVE(r13); \
4031: tdgei r1,-INT_FRAME_SIZE; \
404 EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,0; \
4053: EXCEPTION_PROLOG_COMMON_1(); \
406 kuap_save_amr_and_lock r9, r10, cr1, cr0; \
407 beq 4f; \
408 ACCOUNT_CPU_USER_ENTRY(r13, r9, r10); \
409 SAVE_PPR(area, r9); \
4104: EXCEPTION_PROLOG_COMMON_2(area); \
411 EXCEPTION_PROLOG_COMMON_3(trap); \
412 ACCOUNT_STOLEN_TIME
413
414
415
416
417
418#define EXCEPTION_COMMON_STACK(area, trap) \
419 EXCEPTION_PROLOG_COMMON_1(); \
420 kuap_save_amr_and_lock r9, r10, cr1; \
421 EXCEPTION_PROLOG_COMMON_2(area); \
422 EXCEPTION_PROLOG_COMMON_3(trap)
423
424
425
426
427
428.macro EXCEPTION_RESTORE_REGS hsrr
429
430 ld r9,_MSR(r1)
431 .if \hsrr
432 mtspr SPRN_HSRR1,r9
433 .else
434 mtspr SPRN_SRR1,r9
435 .endif
436 ld r9,_NIP(r1)
437 .if \hsrr
438 mtspr SPRN_HSRR0,r9
439 .else
440 mtspr SPRN_SRR0,r9
441 .endif
442 ld r9,_CTR(r1)
443 mtctr r9
444 ld r9,_XER(r1)
445 mtxer r9
446 ld r9,_LINK(r1)
447 mtlr r9
448 ld r9,_CCR(r1)
449 mtcr r9
450 REST_8GPRS(2, r1)
451 REST_4GPRS(10, r1)
452 REST_GPR(0, r1)
453
454 ld r1,GPR1(r1)
455.endm
456
457#define RUNLATCH_ON \
458BEGIN_FTR_SECTION \
459 ld r3, PACA_THREAD_INFO(r13); \
460 ld r4,TI_LOCAL_FLAGS(r3); \
461 andi. r0,r4,_TLF_RUNLATCH; \
462 beql ppc64_runlatch_on_trampoline; \
463END_FTR_SECTION_IFSET(CPU_FTR_CTRL)
464
465
466
467
468
469
470
471
472#ifdef CONFIG_PPC_970_NAP
473#define FINISH_NAP \
474BEGIN_FTR_SECTION \
475 ld r11, PACA_THREAD_INFO(r13); \
476 ld r9,TI_LOCAL_FLAGS(r11); \
477 andi. r10,r9,_TLF_NAPPING; \
478 bnel power4_fixup_nap; \
479END_FTR_SECTION_IFSET(CPU_FTR_CAN_NAP)
480#else
481#define FINISH_NAP
482#endif
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526#define __EXC_REAL(name, start, size, area) \
527 EXC_REAL_BEGIN(name, start, size); \
528 EXCEPTION_PROLOG_0 area ; \
529 EXCEPTION_PROLOG_1 EXC_STD, area, 1, start, 0, 0, 0 ; \
530 EXCEPTION_PROLOG_2_REAL name
531 EXC_REAL_END(name, start, size)
532
533#define EXC_REAL(name, start, size) \
534 __EXC_REAL(name, start, size, PACA_EXGEN)
535
536#define __EXC_VIRT(name, start, size, realvec, area) \
537 EXC_VIRT_BEGIN(name, start, size); \
538 EXCEPTION_PROLOG_0 area ; \
539 EXCEPTION_PROLOG_1 EXC_STD, area, 0, realvec, 0, 0, 0; \
540 EXCEPTION_PROLOG_2_VIRT name
541 EXC_VIRT_END(name, start, size)
542
543#define EXC_VIRT(name, start, size, realvec) \
544 __EXC_VIRT(name, start, size, realvec, PACA_EXGEN)
545
546#define EXC_REAL_MASKABLE(name, start, size, bitmask) \
547 EXC_REAL_BEGIN(name, start, size); \
548 EXCEPTION_PROLOG_0 PACA_EXGEN ; \
549 EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 1, start, 0, 0, bitmask ; \
550 EXCEPTION_PROLOG_2_REAL name
551 EXC_REAL_END(name, start, size)
552
553#define EXC_VIRT_MASKABLE(name, start, size, realvec, bitmask) \
554 EXC_VIRT_BEGIN(name, start, size); \
555 EXCEPTION_PROLOG_0 PACA_EXGEN ; \
556 EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 0, realvec, 0, 0, bitmask ; \
557 EXCEPTION_PROLOG_2_VIRT name
558 EXC_VIRT_END(name, start, size)
559
560#define EXC_REAL_HV(name, start, size) \
561 EXC_REAL_BEGIN(name, start, size); \
562 EXCEPTION_PROLOG_0 PACA_EXGEN; \
563 EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, start, 0, 0, 0 ; \
564 EXCEPTION_PROLOG_2_REAL name
565 EXC_REAL_END(name, start, size)
566
567#define EXC_VIRT_HV(name, start, size, realvec) \
568 EXC_VIRT_BEGIN(name, start, size); \
569 EXCEPTION_PROLOG_0 PACA_EXGEN; \
570 EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, realvec, 0, 0, 0 ; \
571 EXCEPTION_PROLOG_2_VIRT name
572 EXC_VIRT_END(name, start, size)
573
574#define __EXC_REAL_OOL(name, start, size) \
575 EXC_REAL_BEGIN(name, start, size); \
576 EXCEPTION_PROLOG_0 PACA_EXGEN ; \
577 b tramp_real_
578 EXC_REAL_END(name, start, size)
579
580#define __TRAMP_REAL_OOL(name, vec) \
581 TRAMP_REAL_BEGIN(tramp_real_
582 EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 1, vec, 0, 0, 0 ; \
583 EXCEPTION_PROLOG_2_REAL name
584
585#define EXC_REAL_OOL(name, start, size) \
586 __EXC_REAL_OOL(name, start, size); \
587 __TRAMP_REAL_OOL(name, start)
588
589#define __EXC_REAL_OOL_MASKABLE(name, start, size) \
590 __EXC_REAL_OOL(name, start, size)
591
592#define __TRAMP_REAL_OOL_MASKABLE(name, vec, bitmask) \
593 TRAMP_REAL_BEGIN(tramp_real_
594 EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 1, vec, 0, 0, bitmask ; \
595 EXCEPTION_PROLOG_2_REAL name
596
597#define EXC_REAL_OOL_MASKABLE(name, start, size, bitmask) \
598 __EXC_REAL_OOL_MASKABLE(name, start, size); \
599 __TRAMP_REAL_OOL_MASKABLE(name, start, bitmask)
600
601#define __EXC_REAL_OOL_HV(name, start, size) \
602 __EXC_REAL_OOL(name, start, size)
603
604#define __TRAMP_REAL_OOL_HV(name, vec) \
605 TRAMP_REAL_BEGIN(tramp_real_
606 EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, vec, 0, 0, 0 ; \
607 EXCEPTION_PROLOG_2_REAL name
608
609#define EXC_REAL_OOL_HV(name, start, size) \
610 __EXC_REAL_OOL_HV(name, start, size); \
611 __TRAMP_REAL_OOL_HV(name, start)
612
613#define __EXC_REAL_OOL_MASKABLE_HV(name, start, size) \
614 __EXC_REAL_OOL(name, start, size)
615
616#define __TRAMP_REAL_OOL_MASKABLE_HV(name, vec, bitmask) \
617 TRAMP_REAL_BEGIN(tramp_real_
618 EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, vec, 0, 0, bitmask ; \
619 EXCEPTION_PROLOG_2_REAL name
620
621#define EXC_REAL_OOL_MASKABLE_HV(name, start, size, bitmask) \
622 __EXC_REAL_OOL_MASKABLE_HV(name, start, size); \
623 __TRAMP_REAL_OOL_MASKABLE_HV(name, start, bitmask)
624
625#define __EXC_VIRT_OOL(name, start, size) \
626 EXC_VIRT_BEGIN(name, start, size); \
627 EXCEPTION_PROLOG_0 PACA_EXGEN ; \
628 b tramp_virt_
629 EXC_VIRT_END(name, start, size)
630
631#define __TRAMP_VIRT_OOL(name, realvec) \
632 TRAMP_VIRT_BEGIN(tramp_virt_
633 EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 0, vec, 0, 0, 0 ; \
634 EXCEPTION_PROLOG_2_VIRT name
635
636#define EXC_VIRT_OOL(name, start, size, realvec) \
637 __EXC_VIRT_OOL(name, start, size); \
638 __TRAMP_VIRT_OOL(name, realvec)
639
640#define __EXC_VIRT_OOL_MASKABLE(name, start, size) \
641 __EXC_VIRT_OOL(name, start, size)
642
643#define __TRAMP_VIRT_OOL_MASKABLE(name, realvec, bitmask) \
644 TRAMP_VIRT_BEGIN(tramp_virt_
645 EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 0, realvec, 0, 0, bitmask ; \
646 EXCEPTION_PROLOG_2_REAL name
647
648#define EXC_VIRT_OOL_MASKABLE(name, start, size, realvec, bitmask) \
649 __EXC_VIRT_OOL_MASKABLE(name, start, size); \
650 __TRAMP_VIRT_OOL_MASKABLE(name, realvec, bitmask)
651
652#define __EXC_VIRT_OOL_HV(name, start, size) \
653 __EXC_VIRT_OOL(name, start, size)
654
655#define __TRAMP_VIRT_OOL_HV(name, realvec) \
656 TRAMP_VIRT_BEGIN(tramp_virt_
657 EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, realvec, 0, 0, 0 ; \
658 EXCEPTION_PROLOG_2_VIRT name
659
660#define EXC_VIRT_OOL_HV(name, start, size, realvec) \
661 __EXC_VIRT_OOL_HV(name, start, size); \
662 __TRAMP_VIRT_OOL_HV(name, realvec)
663
664#define __EXC_VIRT_OOL_MASKABLE_HV(name, start, size) \
665 __EXC_VIRT_OOL(name, start, size)
666
667#define __TRAMP_VIRT_OOL_MASKABLE_HV(name, realvec, bitmask) \
668 TRAMP_VIRT_BEGIN(tramp_virt_
669 EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, realvec, 0, 0, bitmask ; \
670 EXCEPTION_PROLOG_2_VIRT name
671
672#define EXC_VIRT_OOL_MASKABLE_HV(name, start, size, realvec, bitmask) \
673 __EXC_VIRT_OOL_MASKABLE_HV(name, start, size); \
674 __TRAMP_VIRT_OOL_MASKABLE_HV(name, realvec, bitmask)
675
676#define TRAMP_KVM(area, n) \
677 TRAMP_KVM_BEGIN(do_kvm_
678 KVM_HANDLER area, EXC_STD, n, 0
679
680#define TRAMP_KVM_SKIP(area, n) \
681 TRAMP_KVM_BEGIN(do_kvm_
682 KVM_HANDLER area, EXC_STD, n, 1
683
684#define TRAMP_KVM_HV(area, n) \
685 TRAMP_KVM_BEGIN(do_kvm_H
686 KVM_HANDLER area, EXC_HV, n, 0
687
688#define TRAMP_KVM_HV_SKIP(area, n) \
689 TRAMP_KVM_BEGIN(do_kvm_H
690 KVM_HANDLER area, EXC_HV, n, 1
691
692#define EXC_COMMON(name, realvec, hdlr) \
693 EXC_COMMON_BEGIN(name); \
694 EXCEPTION_COMMON(PACA_EXGEN, realvec); \
695 bl save_nvgprs; \
696 RECONCILE_IRQ_STATE(r10, r11); \
697 addi r3,r1,STACK_FRAME_OVERHEAD; \
698 bl hdlr; \
699 b ret_from_except
700
701
702
703
704
705#define EXC_COMMON_ASYNC(name, realvec, hdlr) \
706 EXC_COMMON_BEGIN(name); \
707 EXCEPTION_COMMON(PACA_EXGEN, realvec); \
708 FINISH_NAP; \
709 RECONCILE_IRQ_STATE(r10, r11); \
710 RUNLATCH_ON; \
711 addi r3,r1,STACK_FRAME_OVERHEAD; \
712 bl hdlr; \
713 b ret_from_except_lite
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760OPEN_FIXED_SECTION(real_vectors, 0x0100, 0x1900)
761OPEN_FIXED_SECTION(real_trampolines, 0x1900, 0x4000)
762OPEN_FIXED_SECTION(virt_vectors, 0x4000, 0x5900)
763OPEN_FIXED_SECTION(virt_trampolines, 0x5900, 0x7000)
764
765#ifdef CONFIG_PPC_POWERNV
766 .globl start_real_trampolines
767 .globl end_real_trampolines
768 .globl start_virt_trampolines
769 .globl end_virt_trampolines
770#endif
771
772
773
774
775
776
777
778
779ZERO_FIXED_SECTION(fwnmi_page, 0x7000, 0x8000)
780OPEN_TEXT_SECTION(0x8000)
781#else
782OPEN_TEXT_SECTION(0x7000)
783#endif
784
785USE_FIXED_SECTION(real_vectors)
786
787
788
789
790
791
792
793
794
795 .globl __start_interrupts
796__start_interrupts:
797
798
799EXC_VIRT_NONE(0x4000, 0x100)
800
801
802EXC_REAL_BEGIN(system_reset, 0x100, 0x100)
803#ifdef CONFIG_PPC_P7_NAP
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820BEGIN_FTR_SECTION
821 SET_SCRATCH0(r13)
822 GET_PACA(r13)
823 std r3,PACA_EXNMI+0*8(r13)
824 std r4,PACA_EXNMI+1*8(r13)
825 std r5,PACA_EXNMI+2*8(r13)
826 mfspr r3,SPRN_SRR1
827 mfocrf r4,0x80
828 rlwinm. r5,r3,47-31,30,31
829 bne+ system_reset_idle_wake
830
831 mtocrf 0x80,r4
832 ld r3,PACA_EXNMI+0*8(r13)
833 ld r4,PACA_EXNMI+1*8(r13)
834 ld r5,PACA_EXNMI+2*8(r13)
835 GET_SCRATCH0(r13)
836END_FTR_SECTION_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206)
837#endif
838
839 EXCEPTION_PROLOG_0 PACA_EXNMI
840 EXCEPTION_PROLOG_1 EXC_STD, PACA_EXNMI, 1, 0x100, 0, 0, 0
841 EXCEPTION_PROLOG_2_REAL system_reset_common, EXC_STD, 0
842
843
844
845
846
847
848
849
850
851
852EXC_REAL_END(system_reset, 0x100, 0x100)
853
854EXC_VIRT_NONE(0x4100, 0x100)
855TRAMP_KVM(PACA_EXNMI, 0x100)
856
857#ifdef CONFIG_PPC_P7_NAP
858TRAMP_REAL_BEGIN(system_reset_idle_wake)
859
860 cmpwi cr1,r5,2
861 bltlr cr1
862 BRANCH_TO_C000(r12, DOTSYM(idle_return_gpr_loss))
863#endif
864
865#ifdef CONFIG_PPC_PSERIES
866
867
868
869TRAMP_REAL_BEGIN(system_reset_fwnmi)
870
871 EXCEPTION_PROLOG_0 PACA_EXNMI
872 EXCEPTION_PROLOG_1 EXC_STD, PACA_EXNMI, 0, 0x100, 0, 0, 0
873 EXCEPTION_PROLOG_2_REAL system_reset_common, EXC_STD, 0
874
875#endif
876
877EXC_COMMON_BEGIN(system_reset_common)
878
879
880
881
882
883
884 lhz r10,PACA_IN_NMI(r13)
885 addi r10,r10,1
886 sth r10,PACA_IN_NMI(r13)
887 li r10,MSR_RI
888 mtmsrd r10,1
889
890 mr r10,r1
891 ld r1,PACA_NMI_EMERG_SP(r13)
892 subi r1,r1,INT_FRAME_SIZE
893 EXCEPTION_COMMON_STACK(PACA_EXNMI, 0x100)
894 bl save_nvgprs
895
896
897
898
899
900
901
902
903
904 li r10,IRQS_ALL_DISABLED
905 stb r10,PACAIRQSOFTMASK(r13)
906 lbz r10,PACAIRQHAPPENED(r13)
907 std r10,_DAR(r1)
908
909 addi r3,r1,STACK_FRAME_OVERHEAD
910 bl system_reset_exception
911
912
913 li r9,0
914 mtmsrd r9,1
915
916
917
918
919 lhz r10,PACA_IN_NMI(r13)
920 subi r10,r10,1
921 sth r10,PACA_IN_NMI(r13)
922
923
924
925
926 ld r10,_DAR(r1)
927 stb r10,PACAIRQHAPPENED(r13)
928 ld r10,SOFTE(r1)
929 stb r10,PACAIRQSOFTMASK(r13)
930
931 EXCEPTION_RESTORE_REGS EXC_STD
932 RFI_TO_USER_OR_KERNEL
933
934
935EXC_REAL_BEGIN(machine_check, 0x200, 0x100)
936
937
938
939
940 EXCEPTION_PROLOG_0 PACA_EXMC
941BEGIN_FTR_SECTION
942 b machine_check_common_early
943FTR_SECTION_ELSE
944 b machine_check_pSeries_0
945ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE)
946EXC_REAL_END(machine_check, 0x200, 0x100)
947EXC_VIRT_NONE(0x4200, 0x100)
948TRAMP_REAL_BEGIN(machine_check_common_early)
949 EXCEPTION_PROLOG_1 EXC_STD, PACA_EXMC, 0, 0x200, 0, 0, 0
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976 mr r11,r1
977 lhz r10,PACA_IN_MCE(r13)
978 cmpwi r10,0
979 bne 0f
980
981 ld r1,PACAMCEMERGSP(r13)
9820: subi r1,r1,INT_FRAME_SIZE
983 addi r10,r10,1
984 sth r10,PACA_IN_MCE(r13)
985
986 cmpwi r10,MAX_MCE_DEPTH
987 bgt 2f
988 std r11,GPR1(r1)
989 std r11,0(r1)
990 mfspr r11,SPRN_SRR0
991 std r11,_NIP(r1)
992 mfspr r11,SPRN_SRR1
993 std r11,_MSR(r1)
994 mfspr r11,SPRN_DAR
995 std r11,_DAR(r1)
996 mfspr r11,SPRN_DSISR
997 std r11,_DSISR(r1)
998 std r9,_CCR(r1)
999
1000
1001 EXCEPTION_PROLOG_COMMON_2(PACA_EXMC)
1002 mfmsr r11
1003BEGIN_FTR_SECTION
1004 ori r11,r11,MSR_ME
1005END_FTR_SECTION_IFSET(CPU_FTR_HVMODE)
1006 ori r11,r11,MSR_RI
1007 LOAD_HANDLER(r12, machine_check_handle_early)
10081: mtspr SPRN_SRR0,r12
1009 mtspr SPRN_SRR1,r11
1010 RFI_TO_KERNEL
1011 b .
10122:
1013
1014
1015
1016
1017 addi r1,r1,INT_FRAME_SIZE
1018 ld r11,PACAKMSR(r13)
1019 LOAD_HANDLER(r12, unrecover_mce)
1020 li r10,MSR_ME
1021 andc r11,r11,r10
1022 b 1b
1023 b .
1024
1025TRAMP_REAL_BEGIN(machine_check_pSeries)
1026 .globl machine_check_fwnmi
1027machine_check_fwnmi:
1028 EXCEPTION_PROLOG_0 PACA_EXMC
1029BEGIN_FTR_SECTION
1030 b machine_check_common_early
1031END_FTR_SECTION_IFCLR(CPU_FTR_HVMODE)
1032machine_check_pSeries_0:
1033 EXCEPTION_PROLOG_1 EXC_STD, PACA_EXMC, 1, 0x200, 1, 1, 0
1034
1035
1036
1037
1038
1039 EXCEPTION_PROLOG_2_REAL machine_check_common, EXC_STD, 0
1040
1041TRAMP_KVM_SKIP(PACA_EXMC, 0x200)
1042
1043EXC_COMMON_BEGIN(machine_check_common)
1044
1045
1046
1047
1048 EXCEPTION_COMMON(PACA_EXMC, 0x200)
1049 FINISH_NAP
1050 RECONCILE_IRQ_STATE(r10, r11)
1051 ld r3,PACA_EXMC+EX_DAR(r13)
1052 lwz r4,PACA_EXMC+EX_DSISR(r13)
1053
1054 li r10,MSR_RI
1055 mtmsrd r10,1
1056 std r3,_DAR(r1)
1057 std r4,_DSISR(r1)
1058 bl save_nvgprs
1059 addi r3,r1,STACK_FRAME_OVERHEAD
1060 bl machine_check_exception
1061 b ret_from_except
1062
1063#define MACHINE_CHECK_HANDLER_WINDUP \
1064 \
1065 li r9,0; \
1066 mtmsrd r9,1; \
1067 \
1068 lhz r12,PACA_IN_MCE(r13); \
1069 subi r12,r12,1; \
1070 sth r12,PACA_IN_MCE(r13); \
1071 EXCEPTION_RESTORE_REGS EXC_STD
1072
1073#ifdef CONFIG_PPC_P7_NAP
1074
1075
1076
1077
1078EXC_COMMON_BEGIN(machine_check_idle_common)
1079 bl machine_check_queue_event
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091 ld r3,_MSR(r1)
1092 ld r4,_LINK(r1)
1093
1094 lhz r11,PACA_IN_MCE(r13)
1095 subi r11,r11,1
1096 sth r11,PACA_IN_MCE(r13)
1097
1098 mtlr r4
1099 rlwinm r10,r3,47-31,30,31
1100 cmpwi cr1,r10,2
1101 bltlr cr1
1102 b idle_return_gpr_loss
1103#endif
1104
1105
1106
1107
1108EXC_COMMON_BEGIN(machine_check_handle_early)
1109 std r0,GPR0(r1)
1110 EXCEPTION_PROLOG_COMMON_3(0x200)
1111 bl save_nvgprs
1112 addi r3,r1,STACK_FRAME_OVERHEAD
1113 bl machine_check_early
1114 std r3,RESULT(r1)
1115 ld r12,_MSR(r1)
1116BEGIN_FTR_SECTION
1117 b 4f
1118END_FTR_SECTION_IFCLR(CPU_FTR_HVMODE)
1119
1120#ifdef CONFIG_PPC_P7_NAP
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130BEGIN_FTR_SECTION
1131 rlwinm. r11,r12,47-31,30,31
1132 bne machine_check_idle_common
1133END_FTR_SECTION_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206)
1134#endif
1135
1136
1137
1138
1139
1140 rldicl. r11,r12,4,63
1141 beq 5f
11424: andi. r11,r12,MSR_PR
1143 bne 9f
1144
11455:
1146#ifdef CONFIG_KVM_BOOK3S_64_HANDLER
1147BEGIN_FTR_SECTION
1148
1149
1150
1151
1152
1153 lbz r11,HSTATE_IN_GUEST(r13)
1154 cmpwi r11,0
1155 bne 9f
1156END_FTR_SECTION_IFSET(CPU_FTR_HVMODE)
1157#endif
1158
1159
1160
1161
1162
1163
1164 andi. r11,r12,MSR_RI
1165 bne 2f
11661: mfspr r11,SPRN_SRR0
1167 LOAD_HANDLER(r10,unrecover_mce)
1168 mtspr SPRN_SRR0,r10
1169 ld r10,PACAKMSR(r13)
1170
1171
1172
1173
1174
1175
1176
1177 li r3,MSR_ME
1178 andc r10,r10,r3
1179 mtspr SPRN_SRR1,r10
1180 RFI_TO_KERNEL
1181 b .
11822:
1183
1184
1185
1186
1187 ld r3,RESULT(r1)
1188 cmpdi r3,0
1189
1190 beq 1b
1191BEGIN_FTR_SECTION
1192
1193
1194
1195
1196
1197 bl machine_check_queue_event
1198 MACHINE_CHECK_HANDLER_WINDUP
1199 RFI_TO_USER_OR_KERNEL
1200FTR_SECTION_ELSE
1201
1202
1203
1204
1205 LOAD_HANDLER(r10,mce_return)
1206 mtspr SPRN_SRR0,r10
1207 ld r10,PACAKMSR(r13)
1208 mtspr SPRN_SRR1,r10
1209 RFI_TO_KERNEL
1210 b .
1211ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE)
12129:
1213
1214 MACHINE_CHECK_HANDLER_WINDUP
1215 EXCEPTION_PROLOG_0 PACA_EXMC
1216 b machine_check_pSeries_0
1217
1218EXC_COMMON_BEGIN(unrecover_mce)
1219
1220 addi r3,r1,STACK_FRAME_OVERHEAD
1221 bl machine_check_exception
1222
1223
1224
1225
12261: addi r3,r1,STACK_FRAME_OVERHEAD
1227 bl unrecoverable_exception
1228 b 1b
1229
1230EXC_COMMON_BEGIN(mce_return)
1231
1232 addi r3,r1,STACK_FRAME_OVERHEAD
1233 bl machine_check_exception
1234 MACHINE_CHECK_HANDLER_WINDUP
1235 RFI_TO_KERNEL
1236 b .
1237
1238EXC_REAL_BEGIN(data_access, 0x300, 0x80)
1239 EXCEPTION_PROLOG_0 PACA_EXGEN
1240 b tramp_real_data_access
1241EXC_REAL_END(data_access, 0x300, 0x80)
1242
1243TRAMP_REAL_BEGIN(tramp_real_data_access)
1244 EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 1, 0x300, 1, 1, 0
1245 EXCEPTION_PROLOG_2_REAL data_access_common, EXC_STD, 1
1246
1247EXC_VIRT_BEGIN(data_access, 0x4300, 0x80)
1248 EXCEPTION_PROLOG_0 PACA_EXGEN
1249 EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 0, 0x300, 1, 1, 0
1250EXCEPTION_PROLOG_2_VIRT data_access_common, EXC_STD
1251EXC_VIRT_END(data_access, 0x4300, 0x80)
1252
1253TRAMP_KVM_SKIP(PACA_EXGEN, 0x300)
1254
1255EXC_COMMON_BEGIN(data_access_common)
1256
1257
1258
1259
1260
1261
1262 EXCEPTION_COMMON(PACA_EXGEN, 0x300)
1263 RECONCILE_IRQ_STATE(r10, r11)
1264 ld r12,_MSR(r1)
1265 ld r3,PACA_EXGEN+EX_DAR(r13)
1266 lwz r4,PACA_EXGEN+EX_DSISR(r13)
1267 li r5,0x300
1268 std r3,_DAR(r1)
1269 std r4,_DSISR(r1)
1270BEGIN_MMU_FTR_SECTION
1271 b do_hash_page
1272MMU_FTR_SECTION_ELSE
1273 b handle_page_fault
1274ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX)
1275
1276
1277EXC_REAL_BEGIN(data_access_slb, 0x380, 0x80)
1278 EXCEPTION_PROLOG_0 PACA_EXSLB
1279 b tramp_real_data_access_slb
1280EXC_REAL_END(data_access_slb, 0x380, 0x80)
1281
1282TRAMP_REAL_BEGIN(tramp_real_data_access_slb)
1283 EXCEPTION_PROLOG_1 EXC_STD, PACA_EXSLB, 1, 0x380, 1, 0, 0
1284 EXCEPTION_PROLOG_2_REAL data_access_slb_common, EXC_STD, 1
1285
1286EXC_VIRT_BEGIN(data_access_slb, 0x4380, 0x80)
1287 EXCEPTION_PROLOG_0 PACA_EXSLB
1288 EXCEPTION_PROLOG_1 EXC_STD, PACA_EXSLB, 0, 0x380, 1, 0, 0
1289 EXCEPTION_PROLOG_2_VIRT data_access_slb_common, EXC_STD
1290EXC_VIRT_END(data_access_slb, 0x4380, 0x80)
1291
1292TRAMP_KVM_SKIP(PACA_EXSLB, 0x380)
1293
1294EXC_COMMON_BEGIN(data_access_slb_common)
1295 EXCEPTION_COMMON(PACA_EXSLB, 0x380)
1296 ld r4,PACA_EXSLB+EX_DAR(r13)
1297 std r4,_DAR(r1)
1298 addi r3,r1,STACK_FRAME_OVERHEAD
1299BEGIN_MMU_FTR_SECTION
1300
1301 bl do_slb_fault
1302 cmpdi r3,0
1303 bne- 1f
1304 b fast_exception_return
13051:
1306MMU_FTR_SECTION_ELSE
1307
1308 li r3,-EFAULT
1309ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX)
1310 std r3,RESULT(r1)
1311 bl save_nvgprs
1312 RECONCILE_IRQ_STATE(r10, r11)
1313 ld r4,_DAR(r1)
1314 ld r5,RESULT(r1)
1315 addi r3,r1,STACK_FRAME_OVERHEAD
1316 bl do_bad_slb_fault
1317 b ret_from_except
1318
1319
1320EXC_REAL(instruction_access, 0x400, 0x80)
1321EXC_VIRT(instruction_access, 0x4400, 0x80, 0x400)
1322TRAMP_KVM(PACA_EXGEN, 0x400)
1323
1324EXC_COMMON_BEGIN(instruction_access_common)
1325 EXCEPTION_COMMON(PACA_EXGEN, 0x400)
1326 RECONCILE_IRQ_STATE(r10, r11)
1327 ld r12,_MSR(r1)
1328 ld r3,_NIP(r1)
1329 andis. r4,r12,DSISR_SRR1_MATCH_64S@h
1330 li r5,0x400
1331 std r3,_DAR(r1)
1332 std r4,_DSISR(r1)
1333BEGIN_MMU_FTR_SECTION
1334 b do_hash_page
1335MMU_FTR_SECTION_ELSE
1336 b handle_page_fault
1337ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX)
1338
1339
1340__EXC_REAL(instruction_access_slb, 0x480, 0x80, PACA_EXSLB)
1341__EXC_VIRT(instruction_access_slb, 0x4480, 0x80, 0x480, PACA_EXSLB)
1342TRAMP_KVM(PACA_EXSLB, 0x480)
1343
1344EXC_COMMON_BEGIN(instruction_access_slb_common)
1345 EXCEPTION_COMMON(PACA_EXSLB, 0x480)
1346 ld r4,_NIP(r1)
1347 addi r3,r1,STACK_FRAME_OVERHEAD
1348BEGIN_MMU_FTR_SECTION
1349
1350 bl do_slb_fault
1351 cmpdi r3,0
1352 bne- 1f
1353 b fast_exception_return
13541:
1355MMU_FTR_SECTION_ELSE
1356
1357 li r3,-EFAULT
1358ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX)
1359 std r3,RESULT(r1)
1360 bl save_nvgprs
1361 RECONCILE_IRQ_STATE(r10, r11)
1362 ld r4,_NIP(r1)
1363 ld r5,RESULT(r1)
1364 addi r3,r1,STACK_FRAME_OVERHEAD
1365 bl do_bad_slb_fault
1366 b ret_from_except
1367
1368
1369EXC_REAL_BEGIN(hardware_interrupt, 0x500, 0x100)
1370 EXCEPTION_PROLOG_0 PACA_EXGEN
1371BEGIN_FTR_SECTION
1372 EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, 0x500, 0, 0, IRQS_DISABLED
1373 EXCEPTION_PROLOG_2_REAL hardware_interrupt_common, EXC_HV, 1
1374FTR_SECTION_ELSE
1375 EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 1, 0x500, 0, 0, IRQS_DISABLED
1376 EXCEPTION_PROLOG_2_REAL hardware_interrupt_common, EXC_STD, 1
1377ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206)
1378EXC_REAL_END(hardware_interrupt, 0x500, 0x100)
1379
1380EXC_VIRT_BEGIN(hardware_interrupt, 0x4500, 0x100)
1381 EXCEPTION_PROLOG_0 PACA_EXGEN
1382BEGIN_FTR_SECTION
1383 EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, 0x500, 0, 0, IRQS_DISABLED
1384 EXCEPTION_PROLOG_2_VIRT hardware_interrupt_common, EXC_HV
1385FTR_SECTION_ELSE
1386 EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 1, 0x500, 0, 0, IRQS_DISABLED
1387 EXCEPTION_PROLOG_2_VIRT hardware_interrupt_common, EXC_STD
1388ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE)
1389EXC_VIRT_END(hardware_interrupt, 0x4500, 0x100)
1390
1391TRAMP_KVM(PACA_EXGEN, 0x500)
1392TRAMP_KVM_HV(PACA_EXGEN, 0x500)
1393EXC_COMMON_ASYNC(hardware_interrupt_common, 0x500, do_IRQ)
1394
1395
1396EXC_REAL_BEGIN(alignment, 0x600, 0x100)
1397 EXCEPTION_PROLOG_0 PACA_EXGEN
1398 EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 1, 0x600, 1, 1, 0
1399 EXCEPTION_PROLOG_2_REAL alignment_common, EXC_STD, 1
1400EXC_REAL_END(alignment, 0x600, 0x100)
1401
1402EXC_VIRT_BEGIN(alignment, 0x4600, 0x100)
1403 EXCEPTION_PROLOG_0 PACA_EXGEN
1404 EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 0, 0x600, 1, 1, 0
1405 EXCEPTION_PROLOG_2_VIRT alignment_common, EXC_STD
1406EXC_VIRT_END(alignment, 0x4600, 0x100)
1407
1408TRAMP_KVM(PACA_EXGEN, 0x600)
1409EXC_COMMON_BEGIN(alignment_common)
1410 EXCEPTION_COMMON(PACA_EXGEN, 0x600)
1411 ld r3,PACA_EXGEN+EX_DAR(r13)
1412 lwz r4,PACA_EXGEN+EX_DSISR(r13)
1413 std r3,_DAR(r1)
1414 std r4,_DSISR(r1)
1415 bl save_nvgprs
1416 RECONCILE_IRQ_STATE(r10, r11)
1417 addi r3,r1,STACK_FRAME_OVERHEAD
1418 bl alignment_exception
1419 b ret_from_except
1420
1421
1422EXC_REAL(program_check, 0x700, 0x100)
1423EXC_VIRT(program_check, 0x4700, 0x100, 0x700)
1424TRAMP_KVM(PACA_EXGEN, 0x700)
1425EXC_COMMON_BEGIN(program_check_common)
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435 andi. r10,r12,MSR_PR
1436 bne 2f
1437
1438 andis. r10,r12,(SRR1_PROGTM)@h
1439 bne 1f
1440
1441 cmpdi r1,-INT_FRAME_SIZE
1442 blt 2f
1443
1444
14451: andi. r10,r12,MSR_PR
1446
1447 mr r10,r1
1448 ld r1,PACAEMERGSP(r13)
1449 subi r1,r1,INT_FRAME_SIZE
1450 b 3f
14512:
1452 EXCEPTION_COMMON(PACA_EXGEN, 0x700)
1453 bl save_nvgprs
1454 RECONCILE_IRQ_STATE(r10, r11)
1455 addi r3,r1,STACK_FRAME_OVERHEAD
1456 bl program_check_exception
1457 b ret_from_except
1458
1459
1460EXC_REAL(fp_unavailable, 0x800, 0x100)
1461EXC_VIRT(fp_unavailable, 0x4800, 0x100, 0x800)
1462TRAMP_KVM(PACA_EXGEN, 0x800)
1463EXC_COMMON_BEGIN(fp_unavailable_common)
1464 EXCEPTION_COMMON(PACA_EXGEN, 0x800)
1465 bne 1f
1466 bl save_nvgprs
1467 RECONCILE_IRQ_STATE(r10, r11)
1468 addi r3,r1,STACK_FRAME_OVERHEAD
1469 bl kernel_fp_unavailable_exception
1470 BUG_OPCODE
14711:
1472#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1473BEGIN_FTR_SECTION
1474
1475
1476
1477 rldicl. r0, r12, (64-MSR_TS_LG), (64-2)
1478 bne- 2f
1479END_FTR_SECTION_IFSET(CPU_FTR_TM)
1480#endif
1481 bl load_up_fpu
1482 b fast_exception_return
1483#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
14842:
1485 bl save_nvgprs
1486 RECONCILE_IRQ_STATE(r10, r11)
1487 addi r3,r1,STACK_FRAME_OVERHEAD
1488 bl fp_unavailable_tm
1489 b ret_from_except
1490#endif
1491
1492
1493EXC_REAL_OOL_MASKABLE(decrementer, 0x900, 0x80, IRQS_DISABLED)
1494EXC_VIRT_MASKABLE(decrementer, 0x4900, 0x80, 0x900, IRQS_DISABLED)
1495TRAMP_KVM(PACA_EXGEN, 0x900)
1496EXC_COMMON_ASYNC(decrementer_common, 0x900, timer_interrupt)
1497
1498
1499EXC_REAL_HV(hdecrementer, 0x980, 0x80)
1500EXC_VIRT_HV(hdecrementer, 0x4980, 0x80, 0x980)
1501TRAMP_KVM_HV(PACA_EXGEN, 0x980)
1502EXC_COMMON(hdecrementer_common, 0x980, hdec_interrupt)
1503
1504
1505EXC_REAL_MASKABLE(doorbell_super, 0xa00, 0x100, IRQS_DISABLED)
1506EXC_VIRT_MASKABLE(doorbell_super, 0x4a00, 0x100, 0xa00, IRQS_DISABLED)
1507TRAMP_KVM(PACA_EXGEN, 0xa00)
1508#ifdef CONFIG_PPC_DOORBELL
1509EXC_COMMON_ASYNC(doorbell_super_common, 0xa00, doorbell_exception)
1510#else
1511EXC_COMMON_ASYNC(doorbell_super_common, 0xa00, unknown_exception)
1512#endif
1513
1514
1515EXC_REAL(trap_0b, 0xb00, 0x100)
1516EXC_VIRT(trap_0b, 0x4b00, 0x100, 0xb00)
1517TRAMP_KVM(PACA_EXGEN, 0xb00)
1518EXC_COMMON(trap_0b_common, 0xb00, unknown_exception)
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556.macro SYSTEM_CALL virt
1557#ifdef CONFIG_KVM_BOOK3S_64_HANDLER
1558
1559
1560
1561
1562
1563
1564
1565
1566 mtctr r13
1567 GET_PACA(r13)
1568 std r10,PACA_EXGEN+EX_R10(r13)
1569 INTERRUPT_TO_KERNEL
1570 KVMTEST EXC_STD 0xc00
1571 mfctr r9
1572#else
1573 mr r9,r13
1574 GET_PACA(r13)
1575 INTERRUPT_TO_KERNEL
1576#endif
1577
1578#ifdef CONFIG_PPC_FAST_ENDIAN_SWITCH
1579BEGIN_FTR_SECTION
1580 cmpdi r0,0x1ebe
1581 beq- 1f
1582END_FTR_SECTION_IFSET(CPU_FTR_REAL_LE)
1583#endif
1584
1585
1586 mfspr r11,SPRN_SRR0
1587 mfspr r12,SPRN_SRR1
1588
1589 HMT_MEDIUM
1590
1591 .if ! \virt
1592 __LOAD_HANDLER(r10, system_call_common)
1593 mtspr SPRN_SRR0,r10
1594 ld r10,PACAKMSR(r13)
1595 mtspr SPRN_SRR1,r10
1596 RFI_TO_KERNEL
1597 b .
1598 .else
1599 li r10,MSR_RI
1600 mtmsrd r10,1
1601#ifdef CONFIG_RELOCATABLE
1602 __LOAD_HANDLER(r10, system_call_common)
1603 mtctr r10
1604 bctr
1605#else
1606 b system_call_common
1607#endif
1608 .endif
1609
1610#ifdef CONFIG_PPC_FAST_ENDIAN_SWITCH
1611
16121: mfspr r12,SPRN_SRR1
1613 xori r12,r12,MSR_LE
1614 mtspr SPRN_SRR1,r12
1615 mr r13,r9
1616 RFI_TO_USER
1617 b .
1618#endif
1619.endm
1620
1621EXC_REAL_BEGIN(system_call, 0xc00, 0x100)
1622 SYSTEM_CALL 0
1623EXC_REAL_END(system_call, 0xc00, 0x100)
1624
1625EXC_VIRT_BEGIN(system_call, 0x4c00, 0x100)
1626 SYSTEM_CALL 1
1627EXC_VIRT_END(system_call, 0x4c00, 0x100)
1628
1629#ifdef CONFIG_KVM_BOOK3S_64_HANDLER
1630
1631
1632
1633
1634
1635
1636
1637TRAMP_KVM_BEGIN(do_kvm_0xc00)
1638
1639
1640
1641
1642
1643 OPT_GET_SPR(r10, SPRN_PPR, CPU_FTR_HAS_PPR)
1644 HMT_MEDIUM
1645 OPT_SAVE_REG_TO_PACA(PACA_EXGEN+EX_PPR, r10, CPU_FTR_HAS_PPR)
1646 mfctr r10
1647 SET_SCRATCH0(r10)
1648 std r9,PACA_EXGEN+EX_R9(r13)
1649 mfcr r9
1650 KVM_HANDLER PACA_EXGEN, EXC_STD, 0xc00, 0
1651#endif
1652
1653
1654EXC_REAL(single_step, 0xd00, 0x100)
1655EXC_VIRT(single_step, 0x4d00, 0x100, 0xd00)
1656TRAMP_KVM(PACA_EXGEN, 0xd00)
1657EXC_COMMON(single_step_common, 0xd00, single_step_exception)
1658
1659EXC_REAL_OOL_HV(h_data_storage, 0xe00, 0x20)
1660EXC_VIRT_OOL_HV(h_data_storage, 0x4e00, 0x20, 0xe00)
1661TRAMP_KVM_HV_SKIP(PACA_EXGEN, 0xe00)
1662EXC_COMMON_BEGIN(h_data_storage_common)
1663 mfspr r10,SPRN_HDAR
1664 std r10,PACA_EXGEN+EX_DAR(r13)
1665 mfspr r10,SPRN_HDSISR
1666 stw r10,PACA_EXGEN+EX_DSISR(r13)
1667 EXCEPTION_COMMON(PACA_EXGEN, 0xe00)
1668 bl save_nvgprs
1669 RECONCILE_IRQ_STATE(r10, r11)
1670 addi r3,r1,STACK_FRAME_OVERHEAD
1671BEGIN_MMU_FTR_SECTION
1672 ld r4,PACA_EXGEN+EX_DAR(r13)
1673 lwz r5,PACA_EXGEN+EX_DSISR(r13)
1674 std r4,_DAR(r1)
1675 std r5,_DSISR(r1)
1676 li r5,SIGSEGV
1677 bl bad_page_fault
1678MMU_FTR_SECTION_ELSE
1679 bl unknown_exception
1680ALT_MMU_FTR_SECTION_END_IFSET(MMU_FTR_TYPE_RADIX)
1681 b ret_from_except
1682
1683
1684EXC_REAL_OOL_HV(h_instr_storage, 0xe20, 0x20)
1685EXC_VIRT_OOL_HV(h_instr_storage, 0x4e20, 0x20, 0xe20)
1686TRAMP_KVM_HV(PACA_EXGEN, 0xe20)
1687EXC_COMMON(h_instr_storage_common, 0xe20, unknown_exception)
1688
1689
1690EXC_REAL_OOL_HV(emulation_assist, 0xe40, 0x20)
1691EXC_VIRT_OOL_HV(emulation_assist, 0x4e40, 0x20, 0xe40)
1692TRAMP_KVM_HV(PACA_EXGEN, 0xe40)
1693EXC_COMMON(emulation_assist_common, 0xe40, emulation_assist_interrupt)
1694
1695
1696
1697
1698
1699
1700
1701EXC_REAL_BEGIN(hmi_exception, 0xe60, 0x20)
1702 EXCEPTION_PROLOG_0 PACA_EXGEN
1703 b hmi_exception_early
1704EXC_REAL_END(hmi_exception, 0xe60, 0x20)
1705EXC_VIRT_NONE(0x4e60, 0x20)
1706TRAMP_KVM_HV(PACA_EXGEN, 0xe60)
1707TRAMP_REAL_BEGIN(hmi_exception_early)
1708 EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, 0xe60, 0, 0, 0
1709 mfctr r10
1710 BRANCH_TO_C000(r11, hmi_exception_early_common)
1711
1712EXC_COMMON_BEGIN(hmi_exception_early_common)
1713 mtctr r10
1714 mfspr r11,SPRN_HSRR0
1715 mfspr r12,SPRN_HSRR1
1716 mr r10,r1
1717 ld r1,PACAEMERGSP(r13)
1718 subi r1,r1,INT_FRAME_SIZE
1719 EXCEPTION_PROLOG_COMMON_1()
1720
1721 EXCEPTION_PROLOG_COMMON_2(PACA_EXGEN)
1722 EXCEPTION_PROLOG_COMMON_3(0xe60)
1723 addi r3,r1,STACK_FRAME_OVERHEAD
1724 bl hmi_exception_realmode
1725 cmpdi cr0,r3,0
1726 bne 1f
1727
1728 EXCEPTION_RESTORE_REGS EXC_HV
1729 HRFI_TO_USER_OR_KERNEL
1730
17311:
1732
1733
1734
1735
1736 EXCEPTION_RESTORE_REGS EXC_HV
1737 EXCEPTION_PROLOG_0 PACA_EXGEN
1738 EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, 0xe60, 0, 0, IRQS_DISABLED
1739 EXCEPTION_PROLOG_2_REAL hmi_exception_common, EXC_HV, 1
1740
1741EXC_COMMON_BEGIN(hmi_exception_common)
1742 EXCEPTION_COMMON(PACA_EXGEN, 0xe60)
1743 FINISH_NAP
1744 bl save_nvgprs
1745 RECONCILE_IRQ_STATE(r10, r11)
1746 RUNLATCH_ON
1747 addi r3,r1,STACK_FRAME_OVERHEAD
1748 bl handle_hmi_exception
1749 b ret_from_except
1750
1751EXC_REAL_OOL_MASKABLE_HV(h_doorbell, 0xe80, 0x20, IRQS_DISABLED)
1752EXC_VIRT_OOL_MASKABLE_HV(h_doorbell, 0x4e80, 0x20, 0xe80, IRQS_DISABLED)
1753TRAMP_KVM_HV(PACA_EXGEN, 0xe80)
1754#ifdef CONFIG_PPC_DOORBELL
1755EXC_COMMON_ASYNC(h_doorbell_common, 0xe80, doorbell_exception)
1756#else
1757EXC_COMMON_ASYNC(h_doorbell_common, 0xe80, unknown_exception)
1758#endif
1759
1760
1761EXC_REAL_OOL_MASKABLE_HV(h_virt_irq, 0xea0, 0x20, IRQS_DISABLED)
1762EXC_VIRT_OOL_MASKABLE_HV(h_virt_irq, 0x4ea0, 0x20, 0xea0, IRQS_DISABLED)
1763TRAMP_KVM_HV(PACA_EXGEN, 0xea0)
1764EXC_COMMON_ASYNC(h_virt_irq_common, 0xea0, do_IRQ)
1765
1766
1767EXC_REAL_NONE(0xec0, 0x20)
1768EXC_VIRT_NONE(0x4ec0, 0x20)
1769EXC_REAL_NONE(0xee0, 0x20)
1770EXC_VIRT_NONE(0x4ee0, 0x20)
1771
1772
1773EXC_REAL_OOL_MASKABLE(performance_monitor, 0xf00, 0x20, IRQS_PMI_DISABLED)
1774EXC_VIRT_OOL_MASKABLE(performance_monitor, 0x4f00, 0x20, 0xf00, IRQS_PMI_DISABLED)
1775TRAMP_KVM(PACA_EXGEN, 0xf00)
1776EXC_COMMON_ASYNC(performance_monitor_common, 0xf00, performance_monitor_exception)
1777
1778
1779EXC_REAL_OOL(altivec_unavailable, 0xf20, 0x20)
1780EXC_VIRT_OOL(altivec_unavailable, 0x4f20, 0x20, 0xf20)
1781TRAMP_KVM(PACA_EXGEN, 0xf20)
1782EXC_COMMON_BEGIN(altivec_unavailable_common)
1783 EXCEPTION_COMMON(PACA_EXGEN, 0xf20)
1784#ifdef CONFIG_ALTIVEC
1785BEGIN_FTR_SECTION
1786 beq 1f
1787#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1788 BEGIN_FTR_SECTION_NESTED(69)
1789
1790
1791
1792 rldicl. r0, r12, (64-MSR_TS_LG), (64-2)
1793 bne- 2f
1794 END_FTR_SECTION_NESTED(CPU_FTR_TM, CPU_FTR_TM, 69)
1795#endif
1796 bl load_up_altivec
1797 b fast_exception_return
1798#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
17992:
1800 bl save_nvgprs
1801 RECONCILE_IRQ_STATE(r10, r11)
1802 addi r3,r1,STACK_FRAME_OVERHEAD
1803 bl altivec_unavailable_tm
1804 b ret_from_except
1805#endif
18061:
1807END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
1808#endif
1809 bl save_nvgprs
1810 RECONCILE_IRQ_STATE(r10, r11)
1811 addi r3,r1,STACK_FRAME_OVERHEAD
1812 bl altivec_unavailable_exception
1813 b ret_from_except
1814
1815
1816EXC_REAL_OOL(vsx_unavailable, 0xf40, 0x20)
1817EXC_VIRT_OOL(vsx_unavailable, 0x4f40, 0x20, 0xf40)
1818TRAMP_KVM(PACA_EXGEN, 0xf40)
1819EXC_COMMON_BEGIN(vsx_unavailable_common)
1820 EXCEPTION_COMMON(PACA_EXGEN, 0xf40)
1821#ifdef CONFIG_VSX
1822BEGIN_FTR_SECTION
1823 beq 1f
1824#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1825 BEGIN_FTR_SECTION_NESTED(69)
1826
1827
1828
1829 rldicl. r0, r12, (64-MSR_TS_LG), (64-2)
1830 bne- 2f
1831 END_FTR_SECTION_NESTED(CPU_FTR_TM, CPU_FTR_TM, 69)
1832#endif
1833 b load_up_vsx
1834#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
18352:
1836 bl save_nvgprs
1837 RECONCILE_IRQ_STATE(r10, r11)
1838 addi r3,r1,STACK_FRAME_OVERHEAD
1839 bl vsx_unavailable_tm
1840 b ret_from_except
1841#endif
18421:
1843END_FTR_SECTION_IFSET(CPU_FTR_VSX)
1844#endif
1845 bl save_nvgprs
1846 RECONCILE_IRQ_STATE(r10, r11)
1847 addi r3,r1,STACK_FRAME_OVERHEAD
1848 bl vsx_unavailable_exception
1849 b ret_from_except
1850
1851
1852EXC_REAL_OOL(facility_unavailable, 0xf60, 0x20)
1853EXC_VIRT_OOL(facility_unavailable, 0x4f60, 0x20, 0xf60)
1854TRAMP_KVM(PACA_EXGEN, 0xf60)
1855EXC_COMMON(facility_unavailable_common, 0xf60, facility_unavailable_exception)
1856
1857
1858EXC_REAL_OOL_HV(h_facility_unavailable, 0xf80, 0x20)
1859EXC_VIRT_OOL_HV(h_facility_unavailable, 0x4f80, 0x20, 0xf80)
1860TRAMP_KVM_HV(PACA_EXGEN, 0xf80)
1861EXC_COMMON(h_facility_unavailable_common, 0xf80, facility_unavailable_exception)
1862
1863
1864EXC_REAL_NONE(0xfa0, 0x20)
1865EXC_VIRT_NONE(0x4fa0, 0x20)
1866EXC_REAL_NONE(0xfc0, 0x20)
1867EXC_VIRT_NONE(0x4fc0, 0x20)
1868EXC_REAL_NONE(0xfe0, 0x20)
1869EXC_VIRT_NONE(0x4fe0, 0x20)
1870
1871EXC_REAL_NONE(0x1000, 0x100)
1872EXC_VIRT_NONE(0x5000, 0x100)
1873EXC_REAL_NONE(0x1100, 0x100)
1874EXC_VIRT_NONE(0x5100, 0x100)
1875
1876#ifdef CONFIG_CBE_RAS
1877EXC_REAL_HV(cbe_system_error, 0x1200, 0x100)
1878EXC_VIRT_NONE(0x5200, 0x100)
1879TRAMP_KVM_HV_SKIP(PACA_EXGEN, 0x1200)
1880EXC_COMMON(cbe_system_error_common, 0x1200, cbe_system_error_exception)
1881#else
1882EXC_REAL_NONE(0x1200, 0x100)
1883EXC_VIRT_NONE(0x5200, 0x100)
1884#endif
1885
1886
1887EXC_REAL(instruction_breakpoint, 0x1300, 0x100)
1888EXC_VIRT(instruction_breakpoint, 0x5300, 0x100, 0x1300)
1889TRAMP_KVM_SKIP(PACA_EXGEN, 0x1300)
1890EXC_COMMON(instruction_breakpoint_common, 0x1300, instruction_breakpoint_exception)
1891
1892EXC_REAL_NONE(0x1400, 0x100)
1893EXC_VIRT_NONE(0x5400, 0x100)
1894
1895EXC_REAL_BEGIN(denorm_exception_hv, 0x1500, 0x100)
1896 EXCEPTION_PROLOG_0 PACA_EXGEN
1897 EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 0, 0x1500, 0, 0, 0
1898
1899#ifdef CONFIG_PPC_DENORMALISATION
1900 mfspr r10,SPRN_HSRR1
1901 andis. r10,r10,(HSRR1_DENORM)@h
1902 bne+ denorm_assist
1903#endif
1904
1905 KVMTEST EXC_HV 0x1500
1906 EXCEPTION_PROLOG_2_REAL denorm_common, EXC_HV, 1
1907EXC_REAL_END(denorm_exception_hv, 0x1500, 0x100)
1908
1909#ifdef CONFIG_PPC_DENORMALISATION
1910EXC_VIRT_BEGIN(denorm_exception, 0x5500, 0x100)
1911 b exc_real_0x1500_denorm_exception_hv
1912EXC_VIRT_END(denorm_exception, 0x5500, 0x100)
1913#else
1914EXC_VIRT_NONE(0x5500, 0x100)
1915#endif
1916
1917TRAMP_KVM_HV(PACA_EXGEN, 0x1500)
1918
1919#ifdef CONFIG_PPC_DENORMALISATION
1920TRAMP_REAL_BEGIN(denorm_assist)
1921BEGIN_FTR_SECTION
1922
1923
1924
1925
1926 mfmsr r10
1927 ori r10,r10,(MSR_FP|MSR_FE0|MSR_FE1)
1928 xori r10,r10,(MSR_FE0|MSR_FE1)
1929 mtmsrd r10
1930 sync
1931
1932 .Lreg=0
1933 .rept 32
1934 fmr .Lreg,.Lreg
1935 .Lreg=.Lreg+1
1936 .endr
1937
1938FTR_SECTION_ELSE
1939
1940
1941
1942
1943 mfmsr r10
1944 oris r10,r10,MSR_VSX@h
1945 mtmsrd r10
1946 sync
1947
1948 .Lreg=0
1949 .rept 32
1950 XVCPSGNDP(.Lreg,.Lreg,.Lreg)
1951 .Lreg=.Lreg+1
1952 .endr
1953
1954ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_206)
1955
1956BEGIN_FTR_SECTION
1957 b denorm_done
1958END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
1959
1960
1961
1962
1963 .Lreg=32
1964 .rept 32
1965 XVCPSGNDP(.Lreg,.Lreg,.Lreg)
1966 .Lreg=.Lreg+1
1967 .endr
1968
1969denorm_done:
1970 mfspr r11,SPRN_HSRR0
1971 subi r11,r11,4
1972 mtspr SPRN_HSRR0,r11
1973 mtcrf 0x80,r9
1974 ld r9,PACA_EXGEN+EX_R9(r13)
1975 RESTORE_PPR_PACA(PACA_EXGEN, r10)
1976BEGIN_FTR_SECTION
1977 ld r10,PACA_EXGEN+EX_CFAR(r13)
1978 mtspr SPRN_CFAR,r10
1979END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
1980 ld r10,PACA_EXGEN+EX_R10(r13)
1981 ld r11,PACA_EXGEN+EX_R11(r13)
1982 ld r12,PACA_EXGEN+EX_R12(r13)
1983 ld r13,PACA_EXGEN+EX_R13(r13)
1984 HRFI_TO_UNKNOWN
1985 b .
1986#endif
1987
1988EXC_COMMON(denorm_common, 0x1500, unknown_exception)
1989
1990
1991#ifdef CONFIG_CBE_RAS
1992EXC_REAL_HV(cbe_maintenance, 0x1600, 0x100)
1993EXC_VIRT_NONE(0x5600, 0x100)
1994TRAMP_KVM_HV_SKIP(PACA_EXGEN, 0x1600)
1995EXC_COMMON(cbe_maintenance_common, 0x1600, cbe_maintenance_exception)
1996#else
1997EXC_REAL_NONE(0x1600, 0x100)
1998EXC_VIRT_NONE(0x5600, 0x100)
1999#endif
2000
2001
2002EXC_REAL(altivec_assist, 0x1700, 0x100)
2003EXC_VIRT(altivec_assist, 0x5700, 0x100, 0x1700)
2004TRAMP_KVM(PACA_EXGEN, 0x1700)
2005#ifdef CONFIG_ALTIVEC
2006EXC_COMMON(altivec_assist_common, 0x1700, altivec_assist_exception)
2007#else
2008EXC_COMMON(altivec_assist_common, 0x1700, unknown_exception)
2009#endif
2010
2011
2012#ifdef CONFIG_CBE_RAS
2013EXC_REAL_HV(cbe_thermal, 0x1800, 0x100)
2014EXC_VIRT_NONE(0x5800, 0x100)
2015TRAMP_KVM_HV_SKIP(PACA_EXGEN, 0x1800)
2016EXC_COMMON(cbe_thermal_common, 0x1800, cbe_thermal_exception)
2017#else
2018EXC_REAL_NONE(0x1800, 0x100)
2019EXC_VIRT_NONE(0x5800, 0x100)
2020#endif
2021
2022#ifdef CONFIG_PPC_WATCHDOG
2023
2024#define MASKED_DEC_HANDLER_LABEL 3f
2025
2026#define MASKED_DEC_HANDLER(_H) \
20273: \
2028 std r12,PACA_EXGEN+EX_R12(r13); \
2029 GET_SCRATCH0(r10); \
2030 std r10,PACA_EXGEN+EX_R13(r13); \
2031 EXCEPTION_PROLOG_2_REAL soft_nmi_common, _H, 1
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042EXC_COMMON_BEGIN(soft_nmi_common)
2043 mr r10,r1
2044 ld r1,PACAEMERGSP(r13)
2045 subi r1,r1,INT_FRAME_SIZE
2046 EXCEPTION_COMMON_STACK(PACA_EXGEN, 0x900)
2047 bl save_nvgprs
2048 RECONCILE_IRQ_STATE(r10, r11)
2049 addi r3,r1,STACK_FRAME_OVERHEAD
2050 bl soft_nmi_interrupt
2051 b ret_from_except
2052
2053#else
2054#define MASKED_DEC_HANDLER_LABEL 2f
2055#define MASKED_DEC_HANDLER(_H)
2056#endif
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068.macro MASKED_INTERRUPT hsrr
2069 .if \hsrr
2070masked_Hinterrupt:
2071 .else
2072masked_interrupt:
2073 .endif
2074 std r11,PACA_EXGEN+EX_R11(r13)
2075 lbz r11,PACAIRQHAPPENED(r13)
2076 or r11,r11,r10
2077 stb r11,PACAIRQHAPPENED(r13)
2078 cmpwi r10,PACA_IRQ_DEC
2079 bne 1f
2080 lis r10,0x7fff
2081 ori r10,r10,0xffff
2082 mtspr SPRN_DEC,r10
2083 b MASKED_DEC_HANDLER_LABEL
20841: andi. r10,r10,PACA_IRQ_MUST_HARD_MASK
2085 beq 2f
2086 .if \hsrr
2087 mfspr r10,SPRN_HSRR1
2088 xori r10,r10,MSR_EE
2089 mtspr SPRN_HSRR1,r10
2090 .else
2091 mfspr r10,SPRN_SRR1
2092 xori r10,r10,MSR_EE
2093 mtspr SPRN_SRR1,r10
2094 .endif
2095 ori r11,r11,PACA_IRQ_HARD_DIS
2096 stb r11,PACAIRQHAPPENED(r13)
20972:
2098 mtcrf 0x80,r9
2099 std r1,PACAR1(r13)
2100 ld r9,PACA_EXGEN+EX_R9(r13)
2101 ld r10,PACA_EXGEN+EX_R10(r13)
2102 ld r11,PACA_EXGEN+EX_R11(r13)
2103
2104 .if \hsrr
2105 HRFI_TO_KERNEL
2106 .else
2107 RFI_TO_KERNEL
2108 .endif
2109 b .
2110 MASKED_DEC_HANDLER(\hsrr\())
2111.endm
2112
2113TRAMP_REAL_BEGIN(stf_barrier_fallback)
2114 std r9,PACA_EXRFI+EX_R9(r13)
2115 std r10,PACA_EXRFI+EX_R10(r13)
2116 sync
2117 ld r9,PACA_EXRFI+EX_R9(r13)
2118 ld r10,PACA_EXRFI+EX_R10(r13)
2119 ori 31,31,0
2120 .rept 14
2121 b 1f
21221:
2123 .endr
2124 blr
2125
2126TRAMP_REAL_BEGIN(rfi_flush_fallback)
2127 SET_SCRATCH0(r13);
2128 GET_PACA(r13);
2129 std r1,PACA_EXRFI+EX_R12(r13)
2130 ld r1,PACAKSAVE(r13)
2131 std r9,PACA_EXRFI+EX_R9(r13)
2132 std r10,PACA_EXRFI+EX_R10(r13)
2133 std r11,PACA_EXRFI+EX_R11(r13)
2134 mfctr r9
2135 ld r10,PACA_RFI_FLUSH_FALLBACK_AREA(r13)
2136 ld r11,PACA_L1D_FLUSH_SIZE(r13)
2137 srdi r11,r11,(7 + 3)
2138 mtctr r11
2139 DCBT_BOOK3S_STOP_ALL_STREAM_IDS(r11)
2140
2141
2142 sync
2143
2144
2145
2146
2147
2148
21491:
2150 ld r11,(0x80 + 8)*0(r10)
2151 ld r11,(0x80 + 8)*1(r10)
2152 ld r11,(0x80 + 8)*2(r10)
2153 ld r11,(0x80 + 8)*3(r10)
2154 ld r11,(0x80 + 8)*4(r10)
2155 ld r11,(0x80 + 8)*5(r10)
2156 ld r11,(0x80 + 8)*6(r10)
2157 ld r11,(0x80 + 8)*7(r10)
2158 addi r10,r10,0x80*8
2159 bdnz 1b
2160
2161 mtctr r9
2162 ld r9,PACA_EXRFI+EX_R9(r13)
2163 ld r10,PACA_EXRFI+EX_R10(r13)
2164 ld r11,PACA_EXRFI+EX_R11(r13)
2165 ld r1,PACA_EXRFI+EX_R12(r13)
2166 GET_SCRATCH0(r13);
2167 rfid
2168
2169TRAMP_REAL_BEGIN(hrfi_flush_fallback)
2170 SET_SCRATCH0(r13);
2171 GET_PACA(r13);
2172 std r1,PACA_EXRFI+EX_R12(r13)
2173 ld r1,PACAKSAVE(r13)
2174 std r9,PACA_EXRFI+EX_R9(r13)
2175 std r10,PACA_EXRFI+EX_R10(r13)
2176 std r11,PACA_EXRFI+EX_R11(r13)
2177 mfctr r9
2178 ld r10,PACA_RFI_FLUSH_FALLBACK_AREA(r13)
2179 ld r11,PACA_L1D_FLUSH_SIZE(r13)
2180 srdi r11,r11,(7 + 3)
2181 mtctr r11
2182 DCBT_BOOK3S_STOP_ALL_STREAM_IDS(r11)
2183
2184
2185 sync
2186
2187
2188
2189
2190
2191
21921:
2193 ld r11,(0x80 + 8)*0(r10)
2194 ld r11,(0x80 + 8)*1(r10)
2195 ld r11,(0x80 + 8)*2(r10)
2196 ld r11,(0x80 + 8)*3(r10)
2197 ld r11,(0x80 + 8)*4(r10)
2198 ld r11,(0x80 + 8)*5(r10)
2199 ld r11,(0x80 + 8)*6(r10)
2200 ld r11,(0x80 + 8)*7(r10)
2201 addi r10,r10,0x80*8
2202 bdnz 1b
2203
2204 mtctr r9
2205 ld r9,PACA_EXRFI+EX_R9(r13)
2206 ld r10,PACA_EXRFI+EX_R10(r13)
2207 ld r11,PACA_EXRFI+EX_R11(r13)
2208 ld r1,PACA_EXRFI+EX_R12(r13)
2209 GET_SCRATCH0(r13);
2210 hrfid
2211
2212
2213
2214
2215
2216
2217USE_FIXED_SECTION(virt_trampolines)
2218 MASKED_INTERRUPT EXC_STD
2219 MASKED_INTERRUPT EXC_HV
2220
2221#ifdef CONFIG_KVM_BOOK3S_64_HANDLER
2222TRAMP_REAL_BEGIN(kvmppc_skip_interrupt)
2223
2224
2225
2226
2227 mfspr r13, SPRN_SRR0
2228 addi r13, r13, 4
2229 mtspr SPRN_SRR0, r13
2230 GET_SCRATCH0(r13)
2231 RFI_TO_KERNEL
2232 b .
2233
2234TRAMP_REAL_BEGIN(kvmppc_skip_Hinterrupt)
2235
2236
2237
2238
2239 mfspr r13, SPRN_HSRR0
2240 addi r13, r13, 4
2241 mtspr SPRN_HSRR0, r13
2242 GET_SCRATCH0(r13)
2243 HRFI_TO_KERNEL
2244 b .
2245#endif
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273EXC_COMMON_BEGIN(ppc64_runlatch_on_trampoline)
2274 b __ppc64_runlatch_on
2275
2276USE_FIXED_SECTION(virt_trampolines)
2277
2278
2279
2280
2281
2282
2283
2284 .align 7
2285 .globl __end_interrupts
2286__end_interrupts:
2287DEFINE_FIXED_SYMBOL(__end_interrupts)
2288
2289#ifdef CONFIG_PPC_970_NAP
2290EXC_COMMON_BEGIN(power4_fixup_nap)
2291 andc r9,r9,r10
2292 std r9,TI_LOCAL_FLAGS(r11)
2293 ld r10,_LINK(r1)
2294 std r10,_NIP(r1)
2295 blr
2296#endif
2297
2298CLOSE_FIXED_SECTION(real_vectors);
2299CLOSE_FIXED_SECTION(real_trampolines);
2300CLOSE_FIXED_SECTION(virt_vectors);
2301CLOSE_FIXED_SECTION(virt_trampolines);
2302
2303USE_TEXT_SECTION()
2304
2305
2306
2307
2308 .balign IFETCH_ALIGN_BYTES
2309do_hash_page:
2310#ifdef CONFIG_PPC_BOOK3S_64
2311 lis r0,(DSISR_BAD_FAULT_64S | DSISR_DABRMATCH | DSISR_KEYFAULT)@h
2312 ori r0,r0,DSISR_BAD_FAULT_64S@l
2313 and. r0,r4,r0
2314 bne- handle_page_fault
2315 ld r11, PACA_THREAD_INFO(r13)
2316 lwz r0,TI_PREEMPT(r11)
2317 andis. r0,r0,NMI_MASK@h
2318 bne 77f
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328 mr r4,r12
2329 ld r6,_DSISR(r1)
2330 bl __hash_page
2331 cmpdi r3,0
2332
2333
2334 beq fast_exc_return_irq
2335
2336
2337 blt- 13f
2338
2339
2340 ld r4,_DSISR(r1)
2341#endif
2342
2343
2344handle_page_fault:
234511: andis. r0,r4,DSISR_DABRMATCH@h
2346 bne- handle_dabr_fault
2347 ld r4,_DAR(r1)
2348 ld r5,_DSISR(r1)
2349 addi r3,r1,STACK_FRAME_OVERHEAD
2350 bl do_page_fault
2351 cmpdi r3,0
2352 beq+ ret_from_except_lite
2353 bl save_nvgprs
2354 mr r5,r3
2355 addi r3,r1,STACK_FRAME_OVERHEAD
2356 lwz r4,_DAR(r1)
2357 bl bad_page_fault
2358 b ret_from_except
2359
2360
2361handle_dabr_fault:
2362 bl save_nvgprs
2363 ld r4,_DAR(r1)
2364 ld r5,_DSISR(r1)
2365 addi r3,r1,STACK_FRAME_OVERHEAD
2366 bl do_break
2367
2368
2369
2370
2371
2372 b ret_from_except
2373
2374
2375#ifdef CONFIG_PPC_BOOK3S_64
2376
2377
2378
237913: bl save_nvgprs
2380 mr r5,r3
2381 addi r3,r1,STACK_FRAME_OVERHEAD
2382 ld r4,_DAR(r1)
2383 bl low_hash_fault
2384 b ret_from_except
2385#endif
2386
2387
2388
2389
2390
2391
2392
2393
239477: bl save_nvgprs
2395 mr r4,r3
2396 addi r3,r1,STACK_FRAME_OVERHEAD
2397 li r5,SIGSEGV
2398 bl bad_page_fault
2399 b ret_from_except
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410h_doorbell_common_msgclr:
2411 LOAD_REG_IMMEDIATE(r3, PPC_DBELL_MSGTYPE << (63-36))
2412 PPC_MSGCLR(3)
2413 b h_doorbell_common
2414
2415doorbell_super_common_msgclr:
2416 LOAD_REG_IMMEDIATE(r3, PPC_DBELL_MSGTYPE << (63-36))
2417 PPC_MSGCLRP(3)
2418 b doorbell_super_common
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434_GLOBAL(__replay_interrupt)
2435
2436
2437
2438
2439 mfmsr r12
2440 LOAD_REG_ADDR(r11, replay_interrupt_return)
2441 mfcr r9
2442 ori r12,r12,MSR_EE
2443 cmpwi r3,0x900
2444 beq decrementer_common
2445 cmpwi r3,0x500
2446BEGIN_FTR_SECTION
2447 beq h_virt_irq_common
2448FTR_SECTION_ELSE
2449 beq hardware_interrupt_common
2450ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_300)
2451 cmpwi r3,0xf00
2452 beq performance_monitor_common
2453BEGIN_FTR_SECTION
2454 cmpwi r3,0xa00
2455 beq h_doorbell_common_msgclr
2456 cmpwi r3,0xe60
2457 beq hmi_exception_common
2458FTR_SECTION_ELSE
2459 cmpwi r3,0xa00
2460 beq doorbell_super_common_msgclr
2461ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE)
2462replay_interrupt_return:
2463 blr
2464
2465_ASM_NOKPROBE_SYMBOL(__replay_interrupt)
2466