1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21#include <linux/errno.h>
22#include <asm/unistd.h>
23#include <asm/processor.h>
24#include <asm/page.h>
25#include <asm/mmu.h>
26#include <asm/thread_info.h>
27#include <asm/ppc_asm.h>
28#include <asm/asm-offsets.h>
29#include <asm/cputable.h>
30#include <asm/firmware.h>
31#include <asm/bug.h>
32#include <asm/ptrace.h>
33#include <asm/irqflags.h>
34#include <asm/ftrace.h>
35#include <asm/hw_irq.h>
36#include <asm/context_tracking.h>
37
38
39
40
41 .section ".toc","aw"
42.SYS_CALL_TABLE:
43 .tc .sys_call_table[TC],.sys_call_table
44
45
46exception_marker:
47 .tc ID_EXC_MARKER[TC],STACK_FRAME_REGS_MARKER
48
49 .section ".text"
50 .align 7
51
52#undef SHOW_SYSCALLS
53
54 .globl system_call_common
55system_call_common:
56 andi. r10,r12,MSR_PR
57 mr r10,r1
58 addi r1,r1,-INT_FRAME_SIZE
59 beq- 1f
60 ld r1,PACAKSAVE(r13)
611: std r10,0(r1)
62 std r11,_NIP(r1)
63 std r12,_MSR(r1)
64 std r0,GPR0(r1)
65 std r10,GPR1(r1)
66 beq 2f
67 ACCOUNT_CPU_USER_ENTRY(r10, r11)
682: std r2,GPR2(r1)
69 std r3,GPR3(r1)
70 mfcr r2
71 std r4,GPR4(r1)
72 std r5,GPR5(r1)
73 std r6,GPR6(r1)
74 std r7,GPR7(r1)
75 std r8,GPR8(r1)
76 li r11,0
77 std r11,GPR9(r1)
78 std r11,GPR10(r1)
79 std r11,GPR11(r1)
80 std r11,GPR12(r1)
81 std r11,_XER(r1)
82 std r11,_CTR(r1)
83 std r9,GPR13(r1)
84 mflr r10
85
86
87
88
89 rldimi r2,r11,28,(63-28)
90 li r11,0xc01
91 std r10,_LINK(r1)
92 std r11,_TRAP(r1)
93 std r3,ORIG_GPR3(r1)
94 std r2,_CCR(r1)
95 ld r2,PACATOC(r13)
96 addi r9,r1,STACK_FRAME_OVERHEAD
97 ld r11,exception_marker@toc(r2)
98 std r11,-16(r9)
99
100BEGIN_FW_FTR_SECTION
101 beq 33f
102
103 ld r10,PACALPPACAPTR(r13)
104 ld r11,PACA_DTL_RIDX(r13)
105 addi r10,r10,LPPACA_DTLIDX
106 LDX_BE r10,0,r10
107 cmpd cr1,r11,r10
108 beq+ cr1,33f
109 bl .accumulate_stolen_time
110 REST_GPR(0,r1)
111 REST_4GPRS(3,r1)
112 REST_2GPRS(7,r1)
113 addi r9,r1,STACK_FRAME_OVERHEAD
11433:
115END_FW_FTR_SECTION_IFSET(FW_FEATURE_SPLPAR)
116#endif
117
118
119
120
121
122
123
124
125 lbz r10,PACASOFTIRQEN(r13)
126 xori r10,r10,1
1271: tdnei r10,0
128 EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,BUGFLAG_WARNING
129#endif
130
131#ifdef CONFIG_PPC_BOOK3E
132 wrteei 1
133#else
134 ld r11,PACAKMSR(r13)
135 ori r11,r11,MSR_EE
136 mtmsrd r11,1
137#endif
138
139
140
141
142 li r10,1
143 std r10,SOFTE(r1)
144
145#ifdef SHOW_SYSCALLS
146 bl .do_show_syscall
147 REST_GPR(0,r1)
148 REST_4GPRS(3,r1)
149 REST_2GPRS(7,r1)
150 addi r9,r1,STACK_FRAME_OVERHEAD
151#endif
152 CURRENT_THREAD_INFO(r11, r1)
153 ld r10,TI_FLAGS(r11)
154 andi. r11,r10,_TIF_SYSCALL_T_OR_A
155 bne syscall_dotrace
156.Lsyscall_dotrace_cont:
157 cmpldi 0,r0,NR_syscalls
158 bge- syscall_enosys
159
160system_call:
161
162
163
164
165 ld r11,.SYS_CALL_TABLE@toc(2)
166 andi. r10,r10,_TIF_32BIT
167 beq 15f
168 addi r11,r11,8
169 clrldi r3,r3,32
170 clrldi r4,r4,32
171 clrldi r5,r5,32
172 clrldi r6,r6,32
173 clrldi r7,r7,32
174 clrldi r8,r8,32
17515:
176 slwi r0,r0,4
177 ldx r10,r11,r0
178 mtctr r10
179 bctrl
180
181syscall_exit:
182 std r3,RESULT(r1)
183#ifdef SHOW_SYSCALLS
184 bl .do_show_syscall_exit
185 ld r3,RESULT(r1)
186#endif
187 CURRENT_THREAD_INFO(r12, r1)
188
189 ld r8,_MSR(r1)
190#ifdef CONFIG_PPC_BOOK3S
191
192 andi. r10,r8,MSR_RI
193 beq- unrecov_restore
194#endif
195
196
197
198
199#ifdef CONFIG_PPC_BOOK3E
200 wrteei 0
201#else
202 ld r10,PACAKMSR(r13)
203
204
205
206
207
208
209
210 li r9,MSR_RI
211 andc r11,r10,r9
212 mtmsrd r11,1
213#endif
214
215 ld r9,TI_FLAGS(r12)
216 li r11,-_LAST_ERRNO
217 andi. r0,r9,(_TIF_SYSCALL_T_OR_A|_TIF_SINGLESTEP|_TIF_USER_WORK_MASK|_TIF_PERSYSCALL_MASK)
218 bne- syscall_exit_work
219 cmpld r3,r11
220 ld r5,_CCR(r1)
221 bge- syscall_error
222.Lsyscall_error_cont:
223 ld r7,_NIP(r1)
224BEGIN_FTR_SECTION
225 stdcx. r0,0,r1
226END_FTR_SECTION_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
227 andi. r6,r8,MSR_PR
228 ld r4,_LINK(r1)
229
230 beq- 1f
231 ACCOUNT_CPU_USER_EXIT(r11, r12)
232 HMT_MEDIUM_LOW_HAS_PPR
233 ld r13,GPR13(r1)
2341: ld r2,GPR2(r1)
235 ld r1,GPR1(r1)
236 mtlr r4
237 mtcr r5
238 mtspr SPRN_SRR0,r7
239 mtspr SPRN_SRR1,r8
240 RFI
241 b .
242
243syscall_error:
244 oris r5,r5,0x1000
245 neg r3,r3
246 std r5,_CCR(r1)
247 b .Lsyscall_error_cont
248
249
250syscall_dotrace:
251 bl .save_nvgprs
252 addi r3,r1,STACK_FRAME_OVERHEAD
253 bl .do_syscall_trace_enter
254
255
256
257
258
259 mr r0,r3
260 ld r3,GPR3(r1)
261 ld r4,GPR4(r1)
262 ld r5,GPR5(r1)
263 ld r6,GPR6(r1)
264 ld r7,GPR7(r1)
265 ld r8,GPR8(r1)
266 addi r9,r1,STACK_FRAME_OVERHEAD
267 CURRENT_THREAD_INFO(r10, r1)
268 ld r10,TI_FLAGS(r10)
269 b .Lsyscall_dotrace_cont
270
271syscall_enosys:
272 li r3,-ENOSYS
273 b syscall_exit
274
275syscall_exit_work:
276#ifdef CONFIG_PPC_BOOK3S
277 mtmsrd r10,1
278#endif
279
280
281
282 andi. r0,r9,_TIF_RESTOREALL
283 beq+ 0f
284 REST_NVGPRS(r1)
285 b 2f
2860: cmpld r3,r11
287 blt+ 1f
288 andi. r0,r9,_TIF_NOERROR
289 bne- 1f
290 ld r5,_CCR(r1)
291 neg r3,r3
292 oris r5,r5,0x1000
293 std r5,_CCR(r1)
2941: std r3,GPR3(r1)
2952: andi. r0,r9,(_TIF_PERSYSCALL_MASK)
296 beq 4f
297
298
299
300 li r11,_TIF_PERSYSCALL_MASK
301 addi r12,r12,TI_FLAGS
3023: ldarx r10,0,r12
303 andc r10,r10,r11
304 stdcx. r10,0,r12
305 bne- 3b
306 subi r12,r12,TI_FLAGS
307
3084:
309 SET_DEFAULT_THREAD_PPR(r3, r10)
310 andi. r0,r9,(_TIF_SYSCALL_T_OR_A|_TIF_SINGLESTEP)
311 beq .ret_from_except_lite
312
313
314#ifdef CONFIG_PPC_BOOK3E
315 wrteei 1
316#else
317 ld r10,PACAKMSR(r13)
318 ori r10,r10,MSR_EE
319 mtmsrd r10,1
320#endif
321
322 bl .save_nvgprs
323 addi r3,r1,STACK_FRAME_OVERHEAD
324 bl .do_syscall_trace_leave
325 b .ret_from_except
326
327
328_GLOBAL(save_nvgprs)
329 ld r11,_TRAP(r1)
330 andi. r0,r11,1
331 beqlr-
332 SAVE_NVGPRS(r1)
333 clrrdi r0,r11,1
334 std r0,_TRAP(r1)
335 blr
336
337
338
339
340
341
342
343
344
345
346
347_GLOBAL(ppc_fork)
348 bl .save_nvgprs
349 bl .sys_fork
350 b syscall_exit
351
352_GLOBAL(ppc_vfork)
353 bl .save_nvgprs
354 bl .sys_vfork
355 b syscall_exit
356
357_GLOBAL(ppc_clone)
358 bl .save_nvgprs
359 bl .sys_clone
360 b syscall_exit
361
362_GLOBAL(ppc32_swapcontext)
363 bl .save_nvgprs
364 bl .compat_sys_swapcontext
365 b syscall_exit
366
367_GLOBAL(ppc64_swapcontext)
368 bl .save_nvgprs
369 bl .sys_swapcontext
370 b syscall_exit
371
372_GLOBAL(ret_from_fork)
373 bl .schedule_tail
374 REST_NVGPRS(r1)
375 li r3,0
376 b syscall_exit
377
378_GLOBAL(ret_from_kernel_thread)
379 bl .schedule_tail
380 REST_NVGPRS(r1)
381 ld r14, 0(r14)
382 mtlr r14
383 mr r3,r15
384 blrl
385 li r3,0
386 b syscall_exit
387
388 .section ".toc","aw"
389DSCR_DEFAULT:
390 .tc dscr_default[TC],dscr_default
391
392 .section ".text"
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412 .align 7
413_GLOBAL(_switch)
414 mflr r0
415 std r0,16(r1)
416 stdu r1,-SWITCH_FRAME_SIZE(r1)
417
418 SAVE_8GPRS(14, r1)
419 SAVE_10GPRS(22, r1)
420 mflr r20
421 mfmsr r22
422 li r0, MSR_FP
423#ifdef CONFIG_VSX
424BEGIN_FTR_SECTION
425 oris r0,r0,MSR_VSX@h
426END_FTR_SECTION_IFSET(CPU_FTR_VSX)
427#endif
428#ifdef CONFIG_ALTIVEC
429BEGIN_FTR_SECTION
430 oris r0,r0,MSR_VEC@h
431 mfspr r24,SPRN_VRSAVE
432 std r24,THREAD_VRSAVE(r3)
433END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
434#endif
435#ifdef CONFIG_PPC64
436BEGIN_FTR_SECTION
437 mfspr r25,SPRN_DSCR
438 std r25,THREAD_DSCR(r3)
439END_FTR_SECTION_IFSET(CPU_FTR_DSCR)
440#endif
441 and. r0,r0,r22
442 beq+ 1f
443 andc r22,r22,r0
444 MTMSRD(r22)
445 isync
4461: std r20,_NIP(r1)
447 mfcr r23
448 std r23,_CCR(r1)
449 std r1,KSP(r3)
450
451#ifdef CONFIG_PPC_BOOK3S_64
452BEGIN_FTR_SECTION
453
454 mfspr r0, SPRN_BESCR
455 std r0, THREAD_BESCR(r3)
456 mfspr r0, SPRN_EBBHR
457 std r0, THREAD_EBBHR(r3)
458 mfspr r0, SPRN_EBBRR
459 std r0, THREAD_EBBRR(r3)
460END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
461#endif
462
463#ifdef CONFIG_SMP
464
465
466
467
468 sync
469#endif
470
471
472
473
474
475
476
477
478BEGIN_FTR_SECTION
479 ldarx r6,0,r1
480END_FTR_SECTION_IFSET(CPU_FTR_STCX_CHECKS_ADDRESS)
481
482#ifdef CONFIG_PPC_BOOK3S
483
484
485
486 DCBT_STOP_ALL_STREAM_IDS(r6)
487#endif
488
489 addi r6,r4,-THREAD
490 std r6,PACACURRENT(r13)
491
492 ld r8,KSP(r4)
493#ifdef CONFIG_PPC_BOOK3S
494BEGIN_FTR_SECTION
495 BEGIN_FTR_SECTION_NESTED(95)
496 clrrdi r6,r8,28
497 clrrdi r9,r1,28
498 FTR_SECTION_ELSE_NESTED(95)
499 clrrdi r6,r8,40
500 clrrdi r9,r1,40
501 ALT_MMU_FTR_SECTION_END_NESTED_IFCLR(MMU_FTR_1T_SEGMENT, 95)
502FTR_SECTION_ELSE
503 b 2f
504ALT_MMU_FTR_SECTION_END_IFSET(MMU_FTR_SLB)
505 clrldi. r0,r6,2
506 cmpd cr1,r6,r9
507 cror eq,4*cr1+eq,eq
508 beq 2f
509
510
511 ld r7,KSP_VSID(r4)
512 oris r0,r6,(SLB_ESID_V)@h
513 ori r0,r0,(SLB_NUM_BOLTED-1)@l
514BEGIN_FTR_SECTION
515 li r9,MMU_SEGSIZE_1T
516 oris r6,r6,(MMU_SEGSIZE_1T << SLBIE_SSIZE_SHIFT)@h
517 rldimi r7,r9,SLB_VSID_SSIZE_SHIFT,0
518END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT)
519
520
521
522
523
524 ld r9,PACA_SLBSHADOWPTR(r13)
525 li r12,0
526 std r12,SLBSHADOW_STACKESID(r9)
527 li r12,SLBSHADOW_STACKVSID
528 STDX_BE r7,r12,r9
529 li r12,SLBSHADOW_STACKESID
530 STDX_BE r0,r12,r9
531
532
533
534
535
536
537
538 slbie r6
539 slbie r6
540 slbmte r7,r0
541 isync
5422:
543#endif
544
545 CURRENT_THREAD_INFO(r7, r8)
546
547
548
549 addi r7,r7,THREAD_SIZE-SWITCH_FRAME_SIZE
550
551 mr r1,r8
552 std r7,PACAKSAVE(r13)
553
554#ifdef CONFIG_PPC_BOOK3S_64
555BEGIN_FTR_SECTION
556
557 ld r0, THREAD_BESCR(r4)
558 mtspr SPRN_BESCR, r0
559 ld r0, THREAD_EBBHR(r4)
560 mtspr SPRN_EBBHR, r0
561 ld r0, THREAD_EBBRR(r4)
562 mtspr SPRN_EBBRR, r0
563
564 ld r0,THREAD_TAR(r4)
565 mtspr SPRN_TAR,r0
566END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
567#endif
568
569#ifdef CONFIG_ALTIVEC
570BEGIN_FTR_SECTION
571 ld r0,THREAD_VRSAVE(r4)
572 mtspr SPRN_VRSAVE,r0
573END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
574#endif
575#ifdef CONFIG_PPC64
576BEGIN_FTR_SECTION
577 lwz r6,THREAD_DSCR_INHERIT(r4)
578 ld r7,DSCR_DEFAULT@toc(2)
579 ld r0,THREAD_DSCR(r4)
580 cmpwi r6,0
581 bne 1f
582 ld r0,0(r7)
5831:
584BEGIN_FTR_SECTION_NESTED(70)
585 mfspr r8, SPRN_FSCR
586 rldimi r8, r6, FSCR_DSCR_LG, (63 - FSCR_DSCR_LG)
587 mtspr SPRN_FSCR, r8
588END_FTR_SECTION_NESTED(CPU_FTR_ARCH_207S, CPU_FTR_ARCH_207S, 70)
589 cmpd r0,r25
590 beq 2f
591 mtspr SPRN_DSCR,r0
5922:
593END_FTR_SECTION_IFSET(CPU_FTR_DSCR)
594#endif
595
596 ld r6,_CCR(r1)
597 mtcrf 0xFF,r6
598
599
600 REST_8GPRS(14, r1)
601 REST_10GPRS(22, r1)
602
603
604 addi r3,r3,-THREAD
605 ld r7,_NIP(r1)
606 mtlr r7
607 addi r1,r1,SWITCH_FRAME_SIZE
608 blr
609
610 .align 7
611_GLOBAL(ret_from_except)
612 ld r11,_TRAP(r1)
613 andi. r0,r11,1
614 bne .ret_from_except_lite
615 REST_NVGPRS(r1)
616
617_GLOBAL(ret_from_except_lite)
618
619
620
621
622
623#ifdef CONFIG_PPC_BOOK3E
624 wrteei 0
625#else
626 ld r10,PACAKMSR(r13)
627 mtmsrd r10,1
628#endif
629
630 CURRENT_THREAD_INFO(r9, r1)
631 ld r3,_MSR(r1)
632#ifdef CONFIG_PPC_BOOK3E
633 ld r10,PACACURRENT(r13)
634#endif
635 ld r4,TI_FLAGS(r9)
636 andi. r3,r3,MSR_PR
637 beq resume_kernel
638#ifdef CONFIG_PPC_BOOK3E
639 lwz r3,(THREAD+THREAD_DBCR0)(r10)
640#endif
641
642
643 andi. r0,r4,_TIF_USER_WORK_MASK
644#ifdef CONFIG_PPC_BOOK3E
645 bne 1f
646
647
648
649
650 andis. r0,r3,DBCR0_IDM@h
651 beq restore
652 mfmsr r0
653 rlwinm r0,r0,0,~MSR_DE
654 mtmsr r0
655 mtspr SPRN_DBCR0,r3
656 li r10, -1
657 mtspr SPRN_DBSR,r10
658 b restore
659#else
660 beq restore
661#endif
6621: andi. r0,r4,_TIF_NEED_RESCHED
663 beq 2f
664 bl .restore_interrupts
665 SCHEDULE_USER
666 b .ret_from_except_lite
667
6682: bl .save_nvgprs
669 bl .restore_interrupts
670 addi r3,r1,STACK_FRAME_OVERHEAD
671 bl .do_notify_resume
672 b .ret_from_except
673
674resume_kernel:
675
676 CURRENT_THREAD_INFO(r9, r1)
677 ld r8,TI_FLAGS(r9)
678 andis. r8,r8,_TIF_EMULATE_STACK_STORE@h
679 beq+ 1f
680
681 addi r8,r1,INT_FRAME_SIZE
682
683 lwz r3,GPR1(r1)
684 subi r3,r3,INT_FRAME_SIZE
685 mr r4,r1
686 mr r1,r3
687
688
689 li r5,INT_FRAME_SIZE/8
690 li r6,0
691 mtctr r5
6922: ldx r0,r6,r4
693 stdx r0,r6,r3
694 addi r6,r6,8
695 bdnz 2b
696
697
698 lwz r5,GPR1(r1)
699 std r8,0(r5)
700
701
702 lis r11,_TIF_EMULATE_STACK_STORE@h
703 addi r5,r9,TI_FLAGS
7040: ldarx r4,0,r5
705 andc r4,r4,r11
706 stdcx. r4,0,r5
707 bne- 0b
7081:
709
710#ifdef CONFIG_PREEMPT
711
712 andi. r0,r4,_TIF_NEED_RESCHED
713 beq+ restore
714
715 lwz r8,TI_PREEMPT(r9)
716 cmpwi cr1,r8,0
717 ld r0,SOFTE(r1)
718 cmpdi r0,0
719 crandc eq,cr1*4+eq,eq
720 bne restore
721
722
723
724
725
726 RECONCILE_IRQ_STATE(r3,r4)
7271: bl .preempt_schedule_irq
728
729
730 CURRENT_THREAD_INFO(r9, r1)
731 ld r4,TI_FLAGS(r9)
732 andi. r0,r4,_TIF_NEED_RESCHED
733 bne 1b
734
735
736
737
738
739
740
741#ifdef CONFIG_PPC_BOOK3E
742 wrteei 0
743#else
744 ld r10,PACAKMSR(r13)
745 mtmsrd r10,1
746#endif
747#endif
748
749 .globl fast_exc_return_irq
750fast_exc_return_irq:
751restore:
752
753
754
755
756 ld r5,SOFTE(r1)
757 lbz r6,PACASOFTIRQEN(r13)
758 cmpwi cr0,r5,0
759 beq restore_irq_off
760
761
762 cmpwi cr0,r6,1
763 beq cr0,do_restore
764
765
766
767
768
769
770 lbz r0,PACAIRQHAPPENED(r13)
771 cmpwi cr0,r0,0
772 bne- restore_check_irq_replay
773
774
775
776
777
778
779restore_no_replay:
780 TRACE_ENABLE_INTS
781 li r0,1
782 stb r0,PACASOFTIRQEN(r13);
783
784
785
786
787do_restore:
788#ifdef CONFIG_PPC_BOOK3E
789 b .exception_return_book3e
790#else
791
792
793
794
795
796BEGIN_FTR_SECTION
797 stdcx. r0,0,r1
798FTR_SECTION_ELSE
799 ldarx r4,0,r1
800ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
801
802
803
804
805
806
807
808 .globl fast_exception_return
809fast_exception_return:
810 ld r3,_MSR(r1)
811 ld r4,_CTR(r1)
812 ld r0,_LINK(r1)
813 mtctr r4
814 mtlr r0
815 ld r4,_XER(r1)
816 mtspr SPRN_XER,r4
817
818 REST_8GPRS(5, r1)
819
820 andi. r0,r3,MSR_RI
821 beq- unrecov_restore
822
823
824
825
826
827
828 ld r4,PACAKMSR(r13)
829 andc r4,r4,r0
830 mtmsrd r4,1
831
832#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
833
834 std r3, PACATMSCRATCH(r13)
835#endif
836
837
838
839
840
841 andi. r0,r3,MSR_PR
842 beq 1f
843 ACCOUNT_CPU_USER_EXIT(r2, r4)
844 RESTORE_PPR(r2, r4)
845 REST_GPR(13, r1)
8461:
847 mtspr SPRN_SRR1,r3
848
849 ld r2,_CCR(r1)
850 mtcrf 0xFF,r2
851 ld r2,_NIP(r1)
852 mtspr SPRN_SRR0,r2
853
854 ld r0,GPR0(r1)
855 ld r2,GPR2(r1)
856 ld r3,GPR3(r1)
857 ld r4,GPR4(r1)
858 ld r1,GPR1(r1)
859
860 rfid
861 b .
862
863#endif
864
865
866
867
868
869
870
871
872restore_irq_off:
873 ld r3,_MSR(r1)
874 lbz r7,PACAIRQHAPPENED(r13)
875 andi. r0,r3,MSR_EE
876 beq 1f
877 rlwinm r7,r7,0,~PACA_IRQ_HARD_DIS
878 stb r7,PACAIRQHAPPENED(r13)
8791: li r0,0
880 stb r0,PACASOFTIRQEN(r13);
881 TRACE_DISABLE_INTS
882 b do_restore
883
884
885
886
887
888restore_check_irq_replay:
889
890
891
892
893
894
895
896 bl .__check_irq_replay
897 cmpwi cr0,r3,0
898 beq restore_no_replay
899
900
901
902
903
904
905 ld r4,_TRAP(r1)
906 clrldi r4,r4,60
907 or r4,r4,r3
908 std r4,_TRAP(r1)
909
910
911
912
913
914 cmpwi cr0,r3,0x500
915 bne 1f
916 addi r3,r1,STACK_FRAME_OVERHEAD;
917 bl .do_IRQ
918 b .ret_from_except
9191: cmpwi cr0,r3,0x900
920 bne 1f
921 addi r3,r1,STACK_FRAME_OVERHEAD;
922 bl .timer_interrupt
923 b .ret_from_except
924#ifdef CONFIG_PPC_DOORBELL
9251:
926#ifdef CONFIG_PPC_BOOK3E
927 cmpwi cr0,r3,0x280
928#else
929 BEGIN_FTR_SECTION
930 cmpwi cr0,r3,0xe80
931 FTR_SECTION_ELSE
932 cmpwi cr0,r3,0xa00
933 ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE)
934#endif
935 bne 1f
936 addi r3,r1,STACK_FRAME_OVERHEAD;
937 bl .doorbell_exception
938 b .ret_from_except
939#endif
9401: b .ret_from_except
941
942unrecov_restore:
943 addi r3,r1,STACK_FRAME_OVERHEAD
944 bl .unrecoverable_exception
945 b unrecov_restore
946
947#ifdef CONFIG_PPC_RTAS
948
949
950
951
952
953
954
955
956_GLOBAL(enter_rtas)
957 mflr r0
958 std r0,16(r1)
959 stdu r1,-RTAS_FRAME_SIZE(r1)
960
961
962
963
964
965 SAVE_GPR(2, r1)
966 SAVE_GPR(13, r1)
967 SAVE_8GPRS(14, r1)
968 SAVE_10GPRS(22, r1)
969
970 mfcr r4
971 std r4,_CCR(r1)
972 mfctr r5
973 std r5,_CTR(r1)
974 mfspr r6,SPRN_XER
975 std r6,_XER(r1)
976 mfdar r7
977 std r7,_DAR(r1)
978 mfdsisr r8
979 std r8,_DSISR(r1)
980
981
982
983
984 li r0,0
985 mtcr r0
986
987#ifdef CONFIG_BUG
988
989
990
991 lbz r0,PACASOFTIRQEN(r13)
9921: tdnei r0,0
993 EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,BUGFLAG_WARNING
994#endif
995
996
997 mfmsr r6
998 rldicl r7,r6,48,1
999 rotldi r7,r7,16
1000 mtmsrd r7,1
1001
1002
1003
1004
1005
1006 std r1,PACAR1(r13)
1007 std r6,PACASAVEDMSR(r13)
1008
1009
1010 LOAD_REG_ADDR(r4,.rtas_return_loc)
1011 clrldi r4,r4,2
1012 mtlr r4
1013
1014 li r0,0
1015 ori r0,r0,MSR_EE|MSR_SE|MSR_BE|MSR_RI
1016 andc r0,r6,r0
1017
1018 li r9,1
1019 rldicr r9,r9,MSR_SF_LG,(63-MSR_SF_LG)
1020 ori r9,r9,MSR_IR|MSR_DR|MSR_FE0|MSR_FE1|MSR_FP|MSR_RI
1021 andc r6,r0,r9
1022 sync
1023 mtmsrd r0
1024
1025 LOAD_REG_ADDR(r4, rtas)
1026 ld r5,RTASENTRY(r4)
1027 ld r4,RTASBASE(r4)
1028
1029 mtspr SPRN_SRR0,r5
1030 mtspr SPRN_SRR1,r6
1031 rfid
1032 b .
1033
1034_STATIC(rtas_return_loc)
1035
1036 GET_PACA(r4)
1037 clrldi r4,r4,2
1038
1039 bcl 20,31,$+4
10400: mflr r3
1041 ld r3,(1f-0b)(r3)
1042
1043 mfmsr r6
1044 li r0,MSR_RI
1045 andc r6,r6,r0
1046 sync
1047 mtmsrd r6
1048
1049 ld r1,PACAR1(r4)
1050 ld r4,PACASAVEDMSR(r4)
1051
1052 mtspr SPRN_SRR0,r3
1053 mtspr SPRN_SRR1,r4
1054 rfid
1055 b .
1056
1057 .align 3
10581: .llong .rtas_restore_regs
1059
1060_STATIC(rtas_restore_regs)
1061
1062 REST_GPR(2, r1)
1063 REST_GPR(13, r1)
1064 REST_8GPRS(14, r1)
1065 REST_10GPRS(22, r1)
1066
1067 GET_PACA(r13)
1068
1069 ld r4,_CCR(r1)
1070 mtcr r4
1071 ld r5,_CTR(r1)
1072 mtctr r5
1073 ld r6,_XER(r1)
1074 mtspr SPRN_XER,r6
1075 ld r7,_DAR(r1)
1076 mtdar r7
1077 ld r8,_DSISR(r1)
1078 mtdsisr r8
1079
1080 addi r1,r1,RTAS_FRAME_SIZE
1081 ld r0,16(r1)
1082
1083 mtlr r0
1084 blr
1085
1086#endif
1087
1088_GLOBAL(enter_prom)
1089 mflr r0
1090 std r0,16(r1)
1091 stdu r1,-PROM_FRAME_SIZE(r1)
1092
1093
1094
1095
1096
1097 SAVE_GPR(2, r1)
1098 SAVE_GPR(13, r1)
1099 SAVE_8GPRS(14, r1)
1100 SAVE_10GPRS(22, r1)
1101 mfcr r10
1102 mfmsr r11
1103 std r10,_CCR(r1)
1104 std r11,_MSR(r1)
1105
1106
1107 mtlr r4
1108
1109
1110
1111#ifdef CONFIG_PPC_BOOK3E
1112 rlwinm r11,r11,0,1,31
1113 mtmsr r11
1114#else
1115 mfmsr r11
1116 li r12,1
1117 rldicr r12,r12,MSR_SF_LG,(63-MSR_SF_LG)
1118 andc r11,r11,r12
1119 li r12,1
1120 rldicr r12,r12,MSR_ISF_LG,(63-MSR_ISF_LG)
1121 andc r11,r11,r12
1122 mtmsrd r11
1123#endif
1124 isync
1125
1126
1127 blrl
1128
1129
1130
1131
1132 rldicl r1,r1,0,32
1133
1134
1135 ld r0,_MSR(r1)
1136 MTMSRD(r0)
1137 isync
1138
1139
1140 REST_GPR(2, r1)
1141 REST_GPR(13, r1)
1142 REST_8GPRS(14, r1)
1143 REST_10GPRS(22, r1)
1144 ld r4,_CCR(r1)
1145 mtcr r4
1146
1147 addi r1,r1,PROM_FRAME_SIZE
1148 ld r0,16(r1)
1149 mtlr r0
1150 blr
1151
1152#ifdef CONFIG_FUNCTION_TRACER
1153#ifdef CONFIG_DYNAMIC_FTRACE
1154_GLOBAL(mcount)
1155_GLOBAL(_mcount)
1156 blr
1157
1158_GLOBAL(ftrace_caller)
1159
1160 mflr r3
1161 ld r11, 0(r1)
1162 stdu r1, -112(r1)
1163 std r3, 128(r1)
1164 ld r4, 16(r11)
1165 subi r3, r3, MCOUNT_INSN_SIZE
1166.globl ftrace_call
1167ftrace_call:
1168 bl ftrace_stub
1169 nop
1170#ifdef CONFIG_FUNCTION_GRAPH_TRACER
1171.globl ftrace_graph_call
1172ftrace_graph_call:
1173 b ftrace_graph_stub
1174_GLOBAL(ftrace_graph_stub)
1175#endif
1176 ld r0, 128(r1)
1177 mtlr r0
1178 addi r1, r1, 112
1179_GLOBAL(ftrace_stub)
1180 blr
1181#else
1182_GLOBAL(mcount)
1183 blr
1184
1185_GLOBAL(_mcount)
1186
1187 mflr r3
1188 ld r11, 0(r1)
1189 stdu r1, -112(r1)
1190 std r3, 128(r1)
1191 ld r4, 16(r11)
1192
1193 subi r3, r3, MCOUNT_INSN_SIZE
1194 LOAD_REG_ADDR(r5,ftrace_trace_function)
1195 ld r5,0(r5)
1196 ld r5,0(r5)
1197 mtctr r5
1198 bctrl
1199 nop
1200
1201
1202#ifdef CONFIG_FUNCTION_GRAPH_TRACER
1203 b ftrace_graph_caller
1204#endif
1205 ld r0, 128(r1)
1206 mtlr r0
1207 addi r1, r1, 112
1208_GLOBAL(ftrace_stub)
1209 blr
1210
1211#endif
1212
1213#ifdef CONFIG_FUNCTION_GRAPH_TRACER
1214_GLOBAL(ftrace_graph_caller)
1215
1216 ld r4, 128(r1)
1217 subi r4, r4, MCOUNT_INSN_SIZE
1218
1219
1220 ld r11, 112(r1)
1221 addi r3, r11, 16
1222
1223 bl .prepare_ftrace_return
1224 nop
1225
1226 ld r0, 128(r1)
1227 mtlr r0
1228 addi r1, r1, 112
1229 blr
1230
1231_GLOBAL(return_to_handler)
1232
1233 std r4, -24(r1)
1234 std r3, -16(r1)
1235 std r31, -8(r1)
1236 mr r31, r1
1237 stdu r1, -112(r1)
1238
1239 bl .ftrace_return_to_handler
1240 nop
1241
1242
1243 mtlr r3
1244
1245 ld r1, 0(r1)
1246 ld r4, -24(r1)
1247 ld r3, -16(r1)
1248 ld r31, -8(r1)
1249
1250
1251 blr
1252
1253_GLOBAL(mod_return_to_handler)
1254
1255 std r4, -32(r1)
1256 std r3, -24(r1)
1257
1258 std r2, -16(r1)
1259 std r31, -8(r1)
1260 mr r31, r1
1261 stdu r1, -112(r1)
1262
1263
1264
1265
1266
1267 ld r2, PACATOC(r13)
1268
1269 bl .ftrace_return_to_handler
1270 nop
1271
1272
1273 mtlr r3
1274
1275 ld r1, 0(r1)
1276 ld r4, -32(r1)
1277 ld r3, -24(r1)
1278 ld r2, -16(r1)
1279 ld r31, -8(r1)
1280
1281
1282 blr
1283#endif
1284#endif
1285