1
2
3
4
5
6
7
8#include <linux/threads.h>
9#include <asm/reg.h>
10#include <asm/page.h>
11#include <asm/ppc_asm.h>
12#include <asm/asm-offsets.h>
13#include <asm/cputable.h>
14#include <asm/setup.h>
15#include <asm/thread_info.h>
16#include <asm/reg_a2.h>
17#include <asm/exception-64e.h>
18#include <asm/bug.h>
19#include <asm/irqflags.h>
20#include <asm/ptrace.h>
21#include <asm/ppc-opcode.h>
22#include <asm/mmu.h>
23#include <asm/hw_irq.h>
24#include <asm/kvm_asm.h>
25#include <asm/kvm_booke_hv_asm.h>
26#include <asm/feature-fixups.h>
27
28
29
30
31
32
33
34#define SPECIAL_EXC_SRR0 0
35#define SPECIAL_EXC_SRR1 1
36#define SPECIAL_EXC_SPRG_GEN 2
37#define SPECIAL_EXC_SPRG_TLB 3
38#define SPECIAL_EXC_MAS0 4
39#define SPECIAL_EXC_MAS1 5
40#define SPECIAL_EXC_MAS2 6
41#define SPECIAL_EXC_MAS3 7
42#define SPECIAL_EXC_MAS6 8
43#define SPECIAL_EXC_MAS7 9
44#define SPECIAL_EXC_MAS5 10
45#define SPECIAL_EXC_MAS8 11
46#define SPECIAL_EXC_IRQHAPPENED 12
47#define SPECIAL_EXC_DEAR 13
48#define SPECIAL_EXC_ESR 14
49#define SPECIAL_EXC_SOFTE 15
50#define SPECIAL_EXC_CSRR0 16
51#define SPECIAL_EXC_CSRR1 17
52
53#define SPECIAL_EXC_END 18
54
55#define SPECIAL_EXC_FRAME_SIZE (INT_FRAME_SIZE + SPECIAL_EXC_END * 8)
56#define SPECIAL_EXC_FRAME_OFFS (INT_FRAME_SIZE - 288)
57
58#define SPECIAL_EXC_STORE(reg, name) \
59 std reg, (SPECIAL_EXC_
60
61#define SPECIAL_EXC_LOAD(reg, name) \
62 ld reg, (SPECIAL_EXC_
63
64special_reg_save:
65 lbz r9,PACAIRQHAPPENED(r13)
66 RECONCILE_IRQ_STATE(r3,r4)
67
68
69
70
71
72 ld r3,_MSR(r1)
73 andi. r3,r3,MSR_PR
74 bnelr
75
76
77
78
79
80 LOAD_REG_ADDR(r11,extlb_level_exc)
81 lwz r12,0(r11)
82 mfspr r10,SPRN_SPRG_TLB_EXFRAME
83 add r10,r10,r12
84 mtspr SPRN_SPRG_TLB_EXFRAME,r10
85
86
87
88
89
90 mfspr r10,SPRN_SRR0
91 SPECIAL_EXC_STORE(r10,SRR0)
92 mfspr r10,SPRN_SRR1
93 SPECIAL_EXC_STORE(r10,SRR1)
94 mfspr r10,SPRN_SPRG_GEN_SCRATCH
95 SPECIAL_EXC_STORE(r10,SPRG_GEN)
96 mfspr r10,SPRN_SPRG_TLB_SCRATCH
97 SPECIAL_EXC_STORE(r10,SPRG_TLB)
98 mfspr r10,SPRN_MAS0
99 SPECIAL_EXC_STORE(r10,MAS0)
100 mfspr r10,SPRN_MAS1
101 SPECIAL_EXC_STORE(r10,MAS1)
102 mfspr r10,SPRN_MAS2
103 SPECIAL_EXC_STORE(r10,MAS2)
104 mfspr r10,SPRN_MAS3
105 SPECIAL_EXC_STORE(r10,MAS3)
106 mfspr r10,SPRN_MAS6
107 SPECIAL_EXC_STORE(r10,MAS6)
108 mfspr r10,SPRN_MAS7
109 SPECIAL_EXC_STORE(r10,MAS7)
110BEGIN_FTR_SECTION
111 mfspr r10,SPRN_MAS5
112 SPECIAL_EXC_STORE(r10,MAS5)
113 mfspr r10,SPRN_MAS8
114 SPECIAL_EXC_STORE(r10,MAS8)
115
116
117 li r10,0
118 mtspr SPRN_MAS5,r10
119 mtspr SPRN_MAS8,r10
120END_FTR_SECTION_IFSET(CPU_FTR_EMB_HV)
121 SPECIAL_EXC_STORE(r9,IRQHAPPENED)
122
123 mfspr r10,SPRN_DEAR
124 SPECIAL_EXC_STORE(r10,DEAR)
125 mfspr r10,SPRN_ESR
126 SPECIAL_EXC_STORE(r10,ESR)
127
128 lbz r10,PACAIRQSOFTMASK(r13)
129 SPECIAL_EXC_STORE(r10,SOFTE)
130 ld r10,_NIP(r1)
131 SPECIAL_EXC_STORE(r10,CSRR0)
132 ld r10,_MSR(r1)
133 SPECIAL_EXC_STORE(r10,CSRR1)
134
135 blr
136
137ret_from_level_except:
138 ld r3,_MSR(r1)
139 andi. r3,r3,MSR_PR
140 beq 1f
141 b ret_from_except
1421:
143
144 LOAD_REG_ADDR(r11,extlb_level_exc)
145 lwz r12,0(r11)
146 mfspr r10,SPRN_SPRG_TLB_EXFRAME
147 sub r10,r10,r12
148 mtspr SPRN_SPRG_TLB_EXFRAME,r10
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164 PPC_TLBILX_ALL(0,R0)
165
166 REST_NVGPRS(r1)
167
168 SPECIAL_EXC_LOAD(r10,SRR0)
169 mtspr SPRN_SRR0,r10
170 SPECIAL_EXC_LOAD(r10,SRR1)
171 mtspr SPRN_SRR1,r10
172 SPECIAL_EXC_LOAD(r10,SPRG_GEN)
173 mtspr SPRN_SPRG_GEN_SCRATCH,r10
174 SPECIAL_EXC_LOAD(r10,SPRG_TLB)
175 mtspr SPRN_SPRG_TLB_SCRATCH,r10
176 SPECIAL_EXC_LOAD(r10,MAS0)
177 mtspr SPRN_MAS0,r10
178 SPECIAL_EXC_LOAD(r10,MAS1)
179 mtspr SPRN_MAS1,r10
180 SPECIAL_EXC_LOAD(r10,MAS2)
181 mtspr SPRN_MAS2,r10
182 SPECIAL_EXC_LOAD(r10,MAS3)
183 mtspr SPRN_MAS3,r10
184 SPECIAL_EXC_LOAD(r10,MAS6)
185 mtspr SPRN_MAS6,r10
186 SPECIAL_EXC_LOAD(r10,MAS7)
187 mtspr SPRN_MAS7,r10
188BEGIN_FTR_SECTION
189 SPECIAL_EXC_LOAD(r10,MAS5)
190 mtspr SPRN_MAS5,r10
191 SPECIAL_EXC_LOAD(r10,MAS8)
192 mtspr SPRN_MAS8,r10
193END_FTR_SECTION_IFSET(CPU_FTR_EMB_HV)
194
195 lbz r6,PACAIRQSOFTMASK(r13)
196 ld r5,SOFTE(r1)
197
198
199 tweqi r6,IRQS_ENABLED
200
201 andi. r6,r5,IRQS_DISABLED
202 bne 1f
203
204 TRACE_ENABLE_INTS
205 stb r5,PACAIRQSOFTMASK(r13)
2061:
207
208
209
210
211
212
213 SPECIAL_EXC_LOAD(r10,IRQHAPPENED)
214 stb r10,PACAIRQHAPPENED(r13)
215
216 SPECIAL_EXC_LOAD(r10,DEAR)
217 mtspr SPRN_DEAR,r10
218 SPECIAL_EXC_LOAD(r10,ESR)
219 mtspr SPRN_ESR,r10
220
221 stdcx. r0,0,r1
222
223 REST_4GPRS(2, r1)
224 REST_4GPRS(6, r1)
225
226 ld r10,_CTR(r1)
227 ld r11,_XER(r1)
228 mtctr r10
229 mtxer r11
230
231 blr
232
233.macro ret_from_level srr0 srr1 paca_ex scratch
234 bl ret_from_level_except
235
236 ld r10,_LINK(r1)
237 ld r11,_CCR(r1)
238 ld r0,GPR13(r1)
239 mtlr r10
240 mtcr r11
241
242 ld r10,GPR10(r1)
243 ld r11,GPR11(r1)
244 ld r12,GPR12(r1)
245 mtspr \scratch,r0
246
247 std r10,\paca_ex+EX_R10(r13);
248 std r11,\paca_ex+EX_R11(r13);
249 ld r10,_NIP(r1)
250 ld r11,_MSR(r1)
251 ld r0,GPR0(r1)
252 ld r1,GPR1(r1)
253 mtspr \srr0,r10
254 mtspr \srr1,r11
255 ld r10,\paca_ex+EX_R10(r13)
256 ld r11,\paca_ex+EX_R11(r13)
257 mfspr r13,\scratch
258.endm
259
260ret_from_crit_except:
261 ret_from_level SPRN_CSRR0 SPRN_CSRR1 PACA_EXCRIT SPRN_SPRG_CRIT_SCRATCH
262 rfci
263
264ret_from_mc_except:
265 ret_from_level SPRN_MCSRR0 SPRN_MCSRR1 PACA_EXMC SPRN_SPRG_MC_SCRATCH
266 rfmci
267
268
269#define EXCEPTION_PROLOG(n, intnum, type, addition) \
270 mtspr SPRN_SPRG_
271 mfspr r13,SPRN_SPRG_PACA; \
272 std r10,PACA_EX
273 std r11,PACA_EX
274 mfcr r10; \
275 mfspr r11,SPRN_
276 DO_KVM intnum,SPRN_
277 stw r10,PACA_EX
278 addition; \
279 std r1,PACA_EX
280 type
281 andi. r10,r11,MSR_PR; \
282 beq 1f; \
283 ld r1,PACAKSAVE(r13); \
2841: type
285 cmpdi cr1,r1,0; \
286 bge- cr1,exc_
287 mfspr r10,SPRN_
288
289
290#define GEN_SET_KSTACK \
291 subi r1,r1,INT_FRAME_SIZE;
292#define SPRN_GEN_SRR0 SPRN_SRR0
293#define SPRN_GEN_SRR1 SPRN_SRR1
294
295#define GDBELL_SET_KSTACK GEN_SET_KSTACK
296#define SPRN_GDBELL_SRR0 SPRN_GSRR0
297#define SPRN_GDBELL_SRR1 SPRN_GSRR1
298
299#define CRIT_SET_KSTACK \
300 ld r1,PACA_CRIT_STACK(r13); \
301 subi r1,r1,SPECIAL_EXC_FRAME_SIZE
302#define SPRN_CRIT_SRR0 SPRN_CSRR0
303#define SPRN_CRIT_SRR1 SPRN_CSRR1
304
305#define DBG_SET_KSTACK \
306 ld r1,PACA_DBG_STACK(r13); \
307 subi r1,r1,SPECIAL_EXC_FRAME_SIZE
308#define SPRN_DBG_SRR0 SPRN_DSRR0
309#define SPRN_DBG_SRR1 SPRN_DSRR1
310
311#define MC_SET_KSTACK \
312 ld r1,PACA_MC_STACK(r13); \
313 subi r1,r1,SPECIAL_EXC_FRAME_SIZE
314#define SPRN_MC_SRR0 SPRN_MCSRR0
315#define SPRN_MC_SRR1 SPRN_MCSRR1
316
317#ifdef CONFIG_PPC_FSL_BOOK3E
318#define GEN_BTB_FLUSH \
319 START_BTB_FLUSH_SECTION \
320 beq 1f; \
321 BTB_FLUSH(r10) \
322 1: \
323 END_BTB_FLUSH_SECTION
324
325#define CRIT_BTB_FLUSH \
326 START_BTB_FLUSH_SECTION \
327 BTB_FLUSH(r10) \
328 END_BTB_FLUSH_SECTION
329
330#define DBG_BTB_FLUSH CRIT_BTB_FLUSH
331#define MC_BTB_FLUSH CRIT_BTB_FLUSH
332#define GDBELL_BTB_FLUSH GEN_BTB_FLUSH
333#else
334#define GEN_BTB_FLUSH
335#define CRIT_BTB_FLUSH
336#define DBG_BTB_FLUSH
337#define MC_BTB_FLUSH
338#define GDBELL_BTB_FLUSH
339#endif
340
341#define NORMAL_EXCEPTION_PROLOG(n, intnum, addition) \
342 EXCEPTION_PROLOG(n, intnum, GEN, addition
343
344#define CRIT_EXCEPTION_PROLOG(n, intnum, addition) \
345 EXCEPTION_PROLOG(n, intnum, CRIT, addition
346
347#define DBG_EXCEPTION_PROLOG(n, intnum, addition) \
348 EXCEPTION_PROLOG(n, intnum, DBG, addition
349
350#define MC_EXCEPTION_PROLOG(n, intnum, addition) \
351 EXCEPTION_PROLOG(n, intnum, MC, addition
352
353#define GDBELL_EXCEPTION_PROLOG(n, intnum, addition) \
354 EXCEPTION_PROLOG(n, intnum, GDBELL, addition
355
356
357
358#define PROLOG_ADDITION_NONE_GEN(n)
359#define PROLOG_ADDITION_NONE_GDBELL(n)
360#define PROLOG_ADDITION_NONE_CRIT(n)
361#define PROLOG_ADDITION_NONE_DBG(n)
362#define PROLOG_ADDITION_NONE_MC(n)
363
364#define PROLOG_ADDITION_MASKABLE_GEN(n) \
365 lbz r10,PACAIRQSOFTMASK(r13); \
366 andi. r10,r10,IRQS_DISABLED; \
367 bne masked_interrupt_book3e_
368
369#define PROLOG_ADDITION_2REGS_GEN(n) \
370 std r14,PACA_EXGEN+EX_R14(r13); \
371 std r15,PACA_EXGEN+EX_R15(r13)
372
373#define PROLOG_ADDITION_1REG_GEN(n) \
374 std r14,PACA_EXGEN+EX_R14(r13);
375
376#define PROLOG_ADDITION_2REGS_CRIT(n) \
377 std r14,PACA_EXCRIT+EX_R14(r13); \
378 std r15,PACA_EXCRIT+EX_R15(r13)
379
380#define PROLOG_ADDITION_2REGS_DBG(n) \
381 std r14,PACA_EXDBG+EX_R14(r13); \
382 std r15,PACA_EXDBG+EX_R15(r13)
383
384#define PROLOG_ADDITION_2REGS_MC(n) \
385 std r14,PACA_EXMC+EX_R14(r13); \
386 std r15,PACA_EXMC+EX_R15(r13)
387
388
389
390#define EXCEPTION_COMMON_LVL(n, scratch, excf) \
391exc_
392 std r0,GPR0(r1); \
393 std r2,GPR2(r1); \
394 SAVE_4GPRS(3, r1); \
395 SAVE_2GPRS(7, r1); \
396 std r9,GPR9(r1); \
397 std r10,_NIP(r1); \
398 std r11,_MSR(r1); \
399 beq 2f; \
400 ACCOUNT_CPU_USER_ENTRY(r13,r10,r11); \
4012: ld r3,excf+EX_R10(r13); \
402 ld r4,excf+EX_R11(r13); \
403 mfspr r5,scratch; \
404 std r12,GPR12(r1); \
405 ld r2,PACATOC(r13); \
406 mflr r6; \
407 mfctr r7; \
408 mfspr r8,SPRN_XER; \
409 ld r9,excf+EX_R1(r13); \
410 lwz r10,excf+EX_CR(r13); \
411 lbz r11,PACAIRQSOFTMASK(r13); \
412 ld r12,exception_marker@toc(r2); \
413 li r0,0; \
414 std r3,GPR10(r1); \
415 std r4,GPR11(r1); \
416 std r5,GPR13(r1); \
417 std r6,_LINK(r1); \
418 std r7,_CTR(r1); \
419 std r8,_XER(r1); \
420 li r3,(n)+1; \
421 std r9,0(r1); \
422 std r10,_CCR(r1); \
423 std r9,GPR1(r1); \
424 std r11,SOFTE(r1); \
425 std r12,STACK_FRAME_OVERHEAD-16(r1); \
426 std r3,_TRAP(r1); \
427 std r0,RESULT(r1);
428
429#define EXCEPTION_COMMON(n) \
430 EXCEPTION_COMMON_LVL(n, SPRN_SPRG_GEN_SCRATCH, PACA_EXGEN)
431#define EXCEPTION_COMMON_CRIT(n) \
432 EXCEPTION_COMMON_LVL(n, SPRN_SPRG_CRIT_SCRATCH, PACA_EXCRIT)
433#define EXCEPTION_COMMON_MC(n) \
434 EXCEPTION_COMMON_LVL(n, SPRN_SPRG_MC_SCRATCH, PACA_EXMC)
435#define EXCEPTION_COMMON_DBG(n) \
436 EXCEPTION_COMMON_LVL(n, SPRN_SPRG_DBG_SCRATCH, PACA_EXDBG)
437
438
439
440
441
442
443
444#define INTS_DISABLE RECONCILE_IRQ_STATE(r3,r4)
445
446
447
448
449
450
451
452
453
454
455
456#define INTS_RESTORE_HARD \
457 ld r11,_MSR(r1); \
458 wrtee r11;
459
460
461#define BAD_STACK_TRAMPOLINE(n) \
462exc_
463 li r1,(n); \
464 sth r1,PACA_TRAP_SAVE(r13); \
465 b bad_stack_book3e;
466
467
468
469
470
471
472
473#define EXCEPTION_STUB(loc, label) \
474 . = interrupt_base_book3e + loc; \
475 nop; \
476 b exc_
477
478#define ACK_NONE(r)
479#define ACK_DEC(r) \
480 lis r,TSR_DIS@h; \
481 mtspr SPRN_TSR,r
482#define ACK_FIT(r) \
483 lis r,TSR_FIS@h; \
484 mtspr SPRN_TSR,r
485
486
487
488
489
490
491
492#define CHECK_NAPPING() \
493 ld r11, PACA_THREAD_INFO(r13); \
494 ld r10,TI_LOCAL_FLAGS(r11); \
495 andi. r9,r10,_TLF_NAPPING; \
496 beq+ 1f; \
497 ld r8,_LINK(r1); \
498 rlwinm r7,r10,0,~_TLF_NAPPING; \
499 std r8,_NIP(r1); \
500 std r7,TI_LOCAL_FLAGS(r11); \
5011:
502
503
504#define MASKABLE_EXCEPTION(trapnum, intnum, label, hdlr, ack) \
505 START_EXCEPTION(label); \
506 NORMAL_EXCEPTION_PROLOG(trapnum, intnum, PROLOG_ADDITION_MASKABLE)\
507 EXCEPTION_COMMON(trapnum) \
508 INTS_DISABLE; \
509 ack(r8); \
510 CHECK_NAPPING(); \
511 addi r3,r1,STACK_FRAME_OVERHEAD; \
512 bl hdlr; \
513 b ret_from_except_lite;
514
515
516 .section ".toc","aw"
517exception_marker:
518 .tc ID_EXC_MARKER[TC],STACK_FRAME_REGS_MARKER
519
520
521
522
523
524
525 .text
526 .balign 0x1000
527 .globl interrupt_base_book3e
528interrupt_base_book3e:
529 EXCEPTION_STUB(0x000, machine_check)
530 EXCEPTION_STUB(0x020, critical_input)
531 EXCEPTION_STUB(0x040, debug_crit)
532 EXCEPTION_STUB(0x060, data_storage)
533 EXCEPTION_STUB(0x080, instruction_storage)
534 EXCEPTION_STUB(0x0a0, external_input)
535 EXCEPTION_STUB(0x0c0, alignment)
536 EXCEPTION_STUB(0x0e0, program)
537 EXCEPTION_STUB(0x100, fp_unavailable)
538 EXCEPTION_STUB(0x120, system_call)
539 EXCEPTION_STUB(0x140, ap_unavailable)
540 EXCEPTION_STUB(0x160, decrementer)
541 EXCEPTION_STUB(0x180, fixed_interval)
542 EXCEPTION_STUB(0x1a0, watchdog)
543 EXCEPTION_STUB(0x1c0, data_tlb_miss)
544 EXCEPTION_STUB(0x1e0, instruction_tlb_miss)
545 EXCEPTION_STUB(0x200, altivec_unavailable)
546 EXCEPTION_STUB(0x220, altivec_assist)
547 EXCEPTION_STUB(0x260, perfmon)
548 EXCEPTION_STUB(0x280, doorbell)
549 EXCEPTION_STUB(0x2a0, doorbell_crit)
550 EXCEPTION_STUB(0x2c0, guest_doorbell)
551 EXCEPTION_STUB(0x2e0, guest_doorbell_crit)
552 EXCEPTION_STUB(0x300, hypercall)
553 EXCEPTION_STUB(0x320, ehpriv)
554 EXCEPTION_STUB(0x340, lrat_error)
555
556 .globl __end_interrupts
557__end_interrupts:
558
559
560 START_EXCEPTION(critical_input);
561 CRIT_EXCEPTION_PROLOG(0x100, BOOKE_INTERRUPT_CRITICAL,
562 PROLOG_ADDITION_NONE)
563 EXCEPTION_COMMON_CRIT(0x100)
564 bl save_nvgprs
565 bl special_reg_save
566 CHECK_NAPPING();
567 addi r3,r1,STACK_FRAME_OVERHEAD
568 bl unknown_exception
569 b ret_from_crit_except
570
571
572 START_EXCEPTION(machine_check);
573 MC_EXCEPTION_PROLOG(0x000, BOOKE_INTERRUPT_MACHINE_CHECK,
574 PROLOG_ADDITION_NONE)
575 EXCEPTION_COMMON_MC(0x000)
576 bl save_nvgprs
577 bl special_reg_save
578 CHECK_NAPPING();
579 addi r3,r1,STACK_FRAME_OVERHEAD
580 bl machine_check_exception
581 b ret_from_mc_except
582
583
584 START_EXCEPTION(data_storage)
585 NORMAL_EXCEPTION_PROLOG(0x300, BOOKE_INTERRUPT_DATA_STORAGE,
586 PROLOG_ADDITION_2REGS)
587 mfspr r14,SPRN_DEAR
588 mfspr r15,SPRN_ESR
589 EXCEPTION_COMMON(0x300)
590 INTS_DISABLE
591 b storage_fault_common
592
593
594 START_EXCEPTION(instruction_storage);
595 NORMAL_EXCEPTION_PROLOG(0x400, BOOKE_INTERRUPT_INST_STORAGE,
596 PROLOG_ADDITION_2REGS)
597 li r15,0
598 mr r14,r10
599 EXCEPTION_COMMON(0x400)
600 INTS_DISABLE
601 b storage_fault_common
602
603
604 MASKABLE_EXCEPTION(0x500, BOOKE_INTERRUPT_EXTERNAL,
605 external_input, do_IRQ, ACK_NONE)
606
607
608 START_EXCEPTION(alignment);
609 NORMAL_EXCEPTION_PROLOG(0x600, BOOKE_INTERRUPT_ALIGNMENT,
610 PROLOG_ADDITION_2REGS)
611 mfspr r14,SPRN_DEAR
612 mfspr r15,SPRN_ESR
613 EXCEPTION_COMMON(0x600)
614 b alignment_more
615
616
617 START_EXCEPTION(program);
618 NORMAL_EXCEPTION_PROLOG(0x700, BOOKE_INTERRUPT_PROGRAM,
619 PROLOG_ADDITION_1REG)
620 mfspr r14,SPRN_ESR
621 EXCEPTION_COMMON(0x700)
622 INTS_DISABLE
623 std r14,_DSISR(r1)
624 addi r3,r1,STACK_FRAME_OVERHEAD
625 ld r14,PACA_EXGEN+EX_R14(r13)
626 bl save_nvgprs
627 bl program_check_exception
628 b ret_from_except
629
630
631 START_EXCEPTION(fp_unavailable);
632 NORMAL_EXCEPTION_PROLOG(0x800, BOOKE_INTERRUPT_FP_UNAVAIL,
633 PROLOG_ADDITION_NONE)
634
635 EXCEPTION_COMMON(0x800)
636 ld r12,_MSR(r1)
637 andi. r0,r12,MSR_PR;
638 beq- 1f
639 bl load_up_fpu
640 b fast_exception_return
6411: INTS_DISABLE
642 bl save_nvgprs
643 addi r3,r1,STACK_FRAME_OVERHEAD
644 bl kernel_fp_unavailable_exception
645 b ret_from_except
646
647
648 START_EXCEPTION(altivec_unavailable);
649 NORMAL_EXCEPTION_PROLOG(0x200, BOOKE_INTERRUPT_ALTIVEC_UNAVAIL,
650 PROLOG_ADDITION_NONE)
651
652 EXCEPTION_COMMON(0x200)
653#ifdef CONFIG_ALTIVEC
654BEGIN_FTR_SECTION
655 ld r12,_MSR(r1)
656 andi. r0,r12,MSR_PR;
657 beq- 1f
658 bl load_up_altivec
659 b fast_exception_return
6601:
661END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
662#endif
663 INTS_DISABLE
664 bl save_nvgprs
665 addi r3,r1,STACK_FRAME_OVERHEAD
666 bl altivec_unavailable_exception
667 b ret_from_except
668
669
670 START_EXCEPTION(altivec_assist);
671 NORMAL_EXCEPTION_PROLOG(0x220,
672 BOOKE_INTERRUPT_ALTIVEC_ASSIST,
673 PROLOG_ADDITION_NONE)
674 EXCEPTION_COMMON(0x220)
675 INTS_DISABLE
676 bl save_nvgprs
677 addi r3,r1,STACK_FRAME_OVERHEAD
678#ifdef CONFIG_ALTIVEC
679BEGIN_FTR_SECTION
680 bl altivec_assist_exception
681END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
682#else
683 bl unknown_exception
684#endif
685 b ret_from_except
686
687
688
689 MASKABLE_EXCEPTION(0x900, BOOKE_INTERRUPT_DECREMENTER,
690 decrementer, timer_interrupt, ACK_DEC)
691
692
693 MASKABLE_EXCEPTION(0x980, BOOKE_INTERRUPT_FIT,
694 fixed_interval, unknown_exception, ACK_FIT)
695
696
697 START_EXCEPTION(watchdog);
698 CRIT_EXCEPTION_PROLOG(0x9f0, BOOKE_INTERRUPT_WATCHDOG,
699 PROLOG_ADDITION_NONE)
700 EXCEPTION_COMMON_CRIT(0x9f0)
701 bl save_nvgprs
702 bl special_reg_save
703 CHECK_NAPPING();
704 addi r3,r1,STACK_FRAME_OVERHEAD
705#ifdef CONFIG_BOOKE_WDT
706 bl WatchdogException
707#else
708 bl unknown_exception
709#endif
710 b ret_from_crit_except
711
712
713 START_EXCEPTION(system_call)
714 mr r9,r13
715 mfspr r11,SPRN_SRR0
716 mfspr r12,SPRN_SRR1
717 mfspr r13,SPRN_SPRG_PACA
718 b system_call_common
719
720
721 START_EXCEPTION(ap_unavailable);
722 NORMAL_EXCEPTION_PROLOG(0xf20, BOOKE_INTERRUPT_AP_UNAVAIL,
723 PROLOG_ADDITION_NONE)
724 EXCEPTION_COMMON(0xf20)
725 INTS_DISABLE
726 bl save_nvgprs
727 addi r3,r1,STACK_FRAME_OVERHEAD
728 bl unknown_exception
729 b ret_from_except
730
731
732 START_EXCEPTION(debug_crit);
733 CRIT_EXCEPTION_PROLOG(0xd00, BOOKE_INTERRUPT_DEBUG,
734 PROLOG_ADDITION_2REGS)
735
736
737
738
739
740
741
742
743
744
745 mfspr r14,SPRN_DBSR
746 andis. r15,r14,(DBSR_IC|DBSR_BT)@h
747 beq+ 1f
748
749#ifdef CONFIG_RELOCATABLE
750 ld r15,PACATOC(r13)
751 ld r14,interrupt_base_book3e@got(r15)
752 ld r15,__end_interrupts@got(r15)
753#else
754 LOAD_REG_IMMEDIATE(r14,interrupt_base_book3e)
755 LOAD_REG_IMMEDIATE(r15,__end_interrupts)
756#endif
757 cmpld cr0,r10,r14
758 cmpld cr1,r10,r15
759 blt+ cr0,1f
760 bge+ cr1,1f
761
762
763 lis r14,(DBSR_IC|DBSR_BT)@h
764 rlwinm r11,r11,0,~MSR_DE
765 mtspr SPRN_DBSR,r14
766 mtspr SPRN_CSRR1,r11
767 lwz r10,PACA_EXCRIT+EX_CR(r13)
768 ld r1,PACA_EXCRIT+EX_R1(r13)
769 ld r14,PACA_EXCRIT+EX_R14(r13)
770 ld r15,PACA_EXCRIT+EX_R15(r13)
771 mtcr r10
772 ld r10,PACA_EXCRIT+EX_R10(r13)
773 ld r11,PACA_EXCRIT+EX_R11(r13)
774 mfspr r13,SPRN_SPRG_CRIT_SCRATCH
775 rfci
776
777
778
779
780
7811: andi. r14,r11,MSR_PR;
782 beq kernel_dbg_exc;
783
784
785
786
787 mfspr r14,SPRN_DBSR
788 EXCEPTION_COMMON_CRIT(0xd00)
789 std r14,_DSISR(r1)
790 addi r3,r1,STACK_FRAME_OVERHEAD
791 mr r4,r14
792 ld r14,PACA_EXCRIT+EX_R14(r13)
793 ld r15,PACA_EXCRIT+EX_R15(r13)
794 bl save_nvgprs
795 bl DebugException
796 b ret_from_except
797
798kernel_dbg_exc:
799 b .
800
801
802 START_EXCEPTION(debug_debug);
803 DBG_EXCEPTION_PROLOG(0xd00, BOOKE_INTERRUPT_DEBUG,
804 PROLOG_ADDITION_2REGS)
805
806
807
808
809
810
811
812
813
814
815 mfspr r14,SPRN_DBSR
816 andis. r15,r14,(DBSR_IC|DBSR_BT)@h
817 beq+ 1f
818
819#ifdef CONFIG_RELOCATABLE
820 ld r15,PACATOC(r13)
821 ld r14,interrupt_base_book3e@got(r15)
822 ld r15,__end_interrupts@got(r15)
823#else
824 LOAD_REG_IMMEDIATE(r14,interrupt_base_book3e)
825 LOAD_REG_IMMEDIATE(r15,__end_interrupts)
826#endif
827 cmpld cr0,r10,r14
828 cmpld cr1,r10,r15
829 blt+ cr0,1f
830 bge+ cr1,1f
831
832
833 lis r14,(DBSR_IC|DBSR_BT)@h
834 rlwinm r11,r11,0,~MSR_DE
835 mtspr SPRN_DBSR,r14
836 mtspr SPRN_DSRR1,r11
837 lwz r10,PACA_EXDBG+EX_CR(r13)
838 ld r1,PACA_EXDBG+EX_R1(r13)
839 ld r14,PACA_EXDBG+EX_R14(r13)
840 ld r15,PACA_EXDBG+EX_R15(r13)
841 mtcr r10
842 ld r10,PACA_EXDBG+EX_R10(r13)
843 ld r11,PACA_EXDBG+EX_R11(r13)
844 mfspr r13,SPRN_SPRG_DBG_SCRATCH
845 rfdi
846
847
848
849
850
8511: andi. r14,r11,MSR_PR;
852 beq kernel_dbg_exc;
853
854
855
856
857 mfspr r14,SPRN_DBSR
858 EXCEPTION_COMMON_DBG(0xd08)
859 INTS_DISABLE
860 std r14,_DSISR(r1)
861 addi r3,r1,STACK_FRAME_OVERHEAD
862 mr r4,r14
863 ld r14,PACA_EXDBG+EX_R14(r13)
864 ld r15,PACA_EXDBG+EX_R15(r13)
865 bl save_nvgprs
866 bl DebugException
867 b ret_from_except
868
869 START_EXCEPTION(perfmon);
870 NORMAL_EXCEPTION_PROLOG(0x260, BOOKE_INTERRUPT_PERFORMANCE_MONITOR,
871 PROLOG_ADDITION_NONE)
872 EXCEPTION_COMMON(0x260)
873 INTS_DISABLE
874 CHECK_NAPPING()
875 addi r3,r1,STACK_FRAME_OVERHEAD
876 bl performance_monitor_exception
877 b ret_from_except_lite
878
879
880 MASKABLE_EXCEPTION(0x280, BOOKE_INTERRUPT_DOORBELL,
881 doorbell, doorbell_exception, ACK_NONE)
882
883
884 START_EXCEPTION(doorbell_crit);
885 CRIT_EXCEPTION_PROLOG(0x2a0, BOOKE_INTERRUPT_DOORBELL_CRITICAL,
886 PROLOG_ADDITION_NONE)
887 EXCEPTION_COMMON_CRIT(0x2a0)
888 bl save_nvgprs
889 bl special_reg_save
890 CHECK_NAPPING();
891 addi r3,r1,STACK_FRAME_OVERHEAD
892 bl unknown_exception
893 b ret_from_crit_except
894
895
896
897
898
899 START_EXCEPTION(guest_doorbell);
900 GDBELL_EXCEPTION_PROLOG(0x2c0, BOOKE_INTERRUPT_GUEST_DBELL,
901 PROLOG_ADDITION_NONE)
902 EXCEPTION_COMMON(0x2c0)
903 addi r3,r1,STACK_FRAME_OVERHEAD
904 bl save_nvgprs
905 INTS_RESTORE_HARD
906 bl unknown_exception
907 b ret_from_except
908
909
910 START_EXCEPTION(guest_doorbell_crit);
911 CRIT_EXCEPTION_PROLOG(0x2e0, BOOKE_INTERRUPT_GUEST_DBELL_CRIT,
912 PROLOG_ADDITION_NONE)
913 EXCEPTION_COMMON_CRIT(0x2e0)
914 bl save_nvgprs
915 bl special_reg_save
916 CHECK_NAPPING();
917 addi r3,r1,STACK_FRAME_OVERHEAD
918 bl unknown_exception
919 b ret_from_crit_except
920
921
922 START_EXCEPTION(hypercall);
923 NORMAL_EXCEPTION_PROLOG(0x310, BOOKE_INTERRUPT_HV_SYSCALL,
924 PROLOG_ADDITION_NONE)
925 EXCEPTION_COMMON(0x310)
926 addi r3,r1,STACK_FRAME_OVERHEAD
927 bl save_nvgprs
928 INTS_RESTORE_HARD
929 bl unknown_exception
930 b ret_from_except
931
932
933 START_EXCEPTION(ehpriv);
934 NORMAL_EXCEPTION_PROLOG(0x320, BOOKE_INTERRUPT_HV_PRIV,
935 PROLOG_ADDITION_NONE)
936 EXCEPTION_COMMON(0x320)
937 addi r3,r1,STACK_FRAME_OVERHEAD
938 bl save_nvgprs
939 INTS_RESTORE_HARD
940 bl unknown_exception
941 b ret_from_except
942
943
944 START_EXCEPTION(lrat_error);
945 NORMAL_EXCEPTION_PROLOG(0x340, BOOKE_INTERRUPT_LRAT_ERROR,
946 PROLOG_ADDITION_NONE)
947 EXCEPTION_COMMON(0x340)
948 addi r3,r1,STACK_FRAME_OVERHEAD
949 bl save_nvgprs
950 INTS_RESTORE_HARD
951 bl unknown_exception
952 b ret_from_except
953
954
955
956
957
958
959
960
961.macro masked_interrupt_book3e paca_irq full_mask
962 lbz r10,PACAIRQHAPPENED(r13)
963 .if \full_mask == 1
964 ori r10,r10,\paca_irq | PACA_IRQ_HARD_DIS
965 .else
966 ori r10,r10,\paca_irq
967 .endif
968 stb r10,PACAIRQHAPPENED(r13)
969
970 .if \full_mask == 1
971 rldicl r10,r11,48,1
972 rotldi r11,r10,16
973 mtspr SPRN_SRR1,r11
974 .endif
975
976 lwz r11,PACA_EXGEN+EX_CR(r13)
977 mtcr r11
978 ld r10,PACA_EXGEN+EX_R10(r13)
979 ld r11,PACA_EXGEN+EX_R11(r13)
980 mfspr r13,SPRN_SPRG_GEN_SCRATCH
981 rfi
982 b .
983.endm
984
985masked_interrupt_book3e_0x500:
986
987 masked_interrupt_book3e PACA_IRQ_EE 1
988
989masked_interrupt_book3e_0x900:
990 ACK_DEC(r10);
991 masked_interrupt_book3e PACA_IRQ_DEC 0
992
993masked_interrupt_book3e_0x980:
994 ACK_FIT(r10);
995 masked_interrupt_book3e PACA_IRQ_DEC 0
996
997masked_interrupt_book3e_0x280:
998masked_interrupt_book3e_0x2c0:
999 masked_interrupt_book3e PACA_IRQ_DBELL 0
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011_GLOBAL(__replay_interrupt)
1012
1013
1014
1015
1016 mflr r10
1017 mfmsr r11
1018 mfcr r4
1019 mtspr SPRN_SPRG_GEN_SCRATCH,r13;
1020 std r1,PACA_EXGEN+EX_R1(r13);
1021 stw r4,PACA_EXGEN+EX_CR(r13);
1022 ori r11,r11,MSR_EE
1023 subi r1,r1,INT_FRAME_SIZE;
1024 cmpwi cr0,r3,0x500
1025 beq exc_0x500_common
1026 cmpwi cr0,r3,0x900
1027 beq exc_0x900_common
1028 cmpwi cr0,r3,0x280
1029 beq exc_0x280_common
1030 blr
1031
1032
1033
1034
1035
1036
1037
1038storage_fault_common:
1039 std r14,_DAR(r1)
1040 std r15,_DSISR(r1)
1041 addi r3,r1,STACK_FRAME_OVERHEAD
1042 mr r4,r14
1043 mr r5,r15
1044 ld r14,PACA_EXGEN+EX_R14(r13)
1045 ld r15,PACA_EXGEN+EX_R15(r13)
1046 bl do_page_fault
1047 cmpdi r3,0
1048 bne- 1f
1049 b ret_from_except_lite
10501: bl save_nvgprs
1051 mr r5,r3
1052 addi r3,r1,STACK_FRAME_OVERHEAD
1053 ld r4,_DAR(r1)
1054 bl bad_page_fault
1055 b ret_from_except
1056
1057
1058
1059
1060
1061alignment_more:
1062 std r14,_DAR(r1)
1063 std r15,_DSISR(r1)
1064 addi r3,r1,STACK_FRAME_OVERHEAD
1065 ld r14,PACA_EXGEN+EX_R14(r13)
1066 ld r15,PACA_EXGEN+EX_R15(r13)
1067 bl save_nvgprs
1068 INTS_RESTORE_HARD
1069 bl alignment_exception
1070 b ret_from_except
1071
1072
1073
1074
1075
1076_GLOBAL(exception_return_book3e)
1077 b 1f
1078
1079
1080
1081
1082 .globl fast_exception_return
1083fast_exception_return:
1084 wrteei 0
10851: mr r0,r13
1086 ld r10,_MSR(r1)
1087 REST_4GPRS(2, r1)
1088 andi. r6,r10,MSR_PR
1089 REST_2GPRS(6, r1)
1090 beq 1f
1091 ACCOUNT_CPU_USER_EXIT(r13, r10, r11)
1092 ld r0,GPR13(r1)
1093
10941: stdcx. r0,0,r1
1095
1096 ld r8,_CCR(r1)
1097 ld r9,_LINK(r1)
1098 ld r10,_CTR(r1)
1099 ld r11,_XER(r1)
1100 mtcr r8
1101 mtlr r9
1102 mtctr r10
1103 mtxer r11
1104 REST_2GPRS(8, r1)
1105 ld r10,GPR10(r1)
1106 ld r11,GPR11(r1)
1107 ld r12,GPR12(r1)
1108 mtspr SPRN_SPRG_GEN_SCRATCH,r0
1109
1110 std r10,PACA_EXGEN+EX_R10(r13);
1111 std r11,PACA_EXGEN+EX_R11(r13);
1112 ld r10,_NIP(r1)
1113 ld r11,_MSR(r1)
1114 ld r0,GPR0(r1)
1115 ld r1,GPR1(r1)
1116 mtspr SPRN_SRR0,r10
1117 mtspr SPRN_SRR1,r11
1118 ld r10,PACA_EXGEN+EX_R10(r13)
1119 ld r11,PACA_EXGEN+EX_R11(r13)
1120 mfspr r13,SPRN_SPRG_GEN_SCRATCH
1121 rfi
1122
1123
1124
1125
1126
1127
1128
1129
1130BAD_STACK_TRAMPOLINE(0x000)
1131BAD_STACK_TRAMPOLINE(0x100)
1132BAD_STACK_TRAMPOLINE(0x200)
1133BAD_STACK_TRAMPOLINE(0x220)
1134BAD_STACK_TRAMPOLINE(0x260)
1135BAD_STACK_TRAMPOLINE(0x280)
1136BAD_STACK_TRAMPOLINE(0x2a0)
1137BAD_STACK_TRAMPOLINE(0x2c0)
1138BAD_STACK_TRAMPOLINE(0x2e0)
1139BAD_STACK_TRAMPOLINE(0x300)
1140BAD_STACK_TRAMPOLINE(0x310)
1141BAD_STACK_TRAMPOLINE(0x320)
1142BAD_STACK_TRAMPOLINE(0x340)
1143BAD_STACK_TRAMPOLINE(0x400)
1144BAD_STACK_TRAMPOLINE(0x500)
1145BAD_STACK_TRAMPOLINE(0x600)
1146BAD_STACK_TRAMPOLINE(0x700)
1147BAD_STACK_TRAMPOLINE(0x800)
1148BAD_STACK_TRAMPOLINE(0x900)
1149BAD_STACK_TRAMPOLINE(0x980)
1150BAD_STACK_TRAMPOLINE(0x9f0)
1151BAD_STACK_TRAMPOLINE(0xa00)
1152BAD_STACK_TRAMPOLINE(0xb00)
1153BAD_STACK_TRAMPOLINE(0xc00)
1154BAD_STACK_TRAMPOLINE(0xd00)
1155BAD_STACK_TRAMPOLINE(0xd08)
1156BAD_STACK_TRAMPOLINE(0xe00)
1157BAD_STACK_TRAMPOLINE(0xf00)
1158BAD_STACK_TRAMPOLINE(0xf20)
1159
1160 .globl bad_stack_book3e
1161bad_stack_book3e:
1162
1163 mfspr r10,SPRN_SRR0;
1164 ld r1,PACAEMERGSP(r13)
1165 subi r1,r1,64+INT_FRAME_SIZE
1166 std r10,_NIP(r1)
1167 std r11,_MSR(r1)
1168 ld r10,PACA_EXGEN+EX_R1(r13)
1169 lwz r11,PACA_EXGEN+EX_CR(r13)
1170 std r10,GPR1(r1)
1171 std r11,_CCR(r1)
1172 mfspr r10,SPRN_DEAR
1173 mfspr r11,SPRN_ESR
1174 std r10,_DAR(r1)
1175 std r11,_DSISR(r1)
1176 std r0,GPR0(r1); \
1177 std r2,GPR2(r1); \
1178 SAVE_4GPRS(3, r1); \
1179 SAVE_2GPRS(7, r1); \
1180 std r9,GPR9(r1); \
1181 ld r3,PACA_EXGEN+EX_R10(r13); \
1182 ld r4,PACA_EXGEN+EX_R11(r13); \
1183 mfspr r5,SPRN_SPRG_GEN_SCRATCH; \
1184 std r3,GPR10(r1); \
1185 std r4,GPR11(r1); \
1186 std r12,GPR12(r1); \
1187 std r5,GPR13(r1); \
1188 mflr r10
1189 mfctr r11
1190 mfxer r12
1191 std r10,_LINK(r1)
1192 std r11,_CTR(r1)
1193 std r12,_XER(r1)
1194 SAVE_10GPRS(14,r1)
1195 SAVE_8GPRS(24,r1)
1196 lhz r12,PACA_TRAP_SAVE(r13)
1197 std r12,_TRAP(r1)
1198 addi r11,r1,INT_FRAME_SIZE
1199 std r11,0(r1)
1200 li r12,0
1201 std r12,0(r11)
1202 ld r2,PACATOC(r13)
12031: addi r3,r1,STACK_FRAME_OVERHEAD
1204 bl kernel_bad_stack
1205 b 1b
1206
1207
1208
1209
1210
1211
1212_GLOBAL(initial_tlb_book3e)
1213
1214
1215 mfspr r4,SPRN_TLB0CFG
1216 andi. r3,r4,TLBnCFG_IPROT
1217 lis r3,MAS0_TLBSEL(0)@h
1218 bne found_iprot
1219
1220 mfspr r4,SPRN_TLB1CFG
1221 andi. r3,r4,TLBnCFG_IPROT
1222 lis r3,MAS0_TLBSEL(1)@h
1223 bne found_iprot
1224
1225 mfspr r4,SPRN_TLB2CFG
1226 andi. r3,r4,TLBnCFG_IPROT
1227 lis r3,MAS0_TLBSEL(2)@h
1228 bne found_iprot
1229
1230 lis r3,MAS0_TLBSEL(3)@h
1231 mfspr r4,SPRN_TLB3CFG
1232
1233
1234found_iprot:
1235 andi. r5,r4,TLBnCFG_HES
1236 bne have_hes
1237
1238 mflr r8
1239
1240
1241
1242
1243
1244 bl invstr
1245invstr: mflr r6
1246 mfmsr r7
1247 rlwinm r5,r7,27,31,31
1248 mfspr r7,SPRN_PID
1249 slwi r7,r7,16
1250 or r7,r7,r5
1251 mtspr SPRN_MAS6,r7
1252 tlbsx 0,r6
1253
1254 mfspr r3,SPRN_MAS0
1255 rlwinm r5,r3,16,20,31
1256
1257 mfspr r7,SPRN_MAS1
1258 oris r7,r7,MAS1_IPROT@h
1259 mtspr SPRN_MAS1,r7
1260 tlbwe
1261
1262
1263
1264
1265
1266
1267
1268 andi. r4,r4,TLBnCFG_N_ENTRY
1269 li r6,0
12701: mr r7,r3
1271 rlwimi r7,r6,16,4,15
1272 mtspr SPRN_MAS0,r7
1273 tlbre
1274 mfspr r7,SPRN_MAS1
1275 rlwinm r7,r7,0,2,31
1276 cmpw r5,r6
1277 beq skpinv
1278 mtspr SPRN_MAS1,r7
1279 tlbwe
1280 isync
1281skpinv: addi r6,r6,1
1282 cmpw r6,r4
1283 bne 1b
1284
1285
1286 PPC_TLBILX_ALL(0,R0)
1287 sync
1288 isync
1289
1290
1291
1292
1293
1294
1295 andi. r7,r5,0x1
1296 addi r7,r7,0x1
1297 mr r4,r3
1298 mtspr SPRN_MAS0,r4
1299 tlbre
1300
1301 rlwimi r4,r7,16,4,15
1302 mtspr SPRN_MAS0,r4
1303
1304 mfspr r7,SPRN_MAS1
1305 xori r6,r7,MAS1_TS
1306 mtspr SPRN_MAS1,r6
1307
1308 tlbwe
1309
1310 mfmsr r6
1311 xori r6,r6,MSR_IS
1312 mtspr SPRN_SRR1,r6
1313 bl 1f
13141: mflr r6
1315 addi r6,r6,(2f - 1b)
1316 mtspr SPRN_SRR0,r6
1317 rfi
13182:
1319
1320
1321
1322
1323
1324
1325
1326 li r6,0
1327 mtspr SPRN_MAS6,r6
1328 mtspr SPRN_PID,r6
1329
1330
1331
1332
1333
1334
1335
1336 mtspr SPRN_MAS0,r3
1337 tlbre
1338 mfspr r6,SPRN_MAS1
1339 rlwinm r6,r6,0,2,31
1340 mtspr SPRN_MAS1,r6
1341 tlbwe
1342 sync
1343 isync
1344
1345
1346
1347
1348
1349
1350#define M_IF_NEEDED MAS2_M
1351#else
1352#define M_IF_NEEDED 0
1353#endif
1354
1355
1356
1357
1358
1359
1360
1361 rlwinm r3,r3,0,16,3
1362 mtspr SPRN_MAS0,r3
1363 lis r6,(MAS1_VALID|MAS1_IPROT)@h
1364 ori r6,r6,(MAS1_TSIZE(BOOK3E_PAGESZ_1GB))@l
1365 mtspr SPRN_MAS1,r6
1366
1367 LOAD_REG_IMMEDIATE(r6, PAGE_OFFSET | M_IF_NEEDED)
1368 mtspr SPRN_MAS2,r6
1369
1370 rlwinm r5,r5,0,0,25
1371 ori r5,r5,MAS3_SR | MAS3_SW | MAS3_SX
1372 mtspr SPRN_MAS3,r5
1373 li r5,-1
1374 rlwinm r5,r5,0,0,25
1375
1376 tlbwe
1377
1378
1379
1380
1381
1382
1383 bl 1f
13841: mflr r6
1385 addi r6,r6,(2f - 1b)
1386 tovirt(r6,r6)
1387 lis r7,MSR_KERNEL@h
1388 ori r7,r7,MSR_KERNEL@l
1389 mtspr SPRN_SRR0,r6
1390 mtspr SPRN_SRR1,r7
1391 rfi
13922:
1393
1394
1395
1396
1397
1398 mtspr SPRN_MAS0,r4
1399 tlbre
1400 mfspr r5,SPRN_MAS1
1401 rlwinm r5,r5,0,2,31
1402 mtspr SPRN_MAS1,r5
1403 tlbwe
1404 sync
1405 isync
1406
1407
1408 tovirt(r8,r8)
1409 mtlr r8
1410 blr
1411
1412have_hes:
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429 .globl a2_tlbinit_code_start
1430a2_tlbinit_code_start:
1431
1432 ori r11,r3,MAS0_WQ_ALLWAYS
1433 oris r11,r11,MAS0_ESEL(3)@h
1434 mtspr SPRN_MAS0,r11
1435 lis r3,(MAS1_VALID | MAS1_IPROT)@h
1436 ori r3,r3,BOOK3E_PAGESZ_1GB << MAS1_TSIZE_SHIFT
1437 mtspr SPRN_MAS1,r3
1438 LOAD_REG_IMMEDIATE(r3, PAGE_OFFSET | MAS2_M)
1439 mtspr SPRN_MAS2,r3
1440 li r3,MAS3_SR | MAS3_SW | MAS3_SX
1441 mtspr SPRN_MAS7_MAS3,r3
1442 li r3,0
1443 mtspr SPRN_MAS8,r3
1444
1445
1446 tlbwe
1447
1448 .globl a2_tlbinit_after_linear_map
1449a2_tlbinit_after_linear_map:
1450
1451
1452 LOAD_REG_IMMEDIATE(r3,1f)
1453 mtctr r3
1454 bctr
1455
14561:
1457
1458
1459
1460
1461
1462 li r5,0
1463 mtspr SPRN_MAS6,r5
1464 tlbsx 0,r3
1465
1466 rlwinm r9,r4,0,TLBnCFG_N_ENTRY
1467 rlwinm r10,r4,8,0xff
1468 addi r10,r10,-1
1469
1470 li r3,1
1471
1472 mfspr r5,SPRN_MAS1
1473 rlwinm r5,r5,0,(~(MAS1_VALID|MAS1_IPROT))
1474
1475 mfspr r6,SPRN_MAS2
1476 rldicr r6,r6,0,51
1477
1478 mfspr r7,SPRN_MAS0
1479 rlwinm r7,r7,0,0xffff0fff
1480
1481 rlwinm r8,r7,16,0xfff
1482
14832: add r4,r3,r8
1484 and r4,r4,r10
1485
1486 rlwimi r7,r4,16,MAS0_ESEL_MASK
1487
1488 mtspr SPRN_MAS0,r7
1489 mtspr SPRN_MAS1,r5
1490 mtspr SPRN_MAS2,r6
1491 tlbwe
1492
1493 addi r3,r3,1
1494 and. r4,r3,r10
1495
1496 bne 3f
1497 addis r6,r6,(1<<30)@h
14983:
1499 cmpw r3,r9
1500 blt 2b
1501
1502 .globl a2_tlbinit_after_iprot_flush
1503a2_tlbinit_after_iprot_flush:
1504
1505 PPC_TLBILX(0,0,R0)
1506 sync
1507 isync
1508
1509 .globl a2_tlbinit_code_end
1510a2_tlbinit_code_end:
1511
1512
1513 mflr r3
1514 tovirt(r3,r3)
1515 mtlr r3
1516 blr
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535_GLOBAL(start_initialization_book3e)
1536 mflr r28
1537
1538
1539
1540
1541
1542
1543 bl initial_tlb_book3e
1544
1545
1546 bl init_core_book3e
1547
1548
1549 bl init_thread_book3e
1550
1551
1552 tovirt(r28,r28)
1553 mtlr r28
1554 blr
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568_GLOBAL(book3e_secondary_core_init_tlb_set)
1569 li r4,1
1570 b generic_secondary_smp_init
1571
1572_GLOBAL(book3e_secondary_core_init)
1573 mflr r28
1574
1575
1576 cmplwi r4,0
1577 bne 2f
1578
1579
1580 bl initial_tlb_book3e
1581
1582
1583
1584
1585 bl relative_toc
1586
1587
15882: bl init_core_book3e
1589
1590
15913: bl init_thread_book3e
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606 cmpdi cr0,r28,0
1607 blt 1f
1608 lis r3,PAGE_OFFSET@highest
1609 sldi r3,r3,32
1610 or r28,r28,r3
16111: mtlr r28
1612 blr
1613
1614_GLOBAL(book3e_secondary_thread_init)
1615 mflr r28
1616 b 3b
1617
1618 .globl init_core_book3e
1619init_core_book3e:
1620
1621 tovirt(r2,r2)
1622 LOAD_REG_ADDR(r3, interrupt_base_book3e)
1623 mtspr SPRN_IVPR,r3
1624 sync
1625 blr
1626
1627init_thread_book3e:
1628 lis r3,(SPRN_EPCR_ICM | SPRN_EPCR_GICM)@h
1629 mtspr SPRN_EPCR,r3
1630
1631
1632 wrteei 0
1633
1634
1635 li r3,0
1636 mtspr SPRN_TCR,r3
1637 mfspr r3,SPRN_TSR
1638 mtspr SPRN_TSR,r3
1639
1640 blr
1641
1642_GLOBAL(__setup_base_ivors)
1643 SET_IVOR(0, 0x020)
1644 SET_IVOR(1, 0x000)
1645 SET_IVOR(2, 0x060)
1646 SET_IVOR(3, 0x080)
1647 SET_IVOR(4, 0x0a0)
1648 SET_IVOR(5, 0x0c0)
1649 SET_IVOR(6, 0x0e0)
1650 SET_IVOR(7, 0x100)
1651 SET_IVOR(8, 0x120)
1652 SET_IVOR(9, 0x140)
1653 SET_IVOR(10, 0x160)
1654 SET_IVOR(11, 0x180)
1655 SET_IVOR(12, 0x1a0)
1656 SET_IVOR(13, 0x1c0)
1657 SET_IVOR(14, 0x1e0)
1658 SET_IVOR(15, 0x040)
1659
1660 sync
1661
1662 blr
1663
1664_GLOBAL(setup_altivec_ivors)
1665 SET_IVOR(32, 0x200)
1666 SET_IVOR(33, 0x220)
1667 blr
1668
1669_GLOBAL(setup_perfmon_ivor)
1670 SET_IVOR(35, 0x260)
1671 blr
1672
1673_GLOBAL(setup_doorbell_ivors)
1674 SET_IVOR(36, 0x280)
1675 SET_IVOR(37, 0x2a0)
1676 blr
1677
1678_GLOBAL(setup_ehv_ivors)
1679 SET_IVOR(40, 0x300)
1680 SET_IVOR(41, 0x320)
1681 SET_IVOR(38, 0x2c0)
1682 SET_IVOR(39, 0x2e0)
1683 blr
1684
1685_GLOBAL(setup_lrat_ivor)
1686 SET_IVOR(42, 0x340)
1687 blr
1688