1
2
3
4
5
6
7
8
9
10
11
12#include <linux/threads.h>
13#include <asm/reg.h>
14#include <asm/page.h>
15#include <asm/ppc_asm.h>
16#include <asm/asm-offsets.h>
17#include <asm/cputable.h>
18#include <asm/setup.h>
19#include <asm/thread_info.h>
20#include <asm/reg_a2.h>
21#include <asm/exception-64e.h>
22#include <asm/bug.h>
23#include <asm/irqflags.h>
24#include <asm/ptrace.h>
25#include <asm/ppc-opcode.h>
26#include <asm/mmu.h>
27#include <asm/hw_irq.h>
28#include <asm/kvm_asm.h>
29#include <asm/kvm_booke_hv_asm.h>
30
31
32
33
34
35
36
37#define SPECIAL_EXC_SRR0 0
38#define SPECIAL_EXC_SRR1 1
39#define SPECIAL_EXC_SPRG_GEN 2
40#define SPECIAL_EXC_SPRG_TLB 3
41#define SPECIAL_EXC_MAS0 4
42#define SPECIAL_EXC_MAS1 5
43#define SPECIAL_EXC_MAS2 6
44#define SPECIAL_EXC_MAS3 7
45#define SPECIAL_EXC_MAS6 8
46#define SPECIAL_EXC_MAS7 9
47#define SPECIAL_EXC_MAS5 10
48#define SPECIAL_EXC_MAS8 11
49#define SPECIAL_EXC_IRQHAPPENED 12
50#define SPECIAL_EXC_DEAR 13
51#define SPECIAL_EXC_ESR 14
52#define SPECIAL_EXC_SOFTE 15
53#define SPECIAL_EXC_CSRR0 16
54#define SPECIAL_EXC_CSRR1 17
55
56#define SPECIAL_EXC_END 18
57
58#define SPECIAL_EXC_FRAME_SIZE (INT_FRAME_SIZE + SPECIAL_EXC_END * 8)
59#define SPECIAL_EXC_FRAME_OFFS (INT_FRAME_SIZE - 288)
60
61#define SPECIAL_EXC_STORE(reg, name) \
62 std reg, (SPECIAL_EXC_
63
64#define SPECIAL_EXC_LOAD(reg, name) \
65 ld reg, (SPECIAL_EXC_
66
67special_reg_save:
68 lbz r9,PACAIRQHAPPENED(r13)
69 RECONCILE_IRQ_STATE(r3,r4)
70
71
72
73
74
75 ld r3,_MSR(r1)
76 andi. r3,r3,MSR_PR
77 bnelr
78
79
80 ld r11,PACAKSAVE(r13)
81 CURRENT_THREAD_INFO(r11, r11)
82 CURRENT_THREAD_INFO(r12, r1)
83 ld r10,TI_FLAGS(r11)
84 std r10,TI_FLAGS(r12)
85 ld r10,TI_PREEMPT(r11)
86 std r10,TI_PREEMPT(r12)
87 ld r10,TI_TASK(r11)
88 std r10,TI_TASK(r12)
89
90
91
92
93
94 LOAD_REG_ADDR(r11,extlb_level_exc)
95 lwz r12,0(r11)
96 mfspr r10,SPRN_SPRG_TLB_EXFRAME
97 add r10,r10,r12
98 mtspr SPRN_SPRG_TLB_EXFRAME,r10
99
100
101
102
103
104 mfspr r10,SPRN_SRR0
105 SPECIAL_EXC_STORE(r10,SRR0)
106 mfspr r10,SPRN_SRR1
107 SPECIAL_EXC_STORE(r10,SRR1)
108 mfspr r10,SPRN_SPRG_GEN_SCRATCH
109 SPECIAL_EXC_STORE(r10,SPRG_GEN)
110 mfspr r10,SPRN_SPRG_TLB_SCRATCH
111 SPECIAL_EXC_STORE(r10,SPRG_TLB)
112 mfspr r10,SPRN_MAS0
113 SPECIAL_EXC_STORE(r10,MAS0)
114 mfspr r10,SPRN_MAS1
115 SPECIAL_EXC_STORE(r10,MAS1)
116 mfspr r10,SPRN_MAS2
117 SPECIAL_EXC_STORE(r10,MAS2)
118 mfspr r10,SPRN_MAS3
119 SPECIAL_EXC_STORE(r10,MAS3)
120 mfspr r10,SPRN_MAS6
121 SPECIAL_EXC_STORE(r10,MAS6)
122 mfspr r10,SPRN_MAS7
123 SPECIAL_EXC_STORE(r10,MAS7)
124BEGIN_FTR_SECTION
125 mfspr r10,SPRN_MAS5
126 SPECIAL_EXC_STORE(r10,MAS5)
127 mfspr r10,SPRN_MAS8
128 SPECIAL_EXC_STORE(r10,MAS8)
129
130
131 li r10,0
132 mtspr SPRN_MAS5,r10
133 mtspr SPRN_MAS8,r10
134END_FTR_SECTION_IFSET(CPU_FTR_EMB_HV)
135 SPECIAL_EXC_STORE(r9,IRQHAPPENED)
136
137 mfspr r10,SPRN_DEAR
138 SPECIAL_EXC_STORE(r10,DEAR)
139 mfspr r10,SPRN_ESR
140 SPECIAL_EXC_STORE(r10,ESR)
141
142 lbz r10,PACASOFTIRQEN(r13)
143 SPECIAL_EXC_STORE(r10,SOFTE)
144 ld r10,_NIP(r1)
145 SPECIAL_EXC_STORE(r10,CSRR0)
146 ld r10,_MSR(r1)
147 SPECIAL_EXC_STORE(r10,CSRR1)
148
149 blr
150
151ret_from_level_except:
152 ld r3,_MSR(r1)
153 andi. r3,r3,MSR_PR
154 beq 1f
155 b ret_from_except
1561:
157
158 LOAD_REG_ADDR(r11,extlb_level_exc)
159 lwz r12,0(r11)
160 mfspr r10,SPRN_SPRG_TLB_EXFRAME
161 sub r10,r10,r12
162 mtspr SPRN_SPRG_TLB_EXFRAME,r10
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178 PPC_TLBILX_ALL(0,R0)
179
180 REST_NVGPRS(r1)
181
182 SPECIAL_EXC_LOAD(r10,SRR0)
183 mtspr SPRN_SRR0,r10
184 SPECIAL_EXC_LOAD(r10,SRR1)
185 mtspr SPRN_SRR1,r10
186 SPECIAL_EXC_LOAD(r10,SPRG_GEN)
187 mtspr SPRN_SPRG_GEN_SCRATCH,r10
188 SPECIAL_EXC_LOAD(r10,SPRG_TLB)
189 mtspr SPRN_SPRG_TLB_SCRATCH,r10
190 SPECIAL_EXC_LOAD(r10,MAS0)
191 mtspr SPRN_MAS0,r10
192 SPECIAL_EXC_LOAD(r10,MAS1)
193 mtspr SPRN_MAS1,r10
194 SPECIAL_EXC_LOAD(r10,MAS2)
195 mtspr SPRN_MAS2,r10
196 SPECIAL_EXC_LOAD(r10,MAS3)
197 mtspr SPRN_MAS3,r10
198 SPECIAL_EXC_LOAD(r10,MAS6)
199 mtspr SPRN_MAS6,r10
200 SPECIAL_EXC_LOAD(r10,MAS7)
201 mtspr SPRN_MAS7,r10
202BEGIN_FTR_SECTION
203 SPECIAL_EXC_LOAD(r10,MAS5)
204 mtspr SPRN_MAS5,r10
205 SPECIAL_EXC_LOAD(r10,MAS8)
206 mtspr SPRN_MAS8,r10
207END_FTR_SECTION_IFSET(CPU_FTR_EMB_HV)
208
209 lbz r6,PACASOFTIRQEN(r13)
210 ld r5,SOFTE(r1)
211
212
213 twnei r6,0
214
215 cmpwi cr0,r5,0
216 beq 1f
217
218 TRACE_ENABLE_INTS
219 stb r5,PACASOFTIRQEN(r13)
2201:
221
222
223
224
225
226
227 SPECIAL_EXC_LOAD(r10,IRQHAPPENED)
228 stb r10,PACAIRQHAPPENED(r13)
229
230 SPECIAL_EXC_LOAD(r10,DEAR)
231 mtspr SPRN_DEAR,r10
232 SPECIAL_EXC_LOAD(r10,ESR)
233 mtspr SPRN_ESR,r10
234
235 stdcx. r0,0,r1
236
237 REST_4GPRS(2, r1)
238 REST_4GPRS(6, r1)
239
240 ld r10,_CTR(r1)
241 ld r11,_XER(r1)
242 mtctr r10
243 mtxer r11
244
245 blr
246
247.macro ret_from_level srr0 srr1 paca_ex scratch
248 bl ret_from_level_except
249
250 ld r10,_LINK(r1)
251 ld r11,_CCR(r1)
252 ld r0,GPR13(r1)
253 mtlr r10
254 mtcr r11
255
256 ld r10,GPR10(r1)
257 ld r11,GPR11(r1)
258 ld r12,GPR12(r1)
259 mtspr \scratch,r0
260
261 std r10,\paca_ex+EX_R10(r13);
262 std r11,\paca_ex+EX_R11(r13);
263 ld r10,_NIP(r1)
264 ld r11,_MSR(r1)
265 ld r0,GPR0(r1)
266 ld r1,GPR1(r1)
267 mtspr \srr0,r10
268 mtspr \srr1,r11
269 ld r10,\paca_ex+EX_R10(r13)
270 ld r11,\paca_ex+EX_R11(r13)
271 mfspr r13,\scratch
272.endm
273
274ret_from_crit_except:
275 ret_from_level SPRN_CSRR0 SPRN_CSRR1 PACA_EXCRIT SPRN_SPRG_CRIT_SCRATCH
276 rfci
277
278ret_from_mc_except:
279 ret_from_level SPRN_MCSRR0 SPRN_MCSRR1 PACA_EXMC SPRN_SPRG_MC_SCRATCH
280 rfmci
281
282
283#define EXCEPTION_PROLOG(n, intnum, type, addition) \
284 mtspr SPRN_SPRG_
285 mfspr r13,SPRN_SPRG_PACA; \
286 std r10,PACA_EX
287 std r11,PACA_EX
288 mfcr r10; \
289 mfspr r11,SPRN_
290 DO_KVM intnum,SPRN_
291 stw r10,PACA_EX
292 addition; \
293 std r1,PACA_EX
294 type
295 andi. r10,r11,MSR_PR; \
296 beq 1f; \
297 ld r1,PACAKSAVE(r13); \
2981: cmpdi cr1,r1,0; \
299 bge- cr1,exc_
300 mfspr r10,SPRN_
301
302
303#define GEN_SET_KSTACK \
304 subi r1,r1,INT_FRAME_SIZE;
305#define SPRN_GEN_SRR0 SPRN_SRR0
306#define SPRN_GEN_SRR1 SPRN_SRR1
307
308#define GDBELL_SET_KSTACK GEN_SET_KSTACK
309#define SPRN_GDBELL_SRR0 SPRN_GSRR0
310#define SPRN_GDBELL_SRR1 SPRN_GSRR1
311
312#define CRIT_SET_KSTACK \
313 ld r1,PACA_CRIT_STACK(r13); \
314 subi r1,r1,SPECIAL_EXC_FRAME_SIZE
315#define SPRN_CRIT_SRR0 SPRN_CSRR0
316#define SPRN_CRIT_SRR1 SPRN_CSRR1
317
318#define DBG_SET_KSTACK \
319 ld r1,PACA_DBG_STACK(r13); \
320 subi r1,r1,SPECIAL_EXC_FRAME_SIZE
321#define SPRN_DBG_SRR0 SPRN_DSRR0
322#define SPRN_DBG_SRR1 SPRN_DSRR1
323
324#define MC_SET_KSTACK \
325 ld r1,PACA_MC_STACK(r13); \
326 subi r1,r1,SPECIAL_EXC_FRAME_SIZE
327#define SPRN_MC_SRR0 SPRN_MCSRR0
328#define SPRN_MC_SRR1 SPRN_MCSRR1
329
330#define NORMAL_EXCEPTION_PROLOG(n, intnum, addition) \
331 EXCEPTION_PROLOG(n, intnum, GEN, addition
332
333#define CRIT_EXCEPTION_PROLOG(n, intnum, addition) \
334 EXCEPTION_PROLOG(n, intnum, CRIT, addition
335
336#define DBG_EXCEPTION_PROLOG(n, intnum, addition) \
337 EXCEPTION_PROLOG(n, intnum, DBG, addition
338
339#define MC_EXCEPTION_PROLOG(n, intnum, addition) \
340 EXCEPTION_PROLOG(n, intnum, MC, addition
341
342#define GDBELL_EXCEPTION_PROLOG(n, intnum, addition) \
343 EXCEPTION_PROLOG(n, intnum, GDBELL, addition
344
345
346
347#define PROLOG_ADDITION_NONE_GEN(n)
348#define PROLOG_ADDITION_NONE_GDBELL(n)
349#define PROLOG_ADDITION_NONE_CRIT(n)
350#define PROLOG_ADDITION_NONE_DBG(n)
351#define PROLOG_ADDITION_NONE_MC(n)
352
353#define PROLOG_ADDITION_MASKABLE_GEN(n) \
354 lbz r10,PACASOFTIRQEN(r13); \
355 cmpwi cr0,r10,0; \
356 beq masked_interrupt_book3e_
357
358#define PROLOG_ADDITION_2REGS_GEN(n) \
359 std r14,PACA_EXGEN+EX_R14(r13); \
360 std r15,PACA_EXGEN+EX_R15(r13)
361
362#define PROLOG_ADDITION_1REG_GEN(n) \
363 std r14,PACA_EXGEN+EX_R14(r13);
364
365#define PROLOG_ADDITION_2REGS_CRIT(n) \
366 std r14,PACA_EXCRIT+EX_R14(r13); \
367 std r15,PACA_EXCRIT+EX_R15(r13)
368
369#define PROLOG_ADDITION_2REGS_DBG(n) \
370 std r14,PACA_EXDBG+EX_R14(r13); \
371 std r15,PACA_EXDBG+EX_R15(r13)
372
373#define PROLOG_ADDITION_2REGS_MC(n) \
374 std r14,PACA_EXMC+EX_R14(r13); \
375 std r15,PACA_EXMC+EX_R15(r13)
376
377
378
379#define EXCEPTION_COMMON_LVL(n, scratch, excf) \
380exc_
381 std r0,GPR0(r1); \
382 std r2,GPR2(r1); \
383 SAVE_4GPRS(3, r1); \
384 SAVE_2GPRS(7, r1); \
385 std r9,GPR9(r1); \
386 std r10,_NIP(r1); \
387 std r11,_MSR(r1); \
388 beq 2f; \
389 ACCOUNT_CPU_USER_ENTRY(r13,r10,r11); \
3902: ld r3,excf+EX_R10(r13); \
391 ld r4,excf+EX_R11(r13); \
392 mfspr r5,scratch; \
393 std r12,GPR12(r1); \
394 ld r2,PACATOC(r13); \
395 mflr r6; \
396 mfctr r7; \
397 mfspr r8,SPRN_XER; \
398 ld r9,excf+EX_R1(r13); \
399 lwz r10,excf+EX_CR(r13); \
400 lbz r11,PACASOFTIRQEN(r13); \
401 ld r12,exception_marker@toc(r2); \
402 li r0,0; \
403 std r3,GPR10(r1); \
404 std r4,GPR11(r1); \
405 std r5,GPR13(r1); \
406 std r6,_LINK(r1); \
407 std r7,_CTR(r1); \
408 std r8,_XER(r1); \
409 li r3,(n)+1; \
410 std r9,0(r1); \
411 std r10,_CCR(r1); \
412 std r9,GPR1(r1); \
413 std r11,SOFTE(r1); \
414 std r12,STACK_FRAME_OVERHEAD-16(r1); \
415 std r3,_TRAP(r1); \
416 std r0,RESULT(r1);
417
418#define EXCEPTION_COMMON(n) \
419 EXCEPTION_COMMON_LVL(n, SPRN_SPRG_GEN_SCRATCH, PACA_EXGEN)
420#define EXCEPTION_COMMON_CRIT(n) \
421 EXCEPTION_COMMON_LVL(n, SPRN_SPRG_CRIT_SCRATCH, PACA_EXCRIT)
422#define EXCEPTION_COMMON_MC(n) \
423 EXCEPTION_COMMON_LVL(n, SPRN_SPRG_MC_SCRATCH, PACA_EXMC)
424#define EXCEPTION_COMMON_DBG(n) \
425 EXCEPTION_COMMON_LVL(n, SPRN_SPRG_DBG_SCRATCH, PACA_EXDBG)
426
427
428
429
430
431
432
433#define INTS_DISABLE RECONCILE_IRQ_STATE(r3,r4)
434
435
436
437
438
439
440
441
442
443
444
445#define INTS_RESTORE_HARD \
446 ld r11,_MSR(r1); \
447 wrtee r11;
448
449
450#define BAD_STACK_TRAMPOLINE(n) \
451exc_
452 li r1,(n); \
453 sth r1,PACA_TRAP_SAVE(r13); \
454 b bad_stack_book3e;
455
456
457
458
459
460
461
462#define EXCEPTION_STUB(loc, label) \
463 . = interrupt_base_book3e + loc; \
464 nop; \
465 b exc_
466
467#define ACK_NONE(r)
468#define ACK_DEC(r) \
469 lis r,TSR_DIS@h; \
470 mtspr SPRN_TSR,r
471#define ACK_FIT(r) \
472 lis r,TSR_FIS@h; \
473 mtspr SPRN_TSR,r
474
475
476
477
478
479
480
481#define CHECK_NAPPING() \
482 CURRENT_THREAD_INFO(r11, r1); \
483 ld r10,TI_LOCAL_FLAGS(r11); \
484 andi. r9,r10,_TLF_NAPPING; \
485 beq+ 1f; \
486 ld r8,_LINK(r1); \
487 rlwinm r7,r10,0,~_TLF_NAPPING; \
488 std r8,_NIP(r1); \
489 std r7,TI_LOCAL_FLAGS(r11); \
4901:
491
492
493#define MASKABLE_EXCEPTION(trapnum, intnum, label, hdlr, ack) \
494 START_EXCEPTION(label); \
495 NORMAL_EXCEPTION_PROLOG(trapnum, intnum, PROLOG_ADDITION_MASKABLE)\
496 EXCEPTION_COMMON(trapnum) \
497 INTS_DISABLE; \
498 ack(r8); \
499 CHECK_NAPPING(); \
500 addi r3,r1,STACK_FRAME_OVERHEAD; \
501 bl hdlr; \
502 b ret_from_except_lite;
503
504
505 .section ".toc","aw"
506exception_marker:
507 .tc ID_EXC_MARKER[TC],STACK_FRAME_REGS_MARKER
508
509
510
511
512
513
514 .text
515 .balign 0x1000
516 .globl interrupt_base_book3e
517interrupt_base_book3e:
518 EXCEPTION_STUB(0x000, machine_check)
519 EXCEPTION_STUB(0x020, critical_input)
520 EXCEPTION_STUB(0x040, debug_crit)
521 EXCEPTION_STUB(0x060, data_storage)
522 EXCEPTION_STUB(0x080, instruction_storage)
523 EXCEPTION_STUB(0x0a0, external_input)
524 EXCEPTION_STUB(0x0c0, alignment)
525 EXCEPTION_STUB(0x0e0, program)
526 EXCEPTION_STUB(0x100, fp_unavailable)
527 EXCEPTION_STUB(0x120, system_call)
528 EXCEPTION_STUB(0x140, ap_unavailable)
529 EXCEPTION_STUB(0x160, decrementer)
530 EXCEPTION_STUB(0x180, fixed_interval)
531 EXCEPTION_STUB(0x1a0, watchdog)
532 EXCEPTION_STUB(0x1c0, data_tlb_miss)
533 EXCEPTION_STUB(0x1e0, instruction_tlb_miss)
534 EXCEPTION_STUB(0x200, altivec_unavailable)
535 EXCEPTION_STUB(0x220, altivec_assist)
536 EXCEPTION_STUB(0x260, perfmon)
537 EXCEPTION_STUB(0x280, doorbell)
538 EXCEPTION_STUB(0x2a0, doorbell_crit)
539 EXCEPTION_STUB(0x2c0, guest_doorbell)
540 EXCEPTION_STUB(0x2e0, guest_doorbell_crit)
541 EXCEPTION_STUB(0x300, hypercall)
542 EXCEPTION_STUB(0x320, ehpriv)
543 EXCEPTION_STUB(0x340, lrat_error)
544
545 .globl __end_interrupts
546__end_interrupts:
547
548
549 START_EXCEPTION(critical_input);
550 CRIT_EXCEPTION_PROLOG(0x100, BOOKE_INTERRUPT_CRITICAL,
551 PROLOG_ADDITION_NONE)
552 EXCEPTION_COMMON_CRIT(0x100)
553 bl save_nvgprs
554 bl special_reg_save
555 CHECK_NAPPING();
556 addi r3,r1,STACK_FRAME_OVERHEAD
557 bl unknown_exception
558 b ret_from_crit_except
559
560
561 START_EXCEPTION(machine_check);
562 MC_EXCEPTION_PROLOG(0x000, BOOKE_INTERRUPT_MACHINE_CHECK,
563 PROLOG_ADDITION_NONE)
564 EXCEPTION_COMMON_MC(0x000)
565 bl save_nvgprs
566 bl special_reg_save
567 CHECK_NAPPING();
568 addi r3,r1,STACK_FRAME_OVERHEAD
569 bl machine_check_exception
570 b ret_from_mc_except
571
572
573 START_EXCEPTION(data_storage)
574 NORMAL_EXCEPTION_PROLOG(0x300, BOOKE_INTERRUPT_DATA_STORAGE,
575 PROLOG_ADDITION_2REGS)
576 mfspr r14,SPRN_DEAR
577 mfspr r15,SPRN_ESR
578 EXCEPTION_COMMON(0x300)
579 INTS_DISABLE
580 b storage_fault_common
581
582
583 START_EXCEPTION(instruction_storage);
584 NORMAL_EXCEPTION_PROLOG(0x400, BOOKE_INTERRUPT_INST_STORAGE,
585 PROLOG_ADDITION_2REGS)
586 li r15,0
587 mr r14,r10
588 EXCEPTION_COMMON(0x400)
589 INTS_DISABLE
590 b storage_fault_common
591
592
593 MASKABLE_EXCEPTION(0x500, BOOKE_INTERRUPT_EXTERNAL,
594 external_input, do_IRQ, ACK_NONE)
595
596
597 START_EXCEPTION(alignment);
598 NORMAL_EXCEPTION_PROLOG(0x600, BOOKE_INTERRUPT_ALIGNMENT,
599 PROLOG_ADDITION_2REGS)
600 mfspr r14,SPRN_DEAR
601 mfspr r15,SPRN_ESR
602 EXCEPTION_COMMON(0x600)
603 b alignment_more
604
605
606 START_EXCEPTION(program);
607 NORMAL_EXCEPTION_PROLOG(0x700, BOOKE_INTERRUPT_PROGRAM,
608 PROLOG_ADDITION_1REG)
609 mfspr r14,SPRN_ESR
610 EXCEPTION_COMMON(0x700)
611 INTS_DISABLE
612 std r14,_DSISR(r1)
613 addi r3,r1,STACK_FRAME_OVERHEAD
614 ld r14,PACA_EXGEN+EX_R14(r13)
615 bl save_nvgprs
616 bl program_check_exception
617 b ret_from_except
618
619
620 START_EXCEPTION(fp_unavailable);
621 NORMAL_EXCEPTION_PROLOG(0x800, BOOKE_INTERRUPT_FP_UNAVAIL,
622 PROLOG_ADDITION_NONE)
623
624 EXCEPTION_COMMON(0x800)
625 ld r12,_MSR(r1)
626 andi. r0,r12,MSR_PR;
627 beq- 1f
628 bl load_up_fpu
629 b fast_exception_return
6301: INTS_DISABLE
631 bl save_nvgprs
632 addi r3,r1,STACK_FRAME_OVERHEAD
633 bl kernel_fp_unavailable_exception
634 b ret_from_except
635
636
637 START_EXCEPTION(altivec_unavailable);
638 NORMAL_EXCEPTION_PROLOG(0x200, BOOKE_INTERRUPT_ALTIVEC_UNAVAIL,
639 PROLOG_ADDITION_NONE)
640
641 EXCEPTION_COMMON(0x200)
642#ifdef CONFIG_ALTIVEC
643BEGIN_FTR_SECTION
644 ld r12,_MSR(r1)
645 andi. r0,r12,MSR_PR;
646 beq- 1f
647 bl load_up_altivec
648 b fast_exception_return
6491:
650END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
651#endif
652 INTS_DISABLE
653 bl save_nvgprs
654 addi r3,r1,STACK_FRAME_OVERHEAD
655 bl altivec_unavailable_exception
656 b ret_from_except
657
658
659 START_EXCEPTION(altivec_assist);
660 NORMAL_EXCEPTION_PROLOG(0x220,
661 BOOKE_INTERRUPT_ALTIVEC_ASSIST,
662 PROLOG_ADDITION_NONE)
663 EXCEPTION_COMMON(0x220)
664 INTS_DISABLE
665 bl save_nvgprs
666 addi r3,r1,STACK_FRAME_OVERHEAD
667#ifdef CONFIG_ALTIVEC
668BEGIN_FTR_SECTION
669 bl altivec_assist_exception
670END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
671#else
672 bl unknown_exception
673#endif
674 b ret_from_except
675
676
677
678 MASKABLE_EXCEPTION(0x900, BOOKE_INTERRUPT_DECREMENTER,
679 decrementer, timer_interrupt, ACK_DEC)
680
681
682 MASKABLE_EXCEPTION(0x980, BOOKE_INTERRUPT_FIT,
683 fixed_interval, unknown_exception, ACK_FIT)
684
685
686 START_EXCEPTION(watchdog);
687 CRIT_EXCEPTION_PROLOG(0x9f0, BOOKE_INTERRUPT_WATCHDOG,
688 PROLOG_ADDITION_NONE)
689 EXCEPTION_COMMON_CRIT(0x9f0)
690 bl save_nvgprs
691 bl special_reg_save
692 CHECK_NAPPING();
693 addi r3,r1,STACK_FRAME_OVERHEAD
694#ifdef CONFIG_BOOKE_WDT
695 bl WatchdogException
696#else
697 bl unknown_exception
698#endif
699 b ret_from_crit_except
700
701
702 START_EXCEPTION(system_call)
703 mr r9,r13
704 mfspr r11,SPRN_SRR0
705 mfspr r12,SPRN_SRR1
706 mfspr r13,SPRN_SPRG_PACA
707 b system_call_common
708
709
710 START_EXCEPTION(ap_unavailable);
711 NORMAL_EXCEPTION_PROLOG(0xf20, BOOKE_INTERRUPT_AP_UNAVAIL,
712 PROLOG_ADDITION_NONE)
713 EXCEPTION_COMMON(0xf20)
714 INTS_DISABLE
715 bl save_nvgprs
716 addi r3,r1,STACK_FRAME_OVERHEAD
717 bl unknown_exception
718 b ret_from_except
719
720
721 START_EXCEPTION(debug_crit);
722 CRIT_EXCEPTION_PROLOG(0xd00, BOOKE_INTERRUPT_DEBUG,
723 PROLOG_ADDITION_2REGS)
724
725
726
727
728
729
730
731
732
733
734 mfspr r14,SPRN_DBSR
735 andis. r15,r14,(DBSR_IC|DBSR_BT)@h
736 beq+ 1f
737
738#ifdef CONFIG_RELOCATABLE
739 ld r15,PACATOC(r13)
740 ld r14,interrupt_base_book3e@got(r15)
741 ld r15,__end_interrupts@got(r15)
742#else
743 LOAD_REG_IMMEDIATE(r14,interrupt_base_book3e)
744 LOAD_REG_IMMEDIATE(r15,__end_interrupts)
745#endif
746 cmpld cr0,r10,r14
747 cmpld cr1,r10,r15
748 blt+ cr0,1f
749 bge+ cr1,1f
750
751
752 lis r14,(DBSR_IC|DBSR_BT)@h
753 rlwinm r11,r11,0,~MSR_DE
754 mtspr SPRN_DBSR,r14
755 mtspr SPRN_CSRR1,r11
756 lwz r10,PACA_EXCRIT+EX_CR(r13)
757 ld r1,PACA_EXCRIT+EX_R1(r13)
758 ld r14,PACA_EXCRIT+EX_R14(r13)
759 ld r15,PACA_EXCRIT+EX_R15(r13)
760 mtcr r10
761 ld r10,PACA_EXCRIT+EX_R10(r13)
762 ld r11,PACA_EXCRIT+EX_R11(r13)
763 mfspr r13,SPRN_SPRG_CRIT_SCRATCH
764 rfci
765
766
767
768
769
7701: andi. r14,r11,MSR_PR;
771 beq kernel_dbg_exc;
772
773
774
775
776 mfspr r14,SPRN_DBSR
777 EXCEPTION_COMMON_CRIT(0xd00)
778 std r14,_DSISR(r1)
779 addi r3,r1,STACK_FRAME_OVERHEAD
780 mr r4,r14
781 ld r14,PACA_EXCRIT+EX_R14(r13)
782 ld r15,PACA_EXCRIT+EX_R15(r13)
783 bl save_nvgprs
784 bl DebugException
785 b ret_from_except
786
787kernel_dbg_exc:
788 b .
789
790
791 START_EXCEPTION(debug_debug);
792 DBG_EXCEPTION_PROLOG(0xd00, BOOKE_INTERRUPT_DEBUG,
793 PROLOG_ADDITION_2REGS)
794
795
796
797
798
799
800
801
802
803
804 mfspr r14,SPRN_DBSR
805 andis. r15,r14,(DBSR_IC|DBSR_BT)@h
806 beq+ 1f
807
808#ifdef CONFIG_RELOCATABLE
809 ld r15,PACATOC(r13)
810 ld r14,interrupt_base_book3e@got(r15)
811 ld r15,__end_interrupts@got(r15)
812#else
813 LOAD_REG_IMMEDIATE(r14,interrupt_base_book3e)
814 LOAD_REG_IMMEDIATE(r15,__end_interrupts)
815#endif
816 cmpld cr0,r10,r14
817 cmpld cr1,r10,r15
818 blt+ cr0,1f
819 bge+ cr1,1f
820
821
822 lis r14,(DBSR_IC|DBSR_BT)@h
823 rlwinm r11,r11,0,~MSR_DE
824 mtspr SPRN_DBSR,r14
825 mtspr SPRN_DSRR1,r11
826 lwz r10,PACA_EXDBG+EX_CR(r13)
827 ld r1,PACA_EXDBG+EX_R1(r13)
828 ld r14,PACA_EXDBG+EX_R14(r13)
829 ld r15,PACA_EXDBG+EX_R15(r13)
830 mtcr r10
831 ld r10,PACA_EXDBG+EX_R10(r13)
832 ld r11,PACA_EXDBG+EX_R11(r13)
833 mfspr r13,SPRN_SPRG_DBG_SCRATCH
834 rfdi
835
836
837
838
839
8401: andi. r14,r11,MSR_PR;
841 beq kernel_dbg_exc;
842
843
844
845
846 mfspr r14,SPRN_DBSR
847 EXCEPTION_COMMON_DBG(0xd08)
848 INTS_DISABLE
849 std r14,_DSISR(r1)
850 addi r3,r1,STACK_FRAME_OVERHEAD
851 mr r4,r14
852 ld r14,PACA_EXDBG+EX_R14(r13)
853 ld r15,PACA_EXDBG+EX_R15(r13)
854 bl save_nvgprs
855 bl DebugException
856 b ret_from_except
857
858 START_EXCEPTION(perfmon);
859 NORMAL_EXCEPTION_PROLOG(0x260, BOOKE_INTERRUPT_PERFORMANCE_MONITOR,
860 PROLOG_ADDITION_NONE)
861 EXCEPTION_COMMON(0x260)
862 INTS_DISABLE
863 CHECK_NAPPING()
864 addi r3,r1,STACK_FRAME_OVERHEAD
865 bl performance_monitor_exception
866 b ret_from_except_lite
867
868
869 MASKABLE_EXCEPTION(0x280, BOOKE_INTERRUPT_DOORBELL,
870 doorbell, doorbell_exception, ACK_NONE)
871
872
873 START_EXCEPTION(doorbell_crit);
874 CRIT_EXCEPTION_PROLOG(0x2a0, BOOKE_INTERRUPT_DOORBELL_CRITICAL,
875 PROLOG_ADDITION_NONE)
876 EXCEPTION_COMMON_CRIT(0x2a0)
877 bl save_nvgprs
878 bl special_reg_save
879 CHECK_NAPPING();
880 addi r3,r1,STACK_FRAME_OVERHEAD
881 bl unknown_exception
882 b ret_from_crit_except
883
884
885
886
887
888 START_EXCEPTION(guest_doorbell);
889 GDBELL_EXCEPTION_PROLOG(0x2c0, BOOKE_INTERRUPT_GUEST_DBELL,
890 PROLOG_ADDITION_NONE)
891 EXCEPTION_COMMON(0x2c0)
892 addi r3,r1,STACK_FRAME_OVERHEAD
893 bl save_nvgprs
894 INTS_RESTORE_HARD
895 bl unknown_exception
896 b ret_from_except
897
898
899 START_EXCEPTION(guest_doorbell_crit);
900 CRIT_EXCEPTION_PROLOG(0x2e0, BOOKE_INTERRUPT_GUEST_DBELL_CRIT,
901 PROLOG_ADDITION_NONE)
902 EXCEPTION_COMMON_CRIT(0x2e0)
903 bl save_nvgprs
904 bl special_reg_save
905 CHECK_NAPPING();
906 addi r3,r1,STACK_FRAME_OVERHEAD
907 bl unknown_exception
908 b ret_from_crit_except
909
910
911 START_EXCEPTION(hypercall);
912 NORMAL_EXCEPTION_PROLOG(0x310, BOOKE_INTERRUPT_HV_SYSCALL,
913 PROLOG_ADDITION_NONE)
914 EXCEPTION_COMMON(0x310)
915 addi r3,r1,STACK_FRAME_OVERHEAD
916 bl save_nvgprs
917 INTS_RESTORE_HARD
918 bl unknown_exception
919 b ret_from_except
920
921
922 START_EXCEPTION(ehpriv);
923 NORMAL_EXCEPTION_PROLOG(0x320, BOOKE_INTERRUPT_HV_PRIV,
924 PROLOG_ADDITION_NONE)
925 EXCEPTION_COMMON(0x320)
926 addi r3,r1,STACK_FRAME_OVERHEAD
927 bl save_nvgprs
928 INTS_RESTORE_HARD
929 bl unknown_exception
930 b ret_from_except
931
932
933 START_EXCEPTION(lrat_error);
934 NORMAL_EXCEPTION_PROLOG(0x340, BOOKE_INTERRUPT_LRAT_ERROR,
935 PROLOG_ADDITION_NONE)
936 EXCEPTION_COMMON(0x340)
937 addi r3,r1,STACK_FRAME_OVERHEAD
938 bl save_nvgprs
939 INTS_RESTORE_HARD
940 bl unknown_exception
941 b ret_from_except
942
943
944
945
946
947
948.macro masked_interrupt_book3e paca_irq full_mask
949 lbz r10,PACAIRQHAPPENED(r13)
950 ori r10,r10,\paca_irq
951 stb r10,PACAIRQHAPPENED(r13)
952
953 .if \full_mask == 1
954 rldicl r10,r11,48,1
955 rotldi r11,r10,16
956 mtspr SPRN_SRR1,r11
957 .endif
958
959 lwz r11,PACA_EXGEN+EX_CR(r13)
960 mtcr r11
961 ld r10,PACA_EXGEN+EX_R10(r13)
962 ld r11,PACA_EXGEN+EX_R11(r13)
963 mfspr r13,SPRN_SPRG_GEN_SCRATCH
964 rfi
965 b .
966.endm
967
968masked_interrupt_book3e_0x500:
969
970 masked_interrupt_book3e PACA_IRQ_EE 1
971
972masked_interrupt_book3e_0x900:
973 ACK_DEC(r10);
974 masked_interrupt_book3e PACA_IRQ_DEC 0
975
976masked_interrupt_book3e_0x980:
977 ACK_FIT(r10);
978 masked_interrupt_book3e PACA_IRQ_DEC 0
979
980masked_interrupt_book3e_0x280:
981masked_interrupt_book3e_0x2c0:
982 masked_interrupt_book3e PACA_IRQ_DBELL 0
983
984
985
986
987
988
989
990
991
992
993
994_GLOBAL(__replay_interrupt)
995
996
997
998
999 mflr r10
1000 mfmsr r11
1001 mfcr r4
1002 mtspr SPRN_SPRG_GEN_SCRATCH,r13;
1003 std r1,PACA_EXGEN+EX_R1(r13);
1004 stw r4,PACA_EXGEN+EX_CR(r13);
1005 ori r11,r11,MSR_EE
1006 subi r1,r1,INT_FRAME_SIZE;
1007 cmpwi cr0,r3,0x500
1008 beq exc_0x500_common
1009 cmpwi cr0,r3,0x900
1010 beq exc_0x900_common
1011 cmpwi cr0,r3,0x280
1012 beq exc_0x280_common
1013 blr
1014
1015
1016
1017
1018
1019
1020
1021storage_fault_common:
1022 std r14,_DAR(r1)
1023 std r15,_DSISR(r1)
1024 addi r3,r1,STACK_FRAME_OVERHEAD
1025 mr r4,r14
1026 mr r5,r15
1027 ld r14,PACA_EXGEN+EX_R14(r13)
1028 ld r15,PACA_EXGEN+EX_R15(r13)
1029 bl do_page_fault
1030 cmpdi r3,0
1031 bne- 1f
1032 b ret_from_except_lite
10331: bl save_nvgprs
1034 mr r5,r3
1035 addi r3,r1,STACK_FRAME_OVERHEAD
1036 ld r4,_DAR(r1)
1037 bl bad_page_fault
1038 b ret_from_except
1039
1040
1041
1042
1043
1044alignment_more:
1045 std r14,_DAR(r1)
1046 std r15,_DSISR(r1)
1047 addi r3,r1,STACK_FRAME_OVERHEAD
1048 ld r14,PACA_EXGEN+EX_R14(r13)
1049 ld r15,PACA_EXGEN+EX_R15(r13)
1050 bl save_nvgprs
1051 INTS_RESTORE_HARD
1052 bl alignment_exception
1053 b ret_from_except
1054
1055
1056
1057
1058
1059_GLOBAL(exception_return_book3e)
1060 b 1f
1061
1062
1063
1064
1065 .globl fast_exception_return
1066fast_exception_return:
1067 wrteei 0
10681: mr r0,r13
1069 ld r10,_MSR(r1)
1070 REST_4GPRS(2, r1)
1071 andi. r6,r10,MSR_PR
1072 REST_2GPRS(6, r1)
1073 beq 1f
1074 ACCOUNT_CPU_USER_EXIT(r13, r10, r11)
1075 ld r0,GPR13(r1)
1076
10771: stdcx. r0,0,r1
1078
1079 ld r8,_CCR(r1)
1080 ld r9,_LINK(r1)
1081 ld r10,_CTR(r1)
1082 ld r11,_XER(r1)
1083 mtcr r8
1084 mtlr r9
1085 mtctr r10
1086 mtxer r11
1087 REST_2GPRS(8, r1)
1088 ld r10,GPR10(r1)
1089 ld r11,GPR11(r1)
1090 ld r12,GPR12(r1)
1091 mtspr SPRN_SPRG_GEN_SCRATCH,r0
1092
1093 std r10,PACA_EXGEN+EX_R10(r13);
1094 std r11,PACA_EXGEN+EX_R11(r13);
1095 ld r10,_NIP(r1)
1096 ld r11,_MSR(r1)
1097 ld r0,GPR0(r1)
1098 ld r1,GPR1(r1)
1099 mtspr SPRN_SRR0,r10
1100 mtspr SPRN_SRR1,r11
1101 ld r10,PACA_EXGEN+EX_R10(r13)
1102 ld r11,PACA_EXGEN+EX_R11(r13)
1103 mfspr r13,SPRN_SPRG_GEN_SCRATCH
1104 rfi
1105
1106
1107
1108
1109
1110
1111
1112
1113BAD_STACK_TRAMPOLINE(0x000)
1114BAD_STACK_TRAMPOLINE(0x100)
1115BAD_STACK_TRAMPOLINE(0x200)
1116BAD_STACK_TRAMPOLINE(0x220)
1117BAD_STACK_TRAMPOLINE(0x260)
1118BAD_STACK_TRAMPOLINE(0x280)
1119BAD_STACK_TRAMPOLINE(0x2a0)
1120BAD_STACK_TRAMPOLINE(0x2c0)
1121BAD_STACK_TRAMPOLINE(0x2e0)
1122BAD_STACK_TRAMPOLINE(0x300)
1123BAD_STACK_TRAMPOLINE(0x310)
1124BAD_STACK_TRAMPOLINE(0x320)
1125BAD_STACK_TRAMPOLINE(0x340)
1126BAD_STACK_TRAMPOLINE(0x400)
1127BAD_STACK_TRAMPOLINE(0x500)
1128BAD_STACK_TRAMPOLINE(0x600)
1129BAD_STACK_TRAMPOLINE(0x700)
1130BAD_STACK_TRAMPOLINE(0x800)
1131BAD_STACK_TRAMPOLINE(0x900)
1132BAD_STACK_TRAMPOLINE(0x980)
1133BAD_STACK_TRAMPOLINE(0x9f0)
1134BAD_STACK_TRAMPOLINE(0xa00)
1135BAD_STACK_TRAMPOLINE(0xb00)
1136BAD_STACK_TRAMPOLINE(0xc00)
1137BAD_STACK_TRAMPOLINE(0xd00)
1138BAD_STACK_TRAMPOLINE(0xd08)
1139BAD_STACK_TRAMPOLINE(0xe00)
1140BAD_STACK_TRAMPOLINE(0xf00)
1141BAD_STACK_TRAMPOLINE(0xf20)
1142
1143 .globl bad_stack_book3e
1144bad_stack_book3e:
1145
1146 mfspr r10,SPRN_SRR0;
1147 ld r1,PACAEMERGSP(r13)
1148 subi r1,r1,64+INT_FRAME_SIZE
1149 std r10,_NIP(r1)
1150 std r11,_MSR(r1)
1151 ld r10,PACA_EXGEN+EX_R1(r13)
1152 lwz r11,PACA_EXGEN+EX_CR(r13)
1153 std r10,GPR1(r1)
1154 std r11,_CCR(r1)
1155 mfspr r10,SPRN_DEAR
1156 mfspr r11,SPRN_ESR
1157 std r10,_DAR(r1)
1158 std r11,_DSISR(r1)
1159 std r0,GPR0(r1); \
1160 std r2,GPR2(r1); \
1161 SAVE_4GPRS(3, r1); \
1162 SAVE_2GPRS(7, r1); \
1163 std r9,GPR9(r1); \
1164 ld r3,PACA_EXGEN+EX_R10(r13); \
1165 ld r4,PACA_EXGEN+EX_R11(r13); \
1166 mfspr r5,SPRN_SPRG_GEN_SCRATCH; \
1167 std r3,GPR10(r1); \
1168 std r4,GPR11(r1); \
1169 std r12,GPR12(r1); \
1170 std r5,GPR13(r1); \
1171 mflr r10
1172 mfctr r11
1173 mfxer r12
1174 std r10,_LINK(r1)
1175 std r11,_CTR(r1)
1176 std r12,_XER(r1)
1177 SAVE_10GPRS(14,r1)
1178 SAVE_8GPRS(24,r1)
1179 lhz r12,PACA_TRAP_SAVE(r13)
1180 std r12,_TRAP(r1)
1181 addi r11,r1,INT_FRAME_SIZE
1182 std r11,0(r1)
1183 li r12,0
1184 std r12,0(r11)
1185 ld r2,PACATOC(r13)
11861: addi r3,r1,STACK_FRAME_OVERHEAD
1187 bl kernel_bad_stack
1188 b 1b
1189
1190
1191
1192
1193
1194
1195_GLOBAL(initial_tlb_book3e)
1196
1197
1198 mfspr r4,SPRN_TLB0CFG
1199 andi. r3,r4,TLBnCFG_IPROT
1200 lis r3,MAS0_TLBSEL(0)@h
1201 bne found_iprot
1202
1203 mfspr r4,SPRN_TLB1CFG
1204 andi. r3,r4,TLBnCFG_IPROT
1205 lis r3,MAS0_TLBSEL(1)@h
1206 bne found_iprot
1207
1208 mfspr r4,SPRN_TLB2CFG
1209 andi. r3,r4,TLBnCFG_IPROT
1210 lis r3,MAS0_TLBSEL(2)@h
1211 bne found_iprot
1212
1213 lis r3,MAS0_TLBSEL(3)@h
1214 mfspr r4,SPRN_TLB3CFG
1215
1216
1217found_iprot:
1218 andi. r5,r4,TLBnCFG_HES
1219 bne have_hes
1220
1221 mflr r8
1222
1223
1224
1225
1226
1227 bl invstr
1228invstr: mflr r6
1229 mfmsr r7
1230 rlwinm r5,r7,27,31,31
1231 mfspr r7,SPRN_PID
1232 slwi r7,r7,16
1233 or r7,r7,r5
1234 mtspr SPRN_MAS6,r7
1235 tlbsx 0,r6
1236
1237 mfspr r3,SPRN_MAS0
1238 rlwinm r5,r3,16,20,31
1239
1240 mfspr r7,SPRN_MAS1
1241 oris r7,r7,MAS1_IPROT@h
1242 mtspr SPRN_MAS1,r7
1243 tlbwe
1244
1245
1246
1247
1248
1249
1250
1251 andi. r4,r4,TLBnCFG_N_ENTRY
1252 li r6,0
12531: mr r7,r3
1254 rlwimi r7,r6,16,4,15
1255 mtspr SPRN_MAS0,r7
1256 tlbre
1257 mfspr r7,SPRN_MAS1
1258 rlwinm r7,r7,0,2,31
1259 cmpw r5,r6
1260 beq skpinv
1261 mtspr SPRN_MAS1,r7
1262 tlbwe
1263 isync
1264skpinv: addi r6,r6,1
1265 cmpw r6,r4
1266 bne 1b
1267
1268
1269 PPC_TLBILX_ALL(0,R0)
1270 sync
1271 isync
1272
1273
1274
1275
1276
1277
1278 andi. r7,r5,0x1
1279 addi r7,r7,0x1
1280 mr r4,r3
1281 mtspr SPRN_MAS0,r4
1282 tlbre
1283
1284 rlwimi r4,r7,16,4,15
1285 mtspr SPRN_MAS0,r4
1286
1287 mfspr r7,SPRN_MAS1
1288 xori r6,r7,MAS1_TS
1289 mtspr SPRN_MAS1,r6
1290
1291 tlbwe
1292
1293 mfmsr r6
1294 xori r6,r6,MSR_IS
1295 mtspr SPRN_SRR1,r6
1296 bl 1f
12971: mflr r6
1298 addi r6,r6,(2f - 1b)
1299 mtspr SPRN_SRR0,r6
1300 rfi
13012:
1302
1303
1304
1305
1306
1307
1308
1309 li r6,0
1310 mtspr SPRN_MAS6,r6
1311 mtspr SPRN_PID,r6
1312
1313
1314
1315
1316
1317
1318
1319 mtspr SPRN_MAS0,r3
1320 tlbre
1321 mfspr r6,SPRN_MAS1
1322 rlwinm r6,r6,0,2,31
1323 mtspr SPRN_MAS1,r6
1324 tlbwe
1325 sync
1326 isync
1327
1328
1329
1330
1331
1332
1333#define M_IF_NEEDED MAS2_M
1334#else
1335#define M_IF_NEEDED 0
1336#endif
1337
1338
1339
1340
1341
1342
1343
1344 rlwinm r3,r3,0,16,3
1345 mtspr SPRN_MAS0,r3
1346 lis r6,(MAS1_VALID|MAS1_IPROT)@h
1347 ori r6,r6,(MAS1_TSIZE(BOOK3E_PAGESZ_1GB))@l
1348 mtspr SPRN_MAS1,r6
1349
1350 LOAD_REG_IMMEDIATE(r6, PAGE_OFFSET | M_IF_NEEDED)
1351 mtspr SPRN_MAS2,r6
1352
1353 rlwinm r5,r5,0,0,25
1354 ori r5,r5,MAS3_SR | MAS3_SW | MAS3_SX
1355 mtspr SPRN_MAS3,r5
1356 li r5,-1
1357 rlwinm r5,r5,0,0,25
1358
1359 tlbwe
1360
1361
1362
1363
1364
1365
1366 bl 1f
13671: mflr r6
1368 addi r6,r6,(2f - 1b)
1369 tovirt(r6,r6)
1370 lis r7,MSR_KERNEL@h
1371 ori r7,r7,MSR_KERNEL@l
1372 mtspr SPRN_SRR0,r6
1373 mtspr SPRN_SRR1,r7
1374 rfi
13752:
1376
1377
1378
1379
1380
1381 mtspr SPRN_MAS0,r4
1382 tlbre
1383 mfspr r5,SPRN_MAS1
1384 rlwinm r5,r5,0,2,31
1385 mtspr SPRN_MAS1,r5
1386 tlbwe
1387 sync
1388 isync
1389
1390
1391 tovirt(r8,r8)
1392 mtlr r8
1393 blr
1394
1395have_hes:
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412 .globl a2_tlbinit_code_start
1413a2_tlbinit_code_start:
1414
1415 ori r11,r3,MAS0_WQ_ALLWAYS
1416 oris r11,r11,MAS0_ESEL(3)@h
1417 mtspr SPRN_MAS0,r11
1418 lis r3,(MAS1_VALID | MAS1_IPROT)@h
1419 ori r3,r3,BOOK3E_PAGESZ_1GB << MAS1_TSIZE_SHIFT
1420 mtspr SPRN_MAS1,r3
1421 LOAD_REG_IMMEDIATE(r3, PAGE_OFFSET | MAS2_M)
1422 mtspr SPRN_MAS2,r3
1423 li r3,MAS3_SR | MAS3_SW | MAS3_SX
1424 mtspr SPRN_MAS7_MAS3,r3
1425 li r3,0
1426 mtspr SPRN_MAS8,r3
1427
1428
1429 tlbwe
1430
1431 .globl a2_tlbinit_after_linear_map
1432a2_tlbinit_after_linear_map:
1433
1434
1435 LOAD_REG_IMMEDIATE(r3,1f)
1436 mtctr r3
1437 bctr
1438
14391:
1440
1441
1442
1443
1444
1445 li r5,0
1446 mtspr SPRN_MAS6,r5
1447 tlbsx 0,r3
1448
1449 rlwinm r9,r4,0,TLBnCFG_N_ENTRY
1450 rlwinm r10,r4,8,0xff
1451 addi r10,r10,-1
1452
1453 li r3,1
1454
1455 mfspr r5,SPRN_MAS1
1456 rlwinm r5,r5,0,(~(MAS1_VALID|MAS1_IPROT))
1457
1458 mfspr r6,SPRN_MAS2
1459 rldicr r6,r6,0,51
1460
1461 mfspr r7,SPRN_MAS0
1462 rlwinm r7,r7,0,0xffff0fff
1463
1464 rlwinm r8,r7,16,0xfff
1465
14662: add r4,r3,r8
1467 and r4,r4,r10
1468
1469 rlwimi r7,r4,16,MAS0_ESEL_MASK
1470
1471 mtspr SPRN_MAS0,r7
1472 mtspr SPRN_MAS1,r5
1473 mtspr SPRN_MAS2,r6
1474 tlbwe
1475
1476 addi r3,r3,1
1477 and. r4,r3,r10
1478
1479 bne 3f
1480 addis r6,r6,(1<<30)@h
14813:
1482 cmpw r3,r9
1483 blt 2b
1484
1485 .globl a2_tlbinit_after_iprot_flush
1486a2_tlbinit_after_iprot_flush:
1487
1488 PPC_TLBILX(0,0,R0)
1489 sync
1490 isync
1491
1492 .globl a2_tlbinit_code_end
1493a2_tlbinit_code_end:
1494
1495
1496 mflr r3
1497 tovirt(r3,r3)
1498 mtlr r3
1499 blr
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518_GLOBAL(start_initialization_book3e)
1519 mflr r28
1520
1521
1522
1523
1524
1525
1526 bl initial_tlb_book3e
1527
1528
1529 bl init_core_book3e
1530
1531
1532 bl init_thread_book3e
1533
1534
1535 tovirt(r28,r28)
1536 mtlr r28
1537 blr
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551_GLOBAL(book3e_secondary_core_init_tlb_set)
1552 li r4,1
1553 b generic_secondary_smp_init
1554
1555_GLOBAL(book3e_secondary_core_init)
1556 mflr r28
1557
1558
1559 cmplwi r4,0
1560 bne 2f
1561
1562
1563 bl initial_tlb_book3e
1564
1565
1566
1567
1568 bl relative_toc
1569
1570
15712: bl init_core_book3e
1572
1573
15743: bl init_thread_book3e
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589 cmpdi cr0,r28,0
1590 blt 1f
1591 lis r3,PAGE_OFFSET@highest
1592 sldi r3,r3,32
1593 or r28,r28,r3
15941: mtlr r28
1595 blr
1596
1597_GLOBAL(book3e_secondary_thread_init)
1598 mflr r28
1599 b 3b
1600
1601 .globl init_core_book3e
1602init_core_book3e:
1603
1604 tovirt(r2,r2)
1605 LOAD_REG_ADDR(r3, interrupt_base_book3e)
1606 mtspr SPRN_IVPR,r3
1607 sync
1608 blr
1609
1610init_thread_book3e:
1611 lis r3,(SPRN_EPCR_ICM | SPRN_EPCR_GICM)@h
1612 mtspr SPRN_EPCR,r3
1613
1614
1615 wrteei 0
1616
1617
1618 li r3,0
1619 mtspr SPRN_TCR,r3
1620 mfspr r3,SPRN_TSR
1621 mtspr SPRN_TSR,r3
1622
1623 blr
1624
1625_GLOBAL(__setup_base_ivors)
1626 SET_IVOR(0, 0x020)
1627 SET_IVOR(1, 0x000)
1628 SET_IVOR(2, 0x060)
1629 SET_IVOR(3, 0x080)
1630 SET_IVOR(4, 0x0a0)
1631 SET_IVOR(5, 0x0c0)
1632 SET_IVOR(6, 0x0e0)
1633 SET_IVOR(7, 0x100)
1634 SET_IVOR(8, 0x120)
1635 SET_IVOR(9, 0x140)
1636 SET_IVOR(10, 0x160)
1637 SET_IVOR(11, 0x180)
1638 SET_IVOR(12, 0x1a0)
1639 SET_IVOR(13, 0x1c0)
1640 SET_IVOR(14, 0x1e0)
1641 SET_IVOR(15, 0x040)
1642
1643 sync
1644
1645 blr
1646
1647_GLOBAL(setup_altivec_ivors)
1648 SET_IVOR(32, 0x200)
1649 SET_IVOR(33, 0x220)
1650 blr
1651
1652_GLOBAL(setup_perfmon_ivor)
1653 SET_IVOR(35, 0x260)
1654 blr
1655
1656_GLOBAL(setup_doorbell_ivors)
1657 SET_IVOR(36, 0x280)
1658 SET_IVOR(37, 0x2a0)
1659 blr
1660
1661_GLOBAL(setup_ehv_ivors)
1662 SET_IVOR(40, 0x300)
1663 SET_IVOR(41, 0x320)
1664 SET_IVOR(38, 0x2c0)
1665 SET_IVOR(39, 0x2e0)
1666 blr
1667
1668_GLOBAL(setup_lrat_ivor)
1669 SET_IVOR(42, 0x340)
1670 blr
1671