1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19#include <linux/init.h>
20#include <asm/reg.h>
21#include <asm/page.h>
22#include <asm/mmu.h>
23#include <asm/pgtable.h>
24#include <asm/cputable.h>
25#include <asm/cache.h>
26#include <asm/thread_info.h>
27#include <asm/ppc_asm.h>
28#include <asm/asm-offsets.h>
29#include <asm/ptrace.h>
30#include <asm/bug.h>
31#include <asm/kvm_book3s_asm.h>
32#include <asm/export.h>
33#include <asm/feature-fixups.h>
34
35#include "head_32.h"
36
37
38#define LOAD_BAT(n, reg, RA, RB) \
39 \
40 li RA,0; \
41 mtspr SPRN_IBAT
42 mtspr SPRN_DBAT
43 lwz RA,(n*16)+0(reg); \
44 lwz RB,(n*16)+4(reg); \
45 mtspr SPRN_IBAT
46 mtspr SPRN_IBAT
47 beq 1f; \
48 lwz RA,(n*16)+8(reg); \
49 lwz RB,(n*16)+12(reg); \
50 mtspr SPRN_DBAT
51 mtspr SPRN_DBAT
521:
53
54 __HEAD
55 .stabs "arch/powerpc/kernel/",N_SO,0,0,0f
56 .stabs "head_32.S",N_SO,0,0,0f
570:
58_ENTRY(_stext);
59
60
61
62
63
64_ENTRY(_start);
65
66
67
68
69
70
71 nop
72 nop
73 nop
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105 .globl __start
106__start:
107
108
109
110
111
112 cmpwi 0,r5,0
113 beq 1f
114
115#ifdef CONFIG_PPC_OF_BOOT_TRAMPOLINE
116
117 bcl 20,31,$+4
1180: mflr r8
119 addis r8,r8,(_stext - 0b)@ha
120 addi r8,r8,(_stext - 0b)@l
121 bl prom_init
122#endif
123
124
125
126 trap
127
128
129
130
131
132#ifdef CONFIG_PPC_PMAC
1331: lis r31,0x426f
134 ori r31,r31,0x6f58
135 cmpw 0,r3,r31
136 bne 1f
137 bl bootx_init
138 trap
139#endif
140
1411: mr r31,r3
142 li r24,0
143
144
145
146
147
148
149 bl early_init
150
151
152
153
154 bl mmu_off
155__after_mmu_off:
156 bl clear_bats
157 bl flush_tlbs
158
159 bl initial_bats
160 bl load_segment_registers
161#ifdef CONFIG_KASAN
162 bl early_hash_table
163#endif
164
165 bl setup_disp_bat
166#endif
167#ifdef CONFIG_PPC_EARLY_DEBUG_CPM
168 bl setup_cpm_bat
169#endif
170#ifdef CONFIG_PPC_EARLY_DEBUG_USBGECKO
171 bl setup_usbgecko_bat
172#endif
173
174
175
176
177 bl reloc_offset
178 li r24,0
179 bl call_setup_cpu
180#ifdef CONFIG_PPC_BOOK3S_32
181 bl reloc_offset
182 bl init_idle_6xx
183#endif
184
185
186
187
188
189
190
191
192
193 bl reloc_offset
194 mr r26,r3
195 addis r4,r3,KERNELBASE@h
196 lis r5,PHYSICAL_START@h
197 cmplw 0,r4,r5
198 bne relocate_kernel
199
200
201
202
203
204
205
206
207turn_on_mmu:
208 mfmsr r0
209 ori r0,r0,MSR_DR|MSR_IR|MSR_RI
210 mtspr SPRN_SRR1,r0
211 lis r0,start_here@h
212 ori r0,r0,start_here@l
213 mtspr SPRN_SRR0,r0
214 SYNC
215 RFI
216
217
218
219
220
221 . = 0xc0
222 li r3,1
223 .globl __secondary_hold
224__secondary_hold:
225
226 stw r3,__secondary_hold_acknowledge@l(0)
227#ifdef CONFIG_SMP
228100: lwz r4,0(0)
229
230 cmpw 0,r4,r3
231 bne 100b
232
233 mr r24,r3
234 b __secondary_start
235#else
236 b .
237#endif
238
239 .globl __secondary_hold_spinloop
240__secondary_hold_spinloop:
241 .long 0
242 .globl __secondary_hold_acknowledge
243__secondary_hold_acknowledge:
244 .long -1
245
246
247
248
249 EXCEPTION(0x100, Reset, unknown_exception, EXC_XFER_STD)
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265 . = 0x200
266 DO_KVM 0x200
267 mtspr SPRN_SPRG_SCRATCH0,r10
268 mtspr SPRN_SPRG_SCRATCH1,r11
269 mfcr r10
270#ifdef CONFIG_PPC_CHRP
271 mfspr r11, SPRN_SPRG_THREAD
272 lwz r11, RTAS_SP(r11)
273 cmpwi cr1, r11, 0
274 bne cr1, 7f
275#endif
276 EXCEPTION_PROLOG_1
2777: EXCEPTION_PROLOG_2
278 addi r3,r1,STACK_FRAME_OVERHEAD
279#ifdef CONFIG_PPC_CHRP
280 bne cr1,1f
281#endif
282 EXC_XFER_STD(0x200, machine_check_exception)
283#ifdef CONFIG_PPC_CHRP
2841: b machine_check_in_rtas
285#endif
286
287
288 . = 0x300
289 DO_KVM 0x300
290DataAccess:
291 EXCEPTION_PROLOG
292 mfspr r10,SPRN_DSISR
293 stw r10,_DSISR(r11)
294#ifdef CONFIG_PPC_KUAP
295 andis. r0,r10,(DSISR_BAD_FAULT_32S | DSISR_DABRMATCH | DSISR_PROTFAULT)@h
296#else
297 andis. r0,r10,(DSISR_BAD_FAULT_32S|DSISR_DABRMATCH)@h
298#endif
299 bne 1f
300 mfspr r4,SPRN_DAR
301 rlwinm r3,r10,32-15,21,21
302BEGIN_MMU_FTR_SECTION
303 bl hash_page
304END_MMU_FTR_SECTION_IFSET(MMU_FTR_HPTE_TABLE)
3051: lwz r5,_DSISR(r11)
306 mfspr r4,SPRN_DAR
307 EXC_XFER_LITE(0x300, handle_page_fault)
308
309
310
311 . = 0x400
312 DO_KVM 0x400
313InstructionAccess:
314 EXCEPTION_PROLOG
315 andis. r0,r9,SRR1_ISI_NOPT@h
316 beq 1f
317 li r3,0
318 mr r4,r12
319BEGIN_MMU_FTR_SECTION
320 bl hash_page
321END_MMU_FTR_SECTION_IFSET(MMU_FTR_HPTE_TABLE)
3221: mr r4,r12
323 andis. r5,r9,DSISR_SRR1_MATCH_32S@h
324 EXC_XFER_LITE(0x400, handle_page_fault)
325
326
327 EXCEPTION(0x500, HardwareInterrupt, do_IRQ, EXC_XFER_LITE)
328
329
330 . = 0x600
331 DO_KVM 0x600
332Alignment:
333 EXCEPTION_PROLOG
334 mfspr r4,SPRN_DAR
335 stw r4,_DAR(r11)
336 mfspr r5,SPRN_DSISR
337 stw r5,_DSISR(r11)
338 addi r3,r1,STACK_FRAME_OVERHEAD
339 EXC_XFER_STD(0x600, alignment_exception)
340
341
342 EXCEPTION(0x700, ProgramCheck, program_check_exception, EXC_XFER_STD)
343
344
345 . = 0x800
346 DO_KVM 0x800
347FPUnavailable:
348BEGIN_FTR_SECTION
349
350
351
352
353 b ProgramCheck
354END_FTR_SECTION_IFSET(CPU_FTR_FPU_UNAVAILABLE)
355 EXCEPTION_PROLOG
356 beq 1f
357 bl load_up_fpu
358 b fast_exception_return
3591: addi r3,r1,STACK_FRAME_OVERHEAD
360 EXC_XFER_LITE(0x800, kernel_fp_unavailable_exception)
361
362
363 EXCEPTION(0x900, Decrementer, timer_interrupt, EXC_XFER_LITE)
364
365 EXCEPTION(0xa00, Trap_0a, unknown_exception, EXC_XFER_STD)
366 EXCEPTION(0xb00, Trap_0b, unknown_exception, EXC_XFER_STD)
367
368
369 . = 0xc00
370 DO_KVM 0xc00
371SystemCall:
372 SYSCALL_ENTRY 0xc00
373
374
375 EXCEPTION(0xd00, SingleStep, single_step_exception, EXC_XFER_STD)
376 EXCEPTION(0xe00, Trap_0e, unknown_exception, EXC_XFER_STD)
377
378
379
380
381
382
383
384
385
386 . = 0xf00
387 DO_KVM 0xf00
388 b PerformanceMonitor
389
390 . = 0xf20
391 DO_KVM 0xf20
392 b AltiVecUnavailable
393
394
395
396
397
398 . = 0x1000
399InstructionTLBMiss:
400
401
402
403
404
405
406
407 mfspr r3,SPRN_IMISS
408
409 lis r1,PAGE_OFFSET@h
410 cmplw 0,r1,r3
411#endif
412 mfspr r2, SPRN_SPRG_PGDIR
413#ifdef CONFIG_SWAP
414 li r1,_PAGE_PRESENT | _PAGE_ACCESSED | _PAGE_EXEC
415#else
416 li r1,_PAGE_PRESENT | _PAGE_EXEC
417#endif
418
419 bge- 112f
420 lis r2, (swapper_pg_dir - PAGE_OFFSET)@ha
421 addi r2, r2, (swapper_pg_dir - PAGE_OFFSET)@l
422#endif
423112: rlwimi r2,r3,12,20,29
424 lwz r2,0(r2)
425 rlwinm. r2,r2,0,0,19
426 beq- InstructionAddressInvalid
427 rlwimi r2,r3,22,20,29
428 lwz r0,0(r2)
429 andc. r1,r1,r0
430 bne- InstructionAddressInvalid
431
432 rlwimi r0,r0,32-2,31,31
433 ori r1, r1, 0xe06
434 andc r1, r0, r1
435BEGIN_FTR_SECTION
436 rlwinm r1,r1,0,~_PAGE_COHERENT
437END_FTR_SECTION_IFCLR(CPU_FTR_NEED_COHERENT)
438 mtspr SPRN_RPA,r1
439 tlbli r3
440 mfspr r3,SPRN_SRR1
441 mtcrf 0x80,r3
442 rfi
443InstructionAddressInvalid:
444 mfspr r3,SPRN_SRR1
445 rlwinm r1,r3,9,6,6
446
447 addis r1,r1,0x2000
448 mtspr SPRN_DSISR,r1
449 andi. r2,r3,0xFFFF
450 or r2,r2,r1
451 mtspr SPRN_SRR1,r2
452 mfspr r1,SPRN_IMISS
453 rlwinm. r2,r2,0,31,31
454 rlwimi r2,r2,1,30,30
455 xor r1,r1,r2
456 mtspr SPRN_DAR,r1
457 mfmsr r0
458 xoris r0,r0,MSR_TGPR>>16
459 mtcrf 0x80,r3
460 mtmsr r0
461 b InstructionAccess
462
463
464
465
466 . = 0x1100
467DataLoadTLBMiss:
468
469
470
471
472
473
474
475 mfspr r3,SPRN_DMISS
476 lis r1,PAGE_OFFSET@h
477 cmplw 0,r1,r3
478 mfspr r2, SPRN_SPRG_PGDIR
479#ifdef CONFIG_SWAP
480 li r1, _PAGE_PRESENT | _PAGE_ACCESSED
481#else
482 li r1, _PAGE_PRESENT
483#endif
484 bge- 112f
485 lis r2, (swapper_pg_dir - PAGE_OFFSET)@ha
486 addi r2, r2, (swapper_pg_dir - PAGE_OFFSET)@l
487112: rlwimi r2,r3,12,20,29
488 lwz r2,0(r2)
489 rlwinm. r2,r2,0,0,19
490 beq- DataAddressInvalid
491 rlwimi r2,r3,22,20,29
492 lwz r0,0(r2)
493 andc. r1,r1,r0
494 bne- DataAddressInvalid
495
496
497
498
499
500 rlwinm r1,r0,32-9,30,30
501 rlwimi r0,r0,32-1,30,30
502 rlwimi r0,r0,32-1,31,31
503 ori r1,r1,0xe04
504 andc r1,r0,r1
505BEGIN_FTR_SECTION
506 rlwinm r1,r1,0,~_PAGE_COHERENT
507END_FTR_SECTION_IFCLR(CPU_FTR_NEED_COHERENT)
508 mtspr SPRN_RPA,r1
509 mfspr r2,SPRN_SRR1
510 mtcrf 0x80,r2
511BEGIN_MMU_FTR_SECTION
512 li r0,1
513 mfspr r1,SPRN_SPRG_603_LRU
514 rlwinm r2,r3,20,27,31
515 slw r0,r0,r2
516 xor r1,r0,r1
517 srw r0,r1,r2
518 mtspr SPRN_SPRG_603_LRU,r1
519 mfspr r2,SPRN_SRR1
520 rlwimi r2,r0,31-14,14,14
521 mtspr SPRN_SRR1,r2
522END_MMU_FTR_SECTION_IFSET(MMU_FTR_NEED_DTLB_SW_LRU)
523 tlbld r3
524 rfi
525DataAddressInvalid:
526 mfspr r3,SPRN_SRR1
527 rlwinm r1,r3,9,6,6
528 addis r1,r1,0x2000
529 mtspr SPRN_DSISR,r1
530 andi. r2,r3,0xFFFF
531 mtspr SPRN_SRR1,r2
532 mfspr r1,SPRN_DMISS
533 rlwinm. r2,r2,0,31,31
534 beq 20f
535 xori r1,r1,3
53620: mtspr SPRN_DAR,r1
537 mfmsr r0
538 xoris r0,r0,MSR_TGPR>>16
539 mtcrf 0x80,r3
540 mtmsr r0
541 b DataAccess
542
543
544
545
546 . = 0x1200
547DataStoreTLBMiss:
548
549
550
551
552
553
554
555 mfspr r3,SPRN_DMISS
556 lis r1,PAGE_OFFSET@h
557 cmplw 0,r1,r3
558 mfspr r2, SPRN_SPRG_PGDIR
559#ifdef CONFIG_SWAP
560 li r1, _PAGE_RW | _PAGE_PRESENT | _PAGE_ACCESSED
561#else
562 li r1, _PAGE_RW | _PAGE_PRESENT
563#endif
564 bge- 112f
565 lis r2, (swapper_pg_dir - PAGE_OFFSET)@ha
566 addi r2, r2, (swapper_pg_dir - PAGE_OFFSET)@l
567112: rlwimi r2,r3,12,20,29
568 lwz r2,0(r2)
569 rlwinm. r2,r2,0,0,19
570 beq- DataAddressInvalid
571 rlwimi r2,r3,22,20,29
572 lwz r0,0(r2)
573 andc. r1,r1,r0
574 bne- DataAddressInvalid
575
576
577
578
579
580 rlwimi r0,r0,32-2,31,31
581 li r1,0xe06
582 andc r1,r0,r1
583BEGIN_FTR_SECTION
584 rlwinm r1,r1,0,~_PAGE_COHERENT
585END_FTR_SECTION_IFCLR(CPU_FTR_NEED_COHERENT)
586 mtspr SPRN_RPA,r1
587 mfspr r2,SPRN_SRR1
588 mtcrf 0x80,r2
589BEGIN_MMU_FTR_SECTION
590 li r0,1
591 mfspr r1,SPRN_SPRG_603_LRU
592 rlwinm r2,r3,20,27,31
593 slw r0,r0,r2
594 xor r1,r0,r1
595 srw r0,r1,r2
596 mtspr SPRN_SPRG_603_LRU,r1
597 mfspr r2,SPRN_SRR1
598 rlwimi r2,r0,31-14,14,14
599 mtspr SPRN_SRR1,r2
600END_MMU_FTR_SECTION_IFSET(MMU_FTR_NEED_DTLB_SW_LRU)
601 tlbld r3
602 rfi
603
604#ifndef CONFIG_ALTIVEC
605#define altivec_assist_exception unknown_exception
606#endif
607
608 EXCEPTION(0x1300, Trap_13, instruction_breakpoint_exception, EXC_XFER_STD)
609 EXCEPTION(0x1400, SMI, SMIException, EXC_XFER_STD)
610 EXCEPTION(0x1500, Trap_15, unknown_exception, EXC_XFER_STD)
611 EXCEPTION(0x1600, Trap_16, altivec_assist_exception, EXC_XFER_STD)
612 EXCEPTION(0x1700, Trap_17, TAUException, EXC_XFER_STD)
613 EXCEPTION(0x1800, Trap_18, unknown_exception, EXC_XFER_STD)
614 EXCEPTION(0x1900, Trap_19, unknown_exception, EXC_XFER_STD)
615 EXCEPTION(0x1a00, Trap_1a, unknown_exception, EXC_XFER_STD)
616 EXCEPTION(0x1b00, Trap_1b, unknown_exception, EXC_XFER_STD)
617 EXCEPTION(0x1c00, Trap_1c, unknown_exception, EXC_XFER_STD)
618 EXCEPTION(0x1d00, Trap_1d, unknown_exception, EXC_XFER_STD)
619 EXCEPTION(0x1e00, Trap_1e, unknown_exception, EXC_XFER_STD)
620 EXCEPTION(0x1f00, Trap_1f, unknown_exception, EXC_XFER_STD)
621 EXCEPTION(0x2000, RunMode, RunModeException, EXC_XFER_STD)
622 EXCEPTION(0x2100, Trap_21, unknown_exception, EXC_XFER_STD)
623 EXCEPTION(0x2200, Trap_22, unknown_exception, EXC_XFER_STD)
624 EXCEPTION(0x2300, Trap_23, unknown_exception, EXC_XFER_STD)
625 EXCEPTION(0x2400, Trap_24, unknown_exception, EXC_XFER_STD)
626 EXCEPTION(0x2500, Trap_25, unknown_exception, EXC_XFER_STD)
627 EXCEPTION(0x2600, Trap_26, unknown_exception, EXC_XFER_STD)
628 EXCEPTION(0x2700, Trap_27, unknown_exception, EXC_XFER_STD)
629 EXCEPTION(0x2800, Trap_28, unknown_exception, EXC_XFER_STD)
630 EXCEPTION(0x2900, Trap_29, unknown_exception, EXC_XFER_STD)
631 EXCEPTION(0x2a00, Trap_2a, unknown_exception, EXC_XFER_STD)
632 EXCEPTION(0x2b00, Trap_2b, unknown_exception, EXC_XFER_STD)
633 EXCEPTION(0x2c00, Trap_2c, unknown_exception, EXC_XFER_STD)
634 EXCEPTION(0x2d00, Trap_2d, unknown_exception, EXC_XFER_STD)
635 EXCEPTION(0x2e00, Trap_2e, unknown_exception, EXC_XFER_STD)
636 EXCEPTION(0x2f00, Trap_2f, unknown_exception, EXC_XFER_STD)
637
638 . = 0x3000
639
640AltiVecUnavailable:
641 EXCEPTION_PROLOG
642#ifdef CONFIG_ALTIVEC
643 beq 1f
644 bl load_up_altivec
645 b fast_exception_return
646#endif
6471: addi r3,r1,STACK_FRAME_OVERHEAD
648 EXC_XFER_LITE(0xf20, altivec_unavailable_exception)
649
650PerformanceMonitor:
651 EXCEPTION_PROLOG
652 addi r3,r1,STACK_FRAME_OVERHEAD
653 EXC_XFER_STD(0xf00, performance_monitor_exception)
654
655
656
657
658
659
660relocate_kernel:
661 addis r9,r26,klimit@ha
662 lwz r25,klimit@l(r9)
663 addis r25,r25,-KERNELBASE@h
664 lis r3,PHYSICAL_START@h
665 li r6,0
666 li r5,0x4000
667 bl copy_and_flush
668 addi r0,r3,4f@l
669 mtctr r0
670 bctr
6714: mr r5,r25
672 bl copy_and_flush
673 b turn_on_mmu
674
675
676
677
678
679
680
681_ENTRY(copy_and_flush)
682 addi r5,r5,-4
683 addi r6,r6,-4
6844: li r0,L1_CACHE_BYTES/4
685 mtctr r0
6863: addi r6,r6,4
687 lwzx r0,r6,r4
688 stwx r0,r6,r3
689 bdnz 3b
690 dcbst r6,r3
691 sync
692 icbi r6,r3
693 cmplw 0,r6,r5
694 blt 4b
695 sync
696 isync
697 addi r5,r5,4
698 addi r6,r6,4
699 blr
700
701#ifdef CONFIG_SMP
702 .globl __secondary_start_mpc86xx
703__secondary_start_mpc86xx:
704 mfspr r3, SPRN_PIR
705 stw r3, __secondary_hold_acknowledge@l(0)
706 mr r24, r3
707 b __secondary_start
708
709 .globl __secondary_start_pmac_0
710__secondary_start_pmac_0:
711
712 li r24,0
713 b 1f
714 li r24,1
715 b 1f
716 li r24,2
717 b 1f
718 li r24,3
7191:
720
721
722 mfmsr r0
723 rlwinm r0,r0,0,28,26
724 SYNC
725 mtmsr r0
726 isync
727
728 .globl __secondary_start
729__secondary_start:
730
731 bl __restore_cpu_setup
732
733 lis r3,-KERNELBASE@h
734 mr r4,r24
735 bl call_setup_cpu
736#ifdef CONFIG_PPC_BOOK3S_32
737 lis r3,-KERNELBASE@h
738 bl init_idle_6xx
739#endif
740
741
742 lis r2,secondary_current@ha
743 tophys(r2,r2)
744 lwz r2,secondary_current@l(r2)
745 tophys(r1,r2)
746 lwz r1,TASK_STACK(r1)
747
748
749 addi r1,r1,THREAD_SIZE-STACK_FRAME_OVERHEAD
750 li r0,0
751 tophys(r3,r1)
752 stw r0,0(r3)
753
754
755 bl load_segment_registers
756 bl load_up_mmu
757
758
759 tophys(r4,r2)
760 addi r4,r4,THREAD
761 mtspr SPRN_SPRG_THREAD,r4
762 lis r4, (swapper_pg_dir - PAGE_OFFSET)@h
763 ori r4, r4, (swapper_pg_dir - PAGE_OFFSET)@l
764 mtspr SPRN_SPRG_PGDIR, r4
765
766
767 li r4,MSR_KERNEL
768 lis r3,start_secondary@h
769 ori r3,r3,start_secondary@l
770 mtspr SPRN_SRR0,r3
771 mtspr SPRN_SRR1,r4
772 SYNC
773 RFI
774#endif
775
776#ifdef CONFIG_KVM_BOOK3S_HANDLER
777#include "../kvm/book3s_rmhandlers.S"
778#endif
779
780
781
782
783
784
785_ENTRY(__save_cpu_setup)
786 blr
787_ENTRY(__restore_cpu_setup)
788 blr
789#endif
790
791
792
793
794
795#ifdef CONFIG_KASAN
796early_hash_table:
797 sync
798 isync
799 tlbia
800 sync
801 TLBSYNC
802
803 lis r6, early_hash - PAGE_OFFSET@h
804 ori r6, r6, 3
805 mtspr SPRN_SDR1, r6
806 blr
807#endif
808
809load_up_mmu:
810 sync
811 isync
812 tlbia
813 sync
814 TLBSYNC
815
816 lis r6,_SDR1@ha
817 tophys(r6,r6)
818 lwz r6,_SDR1@l(r6)
819 mtspr SPRN_SDR1,r6
820
821
822
823 mfpvr r3
824 srwi r3,r3,16
825 cmpwi r3,1
826 lis r3,BATS@ha
827 addi r3,r3,BATS@l
828 tophys(r3,r3)
829 LOAD_BAT(0,r3,r4,r5)
830 LOAD_BAT(1,r3,r4,r5)
831 LOAD_BAT(2,r3,r4,r5)
832 LOAD_BAT(3,r3,r4,r5)
833BEGIN_MMU_FTR_SECTION
834 LOAD_BAT(4,r3,r4,r5)
835 LOAD_BAT(5,r3,r4,r5)
836 LOAD_BAT(6,r3,r4,r5)
837 LOAD_BAT(7,r3,r4,r5)
838END_MMU_FTR_SECTION_IFSET(MMU_FTR_USE_HIGH_BATS)
839 blr
840
841load_segment_registers:
842 li r0, NUM_USER_SEGMENTS
843 mtctr r0
844 li r3, 0
845#ifdef CONFIG_PPC_KUEP
846 oris r3, r3, SR_NX@h
847#endif
848#ifdef CONFIG_PPC_KUAP
849 oris r3, r3, SR_KS@h
850#endif
851 li r4, 0
8523: mtsrin r3, r4
853 addi r3, r3, 0x111
854 addis r4, r4, 0x1000
855 bdnz 3b
856 li r0, 16 - NUM_USER_SEGMENTS
857 mtctr r0
858 rlwinm r3, r3, 0, ~SR_NX
859 rlwinm r3, r3, 0, ~SR_KS
860 oris r3, r3, SR_KP@h
8613: mtsrin r3, r4
862 addi r3, r3, 0x111
863 addis r4, r4, 0x1000
864 bdnz 3b
865 blr
866
867
868
869
870start_here:
871
872 lis r2,init_task@h
873 ori r2,r2,init_task@l
874
875
876 tophys(r4,r2)
877 addi r4,r4,THREAD
878 mtspr SPRN_SPRG_THREAD,r4
879 lis r4, (swapper_pg_dir - PAGE_OFFSET)@h
880 ori r4, r4, (swapper_pg_dir - PAGE_OFFSET)@l
881 mtspr SPRN_SPRG_PGDIR, r4
882
883
884 lis r1,init_thread_union@ha
885 addi r1,r1,init_thread_union@l
886 li r0,0
887 stwu r0,THREAD_SIZE-STACK_FRAME_OVERHEAD(r1)
888
889
890
891
892#ifdef CONFIG_KASAN
893 bl kasan_early_init
894#endif
895 li r3,0
896 mr r4,r31
897 bl machine_init
898 bl __save_cpu_setup
899 bl MMU_init
900BEGIN_MMU_FTR_SECTION
901 bl MMU_init_hw_patch
902END_MMU_FTR_SECTION_IFSET(MMU_FTR_HPTE_TABLE)
903
904
905
906
907
908
909 lis r4,2f@h
910 ori r4,r4,2f@l
911 tophys(r4,r4)
912 li r3,MSR_KERNEL & ~(MSR_IR|MSR_DR)
913 mtspr SPRN_SRR0,r4
914 mtspr SPRN_SRR1,r3
915 SYNC
916 RFI
917
9182: bl load_up_mmu
919
920#ifdef CONFIG_BDI_SWITCH
921
922
923
924
925 lis r5, abatron_pteptrs@h
926 ori r5, r5, abatron_pteptrs@l
927 stw r5, 0xf0(r0)
928 lis r6, swapper_pg_dir@h
929 ori r6, r6, swapper_pg_dir@l
930 tophys(r5, r5)
931 stw r6, 0(r5)
932#endif
933
934
935 li r4,MSR_KERNEL
936 lis r3,start_kernel@h
937 ori r3,r3,start_kernel@l
938 mtspr SPRN_SRR0,r3
939 mtspr SPRN_SRR1,r4
940 SYNC
941 RFI
942
943
944
945
946
947
948_ENTRY(switch_mmu_context)
949 lwz r3,MMCONTEXTID(r4)
950 cmpwi cr0,r3,0
951 blt- 4f
952 mulli r3,r3,897
953 rlwinm r3,r3,4,8,27
954#ifdef CONFIG_PPC_KUEP
955 oris r3, r3, SR_NX@h
956#endif
957#ifdef CONFIG_PPC_KUAP
958 oris r3, r3, SR_KS@h
959#endif
960 li r0,NUM_USER_SEGMENTS
961 mtctr r0
962
963 lwz r4, MM_PGD(r4)
964#ifdef CONFIG_BDI_SWITCH
965
966
967
968 lis r5, abatron_pteptrs@ha
969 stw r4, abatron_pteptrs@l + 0x4(r5)
970#endif
971 tophys(r4, r4)
972 mtspr SPRN_SPRG_PGDIR, r4
973 li r4,0
974 isync
9753:
976 mtsrin r3,r4
977 addi r3,r3,0x111
978 rlwinm r3,r3,0,8,3
979 addis r4,r4,0x1000
980 bdnz 3b
981 sync
982 isync
983 blr
9844: trap
985 EMIT_BUG_ENTRY 4b,__FILE__,__LINE__,0
986 blr
987EXPORT_SYMBOL(switch_mmu_context)
988
989
990
991
992
993
994
995
996
997clear_bats:
998 li r10,0
999 mfspr r9,SPRN_PVR
1000 rlwinm r9,r9,16,16,31
1001 cmpwi r9, 1
1002 beq 1f
1003
1004 mtspr SPRN_DBAT0U,r10
1005 mtspr SPRN_DBAT0L,r10
1006 mtspr SPRN_DBAT1U,r10
1007 mtspr SPRN_DBAT1L,r10
1008 mtspr SPRN_DBAT2U,r10
1009 mtspr SPRN_DBAT2L,r10
1010 mtspr SPRN_DBAT3U,r10
1011 mtspr SPRN_DBAT3L,r10
10121:
1013 mtspr SPRN_IBAT0U,r10
1014 mtspr SPRN_IBAT0L,r10
1015 mtspr SPRN_IBAT1U,r10
1016 mtspr SPRN_IBAT1L,r10
1017 mtspr SPRN_IBAT2U,r10
1018 mtspr SPRN_IBAT2L,r10
1019 mtspr SPRN_IBAT3U,r10
1020 mtspr SPRN_IBAT3L,r10
1021BEGIN_MMU_FTR_SECTION
1022
1023
1024
1025
1026
1027
1028 mtspr SPRN_DBAT4U,r10
1029 mtspr SPRN_DBAT4L,r10
1030 mtspr SPRN_DBAT5U,r10
1031 mtspr SPRN_DBAT5L,r10
1032 mtspr SPRN_DBAT6U,r10
1033 mtspr SPRN_DBAT6L,r10
1034 mtspr SPRN_DBAT7U,r10
1035 mtspr SPRN_DBAT7L,r10
1036 mtspr SPRN_IBAT4U,r10
1037 mtspr SPRN_IBAT4L,r10
1038 mtspr SPRN_IBAT5U,r10
1039 mtspr SPRN_IBAT5L,r10
1040 mtspr SPRN_IBAT6U,r10
1041 mtspr SPRN_IBAT6L,r10
1042 mtspr SPRN_IBAT7U,r10
1043 mtspr SPRN_IBAT7L,r10
1044END_MMU_FTR_SECTION_IFSET(MMU_FTR_USE_HIGH_BATS)
1045 blr
1046
1047_ENTRY(update_bats)
1048 lis r4, 1f@h
1049 ori r4, r4, 1f@l
1050 tophys(r4, r4)
1051 mfmsr r6
1052 mflr r7
1053 li r3, MSR_KERNEL & ~(MSR_IR | MSR_DR)
1054 rlwinm r0, r6, 0, ~MSR_RI
1055 rlwinm r0, r0, 0, ~MSR_EE
1056 mtmsr r0
1057 mtspr SPRN_SRR0, r4
1058 mtspr SPRN_SRR1, r3
1059 SYNC
1060 RFI
10611: bl clear_bats
1062 lis r3, BATS@ha
1063 addi r3, r3, BATS@l
1064 tophys(r3, r3)
1065 LOAD_BAT(0, r3, r4, r5)
1066 LOAD_BAT(1, r3, r4, r5)
1067 LOAD_BAT(2, r3, r4, r5)
1068 LOAD_BAT(3, r3, r4, r5)
1069BEGIN_MMU_FTR_SECTION
1070 LOAD_BAT(4, r3, r4, r5)
1071 LOAD_BAT(5, r3, r4, r5)
1072 LOAD_BAT(6, r3, r4, r5)
1073 LOAD_BAT(7, r3, r4, r5)
1074END_MMU_FTR_SECTION_IFSET(MMU_FTR_USE_HIGH_BATS)
1075 li r3, MSR_KERNEL & ~(MSR_IR | MSR_DR | MSR_RI)
1076 mtmsr r3
1077 mtspr SPRN_SRR0, r7
1078 mtspr SPRN_SRR1, r6
1079 SYNC
1080 RFI
1081
1082flush_tlbs:
1083 lis r10, 0x40
10841: addic. r10, r10, -0x1000
1085 tlbie r10
1086 bgt 1b
1087 sync
1088 blr
1089
1090mmu_off:
1091 addi r4, r3, __after_mmu_off - _start
1092 mfmsr r3
1093 andi. r0,r3,MSR_DR|MSR_IR
1094 beqlr
1095 andc r3,r3,r0
1096 mtspr SPRN_SRR0,r4
1097 mtspr SPRN_SRR1,r3
1098 sync
1099 RFI
1100
1101
1102
1103
1104
1105initial_bats:
1106 lis r11,PAGE_OFFSET@h
1107 mfspr r9,SPRN_PVR
1108 rlwinm r9,r9,16,16,31
1109 cmpwi 0,r9,1
1110 bne 4f
1111 ori r11,r11,4
1112 li r8,0x7f
1113 mtspr SPRN_IBAT0U,r11
1114 mtspr SPRN_IBAT0L,r8
1115 addis r11,r11,0x800000@h
1116 addis r8,r8,0x800000@h
1117 mtspr SPRN_IBAT1U,r11
1118 mtspr SPRN_IBAT1L,r8
1119 addis r11,r11,0x800000@h
1120 addis r8,r8,0x800000@h
1121 mtspr SPRN_IBAT2U,r11
1122 mtspr SPRN_IBAT2L,r8
1123 isync
1124 blr
1125
11264: tophys(r8,r11)
1127#ifdef CONFIG_SMP
1128 ori r8,r8,0x12
1129#else
1130 ori r8,r8,2
1131#endif
1132 ori r11,r11,BL_256M<<2|0x2
1133
1134 mtspr SPRN_DBAT0L,r8
1135 mtspr SPRN_DBAT0U,r11
1136 mtspr SPRN_IBAT0L,r8
1137 mtspr SPRN_IBAT0U,r11
1138 isync
1139 blr
1140
1141
1142#ifdef CONFIG_BOOTX_TEXT
1143setup_disp_bat:
1144
1145
1146
1147 mflr r8
1148 bl reloc_offset
1149 mtlr r8
1150 addis r8,r3,disp_BAT@ha
1151 addi r8,r8,disp_BAT@l
1152 cmpwi cr0,r8,0
1153 beqlr
1154 lwz r11,0(r8)
1155 lwz r8,4(r8)
1156 mfspr r9,SPRN_PVR
1157 rlwinm r9,r9,16,16,31
1158 cmpwi 0,r9,1
1159 beq 1f
1160 mtspr SPRN_DBAT3L,r8
1161 mtspr SPRN_DBAT3U,r11
1162 blr
11631: mtspr SPRN_IBAT3L,r8
1164 mtspr SPRN_IBAT3U,r11
1165 blr
1166#endif
1167
1168#ifdef CONFIG_PPC_EARLY_DEBUG_CPM
1169setup_cpm_bat:
1170 lis r8, 0xf000
1171 ori r8, r8, 0x002a
1172 mtspr SPRN_DBAT1L, r8
1173
1174 lis r11, 0xf000
1175 ori r11, r11, (BL_1M << 2) | 2
1176 mtspr SPRN_DBAT1U, r11
1177
1178 blr
1179#endif
1180
1181#ifdef CONFIG_PPC_EARLY_DEBUG_USBGECKO
1182setup_usbgecko_bat:
1183
1184
1185 lis r8, 0x0c00
1186
1187 lis r8, 0x0d00
1188#else
1189
1190#endif
1191
1192
1193
1194
1195 lis r11, 0xfffe
1196 ori r8, r8, 0x002a
1197 ori r11, r11, 0x2
1198 mtspr SPRN_DBAT1L, r8
1199 mtspr SPRN_DBAT1U, r11
1200 blr
1201#endif
1202
1203#ifdef CONFIG_8260
1204
1205
1206
1207
1208
1209
1210
1211 .globl m8260_gorom
1212m8260_gorom:
1213 mfmsr r0
1214 rlwinm r0,r0,0,17,15
1215 sync
1216 mtmsr r0
1217 sync
1218 mfspr r11, SPRN_HID0
1219 lis r10, 0
1220 ori r10,r10,HID0_ICE|HID0_DCE
1221 andc r11, r11, r10
1222 mtspr SPRN_HID0, r11
1223 isync
1224 li r5, MSR_ME|MSR_RI
1225 lis r6,2f@h
1226 addis r6,r6,-KERNELBASE@h
1227 ori r6,r6,2f@l
1228 mtspr SPRN_SRR0,r6
1229 mtspr SPRN_SRR1,r5
1230 isync
1231 sync
1232 rfi
12332:
1234 mtlr r4
1235 blr
1236#endif
1237
1238
1239
1240
1241
1242
1243
1244 .data
1245 .globl sdata
1246sdata:
1247 .globl empty_zero_page
1248empty_zero_page:
1249 .space 4096
1250EXPORT_SYMBOL(empty_zero_page)
1251
1252 .globl swapper_pg_dir
1253swapper_pg_dir:
1254 .space PGD_TABLE_SIZE
1255
1256
1257
1258
1259abatron_pteptrs:
1260 .space 8
1261