1
2
3
4
5
6
7
8
9
10
11
12
13#include <asm/processor.h>
14#include <asm/reg.h>
15#include <asm/page.h>
16#include <asm/mmu.h>
17#include <asm/ppc_asm.h>
18#include <asm/asm-offsets.h>
19#include <asm/cputable.h>
20#include <asm/pgtable.h>
21#include <asm/exception-64e.h>
22#include <asm/ppc-opcode.h>
23#include <asm/kvm_asm.h>
24#include <asm/kvm_booke_hv_asm.h>
25
26#ifdef CONFIG_PPC_64K_PAGES
27#define VPTE_PMD_SHIFT (PTE_INDEX_SIZE+1)
28#else
29#define VPTE_PMD_SHIFT (PTE_INDEX_SIZE)
30#endif
31#define VPTE_PUD_SHIFT (VPTE_PMD_SHIFT + PMD_INDEX_SIZE)
32#define VPTE_PGD_SHIFT (VPTE_PUD_SHIFT + PUD_INDEX_SIZE)
33#define VPTE_INDEX_SIZE (VPTE_PGD_SHIFT + PGD_INDEX_SIZE)
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50.macro tlb_prolog_bolted intnum addr
51 mtspr SPRN_SPRG_GEN_SCRATCH,r12
52 mfspr r12,SPRN_SPRG_TLB_EXFRAME
53 std r13,EX_TLB_R13(r12)
54 std r10,EX_TLB_R10(r12)
55 mfspr r13,SPRN_SPRG_PACA
56
57 mfcr r10
58 std r11,EX_TLB_R11(r12)
59#ifdef CONFIG_KVM_BOOKE_HV
60BEGIN_FTR_SECTION
61 mfspr r11, SPRN_SRR1
62END_FTR_SECTION_IFSET(CPU_FTR_EMB_HV)
63#endif
64 DO_KVM \intnum, SPRN_SRR1
65 std r16,EX_TLB_R16(r12)
66 mfspr r16,\addr
67 std r14,EX_TLB_R14(r12)
68 ld r14,PACAPGD(r13)
69 std r15,EX_TLB_R15(r12)
70 std r10,EX_TLB_CR(r12)
71#ifdef CONFIG_PPC_FSL_BOOK3E
72 std r7,EX_TLB_R7(r12)
73#endif
74 TLB_MISS_PROLOG_STATS
75.endm
76
77.macro tlb_epilog_bolted
78 ld r14,EX_TLB_CR(r12)
79#ifdef CONFIG_PPC_FSL_BOOK3E
80 ld r7,EX_TLB_R7(r12)
81#endif
82 ld r10,EX_TLB_R10(r12)
83 ld r11,EX_TLB_R11(r12)
84 ld r13,EX_TLB_R13(r12)
85 mtcr r14
86 ld r14,EX_TLB_R14(r12)
87 ld r15,EX_TLB_R15(r12)
88 TLB_MISS_RESTORE_STATS
89 ld r16,EX_TLB_R16(r12)
90 mfspr r12,SPRN_SPRG_GEN_SCRATCH
91.endm
92
93
94 START_EXCEPTION(data_tlb_miss_bolted)
95 tlb_prolog_bolted BOOKE_INTERRUPT_DTLB_MISS SPRN_DEAR
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119 mfspr r11,SPRN_ESR
120
121 srdi r15,r16,60
122 rldicl. r10,r16,64-PGTABLE_EADDR_SIZE,PGTABLE_EADDR_SIZE+4
123 bne- dtlb_miss_fault_bolted
124
125 rlwinm r10,r11,32-19,27,27
126 rlwimi r10,r11,32-16,19,19
127 cmpwi r15,0
128 ori r10,r10,_PAGE_PRESENT
129 oris r11,r10,_PAGE_ACCESSED@h
130
131 TLB_MISS_STATS_SAVE_INFO_BOLTED
132 bne tlb_miss_kernel_bolted
133
134tlb_miss_common_bolted:
135
136
137
138
139
140
141
142
143
144
145
146 rldicl r15,r16,64-PGDIR_SHIFT+3,64-PGD_INDEX_SIZE-3
147 cmpldi cr0,r14,0
148 clrrdi r15,r15,3
149 beq tlb_miss_fault_bolted
150
151BEGIN_MMU_FTR_SECTION
152
153
154
155 PPC_TLBSRX_DOT(0,R16)
156 ldx r14,r14,r15
157 beq tlb_miss_done_bolted
158MMU_FTR_SECTION_ELSE
159 ldx r14,r14,r15
160ALT_MMU_FTR_SECTION_END_IFSET(MMU_FTR_USE_TLBRSRV)
161
162#ifndef CONFIG_PPC_64K_PAGES
163 rldicl r15,r16,64-PUD_SHIFT+3,64-PUD_INDEX_SIZE-3
164 clrrdi r15,r15,3
165 cmpdi cr0,r14,0
166 bge tlb_miss_fault_bolted
167 ldx r14,r14,r15
168#endif
169
170 rldicl r15,r16,64-PMD_SHIFT+3,64-PMD_INDEX_SIZE-3
171 clrrdi r15,r15,3
172 cmpdi cr0,r14,0
173 bge tlb_miss_fault_bolted
174 ldx r14,r14,r15
175
176 rldicl r15,r16,64-PAGE_SHIFT+3,64-PTE_INDEX_SIZE-3
177 clrrdi r15,r15,3
178 cmpdi cr0,r14,0
179 bge tlb_miss_fault_bolted
180 ldx r14,r14,r15
181
182
183 andc. r15,r11,r14
184 rldicr r15,r14,64-(PTE_RPN_SHIFT-PAGE_SHIFT),63-PAGE_SHIFT
185 bne- tlb_miss_fault_bolted
186
187
188
189
190
191
192
193
194
195
196
197 clrrdi r11,r16,12
198 clrldi r15,r15,12
199 rlwimi r11,r14,32-19,27,31
200 rlwimi r15,r14,32-8,22,25
201 mtspr SPRN_MAS2,r11
202 andi. r11,r14,_PAGE_DIRTY
203 rlwimi r15,r14,32-2,26,31
204
205
206 bne 1f
207 li r11,MAS3_SW|MAS3_UW
208 andc r15,r15,r11
2091:
210 mtspr SPRN_MAS7_MAS3,r15
211 tlbwe
212
213tlb_miss_done_bolted:
214 TLB_MISS_STATS_X(MMSTAT_TLB_MISS_NORM_OK)
215 tlb_epilog_bolted
216 rfi
217
218itlb_miss_kernel_bolted:
219 li r11,_PAGE_PRESENT|_PAGE_BAP_SX
220 oris r11,r11,_PAGE_ACCESSED@h
221tlb_miss_kernel_bolted:
222 mfspr r10,SPRN_MAS1
223 ld r14,PACA_KERNELPGD(r13)
224 cmpldi cr0,r15,8
225 rlwinm r10,r10,0,16,1
226 mtspr SPRN_MAS1,r10
227 beq+ tlb_miss_common_bolted
228
229tlb_miss_fault_bolted:
230
231 andi. r10,r11,_PAGE_EXEC|_PAGE_BAP_SX
232 bne itlb_miss_fault_bolted
233dtlb_miss_fault_bolted:
234 TLB_MISS_STATS_D(MMSTAT_TLB_MISS_NORM_FAULT)
235 tlb_epilog_bolted
236 b exc_data_storage_book3e
237itlb_miss_fault_bolted:
238 TLB_MISS_STATS_I(MMSTAT_TLB_MISS_NORM_FAULT)
239 tlb_epilog_bolted
240 b exc_instruction_storage_book3e
241
242
243 START_EXCEPTION(instruction_tlb_miss_bolted)
244 tlb_prolog_bolted BOOKE_INTERRUPT_ITLB_MISS SPRN_SRR0
245
246 rldicl. r10,r16,64-PGTABLE_EADDR_SIZE,PGTABLE_EADDR_SIZE+4
247 srdi r15,r16,60
248 TLB_MISS_STATS_SAVE_INFO_BOLTED
249 bne- itlb_miss_fault_bolted
250
251 li r11,_PAGE_PRESENT|_PAGE_EXEC
252
253
254
255
256 cmpldi cr0,r15,0
257 oris r11,r11,_PAGE_ACCESSED@h
258 beq tlb_miss_common_bolted
259 b itlb_miss_kernel_bolted
260
261#ifdef CONFIG_PPC_FSL_BOOK3E
262
263
264
265
266
267
268
269
270
271
272
273
274 START_EXCEPTION(instruction_tlb_miss_e6500)
275 tlb_prolog_bolted BOOKE_INTERRUPT_ITLB_MISS SPRN_SRR0
276
277 ld r11,PACA_TCD_PTR(r13)
278 srdi. r15,r16,60
279 ori r16,r16,1
280
281 TLB_MISS_STATS_SAVE_INFO_BOLTED
282 bne tlb_miss_kernel_e6500
283
284 b tlb_miss_common_e6500
285
286 START_EXCEPTION(data_tlb_miss_e6500)
287 tlb_prolog_bolted BOOKE_INTERRUPT_DTLB_MISS SPRN_DEAR
288
289 ld r11,PACA_TCD_PTR(r13)
290 srdi. r15,r16,60
291 rldicr r16,r16,0,62
292
293 TLB_MISS_STATS_SAVE_INFO_BOLTED
294 bne tlb_miss_kernel_e6500
295
296
297
298
299
300
301
302
303
304
305
306
307
308tlb_miss_common_e6500:
309 crmove cr2*4+2,cr0*4+2
310
311BEGIN_FTR_SECTION
312
313
314
315
316
317
318 lhz r10,PACAPACAINDEX(r13)
319 addi r10,r10,1
320 crclr cr1*4+eq
3211: lbarx r15,0,r11
322 cmpdi r15,0
323 bne 2f
324 stbcx. r10,0,r11
325 bne 1b
3263:
327 .subsection 1
3282: cmpd cr1,r15,r10
329 beq cr1,3b
33010: lbz r15,0(r11)
331 cmpdi r15,0
332 bne 10b
333 b 1b
334 .previous
335END_FTR_SECTION_IFSET(CPU_FTR_SMT)
336
337 lbz r7,TCD_ESEL_NEXT(r11)
338
339BEGIN_FTR_SECTION
340
341
342
343
344
345
346
347
348 rlwinm r10,r7,16,0xff0000
349 oris r10,r10,MAS0_TLBSEL(1)@h
350 mtspr SPRN_MAS0,r10
351 isync
352 tlbre
353 mfspr r15,SPRN_MAS1
354 andis. r15,r15,MAS1_VALID@h
355 beq 5f
356
357BEGIN_FTR_SECTION_NESTED(532)
358 mfspr r10,SPRN_MAS8
359 rlwinm r10,r10,0,0x80000fff
360 mtspr SPRN_MAS5,r10
361END_FTR_SECTION_NESTED(CPU_FTR_EMB_HV,CPU_FTR_EMB_HV,532)
362
363 mfspr r10,SPRN_MAS1
364 rlwinm r15,r10,0,0x3fff0000
365 rlwimi r15,r10,20,0x00000003
366 mfspr r10,SPRN_MAS6
367 mtspr SPRN_MAS6,r15
368
369 mfspr r15,SPRN_MAS2
370 isync
371 tlbilxva 0,r15
372 isync
373
374 mtspr SPRN_MAS6,r10
375
3765:
377BEGIN_FTR_SECTION_NESTED(532)
378 li r10,0
379 mtspr SPRN_MAS8,r10
380 mtspr SPRN_MAS5,r10
381END_FTR_SECTION_NESTED(CPU_FTR_EMB_HV,CPU_FTR_EMB_HV,532)
382
383 tlbsx 0,r16
384 mfspr r10,SPRN_MAS1
385 andis. r15,r10,MAS1_VALID@h
386 bne tlb_miss_done_e6500
387FTR_SECTION_ELSE
388 mfspr r10,SPRN_MAS1
389ALT_FTR_SECTION_END_IFSET(CPU_FTR_SMT)
390
391 oris r10,r10,MAS1_VALID@h
392 beq cr2,4f
393 rlwinm r10,r10,0,16,1
3944: mtspr SPRN_MAS1,r10
395
396
397
398
399 rldicl. r10,r16,64-PGTABLE_EADDR_SIZE,PGTABLE_EADDR_SIZE+4
400 bne- tlb_miss_fault_e6500
401
402 rldicl r15,r16,64-PGDIR_SHIFT+3,64-PGD_INDEX_SIZE-3
403 cmpldi cr0,r14,0
404 clrrdi r15,r15,3
405 beq- tlb_miss_fault_e6500
406 ldx r14,r14,r15
407
408 rldicl r15,r16,64-PUD_SHIFT+3,64-PUD_INDEX_SIZE-3
409 clrrdi r15,r15,3
410 cmpdi cr0,r14,0
411 bge tlb_miss_huge_e6500
412 ldx r14,r14,r15
413
414 rldicl r15,r16,64-PMD_SHIFT+3,64-PMD_INDEX_SIZE-3
415 clrrdi r15,r15,3
416 cmpdi cr0,r14,0
417 bge tlb_miss_huge_e6500
418 ldx r14,r14,r15
419
420 mfspr r10,SPRN_MAS0
421 cmpdi cr0,r14,0
422 bge tlb_miss_huge_e6500
423
424
425
426
427
428
429
430
431
432
433
434
435 ori r14,r14,(BOOK3E_PAGESZ_4K << MAS3_SPSIZE_SHIFT)
436 mtspr SPRN_MAS7_MAS3,r14
437
438 clrrdi r15,r16,21
439 mtspr SPRN_MAS2,r15
440
441tlb_miss_huge_done_e6500:
442 lbz r16,TCD_ESEL_MAX(r11)
443 lbz r14,TCD_ESEL_FIRST(r11)
444 rlwimi r10,r7,16,0x00ff0000
445 addi r7,r7,1
446 mtspr SPRN_MAS0,r10
447 cmpw r7,r16
448 iseleq r7,r14,r7
449 stb r7,TCD_ESEL_NEXT(r11)
450
451 tlbwe
452
453tlb_miss_done_e6500:
454 .macro tlb_unlock_e6500
455BEGIN_FTR_SECTION
456 beq cr1,1f
457 li r15,0
458 isync
459 stb r15,0(r11)
4601:
461END_FTR_SECTION_IFSET(CPU_FTR_SMT)
462 .endm
463
464 tlb_unlock_e6500
465 TLB_MISS_STATS_X(MMSTAT_TLB_MISS_NORM_OK)
466 tlb_epilog_bolted
467 rfi
468
469tlb_miss_huge_e6500:
470 beq tlb_miss_fault_e6500
471 li r10,1
472 andi. r15,r14,HUGEPD_SHIFT_MASK@l
473 rldimi r14,r10,63,0
474 xor r14,r14,r15
475 ldx r14,0,r14
476
477
478
479
480
481
482
483
484
485
486 subi r15,r15,10
487 mfspr r10,SPRN_MAS1
488 rlwinm r10,r10,0,~MAS1_IND
489 rlwimi r10,r15,MAS1_TSIZE_SHIFT,MAS1_TSIZE_MASK
490 mtspr SPRN_MAS1,r10
491
492 li r10,-0x400
493 sld r15,r10,r15
494 and r10,r16,r15
495 rldicr r15,r14,64-(PTE_RPN_SHIFT-PAGE_SHIFT),63-PAGE_SHIFT
496 rlwimi r10,r14,32-19,27,31
497 clrldi r15,r15,PAGE_SHIFT
498 rlwimi r15,r14,32-8,22,25
499 mtspr SPRN_MAS2,r10
500 andi. r10,r14,_PAGE_DIRTY
501 rlwimi r15,r14,32-2,26,31
502
503
504 bne 1f
505 li r10,MAS3_SW|MAS3_UW
506 andc r15,r15,r10
5071:
508 mtspr SPRN_MAS7_MAS3,r15
509
510 mfspr r10,SPRN_MAS0
511 b tlb_miss_huge_done_e6500
512
513tlb_miss_kernel_e6500:
514 ld r14,PACA_KERNELPGD(r13)
515 cmpldi cr1,r15,8
516 beq+ cr1,tlb_miss_common_e6500
517
518tlb_miss_fault_e6500:
519 tlb_unlock_e6500
520
521 andi. r16,r16,1
522 bne itlb_miss_fault_e6500
523dtlb_miss_fault_e6500:
524 TLB_MISS_STATS_D(MMSTAT_TLB_MISS_NORM_FAULT)
525 tlb_epilog_bolted
526 b exc_data_storage_book3e
527itlb_miss_fault_e6500:
528 TLB_MISS_STATS_I(MMSTAT_TLB_MISS_NORM_FAULT)
529 tlb_epilog_bolted
530 b exc_instruction_storage_book3e
531#endif
532
533
534
535
536
537
538
539
540
541 START_EXCEPTION(data_tlb_miss)
542 TLB_MISS_PROLOG
543
544
545
546
547
548
549 mfspr r14,SPRN_ESR
550 mfspr r16,SPRN_DEAR
551 srdi r15,r16,60
552 cmpldi cr0,r15,0xc
553 TLB_MISS_STATS_SAVE_INFO
554 beq tlb_load_linear
555
556
557
558
559
560
561 andi. r10,r15,0x1
562 bne- virt_page_table_tlb_miss
563
564 std r14,EX_TLB_ESR(r12);
565 std r16,EX_TLB_DEAR(r12);
566
567
568 li r11,_PAGE_PRESENT
569 oris r11,r11,_PAGE_ACCESSED@h
570
571
572
573 cmpldi cr0,r15,0
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592 rlwimi r11,r14,32-19,27,27
593 rlwimi r11,r14,32-16,19,19
594 beq normal_tlb_miss
595
5961: mfspr r10,SPRN_MAS1
597 cmpldi cr0,r15,8
598 rlwinm r10,r10,0,16,1
599 mtspr SPRN_MAS1,r10
600 beq+ normal_tlb_miss
601
602
603
604
605 TLB_MISS_STATS_D(MMSTAT_TLB_MISS_NORM_FAULT)
606 TLB_MISS_EPILOG_ERROR
607 b exc_data_storage_book3e
608
609
610 START_EXCEPTION(instruction_tlb_miss)
611 TLB_MISS_PROLOG
612
613
614
615
616
617
618 li r14,-1
619
620
621
622
623
624
625
626
627 srdi r15,r16,60
628 cmpldi cr0,r15,0xc
629 TLB_MISS_STATS_SAVE_INFO
630 beq tlb_load_linear
631
632
633
634 li r11,_PAGE_PRESENT|_PAGE_EXEC
635 oris r11,r11,_PAGE_ACCESSED@h
636
637 cmpldi cr0,r15,0
638 std r14,EX_TLB_ESR(r12)
639 beq normal_tlb_miss
640
641 li r11,_PAGE_PRESENT|_PAGE_BAP_SX
642 oris r11,r11,_PAGE_ACCESSED@h
643
644 mfspr r10,SPRN_MAS1
645 cmpldi cr0,r15,8
646 rlwinm r10,r10,0,16,1
647 mtspr SPRN_MAS1,r10
648 beq+ normal_tlb_miss
649
650
651 TLB_MISS_STATS_I(MMSTAT_TLB_MISS_NORM_FAULT)
652 TLB_MISS_EPILOG_ERROR
653 b exc_instruction_storage_book3e
654
655
656
657
658
659
660
661
662
663
664
665
666
667normal_tlb_miss:
668
669
670
671
672
673
674
675
676 ori r10,r15,0x1
677#ifdef CONFIG_PPC_64K_PAGES
678
679 rldicl r14,r16,64-(PAGE_SHIFT-4),PAGE_SHIFT-4+4
680
681
682
683 rldicl r15,r16,64-(PAGE_SHIFT-3),64-15
684
685 rlwimi r14,r15,0,16,31
686#else
687 rldicl r14,r16,64-(PAGE_SHIFT-3),PAGE_SHIFT-3+4
688#endif
689 sldi r15,r10,60
690 clrrdi r14,r14,3
691 or r10,r15,r14
692
693BEGIN_MMU_FTR_SECTION
694
695
696
697 PPC_TLBSRX_DOT(0,R16)
698 ld r14,0(r10)
699 beq normal_tlb_miss_done
700MMU_FTR_SECTION_ELSE
701 ld r14,0(r10)
702ALT_MMU_FTR_SECTION_END_IFSET(MMU_FTR_USE_TLBRSRV)
703
704finish_normal_tlb_miss:
705
706 andc. r15,r11,r14
707 bne- normal_tlb_miss_access_fault
708
709
710
711
712
713
714
715
716
717
718
719
720
721 clrrdi r11,r16,12
722 rlwimi r11,r14,32-19,27,31
723 mtspr SPRN_MAS2,r11
724
725
726 rldicl r11,r14,64-8,64-8
727#ifdef CONFIG_PPC_64K_PAGES
728 cmpldi cr0,r11,BOOK3E_PAGESZ_64K
729#else
730 cmpldi cr0,r11,BOOK3E_PAGESZ_4K
731#endif
732 beq- 1f
733 mfspr r11,SPRN_MAS1
734 rlwimi r11,r14,31,21,24
735 rlwinm r11,r11,0,21,19
736 mtspr SPRN_MAS1,r11
7371:
738
739 rldicr r11,r14,64-(PTE_RPN_SHIFT-PAGE_SHIFT),63-PAGE_SHIFT
740 clrldi r15,r11,12
741 rlwimi r15,r14,32-8,22,25
742 rlwimi r15,r14,32-2,26,31
743
744
745 andi. r11,r14,_PAGE_DIRTY
746 bne 1f
747 li r11,MAS3_SW|MAS3_UW
748 andc r15,r15,r11
7491:
750BEGIN_MMU_FTR_SECTION
751 srdi r16,r15,32
752 mtspr SPRN_MAS3,r15
753 mtspr SPRN_MAS7,r16
754MMU_FTR_SECTION_ELSE
755 mtspr SPRN_MAS7_MAS3,r15
756ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_USE_PAIRED_MAS)
757
758 tlbwe
759
760normal_tlb_miss_done:
761
762
763
764
765 TLB_MISS_STATS_X(MMSTAT_TLB_MISS_NORM_OK)
766 TLB_MISS_EPILOG_SUCCESS
767 rfi
768
769normal_tlb_miss_access_fault:
770
771 andi. r10,r11,_PAGE_EXEC
772 bne 1f
773 ld r14,EX_TLB_DEAR(r12)
774 ld r15,EX_TLB_ESR(r12)
775 mtspr SPRN_DEAR,r14
776 mtspr SPRN_ESR,r15
777 TLB_MISS_STATS_D(MMSTAT_TLB_MISS_NORM_FAULT)
778 TLB_MISS_EPILOG_ERROR
779 b exc_data_storage_book3e
7801: TLB_MISS_STATS_I(MMSTAT_TLB_MISS_NORM_FAULT)
781 TLB_MISS_EPILOG_ERROR
782 b exc_instruction_storage_book3e
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809virt_page_table_tlb_miss:
810
811 andi. r10,r15,0x8
812
813
814
815
816
817 add r11,r10,r13
818
819
820 beq 1f
821
822 mfspr r10,SPRN_MAS1
823 rlwinm r10,r10,0,16,1
824 mtspr SPRN_MAS1,r10
8251:
826BEGIN_MMU_FTR_SECTION
827
828
829
830 PPC_TLBSRX_DOT(0,R16)
831 beq virt_page_table_tlb_miss_done
832END_MMU_FTR_SECTION_IFSET(MMU_FTR_USE_TLBRSRV)
833
834
835
836
837 rldicl. r10,r16,64-(VPTE_INDEX_SIZE+3),VPTE_INDEX_SIZE+3+4
838 bne- virt_page_table_tlb_miss_fault
839
840
841 ld r15,PACAPGD(r11)
842 cmpldi cr0,r15,0
843 beq- virt_page_table_tlb_miss_fault
844
845
846 rldicl r11,r16,64-VPTE_PGD_SHIFT,64-PGD_INDEX_SIZE-3
847 clrrdi r10,r11,3
848 ldx r15,r10,r15
849 cmpdi cr0,r15,0
850 bge virt_page_table_tlb_miss_fault
851
852#ifndef CONFIG_PPC_64K_PAGES
853
854 rldicl r11,r16,64-VPTE_PUD_SHIFT,64-PUD_INDEX_SIZE-3
855 clrrdi r10,r11,3
856 ldx r15,r10,r15
857 cmpdi cr0,r15,0
858 bge virt_page_table_tlb_miss_fault
859#endif
860
861
862 rldicl r11,r16,64-VPTE_PMD_SHIFT,64-PMD_INDEX_SIZE-3
863 clrrdi r10,r11,3
864 ldx r15,r10,r15
865 cmpdi cr0,r15,0
866 bge virt_page_table_tlb_miss_fault
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882 clrldi r11,r15,4
883 ori r10,r11,1
884
885BEGIN_MMU_FTR_SECTION
886 srdi r16,r10,32
887 mtspr SPRN_MAS3,r10
888 mtspr SPRN_MAS7,r16
889MMU_FTR_SECTION_ELSE
890 mtspr SPRN_MAS7_MAS3,r10
891ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_USE_PAIRED_MAS)
892
893 tlbwe
894
895BEGIN_MMU_FTR_SECTION
896virt_page_table_tlb_miss_done:
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912 subf r10,r13,r12
913 cmpldi cr0,r10,PACA_EXTLB+EX_TLB_SIZE
914 bne- 1f
915 ld r11,PACA_EXTLB+EX_TLB_SIZE+EX_TLB_SRR0(r13)
916 addi r10,r11,-4
917 std r10,PACA_EXTLB+EX_TLB_SIZE+EX_TLB_SRR0(r13)
9181:
919END_MMU_FTR_SECTION_IFSET(MMU_FTR_USE_TLBRSRV)
920
921 TLB_MISS_STATS_X(MMSTAT_TLB_MISS_PT_OK);
922 TLB_MISS_EPILOG_SUCCESS
923 rfi
924
925virt_page_table_tlb_miss_fault:
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944 subf r10,r13,r12
945 cmpldi cr0,r10,PACA_EXTLB+EX_TLB_SIZE
946 bne- virt_page_table_tlb_miss_whacko_fault
947
948
949 ld r15,EX_TLB_DEAR+PACA_EXTLB(r13)
950 ld r16,EX_TLB_ESR+PACA_EXTLB(r13)
951
952
953 cmpdi cr0,r16,-1
954 beq 1f
955 mtspr SPRN_DEAR,r15
956 mtspr SPRN_ESR,r16
957 TLB_MISS_STATS_D(MMSTAT_TLB_MISS_PT_FAULT);
958 TLB_MISS_EPILOG_ERROR
959 b exc_data_storage_book3e
9601: TLB_MISS_STATS_I(MMSTAT_TLB_MISS_PT_FAULT);
961 TLB_MISS_EPILOG_ERROR
962 b exc_instruction_storage_book3e
963
964virt_page_table_tlb_miss_whacko_fault:
965
966
967
968 TLB_MISS_STATS_X(MMSTAT_TLB_MISS_PT_FAULT);
969 TLB_MISS_EPILOG_ERROR
970 b exc_data_storage_book3e
971
972
973
974
975
976
977
978
979
980
981 START_EXCEPTION(data_tlb_miss_htw)
982 TLB_MISS_PROLOG
983
984
985
986
987
988
989 mfspr r14,SPRN_ESR
990 mfspr r16,SPRN_DEAR
991 srdi r11,r16,60
992 cmpldi cr0,r11,0xc
993 TLB_MISS_STATS_SAVE_INFO
994 beq tlb_load_linear
995
996
997
998 cmpldi cr0,r11,0
999 ld r15,PACAPGD(r13)
1000 beq htw_tlb_miss
1001
1002
10031: mfspr r10,SPRN_MAS1
1004 cmpldi cr0,r11,8
1005 rlwinm r10,r10,0,16,1
1006 mtspr SPRN_MAS1,r10
1007 ld r15,PACA_KERNELPGD(r13)
1008 beq+ htw_tlb_miss
1009
1010
1011
1012
1013 TLB_MISS_STATS_D(MMSTAT_TLB_MISS_NORM_FAULT)
1014 TLB_MISS_EPILOG_ERROR
1015 b exc_data_storage_book3e
1016
1017
1018 START_EXCEPTION(instruction_tlb_miss_htw)
1019 TLB_MISS_PROLOG
1020
1021
1022
1023
1024
1025
1026 li r14,-1
1027
1028
1029
1030
1031
1032
1033
1034
1035 srdi r11,r16,60
1036 cmpldi cr0,r11,0xc
1037 TLB_MISS_STATS_SAVE_INFO
1038 beq tlb_load_linear
1039
1040
1041
1042 cmpldi cr0,r11,0
1043 ld r15,PACAPGD(r13)
1044 beq htw_tlb_miss
1045
1046
10471: mfspr r10,SPRN_MAS1
1048 cmpldi cr0,r11,8
1049 rlwinm r10,r10,0,16,1
1050 mtspr SPRN_MAS1,r10
1051 ld r15,PACA_KERNELPGD(r13)
1052 beq+ htw_tlb_miss
1053
1054
1055 TLB_MISS_STATS_I(MMSTAT_TLB_MISS_NORM_FAULT)
1056 TLB_MISS_EPILOG_ERROR
1057 b exc_instruction_storage_book3e
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075htw_tlb_miss:
1076
1077
1078
1079
1080
1081 PPC_TLBSRX_DOT(0,R16)
1082 beq htw_tlb_miss_done
1083
1084
1085
1086
1087 rldicl. r10,r16,64-PGTABLE_EADDR_SIZE,PGTABLE_EADDR_SIZE+4
1088 bne- htw_tlb_miss_fault
1089
1090
1091 cmpldi cr0,r15,0
1092 beq- htw_tlb_miss_fault
1093
1094
1095 rldicl r11,r16,64-(PGDIR_SHIFT-3),64-PGD_INDEX_SIZE-3
1096 clrrdi r10,r11,3
1097 ldx r15,r10,r15
1098 cmpdi cr0,r15,0
1099 bge htw_tlb_miss_fault
1100
1101#ifndef CONFIG_PPC_64K_PAGES
1102
1103 rldicl r11,r16,64-(PUD_SHIFT-3),64-PUD_INDEX_SIZE-3
1104 clrrdi r10,r11,3
1105 ldx r15,r10,r15
1106 cmpdi cr0,r15,0
1107 bge htw_tlb_miss_fault
1108#endif
1109
1110
1111 rldicl r11,r16,64-(PMD_SHIFT-3),64-PMD_INDEX_SIZE-3
1112 clrrdi r10,r11,3
1113 ldx r15,r10,r15
1114 cmpdi cr0,r15,0
1115 bge htw_tlb_miss_fault
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127#ifndef CONFIG_PPC_64K_PAGES
1128 rlwimi r15,r16,32-9,20,20
1129#endif
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139#ifdef CONFIG_PPC_64K_PAGES
1140 ori r10,r15,(BOOK3E_PAGESZ_64K << MAS3_SPSIZE_SHIFT)
1141#else
1142 ori r10,r15,(BOOK3E_PAGESZ_4K << MAS3_SPSIZE_SHIFT)
1143#endif
1144
1145BEGIN_MMU_FTR_SECTION
1146 srdi r16,r10,32
1147 mtspr SPRN_MAS3,r10
1148 mtspr SPRN_MAS7,r16
1149MMU_FTR_SECTION_ELSE
1150 mtspr SPRN_MAS7_MAS3,r10
1151ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_USE_PAIRED_MAS)
1152
1153 tlbwe
1154
1155htw_tlb_miss_done:
1156
1157
1158
1159
1160 TLB_MISS_STATS_X(MMSTAT_TLB_MISS_PT_OK)
1161 TLB_MISS_EPILOG_SUCCESS
1162 rfi
1163
1164htw_tlb_miss_fault:
1165
1166
1167
1168 cmpdi cr0,r14,-1
1169 beq 1f
1170 mtspr SPRN_DEAR,r16
1171 mtspr SPRN_ESR,r14
1172 TLB_MISS_STATS_D(MMSTAT_TLB_MISS_PT_FAULT)
1173 TLB_MISS_EPILOG_ERROR
1174 b exc_data_storage_book3e
11751: TLB_MISS_STATS_I(MMSTAT_TLB_MISS_PT_FAULT)
1176 TLB_MISS_EPILOG_ERROR
1177 b exc_instruction_storage_book3e
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203tlb_load_linear:
1204
1205
1206
1207
1208
1209 ld r11,PACATOC(r13)
1210 ld r11,linear_map_top@got(r11)
1211 ld r10,0(r11)
1212 tovirt(10,10)
1213 cmpld cr0,r16,r10
1214 bge tlb_load_linear_fault
1215
1216
1217 li r15,(BOOK3E_PAGESZ_1GB<<MAS1_TSIZE_SHIFT)
1218 oris r15,r15,MAS1_VALID@h
1219 mtspr SPRN_MAS1,r15
1220
1221
1222 PPC_TLBSRX_DOT(0,R16)
1223 beq tlb_load_linear_done
1224
1225
1226
1227
1228
1229
1230
1231 clrrdi r10,r16,30
1232 clrldi r10,r10,4
1233 ori r10,r10,MAS3_SR|MAS3_SW|MAS3_SX
1234
1235BEGIN_MMU_FTR_SECTION
1236 srdi r16,r10,32
1237 mtspr SPRN_MAS3,r10
1238 mtspr SPRN_MAS7,r16
1239MMU_FTR_SECTION_ELSE
1240 mtspr SPRN_MAS7_MAS3,r10
1241ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_USE_PAIRED_MAS)
1242
1243 tlbwe
1244
1245tlb_load_linear_done:
1246
1247
1248
1249
1250
1251 TLB_MISS_STATS_X(MMSTAT_TLB_MISS_LINEAR)
1252 TLB_MISS_EPILOG_ERROR
1253 rfi
1254
1255tlb_load_linear_fault:
1256
1257 cmpdi cr0,r14,-1
1258 beq 1f
1259 TLB_MISS_EPILOG_ERROR_SPECIAL
1260 b exc_data_storage_book3e
12611: TLB_MISS_EPILOG_ERROR_SPECIAL
1262 b exc_instruction_storage_book3e
1263
1264
1265#ifdef CONFIG_BOOK3E_MMU_TLB_STATS
1266.tlb_stat_inc:
12671: ldarx r8,0,r9
1268 addi r8,r8,1
1269 stdcx. r8,0,r9
1270 bne- 1b
1271 blr
1272#endif
1273