1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30#include <linux/kernel.h>
31#include <linux/export.h>
32#include <linux/mm.h>
33#include <linux/init.h>
34#include <linux/highmem.h>
35#include <linux/pagemap.h>
36#include <linux/preempt.h>
37#include <linux/spinlock.h>
38#include <linux/memblock.h>
39#include <linux/of_fdt.h>
40#include <linux/hugetlb.h>
41
42#include <asm/tlbflush.h>
43#include <asm/tlb.h>
44#include <asm/code-patching.h>
45#include <asm/cputhreads.h>
46#include <asm/hugetlb.h>
47#include <asm/paca.h>
48
49#include "mmu_decl.h"
50
51
52
53
54
55
56#if defined(CONFIG_PPC_BOOK3E_MMU) || defined(CONFIG_PPC_8xx)
57#ifdef CONFIG_PPC_FSL_BOOK3E
58struct mmu_psize_def mmu_psize_defs[MMU_PAGE_COUNT] = {
59 [MMU_PAGE_4K] = {
60 .shift = 12,
61 .enc = BOOK3E_PAGESZ_4K,
62 },
63 [MMU_PAGE_2M] = {
64 .shift = 21,
65 .enc = BOOK3E_PAGESZ_2M,
66 },
67 [MMU_PAGE_4M] = {
68 .shift = 22,
69 .enc = BOOK3E_PAGESZ_4M,
70 },
71 [MMU_PAGE_16M] = {
72 .shift = 24,
73 .enc = BOOK3E_PAGESZ_16M,
74 },
75 [MMU_PAGE_64M] = {
76 .shift = 26,
77 .enc = BOOK3E_PAGESZ_64M,
78 },
79 [MMU_PAGE_256M] = {
80 .shift = 28,
81 .enc = BOOK3E_PAGESZ_256M,
82 },
83 [MMU_PAGE_1G] = {
84 .shift = 30,
85 .enc = BOOK3E_PAGESZ_1GB,
86 },
87};
88#elif defined(CONFIG_PPC_8xx)
89struct mmu_psize_def mmu_psize_defs[MMU_PAGE_COUNT] = {
90
91#ifdef CONFIG_PPC_4K_PAGES
92 [MMU_PAGE_4K] = {
93 .shift = 12,
94 },
95#else
96 [MMU_PAGE_16K] = {
97 .shift = 14,
98 },
99#endif
100 [MMU_PAGE_512K] = {
101 .shift = 19,
102 },
103 [MMU_PAGE_8M] = {
104 .shift = 23,
105 },
106};
107#else
108struct mmu_psize_def mmu_psize_defs[MMU_PAGE_COUNT] = {
109 [MMU_PAGE_4K] = {
110 .shift = 12,
111 .ind = 20,
112 .enc = BOOK3E_PAGESZ_4K,
113 },
114 [MMU_PAGE_16K] = {
115 .shift = 14,
116 .enc = BOOK3E_PAGESZ_16K,
117 },
118 [MMU_PAGE_64K] = {
119 .shift = 16,
120 .ind = 28,
121 .enc = BOOK3E_PAGESZ_64K,
122 },
123 [MMU_PAGE_1M] = {
124 .shift = 20,
125 .enc = BOOK3E_PAGESZ_1M,
126 },
127 [MMU_PAGE_16M] = {
128 .shift = 24,
129 .ind = 36,
130 .enc = BOOK3E_PAGESZ_16M,
131 },
132 [MMU_PAGE_256M] = {
133 .shift = 28,
134 .enc = BOOK3E_PAGESZ_256M,
135 },
136 [MMU_PAGE_1G] = {
137 .shift = 30,
138 .enc = BOOK3E_PAGESZ_1GB,
139 },
140};
141#endif
142
143static inline int mmu_get_tsize(int psize)
144{
145 return mmu_psize_defs[psize].enc;
146}
147#else
148static inline int mmu_get_tsize(int psize)
149{
150
151 return 0;
152}
153#endif
154
155
156
157
158
159#ifdef CONFIG_PPC64
160
161int mmu_linear_psize;
162int mmu_pte_psize;
163int mmu_vmemmap_psize;
164int book3e_htw_mode;
165unsigned long linear_map_top;
166
167
168
169
170
171
172
173
174int extlb_level_exc;
175
176#endif
177
178#ifdef CONFIG_PPC_FSL_BOOK3E
179
180DEFINE_PER_CPU(int, next_tlbcam_idx);
181EXPORT_PER_CPU_SYMBOL(next_tlbcam_idx);
182#endif
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199void local_flush_tlb_mm(struct mm_struct *mm)
200{
201 unsigned int pid;
202
203 preempt_disable();
204 pid = mm->context.id;
205 if (pid != MMU_NO_CONTEXT)
206 _tlbil_pid(pid);
207 preempt_enable();
208}
209EXPORT_SYMBOL(local_flush_tlb_mm);
210
211void __local_flush_tlb_page(struct mm_struct *mm, unsigned long vmaddr,
212 int tsize, int ind)
213{
214 unsigned int pid;
215
216 preempt_disable();
217 pid = mm ? mm->context.id : 0;
218 if (pid != MMU_NO_CONTEXT)
219 _tlbil_va(vmaddr, pid, tsize, ind);
220 preempt_enable();
221}
222
223void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr)
224{
225 __local_flush_tlb_page(vma ? vma->vm_mm : NULL, vmaddr,
226 mmu_get_tsize(mmu_virtual_psize), 0);
227}
228EXPORT_SYMBOL(local_flush_tlb_page);
229
230
231
232
233#ifdef CONFIG_SMP
234
235static DEFINE_RAW_SPINLOCK(tlbivax_lock);
236
237struct tlb_flush_param {
238 unsigned long addr;
239 unsigned int pid;
240 unsigned int tsize;
241 unsigned int ind;
242};
243
244static void do_flush_tlb_mm_ipi(void *param)
245{
246 struct tlb_flush_param *p = param;
247
248 _tlbil_pid(p ? p->pid : 0);
249}
250
251static void do_flush_tlb_page_ipi(void *param)
252{
253 struct tlb_flush_param *p = param;
254
255 _tlbil_va(p->addr, p->pid, p->tsize, p->ind);
256}
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275void flush_tlb_mm(struct mm_struct *mm)
276{
277 unsigned int pid;
278
279 preempt_disable();
280 pid = mm->context.id;
281 if (unlikely(pid == MMU_NO_CONTEXT))
282 goto no_context;
283 if (!mm_is_core_local(mm)) {
284 struct tlb_flush_param p = { .pid = pid };
285
286 smp_call_function_many(mm_cpumask(mm),
287 do_flush_tlb_mm_ipi, &p, 1);
288 }
289 _tlbil_pid(pid);
290 no_context:
291 preempt_enable();
292}
293EXPORT_SYMBOL(flush_tlb_mm);
294
295void __flush_tlb_page(struct mm_struct *mm, unsigned long vmaddr,
296 int tsize, int ind)
297{
298 struct cpumask *cpu_mask;
299 unsigned int pid;
300
301
302
303
304
305 if (unlikely(WARN_ON(!mm)))
306 return;
307
308 preempt_disable();
309 pid = mm->context.id;
310 if (unlikely(pid == MMU_NO_CONTEXT))
311 goto bail;
312 cpu_mask = mm_cpumask(mm);
313 if (!mm_is_core_local(mm)) {
314
315 if (mmu_has_feature(MMU_FTR_USE_TLBIVAX_BCAST)) {
316 int lock = mmu_has_feature(MMU_FTR_LOCK_BCAST_INVAL);
317 if (lock)
318 raw_spin_lock(&tlbivax_lock);
319 _tlbivax_bcast(vmaddr, pid, tsize, ind);
320 if (lock)
321 raw_spin_unlock(&tlbivax_lock);
322 goto bail;
323 } else {
324 struct tlb_flush_param p = {
325 .pid = pid,
326 .addr = vmaddr,
327 .tsize = tsize,
328 .ind = ind,
329 };
330
331 smp_call_function_many(cpu_mask,
332 do_flush_tlb_page_ipi, &p, 1);
333 }
334 }
335 _tlbil_va(vmaddr, pid, tsize, ind);
336 bail:
337 preempt_enable();
338}
339
340void flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr)
341{
342#ifdef CONFIG_HUGETLB_PAGE
343 if (vma && is_vm_hugetlb_page(vma))
344 flush_hugetlb_page(vma, vmaddr);
345#endif
346
347 __flush_tlb_page(vma ? vma->vm_mm : NULL, vmaddr,
348 mmu_get_tsize(mmu_virtual_psize), 0);
349}
350EXPORT_SYMBOL(flush_tlb_page);
351
352#endif
353
354#ifdef CONFIG_PPC_47x
355void __init early_init_mmu_47x(void)
356{
357#ifdef CONFIG_SMP
358 unsigned long root = of_get_flat_dt_root();
359 if (of_get_flat_dt_prop(root, "cooperative-partition", NULL))
360 mmu_clear_feature(MMU_FTR_USE_TLBIVAX_BCAST);
361#endif
362}
363#endif
364
365
366
367
368void flush_tlb_kernel_range(unsigned long start, unsigned long end)
369{
370#ifdef CONFIG_SMP
371 preempt_disable();
372 smp_call_function(do_flush_tlb_mm_ipi, NULL, 1);
373 _tlbil_pid(0);
374 preempt_enable();
375#else
376 _tlbil_pid(0);
377#endif
378}
379EXPORT_SYMBOL(flush_tlb_kernel_range);
380
381
382
383
384
385
386
387void flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
388 unsigned long end)
389
390{
391 if (end - start == PAGE_SIZE && !(start & ~PAGE_MASK))
392 flush_tlb_page(vma, start);
393 else
394 flush_tlb_mm(vma->vm_mm);
395}
396EXPORT_SYMBOL(flush_tlb_range);
397
398void tlb_flush(struct mmu_gather *tlb)
399{
400 flush_tlb_mm(tlb->mm);
401}
402
403
404
405
406
407
408#ifdef CONFIG_PPC64
409
410
411
412
413
414void tlb_flush_pgtable(struct mmu_gather *tlb, unsigned long address)
415{
416 int tsize = mmu_psize_defs[mmu_pte_psize].enc;
417
418 if (book3e_htw_mode != PPC_HTW_NONE) {
419 unsigned long start = address & PMD_MASK;
420 unsigned long end = address + PMD_SIZE;
421 unsigned long size = 1UL << mmu_psize_defs[mmu_pte_psize].shift;
422
423
424
425
426
427 while (start < end) {
428 __flush_tlb_page(tlb->mm, start, tsize, 1);
429 start += size;
430 }
431 } else {
432 unsigned long rmask = 0xf000000000000000ul;
433 unsigned long rid = (address & rmask) | 0x1000000000000000ul;
434 unsigned long vpte = address & ~rmask;
435
436#ifdef CONFIG_PPC_64K_PAGES
437 vpte = (vpte >> (PAGE_SHIFT - 4)) & ~0xfffful;
438#else
439 vpte = (vpte >> (PAGE_SHIFT - 3)) & ~0xffful;
440#endif
441 vpte |= rid;
442 __flush_tlb_page(tlb->mm, vpte, tsize, 0);
443 }
444}
445
446static void setup_page_sizes(void)
447{
448 unsigned int tlb0cfg;
449 unsigned int tlb0ps;
450 unsigned int eptcfg;
451 int i, psize;
452
453#ifdef CONFIG_PPC_FSL_BOOK3E
454 unsigned int mmucfg = mfspr(SPRN_MMUCFG);
455 int fsl_mmu = mmu_has_feature(MMU_FTR_TYPE_FSL_E);
456
457 if (fsl_mmu && (mmucfg & MMUCFG_MAVN) == MMUCFG_MAVN_V1) {
458 unsigned int tlb1cfg = mfspr(SPRN_TLB1CFG);
459 unsigned int min_pg, max_pg;
460
461 min_pg = (tlb1cfg & TLBnCFG_MINSIZE) >> TLBnCFG_MINSIZE_SHIFT;
462 max_pg = (tlb1cfg & TLBnCFG_MAXSIZE) >> TLBnCFG_MAXSIZE_SHIFT;
463
464 for (psize = 0; psize < MMU_PAGE_COUNT; ++psize) {
465 struct mmu_psize_def *def;
466 unsigned int shift;
467
468 def = &mmu_psize_defs[psize];
469 shift = def->shift;
470
471 if (shift == 0 || shift & 1)
472 continue;
473
474
475 shift = (shift - 10) >> 1;
476
477 if ((shift >= min_pg) && (shift <= max_pg))
478 def->flags |= MMU_PAGE_SIZE_DIRECT;
479 }
480
481 goto out;
482 }
483
484 if (fsl_mmu && (mmucfg & MMUCFG_MAVN) == MMUCFG_MAVN_V2) {
485 u32 tlb1cfg, tlb1ps;
486
487 tlb0cfg = mfspr(SPRN_TLB0CFG);
488 tlb1cfg = mfspr(SPRN_TLB1CFG);
489 tlb1ps = mfspr(SPRN_TLB1PS);
490 eptcfg = mfspr(SPRN_EPTCFG);
491
492 if ((tlb1cfg & TLBnCFG_IND) && (tlb0cfg & TLBnCFG_PT))
493 book3e_htw_mode = PPC_HTW_E6500;
494
495
496
497
498
499
500 if (eptcfg != 2)
501 book3e_htw_mode = PPC_HTW_NONE;
502
503 for (psize = 0; psize < MMU_PAGE_COUNT; ++psize) {
504 struct mmu_psize_def *def = &mmu_psize_defs[psize];
505
506 if (tlb1ps & (1U << (def->shift - 10))) {
507 def->flags |= MMU_PAGE_SIZE_DIRECT;
508
509 if (book3e_htw_mode && psize == MMU_PAGE_2M)
510 def->flags |= MMU_PAGE_SIZE_INDIRECT;
511 }
512 }
513
514 goto out;
515 }
516#endif
517
518 tlb0cfg = mfspr(SPRN_TLB0CFG);
519 tlb0ps = mfspr(SPRN_TLB0PS);
520 eptcfg = mfspr(SPRN_EPTCFG);
521
522
523 for (psize = 0; psize < MMU_PAGE_COUNT; ++psize) {
524 struct mmu_psize_def *def = &mmu_psize_defs[psize];
525
526 if (tlb0ps & (1U << (def->shift - 10)))
527 def->flags |= MMU_PAGE_SIZE_DIRECT;
528 }
529
530
531 if ((tlb0cfg & TLBnCFG_IND) == 0 ||
532 (tlb0cfg & TLBnCFG_PT) == 0)
533 goto out;
534
535 book3e_htw_mode = PPC_HTW_IBM;
536
537
538
539
540
541
542 for (i = 0; i < 3; i++) {
543 unsigned int ps, sps;
544
545 sps = eptcfg & 0x1f;
546 eptcfg >>= 5;
547 ps = eptcfg & 0x1f;
548 eptcfg >>= 5;
549 if (!ps || !sps)
550 continue;
551 for (psize = 0; psize < MMU_PAGE_COUNT; psize++) {
552 struct mmu_psize_def *def = &mmu_psize_defs[psize];
553
554 if (ps == (def->shift - 10))
555 def->flags |= MMU_PAGE_SIZE_INDIRECT;
556 if (sps == (def->shift - 10))
557 def->ind = ps + 10;
558 }
559 }
560
561out:
562
563 pr_info("MMU: Supported page sizes\n");
564 for (psize = 0; psize < MMU_PAGE_COUNT; ++psize) {
565 struct mmu_psize_def *def = &mmu_psize_defs[psize];
566 const char *__page_type_names[] = {
567 "unsupported",
568 "direct",
569 "indirect",
570 "direct & indirect"
571 };
572 if (def->flags == 0) {
573 def->shift = 0;
574 continue;
575 }
576 pr_info(" %8ld KB as %s\n", 1ul << (def->shift - 10),
577 __page_type_names[def->flags & 0x3]);
578 }
579}
580
581static void setup_mmu_htw(void)
582{
583
584
585
586
587
588 switch (book3e_htw_mode) {
589 case PPC_HTW_IBM:
590 patch_exception(0x1c0, exc_data_tlb_miss_htw_book3e);
591 patch_exception(0x1e0, exc_instruction_tlb_miss_htw_book3e);
592 break;
593#ifdef CONFIG_PPC_FSL_BOOK3E
594 case PPC_HTW_E6500:
595 extlb_level_exc = EX_TLB_SIZE;
596 patch_exception(0x1c0, exc_data_tlb_miss_e6500_book3e);
597 patch_exception(0x1e0, exc_instruction_tlb_miss_e6500_book3e);
598 break;
599#endif
600 }
601 pr_info("MMU: Book3E HW tablewalk %s\n",
602 book3e_htw_mode != PPC_HTW_NONE ? "enabled" : "not supported");
603}
604
605
606
607
608static void early_init_this_mmu(void)
609{
610 unsigned int mas4;
611
612
613
614 mas4 = 0x4 << MAS4_WIMGED_SHIFT;
615 switch (book3e_htw_mode) {
616 case PPC_HTW_E6500:
617 mas4 |= MAS4_INDD;
618 mas4 |= BOOK3E_PAGESZ_2M << MAS4_TSIZED_SHIFT;
619 mas4 |= MAS4_TLBSELD(1);
620 mmu_pte_psize = MMU_PAGE_2M;
621 break;
622
623 case PPC_HTW_IBM:
624 mas4 |= MAS4_INDD;
625#ifdef CONFIG_PPC_64K_PAGES
626 mas4 |= BOOK3E_PAGESZ_256M << MAS4_TSIZED_SHIFT;
627 mmu_pte_psize = MMU_PAGE_256M;
628#else
629 mas4 |= BOOK3E_PAGESZ_1M << MAS4_TSIZED_SHIFT;
630 mmu_pte_psize = MMU_PAGE_1M;
631#endif
632 break;
633
634 case PPC_HTW_NONE:
635#ifdef CONFIG_PPC_64K_PAGES
636 mas4 |= BOOK3E_PAGESZ_64K << MAS4_TSIZED_SHIFT;
637#else
638 mas4 |= BOOK3E_PAGESZ_4K << MAS4_TSIZED_SHIFT;
639#endif
640 mmu_pte_psize = mmu_virtual_psize;
641 break;
642 }
643 mtspr(SPRN_MAS4, mas4);
644
645#ifdef CONFIG_PPC_FSL_BOOK3E
646 if (mmu_has_feature(MMU_FTR_TYPE_FSL_E)) {
647 unsigned int num_cams;
648 int __maybe_unused cpu = smp_processor_id();
649 bool map = true;
650
651
652 num_cams = (mfspr(SPRN_TLB1CFG) & TLBnCFG_N_ENTRY) / 4;
653
654
655
656
657
658#ifdef CONFIG_SMP
659 if (hweight32(get_tensr()) > 1)
660 map = false;
661#endif
662
663 if (map)
664 linear_map_top = map_mem_in_cams(linear_map_top,
665 num_cams, false);
666 }
667#endif
668
669
670
671
672 mb();
673}
674
675static void __init early_init_mmu_global(void)
676{
677
678
679
680
681
682
683 mmu_linear_psize = MMU_PAGE_1G;
684
685
686
687
688
689
690
691 if (mmu_has_feature(MMU_FTR_TYPE_FSL_E))
692 mmu_vmemmap_psize = MMU_PAGE_4K;
693 else
694 mmu_vmemmap_psize = MMU_PAGE_16M;
695
696
697
698
699
700
701
702 setup_page_sizes();
703
704
705 setup_mmu_htw();
706
707#ifdef CONFIG_PPC_FSL_BOOK3E
708 if (mmu_has_feature(MMU_FTR_TYPE_FSL_E)) {
709 if (book3e_htw_mode == PPC_HTW_NONE) {
710 extlb_level_exc = EX_TLB_SIZE;
711 patch_exception(0x1c0, exc_data_tlb_miss_bolted_book3e);
712 patch_exception(0x1e0,
713 exc_instruction_tlb_miss_bolted_book3e);
714 }
715 }
716#endif
717
718
719
720
721 linear_map_top = memblock_end_of_DRAM();
722}
723
724static void __init early_mmu_set_memory_limit(void)
725{
726#ifdef CONFIG_PPC_FSL_BOOK3E
727 if (mmu_has_feature(MMU_FTR_TYPE_FSL_E)) {
728
729
730
731
732
733
734
735 memblock_enforce_memory_limit(linear_map_top);
736 }
737#endif
738
739 memblock_set_current_limit(linear_map_top);
740}
741
742
743void __init early_init_mmu(void)
744{
745 early_init_mmu_global();
746 early_init_this_mmu();
747 early_mmu_set_memory_limit();
748}
749
750void early_init_mmu_secondary(void)
751{
752 early_init_this_mmu();
753}
754
755void setup_initial_memory_limit(phys_addr_t first_memblock_base,
756 phys_addr_t first_memblock_size)
757{
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775#ifdef CONFIG_PPC_FSL_BOOK3E
776 if (early_mmu_has_feature(MMU_FTR_TYPE_FSL_E)) {
777 unsigned long linear_sz;
778 unsigned int num_cams;
779
780
781 num_cams = (mfspr(SPRN_TLB1CFG) & TLBnCFG_N_ENTRY) / 4;
782
783 linear_sz = map_mem_in_cams(first_memblock_size, num_cams,
784 true);
785
786 ppc64_rma_size = min_t(u64, linear_sz, 0x40000000);
787 } else
788#endif
789 ppc64_rma_size = min_t(u64, first_memblock_size, 0x40000000);
790
791
792 memblock_set_current_limit(first_memblock_base + ppc64_rma_size);
793}
794#else
795void __init early_init_mmu(void)
796{
797#ifdef CONFIG_PPC_47x
798 early_init_mmu_47x();
799#endif
800}
801#endif
802