1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54#include <linux/module.h>
55#include <linux/bug.h>
56#include <asm/arcregs.h>
57#include <asm/setup.h>
58#include <asm/mmu_context.h>
59#include <asm/mmu.h>
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103DEFINE_PER_CPU(unsigned int, asid_cache) = MM_CTXT_FIRST_CYCLE;
104
105
106
107
108
109static inline void __tlb_entry_erase(void)
110{
111 write_aux_reg(ARC_REG_TLBPD1, 0);
112
113 if (is_pae40_enabled())
114 write_aux_reg(ARC_REG_TLBPD1HI, 0);
115
116 write_aux_reg(ARC_REG_TLBPD0, 0);
117 write_aux_reg(ARC_REG_TLBCOMMAND, TLBWrite);
118}
119
120#if (CONFIG_ARC_MMU_VER < 4)
121
122static inline unsigned int tlb_entry_lkup(unsigned long vaddr_n_asid)
123{
124 unsigned int idx;
125
126 write_aux_reg(ARC_REG_TLBPD0, vaddr_n_asid);
127
128 write_aux_reg(ARC_REG_TLBCOMMAND, TLBProbe);
129 idx = read_aux_reg(ARC_REG_TLBINDEX);
130
131 return idx;
132}
133
134static void tlb_entry_erase(unsigned int vaddr_n_asid)
135{
136 unsigned int idx;
137
138
139 idx = tlb_entry_lkup(vaddr_n_asid);
140
141
142 if (likely(!(idx & TLB_LKUP_ERR))) {
143 __tlb_entry_erase();
144 } else {
145
146 WARN(idx == TLB_DUP_ERR, "Probe returned Dup PD for %x\n",
147 vaddr_n_asid);
148 }
149}
150
151
152
153
154
155
156
157
158
159
160
161
162static void utlb_invalidate(void)
163{
164#if (CONFIG_ARC_MMU_VER >= 2)
165
166#if (CONFIG_ARC_MMU_VER == 2)
167
168
169
170
171
172
173
174 unsigned int idx;
175
176
177 idx = read_aux_reg(ARC_REG_TLBINDEX);
178
179
180 if (unlikely(idx & TLB_LKUP_ERR))
181 write_aux_reg(ARC_REG_TLBINDEX, 0xa);
182#endif
183
184 write_aux_reg(ARC_REG_TLBCOMMAND, TLBIVUTLB);
185#endif
186
187}
188
189static void tlb_entry_insert(unsigned int pd0, pte_t pd1)
190{
191 unsigned int idx;
192
193
194
195
196
197 idx = tlb_entry_lkup(pd0);
198
199
200
201
202
203
204
205 if (likely(idx & TLB_LKUP_ERR))
206 write_aux_reg(ARC_REG_TLBCOMMAND, TLBGetIndex);
207
208
209 write_aux_reg(ARC_REG_TLBPD1, pd1);
210
211
212
213
214
215
216 write_aux_reg(ARC_REG_TLBCOMMAND, TLBWrite);
217}
218
219#else
220
221static void utlb_invalidate(void)
222{
223
224}
225
226static void tlb_entry_erase(unsigned int vaddr_n_asid)
227{
228 write_aux_reg(ARC_REG_TLBPD0, vaddr_n_asid | _PAGE_PRESENT);
229 write_aux_reg(ARC_REG_TLBCOMMAND, TLBDeleteEntry);
230}
231
232static void tlb_entry_insert(unsigned int pd0, pte_t pd1)
233{
234 write_aux_reg(ARC_REG_TLBPD0, pd0);
235 write_aux_reg(ARC_REG_TLBPD1, pd1);
236
237 if (is_pae40_enabled())
238 write_aux_reg(ARC_REG_TLBPD1HI, (u64)pd1 >> 32);
239
240 write_aux_reg(ARC_REG_TLBCOMMAND, TLBInsertEntry);
241}
242
243#endif
244
245
246
247
248
249noinline void local_flush_tlb_all(void)
250{
251 struct cpuinfo_arc_mmu *mmu = &cpuinfo_arc700[smp_processor_id()].mmu;
252 unsigned long flags;
253 unsigned int entry;
254 int num_tlb = mmu->sets * mmu->ways;
255
256 local_irq_save(flags);
257
258
259 write_aux_reg(ARC_REG_TLBPD1, 0);
260
261 if (is_pae40_enabled())
262 write_aux_reg(ARC_REG_TLBPD1HI, 0);
263
264 write_aux_reg(ARC_REG_TLBPD0, 0);
265
266 for (entry = 0; entry < num_tlb; entry++) {
267
268 write_aux_reg(ARC_REG_TLBINDEX, entry);
269 write_aux_reg(ARC_REG_TLBCOMMAND, TLBWrite);
270 }
271
272 if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) {
273 const int stlb_idx = 0x800;
274
275
276 write_aux_reg(ARC_REG_TLBPD0, _PAGE_HW_SZ);
277
278 for (entry = stlb_idx; entry < stlb_idx + 16; entry++) {
279 write_aux_reg(ARC_REG_TLBINDEX, entry);
280 write_aux_reg(ARC_REG_TLBCOMMAND, TLBWrite);
281 }
282 }
283
284 utlb_invalidate();
285
286 local_irq_restore(flags);
287}
288
289
290
291
292noinline void local_flush_tlb_mm(struct mm_struct *mm)
293{
294
295
296
297
298
299
300 if (atomic_read(&mm->mm_users) == 0)
301 return;
302
303
304
305
306
307
308
309
310 destroy_context(mm);
311 if (current->mm == mm)
312 get_new_mmu_context(mm);
313}
314
315
316
317
318
319
320
321
322
323void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
324 unsigned long end)
325{
326 const unsigned int cpu = smp_processor_id();
327 unsigned long flags;
328
329
330
331
332
333
334
335
336 if (unlikely((end - start) >= PAGE_SIZE * 32)) {
337 local_flush_tlb_mm(vma->vm_mm);
338 return;
339 }
340
341
342
343
344
345
346 start &= PAGE_MASK;
347
348 local_irq_save(flags);
349
350 if (asid_mm(vma->vm_mm, cpu) != MM_CTXT_NO_ASID) {
351 while (start < end) {
352 tlb_entry_erase(start | hw_pid(vma->vm_mm, cpu));
353 start += PAGE_SIZE;
354 }
355 }
356
357 utlb_invalidate();
358
359 local_irq_restore(flags);
360}
361
362
363
364
365
366
367
368void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)
369{
370 unsigned long flags;
371
372
373
374 if (unlikely((end - start) >= PAGE_SIZE * 32)) {
375 local_flush_tlb_all();
376 return;
377 }
378
379 start &= PAGE_MASK;
380
381 local_irq_save(flags);
382 while (start < end) {
383 tlb_entry_erase(start);
384 start += PAGE_SIZE;
385 }
386
387 utlb_invalidate();
388
389 local_irq_restore(flags);
390}
391
392
393
394
395
396
397void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
398{
399 const unsigned int cpu = smp_processor_id();
400 unsigned long flags;
401
402
403
404
405 local_irq_save(flags);
406
407 if (asid_mm(vma->vm_mm, cpu) != MM_CTXT_NO_ASID) {
408 tlb_entry_erase((page & PAGE_MASK) | hw_pid(vma->vm_mm, cpu));
409 utlb_invalidate();
410 }
411
412 local_irq_restore(flags);
413}
414
415#ifdef CONFIG_SMP
416
417struct tlb_args {
418 struct vm_area_struct *ta_vma;
419 unsigned long ta_start;
420 unsigned long ta_end;
421};
422
423static inline void ipi_flush_tlb_page(void *arg)
424{
425 struct tlb_args *ta = arg;
426
427 local_flush_tlb_page(ta->ta_vma, ta->ta_start);
428}
429
430static inline void ipi_flush_tlb_range(void *arg)
431{
432 struct tlb_args *ta = arg;
433
434 local_flush_tlb_range(ta->ta_vma, ta->ta_start, ta->ta_end);
435}
436
437#ifdef CONFIG_TRANSPARENT_HUGEPAGE
438static inline void ipi_flush_pmd_tlb_range(void *arg)
439{
440 struct tlb_args *ta = arg;
441
442 local_flush_pmd_tlb_range(ta->ta_vma, ta->ta_start, ta->ta_end);
443}
444#endif
445
446static inline void ipi_flush_tlb_kernel_range(void *arg)
447{
448 struct tlb_args *ta = (struct tlb_args *)arg;
449
450 local_flush_tlb_kernel_range(ta->ta_start, ta->ta_end);
451}
452
453void flush_tlb_all(void)
454{
455 on_each_cpu((smp_call_func_t)local_flush_tlb_all, NULL, 1);
456}
457
458void flush_tlb_mm(struct mm_struct *mm)
459{
460 on_each_cpu_mask(mm_cpumask(mm), (smp_call_func_t)local_flush_tlb_mm,
461 mm, 1);
462}
463
464void flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr)
465{
466 struct tlb_args ta = {
467 .ta_vma = vma,
468 .ta_start = uaddr
469 };
470
471 on_each_cpu_mask(mm_cpumask(vma->vm_mm), ipi_flush_tlb_page, &ta, 1);
472}
473
474void flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
475 unsigned long end)
476{
477 struct tlb_args ta = {
478 .ta_vma = vma,
479 .ta_start = start,
480 .ta_end = end
481 };
482
483 on_each_cpu_mask(mm_cpumask(vma->vm_mm), ipi_flush_tlb_range, &ta, 1);
484}
485
486#ifdef CONFIG_TRANSPARENT_HUGEPAGE
487void flush_pmd_tlb_range(struct vm_area_struct *vma, unsigned long start,
488 unsigned long end)
489{
490 struct tlb_args ta = {
491 .ta_vma = vma,
492 .ta_start = start,
493 .ta_end = end
494 };
495
496 on_each_cpu_mask(mm_cpumask(vma->vm_mm), ipi_flush_pmd_tlb_range, &ta, 1);
497}
498#endif
499
500void flush_tlb_kernel_range(unsigned long start, unsigned long end)
501{
502 struct tlb_args ta = {
503 .ta_start = start,
504 .ta_end = end
505 };
506
507 on_each_cpu(ipi_flush_tlb_kernel_range, &ta, 1);
508}
509#endif
510
511
512
513
514void create_tlb(struct vm_area_struct *vma, unsigned long vaddr, pte_t *ptep)
515{
516 unsigned long flags;
517 unsigned int asid_or_sasid, rwx;
518 unsigned long pd0;
519 pte_t pd1;
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546 if (current->active_mm != vma->vm_mm)
547 return;
548
549 local_irq_save(flags);
550
551 tlb_paranoid_check(asid_mm(vma->vm_mm, smp_processor_id()), vaddr);
552
553 vaddr &= PAGE_MASK;
554
555
556 pte_val(*ptep) |= (_PAGE_PRESENT | _PAGE_ACCESSED);
557
558
559
560
561 asid_or_sasid = read_aux_reg(ARC_REG_PID) & 0xff;
562
563 pd0 = vaddr | asid_or_sasid | (pte_val(*ptep) & PTE_BITS_IN_PD0);
564
565
566
567
568
569
570
571
572 rwx = pte_val(*ptep) & PTE_BITS_RWX;
573
574 if (pte_val(*ptep) & _PAGE_GLOBAL)
575 rwx <<= 3;
576 else
577 rwx |= (rwx << 3);
578
579 pd1 = rwx | (pte_val(*ptep) & PTE_BITS_NON_RWX_IN_PD1);
580
581 tlb_entry_insert(pd0, pd1);
582
583 local_irq_restore(flags);
584}
585
586
587
588
589
590
591
592
593
594
595void update_mmu_cache(struct vm_area_struct *vma, unsigned long vaddr_unaligned,
596 pte_t *ptep)
597{
598 unsigned long vaddr = vaddr_unaligned & PAGE_MASK;
599 phys_addr_t paddr = pte_val(*ptep) & PAGE_MASK;
600 struct page *page = pfn_to_page(pte_pfn(*ptep));
601
602 create_tlb(vma, vaddr, ptep);
603
604 if (page == ZERO_PAGE(0)) {
605 return;
606 }
607
608
609
610
611
612
613
614
615
616
617 if ((vma->vm_flags & VM_EXEC) ||
618 addr_not_cache_congruent(paddr, vaddr)) {
619
620 int dirty = !test_and_set_bit(PG_dc_clean, &page->flags);
621 if (dirty) {
622
623 __flush_dcache_page(paddr, paddr);
624
625
626 if (vma->vm_flags & VM_EXEC)
627 __inv_icache_page(paddr, vaddr);
628 }
629 }
630}
631
632#ifdef CONFIG_TRANSPARENT_HUGEPAGE
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655void update_mmu_cache_pmd(struct vm_area_struct *vma, unsigned long addr,
656 pmd_t *pmd)
657{
658 pte_t pte = __pte(pmd_val(*pmd));
659 update_mmu_cache(vma, addr, &pte);
660}
661
662void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
663 pgtable_t pgtable)
664{
665 struct list_head *lh = (struct list_head *) pgtable;
666
667 assert_spin_locked(&mm->page_table_lock);
668
669
670 if (!pmd_huge_pte(mm, pmdp))
671 INIT_LIST_HEAD(lh);
672 else
673 list_add(lh, (struct list_head *) pmd_huge_pte(mm, pmdp));
674 pmd_huge_pte(mm, pmdp) = pgtable;
675}
676
677pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp)
678{
679 struct list_head *lh;
680 pgtable_t pgtable;
681
682 assert_spin_locked(&mm->page_table_lock);
683
684 pgtable = pmd_huge_pte(mm, pmdp);
685 lh = (struct list_head *) pgtable;
686 if (list_empty(lh))
687 pmd_huge_pte(mm, pmdp) = NULL;
688 else {
689 pmd_huge_pte(mm, pmdp) = (pgtable_t) lh->next;
690 list_del(lh);
691 }
692
693 pte_val(pgtable[0]) = 0;
694 pte_val(pgtable[1]) = 0;
695
696 return pgtable;
697}
698
699void local_flush_pmd_tlb_range(struct vm_area_struct *vma, unsigned long start,
700 unsigned long end)
701{
702 unsigned int cpu;
703 unsigned long flags;
704
705 local_irq_save(flags);
706
707 cpu = smp_processor_id();
708
709 if (likely(asid_mm(vma->vm_mm, cpu) != MM_CTXT_NO_ASID)) {
710 unsigned int asid = hw_pid(vma->vm_mm, cpu);
711
712
713 tlb_entry_erase(start | _PAGE_HW_SZ | asid);
714 }
715
716 local_irq_restore(flags);
717}
718
719#endif
720
721
722
723
724
725void read_decode_mmu_bcr(void)
726{
727 struct cpuinfo_arc_mmu *mmu = &cpuinfo_arc700[smp_processor_id()].mmu;
728 unsigned int tmp;
729 struct bcr_mmu_1_2 {
730#ifdef CONFIG_CPU_BIG_ENDIAN
731 unsigned int ver:8, ways:4, sets:4, u_itlb:8, u_dtlb:8;
732#else
733 unsigned int u_dtlb:8, u_itlb:8, sets:4, ways:4, ver:8;
734#endif
735 } *mmu2;
736
737 struct bcr_mmu_3 {
738#ifdef CONFIG_CPU_BIG_ENDIAN
739 unsigned int ver:8, ways:4, sets:4, res:3, sasid:1, pg_sz:4,
740 u_itlb:4, u_dtlb:4;
741#else
742 unsigned int u_dtlb:4, u_itlb:4, pg_sz:4, sasid:1, res:3, sets:4,
743 ways:4, ver:8;
744#endif
745 } *mmu3;
746
747 struct bcr_mmu_4 {
748#ifdef CONFIG_CPU_BIG_ENDIAN
749 unsigned int ver:8, sasid:1, sz1:4, sz0:4, res:2, pae:1,
750 n_ways:2, n_entry:2, n_super:2, u_itlb:3, u_dtlb:3;
751#else
752
753 unsigned int u_dtlb:3, u_itlb:3, n_super:2, n_entry:2, n_ways:2,
754 pae:1, res:2, sz0:4, sz1:4, sasid:1, ver:8;
755#endif
756 } *mmu4;
757
758 tmp = read_aux_reg(ARC_REG_MMU_BCR);
759 mmu->ver = (tmp >> 24);
760
761 if (mmu->ver <= 2) {
762 mmu2 = (struct bcr_mmu_1_2 *)&tmp;
763 mmu->pg_sz_k = TO_KB(0x2000);
764 mmu->sets = 1 << mmu2->sets;
765 mmu->ways = 1 << mmu2->ways;
766 mmu->u_dtlb = mmu2->u_dtlb;
767 mmu->u_itlb = mmu2->u_itlb;
768 } else if (mmu->ver == 3) {
769 mmu3 = (struct bcr_mmu_3 *)&tmp;
770 mmu->pg_sz_k = 1 << (mmu3->pg_sz - 1);
771 mmu->sets = 1 << mmu3->sets;
772 mmu->ways = 1 << mmu3->ways;
773 mmu->u_dtlb = mmu3->u_dtlb;
774 mmu->u_itlb = mmu3->u_itlb;
775 mmu->sasid = mmu3->sasid;
776 } else {
777 mmu4 = (struct bcr_mmu_4 *)&tmp;
778 mmu->pg_sz_k = 1 << (mmu4->sz0 - 1);
779 mmu->s_pg_sz_m = 1 << (mmu4->sz1 - 11);
780 mmu->sets = 64 << mmu4->n_entry;
781 mmu->ways = mmu4->n_ways * 2;
782 mmu->u_dtlb = mmu4->u_dtlb * 4;
783 mmu->u_itlb = mmu4->u_itlb * 4;
784 mmu->sasid = mmu4->sasid;
785 mmu->pae = mmu4->pae;
786 }
787}
788
789char *arc_mmu_mumbojumbo(int cpu_id, char *buf, int len)
790{
791 int n = 0;
792 struct cpuinfo_arc_mmu *p_mmu = &cpuinfo_arc700[cpu_id].mmu;
793 char super_pg[64] = "";
794
795 if (p_mmu->s_pg_sz_m)
796 scnprintf(super_pg, 64, "%dM Super Page%s, ",
797 p_mmu->s_pg_sz_m,
798 IS_USED_CFG(CONFIG_TRANSPARENT_HUGEPAGE));
799
800 n += scnprintf(buf + n, len - n,
801 "MMU [v%x]\t: %dk PAGE, %sJTLB %d (%dx%d), uDTLB %d, uITLB %d %s%s\n",
802 p_mmu->ver, p_mmu->pg_sz_k, super_pg,
803 p_mmu->sets * p_mmu->ways, p_mmu->sets, p_mmu->ways,
804 p_mmu->u_dtlb, p_mmu->u_itlb,
805 IS_AVAIL2(p_mmu->pae, "PAE40 ", CONFIG_ARC_HAS_PAE40));
806
807 return buf;
808}
809
810void arc_mmu_init(void)
811{
812 char str[256];
813 struct cpuinfo_arc_mmu *mmu = &cpuinfo_arc700[smp_processor_id()].mmu;
814
815 printk(arc_mmu_mumbojumbo(0, str, sizeof(str)));
816
817
818
819
820
821
822
823
824
825 if (mmu->ver != CONFIG_ARC_MMU_VER) {
826 panic("MMU ver %d doesn't match kernel built for %d...\n",
827 mmu->ver, CONFIG_ARC_MMU_VER);
828 }
829
830 if (mmu->pg_sz_k != TO_KB(PAGE_SIZE))
831 panic("MMU pg size != PAGE_SIZE (%luk)\n", TO_KB(PAGE_SIZE));
832
833 if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) &&
834 mmu->s_pg_sz_m != TO_MB(HPAGE_PMD_SIZE))
835 panic("MMU Super pg size != Linux HPAGE_PMD_SIZE (%luM)\n",
836 (unsigned long)TO_MB(HPAGE_PMD_SIZE));
837
838 if (IS_ENABLED(CONFIG_ARC_HAS_PAE40) && !mmu->pae)
839 panic("Hardware doesn't support PAE40\n");
840
841
842 write_aux_reg(ARC_REG_PID, MMU_ENABLE);
843
844
845#ifndef CONFIG_SMP
846
847 write_aux_reg(ARC_REG_SCRATCH_DATA0, swapper_pg_dir);
848#endif
849}
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866#define SET_WAY_TO_IDX(mmu, set, way) ((set) * mmu->ways + (way))
867
868
869
870
871
872
873
874
875
876volatile int dup_pd_silent;
877
878void do_tlb_overlap_fault(unsigned long cause, unsigned long address,
879 struct pt_regs *regs)
880{
881 struct cpuinfo_arc_mmu *mmu = &cpuinfo_arc700[smp_processor_id()].mmu;
882 unsigned int pd0[mmu->ways];
883 unsigned long flags;
884 int set;
885
886 local_irq_save(flags);
887
888
889 write_aux_reg(ARC_REG_PID, MMU_ENABLE | read_aux_reg(ARC_REG_PID));
890
891
892 for (set = 0; set < mmu->sets; set++) {
893
894 int is_valid, way;
895
896
897 for (way = 0, is_valid = 0; way < mmu->ways; way++) {
898 write_aux_reg(ARC_REG_TLBINDEX,
899 SET_WAY_TO_IDX(mmu, set, way));
900 write_aux_reg(ARC_REG_TLBCOMMAND, TLBRead);
901 pd0[way] = read_aux_reg(ARC_REG_TLBPD0);
902 is_valid |= pd0[way] & _PAGE_PRESENT;
903 pd0[way] &= PAGE_MASK;
904 }
905
906
907 if (!is_valid)
908 continue;
909
910
911 for (way = 0; way < mmu->ways - 1; way++) {
912
913 int n;
914
915 if (!pd0[way])
916 continue;
917
918 for (n = way + 1; n < mmu->ways; n++) {
919 if (pd0[way] != pd0[n])
920 continue;
921
922 if (!dup_pd_silent)
923 pr_info("Dup TLB PD0 %08x @ set %d ways %d,%d\n",
924 pd0[way], set, way, n);
925
926
927
928
929
930 pd0[way] = 0;
931 write_aux_reg(ARC_REG_TLBINDEX,
932 SET_WAY_TO_IDX(mmu, set, way));
933 __tlb_entry_erase();
934 }
935 }
936 }
937
938 local_irq_restore(flags);
939}
940
941
942
943
944
945
946#ifdef CONFIG_ARC_DBG_TLB_PARANOIA
947
948
949
950
951
952void print_asid_mismatch(int mm_asid, int mmu_asid, int is_fast_path)
953{
954 pr_emerg("ASID Mismatch in %s Path Handler: sw-pid=0x%x hw-pid=0x%x\n",
955 is_fast_path ? "Fast" : "Slow", mm_asid, mmu_asid);
956
957 __asm__ __volatile__("flag 1");
958}
959
960void tlb_paranoid_check(unsigned int mm_asid, unsigned long addr)
961{
962 unsigned int mmu_asid;
963
964 mmu_asid = read_aux_reg(ARC_REG_PID) & 0xff;
965
966
967
968
969
970
971 if (addr < 0x70000000 &&
972 ((mm_asid == MM_CTXT_NO_ASID) ||
973 (mmu_asid != (mm_asid & MM_CTXT_ASID_MASK))))
974 print_asid_mismatch(mm_asid, mmu_asid, 0);
975}
976#endif
977