1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17#ifndef __ASMARM_TLB_H
18#define __ASMARM_TLB_H
19
20#include <asm/cacheflush.h>
21
22#ifndef CONFIG_MMU
23
24#include <linux/pagemap.h>
25
26#define tlb_flush(tlb) ((void) tlb)
27
28#include <asm-generic/tlb.h>
29
30#else
31
32#include <linux/swap.h>
33#include <asm/pgalloc.h>
34#include <asm/tlbflush.h>
35
36#define MMU_GATHER_BUNDLE 8
37
38
39
40
41
42struct mmu_gather {
43 struct mm_struct *mm;
44 unsigned int fullmm;
45 struct vm_area_struct *vma;
46 unsigned long start, end;
47 unsigned long range_start;
48 unsigned long range_end;
49 unsigned int nr;
50 unsigned int max;
51 struct page **pages;
52 struct page *local[MMU_GATHER_BUNDLE];
53};
54
55DECLARE_PER_CPU(struct mmu_gather, mmu_gathers);
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70static inline void tlb_flush(struct mmu_gather *tlb)
71{
72 if (tlb->fullmm || !tlb->vma)
73 flush_tlb_mm(tlb->mm);
74 else if (tlb->range_end > 0) {
75 flush_tlb_range(tlb->vma, tlb->range_start, tlb->range_end);
76 tlb->range_start = TASK_SIZE;
77 tlb->range_end = 0;
78 }
79}
80
81static inline void tlb_add_flush(struct mmu_gather *tlb, unsigned long addr)
82{
83 if (!tlb->fullmm) {
84 if (addr < tlb->range_start)
85 tlb->range_start = addr;
86 if (addr + PAGE_SIZE > tlb->range_end)
87 tlb->range_end = addr + PAGE_SIZE;
88 }
89}
90
91static inline void __tlb_alloc_page(struct mmu_gather *tlb)
92{
93 unsigned long addr = __get_free_pages(GFP_NOWAIT | __GFP_NOWARN, 0);
94
95 if (addr) {
96 tlb->pages = (void *)addr;
97 tlb->max = PAGE_SIZE / sizeof(struct page *);
98 }
99}
100
101static inline void tlb_flush_mmu(struct mmu_gather *tlb)
102{
103 tlb_flush(tlb);
104 free_pages_and_swap_cache(tlb->pages, tlb->nr);
105 tlb->nr = 0;
106 if (tlb->pages == tlb->local)
107 __tlb_alloc_page(tlb);
108}
109
110static inline void
111tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, unsigned long start, unsigned long end)
112{
113 tlb->mm = mm;
114 tlb->fullmm = !(start | (end+1));
115 tlb->start = start;
116 tlb->end = end;
117 tlb->vma = NULL;
118 tlb->max = ARRAY_SIZE(tlb->local);
119 tlb->pages = tlb->local;
120 tlb->nr = 0;
121 __tlb_alloc_page(tlb);
122}
123
124static inline void
125tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end)
126{
127 tlb_flush_mmu(tlb);
128
129
130 check_pgt_cache();
131
132 if (tlb->pages != tlb->local)
133 free_pages((unsigned long)tlb->pages, 0);
134}
135
136
137
138
139static inline void
140tlb_remove_tlb_entry(struct mmu_gather *tlb, pte_t *ptep, unsigned long addr)
141{
142 tlb_add_flush(tlb, addr);
143}
144
145
146
147
148
149
150static inline void
151tlb_start_vma(struct mmu_gather *tlb, struct vm_area_struct *vma)
152{
153 if (!tlb->fullmm) {
154 flush_cache_range(vma, vma->vm_start, vma->vm_end);
155 tlb->vma = vma;
156 tlb->range_start = TASK_SIZE;
157 tlb->range_end = 0;
158 }
159}
160
161static inline void
162tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vma)
163{
164 if (!tlb->fullmm)
165 tlb_flush(tlb);
166}
167
168static inline int __tlb_remove_page(struct mmu_gather *tlb, struct page *page)
169{
170 tlb->pages[tlb->nr++] = page;
171 VM_BUG_ON(tlb->nr > tlb->max);
172 return tlb->max - tlb->nr;
173}
174
175static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page)
176{
177 if (!__tlb_remove_page(tlb, page))
178 tlb_flush_mmu(tlb);
179}
180
181static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte,
182 unsigned long addr)
183{
184 pgtable_page_dtor(pte);
185
186#ifdef CONFIG_ARM_LPAE
187 tlb_add_flush(tlb, addr);
188#else
189
190
191
192
193 addr &= PMD_MASK;
194 tlb_add_flush(tlb, addr + SZ_1M - PAGE_SIZE);
195 tlb_add_flush(tlb, addr + SZ_1M);
196#endif
197
198 tlb_remove_page(tlb, pte);
199}
200
201static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmdp,
202 unsigned long addr)
203{
204#ifdef CONFIG_ARM_LPAE
205 tlb_add_flush(tlb, addr);
206 tlb_remove_page(tlb, virt_to_page(pmdp));
207#endif
208}
209
210static inline void
211tlb_remove_pmd_tlb_entry(struct mmu_gather *tlb, pmd_t *pmdp, unsigned long addr)
212{
213 tlb_add_flush(tlb, addr);
214}
215
216#define pte_free_tlb(tlb, ptep, addr) __pte_free_tlb(tlb, ptep, addr)
217#define pmd_free_tlb(tlb, pmdp, addr) __pmd_free_tlb(tlb, pmdp, addr)
218#define pud_free_tlb(tlb, pudp, addr) pud_free((tlb)->mm, pudp)
219
220#define tlb_migrate_finish(mm) do { } while (0)
221
222#endif
223#endif
224