1#include <linux/gfp.h>
2#include <linux/highmem.h>
3#include <linux/kernel.h>
4#include <linux/mmdebug.h>
5#include <linux/mm_types.h>
6#include <linux/pagemap.h>
7#include <linux/rcupdate.h>
8#include <linux/smp.h>
9#include <linux/swap.h>
10
11#include <asm/pgalloc.h>
12#include <asm/tlb.h>
13
14#ifndef CONFIG_MMU_GATHER_NO_GATHER
15
16static bool tlb_next_batch(struct mmu_gather *tlb)
17{
18 struct mmu_gather_batch *batch;
19
20 batch = tlb->active;
21 if (batch->next) {
22 tlb->active = batch->next;
23 return true;
24 }
25
26 if (tlb->batch_count == MAX_GATHER_BATCH_COUNT)
27 return false;
28
29 batch = (void *)__get_free_pages(GFP_NOWAIT | __GFP_NOWARN, 0);
30 if (!batch)
31 return false;
32
33 tlb->batch_count++;
34 batch->next = NULL;
35 batch->nr = 0;
36 batch->max = MAX_GATHER_BATCH;
37
38 tlb->active->next = batch;
39 tlb->active = batch;
40
41 return true;
42}
43
44static void tlb_batch_pages_flush(struct mmu_gather *tlb)
45{
46 struct mmu_gather_batch *batch;
47
48 for (batch = &tlb->local; batch && batch->nr; batch = batch->next) {
49 free_pages_and_swap_cache(batch->pages, batch->nr);
50 batch->nr = 0;
51 }
52 tlb->active = &tlb->local;
53}
54
55static void tlb_batch_list_free(struct mmu_gather *tlb)
56{
57 struct mmu_gather_batch *batch, *next;
58
59 for (batch = tlb->local.next; batch; batch = next) {
60 next = batch->next;
61 free_pages((unsigned long)batch, 0);
62 }
63 tlb->local.next = NULL;
64}
65
66bool __tlb_remove_page_size(struct mmu_gather *tlb, struct page *page, int page_size)
67{
68 struct mmu_gather_batch *batch;
69
70 VM_BUG_ON(!tlb->end);
71
72#ifdef CONFIG_MMU_GATHER_PAGE_SIZE
73 VM_WARN_ON(tlb->page_size != page_size);
74#endif
75
76 batch = tlb->active;
77
78
79
80
81 batch->pages[batch->nr++] = page;
82 if (batch->nr == batch->max) {
83 if (!tlb_next_batch(tlb))
84 return true;
85 batch = tlb->active;
86 }
87 VM_BUG_ON_PAGE(batch->nr > batch->max, page);
88
89 return false;
90}
91
92#endif
93
94#ifdef CONFIG_MMU_GATHER_TABLE_FREE
95
96static void __tlb_remove_table_free(struct mmu_table_batch *batch)
97{
98 int i;
99
100 for (i = 0; i < batch->nr; i++)
101 __tlb_remove_table(batch->tables[i]);
102
103 free_page((unsigned long)batch);
104}
105
106#ifdef CONFIG_MMU_GATHER_RCU_TABLE_FREE
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137static void tlb_remove_table_smp_sync(void *arg)
138{
139
140}
141
142static void tlb_remove_table_sync_one(void)
143{
144
145
146
147
148
149
150
151 smp_call_function(tlb_remove_table_smp_sync, NULL, 1);
152}
153
154static void tlb_remove_table_rcu(struct rcu_head *head)
155{
156 __tlb_remove_table_free(container_of(head, struct mmu_table_batch, rcu));
157}
158
159static void tlb_remove_table_free(struct mmu_table_batch *batch)
160{
161 call_rcu(&batch->rcu, tlb_remove_table_rcu);
162}
163
164#else
165
166static void tlb_remove_table_sync_one(void) { }
167
168static void tlb_remove_table_free(struct mmu_table_batch *batch)
169{
170 __tlb_remove_table_free(batch);
171}
172
173#endif
174
175
176
177
178static inline void tlb_table_invalidate(struct mmu_gather *tlb)
179{
180 if (tlb_needs_table_invalidate()) {
181
182
183
184
185
186 tlb_flush_mmu_tlbonly(tlb);
187 }
188}
189
190static void tlb_remove_table_one(void *table)
191{
192 tlb_remove_table_sync_one();
193 __tlb_remove_table(table);
194}
195
196static void tlb_table_flush(struct mmu_gather *tlb)
197{
198 struct mmu_table_batch **batch = &tlb->batch;
199
200 if (*batch) {
201 tlb_table_invalidate(tlb);
202 tlb_remove_table_free(*batch);
203 *batch = NULL;
204 }
205}
206
207void tlb_remove_table(struct mmu_gather *tlb, void *table)
208{
209 struct mmu_table_batch **batch = &tlb->batch;
210
211 if (*batch == NULL) {
212 *batch = (struct mmu_table_batch *)__get_free_page(GFP_NOWAIT | __GFP_NOWARN);
213 if (*batch == NULL) {
214 tlb_table_invalidate(tlb);
215 tlb_remove_table_one(table);
216 return;
217 }
218 (*batch)->nr = 0;
219 }
220
221 (*batch)->tables[(*batch)->nr++] = table;
222 if ((*batch)->nr == MAX_TABLE_BATCH)
223 tlb_table_flush(tlb);
224}
225
226static inline void tlb_table_init(struct mmu_gather *tlb)
227{
228 tlb->batch = NULL;
229}
230
231#else
232
233static inline void tlb_table_flush(struct mmu_gather *tlb) { }
234static inline void tlb_table_init(struct mmu_gather *tlb) { }
235
236#endif
237
238static void tlb_flush_mmu_free(struct mmu_gather *tlb)
239{
240 tlb_table_flush(tlb);
241#ifndef CONFIG_MMU_GATHER_NO_GATHER
242 tlb_batch_pages_flush(tlb);
243#endif
244}
245
246void tlb_flush_mmu(struct mmu_gather *tlb)
247{
248 tlb_flush_mmu_tlbonly(tlb);
249 tlb_flush_mmu_free(tlb);
250}
251
252static void __tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm,
253 bool fullmm)
254{
255 tlb->mm = mm;
256 tlb->fullmm = fullmm;
257
258#ifndef CONFIG_MMU_GATHER_NO_GATHER
259 tlb->need_flush_all = 0;
260 tlb->local.next = NULL;
261 tlb->local.nr = 0;
262 tlb->local.max = ARRAY_SIZE(tlb->__pages);
263 tlb->active = &tlb->local;
264 tlb->batch_count = 0;
265#endif
266
267 tlb_table_init(tlb);
268#ifdef CONFIG_MMU_GATHER_PAGE_SIZE
269 tlb->page_size = 0;
270#endif
271
272 __tlb_reset_range(tlb);
273 inc_tlb_flush_pending(tlb->mm);
274}
275
276
277
278
279
280
281
282
283
284void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm)
285{
286 __tlb_gather_mmu(tlb, mm, false);
287}
288
289
290
291
292
293
294
295
296
297
298
299
300void tlb_gather_mmu_fullmm(struct mmu_gather *tlb, struct mm_struct *mm)
301{
302 __tlb_gather_mmu(tlb, mm, true);
303}
304
305
306
307
308
309
310
311
312void tlb_finish_mmu(struct mmu_gather *tlb)
313{
314
315
316
317
318
319
320
321
322
323
324
325
326 if (mm_tlb_flush_nested(tlb->mm)) {
327
328
329
330
331
332
333
334
335 tlb->fullmm = 1;
336 __tlb_reset_range(tlb);
337 tlb->freed_tables = 1;
338 }
339
340 tlb_flush_mmu(tlb);
341
342#ifndef CONFIG_MMU_GATHER_NO_GATHER
343 tlb_batch_list_free(tlb);
344#endif
345 dec_tlb_flush_pending(tlb->mm);
346}
347