1
2
3
4
5
6
7
8
9
10
11
12
13
14#include <linux/mman.h>
15#include <linux/swap.h>
16#include <linux/swapops.h>
17#include <linux/security.h>
18#include <linux/module.h>
19#include <linux/debugfs.h>
20#include <linux/frontswap.h>
21#include <linux/swapfile.h>
22
23
24
25
26
27
28
29static struct frontswap_ops *frontswap_ops __read_mostly;
30
31#define for_each_frontswap_ops(ops) \
32 for ((ops) = frontswap_ops; (ops); (ops) = (ops)->next)
33
34
35
36
37
38
39
40
41
42static bool frontswap_writethrough_enabled __read_mostly;
43
44
45
46
47
48
49static bool frontswap_tmem_exclusive_gets_enabled __read_mostly;
50
51#ifdef CONFIG_DEBUG_FS
52
53
54
55
56
57static u64 frontswap_loads;
58static u64 frontswap_succ_stores;
59static u64 frontswap_failed_stores;
60static u64 frontswap_invalidates;
61
62static inline void inc_frontswap_loads(void) {
63 frontswap_loads++;
64}
65static inline void inc_frontswap_succ_stores(void) {
66 frontswap_succ_stores++;
67}
68static inline void inc_frontswap_failed_stores(void) {
69 frontswap_failed_stores++;
70}
71static inline void inc_frontswap_invalidates(void) {
72 frontswap_invalidates++;
73}
74#else
75static inline void inc_frontswap_loads(void) { }
76static inline void inc_frontswap_succ_stores(void) { }
77static inline void inc_frontswap_failed_stores(void) { }
78static inline void inc_frontswap_invalidates(void) { }
79#endif
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112void frontswap_register_ops(struct frontswap_ops *ops)
113{
114 DECLARE_BITMAP(a, MAX_SWAPFILES);
115 DECLARE_BITMAP(b, MAX_SWAPFILES);
116 struct swap_info_struct *si;
117 unsigned int i;
118
119 bitmap_zero(a, MAX_SWAPFILES);
120 bitmap_zero(b, MAX_SWAPFILES);
121
122 spin_lock(&swap_lock);
123 plist_for_each_entry(si, &swap_active_head, list) {
124 if (!WARN_ON(!si->frontswap_map))
125 set_bit(si->type, a);
126 }
127 spin_unlock(&swap_lock);
128
129
130 for_each_set_bit(i, a, MAX_SWAPFILES)
131 ops->init(i);
132
133
134
135
136
137
138 do {
139 ops->next = frontswap_ops;
140 } while (cmpxchg(&frontswap_ops, ops->next, ops) != ops->next);
141
142 spin_lock(&swap_lock);
143 plist_for_each_entry(si, &swap_active_head, list) {
144 if (si->frontswap_map)
145 set_bit(si->type, b);
146 }
147 spin_unlock(&swap_lock);
148
149
150
151
152
153
154
155 if (unlikely(!bitmap_equal(a, b, MAX_SWAPFILES))) {
156 for (i = 0; i < MAX_SWAPFILES; i++) {
157 if (!test_bit(i, a) && test_bit(i, b))
158 ops->init(i);
159 else if (test_bit(i, a) && !test_bit(i, b))
160 ops->invalidate_area(i);
161 }
162 }
163}
164EXPORT_SYMBOL(frontswap_register_ops);
165
166
167
168
169void frontswap_writethrough(bool enable)
170{
171 frontswap_writethrough_enabled = enable;
172}
173EXPORT_SYMBOL(frontswap_writethrough);
174
175
176
177
178void frontswap_tmem_exclusive_gets(bool enable)
179{
180 frontswap_tmem_exclusive_gets_enabled = enable;
181}
182EXPORT_SYMBOL(frontswap_tmem_exclusive_gets);
183
184
185
186
187void __frontswap_init(unsigned type, unsigned long *map)
188{
189 struct swap_info_struct *sis = swap_info[type];
190 struct frontswap_ops *ops;
191
192 BUG_ON(sis == NULL);
193
194
195
196
197
198 if (WARN_ON(!map))
199 return;
200
201
202
203
204
205 frontswap_map_set(sis, map);
206
207 for_each_frontswap_ops(ops)
208 ops->init(type);
209}
210EXPORT_SYMBOL(__frontswap_init);
211
212bool __frontswap_test(struct swap_info_struct *sis,
213 pgoff_t offset)
214{
215 if (sis->frontswap_map)
216 return test_bit(offset, sis->frontswap_map);
217 return false;
218}
219EXPORT_SYMBOL(__frontswap_test);
220
221static inline void __frontswap_set(struct swap_info_struct *sis,
222 pgoff_t offset)
223{
224 set_bit(offset, sis->frontswap_map);
225 atomic_inc(&sis->frontswap_pages);
226}
227
228static inline void __frontswap_clear(struct swap_info_struct *sis,
229 pgoff_t offset)
230{
231 clear_bit(offset, sis->frontswap_map);
232 atomic_dec(&sis->frontswap_pages);
233}
234
235
236
237
238
239
240
241
242int __frontswap_store(struct page *page)
243{
244 int ret = -1;
245 swp_entry_t entry = { .val = page_private(page), };
246 int type = swp_type(entry);
247 struct swap_info_struct *sis = swap_info[type];
248 pgoff_t offset = swp_offset(entry);
249 struct frontswap_ops *ops;
250
251
252
253
254
255 if (!frontswap_ops)
256 return -1;
257
258 BUG_ON(!PageLocked(page));
259 BUG_ON(sis == NULL);
260
261
262
263
264
265
266
267 if (__frontswap_test(sis, offset)) {
268 __frontswap_clear(sis, offset);
269 for_each_frontswap_ops(ops)
270 ops->invalidate_page(type, offset);
271 }
272
273
274 for_each_frontswap_ops(ops) {
275 ret = ops->store(type, offset, page);
276 if (!ret)
277 break;
278 }
279 if (ret == 0) {
280 __frontswap_set(sis, offset);
281 inc_frontswap_succ_stores();
282 } else {
283 inc_frontswap_failed_stores();
284 }
285 if (frontswap_writethrough_enabled)
286
287 ret = -1;
288 return ret;
289}
290EXPORT_SYMBOL(__frontswap_store);
291
292
293
294
295
296
297int __frontswap_load(struct page *page)
298{
299 int ret = -1;
300 swp_entry_t entry = { .val = page_private(page), };
301 int type = swp_type(entry);
302 struct swap_info_struct *sis = swap_info[type];
303 pgoff_t offset = swp_offset(entry);
304 struct frontswap_ops *ops;
305
306 if (!frontswap_ops)
307 return -1;
308
309 BUG_ON(!PageLocked(page));
310 BUG_ON(sis == NULL);
311 if (!__frontswap_test(sis, offset))
312 return -1;
313
314
315 for_each_frontswap_ops(ops) {
316 ret = ops->load(type, offset, page);
317 if (!ret)
318 break;
319 }
320 if (ret == 0) {
321 inc_frontswap_loads();
322 if (frontswap_tmem_exclusive_gets_enabled) {
323 SetPageDirty(page);
324 __frontswap_clear(sis, offset);
325 }
326 }
327 return ret;
328}
329EXPORT_SYMBOL(__frontswap_load);
330
331
332
333
334
335void __frontswap_invalidate_page(unsigned type, pgoff_t offset)
336{
337 struct swap_info_struct *sis = swap_info[type];
338 struct frontswap_ops *ops;
339
340 if (!frontswap_ops)
341 return;
342
343 BUG_ON(sis == NULL);
344 if (!__frontswap_test(sis, offset))
345 return;
346
347 for_each_frontswap_ops(ops)
348 ops->invalidate_page(type, offset);
349 __frontswap_clear(sis, offset);
350 inc_frontswap_invalidates();
351}
352EXPORT_SYMBOL(__frontswap_invalidate_page);
353
354
355
356
357
358void __frontswap_invalidate_area(unsigned type)
359{
360 struct swap_info_struct *sis = swap_info[type];
361 struct frontswap_ops *ops;
362
363 if (!frontswap_ops)
364 return;
365
366 BUG_ON(sis == NULL);
367 if (sis->frontswap_map == NULL)
368 return;
369
370 for_each_frontswap_ops(ops)
371 ops->invalidate_area(type);
372 atomic_set(&sis->frontswap_pages, 0);
373 bitmap_zero(sis->frontswap_map, sis->max);
374}
375EXPORT_SYMBOL(__frontswap_invalidate_area);
376
377static unsigned long __frontswap_curr_pages(void)
378{
379 unsigned long totalpages = 0;
380 struct swap_info_struct *si = NULL;
381
382 assert_spin_locked(&swap_lock);
383 plist_for_each_entry(si, &swap_active_head, list)
384 totalpages += atomic_read(&si->frontswap_pages);
385 return totalpages;
386}
387
388static int __frontswap_unuse_pages(unsigned long total, unsigned long *unused,
389 int *swapid)
390{
391 int ret = -EINVAL;
392 struct swap_info_struct *si = NULL;
393 int si_frontswap_pages;
394 unsigned long total_pages_to_unuse = total;
395 unsigned long pages = 0, pages_to_unuse = 0;
396
397 assert_spin_locked(&swap_lock);
398 plist_for_each_entry(si, &swap_active_head, list) {
399 si_frontswap_pages = atomic_read(&si->frontswap_pages);
400 if (total_pages_to_unuse < si_frontswap_pages) {
401 pages = pages_to_unuse = total_pages_to_unuse;
402 } else {
403 pages = si_frontswap_pages;
404 pages_to_unuse = 0;
405 }
406
407 if (security_vm_enough_memory_mm(current->mm, pages)) {
408 ret = -ENOMEM;
409 continue;
410 }
411 vm_unacct_memory(pages);
412 *unused = pages_to_unuse;
413 *swapid = si->type;
414 ret = 0;
415 break;
416 }
417
418 return ret;
419}
420
421
422
423
424
425
426static int __frontswap_shrink(unsigned long target_pages,
427 unsigned long *pages_to_unuse,
428 int *type)
429{
430 unsigned long total_pages = 0, total_pages_to_unuse;
431
432 assert_spin_locked(&swap_lock);
433
434 total_pages = __frontswap_curr_pages();
435 if (total_pages <= target_pages) {
436
437 *pages_to_unuse = 0;
438 return 1;
439 }
440 total_pages_to_unuse = total_pages - target_pages;
441 return __frontswap_unuse_pages(total_pages_to_unuse, pages_to_unuse, type);
442}
443
444
445
446
447
448
449
450
451
452void frontswap_shrink(unsigned long target_pages)
453{
454 unsigned long pages_to_unuse = 0;
455 int uninitialized_var(type), ret;
456
457
458
459
460
461
462 spin_lock(&swap_lock);
463 ret = __frontswap_shrink(target_pages, &pages_to_unuse, &type);
464 spin_unlock(&swap_lock);
465 if (ret == 0)
466 try_to_unuse(type, true, pages_to_unuse);
467 return;
468}
469EXPORT_SYMBOL(frontswap_shrink);
470
471
472
473
474
475
476unsigned long frontswap_curr_pages(void)
477{
478 unsigned long totalpages = 0;
479
480 spin_lock(&swap_lock);
481 totalpages = __frontswap_curr_pages();
482 spin_unlock(&swap_lock);
483
484 return totalpages;
485}
486EXPORT_SYMBOL(frontswap_curr_pages);
487
488static int __init init_frontswap(void)
489{
490#ifdef CONFIG_DEBUG_FS
491 struct dentry *root = debugfs_create_dir("frontswap", NULL);
492 if (root == NULL)
493 return -ENXIO;
494 debugfs_create_u64("loads", S_IRUGO, root, &frontswap_loads);
495 debugfs_create_u64("succ_stores", S_IRUGO, root, &frontswap_succ_stores);
496 debugfs_create_u64("failed_stores", S_IRUGO, root,
497 &frontswap_failed_stores);
498 debugfs_create_u64("invalidates", S_IRUGO,
499 root, &frontswap_invalidates);
500#endif
501 return 0;
502}
503
504module_init(init_frontswap);
505