1
2
3
4
5
6
7
8
9
10#include <linux/kernel.h>
11#include <linux/backing-dev.h>
12#include <linux/gfp.h>
13#include <linux/mm.h>
14#include <linux/swap.h>
15#include <linux/module.h>
16#include <linux/pagemap.h>
17#include <linux/highmem.h>
18#include <linux/pagevec.h>
19#include <linux/task_io_accounting_ops.h>
20#include <linux/buffer_head.h>
21
22#include "internal.h"
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39void do_invalidatepage(struct page *page, unsigned long offset)
40{
41 void (*invalidatepage)(struct page *, unsigned long);
42 invalidatepage = page->mapping->a_ops->invalidatepage;
43#ifdef CONFIG_BLOCK
44 if (!invalidatepage)
45 invalidatepage = block_invalidatepage;
46#endif
47 if (invalidatepage)
48 (*invalidatepage)(page, offset);
49}
50
51static inline void truncate_partial_page(struct page *page, unsigned partial)
52{
53 zero_user_segment(page, partial, PAGE_CACHE_SIZE);
54 if (page_has_private(page))
55 do_invalidatepage(page, partial);
56}
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72void cancel_dirty_page(struct page *page, unsigned int account_size)
73{
74 if (TestClearPageDirty(page)) {
75 struct address_space *mapping = page->mapping;
76 if (mapping && mapping_cap_account_dirty(mapping)) {
77 dec_zone_page_state(page, NR_FILE_DIRTY);
78 dec_bdi_stat(mapping->backing_dev_info,
79 BDI_RECLAIMABLE);
80 if (account_size)
81 task_io_account_cancelled_write(account_size);
82 }
83 }
84}
85EXPORT_SYMBOL(cancel_dirty_page);
86
87
88
89
90
91
92
93
94
95
96
97static int
98truncate_complete_page(struct address_space *mapping, struct page *page)
99{
100 if (page->mapping != mapping)
101 return -EIO;
102
103 if (page_has_private(page))
104 do_invalidatepage(page, 0);
105
106 cancel_dirty_page(page, PAGE_CACHE_SIZE);
107
108 clear_page_mlock(page);
109 ClearPageMappedToDisk(page);
110 delete_from_page_cache(page);
111 return 0;
112}
113
114
115
116
117
118
119
120
121
122static int
123invalidate_complete_page(struct address_space *mapping, struct page *page)
124{
125 int ret;
126
127 if (page->mapping != mapping)
128 return 0;
129
130 if (page_has_private(page) && !try_to_release_page(page, 0))
131 return 0;
132
133 clear_page_mlock(page);
134 ret = remove_mapping(mapping, page);
135
136 return ret;
137}
138
139int truncate_inode_page(struct address_space *mapping, struct page *page)
140{
141 if (page_mapped(page)) {
142 unmap_mapping_range(mapping,
143 (loff_t)page->index << PAGE_CACHE_SHIFT,
144 PAGE_CACHE_SIZE, 0);
145 }
146 return truncate_complete_page(mapping, page);
147}
148
149
150
151
152int generic_error_remove_page(struct address_space *mapping, struct page *page)
153{
154 if (!mapping)
155 return -EINVAL;
156
157
158
159
160 if (!S_ISREG(mapping->host->i_mode))
161 return -EIO;
162 return truncate_inode_page(mapping, page);
163}
164EXPORT_SYMBOL(generic_error_remove_page);
165
166
167
168
169
170
171
172int invalidate_inode_page(struct page *page)
173{
174 struct address_space *mapping = page_mapping(page);
175 if (!mapping)
176 return 0;
177 if (PageDirty(page) || PageWriteback(page))
178 return 0;
179 if (page_mapped(page))
180 return 0;
181 return invalidate_complete_page(mapping, page);
182}
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207void truncate_inode_pages_range(struct address_space *mapping,
208 loff_t lstart, loff_t lend)
209{
210 const pgoff_t start = (lstart + PAGE_CACHE_SIZE-1) >> PAGE_CACHE_SHIFT;
211 pgoff_t end;
212 const unsigned partial = lstart & (PAGE_CACHE_SIZE - 1);
213 struct pagevec pvec;
214 pgoff_t next;
215 int i;
216
217 if (mapping->nrpages == 0)
218 return;
219
220 BUG_ON((lend & (PAGE_CACHE_SIZE - 1)) != (PAGE_CACHE_SIZE - 1));
221 end = (lend >> PAGE_CACHE_SHIFT);
222
223 pagevec_init(&pvec, 0);
224 next = start;
225 while (next <= end &&
226 pagevec_lookup(&pvec, mapping, next, PAGEVEC_SIZE)) {
227 mem_cgroup_uncharge_start();
228 for (i = 0; i < pagevec_count(&pvec); i++) {
229 struct page *page = pvec.pages[i];
230 pgoff_t page_index = page->index;
231
232 if (page_index > end) {
233 next = page_index;
234 break;
235 }
236
237 if (page_index > next)
238 next = page_index;
239 next++;
240 if (!trylock_page(page))
241 continue;
242 if (PageWriteback(page)) {
243 unlock_page(page);
244 continue;
245 }
246 truncate_inode_page(mapping, page);
247 unlock_page(page);
248 }
249 pagevec_release(&pvec);
250 mem_cgroup_uncharge_end();
251 cond_resched();
252 }
253
254 if (partial) {
255 struct page *page = find_lock_page(mapping, start - 1);
256 if (page) {
257 wait_on_page_writeback(page);
258 truncate_partial_page(page, partial);
259 unlock_page(page);
260 page_cache_release(page);
261 }
262 }
263
264 next = start;
265 for ( ; ; ) {
266 cond_resched();
267 if (!pagevec_lookup(&pvec, mapping, next, PAGEVEC_SIZE)) {
268 if (next == start)
269 break;
270 next = start;
271 continue;
272 }
273 if (pvec.pages[0]->index > end) {
274 pagevec_release(&pvec);
275 break;
276 }
277 mem_cgroup_uncharge_start();
278 for (i = 0; i < pagevec_count(&pvec); i++) {
279 struct page *page = pvec.pages[i];
280
281 if (page->index > end)
282 break;
283 lock_page(page);
284 wait_on_page_writeback(page);
285 truncate_inode_page(mapping, page);
286 if (page->index > next)
287 next = page->index;
288 next++;
289 unlock_page(page);
290 }
291 pagevec_release(&pvec);
292 mem_cgroup_uncharge_end();
293 }
294}
295EXPORT_SYMBOL(truncate_inode_pages_range);
296
297
298
299
300
301
302
303
304void truncate_inode_pages(struct address_space *mapping, loff_t lstart)
305{
306 truncate_inode_pages_range(mapping, lstart, (loff_t)-1);
307}
308EXPORT_SYMBOL(truncate_inode_pages);
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323unsigned long invalidate_mapping_pages(struct address_space *mapping,
324 pgoff_t start, pgoff_t end)
325{
326 struct pagevec pvec;
327 pgoff_t next = start;
328 unsigned long ret;
329 unsigned long count = 0;
330 int i;
331
332 pagevec_init(&pvec, 0);
333 while (next <= end &&
334 pagevec_lookup(&pvec, mapping, next, PAGEVEC_SIZE)) {
335 mem_cgroup_uncharge_start();
336 for (i = 0; i < pagevec_count(&pvec); i++) {
337 struct page *page = pvec.pages[i];
338 pgoff_t index;
339 int lock_failed;
340
341 lock_failed = !trylock_page(page);
342
343
344
345
346
347
348
349 index = page->index;
350 if (index > next)
351 next = index;
352 next++;
353 if (lock_failed)
354 continue;
355
356 ret = invalidate_inode_page(page);
357 unlock_page(page);
358
359
360
361
362 if (!ret)
363 deactivate_page(page);
364 count += ret;
365 if (next > end)
366 break;
367 }
368 pagevec_release(&pvec);
369 mem_cgroup_uncharge_end();
370 cond_resched();
371 }
372 return count;
373}
374EXPORT_SYMBOL(invalidate_mapping_pages);
375
376
377
378
379
380
381
382
383static int
384invalidate_complete_page2(struct address_space *mapping, struct page *page)
385{
386 if (page->mapping != mapping)
387 return 0;
388
389 if (page_has_private(page) && !try_to_release_page(page, GFP_KERNEL))
390 return 0;
391
392 spin_lock_irq(&mapping->tree_lock);
393 if (PageDirty(page))
394 goto failed;
395
396 clear_page_mlock(page);
397 BUG_ON(page_has_private(page));
398 __delete_from_page_cache(page);
399 spin_unlock_irq(&mapping->tree_lock);
400 mem_cgroup_uncharge_cache_page(page);
401
402 if (mapping->a_ops->freepage)
403 mapping->a_ops->freepage(page);
404
405 page_cache_release(page);
406 return 1;
407failed:
408 spin_unlock_irq(&mapping->tree_lock);
409 return 0;
410}
411
412static int do_launder_page(struct address_space *mapping, struct page *page)
413{
414 if (!PageDirty(page))
415 return 0;
416 if (page->mapping != mapping || mapping->a_ops->launder_page == NULL)
417 return 0;
418 return mapping->a_ops->launder_page(page);
419}
420
421
422
423
424
425
426
427
428
429
430
431
432int invalidate_inode_pages2_range(struct address_space *mapping,
433 pgoff_t start, pgoff_t end)
434{
435 struct pagevec pvec;
436 pgoff_t next;
437 int i;
438 int ret = 0;
439 int ret2 = 0;
440 int did_range_unmap = 0;
441 int wrapped = 0;
442
443 pagevec_init(&pvec, 0);
444 next = start;
445 while (next <= end && !wrapped &&
446 pagevec_lookup(&pvec, mapping, next,
447 min(end - next, (pgoff_t)PAGEVEC_SIZE - 1) + 1)) {
448 mem_cgroup_uncharge_start();
449 for (i = 0; i < pagevec_count(&pvec); i++) {
450 struct page *page = pvec.pages[i];
451 pgoff_t page_index;
452
453 lock_page(page);
454 if (page->mapping != mapping) {
455 unlock_page(page);
456 continue;
457 }
458 page_index = page->index;
459 next = page_index + 1;
460 if (next == 0)
461 wrapped = 1;
462 if (page_index > end) {
463 unlock_page(page);
464 break;
465 }
466 wait_on_page_writeback(page);
467 if (page_mapped(page)) {
468 if (!did_range_unmap) {
469
470
471
472 unmap_mapping_range(mapping,
473 (loff_t)page_index<<PAGE_CACHE_SHIFT,
474 (loff_t)(end - page_index + 1)
475 << PAGE_CACHE_SHIFT,
476 0);
477 did_range_unmap = 1;
478 } else {
479
480
481
482 unmap_mapping_range(mapping,
483 (loff_t)page_index<<PAGE_CACHE_SHIFT,
484 PAGE_CACHE_SIZE, 0);
485 }
486 }
487 BUG_ON(page_mapped(page));
488 ret2 = do_launder_page(mapping, page);
489 if (ret2 == 0) {
490 if (!invalidate_complete_page2(mapping, page))
491 ret2 = -EBUSY;
492 }
493 if (ret2 < 0)
494 ret = ret2;
495 unlock_page(page);
496 }
497 pagevec_release(&pvec);
498 mem_cgroup_uncharge_end();
499 cond_resched();
500 }
501 return ret;
502}
503EXPORT_SYMBOL_GPL(invalidate_inode_pages2_range);
504
505
506
507
508
509
510
511
512
513
514int invalidate_inode_pages2(struct address_space *mapping)
515{
516 return invalidate_inode_pages2_range(mapping, 0, -1);
517}
518EXPORT_SYMBOL_GPL(invalidate_inode_pages2);
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536void truncate_pagecache(struct inode *inode, loff_t old, loff_t new)
537{
538 struct address_space *mapping = inode->i_mapping;
539
540
541
542
543
544
545
546
547
548
549 unmap_mapping_range(mapping, new + PAGE_SIZE - 1, 0, 1);
550 truncate_inode_pages(mapping, new);
551 unmap_mapping_range(mapping, new + PAGE_SIZE - 1, 0, 1);
552}
553EXPORT_SYMBOL(truncate_pagecache);
554
555
556
557
558
559
560
561
562
563
564
565
566
567void truncate_setsize(struct inode *inode, loff_t newsize)
568{
569 loff_t oldsize;
570
571 oldsize = inode->i_size;
572 i_size_write(inode, newsize);
573
574 truncate_pagecache(inode, oldsize, newsize);
575}
576EXPORT_SYMBOL(truncate_setsize);
577
578
579
580
581
582
583
584
585
586int vmtruncate(struct inode *inode, loff_t offset)
587{
588 int error;
589
590 error = inode_newsize_ok(inode, offset);
591 if (error)
592 return error;
593
594 truncate_setsize(inode, offset);
595 if (inode->i_op->truncate)
596 inode->i_op->truncate(inode);
597 return 0;
598}
599EXPORT_SYMBOL(vmtruncate);
600