1
2
3
4
5
6
7
8
9
10
11
12
13
14#include <linux/kernel.h>
15#include <linux/module.h>
16#include <linux/spinlock.h>
17#include <linux/fs.h>
18#include <linux/mm.h>
19#include <linux/swap.h>
20#include <linux/slab.h>
21#include <linux/pagemap.h>
22#include <linux/writeback.h>
23#include <linux/init.h>
24#include <linux/backing-dev.h>
25#include <linux/task_io_accounting_ops.h>
26#include <linux/blkdev.h>
27#include <linux/mpage.h>
28#include <linux/rmap.h>
29#include <linux/percpu.h>
30#include <linux/notifier.h>
31#include <linux/smp.h>
32#include <linux/sysctl.h>
33#include <linux/cpu.h>
34#include <linux/syscalls.h>
35#include <linux/buffer_head.h>
36#include <linux/pagevec.h>
37#include <trace/events/writeback.h>
38
39
40
41
42
43static long ratelimit_pages = 32;
44
45
46
47
48
49
50
51static inline long sync_writeback_pages(unsigned long dirtied)
52{
53 if (dirtied < ratelimit_pages)
54 dirtied = ratelimit_pages;
55
56 return dirtied + dirtied / 2;
57}
58
59
60
61
62
63
64int dirty_background_ratio = 10;
65
66
67
68
69
70unsigned long dirty_background_bytes;
71
72
73
74
75
76int vm_highmem_is_dirtyable;
77
78
79
80
81int vm_dirty_ratio = 20;
82
83
84
85
86
87unsigned long vm_dirty_bytes;
88
89
90
91
92unsigned int dirty_writeback_interval = 5 * 100;
93
94
95
96
97unsigned int dirty_expire_interval = 30 * 100;
98
99
100
101
102int block_dump;
103
104
105
106
107
108int laptop_mode;
109
110EXPORT_SYMBOL(laptop_mode);
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131static struct prop_descriptor vm_completions;
132static struct prop_descriptor vm_dirties;
133
134
135
136
137
138
139static int calc_period_shift(void)
140{
141 unsigned long dirty_total;
142
143 if (vm_dirty_bytes)
144 dirty_total = vm_dirty_bytes / PAGE_SIZE;
145 else
146 dirty_total = (vm_dirty_ratio * determine_dirtyable_memory()) /
147 100;
148 return 2 + ilog2(dirty_total - 1);
149}
150
151
152
153
154static void update_completion_period(void)
155{
156 int shift = calc_period_shift();
157 prop_change_shift(&vm_completions, shift);
158 prop_change_shift(&vm_dirties, shift);
159}
160
161int dirty_background_ratio_handler(struct ctl_table *table, int write,
162 void __user *buffer, size_t *lenp,
163 loff_t *ppos)
164{
165 int ret;
166
167 ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
168 if (ret == 0 && write)
169 dirty_background_bytes = 0;
170 return ret;
171}
172
173int dirty_background_bytes_handler(struct ctl_table *table, int write,
174 void __user *buffer, size_t *lenp,
175 loff_t *ppos)
176{
177 int ret;
178
179 ret = proc_doulongvec_minmax(table, write, buffer, lenp, ppos);
180 if (ret == 0 && write)
181 dirty_background_ratio = 0;
182 return ret;
183}
184
185int dirty_ratio_handler(struct ctl_table *table, int write,
186 void __user *buffer, size_t *lenp,
187 loff_t *ppos)
188{
189 int old_ratio = vm_dirty_ratio;
190 int ret;
191
192 ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
193 if (ret == 0 && write && vm_dirty_ratio != old_ratio) {
194 update_completion_period();
195 vm_dirty_bytes = 0;
196 }
197 return ret;
198}
199
200
201int dirty_bytes_handler(struct ctl_table *table, int write,
202 void __user *buffer, size_t *lenp,
203 loff_t *ppos)
204{
205 unsigned long old_bytes = vm_dirty_bytes;
206 int ret;
207
208 ret = proc_doulongvec_minmax(table, write, buffer, lenp, ppos);
209 if (ret == 0 && write && vm_dirty_bytes != old_bytes) {
210 update_completion_period();
211 vm_dirty_ratio = 0;
212 }
213 return ret;
214}
215
216
217
218
219
220static inline void __bdi_writeout_inc(struct backing_dev_info *bdi)
221{
222 __prop_inc_percpu_max(&vm_completions, &bdi->completions,
223 bdi->max_prop_frac);
224}
225
226void bdi_writeout_inc(struct backing_dev_info *bdi)
227{
228 unsigned long flags;
229
230 local_irq_save(flags);
231 __bdi_writeout_inc(bdi);
232 local_irq_restore(flags);
233}
234EXPORT_SYMBOL_GPL(bdi_writeout_inc);
235
236void task_dirty_inc(struct task_struct *tsk)
237{
238 prop_inc_single(&vm_dirties, &tsk->dirties);
239}
240
241
242
243
244static void bdi_writeout_fraction(struct backing_dev_info *bdi,
245 long *numerator, long *denominator)
246{
247 if (bdi_cap_writeback_dirty(bdi)) {
248 prop_fraction_percpu(&vm_completions, &bdi->completions,
249 numerator, denominator);
250 } else {
251 *numerator = 0;
252 *denominator = 1;
253 }
254}
255
256static inline void task_dirties_fraction(struct task_struct *tsk,
257 long *numerator, long *denominator)
258{
259 prop_fraction_single(&vm_dirties, &tsk->dirties,
260 numerator, denominator);
261}
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277static unsigned long task_dirty_limit(struct task_struct *tsk,
278 unsigned long bdi_dirty)
279{
280 long numerator, denominator;
281 unsigned long dirty = bdi_dirty;
282 u64 inv = dirty >> 3;
283
284 task_dirties_fraction(tsk, &numerator, &denominator);
285 inv *= numerator;
286 do_div(inv, denominator);
287
288 dirty -= inv;
289
290 return max(dirty, bdi_dirty/2);
291}
292
293
294
295
296static unsigned int bdi_min_ratio;
297
298int bdi_set_min_ratio(struct backing_dev_info *bdi, unsigned int min_ratio)
299{
300 int ret = 0;
301
302 spin_lock_bh(&bdi_lock);
303 if (min_ratio > bdi->max_ratio) {
304 ret = -EINVAL;
305 } else {
306 min_ratio -= bdi->min_ratio;
307 if (bdi_min_ratio + min_ratio < 100) {
308 bdi_min_ratio += min_ratio;
309 bdi->min_ratio += min_ratio;
310 } else {
311 ret = -EINVAL;
312 }
313 }
314 spin_unlock_bh(&bdi_lock);
315
316 return ret;
317}
318
319int bdi_set_max_ratio(struct backing_dev_info *bdi, unsigned max_ratio)
320{
321 int ret = 0;
322
323 if (max_ratio > 100)
324 return -EINVAL;
325
326 spin_lock_bh(&bdi_lock);
327 if (bdi->min_ratio > max_ratio) {
328 ret = -EINVAL;
329 } else {
330 bdi->max_ratio = max_ratio;
331 bdi->max_prop_frac = (PROP_FRAC_BASE * max_ratio) / 100;
332 }
333 spin_unlock_bh(&bdi_lock);
334
335 return ret;
336}
337EXPORT_SYMBOL(bdi_set_max_ratio);
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357static unsigned long highmem_dirtyable_memory(unsigned long total)
358{
359#ifdef CONFIG_HIGHMEM
360 int node;
361 unsigned long x = 0;
362
363 for_each_node_state(node, N_HIGH_MEMORY) {
364 struct zone *z =
365 &NODE_DATA(node)->node_zones[ZONE_HIGHMEM];
366
367 x += zone_page_state(z, NR_FREE_PAGES) +
368 zone_reclaimable_pages(z);
369 }
370
371
372
373
374
375
376 return min(x, total);
377#else
378 return 0;
379#endif
380}
381
382
383
384
385
386
387
388unsigned long determine_dirtyable_memory(void)
389{
390 unsigned long x;
391
392 x = global_page_state(NR_FREE_PAGES) + global_reclaimable_pages();
393
394 if (!vm_highmem_is_dirtyable)
395 x -= highmem_dirtyable_memory(x);
396
397 return x + 1;
398}
399
400
401
402
403
404
405
406
407
408
409void global_dirty_limits(unsigned long *pbackground, unsigned long *pdirty)
410{
411 unsigned long background;
412 unsigned long dirty;
413 unsigned long uninitialized_var(available_memory);
414 struct task_struct *tsk;
415
416 if (!vm_dirty_bytes || !dirty_background_bytes)
417 available_memory = determine_dirtyable_memory();
418
419 if (vm_dirty_bytes)
420 dirty = DIV_ROUND_UP(vm_dirty_bytes, PAGE_SIZE);
421 else
422 dirty = (vm_dirty_ratio * available_memory) / 100;
423
424 if (dirty_background_bytes)
425 background = DIV_ROUND_UP(dirty_background_bytes, PAGE_SIZE);
426 else
427 background = (dirty_background_ratio * available_memory) / 100;
428
429 if (background >= dirty)
430 background = dirty / 2;
431 tsk = current;
432 if (tsk->flags & PF_LESS_THROTTLE || rt_task(tsk)) {
433 background += background / 4;
434 dirty += dirty / 4;
435 }
436 *pbackground = background;
437 *pdirty = dirty;
438}
439
440
441
442
443
444
445
446
447
448
449
450unsigned long bdi_dirty_limit(struct backing_dev_info *bdi, unsigned long dirty)
451{
452 u64 bdi_dirty;
453 long numerator, denominator;
454
455
456
457
458 bdi_writeout_fraction(bdi, &numerator, &denominator);
459
460 bdi_dirty = (dirty * (100 - bdi_min_ratio)) / 100;
461 bdi_dirty *= numerator;
462 do_div(bdi_dirty, denominator);
463
464 bdi_dirty += (dirty * bdi->min_ratio) / 100;
465 if (bdi_dirty > (dirty * bdi->max_ratio) / 100)
466 bdi_dirty = dirty * bdi->max_ratio / 100;
467
468 return bdi_dirty;
469}
470
471
472
473
474
475
476
477
478static void balance_dirty_pages(struct address_space *mapping,
479 unsigned long write_chunk)
480{
481 long nr_reclaimable, bdi_nr_reclaimable;
482 long nr_writeback, bdi_nr_writeback;
483 unsigned long background_thresh;
484 unsigned long dirty_thresh;
485 unsigned long bdi_thresh;
486 unsigned long pages_written = 0;
487 unsigned long pause = 1;
488 bool dirty_exceeded = false;
489 struct backing_dev_info *bdi = mapping->backing_dev_info;
490
491 for (;;) {
492 struct writeback_control wbc = {
493 .sync_mode = WB_SYNC_NONE,
494 .older_than_this = NULL,
495 .nr_to_write = write_chunk,
496 .range_cyclic = 1,
497 };
498
499 nr_reclaimable = global_page_state(NR_FILE_DIRTY) +
500 global_page_state(NR_UNSTABLE_NFS);
501 nr_writeback = global_page_state(NR_WRITEBACK);
502
503 global_dirty_limits(&background_thresh, &dirty_thresh);
504
505
506
507
508
509
510 if (nr_reclaimable + nr_writeback <=
511 (background_thresh + dirty_thresh) / 2)
512 break;
513
514 bdi_thresh = bdi_dirty_limit(bdi, dirty_thresh);
515 bdi_thresh = task_dirty_limit(current, bdi_thresh);
516
517
518
519
520
521
522
523
524
525
526
527 if (bdi_thresh < 2*bdi_stat_error(bdi)) {
528 bdi_nr_reclaimable = bdi_stat_sum(bdi, BDI_RECLAIMABLE);
529 bdi_nr_writeback = bdi_stat_sum(bdi, BDI_WRITEBACK);
530 } else {
531 bdi_nr_reclaimable = bdi_stat(bdi, BDI_RECLAIMABLE);
532 bdi_nr_writeback = bdi_stat(bdi, BDI_WRITEBACK);
533 }
534
535
536
537
538
539
540
541 dirty_exceeded =
542 (bdi_nr_reclaimable + bdi_nr_writeback > bdi_thresh)
543 || (nr_reclaimable + nr_writeback > dirty_thresh);
544
545 if (!dirty_exceeded)
546 break;
547
548 if (!bdi->dirty_exceeded)
549 bdi->dirty_exceeded = 1;
550
551
552
553
554
555
556
557
558
559
560 trace_wbc_balance_dirty_start(&wbc, bdi);
561 if (bdi_nr_reclaimable > bdi_thresh) {
562 writeback_inodes_wb(&bdi->wb, &wbc);
563 pages_written += write_chunk - wbc.nr_to_write;
564 trace_wbc_balance_dirty_written(&wbc, bdi);
565 if (pages_written >= write_chunk)
566 break;
567 }
568 trace_wbc_balance_dirty_wait(&wbc, bdi);
569 __set_current_state(TASK_UNINTERRUPTIBLE);
570 io_schedule_timeout(pause);
571
572
573
574
575
576 pause <<= 1;
577 if (pause > HZ / 10)
578 pause = HZ / 10;
579 }
580
581 if (!dirty_exceeded && bdi->dirty_exceeded)
582 bdi->dirty_exceeded = 0;
583
584 if (writeback_in_progress(bdi))
585 return;
586
587
588
589
590
591
592
593
594
595 if ((laptop_mode && pages_written) ||
596 (!laptop_mode && (nr_reclaimable > background_thresh)))
597 bdi_start_background_writeback(bdi);
598}
599
600void set_page_dirty_balance(struct page *page, int page_mkwrite)
601{
602 if (set_page_dirty(page) || page_mkwrite) {
603 struct address_space *mapping = page_mapping(page);
604
605 if (mapping)
606 balance_dirty_pages_ratelimited(mapping);
607 }
608}
609
610static DEFINE_PER_CPU(unsigned long, bdp_ratelimits) = 0;
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626void balance_dirty_pages_ratelimited_nr(struct address_space *mapping,
627 unsigned long nr_pages_dirtied)
628{
629 unsigned long ratelimit;
630 unsigned long *p;
631
632 ratelimit = ratelimit_pages;
633 if (mapping->backing_dev_info->dirty_exceeded)
634 ratelimit = 8;
635
636
637
638
639
640 preempt_disable();
641 p = &__get_cpu_var(bdp_ratelimits);
642 *p += nr_pages_dirtied;
643 if (unlikely(*p >= ratelimit)) {
644 ratelimit = sync_writeback_pages(*p);
645 *p = 0;
646 preempt_enable();
647 balance_dirty_pages(mapping, ratelimit);
648 return;
649 }
650 preempt_enable();
651}
652EXPORT_SYMBOL(balance_dirty_pages_ratelimited_nr);
653
654void throttle_vm_writeout(gfp_t gfp_mask)
655{
656 unsigned long background_thresh;
657 unsigned long dirty_thresh;
658
659 for ( ; ; ) {
660 global_dirty_limits(&background_thresh, &dirty_thresh);
661
662
663
664
665
666 dirty_thresh += dirty_thresh / 10;
667
668 if (global_page_state(NR_UNSTABLE_NFS) +
669 global_page_state(NR_WRITEBACK) <= dirty_thresh)
670 break;
671 congestion_wait(BLK_RW_ASYNC, HZ/10);
672
673
674
675
676
677
678 if ((gfp_mask & (__GFP_FS|__GFP_IO)) != (__GFP_FS|__GFP_IO))
679 break;
680 }
681}
682
683
684
685
686int dirty_writeback_centisecs_handler(ctl_table *table, int write,
687 void __user *buffer, size_t *length, loff_t *ppos)
688{
689 proc_dointvec(table, write, buffer, length, ppos);
690 bdi_arm_supers_timer();
691 return 0;
692}
693
694#ifdef CONFIG_BLOCK
695void laptop_mode_timer_fn(unsigned long data)
696{
697 struct request_queue *q = (struct request_queue *)data;
698 int nr_pages = global_page_state(NR_FILE_DIRTY) +
699 global_page_state(NR_UNSTABLE_NFS);
700
701
702
703
704
705 if (bdi_has_dirty_io(&q->backing_dev_info))
706 bdi_start_writeback(&q->backing_dev_info, nr_pages);
707}
708
709
710
711
712
713
714void laptop_io_completion(struct backing_dev_info *info)
715{
716 mod_timer(&info->laptop_mode_wb_timer, jiffies + laptop_mode);
717}
718
719
720
721
722
723
724void laptop_sync_completion(void)
725{
726 struct backing_dev_info *bdi;
727
728 rcu_read_lock();
729
730 list_for_each_entry_rcu(bdi, &bdi_list, bdi_list)
731 del_timer(&bdi->laptop_mode_wb_timer);
732
733 rcu_read_unlock();
734}
735#endif
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754void writeback_set_ratelimit(void)
755{
756 ratelimit_pages = vm_total_pages / (num_online_cpus() * 32);
757 if (ratelimit_pages < 16)
758 ratelimit_pages = 16;
759 if (ratelimit_pages * PAGE_CACHE_SIZE > 4096 * 1024)
760 ratelimit_pages = (4096 * 1024) / PAGE_CACHE_SIZE;
761}
762
763static int __cpuinit
764ratelimit_handler(struct notifier_block *self, unsigned long u, void *v)
765{
766 writeback_set_ratelimit();
767 return NOTIFY_DONE;
768}
769
770static struct notifier_block __cpuinitdata ratelimit_nb = {
771 .notifier_call = ratelimit_handler,
772 .next = NULL,
773};
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793void __init page_writeback_init(void)
794{
795 int shift;
796
797 writeback_set_ratelimit();
798 register_cpu_notifier(&ratelimit_nb);
799
800 shift = calc_period_shift();
801 prop_descriptor_init(&vm_completions, shift);
802 prop_descriptor_init(&vm_dirties, shift);
803}
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822void tag_pages_for_writeback(struct address_space *mapping,
823 pgoff_t start, pgoff_t end)
824{
825#define WRITEBACK_TAG_BATCH 4096
826 unsigned long tagged;
827
828 do {
829 spin_lock_irq(&mapping->tree_lock);
830 tagged = radix_tree_range_tag_if_tagged(&mapping->page_tree,
831 &start, end, WRITEBACK_TAG_BATCH,
832 PAGECACHE_TAG_DIRTY, PAGECACHE_TAG_TOWRITE);
833 spin_unlock_irq(&mapping->tree_lock);
834 WARN_ON_ONCE(tagged > WRITEBACK_TAG_BATCH);
835 cond_resched();
836
837 } while (tagged >= WRITEBACK_TAG_BATCH && start);
838}
839EXPORT_SYMBOL(tag_pages_for_writeback);
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863int write_cache_pages(struct address_space *mapping,
864 struct writeback_control *wbc, writepage_t writepage,
865 void *data)
866{
867 int ret = 0;
868 int done = 0;
869 struct pagevec pvec;
870 int nr_pages;
871 pgoff_t uninitialized_var(writeback_index);
872 pgoff_t index;
873 pgoff_t end;
874 pgoff_t done_index;
875 int cycled;
876 int range_whole = 0;
877 int tag;
878
879 pagevec_init(&pvec, 0);
880 if (wbc->range_cyclic) {
881 writeback_index = mapping->writeback_index;
882 index = writeback_index;
883 if (index == 0)
884 cycled = 1;
885 else
886 cycled = 0;
887 end = -1;
888 } else {
889 index = wbc->range_start >> PAGE_CACHE_SHIFT;
890 end = wbc->range_end >> PAGE_CACHE_SHIFT;
891 if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX)
892 range_whole = 1;
893 cycled = 1;
894 }
895 if (wbc->sync_mode == WB_SYNC_ALL)
896 tag = PAGECACHE_TAG_TOWRITE;
897 else
898 tag = PAGECACHE_TAG_DIRTY;
899retry:
900 if (wbc->sync_mode == WB_SYNC_ALL)
901 tag_pages_for_writeback(mapping, index, end);
902 done_index = index;
903 while (!done && (index <= end)) {
904 int i;
905
906 nr_pages = pagevec_lookup_tag(&pvec, mapping, &index, tag,
907 min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1);
908 if (nr_pages == 0)
909 break;
910
911 for (i = 0; i < nr_pages; i++) {
912 struct page *page = pvec.pages[i];
913
914
915
916
917
918
919
920
921 if (page->index > end) {
922
923
924
925
926 done = 1;
927 break;
928 }
929
930 done_index = page->index + 1;
931
932 lock_page(page);
933
934
935
936
937
938
939
940
941
942 if (unlikely(page->mapping != mapping)) {
943continue_unlock:
944 unlock_page(page);
945 continue;
946 }
947
948 if (!PageDirty(page)) {
949
950 goto continue_unlock;
951 }
952
953 if (PageWriteback(page)) {
954 if (wbc->sync_mode != WB_SYNC_NONE)
955 wait_on_page_writeback(page);
956 else
957 goto continue_unlock;
958 }
959
960 BUG_ON(PageWriteback(page));
961 if (!clear_page_dirty_for_io(page))
962 goto continue_unlock;
963
964 trace_wbc_writepage(wbc, mapping->backing_dev_info);
965 ret = (*writepage)(page, wbc, data);
966 if (unlikely(ret)) {
967 if (ret == AOP_WRITEPAGE_ACTIVATE) {
968 unlock_page(page);
969 ret = 0;
970 } else {
971
972
973
974
975
976
977
978
979
980 done = 1;
981 break;
982 }
983 }
984
985
986
987
988
989
990
991 if (--wbc->nr_to_write <= 0 &&
992 wbc->sync_mode == WB_SYNC_NONE) {
993 done = 1;
994 break;
995 }
996 }
997 pagevec_release(&pvec);
998 cond_resched();
999 }
1000 if (!cycled && !done) {
1001
1002
1003
1004
1005
1006 cycled = 1;
1007 index = 0;
1008 end = writeback_index - 1;
1009 goto retry;
1010 }
1011 if (wbc->range_cyclic || (range_whole && wbc->nr_to_write > 0))
1012 mapping->writeback_index = done_index;
1013
1014 return ret;
1015}
1016EXPORT_SYMBOL(write_cache_pages);
1017
1018
1019
1020
1021
1022static int __writepage(struct page *page, struct writeback_control *wbc,
1023 void *data)
1024{
1025 struct address_space *mapping = data;
1026 int ret = mapping->a_ops->writepage(page, wbc);
1027 mapping_set_error(mapping, ret);
1028 return ret;
1029}
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039int generic_writepages(struct address_space *mapping,
1040 struct writeback_control *wbc)
1041{
1042
1043 if (!mapping->a_ops->writepage)
1044 return 0;
1045
1046 return write_cache_pages(mapping, wbc, __writepage, mapping);
1047}
1048
1049EXPORT_SYMBOL(generic_writepages);
1050
1051int do_writepages(struct address_space *mapping, struct writeback_control *wbc)
1052{
1053 int ret;
1054
1055 if (wbc->nr_to_write <= 0)
1056 return 0;
1057 if (mapping->a_ops->writepages)
1058 ret = mapping->a_ops->writepages(mapping, wbc);
1059 else
1060 ret = generic_writepages(mapping, wbc);
1061 return ret;
1062}
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073int write_one_page(struct page *page, int wait)
1074{
1075 struct address_space *mapping = page->mapping;
1076 int ret = 0;
1077 struct writeback_control wbc = {
1078 .sync_mode = WB_SYNC_ALL,
1079 .nr_to_write = 1,
1080 };
1081
1082 BUG_ON(!PageLocked(page));
1083
1084 if (wait)
1085 wait_on_page_writeback(page);
1086
1087 if (clear_page_dirty_for_io(page)) {
1088 page_cache_get(page);
1089 ret = mapping->a_ops->writepage(page, &wbc);
1090 if (ret == 0 && wait) {
1091 wait_on_page_writeback(page);
1092 if (PageError(page))
1093 ret = -EIO;
1094 }
1095 page_cache_release(page);
1096 } else {
1097 unlock_page(page);
1098 }
1099 return ret;
1100}
1101EXPORT_SYMBOL(write_one_page);
1102
1103
1104
1105
1106int __set_page_dirty_no_writeback(struct page *page)
1107{
1108 if (!PageDirty(page))
1109 return !TestSetPageDirty(page);
1110 return 0;
1111}
1112
1113
1114
1115
1116
1117void account_page_dirtied(struct page *page, struct address_space *mapping)
1118{
1119 if (mapping_cap_account_dirty(mapping)) {
1120 __inc_zone_page_state(page, NR_FILE_DIRTY);
1121 __inc_zone_page_state(page, NR_DIRTIED);
1122 __inc_bdi_stat(mapping->backing_dev_info, BDI_RECLAIMABLE);
1123 task_dirty_inc(current);
1124 task_io_account_write(PAGE_CACHE_SIZE);
1125 }
1126}
1127EXPORT_SYMBOL(account_page_dirtied);
1128
1129
1130
1131
1132
1133
1134void account_page_writeback(struct page *page)
1135{
1136 inc_zone_page_state(page, NR_WRITEBACK);
1137 inc_zone_page_state(page, NR_WRITTEN);
1138}
1139EXPORT_SYMBOL(account_page_writeback);
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156int __set_page_dirty_nobuffers(struct page *page)
1157{
1158 if (!TestSetPageDirty(page)) {
1159 struct address_space *mapping = page_mapping(page);
1160 struct address_space *mapping2;
1161
1162 if (!mapping)
1163 return 1;
1164
1165 spin_lock_irq(&mapping->tree_lock);
1166 mapping2 = page_mapping(page);
1167 if (mapping2) {
1168 BUG_ON(mapping2 != mapping);
1169 WARN_ON_ONCE(!PagePrivate(page) && !PageUptodate(page));
1170 account_page_dirtied(page, mapping);
1171 radix_tree_tag_set(&mapping->page_tree,
1172 page_index(page), PAGECACHE_TAG_DIRTY);
1173 }
1174 spin_unlock_irq(&mapping->tree_lock);
1175 if (mapping->host) {
1176
1177 __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
1178 }
1179 return 1;
1180 }
1181 return 0;
1182}
1183EXPORT_SYMBOL(__set_page_dirty_nobuffers);
1184
1185
1186
1187
1188
1189
1190int redirty_page_for_writepage(struct writeback_control *wbc, struct page *page)
1191{
1192 wbc->pages_skipped++;
1193 return __set_page_dirty_nobuffers(page);
1194}
1195EXPORT_SYMBOL(redirty_page_for_writepage);
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208int set_page_dirty(struct page *page)
1209{
1210 struct address_space *mapping = page_mapping(page);
1211
1212 if (likely(mapping)) {
1213 int (*spd)(struct page *) = mapping->a_ops->set_page_dirty;
1214#ifdef CONFIG_BLOCK
1215 if (!spd)
1216 spd = __set_page_dirty_buffers;
1217#endif
1218 return (*spd)(page);
1219 }
1220 if (!PageDirty(page)) {
1221 if (!TestSetPageDirty(page))
1222 return 1;
1223 }
1224 return 0;
1225}
1226EXPORT_SYMBOL(set_page_dirty);
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238int set_page_dirty_lock(struct page *page)
1239{
1240 int ret;
1241
1242 lock_page_nosync(page);
1243 ret = set_page_dirty(page);
1244 unlock_page(page);
1245 return ret;
1246}
1247EXPORT_SYMBOL(set_page_dirty_lock);
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263int clear_page_dirty_for_io(struct page *page)
1264{
1265 struct address_space *mapping = page_mapping(page);
1266
1267 BUG_ON(!PageLocked(page));
1268
1269 ClearPageReclaim(page);
1270 if (mapping && mapping_cap_account_dirty(mapping)) {
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296 if (page_mkclean(page))
1297 set_page_dirty(page);
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308 if (TestClearPageDirty(page)) {
1309 dec_zone_page_state(page, NR_FILE_DIRTY);
1310 dec_bdi_stat(mapping->backing_dev_info,
1311 BDI_RECLAIMABLE);
1312 return 1;
1313 }
1314 return 0;
1315 }
1316 return TestClearPageDirty(page);
1317}
1318EXPORT_SYMBOL(clear_page_dirty_for_io);
1319
1320int test_clear_page_writeback(struct page *page)
1321{
1322 struct address_space *mapping = page_mapping(page);
1323 int ret;
1324
1325 if (mapping) {
1326 struct backing_dev_info *bdi = mapping->backing_dev_info;
1327 unsigned long flags;
1328
1329 spin_lock_irqsave(&mapping->tree_lock, flags);
1330 ret = TestClearPageWriteback(page);
1331 if (ret) {
1332 radix_tree_tag_clear(&mapping->page_tree,
1333 page_index(page),
1334 PAGECACHE_TAG_WRITEBACK);
1335 if (bdi_cap_account_writeback(bdi)) {
1336 __dec_bdi_stat(bdi, BDI_WRITEBACK);
1337 __bdi_writeout_inc(bdi);
1338 }
1339 }
1340 spin_unlock_irqrestore(&mapping->tree_lock, flags);
1341 } else {
1342 ret = TestClearPageWriteback(page);
1343 }
1344 if (ret)
1345 dec_zone_page_state(page, NR_WRITEBACK);
1346 return ret;
1347}
1348
1349int test_set_page_writeback(struct page *page)
1350{
1351 struct address_space *mapping = page_mapping(page);
1352 int ret;
1353
1354 if (mapping) {
1355 struct backing_dev_info *bdi = mapping->backing_dev_info;
1356 unsigned long flags;
1357
1358 spin_lock_irqsave(&mapping->tree_lock, flags);
1359 ret = TestSetPageWriteback(page);
1360 if (!ret) {
1361 radix_tree_tag_set(&mapping->page_tree,
1362 page_index(page),
1363 PAGECACHE_TAG_WRITEBACK);
1364 if (bdi_cap_account_writeback(bdi))
1365 __inc_bdi_stat(bdi, BDI_WRITEBACK);
1366 }
1367 if (!PageDirty(page))
1368 radix_tree_tag_clear(&mapping->page_tree,
1369 page_index(page),
1370 PAGECACHE_TAG_DIRTY);
1371 radix_tree_tag_clear(&mapping->page_tree,
1372 page_index(page),
1373 PAGECACHE_TAG_TOWRITE);
1374 spin_unlock_irqrestore(&mapping->tree_lock, flags);
1375 } else {
1376 ret = TestSetPageWriteback(page);
1377 }
1378 if (!ret)
1379 account_page_writeback(page);
1380 return ret;
1381
1382}
1383EXPORT_SYMBOL(test_set_page_writeback);
1384
1385
1386
1387
1388
1389int mapping_tagged(struct address_space *mapping, int tag)
1390{
1391 int ret;
1392 rcu_read_lock();
1393 ret = radix_tree_tagged(&mapping->page_tree, tag);
1394 rcu_read_unlock();
1395 return ret;
1396}
1397EXPORT_SYMBOL(mapping_tagged);
1398