1
2
3
4
5
6
7
8
9
10
11
12
13
14
15#include <linux/kernel.h>
16#include <linux/export.h>
17#include <linux/spinlock.h>
18#include <linux/fs.h>
19#include <linux/mm.h>
20#include <linux/swap.h>
21#include <linux/slab.h>
22#include <linux/pagemap.h>
23#include <linux/writeback.h>
24#include <linux/init.h>
25#include <linux/backing-dev.h>
26#include <linux/task_io_accounting_ops.h>
27#include <linux/blkdev.h>
28#include <linux/mpage.h>
29#include <linux/rmap.h>
30#include <linux/percpu.h>
31#include <linux/smp.h>
32#include <linux/sysctl.h>
33#include <linux/cpu.h>
34#include <linux/syscalls.h>
35#include <linux/pagevec.h>
36#include <linux/timer.h>
37#include <linux/sched/rt.h>
38#include <linux/sched/signal.h>
39#include <linux/mm_inline.h>
40#include <trace/events/writeback.h>
41
42#include "internal.h"
43
44
45
46
47#define MAX_PAUSE max(HZ/5, 1)
48
49
50
51
52
53#define DIRTY_POLL_THRESH (128 >> (PAGE_SHIFT - 10))
54
55
56
57
58#define BANDWIDTH_INTERVAL max(HZ/5, 1)
59
60#define RATELIMIT_CALC_SHIFT 10
61
62
63
64
65
66static long ratelimit_pages = 32;
67
68
69
70
71
72
73int dirty_background_ratio = 10;
74
75
76
77
78
79unsigned long dirty_background_bytes;
80
81
82
83
84
85int vm_highmem_is_dirtyable;
86
87
88
89
90int vm_dirty_ratio = 20;
91
92
93
94
95
96unsigned long vm_dirty_bytes;
97
98
99
100
101unsigned int dirty_writeback_interval = 5 * 100;
102
103EXPORT_SYMBOL_GPL(dirty_writeback_interval);
104
105
106
107
108unsigned int dirty_expire_interval = 30 * 100;
109
110
111
112
113
114int laptop_mode;
115
116EXPORT_SYMBOL(laptop_mode);
117
118
119
120struct wb_domain global_wb_domain;
121
122
123struct dirty_throttle_control {
124#ifdef CONFIG_CGROUP_WRITEBACK
125 struct wb_domain *dom;
126 struct dirty_throttle_control *gdtc;
127#endif
128 struct bdi_writeback *wb;
129 struct fprop_local_percpu *wb_completions;
130
131 unsigned long avail;
132 unsigned long dirty;
133 unsigned long thresh;
134 unsigned long bg_thresh;
135
136 unsigned long wb_dirty;
137 unsigned long wb_thresh;
138 unsigned long wb_bg_thresh;
139
140 unsigned long pos_ratio;
141};
142
143
144
145
146
147
148#define VM_COMPLETIONS_PERIOD_LEN (3*HZ)
149
150#ifdef CONFIG_CGROUP_WRITEBACK
151
152#define GDTC_INIT(__wb) .wb = (__wb), \
153 .dom = &global_wb_domain, \
154 .wb_completions = &(__wb)->completions
155
156#define GDTC_INIT_NO_WB .dom = &global_wb_domain
157
158#define MDTC_INIT(__wb, __gdtc) .wb = (__wb), \
159 .dom = mem_cgroup_wb_domain(__wb), \
160 .wb_completions = &(__wb)->memcg_completions, \
161 .gdtc = __gdtc
162
163static bool mdtc_valid(struct dirty_throttle_control *dtc)
164{
165 return dtc->dom;
166}
167
168static struct wb_domain *dtc_dom(struct dirty_throttle_control *dtc)
169{
170 return dtc->dom;
171}
172
173static struct dirty_throttle_control *mdtc_gdtc(struct dirty_throttle_control *mdtc)
174{
175 return mdtc->gdtc;
176}
177
178static struct fprop_local_percpu *wb_memcg_completions(struct bdi_writeback *wb)
179{
180 return &wb->memcg_completions;
181}
182
183static void wb_min_max_ratio(struct bdi_writeback *wb,
184 unsigned long *minp, unsigned long *maxp)
185{
186 unsigned long this_bw = READ_ONCE(wb->avg_write_bandwidth);
187 unsigned long tot_bw = atomic_long_read(&wb->bdi->tot_write_bandwidth);
188 unsigned long long min = wb->bdi->min_ratio;
189 unsigned long long max = wb->bdi->max_ratio;
190
191
192
193
194
195 if (this_bw < tot_bw) {
196 if (min) {
197 min *= this_bw;
198 min = div64_ul(min, tot_bw);
199 }
200 if (max < 100) {
201 max *= this_bw;
202 max = div64_ul(max, tot_bw);
203 }
204 }
205
206 *minp = min;
207 *maxp = max;
208}
209
210#else
211
212#define GDTC_INIT(__wb) .wb = (__wb), \
213 .wb_completions = &(__wb)->completions
214#define GDTC_INIT_NO_WB
215#define MDTC_INIT(__wb, __gdtc)
216
217static bool mdtc_valid(struct dirty_throttle_control *dtc)
218{
219 return false;
220}
221
222static struct wb_domain *dtc_dom(struct dirty_throttle_control *dtc)
223{
224 return &global_wb_domain;
225}
226
227static struct dirty_throttle_control *mdtc_gdtc(struct dirty_throttle_control *mdtc)
228{
229 return NULL;
230}
231
232static struct fprop_local_percpu *wb_memcg_completions(struct bdi_writeback *wb)
233{
234 return NULL;
235}
236
237static void wb_min_max_ratio(struct bdi_writeback *wb,
238 unsigned long *minp, unsigned long *maxp)
239{
240 *minp = wb->bdi->min_ratio;
241 *maxp = wb->bdi->max_ratio;
242}
243
244#endif
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271static unsigned long node_dirtyable_memory(struct pglist_data *pgdat)
272{
273 unsigned long nr_pages = 0;
274 int z;
275
276 for (z = 0; z < MAX_NR_ZONES; z++) {
277 struct zone *zone = pgdat->node_zones + z;
278
279 if (!populated_zone(zone))
280 continue;
281
282 nr_pages += zone_page_state(zone, NR_FREE_PAGES);
283 }
284
285
286
287
288
289
290 nr_pages -= min(nr_pages, pgdat->totalreserve_pages);
291
292 nr_pages += node_page_state(pgdat, NR_INACTIVE_FILE);
293 nr_pages += node_page_state(pgdat, NR_ACTIVE_FILE);
294
295 return nr_pages;
296}
297
298static unsigned long highmem_dirtyable_memory(unsigned long total)
299{
300#ifdef CONFIG_HIGHMEM
301 int node;
302 unsigned long x = 0;
303 int i;
304
305 for_each_node_state(node, N_HIGH_MEMORY) {
306 for (i = ZONE_NORMAL + 1; i < MAX_NR_ZONES; i++) {
307 struct zone *z;
308 unsigned long nr_pages;
309
310 if (!is_highmem_idx(i))
311 continue;
312
313 z = &NODE_DATA(node)->node_zones[i];
314 if (!populated_zone(z))
315 continue;
316
317 nr_pages = zone_page_state(z, NR_FREE_PAGES);
318
319 nr_pages -= min(nr_pages, high_wmark_pages(z));
320 nr_pages += zone_page_state(z, NR_ZONE_INACTIVE_FILE);
321 nr_pages += zone_page_state(z, NR_ZONE_ACTIVE_FILE);
322 x += nr_pages;
323 }
324 }
325
326
327
328
329
330
331
332
333
334
335 if ((long)x < 0)
336 x = 0;
337
338
339
340
341
342
343
344 return min(x, total);
345#else
346 return 0;
347#endif
348}
349
350
351
352
353
354
355
356static unsigned long global_dirtyable_memory(void)
357{
358 unsigned long x;
359
360 x = global_zone_page_state(NR_FREE_PAGES);
361
362
363
364
365
366 x -= min(x, totalreserve_pages);
367
368 x += global_node_page_state(NR_INACTIVE_FILE);
369 x += global_node_page_state(NR_ACTIVE_FILE);
370
371 if (!vm_highmem_is_dirtyable)
372 x -= highmem_dirtyable_memory(x);
373
374 return x + 1;
375}
376
377
378
379
380
381
382
383
384
385
386static void domain_dirty_limits(struct dirty_throttle_control *dtc)
387{
388 const unsigned long available_memory = dtc->avail;
389 struct dirty_throttle_control *gdtc = mdtc_gdtc(dtc);
390 unsigned long bytes = vm_dirty_bytes;
391 unsigned long bg_bytes = dirty_background_bytes;
392
393 unsigned long ratio = (vm_dirty_ratio * PAGE_SIZE) / 100;
394 unsigned long bg_ratio = (dirty_background_ratio * PAGE_SIZE) / 100;
395 unsigned long thresh;
396 unsigned long bg_thresh;
397 struct task_struct *tsk;
398
399
400 if (gdtc) {
401 unsigned long global_avail = gdtc->avail;
402
403
404
405
406
407
408
409
410 if (bytes)
411 ratio = min(DIV_ROUND_UP(bytes, global_avail),
412 PAGE_SIZE);
413 if (bg_bytes)
414 bg_ratio = min(DIV_ROUND_UP(bg_bytes, global_avail),
415 PAGE_SIZE);
416 bytes = bg_bytes = 0;
417 }
418
419 if (bytes)
420 thresh = DIV_ROUND_UP(bytes, PAGE_SIZE);
421 else
422 thresh = (ratio * available_memory) / PAGE_SIZE;
423
424 if (bg_bytes)
425 bg_thresh = DIV_ROUND_UP(bg_bytes, PAGE_SIZE);
426 else
427 bg_thresh = (bg_ratio * available_memory) / PAGE_SIZE;
428
429 if (bg_thresh >= thresh)
430 bg_thresh = thresh / 2;
431 tsk = current;
432 if (rt_task(tsk)) {
433 bg_thresh += bg_thresh / 4 + global_wb_domain.dirty_limit / 32;
434 thresh += thresh / 4 + global_wb_domain.dirty_limit / 32;
435 }
436 dtc->thresh = thresh;
437 dtc->bg_thresh = bg_thresh;
438
439
440 if (!gdtc)
441 trace_global_dirty_state(bg_thresh, thresh);
442}
443
444
445
446
447
448
449
450
451
452void global_dirty_limits(unsigned long *pbackground, unsigned long *pdirty)
453{
454 struct dirty_throttle_control gdtc = { GDTC_INIT_NO_WB };
455
456 gdtc.avail = global_dirtyable_memory();
457 domain_dirty_limits(&gdtc);
458
459 *pbackground = gdtc.bg_thresh;
460 *pdirty = gdtc.thresh;
461}
462
463
464
465
466
467
468
469
470static unsigned long node_dirty_limit(struct pglist_data *pgdat)
471{
472 unsigned long node_memory = node_dirtyable_memory(pgdat);
473 struct task_struct *tsk = current;
474 unsigned long dirty;
475
476 if (vm_dirty_bytes)
477 dirty = DIV_ROUND_UP(vm_dirty_bytes, PAGE_SIZE) *
478 node_memory / global_dirtyable_memory();
479 else
480 dirty = vm_dirty_ratio * node_memory / 100;
481
482 if (rt_task(tsk))
483 dirty += dirty / 4;
484
485 return dirty;
486}
487
488
489
490
491
492
493
494
495bool node_dirty_ok(struct pglist_data *pgdat)
496{
497 unsigned long limit = node_dirty_limit(pgdat);
498 unsigned long nr_pages = 0;
499
500 nr_pages += node_page_state(pgdat, NR_FILE_DIRTY);
501 nr_pages += node_page_state(pgdat, NR_WRITEBACK);
502
503 return nr_pages <= limit;
504}
505
506int dirty_background_ratio_handler(struct ctl_table *table, int write,
507 void *buffer, size_t *lenp, loff_t *ppos)
508{
509 int ret;
510
511 ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
512 if (ret == 0 && write)
513 dirty_background_bytes = 0;
514 return ret;
515}
516
517int dirty_background_bytes_handler(struct ctl_table *table, int write,
518 void *buffer, size_t *lenp, loff_t *ppos)
519{
520 int ret;
521
522 ret = proc_doulongvec_minmax(table, write, buffer, lenp, ppos);
523 if (ret == 0 && write)
524 dirty_background_ratio = 0;
525 return ret;
526}
527
528int dirty_ratio_handler(struct ctl_table *table, int write, void *buffer,
529 size_t *lenp, loff_t *ppos)
530{
531 int old_ratio = vm_dirty_ratio;
532 int ret;
533
534 ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
535 if (ret == 0 && write && vm_dirty_ratio != old_ratio) {
536 writeback_set_ratelimit();
537 vm_dirty_bytes = 0;
538 }
539 return ret;
540}
541
542int dirty_bytes_handler(struct ctl_table *table, int write,
543 void *buffer, size_t *lenp, loff_t *ppos)
544{
545 unsigned long old_bytes = vm_dirty_bytes;
546 int ret;
547
548 ret = proc_doulongvec_minmax(table, write, buffer, lenp, ppos);
549 if (ret == 0 && write && vm_dirty_bytes != old_bytes) {
550 writeback_set_ratelimit();
551 vm_dirty_ratio = 0;
552 }
553 return ret;
554}
555
556static unsigned long wp_next_time(unsigned long cur_time)
557{
558 cur_time += VM_COMPLETIONS_PERIOD_LEN;
559
560 if (!cur_time)
561 return 1;
562 return cur_time;
563}
564
565static void wb_domain_writeout_inc(struct wb_domain *dom,
566 struct fprop_local_percpu *completions,
567 unsigned int max_prop_frac)
568{
569 __fprop_inc_percpu_max(&dom->completions, completions,
570 max_prop_frac);
571
572 if (unlikely(!dom->period_time)) {
573
574
575
576
577
578
579 dom->period_time = wp_next_time(jiffies);
580 mod_timer(&dom->period_timer, dom->period_time);
581 }
582}
583
584
585
586
587
588static inline void __wb_writeout_inc(struct bdi_writeback *wb)
589{
590 struct wb_domain *cgdom;
591
592 inc_wb_stat(wb, WB_WRITTEN);
593 wb_domain_writeout_inc(&global_wb_domain, &wb->completions,
594 wb->bdi->max_prop_frac);
595
596 cgdom = mem_cgroup_wb_domain(wb);
597 if (cgdom)
598 wb_domain_writeout_inc(cgdom, wb_memcg_completions(wb),
599 wb->bdi->max_prop_frac);
600}
601
602void wb_writeout_inc(struct bdi_writeback *wb)
603{
604 unsigned long flags;
605
606 local_irq_save(flags);
607 __wb_writeout_inc(wb);
608 local_irq_restore(flags);
609}
610EXPORT_SYMBOL_GPL(wb_writeout_inc);
611
612
613
614
615
616static void writeout_period(struct timer_list *t)
617{
618 struct wb_domain *dom = from_timer(dom, t, period_timer);
619 int miss_periods = (jiffies - dom->period_time) /
620 VM_COMPLETIONS_PERIOD_LEN;
621
622 if (fprop_new_period(&dom->completions, miss_periods + 1)) {
623 dom->period_time = wp_next_time(dom->period_time +
624 miss_periods * VM_COMPLETIONS_PERIOD_LEN);
625 mod_timer(&dom->period_timer, dom->period_time);
626 } else {
627
628
629
630
631 dom->period_time = 0;
632 }
633}
634
635int wb_domain_init(struct wb_domain *dom, gfp_t gfp)
636{
637 memset(dom, 0, sizeof(*dom));
638
639 spin_lock_init(&dom->lock);
640
641 timer_setup(&dom->period_timer, writeout_period, TIMER_DEFERRABLE);
642
643 dom->dirty_limit_tstamp = jiffies;
644
645 return fprop_global_init(&dom->completions, gfp);
646}
647
648#ifdef CONFIG_CGROUP_WRITEBACK
649void wb_domain_exit(struct wb_domain *dom)
650{
651 del_timer_sync(&dom->period_timer);
652 fprop_global_destroy(&dom->completions);
653}
654#endif
655
656
657
658
659
660
661static unsigned int bdi_min_ratio;
662
663int bdi_set_min_ratio(struct backing_dev_info *bdi, unsigned int min_ratio)
664{
665 int ret = 0;
666
667 spin_lock_bh(&bdi_lock);
668 if (min_ratio > bdi->max_ratio) {
669 ret = -EINVAL;
670 } else {
671 min_ratio -= bdi->min_ratio;
672 if (bdi_min_ratio + min_ratio < 100) {
673 bdi_min_ratio += min_ratio;
674 bdi->min_ratio += min_ratio;
675 } else {
676 ret = -EINVAL;
677 }
678 }
679 spin_unlock_bh(&bdi_lock);
680
681 return ret;
682}
683
684int bdi_set_max_ratio(struct backing_dev_info *bdi, unsigned max_ratio)
685{
686 int ret = 0;
687
688 if (max_ratio > 100)
689 return -EINVAL;
690
691 spin_lock_bh(&bdi_lock);
692 if (bdi->min_ratio > max_ratio) {
693 ret = -EINVAL;
694 } else {
695 bdi->max_ratio = max_ratio;
696 bdi->max_prop_frac = (FPROP_FRAC_BASE * max_ratio) / 100;
697 }
698 spin_unlock_bh(&bdi_lock);
699
700 return ret;
701}
702EXPORT_SYMBOL(bdi_set_max_ratio);
703
704static unsigned long dirty_freerun_ceiling(unsigned long thresh,
705 unsigned long bg_thresh)
706{
707 return (thresh + bg_thresh) / 2;
708}
709
710static unsigned long hard_dirty_limit(struct wb_domain *dom,
711 unsigned long thresh)
712{
713 return max(thresh, dom->dirty_limit);
714}
715
716
717
718
719
720static void mdtc_calc_avail(struct dirty_throttle_control *mdtc,
721 unsigned long filepages, unsigned long headroom)
722{
723 struct dirty_throttle_control *gdtc = mdtc_gdtc(mdtc);
724 unsigned long clean = filepages - min(filepages, mdtc->dirty);
725 unsigned long global_clean = gdtc->avail - min(gdtc->avail, gdtc->dirty);
726 unsigned long other_clean = global_clean - min(global_clean, clean);
727
728 mdtc->avail = filepages + min(headroom, other_clean);
729}
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752static unsigned long __wb_calc_thresh(struct dirty_throttle_control *dtc)
753{
754 struct wb_domain *dom = dtc_dom(dtc);
755 unsigned long thresh = dtc->thresh;
756 u64 wb_thresh;
757 unsigned long numerator, denominator;
758 unsigned long wb_min_ratio, wb_max_ratio;
759
760
761
762
763 fprop_fraction_percpu(&dom->completions, dtc->wb_completions,
764 &numerator, &denominator);
765
766 wb_thresh = (thresh * (100 - bdi_min_ratio)) / 100;
767 wb_thresh *= numerator;
768 wb_thresh = div64_ul(wb_thresh, denominator);
769
770 wb_min_max_ratio(dtc->wb, &wb_min_ratio, &wb_max_ratio);
771
772 wb_thresh += (thresh * wb_min_ratio) / 100;
773 if (wb_thresh > (thresh * wb_max_ratio) / 100)
774 wb_thresh = thresh * wb_max_ratio / 100;
775
776 return wb_thresh;
777}
778
779unsigned long wb_calc_thresh(struct bdi_writeback *wb, unsigned long thresh)
780{
781 struct dirty_throttle_control gdtc = { GDTC_INIT(wb),
782 .thresh = thresh };
783 return __wb_calc_thresh(&gdtc);
784}
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800static long long pos_ratio_polynom(unsigned long setpoint,
801 unsigned long dirty,
802 unsigned long limit)
803{
804 long long pos_ratio;
805 long x;
806
807 x = div64_s64(((s64)setpoint - (s64)dirty) << RATELIMIT_CALC_SHIFT,
808 (limit - setpoint) | 1);
809 pos_ratio = x;
810 pos_ratio = pos_ratio * x >> RATELIMIT_CALC_SHIFT;
811 pos_ratio = pos_ratio * x >> RATELIMIT_CALC_SHIFT;
812 pos_ratio += 1 << RATELIMIT_CALC_SHIFT;
813
814 return clamp(pos_ratio, 0LL, 2LL << RATELIMIT_CALC_SHIFT);
815}
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892static void wb_position_ratio(struct dirty_throttle_control *dtc)
893{
894 struct bdi_writeback *wb = dtc->wb;
895 unsigned long write_bw = READ_ONCE(wb->avg_write_bandwidth);
896 unsigned long freerun = dirty_freerun_ceiling(dtc->thresh, dtc->bg_thresh);
897 unsigned long limit = hard_dirty_limit(dtc_dom(dtc), dtc->thresh);
898 unsigned long wb_thresh = dtc->wb_thresh;
899 unsigned long x_intercept;
900 unsigned long setpoint;
901 unsigned long wb_setpoint;
902 unsigned long span;
903 long long pos_ratio;
904 long x;
905
906 dtc->pos_ratio = 0;
907
908 if (unlikely(dtc->dirty >= limit))
909 return;
910
911
912
913
914
915
916 setpoint = (freerun + limit) / 2;
917 pos_ratio = pos_ratio_polynom(setpoint, dtc->dirty, limit);
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944 if (unlikely(wb->bdi->capabilities & BDI_CAP_STRICTLIMIT)) {
945 long long wb_pos_ratio;
946
947 if (dtc->wb_dirty < 8) {
948 dtc->pos_ratio = min_t(long long, pos_ratio * 2,
949 2 << RATELIMIT_CALC_SHIFT);
950 return;
951 }
952
953 if (dtc->wb_dirty >= wb_thresh)
954 return;
955
956 wb_setpoint = dirty_freerun_ceiling(wb_thresh,
957 dtc->wb_bg_thresh);
958
959 if (wb_setpoint == 0 || wb_setpoint == wb_thresh)
960 return;
961
962 wb_pos_ratio = pos_ratio_polynom(wb_setpoint, dtc->wb_dirty,
963 wb_thresh);
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986 dtc->pos_ratio = min(pos_ratio, wb_pos_ratio);
987 return;
988 }
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021 if (unlikely(wb_thresh > dtc->thresh))
1022 wb_thresh = dtc->thresh;
1023
1024
1025
1026
1027
1028
1029
1030 wb_thresh = max(wb_thresh, (limit - dtc->dirty) / 8);
1031
1032
1033
1034
1035 x = div_u64((u64)wb_thresh << 16, dtc->thresh | 1);
1036 wb_setpoint = setpoint * (u64)x >> 16;
1037
1038
1039
1040
1041
1042
1043
1044
1045 span = (dtc->thresh - wb_thresh + 8 * write_bw) * (u64)x >> 16;
1046 x_intercept = wb_setpoint + span;
1047
1048 if (dtc->wb_dirty < x_intercept - span / 4) {
1049 pos_ratio = div64_u64(pos_ratio * (x_intercept - dtc->wb_dirty),
1050 (x_intercept - wb_setpoint) | 1);
1051 } else
1052 pos_ratio /= 4;
1053
1054
1055
1056
1057
1058
1059 x_intercept = wb_thresh / 2;
1060 if (dtc->wb_dirty < x_intercept) {
1061 if (dtc->wb_dirty > x_intercept / 8)
1062 pos_ratio = div_u64(pos_ratio * x_intercept,
1063 dtc->wb_dirty);
1064 else
1065 pos_ratio *= 8;
1066 }
1067
1068 dtc->pos_ratio = pos_ratio;
1069}
1070
1071static void wb_update_write_bandwidth(struct bdi_writeback *wb,
1072 unsigned long elapsed,
1073 unsigned long written)
1074{
1075 const unsigned long period = roundup_pow_of_two(3 * HZ);
1076 unsigned long avg = wb->avg_write_bandwidth;
1077 unsigned long old = wb->write_bandwidth;
1078 u64 bw;
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090 bw = written - min(written, wb->written_stamp);
1091 bw *= HZ;
1092 if (unlikely(elapsed > period)) {
1093 bw = div64_ul(bw, elapsed);
1094 avg = bw;
1095 goto out;
1096 }
1097 bw += (u64)wb->write_bandwidth * (period - elapsed);
1098 bw >>= ilog2(period);
1099
1100
1101
1102
1103 if (avg > old && old >= (unsigned long)bw)
1104 avg -= (avg - old) >> 3;
1105
1106 if (avg < old && old <= (unsigned long)bw)
1107 avg += (old - avg) >> 3;
1108
1109out:
1110
1111 avg = max(avg, 1LU);
1112 if (wb_has_dirty_io(wb)) {
1113 long delta = avg - wb->avg_write_bandwidth;
1114 WARN_ON_ONCE(atomic_long_add_return(delta,
1115 &wb->bdi->tot_write_bandwidth) <= 0);
1116 }
1117 wb->write_bandwidth = bw;
1118 WRITE_ONCE(wb->avg_write_bandwidth, avg);
1119}
1120
1121static void update_dirty_limit(struct dirty_throttle_control *dtc)
1122{
1123 struct wb_domain *dom = dtc_dom(dtc);
1124 unsigned long thresh = dtc->thresh;
1125 unsigned long limit = dom->dirty_limit;
1126
1127
1128
1129
1130 if (limit < thresh) {
1131 limit = thresh;
1132 goto update;
1133 }
1134
1135
1136
1137
1138
1139
1140 thresh = max(thresh, dtc->dirty);
1141 if (limit > thresh) {
1142 limit -= (limit - thresh) >> 5;
1143 goto update;
1144 }
1145 return;
1146update:
1147 dom->dirty_limit = limit;
1148}
1149
1150static void domain_update_dirty_limit(struct dirty_throttle_control *dtc,
1151 unsigned long now)
1152{
1153 struct wb_domain *dom = dtc_dom(dtc);
1154
1155
1156
1157
1158 if (time_before(now, dom->dirty_limit_tstamp + BANDWIDTH_INTERVAL))
1159 return;
1160
1161 spin_lock(&dom->lock);
1162 if (time_after_eq(now, dom->dirty_limit_tstamp + BANDWIDTH_INTERVAL)) {
1163 update_dirty_limit(dtc);
1164 dom->dirty_limit_tstamp = now;
1165 }
1166 spin_unlock(&dom->lock);
1167}
1168
1169
1170
1171
1172
1173
1174
1175static void wb_update_dirty_ratelimit(struct dirty_throttle_control *dtc,
1176 unsigned long dirtied,
1177 unsigned long elapsed)
1178{
1179 struct bdi_writeback *wb = dtc->wb;
1180 unsigned long dirty = dtc->dirty;
1181 unsigned long freerun = dirty_freerun_ceiling(dtc->thresh, dtc->bg_thresh);
1182 unsigned long limit = hard_dirty_limit(dtc_dom(dtc), dtc->thresh);
1183 unsigned long setpoint = (freerun + limit) / 2;
1184 unsigned long write_bw = wb->avg_write_bandwidth;
1185 unsigned long dirty_ratelimit = wb->dirty_ratelimit;
1186 unsigned long dirty_rate;
1187 unsigned long task_ratelimit;
1188 unsigned long balanced_dirty_ratelimit;
1189 unsigned long step;
1190 unsigned long x;
1191 unsigned long shift;
1192
1193
1194
1195
1196
1197 dirty_rate = (dirtied - wb->dirtied_stamp) * HZ / elapsed;
1198
1199
1200
1201
1202 task_ratelimit = (u64)dirty_ratelimit *
1203 dtc->pos_ratio >> RATELIMIT_CALC_SHIFT;
1204 task_ratelimit++;
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236 balanced_dirty_ratelimit = div_u64((u64)task_ratelimit * write_bw,
1237 dirty_rate | 1);
1238
1239
1240
1241 if (unlikely(balanced_dirty_ratelimit > write_bw))
1242 balanced_dirty_ratelimit = write_bw;
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278 step = 0;
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291 if (unlikely(wb->bdi->capabilities & BDI_CAP_STRICTLIMIT)) {
1292 dirty = dtc->wb_dirty;
1293 if (dtc->wb_dirty < 8)
1294 setpoint = dtc->wb_dirty + 1;
1295 else
1296 setpoint = (dtc->wb_thresh + dtc->wb_bg_thresh) / 2;
1297 }
1298
1299 if (dirty < setpoint) {
1300 x = min3(wb->balanced_dirty_ratelimit,
1301 balanced_dirty_ratelimit, task_ratelimit);
1302 if (dirty_ratelimit < x)
1303 step = x - dirty_ratelimit;
1304 } else {
1305 x = max3(wb->balanced_dirty_ratelimit,
1306 balanced_dirty_ratelimit, task_ratelimit);
1307 if (dirty_ratelimit > x)
1308 step = dirty_ratelimit - x;
1309 }
1310
1311
1312
1313
1314
1315
1316 shift = dirty_ratelimit / (2 * step + 1);
1317 if (shift < BITS_PER_LONG)
1318 step = DIV_ROUND_UP(step >> shift, 8);
1319 else
1320 step = 0;
1321
1322 if (dirty_ratelimit < balanced_dirty_ratelimit)
1323 dirty_ratelimit += step;
1324 else
1325 dirty_ratelimit -= step;
1326
1327 WRITE_ONCE(wb->dirty_ratelimit, max(dirty_ratelimit, 1UL));
1328 wb->balanced_dirty_ratelimit = balanced_dirty_ratelimit;
1329
1330 trace_bdi_dirty_ratelimit(wb, dirty_rate, task_ratelimit);
1331}
1332
1333static void __wb_update_bandwidth(struct dirty_throttle_control *gdtc,
1334 struct dirty_throttle_control *mdtc,
1335 bool update_ratelimit)
1336{
1337 struct bdi_writeback *wb = gdtc->wb;
1338 unsigned long now = jiffies;
1339 unsigned long elapsed;
1340 unsigned long dirtied;
1341 unsigned long written;
1342
1343 spin_lock(&wb->list_lock);
1344
1345
1346
1347
1348
1349
1350
1351 elapsed = max(now - wb->bw_time_stamp, 1UL);
1352 dirtied = percpu_counter_read(&wb->stat[WB_DIRTIED]);
1353 written = percpu_counter_read(&wb->stat[WB_WRITTEN]);
1354
1355 if (update_ratelimit) {
1356 domain_update_dirty_limit(gdtc, now);
1357 wb_update_dirty_ratelimit(gdtc, dirtied, elapsed);
1358
1359
1360
1361
1362
1363 if (IS_ENABLED(CONFIG_CGROUP_WRITEBACK) && mdtc) {
1364 domain_update_dirty_limit(mdtc, now);
1365 wb_update_dirty_ratelimit(mdtc, dirtied, elapsed);
1366 }
1367 }
1368 wb_update_write_bandwidth(wb, elapsed, written);
1369
1370 wb->dirtied_stamp = dirtied;
1371 wb->written_stamp = written;
1372 WRITE_ONCE(wb->bw_time_stamp, now);
1373 spin_unlock(&wb->list_lock);
1374}
1375
1376void wb_update_bandwidth(struct bdi_writeback *wb)
1377{
1378 struct dirty_throttle_control gdtc = { GDTC_INIT(wb) };
1379
1380 __wb_update_bandwidth(&gdtc, NULL, false);
1381}
1382
1383
1384#define WB_BANDWIDTH_IDLE_JIF (HZ)
1385
1386static void wb_bandwidth_estimate_start(struct bdi_writeback *wb)
1387{
1388 unsigned long now = jiffies;
1389 unsigned long elapsed = now - READ_ONCE(wb->bw_time_stamp);
1390
1391 if (elapsed > WB_BANDWIDTH_IDLE_JIF &&
1392 !atomic_read(&wb->writeback_inodes)) {
1393 spin_lock(&wb->list_lock);
1394 wb->dirtied_stamp = wb_stat(wb, WB_DIRTIED);
1395 wb->written_stamp = wb_stat(wb, WB_WRITTEN);
1396 WRITE_ONCE(wb->bw_time_stamp, now);
1397 spin_unlock(&wb->list_lock);
1398 }
1399}
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409static unsigned long dirty_poll_interval(unsigned long dirty,
1410 unsigned long thresh)
1411{
1412 if (thresh > dirty)
1413 return 1UL << (ilog2(thresh - dirty) >> 1);
1414
1415 return 1;
1416}
1417
1418static unsigned long wb_max_pause(struct bdi_writeback *wb,
1419 unsigned long wb_dirty)
1420{
1421 unsigned long bw = READ_ONCE(wb->avg_write_bandwidth);
1422 unsigned long t;
1423
1424
1425
1426
1427
1428
1429
1430
1431 t = wb_dirty / (1 + bw / roundup_pow_of_two(1 + HZ / 8));
1432 t++;
1433
1434 return min_t(unsigned long, t, MAX_PAUSE);
1435}
1436
1437static long wb_min_pause(struct bdi_writeback *wb,
1438 long max_pause,
1439 unsigned long task_ratelimit,
1440 unsigned long dirty_ratelimit,
1441 int *nr_dirtied_pause)
1442{
1443 long hi = ilog2(READ_ONCE(wb->avg_write_bandwidth));
1444 long lo = ilog2(READ_ONCE(wb->dirty_ratelimit));
1445 long t;
1446 long pause;
1447 int pages;
1448
1449
1450 t = max(1, HZ / 100);
1451
1452
1453
1454
1455
1456
1457
1458 if (hi > lo)
1459 t += (hi - lo) * (10 * HZ) / 1024;
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479 t = min(t, 1 + max_pause / 2);
1480 pages = dirty_ratelimit * t / roundup_pow_of_two(HZ);
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490 if (pages < DIRTY_POLL_THRESH) {
1491 t = max_pause;
1492 pages = dirty_ratelimit * t / roundup_pow_of_two(HZ);
1493 if (pages > DIRTY_POLL_THRESH) {
1494 pages = DIRTY_POLL_THRESH;
1495 t = HZ * DIRTY_POLL_THRESH / dirty_ratelimit;
1496 }
1497 }
1498
1499 pause = HZ * pages / (task_ratelimit + 1);
1500 if (pause > max_pause) {
1501 t = max_pause;
1502 pages = task_ratelimit * t / roundup_pow_of_two(HZ);
1503 }
1504
1505 *nr_dirtied_pause = pages;
1506
1507
1508
1509 return pages >= DIRTY_POLL_THRESH ? 1 + t / 2 : t;
1510}
1511
1512static inline void wb_dirty_limits(struct dirty_throttle_control *dtc)
1513{
1514 struct bdi_writeback *wb = dtc->wb;
1515 unsigned long wb_reclaimable;
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530 dtc->wb_thresh = __wb_calc_thresh(dtc);
1531 dtc->wb_bg_thresh = dtc->thresh ?
1532 div_u64((u64)dtc->wb_thresh * dtc->bg_thresh, dtc->thresh) : 0;
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544 if (dtc->wb_thresh < 2 * wb_stat_error()) {
1545 wb_reclaimable = wb_stat_sum(wb, WB_RECLAIMABLE);
1546 dtc->wb_dirty = wb_reclaimable + wb_stat_sum(wb, WB_WRITEBACK);
1547 } else {
1548 wb_reclaimable = wb_stat(wb, WB_RECLAIMABLE);
1549 dtc->wb_dirty = wb_reclaimable + wb_stat(wb, WB_WRITEBACK);
1550 }
1551}
1552
1553
1554
1555
1556
1557
1558
1559
1560static void balance_dirty_pages(struct bdi_writeback *wb,
1561 unsigned long pages_dirtied)
1562{
1563 struct dirty_throttle_control gdtc_stor = { GDTC_INIT(wb) };
1564 struct dirty_throttle_control mdtc_stor = { MDTC_INIT(wb, &gdtc_stor) };
1565 struct dirty_throttle_control * const gdtc = &gdtc_stor;
1566 struct dirty_throttle_control * const mdtc = mdtc_valid(&mdtc_stor) ?
1567 &mdtc_stor : NULL;
1568 struct dirty_throttle_control *sdtc;
1569 unsigned long nr_reclaimable;
1570 long period;
1571 long pause;
1572 long max_pause;
1573 long min_pause;
1574 int nr_dirtied_pause;
1575 bool dirty_exceeded = false;
1576 unsigned long task_ratelimit;
1577 unsigned long dirty_ratelimit;
1578 struct backing_dev_info *bdi = wb->bdi;
1579 bool strictlimit = bdi->capabilities & BDI_CAP_STRICTLIMIT;
1580 unsigned long start_time = jiffies;
1581
1582 for (;;) {
1583 unsigned long now = jiffies;
1584 unsigned long dirty, thresh, bg_thresh;
1585 unsigned long m_dirty = 0;
1586 unsigned long m_thresh = 0;
1587 unsigned long m_bg_thresh = 0;
1588
1589 nr_reclaimable = global_node_page_state(NR_FILE_DIRTY);
1590 gdtc->avail = global_dirtyable_memory();
1591 gdtc->dirty = nr_reclaimable + global_node_page_state(NR_WRITEBACK);
1592
1593 domain_dirty_limits(gdtc);
1594
1595 if (unlikely(strictlimit)) {
1596 wb_dirty_limits(gdtc);
1597
1598 dirty = gdtc->wb_dirty;
1599 thresh = gdtc->wb_thresh;
1600 bg_thresh = gdtc->wb_bg_thresh;
1601 } else {
1602 dirty = gdtc->dirty;
1603 thresh = gdtc->thresh;
1604 bg_thresh = gdtc->bg_thresh;
1605 }
1606
1607 if (mdtc) {
1608 unsigned long filepages, headroom, writeback;
1609
1610
1611
1612
1613
1614 mem_cgroup_wb_stats(wb, &filepages, &headroom,
1615 &mdtc->dirty, &writeback);
1616 mdtc->dirty += writeback;
1617 mdtc_calc_avail(mdtc, filepages, headroom);
1618
1619 domain_dirty_limits(mdtc);
1620
1621 if (unlikely(strictlimit)) {
1622 wb_dirty_limits(mdtc);
1623 m_dirty = mdtc->wb_dirty;
1624 m_thresh = mdtc->wb_thresh;
1625 m_bg_thresh = mdtc->wb_bg_thresh;
1626 } else {
1627 m_dirty = mdtc->dirty;
1628 m_thresh = mdtc->thresh;
1629 m_bg_thresh = mdtc->bg_thresh;
1630 }
1631 }
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645 if (dirty <= dirty_freerun_ceiling(thresh, bg_thresh) &&
1646 (!mdtc ||
1647 m_dirty <= dirty_freerun_ceiling(m_thresh, m_bg_thresh))) {
1648 unsigned long intv;
1649 unsigned long m_intv;
1650
1651free_running:
1652 intv = dirty_poll_interval(dirty, thresh);
1653 m_intv = ULONG_MAX;
1654
1655 current->dirty_paused_when = now;
1656 current->nr_dirtied = 0;
1657 if (mdtc)
1658 m_intv = dirty_poll_interval(m_dirty, m_thresh);
1659 current->nr_dirtied_pause = min(intv, m_intv);
1660 break;
1661 }
1662
1663 if (unlikely(!writeback_in_progress(wb)))
1664 wb_start_background_writeback(wb);
1665
1666 mem_cgroup_flush_foreign(wb);
1667
1668
1669
1670
1671
1672 if (!strictlimit) {
1673 wb_dirty_limits(gdtc);
1674
1675 if ((current->flags & PF_LOCAL_THROTTLE) &&
1676 gdtc->wb_dirty <
1677 dirty_freerun_ceiling(gdtc->wb_thresh,
1678 gdtc->wb_bg_thresh))
1679
1680
1681
1682
1683 goto free_running;
1684 }
1685
1686 dirty_exceeded = (gdtc->wb_dirty > gdtc->wb_thresh) &&
1687 ((gdtc->dirty > gdtc->thresh) || strictlimit);
1688
1689 wb_position_ratio(gdtc);
1690 sdtc = gdtc;
1691
1692 if (mdtc) {
1693
1694
1695
1696
1697
1698
1699 if (!strictlimit) {
1700 wb_dirty_limits(mdtc);
1701
1702 if ((current->flags & PF_LOCAL_THROTTLE) &&
1703 mdtc->wb_dirty <
1704 dirty_freerun_ceiling(mdtc->wb_thresh,
1705 mdtc->wb_bg_thresh))
1706
1707
1708
1709
1710
1711 goto free_running;
1712 }
1713 dirty_exceeded |= (mdtc->wb_dirty > mdtc->wb_thresh) &&
1714 ((mdtc->dirty > mdtc->thresh) || strictlimit);
1715
1716 wb_position_ratio(mdtc);
1717 if (mdtc->pos_ratio < gdtc->pos_ratio)
1718 sdtc = mdtc;
1719 }
1720
1721 if (dirty_exceeded && !wb->dirty_exceeded)
1722 wb->dirty_exceeded = 1;
1723
1724 if (time_is_before_jiffies(READ_ONCE(wb->bw_time_stamp) +
1725 BANDWIDTH_INTERVAL))
1726 __wb_update_bandwidth(gdtc, mdtc, true);
1727
1728
1729 dirty_ratelimit = READ_ONCE(wb->dirty_ratelimit);
1730 task_ratelimit = ((u64)dirty_ratelimit * sdtc->pos_ratio) >>
1731 RATELIMIT_CALC_SHIFT;
1732 max_pause = wb_max_pause(wb, sdtc->wb_dirty);
1733 min_pause = wb_min_pause(wb, max_pause,
1734 task_ratelimit, dirty_ratelimit,
1735 &nr_dirtied_pause);
1736
1737 if (unlikely(task_ratelimit == 0)) {
1738 period = max_pause;
1739 pause = max_pause;
1740 goto pause;
1741 }
1742 period = HZ * pages_dirtied / task_ratelimit;
1743 pause = period;
1744 if (current->dirty_paused_when)
1745 pause -= now - current->dirty_paused_when;
1746
1747
1748
1749
1750
1751
1752
1753 if (pause < min_pause) {
1754 trace_balance_dirty_pages(wb,
1755 sdtc->thresh,
1756 sdtc->bg_thresh,
1757 sdtc->dirty,
1758 sdtc->wb_thresh,
1759 sdtc->wb_dirty,
1760 dirty_ratelimit,
1761 task_ratelimit,
1762 pages_dirtied,
1763 period,
1764 min(pause, 0L),
1765 start_time);
1766 if (pause < -HZ) {
1767 current->dirty_paused_when = now;
1768 current->nr_dirtied = 0;
1769 } else if (period) {
1770 current->dirty_paused_when += period;
1771 current->nr_dirtied = 0;
1772 } else if (current->nr_dirtied_pause <= pages_dirtied)
1773 current->nr_dirtied_pause += pages_dirtied;
1774 break;
1775 }
1776 if (unlikely(pause > max_pause)) {
1777
1778 now += min(pause - max_pause, max_pause);
1779 pause = max_pause;
1780 }
1781
1782pause:
1783 trace_balance_dirty_pages(wb,
1784 sdtc->thresh,
1785 sdtc->bg_thresh,
1786 sdtc->dirty,
1787 sdtc->wb_thresh,
1788 sdtc->wb_dirty,
1789 dirty_ratelimit,
1790 task_ratelimit,
1791 pages_dirtied,
1792 period,
1793 pause,
1794 start_time);
1795 __set_current_state(TASK_KILLABLE);
1796 wb->dirty_sleep = now;
1797 io_schedule_timeout(pause);
1798
1799 current->dirty_paused_when = now + pause;
1800 current->nr_dirtied = 0;
1801 current->nr_dirtied_pause = nr_dirtied_pause;
1802
1803
1804
1805
1806
1807 if (task_ratelimit)
1808 break;
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820 if (sdtc->wb_dirty <= wb_stat_error())
1821 break;
1822
1823 if (fatal_signal_pending(current))
1824 break;
1825 }
1826
1827 if (!dirty_exceeded && wb->dirty_exceeded)
1828 wb->dirty_exceeded = 0;
1829
1830 if (writeback_in_progress(wb))
1831 return;
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841 if (laptop_mode)
1842 return;
1843
1844 if (nr_reclaimable > gdtc->bg_thresh)
1845 wb_start_background_writeback(wb);
1846}
1847
1848static DEFINE_PER_CPU(int, bdp_ratelimits);
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864DEFINE_PER_CPU(int, dirty_throttle_leaks) = 0;
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878void balance_dirty_pages_ratelimited(struct address_space *mapping)
1879{
1880 struct inode *inode = mapping->host;
1881 struct backing_dev_info *bdi = inode_to_bdi(inode);
1882 struct bdi_writeback *wb = NULL;
1883 int ratelimit;
1884 int *p;
1885
1886 if (!(bdi->capabilities & BDI_CAP_WRITEBACK))
1887 return;
1888
1889 if (inode_cgwb_enabled(inode))
1890 wb = wb_get_create_current(bdi, GFP_KERNEL);
1891 if (!wb)
1892 wb = &bdi->wb;
1893
1894 ratelimit = current->nr_dirtied_pause;
1895 if (wb->dirty_exceeded)
1896 ratelimit = min(ratelimit, 32 >> (PAGE_SHIFT - 10));
1897
1898 preempt_disable();
1899
1900
1901
1902
1903
1904
1905 p = this_cpu_ptr(&bdp_ratelimits);
1906 if (unlikely(current->nr_dirtied >= ratelimit))
1907 *p = 0;
1908 else if (unlikely(*p >= ratelimit_pages)) {
1909 *p = 0;
1910 ratelimit = 0;
1911 }
1912
1913
1914
1915
1916
1917 p = this_cpu_ptr(&dirty_throttle_leaks);
1918 if (*p > 0 && current->nr_dirtied < ratelimit) {
1919 unsigned long nr_pages_dirtied;
1920 nr_pages_dirtied = min(*p, ratelimit - current->nr_dirtied);
1921 *p -= nr_pages_dirtied;
1922 current->nr_dirtied += nr_pages_dirtied;
1923 }
1924 preempt_enable();
1925
1926 if (unlikely(current->nr_dirtied >= ratelimit))
1927 balance_dirty_pages(wb, current->nr_dirtied);
1928
1929 wb_put(wb);
1930}
1931EXPORT_SYMBOL(balance_dirty_pages_ratelimited);
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942bool wb_over_bg_thresh(struct bdi_writeback *wb)
1943{
1944 struct dirty_throttle_control gdtc_stor = { GDTC_INIT(wb) };
1945 struct dirty_throttle_control mdtc_stor = { MDTC_INIT(wb, &gdtc_stor) };
1946 struct dirty_throttle_control * const gdtc = &gdtc_stor;
1947 struct dirty_throttle_control * const mdtc = mdtc_valid(&mdtc_stor) ?
1948 &mdtc_stor : NULL;
1949 unsigned long reclaimable;
1950 unsigned long thresh;
1951
1952
1953
1954
1955
1956 gdtc->avail = global_dirtyable_memory();
1957 gdtc->dirty = global_node_page_state(NR_FILE_DIRTY);
1958 domain_dirty_limits(gdtc);
1959
1960 if (gdtc->dirty > gdtc->bg_thresh)
1961 return true;
1962
1963 thresh = wb_calc_thresh(gdtc->wb, gdtc->bg_thresh);
1964 if (thresh < 2 * wb_stat_error())
1965 reclaimable = wb_stat_sum(wb, WB_RECLAIMABLE);
1966 else
1967 reclaimable = wb_stat(wb, WB_RECLAIMABLE);
1968
1969 if (reclaimable > thresh)
1970 return true;
1971
1972 if (mdtc) {
1973 unsigned long filepages, headroom, writeback;
1974
1975 mem_cgroup_wb_stats(wb, &filepages, &headroom, &mdtc->dirty,
1976 &writeback);
1977 mdtc_calc_avail(mdtc, filepages, headroom);
1978 domain_dirty_limits(mdtc);
1979
1980 if (mdtc->dirty > mdtc->bg_thresh)
1981 return true;
1982
1983 thresh = wb_calc_thresh(mdtc->wb, mdtc->bg_thresh);
1984 if (thresh < 2 * wb_stat_error())
1985 reclaimable = wb_stat_sum(wb, WB_RECLAIMABLE);
1986 else
1987 reclaimable = wb_stat(wb, WB_RECLAIMABLE);
1988
1989 if (reclaimable > thresh)
1990 return true;
1991 }
1992
1993 return false;
1994}
1995
1996
1997
1998
1999int dirty_writeback_centisecs_handler(struct ctl_table *table, int write,
2000 void *buffer, size_t *length, loff_t *ppos)
2001{
2002 unsigned int old_interval = dirty_writeback_interval;
2003 int ret;
2004
2005 ret = proc_dointvec(table, write, buffer, length, ppos);
2006
2007
2008
2009
2010
2011
2012
2013
2014 if (!ret && write && dirty_writeback_interval &&
2015 dirty_writeback_interval != old_interval)
2016 wakeup_flusher_threads(WB_REASON_PERIODIC);
2017
2018 return ret;
2019}
2020
2021void laptop_mode_timer_fn(struct timer_list *t)
2022{
2023 struct backing_dev_info *backing_dev_info =
2024 from_timer(backing_dev_info, t, laptop_mode_wb_timer);
2025
2026 wakeup_flusher_threads_bdi(backing_dev_info, WB_REASON_LAPTOP_TIMER);
2027}
2028
2029
2030
2031
2032
2033
2034void laptop_io_completion(struct backing_dev_info *info)
2035{
2036 mod_timer(&info->laptop_mode_wb_timer, jiffies + laptop_mode);
2037}
2038
2039
2040
2041
2042
2043
2044void laptop_sync_completion(void)
2045{
2046 struct backing_dev_info *bdi;
2047
2048 rcu_read_lock();
2049
2050 list_for_each_entry_rcu(bdi, &bdi_list, bdi_list)
2051 del_timer(&bdi->laptop_mode_wb_timer);
2052
2053 rcu_read_unlock();
2054}
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065void writeback_set_ratelimit(void)
2066{
2067 struct wb_domain *dom = &global_wb_domain;
2068 unsigned long background_thresh;
2069 unsigned long dirty_thresh;
2070
2071 global_dirty_limits(&background_thresh, &dirty_thresh);
2072 dom->dirty_limit = dirty_thresh;
2073 ratelimit_pages = dirty_thresh / (num_online_cpus() * 32);
2074 if (ratelimit_pages < 16)
2075 ratelimit_pages = 16;
2076}
2077
2078static int page_writeback_cpu_online(unsigned int cpu)
2079{
2080 writeback_set_ratelimit();
2081 return 0;
2082}
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100void __init page_writeback_init(void)
2101{
2102 BUG_ON(wb_domain_init(&global_wb_domain, GFP_KERNEL));
2103
2104 cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "mm/writeback:online",
2105 page_writeback_cpu_online, NULL);
2106 cpuhp_setup_state(CPUHP_MM_WRITEBACK_DEAD, "mm/writeback:dead", NULL,
2107 page_writeback_cpu_online);
2108}
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124void tag_pages_for_writeback(struct address_space *mapping,
2125 pgoff_t start, pgoff_t end)
2126{
2127 XA_STATE(xas, &mapping->i_pages, start);
2128 unsigned int tagged = 0;
2129 void *page;
2130
2131 xas_lock_irq(&xas);
2132 xas_for_each_marked(&xas, page, end, PAGECACHE_TAG_DIRTY) {
2133 xas_set_mark(&xas, PAGECACHE_TAG_TOWRITE);
2134 if (++tagged % XA_CHECK_SCHED)
2135 continue;
2136
2137 xas_pause(&xas);
2138 xas_unlock_irq(&xas);
2139 cond_resched();
2140 xas_lock_irq(&xas);
2141 }
2142 xas_unlock_irq(&xas);
2143}
2144EXPORT_SYMBOL(tag_pages_for_writeback);
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177int write_cache_pages(struct address_space *mapping,
2178 struct writeback_control *wbc, writepage_t writepage,
2179 void *data)
2180{
2181 int ret = 0;
2182 int done = 0;
2183 int error;
2184 struct pagevec pvec;
2185 int nr_pages;
2186 pgoff_t index;
2187 pgoff_t end;
2188 pgoff_t done_index;
2189 int range_whole = 0;
2190 xa_mark_t tag;
2191
2192 pagevec_init(&pvec);
2193 if (wbc->range_cyclic) {
2194 index = mapping->writeback_index;
2195 end = -1;
2196 } else {
2197 index = wbc->range_start >> PAGE_SHIFT;
2198 end = wbc->range_end >> PAGE_SHIFT;
2199 if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX)
2200 range_whole = 1;
2201 }
2202 if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages) {
2203 tag_pages_for_writeback(mapping, index, end);
2204 tag = PAGECACHE_TAG_TOWRITE;
2205 } else {
2206 tag = PAGECACHE_TAG_DIRTY;
2207 }
2208 done_index = index;
2209 while (!done && (index <= end)) {
2210 int i;
2211
2212 nr_pages = pagevec_lookup_range_tag(&pvec, mapping, &index, end,
2213 tag);
2214 if (nr_pages == 0)
2215 break;
2216
2217 for (i = 0; i < nr_pages; i++) {
2218 struct page *page = pvec.pages[i];
2219
2220 done_index = page->index;
2221
2222 lock_page(page);
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232 if (unlikely(page->mapping != mapping)) {
2233continue_unlock:
2234 unlock_page(page);
2235 continue;
2236 }
2237
2238 if (!PageDirty(page)) {
2239
2240 goto continue_unlock;
2241 }
2242
2243 if (PageWriteback(page)) {
2244 if (wbc->sync_mode != WB_SYNC_NONE)
2245 wait_on_page_writeback(page);
2246 else
2247 goto continue_unlock;
2248 }
2249
2250 BUG_ON(PageWriteback(page));
2251 if (!clear_page_dirty_for_io(page))
2252 goto continue_unlock;
2253
2254 trace_wbc_writepage(wbc, inode_to_bdi(mapping->host));
2255 error = (*writepage)(page, wbc, data);
2256 if (unlikely(error)) {
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269 if (error == AOP_WRITEPAGE_ACTIVATE) {
2270 unlock_page(page);
2271 error = 0;
2272 } else if (wbc->sync_mode != WB_SYNC_ALL) {
2273 ret = error;
2274 done_index = page->index + 1;
2275 done = 1;
2276 break;
2277 }
2278 if (!ret)
2279 ret = error;
2280 }
2281
2282
2283
2284
2285
2286
2287
2288 if (--wbc->nr_to_write <= 0 &&
2289 wbc->sync_mode == WB_SYNC_NONE) {
2290 done = 1;
2291 break;
2292 }
2293 }
2294 pagevec_release(&pvec);
2295 cond_resched();
2296 }
2297
2298
2299
2300
2301
2302
2303 if (wbc->range_cyclic && !done)
2304 done_index = 0;
2305 if (wbc->range_cyclic || (range_whole && wbc->nr_to_write > 0))
2306 mapping->writeback_index = done_index;
2307
2308 return ret;
2309}
2310EXPORT_SYMBOL(write_cache_pages);
2311
2312
2313
2314
2315
2316static int __writepage(struct page *page, struct writeback_control *wbc,
2317 void *data)
2318{
2319 struct address_space *mapping = data;
2320 int ret = mapping->a_ops->writepage(page, wbc);
2321 mapping_set_error(mapping, ret);
2322 return ret;
2323}
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335int generic_writepages(struct address_space *mapping,
2336 struct writeback_control *wbc)
2337{
2338 struct blk_plug plug;
2339 int ret;
2340
2341
2342 if (!mapping->a_ops->writepage)
2343 return 0;
2344
2345 blk_start_plug(&plug);
2346 ret = write_cache_pages(mapping, wbc, __writepage, mapping);
2347 blk_finish_plug(&plug);
2348 return ret;
2349}
2350
2351EXPORT_SYMBOL(generic_writepages);
2352
2353int do_writepages(struct address_space *mapping, struct writeback_control *wbc)
2354{
2355 int ret;
2356 struct bdi_writeback *wb;
2357
2358 if (wbc->nr_to_write <= 0)
2359 return 0;
2360 wb = inode_to_wb_wbc(mapping->host, wbc);
2361 wb_bandwidth_estimate_start(wb);
2362 while (1) {
2363 if (mapping->a_ops->writepages)
2364 ret = mapping->a_ops->writepages(mapping, wbc);
2365 else
2366 ret = generic_writepages(mapping, wbc);
2367 if ((ret != -ENOMEM) || (wbc->sync_mode != WB_SYNC_ALL))
2368 break;
2369 cond_resched();
2370 congestion_wait(BLK_RW_ASYNC, HZ/50);
2371 }
2372
2373
2374
2375
2376
2377 if (time_is_before_jiffies(READ_ONCE(wb->bw_time_stamp) +
2378 BANDWIDTH_INTERVAL))
2379 wb_update_bandwidth(wb);
2380 return ret;
2381}
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394int write_one_page(struct page *page)
2395{
2396 struct address_space *mapping = page->mapping;
2397 int ret = 0;
2398 struct writeback_control wbc = {
2399 .sync_mode = WB_SYNC_ALL,
2400 .nr_to_write = 1,
2401 };
2402
2403 BUG_ON(!PageLocked(page));
2404
2405 wait_on_page_writeback(page);
2406
2407 if (clear_page_dirty_for_io(page)) {
2408 get_page(page);
2409 ret = mapping->a_ops->writepage(page, &wbc);
2410 if (ret == 0)
2411 wait_on_page_writeback(page);
2412 put_page(page);
2413 } else {
2414 unlock_page(page);
2415 }
2416
2417 if (!ret)
2418 ret = filemap_check_errors(mapping);
2419 return ret;
2420}
2421EXPORT_SYMBOL(write_one_page);
2422
2423
2424
2425
2426int __set_page_dirty_no_writeback(struct page *page)
2427{
2428 if (!PageDirty(page))
2429 return !TestSetPageDirty(page);
2430 return 0;
2431}
2432EXPORT_SYMBOL(__set_page_dirty_no_writeback);
2433
2434
2435
2436
2437
2438
2439
2440
2441static void account_page_dirtied(struct page *page,
2442 struct address_space *mapping)
2443{
2444 struct inode *inode = mapping->host;
2445
2446 trace_writeback_dirty_page(page, mapping);
2447
2448 if (mapping_can_writeback(mapping)) {
2449 struct bdi_writeback *wb;
2450
2451 inode_attach_wb(inode, page);
2452 wb = inode_to_wb(inode);
2453
2454 __inc_lruvec_page_state(page, NR_FILE_DIRTY);
2455 __inc_zone_page_state(page, NR_ZONE_WRITE_PENDING);
2456 __inc_node_page_state(page, NR_DIRTIED);
2457 inc_wb_stat(wb, WB_RECLAIMABLE);
2458 inc_wb_stat(wb, WB_DIRTIED);
2459 task_io_account_write(PAGE_SIZE);
2460 current->nr_dirtied++;
2461 __this_cpu_inc(bdp_ratelimits);
2462
2463 mem_cgroup_track_foreign_dirty(page, wb);
2464 }
2465}
2466
2467
2468
2469
2470
2471
2472void account_page_cleaned(struct page *page, struct address_space *mapping,
2473 struct bdi_writeback *wb)
2474{
2475 if (mapping_can_writeback(mapping)) {
2476 dec_lruvec_page_state(page, NR_FILE_DIRTY);
2477 dec_zone_page_state(page, NR_ZONE_WRITE_PENDING);
2478 dec_wb_stat(wb, WB_RECLAIMABLE);
2479 task_io_account_cancelled_write(PAGE_SIZE);
2480 }
2481}
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492void __set_page_dirty(struct page *page, struct address_space *mapping,
2493 int warn)
2494{
2495 unsigned long flags;
2496
2497 xa_lock_irqsave(&mapping->i_pages, flags);
2498 if (page->mapping) {
2499 WARN_ON_ONCE(warn && !PageUptodate(page));
2500 account_page_dirtied(page, mapping);
2501 __xa_set_mark(&mapping->i_pages, page_index(page),
2502 PAGECACHE_TAG_DIRTY);
2503 }
2504 xa_unlock_irqrestore(&mapping->i_pages, flags);
2505}
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519int __set_page_dirty_nobuffers(struct page *page)
2520{
2521 lock_page_memcg(page);
2522 if (!TestSetPageDirty(page)) {
2523 struct address_space *mapping = page_mapping(page);
2524
2525 if (!mapping) {
2526 unlock_page_memcg(page);
2527 return 1;
2528 }
2529 __set_page_dirty(page, mapping, !PagePrivate(page));
2530 unlock_page_memcg(page);
2531
2532 if (mapping->host) {
2533
2534 __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
2535 }
2536 return 1;
2537 }
2538 unlock_page_memcg(page);
2539 return 0;
2540}
2541EXPORT_SYMBOL(__set_page_dirty_nobuffers);
2542
2543
2544
2545
2546
2547
2548
2549
2550void account_page_redirty(struct page *page)
2551{
2552 struct address_space *mapping = page->mapping;
2553
2554 if (mapping && mapping_can_writeback(mapping)) {
2555 struct inode *inode = mapping->host;
2556 struct bdi_writeback *wb;
2557 struct wb_lock_cookie cookie = {};
2558
2559 wb = unlocked_inode_to_wb_begin(inode, &cookie);
2560 current->nr_dirtied--;
2561 dec_node_page_state(page, NR_DIRTIED);
2562 dec_wb_stat(wb, WB_DIRTIED);
2563 unlocked_inode_to_wb_end(inode, &cookie);
2564 }
2565}
2566EXPORT_SYMBOL(account_page_redirty);
2567
2568
2569
2570
2571
2572
2573int redirty_page_for_writepage(struct writeback_control *wbc, struct page *page)
2574{
2575 int ret;
2576
2577 wbc->pages_skipped++;
2578 ret = __set_page_dirty_nobuffers(page);
2579 account_page_redirty(page);
2580 return ret;
2581}
2582EXPORT_SYMBOL(redirty_page_for_writepage);
2583
2584
2585
2586
2587
2588
2589
2590
2591int set_page_dirty(struct page *page)
2592{
2593 struct address_space *mapping = page_mapping(page);
2594
2595 page = compound_head(page);
2596 if (likely(mapping)) {
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607 if (PageReclaim(page))
2608 ClearPageReclaim(page);
2609 return mapping->a_ops->set_page_dirty(page);
2610 }
2611 if (!PageDirty(page)) {
2612 if (!TestSetPageDirty(page))
2613 return 1;
2614 }
2615 return 0;
2616}
2617EXPORT_SYMBOL(set_page_dirty);
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629int set_page_dirty_lock(struct page *page)
2630{
2631 int ret;
2632
2633 lock_page(page);
2634 ret = set_page_dirty(page);
2635 unlock_page(page);
2636 return ret;
2637}
2638EXPORT_SYMBOL(set_page_dirty_lock);
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653void __cancel_dirty_page(struct page *page)
2654{
2655 struct address_space *mapping = page_mapping(page);
2656
2657 if (mapping_can_writeback(mapping)) {
2658 struct inode *inode = mapping->host;
2659 struct bdi_writeback *wb;
2660 struct wb_lock_cookie cookie = {};
2661
2662 lock_page_memcg(page);
2663 wb = unlocked_inode_to_wb_begin(inode, &cookie);
2664
2665 if (TestClearPageDirty(page))
2666 account_page_cleaned(page, mapping, wb);
2667
2668 unlocked_inode_to_wb_end(inode, &cookie);
2669 unlock_page_memcg(page);
2670 } else {
2671 ClearPageDirty(page);
2672 }
2673}
2674EXPORT_SYMBOL(__cancel_dirty_page);
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690int clear_page_dirty_for_io(struct page *page)
2691{
2692 struct address_space *mapping = page_mapping(page);
2693 int ret = 0;
2694
2695 VM_BUG_ON_PAGE(!PageLocked(page), page);
2696
2697 if (mapping && mapping_can_writeback(mapping)) {
2698 struct inode *inode = mapping->host;
2699 struct bdi_writeback *wb;
2700 struct wb_lock_cookie cookie = {};
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727 if (page_mkclean(page))
2728 set_page_dirty(page);
2729
2730
2731
2732
2733
2734
2735
2736
2737 wb = unlocked_inode_to_wb_begin(inode, &cookie);
2738 if (TestClearPageDirty(page)) {
2739 dec_lruvec_page_state(page, NR_FILE_DIRTY);
2740 dec_zone_page_state(page, NR_ZONE_WRITE_PENDING);
2741 dec_wb_stat(wb, WB_RECLAIMABLE);
2742 ret = 1;
2743 }
2744 unlocked_inode_to_wb_end(inode, &cookie);
2745 return ret;
2746 }
2747 return TestClearPageDirty(page);
2748}
2749EXPORT_SYMBOL(clear_page_dirty_for_io);
2750
2751static void wb_inode_writeback_start(struct bdi_writeback *wb)
2752{
2753 atomic_inc(&wb->writeback_inodes);
2754}
2755
2756static void wb_inode_writeback_end(struct bdi_writeback *wb)
2757{
2758 atomic_dec(&wb->writeback_inodes);
2759
2760
2761
2762
2763
2764
2765
2766 queue_delayed_work(bdi_wq, &wb->bw_dwork, BANDWIDTH_INTERVAL);
2767}
2768
2769int test_clear_page_writeback(struct page *page)
2770{
2771 struct address_space *mapping = page_mapping(page);
2772 int ret;
2773
2774 lock_page_memcg(page);
2775 if (mapping && mapping_use_writeback_tags(mapping)) {
2776 struct inode *inode = mapping->host;
2777 struct backing_dev_info *bdi = inode_to_bdi(inode);
2778 unsigned long flags;
2779
2780 xa_lock_irqsave(&mapping->i_pages, flags);
2781 ret = TestClearPageWriteback(page);
2782 if (ret) {
2783 __xa_clear_mark(&mapping->i_pages, page_index(page),
2784 PAGECACHE_TAG_WRITEBACK);
2785 if (bdi->capabilities & BDI_CAP_WRITEBACK_ACCT) {
2786 struct bdi_writeback *wb = inode_to_wb(inode);
2787
2788 dec_wb_stat(wb, WB_WRITEBACK);
2789 __wb_writeout_inc(wb);
2790 if (!mapping_tagged(mapping,
2791 PAGECACHE_TAG_WRITEBACK))
2792 wb_inode_writeback_end(wb);
2793 }
2794 }
2795
2796 if (mapping->host && !mapping_tagged(mapping,
2797 PAGECACHE_TAG_WRITEBACK))
2798 sb_clear_inode_writeback(mapping->host);
2799
2800 xa_unlock_irqrestore(&mapping->i_pages, flags);
2801 } else {
2802 ret = TestClearPageWriteback(page);
2803 }
2804 if (ret) {
2805 dec_lruvec_page_state(page, NR_WRITEBACK);
2806 dec_zone_page_state(page, NR_ZONE_WRITE_PENDING);
2807 inc_node_page_state(page, NR_WRITTEN);
2808 }
2809 unlock_page_memcg(page);
2810 return ret;
2811}
2812
2813int __test_set_page_writeback(struct page *page, bool keep_write)
2814{
2815 struct address_space *mapping = page_mapping(page);
2816 int ret, access_ret;
2817
2818 lock_page_memcg(page);
2819 if (mapping && mapping_use_writeback_tags(mapping)) {
2820 XA_STATE(xas, &mapping->i_pages, page_index(page));
2821 struct inode *inode = mapping->host;
2822 struct backing_dev_info *bdi = inode_to_bdi(inode);
2823 unsigned long flags;
2824
2825 xas_lock_irqsave(&xas, flags);
2826 xas_load(&xas);
2827 ret = TestSetPageWriteback(page);
2828 if (!ret) {
2829 bool on_wblist;
2830
2831 on_wblist = mapping_tagged(mapping,
2832 PAGECACHE_TAG_WRITEBACK);
2833
2834 xas_set_mark(&xas, PAGECACHE_TAG_WRITEBACK);
2835 if (bdi->capabilities & BDI_CAP_WRITEBACK_ACCT) {
2836 struct bdi_writeback *wb = inode_to_wb(inode);
2837
2838 inc_wb_stat(wb, WB_WRITEBACK);
2839 if (!on_wblist)
2840 wb_inode_writeback_start(wb);
2841 }
2842
2843
2844
2845
2846
2847
2848 if (mapping->host && !on_wblist)
2849 sb_mark_inode_writeback(mapping->host);
2850 }
2851 if (!PageDirty(page))
2852 xas_clear_mark(&xas, PAGECACHE_TAG_DIRTY);
2853 if (!keep_write)
2854 xas_clear_mark(&xas, PAGECACHE_TAG_TOWRITE);
2855 xas_unlock_irqrestore(&xas, flags);
2856 } else {
2857 ret = TestSetPageWriteback(page);
2858 }
2859 if (!ret) {
2860 inc_lruvec_page_state(page, NR_WRITEBACK);
2861 inc_zone_page_state(page, NR_ZONE_WRITE_PENDING);
2862 }
2863 unlock_page_memcg(page);
2864 access_ret = arch_make_page_accessible(page);
2865
2866
2867
2868
2869 VM_BUG_ON_PAGE(access_ret != 0, page);
2870
2871 return ret;
2872
2873}
2874EXPORT_SYMBOL(__test_set_page_writeback);
2875
2876
2877
2878
2879void wait_on_page_writeback(struct page *page)
2880{
2881 while (PageWriteback(page)) {
2882 trace_wait_on_page_writeback(page, page_mapping(page));
2883 wait_on_page_bit(page, PG_writeback);
2884 }
2885}
2886EXPORT_SYMBOL_GPL(wait_on_page_writeback);
2887
2888
2889
2890
2891
2892int wait_on_page_writeback_killable(struct page *page)
2893{
2894 while (PageWriteback(page)) {
2895 trace_wait_on_page_writeback(page, page_mapping(page));
2896 if (wait_on_page_bit_killable(page, PG_writeback))
2897 return -EINTR;
2898 }
2899
2900 return 0;
2901}
2902EXPORT_SYMBOL_GPL(wait_on_page_writeback_killable);
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912void wait_for_stable_page(struct page *page)
2913{
2914 page = thp_head(page);
2915 if (page->mapping->host->i_sb->s_iflags & SB_I_STABLE_WRITES)
2916 wait_on_page_writeback(page);
2917}
2918EXPORT_SYMBOL_GPL(wait_for_stable_page);
2919