1
2
3
4
5
6
7
8
9
10
11#include <linux/cpu.h>
12#include <linux/swap.h>
13#include <linux/migrate.h>
14#include <linux/compaction.h>
15#include <linux/mm_inline.h>
16#include <linux/sched/signal.h>
17#include <linux/backing-dev.h>
18#include <linux/sysctl.h>
19#include <linux/sysfs.h>
20#include <linux/page-isolation.h>
21#include <linux/kasan.h>
22#include <linux/kthread.h>
23#include <linux/freezer.h>
24#include <linux/page_owner.h>
25#include <linux/psi.h>
26#include "internal.h"
27
28#ifdef CONFIG_COMPACTION
29static inline void count_compact_event(enum vm_event_item item)
30{
31 count_vm_event(item);
32}
33
34static inline void count_compact_events(enum vm_event_item item, long delta)
35{
36 count_vm_events(item, delta);
37}
38#else
39#define count_compact_event(item) do { } while (0)
40#define count_compact_events(item, delta) do { } while (0)
41#endif
42
43#if defined CONFIG_COMPACTION || defined CONFIG_CMA
44
45#define CREATE_TRACE_POINTS
46#include <trace/events/compaction.h>
47
48#define block_start_pfn(pfn, order) round_down(pfn, 1UL << (order))
49#define block_end_pfn(pfn, order) ALIGN((pfn) + 1, 1UL << (order))
50#define pageblock_start_pfn(pfn) block_start_pfn(pfn, pageblock_order)
51#define pageblock_end_pfn(pfn) block_end_pfn(pfn, pageblock_order)
52
53static unsigned long release_freepages(struct list_head *freelist)
54{
55 struct page *page, *next;
56 unsigned long high_pfn = 0;
57
58 list_for_each_entry_safe(page, next, freelist, lru) {
59 unsigned long pfn = page_to_pfn(page);
60 list_del(&page->lru);
61 __free_page(page);
62 if (pfn > high_pfn)
63 high_pfn = pfn;
64 }
65
66 return high_pfn;
67}
68
69static void split_map_pages(struct list_head *list)
70{
71 unsigned int i, order, nr_pages;
72 struct page *page, *next;
73 LIST_HEAD(tmp_list);
74
75 list_for_each_entry_safe(page, next, list, lru) {
76 list_del(&page->lru);
77
78 order = page_private(page);
79 nr_pages = 1 << order;
80
81 post_alloc_hook(page, order, __GFP_MOVABLE);
82 if (order)
83 split_page(page, order);
84
85 for (i = 0; i < nr_pages; i++) {
86 list_add(&page->lru, &tmp_list);
87 page++;
88 }
89 }
90
91 list_splice(&tmp_list, list);
92}
93
94#ifdef CONFIG_COMPACTION
95
96int PageMovable(struct page *page)
97{
98 struct address_space *mapping;
99
100 VM_BUG_ON_PAGE(!PageLocked(page), page);
101 if (!__PageMovable(page))
102 return 0;
103
104 mapping = page_mapping(page);
105 if (mapping && mapping->a_ops && mapping->a_ops->isolate_page)
106 return 1;
107
108 return 0;
109}
110EXPORT_SYMBOL(PageMovable);
111
112void __SetPageMovable(struct page *page, struct address_space *mapping)
113{
114 VM_BUG_ON_PAGE(!PageLocked(page), page);
115 VM_BUG_ON_PAGE((unsigned long)mapping & PAGE_MAPPING_MOVABLE, page);
116 page->mapping = (void *)((unsigned long)mapping | PAGE_MAPPING_MOVABLE);
117}
118EXPORT_SYMBOL(__SetPageMovable);
119
120void __ClearPageMovable(struct page *page)
121{
122 VM_BUG_ON_PAGE(!PageLocked(page), page);
123 VM_BUG_ON_PAGE(!PageMovable(page), page);
124
125
126
127
128
129 page->mapping = (void *)((unsigned long)page->mapping &
130 PAGE_MAPPING_MOVABLE);
131}
132EXPORT_SYMBOL(__ClearPageMovable);
133
134
135#define COMPACT_MAX_DEFER_SHIFT 6
136
137
138
139
140
141
142void defer_compaction(struct zone *zone, int order)
143{
144 zone->compact_considered = 0;
145 zone->compact_defer_shift++;
146
147 if (order < zone->compact_order_failed)
148 zone->compact_order_failed = order;
149
150 if (zone->compact_defer_shift > COMPACT_MAX_DEFER_SHIFT)
151 zone->compact_defer_shift = COMPACT_MAX_DEFER_SHIFT;
152
153 trace_mm_compaction_defer_compaction(zone, order);
154}
155
156
157bool compaction_deferred(struct zone *zone, int order)
158{
159 unsigned long defer_limit = 1UL << zone->compact_defer_shift;
160
161 if (order < zone->compact_order_failed)
162 return false;
163
164
165 if (++zone->compact_considered > defer_limit)
166 zone->compact_considered = defer_limit;
167
168 if (zone->compact_considered >= defer_limit)
169 return false;
170
171 trace_mm_compaction_deferred(zone, order);
172
173 return true;
174}
175
176
177
178
179
180
181void compaction_defer_reset(struct zone *zone, int order,
182 bool alloc_success)
183{
184 if (alloc_success) {
185 zone->compact_considered = 0;
186 zone->compact_defer_shift = 0;
187 }
188 if (order >= zone->compact_order_failed)
189 zone->compact_order_failed = order + 1;
190
191 trace_mm_compaction_defer_reset(zone, order);
192}
193
194
195bool compaction_restarting(struct zone *zone, int order)
196{
197 if (order < zone->compact_order_failed)
198 return false;
199
200 return zone->compact_defer_shift == COMPACT_MAX_DEFER_SHIFT &&
201 zone->compact_considered >= 1UL << zone->compact_defer_shift;
202}
203
204
205static inline bool isolation_suitable(struct compact_control *cc,
206 struct page *page)
207{
208 if (cc->ignore_skip_hint)
209 return true;
210
211 return !get_pageblock_skip(page);
212}
213
214static void reset_cached_positions(struct zone *zone)
215{
216 zone->compact_cached_migrate_pfn[0] = zone->zone_start_pfn;
217 zone->compact_cached_migrate_pfn[1] = zone->zone_start_pfn;
218 zone->compact_cached_free_pfn =
219 pageblock_start_pfn(zone_end_pfn(zone) - 1);
220}
221
222
223
224
225
226
227static bool pageblock_skip_persistent(struct page *page)
228{
229 if (!PageCompound(page))
230 return false;
231
232 page = compound_head(page);
233
234 if (compound_order(page) >= pageblock_order)
235 return true;
236
237 return false;
238}
239
240static bool
241__reset_isolation_pfn(struct zone *zone, unsigned long pfn, bool check_source,
242 bool check_target)
243{
244 struct page *page = pfn_to_online_page(pfn);
245 struct page *block_page;
246 struct page *end_page;
247 unsigned long block_pfn;
248
249 if (!page)
250 return false;
251 if (zone != page_zone(page))
252 return false;
253 if (pageblock_skip_persistent(page))
254 return false;
255
256
257
258
259
260 if (check_source && check_target && !get_pageblock_skip(page))
261 return true;
262
263
264
265
266
267 if (!check_source && check_target &&
268 get_pageblock_migratetype(page) != MIGRATE_MOVABLE)
269 return false;
270
271
272 block_pfn = pageblock_start_pfn(pfn);
273 block_page = pfn_to_online_page(max(block_pfn, zone->zone_start_pfn));
274 if (block_page) {
275 page = block_page;
276 pfn = block_pfn;
277 }
278
279
280 block_pfn += pageblock_nr_pages;
281 block_pfn = min(block_pfn, zone_end_pfn(zone) - 1);
282 end_page = pfn_to_online_page(block_pfn);
283 if (!end_page)
284 return false;
285
286
287
288
289
290
291 do {
292 if (pfn_valid_within(pfn)) {
293 if (check_source && PageLRU(page)) {
294 clear_pageblock_skip(page);
295 return true;
296 }
297
298 if (check_target && PageBuddy(page)) {
299 clear_pageblock_skip(page);
300 return true;
301 }
302 }
303
304 page += (1 << PAGE_ALLOC_COSTLY_ORDER);
305 pfn += (1 << PAGE_ALLOC_COSTLY_ORDER);
306 } while (page < end_page);
307
308 return false;
309}
310
311
312
313
314
315
316static void __reset_isolation_suitable(struct zone *zone)
317{
318 unsigned long migrate_pfn = zone->zone_start_pfn;
319 unsigned long free_pfn = zone_end_pfn(zone) - 1;
320 unsigned long reset_migrate = free_pfn;
321 unsigned long reset_free = migrate_pfn;
322 bool source_set = false;
323 bool free_set = false;
324
325 if (!zone->compact_blockskip_flush)
326 return;
327
328 zone->compact_blockskip_flush = false;
329
330
331
332
333
334
335
336 for (; migrate_pfn < free_pfn; migrate_pfn += pageblock_nr_pages,
337 free_pfn -= pageblock_nr_pages) {
338 cond_resched();
339
340
341 if (__reset_isolation_pfn(zone, migrate_pfn, true, source_set) &&
342 migrate_pfn < reset_migrate) {
343 source_set = true;
344 reset_migrate = migrate_pfn;
345 zone->compact_init_migrate_pfn = reset_migrate;
346 zone->compact_cached_migrate_pfn[0] = reset_migrate;
347 zone->compact_cached_migrate_pfn[1] = reset_migrate;
348 }
349
350
351 if (__reset_isolation_pfn(zone, free_pfn, free_set, true) &&
352 free_pfn > reset_free) {
353 free_set = true;
354 reset_free = free_pfn;
355 zone->compact_init_free_pfn = reset_free;
356 zone->compact_cached_free_pfn = reset_free;
357 }
358 }
359
360
361 if (reset_migrate >= reset_free) {
362 zone->compact_cached_migrate_pfn[0] = migrate_pfn;
363 zone->compact_cached_migrate_pfn[1] = migrate_pfn;
364 zone->compact_cached_free_pfn = free_pfn;
365 }
366}
367
368void reset_isolation_suitable(pg_data_t *pgdat)
369{
370 int zoneid;
371
372 for (zoneid = 0; zoneid < MAX_NR_ZONES; zoneid++) {
373 struct zone *zone = &pgdat->node_zones[zoneid];
374 if (!populated_zone(zone))
375 continue;
376
377
378 if (zone->compact_blockskip_flush)
379 __reset_isolation_suitable(zone);
380 }
381}
382
383
384
385
386
387static bool test_and_set_skip(struct compact_control *cc, struct page *page,
388 unsigned long pfn)
389{
390 bool skip;
391
392
393 if (cc->ignore_skip_hint)
394 return false;
395
396 if (!IS_ALIGNED(pfn, pageblock_nr_pages))
397 return false;
398
399 skip = get_pageblock_skip(page);
400 if (!skip && !cc->no_set_skip_hint)
401 set_pageblock_skip(page);
402
403 return skip;
404}
405
406static void update_cached_migrate(struct compact_control *cc, unsigned long pfn)
407{
408 struct zone *zone = cc->zone;
409
410 pfn = pageblock_end_pfn(pfn);
411
412
413 if (cc->no_set_skip_hint)
414 return;
415
416 if (pfn > zone->compact_cached_migrate_pfn[0])
417 zone->compact_cached_migrate_pfn[0] = pfn;
418 if (cc->mode != MIGRATE_ASYNC &&
419 pfn > zone->compact_cached_migrate_pfn[1])
420 zone->compact_cached_migrate_pfn[1] = pfn;
421}
422
423
424
425
426
427static void update_pageblock_skip(struct compact_control *cc,
428 struct page *page, unsigned long pfn)
429{
430 struct zone *zone = cc->zone;
431
432 if (cc->no_set_skip_hint)
433 return;
434
435 if (!page)
436 return;
437
438 set_pageblock_skip(page);
439
440
441 if (pfn < zone->compact_cached_free_pfn)
442 zone->compact_cached_free_pfn = pfn;
443}
444#else
445static inline bool isolation_suitable(struct compact_control *cc,
446 struct page *page)
447{
448 return true;
449}
450
451static inline bool pageblock_skip_persistent(struct page *page)
452{
453 return false;
454}
455
456static inline void update_pageblock_skip(struct compact_control *cc,
457 struct page *page, unsigned long pfn)
458{
459}
460
461static void update_cached_migrate(struct compact_control *cc, unsigned long pfn)
462{
463}
464
465static bool test_and_set_skip(struct compact_control *cc, struct page *page,
466 unsigned long pfn)
467{
468 return false;
469}
470#endif
471
472
473
474
475
476
477
478
479
480
481static bool compact_lock_irqsave(spinlock_t *lock, unsigned long *flags,
482 struct compact_control *cc)
483{
484
485 if (cc->mode == MIGRATE_ASYNC && !cc->contended) {
486 if (spin_trylock_irqsave(lock, *flags))
487 return true;
488
489 cc->contended = true;
490 }
491
492 spin_lock_irqsave(lock, *flags);
493 return true;
494}
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511static bool compact_unlock_should_abort(spinlock_t *lock,
512 unsigned long flags, bool *locked, struct compact_control *cc)
513{
514 if (*locked) {
515 spin_unlock_irqrestore(lock, flags);
516 *locked = false;
517 }
518
519 if (fatal_signal_pending(current)) {
520 cc->contended = true;
521 return true;
522 }
523
524 cond_resched();
525
526 return false;
527}
528
529
530
531
532
533
534static unsigned long isolate_freepages_block(struct compact_control *cc,
535 unsigned long *start_pfn,
536 unsigned long end_pfn,
537 struct list_head *freelist,
538 unsigned int stride,
539 bool strict)
540{
541 int nr_scanned = 0, total_isolated = 0;
542 struct page *cursor;
543 unsigned long flags = 0;
544 bool locked = false;
545 unsigned long blockpfn = *start_pfn;
546 unsigned int order;
547
548
549 if (strict)
550 stride = 1;
551
552 cursor = pfn_to_page(blockpfn);
553
554
555 for (; blockpfn < end_pfn; blockpfn += stride, cursor += stride) {
556 int isolated;
557 struct page *page = cursor;
558
559
560
561
562
563
564 if (!(blockpfn % SWAP_CLUSTER_MAX)
565 && compact_unlock_should_abort(&cc->zone->lock, flags,
566 &locked, cc))
567 break;
568
569 nr_scanned++;
570 if (!pfn_valid_within(blockpfn))
571 goto isolate_fail;
572
573
574
575
576
577
578
579 if (PageCompound(page)) {
580 const unsigned int order = compound_order(page);
581
582 if (likely(order < MAX_ORDER)) {
583 blockpfn += (1UL << order) - 1;
584 cursor += (1UL << order) - 1;
585 }
586 goto isolate_fail;
587 }
588
589 if (!PageBuddy(page))
590 goto isolate_fail;
591
592
593
594
595
596
597
598
599 if (!locked) {
600 locked = compact_lock_irqsave(&cc->zone->lock,
601 &flags, cc);
602
603
604 if (!PageBuddy(page))
605 goto isolate_fail;
606 }
607
608
609 order = page_order(page);
610 isolated = __isolate_free_page(page, order);
611 if (!isolated)
612 break;
613 set_page_private(page, order);
614
615 total_isolated += isolated;
616 cc->nr_freepages += isolated;
617 list_add_tail(&page->lru, freelist);
618
619 if (!strict && cc->nr_migratepages <= cc->nr_freepages) {
620 blockpfn += isolated;
621 break;
622 }
623
624 blockpfn += isolated - 1;
625 cursor += isolated - 1;
626 continue;
627
628isolate_fail:
629 if (strict)
630 break;
631 else
632 continue;
633
634 }
635
636 if (locked)
637 spin_unlock_irqrestore(&cc->zone->lock, flags);
638
639
640
641
642
643 if (unlikely(blockpfn > end_pfn))
644 blockpfn = end_pfn;
645
646 trace_mm_compaction_isolate_freepages(*start_pfn, blockpfn,
647 nr_scanned, total_isolated);
648
649
650 *start_pfn = blockpfn;
651
652
653
654
655
656
657 if (strict && blockpfn < end_pfn)
658 total_isolated = 0;
659
660 cc->total_free_scanned += nr_scanned;
661 if (total_isolated)
662 count_compact_events(COMPACTISOLATED, total_isolated);
663 return total_isolated;
664}
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680unsigned long
681isolate_freepages_range(struct compact_control *cc,
682 unsigned long start_pfn, unsigned long end_pfn)
683{
684 unsigned long isolated, pfn, block_start_pfn, block_end_pfn;
685 LIST_HEAD(freelist);
686
687 pfn = start_pfn;
688 block_start_pfn = pageblock_start_pfn(pfn);
689 if (block_start_pfn < cc->zone->zone_start_pfn)
690 block_start_pfn = cc->zone->zone_start_pfn;
691 block_end_pfn = pageblock_end_pfn(pfn);
692
693 for (; pfn < end_pfn; pfn += isolated,
694 block_start_pfn = block_end_pfn,
695 block_end_pfn += pageblock_nr_pages) {
696
697 unsigned long isolate_start_pfn = pfn;
698
699 block_end_pfn = min(block_end_pfn, end_pfn);
700
701
702
703
704
705
706 if (pfn >= block_end_pfn) {
707 block_start_pfn = pageblock_start_pfn(pfn);
708 block_end_pfn = pageblock_end_pfn(pfn);
709 block_end_pfn = min(block_end_pfn, end_pfn);
710 }
711
712 if (!pageblock_pfn_to_page(block_start_pfn,
713 block_end_pfn, cc->zone))
714 break;
715
716 isolated = isolate_freepages_block(cc, &isolate_start_pfn,
717 block_end_pfn, &freelist, 0, true);
718
719
720
721
722
723
724 if (!isolated)
725 break;
726
727
728
729
730
731
732 }
733
734
735 split_map_pages(&freelist);
736
737 if (pfn < end_pfn) {
738
739 release_freepages(&freelist);
740 return 0;
741 }
742
743
744 return pfn;
745}
746
747
748static bool too_many_isolated(pg_data_t *pgdat)
749{
750 unsigned long active, inactive, isolated;
751
752 inactive = node_page_state(pgdat, NR_INACTIVE_FILE) +
753 node_page_state(pgdat, NR_INACTIVE_ANON);
754 active = node_page_state(pgdat, NR_ACTIVE_FILE) +
755 node_page_state(pgdat, NR_ACTIVE_ANON);
756 isolated = node_page_state(pgdat, NR_ISOLATED_FILE) +
757 node_page_state(pgdat, NR_ISOLATED_ANON);
758
759 return isolated > (inactive + active) / 2;
760}
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780static unsigned long
781isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
782 unsigned long end_pfn, isolate_mode_t isolate_mode)
783{
784 pg_data_t *pgdat = cc->zone->zone_pgdat;
785 unsigned long nr_scanned = 0, nr_isolated = 0;
786 struct lruvec *lruvec;
787 unsigned long flags = 0;
788 bool locked = false;
789 struct page *page = NULL, *valid_page = NULL;
790 unsigned long start_pfn = low_pfn;
791 bool skip_on_failure = false;
792 unsigned long next_skip_pfn = 0;
793 bool skip_updated = false;
794
795
796
797
798
799
800 while (unlikely(too_many_isolated(pgdat))) {
801
802 if (cc->mode == MIGRATE_ASYNC)
803 return 0;
804
805 congestion_wait(BLK_RW_ASYNC, HZ/10);
806
807 if (fatal_signal_pending(current))
808 return 0;
809 }
810
811 cond_resched();
812
813 if (cc->direct_compaction && (cc->mode == MIGRATE_ASYNC)) {
814 skip_on_failure = true;
815 next_skip_pfn = block_end_pfn(low_pfn, cc->order);
816 }
817
818
819 for (; low_pfn < end_pfn; low_pfn++) {
820
821 if (skip_on_failure && low_pfn >= next_skip_pfn) {
822
823
824
825
826
827
828 if (nr_isolated)
829 break;
830
831
832
833
834
835
836
837
838
839
840 next_skip_pfn = block_end_pfn(low_pfn, cc->order);
841 }
842
843
844
845
846
847
848 if (!(low_pfn % SWAP_CLUSTER_MAX)
849 && compact_unlock_should_abort(&pgdat->lru_lock,
850 flags, &locked, cc)) {
851 low_pfn = 0;
852 goto fatal_pending;
853 }
854
855 if (!pfn_valid_within(low_pfn))
856 goto isolate_fail;
857 nr_scanned++;
858
859 page = pfn_to_page(low_pfn);
860
861
862
863
864
865
866
867 if (!valid_page && IS_ALIGNED(low_pfn, pageblock_nr_pages)) {
868 if (!cc->ignore_skip_hint && get_pageblock_skip(page)) {
869 low_pfn = end_pfn;
870 goto isolate_abort;
871 }
872 valid_page = page;
873 }
874
875
876
877
878
879
880
881 if (PageBuddy(page)) {
882 unsigned long freepage_order = page_order_unsafe(page);
883
884
885
886
887
888
889 if (freepage_order > 0 && freepage_order < MAX_ORDER)
890 low_pfn += (1UL << freepage_order) - 1;
891 continue;
892 }
893
894
895
896
897
898
899
900
901 if (PageCompound(page)) {
902 const unsigned int order = compound_order(page);
903
904 if (likely(order < MAX_ORDER))
905 low_pfn += (1UL << order) - 1;
906 goto isolate_fail;
907 }
908
909
910
911
912
913
914 if (!PageLRU(page)) {
915
916
917
918
919 if (unlikely(__PageMovable(page)) &&
920 !PageIsolated(page)) {
921 if (locked) {
922 spin_unlock_irqrestore(&pgdat->lru_lock,
923 flags);
924 locked = false;
925 }
926
927 if (!isolate_movable_page(page, isolate_mode))
928 goto isolate_success;
929 }
930
931 goto isolate_fail;
932 }
933
934
935
936
937
938
939 if (!page_mapping(page) &&
940 page_count(page) > page_mapcount(page))
941 goto isolate_fail;
942
943
944
945
946
947 if (!(cc->gfp_mask & __GFP_FS) && page_mapping(page))
948 goto isolate_fail;
949
950
951 if (!locked) {
952 locked = compact_lock_irqsave(&pgdat->lru_lock,
953 &flags, cc);
954
955
956 if (!skip_updated) {
957 skip_updated = true;
958 if (test_and_set_skip(cc, page, low_pfn))
959 goto isolate_abort;
960 }
961
962
963 if (!PageLRU(page))
964 goto isolate_fail;
965
966
967
968
969
970
971 if (unlikely(PageCompound(page))) {
972 low_pfn += (1UL << compound_order(page)) - 1;
973 goto isolate_fail;
974 }
975 }
976
977 lruvec = mem_cgroup_page_lruvec(page, pgdat);
978
979
980 if (__isolate_lru_page(page, isolate_mode) != 0)
981 goto isolate_fail;
982
983 VM_BUG_ON_PAGE(PageCompound(page), page);
984
985
986 del_page_from_lru_list(page, lruvec, page_lru(page));
987 inc_node_page_state(page,
988 NR_ISOLATED_ANON + page_is_file_cache(page));
989
990isolate_success:
991 list_add(&page->lru, &cc->migratepages);
992 cc->nr_migratepages++;
993 nr_isolated++;
994
995
996
997
998
999
1000
1001 if (cc->nr_migratepages == COMPACT_CLUSTER_MAX &&
1002 !cc->rescan && !cc->contended) {
1003 ++low_pfn;
1004 break;
1005 }
1006
1007 continue;
1008isolate_fail:
1009 if (!skip_on_failure)
1010 continue;
1011
1012
1013
1014
1015
1016
1017 if (nr_isolated) {
1018 if (locked) {
1019 spin_unlock_irqrestore(&pgdat->lru_lock, flags);
1020 locked = false;
1021 }
1022 putback_movable_pages(&cc->migratepages);
1023 cc->nr_migratepages = 0;
1024 nr_isolated = 0;
1025 }
1026
1027 if (low_pfn < next_skip_pfn) {
1028 low_pfn = next_skip_pfn - 1;
1029
1030
1031
1032
1033 next_skip_pfn += 1UL << cc->order;
1034 }
1035 }
1036
1037
1038
1039
1040
1041 if (unlikely(low_pfn > end_pfn))
1042 low_pfn = end_pfn;
1043
1044isolate_abort:
1045 if (locked)
1046 spin_unlock_irqrestore(&pgdat->lru_lock, flags);
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056 if (low_pfn == end_pfn && (!nr_isolated || cc->rescan)) {
1057 if (valid_page && !skip_updated)
1058 set_pageblock_skip(valid_page);
1059 update_cached_migrate(cc, low_pfn);
1060 }
1061
1062 trace_mm_compaction_isolate_migratepages(start_pfn, low_pfn,
1063 nr_scanned, nr_isolated);
1064
1065fatal_pending:
1066 cc->total_migrate_scanned += nr_scanned;
1067 if (nr_isolated)
1068 count_compact_events(COMPACTISOLATED, nr_isolated);
1069
1070 return low_pfn;
1071}
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083unsigned long
1084isolate_migratepages_range(struct compact_control *cc, unsigned long start_pfn,
1085 unsigned long end_pfn)
1086{
1087 unsigned long pfn, block_start_pfn, block_end_pfn;
1088
1089
1090 pfn = start_pfn;
1091 block_start_pfn = pageblock_start_pfn(pfn);
1092 if (block_start_pfn < cc->zone->zone_start_pfn)
1093 block_start_pfn = cc->zone->zone_start_pfn;
1094 block_end_pfn = pageblock_end_pfn(pfn);
1095
1096 for (; pfn < end_pfn; pfn = block_end_pfn,
1097 block_start_pfn = block_end_pfn,
1098 block_end_pfn += pageblock_nr_pages) {
1099
1100 block_end_pfn = min(block_end_pfn, end_pfn);
1101
1102 if (!pageblock_pfn_to_page(block_start_pfn,
1103 block_end_pfn, cc->zone))
1104 continue;
1105
1106 pfn = isolate_migratepages_block(cc, pfn, block_end_pfn,
1107 ISOLATE_UNEVICTABLE);
1108
1109 if (!pfn)
1110 break;
1111
1112 if (cc->nr_migratepages == COMPACT_CLUSTER_MAX)
1113 break;
1114 }
1115
1116 return pfn;
1117}
1118
1119#endif
1120#ifdef CONFIG_COMPACTION
1121
1122static bool suitable_migration_source(struct compact_control *cc,
1123 struct page *page)
1124{
1125 int block_mt;
1126
1127 if (pageblock_skip_persistent(page))
1128 return false;
1129
1130 if ((cc->mode != MIGRATE_ASYNC) || !cc->direct_compaction)
1131 return true;
1132
1133 block_mt = get_pageblock_migratetype(page);
1134
1135 if (cc->migratetype == MIGRATE_MOVABLE)
1136 return is_migrate_movable(block_mt);
1137 else
1138 return block_mt == cc->migratetype;
1139}
1140
1141
1142static bool suitable_migration_target(struct compact_control *cc,
1143 struct page *page)
1144{
1145
1146 if (PageBuddy(page)) {
1147
1148
1149
1150
1151
1152 if (page_order_unsafe(page) >= pageblock_order)
1153 return false;
1154 }
1155
1156 if (cc->ignore_block_suitable)
1157 return true;
1158
1159
1160 if (is_migrate_movable(get_pageblock_migratetype(page)))
1161 return true;
1162
1163
1164 return false;
1165}
1166
1167static inline unsigned int
1168freelist_scan_limit(struct compact_control *cc)
1169{
1170 unsigned short shift = BITS_PER_LONG - 1;
1171
1172 return (COMPACT_CLUSTER_MAX >> min(shift, cc->fast_search_fail)) + 1;
1173}
1174
1175
1176
1177
1178
1179static inline bool compact_scanners_met(struct compact_control *cc)
1180{
1181 return (cc->free_pfn >> pageblock_order)
1182 <= (cc->migrate_pfn >> pageblock_order);
1183}
1184
1185
1186
1187
1188
1189
1190static void
1191move_freelist_head(struct list_head *freelist, struct page *freepage)
1192{
1193 LIST_HEAD(sublist);
1194
1195 if (!list_is_last(freelist, &freepage->lru)) {
1196 list_cut_before(&sublist, freelist, &freepage->lru);
1197 if (!list_empty(&sublist))
1198 list_splice_tail(&sublist, freelist);
1199 }
1200}
1201
1202
1203
1204
1205
1206
1207
1208static void
1209move_freelist_tail(struct list_head *freelist, struct page *freepage)
1210{
1211 LIST_HEAD(sublist);
1212
1213 if (!list_is_first(freelist, &freepage->lru)) {
1214 list_cut_position(&sublist, freelist, &freepage->lru);
1215 if (!list_empty(&sublist))
1216 list_splice_tail(&sublist, freelist);
1217 }
1218}
1219
1220static void
1221fast_isolate_around(struct compact_control *cc, unsigned long pfn, unsigned long nr_isolated)
1222{
1223 unsigned long start_pfn, end_pfn;
1224 struct page *page = pfn_to_page(pfn);
1225
1226
1227 if (cc->nr_freepages >= cc->nr_migratepages)
1228 return;
1229
1230
1231 if (cc->direct_compaction && cc->mode == MIGRATE_ASYNC)
1232 return;
1233
1234
1235 start_pfn = pageblock_start_pfn(pfn);
1236 end_pfn = min(pageblock_end_pfn(pfn), zone_end_pfn(cc->zone)) - 1;
1237
1238
1239 if (start_pfn != pfn) {
1240 isolate_freepages_block(cc, &start_pfn, pfn, &cc->freepages, 1, false);
1241 if (cc->nr_freepages >= cc->nr_migratepages)
1242 return;
1243 }
1244
1245
1246 start_pfn = pfn + nr_isolated;
1247 if (start_pfn < end_pfn)
1248 isolate_freepages_block(cc, &start_pfn, end_pfn, &cc->freepages, 1, false);
1249
1250
1251 if (cc->nr_freepages < cc->nr_migratepages)
1252 set_pageblock_skip(page);
1253}
1254
1255
1256static int next_search_order(struct compact_control *cc, int order)
1257{
1258 order--;
1259 if (order < 0)
1260 order = cc->order - 1;
1261
1262
1263 if (order == cc->search_order) {
1264 cc->search_order--;
1265 if (cc->search_order < 0)
1266 cc->search_order = cc->order - 1;
1267 return -1;
1268 }
1269
1270 return order;
1271}
1272
1273static unsigned long
1274fast_isolate_freepages(struct compact_control *cc)
1275{
1276 unsigned int limit = min(1U, freelist_scan_limit(cc) >> 1);
1277 unsigned int nr_scanned = 0;
1278 unsigned long low_pfn, min_pfn, high_pfn = 0, highest = 0;
1279 unsigned long nr_isolated = 0;
1280 unsigned long distance;
1281 struct page *page = NULL;
1282 bool scan_start = false;
1283 int order;
1284
1285
1286 if (cc->order <= 0)
1287 return cc->free_pfn;
1288
1289
1290
1291
1292
1293 if (cc->free_pfn >= cc->zone->compact_init_free_pfn) {
1294 limit = pageblock_nr_pages >> 1;
1295 scan_start = true;
1296 }
1297
1298
1299
1300
1301
1302 distance = (cc->free_pfn - cc->migrate_pfn);
1303 low_pfn = pageblock_start_pfn(cc->free_pfn - (distance >> 2));
1304 min_pfn = pageblock_start_pfn(cc->free_pfn - (distance >> 1));
1305
1306 if (WARN_ON_ONCE(min_pfn > low_pfn))
1307 low_pfn = min_pfn;
1308
1309
1310
1311
1312
1313 cc->search_order = min_t(unsigned int, cc->order - 1, cc->search_order);
1314
1315 for (order = cc->search_order;
1316 !page && order >= 0;
1317 order = next_search_order(cc, order)) {
1318 struct free_area *area = &cc->zone->free_area[order];
1319 struct list_head *freelist;
1320 struct page *freepage;
1321 unsigned long flags;
1322 unsigned int order_scanned = 0;
1323
1324 if (!area->nr_free)
1325 continue;
1326
1327 spin_lock_irqsave(&cc->zone->lock, flags);
1328 freelist = &area->free_list[MIGRATE_MOVABLE];
1329 list_for_each_entry_reverse(freepage, freelist, lru) {
1330 unsigned long pfn;
1331
1332 order_scanned++;
1333 nr_scanned++;
1334 pfn = page_to_pfn(freepage);
1335
1336 if (pfn >= highest)
1337 highest = pageblock_start_pfn(pfn);
1338
1339 if (pfn >= low_pfn) {
1340 cc->fast_search_fail = 0;
1341 cc->search_order = order;
1342 page = freepage;
1343 break;
1344 }
1345
1346 if (pfn >= min_pfn && pfn > high_pfn) {
1347 high_pfn = pfn;
1348
1349
1350 limit >>= 1;
1351 }
1352
1353 if (order_scanned >= limit)
1354 break;
1355 }
1356
1357
1358 if (!page && high_pfn) {
1359 page = pfn_to_page(high_pfn);
1360
1361
1362 freepage = page;
1363 }
1364
1365
1366 move_freelist_head(freelist, freepage);
1367
1368
1369 if (page) {
1370 if (__isolate_free_page(page, order)) {
1371 set_page_private(page, order);
1372 nr_isolated = 1 << order;
1373 cc->nr_freepages += nr_isolated;
1374 list_add_tail(&page->lru, &cc->freepages);
1375 count_compact_events(COMPACTISOLATED, nr_isolated);
1376 } else {
1377
1378 order = cc->search_order + 1;
1379 page = NULL;
1380 }
1381 }
1382
1383 spin_unlock_irqrestore(&cc->zone->lock, flags);
1384
1385
1386
1387
1388
1389 if (order_scanned >= limit)
1390 limit = min(1U, limit >> 1);
1391 }
1392
1393 if (!page) {
1394 cc->fast_search_fail++;
1395 if (scan_start) {
1396
1397
1398
1399
1400
1401 if (highest) {
1402 page = pfn_to_page(highest);
1403 cc->free_pfn = highest;
1404 } else {
1405 if (cc->direct_compaction && pfn_valid(min_pfn)) {
1406 page = pfn_to_page(min_pfn);
1407 cc->free_pfn = min_pfn;
1408 }
1409 }
1410 }
1411 }
1412
1413 if (highest && highest >= cc->zone->compact_cached_free_pfn) {
1414 highest -= pageblock_nr_pages;
1415 cc->zone->compact_cached_free_pfn = highest;
1416 }
1417
1418 cc->total_free_scanned += nr_scanned;
1419 if (!page)
1420 return cc->free_pfn;
1421
1422 low_pfn = page_to_pfn(page);
1423 fast_isolate_around(cc, low_pfn, nr_isolated);
1424 return low_pfn;
1425}
1426
1427
1428
1429
1430
1431static void isolate_freepages(struct compact_control *cc)
1432{
1433 struct zone *zone = cc->zone;
1434 struct page *page;
1435 unsigned long block_start_pfn;
1436 unsigned long isolate_start_pfn;
1437 unsigned long block_end_pfn;
1438 unsigned long low_pfn;
1439 struct list_head *freelist = &cc->freepages;
1440 unsigned int stride;
1441
1442
1443 isolate_start_pfn = fast_isolate_freepages(cc);
1444 if (cc->nr_freepages)
1445 goto splitmap;
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458 isolate_start_pfn = cc->free_pfn;
1459 block_start_pfn = pageblock_start_pfn(isolate_start_pfn);
1460 block_end_pfn = min(block_start_pfn + pageblock_nr_pages,
1461 zone_end_pfn(zone));
1462 low_pfn = pageblock_end_pfn(cc->migrate_pfn);
1463 stride = cc->mode == MIGRATE_ASYNC ? COMPACT_CLUSTER_MAX : 1;
1464
1465
1466
1467
1468
1469
1470 for (; block_start_pfn >= low_pfn;
1471 block_end_pfn = block_start_pfn,
1472 block_start_pfn -= pageblock_nr_pages,
1473 isolate_start_pfn = block_start_pfn) {
1474 unsigned long nr_isolated;
1475
1476
1477
1478
1479
1480 if (!(block_start_pfn % (SWAP_CLUSTER_MAX * pageblock_nr_pages)))
1481 cond_resched();
1482
1483 page = pageblock_pfn_to_page(block_start_pfn, block_end_pfn,
1484 zone);
1485 if (!page)
1486 continue;
1487
1488
1489 if (!suitable_migration_target(cc, page))
1490 continue;
1491
1492
1493 if (!isolation_suitable(cc, page))
1494 continue;
1495
1496
1497 nr_isolated = isolate_freepages_block(cc, &isolate_start_pfn,
1498 block_end_pfn, freelist, stride, false);
1499
1500
1501 if (isolate_start_pfn == block_end_pfn)
1502 update_pageblock_skip(cc, page, block_start_pfn);
1503
1504
1505 if (cc->nr_freepages >= cc->nr_migratepages) {
1506 if (isolate_start_pfn >= block_end_pfn) {
1507
1508
1509
1510
1511 isolate_start_pfn =
1512 block_start_pfn - pageblock_nr_pages;
1513 }
1514 break;
1515 } else if (isolate_start_pfn < block_end_pfn) {
1516
1517
1518
1519
1520 break;
1521 }
1522
1523
1524 if (nr_isolated) {
1525 stride = 1;
1526 continue;
1527 }
1528 stride = min_t(unsigned int, COMPACT_CLUSTER_MAX, stride << 1);
1529 }
1530
1531
1532
1533
1534
1535
1536
1537 cc->free_pfn = isolate_start_pfn;
1538
1539splitmap:
1540
1541 split_map_pages(freelist);
1542}
1543
1544
1545
1546
1547
1548static struct page *compaction_alloc(struct page *migratepage,
1549 unsigned long data)
1550{
1551 struct compact_control *cc = (struct compact_control *)data;
1552 struct page *freepage;
1553
1554 if (list_empty(&cc->freepages)) {
1555 isolate_freepages(cc);
1556
1557 if (list_empty(&cc->freepages))
1558 return NULL;
1559 }
1560
1561 freepage = list_entry(cc->freepages.next, struct page, lru);
1562 list_del(&freepage->lru);
1563 cc->nr_freepages--;
1564
1565 return freepage;
1566}
1567
1568
1569
1570
1571
1572
1573static void compaction_free(struct page *page, unsigned long data)
1574{
1575 struct compact_control *cc = (struct compact_control *)data;
1576
1577 list_add(&page->lru, &cc->freepages);
1578 cc->nr_freepages++;
1579}
1580
1581
1582typedef enum {
1583 ISOLATE_ABORT,
1584 ISOLATE_NONE,
1585 ISOLATE_SUCCESS,
1586} isolate_migrate_t;
1587
1588
1589
1590
1591
1592int sysctl_compact_unevictable_allowed __read_mostly = 1;
1593
1594static inline void
1595update_fast_start_pfn(struct compact_control *cc, unsigned long pfn)
1596{
1597 if (cc->fast_start_pfn == ULONG_MAX)
1598 return;
1599
1600 if (!cc->fast_start_pfn)
1601 cc->fast_start_pfn = pfn;
1602
1603 cc->fast_start_pfn = min(cc->fast_start_pfn, pfn);
1604}
1605
1606static inline unsigned long
1607reinit_migrate_pfn(struct compact_control *cc)
1608{
1609 if (!cc->fast_start_pfn || cc->fast_start_pfn == ULONG_MAX)
1610 return cc->migrate_pfn;
1611
1612 cc->migrate_pfn = cc->fast_start_pfn;
1613 cc->fast_start_pfn = ULONG_MAX;
1614
1615 return cc->migrate_pfn;
1616}
1617
1618
1619
1620
1621
1622
1623static unsigned long fast_find_migrateblock(struct compact_control *cc)
1624{
1625 unsigned int limit = freelist_scan_limit(cc);
1626 unsigned int nr_scanned = 0;
1627 unsigned long distance;
1628 unsigned long pfn = cc->migrate_pfn;
1629 unsigned long high_pfn;
1630 int order;
1631
1632
1633 if (cc->ignore_skip_hint)
1634 return pfn;
1635
1636
1637
1638
1639
1640
1641 if (pfn != cc->zone->zone_start_pfn && pfn != pageblock_start_pfn(pfn))
1642 return pfn;
1643
1644
1645
1646
1647
1648
1649 if (cc->order <= PAGE_ALLOC_COSTLY_ORDER)
1650 return pfn;
1651
1652
1653
1654
1655
1656
1657
1658 if (cc->direct_compaction && cc->migratetype != MIGRATE_MOVABLE)
1659 return pfn;
1660
1661
1662
1663
1664
1665
1666
1667 distance = (cc->free_pfn - cc->migrate_pfn) >> 1;
1668 if (cc->migrate_pfn != cc->zone->zone_start_pfn)
1669 distance >>= 2;
1670 high_pfn = pageblock_start_pfn(cc->migrate_pfn + distance);
1671
1672 for (order = cc->order - 1;
1673 order >= PAGE_ALLOC_COSTLY_ORDER && pfn == cc->migrate_pfn && nr_scanned < limit;
1674 order--) {
1675 struct free_area *area = &cc->zone->free_area[order];
1676 struct list_head *freelist;
1677 unsigned long flags;
1678 struct page *freepage;
1679
1680 if (!area->nr_free)
1681 continue;
1682
1683 spin_lock_irqsave(&cc->zone->lock, flags);
1684 freelist = &area->free_list[MIGRATE_MOVABLE];
1685 list_for_each_entry(freepage, freelist, lru) {
1686 unsigned long free_pfn;
1687
1688 nr_scanned++;
1689 free_pfn = page_to_pfn(freepage);
1690 if (free_pfn < high_pfn) {
1691
1692
1693
1694
1695
1696
1697 if (get_pageblock_skip(freepage)) {
1698 if (list_is_last(freelist, &freepage->lru))
1699 break;
1700
1701 continue;
1702 }
1703
1704
1705 move_freelist_tail(freelist, freepage);
1706
1707 update_fast_start_pfn(cc, free_pfn);
1708 pfn = pageblock_start_pfn(free_pfn);
1709 cc->fast_search_fail = 0;
1710 set_pageblock_skip(freepage);
1711 break;
1712 }
1713
1714 if (nr_scanned >= limit) {
1715 cc->fast_search_fail++;
1716 move_freelist_tail(freelist, freepage);
1717 break;
1718 }
1719 }
1720 spin_unlock_irqrestore(&cc->zone->lock, flags);
1721 }
1722
1723 cc->total_migrate_scanned += nr_scanned;
1724
1725
1726
1727
1728
1729 if (pfn == cc->migrate_pfn)
1730 pfn = reinit_migrate_pfn(cc);
1731
1732 return pfn;
1733}
1734
1735
1736
1737
1738
1739
1740static isolate_migrate_t isolate_migratepages(struct zone *zone,
1741 struct compact_control *cc)
1742{
1743 unsigned long block_start_pfn;
1744 unsigned long block_end_pfn;
1745 unsigned long low_pfn;
1746 struct page *page;
1747 const isolate_mode_t isolate_mode =
1748 (sysctl_compact_unevictable_allowed ? ISOLATE_UNEVICTABLE : 0) |
1749 (cc->mode != MIGRATE_SYNC ? ISOLATE_ASYNC_MIGRATE : 0);
1750 bool fast_find_block;
1751
1752
1753
1754
1755
1756
1757 low_pfn = fast_find_migrateblock(cc);
1758 block_start_pfn = pageblock_start_pfn(low_pfn);
1759 if (block_start_pfn < zone->zone_start_pfn)
1760 block_start_pfn = zone->zone_start_pfn;
1761
1762
1763
1764
1765
1766
1767 fast_find_block = low_pfn != cc->migrate_pfn && !cc->fast_search_fail;
1768
1769
1770 block_end_pfn = pageblock_end_pfn(low_pfn);
1771
1772
1773
1774
1775
1776 for (; block_end_pfn <= cc->free_pfn;
1777 fast_find_block = false,
1778 low_pfn = block_end_pfn,
1779 block_start_pfn = block_end_pfn,
1780 block_end_pfn += pageblock_nr_pages) {
1781
1782
1783
1784
1785
1786
1787 if (!(low_pfn % (SWAP_CLUSTER_MAX * pageblock_nr_pages)))
1788 cond_resched();
1789
1790 page = pageblock_pfn_to_page(block_start_pfn, block_end_pfn,
1791 zone);
1792 if (!page)
1793 continue;
1794
1795
1796
1797
1798
1799
1800
1801
1802 if (IS_ALIGNED(low_pfn, pageblock_nr_pages) &&
1803 !fast_find_block && !isolation_suitable(cc, page))
1804 continue;
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814 if (!suitable_migration_source(cc, page)) {
1815 update_cached_migrate(cc, block_end_pfn);
1816 continue;
1817 }
1818
1819
1820 low_pfn = isolate_migratepages_block(cc, low_pfn,
1821 block_end_pfn, isolate_mode);
1822
1823 if (!low_pfn)
1824 return ISOLATE_ABORT;
1825
1826
1827
1828
1829
1830
1831 break;
1832 }
1833
1834
1835 cc->migrate_pfn = low_pfn;
1836
1837 return cc->nr_migratepages ? ISOLATE_SUCCESS : ISOLATE_NONE;
1838}
1839
1840
1841
1842
1843
1844static inline bool is_via_compact_memory(int order)
1845{
1846 return order == -1;
1847}
1848
1849static enum compact_result __compact_finished(struct compact_control *cc)
1850{
1851 unsigned int order;
1852 const int migratetype = cc->migratetype;
1853 int ret;
1854
1855
1856 if (compact_scanners_met(cc)) {
1857
1858 reset_cached_positions(cc->zone);
1859
1860
1861
1862
1863
1864
1865
1866 if (cc->direct_compaction)
1867 cc->zone->compact_blockskip_flush = true;
1868
1869 if (cc->whole_zone)
1870 return COMPACT_COMPLETE;
1871 else
1872 return COMPACT_PARTIAL_SKIPPED;
1873 }
1874
1875 if (is_via_compact_memory(cc->order))
1876 return COMPACT_CONTINUE;
1877
1878
1879
1880
1881
1882
1883
1884 if (!IS_ALIGNED(cc->migrate_pfn, pageblock_nr_pages))
1885 return COMPACT_CONTINUE;
1886
1887
1888 ret = COMPACT_NO_SUITABLE_PAGE;
1889 for (order = cc->order; order < MAX_ORDER; order++) {
1890 struct free_area *area = &cc->zone->free_area[order];
1891 bool can_steal;
1892
1893
1894 if (!free_area_empty(area, migratetype))
1895 return COMPACT_SUCCESS;
1896
1897#ifdef CONFIG_CMA
1898
1899 if (migratetype == MIGRATE_MOVABLE &&
1900 !free_area_empty(area, MIGRATE_CMA))
1901 return COMPACT_SUCCESS;
1902#endif
1903
1904
1905
1906
1907 if (find_suitable_fallback(area, order, migratetype,
1908 true, &can_steal) != -1) {
1909
1910
1911 if (migratetype == MIGRATE_MOVABLE)
1912 return COMPACT_SUCCESS;
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922 if (cc->mode == MIGRATE_ASYNC ||
1923 IS_ALIGNED(cc->migrate_pfn,
1924 pageblock_nr_pages)) {
1925 return COMPACT_SUCCESS;
1926 }
1927
1928 ret = COMPACT_CONTINUE;
1929 break;
1930 }
1931 }
1932
1933 if (cc->contended || fatal_signal_pending(current))
1934 ret = COMPACT_CONTENDED;
1935
1936 return ret;
1937}
1938
1939static enum compact_result compact_finished(struct compact_control *cc)
1940{
1941 int ret;
1942
1943 ret = __compact_finished(cc);
1944 trace_mm_compaction_finished(cc->zone, cc->order, ret);
1945 if (ret == COMPACT_NO_SUITABLE_PAGE)
1946 ret = COMPACT_CONTINUE;
1947
1948 return ret;
1949}
1950
1951
1952
1953
1954
1955
1956
1957
1958static enum compact_result __compaction_suitable(struct zone *zone, int order,
1959 unsigned int alloc_flags,
1960 int classzone_idx,
1961 unsigned long wmark_target)
1962{
1963 unsigned long watermark;
1964
1965 if (is_via_compact_memory(order))
1966 return COMPACT_CONTINUE;
1967
1968 watermark = wmark_pages(zone, alloc_flags & ALLOC_WMARK_MASK);
1969
1970
1971
1972
1973 if (zone_watermark_ok(zone, order, watermark, classzone_idx,
1974 alloc_flags))
1975 return COMPACT_SUCCESS;
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991 watermark = (order > PAGE_ALLOC_COSTLY_ORDER) ?
1992 low_wmark_pages(zone) : min_wmark_pages(zone);
1993 watermark += compact_gap(order);
1994 if (!__zone_watermark_ok(zone, 0, watermark, classzone_idx,
1995 ALLOC_CMA, wmark_target))
1996 return COMPACT_SKIPPED;
1997
1998 return COMPACT_CONTINUE;
1999}
2000
2001enum compact_result compaction_suitable(struct zone *zone, int order,
2002 unsigned int alloc_flags,
2003 int classzone_idx)
2004{
2005 enum compact_result ret;
2006 int fragindex;
2007
2008 ret = __compaction_suitable(zone, order, alloc_flags, classzone_idx,
2009 zone_page_state(zone, NR_FREE_PAGES));
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026 if (ret == COMPACT_CONTINUE && (order > PAGE_ALLOC_COSTLY_ORDER)) {
2027 fragindex = fragmentation_index(zone, order);
2028 if (fragindex >= 0 && fragindex <= sysctl_extfrag_threshold)
2029 ret = COMPACT_NOT_SUITABLE_ZONE;
2030 }
2031
2032 trace_mm_compaction_suitable(zone, order, ret);
2033 if (ret == COMPACT_NOT_SUITABLE_ZONE)
2034 ret = COMPACT_SKIPPED;
2035
2036 return ret;
2037}
2038
2039bool compaction_zonelist_suitable(struct alloc_context *ac, int order,
2040 int alloc_flags)
2041{
2042 struct zone *zone;
2043 struct zoneref *z;
2044
2045
2046
2047
2048
2049 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx,
2050 ac->nodemask) {
2051 unsigned long available;
2052 enum compact_result compact_result;
2053
2054
2055
2056
2057
2058
2059
2060 available = zone_reclaimable_pages(zone) / order;
2061 available += zone_page_state_snapshot(zone, NR_FREE_PAGES);
2062 compact_result = __compaction_suitable(zone, order, alloc_flags,
2063 ac_classzone_idx(ac), available);
2064 if (compact_result != COMPACT_SKIPPED)
2065 return true;
2066 }
2067
2068 return false;
2069}
2070
2071static enum compact_result
2072compact_zone(struct compact_control *cc, struct capture_control *capc)
2073{
2074 enum compact_result ret;
2075 unsigned long start_pfn = cc->zone->zone_start_pfn;
2076 unsigned long end_pfn = zone_end_pfn(cc->zone);
2077 unsigned long last_migrated_pfn;
2078 const bool sync = cc->mode != MIGRATE_ASYNC;
2079 bool update_cached;
2080
2081 cc->migratetype = gfpflags_to_migratetype(cc->gfp_mask);
2082 ret = compaction_suitable(cc->zone, cc->order, cc->alloc_flags,
2083 cc->classzone_idx);
2084
2085 if (ret == COMPACT_SUCCESS || ret == COMPACT_SKIPPED)
2086 return ret;
2087
2088
2089 VM_BUG_ON(ret != COMPACT_CONTINUE);
2090
2091
2092
2093
2094
2095 if (compaction_restarting(cc->zone, cc->order))
2096 __reset_isolation_suitable(cc->zone);
2097
2098
2099
2100
2101
2102
2103
2104 cc->fast_start_pfn = 0;
2105 if (cc->whole_zone) {
2106 cc->migrate_pfn = start_pfn;
2107 cc->free_pfn = pageblock_start_pfn(end_pfn - 1);
2108 } else {
2109 cc->migrate_pfn = cc->zone->compact_cached_migrate_pfn[sync];
2110 cc->free_pfn = cc->zone->compact_cached_free_pfn;
2111 if (cc->free_pfn < start_pfn || cc->free_pfn >= end_pfn) {
2112 cc->free_pfn = pageblock_start_pfn(end_pfn - 1);
2113 cc->zone->compact_cached_free_pfn = cc->free_pfn;
2114 }
2115 if (cc->migrate_pfn < start_pfn || cc->migrate_pfn >= end_pfn) {
2116 cc->migrate_pfn = start_pfn;
2117 cc->zone->compact_cached_migrate_pfn[0] = cc->migrate_pfn;
2118 cc->zone->compact_cached_migrate_pfn[1] = cc->migrate_pfn;
2119 }
2120
2121 if (cc->migrate_pfn <= cc->zone->compact_init_migrate_pfn)
2122 cc->whole_zone = true;
2123 }
2124
2125 last_migrated_pfn = 0;
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135 update_cached = !sync &&
2136 cc->zone->compact_cached_migrate_pfn[0] == cc->zone->compact_cached_migrate_pfn[1];
2137
2138 trace_mm_compaction_begin(start_pfn, cc->migrate_pfn,
2139 cc->free_pfn, end_pfn, sync);
2140
2141 migrate_prep_local();
2142
2143 while ((ret = compact_finished(cc)) == COMPACT_CONTINUE) {
2144 int err;
2145 unsigned long start_pfn = cc->migrate_pfn;
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155 cc->rescan = false;
2156 if (pageblock_start_pfn(last_migrated_pfn) ==
2157 pageblock_start_pfn(start_pfn)) {
2158 cc->rescan = true;
2159 }
2160
2161 switch (isolate_migratepages(cc->zone, cc)) {
2162 case ISOLATE_ABORT:
2163 ret = COMPACT_CONTENDED;
2164 putback_movable_pages(&cc->migratepages);
2165 cc->nr_migratepages = 0;
2166 last_migrated_pfn = 0;
2167 goto out;
2168 case ISOLATE_NONE:
2169 if (update_cached) {
2170 cc->zone->compact_cached_migrate_pfn[1] =
2171 cc->zone->compact_cached_migrate_pfn[0];
2172 }
2173
2174
2175
2176
2177
2178
2179 goto check_drain;
2180 case ISOLATE_SUCCESS:
2181 update_cached = false;
2182 last_migrated_pfn = start_pfn;
2183 ;
2184 }
2185
2186 err = migrate_pages(&cc->migratepages, compaction_alloc,
2187 compaction_free, (unsigned long)cc, cc->mode,
2188 MR_COMPACTION);
2189
2190 trace_mm_compaction_migratepages(cc->nr_migratepages, err,
2191 &cc->migratepages);
2192
2193
2194 cc->nr_migratepages = 0;
2195 if (err) {
2196 putback_movable_pages(&cc->migratepages);
2197
2198
2199
2200
2201 if (err == -ENOMEM && !compact_scanners_met(cc)) {
2202 ret = COMPACT_CONTENDED;
2203 goto out;
2204 }
2205
2206
2207
2208
2209 if (cc->direct_compaction &&
2210 (cc->mode == MIGRATE_ASYNC)) {
2211 cc->migrate_pfn = block_end_pfn(
2212 cc->migrate_pfn - 1, cc->order);
2213
2214 last_migrated_pfn = 0;
2215 }
2216 }
2217
2218check_drain:
2219
2220
2221
2222
2223
2224
2225
2226 if (cc->order > 0 && last_migrated_pfn) {
2227 int cpu;
2228 unsigned long current_block_start =
2229 block_start_pfn(cc->migrate_pfn, cc->order);
2230
2231 if (last_migrated_pfn < current_block_start) {
2232 cpu = get_cpu();
2233 lru_add_drain_cpu(cpu);
2234 drain_local_pages(cc->zone);
2235 put_cpu();
2236
2237 last_migrated_pfn = 0;
2238 }
2239 }
2240
2241
2242 if (capc && capc->page) {
2243 ret = COMPACT_SUCCESS;
2244 break;
2245 }
2246 }
2247
2248out:
2249
2250
2251
2252
2253 if (cc->nr_freepages > 0) {
2254 unsigned long free_pfn = release_freepages(&cc->freepages);
2255
2256 cc->nr_freepages = 0;
2257 VM_BUG_ON(free_pfn == 0);
2258
2259 free_pfn = pageblock_start_pfn(free_pfn);
2260
2261
2262
2263
2264 if (free_pfn > cc->zone->compact_cached_free_pfn)
2265 cc->zone->compact_cached_free_pfn = free_pfn;
2266 }
2267
2268 count_compact_events(COMPACTMIGRATE_SCANNED, cc->total_migrate_scanned);
2269 count_compact_events(COMPACTFREE_SCANNED, cc->total_free_scanned);
2270
2271 trace_mm_compaction_end(start_pfn, cc->migrate_pfn,
2272 cc->free_pfn, end_pfn, sync, ret);
2273
2274 return ret;
2275}
2276
2277static enum compact_result compact_zone_order(struct zone *zone, int order,
2278 gfp_t gfp_mask, enum compact_priority prio,
2279 unsigned int alloc_flags, int classzone_idx,
2280 struct page **capture)
2281{
2282 enum compact_result ret;
2283 struct compact_control cc = {
2284 .nr_freepages = 0,
2285 .nr_migratepages = 0,
2286 .total_migrate_scanned = 0,
2287 .total_free_scanned = 0,
2288 .order = order,
2289 .search_order = order,
2290 .gfp_mask = gfp_mask,
2291 .zone = zone,
2292 .mode = (prio == COMPACT_PRIO_ASYNC) ?
2293 MIGRATE_ASYNC : MIGRATE_SYNC_LIGHT,
2294 .alloc_flags = alloc_flags,
2295 .classzone_idx = classzone_idx,
2296 .direct_compaction = true,
2297 .whole_zone = (prio == MIN_COMPACT_PRIORITY),
2298 .ignore_skip_hint = (prio == MIN_COMPACT_PRIORITY),
2299 .ignore_block_suitable = (prio == MIN_COMPACT_PRIORITY)
2300 };
2301 struct capture_control capc = {
2302 .cc = &cc,
2303 .page = NULL,
2304 };
2305
2306 if (capture)
2307 current->capture_control = &capc;
2308 INIT_LIST_HEAD(&cc.freepages);
2309 INIT_LIST_HEAD(&cc.migratepages);
2310
2311 ret = compact_zone(&cc, &capc);
2312
2313 VM_BUG_ON(!list_empty(&cc.freepages));
2314 VM_BUG_ON(!list_empty(&cc.migratepages));
2315
2316 *capture = capc.page;
2317 current->capture_control = NULL;
2318
2319 return ret;
2320}
2321
2322int sysctl_extfrag_threshold = 500;
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334enum compact_result try_to_compact_pages(gfp_t gfp_mask, unsigned int order,
2335 unsigned int alloc_flags, const struct alloc_context *ac,
2336 enum compact_priority prio, struct page **capture)
2337{
2338 int may_perform_io = gfp_mask & __GFP_IO;
2339 struct zoneref *z;
2340 struct zone *zone;
2341 enum compact_result rc = COMPACT_SKIPPED;
2342
2343
2344
2345
2346
2347 if (!may_perform_io)
2348 return COMPACT_SKIPPED;
2349
2350 trace_mm_compaction_try_to_compact_pages(order, gfp_mask, prio);
2351
2352
2353 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx,
2354 ac->nodemask) {
2355 enum compact_result status;
2356
2357 if (prio > MIN_COMPACT_PRIORITY
2358 && compaction_deferred(zone, order)) {
2359 rc = max_t(enum compact_result, COMPACT_DEFERRED, rc);
2360 continue;
2361 }
2362
2363 status = compact_zone_order(zone, order, gfp_mask, prio,
2364 alloc_flags, ac_classzone_idx(ac), capture);
2365 rc = max(status, rc);
2366
2367
2368 if (status == COMPACT_SUCCESS) {
2369
2370
2371
2372
2373
2374
2375 compaction_defer_reset(zone, order, false);
2376
2377 break;
2378 }
2379
2380 if (prio != COMPACT_PRIO_ASYNC && (status == COMPACT_COMPLETE ||
2381 status == COMPACT_PARTIAL_SKIPPED))
2382
2383
2384
2385
2386
2387 defer_compaction(zone, order);
2388
2389
2390
2391
2392
2393
2394 if ((prio == COMPACT_PRIO_ASYNC && need_resched())
2395 || fatal_signal_pending(current))
2396 break;
2397 }
2398
2399 return rc;
2400}
2401
2402
2403
2404static void compact_node(int nid)
2405{
2406 pg_data_t *pgdat = NODE_DATA(nid);
2407 int zoneid;
2408 struct zone *zone;
2409 struct compact_control cc = {
2410 .order = -1,
2411 .total_migrate_scanned = 0,
2412 .total_free_scanned = 0,
2413 .mode = MIGRATE_SYNC,
2414 .ignore_skip_hint = true,
2415 .whole_zone = true,
2416 .gfp_mask = GFP_KERNEL,
2417 };
2418
2419
2420 for (zoneid = 0; zoneid < MAX_NR_ZONES; zoneid++) {
2421
2422 zone = &pgdat->node_zones[zoneid];
2423 if (!populated_zone(zone))
2424 continue;
2425
2426 cc.nr_freepages = 0;
2427 cc.nr_migratepages = 0;
2428 cc.zone = zone;
2429 INIT_LIST_HEAD(&cc.freepages);
2430 INIT_LIST_HEAD(&cc.migratepages);
2431
2432 compact_zone(&cc, NULL);
2433
2434 VM_BUG_ON(!list_empty(&cc.freepages));
2435 VM_BUG_ON(!list_empty(&cc.migratepages));
2436 }
2437}
2438
2439
2440static void compact_nodes(void)
2441{
2442 int nid;
2443
2444
2445 lru_add_drain_all();
2446
2447 for_each_online_node(nid)
2448 compact_node(nid);
2449}
2450
2451
2452int sysctl_compact_memory;
2453
2454
2455
2456
2457
2458int sysctl_compaction_handler(struct ctl_table *table, int write,
2459 void __user *buffer, size_t *length, loff_t *ppos)
2460{
2461 if (write)
2462 compact_nodes();
2463
2464 return 0;
2465}
2466
2467#if defined(CONFIG_SYSFS) && defined(CONFIG_NUMA)
2468static ssize_t sysfs_compact_node(struct device *dev,
2469 struct device_attribute *attr,
2470 const char *buf, size_t count)
2471{
2472 int nid = dev->id;
2473
2474 if (nid >= 0 && nid < nr_node_ids && node_online(nid)) {
2475
2476 lru_add_drain_all();
2477
2478 compact_node(nid);
2479 }
2480
2481 return count;
2482}
2483static DEVICE_ATTR(compact, 0200, NULL, sysfs_compact_node);
2484
2485int compaction_register_node(struct node *node)
2486{
2487 return device_create_file(&node->dev, &dev_attr_compact);
2488}
2489
2490void compaction_unregister_node(struct node *node)
2491{
2492 return device_remove_file(&node->dev, &dev_attr_compact);
2493}
2494#endif
2495
2496static inline bool kcompactd_work_requested(pg_data_t *pgdat)
2497{
2498 return pgdat->kcompactd_max_order > 0 || kthread_should_stop();
2499}
2500
2501static bool kcompactd_node_suitable(pg_data_t *pgdat)
2502{
2503 int zoneid;
2504 struct zone *zone;
2505 enum zone_type classzone_idx = pgdat->kcompactd_classzone_idx;
2506
2507 for (zoneid = 0; zoneid <= classzone_idx; zoneid++) {
2508 zone = &pgdat->node_zones[zoneid];
2509
2510 if (!populated_zone(zone))
2511 continue;
2512
2513 if (compaction_suitable(zone, pgdat->kcompactd_max_order, 0,
2514 classzone_idx) == COMPACT_CONTINUE)
2515 return true;
2516 }
2517
2518 return false;
2519}
2520
2521static void kcompactd_do_work(pg_data_t *pgdat)
2522{
2523
2524
2525
2526
2527 int zoneid;
2528 struct zone *zone;
2529 struct compact_control cc = {
2530 .order = pgdat->kcompactd_max_order,
2531 .search_order = pgdat->kcompactd_max_order,
2532 .total_migrate_scanned = 0,
2533 .total_free_scanned = 0,
2534 .classzone_idx = pgdat->kcompactd_classzone_idx,
2535 .mode = MIGRATE_SYNC_LIGHT,
2536 .ignore_skip_hint = false,
2537 .gfp_mask = GFP_KERNEL,
2538 };
2539 trace_mm_compaction_kcompactd_wake(pgdat->node_id, cc.order,
2540 cc.classzone_idx);
2541 count_compact_event(KCOMPACTD_WAKE);
2542
2543 for (zoneid = 0; zoneid <= cc.classzone_idx; zoneid++) {
2544 int status;
2545
2546 zone = &pgdat->node_zones[zoneid];
2547 if (!populated_zone(zone))
2548 continue;
2549
2550 if (compaction_deferred(zone, cc.order))
2551 continue;
2552
2553 if (compaction_suitable(zone, cc.order, 0, zoneid) !=
2554 COMPACT_CONTINUE)
2555 continue;
2556
2557 cc.nr_freepages = 0;
2558 cc.nr_migratepages = 0;
2559 cc.total_migrate_scanned = 0;
2560 cc.total_free_scanned = 0;
2561 cc.zone = zone;
2562 INIT_LIST_HEAD(&cc.freepages);
2563 INIT_LIST_HEAD(&cc.migratepages);
2564
2565 if (kthread_should_stop())
2566 return;
2567 status = compact_zone(&cc, NULL);
2568
2569 if (status == COMPACT_SUCCESS) {
2570 compaction_defer_reset(zone, cc.order, false);
2571 } else if (status == COMPACT_PARTIAL_SKIPPED || status == COMPACT_COMPLETE) {
2572
2573
2574
2575
2576
2577
2578 drain_all_pages(zone);
2579
2580
2581
2582
2583
2584 defer_compaction(zone, cc.order);
2585 }
2586
2587 count_compact_events(KCOMPACTD_MIGRATE_SCANNED,
2588 cc.total_migrate_scanned);
2589 count_compact_events(KCOMPACTD_FREE_SCANNED,
2590 cc.total_free_scanned);
2591
2592 VM_BUG_ON(!list_empty(&cc.freepages));
2593 VM_BUG_ON(!list_empty(&cc.migratepages));
2594 }
2595
2596
2597
2598
2599
2600
2601 if (pgdat->kcompactd_max_order <= cc.order)
2602 pgdat->kcompactd_max_order = 0;
2603 if (pgdat->kcompactd_classzone_idx >= cc.classzone_idx)
2604 pgdat->kcompactd_classzone_idx = pgdat->nr_zones - 1;
2605}
2606
2607void wakeup_kcompactd(pg_data_t *pgdat, int order, int classzone_idx)
2608{
2609 if (!order)
2610 return;
2611
2612 if (pgdat->kcompactd_max_order < order)
2613 pgdat->kcompactd_max_order = order;
2614
2615 if (pgdat->kcompactd_classzone_idx > classzone_idx)
2616 pgdat->kcompactd_classzone_idx = classzone_idx;
2617
2618
2619
2620
2621
2622 if (!wq_has_sleeper(&pgdat->kcompactd_wait))
2623 return;
2624
2625 if (!kcompactd_node_suitable(pgdat))
2626 return;
2627
2628 trace_mm_compaction_wakeup_kcompactd(pgdat->node_id, order,
2629 classzone_idx);
2630 wake_up_interruptible(&pgdat->kcompactd_wait);
2631}
2632
2633
2634
2635
2636
2637static int kcompactd(void *p)
2638{
2639 pg_data_t *pgdat = (pg_data_t*)p;
2640 struct task_struct *tsk = current;
2641
2642 const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id);
2643
2644 if (!cpumask_empty(cpumask))
2645 set_cpus_allowed_ptr(tsk, cpumask);
2646
2647 set_freezable();
2648
2649 pgdat->kcompactd_max_order = 0;
2650 pgdat->kcompactd_classzone_idx = pgdat->nr_zones - 1;
2651
2652 while (!kthread_should_stop()) {
2653 unsigned long pflags;
2654
2655 trace_mm_compaction_kcompactd_sleep(pgdat->node_id);
2656 wait_event_freezable(pgdat->kcompactd_wait,
2657 kcompactd_work_requested(pgdat));
2658
2659 psi_memstall_enter(&pflags);
2660 kcompactd_do_work(pgdat);
2661 psi_memstall_leave(&pflags);
2662 }
2663
2664 return 0;
2665}
2666
2667
2668
2669
2670
2671int kcompactd_run(int nid)
2672{
2673 pg_data_t *pgdat = NODE_DATA(nid);
2674 int ret = 0;
2675
2676 if (pgdat->kcompactd)
2677 return 0;
2678
2679 pgdat->kcompactd = kthread_run(kcompactd, pgdat, "kcompactd%d", nid);
2680 if (IS_ERR(pgdat->kcompactd)) {
2681 pr_err("Failed to start kcompactd on node %d\n", nid);
2682 ret = PTR_ERR(pgdat->kcompactd);
2683 pgdat->kcompactd = NULL;
2684 }
2685 return ret;
2686}
2687
2688
2689
2690
2691
2692void kcompactd_stop(int nid)
2693{
2694 struct task_struct *kcompactd = NODE_DATA(nid)->kcompactd;
2695
2696 if (kcompactd) {
2697 kthread_stop(kcompactd);
2698 NODE_DATA(nid)->kcompactd = NULL;
2699 }
2700}
2701
2702
2703
2704
2705
2706
2707
2708static int kcompactd_cpu_online(unsigned int cpu)
2709{
2710 int nid;
2711
2712 for_each_node_state(nid, N_MEMORY) {
2713 pg_data_t *pgdat = NODE_DATA(nid);
2714 const struct cpumask *mask;
2715
2716 mask = cpumask_of_node(pgdat->node_id);
2717
2718 if (cpumask_any_and(cpu_online_mask, mask) < nr_cpu_ids)
2719
2720 set_cpus_allowed_ptr(pgdat->kcompactd, mask);
2721 }
2722 return 0;
2723}
2724
2725static int __init kcompactd_init(void)
2726{
2727 int nid;
2728 int ret;
2729
2730 ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
2731 "mm/compaction:online",
2732 kcompactd_cpu_online, NULL);
2733 if (ret < 0) {
2734 pr_err("kcompactd: failed to register hotplug callbacks.\n");
2735 return ret;
2736 }
2737
2738 for_each_node_state(nid, N_MEMORY)
2739 kcompactd_run(nid);
2740 return 0;
2741}
2742subsys_initcall(kcompactd_init)
2743
2744#endif
2745