1
2
3
4
5
6
7
8
9
10
11#include <linux/cpu.h>
12#include <linux/swap.h>
13#include <linux/migrate.h>
14#include <linux/compaction.h>
15#include <linux/mm_inline.h>
16#include <linux/sched/signal.h>
17#include <linux/backing-dev.h>
18#include <linux/sysctl.h>
19#include <linux/sysfs.h>
20#include <linux/page-isolation.h>
21#include <linux/kasan.h>
22#include <linux/kthread.h>
23#include <linux/freezer.h>
24#include <linux/page_owner.h>
25#include <linux/psi.h>
26#include "internal.h"
27
28#ifdef CONFIG_COMPACTION
29static inline void count_compact_event(enum vm_event_item item)
30{
31 count_vm_event(item);
32}
33
34static inline void count_compact_events(enum vm_event_item item, long delta)
35{
36 count_vm_events(item, delta);
37}
38#else
39#define count_compact_event(item) do { } while (0)
40#define count_compact_events(item, delta) do { } while (0)
41#endif
42
43#if defined CONFIG_COMPACTION || defined CONFIG_CMA
44
45#define CREATE_TRACE_POINTS
46#include <trace/events/compaction.h>
47
48#define block_start_pfn(pfn, order) round_down(pfn, 1UL << (order))
49#define block_end_pfn(pfn, order) ALIGN((pfn) + 1, 1UL << (order))
50#define pageblock_start_pfn(pfn) block_start_pfn(pfn, pageblock_order)
51#define pageblock_end_pfn(pfn) block_end_pfn(pfn, pageblock_order)
52
53static unsigned long release_freepages(struct list_head *freelist)
54{
55 struct page *page, *next;
56 unsigned long high_pfn = 0;
57
58 list_for_each_entry_safe(page, next, freelist, lru) {
59 unsigned long pfn = page_to_pfn(page);
60 list_del(&page->lru);
61 __free_page(page);
62 if (pfn > high_pfn)
63 high_pfn = pfn;
64 }
65
66 return high_pfn;
67}
68
69static void split_map_pages(struct list_head *list)
70{
71 unsigned int i, order, nr_pages;
72 struct page *page, *next;
73 LIST_HEAD(tmp_list);
74
75 list_for_each_entry_safe(page, next, list, lru) {
76 list_del(&page->lru);
77
78 order = page_private(page);
79 nr_pages = 1 << order;
80
81 post_alloc_hook(page, order, __GFP_MOVABLE);
82 if (order)
83 split_page(page, order);
84
85 for (i = 0; i < nr_pages; i++) {
86 list_add(&page->lru, &tmp_list);
87 page++;
88 }
89 }
90
91 list_splice(&tmp_list, list);
92}
93
94#ifdef CONFIG_COMPACTION
95
96int PageMovable(struct page *page)
97{
98 struct address_space *mapping;
99
100 VM_BUG_ON_PAGE(!PageLocked(page), page);
101 if (!__PageMovable(page))
102 return 0;
103
104 mapping = page_mapping(page);
105 if (mapping && mapping->a_ops && mapping->a_ops->isolate_page)
106 return 1;
107
108 return 0;
109}
110EXPORT_SYMBOL(PageMovable);
111
112void __SetPageMovable(struct page *page, struct address_space *mapping)
113{
114 VM_BUG_ON_PAGE(!PageLocked(page), page);
115 VM_BUG_ON_PAGE((unsigned long)mapping & PAGE_MAPPING_MOVABLE, page);
116 page->mapping = (void *)((unsigned long)mapping | PAGE_MAPPING_MOVABLE);
117}
118EXPORT_SYMBOL(__SetPageMovable);
119
120void __ClearPageMovable(struct page *page)
121{
122 VM_BUG_ON_PAGE(!PageLocked(page), page);
123 VM_BUG_ON_PAGE(!PageMovable(page), page);
124
125
126
127
128
129 page->mapping = (void *)((unsigned long)page->mapping &
130 PAGE_MAPPING_MOVABLE);
131}
132EXPORT_SYMBOL(__ClearPageMovable);
133
134
135#define COMPACT_MAX_DEFER_SHIFT 6
136
137
138
139
140
141
142void defer_compaction(struct zone *zone, int order)
143{
144 zone->compact_considered = 0;
145 zone->compact_defer_shift++;
146
147 if (order < zone->compact_order_failed)
148 zone->compact_order_failed = order;
149
150 if (zone->compact_defer_shift > COMPACT_MAX_DEFER_SHIFT)
151 zone->compact_defer_shift = COMPACT_MAX_DEFER_SHIFT;
152
153 trace_mm_compaction_defer_compaction(zone, order);
154}
155
156
157bool compaction_deferred(struct zone *zone, int order)
158{
159 unsigned long defer_limit = 1UL << zone->compact_defer_shift;
160
161 if (order < zone->compact_order_failed)
162 return false;
163
164
165 if (++zone->compact_considered > defer_limit)
166 zone->compact_considered = defer_limit;
167
168 if (zone->compact_considered >= defer_limit)
169 return false;
170
171 trace_mm_compaction_deferred(zone, order);
172
173 return true;
174}
175
176
177
178
179
180
181void compaction_defer_reset(struct zone *zone, int order,
182 bool alloc_success)
183{
184 if (alloc_success) {
185 zone->compact_considered = 0;
186 zone->compact_defer_shift = 0;
187 }
188 if (order >= zone->compact_order_failed)
189 zone->compact_order_failed = order + 1;
190
191 trace_mm_compaction_defer_reset(zone, order);
192}
193
194
195bool compaction_restarting(struct zone *zone, int order)
196{
197 if (order < zone->compact_order_failed)
198 return false;
199
200 return zone->compact_defer_shift == COMPACT_MAX_DEFER_SHIFT &&
201 zone->compact_considered >= 1UL << zone->compact_defer_shift;
202}
203
204
205static inline bool isolation_suitable(struct compact_control *cc,
206 struct page *page)
207{
208 if (cc->ignore_skip_hint)
209 return true;
210
211 return !get_pageblock_skip(page);
212}
213
214static void reset_cached_positions(struct zone *zone)
215{
216 zone->compact_cached_migrate_pfn[0] = zone->zone_start_pfn;
217 zone->compact_cached_migrate_pfn[1] = zone->zone_start_pfn;
218 zone->compact_cached_free_pfn =
219 pageblock_start_pfn(zone_end_pfn(zone) - 1);
220}
221
222
223
224
225
226
227static bool pageblock_skip_persistent(struct page *page)
228{
229 if (!PageCompound(page))
230 return false;
231
232 page = compound_head(page);
233
234 if (compound_order(page) >= pageblock_order)
235 return true;
236
237 return false;
238}
239
240static bool
241__reset_isolation_pfn(struct zone *zone, unsigned long pfn, bool check_source,
242 bool check_target)
243{
244 struct page *page = pfn_to_online_page(pfn);
245 struct page *block_page;
246 struct page *end_page;
247 unsigned long block_pfn;
248
249 if (!page)
250 return false;
251 if (zone != page_zone(page))
252 return false;
253 if (pageblock_skip_persistent(page))
254 return false;
255
256
257
258
259
260 if (check_source && check_target && !get_pageblock_skip(page))
261 return true;
262
263
264
265
266
267 if (!check_source && check_target &&
268 get_pageblock_migratetype(page) != MIGRATE_MOVABLE)
269 return false;
270
271
272 block_pfn = pageblock_start_pfn(pfn);
273 block_page = pfn_to_online_page(max(block_pfn, zone->zone_start_pfn));
274 if (block_page) {
275 page = block_page;
276 pfn = block_pfn;
277 }
278
279
280 block_pfn += pageblock_nr_pages;
281 block_pfn = min(block_pfn, zone_end_pfn(zone) - 1);
282 end_page = pfn_to_online_page(block_pfn);
283 if (!end_page)
284 return false;
285
286
287
288
289
290
291 do {
292 if (pfn_valid_within(pfn)) {
293 if (check_source && PageLRU(page)) {
294 clear_pageblock_skip(page);
295 return true;
296 }
297
298 if (check_target && PageBuddy(page)) {
299 clear_pageblock_skip(page);
300 return true;
301 }
302 }
303
304 page += (1 << PAGE_ALLOC_COSTLY_ORDER);
305 pfn += (1 << PAGE_ALLOC_COSTLY_ORDER);
306 } while (page < end_page);
307
308 return false;
309}
310
311
312
313
314
315
316static void __reset_isolation_suitable(struct zone *zone)
317{
318 unsigned long migrate_pfn = zone->zone_start_pfn;
319 unsigned long free_pfn = zone_end_pfn(zone) - 1;
320 unsigned long reset_migrate = free_pfn;
321 unsigned long reset_free = migrate_pfn;
322 bool source_set = false;
323 bool free_set = false;
324
325 if (!zone->compact_blockskip_flush)
326 return;
327
328 zone->compact_blockskip_flush = false;
329
330
331
332
333
334
335
336 for (; migrate_pfn < free_pfn; migrate_pfn += pageblock_nr_pages,
337 free_pfn -= pageblock_nr_pages) {
338 cond_resched();
339
340
341 if (__reset_isolation_pfn(zone, migrate_pfn, true, source_set) &&
342 migrate_pfn < reset_migrate) {
343 source_set = true;
344 reset_migrate = migrate_pfn;
345 zone->compact_init_migrate_pfn = reset_migrate;
346 zone->compact_cached_migrate_pfn[0] = reset_migrate;
347 zone->compact_cached_migrate_pfn[1] = reset_migrate;
348 }
349
350
351 if (__reset_isolation_pfn(zone, free_pfn, free_set, true) &&
352 free_pfn > reset_free) {
353 free_set = true;
354 reset_free = free_pfn;
355 zone->compact_init_free_pfn = reset_free;
356 zone->compact_cached_free_pfn = reset_free;
357 }
358 }
359
360
361 if (reset_migrate >= reset_free) {
362 zone->compact_cached_migrate_pfn[0] = migrate_pfn;
363 zone->compact_cached_migrate_pfn[1] = migrate_pfn;
364 zone->compact_cached_free_pfn = free_pfn;
365 }
366}
367
368void reset_isolation_suitable(pg_data_t *pgdat)
369{
370 int zoneid;
371
372 for (zoneid = 0; zoneid < MAX_NR_ZONES; zoneid++) {
373 struct zone *zone = &pgdat->node_zones[zoneid];
374 if (!populated_zone(zone))
375 continue;
376
377
378 if (zone->compact_blockskip_flush)
379 __reset_isolation_suitable(zone);
380 }
381}
382
383
384
385
386
387static bool test_and_set_skip(struct compact_control *cc, struct page *page,
388 unsigned long pfn)
389{
390 bool skip;
391
392
393 if (cc->ignore_skip_hint)
394 return false;
395
396 if (!IS_ALIGNED(pfn, pageblock_nr_pages))
397 return false;
398
399 skip = get_pageblock_skip(page);
400 if (!skip && !cc->no_set_skip_hint)
401 set_pageblock_skip(page);
402
403 return skip;
404}
405
406static void update_cached_migrate(struct compact_control *cc, unsigned long pfn)
407{
408 struct zone *zone = cc->zone;
409
410 pfn = pageblock_end_pfn(pfn);
411
412
413 if (cc->no_set_skip_hint)
414 return;
415
416 if (pfn > zone->compact_cached_migrate_pfn[0])
417 zone->compact_cached_migrate_pfn[0] = pfn;
418 if (cc->mode != MIGRATE_ASYNC &&
419 pfn > zone->compact_cached_migrate_pfn[1])
420 zone->compact_cached_migrate_pfn[1] = pfn;
421}
422
423
424
425
426
427static void update_pageblock_skip(struct compact_control *cc,
428 struct page *page, unsigned long pfn)
429{
430 struct zone *zone = cc->zone;
431
432 if (cc->no_set_skip_hint)
433 return;
434
435 if (!page)
436 return;
437
438 set_pageblock_skip(page);
439
440
441 if (pfn < zone->compact_cached_free_pfn)
442 zone->compact_cached_free_pfn = pfn;
443}
444#else
445static inline bool isolation_suitable(struct compact_control *cc,
446 struct page *page)
447{
448 return true;
449}
450
451static inline bool pageblock_skip_persistent(struct page *page)
452{
453 return false;
454}
455
456static inline void update_pageblock_skip(struct compact_control *cc,
457 struct page *page, unsigned long pfn)
458{
459}
460
461static void update_cached_migrate(struct compact_control *cc, unsigned long pfn)
462{
463}
464
465static bool test_and_set_skip(struct compact_control *cc, struct page *page,
466 unsigned long pfn)
467{
468 return false;
469}
470#endif
471
472
473
474
475
476
477
478
479
480
481static bool compact_lock_irqsave(spinlock_t *lock, unsigned long *flags,
482 struct compact_control *cc)
483{
484
485 if (cc->mode == MIGRATE_ASYNC && !cc->contended) {
486 if (spin_trylock_irqsave(lock, *flags))
487 return true;
488
489 cc->contended = true;
490 }
491
492 spin_lock_irqsave(lock, *flags);
493 return true;
494}
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511static bool compact_unlock_should_abort(spinlock_t *lock,
512 unsigned long flags, bool *locked, struct compact_control *cc)
513{
514 if (*locked) {
515 spin_unlock_irqrestore(lock, flags);
516 *locked = false;
517 }
518
519 if (fatal_signal_pending(current)) {
520 cc->contended = true;
521 return true;
522 }
523
524 cond_resched();
525
526 return false;
527}
528
529
530
531
532
533
534static unsigned long isolate_freepages_block(struct compact_control *cc,
535 unsigned long *start_pfn,
536 unsigned long end_pfn,
537 struct list_head *freelist,
538 unsigned int stride,
539 bool strict)
540{
541 int nr_scanned = 0, total_isolated = 0;
542 struct page *cursor;
543 unsigned long flags = 0;
544 bool locked = false;
545 unsigned long blockpfn = *start_pfn;
546 unsigned int order;
547
548
549 if (strict)
550 stride = 1;
551
552 cursor = pfn_to_page(blockpfn);
553
554
555 for (; blockpfn < end_pfn; blockpfn += stride, cursor += stride) {
556 int isolated;
557 struct page *page = cursor;
558
559
560
561
562
563
564 if (!(blockpfn % SWAP_CLUSTER_MAX)
565 && compact_unlock_should_abort(&cc->zone->lock, flags,
566 &locked, cc))
567 break;
568
569 nr_scanned++;
570 if (!pfn_valid_within(blockpfn))
571 goto isolate_fail;
572
573
574
575
576
577
578
579 if (PageCompound(page)) {
580 const unsigned int order = compound_order(page);
581
582 if (likely(order < MAX_ORDER)) {
583 blockpfn += (1UL << order) - 1;
584 cursor += (1UL << order) - 1;
585 }
586 goto isolate_fail;
587 }
588
589 if (!PageBuddy(page))
590 goto isolate_fail;
591
592
593
594
595
596
597
598
599 if (!locked) {
600 locked = compact_lock_irqsave(&cc->zone->lock,
601 &flags, cc);
602
603
604 if (!PageBuddy(page))
605 goto isolate_fail;
606 }
607
608
609 order = page_order(page);
610 isolated = __isolate_free_page(page, order);
611 if (!isolated)
612 break;
613 set_page_private(page, order);
614
615 total_isolated += isolated;
616 cc->nr_freepages += isolated;
617 list_add_tail(&page->lru, freelist);
618
619 if (!strict && cc->nr_migratepages <= cc->nr_freepages) {
620 blockpfn += isolated;
621 break;
622 }
623
624 blockpfn += isolated - 1;
625 cursor += isolated - 1;
626 continue;
627
628isolate_fail:
629 if (strict)
630 break;
631 else
632 continue;
633
634 }
635
636 if (locked)
637 spin_unlock_irqrestore(&cc->zone->lock, flags);
638
639
640
641
642
643 if (unlikely(blockpfn > end_pfn))
644 blockpfn = end_pfn;
645
646 trace_mm_compaction_isolate_freepages(*start_pfn, blockpfn,
647 nr_scanned, total_isolated);
648
649
650 *start_pfn = blockpfn;
651
652
653
654
655
656
657 if (strict && blockpfn < end_pfn)
658 total_isolated = 0;
659
660 cc->total_free_scanned += nr_scanned;
661 if (total_isolated)
662 count_compact_events(COMPACTISOLATED, total_isolated);
663 return total_isolated;
664}
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680unsigned long
681isolate_freepages_range(struct compact_control *cc,
682 unsigned long start_pfn, unsigned long end_pfn)
683{
684 unsigned long isolated, pfn, block_start_pfn, block_end_pfn;
685 LIST_HEAD(freelist);
686
687 pfn = start_pfn;
688 block_start_pfn = pageblock_start_pfn(pfn);
689 if (block_start_pfn < cc->zone->zone_start_pfn)
690 block_start_pfn = cc->zone->zone_start_pfn;
691 block_end_pfn = pageblock_end_pfn(pfn);
692
693 for (; pfn < end_pfn; pfn += isolated,
694 block_start_pfn = block_end_pfn,
695 block_end_pfn += pageblock_nr_pages) {
696
697 unsigned long isolate_start_pfn = pfn;
698
699 block_end_pfn = min(block_end_pfn, end_pfn);
700
701
702
703
704
705
706 if (pfn >= block_end_pfn) {
707 block_start_pfn = pageblock_start_pfn(pfn);
708 block_end_pfn = pageblock_end_pfn(pfn);
709 block_end_pfn = min(block_end_pfn, end_pfn);
710 }
711
712 if (!pageblock_pfn_to_page(block_start_pfn,
713 block_end_pfn, cc->zone))
714 break;
715
716 isolated = isolate_freepages_block(cc, &isolate_start_pfn,
717 block_end_pfn, &freelist, 0, true);
718
719
720
721
722
723
724 if (!isolated)
725 break;
726
727
728
729
730
731
732 }
733
734
735 split_map_pages(&freelist);
736
737 if (pfn < end_pfn) {
738
739 release_freepages(&freelist);
740 return 0;
741 }
742
743
744 return pfn;
745}
746
747
748static bool too_many_isolated(pg_data_t *pgdat)
749{
750 unsigned long active, inactive, isolated;
751
752 inactive = node_page_state(pgdat, NR_INACTIVE_FILE) +
753 node_page_state(pgdat, NR_INACTIVE_ANON);
754 active = node_page_state(pgdat, NR_ACTIVE_FILE) +
755 node_page_state(pgdat, NR_ACTIVE_ANON);
756 isolated = node_page_state(pgdat, NR_ISOLATED_FILE) +
757 node_page_state(pgdat, NR_ISOLATED_ANON);
758
759 return isolated > (inactive + active) / 2;
760}
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780static unsigned long
781isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
782 unsigned long end_pfn, isolate_mode_t isolate_mode)
783{
784 pg_data_t *pgdat = cc->zone->zone_pgdat;
785 unsigned long nr_scanned = 0, nr_isolated = 0;
786 struct lruvec *lruvec;
787 unsigned long flags = 0;
788 bool locked = false;
789 struct page *page = NULL, *valid_page = NULL;
790 unsigned long start_pfn = low_pfn;
791 bool skip_on_failure = false;
792 unsigned long next_skip_pfn = 0;
793 bool skip_updated = false;
794
795
796
797
798
799
800 while (unlikely(too_many_isolated(pgdat))) {
801
802 if (cc->mode == MIGRATE_ASYNC)
803 return 0;
804
805 congestion_wait(BLK_RW_ASYNC, HZ/10);
806
807 if (fatal_signal_pending(current))
808 return 0;
809 }
810
811 cond_resched();
812
813 if (cc->direct_compaction && (cc->mode == MIGRATE_ASYNC)) {
814 skip_on_failure = true;
815 next_skip_pfn = block_end_pfn(low_pfn, cc->order);
816 }
817
818
819 for (; low_pfn < end_pfn; low_pfn++) {
820
821 if (skip_on_failure && low_pfn >= next_skip_pfn) {
822
823
824
825
826
827
828 if (nr_isolated)
829 break;
830
831
832
833
834
835
836
837
838
839
840 next_skip_pfn = block_end_pfn(low_pfn, cc->order);
841 }
842
843
844
845
846
847
848 if (!(low_pfn % SWAP_CLUSTER_MAX)
849 && compact_unlock_should_abort(&pgdat->lru_lock,
850 flags, &locked, cc))
851 break;
852
853 if (!pfn_valid_within(low_pfn))
854 goto isolate_fail;
855 nr_scanned++;
856
857 page = pfn_to_page(low_pfn);
858
859
860
861
862
863
864
865 if (!valid_page && IS_ALIGNED(low_pfn, pageblock_nr_pages)) {
866 if (!cc->ignore_skip_hint && get_pageblock_skip(page)) {
867 low_pfn = end_pfn;
868 goto isolate_abort;
869 }
870 valid_page = page;
871 }
872
873
874
875
876
877
878
879 if (PageBuddy(page)) {
880 unsigned long freepage_order = page_order_unsafe(page);
881
882
883
884
885
886
887 if (freepage_order > 0 && freepage_order < MAX_ORDER)
888 low_pfn += (1UL << freepage_order) - 1;
889 continue;
890 }
891
892
893
894
895
896
897
898
899 if (PageCompound(page)) {
900 const unsigned int order = compound_order(page);
901
902 if (likely(order < MAX_ORDER))
903 low_pfn += (1UL << order) - 1;
904 goto isolate_fail;
905 }
906
907
908
909
910
911
912 if (!PageLRU(page)) {
913
914
915
916
917 if (unlikely(__PageMovable(page)) &&
918 !PageIsolated(page)) {
919 if (locked) {
920 spin_unlock_irqrestore(&pgdat->lru_lock,
921 flags);
922 locked = false;
923 }
924
925 if (!isolate_movable_page(page, isolate_mode))
926 goto isolate_success;
927 }
928
929 goto isolate_fail;
930 }
931
932
933
934
935
936
937 if (!page_mapping(page) &&
938 page_count(page) > page_mapcount(page))
939 goto isolate_fail;
940
941
942
943
944
945 if (!(cc->gfp_mask & __GFP_FS) && page_mapping(page))
946 goto isolate_fail;
947
948
949 if (!locked) {
950 locked = compact_lock_irqsave(&pgdat->lru_lock,
951 &flags, cc);
952
953
954 if (!skip_updated) {
955 skip_updated = true;
956 if (test_and_set_skip(cc, page, low_pfn))
957 goto isolate_abort;
958 }
959
960
961 if (!PageLRU(page))
962 goto isolate_fail;
963
964
965
966
967
968
969 if (unlikely(PageCompound(page))) {
970 low_pfn += (1UL << compound_order(page)) - 1;
971 goto isolate_fail;
972 }
973 }
974
975 lruvec = mem_cgroup_page_lruvec(page, pgdat);
976
977
978 if (__isolate_lru_page(page, isolate_mode) != 0)
979 goto isolate_fail;
980
981 VM_BUG_ON_PAGE(PageCompound(page), page);
982
983
984 del_page_from_lru_list(page, lruvec, page_lru(page));
985 inc_node_page_state(page,
986 NR_ISOLATED_ANON + page_is_file_cache(page));
987
988isolate_success:
989 list_add(&page->lru, &cc->migratepages);
990 cc->nr_migratepages++;
991 nr_isolated++;
992
993
994
995
996
997
998
999 if (cc->nr_migratepages == COMPACT_CLUSTER_MAX &&
1000 !cc->rescan && !cc->contended) {
1001 ++low_pfn;
1002 break;
1003 }
1004
1005 continue;
1006isolate_fail:
1007 if (!skip_on_failure)
1008 continue;
1009
1010
1011
1012
1013
1014
1015 if (nr_isolated) {
1016 if (locked) {
1017 spin_unlock_irqrestore(&pgdat->lru_lock, flags);
1018 locked = false;
1019 }
1020 putback_movable_pages(&cc->migratepages);
1021 cc->nr_migratepages = 0;
1022 nr_isolated = 0;
1023 }
1024
1025 if (low_pfn < next_skip_pfn) {
1026 low_pfn = next_skip_pfn - 1;
1027
1028
1029
1030
1031 next_skip_pfn += 1UL << cc->order;
1032 }
1033 }
1034
1035
1036
1037
1038
1039 if (unlikely(low_pfn > end_pfn))
1040 low_pfn = end_pfn;
1041
1042isolate_abort:
1043 if (locked)
1044 spin_unlock_irqrestore(&pgdat->lru_lock, flags);
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054 if (low_pfn == end_pfn && (!nr_isolated || cc->rescan)) {
1055 if (valid_page && !skip_updated)
1056 set_pageblock_skip(valid_page);
1057 update_cached_migrate(cc, low_pfn);
1058 }
1059
1060 trace_mm_compaction_isolate_migratepages(start_pfn, low_pfn,
1061 nr_scanned, nr_isolated);
1062
1063 cc->total_migrate_scanned += nr_scanned;
1064 if (nr_isolated)
1065 count_compact_events(COMPACTISOLATED, nr_isolated);
1066
1067 return low_pfn;
1068}
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080unsigned long
1081isolate_migratepages_range(struct compact_control *cc, unsigned long start_pfn,
1082 unsigned long end_pfn)
1083{
1084 unsigned long pfn, block_start_pfn, block_end_pfn;
1085
1086
1087 pfn = start_pfn;
1088 block_start_pfn = pageblock_start_pfn(pfn);
1089 if (block_start_pfn < cc->zone->zone_start_pfn)
1090 block_start_pfn = cc->zone->zone_start_pfn;
1091 block_end_pfn = pageblock_end_pfn(pfn);
1092
1093 for (; pfn < end_pfn; pfn = block_end_pfn,
1094 block_start_pfn = block_end_pfn,
1095 block_end_pfn += pageblock_nr_pages) {
1096
1097 block_end_pfn = min(block_end_pfn, end_pfn);
1098
1099 if (!pageblock_pfn_to_page(block_start_pfn,
1100 block_end_pfn, cc->zone))
1101 continue;
1102
1103 pfn = isolate_migratepages_block(cc, pfn, block_end_pfn,
1104 ISOLATE_UNEVICTABLE);
1105
1106 if (!pfn)
1107 break;
1108
1109 if (cc->nr_migratepages == COMPACT_CLUSTER_MAX)
1110 break;
1111 }
1112
1113 return pfn;
1114}
1115
1116#endif
1117#ifdef CONFIG_COMPACTION
1118
1119static bool suitable_migration_source(struct compact_control *cc,
1120 struct page *page)
1121{
1122 int block_mt;
1123
1124 if (pageblock_skip_persistent(page))
1125 return false;
1126
1127 if ((cc->mode != MIGRATE_ASYNC) || !cc->direct_compaction)
1128 return true;
1129
1130 block_mt = get_pageblock_migratetype(page);
1131
1132 if (cc->migratetype == MIGRATE_MOVABLE)
1133 return is_migrate_movable(block_mt);
1134 else
1135 return block_mt == cc->migratetype;
1136}
1137
1138
1139static bool suitable_migration_target(struct compact_control *cc,
1140 struct page *page)
1141{
1142
1143 if (PageBuddy(page)) {
1144
1145
1146
1147
1148
1149 if (page_order_unsafe(page) >= pageblock_order)
1150 return false;
1151 }
1152
1153 if (cc->ignore_block_suitable)
1154 return true;
1155
1156
1157 if (is_migrate_movable(get_pageblock_migratetype(page)))
1158 return true;
1159
1160
1161 return false;
1162}
1163
1164static inline unsigned int
1165freelist_scan_limit(struct compact_control *cc)
1166{
1167 unsigned short shift = BITS_PER_LONG - 1;
1168
1169 return (COMPACT_CLUSTER_MAX >> min(shift, cc->fast_search_fail)) + 1;
1170}
1171
1172
1173
1174
1175
1176static inline bool compact_scanners_met(struct compact_control *cc)
1177{
1178 return (cc->free_pfn >> pageblock_order)
1179 <= (cc->migrate_pfn >> pageblock_order);
1180}
1181
1182
1183
1184
1185
1186
1187static void
1188move_freelist_head(struct list_head *freelist, struct page *freepage)
1189{
1190 LIST_HEAD(sublist);
1191
1192 if (!list_is_last(freelist, &freepage->lru)) {
1193 list_cut_before(&sublist, freelist, &freepage->lru);
1194 if (!list_empty(&sublist))
1195 list_splice_tail(&sublist, freelist);
1196 }
1197}
1198
1199
1200
1201
1202
1203
1204
1205static void
1206move_freelist_tail(struct list_head *freelist, struct page *freepage)
1207{
1208 LIST_HEAD(sublist);
1209
1210 if (!list_is_first(freelist, &freepage->lru)) {
1211 list_cut_position(&sublist, freelist, &freepage->lru);
1212 if (!list_empty(&sublist))
1213 list_splice_tail(&sublist, freelist);
1214 }
1215}
1216
1217static void
1218fast_isolate_around(struct compact_control *cc, unsigned long pfn, unsigned long nr_isolated)
1219{
1220 unsigned long start_pfn, end_pfn;
1221 struct page *page = pfn_to_page(pfn);
1222
1223
1224 if (cc->nr_freepages >= cc->nr_migratepages)
1225 return;
1226
1227
1228 if (cc->direct_compaction && cc->mode == MIGRATE_ASYNC)
1229 return;
1230
1231
1232 start_pfn = pageblock_start_pfn(pfn);
1233 end_pfn = min(pageblock_end_pfn(pfn), zone_end_pfn(cc->zone)) - 1;
1234
1235
1236 if (start_pfn != pfn) {
1237 isolate_freepages_block(cc, &start_pfn, pfn, &cc->freepages, 1, false);
1238 if (cc->nr_freepages >= cc->nr_migratepages)
1239 return;
1240 }
1241
1242
1243 start_pfn = pfn + nr_isolated;
1244 if (start_pfn < end_pfn)
1245 isolate_freepages_block(cc, &start_pfn, end_pfn, &cc->freepages, 1, false);
1246
1247
1248 if (cc->nr_freepages < cc->nr_migratepages)
1249 set_pageblock_skip(page);
1250}
1251
1252
1253static int next_search_order(struct compact_control *cc, int order)
1254{
1255 order--;
1256 if (order < 0)
1257 order = cc->order - 1;
1258
1259
1260 if (order == cc->search_order) {
1261 cc->search_order--;
1262 if (cc->search_order < 0)
1263 cc->search_order = cc->order - 1;
1264 return -1;
1265 }
1266
1267 return order;
1268}
1269
1270static unsigned long
1271fast_isolate_freepages(struct compact_control *cc)
1272{
1273 unsigned int limit = min(1U, freelist_scan_limit(cc) >> 1);
1274 unsigned int nr_scanned = 0;
1275 unsigned long low_pfn, min_pfn, high_pfn = 0, highest = 0;
1276 unsigned long nr_isolated = 0;
1277 unsigned long distance;
1278 struct page *page = NULL;
1279 bool scan_start = false;
1280 int order;
1281
1282
1283 if (cc->order <= 0)
1284 return cc->free_pfn;
1285
1286
1287
1288
1289
1290 if (cc->free_pfn >= cc->zone->compact_init_free_pfn) {
1291 limit = pageblock_nr_pages >> 1;
1292 scan_start = true;
1293 }
1294
1295
1296
1297
1298
1299 distance = (cc->free_pfn - cc->migrate_pfn);
1300 low_pfn = pageblock_start_pfn(cc->free_pfn - (distance >> 2));
1301 min_pfn = pageblock_start_pfn(cc->free_pfn - (distance >> 1));
1302
1303 if (WARN_ON_ONCE(min_pfn > low_pfn))
1304 low_pfn = min_pfn;
1305
1306
1307
1308
1309
1310 cc->search_order = min_t(unsigned int, cc->order - 1, cc->search_order);
1311
1312 for (order = cc->search_order;
1313 !page && order >= 0;
1314 order = next_search_order(cc, order)) {
1315 struct free_area *area = &cc->zone->free_area[order];
1316 struct list_head *freelist;
1317 struct page *freepage;
1318 unsigned long flags;
1319 unsigned int order_scanned = 0;
1320
1321 if (!area->nr_free)
1322 continue;
1323
1324 spin_lock_irqsave(&cc->zone->lock, flags);
1325 freelist = &area->free_list[MIGRATE_MOVABLE];
1326 list_for_each_entry_reverse(freepage, freelist, lru) {
1327 unsigned long pfn;
1328
1329 order_scanned++;
1330 nr_scanned++;
1331 pfn = page_to_pfn(freepage);
1332
1333 if (pfn >= highest)
1334 highest = pageblock_start_pfn(pfn);
1335
1336 if (pfn >= low_pfn) {
1337 cc->fast_search_fail = 0;
1338 cc->search_order = order;
1339 page = freepage;
1340 break;
1341 }
1342
1343 if (pfn >= min_pfn && pfn > high_pfn) {
1344 high_pfn = pfn;
1345
1346
1347 limit >>= 1;
1348 }
1349
1350 if (order_scanned >= limit)
1351 break;
1352 }
1353
1354
1355 if (!page && high_pfn) {
1356 page = pfn_to_page(high_pfn);
1357
1358
1359 freepage = page;
1360 }
1361
1362
1363 move_freelist_head(freelist, freepage);
1364
1365
1366 if (page) {
1367 if (__isolate_free_page(page, order)) {
1368 set_page_private(page, order);
1369 nr_isolated = 1 << order;
1370 cc->nr_freepages += nr_isolated;
1371 list_add_tail(&page->lru, &cc->freepages);
1372 count_compact_events(COMPACTISOLATED, nr_isolated);
1373 } else {
1374
1375 order = cc->search_order + 1;
1376 page = NULL;
1377 }
1378 }
1379
1380 spin_unlock_irqrestore(&cc->zone->lock, flags);
1381
1382
1383
1384
1385
1386 if (order_scanned >= limit)
1387 limit = min(1U, limit >> 1);
1388 }
1389
1390 if (!page) {
1391 cc->fast_search_fail++;
1392 if (scan_start) {
1393
1394
1395
1396
1397
1398 if (highest) {
1399 page = pfn_to_page(highest);
1400 cc->free_pfn = highest;
1401 } else {
1402 if (cc->direct_compaction && pfn_valid(min_pfn)) {
1403 page = pfn_to_page(min_pfn);
1404 cc->free_pfn = min_pfn;
1405 }
1406 }
1407 }
1408 }
1409
1410 if (highest && highest >= cc->zone->compact_cached_free_pfn) {
1411 highest -= pageblock_nr_pages;
1412 cc->zone->compact_cached_free_pfn = highest;
1413 }
1414
1415 cc->total_free_scanned += nr_scanned;
1416 if (!page)
1417 return cc->free_pfn;
1418
1419 low_pfn = page_to_pfn(page);
1420 fast_isolate_around(cc, low_pfn, nr_isolated);
1421 return low_pfn;
1422}
1423
1424
1425
1426
1427
1428static void isolate_freepages(struct compact_control *cc)
1429{
1430 struct zone *zone = cc->zone;
1431 struct page *page;
1432 unsigned long block_start_pfn;
1433 unsigned long isolate_start_pfn;
1434 unsigned long block_end_pfn;
1435 unsigned long low_pfn;
1436 struct list_head *freelist = &cc->freepages;
1437 unsigned int stride;
1438
1439
1440 isolate_start_pfn = fast_isolate_freepages(cc);
1441 if (cc->nr_freepages)
1442 goto splitmap;
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455 isolate_start_pfn = cc->free_pfn;
1456 block_start_pfn = pageblock_start_pfn(isolate_start_pfn);
1457 block_end_pfn = min(block_start_pfn + pageblock_nr_pages,
1458 zone_end_pfn(zone));
1459 low_pfn = pageblock_end_pfn(cc->migrate_pfn);
1460 stride = cc->mode == MIGRATE_ASYNC ? COMPACT_CLUSTER_MAX : 1;
1461
1462
1463
1464
1465
1466
1467 for (; block_start_pfn >= low_pfn;
1468 block_end_pfn = block_start_pfn,
1469 block_start_pfn -= pageblock_nr_pages,
1470 isolate_start_pfn = block_start_pfn) {
1471 unsigned long nr_isolated;
1472
1473
1474
1475
1476
1477 if (!(block_start_pfn % (SWAP_CLUSTER_MAX * pageblock_nr_pages)))
1478 cond_resched();
1479
1480 page = pageblock_pfn_to_page(block_start_pfn, block_end_pfn,
1481 zone);
1482 if (!page)
1483 continue;
1484
1485
1486 if (!suitable_migration_target(cc, page))
1487 continue;
1488
1489
1490 if (!isolation_suitable(cc, page))
1491 continue;
1492
1493
1494 nr_isolated = isolate_freepages_block(cc, &isolate_start_pfn,
1495 block_end_pfn, freelist, stride, false);
1496
1497
1498 if (isolate_start_pfn == block_end_pfn)
1499 update_pageblock_skip(cc, page, block_start_pfn);
1500
1501
1502 if (cc->nr_freepages >= cc->nr_migratepages) {
1503 if (isolate_start_pfn >= block_end_pfn) {
1504
1505
1506
1507
1508 isolate_start_pfn =
1509 block_start_pfn - pageblock_nr_pages;
1510 }
1511 break;
1512 } else if (isolate_start_pfn < block_end_pfn) {
1513
1514
1515
1516
1517 break;
1518 }
1519
1520
1521 if (nr_isolated) {
1522 stride = 1;
1523 continue;
1524 }
1525 stride = min_t(unsigned int, COMPACT_CLUSTER_MAX, stride << 1);
1526 }
1527
1528
1529
1530
1531
1532
1533
1534 cc->free_pfn = isolate_start_pfn;
1535
1536splitmap:
1537
1538 split_map_pages(freelist);
1539}
1540
1541
1542
1543
1544
1545static struct page *compaction_alloc(struct page *migratepage,
1546 unsigned long data)
1547{
1548 struct compact_control *cc = (struct compact_control *)data;
1549 struct page *freepage;
1550
1551 if (list_empty(&cc->freepages)) {
1552 isolate_freepages(cc);
1553
1554 if (list_empty(&cc->freepages))
1555 return NULL;
1556 }
1557
1558 freepage = list_entry(cc->freepages.next, struct page, lru);
1559 list_del(&freepage->lru);
1560 cc->nr_freepages--;
1561
1562 return freepage;
1563}
1564
1565
1566
1567
1568
1569
1570static void compaction_free(struct page *page, unsigned long data)
1571{
1572 struct compact_control *cc = (struct compact_control *)data;
1573
1574 list_add(&page->lru, &cc->freepages);
1575 cc->nr_freepages++;
1576}
1577
1578
1579typedef enum {
1580 ISOLATE_ABORT,
1581 ISOLATE_NONE,
1582 ISOLATE_SUCCESS,
1583} isolate_migrate_t;
1584
1585
1586
1587
1588
1589int sysctl_compact_unevictable_allowed __read_mostly = 1;
1590
1591static inline void
1592update_fast_start_pfn(struct compact_control *cc, unsigned long pfn)
1593{
1594 if (cc->fast_start_pfn == ULONG_MAX)
1595 return;
1596
1597 if (!cc->fast_start_pfn)
1598 cc->fast_start_pfn = pfn;
1599
1600 cc->fast_start_pfn = min(cc->fast_start_pfn, pfn);
1601}
1602
1603static inline unsigned long
1604reinit_migrate_pfn(struct compact_control *cc)
1605{
1606 if (!cc->fast_start_pfn || cc->fast_start_pfn == ULONG_MAX)
1607 return cc->migrate_pfn;
1608
1609 cc->migrate_pfn = cc->fast_start_pfn;
1610 cc->fast_start_pfn = ULONG_MAX;
1611
1612 return cc->migrate_pfn;
1613}
1614
1615
1616
1617
1618
1619
1620static unsigned long fast_find_migrateblock(struct compact_control *cc)
1621{
1622 unsigned int limit = freelist_scan_limit(cc);
1623 unsigned int nr_scanned = 0;
1624 unsigned long distance;
1625 unsigned long pfn = cc->migrate_pfn;
1626 unsigned long high_pfn;
1627 int order;
1628
1629
1630 if (cc->ignore_skip_hint)
1631 return pfn;
1632
1633
1634
1635
1636
1637
1638 if (pfn != cc->zone->zone_start_pfn && pfn != pageblock_start_pfn(pfn))
1639 return pfn;
1640
1641
1642
1643
1644
1645
1646 if (cc->order <= PAGE_ALLOC_COSTLY_ORDER)
1647 return pfn;
1648
1649
1650
1651
1652
1653
1654
1655 if (cc->direct_compaction && cc->migratetype != MIGRATE_MOVABLE)
1656 return pfn;
1657
1658
1659
1660
1661
1662
1663
1664 distance = (cc->free_pfn - cc->migrate_pfn) >> 1;
1665 if (cc->migrate_pfn != cc->zone->zone_start_pfn)
1666 distance >>= 2;
1667 high_pfn = pageblock_start_pfn(cc->migrate_pfn + distance);
1668
1669 for (order = cc->order - 1;
1670 order >= PAGE_ALLOC_COSTLY_ORDER && pfn == cc->migrate_pfn && nr_scanned < limit;
1671 order--) {
1672 struct free_area *area = &cc->zone->free_area[order];
1673 struct list_head *freelist;
1674 unsigned long flags;
1675 struct page *freepage;
1676
1677 if (!area->nr_free)
1678 continue;
1679
1680 spin_lock_irqsave(&cc->zone->lock, flags);
1681 freelist = &area->free_list[MIGRATE_MOVABLE];
1682 list_for_each_entry(freepage, freelist, lru) {
1683 unsigned long free_pfn;
1684
1685 nr_scanned++;
1686 free_pfn = page_to_pfn(freepage);
1687 if (free_pfn < high_pfn) {
1688
1689
1690
1691
1692
1693
1694 if (get_pageblock_skip(freepage)) {
1695 if (list_is_last(freelist, &freepage->lru))
1696 break;
1697
1698 continue;
1699 }
1700
1701
1702 move_freelist_tail(freelist, freepage);
1703
1704 update_fast_start_pfn(cc, free_pfn);
1705 pfn = pageblock_start_pfn(free_pfn);
1706 cc->fast_search_fail = 0;
1707 set_pageblock_skip(freepage);
1708 break;
1709 }
1710
1711 if (nr_scanned >= limit) {
1712 cc->fast_search_fail++;
1713 move_freelist_tail(freelist, freepage);
1714 break;
1715 }
1716 }
1717 spin_unlock_irqrestore(&cc->zone->lock, flags);
1718 }
1719
1720 cc->total_migrate_scanned += nr_scanned;
1721
1722
1723
1724
1725
1726 if (pfn == cc->migrate_pfn)
1727 pfn = reinit_migrate_pfn(cc);
1728
1729 return pfn;
1730}
1731
1732
1733
1734
1735
1736
1737static isolate_migrate_t isolate_migratepages(struct zone *zone,
1738 struct compact_control *cc)
1739{
1740 unsigned long block_start_pfn;
1741 unsigned long block_end_pfn;
1742 unsigned long low_pfn;
1743 struct page *page;
1744 const isolate_mode_t isolate_mode =
1745 (sysctl_compact_unevictable_allowed ? ISOLATE_UNEVICTABLE : 0) |
1746 (cc->mode != MIGRATE_SYNC ? ISOLATE_ASYNC_MIGRATE : 0);
1747 bool fast_find_block;
1748
1749
1750
1751
1752
1753
1754 low_pfn = fast_find_migrateblock(cc);
1755 block_start_pfn = pageblock_start_pfn(low_pfn);
1756 if (block_start_pfn < zone->zone_start_pfn)
1757 block_start_pfn = zone->zone_start_pfn;
1758
1759
1760
1761
1762
1763
1764 fast_find_block = low_pfn != cc->migrate_pfn && !cc->fast_search_fail;
1765
1766
1767 block_end_pfn = pageblock_end_pfn(low_pfn);
1768
1769
1770
1771
1772
1773 for (; block_end_pfn <= cc->free_pfn;
1774 fast_find_block = false,
1775 low_pfn = block_end_pfn,
1776 block_start_pfn = block_end_pfn,
1777 block_end_pfn += pageblock_nr_pages) {
1778
1779
1780
1781
1782
1783
1784 if (!(low_pfn % (SWAP_CLUSTER_MAX * pageblock_nr_pages)))
1785 cond_resched();
1786
1787 page = pageblock_pfn_to_page(block_start_pfn, block_end_pfn,
1788 zone);
1789 if (!page)
1790 continue;
1791
1792
1793
1794
1795
1796
1797
1798
1799 if (IS_ALIGNED(low_pfn, pageblock_nr_pages) &&
1800 !fast_find_block && !isolation_suitable(cc, page))
1801 continue;
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811 if (!suitable_migration_source(cc, page)) {
1812 update_cached_migrate(cc, block_end_pfn);
1813 continue;
1814 }
1815
1816
1817 low_pfn = isolate_migratepages_block(cc, low_pfn,
1818 block_end_pfn, isolate_mode);
1819
1820 if (!low_pfn)
1821 return ISOLATE_ABORT;
1822
1823
1824
1825
1826
1827
1828 break;
1829 }
1830
1831
1832 cc->migrate_pfn = low_pfn;
1833
1834 return cc->nr_migratepages ? ISOLATE_SUCCESS : ISOLATE_NONE;
1835}
1836
1837
1838
1839
1840
1841static inline bool is_via_compact_memory(int order)
1842{
1843 return order == -1;
1844}
1845
1846static enum compact_result __compact_finished(struct compact_control *cc)
1847{
1848 unsigned int order;
1849 const int migratetype = cc->migratetype;
1850 int ret;
1851
1852
1853 if (compact_scanners_met(cc)) {
1854
1855 reset_cached_positions(cc->zone);
1856
1857
1858
1859
1860
1861
1862
1863 if (cc->direct_compaction)
1864 cc->zone->compact_blockskip_flush = true;
1865
1866 if (cc->whole_zone)
1867 return COMPACT_COMPLETE;
1868 else
1869 return COMPACT_PARTIAL_SKIPPED;
1870 }
1871
1872 if (is_via_compact_memory(cc->order))
1873 return COMPACT_CONTINUE;
1874
1875
1876
1877
1878
1879
1880
1881 if (!IS_ALIGNED(cc->migrate_pfn, pageblock_nr_pages))
1882 return COMPACT_CONTINUE;
1883
1884
1885 ret = COMPACT_NO_SUITABLE_PAGE;
1886 for (order = cc->order; order < MAX_ORDER; order++) {
1887 struct free_area *area = &cc->zone->free_area[order];
1888 bool can_steal;
1889
1890
1891 if (!free_area_empty(area, migratetype))
1892 return COMPACT_SUCCESS;
1893
1894#ifdef CONFIG_CMA
1895
1896 if (migratetype == MIGRATE_MOVABLE &&
1897 !free_area_empty(area, MIGRATE_CMA))
1898 return COMPACT_SUCCESS;
1899#endif
1900
1901
1902
1903
1904 if (find_suitable_fallback(area, order, migratetype,
1905 true, &can_steal) != -1) {
1906
1907
1908 if (migratetype == MIGRATE_MOVABLE)
1909 return COMPACT_SUCCESS;
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919 if (cc->mode == MIGRATE_ASYNC ||
1920 IS_ALIGNED(cc->migrate_pfn,
1921 pageblock_nr_pages)) {
1922 return COMPACT_SUCCESS;
1923 }
1924
1925 ret = COMPACT_CONTINUE;
1926 break;
1927 }
1928 }
1929
1930 if (cc->contended || fatal_signal_pending(current))
1931 ret = COMPACT_CONTENDED;
1932
1933 return ret;
1934}
1935
1936static enum compact_result compact_finished(struct compact_control *cc)
1937{
1938 int ret;
1939
1940 ret = __compact_finished(cc);
1941 trace_mm_compaction_finished(cc->zone, cc->order, ret);
1942 if (ret == COMPACT_NO_SUITABLE_PAGE)
1943 ret = COMPACT_CONTINUE;
1944
1945 return ret;
1946}
1947
1948
1949
1950
1951
1952
1953
1954
1955static enum compact_result __compaction_suitable(struct zone *zone, int order,
1956 unsigned int alloc_flags,
1957 int classzone_idx,
1958 unsigned long wmark_target)
1959{
1960 unsigned long watermark;
1961
1962 if (is_via_compact_memory(order))
1963 return COMPACT_CONTINUE;
1964
1965 watermark = wmark_pages(zone, alloc_flags & ALLOC_WMARK_MASK);
1966
1967
1968
1969
1970 if (zone_watermark_ok(zone, order, watermark, classzone_idx,
1971 alloc_flags))
1972 return COMPACT_SUCCESS;
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988 watermark = (order > PAGE_ALLOC_COSTLY_ORDER) ?
1989 low_wmark_pages(zone) : min_wmark_pages(zone);
1990 watermark += compact_gap(order);
1991 if (!__zone_watermark_ok(zone, 0, watermark, classzone_idx,
1992 ALLOC_CMA, wmark_target))
1993 return COMPACT_SKIPPED;
1994
1995 return COMPACT_CONTINUE;
1996}
1997
1998enum compact_result compaction_suitable(struct zone *zone, int order,
1999 unsigned int alloc_flags,
2000 int classzone_idx)
2001{
2002 enum compact_result ret;
2003 int fragindex;
2004
2005 ret = __compaction_suitable(zone, order, alloc_flags, classzone_idx,
2006 zone_page_state(zone, NR_FREE_PAGES));
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023 if (ret == COMPACT_CONTINUE && (order > PAGE_ALLOC_COSTLY_ORDER)) {
2024 fragindex = fragmentation_index(zone, order);
2025 if (fragindex >= 0 && fragindex <= sysctl_extfrag_threshold)
2026 ret = COMPACT_NOT_SUITABLE_ZONE;
2027 }
2028
2029 trace_mm_compaction_suitable(zone, order, ret);
2030 if (ret == COMPACT_NOT_SUITABLE_ZONE)
2031 ret = COMPACT_SKIPPED;
2032
2033 return ret;
2034}
2035
2036bool compaction_zonelist_suitable(struct alloc_context *ac, int order,
2037 int alloc_flags)
2038{
2039 struct zone *zone;
2040 struct zoneref *z;
2041
2042
2043
2044
2045
2046 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx,
2047 ac->nodemask) {
2048 unsigned long available;
2049 enum compact_result compact_result;
2050
2051
2052
2053
2054
2055
2056
2057 available = zone_reclaimable_pages(zone) / order;
2058 available += zone_page_state_snapshot(zone, NR_FREE_PAGES);
2059 compact_result = __compaction_suitable(zone, order, alloc_flags,
2060 ac_classzone_idx(ac), available);
2061 if (compact_result != COMPACT_SKIPPED)
2062 return true;
2063 }
2064
2065 return false;
2066}
2067
2068static enum compact_result
2069compact_zone(struct compact_control *cc, struct capture_control *capc)
2070{
2071 enum compact_result ret;
2072 unsigned long start_pfn = cc->zone->zone_start_pfn;
2073 unsigned long end_pfn = zone_end_pfn(cc->zone);
2074 unsigned long last_migrated_pfn;
2075 const bool sync = cc->mode != MIGRATE_ASYNC;
2076 bool update_cached;
2077
2078 cc->migratetype = gfpflags_to_migratetype(cc->gfp_mask);
2079 ret = compaction_suitable(cc->zone, cc->order, cc->alloc_flags,
2080 cc->classzone_idx);
2081
2082 if (ret == COMPACT_SUCCESS || ret == COMPACT_SKIPPED)
2083 return ret;
2084
2085
2086 VM_BUG_ON(ret != COMPACT_CONTINUE);
2087
2088
2089
2090
2091
2092 if (compaction_restarting(cc->zone, cc->order))
2093 __reset_isolation_suitable(cc->zone);
2094
2095
2096
2097
2098
2099
2100
2101 cc->fast_start_pfn = 0;
2102 if (cc->whole_zone) {
2103 cc->migrate_pfn = start_pfn;
2104 cc->free_pfn = pageblock_start_pfn(end_pfn - 1);
2105 } else {
2106 cc->migrate_pfn = cc->zone->compact_cached_migrate_pfn[sync];
2107 cc->free_pfn = cc->zone->compact_cached_free_pfn;
2108 if (cc->free_pfn < start_pfn || cc->free_pfn >= end_pfn) {
2109 cc->free_pfn = pageblock_start_pfn(end_pfn - 1);
2110 cc->zone->compact_cached_free_pfn = cc->free_pfn;
2111 }
2112 if (cc->migrate_pfn < start_pfn || cc->migrate_pfn >= end_pfn) {
2113 cc->migrate_pfn = start_pfn;
2114 cc->zone->compact_cached_migrate_pfn[0] = cc->migrate_pfn;
2115 cc->zone->compact_cached_migrate_pfn[1] = cc->migrate_pfn;
2116 }
2117
2118 if (cc->migrate_pfn <= cc->zone->compact_init_migrate_pfn)
2119 cc->whole_zone = true;
2120 }
2121
2122 last_migrated_pfn = 0;
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132 update_cached = !sync &&
2133 cc->zone->compact_cached_migrate_pfn[0] == cc->zone->compact_cached_migrate_pfn[1];
2134
2135 trace_mm_compaction_begin(start_pfn, cc->migrate_pfn,
2136 cc->free_pfn, end_pfn, sync);
2137
2138 migrate_prep_local();
2139
2140 while ((ret = compact_finished(cc)) == COMPACT_CONTINUE) {
2141 int err;
2142 unsigned long start_pfn = cc->migrate_pfn;
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152 cc->rescan = false;
2153 if (pageblock_start_pfn(last_migrated_pfn) ==
2154 pageblock_start_pfn(start_pfn)) {
2155 cc->rescan = true;
2156 }
2157
2158 switch (isolate_migratepages(cc->zone, cc)) {
2159 case ISOLATE_ABORT:
2160 ret = COMPACT_CONTENDED;
2161 putback_movable_pages(&cc->migratepages);
2162 cc->nr_migratepages = 0;
2163 last_migrated_pfn = 0;
2164 goto out;
2165 case ISOLATE_NONE:
2166 if (update_cached) {
2167 cc->zone->compact_cached_migrate_pfn[1] =
2168 cc->zone->compact_cached_migrate_pfn[0];
2169 }
2170
2171
2172
2173
2174
2175
2176 goto check_drain;
2177 case ISOLATE_SUCCESS:
2178 update_cached = false;
2179 last_migrated_pfn = start_pfn;
2180 ;
2181 }
2182
2183 err = migrate_pages(&cc->migratepages, compaction_alloc,
2184 compaction_free, (unsigned long)cc, cc->mode,
2185 MR_COMPACTION);
2186
2187 trace_mm_compaction_migratepages(cc->nr_migratepages, err,
2188 &cc->migratepages);
2189
2190
2191 cc->nr_migratepages = 0;
2192 if (err) {
2193 putback_movable_pages(&cc->migratepages);
2194
2195
2196
2197
2198 if (err == -ENOMEM && !compact_scanners_met(cc)) {
2199 ret = COMPACT_CONTENDED;
2200 goto out;
2201 }
2202
2203
2204
2205
2206 if (cc->direct_compaction &&
2207 (cc->mode == MIGRATE_ASYNC)) {
2208 cc->migrate_pfn = block_end_pfn(
2209 cc->migrate_pfn - 1, cc->order);
2210
2211 last_migrated_pfn = 0;
2212 }
2213 }
2214
2215check_drain:
2216
2217
2218
2219
2220
2221
2222
2223 if (cc->order > 0 && last_migrated_pfn) {
2224 int cpu;
2225 unsigned long current_block_start =
2226 block_start_pfn(cc->migrate_pfn, cc->order);
2227
2228 if (last_migrated_pfn < current_block_start) {
2229 cpu = get_cpu();
2230 lru_add_drain_cpu(cpu);
2231 drain_local_pages(cc->zone);
2232 put_cpu();
2233
2234 last_migrated_pfn = 0;
2235 }
2236 }
2237
2238
2239 if (capc && capc->page) {
2240 ret = COMPACT_SUCCESS;
2241 break;
2242 }
2243 }
2244
2245out:
2246
2247
2248
2249
2250 if (cc->nr_freepages > 0) {
2251 unsigned long free_pfn = release_freepages(&cc->freepages);
2252
2253 cc->nr_freepages = 0;
2254 VM_BUG_ON(free_pfn == 0);
2255
2256 free_pfn = pageblock_start_pfn(free_pfn);
2257
2258
2259
2260
2261 if (free_pfn > cc->zone->compact_cached_free_pfn)
2262 cc->zone->compact_cached_free_pfn = free_pfn;
2263 }
2264
2265 count_compact_events(COMPACTMIGRATE_SCANNED, cc->total_migrate_scanned);
2266 count_compact_events(COMPACTFREE_SCANNED, cc->total_free_scanned);
2267
2268 trace_mm_compaction_end(start_pfn, cc->migrate_pfn,
2269 cc->free_pfn, end_pfn, sync, ret);
2270
2271 return ret;
2272}
2273
2274static enum compact_result compact_zone_order(struct zone *zone, int order,
2275 gfp_t gfp_mask, enum compact_priority prio,
2276 unsigned int alloc_flags, int classzone_idx,
2277 struct page **capture)
2278{
2279 enum compact_result ret;
2280 struct compact_control cc = {
2281 .nr_freepages = 0,
2282 .nr_migratepages = 0,
2283 .total_migrate_scanned = 0,
2284 .total_free_scanned = 0,
2285 .order = order,
2286 .search_order = order,
2287 .gfp_mask = gfp_mask,
2288 .zone = zone,
2289 .mode = (prio == COMPACT_PRIO_ASYNC) ?
2290 MIGRATE_ASYNC : MIGRATE_SYNC_LIGHT,
2291 .alloc_flags = alloc_flags,
2292 .classzone_idx = classzone_idx,
2293 .direct_compaction = true,
2294 .whole_zone = (prio == MIN_COMPACT_PRIORITY),
2295 .ignore_skip_hint = (prio == MIN_COMPACT_PRIORITY),
2296 .ignore_block_suitable = (prio == MIN_COMPACT_PRIORITY)
2297 };
2298 struct capture_control capc = {
2299 .cc = &cc,
2300 .page = NULL,
2301 };
2302
2303 if (capture)
2304 current->capture_control = &capc;
2305 INIT_LIST_HEAD(&cc.freepages);
2306 INIT_LIST_HEAD(&cc.migratepages);
2307
2308 ret = compact_zone(&cc, &capc);
2309
2310 VM_BUG_ON(!list_empty(&cc.freepages));
2311 VM_BUG_ON(!list_empty(&cc.migratepages));
2312
2313 *capture = capc.page;
2314 current->capture_control = NULL;
2315
2316 return ret;
2317}
2318
2319int sysctl_extfrag_threshold = 500;
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331enum compact_result try_to_compact_pages(gfp_t gfp_mask, unsigned int order,
2332 unsigned int alloc_flags, const struct alloc_context *ac,
2333 enum compact_priority prio, struct page **capture)
2334{
2335 int may_perform_io = gfp_mask & __GFP_IO;
2336 struct zoneref *z;
2337 struct zone *zone;
2338 enum compact_result rc = COMPACT_SKIPPED;
2339
2340
2341
2342
2343
2344 if (!may_perform_io)
2345 return COMPACT_SKIPPED;
2346
2347 trace_mm_compaction_try_to_compact_pages(order, gfp_mask, prio);
2348
2349
2350 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx,
2351 ac->nodemask) {
2352 enum compact_result status;
2353
2354 if (prio > MIN_COMPACT_PRIORITY
2355 && compaction_deferred(zone, order)) {
2356 rc = max_t(enum compact_result, COMPACT_DEFERRED, rc);
2357 continue;
2358 }
2359
2360 status = compact_zone_order(zone, order, gfp_mask, prio,
2361 alloc_flags, ac_classzone_idx(ac), capture);
2362 rc = max(status, rc);
2363
2364
2365 if (status == COMPACT_SUCCESS) {
2366
2367
2368
2369
2370
2371
2372 compaction_defer_reset(zone, order, false);
2373
2374 break;
2375 }
2376
2377 if (prio != COMPACT_PRIO_ASYNC && (status == COMPACT_COMPLETE ||
2378 status == COMPACT_PARTIAL_SKIPPED))
2379
2380
2381
2382
2383
2384 defer_compaction(zone, order);
2385
2386
2387
2388
2389
2390
2391 if ((prio == COMPACT_PRIO_ASYNC && need_resched())
2392 || fatal_signal_pending(current))
2393 break;
2394 }
2395
2396 return rc;
2397}
2398
2399
2400
2401static void compact_node(int nid)
2402{
2403 pg_data_t *pgdat = NODE_DATA(nid);
2404 int zoneid;
2405 struct zone *zone;
2406 struct compact_control cc = {
2407 .order = -1,
2408 .total_migrate_scanned = 0,
2409 .total_free_scanned = 0,
2410 .mode = MIGRATE_SYNC,
2411 .ignore_skip_hint = true,
2412 .whole_zone = true,
2413 .gfp_mask = GFP_KERNEL,
2414 };
2415
2416
2417 for (zoneid = 0; zoneid < MAX_NR_ZONES; zoneid++) {
2418
2419 zone = &pgdat->node_zones[zoneid];
2420 if (!populated_zone(zone))
2421 continue;
2422
2423 cc.nr_freepages = 0;
2424 cc.nr_migratepages = 0;
2425 cc.zone = zone;
2426 INIT_LIST_HEAD(&cc.freepages);
2427 INIT_LIST_HEAD(&cc.migratepages);
2428
2429 compact_zone(&cc, NULL);
2430
2431 VM_BUG_ON(!list_empty(&cc.freepages));
2432 VM_BUG_ON(!list_empty(&cc.migratepages));
2433 }
2434}
2435
2436
2437static void compact_nodes(void)
2438{
2439 int nid;
2440
2441
2442 lru_add_drain_all();
2443
2444 for_each_online_node(nid)
2445 compact_node(nid);
2446}
2447
2448
2449int sysctl_compact_memory;
2450
2451
2452
2453
2454
2455int sysctl_compaction_handler(struct ctl_table *table, int write,
2456 void __user *buffer, size_t *length, loff_t *ppos)
2457{
2458 if (write)
2459 compact_nodes();
2460
2461 return 0;
2462}
2463
2464#if defined(CONFIG_SYSFS) && defined(CONFIG_NUMA)
2465static ssize_t sysfs_compact_node(struct device *dev,
2466 struct device_attribute *attr,
2467 const char *buf, size_t count)
2468{
2469 int nid = dev->id;
2470
2471 if (nid >= 0 && nid < nr_node_ids && node_online(nid)) {
2472
2473 lru_add_drain_all();
2474
2475 compact_node(nid);
2476 }
2477
2478 return count;
2479}
2480static DEVICE_ATTR(compact, 0200, NULL, sysfs_compact_node);
2481
2482int compaction_register_node(struct node *node)
2483{
2484 return device_create_file(&node->dev, &dev_attr_compact);
2485}
2486
2487void compaction_unregister_node(struct node *node)
2488{
2489 return device_remove_file(&node->dev, &dev_attr_compact);
2490}
2491#endif
2492
2493static inline bool kcompactd_work_requested(pg_data_t *pgdat)
2494{
2495 return pgdat->kcompactd_max_order > 0 || kthread_should_stop();
2496}
2497
2498static bool kcompactd_node_suitable(pg_data_t *pgdat)
2499{
2500 int zoneid;
2501 struct zone *zone;
2502 enum zone_type classzone_idx = pgdat->kcompactd_classzone_idx;
2503
2504 for (zoneid = 0; zoneid <= classzone_idx; zoneid++) {
2505 zone = &pgdat->node_zones[zoneid];
2506
2507 if (!populated_zone(zone))
2508 continue;
2509
2510 if (compaction_suitable(zone, pgdat->kcompactd_max_order, 0,
2511 classzone_idx) == COMPACT_CONTINUE)
2512 return true;
2513 }
2514
2515 return false;
2516}
2517
2518static void kcompactd_do_work(pg_data_t *pgdat)
2519{
2520
2521
2522
2523
2524 int zoneid;
2525 struct zone *zone;
2526 struct compact_control cc = {
2527 .order = pgdat->kcompactd_max_order,
2528 .search_order = pgdat->kcompactd_max_order,
2529 .total_migrate_scanned = 0,
2530 .total_free_scanned = 0,
2531 .classzone_idx = pgdat->kcompactd_classzone_idx,
2532 .mode = MIGRATE_SYNC_LIGHT,
2533 .ignore_skip_hint = false,
2534 .gfp_mask = GFP_KERNEL,
2535 };
2536 trace_mm_compaction_kcompactd_wake(pgdat->node_id, cc.order,
2537 cc.classzone_idx);
2538 count_compact_event(KCOMPACTD_WAKE);
2539
2540 for (zoneid = 0; zoneid <= cc.classzone_idx; zoneid++) {
2541 int status;
2542
2543 zone = &pgdat->node_zones[zoneid];
2544 if (!populated_zone(zone))
2545 continue;
2546
2547 if (compaction_deferred(zone, cc.order))
2548 continue;
2549
2550 if (compaction_suitable(zone, cc.order, 0, zoneid) !=
2551 COMPACT_CONTINUE)
2552 continue;
2553
2554 cc.nr_freepages = 0;
2555 cc.nr_migratepages = 0;
2556 cc.total_migrate_scanned = 0;
2557 cc.total_free_scanned = 0;
2558 cc.zone = zone;
2559 INIT_LIST_HEAD(&cc.freepages);
2560 INIT_LIST_HEAD(&cc.migratepages);
2561
2562 if (kthread_should_stop())
2563 return;
2564 status = compact_zone(&cc, NULL);
2565
2566 if (status == COMPACT_SUCCESS) {
2567 compaction_defer_reset(zone, cc.order, false);
2568 } else if (status == COMPACT_PARTIAL_SKIPPED || status == COMPACT_COMPLETE) {
2569
2570
2571
2572
2573
2574
2575 drain_all_pages(zone);
2576
2577
2578
2579
2580
2581 defer_compaction(zone, cc.order);
2582 }
2583
2584 count_compact_events(KCOMPACTD_MIGRATE_SCANNED,
2585 cc.total_migrate_scanned);
2586 count_compact_events(KCOMPACTD_FREE_SCANNED,
2587 cc.total_free_scanned);
2588
2589 VM_BUG_ON(!list_empty(&cc.freepages));
2590 VM_BUG_ON(!list_empty(&cc.migratepages));
2591 }
2592
2593
2594
2595
2596
2597
2598 if (pgdat->kcompactd_max_order <= cc.order)
2599 pgdat->kcompactd_max_order = 0;
2600 if (pgdat->kcompactd_classzone_idx >= cc.classzone_idx)
2601 pgdat->kcompactd_classzone_idx = pgdat->nr_zones - 1;
2602}
2603
2604void wakeup_kcompactd(pg_data_t *pgdat, int order, int classzone_idx)
2605{
2606 if (!order)
2607 return;
2608
2609 if (pgdat->kcompactd_max_order < order)
2610 pgdat->kcompactd_max_order = order;
2611
2612 if (pgdat->kcompactd_classzone_idx > classzone_idx)
2613 pgdat->kcompactd_classzone_idx = classzone_idx;
2614
2615
2616
2617
2618
2619 if (!wq_has_sleeper(&pgdat->kcompactd_wait))
2620 return;
2621
2622 if (!kcompactd_node_suitable(pgdat))
2623 return;
2624
2625 trace_mm_compaction_wakeup_kcompactd(pgdat->node_id, order,
2626 classzone_idx);
2627 wake_up_interruptible(&pgdat->kcompactd_wait);
2628}
2629
2630
2631
2632
2633
2634static int kcompactd(void *p)
2635{
2636 pg_data_t *pgdat = (pg_data_t*)p;
2637 struct task_struct *tsk = current;
2638
2639 const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id);
2640
2641 if (!cpumask_empty(cpumask))
2642 set_cpus_allowed_ptr(tsk, cpumask);
2643
2644 set_freezable();
2645
2646 pgdat->kcompactd_max_order = 0;
2647 pgdat->kcompactd_classzone_idx = pgdat->nr_zones - 1;
2648
2649 while (!kthread_should_stop()) {
2650 unsigned long pflags;
2651
2652 trace_mm_compaction_kcompactd_sleep(pgdat->node_id);
2653 wait_event_freezable(pgdat->kcompactd_wait,
2654 kcompactd_work_requested(pgdat));
2655
2656 psi_memstall_enter(&pflags);
2657 kcompactd_do_work(pgdat);
2658 psi_memstall_leave(&pflags);
2659 }
2660
2661 return 0;
2662}
2663
2664
2665
2666
2667
2668int kcompactd_run(int nid)
2669{
2670 pg_data_t *pgdat = NODE_DATA(nid);
2671 int ret = 0;
2672
2673 if (pgdat->kcompactd)
2674 return 0;
2675
2676 pgdat->kcompactd = kthread_run(kcompactd, pgdat, "kcompactd%d", nid);
2677 if (IS_ERR(pgdat->kcompactd)) {
2678 pr_err("Failed to start kcompactd on node %d\n", nid);
2679 ret = PTR_ERR(pgdat->kcompactd);
2680 pgdat->kcompactd = NULL;
2681 }
2682 return ret;
2683}
2684
2685
2686
2687
2688
2689void kcompactd_stop(int nid)
2690{
2691 struct task_struct *kcompactd = NODE_DATA(nid)->kcompactd;
2692
2693 if (kcompactd) {
2694 kthread_stop(kcompactd);
2695 NODE_DATA(nid)->kcompactd = NULL;
2696 }
2697}
2698
2699
2700
2701
2702
2703
2704
2705static int kcompactd_cpu_online(unsigned int cpu)
2706{
2707 int nid;
2708
2709 for_each_node_state(nid, N_MEMORY) {
2710 pg_data_t *pgdat = NODE_DATA(nid);
2711 const struct cpumask *mask;
2712
2713 mask = cpumask_of_node(pgdat->node_id);
2714
2715 if (cpumask_any_and(cpu_online_mask, mask) < nr_cpu_ids)
2716
2717 set_cpus_allowed_ptr(pgdat->kcompactd, mask);
2718 }
2719 return 0;
2720}
2721
2722static int __init kcompactd_init(void)
2723{
2724 int nid;
2725 int ret;
2726
2727 ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
2728 "mm/compaction:online",
2729 kcompactd_cpu_online, NULL);
2730 if (ret < 0) {
2731 pr_err("kcompactd: failed to register hotplug callbacks.\n");
2732 return ret;
2733 }
2734
2735 for_each_node_state(nid, N_MEMORY)
2736 kcompactd_run(nid);
2737 return 0;
2738}
2739subsys_initcall(kcompactd_init)
2740
2741#endif
2742