1
2
3
4
5
6
7
8
9
10
11#include <linux/cpu.h>
12#include <linux/swap.h>
13#include <linux/migrate.h>
14#include <linux/compaction.h>
15#include <linux/mm_inline.h>
16#include <linux/sched/signal.h>
17#include <linux/backing-dev.h>
18#include <linux/sysctl.h>
19#include <linux/sysfs.h>
20#include <linux/page-isolation.h>
21#include <linux/kasan.h>
22#include <linux/kthread.h>
23#include <linux/freezer.h>
24#include <linux/page_owner.h>
25#include <linux/psi.h>
26#include "internal.h"
27
28#ifdef CONFIG_COMPACTION
29static inline void count_compact_event(enum vm_event_item item)
30{
31 count_vm_event(item);
32}
33
34static inline void count_compact_events(enum vm_event_item item, long delta)
35{
36 count_vm_events(item, delta);
37}
38#else
39#define count_compact_event(item) do { } while (0)
40#define count_compact_events(item, delta) do { } while (0)
41#endif
42
43#if defined CONFIG_COMPACTION || defined CONFIG_CMA
44
45#define CREATE_TRACE_POINTS
46#include <trace/events/compaction.h>
47
48#define block_start_pfn(pfn, order) round_down(pfn, 1UL << (order))
49#define block_end_pfn(pfn, order) ALIGN((pfn) + 1, 1UL << (order))
50#define pageblock_start_pfn(pfn) block_start_pfn(pfn, pageblock_order)
51#define pageblock_end_pfn(pfn) block_end_pfn(pfn, pageblock_order)
52
53static unsigned long release_freepages(struct list_head *freelist)
54{
55 struct page *page, *next;
56 unsigned long high_pfn = 0;
57
58 list_for_each_entry_safe(page, next, freelist, lru) {
59 unsigned long pfn = page_to_pfn(page);
60 list_del(&page->lru);
61 __free_page(page);
62 if (pfn > high_pfn)
63 high_pfn = pfn;
64 }
65
66 return high_pfn;
67}
68
69static void split_map_pages(struct list_head *list)
70{
71 unsigned int i, order, nr_pages;
72 struct page *page, *next;
73 LIST_HEAD(tmp_list);
74
75 list_for_each_entry_safe(page, next, list, lru) {
76 list_del(&page->lru);
77
78 order = page_private(page);
79 nr_pages = 1 << order;
80
81 post_alloc_hook(page, order, __GFP_MOVABLE);
82 if (order)
83 split_page(page, order);
84
85 for (i = 0; i < nr_pages; i++) {
86 list_add(&page->lru, &tmp_list);
87 page++;
88 }
89 }
90
91 list_splice(&tmp_list, list);
92}
93
94#ifdef CONFIG_COMPACTION
95
96int PageMovable(struct page *page)
97{
98 struct address_space *mapping;
99
100 VM_BUG_ON_PAGE(!PageLocked(page), page);
101 if (!__PageMovable(page))
102 return 0;
103
104 mapping = page_mapping(page);
105 if (mapping && mapping->a_ops && mapping->a_ops->isolate_page)
106 return 1;
107
108 return 0;
109}
110EXPORT_SYMBOL(PageMovable);
111
112void __SetPageMovable(struct page *page, struct address_space *mapping)
113{
114 VM_BUG_ON_PAGE(!PageLocked(page), page);
115 VM_BUG_ON_PAGE((unsigned long)mapping & PAGE_MAPPING_MOVABLE, page);
116 page->mapping = (void *)((unsigned long)mapping | PAGE_MAPPING_MOVABLE);
117}
118EXPORT_SYMBOL(__SetPageMovable);
119
120void __ClearPageMovable(struct page *page)
121{
122 VM_BUG_ON_PAGE(!PageLocked(page), page);
123 VM_BUG_ON_PAGE(!PageMovable(page), page);
124
125
126
127
128
129 page->mapping = (void *)((unsigned long)page->mapping &
130 PAGE_MAPPING_MOVABLE);
131}
132EXPORT_SYMBOL(__ClearPageMovable);
133
134
135#define COMPACT_MAX_DEFER_SHIFT 6
136
137
138
139
140
141
142void defer_compaction(struct zone *zone, int order)
143{
144 zone->compact_considered = 0;
145 zone->compact_defer_shift++;
146
147 if (order < zone->compact_order_failed)
148 zone->compact_order_failed = order;
149
150 if (zone->compact_defer_shift > COMPACT_MAX_DEFER_SHIFT)
151 zone->compact_defer_shift = COMPACT_MAX_DEFER_SHIFT;
152
153 trace_mm_compaction_defer_compaction(zone, order);
154}
155
156
157bool compaction_deferred(struct zone *zone, int order)
158{
159 unsigned long defer_limit = 1UL << zone->compact_defer_shift;
160
161 if (order < zone->compact_order_failed)
162 return false;
163
164
165 if (++zone->compact_considered > defer_limit)
166 zone->compact_considered = defer_limit;
167
168 if (zone->compact_considered >= defer_limit)
169 return false;
170
171 trace_mm_compaction_deferred(zone, order);
172
173 return true;
174}
175
176
177
178
179
180
181void compaction_defer_reset(struct zone *zone, int order,
182 bool alloc_success)
183{
184 if (alloc_success) {
185 zone->compact_considered = 0;
186 zone->compact_defer_shift = 0;
187 }
188 if (order >= zone->compact_order_failed)
189 zone->compact_order_failed = order + 1;
190
191 trace_mm_compaction_defer_reset(zone, order);
192}
193
194
195bool compaction_restarting(struct zone *zone, int order)
196{
197 if (order < zone->compact_order_failed)
198 return false;
199
200 return zone->compact_defer_shift == COMPACT_MAX_DEFER_SHIFT &&
201 zone->compact_considered >= 1UL << zone->compact_defer_shift;
202}
203
204
205static inline bool isolation_suitable(struct compact_control *cc,
206 struct page *page)
207{
208 if (cc->ignore_skip_hint)
209 return true;
210
211 return !get_pageblock_skip(page);
212}
213
214static void reset_cached_positions(struct zone *zone)
215{
216 zone->compact_cached_migrate_pfn[0] = zone->zone_start_pfn;
217 zone->compact_cached_migrate_pfn[1] = zone->zone_start_pfn;
218 zone->compact_cached_free_pfn =
219 pageblock_start_pfn(zone_end_pfn(zone) - 1);
220}
221
222
223
224
225
226
227static bool pageblock_skip_persistent(struct page *page)
228{
229 if (!PageCompound(page))
230 return false;
231
232 page = compound_head(page);
233
234 if (compound_order(page) >= pageblock_order)
235 return true;
236
237 return false;
238}
239
240static bool
241__reset_isolation_pfn(struct zone *zone, unsigned long pfn, bool check_source,
242 bool check_target)
243{
244 struct page *page = pfn_to_online_page(pfn);
245 struct page *block_page;
246 struct page *end_page;
247 unsigned long block_pfn;
248
249 if (!page)
250 return false;
251 if (zone != page_zone(page))
252 return false;
253 if (pageblock_skip_persistent(page))
254 return false;
255
256
257
258
259
260 if (check_source && check_target && !get_pageblock_skip(page))
261 return true;
262
263
264
265
266
267 if (!check_source && check_target &&
268 get_pageblock_migratetype(page) != MIGRATE_MOVABLE)
269 return false;
270
271
272 block_pfn = pageblock_start_pfn(pfn);
273 block_page = pfn_to_online_page(max(block_pfn, zone->zone_start_pfn));
274 if (block_page) {
275 page = block_page;
276 pfn = block_pfn;
277 }
278
279
280 block_pfn += pageblock_nr_pages;
281 block_pfn = min(block_pfn, zone_end_pfn(zone) - 1);
282 end_page = pfn_to_online_page(block_pfn);
283 if (!end_page)
284 return false;
285
286
287
288
289
290
291 do {
292 if (pfn_valid_within(pfn)) {
293 if (check_source && PageLRU(page)) {
294 clear_pageblock_skip(page);
295 return true;
296 }
297
298 if (check_target && PageBuddy(page)) {
299 clear_pageblock_skip(page);
300 return true;
301 }
302 }
303
304 page += (1 << PAGE_ALLOC_COSTLY_ORDER);
305 pfn += (1 << PAGE_ALLOC_COSTLY_ORDER);
306 } while (page < end_page);
307
308 return false;
309}
310
311
312
313
314
315
316static void __reset_isolation_suitable(struct zone *zone)
317{
318 unsigned long migrate_pfn = zone->zone_start_pfn;
319 unsigned long free_pfn = zone_end_pfn(zone) - 1;
320 unsigned long reset_migrate = free_pfn;
321 unsigned long reset_free = migrate_pfn;
322 bool source_set = false;
323 bool free_set = false;
324
325 if (!zone->compact_blockskip_flush)
326 return;
327
328 zone->compact_blockskip_flush = false;
329
330
331
332
333
334
335
336 for (; migrate_pfn < free_pfn; migrate_pfn += pageblock_nr_pages,
337 free_pfn -= pageblock_nr_pages) {
338 cond_resched();
339
340
341 if (__reset_isolation_pfn(zone, migrate_pfn, true, source_set) &&
342 migrate_pfn < reset_migrate) {
343 source_set = true;
344 reset_migrate = migrate_pfn;
345 zone->compact_init_migrate_pfn = reset_migrate;
346 zone->compact_cached_migrate_pfn[0] = reset_migrate;
347 zone->compact_cached_migrate_pfn[1] = reset_migrate;
348 }
349
350
351 if (__reset_isolation_pfn(zone, free_pfn, free_set, true) &&
352 free_pfn > reset_free) {
353 free_set = true;
354 reset_free = free_pfn;
355 zone->compact_init_free_pfn = reset_free;
356 zone->compact_cached_free_pfn = reset_free;
357 }
358 }
359
360
361 if (reset_migrate >= reset_free) {
362 zone->compact_cached_migrate_pfn[0] = migrate_pfn;
363 zone->compact_cached_migrate_pfn[1] = migrate_pfn;
364 zone->compact_cached_free_pfn = free_pfn;
365 }
366}
367
368void reset_isolation_suitable(pg_data_t *pgdat)
369{
370 int zoneid;
371
372 for (zoneid = 0; zoneid < MAX_NR_ZONES; zoneid++) {
373 struct zone *zone = &pgdat->node_zones[zoneid];
374 if (!populated_zone(zone))
375 continue;
376
377
378 if (zone->compact_blockskip_flush)
379 __reset_isolation_suitable(zone);
380 }
381}
382
383
384
385
386
387static bool test_and_set_skip(struct compact_control *cc, struct page *page,
388 unsigned long pfn)
389{
390 bool skip;
391
392
393 if (cc->ignore_skip_hint)
394 return false;
395
396 if (!IS_ALIGNED(pfn, pageblock_nr_pages))
397 return false;
398
399 skip = get_pageblock_skip(page);
400 if (!skip && !cc->no_set_skip_hint)
401 set_pageblock_skip(page);
402
403 return skip;
404}
405
406static void update_cached_migrate(struct compact_control *cc, unsigned long pfn)
407{
408 struct zone *zone = cc->zone;
409
410 pfn = pageblock_end_pfn(pfn);
411
412
413 if (cc->no_set_skip_hint)
414 return;
415
416 if (pfn > zone->compact_cached_migrate_pfn[0])
417 zone->compact_cached_migrate_pfn[0] = pfn;
418 if (cc->mode != MIGRATE_ASYNC &&
419 pfn > zone->compact_cached_migrate_pfn[1])
420 zone->compact_cached_migrate_pfn[1] = pfn;
421}
422
423
424
425
426
427static void update_pageblock_skip(struct compact_control *cc,
428 struct page *page, unsigned long pfn)
429{
430 struct zone *zone = cc->zone;
431
432 if (cc->no_set_skip_hint)
433 return;
434
435 if (!page)
436 return;
437
438 set_pageblock_skip(page);
439
440
441 if (pfn < zone->compact_cached_free_pfn)
442 zone->compact_cached_free_pfn = pfn;
443}
444#else
445static inline bool isolation_suitable(struct compact_control *cc,
446 struct page *page)
447{
448 return true;
449}
450
451static inline bool pageblock_skip_persistent(struct page *page)
452{
453 return false;
454}
455
456static inline void update_pageblock_skip(struct compact_control *cc,
457 struct page *page, unsigned long pfn)
458{
459}
460
461static void update_cached_migrate(struct compact_control *cc, unsigned long pfn)
462{
463}
464
465static bool test_and_set_skip(struct compact_control *cc, struct page *page,
466 unsigned long pfn)
467{
468 return false;
469}
470#endif
471
472
473
474
475
476
477
478
479
480
481static bool compact_lock_irqsave(spinlock_t *lock, unsigned long *flags,
482 struct compact_control *cc)
483{
484
485 if (cc->mode == MIGRATE_ASYNC && !cc->contended) {
486 if (spin_trylock_irqsave(lock, *flags))
487 return true;
488
489 cc->contended = true;
490 }
491
492 spin_lock_irqsave(lock, *flags);
493 return true;
494}
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511static bool compact_unlock_should_abort(spinlock_t *lock,
512 unsigned long flags, bool *locked, struct compact_control *cc)
513{
514 if (*locked) {
515 spin_unlock_irqrestore(lock, flags);
516 *locked = false;
517 }
518
519 if (fatal_signal_pending(current)) {
520 cc->contended = true;
521 return true;
522 }
523
524 cond_resched();
525
526 return false;
527}
528
529
530
531
532
533
534static unsigned long isolate_freepages_block(struct compact_control *cc,
535 unsigned long *start_pfn,
536 unsigned long end_pfn,
537 struct list_head *freelist,
538 unsigned int stride,
539 bool strict)
540{
541 int nr_scanned = 0, total_isolated = 0;
542 struct page *cursor;
543 unsigned long flags = 0;
544 bool locked = false;
545 unsigned long blockpfn = *start_pfn;
546 unsigned int order;
547
548
549 if (strict)
550 stride = 1;
551
552 cursor = pfn_to_page(blockpfn);
553
554
555 for (; blockpfn < end_pfn; blockpfn += stride, cursor += stride) {
556 int isolated;
557 struct page *page = cursor;
558
559
560
561
562
563
564 if (!(blockpfn % SWAP_CLUSTER_MAX)
565 && compact_unlock_should_abort(&cc->zone->lock, flags,
566 &locked, cc))
567 break;
568
569 nr_scanned++;
570 if (!pfn_valid_within(blockpfn))
571 goto isolate_fail;
572
573
574
575
576
577
578
579 if (PageCompound(page)) {
580 const unsigned int order = compound_order(page);
581
582 if (likely(order < MAX_ORDER)) {
583 blockpfn += (1UL << order) - 1;
584 cursor += (1UL << order) - 1;
585 }
586 goto isolate_fail;
587 }
588
589 if (!PageBuddy(page))
590 goto isolate_fail;
591
592
593
594
595
596
597
598
599 if (!locked) {
600 locked = compact_lock_irqsave(&cc->zone->lock,
601 &flags, cc);
602
603
604 if (!PageBuddy(page))
605 goto isolate_fail;
606 }
607
608
609 order = page_order(page);
610 isolated = __isolate_free_page(page, order);
611 if (!isolated)
612 break;
613 set_page_private(page, order);
614
615 total_isolated += isolated;
616 cc->nr_freepages += isolated;
617 list_add_tail(&page->lru, freelist);
618
619 if (!strict && cc->nr_migratepages <= cc->nr_freepages) {
620 blockpfn += isolated;
621 break;
622 }
623
624 blockpfn += isolated - 1;
625 cursor += isolated - 1;
626 continue;
627
628isolate_fail:
629 if (strict)
630 break;
631 else
632 continue;
633
634 }
635
636 if (locked)
637 spin_unlock_irqrestore(&cc->zone->lock, flags);
638
639
640
641
642
643 if (unlikely(blockpfn > end_pfn))
644 blockpfn = end_pfn;
645
646 trace_mm_compaction_isolate_freepages(*start_pfn, blockpfn,
647 nr_scanned, total_isolated);
648
649
650 *start_pfn = blockpfn;
651
652
653
654
655
656
657 if (strict && blockpfn < end_pfn)
658 total_isolated = 0;
659
660 cc->total_free_scanned += nr_scanned;
661 if (total_isolated)
662 count_compact_events(COMPACTISOLATED, total_isolated);
663 return total_isolated;
664}
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680unsigned long
681isolate_freepages_range(struct compact_control *cc,
682 unsigned long start_pfn, unsigned long end_pfn)
683{
684 unsigned long isolated, pfn, block_start_pfn, block_end_pfn;
685 LIST_HEAD(freelist);
686
687 pfn = start_pfn;
688 block_start_pfn = pageblock_start_pfn(pfn);
689 if (block_start_pfn < cc->zone->zone_start_pfn)
690 block_start_pfn = cc->zone->zone_start_pfn;
691 block_end_pfn = pageblock_end_pfn(pfn);
692
693 for (; pfn < end_pfn; pfn += isolated,
694 block_start_pfn = block_end_pfn,
695 block_end_pfn += pageblock_nr_pages) {
696
697 unsigned long isolate_start_pfn = pfn;
698
699 block_end_pfn = min(block_end_pfn, end_pfn);
700
701
702
703
704
705
706 if (pfn >= block_end_pfn) {
707 block_start_pfn = pageblock_start_pfn(pfn);
708 block_end_pfn = pageblock_end_pfn(pfn);
709 block_end_pfn = min(block_end_pfn, end_pfn);
710 }
711
712 if (!pageblock_pfn_to_page(block_start_pfn,
713 block_end_pfn, cc->zone))
714 break;
715
716 isolated = isolate_freepages_block(cc, &isolate_start_pfn,
717 block_end_pfn, &freelist, 0, true);
718
719
720
721
722
723
724 if (!isolated)
725 break;
726
727
728
729
730
731
732 }
733
734
735 split_map_pages(&freelist);
736
737 if (pfn < end_pfn) {
738
739 release_freepages(&freelist);
740 return 0;
741 }
742
743
744 return pfn;
745}
746
747
748static bool too_many_isolated(pg_data_t *pgdat)
749{
750 unsigned long active, inactive, isolated;
751
752 inactive = node_page_state(pgdat, NR_INACTIVE_FILE) +
753 node_page_state(pgdat, NR_INACTIVE_ANON);
754 active = node_page_state(pgdat, NR_ACTIVE_FILE) +
755 node_page_state(pgdat, NR_ACTIVE_ANON);
756 isolated = node_page_state(pgdat, NR_ISOLATED_FILE) +
757 node_page_state(pgdat, NR_ISOLATED_ANON);
758
759 return isolated > (inactive + active) / 2;
760}
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780static unsigned long
781isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
782 unsigned long end_pfn, isolate_mode_t isolate_mode)
783{
784 pg_data_t *pgdat = cc->zone->zone_pgdat;
785 unsigned long nr_scanned = 0, nr_isolated = 0;
786 struct lruvec *lruvec;
787 unsigned long flags = 0;
788 bool locked = false;
789 struct page *page = NULL, *valid_page = NULL;
790 unsigned long start_pfn = low_pfn;
791 bool skip_on_failure = false;
792 unsigned long next_skip_pfn = 0;
793 bool skip_updated = false;
794
795
796
797
798
799
800 while (unlikely(too_many_isolated(pgdat))) {
801
802 if (cc->mode == MIGRATE_ASYNC)
803 return 0;
804
805 congestion_wait(BLK_RW_ASYNC, HZ/10);
806
807 if (fatal_signal_pending(current))
808 return 0;
809 }
810
811 cond_resched();
812
813 if (cc->direct_compaction && (cc->mode == MIGRATE_ASYNC)) {
814 skip_on_failure = true;
815 next_skip_pfn = block_end_pfn(low_pfn, cc->order);
816 }
817
818
819 for (; low_pfn < end_pfn; low_pfn++) {
820
821 if (skip_on_failure && low_pfn >= next_skip_pfn) {
822
823
824
825
826
827
828 if (nr_isolated)
829 break;
830
831
832
833
834
835
836
837
838
839
840 next_skip_pfn = block_end_pfn(low_pfn, cc->order);
841 }
842
843
844
845
846
847
848 if (!(low_pfn % SWAP_CLUSTER_MAX)
849 && compact_unlock_should_abort(&pgdat->lru_lock,
850 flags, &locked, cc))
851 break;
852
853 if (!pfn_valid_within(low_pfn))
854 goto isolate_fail;
855 nr_scanned++;
856
857 page = pfn_to_page(low_pfn);
858
859
860
861
862
863
864
865 if (!valid_page && IS_ALIGNED(low_pfn, pageblock_nr_pages)) {
866 if (!cc->ignore_skip_hint && get_pageblock_skip(page)) {
867 low_pfn = end_pfn;
868 goto isolate_abort;
869 }
870 valid_page = page;
871 }
872
873
874
875
876
877
878
879 if (PageBuddy(page)) {
880 unsigned long freepage_order = page_order_unsafe(page);
881
882
883
884
885
886
887 if (freepage_order > 0 && freepage_order < MAX_ORDER)
888 low_pfn += (1UL << freepage_order) - 1;
889 continue;
890 }
891
892
893
894
895
896
897
898
899 if (PageCompound(page)) {
900 const unsigned int order = compound_order(page);
901
902 if (likely(order < MAX_ORDER))
903 low_pfn += (1UL << order) - 1;
904 goto isolate_fail;
905 }
906
907
908
909
910
911
912 if (!PageLRU(page)) {
913
914
915
916
917 if (unlikely(__PageMovable(page)) &&
918 !PageIsolated(page)) {
919 if (locked) {
920 spin_unlock_irqrestore(&pgdat->lru_lock,
921 flags);
922 locked = false;
923 }
924
925 if (!isolate_movable_page(page, isolate_mode))
926 goto isolate_success;
927 }
928
929 goto isolate_fail;
930 }
931
932
933
934
935
936
937 if (!page_mapping(page) &&
938 page_count(page) > page_mapcount(page))
939 goto isolate_fail;
940
941
942
943
944
945 if (!(cc->gfp_mask & __GFP_FS) && page_mapping(page))
946 goto isolate_fail;
947
948
949 if (!locked) {
950 locked = compact_lock_irqsave(&pgdat->lru_lock,
951 &flags, cc);
952
953
954 if (!skip_updated) {
955 skip_updated = true;
956 if (test_and_set_skip(cc, page, low_pfn))
957 goto isolate_abort;
958 }
959
960
961 if (!PageLRU(page))
962 goto isolate_fail;
963
964
965
966
967
968
969 if (unlikely(PageCompound(page))) {
970 low_pfn += (1UL << compound_order(page)) - 1;
971 goto isolate_fail;
972 }
973 }
974
975 lruvec = mem_cgroup_page_lruvec(page, pgdat);
976
977
978 if (__isolate_lru_page(page, isolate_mode) != 0)
979 goto isolate_fail;
980
981 VM_BUG_ON_PAGE(PageCompound(page), page);
982
983
984 del_page_from_lru_list(page, lruvec, page_lru(page));
985 inc_node_page_state(page,
986 NR_ISOLATED_ANON + page_is_file_cache(page));
987
988isolate_success:
989 list_add(&page->lru, &cc->migratepages);
990 cc->nr_migratepages++;
991 nr_isolated++;
992
993
994
995
996
997
998
999 if (cc->nr_migratepages == COMPACT_CLUSTER_MAX &&
1000 !cc->rescan && !cc->contended) {
1001 ++low_pfn;
1002 break;
1003 }
1004
1005 continue;
1006isolate_fail:
1007 if (!skip_on_failure)
1008 continue;
1009
1010
1011
1012
1013
1014
1015 if (nr_isolated) {
1016 if (locked) {
1017 spin_unlock_irqrestore(&pgdat->lru_lock, flags);
1018 locked = false;
1019 }
1020 putback_movable_pages(&cc->migratepages);
1021 cc->nr_migratepages = 0;
1022 nr_isolated = 0;
1023 }
1024
1025 if (low_pfn < next_skip_pfn) {
1026 low_pfn = next_skip_pfn - 1;
1027
1028
1029
1030
1031 next_skip_pfn += 1UL << cc->order;
1032 }
1033 }
1034
1035
1036
1037
1038
1039 if (unlikely(low_pfn > end_pfn))
1040 low_pfn = end_pfn;
1041
1042isolate_abort:
1043 if (locked)
1044 spin_unlock_irqrestore(&pgdat->lru_lock, flags);
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054 if (low_pfn == end_pfn && (!nr_isolated || cc->rescan)) {
1055 if (valid_page && !skip_updated)
1056 set_pageblock_skip(valid_page);
1057 update_cached_migrate(cc, low_pfn);
1058 }
1059
1060 trace_mm_compaction_isolate_migratepages(start_pfn, low_pfn,
1061 nr_scanned, nr_isolated);
1062
1063 cc->total_migrate_scanned += nr_scanned;
1064 if (nr_isolated)
1065 count_compact_events(COMPACTISOLATED, nr_isolated);
1066
1067 return low_pfn;
1068}
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080unsigned long
1081isolate_migratepages_range(struct compact_control *cc, unsigned long start_pfn,
1082 unsigned long end_pfn)
1083{
1084 unsigned long pfn, block_start_pfn, block_end_pfn;
1085
1086
1087 pfn = start_pfn;
1088 block_start_pfn = pageblock_start_pfn(pfn);
1089 if (block_start_pfn < cc->zone->zone_start_pfn)
1090 block_start_pfn = cc->zone->zone_start_pfn;
1091 block_end_pfn = pageblock_end_pfn(pfn);
1092
1093 for (; pfn < end_pfn; pfn = block_end_pfn,
1094 block_start_pfn = block_end_pfn,
1095 block_end_pfn += pageblock_nr_pages) {
1096
1097 block_end_pfn = min(block_end_pfn, end_pfn);
1098
1099 if (!pageblock_pfn_to_page(block_start_pfn,
1100 block_end_pfn, cc->zone))
1101 continue;
1102
1103 pfn = isolate_migratepages_block(cc, pfn, block_end_pfn,
1104 ISOLATE_UNEVICTABLE);
1105
1106 if (!pfn)
1107 break;
1108
1109 if (cc->nr_migratepages == COMPACT_CLUSTER_MAX)
1110 break;
1111 }
1112
1113 return pfn;
1114}
1115
1116#endif
1117#ifdef CONFIG_COMPACTION
1118
1119static bool suitable_migration_source(struct compact_control *cc,
1120 struct page *page)
1121{
1122 int block_mt;
1123
1124 if (pageblock_skip_persistent(page))
1125 return false;
1126
1127 if ((cc->mode != MIGRATE_ASYNC) || !cc->direct_compaction)
1128 return true;
1129
1130 block_mt = get_pageblock_migratetype(page);
1131
1132 if (cc->migratetype == MIGRATE_MOVABLE)
1133 return is_migrate_movable(block_mt);
1134 else
1135 return block_mt == cc->migratetype;
1136}
1137
1138
1139static bool suitable_migration_target(struct compact_control *cc,
1140 struct page *page)
1141{
1142
1143 if (PageBuddy(page)) {
1144
1145
1146
1147
1148
1149 if (page_order_unsafe(page) >= pageblock_order)
1150 return false;
1151 }
1152
1153 if (cc->ignore_block_suitable)
1154 return true;
1155
1156
1157 if (is_migrate_movable(get_pageblock_migratetype(page)))
1158 return true;
1159
1160
1161 return false;
1162}
1163
1164static inline unsigned int
1165freelist_scan_limit(struct compact_control *cc)
1166{
1167 return (COMPACT_CLUSTER_MAX >> cc->fast_search_fail) + 1;
1168}
1169
1170
1171
1172
1173
1174static inline bool compact_scanners_met(struct compact_control *cc)
1175{
1176 return (cc->free_pfn >> pageblock_order)
1177 <= (cc->migrate_pfn >> pageblock_order);
1178}
1179
1180
1181
1182
1183
1184
1185static void
1186move_freelist_head(struct list_head *freelist, struct page *freepage)
1187{
1188 LIST_HEAD(sublist);
1189
1190 if (!list_is_last(freelist, &freepage->lru)) {
1191 list_cut_before(&sublist, freelist, &freepage->lru);
1192 if (!list_empty(&sublist))
1193 list_splice_tail(&sublist, freelist);
1194 }
1195}
1196
1197
1198
1199
1200
1201
1202
1203static void
1204move_freelist_tail(struct list_head *freelist, struct page *freepage)
1205{
1206 LIST_HEAD(sublist);
1207
1208 if (!list_is_first(freelist, &freepage->lru)) {
1209 list_cut_position(&sublist, freelist, &freepage->lru);
1210 if (!list_empty(&sublist))
1211 list_splice_tail(&sublist, freelist);
1212 }
1213}
1214
1215static void
1216fast_isolate_around(struct compact_control *cc, unsigned long pfn, unsigned long nr_isolated)
1217{
1218 unsigned long start_pfn, end_pfn;
1219 struct page *page = pfn_to_page(pfn);
1220
1221
1222 if (cc->nr_freepages >= cc->nr_migratepages)
1223 return;
1224
1225
1226 if (cc->direct_compaction && cc->mode == MIGRATE_ASYNC)
1227 return;
1228
1229
1230 start_pfn = pageblock_start_pfn(pfn);
1231 end_pfn = min(start_pfn + pageblock_nr_pages, zone_end_pfn(cc->zone));
1232
1233
1234 if (start_pfn != pfn) {
1235 isolate_freepages_block(cc, &start_pfn, pfn, &cc->freepages, 1, false);
1236 if (cc->nr_freepages >= cc->nr_migratepages)
1237 return;
1238 }
1239
1240
1241 start_pfn = pfn + nr_isolated;
1242 if (start_pfn != end_pfn)
1243 isolate_freepages_block(cc, &start_pfn, end_pfn, &cc->freepages, 1, false);
1244
1245
1246 if (cc->nr_freepages < cc->nr_migratepages)
1247 set_pageblock_skip(page);
1248}
1249
1250
1251static int next_search_order(struct compact_control *cc, int order)
1252{
1253 order--;
1254 if (order < 0)
1255 order = cc->order - 1;
1256
1257
1258 if (order == cc->search_order) {
1259 cc->search_order--;
1260 if (cc->search_order < 0)
1261 cc->search_order = cc->order - 1;
1262 return -1;
1263 }
1264
1265 return order;
1266}
1267
1268static unsigned long
1269fast_isolate_freepages(struct compact_control *cc)
1270{
1271 unsigned int limit = min(1U, freelist_scan_limit(cc) >> 1);
1272 unsigned int nr_scanned = 0;
1273 unsigned long low_pfn, min_pfn, high_pfn = 0, highest = 0;
1274 unsigned long nr_isolated = 0;
1275 unsigned long distance;
1276 struct page *page = NULL;
1277 bool scan_start = false;
1278 int order;
1279
1280
1281 if (cc->order <= 0)
1282 return cc->free_pfn;
1283
1284
1285
1286
1287
1288 if (cc->free_pfn >= cc->zone->compact_init_free_pfn) {
1289 limit = pageblock_nr_pages >> 1;
1290 scan_start = true;
1291 }
1292
1293
1294
1295
1296
1297 distance = (cc->free_pfn - cc->migrate_pfn);
1298 low_pfn = pageblock_start_pfn(cc->free_pfn - (distance >> 2));
1299 min_pfn = pageblock_start_pfn(cc->free_pfn - (distance >> 1));
1300
1301 if (WARN_ON_ONCE(min_pfn > low_pfn))
1302 low_pfn = min_pfn;
1303
1304
1305
1306
1307
1308 cc->search_order = min_t(unsigned int, cc->order - 1, cc->search_order);
1309
1310 for (order = cc->search_order;
1311 !page && order >= 0;
1312 order = next_search_order(cc, order)) {
1313 struct free_area *area = &cc->zone->free_area[order];
1314 struct list_head *freelist;
1315 struct page *freepage;
1316 unsigned long flags;
1317 unsigned int order_scanned = 0;
1318
1319 if (!area->nr_free)
1320 continue;
1321
1322 spin_lock_irqsave(&cc->zone->lock, flags);
1323 freelist = &area->free_list[MIGRATE_MOVABLE];
1324 list_for_each_entry_reverse(freepage, freelist, lru) {
1325 unsigned long pfn;
1326
1327 order_scanned++;
1328 nr_scanned++;
1329 pfn = page_to_pfn(freepage);
1330
1331 if (pfn >= highest)
1332 highest = pageblock_start_pfn(pfn);
1333
1334 if (pfn >= low_pfn) {
1335 cc->fast_search_fail = 0;
1336 cc->search_order = order;
1337 page = freepage;
1338 break;
1339 }
1340
1341 if (pfn >= min_pfn && pfn > high_pfn) {
1342 high_pfn = pfn;
1343
1344
1345 limit >>= 1;
1346 }
1347
1348 if (order_scanned >= limit)
1349 break;
1350 }
1351
1352
1353 if (!page && high_pfn) {
1354 page = pfn_to_page(high_pfn);
1355
1356
1357 freepage = page;
1358 }
1359
1360
1361 move_freelist_head(freelist, freepage);
1362
1363
1364 if (page) {
1365 if (__isolate_free_page(page, order)) {
1366 set_page_private(page, order);
1367 nr_isolated = 1 << order;
1368 cc->nr_freepages += nr_isolated;
1369 list_add_tail(&page->lru, &cc->freepages);
1370 count_compact_events(COMPACTISOLATED, nr_isolated);
1371 } else {
1372
1373 order = cc->search_order + 1;
1374 page = NULL;
1375 }
1376 }
1377
1378 spin_unlock_irqrestore(&cc->zone->lock, flags);
1379
1380
1381
1382
1383
1384 if (order_scanned >= limit)
1385 limit = min(1U, limit >> 1);
1386 }
1387
1388 if (!page) {
1389 cc->fast_search_fail++;
1390 if (scan_start) {
1391
1392
1393
1394
1395
1396 if (highest) {
1397 page = pfn_to_page(highest);
1398 cc->free_pfn = highest;
1399 } else {
1400 if (cc->direct_compaction) {
1401 page = pfn_to_page(min_pfn);
1402 cc->free_pfn = min_pfn;
1403 }
1404 }
1405 }
1406 }
1407
1408 if (highest && highest >= cc->zone->compact_cached_free_pfn) {
1409 highest -= pageblock_nr_pages;
1410 cc->zone->compact_cached_free_pfn = highest;
1411 }
1412
1413 cc->total_free_scanned += nr_scanned;
1414 if (!page)
1415 return cc->free_pfn;
1416
1417 low_pfn = page_to_pfn(page);
1418 fast_isolate_around(cc, low_pfn, nr_isolated);
1419 return low_pfn;
1420}
1421
1422
1423
1424
1425
1426static void isolate_freepages(struct compact_control *cc)
1427{
1428 struct zone *zone = cc->zone;
1429 struct page *page;
1430 unsigned long block_start_pfn;
1431 unsigned long isolate_start_pfn;
1432 unsigned long block_end_pfn;
1433 unsigned long low_pfn;
1434 struct list_head *freelist = &cc->freepages;
1435 unsigned int stride;
1436
1437
1438 isolate_start_pfn = fast_isolate_freepages(cc);
1439 if (cc->nr_freepages)
1440 goto splitmap;
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453 isolate_start_pfn = cc->free_pfn;
1454 block_start_pfn = pageblock_start_pfn(isolate_start_pfn);
1455 block_end_pfn = min(block_start_pfn + pageblock_nr_pages,
1456 zone_end_pfn(zone));
1457 low_pfn = pageblock_end_pfn(cc->migrate_pfn);
1458 stride = cc->mode == MIGRATE_ASYNC ? COMPACT_CLUSTER_MAX : 1;
1459
1460
1461
1462
1463
1464
1465 for (; block_start_pfn >= low_pfn;
1466 block_end_pfn = block_start_pfn,
1467 block_start_pfn -= pageblock_nr_pages,
1468 isolate_start_pfn = block_start_pfn) {
1469 unsigned long nr_isolated;
1470
1471
1472
1473
1474
1475 if (!(block_start_pfn % (SWAP_CLUSTER_MAX * pageblock_nr_pages)))
1476 cond_resched();
1477
1478 page = pageblock_pfn_to_page(block_start_pfn, block_end_pfn,
1479 zone);
1480 if (!page)
1481 continue;
1482
1483
1484 if (!suitable_migration_target(cc, page))
1485 continue;
1486
1487
1488 if (!isolation_suitable(cc, page))
1489 continue;
1490
1491
1492 nr_isolated = isolate_freepages_block(cc, &isolate_start_pfn,
1493 block_end_pfn, freelist, stride, false);
1494
1495
1496 if (isolate_start_pfn == block_end_pfn)
1497 update_pageblock_skip(cc, page, block_start_pfn);
1498
1499
1500 if (cc->nr_freepages >= cc->nr_migratepages) {
1501 if (isolate_start_pfn >= block_end_pfn) {
1502
1503
1504
1505
1506 isolate_start_pfn =
1507 block_start_pfn - pageblock_nr_pages;
1508 }
1509 break;
1510 } else if (isolate_start_pfn < block_end_pfn) {
1511
1512
1513
1514
1515 break;
1516 }
1517
1518
1519 if (nr_isolated) {
1520 stride = 1;
1521 continue;
1522 }
1523 stride = min_t(unsigned int, COMPACT_CLUSTER_MAX, stride << 1);
1524 }
1525
1526
1527
1528
1529
1530
1531
1532 cc->free_pfn = isolate_start_pfn;
1533
1534splitmap:
1535
1536 split_map_pages(freelist);
1537}
1538
1539
1540
1541
1542
1543static struct page *compaction_alloc(struct page *migratepage,
1544 unsigned long data)
1545{
1546 struct compact_control *cc = (struct compact_control *)data;
1547 struct page *freepage;
1548
1549 if (list_empty(&cc->freepages)) {
1550 isolate_freepages(cc);
1551
1552 if (list_empty(&cc->freepages))
1553 return NULL;
1554 }
1555
1556 freepage = list_entry(cc->freepages.next, struct page, lru);
1557 list_del(&freepage->lru);
1558 cc->nr_freepages--;
1559
1560 return freepage;
1561}
1562
1563
1564
1565
1566
1567
1568static void compaction_free(struct page *page, unsigned long data)
1569{
1570 struct compact_control *cc = (struct compact_control *)data;
1571
1572 list_add(&page->lru, &cc->freepages);
1573 cc->nr_freepages++;
1574}
1575
1576
1577typedef enum {
1578 ISOLATE_ABORT,
1579 ISOLATE_NONE,
1580 ISOLATE_SUCCESS,
1581} isolate_migrate_t;
1582
1583
1584
1585
1586
1587int sysctl_compact_unevictable_allowed __read_mostly = 1;
1588
1589static inline void
1590update_fast_start_pfn(struct compact_control *cc, unsigned long pfn)
1591{
1592 if (cc->fast_start_pfn == ULONG_MAX)
1593 return;
1594
1595 if (!cc->fast_start_pfn)
1596 cc->fast_start_pfn = pfn;
1597
1598 cc->fast_start_pfn = min(cc->fast_start_pfn, pfn);
1599}
1600
1601static inline unsigned long
1602reinit_migrate_pfn(struct compact_control *cc)
1603{
1604 if (!cc->fast_start_pfn || cc->fast_start_pfn == ULONG_MAX)
1605 return cc->migrate_pfn;
1606
1607 cc->migrate_pfn = cc->fast_start_pfn;
1608 cc->fast_start_pfn = ULONG_MAX;
1609
1610 return cc->migrate_pfn;
1611}
1612
1613
1614
1615
1616
1617
1618static unsigned long fast_find_migrateblock(struct compact_control *cc)
1619{
1620 unsigned int limit = freelist_scan_limit(cc);
1621 unsigned int nr_scanned = 0;
1622 unsigned long distance;
1623 unsigned long pfn = cc->migrate_pfn;
1624 unsigned long high_pfn;
1625 int order;
1626
1627
1628 if (cc->ignore_skip_hint)
1629 return pfn;
1630
1631
1632
1633
1634
1635
1636 if (pfn != cc->zone->zone_start_pfn && pfn != pageblock_start_pfn(pfn))
1637 return pfn;
1638
1639
1640
1641
1642
1643
1644 if (cc->order <= PAGE_ALLOC_COSTLY_ORDER)
1645 return pfn;
1646
1647
1648
1649
1650
1651
1652
1653 if (cc->direct_compaction && cc->migratetype != MIGRATE_MOVABLE)
1654 return pfn;
1655
1656
1657
1658
1659
1660
1661
1662 distance = (cc->free_pfn - cc->migrate_pfn) >> 1;
1663 if (cc->migrate_pfn != cc->zone->zone_start_pfn)
1664 distance >>= 2;
1665 high_pfn = pageblock_start_pfn(cc->migrate_pfn + distance);
1666
1667 for (order = cc->order - 1;
1668 order >= PAGE_ALLOC_COSTLY_ORDER && pfn == cc->migrate_pfn && nr_scanned < limit;
1669 order--) {
1670 struct free_area *area = &cc->zone->free_area[order];
1671 struct list_head *freelist;
1672 unsigned long flags;
1673 struct page *freepage;
1674
1675 if (!area->nr_free)
1676 continue;
1677
1678 spin_lock_irqsave(&cc->zone->lock, flags);
1679 freelist = &area->free_list[MIGRATE_MOVABLE];
1680 list_for_each_entry(freepage, freelist, lru) {
1681 unsigned long free_pfn;
1682
1683 nr_scanned++;
1684 free_pfn = page_to_pfn(freepage);
1685 if (free_pfn < high_pfn) {
1686
1687
1688
1689
1690
1691
1692 if (get_pageblock_skip(freepage)) {
1693 if (list_is_last(freelist, &freepage->lru))
1694 break;
1695
1696 continue;
1697 }
1698
1699
1700 move_freelist_tail(freelist, freepage);
1701
1702 update_fast_start_pfn(cc, free_pfn);
1703 pfn = pageblock_start_pfn(free_pfn);
1704 cc->fast_search_fail = 0;
1705 set_pageblock_skip(freepage);
1706 break;
1707 }
1708
1709 if (nr_scanned >= limit) {
1710 cc->fast_search_fail++;
1711 move_freelist_tail(freelist, freepage);
1712 break;
1713 }
1714 }
1715 spin_unlock_irqrestore(&cc->zone->lock, flags);
1716 }
1717
1718 cc->total_migrate_scanned += nr_scanned;
1719
1720
1721
1722
1723
1724 if (pfn == cc->migrate_pfn)
1725 pfn = reinit_migrate_pfn(cc);
1726
1727 return pfn;
1728}
1729
1730
1731
1732
1733
1734
1735static isolate_migrate_t isolate_migratepages(struct zone *zone,
1736 struct compact_control *cc)
1737{
1738 unsigned long block_start_pfn;
1739 unsigned long block_end_pfn;
1740 unsigned long low_pfn;
1741 struct page *page;
1742 const isolate_mode_t isolate_mode =
1743 (sysctl_compact_unevictable_allowed ? ISOLATE_UNEVICTABLE : 0) |
1744 (cc->mode != MIGRATE_SYNC ? ISOLATE_ASYNC_MIGRATE : 0);
1745 bool fast_find_block;
1746
1747
1748
1749
1750
1751
1752 low_pfn = fast_find_migrateblock(cc);
1753 block_start_pfn = pageblock_start_pfn(low_pfn);
1754 if (block_start_pfn < zone->zone_start_pfn)
1755 block_start_pfn = zone->zone_start_pfn;
1756
1757
1758
1759
1760
1761
1762 fast_find_block = low_pfn != cc->migrate_pfn && !cc->fast_search_fail;
1763
1764
1765 block_end_pfn = pageblock_end_pfn(low_pfn);
1766
1767
1768
1769
1770
1771 for (; block_end_pfn <= cc->free_pfn;
1772 fast_find_block = false,
1773 low_pfn = block_end_pfn,
1774 block_start_pfn = block_end_pfn,
1775 block_end_pfn += pageblock_nr_pages) {
1776
1777
1778
1779
1780
1781
1782 if (!(low_pfn % (SWAP_CLUSTER_MAX * pageblock_nr_pages)))
1783 cond_resched();
1784
1785 page = pageblock_pfn_to_page(block_start_pfn, block_end_pfn,
1786 zone);
1787 if (!page)
1788 continue;
1789
1790
1791
1792
1793
1794
1795
1796
1797 if (IS_ALIGNED(low_pfn, pageblock_nr_pages) &&
1798 !fast_find_block && !isolation_suitable(cc, page))
1799 continue;
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809 if (!suitable_migration_source(cc, page)) {
1810 update_cached_migrate(cc, block_end_pfn);
1811 continue;
1812 }
1813
1814
1815 low_pfn = isolate_migratepages_block(cc, low_pfn,
1816 block_end_pfn, isolate_mode);
1817
1818 if (!low_pfn)
1819 return ISOLATE_ABORT;
1820
1821
1822
1823
1824
1825
1826 break;
1827 }
1828
1829
1830 cc->migrate_pfn = low_pfn;
1831
1832 return cc->nr_migratepages ? ISOLATE_SUCCESS : ISOLATE_NONE;
1833}
1834
1835
1836
1837
1838
1839static inline bool is_via_compact_memory(int order)
1840{
1841 return order == -1;
1842}
1843
1844static enum compact_result __compact_finished(struct compact_control *cc)
1845{
1846 unsigned int order;
1847 const int migratetype = cc->migratetype;
1848 int ret;
1849
1850
1851 if (compact_scanners_met(cc)) {
1852
1853 reset_cached_positions(cc->zone);
1854
1855
1856
1857
1858
1859
1860
1861 if (cc->direct_compaction)
1862 cc->zone->compact_blockskip_flush = true;
1863
1864 if (cc->whole_zone)
1865 return COMPACT_COMPLETE;
1866 else
1867 return COMPACT_PARTIAL_SKIPPED;
1868 }
1869
1870 if (is_via_compact_memory(cc->order))
1871 return COMPACT_CONTINUE;
1872
1873
1874
1875
1876
1877
1878
1879 if (!IS_ALIGNED(cc->migrate_pfn, pageblock_nr_pages))
1880 return COMPACT_CONTINUE;
1881
1882
1883 ret = COMPACT_NO_SUITABLE_PAGE;
1884 for (order = cc->order; order < MAX_ORDER; order++) {
1885 struct free_area *area = &cc->zone->free_area[order];
1886 bool can_steal;
1887
1888
1889 if (!list_empty(&area->free_list[migratetype]))
1890 return COMPACT_SUCCESS;
1891
1892#ifdef CONFIG_CMA
1893
1894 if (migratetype == MIGRATE_MOVABLE &&
1895 !list_empty(&area->free_list[MIGRATE_CMA]))
1896 return COMPACT_SUCCESS;
1897#endif
1898
1899
1900
1901
1902 if (find_suitable_fallback(area, order, migratetype,
1903 true, &can_steal) != -1) {
1904
1905
1906 if (migratetype == MIGRATE_MOVABLE)
1907 return COMPACT_SUCCESS;
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917 if (cc->mode == MIGRATE_ASYNC ||
1918 IS_ALIGNED(cc->migrate_pfn,
1919 pageblock_nr_pages)) {
1920 return COMPACT_SUCCESS;
1921 }
1922
1923 ret = COMPACT_CONTINUE;
1924 break;
1925 }
1926 }
1927
1928 if (cc->contended || fatal_signal_pending(current))
1929 ret = COMPACT_CONTENDED;
1930
1931 return ret;
1932}
1933
1934static enum compact_result compact_finished(struct compact_control *cc)
1935{
1936 int ret;
1937
1938 ret = __compact_finished(cc);
1939 trace_mm_compaction_finished(cc->zone, cc->order, ret);
1940 if (ret == COMPACT_NO_SUITABLE_PAGE)
1941 ret = COMPACT_CONTINUE;
1942
1943 return ret;
1944}
1945
1946
1947
1948
1949
1950
1951
1952
1953static enum compact_result __compaction_suitable(struct zone *zone, int order,
1954 unsigned int alloc_flags,
1955 int classzone_idx,
1956 unsigned long wmark_target)
1957{
1958 unsigned long watermark;
1959
1960 if (is_via_compact_memory(order))
1961 return COMPACT_CONTINUE;
1962
1963 watermark = wmark_pages(zone, alloc_flags & ALLOC_WMARK_MASK);
1964
1965
1966
1967
1968 if (zone_watermark_ok(zone, order, watermark, classzone_idx,
1969 alloc_flags))
1970 return COMPACT_SUCCESS;
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986 watermark = (order > PAGE_ALLOC_COSTLY_ORDER) ?
1987 low_wmark_pages(zone) : min_wmark_pages(zone);
1988 watermark += compact_gap(order);
1989 if (!__zone_watermark_ok(zone, 0, watermark, classzone_idx,
1990 ALLOC_CMA, wmark_target))
1991 return COMPACT_SKIPPED;
1992
1993 return COMPACT_CONTINUE;
1994}
1995
1996enum compact_result compaction_suitable(struct zone *zone, int order,
1997 unsigned int alloc_flags,
1998 int classzone_idx)
1999{
2000 enum compact_result ret;
2001 int fragindex;
2002
2003 ret = __compaction_suitable(zone, order, alloc_flags, classzone_idx,
2004 zone_page_state(zone, NR_FREE_PAGES));
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021 if (ret == COMPACT_CONTINUE && (order > PAGE_ALLOC_COSTLY_ORDER)) {
2022 fragindex = fragmentation_index(zone, order);
2023 if (fragindex >= 0 && fragindex <= sysctl_extfrag_threshold)
2024 ret = COMPACT_NOT_SUITABLE_ZONE;
2025 }
2026
2027 trace_mm_compaction_suitable(zone, order, ret);
2028 if (ret == COMPACT_NOT_SUITABLE_ZONE)
2029 ret = COMPACT_SKIPPED;
2030
2031 return ret;
2032}
2033
2034bool compaction_zonelist_suitable(struct alloc_context *ac, int order,
2035 int alloc_flags)
2036{
2037 struct zone *zone;
2038 struct zoneref *z;
2039
2040
2041
2042
2043
2044 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx,
2045 ac->nodemask) {
2046 unsigned long available;
2047 enum compact_result compact_result;
2048
2049
2050
2051
2052
2053
2054
2055 available = zone_reclaimable_pages(zone) / order;
2056 available += zone_page_state_snapshot(zone, NR_FREE_PAGES);
2057 compact_result = __compaction_suitable(zone, order, alloc_flags,
2058 ac_classzone_idx(ac), available);
2059 if (compact_result != COMPACT_SKIPPED)
2060 return true;
2061 }
2062
2063 return false;
2064}
2065
2066static enum compact_result
2067compact_zone(struct compact_control *cc, struct capture_control *capc)
2068{
2069 enum compact_result ret;
2070 unsigned long start_pfn = cc->zone->zone_start_pfn;
2071 unsigned long end_pfn = zone_end_pfn(cc->zone);
2072 unsigned long last_migrated_pfn;
2073 const bool sync = cc->mode != MIGRATE_ASYNC;
2074 bool update_cached;
2075
2076 cc->migratetype = gfpflags_to_migratetype(cc->gfp_mask);
2077 ret = compaction_suitable(cc->zone, cc->order, cc->alloc_flags,
2078 cc->classzone_idx);
2079
2080 if (ret == COMPACT_SUCCESS || ret == COMPACT_SKIPPED)
2081 return ret;
2082
2083
2084 VM_BUG_ON(ret != COMPACT_CONTINUE);
2085
2086
2087
2088
2089
2090 if (compaction_restarting(cc->zone, cc->order))
2091 __reset_isolation_suitable(cc->zone);
2092
2093
2094
2095
2096
2097
2098
2099 cc->fast_start_pfn = 0;
2100 if (cc->whole_zone) {
2101 cc->migrate_pfn = start_pfn;
2102 cc->free_pfn = pageblock_start_pfn(end_pfn - 1);
2103 } else {
2104 cc->migrate_pfn = cc->zone->compact_cached_migrate_pfn[sync];
2105 cc->free_pfn = cc->zone->compact_cached_free_pfn;
2106 if (cc->free_pfn < start_pfn || cc->free_pfn >= end_pfn) {
2107 cc->free_pfn = pageblock_start_pfn(end_pfn - 1);
2108 cc->zone->compact_cached_free_pfn = cc->free_pfn;
2109 }
2110 if (cc->migrate_pfn < start_pfn || cc->migrate_pfn >= end_pfn) {
2111 cc->migrate_pfn = start_pfn;
2112 cc->zone->compact_cached_migrate_pfn[0] = cc->migrate_pfn;
2113 cc->zone->compact_cached_migrate_pfn[1] = cc->migrate_pfn;
2114 }
2115
2116 if (cc->migrate_pfn <= cc->zone->compact_init_migrate_pfn)
2117 cc->whole_zone = true;
2118 }
2119
2120 last_migrated_pfn = 0;
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130 update_cached = !sync &&
2131 cc->zone->compact_cached_migrate_pfn[0] == cc->zone->compact_cached_migrate_pfn[1];
2132
2133 trace_mm_compaction_begin(start_pfn, cc->migrate_pfn,
2134 cc->free_pfn, end_pfn, sync);
2135
2136 migrate_prep_local();
2137
2138 while ((ret = compact_finished(cc)) == COMPACT_CONTINUE) {
2139 int err;
2140 unsigned long start_pfn = cc->migrate_pfn;
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150 cc->rescan = false;
2151 if (pageblock_start_pfn(last_migrated_pfn) ==
2152 pageblock_start_pfn(start_pfn)) {
2153 cc->rescan = true;
2154 }
2155
2156 switch (isolate_migratepages(cc->zone, cc)) {
2157 case ISOLATE_ABORT:
2158 ret = COMPACT_CONTENDED;
2159 putback_movable_pages(&cc->migratepages);
2160 cc->nr_migratepages = 0;
2161 last_migrated_pfn = 0;
2162 goto out;
2163 case ISOLATE_NONE:
2164 if (update_cached) {
2165 cc->zone->compact_cached_migrate_pfn[1] =
2166 cc->zone->compact_cached_migrate_pfn[0];
2167 }
2168
2169
2170
2171
2172
2173
2174 goto check_drain;
2175 case ISOLATE_SUCCESS:
2176 update_cached = false;
2177 last_migrated_pfn = start_pfn;
2178 ;
2179 }
2180
2181 err = migrate_pages(&cc->migratepages, compaction_alloc,
2182 compaction_free, (unsigned long)cc, cc->mode,
2183 MR_COMPACTION);
2184
2185 trace_mm_compaction_migratepages(cc->nr_migratepages, err,
2186 &cc->migratepages);
2187
2188
2189 cc->nr_migratepages = 0;
2190 if (err) {
2191 putback_movable_pages(&cc->migratepages);
2192
2193
2194
2195
2196 if (err == -ENOMEM && !compact_scanners_met(cc)) {
2197 ret = COMPACT_CONTENDED;
2198 goto out;
2199 }
2200
2201
2202
2203
2204 if (cc->direct_compaction &&
2205 (cc->mode == MIGRATE_ASYNC)) {
2206 cc->migrate_pfn = block_end_pfn(
2207 cc->migrate_pfn - 1, cc->order);
2208
2209 last_migrated_pfn = 0;
2210 }
2211 }
2212
2213check_drain:
2214
2215
2216
2217
2218
2219
2220
2221 if (cc->order > 0 && last_migrated_pfn) {
2222 int cpu;
2223 unsigned long current_block_start =
2224 block_start_pfn(cc->migrate_pfn, cc->order);
2225
2226 if (last_migrated_pfn < current_block_start) {
2227 cpu = get_cpu();
2228 lru_add_drain_cpu(cpu);
2229 drain_local_pages(cc->zone);
2230 put_cpu();
2231
2232 last_migrated_pfn = 0;
2233 }
2234 }
2235
2236
2237 if (capc && capc->page) {
2238 ret = COMPACT_SUCCESS;
2239 break;
2240 }
2241 }
2242
2243out:
2244
2245
2246
2247
2248 if (cc->nr_freepages > 0) {
2249 unsigned long free_pfn = release_freepages(&cc->freepages);
2250
2251 cc->nr_freepages = 0;
2252 VM_BUG_ON(free_pfn == 0);
2253
2254 free_pfn = pageblock_start_pfn(free_pfn);
2255
2256
2257
2258
2259 if (free_pfn > cc->zone->compact_cached_free_pfn)
2260 cc->zone->compact_cached_free_pfn = free_pfn;
2261 }
2262
2263 count_compact_events(COMPACTMIGRATE_SCANNED, cc->total_migrate_scanned);
2264 count_compact_events(COMPACTFREE_SCANNED, cc->total_free_scanned);
2265
2266 trace_mm_compaction_end(start_pfn, cc->migrate_pfn,
2267 cc->free_pfn, end_pfn, sync, ret);
2268
2269 return ret;
2270}
2271
2272static enum compact_result compact_zone_order(struct zone *zone, int order,
2273 gfp_t gfp_mask, enum compact_priority prio,
2274 unsigned int alloc_flags, int classzone_idx,
2275 struct page **capture)
2276{
2277 enum compact_result ret;
2278 struct compact_control cc = {
2279 .nr_freepages = 0,
2280 .nr_migratepages = 0,
2281 .total_migrate_scanned = 0,
2282 .total_free_scanned = 0,
2283 .order = order,
2284 .search_order = order,
2285 .gfp_mask = gfp_mask,
2286 .zone = zone,
2287 .mode = (prio == COMPACT_PRIO_ASYNC) ?
2288 MIGRATE_ASYNC : MIGRATE_SYNC_LIGHT,
2289 .alloc_flags = alloc_flags,
2290 .classzone_idx = classzone_idx,
2291 .direct_compaction = true,
2292 .whole_zone = (prio == MIN_COMPACT_PRIORITY),
2293 .ignore_skip_hint = (prio == MIN_COMPACT_PRIORITY),
2294 .ignore_block_suitable = (prio == MIN_COMPACT_PRIORITY)
2295 };
2296 struct capture_control capc = {
2297 .cc = &cc,
2298 .page = NULL,
2299 };
2300
2301 if (capture)
2302 current->capture_control = &capc;
2303 INIT_LIST_HEAD(&cc.freepages);
2304 INIT_LIST_HEAD(&cc.migratepages);
2305
2306 ret = compact_zone(&cc, &capc);
2307
2308 VM_BUG_ON(!list_empty(&cc.freepages));
2309 VM_BUG_ON(!list_empty(&cc.migratepages));
2310
2311 *capture = capc.page;
2312 current->capture_control = NULL;
2313
2314 return ret;
2315}
2316
2317int sysctl_extfrag_threshold = 500;
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329enum compact_result try_to_compact_pages(gfp_t gfp_mask, unsigned int order,
2330 unsigned int alloc_flags, const struct alloc_context *ac,
2331 enum compact_priority prio, struct page **capture)
2332{
2333 int may_perform_io = gfp_mask & __GFP_IO;
2334 struct zoneref *z;
2335 struct zone *zone;
2336 enum compact_result rc = COMPACT_SKIPPED;
2337
2338
2339
2340
2341
2342 if (!may_perform_io)
2343 return COMPACT_SKIPPED;
2344
2345 trace_mm_compaction_try_to_compact_pages(order, gfp_mask, prio);
2346
2347
2348 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx,
2349 ac->nodemask) {
2350 enum compact_result status;
2351
2352 if (prio > MIN_COMPACT_PRIORITY
2353 && compaction_deferred(zone, order)) {
2354 rc = max_t(enum compact_result, COMPACT_DEFERRED, rc);
2355 continue;
2356 }
2357
2358 status = compact_zone_order(zone, order, gfp_mask, prio,
2359 alloc_flags, ac_classzone_idx(ac), capture);
2360 rc = max(status, rc);
2361
2362
2363 if (status == COMPACT_SUCCESS) {
2364
2365
2366
2367
2368
2369
2370 compaction_defer_reset(zone, order, false);
2371
2372 break;
2373 }
2374
2375 if (prio != COMPACT_PRIO_ASYNC && (status == COMPACT_COMPLETE ||
2376 status == COMPACT_PARTIAL_SKIPPED))
2377
2378
2379
2380
2381
2382 defer_compaction(zone, order);
2383
2384
2385
2386
2387
2388
2389 if ((prio == COMPACT_PRIO_ASYNC && need_resched())
2390 || fatal_signal_pending(current))
2391 break;
2392 }
2393
2394 return rc;
2395}
2396
2397
2398
2399static void compact_node(int nid)
2400{
2401 pg_data_t *pgdat = NODE_DATA(nid);
2402 int zoneid;
2403 struct zone *zone;
2404 struct compact_control cc = {
2405 .order = -1,
2406 .total_migrate_scanned = 0,
2407 .total_free_scanned = 0,
2408 .mode = MIGRATE_SYNC,
2409 .ignore_skip_hint = true,
2410 .whole_zone = true,
2411 .gfp_mask = GFP_KERNEL,
2412 };
2413
2414
2415 for (zoneid = 0; zoneid < MAX_NR_ZONES; zoneid++) {
2416
2417 zone = &pgdat->node_zones[zoneid];
2418 if (!populated_zone(zone))
2419 continue;
2420
2421 cc.nr_freepages = 0;
2422 cc.nr_migratepages = 0;
2423 cc.zone = zone;
2424 INIT_LIST_HEAD(&cc.freepages);
2425 INIT_LIST_HEAD(&cc.migratepages);
2426
2427 compact_zone(&cc, NULL);
2428
2429 VM_BUG_ON(!list_empty(&cc.freepages));
2430 VM_BUG_ON(!list_empty(&cc.migratepages));
2431 }
2432}
2433
2434
2435static void compact_nodes(void)
2436{
2437 int nid;
2438
2439
2440 lru_add_drain_all();
2441
2442 for_each_online_node(nid)
2443 compact_node(nid);
2444}
2445
2446
2447int sysctl_compact_memory;
2448
2449
2450
2451
2452
2453int sysctl_compaction_handler(struct ctl_table *table, int write,
2454 void __user *buffer, size_t *length, loff_t *ppos)
2455{
2456 if (write)
2457 compact_nodes();
2458
2459 return 0;
2460}
2461
2462#if defined(CONFIG_SYSFS) && defined(CONFIG_NUMA)
2463static ssize_t sysfs_compact_node(struct device *dev,
2464 struct device_attribute *attr,
2465 const char *buf, size_t count)
2466{
2467 int nid = dev->id;
2468
2469 if (nid >= 0 && nid < nr_node_ids && node_online(nid)) {
2470
2471 lru_add_drain_all();
2472
2473 compact_node(nid);
2474 }
2475
2476 return count;
2477}
2478static DEVICE_ATTR(compact, 0200, NULL, sysfs_compact_node);
2479
2480int compaction_register_node(struct node *node)
2481{
2482 return device_create_file(&node->dev, &dev_attr_compact);
2483}
2484
2485void compaction_unregister_node(struct node *node)
2486{
2487 return device_remove_file(&node->dev, &dev_attr_compact);
2488}
2489#endif
2490
2491static inline bool kcompactd_work_requested(pg_data_t *pgdat)
2492{
2493 return pgdat->kcompactd_max_order > 0 || kthread_should_stop();
2494}
2495
2496static bool kcompactd_node_suitable(pg_data_t *pgdat)
2497{
2498 int zoneid;
2499 struct zone *zone;
2500 enum zone_type classzone_idx = pgdat->kcompactd_classzone_idx;
2501
2502 for (zoneid = 0; zoneid <= classzone_idx; zoneid++) {
2503 zone = &pgdat->node_zones[zoneid];
2504
2505 if (!populated_zone(zone))
2506 continue;
2507
2508 if (compaction_suitable(zone, pgdat->kcompactd_max_order, 0,
2509 classzone_idx) == COMPACT_CONTINUE)
2510 return true;
2511 }
2512
2513 return false;
2514}
2515
2516static void kcompactd_do_work(pg_data_t *pgdat)
2517{
2518
2519
2520
2521
2522 int zoneid;
2523 struct zone *zone;
2524 struct compact_control cc = {
2525 .order = pgdat->kcompactd_max_order,
2526 .search_order = pgdat->kcompactd_max_order,
2527 .total_migrate_scanned = 0,
2528 .total_free_scanned = 0,
2529 .classzone_idx = pgdat->kcompactd_classzone_idx,
2530 .mode = MIGRATE_SYNC_LIGHT,
2531 .ignore_skip_hint = false,
2532 .gfp_mask = GFP_KERNEL,
2533 };
2534 trace_mm_compaction_kcompactd_wake(pgdat->node_id, cc.order,
2535 cc.classzone_idx);
2536 count_compact_event(KCOMPACTD_WAKE);
2537
2538 for (zoneid = 0; zoneid <= cc.classzone_idx; zoneid++) {
2539 int status;
2540
2541 zone = &pgdat->node_zones[zoneid];
2542 if (!populated_zone(zone))
2543 continue;
2544
2545 if (compaction_deferred(zone, cc.order))
2546 continue;
2547
2548 if (compaction_suitable(zone, cc.order, 0, zoneid) !=
2549 COMPACT_CONTINUE)
2550 continue;
2551
2552 cc.nr_freepages = 0;
2553 cc.nr_migratepages = 0;
2554 cc.total_migrate_scanned = 0;
2555 cc.total_free_scanned = 0;
2556 cc.zone = zone;
2557 INIT_LIST_HEAD(&cc.freepages);
2558 INIT_LIST_HEAD(&cc.migratepages);
2559
2560 if (kthread_should_stop())
2561 return;
2562 status = compact_zone(&cc, NULL);
2563
2564 if (status == COMPACT_SUCCESS) {
2565 compaction_defer_reset(zone, cc.order, false);
2566 } else if (status == COMPACT_PARTIAL_SKIPPED || status == COMPACT_COMPLETE) {
2567
2568
2569
2570
2571
2572
2573 drain_all_pages(zone);
2574
2575
2576
2577
2578
2579 defer_compaction(zone, cc.order);
2580 }
2581
2582 count_compact_events(KCOMPACTD_MIGRATE_SCANNED,
2583 cc.total_migrate_scanned);
2584 count_compact_events(KCOMPACTD_FREE_SCANNED,
2585 cc.total_free_scanned);
2586
2587 VM_BUG_ON(!list_empty(&cc.freepages));
2588 VM_BUG_ON(!list_empty(&cc.migratepages));
2589 }
2590
2591
2592
2593
2594
2595
2596 if (pgdat->kcompactd_max_order <= cc.order)
2597 pgdat->kcompactd_max_order = 0;
2598 if (pgdat->kcompactd_classzone_idx >= cc.classzone_idx)
2599 pgdat->kcompactd_classzone_idx = pgdat->nr_zones - 1;
2600}
2601
2602void wakeup_kcompactd(pg_data_t *pgdat, int order, int classzone_idx)
2603{
2604 if (!order)
2605 return;
2606
2607 if (pgdat->kcompactd_max_order < order)
2608 pgdat->kcompactd_max_order = order;
2609
2610 if (pgdat->kcompactd_classzone_idx > classzone_idx)
2611 pgdat->kcompactd_classzone_idx = classzone_idx;
2612
2613
2614
2615
2616
2617 if (!wq_has_sleeper(&pgdat->kcompactd_wait))
2618 return;
2619
2620 if (!kcompactd_node_suitable(pgdat))
2621 return;
2622
2623 trace_mm_compaction_wakeup_kcompactd(pgdat->node_id, order,
2624 classzone_idx);
2625 wake_up_interruptible(&pgdat->kcompactd_wait);
2626}
2627
2628
2629
2630
2631
2632static int kcompactd(void *p)
2633{
2634 pg_data_t *pgdat = (pg_data_t*)p;
2635 struct task_struct *tsk = current;
2636
2637 const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id);
2638
2639 if (!cpumask_empty(cpumask))
2640 set_cpus_allowed_ptr(tsk, cpumask);
2641
2642 set_freezable();
2643
2644 pgdat->kcompactd_max_order = 0;
2645 pgdat->kcompactd_classzone_idx = pgdat->nr_zones - 1;
2646
2647 while (!kthread_should_stop()) {
2648 unsigned long pflags;
2649
2650 trace_mm_compaction_kcompactd_sleep(pgdat->node_id);
2651 wait_event_freezable(pgdat->kcompactd_wait,
2652 kcompactd_work_requested(pgdat));
2653
2654 psi_memstall_enter(&pflags);
2655 kcompactd_do_work(pgdat);
2656 psi_memstall_leave(&pflags);
2657 }
2658
2659 return 0;
2660}
2661
2662
2663
2664
2665
2666int kcompactd_run(int nid)
2667{
2668 pg_data_t *pgdat = NODE_DATA(nid);
2669 int ret = 0;
2670
2671 if (pgdat->kcompactd)
2672 return 0;
2673
2674 pgdat->kcompactd = kthread_run(kcompactd, pgdat, "kcompactd%d", nid);
2675 if (IS_ERR(pgdat->kcompactd)) {
2676 pr_err("Failed to start kcompactd on node %d\n", nid);
2677 ret = PTR_ERR(pgdat->kcompactd);
2678 pgdat->kcompactd = NULL;
2679 }
2680 return ret;
2681}
2682
2683
2684
2685
2686
2687void kcompactd_stop(int nid)
2688{
2689 struct task_struct *kcompactd = NODE_DATA(nid)->kcompactd;
2690
2691 if (kcompactd) {
2692 kthread_stop(kcompactd);
2693 NODE_DATA(nid)->kcompactd = NULL;
2694 }
2695}
2696
2697
2698
2699
2700
2701
2702
2703static int kcompactd_cpu_online(unsigned int cpu)
2704{
2705 int nid;
2706
2707 for_each_node_state(nid, N_MEMORY) {
2708 pg_data_t *pgdat = NODE_DATA(nid);
2709 const struct cpumask *mask;
2710
2711 mask = cpumask_of_node(pgdat->node_id);
2712
2713 if (cpumask_any_and(cpu_online_mask, mask) < nr_cpu_ids)
2714
2715 set_cpus_allowed_ptr(pgdat->kcompactd, mask);
2716 }
2717 return 0;
2718}
2719
2720static int __init kcompactd_init(void)
2721{
2722 int nid;
2723 int ret;
2724
2725 ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
2726 "mm/compaction:online",
2727 kcompactd_cpu_online, NULL);
2728 if (ret < 0) {
2729 pr_err("kcompactd: failed to register hotplug callbacks.\n");
2730 return ret;
2731 }
2732
2733 for_each_node_state(nid, N_MEMORY)
2734 kcompactd_run(nid);
2735 return 0;
2736}
2737subsys_initcall(kcompactd_init)
2738
2739#endif
2740