1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18#include <linux/stddef.h>
19#include <linux/mm.h>
20#include <linux/highmem.h>
21#include <linux/swap.h>
22#include <linux/interrupt.h>
23#include <linux/pagemap.h>
24#include <linux/jiffies.h>
25#include <linux/memblock.h>
26#include <linux/compiler.h>
27#include <linux/kernel.h>
28#include <linux/kasan.h>
29#include <linux/module.h>
30#include <linux/suspend.h>
31#include <linux/pagevec.h>
32#include <linux/blkdev.h>
33#include <linux/slab.h>
34#include <linux/ratelimit.h>
35#include <linux/oom.h>
36#include <linux/topology.h>
37#include <linux/sysctl.h>
38#include <linux/cpu.h>
39#include <linux/cpuset.h>
40#include <linux/memory_hotplug.h>
41#include <linux/nodemask.h>
42#include <linux/vmalloc.h>
43#include <linux/vmstat.h>
44#include <linux/mempolicy.h>
45#include <linux/memremap.h>
46#include <linux/stop_machine.h>
47#include <linux/random.h>
48#include <linux/sort.h>
49#include <linux/pfn.h>
50#include <linux/backing-dev.h>
51#include <linux/fault-inject.h>
52#include <linux/page-isolation.h>
53#include <linux/page_ext.h>
54#include <linux/debugobjects.h>
55#include <linux/kmemleak.h>
56#include <linux/compaction.h>
57#include <trace/events/kmem.h>
58#include <trace/events/oom.h>
59#include <linux/prefetch.h>
60#include <linux/mm_inline.h>
61#include <linux/migrate.h>
62#include <linux/hugetlb.h>
63#include <linux/sched/rt.h>
64#include <linux/sched/mm.h>
65#include <linux/page_owner.h>
66#include <linux/kthread.h>
67#include <linux/memcontrol.h>
68#include <linux/ftrace.h>
69#include <linux/lockdep.h>
70#include <linux/nmi.h>
71#include <linux/psi.h>
72
73#include <asm/sections.h>
74#include <asm/tlbflush.h>
75#include <asm/div64.h>
76#include "internal.h"
77#include "shuffle.h"
78
79
80static DEFINE_MUTEX(pcp_batch_high_lock);
81#define MIN_PERCPU_PAGELIST_FRACTION (8)
82
83#ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID
84DEFINE_PER_CPU(int, numa_node);
85EXPORT_PER_CPU_SYMBOL(numa_node);
86#endif
87
88DEFINE_STATIC_KEY_TRUE(vm_numa_stat_key);
89
90#ifdef CONFIG_HAVE_MEMORYLESS_NODES
91
92
93
94
95
96
97DEFINE_PER_CPU(int, _numa_mem_);
98EXPORT_PER_CPU_SYMBOL(_numa_mem_);
99int _node_numa_mem_[MAX_NUMNODES];
100#endif
101
102
103struct pcpu_drain {
104 struct zone *zone;
105 struct work_struct work;
106};
107DEFINE_MUTEX(pcpu_drain_mutex);
108DEFINE_PER_CPU(struct pcpu_drain, pcpu_drain);
109
110#ifdef CONFIG_GCC_PLUGIN_LATENT_ENTROPY
111volatile unsigned long latent_entropy __latent_entropy;
112EXPORT_SYMBOL(latent_entropy);
113#endif
114
115
116
117
118nodemask_t node_states[NR_NODE_STATES] __read_mostly = {
119 [N_POSSIBLE] = NODE_MASK_ALL,
120 [N_ONLINE] = { { [0] = 1UL } },
121#ifndef CONFIG_NUMA
122 [N_NORMAL_MEMORY] = { { [0] = 1UL } },
123#ifdef CONFIG_HIGHMEM
124 [N_HIGH_MEMORY] = { { [0] = 1UL } },
125#endif
126 [N_MEMORY] = { { [0] = 1UL } },
127 [N_CPU] = { { [0] = 1UL } },
128#endif
129};
130EXPORT_SYMBOL(node_states);
131
132atomic_long_t _totalram_pages __read_mostly;
133EXPORT_SYMBOL(_totalram_pages);
134unsigned long totalreserve_pages __read_mostly;
135unsigned long totalcma_pages __read_mostly;
136
137int percpu_pagelist_fraction;
138gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK;
139
140
141
142
143
144
145
146
147
148static inline int get_pcppage_migratetype(struct page *page)
149{
150 return page->index;
151}
152
153static inline void set_pcppage_migratetype(struct page *page, int migratetype)
154{
155 page->index = migratetype;
156}
157
158#ifdef CONFIG_PM_SLEEP
159
160
161
162
163
164
165
166
167
168
169static gfp_t saved_gfp_mask;
170
171void pm_restore_gfp_mask(void)
172{
173 WARN_ON(!mutex_is_locked(&system_transition_mutex));
174 if (saved_gfp_mask) {
175 gfp_allowed_mask = saved_gfp_mask;
176 saved_gfp_mask = 0;
177 }
178}
179
180void pm_restrict_gfp_mask(void)
181{
182 WARN_ON(!mutex_is_locked(&system_transition_mutex));
183 WARN_ON(saved_gfp_mask);
184 saved_gfp_mask = gfp_allowed_mask;
185 gfp_allowed_mask &= ~(__GFP_IO | __GFP_FS);
186}
187
188bool pm_suspended_storage(void)
189{
190 if ((gfp_allowed_mask & (__GFP_IO | __GFP_FS)) == (__GFP_IO | __GFP_FS))
191 return false;
192 return true;
193}
194#endif
195
196#ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE
197unsigned int pageblock_order __read_mostly;
198#endif
199
200static void __free_pages_ok(struct page *page, unsigned int order);
201
202
203
204
205
206
207
208
209
210
211
212
213int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES] = {
214#ifdef CONFIG_ZONE_DMA
215 [ZONE_DMA] = 256,
216#endif
217#ifdef CONFIG_ZONE_DMA32
218 [ZONE_DMA32] = 256,
219#endif
220 [ZONE_NORMAL] = 32,
221#ifdef CONFIG_HIGHMEM
222 [ZONE_HIGHMEM] = 0,
223#endif
224 [ZONE_MOVABLE] = 0,
225};
226
227EXPORT_SYMBOL(totalram_pages);
228
229static char * const zone_names[MAX_NR_ZONES] = {
230#ifdef CONFIG_ZONE_DMA
231 "DMA",
232#endif
233#ifdef CONFIG_ZONE_DMA32
234 "DMA32",
235#endif
236 "Normal",
237#ifdef CONFIG_HIGHMEM
238 "HighMem",
239#endif
240 "Movable",
241#ifdef CONFIG_ZONE_DEVICE
242 "Device",
243#endif
244};
245
246const char * const migratetype_names[MIGRATE_TYPES] = {
247 "Unmovable",
248 "Movable",
249 "Reclaimable",
250 "HighAtomic",
251#ifdef CONFIG_CMA
252 "CMA",
253#endif
254#ifdef CONFIG_MEMORY_ISOLATION
255 "Isolate",
256#endif
257};
258
259compound_page_dtor * const compound_page_dtors[] = {
260 NULL,
261 free_compound_page,
262#ifdef CONFIG_HUGETLB_PAGE
263 free_huge_page,
264#endif
265#ifdef CONFIG_TRANSPARENT_HUGEPAGE
266 free_transhuge_page,
267#endif
268};
269
270int min_free_kbytes = 1024;
271int user_min_free_kbytes = -1;
272#ifdef CONFIG_DISCONTIGMEM
273
274
275
276
277
278
279
280
281
282int watermark_boost_factor __read_mostly;
283#else
284int watermark_boost_factor __read_mostly = 15000;
285#endif
286int watermark_scale_factor = 10;
287
288static unsigned long nr_kernel_pages __initdata;
289static unsigned long nr_all_pages __initdata;
290static unsigned long dma_reserve __initdata;
291
292#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
293static unsigned long arch_zone_lowest_possible_pfn[MAX_NR_ZONES] __initdata;
294static unsigned long arch_zone_highest_possible_pfn[MAX_NR_ZONES] __initdata;
295static unsigned long required_kernelcore __initdata;
296static unsigned long required_kernelcore_percent __initdata;
297static unsigned long required_movablecore __initdata;
298static unsigned long required_movablecore_percent __initdata;
299static unsigned long zone_movable_pfn[MAX_NUMNODES] __initdata;
300static bool mirrored_kernelcore __meminitdata;
301
302
303int movable_zone;
304EXPORT_SYMBOL(movable_zone);
305#endif
306
307#if MAX_NUMNODES > 1
308unsigned int nr_node_ids __read_mostly = MAX_NUMNODES;
309unsigned int nr_online_nodes __read_mostly = 1;
310EXPORT_SYMBOL(nr_node_ids);
311EXPORT_SYMBOL(nr_online_nodes);
312#endif
313
314int page_group_by_mobility_disabled __read_mostly;
315
316#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
317
318
319
320
321
322static DEFINE_STATIC_KEY_TRUE(deferred_pages);
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337static inline void kasan_free_nondeferred_pages(struct page *page, int order)
338{
339 if (!static_branch_unlikely(&deferred_pages))
340 kasan_free_pages(page, order);
341}
342
343
344static inline bool __meminit early_page_uninitialised(unsigned long pfn)
345{
346 int nid = early_pfn_to_nid(pfn);
347
348 if (node_online(nid) && pfn >= NODE_DATA(nid)->first_deferred_pfn)
349 return true;
350
351 return false;
352}
353
354
355
356
357
358static bool __meminit
359defer_init(int nid, unsigned long pfn, unsigned long end_pfn)
360{
361 static unsigned long prev_end_pfn, nr_initialised;
362
363
364
365
366
367 if (prev_end_pfn != end_pfn) {
368 prev_end_pfn = end_pfn;
369 nr_initialised = 0;
370 }
371
372
373 if (end_pfn < pgdat_end_pfn(NODE_DATA(nid)))
374 return false;
375
376
377
378
379
380 nr_initialised++;
381 if ((nr_initialised > PAGES_PER_SECTION) &&
382 (pfn & (PAGES_PER_SECTION - 1)) == 0) {
383 NODE_DATA(nid)->first_deferred_pfn = pfn;
384 return true;
385 }
386 return false;
387}
388#else
389#define kasan_free_nondeferred_pages(p, o) kasan_free_pages(p, o)
390
391static inline bool early_page_uninitialised(unsigned long pfn)
392{
393 return false;
394}
395
396static inline bool defer_init(int nid, unsigned long pfn, unsigned long end_pfn)
397{
398 return false;
399}
400#endif
401
402
403static inline unsigned long *get_pageblock_bitmap(struct page *page,
404 unsigned long pfn)
405{
406#ifdef CONFIG_SPARSEMEM
407 return __pfn_to_section(pfn)->pageblock_flags;
408#else
409 return page_zone(page)->pageblock_flags;
410#endif
411}
412
413static inline int pfn_to_bitidx(struct page *page, unsigned long pfn)
414{
415#ifdef CONFIG_SPARSEMEM
416 pfn &= (PAGES_PER_SECTION-1);
417 return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS;
418#else
419 pfn = pfn - round_down(page_zone(page)->zone_start_pfn, pageblock_nr_pages);
420 return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS;
421#endif
422}
423
424
425
426
427
428
429
430
431
432
433static __always_inline unsigned long __get_pfnblock_flags_mask(struct page *page,
434 unsigned long pfn,
435 unsigned long end_bitidx,
436 unsigned long mask)
437{
438 unsigned long *bitmap;
439 unsigned long bitidx, word_bitidx;
440 unsigned long word;
441
442 bitmap = get_pageblock_bitmap(page, pfn);
443 bitidx = pfn_to_bitidx(page, pfn);
444 word_bitidx = bitidx / BITS_PER_LONG;
445 bitidx &= (BITS_PER_LONG-1);
446
447 word = bitmap[word_bitidx];
448 bitidx += end_bitidx;
449 return (word >> (BITS_PER_LONG - bitidx - 1)) & mask;
450}
451
452unsigned long get_pfnblock_flags_mask(struct page *page, unsigned long pfn,
453 unsigned long end_bitidx,
454 unsigned long mask)
455{
456 return __get_pfnblock_flags_mask(page, pfn, end_bitidx, mask);
457}
458
459static __always_inline int get_pfnblock_migratetype(struct page *page, unsigned long pfn)
460{
461 return __get_pfnblock_flags_mask(page, pfn, PB_migrate_end, MIGRATETYPE_MASK);
462}
463
464
465
466
467
468
469
470
471
472void set_pfnblock_flags_mask(struct page *page, unsigned long flags,
473 unsigned long pfn,
474 unsigned long end_bitidx,
475 unsigned long mask)
476{
477 unsigned long *bitmap;
478 unsigned long bitidx, word_bitidx;
479 unsigned long old_word, word;
480
481 BUILD_BUG_ON(NR_PAGEBLOCK_BITS != 4);
482 BUILD_BUG_ON(MIGRATE_TYPES > (1 << PB_migratetype_bits));
483
484 bitmap = get_pageblock_bitmap(page, pfn);
485 bitidx = pfn_to_bitidx(page, pfn);
486 word_bitidx = bitidx / BITS_PER_LONG;
487 bitidx &= (BITS_PER_LONG-1);
488
489 VM_BUG_ON_PAGE(!zone_spans_pfn(page_zone(page), pfn), page);
490
491 bitidx += end_bitidx;
492 mask <<= (BITS_PER_LONG - bitidx - 1);
493 flags <<= (BITS_PER_LONG - bitidx - 1);
494
495 word = READ_ONCE(bitmap[word_bitidx]);
496 for (;;) {
497 old_word = cmpxchg(&bitmap[word_bitidx], word, (word & ~mask) | flags);
498 if (word == old_word)
499 break;
500 word = old_word;
501 }
502}
503
504void set_pageblock_migratetype(struct page *page, int migratetype)
505{
506 if (unlikely(page_group_by_mobility_disabled &&
507 migratetype < MIGRATE_PCPTYPES))
508 migratetype = MIGRATE_UNMOVABLE;
509
510 set_pageblock_flags_group(page, (unsigned long)migratetype,
511 PB_migrate, PB_migrate_end);
512}
513
514#ifdef CONFIG_DEBUG_VM
515static int page_outside_zone_boundaries(struct zone *zone, struct page *page)
516{
517 int ret = 0;
518 unsigned seq;
519 unsigned long pfn = page_to_pfn(page);
520 unsigned long sp, start_pfn;
521
522 do {
523 seq = zone_span_seqbegin(zone);
524 start_pfn = zone->zone_start_pfn;
525 sp = zone->spanned_pages;
526 if (!zone_spans_pfn(zone, pfn))
527 ret = 1;
528 } while (zone_span_seqretry(zone, seq));
529
530 if (ret)
531 pr_err("page 0x%lx outside node %d zone %s [ 0x%lx - 0x%lx ]\n",
532 pfn, zone_to_nid(zone), zone->name,
533 start_pfn, start_pfn + sp);
534
535 return ret;
536}
537
538static int page_is_consistent(struct zone *zone, struct page *page)
539{
540 if (!pfn_valid_within(page_to_pfn(page)))
541 return 0;
542 if (zone != page_zone(page))
543 return 0;
544
545 return 1;
546}
547
548
549
550static int __maybe_unused bad_range(struct zone *zone, struct page *page)
551{
552 if (page_outside_zone_boundaries(zone, page))
553 return 1;
554 if (!page_is_consistent(zone, page))
555 return 1;
556
557 return 0;
558}
559#else
560static inline int __maybe_unused bad_range(struct zone *zone, struct page *page)
561{
562 return 0;
563}
564#endif
565
566static void bad_page(struct page *page, const char *reason,
567 unsigned long bad_flags)
568{
569 static unsigned long resume;
570 static unsigned long nr_shown;
571 static unsigned long nr_unshown;
572
573
574
575
576
577 if (nr_shown == 60) {
578 if (time_before(jiffies, resume)) {
579 nr_unshown++;
580 goto out;
581 }
582 if (nr_unshown) {
583 pr_alert(
584 "BUG: Bad page state: %lu messages suppressed\n",
585 nr_unshown);
586 nr_unshown = 0;
587 }
588 nr_shown = 0;
589 }
590 if (nr_shown++ == 0)
591 resume = jiffies + 60 * HZ;
592
593 pr_alert("BUG: Bad page state in process %s pfn:%05lx\n",
594 current->comm, page_to_pfn(page));
595 __dump_page(page, reason);
596 bad_flags &= page->flags;
597 if (bad_flags)
598 pr_alert("bad because of flags: %#lx(%pGp)\n",
599 bad_flags, &bad_flags);
600 dump_page_owner(page);
601
602 print_modules();
603 dump_stack();
604out:
605
606 page_mapcount_reset(page);
607 add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
608}
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625void free_compound_page(struct page *page)
626{
627 __free_pages_ok(page, compound_order(page));
628}
629
630void prep_compound_page(struct page *page, unsigned int order)
631{
632 int i;
633 int nr_pages = 1 << order;
634
635 set_compound_page_dtor(page, COMPOUND_PAGE_DTOR);
636 set_compound_order(page, order);
637 __SetPageHead(page);
638 for (i = 1; i < nr_pages; i++) {
639 struct page *p = page + i;
640 set_page_count(p, 0);
641 p->mapping = TAIL_MAPPING;
642 set_compound_head(p, page);
643 }
644 atomic_set(compound_mapcount_ptr(page), -1);
645}
646
647#ifdef CONFIG_DEBUG_PAGEALLOC
648unsigned int _debug_guardpage_minorder;
649bool _debug_pagealloc_enabled __read_mostly
650 = IS_ENABLED(CONFIG_DEBUG_PAGEALLOC_ENABLE_DEFAULT);
651EXPORT_SYMBOL(_debug_pagealloc_enabled);
652bool _debug_guardpage_enabled __read_mostly;
653
654static int __init early_debug_pagealloc(char *buf)
655{
656 if (!buf)
657 return -EINVAL;
658 return kstrtobool(buf, &_debug_pagealloc_enabled);
659}
660early_param("debug_pagealloc", early_debug_pagealloc);
661
662static bool need_debug_guardpage(void)
663{
664
665 if (!debug_pagealloc_enabled())
666 return false;
667
668 if (!debug_guardpage_minorder())
669 return false;
670
671 return true;
672}
673
674static void init_debug_guardpage(void)
675{
676 if (!debug_pagealloc_enabled())
677 return;
678
679 if (!debug_guardpage_minorder())
680 return;
681
682 _debug_guardpage_enabled = true;
683}
684
685struct page_ext_operations debug_guardpage_ops = {
686 .need = need_debug_guardpage,
687 .init = init_debug_guardpage,
688};
689
690static int __init debug_guardpage_minorder_setup(char *buf)
691{
692 unsigned long res;
693
694 if (kstrtoul(buf, 10, &res) < 0 || res > MAX_ORDER / 2) {
695 pr_err("Bad debug_guardpage_minorder value\n");
696 return 0;
697 }
698 _debug_guardpage_minorder = res;
699 pr_info("Setting debug_guardpage_minorder to %lu\n", res);
700 return 0;
701}
702early_param("debug_guardpage_minorder", debug_guardpage_minorder_setup);
703
704static inline bool set_page_guard(struct zone *zone, struct page *page,
705 unsigned int order, int migratetype)
706{
707 struct page_ext *page_ext;
708
709 if (!debug_guardpage_enabled())
710 return false;
711
712 if (order >= debug_guardpage_minorder())
713 return false;
714
715 page_ext = lookup_page_ext(page);
716 if (unlikely(!page_ext))
717 return false;
718
719 __set_bit(PAGE_EXT_DEBUG_GUARD, &page_ext->flags);
720
721 INIT_LIST_HEAD(&page->lru);
722 set_page_private(page, order);
723
724 __mod_zone_freepage_state(zone, -(1 << order), migratetype);
725
726 return true;
727}
728
729static inline void clear_page_guard(struct zone *zone, struct page *page,
730 unsigned int order, int migratetype)
731{
732 struct page_ext *page_ext;
733
734 if (!debug_guardpage_enabled())
735 return;
736
737 page_ext = lookup_page_ext(page);
738 if (unlikely(!page_ext))
739 return;
740
741 __clear_bit(PAGE_EXT_DEBUG_GUARD, &page_ext->flags);
742
743 set_page_private(page, 0);
744 if (!is_migrate_isolate(migratetype))
745 __mod_zone_freepage_state(zone, (1 << order), migratetype);
746}
747#else
748struct page_ext_operations debug_guardpage_ops;
749static inline bool set_page_guard(struct zone *zone, struct page *page,
750 unsigned int order, int migratetype) { return false; }
751static inline void clear_page_guard(struct zone *zone, struct page *page,
752 unsigned int order, int migratetype) {}
753#endif
754
755static inline void set_page_order(struct page *page, unsigned int order)
756{
757 set_page_private(page, order);
758 __SetPageBuddy(page);
759}
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774static inline int page_is_buddy(struct page *page, struct page *buddy,
775 unsigned int order)
776{
777 if (page_is_guard(buddy) && page_order(buddy) == order) {
778 if (page_zone_id(page) != page_zone_id(buddy))
779 return 0;
780
781 VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy);
782
783 return 1;
784 }
785
786 if (PageBuddy(buddy) && page_order(buddy) == order) {
787
788
789
790
791
792 if (page_zone_id(page) != page_zone_id(buddy))
793 return 0;
794
795 VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy);
796
797 return 1;
798 }
799 return 0;
800}
801
802#ifdef CONFIG_COMPACTION
803static inline struct capture_control *task_capc(struct zone *zone)
804{
805 struct capture_control *capc = current->capture_control;
806
807 return capc &&
808 !(current->flags & PF_KTHREAD) &&
809 !capc->page &&
810 capc->cc->zone == zone &&
811 capc->cc->direct_compaction ? capc : NULL;
812}
813
814static inline bool
815compaction_capture(struct capture_control *capc, struct page *page,
816 int order, int migratetype)
817{
818 if (!capc || order != capc->cc->order)
819 return false;
820
821
822 if (is_migrate_cma(migratetype) ||
823 is_migrate_isolate(migratetype))
824 return false;
825
826
827
828
829
830
831
832 if (order < pageblock_order && migratetype == MIGRATE_MOVABLE)
833 return false;
834
835 capc->page = page;
836 return true;
837}
838
839#else
840static inline struct capture_control *task_capc(struct zone *zone)
841{
842 return NULL;
843}
844
845static inline bool
846compaction_capture(struct capture_control *capc, struct page *page,
847 int order, int migratetype)
848{
849 return false;
850}
851#endif
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877static inline void __free_one_page(struct page *page,
878 unsigned long pfn,
879 struct zone *zone, unsigned int order,
880 int migratetype)
881{
882 unsigned long combined_pfn;
883 unsigned long uninitialized_var(buddy_pfn);
884 struct page *buddy;
885 unsigned int max_order;
886 struct capture_control *capc = task_capc(zone);
887
888 max_order = min_t(unsigned int, MAX_ORDER, pageblock_order + 1);
889
890 VM_BUG_ON(!zone_is_initialized(zone));
891 VM_BUG_ON_PAGE(page->flags & PAGE_FLAGS_CHECK_AT_PREP, page);
892
893 VM_BUG_ON(migratetype == -1);
894 if (likely(!is_migrate_isolate(migratetype)))
895 __mod_zone_freepage_state(zone, 1 << order, migratetype);
896
897 VM_BUG_ON_PAGE(pfn & ((1 << order) - 1), page);
898 VM_BUG_ON_PAGE(bad_range(zone, page), page);
899
900continue_merging:
901 while (order < max_order - 1) {
902 if (compaction_capture(capc, page, order, migratetype)) {
903 __mod_zone_freepage_state(zone, -(1 << order),
904 migratetype);
905 return;
906 }
907 buddy_pfn = __find_buddy_pfn(pfn, order);
908 buddy = page + (buddy_pfn - pfn);
909
910 if (!pfn_valid_within(buddy_pfn))
911 goto done_merging;
912 if (!page_is_buddy(page, buddy, order))
913 goto done_merging;
914
915
916
917
918 if (page_is_guard(buddy))
919 clear_page_guard(zone, buddy, order, migratetype);
920 else
921 del_page_from_free_area(buddy, &zone->free_area[order]);
922 combined_pfn = buddy_pfn & pfn;
923 page = page + (combined_pfn - pfn);
924 pfn = combined_pfn;
925 order++;
926 }
927 if (max_order < MAX_ORDER) {
928
929
930
931
932
933
934
935
936 if (unlikely(has_isolate_pageblock(zone))) {
937 int buddy_mt;
938
939 buddy_pfn = __find_buddy_pfn(pfn, order);
940 buddy = page + (buddy_pfn - pfn);
941 buddy_mt = get_pageblock_migratetype(buddy);
942
943 if (migratetype != buddy_mt
944 && (is_migrate_isolate(migratetype) ||
945 is_migrate_isolate(buddy_mt)))
946 goto done_merging;
947 }
948 max_order++;
949 goto continue_merging;
950 }
951
952done_merging:
953 set_page_order(page, order);
954
955
956
957
958
959
960
961
962
963 if ((order < MAX_ORDER-2) && pfn_valid_within(buddy_pfn)
964 && !is_shuffle_order(order)) {
965 struct page *higher_page, *higher_buddy;
966 combined_pfn = buddy_pfn & pfn;
967 higher_page = page + (combined_pfn - pfn);
968 buddy_pfn = __find_buddy_pfn(combined_pfn, order + 1);
969 higher_buddy = higher_page + (buddy_pfn - combined_pfn);
970 if (pfn_valid_within(buddy_pfn) &&
971 page_is_buddy(higher_page, higher_buddy, order + 1)) {
972 add_to_free_area_tail(page, &zone->free_area[order],
973 migratetype);
974 return;
975 }
976 }
977
978 if (is_shuffle_order(order))
979 add_to_free_area_random(page, &zone->free_area[order],
980 migratetype);
981 else
982 add_to_free_area(page, &zone->free_area[order], migratetype);
983
984}
985
986
987
988
989
990
991static inline bool page_expected_state(struct page *page,
992 unsigned long check_flags)
993{
994 if (unlikely(atomic_read(&page->_mapcount) != -1))
995 return false;
996
997 if (unlikely((unsigned long)page->mapping |
998 page_ref_count(page) |
999#ifdef CONFIG_MEMCG
1000 (unsigned long)page->mem_cgroup |
1001#endif
1002 (page->flags & check_flags)))
1003 return false;
1004
1005 return true;
1006}
1007
1008static void free_pages_check_bad(struct page *page)
1009{
1010 const char *bad_reason;
1011 unsigned long bad_flags;
1012
1013 bad_reason = NULL;
1014 bad_flags = 0;
1015
1016 if (unlikely(atomic_read(&page->_mapcount) != -1))
1017 bad_reason = "nonzero mapcount";
1018 if (unlikely(page->mapping != NULL))
1019 bad_reason = "non-NULL mapping";
1020 if (unlikely(page_ref_count(page) != 0))
1021 bad_reason = "nonzero _refcount";
1022 if (unlikely(page->flags & PAGE_FLAGS_CHECK_AT_FREE)) {
1023 bad_reason = "PAGE_FLAGS_CHECK_AT_FREE flag(s) set";
1024 bad_flags = PAGE_FLAGS_CHECK_AT_FREE;
1025 }
1026#ifdef CONFIG_MEMCG
1027 if (unlikely(page->mem_cgroup))
1028 bad_reason = "page still charged to cgroup";
1029#endif
1030 bad_page(page, bad_reason, bad_flags);
1031}
1032
1033static inline int free_pages_check(struct page *page)
1034{
1035 if (likely(page_expected_state(page, PAGE_FLAGS_CHECK_AT_FREE)))
1036 return 0;
1037
1038
1039 free_pages_check_bad(page);
1040 return 1;
1041}
1042
1043static int free_tail_pages_check(struct page *head_page, struct page *page)
1044{
1045 int ret = 1;
1046
1047
1048
1049
1050
1051 BUILD_BUG_ON((unsigned long)LIST_POISON1 & 1);
1052
1053 if (!IS_ENABLED(CONFIG_DEBUG_VM)) {
1054 ret = 0;
1055 goto out;
1056 }
1057 switch (page - head_page) {
1058 case 1:
1059
1060 if (unlikely(compound_mapcount(page))) {
1061 bad_page(page, "nonzero compound_mapcount", 0);
1062 goto out;
1063 }
1064 break;
1065 case 2:
1066
1067
1068
1069
1070 break;
1071 default:
1072 if (page->mapping != TAIL_MAPPING) {
1073 bad_page(page, "corrupted mapping in tail page", 0);
1074 goto out;
1075 }
1076 break;
1077 }
1078 if (unlikely(!PageTail(page))) {
1079 bad_page(page, "PageTail not set", 0);
1080 goto out;
1081 }
1082 if (unlikely(compound_head(page) != head_page)) {
1083 bad_page(page, "compound_head not consistent", 0);
1084 goto out;
1085 }
1086 ret = 0;
1087out:
1088 page->mapping = NULL;
1089 clear_compound_head(page);
1090 return ret;
1091}
1092
1093static __always_inline bool free_pages_prepare(struct page *page,
1094 unsigned int order, bool check_free)
1095{
1096 int bad = 0;
1097
1098 VM_BUG_ON_PAGE(PageTail(page), page);
1099
1100 trace_mm_page_free(page, order);
1101
1102
1103
1104
1105
1106 if (unlikely(order)) {
1107 bool compound = PageCompound(page);
1108 int i;
1109
1110 VM_BUG_ON_PAGE(compound && compound_order(page) != order, page);
1111
1112 if (compound)
1113 ClearPageDoubleMap(page);
1114 for (i = 1; i < (1 << order); i++) {
1115 if (compound)
1116 bad += free_tail_pages_check(page, page + i);
1117 if (unlikely(free_pages_check(page + i))) {
1118 bad++;
1119 continue;
1120 }
1121 (page + i)->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
1122 }
1123 }
1124 if (PageMappingFlags(page))
1125 page->mapping = NULL;
1126 if (memcg_kmem_enabled() && PageKmemcg(page))
1127 __memcg_kmem_uncharge(page, order);
1128 if (check_free)
1129 bad += free_pages_check(page);
1130 if (bad)
1131 return false;
1132
1133 page_cpupid_reset_last(page);
1134 page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
1135 reset_page_owner(page, order);
1136
1137 if (!PageHighMem(page)) {
1138 debug_check_no_locks_freed(page_address(page),
1139 PAGE_SIZE << order);
1140 debug_check_no_obj_freed(page_address(page),
1141 PAGE_SIZE << order);
1142 }
1143 arch_free_page(page, order);
1144 kernel_poison_pages(page, 1 << order, 0);
1145 if (debug_pagealloc_enabled())
1146 kernel_map_pages(page, 1 << order, 0);
1147
1148 kasan_free_nondeferred_pages(page, order);
1149
1150 return true;
1151}
1152
1153#ifdef CONFIG_DEBUG_VM
1154static inline bool free_pcp_prepare(struct page *page)
1155{
1156 return free_pages_prepare(page, 0, true);
1157}
1158
1159static inline bool bulkfree_pcp_prepare(struct page *page)
1160{
1161 return false;
1162}
1163#else
1164static bool free_pcp_prepare(struct page *page)
1165{
1166 return free_pages_prepare(page, 0, false);
1167}
1168
1169static bool bulkfree_pcp_prepare(struct page *page)
1170{
1171 return free_pages_check(page);
1172}
1173#endif
1174
1175static inline void prefetch_buddy(struct page *page)
1176{
1177 unsigned long pfn = page_to_pfn(page);
1178 unsigned long buddy_pfn = __find_buddy_pfn(pfn, 0);
1179 struct page *buddy = page + (buddy_pfn - pfn);
1180
1181 prefetch(buddy);
1182}
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195static void free_pcppages_bulk(struct zone *zone, int count,
1196 struct per_cpu_pages *pcp)
1197{
1198 int migratetype = 0;
1199 int batch_free = 0;
1200 int prefetch_nr = 0;
1201 bool isolated_pageblocks;
1202 struct page *page, *tmp;
1203 LIST_HEAD(head);
1204
1205 while (count) {
1206 struct list_head *list;
1207
1208
1209
1210
1211
1212
1213
1214
1215 do {
1216 batch_free++;
1217 if (++migratetype == MIGRATE_PCPTYPES)
1218 migratetype = 0;
1219 list = &pcp->lists[migratetype];
1220 } while (list_empty(list));
1221
1222
1223 if (batch_free == MIGRATE_PCPTYPES)
1224 batch_free = count;
1225
1226 do {
1227 page = list_last_entry(list, struct page, lru);
1228
1229 list_del(&page->lru);
1230 pcp->count--;
1231
1232 if (bulkfree_pcp_prepare(page))
1233 continue;
1234
1235 list_add_tail(&page->lru, &head);
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246 if (prefetch_nr++ < pcp->batch)
1247 prefetch_buddy(page);
1248 } while (--count && --batch_free && !list_empty(list));
1249 }
1250
1251 spin_lock(&zone->lock);
1252 isolated_pageblocks = has_isolate_pageblock(zone);
1253
1254
1255
1256
1257
1258 list_for_each_entry_safe(page, tmp, &head, lru) {
1259 int mt = get_pcppage_migratetype(page);
1260
1261 VM_BUG_ON_PAGE(is_migrate_isolate(mt), page);
1262
1263 if (unlikely(isolated_pageblocks))
1264 mt = get_pageblock_migratetype(page);
1265
1266 __free_one_page(page, page_to_pfn(page), zone, 0, mt);
1267 trace_mm_page_pcpu_drain(page, 0, mt);
1268 }
1269 spin_unlock(&zone->lock);
1270}
1271
1272static void free_one_page(struct zone *zone,
1273 struct page *page, unsigned long pfn,
1274 unsigned int order,
1275 int migratetype)
1276{
1277 spin_lock(&zone->lock);
1278 if (unlikely(has_isolate_pageblock(zone) ||
1279 is_migrate_isolate(migratetype))) {
1280 migratetype = get_pfnblock_migratetype(page, pfn);
1281 }
1282 __free_one_page(page, pfn, zone, order, migratetype);
1283 spin_unlock(&zone->lock);
1284}
1285
1286static void __meminit __init_single_page(struct page *page, unsigned long pfn,
1287 unsigned long zone, int nid)
1288{
1289 mm_zero_struct_page(page);
1290 set_page_links(page, zone, nid, pfn);
1291 init_page_count(page);
1292 page_mapcount_reset(page);
1293 page_cpupid_reset_last(page);
1294 page_kasan_tag_reset(page);
1295
1296 INIT_LIST_HEAD(&page->lru);
1297#ifdef WANT_PAGE_VIRTUAL
1298
1299 if (!is_highmem_idx(zone))
1300 set_page_address(page, __va(pfn << PAGE_SHIFT));
1301#endif
1302}
1303
1304#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
1305static void __meminit init_reserved_page(unsigned long pfn)
1306{
1307 pg_data_t *pgdat;
1308 int nid, zid;
1309
1310 if (!early_page_uninitialised(pfn))
1311 return;
1312
1313 nid = early_pfn_to_nid(pfn);
1314 pgdat = NODE_DATA(nid);
1315
1316 for (zid = 0; zid < MAX_NR_ZONES; zid++) {
1317 struct zone *zone = &pgdat->node_zones[zid];
1318
1319 if (pfn >= zone->zone_start_pfn && pfn < zone_end_pfn(zone))
1320 break;
1321 }
1322 __init_single_page(pfn_to_page(pfn), pfn, zid, nid);
1323}
1324#else
1325static inline void init_reserved_page(unsigned long pfn)
1326{
1327}
1328#endif
1329
1330
1331
1332
1333
1334
1335
1336void __meminit reserve_bootmem_region(phys_addr_t start, phys_addr_t end)
1337{
1338 unsigned long start_pfn = PFN_DOWN(start);
1339 unsigned long end_pfn = PFN_UP(end);
1340
1341 for (; start_pfn < end_pfn; start_pfn++) {
1342 if (pfn_valid(start_pfn)) {
1343 struct page *page = pfn_to_page(start_pfn);
1344
1345 init_reserved_page(start_pfn);
1346
1347
1348 INIT_LIST_HEAD(&page->lru);
1349
1350
1351
1352
1353
1354
1355 __SetPageReserved(page);
1356 }
1357 }
1358}
1359
1360static void __free_pages_ok(struct page *page, unsigned int order)
1361{
1362 unsigned long flags;
1363 int migratetype;
1364 unsigned long pfn = page_to_pfn(page);
1365
1366 if (!free_pages_prepare(page, order, true))
1367 return;
1368
1369 migratetype = get_pfnblock_migratetype(page, pfn);
1370 local_irq_save(flags);
1371 __count_vm_events(PGFREE, 1 << order);
1372 free_one_page(page_zone(page), page, pfn, order, migratetype);
1373 local_irq_restore(flags);
1374}
1375
1376void __free_pages_core(struct page *page, unsigned int order)
1377{
1378 unsigned int nr_pages = 1 << order;
1379 struct page *p = page;
1380 unsigned int loop;
1381
1382 prefetchw(p);
1383 for (loop = 0; loop < (nr_pages - 1); loop++, p++) {
1384 prefetchw(p + 1);
1385 __ClearPageReserved(p);
1386 set_page_count(p, 0);
1387 }
1388 __ClearPageReserved(p);
1389 set_page_count(p, 0);
1390
1391 atomic_long_add(nr_pages, &page_zone(page)->managed_pages);
1392 set_page_refcounted(page);
1393 __free_pages(page, order);
1394}
1395
1396#if defined(CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID) || \
1397 defined(CONFIG_HAVE_MEMBLOCK_NODE_MAP)
1398
1399static struct mminit_pfnnid_cache early_pfnnid_cache __meminitdata;
1400
1401int __meminit early_pfn_to_nid(unsigned long pfn)
1402{
1403 static DEFINE_SPINLOCK(early_pfn_lock);
1404 int nid;
1405
1406 spin_lock(&early_pfn_lock);
1407 nid = __early_pfn_to_nid(pfn, &early_pfnnid_cache);
1408 if (nid < 0)
1409 nid = first_online_node;
1410 spin_unlock(&early_pfn_lock);
1411
1412 return nid;
1413}
1414#endif
1415
1416#ifdef CONFIG_NODES_SPAN_OTHER_NODES
1417
1418static inline bool __meminit early_pfn_in_nid(unsigned long pfn, int node)
1419{
1420 int nid;
1421
1422 nid = __early_pfn_to_nid(pfn, &early_pfnnid_cache);
1423 if (nid >= 0 && nid != node)
1424 return false;
1425 return true;
1426}
1427
1428#else
1429static inline bool __meminit early_pfn_in_nid(unsigned long pfn, int node)
1430{
1431 return true;
1432}
1433#endif
1434
1435
1436void __init memblock_free_pages(struct page *page, unsigned long pfn,
1437 unsigned int order)
1438{
1439 if (early_page_uninitialised(pfn))
1440 return;
1441 __free_pages_core(page, order);
1442}
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461struct page *__pageblock_pfn_to_page(unsigned long start_pfn,
1462 unsigned long end_pfn, struct zone *zone)
1463{
1464 struct page *start_page;
1465 struct page *end_page;
1466
1467
1468 end_pfn--;
1469
1470 if (!pfn_valid(start_pfn) || !pfn_valid(end_pfn))
1471 return NULL;
1472
1473 start_page = pfn_to_online_page(start_pfn);
1474 if (!start_page)
1475 return NULL;
1476
1477 if (page_zone(start_page) != zone)
1478 return NULL;
1479
1480 end_page = pfn_to_page(end_pfn);
1481
1482
1483 if (page_zone_id(start_page) != page_zone_id(end_page))
1484 return NULL;
1485
1486 return start_page;
1487}
1488
1489void set_zone_contiguous(struct zone *zone)
1490{
1491 unsigned long block_start_pfn = zone->zone_start_pfn;
1492 unsigned long block_end_pfn;
1493
1494 block_end_pfn = ALIGN(block_start_pfn + 1, pageblock_nr_pages);
1495 for (; block_start_pfn < zone_end_pfn(zone);
1496 block_start_pfn = block_end_pfn,
1497 block_end_pfn += pageblock_nr_pages) {
1498
1499 block_end_pfn = min(block_end_pfn, zone_end_pfn(zone));
1500
1501 if (!__pageblock_pfn_to_page(block_start_pfn,
1502 block_end_pfn, zone))
1503 return;
1504 }
1505
1506
1507 zone->contiguous = true;
1508}
1509
1510void clear_zone_contiguous(struct zone *zone)
1511{
1512 zone->contiguous = false;
1513}
1514
1515#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
1516static void __init deferred_free_range(unsigned long pfn,
1517 unsigned long nr_pages)
1518{
1519 struct page *page;
1520 unsigned long i;
1521
1522 if (!nr_pages)
1523 return;
1524
1525 page = pfn_to_page(pfn);
1526
1527
1528 if (nr_pages == pageblock_nr_pages &&
1529 (pfn & (pageblock_nr_pages - 1)) == 0) {
1530 set_pageblock_migratetype(page, MIGRATE_MOVABLE);
1531 __free_pages_core(page, pageblock_order);
1532 return;
1533 }
1534
1535 for (i = 0; i < nr_pages; i++, page++, pfn++) {
1536 if ((pfn & (pageblock_nr_pages - 1)) == 0)
1537 set_pageblock_migratetype(page, MIGRATE_MOVABLE);
1538 __free_pages_core(page, 0);
1539 }
1540}
1541
1542
1543static atomic_t pgdat_init_n_undone __initdata;
1544static __initdata DECLARE_COMPLETION(pgdat_init_all_done_comp);
1545
1546static inline void __init pgdat_init_report_one_done(void)
1547{
1548 if (atomic_dec_and_test(&pgdat_init_n_undone))
1549 complete(&pgdat_init_all_done_comp);
1550}
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562static inline bool __init deferred_pfn_valid(unsigned long pfn)
1563{
1564 if (!pfn_valid_within(pfn))
1565 return false;
1566 if (!(pfn & (pageblock_nr_pages - 1)) && !pfn_valid(pfn))
1567 return false;
1568 return true;
1569}
1570
1571
1572
1573
1574
1575static void __init deferred_free_pages(unsigned long pfn,
1576 unsigned long end_pfn)
1577{
1578 unsigned long nr_pgmask = pageblock_nr_pages - 1;
1579 unsigned long nr_free = 0;
1580
1581 for (; pfn < end_pfn; pfn++) {
1582 if (!deferred_pfn_valid(pfn)) {
1583 deferred_free_range(pfn - nr_free, nr_free);
1584 nr_free = 0;
1585 } else if (!(pfn & nr_pgmask)) {
1586 deferred_free_range(pfn - nr_free, nr_free);
1587 nr_free = 1;
1588 touch_nmi_watchdog();
1589 } else {
1590 nr_free++;
1591 }
1592 }
1593
1594 deferred_free_range(pfn - nr_free, nr_free);
1595}
1596
1597
1598
1599
1600
1601
1602static unsigned long __init deferred_init_pages(struct zone *zone,
1603 unsigned long pfn,
1604 unsigned long end_pfn)
1605{
1606 unsigned long nr_pgmask = pageblock_nr_pages - 1;
1607 int nid = zone_to_nid(zone);
1608 unsigned long nr_pages = 0;
1609 int zid = zone_idx(zone);
1610 struct page *page = NULL;
1611
1612 for (; pfn < end_pfn; pfn++) {
1613 if (!deferred_pfn_valid(pfn)) {
1614 page = NULL;
1615 continue;
1616 } else if (!page || !(pfn & nr_pgmask)) {
1617 page = pfn_to_page(pfn);
1618 touch_nmi_watchdog();
1619 } else {
1620 page++;
1621 }
1622 __init_single_page(page, pfn, zid, nid);
1623 nr_pages++;
1624 }
1625 return (nr_pages);
1626}
1627
1628
1629
1630
1631
1632
1633
1634static bool __init
1635deferred_init_mem_pfn_range_in_zone(u64 *i, struct zone *zone,
1636 unsigned long *spfn, unsigned long *epfn,
1637 unsigned long first_init_pfn)
1638{
1639 u64 j;
1640
1641
1642
1643
1644
1645
1646 for_each_free_mem_pfn_range_in_zone(j, zone, spfn, epfn) {
1647 if (*epfn <= first_init_pfn)
1648 continue;
1649 if (*spfn < first_init_pfn)
1650 *spfn = first_init_pfn;
1651 *i = j;
1652 return true;
1653 }
1654
1655 return false;
1656}
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668static unsigned long __init
1669deferred_init_maxorder(u64 *i, struct zone *zone, unsigned long *start_pfn,
1670 unsigned long *end_pfn)
1671{
1672 unsigned long mo_pfn = ALIGN(*start_pfn + 1, MAX_ORDER_NR_PAGES);
1673 unsigned long spfn = *start_pfn, epfn = *end_pfn;
1674 unsigned long nr_pages = 0;
1675 u64 j = *i;
1676
1677
1678 for_each_free_mem_pfn_range_in_zone_from(j, zone, start_pfn, end_pfn) {
1679 unsigned long t;
1680
1681 if (mo_pfn <= *start_pfn)
1682 break;
1683
1684 t = min(mo_pfn, *end_pfn);
1685 nr_pages += deferred_init_pages(zone, *start_pfn, t);
1686
1687 if (mo_pfn < *end_pfn) {
1688 *start_pfn = mo_pfn;
1689 break;
1690 }
1691 }
1692
1693
1694 swap(j, *i);
1695
1696 for_each_free_mem_pfn_range_in_zone_from(j, zone, &spfn, &epfn) {
1697 unsigned long t;
1698
1699 if (mo_pfn <= spfn)
1700 break;
1701
1702 t = min(mo_pfn, epfn);
1703 deferred_free_pages(spfn, t);
1704
1705 if (mo_pfn <= epfn)
1706 break;
1707 }
1708
1709 return nr_pages;
1710}
1711
1712
1713static int __init deferred_init_memmap(void *data)
1714{
1715 pg_data_t *pgdat = data;
1716 const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id);
1717 unsigned long spfn = 0, epfn = 0, nr_pages = 0;
1718 unsigned long first_init_pfn, flags;
1719 unsigned long start = jiffies;
1720 struct zone *zone;
1721 int zid;
1722 u64 i;
1723
1724
1725 if (!cpumask_empty(cpumask))
1726 set_cpus_allowed_ptr(current, cpumask);
1727
1728 pgdat_resize_lock(pgdat, &flags);
1729 first_init_pfn = pgdat->first_deferred_pfn;
1730 if (first_init_pfn == ULONG_MAX) {
1731 pgdat_resize_unlock(pgdat, &flags);
1732 pgdat_init_report_one_done();
1733 return 0;
1734 }
1735
1736
1737 BUG_ON(pgdat->first_deferred_pfn < pgdat->node_start_pfn);
1738 BUG_ON(pgdat->first_deferred_pfn > pgdat_end_pfn(pgdat));
1739 pgdat->first_deferred_pfn = ULONG_MAX;
1740
1741
1742 for (zid = 0; zid < MAX_NR_ZONES; zid++) {
1743 zone = pgdat->node_zones + zid;
1744 if (first_init_pfn < zone_end_pfn(zone))
1745 break;
1746 }
1747
1748
1749 if (!deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn,
1750 first_init_pfn))
1751 goto zone_empty;
1752
1753
1754
1755
1756
1757
1758 while (spfn < epfn)
1759 nr_pages += deferred_init_maxorder(&i, zone, &spfn, &epfn);
1760zone_empty:
1761 pgdat_resize_unlock(pgdat, &flags);
1762
1763
1764 WARN_ON(++zid < MAX_NR_ZONES && populated_zone(++zone));
1765
1766 pr_info("node %d initialised, %lu pages in %ums\n",
1767 pgdat->node_id, nr_pages, jiffies_to_msecs(jiffies - start));
1768
1769 pgdat_init_report_one_done();
1770 return 0;
1771}
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788static noinline bool __init
1789deferred_grow_zone(struct zone *zone, unsigned int order)
1790{
1791 unsigned long nr_pages_needed = ALIGN(1 << order, PAGES_PER_SECTION);
1792 pg_data_t *pgdat = zone->zone_pgdat;
1793 unsigned long first_deferred_pfn = pgdat->first_deferred_pfn;
1794 unsigned long spfn, epfn, flags;
1795 unsigned long nr_pages = 0;
1796 u64 i;
1797
1798
1799 if (zone_end_pfn(zone) != pgdat_end_pfn(pgdat))
1800 return false;
1801
1802 pgdat_resize_lock(pgdat, &flags);
1803
1804
1805
1806
1807
1808
1809
1810 if (!static_branch_unlikely(&deferred_pages)) {
1811 pgdat_resize_unlock(pgdat, &flags);
1812 return true;
1813 }
1814
1815
1816
1817
1818
1819 if (first_deferred_pfn != pgdat->first_deferred_pfn) {
1820 pgdat_resize_unlock(pgdat, &flags);
1821 return true;
1822 }
1823
1824
1825 if (!deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn,
1826 first_deferred_pfn)) {
1827 pgdat->first_deferred_pfn = ULONG_MAX;
1828 pgdat_resize_unlock(pgdat, &flags);
1829
1830 return first_deferred_pfn != ULONG_MAX;
1831 }
1832
1833
1834
1835
1836
1837
1838 while (spfn < epfn) {
1839
1840 first_deferred_pfn = spfn;
1841
1842 nr_pages += deferred_init_maxorder(&i, zone, &spfn, &epfn);
1843
1844
1845 if ((first_deferred_pfn ^ spfn) < PAGES_PER_SECTION)
1846 continue;
1847
1848
1849 if (nr_pages >= nr_pages_needed)
1850 break;
1851 }
1852
1853 pgdat->first_deferred_pfn = spfn;
1854 pgdat_resize_unlock(pgdat, &flags);
1855
1856 return nr_pages > 0;
1857}
1858
1859
1860
1861
1862
1863
1864
1865static bool __ref
1866_deferred_grow_zone(struct zone *zone, unsigned int order)
1867{
1868 return deferred_grow_zone(zone, order);
1869}
1870
1871#endif
1872
1873void __init page_alloc_init_late(void)
1874{
1875 struct zone *zone;
1876 int nid;
1877
1878#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
1879
1880
1881 atomic_set(&pgdat_init_n_undone, num_node_state(N_MEMORY));
1882 for_each_node_state(nid, N_MEMORY) {
1883 kthread_run(deferred_init_memmap, NODE_DATA(nid), "pgdatinit%d", nid);
1884 }
1885
1886
1887 wait_for_completion(&pgdat_init_all_done_comp);
1888
1889
1890
1891
1892
1893 static_branch_disable(&deferred_pages);
1894
1895
1896 files_maxfiles_init();
1897#endif
1898
1899
1900 memblock_discard();
1901
1902 for_each_node_state(nid, N_MEMORY)
1903 shuffle_free_memory(NODE_DATA(nid));
1904
1905 for_each_populated_zone(zone)
1906 set_zone_contiguous(zone);
1907}
1908
1909#ifdef CONFIG_CMA
1910
1911void __init init_cma_reserved_pageblock(struct page *page)
1912{
1913 unsigned i = pageblock_nr_pages;
1914 struct page *p = page;
1915
1916 do {
1917 __ClearPageReserved(p);
1918 set_page_count(p, 0);
1919 } while (++p, --i);
1920
1921 set_pageblock_migratetype(page, MIGRATE_CMA);
1922
1923 if (pageblock_order >= MAX_ORDER) {
1924 i = pageblock_nr_pages;
1925 p = page;
1926 do {
1927 set_page_refcounted(p);
1928 __free_pages(p, MAX_ORDER - 1);
1929 p += MAX_ORDER_NR_PAGES;
1930 } while (i -= MAX_ORDER_NR_PAGES);
1931 } else {
1932 set_page_refcounted(page);
1933 __free_pages(page, pageblock_order);
1934 }
1935
1936 adjust_managed_page_count(page, pageblock_nr_pages);
1937}
1938#endif
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954static inline void expand(struct zone *zone, struct page *page,
1955 int low, int high, struct free_area *area,
1956 int migratetype)
1957{
1958 unsigned long size = 1 << high;
1959
1960 while (high > low) {
1961 area--;
1962 high--;
1963 size >>= 1;
1964 VM_BUG_ON_PAGE(bad_range(zone, &page[size]), &page[size]);
1965
1966
1967
1968
1969
1970
1971
1972 if (set_page_guard(zone, &page[size], high, migratetype))
1973 continue;
1974
1975 add_to_free_area(&page[size], area, migratetype);
1976 set_page_order(&page[size], high);
1977 }
1978}
1979
1980static void check_new_page_bad(struct page *page)
1981{
1982 const char *bad_reason = NULL;
1983 unsigned long bad_flags = 0;
1984
1985 if (unlikely(atomic_read(&page->_mapcount) != -1))
1986 bad_reason = "nonzero mapcount";
1987 if (unlikely(page->mapping != NULL))
1988 bad_reason = "non-NULL mapping";
1989 if (unlikely(page_ref_count(page) != 0))
1990 bad_reason = "nonzero _refcount";
1991 if (unlikely(page->flags & __PG_HWPOISON)) {
1992 bad_reason = "HWPoisoned (hardware-corrupted)";
1993 bad_flags = __PG_HWPOISON;
1994
1995 page_mapcount_reset(page);
1996 return;
1997 }
1998 if (unlikely(page->flags & PAGE_FLAGS_CHECK_AT_PREP)) {
1999 bad_reason = "PAGE_FLAGS_CHECK_AT_PREP flag set";
2000 bad_flags = PAGE_FLAGS_CHECK_AT_PREP;
2001 }
2002#ifdef CONFIG_MEMCG
2003 if (unlikely(page->mem_cgroup))
2004 bad_reason = "page still charged to cgroup";
2005#endif
2006 bad_page(page, bad_reason, bad_flags);
2007}
2008
2009
2010
2011
2012static inline int check_new_page(struct page *page)
2013{
2014 if (likely(page_expected_state(page,
2015 PAGE_FLAGS_CHECK_AT_PREP|__PG_HWPOISON)))
2016 return 0;
2017
2018 check_new_page_bad(page);
2019 return 1;
2020}
2021
2022static inline bool free_pages_prezeroed(void)
2023{
2024 return IS_ENABLED(CONFIG_PAGE_POISONING_ZERO) &&
2025 page_poisoning_enabled();
2026}
2027
2028#ifdef CONFIG_DEBUG_VM
2029static bool check_pcp_refill(struct page *page)
2030{
2031 return false;
2032}
2033
2034static bool check_new_pcp(struct page *page)
2035{
2036 return check_new_page(page);
2037}
2038#else
2039static bool check_pcp_refill(struct page *page)
2040{
2041 return check_new_page(page);
2042}
2043static bool check_new_pcp(struct page *page)
2044{
2045 return false;
2046}
2047#endif
2048
2049static bool check_new_pages(struct page *page, unsigned int order)
2050{
2051 int i;
2052 for (i = 0; i < (1 << order); i++) {
2053 struct page *p = page + i;
2054
2055 if (unlikely(check_new_page(p)))
2056 return true;
2057 }
2058
2059 return false;
2060}
2061
2062inline void post_alloc_hook(struct page *page, unsigned int order,
2063 gfp_t gfp_flags)
2064{
2065 set_page_private(page, 0);
2066 set_page_refcounted(page);
2067
2068 arch_alloc_page(page, order);
2069 if (debug_pagealloc_enabled())
2070 kernel_map_pages(page, 1 << order, 1);
2071 kasan_alloc_pages(page, order);
2072 kernel_poison_pages(page, 1 << order, 1);
2073 set_page_owner(page, order, gfp_flags);
2074}
2075
2076static void prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags,
2077 unsigned int alloc_flags)
2078{
2079 int i;
2080
2081 post_alloc_hook(page, order, gfp_flags);
2082
2083 if (!free_pages_prezeroed() && (gfp_flags & __GFP_ZERO))
2084 for (i = 0; i < (1 << order); i++)
2085 clear_highpage(page + i);
2086
2087 if (order && (gfp_flags & __GFP_COMP))
2088 prep_compound_page(page, order);
2089
2090
2091
2092
2093
2094
2095
2096 if (alloc_flags & ALLOC_NO_WATERMARKS)
2097 set_page_pfmemalloc(page);
2098 else
2099 clear_page_pfmemalloc(page);
2100}
2101
2102
2103
2104
2105
2106static __always_inline
2107struct page *__rmqueue_smallest(struct zone *zone, unsigned int order,
2108 int migratetype)
2109{
2110 unsigned int current_order;
2111 struct free_area *area;
2112 struct page *page;
2113
2114
2115 for (current_order = order; current_order < MAX_ORDER; ++current_order) {
2116 area = &(zone->free_area[current_order]);
2117 page = get_page_from_free_area(area, migratetype);
2118 if (!page)
2119 continue;
2120 del_page_from_free_area(page, area);
2121 expand(zone, page, order, current_order, area, migratetype);
2122 set_pcppage_migratetype(page, migratetype);
2123 return page;
2124 }
2125
2126 return NULL;
2127}
2128
2129
2130
2131
2132
2133
2134static int fallbacks[MIGRATE_TYPES][4] = {
2135 [MIGRATE_UNMOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_MOVABLE, MIGRATE_TYPES },
2136 [MIGRATE_MOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE, MIGRATE_TYPES },
2137 [MIGRATE_RECLAIMABLE] = { MIGRATE_UNMOVABLE, MIGRATE_MOVABLE, MIGRATE_TYPES },
2138#ifdef CONFIG_CMA
2139 [MIGRATE_CMA] = { MIGRATE_TYPES },
2140#endif
2141#ifdef CONFIG_MEMORY_ISOLATION
2142 [MIGRATE_ISOLATE] = { MIGRATE_TYPES },
2143#endif
2144};
2145
2146#ifdef CONFIG_CMA
2147static __always_inline struct page *__rmqueue_cma_fallback(struct zone *zone,
2148 unsigned int order)
2149{
2150 return __rmqueue_smallest(zone, order, MIGRATE_CMA);
2151}
2152#else
2153static inline struct page *__rmqueue_cma_fallback(struct zone *zone,
2154 unsigned int order) { return NULL; }
2155#endif
2156
2157
2158
2159
2160
2161
2162static int move_freepages(struct zone *zone,
2163 struct page *start_page, struct page *end_page,
2164 int migratetype, int *num_movable)
2165{
2166 struct page *page;
2167 unsigned int order;
2168 int pages_moved = 0;
2169
2170#ifndef CONFIG_HOLES_IN_ZONE
2171
2172
2173
2174
2175
2176
2177
2178 VM_BUG_ON(pfn_valid(page_to_pfn(start_page)) &&
2179 pfn_valid(page_to_pfn(end_page)) &&
2180 page_zone(start_page) != page_zone(end_page));
2181#endif
2182 for (page = start_page; page <= end_page;) {
2183 if (!pfn_valid_within(page_to_pfn(page))) {
2184 page++;
2185 continue;
2186 }
2187
2188
2189 VM_BUG_ON_PAGE(page_to_nid(page) != zone_to_nid(zone), page);
2190
2191 if (!PageBuddy(page)) {
2192
2193
2194
2195
2196
2197 if (num_movable &&
2198 (PageLRU(page) || __PageMovable(page)))
2199 (*num_movable)++;
2200
2201 page++;
2202 continue;
2203 }
2204
2205 order = page_order(page);
2206 move_to_free_area(page, &zone->free_area[order], migratetype);
2207 page += 1 << order;
2208 pages_moved += 1 << order;
2209 }
2210
2211 return pages_moved;
2212}
2213
2214int move_freepages_block(struct zone *zone, struct page *page,
2215 int migratetype, int *num_movable)
2216{
2217 unsigned long start_pfn, end_pfn;
2218 struct page *start_page, *end_page;
2219
2220 if (num_movable)
2221 *num_movable = 0;
2222
2223 start_pfn = page_to_pfn(page);
2224 start_pfn = start_pfn & ~(pageblock_nr_pages-1);
2225 start_page = pfn_to_page(start_pfn);
2226 end_page = start_page + pageblock_nr_pages - 1;
2227 end_pfn = start_pfn + pageblock_nr_pages - 1;
2228
2229
2230 if (!zone_spans_pfn(zone, start_pfn))
2231 start_page = page;
2232 if (!zone_spans_pfn(zone, end_pfn))
2233 return 0;
2234
2235 return move_freepages(zone, start_page, end_page, migratetype,
2236 num_movable);
2237}
2238
2239static void change_pageblock_range(struct page *pageblock_page,
2240 int start_order, int migratetype)
2241{
2242 int nr_pageblocks = 1 << (start_order - pageblock_order);
2243
2244 while (nr_pageblocks--) {
2245 set_pageblock_migratetype(pageblock_page, migratetype);
2246 pageblock_page += pageblock_nr_pages;
2247 }
2248}
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262static bool can_steal_fallback(unsigned int order, int start_mt)
2263{
2264
2265
2266
2267
2268
2269
2270
2271 if (order >= pageblock_order)
2272 return true;
2273
2274 if (order >= pageblock_order / 2 ||
2275 start_mt == MIGRATE_RECLAIMABLE ||
2276 start_mt == MIGRATE_UNMOVABLE ||
2277 page_group_by_mobility_disabled)
2278 return true;
2279
2280 return false;
2281}
2282
2283static inline void boost_watermark(struct zone *zone)
2284{
2285 unsigned long max_boost;
2286
2287 if (!watermark_boost_factor)
2288 return;
2289
2290 max_boost = mult_frac(zone->_watermark[WMARK_HIGH],
2291 watermark_boost_factor, 10000);
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301 if (!max_boost)
2302 return;
2303
2304 max_boost = max(pageblock_nr_pages, max_boost);
2305
2306 zone->watermark_boost = min(zone->watermark_boost + pageblock_nr_pages,
2307 max_boost);
2308}
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318static void steal_suitable_fallback(struct zone *zone, struct page *page,
2319 unsigned int alloc_flags, int start_type, bool whole_block)
2320{
2321 unsigned int current_order = page_order(page);
2322 struct free_area *area;
2323 int free_pages, movable_pages, alike_pages;
2324 int old_block_type;
2325
2326 old_block_type = get_pageblock_migratetype(page);
2327
2328
2329
2330
2331
2332 if (is_migrate_highatomic(old_block_type))
2333 goto single_page;
2334
2335
2336 if (current_order >= pageblock_order) {
2337 change_pageblock_range(page, current_order, start_type);
2338 goto single_page;
2339 }
2340
2341
2342
2343
2344
2345
2346 boost_watermark(zone);
2347 if (alloc_flags & ALLOC_KSWAPD)
2348 set_bit(ZONE_BOOSTED_WATERMARK, &zone->flags);
2349
2350
2351 if (!whole_block)
2352 goto single_page;
2353
2354 free_pages = move_freepages_block(zone, page, start_type,
2355 &movable_pages);
2356
2357
2358
2359
2360
2361 if (start_type == MIGRATE_MOVABLE) {
2362 alike_pages = movable_pages;
2363 } else {
2364
2365
2366
2367
2368
2369
2370
2371 if (old_block_type == MIGRATE_MOVABLE)
2372 alike_pages = pageblock_nr_pages
2373 - (free_pages + movable_pages);
2374 else
2375 alike_pages = 0;
2376 }
2377
2378
2379 if (!free_pages)
2380 goto single_page;
2381
2382
2383
2384
2385
2386 if (free_pages + alike_pages >= (1 << (pageblock_order-1)) ||
2387 page_group_by_mobility_disabled)
2388 set_pageblock_migratetype(page, start_type);
2389
2390 return;
2391
2392single_page:
2393 area = &zone->free_area[current_order];
2394 move_to_free_area(page, area, start_type);
2395}
2396
2397
2398
2399
2400
2401
2402
2403int find_suitable_fallback(struct free_area *area, unsigned int order,
2404 int migratetype, bool only_stealable, bool *can_steal)
2405{
2406 int i;
2407 int fallback_mt;
2408
2409 if (area->nr_free == 0)
2410 return -1;
2411
2412 *can_steal = false;
2413 for (i = 0;; i++) {
2414 fallback_mt = fallbacks[migratetype][i];
2415 if (fallback_mt == MIGRATE_TYPES)
2416 break;
2417
2418 if (free_area_empty(area, fallback_mt))
2419 continue;
2420
2421 if (can_steal_fallback(order, migratetype))
2422 *can_steal = true;
2423
2424 if (!only_stealable)
2425 return fallback_mt;
2426
2427 if (*can_steal)
2428 return fallback_mt;
2429 }
2430
2431 return -1;
2432}
2433
2434
2435
2436
2437
2438static void reserve_highatomic_pageblock(struct page *page, struct zone *zone,
2439 unsigned int alloc_order)
2440{
2441 int mt;
2442 unsigned long max_managed, flags;
2443
2444
2445
2446
2447
2448 max_managed = (zone_managed_pages(zone) / 100) + pageblock_nr_pages;
2449 if (zone->nr_reserved_highatomic >= max_managed)
2450 return;
2451
2452 spin_lock_irqsave(&zone->lock, flags);
2453
2454
2455 if (zone->nr_reserved_highatomic >= max_managed)
2456 goto out_unlock;
2457
2458
2459 mt = get_pageblock_migratetype(page);
2460 if (!is_migrate_highatomic(mt) && !is_migrate_isolate(mt)
2461 && !is_migrate_cma(mt)) {
2462 zone->nr_reserved_highatomic += pageblock_nr_pages;
2463 set_pageblock_migratetype(page, MIGRATE_HIGHATOMIC);
2464 move_freepages_block(zone, page, MIGRATE_HIGHATOMIC, NULL);
2465 }
2466
2467out_unlock:
2468 spin_unlock_irqrestore(&zone->lock, flags);
2469}
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480static bool unreserve_highatomic_pageblock(const struct alloc_context *ac,
2481 bool force)
2482{
2483 struct zonelist *zonelist = ac->zonelist;
2484 unsigned long flags;
2485 struct zoneref *z;
2486 struct zone *zone;
2487 struct page *page;
2488 int order;
2489 bool ret;
2490
2491 for_each_zone_zonelist_nodemask(zone, z, zonelist, ac->high_zoneidx,
2492 ac->nodemask) {
2493
2494
2495
2496
2497 if (!force && zone->nr_reserved_highatomic <=
2498 pageblock_nr_pages)
2499 continue;
2500
2501 spin_lock_irqsave(&zone->lock, flags);
2502 for (order = 0; order < MAX_ORDER; order++) {
2503 struct free_area *area = &(zone->free_area[order]);
2504
2505 page = get_page_from_free_area(area, MIGRATE_HIGHATOMIC);
2506 if (!page)
2507 continue;
2508
2509
2510
2511
2512
2513
2514
2515
2516 if (is_migrate_highatomic_page(page)) {
2517
2518
2519
2520
2521
2522
2523
2524 zone->nr_reserved_highatomic -= min(
2525 pageblock_nr_pages,
2526 zone->nr_reserved_highatomic);
2527 }
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538 set_pageblock_migratetype(page, ac->migratetype);
2539 ret = move_freepages_block(zone, page, ac->migratetype,
2540 NULL);
2541 if (ret) {
2542 spin_unlock_irqrestore(&zone->lock, flags);
2543 return ret;
2544 }
2545 }
2546 spin_unlock_irqrestore(&zone->lock, flags);
2547 }
2548
2549 return false;
2550}
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562static __always_inline bool
2563__rmqueue_fallback(struct zone *zone, int order, int start_migratetype,
2564 unsigned int alloc_flags)
2565{
2566 struct free_area *area;
2567 int current_order;
2568 int min_order = order;
2569 struct page *page;
2570 int fallback_mt;
2571 bool can_steal;
2572
2573
2574
2575
2576
2577
2578 if (alloc_flags & ALLOC_NOFRAGMENT)
2579 min_order = pageblock_order;
2580
2581
2582
2583
2584
2585
2586 for (current_order = MAX_ORDER - 1; current_order >= min_order;
2587 --current_order) {
2588 area = &(zone->free_area[current_order]);
2589 fallback_mt = find_suitable_fallback(area, current_order,
2590 start_migratetype, false, &can_steal);
2591 if (fallback_mt == -1)
2592 continue;
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602 if (!can_steal && start_migratetype == MIGRATE_MOVABLE
2603 && current_order > order)
2604 goto find_smallest;
2605
2606 goto do_steal;
2607 }
2608
2609 return false;
2610
2611find_smallest:
2612 for (current_order = order; current_order < MAX_ORDER;
2613 current_order++) {
2614 area = &(zone->free_area[current_order]);
2615 fallback_mt = find_suitable_fallback(area, current_order,
2616 start_migratetype, false, &can_steal);
2617 if (fallback_mt != -1)
2618 break;
2619 }
2620
2621
2622
2623
2624
2625 VM_BUG_ON(current_order == MAX_ORDER);
2626
2627do_steal:
2628 page = get_page_from_free_area(area, fallback_mt);
2629
2630 steal_suitable_fallback(zone, page, alloc_flags, start_migratetype,
2631 can_steal);
2632
2633 trace_mm_page_alloc_extfrag(page, order, current_order,
2634 start_migratetype, fallback_mt);
2635
2636 return true;
2637
2638}
2639
2640
2641
2642
2643
2644static __always_inline struct page *
2645__rmqueue(struct zone *zone, unsigned int order, int migratetype,
2646 unsigned int alloc_flags)
2647{
2648 struct page *page;
2649
2650retry:
2651 page = __rmqueue_smallest(zone, order, migratetype);
2652 if (unlikely(!page)) {
2653 if (migratetype == MIGRATE_MOVABLE)
2654 page = __rmqueue_cma_fallback(zone, order);
2655
2656 if (!page && __rmqueue_fallback(zone, order, migratetype,
2657 alloc_flags))
2658 goto retry;
2659 }
2660
2661 trace_mm_page_alloc_zone_locked(page, order, migratetype);
2662 return page;
2663}
2664
2665
2666
2667
2668
2669
2670static int rmqueue_bulk(struct zone *zone, unsigned int order,
2671 unsigned long count, struct list_head *list,
2672 int migratetype, unsigned int alloc_flags)
2673{
2674 int i, alloced = 0;
2675
2676 spin_lock(&zone->lock);
2677 for (i = 0; i < count; ++i) {
2678 struct page *page = __rmqueue(zone, order, migratetype,
2679 alloc_flags);
2680 if (unlikely(page == NULL))
2681 break;
2682
2683 if (unlikely(check_pcp_refill(page)))
2684 continue;
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696 list_add_tail(&page->lru, list);
2697 alloced++;
2698 if (is_migrate_cma(get_pcppage_migratetype(page)))
2699 __mod_zone_page_state(zone, NR_FREE_CMA_PAGES,
2700 -(1 << order));
2701 }
2702
2703
2704
2705
2706
2707
2708
2709 __mod_zone_page_state(zone, NR_FREE_PAGES, -(i << order));
2710 spin_unlock(&zone->lock);
2711 return alloced;
2712}
2713
2714#ifdef CONFIG_NUMA
2715
2716
2717
2718
2719
2720
2721
2722
2723void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp)
2724{
2725 unsigned long flags;
2726 int to_drain, batch;
2727
2728 local_irq_save(flags);
2729 batch = READ_ONCE(pcp->batch);
2730 to_drain = min(pcp->count, batch);
2731 if (to_drain > 0)
2732 free_pcppages_bulk(zone, to_drain, pcp);
2733 local_irq_restore(flags);
2734}
2735#endif
2736
2737
2738
2739
2740
2741
2742
2743
2744static void drain_pages_zone(unsigned int cpu, struct zone *zone)
2745{
2746 unsigned long flags;
2747 struct per_cpu_pageset *pset;
2748 struct per_cpu_pages *pcp;
2749
2750 local_irq_save(flags);
2751 pset = per_cpu_ptr(zone->pageset, cpu);
2752
2753 pcp = &pset->pcp;
2754 if (pcp->count)
2755 free_pcppages_bulk(zone, pcp->count, pcp);
2756 local_irq_restore(flags);
2757}
2758
2759
2760
2761
2762
2763
2764
2765
2766static void drain_pages(unsigned int cpu)
2767{
2768 struct zone *zone;
2769
2770 for_each_populated_zone(zone) {
2771 drain_pages_zone(cpu, zone);
2772 }
2773}
2774
2775
2776
2777
2778
2779
2780
2781void drain_local_pages(struct zone *zone)
2782{
2783 int cpu = smp_processor_id();
2784
2785 if (zone)
2786 drain_pages_zone(cpu, zone);
2787 else
2788 drain_pages(cpu);
2789}
2790
2791static void drain_local_pages_wq(struct work_struct *work)
2792{
2793 struct pcpu_drain *drain;
2794
2795 drain = container_of(work, struct pcpu_drain, work);
2796
2797
2798
2799
2800
2801
2802
2803
2804 preempt_disable();
2805 drain_local_pages(drain->zone);
2806 preempt_enable();
2807}
2808
2809
2810
2811
2812
2813
2814
2815
2816void drain_all_pages(struct zone *zone)
2817{
2818 int cpu;
2819
2820
2821
2822
2823
2824 static cpumask_t cpus_with_pcps;
2825
2826
2827
2828
2829
2830 if (WARN_ON_ONCE(!mm_percpu_wq))
2831 return;
2832
2833
2834
2835
2836
2837
2838 if (unlikely(!mutex_trylock(&pcpu_drain_mutex))) {
2839 if (!zone)
2840 return;
2841 mutex_lock(&pcpu_drain_mutex);
2842 }
2843
2844
2845
2846
2847
2848
2849
2850 for_each_online_cpu(cpu) {
2851 struct per_cpu_pageset *pcp;
2852 struct zone *z;
2853 bool has_pcps = false;
2854
2855 if (zone) {
2856 pcp = per_cpu_ptr(zone->pageset, cpu);
2857 if (pcp->pcp.count)
2858 has_pcps = true;
2859 } else {
2860 for_each_populated_zone(z) {
2861 pcp = per_cpu_ptr(z->pageset, cpu);
2862 if (pcp->pcp.count) {
2863 has_pcps = true;
2864 break;
2865 }
2866 }
2867 }
2868
2869 if (has_pcps)
2870 cpumask_set_cpu(cpu, &cpus_with_pcps);
2871 else
2872 cpumask_clear_cpu(cpu, &cpus_with_pcps);
2873 }
2874
2875 for_each_cpu(cpu, &cpus_with_pcps) {
2876 struct pcpu_drain *drain = per_cpu_ptr(&pcpu_drain, cpu);
2877
2878 drain->zone = zone;
2879 INIT_WORK(&drain->work, drain_local_pages_wq);
2880 queue_work_on(cpu, mm_percpu_wq, &drain->work);
2881 }
2882 for_each_cpu(cpu, &cpus_with_pcps)
2883 flush_work(&per_cpu_ptr(&pcpu_drain, cpu)->work);
2884
2885 mutex_unlock(&pcpu_drain_mutex);
2886}
2887
2888#ifdef CONFIG_HIBERNATION
2889
2890
2891
2892
2893#define WD_PAGE_COUNT (128*1024)
2894
2895void mark_free_pages(struct zone *zone)
2896{
2897 unsigned long pfn, max_zone_pfn, page_count = WD_PAGE_COUNT;
2898 unsigned long flags;
2899 unsigned int order, t;
2900 struct page *page;
2901
2902 if (zone_is_empty(zone))
2903 return;
2904
2905 spin_lock_irqsave(&zone->lock, flags);
2906
2907 max_zone_pfn = zone_end_pfn(zone);
2908 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
2909 if (pfn_valid(pfn)) {
2910 page = pfn_to_page(pfn);
2911
2912 if (!--page_count) {
2913 touch_nmi_watchdog();
2914 page_count = WD_PAGE_COUNT;
2915 }
2916
2917 if (page_zone(page) != zone)
2918 continue;
2919
2920 if (!swsusp_page_is_forbidden(page))
2921 swsusp_unset_page_free(page);
2922 }
2923
2924 for_each_migratetype_order(order, t) {
2925 list_for_each_entry(page,
2926 &zone->free_area[order].free_list[t], lru) {
2927 unsigned long i;
2928
2929 pfn = page_to_pfn(page);
2930 for (i = 0; i < (1UL << order); i++) {
2931 if (!--page_count) {
2932 touch_nmi_watchdog();
2933 page_count = WD_PAGE_COUNT;
2934 }
2935 swsusp_set_page_free(pfn_to_page(pfn + i));
2936 }
2937 }
2938 }
2939 spin_unlock_irqrestore(&zone->lock, flags);
2940}
2941#endif
2942
2943static bool free_unref_page_prepare(struct page *page, unsigned long pfn)
2944{
2945 int migratetype;
2946
2947 if (!free_pcp_prepare(page))
2948 return false;
2949
2950 migratetype = get_pfnblock_migratetype(page, pfn);
2951 set_pcppage_migratetype(page, migratetype);
2952 return true;
2953}
2954
2955static void free_unref_page_commit(struct page *page, unsigned long pfn)
2956{
2957 struct zone *zone = page_zone(page);
2958 struct per_cpu_pages *pcp;
2959 int migratetype;
2960
2961 migratetype = get_pcppage_migratetype(page);
2962 __count_vm_event(PGFREE);
2963
2964
2965
2966
2967
2968
2969
2970
2971 if (migratetype >= MIGRATE_PCPTYPES) {
2972 if (unlikely(is_migrate_isolate(migratetype))) {
2973 free_one_page(zone, page, pfn, 0, migratetype);
2974 return;
2975 }
2976 migratetype = MIGRATE_MOVABLE;
2977 }
2978
2979 pcp = &this_cpu_ptr(zone->pageset)->pcp;
2980 list_add(&page->lru, &pcp->lists[migratetype]);
2981 pcp->count++;
2982 if (pcp->count >= pcp->high) {
2983 unsigned long batch = READ_ONCE(pcp->batch);
2984 free_pcppages_bulk(zone, batch, pcp);
2985 }
2986}
2987
2988
2989
2990
2991void free_unref_page(struct page *page)
2992{
2993 unsigned long flags;
2994 unsigned long pfn = page_to_pfn(page);
2995
2996 if (!free_unref_page_prepare(page, pfn))
2997 return;
2998
2999 local_irq_save(flags);
3000 free_unref_page_commit(page, pfn);
3001 local_irq_restore(flags);
3002}
3003
3004
3005
3006
3007void free_unref_page_list(struct list_head *list)
3008{
3009 struct page *page, *next;
3010 unsigned long flags, pfn;
3011 int batch_count = 0;
3012
3013
3014 list_for_each_entry_safe(page, next, list, lru) {
3015 pfn = page_to_pfn(page);
3016 if (!free_unref_page_prepare(page, pfn))
3017 list_del(&page->lru);
3018 set_page_private(page, pfn);
3019 }
3020
3021 local_irq_save(flags);
3022 list_for_each_entry_safe(page, next, list, lru) {
3023 unsigned long pfn = page_private(page);
3024
3025 set_page_private(page, 0);
3026 trace_mm_page_free_batched(page);
3027 free_unref_page_commit(page, pfn);
3028
3029
3030
3031
3032
3033 if (++batch_count == SWAP_CLUSTER_MAX) {
3034 local_irq_restore(flags);
3035 batch_count = 0;
3036 local_irq_save(flags);
3037 }
3038 }
3039 local_irq_restore(flags);
3040}
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050void split_page(struct page *page, unsigned int order)
3051{
3052 int i;
3053
3054 VM_BUG_ON_PAGE(PageCompound(page), page);
3055 VM_BUG_ON_PAGE(!page_count(page), page);
3056
3057 for (i = 1; i < (1 << order); i++)
3058 set_page_refcounted(page + i);
3059 split_page_owner(page, order);
3060}
3061EXPORT_SYMBOL_GPL(split_page);
3062
3063int __isolate_free_page(struct page *page, unsigned int order)
3064{
3065 struct free_area *area = &page_zone(page)->free_area[order];
3066 unsigned long watermark;
3067 struct zone *zone;
3068 int mt;
3069
3070 BUG_ON(!PageBuddy(page));
3071
3072 zone = page_zone(page);
3073 mt = get_pageblock_migratetype(page);
3074
3075 if (!is_migrate_isolate(mt)) {
3076
3077
3078
3079
3080
3081
3082 watermark = zone->_watermark[WMARK_MIN] + (1UL << order);
3083 if (!zone_watermark_ok(zone, 0, watermark, 0, ALLOC_CMA))
3084 return 0;
3085
3086 __mod_zone_freepage_state(zone, -(1UL << order), mt);
3087 }
3088
3089
3090
3091 del_page_from_free_area(page, area);
3092
3093
3094
3095
3096
3097 if (order >= pageblock_order - 1) {
3098 struct page *endpage = page + (1 << order) - 1;
3099 for (; page < endpage; page += pageblock_nr_pages) {
3100 int mt = get_pageblock_migratetype(page);
3101 if (!is_migrate_isolate(mt) && !is_migrate_cma(mt)
3102 && !is_migrate_highatomic(mt))
3103 set_pageblock_migratetype(page,
3104 MIGRATE_MOVABLE);
3105 }
3106 }
3107
3108
3109 return 1UL << order;
3110}
3111
3112
3113
3114
3115
3116
3117static inline void zone_statistics(struct zone *preferred_zone, struct zone *z)
3118{
3119#ifdef CONFIG_NUMA
3120 enum numa_stat_item local_stat = NUMA_LOCAL;
3121
3122
3123 if (!static_branch_likely(&vm_numa_stat_key))
3124 return;
3125
3126 if (zone_to_nid(z) != numa_node_id())
3127 local_stat = NUMA_OTHER;
3128
3129 if (zone_to_nid(z) == zone_to_nid(preferred_zone))
3130 __inc_numa_state(z, NUMA_HIT);
3131 else {
3132 __inc_numa_state(z, NUMA_MISS);
3133 __inc_numa_state(preferred_zone, NUMA_FOREIGN);
3134 }
3135 __inc_numa_state(z, local_stat);
3136#endif
3137}
3138
3139
3140static struct page *__rmqueue_pcplist(struct zone *zone, int migratetype,
3141 unsigned int alloc_flags,
3142 struct per_cpu_pages *pcp,
3143 struct list_head *list)
3144{
3145 struct page *page;
3146
3147 do {
3148 if (list_empty(list)) {
3149 pcp->count += rmqueue_bulk(zone, 0,
3150 pcp->batch, list,
3151 migratetype, alloc_flags);
3152 if (unlikely(list_empty(list)))
3153 return NULL;
3154 }
3155
3156 page = list_first_entry(list, struct page, lru);
3157 list_del(&page->lru);
3158 pcp->count--;
3159 } while (check_new_pcp(page));
3160
3161 return page;
3162}
3163
3164
3165static struct page *rmqueue_pcplist(struct zone *preferred_zone,
3166 struct zone *zone, gfp_t gfp_flags,
3167 int migratetype, unsigned int alloc_flags)
3168{
3169 struct per_cpu_pages *pcp;
3170 struct list_head *list;
3171 struct page *page;
3172 unsigned long flags;
3173
3174 local_irq_save(flags);
3175 pcp = &this_cpu_ptr(zone->pageset)->pcp;
3176 list = &pcp->lists[migratetype];
3177 page = __rmqueue_pcplist(zone, migratetype, alloc_flags, pcp, list);
3178 if (page) {
3179 __count_zid_vm_events(PGALLOC, page_zonenum(page), 1);
3180 zone_statistics(preferred_zone, zone);
3181 }
3182 local_irq_restore(flags);
3183 return page;
3184}
3185
3186
3187
3188
3189static inline
3190struct page *rmqueue(struct zone *preferred_zone,
3191 struct zone *zone, unsigned int order,
3192 gfp_t gfp_flags, unsigned int alloc_flags,
3193 int migratetype)
3194{
3195 unsigned long flags;
3196 struct page *page;
3197
3198 if (likely(order == 0)) {
3199 page = rmqueue_pcplist(preferred_zone, zone, gfp_flags,
3200 migratetype, alloc_flags);
3201 goto out;
3202 }
3203
3204
3205
3206
3207
3208 WARN_ON_ONCE((gfp_flags & __GFP_NOFAIL) && (order > 1));
3209 spin_lock_irqsave(&zone->lock, flags);
3210
3211 do {
3212 page = NULL;
3213 if (alloc_flags & ALLOC_HARDER) {
3214 page = __rmqueue_smallest(zone, order, MIGRATE_HIGHATOMIC);
3215 if (page)
3216 trace_mm_page_alloc_zone_locked(page, order, migratetype);
3217 }
3218 if (!page)
3219 page = __rmqueue(zone, order, migratetype, alloc_flags);
3220 } while (page && check_new_pages(page, order));
3221 spin_unlock(&zone->lock);
3222 if (!page)
3223 goto failed;
3224 __mod_zone_freepage_state(zone, -(1 << order),
3225 get_pcppage_migratetype(page));
3226
3227 __count_zid_vm_events(PGALLOC, page_zonenum(page), 1 << order);
3228 zone_statistics(preferred_zone, zone);
3229 local_irq_restore(flags);
3230
3231out:
3232
3233 if (test_bit(ZONE_BOOSTED_WATERMARK, &zone->flags)) {
3234 clear_bit(ZONE_BOOSTED_WATERMARK, &zone->flags);
3235 wakeup_kswapd(zone, 0, 0, zone_idx(zone));
3236 }
3237
3238 VM_BUG_ON_PAGE(page && bad_range(zone, page), page);
3239 return page;
3240
3241failed:
3242 local_irq_restore(flags);
3243 return NULL;
3244}
3245
3246#ifdef CONFIG_FAIL_PAGE_ALLOC
3247
3248static struct {
3249 struct fault_attr attr;
3250
3251 bool ignore_gfp_highmem;
3252 bool ignore_gfp_reclaim;
3253 u32 min_order;
3254} fail_page_alloc = {
3255 .attr = FAULT_ATTR_INITIALIZER,
3256 .ignore_gfp_reclaim = true,
3257 .ignore_gfp_highmem = true,
3258 .min_order = 1,
3259};
3260
3261static int __init setup_fail_page_alloc(char *str)
3262{
3263 return setup_fault_attr(&fail_page_alloc.attr, str);
3264}
3265__setup("fail_page_alloc=", setup_fail_page_alloc);
3266
3267static bool __should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
3268{
3269 if (order < fail_page_alloc.min_order)
3270 return false;
3271 if (gfp_mask & __GFP_NOFAIL)
3272 return false;
3273 if (fail_page_alloc.ignore_gfp_highmem && (gfp_mask & __GFP_HIGHMEM))
3274 return false;
3275 if (fail_page_alloc.ignore_gfp_reclaim &&
3276 (gfp_mask & __GFP_DIRECT_RECLAIM))
3277 return false;
3278
3279 return should_fail(&fail_page_alloc.attr, 1 << order);
3280}
3281
3282#ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
3283
3284static int __init fail_page_alloc_debugfs(void)
3285{
3286 umode_t mode = S_IFREG | 0600;
3287 struct dentry *dir;
3288
3289 dir = fault_create_debugfs_attr("fail_page_alloc", NULL,
3290 &fail_page_alloc.attr);
3291
3292 debugfs_create_bool("ignore-gfp-wait", mode, dir,
3293 &fail_page_alloc.ignore_gfp_reclaim);
3294 debugfs_create_bool("ignore-gfp-highmem", mode, dir,
3295 &fail_page_alloc.ignore_gfp_highmem);
3296 debugfs_create_u32("min-order", mode, dir, &fail_page_alloc.min_order);
3297
3298 return 0;
3299}
3300
3301late_initcall(fail_page_alloc_debugfs);
3302
3303#endif
3304
3305#else
3306
3307static inline bool __should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
3308{
3309 return false;
3310}
3311
3312#endif
3313
3314static noinline bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
3315{
3316 return __should_fail_alloc_page(gfp_mask, order);
3317}
3318ALLOW_ERROR_INJECTION(should_fail_alloc_page, TRUE);
3319
3320
3321
3322
3323
3324
3325
3326bool __zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark,
3327 int classzone_idx, unsigned int alloc_flags,
3328 long free_pages)
3329{
3330 long min = mark;
3331 int o;
3332 const bool alloc_harder = (alloc_flags & (ALLOC_HARDER|ALLOC_OOM));
3333
3334
3335 free_pages -= (1 << order) - 1;
3336
3337 if (alloc_flags & ALLOC_HIGH)
3338 min -= min / 2;
3339
3340
3341
3342
3343
3344
3345 if (likely(!alloc_harder)) {
3346 free_pages -= z->nr_reserved_highatomic;
3347 } else {
3348
3349
3350
3351
3352
3353
3354 if (alloc_flags & ALLOC_OOM)
3355 min -= min / 2;
3356 else
3357 min -= min / 4;
3358 }
3359
3360
3361#ifdef CONFIG_CMA
3362
3363 if (!(alloc_flags & ALLOC_CMA))
3364 free_pages -= zone_page_state(z, NR_FREE_CMA_PAGES);
3365#endif
3366
3367
3368
3369
3370
3371
3372 if (free_pages <= min + z->lowmem_reserve[classzone_idx])
3373 return false;
3374
3375
3376 if (!order)
3377 return true;
3378
3379
3380 for (o = order; o < MAX_ORDER; o++) {
3381 struct free_area *area = &z->free_area[o];
3382 int mt;
3383
3384 if (!area->nr_free)
3385 continue;
3386
3387 for (mt = 0; mt < MIGRATE_PCPTYPES; mt++) {
3388 if (!free_area_empty(area, mt))
3389 return true;
3390 }
3391
3392#ifdef CONFIG_CMA
3393 if ((alloc_flags & ALLOC_CMA) &&
3394 !free_area_empty(area, MIGRATE_CMA)) {
3395 return true;
3396 }
3397#endif
3398 if (alloc_harder &&
3399 !list_empty(&area->free_list[MIGRATE_HIGHATOMIC]))
3400 return true;
3401 }
3402 return false;
3403}
3404
3405bool zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark,
3406 int classzone_idx, unsigned int alloc_flags)
3407{
3408 return __zone_watermark_ok(z, order, mark, classzone_idx, alloc_flags,
3409 zone_page_state(z, NR_FREE_PAGES));
3410}
3411
3412static inline bool zone_watermark_fast(struct zone *z, unsigned int order,
3413 unsigned long mark, int classzone_idx, unsigned int alloc_flags)
3414{
3415 long free_pages = zone_page_state(z, NR_FREE_PAGES);
3416 long cma_pages = 0;
3417
3418#ifdef CONFIG_CMA
3419
3420 if (!(alloc_flags & ALLOC_CMA))
3421 cma_pages = zone_page_state(z, NR_FREE_CMA_PAGES);
3422#endif
3423
3424
3425
3426
3427
3428
3429
3430
3431 if (!order && (free_pages - cma_pages) > mark + z->lowmem_reserve[classzone_idx])
3432 return true;
3433
3434 return __zone_watermark_ok(z, order, mark, classzone_idx, alloc_flags,
3435 free_pages);
3436}
3437
3438bool zone_watermark_ok_safe(struct zone *z, unsigned int order,
3439 unsigned long mark, int classzone_idx)
3440{
3441 long free_pages = zone_page_state(z, NR_FREE_PAGES);
3442
3443 if (z->percpu_drift_mark && free_pages < z->percpu_drift_mark)
3444 free_pages = zone_page_state_snapshot(z, NR_FREE_PAGES);
3445
3446 return __zone_watermark_ok(z, order, mark, classzone_idx, 0,
3447 free_pages);
3448}
3449
3450#ifdef CONFIG_NUMA
3451static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone)
3452{
3453 return node_distance(zone_to_nid(local_zone), zone_to_nid(zone)) <=
3454 RECLAIM_DISTANCE;
3455}
3456#else
3457static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone)
3458{
3459 return true;
3460}
3461#endif
3462
3463
3464
3465
3466
3467
3468
3469
3470
3471static inline unsigned int
3472alloc_flags_nofragment(struct zone *zone, gfp_t gfp_mask)
3473{
3474 unsigned int alloc_flags = 0;
3475
3476 if (gfp_mask & __GFP_KSWAPD_RECLAIM)
3477 alloc_flags |= ALLOC_KSWAPD;
3478
3479#ifdef CONFIG_ZONE_DMA32
3480 if (!zone)
3481 return alloc_flags;
3482
3483 if (zone_idx(zone) != ZONE_NORMAL)
3484 return alloc_flags;
3485
3486
3487
3488
3489
3490
3491 BUILD_BUG_ON(ZONE_NORMAL - ZONE_DMA32 != 1);
3492 if (nr_online_nodes > 1 && !populated_zone(--zone))
3493 return alloc_flags;
3494
3495 alloc_flags |= ALLOC_NOFRAGMENT;
3496#endif
3497 return alloc_flags;
3498}
3499
3500
3501
3502
3503
3504static struct page *
3505get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags,
3506 const struct alloc_context *ac)
3507{
3508 struct zoneref *z;
3509 struct zone *zone;
3510 struct pglist_data *last_pgdat_dirty_limit = NULL;
3511 bool no_fallback;
3512
3513retry:
3514
3515
3516
3517
3518 no_fallback = alloc_flags & ALLOC_NOFRAGMENT;
3519 z = ac->preferred_zoneref;
3520 for_next_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx,
3521 ac->nodemask) {
3522 struct page *page;
3523 unsigned long mark;
3524
3525 if (cpusets_enabled() &&
3526 (alloc_flags & ALLOC_CPUSET) &&
3527 !__cpuset_zone_allowed(zone, gfp_mask))
3528 continue;
3529
3530
3531
3532
3533
3534
3535
3536
3537
3538
3539
3540
3541
3542
3543
3544
3545
3546
3547
3548 if (ac->spread_dirty_pages) {
3549 if (last_pgdat_dirty_limit == zone->zone_pgdat)
3550 continue;
3551
3552 if (!node_dirty_ok(zone->zone_pgdat)) {
3553 last_pgdat_dirty_limit = zone->zone_pgdat;
3554 continue;
3555 }
3556 }
3557
3558 if (no_fallback && nr_online_nodes > 1 &&
3559 zone != ac->preferred_zoneref->zone) {
3560 int local_nid;
3561
3562
3563
3564
3565
3566
3567 local_nid = zone_to_nid(ac->preferred_zoneref->zone);
3568 if (zone_to_nid(zone) != local_nid) {
3569 alloc_flags &= ~ALLOC_NOFRAGMENT;
3570 goto retry;
3571 }
3572 }
3573
3574 mark = wmark_pages(zone, alloc_flags & ALLOC_WMARK_MASK);
3575 if (!zone_watermark_fast(zone, order, mark,
3576 ac_classzone_idx(ac), alloc_flags)) {
3577 int ret;
3578
3579#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
3580
3581
3582
3583
3584 if (static_branch_unlikely(&deferred_pages)) {
3585 if (_deferred_grow_zone(zone, order))
3586 goto try_this_zone;
3587 }
3588#endif
3589
3590 BUILD_BUG_ON(ALLOC_NO_WATERMARKS < NR_WMARK);
3591 if (alloc_flags & ALLOC_NO_WATERMARKS)
3592 goto try_this_zone;
3593
3594 if (node_reclaim_mode == 0 ||
3595 !zone_allows_reclaim(ac->preferred_zoneref->zone, zone))
3596 continue;
3597
3598 ret = node_reclaim(zone->zone_pgdat, gfp_mask, order);
3599 switch (ret) {
3600 case NODE_RECLAIM_NOSCAN:
3601
3602 continue;
3603 case NODE_RECLAIM_FULL:
3604
3605 continue;
3606 default:
3607
3608 if (zone_watermark_ok(zone, order, mark,
3609 ac_classzone_idx(ac), alloc_flags))
3610 goto try_this_zone;
3611
3612 continue;
3613 }
3614 }
3615
3616try_this_zone:
3617 page = rmqueue(ac->preferred_zoneref->zone, zone, order,
3618 gfp_mask, alloc_flags, ac->migratetype);
3619 if (page) {
3620 prep_new_page(page, order, gfp_mask, alloc_flags);
3621
3622
3623
3624
3625
3626 if (unlikely(order && (alloc_flags & ALLOC_HARDER)))
3627 reserve_highatomic_pageblock(page, zone, order);
3628
3629 return page;
3630 } else {
3631#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
3632
3633 if (static_branch_unlikely(&deferred_pages)) {
3634 if (_deferred_grow_zone(zone, order))
3635 goto try_this_zone;
3636 }
3637#endif
3638 }
3639 }
3640
3641
3642
3643
3644
3645 if (no_fallback) {
3646 alloc_flags &= ~ALLOC_NOFRAGMENT;
3647 goto retry;
3648 }
3649
3650 return NULL;
3651}
3652
3653static void warn_alloc_show_mem(gfp_t gfp_mask, nodemask_t *nodemask)
3654{
3655 unsigned int filter = SHOW_MEM_FILTER_NODES;
3656 static DEFINE_RATELIMIT_STATE(show_mem_rs, HZ, 1);
3657
3658 if (!__ratelimit(&show_mem_rs))
3659 return;
3660
3661
3662
3663
3664
3665
3666 if (!(gfp_mask & __GFP_NOMEMALLOC))
3667 if (tsk_is_oom_victim(current) ||
3668 (current->flags & (PF_MEMALLOC | PF_EXITING)))
3669 filter &= ~SHOW_MEM_FILTER_NODES;
3670 if (in_interrupt() || !(gfp_mask & __GFP_DIRECT_RECLAIM))
3671 filter &= ~SHOW_MEM_FILTER_NODES;
3672
3673 show_mem(filter, nodemask);
3674}
3675
3676void warn_alloc(gfp_t gfp_mask, nodemask_t *nodemask, const char *fmt, ...)
3677{
3678 struct va_format vaf;
3679 va_list args;
3680 static DEFINE_RATELIMIT_STATE(nopage_rs, DEFAULT_RATELIMIT_INTERVAL,
3681 DEFAULT_RATELIMIT_BURST);
3682
3683 if ((gfp_mask & __GFP_NOWARN) || !__ratelimit(&nopage_rs))
3684 return;
3685
3686 va_start(args, fmt);
3687 vaf.fmt = fmt;
3688 vaf.va = &args;
3689 pr_warn("%s: %pV, mode:%#x(%pGg), nodemask=%*pbl",
3690 current->comm, &vaf, gfp_mask, &gfp_mask,
3691 nodemask_pr_args(nodemask));
3692 va_end(args);
3693
3694 cpuset_print_current_mems_allowed();
3695 pr_cont("\n");
3696 dump_stack();
3697 warn_alloc_show_mem(gfp_mask, nodemask);
3698}
3699
3700static inline struct page *
3701__alloc_pages_cpuset_fallback(gfp_t gfp_mask, unsigned int order,
3702 unsigned int alloc_flags,
3703 const struct alloc_context *ac)
3704{
3705 struct page *page;
3706
3707 page = get_page_from_freelist(gfp_mask, order,
3708 alloc_flags|ALLOC_CPUSET, ac);
3709
3710
3711
3712
3713 if (!page)
3714 page = get_page_from_freelist(gfp_mask, order,
3715 alloc_flags, ac);
3716
3717 return page;
3718}
3719
3720static inline struct page *
3721__alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order,
3722 const struct alloc_context *ac, unsigned long *did_some_progress)
3723{
3724 struct oom_control oc = {
3725 .zonelist = ac->zonelist,
3726 .nodemask = ac->nodemask,
3727 .memcg = NULL,
3728 .gfp_mask = gfp_mask,
3729 .order = order,
3730 };
3731 struct page *page;
3732
3733 *did_some_progress = 0;
3734
3735
3736
3737
3738
3739 if (!mutex_trylock(&oom_lock)) {
3740 *did_some_progress = 1;
3741 schedule_timeout_uninterruptible(1);
3742 return NULL;
3743 }
3744
3745
3746
3747
3748
3749
3750
3751
3752 page = get_page_from_freelist((gfp_mask | __GFP_HARDWALL) &
3753 ~__GFP_DIRECT_RECLAIM, order,
3754 ALLOC_WMARK_HIGH|ALLOC_CPUSET, ac);
3755 if (page)
3756 goto out;
3757
3758
3759 if (current->flags & PF_DUMPCORE)
3760 goto out;
3761
3762 if (order > PAGE_ALLOC_COSTLY_ORDER)
3763 goto out;
3764
3765
3766
3767
3768
3769
3770 if (gfp_mask & __GFP_RETRY_MAYFAIL)
3771 goto out;
3772
3773 if (ac->high_zoneidx < ZONE_NORMAL)
3774 goto out;
3775 if (pm_suspended_storage())
3776 goto out;
3777
3778
3779
3780
3781
3782
3783
3784
3785
3786
3787
3788 if (gfp_mask & __GFP_THISNODE)
3789 goto out;
3790
3791
3792 if (out_of_memory(&oc) || WARN_ON_ONCE(gfp_mask & __GFP_NOFAIL)) {
3793 *did_some_progress = 1;
3794
3795
3796
3797
3798
3799 if (gfp_mask & __GFP_NOFAIL)
3800 page = __alloc_pages_cpuset_fallback(gfp_mask, order,
3801 ALLOC_NO_WATERMARKS, ac);
3802 }
3803out:
3804 mutex_unlock(&oom_lock);
3805 return page;
3806}
3807
3808
3809
3810
3811
3812#define MAX_COMPACT_RETRIES 16
3813
3814#ifdef CONFIG_COMPACTION
3815
3816static struct page *
3817__alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
3818 unsigned int alloc_flags, const struct alloc_context *ac,
3819 enum compact_priority prio, enum compact_result *compact_result)
3820{
3821 struct page *page = NULL;
3822 unsigned long pflags;
3823 unsigned int noreclaim_flag;
3824
3825 if (!order)
3826 return NULL;
3827
3828 psi_memstall_enter(&pflags);
3829 noreclaim_flag = memalloc_noreclaim_save();
3830
3831 *compact_result = try_to_compact_pages(gfp_mask, order, alloc_flags, ac,
3832 prio, &page);
3833
3834 memalloc_noreclaim_restore(noreclaim_flag);
3835 psi_memstall_leave(&pflags);
3836
3837
3838
3839
3840
3841 count_vm_event(COMPACTSTALL);
3842
3843
3844 if (page)
3845 prep_new_page(page, order, gfp_mask, alloc_flags);
3846
3847
3848 if (!page)
3849 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
3850
3851 if (page) {
3852 struct zone *zone = page_zone(page);
3853
3854 zone->compact_blockskip_flush = false;
3855 compaction_defer_reset(zone, order, true);
3856 count_vm_event(COMPACTSUCCESS);
3857 return page;
3858 }
3859
3860
3861
3862
3863
3864 count_vm_event(COMPACTFAIL);
3865
3866 cond_resched();
3867
3868 return NULL;
3869}
3870
3871static inline bool
3872should_compact_retry(struct alloc_context *ac, int order, int alloc_flags,
3873 enum compact_result compact_result,
3874 enum compact_priority *compact_priority,
3875 int *compaction_retries)
3876{
3877 int max_retries = MAX_COMPACT_RETRIES;
3878 int min_priority;
3879 bool ret = false;
3880 int retries = *compaction_retries;
3881 enum compact_priority priority = *compact_priority;
3882
3883 if (!order)
3884 return false;
3885
3886 if (compaction_made_progress(compact_result))
3887 (*compaction_retries)++;
3888
3889
3890
3891
3892
3893
3894 if (compaction_failed(compact_result))
3895 goto check_priority;
3896
3897
3898
3899
3900
3901
3902
3903 if (compaction_withdrawn(compact_result)) {
3904 ret = compaction_zonelist_suitable(ac, order, alloc_flags);
3905 goto out;
3906 }
3907
3908
3909
3910
3911
3912
3913
3914
3915
3916 if (order > PAGE_ALLOC_COSTLY_ORDER)
3917 max_retries /= 4;
3918 if (*compaction_retries <= max_retries) {
3919 ret = true;
3920 goto out;
3921 }
3922
3923
3924
3925
3926
3927check_priority:
3928 min_priority = (order > PAGE_ALLOC_COSTLY_ORDER) ?
3929 MIN_COMPACT_COSTLY_PRIORITY : MIN_COMPACT_PRIORITY;
3930
3931 if (*compact_priority > min_priority) {
3932 (*compact_priority)--;
3933 *compaction_retries = 0;
3934 ret = true;
3935 }
3936out:
3937 trace_compact_retry(order, priority, compact_result, retries, max_retries, ret);
3938 return ret;
3939}
3940#else
3941static inline struct page *
3942__alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
3943 unsigned int alloc_flags, const struct alloc_context *ac,
3944 enum compact_priority prio, enum compact_result *compact_result)
3945{
3946 *compact_result = COMPACT_SKIPPED;
3947 return NULL;
3948}
3949
3950static inline bool
3951should_compact_retry(struct alloc_context *ac, unsigned int order, int alloc_flags,
3952 enum compact_result compact_result,
3953 enum compact_priority *compact_priority,
3954 int *compaction_retries)
3955{
3956 struct zone *zone;
3957 struct zoneref *z;
3958
3959 if (!order || order > PAGE_ALLOC_COSTLY_ORDER)
3960 return false;
3961
3962
3963
3964
3965
3966
3967
3968 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx,
3969 ac->nodemask) {
3970 if (zone_watermark_ok(zone, 0, min_wmark_pages(zone),
3971 ac_classzone_idx(ac), alloc_flags))
3972 return true;
3973 }
3974 return false;
3975}
3976#endif
3977
3978#ifdef CONFIG_LOCKDEP
3979static struct lockdep_map __fs_reclaim_map =
3980 STATIC_LOCKDEP_MAP_INIT("fs_reclaim", &__fs_reclaim_map);
3981
3982static bool __need_fs_reclaim(gfp_t gfp_mask)
3983{
3984 gfp_mask = current_gfp_context(gfp_mask);
3985
3986
3987 if (!(gfp_mask & __GFP_DIRECT_RECLAIM))
3988 return false;
3989
3990
3991 if (current->flags & PF_MEMALLOC)
3992 return false;
3993
3994
3995 if (!(gfp_mask & __GFP_FS))
3996 return false;
3997
3998 if (gfp_mask & __GFP_NOLOCKDEP)
3999 return false;
4000
4001 return true;
4002}
4003
4004void __fs_reclaim_acquire(void)
4005{
4006 lock_map_acquire(&__fs_reclaim_map);
4007}
4008
4009void __fs_reclaim_release(void)
4010{
4011 lock_map_release(&__fs_reclaim_map);
4012}
4013
4014void fs_reclaim_acquire(gfp_t gfp_mask)
4015{
4016 if (__need_fs_reclaim(gfp_mask))
4017 __fs_reclaim_acquire();
4018}
4019EXPORT_SYMBOL_GPL(fs_reclaim_acquire);
4020
4021void fs_reclaim_release(gfp_t gfp_mask)
4022{
4023 if (__need_fs_reclaim(gfp_mask))
4024 __fs_reclaim_release();
4025}
4026EXPORT_SYMBOL_GPL(fs_reclaim_release);
4027#endif
4028
4029
4030static int
4031__perform_reclaim(gfp_t gfp_mask, unsigned int order,
4032 const struct alloc_context *ac)
4033{
4034 struct reclaim_state reclaim_state;
4035 int progress;
4036 unsigned int noreclaim_flag;
4037 unsigned long pflags;
4038
4039 cond_resched();
4040
4041
4042 cpuset_memory_pressure_bump();
4043 psi_memstall_enter(&pflags);
4044 fs_reclaim_acquire(gfp_mask);
4045 noreclaim_flag = memalloc_noreclaim_save();
4046 reclaim_state.reclaimed_slab = 0;
4047 current->reclaim_state = &reclaim_state;
4048
4049 progress = try_to_free_pages(ac->zonelist, order, gfp_mask,
4050 ac->nodemask);
4051
4052 current->reclaim_state = NULL;
4053 memalloc_noreclaim_restore(noreclaim_flag);
4054 fs_reclaim_release(gfp_mask);
4055 psi_memstall_leave(&pflags);
4056
4057 cond_resched();
4058
4059 return progress;
4060}
4061
4062
4063static inline struct page *
4064__alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order,
4065 unsigned int alloc_flags, const struct alloc_context *ac,
4066 unsigned long *did_some_progress)
4067{
4068 struct page *page = NULL;
4069 bool drained = false;
4070
4071 *did_some_progress = __perform_reclaim(gfp_mask, order, ac);
4072 if (unlikely(!(*did_some_progress)))
4073 return NULL;
4074
4075retry:
4076 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
4077
4078
4079
4080
4081
4082
4083 if (!page && !drained) {
4084 unreserve_highatomic_pageblock(ac, false);
4085 drain_all_pages(NULL);
4086 drained = true;
4087 goto retry;
4088 }
4089
4090 return page;
4091}
4092
4093static void wake_all_kswapds(unsigned int order, gfp_t gfp_mask,
4094 const struct alloc_context *ac)
4095{
4096 struct zoneref *z;
4097 struct zone *zone;
4098 pg_data_t *last_pgdat = NULL;
4099 enum zone_type high_zoneidx = ac->high_zoneidx;
4100
4101 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, high_zoneidx,
4102 ac->nodemask) {
4103 if (last_pgdat != zone->zone_pgdat)
4104 wakeup_kswapd(zone, gfp_mask, order, high_zoneidx);
4105 last_pgdat = zone->zone_pgdat;
4106 }
4107}
4108
4109static inline unsigned int
4110gfp_to_alloc_flags(gfp_t gfp_mask)
4111{
4112 unsigned int alloc_flags = ALLOC_WMARK_MIN | ALLOC_CPUSET;
4113
4114
4115 BUILD_BUG_ON(__GFP_HIGH != (__force gfp_t) ALLOC_HIGH);
4116
4117
4118
4119
4120
4121
4122
4123 alloc_flags |= (__force int) (gfp_mask & __GFP_HIGH);
4124
4125 if (gfp_mask & __GFP_ATOMIC) {
4126
4127
4128
4129
4130 if (!(gfp_mask & __GFP_NOMEMALLOC))
4131 alloc_flags |= ALLOC_HARDER;
4132
4133
4134
4135
4136 alloc_flags &= ~ALLOC_CPUSET;
4137 } else if (unlikely(rt_task(current)) && !in_interrupt())
4138 alloc_flags |= ALLOC_HARDER;
4139
4140 if (gfp_mask & __GFP_KSWAPD_RECLAIM)
4141 alloc_flags |= ALLOC_KSWAPD;
4142
4143#ifdef CONFIG_CMA
4144 if (gfpflags_to_migratetype(gfp_mask) == MIGRATE_MOVABLE)
4145 alloc_flags |= ALLOC_CMA;
4146#endif
4147 return alloc_flags;
4148}
4149
4150static bool oom_reserves_allowed(struct task_struct *tsk)
4151{
4152 if (!tsk_is_oom_victim(tsk))
4153 return false;
4154
4155
4156
4157
4158
4159 if (!IS_ENABLED(CONFIG_MMU) && !test_thread_flag(TIF_MEMDIE))
4160 return false;
4161
4162 return true;
4163}
4164
4165
4166
4167
4168
4169static inline int __gfp_pfmemalloc_flags(gfp_t gfp_mask)
4170{
4171 if (unlikely(gfp_mask & __GFP_NOMEMALLOC))
4172 return 0;
4173 if (gfp_mask & __GFP_MEMALLOC)
4174 return ALLOC_NO_WATERMARKS;
4175 if (in_serving_softirq() && (current->flags & PF_MEMALLOC))
4176 return ALLOC_NO_WATERMARKS;
4177 if (!in_interrupt()) {
4178 if (current->flags & PF_MEMALLOC)
4179 return ALLOC_NO_WATERMARKS;
4180 else if (oom_reserves_allowed(current))
4181 return ALLOC_OOM;
4182 }
4183
4184 return 0;
4185}
4186
4187bool gfp_pfmemalloc_allowed(gfp_t gfp_mask)
4188{
4189 return !!__gfp_pfmemalloc_flags(gfp_mask);
4190}
4191
4192
4193
4194
4195
4196
4197
4198
4199
4200
4201
4202static inline bool
4203should_reclaim_retry(gfp_t gfp_mask, unsigned order,
4204 struct alloc_context *ac, int alloc_flags,
4205 bool did_some_progress, int *no_progress_loops)
4206{
4207 struct zone *zone;
4208 struct zoneref *z;
4209 bool ret = false;
4210
4211
4212
4213
4214
4215
4216 if (did_some_progress && order <= PAGE_ALLOC_COSTLY_ORDER)
4217 *no_progress_loops = 0;
4218 else
4219 (*no_progress_loops)++;
4220
4221
4222
4223
4224
4225 if (*no_progress_loops > MAX_RECLAIM_RETRIES) {
4226
4227 return unreserve_highatomic_pageblock(ac, true);
4228 }
4229
4230
4231
4232
4233
4234
4235
4236 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx,
4237 ac->nodemask) {
4238 unsigned long available;
4239 unsigned long reclaimable;
4240 unsigned long min_wmark = min_wmark_pages(zone);
4241 bool wmark;
4242
4243 available = reclaimable = zone_reclaimable_pages(zone);
4244 available += zone_page_state_snapshot(zone, NR_FREE_PAGES);
4245
4246
4247
4248
4249
4250 wmark = __zone_watermark_ok(zone, order, min_wmark,
4251 ac_classzone_idx(ac), alloc_flags, available);
4252 trace_reclaim_retry_zone(z, order, reclaimable,
4253 available, min_wmark, *no_progress_loops, wmark);
4254 if (wmark) {
4255
4256
4257
4258
4259
4260
4261 if (!did_some_progress) {
4262 unsigned long write_pending;
4263
4264 write_pending = zone_page_state_snapshot(zone,
4265 NR_ZONE_WRITE_PENDING);
4266
4267 if (2 * write_pending > reclaimable) {
4268 congestion_wait(BLK_RW_ASYNC, HZ/10);
4269 return true;
4270 }
4271 }
4272
4273 ret = true;
4274 goto out;
4275 }
4276 }
4277
4278out:
4279
4280
4281
4282
4283
4284
4285
4286 if (current->flags & PF_WQ_WORKER)
4287 schedule_timeout_uninterruptible(1);
4288 else
4289 cond_resched();
4290 return ret;
4291}
4292
4293static inline bool
4294check_retry_cpuset(int cpuset_mems_cookie, struct alloc_context *ac)
4295{
4296
4297
4298
4299
4300
4301
4302
4303
4304
4305
4306
4307 if (cpusets_enabled() && ac->nodemask &&
4308 !cpuset_nodemask_valid_mems_allowed(ac->nodemask)) {
4309 ac->nodemask = NULL;
4310 return true;
4311 }
4312
4313
4314
4315
4316
4317
4318
4319
4320 if (read_mems_allowed_retry(cpuset_mems_cookie))
4321 return true;
4322
4323 return false;
4324}
4325
4326static inline struct page *
4327__alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
4328 struct alloc_context *ac)
4329{
4330 bool can_direct_reclaim = gfp_mask & __GFP_DIRECT_RECLAIM;
4331 const bool costly_order = order > PAGE_ALLOC_COSTLY_ORDER;
4332 struct page *page = NULL;
4333 unsigned int alloc_flags;
4334 unsigned long did_some_progress;
4335 enum compact_priority compact_priority;
4336 enum compact_result compact_result;
4337 int compaction_retries;
4338 int no_progress_loops;
4339 unsigned int cpuset_mems_cookie;
4340 int reserve_flags;
4341
4342
4343
4344
4345
4346 if (WARN_ON_ONCE((gfp_mask & (__GFP_ATOMIC|__GFP_DIRECT_RECLAIM)) ==
4347 (__GFP_ATOMIC|__GFP_DIRECT_RECLAIM)))
4348 gfp_mask &= ~__GFP_ATOMIC;
4349
4350retry_cpuset:
4351 compaction_retries = 0;
4352 no_progress_loops = 0;
4353 compact_priority = DEF_COMPACT_PRIORITY;
4354 cpuset_mems_cookie = read_mems_allowed_begin();
4355
4356
4357
4358
4359
4360
4361 alloc_flags = gfp_to_alloc_flags(gfp_mask);
4362
4363
4364
4365
4366
4367
4368
4369 ac->preferred_zoneref = first_zones_zonelist(ac->zonelist,
4370 ac->high_zoneidx, ac->nodemask);
4371 if (!ac->preferred_zoneref->zone)
4372 goto nopage;
4373
4374 if (alloc_flags & ALLOC_KSWAPD)
4375 wake_all_kswapds(order, gfp_mask, ac);
4376
4377
4378
4379
4380
4381 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
4382 if (page)
4383 goto got_pg;
4384
4385
4386
4387
4388
4389
4390
4391
4392
4393
4394 if (can_direct_reclaim &&
4395 (costly_order ||
4396 (order > 0 && ac->migratetype != MIGRATE_MOVABLE))
4397 && !gfp_pfmemalloc_allowed(gfp_mask)) {
4398 page = __alloc_pages_direct_compact(gfp_mask, order,
4399 alloc_flags, ac,
4400 INIT_COMPACT_PRIORITY,
4401 &compact_result);
4402 if (page)
4403 goto got_pg;
4404
4405
4406
4407
4408
4409 if (costly_order && (gfp_mask & __GFP_NORETRY)) {
4410
4411
4412
4413
4414
4415
4416
4417
4418 if (compact_result == COMPACT_DEFERRED)
4419 goto nopage;
4420
4421
4422
4423
4424
4425
4426 compact_priority = INIT_COMPACT_PRIORITY;
4427 }
4428 }
4429
4430retry:
4431
4432 if (alloc_flags & ALLOC_KSWAPD)
4433 wake_all_kswapds(order, gfp_mask, ac);
4434
4435 reserve_flags = __gfp_pfmemalloc_flags(gfp_mask);
4436 if (reserve_flags)
4437 alloc_flags = reserve_flags;
4438
4439
4440
4441
4442
4443
4444 if (!(alloc_flags & ALLOC_CPUSET) || reserve_flags) {
4445 ac->nodemask = NULL;
4446 ac->preferred_zoneref = first_zones_zonelist(ac->zonelist,
4447 ac->high_zoneidx, ac->nodemask);
4448 }
4449
4450
4451 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
4452 if (page)
4453 goto got_pg;
4454
4455
4456 if (!can_direct_reclaim)
4457 goto nopage;
4458
4459
4460 if (current->flags & PF_MEMALLOC)
4461 goto nopage;
4462
4463
4464 page = __alloc_pages_direct_reclaim(gfp_mask, order, alloc_flags, ac,
4465 &did_some_progress);
4466 if (page)
4467 goto got_pg;
4468
4469
4470 page = __alloc_pages_direct_compact(gfp_mask, order, alloc_flags, ac,
4471 compact_priority, &compact_result);
4472 if (page)
4473 goto got_pg;
4474
4475
4476 if (gfp_mask & __GFP_NORETRY)
4477 goto nopage;
4478
4479
4480
4481
4482
4483 if (costly_order && !(gfp_mask & __GFP_RETRY_MAYFAIL))
4484 goto nopage;
4485
4486 if (should_reclaim_retry(gfp_mask, order, ac, alloc_flags,
4487 did_some_progress > 0, &no_progress_loops))
4488 goto retry;
4489
4490
4491
4492
4493
4494
4495
4496 if (did_some_progress > 0 &&
4497 should_compact_retry(ac, order, alloc_flags,
4498 compact_result, &compact_priority,
4499 &compaction_retries))
4500 goto retry;
4501
4502
4503
4504 if (check_retry_cpuset(cpuset_mems_cookie, ac))
4505 goto retry_cpuset;
4506
4507
4508 page = __alloc_pages_may_oom(gfp_mask, order, ac, &did_some_progress);
4509 if (page)
4510 goto got_pg;
4511
4512
4513 if (tsk_is_oom_victim(current) &&
4514 (alloc_flags == ALLOC_OOM ||
4515 (gfp_mask & __GFP_NOMEMALLOC)))
4516 goto nopage;
4517
4518
4519 if (did_some_progress) {
4520 no_progress_loops = 0;
4521 goto retry;
4522 }
4523
4524nopage:
4525
4526 if (check_retry_cpuset(cpuset_mems_cookie, ac))
4527 goto retry_cpuset;
4528
4529
4530
4531
4532
4533 if (gfp_mask & __GFP_NOFAIL) {
4534
4535
4536
4537
4538 if (WARN_ON_ONCE(!can_direct_reclaim))
4539 goto fail;
4540
4541
4542
4543
4544
4545
4546 WARN_ON_ONCE(current->flags & PF_MEMALLOC);
4547
4548
4549
4550
4551
4552
4553
4554 WARN_ON_ONCE(order > PAGE_ALLOC_COSTLY_ORDER);
4555
4556
4557
4558
4559
4560
4561
4562 page = __alloc_pages_cpuset_fallback(gfp_mask, order, ALLOC_HARDER, ac);
4563 if (page)
4564 goto got_pg;
4565
4566 cond_resched();
4567 goto retry;
4568 }
4569fail:
4570 warn_alloc(gfp_mask, ac->nodemask,
4571 "page allocation failure: order:%u", order);
4572got_pg:
4573 return page;
4574}
4575
4576static inline bool prepare_alloc_pages(gfp_t gfp_mask, unsigned int order,
4577 int preferred_nid, nodemask_t *nodemask,
4578 struct alloc_context *ac, gfp_t *alloc_mask,
4579 unsigned int *alloc_flags)
4580{
4581 ac->high_zoneidx = gfp_zone(gfp_mask);
4582 ac->zonelist = node_zonelist(preferred_nid, gfp_mask);
4583 ac->nodemask = nodemask;
4584 ac->migratetype = gfpflags_to_migratetype(gfp_mask);
4585
4586 if (cpusets_enabled()) {
4587 *alloc_mask |= __GFP_HARDWALL;
4588 if (!ac->nodemask)
4589 ac->nodemask = &cpuset_current_mems_allowed;
4590 else
4591 *alloc_flags |= ALLOC_CPUSET;
4592 }
4593
4594 fs_reclaim_acquire(gfp_mask);
4595 fs_reclaim_release(gfp_mask);
4596
4597 might_sleep_if(gfp_mask & __GFP_DIRECT_RECLAIM);
4598
4599 if (should_fail_alloc_page(gfp_mask, order))
4600 return false;
4601
4602 if (IS_ENABLED(CONFIG_CMA) && ac->migratetype == MIGRATE_MOVABLE)
4603 *alloc_flags |= ALLOC_CMA;
4604
4605 return true;
4606}
4607
4608
4609static inline void finalise_ac(gfp_t gfp_mask, struct alloc_context *ac)
4610{
4611
4612 ac->spread_dirty_pages = (gfp_mask & __GFP_WRITE);
4613
4614
4615
4616
4617
4618
4619 ac->preferred_zoneref = first_zones_zonelist(ac->zonelist,
4620 ac->high_zoneidx, ac->nodemask);
4621}
4622
4623
4624
4625
4626struct page *
4627__alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order, int preferred_nid,
4628 nodemask_t *nodemask)
4629{
4630 struct page *page;
4631 unsigned int alloc_flags = ALLOC_WMARK_LOW;
4632 gfp_t alloc_mask;
4633 struct alloc_context ac = { };
4634
4635
4636
4637
4638
4639 if (unlikely(order >= MAX_ORDER)) {
4640 WARN_ON_ONCE(!(gfp_mask & __GFP_NOWARN));
4641 return NULL;
4642 }
4643
4644 gfp_mask &= gfp_allowed_mask;
4645 alloc_mask = gfp_mask;
4646 if (!prepare_alloc_pages(gfp_mask, order, preferred_nid, nodemask, &ac, &alloc_mask, &alloc_flags))
4647 return NULL;
4648
4649 finalise_ac(gfp_mask, &ac);
4650
4651
4652
4653
4654
4655 alloc_flags |= alloc_flags_nofragment(ac.preferred_zoneref->zone, gfp_mask);
4656
4657
4658 page = get_page_from_freelist(alloc_mask, order, alloc_flags, &ac);
4659 if (likely(page))
4660 goto out;
4661
4662
4663
4664
4665
4666
4667
4668 alloc_mask = current_gfp_context(gfp_mask);
4669 ac.spread_dirty_pages = false;
4670
4671
4672
4673
4674
4675 if (unlikely(ac.nodemask != nodemask))
4676 ac.nodemask = nodemask;
4677
4678 page = __alloc_pages_slowpath(alloc_mask, order, &ac);
4679
4680out:
4681 if (memcg_kmem_enabled() && (gfp_mask & __GFP_ACCOUNT) && page &&
4682 unlikely(__memcg_kmem_charge(page, gfp_mask, order) != 0)) {
4683 __free_pages(page, order);
4684 page = NULL;
4685 }
4686
4687 trace_mm_page_alloc(page, order, alloc_mask, ac.migratetype);
4688
4689 return page;
4690}
4691EXPORT_SYMBOL(__alloc_pages_nodemask);
4692
4693
4694
4695
4696
4697
4698unsigned long __get_free_pages(gfp_t gfp_mask, unsigned int order)
4699{
4700 struct page *page;
4701
4702 page = alloc_pages(gfp_mask & ~__GFP_HIGHMEM, order);
4703 if (!page)
4704 return 0;
4705 return (unsigned long) page_address(page);
4706}
4707EXPORT_SYMBOL(__get_free_pages);
4708
4709unsigned long get_zeroed_page(gfp_t gfp_mask)
4710{
4711 return __get_free_pages(gfp_mask | __GFP_ZERO, 0);
4712}
4713EXPORT_SYMBOL(get_zeroed_page);
4714
4715static inline void free_the_page(struct page *page, unsigned int order)
4716{
4717 if (order == 0)
4718 free_unref_page(page);
4719 else
4720 __free_pages_ok(page, order);
4721}
4722
4723void __free_pages(struct page *page, unsigned int order)
4724{
4725 if (put_page_testzero(page))
4726 free_the_page(page, order);
4727}
4728EXPORT_SYMBOL(__free_pages);
4729
4730void free_pages(unsigned long addr, unsigned int order)
4731{
4732 if (addr != 0) {
4733 VM_BUG_ON(!virt_addr_valid((void *)addr));
4734 __free_pages(virt_to_page((void *)addr), order);
4735 }
4736}
4737
4738EXPORT_SYMBOL(free_pages);
4739
4740
4741
4742
4743
4744
4745
4746
4747
4748
4749
4750
4751static struct page *__page_frag_cache_refill(struct page_frag_cache *nc,
4752 gfp_t gfp_mask)
4753{
4754 struct page *page = NULL;
4755 gfp_t gfp = gfp_mask;
4756
4757#if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE)
4758 gfp_mask |= __GFP_COMP | __GFP_NOWARN | __GFP_NORETRY |
4759 __GFP_NOMEMALLOC;
4760 page = alloc_pages_node(NUMA_NO_NODE, gfp_mask,
4761 PAGE_FRAG_CACHE_MAX_ORDER);
4762 nc->size = page ? PAGE_FRAG_CACHE_MAX_SIZE : PAGE_SIZE;
4763#endif
4764 if (unlikely(!page))
4765 page = alloc_pages_node(NUMA_NO_NODE, gfp, 0);
4766
4767 nc->va = page ? page_address(page) : NULL;
4768
4769 return page;
4770}
4771
4772void __page_frag_cache_drain(struct page *page, unsigned int count)
4773{
4774 VM_BUG_ON_PAGE(page_ref_count(page) == 0, page);
4775
4776 if (page_ref_sub_and_test(page, count))
4777 free_the_page(page, compound_order(page));
4778}
4779EXPORT_SYMBOL(__page_frag_cache_drain);
4780
4781void *page_frag_alloc(struct page_frag_cache *nc,
4782 unsigned int fragsz, gfp_t gfp_mask)
4783{
4784 unsigned int size = PAGE_SIZE;
4785 struct page *page;
4786 int offset;
4787
4788 if (unlikely(!nc->va)) {
4789refill:
4790 page = __page_frag_cache_refill(nc, gfp_mask);
4791 if (!page)
4792 return NULL;
4793
4794#if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE)
4795
4796 size = nc->size;
4797#endif
4798
4799
4800
4801 page_ref_add(page, PAGE_FRAG_CACHE_MAX_SIZE);
4802
4803
4804 nc->pfmemalloc = page_is_pfmemalloc(page);
4805 nc->pagecnt_bias = PAGE_FRAG_CACHE_MAX_SIZE + 1;
4806 nc->offset = size;
4807 }
4808
4809 offset = nc->offset - fragsz;
4810 if (unlikely(offset < 0)) {
4811 page = virt_to_page(nc->va);
4812
4813 if (!page_ref_sub_and_test(page, nc->pagecnt_bias))
4814 goto refill;
4815
4816#if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE)
4817
4818 size = nc->size;
4819#endif
4820
4821 set_page_count(page, PAGE_FRAG_CACHE_MAX_SIZE + 1);
4822
4823
4824 nc->pagecnt_bias = PAGE_FRAG_CACHE_MAX_SIZE + 1;
4825 offset = size - fragsz;
4826 }
4827
4828 nc->pagecnt_bias--;
4829 nc->offset = offset;
4830
4831 return nc->va + offset;
4832}
4833EXPORT_SYMBOL(page_frag_alloc);
4834
4835
4836
4837
4838void page_frag_free(void *addr)
4839{
4840 struct page *page = virt_to_head_page(addr);
4841
4842 if (unlikely(put_page_testzero(page)))
4843 free_the_page(page, compound_order(page));
4844}
4845EXPORT_SYMBOL(page_frag_free);
4846
4847static void *make_alloc_exact(unsigned long addr, unsigned int order,
4848 size_t size)
4849{
4850 if (addr) {
4851 unsigned long alloc_end = addr + (PAGE_SIZE << order);
4852 unsigned long used = addr + PAGE_ALIGN(size);
4853
4854 split_page(virt_to_page((void *)addr), order);
4855 while (used < alloc_end) {
4856 free_page(used);
4857 used += PAGE_SIZE;
4858 }
4859 }
4860 return (void *)addr;
4861}
4862
4863
4864
4865
4866
4867
4868
4869
4870
4871
4872
4873
4874
4875
4876
4877
4878void *alloc_pages_exact(size_t size, gfp_t gfp_mask)
4879{
4880 unsigned int order = get_order(size);
4881 unsigned long addr;
4882
4883 if (WARN_ON_ONCE(gfp_mask & __GFP_COMP))
4884 gfp_mask &= ~__GFP_COMP;
4885
4886 addr = __get_free_pages(gfp_mask, order);
4887 return make_alloc_exact(addr, order, size);
4888}
4889EXPORT_SYMBOL(alloc_pages_exact);
4890
4891
4892
4893
4894
4895
4896
4897
4898
4899
4900
4901
4902
4903void * __meminit alloc_pages_exact_nid(int nid, size_t size, gfp_t gfp_mask)
4904{
4905 unsigned int order = get_order(size);
4906 struct page *p;
4907
4908 if (WARN_ON_ONCE(gfp_mask & __GFP_COMP))
4909 gfp_mask &= ~__GFP_COMP;
4910
4911 p = alloc_pages_node(nid, gfp_mask, order);
4912 if (!p)
4913 return NULL;
4914 return make_alloc_exact((unsigned long)page_address(p), order, size);
4915}
4916
4917
4918
4919
4920
4921
4922
4923
4924void free_pages_exact(void *virt, size_t size)
4925{
4926 unsigned long addr = (unsigned long)virt;
4927 unsigned long end = addr + PAGE_ALIGN(size);
4928
4929 while (addr < end) {
4930 free_page(addr);
4931 addr += PAGE_SIZE;
4932 }
4933}
4934EXPORT_SYMBOL(free_pages_exact);
4935
4936
4937
4938
4939
4940
4941
4942
4943
4944
4945
4946
4947
4948static unsigned long nr_free_zone_pages(int offset)
4949{
4950 struct zoneref *z;
4951 struct zone *zone;
4952
4953
4954 unsigned long sum = 0;
4955
4956 struct zonelist *zonelist = node_zonelist(numa_node_id(), GFP_KERNEL);
4957
4958 for_each_zone_zonelist(zone, z, zonelist, offset) {
4959 unsigned long size = zone_managed_pages(zone);
4960 unsigned long high = high_wmark_pages(zone);
4961 if (size > high)
4962 sum += size - high;
4963 }
4964
4965 return sum;
4966}
4967
4968
4969
4970
4971
4972
4973
4974
4975
4976
4977unsigned long nr_free_buffer_pages(void)
4978{
4979 return nr_free_zone_pages(gfp_zone(GFP_USER));
4980}
4981EXPORT_SYMBOL_GPL(nr_free_buffer_pages);
4982
4983
4984
4985
4986
4987
4988
4989
4990
4991unsigned long nr_free_pagecache_pages(void)
4992{
4993 return nr_free_zone_pages(gfp_zone(GFP_HIGHUSER_MOVABLE));
4994}
4995
4996static inline void show_node(struct zone *zone)
4997{
4998 if (IS_ENABLED(CONFIG_NUMA))
4999 printk("Node %d ", zone_to_nid(zone));
5000}
5001
5002long si_mem_available(void)
5003{
5004 long available;
5005 unsigned long pagecache;
5006 unsigned long wmark_low = 0;
5007 unsigned long pages[NR_LRU_LISTS];
5008 unsigned long reclaimable;
5009 struct zone *zone;
5010 int lru;
5011
5012 for (lru = LRU_BASE; lru < NR_LRU_LISTS; lru++)
5013 pages[lru] = global_node_page_state(NR_LRU_BASE + lru);
5014
5015 for_each_zone(zone)
5016 wmark_low += low_wmark_pages(zone);
5017
5018
5019
5020
5021
5022 available = global_zone_page_state(NR_FREE_PAGES) - totalreserve_pages;
5023
5024
5025
5026
5027
5028
5029 pagecache = pages[LRU_ACTIVE_FILE] + pages[LRU_INACTIVE_FILE];
5030 pagecache -= min(pagecache / 2, wmark_low);
5031 available += pagecache;
5032
5033
5034
5035
5036
5037
5038 reclaimable = global_node_page_state(NR_SLAB_RECLAIMABLE) +
5039 global_node_page_state(NR_KERNEL_MISC_RECLAIMABLE);
5040 available += reclaimable - min(reclaimable / 2, wmark_low);
5041
5042 if (available < 0)
5043 available = 0;
5044 return available;
5045}
5046EXPORT_SYMBOL_GPL(si_mem_available);
5047
5048void si_meminfo(struct sysinfo *val)
5049{
5050 val->totalram = totalram_pages();
5051 val->sharedram = global_node_page_state(NR_SHMEM);
5052 val->freeram = global_zone_page_state(NR_FREE_PAGES);
5053 val->bufferram = nr_blockdev_pages();
5054 val->totalhigh = totalhigh_pages();
5055 val->freehigh = nr_free_highpages();
5056 val->mem_unit = PAGE_SIZE;
5057}
5058
5059EXPORT_SYMBOL(si_meminfo);
5060
5061#ifdef CONFIG_NUMA
5062void si_meminfo_node(struct sysinfo *val, int nid)
5063{
5064 int zone_type;
5065 unsigned long managed_pages = 0;
5066 unsigned long managed_highpages = 0;
5067 unsigned long free_highpages = 0;
5068 pg_data_t *pgdat = NODE_DATA(nid);
5069
5070 for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++)
5071 managed_pages += zone_managed_pages(&pgdat->node_zones[zone_type]);
5072 val->totalram = managed_pages;
5073 val->sharedram = node_page_state(pgdat, NR_SHMEM);
5074 val->freeram = sum_zone_node_page_state(nid, NR_FREE_PAGES);
5075#ifdef CONFIG_HIGHMEM
5076 for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) {
5077 struct zone *zone = &pgdat->node_zones[zone_type];
5078
5079 if (is_highmem(zone)) {
5080 managed_highpages += zone_managed_pages(zone);
5081 free_highpages += zone_page_state(zone, NR_FREE_PAGES);
5082 }
5083 }
5084 val->totalhigh = managed_highpages;
5085 val->freehigh = free_highpages;
5086#else
5087 val->totalhigh = managed_highpages;
5088 val->freehigh = free_highpages;
5089#endif
5090 val->mem_unit = PAGE_SIZE;
5091}
5092#endif
5093
5094
5095
5096
5097
5098static bool show_mem_node_skip(unsigned int flags, int nid, nodemask_t *nodemask)
5099{
5100 if (!(flags & SHOW_MEM_FILTER_NODES))
5101 return false;
5102
5103
5104
5105
5106
5107
5108 if (!nodemask)
5109 nodemask = &cpuset_current_mems_allowed;
5110
5111 return !node_isset(nid, *nodemask);
5112}
5113
5114#define K(x) ((x) << (PAGE_SHIFT-10))
5115
5116static void show_migration_types(unsigned char type)
5117{
5118 static const char types[MIGRATE_TYPES] = {
5119 [MIGRATE_UNMOVABLE] = 'U',
5120 [MIGRATE_MOVABLE] = 'M',
5121 [MIGRATE_RECLAIMABLE] = 'E',
5122 [MIGRATE_HIGHATOMIC] = 'H',
5123#ifdef CONFIG_CMA
5124 [MIGRATE_CMA] = 'C',
5125#endif
5126#ifdef CONFIG_MEMORY_ISOLATION
5127 [MIGRATE_ISOLATE] = 'I',
5128#endif
5129 };
5130 char tmp[MIGRATE_TYPES + 1];
5131 char *p = tmp;
5132 int i;
5133
5134 for (i = 0; i < MIGRATE_TYPES; i++) {
5135 if (type & (1 << i))
5136 *p++ = types[i];
5137 }
5138
5139 *p = '\0';
5140 printk(KERN_CONT "(%s) ", tmp);
5141}
5142
5143
5144
5145
5146
5147
5148
5149
5150
5151
5152void show_free_areas(unsigned int filter, nodemask_t *nodemask)
5153{
5154 unsigned long free_pcp = 0;
5155 int cpu;
5156 struct zone *zone;
5157 pg_data_t *pgdat;
5158
5159 for_each_populated_zone(zone) {
5160 if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask))
5161 continue;
5162
5163 for_each_online_cpu(cpu)
5164 free_pcp += per_cpu_ptr(zone->pageset, cpu)->pcp.count;
5165 }
5166
5167 printk("active_anon:%lu inactive_anon:%lu isolated_anon:%lu\n"
5168 " active_file:%lu inactive_file:%lu isolated_file:%lu\n"
5169 " unevictable:%lu dirty:%lu writeback:%lu unstable:%lu\n"
5170 " slab_reclaimable:%lu slab_unreclaimable:%lu\n"
5171 " mapped:%lu shmem:%lu pagetables:%lu bounce:%lu\n"
5172 " free:%lu free_pcp:%lu free_cma:%lu\n",
5173 global_node_page_state(NR_ACTIVE_ANON),
5174 global_node_page_state(NR_INACTIVE_ANON),
5175 global_node_page_state(NR_ISOLATED_ANON),
5176 global_node_page_state(NR_ACTIVE_FILE),
5177 global_node_page_state(NR_INACTIVE_FILE),
5178 global_node_page_state(NR_ISOLATED_FILE),
5179 global_node_page_state(NR_UNEVICTABLE),
5180 global_node_page_state(NR_FILE_DIRTY),
5181 global_node_page_state(NR_WRITEBACK),
5182 global_node_page_state(NR_UNSTABLE_NFS),
5183 global_node_page_state(NR_SLAB_RECLAIMABLE),
5184 global_node_page_state(NR_SLAB_UNRECLAIMABLE),
5185 global_node_page_state(NR_FILE_MAPPED),
5186 global_node_page_state(NR_SHMEM),
5187 global_zone_page_state(NR_PAGETABLE),
5188 global_zone_page_state(NR_BOUNCE),
5189 global_zone_page_state(NR_FREE_PAGES),
5190 free_pcp,
5191 global_zone_page_state(NR_FREE_CMA_PAGES));
5192
5193 for_each_online_pgdat(pgdat) {
5194 if (show_mem_node_skip(filter, pgdat->node_id, nodemask))
5195 continue;
5196
5197 printk("Node %d"
5198 " active_anon:%lukB"
5199 " inactive_anon:%lukB"
5200 " active_file:%lukB"
5201 " inactive_file:%lukB"
5202 " unevictable:%lukB"
5203 " isolated(anon):%lukB"
5204 " isolated(file):%lukB"
5205 " mapped:%lukB"
5206 " dirty:%lukB"
5207 " writeback:%lukB"
5208 " shmem:%lukB"
5209#ifdef CONFIG_TRANSPARENT_HUGEPAGE
5210 " shmem_thp: %lukB"
5211 " shmem_pmdmapped: %lukB"
5212 " anon_thp: %lukB"
5213#endif
5214 " writeback_tmp:%lukB"
5215 " unstable:%lukB"
5216 " all_unreclaimable? %s"
5217 "\n",
5218 pgdat->node_id,
5219 K(node_page_state(pgdat, NR_ACTIVE_ANON)),
5220 K(node_page_state(pgdat, NR_INACTIVE_ANON)),
5221 K(node_page_state(pgdat, NR_ACTIVE_FILE)),
5222 K(node_page_state(pgdat, NR_INACTIVE_FILE)),
5223 K(node_page_state(pgdat, NR_UNEVICTABLE)),
5224 K(node_page_state(pgdat, NR_ISOLATED_ANON)),
5225 K(node_page_state(pgdat, NR_ISOLATED_FILE)),
5226 K(node_page_state(pgdat, NR_FILE_MAPPED)),
5227 K(node_page_state(pgdat, NR_FILE_DIRTY)),
5228 K(node_page_state(pgdat, NR_WRITEBACK)),
5229 K(node_page_state(pgdat, NR_SHMEM)),
5230#ifdef CONFIG_TRANSPARENT_HUGEPAGE
5231 K(node_page_state(pgdat, NR_SHMEM_THPS) * HPAGE_PMD_NR),
5232 K(node_page_state(pgdat, NR_SHMEM_PMDMAPPED)
5233 * HPAGE_PMD_NR),
5234 K(node_page_state(pgdat, NR_ANON_THPS) * HPAGE_PMD_NR),
5235#endif
5236 K(node_page_state(pgdat, NR_WRITEBACK_TEMP)),
5237 K(node_page_state(pgdat, NR_UNSTABLE_NFS)),
5238 pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES ?
5239 "yes" : "no");
5240 }
5241
5242 for_each_populated_zone(zone) {
5243 int i;
5244
5245 if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask))
5246 continue;
5247
5248 free_pcp = 0;
5249 for_each_online_cpu(cpu)
5250 free_pcp += per_cpu_ptr(zone->pageset, cpu)->pcp.count;
5251
5252 show_node(zone);
5253 printk(KERN_CONT
5254 "%s"
5255 " free:%lukB"
5256 " min:%lukB"
5257 " low:%lukB"
5258 " high:%lukB"
5259 " active_anon:%lukB"
5260 " inactive_anon:%lukB"
5261 " active_file:%lukB"
5262 " inactive_file:%lukB"
5263 " unevictable:%lukB"
5264 " writepending:%lukB"
5265 " present:%lukB"
5266 " managed:%lukB"
5267 " mlocked:%lukB"
5268 " kernel_stack:%lukB"
5269 " pagetables:%lukB"
5270 " bounce:%lukB"
5271 " free_pcp:%lukB"
5272 " local_pcp:%ukB"
5273 " free_cma:%lukB"
5274 "\n",
5275 zone->name,
5276 K(zone_page_state(zone, NR_FREE_PAGES)),
5277 K(min_wmark_pages(zone)),
5278 K(low_wmark_pages(zone)),
5279 K(high_wmark_pages(zone)),
5280 K(zone_page_state(zone, NR_ZONE_ACTIVE_ANON)),
5281 K(zone_page_state(zone, NR_ZONE_INACTIVE_ANON)),
5282 K(zone_page_state(zone, NR_ZONE_ACTIVE_FILE)),
5283 K(zone_page_state(zone, NR_ZONE_INACTIVE_FILE)),
5284 K(zone_page_state(zone, NR_ZONE_UNEVICTABLE)),
5285 K(zone_page_state(zone, NR_ZONE_WRITE_PENDING)),
5286 K(zone->present_pages),
5287 K(zone_managed_pages(zone)),
5288 K(zone_page_state(zone, NR_MLOCK)),
5289 zone_page_state(zone, NR_KERNEL_STACK_KB),
5290 K(zone_page_state(zone, NR_PAGETABLE)),
5291 K(zone_page_state(zone, NR_BOUNCE)),
5292 K(free_pcp),
5293 K(this_cpu_read(zone->pageset->pcp.count)),
5294 K(zone_page_state(zone, NR_FREE_CMA_PAGES)));
5295 printk("lowmem_reserve[]:");
5296 for (i = 0; i < MAX_NR_ZONES; i++)
5297 printk(KERN_CONT " %ld", zone->lowmem_reserve[i]);
5298 printk(KERN_CONT "\n");
5299 }
5300
5301 for_each_populated_zone(zone) {
5302 unsigned int order;
5303 unsigned long nr[MAX_ORDER], flags, total = 0;
5304 unsigned char types[MAX_ORDER];
5305
5306 if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask))
5307 continue;
5308 show_node(zone);
5309 printk(KERN_CONT "%s: ", zone->name);
5310
5311 spin_lock_irqsave(&zone->lock, flags);
5312 for (order = 0; order < MAX_ORDER; order++) {
5313 struct free_area *area = &zone->free_area[order];
5314 int type;
5315
5316 nr[order] = area->nr_free;
5317 total += nr[order] << order;
5318
5319 types[order] = 0;
5320 for (type = 0; type < MIGRATE_TYPES; type++) {
5321 if (!free_area_empty(area, type))
5322 types[order] |= 1 << type;
5323 }
5324 }
5325 spin_unlock_irqrestore(&zone->lock, flags);
5326 for (order = 0; order < MAX_ORDER; order++) {
5327 printk(KERN_CONT "%lu*%lukB ",
5328 nr[order], K(1UL) << order);
5329 if (nr[order])
5330 show_migration_types(types[order]);
5331 }
5332 printk(KERN_CONT "= %lukB\n", K(total));
5333 }
5334
5335 hugetlb_show_meminfo();
5336
5337 printk("%ld total pagecache pages\n", global_node_page_state(NR_FILE_PAGES));
5338
5339 show_swap_cache_info();
5340}
5341
5342static void zoneref_set_zone(struct zone *zone, struct zoneref *zoneref)
5343{
5344 zoneref->zone = zone;
5345 zoneref->zone_idx = zone_idx(zone);
5346}
5347
5348
5349
5350
5351
5352
5353static int build_zonerefs_node(pg_data_t *pgdat, struct zoneref *zonerefs)
5354{
5355 struct zone *zone;
5356 enum zone_type zone_type = MAX_NR_ZONES;
5357 int nr_zones = 0;
5358
5359 do {
5360 zone_type--;
5361 zone = pgdat->node_zones + zone_type;
5362 if (managed_zone(zone)) {
5363 zoneref_set_zone(zone, &zonerefs[nr_zones++]);
5364 check_highest_zone(zone_type);
5365 }
5366 } while (zone_type);
5367
5368 return nr_zones;
5369}
5370
5371#ifdef CONFIG_NUMA
5372
5373static int __parse_numa_zonelist_order(char *s)
5374{
5375
5376
5377
5378
5379
5380
5381 if (!(*s == 'd' || *s == 'D' || *s == 'n' || *s == 'N')) {
5382 pr_warn("Ignoring unsupported numa_zonelist_order value: %s\n", s);
5383 return -EINVAL;
5384 }
5385 return 0;
5386}
5387
5388static __init int setup_numa_zonelist_order(char *s)
5389{
5390 if (!s)
5391 return 0;
5392
5393 return __parse_numa_zonelist_order(s);
5394}
5395early_param("numa_zonelist_order", setup_numa_zonelist_order);
5396
5397char numa_zonelist_order[] = "Node";
5398
5399
5400
5401
5402int numa_zonelist_order_handler(struct ctl_table *table, int write,
5403 void __user *buffer, size_t *length,
5404 loff_t *ppos)
5405{
5406 char *str;
5407 int ret;
5408
5409 if (!write)
5410 return proc_dostring(table, write, buffer, length, ppos);
5411 str = memdup_user_nul(buffer, 16);
5412 if (IS_ERR(str))
5413 return PTR_ERR(str);
5414
5415 ret = __parse_numa_zonelist_order(str);
5416 kfree(str);
5417 return ret;
5418}
5419
5420
5421#define MAX_NODE_LOAD (nr_online_nodes)
5422static int node_load[MAX_NUMNODES];
5423
5424
5425
5426
5427
5428
5429
5430
5431
5432
5433
5434
5435
5436
5437
5438
5439static int find_next_best_node(int node, nodemask_t *used_node_mask)
5440{
5441 int n, val;
5442 int min_val = INT_MAX;
5443 int best_node = NUMA_NO_NODE;
5444 const struct cpumask *tmp = cpumask_of_node(0);
5445
5446
5447 if (!node_isset(node, *used_node_mask)) {
5448 node_set(node, *used_node_mask);
5449 return node;
5450 }
5451
5452 for_each_node_state(n, N_MEMORY) {
5453
5454
5455 if (node_isset(n, *used_node_mask))
5456 continue;
5457
5458
5459 val = node_distance(node, n);
5460
5461
5462 val += (n < node);
5463
5464
5465 tmp = cpumask_of_node(n);
5466 if (!cpumask_empty(tmp))
5467 val += PENALTY_FOR_NODE_WITH_CPUS;
5468
5469
5470 val *= (MAX_NODE_LOAD*MAX_NUMNODES);
5471 val += node_load[n];
5472
5473 if (val < min_val) {
5474 min_val = val;
5475 best_node = n;
5476 }
5477 }
5478
5479 if (best_node >= 0)
5480 node_set(best_node, *used_node_mask);
5481
5482 return best_node;
5483}
5484
5485
5486
5487
5488
5489
5490
5491static void build_zonelists_in_node_order(pg_data_t *pgdat, int *node_order,
5492 unsigned nr_nodes)
5493{
5494 struct zoneref *zonerefs;
5495 int i;
5496
5497 zonerefs = pgdat->node_zonelists[ZONELIST_FALLBACK]._zonerefs;
5498
5499 for (i = 0; i < nr_nodes; i++) {
5500 int nr_zones;
5501
5502 pg_data_t *node = NODE_DATA(node_order[i]);
5503
5504 nr_zones = build_zonerefs_node(node, zonerefs);
5505 zonerefs += nr_zones;
5506 }
5507 zonerefs->zone = NULL;
5508 zonerefs->zone_idx = 0;
5509}
5510
5511
5512
5513
5514static void build_thisnode_zonelists(pg_data_t *pgdat)
5515{
5516 struct zoneref *zonerefs;
5517 int nr_zones;
5518
5519 zonerefs = pgdat->node_zonelists[ZONELIST_NOFALLBACK]._zonerefs;
5520 nr_zones = build_zonerefs_node(pgdat, zonerefs);
5521 zonerefs += nr_zones;
5522 zonerefs->zone = NULL;
5523 zonerefs->zone_idx = 0;
5524}
5525
5526
5527
5528
5529
5530
5531
5532
5533static void build_zonelists(pg_data_t *pgdat)
5534{
5535 static int node_order[MAX_NUMNODES];
5536 int node, load, nr_nodes = 0;
5537 nodemask_t used_mask;
5538 int local_node, prev_node;
5539
5540
5541 local_node = pgdat->node_id;
5542 load = nr_online_nodes;
5543 prev_node = local_node;
5544 nodes_clear(used_mask);
5545
5546 memset(node_order, 0, sizeof(node_order));
5547 while ((node = find_next_best_node(local_node, &used_mask)) >= 0) {
5548
5549
5550
5551
5552
5553 if (node_distance(local_node, node) !=
5554 node_distance(local_node, prev_node))
5555 node_load[node] = load;
5556
5557 node_order[nr_nodes++] = node;
5558 prev_node = node;
5559 load--;
5560 }
5561
5562 build_zonelists_in_node_order(pgdat, node_order, nr_nodes);
5563 build_thisnode_zonelists(pgdat);
5564}
5565
5566#ifdef CONFIG_HAVE_MEMORYLESS_NODES
5567
5568
5569
5570
5571
5572
5573int local_memory_node(int node)
5574{
5575 struct zoneref *z;
5576
5577 z = first_zones_zonelist(node_zonelist(node, GFP_KERNEL),
5578 gfp_zone(GFP_KERNEL),
5579 NULL);
5580 return zone_to_nid(z->zone);
5581}
5582#endif
5583
5584static void setup_min_unmapped_ratio(void);
5585static void setup_min_slab_ratio(void);
5586#else
5587
5588static void build_zonelists(pg_data_t *pgdat)
5589{
5590 int node, local_node;
5591 struct zoneref *zonerefs;
5592 int nr_zones;
5593
5594 local_node = pgdat->node_id;
5595
5596 zonerefs = pgdat->node_zonelists[ZONELIST_FALLBACK]._zonerefs;
5597 nr_zones = build_zonerefs_node(pgdat, zonerefs);
5598 zonerefs += nr_zones;
5599
5600
5601
5602
5603
5604
5605
5606
5607
5608 for (node = local_node + 1; node < MAX_NUMNODES; node++) {
5609 if (!node_online(node))
5610 continue;
5611 nr_zones = build_zonerefs_node(NODE_DATA(node), zonerefs);
5612 zonerefs += nr_zones;
5613 }
5614 for (node = 0; node < local_node; node++) {
5615 if (!node_online(node))
5616 continue;
5617 nr_zones = build_zonerefs_node(NODE_DATA(node), zonerefs);
5618 zonerefs += nr_zones;
5619 }
5620
5621 zonerefs->zone = NULL;
5622 zonerefs->zone_idx = 0;
5623}
5624
5625#endif
5626
5627
5628
5629
5630
5631
5632
5633
5634
5635
5636
5637
5638
5639
5640
5641
5642static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch);
5643static DEFINE_PER_CPU(struct per_cpu_pageset, boot_pageset);
5644static DEFINE_PER_CPU(struct per_cpu_nodestat, boot_nodestats);
5645
5646static void __build_all_zonelists(void *data)
5647{
5648 int nid;
5649 int __maybe_unused cpu;
5650 pg_data_t *self = data;
5651 static DEFINE_SPINLOCK(lock);
5652
5653 spin_lock(&lock);
5654
5655#ifdef CONFIG_NUMA
5656 memset(node_load, 0, sizeof(node_load));
5657#endif
5658
5659
5660
5661
5662
5663 if (self && !node_online(self->node_id)) {
5664 build_zonelists(self);
5665 } else {
5666 for_each_online_node(nid) {
5667 pg_data_t *pgdat = NODE_DATA(nid);
5668
5669 build_zonelists(pgdat);
5670 }
5671
5672#ifdef CONFIG_HAVE_MEMORYLESS_NODES
5673
5674
5675
5676
5677
5678
5679
5680
5681 for_each_online_cpu(cpu)
5682 set_cpu_numa_mem(cpu, local_memory_node(cpu_to_node(cpu)));
5683#endif
5684 }
5685
5686 spin_unlock(&lock);
5687}
5688
5689static noinline void __init
5690build_all_zonelists_init(void)
5691{
5692 int cpu;
5693
5694 __build_all_zonelists(NULL);
5695
5696
5697
5698
5699
5700
5701
5702
5703
5704
5705
5706
5707
5708
5709 for_each_possible_cpu(cpu)
5710 setup_pageset(&per_cpu(boot_pageset, cpu), 0);
5711
5712 mminit_verify_zonelist();
5713 cpuset_init_current_mems_allowed();
5714}
5715
5716
5717
5718
5719
5720
5721
5722void __ref build_all_zonelists(pg_data_t *pgdat)
5723{
5724 if (system_state == SYSTEM_BOOTING) {
5725 build_all_zonelists_init();
5726 } else {
5727 __build_all_zonelists(pgdat);
5728
5729 }
5730 vm_total_pages = nr_free_pagecache_pages();
5731
5732
5733
5734
5735
5736
5737
5738 if (vm_total_pages < (pageblock_nr_pages * MIGRATE_TYPES))
5739 page_group_by_mobility_disabled = 1;
5740 else
5741 page_group_by_mobility_disabled = 0;
5742
5743 pr_info("Built %u zonelists, mobility grouping %s. Total pages: %ld\n",
5744 nr_online_nodes,
5745 page_group_by_mobility_disabled ? "off" : "on",
5746 vm_total_pages);
5747#ifdef CONFIG_NUMA
5748 pr_info("Policy zone: %s\n", zone_names[policy_zone]);
5749#endif
5750}
5751
5752
5753static bool __meminit
5754overlap_memmap_init(unsigned long zone, unsigned long *pfn)
5755{
5756#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
5757 static struct memblock_region *r;
5758
5759 if (mirrored_kernelcore && zone == ZONE_MOVABLE) {
5760 if (!r || *pfn >= memblock_region_memory_end_pfn(r)) {
5761 for_each_memblock(memory, r) {
5762 if (*pfn < memblock_region_memory_end_pfn(r))
5763 break;
5764 }
5765 }
5766 if (*pfn >= memblock_region_memory_base_pfn(r) &&
5767 memblock_is_mirror(r)) {
5768 *pfn = memblock_region_memory_end_pfn(r);
5769 return true;
5770 }
5771 }
5772#endif
5773 return false;
5774}
5775
5776
5777
5778
5779
5780
5781void __meminit memmap_init_zone(unsigned long size, int nid, unsigned long zone,
5782 unsigned long start_pfn, enum memmap_context context,
5783 struct vmem_altmap *altmap)
5784{
5785 unsigned long pfn, end_pfn = start_pfn + size;
5786 struct page *page;
5787
5788 if (highest_memmap_pfn < end_pfn - 1)
5789 highest_memmap_pfn = end_pfn - 1;
5790
5791#ifdef CONFIG_ZONE_DEVICE
5792
5793
5794
5795
5796
5797
5798
5799 if (zone == ZONE_DEVICE) {
5800 if (!altmap)
5801 return;
5802
5803 if (start_pfn == altmap->base_pfn)
5804 start_pfn += altmap->reserve;
5805 end_pfn = altmap->base_pfn + vmem_altmap_offset(altmap);
5806 }
5807#endif
5808
5809 for (pfn = start_pfn; pfn < end_pfn; pfn++) {
5810
5811
5812
5813
5814 if (context == MEMMAP_EARLY) {
5815 if (!early_pfn_valid(pfn))
5816 continue;
5817 if (!early_pfn_in_nid(pfn, nid))
5818 continue;
5819 if (overlap_memmap_init(zone, &pfn))
5820 continue;
5821 if (defer_init(nid, pfn, end_pfn))
5822 break;
5823 }
5824
5825 page = pfn_to_page(pfn);
5826 __init_single_page(page, pfn, zone, nid);
5827 if (context == MEMMAP_HOTPLUG)
5828 __SetPageReserved(page);
5829
5830
5831
5832
5833
5834
5835
5836
5837
5838
5839
5840
5841
5842 if (!(pfn & (pageblock_nr_pages - 1))) {
5843 set_pageblock_migratetype(page, MIGRATE_MOVABLE);
5844 cond_resched();
5845 }
5846 }
5847}
5848
5849#ifdef CONFIG_ZONE_DEVICE
5850void __ref memmap_init_zone_device(struct zone *zone,
5851 unsigned long start_pfn,
5852 unsigned long size,
5853 struct dev_pagemap *pgmap)
5854{
5855 unsigned long pfn, end_pfn = start_pfn + size;
5856 struct pglist_data *pgdat = zone->zone_pgdat;
5857 unsigned long zone_idx = zone_idx(zone);
5858 unsigned long start = jiffies;
5859 int nid = pgdat->node_id;
5860
5861 if (WARN_ON_ONCE(!pgmap || !is_dev_zone(zone)))
5862 return;
5863
5864
5865
5866
5867
5868
5869 if (pgmap->altmap_valid) {
5870 struct vmem_altmap *altmap = &pgmap->altmap;
5871
5872 start_pfn = altmap->base_pfn + vmem_altmap_offset(altmap);
5873 size = end_pfn - start_pfn;
5874 }
5875
5876 for (pfn = start_pfn; pfn < end_pfn; pfn++) {
5877 struct page *page = pfn_to_page(pfn);
5878
5879 __init_single_page(page, pfn, zone_idx, nid);
5880
5881
5882
5883
5884
5885
5886
5887
5888 __SetPageReserved(page);
5889
5890
5891
5892
5893
5894
5895 page->pgmap = pgmap;
5896 page->hmm_data = 0;
5897
5898
5899
5900
5901
5902
5903
5904
5905
5906
5907
5908
5909
5910
5911
5912
5913 if (!(pfn & (pageblock_nr_pages - 1))) {
5914 set_pageblock_migratetype(page, MIGRATE_MOVABLE);
5915 cond_resched();
5916 }
5917 }
5918
5919 pr_info("%s initialised, %lu pages in %ums\n", dev_name(pgmap->dev),
5920 size, jiffies_to_msecs(jiffies - start));
5921}
5922
5923#endif
5924static void __meminit zone_init_free_lists(struct zone *zone)
5925{
5926 unsigned int order, t;
5927 for_each_migratetype_order(order, t) {
5928 INIT_LIST_HEAD(&zone->free_area[order].free_list[t]);
5929 zone->free_area[order].nr_free = 0;
5930 }
5931}
5932
5933void __meminit __weak memmap_init(unsigned long size, int nid,
5934 unsigned long zone, unsigned long start_pfn)
5935{
5936 memmap_init_zone(size, nid, zone, start_pfn, MEMMAP_EARLY, NULL);
5937}
5938
5939static int zone_batchsize(struct zone *zone)
5940{
5941#ifdef CONFIG_MMU
5942 int batch;
5943
5944
5945
5946
5947
5948 batch = zone_managed_pages(zone) / 1024;
5949
5950 if (batch * PAGE_SIZE > 1024 * 1024)
5951 batch = (1024 * 1024) / PAGE_SIZE;
5952 batch /= 4;
5953 if (batch < 1)
5954 batch = 1;
5955
5956
5957
5958
5959
5960
5961
5962
5963
5964
5965
5966 batch = rounddown_pow_of_two(batch + batch/2) - 1;
5967
5968 return batch;
5969
5970#else
5971
5972
5973
5974
5975
5976
5977
5978
5979
5980
5981
5982
5983
5984 return 0;
5985#endif
5986}
5987
5988
5989
5990
5991
5992
5993
5994
5995
5996
5997
5998
5999
6000
6001static void pageset_update(struct per_cpu_pages *pcp, unsigned long high,
6002 unsigned long batch)
6003{
6004
6005 pcp->batch = 1;
6006 smp_wmb();
6007
6008
6009 pcp->high = high;
6010 smp_wmb();
6011
6012 pcp->batch = batch;
6013}
6014
6015
6016static void pageset_set_batch(struct per_cpu_pageset *p, unsigned long batch)
6017{
6018 pageset_update(&p->pcp, 6 * batch, max(1UL, 1 * batch));
6019}
6020
6021static void pageset_init(struct per_cpu_pageset *p)
6022{
6023 struct per_cpu_pages *pcp;
6024 int migratetype;
6025
6026 memset(p, 0, sizeof(*p));
6027
6028 pcp = &p->pcp;
6029 for (migratetype = 0; migratetype < MIGRATE_PCPTYPES; migratetype++)
6030 INIT_LIST_HEAD(&pcp->lists[migratetype]);
6031}
6032
6033static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch)
6034{
6035 pageset_init(p);
6036 pageset_set_batch(p, batch);
6037}
6038
6039
6040
6041
6042
6043static void pageset_set_high(struct per_cpu_pageset *p,
6044 unsigned long high)
6045{
6046 unsigned long batch = max(1UL, high / 4);
6047 if ((high / 4) > (PAGE_SHIFT * 8))
6048 batch = PAGE_SHIFT * 8;
6049
6050 pageset_update(&p->pcp, high, batch);
6051}
6052
6053static void pageset_set_high_and_batch(struct zone *zone,
6054 struct per_cpu_pageset *pcp)
6055{
6056 if (percpu_pagelist_fraction)
6057 pageset_set_high(pcp,
6058 (zone_managed_pages(zone) /
6059 percpu_pagelist_fraction));
6060 else
6061 pageset_set_batch(pcp, zone_batchsize(zone));
6062}
6063
6064static void __meminit zone_pageset_init(struct zone *zone, int cpu)
6065{
6066 struct per_cpu_pageset *pcp = per_cpu_ptr(zone->pageset, cpu);
6067
6068 pageset_init(pcp);
6069 pageset_set_high_and_batch(zone, pcp);
6070}
6071
6072void __meminit setup_zone_pageset(struct zone *zone)
6073{
6074 int cpu;
6075 zone->pageset = alloc_percpu(struct per_cpu_pageset);
6076 for_each_possible_cpu(cpu)
6077 zone_pageset_init(zone, cpu);
6078}
6079
6080
6081
6082
6083
6084void __init setup_per_cpu_pageset(void)
6085{
6086 struct pglist_data *pgdat;
6087 struct zone *zone;
6088
6089 for_each_populated_zone(zone)
6090 setup_zone_pageset(zone);
6091
6092 for_each_online_pgdat(pgdat)
6093 pgdat->per_cpu_nodestats =
6094 alloc_percpu(struct per_cpu_nodestat);
6095}
6096
6097static __meminit void zone_pcp_init(struct zone *zone)
6098{
6099
6100
6101
6102
6103
6104 zone->pageset = &boot_pageset;
6105
6106 if (populated_zone(zone))
6107 printk(KERN_DEBUG " %s zone: %lu pages, LIFO batch:%u\n",
6108 zone->name, zone->present_pages,
6109 zone_batchsize(zone));
6110}
6111
6112void __meminit init_currently_empty_zone(struct zone *zone,
6113 unsigned long zone_start_pfn,
6114 unsigned long size)
6115{
6116 struct pglist_data *pgdat = zone->zone_pgdat;
6117 int zone_idx = zone_idx(zone) + 1;
6118
6119 if (zone_idx > pgdat->nr_zones)
6120 pgdat->nr_zones = zone_idx;
6121
6122 zone->zone_start_pfn = zone_start_pfn;
6123
6124 mminit_dprintk(MMINIT_TRACE, "memmap_init",
6125 "Initialising map node %d zone %lu pfns %lu -> %lu\n",
6126 pgdat->node_id,
6127 (unsigned long)zone_idx(zone),
6128 zone_start_pfn, (zone_start_pfn + size));
6129
6130 zone_init_free_lists(zone);
6131 zone->initialized = 1;
6132}
6133
6134#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
6135#ifndef CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID
6136
6137
6138
6139
6140int __meminit __early_pfn_to_nid(unsigned long pfn,
6141 struct mminit_pfnnid_cache *state)
6142{
6143 unsigned long start_pfn, end_pfn;
6144 int nid;
6145
6146 if (state->last_start <= pfn && pfn < state->last_end)
6147 return state->last_nid;
6148
6149 nid = memblock_search_pfn_nid(pfn, &start_pfn, &end_pfn);
6150 if (nid != NUMA_NO_NODE) {
6151 state->last_start = start_pfn;
6152 state->last_end = end_pfn;
6153 state->last_nid = nid;
6154 }
6155
6156 return nid;
6157}
6158#endif
6159
6160
6161
6162
6163
6164
6165
6166
6167
6168
6169void __init free_bootmem_with_active_regions(int nid, unsigned long max_low_pfn)
6170{
6171 unsigned long start_pfn, end_pfn;
6172 int i, this_nid;
6173
6174 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, &this_nid) {
6175 start_pfn = min(start_pfn, max_low_pfn);
6176 end_pfn = min(end_pfn, max_low_pfn);
6177
6178 if (start_pfn < end_pfn)
6179 memblock_free_early_nid(PFN_PHYS(start_pfn),
6180 (end_pfn - start_pfn) << PAGE_SHIFT,
6181 this_nid);
6182 }
6183}
6184
6185
6186
6187
6188
6189
6190
6191
6192void __init sparse_memory_present_with_active_regions(int nid)
6193{
6194 unsigned long start_pfn, end_pfn;
6195 int i, this_nid;
6196
6197 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, &this_nid)
6198 memory_present(this_nid, start_pfn, end_pfn);
6199}
6200
6201
6202
6203
6204
6205
6206
6207
6208
6209
6210
6211
6212void __init get_pfn_range_for_nid(unsigned int nid,
6213 unsigned long *start_pfn, unsigned long *end_pfn)
6214{
6215 unsigned long this_start_pfn, this_end_pfn;
6216 int i;
6217
6218 *start_pfn = -1UL;
6219 *end_pfn = 0;
6220
6221 for_each_mem_pfn_range(i, nid, &this_start_pfn, &this_end_pfn, NULL) {
6222 *start_pfn = min(*start_pfn, this_start_pfn);
6223 *end_pfn = max(*end_pfn, this_end_pfn);
6224 }
6225
6226 if (*start_pfn == -1UL)
6227 *start_pfn = 0;
6228}
6229
6230
6231
6232
6233
6234
6235static void __init find_usable_zone_for_movable(void)
6236{
6237 int zone_index;
6238 for (zone_index = MAX_NR_ZONES - 1; zone_index >= 0; zone_index--) {
6239 if (zone_index == ZONE_MOVABLE)
6240 continue;
6241
6242 if (arch_zone_highest_possible_pfn[zone_index] >
6243 arch_zone_lowest_possible_pfn[zone_index])
6244 break;
6245 }
6246
6247 VM_BUG_ON(zone_index == -1);
6248 movable_zone = zone_index;
6249}
6250
6251
6252
6253
6254
6255
6256
6257
6258
6259
6260
6261static void __init adjust_zone_range_for_zone_movable(int nid,
6262 unsigned long zone_type,
6263 unsigned long node_start_pfn,
6264 unsigned long node_end_pfn,
6265 unsigned long *zone_start_pfn,
6266 unsigned long *zone_end_pfn)
6267{
6268
6269 if (zone_movable_pfn[nid]) {
6270
6271 if (zone_type == ZONE_MOVABLE) {
6272 *zone_start_pfn = zone_movable_pfn[nid];
6273 *zone_end_pfn = min(node_end_pfn,
6274 arch_zone_highest_possible_pfn[movable_zone]);
6275
6276
6277 } else if (!mirrored_kernelcore &&
6278 *zone_start_pfn < zone_movable_pfn[nid] &&
6279 *zone_end_pfn > zone_movable_pfn[nid]) {
6280 *zone_end_pfn = zone_movable_pfn[nid];
6281
6282
6283 } else if (*zone_start_pfn >= zone_movable_pfn[nid])
6284 *zone_start_pfn = *zone_end_pfn;
6285 }
6286}
6287
6288
6289
6290
6291
6292static unsigned long __init zone_spanned_pages_in_node(int nid,
6293 unsigned long zone_type,
6294 unsigned long node_start_pfn,
6295 unsigned long node_end_pfn,
6296 unsigned long *zone_start_pfn,
6297 unsigned long *zone_end_pfn,
6298 unsigned long *ignored)
6299{
6300 unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type];
6301 unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type];
6302
6303 if (!node_start_pfn && !node_end_pfn)
6304 return 0;
6305
6306
6307 *zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high);
6308 *zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high);
6309 adjust_zone_range_for_zone_movable(nid, zone_type,
6310 node_start_pfn, node_end_pfn,
6311 zone_start_pfn, zone_end_pfn);
6312
6313
6314 if (*zone_end_pfn < node_start_pfn || *zone_start_pfn > node_end_pfn)
6315 return 0;
6316
6317
6318 *zone_end_pfn = min(*zone_end_pfn, node_end_pfn);
6319 *zone_start_pfn = max(*zone_start_pfn, node_start_pfn);
6320
6321
6322 return *zone_end_pfn - *zone_start_pfn;
6323}
6324
6325
6326
6327
6328
6329unsigned long __init __absent_pages_in_range(int nid,
6330 unsigned long range_start_pfn,
6331 unsigned long range_end_pfn)
6332{
6333 unsigned long nr_absent = range_end_pfn - range_start_pfn;
6334 unsigned long start_pfn, end_pfn;
6335 int i;
6336
6337 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
6338 start_pfn = clamp(start_pfn, range_start_pfn, range_end_pfn);
6339 end_pfn = clamp(end_pfn, range_start_pfn, range_end_pfn);
6340 nr_absent -= end_pfn - start_pfn;
6341 }
6342 return nr_absent;
6343}
6344
6345
6346
6347
6348
6349
6350
6351
6352unsigned long __init absent_pages_in_range(unsigned long start_pfn,
6353 unsigned long end_pfn)
6354{
6355 return __absent_pages_in_range(MAX_NUMNODES, start_pfn, end_pfn);
6356}
6357
6358
6359static unsigned long __init zone_absent_pages_in_node(int nid,
6360 unsigned long zone_type,
6361 unsigned long node_start_pfn,
6362 unsigned long node_end_pfn,
6363 unsigned long *ignored)
6364{
6365 unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type];
6366 unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type];
6367 unsigned long zone_start_pfn, zone_end_pfn;
6368 unsigned long nr_absent;
6369
6370
6371 if (!node_start_pfn && !node_end_pfn)
6372 return 0;
6373
6374 zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high);
6375 zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high);
6376
6377 adjust_zone_range_for_zone_movable(nid, zone_type,
6378 node_start_pfn, node_end_pfn,
6379 &zone_start_pfn, &zone_end_pfn);
6380 nr_absent = __absent_pages_in_range(nid, zone_start_pfn, zone_end_pfn);
6381
6382
6383
6384
6385
6386
6387 if (mirrored_kernelcore && zone_movable_pfn[nid]) {
6388 unsigned long start_pfn, end_pfn;
6389 struct memblock_region *r;
6390
6391 for_each_memblock(memory, r) {
6392 start_pfn = clamp(memblock_region_memory_base_pfn(r),
6393 zone_start_pfn, zone_end_pfn);
6394 end_pfn = clamp(memblock_region_memory_end_pfn(r),
6395 zone_start_pfn, zone_end_pfn);
6396
6397 if (zone_type == ZONE_MOVABLE &&
6398 memblock_is_mirror(r))
6399 nr_absent += end_pfn - start_pfn;
6400
6401 if (zone_type == ZONE_NORMAL &&
6402 !memblock_is_mirror(r))
6403 nr_absent += end_pfn - start_pfn;
6404 }
6405 }
6406
6407 return nr_absent;
6408}
6409
6410#else
6411static inline unsigned long __init zone_spanned_pages_in_node(int nid,
6412 unsigned long zone_type,
6413 unsigned long node_start_pfn,
6414 unsigned long node_end_pfn,
6415 unsigned long *zone_start_pfn,
6416 unsigned long *zone_end_pfn,
6417 unsigned long *zones_size)
6418{
6419 unsigned int zone;
6420
6421 *zone_start_pfn = node_start_pfn;
6422 for (zone = 0; zone < zone_type; zone++)
6423 *zone_start_pfn += zones_size[zone];
6424
6425 *zone_end_pfn = *zone_start_pfn + zones_size[zone_type];
6426
6427 return zones_size[zone_type];
6428}
6429
6430static inline unsigned long __init zone_absent_pages_in_node(int nid,
6431 unsigned long zone_type,
6432 unsigned long node_start_pfn,
6433 unsigned long node_end_pfn,
6434 unsigned long *zholes_size)
6435{
6436 if (!zholes_size)
6437 return 0;
6438
6439 return zholes_size[zone_type];
6440}
6441
6442#endif
6443
6444static void __init calculate_node_totalpages(struct pglist_data *pgdat,
6445 unsigned long node_start_pfn,
6446 unsigned long node_end_pfn,
6447 unsigned long *zones_size,
6448 unsigned long *zholes_size)
6449{
6450 unsigned long realtotalpages = 0, totalpages = 0;
6451 enum zone_type i;
6452
6453 for (i = 0; i < MAX_NR_ZONES; i++) {
6454 struct zone *zone = pgdat->node_zones + i;
6455 unsigned long zone_start_pfn, zone_end_pfn;
6456 unsigned long size, real_size;
6457
6458 size = zone_spanned_pages_in_node(pgdat->node_id, i,
6459 node_start_pfn,
6460 node_end_pfn,
6461 &zone_start_pfn,
6462 &zone_end_pfn,
6463 zones_size);
6464 real_size = size - zone_absent_pages_in_node(pgdat->node_id, i,
6465 node_start_pfn, node_end_pfn,
6466 zholes_size);
6467 if (size)
6468 zone->zone_start_pfn = zone_start_pfn;
6469 else
6470 zone->zone_start_pfn = 0;
6471 zone->spanned_pages = size;
6472 zone->present_pages = real_size;
6473
6474 totalpages += size;
6475 realtotalpages += real_size;
6476 }
6477
6478 pgdat->node_spanned_pages = totalpages;
6479 pgdat->node_present_pages = realtotalpages;
6480 printk(KERN_DEBUG "On node %d totalpages: %lu\n", pgdat->node_id,
6481 realtotalpages);
6482}
6483
6484#ifndef CONFIG_SPARSEMEM
6485
6486
6487
6488
6489
6490
6491
6492static unsigned long __init usemap_size(unsigned long zone_start_pfn, unsigned long zonesize)
6493{
6494 unsigned long usemapsize;
6495
6496 zonesize += zone_start_pfn & (pageblock_nr_pages-1);
6497 usemapsize = roundup(zonesize, pageblock_nr_pages);
6498 usemapsize = usemapsize >> pageblock_order;
6499 usemapsize *= NR_PAGEBLOCK_BITS;
6500 usemapsize = roundup(usemapsize, 8 * sizeof(unsigned long));
6501
6502 return usemapsize / 8;
6503}
6504
6505static void __ref setup_usemap(struct pglist_data *pgdat,
6506 struct zone *zone,
6507 unsigned long zone_start_pfn,
6508 unsigned long zonesize)
6509{
6510 unsigned long usemapsize = usemap_size(zone_start_pfn, zonesize);
6511 zone->pageblock_flags = NULL;
6512 if (usemapsize) {
6513 zone->pageblock_flags =
6514 memblock_alloc_node(usemapsize, SMP_CACHE_BYTES,
6515 pgdat->node_id);
6516 if (!zone->pageblock_flags)
6517 panic("Failed to allocate %ld bytes for zone %s pageblock flags on node %d\n",
6518 usemapsize, zone->name, pgdat->node_id);
6519 }
6520}
6521#else
6522static inline void setup_usemap(struct pglist_data *pgdat, struct zone *zone,
6523 unsigned long zone_start_pfn, unsigned long zonesize) {}
6524#endif
6525
6526#ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE
6527
6528
6529void __init set_pageblock_order(void)
6530{
6531 unsigned int order;
6532
6533
6534 if (pageblock_order)
6535 return;
6536
6537 if (HPAGE_SHIFT > PAGE_SHIFT)
6538 order = HUGETLB_PAGE_ORDER;
6539 else
6540 order = MAX_ORDER - 1;
6541
6542
6543
6544
6545
6546
6547 pageblock_order = order;
6548}
6549#else
6550
6551
6552
6553
6554
6555
6556
6557void __init set_pageblock_order(void)
6558{
6559}
6560
6561#endif
6562
6563static unsigned long __init calc_memmap_size(unsigned long spanned_pages,
6564 unsigned long present_pages)
6565{
6566 unsigned long pages = spanned_pages;
6567
6568
6569
6570
6571
6572
6573
6574
6575
6576 if (spanned_pages > present_pages + (present_pages >> 4) &&
6577 IS_ENABLED(CONFIG_SPARSEMEM))
6578 pages = present_pages;
6579
6580 return PAGE_ALIGN(pages * sizeof(struct page)) >> PAGE_SHIFT;
6581}
6582
6583#ifdef CONFIG_TRANSPARENT_HUGEPAGE
6584static void pgdat_init_split_queue(struct pglist_data *pgdat)
6585{
6586 spin_lock_init(&pgdat->split_queue_lock);
6587 INIT_LIST_HEAD(&pgdat->split_queue);
6588 pgdat->split_queue_len = 0;
6589}
6590#else
6591static void pgdat_init_split_queue(struct pglist_data *pgdat) {}
6592#endif
6593
6594#ifdef CONFIG_COMPACTION
6595static void pgdat_init_kcompactd(struct pglist_data *pgdat)
6596{
6597 init_waitqueue_head(&pgdat->kcompactd_wait);
6598}
6599#else
6600static void pgdat_init_kcompactd(struct pglist_data *pgdat) {}
6601#endif
6602
6603static void __meminit pgdat_init_internals(struct pglist_data *pgdat)
6604{
6605 pgdat_resize_init(pgdat);
6606
6607 pgdat_init_split_queue(pgdat);
6608 pgdat_init_kcompactd(pgdat);
6609
6610 init_waitqueue_head(&pgdat->kswapd_wait);
6611 init_waitqueue_head(&pgdat->pfmemalloc_wait);
6612
6613 pgdat_page_ext_init(pgdat);
6614 spin_lock_init(&pgdat->lru_lock);
6615 lruvec_init(node_lruvec(pgdat));
6616}
6617
6618static void __meminit zone_init_internals(struct zone *zone, enum zone_type idx, int nid,
6619 unsigned long remaining_pages)
6620{
6621 atomic_long_set(&zone->managed_pages, remaining_pages);
6622 zone_set_nid(zone, nid);
6623 zone->name = zone_names[idx];
6624 zone->zone_pgdat = NODE_DATA(nid);
6625 spin_lock_init(&zone->lock);
6626 zone_seqlock_init(zone);
6627 zone_pcp_init(zone);
6628}
6629
6630
6631
6632
6633
6634
6635
6636
6637#ifdef CONFIG_MEMORY_HOTPLUG
6638void __ref free_area_init_core_hotplug(int nid)
6639{
6640 enum zone_type z;
6641 pg_data_t *pgdat = NODE_DATA(nid);
6642
6643 pgdat_init_internals(pgdat);
6644 for (z = 0; z < MAX_NR_ZONES; z++)
6645 zone_init_internals(&pgdat->node_zones[z], z, nid, 0);
6646}
6647#endif
6648
6649
6650
6651
6652
6653
6654
6655
6656
6657
6658static void __init free_area_init_core(struct pglist_data *pgdat)
6659{
6660 enum zone_type j;
6661 int nid = pgdat->node_id;
6662
6663 pgdat_init_internals(pgdat);
6664 pgdat->per_cpu_nodestats = &boot_nodestats;
6665
6666 for (j = 0; j < MAX_NR_ZONES; j++) {
6667 struct zone *zone = pgdat->node_zones + j;
6668 unsigned long size, freesize, memmap_pages;
6669 unsigned long zone_start_pfn = zone->zone_start_pfn;
6670
6671 size = zone->spanned_pages;
6672 freesize = zone->present_pages;
6673
6674
6675
6676
6677
6678
6679 memmap_pages = calc_memmap_size(size, freesize);
6680 if (!is_highmem_idx(j)) {
6681 if (freesize >= memmap_pages) {
6682 freesize -= memmap_pages;
6683 if (memmap_pages)
6684 printk(KERN_DEBUG
6685 " %s zone: %lu pages used for memmap\n",
6686 zone_names[j], memmap_pages);
6687 } else
6688 pr_warn(" %s zone: %lu pages exceeds freesize %lu\n",
6689 zone_names[j], memmap_pages, freesize);
6690 }
6691
6692
6693 if (j == 0 && freesize > dma_reserve) {
6694 freesize -= dma_reserve;
6695 printk(KERN_DEBUG " %s zone: %lu pages reserved\n",
6696 zone_names[0], dma_reserve);
6697 }
6698
6699 if (!is_highmem_idx(j))
6700 nr_kernel_pages += freesize;
6701
6702 else if (nr_kernel_pages > memmap_pages * 2)
6703 nr_kernel_pages -= memmap_pages;
6704 nr_all_pages += freesize;
6705
6706
6707
6708
6709
6710
6711 zone_init_internals(zone, j, nid, freesize);
6712
6713 if (!size)
6714 continue;
6715
6716 set_pageblock_order();
6717 setup_usemap(pgdat, zone, zone_start_pfn, size);
6718 init_currently_empty_zone(zone, zone_start_pfn, size);
6719 memmap_init(size, nid, j, zone_start_pfn);
6720 }
6721}
6722
6723#ifdef CONFIG_FLAT_NODE_MEM_MAP
6724static void __ref alloc_node_mem_map(struct pglist_data *pgdat)
6725{
6726 unsigned long __maybe_unused start = 0;
6727 unsigned long __maybe_unused offset = 0;
6728
6729
6730 if (!pgdat->node_spanned_pages)
6731 return;
6732
6733 start = pgdat->node_start_pfn & ~(MAX_ORDER_NR_PAGES - 1);
6734 offset = pgdat->node_start_pfn - start;
6735
6736 if (!pgdat->node_mem_map) {
6737 unsigned long size, end;
6738 struct page *map;
6739
6740
6741
6742
6743
6744
6745 end = pgdat_end_pfn(pgdat);
6746 end = ALIGN(end, MAX_ORDER_NR_PAGES);
6747 size = (end - start) * sizeof(struct page);
6748 map = memblock_alloc_node(size, SMP_CACHE_BYTES,
6749 pgdat->node_id);
6750 if (!map)
6751 panic("Failed to allocate %ld bytes for node %d memory map\n",
6752 size, pgdat->node_id);
6753 pgdat->node_mem_map = map + offset;
6754 }
6755 pr_debug("%s: node %d, pgdat %08lx, node_mem_map %08lx\n",
6756 __func__, pgdat->node_id, (unsigned long)pgdat,
6757 (unsigned long)pgdat->node_mem_map);
6758#ifndef CONFIG_NEED_MULTIPLE_NODES
6759
6760
6761
6762 if (pgdat == NODE_DATA(0)) {
6763 mem_map = NODE_DATA(0)->node_mem_map;
6764#if defined(CONFIG_HAVE_MEMBLOCK_NODE_MAP) || defined(CONFIG_FLATMEM)
6765 if (page_to_pfn(mem_map) != pgdat->node_start_pfn)
6766 mem_map -= offset;
6767#endif
6768 }
6769#endif
6770}
6771#else
6772static void __ref alloc_node_mem_map(struct pglist_data *pgdat) { }
6773#endif
6774
6775#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
6776static inline void pgdat_set_deferred_range(pg_data_t *pgdat)
6777{
6778 pgdat->first_deferred_pfn = ULONG_MAX;
6779}
6780#else
6781static inline void pgdat_set_deferred_range(pg_data_t *pgdat) {}
6782#endif
6783
6784void __init free_area_init_node(int nid, unsigned long *zones_size,
6785 unsigned long node_start_pfn,
6786 unsigned long *zholes_size)
6787{
6788 pg_data_t *pgdat = NODE_DATA(nid);
6789 unsigned long start_pfn = 0;
6790 unsigned long end_pfn = 0;
6791
6792
6793 WARN_ON(pgdat->nr_zones || pgdat->kswapd_classzone_idx);
6794
6795 pgdat->node_id = nid;
6796 pgdat->node_start_pfn = node_start_pfn;
6797 pgdat->per_cpu_nodestats = NULL;
6798#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
6799 get_pfn_range_for_nid(nid, &start_pfn, &end_pfn);
6800 pr_info("Initmem setup node %d [mem %#018Lx-%#018Lx]\n", nid,
6801 (u64)start_pfn << PAGE_SHIFT,
6802 end_pfn ? ((u64)end_pfn << PAGE_SHIFT) - 1 : 0);
6803#else
6804 start_pfn = node_start_pfn;
6805#endif
6806 calculate_node_totalpages(pgdat, start_pfn, end_pfn,
6807 zones_size, zholes_size);
6808
6809 alloc_node_mem_map(pgdat);
6810 pgdat_set_deferred_range(pgdat);
6811
6812 free_area_init_core(pgdat);
6813}
6814
6815#if !defined(CONFIG_FLAT_NODE_MEM_MAP)
6816
6817
6818
6819
6820static u64 zero_pfn_range(unsigned long spfn, unsigned long epfn)
6821{
6822 unsigned long pfn;
6823 u64 pgcnt = 0;
6824
6825 for (pfn = spfn; pfn < epfn; pfn++) {
6826 if (!pfn_valid(ALIGN_DOWN(pfn, pageblock_nr_pages))) {
6827 pfn = ALIGN_DOWN(pfn, pageblock_nr_pages)
6828 + pageblock_nr_pages - 1;
6829 continue;
6830 }
6831 mm_zero_struct_page(pfn_to_page(pfn));
6832 pgcnt++;
6833 }
6834
6835 return pgcnt;
6836}
6837
6838
6839
6840
6841
6842
6843
6844
6845
6846
6847
6848
6849
6850void __init zero_resv_unavail(void)
6851{
6852 phys_addr_t start, end;
6853 u64 i, pgcnt;
6854 phys_addr_t next = 0;
6855
6856
6857
6858
6859 pgcnt = 0;
6860 for_each_mem_range(i, &memblock.memory, NULL,
6861 NUMA_NO_NODE, MEMBLOCK_NONE, &start, &end, NULL) {
6862 if (next < start)
6863 pgcnt += zero_pfn_range(PFN_DOWN(next), PFN_UP(start));
6864 next = end;
6865 }
6866 pgcnt += zero_pfn_range(PFN_DOWN(next), max_pfn);
6867
6868
6869
6870
6871
6872 if (pgcnt)
6873 pr_info("Zeroed struct page in unavailable ranges: %lld pages", pgcnt);
6874}
6875#endif
6876
6877#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
6878
6879#if MAX_NUMNODES > 1
6880
6881
6882
6883void __init setup_nr_node_ids(void)
6884{
6885 unsigned int highest;
6886
6887 highest = find_last_bit(node_possible_map.bits, MAX_NUMNODES);
6888 nr_node_ids = highest + 1;
6889}
6890#endif
6891
6892
6893
6894
6895
6896
6897
6898
6899
6900
6901
6902
6903
6904
6905
6906
6907
6908
6909
6910
6911unsigned long __init node_map_pfn_alignment(void)
6912{
6913 unsigned long accl_mask = 0, last_end = 0;
6914 unsigned long start, end, mask;
6915 int last_nid = NUMA_NO_NODE;
6916 int i, nid;
6917
6918 for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, &nid) {
6919 if (!start || last_nid < 0 || last_nid == nid) {
6920 last_nid = nid;
6921 last_end = end;
6922 continue;
6923 }
6924
6925
6926
6927
6928
6929
6930 mask = ~((1 << __ffs(start)) - 1);
6931 while (mask && last_end <= (start & (mask << 1)))
6932 mask <<= 1;
6933
6934
6935 accl_mask |= mask;
6936 }
6937
6938
6939 return ~accl_mask + 1;
6940}
6941
6942
6943static unsigned long __init find_min_pfn_for_node(int nid)
6944{
6945 unsigned long min_pfn = ULONG_MAX;
6946 unsigned long start_pfn;
6947 int i;
6948
6949 for_each_mem_pfn_range(i, nid, &start_pfn, NULL, NULL)
6950 min_pfn = min(min_pfn, start_pfn);
6951
6952 if (min_pfn == ULONG_MAX) {
6953 pr_warn("Could not find start_pfn for node %d\n", nid);
6954 return 0;
6955 }
6956
6957 return min_pfn;
6958}
6959
6960
6961
6962
6963
6964
6965
6966unsigned long __init find_min_pfn_with_active_regions(void)
6967{
6968 return find_min_pfn_for_node(MAX_NUMNODES);
6969}
6970
6971
6972
6973
6974
6975
6976static unsigned long __init early_calculate_totalpages(void)
6977{
6978 unsigned long totalpages = 0;
6979 unsigned long start_pfn, end_pfn;
6980 int i, nid;
6981
6982 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) {
6983 unsigned long pages = end_pfn - start_pfn;
6984
6985 totalpages += pages;
6986 if (pages)
6987 node_set_state(nid, N_MEMORY);
6988 }
6989 return totalpages;
6990}
6991
6992
6993
6994
6995
6996
6997
6998static void __init find_zone_movable_pfns_for_nodes(void)
6999{
7000 int i, nid;
7001 unsigned long usable_startpfn;
7002 unsigned long kernelcore_node, kernelcore_remaining;
7003
7004 nodemask_t saved_node_state = node_states[N_MEMORY];
7005 unsigned long totalpages = early_calculate_totalpages();
7006 int usable_nodes = nodes_weight(node_states[N_MEMORY]);
7007 struct memblock_region *r;
7008
7009
7010 find_usable_zone_for_movable();
7011
7012
7013
7014
7015
7016 if (movable_node_is_enabled()) {
7017 for_each_memblock(memory, r) {
7018 if (!memblock_is_hotpluggable(r))
7019 continue;
7020
7021 nid = r->nid;
7022
7023 usable_startpfn = PFN_DOWN(r->base);
7024 zone_movable_pfn[nid] = zone_movable_pfn[nid] ?
7025 min(usable_startpfn, zone_movable_pfn[nid]) :
7026 usable_startpfn;
7027 }
7028
7029 goto out2;
7030 }
7031
7032
7033
7034
7035 if (mirrored_kernelcore) {
7036 bool mem_below_4gb_not_mirrored = false;
7037
7038 for_each_memblock(memory, r) {
7039 if (memblock_is_mirror(r))
7040 continue;
7041
7042 nid = r->nid;
7043
7044 usable_startpfn = memblock_region_memory_base_pfn(r);
7045
7046 if (usable_startpfn < 0x100000) {
7047 mem_below_4gb_not_mirrored = true;
7048 continue;
7049 }
7050
7051 zone_movable_pfn[nid] = zone_movable_pfn[nid] ?
7052 min(usable_startpfn, zone_movable_pfn[nid]) :
7053 usable_startpfn;
7054 }
7055
7056 if (mem_below_4gb_not_mirrored)
7057 pr_warn("This configuration results in unmirrored kernel memory.");
7058
7059 goto out2;
7060 }
7061
7062
7063
7064
7065
7066 if (required_kernelcore_percent)
7067 required_kernelcore = (totalpages * 100 * required_kernelcore_percent) /
7068 10000UL;
7069 if (required_movablecore_percent)
7070 required_movablecore = (totalpages * 100 * required_movablecore_percent) /
7071 10000UL;
7072
7073
7074
7075
7076
7077
7078
7079
7080
7081 if (required_movablecore) {
7082 unsigned long corepages;
7083
7084
7085
7086
7087
7088 required_movablecore =
7089 roundup(required_movablecore, MAX_ORDER_NR_PAGES);
7090 required_movablecore = min(totalpages, required_movablecore);
7091 corepages = totalpages - required_movablecore;
7092
7093 required_kernelcore = max(required_kernelcore, corepages);
7094 }
7095
7096
7097
7098
7099
7100 if (!required_kernelcore || required_kernelcore >= totalpages)
7101 goto out;
7102
7103
7104 usable_startpfn = arch_zone_lowest_possible_pfn[movable_zone];
7105
7106restart:
7107
7108 kernelcore_node = required_kernelcore / usable_nodes;
7109 for_each_node_state(nid, N_MEMORY) {
7110 unsigned long start_pfn, end_pfn;
7111
7112
7113
7114
7115
7116
7117 if (required_kernelcore < kernelcore_node)
7118 kernelcore_node = required_kernelcore / usable_nodes;
7119
7120
7121
7122
7123
7124
7125 kernelcore_remaining = kernelcore_node;
7126
7127
7128 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
7129 unsigned long size_pages;
7130
7131 start_pfn = max(start_pfn, zone_movable_pfn[nid]);
7132 if (start_pfn >= end_pfn)
7133 continue;
7134
7135
7136 if (start_pfn < usable_startpfn) {
7137 unsigned long kernel_pages;
7138 kernel_pages = min(end_pfn, usable_startpfn)
7139 - start_pfn;
7140
7141 kernelcore_remaining -= min(kernel_pages,
7142 kernelcore_remaining);
7143 required_kernelcore -= min(kernel_pages,
7144 required_kernelcore);
7145
7146
7147 if (end_pfn <= usable_startpfn) {
7148
7149
7150
7151
7152
7153
7154
7155 zone_movable_pfn[nid] = end_pfn;
7156 continue;
7157 }
7158 start_pfn = usable_startpfn;
7159 }
7160
7161
7162
7163
7164
7165
7166 size_pages = end_pfn - start_pfn;
7167 if (size_pages > kernelcore_remaining)
7168 size_pages = kernelcore_remaining;
7169 zone_movable_pfn[nid] = start_pfn + size_pages;
7170
7171
7172
7173
7174
7175
7176 required_kernelcore -= min(required_kernelcore,
7177 size_pages);
7178 kernelcore_remaining -= size_pages;
7179 if (!kernelcore_remaining)
7180 break;
7181 }
7182 }
7183
7184
7185
7186
7187
7188
7189
7190 usable_nodes--;
7191 if (usable_nodes && required_kernelcore > usable_nodes)
7192 goto restart;
7193
7194out2:
7195
7196 for (nid = 0; nid < MAX_NUMNODES; nid++)
7197 zone_movable_pfn[nid] =
7198 roundup(zone_movable_pfn[nid], MAX_ORDER_NR_PAGES);
7199
7200out:
7201
7202 node_states[N_MEMORY] = saved_node_state;
7203}
7204
7205
7206static void check_for_memory(pg_data_t *pgdat, int nid)
7207{
7208 enum zone_type zone_type;
7209
7210 for (zone_type = 0; zone_type <= ZONE_MOVABLE - 1; zone_type++) {
7211 struct zone *zone = &pgdat->node_zones[zone_type];
7212 if (populated_zone(zone)) {
7213 if (IS_ENABLED(CONFIG_HIGHMEM))
7214 node_set_state(nid, N_HIGH_MEMORY);
7215 if (zone_type <= ZONE_NORMAL)
7216 node_set_state(nid, N_NORMAL_MEMORY);
7217 break;
7218 }
7219 }
7220}
7221
7222
7223
7224
7225
7226
7227
7228
7229
7230
7231
7232
7233
7234
7235void __init free_area_init_nodes(unsigned long *max_zone_pfn)
7236{
7237 unsigned long start_pfn, end_pfn;
7238 int i, nid;
7239
7240
7241 memset(arch_zone_lowest_possible_pfn, 0,
7242 sizeof(arch_zone_lowest_possible_pfn));
7243 memset(arch_zone_highest_possible_pfn, 0,
7244 sizeof(arch_zone_highest_possible_pfn));
7245
7246 start_pfn = find_min_pfn_with_active_regions();
7247
7248 for (i = 0; i < MAX_NR_ZONES; i++) {
7249 if (i == ZONE_MOVABLE)
7250 continue;
7251
7252 end_pfn = max(max_zone_pfn[i], start_pfn);
7253 arch_zone_lowest_possible_pfn[i] = start_pfn;
7254 arch_zone_highest_possible_pfn[i] = end_pfn;
7255
7256 start_pfn = end_pfn;
7257 }
7258
7259
7260 memset(zone_movable_pfn, 0, sizeof(zone_movable_pfn));
7261 find_zone_movable_pfns_for_nodes();
7262
7263
7264 pr_info("Zone ranges:\n");
7265 for (i = 0; i < MAX_NR_ZONES; i++) {
7266 if (i == ZONE_MOVABLE)
7267 continue;
7268 pr_info(" %-8s ", zone_names[i]);
7269 if (arch_zone_lowest_possible_pfn[i] ==
7270 arch_zone_highest_possible_pfn[i])
7271 pr_cont("empty\n");
7272 else
7273 pr_cont("[mem %#018Lx-%#018Lx]\n",
7274 (u64)arch_zone_lowest_possible_pfn[i]
7275 << PAGE_SHIFT,
7276 ((u64)arch_zone_highest_possible_pfn[i]
7277 << PAGE_SHIFT) - 1);
7278 }
7279
7280
7281 pr_info("Movable zone start for each node\n");
7282 for (i = 0; i < MAX_NUMNODES; i++) {
7283 if (zone_movable_pfn[i])
7284 pr_info(" Node %d: %#018Lx\n", i,
7285 (u64)zone_movable_pfn[i] << PAGE_SHIFT);
7286 }
7287
7288
7289 pr_info("Early memory node ranges\n");
7290 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid)
7291 pr_info(" node %3d: [mem %#018Lx-%#018Lx]\n", nid,
7292 (u64)start_pfn << PAGE_SHIFT,
7293 ((u64)end_pfn << PAGE_SHIFT) - 1);
7294
7295
7296 mminit_verify_pageflags_layout();
7297 setup_nr_node_ids();
7298 zero_resv_unavail();
7299 for_each_online_node(nid) {
7300 pg_data_t *pgdat = NODE_DATA(nid);
7301 free_area_init_node(nid, NULL,
7302 find_min_pfn_for_node(nid), NULL);
7303
7304
7305 if (pgdat->node_present_pages)
7306 node_set_state(nid, N_MEMORY);
7307 check_for_memory(pgdat, nid);
7308 }
7309}
7310
7311static int __init cmdline_parse_core(char *p, unsigned long *core,
7312 unsigned long *percent)
7313{
7314 unsigned long long coremem;
7315 char *endptr;
7316
7317 if (!p)
7318 return -EINVAL;
7319
7320
7321 coremem = simple_strtoull(p, &endptr, 0);
7322 if (*endptr == '%') {
7323
7324 WARN_ON(coremem > 100);
7325
7326 *percent = coremem;
7327 } else {
7328 coremem = memparse(p, &p);
7329
7330 WARN_ON((coremem >> PAGE_SHIFT) > ULONG_MAX);
7331
7332 *core = coremem >> PAGE_SHIFT;
7333 *percent = 0UL;
7334 }
7335 return 0;
7336}
7337
7338
7339
7340
7341
7342static int __init cmdline_parse_kernelcore(char *p)
7343{
7344
7345 if (parse_option_str(p, "mirror")) {
7346 mirrored_kernelcore = true;
7347 return 0;
7348 }
7349
7350 return cmdline_parse_core(p, &required_kernelcore,
7351 &required_kernelcore_percent);
7352}
7353
7354
7355
7356
7357
7358static int __init cmdline_parse_movablecore(char *p)
7359{
7360 return cmdline_parse_core(p, &required_movablecore,
7361 &required_movablecore_percent);
7362}
7363
7364early_param("kernelcore", cmdline_parse_kernelcore);
7365early_param("movablecore", cmdline_parse_movablecore);
7366
7367#endif
7368
7369void adjust_managed_page_count(struct page *page, long count)
7370{
7371 atomic_long_add(count, &page_zone(page)->managed_pages);
7372 totalram_pages_add(count);
7373#ifdef CONFIG_HIGHMEM
7374 if (PageHighMem(page))
7375 totalhigh_pages_add(count);
7376#endif
7377}
7378EXPORT_SYMBOL(adjust_managed_page_count);
7379
7380unsigned long free_reserved_area(void *start, void *end, int poison, const char *s)
7381{
7382 void *pos;
7383 unsigned long pages = 0;
7384
7385 start = (void *)PAGE_ALIGN((unsigned long)start);
7386 end = (void *)((unsigned long)end & PAGE_MASK);
7387 for (pos = start; pos < end; pos += PAGE_SIZE, pages++) {
7388 struct page *page = virt_to_page(pos);
7389 void *direct_map_addr;
7390
7391
7392
7393
7394
7395
7396
7397
7398 direct_map_addr = page_address(page);
7399 if ((unsigned int)poison <= 0xFF)
7400 memset(direct_map_addr, poison, PAGE_SIZE);
7401
7402 free_reserved_page(page);
7403 }
7404
7405 if (pages && s)
7406 pr_info("Freeing %s memory: %ldK\n",
7407 s, pages << (PAGE_SHIFT - 10));
7408
7409 return pages;
7410}
7411
7412#ifdef CONFIG_HIGHMEM
7413void free_highmem_page(struct page *page)
7414{
7415 __free_reserved_page(page);
7416 totalram_pages_inc();
7417 atomic_long_inc(&page_zone(page)->managed_pages);
7418 totalhigh_pages_inc();
7419}
7420#endif
7421
7422
7423void __init mem_init_print_info(const char *str)
7424{
7425 unsigned long physpages, codesize, datasize, rosize, bss_size;
7426 unsigned long init_code_size, init_data_size;
7427
7428 physpages = get_num_physpages();
7429 codesize = _etext - _stext;
7430 datasize = _edata - _sdata;
7431 rosize = __end_rodata - __start_rodata;
7432 bss_size = __bss_stop - __bss_start;
7433 init_data_size = __init_end - __init_begin;
7434 init_code_size = _einittext - _sinittext;
7435
7436
7437
7438
7439
7440
7441
7442
7443#define adj_init_size(start, end, size, pos, adj) \
7444 do { \
7445 if (start <= pos && pos < end && size > adj) \
7446 size -= adj; \
7447 } while (0)
7448
7449 adj_init_size(__init_begin, __init_end, init_data_size,
7450 _sinittext, init_code_size);
7451 adj_init_size(_stext, _etext, codesize, _sinittext, init_code_size);
7452 adj_init_size(_sdata, _edata, datasize, __init_begin, init_data_size);
7453 adj_init_size(_stext, _etext, codesize, __start_rodata, rosize);
7454 adj_init_size(_sdata, _edata, datasize, __start_rodata, rosize);
7455
7456#undef adj_init_size
7457
7458 pr_info("Memory: %luK/%luK available (%luK kernel code, %luK rwdata, %luK rodata, %luK init, %luK bss, %luK reserved, %luK cma-reserved"
7459#ifdef CONFIG_HIGHMEM
7460 ", %luK highmem"
7461#endif
7462 "%s%s)\n",
7463 nr_free_pages() << (PAGE_SHIFT - 10),
7464 physpages << (PAGE_SHIFT - 10),
7465 codesize >> 10, datasize >> 10, rosize >> 10,
7466 (init_data_size + init_code_size) >> 10, bss_size >> 10,
7467 (physpages - totalram_pages() - totalcma_pages) << (PAGE_SHIFT - 10),
7468 totalcma_pages << (PAGE_SHIFT - 10),
7469#ifdef CONFIG_HIGHMEM
7470 totalhigh_pages() << (PAGE_SHIFT - 10),
7471#endif
7472 str ? ", " : "", str ? str : "");
7473}
7474
7475
7476
7477
7478
7479
7480
7481
7482
7483
7484
7485
7486void __init set_dma_reserve(unsigned long new_dma_reserve)
7487{
7488 dma_reserve = new_dma_reserve;
7489}
7490
7491void __init free_area_init(unsigned long *zones_size)
7492{
7493 zero_resv_unavail();
7494 free_area_init_node(0, zones_size,
7495 __pa(PAGE_OFFSET) >> PAGE_SHIFT, NULL);
7496}
7497
7498static int page_alloc_cpu_dead(unsigned int cpu)
7499{
7500
7501 lru_add_drain_cpu(cpu);
7502 drain_pages(cpu);
7503
7504
7505
7506
7507
7508
7509
7510 vm_events_fold_cpu(cpu);
7511
7512
7513
7514
7515
7516
7517
7518
7519 cpu_vm_stats_fold(cpu);
7520 return 0;
7521}
7522
7523void __init page_alloc_init(void)
7524{
7525 int ret;
7526
7527 ret = cpuhp_setup_state_nocalls(CPUHP_PAGE_ALLOC_DEAD,
7528 "mm/page_alloc:dead", NULL,
7529 page_alloc_cpu_dead);
7530 WARN_ON(ret < 0);
7531}
7532
7533
7534
7535
7536
7537static void calculate_totalreserve_pages(void)
7538{
7539 struct pglist_data *pgdat;
7540 unsigned long reserve_pages = 0;
7541 enum zone_type i, j;
7542
7543 for_each_online_pgdat(pgdat) {
7544
7545 pgdat->totalreserve_pages = 0;
7546
7547 for (i = 0; i < MAX_NR_ZONES; i++) {
7548 struct zone *zone = pgdat->node_zones + i;
7549 long max = 0;
7550 unsigned long managed_pages = zone_managed_pages(zone);
7551
7552
7553 for (j = i; j < MAX_NR_ZONES; j++) {
7554 if (zone->lowmem_reserve[j] > max)
7555 max = zone->lowmem_reserve[j];
7556 }
7557
7558
7559 max += high_wmark_pages(zone);
7560
7561 if (max > managed_pages)
7562 max = managed_pages;
7563
7564 pgdat->totalreserve_pages += max;
7565
7566 reserve_pages += max;
7567 }
7568 }
7569 totalreserve_pages = reserve_pages;
7570}
7571
7572
7573
7574
7575
7576
7577
7578static void setup_per_zone_lowmem_reserve(void)
7579{
7580 struct pglist_data *pgdat;
7581 enum zone_type j, idx;
7582
7583 for_each_online_pgdat(pgdat) {
7584 for (j = 0; j < MAX_NR_ZONES; j++) {
7585 struct zone *zone = pgdat->node_zones + j;
7586 unsigned long managed_pages = zone_managed_pages(zone);
7587
7588 zone->lowmem_reserve[j] = 0;
7589
7590 idx = j;
7591 while (idx) {
7592 struct zone *lower_zone;
7593
7594 idx--;
7595 lower_zone = pgdat->node_zones + idx;
7596
7597 if (sysctl_lowmem_reserve_ratio[idx] < 1) {
7598 sysctl_lowmem_reserve_ratio[idx] = 0;
7599 lower_zone->lowmem_reserve[j] = 0;
7600 } else {
7601 lower_zone->lowmem_reserve[j] =
7602 managed_pages / sysctl_lowmem_reserve_ratio[idx];
7603 }
7604 managed_pages += zone_managed_pages(lower_zone);
7605 }
7606 }
7607 }
7608
7609
7610 calculate_totalreserve_pages();
7611}
7612
7613static void __setup_per_zone_wmarks(void)
7614{
7615 unsigned long pages_min = min_free_kbytes >> (PAGE_SHIFT - 10);
7616 unsigned long lowmem_pages = 0;
7617 struct zone *zone;
7618 unsigned long flags;
7619
7620
7621 for_each_zone(zone) {
7622 if (!is_highmem(zone))
7623 lowmem_pages += zone_managed_pages(zone);
7624 }
7625
7626 for_each_zone(zone) {
7627 u64 tmp;
7628
7629 spin_lock_irqsave(&zone->lock, flags);
7630 tmp = (u64)pages_min * zone_managed_pages(zone);
7631 do_div(tmp, lowmem_pages);
7632 if (is_highmem(zone)) {
7633
7634
7635
7636
7637
7638
7639
7640
7641
7642 unsigned long min_pages;
7643
7644 min_pages = zone_managed_pages(zone) / 1024;
7645 min_pages = clamp(min_pages, SWAP_CLUSTER_MAX, 128UL);
7646 zone->_watermark[WMARK_MIN] = min_pages;
7647 } else {
7648
7649
7650
7651
7652 zone->_watermark[WMARK_MIN] = tmp;
7653 }
7654
7655
7656
7657
7658
7659
7660 tmp = max_t(u64, tmp >> 2,
7661 mult_frac(zone_managed_pages(zone),
7662 watermark_scale_factor, 10000));
7663
7664 zone->_watermark[WMARK_LOW] = min_wmark_pages(zone) + tmp;
7665 zone->_watermark[WMARK_HIGH] = min_wmark_pages(zone) + tmp * 2;
7666 zone->watermark_boost = 0;
7667
7668 spin_unlock_irqrestore(&zone->lock, flags);
7669 }
7670
7671
7672 calculate_totalreserve_pages();
7673}
7674
7675
7676
7677
7678
7679
7680
7681
7682void setup_per_zone_wmarks(void)
7683{
7684 static DEFINE_SPINLOCK(lock);
7685
7686 spin_lock(&lock);
7687 __setup_per_zone_wmarks();
7688 spin_unlock(&lock);
7689}
7690
7691
7692
7693
7694
7695
7696
7697
7698
7699
7700
7701
7702
7703
7704
7705
7706
7707
7708
7709
7710
7711
7712
7713
7714
7715int __meminit init_per_zone_wmark_min(void)
7716{
7717 unsigned long lowmem_kbytes;
7718 int new_min_free_kbytes;
7719
7720 lowmem_kbytes = nr_free_buffer_pages() * (PAGE_SIZE >> 10);
7721 new_min_free_kbytes = int_sqrt(lowmem_kbytes * 16);
7722
7723 if (new_min_free_kbytes > user_min_free_kbytes) {
7724 min_free_kbytes = new_min_free_kbytes;
7725 if (min_free_kbytes < 128)
7726 min_free_kbytes = 128;
7727 if (min_free_kbytes > 65536)
7728 min_free_kbytes = 65536;
7729 } else {
7730 pr_warn("min_free_kbytes is not updated to %d because user defined value %d is preferred\n",
7731 new_min_free_kbytes, user_min_free_kbytes);
7732 }
7733 setup_per_zone_wmarks();
7734 refresh_zone_stat_thresholds();
7735 setup_per_zone_lowmem_reserve();
7736
7737#ifdef CONFIG_NUMA
7738 setup_min_unmapped_ratio();
7739 setup_min_slab_ratio();
7740#endif
7741
7742 return 0;
7743}
7744core_initcall(init_per_zone_wmark_min)
7745
7746
7747
7748
7749
7750
7751int min_free_kbytes_sysctl_handler(struct ctl_table *table, int write,
7752 void __user *buffer, size_t *length, loff_t *ppos)
7753{
7754 int rc;
7755
7756 rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
7757 if (rc)
7758 return rc;
7759
7760 if (write) {
7761 user_min_free_kbytes = min_free_kbytes;
7762 setup_per_zone_wmarks();
7763 }
7764 return 0;
7765}
7766
7767int watermark_boost_factor_sysctl_handler(struct ctl_table *table, int write,
7768 void __user *buffer, size_t *length, loff_t *ppos)
7769{
7770 int rc;
7771
7772 rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
7773 if (rc)
7774 return rc;
7775
7776 return 0;
7777}
7778
7779int watermark_scale_factor_sysctl_handler(struct ctl_table *table, int write,
7780 void __user *buffer, size_t *length, loff_t *ppos)
7781{
7782 int rc;
7783
7784 rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
7785 if (rc)
7786 return rc;
7787
7788 if (write)
7789 setup_per_zone_wmarks();
7790
7791 return 0;
7792}
7793
7794#ifdef CONFIG_NUMA
7795static void setup_min_unmapped_ratio(void)
7796{
7797 pg_data_t *pgdat;
7798 struct zone *zone;
7799
7800 for_each_online_pgdat(pgdat)
7801 pgdat->min_unmapped_pages = 0;
7802
7803 for_each_zone(zone)
7804 zone->zone_pgdat->min_unmapped_pages += (zone_managed_pages(zone) *
7805 sysctl_min_unmapped_ratio) / 100;
7806}
7807
7808
7809int sysctl_min_unmapped_ratio_sysctl_handler(struct ctl_table *table, int write,
7810 void __user *buffer, size_t *length, loff_t *ppos)
7811{
7812 int rc;
7813
7814 rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
7815 if (rc)
7816 return rc;
7817
7818 setup_min_unmapped_ratio();
7819
7820 return 0;
7821}
7822
7823static void setup_min_slab_ratio(void)
7824{
7825 pg_data_t *pgdat;
7826 struct zone *zone;
7827
7828 for_each_online_pgdat(pgdat)
7829 pgdat->min_slab_pages = 0;
7830
7831 for_each_zone(zone)
7832 zone->zone_pgdat->min_slab_pages += (zone_managed_pages(zone) *
7833 sysctl_min_slab_ratio) / 100;
7834}
7835
7836int sysctl_min_slab_ratio_sysctl_handler(struct ctl_table *table, int write,
7837 void __user *buffer, size_t *length, loff_t *ppos)
7838{
7839 int rc;
7840
7841 rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
7842 if (rc)
7843 return rc;
7844
7845 setup_min_slab_ratio();
7846
7847 return 0;
7848}
7849#endif
7850
7851
7852
7853
7854
7855
7856
7857
7858
7859
7860int lowmem_reserve_ratio_sysctl_handler(struct ctl_table *table, int write,
7861 void __user *buffer, size_t *length, loff_t *ppos)
7862{
7863 proc_dointvec_minmax(table, write, buffer, length, ppos);
7864 setup_per_zone_lowmem_reserve();
7865 return 0;
7866}
7867
7868
7869
7870
7871
7872
7873int percpu_pagelist_fraction_sysctl_handler(struct ctl_table *table, int write,
7874 void __user *buffer, size_t *length, loff_t *ppos)
7875{
7876 struct zone *zone;
7877 int old_percpu_pagelist_fraction;
7878 int ret;
7879
7880 mutex_lock(&pcp_batch_high_lock);
7881 old_percpu_pagelist_fraction = percpu_pagelist_fraction;
7882
7883 ret = proc_dointvec_minmax(table, write, buffer, length, ppos);
7884 if (!write || ret < 0)
7885 goto out;
7886
7887
7888 if (percpu_pagelist_fraction &&
7889 percpu_pagelist_fraction < MIN_PERCPU_PAGELIST_FRACTION) {
7890 percpu_pagelist_fraction = old_percpu_pagelist_fraction;
7891 ret = -EINVAL;
7892 goto out;
7893 }
7894
7895
7896 if (percpu_pagelist_fraction == old_percpu_pagelist_fraction)
7897 goto out;
7898
7899 for_each_populated_zone(zone) {
7900 unsigned int cpu;
7901
7902 for_each_possible_cpu(cpu)
7903 pageset_set_high_and_batch(zone,
7904 per_cpu_ptr(zone->pageset, cpu));
7905 }
7906out:
7907 mutex_unlock(&pcp_batch_high_lock);
7908 return ret;
7909}
7910
7911#ifdef CONFIG_NUMA
7912int hashdist = HASHDIST_DEFAULT;
7913
7914static int __init set_hashdist(char *str)
7915{
7916 if (!str)
7917 return 0;
7918 hashdist = simple_strtoul(str, &str, 0);
7919 return 1;
7920}
7921__setup("hashdist=", set_hashdist);
7922#endif
7923
7924#ifndef __HAVE_ARCH_RESERVED_KERNEL_PAGES
7925
7926
7927
7928
7929static unsigned long __init arch_reserved_kernel_pages(void)
7930{
7931 return 0;
7932}
7933#endif
7934
7935
7936
7937
7938
7939
7940
7941
7942
7943
7944#if __BITS_PER_LONG > 32
7945#define ADAPT_SCALE_BASE (64ul << 30)
7946#define ADAPT_SCALE_SHIFT 2
7947#define ADAPT_SCALE_NPAGES (ADAPT_SCALE_BASE >> PAGE_SHIFT)
7948#endif
7949
7950
7951
7952
7953
7954
7955
7956void *__init alloc_large_system_hash(const char *tablename,
7957 unsigned long bucketsize,
7958 unsigned long numentries,
7959 int scale,
7960 int flags,
7961 unsigned int *_hash_shift,
7962 unsigned int *_hash_mask,
7963 unsigned long low_limit,
7964 unsigned long high_limit)
7965{
7966 unsigned long long max = high_limit;
7967 unsigned long log2qty, size;
7968 void *table = NULL;
7969 gfp_t gfp_flags;
7970
7971
7972 if (!numentries) {
7973
7974 numentries = nr_kernel_pages;
7975 numentries -= arch_reserved_kernel_pages();
7976
7977
7978 if (PAGE_SHIFT < 20)
7979 numentries = round_up(numentries, (1<<20)/PAGE_SIZE);
7980
7981#if __BITS_PER_LONG > 32
7982 if (!high_limit) {
7983 unsigned long adapt;
7984
7985 for (adapt = ADAPT_SCALE_NPAGES; adapt < numentries;
7986 adapt <<= ADAPT_SCALE_SHIFT)
7987 scale++;
7988 }
7989#endif
7990
7991
7992 if (scale > PAGE_SHIFT)
7993 numentries >>= (scale - PAGE_SHIFT);
7994 else
7995 numentries <<= (PAGE_SHIFT - scale);
7996
7997
7998 if (unlikely(flags & HASH_SMALL)) {
7999
8000 WARN_ON(!(flags & HASH_EARLY));
8001 if (!(numentries >> *_hash_shift)) {
8002 numentries = 1UL << *_hash_shift;
8003 BUG_ON(!numentries);
8004 }
8005 } else if (unlikely((numentries * bucketsize) < PAGE_SIZE))
8006 numentries = PAGE_SIZE / bucketsize;
8007 }
8008 numentries = roundup_pow_of_two(numentries);
8009
8010
8011 if (max == 0) {
8012 max = ((unsigned long long)nr_all_pages << PAGE_SHIFT) >> 4;
8013 do_div(max, bucketsize);
8014 }
8015 max = min(max, 0x80000000ULL);
8016
8017 if (numentries < low_limit)
8018 numentries = low_limit;
8019 if (numentries > max)
8020 numentries = max;
8021
8022 log2qty = ilog2(numentries);
8023
8024 gfp_flags = (flags & HASH_ZERO) ? GFP_ATOMIC | __GFP_ZERO : GFP_ATOMIC;
8025 do {
8026 size = bucketsize << log2qty;
8027 if (flags & HASH_EARLY) {
8028 if (flags & HASH_ZERO)
8029 table = memblock_alloc(size, SMP_CACHE_BYTES);
8030 else
8031 table = memblock_alloc_raw(size,
8032 SMP_CACHE_BYTES);
8033 } else if (hashdist) {
8034 table = __vmalloc(size, gfp_flags, PAGE_KERNEL);
8035 } else {
8036
8037
8038
8039
8040
8041 if (get_order(size) < MAX_ORDER) {
8042 table = alloc_pages_exact(size, gfp_flags);
8043 kmemleak_alloc(table, size, 1, gfp_flags);
8044 }
8045 }
8046 } while (!table && size > PAGE_SIZE && --log2qty);
8047
8048 if (!table)
8049 panic("Failed to allocate %s hash table\n", tablename);
8050
8051 pr_info("%s hash table entries: %ld (order: %d, %lu bytes)\n",
8052 tablename, 1UL << log2qty, ilog2(size) - PAGE_SHIFT, size);
8053
8054 if (_hash_shift)
8055 *_hash_shift = log2qty;
8056 if (_hash_mask)
8057 *_hash_mask = (1 << log2qty) - 1;
8058
8059 return table;
8060}
8061
8062
8063
8064
8065
8066
8067
8068
8069
8070
8071bool has_unmovable_pages(struct zone *zone, struct page *page, int count,
8072 int migratetype, int flags)
8073{
8074 unsigned long found;
8075 unsigned long iter = 0;
8076 unsigned long pfn = page_to_pfn(page);
8077 const char *reason = "unmovable page";
8078
8079
8080
8081
8082
8083
8084
8085
8086
8087 if (is_migrate_cma_page(page)) {
8088
8089
8090
8091
8092
8093 if (is_migrate_cma(migratetype))
8094 return false;
8095
8096 reason = "CMA page";
8097 goto unmovable;
8098 }
8099
8100 for (found = 0; iter < pageblock_nr_pages; iter++) {
8101 unsigned long check = pfn + iter;
8102
8103 if (!pfn_valid_within(check))
8104 continue;
8105
8106 page = pfn_to_page(check);
8107
8108 if (PageReserved(page))
8109 goto unmovable;
8110
8111
8112
8113
8114
8115
8116 if (zone_idx(zone) == ZONE_MOVABLE)
8117 continue;
8118
8119
8120
8121
8122
8123
8124 if (PageHuge(page)) {
8125 struct page *head = compound_head(page);
8126 unsigned int skip_pages;
8127
8128 if (!hugepage_migration_supported(page_hstate(head)))
8129 goto unmovable;
8130
8131 skip_pages = (1 << compound_order(head)) - (page - head);
8132 iter += skip_pages - 1;
8133 continue;
8134 }
8135
8136
8137
8138
8139
8140
8141
8142 if (!page_ref_count(page)) {
8143 if (PageBuddy(page))
8144 iter += (1 << page_order(page)) - 1;
8145 continue;
8146 }
8147
8148
8149
8150
8151
8152 if ((flags & SKIP_HWPOISON) && PageHWPoison(page))
8153 continue;
8154
8155 if (__PageMovable(page))
8156 continue;
8157
8158 if (!PageLRU(page))
8159 found++;
8160
8161
8162
8163
8164
8165
8166
8167
8168
8169
8170
8171
8172
8173 if (found > count)
8174 goto unmovable;
8175 }
8176 return false;
8177unmovable:
8178 WARN_ON_ONCE(zone_idx(zone) == ZONE_MOVABLE);
8179 if (flags & REPORT_FAILURE)
8180 dump_page(pfn_to_page(pfn + iter), reason);
8181 return true;
8182}
8183
8184#ifdef CONFIG_CONTIG_ALLOC
8185static unsigned long pfn_max_align_down(unsigned long pfn)
8186{
8187 return pfn & ~(max_t(unsigned long, MAX_ORDER_NR_PAGES,
8188 pageblock_nr_pages) - 1);
8189}
8190
8191static unsigned long pfn_max_align_up(unsigned long pfn)
8192{
8193 return ALIGN(pfn, max_t(unsigned long, MAX_ORDER_NR_PAGES,
8194 pageblock_nr_pages));
8195}
8196
8197
8198static int __alloc_contig_migrate_range(struct compact_control *cc,
8199 unsigned long start, unsigned long end)
8200{
8201
8202 unsigned long nr_reclaimed;
8203 unsigned long pfn = start;
8204 unsigned int tries = 0;
8205 int ret = 0;
8206
8207 migrate_prep();
8208
8209 while (pfn < end || !list_empty(&cc->migratepages)) {
8210 if (fatal_signal_pending(current)) {
8211 ret = -EINTR;
8212 break;
8213 }
8214
8215 if (list_empty(&cc->migratepages)) {
8216 cc->nr_migratepages = 0;
8217 pfn = isolate_migratepages_range(cc, pfn, end);
8218 if (!pfn) {
8219 ret = -EINTR;
8220 break;
8221 }
8222 tries = 0;
8223 } else if (++tries == 5) {
8224 ret = ret < 0 ? ret : -EBUSY;
8225 break;
8226 }
8227
8228 nr_reclaimed = reclaim_clean_pages_from_list(cc->zone,
8229 &cc->migratepages);
8230 cc->nr_migratepages -= nr_reclaimed;
8231
8232 ret = migrate_pages(&cc->migratepages, alloc_migrate_target,
8233 NULL, 0, cc->mode, MR_CONTIG_RANGE);
8234 }
8235 if (ret < 0) {
8236 putback_movable_pages(&cc->migratepages);
8237 return ret;
8238 }
8239 return 0;
8240}
8241
8242
8243
8244
8245
8246
8247
8248
8249
8250
8251
8252
8253
8254
8255
8256
8257
8258
8259
8260
8261
8262
8263int alloc_contig_range(unsigned long start, unsigned long end,
8264 unsigned migratetype, gfp_t gfp_mask)
8265{
8266 unsigned long outer_start, outer_end;
8267 unsigned int order;
8268 int ret = 0;
8269
8270 struct compact_control cc = {
8271 .nr_migratepages = 0,
8272 .order = -1,
8273 .zone = page_zone(pfn_to_page(start)),
8274 .mode = MIGRATE_SYNC,
8275 .ignore_skip_hint = true,
8276 .no_set_skip_hint = true,
8277 .gfp_mask = current_gfp_context(gfp_mask),
8278 };
8279 INIT_LIST_HEAD(&cc.migratepages);
8280
8281
8282
8283
8284
8285
8286
8287
8288
8289
8290
8291
8292
8293
8294
8295
8296
8297
8298
8299
8300
8301
8302
8303
8304
8305 ret = start_isolate_page_range(pfn_max_align_down(start),
8306 pfn_max_align_up(end), migratetype, 0);
8307 if (ret < 0)
8308 return ret;
8309
8310
8311
8312
8313
8314
8315
8316
8317
8318
8319
8320 ret = __alloc_contig_migrate_range(&cc, start, end);
8321 if (ret && ret != -EBUSY)
8322 goto done;
8323 ret =0;
8324
8325
8326
8327
8328
8329
8330
8331
8332
8333
8334
8335
8336
8337
8338
8339
8340
8341
8342 lru_add_drain_all();
8343
8344 order = 0;
8345 outer_start = start;
8346 while (!PageBuddy(pfn_to_page(outer_start))) {
8347 if (++order >= MAX_ORDER) {
8348 outer_start = start;
8349 break;
8350 }
8351 outer_start &= ~0UL << order;
8352 }
8353
8354 if (outer_start != start) {
8355 order = page_order(pfn_to_page(outer_start));
8356
8357
8358
8359
8360
8361
8362
8363 if (outer_start + (1UL << order) <= start)
8364 outer_start = start;
8365 }
8366
8367
8368 if (test_pages_isolated(outer_start, end, false)) {
8369 pr_info_ratelimited("%s: [%lx, %lx) PFNs busy\n",
8370 __func__, outer_start, end);
8371 ret = -EBUSY;
8372 goto done;
8373 }
8374
8375
8376 outer_end = isolate_freepages_range(&cc, outer_start, end);
8377 if (!outer_end) {
8378 ret = -EBUSY;
8379 goto done;
8380 }
8381
8382
8383 if (start != outer_start)
8384 free_contig_range(outer_start, start - outer_start);
8385 if (end != outer_end)
8386 free_contig_range(end, outer_end - end);
8387
8388done:
8389 undo_isolate_page_range(pfn_max_align_down(start),
8390 pfn_max_align_up(end), migratetype);
8391 return ret;
8392}
8393#endif
8394
8395void free_contig_range(unsigned long pfn, unsigned int nr_pages)
8396{
8397 unsigned int count = 0;
8398
8399 for (; nr_pages--; pfn++) {
8400 struct page *page = pfn_to_page(pfn);
8401
8402 count += page_count(page) != 1;
8403 __free_page(page);
8404 }
8405 WARN(count != 0, "%d pages are still in use!\n", count);
8406}
8407
8408#ifdef CONFIG_MEMORY_HOTPLUG
8409
8410
8411
8412
8413void __meminit zone_pcp_update(struct zone *zone)
8414{
8415 unsigned cpu;
8416 mutex_lock(&pcp_batch_high_lock);
8417 for_each_possible_cpu(cpu)
8418 pageset_set_high_and_batch(zone,
8419 per_cpu_ptr(zone->pageset, cpu));
8420 mutex_unlock(&pcp_batch_high_lock);
8421}
8422#endif
8423
8424void zone_pcp_reset(struct zone *zone)
8425{
8426 unsigned long flags;
8427 int cpu;
8428 struct per_cpu_pageset *pset;
8429
8430
8431 local_irq_save(flags);
8432 if (zone->pageset != &boot_pageset) {
8433 for_each_online_cpu(cpu) {
8434 pset = per_cpu_ptr(zone->pageset, cpu);
8435 drain_zonestat(zone, pset);
8436 }
8437 free_percpu(zone->pageset);
8438 zone->pageset = &boot_pageset;
8439 }
8440 local_irq_restore(flags);
8441}
8442
8443#ifdef CONFIG_MEMORY_HOTREMOVE
8444
8445
8446
8447
8448unsigned long
8449__offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn)
8450{
8451 struct page *page;
8452 struct zone *zone;
8453 unsigned int order, i;
8454 unsigned long pfn;
8455 unsigned long flags;
8456 unsigned long offlined_pages = 0;
8457
8458
8459 for (pfn = start_pfn; pfn < end_pfn; pfn++)
8460 if (pfn_valid(pfn))
8461 break;
8462 if (pfn == end_pfn)
8463 return offlined_pages;
8464
8465 offline_mem_sections(pfn, end_pfn);
8466 zone = page_zone(pfn_to_page(pfn));
8467 spin_lock_irqsave(&zone->lock, flags);
8468 pfn = start_pfn;
8469 while (pfn < end_pfn) {
8470 if (!pfn_valid(pfn)) {
8471 pfn++;
8472 continue;
8473 }
8474 page = pfn_to_page(pfn);
8475
8476
8477
8478
8479 if (unlikely(!PageBuddy(page) && PageHWPoison(page))) {
8480 pfn++;
8481 SetPageReserved(page);
8482 offlined_pages++;
8483 continue;
8484 }
8485
8486 BUG_ON(page_count(page));
8487 BUG_ON(!PageBuddy(page));
8488 order = page_order(page);
8489 offlined_pages += 1 << order;
8490#ifdef CONFIG_DEBUG_VM
8491 pr_info("remove from free list %lx %d %lx\n",
8492 pfn, 1 << order, end_pfn);
8493#endif
8494 del_page_from_free_area(page, &zone->free_area[order]);
8495 for (i = 0; i < (1 << order); i++)
8496 SetPageReserved((page+i));
8497 pfn += (1 << order);
8498 }
8499 spin_unlock_irqrestore(&zone->lock, flags);
8500
8501 return offlined_pages;
8502}
8503#endif
8504
8505bool is_free_buddy_page(struct page *page)
8506{
8507 struct zone *zone = page_zone(page);
8508 unsigned long pfn = page_to_pfn(page);
8509 unsigned long flags;
8510 unsigned int order;
8511
8512 spin_lock_irqsave(&zone->lock, flags);
8513 for (order = 0; order < MAX_ORDER; order++) {
8514 struct page *page_head = page - (pfn & ((1 << order) - 1));
8515
8516 if (PageBuddy(page_head) && page_order(page_head) >= order)
8517 break;
8518 }
8519 spin_unlock_irqrestore(&zone->lock, flags);
8520
8521 return order < MAX_ORDER;
8522}
8523
8524#ifdef CONFIG_MEMORY_FAILURE
8525
8526
8527
8528
8529
8530bool set_hwpoison_free_buddy_page(struct page *page)
8531{
8532 struct zone *zone = page_zone(page);
8533 unsigned long pfn = page_to_pfn(page);
8534 unsigned long flags;
8535 unsigned int order;
8536 bool hwpoisoned = false;
8537
8538 spin_lock_irqsave(&zone->lock, flags);
8539 for (order = 0; order < MAX_ORDER; order++) {
8540 struct page *page_head = page - (pfn & ((1 << order) - 1));
8541
8542 if (PageBuddy(page_head) && page_order(page_head) >= order) {
8543 if (!TestSetPageHWPoison(page))
8544 hwpoisoned = true;
8545 break;
8546 }
8547 }
8548 spin_unlock_irqrestore(&zone->lock, flags);
8549
8550 return hwpoisoned;
8551}
8552#endif
8553