1
2#ifndef _LINUX_MMZONE_H
3#define _LINUX_MMZONE_H
4
5#ifndef __ASSEMBLY__
6#ifndef __GENERATING_BOUNDS_H
7
8#include <linux/spinlock.h>
9#include <linux/list.h>
10#include <linux/wait.h>
11#include <linux/bitops.h>
12#include <linux/cache.h>
13#include <linux/threads.h>
14#include <linux/numa.h>
15#include <linux/init.h>
16#include <linux/seqlock.h>
17#include <linux/nodemask.h>
18#include <linux/pageblock-flags.h>
19#include <linux/page-flags-layout.h>
20#include <linux/atomic.h>
21#include <linux/mm_types.h>
22#include <linux/page-flags.h>
23#include <asm/page.h>
24
25
26#ifndef CONFIG_FORCE_MAX_ZONEORDER
27#define MAX_ORDER 11
28#else
29#define MAX_ORDER CONFIG_FORCE_MAX_ZONEORDER
30#endif
31#define MAX_ORDER_NR_PAGES (1 << (MAX_ORDER - 1))
32
33
34
35
36
37
38
39#define PAGE_ALLOC_COSTLY_ORDER 3
40
41enum migratetype {
42 MIGRATE_UNMOVABLE,
43 MIGRATE_MOVABLE,
44 MIGRATE_RECLAIMABLE,
45 MIGRATE_PCPTYPES,
46 MIGRATE_HIGHATOMIC = MIGRATE_PCPTYPES,
47#ifdef CONFIG_CMA
48
49
50
51
52
53
54
55
56
57
58
59
60
61 MIGRATE_CMA,
62#endif
63#ifdef CONFIG_MEMORY_ISOLATION
64 MIGRATE_ISOLATE,
65#endif
66 MIGRATE_TYPES
67};
68
69
70extern const char * const migratetype_names[MIGRATE_TYPES];
71
72#ifdef CONFIG_CMA
73# define is_migrate_cma(migratetype) unlikely((migratetype) == MIGRATE_CMA)
74# define is_migrate_cma_page(_page) (get_pageblock_migratetype(_page) == MIGRATE_CMA)
75#else
76# define is_migrate_cma(migratetype) false
77# define is_migrate_cma_page(_page) false
78#endif
79
80static inline bool is_migrate_movable(int mt)
81{
82 return is_migrate_cma(mt) || mt == MIGRATE_MOVABLE;
83}
84
85#define for_each_migratetype_order(order, type) \
86 for (order = 0; order < MAX_ORDER; order++) \
87 for (type = 0; type < MIGRATE_TYPES; type++)
88
89extern int page_group_by_mobility_disabled;
90
91#define NR_MIGRATETYPE_BITS (PB_migrate_end - PB_migrate + 1)
92#define MIGRATETYPE_MASK ((1UL << NR_MIGRATETYPE_BITS) - 1)
93
94#define get_pageblock_migratetype(page) \
95 get_pfnblock_flags_mask(page, page_to_pfn(page), \
96 PB_migrate_end, MIGRATETYPE_MASK)
97
98struct free_area {
99 struct list_head free_list[MIGRATE_TYPES];
100 unsigned long nr_free;
101};
102
103
104static inline void add_to_free_area(struct page *page, struct free_area *area,
105 int migratetype)
106{
107 list_add(&page->lru, &area->free_list[migratetype]);
108 area->nr_free++;
109}
110
111
112static inline void add_to_free_area_tail(struct page *page, struct free_area *area,
113 int migratetype)
114{
115 list_add_tail(&page->lru, &area->free_list[migratetype]);
116 area->nr_free++;
117}
118
119#ifdef CONFIG_SHUFFLE_PAGE_ALLOCATOR
120
121void add_to_free_area_random(struct page *page, struct free_area *area,
122 int migratetype);
123#else
124static inline void add_to_free_area_random(struct page *page,
125 struct free_area *area, int migratetype)
126{
127 add_to_free_area(page, area, migratetype);
128}
129#endif
130
131
132static inline void move_to_free_area(struct page *page, struct free_area *area,
133 int migratetype)
134{
135 list_move(&page->lru, &area->free_list[migratetype]);
136}
137
138static inline struct page *get_page_from_free_area(struct free_area *area,
139 int migratetype)
140{
141 return list_first_entry_or_null(&area->free_list[migratetype],
142 struct page, lru);
143}
144
145static inline void del_page_from_free_area(struct page *page,
146 struct free_area *area)
147{
148 list_del(&page->lru);
149 __ClearPageBuddy(page);
150 set_page_private(page, 0);
151 area->nr_free--;
152}
153
154static inline bool free_area_empty(struct free_area *area, int migratetype)
155{
156 return list_empty(&area->free_list[migratetype]);
157}
158
159struct pglist_data;
160
161
162
163
164
165
166
167#if defined(CONFIG_SMP)
168struct zone_padding {
169 char x[0];
170} ____cacheline_internodealigned_in_smp;
171#define ZONE_PADDING(name) struct zone_padding name;
172#else
173#define ZONE_PADDING(name)
174#endif
175
176#ifdef CONFIG_NUMA
177enum numa_stat_item {
178 NUMA_HIT,
179 NUMA_MISS,
180 NUMA_FOREIGN,
181 NUMA_INTERLEAVE_HIT,
182 NUMA_LOCAL,
183 NUMA_OTHER,
184 NR_VM_NUMA_STAT_ITEMS
185};
186#else
187#define NR_VM_NUMA_STAT_ITEMS 0
188#endif
189
190enum zone_stat_item {
191
192 NR_FREE_PAGES,
193 NR_ZONE_LRU_BASE,
194 NR_ZONE_INACTIVE_ANON = NR_ZONE_LRU_BASE,
195 NR_ZONE_ACTIVE_ANON,
196 NR_ZONE_INACTIVE_FILE,
197 NR_ZONE_ACTIVE_FILE,
198 NR_ZONE_UNEVICTABLE,
199 NR_ZONE_WRITE_PENDING,
200 NR_MLOCK,
201 NR_PAGETABLE,
202 NR_KERNEL_STACK_KB,
203
204 NR_BOUNCE,
205#if IS_ENABLED(CONFIG_ZSMALLOC)
206 NR_ZSPAGES,
207#endif
208 NR_FREE_CMA_PAGES,
209 NR_VM_ZONE_STAT_ITEMS };
210
211enum node_stat_item {
212 NR_LRU_BASE,
213 NR_INACTIVE_ANON = NR_LRU_BASE,
214 NR_ACTIVE_ANON,
215 NR_INACTIVE_FILE,
216 NR_ACTIVE_FILE,
217 NR_UNEVICTABLE,
218 NR_SLAB_RECLAIMABLE,
219 NR_SLAB_UNRECLAIMABLE,
220
221 NR_ISOLATED_ANON,
222 NR_ISOLATED_FILE,
223 WORKINGSET_NODES,
224 WORKINGSET_REFAULT,
225 WORKINGSET_ACTIVATE,
226 WORKINGSET_RESTORE,
227 WORKINGSET_NODERECLAIM,
228 NR_ANON_MAPPED,
229 NR_FILE_MAPPED,
230
231 NR_FILE_PAGES,
232 NR_FILE_DIRTY,
233 NR_WRITEBACK,
234 NR_WRITEBACK_TEMP,
235 NR_SHMEM,
236 NR_SHMEM_THPS,
237 NR_SHMEM_PMDMAPPED,
238 NR_ANON_THPS,
239 NR_UNSTABLE_NFS,
240 NR_VMSCAN_WRITE,
241 NR_VMSCAN_IMMEDIATE,
242 NR_DIRTIED,
243 NR_WRITTEN,
244 NR_KERNEL_MISC_RECLAIMABLE,
245 NR_VM_NODE_STAT_ITEMS
246};
247
248
249
250
251
252
253
254
255
256
257#define LRU_BASE 0
258#define LRU_ACTIVE 1
259#define LRU_FILE 2
260
261enum lru_list {
262 LRU_INACTIVE_ANON = LRU_BASE,
263 LRU_ACTIVE_ANON = LRU_BASE + LRU_ACTIVE,
264 LRU_INACTIVE_FILE = LRU_BASE + LRU_FILE,
265 LRU_ACTIVE_FILE = LRU_BASE + LRU_FILE + LRU_ACTIVE,
266 LRU_UNEVICTABLE,
267 NR_LRU_LISTS
268};
269
270#define for_each_lru(lru) for (lru = 0; lru < NR_LRU_LISTS; lru++)
271
272#define for_each_evictable_lru(lru) for (lru = 0; lru <= LRU_ACTIVE_FILE; lru++)
273
274static inline int is_file_lru(enum lru_list lru)
275{
276 return (lru == LRU_INACTIVE_FILE || lru == LRU_ACTIVE_FILE);
277}
278
279static inline int is_active_lru(enum lru_list lru)
280{
281 return (lru == LRU_ACTIVE_ANON || lru == LRU_ACTIVE_FILE);
282}
283
284struct zone_reclaim_stat {
285
286
287
288
289
290
291
292
293 unsigned long recent_rotated[2];
294 unsigned long recent_scanned[2];
295};
296
297struct lruvec {
298 struct list_head lists[NR_LRU_LISTS];
299 struct zone_reclaim_stat reclaim_stat;
300
301 atomic_long_t inactive_age;
302
303 unsigned long refaults;
304#ifdef CONFIG_MEMCG
305 struct pglist_data *pgdat;
306#endif
307};
308
309
310#define ISOLATE_UNMAPPED ((__force isolate_mode_t)0x2)
311
312#define ISOLATE_ASYNC_MIGRATE ((__force isolate_mode_t)0x4)
313
314#define ISOLATE_UNEVICTABLE ((__force isolate_mode_t)0x8)
315
316
317typedef unsigned __bitwise isolate_mode_t;
318
319enum zone_watermarks {
320 WMARK_MIN,
321 WMARK_LOW,
322 WMARK_HIGH,
323 NR_WMARK
324};
325
326#define min_wmark_pages(z) (z->_watermark[WMARK_MIN] + z->watermark_boost)
327#define low_wmark_pages(z) (z->_watermark[WMARK_LOW] + z->watermark_boost)
328#define high_wmark_pages(z) (z->_watermark[WMARK_HIGH] + z->watermark_boost)
329#define wmark_pages(z, i) (z->_watermark[i] + z->watermark_boost)
330
331struct per_cpu_pages {
332 int count;
333 int high;
334 int batch;
335
336
337 struct list_head lists[MIGRATE_PCPTYPES];
338};
339
340struct per_cpu_pageset {
341 struct per_cpu_pages pcp;
342#ifdef CONFIG_NUMA
343 s8 expire;
344 u16 vm_numa_stat_diff[NR_VM_NUMA_STAT_ITEMS];
345#endif
346#ifdef CONFIG_SMP
347 s8 stat_threshold;
348 s8 vm_stat_diff[NR_VM_ZONE_STAT_ITEMS];
349#endif
350};
351
352struct per_cpu_nodestat {
353 s8 stat_threshold;
354 s8 vm_node_stat_diff[NR_VM_NODE_STAT_ITEMS];
355};
356
357#endif
358
359enum zone_type {
360#ifdef CONFIG_ZONE_DMA
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379 ZONE_DMA,
380#endif
381#ifdef CONFIG_ZONE_DMA32
382
383
384
385
386
387 ZONE_DMA32,
388#endif
389
390
391
392
393
394 ZONE_NORMAL,
395#ifdef CONFIG_HIGHMEM
396
397
398
399
400
401
402
403
404 ZONE_HIGHMEM,
405#endif
406 ZONE_MOVABLE,
407#ifdef CONFIG_ZONE_DEVICE
408 ZONE_DEVICE,
409#endif
410 __MAX_NR_ZONES
411
412};
413
414#ifndef __GENERATING_BOUNDS_H
415
416struct zone {
417
418
419
420 unsigned long _watermark[NR_WMARK];
421 unsigned long watermark_boost;
422
423 unsigned long nr_reserved_highatomic;
424
425
426
427
428
429
430
431
432
433
434 long lowmem_reserve[MAX_NR_ZONES];
435
436#ifdef CONFIG_NUMA
437 int node;
438#endif
439 struct pglist_data *zone_pgdat;
440 struct per_cpu_pageset __percpu *pageset;
441
442#ifndef CONFIG_SPARSEMEM
443
444
445
446
447 unsigned long *pageblock_flags;
448#endif
449
450
451 unsigned long zone_start_pfn;
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488 atomic_long_t managed_pages;
489 unsigned long spanned_pages;
490 unsigned long present_pages;
491
492 const char *name;
493
494#ifdef CONFIG_MEMORY_ISOLATION
495
496
497
498
499
500 unsigned long nr_isolate_pageblock;
501#endif
502
503#ifdef CONFIG_MEMORY_HOTPLUG
504
505 seqlock_t span_seqlock;
506#endif
507
508 int initialized;
509
510
511 ZONE_PADDING(_pad1_)
512
513
514 struct free_area free_area[MAX_ORDER];
515
516
517 unsigned long flags;
518
519
520 spinlock_t lock;
521
522
523 ZONE_PADDING(_pad2_)
524
525
526
527
528
529
530 unsigned long percpu_drift_mark;
531
532#if defined CONFIG_COMPACTION || defined CONFIG_CMA
533
534 unsigned long compact_cached_free_pfn;
535
536 unsigned long compact_cached_migrate_pfn[2];
537 unsigned long compact_init_migrate_pfn;
538 unsigned long compact_init_free_pfn;
539#endif
540
541#ifdef CONFIG_COMPACTION
542
543
544
545
546
547 unsigned int compact_considered;
548 unsigned int compact_defer_shift;
549 int compact_order_failed;
550#endif
551
552#if defined CONFIG_COMPACTION || defined CONFIG_CMA
553
554 bool compact_blockskip_flush;
555#endif
556
557 bool contiguous;
558
559 ZONE_PADDING(_pad3_)
560
561 atomic_long_t vm_stat[NR_VM_ZONE_STAT_ITEMS];
562 atomic_long_t vm_numa_stat[NR_VM_NUMA_STAT_ITEMS];
563} ____cacheline_internodealigned_in_smp;
564
565enum pgdat_flags {
566 PGDAT_CONGESTED,
567
568
569 PGDAT_DIRTY,
570
571
572
573 PGDAT_WRITEBACK,
574
575
576 PGDAT_RECLAIM_LOCKED,
577};
578
579enum zone_flags {
580 ZONE_BOOSTED_WATERMARK,
581
582
583};
584
585static inline unsigned long zone_managed_pages(struct zone *zone)
586{
587 return (unsigned long)atomic_long_read(&zone->managed_pages);
588}
589
590static inline unsigned long zone_end_pfn(const struct zone *zone)
591{
592 return zone->zone_start_pfn + zone->spanned_pages;
593}
594
595static inline bool zone_spans_pfn(const struct zone *zone, unsigned long pfn)
596{
597 return zone->zone_start_pfn <= pfn && pfn < zone_end_pfn(zone);
598}
599
600static inline bool zone_is_initialized(struct zone *zone)
601{
602 return zone->initialized;
603}
604
605static inline bool zone_is_empty(struct zone *zone)
606{
607 return zone->spanned_pages == 0;
608}
609
610
611
612
613
614static inline bool zone_intersects(struct zone *zone,
615 unsigned long start_pfn, unsigned long nr_pages)
616{
617 if (zone_is_empty(zone))
618 return false;
619 if (start_pfn >= zone_end_pfn(zone) ||
620 start_pfn + nr_pages <= zone->zone_start_pfn)
621 return false;
622
623 return true;
624}
625
626
627
628
629
630
631#define DEF_PRIORITY 12
632
633
634#define MAX_ZONES_PER_ZONELIST (MAX_NUMNODES * MAX_NR_ZONES)
635
636enum {
637 ZONELIST_FALLBACK,
638#ifdef CONFIG_NUMA
639
640
641
642
643 ZONELIST_NOFALLBACK,
644#endif
645 MAX_ZONELISTS
646};
647
648
649
650
651
652struct zoneref {
653 struct zone *zone;
654 int zone_idx;
655};
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671struct zonelist {
672 struct zoneref _zonerefs[MAX_ZONES_PER_ZONELIST + 1];
673};
674
675#ifndef CONFIG_DISCONTIGMEM
676
677extern struct page *mem_map;
678#endif
679
680
681
682
683
684
685
686
687
688struct bootmem_data;
689typedef struct pglist_data {
690 struct zone node_zones[MAX_NR_ZONES];
691 struct zonelist node_zonelists[MAX_ZONELISTS];
692 int nr_zones;
693#ifdef CONFIG_FLAT_NODE_MEM_MAP
694 struct page *node_mem_map;
695#ifdef CONFIG_PAGE_EXTENSION
696 struct page_ext *node_page_ext;
697#endif
698#endif
699#if defined(CONFIG_MEMORY_HOTPLUG) || defined(CONFIG_DEFERRED_STRUCT_PAGE_INIT)
700
701
702
703
704
705
706
707
708
709
710 spinlock_t node_size_lock;
711#endif
712 unsigned long node_start_pfn;
713 unsigned long node_present_pages;
714 unsigned long node_spanned_pages;
715
716 int node_id;
717 wait_queue_head_t kswapd_wait;
718 wait_queue_head_t pfmemalloc_wait;
719 struct task_struct *kswapd;
720
721 int kswapd_order;
722 enum zone_type kswapd_classzone_idx;
723
724 int kswapd_failures;
725
726#ifdef CONFIG_COMPACTION
727 int kcompactd_max_order;
728 enum zone_type kcompactd_classzone_idx;
729 wait_queue_head_t kcompactd_wait;
730 struct task_struct *kcompactd;
731#endif
732
733
734
735
736 unsigned long totalreserve_pages;
737
738#ifdef CONFIG_NUMA
739
740
741
742 unsigned long min_unmapped_pages;
743 unsigned long min_slab_pages;
744#endif
745
746
747 ZONE_PADDING(_pad1_)
748 spinlock_t lru_lock;
749
750#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
751
752
753
754
755 unsigned long first_deferred_pfn;
756#endif
757
758#ifdef CONFIG_TRANSPARENT_HUGEPAGE
759 spinlock_t split_queue_lock;
760 struct list_head split_queue;
761 unsigned long split_queue_len;
762#endif
763
764
765 struct lruvec lruvec;
766
767 unsigned long flags;
768
769 ZONE_PADDING(_pad2_)
770
771
772 struct per_cpu_nodestat __percpu *per_cpu_nodestats;
773 atomic_long_t vm_stat[NR_VM_NODE_STAT_ITEMS];
774} pg_data_t;
775
776#define node_present_pages(nid) (NODE_DATA(nid)->node_present_pages)
777#define node_spanned_pages(nid) (NODE_DATA(nid)->node_spanned_pages)
778#ifdef CONFIG_FLAT_NODE_MEM_MAP
779#define pgdat_page_nr(pgdat, pagenr) ((pgdat)->node_mem_map + (pagenr))
780#else
781#define pgdat_page_nr(pgdat, pagenr) pfn_to_page((pgdat)->node_start_pfn + (pagenr))
782#endif
783#define nid_page_nr(nid, pagenr) pgdat_page_nr(NODE_DATA(nid),(pagenr))
784
785#define node_start_pfn(nid) (NODE_DATA(nid)->node_start_pfn)
786#define node_end_pfn(nid) pgdat_end_pfn(NODE_DATA(nid))
787
788static inline struct lruvec *node_lruvec(struct pglist_data *pgdat)
789{
790 return &pgdat->lruvec;
791}
792
793static inline unsigned long pgdat_end_pfn(pg_data_t *pgdat)
794{
795 return pgdat->node_start_pfn + pgdat->node_spanned_pages;
796}
797
798static inline bool pgdat_is_empty(pg_data_t *pgdat)
799{
800 return !pgdat->node_start_pfn && !pgdat->node_spanned_pages;
801}
802
803#include <linux/memory_hotplug.h>
804
805void build_all_zonelists(pg_data_t *pgdat);
806void wakeup_kswapd(struct zone *zone, gfp_t gfp_mask, int order,
807 enum zone_type classzone_idx);
808bool __zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark,
809 int classzone_idx, unsigned int alloc_flags,
810 long free_pages);
811bool zone_watermark_ok(struct zone *z, unsigned int order,
812 unsigned long mark, int classzone_idx,
813 unsigned int alloc_flags);
814bool zone_watermark_ok_safe(struct zone *z, unsigned int order,
815 unsigned long mark, int classzone_idx);
816enum memmap_context {
817 MEMMAP_EARLY,
818 MEMMAP_HOTPLUG,
819};
820extern void init_currently_empty_zone(struct zone *zone, unsigned long start_pfn,
821 unsigned long size);
822
823extern void lruvec_init(struct lruvec *lruvec);
824
825static inline struct pglist_data *lruvec_pgdat(struct lruvec *lruvec)
826{
827#ifdef CONFIG_MEMCG
828 return lruvec->pgdat;
829#else
830 return container_of(lruvec, struct pglist_data, lruvec);
831#endif
832}
833
834extern unsigned long lruvec_lru_size(struct lruvec *lruvec, enum lru_list lru, int zone_idx);
835
836#ifdef CONFIG_HAVE_MEMORY_PRESENT
837void memory_present(int nid, unsigned long start, unsigned long end);
838#else
839static inline void memory_present(int nid, unsigned long start, unsigned long end) {}
840#endif
841
842#if defined(CONFIG_SPARSEMEM)
843void memblocks_present(void);
844#else
845static inline void memblocks_present(void) {}
846#endif
847
848#ifdef CONFIG_HAVE_MEMORYLESS_NODES
849int local_memory_node(int node_id);
850#else
851static inline int local_memory_node(int node_id) { return node_id; };
852#endif
853
854
855
856
857#define zone_idx(zone) ((zone) - (zone)->zone_pgdat->node_zones)
858
859
860
861
862
863
864
865static inline bool managed_zone(struct zone *zone)
866{
867 return zone_managed_pages(zone);
868}
869
870
871static inline bool populated_zone(struct zone *zone)
872{
873 return zone->present_pages;
874}
875
876#ifdef CONFIG_NUMA
877static inline int zone_to_nid(struct zone *zone)
878{
879 return zone->node;
880}
881
882static inline void zone_set_nid(struct zone *zone, int nid)
883{
884 zone->node = nid;
885}
886#else
887static inline int zone_to_nid(struct zone *zone)
888{
889 return 0;
890}
891
892static inline void zone_set_nid(struct zone *zone, int nid) {}
893#endif
894
895extern int movable_zone;
896
897#ifdef CONFIG_HIGHMEM
898static inline int zone_movable_is_highmem(void)
899{
900#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
901 return movable_zone == ZONE_HIGHMEM;
902#else
903 return (ZONE_MOVABLE - 1) == ZONE_HIGHMEM;
904#endif
905}
906#endif
907
908static inline int is_highmem_idx(enum zone_type idx)
909{
910#ifdef CONFIG_HIGHMEM
911 return (idx == ZONE_HIGHMEM ||
912 (idx == ZONE_MOVABLE && zone_movable_is_highmem()));
913#else
914 return 0;
915#endif
916}
917
918
919
920
921
922
923
924static inline int is_highmem(struct zone *zone)
925{
926#ifdef CONFIG_HIGHMEM
927 return is_highmem_idx(zone_idx(zone));
928#else
929 return 0;
930#endif
931}
932
933
934struct ctl_table;
935int min_free_kbytes_sysctl_handler(struct ctl_table *, int,
936 void __user *, size_t *, loff_t *);
937int watermark_boost_factor_sysctl_handler(struct ctl_table *, int,
938 void __user *, size_t *, loff_t *);
939int watermark_scale_factor_sysctl_handler(struct ctl_table *, int,
940 void __user *, size_t *, loff_t *);
941extern int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES];
942int lowmem_reserve_ratio_sysctl_handler(struct ctl_table *, int,
943 void __user *, size_t *, loff_t *);
944int percpu_pagelist_fraction_sysctl_handler(struct ctl_table *, int,
945 void __user *, size_t *, loff_t *);
946int sysctl_min_unmapped_ratio_sysctl_handler(struct ctl_table *, int,
947 void __user *, size_t *, loff_t *);
948int sysctl_min_slab_ratio_sysctl_handler(struct ctl_table *, int,
949 void __user *, size_t *, loff_t *);
950
951extern int numa_zonelist_order_handler(struct ctl_table *, int,
952 void __user *, size_t *, loff_t *);
953extern char numa_zonelist_order[];
954#define NUMA_ZONELIST_ORDER_LEN 16
955
956#ifndef CONFIG_NEED_MULTIPLE_NODES
957
958extern struct pglist_data contig_page_data;
959#define NODE_DATA(nid) (&contig_page_data)
960#define NODE_MEM_MAP(nid) mem_map
961
962#else
963
964#include <asm/mmzone.h>
965
966#endif
967
968extern struct pglist_data *first_online_pgdat(void);
969extern struct pglist_data *next_online_pgdat(struct pglist_data *pgdat);
970extern struct zone *next_zone(struct zone *zone);
971
972
973
974
975
976#define for_each_online_pgdat(pgdat) \
977 for (pgdat = first_online_pgdat(); \
978 pgdat; \
979 pgdat = next_online_pgdat(pgdat))
980
981
982
983
984
985
986
987#define for_each_zone(zone) \
988 for (zone = (first_online_pgdat())->node_zones; \
989 zone; \
990 zone = next_zone(zone))
991
992#define for_each_populated_zone(zone) \
993 for (zone = (first_online_pgdat())->node_zones; \
994 zone; \
995 zone = next_zone(zone)) \
996 if (!populated_zone(zone)) \
997 ; \
998 else
999
1000static inline struct zone *zonelist_zone(struct zoneref *zoneref)
1001{
1002 return zoneref->zone;
1003}
1004
1005static inline int zonelist_zone_idx(struct zoneref *zoneref)
1006{
1007 return zoneref->zone_idx;
1008}
1009
1010static inline int zonelist_node_idx(struct zoneref *zoneref)
1011{
1012 return zone_to_nid(zoneref->zone);
1013}
1014
1015struct zoneref *__next_zones_zonelist(struct zoneref *z,
1016 enum zone_type highest_zoneidx,
1017 nodemask_t *nodes);
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031static __always_inline struct zoneref *next_zones_zonelist(struct zoneref *z,
1032 enum zone_type highest_zoneidx,
1033 nodemask_t *nodes)
1034{
1035 if (likely(!nodes && zonelist_zone_idx(z) <= highest_zoneidx))
1036 return z;
1037 return __next_zones_zonelist(z, highest_zoneidx, nodes);
1038}
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056static inline struct zoneref *first_zones_zonelist(struct zonelist *zonelist,
1057 enum zone_type highest_zoneidx,
1058 nodemask_t *nodes)
1059{
1060 return next_zones_zonelist(zonelist->_zonerefs,
1061 highest_zoneidx, nodes);
1062}
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075#define for_each_zone_zonelist_nodemask(zone, z, zlist, highidx, nodemask) \
1076 for (z = first_zones_zonelist(zlist, highidx, nodemask), zone = zonelist_zone(z); \
1077 zone; \
1078 z = next_zones_zonelist(++z, highidx, nodemask), \
1079 zone = zonelist_zone(z))
1080
1081#define for_next_zone_zonelist_nodemask(zone, z, zlist, highidx, nodemask) \
1082 for (zone = z->zone; \
1083 zone; \
1084 z = next_zones_zonelist(++z, highidx, nodemask), \
1085 zone = zonelist_zone(z))
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097#define for_each_zone_zonelist(zone, z, zlist, highidx) \
1098 for_each_zone_zonelist_nodemask(zone, z, zlist, highidx, NULL)
1099
1100#ifdef CONFIG_SPARSEMEM
1101#include <asm/sparsemem.h>
1102#endif
1103
1104#if !defined(CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID) && \
1105 !defined(CONFIG_HAVE_MEMBLOCK_NODE_MAP)
1106static inline unsigned long early_pfn_to_nid(unsigned long pfn)
1107{
1108 BUILD_BUG_ON(IS_ENABLED(CONFIG_NUMA));
1109 return 0;
1110}
1111#endif
1112
1113#ifdef CONFIG_FLATMEM
1114#define pfn_to_nid(pfn) (0)
1115#endif
1116
1117#ifdef CONFIG_SPARSEMEM
1118
1119
1120
1121
1122
1123
1124
1125#define PA_SECTION_SHIFT (SECTION_SIZE_BITS)
1126#define PFN_SECTION_SHIFT (SECTION_SIZE_BITS - PAGE_SHIFT)
1127
1128#define NR_MEM_SECTIONS (1UL << SECTIONS_SHIFT)
1129
1130#define PAGES_PER_SECTION (1UL << PFN_SECTION_SHIFT)
1131#define PAGE_SECTION_MASK (~(PAGES_PER_SECTION-1))
1132
1133#define SECTION_BLOCKFLAGS_BITS \
1134 ((1UL << (PFN_SECTION_SHIFT - pageblock_order)) * NR_PAGEBLOCK_BITS)
1135
1136#if (MAX_ORDER - 1 + PAGE_SHIFT) > SECTION_SIZE_BITS
1137#error Allocator MAX_ORDER exceeds SECTION_SIZE
1138#endif
1139
1140static inline unsigned long pfn_to_section_nr(unsigned long pfn)
1141{
1142 return pfn >> PFN_SECTION_SHIFT;
1143}
1144static inline unsigned long section_nr_to_pfn(unsigned long sec)
1145{
1146 return sec << PFN_SECTION_SHIFT;
1147}
1148
1149#define SECTION_ALIGN_UP(pfn) (((pfn) + PAGES_PER_SECTION - 1) & PAGE_SECTION_MASK)
1150#define SECTION_ALIGN_DOWN(pfn) ((pfn) & PAGE_SECTION_MASK)
1151
1152#define SUBSECTION_SHIFT 21
1153
1154#define PFN_SUBSECTION_SHIFT (SUBSECTION_SHIFT - PAGE_SHIFT)
1155#define PAGES_PER_SUBSECTION (1UL << PFN_SUBSECTION_SHIFT)
1156#define PAGE_SUBSECTION_MASK (~(PAGES_PER_SUBSECTION-1))
1157
1158#if SUBSECTION_SHIFT > SECTION_SIZE_BITS
1159#error Subsection size exceeds section size
1160#else
1161#define SUBSECTIONS_PER_SECTION (1UL << (SECTION_SIZE_BITS - SUBSECTION_SHIFT))
1162#endif
1163
1164#define SUBSECTION_ALIGN_UP(pfn) ALIGN((pfn), PAGES_PER_SUBSECTION)
1165#define SUBSECTION_ALIGN_DOWN(pfn) ((pfn) & PAGE_SUBSECTION_MASK)
1166
1167struct mem_section_usage {
1168 DECLARE_BITMAP(subsection_map, SUBSECTIONS_PER_SECTION);
1169
1170 unsigned long pageblock_flags[0];
1171};
1172
1173void subsection_map_init(unsigned long pfn, unsigned long nr_pages);
1174
1175struct page;
1176struct page_ext;
1177struct mem_section {
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190 unsigned long section_mem_map;
1191
1192 struct mem_section_usage *usage;
1193#ifdef CONFIG_PAGE_EXTENSION
1194
1195
1196
1197
1198 struct page_ext *page_ext;
1199 unsigned long pad;
1200#endif
1201
1202
1203
1204
1205};
1206
1207#ifdef CONFIG_SPARSEMEM_EXTREME
1208#define SECTIONS_PER_ROOT (PAGE_SIZE / sizeof (struct mem_section))
1209#else
1210#define SECTIONS_PER_ROOT 1
1211#endif
1212
1213#define SECTION_NR_TO_ROOT(sec) ((sec) / SECTIONS_PER_ROOT)
1214#define NR_SECTION_ROOTS DIV_ROUND_UP(NR_MEM_SECTIONS, SECTIONS_PER_ROOT)
1215#define SECTION_ROOT_MASK (SECTIONS_PER_ROOT - 1)
1216
1217#ifdef CONFIG_SPARSEMEM_EXTREME
1218extern struct mem_section **mem_section;
1219#else
1220extern struct mem_section mem_section[NR_SECTION_ROOTS][SECTIONS_PER_ROOT];
1221#endif
1222
1223static inline unsigned long *section_to_usemap(struct mem_section *ms)
1224{
1225 return ms->usage->pageblock_flags;
1226}
1227
1228static inline struct mem_section *__nr_to_section(unsigned long nr)
1229{
1230#ifdef CONFIG_SPARSEMEM_EXTREME
1231 if (!mem_section)
1232 return NULL;
1233#endif
1234 if (!mem_section[SECTION_NR_TO_ROOT(nr)])
1235 return NULL;
1236 return &mem_section[SECTION_NR_TO_ROOT(nr)][nr & SECTION_ROOT_MASK];
1237}
1238extern unsigned long __section_nr(struct mem_section *ms);
1239extern size_t mem_section_usage_size(void);
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254#define SECTION_MARKED_PRESENT (1UL<<0)
1255#define SECTION_HAS_MEM_MAP (1UL<<1)
1256#define SECTION_IS_ONLINE (1UL<<2)
1257#define SECTION_IS_EARLY (1UL<<3)
1258#define SECTION_MAP_LAST_BIT (1UL<<4)
1259#define SECTION_MAP_MASK (~(SECTION_MAP_LAST_BIT-1))
1260#define SECTION_NID_SHIFT 3
1261
1262static inline struct page *__section_mem_map_addr(struct mem_section *section)
1263{
1264 unsigned long map = section->section_mem_map;
1265 map &= SECTION_MAP_MASK;
1266 return (struct page *)map;
1267}
1268
1269static inline int present_section(struct mem_section *section)
1270{
1271 return (section && (section->section_mem_map & SECTION_MARKED_PRESENT));
1272}
1273
1274static inline int present_section_nr(unsigned long nr)
1275{
1276 return present_section(__nr_to_section(nr));
1277}
1278
1279static inline int valid_section(struct mem_section *section)
1280{
1281 return (section && (section->section_mem_map & SECTION_HAS_MEM_MAP));
1282}
1283
1284static inline int early_section(struct mem_section *section)
1285{
1286 return (section && (section->section_mem_map & SECTION_IS_EARLY));
1287}
1288
1289static inline int valid_section_nr(unsigned long nr)
1290{
1291 return valid_section(__nr_to_section(nr));
1292}
1293
1294static inline int online_section(struct mem_section *section)
1295{
1296 return (section && (section->section_mem_map & SECTION_IS_ONLINE));
1297}
1298
1299static inline int online_section_nr(unsigned long nr)
1300{
1301 return online_section(__nr_to_section(nr));
1302}
1303
1304#ifdef CONFIG_MEMORY_HOTPLUG
1305void online_mem_sections(unsigned long start_pfn, unsigned long end_pfn);
1306#ifdef CONFIG_MEMORY_HOTREMOVE
1307void offline_mem_sections(unsigned long start_pfn, unsigned long end_pfn);
1308#endif
1309#endif
1310
1311static inline struct mem_section *__pfn_to_section(unsigned long pfn)
1312{
1313 return __nr_to_section(pfn_to_section_nr(pfn));
1314}
1315
1316extern unsigned long __highest_present_section_nr;
1317
1318static inline int subsection_map_index(unsigned long pfn)
1319{
1320 return (pfn & ~(PAGE_SECTION_MASK)) / PAGES_PER_SUBSECTION;
1321}
1322
1323#ifdef CONFIG_SPARSEMEM_VMEMMAP
1324static inline int pfn_section_valid(struct mem_section *ms, unsigned long pfn)
1325{
1326 int idx = subsection_map_index(pfn);
1327
1328 return test_bit(idx, ms->usage->subsection_map);
1329}
1330#else
1331static inline int pfn_section_valid(struct mem_section *ms, unsigned long pfn)
1332{
1333 return 1;
1334}
1335#endif
1336
1337#ifndef CONFIG_HAVE_ARCH_PFN_VALID
1338static inline int pfn_valid(unsigned long pfn)
1339{
1340 struct mem_section *ms;
1341
1342 if (pfn_to_section_nr(pfn) >= NR_MEM_SECTIONS)
1343 return 0;
1344 ms = __nr_to_section(pfn_to_section_nr(pfn));
1345 if (!valid_section(ms))
1346 return 0;
1347
1348
1349
1350
1351 return early_section(ms) || pfn_section_valid(ms, pfn);
1352}
1353#endif
1354
1355static inline int pfn_present(unsigned long pfn)
1356{
1357 if (pfn_to_section_nr(pfn) >= NR_MEM_SECTIONS)
1358 return 0;
1359 return present_section(__nr_to_section(pfn_to_section_nr(pfn)));
1360}
1361
1362
1363
1364
1365
1366
1367#ifdef CONFIG_NUMA
1368#define pfn_to_nid(pfn) \
1369({ \
1370 unsigned long __pfn_to_nid_pfn = (pfn); \
1371 page_to_nid(pfn_to_page(__pfn_to_nid_pfn)); \
1372})
1373#else
1374#define pfn_to_nid(pfn) (0)
1375#endif
1376
1377#define early_pfn_valid(pfn) pfn_valid(pfn)
1378void sparse_init(void);
1379#else
1380#define sparse_init() do {} while (0)
1381#define sparse_index_init(_sec, _nid) do {} while (0)
1382#define pfn_present pfn_valid
1383#define subsection_map_init(_pfn, _nr_pages) do {} while (0)
1384#endif
1385
1386
1387
1388
1389
1390
1391struct mminit_pfnnid_cache {
1392 unsigned long last_start;
1393 unsigned long last_end;
1394 int last_nid;
1395};
1396
1397#ifndef early_pfn_valid
1398#define early_pfn_valid(pfn) (1)
1399#endif
1400
1401void memory_present(int nid, unsigned long start, unsigned long end);
1402
1403
1404
1405
1406
1407
1408
1409#ifdef CONFIG_HOLES_IN_ZONE
1410#define pfn_valid_within(pfn) pfn_valid(pfn)
1411#else
1412#define pfn_valid_within(pfn) (1)
1413#endif
1414
1415#ifdef CONFIG_ARCH_HAS_HOLES_MEMORYMODEL
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436bool memmap_valid_within(unsigned long pfn,
1437 struct page *page, struct zone *zone);
1438#else
1439static inline bool memmap_valid_within(unsigned long pfn,
1440 struct page *page, struct zone *zone)
1441{
1442 return true;
1443}
1444#endif
1445
1446#endif
1447#endif
1448#endif
1449