1#ifndef _LINUX_MM_H
2#define _LINUX_MM_H
3
4#include <linux/errno.h>
5
6#ifdef __KERNEL__
7
8#include <linux/mmdebug.h>
9#include <linux/gfp.h>
10#include <linux/bug.h>
11#include <linux/list.h>
12#include <linux/mmzone.h>
13#include <linux/rbtree.h>
14#include <linux/atomic.h>
15#include <linux/debug_locks.h>
16#include <linux/mm_types.h>
17#include <linux/range.h>
18#include <linux/pfn.h>
19#include <linux/bit_spinlock.h>
20#include <linux/shrinker.h>
21#include <linux/resource.h>
22#include <linux/page_ext.h>
23#include <linux/err.h>
24
25struct mempolicy;
26struct anon_vma;
27struct anon_vma_chain;
28struct file_ra_state;
29struct user_struct;
30struct writeback_control;
31struct bdi_writeback;
32
33#ifndef CONFIG_NEED_MULTIPLE_NODES
34extern unsigned long max_mapnr;
35
36static inline void set_max_mapnr(unsigned long limit)
37{
38 max_mapnr = limit;
39}
40#else
41static inline void set_max_mapnr(unsigned long limit) { }
42#endif
43
44extern unsigned long totalram_pages;
45extern void * high_memory;
46extern int page_cluster;
47
48#ifdef CONFIG_SYSCTL
49extern int sysctl_legacy_va_layout;
50#else
51#define sysctl_legacy_va_layout 0
52#endif
53
54#include <asm/page.h>
55#include <asm/pgtable.h>
56#include <asm/processor.h>
57
58#ifndef __pa_symbol
59#define __pa_symbol(x) __pa(RELOC_HIDE((unsigned long)(x), 0))
60#endif
61
62
63
64
65
66
67
68
69#ifndef mm_forbids_zeropage
70#define mm_forbids_zeropage(X) (0)
71#endif
72
73extern unsigned long sysctl_user_reserve_kbytes;
74extern unsigned long sysctl_admin_reserve_kbytes;
75
76extern int sysctl_overcommit_memory;
77extern int sysctl_overcommit_ratio;
78extern unsigned long sysctl_overcommit_kbytes;
79
80extern int overcommit_ratio_handler(struct ctl_table *, int, void __user *,
81 size_t *, loff_t *);
82extern int overcommit_kbytes_handler(struct ctl_table *, int, void __user *,
83 size_t *, loff_t *);
84
85#define nth_page(page,n) pfn_to_page(page_to_pfn((page)) + (n))
86
87
88#define PAGE_ALIGN(addr) ALIGN(addr, PAGE_SIZE)
89
90
91#define PAGE_ALIGNED(addr) IS_ALIGNED((unsigned long)addr, PAGE_SIZE)
92
93
94
95
96
97
98
99
100
101
102extern struct kmem_cache *vm_area_cachep;
103
104#ifndef CONFIG_MMU
105extern struct rb_root nommu_region_tree;
106extern struct rw_semaphore nommu_region_sem;
107
108extern unsigned int kobjsize(const void *objp);
109#endif
110
111
112
113
114#define VM_NONE 0x00000000
115
116#define VM_READ 0x00000001
117#define VM_WRITE 0x00000002
118#define VM_EXEC 0x00000004
119#define VM_SHARED 0x00000008
120
121
122#define VM_MAYREAD 0x00000010
123#define VM_MAYWRITE 0x00000020
124#define VM_MAYEXEC 0x00000040
125#define VM_MAYSHARE 0x00000080
126
127#define VM_GROWSDOWN 0x00000100
128#define VM_UFFD_MISSING 0x00000200
129#define VM_PFNMAP 0x00000400
130#define VM_DENYWRITE 0x00000800
131#define VM_UFFD_WP 0x00001000
132
133#define VM_LOCKED 0x00002000
134#define VM_IO 0x00004000
135
136
137#define VM_SEQ_READ 0x00008000
138#define VM_RAND_READ 0x00010000
139
140#define VM_DONTCOPY 0x00020000
141#define VM_DONTEXPAND 0x00040000
142#define VM_LOCKONFAULT 0x00080000
143#define VM_ACCOUNT 0x00100000
144#define VM_NORESERVE 0x00200000
145#define VM_HUGETLB 0x00400000
146#define VM_ARCH_1 0x01000000
147#define VM_ARCH_2 0x02000000
148#define VM_DONTDUMP 0x04000000
149
150#ifdef CONFIG_MEM_SOFT_DIRTY
151# define VM_SOFTDIRTY 0x08000000
152#else
153# define VM_SOFTDIRTY 0
154#endif
155
156#define VM_MIXEDMAP 0x10000000
157#define VM_HUGEPAGE 0x20000000
158#define VM_NOHUGEPAGE 0x40000000
159#define VM_MERGEABLE 0x80000000
160
161#if defined(CONFIG_X86)
162# define VM_PAT VM_ARCH_1
163#elif defined(CONFIG_PPC)
164# define VM_SAO VM_ARCH_1
165#elif defined(CONFIG_PARISC)
166# define VM_GROWSUP VM_ARCH_1
167#elif defined(CONFIG_METAG)
168# define VM_GROWSUP VM_ARCH_1
169#elif defined(CONFIG_IA64)
170# define VM_GROWSUP VM_ARCH_1
171#elif !defined(CONFIG_MMU)
172# define VM_MAPPED_COPY VM_ARCH_1
173#endif
174
175#if defined(CONFIG_X86)
176
177# define VM_MPX VM_ARCH_2
178#endif
179
180#ifndef VM_GROWSUP
181# define VM_GROWSUP VM_NONE
182#endif
183
184
185#define VM_STACK_INCOMPLETE_SETUP (VM_RAND_READ | VM_SEQ_READ)
186
187#ifndef VM_STACK_DEFAULT_FLAGS
188#define VM_STACK_DEFAULT_FLAGS VM_DATA_DEFAULT_FLAGS
189#endif
190
191#ifdef CONFIG_STACK_GROWSUP
192#define VM_STACK_FLAGS (VM_GROWSUP | VM_STACK_DEFAULT_FLAGS | VM_ACCOUNT)
193#else
194#define VM_STACK_FLAGS (VM_GROWSDOWN | VM_STACK_DEFAULT_FLAGS | VM_ACCOUNT)
195#endif
196
197
198
199
200
201#define VM_SPECIAL (VM_IO | VM_DONTEXPAND | VM_PFNMAP | VM_MIXEDMAP)
202
203
204#define VM_INIT_DEF_MASK VM_NOHUGEPAGE
205
206
207#define VM_LOCKED_CLEAR_MASK (~(VM_LOCKED | VM_LOCKONFAULT))
208
209
210
211
212
213extern pgprot_t protection_map[16];
214
215#define FAULT_FLAG_WRITE 0x01
216#define FAULT_FLAG_MKWRITE 0x02
217#define FAULT_FLAG_ALLOW_RETRY 0x04
218#define FAULT_FLAG_RETRY_NOWAIT 0x08
219#define FAULT_FLAG_KILLABLE 0x10
220#define FAULT_FLAG_TRIED 0x20
221#define FAULT_FLAG_USER 0x40
222
223
224
225
226
227
228
229
230struct vm_fault {
231 unsigned int flags;
232 pgoff_t pgoff;
233 void __user *virtual_address;
234
235 struct page *cow_page;
236 struct page *page;
237
238
239
240
241
242 pgoff_t max_pgoff;
243
244 pte_t *pte;
245};
246
247
248
249
250
251
252struct vm_operations_struct {
253 void (*open)(struct vm_area_struct * area);
254 void (*close)(struct vm_area_struct * area);
255 int (*mremap)(struct vm_area_struct * area);
256 int (*fault)(struct vm_area_struct *vma, struct vm_fault *vmf);
257 int (*pmd_fault)(struct vm_area_struct *, unsigned long address,
258 pmd_t *, unsigned int flags);
259 void (*map_pages)(struct vm_area_struct *vma, struct vm_fault *vmf);
260
261
262
263 int (*page_mkwrite)(struct vm_area_struct *vma, struct vm_fault *vmf);
264
265
266 int (*pfn_mkwrite)(struct vm_area_struct *vma, struct vm_fault *vmf);
267
268
269
270
271 int (*access)(struct vm_area_struct *vma, unsigned long addr,
272 void *buf, int len, int write);
273
274
275
276
277 const char *(*name)(struct vm_area_struct *vma);
278
279#ifdef CONFIG_NUMA
280
281
282
283
284
285
286
287 int (*set_policy)(struct vm_area_struct *vma, struct mempolicy *new);
288
289
290
291
292
293
294
295
296
297
298
299 struct mempolicy *(*get_policy)(struct vm_area_struct *vma,
300 unsigned long addr);
301#endif
302
303
304
305
306
307 struct page *(*find_special_page)(struct vm_area_struct *vma,
308 unsigned long addr);
309};
310
311struct mmu_gather;
312struct inode;
313
314#define page_private(page) ((page)->private)
315#define set_page_private(page, v) ((page)->private = (v))
316
317
318
319
320
321#include <linux/page-flags.h>
322#include <linux/huge_mm.h>
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340static inline int put_page_testzero(struct page *page)
341{
342 VM_BUG_ON_PAGE(atomic_read(&page->_count) == 0, page);
343 return atomic_dec_and_test(&page->_count);
344}
345
346
347
348
349
350
351
352static inline int get_page_unless_zero(struct page *page)
353{
354 return atomic_inc_not_zero(&page->_count);
355}
356
357extern int page_is_ram(unsigned long pfn);
358
359enum {
360 REGION_INTERSECTS,
361 REGION_DISJOINT,
362 REGION_MIXED,
363};
364
365int region_intersects(resource_size_t offset, size_t size, const char *type);
366
367
368struct page *vmalloc_to_page(const void *addr);
369unsigned long vmalloc_to_pfn(const void *addr);
370
371
372
373
374
375
376
377static inline int is_vmalloc_addr(const void *x)
378{
379#ifdef CONFIG_MMU
380 unsigned long addr = (unsigned long)x;
381
382 return addr >= VMALLOC_START && addr < VMALLOC_END;
383#else
384 return 0;
385#endif
386}
387#ifdef CONFIG_MMU
388extern int is_vmalloc_or_module_addr(const void *x);
389#else
390static inline int is_vmalloc_or_module_addr(const void *x)
391{
392 return 0;
393}
394#endif
395
396extern void kvfree(const void *addr);
397
398static inline void compound_lock(struct page *page)
399{
400#ifdef CONFIG_TRANSPARENT_HUGEPAGE
401 VM_BUG_ON_PAGE(PageSlab(page), page);
402 bit_spin_lock(PG_compound_lock, &page->flags);
403#endif
404}
405
406static inline void compound_unlock(struct page *page)
407{
408#ifdef CONFIG_TRANSPARENT_HUGEPAGE
409 VM_BUG_ON_PAGE(PageSlab(page), page);
410 bit_spin_unlock(PG_compound_lock, &page->flags);
411#endif
412}
413
414static inline unsigned long compound_lock_irqsave(struct page *page)
415{
416 unsigned long uninitialized_var(flags);
417#ifdef CONFIG_TRANSPARENT_HUGEPAGE
418 local_irq_save(flags);
419 compound_lock(page);
420#endif
421 return flags;
422}
423
424static inline void compound_unlock_irqrestore(struct page *page,
425 unsigned long flags)
426{
427#ifdef CONFIG_TRANSPARENT_HUGEPAGE
428 compound_unlock(page);
429 local_irq_restore(flags);
430#endif
431}
432
433
434
435
436
437
438static inline void page_mapcount_reset(struct page *page)
439{
440 atomic_set(&(page)->_mapcount, -1);
441}
442
443static inline int page_mapcount(struct page *page)
444{
445 VM_BUG_ON_PAGE(PageSlab(page), page);
446 return atomic_read(&page->_mapcount) + 1;
447}
448
449static inline int page_count(struct page *page)
450{
451 return atomic_read(&compound_head(page)->_count);
452}
453
454static inline bool __compound_tail_refcounted(struct page *page)
455{
456 return PageAnon(page) && !PageSlab(page) && !PageHeadHuge(page);
457}
458
459
460
461
462
463
464
465
466
467static inline bool compound_tail_refcounted(struct page *page)
468{
469 VM_BUG_ON_PAGE(!PageHead(page), page);
470 return __compound_tail_refcounted(page);
471}
472
473static inline void get_huge_page_tail(struct page *page)
474{
475
476
477
478 VM_BUG_ON_PAGE(!PageTail(page), page);
479 VM_BUG_ON_PAGE(page_mapcount(page) < 0, page);
480 VM_BUG_ON_PAGE(atomic_read(&page->_count) != 0, page);
481 if (compound_tail_refcounted(compound_head(page)))
482 atomic_inc(&page->_mapcount);
483}
484
485extern bool __get_page_tail(struct page *page);
486
487static inline void get_page(struct page *page)
488{
489 if (unlikely(PageTail(page)))
490 if (likely(__get_page_tail(page)))
491 return;
492
493
494
495
496 VM_BUG_ON_PAGE(atomic_read(&page->_count) <= 0, page);
497 atomic_inc(&page->_count);
498}
499
500static inline struct page *virt_to_head_page(const void *x)
501{
502 struct page *page = virt_to_page(x);
503
504 return compound_head(page);
505}
506
507
508
509
510
511static inline void init_page_count(struct page *page)
512{
513 atomic_set(&page->_count, 1);
514}
515
516void put_page(struct page *page);
517void put_pages_list(struct list_head *pages);
518
519void split_page(struct page *page, unsigned int order);
520int split_free_page(struct page *page);
521
522
523
524
525
526
527typedef void compound_page_dtor(struct page *);
528
529
530enum compound_dtor_id {
531 NULL_COMPOUND_DTOR,
532 COMPOUND_PAGE_DTOR,
533#ifdef CONFIG_HUGETLB_PAGE
534 HUGETLB_PAGE_DTOR,
535#endif
536 NR_COMPOUND_DTORS,
537};
538extern compound_page_dtor * const compound_page_dtors[];
539
540static inline void set_compound_page_dtor(struct page *page,
541 enum compound_dtor_id compound_dtor)
542{
543 VM_BUG_ON_PAGE(compound_dtor >= NR_COMPOUND_DTORS, page);
544 page[1].compound_dtor = compound_dtor;
545}
546
547static inline compound_page_dtor *get_compound_page_dtor(struct page *page)
548{
549 VM_BUG_ON_PAGE(page[1].compound_dtor >= NR_COMPOUND_DTORS, page);
550 return compound_page_dtors[page[1].compound_dtor];
551}
552
553static inline unsigned int compound_order(struct page *page)
554{
555 if (!PageHead(page))
556 return 0;
557 return page[1].compound_order;
558}
559
560static inline void set_compound_order(struct page *page, unsigned int order)
561{
562 page[1].compound_order = order;
563}
564
565#ifdef CONFIG_MMU
566
567
568
569
570
571
572static inline pte_t maybe_mkwrite(pte_t pte, struct vm_area_struct *vma)
573{
574 if (likely(vma->vm_flags & VM_WRITE))
575 pte = pte_mkwrite(pte);
576 return pte;
577}
578
579void do_set_pte(struct vm_area_struct *vma, unsigned long address,
580 struct page *page, pte_t *pte, bool write, bool anon);
581#endif
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649#define SECTIONS_PGOFF ((sizeof(unsigned long)*8) - SECTIONS_WIDTH)
650#define NODES_PGOFF (SECTIONS_PGOFF - NODES_WIDTH)
651#define ZONES_PGOFF (NODES_PGOFF - ZONES_WIDTH)
652#define LAST_CPUPID_PGOFF (ZONES_PGOFF - LAST_CPUPID_WIDTH)
653
654
655
656
657
658
659#define SECTIONS_PGSHIFT (SECTIONS_PGOFF * (SECTIONS_WIDTH != 0))
660#define NODES_PGSHIFT (NODES_PGOFF * (NODES_WIDTH != 0))
661#define ZONES_PGSHIFT (ZONES_PGOFF * (ZONES_WIDTH != 0))
662#define LAST_CPUPID_PGSHIFT (LAST_CPUPID_PGOFF * (LAST_CPUPID_WIDTH != 0))
663
664
665#ifdef NODE_NOT_IN_PAGE_FLAGS
666#define ZONEID_SHIFT (SECTIONS_SHIFT + ZONES_SHIFT)
667#define ZONEID_PGOFF ((SECTIONS_PGOFF < ZONES_PGOFF)? \
668 SECTIONS_PGOFF : ZONES_PGOFF)
669#else
670#define ZONEID_SHIFT (NODES_SHIFT + ZONES_SHIFT)
671#define ZONEID_PGOFF ((NODES_PGOFF < ZONES_PGOFF)? \
672 NODES_PGOFF : ZONES_PGOFF)
673#endif
674
675#define ZONEID_PGSHIFT (ZONEID_PGOFF * (ZONEID_SHIFT != 0))
676
677#if SECTIONS_WIDTH+NODES_WIDTH+ZONES_WIDTH > BITS_PER_LONG - NR_PAGEFLAGS
678#error SECTIONS_WIDTH+NODES_WIDTH+ZONES_WIDTH > BITS_PER_LONG - NR_PAGEFLAGS
679#endif
680
681#define ZONES_MASK ((1UL << ZONES_WIDTH) - 1)
682#define NODES_MASK ((1UL << NODES_WIDTH) - 1)
683#define SECTIONS_MASK ((1UL << SECTIONS_WIDTH) - 1)
684#define LAST_CPUPID_MASK ((1UL << LAST_CPUPID_SHIFT) - 1)
685#define ZONEID_MASK ((1UL << ZONEID_SHIFT) - 1)
686
687static inline enum zone_type page_zonenum(const struct page *page)
688{
689 return (page->flags >> ZONES_PGSHIFT) & ZONES_MASK;
690}
691
692#if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP)
693#define SECTION_IN_PAGE_FLAGS
694#endif
695
696
697
698
699
700
701
702
703
704static inline int page_zone_id(struct page *page)
705{
706 return (page->flags >> ZONEID_PGSHIFT) & ZONEID_MASK;
707}
708
709static inline int zone_to_nid(struct zone *zone)
710{
711#ifdef CONFIG_NUMA
712 return zone->node;
713#else
714 return 0;
715#endif
716}
717
718#ifdef NODE_NOT_IN_PAGE_FLAGS
719extern int page_to_nid(const struct page *page);
720#else
721static inline int page_to_nid(const struct page *page)
722{
723 return (page->flags >> NODES_PGSHIFT) & NODES_MASK;
724}
725#endif
726
727#ifdef CONFIG_NUMA_BALANCING
728static inline int cpu_pid_to_cpupid(int cpu, int pid)
729{
730 return ((cpu & LAST__CPU_MASK) << LAST__PID_SHIFT) | (pid & LAST__PID_MASK);
731}
732
733static inline int cpupid_to_pid(int cpupid)
734{
735 return cpupid & LAST__PID_MASK;
736}
737
738static inline int cpupid_to_cpu(int cpupid)
739{
740 return (cpupid >> LAST__PID_SHIFT) & LAST__CPU_MASK;
741}
742
743static inline int cpupid_to_nid(int cpupid)
744{
745 return cpu_to_node(cpupid_to_cpu(cpupid));
746}
747
748static inline bool cpupid_pid_unset(int cpupid)
749{
750 return cpupid_to_pid(cpupid) == (-1 & LAST__PID_MASK);
751}
752
753static inline bool cpupid_cpu_unset(int cpupid)
754{
755 return cpupid_to_cpu(cpupid) == (-1 & LAST__CPU_MASK);
756}
757
758static inline bool __cpupid_match_pid(pid_t task_pid, int cpupid)
759{
760 return (task_pid & LAST__PID_MASK) == cpupid_to_pid(cpupid);
761}
762
763#define cpupid_match_pid(task, cpupid) __cpupid_match_pid(task->pid, cpupid)
764#ifdef LAST_CPUPID_NOT_IN_PAGE_FLAGS
765static inline int page_cpupid_xchg_last(struct page *page, int cpupid)
766{
767 return xchg(&page->_last_cpupid, cpupid & LAST_CPUPID_MASK);
768}
769
770static inline int page_cpupid_last(struct page *page)
771{
772 return page->_last_cpupid;
773}
774static inline void page_cpupid_reset_last(struct page *page)
775{
776 page->_last_cpupid = -1 & LAST_CPUPID_MASK;
777}
778#else
779static inline int page_cpupid_last(struct page *page)
780{
781 return (page->flags >> LAST_CPUPID_PGSHIFT) & LAST_CPUPID_MASK;
782}
783
784extern int page_cpupid_xchg_last(struct page *page, int cpupid);
785
786static inline void page_cpupid_reset_last(struct page *page)
787{
788 int cpupid = (1 << LAST_CPUPID_SHIFT) - 1;
789
790 page->flags &= ~(LAST_CPUPID_MASK << LAST_CPUPID_PGSHIFT);
791 page->flags |= (cpupid & LAST_CPUPID_MASK) << LAST_CPUPID_PGSHIFT;
792}
793#endif
794#else
795static inline int page_cpupid_xchg_last(struct page *page, int cpupid)
796{
797 return page_to_nid(page);
798}
799
800static inline int page_cpupid_last(struct page *page)
801{
802 return page_to_nid(page);
803}
804
805static inline int cpupid_to_nid(int cpupid)
806{
807 return -1;
808}
809
810static inline int cpupid_to_pid(int cpupid)
811{
812 return -1;
813}
814
815static inline int cpupid_to_cpu(int cpupid)
816{
817 return -1;
818}
819
820static inline int cpu_pid_to_cpupid(int nid, int pid)
821{
822 return -1;
823}
824
825static inline bool cpupid_pid_unset(int cpupid)
826{
827 return 1;
828}
829
830static inline void page_cpupid_reset_last(struct page *page)
831{
832}
833
834static inline bool cpupid_match_pid(struct task_struct *task, int cpupid)
835{
836 return false;
837}
838#endif
839
840static inline struct zone *page_zone(const struct page *page)
841{
842 return &NODE_DATA(page_to_nid(page))->node_zones[page_zonenum(page)];
843}
844
845#ifdef SECTION_IN_PAGE_FLAGS
846static inline void set_page_section(struct page *page, unsigned long section)
847{
848 page->flags &= ~(SECTIONS_MASK << SECTIONS_PGSHIFT);
849 page->flags |= (section & SECTIONS_MASK) << SECTIONS_PGSHIFT;
850}
851
852static inline unsigned long page_to_section(const struct page *page)
853{
854 return (page->flags >> SECTIONS_PGSHIFT) & SECTIONS_MASK;
855}
856#endif
857
858static inline void set_page_zone(struct page *page, enum zone_type zone)
859{
860 page->flags &= ~(ZONES_MASK << ZONES_PGSHIFT);
861 page->flags |= (zone & ZONES_MASK) << ZONES_PGSHIFT;
862}
863
864static inline void set_page_node(struct page *page, unsigned long node)
865{
866 page->flags &= ~(NODES_MASK << NODES_PGSHIFT);
867 page->flags |= (node & NODES_MASK) << NODES_PGSHIFT;
868}
869
870static inline void set_page_links(struct page *page, enum zone_type zone,
871 unsigned long node, unsigned long pfn)
872{
873 set_page_zone(page, zone);
874 set_page_node(page, node);
875#ifdef SECTION_IN_PAGE_FLAGS
876 set_page_section(page, pfn_to_section_nr(pfn));
877#endif
878}
879
880#ifdef CONFIG_MEMCG
881static inline struct mem_cgroup *page_memcg(struct page *page)
882{
883 return page->mem_cgroup;
884}
885
886static inline void set_page_memcg(struct page *page, struct mem_cgroup *memcg)
887{
888 page->mem_cgroup = memcg;
889}
890#else
891static inline struct mem_cgroup *page_memcg(struct page *page)
892{
893 return NULL;
894}
895
896static inline void set_page_memcg(struct page *page, struct mem_cgroup *memcg)
897{
898}
899#endif
900
901
902
903
904#include <linux/vmstat.h>
905
906static __always_inline void *lowmem_page_address(const struct page *page)
907{
908 return __va(PFN_PHYS(page_to_pfn(page)));
909}
910
911#if defined(CONFIG_HIGHMEM) && !defined(WANT_PAGE_VIRTUAL)
912#define HASHED_PAGE_VIRTUAL
913#endif
914
915#if defined(WANT_PAGE_VIRTUAL)
916static inline void *page_address(const struct page *page)
917{
918 return page->virtual;
919}
920static inline void set_page_address(struct page *page, void *address)
921{
922 page->virtual = address;
923}
924#define page_address_init() do { } while(0)
925#endif
926
927#if defined(HASHED_PAGE_VIRTUAL)
928void *page_address(const struct page *page);
929void set_page_address(struct page *page, void *virtual);
930void page_address_init(void);
931#endif
932
933#if !defined(HASHED_PAGE_VIRTUAL) && !defined(WANT_PAGE_VIRTUAL)
934#define page_address(page) lowmem_page_address(page)
935#define set_page_address(page, address) do { } while(0)
936#define page_address_init() do { } while(0)
937#endif
938
939extern void *page_rmapping(struct page *page);
940extern struct anon_vma *page_anon_vma(struct page *page);
941extern struct address_space *page_mapping(struct page *page);
942
943extern struct address_space *__page_file_mapping(struct page *);
944
945static inline
946struct address_space *page_file_mapping(struct page *page)
947{
948 if (unlikely(PageSwapCache(page)))
949 return __page_file_mapping(page);
950
951 return page->mapping;
952}
953
954
955
956
957
958static inline pgoff_t page_index(struct page *page)
959{
960 if (unlikely(PageSwapCache(page)))
961 return page_private(page);
962 return page->index;
963}
964
965extern pgoff_t __page_file_index(struct page *page);
966
967
968
969
970
971static inline pgoff_t page_file_index(struct page *page)
972{
973 if (unlikely(PageSwapCache(page)))
974 return __page_file_index(page);
975
976 return page->index;
977}
978
979
980
981
982static inline int page_mapped(struct page *page)
983{
984 return atomic_read(&(page)->_mapcount) >= 0;
985}
986
987
988
989
990
991
992static inline bool page_is_pfmemalloc(struct page *page)
993{
994
995
996
997
998 return page->index == -1UL;
999}
1000
1001
1002
1003
1004
1005static inline void set_page_pfmemalloc(struct page *page)
1006{
1007 page->index = -1UL;
1008}
1009
1010static inline void clear_page_pfmemalloc(struct page *page)
1011{
1012 page->index = 0;
1013}
1014
1015
1016
1017
1018
1019
1020
1021#define VM_FAULT_MINOR 0
1022
1023#define VM_FAULT_OOM 0x0001
1024#define VM_FAULT_SIGBUS 0x0002
1025#define VM_FAULT_MAJOR 0x0004
1026#define VM_FAULT_WRITE 0x0008
1027#define VM_FAULT_HWPOISON 0x0010
1028#define VM_FAULT_HWPOISON_LARGE 0x0020
1029#define VM_FAULT_SIGSEGV 0x0040
1030
1031#define VM_FAULT_NOPAGE 0x0100
1032#define VM_FAULT_LOCKED 0x0200
1033#define VM_FAULT_RETRY 0x0400
1034#define VM_FAULT_FALLBACK 0x0800
1035
1036#define VM_FAULT_HWPOISON_LARGE_MASK 0xf000
1037
1038#define VM_FAULT_ERROR (VM_FAULT_OOM | VM_FAULT_SIGBUS | VM_FAULT_SIGSEGV | \
1039 VM_FAULT_HWPOISON | VM_FAULT_HWPOISON_LARGE | \
1040 VM_FAULT_FALLBACK)
1041
1042
1043#define VM_FAULT_SET_HINDEX(x) ((x) << 12)
1044#define VM_FAULT_GET_HINDEX(x) (((x) >> 12) & 0xf)
1045
1046
1047
1048
1049extern void pagefault_out_of_memory(void);
1050
1051#define offset_in_page(p) ((unsigned long)(p) & ~PAGE_MASK)
1052
1053
1054
1055
1056
1057#define SHOW_MEM_FILTER_NODES (0x0001u)
1058
1059extern void show_free_areas(unsigned int flags);
1060extern bool skip_free_areas_node(unsigned int flags, int nid);
1061
1062int shmem_zero_setup(struct vm_area_struct *);
1063#ifdef CONFIG_SHMEM
1064bool shmem_mapping(struct address_space *mapping);
1065#else
1066static inline bool shmem_mapping(struct address_space *mapping)
1067{
1068 return false;
1069}
1070#endif
1071
1072extern int can_do_mlock(void);
1073extern int user_shm_lock(size_t, struct user_struct *);
1074extern void user_shm_unlock(size_t, struct user_struct *);
1075
1076
1077
1078
1079struct zap_details {
1080 struct address_space *check_mapping;
1081 pgoff_t first_index;
1082 pgoff_t last_index;
1083};
1084
1085struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr,
1086 pte_t pte);
1087
1088int zap_vma_ptes(struct vm_area_struct *vma, unsigned long address,
1089 unsigned long size);
1090void zap_page_range(struct vm_area_struct *vma, unsigned long address,
1091 unsigned long size, struct zap_details *);
1092void unmap_vmas(struct mmu_gather *tlb, struct vm_area_struct *start_vma,
1093 unsigned long start, unsigned long end);
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115struct mm_walk {
1116 int (*pmd_entry)(pmd_t *pmd, unsigned long addr,
1117 unsigned long next, struct mm_walk *walk);
1118 int (*pte_entry)(pte_t *pte, unsigned long addr,
1119 unsigned long next, struct mm_walk *walk);
1120 int (*pte_hole)(unsigned long addr, unsigned long next,
1121 struct mm_walk *walk);
1122 int (*hugetlb_entry)(pte_t *pte, unsigned long hmask,
1123 unsigned long addr, unsigned long next,
1124 struct mm_walk *walk);
1125 int (*test_walk)(unsigned long addr, unsigned long next,
1126 struct mm_walk *walk);
1127 struct mm_struct *mm;
1128 struct vm_area_struct *vma;
1129 void *private;
1130};
1131
1132int walk_page_range(unsigned long addr, unsigned long end,
1133 struct mm_walk *walk);
1134int walk_page_vma(struct vm_area_struct *vma, struct mm_walk *walk);
1135void free_pgd_range(struct mmu_gather *tlb, unsigned long addr,
1136 unsigned long end, unsigned long floor, unsigned long ceiling);
1137int copy_page_range(struct mm_struct *dst, struct mm_struct *src,
1138 struct vm_area_struct *vma);
1139void unmap_mapping_range(struct address_space *mapping,
1140 loff_t const holebegin, loff_t const holelen, int even_cows);
1141int follow_pfn(struct vm_area_struct *vma, unsigned long address,
1142 unsigned long *pfn);
1143int follow_phys(struct vm_area_struct *vma, unsigned long address,
1144 unsigned int flags, unsigned long *prot, resource_size_t *phys);
1145int generic_access_phys(struct vm_area_struct *vma, unsigned long addr,
1146 void *buf, int len, int write);
1147
1148static inline void unmap_shared_mapping_range(struct address_space *mapping,
1149 loff_t const holebegin, loff_t const holelen)
1150{
1151 unmap_mapping_range(mapping, holebegin, holelen, 0);
1152}
1153
1154extern void truncate_pagecache(struct inode *inode, loff_t new);
1155extern void truncate_setsize(struct inode *inode, loff_t newsize);
1156void pagecache_isize_extended(struct inode *inode, loff_t from, loff_t to);
1157void truncate_pagecache_range(struct inode *inode, loff_t offset, loff_t end);
1158int truncate_inode_page(struct address_space *mapping, struct page *page);
1159int generic_error_remove_page(struct address_space *mapping, struct page *page);
1160int invalidate_inode_page(struct page *page);
1161
1162#ifdef CONFIG_MMU
1163extern int handle_mm_fault(struct mm_struct *mm, struct vm_area_struct *vma,
1164 unsigned long address, unsigned int flags);
1165extern int fixup_user_fault(struct task_struct *tsk, struct mm_struct *mm,
1166 unsigned long address, unsigned int fault_flags);
1167#else
1168static inline int handle_mm_fault(struct mm_struct *mm,
1169 struct vm_area_struct *vma, unsigned long address,
1170 unsigned int flags)
1171{
1172
1173 BUG();
1174 return VM_FAULT_SIGBUS;
1175}
1176static inline int fixup_user_fault(struct task_struct *tsk,
1177 struct mm_struct *mm, unsigned long address,
1178 unsigned int fault_flags)
1179{
1180
1181 BUG();
1182 return -EFAULT;
1183}
1184#endif
1185
1186extern int access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, int len, int write);
1187extern int access_remote_vm(struct mm_struct *mm, unsigned long addr,
1188 void *buf, int len, int write);
1189
1190long __get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
1191 unsigned long start, unsigned long nr_pages,
1192 unsigned int foll_flags, struct page **pages,
1193 struct vm_area_struct **vmas, int *nonblocking);
1194long get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
1195 unsigned long start, unsigned long nr_pages,
1196 int write, int force, struct page **pages,
1197 struct vm_area_struct **vmas);
1198long get_user_pages_locked(struct task_struct *tsk, struct mm_struct *mm,
1199 unsigned long start, unsigned long nr_pages,
1200 int write, int force, struct page **pages,
1201 int *locked);
1202long __get_user_pages_unlocked(struct task_struct *tsk, struct mm_struct *mm,
1203 unsigned long start, unsigned long nr_pages,
1204 int write, int force, struct page **pages,
1205 unsigned int gup_flags);
1206long get_user_pages_unlocked(struct task_struct *tsk, struct mm_struct *mm,
1207 unsigned long start, unsigned long nr_pages,
1208 int write, int force, struct page **pages);
1209int get_user_pages_fast(unsigned long start, int nr_pages, int write,
1210 struct page **pages);
1211
1212
1213struct frame_vector {
1214 unsigned int nr_allocated;
1215 unsigned int nr_frames;
1216 bool got_ref;
1217 bool is_pfns;
1218 void *ptrs[0];
1219
1220
1221};
1222
1223struct frame_vector *frame_vector_create(unsigned int nr_frames);
1224void frame_vector_destroy(struct frame_vector *vec);
1225int get_vaddr_frames(unsigned long start, unsigned int nr_pfns,
1226 bool write, bool force, struct frame_vector *vec);
1227void put_vaddr_frames(struct frame_vector *vec);
1228int frame_vector_to_pages(struct frame_vector *vec);
1229void frame_vector_to_pfns(struct frame_vector *vec);
1230
1231static inline unsigned int frame_vector_count(struct frame_vector *vec)
1232{
1233 return vec->nr_frames;
1234}
1235
1236static inline struct page **frame_vector_pages(struct frame_vector *vec)
1237{
1238 if (vec->is_pfns) {
1239 int err = frame_vector_to_pages(vec);
1240
1241 if (err)
1242 return ERR_PTR(err);
1243 }
1244 return (struct page **)(vec->ptrs);
1245}
1246
1247static inline unsigned long *frame_vector_pfns(struct frame_vector *vec)
1248{
1249 if (!vec->is_pfns)
1250 frame_vector_to_pfns(vec);
1251 return (unsigned long *)(vec->ptrs);
1252}
1253
1254struct kvec;
1255int get_kernel_pages(const struct kvec *iov, int nr_pages, int write,
1256 struct page **pages);
1257int get_kernel_page(unsigned long start, int write, struct page **pages);
1258struct page *get_dump_page(unsigned long addr);
1259
1260extern int try_to_release_page(struct page * page, gfp_t gfp_mask);
1261extern void do_invalidatepage(struct page *page, unsigned int offset,
1262 unsigned int length);
1263
1264int __set_page_dirty_nobuffers(struct page *page);
1265int __set_page_dirty_no_writeback(struct page *page);
1266int redirty_page_for_writepage(struct writeback_control *wbc,
1267 struct page *page);
1268void account_page_dirtied(struct page *page, struct address_space *mapping,
1269 struct mem_cgroup *memcg);
1270void account_page_cleaned(struct page *page, struct address_space *mapping,
1271 struct mem_cgroup *memcg, struct bdi_writeback *wb);
1272int set_page_dirty(struct page *page);
1273int set_page_dirty_lock(struct page *page);
1274void cancel_dirty_page(struct page *page);
1275int clear_page_dirty_for_io(struct page *page);
1276
1277int get_cmdline(struct task_struct *task, char *buffer, int buflen);
1278
1279
1280static inline int vma_growsdown(struct vm_area_struct *vma, unsigned long addr)
1281{
1282 return vma && (vma->vm_end == addr) && (vma->vm_flags & VM_GROWSDOWN);
1283}
1284
1285static inline bool vma_is_anonymous(struct vm_area_struct *vma)
1286{
1287 return !vma->vm_ops;
1288}
1289
1290static inline int stack_guard_page_start(struct vm_area_struct *vma,
1291 unsigned long addr)
1292{
1293 return (vma->vm_flags & VM_GROWSDOWN) &&
1294 (vma->vm_start == addr) &&
1295 !vma_growsdown(vma->vm_prev, addr);
1296}
1297
1298
1299static inline int vma_growsup(struct vm_area_struct *vma, unsigned long addr)
1300{
1301 return vma && (vma->vm_start == addr) && (vma->vm_flags & VM_GROWSUP);
1302}
1303
1304static inline int stack_guard_page_end(struct vm_area_struct *vma,
1305 unsigned long addr)
1306{
1307 return (vma->vm_flags & VM_GROWSUP) &&
1308 (vma->vm_end == addr) &&
1309 !vma_growsup(vma->vm_next, addr);
1310}
1311
1312extern struct task_struct *task_of_stack(struct task_struct *task,
1313 struct vm_area_struct *vma, bool in_group);
1314
1315extern unsigned long move_page_tables(struct vm_area_struct *vma,
1316 unsigned long old_addr, struct vm_area_struct *new_vma,
1317 unsigned long new_addr, unsigned long len,
1318 bool need_rmap_locks);
1319extern unsigned long change_protection(struct vm_area_struct *vma, unsigned long start,
1320 unsigned long end, pgprot_t newprot,
1321 int dirty_accountable, int prot_numa);
1322extern int mprotect_fixup(struct vm_area_struct *vma,
1323 struct vm_area_struct **pprev, unsigned long start,
1324 unsigned long end, unsigned long newflags);
1325
1326
1327
1328
1329int __get_user_pages_fast(unsigned long start, int nr_pages, int write,
1330 struct page **pages);
1331
1332
1333
1334static inline unsigned long get_mm_counter(struct mm_struct *mm, int member)
1335{
1336 long val = atomic_long_read(&mm->rss_stat.count[member]);
1337
1338#ifdef SPLIT_RSS_COUNTING
1339
1340
1341
1342
1343 if (val < 0)
1344 val = 0;
1345#endif
1346 return (unsigned long)val;
1347}
1348
1349static inline void add_mm_counter(struct mm_struct *mm, int member, long value)
1350{
1351 atomic_long_add(value, &mm->rss_stat.count[member]);
1352}
1353
1354static inline void inc_mm_counter(struct mm_struct *mm, int member)
1355{
1356 atomic_long_inc(&mm->rss_stat.count[member]);
1357}
1358
1359static inline void dec_mm_counter(struct mm_struct *mm, int member)
1360{
1361 atomic_long_dec(&mm->rss_stat.count[member]);
1362}
1363
1364static inline unsigned long get_mm_rss(struct mm_struct *mm)
1365{
1366 return get_mm_counter(mm, MM_FILEPAGES) +
1367 get_mm_counter(mm, MM_ANONPAGES);
1368}
1369
1370static inline unsigned long get_mm_hiwater_rss(struct mm_struct *mm)
1371{
1372 return max(mm->hiwater_rss, get_mm_rss(mm));
1373}
1374
1375static inline unsigned long get_mm_hiwater_vm(struct mm_struct *mm)
1376{
1377 return max(mm->hiwater_vm, mm->total_vm);
1378}
1379
1380static inline void update_hiwater_rss(struct mm_struct *mm)
1381{
1382 unsigned long _rss = get_mm_rss(mm);
1383
1384 if ((mm)->hiwater_rss < _rss)
1385 (mm)->hiwater_rss = _rss;
1386}
1387
1388static inline void update_hiwater_vm(struct mm_struct *mm)
1389{
1390 if (mm->hiwater_vm < mm->total_vm)
1391 mm->hiwater_vm = mm->total_vm;
1392}
1393
1394static inline void reset_mm_hiwater_rss(struct mm_struct *mm)
1395{
1396 mm->hiwater_rss = get_mm_rss(mm);
1397}
1398
1399static inline void setmax_mm_hiwater_rss(unsigned long *maxrss,
1400 struct mm_struct *mm)
1401{
1402 unsigned long hiwater_rss = get_mm_hiwater_rss(mm);
1403
1404 if (*maxrss < hiwater_rss)
1405 *maxrss = hiwater_rss;
1406}
1407
1408#if defined(SPLIT_RSS_COUNTING)
1409void sync_mm_rss(struct mm_struct *mm);
1410#else
1411static inline void sync_mm_rss(struct mm_struct *mm)
1412{
1413}
1414#endif
1415
1416int vma_wants_writenotify(struct vm_area_struct *vma);
1417
1418extern pte_t *__get_locked_pte(struct mm_struct *mm, unsigned long addr,
1419 spinlock_t **ptl);
1420static inline pte_t *get_locked_pte(struct mm_struct *mm, unsigned long addr,
1421 spinlock_t **ptl)
1422{
1423 pte_t *ptep;
1424 __cond_lock(*ptl, ptep = __get_locked_pte(mm, addr, ptl));
1425 return ptep;
1426}
1427
1428#ifdef __PAGETABLE_PUD_FOLDED
1429static inline int __pud_alloc(struct mm_struct *mm, pgd_t *pgd,
1430 unsigned long address)
1431{
1432 return 0;
1433}
1434#else
1435int __pud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address);
1436#endif
1437
1438#if defined(__PAGETABLE_PMD_FOLDED) || !defined(CONFIG_MMU)
1439static inline int __pmd_alloc(struct mm_struct *mm, pud_t *pud,
1440 unsigned long address)
1441{
1442 return 0;
1443}
1444
1445static inline void mm_nr_pmds_init(struct mm_struct *mm) {}
1446
1447static inline unsigned long mm_nr_pmds(struct mm_struct *mm)
1448{
1449 return 0;
1450}
1451
1452static inline void mm_inc_nr_pmds(struct mm_struct *mm) {}
1453static inline void mm_dec_nr_pmds(struct mm_struct *mm) {}
1454
1455#else
1456int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address);
1457
1458static inline void mm_nr_pmds_init(struct mm_struct *mm)
1459{
1460 atomic_long_set(&mm->nr_pmds, 0);
1461}
1462
1463static inline unsigned long mm_nr_pmds(struct mm_struct *mm)
1464{
1465 return atomic_long_read(&mm->nr_pmds);
1466}
1467
1468static inline void mm_inc_nr_pmds(struct mm_struct *mm)
1469{
1470 atomic_long_inc(&mm->nr_pmds);
1471}
1472
1473static inline void mm_dec_nr_pmds(struct mm_struct *mm)
1474{
1475 atomic_long_dec(&mm->nr_pmds);
1476}
1477#endif
1478
1479int __pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma,
1480 pmd_t *pmd, unsigned long address);
1481int __pte_alloc_kernel(pmd_t *pmd, unsigned long address);
1482
1483
1484
1485
1486
1487#if defined(CONFIG_MMU) && !defined(__ARCH_HAS_4LEVEL_HACK)
1488static inline pud_t *pud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address)
1489{
1490 return (unlikely(pgd_none(*pgd)) && __pud_alloc(mm, pgd, address))?
1491 NULL: pud_offset(pgd, address);
1492}
1493
1494static inline pmd_t *pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address)
1495{
1496 return (unlikely(pud_none(*pud)) && __pmd_alloc(mm, pud, address))?
1497 NULL: pmd_offset(pud, address);
1498}
1499#endif
1500
1501#if USE_SPLIT_PTE_PTLOCKS
1502#if ALLOC_SPLIT_PTLOCKS
1503void __init ptlock_cache_init(void);
1504extern bool ptlock_alloc(struct page *page);
1505extern void ptlock_free(struct page *page);
1506
1507static inline spinlock_t *ptlock_ptr(struct page *page)
1508{
1509 return page->ptl;
1510}
1511#else
1512static inline void ptlock_cache_init(void)
1513{
1514}
1515
1516static inline bool ptlock_alloc(struct page *page)
1517{
1518 return true;
1519}
1520
1521static inline void ptlock_free(struct page *page)
1522{
1523}
1524
1525static inline spinlock_t *ptlock_ptr(struct page *page)
1526{
1527 return &page->ptl;
1528}
1529#endif
1530
1531static inline spinlock_t *pte_lockptr(struct mm_struct *mm, pmd_t *pmd)
1532{
1533 return ptlock_ptr(pmd_page(*pmd));
1534}
1535
1536static inline bool ptlock_init(struct page *page)
1537{
1538
1539
1540
1541
1542
1543
1544
1545 VM_BUG_ON_PAGE(*(unsigned long *)&page->ptl, page);
1546 if (!ptlock_alloc(page))
1547 return false;
1548 spin_lock_init(ptlock_ptr(page));
1549 return true;
1550}
1551
1552
1553static inline void pte_lock_deinit(struct page *page)
1554{
1555 page->mapping = NULL;
1556 ptlock_free(page);
1557}
1558
1559#else
1560
1561
1562
1563static inline spinlock_t *pte_lockptr(struct mm_struct *mm, pmd_t *pmd)
1564{
1565 return &mm->page_table_lock;
1566}
1567static inline void ptlock_cache_init(void) {}
1568static inline bool ptlock_init(struct page *page) { return true; }
1569static inline void pte_lock_deinit(struct page *page) {}
1570#endif
1571
1572static inline void pgtable_init(void)
1573{
1574 ptlock_cache_init();
1575 pgtable_cache_init();
1576}
1577
1578static inline bool pgtable_page_ctor(struct page *page)
1579{
1580 if (!ptlock_init(page))
1581 return false;
1582 inc_zone_page_state(page, NR_PAGETABLE);
1583 return true;
1584}
1585
1586static inline void pgtable_page_dtor(struct page *page)
1587{
1588 pte_lock_deinit(page);
1589 dec_zone_page_state(page, NR_PAGETABLE);
1590}
1591
1592#define pte_offset_map_lock(mm, pmd, address, ptlp) \
1593({ \
1594 spinlock_t *__ptl = pte_lockptr(mm, pmd); \
1595 pte_t *__pte = pte_offset_map(pmd, address); \
1596 *(ptlp) = __ptl; \
1597 spin_lock(__ptl); \
1598 __pte; \
1599})
1600
1601#define pte_unmap_unlock(pte, ptl) do { \
1602 spin_unlock(ptl); \
1603 pte_unmap(pte); \
1604} while (0)
1605
1606#define pte_alloc_map(mm, vma, pmd, address) \
1607 ((unlikely(pmd_none(*(pmd))) && __pte_alloc(mm, vma, \
1608 pmd, address))? \
1609 NULL: pte_offset_map(pmd, address))
1610
1611#define pte_alloc_map_lock(mm, pmd, address, ptlp) \
1612 ((unlikely(pmd_none(*(pmd))) && __pte_alloc(mm, NULL, \
1613 pmd, address))? \
1614 NULL: pte_offset_map_lock(mm, pmd, address, ptlp))
1615
1616#define pte_alloc_kernel(pmd, address) \
1617 ((unlikely(pmd_none(*(pmd))) && __pte_alloc_kernel(pmd, address))? \
1618 NULL: pte_offset_kernel(pmd, address))
1619
1620#if USE_SPLIT_PMD_PTLOCKS
1621
1622static struct page *pmd_to_page(pmd_t *pmd)
1623{
1624 unsigned long mask = ~(PTRS_PER_PMD * sizeof(pmd_t) - 1);
1625 return virt_to_page((void *)((unsigned long) pmd & mask));
1626}
1627
1628static inline spinlock_t *pmd_lockptr(struct mm_struct *mm, pmd_t *pmd)
1629{
1630 return ptlock_ptr(pmd_to_page(pmd));
1631}
1632
1633static inline bool pgtable_pmd_page_ctor(struct page *page)
1634{
1635#ifdef CONFIG_TRANSPARENT_HUGEPAGE
1636 page->pmd_huge_pte = NULL;
1637#endif
1638 return ptlock_init(page);
1639}
1640
1641static inline void pgtable_pmd_page_dtor(struct page *page)
1642{
1643#ifdef CONFIG_TRANSPARENT_HUGEPAGE
1644 VM_BUG_ON_PAGE(page->pmd_huge_pte, page);
1645#endif
1646 ptlock_free(page);
1647}
1648
1649#define pmd_huge_pte(mm, pmd) (pmd_to_page(pmd)->pmd_huge_pte)
1650
1651#else
1652
1653static inline spinlock_t *pmd_lockptr(struct mm_struct *mm, pmd_t *pmd)
1654{
1655 return &mm->page_table_lock;
1656}
1657
1658static inline bool pgtable_pmd_page_ctor(struct page *page) { return true; }
1659static inline void pgtable_pmd_page_dtor(struct page *page) {}
1660
1661#define pmd_huge_pte(mm, pmd) ((mm)->pmd_huge_pte)
1662
1663#endif
1664
1665static inline spinlock_t *pmd_lock(struct mm_struct *mm, pmd_t *pmd)
1666{
1667 spinlock_t *ptl = pmd_lockptr(mm, pmd);
1668 spin_lock(ptl);
1669 return ptl;
1670}
1671
1672extern void free_area_init(unsigned long * zones_size);
1673extern void free_area_init_node(int nid, unsigned long * zones_size,
1674 unsigned long zone_start_pfn, unsigned long *zholes_size);
1675extern void free_initmem(void);
1676
1677
1678
1679
1680
1681
1682
1683extern unsigned long free_reserved_area(void *start, void *end,
1684 int poison, char *s);
1685
1686#ifdef CONFIG_HIGHMEM
1687
1688
1689
1690
1691extern void free_highmem_page(struct page *page);
1692#endif
1693
1694extern void adjust_managed_page_count(struct page *page, long count);
1695extern void mem_init_print_info(const char *str);
1696
1697extern void reserve_bootmem_region(unsigned long start, unsigned long end);
1698
1699
1700static inline void __free_reserved_page(struct page *page)
1701{
1702 ClearPageReserved(page);
1703 init_page_count(page);
1704 __free_page(page);
1705}
1706
1707static inline void free_reserved_page(struct page *page)
1708{
1709 __free_reserved_page(page);
1710 adjust_managed_page_count(page, 1);
1711}
1712
1713static inline void mark_page_reserved(struct page *page)
1714{
1715 SetPageReserved(page);
1716 adjust_managed_page_count(page, -1);
1717}
1718
1719
1720
1721
1722
1723
1724
1725static inline unsigned long free_initmem_default(int poison)
1726{
1727 extern char __init_begin[], __init_end[];
1728
1729 return free_reserved_area(&__init_begin, &__init_end,
1730 poison, "unused kernel");
1731}
1732
1733static inline unsigned long get_num_physpages(void)
1734{
1735 int nid;
1736 unsigned long phys_pages = 0;
1737
1738 for_each_online_node(nid)
1739 phys_pages += node_present_pages(nid);
1740
1741 return phys_pages;
1742}
1743
1744#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771extern void free_area_init_nodes(unsigned long *max_zone_pfn);
1772unsigned long node_map_pfn_alignment(void);
1773unsigned long __absent_pages_in_range(int nid, unsigned long start_pfn,
1774 unsigned long end_pfn);
1775extern unsigned long absent_pages_in_range(unsigned long start_pfn,
1776 unsigned long end_pfn);
1777extern void get_pfn_range_for_nid(unsigned int nid,
1778 unsigned long *start_pfn, unsigned long *end_pfn);
1779extern unsigned long find_min_pfn_with_active_regions(void);
1780extern void free_bootmem_with_active_regions(int nid,
1781 unsigned long max_low_pfn);
1782extern void sparse_memory_present_with_active_regions(int nid);
1783
1784#endif
1785
1786#if !defined(CONFIG_HAVE_MEMBLOCK_NODE_MAP) && \
1787 !defined(CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID)
1788static inline int __early_pfn_to_nid(unsigned long pfn,
1789 struct mminit_pfnnid_cache *state)
1790{
1791 return 0;
1792}
1793#else
1794
1795extern int __meminit early_pfn_to_nid(unsigned long pfn);
1796
1797extern int __meminit __early_pfn_to_nid(unsigned long pfn,
1798 struct mminit_pfnnid_cache *state);
1799#endif
1800
1801extern void set_dma_reserve(unsigned long new_dma_reserve);
1802extern void memmap_init_zone(unsigned long, int, unsigned long,
1803 unsigned long, enum memmap_context);
1804extern void setup_per_zone_wmarks(void);
1805extern int __meminit init_per_zone_wmark_min(void);
1806extern void mem_init(void);
1807extern void __init mmap_init(void);
1808extern void show_mem(unsigned int flags);
1809extern void si_meminfo(struct sysinfo * val);
1810extern void si_meminfo_node(struct sysinfo *val, int nid);
1811
1812extern __printf(3, 4)
1813void warn_alloc_failed(gfp_t gfp_mask, unsigned int order,
1814 const char *fmt, ...);
1815
1816extern void setup_per_cpu_pageset(void);
1817
1818extern void zone_pcp_update(struct zone *zone);
1819extern void zone_pcp_reset(struct zone *zone);
1820
1821
1822extern int min_free_kbytes;
1823
1824
1825extern atomic_long_t mmap_pages_allocated;
1826extern int nommu_shrink_inode_mappings(struct inode *, size_t, size_t);
1827
1828
1829void vma_interval_tree_insert(struct vm_area_struct *node,
1830 struct rb_root *root);
1831void vma_interval_tree_insert_after(struct vm_area_struct *node,
1832 struct vm_area_struct *prev,
1833 struct rb_root *root);
1834void vma_interval_tree_remove(struct vm_area_struct *node,
1835 struct rb_root *root);
1836struct vm_area_struct *vma_interval_tree_iter_first(struct rb_root *root,
1837 unsigned long start, unsigned long last);
1838struct vm_area_struct *vma_interval_tree_iter_next(struct vm_area_struct *node,
1839 unsigned long start, unsigned long last);
1840
1841#define vma_interval_tree_foreach(vma, root, start, last) \
1842 for (vma = vma_interval_tree_iter_first(root, start, last); \
1843 vma; vma = vma_interval_tree_iter_next(vma, start, last))
1844
1845void anon_vma_interval_tree_insert(struct anon_vma_chain *node,
1846 struct rb_root *root);
1847void anon_vma_interval_tree_remove(struct anon_vma_chain *node,
1848 struct rb_root *root);
1849struct anon_vma_chain *anon_vma_interval_tree_iter_first(
1850 struct rb_root *root, unsigned long start, unsigned long last);
1851struct anon_vma_chain *anon_vma_interval_tree_iter_next(
1852 struct anon_vma_chain *node, unsigned long start, unsigned long last);
1853#ifdef CONFIG_DEBUG_VM_RB
1854void anon_vma_interval_tree_verify(struct anon_vma_chain *node);
1855#endif
1856
1857#define anon_vma_interval_tree_foreach(avc, root, start, last) \
1858 for (avc = anon_vma_interval_tree_iter_first(root, start, last); \
1859 avc; avc = anon_vma_interval_tree_iter_next(avc, start, last))
1860
1861
1862extern int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin);
1863extern int vma_adjust(struct vm_area_struct *vma, unsigned long start,
1864 unsigned long end, pgoff_t pgoff, struct vm_area_struct *insert);
1865extern struct vm_area_struct *vma_merge(struct mm_struct *,
1866 struct vm_area_struct *prev, unsigned long addr, unsigned long end,
1867 unsigned long vm_flags, struct anon_vma *, struct file *, pgoff_t,
1868 struct mempolicy *, struct vm_userfaultfd_ctx);
1869extern struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *);
1870extern int split_vma(struct mm_struct *,
1871 struct vm_area_struct *, unsigned long addr, int new_below);
1872extern int insert_vm_struct(struct mm_struct *, struct vm_area_struct *);
1873extern void __vma_link_rb(struct mm_struct *, struct vm_area_struct *,
1874 struct rb_node **, struct rb_node *);
1875extern void unlink_file_vma(struct vm_area_struct *);
1876extern struct vm_area_struct *copy_vma(struct vm_area_struct **,
1877 unsigned long addr, unsigned long len, pgoff_t pgoff,
1878 bool *need_rmap_locks);
1879extern void exit_mmap(struct mm_struct *);
1880
1881static inline int check_data_rlimit(unsigned long rlim,
1882 unsigned long new,
1883 unsigned long start,
1884 unsigned long end_data,
1885 unsigned long start_data)
1886{
1887 if (rlim < RLIM_INFINITY) {
1888 if (((new - start) + (end_data - start_data)) > rlim)
1889 return -ENOSPC;
1890 }
1891
1892 return 0;
1893}
1894
1895extern int mm_take_all_locks(struct mm_struct *mm);
1896extern void mm_drop_all_locks(struct mm_struct *mm);
1897
1898extern void set_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file);
1899extern struct file *get_mm_exe_file(struct mm_struct *mm);
1900
1901extern int may_expand_vm(struct mm_struct *mm, unsigned long npages);
1902extern struct vm_area_struct *_install_special_mapping(struct mm_struct *mm,
1903 unsigned long addr, unsigned long len,
1904 unsigned long flags,
1905 const struct vm_special_mapping *spec);
1906
1907extern int install_special_mapping(struct mm_struct *mm,
1908 unsigned long addr, unsigned long len,
1909 unsigned long flags, struct page **pages);
1910
1911extern unsigned long get_unmapped_area(struct file *, unsigned long, unsigned long, unsigned long, unsigned long);
1912
1913extern unsigned long mmap_region(struct file *file, unsigned long addr,
1914 unsigned long len, vm_flags_t vm_flags, unsigned long pgoff);
1915extern unsigned long do_mmap(struct file *file, unsigned long addr,
1916 unsigned long len, unsigned long prot, unsigned long flags,
1917 vm_flags_t vm_flags, unsigned long pgoff, unsigned long *populate);
1918extern int do_munmap(struct mm_struct *, unsigned long, size_t);
1919
1920static inline unsigned long
1921do_mmap_pgoff(struct file *file, unsigned long addr,
1922 unsigned long len, unsigned long prot, unsigned long flags,
1923 unsigned long pgoff, unsigned long *populate)
1924{
1925 return do_mmap(file, addr, len, prot, flags, 0, pgoff, populate);
1926}
1927
1928#ifdef CONFIG_MMU
1929extern int __mm_populate(unsigned long addr, unsigned long len,
1930 int ignore_errors);
1931static inline void mm_populate(unsigned long addr, unsigned long len)
1932{
1933
1934 (void) __mm_populate(addr, len, 1);
1935}
1936#else
1937static inline void mm_populate(unsigned long addr, unsigned long len) {}
1938#endif
1939
1940
1941extern unsigned long vm_brk(unsigned long, unsigned long);
1942extern int vm_munmap(unsigned long, size_t);
1943extern unsigned long vm_mmap(struct file *, unsigned long,
1944 unsigned long, unsigned long,
1945 unsigned long, unsigned long);
1946
1947struct vm_unmapped_area_info {
1948#define VM_UNMAPPED_AREA_TOPDOWN 1
1949 unsigned long flags;
1950 unsigned long length;
1951 unsigned long low_limit;
1952 unsigned long high_limit;
1953 unsigned long align_mask;
1954 unsigned long align_offset;
1955};
1956
1957extern unsigned long unmapped_area(struct vm_unmapped_area_info *info);
1958extern unsigned long unmapped_area_topdown(struct vm_unmapped_area_info *info);
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969static inline unsigned long
1970vm_unmapped_area(struct vm_unmapped_area_info *info)
1971{
1972 if (info->flags & VM_UNMAPPED_AREA_TOPDOWN)
1973 return unmapped_area_topdown(info);
1974 else
1975 return unmapped_area(info);
1976}
1977
1978
1979extern void truncate_inode_pages(struct address_space *, loff_t);
1980extern void truncate_inode_pages_range(struct address_space *,
1981 loff_t lstart, loff_t lend);
1982extern void truncate_inode_pages_final(struct address_space *);
1983
1984
1985extern int filemap_fault(struct vm_area_struct *, struct vm_fault *);
1986extern void filemap_map_pages(struct vm_area_struct *vma, struct vm_fault *vmf);
1987extern int filemap_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf);
1988
1989
1990int write_one_page(struct page *page, int wait);
1991void task_dirty_inc(struct task_struct *tsk);
1992
1993
1994#define VM_MAX_READAHEAD 128
1995#define VM_MIN_READAHEAD 16
1996
1997int force_page_cache_readahead(struct address_space *mapping, struct file *filp,
1998 pgoff_t offset, unsigned long nr_to_read);
1999
2000void page_cache_sync_readahead(struct address_space *mapping,
2001 struct file_ra_state *ra,
2002 struct file *filp,
2003 pgoff_t offset,
2004 unsigned long size);
2005
2006void page_cache_async_readahead(struct address_space *mapping,
2007 struct file_ra_state *ra,
2008 struct file *filp,
2009 struct page *pg,
2010 pgoff_t offset,
2011 unsigned long size);
2012
2013
2014extern int expand_stack(struct vm_area_struct *vma, unsigned long address);
2015
2016
2017extern int expand_downwards(struct vm_area_struct *vma,
2018 unsigned long address);
2019#if VM_GROWSUP
2020extern int expand_upwards(struct vm_area_struct *vma, unsigned long address);
2021#else
2022 #define expand_upwards(vma, address) (0)
2023#endif
2024
2025
2026extern struct vm_area_struct * find_vma(struct mm_struct * mm, unsigned long addr);
2027extern struct vm_area_struct * find_vma_prev(struct mm_struct * mm, unsigned long addr,
2028 struct vm_area_struct **pprev);
2029
2030
2031
2032static inline struct vm_area_struct * find_vma_intersection(struct mm_struct * mm, unsigned long start_addr, unsigned long end_addr)
2033{
2034 struct vm_area_struct * vma = find_vma(mm,start_addr);
2035
2036 if (vma && end_addr <= vma->vm_start)
2037 vma = NULL;
2038 return vma;
2039}
2040
2041static inline unsigned long vma_pages(struct vm_area_struct *vma)
2042{
2043 return (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
2044}
2045
2046
2047static inline struct vm_area_struct *find_exact_vma(struct mm_struct *mm,
2048 unsigned long vm_start, unsigned long vm_end)
2049{
2050 struct vm_area_struct *vma = find_vma(mm, vm_start);
2051
2052 if (vma && (vma->vm_start != vm_start || vma->vm_end != vm_end))
2053 vma = NULL;
2054
2055 return vma;
2056}
2057
2058#ifdef CONFIG_MMU
2059pgprot_t vm_get_page_prot(unsigned long vm_flags);
2060void vma_set_page_prot(struct vm_area_struct *vma);
2061#else
2062static inline pgprot_t vm_get_page_prot(unsigned long vm_flags)
2063{
2064 return __pgprot(0);
2065}
2066static inline void vma_set_page_prot(struct vm_area_struct *vma)
2067{
2068 vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
2069}
2070#endif
2071
2072#ifdef CONFIG_NUMA_BALANCING
2073unsigned long change_prot_numa(struct vm_area_struct *vma,
2074 unsigned long start, unsigned long end);
2075#endif
2076
2077struct vm_area_struct *find_extend_vma(struct mm_struct *, unsigned long addr);
2078int remap_pfn_range(struct vm_area_struct *, unsigned long addr,
2079 unsigned long pfn, unsigned long size, pgprot_t);
2080int vm_insert_page(struct vm_area_struct *, unsigned long addr, struct page *);
2081int vm_insert_pfn(struct vm_area_struct *vma, unsigned long addr,
2082 unsigned long pfn);
2083int vm_insert_mixed(struct vm_area_struct *vma, unsigned long addr,
2084 unsigned long pfn);
2085int vm_iomap_memory(struct vm_area_struct *vma, phys_addr_t start, unsigned long len);
2086
2087
2088struct page *follow_page_mask(struct vm_area_struct *vma,
2089 unsigned long address, unsigned int foll_flags,
2090 unsigned int *page_mask);
2091
2092static inline struct page *follow_page(struct vm_area_struct *vma,
2093 unsigned long address, unsigned int foll_flags)
2094{
2095 unsigned int unused_page_mask;
2096 return follow_page_mask(vma, address, foll_flags, &unused_page_mask);
2097}
2098
2099#define FOLL_WRITE 0x01
2100#define FOLL_TOUCH 0x02
2101#define FOLL_GET 0x04
2102#define FOLL_DUMP 0x08
2103#define FOLL_FORCE 0x10
2104#define FOLL_NOWAIT 0x20
2105
2106#define FOLL_POPULATE 0x40
2107#define FOLL_SPLIT 0x80
2108#define FOLL_HWPOISON 0x100
2109#define FOLL_NUMA 0x200
2110#define FOLL_MIGRATION 0x400
2111#define FOLL_TRIED 0x800
2112#define FOLL_MLOCK 0x1000
2113
2114typedef int (*pte_fn_t)(pte_t *pte, pgtable_t token, unsigned long addr,
2115 void *data);
2116extern int apply_to_page_range(struct mm_struct *mm, unsigned long address,
2117 unsigned long size, pte_fn_t fn, void *data);
2118
2119#ifdef CONFIG_PROC_FS
2120void vm_stat_account(struct mm_struct *, unsigned long, struct file *, long);
2121#else
2122static inline void vm_stat_account(struct mm_struct *mm,
2123 unsigned long flags, struct file *file, long pages)
2124{
2125 mm->total_vm += pages;
2126}
2127#endif
2128
2129#ifdef CONFIG_DEBUG_PAGEALLOC
2130extern bool _debug_pagealloc_enabled;
2131extern void __kernel_map_pages(struct page *page, int numpages, int enable);
2132
2133static inline bool debug_pagealloc_enabled(void)
2134{
2135 return _debug_pagealloc_enabled;
2136}
2137
2138static inline void
2139kernel_map_pages(struct page *page, int numpages, int enable)
2140{
2141 if (!debug_pagealloc_enabled())
2142 return;
2143
2144 __kernel_map_pages(page, numpages, enable);
2145}
2146#ifdef CONFIG_HIBERNATION
2147extern bool kernel_page_present(struct page *page);
2148#endif
2149#else
2150static inline void
2151kernel_map_pages(struct page *page, int numpages, int enable) {}
2152#ifdef CONFIG_HIBERNATION
2153static inline bool kernel_page_present(struct page *page) { return true; }
2154#endif
2155#endif
2156
2157#ifdef __HAVE_ARCH_GATE_AREA
2158extern struct vm_area_struct *get_gate_vma(struct mm_struct *mm);
2159extern int in_gate_area_no_mm(unsigned long addr);
2160extern int in_gate_area(struct mm_struct *mm, unsigned long addr);
2161#else
2162static inline struct vm_area_struct *get_gate_vma(struct mm_struct *mm)
2163{
2164 return NULL;
2165}
2166static inline int in_gate_area_no_mm(unsigned long addr) { return 0; }
2167static inline int in_gate_area(struct mm_struct *mm, unsigned long addr)
2168{
2169 return 0;
2170}
2171#endif
2172
2173#ifdef CONFIG_SYSCTL
2174extern int sysctl_drop_caches;
2175int drop_caches_sysctl_handler(struct ctl_table *, int,
2176 void __user *, size_t *, loff_t *);
2177#endif
2178
2179void drop_slab(void);
2180void drop_slab_node(int nid);
2181
2182#ifndef CONFIG_MMU
2183#define randomize_va_space 0
2184#else
2185extern int randomize_va_space;
2186#endif
2187
2188const char * arch_vma_name(struct vm_area_struct *vma);
2189void print_vma_addr(char *prefix, unsigned long rip);
2190
2191void sparse_mem_maps_populate_node(struct page **map_map,
2192 unsigned long pnum_begin,
2193 unsigned long pnum_end,
2194 unsigned long map_count,
2195 int nodeid);
2196
2197struct page *sparse_mem_map_populate(unsigned long pnum, int nid);
2198pgd_t *vmemmap_pgd_populate(unsigned long addr, int node);
2199pud_t *vmemmap_pud_populate(pgd_t *pgd, unsigned long addr, int node);
2200pmd_t *vmemmap_pmd_populate(pud_t *pud, unsigned long addr, int node);
2201pte_t *vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node);
2202void *vmemmap_alloc_block(unsigned long size, int node);
2203void *vmemmap_alloc_block_buf(unsigned long size, int node);
2204void vmemmap_verify(pte_t *, int, unsigned long, unsigned long);
2205int vmemmap_populate_basepages(unsigned long start, unsigned long end,
2206 int node);
2207int vmemmap_populate(unsigned long start, unsigned long end, int node);
2208void vmemmap_populate_print_last(void);
2209#ifdef CONFIG_MEMORY_HOTPLUG
2210void vmemmap_free(unsigned long start, unsigned long end);
2211#endif
2212void register_page_bootmem_memmap(unsigned long section_nr, struct page *map,
2213 unsigned long size);
2214
2215enum mf_flags {
2216 MF_COUNT_INCREASED = 1 << 0,
2217 MF_ACTION_REQUIRED = 1 << 1,
2218 MF_MUST_KILL = 1 << 2,
2219 MF_SOFT_OFFLINE = 1 << 3,
2220};
2221extern int memory_failure(unsigned long pfn, int trapno, int flags);
2222extern void memory_failure_queue(unsigned long pfn, int trapno, int flags);
2223extern int unpoison_memory(unsigned long pfn);
2224extern int get_hwpoison_page(struct page *page);
2225extern void put_hwpoison_page(struct page *page);
2226extern int sysctl_memory_failure_early_kill;
2227extern int sysctl_memory_failure_recovery;
2228extern void shake_page(struct page *p, int access);
2229extern atomic_long_t num_poisoned_pages;
2230extern int soft_offline_page(struct page *page, int flags);
2231
2232
2233
2234
2235
2236enum mf_result {
2237 MF_IGNORED,
2238 MF_FAILED,
2239 MF_DELAYED,
2240 MF_RECOVERED,
2241};
2242
2243enum mf_action_page_type {
2244 MF_MSG_KERNEL,
2245 MF_MSG_KERNEL_HIGH_ORDER,
2246 MF_MSG_SLAB,
2247 MF_MSG_DIFFERENT_COMPOUND,
2248 MF_MSG_POISONED_HUGE,
2249 MF_MSG_HUGE,
2250 MF_MSG_FREE_HUGE,
2251 MF_MSG_UNMAP_FAILED,
2252 MF_MSG_DIRTY_SWAPCACHE,
2253 MF_MSG_CLEAN_SWAPCACHE,
2254 MF_MSG_DIRTY_MLOCKED_LRU,
2255 MF_MSG_CLEAN_MLOCKED_LRU,
2256 MF_MSG_DIRTY_UNEVICTABLE_LRU,
2257 MF_MSG_CLEAN_UNEVICTABLE_LRU,
2258 MF_MSG_DIRTY_LRU,
2259 MF_MSG_CLEAN_LRU,
2260 MF_MSG_TRUNCATED_LRU,
2261 MF_MSG_BUDDY,
2262 MF_MSG_BUDDY_2ND,
2263 MF_MSG_UNKNOWN,
2264};
2265
2266#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLBFS)
2267extern void clear_huge_page(struct page *page,
2268 unsigned long addr,
2269 unsigned int pages_per_huge_page);
2270extern void copy_user_huge_page(struct page *dst, struct page *src,
2271 unsigned long addr, struct vm_area_struct *vma,
2272 unsigned int pages_per_huge_page);
2273#endif
2274
2275extern struct page_ext_operations debug_guardpage_ops;
2276extern struct page_ext_operations page_poisoning_ops;
2277
2278#ifdef CONFIG_DEBUG_PAGEALLOC
2279extern unsigned int _debug_guardpage_minorder;
2280extern bool _debug_guardpage_enabled;
2281
2282static inline unsigned int debug_guardpage_minorder(void)
2283{
2284 return _debug_guardpage_minorder;
2285}
2286
2287static inline bool debug_guardpage_enabled(void)
2288{
2289 return _debug_guardpage_enabled;
2290}
2291
2292static inline bool page_is_guard(struct page *page)
2293{
2294 struct page_ext *page_ext;
2295
2296 if (!debug_guardpage_enabled())
2297 return false;
2298
2299 page_ext = lookup_page_ext(page);
2300 return test_bit(PAGE_EXT_DEBUG_GUARD, &page_ext->flags);
2301}
2302#else
2303static inline unsigned int debug_guardpage_minorder(void) { return 0; }
2304static inline bool debug_guardpage_enabled(void) { return false; }
2305static inline bool page_is_guard(struct page *page) { return false; }
2306#endif
2307
2308#if MAX_NUMNODES > 1
2309void __init setup_nr_node_ids(void);
2310#else
2311static inline void setup_nr_node_ids(void) {}
2312#endif
2313
2314#endif
2315#endif
2316