1
2
3
4
5
6
7
8
9
10
11#ifndef __MM_INTERNAL_H
12#define __MM_INTERNAL_H
13
14#include <linux/mm.h>
15
16void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *start_vma,
17 unsigned long floor, unsigned long ceiling);
18
19static inline void set_page_count(struct page *page, int v)
20{
21 atomic_set(&page->_count, v);
22}
23
24
25
26
27
28static inline void set_page_refcounted(struct page *page)
29{
30 VM_BUG_ON(PageTail(page));
31 VM_BUG_ON(atomic_read(&page->_count));
32 set_page_count(page, 1);
33}
34
35static inline void __put_page(struct page *page)
36{
37 atomic_dec(&page->_count);
38}
39
40extern unsigned long highest_memmap_pfn;
41
42
43
44
45extern int isolate_lru_page(struct page *page);
46extern void putback_lru_page(struct page *page);
47
48
49
50
51extern void __free_pages_bootmem(struct page *page, unsigned int order);
52extern void prep_compound_page(struct page *page, unsigned long order);
53
54
55
56
57
58
59
60static inline unsigned long page_order(struct page *page)
61{
62 VM_BUG_ON(!PageBuddy(page));
63 return page_private(page);
64}
65
66#ifdef CONFIG_HAVE_MLOCK
67extern long mlock_vma_pages_range(struct vm_area_struct *vma,
68 unsigned long start, unsigned long end);
69extern void munlock_vma_pages_range(struct vm_area_struct *vma,
70 unsigned long start, unsigned long end);
71static inline void munlock_vma_pages_all(struct vm_area_struct *vma)
72{
73 munlock_vma_pages_range(vma, vma->vm_start, vma->vm_end);
74}
75#endif
76
77
78
79
80
81
82
83static inline void unevictable_migrate_page(struct page *new, struct page *old)
84{
85 if (TestClearPageUnevictable(old))
86 SetPageUnevictable(new);
87}
88
89#ifdef CONFIG_HAVE_MLOCKED_PAGE_BIT
90
91
92
93
94
95static inline int is_mlocked_vma(struct vm_area_struct *vma, struct page *page)
96{
97 VM_BUG_ON(PageLRU(page));
98
99 if (likely((vma->vm_flags & (VM_LOCKED | VM_SPECIAL)) != VM_LOCKED))
100 return 0;
101
102 if (!TestSetPageMlocked(page)) {
103 inc_zone_page_state(page, NR_MLOCK);
104 count_vm_event(UNEVICTABLE_PGMLOCKED);
105 }
106 return 1;
107}
108
109
110
111
112extern void mlock_vma_page(struct page *page);
113
114
115
116
117
118
119
120
121
122
123extern void __clear_page_mlock(struct page *page);
124static inline void clear_page_mlock(struct page *page)
125{
126 if (unlikely(TestClearPageMlocked(page)))
127 __clear_page_mlock(page);
128}
129
130
131
132
133
134static inline void mlock_migrate_page(struct page *newpage, struct page *page)
135{
136 if (TestClearPageMlocked(page)) {
137 unsigned long flags;
138
139 local_irq_save(flags);
140 __dec_zone_page_state(page, NR_MLOCK);
141 SetPageMlocked(newpage);
142 __inc_zone_page_state(newpage, NR_MLOCK);
143 local_irq_restore(flags);
144 }
145}
146
147#else
148static inline int is_mlocked_vma(struct vm_area_struct *v, struct page *p)
149{
150 return 0;
151}
152static inline void clear_page_mlock(struct page *page) { }
153static inline void mlock_vma_page(struct page *page) { }
154static inline void mlock_migrate_page(struct page *new, struct page *old) { }
155
156#endif
157
158
159
160
161
162
163static inline struct page *mem_map_offset(struct page *base, int offset)
164{
165 if (unlikely(offset >= MAX_ORDER_NR_PAGES))
166 return pfn_to_page(page_to_pfn(base) + offset);
167 return base + offset;
168}
169
170
171
172
173
174static inline struct page *mem_map_next(struct page *iter,
175 struct page *base, int offset)
176{
177 if (unlikely((offset & (MAX_ORDER_NR_PAGES - 1)) == 0)) {
178 unsigned long pfn = page_to_pfn(base) + offset;
179 if (!pfn_valid(pfn))
180 return NULL;
181 return pfn_to_page(pfn);
182 }
183 return iter + 1;
184}
185
186
187
188
189
190
191
192#ifdef CONFIG_SPARSEMEM
193#define __paginginit __meminit
194#else
195#define __paginginit __init
196#endif
197
198
199enum mminit_level {
200 MMINIT_WARNING,
201 MMINIT_VERIFY,
202 MMINIT_TRACE
203};
204
205#ifdef CONFIG_DEBUG_MEMORY_INIT
206
207extern int mminit_loglevel;
208
209#define mminit_dprintk(level, prefix, fmt, arg...) \
210do { \
211 if (level < mminit_loglevel) { \
212 printk(level <= MMINIT_WARNING ? KERN_WARNING : KERN_DEBUG); \
213 printk(KERN_CONT "mminit::" prefix " " fmt, ##arg); \
214 } \
215} while (0)
216
217extern void mminit_verify_pageflags_layout(void);
218extern void mminit_verify_page_links(struct page *page,
219 enum zone_type zone, unsigned long nid, unsigned long pfn);
220extern void mminit_verify_zonelist(void);
221
222#else
223
224static inline void mminit_dprintk(enum mminit_level level,
225 const char *prefix, const char *fmt, ...)
226{
227}
228
229static inline void mminit_verify_pageflags_layout(void)
230{
231}
232
233static inline void mminit_verify_page_links(struct page *page,
234 enum zone_type zone, unsigned long nid, unsigned long pfn)
235{
236}
237
238static inline void mminit_verify_zonelist(void)
239{
240}
241#endif
242
243
244#if defined(CONFIG_SPARSEMEM)
245extern void mminit_validate_memmodel_limits(unsigned long *start_pfn,
246 unsigned long *end_pfn);
247#else
248static inline void mminit_validate_memmodel_limits(unsigned long *start_pfn,
249 unsigned long *end_pfn)
250{
251}
252#endif
253
254int __get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
255 unsigned long start, int len, unsigned int foll_flags,
256 struct page **pages, struct vm_area_struct **vmas);
257
258#define ZONE_RECLAIM_NOSCAN -2
259#define ZONE_RECLAIM_FULL -1
260#define ZONE_RECLAIM_SOME 0
261#define ZONE_RECLAIM_SUCCESS 1
262#endif
263