1#ifndef _LINUX_HUGETLB_H
2#define _LINUX_HUGETLB_H
3
4#include <linux/mm_types.h>
5#include <linux/fs.h>
6#include <linux/hugetlb_inline.h>
7#include <linux/cgroup.h>
8
9struct ctl_table;
10struct user_struct;
11struct mmu_gather;
12
13#ifdef CONFIG_HUGETLB_PAGE
14
15#include <linux/mempolicy.h>
16#include <linux/shm.h>
17#include <asm/tlbflush.h>
18
19struct hugepage_subpool {
20 spinlock_t lock;
21 long count;
22 long max_hpages, used_hpages;
23};
24
25extern spinlock_t hugetlb_lock;
26extern int hugetlb_max_hstate __read_mostly;
27#define for_each_hstate(h) \
28 for ((h) = hstates; (h) < &hstates[hugetlb_max_hstate]; (h)++)
29
30struct hugepage_subpool *hugepage_new_subpool(long nr_blocks);
31void hugepage_put_subpool(struct hugepage_subpool *spool);
32
33int PageHuge(struct page *page);
34
35void reset_vma_resv_huge_pages(struct vm_area_struct *vma);
36int hugetlb_sysctl_handler(struct ctl_table *, int, void __user *, size_t *, loff_t *);
37int hugetlb_overcommit_handler(struct ctl_table *, int, void __user *, size_t *, loff_t *);
38int hugetlb_treat_movable_handler(struct ctl_table *, int, void __user *, size_t *, loff_t *);
39
40#ifdef CONFIG_NUMA
41int hugetlb_mempolicy_sysctl_handler(struct ctl_table *, int,
42 void __user *, size_t *, loff_t *);
43#endif
44
45int copy_hugetlb_page_range(struct mm_struct *, struct mm_struct *, struct vm_area_struct *);
46long follow_hugetlb_page(struct mm_struct *, struct vm_area_struct *,
47 struct page **, struct vm_area_struct **,
48 unsigned long *, unsigned long *, long, unsigned int);
49void unmap_hugepage_range(struct vm_area_struct *,
50 unsigned long, unsigned long, struct page *);
51void __unmap_hugepage_range_final(struct mmu_gather *tlb,
52 struct vm_area_struct *vma,
53 unsigned long start, unsigned long end,
54 struct page *ref_page);
55void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma,
56 unsigned long start, unsigned long end,
57 struct page *ref_page);
58void hugetlb_report_meminfo(struct seq_file *);
59int hugetlb_report_node_meminfo(int, char *);
60void hugetlb_show_meminfo(void);
61unsigned long hugetlb_total_pages(void);
62int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
63 unsigned long address, unsigned int flags);
64int hugetlb_reserve_pages(struct inode *inode, long from, long to,
65 struct vm_area_struct *vma,
66 vm_flags_t vm_flags);
67void hugetlb_unreserve_pages(struct inode *inode, long offset, long freed);
68int dequeue_hwpoisoned_huge_page(struct page *page);
69void copy_huge_page(struct page *dst, struct page *src);
70
71#ifdef CONFIG_ARCH_WANT_HUGE_PMD_SHARE
72pte_t *huge_pmd_share(struct mm_struct *mm, unsigned long addr, pud_t *pud);
73#endif
74
75extern unsigned long hugepages_treat_as_movable;
76extern const unsigned long hugetlb_zero, hugetlb_infinity;
77extern int sysctl_hugetlb_shm_group;
78extern struct list_head huge_boot_pages;
79
80
81
82pte_t *huge_pte_alloc(struct mm_struct *mm,
83 unsigned long addr, unsigned long sz);
84pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr);
85int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep);
86struct page *follow_huge_addr(struct mm_struct *mm, unsigned long address,
87 int write);
88struct page *follow_huge_pmd(struct mm_struct *mm, unsigned long address,
89 pmd_t *pmd, int write);
90struct page *follow_huge_pud(struct mm_struct *mm, unsigned long address,
91 pud_t *pud, int write);
92int pmd_huge(pmd_t pmd);
93int pud_huge(pud_t pmd);
94unsigned long hugetlb_change_protection(struct vm_area_struct *vma,
95 unsigned long address, unsigned long end, pgprot_t newprot);
96
97#else
98
99static inline int PageHuge(struct page *page)
100{
101 return 0;
102}
103
104static inline void reset_vma_resv_huge_pages(struct vm_area_struct *vma)
105{
106}
107
108static inline unsigned long hugetlb_total_pages(void)
109{
110 return 0;
111}
112
113#define follow_hugetlb_page(m,v,p,vs,a,b,i,w) ({ BUG(); 0; })
114#define follow_huge_addr(mm, addr, write) ERR_PTR(-EINVAL)
115#define copy_hugetlb_page_range(src, dst, vma) ({ BUG(); 0; })
116static inline void hugetlb_report_meminfo(struct seq_file *m)
117{
118}
119#define hugetlb_report_node_meminfo(n, buf) 0
120static inline void hugetlb_show_meminfo(void)
121{
122}
123#define follow_huge_pmd(mm, addr, pmd, write) NULL
124#define follow_huge_pud(mm, addr, pud, write) NULL
125#define prepare_hugepage_range(file, addr, len) (-EINVAL)
126#define pmd_huge(x) 0
127#define pud_huge(x) 0
128#define is_hugepage_only_range(mm, addr, len) 0
129#define hugetlb_free_pgd_range(tlb, addr, end, floor, ceiling) ({BUG(); 0; })
130#define hugetlb_fault(mm, vma, addr, flags) ({ BUG(); 0; })
131#define huge_pte_offset(mm, address) 0
132static inline int dequeue_hwpoisoned_huge_page(struct page *page)
133{
134 return 0;
135}
136
137static inline void copy_huge_page(struct page *dst, struct page *src)
138{
139}
140
141static inline unsigned long hugetlb_change_protection(struct vm_area_struct *vma,
142 unsigned long address, unsigned long end, pgprot_t newprot)
143{
144 return 0;
145}
146
147static inline void __unmap_hugepage_range_final(struct mmu_gather *tlb,
148 struct vm_area_struct *vma, unsigned long start,
149 unsigned long end, struct page *ref_page)
150{
151 BUG();
152}
153
154static inline void __unmap_hugepage_range(struct mmu_gather *tlb,
155 struct vm_area_struct *vma, unsigned long start,
156 unsigned long end, struct page *ref_page)
157{
158 BUG();
159}
160
161#endif
162
163#define HUGETLB_ANON_FILE "anon_hugepage"
164
165enum {
166
167
168
169
170 HUGETLB_SHMFS_INODE = 1,
171
172
173
174
175 HUGETLB_ANONHUGE_INODE = 2,
176};
177
178#ifdef CONFIG_HUGETLBFS
179struct hugetlbfs_sb_info {
180 long max_inodes;
181 long free_inodes;
182 spinlock_t stat_lock;
183 struct hstate *hstate;
184 struct hugepage_subpool *spool;
185};
186
187static inline struct hugetlbfs_sb_info *HUGETLBFS_SB(struct super_block *sb)
188{
189 return sb->s_fs_info;
190}
191
192extern const struct file_operations hugetlbfs_file_operations;
193extern const struct vm_operations_struct hugetlb_vm_ops;
194struct file *hugetlb_file_setup(const char *name, size_t size, vm_flags_t acct,
195 struct user_struct **user, int creat_flags,
196 int page_size_log);
197
198static inline int is_file_hugepages(struct file *file)
199{
200 if (file->f_op == &hugetlbfs_file_operations)
201 return 1;
202 if (is_file_shm_hugepages(file))
203 return 1;
204
205 return 0;
206}
207
208
209#else
210
211#define is_file_hugepages(file) 0
212static inline struct file *
213hugetlb_file_setup(const char *name, size_t size, vm_flags_t acctflag,
214 struct user_struct **user, int creat_flags,
215 int page_size_log)
216{
217 return ERR_PTR(-ENOSYS);
218}
219
220#endif
221
222#ifdef HAVE_ARCH_HUGETLB_UNMAPPED_AREA
223unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
224 unsigned long len, unsigned long pgoff,
225 unsigned long flags);
226#endif
227
228#ifdef CONFIG_HUGETLB_PAGE
229
230#define HSTATE_NAME_LEN 32
231
232struct hstate {
233 int next_nid_to_alloc;
234 int next_nid_to_free;
235 unsigned int order;
236 unsigned long mask;
237 unsigned long max_huge_pages;
238 unsigned long nr_huge_pages;
239 unsigned long free_huge_pages;
240 unsigned long resv_huge_pages;
241 unsigned long surplus_huge_pages;
242 unsigned long nr_overcommit_huge_pages;
243 struct list_head hugepage_activelist;
244 struct list_head hugepage_freelists[MAX_NUMNODES];
245 unsigned int nr_huge_pages_node[MAX_NUMNODES];
246 unsigned int free_huge_pages_node[MAX_NUMNODES];
247 unsigned int surplus_huge_pages_node[MAX_NUMNODES];
248#ifdef CONFIG_CGROUP_HUGETLB
249
250 struct cftype cgroup_files[5];
251#endif
252 char name[HSTATE_NAME_LEN];
253};
254
255struct huge_bootmem_page {
256 struct list_head list;
257 struct hstate *hstate;
258#ifdef CONFIG_HIGHMEM
259 phys_addr_t phys;
260#endif
261};
262
263struct page *alloc_huge_page_node(struct hstate *h, int nid);
264
265
266int __init alloc_bootmem_huge_page(struct hstate *h);
267
268void __init hugetlb_add_hstate(unsigned order);
269struct hstate *size_to_hstate(unsigned long size);
270
271#ifndef HUGE_MAX_HSTATE
272#define HUGE_MAX_HSTATE 1
273#endif
274
275extern struct hstate hstates[HUGE_MAX_HSTATE];
276extern unsigned int default_hstate_idx;
277
278#define default_hstate (hstates[default_hstate_idx])
279
280static inline struct hstate *hstate_inode(struct inode *i)
281{
282 struct hugetlbfs_sb_info *hsb;
283 hsb = HUGETLBFS_SB(i->i_sb);
284 return hsb->hstate;
285}
286
287static inline struct hstate *hstate_file(struct file *f)
288{
289 return hstate_inode(file_inode(f));
290}
291
292static inline struct hstate *hstate_sizelog(int page_size_log)
293{
294 if (!page_size_log)
295 return &default_hstate;
296 return size_to_hstate(1 << page_size_log);
297}
298
299static inline struct hstate *hstate_vma(struct vm_area_struct *vma)
300{
301 return hstate_file(vma->vm_file);
302}
303
304static inline unsigned long huge_page_size(struct hstate *h)
305{
306 return (unsigned long)PAGE_SIZE << h->order;
307}
308
309extern unsigned long vma_kernel_pagesize(struct vm_area_struct *vma);
310
311extern unsigned long vma_mmu_pagesize(struct vm_area_struct *vma);
312
313static inline unsigned long huge_page_mask(struct hstate *h)
314{
315 return h->mask;
316}
317
318static inline unsigned int huge_page_order(struct hstate *h)
319{
320 return h->order;
321}
322
323static inline unsigned huge_page_shift(struct hstate *h)
324{
325 return h->order + PAGE_SHIFT;
326}
327
328static inline unsigned int pages_per_huge_page(struct hstate *h)
329{
330 return 1 << h->order;
331}
332
333static inline unsigned int blocks_per_huge_page(struct hstate *h)
334{
335 return huge_page_size(h) / 512;
336}
337
338#include <asm/hugetlb.h>
339
340#ifndef arch_make_huge_pte
341static inline pte_t arch_make_huge_pte(pte_t entry, struct vm_area_struct *vma,
342 struct page *page, int writable)
343{
344 return entry;
345}
346#endif
347
348static inline struct hstate *page_hstate(struct page *page)
349{
350 return size_to_hstate(PAGE_SIZE << compound_order(page));
351}
352
353static inline unsigned hstate_index_to_shift(unsigned index)
354{
355 return hstates[index].order + PAGE_SHIFT;
356}
357
358static inline int hstate_index(struct hstate *h)
359{
360 return h - hstates;
361}
362
363pgoff_t __basepage_index(struct page *page);
364
365
366static inline pgoff_t basepage_index(struct page *page)
367{
368 if (!PageCompound(page))
369 return page->index;
370
371 return __basepage_index(page);
372}
373
374#else
375struct hstate {};
376#define alloc_huge_page_node(h, nid) NULL
377#define alloc_bootmem_huge_page(h) NULL
378#define hstate_file(f) NULL
379#define hstate_sizelog(s) NULL
380#define hstate_vma(v) NULL
381#define hstate_inode(i) NULL
382#define huge_page_size(h) PAGE_SIZE
383#define huge_page_mask(h) PAGE_MASK
384#define vma_kernel_pagesize(v) PAGE_SIZE
385#define vma_mmu_pagesize(v) PAGE_SIZE
386#define huge_page_order(h) 0
387#define huge_page_shift(h) PAGE_SHIFT
388static inline unsigned int pages_per_huge_page(struct hstate *h)
389{
390 return 1;
391}
392#define hstate_index_to_shift(index) 0
393#define hstate_index(h) 0
394
395static inline pgoff_t basepage_index(struct page *page)
396{
397 return page->index;
398}
399#endif
400
401#endif
402