1
2
3
4
5
6
7
8
9
10#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
11
12#include <linux/thread_info.h>
13#include <asm/current.h>
14#include <linux/sched/signal.h>
15#include <linux/falloc.h>
16#include <linux/fs.h>
17#include <linux/mount.h>
18#include <linux/file.h>
19#include <linux/kernel.h>
20#include <linux/writeback.h>
21#include <linux/pagemap.h>
22#include <linux/highmem.h>
23#include <linux/init.h>
24#include <linux/string.h>
25#include <linux/capability.h>
26#include <linux/ctype.h>
27#include <linux/backing-dev.h>
28#include <linux/hugetlb.h>
29#include <linux/pagevec.h>
30#include <linux/parser.h>
31#include <linux/mman.h>
32#include <linux/slab.h>
33#include <linux/dnotify.h>
34#include <linux/statfs.h>
35#include <linux/security.h>
36#include <linux/magic.h>
37#include <linux/migrate.h>
38#include <linux/uio.h>
39
40#include <linux/uaccess.h>
41
42static const struct super_operations hugetlbfs_ops;
43static const struct address_space_operations hugetlbfs_aops;
44const struct file_operations hugetlbfs_file_operations;
45static const struct inode_operations hugetlbfs_dir_inode_operations;
46static const struct inode_operations hugetlbfs_inode_operations;
47
48struct hugetlbfs_config {
49 struct hstate *hstate;
50 long max_hpages;
51 long nr_inodes;
52 long min_hpages;
53 kuid_t uid;
54 kgid_t gid;
55 umode_t mode;
56};
57
58int sysctl_hugetlb_shm_group;
59
60enum {
61 Opt_size, Opt_nr_inodes,
62 Opt_mode, Opt_uid, Opt_gid,
63 Opt_pagesize, Opt_min_size,
64 Opt_err,
65};
66
67static const match_table_t tokens = {
68 {Opt_size, "size=%s"},
69 {Opt_nr_inodes, "nr_inodes=%s"},
70 {Opt_mode, "mode=%o"},
71 {Opt_uid, "uid=%u"},
72 {Opt_gid, "gid=%u"},
73 {Opt_pagesize, "pagesize=%s"},
74 {Opt_min_size, "min_size=%s"},
75 {Opt_err, NULL},
76};
77
78#ifdef CONFIG_NUMA
79static inline void hugetlb_set_vma_policy(struct vm_area_struct *vma,
80 struct inode *inode, pgoff_t index)
81{
82 vma->vm_policy = mpol_shared_policy_lookup(&HUGETLBFS_I(inode)->policy,
83 index);
84}
85
86static inline void hugetlb_drop_vma_policy(struct vm_area_struct *vma)
87{
88 mpol_cond_put(vma->vm_policy);
89}
90#else
91static inline void hugetlb_set_vma_policy(struct vm_area_struct *vma,
92 struct inode *inode, pgoff_t index)
93{
94}
95
96static inline void hugetlb_drop_vma_policy(struct vm_area_struct *vma)
97{
98}
99#endif
100
101static void huge_pagevec_release(struct pagevec *pvec)
102{
103 int i;
104
105 for (i = 0; i < pagevec_count(pvec); ++i)
106 put_page(pvec->pages[i]);
107
108 pagevec_reinit(pvec);
109}
110
111
112
113
114
115
116
117
118#define PGOFF_LOFFT_MAX \
119 (((1UL << (PAGE_SHIFT + 1)) - 1) << (BITS_PER_LONG - (PAGE_SHIFT + 1)))
120
121static int hugetlbfs_file_mmap(struct file *file, struct vm_area_struct *vma)
122{
123 struct inode *inode = file_inode(file);
124 loff_t len, vma_len;
125 int ret;
126 struct hstate *h = hstate_file(file);
127
128
129
130
131
132
133
134
135
136 vma->vm_flags |= VM_HUGETLB | VM_DONTEXPAND;
137 vma->vm_ops = &hugetlb_vm_ops;
138
139
140
141
142
143
144
145 if (sizeof(unsigned long) == sizeof(loff_t)) {
146 if (vma->vm_pgoff & PGOFF_LOFFT_MAX)
147 return -EINVAL;
148 }
149
150
151 if (vma->vm_pgoff & (~huge_page_mask(h) >> PAGE_SHIFT))
152 return -EINVAL;
153
154 vma_len = (loff_t)(vma->vm_end - vma->vm_start);
155 len = vma_len + ((loff_t)vma->vm_pgoff << PAGE_SHIFT);
156
157 if (len < vma_len)
158 return -EINVAL;
159
160 inode_lock(inode);
161 file_accessed(file);
162
163 ret = -ENOMEM;
164 if (hugetlb_reserve_pages(inode,
165 vma->vm_pgoff >> huge_page_order(h),
166 len >> huge_page_shift(h), vma,
167 vma->vm_flags))
168 goto out;
169
170 ret = 0;
171 if (vma->vm_flags & VM_WRITE && inode->i_size < len)
172 i_size_write(inode, len);
173out:
174 inode_unlock(inode);
175
176 return ret;
177}
178
179
180
181
182
183#ifndef HAVE_ARCH_HUGETLB_UNMAPPED_AREA
184static unsigned long
185hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
186 unsigned long len, unsigned long pgoff, unsigned long flags)
187{
188 struct mm_struct *mm = current->mm;
189 struct vm_area_struct *vma;
190 struct hstate *h = hstate_file(file);
191 struct vm_unmapped_area_info info;
192
193 if (len & ~huge_page_mask(h))
194 return -EINVAL;
195 if (len > TASK_SIZE)
196 return -ENOMEM;
197
198 if (flags & MAP_FIXED) {
199 if (prepare_hugepage_range(file, addr, len))
200 return -EINVAL;
201 return addr;
202 }
203
204 if (addr) {
205 addr = ALIGN(addr, huge_page_size(h));
206 vma = find_vma(mm, addr);
207 if (TASK_SIZE - len >= addr &&
208 (!vma || addr + len <= vm_start_gap(vma)))
209 return addr;
210 }
211
212 info.flags = 0;
213 info.length = len;
214 info.low_limit = TASK_UNMAPPED_BASE;
215 info.high_limit = TASK_SIZE;
216 info.align_mask = PAGE_MASK & ~huge_page_mask(h);
217 info.align_offset = 0;
218 return vm_unmapped_area(&info);
219}
220#endif
221
222static size_t
223hugetlbfs_read_actor(struct page *page, unsigned long offset,
224 struct iov_iter *to, unsigned long size)
225{
226 size_t copied = 0;
227 int i, chunksize;
228
229
230 i = offset >> PAGE_SHIFT;
231 offset = offset & ~PAGE_MASK;
232
233 while (size) {
234 size_t n;
235 chunksize = PAGE_SIZE;
236 if (offset)
237 chunksize -= offset;
238 if (chunksize > size)
239 chunksize = size;
240 n = copy_page_to_iter(&page[i], offset, chunksize, to);
241 copied += n;
242 if (n != chunksize)
243 return copied;
244 offset = 0;
245 size -= chunksize;
246 i++;
247 }
248 return copied;
249}
250
251
252
253
254
255
256static ssize_t hugetlbfs_read_iter(struct kiocb *iocb, struct iov_iter *to)
257{
258 struct file *file = iocb->ki_filp;
259 struct hstate *h = hstate_file(file);
260 struct address_space *mapping = file->f_mapping;
261 struct inode *inode = mapping->host;
262 unsigned long index = iocb->ki_pos >> huge_page_shift(h);
263 unsigned long offset = iocb->ki_pos & ~huge_page_mask(h);
264 unsigned long end_index;
265 loff_t isize;
266 ssize_t retval = 0;
267
268 while (iov_iter_count(to)) {
269 struct page *page;
270 size_t nr, copied;
271
272
273 nr = huge_page_size(h);
274 isize = i_size_read(inode);
275 if (!isize)
276 break;
277 end_index = (isize - 1) >> huge_page_shift(h);
278 if (index > end_index)
279 break;
280 if (index == end_index) {
281 nr = ((isize - 1) & ~huge_page_mask(h)) + 1;
282 if (nr <= offset)
283 break;
284 }
285 nr = nr - offset;
286
287
288 page = find_lock_page(mapping, index);
289 if (unlikely(page == NULL)) {
290
291
292
293
294 copied = iov_iter_zero(nr, to);
295 } else {
296 unlock_page(page);
297
298
299
300
301 copied = hugetlbfs_read_actor(page, offset, to, nr);
302 put_page(page);
303 }
304 offset += copied;
305 retval += copied;
306 if (copied != nr && iov_iter_count(to)) {
307 if (!retval)
308 retval = -EFAULT;
309 break;
310 }
311 index += offset >> huge_page_shift(h);
312 offset &= ~huge_page_mask(h);
313 }
314 iocb->ki_pos = ((loff_t)index << huge_page_shift(h)) + offset;
315 return retval;
316}
317
318static int hugetlbfs_write_begin(struct file *file,
319 struct address_space *mapping,
320 loff_t pos, unsigned len, unsigned flags,
321 struct page **pagep, void **fsdata)
322{
323 return -EINVAL;
324}
325
326static int hugetlbfs_write_end(struct file *file, struct address_space *mapping,
327 loff_t pos, unsigned len, unsigned copied,
328 struct page *page, void *fsdata)
329{
330 BUG();
331 return -EINVAL;
332}
333
334static void remove_huge_page(struct page *page)
335{
336 ClearPageDirty(page);
337 ClearPageUptodate(page);
338 delete_from_page_cache(page);
339}
340
341static void
342hugetlb_vmdelete_list(struct rb_root_cached *root, pgoff_t start, pgoff_t end)
343{
344 struct vm_area_struct *vma;
345
346
347
348
349
350 vma_interval_tree_foreach(vma, root, start, end ? end : ULONG_MAX) {
351 unsigned long v_offset;
352 unsigned long v_end;
353
354
355
356
357
358
359
360 if (vma->vm_pgoff < start)
361 v_offset = (start - vma->vm_pgoff) << PAGE_SHIFT;
362 else
363 v_offset = 0;
364
365 if (!end)
366 v_end = vma->vm_end;
367 else {
368 v_end = ((end - vma->vm_pgoff) << PAGE_SHIFT)
369 + vma->vm_start;
370 if (v_end > vma->vm_end)
371 v_end = vma->vm_end;
372 }
373
374 unmap_hugepage_range(vma, vma->vm_start + v_offset, v_end,
375 NULL);
376 }
377}
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400static void remove_inode_hugepages(struct inode *inode, loff_t lstart,
401 loff_t lend)
402{
403 struct hstate *h = hstate_inode(inode);
404 struct address_space *mapping = &inode->i_data;
405 const pgoff_t start = lstart >> huge_page_shift(h);
406 const pgoff_t end = lend >> huge_page_shift(h);
407 struct vm_area_struct pseudo_vma;
408 struct pagevec pvec;
409 pgoff_t next, index;
410 int i, freed = 0;
411 bool truncate_op = (lend == LLONG_MAX);
412
413 memset(&pseudo_vma, 0, sizeof(struct vm_area_struct));
414 pseudo_vma.vm_flags = (VM_HUGETLB | VM_MAYSHARE | VM_SHARED);
415 pagevec_init(&pvec);
416 next = start;
417 while (next < end) {
418
419
420
421 if (!pagevec_lookup_range(&pvec, mapping, &next, end - 1))
422 break;
423
424 for (i = 0; i < pagevec_count(&pvec); ++i) {
425 struct page *page = pvec.pages[i];
426 u32 hash;
427
428 index = page->index;
429 hash = hugetlb_fault_mutex_hash(h, current->mm,
430 &pseudo_vma,
431 mapping, index, 0);
432 mutex_lock(&hugetlb_fault_mutex_table[hash]);
433
434
435
436
437
438
439
440
441
442
443 if (unlikely(page_mapped(page))) {
444 BUG_ON(truncate_op);
445
446 i_mmap_lock_write(mapping);
447 hugetlb_vmdelete_list(&mapping->i_mmap,
448 index * pages_per_huge_page(h),
449 (index + 1) * pages_per_huge_page(h));
450 i_mmap_unlock_write(mapping);
451 }
452
453 lock_page(page);
454
455
456
457
458
459
460
461
462
463 VM_BUG_ON(PagePrivate(page));
464 remove_huge_page(page);
465 freed++;
466 if (!truncate_op) {
467 if (unlikely(hugetlb_unreserve_pages(inode,
468 index, index + 1, 1)))
469 hugetlb_fix_reserve_counts(inode);
470 }
471
472 unlock_page(page);
473 mutex_unlock(&hugetlb_fault_mutex_table[hash]);
474 }
475 huge_pagevec_release(&pvec);
476 cond_resched();
477 }
478
479 if (truncate_op)
480 (void)hugetlb_unreserve_pages(inode, start, LONG_MAX, freed);
481}
482
483static void hugetlbfs_evict_inode(struct inode *inode)
484{
485 struct resv_map *resv_map;
486
487 remove_inode_hugepages(inode, 0, LLONG_MAX);
488 resv_map = (struct resv_map *)inode->i_mapping->private_data;
489
490 if (resv_map)
491 resv_map_release(&resv_map->refs);
492 clear_inode(inode);
493}
494
495static int hugetlb_vmtruncate(struct inode *inode, loff_t offset)
496{
497 pgoff_t pgoff;
498 struct address_space *mapping = inode->i_mapping;
499 struct hstate *h = hstate_inode(inode);
500
501 BUG_ON(offset & ~huge_page_mask(h));
502 pgoff = offset >> PAGE_SHIFT;
503
504 i_size_write(inode, offset);
505 i_mmap_lock_write(mapping);
506 if (!RB_EMPTY_ROOT(&mapping->i_mmap.rb_root))
507 hugetlb_vmdelete_list(&mapping->i_mmap, pgoff, 0);
508 i_mmap_unlock_write(mapping);
509 remove_inode_hugepages(inode, offset, LLONG_MAX);
510 return 0;
511}
512
513static long hugetlbfs_punch_hole(struct inode *inode, loff_t offset, loff_t len)
514{
515 struct hstate *h = hstate_inode(inode);
516 loff_t hpage_size = huge_page_size(h);
517 loff_t hole_start, hole_end;
518
519
520
521
522
523 hole_start = round_up(offset, hpage_size);
524 hole_end = round_down(offset + len, hpage_size);
525
526 if (hole_end > hole_start) {
527 struct address_space *mapping = inode->i_mapping;
528 struct hugetlbfs_inode_info *info = HUGETLBFS_I(inode);
529
530 inode_lock(inode);
531
532
533 if (info->seals & F_SEAL_WRITE) {
534 inode_unlock(inode);
535 return -EPERM;
536 }
537
538 i_mmap_lock_write(mapping);
539 if (!RB_EMPTY_ROOT(&mapping->i_mmap.rb_root))
540 hugetlb_vmdelete_list(&mapping->i_mmap,
541 hole_start >> PAGE_SHIFT,
542 hole_end >> PAGE_SHIFT);
543 i_mmap_unlock_write(mapping);
544 remove_inode_hugepages(inode, hole_start, hole_end);
545 inode_unlock(inode);
546 }
547
548 return 0;
549}
550
551static long hugetlbfs_fallocate(struct file *file, int mode, loff_t offset,
552 loff_t len)
553{
554 struct inode *inode = file_inode(file);
555 struct hugetlbfs_inode_info *info = HUGETLBFS_I(inode);
556 struct address_space *mapping = inode->i_mapping;
557 struct hstate *h = hstate_inode(inode);
558 struct vm_area_struct pseudo_vma;
559 struct mm_struct *mm = current->mm;
560 loff_t hpage_size = huge_page_size(h);
561 unsigned long hpage_shift = huge_page_shift(h);
562 pgoff_t start, index, end;
563 int error;
564 u32 hash;
565
566 if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE))
567 return -EOPNOTSUPP;
568
569 if (mode & FALLOC_FL_PUNCH_HOLE)
570 return hugetlbfs_punch_hole(inode, offset, len);
571
572
573
574
575
576
577 start = offset >> hpage_shift;
578 end = (offset + len + hpage_size - 1) >> hpage_shift;
579
580 inode_lock(inode);
581
582
583 error = inode_newsize_ok(inode, offset + len);
584 if (error)
585 goto out;
586
587 if ((info->seals & F_SEAL_GROW) && offset + len > inode->i_size) {
588 error = -EPERM;
589 goto out;
590 }
591
592
593
594
595
596
597 memset(&pseudo_vma, 0, sizeof(struct vm_area_struct));
598 pseudo_vma.vm_flags = (VM_HUGETLB | VM_MAYSHARE | VM_SHARED);
599 pseudo_vma.vm_file = file;
600
601 for (index = start; index < end; index++) {
602
603
604
605
606 struct page *page;
607 unsigned long addr;
608 int avoid_reserve = 0;
609
610 cond_resched();
611
612
613
614
615
616 if (signal_pending(current)) {
617 error = -EINTR;
618 break;
619 }
620
621
622 hugetlb_set_vma_policy(&pseudo_vma, inode, index);
623
624
625 addr = index * hpage_size;
626
627
628 hash = hugetlb_fault_mutex_hash(h, mm, &pseudo_vma, mapping,
629 index, addr);
630 mutex_lock(&hugetlb_fault_mutex_table[hash]);
631
632
633 page = find_get_page(mapping, index);
634 if (page) {
635 put_page(page);
636 mutex_unlock(&hugetlb_fault_mutex_table[hash]);
637 hugetlb_drop_vma_policy(&pseudo_vma);
638 continue;
639 }
640
641
642 page = alloc_huge_page(&pseudo_vma, addr, avoid_reserve);
643 hugetlb_drop_vma_policy(&pseudo_vma);
644 if (IS_ERR(page)) {
645 mutex_unlock(&hugetlb_fault_mutex_table[hash]);
646 error = PTR_ERR(page);
647 goto out;
648 }
649 clear_huge_page(page, addr, pages_per_huge_page(h));
650 __SetPageUptodate(page);
651 error = huge_add_to_page_cache(page, mapping, index);
652 if (unlikely(error)) {
653 put_page(page);
654 mutex_unlock(&hugetlb_fault_mutex_table[hash]);
655 goto out;
656 }
657
658 mutex_unlock(&hugetlb_fault_mutex_table[hash]);
659
660
661
662
663
664 unlock_page(page);
665 put_page(page);
666 }
667
668 if (!(mode & FALLOC_FL_KEEP_SIZE) && offset + len > inode->i_size)
669 i_size_write(inode, offset + len);
670 inode->i_ctime = current_time(inode);
671out:
672 inode_unlock(inode);
673 return error;
674}
675
676static int hugetlbfs_setattr(struct dentry *dentry, struct iattr *attr)
677{
678 struct inode *inode = d_inode(dentry);
679 struct hstate *h = hstate_inode(inode);
680 int error;
681 unsigned int ia_valid = attr->ia_valid;
682 struct hugetlbfs_inode_info *info = HUGETLBFS_I(inode);
683
684 BUG_ON(!inode);
685
686 error = setattr_prepare(dentry, attr);
687 if (error)
688 return error;
689
690 if (ia_valid & ATTR_SIZE) {
691 loff_t oldsize = inode->i_size;
692 loff_t newsize = attr->ia_size;
693
694 if (newsize & ~huge_page_mask(h))
695 return -EINVAL;
696
697 if ((newsize < oldsize && (info->seals & F_SEAL_SHRINK)) ||
698 (newsize > oldsize && (info->seals & F_SEAL_GROW)))
699 return -EPERM;
700 error = hugetlb_vmtruncate(inode, newsize);
701 if (error)
702 return error;
703 }
704
705 setattr_copy(inode, attr);
706 mark_inode_dirty(inode);
707 return 0;
708}
709
710static struct inode *hugetlbfs_get_root(struct super_block *sb,
711 struct hugetlbfs_config *config)
712{
713 struct inode *inode;
714
715 inode = new_inode(sb);
716 if (inode) {
717 inode->i_ino = get_next_ino();
718 inode->i_mode = S_IFDIR | config->mode;
719 inode->i_uid = config->uid;
720 inode->i_gid = config->gid;
721 inode->i_atime = inode->i_mtime = inode->i_ctime = current_time(inode);
722 inode->i_op = &hugetlbfs_dir_inode_operations;
723 inode->i_fop = &simple_dir_operations;
724
725 inc_nlink(inode);
726 lockdep_annotate_inode_mutex_key(inode);
727 }
728 return inode;
729}
730
731
732
733
734
735
736
737static struct lock_class_key hugetlbfs_i_mmap_rwsem_key;
738
739static struct inode *hugetlbfs_get_inode(struct super_block *sb,
740 struct inode *dir,
741 umode_t mode, dev_t dev)
742{
743 struct inode *inode;
744 struct resv_map *resv_map;
745
746 resv_map = resv_map_alloc();
747 if (!resv_map)
748 return NULL;
749
750 inode = new_inode(sb);
751 if (inode) {
752 struct hugetlbfs_inode_info *info = HUGETLBFS_I(inode);
753
754 inode->i_ino = get_next_ino();
755 inode_init_owner(inode, dir, mode);
756 lockdep_set_class(&inode->i_mapping->i_mmap_rwsem,
757 &hugetlbfs_i_mmap_rwsem_key);
758 inode->i_mapping->a_ops = &hugetlbfs_aops;
759 inode->i_atime = inode->i_mtime = inode->i_ctime = current_time(inode);
760 inode->i_mapping->private_data = resv_map;
761 info->seals = F_SEAL_SEAL;
762 switch (mode & S_IFMT) {
763 default:
764 init_special_inode(inode, mode, dev);
765 break;
766 case S_IFREG:
767 inode->i_op = &hugetlbfs_inode_operations;
768 inode->i_fop = &hugetlbfs_file_operations;
769 break;
770 case S_IFDIR:
771 inode->i_op = &hugetlbfs_dir_inode_operations;
772 inode->i_fop = &simple_dir_operations;
773
774
775 inc_nlink(inode);
776 break;
777 case S_IFLNK:
778 inode->i_op = &page_symlink_inode_operations;
779 inode_nohighmem(inode);
780 break;
781 }
782 lockdep_annotate_inode_mutex_key(inode);
783 } else
784 kref_put(&resv_map->refs, resv_map_release);
785
786 return inode;
787}
788
789
790
791
792static int hugetlbfs_mknod(struct inode *dir,
793 struct dentry *dentry, umode_t mode, dev_t dev)
794{
795 struct inode *inode;
796 int error = -ENOSPC;
797
798 inode = hugetlbfs_get_inode(dir->i_sb, dir, mode, dev);
799 if (inode) {
800 dir->i_ctime = dir->i_mtime = current_time(dir);
801 d_instantiate(dentry, inode);
802 dget(dentry);
803 error = 0;
804 }
805 return error;
806}
807
808static int hugetlbfs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode)
809{
810 int retval = hugetlbfs_mknod(dir, dentry, mode | S_IFDIR, 0);
811 if (!retval)
812 inc_nlink(dir);
813 return retval;
814}
815
816static int hugetlbfs_create(struct inode *dir, struct dentry *dentry, umode_t mode, bool excl)
817{
818 return hugetlbfs_mknod(dir, dentry, mode | S_IFREG, 0);
819}
820
821static int hugetlbfs_symlink(struct inode *dir,
822 struct dentry *dentry, const char *symname)
823{
824 struct inode *inode;
825 int error = -ENOSPC;
826
827 inode = hugetlbfs_get_inode(dir->i_sb, dir, S_IFLNK|S_IRWXUGO, 0);
828 if (inode) {
829 int l = strlen(symname)+1;
830 error = page_symlink(inode, symname, l);
831 if (!error) {
832 d_instantiate(dentry, inode);
833 dget(dentry);
834 } else
835 iput(inode);
836 }
837 dir->i_ctime = dir->i_mtime = current_time(dir);
838
839 return error;
840}
841
842
843
844
845static int hugetlbfs_set_page_dirty(struct page *page)
846{
847 struct page *head = compound_head(page);
848
849 SetPageDirty(head);
850 return 0;
851}
852
853static int hugetlbfs_migrate_page(struct address_space *mapping,
854 struct page *newpage, struct page *page,
855 enum migrate_mode mode)
856{
857 int rc;
858
859 rc = migrate_huge_page_move_mapping(mapping, newpage, page);
860 if (rc != MIGRATEPAGE_SUCCESS)
861 return rc;
862 if (mode != MIGRATE_SYNC_NO_COPY)
863 migrate_page_copy(newpage, page);
864 else
865 migrate_page_states(newpage, page);
866
867 return MIGRATEPAGE_SUCCESS;
868}
869
870static int hugetlbfs_error_remove_page(struct address_space *mapping,
871 struct page *page)
872{
873 struct inode *inode = mapping->host;
874 pgoff_t index = page->index;
875
876 remove_huge_page(page);
877 if (unlikely(hugetlb_unreserve_pages(inode, index, index + 1, 1)))
878 hugetlb_fix_reserve_counts(inode);
879
880 return 0;
881}
882
883
884
885
886static int hugetlbfs_show_options(struct seq_file *m, struct dentry *root)
887{
888 struct hugetlbfs_sb_info *sbinfo = HUGETLBFS_SB(root->d_sb);
889 struct hugepage_subpool *spool = sbinfo->spool;
890 unsigned long hpage_size = huge_page_size(sbinfo->hstate);
891 unsigned hpage_shift = huge_page_shift(sbinfo->hstate);
892 char mod;
893
894 if (!uid_eq(sbinfo->uid, GLOBAL_ROOT_UID))
895 seq_printf(m, ",uid=%u",
896 from_kuid_munged(&init_user_ns, sbinfo->uid));
897 if (!gid_eq(sbinfo->gid, GLOBAL_ROOT_GID))
898 seq_printf(m, ",gid=%u",
899 from_kgid_munged(&init_user_ns, sbinfo->gid));
900 if (sbinfo->mode != 0755)
901 seq_printf(m, ",mode=%o", sbinfo->mode);
902 if (sbinfo->max_inodes != -1)
903 seq_printf(m, ",nr_inodes=%lu", sbinfo->max_inodes);
904
905 hpage_size /= 1024;
906 mod = 'K';
907 if (hpage_size >= 1024) {
908 hpage_size /= 1024;
909 mod = 'M';
910 }
911 seq_printf(m, ",pagesize=%lu%c", hpage_size, mod);
912 if (spool) {
913 if (spool->max_hpages != -1)
914 seq_printf(m, ",size=%llu",
915 (unsigned long long)spool->max_hpages << hpage_shift);
916 if (spool->min_hpages != -1)
917 seq_printf(m, ",min_size=%llu",
918 (unsigned long long)spool->min_hpages << hpage_shift);
919 }
920 return 0;
921}
922
923static int hugetlbfs_statfs(struct dentry *dentry, struct kstatfs *buf)
924{
925 struct hugetlbfs_sb_info *sbinfo = HUGETLBFS_SB(dentry->d_sb);
926 struct hstate *h = hstate_inode(d_inode(dentry));
927
928 buf->f_type = HUGETLBFS_MAGIC;
929 buf->f_bsize = huge_page_size(h);
930 if (sbinfo) {
931 spin_lock(&sbinfo->stat_lock);
932
933
934 if (sbinfo->spool) {
935 long free_pages;
936
937 spin_lock(&sbinfo->spool->lock);
938 buf->f_blocks = sbinfo->spool->max_hpages;
939 free_pages = sbinfo->spool->max_hpages
940 - sbinfo->spool->used_hpages;
941 buf->f_bavail = buf->f_bfree = free_pages;
942 spin_unlock(&sbinfo->spool->lock);
943 buf->f_files = sbinfo->max_inodes;
944 buf->f_ffree = sbinfo->free_inodes;
945 }
946 spin_unlock(&sbinfo->stat_lock);
947 }
948 buf->f_namelen = NAME_MAX;
949 return 0;
950}
951
952static void hugetlbfs_put_super(struct super_block *sb)
953{
954 struct hugetlbfs_sb_info *sbi = HUGETLBFS_SB(sb);
955
956 if (sbi) {
957 sb->s_fs_info = NULL;
958
959 if (sbi->spool)
960 hugepage_put_subpool(sbi->spool);
961
962 kfree(sbi);
963 }
964}
965
966static inline int hugetlbfs_dec_free_inodes(struct hugetlbfs_sb_info *sbinfo)
967{
968 if (sbinfo->free_inodes >= 0) {
969 spin_lock(&sbinfo->stat_lock);
970 if (unlikely(!sbinfo->free_inodes)) {
971 spin_unlock(&sbinfo->stat_lock);
972 return 0;
973 }
974 sbinfo->free_inodes--;
975 spin_unlock(&sbinfo->stat_lock);
976 }
977
978 return 1;
979}
980
981static void hugetlbfs_inc_free_inodes(struct hugetlbfs_sb_info *sbinfo)
982{
983 if (sbinfo->free_inodes >= 0) {
984 spin_lock(&sbinfo->stat_lock);
985 sbinfo->free_inodes++;
986 spin_unlock(&sbinfo->stat_lock);
987 }
988}
989
990
991static struct kmem_cache *hugetlbfs_inode_cachep;
992
993static struct inode *hugetlbfs_alloc_inode(struct super_block *sb)
994{
995 struct hugetlbfs_sb_info *sbinfo = HUGETLBFS_SB(sb);
996 struct hugetlbfs_inode_info *p;
997
998 if (unlikely(!hugetlbfs_dec_free_inodes(sbinfo)))
999 return NULL;
1000 p = kmem_cache_alloc(hugetlbfs_inode_cachep, GFP_KERNEL);
1001 if (unlikely(!p)) {
1002 hugetlbfs_inc_free_inodes(sbinfo);
1003 return NULL;
1004 }
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015 mpol_shared_policy_init(&p->policy, NULL);
1016
1017 return &p->vfs_inode;
1018}
1019
1020static void hugetlbfs_i_callback(struct rcu_head *head)
1021{
1022 struct inode *inode = container_of(head, struct inode, i_rcu);
1023 kmem_cache_free(hugetlbfs_inode_cachep, HUGETLBFS_I(inode));
1024}
1025
1026static void hugetlbfs_destroy_inode(struct inode *inode)
1027{
1028 hugetlbfs_inc_free_inodes(HUGETLBFS_SB(inode->i_sb));
1029 mpol_free_shared_policy(&HUGETLBFS_I(inode)->policy);
1030 call_rcu(&inode->i_rcu, hugetlbfs_i_callback);
1031}
1032
1033static const struct address_space_operations hugetlbfs_aops = {
1034 .write_begin = hugetlbfs_write_begin,
1035 .write_end = hugetlbfs_write_end,
1036 .set_page_dirty = hugetlbfs_set_page_dirty,
1037 .migratepage = hugetlbfs_migrate_page,
1038 .error_remove_page = hugetlbfs_error_remove_page,
1039};
1040
1041
1042static void init_once(void *foo)
1043{
1044 struct hugetlbfs_inode_info *ei = (struct hugetlbfs_inode_info *)foo;
1045
1046 inode_init_once(&ei->vfs_inode);
1047}
1048
1049const struct file_operations hugetlbfs_file_operations = {
1050 .read_iter = hugetlbfs_read_iter,
1051 .mmap = hugetlbfs_file_mmap,
1052 .fsync = noop_fsync,
1053 .get_unmapped_area = hugetlb_get_unmapped_area,
1054 .llseek = default_llseek,
1055 .fallocate = hugetlbfs_fallocate,
1056};
1057
1058static const struct inode_operations hugetlbfs_dir_inode_operations = {
1059 .create = hugetlbfs_create,
1060 .lookup = simple_lookup,
1061 .link = simple_link,
1062 .unlink = simple_unlink,
1063 .symlink = hugetlbfs_symlink,
1064 .mkdir = hugetlbfs_mkdir,
1065 .rmdir = simple_rmdir,
1066 .mknod = hugetlbfs_mknod,
1067 .rename = simple_rename,
1068 .setattr = hugetlbfs_setattr,
1069};
1070
1071static const struct inode_operations hugetlbfs_inode_operations = {
1072 .setattr = hugetlbfs_setattr,
1073};
1074
1075static const struct super_operations hugetlbfs_ops = {
1076 .alloc_inode = hugetlbfs_alloc_inode,
1077 .destroy_inode = hugetlbfs_destroy_inode,
1078 .evict_inode = hugetlbfs_evict_inode,
1079 .statfs = hugetlbfs_statfs,
1080 .put_super = hugetlbfs_put_super,
1081 .show_options = hugetlbfs_show_options,
1082};
1083
1084enum hugetlbfs_size_type { NO_SIZE, SIZE_STD, SIZE_PERCENT };
1085
1086
1087
1088
1089
1090
1091static long
1092hugetlbfs_size_to_hpages(struct hstate *h, unsigned long long size_opt,
1093 enum hugetlbfs_size_type val_type)
1094{
1095 if (val_type == NO_SIZE)
1096 return -1;
1097
1098 if (val_type == SIZE_PERCENT) {
1099 size_opt <<= huge_page_shift(h);
1100 size_opt *= h->max_huge_pages;
1101 do_div(size_opt, 100);
1102 }
1103
1104 size_opt >>= huge_page_shift(h);
1105 return size_opt;
1106}
1107
1108static int
1109hugetlbfs_parse_options(char *options, struct hugetlbfs_config *pconfig)
1110{
1111 char *p, *rest;
1112 substring_t args[MAX_OPT_ARGS];
1113 int option;
1114 unsigned long long max_size_opt = 0, min_size_opt = 0;
1115 enum hugetlbfs_size_type max_val_type = NO_SIZE, min_val_type = NO_SIZE;
1116
1117 if (!options)
1118 return 0;
1119
1120 while ((p = strsep(&options, ",")) != NULL) {
1121 int token;
1122 if (!*p)
1123 continue;
1124
1125 token = match_token(p, tokens, args);
1126 switch (token) {
1127 case Opt_uid:
1128 if (match_int(&args[0], &option))
1129 goto bad_val;
1130 pconfig->uid = make_kuid(current_user_ns(), option);
1131 if (!uid_valid(pconfig->uid))
1132 goto bad_val;
1133 break;
1134
1135 case Opt_gid:
1136 if (match_int(&args[0], &option))
1137 goto bad_val;
1138 pconfig->gid = make_kgid(current_user_ns(), option);
1139 if (!gid_valid(pconfig->gid))
1140 goto bad_val;
1141 break;
1142
1143 case Opt_mode:
1144 if (match_octal(&args[0], &option))
1145 goto bad_val;
1146 pconfig->mode = option & 01777U;
1147 break;
1148
1149 case Opt_size: {
1150
1151 if (!isdigit(*args[0].from))
1152 goto bad_val;
1153 max_size_opt = memparse(args[0].from, &rest);
1154 max_val_type = SIZE_STD;
1155 if (*rest == '%')
1156 max_val_type = SIZE_PERCENT;
1157 break;
1158 }
1159
1160 case Opt_nr_inodes:
1161
1162 if (!isdigit(*args[0].from))
1163 goto bad_val;
1164 pconfig->nr_inodes = memparse(args[0].from, &rest);
1165 break;
1166
1167 case Opt_pagesize: {
1168 unsigned long ps;
1169 ps = memparse(args[0].from, &rest);
1170 pconfig->hstate = size_to_hstate(ps);
1171 if (!pconfig->hstate) {
1172 pr_err("Unsupported page size %lu MB\n",
1173 ps >> 20);
1174 return -EINVAL;
1175 }
1176 break;
1177 }
1178
1179 case Opt_min_size: {
1180
1181 if (!isdigit(*args[0].from))
1182 goto bad_val;
1183 min_size_opt = memparse(args[0].from, &rest);
1184 min_val_type = SIZE_STD;
1185 if (*rest == '%')
1186 min_val_type = SIZE_PERCENT;
1187 break;
1188 }
1189
1190 default:
1191 pr_err("Bad mount option: \"%s\"\n", p);
1192 return -EINVAL;
1193 break;
1194 }
1195 }
1196
1197
1198
1199
1200
1201 pconfig->max_hpages = hugetlbfs_size_to_hpages(pconfig->hstate,
1202 max_size_opt, max_val_type);
1203 pconfig->min_hpages = hugetlbfs_size_to_hpages(pconfig->hstate,
1204 min_size_opt, min_val_type);
1205
1206
1207
1208
1209 if (max_val_type > NO_SIZE &&
1210 pconfig->min_hpages > pconfig->max_hpages) {
1211 pr_err("minimum size can not be greater than maximum size\n");
1212 return -EINVAL;
1213 }
1214
1215 return 0;
1216
1217bad_val:
1218 pr_err("Bad value '%s' for mount option '%s'\n", args[0].from, p);
1219 return -EINVAL;
1220}
1221
1222static int
1223hugetlbfs_fill_super(struct super_block *sb, void *data, int silent)
1224{
1225 int ret;
1226 struct hugetlbfs_config config;
1227 struct hugetlbfs_sb_info *sbinfo;
1228
1229 config.max_hpages = -1;
1230 config.nr_inodes = -1;
1231 config.uid = current_fsuid();
1232 config.gid = current_fsgid();
1233 config.mode = 0755;
1234 config.hstate = &default_hstate;
1235 config.min_hpages = -1;
1236 ret = hugetlbfs_parse_options(data, &config);
1237 if (ret)
1238 return ret;
1239
1240 sbinfo = kmalloc(sizeof(struct hugetlbfs_sb_info), GFP_KERNEL);
1241 if (!sbinfo)
1242 return -ENOMEM;
1243 sb->s_fs_info = sbinfo;
1244 sbinfo->hstate = config.hstate;
1245 spin_lock_init(&sbinfo->stat_lock);
1246 sbinfo->max_inodes = config.nr_inodes;
1247 sbinfo->free_inodes = config.nr_inodes;
1248 sbinfo->spool = NULL;
1249 sbinfo->uid = config.uid;
1250 sbinfo->gid = config.gid;
1251 sbinfo->mode = config.mode;
1252
1253
1254
1255
1256
1257
1258 if (config.max_hpages != -1 || config.min_hpages != -1) {
1259 sbinfo->spool = hugepage_new_subpool(config.hstate,
1260 config.max_hpages,
1261 config.min_hpages);
1262 if (!sbinfo->spool)
1263 goto out_free;
1264 }
1265 sb->s_maxbytes = MAX_LFS_FILESIZE;
1266 sb->s_blocksize = huge_page_size(config.hstate);
1267 sb->s_blocksize_bits = huge_page_shift(config.hstate);
1268 sb->s_magic = HUGETLBFS_MAGIC;
1269 sb->s_op = &hugetlbfs_ops;
1270 sb->s_time_gran = 1;
1271 sb->s_root = d_make_root(hugetlbfs_get_root(sb, &config));
1272 if (!sb->s_root)
1273 goto out_free;
1274 return 0;
1275out_free:
1276 kfree(sbinfo->spool);
1277 kfree(sbinfo);
1278 return -ENOMEM;
1279}
1280
1281static struct dentry *hugetlbfs_mount(struct file_system_type *fs_type,
1282 int flags, const char *dev_name, void *data)
1283{
1284 return mount_nodev(fs_type, flags, data, hugetlbfs_fill_super);
1285}
1286
1287static struct file_system_type hugetlbfs_fs_type = {
1288 .name = "hugetlbfs",
1289 .mount = hugetlbfs_mount,
1290 .kill_sb = kill_litter_super,
1291};
1292
1293static struct vfsmount *hugetlbfs_vfsmount[HUGE_MAX_HSTATE];
1294
1295static int can_do_hugetlb_shm(void)
1296{
1297 kgid_t shm_group;
1298 shm_group = make_kgid(&init_user_ns, sysctl_hugetlb_shm_group);
1299 return capable(CAP_IPC_LOCK) || in_group_p(shm_group);
1300}
1301
1302static int get_hstate_idx(int page_size_log)
1303{
1304 struct hstate *h = hstate_sizelog(page_size_log);
1305
1306 if (!h)
1307 return -1;
1308 return h - hstates;
1309}
1310
1311static const struct dentry_operations anon_ops = {
1312 .d_dname = simple_dname
1313};
1314
1315
1316
1317
1318
1319struct file *hugetlb_file_setup(const char *name, size_t size,
1320 vm_flags_t acctflag, struct user_struct **user,
1321 int creat_flags, int page_size_log)
1322{
1323 struct file *file = ERR_PTR(-ENOMEM);
1324 struct inode *inode;
1325 struct path path;
1326 struct super_block *sb;
1327 struct qstr quick_string;
1328 int hstate_idx;
1329
1330 hstate_idx = get_hstate_idx(page_size_log);
1331 if (hstate_idx < 0)
1332 return ERR_PTR(-ENODEV);
1333
1334 *user = NULL;
1335 if (!hugetlbfs_vfsmount[hstate_idx])
1336 return ERR_PTR(-ENOENT);
1337
1338 if (creat_flags == HUGETLB_SHMFS_INODE && !can_do_hugetlb_shm()) {
1339 *user = current_user();
1340 if (user_shm_lock(size, *user)) {
1341 task_lock(current);
1342 pr_warn_once("%s (%d): Using mlock ulimits for SHM_HUGETLB is deprecated\n",
1343 current->comm, current->pid);
1344 task_unlock(current);
1345 } else {
1346 *user = NULL;
1347 return ERR_PTR(-EPERM);
1348 }
1349 }
1350
1351 sb = hugetlbfs_vfsmount[hstate_idx]->mnt_sb;
1352 quick_string.name = name;
1353 quick_string.len = strlen(quick_string.name);
1354 quick_string.hash = 0;
1355 path.dentry = d_alloc_pseudo(sb, &quick_string);
1356 if (!path.dentry)
1357 goto out_shm_unlock;
1358
1359 d_set_d_op(path.dentry, &anon_ops);
1360 path.mnt = mntget(hugetlbfs_vfsmount[hstate_idx]);
1361 file = ERR_PTR(-ENOSPC);
1362 inode = hugetlbfs_get_inode(sb, NULL, S_IFREG | S_IRWXUGO, 0);
1363 if (!inode)
1364 goto out_dentry;
1365 if (creat_flags == HUGETLB_SHMFS_INODE)
1366 inode->i_flags |= S_PRIVATE;
1367
1368 file = ERR_PTR(-ENOMEM);
1369 if (hugetlb_reserve_pages(inode, 0,
1370 size >> huge_page_shift(hstate_inode(inode)), NULL,
1371 acctflag))
1372 goto out_inode;
1373
1374 d_instantiate(path.dentry, inode);
1375 inode->i_size = size;
1376 clear_nlink(inode);
1377
1378 file = alloc_file(&path, FMODE_WRITE | FMODE_READ,
1379 &hugetlbfs_file_operations);
1380 if (IS_ERR(file))
1381 goto out_dentry;
1382
1383 return file;
1384
1385out_inode:
1386 iput(inode);
1387out_dentry:
1388 path_put(&path);
1389out_shm_unlock:
1390 if (*user) {
1391 user_shm_unlock(size, *user);
1392 *user = NULL;
1393 }
1394 return file;
1395}
1396
1397static int __init init_hugetlbfs_fs(void)
1398{
1399 struct hstate *h;
1400 int error;
1401 int i;
1402
1403 if (!hugepages_supported()) {
1404 pr_info("disabling because there are no supported hugepage sizes\n");
1405 return -ENOTSUPP;
1406 }
1407
1408 error = -ENOMEM;
1409 hugetlbfs_inode_cachep = kmem_cache_create("hugetlbfs_inode_cache",
1410 sizeof(struct hugetlbfs_inode_info),
1411 0, SLAB_ACCOUNT, init_once);
1412 if (hugetlbfs_inode_cachep == NULL)
1413 goto out2;
1414
1415 error = register_filesystem(&hugetlbfs_fs_type);
1416 if (error)
1417 goto out;
1418
1419 i = 0;
1420 for_each_hstate(h) {
1421 char buf[50];
1422 unsigned ps_kb = 1U << (h->order + PAGE_SHIFT - 10);
1423
1424 snprintf(buf, sizeof(buf), "pagesize=%uK", ps_kb);
1425 hugetlbfs_vfsmount[i] = kern_mount_data(&hugetlbfs_fs_type,
1426 buf);
1427
1428 if (IS_ERR(hugetlbfs_vfsmount[i])) {
1429 pr_err("Cannot mount internal hugetlbfs for "
1430 "page size %uK", ps_kb);
1431 error = PTR_ERR(hugetlbfs_vfsmount[i]);
1432 hugetlbfs_vfsmount[i] = NULL;
1433 }
1434 i++;
1435 }
1436
1437 if (!IS_ERR_OR_NULL(hugetlbfs_vfsmount[default_hstate_idx]))
1438 return 0;
1439
1440 out:
1441 kmem_cache_destroy(hugetlbfs_inode_cachep);
1442 out2:
1443 return error;
1444}
1445fs_initcall(init_hugetlbfs_fs)
1446