1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24#include <linux/fs.h>
25#include <linux/init.h>
26#include <linux/vfs.h>
27#include <linux/mount.h>
28#include <linux/ramfs.h>
29#include <linux/pagemap.h>
30#include <linux/file.h>
31#include <linux/mm.h>
32#include <linux/sched/signal.h>
33#include <linux/export.h>
34#include <linux/swap.h>
35#include <linux/uio.h>
36#include <linux/khugepaged.h>
37#include <linux/hugetlb.h>
38
39#include <asm/tlbflush.h>
40
41static struct vfsmount *shm_mnt;
42
43#ifdef CONFIG_SHMEM
44
45
46
47
48
49
50#include <linux/xattr.h>
51#include <linux/exportfs.h>
52#include <linux/posix_acl.h>
53#include <linux/posix_acl_xattr.h>
54#include <linux/mman.h>
55#include <linux/string.h>
56#include <linux/slab.h>
57#include <linux/backing-dev.h>
58#include <linux/shmem_fs.h>
59#include <linux/writeback.h>
60#include <linux/blkdev.h>
61#include <linux/pagevec.h>
62#include <linux/percpu_counter.h>
63#include <linux/falloc.h>
64#include <linux/splice.h>
65#include <linux/security.h>
66#include <linux/swapops.h>
67#include <linux/mempolicy.h>
68#include <linux/namei.h>
69#include <linux/ctype.h>
70#include <linux/migrate.h>
71#include <linux/highmem.h>
72#include <linux/seq_file.h>
73#include <linux/magic.h>
74#include <linux/syscalls.h>
75#include <linux/fcntl.h>
76#include <uapi/linux/memfd.h>
77#include <linux/userfaultfd_k.h>
78#include <linux/rmap.h>
79#include <linux/uuid.h>
80
81#include <linux/uaccess.h>
82#include <asm/pgtable.h>
83
84#include "internal.h"
85
86#define BLOCKS_PER_PAGE (PAGE_SIZE/512)
87#define VM_ACCT(size) (PAGE_ALIGN(size) >> PAGE_SHIFT)
88
89
90#define BOGO_DIRENT_SIZE 20
91
92
93#define SHORT_SYMLINK_LEN 128
94
95
96
97
98
99
100struct shmem_falloc {
101 wait_queue_head_t *waitq;
102 pgoff_t start;
103 pgoff_t next;
104 pgoff_t nr_falloced;
105 pgoff_t nr_unswapped;
106};
107
108#ifdef CONFIG_TMPFS
109static unsigned long shmem_default_max_blocks(void)
110{
111 return totalram_pages / 2;
112}
113
114static unsigned long shmem_default_max_inodes(void)
115{
116 return min(totalram_pages - totalhigh_pages, totalram_pages / 2);
117}
118#endif
119
120static bool shmem_should_replace_page(struct page *page, gfp_t gfp);
121static int shmem_replace_page(struct page **pagep, gfp_t gfp,
122 struct shmem_inode_info *info, pgoff_t index);
123static int shmem_getpage_gfp(struct inode *inode, pgoff_t index,
124 struct page **pagep, enum sgp_type sgp,
125 gfp_t gfp, struct vm_area_struct *vma,
126 struct vm_fault *vmf, int *fault_type);
127
128int shmem_getpage(struct inode *inode, pgoff_t index,
129 struct page **pagep, enum sgp_type sgp)
130{
131 return shmem_getpage_gfp(inode, index, pagep, sgp,
132 mapping_gfp_mask(inode->i_mapping), NULL, NULL, NULL);
133}
134
135static inline struct shmem_sb_info *SHMEM_SB(struct super_block *sb)
136{
137 return sb->s_fs_info;
138}
139
140
141
142
143
144
145
146static inline int shmem_acct_size(unsigned long flags, loff_t size)
147{
148 return (flags & VM_NORESERVE) ?
149 0 : security_vm_enough_memory_mm(current->mm, VM_ACCT(size));
150}
151
152static inline void shmem_unacct_size(unsigned long flags, loff_t size)
153{
154 if (!(flags & VM_NORESERVE))
155 vm_unacct_memory(VM_ACCT(size));
156}
157
158static inline int shmem_reacct_size(unsigned long flags,
159 loff_t oldsize, loff_t newsize)
160{
161 if (!(flags & VM_NORESERVE)) {
162 if (VM_ACCT(newsize) > VM_ACCT(oldsize))
163 return security_vm_enough_memory_mm(current->mm,
164 VM_ACCT(newsize) - VM_ACCT(oldsize));
165 else if (VM_ACCT(newsize) < VM_ACCT(oldsize))
166 vm_unacct_memory(VM_ACCT(oldsize) - VM_ACCT(newsize));
167 }
168 return 0;
169}
170
171
172
173
174
175
176
177static inline int shmem_acct_block(unsigned long flags, long pages)
178{
179 if (!(flags & VM_NORESERVE))
180 return 0;
181
182 return security_vm_enough_memory_mm(current->mm,
183 pages * VM_ACCT(PAGE_SIZE));
184}
185
186static inline void shmem_unacct_blocks(unsigned long flags, long pages)
187{
188 if (flags & VM_NORESERVE)
189 vm_unacct_memory(pages * VM_ACCT(PAGE_SIZE));
190}
191
192static inline bool shmem_inode_acct_block(struct inode *inode, long pages)
193{
194 struct shmem_inode_info *info = SHMEM_I(inode);
195 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
196
197 if (shmem_acct_block(info->flags, pages))
198 return false;
199
200 if (sbinfo->max_blocks) {
201 if (percpu_counter_compare(&sbinfo->used_blocks,
202 sbinfo->max_blocks - pages) > 0)
203 goto unacct;
204 percpu_counter_add(&sbinfo->used_blocks, pages);
205 }
206
207 return true;
208
209unacct:
210 shmem_unacct_blocks(info->flags, pages);
211 return false;
212}
213
214static inline void shmem_inode_unacct_blocks(struct inode *inode, long pages)
215{
216 struct shmem_inode_info *info = SHMEM_I(inode);
217 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
218
219 if (sbinfo->max_blocks)
220 percpu_counter_sub(&sbinfo->used_blocks, pages);
221 shmem_unacct_blocks(info->flags, pages);
222}
223
224static const struct super_operations shmem_ops;
225static const struct address_space_operations shmem_aops;
226static const struct file_operations shmem_file_operations;
227static const struct inode_operations shmem_inode_operations;
228static const struct inode_operations shmem_dir_inode_operations;
229static const struct inode_operations shmem_special_inode_operations;
230static const struct vm_operations_struct shmem_vm_ops;
231static struct file_system_type shmem_fs_type;
232
233bool vma_is_shmem(struct vm_area_struct *vma)
234{
235 return vma->vm_ops == &shmem_vm_ops;
236}
237
238static LIST_HEAD(shmem_swaplist);
239static DEFINE_MUTEX(shmem_swaplist_mutex);
240
241static int shmem_reserve_inode(struct super_block *sb)
242{
243 struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
244 if (sbinfo->max_inodes) {
245 spin_lock(&sbinfo->stat_lock);
246 if (!sbinfo->free_inodes) {
247 spin_unlock(&sbinfo->stat_lock);
248 return -ENOSPC;
249 }
250 sbinfo->free_inodes--;
251 spin_unlock(&sbinfo->stat_lock);
252 }
253 return 0;
254}
255
256static void shmem_free_inode(struct super_block *sb)
257{
258 struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
259 if (sbinfo->max_inodes) {
260 spin_lock(&sbinfo->stat_lock);
261 sbinfo->free_inodes++;
262 spin_unlock(&sbinfo->stat_lock);
263 }
264}
265
266
267
268
269
270
271
272
273
274
275
276
277
278static void shmem_recalc_inode(struct inode *inode)
279{
280 struct shmem_inode_info *info = SHMEM_I(inode);
281 long freed;
282
283 freed = info->alloced - info->swapped - inode->i_mapping->nrpages;
284 if (freed > 0) {
285 info->alloced -= freed;
286 inode->i_blocks -= freed * BLOCKS_PER_PAGE;
287 shmem_inode_unacct_blocks(inode, freed);
288 }
289}
290
291bool shmem_charge(struct inode *inode, long pages)
292{
293 struct shmem_inode_info *info = SHMEM_I(inode);
294 unsigned long flags;
295
296 if (!shmem_inode_acct_block(inode, pages))
297 return false;
298
299 spin_lock_irqsave(&info->lock, flags);
300 info->alloced += pages;
301 inode->i_blocks += pages * BLOCKS_PER_PAGE;
302 shmem_recalc_inode(inode);
303 spin_unlock_irqrestore(&info->lock, flags);
304 inode->i_mapping->nrpages += pages;
305
306 return true;
307}
308
309void shmem_uncharge(struct inode *inode, long pages)
310{
311 struct shmem_inode_info *info = SHMEM_I(inode);
312 unsigned long flags;
313
314 spin_lock_irqsave(&info->lock, flags);
315 info->alloced -= pages;
316 inode->i_blocks -= pages * BLOCKS_PER_PAGE;
317 shmem_recalc_inode(inode);
318 spin_unlock_irqrestore(&info->lock, flags);
319
320 shmem_inode_unacct_blocks(inode, pages);
321}
322
323
324
325
326static int shmem_radix_tree_replace(struct address_space *mapping,
327 pgoff_t index, void *expected, void *replacement)
328{
329 struct radix_tree_node *node;
330 void **pslot;
331 void *item;
332
333 VM_BUG_ON(!expected);
334 VM_BUG_ON(!replacement);
335 item = __radix_tree_lookup(&mapping->i_pages, index, &node, &pslot);
336 if (!item)
337 return -ENOENT;
338 if (item != expected)
339 return -ENOENT;
340 __radix_tree_replace(&mapping->i_pages, node, pslot,
341 replacement, NULL);
342 return 0;
343}
344
345
346
347
348
349
350
351
352static bool shmem_confirm_swap(struct address_space *mapping,
353 pgoff_t index, swp_entry_t swap)
354{
355 void *item;
356
357 rcu_read_lock();
358 item = radix_tree_lookup(&mapping->i_pages, index);
359 rcu_read_unlock();
360 return item == swp_to_radix_entry(swap);
361}
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377#define SHMEM_HUGE_NEVER 0
378#define SHMEM_HUGE_ALWAYS 1
379#define SHMEM_HUGE_WITHIN_SIZE 2
380#define SHMEM_HUGE_ADVISE 3
381
382
383
384
385
386
387
388
389
390
391
392#define SHMEM_HUGE_DENY (-1)
393#define SHMEM_HUGE_FORCE (-2)
394
395#ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE
396
397
398int shmem_huge __read_mostly;
399
400#if defined(CONFIG_SYSFS) || defined(CONFIG_TMPFS)
401static int shmem_parse_huge(const char *str)
402{
403 if (!strcmp(str, "never"))
404 return SHMEM_HUGE_NEVER;
405 if (!strcmp(str, "always"))
406 return SHMEM_HUGE_ALWAYS;
407 if (!strcmp(str, "within_size"))
408 return SHMEM_HUGE_WITHIN_SIZE;
409 if (!strcmp(str, "advise"))
410 return SHMEM_HUGE_ADVISE;
411 if (!strcmp(str, "deny"))
412 return SHMEM_HUGE_DENY;
413 if (!strcmp(str, "force"))
414 return SHMEM_HUGE_FORCE;
415 return -EINVAL;
416}
417
418static const char *shmem_format_huge(int huge)
419{
420 switch (huge) {
421 case SHMEM_HUGE_NEVER:
422 return "never";
423 case SHMEM_HUGE_ALWAYS:
424 return "always";
425 case SHMEM_HUGE_WITHIN_SIZE:
426 return "within_size";
427 case SHMEM_HUGE_ADVISE:
428 return "advise";
429 case SHMEM_HUGE_DENY:
430 return "deny";
431 case SHMEM_HUGE_FORCE:
432 return "force";
433 default:
434 VM_BUG_ON(1);
435 return "bad_val";
436 }
437}
438#endif
439
440static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo,
441 struct shrink_control *sc, unsigned long nr_to_split)
442{
443 LIST_HEAD(list), *pos, *next;
444 LIST_HEAD(to_remove);
445 struct inode *inode;
446 struct shmem_inode_info *info;
447 struct page *page;
448 unsigned long batch = sc ? sc->nr_to_scan : 128;
449 int removed = 0, split = 0;
450
451 if (list_empty(&sbinfo->shrinklist))
452 return SHRINK_STOP;
453
454 spin_lock(&sbinfo->shrinklist_lock);
455 list_for_each_safe(pos, next, &sbinfo->shrinklist) {
456 info = list_entry(pos, struct shmem_inode_info, shrinklist);
457
458
459 inode = igrab(&info->vfs_inode);
460
461
462 if (!inode) {
463 list_del_init(&info->shrinklist);
464 removed++;
465 goto next;
466 }
467
468
469 if (round_up(inode->i_size, PAGE_SIZE) ==
470 round_up(inode->i_size, HPAGE_PMD_SIZE)) {
471 list_move(&info->shrinklist, &to_remove);
472 removed++;
473 goto next;
474 }
475
476 list_move(&info->shrinklist, &list);
477next:
478 if (!--batch)
479 break;
480 }
481 spin_unlock(&sbinfo->shrinklist_lock);
482
483 list_for_each_safe(pos, next, &to_remove) {
484 info = list_entry(pos, struct shmem_inode_info, shrinklist);
485 inode = &info->vfs_inode;
486 list_del_init(&info->shrinklist);
487 iput(inode);
488 }
489
490 list_for_each_safe(pos, next, &list) {
491 int ret;
492
493 info = list_entry(pos, struct shmem_inode_info, shrinklist);
494 inode = &info->vfs_inode;
495
496 if (nr_to_split && split >= nr_to_split)
497 goto leave;
498
499 page = find_get_page(inode->i_mapping,
500 (inode->i_size & HPAGE_PMD_MASK) >> PAGE_SHIFT);
501 if (!page)
502 goto drop;
503
504
505 if (!PageTransHuge(page)) {
506 put_page(page);
507 goto drop;
508 }
509
510
511
512
513
514
515
516
517 if (!trylock_page(page)) {
518 put_page(page);
519 goto leave;
520 }
521
522 ret = split_huge_page(page);
523 unlock_page(page);
524 put_page(page);
525
526
527 if (ret)
528 goto leave;
529
530 split++;
531drop:
532 list_del_init(&info->shrinklist);
533 removed++;
534leave:
535 iput(inode);
536 }
537
538 spin_lock(&sbinfo->shrinklist_lock);
539 list_splice_tail(&list, &sbinfo->shrinklist);
540 sbinfo->shrinklist_len -= removed;
541 spin_unlock(&sbinfo->shrinklist_lock);
542
543 return split;
544}
545
546static long shmem_unused_huge_scan(struct super_block *sb,
547 struct shrink_control *sc)
548{
549 struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
550
551 if (!READ_ONCE(sbinfo->shrinklist_len))
552 return SHRINK_STOP;
553
554 return shmem_unused_huge_shrink(sbinfo, sc, 0);
555}
556
557static long shmem_unused_huge_count(struct super_block *sb,
558 struct shrink_control *sc)
559{
560 struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
561 return READ_ONCE(sbinfo->shrinklist_len);
562}
563#else
564
565#define shmem_huge SHMEM_HUGE_DENY
566
567static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo,
568 struct shrink_control *sc, unsigned long nr_to_split)
569{
570 return 0;
571}
572#endif
573
574
575
576
577static int shmem_add_to_page_cache(struct page *page,
578 struct address_space *mapping,
579 pgoff_t index, void *expected)
580{
581 int error, nr = hpage_nr_pages(page);
582
583 VM_BUG_ON_PAGE(PageTail(page), page);
584 VM_BUG_ON_PAGE(index != round_down(index, nr), page);
585 VM_BUG_ON_PAGE(!PageLocked(page), page);
586 VM_BUG_ON_PAGE(!PageSwapBacked(page), page);
587 VM_BUG_ON(expected && PageTransHuge(page));
588
589 page_ref_add(page, nr);
590 page->mapping = mapping;
591 page->index = index;
592
593 xa_lock_irq(&mapping->i_pages);
594 if (PageTransHuge(page)) {
595 void __rcu **results;
596 pgoff_t idx;
597 int i;
598
599 error = 0;
600 if (radix_tree_gang_lookup_slot(&mapping->i_pages,
601 &results, &idx, index, 1) &&
602 idx < index + HPAGE_PMD_NR) {
603 error = -EEXIST;
604 }
605
606 if (!error) {
607 for (i = 0; i < HPAGE_PMD_NR; i++) {
608 error = radix_tree_insert(&mapping->i_pages,
609 index + i, page + i);
610 VM_BUG_ON(error);
611 }
612 count_vm_event(THP_FILE_ALLOC);
613 }
614 } else if (!expected) {
615 error = radix_tree_insert(&mapping->i_pages, index, page);
616 } else {
617 error = shmem_radix_tree_replace(mapping, index, expected,
618 page);
619 }
620
621 if (!error) {
622 mapping->nrpages += nr;
623 if (PageTransHuge(page))
624 __inc_node_page_state(page, NR_SHMEM_THPS);
625 __mod_node_page_state(page_pgdat(page), NR_FILE_PAGES, nr);
626 __mod_node_page_state(page_pgdat(page), NR_SHMEM, nr);
627 xa_unlock_irq(&mapping->i_pages);
628 } else {
629 page->mapping = NULL;
630 xa_unlock_irq(&mapping->i_pages);
631 page_ref_sub(page, nr);
632 }
633 return error;
634}
635
636
637
638
639static void shmem_delete_from_page_cache(struct page *page, void *radswap)
640{
641 struct address_space *mapping = page->mapping;
642 int error;
643
644 VM_BUG_ON_PAGE(PageCompound(page), page);
645
646 xa_lock_irq(&mapping->i_pages);
647 error = shmem_radix_tree_replace(mapping, page->index, page, radswap);
648 page->mapping = NULL;
649 mapping->nrpages--;
650 __dec_node_page_state(page, NR_FILE_PAGES);
651 __dec_node_page_state(page, NR_SHMEM);
652 xa_unlock_irq(&mapping->i_pages);
653 put_page(page);
654 BUG_ON(error);
655}
656
657
658
659
660static int shmem_free_swap(struct address_space *mapping,
661 pgoff_t index, void *radswap)
662{
663 void *old;
664
665 xa_lock_irq(&mapping->i_pages);
666 old = radix_tree_delete_item(&mapping->i_pages, index, radswap);
667 xa_unlock_irq(&mapping->i_pages);
668 if (old != radswap)
669 return -ENOENT;
670 free_swap_and_cache(radix_to_swp_entry(radswap));
671 return 0;
672}
673
674
675
676
677
678
679
680
681unsigned long shmem_partial_swap_usage(struct address_space *mapping,
682 pgoff_t start, pgoff_t end)
683{
684 struct radix_tree_iter iter;
685 void **slot;
686 struct page *page;
687 unsigned long swapped = 0;
688
689 rcu_read_lock();
690
691 radix_tree_for_each_slot(slot, &mapping->i_pages, &iter, start) {
692 if (iter.index >= end)
693 break;
694
695 page = radix_tree_deref_slot(slot);
696
697 if (radix_tree_deref_retry(page)) {
698 slot = radix_tree_iter_retry(&iter);
699 continue;
700 }
701
702 if (radix_tree_exceptional_entry(page))
703 swapped++;
704
705 if (need_resched()) {
706 slot = radix_tree_iter_resume(slot, &iter);
707 cond_resched_rcu();
708 }
709 }
710
711 rcu_read_unlock();
712
713 return swapped << PAGE_SHIFT;
714}
715
716
717
718
719
720
721
722
723unsigned long shmem_swap_usage(struct vm_area_struct *vma)
724{
725 struct inode *inode = file_inode(vma->vm_file);
726 struct shmem_inode_info *info = SHMEM_I(inode);
727 struct address_space *mapping = inode->i_mapping;
728 unsigned long swapped;
729
730
731 swapped = READ_ONCE(info->swapped);
732
733
734
735
736
737
738 if (!swapped)
739 return 0;
740
741 if (!vma->vm_pgoff && vma->vm_end - vma->vm_start >= inode->i_size)
742 return swapped << PAGE_SHIFT;
743
744
745 return shmem_partial_swap_usage(mapping,
746 linear_page_index(vma, vma->vm_start),
747 linear_page_index(vma, vma->vm_end));
748}
749
750
751
752
753void shmem_unlock_mapping(struct address_space *mapping)
754{
755 struct pagevec pvec;
756 pgoff_t indices[PAGEVEC_SIZE];
757 pgoff_t index = 0;
758
759 pagevec_init(&pvec);
760
761
762
763 while (!mapping_unevictable(mapping)) {
764
765
766
767
768 pvec.nr = find_get_entries(mapping, index,
769 PAGEVEC_SIZE, pvec.pages, indices);
770 if (!pvec.nr)
771 break;
772 index = indices[pvec.nr - 1] + 1;
773 pagevec_remove_exceptionals(&pvec);
774 check_move_unevictable_pages(pvec.pages, pvec.nr);
775 pagevec_release(&pvec);
776 cond_resched();
777 }
778}
779
780
781
782
783
784static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend,
785 bool unfalloc)
786{
787 struct address_space *mapping = inode->i_mapping;
788 struct shmem_inode_info *info = SHMEM_I(inode);
789 pgoff_t start = (lstart + PAGE_SIZE - 1) >> PAGE_SHIFT;
790 pgoff_t end = (lend + 1) >> PAGE_SHIFT;
791 unsigned int partial_start = lstart & (PAGE_SIZE - 1);
792 unsigned int partial_end = (lend + 1) & (PAGE_SIZE - 1);
793 struct pagevec pvec;
794 pgoff_t indices[PAGEVEC_SIZE];
795 long nr_swaps_freed = 0;
796 pgoff_t index;
797 int i;
798
799 if (lend == -1)
800 end = -1;
801
802 pagevec_init(&pvec);
803 index = start;
804 while (index < end) {
805 pvec.nr = find_get_entries(mapping, index,
806 min(end - index, (pgoff_t)PAGEVEC_SIZE),
807 pvec.pages, indices);
808 if (!pvec.nr)
809 break;
810 for (i = 0; i < pagevec_count(&pvec); i++) {
811 struct page *page = pvec.pages[i];
812
813 index = indices[i];
814 if (index >= end)
815 break;
816
817 if (radix_tree_exceptional_entry(page)) {
818 if (unfalloc)
819 continue;
820 nr_swaps_freed += !shmem_free_swap(mapping,
821 index, page);
822 continue;
823 }
824
825 VM_BUG_ON_PAGE(page_to_pgoff(page) != index, page);
826
827 if (!trylock_page(page))
828 continue;
829
830 if (PageTransTail(page)) {
831
832 clear_highpage(page);
833 unlock_page(page);
834 continue;
835 } else if (PageTransHuge(page)) {
836 if (index == round_down(end, HPAGE_PMD_NR)) {
837
838
839
840
841 clear_highpage(page);
842 unlock_page(page);
843 continue;
844 }
845 index += HPAGE_PMD_NR - 1;
846 i += HPAGE_PMD_NR - 1;
847 }
848
849 if (!unfalloc || !PageUptodate(page)) {
850 VM_BUG_ON_PAGE(PageTail(page), page);
851 if (page_mapping(page) == mapping) {
852 VM_BUG_ON_PAGE(PageWriteback(page), page);
853 truncate_inode_page(mapping, page);
854 }
855 }
856 unlock_page(page);
857 }
858 pagevec_remove_exceptionals(&pvec);
859 pagevec_release(&pvec);
860 cond_resched();
861 index++;
862 }
863
864 if (partial_start) {
865 struct page *page = NULL;
866 shmem_getpage(inode, start - 1, &page, SGP_READ);
867 if (page) {
868 unsigned int top = PAGE_SIZE;
869 if (start > end) {
870 top = partial_end;
871 partial_end = 0;
872 }
873 zero_user_segment(page, partial_start, top);
874 set_page_dirty(page);
875 unlock_page(page);
876 put_page(page);
877 }
878 }
879 if (partial_end) {
880 struct page *page = NULL;
881 shmem_getpage(inode, end, &page, SGP_READ);
882 if (page) {
883 zero_user_segment(page, 0, partial_end);
884 set_page_dirty(page);
885 unlock_page(page);
886 put_page(page);
887 }
888 }
889 if (start >= end)
890 return;
891
892 index = start;
893 while (index < end) {
894 cond_resched();
895
896 pvec.nr = find_get_entries(mapping, index,
897 min(end - index, (pgoff_t)PAGEVEC_SIZE),
898 pvec.pages, indices);
899 if (!pvec.nr) {
900
901 if (index == start || end != -1)
902 break;
903
904 index = start;
905 continue;
906 }
907 for (i = 0; i < pagevec_count(&pvec); i++) {
908 struct page *page = pvec.pages[i];
909
910 index = indices[i];
911 if (index >= end)
912 break;
913
914 if (radix_tree_exceptional_entry(page)) {
915 if (unfalloc)
916 continue;
917 if (shmem_free_swap(mapping, index, page)) {
918
919 index--;
920 break;
921 }
922 nr_swaps_freed++;
923 continue;
924 }
925
926 lock_page(page);
927
928 if (PageTransTail(page)) {
929
930 clear_highpage(page);
931 unlock_page(page);
932
933
934
935
936
937 if (index != round_down(end, HPAGE_PMD_NR))
938 start++;
939 continue;
940 } else if (PageTransHuge(page)) {
941 if (index == round_down(end, HPAGE_PMD_NR)) {
942
943
944
945
946 clear_highpage(page);
947 unlock_page(page);
948 continue;
949 }
950 index += HPAGE_PMD_NR - 1;
951 i += HPAGE_PMD_NR - 1;
952 }
953
954 if (!unfalloc || !PageUptodate(page)) {
955 VM_BUG_ON_PAGE(PageTail(page), page);
956 if (page_mapping(page) == mapping) {
957 VM_BUG_ON_PAGE(PageWriteback(page), page);
958 truncate_inode_page(mapping, page);
959 } else {
960
961 unlock_page(page);
962 index--;
963 break;
964 }
965 }
966 unlock_page(page);
967 }
968 pagevec_remove_exceptionals(&pvec);
969 pagevec_release(&pvec);
970 index++;
971 }
972
973 spin_lock_irq(&info->lock);
974 info->swapped -= nr_swaps_freed;
975 shmem_recalc_inode(inode);
976 spin_unlock_irq(&info->lock);
977}
978
979void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend)
980{
981 shmem_undo_range(inode, lstart, lend, false);
982 inode->i_ctime = inode->i_mtime = current_time(inode);
983}
984EXPORT_SYMBOL_GPL(shmem_truncate_range);
985
986static int shmem_getattr(const struct path *path, struct kstat *stat,
987 u32 request_mask, unsigned int query_flags)
988{
989 struct inode *inode = path->dentry->d_inode;
990 struct shmem_inode_info *info = SHMEM_I(inode);
991
992 if (info->alloced - info->swapped != inode->i_mapping->nrpages) {
993 spin_lock_irq(&info->lock);
994 shmem_recalc_inode(inode);
995 spin_unlock_irq(&info->lock);
996 }
997 generic_fillattr(inode, stat);
998 return 0;
999}
1000
1001static int shmem_setattr(struct dentry *dentry, struct iattr *attr)
1002{
1003 struct inode *inode = d_inode(dentry);
1004 struct shmem_inode_info *info = SHMEM_I(inode);
1005 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
1006 int error;
1007
1008 error = setattr_prepare(dentry, attr);
1009 if (error)
1010 return error;
1011
1012 if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) {
1013 loff_t oldsize = inode->i_size;
1014 loff_t newsize = attr->ia_size;
1015
1016
1017 if ((newsize < oldsize && (info->seals & F_SEAL_SHRINK)) ||
1018 (newsize > oldsize && (info->seals & F_SEAL_GROW)))
1019 return -EPERM;
1020
1021 if (newsize != oldsize) {
1022 error = shmem_reacct_size(SHMEM_I(inode)->flags,
1023 oldsize, newsize);
1024 if (error)
1025 return error;
1026 i_size_write(inode, newsize);
1027 inode->i_ctime = inode->i_mtime = current_time(inode);
1028 }
1029 if (newsize <= oldsize) {
1030 loff_t holebegin = round_up(newsize, PAGE_SIZE);
1031 if (oldsize > holebegin)
1032 unmap_mapping_range(inode->i_mapping,
1033 holebegin, 0, 1);
1034 if (info->alloced)
1035 shmem_truncate_range(inode,
1036 newsize, (loff_t)-1);
1037
1038 if (oldsize > holebegin)
1039 unmap_mapping_range(inode->i_mapping,
1040 holebegin, 0, 1);
1041
1042
1043
1044
1045
1046 if (IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE)) {
1047 spin_lock(&sbinfo->shrinklist_lock);
1048
1049
1050
1051
1052 if (list_empty_careful(&info->shrinklist)) {
1053 list_add_tail(&info->shrinklist,
1054 &sbinfo->shrinklist);
1055 sbinfo->shrinklist_len++;
1056 }
1057 spin_unlock(&sbinfo->shrinklist_lock);
1058 }
1059 }
1060 }
1061
1062 setattr_copy(inode, attr);
1063 if (attr->ia_valid & ATTR_MODE)
1064 error = posix_acl_chmod(inode, inode->i_mode);
1065 return error;
1066}
1067
1068static void shmem_evict_inode(struct inode *inode)
1069{
1070 struct shmem_inode_info *info = SHMEM_I(inode);
1071 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
1072
1073 if (inode->i_mapping->a_ops == &shmem_aops) {
1074 shmem_unacct_size(info->flags, inode->i_size);
1075 inode->i_size = 0;
1076 shmem_truncate_range(inode, 0, (loff_t)-1);
1077 if (!list_empty(&info->shrinklist)) {
1078 spin_lock(&sbinfo->shrinklist_lock);
1079 if (!list_empty(&info->shrinklist)) {
1080 list_del_init(&info->shrinklist);
1081 sbinfo->shrinklist_len--;
1082 }
1083 spin_unlock(&sbinfo->shrinklist_lock);
1084 }
1085 if (!list_empty(&info->swaplist)) {
1086 mutex_lock(&shmem_swaplist_mutex);
1087 list_del_init(&info->swaplist);
1088 mutex_unlock(&shmem_swaplist_mutex);
1089 }
1090 }
1091
1092 simple_xattrs_free(&info->xattrs);
1093 WARN_ON(inode->i_blocks);
1094 shmem_free_inode(inode->i_sb);
1095 clear_inode(inode);
1096}
1097
1098static unsigned long find_swap_entry(struct radix_tree_root *root, void *item)
1099{
1100 struct radix_tree_iter iter;
1101 void **slot;
1102 unsigned long found = -1;
1103 unsigned int checked = 0;
1104
1105 rcu_read_lock();
1106 radix_tree_for_each_slot(slot, root, &iter, 0) {
1107 if (*slot == item) {
1108 found = iter.index;
1109 break;
1110 }
1111 checked++;
1112 if ((checked % 4096) != 0)
1113 continue;
1114 slot = radix_tree_iter_resume(slot, &iter);
1115 cond_resched_rcu();
1116 }
1117
1118 rcu_read_unlock();
1119 return found;
1120}
1121
1122
1123
1124
1125static int shmem_unuse_inode(struct shmem_inode_info *info,
1126 swp_entry_t swap, struct page **pagep)
1127{
1128 struct address_space *mapping = info->vfs_inode.i_mapping;
1129 void *radswap;
1130 pgoff_t index;
1131 gfp_t gfp;
1132 int error = 0;
1133
1134 radswap = swp_to_radix_entry(swap);
1135 index = find_swap_entry(&mapping->i_pages, radswap);
1136 if (index == -1)
1137 return -EAGAIN;
1138
1139
1140
1141
1142
1143
1144
1145 if (shmem_swaplist.next != &info->swaplist)
1146 list_move_tail(&shmem_swaplist, &info->swaplist);
1147
1148 gfp = mapping_gfp_mask(mapping);
1149 if (shmem_should_replace_page(*pagep, gfp)) {
1150 mutex_unlock(&shmem_swaplist_mutex);
1151 error = shmem_replace_page(pagep, gfp, info, index);
1152 mutex_lock(&shmem_swaplist_mutex);
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171 if (!page_swapcount(*pagep))
1172 error = -ENOENT;
1173 }
1174
1175
1176
1177
1178
1179
1180 if (!error)
1181 error = shmem_add_to_page_cache(*pagep, mapping, index,
1182 radswap);
1183 if (error != -ENOMEM) {
1184
1185
1186
1187
1188 delete_from_swap_cache(*pagep);
1189 set_page_dirty(*pagep);
1190 if (!error) {
1191 spin_lock_irq(&info->lock);
1192 info->swapped--;
1193 spin_unlock_irq(&info->lock);
1194 swap_free(swap);
1195 }
1196 }
1197 return error;
1198}
1199
1200
1201
1202
1203int shmem_unuse(swp_entry_t swap, struct page *page)
1204{
1205 struct list_head *this, *next;
1206 struct shmem_inode_info *info;
1207 struct mem_cgroup *memcg;
1208 int error = 0;
1209
1210
1211
1212
1213
1214 if (unlikely(!PageSwapCache(page) || page_private(page) != swap.val))
1215 goto out;
1216
1217
1218
1219
1220
1221
1222 error = mem_cgroup_try_charge(page, current->mm, GFP_KERNEL, &memcg,
1223 false);
1224 if (error)
1225 goto out;
1226
1227 error = -EAGAIN;
1228
1229 mutex_lock(&shmem_swaplist_mutex);
1230 list_for_each_safe(this, next, &shmem_swaplist) {
1231 info = list_entry(this, struct shmem_inode_info, swaplist);
1232 if (info->swapped)
1233 error = shmem_unuse_inode(info, swap, &page);
1234 else
1235 list_del_init(&info->swaplist);
1236 cond_resched();
1237 if (error != -EAGAIN)
1238 break;
1239
1240 }
1241 mutex_unlock(&shmem_swaplist_mutex);
1242
1243 if (error) {
1244 if (error != -ENOMEM)
1245 error = 0;
1246 mem_cgroup_cancel_charge(page, memcg, false);
1247 } else
1248 mem_cgroup_commit_charge(page, memcg, true, false);
1249out:
1250 unlock_page(page);
1251 put_page(page);
1252 return error;
1253}
1254
1255
1256
1257
1258static int shmem_writepage(struct page *page, struct writeback_control *wbc)
1259{
1260 struct shmem_inode_info *info;
1261 struct address_space *mapping;
1262 struct inode *inode;
1263 swp_entry_t swap;
1264 pgoff_t index;
1265
1266 VM_BUG_ON_PAGE(PageCompound(page), page);
1267 BUG_ON(!PageLocked(page));
1268 mapping = page->mapping;
1269 index = page->index;
1270 inode = mapping->host;
1271 info = SHMEM_I(inode);
1272 if (info->flags & VM_LOCKED)
1273 goto redirty;
1274 if (!total_swap_pages)
1275 goto redirty;
1276
1277
1278
1279
1280
1281
1282
1283
1284 if (!wbc->for_reclaim) {
1285 WARN_ON_ONCE(1);
1286 goto redirty;
1287 }
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300 if (!PageUptodate(page)) {
1301 if (inode->i_private) {
1302 struct shmem_falloc *shmem_falloc;
1303 spin_lock(&inode->i_lock);
1304 shmem_falloc = inode->i_private;
1305 if (shmem_falloc &&
1306 !shmem_falloc->waitq &&
1307 index >= shmem_falloc->start &&
1308 index < shmem_falloc->next)
1309 shmem_falloc->nr_unswapped++;
1310 else
1311 shmem_falloc = NULL;
1312 spin_unlock(&inode->i_lock);
1313 if (shmem_falloc)
1314 goto redirty;
1315 }
1316 clear_highpage(page);
1317 flush_dcache_page(page);
1318 SetPageUptodate(page);
1319 }
1320
1321 swap = get_swap_page(page);
1322 if (!swap.val)
1323 goto redirty;
1324
1325 if (mem_cgroup_try_charge_swap(page, swap))
1326 goto free_swap;
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336 mutex_lock(&shmem_swaplist_mutex);
1337 if (list_empty(&info->swaplist))
1338 list_add_tail(&info->swaplist, &shmem_swaplist);
1339
1340 if (add_to_swap_cache(page, swap, GFP_ATOMIC) == 0) {
1341 spin_lock_irq(&info->lock);
1342 shmem_recalc_inode(inode);
1343 info->swapped++;
1344 spin_unlock_irq(&info->lock);
1345
1346 swap_shmem_alloc(swap);
1347 shmem_delete_from_page_cache(page, swp_to_radix_entry(swap));
1348
1349 mutex_unlock(&shmem_swaplist_mutex);
1350 BUG_ON(page_mapped(page));
1351 swap_writepage(page, wbc);
1352 return 0;
1353 }
1354
1355 mutex_unlock(&shmem_swaplist_mutex);
1356free_swap:
1357 put_swap_page(page, swap);
1358redirty:
1359 set_page_dirty(page);
1360 if (wbc->for_reclaim)
1361 return AOP_WRITEPAGE_ACTIVATE;
1362 unlock_page(page);
1363 return 0;
1364}
1365
1366#if defined(CONFIG_NUMA) && defined(CONFIG_TMPFS)
1367static void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol)
1368{
1369 char buffer[64];
1370
1371 if (!mpol || mpol->mode == MPOL_DEFAULT)
1372 return;
1373
1374 mpol_to_str(buffer, sizeof(buffer), mpol);
1375
1376 seq_printf(seq, ",mpol=%s", buffer);
1377}
1378
1379static struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo)
1380{
1381 struct mempolicy *mpol = NULL;
1382 if (sbinfo->mpol) {
1383 spin_lock(&sbinfo->stat_lock);
1384 mpol = sbinfo->mpol;
1385 mpol_get(mpol);
1386 spin_unlock(&sbinfo->stat_lock);
1387 }
1388 return mpol;
1389}
1390#else
1391static inline void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol)
1392{
1393}
1394static inline struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo)
1395{
1396 return NULL;
1397}
1398#endif
1399#ifndef CONFIG_NUMA
1400#define vm_policy vm_private_data
1401#endif
1402
1403static void shmem_pseudo_vma_init(struct vm_area_struct *vma,
1404 struct shmem_inode_info *info, pgoff_t index)
1405{
1406
1407 vma->vm_start = 0;
1408
1409 vma->vm_pgoff = index + info->vfs_inode.i_ino;
1410 vma->vm_ops = NULL;
1411 vma->vm_policy = mpol_shared_policy_lookup(&info->policy, index);
1412}
1413
1414static void shmem_pseudo_vma_destroy(struct vm_area_struct *vma)
1415{
1416
1417 mpol_cond_put(vma->vm_policy);
1418}
1419
1420static struct page *shmem_swapin(swp_entry_t swap, gfp_t gfp,
1421 struct shmem_inode_info *info, pgoff_t index)
1422{
1423 struct vm_area_struct pvma;
1424 struct page *page;
1425 struct vm_fault vmf;
1426
1427 shmem_pseudo_vma_init(&pvma, info, index);
1428 vmf.vma = &pvma;
1429 vmf.address = 0;
1430 page = swap_cluster_readahead(swap, gfp, &vmf);
1431 shmem_pseudo_vma_destroy(&pvma);
1432
1433 return page;
1434}
1435
1436static struct page *shmem_alloc_hugepage(gfp_t gfp,
1437 struct shmem_inode_info *info, pgoff_t index)
1438{
1439 struct vm_area_struct pvma;
1440 struct inode *inode = &info->vfs_inode;
1441 struct address_space *mapping = inode->i_mapping;
1442 pgoff_t idx, hindex;
1443 void __rcu **results;
1444 struct page *page;
1445
1446 if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE))
1447 return NULL;
1448
1449 hindex = round_down(index, HPAGE_PMD_NR);
1450 rcu_read_lock();
1451 if (radix_tree_gang_lookup_slot(&mapping->i_pages, &results, &idx,
1452 hindex, 1) && idx < hindex + HPAGE_PMD_NR) {
1453 rcu_read_unlock();
1454 return NULL;
1455 }
1456 rcu_read_unlock();
1457
1458 shmem_pseudo_vma_init(&pvma, info, hindex);
1459 page = alloc_pages_vma(gfp | __GFP_COMP | __GFP_NORETRY | __GFP_NOWARN,
1460 HPAGE_PMD_ORDER, &pvma, 0, numa_node_id(), true);
1461 shmem_pseudo_vma_destroy(&pvma);
1462 if (page)
1463 prep_transhuge_page(page);
1464 return page;
1465}
1466
1467static struct page *shmem_alloc_page(gfp_t gfp,
1468 struct shmem_inode_info *info, pgoff_t index)
1469{
1470 struct vm_area_struct pvma;
1471 struct page *page;
1472
1473 shmem_pseudo_vma_init(&pvma, info, index);
1474 page = alloc_page_vma(gfp, &pvma, 0);
1475 shmem_pseudo_vma_destroy(&pvma);
1476
1477 return page;
1478}
1479
1480static struct page *shmem_alloc_and_acct_page(gfp_t gfp,
1481 struct inode *inode,
1482 pgoff_t index, bool huge)
1483{
1484 struct shmem_inode_info *info = SHMEM_I(inode);
1485 struct page *page;
1486 int nr;
1487 int err = -ENOSPC;
1488
1489 if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE))
1490 huge = false;
1491 nr = huge ? HPAGE_PMD_NR : 1;
1492
1493 if (!shmem_inode_acct_block(inode, nr))
1494 goto failed;
1495
1496 if (huge)
1497 page = shmem_alloc_hugepage(gfp, info, index);
1498 else
1499 page = shmem_alloc_page(gfp, info, index);
1500 if (page) {
1501 __SetPageLocked(page);
1502 __SetPageSwapBacked(page);
1503 return page;
1504 }
1505
1506 err = -ENOMEM;
1507 shmem_inode_unacct_blocks(inode, nr);
1508failed:
1509 return ERR_PTR(err);
1510}
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524static bool shmem_should_replace_page(struct page *page, gfp_t gfp)
1525{
1526 return page_zonenum(page) > gfp_zone(gfp);
1527}
1528
1529static int shmem_replace_page(struct page **pagep, gfp_t gfp,
1530 struct shmem_inode_info *info, pgoff_t index)
1531{
1532 struct page *oldpage, *newpage;
1533 struct address_space *swap_mapping;
1534 pgoff_t swap_index;
1535 int error;
1536
1537 oldpage = *pagep;
1538 swap_index = page_private(oldpage);
1539 swap_mapping = page_mapping(oldpage);
1540
1541
1542
1543
1544
1545 gfp &= ~GFP_CONSTRAINT_MASK;
1546 newpage = shmem_alloc_page(gfp, info, index);
1547 if (!newpage)
1548 return -ENOMEM;
1549
1550 get_page(newpage);
1551 copy_highpage(newpage, oldpage);
1552 flush_dcache_page(newpage);
1553
1554 __SetPageLocked(newpage);
1555 __SetPageSwapBacked(newpage);
1556 SetPageUptodate(newpage);
1557 set_page_private(newpage, swap_index);
1558 SetPageSwapCache(newpage);
1559
1560
1561
1562
1563
1564 xa_lock_irq(&swap_mapping->i_pages);
1565 error = shmem_radix_tree_replace(swap_mapping, swap_index, oldpage,
1566 newpage);
1567 if (!error) {
1568 __inc_node_page_state(newpage, NR_FILE_PAGES);
1569 __dec_node_page_state(oldpage, NR_FILE_PAGES);
1570 }
1571 xa_unlock_irq(&swap_mapping->i_pages);
1572
1573 if (unlikely(error)) {
1574
1575
1576
1577
1578
1579 oldpage = newpage;
1580 } else {
1581 mem_cgroup_migrate(oldpage, newpage);
1582 lru_cache_add_anon(newpage);
1583 *pagep = newpage;
1584 }
1585
1586 ClearPageSwapCache(oldpage);
1587 set_page_private(oldpage, 0);
1588
1589 unlock_page(oldpage);
1590 put_page(oldpage);
1591 put_page(oldpage);
1592 return error;
1593}
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605static int shmem_getpage_gfp(struct inode *inode, pgoff_t index,
1606 struct page **pagep, enum sgp_type sgp, gfp_t gfp,
1607 struct vm_area_struct *vma, struct vm_fault *vmf, int *fault_type)
1608{
1609 struct address_space *mapping = inode->i_mapping;
1610 struct shmem_inode_info *info = SHMEM_I(inode);
1611 struct shmem_sb_info *sbinfo;
1612 struct mm_struct *charge_mm;
1613 struct mem_cgroup *memcg;
1614 struct page *page;
1615 swp_entry_t swap;
1616 enum sgp_type sgp_huge = sgp;
1617 pgoff_t hindex = index;
1618 int error;
1619 int once = 0;
1620 int alloced = 0;
1621
1622 if (index > (MAX_LFS_FILESIZE >> PAGE_SHIFT))
1623 return -EFBIG;
1624 if (sgp == SGP_NOHUGE || sgp == SGP_HUGE)
1625 sgp = SGP_CACHE;
1626repeat:
1627 swap.val = 0;
1628 page = find_lock_entry(mapping, index);
1629 if (radix_tree_exceptional_entry(page)) {
1630 swap = radix_to_swp_entry(page);
1631 page = NULL;
1632 }
1633
1634 if (sgp <= SGP_CACHE &&
1635 ((loff_t)index << PAGE_SHIFT) >= i_size_read(inode)) {
1636 error = -EINVAL;
1637 goto unlock;
1638 }
1639
1640 if (page && sgp == SGP_WRITE)
1641 mark_page_accessed(page);
1642
1643
1644 if (page && !PageUptodate(page)) {
1645 if (sgp != SGP_READ)
1646 goto clear;
1647 unlock_page(page);
1648 put_page(page);
1649 page = NULL;
1650 }
1651 if (page || (sgp == SGP_READ && !swap.val)) {
1652 *pagep = page;
1653 return 0;
1654 }
1655
1656
1657
1658
1659
1660 sbinfo = SHMEM_SB(inode->i_sb);
1661 charge_mm = vma ? vma->vm_mm : current->mm;
1662
1663 if (swap.val) {
1664
1665 page = lookup_swap_cache(swap, NULL, 0);
1666 if (!page) {
1667
1668 if (fault_type) {
1669 *fault_type |= VM_FAULT_MAJOR;
1670 count_vm_event(PGMAJFAULT);
1671 count_memcg_event_mm(charge_mm, PGMAJFAULT);
1672 }
1673
1674 page = shmem_swapin(swap, gfp, info, index);
1675 if (!page) {
1676 error = -ENOMEM;
1677 goto failed;
1678 }
1679 }
1680
1681
1682 lock_page(page);
1683 if (!PageSwapCache(page) || page_private(page) != swap.val ||
1684 !shmem_confirm_swap(mapping, index, swap)) {
1685 error = -EEXIST;
1686 goto unlock;
1687 }
1688 if (!PageUptodate(page)) {
1689 error = -EIO;
1690 goto failed;
1691 }
1692 wait_on_page_writeback(page);
1693
1694 if (shmem_should_replace_page(page, gfp)) {
1695 error = shmem_replace_page(&page, gfp, info, index);
1696 if (error)
1697 goto failed;
1698 }
1699
1700 error = mem_cgroup_try_charge(page, charge_mm, gfp, &memcg,
1701 false);
1702 if (!error) {
1703 error = shmem_add_to_page_cache(page, mapping, index,
1704 swp_to_radix_entry(swap));
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717 if (error) {
1718 mem_cgroup_cancel_charge(page, memcg, false);
1719 delete_from_swap_cache(page);
1720 }
1721 }
1722 if (error)
1723 goto failed;
1724
1725 mem_cgroup_commit_charge(page, memcg, true, false);
1726
1727 spin_lock_irq(&info->lock);
1728 info->swapped--;
1729 shmem_recalc_inode(inode);
1730 spin_unlock_irq(&info->lock);
1731
1732 if (sgp == SGP_WRITE)
1733 mark_page_accessed(page);
1734
1735 delete_from_swap_cache(page);
1736 set_page_dirty(page);
1737 swap_free(swap);
1738
1739 } else {
1740 if (vma && userfaultfd_missing(vma)) {
1741 *fault_type = handle_userfault(vmf, VM_UFFD_MISSING);
1742 return 0;
1743 }
1744
1745
1746 if (mapping->a_ops != &shmem_aops)
1747 goto alloc_nohuge;
1748 if (shmem_huge == SHMEM_HUGE_DENY || sgp_huge == SGP_NOHUGE)
1749 goto alloc_nohuge;
1750 if (shmem_huge == SHMEM_HUGE_FORCE)
1751 goto alloc_huge;
1752 switch (sbinfo->huge) {
1753 loff_t i_size;
1754 pgoff_t off;
1755 case SHMEM_HUGE_NEVER:
1756 goto alloc_nohuge;
1757 case SHMEM_HUGE_WITHIN_SIZE:
1758 off = round_up(index, HPAGE_PMD_NR);
1759 i_size = round_up(i_size_read(inode), PAGE_SIZE);
1760 if (i_size >= HPAGE_PMD_SIZE &&
1761 i_size >> PAGE_SHIFT >= off)
1762 goto alloc_huge;
1763
1764 case SHMEM_HUGE_ADVISE:
1765 if (sgp_huge == SGP_HUGE)
1766 goto alloc_huge;
1767
1768 goto alloc_nohuge;
1769 }
1770
1771alloc_huge:
1772 page = shmem_alloc_and_acct_page(gfp, inode, index, true);
1773 if (IS_ERR(page)) {
1774alloc_nohuge: page = shmem_alloc_and_acct_page(gfp, inode,
1775 index, false);
1776 }
1777 if (IS_ERR(page)) {
1778 int retry = 5;
1779 error = PTR_ERR(page);
1780 page = NULL;
1781 if (error != -ENOSPC)
1782 goto failed;
1783
1784
1785
1786
1787 while (retry--) {
1788 int ret;
1789 ret = shmem_unused_huge_shrink(sbinfo, NULL, 1);
1790 if (ret == SHRINK_STOP)
1791 break;
1792 if (ret)
1793 goto alloc_nohuge;
1794 }
1795 goto failed;
1796 }
1797
1798 if (PageTransHuge(page))
1799 hindex = round_down(index, HPAGE_PMD_NR);
1800 else
1801 hindex = index;
1802
1803 if (sgp == SGP_WRITE)
1804 __SetPageReferenced(page);
1805
1806 error = mem_cgroup_try_charge(page, charge_mm, gfp, &memcg,
1807 PageTransHuge(page));
1808 if (error)
1809 goto unacct;
1810 error = radix_tree_maybe_preload_order(gfp & GFP_RECLAIM_MASK,
1811 compound_order(page));
1812 if (!error) {
1813 error = shmem_add_to_page_cache(page, mapping, hindex,
1814 NULL);
1815 radix_tree_preload_end();
1816 }
1817 if (error) {
1818 mem_cgroup_cancel_charge(page, memcg,
1819 PageTransHuge(page));
1820 goto unacct;
1821 }
1822 mem_cgroup_commit_charge(page, memcg, false,
1823 PageTransHuge(page));
1824 lru_cache_add_anon(page);
1825
1826 spin_lock_irq(&info->lock);
1827 info->alloced += 1 << compound_order(page);
1828 inode->i_blocks += BLOCKS_PER_PAGE << compound_order(page);
1829 shmem_recalc_inode(inode);
1830 spin_unlock_irq(&info->lock);
1831 alloced = true;
1832
1833 if (PageTransHuge(page) &&
1834 DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE) <
1835 hindex + HPAGE_PMD_NR - 1) {
1836
1837
1838
1839
1840 spin_lock(&sbinfo->shrinklist_lock);
1841
1842
1843
1844
1845 if (list_empty_careful(&info->shrinklist)) {
1846 list_add_tail(&info->shrinklist,
1847 &sbinfo->shrinklist);
1848 sbinfo->shrinklist_len++;
1849 }
1850 spin_unlock(&sbinfo->shrinklist_lock);
1851 }
1852
1853
1854
1855
1856 if (sgp == SGP_FALLOC)
1857 sgp = SGP_WRITE;
1858clear:
1859
1860
1861
1862
1863
1864 if (sgp != SGP_WRITE && !PageUptodate(page)) {
1865 struct page *head = compound_head(page);
1866 int i;
1867
1868 for (i = 0; i < (1 << compound_order(head)); i++) {
1869 clear_highpage(head + i);
1870 flush_dcache_page(head + i);
1871 }
1872 SetPageUptodate(head);
1873 }
1874 }
1875
1876
1877 if (sgp <= SGP_CACHE &&
1878 ((loff_t)index << PAGE_SHIFT) >= i_size_read(inode)) {
1879 if (alloced) {
1880 ClearPageDirty(page);
1881 delete_from_page_cache(page);
1882 spin_lock_irq(&info->lock);
1883 shmem_recalc_inode(inode);
1884 spin_unlock_irq(&info->lock);
1885 }
1886 error = -EINVAL;
1887 goto unlock;
1888 }
1889 *pagep = page + index - hindex;
1890 return 0;
1891
1892
1893
1894
1895unacct:
1896 shmem_inode_unacct_blocks(inode, 1 << compound_order(page));
1897
1898 if (PageTransHuge(page)) {
1899 unlock_page(page);
1900 put_page(page);
1901 goto alloc_nohuge;
1902 }
1903failed:
1904 if (swap.val && !shmem_confirm_swap(mapping, index, swap))
1905 error = -EEXIST;
1906unlock:
1907 if (page) {
1908 unlock_page(page);
1909 put_page(page);
1910 }
1911 if (error == -ENOSPC && !once++) {
1912 spin_lock_irq(&info->lock);
1913 shmem_recalc_inode(inode);
1914 spin_unlock_irq(&info->lock);
1915 goto repeat;
1916 }
1917 if (error == -EEXIST)
1918 goto repeat;
1919 return error;
1920}
1921
1922
1923
1924
1925
1926
1927static int synchronous_wake_function(wait_queue_entry_t *wait, unsigned mode, int sync, void *key)
1928{
1929 int ret = default_wake_function(wait, mode, sync, key);
1930 list_del_init(&wait->entry);
1931 return ret;
1932}
1933
1934static int shmem_fault(struct vm_fault *vmf)
1935{
1936 struct vm_area_struct *vma = vmf->vma;
1937 struct inode *inode = file_inode(vma->vm_file);
1938 gfp_t gfp = mapping_gfp_mask(inode->i_mapping);
1939 enum sgp_type sgp;
1940 int error;
1941 int ret = VM_FAULT_LOCKED;
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960 if (unlikely(inode->i_private)) {
1961 struct shmem_falloc *shmem_falloc;
1962
1963 spin_lock(&inode->i_lock);
1964 shmem_falloc = inode->i_private;
1965 if (shmem_falloc &&
1966 shmem_falloc->waitq &&
1967 vmf->pgoff >= shmem_falloc->start &&
1968 vmf->pgoff < shmem_falloc->next) {
1969 wait_queue_head_t *shmem_falloc_waitq;
1970 DEFINE_WAIT_FUNC(shmem_fault_wait, synchronous_wake_function);
1971
1972 ret = VM_FAULT_NOPAGE;
1973 if ((vmf->flags & FAULT_FLAG_ALLOW_RETRY) &&
1974 !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) {
1975
1976 up_read(&vma->vm_mm->mmap_sem);
1977 ret = VM_FAULT_RETRY;
1978 }
1979
1980 shmem_falloc_waitq = shmem_falloc->waitq;
1981 prepare_to_wait(shmem_falloc_waitq, &shmem_fault_wait,
1982 TASK_UNINTERRUPTIBLE);
1983 spin_unlock(&inode->i_lock);
1984 schedule();
1985
1986
1987
1988
1989
1990
1991
1992
1993 spin_lock(&inode->i_lock);
1994 finish_wait(shmem_falloc_waitq, &shmem_fault_wait);
1995 spin_unlock(&inode->i_lock);
1996 return ret;
1997 }
1998 spin_unlock(&inode->i_lock);
1999 }
2000
2001 sgp = SGP_CACHE;
2002
2003 if ((vma->vm_flags & VM_NOHUGEPAGE) ||
2004 test_bit(MMF_DISABLE_THP, &vma->vm_mm->flags))
2005 sgp = SGP_NOHUGE;
2006 else if (vma->vm_flags & VM_HUGEPAGE)
2007 sgp = SGP_HUGE;
2008
2009 error = shmem_getpage_gfp(inode, vmf->pgoff, &vmf->page, sgp,
2010 gfp, vma, vmf, &ret);
2011 if (error)
2012 return ((error == -ENOMEM) ? VM_FAULT_OOM : VM_FAULT_SIGBUS);
2013 return ret;
2014}
2015
2016unsigned long shmem_get_unmapped_area(struct file *file,
2017 unsigned long uaddr, unsigned long len,
2018 unsigned long pgoff, unsigned long flags)
2019{
2020 unsigned long (*get_area)(struct file *,
2021 unsigned long, unsigned long, unsigned long, unsigned long);
2022 unsigned long addr;
2023 unsigned long offset;
2024 unsigned long inflated_len;
2025 unsigned long inflated_addr;
2026 unsigned long inflated_offset;
2027
2028 if (len > TASK_SIZE)
2029 return -ENOMEM;
2030
2031 get_area = current->mm->get_unmapped_area;
2032 addr = get_area(file, uaddr, len, pgoff, flags);
2033
2034 if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE))
2035 return addr;
2036 if (IS_ERR_VALUE(addr))
2037 return addr;
2038 if (addr & ~PAGE_MASK)
2039 return addr;
2040 if (addr > TASK_SIZE - len)
2041 return addr;
2042
2043 if (shmem_huge == SHMEM_HUGE_DENY)
2044 return addr;
2045 if (len < HPAGE_PMD_SIZE)
2046 return addr;
2047 if (flags & MAP_FIXED)
2048 return addr;
2049
2050
2051
2052
2053
2054 if (uaddr)
2055 return addr;
2056
2057 if (shmem_huge != SHMEM_HUGE_FORCE) {
2058 struct super_block *sb;
2059
2060 if (file) {
2061 VM_BUG_ON(file->f_op != &shmem_file_operations);
2062 sb = file_inode(file)->i_sb;
2063 } else {
2064
2065
2066
2067
2068 if (IS_ERR(shm_mnt))
2069 return addr;
2070 sb = shm_mnt->mnt_sb;
2071 }
2072 if (SHMEM_SB(sb)->huge == SHMEM_HUGE_NEVER)
2073 return addr;
2074 }
2075
2076 offset = (pgoff << PAGE_SHIFT) & (HPAGE_PMD_SIZE-1);
2077 if (offset && offset + len < 2 * HPAGE_PMD_SIZE)
2078 return addr;
2079 if ((addr & (HPAGE_PMD_SIZE-1)) == offset)
2080 return addr;
2081
2082 inflated_len = len + HPAGE_PMD_SIZE - PAGE_SIZE;
2083 if (inflated_len > TASK_SIZE)
2084 return addr;
2085 if (inflated_len < len)
2086 return addr;
2087
2088 inflated_addr = get_area(NULL, 0, inflated_len, 0, flags);
2089 if (IS_ERR_VALUE(inflated_addr))
2090 return addr;
2091 if (inflated_addr & ~PAGE_MASK)
2092 return addr;
2093
2094 inflated_offset = inflated_addr & (HPAGE_PMD_SIZE-1);
2095 inflated_addr += offset - inflated_offset;
2096 if (inflated_offset > offset)
2097 inflated_addr += HPAGE_PMD_SIZE;
2098
2099 if (inflated_addr > TASK_SIZE - len)
2100 return addr;
2101 return inflated_addr;
2102}
2103
2104#ifdef CONFIG_NUMA
2105static int shmem_set_policy(struct vm_area_struct *vma, struct mempolicy *mpol)
2106{
2107 struct inode *inode = file_inode(vma->vm_file);
2108 return mpol_set_shared_policy(&SHMEM_I(inode)->policy, vma, mpol);
2109}
2110
2111static struct mempolicy *shmem_get_policy(struct vm_area_struct *vma,
2112 unsigned long addr)
2113{
2114 struct inode *inode = file_inode(vma->vm_file);
2115 pgoff_t index;
2116
2117 index = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
2118 return mpol_shared_policy_lookup(&SHMEM_I(inode)->policy, index);
2119}
2120#endif
2121
2122int shmem_lock(struct file *file, int lock, struct user_struct *user)
2123{
2124 struct inode *inode = file_inode(file);
2125 struct shmem_inode_info *info = SHMEM_I(inode);
2126 int retval = -ENOMEM;
2127
2128 spin_lock_irq(&info->lock);
2129 if (lock && !(info->flags & VM_LOCKED)) {
2130 if (!user_shm_lock(inode->i_size, user))
2131 goto out_nomem;
2132 info->flags |= VM_LOCKED;
2133 mapping_set_unevictable(file->f_mapping);
2134 }
2135 if (!lock && (info->flags & VM_LOCKED) && user) {
2136 user_shm_unlock(inode->i_size, user);
2137 info->flags &= ~VM_LOCKED;
2138 mapping_clear_unevictable(file->f_mapping);
2139 }
2140 retval = 0;
2141
2142out_nomem:
2143 spin_unlock_irq(&info->lock);
2144 return retval;
2145}
2146
2147static int shmem_mmap(struct file *file, struct vm_area_struct *vma)
2148{
2149 file_accessed(file);
2150 vma->vm_ops = &shmem_vm_ops;
2151 if (IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE) &&
2152 ((vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK) <
2153 (vma->vm_end & HPAGE_PMD_MASK)) {
2154 khugepaged_enter(vma, vma->vm_flags);
2155 }
2156 return 0;
2157}
2158
2159static struct inode *shmem_get_inode(struct super_block *sb, const struct inode *dir,
2160 umode_t mode, dev_t dev, unsigned long flags)
2161{
2162 struct inode *inode;
2163 struct shmem_inode_info *info;
2164 struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
2165
2166 if (shmem_reserve_inode(sb))
2167 return NULL;
2168
2169 inode = new_inode(sb);
2170 if (inode) {
2171 inode->i_ino = get_next_ino();
2172 inode_init_owner(inode, dir, mode);
2173 inode->i_blocks = 0;
2174 inode->i_atime = inode->i_mtime = inode->i_ctime = current_time(inode);
2175 inode->i_generation = get_seconds();
2176 info = SHMEM_I(inode);
2177 memset(info, 0, (char *)inode - (char *)info);
2178 spin_lock_init(&info->lock);
2179 info->seals = F_SEAL_SEAL;
2180 info->flags = flags & VM_NORESERVE;
2181 INIT_LIST_HEAD(&info->shrinklist);
2182 INIT_LIST_HEAD(&info->swaplist);
2183 simple_xattrs_init(&info->xattrs);
2184 cache_no_acl(inode);
2185
2186 switch (mode & S_IFMT) {
2187 default:
2188 inode->i_op = &shmem_special_inode_operations;
2189 init_special_inode(inode, mode, dev);
2190 break;
2191 case S_IFREG:
2192 inode->i_mapping->a_ops = &shmem_aops;
2193 inode->i_op = &shmem_inode_operations;
2194 inode->i_fop = &shmem_file_operations;
2195 mpol_shared_policy_init(&info->policy,
2196 shmem_get_sbmpol(sbinfo));
2197 break;
2198 case S_IFDIR:
2199 inc_nlink(inode);
2200
2201 inode->i_size = 2 * BOGO_DIRENT_SIZE;
2202 inode->i_op = &shmem_dir_inode_operations;
2203 inode->i_fop = &simple_dir_operations;
2204 break;
2205 case S_IFLNK:
2206
2207
2208
2209
2210 mpol_shared_policy_init(&info->policy, NULL);
2211 break;
2212 }
2213 } else
2214 shmem_free_inode(sb);
2215 return inode;
2216}
2217
2218bool shmem_mapping(struct address_space *mapping)
2219{
2220 return mapping->a_ops == &shmem_aops;
2221}
2222
2223static int shmem_mfill_atomic_pte(struct mm_struct *dst_mm,
2224 pmd_t *dst_pmd,
2225 struct vm_area_struct *dst_vma,
2226 unsigned long dst_addr,
2227 unsigned long src_addr,
2228 bool zeropage,
2229 struct page **pagep)
2230{
2231 struct inode *inode = file_inode(dst_vma->vm_file);
2232 struct shmem_inode_info *info = SHMEM_I(inode);
2233 struct address_space *mapping = inode->i_mapping;
2234 gfp_t gfp = mapping_gfp_mask(mapping);
2235 pgoff_t pgoff = linear_page_index(dst_vma, dst_addr);
2236 struct mem_cgroup *memcg;
2237 spinlock_t *ptl;
2238 void *page_kaddr;
2239 struct page *page;
2240 pte_t _dst_pte, *dst_pte;
2241 int ret;
2242
2243 ret = -ENOMEM;
2244 if (!shmem_inode_acct_block(inode, 1))
2245 goto out;
2246
2247 if (!*pagep) {
2248 page = shmem_alloc_page(gfp, info, pgoff);
2249 if (!page)
2250 goto out_unacct_blocks;
2251
2252 if (!zeropage) {
2253 page_kaddr = kmap_atomic(page);
2254 ret = copy_from_user(page_kaddr,
2255 (const void __user *)src_addr,
2256 PAGE_SIZE);
2257 kunmap_atomic(page_kaddr);
2258
2259
2260 if (unlikely(ret)) {
2261 *pagep = page;
2262 shmem_inode_unacct_blocks(inode, 1);
2263
2264 return -EFAULT;
2265 }
2266 } else {
2267 clear_highpage(page);
2268 }
2269 } else {
2270 page = *pagep;
2271 *pagep = NULL;
2272 }
2273
2274 VM_BUG_ON(PageLocked(page) || PageSwapBacked(page));
2275 __SetPageLocked(page);
2276 __SetPageSwapBacked(page);
2277 __SetPageUptodate(page);
2278
2279 ret = mem_cgroup_try_charge(page, dst_mm, gfp, &memcg, false);
2280 if (ret)
2281 goto out_release;
2282
2283 ret = radix_tree_maybe_preload(gfp & GFP_RECLAIM_MASK);
2284 if (!ret) {
2285 ret = shmem_add_to_page_cache(page, mapping, pgoff, NULL);
2286 radix_tree_preload_end();
2287 }
2288 if (ret)
2289 goto out_release_uncharge;
2290
2291 mem_cgroup_commit_charge(page, memcg, false, false);
2292
2293 _dst_pte = mk_pte(page, dst_vma->vm_page_prot);
2294 if (dst_vma->vm_flags & VM_WRITE)
2295 _dst_pte = pte_mkwrite(pte_mkdirty(_dst_pte));
2296
2297 ret = -EEXIST;
2298 dst_pte = pte_offset_map_lock(dst_mm, dst_pmd, dst_addr, &ptl);
2299 if (!pte_none(*dst_pte))
2300 goto out_release_uncharge_unlock;
2301
2302 lru_cache_add_anon(page);
2303
2304 spin_lock(&info->lock);
2305 info->alloced++;
2306 inode->i_blocks += BLOCKS_PER_PAGE;
2307 shmem_recalc_inode(inode);
2308 spin_unlock(&info->lock);
2309
2310 inc_mm_counter(dst_mm, mm_counter_file(page));
2311 page_add_file_rmap(page, false);
2312 set_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte);
2313
2314
2315 update_mmu_cache(dst_vma, dst_addr, dst_pte);
2316 unlock_page(page);
2317 pte_unmap_unlock(dst_pte, ptl);
2318 ret = 0;
2319out:
2320 return ret;
2321out_release_uncharge_unlock:
2322 pte_unmap_unlock(dst_pte, ptl);
2323out_release_uncharge:
2324 mem_cgroup_cancel_charge(page, memcg, false);
2325out_release:
2326 unlock_page(page);
2327 put_page(page);
2328out_unacct_blocks:
2329 shmem_inode_unacct_blocks(inode, 1);
2330 goto out;
2331}
2332
2333int shmem_mcopy_atomic_pte(struct mm_struct *dst_mm,
2334 pmd_t *dst_pmd,
2335 struct vm_area_struct *dst_vma,
2336 unsigned long dst_addr,
2337 unsigned long src_addr,
2338 struct page **pagep)
2339{
2340 return shmem_mfill_atomic_pte(dst_mm, dst_pmd, dst_vma,
2341 dst_addr, src_addr, false, pagep);
2342}
2343
2344int shmem_mfill_zeropage_pte(struct mm_struct *dst_mm,
2345 pmd_t *dst_pmd,
2346 struct vm_area_struct *dst_vma,
2347 unsigned long dst_addr)
2348{
2349 struct page *page = NULL;
2350
2351 return shmem_mfill_atomic_pte(dst_mm, dst_pmd, dst_vma,
2352 dst_addr, 0, true, &page);
2353}
2354
2355#ifdef CONFIG_TMPFS
2356static const struct inode_operations shmem_symlink_inode_operations;
2357static const struct inode_operations shmem_short_symlink_operations;
2358
2359#ifdef CONFIG_TMPFS_XATTR
2360static int shmem_initxattrs(struct inode *, const struct xattr *, void *);
2361#else
2362#define shmem_initxattrs NULL
2363#endif
2364
2365static int
2366shmem_write_begin(struct file *file, struct address_space *mapping,
2367 loff_t pos, unsigned len, unsigned flags,
2368 struct page **pagep, void **fsdata)
2369{
2370 struct inode *inode = mapping->host;
2371 struct shmem_inode_info *info = SHMEM_I(inode);
2372 pgoff_t index = pos >> PAGE_SHIFT;
2373
2374
2375 if (unlikely(info->seals & (F_SEAL_WRITE | F_SEAL_GROW))) {
2376 if (info->seals & F_SEAL_WRITE)
2377 return -EPERM;
2378 if ((info->seals & F_SEAL_GROW) && pos + len > inode->i_size)
2379 return -EPERM;
2380 }
2381
2382 return shmem_getpage(inode, index, pagep, SGP_WRITE);
2383}
2384
2385static int
2386shmem_write_end(struct file *file, struct address_space *mapping,
2387 loff_t pos, unsigned len, unsigned copied,
2388 struct page *page, void *fsdata)
2389{
2390 struct inode *inode = mapping->host;
2391
2392 if (pos + copied > inode->i_size)
2393 i_size_write(inode, pos + copied);
2394
2395 if (!PageUptodate(page)) {
2396 struct page *head = compound_head(page);
2397 if (PageTransCompound(page)) {
2398 int i;
2399
2400 for (i = 0; i < HPAGE_PMD_NR; i++) {
2401 if (head + i == page)
2402 continue;
2403 clear_highpage(head + i);
2404 flush_dcache_page(head + i);
2405 }
2406 }
2407 if (copied < PAGE_SIZE) {
2408 unsigned from = pos & (PAGE_SIZE - 1);
2409 zero_user_segments(page, 0, from,
2410 from + copied, PAGE_SIZE);
2411 }
2412 SetPageUptodate(head);
2413 }
2414 set_page_dirty(page);
2415 unlock_page(page);
2416 put_page(page);
2417
2418 return copied;
2419}
2420
2421static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
2422{
2423 struct file *file = iocb->ki_filp;
2424 struct inode *inode = file_inode(file);
2425 struct address_space *mapping = inode->i_mapping;
2426 pgoff_t index;
2427 unsigned long offset;
2428 enum sgp_type sgp = SGP_READ;
2429 int error = 0;
2430 ssize_t retval = 0;
2431 loff_t *ppos = &iocb->ki_pos;
2432
2433
2434
2435
2436
2437
2438 if (!iter_is_iovec(to))
2439 sgp = SGP_CACHE;
2440
2441 index = *ppos >> PAGE_SHIFT;
2442 offset = *ppos & ~PAGE_MASK;
2443
2444 for (;;) {
2445 struct page *page = NULL;
2446 pgoff_t end_index;
2447 unsigned long nr, ret;
2448 loff_t i_size = i_size_read(inode);
2449
2450 end_index = i_size >> PAGE_SHIFT;
2451 if (index > end_index)
2452 break;
2453 if (index == end_index) {
2454 nr = i_size & ~PAGE_MASK;
2455 if (nr <= offset)
2456 break;
2457 }
2458
2459 error = shmem_getpage(inode, index, &page, sgp);
2460 if (error) {
2461 if (error == -EINVAL)
2462 error = 0;
2463 break;
2464 }
2465 if (page) {
2466 if (sgp == SGP_CACHE)
2467 set_page_dirty(page);
2468 unlock_page(page);
2469 }
2470
2471
2472
2473
2474
2475 nr = PAGE_SIZE;
2476 i_size = i_size_read(inode);
2477 end_index = i_size >> PAGE_SHIFT;
2478 if (index == end_index) {
2479 nr = i_size & ~PAGE_MASK;
2480 if (nr <= offset) {
2481 if (page)
2482 put_page(page);
2483 break;
2484 }
2485 }
2486 nr -= offset;
2487
2488 if (page) {
2489
2490
2491
2492
2493
2494 if (mapping_writably_mapped(mapping))
2495 flush_dcache_page(page);
2496
2497
2498
2499 if (!offset)
2500 mark_page_accessed(page);
2501 } else {
2502 page = ZERO_PAGE(0);
2503 get_page(page);
2504 }
2505
2506
2507
2508
2509
2510 ret = copy_page_to_iter(page, offset, nr, to);
2511 retval += ret;
2512 offset += ret;
2513 index += offset >> PAGE_SHIFT;
2514 offset &= ~PAGE_MASK;
2515
2516 put_page(page);
2517 if (!iov_iter_count(to))
2518 break;
2519 if (ret < nr) {
2520 error = -EFAULT;
2521 break;
2522 }
2523 cond_resched();
2524 }
2525
2526 *ppos = ((loff_t) index << PAGE_SHIFT) + offset;
2527 file_accessed(file);
2528 return retval ? retval : error;
2529}
2530
2531
2532
2533
2534static pgoff_t shmem_seek_hole_data(struct address_space *mapping,
2535 pgoff_t index, pgoff_t end, int whence)
2536{
2537 struct page *page;
2538 struct pagevec pvec;
2539 pgoff_t indices[PAGEVEC_SIZE];
2540 bool done = false;
2541 int i;
2542
2543 pagevec_init(&pvec);
2544 pvec.nr = 1;
2545 while (!done) {
2546 pvec.nr = find_get_entries(mapping, index,
2547 pvec.nr, pvec.pages, indices);
2548 if (!pvec.nr) {
2549 if (whence == SEEK_DATA)
2550 index = end;
2551 break;
2552 }
2553 for (i = 0; i < pvec.nr; i++, index++) {
2554 if (index < indices[i]) {
2555 if (whence == SEEK_HOLE) {
2556 done = true;
2557 break;
2558 }
2559 index = indices[i];
2560 }
2561 page = pvec.pages[i];
2562 if (page && !radix_tree_exceptional_entry(page)) {
2563 if (!PageUptodate(page))
2564 page = NULL;
2565 }
2566 if (index >= end ||
2567 (page && whence == SEEK_DATA) ||
2568 (!page && whence == SEEK_HOLE)) {
2569 done = true;
2570 break;
2571 }
2572 }
2573 pagevec_remove_exceptionals(&pvec);
2574 pagevec_release(&pvec);
2575 pvec.nr = PAGEVEC_SIZE;
2576 cond_resched();
2577 }
2578 return index;
2579}
2580
2581static loff_t shmem_file_llseek(struct file *file, loff_t offset, int whence)
2582{
2583 struct address_space *mapping = file->f_mapping;
2584 struct inode *inode = mapping->host;
2585 pgoff_t start, end;
2586 loff_t new_offset;
2587
2588 if (whence != SEEK_DATA && whence != SEEK_HOLE)
2589 return generic_file_llseek_size(file, offset, whence,
2590 MAX_LFS_FILESIZE, i_size_read(inode));
2591 inode_lock(inode);
2592
2593
2594 if (offset < 0)
2595 offset = -EINVAL;
2596 else if (offset >= inode->i_size)
2597 offset = -ENXIO;
2598 else {
2599 start = offset >> PAGE_SHIFT;
2600 end = (inode->i_size + PAGE_SIZE - 1) >> PAGE_SHIFT;
2601 new_offset = shmem_seek_hole_data(mapping, start, end, whence);
2602 new_offset <<= PAGE_SHIFT;
2603 if (new_offset > offset) {
2604 if (new_offset < inode->i_size)
2605 offset = new_offset;
2606 else if (whence == SEEK_DATA)
2607 offset = -ENXIO;
2608 else
2609 offset = inode->i_size;
2610 }
2611 }
2612
2613 if (offset >= 0)
2614 offset = vfs_setpos(file, offset, MAX_LFS_FILESIZE);
2615 inode_unlock(inode);
2616 return offset;
2617}
2618
2619
2620
2621
2622
2623#define SHMEM_TAG_PINNED PAGECACHE_TAG_TOWRITE
2624#define LAST_SCAN 4
2625
2626static void shmem_tag_pins(struct address_space *mapping)
2627{
2628 struct radix_tree_iter iter;
2629 void **slot;
2630 pgoff_t start;
2631 struct page *page;
2632
2633 lru_add_drain();
2634 start = 0;
2635 rcu_read_lock();
2636
2637 radix_tree_for_each_slot(slot, &mapping->i_pages, &iter, start) {
2638 page = radix_tree_deref_slot(slot);
2639 if (!page || radix_tree_exception(page)) {
2640 if (radix_tree_deref_retry(page)) {
2641 slot = radix_tree_iter_retry(&iter);
2642 continue;
2643 }
2644 } else if (page_count(page) - page_mapcount(page) > 1) {
2645 xa_lock_irq(&mapping->i_pages);
2646 radix_tree_tag_set(&mapping->i_pages, iter.index,
2647 SHMEM_TAG_PINNED);
2648 xa_unlock_irq(&mapping->i_pages);
2649 }
2650
2651 if (need_resched()) {
2652 slot = radix_tree_iter_resume(slot, &iter);
2653 cond_resched_rcu();
2654 }
2655 }
2656 rcu_read_unlock();
2657}
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668static int shmem_wait_for_pins(struct address_space *mapping)
2669{
2670 struct radix_tree_iter iter;
2671 void **slot;
2672 pgoff_t start;
2673 struct page *page;
2674 int error, scan;
2675
2676 shmem_tag_pins(mapping);
2677
2678 error = 0;
2679 for (scan = 0; scan <= LAST_SCAN; scan++) {
2680 if (!radix_tree_tagged(&mapping->i_pages, SHMEM_TAG_PINNED))
2681 break;
2682
2683 if (!scan)
2684 lru_add_drain_all();
2685 else if (schedule_timeout_killable((HZ << scan) / 200))
2686 scan = LAST_SCAN;
2687
2688 start = 0;
2689 rcu_read_lock();
2690 radix_tree_for_each_tagged(slot, &mapping->i_pages, &iter,
2691 start, SHMEM_TAG_PINNED) {
2692
2693 page = radix_tree_deref_slot(slot);
2694 if (radix_tree_exception(page)) {
2695 if (radix_tree_deref_retry(page)) {
2696 slot = radix_tree_iter_retry(&iter);
2697 continue;
2698 }
2699
2700 page = NULL;
2701 }
2702
2703 if (page &&
2704 page_count(page) - page_mapcount(page) != 1) {
2705 if (scan < LAST_SCAN)
2706 goto continue_resched;
2707
2708
2709
2710
2711
2712
2713 error = -EBUSY;
2714 }
2715
2716 xa_lock_irq(&mapping->i_pages);
2717 radix_tree_tag_clear(&mapping->i_pages,
2718 iter.index, SHMEM_TAG_PINNED);
2719 xa_unlock_irq(&mapping->i_pages);
2720continue_resched:
2721 if (need_resched()) {
2722 slot = radix_tree_iter_resume(slot, &iter);
2723 cond_resched_rcu();
2724 }
2725 }
2726 rcu_read_unlock();
2727 }
2728
2729 return error;
2730}
2731
2732static unsigned int *memfd_file_seals_ptr(struct file *file)
2733{
2734 if (file->f_op == &shmem_file_operations)
2735 return &SHMEM_I(file_inode(file))->seals;
2736
2737#ifdef CONFIG_HUGETLBFS
2738 if (file->f_op == &hugetlbfs_file_operations)
2739 return &HUGETLBFS_I(file_inode(file))->seals;
2740#endif
2741
2742 return NULL;
2743}
2744
2745#define F_ALL_SEALS (F_SEAL_SEAL | \
2746 F_SEAL_SHRINK | \
2747 F_SEAL_GROW | \
2748 F_SEAL_WRITE)
2749
2750static int memfd_add_seals(struct file *file, unsigned int seals)
2751{
2752 struct inode *inode = file_inode(file);
2753 unsigned int *file_seals;
2754 int error;
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786 if (!(file->f_mode & FMODE_WRITE))
2787 return -EPERM;
2788 if (seals & ~(unsigned int)F_ALL_SEALS)
2789 return -EINVAL;
2790
2791 inode_lock(inode);
2792
2793 file_seals = memfd_file_seals_ptr(file);
2794 if (!file_seals) {
2795 error = -EINVAL;
2796 goto unlock;
2797 }
2798
2799 if (*file_seals & F_SEAL_SEAL) {
2800 error = -EPERM;
2801 goto unlock;
2802 }
2803
2804 if ((seals & F_SEAL_WRITE) && !(*file_seals & F_SEAL_WRITE)) {
2805 error = mapping_deny_writable(file->f_mapping);
2806 if (error)
2807 goto unlock;
2808
2809 error = shmem_wait_for_pins(file->f_mapping);
2810 if (error) {
2811 mapping_allow_writable(file->f_mapping);
2812 goto unlock;
2813 }
2814 }
2815
2816 *file_seals |= seals;
2817 error = 0;
2818
2819unlock:
2820 inode_unlock(inode);
2821 return error;
2822}
2823
2824static int memfd_get_seals(struct file *file)
2825{
2826 unsigned int *seals = memfd_file_seals_ptr(file);
2827
2828 return seals ? *seals : -EINVAL;
2829}
2830
2831long memfd_fcntl(struct file *file, unsigned int cmd, unsigned long arg)
2832{
2833 long error;
2834
2835 switch (cmd) {
2836 case F_ADD_SEALS:
2837
2838 if (arg > UINT_MAX)
2839 return -EINVAL;
2840
2841 error = memfd_add_seals(file, arg);
2842 break;
2843 case F_GET_SEALS:
2844 error = memfd_get_seals(file);
2845 break;
2846 default:
2847 error = -EINVAL;
2848 break;
2849 }
2850
2851 return error;
2852}
2853
2854static long shmem_fallocate(struct file *file, int mode, loff_t offset,
2855 loff_t len)
2856{
2857 struct inode *inode = file_inode(file);
2858 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
2859 struct shmem_inode_info *info = SHMEM_I(inode);
2860 struct shmem_falloc shmem_falloc;
2861 pgoff_t start, index, end;
2862 int error;
2863
2864 if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE))
2865 return -EOPNOTSUPP;
2866
2867 inode_lock(inode);
2868
2869 if (mode & FALLOC_FL_PUNCH_HOLE) {
2870 struct address_space *mapping = file->f_mapping;
2871 loff_t unmap_start = round_up(offset, PAGE_SIZE);
2872 loff_t unmap_end = round_down(offset + len, PAGE_SIZE) - 1;
2873 DECLARE_WAIT_QUEUE_HEAD_ONSTACK(shmem_falloc_waitq);
2874
2875
2876 if (info->seals & F_SEAL_WRITE) {
2877 error = -EPERM;
2878 goto out;
2879 }
2880
2881 shmem_falloc.waitq = &shmem_falloc_waitq;
2882 shmem_falloc.start = unmap_start >> PAGE_SHIFT;
2883 shmem_falloc.next = (unmap_end + 1) >> PAGE_SHIFT;
2884 spin_lock(&inode->i_lock);
2885 inode->i_private = &shmem_falloc;
2886 spin_unlock(&inode->i_lock);
2887
2888 if ((u64)unmap_end > (u64)unmap_start)
2889 unmap_mapping_range(mapping, unmap_start,
2890 1 + unmap_end - unmap_start, 0);
2891 shmem_truncate_range(inode, offset, offset + len - 1);
2892
2893
2894 spin_lock(&inode->i_lock);
2895 inode->i_private = NULL;
2896 wake_up_all(&shmem_falloc_waitq);
2897 WARN_ON_ONCE(!list_empty(&shmem_falloc_waitq.head));
2898 spin_unlock(&inode->i_lock);
2899 error = 0;
2900 goto out;
2901 }
2902
2903
2904 error = inode_newsize_ok(inode, offset + len);
2905 if (error)
2906 goto out;
2907
2908 if ((info->seals & F_SEAL_GROW) && offset + len > inode->i_size) {
2909 error = -EPERM;
2910 goto out;
2911 }
2912
2913 start = offset >> PAGE_SHIFT;
2914 end = (offset + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
2915
2916 if (sbinfo->max_blocks && end - start > sbinfo->max_blocks) {
2917 error = -ENOSPC;
2918 goto out;
2919 }
2920
2921 shmem_falloc.waitq = NULL;
2922 shmem_falloc.start = start;
2923 shmem_falloc.next = start;
2924 shmem_falloc.nr_falloced = 0;
2925 shmem_falloc.nr_unswapped = 0;
2926 spin_lock(&inode->i_lock);
2927 inode->i_private = &shmem_falloc;
2928 spin_unlock(&inode->i_lock);
2929
2930 for (index = start; index < end; index++) {
2931 struct page *page;
2932
2933
2934
2935
2936
2937 if (signal_pending(current))
2938 error = -EINTR;
2939 else if (shmem_falloc.nr_unswapped > shmem_falloc.nr_falloced)
2940 error = -ENOMEM;
2941 else
2942 error = shmem_getpage(inode, index, &page, SGP_FALLOC);
2943 if (error) {
2944
2945 if (index > start) {
2946 shmem_undo_range(inode,
2947 (loff_t)start << PAGE_SHIFT,
2948 ((loff_t)index << PAGE_SHIFT) - 1, true);
2949 }
2950 goto undone;
2951 }
2952
2953
2954
2955
2956
2957 shmem_falloc.next++;
2958 if (!PageUptodate(page))
2959 shmem_falloc.nr_falloced++;
2960
2961
2962
2963
2964
2965
2966
2967
2968 set_page_dirty(page);
2969 unlock_page(page);
2970 put_page(page);
2971 cond_resched();
2972 }
2973
2974 if (!(mode & FALLOC_FL_KEEP_SIZE) && offset + len > inode->i_size)
2975 i_size_write(inode, offset + len);
2976 inode->i_ctime = current_time(inode);
2977undone:
2978 spin_lock(&inode->i_lock);
2979 inode->i_private = NULL;
2980 spin_unlock(&inode->i_lock);
2981out:
2982 inode_unlock(inode);
2983 return error;
2984}
2985
2986static int shmem_statfs(struct dentry *dentry, struct kstatfs *buf)
2987{
2988 struct shmem_sb_info *sbinfo = SHMEM_SB(dentry->d_sb);
2989
2990 buf->f_type = TMPFS_MAGIC;
2991 buf->f_bsize = PAGE_SIZE;
2992 buf->f_namelen = NAME_MAX;
2993 if (sbinfo->max_blocks) {
2994 buf->f_blocks = sbinfo->max_blocks;
2995 buf->f_bavail =
2996 buf->f_bfree = sbinfo->max_blocks -
2997 percpu_counter_sum(&sbinfo->used_blocks);
2998 }
2999 if (sbinfo->max_inodes) {
3000 buf->f_files = sbinfo->max_inodes;
3001 buf->f_ffree = sbinfo->free_inodes;
3002 }
3003
3004 return 0;
3005}
3006
3007
3008
3009
3010static int
3011shmem_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, dev_t dev)
3012{
3013 struct inode *inode;
3014 int error = -ENOSPC;
3015
3016 inode = shmem_get_inode(dir->i_sb, dir, mode, dev, VM_NORESERVE);
3017 if (inode) {
3018 error = simple_acl_create(dir, inode);
3019 if (error)
3020 goto out_iput;
3021 error = security_inode_init_security(inode, dir,
3022 &dentry->d_name,
3023 shmem_initxattrs, NULL);
3024 if (error && error != -EOPNOTSUPP)
3025 goto out_iput;
3026
3027 error = 0;
3028 dir->i_size += BOGO_DIRENT_SIZE;
3029 dir->i_ctime = dir->i_mtime = current_time(dir);
3030 d_instantiate(dentry, inode);
3031 dget(dentry);
3032 }
3033 return error;
3034out_iput:
3035 iput(inode);
3036 return error;
3037}
3038
3039static int
3040shmem_tmpfile(struct inode *dir, struct dentry *dentry, umode_t mode)
3041{
3042 struct inode *inode;
3043 int error = -ENOSPC;
3044
3045 inode = shmem_get_inode(dir->i_sb, dir, mode, 0, VM_NORESERVE);
3046 if (inode) {
3047 error = security_inode_init_security(inode, dir,
3048 NULL,
3049 shmem_initxattrs, NULL);
3050 if (error && error != -EOPNOTSUPP)
3051 goto out_iput;
3052 error = simple_acl_create(dir, inode);
3053 if (error)
3054 goto out_iput;
3055 d_tmpfile(dentry, inode);
3056 }
3057 return error;
3058out_iput:
3059 iput(inode);
3060 return error;
3061}
3062
3063static int shmem_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode)
3064{
3065 int error;
3066
3067 if ((error = shmem_mknod(dir, dentry, mode | S_IFDIR, 0)))
3068 return error;
3069 inc_nlink(dir);
3070 return 0;
3071}
3072
3073static int shmem_create(struct inode *dir, struct dentry *dentry, umode_t mode,
3074 bool excl)
3075{
3076 return shmem_mknod(dir, dentry, mode | S_IFREG, 0);
3077}
3078
3079
3080
3081
3082static int shmem_link(struct dentry *old_dentry, struct inode *dir, struct dentry *dentry)
3083{
3084 struct inode *inode = d_inode(old_dentry);
3085 int ret;
3086
3087
3088
3089
3090
3091
3092 ret = shmem_reserve_inode(inode->i_sb);
3093 if (ret)
3094 goto out;
3095
3096 dir->i_size += BOGO_DIRENT_SIZE;
3097 inode->i_ctime = dir->i_ctime = dir->i_mtime = current_time(inode);
3098 inc_nlink(inode);
3099 ihold(inode);
3100 dget(dentry);
3101 d_instantiate(dentry, inode);
3102out:
3103 return ret;
3104}
3105
3106static int shmem_unlink(struct inode *dir, struct dentry *dentry)
3107{
3108 struct inode *inode = d_inode(dentry);
3109
3110 if (inode->i_nlink > 1 && !S_ISDIR(inode->i_mode))
3111 shmem_free_inode(inode->i_sb);
3112
3113 dir->i_size -= BOGO_DIRENT_SIZE;
3114 inode->i_ctime = dir->i_ctime = dir->i_mtime = current_time(inode);
3115 drop_nlink(inode);
3116 dput(dentry);
3117 return 0;
3118}
3119
3120static int shmem_rmdir(struct inode *dir, struct dentry *dentry)
3121{
3122 if (!simple_empty(dentry))
3123 return -ENOTEMPTY;
3124
3125 drop_nlink(d_inode(dentry));
3126 drop_nlink(dir);
3127 return shmem_unlink(dir, dentry);
3128}
3129
3130static int shmem_exchange(struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry)
3131{
3132 bool old_is_dir = d_is_dir(old_dentry);
3133 bool new_is_dir = d_is_dir(new_dentry);
3134
3135 if (old_dir != new_dir && old_is_dir != new_is_dir) {
3136 if (old_is_dir) {
3137 drop_nlink(old_dir);
3138 inc_nlink(new_dir);
3139 } else {
3140 drop_nlink(new_dir);
3141 inc_nlink(old_dir);
3142 }
3143 }
3144 old_dir->i_ctime = old_dir->i_mtime =
3145 new_dir->i_ctime = new_dir->i_mtime =
3146 d_inode(old_dentry)->i_ctime =
3147 d_inode(new_dentry)->i_ctime = current_time(old_dir);
3148
3149 return 0;
3150}
3151
3152static int shmem_whiteout(struct inode *old_dir, struct dentry *old_dentry)
3153{
3154 struct dentry *whiteout;
3155 int error;
3156
3157 whiteout = d_alloc(old_dentry->d_parent, &old_dentry->d_name);
3158 if (!whiteout)
3159 return -ENOMEM;
3160
3161 error = shmem_mknod(old_dir, whiteout,
3162 S_IFCHR | WHITEOUT_MODE, WHITEOUT_DEV);
3163 dput(whiteout);
3164 if (error)
3165 return error;
3166
3167
3168
3169
3170
3171
3172
3173
3174 d_rehash(whiteout);
3175 return 0;
3176}
3177
3178
3179
3180
3181
3182
3183
3184static int shmem_rename2(struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry, unsigned int flags)
3185{
3186 struct inode *inode = d_inode(old_dentry);
3187 int they_are_dirs = S_ISDIR(inode->i_mode);
3188
3189 if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE | RENAME_WHITEOUT))
3190 return -EINVAL;
3191
3192 if (flags & RENAME_EXCHANGE)
3193 return shmem_exchange(old_dir, old_dentry, new_dir, new_dentry);
3194
3195 if (!simple_empty(new_dentry))
3196 return -ENOTEMPTY;
3197
3198 if (flags & RENAME_WHITEOUT) {
3199 int error;
3200
3201 error = shmem_whiteout(old_dir, old_dentry);
3202 if (error)
3203 return error;
3204 }
3205
3206 if (d_really_is_positive(new_dentry)) {
3207 (void) shmem_unlink(new_dir, new_dentry);
3208 if (they_are_dirs) {
3209 drop_nlink(d_inode(new_dentry));
3210 drop_nlink(old_dir);
3211 }
3212 } else if (they_are_dirs) {
3213 drop_nlink(old_dir);
3214 inc_nlink(new_dir);
3215 }
3216
3217 old_dir->i_size -= BOGO_DIRENT_SIZE;
3218 new_dir->i_size += BOGO_DIRENT_SIZE;
3219 old_dir->i_ctime = old_dir->i_mtime =
3220 new_dir->i_ctime = new_dir->i_mtime =
3221 inode->i_ctime = current_time(old_dir);
3222 return 0;
3223}
3224
3225static int shmem_symlink(struct inode *dir, struct dentry *dentry, const char *symname)
3226{
3227 int error;
3228 int len;
3229 struct inode *inode;
3230 struct page *page;
3231
3232 len = strlen(symname) + 1;
3233 if (len > PAGE_SIZE)
3234 return -ENAMETOOLONG;
3235
3236 inode = shmem_get_inode(dir->i_sb, dir, S_IFLNK|S_IRWXUGO, 0, VM_NORESERVE);
3237 if (!inode)
3238 return -ENOSPC;
3239
3240 error = security_inode_init_security(inode, dir, &dentry->d_name,
3241 shmem_initxattrs, NULL);
3242 if (error) {
3243 if (error != -EOPNOTSUPP) {
3244 iput(inode);
3245 return error;
3246 }
3247 error = 0;
3248 }
3249
3250 inode->i_size = len-1;
3251 if (len <= SHORT_SYMLINK_LEN) {
3252 inode->i_link = kmemdup(symname, len, GFP_KERNEL);
3253 if (!inode->i_link) {
3254 iput(inode);
3255 return -ENOMEM;
3256 }
3257 inode->i_op = &shmem_short_symlink_operations;
3258 } else {
3259 inode_nohighmem(inode);
3260 error = shmem_getpage(inode, 0, &page, SGP_WRITE);
3261 if (error) {
3262 iput(inode);
3263 return error;
3264 }
3265 inode->i_mapping->a_ops = &shmem_aops;
3266 inode->i_op = &shmem_symlink_inode_operations;
3267 memcpy(page_address(page), symname, len);
3268 SetPageUptodate(page);
3269 set_page_dirty(page);
3270 unlock_page(page);
3271 put_page(page);
3272 }
3273 dir->i_size += BOGO_DIRENT_SIZE;
3274 dir->i_ctime = dir->i_mtime = current_time(dir);
3275 d_instantiate(dentry, inode);
3276 dget(dentry);
3277 return 0;
3278}
3279
3280static void shmem_put_link(void *arg)
3281{
3282 mark_page_accessed(arg);
3283 put_page(arg);
3284}
3285
3286static const char *shmem_get_link(struct dentry *dentry,
3287 struct inode *inode,
3288 struct delayed_call *done)
3289{
3290 struct page *page = NULL;
3291 int error;
3292 if (!dentry) {
3293 page = find_get_page(inode->i_mapping, 0);
3294 if (!page)
3295 return ERR_PTR(-ECHILD);
3296 if (!PageUptodate(page)) {
3297 put_page(page);
3298 return ERR_PTR(-ECHILD);
3299 }
3300 } else {
3301 error = shmem_getpage(inode, 0, &page, SGP_READ);
3302 if (error)
3303 return ERR_PTR(error);
3304 unlock_page(page);
3305 }
3306 set_delayed_call(done, shmem_put_link, page);
3307 return page_address(page);
3308}
3309
3310#ifdef CONFIG_TMPFS_XATTR
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321static int shmem_initxattrs(struct inode *inode,
3322 const struct xattr *xattr_array,
3323 void *fs_info)
3324{
3325 struct shmem_inode_info *info = SHMEM_I(inode);
3326 const struct xattr *xattr;
3327 struct simple_xattr *new_xattr;
3328 size_t len;
3329
3330 for (xattr = xattr_array; xattr->name != NULL; xattr++) {
3331 new_xattr = simple_xattr_alloc(xattr->value, xattr->value_len);
3332 if (!new_xattr)
3333 return -ENOMEM;
3334
3335 len = strlen(xattr->name) + 1;
3336 new_xattr->name = kmalloc(XATTR_SECURITY_PREFIX_LEN + len,
3337 GFP_KERNEL);
3338 if (!new_xattr->name) {
3339 kfree(new_xattr);
3340 return -ENOMEM;
3341 }
3342
3343 memcpy(new_xattr->name, XATTR_SECURITY_PREFIX,
3344 XATTR_SECURITY_PREFIX_LEN);
3345 memcpy(new_xattr->name + XATTR_SECURITY_PREFIX_LEN,
3346 xattr->name, len);
3347
3348 simple_xattr_list_add(&info->xattrs, new_xattr);
3349 }
3350
3351 return 0;
3352}
3353
3354static int shmem_xattr_handler_get(const struct xattr_handler *handler,
3355 struct dentry *unused, struct inode *inode,
3356 const char *name, void *buffer, size_t size)
3357{
3358 struct shmem_inode_info *info = SHMEM_I(inode);
3359
3360 name = xattr_full_name(handler, name);
3361 return simple_xattr_get(&info->xattrs, name, buffer, size);
3362}
3363
3364static int shmem_xattr_handler_set(const struct xattr_handler *handler,
3365 struct dentry *unused, struct inode *inode,
3366 const char *name, const void *value,
3367 size_t size, int flags)
3368{
3369 struct shmem_inode_info *info = SHMEM_I(inode);
3370
3371 name = xattr_full_name(handler, name);
3372 return simple_xattr_set(&info->xattrs, name, value, size, flags);
3373}
3374
3375static const struct xattr_handler shmem_security_xattr_handler = {
3376 .prefix = XATTR_SECURITY_PREFIX,
3377 .get = shmem_xattr_handler_get,
3378 .set = shmem_xattr_handler_set,
3379};
3380
3381static const struct xattr_handler shmem_trusted_xattr_handler = {
3382 .prefix = XATTR_TRUSTED_PREFIX,
3383 .get = shmem_xattr_handler_get,
3384 .set = shmem_xattr_handler_set,
3385};
3386
3387static const struct xattr_handler *shmem_xattr_handlers[] = {
3388#ifdef CONFIG_TMPFS_POSIX_ACL
3389 &posix_acl_access_xattr_handler,
3390 &posix_acl_default_xattr_handler,
3391#endif
3392 &shmem_security_xattr_handler,
3393 &shmem_trusted_xattr_handler,
3394 NULL
3395};
3396
3397static ssize_t shmem_listxattr(struct dentry *dentry, char *buffer, size_t size)
3398{
3399 struct shmem_inode_info *info = SHMEM_I(d_inode(dentry));
3400 return simple_xattr_list(d_inode(dentry), &info->xattrs, buffer, size);
3401}
3402#endif
3403
3404static const struct inode_operations shmem_short_symlink_operations = {
3405 .get_link = simple_get_link,
3406#ifdef CONFIG_TMPFS_XATTR
3407 .listxattr = shmem_listxattr,
3408#endif
3409};
3410
3411static const struct inode_operations shmem_symlink_inode_operations = {
3412 .get_link = shmem_get_link,
3413#ifdef CONFIG_TMPFS_XATTR
3414 .listxattr = shmem_listxattr,
3415#endif
3416};
3417
3418static struct dentry *shmem_get_parent(struct dentry *child)
3419{
3420 return ERR_PTR(-ESTALE);
3421}
3422
3423static int shmem_match(struct inode *ino, void *vfh)
3424{
3425 __u32 *fh = vfh;
3426 __u64 inum = fh[2];
3427 inum = (inum << 32) | fh[1];
3428 return ino->i_ino == inum && fh[0] == ino->i_generation;
3429}
3430
3431static struct dentry *shmem_fh_to_dentry(struct super_block *sb,
3432 struct fid *fid, int fh_len, int fh_type)
3433{
3434 struct inode *inode;
3435 struct dentry *dentry = NULL;
3436 u64 inum;
3437
3438 if (fh_len < 3)
3439 return NULL;
3440
3441 inum = fid->raw[2];
3442 inum = (inum << 32) | fid->raw[1];
3443
3444 inode = ilookup5(sb, (unsigned long)(inum + fid->raw[0]),
3445 shmem_match, fid->raw);
3446 if (inode) {
3447 dentry = d_find_alias(inode);
3448 iput(inode);
3449 }
3450
3451 return dentry;
3452}
3453
3454static int shmem_encode_fh(struct inode *inode, __u32 *fh, int *len,
3455 struct inode *parent)
3456{
3457 if (*len < 3) {
3458 *len = 3;
3459 return FILEID_INVALID;
3460 }
3461
3462 if (inode_unhashed(inode)) {
3463
3464
3465
3466
3467
3468 static DEFINE_SPINLOCK(lock);
3469 spin_lock(&lock);
3470 if (inode_unhashed(inode))
3471 __insert_inode_hash(inode,
3472 inode->i_ino + inode->i_generation);
3473 spin_unlock(&lock);
3474 }
3475
3476 fh[0] = inode->i_generation;
3477 fh[1] = inode->i_ino;
3478 fh[2] = ((__u64)inode->i_ino) >> 32;
3479
3480 *len = 3;
3481 return 1;
3482}
3483
3484static const struct export_operations shmem_export_ops = {
3485 .get_parent = shmem_get_parent,
3486 .encode_fh = shmem_encode_fh,
3487 .fh_to_dentry = shmem_fh_to_dentry,
3488};
3489
3490static int shmem_parse_options(char *options, struct shmem_sb_info *sbinfo,
3491 bool remount)
3492{
3493 char *this_char, *value, *rest;
3494 struct mempolicy *mpol = NULL;
3495 uid_t uid;
3496 gid_t gid;
3497
3498 while (options != NULL) {
3499 this_char = options;
3500 for (;;) {
3501
3502
3503
3504
3505
3506 options = strchr(options, ',');
3507 if (options == NULL)
3508 break;
3509 options++;
3510 if (!isdigit(*options)) {
3511 options[-1] = '\0';
3512 break;
3513 }
3514 }
3515 if (!*this_char)
3516 continue;
3517 if ((value = strchr(this_char,'=')) != NULL) {
3518 *value++ = 0;
3519 } else {
3520 pr_err("tmpfs: No value for mount option '%s'\n",
3521 this_char);
3522 goto error;
3523 }
3524
3525 if (!strcmp(this_char,"size")) {
3526 unsigned long long size;
3527 size = memparse(value,&rest);
3528 if (*rest == '%') {
3529 size <<= PAGE_SHIFT;
3530 size *= totalram_pages;
3531 do_div(size, 100);
3532 rest++;
3533 }
3534 if (*rest)
3535 goto bad_val;
3536 sbinfo->max_blocks =
3537 DIV_ROUND_UP(size, PAGE_SIZE);
3538 } else if (!strcmp(this_char,"nr_blocks")) {
3539 sbinfo->max_blocks = memparse(value, &rest);
3540 if (*rest)
3541 goto bad_val;
3542 } else if (!strcmp(this_char,"nr_inodes")) {
3543 sbinfo->max_inodes = memparse(value, &rest);
3544 if (*rest)
3545 goto bad_val;
3546 } else if (!strcmp(this_char,"mode")) {
3547 if (remount)
3548 continue;
3549 sbinfo->mode = simple_strtoul(value, &rest, 8) & 07777;
3550 if (*rest)
3551 goto bad_val;
3552 } else if (!strcmp(this_char,"uid")) {
3553 if (remount)
3554 continue;
3555 uid = simple_strtoul(value, &rest, 0);
3556 if (*rest)
3557 goto bad_val;
3558 sbinfo->uid = make_kuid(current_user_ns(), uid);
3559 if (!uid_valid(sbinfo->uid))
3560 goto bad_val;
3561 } else if (!strcmp(this_char,"gid")) {
3562 if (remount)
3563 continue;
3564 gid = simple_strtoul(value, &rest, 0);
3565 if (*rest)
3566 goto bad_val;
3567 sbinfo->gid = make_kgid(current_user_ns(), gid);
3568 if (!gid_valid(sbinfo->gid))
3569 goto bad_val;
3570#ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE
3571 } else if (!strcmp(this_char, "huge")) {
3572 int huge;
3573 huge = shmem_parse_huge(value);
3574 if (huge < 0)
3575 goto bad_val;
3576 if (!has_transparent_hugepage() &&
3577 huge != SHMEM_HUGE_NEVER)
3578 goto bad_val;
3579 sbinfo->huge = huge;
3580#endif
3581#ifdef CONFIG_NUMA
3582 } else if (!strcmp(this_char,"mpol")) {
3583 mpol_put(mpol);
3584 mpol = NULL;
3585 if (mpol_parse_str(value, &mpol))
3586 goto bad_val;
3587#endif
3588 } else {
3589 pr_err("tmpfs: Bad mount option %s\n", this_char);
3590 goto error;
3591 }
3592 }
3593 sbinfo->mpol = mpol;
3594 return 0;
3595
3596bad_val:
3597 pr_err("tmpfs: Bad value '%s' for mount option '%s'\n",
3598 value, this_char);
3599error:
3600 mpol_put(mpol);
3601 return 1;
3602
3603}
3604
3605static int shmem_remount_fs(struct super_block *sb, int *flags, char *data)
3606{
3607 struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
3608 struct shmem_sb_info config = *sbinfo;
3609 unsigned long inodes;
3610 int error = -EINVAL;
3611
3612 config.mpol = NULL;
3613 if (shmem_parse_options(data, &config, true))
3614 return error;
3615
3616 spin_lock(&sbinfo->stat_lock);
3617 inodes = sbinfo->max_inodes - sbinfo->free_inodes;
3618 if (percpu_counter_compare(&sbinfo->used_blocks, config.max_blocks) > 0)
3619 goto out;
3620 if (config.max_inodes < inodes)
3621 goto out;
3622
3623
3624
3625
3626
3627 if (config.max_blocks && !sbinfo->max_blocks)
3628 goto out;
3629 if (config.max_inodes && !sbinfo->max_inodes)
3630 goto out;
3631
3632 error = 0;
3633 sbinfo->huge = config.huge;
3634 sbinfo->max_blocks = config.max_blocks;
3635 sbinfo->max_inodes = config.max_inodes;
3636 sbinfo->free_inodes = config.max_inodes - inodes;
3637
3638
3639
3640
3641 if (config.mpol) {
3642 mpol_put(sbinfo->mpol);
3643 sbinfo->mpol = config.mpol;
3644 }
3645out:
3646 spin_unlock(&sbinfo->stat_lock);
3647 return error;
3648}
3649
3650static int shmem_show_options(struct seq_file *seq, struct dentry *root)
3651{
3652 struct shmem_sb_info *sbinfo = SHMEM_SB(root->d_sb);
3653
3654 if (sbinfo->max_blocks != shmem_default_max_blocks())
3655 seq_printf(seq, ",size=%luk",
3656 sbinfo->max_blocks << (PAGE_SHIFT - 10));
3657 if (sbinfo->max_inodes != shmem_default_max_inodes())
3658 seq_printf(seq, ",nr_inodes=%lu", sbinfo->max_inodes);
3659 if (sbinfo->mode != (S_IRWXUGO | S_ISVTX))
3660 seq_printf(seq, ",mode=%03ho", sbinfo->mode);
3661 if (!uid_eq(sbinfo->uid, GLOBAL_ROOT_UID))
3662 seq_printf(seq, ",uid=%u",
3663 from_kuid_munged(&init_user_ns, sbinfo->uid));
3664 if (!gid_eq(sbinfo->gid, GLOBAL_ROOT_GID))
3665 seq_printf(seq, ",gid=%u",
3666 from_kgid_munged(&init_user_ns, sbinfo->gid));
3667#ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE
3668
3669 if (sbinfo->huge)
3670 seq_printf(seq, ",huge=%s", shmem_format_huge(sbinfo->huge));
3671#endif
3672 shmem_show_mpol(seq, sbinfo->mpol);
3673 return 0;
3674}
3675
3676#define MFD_NAME_PREFIX "memfd:"
3677#define MFD_NAME_PREFIX_LEN (sizeof(MFD_NAME_PREFIX) - 1)
3678#define MFD_NAME_MAX_LEN (NAME_MAX - MFD_NAME_PREFIX_LEN)
3679
3680#define MFD_ALL_FLAGS (MFD_CLOEXEC | MFD_ALLOW_SEALING | MFD_HUGETLB)
3681
3682SYSCALL_DEFINE2(memfd_create,
3683 const char __user *, uname,
3684 unsigned int, flags)
3685{
3686 unsigned int *file_seals;
3687 struct file *file;
3688 int fd, error;
3689 char *name;
3690 long len;
3691
3692 if (!(flags & MFD_HUGETLB)) {
3693 if (flags & ~(unsigned int)MFD_ALL_FLAGS)
3694 return -EINVAL;
3695 } else {
3696
3697 if (flags & ~(unsigned int)(MFD_ALL_FLAGS |
3698 (MFD_HUGE_MASK << MFD_HUGE_SHIFT)))
3699 return -EINVAL;
3700 }
3701
3702
3703 len = strnlen_user(uname, MFD_NAME_MAX_LEN + 1);
3704 if (len <= 0)
3705 return -EFAULT;
3706 if (len > MFD_NAME_MAX_LEN + 1)
3707 return -EINVAL;
3708
3709 name = kmalloc(len + MFD_NAME_PREFIX_LEN, GFP_KERNEL);
3710 if (!name)
3711 return -ENOMEM;
3712
3713 strcpy(name, MFD_NAME_PREFIX);
3714 if (copy_from_user(&name[MFD_NAME_PREFIX_LEN], uname, len)) {
3715 error = -EFAULT;
3716 goto err_name;
3717 }
3718
3719
3720 if (name[len + MFD_NAME_PREFIX_LEN - 1]) {
3721 error = -EFAULT;
3722 goto err_name;
3723 }
3724
3725 fd = get_unused_fd_flags((flags & MFD_CLOEXEC) ? O_CLOEXEC : 0);
3726 if (fd < 0) {
3727 error = fd;
3728 goto err_name;
3729 }
3730
3731 if (flags & MFD_HUGETLB) {
3732 struct user_struct *user = NULL;
3733
3734 file = hugetlb_file_setup(name, 0, VM_NORESERVE, &user,
3735 HUGETLB_ANONHUGE_INODE,
3736 (flags >> MFD_HUGE_SHIFT) &
3737 MFD_HUGE_MASK);
3738 } else
3739 file = shmem_file_setup(name, 0, VM_NORESERVE);
3740 if (IS_ERR(file)) {
3741 error = PTR_ERR(file);
3742 goto err_fd;
3743 }
3744 file->f_mode |= FMODE_LSEEK | FMODE_PREAD | FMODE_PWRITE;
3745 file->f_flags |= O_RDWR | O_LARGEFILE;
3746
3747 if (flags & MFD_ALLOW_SEALING) {
3748 file_seals = memfd_file_seals_ptr(file);
3749 *file_seals &= ~F_SEAL_SEAL;
3750 }
3751
3752 fd_install(fd, file);
3753 kfree(name);
3754 return fd;
3755
3756err_fd:
3757 put_unused_fd(fd);
3758err_name:
3759 kfree(name);
3760 return error;
3761}
3762
3763#endif
3764
3765static void shmem_put_super(struct super_block *sb)
3766{
3767 struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
3768
3769 percpu_counter_destroy(&sbinfo->used_blocks);
3770 mpol_put(sbinfo->mpol);
3771 kfree(sbinfo);
3772 sb->s_fs_info = NULL;
3773}
3774
3775int shmem_fill_super(struct super_block *sb, void *data, int silent)
3776{
3777 struct inode *inode;
3778 struct shmem_sb_info *sbinfo;
3779 int err = -ENOMEM;
3780
3781
3782 sbinfo = kzalloc(max((int)sizeof(struct shmem_sb_info),
3783 L1_CACHE_BYTES), GFP_KERNEL);
3784 if (!sbinfo)
3785 return -ENOMEM;
3786
3787 sbinfo->mode = S_IRWXUGO | S_ISVTX;
3788 sbinfo->uid = current_fsuid();
3789 sbinfo->gid = current_fsgid();
3790 sb->s_fs_info = sbinfo;
3791
3792#ifdef CONFIG_TMPFS
3793
3794
3795
3796
3797
3798 if (!(sb->s_flags & SB_KERNMOUNT)) {
3799 sbinfo->max_blocks = shmem_default_max_blocks();
3800 sbinfo->max_inodes = shmem_default_max_inodes();
3801 if (shmem_parse_options(data, sbinfo, false)) {
3802 err = -EINVAL;
3803 goto failed;
3804 }
3805 } else {
3806 sb->s_flags |= SB_NOUSER;
3807 }
3808 sb->s_export_op = &shmem_export_ops;
3809 sb->s_flags |= SB_NOSEC;
3810#else
3811 sb->s_flags |= SB_NOUSER;
3812#endif
3813
3814 spin_lock_init(&sbinfo->stat_lock);
3815 if (percpu_counter_init(&sbinfo->used_blocks, 0, GFP_KERNEL))
3816 goto failed;
3817 sbinfo->free_inodes = sbinfo->max_inodes;
3818 spin_lock_init(&sbinfo->shrinklist_lock);
3819 INIT_LIST_HEAD(&sbinfo->shrinklist);
3820
3821 sb->s_maxbytes = MAX_LFS_FILESIZE;
3822 sb->s_blocksize = PAGE_SIZE;
3823 sb->s_blocksize_bits = PAGE_SHIFT;
3824 sb->s_magic = TMPFS_MAGIC;
3825 sb->s_op = &shmem_ops;
3826 sb->s_time_gran = 1;
3827#ifdef CONFIG_TMPFS_XATTR
3828 sb->s_xattr = shmem_xattr_handlers;
3829#endif
3830#ifdef CONFIG_TMPFS_POSIX_ACL
3831 sb->s_flags |= SB_POSIXACL;
3832#endif
3833 uuid_gen(&sb->s_uuid);
3834
3835 inode = shmem_get_inode(sb, NULL, S_IFDIR | sbinfo->mode, 0, VM_NORESERVE);
3836 if (!inode)
3837 goto failed;
3838 inode->i_uid = sbinfo->uid;
3839 inode->i_gid = sbinfo->gid;
3840 sb->s_root = d_make_root(inode);
3841 if (!sb->s_root)
3842 goto failed;
3843 return 0;
3844
3845failed:
3846 shmem_put_super(sb);
3847 return err;
3848}
3849
3850static struct kmem_cache *shmem_inode_cachep;
3851
3852static struct inode *shmem_alloc_inode(struct super_block *sb)
3853{
3854 struct shmem_inode_info *info;
3855 info = kmem_cache_alloc(shmem_inode_cachep, GFP_KERNEL);
3856 if (!info)
3857 return NULL;
3858 return &info->vfs_inode;
3859}
3860
3861static void shmem_destroy_callback(struct rcu_head *head)
3862{
3863 struct inode *inode = container_of(head, struct inode, i_rcu);
3864 if (S_ISLNK(inode->i_mode))
3865 kfree(inode->i_link);
3866 kmem_cache_free(shmem_inode_cachep, SHMEM_I(inode));
3867}
3868
3869static void shmem_destroy_inode(struct inode *inode)
3870{
3871 if (S_ISREG(inode->i_mode))
3872 mpol_free_shared_policy(&SHMEM_I(inode)->policy);
3873 call_rcu(&inode->i_rcu, shmem_destroy_callback);
3874}
3875
3876static void shmem_init_inode(void *foo)
3877{
3878 struct shmem_inode_info *info = foo;
3879 inode_init_once(&info->vfs_inode);
3880}
3881
3882static void shmem_init_inodecache(void)
3883{
3884 shmem_inode_cachep = kmem_cache_create("shmem_inode_cache",
3885 sizeof(struct shmem_inode_info),
3886 0, SLAB_PANIC|SLAB_ACCOUNT, shmem_init_inode);
3887}
3888
3889static void shmem_destroy_inodecache(void)
3890{
3891 kmem_cache_destroy(shmem_inode_cachep);
3892}
3893
3894static const struct address_space_operations shmem_aops = {
3895 .writepage = shmem_writepage,
3896 .set_page_dirty = __set_page_dirty_no_writeback,
3897#ifdef CONFIG_TMPFS
3898 .write_begin = shmem_write_begin,
3899 .write_end = shmem_write_end,
3900#endif
3901#ifdef CONFIG_MIGRATION
3902 .migratepage = migrate_page,
3903#endif
3904 .error_remove_page = generic_error_remove_page,
3905};
3906
3907static const struct file_operations shmem_file_operations = {
3908 .mmap = shmem_mmap,
3909 .get_unmapped_area = shmem_get_unmapped_area,
3910#ifdef CONFIG_TMPFS
3911 .llseek = shmem_file_llseek,
3912 .read_iter = shmem_file_read_iter,
3913 .write_iter = generic_file_write_iter,
3914 .fsync = noop_fsync,
3915 .splice_read = generic_file_splice_read,
3916 .splice_write = iter_file_splice_write,
3917 .fallocate = shmem_fallocate,
3918#endif
3919};
3920
3921static const struct inode_operations shmem_inode_operations = {
3922 .getattr = shmem_getattr,
3923 .setattr = shmem_setattr,
3924#ifdef CONFIG_TMPFS_XATTR
3925 .listxattr = shmem_listxattr,
3926 .set_acl = simple_set_acl,
3927#endif
3928};
3929
3930static const struct inode_operations shmem_dir_inode_operations = {
3931#ifdef CONFIG_TMPFS
3932 .create = shmem_create,
3933 .lookup = simple_lookup,
3934 .link = shmem_link,
3935 .unlink = shmem_unlink,
3936 .symlink = shmem_symlink,
3937 .mkdir = shmem_mkdir,
3938 .rmdir = shmem_rmdir,
3939 .mknod = shmem_mknod,
3940 .rename = shmem_rename2,
3941 .tmpfile = shmem_tmpfile,
3942#endif
3943#ifdef CONFIG_TMPFS_XATTR
3944 .listxattr = shmem_listxattr,
3945#endif
3946#ifdef CONFIG_TMPFS_POSIX_ACL
3947 .setattr = shmem_setattr,
3948 .set_acl = simple_set_acl,
3949#endif
3950};
3951
3952static const struct inode_operations shmem_special_inode_operations = {
3953#ifdef CONFIG_TMPFS_XATTR
3954 .listxattr = shmem_listxattr,
3955#endif
3956#ifdef CONFIG_TMPFS_POSIX_ACL
3957 .setattr = shmem_setattr,
3958 .set_acl = simple_set_acl,
3959#endif
3960};
3961
3962static const struct super_operations shmem_ops = {
3963 .alloc_inode = shmem_alloc_inode,
3964 .destroy_inode = shmem_destroy_inode,
3965#ifdef CONFIG_TMPFS
3966 .statfs = shmem_statfs,
3967 .remount_fs = shmem_remount_fs,
3968 .show_options = shmem_show_options,
3969#endif
3970 .evict_inode = shmem_evict_inode,
3971 .drop_inode = generic_delete_inode,
3972 .put_super = shmem_put_super,
3973#ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE
3974 .nr_cached_objects = shmem_unused_huge_count,
3975 .free_cached_objects = shmem_unused_huge_scan,
3976#endif
3977};
3978
3979static const struct vm_operations_struct shmem_vm_ops = {
3980 .fault = shmem_fault,
3981 .map_pages = filemap_map_pages,
3982#ifdef CONFIG_NUMA
3983 .set_policy = shmem_set_policy,
3984 .get_policy = shmem_get_policy,
3985#endif
3986};
3987
3988static struct dentry *shmem_mount(struct file_system_type *fs_type,
3989 int flags, const char *dev_name, void *data)
3990{
3991 return mount_nodev(fs_type, flags, data, shmem_fill_super);
3992}
3993
3994static struct file_system_type shmem_fs_type = {
3995 .owner = THIS_MODULE,
3996 .name = "tmpfs",
3997 .mount = shmem_mount,
3998 .kill_sb = kill_litter_super,
3999 .fs_flags = FS_USERNS_MOUNT,
4000};
4001
4002int __init shmem_init(void)
4003{
4004 int error;
4005
4006
4007 if (shmem_inode_cachep)
4008 return 0;
4009
4010 shmem_init_inodecache();
4011
4012 error = register_filesystem(&shmem_fs_type);
4013 if (error) {
4014 pr_err("Could not register tmpfs\n");
4015 goto out2;
4016 }
4017
4018 shm_mnt = kern_mount(&shmem_fs_type);
4019 if (IS_ERR(shm_mnt)) {
4020 error = PTR_ERR(shm_mnt);
4021 pr_err("Could not kern_mount tmpfs\n");
4022 goto out1;
4023 }
4024
4025#ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE
4026 if (has_transparent_hugepage() && shmem_huge > SHMEM_HUGE_DENY)
4027 SHMEM_SB(shm_mnt->mnt_sb)->huge = shmem_huge;
4028 else
4029 shmem_huge = 0;
4030#endif
4031 return 0;
4032
4033out1:
4034 unregister_filesystem(&shmem_fs_type);
4035out2:
4036 shmem_destroy_inodecache();
4037 shm_mnt = ERR_PTR(error);
4038 return error;
4039}
4040
4041#if defined(CONFIG_TRANSPARENT_HUGE_PAGECACHE) && defined(CONFIG_SYSFS)
4042static ssize_t shmem_enabled_show(struct kobject *kobj,
4043 struct kobj_attribute *attr, char *buf)
4044{
4045 int values[] = {
4046 SHMEM_HUGE_ALWAYS,
4047 SHMEM_HUGE_WITHIN_SIZE,
4048 SHMEM_HUGE_ADVISE,
4049 SHMEM_HUGE_NEVER,
4050 SHMEM_HUGE_DENY,
4051 SHMEM_HUGE_FORCE,
4052 };
4053 int i, count;
4054
4055 for (i = 0, count = 0; i < ARRAY_SIZE(values); i++) {
4056 const char *fmt = shmem_huge == values[i] ? "[%s] " : "%s ";
4057
4058 count += sprintf(buf + count, fmt,
4059 shmem_format_huge(values[i]));
4060 }
4061 buf[count - 1] = '\n';
4062 return count;
4063}
4064
4065static ssize_t shmem_enabled_store(struct kobject *kobj,
4066 struct kobj_attribute *attr, const char *buf, size_t count)
4067{
4068 char tmp[16];
4069 int huge;
4070
4071 if (count + 1 > sizeof(tmp))
4072 return -EINVAL;
4073 memcpy(tmp, buf, count);
4074 tmp[count] = '\0';
4075 if (count && tmp[count - 1] == '\n')
4076 tmp[count - 1] = '\0';
4077
4078 huge = shmem_parse_huge(tmp);
4079 if (huge == -EINVAL)
4080 return -EINVAL;
4081 if (!has_transparent_hugepage() &&
4082 huge != SHMEM_HUGE_NEVER && huge != SHMEM_HUGE_DENY)
4083 return -EINVAL;
4084
4085 shmem_huge = huge;
4086 if (shmem_huge > SHMEM_HUGE_DENY)
4087 SHMEM_SB(shm_mnt->mnt_sb)->huge = shmem_huge;
4088 return count;
4089}
4090
4091struct kobj_attribute shmem_enabled_attr =
4092 __ATTR(shmem_enabled, 0644, shmem_enabled_show, shmem_enabled_store);
4093#endif
4094
4095#ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE
4096bool shmem_huge_enabled(struct vm_area_struct *vma)
4097{
4098 struct inode *inode = file_inode(vma->vm_file);
4099 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
4100 loff_t i_size;
4101 pgoff_t off;
4102
4103 if (shmem_huge == SHMEM_HUGE_FORCE)
4104 return true;
4105 if (shmem_huge == SHMEM_HUGE_DENY)
4106 return false;
4107 switch (sbinfo->huge) {
4108 case SHMEM_HUGE_NEVER:
4109 return false;
4110 case SHMEM_HUGE_ALWAYS:
4111 return true;
4112 case SHMEM_HUGE_WITHIN_SIZE:
4113 off = round_up(vma->vm_pgoff, HPAGE_PMD_NR);
4114 i_size = round_up(i_size_read(inode), PAGE_SIZE);
4115 if (i_size >= HPAGE_PMD_SIZE &&
4116 i_size >> PAGE_SHIFT >= off)
4117 return true;
4118
4119 case SHMEM_HUGE_ADVISE:
4120
4121 return (vma->vm_flags & VM_HUGEPAGE);
4122 default:
4123 VM_BUG_ON(1);
4124 return false;
4125 }
4126}
4127#endif
4128
4129#else
4130
4131
4132
4133
4134
4135
4136
4137
4138
4139
4140static struct file_system_type shmem_fs_type = {
4141 .name = "tmpfs",
4142 .mount = ramfs_mount,
4143 .kill_sb = kill_litter_super,
4144 .fs_flags = FS_USERNS_MOUNT,
4145};
4146
4147int __init shmem_init(void)
4148{
4149 BUG_ON(register_filesystem(&shmem_fs_type) != 0);
4150
4151 shm_mnt = kern_mount(&shmem_fs_type);
4152 BUG_ON(IS_ERR(shm_mnt));
4153
4154 return 0;
4155}
4156
4157int shmem_unuse(swp_entry_t swap, struct page *page)
4158{
4159 return 0;
4160}
4161
4162int shmem_lock(struct file *file, int lock, struct user_struct *user)
4163{
4164 return 0;
4165}
4166
4167void shmem_unlock_mapping(struct address_space *mapping)
4168{
4169}
4170
4171#ifdef CONFIG_MMU
4172unsigned long shmem_get_unmapped_area(struct file *file,
4173 unsigned long addr, unsigned long len,
4174 unsigned long pgoff, unsigned long flags)
4175{
4176 return current->mm->get_unmapped_area(file, addr, len, pgoff, flags);
4177}
4178#endif
4179
4180void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend)
4181{
4182 truncate_inode_pages_range(inode->i_mapping, lstart, lend);
4183}
4184EXPORT_SYMBOL_GPL(shmem_truncate_range);
4185
4186#define shmem_vm_ops generic_file_vm_ops
4187#define shmem_file_operations ramfs_file_operations
4188#define shmem_get_inode(sb, dir, mode, dev, flags) ramfs_get_inode(sb, dir, mode, dev)
4189#define shmem_acct_size(flags, size) 0
4190#define shmem_unacct_size(flags, size) do {} while (0)
4191
4192#endif
4193
4194
4195
4196static const struct dentry_operations anon_ops = {
4197 .d_dname = simple_dname
4198};
4199
4200static struct file *__shmem_file_setup(struct vfsmount *mnt, const char *name, loff_t size,
4201 unsigned long flags, unsigned int i_flags)
4202{
4203 struct file *res;
4204 struct inode *inode;
4205 struct path path;
4206 struct super_block *sb;
4207 struct qstr this;
4208
4209 if (IS_ERR(mnt))
4210 return ERR_CAST(mnt);
4211
4212 if (size < 0 || size > MAX_LFS_FILESIZE)
4213 return ERR_PTR(-EINVAL);
4214
4215 if (shmem_acct_size(flags, size))
4216 return ERR_PTR(-ENOMEM);
4217
4218 res = ERR_PTR(-ENOMEM);
4219 this.name = name;
4220 this.len = strlen(name);
4221 this.hash = 0;
4222 sb = mnt->mnt_sb;
4223 path.mnt = mntget(mnt);
4224 path.dentry = d_alloc_pseudo(sb, &this);
4225 if (!path.dentry)
4226 goto put_memory;
4227 d_set_d_op(path.dentry, &anon_ops);
4228
4229 res = ERR_PTR(-ENOSPC);
4230 inode = shmem_get_inode(sb, NULL, S_IFREG | S_IRWXUGO, 0, flags);
4231 if (!inode)
4232 goto put_memory;
4233
4234 inode->i_flags |= i_flags;
4235 d_instantiate(path.dentry, inode);
4236 inode->i_size = size;
4237 clear_nlink(inode);
4238 res = ERR_PTR(ramfs_nommu_expand_for_mapping(inode, size));
4239 if (IS_ERR(res))
4240 goto put_path;
4241
4242 res = alloc_file(&path, FMODE_WRITE | FMODE_READ,
4243 &shmem_file_operations);
4244 if (IS_ERR(res))
4245 goto put_path;
4246
4247 return res;
4248
4249put_memory:
4250 shmem_unacct_size(flags, size);
4251put_path:
4252 path_put(&path);
4253 return res;
4254}
4255
4256
4257
4258
4259
4260
4261
4262
4263
4264
4265
4266struct file *shmem_kernel_file_setup(const char *name, loff_t size, unsigned long flags)
4267{
4268 return __shmem_file_setup(shm_mnt, name, size, flags, S_PRIVATE);
4269}
4270
4271
4272
4273
4274
4275
4276
4277struct file *shmem_file_setup(const char *name, loff_t size, unsigned long flags)
4278{
4279 return __shmem_file_setup(shm_mnt, name, size, flags, 0);
4280}
4281EXPORT_SYMBOL_GPL(shmem_file_setup);
4282
4283
4284
4285
4286
4287
4288
4289
4290struct file *shmem_file_setup_with_mnt(struct vfsmount *mnt, const char *name,
4291 loff_t size, unsigned long flags)
4292{
4293 return __shmem_file_setup(mnt, name, size, flags, 0);
4294}
4295EXPORT_SYMBOL_GPL(shmem_file_setup_with_mnt);
4296
4297
4298
4299
4300
4301int shmem_zero_setup(struct vm_area_struct *vma)
4302{
4303 struct file *file;
4304 loff_t size = vma->vm_end - vma->vm_start;
4305
4306
4307
4308
4309
4310
4311
4312 file = shmem_kernel_file_setup("dev/zero", size, vma->vm_flags);
4313 if (IS_ERR(file))
4314 return PTR_ERR(file);
4315
4316 if (vma->vm_file)
4317 fput(vma->vm_file);
4318 vma->vm_file = file;
4319 vma->vm_ops = &shmem_vm_ops;
4320
4321 if (IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE) &&
4322 ((vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK) <
4323 (vma->vm_end & HPAGE_PMD_MASK)) {
4324 khugepaged_enter(vma, vma->vm_flags);
4325 }
4326
4327 return 0;
4328}
4329
4330
4331
4332
4333
4334
4335
4336
4337
4338
4339
4340
4341
4342
4343
4344
4345struct page *shmem_read_mapping_page_gfp(struct address_space *mapping,
4346 pgoff_t index, gfp_t gfp)
4347{
4348#ifdef CONFIG_SHMEM
4349 struct inode *inode = mapping->host;
4350 struct page *page;
4351 int error;
4352
4353 BUG_ON(mapping->a_ops != &shmem_aops);
4354 error = shmem_getpage_gfp(inode, index, &page, SGP_CACHE,
4355 gfp, NULL, NULL, NULL);
4356 if (error)
4357 page = ERR_PTR(error);
4358 else
4359 unlock_page(page);
4360 return page;
4361#else
4362
4363
4364
4365 return read_cache_page_gfp(mapping, index, gfp);
4366#endif
4367}
4368EXPORT_SYMBOL_GPL(shmem_read_mapping_page_gfp);
4369