1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23#include <linux/fs.h>
24#include <linux/init.h>
25#include <linux/vfs.h>
26#include <linux/mount.h>
27#include <linux/pagemap.h>
28#include <linux/file.h>
29#include <linux/mm.h>
30#include <linux/module.h>
31#include <linux/swap.h>
32#include <linux/ima.h>
33
34static struct vfsmount *shm_mnt;
35
36#ifdef CONFIG_SHMEM
37
38
39
40
41
42
43#include <linux/xattr.h>
44#include <linux/exportfs.h>
45#include <linux/generic_acl.h>
46#include <linux/mman.h>
47#include <linux/string.h>
48#include <linux/slab.h>
49#include <linux/backing-dev.h>
50#include <linux/shmem_fs.h>
51#include <linux/writeback.h>
52#include <linux/blkdev.h>
53#include <linux/security.h>
54#include <linux/swapops.h>
55#include <linux/mempolicy.h>
56#include <linux/namei.h>
57#include <linux/ctype.h>
58#include <linux/migrate.h>
59#include <linux/highmem.h>
60#include <linux/seq_file.h>
61#include <linux/magic.h>
62
63#include <asm/uaccess.h>
64#include <asm/div64.h>
65#include <asm/pgtable.h>
66
67
68
69
70
71
72
73
74
75
76
77
78
79#define ENTRIES_PER_PAGE (PAGE_CACHE_SIZE/sizeof(unsigned long))
80#define ENTRIES_PER_PAGEPAGE ((unsigned long long)ENTRIES_PER_PAGE*ENTRIES_PER_PAGE)
81
82#define SHMSWP_MAX_INDEX (SHMEM_NR_DIRECT + (ENTRIES_PER_PAGEPAGE/2) * (ENTRIES_PER_PAGE+1))
83#define SHMSWP_MAX_BYTES (SHMSWP_MAX_INDEX << PAGE_CACHE_SHIFT)
84
85#define SHMEM_MAX_BYTES min_t(unsigned long long, SHMSWP_MAX_BYTES, MAX_LFS_FILESIZE)
86#define SHMEM_MAX_INDEX ((unsigned long)((SHMEM_MAX_BYTES+1) >> PAGE_CACHE_SHIFT))
87
88#define BLOCKS_PER_PAGE (PAGE_CACHE_SIZE/512)
89#define VM_ACCT(size) (PAGE_CACHE_ALIGN(size) >> PAGE_SHIFT)
90
91
92#define SHMEM_PAGEIN VM_READ
93#define SHMEM_TRUNCATE VM_WRITE
94
95
96#define LATENCY_LIMIT 64
97
98
99#define BOGO_DIRENT_SIZE 20
100
101
102enum sgp_type {
103 SGP_READ,
104 SGP_CACHE,
105 SGP_DIRTY,
106 SGP_WRITE,
107};
108
109#ifdef CONFIG_TMPFS
110static unsigned long shmem_default_max_blocks(void)
111{
112 return totalram_pages / 2;
113}
114
115static unsigned long shmem_default_max_inodes(void)
116{
117 return min(totalram_pages - totalhigh_pages, totalram_pages / 2);
118}
119#endif
120
121static int shmem_getpage(struct inode *inode, unsigned long idx,
122 struct page **pagep, enum sgp_type sgp, int *type);
123
124static inline struct page *shmem_dir_alloc(gfp_t gfp_mask)
125{
126
127
128
129
130
131
132
133 return alloc_pages((gfp_mask & ~GFP_MOVABLE_MASK) | __GFP_ZERO,
134 PAGE_CACHE_SHIFT-PAGE_SHIFT);
135}
136
137static inline void shmem_dir_free(struct page *page)
138{
139 __free_pages(page, PAGE_CACHE_SHIFT-PAGE_SHIFT);
140}
141
142static struct page **shmem_dir_map(struct page *page)
143{
144 return (struct page **)kmap_atomic(page, KM_USER0);
145}
146
147static inline void shmem_dir_unmap(struct page **dir)
148{
149 kunmap_atomic(dir, KM_USER0);
150}
151
152static swp_entry_t *shmem_swp_map(struct page *page)
153{
154 return (swp_entry_t *)kmap_atomic(page, KM_USER1);
155}
156
157static inline void shmem_swp_balance_unmap(void)
158{
159
160
161
162
163
164
165
166 (void) kmap_atomic(ZERO_PAGE(0), KM_USER1);
167}
168
169static inline void shmem_swp_unmap(swp_entry_t *entry)
170{
171 kunmap_atomic(entry, KM_USER1);
172}
173
174static inline struct shmem_sb_info *SHMEM_SB(struct super_block *sb)
175{
176 return sb->s_fs_info;
177}
178
179
180
181
182
183
184
185static inline int shmem_acct_size(unsigned long flags, loff_t size)
186{
187 return (flags & VM_NORESERVE) ?
188 0 : security_vm_enough_memory_kern(VM_ACCT(size));
189}
190
191static inline void shmem_unacct_size(unsigned long flags, loff_t size)
192{
193 if (!(flags & VM_NORESERVE))
194 vm_unacct_memory(VM_ACCT(size));
195}
196
197
198
199
200
201
202
203static inline int shmem_acct_block(unsigned long flags)
204{
205 return (flags & VM_NORESERVE) ?
206 security_vm_enough_memory_kern(VM_ACCT(PAGE_CACHE_SIZE)) : 0;
207}
208
209static inline void shmem_unacct_blocks(unsigned long flags, long pages)
210{
211 if (flags & VM_NORESERVE)
212 vm_unacct_memory(pages * VM_ACCT(PAGE_CACHE_SIZE));
213}
214
215static const struct super_operations shmem_ops;
216static const struct address_space_operations shmem_aops;
217static const struct file_operations shmem_file_operations;
218static const struct inode_operations shmem_inode_operations;
219static const struct inode_operations shmem_dir_inode_operations;
220static const struct inode_operations shmem_special_inode_operations;
221static const struct vm_operations_struct shmem_vm_ops;
222
223static struct backing_dev_info shmem_backing_dev_info __read_mostly = {
224 .ra_pages = 0,
225 .capabilities = BDI_CAP_NO_ACCT_AND_WRITEBACK | BDI_CAP_SWAP_BACKED,
226 .unplug_io_fn = default_unplug_io_fn,
227};
228
229static LIST_HEAD(shmem_swaplist);
230static DEFINE_MUTEX(shmem_swaplist_mutex);
231
232static void shmem_free_blocks(struct inode *inode, long pages)
233{
234 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
235 if (sbinfo->max_blocks) {
236 spin_lock(&sbinfo->stat_lock);
237 sbinfo->free_blocks += pages;
238 inode->i_blocks -= pages*BLOCKS_PER_PAGE;
239 spin_unlock(&sbinfo->stat_lock);
240 }
241}
242
243static int shmem_reserve_inode(struct super_block *sb)
244{
245 struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
246 if (sbinfo->max_inodes) {
247 spin_lock(&sbinfo->stat_lock);
248 if (!sbinfo->free_inodes) {
249 spin_unlock(&sbinfo->stat_lock);
250 return -ENOSPC;
251 }
252 sbinfo->free_inodes--;
253 spin_unlock(&sbinfo->stat_lock);
254 }
255 return 0;
256}
257
258static void shmem_free_inode(struct super_block *sb)
259{
260 struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
261 if (sbinfo->max_inodes) {
262 spin_lock(&sbinfo->stat_lock);
263 sbinfo->free_inodes++;
264 spin_unlock(&sbinfo->stat_lock);
265 }
266}
267
268
269
270
271
272
273
274
275
276
277
278
279
280static void shmem_recalc_inode(struct inode *inode)
281{
282 struct shmem_inode_info *info = SHMEM_I(inode);
283 long freed;
284
285 freed = info->alloced - info->swapped - inode->i_mapping->nrpages;
286 if (freed > 0) {
287 info->alloced -= freed;
288 shmem_unacct_blocks(info->flags, freed);
289 shmem_free_blocks(inode, freed);
290 }
291}
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330static swp_entry_t *shmem_swp_entry(struct shmem_inode_info *info, unsigned long index, struct page **page)
331{
332 unsigned long offset;
333 struct page **dir;
334 struct page *subdir;
335
336 if (index < SHMEM_NR_DIRECT) {
337 shmem_swp_balance_unmap();
338 return info->i_direct+index;
339 }
340 if (!info->i_indirect) {
341 if (page) {
342 info->i_indirect = *page;
343 *page = NULL;
344 }
345 return NULL;
346 }
347
348 index -= SHMEM_NR_DIRECT;
349 offset = index % ENTRIES_PER_PAGE;
350 index /= ENTRIES_PER_PAGE;
351 dir = shmem_dir_map(info->i_indirect);
352
353 if (index >= ENTRIES_PER_PAGE/2) {
354 index -= ENTRIES_PER_PAGE/2;
355 dir += ENTRIES_PER_PAGE/2 + index/ENTRIES_PER_PAGE;
356 index %= ENTRIES_PER_PAGE;
357 subdir = *dir;
358 if (!subdir) {
359 if (page) {
360 *dir = *page;
361 *page = NULL;
362 }
363 shmem_dir_unmap(dir);
364 return NULL;
365 }
366 shmem_dir_unmap(dir);
367 dir = shmem_dir_map(subdir);
368 }
369
370 dir += index;
371 subdir = *dir;
372 if (!subdir) {
373 if (!page || !(subdir = *page)) {
374 shmem_dir_unmap(dir);
375 return NULL;
376 }
377 *dir = subdir;
378 *page = NULL;
379 }
380 shmem_dir_unmap(dir);
381 return shmem_swp_map(subdir) + offset;
382}
383
384static void shmem_swp_set(struct shmem_inode_info *info, swp_entry_t *entry, unsigned long value)
385{
386 long incdec = value? 1: -1;
387
388 entry->val = value;
389 info->swapped += incdec;
390 if ((unsigned long)(entry - info->i_direct) >= SHMEM_NR_DIRECT) {
391 struct page *page = kmap_atomic_to_page(entry);
392 set_page_private(page, page_private(page) + incdec);
393 }
394}
395
396
397
398
399
400
401
402
403
404static swp_entry_t *shmem_swp_alloc(struct shmem_inode_info *info, unsigned long index, enum sgp_type sgp)
405{
406 struct inode *inode = &info->vfs_inode;
407 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
408 struct page *page = NULL;
409 swp_entry_t *entry;
410
411 if (sgp != SGP_WRITE &&
412 ((loff_t) index << PAGE_CACHE_SHIFT) >= i_size_read(inode))
413 return ERR_PTR(-EINVAL);
414
415 while (!(entry = shmem_swp_entry(info, index, &page))) {
416 if (sgp == SGP_READ)
417 return shmem_swp_map(ZERO_PAGE(0));
418
419
420
421
422
423 if (sbinfo->max_blocks) {
424 spin_lock(&sbinfo->stat_lock);
425 if (sbinfo->free_blocks <= 1) {
426 spin_unlock(&sbinfo->stat_lock);
427 return ERR_PTR(-ENOSPC);
428 }
429 sbinfo->free_blocks--;
430 inode->i_blocks += BLOCKS_PER_PAGE;
431 spin_unlock(&sbinfo->stat_lock);
432 }
433
434 spin_unlock(&info->lock);
435 page = shmem_dir_alloc(mapping_gfp_mask(inode->i_mapping));
436 if (page)
437 set_page_private(page, 0);
438 spin_lock(&info->lock);
439
440 if (!page) {
441 shmem_free_blocks(inode, 1);
442 return ERR_PTR(-ENOMEM);
443 }
444 if (sgp != SGP_WRITE &&
445 ((loff_t) index << PAGE_CACHE_SHIFT) >= i_size_read(inode)) {
446 entry = ERR_PTR(-EINVAL);
447 break;
448 }
449 if (info->next_index <= index)
450 info->next_index = index + 1;
451 }
452 if (page) {
453
454 shmem_free_blocks(inode, 1);
455 shmem_dir_free(page);
456 }
457 if (info->next_index <= index && !IS_ERR(entry))
458 info->next_index = index + 1;
459 return entry;
460}
461
462
463
464
465
466
467
468static int shmem_free_swp(swp_entry_t *dir, swp_entry_t *edir,
469 spinlock_t *punch_lock)
470{
471 spinlock_t *punch_unlock = NULL;
472 swp_entry_t *ptr;
473 int freed = 0;
474
475 for (ptr = dir; ptr < edir; ptr++) {
476 if (ptr->val) {
477 if (unlikely(punch_lock)) {
478 punch_unlock = punch_lock;
479 punch_lock = NULL;
480 spin_lock(punch_unlock);
481 if (!ptr->val)
482 continue;
483 }
484 free_swap_and_cache(*ptr);
485 *ptr = (swp_entry_t){0};
486 freed++;
487 }
488 }
489 if (punch_unlock)
490 spin_unlock(punch_unlock);
491 return freed;
492}
493
494static int shmem_map_and_free_swp(struct page *subdir, int offset,
495 int limit, struct page ***dir, spinlock_t *punch_lock)
496{
497 swp_entry_t *ptr;
498 int freed = 0;
499
500 ptr = shmem_swp_map(subdir);
501 for (; offset < limit; offset += LATENCY_LIMIT) {
502 int size = limit - offset;
503 if (size > LATENCY_LIMIT)
504 size = LATENCY_LIMIT;
505 freed += shmem_free_swp(ptr+offset, ptr+offset+size,
506 punch_lock);
507 if (need_resched()) {
508 shmem_swp_unmap(ptr);
509 if (*dir) {
510 shmem_dir_unmap(*dir);
511 *dir = NULL;
512 }
513 cond_resched();
514 ptr = shmem_swp_map(subdir);
515 }
516 }
517 shmem_swp_unmap(ptr);
518 return freed;
519}
520
521static void shmem_free_pages(struct list_head *next)
522{
523 struct page *page;
524 int freed = 0;
525
526 do {
527 page = container_of(next, struct page, lru);
528 next = next->next;
529 shmem_dir_free(page);
530 freed++;
531 if (freed >= LATENCY_LIMIT) {
532 cond_resched();
533 freed = 0;
534 }
535 } while (next);
536}
537
538static void shmem_truncate_range(struct inode *inode, loff_t start, loff_t end)
539{
540 struct shmem_inode_info *info = SHMEM_I(inode);
541 unsigned long idx;
542 unsigned long size;
543 unsigned long limit;
544 unsigned long stage;
545 unsigned long diroff;
546 struct page **dir;
547 struct page *topdir;
548 struct page *middir;
549 struct page *subdir;
550 swp_entry_t *ptr;
551 LIST_HEAD(pages_to_free);
552 long nr_pages_to_free = 0;
553 long nr_swaps_freed = 0;
554 int offset;
555 int freed;
556 int punch_hole;
557 spinlock_t *needs_lock;
558 spinlock_t *punch_lock;
559 unsigned long upper_limit;
560
561 inode->i_ctime = inode->i_mtime = CURRENT_TIME;
562 idx = (start + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
563 if (idx >= info->next_index)
564 return;
565
566 spin_lock(&info->lock);
567 info->flags |= SHMEM_TRUNCATE;
568 if (likely(end == (loff_t) -1)) {
569 limit = info->next_index;
570 upper_limit = SHMEM_MAX_INDEX;
571 info->next_index = idx;
572 needs_lock = NULL;
573 punch_hole = 0;
574 } else {
575 if (end + 1 >= inode->i_size) {
576 limit = (inode->i_size + PAGE_CACHE_SIZE - 1) >>
577 PAGE_CACHE_SHIFT;
578 upper_limit = SHMEM_MAX_INDEX;
579 } else {
580 limit = (end + 1) >> PAGE_CACHE_SHIFT;
581 upper_limit = limit;
582 }
583 needs_lock = &info->lock;
584 punch_hole = 1;
585 }
586
587 topdir = info->i_indirect;
588 if (topdir && idx <= SHMEM_NR_DIRECT && !punch_hole) {
589 info->i_indirect = NULL;
590 nr_pages_to_free++;
591 list_add(&topdir->lru, &pages_to_free);
592 }
593 spin_unlock(&info->lock);
594
595 if (info->swapped && idx < SHMEM_NR_DIRECT) {
596 ptr = info->i_direct;
597 size = limit;
598 if (size > SHMEM_NR_DIRECT)
599 size = SHMEM_NR_DIRECT;
600 nr_swaps_freed = shmem_free_swp(ptr+idx, ptr+size, needs_lock);
601 }
602
603
604
605
606
607 if (!topdir || limit <= SHMEM_NR_DIRECT)
608 goto done2;
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623 upper_limit -= SHMEM_NR_DIRECT;
624 limit -= SHMEM_NR_DIRECT;
625 idx = (idx > SHMEM_NR_DIRECT)? (idx - SHMEM_NR_DIRECT): 0;
626 offset = idx % ENTRIES_PER_PAGE;
627 idx -= offset;
628
629 dir = shmem_dir_map(topdir);
630 stage = ENTRIES_PER_PAGEPAGE/2;
631 if (idx < ENTRIES_PER_PAGEPAGE/2) {
632 middir = topdir;
633 diroff = idx/ENTRIES_PER_PAGE;
634 } else {
635 dir += ENTRIES_PER_PAGE/2;
636 dir += (idx - ENTRIES_PER_PAGEPAGE/2)/ENTRIES_PER_PAGEPAGE;
637 while (stage <= idx)
638 stage += ENTRIES_PER_PAGEPAGE;
639 middir = *dir;
640 if (*dir) {
641 diroff = ((idx - ENTRIES_PER_PAGEPAGE/2) %
642 ENTRIES_PER_PAGEPAGE) / ENTRIES_PER_PAGE;
643 if (!diroff && !offset && upper_limit >= stage) {
644 if (needs_lock) {
645 spin_lock(needs_lock);
646 *dir = NULL;
647 spin_unlock(needs_lock);
648 needs_lock = NULL;
649 } else
650 *dir = NULL;
651 nr_pages_to_free++;
652 list_add(&middir->lru, &pages_to_free);
653 }
654 shmem_dir_unmap(dir);
655 dir = shmem_dir_map(middir);
656 } else {
657 diroff = 0;
658 offset = 0;
659 idx = stage;
660 }
661 }
662
663 for (; idx < limit; idx += ENTRIES_PER_PAGE, diroff++) {
664 if (unlikely(idx == stage)) {
665 shmem_dir_unmap(dir);
666 dir = shmem_dir_map(topdir) +
667 ENTRIES_PER_PAGE/2 + idx/ENTRIES_PER_PAGEPAGE;
668 while (!*dir) {
669 dir++;
670 idx += ENTRIES_PER_PAGEPAGE;
671 if (idx >= limit)
672 goto done1;
673 }
674 stage = idx + ENTRIES_PER_PAGEPAGE;
675 middir = *dir;
676 if (punch_hole)
677 needs_lock = &info->lock;
678 if (upper_limit >= stage) {
679 if (needs_lock) {
680 spin_lock(needs_lock);
681 *dir = NULL;
682 spin_unlock(needs_lock);
683 needs_lock = NULL;
684 } else
685 *dir = NULL;
686 nr_pages_to_free++;
687 list_add(&middir->lru, &pages_to_free);
688 }
689 shmem_dir_unmap(dir);
690 cond_resched();
691 dir = shmem_dir_map(middir);
692 diroff = 0;
693 }
694 punch_lock = needs_lock;
695 subdir = dir[diroff];
696 if (subdir && !offset && upper_limit-idx >= ENTRIES_PER_PAGE) {
697 if (needs_lock) {
698 spin_lock(needs_lock);
699 dir[diroff] = NULL;
700 spin_unlock(needs_lock);
701 punch_lock = NULL;
702 } else
703 dir[diroff] = NULL;
704 nr_pages_to_free++;
705 list_add(&subdir->lru, &pages_to_free);
706 }
707 if (subdir && page_private(subdir) ) {
708 size = limit - idx;
709 if (size > ENTRIES_PER_PAGE)
710 size = ENTRIES_PER_PAGE;
711 freed = shmem_map_and_free_swp(subdir,
712 offset, size, &dir, punch_lock);
713 if (!dir)
714 dir = shmem_dir_map(middir);
715 nr_swaps_freed += freed;
716 if (offset || punch_lock) {
717 spin_lock(&info->lock);
718 set_page_private(subdir,
719 page_private(subdir) - freed);
720 spin_unlock(&info->lock);
721 } else
722 BUG_ON(page_private(subdir) != freed);
723 }
724 offset = 0;
725 }
726done1:
727 shmem_dir_unmap(dir);
728done2:
729 if (inode->i_mapping->nrpages && (info->flags & SHMEM_PAGEIN)) {
730
731
732
733
734
735
736
737
738
739
740
741
742 truncate_inode_pages_range(inode->i_mapping, start, end);
743 if (punch_hole)
744 unmap_mapping_range(inode->i_mapping, start,
745 end - start, 1);
746 }
747
748 spin_lock(&info->lock);
749 info->flags &= ~SHMEM_TRUNCATE;
750 info->swapped -= nr_swaps_freed;
751 if (nr_pages_to_free)
752 shmem_free_blocks(inode, nr_pages_to_free);
753 shmem_recalc_inode(inode);
754 spin_unlock(&info->lock);
755
756
757
758
759 if (!list_empty(&pages_to_free)) {
760 pages_to_free.prev->next = NULL;
761 shmem_free_pages(pages_to_free.next);
762 }
763}
764
765static void shmem_truncate(struct inode *inode)
766{
767 shmem_truncate_range(inode, inode->i_size, (loff_t)-1);
768}
769
770static int shmem_notify_change(struct dentry *dentry, struct iattr *attr)
771{
772 struct inode *inode = dentry->d_inode;
773 struct page *page = NULL;
774 int error;
775
776 if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) {
777 if (attr->ia_size < inode->i_size) {
778
779
780
781
782
783
784
785 if (attr->ia_size & (PAGE_CACHE_SIZE-1)) {
786 (void) shmem_getpage(inode,
787 attr->ia_size>>PAGE_CACHE_SHIFT,
788 &page, SGP_READ, NULL);
789 if (page)
790 unlock_page(page);
791 }
792
793
794
795
796
797
798
799 if (attr->ia_size) {
800 struct shmem_inode_info *info = SHMEM_I(inode);
801 spin_lock(&info->lock);
802 info->flags &= ~SHMEM_PAGEIN;
803 spin_unlock(&info->lock);
804 }
805 }
806 }
807
808 error = inode_change_ok(inode, attr);
809 if (!error)
810 error = inode_setattr(inode, attr);
811#ifdef CONFIG_TMPFS_POSIX_ACL
812 if (!error && (attr->ia_valid & ATTR_MODE))
813 error = generic_acl_chmod(inode, &shmem_acl_ops);
814#endif
815 if (page)
816 page_cache_release(page);
817 return error;
818}
819
820static void shmem_delete_inode(struct inode *inode)
821{
822 struct shmem_inode_info *info = SHMEM_I(inode);
823
824 if (inode->i_op->truncate == shmem_truncate) {
825 truncate_inode_pages(inode->i_mapping, 0);
826 shmem_unacct_size(info->flags, inode->i_size);
827 inode->i_size = 0;
828 shmem_truncate(inode);
829 if (!list_empty(&info->swaplist)) {
830 mutex_lock(&shmem_swaplist_mutex);
831 list_del_init(&info->swaplist);
832 mutex_unlock(&shmem_swaplist_mutex);
833 }
834 }
835 BUG_ON(inode->i_blocks);
836 shmem_free_inode(inode->i_sb);
837 clear_inode(inode);
838}
839
840static inline int shmem_find_swp(swp_entry_t entry, swp_entry_t *dir, swp_entry_t *edir)
841{
842 swp_entry_t *ptr;
843
844 for (ptr = dir; ptr < edir; ptr++) {
845 if (ptr->val == entry.val)
846 return ptr - dir;
847 }
848 return -1;
849}
850
851static int shmem_unuse_inode(struct shmem_inode_info *info, swp_entry_t entry, struct page *page)
852{
853 struct inode *inode;
854 unsigned long idx;
855 unsigned long size;
856 unsigned long limit;
857 unsigned long stage;
858 struct page **dir;
859 struct page *subdir;
860 swp_entry_t *ptr;
861 int offset;
862 int error;
863
864 idx = 0;
865 ptr = info->i_direct;
866 spin_lock(&info->lock);
867 if (!info->swapped) {
868 list_del_init(&info->swaplist);
869 goto lost2;
870 }
871 limit = info->next_index;
872 size = limit;
873 if (size > SHMEM_NR_DIRECT)
874 size = SHMEM_NR_DIRECT;
875 offset = shmem_find_swp(entry, ptr, ptr+size);
876 if (offset >= 0)
877 goto found;
878 if (!info->i_indirect)
879 goto lost2;
880
881 dir = shmem_dir_map(info->i_indirect);
882 stage = SHMEM_NR_DIRECT + ENTRIES_PER_PAGEPAGE/2;
883
884 for (idx = SHMEM_NR_DIRECT; idx < limit; idx += ENTRIES_PER_PAGE, dir++) {
885 if (unlikely(idx == stage)) {
886 shmem_dir_unmap(dir-1);
887 if (cond_resched_lock(&info->lock)) {
888
889 if (limit > info->next_index) {
890 limit = info->next_index;
891 if (idx >= limit)
892 goto lost2;
893 }
894 }
895 dir = shmem_dir_map(info->i_indirect) +
896 ENTRIES_PER_PAGE/2 + idx/ENTRIES_PER_PAGEPAGE;
897 while (!*dir) {
898 dir++;
899 idx += ENTRIES_PER_PAGEPAGE;
900 if (idx >= limit)
901 goto lost1;
902 }
903 stage = idx + ENTRIES_PER_PAGEPAGE;
904 subdir = *dir;
905 shmem_dir_unmap(dir);
906 dir = shmem_dir_map(subdir);
907 }
908 subdir = *dir;
909 if (subdir && page_private(subdir)) {
910 ptr = shmem_swp_map(subdir);
911 size = limit - idx;
912 if (size > ENTRIES_PER_PAGE)
913 size = ENTRIES_PER_PAGE;
914 offset = shmem_find_swp(entry, ptr, ptr+size);
915 shmem_swp_unmap(ptr);
916 if (offset >= 0) {
917 shmem_dir_unmap(dir);
918 goto found;
919 }
920 }
921 }
922lost1:
923 shmem_dir_unmap(dir-1);
924lost2:
925 spin_unlock(&info->lock);
926 return 0;
927found:
928 idx += offset;
929 inode = igrab(&info->vfs_inode);
930 spin_unlock(&info->lock);
931
932
933
934
935
936
937
938
939 if (shmem_swaplist.next != &info->swaplist)
940 list_move_tail(&shmem_swaplist, &info->swaplist);
941 mutex_unlock(&shmem_swaplist_mutex);
942
943 error = 1;
944 if (!inode)
945 goto out;
946
947
948
949
950
951 error = mem_cgroup_cache_charge(page, current->mm, GFP_KERNEL);
952 if (error)
953 goto out;
954 error = radix_tree_preload(GFP_KERNEL);
955 if (error) {
956 mem_cgroup_uncharge_cache_page(page);
957 goto out;
958 }
959 error = 1;
960
961 spin_lock(&info->lock);
962 ptr = shmem_swp_entry(info, idx, NULL);
963 if (ptr && ptr->val == entry.val) {
964 error = add_to_page_cache_locked(page, inode->i_mapping,
965 idx, GFP_NOWAIT);
966
967 } else
968 mem_cgroup_uncharge_cache_page(page);
969
970 if (error == -EEXIST) {
971 struct page *filepage = find_get_page(inode->i_mapping, idx);
972 error = 1;
973 if (filepage) {
974
975
976
977
978 if (PageUptodate(filepage))
979 error = 0;
980 page_cache_release(filepage);
981 }
982 }
983 if (!error) {
984 delete_from_swap_cache(page);
985 set_page_dirty(page);
986 info->flags |= SHMEM_PAGEIN;
987 shmem_swp_set(info, ptr, 0);
988 swap_free(entry);
989 error = 1;
990 }
991 if (ptr)
992 shmem_swp_unmap(ptr);
993 spin_unlock(&info->lock);
994 radix_tree_preload_end();
995out:
996 unlock_page(page);
997 page_cache_release(page);
998 iput(inode);
999 return error;
1000}
1001
1002
1003
1004
1005int shmem_unuse(swp_entry_t entry, struct page *page)
1006{
1007 struct list_head *p, *next;
1008 struct shmem_inode_info *info;
1009 int found = 0;
1010
1011 mutex_lock(&shmem_swaplist_mutex);
1012 list_for_each_safe(p, next, &shmem_swaplist) {
1013 info = list_entry(p, struct shmem_inode_info, swaplist);
1014 found = shmem_unuse_inode(info, entry, page);
1015 cond_resched();
1016 if (found)
1017 goto out;
1018 }
1019 mutex_unlock(&shmem_swaplist_mutex);
1020out: return found;
1021}
1022
1023
1024
1025
1026static int shmem_writepage(struct page *page, struct writeback_control *wbc)
1027{
1028 struct shmem_inode_info *info;
1029 swp_entry_t *entry, swap;
1030 struct address_space *mapping;
1031 unsigned long index;
1032 struct inode *inode;
1033
1034 BUG_ON(!PageLocked(page));
1035 mapping = page->mapping;
1036 index = page->index;
1037 inode = mapping->host;
1038 info = SHMEM_I(inode);
1039 if (info->flags & VM_LOCKED)
1040 goto redirty;
1041 if (!total_swap_pages)
1042 goto redirty;
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053 if (wbc->for_reclaim)
1054 swap = get_swap_page();
1055 else
1056 swap.val = 0;
1057
1058 spin_lock(&info->lock);
1059 if (index >= info->next_index) {
1060 BUG_ON(!(info->flags & SHMEM_TRUNCATE));
1061 goto unlock;
1062 }
1063 entry = shmem_swp_entry(info, index, NULL);
1064 if (entry->val) {
1065
1066
1067
1068
1069 free_swap_and_cache(*entry);
1070 shmem_swp_set(info, entry, 0);
1071 }
1072 shmem_recalc_inode(inode);
1073
1074 if (swap.val && add_to_swap_cache(page, swap, GFP_ATOMIC) == 0) {
1075 remove_from_page_cache(page);
1076 shmem_swp_set(info, entry, swap.val);
1077 shmem_swp_unmap(entry);
1078 if (list_empty(&info->swaplist))
1079 inode = igrab(inode);
1080 else
1081 inode = NULL;
1082 spin_unlock(&info->lock);
1083 swap_duplicate(swap);
1084 BUG_ON(page_mapped(page));
1085 page_cache_release(page);
1086 swap_writepage(page, wbc);
1087 if (inode) {
1088 mutex_lock(&shmem_swaplist_mutex);
1089
1090 list_move_tail(&info->swaplist, &shmem_swaplist);
1091 mutex_unlock(&shmem_swaplist_mutex);
1092 iput(inode);
1093 }
1094 return 0;
1095 }
1096
1097 shmem_swp_unmap(entry);
1098unlock:
1099 spin_unlock(&info->lock);
1100
1101
1102
1103
1104 swapcache_free(swap, NULL);
1105redirty:
1106 set_page_dirty(page);
1107 if (wbc->for_reclaim)
1108 return AOP_WRITEPAGE_ACTIVATE;
1109 unlock_page(page);
1110 return 0;
1111}
1112
1113#ifdef CONFIG_NUMA
1114#ifdef CONFIG_TMPFS
1115static void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol)
1116{
1117 char buffer[64];
1118
1119 if (!mpol || mpol->mode == MPOL_DEFAULT)
1120 return;
1121
1122 mpol_to_str(buffer, sizeof(buffer), mpol, 1);
1123
1124 seq_printf(seq, ",mpol=%s", buffer);
1125}
1126
1127static struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo)
1128{
1129 struct mempolicy *mpol = NULL;
1130 if (sbinfo->mpol) {
1131 spin_lock(&sbinfo->stat_lock);
1132 mpol = sbinfo->mpol;
1133 mpol_get(mpol);
1134 spin_unlock(&sbinfo->stat_lock);
1135 }
1136 return mpol;
1137}
1138#endif
1139
1140static struct page *shmem_swapin(swp_entry_t entry, gfp_t gfp,
1141 struct shmem_inode_info *info, unsigned long idx)
1142{
1143 struct mempolicy mpol, *spol;
1144 struct vm_area_struct pvma;
1145 struct page *page;
1146
1147 spol = mpol_cond_copy(&mpol,
1148 mpol_shared_policy_lookup(&info->policy, idx));
1149
1150
1151 pvma.vm_start = 0;
1152 pvma.vm_pgoff = idx;
1153 pvma.vm_ops = NULL;
1154 pvma.vm_policy = spol;
1155 page = swapin_readahead(entry, gfp, &pvma, 0);
1156 return page;
1157}
1158
1159static struct page *shmem_alloc_page(gfp_t gfp,
1160 struct shmem_inode_info *info, unsigned long idx)
1161{
1162 struct vm_area_struct pvma;
1163
1164
1165 pvma.vm_start = 0;
1166 pvma.vm_pgoff = idx;
1167 pvma.vm_ops = NULL;
1168 pvma.vm_policy = mpol_shared_policy_lookup(&info->policy, idx);
1169
1170
1171
1172
1173 return alloc_page_vma(gfp, &pvma, 0);
1174}
1175#else
1176#ifdef CONFIG_TMPFS
1177static inline void shmem_show_mpol(struct seq_file *seq, struct mempolicy *p)
1178{
1179}
1180#endif
1181
1182static inline struct page *shmem_swapin(swp_entry_t entry, gfp_t gfp,
1183 struct shmem_inode_info *info, unsigned long idx)
1184{
1185 return swapin_readahead(entry, gfp, NULL, 0);
1186}
1187
1188static inline struct page *shmem_alloc_page(gfp_t gfp,
1189 struct shmem_inode_info *info, unsigned long idx)
1190{
1191 return alloc_page(gfp);
1192}
1193#endif
1194
1195#if !defined(CONFIG_NUMA) || !defined(CONFIG_TMPFS)
1196static inline struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo)
1197{
1198 return NULL;
1199}
1200#endif
1201
1202
1203
1204
1205
1206
1207
1208
1209static int shmem_getpage(struct inode *inode, unsigned long idx,
1210 struct page **pagep, enum sgp_type sgp, int *type)
1211{
1212 struct address_space *mapping = inode->i_mapping;
1213 struct shmem_inode_info *info = SHMEM_I(inode);
1214 struct shmem_sb_info *sbinfo;
1215 struct page *filepage = *pagep;
1216 struct page *swappage;
1217 swp_entry_t *entry;
1218 swp_entry_t swap;
1219 gfp_t gfp;
1220 int error;
1221
1222 if (idx >= SHMEM_MAX_INDEX)
1223 return -EFBIG;
1224
1225 if (type)
1226 *type = 0;
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236repeat:
1237 if (!filepage)
1238 filepage = find_lock_page(mapping, idx);
1239 if (filepage && PageUptodate(filepage))
1240 goto done;
1241 error = 0;
1242 gfp = mapping_gfp_mask(mapping);
1243 if (!filepage) {
1244
1245
1246
1247
1248 error = radix_tree_preload(gfp & ~__GFP_HIGHMEM);
1249 if (error)
1250 goto failed;
1251 radix_tree_preload_end();
1252 }
1253
1254 spin_lock(&info->lock);
1255 shmem_recalc_inode(inode);
1256 entry = shmem_swp_alloc(info, idx, sgp);
1257 if (IS_ERR(entry)) {
1258 spin_unlock(&info->lock);
1259 error = PTR_ERR(entry);
1260 goto failed;
1261 }
1262 swap = *entry;
1263
1264 if (swap.val) {
1265
1266 swappage = lookup_swap_cache(swap);
1267 if (!swappage) {
1268 shmem_swp_unmap(entry);
1269
1270 if (type && !(*type & VM_FAULT_MAJOR)) {
1271 __count_vm_event(PGMAJFAULT);
1272 *type |= VM_FAULT_MAJOR;
1273 }
1274 spin_unlock(&info->lock);
1275 swappage = shmem_swapin(swap, gfp, info, idx);
1276 if (!swappage) {
1277 spin_lock(&info->lock);
1278 entry = shmem_swp_alloc(info, idx, sgp);
1279 if (IS_ERR(entry))
1280 error = PTR_ERR(entry);
1281 else {
1282 if (entry->val == swap.val)
1283 error = -ENOMEM;
1284 shmem_swp_unmap(entry);
1285 }
1286 spin_unlock(&info->lock);
1287 if (error)
1288 goto failed;
1289 goto repeat;
1290 }
1291 wait_on_page_locked(swappage);
1292 page_cache_release(swappage);
1293 goto repeat;
1294 }
1295
1296
1297 if (!trylock_page(swappage)) {
1298 shmem_swp_unmap(entry);
1299 spin_unlock(&info->lock);
1300 wait_on_page_locked(swappage);
1301 page_cache_release(swappage);
1302 goto repeat;
1303 }
1304 if (PageWriteback(swappage)) {
1305 shmem_swp_unmap(entry);
1306 spin_unlock(&info->lock);
1307 wait_on_page_writeback(swappage);
1308 unlock_page(swappage);
1309 page_cache_release(swappage);
1310 goto repeat;
1311 }
1312 if (!PageUptodate(swappage)) {
1313 shmem_swp_unmap(entry);
1314 spin_unlock(&info->lock);
1315 unlock_page(swappage);
1316 page_cache_release(swappage);
1317 error = -EIO;
1318 goto failed;
1319 }
1320
1321 if (filepage) {
1322 shmem_swp_set(info, entry, 0);
1323 shmem_swp_unmap(entry);
1324 delete_from_swap_cache(swappage);
1325 spin_unlock(&info->lock);
1326 copy_highpage(filepage, swappage);
1327 unlock_page(swappage);
1328 page_cache_release(swappage);
1329 flush_dcache_page(filepage);
1330 SetPageUptodate(filepage);
1331 set_page_dirty(filepage);
1332 swap_free(swap);
1333 } else if (!(error = add_to_page_cache_locked(swappage, mapping,
1334 idx, GFP_NOWAIT))) {
1335 info->flags |= SHMEM_PAGEIN;
1336 shmem_swp_set(info, entry, 0);
1337 shmem_swp_unmap(entry);
1338 delete_from_swap_cache(swappage);
1339 spin_unlock(&info->lock);
1340 filepage = swappage;
1341 set_page_dirty(filepage);
1342 swap_free(swap);
1343 } else {
1344 shmem_swp_unmap(entry);
1345 spin_unlock(&info->lock);
1346 if (error == -ENOMEM) {
1347
1348
1349
1350
1351 error = mem_cgroup_shmem_charge_fallback(
1352 swappage,
1353 current->mm,
1354 gfp);
1355 if (error) {
1356 unlock_page(swappage);
1357 page_cache_release(swappage);
1358 goto failed;
1359 }
1360 }
1361 unlock_page(swappage);
1362 page_cache_release(swappage);
1363 goto repeat;
1364 }
1365 } else if (sgp == SGP_READ && !filepage) {
1366 shmem_swp_unmap(entry);
1367 filepage = find_get_page(mapping, idx);
1368 if (filepage &&
1369 (!PageUptodate(filepage) || !trylock_page(filepage))) {
1370 spin_unlock(&info->lock);
1371 wait_on_page_locked(filepage);
1372 page_cache_release(filepage);
1373 filepage = NULL;
1374 goto repeat;
1375 }
1376 spin_unlock(&info->lock);
1377 } else {
1378 shmem_swp_unmap(entry);
1379 sbinfo = SHMEM_SB(inode->i_sb);
1380 if (sbinfo->max_blocks) {
1381 spin_lock(&sbinfo->stat_lock);
1382 if (sbinfo->free_blocks == 0 ||
1383 shmem_acct_block(info->flags)) {
1384 spin_unlock(&sbinfo->stat_lock);
1385 spin_unlock(&info->lock);
1386 error = -ENOSPC;
1387 goto failed;
1388 }
1389 sbinfo->free_blocks--;
1390 inode->i_blocks += BLOCKS_PER_PAGE;
1391 spin_unlock(&sbinfo->stat_lock);
1392 } else if (shmem_acct_block(info->flags)) {
1393 spin_unlock(&info->lock);
1394 error = -ENOSPC;
1395 goto failed;
1396 }
1397
1398 if (!filepage) {
1399 int ret;
1400
1401 spin_unlock(&info->lock);
1402 filepage = shmem_alloc_page(gfp, info, idx);
1403 if (!filepage) {
1404 shmem_unacct_blocks(info->flags, 1);
1405 shmem_free_blocks(inode, 1);
1406 error = -ENOMEM;
1407 goto failed;
1408 }
1409 SetPageSwapBacked(filepage);
1410
1411
1412 error = mem_cgroup_cache_charge(filepage, current->mm,
1413 GFP_KERNEL);
1414 if (error) {
1415 page_cache_release(filepage);
1416 shmem_unacct_blocks(info->flags, 1);
1417 shmem_free_blocks(inode, 1);
1418 filepage = NULL;
1419 goto failed;
1420 }
1421
1422 spin_lock(&info->lock);
1423 entry = shmem_swp_alloc(info, idx, sgp);
1424 if (IS_ERR(entry))
1425 error = PTR_ERR(entry);
1426 else {
1427 swap = *entry;
1428 shmem_swp_unmap(entry);
1429 }
1430 ret = error || swap.val;
1431 if (ret)
1432 mem_cgroup_uncharge_cache_page(filepage);
1433 else
1434 ret = add_to_page_cache_lru(filepage, mapping,
1435 idx, GFP_NOWAIT);
1436
1437
1438
1439
1440 if (ret) {
1441 spin_unlock(&info->lock);
1442 page_cache_release(filepage);
1443 shmem_unacct_blocks(info->flags, 1);
1444 shmem_free_blocks(inode, 1);
1445 filepage = NULL;
1446 if (error)
1447 goto failed;
1448 goto repeat;
1449 }
1450 info->flags |= SHMEM_PAGEIN;
1451 }
1452
1453 info->alloced++;
1454 spin_unlock(&info->lock);
1455 clear_highpage(filepage);
1456 flush_dcache_page(filepage);
1457 SetPageUptodate(filepage);
1458 if (sgp == SGP_DIRTY)
1459 set_page_dirty(filepage);
1460 }
1461done:
1462 *pagep = filepage;
1463 return 0;
1464
1465failed:
1466 if (*pagep != filepage) {
1467 unlock_page(filepage);
1468 page_cache_release(filepage);
1469 }
1470 return error;
1471}
1472
1473static int shmem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1474{
1475 struct inode *inode = vma->vm_file->f_path.dentry->d_inode;
1476 int error;
1477 int ret;
1478
1479 if (((loff_t)vmf->pgoff << PAGE_CACHE_SHIFT) >= i_size_read(inode))
1480 return VM_FAULT_SIGBUS;
1481
1482 error = shmem_getpage(inode, vmf->pgoff, &vmf->page, SGP_CACHE, &ret);
1483 if (error)
1484 return ((error == -ENOMEM) ? VM_FAULT_OOM : VM_FAULT_SIGBUS);
1485
1486 return ret | VM_FAULT_LOCKED;
1487}
1488
1489#ifdef CONFIG_NUMA
1490static int shmem_set_policy(struct vm_area_struct *vma, struct mempolicy *new)
1491{
1492 struct inode *i = vma->vm_file->f_path.dentry->d_inode;
1493 return mpol_set_shared_policy(&SHMEM_I(i)->policy, vma, new);
1494}
1495
1496static struct mempolicy *shmem_get_policy(struct vm_area_struct *vma,
1497 unsigned long addr)
1498{
1499 struct inode *i = vma->vm_file->f_path.dentry->d_inode;
1500 unsigned long idx;
1501
1502 idx = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
1503 return mpol_shared_policy_lookup(&SHMEM_I(i)->policy, idx);
1504}
1505#endif
1506
1507int shmem_lock(struct file *file, int lock, struct user_struct *user)
1508{
1509 struct inode *inode = file->f_path.dentry->d_inode;
1510 struct shmem_inode_info *info = SHMEM_I(inode);
1511 int retval = -ENOMEM;
1512
1513 spin_lock(&info->lock);
1514 if (lock && !(info->flags & VM_LOCKED)) {
1515 if (!user_shm_lock(inode->i_size, user))
1516 goto out_nomem;
1517 info->flags |= VM_LOCKED;
1518 mapping_set_unevictable(file->f_mapping);
1519 }
1520 if (!lock && (info->flags & VM_LOCKED) && user) {
1521 user_shm_unlock(inode->i_size, user);
1522 info->flags &= ~VM_LOCKED;
1523 mapping_clear_unevictable(file->f_mapping);
1524 scan_mapping_unevictable_pages(file->f_mapping);
1525 }
1526 retval = 0;
1527
1528out_nomem:
1529 spin_unlock(&info->lock);
1530 return retval;
1531}
1532
1533static int shmem_mmap(struct file *file, struct vm_area_struct *vma)
1534{
1535 file_accessed(file);
1536 vma->vm_ops = &shmem_vm_ops;
1537 vma->vm_flags |= VM_CAN_NONLINEAR;
1538 return 0;
1539}
1540
1541static struct inode *shmem_get_inode(struct super_block *sb, int mode,
1542 dev_t dev, unsigned long flags)
1543{
1544 struct inode *inode;
1545 struct shmem_inode_info *info;
1546 struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
1547
1548 if (shmem_reserve_inode(sb))
1549 return NULL;
1550
1551 inode = new_inode(sb);
1552 if (inode) {
1553 inode->i_mode = mode;
1554 inode->i_uid = current_fsuid();
1555 inode->i_gid = current_fsgid();
1556 inode->i_blocks = 0;
1557 inode->i_mapping->backing_dev_info = &shmem_backing_dev_info;
1558 inode->i_atime = inode->i_mtime = inode->i_ctime = CURRENT_TIME;
1559 inode->i_generation = get_seconds();
1560 info = SHMEM_I(inode);
1561 memset(info, 0, (char *)inode - (char *)info);
1562 spin_lock_init(&info->lock);
1563 info->flags = flags & VM_NORESERVE;
1564 INIT_LIST_HEAD(&info->swaplist);
1565 cache_no_acl(inode);
1566
1567 switch (mode & S_IFMT) {
1568 default:
1569 inode->i_op = &shmem_special_inode_operations;
1570 init_special_inode(inode, mode, dev);
1571 break;
1572 case S_IFREG:
1573 inode->i_mapping->a_ops = &shmem_aops;
1574 inode->i_op = &shmem_inode_operations;
1575 inode->i_fop = &shmem_file_operations;
1576 mpol_shared_policy_init(&info->policy,
1577 shmem_get_sbmpol(sbinfo));
1578 break;
1579 case S_IFDIR:
1580 inc_nlink(inode);
1581
1582 inode->i_size = 2 * BOGO_DIRENT_SIZE;
1583 inode->i_op = &shmem_dir_inode_operations;
1584 inode->i_fop = &simple_dir_operations;
1585 break;
1586 case S_IFLNK:
1587
1588
1589
1590
1591 mpol_shared_policy_init(&info->policy, NULL);
1592 break;
1593 }
1594 } else
1595 shmem_free_inode(sb);
1596 return inode;
1597}
1598
1599#ifdef CONFIG_TMPFS
1600static const struct inode_operations shmem_symlink_inode_operations;
1601static const struct inode_operations shmem_symlink_inline_operations;
1602
1603
1604
1605
1606
1607
1608static int shmem_readpage(struct file *file, struct page *page)
1609{
1610 struct inode *inode = page->mapping->host;
1611 int error = shmem_getpage(inode, page->index, &page, SGP_CACHE, NULL);
1612 unlock_page(page);
1613 return error;
1614}
1615
1616static int
1617shmem_write_begin(struct file *file, struct address_space *mapping,
1618 loff_t pos, unsigned len, unsigned flags,
1619 struct page **pagep, void **fsdata)
1620{
1621 struct inode *inode = mapping->host;
1622 pgoff_t index = pos >> PAGE_CACHE_SHIFT;
1623 *pagep = NULL;
1624 return shmem_getpage(inode, index, pagep, SGP_WRITE, NULL);
1625}
1626
1627static int
1628shmem_write_end(struct file *file, struct address_space *mapping,
1629 loff_t pos, unsigned len, unsigned copied,
1630 struct page *page, void *fsdata)
1631{
1632 struct inode *inode = mapping->host;
1633
1634 if (pos + copied > inode->i_size)
1635 i_size_write(inode, pos + copied);
1636
1637 set_page_dirty(page);
1638 unlock_page(page);
1639 page_cache_release(page);
1640
1641 return copied;
1642}
1643
1644static void do_shmem_file_read(struct file *filp, loff_t *ppos, read_descriptor_t *desc, read_actor_t actor)
1645{
1646 struct inode *inode = filp->f_path.dentry->d_inode;
1647 struct address_space *mapping = inode->i_mapping;
1648 unsigned long index, offset;
1649 enum sgp_type sgp = SGP_READ;
1650
1651
1652
1653
1654
1655
1656 if (segment_eq(get_fs(), KERNEL_DS))
1657 sgp = SGP_DIRTY;
1658
1659 index = *ppos >> PAGE_CACHE_SHIFT;
1660 offset = *ppos & ~PAGE_CACHE_MASK;
1661
1662 for (;;) {
1663 struct page *page = NULL;
1664 unsigned long end_index, nr, ret;
1665 loff_t i_size = i_size_read(inode);
1666
1667 end_index = i_size >> PAGE_CACHE_SHIFT;
1668 if (index > end_index)
1669 break;
1670 if (index == end_index) {
1671 nr = i_size & ~PAGE_CACHE_MASK;
1672 if (nr <= offset)
1673 break;
1674 }
1675
1676 desc->error = shmem_getpage(inode, index, &page, sgp, NULL);
1677 if (desc->error) {
1678 if (desc->error == -EINVAL)
1679 desc->error = 0;
1680 break;
1681 }
1682 if (page)
1683 unlock_page(page);
1684
1685
1686
1687
1688
1689 nr = PAGE_CACHE_SIZE;
1690 i_size = i_size_read(inode);
1691 end_index = i_size >> PAGE_CACHE_SHIFT;
1692 if (index == end_index) {
1693 nr = i_size & ~PAGE_CACHE_MASK;
1694 if (nr <= offset) {
1695 if (page)
1696 page_cache_release(page);
1697 break;
1698 }
1699 }
1700 nr -= offset;
1701
1702 if (page) {
1703
1704
1705
1706
1707
1708 if (mapping_writably_mapped(mapping))
1709 flush_dcache_page(page);
1710
1711
1712
1713 if (!offset)
1714 mark_page_accessed(page);
1715 } else {
1716 page = ZERO_PAGE(0);
1717 page_cache_get(page);
1718 }
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730 ret = actor(desc, page, offset, nr);
1731 offset += ret;
1732 index += offset >> PAGE_CACHE_SHIFT;
1733 offset &= ~PAGE_CACHE_MASK;
1734
1735 page_cache_release(page);
1736 if (ret != nr || !desc->count)
1737 break;
1738
1739 cond_resched();
1740 }
1741
1742 *ppos = ((loff_t) index << PAGE_CACHE_SHIFT) + offset;
1743 file_accessed(filp);
1744}
1745
1746static ssize_t shmem_file_aio_read(struct kiocb *iocb,
1747 const struct iovec *iov, unsigned long nr_segs, loff_t pos)
1748{
1749 struct file *filp = iocb->ki_filp;
1750 ssize_t retval;
1751 unsigned long seg;
1752 size_t count;
1753 loff_t *ppos = &iocb->ki_pos;
1754
1755 retval = generic_segment_checks(iov, &nr_segs, &count, VERIFY_WRITE);
1756 if (retval)
1757 return retval;
1758
1759 for (seg = 0; seg < nr_segs; seg++) {
1760 read_descriptor_t desc;
1761
1762 desc.written = 0;
1763 desc.arg.buf = iov[seg].iov_base;
1764 desc.count = iov[seg].iov_len;
1765 if (desc.count == 0)
1766 continue;
1767 desc.error = 0;
1768 do_shmem_file_read(filp, ppos, &desc, file_read_actor);
1769 retval += desc.written;
1770 if (desc.error) {
1771 retval = retval ?: desc.error;
1772 break;
1773 }
1774 if (desc.count > 0)
1775 break;
1776 }
1777 return retval;
1778}
1779
1780static int shmem_statfs(struct dentry *dentry, struct kstatfs *buf)
1781{
1782 struct shmem_sb_info *sbinfo = SHMEM_SB(dentry->d_sb);
1783
1784 buf->f_type = TMPFS_MAGIC;
1785 buf->f_bsize = PAGE_CACHE_SIZE;
1786 buf->f_namelen = NAME_MAX;
1787 spin_lock(&sbinfo->stat_lock);
1788 if (sbinfo->max_blocks) {
1789 buf->f_blocks = sbinfo->max_blocks;
1790 buf->f_bavail = buf->f_bfree = sbinfo->free_blocks;
1791 }
1792 if (sbinfo->max_inodes) {
1793 buf->f_files = sbinfo->max_inodes;
1794 buf->f_ffree = sbinfo->free_inodes;
1795 }
1796
1797 spin_unlock(&sbinfo->stat_lock);
1798 return 0;
1799}
1800
1801
1802
1803
1804static int
1805shmem_mknod(struct inode *dir, struct dentry *dentry, int mode, dev_t dev)
1806{
1807 struct inode *inode;
1808 int error = -ENOSPC;
1809
1810 inode = shmem_get_inode(dir->i_sb, mode, dev, VM_NORESERVE);
1811 if (inode) {
1812 error = security_inode_init_security(inode, dir, NULL, NULL,
1813 NULL);
1814 if (error) {
1815 if (error != -EOPNOTSUPP) {
1816 iput(inode);
1817 return error;
1818 }
1819 }
1820 error = shmem_acl_init(inode, dir);
1821 if (error) {
1822 iput(inode);
1823 return error;
1824 }
1825 if (dir->i_mode & S_ISGID) {
1826 inode->i_gid = dir->i_gid;
1827 if (S_ISDIR(mode))
1828 inode->i_mode |= S_ISGID;
1829 }
1830 dir->i_size += BOGO_DIRENT_SIZE;
1831 dir->i_ctime = dir->i_mtime = CURRENT_TIME;
1832 d_instantiate(dentry, inode);
1833 dget(dentry);
1834 }
1835 return error;
1836}
1837
1838static int shmem_mkdir(struct inode *dir, struct dentry *dentry, int mode)
1839{
1840 int error;
1841
1842 if ((error = shmem_mknod(dir, dentry, mode | S_IFDIR, 0)))
1843 return error;
1844 inc_nlink(dir);
1845 return 0;
1846}
1847
1848static int shmem_create(struct inode *dir, struct dentry *dentry, int mode,
1849 struct nameidata *nd)
1850{
1851 return shmem_mknod(dir, dentry, mode | S_IFREG, 0);
1852}
1853
1854
1855
1856
1857static int shmem_link(struct dentry *old_dentry, struct inode *dir, struct dentry *dentry)
1858{
1859 struct inode *inode = old_dentry->d_inode;
1860 int ret;
1861
1862
1863
1864
1865
1866
1867 ret = shmem_reserve_inode(inode->i_sb);
1868 if (ret)
1869 goto out;
1870
1871 dir->i_size += BOGO_DIRENT_SIZE;
1872 inode->i_ctime = dir->i_ctime = dir->i_mtime = CURRENT_TIME;
1873 inc_nlink(inode);
1874 atomic_inc(&inode->i_count);
1875 dget(dentry);
1876 d_instantiate(dentry, inode);
1877out:
1878 return ret;
1879}
1880
1881static int shmem_unlink(struct inode *dir, struct dentry *dentry)
1882{
1883 struct inode *inode = dentry->d_inode;
1884
1885 if (inode->i_nlink > 1 && !S_ISDIR(inode->i_mode))
1886 shmem_free_inode(inode->i_sb);
1887
1888 dir->i_size -= BOGO_DIRENT_SIZE;
1889 inode->i_ctime = dir->i_ctime = dir->i_mtime = CURRENT_TIME;
1890 drop_nlink(inode);
1891 dput(dentry);
1892 return 0;
1893}
1894
1895static int shmem_rmdir(struct inode *dir, struct dentry *dentry)
1896{
1897 if (!simple_empty(dentry))
1898 return -ENOTEMPTY;
1899
1900 drop_nlink(dentry->d_inode);
1901 drop_nlink(dir);
1902 return shmem_unlink(dir, dentry);
1903}
1904
1905
1906
1907
1908
1909
1910
1911static int shmem_rename(struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry)
1912{
1913 struct inode *inode = old_dentry->d_inode;
1914 int they_are_dirs = S_ISDIR(inode->i_mode);
1915
1916 if (!simple_empty(new_dentry))
1917 return -ENOTEMPTY;
1918
1919 if (new_dentry->d_inode) {
1920 (void) shmem_unlink(new_dir, new_dentry);
1921 if (they_are_dirs)
1922 drop_nlink(old_dir);
1923 } else if (they_are_dirs) {
1924 drop_nlink(old_dir);
1925 inc_nlink(new_dir);
1926 }
1927
1928 old_dir->i_size -= BOGO_DIRENT_SIZE;
1929 new_dir->i_size += BOGO_DIRENT_SIZE;
1930 old_dir->i_ctime = old_dir->i_mtime =
1931 new_dir->i_ctime = new_dir->i_mtime =
1932 inode->i_ctime = CURRENT_TIME;
1933 return 0;
1934}
1935
1936static int shmem_symlink(struct inode *dir, struct dentry *dentry, const char *symname)
1937{
1938 int error;
1939 int len;
1940 struct inode *inode;
1941 struct page *page = NULL;
1942 char *kaddr;
1943 struct shmem_inode_info *info;
1944
1945 len = strlen(symname) + 1;
1946 if (len > PAGE_CACHE_SIZE)
1947 return -ENAMETOOLONG;
1948
1949 inode = shmem_get_inode(dir->i_sb, S_IFLNK|S_IRWXUGO, 0, VM_NORESERVE);
1950 if (!inode)
1951 return -ENOSPC;
1952
1953 error = security_inode_init_security(inode, dir, NULL, NULL,
1954 NULL);
1955 if (error) {
1956 if (error != -EOPNOTSUPP) {
1957 iput(inode);
1958 return error;
1959 }
1960 error = 0;
1961 }
1962
1963 info = SHMEM_I(inode);
1964 inode->i_size = len-1;
1965 if (len <= (char *)inode - (char *)info) {
1966
1967 memcpy(info, symname, len);
1968 inode->i_op = &shmem_symlink_inline_operations;
1969 } else {
1970 error = shmem_getpage(inode, 0, &page, SGP_WRITE, NULL);
1971 if (error) {
1972 iput(inode);
1973 return error;
1974 }
1975 inode->i_mapping->a_ops = &shmem_aops;
1976 inode->i_op = &shmem_symlink_inode_operations;
1977 kaddr = kmap_atomic(page, KM_USER0);
1978 memcpy(kaddr, symname, len);
1979 kunmap_atomic(kaddr, KM_USER0);
1980 set_page_dirty(page);
1981 unlock_page(page);
1982 page_cache_release(page);
1983 }
1984 if (dir->i_mode & S_ISGID)
1985 inode->i_gid = dir->i_gid;
1986 dir->i_size += BOGO_DIRENT_SIZE;
1987 dir->i_ctime = dir->i_mtime = CURRENT_TIME;
1988 d_instantiate(dentry, inode);
1989 dget(dentry);
1990 return 0;
1991}
1992
1993static void *shmem_follow_link_inline(struct dentry *dentry, struct nameidata *nd)
1994{
1995 nd_set_link(nd, (char *)SHMEM_I(dentry->d_inode));
1996 return NULL;
1997}
1998
1999static void *shmem_follow_link(struct dentry *dentry, struct nameidata *nd)
2000{
2001 struct page *page = NULL;
2002 int res = shmem_getpage(dentry->d_inode, 0, &page, SGP_READ, NULL);
2003 nd_set_link(nd, res ? ERR_PTR(res) : kmap(page));
2004 if (page)
2005 unlock_page(page);
2006 return page;
2007}
2008
2009static void shmem_put_link(struct dentry *dentry, struct nameidata *nd, void *cookie)
2010{
2011 if (!IS_ERR(nd_get_link(nd))) {
2012 struct page *page = cookie;
2013 kunmap(page);
2014 mark_page_accessed(page);
2015 page_cache_release(page);
2016 }
2017}
2018
2019static const struct inode_operations shmem_symlink_inline_operations = {
2020 .readlink = generic_readlink,
2021 .follow_link = shmem_follow_link_inline,
2022};
2023
2024static const struct inode_operations shmem_symlink_inode_operations = {
2025 .truncate = shmem_truncate,
2026 .readlink = generic_readlink,
2027 .follow_link = shmem_follow_link,
2028 .put_link = shmem_put_link,
2029};
2030
2031#ifdef CONFIG_TMPFS_POSIX_ACL
2032
2033
2034
2035
2036
2037
2038
2039static size_t shmem_xattr_security_list(struct inode *inode, char *list,
2040 size_t list_len, const char *name,
2041 size_t name_len)
2042{
2043 return security_inode_listsecurity(inode, list, list_len);
2044}
2045
2046static int shmem_xattr_security_get(struct inode *inode, const char *name,
2047 void *buffer, size_t size)
2048{
2049 if (strcmp(name, "") == 0)
2050 return -EINVAL;
2051 return xattr_getsecurity(inode, name, buffer, size);
2052}
2053
2054static int shmem_xattr_security_set(struct inode *inode, const char *name,
2055 const void *value, size_t size, int flags)
2056{
2057 if (strcmp(name, "") == 0)
2058 return -EINVAL;
2059 return security_inode_setsecurity(inode, name, value, size, flags);
2060}
2061
2062static struct xattr_handler shmem_xattr_security_handler = {
2063 .prefix = XATTR_SECURITY_PREFIX,
2064 .list = shmem_xattr_security_list,
2065 .get = shmem_xattr_security_get,
2066 .set = shmem_xattr_security_set,
2067};
2068
2069static struct xattr_handler *shmem_xattr_handlers[] = {
2070 &shmem_xattr_acl_access_handler,
2071 &shmem_xattr_acl_default_handler,
2072 &shmem_xattr_security_handler,
2073 NULL
2074};
2075#endif
2076
2077static struct dentry *shmem_get_parent(struct dentry *child)
2078{
2079 return ERR_PTR(-ESTALE);
2080}
2081
2082static int shmem_match(struct inode *ino, void *vfh)
2083{
2084 __u32 *fh = vfh;
2085 __u64 inum = fh[2];
2086 inum = (inum << 32) | fh[1];
2087 return ino->i_ino == inum && fh[0] == ino->i_generation;
2088}
2089
2090static struct dentry *shmem_fh_to_dentry(struct super_block *sb,
2091 struct fid *fid, int fh_len, int fh_type)
2092{
2093 struct inode *inode;
2094 struct dentry *dentry = NULL;
2095 u64 inum = fid->raw[2];
2096 inum = (inum << 32) | fid->raw[1];
2097
2098 if (fh_len < 3)
2099 return NULL;
2100
2101 inode = ilookup5(sb, (unsigned long)(inum + fid->raw[0]),
2102 shmem_match, fid->raw);
2103 if (inode) {
2104 dentry = d_find_alias(inode);
2105 iput(inode);
2106 }
2107
2108 return dentry;
2109}
2110
2111static int shmem_encode_fh(struct dentry *dentry, __u32 *fh, int *len,
2112 int connectable)
2113{
2114 struct inode *inode = dentry->d_inode;
2115
2116 if (*len < 3)
2117 return 255;
2118
2119 if (hlist_unhashed(&inode->i_hash)) {
2120
2121
2122
2123
2124
2125 static DEFINE_SPINLOCK(lock);
2126 spin_lock(&lock);
2127 if (hlist_unhashed(&inode->i_hash))
2128 __insert_inode_hash(inode,
2129 inode->i_ino + inode->i_generation);
2130 spin_unlock(&lock);
2131 }
2132
2133 fh[0] = inode->i_generation;
2134 fh[1] = inode->i_ino;
2135 fh[2] = ((__u64)inode->i_ino) >> 32;
2136
2137 *len = 3;
2138 return 1;
2139}
2140
2141static const struct export_operations shmem_export_ops = {
2142 .get_parent = shmem_get_parent,
2143 .encode_fh = shmem_encode_fh,
2144 .fh_to_dentry = shmem_fh_to_dentry,
2145};
2146
2147static int shmem_parse_options(char *options, struct shmem_sb_info *sbinfo,
2148 bool remount)
2149{
2150 char *this_char, *value, *rest;
2151
2152 while (options != NULL) {
2153 this_char = options;
2154 for (;;) {
2155
2156
2157
2158
2159
2160 options = strchr(options, ',');
2161 if (options == NULL)
2162 break;
2163 options++;
2164 if (!isdigit(*options)) {
2165 options[-1] = '\0';
2166 break;
2167 }
2168 }
2169 if (!*this_char)
2170 continue;
2171 if ((value = strchr(this_char,'=')) != NULL) {
2172 *value++ = 0;
2173 } else {
2174 printk(KERN_ERR
2175 "tmpfs: No value for mount option '%s'\n",
2176 this_char);
2177 return 1;
2178 }
2179
2180 if (!strcmp(this_char,"size")) {
2181 unsigned long long size;
2182 size = memparse(value,&rest);
2183 if (*rest == '%') {
2184 size <<= PAGE_SHIFT;
2185 size *= totalram_pages;
2186 do_div(size, 100);
2187 rest++;
2188 }
2189 if (*rest)
2190 goto bad_val;
2191 sbinfo->max_blocks =
2192 DIV_ROUND_UP(size, PAGE_CACHE_SIZE);
2193 } else if (!strcmp(this_char,"nr_blocks")) {
2194 sbinfo->max_blocks = memparse(value, &rest);
2195 if (*rest)
2196 goto bad_val;
2197 } else if (!strcmp(this_char,"nr_inodes")) {
2198 sbinfo->max_inodes = memparse(value, &rest);
2199 if (*rest)
2200 goto bad_val;
2201 } else if (!strcmp(this_char,"mode")) {
2202 if (remount)
2203 continue;
2204 sbinfo->mode = simple_strtoul(value, &rest, 8) & 07777;
2205 if (*rest)
2206 goto bad_val;
2207 } else if (!strcmp(this_char,"uid")) {
2208 if (remount)
2209 continue;
2210 sbinfo->uid = simple_strtoul(value, &rest, 0);
2211 if (*rest)
2212 goto bad_val;
2213 } else if (!strcmp(this_char,"gid")) {
2214 if (remount)
2215 continue;
2216 sbinfo->gid = simple_strtoul(value, &rest, 0);
2217 if (*rest)
2218 goto bad_val;
2219 } else if (!strcmp(this_char,"mpol")) {
2220 if (mpol_parse_str(value, &sbinfo->mpol, 1))
2221 goto bad_val;
2222 } else {
2223 printk(KERN_ERR "tmpfs: Bad mount option %s\n",
2224 this_char);
2225 return 1;
2226 }
2227 }
2228 return 0;
2229
2230bad_val:
2231 printk(KERN_ERR "tmpfs: Bad value '%s' for mount option '%s'\n",
2232 value, this_char);
2233 return 1;
2234
2235}
2236
2237static int shmem_remount_fs(struct super_block *sb, int *flags, char *data)
2238{
2239 struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
2240 struct shmem_sb_info config = *sbinfo;
2241 unsigned long blocks;
2242 unsigned long inodes;
2243 int error = -EINVAL;
2244
2245 if (shmem_parse_options(data, &config, true))
2246 return error;
2247
2248 spin_lock(&sbinfo->stat_lock);
2249 blocks = sbinfo->max_blocks - sbinfo->free_blocks;
2250 inodes = sbinfo->max_inodes - sbinfo->free_inodes;
2251 if (config.max_blocks < blocks)
2252 goto out;
2253 if (config.max_inodes < inodes)
2254 goto out;
2255
2256
2257
2258
2259
2260
2261 if (config.max_blocks && !sbinfo->max_blocks)
2262 goto out;
2263 if (config.max_inodes && !sbinfo->max_inodes)
2264 goto out;
2265
2266 error = 0;
2267 sbinfo->max_blocks = config.max_blocks;
2268 sbinfo->free_blocks = config.max_blocks - blocks;
2269 sbinfo->max_inodes = config.max_inodes;
2270 sbinfo->free_inodes = config.max_inodes - inodes;
2271
2272 mpol_put(sbinfo->mpol);
2273 sbinfo->mpol = config.mpol;
2274out:
2275 spin_unlock(&sbinfo->stat_lock);
2276 return error;
2277}
2278
2279static int shmem_show_options(struct seq_file *seq, struct vfsmount *vfs)
2280{
2281 struct shmem_sb_info *sbinfo = SHMEM_SB(vfs->mnt_sb);
2282
2283 if (sbinfo->max_blocks != shmem_default_max_blocks())
2284 seq_printf(seq, ",size=%luk",
2285 sbinfo->max_blocks << (PAGE_CACHE_SHIFT - 10));
2286 if (sbinfo->max_inodes != shmem_default_max_inodes())
2287 seq_printf(seq, ",nr_inodes=%lu", sbinfo->max_inodes);
2288 if (sbinfo->mode != (S_IRWXUGO | S_ISVTX))
2289 seq_printf(seq, ",mode=%03o", sbinfo->mode);
2290 if (sbinfo->uid != 0)
2291 seq_printf(seq, ",uid=%u", sbinfo->uid);
2292 if (sbinfo->gid != 0)
2293 seq_printf(seq, ",gid=%u", sbinfo->gid);
2294 shmem_show_mpol(seq, sbinfo->mpol);
2295 return 0;
2296}
2297#endif
2298
2299static void shmem_put_super(struct super_block *sb)
2300{
2301 kfree(sb->s_fs_info);
2302 sb->s_fs_info = NULL;
2303}
2304
2305int shmem_fill_super(struct super_block *sb, void *data, int silent)
2306{
2307 struct inode *inode;
2308 struct dentry *root;
2309 struct shmem_sb_info *sbinfo;
2310 int err = -ENOMEM;
2311
2312
2313 sbinfo = kzalloc(max((int)sizeof(struct shmem_sb_info),
2314 L1_CACHE_BYTES), GFP_KERNEL);
2315 if (!sbinfo)
2316 return -ENOMEM;
2317
2318 sbinfo->mode = S_IRWXUGO | S_ISVTX;
2319 sbinfo->uid = current_fsuid();
2320 sbinfo->gid = current_fsgid();
2321 sb->s_fs_info = sbinfo;
2322
2323#ifdef CONFIG_TMPFS
2324
2325
2326
2327
2328
2329 if (!(sb->s_flags & MS_NOUSER)) {
2330 sbinfo->max_blocks = shmem_default_max_blocks();
2331 sbinfo->max_inodes = shmem_default_max_inodes();
2332 if (shmem_parse_options(data, sbinfo, false)) {
2333 err = -EINVAL;
2334 goto failed;
2335 }
2336 }
2337 sb->s_export_op = &shmem_export_ops;
2338#else
2339 sb->s_flags |= MS_NOUSER;
2340#endif
2341
2342 spin_lock_init(&sbinfo->stat_lock);
2343 sbinfo->free_blocks = sbinfo->max_blocks;
2344 sbinfo->free_inodes = sbinfo->max_inodes;
2345
2346 sb->s_maxbytes = SHMEM_MAX_BYTES;
2347 sb->s_blocksize = PAGE_CACHE_SIZE;
2348 sb->s_blocksize_bits = PAGE_CACHE_SHIFT;
2349 sb->s_magic = TMPFS_MAGIC;
2350 sb->s_op = &shmem_ops;
2351 sb->s_time_gran = 1;
2352#ifdef CONFIG_TMPFS_POSIX_ACL
2353 sb->s_xattr = shmem_xattr_handlers;
2354 sb->s_flags |= MS_POSIXACL;
2355#endif
2356
2357 inode = shmem_get_inode(sb, S_IFDIR | sbinfo->mode, 0, VM_NORESERVE);
2358 if (!inode)
2359 goto failed;
2360 inode->i_uid = sbinfo->uid;
2361 inode->i_gid = sbinfo->gid;
2362 root = d_alloc_root(inode);
2363 if (!root)
2364 goto failed_iput;
2365 sb->s_root = root;
2366 return 0;
2367
2368failed_iput:
2369 iput(inode);
2370failed:
2371 shmem_put_super(sb);
2372 return err;
2373}
2374
2375static struct kmem_cache *shmem_inode_cachep;
2376
2377static struct inode *shmem_alloc_inode(struct super_block *sb)
2378{
2379 struct shmem_inode_info *p;
2380 p = (struct shmem_inode_info *)kmem_cache_alloc(shmem_inode_cachep, GFP_KERNEL);
2381 if (!p)
2382 return NULL;
2383 return &p->vfs_inode;
2384}
2385
2386static void shmem_destroy_inode(struct inode *inode)
2387{
2388 if ((inode->i_mode & S_IFMT) == S_IFREG) {
2389
2390 mpol_free_shared_policy(&SHMEM_I(inode)->policy);
2391 }
2392 kmem_cache_free(shmem_inode_cachep, SHMEM_I(inode));
2393}
2394
2395static void init_once(void *foo)
2396{
2397 struct shmem_inode_info *p = (struct shmem_inode_info *) foo;
2398
2399 inode_init_once(&p->vfs_inode);
2400}
2401
2402static int init_inodecache(void)
2403{
2404 shmem_inode_cachep = kmem_cache_create("shmem_inode_cache",
2405 sizeof(struct shmem_inode_info),
2406 0, SLAB_PANIC, init_once);
2407 return 0;
2408}
2409
2410static void destroy_inodecache(void)
2411{
2412 kmem_cache_destroy(shmem_inode_cachep);
2413}
2414
2415static const struct address_space_operations shmem_aops = {
2416 .writepage = shmem_writepage,
2417 .set_page_dirty = __set_page_dirty_no_writeback,
2418#ifdef CONFIG_TMPFS
2419 .readpage = shmem_readpage,
2420 .write_begin = shmem_write_begin,
2421 .write_end = shmem_write_end,
2422#endif
2423 .migratepage = migrate_page,
2424 .error_remove_page = generic_error_remove_page,
2425};
2426
2427static const struct file_operations shmem_file_operations = {
2428 .mmap = shmem_mmap,
2429#ifdef CONFIG_TMPFS
2430 .llseek = generic_file_llseek,
2431 .read = do_sync_read,
2432 .write = do_sync_write,
2433 .aio_read = shmem_file_aio_read,
2434 .aio_write = generic_file_aio_write,
2435 .fsync = simple_sync_file,
2436 .splice_read = generic_file_splice_read,
2437 .splice_write = generic_file_splice_write,
2438#endif
2439};
2440
2441static const struct inode_operations shmem_inode_operations = {
2442 .truncate = shmem_truncate,
2443 .setattr = shmem_notify_change,
2444 .truncate_range = shmem_truncate_range,
2445#ifdef CONFIG_TMPFS_POSIX_ACL
2446 .setxattr = generic_setxattr,
2447 .getxattr = generic_getxattr,
2448 .listxattr = generic_listxattr,
2449 .removexattr = generic_removexattr,
2450 .check_acl = shmem_check_acl,
2451#endif
2452
2453};
2454
2455static const struct inode_operations shmem_dir_inode_operations = {
2456#ifdef CONFIG_TMPFS
2457 .create = shmem_create,
2458 .lookup = simple_lookup,
2459 .link = shmem_link,
2460 .unlink = shmem_unlink,
2461 .symlink = shmem_symlink,
2462 .mkdir = shmem_mkdir,
2463 .rmdir = shmem_rmdir,
2464 .mknod = shmem_mknod,
2465 .rename = shmem_rename,
2466#endif
2467#ifdef CONFIG_TMPFS_POSIX_ACL
2468 .setattr = shmem_notify_change,
2469 .setxattr = generic_setxattr,
2470 .getxattr = generic_getxattr,
2471 .listxattr = generic_listxattr,
2472 .removexattr = generic_removexattr,
2473 .check_acl = shmem_check_acl,
2474#endif
2475};
2476
2477static const struct inode_operations shmem_special_inode_operations = {
2478#ifdef CONFIG_TMPFS_POSIX_ACL
2479 .setattr = shmem_notify_change,
2480 .setxattr = generic_setxattr,
2481 .getxattr = generic_getxattr,
2482 .listxattr = generic_listxattr,
2483 .removexattr = generic_removexattr,
2484 .check_acl = shmem_check_acl,
2485#endif
2486};
2487
2488static const struct super_operations shmem_ops = {
2489 .alloc_inode = shmem_alloc_inode,
2490 .destroy_inode = shmem_destroy_inode,
2491#ifdef CONFIG_TMPFS
2492 .statfs = shmem_statfs,
2493 .remount_fs = shmem_remount_fs,
2494 .show_options = shmem_show_options,
2495#endif
2496 .delete_inode = shmem_delete_inode,
2497 .drop_inode = generic_delete_inode,
2498 .put_super = shmem_put_super,
2499};
2500
2501static const struct vm_operations_struct shmem_vm_ops = {
2502 .fault = shmem_fault,
2503#ifdef CONFIG_NUMA
2504 .set_policy = shmem_set_policy,
2505 .get_policy = shmem_get_policy,
2506#endif
2507};
2508
2509
2510static int shmem_get_sb(struct file_system_type *fs_type,
2511 int flags, const char *dev_name, void *data, struct vfsmount *mnt)
2512{
2513 return get_sb_nodev(fs_type, flags, data, shmem_fill_super, mnt);
2514}
2515
2516static struct file_system_type tmpfs_fs_type = {
2517 .owner = THIS_MODULE,
2518 .name = "tmpfs",
2519 .get_sb = shmem_get_sb,
2520 .kill_sb = kill_litter_super,
2521};
2522
2523int __init init_tmpfs(void)
2524{
2525 int error;
2526
2527 error = bdi_init(&shmem_backing_dev_info);
2528 if (error)
2529 goto out4;
2530
2531 error = init_inodecache();
2532 if (error)
2533 goto out3;
2534
2535 error = register_filesystem(&tmpfs_fs_type);
2536 if (error) {
2537 printk(KERN_ERR "Could not register tmpfs\n");
2538 goto out2;
2539 }
2540
2541 shm_mnt = vfs_kern_mount(&tmpfs_fs_type, MS_NOUSER,
2542 tmpfs_fs_type.name, NULL);
2543 if (IS_ERR(shm_mnt)) {
2544 error = PTR_ERR(shm_mnt);
2545 printk(KERN_ERR "Could not kern_mount tmpfs\n");
2546 goto out1;
2547 }
2548 return 0;
2549
2550out1:
2551 unregister_filesystem(&tmpfs_fs_type);
2552out2:
2553 destroy_inodecache();
2554out3:
2555 bdi_destroy(&shmem_backing_dev_info);
2556out4:
2557 shm_mnt = ERR_PTR(error);
2558 return error;
2559}
2560
2561#else
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572#include <linux/ramfs.h>
2573
2574static struct file_system_type tmpfs_fs_type = {
2575 .name = "tmpfs",
2576 .get_sb = ramfs_get_sb,
2577 .kill_sb = kill_litter_super,
2578};
2579
2580int __init init_tmpfs(void)
2581{
2582 BUG_ON(register_filesystem(&tmpfs_fs_type) != 0);
2583
2584 shm_mnt = kern_mount(&tmpfs_fs_type);
2585 BUG_ON(IS_ERR(shm_mnt));
2586
2587 return 0;
2588}
2589
2590int shmem_unuse(swp_entry_t entry, struct page *page)
2591{
2592 return 0;
2593}
2594
2595int shmem_lock(struct file *file, int lock, struct user_struct *user)
2596{
2597 return 0;
2598}
2599
2600#define shmem_vm_ops generic_file_vm_ops
2601#define shmem_file_operations ramfs_file_operations
2602#define shmem_get_inode(sb, mode, dev, flags) ramfs_get_inode(sb, mode, dev)
2603#define shmem_acct_size(flags, size) 0
2604#define shmem_unacct_size(flags, size) do {} while (0)
2605#define SHMEM_MAX_BYTES MAX_LFS_FILESIZE
2606
2607#endif
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617struct file *shmem_file_setup(const char *name, loff_t size, unsigned long flags)
2618{
2619 int error;
2620 struct file *file;
2621 struct inode *inode;
2622 struct dentry *dentry, *root;
2623 struct qstr this;
2624
2625 if (IS_ERR(shm_mnt))
2626 return (void *)shm_mnt;
2627
2628 if (size < 0 || size > SHMEM_MAX_BYTES)
2629 return ERR_PTR(-EINVAL);
2630
2631 if (shmem_acct_size(flags, size))
2632 return ERR_PTR(-ENOMEM);
2633
2634 error = -ENOMEM;
2635 this.name = name;
2636 this.len = strlen(name);
2637 this.hash = 0;
2638 root = shm_mnt->mnt_root;
2639 dentry = d_alloc(root, &this);
2640 if (!dentry)
2641 goto put_memory;
2642
2643 error = -ENFILE;
2644 file = get_empty_filp();
2645 if (!file)
2646 goto put_dentry;
2647
2648 error = -ENOSPC;
2649 inode = shmem_get_inode(root->d_sb, S_IFREG | S_IRWXUGO, 0, flags);
2650 if (!inode)
2651 goto close_file;
2652
2653 d_instantiate(dentry, inode);
2654 inode->i_size = size;
2655 inode->i_nlink = 0;
2656 init_file(file, shm_mnt, dentry, FMODE_WRITE | FMODE_READ,
2657 &shmem_file_operations);
2658
2659#ifndef CONFIG_MMU
2660 error = ramfs_nommu_expand_for_mapping(inode, size);
2661 if (error)
2662 goto close_file;
2663#endif
2664 ima_counts_get(file);
2665 return file;
2666
2667close_file:
2668 put_filp(file);
2669put_dentry:
2670 dput(dentry);
2671put_memory:
2672 shmem_unacct_size(flags, size);
2673 return ERR_PTR(error);
2674}
2675EXPORT_SYMBOL_GPL(shmem_file_setup);
2676
2677
2678
2679
2680
2681int shmem_zero_setup(struct vm_area_struct *vma)
2682{
2683 struct file *file;
2684 loff_t size = vma->vm_end - vma->vm_start;
2685
2686 file = shmem_file_setup("dev/zero", size, vma->vm_flags);
2687 if (IS_ERR(file))
2688 return PTR_ERR(file);
2689
2690 if (vma->vm_file)
2691 fput(vma->vm_file);
2692 vma->vm_file = file;
2693 vma->vm_ops = &shmem_vm_ops;
2694 return 0;
2695}
2696