1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23#include <linux/fs.h>
24#include <linux/init.h>
25#include <linux/vfs.h>
26#include <linux/mount.h>
27#include <linux/pagemap.h>
28#include <linux/file.h>
29#include <linux/mm.h>
30#include <linux/module.h>
31#include <linux/percpu_counter.h>
32#include <linux/swap.h>
33
34static struct vfsmount *shm_mnt;
35
36#ifdef CONFIG_SHMEM
37
38
39
40
41
42
43#include <linux/xattr.h>
44#include <linux/exportfs.h>
45#include <linux/posix_acl.h>
46#include <linux/generic_acl.h>
47#include <linux/mman.h>
48#include <linux/string.h>
49#include <linux/slab.h>
50#include <linux/backing-dev.h>
51#include <linux/shmem_fs.h>
52#include <linux/writeback.h>
53#include <linux/blkdev.h>
54#include <linux/security.h>
55#include <linux/swapops.h>
56#include <linux/mempolicy.h>
57#include <linux/namei.h>
58#include <linux/ctype.h>
59#include <linux/migrate.h>
60#include <linux/highmem.h>
61#include <linux/seq_file.h>
62#include <linux/magic.h>
63
64#include <asm/uaccess.h>
65#include <asm/div64.h>
66#include <asm/pgtable.h>
67
68
69
70
71
72
73
74
75
76
77
78
79
80#define ENTRIES_PER_PAGE (PAGE_CACHE_SIZE/sizeof(unsigned long))
81#define ENTRIES_PER_PAGEPAGE ((unsigned long long)ENTRIES_PER_PAGE*ENTRIES_PER_PAGE)
82
83#define SHMSWP_MAX_INDEX (SHMEM_NR_DIRECT + (ENTRIES_PER_PAGEPAGE/2) * (ENTRIES_PER_PAGE+1))
84#define SHMSWP_MAX_BYTES (SHMSWP_MAX_INDEX << PAGE_CACHE_SHIFT)
85
86#define SHMEM_MAX_BYTES min_t(unsigned long long, SHMSWP_MAX_BYTES, MAX_LFS_FILESIZE)
87#define SHMEM_MAX_INDEX ((unsigned long)((SHMEM_MAX_BYTES+1) >> PAGE_CACHE_SHIFT))
88
89#define BLOCKS_PER_PAGE (PAGE_CACHE_SIZE/512)
90#define VM_ACCT(size) (PAGE_CACHE_ALIGN(size) >> PAGE_SHIFT)
91
92
93#define SHMEM_PAGEIN VM_READ
94#define SHMEM_TRUNCATE VM_WRITE
95
96
97#define LATENCY_LIMIT 64
98
99
100#define BOGO_DIRENT_SIZE 20
101
102
103enum sgp_type {
104 SGP_READ,
105 SGP_CACHE,
106 SGP_DIRTY,
107 SGP_WRITE,
108};
109
110#ifdef CONFIG_TMPFS
111static unsigned long shmem_default_max_blocks(void)
112{
113 return totalram_pages / 2;
114}
115
116static unsigned long shmem_default_max_inodes(void)
117{
118 return min(totalram_pages - totalhigh_pages, totalram_pages / 2);
119}
120#endif
121
122static int shmem_getpage(struct inode *inode, unsigned long idx,
123 struct page **pagep, enum sgp_type sgp, int *type);
124
125static inline struct page *shmem_dir_alloc(gfp_t gfp_mask)
126{
127
128
129
130
131
132
133
134 return alloc_pages((gfp_mask & ~GFP_MOVABLE_MASK) | __GFP_ZERO,
135 PAGE_CACHE_SHIFT-PAGE_SHIFT);
136}
137
138static inline void shmem_dir_free(struct page *page)
139{
140 __free_pages(page, PAGE_CACHE_SHIFT-PAGE_SHIFT);
141}
142
143static struct page **shmem_dir_map(struct page *page)
144{
145 return (struct page **)kmap_atomic(page, KM_USER0);
146}
147
148static inline void shmem_dir_unmap(struct page **dir)
149{
150 kunmap_atomic(dir, KM_USER0);
151}
152
153static swp_entry_t *shmem_swp_map(struct page *page)
154{
155 return (swp_entry_t *)kmap_atomic(page, KM_USER1);
156}
157
158static inline void shmem_swp_balance_unmap(void)
159{
160
161
162
163
164
165
166
167 (void) kmap_atomic(ZERO_PAGE(0), KM_USER1);
168}
169
170static inline void shmem_swp_unmap(swp_entry_t *entry)
171{
172 kunmap_atomic(entry, KM_USER1);
173}
174
175static inline struct shmem_sb_info *SHMEM_SB(struct super_block *sb)
176{
177 return sb->s_fs_info;
178}
179
180
181
182
183
184
185
186static inline int shmem_acct_size(unsigned long flags, loff_t size)
187{
188 return (flags & VM_NORESERVE) ?
189 0 : security_vm_enough_memory_kern(VM_ACCT(size));
190}
191
192static inline void shmem_unacct_size(unsigned long flags, loff_t size)
193{
194 if (!(flags & VM_NORESERVE))
195 vm_unacct_memory(VM_ACCT(size));
196}
197
198
199
200
201
202
203
204static inline int shmem_acct_block(unsigned long flags)
205{
206 return (flags & VM_NORESERVE) ?
207 security_vm_enough_memory_kern(VM_ACCT(PAGE_CACHE_SIZE)) : 0;
208}
209
210static inline void shmem_unacct_blocks(unsigned long flags, long pages)
211{
212 if (flags & VM_NORESERVE)
213 vm_unacct_memory(pages * VM_ACCT(PAGE_CACHE_SIZE));
214}
215
216static const struct super_operations shmem_ops;
217static const struct address_space_operations shmem_aops;
218static const struct file_operations shmem_file_operations;
219static const struct inode_operations shmem_inode_operations;
220static const struct inode_operations shmem_dir_inode_operations;
221static const struct inode_operations shmem_special_inode_operations;
222static const struct vm_operations_struct shmem_vm_ops;
223
224static struct backing_dev_info shmem_backing_dev_info __read_mostly = {
225 .ra_pages = 0,
226 .capabilities = BDI_CAP_NO_ACCT_AND_WRITEBACK | BDI_CAP_SWAP_BACKED,
227};
228
229static LIST_HEAD(shmem_swaplist);
230static DEFINE_MUTEX(shmem_swaplist_mutex);
231
232static void shmem_free_blocks(struct inode *inode, long pages)
233{
234 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
235 if (sbinfo->max_blocks) {
236 percpu_counter_add(&sbinfo->used_blocks, -pages);
237 spin_lock(&inode->i_lock);
238 inode->i_blocks -= pages*BLOCKS_PER_PAGE;
239 spin_unlock(&inode->i_lock);
240 }
241}
242
243static int shmem_reserve_inode(struct super_block *sb)
244{
245 struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
246 if (sbinfo->max_inodes) {
247 spin_lock(&sbinfo->stat_lock);
248 if (!sbinfo->free_inodes) {
249 spin_unlock(&sbinfo->stat_lock);
250 return -ENOSPC;
251 }
252 sbinfo->free_inodes--;
253 spin_unlock(&sbinfo->stat_lock);
254 }
255 return 0;
256}
257
258static void shmem_free_inode(struct super_block *sb)
259{
260 struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
261 if (sbinfo->max_inodes) {
262 spin_lock(&sbinfo->stat_lock);
263 sbinfo->free_inodes++;
264 spin_unlock(&sbinfo->stat_lock);
265 }
266}
267
268
269
270
271
272
273
274
275
276
277
278
279
280static void shmem_recalc_inode(struct inode *inode)
281{
282 struct shmem_inode_info *info = SHMEM_I(inode);
283 long freed;
284
285 freed = info->alloced - info->swapped - inode->i_mapping->nrpages;
286 if (freed > 0) {
287 info->alloced -= freed;
288 shmem_unacct_blocks(info->flags, freed);
289 shmem_free_blocks(inode, freed);
290 }
291}
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330static swp_entry_t *shmem_swp_entry(struct shmem_inode_info *info, unsigned long index, struct page **page)
331{
332 unsigned long offset;
333 struct page **dir;
334 struct page *subdir;
335
336 if (index < SHMEM_NR_DIRECT) {
337 shmem_swp_balance_unmap();
338 return info->i_direct+index;
339 }
340 if (!info->i_indirect) {
341 if (page) {
342 info->i_indirect = *page;
343 *page = NULL;
344 }
345 return NULL;
346 }
347
348 index -= SHMEM_NR_DIRECT;
349 offset = index % ENTRIES_PER_PAGE;
350 index /= ENTRIES_PER_PAGE;
351 dir = shmem_dir_map(info->i_indirect);
352
353 if (index >= ENTRIES_PER_PAGE/2) {
354 index -= ENTRIES_PER_PAGE/2;
355 dir += ENTRIES_PER_PAGE/2 + index/ENTRIES_PER_PAGE;
356 index %= ENTRIES_PER_PAGE;
357 subdir = *dir;
358 if (!subdir) {
359 if (page) {
360 *dir = *page;
361 *page = NULL;
362 }
363 shmem_dir_unmap(dir);
364 return NULL;
365 }
366 shmem_dir_unmap(dir);
367 dir = shmem_dir_map(subdir);
368 }
369
370 dir += index;
371 subdir = *dir;
372 if (!subdir) {
373 if (!page || !(subdir = *page)) {
374 shmem_dir_unmap(dir);
375 return NULL;
376 }
377 *dir = subdir;
378 *page = NULL;
379 }
380 shmem_dir_unmap(dir);
381 return shmem_swp_map(subdir) + offset;
382}
383
384static void shmem_swp_set(struct shmem_inode_info *info, swp_entry_t *entry, unsigned long value)
385{
386 long incdec = value? 1: -1;
387
388 entry->val = value;
389 info->swapped += incdec;
390 if ((unsigned long)(entry - info->i_direct) >= SHMEM_NR_DIRECT) {
391 struct page *page = kmap_atomic_to_page(entry);
392 set_page_private(page, page_private(page) + incdec);
393 }
394}
395
396
397
398
399
400
401
402
403
404static swp_entry_t *shmem_swp_alloc(struct shmem_inode_info *info, unsigned long index, enum sgp_type sgp)
405{
406 struct inode *inode = &info->vfs_inode;
407 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
408 struct page *page = NULL;
409 swp_entry_t *entry;
410
411 if (sgp != SGP_WRITE &&
412 ((loff_t) index << PAGE_CACHE_SHIFT) >= i_size_read(inode))
413 return ERR_PTR(-EINVAL);
414
415 while (!(entry = shmem_swp_entry(info, index, &page))) {
416 if (sgp == SGP_READ)
417 return shmem_swp_map(ZERO_PAGE(0));
418
419
420
421
422
423 if (sbinfo->max_blocks) {
424 if (percpu_counter_compare(&sbinfo->used_blocks,
425 sbinfo->max_blocks - 1) >= 0)
426 return ERR_PTR(-ENOSPC);
427 percpu_counter_inc(&sbinfo->used_blocks);
428 spin_lock(&inode->i_lock);
429 inode->i_blocks += BLOCKS_PER_PAGE;
430 spin_unlock(&inode->i_lock);
431 }
432
433 spin_unlock(&info->lock);
434 page = shmem_dir_alloc(mapping_gfp_mask(inode->i_mapping));
435 spin_lock(&info->lock);
436
437 if (!page) {
438 shmem_free_blocks(inode, 1);
439 return ERR_PTR(-ENOMEM);
440 }
441 if (sgp != SGP_WRITE &&
442 ((loff_t) index << PAGE_CACHE_SHIFT) >= i_size_read(inode)) {
443 entry = ERR_PTR(-EINVAL);
444 break;
445 }
446 if (info->next_index <= index)
447 info->next_index = index + 1;
448 }
449 if (page) {
450
451 shmem_free_blocks(inode, 1);
452 shmem_dir_free(page);
453 }
454 if (info->next_index <= index && !IS_ERR(entry))
455 info->next_index = index + 1;
456 return entry;
457}
458
459
460
461
462
463
464
465static int shmem_free_swp(swp_entry_t *dir, swp_entry_t *edir,
466 spinlock_t *punch_lock)
467{
468 spinlock_t *punch_unlock = NULL;
469 swp_entry_t *ptr;
470 int freed = 0;
471
472 for (ptr = dir; ptr < edir; ptr++) {
473 if (ptr->val) {
474 if (unlikely(punch_lock)) {
475 punch_unlock = punch_lock;
476 punch_lock = NULL;
477 spin_lock(punch_unlock);
478 if (!ptr->val)
479 continue;
480 }
481 free_swap_and_cache(*ptr);
482 *ptr = (swp_entry_t){0};
483 freed++;
484 }
485 }
486 if (punch_unlock)
487 spin_unlock(punch_unlock);
488 return freed;
489}
490
491static int shmem_map_and_free_swp(struct page *subdir, int offset,
492 int limit, struct page ***dir, spinlock_t *punch_lock)
493{
494 swp_entry_t *ptr;
495 int freed = 0;
496
497 ptr = shmem_swp_map(subdir);
498 for (; offset < limit; offset += LATENCY_LIMIT) {
499 int size = limit - offset;
500 if (size > LATENCY_LIMIT)
501 size = LATENCY_LIMIT;
502 freed += shmem_free_swp(ptr+offset, ptr+offset+size,
503 punch_lock);
504 if (need_resched()) {
505 shmem_swp_unmap(ptr);
506 if (*dir) {
507 shmem_dir_unmap(*dir);
508 *dir = NULL;
509 }
510 cond_resched();
511 ptr = shmem_swp_map(subdir);
512 }
513 }
514 shmem_swp_unmap(ptr);
515 return freed;
516}
517
518static void shmem_free_pages(struct list_head *next)
519{
520 struct page *page;
521 int freed = 0;
522
523 do {
524 page = container_of(next, struct page, lru);
525 next = next->next;
526 shmem_dir_free(page);
527 freed++;
528 if (freed >= LATENCY_LIMIT) {
529 cond_resched();
530 freed = 0;
531 }
532 } while (next);
533}
534
535static void shmem_truncate_range(struct inode *inode, loff_t start, loff_t end)
536{
537 struct shmem_inode_info *info = SHMEM_I(inode);
538 unsigned long idx;
539 unsigned long size;
540 unsigned long limit;
541 unsigned long stage;
542 unsigned long diroff;
543 struct page **dir;
544 struct page *topdir;
545 struct page *middir;
546 struct page *subdir;
547 swp_entry_t *ptr;
548 LIST_HEAD(pages_to_free);
549 long nr_pages_to_free = 0;
550 long nr_swaps_freed = 0;
551 int offset;
552 int freed;
553 int punch_hole;
554 spinlock_t *needs_lock;
555 spinlock_t *punch_lock;
556 unsigned long upper_limit;
557
558 inode->i_ctime = inode->i_mtime = CURRENT_TIME;
559 idx = (start + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
560 if (idx >= info->next_index)
561 return;
562
563 spin_lock(&info->lock);
564 info->flags |= SHMEM_TRUNCATE;
565 if (likely(end == (loff_t) -1)) {
566 limit = info->next_index;
567 upper_limit = SHMEM_MAX_INDEX;
568 info->next_index = idx;
569 needs_lock = NULL;
570 punch_hole = 0;
571 } else {
572 if (end + 1 >= inode->i_size) {
573 limit = (inode->i_size + PAGE_CACHE_SIZE - 1) >>
574 PAGE_CACHE_SHIFT;
575 upper_limit = SHMEM_MAX_INDEX;
576 } else {
577 limit = (end + 1) >> PAGE_CACHE_SHIFT;
578 upper_limit = limit;
579 }
580 needs_lock = &info->lock;
581 punch_hole = 1;
582 }
583
584 topdir = info->i_indirect;
585 if (topdir && idx <= SHMEM_NR_DIRECT && !punch_hole) {
586 info->i_indirect = NULL;
587 nr_pages_to_free++;
588 list_add(&topdir->lru, &pages_to_free);
589 }
590 spin_unlock(&info->lock);
591
592 if (info->swapped && idx < SHMEM_NR_DIRECT) {
593 ptr = info->i_direct;
594 size = limit;
595 if (size > SHMEM_NR_DIRECT)
596 size = SHMEM_NR_DIRECT;
597 nr_swaps_freed = shmem_free_swp(ptr+idx, ptr+size, needs_lock);
598 }
599
600
601
602
603
604 if (!topdir || limit <= SHMEM_NR_DIRECT)
605 goto done2;
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620 upper_limit -= SHMEM_NR_DIRECT;
621 limit -= SHMEM_NR_DIRECT;
622 idx = (idx > SHMEM_NR_DIRECT)? (idx - SHMEM_NR_DIRECT): 0;
623 offset = idx % ENTRIES_PER_PAGE;
624 idx -= offset;
625
626 dir = shmem_dir_map(topdir);
627 stage = ENTRIES_PER_PAGEPAGE/2;
628 if (idx < ENTRIES_PER_PAGEPAGE/2) {
629 middir = topdir;
630 diroff = idx/ENTRIES_PER_PAGE;
631 } else {
632 dir += ENTRIES_PER_PAGE/2;
633 dir += (idx - ENTRIES_PER_PAGEPAGE/2)/ENTRIES_PER_PAGEPAGE;
634 while (stage <= idx)
635 stage += ENTRIES_PER_PAGEPAGE;
636 middir = *dir;
637 if (*dir) {
638 diroff = ((idx - ENTRIES_PER_PAGEPAGE/2) %
639 ENTRIES_PER_PAGEPAGE) / ENTRIES_PER_PAGE;
640 if (!diroff && !offset && upper_limit >= stage) {
641 if (needs_lock) {
642 spin_lock(needs_lock);
643 *dir = NULL;
644 spin_unlock(needs_lock);
645 needs_lock = NULL;
646 } else
647 *dir = NULL;
648 nr_pages_to_free++;
649 list_add(&middir->lru, &pages_to_free);
650 }
651 shmem_dir_unmap(dir);
652 dir = shmem_dir_map(middir);
653 } else {
654 diroff = 0;
655 offset = 0;
656 idx = stage;
657 }
658 }
659
660 for (; idx < limit; idx += ENTRIES_PER_PAGE, diroff++) {
661 if (unlikely(idx == stage)) {
662 shmem_dir_unmap(dir);
663 dir = shmem_dir_map(topdir) +
664 ENTRIES_PER_PAGE/2 + idx/ENTRIES_PER_PAGEPAGE;
665 while (!*dir) {
666 dir++;
667 idx += ENTRIES_PER_PAGEPAGE;
668 if (idx >= limit)
669 goto done1;
670 }
671 stage = idx + ENTRIES_PER_PAGEPAGE;
672 middir = *dir;
673 if (punch_hole)
674 needs_lock = &info->lock;
675 if (upper_limit >= stage) {
676 if (needs_lock) {
677 spin_lock(needs_lock);
678 *dir = NULL;
679 spin_unlock(needs_lock);
680 needs_lock = NULL;
681 } else
682 *dir = NULL;
683 nr_pages_to_free++;
684 list_add(&middir->lru, &pages_to_free);
685 }
686 shmem_dir_unmap(dir);
687 cond_resched();
688 dir = shmem_dir_map(middir);
689 diroff = 0;
690 }
691 punch_lock = needs_lock;
692 subdir = dir[diroff];
693 if (subdir && !offset && upper_limit-idx >= ENTRIES_PER_PAGE) {
694 if (needs_lock) {
695 spin_lock(needs_lock);
696 dir[diroff] = NULL;
697 spin_unlock(needs_lock);
698 punch_lock = NULL;
699 } else
700 dir[diroff] = NULL;
701 nr_pages_to_free++;
702 list_add(&subdir->lru, &pages_to_free);
703 }
704 if (subdir && page_private(subdir) ) {
705 size = limit - idx;
706 if (size > ENTRIES_PER_PAGE)
707 size = ENTRIES_PER_PAGE;
708 freed = shmem_map_and_free_swp(subdir,
709 offset, size, &dir, punch_lock);
710 if (!dir)
711 dir = shmem_dir_map(middir);
712 nr_swaps_freed += freed;
713 if (offset || punch_lock) {
714 spin_lock(&info->lock);
715 set_page_private(subdir,
716 page_private(subdir) - freed);
717 spin_unlock(&info->lock);
718 } else
719 BUG_ON(page_private(subdir) != freed);
720 }
721 offset = 0;
722 }
723done1:
724 shmem_dir_unmap(dir);
725done2:
726 if (inode->i_mapping->nrpages && (info->flags & SHMEM_PAGEIN)) {
727
728
729
730
731
732
733
734
735
736
737
738
739
740 truncate_inode_pages_range(inode->i_mapping, start, end);
741 if (punch_hole)
742 unmap_mapping_range(inode->i_mapping, start,
743 end - start, 1);
744 }
745
746 spin_lock(&info->lock);
747 info->flags &= ~SHMEM_TRUNCATE;
748 info->swapped -= nr_swaps_freed;
749 if (nr_pages_to_free)
750 shmem_free_blocks(inode, nr_pages_to_free);
751 shmem_recalc_inode(inode);
752 spin_unlock(&info->lock);
753
754
755
756
757 if (!list_empty(&pages_to_free)) {
758 pages_to_free.prev->next = NULL;
759 shmem_free_pages(pages_to_free.next);
760 }
761}
762
763static int shmem_notify_change(struct dentry *dentry, struct iattr *attr)
764{
765 struct inode *inode = dentry->d_inode;
766 loff_t newsize = attr->ia_size;
767 int error;
768
769 error = inode_change_ok(inode, attr);
770 if (error)
771 return error;
772
773 if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)
774 && newsize != inode->i_size) {
775 struct page *page = NULL;
776
777 if (newsize < inode->i_size) {
778
779
780
781
782
783
784
785 if (newsize & (PAGE_CACHE_SIZE-1)) {
786 (void) shmem_getpage(inode,
787 newsize >> PAGE_CACHE_SHIFT,
788 &page, SGP_READ, NULL);
789 if (page)
790 unlock_page(page);
791 }
792
793
794
795
796
797
798
799 if (newsize) {
800 struct shmem_inode_info *info = SHMEM_I(inode);
801 spin_lock(&info->lock);
802 info->flags &= ~SHMEM_PAGEIN;
803 spin_unlock(&info->lock);
804 }
805 }
806
807
808 truncate_setsize(inode, newsize);
809 if (page)
810 page_cache_release(page);
811 shmem_truncate_range(inode, newsize, (loff_t)-1);
812 }
813
814 setattr_copy(inode, attr);
815#ifdef CONFIG_TMPFS_POSIX_ACL
816 if (attr->ia_valid & ATTR_MODE)
817 error = generic_acl_chmod(inode);
818#endif
819 return error;
820}
821
822static void shmem_evict_inode(struct inode *inode)
823{
824 struct shmem_inode_info *info = SHMEM_I(inode);
825
826 if (inode->i_mapping->a_ops == &shmem_aops) {
827 truncate_inode_pages(inode->i_mapping, 0);
828 shmem_unacct_size(info->flags, inode->i_size);
829 inode->i_size = 0;
830 shmem_truncate_range(inode, 0, (loff_t)-1);
831 if (!list_empty(&info->swaplist)) {
832 mutex_lock(&shmem_swaplist_mutex);
833 list_del_init(&info->swaplist);
834 mutex_unlock(&shmem_swaplist_mutex);
835 }
836 }
837 BUG_ON(inode->i_blocks);
838 shmem_free_inode(inode->i_sb);
839 end_writeback(inode);
840}
841
842static inline int shmem_find_swp(swp_entry_t entry, swp_entry_t *dir, swp_entry_t *edir)
843{
844 swp_entry_t *ptr;
845
846 for (ptr = dir; ptr < edir; ptr++) {
847 if (ptr->val == entry.val)
848 return ptr - dir;
849 }
850 return -1;
851}
852
853static int shmem_unuse_inode(struct shmem_inode_info *info, swp_entry_t entry, struct page *page)
854{
855 struct address_space *mapping;
856 unsigned long idx;
857 unsigned long size;
858 unsigned long limit;
859 unsigned long stage;
860 struct page **dir;
861 struct page *subdir;
862 swp_entry_t *ptr;
863 int offset;
864 int error;
865
866 idx = 0;
867 ptr = info->i_direct;
868 spin_lock(&info->lock);
869 if (!info->swapped) {
870 list_del_init(&info->swaplist);
871 goto lost2;
872 }
873 limit = info->next_index;
874 size = limit;
875 if (size > SHMEM_NR_DIRECT)
876 size = SHMEM_NR_DIRECT;
877 offset = shmem_find_swp(entry, ptr, ptr+size);
878 if (offset >= 0) {
879 shmem_swp_balance_unmap();
880 goto found;
881 }
882 if (!info->i_indirect)
883 goto lost2;
884
885 dir = shmem_dir_map(info->i_indirect);
886 stage = SHMEM_NR_DIRECT + ENTRIES_PER_PAGEPAGE/2;
887
888 for (idx = SHMEM_NR_DIRECT; idx < limit; idx += ENTRIES_PER_PAGE, dir++) {
889 if (unlikely(idx == stage)) {
890 shmem_dir_unmap(dir-1);
891 if (cond_resched_lock(&info->lock)) {
892
893 if (limit > info->next_index) {
894 limit = info->next_index;
895 if (idx >= limit)
896 goto lost2;
897 }
898 }
899 dir = shmem_dir_map(info->i_indirect) +
900 ENTRIES_PER_PAGE/2 + idx/ENTRIES_PER_PAGEPAGE;
901 while (!*dir) {
902 dir++;
903 idx += ENTRIES_PER_PAGEPAGE;
904 if (idx >= limit)
905 goto lost1;
906 }
907 stage = idx + ENTRIES_PER_PAGEPAGE;
908 subdir = *dir;
909 shmem_dir_unmap(dir);
910 dir = shmem_dir_map(subdir);
911 }
912 subdir = *dir;
913 if (subdir && page_private(subdir)) {
914 ptr = shmem_swp_map(subdir);
915 size = limit - idx;
916 if (size > ENTRIES_PER_PAGE)
917 size = ENTRIES_PER_PAGE;
918 offset = shmem_find_swp(entry, ptr, ptr+size);
919 if (offset >= 0) {
920 shmem_dir_unmap(dir);
921 goto found;
922 }
923 shmem_swp_unmap(ptr);
924 }
925 }
926lost1:
927 shmem_dir_unmap(dir-1);
928lost2:
929 spin_unlock(&info->lock);
930 return 0;
931found:
932 idx += offset;
933 ptr += offset;
934
935
936
937
938
939
940
941
942 if (shmem_swaplist.next != &info->swaplist)
943 list_move_tail(&shmem_swaplist, &info->swaplist);
944
945
946
947
948
949
950 mapping = info->vfs_inode.i_mapping;
951 error = add_to_page_cache_locked(page, mapping, idx, GFP_NOWAIT);
952
953
954 if (error == -EEXIST) {
955 struct page *filepage = find_get_page(mapping, idx);
956 error = 1;
957 if (filepage) {
958
959
960
961
962 if (PageUptodate(filepage))
963 error = 0;
964 page_cache_release(filepage);
965 }
966 }
967 if (!error) {
968 delete_from_swap_cache(page);
969 set_page_dirty(page);
970 info->flags |= SHMEM_PAGEIN;
971 shmem_swp_set(info, ptr, 0);
972 swap_free(entry);
973 error = 1;
974 }
975 shmem_swp_unmap(ptr);
976 spin_unlock(&info->lock);
977 return error;
978}
979
980
981
982
983int shmem_unuse(swp_entry_t entry, struct page *page)
984{
985 struct list_head *p, *next;
986 struct shmem_inode_info *info;
987 int found = 0;
988 int error;
989
990
991
992
993
994
995
996 error = mem_cgroup_cache_charge(page, current->mm, GFP_KERNEL);
997 if (error)
998 goto out;
999
1000
1001
1002
1003
1004 error = radix_tree_preload(GFP_KERNEL);
1005 if (error)
1006 goto uncharge;
1007 radix_tree_preload_end();
1008
1009 mutex_lock(&shmem_swaplist_mutex);
1010 list_for_each_safe(p, next, &shmem_swaplist) {
1011 info = list_entry(p, struct shmem_inode_info, swaplist);
1012 found = shmem_unuse_inode(info, entry, page);
1013 cond_resched();
1014 if (found)
1015 break;
1016 }
1017 mutex_unlock(&shmem_swaplist_mutex);
1018
1019uncharge:
1020 if (!found)
1021 mem_cgroup_uncharge_cache_page(page);
1022 if (found < 0)
1023 error = found;
1024out:
1025 unlock_page(page);
1026 page_cache_release(page);
1027 return error;
1028}
1029
1030
1031
1032
1033static int shmem_writepage(struct page *page, struct writeback_control *wbc)
1034{
1035 struct shmem_inode_info *info;
1036 swp_entry_t *entry, swap;
1037 struct address_space *mapping;
1038 unsigned long index;
1039 struct inode *inode;
1040
1041 BUG_ON(!PageLocked(page));
1042 mapping = page->mapping;
1043 index = page->index;
1044 inode = mapping->host;
1045 info = SHMEM_I(inode);
1046 if (info->flags & VM_LOCKED)
1047 goto redirty;
1048 if (!total_swap_pages)
1049 goto redirty;
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060 if (wbc->for_reclaim)
1061 swap = get_swap_page();
1062 else
1063 swap.val = 0;
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074 if (swap.val) {
1075 mutex_lock(&shmem_swaplist_mutex);
1076 if (list_empty(&info->swaplist))
1077 list_add_tail(&info->swaplist, &shmem_swaplist);
1078 }
1079
1080 spin_lock(&info->lock);
1081 if (swap.val)
1082 mutex_unlock(&shmem_swaplist_mutex);
1083
1084 if (index >= info->next_index) {
1085 BUG_ON(!(info->flags & SHMEM_TRUNCATE));
1086 goto unlock;
1087 }
1088 entry = shmem_swp_entry(info, index, NULL);
1089 if (entry->val) {
1090
1091
1092
1093
1094 free_swap_and_cache(*entry);
1095 shmem_swp_set(info, entry, 0);
1096 }
1097 shmem_recalc_inode(inode);
1098
1099 if (swap.val && add_to_swap_cache(page, swap, GFP_ATOMIC) == 0) {
1100 delete_from_page_cache(page);
1101 shmem_swp_set(info, entry, swap.val);
1102 shmem_swp_unmap(entry);
1103 spin_unlock(&info->lock);
1104 swap_shmem_alloc(swap);
1105 BUG_ON(page_mapped(page));
1106 swap_writepage(page, wbc);
1107 return 0;
1108 }
1109
1110 shmem_swp_unmap(entry);
1111unlock:
1112 spin_unlock(&info->lock);
1113
1114
1115
1116
1117 swapcache_free(swap, NULL);
1118redirty:
1119 set_page_dirty(page);
1120 if (wbc->for_reclaim)
1121 return AOP_WRITEPAGE_ACTIVATE;
1122 unlock_page(page);
1123 return 0;
1124}
1125
1126#ifdef CONFIG_NUMA
1127#ifdef CONFIG_TMPFS
1128static void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol)
1129{
1130 char buffer[64];
1131
1132 if (!mpol || mpol->mode == MPOL_DEFAULT)
1133 return;
1134
1135 mpol_to_str(buffer, sizeof(buffer), mpol, 1);
1136
1137 seq_printf(seq, ",mpol=%s", buffer);
1138}
1139
1140static struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo)
1141{
1142 struct mempolicy *mpol = NULL;
1143 if (sbinfo->mpol) {
1144 spin_lock(&sbinfo->stat_lock);
1145 mpol = sbinfo->mpol;
1146 mpol_get(mpol);
1147 spin_unlock(&sbinfo->stat_lock);
1148 }
1149 return mpol;
1150}
1151#endif
1152
1153static struct page *shmem_swapin(swp_entry_t entry, gfp_t gfp,
1154 struct shmem_inode_info *info, unsigned long idx)
1155{
1156 struct mempolicy mpol, *spol;
1157 struct vm_area_struct pvma;
1158 struct page *page;
1159
1160 spol = mpol_cond_copy(&mpol,
1161 mpol_shared_policy_lookup(&info->policy, idx));
1162
1163
1164 pvma.vm_start = 0;
1165 pvma.vm_pgoff = idx;
1166 pvma.vm_ops = NULL;
1167 pvma.vm_policy = spol;
1168 page = swapin_readahead(entry, gfp, &pvma, 0);
1169 return page;
1170}
1171
1172static struct page *shmem_alloc_page(gfp_t gfp,
1173 struct shmem_inode_info *info, unsigned long idx)
1174{
1175 struct vm_area_struct pvma;
1176
1177
1178 pvma.vm_start = 0;
1179 pvma.vm_pgoff = idx;
1180 pvma.vm_ops = NULL;
1181 pvma.vm_policy = mpol_shared_policy_lookup(&info->policy, idx);
1182
1183
1184
1185
1186 return alloc_page_vma(gfp, &pvma, 0);
1187}
1188#else
1189#ifdef CONFIG_TMPFS
1190static inline void shmem_show_mpol(struct seq_file *seq, struct mempolicy *p)
1191{
1192}
1193#endif
1194
1195static inline struct page *shmem_swapin(swp_entry_t entry, gfp_t gfp,
1196 struct shmem_inode_info *info, unsigned long idx)
1197{
1198 return swapin_readahead(entry, gfp, NULL, 0);
1199}
1200
1201static inline struct page *shmem_alloc_page(gfp_t gfp,
1202 struct shmem_inode_info *info, unsigned long idx)
1203{
1204 return alloc_page(gfp);
1205}
1206#endif
1207
1208#if !defined(CONFIG_NUMA) || !defined(CONFIG_TMPFS)
1209static inline struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo)
1210{
1211 return NULL;
1212}
1213#endif
1214
1215
1216
1217
1218
1219
1220
1221
1222static int shmem_getpage(struct inode *inode, unsigned long idx,
1223 struct page **pagep, enum sgp_type sgp, int *type)
1224{
1225 struct address_space *mapping = inode->i_mapping;
1226 struct shmem_inode_info *info = SHMEM_I(inode);
1227 struct shmem_sb_info *sbinfo;
1228 struct page *filepage = *pagep;
1229 struct page *swappage;
1230 struct page *prealloc_page = NULL;
1231 swp_entry_t *entry;
1232 swp_entry_t swap;
1233 gfp_t gfp;
1234 int error;
1235
1236 if (idx >= SHMEM_MAX_INDEX)
1237 return -EFBIG;
1238
1239 if (type)
1240 *type = 0;
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250repeat:
1251 if (!filepage)
1252 filepage = find_lock_page(mapping, idx);
1253 if (filepage && PageUptodate(filepage))
1254 goto done;
1255 gfp = mapping_gfp_mask(mapping);
1256 if (!filepage) {
1257
1258
1259
1260
1261 error = radix_tree_preload(gfp & ~__GFP_HIGHMEM);
1262 if (error)
1263 goto failed;
1264 radix_tree_preload_end();
1265 if (sgp != SGP_READ && !prealloc_page) {
1266
1267 prealloc_page = shmem_alloc_page(gfp, info, idx);
1268 if (prealloc_page) {
1269 if (mem_cgroup_cache_charge(prealloc_page,
1270 current->mm, GFP_KERNEL)) {
1271 page_cache_release(prealloc_page);
1272 prealloc_page = NULL;
1273 }
1274 }
1275 }
1276 }
1277 error = 0;
1278
1279 spin_lock(&info->lock);
1280 shmem_recalc_inode(inode);
1281 entry = shmem_swp_alloc(info, idx, sgp);
1282 if (IS_ERR(entry)) {
1283 spin_unlock(&info->lock);
1284 error = PTR_ERR(entry);
1285 goto failed;
1286 }
1287 swap = *entry;
1288
1289 if (swap.val) {
1290
1291 swappage = lookup_swap_cache(swap);
1292 if (!swappage) {
1293 shmem_swp_unmap(entry);
1294
1295 if (type && !(*type & VM_FAULT_MAJOR)) {
1296 __count_vm_event(PGMAJFAULT);
1297 *type |= VM_FAULT_MAJOR;
1298 }
1299 spin_unlock(&info->lock);
1300 swappage = shmem_swapin(swap, gfp, info, idx);
1301 if (!swappage) {
1302 spin_lock(&info->lock);
1303 entry = shmem_swp_alloc(info, idx, sgp);
1304 if (IS_ERR(entry))
1305 error = PTR_ERR(entry);
1306 else {
1307 if (entry->val == swap.val)
1308 error = -ENOMEM;
1309 shmem_swp_unmap(entry);
1310 }
1311 spin_unlock(&info->lock);
1312 if (error)
1313 goto failed;
1314 goto repeat;
1315 }
1316 wait_on_page_locked(swappage);
1317 page_cache_release(swappage);
1318 goto repeat;
1319 }
1320
1321
1322 if (!trylock_page(swappage)) {
1323 shmem_swp_unmap(entry);
1324 spin_unlock(&info->lock);
1325 wait_on_page_locked(swappage);
1326 page_cache_release(swappage);
1327 goto repeat;
1328 }
1329 if (PageWriteback(swappage)) {
1330 shmem_swp_unmap(entry);
1331 spin_unlock(&info->lock);
1332 wait_on_page_writeback(swappage);
1333 unlock_page(swappage);
1334 page_cache_release(swappage);
1335 goto repeat;
1336 }
1337 if (!PageUptodate(swappage)) {
1338 shmem_swp_unmap(entry);
1339 spin_unlock(&info->lock);
1340 unlock_page(swappage);
1341 page_cache_release(swappage);
1342 error = -EIO;
1343 goto failed;
1344 }
1345
1346 if (filepage) {
1347 shmem_swp_set(info, entry, 0);
1348 shmem_swp_unmap(entry);
1349 delete_from_swap_cache(swappage);
1350 spin_unlock(&info->lock);
1351 copy_highpage(filepage, swappage);
1352 unlock_page(swappage);
1353 page_cache_release(swappage);
1354 flush_dcache_page(filepage);
1355 SetPageUptodate(filepage);
1356 set_page_dirty(filepage);
1357 swap_free(swap);
1358 } else if (!(error = add_to_page_cache_locked(swappage, mapping,
1359 idx, GFP_NOWAIT))) {
1360 info->flags |= SHMEM_PAGEIN;
1361 shmem_swp_set(info, entry, 0);
1362 shmem_swp_unmap(entry);
1363 delete_from_swap_cache(swappage);
1364 spin_unlock(&info->lock);
1365 filepage = swappage;
1366 set_page_dirty(filepage);
1367 swap_free(swap);
1368 } else {
1369 shmem_swp_unmap(entry);
1370 spin_unlock(&info->lock);
1371 if (error == -ENOMEM) {
1372
1373
1374
1375
1376 error = mem_cgroup_shmem_charge_fallback(
1377 swappage,
1378 current->mm,
1379 gfp);
1380 if (error) {
1381 unlock_page(swappage);
1382 page_cache_release(swappage);
1383 goto failed;
1384 }
1385 }
1386 unlock_page(swappage);
1387 page_cache_release(swappage);
1388 goto repeat;
1389 }
1390 } else if (sgp == SGP_READ && !filepage) {
1391 shmem_swp_unmap(entry);
1392 filepage = find_get_page(mapping, idx);
1393 if (filepage &&
1394 (!PageUptodate(filepage) || !trylock_page(filepage))) {
1395 spin_unlock(&info->lock);
1396 wait_on_page_locked(filepage);
1397 page_cache_release(filepage);
1398 filepage = NULL;
1399 goto repeat;
1400 }
1401 spin_unlock(&info->lock);
1402 } else {
1403 shmem_swp_unmap(entry);
1404 sbinfo = SHMEM_SB(inode->i_sb);
1405 if (sbinfo->max_blocks) {
1406 if (percpu_counter_compare(&sbinfo->used_blocks,
1407 sbinfo->max_blocks) >= 0 ||
1408 shmem_acct_block(info->flags))
1409 goto nospace;
1410 percpu_counter_inc(&sbinfo->used_blocks);
1411 spin_lock(&inode->i_lock);
1412 inode->i_blocks += BLOCKS_PER_PAGE;
1413 spin_unlock(&inode->i_lock);
1414 } else if (shmem_acct_block(info->flags))
1415 goto nospace;
1416
1417 if (!filepage) {
1418 int ret;
1419
1420 if (!prealloc_page) {
1421 spin_unlock(&info->lock);
1422 filepage = shmem_alloc_page(gfp, info, idx);
1423 if (!filepage) {
1424 shmem_unacct_blocks(info->flags, 1);
1425 shmem_free_blocks(inode, 1);
1426 error = -ENOMEM;
1427 goto failed;
1428 }
1429 SetPageSwapBacked(filepage);
1430
1431
1432
1433
1434
1435 error = mem_cgroup_cache_charge(filepage,
1436 current->mm, GFP_KERNEL);
1437 if (error) {
1438 page_cache_release(filepage);
1439 shmem_unacct_blocks(info->flags, 1);
1440 shmem_free_blocks(inode, 1);
1441 filepage = NULL;
1442 goto failed;
1443 }
1444
1445 spin_lock(&info->lock);
1446 } else {
1447 filepage = prealloc_page;
1448 prealloc_page = NULL;
1449 SetPageSwapBacked(filepage);
1450 }
1451
1452 entry = shmem_swp_alloc(info, idx, sgp);
1453 if (IS_ERR(entry))
1454 error = PTR_ERR(entry);
1455 else {
1456 swap = *entry;
1457 shmem_swp_unmap(entry);
1458 }
1459 ret = error || swap.val;
1460 if (ret)
1461 mem_cgroup_uncharge_cache_page(filepage);
1462 else
1463 ret = add_to_page_cache_lru(filepage, mapping,
1464 idx, GFP_NOWAIT);
1465
1466
1467
1468
1469 if (ret) {
1470 spin_unlock(&info->lock);
1471 page_cache_release(filepage);
1472 shmem_unacct_blocks(info->flags, 1);
1473 shmem_free_blocks(inode, 1);
1474 filepage = NULL;
1475 if (error)
1476 goto failed;
1477 goto repeat;
1478 }
1479 info->flags |= SHMEM_PAGEIN;
1480 }
1481
1482 info->alloced++;
1483 spin_unlock(&info->lock);
1484 clear_highpage(filepage);
1485 flush_dcache_page(filepage);
1486 SetPageUptodate(filepage);
1487 if (sgp == SGP_DIRTY)
1488 set_page_dirty(filepage);
1489 }
1490done:
1491 *pagep = filepage;
1492 error = 0;
1493 goto out;
1494
1495nospace:
1496
1497
1498
1499
1500
1501
1502
1503 if (!filepage) {
1504 struct page *page = find_get_page(mapping, idx);
1505 if (page) {
1506 spin_unlock(&info->lock);
1507 page_cache_release(page);
1508 goto repeat;
1509 }
1510 }
1511 spin_unlock(&info->lock);
1512 error = -ENOSPC;
1513failed:
1514 if (*pagep != filepage) {
1515 unlock_page(filepage);
1516 page_cache_release(filepage);
1517 }
1518out:
1519 if (prealloc_page) {
1520 mem_cgroup_uncharge_cache_page(prealloc_page);
1521 page_cache_release(prealloc_page);
1522 }
1523 return error;
1524}
1525
1526static int shmem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1527{
1528 struct inode *inode = vma->vm_file->f_path.dentry->d_inode;
1529 int error;
1530 int ret;
1531
1532 if (((loff_t)vmf->pgoff << PAGE_CACHE_SHIFT) >= i_size_read(inode))
1533 return VM_FAULT_SIGBUS;
1534
1535 error = shmem_getpage(inode, vmf->pgoff, &vmf->page, SGP_CACHE, &ret);
1536 if (error)
1537 return ((error == -ENOMEM) ? VM_FAULT_OOM : VM_FAULT_SIGBUS);
1538
1539 return ret | VM_FAULT_LOCKED;
1540}
1541
1542#ifdef CONFIG_NUMA
1543static int shmem_set_policy(struct vm_area_struct *vma, struct mempolicy *new)
1544{
1545 struct inode *i = vma->vm_file->f_path.dentry->d_inode;
1546 return mpol_set_shared_policy(&SHMEM_I(i)->policy, vma, new);
1547}
1548
1549static struct mempolicy *shmem_get_policy(struct vm_area_struct *vma,
1550 unsigned long addr)
1551{
1552 struct inode *i = vma->vm_file->f_path.dentry->d_inode;
1553 unsigned long idx;
1554
1555 idx = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
1556 return mpol_shared_policy_lookup(&SHMEM_I(i)->policy, idx);
1557}
1558#endif
1559
1560int shmem_lock(struct file *file, int lock, struct user_struct *user)
1561{
1562 struct inode *inode = file->f_path.dentry->d_inode;
1563 struct shmem_inode_info *info = SHMEM_I(inode);
1564 int retval = -ENOMEM;
1565
1566 spin_lock(&info->lock);
1567 if (lock && !(info->flags & VM_LOCKED)) {
1568 if (!user_shm_lock(inode->i_size, user))
1569 goto out_nomem;
1570 info->flags |= VM_LOCKED;
1571 mapping_set_unevictable(file->f_mapping);
1572 }
1573 if (!lock && (info->flags & VM_LOCKED) && user) {
1574 user_shm_unlock(inode->i_size, user);
1575 info->flags &= ~VM_LOCKED;
1576 mapping_clear_unevictable(file->f_mapping);
1577 scan_mapping_unevictable_pages(file->f_mapping);
1578 }
1579 retval = 0;
1580
1581out_nomem:
1582 spin_unlock(&info->lock);
1583 return retval;
1584}
1585
1586static int shmem_mmap(struct file *file, struct vm_area_struct *vma)
1587{
1588 file_accessed(file);
1589 vma->vm_ops = &shmem_vm_ops;
1590 vma->vm_flags |= VM_CAN_NONLINEAR;
1591 return 0;
1592}
1593
1594static struct inode *shmem_get_inode(struct super_block *sb, const struct inode *dir,
1595 int mode, dev_t dev, unsigned long flags)
1596{
1597 struct inode *inode;
1598 struct shmem_inode_info *info;
1599 struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
1600
1601 if (shmem_reserve_inode(sb))
1602 return NULL;
1603
1604 inode = new_inode(sb);
1605 if (inode) {
1606 inode->i_ino = get_next_ino();
1607 inode_init_owner(inode, dir, mode);
1608 inode->i_blocks = 0;
1609 inode->i_mapping->backing_dev_info = &shmem_backing_dev_info;
1610 inode->i_atime = inode->i_mtime = inode->i_ctime = CURRENT_TIME;
1611 inode->i_generation = get_seconds();
1612 info = SHMEM_I(inode);
1613 memset(info, 0, (char *)inode - (char *)info);
1614 spin_lock_init(&info->lock);
1615 info->flags = flags & VM_NORESERVE;
1616 INIT_LIST_HEAD(&info->swaplist);
1617 cache_no_acl(inode);
1618
1619 switch (mode & S_IFMT) {
1620 default:
1621 inode->i_op = &shmem_special_inode_operations;
1622 init_special_inode(inode, mode, dev);
1623 break;
1624 case S_IFREG:
1625 inode->i_mapping->a_ops = &shmem_aops;
1626 inode->i_op = &shmem_inode_operations;
1627 inode->i_fop = &shmem_file_operations;
1628 mpol_shared_policy_init(&info->policy,
1629 shmem_get_sbmpol(sbinfo));
1630 break;
1631 case S_IFDIR:
1632 inc_nlink(inode);
1633
1634 inode->i_size = 2 * BOGO_DIRENT_SIZE;
1635 inode->i_op = &shmem_dir_inode_operations;
1636 inode->i_fop = &simple_dir_operations;
1637 break;
1638 case S_IFLNK:
1639
1640
1641
1642
1643 mpol_shared_policy_init(&info->policy, NULL);
1644 break;
1645 }
1646 } else
1647 shmem_free_inode(sb);
1648 return inode;
1649}
1650
1651#ifdef CONFIG_TMPFS
1652static const struct inode_operations shmem_symlink_inode_operations;
1653static const struct inode_operations shmem_symlink_inline_operations;
1654
1655
1656
1657
1658
1659
1660static int shmem_readpage(struct file *file, struct page *page)
1661{
1662 struct inode *inode = page->mapping->host;
1663 int error = shmem_getpage(inode, page->index, &page, SGP_CACHE, NULL);
1664 unlock_page(page);
1665 return error;
1666}
1667
1668static int
1669shmem_write_begin(struct file *file, struct address_space *mapping,
1670 loff_t pos, unsigned len, unsigned flags,
1671 struct page **pagep, void **fsdata)
1672{
1673 struct inode *inode = mapping->host;
1674 pgoff_t index = pos >> PAGE_CACHE_SHIFT;
1675 *pagep = NULL;
1676 return shmem_getpage(inode, index, pagep, SGP_WRITE, NULL);
1677}
1678
1679static int
1680shmem_write_end(struct file *file, struct address_space *mapping,
1681 loff_t pos, unsigned len, unsigned copied,
1682 struct page *page, void *fsdata)
1683{
1684 struct inode *inode = mapping->host;
1685
1686 if (pos + copied > inode->i_size)
1687 i_size_write(inode, pos + copied);
1688
1689 set_page_dirty(page);
1690 unlock_page(page);
1691 page_cache_release(page);
1692
1693 return copied;
1694}
1695
1696static void do_shmem_file_read(struct file *filp, loff_t *ppos, read_descriptor_t *desc, read_actor_t actor)
1697{
1698 struct inode *inode = filp->f_path.dentry->d_inode;
1699 struct address_space *mapping = inode->i_mapping;
1700 unsigned long index, offset;
1701 enum sgp_type sgp = SGP_READ;
1702
1703
1704
1705
1706
1707
1708 if (segment_eq(get_fs(), KERNEL_DS))
1709 sgp = SGP_DIRTY;
1710
1711 index = *ppos >> PAGE_CACHE_SHIFT;
1712 offset = *ppos & ~PAGE_CACHE_MASK;
1713
1714 for (;;) {
1715 struct page *page = NULL;
1716 unsigned long end_index, nr, ret;
1717 loff_t i_size = i_size_read(inode);
1718
1719 end_index = i_size >> PAGE_CACHE_SHIFT;
1720 if (index > end_index)
1721 break;
1722 if (index == end_index) {
1723 nr = i_size & ~PAGE_CACHE_MASK;
1724 if (nr <= offset)
1725 break;
1726 }
1727
1728 desc->error = shmem_getpage(inode, index, &page, sgp, NULL);
1729 if (desc->error) {
1730 if (desc->error == -EINVAL)
1731 desc->error = 0;
1732 break;
1733 }
1734 if (page)
1735 unlock_page(page);
1736
1737
1738
1739
1740
1741 nr = PAGE_CACHE_SIZE;
1742 i_size = i_size_read(inode);
1743 end_index = i_size >> PAGE_CACHE_SHIFT;
1744 if (index == end_index) {
1745 nr = i_size & ~PAGE_CACHE_MASK;
1746 if (nr <= offset) {
1747 if (page)
1748 page_cache_release(page);
1749 break;
1750 }
1751 }
1752 nr -= offset;
1753
1754 if (page) {
1755
1756
1757
1758
1759
1760 if (mapping_writably_mapped(mapping))
1761 flush_dcache_page(page);
1762
1763
1764
1765 if (!offset)
1766 mark_page_accessed(page);
1767 } else {
1768 page = ZERO_PAGE(0);
1769 page_cache_get(page);
1770 }
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782 ret = actor(desc, page, offset, nr);
1783 offset += ret;
1784 index += offset >> PAGE_CACHE_SHIFT;
1785 offset &= ~PAGE_CACHE_MASK;
1786
1787 page_cache_release(page);
1788 if (ret != nr || !desc->count)
1789 break;
1790
1791 cond_resched();
1792 }
1793
1794 *ppos = ((loff_t) index << PAGE_CACHE_SHIFT) + offset;
1795 file_accessed(filp);
1796}
1797
1798static ssize_t shmem_file_aio_read(struct kiocb *iocb,
1799 const struct iovec *iov, unsigned long nr_segs, loff_t pos)
1800{
1801 struct file *filp = iocb->ki_filp;
1802 ssize_t retval;
1803 unsigned long seg;
1804 size_t count;
1805 loff_t *ppos = &iocb->ki_pos;
1806
1807 retval = generic_segment_checks(iov, &nr_segs, &count, VERIFY_WRITE);
1808 if (retval)
1809 return retval;
1810
1811 for (seg = 0; seg < nr_segs; seg++) {
1812 read_descriptor_t desc;
1813
1814 desc.written = 0;
1815 desc.arg.buf = iov[seg].iov_base;
1816 desc.count = iov[seg].iov_len;
1817 if (desc.count == 0)
1818 continue;
1819 desc.error = 0;
1820 do_shmem_file_read(filp, ppos, &desc, file_read_actor);
1821 retval += desc.written;
1822 if (desc.error) {
1823 retval = retval ?: desc.error;
1824 break;
1825 }
1826 if (desc.count > 0)
1827 break;
1828 }
1829 return retval;
1830}
1831
1832static int shmem_statfs(struct dentry *dentry, struct kstatfs *buf)
1833{
1834 struct shmem_sb_info *sbinfo = SHMEM_SB(dentry->d_sb);
1835
1836 buf->f_type = TMPFS_MAGIC;
1837 buf->f_bsize = PAGE_CACHE_SIZE;
1838 buf->f_namelen = NAME_MAX;
1839 if (sbinfo->max_blocks) {
1840 buf->f_blocks = sbinfo->max_blocks;
1841 buf->f_bavail = buf->f_bfree =
1842 sbinfo->max_blocks - percpu_counter_sum(&sbinfo->used_blocks);
1843 }
1844 if (sbinfo->max_inodes) {
1845 buf->f_files = sbinfo->max_inodes;
1846 buf->f_ffree = sbinfo->free_inodes;
1847 }
1848
1849 return 0;
1850}
1851
1852
1853
1854
1855static int
1856shmem_mknod(struct inode *dir, struct dentry *dentry, int mode, dev_t dev)
1857{
1858 struct inode *inode;
1859 int error = -ENOSPC;
1860
1861 inode = shmem_get_inode(dir->i_sb, dir, mode, dev, VM_NORESERVE);
1862 if (inode) {
1863 error = security_inode_init_security(inode, dir,
1864 &dentry->d_name, NULL,
1865 NULL, NULL);
1866 if (error) {
1867 if (error != -EOPNOTSUPP) {
1868 iput(inode);
1869 return error;
1870 }
1871 }
1872#ifdef CONFIG_TMPFS_POSIX_ACL
1873 error = generic_acl_init(inode, dir);
1874 if (error) {
1875 iput(inode);
1876 return error;
1877 }
1878#else
1879 error = 0;
1880#endif
1881 dir->i_size += BOGO_DIRENT_SIZE;
1882 dir->i_ctime = dir->i_mtime = CURRENT_TIME;
1883 d_instantiate(dentry, inode);
1884 dget(dentry);
1885 }
1886 return error;
1887}
1888
1889static int shmem_mkdir(struct inode *dir, struct dentry *dentry, int mode)
1890{
1891 int error;
1892
1893 if ((error = shmem_mknod(dir, dentry, mode | S_IFDIR, 0)))
1894 return error;
1895 inc_nlink(dir);
1896 return 0;
1897}
1898
1899static int shmem_create(struct inode *dir, struct dentry *dentry, int mode,
1900 struct nameidata *nd)
1901{
1902 return shmem_mknod(dir, dentry, mode | S_IFREG, 0);
1903}
1904
1905
1906
1907
1908static int shmem_link(struct dentry *old_dentry, struct inode *dir, struct dentry *dentry)
1909{
1910 struct inode *inode = old_dentry->d_inode;
1911 int ret;
1912
1913
1914
1915
1916
1917
1918 ret = shmem_reserve_inode(inode->i_sb);
1919 if (ret)
1920 goto out;
1921
1922 dir->i_size += BOGO_DIRENT_SIZE;
1923 inode->i_ctime = dir->i_ctime = dir->i_mtime = CURRENT_TIME;
1924 inc_nlink(inode);
1925 ihold(inode);
1926 dget(dentry);
1927 d_instantiate(dentry, inode);
1928out:
1929 return ret;
1930}
1931
1932static int shmem_unlink(struct inode *dir, struct dentry *dentry)
1933{
1934 struct inode *inode = dentry->d_inode;
1935
1936 if (inode->i_nlink > 1 && !S_ISDIR(inode->i_mode))
1937 shmem_free_inode(inode->i_sb);
1938
1939 dir->i_size -= BOGO_DIRENT_SIZE;
1940 inode->i_ctime = dir->i_ctime = dir->i_mtime = CURRENT_TIME;
1941 drop_nlink(inode);
1942 dput(dentry);
1943 return 0;
1944}
1945
1946static int shmem_rmdir(struct inode *dir, struct dentry *dentry)
1947{
1948 if (!simple_empty(dentry))
1949 return -ENOTEMPTY;
1950
1951 drop_nlink(dentry->d_inode);
1952 drop_nlink(dir);
1953 return shmem_unlink(dir, dentry);
1954}
1955
1956
1957
1958
1959
1960
1961
1962static int shmem_rename(struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry)
1963{
1964 struct inode *inode = old_dentry->d_inode;
1965 int they_are_dirs = S_ISDIR(inode->i_mode);
1966
1967 if (!simple_empty(new_dentry))
1968 return -ENOTEMPTY;
1969
1970 if (new_dentry->d_inode) {
1971 (void) shmem_unlink(new_dir, new_dentry);
1972 if (they_are_dirs)
1973 drop_nlink(old_dir);
1974 } else if (they_are_dirs) {
1975 drop_nlink(old_dir);
1976 inc_nlink(new_dir);
1977 }
1978
1979 old_dir->i_size -= BOGO_DIRENT_SIZE;
1980 new_dir->i_size += BOGO_DIRENT_SIZE;
1981 old_dir->i_ctime = old_dir->i_mtime =
1982 new_dir->i_ctime = new_dir->i_mtime =
1983 inode->i_ctime = CURRENT_TIME;
1984 return 0;
1985}
1986
1987static int shmem_symlink(struct inode *dir, struct dentry *dentry, const char *symname)
1988{
1989 int error;
1990 int len;
1991 struct inode *inode;
1992 struct page *page = NULL;
1993 char *kaddr;
1994 struct shmem_inode_info *info;
1995
1996 len = strlen(symname) + 1;
1997 if (len > PAGE_CACHE_SIZE)
1998 return -ENAMETOOLONG;
1999
2000 inode = shmem_get_inode(dir->i_sb, dir, S_IFLNK|S_IRWXUGO, 0, VM_NORESERVE);
2001 if (!inode)
2002 return -ENOSPC;
2003
2004 error = security_inode_init_security(inode, dir, &dentry->d_name, NULL,
2005 NULL, NULL);
2006 if (error) {
2007 if (error != -EOPNOTSUPP) {
2008 iput(inode);
2009 return error;
2010 }
2011 error = 0;
2012 }
2013
2014 info = SHMEM_I(inode);
2015 inode->i_size = len-1;
2016 if (len <= (char *)inode - (char *)info) {
2017
2018 memcpy(info, symname, len);
2019 inode->i_op = &shmem_symlink_inline_operations;
2020 } else {
2021 error = shmem_getpage(inode, 0, &page, SGP_WRITE, NULL);
2022 if (error) {
2023 iput(inode);
2024 return error;
2025 }
2026 inode->i_mapping->a_ops = &shmem_aops;
2027 inode->i_op = &shmem_symlink_inode_operations;
2028 kaddr = kmap_atomic(page, KM_USER0);
2029 memcpy(kaddr, symname, len);
2030 kunmap_atomic(kaddr, KM_USER0);
2031 set_page_dirty(page);
2032 unlock_page(page);
2033 page_cache_release(page);
2034 }
2035 dir->i_size += BOGO_DIRENT_SIZE;
2036 dir->i_ctime = dir->i_mtime = CURRENT_TIME;
2037 d_instantiate(dentry, inode);
2038 dget(dentry);
2039 return 0;
2040}
2041
2042static void *shmem_follow_link_inline(struct dentry *dentry, struct nameidata *nd)
2043{
2044 nd_set_link(nd, (char *)SHMEM_I(dentry->d_inode));
2045 return NULL;
2046}
2047
2048static void *shmem_follow_link(struct dentry *dentry, struct nameidata *nd)
2049{
2050 struct page *page = NULL;
2051 int res = shmem_getpage(dentry->d_inode, 0, &page, SGP_READ, NULL);
2052 nd_set_link(nd, res ? ERR_PTR(res) : kmap(page));
2053 if (page)
2054 unlock_page(page);
2055 return page;
2056}
2057
2058static void shmem_put_link(struct dentry *dentry, struct nameidata *nd, void *cookie)
2059{
2060 if (!IS_ERR(nd_get_link(nd))) {
2061 struct page *page = cookie;
2062 kunmap(page);
2063 mark_page_accessed(page);
2064 page_cache_release(page);
2065 }
2066}
2067
2068static const struct inode_operations shmem_symlink_inline_operations = {
2069 .readlink = generic_readlink,
2070 .follow_link = shmem_follow_link_inline,
2071};
2072
2073static const struct inode_operations shmem_symlink_inode_operations = {
2074 .readlink = generic_readlink,
2075 .follow_link = shmem_follow_link,
2076 .put_link = shmem_put_link,
2077};
2078
2079#ifdef CONFIG_TMPFS_POSIX_ACL
2080
2081
2082
2083
2084
2085
2086
2087static size_t shmem_xattr_security_list(struct dentry *dentry, char *list,
2088 size_t list_len, const char *name,
2089 size_t name_len, int handler_flags)
2090{
2091 return security_inode_listsecurity(dentry->d_inode, list, list_len);
2092}
2093
2094static int shmem_xattr_security_get(struct dentry *dentry, const char *name,
2095 void *buffer, size_t size, int handler_flags)
2096{
2097 if (strcmp(name, "") == 0)
2098 return -EINVAL;
2099 return xattr_getsecurity(dentry->d_inode, name, buffer, size);
2100}
2101
2102static int shmem_xattr_security_set(struct dentry *dentry, const char *name,
2103 const void *value, size_t size, int flags, int handler_flags)
2104{
2105 if (strcmp(name, "") == 0)
2106 return -EINVAL;
2107 return security_inode_setsecurity(dentry->d_inode, name, value,
2108 size, flags);
2109}
2110
2111static const struct xattr_handler shmem_xattr_security_handler = {
2112 .prefix = XATTR_SECURITY_PREFIX,
2113 .list = shmem_xattr_security_list,
2114 .get = shmem_xattr_security_get,
2115 .set = shmem_xattr_security_set,
2116};
2117
2118static const struct xattr_handler *shmem_xattr_handlers[] = {
2119 &generic_acl_access_handler,
2120 &generic_acl_default_handler,
2121 &shmem_xattr_security_handler,
2122 NULL
2123};
2124#endif
2125
2126static struct dentry *shmem_get_parent(struct dentry *child)
2127{
2128 return ERR_PTR(-ESTALE);
2129}
2130
2131static int shmem_match(struct inode *ino, void *vfh)
2132{
2133 __u32 *fh = vfh;
2134 __u64 inum = fh[2];
2135 inum = (inum << 32) | fh[1];
2136 return ino->i_ino == inum && fh[0] == ino->i_generation;
2137}
2138
2139static struct dentry *shmem_fh_to_dentry(struct super_block *sb,
2140 struct fid *fid, int fh_len, int fh_type)
2141{
2142 struct inode *inode;
2143 struct dentry *dentry = NULL;
2144 u64 inum = fid->raw[2];
2145 inum = (inum << 32) | fid->raw[1];
2146
2147 if (fh_len < 3)
2148 return NULL;
2149
2150 inode = ilookup5(sb, (unsigned long)(inum + fid->raw[0]),
2151 shmem_match, fid->raw);
2152 if (inode) {
2153 dentry = d_find_alias(inode);
2154 iput(inode);
2155 }
2156
2157 return dentry;
2158}
2159
2160static int shmem_encode_fh(struct dentry *dentry, __u32 *fh, int *len,
2161 int connectable)
2162{
2163 struct inode *inode = dentry->d_inode;
2164
2165 if (*len < 3) {
2166 *len = 3;
2167 return 255;
2168 }
2169
2170 if (inode_unhashed(inode)) {
2171
2172
2173
2174
2175
2176 static DEFINE_SPINLOCK(lock);
2177 spin_lock(&lock);
2178 if (inode_unhashed(inode))
2179 __insert_inode_hash(inode,
2180 inode->i_ino + inode->i_generation);
2181 spin_unlock(&lock);
2182 }
2183
2184 fh[0] = inode->i_generation;
2185 fh[1] = inode->i_ino;
2186 fh[2] = ((__u64)inode->i_ino) >> 32;
2187
2188 *len = 3;
2189 return 1;
2190}
2191
2192static const struct export_operations shmem_export_ops = {
2193 .get_parent = shmem_get_parent,
2194 .encode_fh = shmem_encode_fh,
2195 .fh_to_dentry = shmem_fh_to_dentry,
2196};
2197
2198static int shmem_parse_options(char *options, struct shmem_sb_info *sbinfo,
2199 bool remount)
2200{
2201 char *this_char, *value, *rest;
2202
2203 while (options != NULL) {
2204 this_char = options;
2205 for (;;) {
2206
2207
2208
2209
2210
2211 options = strchr(options, ',');
2212 if (options == NULL)
2213 break;
2214 options++;
2215 if (!isdigit(*options)) {
2216 options[-1] = '\0';
2217 break;
2218 }
2219 }
2220 if (!*this_char)
2221 continue;
2222 if ((value = strchr(this_char,'=')) != NULL) {
2223 *value++ = 0;
2224 } else {
2225 printk(KERN_ERR
2226 "tmpfs: No value for mount option '%s'\n",
2227 this_char);
2228 return 1;
2229 }
2230
2231 if (!strcmp(this_char,"size")) {
2232 unsigned long long size;
2233 size = memparse(value,&rest);
2234 if (*rest == '%') {
2235 size <<= PAGE_SHIFT;
2236 size *= totalram_pages;
2237 do_div(size, 100);
2238 rest++;
2239 }
2240 if (*rest)
2241 goto bad_val;
2242 sbinfo->max_blocks =
2243 DIV_ROUND_UP(size, PAGE_CACHE_SIZE);
2244 } else if (!strcmp(this_char,"nr_blocks")) {
2245 sbinfo->max_blocks = memparse(value, &rest);
2246 if (*rest)
2247 goto bad_val;
2248 } else if (!strcmp(this_char,"nr_inodes")) {
2249 sbinfo->max_inodes = memparse(value, &rest);
2250 if (*rest)
2251 goto bad_val;
2252 } else if (!strcmp(this_char,"mode")) {
2253 if (remount)
2254 continue;
2255 sbinfo->mode = simple_strtoul(value, &rest, 8) & 07777;
2256 if (*rest)
2257 goto bad_val;
2258 } else if (!strcmp(this_char,"uid")) {
2259 if (remount)
2260 continue;
2261 sbinfo->uid = simple_strtoul(value, &rest, 0);
2262 if (*rest)
2263 goto bad_val;
2264 } else if (!strcmp(this_char,"gid")) {
2265 if (remount)
2266 continue;
2267 sbinfo->gid = simple_strtoul(value, &rest, 0);
2268 if (*rest)
2269 goto bad_val;
2270 } else if (!strcmp(this_char,"mpol")) {
2271 if (mpol_parse_str(value, &sbinfo->mpol, 1))
2272 goto bad_val;
2273 } else {
2274 printk(KERN_ERR "tmpfs: Bad mount option %s\n",
2275 this_char);
2276 return 1;
2277 }
2278 }
2279 return 0;
2280
2281bad_val:
2282 printk(KERN_ERR "tmpfs: Bad value '%s' for mount option '%s'\n",
2283 value, this_char);
2284 return 1;
2285
2286}
2287
2288static int shmem_remount_fs(struct super_block *sb, int *flags, char *data)
2289{
2290 struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
2291 struct shmem_sb_info config = *sbinfo;
2292 unsigned long inodes;
2293 int error = -EINVAL;
2294
2295 if (shmem_parse_options(data, &config, true))
2296 return error;
2297
2298 spin_lock(&sbinfo->stat_lock);
2299 inodes = sbinfo->max_inodes - sbinfo->free_inodes;
2300 if (percpu_counter_compare(&sbinfo->used_blocks, config.max_blocks) > 0)
2301 goto out;
2302 if (config.max_inodes < inodes)
2303 goto out;
2304
2305
2306
2307
2308
2309
2310 if (config.max_blocks && !sbinfo->max_blocks)
2311 goto out;
2312 if (config.max_inodes && !sbinfo->max_inodes)
2313 goto out;
2314
2315 error = 0;
2316 sbinfo->max_blocks = config.max_blocks;
2317 sbinfo->max_inodes = config.max_inodes;
2318 sbinfo->free_inodes = config.max_inodes - inodes;
2319
2320 mpol_put(sbinfo->mpol);
2321 sbinfo->mpol = config.mpol;
2322out:
2323 spin_unlock(&sbinfo->stat_lock);
2324 return error;
2325}
2326
2327static int shmem_show_options(struct seq_file *seq, struct vfsmount *vfs)
2328{
2329 struct shmem_sb_info *sbinfo = SHMEM_SB(vfs->mnt_sb);
2330
2331 if (sbinfo->max_blocks != shmem_default_max_blocks())
2332 seq_printf(seq, ",size=%luk",
2333 sbinfo->max_blocks << (PAGE_CACHE_SHIFT - 10));
2334 if (sbinfo->max_inodes != shmem_default_max_inodes())
2335 seq_printf(seq, ",nr_inodes=%lu", sbinfo->max_inodes);
2336 if (sbinfo->mode != (S_IRWXUGO | S_ISVTX))
2337 seq_printf(seq, ",mode=%03o", sbinfo->mode);
2338 if (sbinfo->uid != 0)
2339 seq_printf(seq, ",uid=%u", sbinfo->uid);
2340 if (sbinfo->gid != 0)
2341 seq_printf(seq, ",gid=%u", sbinfo->gid);
2342 shmem_show_mpol(seq, sbinfo->mpol);
2343 return 0;
2344}
2345#endif
2346
2347static void shmem_put_super(struct super_block *sb)
2348{
2349 struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
2350
2351 percpu_counter_destroy(&sbinfo->used_blocks);
2352 kfree(sbinfo);
2353 sb->s_fs_info = NULL;
2354}
2355
2356int shmem_fill_super(struct super_block *sb, void *data, int silent)
2357{
2358 struct inode *inode;
2359 struct dentry *root;
2360 struct shmem_sb_info *sbinfo;
2361 int err = -ENOMEM;
2362
2363
2364 sbinfo = kzalloc(max((int)sizeof(struct shmem_sb_info),
2365 L1_CACHE_BYTES), GFP_KERNEL);
2366 if (!sbinfo)
2367 return -ENOMEM;
2368
2369 sbinfo->mode = S_IRWXUGO | S_ISVTX;
2370 sbinfo->uid = current_fsuid();
2371 sbinfo->gid = current_fsgid();
2372 sb->s_fs_info = sbinfo;
2373
2374#ifdef CONFIG_TMPFS
2375
2376
2377
2378
2379
2380 if (!(sb->s_flags & MS_NOUSER)) {
2381 sbinfo->max_blocks = shmem_default_max_blocks();
2382 sbinfo->max_inodes = shmem_default_max_inodes();
2383 if (shmem_parse_options(data, sbinfo, false)) {
2384 err = -EINVAL;
2385 goto failed;
2386 }
2387 }
2388 sb->s_export_op = &shmem_export_ops;
2389#else
2390 sb->s_flags |= MS_NOUSER;
2391#endif
2392
2393 spin_lock_init(&sbinfo->stat_lock);
2394 if (percpu_counter_init(&sbinfo->used_blocks, 0))
2395 goto failed;
2396 sbinfo->free_inodes = sbinfo->max_inodes;
2397
2398 sb->s_maxbytes = SHMEM_MAX_BYTES;
2399 sb->s_blocksize = PAGE_CACHE_SIZE;
2400 sb->s_blocksize_bits = PAGE_CACHE_SHIFT;
2401 sb->s_magic = TMPFS_MAGIC;
2402 sb->s_op = &shmem_ops;
2403 sb->s_time_gran = 1;
2404#ifdef CONFIG_TMPFS_POSIX_ACL
2405 sb->s_xattr = shmem_xattr_handlers;
2406 sb->s_flags |= MS_POSIXACL;
2407#endif
2408
2409 inode = shmem_get_inode(sb, NULL, S_IFDIR | sbinfo->mode, 0, VM_NORESERVE);
2410 if (!inode)
2411 goto failed;
2412 inode->i_uid = sbinfo->uid;
2413 inode->i_gid = sbinfo->gid;
2414 root = d_alloc_root(inode);
2415 if (!root)
2416 goto failed_iput;
2417 sb->s_root = root;
2418 return 0;
2419
2420failed_iput:
2421 iput(inode);
2422failed:
2423 shmem_put_super(sb);
2424 return err;
2425}
2426
2427static struct kmem_cache *shmem_inode_cachep;
2428
2429static struct inode *shmem_alloc_inode(struct super_block *sb)
2430{
2431 struct shmem_inode_info *p;
2432 p = (struct shmem_inode_info *)kmem_cache_alloc(shmem_inode_cachep, GFP_KERNEL);
2433 if (!p)
2434 return NULL;
2435 return &p->vfs_inode;
2436}
2437
2438static void shmem_i_callback(struct rcu_head *head)
2439{
2440 struct inode *inode = container_of(head, struct inode, i_rcu);
2441 INIT_LIST_HEAD(&inode->i_dentry);
2442 kmem_cache_free(shmem_inode_cachep, SHMEM_I(inode));
2443}
2444
2445static void shmem_destroy_inode(struct inode *inode)
2446{
2447 if ((inode->i_mode & S_IFMT) == S_IFREG) {
2448
2449 mpol_free_shared_policy(&SHMEM_I(inode)->policy);
2450 }
2451 call_rcu(&inode->i_rcu, shmem_i_callback);
2452}
2453
2454static void init_once(void *foo)
2455{
2456 struct shmem_inode_info *p = (struct shmem_inode_info *) foo;
2457
2458 inode_init_once(&p->vfs_inode);
2459}
2460
2461static int init_inodecache(void)
2462{
2463 shmem_inode_cachep = kmem_cache_create("shmem_inode_cache",
2464 sizeof(struct shmem_inode_info),
2465 0, SLAB_PANIC, init_once);
2466 return 0;
2467}
2468
2469static void destroy_inodecache(void)
2470{
2471 kmem_cache_destroy(shmem_inode_cachep);
2472}
2473
2474static const struct address_space_operations shmem_aops = {
2475 .writepage = shmem_writepage,
2476 .set_page_dirty = __set_page_dirty_no_writeback,
2477#ifdef CONFIG_TMPFS
2478 .readpage = shmem_readpage,
2479 .write_begin = shmem_write_begin,
2480 .write_end = shmem_write_end,
2481#endif
2482 .migratepage = migrate_page,
2483 .error_remove_page = generic_error_remove_page,
2484};
2485
2486static const struct file_operations shmem_file_operations = {
2487 .mmap = shmem_mmap,
2488#ifdef CONFIG_TMPFS
2489 .llseek = generic_file_llseek,
2490 .read = do_sync_read,
2491 .write = do_sync_write,
2492 .aio_read = shmem_file_aio_read,
2493 .aio_write = generic_file_aio_write,
2494 .fsync = noop_fsync,
2495 .splice_read = generic_file_splice_read,
2496 .splice_write = generic_file_splice_write,
2497#endif
2498};
2499
2500static const struct inode_operations shmem_inode_operations = {
2501 .setattr = shmem_notify_change,
2502 .truncate_range = shmem_truncate_range,
2503#ifdef CONFIG_TMPFS_POSIX_ACL
2504 .setxattr = generic_setxattr,
2505 .getxattr = generic_getxattr,
2506 .listxattr = generic_listxattr,
2507 .removexattr = generic_removexattr,
2508 .check_acl = generic_check_acl,
2509#endif
2510
2511};
2512
2513static const struct inode_operations shmem_dir_inode_operations = {
2514#ifdef CONFIG_TMPFS
2515 .create = shmem_create,
2516 .lookup = simple_lookup,
2517 .link = shmem_link,
2518 .unlink = shmem_unlink,
2519 .symlink = shmem_symlink,
2520 .mkdir = shmem_mkdir,
2521 .rmdir = shmem_rmdir,
2522 .mknod = shmem_mknod,
2523 .rename = shmem_rename,
2524#endif
2525#ifdef CONFIG_TMPFS_POSIX_ACL
2526 .setattr = shmem_notify_change,
2527 .setxattr = generic_setxattr,
2528 .getxattr = generic_getxattr,
2529 .listxattr = generic_listxattr,
2530 .removexattr = generic_removexattr,
2531 .check_acl = generic_check_acl,
2532#endif
2533};
2534
2535static const struct inode_operations shmem_special_inode_operations = {
2536#ifdef CONFIG_TMPFS_POSIX_ACL
2537 .setattr = shmem_notify_change,
2538 .setxattr = generic_setxattr,
2539 .getxattr = generic_getxattr,
2540 .listxattr = generic_listxattr,
2541 .removexattr = generic_removexattr,
2542 .check_acl = generic_check_acl,
2543#endif
2544};
2545
2546static const struct super_operations shmem_ops = {
2547 .alloc_inode = shmem_alloc_inode,
2548 .destroy_inode = shmem_destroy_inode,
2549#ifdef CONFIG_TMPFS
2550 .statfs = shmem_statfs,
2551 .remount_fs = shmem_remount_fs,
2552 .show_options = shmem_show_options,
2553#endif
2554 .evict_inode = shmem_evict_inode,
2555 .drop_inode = generic_delete_inode,
2556 .put_super = shmem_put_super,
2557};
2558
2559static const struct vm_operations_struct shmem_vm_ops = {
2560 .fault = shmem_fault,
2561#ifdef CONFIG_NUMA
2562 .set_policy = shmem_set_policy,
2563 .get_policy = shmem_get_policy,
2564#endif
2565};
2566
2567
2568static struct dentry *shmem_mount(struct file_system_type *fs_type,
2569 int flags, const char *dev_name, void *data)
2570{
2571 return mount_nodev(fs_type, flags, data, shmem_fill_super);
2572}
2573
2574static struct file_system_type tmpfs_fs_type = {
2575 .owner = THIS_MODULE,
2576 .name = "tmpfs",
2577 .mount = shmem_mount,
2578 .kill_sb = kill_litter_super,
2579};
2580
2581int __init init_tmpfs(void)
2582{
2583 int error;
2584
2585 error = bdi_init(&shmem_backing_dev_info);
2586 if (error)
2587 goto out4;
2588
2589 error = init_inodecache();
2590 if (error)
2591 goto out3;
2592
2593 error = register_filesystem(&tmpfs_fs_type);
2594 if (error) {
2595 printk(KERN_ERR "Could not register tmpfs\n");
2596 goto out2;
2597 }
2598
2599 shm_mnt = vfs_kern_mount(&tmpfs_fs_type, MS_NOUSER,
2600 tmpfs_fs_type.name, NULL);
2601 if (IS_ERR(shm_mnt)) {
2602 error = PTR_ERR(shm_mnt);
2603 printk(KERN_ERR "Could not kern_mount tmpfs\n");
2604 goto out1;
2605 }
2606 return 0;
2607
2608out1:
2609 unregister_filesystem(&tmpfs_fs_type);
2610out2:
2611 destroy_inodecache();
2612out3:
2613 bdi_destroy(&shmem_backing_dev_info);
2614out4:
2615 shm_mnt = ERR_PTR(error);
2616 return error;
2617}
2618
2619#ifdef CONFIG_CGROUP_MEM_RES_CTLR
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630void mem_cgroup_get_shmem_target(struct inode *inode, pgoff_t pgoff,
2631 struct page **pagep, swp_entry_t *ent)
2632{
2633 swp_entry_t entry = { .val = 0 }, *ptr;
2634 struct page *page = NULL;
2635 struct shmem_inode_info *info = SHMEM_I(inode);
2636
2637 if ((pgoff << PAGE_CACHE_SHIFT) >= i_size_read(inode))
2638 goto out;
2639
2640 spin_lock(&info->lock);
2641 ptr = shmem_swp_entry(info, pgoff, NULL);
2642#ifdef CONFIG_SWAP
2643 if (ptr && ptr->val) {
2644 entry.val = ptr->val;
2645 page = find_get_page(&swapper_space, entry.val);
2646 } else
2647#endif
2648 page = find_get_page(inode->i_mapping, pgoff);
2649 if (ptr)
2650 shmem_swp_unmap(ptr);
2651 spin_unlock(&info->lock);
2652out:
2653 *pagep = page;
2654 *ent = entry;
2655}
2656#endif
2657
2658#else
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669#include <linux/ramfs.h>
2670
2671static struct file_system_type tmpfs_fs_type = {
2672 .name = "tmpfs",
2673 .mount = ramfs_mount,
2674 .kill_sb = kill_litter_super,
2675};
2676
2677int __init init_tmpfs(void)
2678{
2679 BUG_ON(register_filesystem(&tmpfs_fs_type) != 0);
2680
2681 shm_mnt = kern_mount(&tmpfs_fs_type);
2682 BUG_ON(IS_ERR(shm_mnt));
2683
2684 return 0;
2685}
2686
2687int shmem_unuse(swp_entry_t entry, struct page *page)
2688{
2689 return 0;
2690}
2691
2692int shmem_lock(struct file *file, int lock, struct user_struct *user)
2693{
2694 return 0;
2695}
2696
2697#ifdef CONFIG_CGROUP_MEM_RES_CTLR
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708void mem_cgroup_get_shmem_target(struct inode *inode, pgoff_t pgoff,
2709 struct page **pagep, swp_entry_t *ent)
2710{
2711 struct page *page = NULL;
2712
2713 if ((pgoff << PAGE_CACHE_SHIFT) >= i_size_read(inode))
2714 goto out;
2715 page = find_get_page(inode->i_mapping, pgoff);
2716out:
2717 *pagep = page;
2718 *ent = (swp_entry_t){ .val = 0 };
2719}
2720#endif
2721
2722#define shmem_vm_ops generic_file_vm_ops
2723#define shmem_file_operations ramfs_file_operations
2724#define shmem_get_inode(sb, dir, mode, dev, flags) ramfs_get_inode(sb, dir, mode, dev)
2725#define shmem_acct_size(flags, size) 0
2726#define shmem_unacct_size(flags, size) do {} while (0)
2727#define SHMEM_MAX_BYTES MAX_LFS_FILESIZE
2728
2729#endif
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739struct file *shmem_file_setup(const char *name, loff_t size, unsigned long flags)
2740{
2741 int error;
2742 struct file *file;
2743 struct inode *inode;
2744 struct path path;
2745 struct dentry *root;
2746 struct qstr this;
2747
2748 if (IS_ERR(shm_mnt))
2749 return (void *)shm_mnt;
2750
2751 if (size < 0 || size > SHMEM_MAX_BYTES)
2752 return ERR_PTR(-EINVAL);
2753
2754 if (shmem_acct_size(flags, size))
2755 return ERR_PTR(-ENOMEM);
2756
2757 error = -ENOMEM;
2758 this.name = name;
2759 this.len = strlen(name);
2760 this.hash = 0;
2761 root = shm_mnt->mnt_root;
2762 path.dentry = d_alloc(root, &this);
2763 if (!path.dentry)
2764 goto put_memory;
2765 path.mnt = mntget(shm_mnt);
2766
2767 error = -ENOSPC;
2768 inode = shmem_get_inode(root->d_sb, NULL, S_IFREG | S_IRWXUGO, 0, flags);
2769 if (!inode)
2770 goto put_dentry;
2771
2772 d_instantiate(path.dentry, inode);
2773 inode->i_size = size;
2774 inode->i_nlink = 0;
2775#ifndef CONFIG_MMU
2776 error = ramfs_nommu_expand_for_mapping(inode, size);
2777 if (error)
2778 goto put_dentry;
2779#endif
2780
2781 error = -ENFILE;
2782 file = alloc_file(&path, FMODE_WRITE | FMODE_READ,
2783 &shmem_file_operations);
2784 if (!file)
2785 goto put_dentry;
2786
2787 return file;
2788
2789put_dentry:
2790 path_put(&path);
2791put_memory:
2792 shmem_unacct_size(flags, size);
2793 return ERR_PTR(error);
2794}
2795EXPORT_SYMBOL_GPL(shmem_file_setup);
2796
2797
2798
2799
2800
2801int shmem_zero_setup(struct vm_area_struct *vma)
2802{
2803 struct file *file;
2804 loff_t size = vma->vm_end - vma->vm_start;
2805
2806 file = shmem_file_setup("dev/zero", size, vma->vm_flags);
2807 if (IS_ERR(file))
2808 return PTR_ERR(file);
2809
2810 if (vma->vm_file)
2811 fput(vma->vm_file);
2812 vma->vm_file = file;
2813 vma->vm_ops = &shmem_vm_ops;
2814 vma->vm_flags |= VM_CAN_NONLINEAR;
2815 return 0;
2816}
2817