1
2
3
4
5
6#include <linux/export.h>
7#include <linux/fs.h>
8#include <linux/mm.h>
9#include <linux/backing-dev.h>
10#include <linux/hash.h>
11#include <linux/swap.h>
12#include <linux/security.h>
13#include <linux/cdev.h>
14#include <linux/memblock.h>
15#include <linux/fscrypt.h>
16#include <linux/fsnotify.h>
17#include <linux/mount.h>
18#include <linux/posix_acl.h>
19#include <linux/prefetch.h>
20#include <linux/buffer_head.h>
21#include <linux/ratelimit.h>
22#include <linux/list_lru.h>
23#include <linux/iversion.h>
24#include <trace/events/writeback.h>
25#include "internal.h"
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58static unsigned int i_hash_mask __read_mostly;
59static unsigned int i_hash_shift __read_mostly;
60static struct hlist_head *inode_hashtable __read_mostly;
61static __cacheline_aligned_in_smp DEFINE_SPINLOCK(inode_hash_lock);
62
63
64
65
66
67const struct address_space_operations empty_aops = {
68};
69EXPORT_SYMBOL(empty_aops);
70
71
72
73
74struct inodes_stat_t inodes_stat;
75
76static DEFINE_PER_CPU(unsigned long, nr_inodes);
77static DEFINE_PER_CPU(unsigned long, nr_unused);
78
79static struct kmem_cache *inode_cachep __read_mostly;
80
81static long get_nr_inodes(void)
82{
83 int i;
84 long sum = 0;
85 for_each_possible_cpu(i)
86 sum += per_cpu(nr_inodes, i);
87 return sum < 0 ? 0 : sum;
88}
89
90static inline long get_nr_inodes_unused(void)
91{
92 int i;
93 long sum = 0;
94 for_each_possible_cpu(i)
95 sum += per_cpu(nr_unused, i);
96 return sum < 0 ? 0 : sum;
97}
98
99long get_nr_dirty_inodes(void)
100{
101
102 long nr_dirty = get_nr_inodes() - get_nr_inodes_unused();
103 return nr_dirty > 0 ? nr_dirty : 0;
104}
105
106
107
108
109#ifdef CONFIG_SYSCTL
110int proc_nr_inodes(struct ctl_table *table, int write,
111 void *buffer, size_t *lenp, loff_t *ppos)
112{
113 inodes_stat.nr_inodes = get_nr_inodes();
114 inodes_stat.nr_unused = get_nr_inodes_unused();
115 return proc_doulongvec_minmax(table, write, buffer, lenp, ppos);
116}
117#endif
118
119static int no_open(struct inode *inode, struct file *file)
120{
121 return -ENXIO;
122}
123
124
125
126
127
128
129
130
131
132int inode_init_always(struct super_block *sb, struct inode *inode)
133{
134 static const struct inode_operations empty_iops;
135 static const struct file_operations no_open_fops = {.open = no_open};
136 struct address_space *const mapping = &inode->i_data;
137
138 inode->i_sb = sb;
139 inode->i_blkbits = sb->s_blocksize_bits;
140 inode->i_flags = 0;
141 atomic64_set(&inode->i_sequence, 0);
142 atomic_set(&inode->i_count, 1);
143 inode->i_op = &empty_iops;
144 inode->i_fop = &no_open_fops;
145 inode->__i_nlink = 1;
146 inode->i_opflags = 0;
147 if (sb->s_xattr)
148 inode->i_opflags |= IOP_XATTR;
149 i_uid_write(inode, 0);
150 i_gid_write(inode, 0);
151 atomic_set(&inode->i_writecount, 0);
152 inode->i_size = 0;
153 inode->i_write_hint = WRITE_LIFE_NOT_SET;
154 inode->i_blocks = 0;
155 inode->i_bytes = 0;
156 inode->i_generation = 0;
157 inode->i_pipe = NULL;
158 inode->i_bdev = NULL;
159 inode->i_cdev = NULL;
160 inode->i_link = NULL;
161 inode->i_dir_seq = 0;
162 inode->i_rdev = 0;
163 inode->dirtied_when = 0;
164
165#ifdef CONFIG_CGROUP_WRITEBACK
166 inode->i_wb_frn_winner = 0;
167 inode->i_wb_frn_avg_time = 0;
168 inode->i_wb_frn_history = 0;
169#endif
170
171 if (security_inode_alloc(inode))
172 goto out;
173 spin_lock_init(&inode->i_lock);
174 lockdep_set_class(&inode->i_lock, &sb->s_type->i_lock_key);
175
176 init_rwsem(&inode->i_rwsem);
177 lockdep_set_class(&inode->i_rwsem, &sb->s_type->i_mutex_key);
178
179 atomic_set(&inode->i_dio_count, 0);
180
181 mapping->a_ops = &empty_aops;
182 mapping->host = inode;
183 mapping->flags = 0;
184 mapping->wb_err = 0;
185 atomic_set(&mapping->i_mmap_writable, 0);
186#ifdef CONFIG_READ_ONLY_THP_FOR_FS
187 atomic_set(&mapping->nr_thps, 0);
188#endif
189 mapping_set_gfp_mask(mapping, GFP_HIGHUSER_MOVABLE);
190 mapping->private_data = NULL;
191 mapping->writeback_index = 0;
192 inode->i_private = NULL;
193 inode->i_mapping = mapping;
194 INIT_HLIST_HEAD(&inode->i_dentry);
195#ifdef CONFIG_FS_POSIX_ACL
196 inode->i_acl = inode->i_default_acl = ACL_NOT_CACHED;
197#endif
198
199#ifdef CONFIG_FSNOTIFY
200 inode->i_fsnotify_mask = 0;
201#endif
202 inode->i_flctx = NULL;
203 this_cpu_inc(nr_inodes);
204
205 return 0;
206out:
207 return -ENOMEM;
208}
209EXPORT_SYMBOL(inode_init_always);
210
211void free_inode_nonrcu(struct inode *inode)
212{
213 kmem_cache_free(inode_cachep, inode);
214}
215EXPORT_SYMBOL(free_inode_nonrcu);
216
217static void i_callback(struct rcu_head *head)
218{
219 struct inode *inode = container_of(head, struct inode, i_rcu);
220 if (inode->free_inode)
221 inode->free_inode(inode);
222 else
223 free_inode_nonrcu(inode);
224}
225
226static struct inode *alloc_inode(struct super_block *sb)
227{
228 const struct super_operations *ops = sb->s_op;
229 struct inode *inode;
230
231 if (ops->alloc_inode)
232 inode = ops->alloc_inode(sb);
233 else
234 inode = kmem_cache_alloc(inode_cachep, GFP_KERNEL);
235
236 if (!inode)
237 return NULL;
238
239 if (unlikely(inode_init_always(sb, inode))) {
240 if (ops->destroy_inode) {
241 ops->destroy_inode(inode);
242 if (!ops->free_inode)
243 return NULL;
244 }
245 inode->free_inode = ops->free_inode;
246 i_callback(&inode->i_rcu);
247 return NULL;
248 }
249
250 return inode;
251}
252
253void __destroy_inode(struct inode *inode)
254{
255 BUG_ON(inode_has_buffers(inode));
256 inode_detach_wb(inode);
257 security_inode_free(inode);
258 fsnotify_inode_delete(inode);
259 locks_free_lock_context(inode);
260 if (!inode->i_nlink) {
261 WARN_ON(atomic_long_read(&inode->i_sb->s_remove_count) == 0);
262 atomic_long_dec(&inode->i_sb->s_remove_count);
263 }
264
265#ifdef CONFIG_FS_POSIX_ACL
266 if (inode->i_acl && !is_uncached_acl(inode->i_acl))
267 posix_acl_release(inode->i_acl);
268 if (inode->i_default_acl && !is_uncached_acl(inode->i_default_acl))
269 posix_acl_release(inode->i_default_acl);
270#endif
271 this_cpu_dec(nr_inodes);
272}
273EXPORT_SYMBOL(__destroy_inode);
274
275static void destroy_inode(struct inode *inode)
276{
277 const struct super_operations *ops = inode->i_sb->s_op;
278
279 BUG_ON(!list_empty(&inode->i_lru));
280 __destroy_inode(inode);
281 if (ops->destroy_inode) {
282 ops->destroy_inode(inode);
283 if (!ops->free_inode)
284 return;
285 }
286 inode->free_inode = ops->free_inode;
287 call_rcu(&inode->i_rcu, i_callback);
288}
289
290
291
292
293
294
295
296
297
298
299
300
301void drop_nlink(struct inode *inode)
302{
303 WARN_ON(inode->i_nlink == 0);
304 inode->__i_nlink--;
305 if (!inode->i_nlink)
306 atomic_long_inc(&inode->i_sb->s_remove_count);
307}
308EXPORT_SYMBOL(drop_nlink);
309
310
311
312
313
314
315
316
317
318void clear_nlink(struct inode *inode)
319{
320 if (inode->i_nlink) {
321 inode->__i_nlink = 0;
322 atomic_long_inc(&inode->i_sb->s_remove_count);
323 }
324}
325EXPORT_SYMBOL(clear_nlink);
326
327
328
329
330
331
332
333
334
335void set_nlink(struct inode *inode, unsigned int nlink)
336{
337 if (!nlink) {
338 clear_nlink(inode);
339 } else {
340
341 if (inode->i_nlink == 0)
342 atomic_long_dec(&inode->i_sb->s_remove_count);
343
344 inode->__i_nlink = nlink;
345 }
346}
347EXPORT_SYMBOL(set_nlink);
348
349
350
351
352
353
354
355
356
357void inc_nlink(struct inode *inode)
358{
359 if (unlikely(inode->i_nlink == 0)) {
360 WARN_ON(!(inode->i_state & I_LINKABLE));
361 atomic_long_dec(&inode->i_sb->s_remove_count);
362 }
363
364 inode->__i_nlink++;
365}
366EXPORT_SYMBOL(inc_nlink);
367
368static void __address_space_init_once(struct address_space *mapping)
369{
370 xa_init_flags(&mapping->i_pages, XA_FLAGS_LOCK_IRQ | XA_FLAGS_ACCOUNT);
371 init_rwsem(&mapping->i_mmap_rwsem);
372 INIT_LIST_HEAD(&mapping->private_list);
373 spin_lock_init(&mapping->private_lock);
374 mapping->i_mmap = RB_ROOT_CACHED;
375}
376
377void address_space_init_once(struct address_space *mapping)
378{
379 memset(mapping, 0, sizeof(*mapping));
380 __address_space_init_once(mapping);
381}
382EXPORT_SYMBOL(address_space_init_once);
383
384
385
386
387
388
389void inode_init_once(struct inode *inode)
390{
391 memset(inode, 0, sizeof(*inode));
392 INIT_HLIST_NODE(&inode->i_hash);
393 INIT_LIST_HEAD(&inode->i_devices);
394 INIT_LIST_HEAD(&inode->i_io_list);
395 INIT_LIST_HEAD(&inode->i_wb_list);
396 INIT_LIST_HEAD(&inode->i_lru);
397 __address_space_init_once(&inode->i_data);
398 i_size_ordered_init(inode);
399}
400EXPORT_SYMBOL(inode_init_once);
401
402static void init_once(void *foo)
403{
404 struct inode *inode = (struct inode *) foo;
405
406 inode_init_once(inode);
407}
408
409
410
411
412void __iget(struct inode *inode)
413{
414 atomic_inc(&inode->i_count);
415}
416
417
418
419
420void ihold(struct inode *inode)
421{
422 WARN_ON(atomic_inc_return(&inode->i_count) < 2);
423}
424EXPORT_SYMBOL(ihold);
425
426static void inode_lru_list_add(struct inode *inode)
427{
428 if (list_lru_add(&inode->i_sb->s_inode_lru, &inode->i_lru))
429 this_cpu_inc(nr_unused);
430 else
431 inode->i_state |= I_REFERENCED;
432}
433
434
435
436
437
438
439void inode_add_lru(struct inode *inode)
440{
441 if (!(inode->i_state & (I_DIRTY_ALL | I_SYNC |
442 I_FREEING | I_WILL_FREE)) &&
443 !atomic_read(&inode->i_count) && inode->i_sb->s_flags & SB_ACTIVE)
444 inode_lru_list_add(inode);
445}
446
447
448static void inode_lru_list_del(struct inode *inode)
449{
450
451 if (list_lru_del(&inode->i_sb->s_inode_lru, &inode->i_lru))
452 this_cpu_dec(nr_unused);
453}
454
455
456
457
458
459void inode_sb_list_add(struct inode *inode)
460{
461 spin_lock(&inode->i_sb->s_inode_list_lock);
462 list_add(&inode->i_sb_list, &inode->i_sb->s_inodes);
463 spin_unlock(&inode->i_sb->s_inode_list_lock);
464}
465EXPORT_SYMBOL_GPL(inode_sb_list_add);
466
467static inline void inode_sb_list_del(struct inode *inode)
468{
469 if (!list_empty(&inode->i_sb_list)) {
470 spin_lock(&inode->i_sb->s_inode_list_lock);
471 list_del_init(&inode->i_sb_list);
472 spin_unlock(&inode->i_sb->s_inode_list_lock);
473 }
474}
475
476static unsigned long hash(struct super_block *sb, unsigned long hashval)
477{
478 unsigned long tmp;
479
480 tmp = (hashval * (unsigned long)sb) ^ (GOLDEN_RATIO_PRIME + hashval) /
481 L1_CACHE_BYTES;
482 tmp = tmp ^ ((tmp ^ GOLDEN_RATIO_PRIME) >> i_hash_shift);
483 return tmp & i_hash_mask;
484}
485
486
487
488
489
490
491
492
493
494void __insert_inode_hash(struct inode *inode, unsigned long hashval)
495{
496 struct hlist_head *b = inode_hashtable + hash(inode->i_sb, hashval);
497
498 spin_lock(&inode_hash_lock);
499 spin_lock(&inode->i_lock);
500 hlist_add_head_rcu(&inode->i_hash, b);
501 spin_unlock(&inode->i_lock);
502 spin_unlock(&inode_hash_lock);
503}
504EXPORT_SYMBOL(__insert_inode_hash);
505
506
507
508
509
510
511
512void __remove_inode_hash(struct inode *inode)
513{
514 spin_lock(&inode_hash_lock);
515 spin_lock(&inode->i_lock);
516 hlist_del_init_rcu(&inode->i_hash);
517 spin_unlock(&inode->i_lock);
518 spin_unlock(&inode_hash_lock);
519}
520EXPORT_SYMBOL(__remove_inode_hash);
521
522void clear_inode(struct inode *inode)
523{
524
525
526
527
528
529 xa_lock_irq(&inode->i_data.i_pages);
530 BUG_ON(inode->i_data.nrpages);
531 BUG_ON(inode->i_data.nrexceptional);
532 xa_unlock_irq(&inode->i_data.i_pages);
533 BUG_ON(!list_empty(&inode->i_data.private_list));
534 BUG_ON(!(inode->i_state & I_FREEING));
535 BUG_ON(inode->i_state & I_CLEAR);
536 BUG_ON(!list_empty(&inode->i_wb_list));
537
538 inode->i_state = I_FREEING | I_CLEAR;
539}
540EXPORT_SYMBOL(clear_inode);
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555static void evict(struct inode *inode)
556{
557 const struct super_operations *op = inode->i_sb->s_op;
558
559 BUG_ON(!(inode->i_state & I_FREEING));
560 BUG_ON(!list_empty(&inode->i_lru));
561
562 if (!list_empty(&inode->i_io_list))
563 inode_io_list_del(inode);
564
565 inode_sb_list_del(inode);
566
567
568
569
570
571
572
573 inode_wait_for_writeback(inode);
574
575 if (op->evict_inode) {
576 op->evict_inode(inode);
577 } else {
578 truncate_inode_pages_final(&inode->i_data);
579 clear_inode(inode);
580 }
581 if (S_ISBLK(inode->i_mode) && inode->i_bdev)
582 bd_forget(inode);
583 if (S_ISCHR(inode->i_mode) && inode->i_cdev)
584 cd_forget(inode);
585
586 remove_inode_hash(inode);
587
588 spin_lock(&inode->i_lock);
589 wake_up_bit(&inode->i_state, __I_NEW);
590 BUG_ON(inode->i_state != (I_FREEING | I_CLEAR));
591 spin_unlock(&inode->i_lock);
592
593 destroy_inode(inode);
594}
595
596
597
598
599
600
601
602
603static void dispose_list(struct list_head *head)
604{
605 while (!list_empty(head)) {
606 struct inode *inode;
607
608 inode = list_first_entry(head, struct inode, i_lru);
609 list_del_init(&inode->i_lru);
610
611 evict(inode);
612 cond_resched();
613 }
614}
615
616
617
618
619
620
621
622
623
624
625void evict_inodes(struct super_block *sb)
626{
627 struct inode *inode, *next;
628 LIST_HEAD(dispose);
629
630again:
631 spin_lock(&sb->s_inode_list_lock);
632 list_for_each_entry_safe(inode, next, &sb->s_inodes, i_sb_list) {
633 if (atomic_read(&inode->i_count))
634 continue;
635
636 spin_lock(&inode->i_lock);
637 if (inode->i_state & (I_NEW | I_FREEING | I_WILL_FREE)) {
638 spin_unlock(&inode->i_lock);
639 continue;
640 }
641
642 inode->i_state |= I_FREEING;
643 inode_lru_list_del(inode);
644 spin_unlock(&inode->i_lock);
645 list_add(&inode->i_lru, &dispose);
646
647
648
649
650
651
652 if (need_resched()) {
653 spin_unlock(&sb->s_inode_list_lock);
654 cond_resched();
655 dispose_list(&dispose);
656 goto again;
657 }
658 }
659 spin_unlock(&sb->s_inode_list_lock);
660
661 dispose_list(&dispose);
662}
663EXPORT_SYMBOL_GPL(evict_inodes);
664
665
666
667
668
669
670
671
672
673
674
675int invalidate_inodes(struct super_block *sb, bool kill_dirty)
676{
677 int busy = 0;
678 struct inode *inode, *next;
679 LIST_HEAD(dispose);
680
681again:
682 spin_lock(&sb->s_inode_list_lock);
683 list_for_each_entry_safe(inode, next, &sb->s_inodes, i_sb_list) {
684 spin_lock(&inode->i_lock);
685 if (inode->i_state & (I_NEW | I_FREEING | I_WILL_FREE)) {
686 spin_unlock(&inode->i_lock);
687 continue;
688 }
689 if (inode->i_state & I_DIRTY_ALL && !kill_dirty) {
690 spin_unlock(&inode->i_lock);
691 busy = 1;
692 continue;
693 }
694 if (atomic_read(&inode->i_count)) {
695 spin_unlock(&inode->i_lock);
696 busy = 1;
697 continue;
698 }
699
700 inode->i_state |= I_FREEING;
701 inode_lru_list_del(inode);
702 spin_unlock(&inode->i_lock);
703 list_add(&inode->i_lru, &dispose);
704 if (need_resched()) {
705 spin_unlock(&sb->s_inode_list_lock);
706 cond_resched();
707 dispose_list(&dispose);
708 goto again;
709 }
710 }
711 spin_unlock(&sb->s_inode_list_lock);
712
713 dispose_list(&dispose);
714
715 return busy;
716}
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733static enum lru_status inode_lru_isolate(struct list_head *item,
734 struct list_lru_one *lru, spinlock_t *lru_lock, void *arg)
735{
736 struct list_head *freeable = arg;
737 struct inode *inode = container_of(item, struct inode, i_lru);
738
739
740
741
742
743 if (!spin_trylock(&inode->i_lock))
744 return LRU_SKIP;
745
746
747
748
749
750 if (atomic_read(&inode->i_count) ||
751 (inode->i_state & ~I_REFERENCED)) {
752 list_lru_isolate(lru, &inode->i_lru);
753 spin_unlock(&inode->i_lock);
754 this_cpu_dec(nr_unused);
755 return LRU_REMOVED;
756 }
757
758
759 if (inode->i_state & I_REFERENCED) {
760 inode->i_state &= ~I_REFERENCED;
761 spin_unlock(&inode->i_lock);
762 return LRU_ROTATE;
763 }
764
765 if (inode_has_buffers(inode) || inode->i_data.nrpages) {
766 __iget(inode);
767 spin_unlock(&inode->i_lock);
768 spin_unlock(lru_lock);
769 if (remove_inode_buffers(inode)) {
770 unsigned long reap;
771 reap = invalidate_mapping_pages(&inode->i_data, 0, -1);
772 if (current_is_kswapd())
773 __count_vm_events(KSWAPD_INODESTEAL, reap);
774 else
775 __count_vm_events(PGINODESTEAL, reap);
776 if (current->reclaim_state)
777 current->reclaim_state->reclaimed_slab += reap;
778 }
779 iput(inode);
780 spin_lock(lru_lock);
781 return LRU_RETRY;
782 }
783
784 WARN_ON(inode->i_state & I_NEW);
785 inode->i_state |= I_FREEING;
786 list_lru_isolate_move(lru, &inode->i_lru, freeable);
787 spin_unlock(&inode->i_lock);
788
789 this_cpu_dec(nr_unused);
790 return LRU_REMOVED;
791}
792
793
794
795
796
797
798
799long prune_icache_sb(struct super_block *sb, struct shrink_control *sc)
800{
801 LIST_HEAD(freeable);
802 long freed;
803
804 freed = list_lru_shrink_walk(&sb->s_inode_lru, sc,
805 inode_lru_isolate, &freeable);
806 dispose_list(&freeable);
807 return freed;
808}
809
810static void __wait_on_freeing_inode(struct inode *inode);
811
812
813
814static struct inode *find_inode(struct super_block *sb,
815 struct hlist_head *head,
816 int (*test)(struct inode *, void *),
817 void *data)
818{
819 struct inode *inode = NULL;
820
821repeat:
822 hlist_for_each_entry(inode, head, i_hash) {
823 if (inode->i_sb != sb)
824 continue;
825 if (!test(inode, data))
826 continue;
827 spin_lock(&inode->i_lock);
828 if (inode->i_state & (I_FREEING|I_WILL_FREE)) {
829 __wait_on_freeing_inode(inode);
830 goto repeat;
831 }
832 if (unlikely(inode->i_state & I_CREATING)) {
833 spin_unlock(&inode->i_lock);
834 return ERR_PTR(-ESTALE);
835 }
836 __iget(inode);
837 spin_unlock(&inode->i_lock);
838 return inode;
839 }
840 return NULL;
841}
842
843
844
845
846
847static struct inode *find_inode_fast(struct super_block *sb,
848 struct hlist_head *head, unsigned long ino)
849{
850 struct inode *inode = NULL;
851
852repeat:
853 hlist_for_each_entry(inode, head, i_hash) {
854 if (inode->i_ino != ino)
855 continue;
856 if (inode->i_sb != sb)
857 continue;
858 spin_lock(&inode->i_lock);
859 if (inode->i_state & (I_FREEING|I_WILL_FREE)) {
860 __wait_on_freeing_inode(inode);
861 goto repeat;
862 }
863 if (unlikely(inode->i_state & I_CREATING)) {
864 spin_unlock(&inode->i_lock);
865 return ERR_PTR(-ESTALE);
866 }
867 __iget(inode);
868 spin_unlock(&inode->i_lock);
869 return inode;
870 }
871 return NULL;
872}
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889#define LAST_INO_BATCH 1024
890static DEFINE_PER_CPU(unsigned int, last_ino);
891
892unsigned int get_next_ino(void)
893{
894 unsigned int *p = &get_cpu_var(last_ino);
895 unsigned int res = *p;
896
897#ifdef CONFIG_SMP
898 if (unlikely((res & (LAST_INO_BATCH-1)) == 0)) {
899 static atomic_t shared_last_ino;
900 int next = atomic_add_return(LAST_INO_BATCH, &shared_last_ino);
901
902 res = next - LAST_INO_BATCH;
903 }
904#endif
905
906 res++;
907
908 if (unlikely(!res))
909 res++;
910 *p = res;
911 put_cpu_var(last_ino);
912 return res;
913}
914EXPORT_SYMBOL(get_next_ino);
915
916
917
918
919
920
921
922
923
924
925
926struct inode *new_inode_pseudo(struct super_block *sb)
927{
928 struct inode *inode = alloc_inode(sb);
929
930 if (inode) {
931 spin_lock(&inode->i_lock);
932 inode->i_state = 0;
933 spin_unlock(&inode->i_lock);
934 INIT_LIST_HEAD(&inode->i_sb_list);
935 }
936 return inode;
937}
938
939
940
941
942
943
944
945
946
947
948
949
950
951struct inode *new_inode(struct super_block *sb)
952{
953 struct inode *inode;
954
955 spin_lock_prefetch(&sb->s_inode_list_lock);
956
957 inode = new_inode_pseudo(sb);
958 if (inode)
959 inode_sb_list_add(inode);
960 return inode;
961}
962EXPORT_SYMBOL(new_inode);
963
964#ifdef CONFIG_DEBUG_LOCK_ALLOC
965void lockdep_annotate_inode_mutex_key(struct inode *inode)
966{
967 if (S_ISDIR(inode->i_mode)) {
968 struct file_system_type *type = inode->i_sb->s_type;
969
970
971 if (lockdep_match_class(&inode->i_rwsem, &type->i_mutex_key)) {
972
973
974
975
976 init_rwsem(&inode->i_rwsem);
977 lockdep_set_class(&inode->i_rwsem,
978 &type->i_mutex_dir_key);
979 }
980 }
981}
982EXPORT_SYMBOL(lockdep_annotate_inode_mutex_key);
983#endif
984
985
986
987
988
989
990
991
992void unlock_new_inode(struct inode *inode)
993{
994 lockdep_annotate_inode_mutex_key(inode);
995 spin_lock(&inode->i_lock);
996 WARN_ON(!(inode->i_state & I_NEW));
997 inode->i_state &= ~I_NEW & ~I_CREATING;
998 smp_mb();
999 wake_up_bit(&inode->i_state, __I_NEW);
1000 spin_unlock(&inode->i_lock);
1001}
1002EXPORT_SYMBOL(unlock_new_inode);
1003
1004void discard_new_inode(struct inode *inode)
1005{
1006 lockdep_annotate_inode_mutex_key(inode);
1007 spin_lock(&inode->i_lock);
1008 WARN_ON(!(inode->i_state & I_NEW));
1009 inode->i_state &= ~I_NEW;
1010 smp_mb();
1011 wake_up_bit(&inode->i_state, __I_NEW);
1012 spin_unlock(&inode->i_lock);
1013 iput(inode);
1014}
1015EXPORT_SYMBOL(discard_new_inode);
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026void lock_two_nondirectories(struct inode *inode1, struct inode *inode2)
1027{
1028 if (inode1 > inode2)
1029 swap(inode1, inode2);
1030
1031 if (inode1 && !S_ISDIR(inode1->i_mode))
1032 inode_lock(inode1);
1033 if (inode2 && !S_ISDIR(inode2->i_mode) && inode2 != inode1)
1034 inode_lock_nested(inode2, I_MUTEX_NONDIR2);
1035}
1036EXPORT_SYMBOL(lock_two_nondirectories);
1037
1038
1039
1040
1041
1042
1043void unlock_two_nondirectories(struct inode *inode1, struct inode *inode2)
1044{
1045 if (inode1 && !S_ISDIR(inode1->i_mode))
1046 inode_unlock(inode1);
1047 if (inode2 && !S_ISDIR(inode2->i_mode) && inode2 != inode1)
1048 inode_unlock(inode2);
1049}
1050EXPORT_SYMBOL(unlock_two_nondirectories);
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072struct inode *inode_insert5(struct inode *inode, unsigned long hashval,
1073 int (*test)(struct inode *, void *),
1074 int (*set)(struct inode *, void *), void *data)
1075{
1076 struct hlist_head *head = inode_hashtable + hash(inode->i_sb, hashval);
1077 struct inode *old;
1078 bool creating = inode->i_state & I_CREATING;
1079
1080again:
1081 spin_lock(&inode_hash_lock);
1082 old = find_inode(inode->i_sb, head, test, data);
1083 if (unlikely(old)) {
1084
1085
1086
1087
1088 spin_unlock(&inode_hash_lock);
1089 if (IS_ERR(old))
1090 return NULL;
1091 wait_on_inode(old);
1092 if (unlikely(inode_unhashed(old))) {
1093 iput(old);
1094 goto again;
1095 }
1096 return old;
1097 }
1098
1099 if (set && unlikely(set(inode, data))) {
1100 inode = NULL;
1101 goto unlock;
1102 }
1103
1104
1105
1106
1107
1108 spin_lock(&inode->i_lock);
1109 inode->i_state |= I_NEW;
1110 hlist_add_head_rcu(&inode->i_hash, head);
1111 spin_unlock(&inode->i_lock);
1112 if (!creating)
1113 inode_sb_list_add(inode);
1114unlock:
1115 spin_unlock(&inode_hash_lock);
1116
1117 return inode;
1118}
1119EXPORT_SYMBOL(inode_insert5);
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141struct inode *iget5_locked(struct super_block *sb, unsigned long hashval,
1142 int (*test)(struct inode *, void *),
1143 int (*set)(struct inode *, void *), void *data)
1144{
1145 struct inode *inode = ilookup5(sb, hashval, test, data);
1146
1147 if (!inode) {
1148 struct inode *new = alloc_inode(sb);
1149
1150 if (new) {
1151 new->i_state = 0;
1152 inode = inode_insert5(new, hashval, test, set, data);
1153 if (unlikely(inode != new))
1154 destroy_inode(new);
1155 }
1156 }
1157 return inode;
1158}
1159EXPORT_SYMBOL(iget5_locked);
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174struct inode *iget_locked(struct super_block *sb, unsigned long ino)
1175{
1176 struct hlist_head *head = inode_hashtable + hash(sb, ino);
1177 struct inode *inode;
1178again:
1179 spin_lock(&inode_hash_lock);
1180 inode = find_inode_fast(sb, head, ino);
1181 spin_unlock(&inode_hash_lock);
1182 if (inode) {
1183 if (IS_ERR(inode))
1184 return NULL;
1185 wait_on_inode(inode);
1186 if (unlikely(inode_unhashed(inode))) {
1187 iput(inode);
1188 goto again;
1189 }
1190 return inode;
1191 }
1192
1193 inode = alloc_inode(sb);
1194 if (inode) {
1195 struct inode *old;
1196
1197 spin_lock(&inode_hash_lock);
1198
1199 old = find_inode_fast(sb, head, ino);
1200 if (!old) {
1201 inode->i_ino = ino;
1202 spin_lock(&inode->i_lock);
1203 inode->i_state = I_NEW;
1204 hlist_add_head_rcu(&inode->i_hash, head);
1205 spin_unlock(&inode->i_lock);
1206 inode_sb_list_add(inode);
1207 spin_unlock(&inode_hash_lock);
1208
1209
1210
1211
1212 return inode;
1213 }
1214
1215
1216
1217
1218
1219
1220 spin_unlock(&inode_hash_lock);
1221 destroy_inode(inode);
1222 if (IS_ERR(old))
1223 return NULL;
1224 inode = old;
1225 wait_on_inode(inode);
1226 if (unlikely(inode_unhashed(inode))) {
1227 iput(inode);
1228 goto again;
1229 }
1230 }
1231 return inode;
1232}
1233EXPORT_SYMBOL(iget_locked);
1234
1235
1236
1237
1238
1239
1240
1241
1242static int test_inode_iunique(struct super_block *sb, unsigned long ino)
1243{
1244 struct hlist_head *b = inode_hashtable + hash(sb, ino);
1245 struct inode *inode;
1246
1247 hlist_for_each_entry_rcu(inode, b, i_hash) {
1248 if (inode->i_ino == ino && inode->i_sb == sb)
1249 return 0;
1250 }
1251 return 1;
1252}
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268ino_t iunique(struct super_block *sb, ino_t max_reserved)
1269{
1270
1271
1272
1273
1274
1275 static DEFINE_SPINLOCK(iunique_lock);
1276 static unsigned int counter;
1277 ino_t res;
1278
1279 rcu_read_lock();
1280 spin_lock(&iunique_lock);
1281 do {
1282 if (counter <= max_reserved)
1283 counter = max_reserved + 1;
1284 res = counter++;
1285 } while (!test_inode_iunique(sb, res));
1286 spin_unlock(&iunique_lock);
1287 rcu_read_unlock();
1288
1289 return res;
1290}
1291EXPORT_SYMBOL(iunique);
1292
1293struct inode *igrab(struct inode *inode)
1294{
1295 spin_lock(&inode->i_lock);
1296 if (!(inode->i_state & (I_FREEING|I_WILL_FREE))) {
1297 __iget(inode);
1298 spin_unlock(&inode->i_lock);
1299 } else {
1300 spin_unlock(&inode->i_lock);
1301
1302
1303
1304
1305
1306 inode = NULL;
1307 }
1308 return inode;
1309}
1310EXPORT_SYMBOL(igrab);
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328struct inode *ilookup5_nowait(struct super_block *sb, unsigned long hashval,
1329 int (*test)(struct inode *, void *), void *data)
1330{
1331 struct hlist_head *head = inode_hashtable + hash(sb, hashval);
1332 struct inode *inode;
1333
1334 spin_lock(&inode_hash_lock);
1335 inode = find_inode(sb, head, test, data);
1336 spin_unlock(&inode_hash_lock);
1337
1338 return IS_ERR(inode) ? NULL : inode;
1339}
1340EXPORT_SYMBOL(ilookup5_nowait);
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359struct inode *ilookup5(struct super_block *sb, unsigned long hashval,
1360 int (*test)(struct inode *, void *), void *data)
1361{
1362 struct inode *inode;
1363again:
1364 inode = ilookup5_nowait(sb, hashval, test, data);
1365 if (inode) {
1366 wait_on_inode(inode);
1367 if (unlikely(inode_unhashed(inode))) {
1368 iput(inode);
1369 goto again;
1370 }
1371 }
1372 return inode;
1373}
1374EXPORT_SYMBOL(ilookup5);
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384struct inode *ilookup(struct super_block *sb, unsigned long ino)
1385{
1386 struct hlist_head *head = inode_hashtable + hash(sb, ino);
1387 struct inode *inode;
1388again:
1389 spin_lock(&inode_hash_lock);
1390 inode = find_inode_fast(sb, head, ino);
1391 spin_unlock(&inode_hash_lock);
1392
1393 if (inode) {
1394 if (IS_ERR(inode))
1395 return NULL;
1396 wait_on_inode(inode);
1397 if (unlikely(inode_unhashed(inode))) {
1398 iput(inode);
1399 goto again;
1400 }
1401 }
1402 return inode;
1403}
1404EXPORT_SYMBOL(ilookup);
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429struct inode *find_inode_nowait(struct super_block *sb,
1430 unsigned long hashval,
1431 int (*match)(struct inode *, unsigned long,
1432 void *),
1433 void *data)
1434{
1435 struct hlist_head *head = inode_hashtable + hash(sb, hashval);
1436 struct inode *inode, *ret_inode = NULL;
1437 int mval;
1438
1439 spin_lock(&inode_hash_lock);
1440 hlist_for_each_entry(inode, head, i_hash) {
1441 if (inode->i_sb != sb)
1442 continue;
1443 mval = match(inode, hashval, data);
1444 if (mval == 0)
1445 continue;
1446 if (mval == 1)
1447 ret_inode = inode;
1448 goto out;
1449 }
1450out:
1451 spin_unlock(&inode_hash_lock);
1452 return ret_inode;
1453}
1454EXPORT_SYMBOL(find_inode_nowait);
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477struct inode *find_inode_rcu(struct super_block *sb, unsigned long hashval,
1478 int (*test)(struct inode *, void *), void *data)
1479{
1480 struct hlist_head *head = inode_hashtable + hash(sb, hashval);
1481 struct inode *inode;
1482
1483 RCU_LOCKDEP_WARN(!rcu_read_lock_held(),
1484 "suspicious find_inode_rcu() usage");
1485
1486 hlist_for_each_entry_rcu(inode, head, i_hash) {
1487 if (inode->i_sb == sb &&
1488 !(READ_ONCE(inode->i_state) & (I_FREEING | I_WILL_FREE)) &&
1489 test(inode, data))
1490 return inode;
1491 }
1492 return NULL;
1493}
1494EXPORT_SYMBOL(find_inode_rcu);
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515struct inode *find_inode_by_ino_rcu(struct super_block *sb,
1516 unsigned long ino)
1517{
1518 struct hlist_head *head = inode_hashtable + hash(sb, ino);
1519 struct inode *inode;
1520
1521 RCU_LOCKDEP_WARN(!rcu_read_lock_held(),
1522 "suspicious find_inode_by_ino_rcu() usage");
1523
1524 hlist_for_each_entry_rcu(inode, head, i_hash) {
1525 if (inode->i_ino == ino &&
1526 inode->i_sb == sb &&
1527 !(READ_ONCE(inode->i_state) & (I_FREEING | I_WILL_FREE)))
1528 return inode;
1529 }
1530 return NULL;
1531}
1532EXPORT_SYMBOL(find_inode_by_ino_rcu);
1533
1534int insert_inode_locked(struct inode *inode)
1535{
1536 struct super_block *sb = inode->i_sb;
1537 ino_t ino = inode->i_ino;
1538 struct hlist_head *head = inode_hashtable + hash(sb, ino);
1539
1540 while (1) {
1541 struct inode *old = NULL;
1542 spin_lock(&inode_hash_lock);
1543 hlist_for_each_entry(old, head, i_hash) {
1544 if (old->i_ino != ino)
1545 continue;
1546 if (old->i_sb != sb)
1547 continue;
1548 spin_lock(&old->i_lock);
1549 if (old->i_state & (I_FREEING|I_WILL_FREE)) {
1550 spin_unlock(&old->i_lock);
1551 continue;
1552 }
1553 break;
1554 }
1555 if (likely(!old)) {
1556 spin_lock(&inode->i_lock);
1557 inode->i_state |= I_NEW | I_CREATING;
1558 hlist_add_head_rcu(&inode->i_hash, head);
1559 spin_unlock(&inode->i_lock);
1560 spin_unlock(&inode_hash_lock);
1561 return 0;
1562 }
1563 if (unlikely(old->i_state & I_CREATING)) {
1564 spin_unlock(&old->i_lock);
1565 spin_unlock(&inode_hash_lock);
1566 return -EBUSY;
1567 }
1568 __iget(old);
1569 spin_unlock(&old->i_lock);
1570 spin_unlock(&inode_hash_lock);
1571 wait_on_inode(old);
1572 if (unlikely(!inode_unhashed(old))) {
1573 iput(old);
1574 return -EBUSY;
1575 }
1576 iput(old);
1577 }
1578}
1579EXPORT_SYMBOL(insert_inode_locked);
1580
1581int insert_inode_locked4(struct inode *inode, unsigned long hashval,
1582 int (*test)(struct inode *, void *), void *data)
1583{
1584 struct inode *old;
1585
1586 inode->i_state |= I_CREATING;
1587 old = inode_insert5(inode, hashval, test, NULL, data);
1588
1589 if (old != inode) {
1590 iput(old);
1591 return -EBUSY;
1592 }
1593 return 0;
1594}
1595EXPORT_SYMBOL(insert_inode_locked4);
1596
1597
1598int generic_delete_inode(struct inode *inode)
1599{
1600 return 1;
1601}
1602EXPORT_SYMBOL(generic_delete_inode);
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614static void iput_final(struct inode *inode)
1615{
1616 struct super_block *sb = inode->i_sb;
1617 const struct super_operations *op = inode->i_sb->s_op;
1618 unsigned long state;
1619 int drop;
1620
1621 WARN_ON(inode->i_state & I_NEW);
1622
1623 if (op->drop_inode)
1624 drop = op->drop_inode(inode);
1625 else
1626 drop = generic_drop_inode(inode);
1627
1628 if (!drop && (sb->s_flags & SB_ACTIVE)) {
1629 inode_add_lru(inode);
1630 spin_unlock(&inode->i_lock);
1631 return;
1632 }
1633
1634 state = inode->i_state;
1635 if (!drop) {
1636 WRITE_ONCE(inode->i_state, state | I_WILL_FREE);
1637 spin_unlock(&inode->i_lock);
1638
1639 write_inode_now(inode, 1);
1640
1641 spin_lock(&inode->i_lock);
1642 state = inode->i_state;
1643 WARN_ON(state & I_NEW);
1644 state &= ~I_WILL_FREE;
1645 }
1646
1647 WRITE_ONCE(inode->i_state, state | I_FREEING);
1648 if (!list_empty(&inode->i_lru))
1649 inode_lru_list_del(inode);
1650 spin_unlock(&inode->i_lock);
1651
1652 evict(inode);
1653}
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664void iput(struct inode *inode)
1665{
1666 if (!inode)
1667 return;
1668 BUG_ON(inode->i_state & I_CLEAR);
1669retry:
1670 if (atomic_dec_and_lock(&inode->i_count, &inode->i_lock)) {
1671 if (inode->i_nlink && (inode->i_state & I_DIRTY_TIME)) {
1672 atomic_inc(&inode->i_count);
1673 spin_unlock(&inode->i_lock);
1674 trace_writeback_lazytime_iput(inode);
1675 mark_inode_dirty_sync(inode);
1676 goto retry;
1677 }
1678 iput_final(inode);
1679 }
1680}
1681EXPORT_SYMBOL(iput);
1682
1683#ifdef CONFIG_BLOCK
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698int bmap(struct inode *inode, sector_t *block)
1699{
1700 if (!inode->i_mapping->a_ops->bmap)
1701 return -EINVAL;
1702
1703 *block = inode->i_mapping->a_ops->bmap(inode->i_mapping, *block);
1704 return 0;
1705}
1706EXPORT_SYMBOL(bmap);
1707#endif
1708
1709
1710
1711
1712
1713
1714static int relatime_need_update(struct vfsmount *mnt, struct inode *inode,
1715 struct timespec64 now)
1716{
1717
1718 if (!(mnt->mnt_flags & MNT_RELATIME))
1719 return 1;
1720
1721
1722
1723 if (timespec64_compare(&inode->i_mtime, &inode->i_atime) >= 0)
1724 return 1;
1725
1726
1727
1728 if (timespec64_compare(&inode->i_ctime, &inode->i_atime) >= 0)
1729 return 1;
1730
1731
1732
1733
1734
1735 if ((long)(now.tv_sec - inode->i_atime.tv_sec) >= 24*60*60)
1736 return 1;
1737
1738
1739
1740 return 0;
1741}
1742
1743int generic_update_time(struct inode *inode, struct timespec64 *time, int flags)
1744{
1745 int iflags = I_DIRTY_TIME;
1746 bool dirty = false;
1747
1748 if (flags & S_ATIME)
1749 inode->i_atime = *time;
1750 if (flags & S_VERSION)
1751 dirty = inode_maybe_inc_iversion(inode, false);
1752 if (flags & S_CTIME)
1753 inode->i_ctime = *time;
1754 if (flags & S_MTIME)
1755 inode->i_mtime = *time;
1756 if ((flags & (S_ATIME | S_CTIME | S_MTIME)) &&
1757 !(inode->i_sb->s_flags & SB_LAZYTIME))
1758 dirty = true;
1759
1760 if (dirty)
1761 iflags |= I_DIRTY_SYNC;
1762 __mark_inode_dirty(inode, iflags);
1763 return 0;
1764}
1765EXPORT_SYMBOL(generic_update_time);
1766
1767
1768
1769
1770
1771static int update_time(struct inode *inode, struct timespec64 *time, int flags)
1772{
1773 if (inode->i_op->update_time)
1774 return inode->i_op->update_time(inode, time, flags);
1775 return generic_update_time(inode, time, flags);
1776}
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787bool atime_needs_update(const struct path *path, struct inode *inode)
1788{
1789 struct vfsmount *mnt = path->mnt;
1790 struct timespec64 now;
1791
1792 if (inode->i_flags & S_NOATIME)
1793 return false;
1794
1795
1796
1797
1798 if (HAS_UNMAPPED_ID(inode))
1799 return false;
1800
1801 if (IS_NOATIME(inode))
1802 return false;
1803 if ((inode->i_sb->s_flags & SB_NODIRATIME) && S_ISDIR(inode->i_mode))
1804 return false;
1805
1806 if (mnt->mnt_flags & MNT_NOATIME)
1807 return false;
1808 if ((mnt->mnt_flags & MNT_NODIRATIME) && S_ISDIR(inode->i_mode))
1809 return false;
1810
1811 now = current_time(inode);
1812
1813 if (!relatime_need_update(mnt, inode, now))
1814 return false;
1815
1816 if (timespec64_equal(&inode->i_atime, &now))
1817 return false;
1818
1819 return true;
1820}
1821
1822void touch_atime(const struct path *path)
1823{
1824 struct vfsmount *mnt = path->mnt;
1825 struct inode *inode = d_inode(path->dentry);
1826 struct timespec64 now;
1827
1828 if (!atime_needs_update(path, inode))
1829 return;
1830
1831 if (!sb_start_write_trylock(inode->i_sb))
1832 return;
1833
1834 if (__mnt_want_write(mnt) != 0)
1835 goto skip_update;
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845 now = current_time(inode);
1846 update_time(inode, &now, S_ATIME);
1847 __mnt_drop_write(mnt);
1848skip_update:
1849 sb_end_write(inode->i_sb);
1850}
1851EXPORT_SYMBOL(touch_atime);
1852
1853
1854
1855
1856
1857
1858
1859int should_remove_suid(struct dentry *dentry)
1860{
1861 umode_t mode = d_inode(dentry)->i_mode;
1862 int kill = 0;
1863
1864
1865 if (unlikely(mode & S_ISUID))
1866 kill = ATTR_KILL_SUID;
1867
1868
1869
1870
1871
1872 if (unlikely((mode & S_ISGID) && (mode & S_IXGRP)))
1873 kill |= ATTR_KILL_SGID;
1874
1875 if (unlikely(kill && !capable(CAP_FSETID) && S_ISREG(mode)))
1876 return kill;
1877
1878 return 0;
1879}
1880EXPORT_SYMBOL(should_remove_suid);
1881
1882
1883
1884
1885
1886
1887int dentry_needs_remove_privs(struct dentry *dentry)
1888{
1889 struct inode *inode = d_inode(dentry);
1890 int mask = 0;
1891 int ret;
1892
1893 if (IS_NOSEC(inode))
1894 return 0;
1895
1896 mask = should_remove_suid(dentry);
1897 ret = security_inode_need_killpriv(dentry);
1898 if (ret < 0)
1899 return ret;
1900 if (ret)
1901 mask |= ATTR_KILL_PRIV;
1902 return mask;
1903}
1904
1905static int __remove_privs(struct dentry *dentry, int kill)
1906{
1907 struct iattr newattrs;
1908
1909 newattrs.ia_valid = ATTR_FORCE | kill;
1910
1911
1912
1913
1914 return notify_change(dentry, &newattrs, NULL);
1915}
1916
1917
1918
1919
1920
1921int file_remove_privs(struct file *file)
1922{
1923 struct dentry *dentry = file_dentry(file);
1924 struct inode *inode = file_inode(file);
1925 int kill;
1926 int error = 0;
1927
1928
1929
1930
1931
1932
1933
1934 if (IS_NOSEC(inode) || !S_ISREG(inode->i_mode))
1935 return 0;
1936
1937 kill = dentry_needs_remove_privs(dentry);
1938 if (kill < 0)
1939 return kill;
1940 if (kill)
1941 error = __remove_privs(dentry, kill);
1942 if (!error)
1943 inode_has_no_xattr(inode);
1944
1945 return error;
1946}
1947EXPORT_SYMBOL(file_remove_privs);
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962int file_update_time(struct file *file)
1963{
1964 struct inode *inode = file_inode(file);
1965 struct timespec64 now;
1966 int sync_it = 0;
1967 int ret;
1968
1969
1970 if (IS_NOCMTIME(inode))
1971 return 0;
1972
1973 now = current_time(inode);
1974 if (!timespec64_equal(&inode->i_mtime, &now))
1975 sync_it = S_MTIME;
1976
1977 if (!timespec64_equal(&inode->i_ctime, &now))
1978 sync_it |= S_CTIME;
1979
1980 if (IS_I_VERSION(inode) && inode_iversion_need_inc(inode))
1981 sync_it |= S_VERSION;
1982
1983 if (!sync_it)
1984 return 0;
1985
1986
1987 if (__mnt_want_write_file(file))
1988 return 0;
1989
1990 ret = update_time(inode, &now, sync_it);
1991 __mnt_drop_write_file(file);
1992
1993 return ret;
1994}
1995EXPORT_SYMBOL(file_update_time);
1996
1997
1998int file_modified(struct file *file)
1999{
2000 int err;
2001
2002
2003
2004
2005
2006 err = file_remove_privs(file);
2007 if (err)
2008 return err;
2009
2010 if (unlikely(file->f_mode & FMODE_NOCMTIME))
2011 return 0;
2012
2013 return file_update_time(file);
2014}
2015EXPORT_SYMBOL(file_modified);
2016
2017int inode_needs_sync(struct inode *inode)
2018{
2019 if (IS_SYNC(inode))
2020 return 1;
2021 if (S_ISDIR(inode->i_mode) && IS_DIRSYNC(inode))
2022 return 1;
2023 return 0;
2024}
2025EXPORT_SYMBOL(inode_needs_sync);
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038static void __wait_on_freeing_inode(struct inode *inode)
2039{
2040 wait_queue_head_t *wq;
2041 DEFINE_WAIT_BIT(wait, &inode->i_state, __I_NEW);
2042 wq = bit_waitqueue(&inode->i_state, __I_NEW);
2043 prepare_to_wait(wq, &wait.wq_entry, TASK_UNINTERRUPTIBLE);
2044 spin_unlock(&inode->i_lock);
2045 spin_unlock(&inode_hash_lock);
2046 schedule();
2047 finish_wait(wq, &wait.wq_entry);
2048 spin_lock(&inode_hash_lock);
2049}
2050
2051static __initdata unsigned long ihash_entries;
2052static int __init set_ihash_entries(char *str)
2053{
2054 if (!str)
2055 return 0;
2056 ihash_entries = simple_strtoul(str, &str, 0);
2057 return 1;
2058}
2059__setup("ihash_entries=", set_ihash_entries);
2060
2061
2062
2063
2064void __init inode_init_early(void)
2065{
2066
2067
2068
2069 if (hashdist)
2070 return;
2071
2072 inode_hashtable =
2073 alloc_large_system_hash("Inode-cache",
2074 sizeof(struct hlist_head),
2075 ihash_entries,
2076 14,
2077 HASH_EARLY | HASH_ZERO,
2078 &i_hash_shift,
2079 &i_hash_mask,
2080 0,
2081 0);
2082}
2083
2084void __init inode_init(void)
2085{
2086
2087 inode_cachep = kmem_cache_create("inode_cache",
2088 sizeof(struct inode),
2089 0,
2090 (SLAB_RECLAIM_ACCOUNT|SLAB_PANIC|
2091 SLAB_MEM_SPREAD|SLAB_ACCOUNT),
2092 init_once);
2093
2094
2095 if (!hashdist)
2096 return;
2097
2098 inode_hashtable =
2099 alloc_large_system_hash("Inode-cache",
2100 sizeof(struct hlist_head),
2101 ihash_entries,
2102 14,
2103 HASH_ZERO,
2104 &i_hash_shift,
2105 &i_hash_mask,
2106 0,
2107 0);
2108}
2109
2110void init_special_inode(struct inode *inode, umode_t mode, dev_t rdev)
2111{
2112 inode->i_mode = mode;
2113 if (S_ISCHR(mode)) {
2114 inode->i_fop = &def_chr_fops;
2115 inode->i_rdev = rdev;
2116 } else if (S_ISBLK(mode)) {
2117 inode->i_fop = &def_blk_fops;
2118 inode->i_rdev = rdev;
2119 } else if (S_ISFIFO(mode))
2120 inode->i_fop = &pipefifo_fops;
2121 else if (S_ISSOCK(mode))
2122 ;
2123 else
2124 printk(KERN_DEBUG "init_special_inode: bogus i_mode (%o) for"
2125 " inode %s:%lu\n", mode, inode->i_sb->s_id,
2126 inode->i_ino);
2127}
2128EXPORT_SYMBOL(init_special_inode);
2129
2130
2131
2132
2133
2134
2135
2136void inode_init_owner(struct inode *inode, const struct inode *dir,
2137 umode_t mode)
2138{
2139 inode->i_uid = current_fsuid();
2140 if (dir && dir->i_mode & S_ISGID) {
2141 inode->i_gid = dir->i_gid;
2142
2143
2144 if (S_ISDIR(mode))
2145 mode |= S_ISGID;
2146 else if ((mode & (S_ISGID | S_IXGRP)) == (S_ISGID | S_IXGRP) &&
2147 !in_group_p(inode->i_gid) &&
2148 !capable_wrt_inode_uidgid(dir, CAP_FSETID))
2149 mode &= ~S_ISGID;
2150 } else
2151 inode->i_gid = current_fsgid();
2152 inode->i_mode = mode;
2153}
2154EXPORT_SYMBOL(inode_init_owner);
2155
2156
2157
2158
2159
2160
2161
2162
2163bool inode_owner_or_capable(const struct inode *inode)
2164{
2165 struct user_namespace *ns;
2166
2167 if (uid_eq(current_fsuid(), inode->i_uid))
2168 return true;
2169
2170 ns = current_user_ns();
2171 if (kuid_has_mapping(ns, inode->i_uid) && ns_capable(ns, CAP_FOWNER))
2172 return true;
2173 return false;
2174}
2175EXPORT_SYMBOL(inode_owner_or_capable);
2176
2177
2178
2179
2180static void __inode_dio_wait(struct inode *inode)
2181{
2182 wait_queue_head_t *wq = bit_waitqueue(&inode->i_state, __I_DIO_WAKEUP);
2183 DEFINE_WAIT_BIT(q, &inode->i_state, __I_DIO_WAKEUP);
2184
2185 do {
2186 prepare_to_wait(wq, &q.wq_entry, TASK_UNINTERRUPTIBLE);
2187 if (atomic_read(&inode->i_dio_count))
2188 schedule();
2189 } while (atomic_read(&inode->i_dio_count));
2190 finish_wait(wq, &q.wq_entry);
2191}
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203void inode_dio_wait(struct inode *inode)
2204{
2205 if (atomic_read(&inode->i_dio_count))
2206 __inode_dio_wait(inode);
2207}
2208EXPORT_SYMBOL(inode_dio_wait);
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226void inode_set_flags(struct inode *inode, unsigned int flags,
2227 unsigned int mask)
2228{
2229 WARN_ON_ONCE(flags & ~mask);
2230 set_mask_bits(&inode->i_flags, mask, flags);
2231}
2232EXPORT_SYMBOL(inode_set_flags);
2233
2234void inode_nohighmem(struct inode *inode)
2235{
2236 mapping_set_gfp_mask(inode->i_mapping, GFP_USER);
2237}
2238EXPORT_SYMBOL(inode_nohighmem);
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249struct timespec64 timestamp_truncate(struct timespec64 t, struct inode *inode)
2250{
2251 struct super_block *sb = inode->i_sb;
2252 unsigned int gran = sb->s_time_gran;
2253
2254 t.tv_sec = clamp(t.tv_sec, sb->s_time_min, sb->s_time_max);
2255 if (unlikely(t.tv_sec == sb->s_time_max || t.tv_sec == sb->s_time_min))
2256 t.tv_nsec = 0;
2257
2258
2259 if (gran == 1)
2260 ;
2261 else if (gran == NSEC_PER_SEC)
2262 t.tv_nsec = 0;
2263 else if (gran > 1 && gran < NSEC_PER_SEC)
2264 t.tv_nsec -= t.tv_nsec % gran;
2265 else
2266 WARN(1, "invalid file time granularity: %u", gran);
2267 return t;
2268}
2269EXPORT_SYMBOL(timestamp_truncate);
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281struct timespec64 current_time(struct inode *inode)
2282{
2283 struct timespec64 now;
2284
2285 ktime_get_coarse_real_ts64(&now);
2286
2287 if (unlikely(!inode->i_sb)) {
2288 WARN(1, "current_time() called with uninitialized super_block in the inode");
2289 return now;
2290 }
2291
2292 return timestamp_truncate(now, inode);
2293}
2294EXPORT_SYMBOL(current_time);
2295
2296
2297
2298
2299
2300
2301
2302
2303int vfs_ioc_setflags_prepare(struct inode *inode, unsigned int oldflags,
2304 unsigned int flags)
2305{
2306
2307
2308
2309
2310
2311
2312 if ((flags ^ oldflags) & (FS_APPEND_FL | FS_IMMUTABLE_FL) &&
2313 !capable(CAP_LINUX_IMMUTABLE))
2314 return -EPERM;
2315
2316 return fscrypt_prepare_setflags(inode, oldflags, flags);
2317}
2318EXPORT_SYMBOL(vfs_ioc_setflags_prepare);
2319
2320
2321
2322
2323
2324
2325
2326
2327int vfs_ioc_fssetxattr_check(struct inode *inode, const struct fsxattr *old_fa,
2328 struct fsxattr *fa)
2329{
2330
2331
2332
2333
2334 if ((old_fa->fsx_xflags ^ fa->fsx_xflags) &
2335 (FS_XFLAG_IMMUTABLE | FS_XFLAG_APPEND) &&
2336 !capable(CAP_LINUX_IMMUTABLE))
2337 return -EPERM;
2338
2339
2340
2341
2342
2343
2344 if (current_user_ns() != &init_user_ns) {
2345 if (old_fa->fsx_projid != fa->fsx_projid)
2346 return -EINVAL;
2347 if ((old_fa->fsx_xflags ^ fa->fsx_xflags) &
2348 FS_XFLAG_PROJINHERIT)
2349 return -EINVAL;
2350 }
2351
2352
2353 if ((fa->fsx_xflags & FS_XFLAG_EXTSIZE) && !S_ISREG(inode->i_mode))
2354 return -EINVAL;
2355
2356 if ((fa->fsx_xflags & FS_XFLAG_EXTSZINHERIT) &&
2357 !S_ISDIR(inode->i_mode))
2358 return -EINVAL;
2359
2360 if ((fa->fsx_xflags & FS_XFLAG_COWEXTSIZE) &&
2361 !S_ISREG(inode->i_mode) && !S_ISDIR(inode->i_mode))
2362 return -EINVAL;
2363
2364
2365
2366
2367
2368 if ((fa->fsx_xflags & FS_XFLAG_DAX) &&
2369 !(S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode)))
2370 return -EINVAL;
2371
2372
2373 if (fa->fsx_extsize == 0)
2374 fa->fsx_xflags &= ~(FS_XFLAG_EXTSIZE | FS_XFLAG_EXTSZINHERIT);
2375 if (fa->fsx_cowextsize == 0)
2376 fa->fsx_xflags &= ~FS_XFLAG_COWEXTSIZE;
2377
2378 return 0;
2379}
2380EXPORT_SYMBOL(vfs_ioc_fssetxattr_check);
2381