1
2
3
4
5
6#include <linux/export.h>
7#include <linux/fs.h>
8#include <linux/mm.h>
9#include <linux/backing-dev.h>
10#include <linux/hash.h>
11#include <linux/swap.h>
12#include <linux/security.h>
13#include <linux/cdev.h>
14#include <linux/memblock.h>
15#include <linux/fsnotify.h>
16#include <linux/mount.h>
17#include <linux/posix_acl.h>
18#include <linux/prefetch.h>
19#include <linux/buffer_head.h>
20#include <linux/ratelimit.h>
21#include <linux/list_lru.h>
22#include <linux/iversion.h>
23#include <trace/events/writeback.h>
24#include "internal.h"
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57static unsigned int i_hash_mask __read_mostly;
58static unsigned int i_hash_shift __read_mostly;
59static struct hlist_head *inode_hashtable __read_mostly;
60static __cacheline_aligned_in_smp DEFINE_SPINLOCK(inode_hash_lock);
61
62
63
64
65
66const struct address_space_operations empty_aops = {
67};
68EXPORT_SYMBOL(empty_aops);
69
70
71
72
73struct inodes_stat_t inodes_stat;
74
75static DEFINE_PER_CPU(unsigned long, nr_inodes);
76static DEFINE_PER_CPU(unsigned long, nr_unused);
77
78static struct kmem_cache *inode_cachep __read_mostly;
79
80static long get_nr_inodes(void)
81{
82 int i;
83 long sum = 0;
84 for_each_possible_cpu(i)
85 sum += per_cpu(nr_inodes, i);
86 return sum < 0 ? 0 : sum;
87}
88
89static inline long get_nr_inodes_unused(void)
90{
91 int i;
92 long sum = 0;
93 for_each_possible_cpu(i)
94 sum += per_cpu(nr_unused, i);
95 return sum < 0 ? 0 : sum;
96}
97
98long get_nr_dirty_inodes(void)
99{
100
101 long nr_dirty = get_nr_inodes() - get_nr_inodes_unused();
102 return nr_dirty > 0 ? nr_dirty : 0;
103}
104
105
106
107
108#ifdef CONFIG_SYSCTL
109int proc_nr_inodes(struct ctl_table *table, int write,
110 void *buffer, size_t *lenp, loff_t *ppos)
111{
112 inodes_stat.nr_inodes = get_nr_inodes();
113 inodes_stat.nr_unused = get_nr_inodes_unused();
114 return proc_doulongvec_minmax(table, write, buffer, lenp, ppos);
115}
116#endif
117
118static int no_open(struct inode *inode, struct file *file)
119{
120 return -ENXIO;
121}
122
123
124
125
126
127
128
129
130
131int inode_init_always(struct super_block *sb, struct inode *inode)
132{
133 static const struct inode_operations empty_iops;
134 static const struct file_operations no_open_fops = {.open = no_open};
135 struct address_space *const mapping = &inode->i_data;
136
137 inode->i_sb = sb;
138 inode->i_blkbits = sb->s_blocksize_bits;
139 inode->i_flags = 0;
140 atomic64_set(&inode->i_sequence, 0);
141 atomic_set(&inode->i_count, 1);
142 inode->i_op = &empty_iops;
143 inode->i_fop = &no_open_fops;
144 inode->i_ino = 0;
145 inode->__i_nlink = 1;
146 inode->i_opflags = 0;
147 if (sb->s_xattr)
148 inode->i_opflags |= IOP_XATTR;
149 i_uid_write(inode, 0);
150 i_gid_write(inode, 0);
151 atomic_set(&inode->i_writecount, 0);
152 inode->i_size = 0;
153 inode->i_write_hint = WRITE_LIFE_NOT_SET;
154 inode->i_blocks = 0;
155 inode->i_bytes = 0;
156 inode->i_generation = 0;
157 inode->i_pipe = NULL;
158 inode->i_cdev = NULL;
159 inode->i_link = NULL;
160 inode->i_dir_seq = 0;
161 inode->i_rdev = 0;
162 inode->dirtied_when = 0;
163
164#ifdef CONFIG_CGROUP_WRITEBACK
165 inode->i_wb_frn_winner = 0;
166 inode->i_wb_frn_avg_time = 0;
167 inode->i_wb_frn_history = 0;
168#endif
169
170 if (security_inode_alloc(inode))
171 goto out;
172 spin_lock_init(&inode->i_lock);
173 lockdep_set_class(&inode->i_lock, &sb->s_type->i_lock_key);
174
175 init_rwsem(&inode->i_rwsem);
176 lockdep_set_class(&inode->i_rwsem, &sb->s_type->i_mutex_key);
177
178 atomic_set(&inode->i_dio_count, 0);
179
180 mapping->a_ops = &empty_aops;
181 mapping->host = inode;
182 mapping->flags = 0;
183 if (sb->s_type->fs_flags & FS_THP_SUPPORT)
184 __set_bit(AS_THP_SUPPORT, &mapping->flags);
185 mapping->wb_err = 0;
186 atomic_set(&mapping->i_mmap_writable, 0);
187#ifdef CONFIG_READ_ONLY_THP_FOR_FS
188 atomic_set(&mapping->nr_thps, 0);
189#endif
190 mapping_set_gfp_mask(mapping, GFP_HIGHUSER_MOVABLE);
191 mapping->private_data = NULL;
192 mapping->writeback_index = 0;
193 inode->i_private = NULL;
194 inode->i_mapping = mapping;
195 INIT_HLIST_HEAD(&inode->i_dentry);
196#ifdef CONFIG_FS_POSIX_ACL
197 inode->i_acl = inode->i_default_acl = ACL_NOT_CACHED;
198#endif
199
200#ifdef CONFIG_FSNOTIFY
201 inode->i_fsnotify_mask = 0;
202#endif
203 inode->i_flctx = NULL;
204 this_cpu_inc(nr_inodes);
205
206 return 0;
207out:
208 return -ENOMEM;
209}
210EXPORT_SYMBOL(inode_init_always);
211
212void free_inode_nonrcu(struct inode *inode)
213{
214 kmem_cache_free(inode_cachep, inode);
215}
216EXPORT_SYMBOL(free_inode_nonrcu);
217
218static void i_callback(struct rcu_head *head)
219{
220 struct inode *inode = container_of(head, struct inode, i_rcu);
221 if (inode->free_inode)
222 inode->free_inode(inode);
223 else
224 free_inode_nonrcu(inode);
225}
226
227static struct inode *alloc_inode(struct super_block *sb)
228{
229 const struct super_operations *ops = sb->s_op;
230 struct inode *inode;
231
232 if (ops->alloc_inode)
233 inode = ops->alloc_inode(sb);
234 else
235 inode = kmem_cache_alloc(inode_cachep, GFP_KERNEL);
236
237 if (!inode)
238 return NULL;
239
240 if (unlikely(inode_init_always(sb, inode))) {
241 if (ops->destroy_inode) {
242 ops->destroy_inode(inode);
243 if (!ops->free_inode)
244 return NULL;
245 }
246 inode->free_inode = ops->free_inode;
247 i_callback(&inode->i_rcu);
248 return NULL;
249 }
250
251 return inode;
252}
253
254void __destroy_inode(struct inode *inode)
255{
256 BUG_ON(inode_has_buffers(inode));
257 inode_detach_wb(inode);
258 security_inode_free(inode);
259 fsnotify_inode_delete(inode);
260 locks_free_lock_context(inode);
261 if (!inode->i_nlink) {
262 WARN_ON(atomic_long_read(&inode->i_sb->s_remove_count) == 0);
263 atomic_long_dec(&inode->i_sb->s_remove_count);
264 }
265
266#ifdef CONFIG_FS_POSIX_ACL
267 if (inode->i_acl && !is_uncached_acl(inode->i_acl))
268 posix_acl_release(inode->i_acl);
269 if (inode->i_default_acl && !is_uncached_acl(inode->i_default_acl))
270 posix_acl_release(inode->i_default_acl);
271#endif
272 this_cpu_dec(nr_inodes);
273}
274EXPORT_SYMBOL(__destroy_inode);
275
276static void destroy_inode(struct inode *inode)
277{
278 const struct super_operations *ops = inode->i_sb->s_op;
279
280 BUG_ON(!list_empty(&inode->i_lru));
281 __destroy_inode(inode);
282 if (ops->destroy_inode) {
283 ops->destroy_inode(inode);
284 if (!ops->free_inode)
285 return;
286 }
287 inode->free_inode = ops->free_inode;
288 call_rcu(&inode->i_rcu, i_callback);
289}
290
291
292
293
294
295
296
297
298
299
300
301
302void drop_nlink(struct inode *inode)
303{
304 WARN_ON(inode->i_nlink == 0);
305 inode->__i_nlink--;
306 if (!inode->i_nlink)
307 atomic_long_inc(&inode->i_sb->s_remove_count);
308}
309EXPORT_SYMBOL(drop_nlink);
310
311
312
313
314
315
316
317
318
319void clear_nlink(struct inode *inode)
320{
321 if (inode->i_nlink) {
322 inode->__i_nlink = 0;
323 atomic_long_inc(&inode->i_sb->s_remove_count);
324 }
325}
326EXPORT_SYMBOL(clear_nlink);
327
328
329
330
331
332
333
334
335
336void set_nlink(struct inode *inode, unsigned int nlink)
337{
338 if (!nlink) {
339 clear_nlink(inode);
340 } else {
341
342 if (inode->i_nlink == 0)
343 atomic_long_dec(&inode->i_sb->s_remove_count);
344
345 inode->__i_nlink = nlink;
346 }
347}
348EXPORT_SYMBOL(set_nlink);
349
350
351
352
353
354
355
356
357
358void inc_nlink(struct inode *inode)
359{
360 if (unlikely(inode->i_nlink == 0)) {
361 WARN_ON(!(inode->i_state & I_LINKABLE));
362 atomic_long_dec(&inode->i_sb->s_remove_count);
363 }
364
365 inode->__i_nlink++;
366}
367EXPORT_SYMBOL(inc_nlink);
368
369static void __address_space_init_once(struct address_space *mapping)
370{
371 xa_init_flags(&mapping->i_pages, XA_FLAGS_LOCK_IRQ | XA_FLAGS_ACCOUNT);
372 init_rwsem(&mapping->i_mmap_rwsem);
373 INIT_LIST_HEAD(&mapping->private_list);
374 spin_lock_init(&mapping->private_lock);
375 mapping->i_mmap = RB_ROOT_CACHED;
376}
377
378void address_space_init_once(struct address_space *mapping)
379{
380 memset(mapping, 0, sizeof(*mapping));
381 __address_space_init_once(mapping);
382}
383EXPORT_SYMBOL(address_space_init_once);
384
385
386
387
388
389
390void inode_init_once(struct inode *inode)
391{
392 memset(inode, 0, sizeof(*inode));
393 INIT_HLIST_NODE(&inode->i_hash);
394 INIT_LIST_HEAD(&inode->i_devices);
395 INIT_LIST_HEAD(&inode->i_io_list);
396 INIT_LIST_HEAD(&inode->i_wb_list);
397 INIT_LIST_HEAD(&inode->i_lru);
398 __address_space_init_once(&inode->i_data);
399 i_size_ordered_init(inode);
400}
401EXPORT_SYMBOL(inode_init_once);
402
403static void init_once(void *foo)
404{
405 struct inode *inode = (struct inode *) foo;
406
407 inode_init_once(inode);
408}
409
410
411
412
413void __iget(struct inode *inode)
414{
415 atomic_inc(&inode->i_count);
416}
417
418
419
420
421void ihold(struct inode *inode)
422{
423 WARN_ON(atomic_inc_return(&inode->i_count) < 2);
424}
425EXPORT_SYMBOL(ihold);
426
427static void inode_lru_list_add(struct inode *inode)
428{
429 if (list_lru_add(&inode->i_sb->s_inode_lru, &inode->i_lru))
430 this_cpu_inc(nr_unused);
431 else
432 inode->i_state |= I_REFERENCED;
433}
434
435
436
437
438
439
440void inode_add_lru(struct inode *inode)
441{
442 if (!(inode->i_state & (I_DIRTY_ALL | I_SYNC |
443 I_FREEING | I_WILL_FREE)) &&
444 !atomic_read(&inode->i_count) && inode->i_sb->s_flags & SB_ACTIVE)
445 inode_lru_list_add(inode);
446}
447
448
449static void inode_lru_list_del(struct inode *inode)
450{
451
452 if (list_lru_del(&inode->i_sb->s_inode_lru, &inode->i_lru))
453 this_cpu_dec(nr_unused);
454}
455
456
457
458
459
460void inode_sb_list_add(struct inode *inode)
461{
462 spin_lock(&inode->i_sb->s_inode_list_lock);
463 list_add(&inode->i_sb_list, &inode->i_sb->s_inodes);
464 spin_unlock(&inode->i_sb->s_inode_list_lock);
465}
466EXPORT_SYMBOL_GPL(inode_sb_list_add);
467
468static inline void inode_sb_list_del(struct inode *inode)
469{
470 if (!list_empty(&inode->i_sb_list)) {
471 spin_lock(&inode->i_sb->s_inode_list_lock);
472 list_del_init(&inode->i_sb_list);
473 spin_unlock(&inode->i_sb->s_inode_list_lock);
474 }
475}
476
477static unsigned long hash(struct super_block *sb, unsigned long hashval)
478{
479 unsigned long tmp;
480
481 tmp = (hashval * (unsigned long)sb) ^ (GOLDEN_RATIO_PRIME + hashval) /
482 L1_CACHE_BYTES;
483 tmp = tmp ^ ((tmp ^ GOLDEN_RATIO_PRIME) >> i_hash_shift);
484 return tmp & i_hash_mask;
485}
486
487
488
489
490
491
492
493
494
495void __insert_inode_hash(struct inode *inode, unsigned long hashval)
496{
497 struct hlist_head *b = inode_hashtable + hash(inode->i_sb, hashval);
498
499 spin_lock(&inode_hash_lock);
500 spin_lock(&inode->i_lock);
501 hlist_add_head_rcu(&inode->i_hash, b);
502 spin_unlock(&inode->i_lock);
503 spin_unlock(&inode_hash_lock);
504}
505EXPORT_SYMBOL(__insert_inode_hash);
506
507
508
509
510
511
512
513void __remove_inode_hash(struct inode *inode)
514{
515 spin_lock(&inode_hash_lock);
516 spin_lock(&inode->i_lock);
517 hlist_del_init_rcu(&inode->i_hash);
518 spin_unlock(&inode->i_lock);
519 spin_unlock(&inode_hash_lock);
520}
521EXPORT_SYMBOL(__remove_inode_hash);
522
523void clear_inode(struct inode *inode)
524{
525
526
527
528
529
530 xa_lock_irq(&inode->i_data.i_pages);
531 BUG_ON(inode->i_data.nrpages);
532
533
534
535
536
537
538
539
540 xa_unlock_irq(&inode->i_data.i_pages);
541 BUG_ON(!list_empty(&inode->i_data.private_list));
542 BUG_ON(!(inode->i_state & I_FREEING));
543 BUG_ON(inode->i_state & I_CLEAR);
544 BUG_ON(!list_empty(&inode->i_wb_list));
545
546 inode->i_state = I_FREEING | I_CLEAR;
547}
548EXPORT_SYMBOL(clear_inode);
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563static void evict(struct inode *inode)
564{
565 const struct super_operations *op = inode->i_sb->s_op;
566
567 BUG_ON(!(inode->i_state & I_FREEING));
568 BUG_ON(!list_empty(&inode->i_lru));
569
570 if (!list_empty(&inode->i_io_list))
571 inode_io_list_del(inode);
572
573 inode_sb_list_del(inode);
574
575
576
577
578
579
580
581 inode_wait_for_writeback(inode);
582
583 if (op->evict_inode) {
584 op->evict_inode(inode);
585 } else {
586 truncate_inode_pages_final(&inode->i_data);
587 clear_inode(inode);
588 }
589 if (S_ISCHR(inode->i_mode) && inode->i_cdev)
590 cd_forget(inode);
591
592 remove_inode_hash(inode);
593
594 spin_lock(&inode->i_lock);
595 wake_up_bit(&inode->i_state, __I_NEW);
596 BUG_ON(inode->i_state != (I_FREEING | I_CLEAR));
597 spin_unlock(&inode->i_lock);
598
599 destroy_inode(inode);
600}
601
602
603
604
605
606
607
608
609static void dispose_list(struct list_head *head)
610{
611 while (!list_empty(head)) {
612 struct inode *inode;
613
614 inode = list_first_entry(head, struct inode, i_lru);
615 list_del_init(&inode->i_lru);
616
617 evict(inode);
618 cond_resched();
619 }
620}
621
622
623
624
625
626
627
628
629
630
631void evict_inodes(struct super_block *sb)
632{
633 struct inode *inode, *next;
634 LIST_HEAD(dispose);
635
636again:
637 spin_lock(&sb->s_inode_list_lock);
638 list_for_each_entry_safe(inode, next, &sb->s_inodes, i_sb_list) {
639 if (atomic_read(&inode->i_count))
640 continue;
641
642 spin_lock(&inode->i_lock);
643 if (inode->i_state & (I_NEW | I_FREEING | I_WILL_FREE)) {
644 spin_unlock(&inode->i_lock);
645 continue;
646 }
647
648 inode->i_state |= I_FREEING;
649 inode_lru_list_del(inode);
650 spin_unlock(&inode->i_lock);
651 list_add(&inode->i_lru, &dispose);
652
653
654
655
656
657
658 if (need_resched()) {
659 spin_unlock(&sb->s_inode_list_lock);
660 cond_resched();
661 dispose_list(&dispose);
662 goto again;
663 }
664 }
665 spin_unlock(&sb->s_inode_list_lock);
666
667 dispose_list(&dispose);
668}
669EXPORT_SYMBOL_GPL(evict_inodes);
670
671
672
673
674
675
676
677
678
679
680
681int invalidate_inodes(struct super_block *sb, bool kill_dirty)
682{
683 int busy = 0;
684 struct inode *inode, *next;
685 LIST_HEAD(dispose);
686
687again:
688 spin_lock(&sb->s_inode_list_lock);
689 list_for_each_entry_safe(inode, next, &sb->s_inodes, i_sb_list) {
690 spin_lock(&inode->i_lock);
691 if (inode->i_state & (I_NEW | I_FREEING | I_WILL_FREE)) {
692 spin_unlock(&inode->i_lock);
693 continue;
694 }
695 if (inode->i_state & I_DIRTY_ALL && !kill_dirty) {
696 spin_unlock(&inode->i_lock);
697 busy = 1;
698 continue;
699 }
700 if (atomic_read(&inode->i_count)) {
701 spin_unlock(&inode->i_lock);
702 busy = 1;
703 continue;
704 }
705
706 inode->i_state |= I_FREEING;
707 inode_lru_list_del(inode);
708 spin_unlock(&inode->i_lock);
709 list_add(&inode->i_lru, &dispose);
710 if (need_resched()) {
711 spin_unlock(&sb->s_inode_list_lock);
712 cond_resched();
713 dispose_list(&dispose);
714 goto again;
715 }
716 }
717 spin_unlock(&sb->s_inode_list_lock);
718
719 dispose_list(&dispose);
720
721 return busy;
722}
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739static enum lru_status inode_lru_isolate(struct list_head *item,
740 struct list_lru_one *lru, spinlock_t *lru_lock, void *arg)
741{
742 struct list_head *freeable = arg;
743 struct inode *inode = container_of(item, struct inode, i_lru);
744
745
746
747
748
749 if (!spin_trylock(&inode->i_lock))
750 return LRU_SKIP;
751
752
753
754
755
756 if (atomic_read(&inode->i_count) ||
757 (inode->i_state & ~I_REFERENCED)) {
758 list_lru_isolate(lru, &inode->i_lru);
759 spin_unlock(&inode->i_lock);
760 this_cpu_dec(nr_unused);
761 return LRU_REMOVED;
762 }
763
764
765 if (inode->i_state & I_REFERENCED) {
766 inode->i_state &= ~I_REFERENCED;
767 spin_unlock(&inode->i_lock);
768 return LRU_ROTATE;
769 }
770
771 if (inode_has_buffers(inode) || inode->i_data.nrpages) {
772 __iget(inode);
773 spin_unlock(&inode->i_lock);
774 spin_unlock(lru_lock);
775 if (remove_inode_buffers(inode)) {
776 unsigned long reap;
777 reap = invalidate_mapping_pages(&inode->i_data, 0, -1);
778 if (current_is_kswapd())
779 __count_vm_events(KSWAPD_INODESTEAL, reap);
780 else
781 __count_vm_events(PGINODESTEAL, reap);
782 if (current->reclaim_state)
783 current->reclaim_state->reclaimed_slab += reap;
784 }
785 iput(inode);
786 spin_lock(lru_lock);
787 return LRU_RETRY;
788 }
789
790 WARN_ON(inode->i_state & I_NEW);
791 inode->i_state |= I_FREEING;
792 list_lru_isolate_move(lru, &inode->i_lru, freeable);
793 spin_unlock(&inode->i_lock);
794
795 this_cpu_dec(nr_unused);
796 return LRU_REMOVED;
797}
798
799
800
801
802
803
804
805long prune_icache_sb(struct super_block *sb, struct shrink_control *sc)
806{
807 LIST_HEAD(freeable);
808 long freed;
809
810 freed = list_lru_shrink_walk(&sb->s_inode_lru, sc,
811 inode_lru_isolate, &freeable);
812 dispose_list(&freeable);
813 return freed;
814}
815
816static void __wait_on_freeing_inode(struct inode *inode);
817
818
819
820static struct inode *find_inode(struct super_block *sb,
821 struct hlist_head *head,
822 int (*test)(struct inode *, void *),
823 void *data)
824{
825 struct inode *inode = NULL;
826
827repeat:
828 hlist_for_each_entry(inode, head, i_hash) {
829 if (inode->i_sb != sb)
830 continue;
831 if (!test(inode, data))
832 continue;
833 spin_lock(&inode->i_lock);
834 if (inode->i_state & (I_FREEING|I_WILL_FREE)) {
835 __wait_on_freeing_inode(inode);
836 goto repeat;
837 }
838 if (unlikely(inode->i_state & I_CREATING)) {
839 spin_unlock(&inode->i_lock);
840 return ERR_PTR(-ESTALE);
841 }
842 __iget(inode);
843 spin_unlock(&inode->i_lock);
844 return inode;
845 }
846 return NULL;
847}
848
849
850
851
852
853static struct inode *find_inode_fast(struct super_block *sb,
854 struct hlist_head *head, unsigned long ino)
855{
856 struct inode *inode = NULL;
857
858repeat:
859 hlist_for_each_entry(inode, head, i_hash) {
860 if (inode->i_ino != ino)
861 continue;
862 if (inode->i_sb != sb)
863 continue;
864 spin_lock(&inode->i_lock);
865 if (inode->i_state & (I_FREEING|I_WILL_FREE)) {
866 __wait_on_freeing_inode(inode);
867 goto repeat;
868 }
869 if (unlikely(inode->i_state & I_CREATING)) {
870 spin_unlock(&inode->i_lock);
871 return ERR_PTR(-ESTALE);
872 }
873 __iget(inode);
874 spin_unlock(&inode->i_lock);
875 return inode;
876 }
877 return NULL;
878}
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895#define LAST_INO_BATCH 1024
896static DEFINE_PER_CPU(unsigned int, last_ino);
897
898unsigned int get_next_ino(void)
899{
900 unsigned int *p = &get_cpu_var(last_ino);
901 unsigned int res = *p;
902
903#ifdef CONFIG_SMP
904 if (unlikely((res & (LAST_INO_BATCH-1)) == 0)) {
905 static atomic_t shared_last_ino;
906 int next = atomic_add_return(LAST_INO_BATCH, &shared_last_ino);
907
908 res = next - LAST_INO_BATCH;
909 }
910#endif
911
912 res++;
913
914 if (unlikely(!res))
915 res++;
916 *p = res;
917 put_cpu_var(last_ino);
918 return res;
919}
920EXPORT_SYMBOL(get_next_ino);
921
922
923
924
925
926
927
928
929
930
931
932struct inode *new_inode_pseudo(struct super_block *sb)
933{
934 struct inode *inode = alloc_inode(sb);
935
936 if (inode) {
937 spin_lock(&inode->i_lock);
938 inode->i_state = 0;
939 spin_unlock(&inode->i_lock);
940 INIT_LIST_HEAD(&inode->i_sb_list);
941 }
942 return inode;
943}
944
945
946
947
948
949
950
951
952
953
954
955
956
957struct inode *new_inode(struct super_block *sb)
958{
959 struct inode *inode;
960
961 spin_lock_prefetch(&sb->s_inode_list_lock);
962
963 inode = new_inode_pseudo(sb);
964 if (inode)
965 inode_sb_list_add(inode);
966 return inode;
967}
968EXPORT_SYMBOL(new_inode);
969
970#ifdef CONFIG_DEBUG_LOCK_ALLOC
971void lockdep_annotate_inode_mutex_key(struct inode *inode)
972{
973 if (S_ISDIR(inode->i_mode)) {
974 struct file_system_type *type = inode->i_sb->s_type;
975
976
977 if (lockdep_match_class(&inode->i_rwsem, &type->i_mutex_key)) {
978
979
980
981
982 init_rwsem(&inode->i_rwsem);
983 lockdep_set_class(&inode->i_rwsem,
984 &type->i_mutex_dir_key);
985 }
986 }
987}
988EXPORT_SYMBOL(lockdep_annotate_inode_mutex_key);
989#endif
990
991
992
993
994
995
996
997
998void unlock_new_inode(struct inode *inode)
999{
1000 lockdep_annotate_inode_mutex_key(inode);
1001 spin_lock(&inode->i_lock);
1002 WARN_ON(!(inode->i_state & I_NEW));
1003 inode->i_state &= ~I_NEW & ~I_CREATING;
1004 smp_mb();
1005 wake_up_bit(&inode->i_state, __I_NEW);
1006 spin_unlock(&inode->i_lock);
1007}
1008EXPORT_SYMBOL(unlock_new_inode);
1009
1010void discard_new_inode(struct inode *inode)
1011{
1012 lockdep_annotate_inode_mutex_key(inode);
1013 spin_lock(&inode->i_lock);
1014 WARN_ON(!(inode->i_state & I_NEW));
1015 inode->i_state &= ~I_NEW;
1016 smp_mb();
1017 wake_up_bit(&inode->i_state, __I_NEW);
1018 spin_unlock(&inode->i_lock);
1019 iput(inode);
1020}
1021EXPORT_SYMBOL(discard_new_inode);
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032void lock_two_nondirectories(struct inode *inode1, struct inode *inode2)
1033{
1034 if (inode1 > inode2)
1035 swap(inode1, inode2);
1036
1037 if (inode1 && !S_ISDIR(inode1->i_mode))
1038 inode_lock(inode1);
1039 if (inode2 && !S_ISDIR(inode2->i_mode) && inode2 != inode1)
1040 inode_lock_nested(inode2, I_MUTEX_NONDIR2);
1041}
1042EXPORT_SYMBOL(lock_two_nondirectories);
1043
1044
1045
1046
1047
1048
1049void unlock_two_nondirectories(struct inode *inode1, struct inode *inode2)
1050{
1051 if (inode1 && !S_ISDIR(inode1->i_mode))
1052 inode_unlock(inode1);
1053 if (inode2 && !S_ISDIR(inode2->i_mode) && inode2 != inode1)
1054 inode_unlock(inode2);
1055}
1056EXPORT_SYMBOL(unlock_two_nondirectories);
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078struct inode *inode_insert5(struct inode *inode, unsigned long hashval,
1079 int (*test)(struct inode *, void *),
1080 int (*set)(struct inode *, void *), void *data)
1081{
1082 struct hlist_head *head = inode_hashtable + hash(inode->i_sb, hashval);
1083 struct inode *old;
1084 bool creating = inode->i_state & I_CREATING;
1085
1086again:
1087 spin_lock(&inode_hash_lock);
1088 old = find_inode(inode->i_sb, head, test, data);
1089 if (unlikely(old)) {
1090
1091
1092
1093
1094 spin_unlock(&inode_hash_lock);
1095 if (IS_ERR(old))
1096 return NULL;
1097 wait_on_inode(old);
1098 if (unlikely(inode_unhashed(old))) {
1099 iput(old);
1100 goto again;
1101 }
1102 return old;
1103 }
1104
1105 if (set && unlikely(set(inode, data))) {
1106 inode = NULL;
1107 goto unlock;
1108 }
1109
1110
1111
1112
1113
1114 spin_lock(&inode->i_lock);
1115 inode->i_state |= I_NEW;
1116 hlist_add_head_rcu(&inode->i_hash, head);
1117 spin_unlock(&inode->i_lock);
1118 if (!creating)
1119 inode_sb_list_add(inode);
1120unlock:
1121 spin_unlock(&inode_hash_lock);
1122
1123 return inode;
1124}
1125EXPORT_SYMBOL(inode_insert5);
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147struct inode *iget5_locked(struct super_block *sb, unsigned long hashval,
1148 int (*test)(struct inode *, void *),
1149 int (*set)(struct inode *, void *), void *data)
1150{
1151 struct inode *inode = ilookup5(sb, hashval, test, data);
1152
1153 if (!inode) {
1154 struct inode *new = alloc_inode(sb);
1155
1156 if (new) {
1157 new->i_state = 0;
1158 inode = inode_insert5(new, hashval, test, set, data);
1159 if (unlikely(inode != new))
1160 destroy_inode(new);
1161 }
1162 }
1163 return inode;
1164}
1165EXPORT_SYMBOL(iget5_locked);
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180struct inode *iget_locked(struct super_block *sb, unsigned long ino)
1181{
1182 struct hlist_head *head = inode_hashtable + hash(sb, ino);
1183 struct inode *inode;
1184again:
1185 spin_lock(&inode_hash_lock);
1186 inode = find_inode_fast(sb, head, ino);
1187 spin_unlock(&inode_hash_lock);
1188 if (inode) {
1189 if (IS_ERR(inode))
1190 return NULL;
1191 wait_on_inode(inode);
1192 if (unlikely(inode_unhashed(inode))) {
1193 iput(inode);
1194 goto again;
1195 }
1196 return inode;
1197 }
1198
1199 inode = alloc_inode(sb);
1200 if (inode) {
1201 struct inode *old;
1202
1203 spin_lock(&inode_hash_lock);
1204
1205 old = find_inode_fast(sb, head, ino);
1206 if (!old) {
1207 inode->i_ino = ino;
1208 spin_lock(&inode->i_lock);
1209 inode->i_state = I_NEW;
1210 hlist_add_head_rcu(&inode->i_hash, head);
1211 spin_unlock(&inode->i_lock);
1212 inode_sb_list_add(inode);
1213 spin_unlock(&inode_hash_lock);
1214
1215
1216
1217
1218 return inode;
1219 }
1220
1221
1222
1223
1224
1225
1226 spin_unlock(&inode_hash_lock);
1227 destroy_inode(inode);
1228 if (IS_ERR(old))
1229 return NULL;
1230 inode = old;
1231 wait_on_inode(inode);
1232 if (unlikely(inode_unhashed(inode))) {
1233 iput(inode);
1234 goto again;
1235 }
1236 }
1237 return inode;
1238}
1239EXPORT_SYMBOL(iget_locked);
1240
1241
1242
1243
1244
1245
1246
1247
1248static int test_inode_iunique(struct super_block *sb, unsigned long ino)
1249{
1250 struct hlist_head *b = inode_hashtable + hash(sb, ino);
1251 struct inode *inode;
1252
1253 hlist_for_each_entry_rcu(inode, b, i_hash) {
1254 if (inode->i_ino == ino && inode->i_sb == sb)
1255 return 0;
1256 }
1257 return 1;
1258}
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274ino_t iunique(struct super_block *sb, ino_t max_reserved)
1275{
1276
1277
1278
1279
1280
1281 static DEFINE_SPINLOCK(iunique_lock);
1282 static unsigned int counter;
1283 ino_t res;
1284
1285 rcu_read_lock();
1286 spin_lock(&iunique_lock);
1287 do {
1288 if (counter <= max_reserved)
1289 counter = max_reserved + 1;
1290 res = counter++;
1291 } while (!test_inode_iunique(sb, res));
1292 spin_unlock(&iunique_lock);
1293 rcu_read_unlock();
1294
1295 return res;
1296}
1297EXPORT_SYMBOL(iunique);
1298
1299struct inode *igrab(struct inode *inode)
1300{
1301 spin_lock(&inode->i_lock);
1302 if (!(inode->i_state & (I_FREEING|I_WILL_FREE))) {
1303 __iget(inode);
1304 spin_unlock(&inode->i_lock);
1305 } else {
1306 spin_unlock(&inode->i_lock);
1307
1308
1309
1310
1311
1312 inode = NULL;
1313 }
1314 return inode;
1315}
1316EXPORT_SYMBOL(igrab);
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334struct inode *ilookup5_nowait(struct super_block *sb, unsigned long hashval,
1335 int (*test)(struct inode *, void *), void *data)
1336{
1337 struct hlist_head *head = inode_hashtable + hash(sb, hashval);
1338 struct inode *inode;
1339
1340 spin_lock(&inode_hash_lock);
1341 inode = find_inode(sb, head, test, data);
1342 spin_unlock(&inode_hash_lock);
1343
1344 return IS_ERR(inode) ? NULL : inode;
1345}
1346EXPORT_SYMBOL(ilookup5_nowait);
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365struct inode *ilookup5(struct super_block *sb, unsigned long hashval,
1366 int (*test)(struct inode *, void *), void *data)
1367{
1368 struct inode *inode;
1369again:
1370 inode = ilookup5_nowait(sb, hashval, test, data);
1371 if (inode) {
1372 wait_on_inode(inode);
1373 if (unlikely(inode_unhashed(inode))) {
1374 iput(inode);
1375 goto again;
1376 }
1377 }
1378 return inode;
1379}
1380EXPORT_SYMBOL(ilookup5);
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390struct inode *ilookup(struct super_block *sb, unsigned long ino)
1391{
1392 struct hlist_head *head = inode_hashtable + hash(sb, ino);
1393 struct inode *inode;
1394again:
1395 spin_lock(&inode_hash_lock);
1396 inode = find_inode_fast(sb, head, ino);
1397 spin_unlock(&inode_hash_lock);
1398
1399 if (inode) {
1400 if (IS_ERR(inode))
1401 return NULL;
1402 wait_on_inode(inode);
1403 if (unlikely(inode_unhashed(inode))) {
1404 iput(inode);
1405 goto again;
1406 }
1407 }
1408 return inode;
1409}
1410EXPORT_SYMBOL(ilookup);
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435struct inode *find_inode_nowait(struct super_block *sb,
1436 unsigned long hashval,
1437 int (*match)(struct inode *, unsigned long,
1438 void *),
1439 void *data)
1440{
1441 struct hlist_head *head = inode_hashtable + hash(sb, hashval);
1442 struct inode *inode, *ret_inode = NULL;
1443 int mval;
1444
1445 spin_lock(&inode_hash_lock);
1446 hlist_for_each_entry(inode, head, i_hash) {
1447 if (inode->i_sb != sb)
1448 continue;
1449 mval = match(inode, hashval, data);
1450 if (mval == 0)
1451 continue;
1452 if (mval == 1)
1453 ret_inode = inode;
1454 goto out;
1455 }
1456out:
1457 spin_unlock(&inode_hash_lock);
1458 return ret_inode;
1459}
1460EXPORT_SYMBOL(find_inode_nowait);
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483struct inode *find_inode_rcu(struct super_block *sb, unsigned long hashval,
1484 int (*test)(struct inode *, void *), void *data)
1485{
1486 struct hlist_head *head = inode_hashtable + hash(sb, hashval);
1487 struct inode *inode;
1488
1489 RCU_LOCKDEP_WARN(!rcu_read_lock_held(),
1490 "suspicious find_inode_rcu() usage");
1491
1492 hlist_for_each_entry_rcu(inode, head, i_hash) {
1493 if (inode->i_sb == sb &&
1494 !(READ_ONCE(inode->i_state) & (I_FREEING | I_WILL_FREE)) &&
1495 test(inode, data))
1496 return inode;
1497 }
1498 return NULL;
1499}
1500EXPORT_SYMBOL(find_inode_rcu);
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521struct inode *find_inode_by_ino_rcu(struct super_block *sb,
1522 unsigned long ino)
1523{
1524 struct hlist_head *head = inode_hashtable + hash(sb, ino);
1525 struct inode *inode;
1526
1527 RCU_LOCKDEP_WARN(!rcu_read_lock_held(),
1528 "suspicious find_inode_by_ino_rcu() usage");
1529
1530 hlist_for_each_entry_rcu(inode, head, i_hash) {
1531 if (inode->i_ino == ino &&
1532 inode->i_sb == sb &&
1533 !(READ_ONCE(inode->i_state) & (I_FREEING | I_WILL_FREE)))
1534 return inode;
1535 }
1536 return NULL;
1537}
1538EXPORT_SYMBOL(find_inode_by_ino_rcu);
1539
1540int insert_inode_locked(struct inode *inode)
1541{
1542 struct super_block *sb = inode->i_sb;
1543 ino_t ino = inode->i_ino;
1544 struct hlist_head *head = inode_hashtable + hash(sb, ino);
1545
1546 while (1) {
1547 struct inode *old = NULL;
1548 spin_lock(&inode_hash_lock);
1549 hlist_for_each_entry(old, head, i_hash) {
1550 if (old->i_ino != ino)
1551 continue;
1552 if (old->i_sb != sb)
1553 continue;
1554 spin_lock(&old->i_lock);
1555 if (old->i_state & (I_FREEING|I_WILL_FREE)) {
1556 spin_unlock(&old->i_lock);
1557 continue;
1558 }
1559 break;
1560 }
1561 if (likely(!old)) {
1562 spin_lock(&inode->i_lock);
1563 inode->i_state |= I_NEW | I_CREATING;
1564 hlist_add_head_rcu(&inode->i_hash, head);
1565 spin_unlock(&inode->i_lock);
1566 spin_unlock(&inode_hash_lock);
1567 return 0;
1568 }
1569 if (unlikely(old->i_state & I_CREATING)) {
1570 spin_unlock(&old->i_lock);
1571 spin_unlock(&inode_hash_lock);
1572 return -EBUSY;
1573 }
1574 __iget(old);
1575 spin_unlock(&old->i_lock);
1576 spin_unlock(&inode_hash_lock);
1577 wait_on_inode(old);
1578 if (unlikely(!inode_unhashed(old))) {
1579 iput(old);
1580 return -EBUSY;
1581 }
1582 iput(old);
1583 }
1584}
1585EXPORT_SYMBOL(insert_inode_locked);
1586
1587int insert_inode_locked4(struct inode *inode, unsigned long hashval,
1588 int (*test)(struct inode *, void *), void *data)
1589{
1590 struct inode *old;
1591
1592 inode->i_state |= I_CREATING;
1593 old = inode_insert5(inode, hashval, test, NULL, data);
1594
1595 if (old != inode) {
1596 iput(old);
1597 return -EBUSY;
1598 }
1599 return 0;
1600}
1601EXPORT_SYMBOL(insert_inode_locked4);
1602
1603
1604int generic_delete_inode(struct inode *inode)
1605{
1606 return 1;
1607}
1608EXPORT_SYMBOL(generic_delete_inode);
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620static void iput_final(struct inode *inode)
1621{
1622 struct super_block *sb = inode->i_sb;
1623 const struct super_operations *op = inode->i_sb->s_op;
1624 unsigned long state;
1625 int drop;
1626
1627 WARN_ON(inode->i_state & I_NEW);
1628
1629 if (op->drop_inode)
1630 drop = op->drop_inode(inode);
1631 else
1632 drop = generic_drop_inode(inode);
1633
1634 if (!drop &&
1635 !(inode->i_state & I_DONTCACHE) &&
1636 (sb->s_flags & SB_ACTIVE)) {
1637 inode_add_lru(inode);
1638 spin_unlock(&inode->i_lock);
1639 return;
1640 }
1641
1642 state = inode->i_state;
1643 if (!drop) {
1644 WRITE_ONCE(inode->i_state, state | I_WILL_FREE);
1645 spin_unlock(&inode->i_lock);
1646
1647 write_inode_now(inode, 1);
1648
1649 spin_lock(&inode->i_lock);
1650 state = inode->i_state;
1651 WARN_ON(state & I_NEW);
1652 state &= ~I_WILL_FREE;
1653 }
1654
1655 WRITE_ONCE(inode->i_state, state | I_FREEING);
1656 if (!list_empty(&inode->i_lru))
1657 inode_lru_list_del(inode);
1658 spin_unlock(&inode->i_lock);
1659
1660 evict(inode);
1661}
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672void iput(struct inode *inode)
1673{
1674 if (!inode)
1675 return;
1676 BUG_ON(inode->i_state & I_CLEAR);
1677retry:
1678 if (atomic_dec_and_lock(&inode->i_count, &inode->i_lock)) {
1679 if (inode->i_nlink && (inode->i_state & I_DIRTY_TIME)) {
1680 atomic_inc(&inode->i_count);
1681 spin_unlock(&inode->i_lock);
1682 trace_writeback_lazytime_iput(inode);
1683 mark_inode_dirty_sync(inode);
1684 goto retry;
1685 }
1686 iput_final(inode);
1687 }
1688}
1689EXPORT_SYMBOL(iput);
1690
1691#ifdef CONFIG_BLOCK
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706int bmap(struct inode *inode, sector_t *block)
1707{
1708 if (!inode->i_mapping->a_ops->bmap)
1709 return -EINVAL;
1710
1711 *block = inode->i_mapping->a_ops->bmap(inode->i_mapping, *block);
1712 return 0;
1713}
1714EXPORT_SYMBOL(bmap);
1715#endif
1716
1717
1718
1719
1720
1721
1722static int relatime_need_update(struct vfsmount *mnt, struct inode *inode,
1723 struct timespec64 now)
1724{
1725
1726 if (!(mnt->mnt_flags & MNT_RELATIME))
1727 return 1;
1728
1729
1730
1731 if (timespec64_compare(&inode->i_mtime, &inode->i_atime) >= 0)
1732 return 1;
1733
1734
1735
1736 if (timespec64_compare(&inode->i_ctime, &inode->i_atime) >= 0)
1737 return 1;
1738
1739
1740
1741
1742
1743 if ((long)(now.tv_sec - inode->i_atime.tv_sec) >= 24*60*60)
1744 return 1;
1745
1746
1747
1748 return 0;
1749}
1750
1751int generic_update_time(struct inode *inode, struct timespec64 *time, int flags)
1752{
1753 int dirty_flags = 0;
1754
1755 if (flags & (S_ATIME | S_CTIME | S_MTIME)) {
1756 if (flags & S_ATIME)
1757 inode->i_atime = *time;
1758 if (flags & S_CTIME)
1759 inode->i_ctime = *time;
1760 if (flags & S_MTIME)
1761 inode->i_mtime = *time;
1762
1763 if (inode->i_sb->s_flags & SB_LAZYTIME)
1764 dirty_flags |= I_DIRTY_TIME;
1765 else
1766 dirty_flags |= I_DIRTY_SYNC;
1767 }
1768
1769 if ((flags & S_VERSION) && inode_maybe_inc_iversion(inode, false))
1770 dirty_flags |= I_DIRTY_SYNC;
1771
1772 __mark_inode_dirty(inode, dirty_flags);
1773 return 0;
1774}
1775EXPORT_SYMBOL(generic_update_time);
1776
1777
1778
1779
1780
1781static int update_time(struct inode *inode, struct timespec64 *time, int flags)
1782{
1783 if (inode->i_op->update_time)
1784 return inode->i_op->update_time(inode, time, flags);
1785 return generic_update_time(inode, time, flags);
1786}
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797bool atime_needs_update(const struct path *path, struct inode *inode)
1798{
1799 struct vfsmount *mnt = path->mnt;
1800 struct timespec64 now;
1801
1802 if (inode->i_flags & S_NOATIME)
1803 return false;
1804
1805
1806
1807
1808 if (HAS_UNMAPPED_ID(mnt_user_ns(mnt), inode))
1809 return false;
1810
1811 if (IS_NOATIME(inode))
1812 return false;
1813 if ((inode->i_sb->s_flags & SB_NODIRATIME) && S_ISDIR(inode->i_mode))
1814 return false;
1815
1816 if (mnt->mnt_flags & MNT_NOATIME)
1817 return false;
1818 if ((mnt->mnt_flags & MNT_NODIRATIME) && S_ISDIR(inode->i_mode))
1819 return false;
1820
1821 now = current_time(inode);
1822
1823 if (!relatime_need_update(mnt, inode, now))
1824 return false;
1825
1826 if (timespec64_equal(&inode->i_atime, &now))
1827 return false;
1828
1829 return true;
1830}
1831
1832void touch_atime(const struct path *path)
1833{
1834 struct vfsmount *mnt = path->mnt;
1835 struct inode *inode = d_inode(path->dentry);
1836 struct timespec64 now;
1837
1838 if (!atime_needs_update(path, inode))
1839 return;
1840
1841 if (!sb_start_write_trylock(inode->i_sb))
1842 return;
1843
1844 if (__mnt_want_write(mnt) != 0)
1845 goto skip_update;
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855 now = current_time(inode);
1856 update_time(inode, &now, S_ATIME);
1857 __mnt_drop_write(mnt);
1858skip_update:
1859 sb_end_write(inode->i_sb);
1860}
1861EXPORT_SYMBOL(touch_atime);
1862
1863
1864
1865
1866
1867
1868
1869int should_remove_suid(struct dentry *dentry)
1870{
1871 umode_t mode = d_inode(dentry)->i_mode;
1872 int kill = 0;
1873
1874
1875 if (unlikely(mode & S_ISUID))
1876 kill = ATTR_KILL_SUID;
1877
1878
1879
1880
1881
1882 if (unlikely((mode & S_ISGID) && (mode & S_IXGRP)))
1883 kill |= ATTR_KILL_SGID;
1884
1885 if (unlikely(kill && !capable(CAP_FSETID) && S_ISREG(mode)))
1886 return kill;
1887
1888 return 0;
1889}
1890EXPORT_SYMBOL(should_remove_suid);
1891
1892
1893
1894
1895
1896
1897int dentry_needs_remove_privs(struct dentry *dentry)
1898{
1899 struct inode *inode = d_inode(dentry);
1900 int mask = 0;
1901 int ret;
1902
1903 if (IS_NOSEC(inode))
1904 return 0;
1905
1906 mask = should_remove_suid(dentry);
1907 ret = security_inode_need_killpriv(dentry);
1908 if (ret < 0)
1909 return ret;
1910 if (ret)
1911 mask |= ATTR_KILL_PRIV;
1912 return mask;
1913}
1914
1915static int __remove_privs(struct user_namespace *mnt_userns,
1916 struct dentry *dentry, int kill)
1917{
1918 struct iattr newattrs;
1919
1920 newattrs.ia_valid = ATTR_FORCE | kill;
1921
1922
1923
1924
1925 return notify_change(mnt_userns, dentry, &newattrs, NULL);
1926}
1927
1928
1929
1930
1931
1932int file_remove_privs(struct file *file)
1933{
1934 struct dentry *dentry = file_dentry(file);
1935 struct inode *inode = file_inode(file);
1936 int kill;
1937 int error = 0;
1938
1939
1940
1941
1942
1943
1944
1945 if (IS_NOSEC(inode) || !S_ISREG(inode->i_mode))
1946 return 0;
1947
1948 kill = dentry_needs_remove_privs(dentry);
1949 if (kill < 0)
1950 return kill;
1951 if (kill)
1952 error = __remove_privs(file_mnt_user_ns(file), dentry, kill);
1953 if (!error)
1954 inode_has_no_xattr(inode);
1955
1956 return error;
1957}
1958EXPORT_SYMBOL(file_remove_privs);
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973int file_update_time(struct file *file)
1974{
1975 struct inode *inode = file_inode(file);
1976 struct timespec64 now;
1977 int sync_it = 0;
1978 int ret;
1979
1980
1981 if (IS_NOCMTIME(inode))
1982 return 0;
1983
1984 now = current_time(inode);
1985 if (!timespec64_equal(&inode->i_mtime, &now))
1986 sync_it = S_MTIME;
1987
1988 if (!timespec64_equal(&inode->i_ctime, &now))
1989 sync_it |= S_CTIME;
1990
1991 if (IS_I_VERSION(inode) && inode_iversion_need_inc(inode))
1992 sync_it |= S_VERSION;
1993
1994 if (!sync_it)
1995 return 0;
1996
1997
1998 if (__mnt_want_write_file(file))
1999 return 0;
2000
2001 ret = update_time(inode, &now, sync_it);
2002 __mnt_drop_write_file(file);
2003
2004 return ret;
2005}
2006EXPORT_SYMBOL(file_update_time);
2007
2008
2009int file_modified(struct file *file)
2010{
2011 int err;
2012
2013
2014
2015
2016
2017 err = file_remove_privs(file);
2018 if (err)
2019 return err;
2020
2021 if (unlikely(file->f_mode & FMODE_NOCMTIME))
2022 return 0;
2023
2024 return file_update_time(file);
2025}
2026EXPORT_SYMBOL(file_modified);
2027
2028int inode_needs_sync(struct inode *inode)
2029{
2030 if (IS_SYNC(inode))
2031 return 1;
2032 if (S_ISDIR(inode->i_mode) && IS_DIRSYNC(inode))
2033 return 1;
2034 return 0;
2035}
2036EXPORT_SYMBOL(inode_needs_sync);
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049static void __wait_on_freeing_inode(struct inode *inode)
2050{
2051 wait_queue_head_t *wq;
2052 DEFINE_WAIT_BIT(wait, &inode->i_state, __I_NEW);
2053 wq = bit_waitqueue(&inode->i_state, __I_NEW);
2054 prepare_to_wait(wq, &wait.wq_entry, TASK_UNINTERRUPTIBLE);
2055 spin_unlock(&inode->i_lock);
2056 spin_unlock(&inode_hash_lock);
2057 schedule();
2058 finish_wait(wq, &wait.wq_entry);
2059 spin_lock(&inode_hash_lock);
2060}
2061
2062static __initdata unsigned long ihash_entries;
2063static int __init set_ihash_entries(char *str)
2064{
2065 if (!str)
2066 return 0;
2067 ihash_entries = simple_strtoul(str, &str, 0);
2068 return 1;
2069}
2070__setup("ihash_entries=", set_ihash_entries);
2071
2072
2073
2074
2075void __init inode_init_early(void)
2076{
2077
2078
2079
2080 if (hashdist)
2081 return;
2082
2083 inode_hashtable =
2084 alloc_large_system_hash("Inode-cache",
2085 sizeof(struct hlist_head),
2086 ihash_entries,
2087 14,
2088 HASH_EARLY | HASH_ZERO,
2089 &i_hash_shift,
2090 &i_hash_mask,
2091 0,
2092 0);
2093}
2094
2095void __init inode_init(void)
2096{
2097
2098 inode_cachep = kmem_cache_create("inode_cache",
2099 sizeof(struct inode),
2100 0,
2101 (SLAB_RECLAIM_ACCOUNT|SLAB_PANIC|
2102 SLAB_MEM_SPREAD|SLAB_ACCOUNT),
2103 init_once);
2104
2105
2106 if (!hashdist)
2107 return;
2108
2109 inode_hashtable =
2110 alloc_large_system_hash("Inode-cache",
2111 sizeof(struct hlist_head),
2112 ihash_entries,
2113 14,
2114 HASH_ZERO,
2115 &i_hash_shift,
2116 &i_hash_mask,
2117 0,
2118 0);
2119}
2120
2121void init_special_inode(struct inode *inode, umode_t mode, dev_t rdev)
2122{
2123 inode->i_mode = mode;
2124 if (S_ISCHR(mode)) {
2125 inode->i_fop = &def_chr_fops;
2126 inode->i_rdev = rdev;
2127 } else if (S_ISBLK(mode)) {
2128 inode->i_fop = &def_blk_fops;
2129 inode->i_rdev = rdev;
2130 } else if (S_ISFIFO(mode))
2131 inode->i_fop = &pipefifo_fops;
2132 else if (S_ISSOCK(mode))
2133 ;
2134 else
2135 printk(KERN_DEBUG "init_special_inode: bogus i_mode (%o) for"
2136 " inode %s:%lu\n", mode, inode->i_sb->s_id,
2137 inode->i_ino);
2138}
2139EXPORT_SYMBOL(init_special_inode);
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154void inode_init_owner(struct user_namespace *mnt_userns, struct inode *inode,
2155 const struct inode *dir, umode_t mode)
2156{
2157 inode_fsuid_set(inode, mnt_userns);
2158 if (dir && dir->i_mode & S_ISGID) {
2159 inode->i_gid = dir->i_gid;
2160
2161
2162 if (S_ISDIR(mode))
2163 mode |= S_ISGID;
2164 else if ((mode & (S_ISGID | S_IXGRP)) == (S_ISGID | S_IXGRP) &&
2165 !in_group_p(i_gid_into_mnt(mnt_userns, dir)) &&
2166 !capable_wrt_inode_uidgid(mnt_userns, dir, CAP_FSETID))
2167 mode &= ~S_ISGID;
2168 } else
2169 inode_fsgid_set(inode, mnt_userns);
2170 inode->i_mode = mode;
2171}
2172EXPORT_SYMBOL(inode_init_owner);
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188bool inode_owner_or_capable(struct user_namespace *mnt_userns,
2189 const struct inode *inode)
2190{
2191 kuid_t i_uid;
2192 struct user_namespace *ns;
2193
2194 i_uid = i_uid_into_mnt(mnt_userns, inode);
2195 if (uid_eq(current_fsuid(), i_uid))
2196 return true;
2197
2198 ns = current_user_ns();
2199 if (kuid_has_mapping(ns, i_uid) && ns_capable(ns, CAP_FOWNER))
2200 return true;
2201 return false;
2202}
2203EXPORT_SYMBOL(inode_owner_or_capable);
2204
2205
2206
2207
2208static void __inode_dio_wait(struct inode *inode)
2209{
2210 wait_queue_head_t *wq = bit_waitqueue(&inode->i_state, __I_DIO_WAKEUP);
2211 DEFINE_WAIT_BIT(q, &inode->i_state, __I_DIO_WAKEUP);
2212
2213 do {
2214 prepare_to_wait(wq, &q.wq_entry, TASK_UNINTERRUPTIBLE);
2215 if (atomic_read(&inode->i_dio_count))
2216 schedule();
2217 } while (atomic_read(&inode->i_dio_count));
2218 finish_wait(wq, &q.wq_entry);
2219}
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231void inode_dio_wait(struct inode *inode)
2232{
2233 if (atomic_read(&inode->i_dio_count))
2234 __inode_dio_wait(inode);
2235}
2236EXPORT_SYMBOL(inode_dio_wait);
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254void inode_set_flags(struct inode *inode, unsigned int flags,
2255 unsigned int mask)
2256{
2257 WARN_ON_ONCE(flags & ~mask);
2258 set_mask_bits(&inode->i_flags, mask, flags);
2259}
2260EXPORT_SYMBOL(inode_set_flags);
2261
2262void inode_nohighmem(struct inode *inode)
2263{
2264 mapping_set_gfp_mask(inode->i_mapping, GFP_USER);
2265}
2266EXPORT_SYMBOL(inode_nohighmem);
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277struct timespec64 timestamp_truncate(struct timespec64 t, struct inode *inode)
2278{
2279 struct super_block *sb = inode->i_sb;
2280 unsigned int gran = sb->s_time_gran;
2281
2282 t.tv_sec = clamp(t.tv_sec, sb->s_time_min, sb->s_time_max);
2283 if (unlikely(t.tv_sec == sb->s_time_max || t.tv_sec == sb->s_time_min))
2284 t.tv_nsec = 0;
2285
2286
2287 if (gran == 1)
2288 ;
2289 else if (gran == NSEC_PER_SEC)
2290 t.tv_nsec = 0;
2291 else if (gran > 1 && gran < NSEC_PER_SEC)
2292 t.tv_nsec -= t.tv_nsec % gran;
2293 else
2294 WARN(1, "invalid file time granularity: %u", gran);
2295 return t;
2296}
2297EXPORT_SYMBOL(timestamp_truncate);
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309struct timespec64 current_time(struct inode *inode)
2310{
2311 struct timespec64 now;
2312
2313 ktime_get_coarse_real_ts64(&now);
2314
2315 if (unlikely(!inode->i_sb)) {
2316 WARN(1, "current_time() called with uninitialized super_block in the inode");
2317 return now;
2318 }
2319
2320 return timestamp_truncate(now, inode);
2321}
2322EXPORT_SYMBOL(current_time);
2323