1
2
3
4
5
6#include <linux/export.h>
7#include <linux/fs.h>
8#include <linux/mm.h>
9#include <linux/backing-dev.h>
10#include <linux/hash.h>
11#include <linux/swap.h>
12#include <linux/security.h>
13#include <linux/cdev.h>
14#include <linux/memblock.h>
15#include <linux/fsnotify.h>
16#include <linux/mount.h>
17#include <linux/posix_acl.h>
18#include <linux/prefetch.h>
19#include <linux/buffer_head.h>
20#include <linux/ratelimit.h>
21#include <linux/list_lru.h>
22#include <linux/iversion.h>
23#include <trace/events/writeback.h>
24#include "internal.h"
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57static unsigned int i_hash_mask __read_mostly;
58static unsigned int i_hash_shift __read_mostly;
59static struct hlist_head *inode_hashtable __read_mostly;
60static __cacheline_aligned_in_smp DEFINE_SPINLOCK(inode_hash_lock);
61
62
63
64
65
66const struct address_space_operations empty_aops = {
67};
68EXPORT_SYMBOL(empty_aops);
69
70
71
72
73struct inodes_stat_t inodes_stat;
74
75static DEFINE_PER_CPU(unsigned long, nr_inodes);
76static DEFINE_PER_CPU(unsigned long, nr_unused);
77
78static struct kmem_cache *inode_cachep __read_mostly;
79
80static long get_nr_inodes(void)
81{
82 int i;
83 long sum = 0;
84 for_each_possible_cpu(i)
85 sum += per_cpu(nr_inodes, i);
86 return sum < 0 ? 0 : sum;
87}
88
89static inline long get_nr_inodes_unused(void)
90{
91 int i;
92 long sum = 0;
93 for_each_possible_cpu(i)
94 sum += per_cpu(nr_unused, i);
95 return sum < 0 ? 0 : sum;
96}
97
98long get_nr_dirty_inodes(void)
99{
100
101 long nr_dirty = get_nr_inodes() - get_nr_inodes_unused();
102 return nr_dirty > 0 ? nr_dirty : 0;
103}
104
105
106
107
108#ifdef CONFIG_SYSCTL
109int proc_nr_inodes(struct ctl_table *table, int write,
110 void __user *buffer, size_t *lenp, loff_t *ppos)
111{
112 inodes_stat.nr_inodes = get_nr_inodes();
113 inodes_stat.nr_unused = get_nr_inodes_unused();
114 return proc_doulongvec_minmax(table, write, buffer, lenp, ppos);
115}
116#endif
117
118static int no_open(struct inode *inode, struct file *file)
119{
120 return -ENXIO;
121}
122
123
124
125
126
127
128
129
130
131int inode_init_always(struct super_block *sb, struct inode *inode)
132{
133 static const struct inode_operations empty_iops;
134 static const struct file_operations no_open_fops = {.open = no_open};
135 struct address_space *const mapping = &inode->i_data;
136
137 inode->i_sb = sb;
138 inode->i_blkbits = sb->s_blocksize_bits;
139 inode->i_flags = 0;
140 atomic_set(&inode->i_count, 1);
141 inode->i_op = &empty_iops;
142 inode->i_fop = &no_open_fops;
143 inode->__i_nlink = 1;
144 inode->i_opflags = 0;
145 if (sb->s_xattr)
146 inode->i_opflags |= IOP_XATTR;
147 i_uid_write(inode, 0);
148 i_gid_write(inode, 0);
149 atomic_set(&inode->i_writecount, 0);
150 inode->i_size = 0;
151 inode->i_write_hint = WRITE_LIFE_NOT_SET;
152 inode->i_blocks = 0;
153 inode->i_bytes = 0;
154 inode->i_generation = 0;
155 inode->i_pipe = NULL;
156 inode->i_bdev = NULL;
157 inode->i_cdev = NULL;
158 inode->i_link = NULL;
159 inode->i_dir_seq = 0;
160 inode->i_rdev = 0;
161 inode->dirtied_when = 0;
162
163#ifdef CONFIG_CGROUP_WRITEBACK
164 inode->i_wb_frn_winner = 0;
165 inode->i_wb_frn_avg_time = 0;
166 inode->i_wb_frn_history = 0;
167#endif
168
169 if (security_inode_alloc(inode))
170 goto out;
171 spin_lock_init(&inode->i_lock);
172 lockdep_set_class(&inode->i_lock, &sb->s_type->i_lock_key);
173
174 init_rwsem(&inode->i_rwsem);
175 lockdep_set_class(&inode->i_rwsem, &sb->s_type->i_mutex_key);
176
177 atomic_set(&inode->i_dio_count, 0);
178
179 mapping->a_ops = &empty_aops;
180 mapping->host = inode;
181 mapping->flags = 0;
182 mapping->wb_err = 0;
183 atomic_set(&mapping->i_mmap_writable, 0);
184 mapping_set_gfp_mask(mapping, GFP_HIGHUSER_MOVABLE);
185 mapping->private_data = NULL;
186 mapping->writeback_index = 0;
187 inode->i_private = NULL;
188 inode->i_mapping = mapping;
189 INIT_HLIST_HEAD(&inode->i_dentry);
190#ifdef CONFIG_FS_POSIX_ACL
191 inode->i_acl = inode->i_default_acl = ACL_NOT_CACHED;
192#endif
193
194#ifdef CONFIG_FSNOTIFY
195 inode->i_fsnotify_mask = 0;
196#endif
197 inode->i_flctx = NULL;
198 this_cpu_inc(nr_inodes);
199
200 return 0;
201out:
202 return -ENOMEM;
203}
204EXPORT_SYMBOL(inode_init_always);
205
206void free_inode_nonrcu(struct inode *inode)
207{
208 kmem_cache_free(inode_cachep, inode);
209}
210EXPORT_SYMBOL(free_inode_nonrcu);
211
212static void i_callback(struct rcu_head *head)
213{
214 struct inode *inode = container_of(head, struct inode, i_rcu);
215 if (inode->free_inode)
216 inode->free_inode(inode);
217 else
218 free_inode_nonrcu(inode);
219}
220
221static struct inode *alloc_inode(struct super_block *sb)
222{
223 const struct super_operations *ops = sb->s_op;
224 struct inode *inode;
225
226 if (ops->alloc_inode)
227 inode = ops->alloc_inode(sb);
228 else
229 inode = kmem_cache_alloc(inode_cachep, GFP_KERNEL);
230
231 if (!inode)
232 return NULL;
233
234 if (unlikely(inode_init_always(sb, inode))) {
235 if (ops->destroy_inode) {
236 ops->destroy_inode(inode);
237 if (!ops->free_inode)
238 return NULL;
239 }
240 inode->free_inode = ops->free_inode;
241 i_callback(&inode->i_rcu);
242 return NULL;
243 }
244
245 return inode;
246}
247
248void __destroy_inode(struct inode *inode)
249{
250 BUG_ON(inode_has_buffers(inode));
251 inode_detach_wb(inode);
252 security_inode_free(inode);
253 fsnotify_inode_delete(inode);
254 locks_free_lock_context(inode);
255 if (!inode->i_nlink) {
256 WARN_ON(atomic_long_read(&inode->i_sb->s_remove_count) == 0);
257 atomic_long_dec(&inode->i_sb->s_remove_count);
258 }
259
260#ifdef CONFIG_FS_POSIX_ACL
261 if (inode->i_acl && !is_uncached_acl(inode->i_acl))
262 posix_acl_release(inode->i_acl);
263 if (inode->i_default_acl && !is_uncached_acl(inode->i_default_acl))
264 posix_acl_release(inode->i_default_acl);
265#endif
266 this_cpu_dec(nr_inodes);
267}
268EXPORT_SYMBOL(__destroy_inode);
269
270static void destroy_inode(struct inode *inode)
271{
272 const struct super_operations *ops = inode->i_sb->s_op;
273
274 BUG_ON(!list_empty(&inode->i_lru));
275 __destroy_inode(inode);
276 if (ops->destroy_inode) {
277 ops->destroy_inode(inode);
278 if (!ops->free_inode)
279 return;
280 }
281 inode->free_inode = ops->free_inode;
282 call_rcu(&inode->i_rcu, i_callback);
283}
284
285
286
287
288
289
290
291
292
293
294
295
296void drop_nlink(struct inode *inode)
297{
298 WARN_ON(inode->i_nlink == 0);
299 inode->__i_nlink--;
300 if (!inode->i_nlink)
301 atomic_long_inc(&inode->i_sb->s_remove_count);
302}
303EXPORT_SYMBOL(drop_nlink);
304
305
306
307
308
309
310
311
312
313void clear_nlink(struct inode *inode)
314{
315 if (inode->i_nlink) {
316 inode->__i_nlink = 0;
317 atomic_long_inc(&inode->i_sb->s_remove_count);
318 }
319}
320EXPORT_SYMBOL(clear_nlink);
321
322
323
324
325
326
327
328
329
330void set_nlink(struct inode *inode, unsigned int nlink)
331{
332 if (!nlink) {
333 clear_nlink(inode);
334 } else {
335
336 if (inode->i_nlink == 0)
337 atomic_long_dec(&inode->i_sb->s_remove_count);
338
339 inode->__i_nlink = nlink;
340 }
341}
342EXPORT_SYMBOL(set_nlink);
343
344
345
346
347
348
349
350
351
352void inc_nlink(struct inode *inode)
353{
354 if (unlikely(inode->i_nlink == 0)) {
355 WARN_ON(!(inode->i_state & I_LINKABLE));
356 atomic_long_dec(&inode->i_sb->s_remove_count);
357 }
358
359 inode->__i_nlink++;
360}
361EXPORT_SYMBOL(inc_nlink);
362
363static void __address_space_init_once(struct address_space *mapping)
364{
365 xa_init_flags(&mapping->i_pages, XA_FLAGS_LOCK_IRQ | XA_FLAGS_ACCOUNT);
366 init_rwsem(&mapping->i_mmap_rwsem);
367 INIT_LIST_HEAD(&mapping->private_list);
368 spin_lock_init(&mapping->private_lock);
369 mapping->i_mmap = RB_ROOT_CACHED;
370}
371
372void address_space_init_once(struct address_space *mapping)
373{
374 memset(mapping, 0, sizeof(*mapping));
375 __address_space_init_once(mapping);
376}
377EXPORT_SYMBOL(address_space_init_once);
378
379
380
381
382
383
384void inode_init_once(struct inode *inode)
385{
386 memset(inode, 0, sizeof(*inode));
387 INIT_HLIST_NODE(&inode->i_hash);
388 INIT_LIST_HEAD(&inode->i_devices);
389 INIT_LIST_HEAD(&inode->i_io_list);
390 INIT_LIST_HEAD(&inode->i_wb_list);
391 INIT_LIST_HEAD(&inode->i_lru);
392 __address_space_init_once(&inode->i_data);
393 i_size_ordered_init(inode);
394}
395EXPORT_SYMBOL(inode_init_once);
396
397static void init_once(void *foo)
398{
399 struct inode *inode = (struct inode *) foo;
400
401 inode_init_once(inode);
402}
403
404
405
406
407void __iget(struct inode *inode)
408{
409 atomic_inc(&inode->i_count);
410}
411
412
413
414
415void ihold(struct inode *inode)
416{
417 WARN_ON(atomic_inc_return(&inode->i_count) < 2);
418}
419EXPORT_SYMBOL(ihold);
420
421static void inode_lru_list_add(struct inode *inode)
422{
423 if (list_lru_add(&inode->i_sb->s_inode_lru, &inode->i_lru))
424 this_cpu_inc(nr_unused);
425 else
426 inode->i_state |= I_REFERENCED;
427}
428
429
430
431
432
433
434void inode_add_lru(struct inode *inode)
435{
436 if (!(inode->i_state & (I_DIRTY_ALL | I_SYNC |
437 I_FREEING | I_WILL_FREE)) &&
438 !atomic_read(&inode->i_count) && inode->i_sb->s_flags & SB_ACTIVE)
439 inode_lru_list_add(inode);
440}
441
442
443static void inode_lru_list_del(struct inode *inode)
444{
445
446 if (list_lru_del(&inode->i_sb->s_inode_lru, &inode->i_lru))
447 this_cpu_dec(nr_unused);
448}
449
450
451
452
453
454void inode_sb_list_add(struct inode *inode)
455{
456 spin_lock(&inode->i_sb->s_inode_list_lock);
457 list_add(&inode->i_sb_list, &inode->i_sb->s_inodes);
458 spin_unlock(&inode->i_sb->s_inode_list_lock);
459}
460EXPORT_SYMBOL_GPL(inode_sb_list_add);
461
462static inline void inode_sb_list_del(struct inode *inode)
463{
464 if (!list_empty(&inode->i_sb_list)) {
465 spin_lock(&inode->i_sb->s_inode_list_lock);
466 list_del_init(&inode->i_sb_list);
467 spin_unlock(&inode->i_sb->s_inode_list_lock);
468 }
469}
470
471static unsigned long hash(struct super_block *sb, unsigned long hashval)
472{
473 unsigned long tmp;
474
475 tmp = (hashval * (unsigned long)sb) ^ (GOLDEN_RATIO_PRIME + hashval) /
476 L1_CACHE_BYTES;
477 tmp = tmp ^ ((tmp ^ GOLDEN_RATIO_PRIME) >> i_hash_shift);
478 return tmp & i_hash_mask;
479}
480
481
482
483
484
485
486
487
488
489void __insert_inode_hash(struct inode *inode, unsigned long hashval)
490{
491 struct hlist_head *b = inode_hashtable + hash(inode->i_sb, hashval);
492
493 spin_lock(&inode_hash_lock);
494 spin_lock(&inode->i_lock);
495 hlist_add_head(&inode->i_hash, b);
496 spin_unlock(&inode->i_lock);
497 spin_unlock(&inode_hash_lock);
498}
499EXPORT_SYMBOL(__insert_inode_hash);
500
501
502
503
504
505
506
507void __remove_inode_hash(struct inode *inode)
508{
509 spin_lock(&inode_hash_lock);
510 spin_lock(&inode->i_lock);
511 hlist_del_init(&inode->i_hash);
512 spin_unlock(&inode->i_lock);
513 spin_unlock(&inode_hash_lock);
514}
515EXPORT_SYMBOL(__remove_inode_hash);
516
517void clear_inode(struct inode *inode)
518{
519
520
521
522
523
524 xa_lock_irq(&inode->i_data.i_pages);
525 BUG_ON(inode->i_data.nrpages);
526 BUG_ON(inode->i_data.nrexceptional);
527 xa_unlock_irq(&inode->i_data.i_pages);
528 BUG_ON(!list_empty(&inode->i_data.private_list));
529 BUG_ON(!(inode->i_state & I_FREEING));
530 BUG_ON(inode->i_state & I_CLEAR);
531 BUG_ON(!list_empty(&inode->i_wb_list));
532
533 inode->i_state = I_FREEING | I_CLEAR;
534}
535EXPORT_SYMBOL(clear_inode);
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550static void evict(struct inode *inode)
551{
552 const struct super_operations *op = inode->i_sb->s_op;
553
554 BUG_ON(!(inode->i_state & I_FREEING));
555 BUG_ON(!list_empty(&inode->i_lru));
556
557 if (!list_empty(&inode->i_io_list))
558 inode_io_list_del(inode);
559
560 inode_sb_list_del(inode);
561
562
563
564
565
566
567
568 inode_wait_for_writeback(inode);
569
570 if (op->evict_inode) {
571 op->evict_inode(inode);
572 } else {
573 truncate_inode_pages_final(&inode->i_data);
574 clear_inode(inode);
575 }
576 if (S_ISBLK(inode->i_mode) && inode->i_bdev)
577 bd_forget(inode);
578 if (S_ISCHR(inode->i_mode) && inode->i_cdev)
579 cd_forget(inode);
580
581 remove_inode_hash(inode);
582
583 spin_lock(&inode->i_lock);
584 wake_up_bit(&inode->i_state, __I_NEW);
585 BUG_ON(inode->i_state != (I_FREEING | I_CLEAR));
586 spin_unlock(&inode->i_lock);
587
588 destroy_inode(inode);
589}
590
591
592
593
594
595
596
597
598static void dispose_list(struct list_head *head)
599{
600 while (!list_empty(head)) {
601 struct inode *inode;
602
603 inode = list_first_entry(head, struct inode, i_lru);
604 list_del_init(&inode->i_lru);
605
606 evict(inode);
607 cond_resched();
608 }
609}
610
611
612
613
614
615
616
617
618
619
620void evict_inodes(struct super_block *sb)
621{
622 struct inode *inode, *next;
623 LIST_HEAD(dispose);
624
625again:
626 spin_lock(&sb->s_inode_list_lock);
627 list_for_each_entry_safe(inode, next, &sb->s_inodes, i_sb_list) {
628 if (atomic_read(&inode->i_count))
629 continue;
630
631 spin_lock(&inode->i_lock);
632 if (inode->i_state & (I_NEW | I_FREEING | I_WILL_FREE)) {
633 spin_unlock(&inode->i_lock);
634 continue;
635 }
636
637 inode->i_state |= I_FREEING;
638 inode_lru_list_del(inode);
639 spin_unlock(&inode->i_lock);
640 list_add(&inode->i_lru, &dispose);
641
642
643
644
645
646
647 if (need_resched()) {
648 spin_unlock(&sb->s_inode_list_lock);
649 cond_resched();
650 dispose_list(&dispose);
651 goto again;
652 }
653 }
654 spin_unlock(&sb->s_inode_list_lock);
655
656 dispose_list(&dispose);
657}
658EXPORT_SYMBOL_GPL(evict_inodes);
659
660
661
662
663
664
665
666
667
668
669
670int invalidate_inodes(struct super_block *sb, bool kill_dirty)
671{
672 int busy = 0;
673 struct inode *inode, *next;
674 LIST_HEAD(dispose);
675
676 spin_lock(&sb->s_inode_list_lock);
677 list_for_each_entry_safe(inode, next, &sb->s_inodes, i_sb_list) {
678 spin_lock(&inode->i_lock);
679 if (inode->i_state & (I_NEW | I_FREEING | I_WILL_FREE)) {
680 spin_unlock(&inode->i_lock);
681 continue;
682 }
683 if (inode->i_state & I_DIRTY_ALL && !kill_dirty) {
684 spin_unlock(&inode->i_lock);
685 busy = 1;
686 continue;
687 }
688 if (atomic_read(&inode->i_count)) {
689 spin_unlock(&inode->i_lock);
690 busy = 1;
691 continue;
692 }
693
694 inode->i_state |= I_FREEING;
695 inode_lru_list_del(inode);
696 spin_unlock(&inode->i_lock);
697 list_add(&inode->i_lru, &dispose);
698 }
699 spin_unlock(&sb->s_inode_list_lock);
700
701 dispose_list(&dispose);
702
703 return busy;
704}
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721static enum lru_status inode_lru_isolate(struct list_head *item,
722 struct list_lru_one *lru, spinlock_t *lru_lock, void *arg)
723{
724 struct list_head *freeable = arg;
725 struct inode *inode = container_of(item, struct inode, i_lru);
726
727
728
729
730
731 if (!spin_trylock(&inode->i_lock))
732 return LRU_SKIP;
733
734
735
736
737
738 if (atomic_read(&inode->i_count) ||
739 (inode->i_state & ~I_REFERENCED)) {
740 list_lru_isolate(lru, &inode->i_lru);
741 spin_unlock(&inode->i_lock);
742 this_cpu_dec(nr_unused);
743 return LRU_REMOVED;
744 }
745
746
747 if (inode->i_state & I_REFERENCED) {
748 inode->i_state &= ~I_REFERENCED;
749 spin_unlock(&inode->i_lock);
750 return LRU_ROTATE;
751 }
752
753 if (inode_has_buffers(inode) || inode->i_data.nrpages) {
754 __iget(inode);
755 spin_unlock(&inode->i_lock);
756 spin_unlock(lru_lock);
757 if (remove_inode_buffers(inode)) {
758 unsigned long reap;
759 reap = invalidate_mapping_pages(&inode->i_data, 0, -1);
760 if (current_is_kswapd())
761 __count_vm_events(KSWAPD_INODESTEAL, reap);
762 else
763 __count_vm_events(PGINODESTEAL, reap);
764 if (current->reclaim_state)
765 current->reclaim_state->reclaimed_slab += reap;
766 }
767 iput(inode);
768 spin_lock(lru_lock);
769 return LRU_RETRY;
770 }
771
772 WARN_ON(inode->i_state & I_NEW);
773 inode->i_state |= I_FREEING;
774 list_lru_isolate_move(lru, &inode->i_lru, freeable);
775 spin_unlock(&inode->i_lock);
776
777 this_cpu_dec(nr_unused);
778 return LRU_REMOVED;
779}
780
781
782
783
784
785
786
787long prune_icache_sb(struct super_block *sb, struct shrink_control *sc)
788{
789 LIST_HEAD(freeable);
790 long freed;
791
792 freed = list_lru_shrink_walk(&sb->s_inode_lru, sc,
793 inode_lru_isolate, &freeable);
794 dispose_list(&freeable);
795 return freed;
796}
797
798static void __wait_on_freeing_inode(struct inode *inode);
799
800
801
802static struct inode *find_inode(struct super_block *sb,
803 struct hlist_head *head,
804 int (*test)(struct inode *, void *),
805 void *data)
806{
807 struct inode *inode = NULL;
808
809repeat:
810 hlist_for_each_entry(inode, head, i_hash) {
811 if (inode->i_sb != sb)
812 continue;
813 if (!test(inode, data))
814 continue;
815 spin_lock(&inode->i_lock);
816 if (inode->i_state & (I_FREEING|I_WILL_FREE)) {
817 __wait_on_freeing_inode(inode);
818 goto repeat;
819 }
820 if (unlikely(inode->i_state & I_CREATING)) {
821 spin_unlock(&inode->i_lock);
822 return ERR_PTR(-ESTALE);
823 }
824 __iget(inode);
825 spin_unlock(&inode->i_lock);
826 return inode;
827 }
828 return NULL;
829}
830
831
832
833
834
835static struct inode *find_inode_fast(struct super_block *sb,
836 struct hlist_head *head, unsigned long ino)
837{
838 struct inode *inode = NULL;
839
840repeat:
841 hlist_for_each_entry(inode, head, i_hash) {
842 if (inode->i_ino != ino)
843 continue;
844 if (inode->i_sb != sb)
845 continue;
846 spin_lock(&inode->i_lock);
847 if (inode->i_state & (I_FREEING|I_WILL_FREE)) {
848 __wait_on_freeing_inode(inode);
849 goto repeat;
850 }
851 if (unlikely(inode->i_state & I_CREATING)) {
852 spin_unlock(&inode->i_lock);
853 return ERR_PTR(-ESTALE);
854 }
855 __iget(inode);
856 spin_unlock(&inode->i_lock);
857 return inode;
858 }
859 return NULL;
860}
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877#define LAST_INO_BATCH 1024
878static DEFINE_PER_CPU(unsigned int, last_ino);
879
880unsigned int get_next_ino(void)
881{
882 unsigned int *p = &get_cpu_var(last_ino);
883 unsigned int res = *p;
884
885#ifdef CONFIG_SMP
886 if (unlikely((res & (LAST_INO_BATCH-1)) == 0)) {
887 static atomic_t shared_last_ino;
888 int next = atomic_add_return(LAST_INO_BATCH, &shared_last_ino);
889
890 res = next - LAST_INO_BATCH;
891 }
892#endif
893
894 res++;
895
896 if (unlikely(!res))
897 res++;
898 *p = res;
899 put_cpu_var(last_ino);
900 return res;
901}
902EXPORT_SYMBOL(get_next_ino);
903
904
905
906
907
908
909
910
911
912
913
914struct inode *new_inode_pseudo(struct super_block *sb)
915{
916 struct inode *inode = alloc_inode(sb);
917
918 if (inode) {
919 spin_lock(&inode->i_lock);
920 inode->i_state = 0;
921 spin_unlock(&inode->i_lock);
922 INIT_LIST_HEAD(&inode->i_sb_list);
923 }
924 return inode;
925}
926
927
928
929
930
931
932
933
934
935
936
937
938
939struct inode *new_inode(struct super_block *sb)
940{
941 struct inode *inode;
942
943 spin_lock_prefetch(&sb->s_inode_list_lock);
944
945 inode = new_inode_pseudo(sb);
946 if (inode)
947 inode_sb_list_add(inode);
948 return inode;
949}
950EXPORT_SYMBOL(new_inode);
951
952#ifdef CONFIG_DEBUG_LOCK_ALLOC
953void lockdep_annotate_inode_mutex_key(struct inode *inode)
954{
955 if (S_ISDIR(inode->i_mode)) {
956 struct file_system_type *type = inode->i_sb->s_type;
957
958
959 if (lockdep_match_class(&inode->i_rwsem, &type->i_mutex_key)) {
960
961
962
963
964 init_rwsem(&inode->i_rwsem);
965 lockdep_set_class(&inode->i_rwsem,
966 &type->i_mutex_dir_key);
967 }
968 }
969}
970EXPORT_SYMBOL(lockdep_annotate_inode_mutex_key);
971#endif
972
973
974
975
976
977
978
979
980void unlock_new_inode(struct inode *inode)
981{
982 lockdep_annotate_inode_mutex_key(inode);
983 spin_lock(&inode->i_lock);
984 WARN_ON(!(inode->i_state & I_NEW));
985 inode->i_state &= ~I_NEW & ~I_CREATING;
986 smp_mb();
987 wake_up_bit(&inode->i_state, __I_NEW);
988 spin_unlock(&inode->i_lock);
989}
990EXPORT_SYMBOL(unlock_new_inode);
991
992void discard_new_inode(struct inode *inode)
993{
994 lockdep_annotate_inode_mutex_key(inode);
995 spin_lock(&inode->i_lock);
996 WARN_ON(!(inode->i_state & I_NEW));
997 inode->i_state &= ~I_NEW;
998 smp_mb();
999 wake_up_bit(&inode->i_state, __I_NEW);
1000 spin_unlock(&inode->i_lock);
1001 iput(inode);
1002}
1003EXPORT_SYMBOL(discard_new_inode);
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014void lock_two_nondirectories(struct inode *inode1, struct inode *inode2)
1015{
1016 if (inode1 > inode2)
1017 swap(inode1, inode2);
1018
1019 if (inode1 && !S_ISDIR(inode1->i_mode))
1020 inode_lock(inode1);
1021 if (inode2 && !S_ISDIR(inode2->i_mode) && inode2 != inode1)
1022 inode_lock_nested(inode2, I_MUTEX_NONDIR2);
1023}
1024EXPORT_SYMBOL(lock_two_nondirectories);
1025
1026
1027
1028
1029
1030
1031void unlock_two_nondirectories(struct inode *inode1, struct inode *inode2)
1032{
1033 if (inode1 && !S_ISDIR(inode1->i_mode))
1034 inode_unlock(inode1);
1035 if (inode2 && !S_ISDIR(inode2->i_mode) && inode2 != inode1)
1036 inode_unlock(inode2);
1037}
1038EXPORT_SYMBOL(unlock_two_nondirectories);
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060struct inode *inode_insert5(struct inode *inode, unsigned long hashval,
1061 int (*test)(struct inode *, void *),
1062 int (*set)(struct inode *, void *), void *data)
1063{
1064 struct hlist_head *head = inode_hashtable + hash(inode->i_sb, hashval);
1065 struct inode *old;
1066 bool creating = inode->i_state & I_CREATING;
1067
1068again:
1069 spin_lock(&inode_hash_lock);
1070 old = find_inode(inode->i_sb, head, test, data);
1071 if (unlikely(old)) {
1072
1073
1074
1075
1076 spin_unlock(&inode_hash_lock);
1077 if (IS_ERR(old))
1078 return NULL;
1079 wait_on_inode(old);
1080 if (unlikely(inode_unhashed(old))) {
1081 iput(old);
1082 goto again;
1083 }
1084 return old;
1085 }
1086
1087 if (set && unlikely(set(inode, data))) {
1088 inode = NULL;
1089 goto unlock;
1090 }
1091
1092
1093
1094
1095
1096 spin_lock(&inode->i_lock);
1097 inode->i_state |= I_NEW;
1098 hlist_add_head(&inode->i_hash, head);
1099 spin_unlock(&inode->i_lock);
1100 if (!creating)
1101 inode_sb_list_add(inode);
1102unlock:
1103 spin_unlock(&inode_hash_lock);
1104
1105 return inode;
1106}
1107EXPORT_SYMBOL(inode_insert5);
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129struct inode *iget5_locked(struct super_block *sb, unsigned long hashval,
1130 int (*test)(struct inode *, void *),
1131 int (*set)(struct inode *, void *), void *data)
1132{
1133 struct inode *inode = ilookup5(sb, hashval, test, data);
1134
1135 if (!inode) {
1136 struct inode *new = alloc_inode(sb);
1137
1138 if (new) {
1139 new->i_state = 0;
1140 inode = inode_insert5(new, hashval, test, set, data);
1141 if (unlikely(inode != new))
1142 destroy_inode(new);
1143 }
1144 }
1145 return inode;
1146}
1147EXPORT_SYMBOL(iget5_locked);
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162struct inode *iget_locked(struct super_block *sb, unsigned long ino)
1163{
1164 struct hlist_head *head = inode_hashtable + hash(sb, ino);
1165 struct inode *inode;
1166again:
1167 spin_lock(&inode_hash_lock);
1168 inode = find_inode_fast(sb, head, ino);
1169 spin_unlock(&inode_hash_lock);
1170 if (inode) {
1171 if (IS_ERR(inode))
1172 return NULL;
1173 wait_on_inode(inode);
1174 if (unlikely(inode_unhashed(inode))) {
1175 iput(inode);
1176 goto again;
1177 }
1178 return inode;
1179 }
1180
1181 inode = alloc_inode(sb);
1182 if (inode) {
1183 struct inode *old;
1184
1185 spin_lock(&inode_hash_lock);
1186
1187 old = find_inode_fast(sb, head, ino);
1188 if (!old) {
1189 inode->i_ino = ino;
1190 spin_lock(&inode->i_lock);
1191 inode->i_state = I_NEW;
1192 hlist_add_head(&inode->i_hash, head);
1193 spin_unlock(&inode->i_lock);
1194 inode_sb_list_add(inode);
1195 spin_unlock(&inode_hash_lock);
1196
1197
1198
1199
1200 return inode;
1201 }
1202
1203
1204
1205
1206
1207
1208 spin_unlock(&inode_hash_lock);
1209 destroy_inode(inode);
1210 if (IS_ERR(old))
1211 return NULL;
1212 inode = old;
1213 wait_on_inode(inode);
1214 if (unlikely(inode_unhashed(inode))) {
1215 iput(inode);
1216 goto again;
1217 }
1218 }
1219 return inode;
1220}
1221EXPORT_SYMBOL(iget_locked);
1222
1223
1224
1225
1226
1227
1228
1229
1230static int test_inode_iunique(struct super_block *sb, unsigned long ino)
1231{
1232 struct hlist_head *b = inode_hashtable + hash(sb, ino);
1233 struct inode *inode;
1234
1235 spin_lock(&inode_hash_lock);
1236 hlist_for_each_entry(inode, b, i_hash) {
1237 if (inode->i_ino == ino && inode->i_sb == sb) {
1238 spin_unlock(&inode_hash_lock);
1239 return 0;
1240 }
1241 }
1242 spin_unlock(&inode_hash_lock);
1243
1244 return 1;
1245}
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261ino_t iunique(struct super_block *sb, ino_t max_reserved)
1262{
1263
1264
1265
1266
1267
1268 static DEFINE_SPINLOCK(iunique_lock);
1269 static unsigned int counter;
1270 ino_t res;
1271
1272 spin_lock(&iunique_lock);
1273 do {
1274 if (counter <= max_reserved)
1275 counter = max_reserved + 1;
1276 res = counter++;
1277 } while (!test_inode_iunique(sb, res));
1278 spin_unlock(&iunique_lock);
1279
1280 return res;
1281}
1282EXPORT_SYMBOL(iunique);
1283
1284struct inode *igrab(struct inode *inode)
1285{
1286 spin_lock(&inode->i_lock);
1287 if (!(inode->i_state & (I_FREEING|I_WILL_FREE))) {
1288 __iget(inode);
1289 spin_unlock(&inode->i_lock);
1290 } else {
1291 spin_unlock(&inode->i_lock);
1292
1293
1294
1295
1296
1297 inode = NULL;
1298 }
1299 return inode;
1300}
1301EXPORT_SYMBOL(igrab);
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319struct inode *ilookup5_nowait(struct super_block *sb, unsigned long hashval,
1320 int (*test)(struct inode *, void *), void *data)
1321{
1322 struct hlist_head *head = inode_hashtable + hash(sb, hashval);
1323 struct inode *inode;
1324
1325 spin_lock(&inode_hash_lock);
1326 inode = find_inode(sb, head, test, data);
1327 spin_unlock(&inode_hash_lock);
1328
1329 return IS_ERR(inode) ? NULL : inode;
1330}
1331EXPORT_SYMBOL(ilookup5_nowait);
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350struct inode *ilookup5(struct super_block *sb, unsigned long hashval,
1351 int (*test)(struct inode *, void *), void *data)
1352{
1353 struct inode *inode;
1354again:
1355 inode = ilookup5_nowait(sb, hashval, test, data);
1356 if (inode) {
1357 wait_on_inode(inode);
1358 if (unlikely(inode_unhashed(inode))) {
1359 iput(inode);
1360 goto again;
1361 }
1362 }
1363 return inode;
1364}
1365EXPORT_SYMBOL(ilookup5);
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375struct inode *ilookup(struct super_block *sb, unsigned long ino)
1376{
1377 struct hlist_head *head = inode_hashtable + hash(sb, ino);
1378 struct inode *inode;
1379again:
1380 spin_lock(&inode_hash_lock);
1381 inode = find_inode_fast(sb, head, ino);
1382 spin_unlock(&inode_hash_lock);
1383
1384 if (inode) {
1385 if (IS_ERR(inode))
1386 return NULL;
1387 wait_on_inode(inode);
1388 if (unlikely(inode_unhashed(inode))) {
1389 iput(inode);
1390 goto again;
1391 }
1392 }
1393 return inode;
1394}
1395EXPORT_SYMBOL(ilookup);
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420struct inode *find_inode_nowait(struct super_block *sb,
1421 unsigned long hashval,
1422 int (*match)(struct inode *, unsigned long,
1423 void *),
1424 void *data)
1425{
1426 struct hlist_head *head = inode_hashtable + hash(sb, hashval);
1427 struct inode *inode, *ret_inode = NULL;
1428 int mval;
1429
1430 spin_lock(&inode_hash_lock);
1431 hlist_for_each_entry(inode, head, i_hash) {
1432 if (inode->i_sb != sb)
1433 continue;
1434 mval = match(inode, hashval, data);
1435 if (mval == 0)
1436 continue;
1437 if (mval == 1)
1438 ret_inode = inode;
1439 goto out;
1440 }
1441out:
1442 spin_unlock(&inode_hash_lock);
1443 return ret_inode;
1444}
1445EXPORT_SYMBOL(find_inode_nowait);
1446
1447int insert_inode_locked(struct inode *inode)
1448{
1449 struct super_block *sb = inode->i_sb;
1450 ino_t ino = inode->i_ino;
1451 struct hlist_head *head = inode_hashtable + hash(sb, ino);
1452
1453 while (1) {
1454 struct inode *old = NULL;
1455 spin_lock(&inode_hash_lock);
1456 hlist_for_each_entry(old, head, i_hash) {
1457 if (old->i_ino != ino)
1458 continue;
1459 if (old->i_sb != sb)
1460 continue;
1461 spin_lock(&old->i_lock);
1462 if (old->i_state & (I_FREEING|I_WILL_FREE)) {
1463 spin_unlock(&old->i_lock);
1464 continue;
1465 }
1466 break;
1467 }
1468 if (likely(!old)) {
1469 spin_lock(&inode->i_lock);
1470 inode->i_state |= I_NEW | I_CREATING;
1471 hlist_add_head(&inode->i_hash, head);
1472 spin_unlock(&inode->i_lock);
1473 spin_unlock(&inode_hash_lock);
1474 return 0;
1475 }
1476 if (unlikely(old->i_state & I_CREATING)) {
1477 spin_unlock(&old->i_lock);
1478 spin_unlock(&inode_hash_lock);
1479 return -EBUSY;
1480 }
1481 __iget(old);
1482 spin_unlock(&old->i_lock);
1483 spin_unlock(&inode_hash_lock);
1484 wait_on_inode(old);
1485 if (unlikely(!inode_unhashed(old))) {
1486 iput(old);
1487 return -EBUSY;
1488 }
1489 iput(old);
1490 }
1491}
1492EXPORT_SYMBOL(insert_inode_locked);
1493
1494int insert_inode_locked4(struct inode *inode, unsigned long hashval,
1495 int (*test)(struct inode *, void *), void *data)
1496{
1497 struct inode *old;
1498
1499 inode->i_state |= I_CREATING;
1500 old = inode_insert5(inode, hashval, test, NULL, data);
1501
1502 if (old != inode) {
1503 iput(old);
1504 return -EBUSY;
1505 }
1506 return 0;
1507}
1508EXPORT_SYMBOL(insert_inode_locked4);
1509
1510
1511int generic_delete_inode(struct inode *inode)
1512{
1513 return 1;
1514}
1515EXPORT_SYMBOL(generic_delete_inode);
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527static void iput_final(struct inode *inode)
1528{
1529 struct super_block *sb = inode->i_sb;
1530 const struct super_operations *op = inode->i_sb->s_op;
1531 int drop;
1532
1533 WARN_ON(inode->i_state & I_NEW);
1534
1535 if (op->drop_inode)
1536 drop = op->drop_inode(inode);
1537 else
1538 drop = generic_drop_inode(inode);
1539
1540 if (!drop && (sb->s_flags & SB_ACTIVE)) {
1541 inode_add_lru(inode);
1542 spin_unlock(&inode->i_lock);
1543 return;
1544 }
1545
1546 if (!drop) {
1547 inode->i_state |= I_WILL_FREE;
1548 spin_unlock(&inode->i_lock);
1549 write_inode_now(inode, 1);
1550 spin_lock(&inode->i_lock);
1551 WARN_ON(inode->i_state & I_NEW);
1552 inode->i_state &= ~I_WILL_FREE;
1553 }
1554
1555 inode->i_state |= I_FREEING;
1556 if (!list_empty(&inode->i_lru))
1557 inode_lru_list_del(inode);
1558 spin_unlock(&inode->i_lock);
1559
1560 evict(inode);
1561}
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572void iput(struct inode *inode)
1573{
1574 if (!inode)
1575 return;
1576 BUG_ON(inode->i_state & I_CLEAR);
1577retry:
1578 if (atomic_dec_and_lock(&inode->i_count, &inode->i_lock)) {
1579 if (inode->i_nlink && (inode->i_state & I_DIRTY_TIME)) {
1580 atomic_inc(&inode->i_count);
1581 spin_unlock(&inode->i_lock);
1582 trace_writeback_lazytime_iput(inode);
1583 mark_inode_dirty_sync(inode);
1584 goto retry;
1585 }
1586 iput_final(inode);
1587 }
1588}
1589EXPORT_SYMBOL(iput);
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602sector_t bmap(struct inode *inode, sector_t block)
1603{
1604 sector_t res = 0;
1605 if (inode->i_mapping->a_ops->bmap)
1606 res = inode->i_mapping->a_ops->bmap(inode->i_mapping, block);
1607 return res;
1608}
1609EXPORT_SYMBOL(bmap);
1610
1611
1612
1613
1614
1615
1616static int relatime_need_update(struct vfsmount *mnt, struct inode *inode,
1617 struct timespec64 now)
1618{
1619
1620 if (!(mnt->mnt_flags & MNT_RELATIME))
1621 return 1;
1622
1623
1624
1625 if (timespec64_compare(&inode->i_mtime, &inode->i_atime) >= 0)
1626 return 1;
1627
1628
1629
1630 if (timespec64_compare(&inode->i_ctime, &inode->i_atime) >= 0)
1631 return 1;
1632
1633
1634
1635
1636
1637 if ((long)(now.tv_sec - inode->i_atime.tv_sec) >= 24*60*60)
1638 return 1;
1639
1640
1641
1642 return 0;
1643}
1644
1645int generic_update_time(struct inode *inode, struct timespec64 *time, int flags)
1646{
1647 int iflags = I_DIRTY_TIME;
1648 bool dirty = false;
1649
1650 if (flags & S_ATIME)
1651 inode->i_atime = *time;
1652 if (flags & S_VERSION)
1653 dirty = inode_maybe_inc_iversion(inode, false);
1654 if (flags & S_CTIME)
1655 inode->i_ctime = *time;
1656 if (flags & S_MTIME)
1657 inode->i_mtime = *time;
1658 if ((flags & (S_ATIME | S_CTIME | S_MTIME)) &&
1659 !(inode->i_sb->s_flags & SB_LAZYTIME))
1660 dirty = true;
1661
1662 if (dirty)
1663 iflags |= I_DIRTY_SYNC;
1664 __mark_inode_dirty(inode, iflags);
1665 return 0;
1666}
1667EXPORT_SYMBOL(generic_update_time);
1668
1669
1670
1671
1672
1673static int update_time(struct inode *inode, struct timespec64 *time, int flags)
1674{
1675 int (*update_time)(struct inode *, struct timespec64 *, int);
1676
1677 update_time = inode->i_op->update_time ? inode->i_op->update_time :
1678 generic_update_time;
1679
1680 return update_time(inode, time, flags);
1681}
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692bool atime_needs_update(const struct path *path, struct inode *inode)
1693{
1694 struct vfsmount *mnt = path->mnt;
1695 struct timespec64 now;
1696
1697 if (inode->i_flags & S_NOATIME)
1698 return false;
1699
1700
1701
1702
1703 if (HAS_UNMAPPED_ID(inode))
1704 return false;
1705
1706 if (IS_NOATIME(inode))
1707 return false;
1708 if ((inode->i_sb->s_flags & SB_NODIRATIME) && S_ISDIR(inode->i_mode))
1709 return false;
1710
1711 if (mnt->mnt_flags & MNT_NOATIME)
1712 return false;
1713 if ((mnt->mnt_flags & MNT_NODIRATIME) && S_ISDIR(inode->i_mode))
1714 return false;
1715
1716 now = current_time(inode);
1717
1718 if (!relatime_need_update(mnt, inode, now))
1719 return false;
1720
1721 if (timespec64_equal(&inode->i_atime, &now))
1722 return false;
1723
1724 return true;
1725}
1726
1727void touch_atime(const struct path *path)
1728{
1729 struct vfsmount *mnt = path->mnt;
1730 struct inode *inode = d_inode(path->dentry);
1731 struct timespec64 now;
1732
1733 if (!atime_needs_update(path, inode))
1734 return;
1735
1736 if (!sb_start_write_trylock(inode->i_sb))
1737 return;
1738
1739 if (__mnt_want_write(mnt) != 0)
1740 goto skip_update;
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750 now = current_time(inode);
1751 update_time(inode, &now, S_ATIME);
1752 __mnt_drop_write(mnt);
1753skip_update:
1754 sb_end_write(inode->i_sb);
1755}
1756EXPORT_SYMBOL(touch_atime);
1757
1758
1759
1760
1761
1762
1763
1764int should_remove_suid(struct dentry *dentry)
1765{
1766 umode_t mode = d_inode(dentry)->i_mode;
1767 int kill = 0;
1768
1769
1770 if (unlikely(mode & S_ISUID))
1771 kill = ATTR_KILL_SUID;
1772
1773
1774
1775
1776
1777 if (unlikely((mode & S_ISGID) && (mode & S_IXGRP)))
1778 kill |= ATTR_KILL_SGID;
1779
1780 if (unlikely(kill && !capable(CAP_FSETID) && S_ISREG(mode)))
1781 return kill;
1782
1783 return 0;
1784}
1785EXPORT_SYMBOL(should_remove_suid);
1786
1787
1788
1789
1790
1791
1792int dentry_needs_remove_privs(struct dentry *dentry)
1793{
1794 struct inode *inode = d_inode(dentry);
1795 int mask = 0;
1796 int ret;
1797
1798 if (IS_NOSEC(inode))
1799 return 0;
1800
1801 mask = should_remove_suid(dentry);
1802 ret = security_inode_need_killpriv(dentry);
1803 if (ret < 0)
1804 return ret;
1805 if (ret)
1806 mask |= ATTR_KILL_PRIV;
1807 return mask;
1808}
1809
1810static int __remove_privs(struct dentry *dentry, int kill)
1811{
1812 struct iattr newattrs;
1813
1814 newattrs.ia_valid = ATTR_FORCE | kill;
1815
1816
1817
1818
1819 return notify_change(dentry, &newattrs, NULL);
1820}
1821
1822
1823
1824
1825
1826int file_remove_privs(struct file *file)
1827{
1828 struct dentry *dentry = file_dentry(file);
1829 struct inode *inode = file_inode(file);
1830 int kill;
1831 int error = 0;
1832
1833
1834
1835
1836
1837
1838
1839 if (IS_NOSEC(inode) || !S_ISREG(inode->i_mode))
1840 return 0;
1841
1842 kill = dentry_needs_remove_privs(dentry);
1843 if (kill < 0)
1844 return kill;
1845 if (kill)
1846 error = __remove_privs(dentry, kill);
1847 if (!error)
1848 inode_has_no_xattr(inode);
1849
1850 return error;
1851}
1852EXPORT_SYMBOL(file_remove_privs);
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867int file_update_time(struct file *file)
1868{
1869 struct inode *inode = file_inode(file);
1870 struct timespec64 now;
1871 int sync_it = 0;
1872 int ret;
1873
1874
1875 if (IS_NOCMTIME(inode))
1876 return 0;
1877
1878 now = current_time(inode);
1879 if (!timespec64_equal(&inode->i_mtime, &now))
1880 sync_it = S_MTIME;
1881
1882 if (!timespec64_equal(&inode->i_ctime, &now))
1883 sync_it |= S_CTIME;
1884
1885 if (IS_I_VERSION(inode) && inode_iversion_need_inc(inode))
1886 sync_it |= S_VERSION;
1887
1888 if (!sync_it)
1889 return 0;
1890
1891
1892 if (__mnt_want_write_file(file))
1893 return 0;
1894
1895 ret = update_time(inode, &now, sync_it);
1896 __mnt_drop_write_file(file);
1897
1898 return ret;
1899}
1900EXPORT_SYMBOL(file_update_time);
1901
1902int inode_needs_sync(struct inode *inode)
1903{
1904 if (IS_SYNC(inode))
1905 return 1;
1906 if (S_ISDIR(inode->i_mode) && IS_DIRSYNC(inode))
1907 return 1;
1908 return 0;
1909}
1910EXPORT_SYMBOL(inode_needs_sync);
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923static void __wait_on_freeing_inode(struct inode *inode)
1924{
1925 wait_queue_head_t *wq;
1926 DEFINE_WAIT_BIT(wait, &inode->i_state, __I_NEW);
1927 wq = bit_waitqueue(&inode->i_state, __I_NEW);
1928 prepare_to_wait(wq, &wait.wq_entry, TASK_UNINTERRUPTIBLE);
1929 spin_unlock(&inode->i_lock);
1930 spin_unlock(&inode_hash_lock);
1931 schedule();
1932 finish_wait(wq, &wait.wq_entry);
1933 spin_lock(&inode_hash_lock);
1934}
1935
1936static __initdata unsigned long ihash_entries;
1937static int __init set_ihash_entries(char *str)
1938{
1939 if (!str)
1940 return 0;
1941 ihash_entries = simple_strtoul(str, &str, 0);
1942 return 1;
1943}
1944__setup("ihash_entries=", set_ihash_entries);
1945
1946
1947
1948
1949void __init inode_init_early(void)
1950{
1951
1952
1953
1954 if (hashdist)
1955 return;
1956
1957 inode_hashtable =
1958 alloc_large_system_hash("Inode-cache",
1959 sizeof(struct hlist_head),
1960 ihash_entries,
1961 14,
1962 HASH_EARLY | HASH_ZERO,
1963 &i_hash_shift,
1964 &i_hash_mask,
1965 0,
1966 0);
1967}
1968
1969void __init inode_init(void)
1970{
1971
1972 inode_cachep = kmem_cache_create("inode_cache",
1973 sizeof(struct inode),
1974 0,
1975 (SLAB_RECLAIM_ACCOUNT|SLAB_PANIC|
1976 SLAB_MEM_SPREAD|SLAB_ACCOUNT),
1977 init_once);
1978
1979
1980 if (!hashdist)
1981 return;
1982
1983 inode_hashtable =
1984 alloc_large_system_hash("Inode-cache",
1985 sizeof(struct hlist_head),
1986 ihash_entries,
1987 14,
1988 HASH_ZERO,
1989 &i_hash_shift,
1990 &i_hash_mask,
1991 0,
1992 0);
1993}
1994
1995void init_special_inode(struct inode *inode, umode_t mode, dev_t rdev)
1996{
1997 inode->i_mode = mode;
1998 if (S_ISCHR(mode)) {
1999 inode->i_fop = &def_chr_fops;
2000 inode->i_rdev = rdev;
2001 } else if (S_ISBLK(mode)) {
2002 inode->i_fop = &def_blk_fops;
2003 inode->i_rdev = rdev;
2004 } else if (S_ISFIFO(mode))
2005 inode->i_fop = &pipefifo_fops;
2006 else if (S_ISSOCK(mode))
2007 ;
2008 else
2009 printk(KERN_DEBUG "init_special_inode: bogus i_mode (%o) for"
2010 " inode %s:%lu\n", mode, inode->i_sb->s_id,
2011 inode->i_ino);
2012}
2013EXPORT_SYMBOL(init_special_inode);
2014
2015
2016
2017
2018
2019
2020
2021void inode_init_owner(struct inode *inode, const struct inode *dir,
2022 umode_t mode)
2023{
2024 inode->i_uid = current_fsuid();
2025 if (dir && dir->i_mode & S_ISGID) {
2026 inode->i_gid = dir->i_gid;
2027
2028
2029 if (S_ISDIR(mode))
2030 mode |= S_ISGID;
2031 else if ((mode & (S_ISGID | S_IXGRP)) == (S_ISGID | S_IXGRP) &&
2032 !in_group_p(inode->i_gid) &&
2033 !capable_wrt_inode_uidgid(dir, CAP_FSETID))
2034 mode &= ~S_ISGID;
2035 } else
2036 inode->i_gid = current_fsgid();
2037 inode->i_mode = mode;
2038}
2039EXPORT_SYMBOL(inode_init_owner);
2040
2041
2042
2043
2044
2045
2046
2047
2048bool inode_owner_or_capable(const struct inode *inode)
2049{
2050 struct user_namespace *ns;
2051
2052 if (uid_eq(current_fsuid(), inode->i_uid))
2053 return true;
2054
2055 ns = current_user_ns();
2056 if (kuid_has_mapping(ns, inode->i_uid) && ns_capable(ns, CAP_FOWNER))
2057 return true;
2058 return false;
2059}
2060EXPORT_SYMBOL(inode_owner_or_capable);
2061
2062
2063
2064
2065static void __inode_dio_wait(struct inode *inode)
2066{
2067 wait_queue_head_t *wq = bit_waitqueue(&inode->i_state, __I_DIO_WAKEUP);
2068 DEFINE_WAIT_BIT(q, &inode->i_state, __I_DIO_WAKEUP);
2069
2070 do {
2071 prepare_to_wait(wq, &q.wq_entry, TASK_UNINTERRUPTIBLE);
2072 if (atomic_read(&inode->i_dio_count))
2073 schedule();
2074 } while (atomic_read(&inode->i_dio_count));
2075 finish_wait(wq, &q.wq_entry);
2076}
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088void inode_dio_wait(struct inode *inode)
2089{
2090 if (atomic_read(&inode->i_dio_count))
2091 __inode_dio_wait(inode);
2092}
2093EXPORT_SYMBOL(inode_dio_wait);
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111void inode_set_flags(struct inode *inode, unsigned int flags,
2112 unsigned int mask)
2113{
2114 WARN_ON_ONCE(flags & ~mask);
2115 set_mask_bits(&inode->i_flags, mask, flags);
2116}
2117EXPORT_SYMBOL(inode_set_flags);
2118
2119void inode_nohighmem(struct inode *inode)
2120{
2121 mapping_set_gfp_mask(inode->i_mapping, GFP_USER);
2122}
2123EXPORT_SYMBOL(inode_nohighmem);
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133struct timespec64 timespec64_trunc(struct timespec64 t, unsigned gran)
2134{
2135
2136 if (gran == 1) {
2137
2138 } else if (gran == NSEC_PER_SEC) {
2139 t.tv_nsec = 0;
2140 } else if (gran > 1 && gran < NSEC_PER_SEC) {
2141 t.tv_nsec -= t.tv_nsec % gran;
2142 } else {
2143 WARN(1, "illegal file time granularity: %u", gran);
2144 }
2145 return t;
2146}
2147EXPORT_SYMBOL(timespec64_trunc);
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159struct timespec64 current_time(struct inode *inode)
2160{
2161 struct timespec64 now;
2162
2163 ktime_get_coarse_real_ts64(&now);
2164
2165 if (unlikely(!inode->i_sb)) {
2166 WARN(1, "current_time() called with uninitialized super_block in the inode");
2167 return now;
2168 }
2169
2170 return timespec64_trunc(now, inode->i_sb->s_time_gran);
2171}
2172EXPORT_SYMBOL(current_time);
2173