1
2
3
4
5#include <linux/export.h>
6#include <linux/fs.h>
7#include <linux/mm.h>
8#include <linux/backing-dev.h>
9#include <linux/hash.h>
10#include <linux/swap.h>
11#include <linux/security.h>
12#include <linux/cdev.h>
13#include <linux/bootmem.h>
14#include <linux/fsnotify.h>
15#include <linux/mount.h>
16#include <linux/posix_acl.h>
17#include <linux/prefetch.h>
18#include <linux/buffer_head.h>
19#include <linux/ratelimit.h>
20#include <linux/list_lru.h>
21#include <linux/iversion.h>
22#include <trace/events/writeback.h>
23#include "internal.h"
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56static unsigned int i_hash_mask __read_mostly;
57static unsigned int i_hash_shift __read_mostly;
58static struct hlist_head *inode_hashtable __read_mostly;
59static __cacheline_aligned_in_smp DEFINE_SPINLOCK(inode_hash_lock);
60
61
62
63
64
65const struct address_space_operations empty_aops = {
66};
67EXPORT_SYMBOL(empty_aops);
68
69
70
71
72struct inodes_stat_t inodes_stat;
73
74static DEFINE_PER_CPU(unsigned long, nr_inodes);
75static DEFINE_PER_CPU(unsigned long, nr_unused);
76
77static struct kmem_cache *inode_cachep __read_mostly;
78
79static long get_nr_inodes(void)
80{
81 int i;
82 long sum = 0;
83 for_each_possible_cpu(i)
84 sum += per_cpu(nr_inodes, i);
85 return sum < 0 ? 0 : sum;
86}
87
88static inline long get_nr_inodes_unused(void)
89{
90 int i;
91 long sum = 0;
92 for_each_possible_cpu(i)
93 sum += per_cpu(nr_unused, i);
94 return sum < 0 ? 0 : sum;
95}
96
97long get_nr_dirty_inodes(void)
98{
99
100 long nr_dirty = get_nr_inodes() - get_nr_inodes_unused();
101 return nr_dirty > 0 ? nr_dirty : 0;
102}
103
104
105
106
107#ifdef CONFIG_SYSCTL
108int proc_nr_inodes(struct ctl_table *table, int write,
109 void __user *buffer, size_t *lenp, loff_t *ppos)
110{
111 inodes_stat.nr_inodes = get_nr_inodes();
112 inodes_stat.nr_unused = get_nr_inodes_unused();
113 return proc_doulongvec_minmax(table, write, buffer, lenp, ppos);
114}
115#endif
116
117static int no_open(struct inode *inode, struct file *file)
118{
119 return -ENXIO;
120}
121
122
123
124
125
126
127
128
129
130int inode_init_always(struct super_block *sb, struct inode *inode)
131{
132 static const struct inode_operations empty_iops;
133 static const struct file_operations no_open_fops = {.open = no_open};
134 struct address_space *const mapping = &inode->i_data;
135
136 inode->i_sb = sb;
137 inode->i_blkbits = sb->s_blocksize_bits;
138 inode->i_flags = 0;
139 atomic_set(&inode->i_count, 1);
140 inode->i_op = &empty_iops;
141 inode->i_fop = &no_open_fops;
142 inode->__i_nlink = 1;
143 inode->i_opflags = 0;
144 if (sb->s_xattr)
145 inode->i_opflags |= IOP_XATTR;
146 i_uid_write(inode, 0);
147 i_gid_write(inode, 0);
148 atomic_set(&inode->i_writecount, 0);
149 inode->i_size = 0;
150 inode->i_write_hint = WRITE_LIFE_NOT_SET;
151 inode->i_blocks = 0;
152 inode->i_bytes = 0;
153 inode->i_generation = 0;
154 inode->i_pipe = NULL;
155 inode->i_bdev = NULL;
156 inode->i_cdev = NULL;
157 inode->i_link = NULL;
158 inode->i_dir_seq = 0;
159 inode->i_rdev = 0;
160 inode->dirtied_when = 0;
161
162#ifdef CONFIG_CGROUP_WRITEBACK
163 inode->i_wb_frn_winner = 0;
164 inode->i_wb_frn_avg_time = 0;
165 inode->i_wb_frn_history = 0;
166#endif
167
168 if (security_inode_alloc(inode))
169 goto out;
170 spin_lock_init(&inode->i_lock);
171 lockdep_set_class(&inode->i_lock, &sb->s_type->i_lock_key);
172
173 init_rwsem(&inode->i_rwsem);
174 lockdep_set_class(&inode->i_rwsem, &sb->s_type->i_mutex_key);
175
176 atomic_set(&inode->i_dio_count, 0);
177
178 mapping->a_ops = &empty_aops;
179 mapping->host = inode;
180 mapping->flags = 0;
181 mapping->wb_err = 0;
182 atomic_set(&mapping->i_mmap_writable, 0);
183 mapping_set_gfp_mask(mapping, GFP_HIGHUSER_MOVABLE);
184 mapping->private_data = NULL;
185 mapping->writeback_index = 0;
186 inode->i_private = NULL;
187 inode->i_mapping = mapping;
188 INIT_HLIST_HEAD(&inode->i_dentry);
189#ifdef CONFIG_FS_POSIX_ACL
190 inode->i_acl = inode->i_default_acl = ACL_NOT_CACHED;
191#endif
192
193#ifdef CONFIG_FSNOTIFY
194 inode->i_fsnotify_mask = 0;
195#endif
196 inode->i_flctx = NULL;
197 this_cpu_inc(nr_inodes);
198
199 return 0;
200out:
201 return -ENOMEM;
202}
203EXPORT_SYMBOL(inode_init_always);
204
205static struct inode *alloc_inode(struct super_block *sb)
206{
207 struct inode *inode;
208
209 if (sb->s_op->alloc_inode)
210 inode = sb->s_op->alloc_inode(sb);
211 else
212 inode = kmem_cache_alloc(inode_cachep, GFP_KERNEL);
213
214 if (!inode)
215 return NULL;
216
217 if (unlikely(inode_init_always(sb, inode))) {
218 if (inode->i_sb->s_op->destroy_inode)
219 inode->i_sb->s_op->destroy_inode(inode);
220 else
221 kmem_cache_free(inode_cachep, inode);
222 return NULL;
223 }
224
225 return inode;
226}
227
228void free_inode_nonrcu(struct inode *inode)
229{
230 kmem_cache_free(inode_cachep, inode);
231}
232EXPORT_SYMBOL(free_inode_nonrcu);
233
234void __destroy_inode(struct inode *inode)
235{
236 BUG_ON(inode_has_buffers(inode));
237 inode_detach_wb(inode);
238 security_inode_free(inode);
239 fsnotify_inode_delete(inode);
240 locks_free_lock_context(inode);
241 if (!inode->i_nlink) {
242 WARN_ON(atomic_long_read(&inode->i_sb->s_remove_count) == 0);
243 atomic_long_dec(&inode->i_sb->s_remove_count);
244 }
245
246#ifdef CONFIG_FS_POSIX_ACL
247 if (inode->i_acl && !is_uncached_acl(inode->i_acl))
248 posix_acl_release(inode->i_acl);
249 if (inode->i_default_acl && !is_uncached_acl(inode->i_default_acl))
250 posix_acl_release(inode->i_default_acl);
251#endif
252 this_cpu_dec(nr_inodes);
253}
254EXPORT_SYMBOL(__destroy_inode);
255
256static void i_callback(struct rcu_head *head)
257{
258 struct inode *inode = container_of(head, struct inode, i_rcu);
259 kmem_cache_free(inode_cachep, inode);
260}
261
262static void destroy_inode(struct inode *inode)
263{
264 BUG_ON(!list_empty(&inode->i_lru));
265 __destroy_inode(inode);
266 if (inode->i_sb->s_op->destroy_inode)
267 inode->i_sb->s_op->destroy_inode(inode);
268 else
269 call_rcu(&inode->i_rcu, i_callback);
270}
271
272
273
274
275
276
277
278
279
280
281
282
283void drop_nlink(struct inode *inode)
284{
285 WARN_ON(inode->i_nlink == 0);
286 inode->__i_nlink--;
287 if (!inode->i_nlink)
288 atomic_long_inc(&inode->i_sb->s_remove_count);
289}
290EXPORT_SYMBOL(drop_nlink);
291
292
293
294
295
296
297
298
299
300void clear_nlink(struct inode *inode)
301{
302 if (inode->i_nlink) {
303 inode->__i_nlink = 0;
304 atomic_long_inc(&inode->i_sb->s_remove_count);
305 }
306}
307EXPORT_SYMBOL(clear_nlink);
308
309
310
311
312
313
314
315
316
317void set_nlink(struct inode *inode, unsigned int nlink)
318{
319 if (!nlink) {
320 clear_nlink(inode);
321 } else {
322
323 if (inode->i_nlink == 0)
324 atomic_long_dec(&inode->i_sb->s_remove_count);
325
326 inode->__i_nlink = nlink;
327 }
328}
329EXPORT_SYMBOL(set_nlink);
330
331
332
333
334
335
336
337
338
339void inc_nlink(struct inode *inode)
340{
341 if (unlikely(inode->i_nlink == 0)) {
342 WARN_ON(!(inode->i_state & I_LINKABLE));
343 atomic_long_dec(&inode->i_sb->s_remove_count);
344 }
345
346 inode->__i_nlink++;
347}
348EXPORT_SYMBOL(inc_nlink);
349
350static void __address_space_init_once(struct address_space *mapping)
351{
352 INIT_RADIX_TREE(&mapping->i_pages, GFP_ATOMIC | __GFP_ACCOUNT);
353 init_rwsem(&mapping->i_mmap_rwsem);
354 INIT_LIST_HEAD(&mapping->private_list);
355 spin_lock_init(&mapping->private_lock);
356 mapping->i_mmap = RB_ROOT_CACHED;
357}
358
359void address_space_init_once(struct address_space *mapping)
360{
361 memset(mapping, 0, sizeof(*mapping));
362 __address_space_init_once(mapping);
363}
364EXPORT_SYMBOL(address_space_init_once);
365
366
367
368
369
370
371void inode_init_once(struct inode *inode)
372{
373 memset(inode, 0, sizeof(*inode));
374 INIT_HLIST_NODE(&inode->i_hash);
375 INIT_LIST_HEAD(&inode->i_devices);
376 INIT_LIST_HEAD(&inode->i_io_list);
377 INIT_LIST_HEAD(&inode->i_wb_list);
378 INIT_LIST_HEAD(&inode->i_lru);
379 __address_space_init_once(&inode->i_data);
380 i_size_ordered_init(inode);
381}
382EXPORT_SYMBOL(inode_init_once);
383
384static void init_once(void *foo)
385{
386 struct inode *inode = (struct inode *) foo;
387
388 inode_init_once(inode);
389}
390
391
392
393
394void __iget(struct inode *inode)
395{
396 atomic_inc(&inode->i_count);
397}
398
399
400
401
402void ihold(struct inode *inode)
403{
404 WARN_ON(atomic_inc_return(&inode->i_count) < 2);
405}
406EXPORT_SYMBOL(ihold);
407
408static void inode_lru_list_add(struct inode *inode)
409{
410 if (list_lru_add(&inode->i_sb->s_inode_lru, &inode->i_lru))
411 this_cpu_inc(nr_unused);
412 else
413 inode->i_state |= I_REFERENCED;
414}
415
416
417
418
419
420
421void inode_add_lru(struct inode *inode)
422{
423 if (!(inode->i_state & (I_DIRTY_ALL | I_SYNC |
424 I_FREEING | I_WILL_FREE)) &&
425 !atomic_read(&inode->i_count) && inode->i_sb->s_flags & SB_ACTIVE)
426 inode_lru_list_add(inode);
427}
428
429
430static void inode_lru_list_del(struct inode *inode)
431{
432
433 if (list_lru_del(&inode->i_sb->s_inode_lru, &inode->i_lru))
434 this_cpu_dec(nr_unused);
435}
436
437
438
439
440
441void inode_sb_list_add(struct inode *inode)
442{
443 spin_lock(&inode->i_sb->s_inode_list_lock);
444 list_add(&inode->i_sb_list, &inode->i_sb->s_inodes);
445 spin_unlock(&inode->i_sb->s_inode_list_lock);
446}
447EXPORT_SYMBOL_GPL(inode_sb_list_add);
448
449static inline void inode_sb_list_del(struct inode *inode)
450{
451 if (!list_empty(&inode->i_sb_list)) {
452 spin_lock(&inode->i_sb->s_inode_list_lock);
453 list_del_init(&inode->i_sb_list);
454 spin_unlock(&inode->i_sb->s_inode_list_lock);
455 }
456}
457
458static unsigned long hash(struct super_block *sb, unsigned long hashval)
459{
460 unsigned long tmp;
461
462 tmp = (hashval * (unsigned long)sb) ^ (GOLDEN_RATIO_PRIME + hashval) /
463 L1_CACHE_BYTES;
464 tmp = tmp ^ ((tmp ^ GOLDEN_RATIO_PRIME) >> i_hash_shift);
465 return tmp & i_hash_mask;
466}
467
468
469
470
471
472
473
474
475
476void __insert_inode_hash(struct inode *inode, unsigned long hashval)
477{
478 struct hlist_head *b = inode_hashtable + hash(inode->i_sb, hashval);
479
480 spin_lock(&inode_hash_lock);
481 spin_lock(&inode->i_lock);
482 hlist_add_head(&inode->i_hash, b);
483 spin_unlock(&inode->i_lock);
484 spin_unlock(&inode_hash_lock);
485}
486EXPORT_SYMBOL(__insert_inode_hash);
487
488
489
490
491
492
493
494void __remove_inode_hash(struct inode *inode)
495{
496 spin_lock(&inode_hash_lock);
497 spin_lock(&inode->i_lock);
498 hlist_del_init(&inode->i_hash);
499 spin_unlock(&inode->i_lock);
500 spin_unlock(&inode_hash_lock);
501}
502EXPORT_SYMBOL(__remove_inode_hash);
503
504void clear_inode(struct inode *inode)
505{
506
507
508
509
510
511 xa_lock_irq(&inode->i_data.i_pages);
512 BUG_ON(inode->i_data.nrpages);
513 BUG_ON(inode->i_data.nrexceptional);
514 xa_unlock_irq(&inode->i_data.i_pages);
515 BUG_ON(!list_empty(&inode->i_data.private_list));
516 BUG_ON(!(inode->i_state & I_FREEING));
517 BUG_ON(inode->i_state & I_CLEAR);
518 BUG_ON(!list_empty(&inode->i_wb_list));
519
520 inode->i_state = I_FREEING | I_CLEAR;
521}
522EXPORT_SYMBOL(clear_inode);
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537static void evict(struct inode *inode)
538{
539 const struct super_operations *op = inode->i_sb->s_op;
540
541 BUG_ON(!(inode->i_state & I_FREEING));
542 BUG_ON(!list_empty(&inode->i_lru));
543
544 if (!list_empty(&inode->i_io_list))
545 inode_io_list_del(inode);
546
547 inode_sb_list_del(inode);
548
549
550
551
552
553
554
555 inode_wait_for_writeback(inode);
556
557 if (op->evict_inode) {
558 op->evict_inode(inode);
559 } else {
560 truncate_inode_pages_final(&inode->i_data);
561 clear_inode(inode);
562 }
563 if (S_ISBLK(inode->i_mode) && inode->i_bdev)
564 bd_forget(inode);
565 if (S_ISCHR(inode->i_mode) && inode->i_cdev)
566 cd_forget(inode);
567
568 remove_inode_hash(inode);
569
570 spin_lock(&inode->i_lock);
571 wake_up_bit(&inode->i_state, __I_NEW);
572 BUG_ON(inode->i_state != (I_FREEING | I_CLEAR));
573 spin_unlock(&inode->i_lock);
574
575 destroy_inode(inode);
576}
577
578
579
580
581
582
583
584
585static void dispose_list(struct list_head *head)
586{
587 while (!list_empty(head)) {
588 struct inode *inode;
589
590 inode = list_first_entry(head, struct inode, i_lru);
591 list_del_init(&inode->i_lru);
592
593 evict(inode);
594 cond_resched();
595 }
596}
597
598
599
600
601
602
603
604
605
606
607void evict_inodes(struct super_block *sb)
608{
609 struct inode *inode, *next;
610 LIST_HEAD(dispose);
611
612again:
613 spin_lock(&sb->s_inode_list_lock);
614 list_for_each_entry_safe(inode, next, &sb->s_inodes, i_sb_list) {
615 if (atomic_read(&inode->i_count))
616 continue;
617
618 spin_lock(&inode->i_lock);
619 if (inode->i_state & (I_NEW | I_FREEING | I_WILL_FREE)) {
620 spin_unlock(&inode->i_lock);
621 continue;
622 }
623
624 inode->i_state |= I_FREEING;
625 inode_lru_list_del(inode);
626 spin_unlock(&inode->i_lock);
627 list_add(&inode->i_lru, &dispose);
628
629
630
631
632
633
634 if (need_resched()) {
635 spin_unlock(&sb->s_inode_list_lock);
636 cond_resched();
637 dispose_list(&dispose);
638 goto again;
639 }
640 }
641 spin_unlock(&sb->s_inode_list_lock);
642
643 dispose_list(&dispose);
644}
645EXPORT_SYMBOL_GPL(evict_inodes);
646
647
648
649
650
651
652
653
654
655
656
657int invalidate_inodes(struct super_block *sb, bool kill_dirty)
658{
659 int busy = 0;
660 struct inode *inode, *next;
661 LIST_HEAD(dispose);
662
663 spin_lock(&sb->s_inode_list_lock);
664 list_for_each_entry_safe(inode, next, &sb->s_inodes, i_sb_list) {
665 spin_lock(&inode->i_lock);
666 if (inode->i_state & (I_NEW | I_FREEING | I_WILL_FREE)) {
667 spin_unlock(&inode->i_lock);
668 continue;
669 }
670 if (inode->i_state & I_DIRTY_ALL && !kill_dirty) {
671 spin_unlock(&inode->i_lock);
672 busy = 1;
673 continue;
674 }
675 if (atomic_read(&inode->i_count)) {
676 spin_unlock(&inode->i_lock);
677 busy = 1;
678 continue;
679 }
680
681 inode->i_state |= I_FREEING;
682 inode_lru_list_del(inode);
683 spin_unlock(&inode->i_lock);
684 list_add(&inode->i_lru, &dispose);
685 }
686 spin_unlock(&sb->s_inode_list_lock);
687
688 dispose_list(&dispose);
689
690 return busy;
691}
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708static enum lru_status inode_lru_isolate(struct list_head *item,
709 struct list_lru_one *lru, spinlock_t *lru_lock, void *arg)
710{
711 struct list_head *freeable = arg;
712 struct inode *inode = container_of(item, struct inode, i_lru);
713
714
715
716
717
718 if (!spin_trylock(&inode->i_lock))
719 return LRU_SKIP;
720
721
722
723
724
725 if (atomic_read(&inode->i_count) ||
726 (inode->i_state & ~I_REFERENCED)) {
727 list_lru_isolate(lru, &inode->i_lru);
728 spin_unlock(&inode->i_lock);
729 this_cpu_dec(nr_unused);
730 return LRU_REMOVED;
731 }
732
733
734 if (inode->i_state & I_REFERENCED) {
735 inode->i_state &= ~I_REFERENCED;
736 spin_unlock(&inode->i_lock);
737 return LRU_ROTATE;
738 }
739
740 if (inode_has_buffers(inode) || inode->i_data.nrpages) {
741 __iget(inode);
742 spin_unlock(&inode->i_lock);
743 spin_unlock(lru_lock);
744 if (remove_inode_buffers(inode)) {
745 unsigned long reap;
746 reap = invalidate_mapping_pages(&inode->i_data, 0, -1);
747 if (current_is_kswapd())
748 __count_vm_events(KSWAPD_INODESTEAL, reap);
749 else
750 __count_vm_events(PGINODESTEAL, reap);
751 if (current->reclaim_state)
752 current->reclaim_state->reclaimed_slab += reap;
753 }
754 iput(inode);
755 spin_lock(lru_lock);
756 return LRU_RETRY;
757 }
758
759 WARN_ON(inode->i_state & I_NEW);
760 inode->i_state |= I_FREEING;
761 list_lru_isolate_move(lru, &inode->i_lru, freeable);
762 spin_unlock(&inode->i_lock);
763
764 this_cpu_dec(nr_unused);
765 return LRU_REMOVED;
766}
767
768
769
770
771
772
773
774long prune_icache_sb(struct super_block *sb, struct shrink_control *sc)
775{
776 LIST_HEAD(freeable);
777 long freed;
778
779 freed = list_lru_shrink_walk(&sb->s_inode_lru, sc,
780 inode_lru_isolate, &freeable);
781 dispose_list(&freeable);
782 return freed;
783}
784
785static void __wait_on_freeing_inode(struct inode *inode);
786
787
788
789static struct inode *find_inode(struct super_block *sb,
790 struct hlist_head *head,
791 int (*test)(struct inode *, void *),
792 void *data)
793{
794 struct inode *inode = NULL;
795
796repeat:
797 hlist_for_each_entry(inode, head, i_hash) {
798 if (inode->i_sb != sb)
799 continue;
800 if (!test(inode, data))
801 continue;
802 spin_lock(&inode->i_lock);
803 if (inode->i_state & (I_FREEING|I_WILL_FREE)) {
804 __wait_on_freeing_inode(inode);
805 goto repeat;
806 }
807 if (unlikely(inode->i_state & I_CREATING)) {
808 spin_unlock(&inode->i_lock);
809 return ERR_PTR(-ESTALE);
810 }
811 __iget(inode);
812 spin_unlock(&inode->i_lock);
813 return inode;
814 }
815 return NULL;
816}
817
818
819
820
821
822static struct inode *find_inode_fast(struct super_block *sb,
823 struct hlist_head *head, unsigned long ino)
824{
825 struct inode *inode = NULL;
826
827repeat:
828 hlist_for_each_entry(inode, head, i_hash) {
829 if (inode->i_ino != ino)
830 continue;
831 if (inode->i_sb != sb)
832 continue;
833 spin_lock(&inode->i_lock);
834 if (inode->i_state & (I_FREEING|I_WILL_FREE)) {
835 __wait_on_freeing_inode(inode);
836 goto repeat;
837 }
838 if (unlikely(inode->i_state & I_CREATING)) {
839 spin_unlock(&inode->i_lock);
840 return ERR_PTR(-ESTALE);
841 }
842 __iget(inode);
843 spin_unlock(&inode->i_lock);
844 return inode;
845 }
846 return NULL;
847}
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864#define LAST_INO_BATCH 1024
865static DEFINE_PER_CPU(unsigned int, last_ino);
866
867unsigned int get_next_ino(void)
868{
869 unsigned int *p = &get_cpu_var(last_ino);
870 unsigned int res = *p;
871
872#ifdef CONFIG_SMP
873 if (unlikely((res & (LAST_INO_BATCH-1)) == 0)) {
874 static atomic_t shared_last_ino;
875 int next = atomic_add_return(LAST_INO_BATCH, &shared_last_ino);
876
877 res = next - LAST_INO_BATCH;
878 }
879#endif
880
881 res++;
882
883 if (unlikely(!res))
884 res++;
885 *p = res;
886 put_cpu_var(last_ino);
887 return res;
888}
889EXPORT_SYMBOL(get_next_ino);
890
891
892
893
894
895
896
897
898
899
900
901struct inode *new_inode_pseudo(struct super_block *sb)
902{
903 struct inode *inode = alloc_inode(sb);
904
905 if (inode) {
906 spin_lock(&inode->i_lock);
907 inode->i_state = 0;
908 spin_unlock(&inode->i_lock);
909 INIT_LIST_HEAD(&inode->i_sb_list);
910 }
911 return inode;
912}
913
914
915
916
917
918
919
920
921
922
923
924
925
926struct inode *new_inode(struct super_block *sb)
927{
928 struct inode *inode;
929
930 spin_lock_prefetch(&sb->s_inode_list_lock);
931
932 inode = new_inode_pseudo(sb);
933 if (inode)
934 inode_sb_list_add(inode);
935 return inode;
936}
937EXPORT_SYMBOL(new_inode);
938
939#ifdef CONFIG_DEBUG_LOCK_ALLOC
940void lockdep_annotate_inode_mutex_key(struct inode *inode)
941{
942 if (S_ISDIR(inode->i_mode)) {
943 struct file_system_type *type = inode->i_sb->s_type;
944
945
946 if (lockdep_match_class(&inode->i_rwsem, &type->i_mutex_key)) {
947
948
949
950
951 init_rwsem(&inode->i_rwsem);
952 lockdep_set_class(&inode->i_rwsem,
953 &type->i_mutex_dir_key);
954 }
955 }
956}
957EXPORT_SYMBOL(lockdep_annotate_inode_mutex_key);
958#endif
959
960
961
962
963
964
965
966
967void unlock_new_inode(struct inode *inode)
968{
969 lockdep_annotate_inode_mutex_key(inode);
970 spin_lock(&inode->i_lock);
971 WARN_ON(!(inode->i_state & I_NEW));
972 inode->i_state &= ~I_NEW & ~I_CREATING;
973 smp_mb();
974 wake_up_bit(&inode->i_state, __I_NEW);
975 spin_unlock(&inode->i_lock);
976}
977EXPORT_SYMBOL(unlock_new_inode);
978
979void discard_new_inode(struct inode *inode)
980{
981 lockdep_annotate_inode_mutex_key(inode);
982 spin_lock(&inode->i_lock);
983 WARN_ON(!(inode->i_state & I_NEW));
984 inode->i_state &= ~I_NEW;
985 smp_mb();
986 wake_up_bit(&inode->i_state, __I_NEW);
987 spin_unlock(&inode->i_lock);
988 iput(inode);
989}
990EXPORT_SYMBOL(discard_new_inode);
991
992
993
994
995
996
997
998
999
1000
1001void lock_two_nondirectories(struct inode *inode1, struct inode *inode2)
1002{
1003 if (inode1 > inode2)
1004 swap(inode1, inode2);
1005
1006 if (inode1 && !S_ISDIR(inode1->i_mode))
1007 inode_lock(inode1);
1008 if (inode2 && !S_ISDIR(inode2->i_mode) && inode2 != inode1)
1009 inode_lock_nested(inode2, I_MUTEX_NONDIR2);
1010}
1011EXPORT_SYMBOL(lock_two_nondirectories);
1012
1013
1014
1015
1016
1017
1018void unlock_two_nondirectories(struct inode *inode1, struct inode *inode2)
1019{
1020 if (inode1 && !S_ISDIR(inode1->i_mode))
1021 inode_unlock(inode1);
1022 if (inode2 && !S_ISDIR(inode2->i_mode) && inode2 != inode1)
1023 inode_unlock(inode2);
1024}
1025EXPORT_SYMBOL(unlock_two_nondirectories);
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047struct inode *inode_insert5(struct inode *inode, unsigned long hashval,
1048 int (*test)(struct inode *, void *),
1049 int (*set)(struct inode *, void *), void *data)
1050{
1051 struct hlist_head *head = inode_hashtable + hash(inode->i_sb, hashval);
1052 struct inode *old;
1053 bool creating = inode->i_state & I_CREATING;
1054
1055again:
1056 spin_lock(&inode_hash_lock);
1057 old = find_inode(inode->i_sb, head, test, data);
1058 if (unlikely(old)) {
1059
1060
1061
1062
1063 spin_unlock(&inode_hash_lock);
1064 if (IS_ERR(old))
1065 return NULL;
1066 wait_on_inode(old);
1067 if (unlikely(inode_unhashed(old))) {
1068 iput(old);
1069 goto again;
1070 }
1071 return old;
1072 }
1073
1074 if (set && unlikely(set(inode, data))) {
1075 inode = NULL;
1076 goto unlock;
1077 }
1078
1079
1080
1081
1082
1083 spin_lock(&inode->i_lock);
1084 inode->i_state |= I_NEW;
1085 hlist_add_head(&inode->i_hash, head);
1086 spin_unlock(&inode->i_lock);
1087 if (!creating)
1088 inode_sb_list_add(inode);
1089unlock:
1090 spin_unlock(&inode_hash_lock);
1091
1092 return inode;
1093}
1094EXPORT_SYMBOL(inode_insert5);
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116struct inode *iget5_locked(struct super_block *sb, unsigned long hashval,
1117 int (*test)(struct inode *, void *),
1118 int (*set)(struct inode *, void *), void *data)
1119{
1120 struct inode *inode = ilookup5(sb, hashval, test, data);
1121
1122 if (!inode) {
1123 struct inode *new = alloc_inode(sb);
1124
1125 if (new) {
1126 new->i_state = 0;
1127 inode = inode_insert5(new, hashval, test, set, data);
1128 if (unlikely(inode != new))
1129 destroy_inode(new);
1130 }
1131 }
1132 return inode;
1133}
1134EXPORT_SYMBOL(iget5_locked);
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149struct inode *iget_locked(struct super_block *sb, unsigned long ino)
1150{
1151 struct hlist_head *head = inode_hashtable + hash(sb, ino);
1152 struct inode *inode;
1153again:
1154 spin_lock(&inode_hash_lock);
1155 inode = find_inode_fast(sb, head, ino);
1156 spin_unlock(&inode_hash_lock);
1157 if (inode) {
1158 if (IS_ERR(inode))
1159 return NULL;
1160 wait_on_inode(inode);
1161 if (unlikely(inode_unhashed(inode))) {
1162 iput(inode);
1163 goto again;
1164 }
1165 return inode;
1166 }
1167
1168 inode = alloc_inode(sb);
1169 if (inode) {
1170 struct inode *old;
1171
1172 spin_lock(&inode_hash_lock);
1173
1174 old = find_inode_fast(sb, head, ino);
1175 if (!old) {
1176 inode->i_ino = ino;
1177 spin_lock(&inode->i_lock);
1178 inode->i_state = I_NEW;
1179 hlist_add_head(&inode->i_hash, head);
1180 spin_unlock(&inode->i_lock);
1181 inode_sb_list_add(inode);
1182 spin_unlock(&inode_hash_lock);
1183
1184
1185
1186
1187 return inode;
1188 }
1189
1190
1191
1192
1193
1194
1195 spin_unlock(&inode_hash_lock);
1196 destroy_inode(inode);
1197 if (IS_ERR(old))
1198 return NULL;
1199 inode = old;
1200 wait_on_inode(inode);
1201 if (unlikely(inode_unhashed(inode))) {
1202 iput(inode);
1203 goto again;
1204 }
1205 }
1206 return inode;
1207}
1208EXPORT_SYMBOL(iget_locked);
1209
1210
1211
1212
1213
1214
1215
1216
1217static int test_inode_iunique(struct super_block *sb, unsigned long ino)
1218{
1219 struct hlist_head *b = inode_hashtable + hash(sb, ino);
1220 struct inode *inode;
1221
1222 spin_lock(&inode_hash_lock);
1223 hlist_for_each_entry(inode, b, i_hash) {
1224 if (inode->i_ino == ino && inode->i_sb == sb) {
1225 spin_unlock(&inode_hash_lock);
1226 return 0;
1227 }
1228 }
1229 spin_unlock(&inode_hash_lock);
1230
1231 return 1;
1232}
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248ino_t iunique(struct super_block *sb, ino_t max_reserved)
1249{
1250
1251
1252
1253
1254
1255 static DEFINE_SPINLOCK(iunique_lock);
1256 static unsigned int counter;
1257 ino_t res;
1258
1259 spin_lock(&iunique_lock);
1260 do {
1261 if (counter <= max_reserved)
1262 counter = max_reserved + 1;
1263 res = counter++;
1264 } while (!test_inode_iunique(sb, res));
1265 spin_unlock(&iunique_lock);
1266
1267 return res;
1268}
1269EXPORT_SYMBOL(iunique);
1270
1271struct inode *igrab(struct inode *inode)
1272{
1273 spin_lock(&inode->i_lock);
1274 if (!(inode->i_state & (I_FREEING|I_WILL_FREE))) {
1275 __iget(inode);
1276 spin_unlock(&inode->i_lock);
1277 } else {
1278 spin_unlock(&inode->i_lock);
1279
1280
1281
1282
1283
1284 inode = NULL;
1285 }
1286 return inode;
1287}
1288EXPORT_SYMBOL(igrab);
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306struct inode *ilookup5_nowait(struct super_block *sb, unsigned long hashval,
1307 int (*test)(struct inode *, void *), void *data)
1308{
1309 struct hlist_head *head = inode_hashtable + hash(sb, hashval);
1310 struct inode *inode;
1311
1312 spin_lock(&inode_hash_lock);
1313 inode = find_inode(sb, head, test, data);
1314 spin_unlock(&inode_hash_lock);
1315
1316 return IS_ERR(inode) ? NULL : inode;
1317}
1318EXPORT_SYMBOL(ilookup5_nowait);
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337struct inode *ilookup5(struct super_block *sb, unsigned long hashval,
1338 int (*test)(struct inode *, void *), void *data)
1339{
1340 struct inode *inode;
1341again:
1342 inode = ilookup5_nowait(sb, hashval, test, data);
1343 if (inode) {
1344 wait_on_inode(inode);
1345 if (unlikely(inode_unhashed(inode))) {
1346 iput(inode);
1347 goto again;
1348 }
1349 }
1350 return inode;
1351}
1352EXPORT_SYMBOL(ilookup5);
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362struct inode *ilookup(struct super_block *sb, unsigned long ino)
1363{
1364 struct hlist_head *head = inode_hashtable + hash(sb, ino);
1365 struct inode *inode;
1366again:
1367 spin_lock(&inode_hash_lock);
1368 inode = find_inode_fast(sb, head, ino);
1369 spin_unlock(&inode_hash_lock);
1370
1371 if (inode) {
1372 if (IS_ERR(inode))
1373 return NULL;
1374 wait_on_inode(inode);
1375 if (unlikely(inode_unhashed(inode))) {
1376 iput(inode);
1377 goto again;
1378 }
1379 }
1380 return inode;
1381}
1382EXPORT_SYMBOL(ilookup);
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407struct inode *find_inode_nowait(struct super_block *sb,
1408 unsigned long hashval,
1409 int (*match)(struct inode *, unsigned long,
1410 void *),
1411 void *data)
1412{
1413 struct hlist_head *head = inode_hashtable + hash(sb, hashval);
1414 struct inode *inode, *ret_inode = NULL;
1415 int mval;
1416
1417 spin_lock(&inode_hash_lock);
1418 hlist_for_each_entry(inode, head, i_hash) {
1419 if (inode->i_sb != sb)
1420 continue;
1421 mval = match(inode, hashval, data);
1422 if (mval == 0)
1423 continue;
1424 if (mval == 1)
1425 ret_inode = inode;
1426 goto out;
1427 }
1428out:
1429 spin_unlock(&inode_hash_lock);
1430 return ret_inode;
1431}
1432EXPORT_SYMBOL(find_inode_nowait);
1433
1434int insert_inode_locked(struct inode *inode)
1435{
1436 struct super_block *sb = inode->i_sb;
1437 ino_t ino = inode->i_ino;
1438 struct hlist_head *head = inode_hashtable + hash(sb, ino);
1439
1440 while (1) {
1441 struct inode *old = NULL;
1442 spin_lock(&inode_hash_lock);
1443 hlist_for_each_entry(old, head, i_hash) {
1444 if (old->i_ino != ino)
1445 continue;
1446 if (old->i_sb != sb)
1447 continue;
1448 spin_lock(&old->i_lock);
1449 if (old->i_state & (I_FREEING|I_WILL_FREE)) {
1450 spin_unlock(&old->i_lock);
1451 continue;
1452 }
1453 break;
1454 }
1455 if (likely(!old)) {
1456 spin_lock(&inode->i_lock);
1457 inode->i_state |= I_NEW | I_CREATING;
1458 hlist_add_head(&inode->i_hash, head);
1459 spin_unlock(&inode->i_lock);
1460 spin_unlock(&inode_hash_lock);
1461 return 0;
1462 }
1463 if (unlikely(old->i_state & I_CREATING)) {
1464 spin_unlock(&old->i_lock);
1465 spin_unlock(&inode_hash_lock);
1466 return -EBUSY;
1467 }
1468 __iget(old);
1469 spin_unlock(&old->i_lock);
1470 spin_unlock(&inode_hash_lock);
1471 wait_on_inode(old);
1472 if (unlikely(!inode_unhashed(old))) {
1473 iput(old);
1474 return -EBUSY;
1475 }
1476 iput(old);
1477 }
1478}
1479EXPORT_SYMBOL(insert_inode_locked);
1480
1481int insert_inode_locked4(struct inode *inode, unsigned long hashval,
1482 int (*test)(struct inode *, void *), void *data)
1483{
1484 struct inode *old;
1485
1486 inode->i_state |= I_CREATING;
1487 old = inode_insert5(inode, hashval, test, NULL, data);
1488
1489 if (old != inode) {
1490 iput(old);
1491 return -EBUSY;
1492 }
1493 return 0;
1494}
1495EXPORT_SYMBOL(insert_inode_locked4);
1496
1497
1498int generic_delete_inode(struct inode *inode)
1499{
1500 return 1;
1501}
1502EXPORT_SYMBOL(generic_delete_inode);
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514static void iput_final(struct inode *inode)
1515{
1516 struct super_block *sb = inode->i_sb;
1517 const struct super_operations *op = inode->i_sb->s_op;
1518 int drop;
1519
1520 WARN_ON(inode->i_state & I_NEW);
1521
1522 if (op->drop_inode)
1523 drop = op->drop_inode(inode);
1524 else
1525 drop = generic_drop_inode(inode);
1526
1527 if (!drop && (sb->s_flags & SB_ACTIVE)) {
1528 inode_add_lru(inode);
1529 spin_unlock(&inode->i_lock);
1530 return;
1531 }
1532
1533 if (!drop) {
1534 inode->i_state |= I_WILL_FREE;
1535 spin_unlock(&inode->i_lock);
1536 write_inode_now(inode, 1);
1537 spin_lock(&inode->i_lock);
1538 WARN_ON(inode->i_state & I_NEW);
1539 inode->i_state &= ~I_WILL_FREE;
1540 }
1541
1542 inode->i_state |= I_FREEING;
1543 if (!list_empty(&inode->i_lru))
1544 inode_lru_list_del(inode);
1545 spin_unlock(&inode->i_lock);
1546
1547 evict(inode);
1548}
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559void iput(struct inode *inode)
1560{
1561 if (!inode)
1562 return;
1563 BUG_ON(inode->i_state & I_CLEAR);
1564retry:
1565 if (atomic_dec_and_lock(&inode->i_count, &inode->i_lock)) {
1566 if (inode->i_nlink && (inode->i_state & I_DIRTY_TIME)) {
1567 atomic_inc(&inode->i_count);
1568 spin_unlock(&inode->i_lock);
1569 trace_writeback_lazytime_iput(inode);
1570 mark_inode_dirty_sync(inode);
1571 goto retry;
1572 }
1573 iput_final(inode);
1574 }
1575}
1576EXPORT_SYMBOL(iput);
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589sector_t bmap(struct inode *inode, sector_t block)
1590{
1591 sector_t res = 0;
1592 if (inode->i_mapping->a_ops->bmap)
1593 res = inode->i_mapping->a_ops->bmap(inode->i_mapping, block);
1594 return res;
1595}
1596EXPORT_SYMBOL(bmap);
1597
1598
1599
1600
1601
1602
1603static int relatime_need_update(struct vfsmount *mnt, struct inode *inode,
1604 struct timespec now)
1605{
1606
1607 if (!(mnt->mnt_flags & MNT_RELATIME))
1608 return 1;
1609
1610
1611
1612 if (timespec64_compare(&inode->i_mtime, &inode->i_atime) >= 0)
1613 return 1;
1614
1615
1616
1617 if (timespec64_compare(&inode->i_ctime, &inode->i_atime) >= 0)
1618 return 1;
1619
1620
1621
1622
1623
1624 if ((long)(now.tv_sec - inode->i_atime.tv_sec) >= 24*60*60)
1625 return 1;
1626
1627
1628
1629 return 0;
1630}
1631
1632int generic_update_time(struct inode *inode, struct timespec64 *time, int flags)
1633{
1634 int iflags = I_DIRTY_TIME;
1635 bool dirty = false;
1636
1637 if (flags & S_ATIME)
1638 inode->i_atime = *time;
1639 if (flags & S_VERSION)
1640 dirty = inode_maybe_inc_iversion(inode, false);
1641 if (flags & S_CTIME)
1642 inode->i_ctime = *time;
1643 if (flags & S_MTIME)
1644 inode->i_mtime = *time;
1645 if ((flags & (S_ATIME | S_CTIME | S_MTIME)) &&
1646 !(inode->i_sb->s_flags & SB_LAZYTIME))
1647 dirty = true;
1648
1649 if (dirty)
1650 iflags |= I_DIRTY_SYNC;
1651 __mark_inode_dirty(inode, iflags);
1652 return 0;
1653}
1654EXPORT_SYMBOL(generic_update_time);
1655
1656
1657
1658
1659
1660static int update_time(struct inode *inode, struct timespec64 *time, int flags)
1661{
1662 int (*update_time)(struct inode *, struct timespec64 *, int);
1663
1664 update_time = inode->i_op->update_time ? inode->i_op->update_time :
1665 generic_update_time;
1666
1667 return update_time(inode, time, flags);
1668}
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679bool atime_needs_update(const struct path *path, struct inode *inode)
1680{
1681 struct vfsmount *mnt = path->mnt;
1682 struct timespec64 now;
1683
1684 if (inode->i_flags & S_NOATIME)
1685 return false;
1686
1687
1688
1689
1690 if (HAS_UNMAPPED_ID(inode))
1691 return false;
1692
1693 if (IS_NOATIME(inode))
1694 return false;
1695 if ((inode->i_sb->s_flags & SB_NODIRATIME) && S_ISDIR(inode->i_mode))
1696 return false;
1697
1698 if (mnt->mnt_flags & MNT_NOATIME)
1699 return false;
1700 if ((mnt->mnt_flags & MNT_NODIRATIME) && S_ISDIR(inode->i_mode))
1701 return false;
1702
1703 now = current_time(inode);
1704
1705 if (!relatime_need_update(mnt, inode, timespec64_to_timespec(now)))
1706 return false;
1707
1708 if (timespec64_equal(&inode->i_atime, &now))
1709 return false;
1710
1711 return true;
1712}
1713
1714void touch_atime(const struct path *path)
1715{
1716 struct vfsmount *mnt = path->mnt;
1717 struct inode *inode = d_inode(path->dentry);
1718 struct timespec64 now;
1719
1720 if (!atime_needs_update(path, inode))
1721 return;
1722
1723 if (!sb_start_write_trylock(inode->i_sb))
1724 return;
1725
1726 if (__mnt_want_write(mnt) != 0)
1727 goto skip_update;
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737 now = current_time(inode);
1738 update_time(inode, &now, S_ATIME);
1739 __mnt_drop_write(mnt);
1740skip_update:
1741 sb_end_write(inode->i_sb);
1742}
1743EXPORT_SYMBOL(touch_atime);
1744
1745
1746
1747
1748
1749
1750
1751int should_remove_suid(struct dentry *dentry)
1752{
1753 umode_t mode = d_inode(dentry)->i_mode;
1754 int kill = 0;
1755
1756
1757 if (unlikely(mode & S_ISUID))
1758 kill = ATTR_KILL_SUID;
1759
1760
1761
1762
1763
1764 if (unlikely((mode & S_ISGID) && (mode & S_IXGRP)))
1765 kill |= ATTR_KILL_SGID;
1766
1767 if (unlikely(kill && !capable(CAP_FSETID) && S_ISREG(mode)))
1768 return kill;
1769
1770 return 0;
1771}
1772EXPORT_SYMBOL(should_remove_suid);
1773
1774
1775
1776
1777
1778
1779int dentry_needs_remove_privs(struct dentry *dentry)
1780{
1781 struct inode *inode = d_inode(dentry);
1782 int mask = 0;
1783 int ret;
1784
1785 if (IS_NOSEC(inode))
1786 return 0;
1787
1788 mask = should_remove_suid(dentry);
1789 ret = security_inode_need_killpriv(dentry);
1790 if (ret < 0)
1791 return ret;
1792 if (ret)
1793 mask |= ATTR_KILL_PRIV;
1794 return mask;
1795}
1796
1797static int __remove_privs(struct dentry *dentry, int kill)
1798{
1799 struct iattr newattrs;
1800
1801 newattrs.ia_valid = ATTR_FORCE | kill;
1802
1803
1804
1805
1806 return notify_change(dentry, &newattrs, NULL);
1807}
1808
1809
1810
1811
1812
1813int file_remove_privs(struct file *file)
1814{
1815 struct dentry *dentry = file_dentry(file);
1816 struct inode *inode = file_inode(file);
1817 int kill;
1818 int error = 0;
1819
1820
1821 if (IS_NOSEC(inode))
1822 return 0;
1823
1824 kill = dentry_needs_remove_privs(dentry);
1825 if (kill < 0)
1826 return kill;
1827 if (kill)
1828 error = __remove_privs(dentry, kill);
1829 if (!error)
1830 inode_has_no_xattr(inode);
1831
1832 return error;
1833}
1834EXPORT_SYMBOL(file_remove_privs);
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849int file_update_time(struct file *file)
1850{
1851 struct inode *inode = file_inode(file);
1852 struct timespec64 now;
1853 int sync_it = 0;
1854 int ret;
1855
1856
1857 if (IS_NOCMTIME(inode))
1858 return 0;
1859
1860 now = current_time(inode);
1861 if (!timespec64_equal(&inode->i_mtime, &now))
1862 sync_it = S_MTIME;
1863
1864 if (!timespec64_equal(&inode->i_ctime, &now))
1865 sync_it |= S_CTIME;
1866
1867 if (IS_I_VERSION(inode) && inode_iversion_need_inc(inode))
1868 sync_it |= S_VERSION;
1869
1870 if (!sync_it)
1871 return 0;
1872
1873
1874 if (__mnt_want_write_file(file))
1875 return 0;
1876
1877 ret = update_time(inode, &now, sync_it);
1878 __mnt_drop_write_file(file);
1879
1880 return ret;
1881}
1882EXPORT_SYMBOL(file_update_time);
1883
1884int inode_needs_sync(struct inode *inode)
1885{
1886 if (IS_SYNC(inode))
1887 return 1;
1888 if (S_ISDIR(inode->i_mode) && IS_DIRSYNC(inode))
1889 return 1;
1890 return 0;
1891}
1892EXPORT_SYMBOL(inode_needs_sync);
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905static void __wait_on_freeing_inode(struct inode *inode)
1906{
1907 wait_queue_head_t *wq;
1908 DEFINE_WAIT_BIT(wait, &inode->i_state, __I_NEW);
1909 wq = bit_waitqueue(&inode->i_state, __I_NEW);
1910 prepare_to_wait(wq, &wait.wq_entry, TASK_UNINTERRUPTIBLE);
1911 spin_unlock(&inode->i_lock);
1912 spin_unlock(&inode_hash_lock);
1913 schedule();
1914 finish_wait(wq, &wait.wq_entry);
1915 spin_lock(&inode_hash_lock);
1916}
1917
1918static __initdata unsigned long ihash_entries;
1919static int __init set_ihash_entries(char *str)
1920{
1921 if (!str)
1922 return 0;
1923 ihash_entries = simple_strtoul(str, &str, 0);
1924 return 1;
1925}
1926__setup("ihash_entries=", set_ihash_entries);
1927
1928
1929
1930
1931void __init inode_init_early(void)
1932{
1933
1934
1935
1936 if (hashdist)
1937 return;
1938
1939 inode_hashtable =
1940 alloc_large_system_hash("Inode-cache",
1941 sizeof(struct hlist_head),
1942 ihash_entries,
1943 14,
1944 HASH_EARLY | HASH_ZERO,
1945 &i_hash_shift,
1946 &i_hash_mask,
1947 0,
1948 0);
1949}
1950
1951void __init inode_init(void)
1952{
1953
1954 inode_cachep = kmem_cache_create("inode_cache",
1955 sizeof(struct inode),
1956 0,
1957 (SLAB_RECLAIM_ACCOUNT|SLAB_PANIC|
1958 SLAB_MEM_SPREAD|SLAB_ACCOUNT),
1959 init_once);
1960
1961
1962 if (!hashdist)
1963 return;
1964
1965 inode_hashtable =
1966 alloc_large_system_hash("Inode-cache",
1967 sizeof(struct hlist_head),
1968 ihash_entries,
1969 14,
1970 HASH_ZERO,
1971 &i_hash_shift,
1972 &i_hash_mask,
1973 0,
1974 0);
1975}
1976
1977void init_special_inode(struct inode *inode, umode_t mode, dev_t rdev)
1978{
1979 inode->i_mode = mode;
1980 if (S_ISCHR(mode)) {
1981 inode->i_fop = &def_chr_fops;
1982 inode->i_rdev = rdev;
1983 } else if (S_ISBLK(mode)) {
1984 inode->i_fop = &def_blk_fops;
1985 inode->i_rdev = rdev;
1986 } else if (S_ISFIFO(mode))
1987 inode->i_fop = &pipefifo_fops;
1988 else if (S_ISSOCK(mode))
1989 ;
1990 else
1991 printk(KERN_DEBUG "init_special_inode: bogus i_mode (%o) for"
1992 " inode %s:%lu\n", mode, inode->i_sb->s_id,
1993 inode->i_ino);
1994}
1995EXPORT_SYMBOL(init_special_inode);
1996
1997
1998
1999
2000
2001
2002
2003void inode_init_owner(struct inode *inode, const struct inode *dir,
2004 umode_t mode)
2005{
2006 inode->i_uid = current_fsuid();
2007 if (dir && dir->i_mode & S_ISGID) {
2008 inode->i_gid = dir->i_gid;
2009
2010
2011 if (S_ISDIR(mode))
2012 mode |= S_ISGID;
2013 else if ((mode & (S_ISGID | S_IXGRP)) == (S_ISGID | S_IXGRP) &&
2014 !in_group_p(inode->i_gid) &&
2015 !capable_wrt_inode_uidgid(dir, CAP_FSETID))
2016 mode &= ~S_ISGID;
2017 } else
2018 inode->i_gid = current_fsgid();
2019 inode->i_mode = mode;
2020}
2021EXPORT_SYMBOL(inode_init_owner);
2022
2023
2024
2025
2026
2027
2028
2029
2030bool inode_owner_or_capable(const struct inode *inode)
2031{
2032 struct user_namespace *ns;
2033
2034 if (uid_eq(current_fsuid(), inode->i_uid))
2035 return true;
2036
2037 ns = current_user_ns();
2038 if (kuid_has_mapping(ns, inode->i_uid) && ns_capable(ns, CAP_FOWNER))
2039 return true;
2040 return false;
2041}
2042EXPORT_SYMBOL(inode_owner_or_capable);
2043
2044
2045
2046
2047static void __inode_dio_wait(struct inode *inode)
2048{
2049 wait_queue_head_t *wq = bit_waitqueue(&inode->i_state, __I_DIO_WAKEUP);
2050 DEFINE_WAIT_BIT(q, &inode->i_state, __I_DIO_WAKEUP);
2051
2052 do {
2053 prepare_to_wait(wq, &q.wq_entry, TASK_UNINTERRUPTIBLE);
2054 if (atomic_read(&inode->i_dio_count))
2055 schedule();
2056 } while (atomic_read(&inode->i_dio_count));
2057 finish_wait(wq, &q.wq_entry);
2058}
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070void inode_dio_wait(struct inode *inode)
2071{
2072 if (atomic_read(&inode->i_dio_count))
2073 __inode_dio_wait(inode);
2074}
2075EXPORT_SYMBOL(inode_dio_wait);
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093void inode_set_flags(struct inode *inode, unsigned int flags,
2094 unsigned int mask)
2095{
2096 unsigned int old_flags, new_flags;
2097
2098 WARN_ON_ONCE(flags & ~mask);
2099 do {
2100 old_flags = READ_ONCE(inode->i_flags);
2101 new_flags = (old_flags & ~mask) | flags;
2102 } while (unlikely(cmpxchg(&inode->i_flags, old_flags,
2103 new_flags) != old_flags));
2104}
2105EXPORT_SYMBOL(inode_set_flags);
2106
2107void inode_nohighmem(struct inode *inode)
2108{
2109 mapping_set_gfp_mask(inode->i_mapping, GFP_USER);
2110}
2111EXPORT_SYMBOL(inode_nohighmem);
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121struct timespec64 timespec64_trunc(struct timespec64 t, unsigned gran)
2122{
2123
2124 if (gran == 1) {
2125
2126 } else if (gran == NSEC_PER_SEC) {
2127 t.tv_nsec = 0;
2128 } else if (gran > 1 && gran < NSEC_PER_SEC) {
2129 t.tv_nsec -= t.tv_nsec % gran;
2130 } else {
2131 WARN(1, "illegal file time granularity: %u", gran);
2132 }
2133 return t;
2134}
2135EXPORT_SYMBOL(timespec64_trunc);
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147struct timespec64 current_time(struct inode *inode)
2148{
2149 struct timespec64 now = current_kernel_time64();
2150
2151 if (unlikely(!inode->i_sb)) {
2152 WARN(1, "current_time() called with uninitialized super_block in the inode");
2153 return now;
2154 }
2155
2156 return timespec64_trunc(now, inode->i_sb->s_time_gran);
2157}
2158EXPORT_SYMBOL(current_time);
2159