1
2
3
4
5#include <linux/export.h>
6#include <linux/fs.h>
7#include <linux/mm.h>
8#include <linux/backing-dev.h>
9#include <linux/hash.h>
10#include <linux/swap.h>
11#include <linux/security.h>
12#include <linux/cdev.h>
13#include <linux/bootmem.h>
14#include <linux/fsnotify.h>
15#include <linux/mount.h>
16#include <linux/posix_acl.h>
17#include <linux/prefetch.h>
18#include <linux/buffer_head.h>
19#include <linux/ratelimit.h>
20#include <linux/list_lru.h>
21#include <linux/iversion.h>
22#include <trace/events/writeback.h>
23#include "internal.h"
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56static unsigned int i_hash_mask __read_mostly;
57static unsigned int i_hash_shift __read_mostly;
58static struct hlist_head *inode_hashtable __read_mostly;
59static __cacheline_aligned_in_smp DEFINE_SPINLOCK(inode_hash_lock);
60
61
62
63
64
65const struct address_space_operations empty_aops = {
66};
67EXPORT_SYMBOL(empty_aops);
68
69
70
71
72struct inodes_stat_t inodes_stat;
73
74static DEFINE_PER_CPU(unsigned long, nr_inodes);
75static DEFINE_PER_CPU(unsigned long, nr_unused);
76
77static struct kmem_cache *inode_cachep __read_mostly;
78
79static long get_nr_inodes(void)
80{
81 int i;
82 long sum = 0;
83 for_each_possible_cpu(i)
84 sum += per_cpu(nr_inodes, i);
85 return sum < 0 ? 0 : sum;
86}
87
88static inline long get_nr_inodes_unused(void)
89{
90 int i;
91 long sum = 0;
92 for_each_possible_cpu(i)
93 sum += per_cpu(nr_unused, i);
94 return sum < 0 ? 0 : sum;
95}
96
97long get_nr_dirty_inodes(void)
98{
99
100 long nr_dirty = get_nr_inodes() - get_nr_inodes_unused();
101 return nr_dirty > 0 ? nr_dirty : 0;
102}
103
104
105
106
107#ifdef CONFIG_SYSCTL
108int proc_nr_inodes(struct ctl_table *table, int write,
109 void __user *buffer, size_t *lenp, loff_t *ppos)
110{
111 inodes_stat.nr_inodes = get_nr_inodes();
112 inodes_stat.nr_unused = get_nr_inodes_unused();
113 return proc_doulongvec_minmax(table, write, buffer, lenp, ppos);
114}
115#endif
116
117static int no_open(struct inode *inode, struct file *file)
118{
119 return -ENXIO;
120}
121
122
123
124
125
126
127
128
129
130int inode_init_always(struct super_block *sb, struct inode *inode)
131{
132 static const struct inode_operations empty_iops;
133 static const struct file_operations no_open_fops = {.open = no_open};
134 struct address_space *const mapping = &inode->i_data;
135
136 inode->i_sb = sb;
137 inode->i_blkbits = sb->s_blocksize_bits;
138 inode->i_flags = 0;
139 atomic_set(&inode->i_count, 1);
140 inode->i_op = &empty_iops;
141 inode->i_fop = &no_open_fops;
142 inode->__i_nlink = 1;
143 inode->i_opflags = 0;
144 if (sb->s_xattr)
145 inode->i_opflags |= IOP_XATTR;
146 i_uid_write(inode, 0);
147 i_gid_write(inode, 0);
148 atomic_set(&inode->i_writecount, 0);
149 inode->i_size = 0;
150 inode->i_write_hint = WRITE_LIFE_NOT_SET;
151 inode->i_blocks = 0;
152 inode->i_bytes = 0;
153 inode->i_generation = 0;
154 inode->i_pipe = NULL;
155 inode->i_bdev = NULL;
156 inode->i_cdev = NULL;
157 inode->i_link = NULL;
158 inode->i_dir_seq = 0;
159 inode->i_rdev = 0;
160 inode->dirtied_when = 0;
161
162#ifdef CONFIG_CGROUP_WRITEBACK
163 inode->i_wb_frn_winner = 0;
164 inode->i_wb_frn_avg_time = 0;
165 inode->i_wb_frn_history = 0;
166#endif
167
168 if (security_inode_alloc(inode))
169 goto out;
170 spin_lock_init(&inode->i_lock);
171 lockdep_set_class(&inode->i_lock, &sb->s_type->i_lock_key);
172
173 init_rwsem(&inode->i_rwsem);
174 lockdep_set_class(&inode->i_rwsem, &sb->s_type->i_mutex_key);
175
176 atomic_set(&inode->i_dio_count, 0);
177
178 mapping->a_ops = &empty_aops;
179 mapping->host = inode;
180 mapping->flags = 0;
181 mapping->wb_err = 0;
182 atomic_set(&mapping->i_mmap_writable, 0);
183 mapping_set_gfp_mask(mapping, GFP_HIGHUSER_MOVABLE);
184 mapping->private_data = NULL;
185 mapping->writeback_index = 0;
186 inode->i_private = NULL;
187 inode->i_mapping = mapping;
188 INIT_HLIST_HEAD(&inode->i_dentry);
189#ifdef CONFIG_FS_POSIX_ACL
190 inode->i_acl = inode->i_default_acl = ACL_NOT_CACHED;
191#endif
192
193#ifdef CONFIG_FSNOTIFY
194 inode->i_fsnotify_mask = 0;
195#endif
196 inode->i_flctx = NULL;
197 this_cpu_inc(nr_inodes);
198
199 return 0;
200out:
201 return -ENOMEM;
202}
203EXPORT_SYMBOL(inode_init_always);
204
205static struct inode *alloc_inode(struct super_block *sb)
206{
207 struct inode *inode;
208
209 if (sb->s_op->alloc_inode)
210 inode = sb->s_op->alloc_inode(sb);
211 else
212 inode = kmem_cache_alloc(inode_cachep, GFP_KERNEL);
213
214 if (!inode)
215 return NULL;
216
217 if (unlikely(inode_init_always(sb, inode))) {
218 if (inode->i_sb->s_op->destroy_inode)
219 inode->i_sb->s_op->destroy_inode(inode);
220 else
221 kmem_cache_free(inode_cachep, inode);
222 return NULL;
223 }
224
225 return inode;
226}
227
228void free_inode_nonrcu(struct inode *inode)
229{
230 kmem_cache_free(inode_cachep, inode);
231}
232EXPORT_SYMBOL(free_inode_nonrcu);
233
234void __destroy_inode(struct inode *inode)
235{
236 BUG_ON(inode_has_buffers(inode));
237 inode_detach_wb(inode);
238 security_inode_free(inode);
239 fsnotify_inode_delete(inode);
240 locks_free_lock_context(inode);
241 if (!inode->i_nlink) {
242 WARN_ON(atomic_long_read(&inode->i_sb->s_remove_count) == 0);
243 atomic_long_dec(&inode->i_sb->s_remove_count);
244 }
245
246#ifdef CONFIG_FS_POSIX_ACL
247 if (inode->i_acl && !is_uncached_acl(inode->i_acl))
248 posix_acl_release(inode->i_acl);
249 if (inode->i_default_acl && !is_uncached_acl(inode->i_default_acl))
250 posix_acl_release(inode->i_default_acl);
251#endif
252 this_cpu_dec(nr_inodes);
253}
254EXPORT_SYMBOL(__destroy_inode);
255
256static void i_callback(struct rcu_head *head)
257{
258 struct inode *inode = container_of(head, struct inode, i_rcu);
259 kmem_cache_free(inode_cachep, inode);
260}
261
262static void destroy_inode(struct inode *inode)
263{
264 BUG_ON(!list_empty(&inode->i_lru));
265 __destroy_inode(inode);
266 if (inode->i_sb->s_op->destroy_inode)
267 inode->i_sb->s_op->destroy_inode(inode);
268 else
269 call_rcu(&inode->i_rcu, i_callback);
270}
271
272
273
274
275
276
277
278
279
280
281
282
283void drop_nlink(struct inode *inode)
284{
285 WARN_ON(inode->i_nlink == 0);
286 inode->__i_nlink--;
287 if (!inode->i_nlink)
288 atomic_long_inc(&inode->i_sb->s_remove_count);
289}
290EXPORT_SYMBOL(drop_nlink);
291
292
293
294
295
296
297
298
299
300void clear_nlink(struct inode *inode)
301{
302 if (inode->i_nlink) {
303 inode->__i_nlink = 0;
304 atomic_long_inc(&inode->i_sb->s_remove_count);
305 }
306}
307EXPORT_SYMBOL(clear_nlink);
308
309
310
311
312
313
314
315
316
317void set_nlink(struct inode *inode, unsigned int nlink)
318{
319 if (!nlink) {
320 clear_nlink(inode);
321 } else {
322
323 if (inode->i_nlink == 0)
324 atomic_long_dec(&inode->i_sb->s_remove_count);
325
326 inode->__i_nlink = nlink;
327 }
328}
329EXPORT_SYMBOL(set_nlink);
330
331
332
333
334
335
336
337
338
339void inc_nlink(struct inode *inode)
340{
341 if (unlikely(inode->i_nlink == 0)) {
342 WARN_ON(!(inode->i_state & I_LINKABLE));
343 atomic_long_dec(&inode->i_sb->s_remove_count);
344 }
345
346 inode->__i_nlink++;
347}
348EXPORT_SYMBOL(inc_nlink);
349
350static void __address_space_init_once(struct address_space *mapping)
351{
352 INIT_RADIX_TREE(&mapping->i_pages, GFP_ATOMIC | __GFP_ACCOUNT);
353 init_rwsem(&mapping->i_mmap_rwsem);
354 INIT_LIST_HEAD(&mapping->private_list);
355 spin_lock_init(&mapping->private_lock);
356 mapping->i_mmap = RB_ROOT_CACHED;
357}
358
359void address_space_init_once(struct address_space *mapping)
360{
361 memset(mapping, 0, sizeof(*mapping));
362 __address_space_init_once(mapping);
363}
364EXPORT_SYMBOL(address_space_init_once);
365
366
367
368
369
370
371void inode_init_once(struct inode *inode)
372{
373 memset(inode, 0, sizeof(*inode));
374 INIT_HLIST_NODE(&inode->i_hash);
375 INIT_LIST_HEAD(&inode->i_devices);
376 INIT_LIST_HEAD(&inode->i_io_list);
377 INIT_LIST_HEAD(&inode->i_wb_list);
378 INIT_LIST_HEAD(&inode->i_lru);
379 __address_space_init_once(&inode->i_data);
380 i_size_ordered_init(inode);
381}
382EXPORT_SYMBOL(inode_init_once);
383
384static void init_once(void *foo)
385{
386 struct inode *inode = (struct inode *) foo;
387
388 inode_init_once(inode);
389}
390
391
392
393
394void __iget(struct inode *inode)
395{
396 atomic_inc(&inode->i_count);
397}
398
399
400
401
402void ihold(struct inode *inode)
403{
404 WARN_ON(atomic_inc_return(&inode->i_count) < 2);
405}
406EXPORT_SYMBOL(ihold);
407
408static void inode_lru_list_add(struct inode *inode)
409{
410 if (list_lru_add(&inode->i_sb->s_inode_lru, &inode->i_lru))
411 this_cpu_inc(nr_unused);
412 else
413 inode->i_state |= I_REFERENCED;
414}
415
416
417
418
419
420
421void inode_add_lru(struct inode *inode)
422{
423 if (!(inode->i_state & (I_DIRTY_ALL | I_SYNC |
424 I_FREEING | I_WILL_FREE)) &&
425 !atomic_read(&inode->i_count) && inode->i_sb->s_flags & SB_ACTIVE)
426 inode_lru_list_add(inode);
427}
428
429
430static void inode_lru_list_del(struct inode *inode)
431{
432
433 if (list_lru_del(&inode->i_sb->s_inode_lru, &inode->i_lru))
434 this_cpu_dec(nr_unused);
435}
436
437
438
439
440
441void inode_sb_list_add(struct inode *inode)
442{
443 spin_lock(&inode->i_sb->s_inode_list_lock);
444 list_add(&inode->i_sb_list, &inode->i_sb->s_inodes);
445 spin_unlock(&inode->i_sb->s_inode_list_lock);
446}
447EXPORT_SYMBOL_GPL(inode_sb_list_add);
448
449static inline void inode_sb_list_del(struct inode *inode)
450{
451 if (!list_empty(&inode->i_sb_list)) {
452 spin_lock(&inode->i_sb->s_inode_list_lock);
453 list_del_init(&inode->i_sb_list);
454 spin_unlock(&inode->i_sb->s_inode_list_lock);
455 }
456}
457
458static unsigned long hash(struct super_block *sb, unsigned long hashval)
459{
460 unsigned long tmp;
461
462 tmp = (hashval * (unsigned long)sb) ^ (GOLDEN_RATIO_PRIME + hashval) /
463 L1_CACHE_BYTES;
464 tmp = tmp ^ ((tmp ^ GOLDEN_RATIO_PRIME) >> i_hash_shift);
465 return tmp & i_hash_mask;
466}
467
468
469
470
471
472
473
474
475
476void __insert_inode_hash(struct inode *inode, unsigned long hashval)
477{
478 struct hlist_head *b = inode_hashtable + hash(inode->i_sb, hashval);
479
480 spin_lock(&inode_hash_lock);
481 spin_lock(&inode->i_lock);
482 hlist_add_head(&inode->i_hash, b);
483 spin_unlock(&inode->i_lock);
484 spin_unlock(&inode_hash_lock);
485}
486EXPORT_SYMBOL(__insert_inode_hash);
487
488
489
490
491
492
493
494void __remove_inode_hash(struct inode *inode)
495{
496 spin_lock(&inode_hash_lock);
497 spin_lock(&inode->i_lock);
498 hlist_del_init(&inode->i_hash);
499 spin_unlock(&inode->i_lock);
500 spin_unlock(&inode_hash_lock);
501}
502EXPORT_SYMBOL(__remove_inode_hash);
503
504void clear_inode(struct inode *inode)
505{
506
507
508
509
510
511 xa_lock_irq(&inode->i_data.i_pages);
512 BUG_ON(inode->i_data.nrpages);
513 BUG_ON(inode->i_data.nrexceptional);
514 xa_unlock_irq(&inode->i_data.i_pages);
515 BUG_ON(!list_empty(&inode->i_data.private_list));
516 BUG_ON(!(inode->i_state & I_FREEING));
517 BUG_ON(inode->i_state & I_CLEAR);
518 BUG_ON(!list_empty(&inode->i_wb_list));
519
520 inode->i_state = I_FREEING | I_CLEAR;
521}
522EXPORT_SYMBOL(clear_inode);
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537static void evict(struct inode *inode)
538{
539 const struct super_operations *op = inode->i_sb->s_op;
540
541 BUG_ON(!(inode->i_state & I_FREEING));
542 BUG_ON(!list_empty(&inode->i_lru));
543
544 if (!list_empty(&inode->i_io_list))
545 inode_io_list_del(inode);
546
547 inode_sb_list_del(inode);
548
549
550
551
552
553
554
555 inode_wait_for_writeback(inode);
556
557 if (op->evict_inode) {
558 op->evict_inode(inode);
559 } else {
560 truncate_inode_pages_final(&inode->i_data);
561 clear_inode(inode);
562 }
563 if (S_ISBLK(inode->i_mode) && inode->i_bdev)
564 bd_forget(inode);
565 if (S_ISCHR(inode->i_mode) && inode->i_cdev)
566 cd_forget(inode);
567
568 remove_inode_hash(inode);
569
570 spin_lock(&inode->i_lock);
571 wake_up_bit(&inode->i_state, __I_NEW);
572 BUG_ON(inode->i_state != (I_FREEING | I_CLEAR));
573 spin_unlock(&inode->i_lock);
574
575 destroy_inode(inode);
576}
577
578
579
580
581
582
583
584
585static void dispose_list(struct list_head *head)
586{
587 while (!list_empty(head)) {
588 struct inode *inode;
589
590 inode = list_first_entry(head, struct inode, i_lru);
591 list_del_init(&inode->i_lru);
592
593 evict(inode);
594 cond_resched();
595 }
596}
597
598
599
600
601
602
603
604
605
606
607void evict_inodes(struct super_block *sb)
608{
609 struct inode *inode, *next;
610 LIST_HEAD(dispose);
611
612again:
613 spin_lock(&sb->s_inode_list_lock);
614 list_for_each_entry_safe(inode, next, &sb->s_inodes, i_sb_list) {
615 if (atomic_read(&inode->i_count))
616 continue;
617
618 spin_lock(&inode->i_lock);
619 if (inode->i_state & (I_NEW | I_FREEING | I_WILL_FREE)) {
620 spin_unlock(&inode->i_lock);
621 continue;
622 }
623
624 inode->i_state |= I_FREEING;
625 inode_lru_list_del(inode);
626 spin_unlock(&inode->i_lock);
627 list_add(&inode->i_lru, &dispose);
628
629
630
631
632
633
634 if (need_resched()) {
635 spin_unlock(&sb->s_inode_list_lock);
636 cond_resched();
637 dispose_list(&dispose);
638 goto again;
639 }
640 }
641 spin_unlock(&sb->s_inode_list_lock);
642
643 dispose_list(&dispose);
644}
645EXPORT_SYMBOL_GPL(evict_inodes);
646
647
648
649
650
651
652
653
654
655
656
657int invalidate_inodes(struct super_block *sb, bool kill_dirty)
658{
659 int busy = 0;
660 struct inode *inode, *next;
661 LIST_HEAD(dispose);
662
663 spin_lock(&sb->s_inode_list_lock);
664 list_for_each_entry_safe(inode, next, &sb->s_inodes, i_sb_list) {
665 spin_lock(&inode->i_lock);
666 if (inode->i_state & (I_NEW | I_FREEING | I_WILL_FREE)) {
667 spin_unlock(&inode->i_lock);
668 continue;
669 }
670 if (inode->i_state & I_DIRTY_ALL && !kill_dirty) {
671 spin_unlock(&inode->i_lock);
672 busy = 1;
673 continue;
674 }
675 if (atomic_read(&inode->i_count)) {
676 spin_unlock(&inode->i_lock);
677 busy = 1;
678 continue;
679 }
680
681 inode->i_state |= I_FREEING;
682 inode_lru_list_del(inode);
683 spin_unlock(&inode->i_lock);
684 list_add(&inode->i_lru, &dispose);
685 }
686 spin_unlock(&sb->s_inode_list_lock);
687
688 dispose_list(&dispose);
689
690 return busy;
691}
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708static enum lru_status inode_lru_isolate(struct list_head *item,
709 struct list_lru_one *lru, spinlock_t *lru_lock, void *arg)
710{
711 struct list_head *freeable = arg;
712 struct inode *inode = container_of(item, struct inode, i_lru);
713
714
715
716
717
718 if (!spin_trylock(&inode->i_lock))
719 return LRU_SKIP;
720
721
722
723
724
725 if (atomic_read(&inode->i_count) ||
726 (inode->i_state & ~I_REFERENCED)) {
727 list_lru_isolate(lru, &inode->i_lru);
728 spin_unlock(&inode->i_lock);
729 this_cpu_dec(nr_unused);
730 return LRU_REMOVED;
731 }
732
733
734 if (inode->i_state & I_REFERENCED) {
735 inode->i_state &= ~I_REFERENCED;
736 spin_unlock(&inode->i_lock);
737 return LRU_ROTATE;
738 }
739
740 if (inode_has_buffers(inode) || inode->i_data.nrpages) {
741 __iget(inode);
742 spin_unlock(&inode->i_lock);
743 spin_unlock(lru_lock);
744 if (remove_inode_buffers(inode)) {
745 unsigned long reap;
746 reap = invalidate_mapping_pages(&inode->i_data, 0, -1);
747 if (current_is_kswapd())
748 __count_vm_events(KSWAPD_INODESTEAL, reap);
749 else
750 __count_vm_events(PGINODESTEAL, reap);
751 if (current->reclaim_state)
752 current->reclaim_state->reclaimed_slab += reap;
753 }
754 iput(inode);
755 spin_lock(lru_lock);
756 return LRU_RETRY;
757 }
758
759 WARN_ON(inode->i_state & I_NEW);
760 inode->i_state |= I_FREEING;
761 list_lru_isolate_move(lru, &inode->i_lru, freeable);
762 spin_unlock(&inode->i_lock);
763
764 this_cpu_dec(nr_unused);
765 return LRU_REMOVED;
766}
767
768
769
770
771
772
773
774long prune_icache_sb(struct super_block *sb, struct shrink_control *sc)
775{
776 LIST_HEAD(freeable);
777 long freed;
778
779 freed = list_lru_shrink_walk(&sb->s_inode_lru, sc,
780 inode_lru_isolate, &freeable);
781 dispose_list(&freeable);
782 return freed;
783}
784
785static void __wait_on_freeing_inode(struct inode *inode);
786
787
788
789static struct inode *find_inode(struct super_block *sb,
790 struct hlist_head *head,
791 int (*test)(struct inode *, void *),
792 void *data)
793{
794 struct inode *inode = NULL;
795
796repeat:
797 hlist_for_each_entry(inode, head, i_hash) {
798 if (inode->i_sb != sb)
799 continue;
800 if (!test(inode, data))
801 continue;
802 spin_lock(&inode->i_lock);
803 if (inode->i_state & (I_FREEING|I_WILL_FREE)) {
804 __wait_on_freeing_inode(inode);
805 goto repeat;
806 }
807 __iget(inode);
808 spin_unlock(&inode->i_lock);
809 return inode;
810 }
811 return NULL;
812}
813
814
815
816
817
818static struct inode *find_inode_fast(struct super_block *sb,
819 struct hlist_head *head, unsigned long ino)
820{
821 struct inode *inode = NULL;
822
823repeat:
824 hlist_for_each_entry(inode, head, i_hash) {
825 if (inode->i_ino != ino)
826 continue;
827 if (inode->i_sb != sb)
828 continue;
829 spin_lock(&inode->i_lock);
830 if (inode->i_state & (I_FREEING|I_WILL_FREE)) {
831 __wait_on_freeing_inode(inode);
832 goto repeat;
833 }
834 __iget(inode);
835 spin_unlock(&inode->i_lock);
836 return inode;
837 }
838 return NULL;
839}
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856#define LAST_INO_BATCH 1024
857static DEFINE_PER_CPU(unsigned int, last_ino);
858
859unsigned int get_next_ino(void)
860{
861 unsigned int *p = &get_cpu_var(last_ino);
862 unsigned int res = *p;
863
864#ifdef CONFIG_SMP
865 if (unlikely((res & (LAST_INO_BATCH-1)) == 0)) {
866 static atomic_t shared_last_ino;
867 int next = atomic_add_return(LAST_INO_BATCH, &shared_last_ino);
868
869 res = next - LAST_INO_BATCH;
870 }
871#endif
872
873 res++;
874
875 if (unlikely(!res))
876 res++;
877 *p = res;
878 put_cpu_var(last_ino);
879 return res;
880}
881EXPORT_SYMBOL(get_next_ino);
882
883
884
885
886
887
888
889
890
891
892
893struct inode *new_inode_pseudo(struct super_block *sb)
894{
895 struct inode *inode = alloc_inode(sb);
896
897 if (inode) {
898 spin_lock(&inode->i_lock);
899 inode->i_state = 0;
900 spin_unlock(&inode->i_lock);
901 INIT_LIST_HEAD(&inode->i_sb_list);
902 }
903 return inode;
904}
905
906
907
908
909
910
911
912
913
914
915
916
917
918struct inode *new_inode(struct super_block *sb)
919{
920 struct inode *inode;
921
922 spin_lock_prefetch(&sb->s_inode_list_lock);
923
924 inode = new_inode_pseudo(sb);
925 if (inode)
926 inode_sb_list_add(inode);
927 return inode;
928}
929EXPORT_SYMBOL(new_inode);
930
931#ifdef CONFIG_DEBUG_LOCK_ALLOC
932void lockdep_annotate_inode_mutex_key(struct inode *inode)
933{
934 if (S_ISDIR(inode->i_mode)) {
935 struct file_system_type *type = inode->i_sb->s_type;
936
937
938 if (lockdep_match_class(&inode->i_rwsem, &type->i_mutex_key)) {
939
940
941
942
943 init_rwsem(&inode->i_rwsem);
944 lockdep_set_class(&inode->i_rwsem,
945 &type->i_mutex_dir_key);
946 }
947 }
948}
949EXPORT_SYMBOL(lockdep_annotate_inode_mutex_key);
950#endif
951
952
953
954
955
956
957
958
959void unlock_new_inode(struct inode *inode)
960{
961 lockdep_annotate_inode_mutex_key(inode);
962 spin_lock(&inode->i_lock);
963 WARN_ON(!(inode->i_state & I_NEW));
964 inode->i_state &= ~I_NEW;
965 smp_mb();
966 wake_up_bit(&inode->i_state, __I_NEW);
967 spin_unlock(&inode->i_lock);
968}
969EXPORT_SYMBOL(unlock_new_inode);
970
971
972
973
974
975
976
977
978
979
980void lock_two_nondirectories(struct inode *inode1, struct inode *inode2)
981{
982 if (inode1 > inode2)
983 swap(inode1, inode2);
984
985 if (inode1 && !S_ISDIR(inode1->i_mode))
986 inode_lock(inode1);
987 if (inode2 && !S_ISDIR(inode2->i_mode) && inode2 != inode1)
988 inode_lock_nested(inode2, I_MUTEX_NONDIR2);
989}
990EXPORT_SYMBOL(lock_two_nondirectories);
991
992
993
994
995
996
997void unlock_two_nondirectories(struct inode *inode1, struct inode *inode2)
998{
999 if (inode1 && !S_ISDIR(inode1->i_mode))
1000 inode_unlock(inode1);
1001 if (inode2 && !S_ISDIR(inode2->i_mode) && inode2 != inode1)
1002 inode_unlock(inode2);
1003}
1004EXPORT_SYMBOL(unlock_two_nondirectories);
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026struct inode *iget5_locked(struct super_block *sb, unsigned long hashval,
1027 int (*test)(struct inode *, void *),
1028 int (*set)(struct inode *, void *), void *data)
1029{
1030 struct hlist_head *head = inode_hashtable + hash(sb, hashval);
1031 struct inode *inode;
1032again:
1033 spin_lock(&inode_hash_lock);
1034 inode = find_inode(sb, head, test, data);
1035 spin_unlock(&inode_hash_lock);
1036
1037 if (inode) {
1038 wait_on_inode(inode);
1039 if (unlikely(inode_unhashed(inode))) {
1040 iput(inode);
1041 goto again;
1042 }
1043 return inode;
1044 }
1045
1046 inode = alloc_inode(sb);
1047 if (inode) {
1048 struct inode *old;
1049
1050 spin_lock(&inode_hash_lock);
1051
1052 old = find_inode(sb, head, test, data);
1053 if (!old) {
1054 if (set(inode, data))
1055 goto set_failed;
1056
1057 spin_lock(&inode->i_lock);
1058 inode->i_state = I_NEW;
1059 hlist_add_head(&inode->i_hash, head);
1060 spin_unlock(&inode->i_lock);
1061 inode_sb_list_add(inode);
1062 spin_unlock(&inode_hash_lock);
1063
1064
1065
1066
1067 return inode;
1068 }
1069
1070
1071
1072
1073
1074
1075 spin_unlock(&inode_hash_lock);
1076 destroy_inode(inode);
1077 inode = old;
1078 wait_on_inode(inode);
1079 if (unlikely(inode_unhashed(inode))) {
1080 iput(inode);
1081 goto again;
1082 }
1083 }
1084 return inode;
1085
1086set_failed:
1087 spin_unlock(&inode_hash_lock);
1088 destroy_inode(inode);
1089 return NULL;
1090}
1091EXPORT_SYMBOL(iget5_locked);
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106struct inode *iget_locked(struct super_block *sb, unsigned long ino)
1107{
1108 struct hlist_head *head = inode_hashtable + hash(sb, ino);
1109 struct inode *inode;
1110again:
1111 spin_lock(&inode_hash_lock);
1112 inode = find_inode_fast(sb, head, ino);
1113 spin_unlock(&inode_hash_lock);
1114 if (inode) {
1115 wait_on_inode(inode);
1116 if (unlikely(inode_unhashed(inode))) {
1117 iput(inode);
1118 goto again;
1119 }
1120 return inode;
1121 }
1122
1123 inode = alloc_inode(sb);
1124 if (inode) {
1125 struct inode *old;
1126
1127 spin_lock(&inode_hash_lock);
1128
1129 old = find_inode_fast(sb, head, ino);
1130 if (!old) {
1131 inode->i_ino = ino;
1132 spin_lock(&inode->i_lock);
1133 inode->i_state = I_NEW;
1134 hlist_add_head(&inode->i_hash, head);
1135 spin_unlock(&inode->i_lock);
1136 inode_sb_list_add(inode);
1137 spin_unlock(&inode_hash_lock);
1138
1139
1140
1141
1142 return inode;
1143 }
1144
1145
1146
1147
1148
1149
1150 spin_unlock(&inode_hash_lock);
1151 destroy_inode(inode);
1152 inode = old;
1153 wait_on_inode(inode);
1154 if (unlikely(inode_unhashed(inode))) {
1155 iput(inode);
1156 goto again;
1157 }
1158 }
1159 return inode;
1160}
1161EXPORT_SYMBOL(iget_locked);
1162
1163
1164
1165
1166
1167
1168
1169
1170static int test_inode_iunique(struct super_block *sb, unsigned long ino)
1171{
1172 struct hlist_head *b = inode_hashtable + hash(sb, ino);
1173 struct inode *inode;
1174
1175 spin_lock(&inode_hash_lock);
1176 hlist_for_each_entry(inode, b, i_hash) {
1177 if (inode->i_ino == ino && inode->i_sb == sb) {
1178 spin_unlock(&inode_hash_lock);
1179 return 0;
1180 }
1181 }
1182 spin_unlock(&inode_hash_lock);
1183
1184 return 1;
1185}
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201ino_t iunique(struct super_block *sb, ino_t max_reserved)
1202{
1203
1204
1205
1206
1207
1208 static DEFINE_SPINLOCK(iunique_lock);
1209 static unsigned int counter;
1210 ino_t res;
1211
1212 spin_lock(&iunique_lock);
1213 do {
1214 if (counter <= max_reserved)
1215 counter = max_reserved + 1;
1216 res = counter++;
1217 } while (!test_inode_iunique(sb, res));
1218 spin_unlock(&iunique_lock);
1219
1220 return res;
1221}
1222EXPORT_SYMBOL(iunique);
1223
1224struct inode *igrab(struct inode *inode)
1225{
1226 spin_lock(&inode->i_lock);
1227 if (!(inode->i_state & (I_FREEING|I_WILL_FREE))) {
1228 __iget(inode);
1229 spin_unlock(&inode->i_lock);
1230 } else {
1231 spin_unlock(&inode->i_lock);
1232
1233
1234
1235
1236
1237 inode = NULL;
1238 }
1239 return inode;
1240}
1241EXPORT_SYMBOL(igrab);
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259struct inode *ilookup5_nowait(struct super_block *sb, unsigned long hashval,
1260 int (*test)(struct inode *, void *), void *data)
1261{
1262 struct hlist_head *head = inode_hashtable + hash(sb, hashval);
1263 struct inode *inode;
1264
1265 spin_lock(&inode_hash_lock);
1266 inode = find_inode(sb, head, test, data);
1267 spin_unlock(&inode_hash_lock);
1268
1269 return inode;
1270}
1271EXPORT_SYMBOL(ilookup5_nowait);
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290struct inode *ilookup5(struct super_block *sb, unsigned long hashval,
1291 int (*test)(struct inode *, void *), void *data)
1292{
1293 struct inode *inode;
1294again:
1295 inode = ilookup5_nowait(sb, hashval, test, data);
1296 if (inode) {
1297 wait_on_inode(inode);
1298 if (unlikely(inode_unhashed(inode))) {
1299 iput(inode);
1300 goto again;
1301 }
1302 }
1303 return inode;
1304}
1305EXPORT_SYMBOL(ilookup5);
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315struct inode *ilookup(struct super_block *sb, unsigned long ino)
1316{
1317 struct hlist_head *head = inode_hashtable + hash(sb, ino);
1318 struct inode *inode;
1319again:
1320 spin_lock(&inode_hash_lock);
1321 inode = find_inode_fast(sb, head, ino);
1322 spin_unlock(&inode_hash_lock);
1323
1324 if (inode) {
1325 wait_on_inode(inode);
1326 if (unlikely(inode_unhashed(inode))) {
1327 iput(inode);
1328 goto again;
1329 }
1330 }
1331 return inode;
1332}
1333EXPORT_SYMBOL(ilookup);
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358struct inode *find_inode_nowait(struct super_block *sb,
1359 unsigned long hashval,
1360 int (*match)(struct inode *, unsigned long,
1361 void *),
1362 void *data)
1363{
1364 struct hlist_head *head = inode_hashtable + hash(sb, hashval);
1365 struct inode *inode, *ret_inode = NULL;
1366 int mval;
1367
1368 spin_lock(&inode_hash_lock);
1369 hlist_for_each_entry(inode, head, i_hash) {
1370 if (inode->i_sb != sb)
1371 continue;
1372 mval = match(inode, hashval, data);
1373 if (mval == 0)
1374 continue;
1375 if (mval == 1)
1376 ret_inode = inode;
1377 goto out;
1378 }
1379out:
1380 spin_unlock(&inode_hash_lock);
1381 return ret_inode;
1382}
1383EXPORT_SYMBOL(find_inode_nowait);
1384
1385int insert_inode_locked(struct inode *inode)
1386{
1387 struct super_block *sb = inode->i_sb;
1388 ino_t ino = inode->i_ino;
1389 struct hlist_head *head = inode_hashtable + hash(sb, ino);
1390
1391 while (1) {
1392 struct inode *old = NULL;
1393 spin_lock(&inode_hash_lock);
1394 hlist_for_each_entry(old, head, i_hash) {
1395 if (old->i_ino != ino)
1396 continue;
1397 if (old->i_sb != sb)
1398 continue;
1399 spin_lock(&old->i_lock);
1400 if (old->i_state & (I_FREEING|I_WILL_FREE)) {
1401 spin_unlock(&old->i_lock);
1402 continue;
1403 }
1404 break;
1405 }
1406 if (likely(!old)) {
1407 spin_lock(&inode->i_lock);
1408 inode->i_state |= I_NEW;
1409 hlist_add_head(&inode->i_hash, head);
1410 spin_unlock(&inode->i_lock);
1411 spin_unlock(&inode_hash_lock);
1412 return 0;
1413 }
1414 __iget(old);
1415 spin_unlock(&old->i_lock);
1416 spin_unlock(&inode_hash_lock);
1417 wait_on_inode(old);
1418 if (unlikely(!inode_unhashed(old))) {
1419 iput(old);
1420 return -EBUSY;
1421 }
1422 iput(old);
1423 }
1424}
1425EXPORT_SYMBOL(insert_inode_locked);
1426
1427int insert_inode_locked4(struct inode *inode, unsigned long hashval,
1428 int (*test)(struct inode *, void *), void *data)
1429{
1430 struct super_block *sb = inode->i_sb;
1431 struct hlist_head *head = inode_hashtable + hash(sb, hashval);
1432
1433 while (1) {
1434 struct inode *old = NULL;
1435
1436 spin_lock(&inode_hash_lock);
1437 hlist_for_each_entry(old, head, i_hash) {
1438 if (old->i_sb != sb)
1439 continue;
1440 if (!test(old, data))
1441 continue;
1442 spin_lock(&old->i_lock);
1443 if (old->i_state & (I_FREEING|I_WILL_FREE)) {
1444 spin_unlock(&old->i_lock);
1445 continue;
1446 }
1447 break;
1448 }
1449 if (likely(!old)) {
1450 spin_lock(&inode->i_lock);
1451 inode->i_state |= I_NEW;
1452 hlist_add_head(&inode->i_hash, head);
1453 spin_unlock(&inode->i_lock);
1454 spin_unlock(&inode_hash_lock);
1455 return 0;
1456 }
1457 __iget(old);
1458 spin_unlock(&old->i_lock);
1459 spin_unlock(&inode_hash_lock);
1460 wait_on_inode(old);
1461 if (unlikely(!inode_unhashed(old))) {
1462 iput(old);
1463 return -EBUSY;
1464 }
1465 iput(old);
1466 }
1467}
1468EXPORT_SYMBOL(insert_inode_locked4);
1469
1470
1471int generic_delete_inode(struct inode *inode)
1472{
1473 return 1;
1474}
1475EXPORT_SYMBOL(generic_delete_inode);
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487static void iput_final(struct inode *inode)
1488{
1489 struct super_block *sb = inode->i_sb;
1490 const struct super_operations *op = inode->i_sb->s_op;
1491 int drop;
1492
1493 WARN_ON(inode->i_state & I_NEW);
1494
1495 if (op->drop_inode)
1496 drop = op->drop_inode(inode);
1497 else
1498 drop = generic_drop_inode(inode);
1499
1500 if (!drop && (sb->s_flags & SB_ACTIVE)) {
1501 inode_add_lru(inode);
1502 spin_unlock(&inode->i_lock);
1503 return;
1504 }
1505
1506 if (!drop) {
1507 inode->i_state |= I_WILL_FREE;
1508 spin_unlock(&inode->i_lock);
1509 write_inode_now(inode, 1);
1510 spin_lock(&inode->i_lock);
1511 WARN_ON(inode->i_state & I_NEW);
1512 inode->i_state &= ~I_WILL_FREE;
1513 }
1514
1515 inode->i_state |= I_FREEING;
1516 if (!list_empty(&inode->i_lru))
1517 inode_lru_list_del(inode);
1518 spin_unlock(&inode->i_lock);
1519
1520 evict(inode);
1521}
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532void iput(struct inode *inode)
1533{
1534 if (!inode)
1535 return;
1536 BUG_ON(inode->i_state & I_CLEAR);
1537retry:
1538 if (atomic_dec_and_lock(&inode->i_count, &inode->i_lock)) {
1539 if (inode->i_nlink && (inode->i_state & I_DIRTY_TIME)) {
1540 atomic_inc(&inode->i_count);
1541 spin_unlock(&inode->i_lock);
1542 trace_writeback_lazytime_iput(inode);
1543 mark_inode_dirty_sync(inode);
1544 goto retry;
1545 }
1546 iput_final(inode);
1547 }
1548}
1549EXPORT_SYMBOL(iput);
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562sector_t bmap(struct inode *inode, sector_t block)
1563{
1564 sector_t res = 0;
1565 if (inode->i_mapping->a_ops->bmap)
1566 res = inode->i_mapping->a_ops->bmap(inode->i_mapping, block);
1567 return res;
1568}
1569EXPORT_SYMBOL(bmap);
1570
1571
1572
1573
1574static void update_ovl_inode_times(struct dentry *dentry, struct inode *inode,
1575 bool rcu)
1576{
1577 struct dentry *upperdentry;
1578
1579
1580
1581
1582 if (rcu || likely(!(dentry->d_flags & DCACHE_OP_REAL)))
1583 return;
1584
1585 upperdentry = d_real(dentry, NULL, 0, D_REAL_UPPER);
1586
1587
1588
1589
1590
1591 if (upperdentry) {
1592 struct inode *realinode = d_inode(upperdentry);
1593
1594 if ((!timespec_equal(&inode->i_mtime, &realinode->i_mtime) ||
1595 !timespec_equal(&inode->i_ctime, &realinode->i_ctime))) {
1596 inode->i_mtime = realinode->i_mtime;
1597 inode->i_ctime = realinode->i_ctime;
1598 }
1599 }
1600}
1601
1602
1603
1604
1605
1606
1607static int relatime_need_update(const struct path *path, struct inode *inode,
1608 struct timespec now, bool rcu)
1609{
1610
1611 if (!(path->mnt->mnt_flags & MNT_RELATIME))
1612 return 1;
1613
1614 update_ovl_inode_times(path->dentry, inode, rcu);
1615
1616
1617
1618 if (timespec_compare(&inode->i_mtime, &inode->i_atime) >= 0)
1619 return 1;
1620
1621
1622
1623 if (timespec_compare(&inode->i_ctime, &inode->i_atime) >= 0)
1624 return 1;
1625
1626
1627
1628
1629
1630 if ((long)(now.tv_sec - inode->i_atime.tv_sec) >= 24*60*60)
1631 return 1;
1632
1633
1634
1635 return 0;
1636}
1637
1638int generic_update_time(struct inode *inode, struct timespec *time, int flags)
1639{
1640 int iflags = I_DIRTY_TIME;
1641 bool dirty = false;
1642
1643 if (flags & S_ATIME)
1644 inode->i_atime = *time;
1645 if (flags & S_VERSION)
1646 dirty = inode_maybe_inc_iversion(inode, false);
1647 if (flags & S_CTIME)
1648 inode->i_ctime = *time;
1649 if (flags & S_MTIME)
1650 inode->i_mtime = *time;
1651 if ((flags & (S_ATIME | S_CTIME | S_MTIME)) &&
1652 !(inode->i_sb->s_flags & SB_LAZYTIME))
1653 dirty = true;
1654
1655 if (dirty)
1656 iflags |= I_DIRTY_SYNC;
1657 __mark_inode_dirty(inode, iflags);
1658 return 0;
1659}
1660EXPORT_SYMBOL(generic_update_time);
1661
1662
1663
1664
1665
1666static int update_time(struct inode *inode, struct timespec *time, int flags)
1667{
1668 int (*update_time)(struct inode *, struct timespec *, int);
1669
1670 update_time = inode->i_op->update_time ? inode->i_op->update_time :
1671 generic_update_time;
1672
1673 return update_time(inode, time, flags);
1674}
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685bool __atime_needs_update(const struct path *path, struct inode *inode,
1686 bool rcu)
1687{
1688 struct vfsmount *mnt = path->mnt;
1689 struct timespec now;
1690
1691 if (inode->i_flags & S_NOATIME)
1692 return false;
1693
1694
1695
1696
1697 if (HAS_UNMAPPED_ID(inode))
1698 return false;
1699
1700 if (IS_NOATIME(inode))
1701 return false;
1702 if ((inode->i_sb->s_flags & SB_NODIRATIME) && S_ISDIR(inode->i_mode))
1703 return false;
1704
1705 if (mnt->mnt_flags & MNT_NOATIME)
1706 return false;
1707 if ((mnt->mnt_flags & MNT_NODIRATIME) && S_ISDIR(inode->i_mode))
1708 return false;
1709
1710 now = current_time(inode);
1711
1712 if (!relatime_need_update(path, inode, now, rcu))
1713 return false;
1714
1715 if (timespec_equal(&inode->i_atime, &now))
1716 return false;
1717
1718 return true;
1719}
1720
1721void touch_atime(const struct path *path)
1722{
1723 struct vfsmount *mnt = path->mnt;
1724 struct inode *inode = d_inode(path->dentry);
1725 struct timespec now;
1726
1727 if (!__atime_needs_update(path, inode, false))
1728 return;
1729
1730 if (!sb_start_write_trylock(inode->i_sb))
1731 return;
1732
1733 if (__mnt_want_write(mnt) != 0)
1734 goto skip_update;
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744 now = current_time(inode);
1745 update_time(inode, &now, S_ATIME);
1746 __mnt_drop_write(mnt);
1747skip_update:
1748 sb_end_write(inode->i_sb);
1749}
1750EXPORT_SYMBOL(touch_atime);
1751
1752
1753
1754
1755
1756
1757
1758int should_remove_suid(struct dentry *dentry)
1759{
1760 umode_t mode = d_inode(dentry)->i_mode;
1761 int kill = 0;
1762
1763
1764 if (unlikely(mode & S_ISUID))
1765 kill = ATTR_KILL_SUID;
1766
1767
1768
1769
1770
1771 if (unlikely((mode & S_ISGID) && (mode & S_IXGRP)))
1772 kill |= ATTR_KILL_SGID;
1773
1774 if (unlikely(kill && !capable(CAP_FSETID) && S_ISREG(mode)))
1775 return kill;
1776
1777 return 0;
1778}
1779EXPORT_SYMBOL(should_remove_suid);
1780
1781
1782
1783
1784
1785
1786int dentry_needs_remove_privs(struct dentry *dentry)
1787{
1788 struct inode *inode = d_inode(dentry);
1789 int mask = 0;
1790 int ret;
1791
1792 if (IS_NOSEC(inode))
1793 return 0;
1794
1795 mask = should_remove_suid(dentry);
1796 ret = security_inode_need_killpriv(dentry);
1797 if (ret < 0)
1798 return ret;
1799 if (ret)
1800 mask |= ATTR_KILL_PRIV;
1801 return mask;
1802}
1803
1804static int __remove_privs(struct dentry *dentry, int kill)
1805{
1806 struct iattr newattrs;
1807
1808 newattrs.ia_valid = ATTR_FORCE | kill;
1809
1810
1811
1812
1813 return notify_change(dentry, &newattrs, NULL);
1814}
1815
1816
1817
1818
1819
1820int file_remove_privs(struct file *file)
1821{
1822 struct dentry *dentry = file_dentry(file);
1823 struct inode *inode = file_inode(file);
1824 int kill;
1825 int error = 0;
1826
1827
1828 if (IS_NOSEC(inode))
1829 return 0;
1830
1831 kill = dentry_needs_remove_privs(dentry);
1832 if (kill < 0)
1833 return kill;
1834 if (kill)
1835 error = __remove_privs(dentry, kill);
1836 if (!error)
1837 inode_has_no_xattr(inode);
1838
1839 return error;
1840}
1841EXPORT_SYMBOL(file_remove_privs);
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856int file_update_time(struct file *file)
1857{
1858 struct inode *inode = file_inode(file);
1859 struct timespec now;
1860 int sync_it = 0;
1861 int ret;
1862
1863
1864 if (IS_NOCMTIME(inode))
1865 return 0;
1866
1867 now = current_time(inode);
1868 if (!timespec_equal(&inode->i_mtime, &now))
1869 sync_it = S_MTIME;
1870
1871 if (!timespec_equal(&inode->i_ctime, &now))
1872 sync_it |= S_CTIME;
1873
1874 if (IS_I_VERSION(inode) && inode_iversion_need_inc(inode))
1875 sync_it |= S_VERSION;
1876
1877 if (!sync_it)
1878 return 0;
1879
1880
1881 if (__mnt_want_write_file(file))
1882 return 0;
1883
1884 ret = update_time(inode, &now, sync_it);
1885 __mnt_drop_write_file(file);
1886
1887 return ret;
1888}
1889EXPORT_SYMBOL(file_update_time);
1890
1891int inode_needs_sync(struct inode *inode)
1892{
1893 if (IS_SYNC(inode))
1894 return 1;
1895 if (S_ISDIR(inode->i_mode) && IS_DIRSYNC(inode))
1896 return 1;
1897 return 0;
1898}
1899EXPORT_SYMBOL(inode_needs_sync);
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912static void __wait_on_freeing_inode(struct inode *inode)
1913{
1914 wait_queue_head_t *wq;
1915 DEFINE_WAIT_BIT(wait, &inode->i_state, __I_NEW);
1916 wq = bit_waitqueue(&inode->i_state, __I_NEW);
1917 prepare_to_wait(wq, &wait.wq_entry, TASK_UNINTERRUPTIBLE);
1918 spin_unlock(&inode->i_lock);
1919 spin_unlock(&inode_hash_lock);
1920 schedule();
1921 finish_wait(wq, &wait.wq_entry);
1922 spin_lock(&inode_hash_lock);
1923}
1924
1925static __initdata unsigned long ihash_entries;
1926static int __init set_ihash_entries(char *str)
1927{
1928 if (!str)
1929 return 0;
1930 ihash_entries = simple_strtoul(str, &str, 0);
1931 return 1;
1932}
1933__setup("ihash_entries=", set_ihash_entries);
1934
1935
1936
1937
1938void __init inode_init_early(void)
1939{
1940
1941
1942
1943 if (hashdist)
1944 return;
1945
1946 inode_hashtable =
1947 alloc_large_system_hash("Inode-cache",
1948 sizeof(struct hlist_head),
1949 ihash_entries,
1950 14,
1951 HASH_EARLY | HASH_ZERO,
1952 &i_hash_shift,
1953 &i_hash_mask,
1954 0,
1955 0);
1956}
1957
1958void __init inode_init(void)
1959{
1960
1961 inode_cachep = kmem_cache_create("inode_cache",
1962 sizeof(struct inode),
1963 0,
1964 (SLAB_RECLAIM_ACCOUNT|SLAB_PANIC|
1965 SLAB_MEM_SPREAD|SLAB_ACCOUNT),
1966 init_once);
1967
1968
1969 if (!hashdist)
1970 return;
1971
1972 inode_hashtable =
1973 alloc_large_system_hash("Inode-cache",
1974 sizeof(struct hlist_head),
1975 ihash_entries,
1976 14,
1977 HASH_ZERO,
1978 &i_hash_shift,
1979 &i_hash_mask,
1980 0,
1981 0);
1982}
1983
1984void init_special_inode(struct inode *inode, umode_t mode, dev_t rdev)
1985{
1986 inode->i_mode = mode;
1987 if (S_ISCHR(mode)) {
1988 inode->i_fop = &def_chr_fops;
1989 inode->i_rdev = rdev;
1990 } else if (S_ISBLK(mode)) {
1991 inode->i_fop = &def_blk_fops;
1992 inode->i_rdev = rdev;
1993 } else if (S_ISFIFO(mode))
1994 inode->i_fop = &pipefifo_fops;
1995 else if (S_ISSOCK(mode))
1996 ;
1997 else
1998 printk(KERN_DEBUG "init_special_inode: bogus i_mode (%o) for"
1999 " inode %s:%lu\n", mode, inode->i_sb->s_id,
2000 inode->i_ino);
2001}
2002EXPORT_SYMBOL(init_special_inode);
2003
2004
2005
2006
2007
2008
2009
2010void inode_init_owner(struct inode *inode, const struct inode *dir,
2011 umode_t mode)
2012{
2013 inode->i_uid = current_fsuid();
2014 if (dir && dir->i_mode & S_ISGID) {
2015 inode->i_gid = dir->i_gid;
2016 if (S_ISDIR(mode))
2017 mode |= S_ISGID;
2018 } else
2019 inode->i_gid = current_fsgid();
2020 inode->i_mode = mode;
2021}
2022EXPORT_SYMBOL(inode_init_owner);
2023
2024
2025
2026
2027
2028
2029
2030
2031bool inode_owner_or_capable(const struct inode *inode)
2032{
2033 struct user_namespace *ns;
2034
2035 if (uid_eq(current_fsuid(), inode->i_uid))
2036 return true;
2037
2038 ns = current_user_ns();
2039 if (kuid_has_mapping(ns, inode->i_uid) && ns_capable(ns, CAP_FOWNER))
2040 return true;
2041 return false;
2042}
2043EXPORT_SYMBOL(inode_owner_or_capable);
2044
2045
2046
2047
2048static void __inode_dio_wait(struct inode *inode)
2049{
2050 wait_queue_head_t *wq = bit_waitqueue(&inode->i_state, __I_DIO_WAKEUP);
2051 DEFINE_WAIT_BIT(q, &inode->i_state, __I_DIO_WAKEUP);
2052
2053 do {
2054 prepare_to_wait(wq, &q.wq_entry, TASK_UNINTERRUPTIBLE);
2055 if (atomic_read(&inode->i_dio_count))
2056 schedule();
2057 } while (atomic_read(&inode->i_dio_count));
2058 finish_wait(wq, &q.wq_entry);
2059}
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071void inode_dio_wait(struct inode *inode)
2072{
2073 if (atomic_read(&inode->i_dio_count))
2074 __inode_dio_wait(inode);
2075}
2076EXPORT_SYMBOL(inode_dio_wait);
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094void inode_set_flags(struct inode *inode, unsigned int flags,
2095 unsigned int mask)
2096{
2097 unsigned int old_flags, new_flags;
2098
2099 WARN_ON_ONCE(flags & ~mask);
2100 do {
2101 old_flags = READ_ONCE(inode->i_flags);
2102 new_flags = (old_flags & ~mask) | flags;
2103 } while (unlikely(cmpxchg(&inode->i_flags, old_flags,
2104 new_flags) != old_flags));
2105}
2106EXPORT_SYMBOL(inode_set_flags);
2107
2108void inode_nohighmem(struct inode *inode)
2109{
2110 mapping_set_gfp_mask(inode->i_mapping, GFP_USER);
2111}
2112EXPORT_SYMBOL(inode_nohighmem);
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124struct timespec current_time(struct inode *inode)
2125{
2126 struct timespec now = current_kernel_time();
2127
2128 if (unlikely(!inode->i_sb)) {
2129 WARN(1, "current_time() called with uninitialized super_block in the inode");
2130 return now;
2131 }
2132
2133 return timespec_trunc(now, inode->i_sb->s_time_gran);
2134}
2135EXPORT_SYMBOL(current_time);
2136