1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18#include <linux/ratelimit.h>
19#include <linux/string.h>
20#include <linux/mm.h>
21#include <linux/fs.h>
22#include <linux/fscrypt.h>
23#include <linux/fsnotify.h>
24#include <linux/slab.h>
25#include <linux/init.h>
26#include <linux/hash.h>
27#include <linux/cache.h>
28#include <linux/export.h>
29#include <linux/security.h>
30#include <linux/seqlock.h>
31#include <linux/memblock.h>
32#include <linux/bit_spinlock.h>
33#include <linux/rculist_bl.h>
34#include <linux/list_lru.h>
35#include "internal.h"
36#include "mount.h"
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74int sysctl_vfs_cache_pressure __read_mostly = 100;
75EXPORT_SYMBOL_GPL(sysctl_vfs_cache_pressure);
76
77__cacheline_aligned_in_smp DEFINE_SEQLOCK(rename_lock);
78
79EXPORT_SYMBOL(rename_lock);
80
81static struct kmem_cache *dentry_cache __read_mostly;
82
83const struct qstr empty_name = QSTR_INIT("", 0);
84EXPORT_SYMBOL(empty_name);
85const struct qstr slash_name = QSTR_INIT("/", 1);
86EXPORT_SYMBOL(slash_name);
87
88
89
90
91
92
93
94
95
96
97static unsigned int d_hash_shift __read_mostly;
98
99static struct hlist_bl_head *dentry_hashtable __read_mostly;
100
101static inline struct hlist_bl_head *d_hash(unsigned int hash)
102{
103 return dentry_hashtable + (hash >> d_hash_shift);
104}
105
106#define IN_LOOKUP_SHIFT 10
107static struct hlist_bl_head in_lookup_hashtable[1 << IN_LOOKUP_SHIFT];
108
109static inline struct hlist_bl_head *in_lookup_hash(const struct dentry *parent,
110 unsigned int hash)
111{
112 hash += (unsigned long) parent / L1_CACHE_BYTES;
113 return in_lookup_hashtable + hash_32(hash, IN_LOOKUP_SHIFT);
114}
115
116
117
118struct dentry_stat_t dentry_stat = {
119 .age_limit = 45,
120};
121
122static DEFINE_PER_CPU(long, nr_dentry);
123static DEFINE_PER_CPU(long, nr_dentry_unused);
124static DEFINE_PER_CPU(long, nr_dentry_negative);
125
126#if defined(CONFIG_SYSCTL) && defined(CONFIG_PROC_FS)
127
128
129
130
131
132
133
134
135
136
137
138
139
140static long get_nr_dentry(void)
141{
142 int i;
143 long sum = 0;
144 for_each_possible_cpu(i)
145 sum += per_cpu(nr_dentry, i);
146 return sum < 0 ? 0 : sum;
147}
148
149static long get_nr_dentry_unused(void)
150{
151 int i;
152 long sum = 0;
153 for_each_possible_cpu(i)
154 sum += per_cpu(nr_dentry_unused, i);
155 return sum < 0 ? 0 : sum;
156}
157
158static long get_nr_dentry_negative(void)
159{
160 int i;
161 long sum = 0;
162
163 for_each_possible_cpu(i)
164 sum += per_cpu(nr_dentry_negative, i);
165 return sum < 0 ? 0 : sum;
166}
167
168int proc_nr_dentry(struct ctl_table *table, int write, void __user *buffer,
169 size_t *lenp, loff_t *ppos)
170{
171 dentry_stat.nr_dentry = get_nr_dentry();
172 dentry_stat.nr_unused = get_nr_dentry_unused();
173 dentry_stat.nr_negative = get_nr_dentry_negative();
174 return proc_doulongvec_minmax(table, write, buffer, lenp, ppos);
175}
176#endif
177
178
179
180
181
182#ifdef CONFIG_DCACHE_WORD_ACCESS
183
184#include <asm/word-at-a-time.h>
185
186
187
188
189
190
191
192
193
194static inline int dentry_string_cmp(const unsigned char *cs, const unsigned char *ct, unsigned tcount)
195{
196 unsigned long a,b,mask;
197
198 for (;;) {
199 a = read_word_at_a_time(cs);
200 b = load_unaligned_zeropad(ct);
201 if (tcount < sizeof(unsigned long))
202 break;
203 if (unlikely(a != b))
204 return 1;
205 cs += sizeof(unsigned long);
206 ct += sizeof(unsigned long);
207 tcount -= sizeof(unsigned long);
208 if (!tcount)
209 return 0;
210 }
211 mask = bytemask_from_count(tcount);
212 return unlikely(!!((a ^ b) & mask));
213}
214
215#else
216
217static inline int dentry_string_cmp(const unsigned char *cs, const unsigned char *ct, unsigned tcount)
218{
219 do {
220 if (*cs != *ct)
221 return 1;
222 cs++;
223 ct++;
224 tcount--;
225 } while (tcount);
226 return 0;
227}
228
229#endif
230
231static inline int dentry_cmp(const struct dentry *dentry, const unsigned char *ct, unsigned tcount)
232{
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249 const unsigned char *cs = READ_ONCE(dentry->d_name.name);
250
251 return dentry_string_cmp(cs, ct, tcount);
252}
253
254struct external_name {
255 union {
256 atomic_t count;
257 struct rcu_head head;
258 } u;
259 unsigned char name[];
260};
261
262static inline struct external_name *external_name(struct dentry *dentry)
263{
264 return container_of(dentry->d_name.name, struct external_name, name[0]);
265}
266
267static void __d_free(struct rcu_head *head)
268{
269 struct dentry *dentry = container_of(head, struct dentry, d_u.d_rcu);
270
271 kmem_cache_free(dentry_cache, dentry);
272}
273
274static void __d_free_external(struct rcu_head *head)
275{
276 struct dentry *dentry = container_of(head, struct dentry, d_u.d_rcu);
277 kfree(external_name(dentry));
278 kmem_cache_free(dentry_cache, dentry);
279}
280
281static inline int dname_external(const struct dentry *dentry)
282{
283 return dentry->d_name.name != dentry->d_iname;
284}
285
286void take_dentry_name_snapshot(struct name_snapshot *name, struct dentry *dentry)
287{
288 spin_lock(&dentry->d_lock);
289 name->name = dentry->d_name;
290 if (unlikely(dname_external(dentry))) {
291 atomic_inc(&external_name(dentry)->u.count);
292 } else {
293 memcpy(name->inline_name, dentry->d_iname,
294 dentry->d_name.len + 1);
295 name->name.name = name->inline_name;
296 }
297 spin_unlock(&dentry->d_lock);
298}
299EXPORT_SYMBOL(take_dentry_name_snapshot);
300
301void release_dentry_name_snapshot(struct name_snapshot *name)
302{
303 if (unlikely(name->name.name != name->inline_name)) {
304 struct external_name *p;
305 p = container_of(name->name.name, struct external_name, name[0]);
306 if (unlikely(atomic_dec_and_test(&p->u.count)))
307 kfree_rcu(p, u.head);
308 }
309}
310EXPORT_SYMBOL(release_dentry_name_snapshot);
311
312static inline void __d_set_inode_and_type(struct dentry *dentry,
313 struct inode *inode,
314 unsigned type_flags)
315{
316 unsigned flags;
317
318 dentry->d_inode = inode;
319 flags = READ_ONCE(dentry->d_flags);
320 flags &= ~(DCACHE_ENTRY_TYPE | DCACHE_FALLTHRU);
321 flags |= type_flags;
322 WRITE_ONCE(dentry->d_flags, flags);
323}
324
325static inline void __d_clear_type_and_inode(struct dentry *dentry)
326{
327 unsigned flags = READ_ONCE(dentry->d_flags);
328
329 flags &= ~(DCACHE_ENTRY_TYPE | DCACHE_FALLTHRU);
330 WRITE_ONCE(dentry->d_flags, flags);
331 dentry->d_inode = NULL;
332 if (dentry->d_flags & DCACHE_LRU_LIST)
333 this_cpu_inc(nr_dentry_negative);
334}
335
336static void dentry_free(struct dentry *dentry)
337{
338 WARN_ON(!hlist_unhashed(&dentry->d_u.d_alias));
339 if (unlikely(dname_external(dentry))) {
340 struct external_name *p = external_name(dentry);
341 if (likely(atomic_dec_and_test(&p->u.count))) {
342 call_rcu(&dentry->d_u.d_rcu, __d_free_external);
343 return;
344 }
345 }
346
347 if (dentry->d_flags & DCACHE_NORCU)
348 __d_free(&dentry->d_u.d_rcu);
349 else
350 call_rcu(&dentry->d_u.d_rcu, __d_free);
351}
352
353
354
355
356
357static void dentry_unlink_inode(struct dentry * dentry)
358 __releases(dentry->d_lock)
359 __releases(dentry->d_inode->i_lock)
360{
361 struct inode *inode = dentry->d_inode;
362
363 raw_write_seqcount_begin(&dentry->d_seq);
364 __d_clear_type_and_inode(dentry);
365 hlist_del_init(&dentry->d_u.d_alias);
366 raw_write_seqcount_end(&dentry->d_seq);
367 spin_unlock(&dentry->d_lock);
368 spin_unlock(&inode->i_lock);
369 if (!inode->i_nlink)
370 fsnotify_inoderemove(inode);
371 if (dentry->d_op && dentry->d_op->d_iput)
372 dentry->d_op->d_iput(dentry, inode);
373 else
374 iput(inode);
375}
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396#define D_FLAG_VERIFY(dentry,x) WARN_ON_ONCE(((dentry)->d_flags & (DCACHE_LRU_LIST | DCACHE_SHRINK_LIST)) != (x))
397static void d_lru_add(struct dentry *dentry)
398{
399 D_FLAG_VERIFY(dentry, 0);
400 dentry->d_flags |= DCACHE_LRU_LIST;
401 this_cpu_inc(nr_dentry_unused);
402 if (d_is_negative(dentry))
403 this_cpu_inc(nr_dentry_negative);
404 WARN_ON_ONCE(!list_lru_add(&dentry->d_sb->s_dentry_lru, &dentry->d_lru));
405}
406
407static void d_lru_del(struct dentry *dentry)
408{
409 D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST);
410 dentry->d_flags &= ~DCACHE_LRU_LIST;
411 this_cpu_dec(nr_dentry_unused);
412 if (d_is_negative(dentry))
413 this_cpu_dec(nr_dentry_negative);
414 WARN_ON_ONCE(!list_lru_del(&dentry->d_sb->s_dentry_lru, &dentry->d_lru));
415}
416
417static void d_shrink_del(struct dentry *dentry)
418{
419 D_FLAG_VERIFY(dentry, DCACHE_SHRINK_LIST | DCACHE_LRU_LIST);
420 list_del_init(&dentry->d_lru);
421 dentry->d_flags &= ~(DCACHE_SHRINK_LIST | DCACHE_LRU_LIST);
422 this_cpu_dec(nr_dentry_unused);
423}
424
425static void d_shrink_add(struct dentry *dentry, struct list_head *list)
426{
427 D_FLAG_VERIFY(dentry, 0);
428 list_add(&dentry->d_lru, list);
429 dentry->d_flags |= DCACHE_SHRINK_LIST | DCACHE_LRU_LIST;
430 this_cpu_inc(nr_dentry_unused);
431}
432
433
434
435
436
437
438
439static void d_lru_isolate(struct list_lru_one *lru, struct dentry *dentry)
440{
441 D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST);
442 dentry->d_flags &= ~DCACHE_LRU_LIST;
443 this_cpu_dec(nr_dentry_unused);
444 if (d_is_negative(dentry))
445 this_cpu_dec(nr_dentry_negative);
446 list_lru_isolate(lru, &dentry->d_lru);
447}
448
449static void d_lru_shrink_move(struct list_lru_one *lru, struct dentry *dentry,
450 struct list_head *list)
451{
452 D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST);
453 dentry->d_flags |= DCACHE_SHRINK_LIST;
454 if (d_is_negative(dentry))
455 this_cpu_dec(nr_dentry_negative);
456 list_lru_isolate_move(lru, &dentry->d_lru, list);
457}
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476static void ___d_drop(struct dentry *dentry)
477{
478 struct hlist_bl_head *b;
479
480
481
482
483
484 if (unlikely(IS_ROOT(dentry)))
485 b = &dentry->d_sb->s_roots;
486 else
487 b = d_hash(dentry->d_name.hash);
488
489 hlist_bl_lock(b);
490 __hlist_bl_del(&dentry->d_hash);
491 hlist_bl_unlock(b);
492}
493
494void __d_drop(struct dentry *dentry)
495{
496 if (!d_unhashed(dentry)) {
497 ___d_drop(dentry);
498 dentry->d_hash.pprev = NULL;
499 write_seqcount_invalidate(&dentry->d_seq);
500 }
501}
502EXPORT_SYMBOL(__d_drop);
503
504void d_drop(struct dentry *dentry)
505{
506 spin_lock(&dentry->d_lock);
507 __d_drop(dentry);
508 spin_unlock(&dentry->d_lock);
509}
510EXPORT_SYMBOL(d_drop);
511
512static inline void dentry_unlist(struct dentry *dentry, struct dentry *parent)
513{
514 struct dentry *next;
515
516
517
518
519 dentry->d_flags |= DCACHE_DENTRY_KILLED;
520 if (unlikely(list_empty(&dentry->d_child)))
521 return;
522 __list_del_entry(&dentry->d_child);
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542 while (dentry->d_child.next != &parent->d_subdirs) {
543 next = list_entry(dentry->d_child.next, struct dentry, d_child);
544 if (likely(!(next->d_flags & DCACHE_DENTRY_CURSOR)))
545 break;
546 dentry->d_child.next = next->d_child.next;
547 }
548}
549
550static void __dentry_kill(struct dentry *dentry)
551{
552 struct dentry *parent = NULL;
553 bool can_free = true;
554 if (!IS_ROOT(dentry))
555 parent = dentry->d_parent;
556
557
558
559
560 lockref_mark_dead(&dentry->d_lockref);
561
562
563
564
565
566 if (dentry->d_flags & DCACHE_OP_PRUNE)
567 dentry->d_op->d_prune(dentry);
568
569 if (dentry->d_flags & DCACHE_LRU_LIST) {
570 if (!(dentry->d_flags & DCACHE_SHRINK_LIST))
571 d_lru_del(dentry);
572 }
573
574 __d_drop(dentry);
575 dentry_unlist(dentry, parent);
576 if (parent)
577 spin_unlock(&parent->d_lock);
578 if (dentry->d_inode)
579 dentry_unlink_inode(dentry);
580 else
581 spin_unlock(&dentry->d_lock);
582 this_cpu_dec(nr_dentry);
583 if (dentry->d_op && dentry->d_op->d_release)
584 dentry->d_op->d_release(dentry);
585
586 spin_lock(&dentry->d_lock);
587 if (dentry->d_flags & DCACHE_SHRINK_LIST) {
588 dentry->d_flags |= DCACHE_MAY_FREE;
589 can_free = false;
590 }
591 spin_unlock(&dentry->d_lock);
592 if (likely(can_free))
593 dentry_free(dentry);
594 cond_resched();
595}
596
597static struct dentry *__lock_parent(struct dentry *dentry)
598{
599 struct dentry *parent;
600 rcu_read_lock();
601 spin_unlock(&dentry->d_lock);
602again:
603 parent = READ_ONCE(dentry->d_parent);
604 spin_lock(&parent->d_lock);
605
606
607
608
609
610
611
612
613 if (unlikely(parent != dentry->d_parent)) {
614 spin_unlock(&parent->d_lock);
615 goto again;
616 }
617 rcu_read_unlock();
618 if (parent != dentry)
619 spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED);
620 else
621 parent = NULL;
622 return parent;
623}
624
625static inline struct dentry *lock_parent(struct dentry *dentry)
626{
627 struct dentry *parent = dentry->d_parent;
628 if (IS_ROOT(dentry))
629 return NULL;
630 if (likely(spin_trylock(&parent->d_lock)))
631 return parent;
632 return __lock_parent(dentry);
633}
634
635static inline bool retain_dentry(struct dentry *dentry)
636{
637 WARN_ON(d_in_lookup(dentry));
638
639
640 if (unlikely(d_unhashed(dentry)))
641 return false;
642
643 if (unlikely(dentry->d_flags & DCACHE_DISCONNECTED))
644 return false;
645
646 if (unlikely(dentry->d_flags & DCACHE_OP_DELETE)) {
647 if (dentry->d_op->d_delete(dentry))
648 return false;
649 }
650
651 dentry->d_lockref.count--;
652 if (unlikely(!(dentry->d_flags & DCACHE_LRU_LIST)))
653 d_lru_add(dentry);
654 else if (unlikely(!(dentry->d_flags & DCACHE_REFERENCED)))
655 dentry->d_flags |= DCACHE_REFERENCED;
656 return true;
657}
658
659
660
661
662
663
664static struct dentry *dentry_kill(struct dentry *dentry)
665 __releases(dentry->d_lock)
666{
667 struct inode *inode = dentry->d_inode;
668 struct dentry *parent = NULL;
669
670 if (inode && unlikely(!spin_trylock(&inode->i_lock)))
671 goto slow_positive;
672
673 if (!IS_ROOT(dentry)) {
674 parent = dentry->d_parent;
675 if (unlikely(!spin_trylock(&parent->d_lock))) {
676 parent = __lock_parent(dentry);
677 if (likely(inode || !dentry->d_inode))
678 goto got_locks;
679
680 if (parent)
681 spin_unlock(&parent->d_lock);
682 inode = dentry->d_inode;
683 goto slow_positive;
684 }
685 }
686 __dentry_kill(dentry);
687 return parent;
688
689slow_positive:
690 spin_unlock(&dentry->d_lock);
691 spin_lock(&inode->i_lock);
692 spin_lock(&dentry->d_lock);
693 parent = lock_parent(dentry);
694got_locks:
695 if (unlikely(dentry->d_lockref.count != 1)) {
696 dentry->d_lockref.count--;
697 } else if (likely(!retain_dentry(dentry))) {
698 __dentry_kill(dentry);
699 return parent;
700 }
701
702 if (inode)
703 spin_unlock(&inode->i_lock);
704 if (parent)
705 spin_unlock(&parent->d_lock);
706 spin_unlock(&dentry->d_lock);
707 return NULL;
708}
709
710
711
712
713
714
715
716
717
718static inline bool fast_dput(struct dentry *dentry)
719{
720 int ret;
721 unsigned int d_flags;
722
723
724
725
726
727 if (unlikely(dentry->d_flags & DCACHE_OP_DELETE))
728 return lockref_put_or_lock(&dentry->d_lockref);
729
730
731
732
733
734 ret = lockref_put_return(&dentry->d_lockref);
735
736
737
738
739
740
741 if (unlikely(ret < 0)) {
742 spin_lock(&dentry->d_lock);
743 if (dentry->d_lockref.count > 1) {
744 dentry->d_lockref.count--;
745 spin_unlock(&dentry->d_lock);
746 return true;
747 }
748 return false;
749 }
750
751
752
753
754 if (ret)
755 return true;
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778 smp_rmb();
779 d_flags = READ_ONCE(dentry->d_flags);
780 d_flags &= DCACHE_REFERENCED | DCACHE_LRU_LIST | DCACHE_DISCONNECTED;
781
782
783 if (d_flags == (DCACHE_REFERENCED | DCACHE_LRU_LIST) && !d_unhashed(dentry))
784 return true;
785
786
787
788
789
790
791 spin_lock(&dentry->d_lock);
792
793
794
795
796
797
798
799 if (dentry->d_lockref.count) {
800 spin_unlock(&dentry->d_lock);
801 return true;
802 }
803
804
805
806
807
808
809 dentry->d_lockref.count = 1;
810 return false;
811}
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840void dput(struct dentry *dentry)
841{
842 while (dentry) {
843 might_sleep();
844
845 rcu_read_lock();
846 if (likely(fast_dput(dentry))) {
847 rcu_read_unlock();
848 return;
849 }
850
851
852 rcu_read_unlock();
853
854 if (likely(retain_dentry(dentry))) {
855 spin_unlock(&dentry->d_lock);
856 return;
857 }
858
859 dentry = dentry_kill(dentry);
860 }
861}
862EXPORT_SYMBOL(dput);
863
864
865
866static inline void __dget_dlock(struct dentry *dentry)
867{
868 dentry->d_lockref.count++;
869}
870
871static inline void __dget(struct dentry *dentry)
872{
873 lockref_get(&dentry->d_lockref);
874}
875
876struct dentry *dget_parent(struct dentry *dentry)
877{
878 int gotref;
879 struct dentry *ret;
880
881
882
883
884
885 rcu_read_lock();
886 ret = READ_ONCE(dentry->d_parent);
887 gotref = lockref_get_not_zero(&ret->d_lockref);
888 rcu_read_unlock();
889 if (likely(gotref)) {
890 if (likely(ret == READ_ONCE(dentry->d_parent)))
891 return ret;
892 dput(ret);
893 }
894
895repeat:
896
897
898
899
900 rcu_read_lock();
901 ret = dentry->d_parent;
902 spin_lock(&ret->d_lock);
903 if (unlikely(ret != dentry->d_parent)) {
904 spin_unlock(&ret->d_lock);
905 rcu_read_unlock();
906 goto repeat;
907 }
908 rcu_read_unlock();
909 BUG_ON(!ret->d_lockref.count);
910 ret->d_lockref.count++;
911 spin_unlock(&ret->d_lock);
912 return ret;
913}
914EXPORT_SYMBOL(dget_parent);
915
916static struct dentry * __d_find_any_alias(struct inode *inode)
917{
918 struct dentry *alias;
919
920 if (hlist_empty(&inode->i_dentry))
921 return NULL;
922 alias = hlist_entry(inode->i_dentry.first, struct dentry, d_u.d_alias);
923 __dget(alias);
924 return alias;
925}
926
927
928
929
930
931
932
933
934struct dentry *d_find_any_alias(struct inode *inode)
935{
936 struct dentry *de;
937
938 spin_lock(&inode->i_lock);
939 de = __d_find_any_alias(inode);
940 spin_unlock(&inode->i_lock);
941 return de;
942}
943EXPORT_SYMBOL(d_find_any_alias);
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959static struct dentry *__d_find_alias(struct inode *inode)
960{
961 struct dentry *alias;
962
963 if (S_ISDIR(inode->i_mode))
964 return __d_find_any_alias(inode);
965
966 hlist_for_each_entry(alias, &inode->i_dentry, d_u.d_alias) {
967 spin_lock(&alias->d_lock);
968 if (!d_unhashed(alias)) {
969 __dget_dlock(alias);
970 spin_unlock(&alias->d_lock);
971 return alias;
972 }
973 spin_unlock(&alias->d_lock);
974 }
975 return NULL;
976}
977
978struct dentry *d_find_alias(struct inode *inode)
979{
980 struct dentry *de = NULL;
981
982 if (!hlist_empty(&inode->i_dentry)) {
983 spin_lock(&inode->i_lock);
984 de = __d_find_alias(inode);
985 spin_unlock(&inode->i_lock);
986 }
987 return de;
988}
989EXPORT_SYMBOL(d_find_alias);
990
991
992
993
994
995void d_prune_aliases(struct inode *inode)
996{
997 struct dentry *dentry;
998restart:
999 spin_lock(&inode->i_lock);
1000 hlist_for_each_entry(dentry, &inode->i_dentry, d_u.d_alias) {
1001 spin_lock(&dentry->d_lock);
1002 if (!dentry->d_lockref.count) {
1003 struct dentry *parent = lock_parent(dentry);
1004 if (likely(!dentry->d_lockref.count)) {
1005 __dentry_kill(dentry);
1006 dput(parent);
1007 goto restart;
1008 }
1009 if (parent)
1010 spin_unlock(&parent->d_lock);
1011 }
1012 spin_unlock(&dentry->d_lock);
1013 }
1014 spin_unlock(&inode->i_lock);
1015}
1016EXPORT_SYMBOL(d_prune_aliases);
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029static bool shrink_lock_dentry(struct dentry *dentry)
1030{
1031 struct inode *inode;
1032 struct dentry *parent;
1033
1034 if (dentry->d_lockref.count)
1035 return false;
1036
1037 inode = dentry->d_inode;
1038 if (inode && unlikely(!spin_trylock(&inode->i_lock))) {
1039 spin_unlock(&dentry->d_lock);
1040 spin_lock(&inode->i_lock);
1041 spin_lock(&dentry->d_lock);
1042 if (unlikely(dentry->d_lockref.count))
1043 goto out;
1044
1045 if (unlikely(inode != dentry->d_inode))
1046 goto out;
1047 }
1048
1049 parent = dentry->d_parent;
1050 if (IS_ROOT(dentry) || likely(spin_trylock(&parent->d_lock)))
1051 return true;
1052
1053 spin_unlock(&dentry->d_lock);
1054 spin_lock(&parent->d_lock);
1055 if (unlikely(parent != dentry->d_parent)) {
1056 spin_unlock(&parent->d_lock);
1057 spin_lock(&dentry->d_lock);
1058 goto out;
1059 }
1060 spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED);
1061 if (likely(!dentry->d_lockref.count))
1062 return true;
1063 spin_unlock(&parent->d_lock);
1064out:
1065 if (inode)
1066 spin_unlock(&inode->i_lock);
1067 return false;
1068}
1069
1070static void shrink_dentry_list(struct list_head *list)
1071{
1072 while (!list_empty(list)) {
1073 struct dentry *dentry, *parent;
1074
1075 dentry = list_entry(list->prev, struct dentry, d_lru);
1076 spin_lock(&dentry->d_lock);
1077 rcu_read_lock();
1078 if (!shrink_lock_dentry(dentry)) {
1079 bool can_free = false;
1080 rcu_read_unlock();
1081 d_shrink_del(dentry);
1082 if (dentry->d_lockref.count < 0)
1083 can_free = dentry->d_flags & DCACHE_MAY_FREE;
1084 spin_unlock(&dentry->d_lock);
1085 if (can_free)
1086 dentry_free(dentry);
1087 continue;
1088 }
1089 rcu_read_unlock();
1090 d_shrink_del(dentry);
1091 parent = dentry->d_parent;
1092 __dentry_kill(dentry);
1093 if (parent == dentry)
1094 continue;
1095
1096
1097
1098
1099
1100
1101 dentry = parent;
1102 while (dentry && !lockref_put_or_lock(&dentry->d_lockref))
1103 dentry = dentry_kill(dentry);
1104 }
1105}
1106
1107static enum lru_status dentry_lru_isolate(struct list_head *item,
1108 struct list_lru_one *lru, spinlock_t *lru_lock, void *arg)
1109{
1110 struct list_head *freeable = arg;
1111 struct dentry *dentry = container_of(item, struct dentry, d_lru);
1112
1113
1114
1115
1116
1117
1118
1119 if (!spin_trylock(&dentry->d_lock))
1120 return LRU_SKIP;
1121
1122
1123
1124
1125
1126
1127 if (dentry->d_lockref.count) {
1128 d_lru_isolate(lru, dentry);
1129 spin_unlock(&dentry->d_lock);
1130 return LRU_REMOVED;
1131 }
1132
1133 if (dentry->d_flags & DCACHE_REFERENCED) {
1134 dentry->d_flags &= ~DCACHE_REFERENCED;
1135 spin_unlock(&dentry->d_lock);
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156 return LRU_ROTATE;
1157 }
1158
1159 d_lru_shrink_move(lru, dentry, freeable);
1160 spin_unlock(&dentry->d_lock);
1161
1162 return LRU_REMOVED;
1163}
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177long prune_dcache_sb(struct super_block *sb, struct shrink_control *sc)
1178{
1179 LIST_HEAD(dispose);
1180 long freed;
1181
1182 freed = list_lru_shrink_walk(&sb->s_dentry_lru, sc,
1183 dentry_lru_isolate, &dispose);
1184 shrink_dentry_list(&dispose);
1185 return freed;
1186}
1187
1188static enum lru_status dentry_lru_isolate_shrink(struct list_head *item,
1189 struct list_lru_one *lru, spinlock_t *lru_lock, void *arg)
1190{
1191 struct list_head *freeable = arg;
1192 struct dentry *dentry = container_of(item, struct dentry, d_lru);
1193
1194
1195
1196
1197
1198
1199 if (!spin_trylock(&dentry->d_lock))
1200 return LRU_SKIP;
1201
1202 d_lru_shrink_move(lru, dentry, freeable);
1203 spin_unlock(&dentry->d_lock);
1204
1205 return LRU_REMOVED;
1206}
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216void shrink_dcache_sb(struct super_block *sb)
1217{
1218 do {
1219 LIST_HEAD(dispose);
1220
1221 list_lru_walk(&sb->s_dentry_lru,
1222 dentry_lru_isolate_shrink, &dispose, 1024);
1223 shrink_dentry_list(&dispose);
1224 } while (list_lru_count(&sb->s_dentry_lru) > 0);
1225}
1226EXPORT_SYMBOL(shrink_dcache_sb);
1227
1228
1229
1230
1231
1232
1233
1234
1235enum d_walk_ret {
1236 D_WALK_CONTINUE,
1237 D_WALK_QUIT,
1238 D_WALK_NORETRY,
1239 D_WALK_SKIP,
1240};
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250static void d_walk(struct dentry *parent, void *data,
1251 enum d_walk_ret (*enter)(void *, struct dentry *))
1252{
1253 struct dentry *this_parent;
1254 struct list_head *next;
1255 unsigned seq = 0;
1256 enum d_walk_ret ret;
1257 bool retry = true;
1258
1259again:
1260 read_seqbegin_or_lock(&rename_lock, &seq);
1261 this_parent = parent;
1262 spin_lock(&this_parent->d_lock);
1263
1264 ret = enter(data, this_parent);
1265 switch (ret) {
1266 case D_WALK_CONTINUE:
1267 break;
1268 case D_WALK_QUIT:
1269 case D_WALK_SKIP:
1270 goto out_unlock;
1271 case D_WALK_NORETRY:
1272 retry = false;
1273 break;
1274 }
1275repeat:
1276 next = this_parent->d_subdirs.next;
1277resume:
1278 while (next != &this_parent->d_subdirs) {
1279 struct list_head *tmp = next;
1280 struct dentry *dentry = list_entry(tmp, struct dentry, d_child);
1281 next = tmp->next;
1282
1283 if (unlikely(dentry->d_flags & DCACHE_DENTRY_CURSOR))
1284 continue;
1285
1286 spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED);
1287
1288 ret = enter(data, dentry);
1289 switch (ret) {
1290 case D_WALK_CONTINUE:
1291 break;
1292 case D_WALK_QUIT:
1293 spin_unlock(&dentry->d_lock);
1294 goto out_unlock;
1295 case D_WALK_NORETRY:
1296 retry = false;
1297 break;
1298 case D_WALK_SKIP:
1299 spin_unlock(&dentry->d_lock);
1300 continue;
1301 }
1302
1303 if (!list_empty(&dentry->d_subdirs)) {
1304 spin_unlock(&this_parent->d_lock);
1305 spin_release(&dentry->d_lock.dep_map, 1, _RET_IP_);
1306 this_parent = dentry;
1307 spin_acquire(&this_parent->d_lock.dep_map, 0, 1, _RET_IP_);
1308 goto repeat;
1309 }
1310 spin_unlock(&dentry->d_lock);
1311 }
1312
1313
1314
1315 rcu_read_lock();
1316ascend:
1317 if (this_parent != parent) {
1318 struct dentry *child = this_parent;
1319 this_parent = child->d_parent;
1320
1321 spin_unlock(&child->d_lock);
1322 spin_lock(&this_parent->d_lock);
1323
1324
1325 if (need_seqretry(&rename_lock, seq))
1326 goto rename_retry;
1327
1328 do {
1329 next = child->d_child.next;
1330 if (next == &this_parent->d_subdirs)
1331 goto ascend;
1332 child = list_entry(next, struct dentry, d_child);
1333 } while (unlikely(child->d_flags & DCACHE_DENTRY_KILLED));
1334 rcu_read_unlock();
1335 goto resume;
1336 }
1337 if (need_seqretry(&rename_lock, seq))
1338 goto rename_retry;
1339 rcu_read_unlock();
1340
1341out_unlock:
1342 spin_unlock(&this_parent->d_lock);
1343 done_seqretry(&rename_lock, seq);
1344 return;
1345
1346rename_retry:
1347 spin_unlock(&this_parent->d_lock);
1348 rcu_read_unlock();
1349 BUG_ON(seq & 1);
1350 if (!retry)
1351 return;
1352 seq = 1;
1353 goto again;
1354}
1355
1356struct check_mount {
1357 struct vfsmount *mnt;
1358 unsigned int mounted;
1359};
1360
1361static enum d_walk_ret path_check_mount(void *data, struct dentry *dentry)
1362{
1363 struct check_mount *info = data;
1364 struct path path = { .mnt = info->mnt, .dentry = dentry };
1365
1366 if (likely(!d_mountpoint(dentry)))
1367 return D_WALK_CONTINUE;
1368 if (__path_is_mountpoint(&path)) {
1369 info->mounted = 1;
1370 return D_WALK_QUIT;
1371 }
1372 return D_WALK_CONTINUE;
1373}
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383int path_has_submounts(const struct path *parent)
1384{
1385 struct check_mount data = { .mnt = parent->mnt, .mounted = 0 };
1386
1387 read_seqlock_excl(&mount_lock);
1388 d_walk(parent->dentry, &data, path_check_mount);
1389 read_sequnlock_excl(&mount_lock);
1390
1391 return data.mounted;
1392}
1393EXPORT_SYMBOL(path_has_submounts);
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403int d_set_mounted(struct dentry *dentry)
1404{
1405 struct dentry *p;
1406 int ret = -ENOENT;
1407 write_seqlock(&rename_lock);
1408 for (p = dentry->d_parent; !IS_ROOT(p); p = p->d_parent) {
1409
1410 spin_lock(&p->d_lock);
1411 if (unlikely(d_unhashed(p))) {
1412 spin_unlock(&p->d_lock);
1413 goto out;
1414 }
1415 spin_unlock(&p->d_lock);
1416 }
1417 spin_lock(&dentry->d_lock);
1418 if (!d_unlinked(dentry)) {
1419 ret = -EBUSY;
1420 if (!d_mountpoint(dentry)) {
1421 dentry->d_flags |= DCACHE_MOUNTED;
1422 ret = 0;
1423 }
1424 }
1425 spin_unlock(&dentry->d_lock);
1426out:
1427 write_sequnlock(&rename_lock);
1428 return ret;
1429}
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446struct select_data {
1447 struct dentry *start;
1448 struct list_head dispose;
1449 int found;
1450};
1451
1452static enum d_walk_ret select_collect(void *_data, struct dentry *dentry)
1453{
1454 struct select_data *data = _data;
1455 enum d_walk_ret ret = D_WALK_CONTINUE;
1456
1457 if (data->start == dentry)
1458 goto out;
1459
1460 if (dentry->d_flags & DCACHE_SHRINK_LIST) {
1461 data->found++;
1462 } else {
1463 if (dentry->d_flags & DCACHE_LRU_LIST)
1464 d_lru_del(dentry);
1465 if (!dentry->d_lockref.count) {
1466 d_shrink_add(dentry, &data->dispose);
1467 data->found++;
1468 }
1469 }
1470
1471
1472
1473
1474
1475 if (!list_empty(&data->dispose))
1476 ret = need_resched() ? D_WALK_QUIT : D_WALK_NORETRY;
1477out:
1478 return ret;
1479}
1480
1481
1482
1483
1484
1485
1486
1487void shrink_dcache_parent(struct dentry *parent)
1488{
1489 for (;;) {
1490 struct select_data data;
1491
1492 INIT_LIST_HEAD(&data.dispose);
1493 data.start = parent;
1494 data.found = 0;
1495
1496 d_walk(parent, &data, select_collect);
1497
1498 if (!list_empty(&data.dispose)) {
1499 shrink_dentry_list(&data.dispose);
1500 continue;
1501 }
1502
1503 cond_resched();
1504 if (!data.found)
1505 break;
1506 }
1507}
1508EXPORT_SYMBOL(shrink_dcache_parent);
1509
1510static enum d_walk_ret umount_check(void *_data, struct dentry *dentry)
1511{
1512
1513 if (!list_empty(&dentry->d_subdirs))
1514 return D_WALK_CONTINUE;
1515
1516
1517 if (dentry == _data && dentry->d_lockref.count == 1)
1518 return D_WALK_CONTINUE;
1519
1520 printk(KERN_ERR "BUG: Dentry %p{i=%lx,n=%pd} "
1521 " still in use (%d) [unmount of %s %s]\n",
1522 dentry,
1523 dentry->d_inode ?
1524 dentry->d_inode->i_ino : 0UL,
1525 dentry,
1526 dentry->d_lockref.count,
1527 dentry->d_sb->s_type->name,
1528 dentry->d_sb->s_id);
1529 WARN_ON(1);
1530 return D_WALK_CONTINUE;
1531}
1532
1533static void do_one_tree(struct dentry *dentry)
1534{
1535 shrink_dcache_parent(dentry);
1536 d_walk(dentry, dentry, umount_check);
1537 d_drop(dentry);
1538 dput(dentry);
1539}
1540
1541
1542
1543
1544void shrink_dcache_for_umount(struct super_block *sb)
1545{
1546 struct dentry *dentry;
1547
1548 WARN(down_read_trylock(&sb->s_umount), "s_umount should've been locked");
1549
1550 dentry = sb->s_root;
1551 sb->s_root = NULL;
1552 do_one_tree(dentry);
1553
1554 while (!hlist_bl_empty(&sb->s_roots)) {
1555 dentry = dget(hlist_bl_entry(hlist_bl_first(&sb->s_roots), struct dentry, d_hash));
1556 do_one_tree(dentry);
1557 }
1558}
1559
1560static enum d_walk_ret find_submount(void *_data, struct dentry *dentry)
1561{
1562 struct dentry **victim = _data;
1563 if (d_mountpoint(dentry)) {
1564 __dget_dlock(dentry);
1565 *victim = dentry;
1566 return D_WALK_QUIT;
1567 }
1568 return D_WALK_CONTINUE;
1569}
1570
1571
1572
1573
1574
1575void d_invalidate(struct dentry *dentry)
1576{
1577 bool had_submounts = false;
1578 spin_lock(&dentry->d_lock);
1579 if (d_unhashed(dentry)) {
1580 spin_unlock(&dentry->d_lock);
1581 return;
1582 }
1583 __d_drop(dentry);
1584 spin_unlock(&dentry->d_lock);
1585
1586
1587 if (!dentry->d_inode)
1588 return;
1589
1590 shrink_dcache_parent(dentry);
1591 for (;;) {
1592 struct dentry *victim = NULL;
1593 d_walk(dentry, &victim, find_submount);
1594 if (!victim) {
1595 if (had_submounts)
1596 shrink_dcache_parent(dentry);
1597 return;
1598 }
1599 had_submounts = true;
1600 detach_mounts(victim);
1601 dput(victim);
1602 }
1603}
1604EXPORT_SYMBOL(d_invalidate);
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616struct dentry *__d_alloc(struct super_block *sb, const struct qstr *name)
1617{
1618 struct dentry *dentry;
1619 char *dname;
1620 int err;
1621
1622 dentry = kmem_cache_alloc(dentry_cache, GFP_KERNEL);
1623 if (!dentry)
1624 return NULL;
1625
1626
1627
1628
1629
1630
1631
1632 dentry->d_iname[DNAME_INLINE_LEN-1] = 0;
1633 if (unlikely(!name)) {
1634 name = &slash_name;
1635 dname = dentry->d_iname;
1636 } else if (name->len > DNAME_INLINE_LEN-1) {
1637 size_t size = offsetof(struct external_name, name[1]);
1638 struct external_name *p = kmalloc(size + name->len,
1639 GFP_KERNEL_ACCOUNT |
1640 __GFP_RECLAIMABLE);
1641 if (!p) {
1642 kmem_cache_free(dentry_cache, dentry);
1643 return NULL;
1644 }
1645 atomic_set(&p->u.count, 1);
1646 dname = p->name;
1647 } else {
1648 dname = dentry->d_iname;
1649 }
1650
1651 dentry->d_name.len = name->len;
1652 dentry->d_name.hash = name->hash;
1653 memcpy(dname, name->name, name->len);
1654 dname[name->len] = 0;
1655
1656
1657 smp_store_release(&dentry->d_name.name, dname);
1658
1659 dentry->d_lockref.count = 1;
1660 dentry->d_flags = 0;
1661 spin_lock_init(&dentry->d_lock);
1662 seqcount_init(&dentry->d_seq);
1663 dentry->d_inode = NULL;
1664 dentry->d_parent = dentry;
1665 dentry->d_sb = sb;
1666 dentry->d_op = NULL;
1667 dentry->d_fsdata = NULL;
1668 INIT_HLIST_BL_NODE(&dentry->d_hash);
1669 INIT_LIST_HEAD(&dentry->d_lru);
1670 INIT_LIST_HEAD(&dentry->d_subdirs);
1671 INIT_HLIST_NODE(&dentry->d_u.d_alias);
1672 INIT_LIST_HEAD(&dentry->d_child);
1673 d_set_d_op(dentry, dentry->d_sb->s_d_op);
1674
1675 if (dentry->d_op && dentry->d_op->d_init) {
1676 err = dentry->d_op->d_init(dentry);
1677 if (err) {
1678 if (dname_external(dentry))
1679 kfree(external_name(dentry));
1680 kmem_cache_free(dentry_cache, dentry);
1681 return NULL;
1682 }
1683 }
1684
1685 this_cpu_inc(nr_dentry);
1686
1687 return dentry;
1688}
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699struct dentry *d_alloc(struct dentry * parent, const struct qstr *name)
1700{
1701 struct dentry *dentry = __d_alloc(parent->d_sb, name);
1702 if (!dentry)
1703 return NULL;
1704 spin_lock(&parent->d_lock);
1705
1706
1707
1708
1709 __dget_dlock(parent);
1710 dentry->d_parent = parent;
1711 list_add(&dentry->d_child, &parent->d_subdirs);
1712 spin_unlock(&parent->d_lock);
1713
1714 return dentry;
1715}
1716EXPORT_SYMBOL(d_alloc);
1717
1718struct dentry *d_alloc_anon(struct super_block *sb)
1719{
1720 return __d_alloc(sb, NULL);
1721}
1722EXPORT_SYMBOL(d_alloc_anon);
1723
1724struct dentry *d_alloc_cursor(struct dentry * parent)
1725{
1726 struct dentry *dentry = d_alloc_anon(parent->d_sb);
1727 if (dentry) {
1728 dentry->d_flags |= DCACHE_DENTRY_CURSOR;
1729 dentry->d_parent = dget(parent);
1730 }
1731 return dentry;
1732}
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749struct dentry *d_alloc_pseudo(struct super_block *sb, const struct qstr *name)
1750{
1751 struct dentry *dentry = __d_alloc(sb, name);
1752 if (likely(dentry))
1753 dentry->d_flags |= DCACHE_NORCU;
1754 return dentry;
1755}
1756
1757struct dentry *d_alloc_name(struct dentry *parent, const char *name)
1758{
1759 struct qstr q;
1760
1761 q.name = name;
1762 q.hash_len = hashlen_string(parent, name);
1763 return d_alloc(parent, &q);
1764}
1765EXPORT_SYMBOL(d_alloc_name);
1766
1767void d_set_d_op(struct dentry *dentry, const struct dentry_operations *op)
1768{
1769 WARN_ON_ONCE(dentry->d_op);
1770 WARN_ON_ONCE(dentry->d_flags & (DCACHE_OP_HASH |
1771 DCACHE_OP_COMPARE |
1772 DCACHE_OP_REVALIDATE |
1773 DCACHE_OP_WEAK_REVALIDATE |
1774 DCACHE_OP_DELETE |
1775 DCACHE_OP_REAL));
1776 dentry->d_op = op;
1777 if (!op)
1778 return;
1779 if (op->d_hash)
1780 dentry->d_flags |= DCACHE_OP_HASH;
1781 if (op->d_compare)
1782 dentry->d_flags |= DCACHE_OP_COMPARE;
1783 if (op->d_revalidate)
1784 dentry->d_flags |= DCACHE_OP_REVALIDATE;
1785 if (op->d_weak_revalidate)
1786 dentry->d_flags |= DCACHE_OP_WEAK_REVALIDATE;
1787 if (op->d_delete)
1788 dentry->d_flags |= DCACHE_OP_DELETE;
1789 if (op->d_prune)
1790 dentry->d_flags |= DCACHE_OP_PRUNE;
1791 if (op->d_real)
1792 dentry->d_flags |= DCACHE_OP_REAL;
1793
1794}
1795EXPORT_SYMBOL(d_set_d_op);
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805void d_set_fallthru(struct dentry *dentry)
1806{
1807 spin_lock(&dentry->d_lock);
1808 dentry->d_flags |= DCACHE_FALLTHRU;
1809 spin_unlock(&dentry->d_lock);
1810}
1811EXPORT_SYMBOL(d_set_fallthru);
1812
1813static unsigned d_flags_for_inode(struct inode *inode)
1814{
1815 unsigned add_flags = DCACHE_REGULAR_TYPE;
1816
1817 if (!inode)
1818 return DCACHE_MISS_TYPE;
1819
1820 if (S_ISDIR(inode->i_mode)) {
1821 add_flags = DCACHE_DIRECTORY_TYPE;
1822 if (unlikely(!(inode->i_opflags & IOP_LOOKUP))) {
1823 if (unlikely(!inode->i_op->lookup))
1824 add_flags = DCACHE_AUTODIR_TYPE;
1825 else
1826 inode->i_opflags |= IOP_LOOKUP;
1827 }
1828 goto type_determined;
1829 }
1830
1831 if (unlikely(!(inode->i_opflags & IOP_NOFOLLOW))) {
1832 if (unlikely(inode->i_op->get_link)) {
1833 add_flags = DCACHE_SYMLINK_TYPE;
1834 goto type_determined;
1835 }
1836 inode->i_opflags |= IOP_NOFOLLOW;
1837 }
1838
1839 if (unlikely(!S_ISREG(inode->i_mode)))
1840 add_flags = DCACHE_SPECIAL_TYPE;
1841
1842type_determined:
1843 if (unlikely(IS_AUTOMOUNT(inode)))
1844 add_flags |= DCACHE_NEED_AUTOMOUNT;
1845 return add_flags;
1846}
1847
1848static void __d_instantiate(struct dentry *dentry, struct inode *inode)
1849{
1850 unsigned add_flags = d_flags_for_inode(inode);
1851 WARN_ON(d_in_lookup(dentry));
1852
1853 spin_lock(&dentry->d_lock);
1854
1855
1856
1857 if (dentry->d_flags & DCACHE_LRU_LIST)
1858 this_cpu_dec(nr_dentry_negative);
1859 hlist_add_head(&dentry->d_u.d_alias, &inode->i_dentry);
1860 raw_write_seqcount_begin(&dentry->d_seq);
1861 __d_set_inode_and_type(dentry, inode, add_flags);
1862 raw_write_seqcount_end(&dentry->d_seq);
1863 fsnotify_update_flags(dentry);
1864 spin_unlock(&dentry->d_lock);
1865}
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882void d_instantiate(struct dentry *entry, struct inode * inode)
1883{
1884 BUG_ON(!hlist_unhashed(&entry->d_u.d_alias));
1885 if (inode) {
1886 security_d_instantiate(entry, inode);
1887 spin_lock(&inode->i_lock);
1888 __d_instantiate(entry, inode);
1889 spin_unlock(&inode->i_lock);
1890 }
1891}
1892EXPORT_SYMBOL(d_instantiate);
1893
1894
1895
1896
1897
1898
1899
1900void d_instantiate_new(struct dentry *entry, struct inode *inode)
1901{
1902 BUG_ON(!hlist_unhashed(&entry->d_u.d_alias));
1903 BUG_ON(!inode);
1904 lockdep_annotate_inode_mutex_key(inode);
1905 security_d_instantiate(entry, inode);
1906 spin_lock(&inode->i_lock);
1907 __d_instantiate(entry, inode);
1908 WARN_ON(!(inode->i_state & I_NEW));
1909 inode->i_state &= ~I_NEW & ~I_CREATING;
1910 smp_mb();
1911 wake_up_bit(&inode->i_state, __I_NEW);
1912 spin_unlock(&inode->i_lock);
1913}
1914EXPORT_SYMBOL(d_instantiate_new);
1915
1916struct dentry *d_make_root(struct inode *root_inode)
1917{
1918 struct dentry *res = NULL;
1919
1920 if (root_inode) {
1921 res = d_alloc_anon(root_inode->i_sb);
1922 if (res)
1923 d_instantiate(res, root_inode);
1924 else
1925 iput(root_inode);
1926 }
1927 return res;
1928}
1929EXPORT_SYMBOL(d_make_root);
1930
1931static struct dentry *__d_instantiate_anon(struct dentry *dentry,
1932 struct inode *inode,
1933 bool disconnected)
1934{
1935 struct dentry *res;
1936 unsigned add_flags;
1937
1938 security_d_instantiate(dentry, inode);
1939 spin_lock(&inode->i_lock);
1940 res = __d_find_any_alias(inode);
1941 if (res) {
1942 spin_unlock(&inode->i_lock);
1943 dput(dentry);
1944 goto out_iput;
1945 }
1946
1947
1948 add_flags = d_flags_for_inode(inode);
1949
1950 if (disconnected)
1951 add_flags |= DCACHE_DISCONNECTED;
1952
1953 spin_lock(&dentry->d_lock);
1954 __d_set_inode_and_type(dentry, inode, add_flags);
1955 hlist_add_head(&dentry->d_u.d_alias, &inode->i_dentry);
1956 if (!disconnected) {
1957 hlist_bl_lock(&dentry->d_sb->s_roots);
1958 hlist_bl_add_head(&dentry->d_hash, &dentry->d_sb->s_roots);
1959 hlist_bl_unlock(&dentry->d_sb->s_roots);
1960 }
1961 spin_unlock(&dentry->d_lock);
1962 spin_unlock(&inode->i_lock);
1963
1964 return dentry;
1965
1966 out_iput:
1967 iput(inode);
1968 return res;
1969}
1970
1971struct dentry *d_instantiate_anon(struct dentry *dentry, struct inode *inode)
1972{
1973 return __d_instantiate_anon(dentry, inode, true);
1974}
1975EXPORT_SYMBOL(d_instantiate_anon);
1976
1977static struct dentry *__d_obtain_alias(struct inode *inode, bool disconnected)
1978{
1979 struct dentry *tmp;
1980 struct dentry *res;
1981
1982 if (!inode)
1983 return ERR_PTR(-ESTALE);
1984 if (IS_ERR(inode))
1985 return ERR_CAST(inode);
1986
1987 res = d_find_any_alias(inode);
1988 if (res)
1989 goto out_iput;
1990
1991 tmp = d_alloc_anon(inode->i_sb);
1992 if (!tmp) {
1993 res = ERR_PTR(-ENOMEM);
1994 goto out_iput;
1995 }
1996
1997 return __d_instantiate_anon(tmp, inode, disconnected);
1998
1999out_iput:
2000 iput(inode);
2001 return res;
2002}
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022struct dentry *d_obtain_alias(struct inode *inode)
2023{
2024 return __d_obtain_alias(inode, true);
2025}
2026EXPORT_SYMBOL(d_obtain_alias);
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043struct dentry *d_obtain_root(struct inode *inode)
2044{
2045 return __d_obtain_alias(inode, false);
2046}
2047EXPORT_SYMBOL(d_obtain_root);
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065struct dentry *d_add_ci(struct dentry *dentry, struct inode *inode,
2066 struct qstr *name)
2067{
2068 struct dentry *found, *res;
2069
2070
2071
2072
2073
2074 found = d_hash_and_lookup(dentry->d_parent, name);
2075 if (found) {
2076 iput(inode);
2077 return found;
2078 }
2079 if (d_in_lookup(dentry)) {
2080 found = d_alloc_parallel(dentry->d_parent, name,
2081 dentry->d_wait);
2082 if (IS_ERR(found) || !d_in_lookup(found)) {
2083 iput(inode);
2084 return found;
2085 }
2086 } else {
2087 found = d_alloc(dentry->d_parent, name);
2088 if (!found) {
2089 iput(inode);
2090 return ERR_PTR(-ENOMEM);
2091 }
2092 }
2093 res = d_splice_alias(inode, found);
2094 if (res) {
2095 dput(found);
2096 return res;
2097 }
2098 return found;
2099}
2100EXPORT_SYMBOL(d_add_ci);
2101
2102
2103static inline bool d_same_name(const struct dentry *dentry,
2104 const struct dentry *parent,
2105 const struct qstr *name)
2106{
2107 if (likely(!(parent->d_flags & DCACHE_OP_COMPARE))) {
2108 if (dentry->d_name.len != name->len)
2109 return false;
2110 return dentry_cmp(dentry, name->name, name->len) == 0;
2111 }
2112 return parent->d_op->d_compare(dentry,
2113 dentry->d_name.len, dentry->d_name.name,
2114 name) == 0;
2115}
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146struct dentry *__d_lookup_rcu(const struct dentry *parent,
2147 const struct qstr *name,
2148 unsigned *seqp)
2149{
2150 u64 hashlen = name->hash_len;
2151 const unsigned char *str = name->name;
2152 struct hlist_bl_head *b = d_hash(hashlen_hash(hashlen));
2153 struct hlist_bl_node *node;
2154 struct dentry *dentry;
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176 hlist_bl_for_each_entry_rcu(dentry, node, b, d_hash) {
2177 unsigned seq;
2178
2179seqretry:
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197 seq = raw_seqcount_begin(&dentry->d_seq);
2198 if (dentry->d_parent != parent)
2199 continue;
2200 if (d_unhashed(dentry))
2201 continue;
2202
2203 if (unlikely(parent->d_flags & DCACHE_OP_COMPARE)) {
2204 int tlen;
2205 const char *tname;
2206 if (dentry->d_name.hash != hashlen_hash(hashlen))
2207 continue;
2208 tlen = dentry->d_name.len;
2209 tname = dentry->d_name.name;
2210
2211 if (read_seqcount_retry(&dentry->d_seq, seq)) {
2212 cpu_relax();
2213 goto seqretry;
2214 }
2215 if (parent->d_op->d_compare(dentry,
2216 tlen, tname, name) != 0)
2217 continue;
2218 } else {
2219 if (dentry->d_name.hash_len != hashlen)
2220 continue;
2221 if (dentry_cmp(dentry, str, hashlen_len(hashlen)) != 0)
2222 continue;
2223 }
2224 *seqp = seq;
2225 return dentry;
2226 }
2227 return NULL;
2228}
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241struct dentry *d_lookup(const struct dentry *parent, const struct qstr *name)
2242{
2243 struct dentry *dentry;
2244 unsigned seq;
2245
2246 do {
2247 seq = read_seqbegin(&rename_lock);
2248 dentry = __d_lookup(parent, name);
2249 if (dentry)
2250 break;
2251 } while (read_seqretry(&rename_lock, seq));
2252 return dentry;
2253}
2254EXPORT_SYMBOL(d_lookup);
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271struct dentry *__d_lookup(const struct dentry *parent, const struct qstr *name)
2272{
2273 unsigned int hash = name->hash;
2274 struct hlist_bl_head *b = d_hash(hash);
2275 struct hlist_bl_node *node;
2276 struct dentry *found = NULL;
2277 struct dentry *dentry;
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299 rcu_read_lock();
2300
2301 hlist_bl_for_each_entry_rcu(dentry, node, b, d_hash) {
2302
2303 if (dentry->d_name.hash != hash)
2304 continue;
2305
2306 spin_lock(&dentry->d_lock);
2307 if (dentry->d_parent != parent)
2308 goto next;
2309 if (d_unhashed(dentry))
2310 goto next;
2311
2312 if (!d_same_name(dentry, parent, name))
2313 goto next;
2314
2315 dentry->d_lockref.count++;
2316 found = dentry;
2317 spin_unlock(&dentry->d_lock);
2318 break;
2319next:
2320 spin_unlock(&dentry->d_lock);
2321 }
2322 rcu_read_unlock();
2323
2324 return found;
2325}
2326
2327
2328
2329
2330
2331
2332
2333
2334struct dentry *d_hash_and_lookup(struct dentry *dir, struct qstr *name)
2335{
2336
2337
2338
2339
2340
2341 name->hash = full_name_hash(dir, name->name, name->len);
2342 if (dir->d_flags & DCACHE_OP_HASH) {
2343 int err = dir->d_op->d_hash(dir, name);
2344 if (unlikely(err < 0))
2345 return ERR_PTR(err);
2346 }
2347 return d_lookup(dir, name);
2348}
2349EXPORT_SYMBOL(d_hash_and_lookup);
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372void d_delete(struct dentry * dentry)
2373{
2374 struct inode *inode = dentry->d_inode;
2375 int isdir = d_is_dir(dentry);
2376
2377 spin_lock(&inode->i_lock);
2378 spin_lock(&dentry->d_lock);
2379
2380
2381
2382 if (dentry->d_lockref.count == 1) {
2383 dentry->d_flags &= ~DCACHE_CANT_MOUNT;
2384 dentry_unlink_inode(dentry);
2385 } else {
2386 __d_drop(dentry);
2387 spin_unlock(&dentry->d_lock);
2388 spin_unlock(&inode->i_lock);
2389 }
2390 fsnotify_nameremove(dentry, isdir);
2391}
2392EXPORT_SYMBOL(d_delete);
2393
2394static void __d_rehash(struct dentry *entry)
2395{
2396 struct hlist_bl_head *b = d_hash(entry->d_name.hash);
2397
2398 hlist_bl_lock(b);
2399 hlist_bl_add_head_rcu(&entry->d_hash, b);
2400 hlist_bl_unlock(b);
2401}
2402
2403
2404
2405
2406
2407
2408
2409
2410void d_rehash(struct dentry * entry)
2411{
2412 spin_lock(&entry->d_lock);
2413 __d_rehash(entry);
2414 spin_unlock(&entry->d_lock);
2415}
2416EXPORT_SYMBOL(d_rehash);
2417
2418static inline unsigned start_dir_add(struct inode *dir)
2419{
2420
2421 for (;;) {
2422 unsigned n = dir->i_dir_seq;
2423 if (!(n & 1) && cmpxchg(&dir->i_dir_seq, n, n + 1) == n)
2424 return n;
2425 cpu_relax();
2426 }
2427}
2428
2429static inline void end_dir_add(struct inode *dir, unsigned n)
2430{
2431 smp_store_release(&dir->i_dir_seq, n + 2);
2432}
2433
2434static void d_wait_lookup(struct dentry *dentry)
2435{
2436 if (d_in_lookup(dentry)) {
2437 DECLARE_WAITQUEUE(wait, current);
2438 add_wait_queue(dentry->d_wait, &wait);
2439 do {
2440 set_current_state(TASK_UNINTERRUPTIBLE);
2441 spin_unlock(&dentry->d_lock);
2442 schedule();
2443 spin_lock(&dentry->d_lock);
2444 } while (d_in_lookup(dentry));
2445 }
2446}
2447
2448struct dentry *d_alloc_parallel(struct dentry *parent,
2449 const struct qstr *name,
2450 wait_queue_head_t *wq)
2451{
2452 unsigned int hash = name->hash;
2453 struct hlist_bl_head *b = in_lookup_hash(parent, hash);
2454 struct hlist_bl_node *node;
2455 struct dentry *new = d_alloc(parent, name);
2456 struct dentry *dentry;
2457 unsigned seq, r_seq, d_seq;
2458
2459 if (unlikely(!new))
2460 return ERR_PTR(-ENOMEM);
2461
2462retry:
2463 rcu_read_lock();
2464 seq = smp_load_acquire(&parent->d_inode->i_dir_seq);
2465 r_seq = read_seqbegin(&rename_lock);
2466 dentry = __d_lookup_rcu(parent, name, &d_seq);
2467 if (unlikely(dentry)) {
2468 if (!lockref_get_not_dead(&dentry->d_lockref)) {
2469 rcu_read_unlock();
2470 goto retry;
2471 }
2472 if (read_seqcount_retry(&dentry->d_seq, d_seq)) {
2473 rcu_read_unlock();
2474 dput(dentry);
2475 goto retry;
2476 }
2477 rcu_read_unlock();
2478 dput(new);
2479 return dentry;
2480 }
2481 if (unlikely(read_seqretry(&rename_lock, r_seq))) {
2482 rcu_read_unlock();
2483 goto retry;
2484 }
2485
2486 if (unlikely(seq & 1)) {
2487 rcu_read_unlock();
2488 goto retry;
2489 }
2490
2491 hlist_bl_lock(b);
2492 if (unlikely(READ_ONCE(parent->d_inode->i_dir_seq) != seq)) {
2493 hlist_bl_unlock(b);
2494 rcu_read_unlock();
2495 goto retry;
2496 }
2497
2498
2499
2500
2501
2502
2503
2504 hlist_bl_for_each_entry(dentry, node, b, d_u.d_in_lookup_hash) {
2505 if (dentry->d_name.hash != hash)
2506 continue;
2507 if (dentry->d_parent != parent)
2508 continue;
2509 if (!d_same_name(dentry, parent, name))
2510 continue;
2511 hlist_bl_unlock(b);
2512
2513 if (!lockref_get_not_dead(&dentry->d_lockref)) {
2514 rcu_read_unlock();
2515 goto retry;
2516 }
2517
2518 rcu_read_unlock();
2519
2520
2521
2522
2523 spin_lock(&dentry->d_lock);
2524 d_wait_lookup(dentry);
2525
2526
2527
2528
2529
2530
2531 if (unlikely(dentry->d_name.hash != hash))
2532 goto mismatch;
2533 if (unlikely(dentry->d_parent != parent))
2534 goto mismatch;
2535 if (unlikely(d_unhashed(dentry)))
2536 goto mismatch;
2537 if (unlikely(!d_same_name(dentry, parent, name)))
2538 goto mismatch;
2539
2540 spin_unlock(&dentry->d_lock);
2541 dput(new);
2542 return dentry;
2543 }
2544 rcu_read_unlock();
2545
2546 new->d_flags |= DCACHE_PAR_LOOKUP;
2547 new->d_wait = wq;
2548 hlist_bl_add_head_rcu(&new->d_u.d_in_lookup_hash, b);
2549 hlist_bl_unlock(b);
2550 return new;
2551mismatch:
2552 spin_unlock(&dentry->d_lock);
2553 dput(dentry);
2554 goto retry;
2555}
2556EXPORT_SYMBOL(d_alloc_parallel);
2557
2558void __d_lookup_done(struct dentry *dentry)
2559{
2560 struct hlist_bl_head *b = in_lookup_hash(dentry->d_parent,
2561 dentry->d_name.hash);
2562 hlist_bl_lock(b);
2563 dentry->d_flags &= ~DCACHE_PAR_LOOKUP;
2564 __hlist_bl_del(&dentry->d_u.d_in_lookup_hash);
2565 wake_up_all(dentry->d_wait);
2566 dentry->d_wait = NULL;
2567 hlist_bl_unlock(b);
2568 INIT_HLIST_NODE(&dentry->d_u.d_alias);
2569 INIT_LIST_HEAD(&dentry->d_lru);
2570}
2571EXPORT_SYMBOL(__d_lookup_done);
2572
2573
2574
2575static inline void __d_add(struct dentry *dentry, struct inode *inode)
2576{
2577 struct inode *dir = NULL;
2578 unsigned n;
2579 spin_lock(&dentry->d_lock);
2580 if (unlikely(d_in_lookup(dentry))) {
2581 dir = dentry->d_parent->d_inode;
2582 n = start_dir_add(dir);
2583 __d_lookup_done(dentry);
2584 }
2585 if (inode) {
2586 unsigned add_flags = d_flags_for_inode(inode);
2587 hlist_add_head(&dentry->d_u.d_alias, &inode->i_dentry);
2588 raw_write_seqcount_begin(&dentry->d_seq);
2589 __d_set_inode_and_type(dentry, inode, add_flags);
2590 raw_write_seqcount_end(&dentry->d_seq);
2591 fsnotify_update_flags(dentry);
2592 }
2593 __d_rehash(dentry);
2594 if (dir)
2595 end_dir_add(dir, n);
2596 spin_unlock(&dentry->d_lock);
2597 if (inode)
2598 spin_unlock(&inode->i_lock);
2599}
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610void d_add(struct dentry *entry, struct inode *inode)
2611{
2612 if (inode) {
2613 security_d_instantiate(entry, inode);
2614 spin_lock(&inode->i_lock);
2615 }
2616 __d_add(entry, inode);
2617}
2618EXPORT_SYMBOL(d_add);
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631struct dentry *d_exact_alias(struct dentry *entry, struct inode *inode)
2632{
2633 struct dentry *alias;
2634 unsigned int hash = entry->d_name.hash;
2635
2636 spin_lock(&inode->i_lock);
2637 hlist_for_each_entry(alias, &inode->i_dentry, d_u.d_alias) {
2638
2639
2640
2641
2642
2643 if (alias->d_name.hash != hash)
2644 continue;
2645 if (alias->d_parent != entry->d_parent)
2646 continue;
2647 if (!d_same_name(alias, entry->d_parent, &entry->d_name))
2648 continue;
2649 spin_lock(&alias->d_lock);
2650 if (!d_unhashed(alias)) {
2651 spin_unlock(&alias->d_lock);
2652 alias = NULL;
2653 } else {
2654 __dget_dlock(alias);
2655 __d_rehash(alias);
2656 spin_unlock(&alias->d_lock);
2657 }
2658 spin_unlock(&inode->i_lock);
2659 return alias;
2660 }
2661 spin_unlock(&inode->i_lock);
2662 return NULL;
2663}
2664EXPORT_SYMBOL(d_exact_alias);
2665
2666static void swap_names(struct dentry *dentry, struct dentry *target)
2667{
2668 if (unlikely(dname_external(target))) {
2669 if (unlikely(dname_external(dentry))) {
2670
2671
2672
2673 swap(target->d_name.name, dentry->d_name.name);
2674 } else {
2675
2676
2677
2678
2679 memcpy(target->d_iname, dentry->d_name.name,
2680 dentry->d_name.len + 1);
2681 dentry->d_name.name = target->d_name.name;
2682 target->d_name.name = target->d_iname;
2683 }
2684 } else {
2685 if (unlikely(dname_external(dentry))) {
2686
2687
2688
2689
2690 memcpy(dentry->d_iname, target->d_name.name,
2691 target->d_name.len + 1);
2692 target->d_name.name = dentry->d_name.name;
2693 dentry->d_name.name = dentry->d_iname;
2694 } else {
2695
2696
2697
2698 unsigned int i;
2699 BUILD_BUG_ON(!IS_ALIGNED(DNAME_INLINE_LEN, sizeof(long)));
2700 for (i = 0; i < DNAME_INLINE_LEN / sizeof(long); i++) {
2701 swap(((long *) &dentry->d_iname)[i],
2702 ((long *) &target->d_iname)[i]);
2703 }
2704 }
2705 }
2706 swap(dentry->d_name.hash_len, target->d_name.hash_len);
2707}
2708
2709static void copy_name(struct dentry *dentry, struct dentry *target)
2710{
2711 struct external_name *old_name = NULL;
2712 if (unlikely(dname_external(dentry)))
2713 old_name = external_name(dentry);
2714 if (unlikely(dname_external(target))) {
2715 atomic_inc(&external_name(target)->u.count);
2716 dentry->d_name = target->d_name;
2717 } else {
2718 memcpy(dentry->d_iname, target->d_name.name,
2719 target->d_name.len + 1);
2720 dentry->d_name.name = dentry->d_iname;
2721 dentry->d_name.hash_len = target->d_name.hash_len;
2722 }
2723 if (old_name && likely(atomic_dec_and_test(&old_name->u.count)))
2724 kfree_rcu(old_name, u.head);
2725}
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738static void __d_move(struct dentry *dentry, struct dentry *target,
2739 bool exchange)
2740{
2741 struct dentry *old_parent, *p;
2742 struct inode *dir = NULL;
2743 unsigned n;
2744
2745 WARN_ON(!dentry->d_inode);
2746 if (WARN_ON(dentry == target))
2747 return;
2748
2749 BUG_ON(d_ancestor(target, dentry));
2750 old_parent = dentry->d_parent;
2751 p = d_ancestor(old_parent, target);
2752 if (IS_ROOT(dentry)) {
2753 BUG_ON(p);
2754 spin_lock(&target->d_parent->d_lock);
2755 } else if (!p) {
2756
2757 spin_lock(&target->d_parent->d_lock);
2758 spin_lock_nested(&old_parent->d_lock, DENTRY_D_LOCK_NESTED);
2759 } else {
2760 BUG_ON(p == dentry);
2761 spin_lock(&old_parent->d_lock);
2762 if (p != target)
2763 spin_lock_nested(&target->d_parent->d_lock,
2764 DENTRY_D_LOCK_NESTED);
2765 }
2766 spin_lock_nested(&dentry->d_lock, 2);
2767 spin_lock_nested(&target->d_lock, 3);
2768
2769 if (unlikely(d_in_lookup(target))) {
2770 dir = target->d_parent->d_inode;
2771 n = start_dir_add(dir);
2772 __d_lookup_done(target);
2773 }
2774
2775 write_seqcount_begin(&dentry->d_seq);
2776 write_seqcount_begin_nested(&target->d_seq, DENTRY_D_LOCK_NESTED);
2777
2778
2779 if (!d_unhashed(dentry))
2780 ___d_drop(dentry);
2781 if (!d_unhashed(target))
2782 ___d_drop(target);
2783
2784
2785 dentry->d_parent = target->d_parent;
2786 if (!exchange) {
2787 copy_name(dentry, target);
2788 target->d_hash.pprev = NULL;
2789 dentry->d_parent->d_lockref.count++;
2790 if (dentry != old_parent)
2791 WARN_ON(!--old_parent->d_lockref.count);
2792 } else {
2793 target->d_parent = old_parent;
2794 swap_names(dentry, target);
2795 list_move(&target->d_child, &target->d_parent->d_subdirs);
2796 __d_rehash(target);
2797 fsnotify_update_flags(target);
2798 }
2799 list_move(&dentry->d_child, &dentry->d_parent->d_subdirs);
2800 __d_rehash(dentry);
2801 fsnotify_update_flags(dentry);
2802 fscrypt_handle_d_move(dentry);
2803
2804 write_seqcount_end(&target->d_seq);
2805 write_seqcount_end(&dentry->d_seq);
2806
2807 if (dir)
2808 end_dir_add(dir, n);
2809
2810 if (dentry->d_parent != old_parent)
2811 spin_unlock(&dentry->d_parent->d_lock);
2812 if (dentry != old_parent)
2813 spin_unlock(&old_parent->d_lock);
2814 spin_unlock(&target->d_lock);
2815 spin_unlock(&dentry->d_lock);
2816}
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827void d_move(struct dentry *dentry, struct dentry *target)
2828{
2829 write_seqlock(&rename_lock);
2830 __d_move(dentry, target, false);
2831 write_sequnlock(&rename_lock);
2832}
2833EXPORT_SYMBOL(d_move);
2834
2835
2836
2837
2838
2839
2840void d_exchange(struct dentry *dentry1, struct dentry *dentry2)
2841{
2842 write_seqlock(&rename_lock);
2843
2844 WARN_ON(!dentry1->d_inode);
2845 WARN_ON(!dentry2->d_inode);
2846 WARN_ON(IS_ROOT(dentry1));
2847 WARN_ON(IS_ROOT(dentry2));
2848
2849 __d_move(dentry1, dentry2, true);
2850
2851 write_sequnlock(&rename_lock);
2852}
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862struct dentry *d_ancestor(struct dentry *p1, struct dentry *p2)
2863{
2864 struct dentry *p;
2865
2866 for (p = p2; !IS_ROOT(p); p = p->d_parent) {
2867 if (p->d_parent == p1)
2868 return p;
2869 }
2870 return NULL;
2871}
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882static int __d_unalias(struct inode *inode,
2883 struct dentry *dentry, struct dentry *alias)
2884{
2885 struct mutex *m1 = NULL;
2886 struct rw_semaphore *m2 = NULL;
2887 int ret = -ESTALE;
2888
2889
2890 if (alias->d_parent == dentry->d_parent)
2891 goto out_unalias;
2892
2893
2894 if (!mutex_trylock(&dentry->d_sb->s_vfs_rename_mutex))
2895 goto out_err;
2896 m1 = &dentry->d_sb->s_vfs_rename_mutex;
2897 if (!inode_trylock_shared(alias->d_parent->d_inode))
2898 goto out_err;
2899 m2 = &alias->d_parent->d_inode->i_rwsem;
2900out_unalias:
2901 __d_move(alias, dentry, false);
2902 ret = 0;
2903out_err:
2904 if (m2)
2905 up_read(m2);
2906 if (m1)
2907 mutex_unlock(m1);
2908 return ret;
2909}
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934struct dentry *d_splice_alias(struct inode *inode, struct dentry *dentry)
2935{
2936 if (IS_ERR(inode))
2937 return ERR_CAST(inode);
2938
2939 BUG_ON(!d_unhashed(dentry));
2940
2941 if (!inode)
2942 goto out;
2943
2944 security_d_instantiate(dentry, inode);
2945 spin_lock(&inode->i_lock);
2946 if (S_ISDIR(inode->i_mode)) {
2947 struct dentry *new = __d_find_any_alias(inode);
2948 if (unlikely(new)) {
2949
2950 spin_unlock(&inode->i_lock);
2951 write_seqlock(&rename_lock);
2952 if (unlikely(d_ancestor(new, dentry))) {
2953 write_sequnlock(&rename_lock);
2954 dput(new);
2955 new = ERR_PTR(-ELOOP);
2956 pr_warn_ratelimited(
2957 "VFS: Lookup of '%s' in %s %s"
2958 " would have caused loop\n",
2959 dentry->d_name.name,
2960 inode->i_sb->s_type->name,
2961 inode->i_sb->s_id);
2962 } else if (!IS_ROOT(new)) {
2963 struct dentry *old_parent = dget(new->d_parent);
2964 int err = __d_unalias(inode, dentry, new);
2965 write_sequnlock(&rename_lock);
2966 if (err) {
2967 dput(new);
2968 new = ERR_PTR(err);
2969 }
2970 dput(old_parent);
2971 } else {
2972 __d_move(new, dentry, false);
2973 write_sequnlock(&rename_lock);
2974 }
2975 iput(inode);
2976 return new;
2977 }
2978 }
2979out:
2980 __d_add(dentry, inode);
2981 return NULL;
2982}
2983EXPORT_SYMBOL(d_splice_alias);
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001bool is_subdir(struct dentry *new_dentry, struct dentry *old_dentry)
3002{
3003 bool result;
3004 unsigned seq;
3005
3006 if (new_dentry == old_dentry)
3007 return true;
3008
3009 do {
3010
3011 seq = read_seqbegin(&rename_lock);
3012
3013
3014
3015
3016 rcu_read_lock();
3017 if (d_ancestor(old_dentry, new_dentry))
3018 result = true;
3019 else
3020 result = false;
3021 rcu_read_unlock();
3022 } while (read_seqretry(&rename_lock, seq));
3023
3024 return result;
3025}
3026EXPORT_SYMBOL(is_subdir);
3027
3028static enum d_walk_ret d_genocide_kill(void *data, struct dentry *dentry)
3029{
3030 struct dentry *root = data;
3031 if (dentry != root) {
3032 if (d_unhashed(dentry) || !dentry->d_inode)
3033 return D_WALK_SKIP;
3034
3035 if (!(dentry->d_flags & DCACHE_GENOCIDE)) {
3036 dentry->d_flags |= DCACHE_GENOCIDE;
3037 dentry->d_lockref.count--;
3038 }
3039 }
3040 return D_WALK_CONTINUE;
3041}
3042
3043void d_genocide(struct dentry *parent)
3044{
3045 d_walk(parent, parent, d_genocide_kill);
3046}
3047
3048EXPORT_SYMBOL(d_genocide);
3049
3050void d_tmpfile(struct dentry *dentry, struct inode *inode)
3051{
3052 inode_dec_link_count(inode);
3053 BUG_ON(dentry->d_name.name != dentry->d_iname ||
3054 !hlist_unhashed(&dentry->d_u.d_alias) ||
3055 !d_unlinked(dentry));
3056 spin_lock(&dentry->d_parent->d_lock);
3057 spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED);
3058 dentry->d_name.len = sprintf(dentry->d_iname, "#%llu",
3059 (unsigned long long)inode->i_ino);
3060 spin_unlock(&dentry->d_lock);
3061 spin_unlock(&dentry->d_parent->d_lock);
3062 d_instantiate(dentry, inode);
3063}
3064EXPORT_SYMBOL(d_tmpfile);
3065
3066static __initdata unsigned long dhash_entries;
3067static int __init set_dhash_entries(char *str)
3068{
3069 if (!str)
3070 return 0;
3071 dhash_entries = simple_strtoul(str, &str, 0);
3072 return 1;
3073}
3074__setup("dhash_entries=", set_dhash_entries);
3075
3076static void __init dcache_init_early(void)
3077{
3078
3079
3080
3081 if (hashdist)
3082 return;
3083
3084 dentry_hashtable =
3085 alloc_large_system_hash("Dentry cache",
3086 sizeof(struct hlist_bl_head),
3087 dhash_entries,
3088 13,
3089 HASH_EARLY | HASH_ZERO,
3090 &d_hash_shift,
3091 NULL,
3092 0,
3093 0);
3094 d_hash_shift = 32 - d_hash_shift;
3095}
3096
3097static void __init dcache_init(void)
3098{
3099
3100
3101
3102
3103
3104 dentry_cache = KMEM_CACHE_USERCOPY(dentry,
3105 SLAB_RECLAIM_ACCOUNT|SLAB_PANIC|SLAB_MEM_SPREAD|SLAB_ACCOUNT,
3106 d_iname);
3107
3108
3109 if (!hashdist)
3110 return;
3111
3112 dentry_hashtable =
3113 alloc_large_system_hash("Dentry cache",
3114 sizeof(struct hlist_bl_head),
3115 dhash_entries,
3116 13,
3117 HASH_ZERO,
3118 &d_hash_shift,
3119 NULL,
3120 0,
3121 0);
3122 d_hash_shift = 32 - d_hash_shift;
3123}
3124
3125
3126struct kmem_cache *names_cachep __read_mostly;
3127EXPORT_SYMBOL(names_cachep);
3128
3129void __init vfs_caches_init_early(void)
3130{
3131 int i;
3132
3133 for (i = 0; i < ARRAY_SIZE(in_lookup_hashtable); i++)
3134 INIT_HLIST_BL_HEAD(&in_lookup_hashtable[i]);
3135
3136 dcache_init_early();
3137 inode_init_early();
3138}
3139
3140void __init vfs_caches_init(void)
3141{
3142 names_cachep = kmem_cache_create_usercopy("names_cache", PATH_MAX, 0,
3143 SLAB_HWCACHE_ALIGN|SLAB_PANIC, 0, PATH_MAX, NULL);
3144
3145 dcache_init();
3146 inode_init();
3147 files_init();
3148 files_maxfiles_init();
3149 mnt_init();
3150 bdev_cache_init();
3151 chrdev_init();
3152}
3153