1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18#include <linux/ratelimit.h>
19#include <linux/string.h>
20#include <linux/mm.h>
21#include <linux/fs.h>
22#include <linux/fscrypt.h>
23#include <linux/fsnotify.h>
24#include <linux/slab.h>
25#include <linux/init.h>
26#include <linux/hash.h>
27#include <linux/cache.h>
28#include <linux/export.h>
29#include <linux/security.h>
30#include <linux/seqlock.h>
31#include <linux/memblock.h>
32#include <linux/bit_spinlock.h>
33#include <linux/rculist_bl.h>
34#include <linux/list_lru.h>
35#include "internal.h"
36#include "mount.h"
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74int sysctl_vfs_cache_pressure __read_mostly = 100;
75EXPORT_SYMBOL_GPL(sysctl_vfs_cache_pressure);
76
77__cacheline_aligned_in_smp DEFINE_SEQLOCK(rename_lock);
78
79EXPORT_SYMBOL(rename_lock);
80
81static struct kmem_cache *dentry_cache __read_mostly;
82
83const struct qstr empty_name = QSTR_INIT("", 0);
84EXPORT_SYMBOL(empty_name);
85const struct qstr slash_name = QSTR_INIT("/", 1);
86EXPORT_SYMBOL(slash_name);
87const struct qstr dotdot_name = QSTR_INIT("..", 2);
88EXPORT_SYMBOL(dotdot_name);
89
90
91
92
93
94
95
96
97
98
99static unsigned int d_hash_shift __read_mostly;
100
101static struct hlist_bl_head *dentry_hashtable __read_mostly;
102
103static inline struct hlist_bl_head *d_hash(unsigned int hash)
104{
105 return dentry_hashtable + (hash >> d_hash_shift);
106}
107
108#define IN_LOOKUP_SHIFT 10
109static struct hlist_bl_head in_lookup_hashtable[1 << IN_LOOKUP_SHIFT];
110
111static inline struct hlist_bl_head *in_lookup_hash(const struct dentry *parent,
112 unsigned int hash)
113{
114 hash += (unsigned long) parent / L1_CACHE_BYTES;
115 return in_lookup_hashtable + hash_32(hash, IN_LOOKUP_SHIFT);
116}
117
118
119
120struct dentry_stat_t dentry_stat = {
121 .age_limit = 45,
122};
123
124static DEFINE_PER_CPU(long, nr_dentry);
125static DEFINE_PER_CPU(long, nr_dentry_unused);
126static DEFINE_PER_CPU(long, nr_dentry_negative);
127
128#if defined(CONFIG_SYSCTL) && defined(CONFIG_PROC_FS)
129
130
131
132
133
134
135
136
137
138
139
140
141
142static long get_nr_dentry(void)
143{
144 int i;
145 long sum = 0;
146 for_each_possible_cpu(i)
147 sum += per_cpu(nr_dentry, i);
148 return sum < 0 ? 0 : sum;
149}
150
151static long get_nr_dentry_unused(void)
152{
153 int i;
154 long sum = 0;
155 for_each_possible_cpu(i)
156 sum += per_cpu(nr_dentry_unused, i);
157 return sum < 0 ? 0 : sum;
158}
159
160static long get_nr_dentry_negative(void)
161{
162 int i;
163 long sum = 0;
164
165 for_each_possible_cpu(i)
166 sum += per_cpu(nr_dentry_negative, i);
167 return sum < 0 ? 0 : sum;
168}
169
170int proc_nr_dentry(struct ctl_table *table, int write, void *buffer,
171 size_t *lenp, loff_t *ppos)
172{
173 dentry_stat.nr_dentry = get_nr_dentry();
174 dentry_stat.nr_unused = get_nr_dentry_unused();
175 dentry_stat.nr_negative = get_nr_dentry_negative();
176 return proc_doulongvec_minmax(table, write, buffer, lenp, ppos);
177}
178#endif
179
180
181
182
183
184#ifdef CONFIG_DCACHE_WORD_ACCESS
185
186#include <asm/word-at-a-time.h>
187
188
189
190
191
192
193
194
195
196static inline int dentry_string_cmp(const unsigned char *cs, const unsigned char *ct, unsigned tcount)
197{
198 unsigned long a,b,mask;
199
200 for (;;) {
201 a = read_word_at_a_time(cs);
202 b = load_unaligned_zeropad(ct);
203 if (tcount < sizeof(unsigned long))
204 break;
205 if (unlikely(a != b))
206 return 1;
207 cs += sizeof(unsigned long);
208 ct += sizeof(unsigned long);
209 tcount -= sizeof(unsigned long);
210 if (!tcount)
211 return 0;
212 }
213 mask = bytemask_from_count(tcount);
214 return unlikely(!!((a ^ b) & mask));
215}
216
217#else
218
219static inline int dentry_string_cmp(const unsigned char *cs, const unsigned char *ct, unsigned tcount)
220{
221 do {
222 if (*cs != *ct)
223 return 1;
224 cs++;
225 ct++;
226 tcount--;
227 } while (tcount);
228 return 0;
229}
230
231#endif
232
233static inline int dentry_cmp(const struct dentry *dentry, const unsigned char *ct, unsigned tcount)
234{
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251 const unsigned char *cs = READ_ONCE(dentry->d_name.name);
252
253 return dentry_string_cmp(cs, ct, tcount);
254}
255
256struct external_name {
257 union {
258 atomic_t count;
259 struct rcu_head head;
260 } u;
261 unsigned char name[];
262};
263
264static inline struct external_name *external_name(struct dentry *dentry)
265{
266 return container_of(dentry->d_name.name, struct external_name, name[0]);
267}
268
269static void __d_free(struct rcu_head *head)
270{
271 struct dentry *dentry = container_of(head, struct dentry, d_u.d_rcu);
272
273 kmem_cache_free(dentry_cache, dentry);
274}
275
276static void __d_free_external(struct rcu_head *head)
277{
278 struct dentry *dentry = container_of(head, struct dentry, d_u.d_rcu);
279 kfree(external_name(dentry));
280 kmem_cache_free(dentry_cache, dentry);
281}
282
283static inline int dname_external(const struct dentry *dentry)
284{
285 return dentry->d_name.name != dentry->d_iname;
286}
287
288void take_dentry_name_snapshot(struct name_snapshot *name, struct dentry *dentry)
289{
290 spin_lock(&dentry->d_lock);
291 name->name = dentry->d_name;
292 if (unlikely(dname_external(dentry))) {
293 atomic_inc(&external_name(dentry)->u.count);
294 } else {
295 memcpy(name->inline_name, dentry->d_iname,
296 dentry->d_name.len + 1);
297 name->name.name = name->inline_name;
298 }
299 spin_unlock(&dentry->d_lock);
300}
301EXPORT_SYMBOL(take_dentry_name_snapshot);
302
303void release_dentry_name_snapshot(struct name_snapshot *name)
304{
305 if (unlikely(name->name.name != name->inline_name)) {
306 struct external_name *p;
307 p = container_of(name->name.name, struct external_name, name[0]);
308 if (unlikely(atomic_dec_and_test(&p->u.count)))
309 kfree_rcu(p, u.head);
310 }
311}
312EXPORT_SYMBOL(release_dentry_name_snapshot);
313
314static inline void __d_set_inode_and_type(struct dentry *dentry,
315 struct inode *inode,
316 unsigned type_flags)
317{
318 unsigned flags;
319
320 dentry->d_inode = inode;
321 flags = READ_ONCE(dentry->d_flags);
322 flags &= ~(DCACHE_ENTRY_TYPE | DCACHE_FALLTHRU);
323 flags |= type_flags;
324 smp_store_release(&dentry->d_flags, flags);
325}
326
327static inline void __d_clear_type_and_inode(struct dentry *dentry)
328{
329 unsigned flags = READ_ONCE(dentry->d_flags);
330
331 flags &= ~(DCACHE_ENTRY_TYPE | DCACHE_FALLTHRU);
332 WRITE_ONCE(dentry->d_flags, flags);
333 dentry->d_inode = NULL;
334 if (dentry->d_flags & DCACHE_LRU_LIST)
335 this_cpu_inc(nr_dentry_negative);
336}
337
338static void dentry_free(struct dentry *dentry)
339{
340 WARN_ON(!hlist_unhashed(&dentry->d_u.d_alias));
341 if (unlikely(dname_external(dentry))) {
342 struct external_name *p = external_name(dentry);
343 if (likely(atomic_dec_and_test(&p->u.count))) {
344 call_rcu(&dentry->d_u.d_rcu, __d_free_external);
345 return;
346 }
347 }
348
349 if (dentry->d_flags & DCACHE_NORCU)
350 __d_free(&dentry->d_u.d_rcu);
351 else
352 call_rcu(&dentry->d_u.d_rcu, __d_free);
353}
354
355
356
357
358
359static void dentry_unlink_inode(struct dentry * dentry)
360 __releases(dentry->d_lock)
361 __releases(dentry->d_inode->i_lock)
362{
363 struct inode *inode = dentry->d_inode;
364
365 raw_write_seqcount_begin(&dentry->d_seq);
366 __d_clear_type_and_inode(dentry);
367 hlist_del_init(&dentry->d_u.d_alias);
368 raw_write_seqcount_end(&dentry->d_seq);
369 spin_unlock(&dentry->d_lock);
370 spin_unlock(&inode->i_lock);
371 if (!inode->i_nlink)
372 fsnotify_inoderemove(inode);
373 if (dentry->d_op && dentry->d_op->d_iput)
374 dentry->d_op->d_iput(dentry, inode);
375 else
376 iput(inode);
377}
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398#define D_FLAG_VERIFY(dentry,x) WARN_ON_ONCE(((dentry)->d_flags & (DCACHE_LRU_LIST | DCACHE_SHRINK_LIST)) != (x))
399static void d_lru_add(struct dentry *dentry)
400{
401 D_FLAG_VERIFY(dentry, 0);
402 dentry->d_flags |= DCACHE_LRU_LIST;
403 this_cpu_inc(nr_dentry_unused);
404 if (d_is_negative(dentry))
405 this_cpu_inc(nr_dentry_negative);
406 WARN_ON_ONCE(!list_lru_add(&dentry->d_sb->s_dentry_lru, &dentry->d_lru));
407}
408
409static void d_lru_del(struct dentry *dentry)
410{
411 D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST);
412 dentry->d_flags &= ~DCACHE_LRU_LIST;
413 this_cpu_dec(nr_dentry_unused);
414 if (d_is_negative(dentry))
415 this_cpu_dec(nr_dentry_negative);
416 WARN_ON_ONCE(!list_lru_del(&dentry->d_sb->s_dentry_lru, &dentry->d_lru));
417}
418
419static void d_shrink_del(struct dentry *dentry)
420{
421 D_FLAG_VERIFY(dentry, DCACHE_SHRINK_LIST | DCACHE_LRU_LIST);
422 list_del_init(&dentry->d_lru);
423 dentry->d_flags &= ~(DCACHE_SHRINK_LIST | DCACHE_LRU_LIST);
424 this_cpu_dec(nr_dentry_unused);
425}
426
427static void d_shrink_add(struct dentry *dentry, struct list_head *list)
428{
429 D_FLAG_VERIFY(dentry, 0);
430 list_add(&dentry->d_lru, list);
431 dentry->d_flags |= DCACHE_SHRINK_LIST | DCACHE_LRU_LIST;
432 this_cpu_inc(nr_dentry_unused);
433}
434
435
436
437
438
439
440
441static void d_lru_isolate(struct list_lru_one *lru, struct dentry *dentry)
442{
443 D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST);
444 dentry->d_flags &= ~DCACHE_LRU_LIST;
445 this_cpu_dec(nr_dentry_unused);
446 if (d_is_negative(dentry))
447 this_cpu_dec(nr_dentry_negative);
448 list_lru_isolate(lru, &dentry->d_lru);
449}
450
451static void d_lru_shrink_move(struct list_lru_one *lru, struct dentry *dentry,
452 struct list_head *list)
453{
454 D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST);
455 dentry->d_flags |= DCACHE_SHRINK_LIST;
456 if (d_is_negative(dentry))
457 this_cpu_dec(nr_dentry_negative);
458 list_lru_isolate_move(lru, &dentry->d_lru, list);
459}
460
461static void ___d_drop(struct dentry *dentry)
462{
463 struct hlist_bl_head *b;
464
465
466
467
468
469 if (unlikely(IS_ROOT(dentry)))
470 b = &dentry->d_sb->s_roots;
471 else
472 b = d_hash(dentry->d_name.hash);
473
474 hlist_bl_lock(b);
475 __hlist_bl_del(&dentry->d_hash);
476 hlist_bl_unlock(b);
477}
478
479void __d_drop(struct dentry *dentry)
480{
481 if (!d_unhashed(dentry)) {
482 ___d_drop(dentry);
483 dentry->d_hash.pprev = NULL;
484 write_seqcount_invalidate(&dentry->d_seq);
485 }
486}
487EXPORT_SYMBOL(__d_drop);
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507void d_drop(struct dentry *dentry)
508{
509 spin_lock(&dentry->d_lock);
510 __d_drop(dentry);
511 spin_unlock(&dentry->d_lock);
512}
513EXPORT_SYMBOL(d_drop);
514
515static inline void dentry_unlist(struct dentry *dentry, struct dentry *parent)
516{
517 struct dentry *next;
518
519
520
521
522 dentry->d_flags |= DCACHE_DENTRY_KILLED;
523 if (unlikely(list_empty(&dentry->d_child)))
524 return;
525 __list_del_entry(&dentry->d_child);
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545 while (dentry->d_child.next != &parent->d_subdirs) {
546 next = list_entry(dentry->d_child.next, struct dentry, d_child);
547 if (likely(!(next->d_flags & DCACHE_DENTRY_CURSOR)))
548 break;
549 dentry->d_child.next = next->d_child.next;
550 }
551}
552
553static void __dentry_kill(struct dentry *dentry)
554{
555 struct dentry *parent = NULL;
556 bool can_free = true;
557 if (!IS_ROOT(dentry))
558 parent = dentry->d_parent;
559
560
561
562
563 lockref_mark_dead(&dentry->d_lockref);
564
565
566
567
568
569 if (dentry->d_flags & DCACHE_OP_PRUNE)
570 dentry->d_op->d_prune(dentry);
571
572 if (dentry->d_flags & DCACHE_LRU_LIST) {
573 if (!(dentry->d_flags & DCACHE_SHRINK_LIST))
574 d_lru_del(dentry);
575 }
576
577 __d_drop(dentry);
578 dentry_unlist(dentry, parent);
579 if (parent)
580 spin_unlock(&parent->d_lock);
581 if (dentry->d_inode)
582 dentry_unlink_inode(dentry);
583 else
584 spin_unlock(&dentry->d_lock);
585 this_cpu_dec(nr_dentry);
586 if (dentry->d_op && dentry->d_op->d_release)
587 dentry->d_op->d_release(dentry);
588
589 spin_lock(&dentry->d_lock);
590 if (dentry->d_flags & DCACHE_SHRINK_LIST) {
591 dentry->d_flags |= DCACHE_MAY_FREE;
592 can_free = false;
593 }
594 spin_unlock(&dentry->d_lock);
595 if (likely(can_free))
596 dentry_free(dentry);
597 cond_resched();
598}
599
600static struct dentry *__lock_parent(struct dentry *dentry)
601{
602 struct dentry *parent;
603 rcu_read_lock();
604 spin_unlock(&dentry->d_lock);
605again:
606 parent = READ_ONCE(dentry->d_parent);
607 spin_lock(&parent->d_lock);
608
609
610
611
612
613
614
615
616 if (unlikely(parent != dentry->d_parent)) {
617 spin_unlock(&parent->d_lock);
618 goto again;
619 }
620 rcu_read_unlock();
621 if (parent != dentry)
622 spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED);
623 else
624 parent = NULL;
625 return parent;
626}
627
628static inline struct dentry *lock_parent(struct dentry *dentry)
629{
630 struct dentry *parent = dentry->d_parent;
631 if (IS_ROOT(dentry))
632 return NULL;
633 if (likely(spin_trylock(&parent->d_lock)))
634 return parent;
635 return __lock_parent(dentry);
636}
637
638static inline bool retain_dentry(struct dentry *dentry)
639{
640 WARN_ON(d_in_lookup(dentry));
641
642
643 if (unlikely(d_unhashed(dentry)))
644 return false;
645
646 if (unlikely(dentry->d_flags & DCACHE_DISCONNECTED))
647 return false;
648
649 if (unlikely(dentry->d_flags & DCACHE_OP_DELETE)) {
650 if (dentry->d_op->d_delete(dentry))
651 return false;
652 }
653
654 if (unlikely(dentry->d_flags & DCACHE_DONTCACHE))
655 return false;
656
657
658 dentry->d_lockref.count--;
659 if (unlikely(!(dentry->d_flags & DCACHE_LRU_LIST)))
660 d_lru_add(dentry);
661 else if (unlikely(!(dentry->d_flags & DCACHE_REFERENCED)))
662 dentry->d_flags |= DCACHE_REFERENCED;
663 return true;
664}
665
666void d_mark_dontcache(struct inode *inode)
667{
668 struct dentry *de;
669
670 spin_lock(&inode->i_lock);
671 hlist_for_each_entry(de, &inode->i_dentry, d_u.d_alias) {
672 spin_lock(&de->d_lock);
673 de->d_flags |= DCACHE_DONTCACHE;
674 spin_unlock(&de->d_lock);
675 }
676 inode->i_state |= I_DONTCACHE;
677 spin_unlock(&inode->i_lock);
678}
679EXPORT_SYMBOL(d_mark_dontcache);
680
681
682
683
684
685
686static struct dentry *dentry_kill(struct dentry *dentry)
687 __releases(dentry->d_lock)
688{
689 struct inode *inode = dentry->d_inode;
690 struct dentry *parent = NULL;
691
692 if (inode && unlikely(!spin_trylock(&inode->i_lock)))
693 goto slow_positive;
694
695 if (!IS_ROOT(dentry)) {
696 parent = dentry->d_parent;
697 if (unlikely(!spin_trylock(&parent->d_lock))) {
698 parent = __lock_parent(dentry);
699 if (likely(inode || !dentry->d_inode))
700 goto got_locks;
701
702 if (parent)
703 spin_unlock(&parent->d_lock);
704 inode = dentry->d_inode;
705 goto slow_positive;
706 }
707 }
708 __dentry_kill(dentry);
709 return parent;
710
711slow_positive:
712 spin_unlock(&dentry->d_lock);
713 spin_lock(&inode->i_lock);
714 spin_lock(&dentry->d_lock);
715 parent = lock_parent(dentry);
716got_locks:
717 if (unlikely(dentry->d_lockref.count != 1)) {
718 dentry->d_lockref.count--;
719 } else if (likely(!retain_dentry(dentry))) {
720 __dentry_kill(dentry);
721 return parent;
722 }
723
724 if (inode)
725 spin_unlock(&inode->i_lock);
726 if (parent)
727 spin_unlock(&parent->d_lock);
728 spin_unlock(&dentry->d_lock);
729 return NULL;
730}
731
732
733
734
735
736
737
738
739
740static inline bool fast_dput(struct dentry *dentry)
741{
742 int ret;
743 unsigned int d_flags;
744
745
746
747
748
749 if (unlikely(dentry->d_flags & DCACHE_OP_DELETE))
750 return lockref_put_or_lock(&dentry->d_lockref);
751
752
753
754
755
756 ret = lockref_put_return(&dentry->d_lockref);
757
758
759
760
761
762
763 if (unlikely(ret < 0)) {
764 spin_lock(&dentry->d_lock);
765 if (dentry->d_lockref.count > 1) {
766 dentry->d_lockref.count--;
767 spin_unlock(&dentry->d_lock);
768 return true;
769 }
770 return false;
771 }
772
773
774
775
776 if (ret)
777 return true;
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806 smp_rmb();
807 d_flags = READ_ONCE(dentry->d_flags);
808 d_flags &= DCACHE_REFERENCED | DCACHE_LRU_LIST |
809 DCACHE_DISCONNECTED | DCACHE_DONTCACHE;
810
811
812 if (d_flags == (DCACHE_REFERENCED | DCACHE_LRU_LIST) && !d_unhashed(dentry))
813 return true;
814
815
816
817
818
819
820 spin_lock(&dentry->d_lock);
821
822
823
824
825
826
827
828 if (dentry->d_lockref.count) {
829 spin_unlock(&dentry->d_lock);
830 return true;
831 }
832
833
834
835
836
837
838 dentry->d_lockref.count = 1;
839 return false;
840}
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869void dput(struct dentry *dentry)
870{
871 while (dentry) {
872 might_sleep();
873
874 rcu_read_lock();
875 if (likely(fast_dput(dentry))) {
876 rcu_read_unlock();
877 return;
878 }
879
880
881 rcu_read_unlock();
882
883 if (likely(retain_dentry(dentry))) {
884 spin_unlock(&dentry->d_lock);
885 return;
886 }
887
888 dentry = dentry_kill(dentry);
889 }
890}
891EXPORT_SYMBOL(dput);
892
893static void __dput_to_list(struct dentry *dentry, struct list_head *list)
894__must_hold(&dentry->d_lock)
895{
896 if (dentry->d_flags & DCACHE_SHRINK_LIST) {
897
898 --dentry->d_lockref.count;
899 } else {
900 if (dentry->d_flags & DCACHE_LRU_LIST)
901 d_lru_del(dentry);
902 if (!--dentry->d_lockref.count)
903 d_shrink_add(dentry, list);
904 }
905}
906
907void dput_to_list(struct dentry *dentry, struct list_head *list)
908{
909 rcu_read_lock();
910 if (likely(fast_dput(dentry))) {
911 rcu_read_unlock();
912 return;
913 }
914 rcu_read_unlock();
915 if (!retain_dentry(dentry))
916 __dput_to_list(dentry, list);
917 spin_unlock(&dentry->d_lock);
918}
919
920
921static inline void __dget_dlock(struct dentry *dentry)
922{
923 dentry->d_lockref.count++;
924}
925
926static inline void __dget(struct dentry *dentry)
927{
928 lockref_get(&dentry->d_lockref);
929}
930
931struct dentry *dget_parent(struct dentry *dentry)
932{
933 int gotref;
934 struct dentry *ret;
935 unsigned seq;
936
937
938
939
940
941 rcu_read_lock();
942 seq = raw_seqcount_begin(&dentry->d_seq);
943 ret = READ_ONCE(dentry->d_parent);
944 gotref = lockref_get_not_zero(&ret->d_lockref);
945 rcu_read_unlock();
946 if (likely(gotref)) {
947 if (!read_seqcount_retry(&dentry->d_seq, seq))
948 return ret;
949 dput(ret);
950 }
951
952repeat:
953
954
955
956
957 rcu_read_lock();
958 ret = dentry->d_parent;
959 spin_lock(&ret->d_lock);
960 if (unlikely(ret != dentry->d_parent)) {
961 spin_unlock(&ret->d_lock);
962 rcu_read_unlock();
963 goto repeat;
964 }
965 rcu_read_unlock();
966 BUG_ON(!ret->d_lockref.count);
967 ret->d_lockref.count++;
968 spin_unlock(&ret->d_lock);
969 return ret;
970}
971EXPORT_SYMBOL(dget_parent);
972
973static struct dentry * __d_find_any_alias(struct inode *inode)
974{
975 struct dentry *alias;
976
977 if (hlist_empty(&inode->i_dentry))
978 return NULL;
979 alias = hlist_entry(inode->i_dentry.first, struct dentry, d_u.d_alias);
980 __dget(alias);
981 return alias;
982}
983
984
985
986
987
988
989
990
991struct dentry *d_find_any_alias(struct inode *inode)
992{
993 struct dentry *de;
994
995 spin_lock(&inode->i_lock);
996 de = __d_find_any_alias(inode);
997 spin_unlock(&inode->i_lock);
998 return de;
999}
1000EXPORT_SYMBOL(d_find_any_alias);
1001
1002static struct dentry *__d_find_alias(struct inode *inode)
1003{
1004 struct dentry *alias;
1005
1006 if (S_ISDIR(inode->i_mode))
1007 return __d_find_any_alias(inode);
1008
1009 hlist_for_each_entry(alias, &inode->i_dentry, d_u.d_alias) {
1010 spin_lock(&alias->d_lock);
1011 if (!d_unhashed(alias)) {
1012 __dget_dlock(alias);
1013 spin_unlock(&alias->d_lock);
1014 return alias;
1015 }
1016 spin_unlock(&alias->d_lock);
1017 }
1018 return NULL;
1019}
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035struct dentry *d_find_alias(struct inode *inode)
1036{
1037 struct dentry *de = NULL;
1038
1039 if (!hlist_empty(&inode->i_dentry)) {
1040 spin_lock(&inode->i_lock);
1041 de = __d_find_alias(inode);
1042 spin_unlock(&inode->i_lock);
1043 }
1044 return de;
1045}
1046EXPORT_SYMBOL(d_find_alias);
1047
1048
1049
1050
1051
1052struct dentry *d_find_alias_rcu(struct inode *inode)
1053{
1054 struct hlist_head *l = &inode->i_dentry;
1055 struct dentry *de = NULL;
1056
1057 spin_lock(&inode->i_lock);
1058
1059
1060 if (likely(!(inode->i_state & I_FREEING) && !hlist_empty(l))) {
1061 if (S_ISDIR(inode->i_mode)) {
1062 de = hlist_entry(l->first, struct dentry, d_u.d_alias);
1063 } else {
1064 hlist_for_each_entry(de, l, d_u.d_alias)
1065 if (!d_unhashed(de))
1066 break;
1067 }
1068 }
1069 spin_unlock(&inode->i_lock);
1070 return de;
1071}
1072
1073
1074
1075
1076
1077void d_prune_aliases(struct inode *inode)
1078{
1079 struct dentry *dentry;
1080restart:
1081 spin_lock(&inode->i_lock);
1082 hlist_for_each_entry(dentry, &inode->i_dentry, d_u.d_alias) {
1083 spin_lock(&dentry->d_lock);
1084 if (!dentry->d_lockref.count) {
1085 struct dentry *parent = lock_parent(dentry);
1086 if (likely(!dentry->d_lockref.count)) {
1087 __dentry_kill(dentry);
1088 dput(parent);
1089 goto restart;
1090 }
1091 if (parent)
1092 spin_unlock(&parent->d_lock);
1093 }
1094 spin_unlock(&dentry->d_lock);
1095 }
1096 spin_unlock(&inode->i_lock);
1097}
1098EXPORT_SYMBOL(d_prune_aliases);
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111static bool shrink_lock_dentry(struct dentry *dentry)
1112{
1113 struct inode *inode;
1114 struct dentry *parent;
1115
1116 if (dentry->d_lockref.count)
1117 return false;
1118
1119 inode = dentry->d_inode;
1120 if (inode && unlikely(!spin_trylock(&inode->i_lock))) {
1121 spin_unlock(&dentry->d_lock);
1122 spin_lock(&inode->i_lock);
1123 spin_lock(&dentry->d_lock);
1124 if (unlikely(dentry->d_lockref.count))
1125 goto out;
1126
1127 if (unlikely(inode != dentry->d_inode))
1128 goto out;
1129 }
1130
1131 parent = dentry->d_parent;
1132 if (IS_ROOT(dentry) || likely(spin_trylock(&parent->d_lock)))
1133 return true;
1134
1135 spin_unlock(&dentry->d_lock);
1136 spin_lock(&parent->d_lock);
1137 if (unlikely(parent != dentry->d_parent)) {
1138 spin_unlock(&parent->d_lock);
1139 spin_lock(&dentry->d_lock);
1140 goto out;
1141 }
1142 spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED);
1143 if (likely(!dentry->d_lockref.count))
1144 return true;
1145 spin_unlock(&parent->d_lock);
1146out:
1147 if (inode)
1148 spin_unlock(&inode->i_lock);
1149 return false;
1150}
1151
1152void shrink_dentry_list(struct list_head *list)
1153{
1154 while (!list_empty(list)) {
1155 struct dentry *dentry, *parent;
1156
1157 dentry = list_entry(list->prev, struct dentry, d_lru);
1158 spin_lock(&dentry->d_lock);
1159 rcu_read_lock();
1160 if (!shrink_lock_dentry(dentry)) {
1161 bool can_free = false;
1162 rcu_read_unlock();
1163 d_shrink_del(dentry);
1164 if (dentry->d_lockref.count < 0)
1165 can_free = dentry->d_flags & DCACHE_MAY_FREE;
1166 spin_unlock(&dentry->d_lock);
1167 if (can_free)
1168 dentry_free(dentry);
1169 continue;
1170 }
1171 rcu_read_unlock();
1172 d_shrink_del(dentry);
1173 parent = dentry->d_parent;
1174 if (parent != dentry)
1175 __dput_to_list(parent, list);
1176 __dentry_kill(dentry);
1177 }
1178}
1179
1180static enum lru_status dentry_lru_isolate(struct list_head *item,
1181 struct list_lru_one *lru, spinlock_t *lru_lock, void *arg)
1182{
1183 struct list_head *freeable = arg;
1184 struct dentry *dentry = container_of(item, struct dentry, d_lru);
1185
1186
1187
1188
1189
1190
1191
1192 if (!spin_trylock(&dentry->d_lock))
1193 return LRU_SKIP;
1194
1195
1196
1197
1198
1199
1200 if (dentry->d_lockref.count) {
1201 d_lru_isolate(lru, dentry);
1202 spin_unlock(&dentry->d_lock);
1203 return LRU_REMOVED;
1204 }
1205
1206 if (dentry->d_flags & DCACHE_REFERENCED) {
1207 dentry->d_flags &= ~DCACHE_REFERENCED;
1208 spin_unlock(&dentry->d_lock);
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229 return LRU_ROTATE;
1230 }
1231
1232 d_lru_shrink_move(lru, dentry, freeable);
1233 spin_unlock(&dentry->d_lock);
1234
1235 return LRU_REMOVED;
1236}
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250long prune_dcache_sb(struct super_block *sb, struct shrink_control *sc)
1251{
1252 LIST_HEAD(dispose);
1253 long freed;
1254
1255 freed = list_lru_shrink_walk(&sb->s_dentry_lru, sc,
1256 dentry_lru_isolate, &dispose);
1257 shrink_dentry_list(&dispose);
1258 return freed;
1259}
1260
1261static enum lru_status dentry_lru_isolate_shrink(struct list_head *item,
1262 struct list_lru_one *lru, spinlock_t *lru_lock, void *arg)
1263{
1264 struct list_head *freeable = arg;
1265 struct dentry *dentry = container_of(item, struct dentry, d_lru);
1266
1267
1268
1269
1270
1271
1272 if (!spin_trylock(&dentry->d_lock))
1273 return LRU_SKIP;
1274
1275 d_lru_shrink_move(lru, dentry, freeable);
1276 spin_unlock(&dentry->d_lock);
1277
1278 return LRU_REMOVED;
1279}
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289void shrink_dcache_sb(struct super_block *sb)
1290{
1291 do {
1292 LIST_HEAD(dispose);
1293
1294 list_lru_walk(&sb->s_dentry_lru,
1295 dentry_lru_isolate_shrink, &dispose, 1024);
1296 shrink_dentry_list(&dispose);
1297 } while (list_lru_count(&sb->s_dentry_lru) > 0);
1298}
1299EXPORT_SYMBOL(shrink_dcache_sb);
1300
1301
1302
1303
1304
1305
1306
1307
1308enum d_walk_ret {
1309 D_WALK_CONTINUE,
1310 D_WALK_QUIT,
1311 D_WALK_NORETRY,
1312 D_WALK_SKIP,
1313};
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323static void d_walk(struct dentry *parent, void *data,
1324 enum d_walk_ret (*enter)(void *, struct dentry *))
1325{
1326 struct dentry *this_parent;
1327 struct list_head *next;
1328 unsigned seq = 0;
1329 enum d_walk_ret ret;
1330 bool retry = true;
1331
1332again:
1333 read_seqbegin_or_lock(&rename_lock, &seq);
1334 this_parent = parent;
1335 spin_lock(&this_parent->d_lock);
1336
1337 ret = enter(data, this_parent);
1338 switch (ret) {
1339 case D_WALK_CONTINUE:
1340 break;
1341 case D_WALK_QUIT:
1342 case D_WALK_SKIP:
1343 goto out_unlock;
1344 case D_WALK_NORETRY:
1345 retry = false;
1346 break;
1347 }
1348repeat:
1349 next = this_parent->d_subdirs.next;
1350resume:
1351 while (next != &this_parent->d_subdirs) {
1352 struct list_head *tmp = next;
1353 struct dentry *dentry = list_entry(tmp, struct dentry, d_child);
1354 next = tmp->next;
1355
1356 if (unlikely(dentry->d_flags & DCACHE_DENTRY_CURSOR))
1357 continue;
1358
1359 spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED);
1360
1361 ret = enter(data, dentry);
1362 switch (ret) {
1363 case D_WALK_CONTINUE:
1364 break;
1365 case D_WALK_QUIT:
1366 spin_unlock(&dentry->d_lock);
1367 goto out_unlock;
1368 case D_WALK_NORETRY:
1369 retry = false;
1370 break;
1371 case D_WALK_SKIP:
1372 spin_unlock(&dentry->d_lock);
1373 continue;
1374 }
1375
1376 if (!list_empty(&dentry->d_subdirs)) {
1377 spin_unlock(&this_parent->d_lock);
1378 spin_release(&dentry->d_lock.dep_map, _RET_IP_);
1379 this_parent = dentry;
1380 spin_acquire(&this_parent->d_lock.dep_map, 0, 1, _RET_IP_);
1381 goto repeat;
1382 }
1383 spin_unlock(&dentry->d_lock);
1384 }
1385
1386
1387
1388 rcu_read_lock();
1389ascend:
1390 if (this_parent != parent) {
1391 struct dentry *child = this_parent;
1392 this_parent = child->d_parent;
1393
1394 spin_unlock(&child->d_lock);
1395 spin_lock(&this_parent->d_lock);
1396
1397
1398 if (need_seqretry(&rename_lock, seq))
1399 goto rename_retry;
1400
1401 do {
1402 next = child->d_child.next;
1403 if (next == &this_parent->d_subdirs)
1404 goto ascend;
1405 child = list_entry(next, struct dentry, d_child);
1406 } while (unlikely(child->d_flags & DCACHE_DENTRY_KILLED));
1407 rcu_read_unlock();
1408 goto resume;
1409 }
1410 if (need_seqretry(&rename_lock, seq))
1411 goto rename_retry;
1412 rcu_read_unlock();
1413
1414out_unlock:
1415 spin_unlock(&this_parent->d_lock);
1416 done_seqretry(&rename_lock, seq);
1417 return;
1418
1419rename_retry:
1420 spin_unlock(&this_parent->d_lock);
1421 rcu_read_unlock();
1422 BUG_ON(seq & 1);
1423 if (!retry)
1424 return;
1425 seq = 1;
1426 goto again;
1427}
1428
1429struct check_mount {
1430 struct vfsmount *mnt;
1431 unsigned int mounted;
1432};
1433
1434static enum d_walk_ret path_check_mount(void *data, struct dentry *dentry)
1435{
1436 struct check_mount *info = data;
1437 struct path path = { .mnt = info->mnt, .dentry = dentry };
1438
1439 if (likely(!d_mountpoint(dentry)))
1440 return D_WALK_CONTINUE;
1441 if (__path_is_mountpoint(&path)) {
1442 info->mounted = 1;
1443 return D_WALK_QUIT;
1444 }
1445 return D_WALK_CONTINUE;
1446}
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456int path_has_submounts(const struct path *parent)
1457{
1458 struct check_mount data = { .mnt = parent->mnt, .mounted = 0 };
1459
1460 read_seqlock_excl(&mount_lock);
1461 d_walk(parent->dentry, &data, path_check_mount);
1462 read_sequnlock_excl(&mount_lock);
1463
1464 return data.mounted;
1465}
1466EXPORT_SYMBOL(path_has_submounts);
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476int d_set_mounted(struct dentry *dentry)
1477{
1478 struct dentry *p;
1479 int ret = -ENOENT;
1480 write_seqlock(&rename_lock);
1481 for (p = dentry->d_parent; !IS_ROOT(p); p = p->d_parent) {
1482
1483 spin_lock(&p->d_lock);
1484 if (unlikely(d_unhashed(p))) {
1485 spin_unlock(&p->d_lock);
1486 goto out;
1487 }
1488 spin_unlock(&p->d_lock);
1489 }
1490 spin_lock(&dentry->d_lock);
1491 if (!d_unlinked(dentry)) {
1492 ret = -EBUSY;
1493 if (!d_mountpoint(dentry)) {
1494 dentry->d_flags |= DCACHE_MOUNTED;
1495 ret = 0;
1496 }
1497 }
1498 spin_unlock(&dentry->d_lock);
1499out:
1500 write_sequnlock(&rename_lock);
1501 return ret;
1502}
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519struct select_data {
1520 struct dentry *start;
1521 union {
1522 long found;
1523 struct dentry *victim;
1524 };
1525 struct list_head dispose;
1526};
1527
1528static enum d_walk_ret select_collect(void *_data, struct dentry *dentry)
1529{
1530 struct select_data *data = _data;
1531 enum d_walk_ret ret = D_WALK_CONTINUE;
1532
1533 if (data->start == dentry)
1534 goto out;
1535
1536 if (dentry->d_flags & DCACHE_SHRINK_LIST) {
1537 data->found++;
1538 } else {
1539 if (dentry->d_flags & DCACHE_LRU_LIST)
1540 d_lru_del(dentry);
1541 if (!dentry->d_lockref.count) {
1542 d_shrink_add(dentry, &data->dispose);
1543 data->found++;
1544 }
1545 }
1546
1547
1548
1549
1550
1551 if (!list_empty(&data->dispose))
1552 ret = need_resched() ? D_WALK_QUIT : D_WALK_NORETRY;
1553out:
1554 return ret;
1555}
1556
1557static enum d_walk_ret select_collect2(void *_data, struct dentry *dentry)
1558{
1559 struct select_data *data = _data;
1560 enum d_walk_ret ret = D_WALK_CONTINUE;
1561
1562 if (data->start == dentry)
1563 goto out;
1564
1565 if (dentry->d_flags & DCACHE_SHRINK_LIST) {
1566 if (!dentry->d_lockref.count) {
1567 rcu_read_lock();
1568 data->victim = dentry;
1569 return D_WALK_QUIT;
1570 }
1571 } else {
1572 if (dentry->d_flags & DCACHE_LRU_LIST)
1573 d_lru_del(dentry);
1574 if (!dentry->d_lockref.count)
1575 d_shrink_add(dentry, &data->dispose);
1576 }
1577
1578
1579
1580
1581
1582 if (!list_empty(&data->dispose))
1583 ret = need_resched() ? D_WALK_QUIT : D_WALK_NORETRY;
1584out:
1585 return ret;
1586}
1587
1588
1589
1590
1591
1592
1593
1594void shrink_dcache_parent(struct dentry *parent)
1595{
1596 for (;;) {
1597 struct select_data data = {.start = parent};
1598
1599 INIT_LIST_HEAD(&data.dispose);
1600 d_walk(parent, &data, select_collect);
1601
1602 if (!list_empty(&data.dispose)) {
1603 shrink_dentry_list(&data.dispose);
1604 continue;
1605 }
1606
1607 cond_resched();
1608 if (!data.found)
1609 break;
1610 data.victim = NULL;
1611 d_walk(parent, &data, select_collect2);
1612 if (data.victim) {
1613 struct dentry *parent;
1614 spin_lock(&data.victim->d_lock);
1615 if (!shrink_lock_dentry(data.victim)) {
1616 spin_unlock(&data.victim->d_lock);
1617 rcu_read_unlock();
1618 } else {
1619 rcu_read_unlock();
1620 parent = data.victim->d_parent;
1621 if (parent != data.victim)
1622 __dput_to_list(parent, &data.dispose);
1623 __dentry_kill(data.victim);
1624 }
1625 }
1626 if (!list_empty(&data.dispose))
1627 shrink_dentry_list(&data.dispose);
1628 }
1629}
1630EXPORT_SYMBOL(shrink_dcache_parent);
1631
1632static enum d_walk_ret umount_check(void *_data, struct dentry *dentry)
1633{
1634
1635 if (!list_empty(&dentry->d_subdirs))
1636 return D_WALK_CONTINUE;
1637
1638
1639 if (dentry == _data && dentry->d_lockref.count == 1)
1640 return D_WALK_CONTINUE;
1641
1642 printk(KERN_ERR "BUG: Dentry %p{i=%lx,n=%pd} "
1643 " still in use (%d) [unmount of %s %s]\n",
1644 dentry,
1645 dentry->d_inode ?
1646 dentry->d_inode->i_ino : 0UL,
1647 dentry,
1648 dentry->d_lockref.count,
1649 dentry->d_sb->s_type->name,
1650 dentry->d_sb->s_id);
1651 WARN_ON(1);
1652 return D_WALK_CONTINUE;
1653}
1654
1655static void do_one_tree(struct dentry *dentry)
1656{
1657 shrink_dcache_parent(dentry);
1658 d_walk(dentry, dentry, umount_check);
1659 d_drop(dentry);
1660 dput(dentry);
1661}
1662
1663
1664
1665
1666void shrink_dcache_for_umount(struct super_block *sb)
1667{
1668 struct dentry *dentry;
1669
1670 WARN(down_read_trylock(&sb->s_umount), "s_umount should've been locked");
1671
1672 dentry = sb->s_root;
1673 sb->s_root = NULL;
1674 do_one_tree(dentry);
1675
1676 while (!hlist_bl_empty(&sb->s_roots)) {
1677 dentry = dget(hlist_bl_entry(hlist_bl_first(&sb->s_roots), struct dentry, d_hash));
1678 do_one_tree(dentry);
1679 }
1680}
1681
1682static enum d_walk_ret find_submount(void *_data, struct dentry *dentry)
1683{
1684 struct dentry **victim = _data;
1685 if (d_mountpoint(dentry)) {
1686 __dget_dlock(dentry);
1687 *victim = dentry;
1688 return D_WALK_QUIT;
1689 }
1690 return D_WALK_CONTINUE;
1691}
1692
1693
1694
1695
1696
1697void d_invalidate(struct dentry *dentry)
1698{
1699 bool had_submounts = false;
1700 spin_lock(&dentry->d_lock);
1701 if (d_unhashed(dentry)) {
1702 spin_unlock(&dentry->d_lock);
1703 return;
1704 }
1705 __d_drop(dentry);
1706 spin_unlock(&dentry->d_lock);
1707
1708
1709 if (!dentry->d_inode)
1710 return;
1711
1712 shrink_dcache_parent(dentry);
1713 for (;;) {
1714 struct dentry *victim = NULL;
1715 d_walk(dentry, &victim, find_submount);
1716 if (!victim) {
1717 if (had_submounts)
1718 shrink_dcache_parent(dentry);
1719 return;
1720 }
1721 had_submounts = true;
1722 detach_mounts(victim);
1723 dput(victim);
1724 }
1725}
1726EXPORT_SYMBOL(d_invalidate);
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738static struct dentry *__d_alloc(struct super_block *sb, const struct qstr *name)
1739{
1740 struct dentry *dentry;
1741 char *dname;
1742 int err;
1743
1744 dentry = kmem_cache_alloc(dentry_cache, GFP_KERNEL);
1745 if (!dentry)
1746 return NULL;
1747
1748
1749
1750
1751
1752
1753
1754 dentry->d_iname[DNAME_INLINE_LEN-1] = 0;
1755 if (unlikely(!name)) {
1756 name = &slash_name;
1757 dname = dentry->d_iname;
1758 } else if (name->len > DNAME_INLINE_LEN-1) {
1759 size_t size = offsetof(struct external_name, name[1]);
1760 struct external_name *p = kmalloc(size + name->len,
1761 GFP_KERNEL_ACCOUNT |
1762 __GFP_RECLAIMABLE);
1763 if (!p) {
1764 kmem_cache_free(dentry_cache, dentry);
1765 return NULL;
1766 }
1767 atomic_set(&p->u.count, 1);
1768 dname = p->name;
1769 } else {
1770 dname = dentry->d_iname;
1771 }
1772
1773 dentry->d_name.len = name->len;
1774 dentry->d_name.hash = name->hash;
1775 memcpy(dname, name->name, name->len);
1776 dname[name->len] = 0;
1777
1778
1779 smp_store_release(&dentry->d_name.name, dname);
1780
1781 dentry->d_lockref.count = 1;
1782 dentry->d_flags = 0;
1783 spin_lock_init(&dentry->d_lock);
1784 seqcount_spinlock_init(&dentry->d_seq, &dentry->d_lock);
1785 dentry->d_inode = NULL;
1786 dentry->d_parent = dentry;
1787 dentry->d_sb = sb;
1788 dentry->d_op = NULL;
1789 dentry->d_fsdata = NULL;
1790 INIT_HLIST_BL_NODE(&dentry->d_hash);
1791 INIT_LIST_HEAD(&dentry->d_lru);
1792 INIT_LIST_HEAD(&dentry->d_subdirs);
1793 INIT_HLIST_NODE(&dentry->d_u.d_alias);
1794 INIT_LIST_HEAD(&dentry->d_child);
1795 d_set_d_op(dentry, dentry->d_sb->s_d_op);
1796
1797 if (dentry->d_op && dentry->d_op->d_init) {
1798 err = dentry->d_op->d_init(dentry);
1799 if (err) {
1800 if (dname_external(dentry))
1801 kfree(external_name(dentry));
1802 kmem_cache_free(dentry_cache, dentry);
1803 return NULL;
1804 }
1805 }
1806
1807 this_cpu_inc(nr_dentry);
1808
1809 return dentry;
1810}
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821struct dentry *d_alloc(struct dentry * parent, const struct qstr *name)
1822{
1823 struct dentry *dentry = __d_alloc(parent->d_sb, name);
1824 if (!dentry)
1825 return NULL;
1826 spin_lock(&parent->d_lock);
1827
1828
1829
1830
1831 __dget_dlock(parent);
1832 dentry->d_parent = parent;
1833 list_add(&dentry->d_child, &parent->d_subdirs);
1834 spin_unlock(&parent->d_lock);
1835
1836 return dentry;
1837}
1838EXPORT_SYMBOL(d_alloc);
1839
1840struct dentry *d_alloc_anon(struct super_block *sb)
1841{
1842 return __d_alloc(sb, NULL);
1843}
1844EXPORT_SYMBOL(d_alloc_anon);
1845
1846struct dentry *d_alloc_cursor(struct dentry * parent)
1847{
1848 struct dentry *dentry = d_alloc_anon(parent->d_sb);
1849 if (dentry) {
1850 dentry->d_flags |= DCACHE_DENTRY_CURSOR;
1851 dentry->d_parent = dget(parent);
1852 }
1853 return dentry;
1854}
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871struct dentry *d_alloc_pseudo(struct super_block *sb, const struct qstr *name)
1872{
1873 struct dentry *dentry = __d_alloc(sb, name);
1874 if (likely(dentry))
1875 dentry->d_flags |= DCACHE_NORCU;
1876 return dentry;
1877}
1878
1879struct dentry *d_alloc_name(struct dentry *parent, const char *name)
1880{
1881 struct qstr q;
1882
1883 q.name = name;
1884 q.hash_len = hashlen_string(parent, name);
1885 return d_alloc(parent, &q);
1886}
1887EXPORT_SYMBOL(d_alloc_name);
1888
1889void d_set_d_op(struct dentry *dentry, const struct dentry_operations *op)
1890{
1891 WARN_ON_ONCE(dentry->d_op);
1892 WARN_ON_ONCE(dentry->d_flags & (DCACHE_OP_HASH |
1893 DCACHE_OP_COMPARE |
1894 DCACHE_OP_REVALIDATE |
1895 DCACHE_OP_WEAK_REVALIDATE |
1896 DCACHE_OP_DELETE |
1897 DCACHE_OP_REAL));
1898 dentry->d_op = op;
1899 if (!op)
1900 return;
1901 if (op->d_hash)
1902 dentry->d_flags |= DCACHE_OP_HASH;
1903 if (op->d_compare)
1904 dentry->d_flags |= DCACHE_OP_COMPARE;
1905 if (op->d_revalidate)
1906 dentry->d_flags |= DCACHE_OP_REVALIDATE;
1907 if (op->d_weak_revalidate)
1908 dentry->d_flags |= DCACHE_OP_WEAK_REVALIDATE;
1909 if (op->d_delete)
1910 dentry->d_flags |= DCACHE_OP_DELETE;
1911 if (op->d_prune)
1912 dentry->d_flags |= DCACHE_OP_PRUNE;
1913 if (op->d_real)
1914 dentry->d_flags |= DCACHE_OP_REAL;
1915
1916}
1917EXPORT_SYMBOL(d_set_d_op);
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927void d_set_fallthru(struct dentry *dentry)
1928{
1929 spin_lock(&dentry->d_lock);
1930 dentry->d_flags |= DCACHE_FALLTHRU;
1931 spin_unlock(&dentry->d_lock);
1932}
1933EXPORT_SYMBOL(d_set_fallthru);
1934
1935static unsigned d_flags_for_inode(struct inode *inode)
1936{
1937 unsigned add_flags = DCACHE_REGULAR_TYPE;
1938
1939 if (!inode)
1940 return DCACHE_MISS_TYPE;
1941
1942 if (S_ISDIR(inode->i_mode)) {
1943 add_flags = DCACHE_DIRECTORY_TYPE;
1944 if (unlikely(!(inode->i_opflags & IOP_LOOKUP))) {
1945 if (unlikely(!inode->i_op->lookup))
1946 add_flags = DCACHE_AUTODIR_TYPE;
1947 else
1948 inode->i_opflags |= IOP_LOOKUP;
1949 }
1950 goto type_determined;
1951 }
1952
1953 if (unlikely(!(inode->i_opflags & IOP_NOFOLLOW))) {
1954 if (unlikely(inode->i_op->get_link)) {
1955 add_flags = DCACHE_SYMLINK_TYPE;
1956 goto type_determined;
1957 }
1958 inode->i_opflags |= IOP_NOFOLLOW;
1959 }
1960
1961 if (unlikely(!S_ISREG(inode->i_mode)))
1962 add_flags = DCACHE_SPECIAL_TYPE;
1963
1964type_determined:
1965 if (unlikely(IS_AUTOMOUNT(inode)))
1966 add_flags |= DCACHE_NEED_AUTOMOUNT;
1967 return add_flags;
1968}
1969
1970static void __d_instantiate(struct dentry *dentry, struct inode *inode)
1971{
1972 unsigned add_flags = d_flags_for_inode(inode);
1973 WARN_ON(d_in_lookup(dentry));
1974
1975 spin_lock(&dentry->d_lock);
1976
1977
1978
1979 if (dentry->d_flags & DCACHE_LRU_LIST)
1980 this_cpu_dec(nr_dentry_negative);
1981 hlist_add_head(&dentry->d_u.d_alias, &inode->i_dentry);
1982 raw_write_seqcount_begin(&dentry->d_seq);
1983 __d_set_inode_and_type(dentry, inode, add_flags);
1984 raw_write_seqcount_end(&dentry->d_seq);
1985 fsnotify_update_flags(dentry);
1986 spin_unlock(&dentry->d_lock);
1987}
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004void d_instantiate(struct dentry *entry, struct inode * inode)
2005{
2006 BUG_ON(!hlist_unhashed(&entry->d_u.d_alias));
2007 if (inode) {
2008 security_d_instantiate(entry, inode);
2009 spin_lock(&inode->i_lock);
2010 __d_instantiate(entry, inode);
2011 spin_unlock(&inode->i_lock);
2012 }
2013}
2014EXPORT_SYMBOL(d_instantiate);
2015
2016
2017
2018
2019
2020
2021
2022void d_instantiate_new(struct dentry *entry, struct inode *inode)
2023{
2024 BUG_ON(!hlist_unhashed(&entry->d_u.d_alias));
2025 BUG_ON(!inode);
2026 lockdep_annotate_inode_mutex_key(inode);
2027 security_d_instantiate(entry, inode);
2028 spin_lock(&inode->i_lock);
2029 __d_instantiate(entry, inode);
2030 WARN_ON(!(inode->i_state & I_NEW));
2031 inode->i_state &= ~I_NEW & ~I_CREATING;
2032 smp_mb();
2033 wake_up_bit(&inode->i_state, __I_NEW);
2034 spin_unlock(&inode->i_lock);
2035}
2036EXPORT_SYMBOL(d_instantiate_new);
2037
2038struct dentry *d_make_root(struct inode *root_inode)
2039{
2040 struct dentry *res = NULL;
2041
2042 if (root_inode) {
2043 res = d_alloc_anon(root_inode->i_sb);
2044 if (res)
2045 d_instantiate(res, root_inode);
2046 else
2047 iput(root_inode);
2048 }
2049 return res;
2050}
2051EXPORT_SYMBOL(d_make_root);
2052
2053static struct dentry *__d_instantiate_anon(struct dentry *dentry,
2054 struct inode *inode,
2055 bool disconnected)
2056{
2057 struct dentry *res;
2058 unsigned add_flags;
2059
2060 security_d_instantiate(dentry, inode);
2061 spin_lock(&inode->i_lock);
2062 res = __d_find_any_alias(inode);
2063 if (res) {
2064 spin_unlock(&inode->i_lock);
2065 dput(dentry);
2066 goto out_iput;
2067 }
2068
2069
2070 add_flags = d_flags_for_inode(inode);
2071
2072 if (disconnected)
2073 add_flags |= DCACHE_DISCONNECTED;
2074
2075 spin_lock(&dentry->d_lock);
2076 __d_set_inode_and_type(dentry, inode, add_flags);
2077 hlist_add_head(&dentry->d_u.d_alias, &inode->i_dentry);
2078 if (!disconnected) {
2079 hlist_bl_lock(&dentry->d_sb->s_roots);
2080 hlist_bl_add_head(&dentry->d_hash, &dentry->d_sb->s_roots);
2081 hlist_bl_unlock(&dentry->d_sb->s_roots);
2082 }
2083 spin_unlock(&dentry->d_lock);
2084 spin_unlock(&inode->i_lock);
2085
2086 return dentry;
2087
2088 out_iput:
2089 iput(inode);
2090 return res;
2091}
2092
2093struct dentry *d_instantiate_anon(struct dentry *dentry, struct inode *inode)
2094{
2095 return __d_instantiate_anon(dentry, inode, true);
2096}
2097EXPORT_SYMBOL(d_instantiate_anon);
2098
2099static struct dentry *__d_obtain_alias(struct inode *inode, bool disconnected)
2100{
2101 struct dentry *tmp;
2102 struct dentry *res;
2103
2104 if (!inode)
2105 return ERR_PTR(-ESTALE);
2106 if (IS_ERR(inode))
2107 return ERR_CAST(inode);
2108
2109 res = d_find_any_alias(inode);
2110 if (res)
2111 goto out_iput;
2112
2113 tmp = d_alloc_anon(inode->i_sb);
2114 if (!tmp) {
2115 res = ERR_PTR(-ENOMEM);
2116 goto out_iput;
2117 }
2118
2119 return __d_instantiate_anon(tmp, inode, disconnected);
2120
2121out_iput:
2122 iput(inode);
2123 return res;
2124}
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144struct dentry *d_obtain_alias(struct inode *inode)
2145{
2146 return __d_obtain_alias(inode, true);
2147}
2148EXPORT_SYMBOL(d_obtain_alias);
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165struct dentry *d_obtain_root(struct inode *inode)
2166{
2167 return __d_obtain_alias(inode, false);
2168}
2169EXPORT_SYMBOL(d_obtain_root);
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187struct dentry *d_add_ci(struct dentry *dentry, struct inode *inode,
2188 struct qstr *name)
2189{
2190 struct dentry *found, *res;
2191
2192
2193
2194
2195
2196 found = d_hash_and_lookup(dentry->d_parent, name);
2197 if (found) {
2198 iput(inode);
2199 return found;
2200 }
2201 if (d_in_lookup(dentry)) {
2202 found = d_alloc_parallel(dentry->d_parent, name,
2203 dentry->d_wait);
2204 if (IS_ERR(found) || !d_in_lookup(found)) {
2205 iput(inode);
2206 return found;
2207 }
2208 } else {
2209 found = d_alloc(dentry->d_parent, name);
2210 if (!found) {
2211 iput(inode);
2212 return ERR_PTR(-ENOMEM);
2213 }
2214 }
2215 res = d_splice_alias(inode, found);
2216 if (res) {
2217 dput(found);
2218 return res;
2219 }
2220 return found;
2221}
2222EXPORT_SYMBOL(d_add_ci);
2223
2224
2225static inline bool d_same_name(const struct dentry *dentry,
2226 const struct dentry *parent,
2227 const struct qstr *name)
2228{
2229 if (likely(!(parent->d_flags & DCACHE_OP_COMPARE))) {
2230 if (dentry->d_name.len != name->len)
2231 return false;
2232 return dentry_cmp(dentry, name->name, name->len) == 0;
2233 }
2234 return parent->d_op->d_compare(dentry,
2235 dentry->d_name.len, dentry->d_name.name,
2236 name) == 0;
2237}
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268struct dentry *__d_lookup_rcu(const struct dentry *parent,
2269 const struct qstr *name,
2270 unsigned *seqp)
2271{
2272 u64 hashlen = name->hash_len;
2273 const unsigned char *str = name->name;
2274 struct hlist_bl_head *b = d_hash(hashlen_hash(hashlen));
2275 struct hlist_bl_node *node;
2276 struct dentry *dentry;
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298 hlist_bl_for_each_entry_rcu(dentry, node, b, d_hash) {
2299 unsigned seq;
2300
2301seqretry:
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319 seq = raw_seqcount_begin(&dentry->d_seq);
2320 if (dentry->d_parent != parent)
2321 continue;
2322 if (d_unhashed(dentry))
2323 continue;
2324
2325 if (unlikely(parent->d_flags & DCACHE_OP_COMPARE)) {
2326 int tlen;
2327 const char *tname;
2328 if (dentry->d_name.hash != hashlen_hash(hashlen))
2329 continue;
2330 tlen = dentry->d_name.len;
2331 tname = dentry->d_name.name;
2332
2333 if (read_seqcount_retry(&dentry->d_seq, seq)) {
2334 cpu_relax();
2335 goto seqretry;
2336 }
2337 if (parent->d_op->d_compare(dentry,
2338 tlen, tname, name) != 0)
2339 continue;
2340 } else {
2341 if (dentry->d_name.hash_len != hashlen)
2342 continue;
2343 if (dentry_cmp(dentry, str, hashlen_len(hashlen)) != 0)
2344 continue;
2345 }
2346 *seqp = seq;
2347 return dentry;
2348 }
2349 return NULL;
2350}
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363struct dentry *d_lookup(const struct dentry *parent, const struct qstr *name)
2364{
2365 struct dentry *dentry;
2366 unsigned seq;
2367
2368 do {
2369 seq = read_seqbegin(&rename_lock);
2370 dentry = __d_lookup(parent, name);
2371 if (dentry)
2372 break;
2373 } while (read_seqretry(&rename_lock, seq));
2374 return dentry;
2375}
2376EXPORT_SYMBOL(d_lookup);
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393struct dentry *__d_lookup(const struct dentry *parent, const struct qstr *name)
2394{
2395 unsigned int hash = name->hash;
2396 struct hlist_bl_head *b = d_hash(hash);
2397 struct hlist_bl_node *node;
2398 struct dentry *found = NULL;
2399 struct dentry *dentry;
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421 rcu_read_lock();
2422
2423 hlist_bl_for_each_entry_rcu(dentry, node, b, d_hash) {
2424
2425 if (dentry->d_name.hash != hash)
2426 continue;
2427
2428 spin_lock(&dentry->d_lock);
2429 if (dentry->d_parent != parent)
2430 goto next;
2431 if (d_unhashed(dentry))
2432 goto next;
2433
2434 if (!d_same_name(dentry, parent, name))
2435 goto next;
2436
2437 dentry->d_lockref.count++;
2438 found = dentry;
2439 spin_unlock(&dentry->d_lock);
2440 break;
2441next:
2442 spin_unlock(&dentry->d_lock);
2443 }
2444 rcu_read_unlock();
2445
2446 return found;
2447}
2448
2449
2450
2451
2452
2453
2454
2455
2456struct dentry *d_hash_and_lookup(struct dentry *dir, struct qstr *name)
2457{
2458
2459
2460
2461
2462
2463 name->hash = full_name_hash(dir, name->name, name->len);
2464 if (dir->d_flags & DCACHE_OP_HASH) {
2465 int err = dir->d_op->d_hash(dir, name);
2466 if (unlikely(err < 0))
2467 return ERR_PTR(err);
2468 }
2469 return d_lookup(dir, name);
2470}
2471EXPORT_SYMBOL(d_hash_and_lookup);
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494void d_delete(struct dentry * dentry)
2495{
2496 struct inode *inode = dentry->d_inode;
2497
2498 spin_lock(&inode->i_lock);
2499 spin_lock(&dentry->d_lock);
2500
2501
2502
2503 if (dentry->d_lockref.count == 1) {
2504 dentry->d_flags &= ~DCACHE_CANT_MOUNT;
2505 dentry_unlink_inode(dentry);
2506 } else {
2507 __d_drop(dentry);
2508 spin_unlock(&dentry->d_lock);
2509 spin_unlock(&inode->i_lock);
2510 }
2511}
2512EXPORT_SYMBOL(d_delete);
2513
2514static void __d_rehash(struct dentry *entry)
2515{
2516 struct hlist_bl_head *b = d_hash(entry->d_name.hash);
2517
2518 hlist_bl_lock(b);
2519 hlist_bl_add_head_rcu(&entry->d_hash, b);
2520 hlist_bl_unlock(b);
2521}
2522
2523
2524
2525
2526
2527
2528
2529
2530void d_rehash(struct dentry * entry)
2531{
2532 spin_lock(&entry->d_lock);
2533 __d_rehash(entry);
2534 spin_unlock(&entry->d_lock);
2535}
2536EXPORT_SYMBOL(d_rehash);
2537
2538static inline unsigned start_dir_add(struct inode *dir)
2539{
2540
2541 for (;;) {
2542 unsigned n = dir->i_dir_seq;
2543 if (!(n & 1) && cmpxchg(&dir->i_dir_seq, n, n + 1) == n)
2544 return n;
2545 cpu_relax();
2546 }
2547}
2548
2549static inline void end_dir_add(struct inode *dir, unsigned n)
2550{
2551 smp_store_release(&dir->i_dir_seq, n + 2);
2552}
2553
2554static void d_wait_lookup(struct dentry *dentry)
2555{
2556 if (d_in_lookup(dentry)) {
2557 DECLARE_WAITQUEUE(wait, current);
2558 add_wait_queue(dentry->d_wait, &wait);
2559 do {
2560 set_current_state(TASK_UNINTERRUPTIBLE);
2561 spin_unlock(&dentry->d_lock);
2562 schedule();
2563 spin_lock(&dentry->d_lock);
2564 } while (d_in_lookup(dentry));
2565 }
2566}
2567
2568struct dentry *d_alloc_parallel(struct dentry *parent,
2569 const struct qstr *name,
2570 wait_queue_head_t *wq)
2571{
2572 unsigned int hash = name->hash;
2573 struct hlist_bl_head *b = in_lookup_hash(parent, hash);
2574 struct hlist_bl_node *node;
2575 struct dentry *new = d_alloc(parent, name);
2576 struct dentry *dentry;
2577 unsigned seq, r_seq, d_seq;
2578
2579 if (unlikely(!new))
2580 return ERR_PTR(-ENOMEM);
2581
2582retry:
2583 rcu_read_lock();
2584 seq = smp_load_acquire(&parent->d_inode->i_dir_seq);
2585 r_seq = read_seqbegin(&rename_lock);
2586 dentry = __d_lookup_rcu(parent, name, &d_seq);
2587 if (unlikely(dentry)) {
2588 if (!lockref_get_not_dead(&dentry->d_lockref)) {
2589 rcu_read_unlock();
2590 goto retry;
2591 }
2592 if (read_seqcount_retry(&dentry->d_seq, d_seq)) {
2593 rcu_read_unlock();
2594 dput(dentry);
2595 goto retry;
2596 }
2597 rcu_read_unlock();
2598 dput(new);
2599 return dentry;
2600 }
2601 if (unlikely(read_seqretry(&rename_lock, r_seq))) {
2602 rcu_read_unlock();
2603 goto retry;
2604 }
2605
2606 if (unlikely(seq & 1)) {
2607 rcu_read_unlock();
2608 goto retry;
2609 }
2610
2611 hlist_bl_lock(b);
2612 if (unlikely(READ_ONCE(parent->d_inode->i_dir_seq) != seq)) {
2613 hlist_bl_unlock(b);
2614 rcu_read_unlock();
2615 goto retry;
2616 }
2617
2618
2619
2620
2621
2622
2623
2624 hlist_bl_for_each_entry(dentry, node, b, d_u.d_in_lookup_hash) {
2625 if (dentry->d_name.hash != hash)
2626 continue;
2627 if (dentry->d_parent != parent)
2628 continue;
2629 if (!d_same_name(dentry, parent, name))
2630 continue;
2631 hlist_bl_unlock(b);
2632
2633 if (!lockref_get_not_dead(&dentry->d_lockref)) {
2634 rcu_read_unlock();
2635 goto retry;
2636 }
2637
2638 rcu_read_unlock();
2639
2640
2641
2642
2643 spin_lock(&dentry->d_lock);
2644 d_wait_lookup(dentry);
2645
2646
2647
2648
2649
2650
2651 if (unlikely(dentry->d_name.hash != hash))
2652 goto mismatch;
2653 if (unlikely(dentry->d_parent != parent))
2654 goto mismatch;
2655 if (unlikely(d_unhashed(dentry)))
2656 goto mismatch;
2657 if (unlikely(!d_same_name(dentry, parent, name)))
2658 goto mismatch;
2659
2660 spin_unlock(&dentry->d_lock);
2661 dput(new);
2662 return dentry;
2663 }
2664 rcu_read_unlock();
2665
2666 new->d_flags |= DCACHE_PAR_LOOKUP;
2667 new->d_wait = wq;
2668 hlist_bl_add_head_rcu(&new->d_u.d_in_lookup_hash, b);
2669 hlist_bl_unlock(b);
2670 return new;
2671mismatch:
2672 spin_unlock(&dentry->d_lock);
2673 dput(dentry);
2674 goto retry;
2675}
2676EXPORT_SYMBOL(d_alloc_parallel);
2677
2678void __d_lookup_done(struct dentry *dentry)
2679{
2680 struct hlist_bl_head *b = in_lookup_hash(dentry->d_parent,
2681 dentry->d_name.hash);
2682 hlist_bl_lock(b);
2683 dentry->d_flags &= ~DCACHE_PAR_LOOKUP;
2684 __hlist_bl_del(&dentry->d_u.d_in_lookup_hash);
2685 wake_up_all(dentry->d_wait);
2686 dentry->d_wait = NULL;
2687 hlist_bl_unlock(b);
2688 INIT_HLIST_NODE(&dentry->d_u.d_alias);
2689 INIT_LIST_HEAD(&dentry->d_lru);
2690}
2691EXPORT_SYMBOL(__d_lookup_done);
2692
2693
2694
2695static inline void __d_add(struct dentry *dentry, struct inode *inode)
2696{
2697 struct inode *dir = NULL;
2698 unsigned n;
2699 spin_lock(&dentry->d_lock);
2700 if (unlikely(d_in_lookup(dentry))) {
2701 dir = dentry->d_parent->d_inode;
2702 n = start_dir_add(dir);
2703 __d_lookup_done(dentry);
2704 }
2705 if (inode) {
2706 unsigned add_flags = d_flags_for_inode(inode);
2707 hlist_add_head(&dentry->d_u.d_alias, &inode->i_dentry);
2708 raw_write_seqcount_begin(&dentry->d_seq);
2709 __d_set_inode_and_type(dentry, inode, add_flags);
2710 raw_write_seqcount_end(&dentry->d_seq);
2711 fsnotify_update_flags(dentry);
2712 }
2713 __d_rehash(dentry);
2714 if (dir)
2715 end_dir_add(dir, n);
2716 spin_unlock(&dentry->d_lock);
2717 if (inode)
2718 spin_unlock(&inode->i_lock);
2719}
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730void d_add(struct dentry *entry, struct inode *inode)
2731{
2732 if (inode) {
2733 security_d_instantiate(entry, inode);
2734 spin_lock(&inode->i_lock);
2735 }
2736 __d_add(entry, inode);
2737}
2738EXPORT_SYMBOL(d_add);
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751struct dentry *d_exact_alias(struct dentry *entry, struct inode *inode)
2752{
2753 struct dentry *alias;
2754 unsigned int hash = entry->d_name.hash;
2755
2756 spin_lock(&inode->i_lock);
2757 hlist_for_each_entry(alias, &inode->i_dentry, d_u.d_alias) {
2758
2759
2760
2761
2762
2763 if (alias->d_name.hash != hash)
2764 continue;
2765 if (alias->d_parent != entry->d_parent)
2766 continue;
2767 if (!d_same_name(alias, entry->d_parent, &entry->d_name))
2768 continue;
2769 spin_lock(&alias->d_lock);
2770 if (!d_unhashed(alias)) {
2771 spin_unlock(&alias->d_lock);
2772 alias = NULL;
2773 } else {
2774 __dget_dlock(alias);
2775 __d_rehash(alias);
2776 spin_unlock(&alias->d_lock);
2777 }
2778 spin_unlock(&inode->i_lock);
2779 return alias;
2780 }
2781 spin_unlock(&inode->i_lock);
2782 return NULL;
2783}
2784EXPORT_SYMBOL(d_exact_alias);
2785
2786static void swap_names(struct dentry *dentry, struct dentry *target)
2787{
2788 if (unlikely(dname_external(target))) {
2789 if (unlikely(dname_external(dentry))) {
2790
2791
2792
2793 swap(target->d_name.name, dentry->d_name.name);
2794 } else {
2795
2796
2797
2798
2799 memcpy(target->d_iname, dentry->d_name.name,
2800 dentry->d_name.len + 1);
2801 dentry->d_name.name = target->d_name.name;
2802 target->d_name.name = target->d_iname;
2803 }
2804 } else {
2805 if (unlikely(dname_external(dentry))) {
2806
2807
2808
2809
2810 memcpy(dentry->d_iname, target->d_name.name,
2811 target->d_name.len + 1);
2812 target->d_name.name = dentry->d_name.name;
2813 dentry->d_name.name = dentry->d_iname;
2814 } else {
2815
2816
2817
2818 unsigned int i;
2819 BUILD_BUG_ON(!IS_ALIGNED(DNAME_INLINE_LEN, sizeof(long)));
2820 for (i = 0; i < DNAME_INLINE_LEN / sizeof(long); i++) {
2821 swap(((long *) &dentry->d_iname)[i],
2822 ((long *) &target->d_iname)[i]);
2823 }
2824 }
2825 }
2826 swap(dentry->d_name.hash_len, target->d_name.hash_len);
2827}
2828
2829static void copy_name(struct dentry *dentry, struct dentry *target)
2830{
2831 struct external_name *old_name = NULL;
2832 if (unlikely(dname_external(dentry)))
2833 old_name = external_name(dentry);
2834 if (unlikely(dname_external(target))) {
2835 atomic_inc(&external_name(target)->u.count);
2836 dentry->d_name = target->d_name;
2837 } else {
2838 memcpy(dentry->d_iname, target->d_name.name,
2839 target->d_name.len + 1);
2840 dentry->d_name.name = dentry->d_iname;
2841 dentry->d_name.hash_len = target->d_name.hash_len;
2842 }
2843 if (old_name && likely(atomic_dec_and_test(&old_name->u.count)))
2844 kfree_rcu(old_name, u.head);
2845}
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858static void __d_move(struct dentry *dentry, struct dentry *target,
2859 bool exchange)
2860{
2861 struct dentry *old_parent, *p;
2862 struct inode *dir = NULL;
2863 unsigned n;
2864
2865 WARN_ON(!dentry->d_inode);
2866 if (WARN_ON(dentry == target))
2867 return;
2868
2869 BUG_ON(d_ancestor(target, dentry));
2870 old_parent = dentry->d_parent;
2871 p = d_ancestor(old_parent, target);
2872 if (IS_ROOT(dentry)) {
2873 BUG_ON(p);
2874 spin_lock(&target->d_parent->d_lock);
2875 } else if (!p) {
2876
2877 spin_lock(&target->d_parent->d_lock);
2878 spin_lock_nested(&old_parent->d_lock, DENTRY_D_LOCK_NESTED);
2879 } else {
2880 BUG_ON(p == dentry);
2881 spin_lock(&old_parent->d_lock);
2882 if (p != target)
2883 spin_lock_nested(&target->d_parent->d_lock,
2884 DENTRY_D_LOCK_NESTED);
2885 }
2886 spin_lock_nested(&dentry->d_lock, 2);
2887 spin_lock_nested(&target->d_lock, 3);
2888
2889 if (unlikely(d_in_lookup(target))) {
2890 dir = target->d_parent->d_inode;
2891 n = start_dir_add(dir);
2892 __d_lookup_done(target);
2893 }
2894
2895 write_seqcount_begin(&dentry->d_seq);
2896 write_seqcount_begin_nested(&target->d_seq, DENTRY_D_LOCK_NESTED);
2897
2898
2899 if (!d_unhashed(dentry))
2900 ___d_drop(dentry);
2901 if (!d_unhashed(target))
2902 ___d_drop(target);
2903
2904
2905 dentry->d_parent = target->d_parent;
2906 if (!exchange) {
2907 copy_name(dentry, target);
2908 target->d_hash.pprev = NULL;
2909 dentry->d_parent->d_lockref.count++;
2910 if (dentry != old_parent)
2911 WARN_ON(!--old_parent->d_lockref.count);
2912 } else {
2913 target->d_parent = old_parent;
2914 swap_names(dentry, target);
2915 list_move(&target->d_child, &target->d_parent->d_subdirs);
2916 __d_rehash(target);
2917 fsnotify_update_flags(target);
2918 }
2919 list_move(&dentry->d_child, &dentry->d_parent->d_subdirs);
2920 __d_rehash(dentry);
2921 fsnotify_update_flags(dentry);
2922 fscrypt_handle_d_move(dentry);
2923
2924 write_seqcount_end(&target->d_seq);
2925 write_seqcount_end(&dentry->d_seq);
2926
2927 if (dir)
2928 end_dir_add(dir, n);
2929
2930 if (dentry->d_parent != old_parent)
2931 spin_unlock(&dentry->d_parent->d_lock);
2932 if (dentry != old_parent)
2933 spin_unlock(&old_parent->d_lock);
2934 spin_unlock(&target->d_lock);
2935 spin_unlock(&dentry->d_lock);
2936}
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947void d_move(struct dentry *dentry, struct dentry *target)
2948{
2949 write_seqlock(&rename_lock);
2950 __d_move(dentry, target, false);
2951 write_sequnlock(&rename_lock);
2952}
2953EXPORT_SYMBOL(d_move);
2954
2955
2956
2957
2958
2959
2960void d_exchange(struct dentry *dentry1, struct dentry *dentry2)
2961{
2962 write_seqlock(&rename_lock);
2963
2964 WARN_ON(!dentry1->d_inode);
2965 WARN_ON(!dentry2->d_inode);
2966 WARN_ON(IS_ROOT(dentry1));
2967 WARN_ON(IS_ROOT(dentry2));
2968
2969 __d_move(dentry1, dentry2, true);
2970
2971 write_sequnlock(&rename_lock);
2972}
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982struct dentry *d_ancestor(struct dentry *p1, struct dentry *p2)
2983{
2984 struct dentry *p;
2985
2986 for (p = p2; !IS_ROOT(p); p = p->d_parent) {
2987 if (p->d_parent == p1)
2988 return p;
2989 }
2990 return NULL;
2991}
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002static int __d_unalias(struct inode *inode,
3003 struct dentry *dentry, struct dentry *alias)
3004{
3005 struct mutex *m1 = NULL;
3006 struct rw_semaphore *m2 = NULL;
3007 int ret = -ESTALE;
3008
3009
3010 if (alias->d_parent == dentry->d_parent)
3011 goto out_unalias;
3012
3013
3014 if (!mutex_trylock(&dentry->d_sb->s_vfs_rename_mutex))
3015 goto out_err;
3016 m1 = &dentry->d_sb->s_vfs_rename_mutex;
3017 if (!inode_trylock_shared(alias->d_parent->d_inode))
3018 goto out_err;
3019 m2 = &alias->d_parent->d_inode->i_rwsem;
3020out_unalias:
3021 __d_move(alias, dentry, false);
3022 ret = 0;
3023out_err:
3024 if (m2)
3025 up_read(m2);
3026 if (m1)
3027 mutex_unlock(m1);
3028 return ret;
3029}
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054struct dentry *d_splice_alias(struct inode *inode, struct dentry *dentry)
3055{
3056 if (IS_ERR(inode))
3057 return ERR_CAST(inode);
3058
3059 BUG_ON(!d_unhashed(dentry));
3060
3061 if (!inode)
3062 goto out;
3063
3064 security_d_instantiate(dentry, inode);
3065 spin_lock(&inode->i_lock);
3066 if (S_ISDIR(inode->i_mode)) {
3067 struct dentry *new = __d_find_any_alias(inode);
3068 if (unlikely(new)) {
3069
3070 spin_unlock(&inode->i_lock);
3071 write_seqlock(&rename_lock);
3072 if (unlikely(d_ancestor(new, dentry))) {
3073 write_sequnlock(&rename_lock);
3074 dput(new);
3075 new = ERR_PTR(-ELOOP);
3076 pr_warn_ratelimited(
3077 "VFS: Lookup of '%s' in %s %s"
3078 " would have caused loop\n",
3079 dentry->d_name.name,
3080 inode->i_sb->s_type->name,
3081 inode->i_sb->s_id);
3082 } else if (!IS_ROOT(new)) {
3083 struct dentry *old_parent = dget(new->d_parent);
3084 int err = __d_unalias(inode, dentry, new);
3085 write_sequnlock(&rename_lock);
3086 if (err) {
3087 dput(new);
3088 new = ERR_PTR(err);
3089 }
3090 dput(old_parent);
3091 } else {
3092 __d_move(new, dentry, false);
3093 write_sequnlock(&rename_lock);
3094 }
3095 iput(inode);
3096 return new;
3097 }
3098 }
3099out:
3100 __d_add(dentry, inode);
3101 return NULL;
3102}
3103EXPORT_SYMBOL(d_splice_alias);
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121bool is_subdir(struct dentry *new_dentry, struct dentry *old_dentry)
3122{
3123 bool result;
3124 unsigned seq;
3125
3126 if (new_dentry == old_dentry)
3127 return true;
3128
3129 do {
3130
3131 seq = read_seqbegin(&rename_lock);
3132
3133
3134
3135
3136 rcu_read_lock();
3137 if (d_ancestor(old_dentry, new_dentry))
3138 result = true;
3139 else
3140 result = false;
3141 rcu_read_unlock();
3142 } while (read_seqretry(&rename_lock, seq));
3143
3144 return result;
3145}
3146EXPORT_SYMBOL(is_subdir);
3147
3148static enum d_walk_ret d_genocide_kill(void *data, struct dentry *dentry)
3149{
3150 struct dentry *root = data;
3151 if (dentry != root) {
3152 if (d_unhashed(dentry) || !dentry->d_inode)
3153 return D_WALK_SKIP;
3154
3155 if (!(dentry->d_flags & DCACHE_GENOCIDE)) {
3156 dentry->d_flags |= DCACHE_GENOCIDE;
3157 dentry->d_lockref.count--;
3158 }
3159 }
3160 return D_WALK_CONTINUE;
3161}
3162
3163void d_genocide(struct dentry *parent)
3164{
3165 d_walk(parent, parent, d_genocide_kill);
3166}
3167
3168EXPORT_SYMBOL(d_genocide);
3169
3170void d_tmpfile(struct dentry *dentry, struct inode *inode)
3171{
3172 inode_dec_link_count(inode);
3173 BUG_ON(dentry->d_name.name != dentry->d_iname ||
3174 !hlist_unhashed(&dentry->d_u.d_alias) ||
3175 !d_unlinked(dentry));
3176 spin_lock(&dentry->d_parent->d_lock);
3177 spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED);
3178 dentry->d_name.len = sprintf(dentry->d_iname, "#%llu",
3179 (unsigned long long)inode->i_ino);
3180 spin_unlock(&dentry->d_lock);
3181 spin_unlock(&dentry->d_parent->d_lock);
3182 d_instantiate(dentry, inode);
3183}
3184EXPORT_SYMBOL(d_tmpfile);
3185
3186static __initdata unsigned long dhash_entries;
3187static int __init set_dhash_entries(char *str)
3188{
3189 if (!str)
3190 return 0;
3191 dhash_entries = simple_strtoul(str, &str, 0);
3192 return 1;
3193}
3194__setup("dhash_entries=", set_dhash_entries);
3195
3196static void __init dcache_init_early(void)
3197{
3198
3199
3200
3201 if (hashdist)
3202 return;
3203
3204 dentry_hashtable =
3205 alloc_large_system_hash("Dentry cache",
3206 sizeof(struct hlist_bl_head),
3207 dhash_entries,
3208 13,
3209 HASH_EARLY | HASH_ZERO,
3210 &d_hash_shift,
3211 NULL,
3212 0,
3213 0);
3214 d_hash_shift = 32 - d_hash_shift;
3215}
3216
3217static void __init dcache_init(void)
3218{
3219
3220
3221
3222
3223
3224 dentry_cache = KMEM_CACHE_USERCOPY(dentry,
3225 SLAB_RECLAIM_ACCOUNT|SLAB_PANIC|SLAB_MEM_SPREAD|SLAB_ACCOUNT,
3226 d_iname);
3227
3228
3229 if (!hashdist)
3230 return;
3231
3232 dentry_hashtable =
3233 alloc_large_system_hash("Dentry cache",
3234 sizeof(struct hlist_bl_head),
3235 dhash_entries,
3236 13,
3237 HASH_ZERO,
3238 &d_hash_shift,
3239 NULL,
3240 0,
3241 0);
3242 d_hash_shift = 32 - d_hash_shift;
3243}
3244
3245
3246struct kmem_cache *names_cachep __read_mostly;
3247EXPORT_SYMBOL(names_cachep);
3248
3249void __init vfs_caches_init_early(void)
3250{
3251 int i;
3252
3253 for (i = 0; i < ARRAY_SIZE(in_lookup_hashtable); i++)
3254 INIT_HLIST_BL_HEAD(&in_lookup_hashtable[i]);
3255
3256 dcache_init_early();
3257 inode_init_early();
3258}
3259
3260void __init vfs_caches_init(void)
3261{
3262 names_cachep = kmem_cache_create_usercopy("names_cache", PATH_MAX, 0,
3263 SLAB_HWCACHE_ALIGN|SLAB_PANIC, 0, PATH_MAX, NULL);
3264
3265 dcache_init();
3266 inode_init();
3267 files_init();
3268 files_maxfiles_init();
3269 mnt_init();
3270 bdev_cache_init();
3271 chrdev_init();
3272}
3273