1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18#include <linux/init.h>
19#include <linux/export.h>
20#include <linux/kernel.h>
21#include <linux/slab.h>
22#include <linux/fs.h>
23#include <linux/namei.h>
24#include <linux/pagemap.h>
25#include <linux/fsnotify.h>
26#include <linux/personality.h>
27#include <linux/security.h>
28#include <linux/ima.h>
29#include <linux/syscalls.h>
30#include <linux/mount.h>
31#include <linux/audit.h>
32#include <linux/capability.h>
33#include <linux/file.h>
34#include <linux/fcntl.h>
35#include <linux/device_cgroup.h>
36#include <linux/fs_struct.h>
37#include <linux/posix_acl.h>
38#include <linux/hash.h>
39#include <linux/bitops.h>
40#include <linux/init_task.h>
41#include <linux/uaccess.h>
42
43#include "internal.h"
44#include "mount.h"
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125#define EMBEDDED_NAME_MAX (PATH_MAX - offsetof(struct filename, iname))
126
127struct filename *
128getname_flags(const char __user *filename, int flags, int *empty)
129{
130 struct filename *result;
131 char *kname;
132 int len;
133
134 result = audit_reusename(filename);
135 if (result)
136 return result;
137
138 result = __getname();
139 if (unlikely(!result))
140 return ERR_PTR(-ENOMEM);
141
142
143
144
145
146 kname = (char *)result->iname;
147 result->name = kname;
148
149 len = strncpy_from_user(kname, filename, EMBEDDED_NAME_MAX);
150 if (unlikely(len < 0)) {
151 __putname(result);
152 return ERR_PTR(len);
153 }
154
155
156
157
158
159
160
161 if (unlikely(len == EMBEDDED_NAME_MAX)) {
162 const size_t size = offsetof(struct filename, iname[1]);
163 kname = (char *)result;
164
165
166
167
168
169
170 result = kzalloc(size, GFP_KERNEL);
171 if (unlikely(!result)) {
172 __putname(kname);
173 return ERR_PTR(-ENOMEM);
174 }
175 result->name = kname;
176 len = strncpy_from_user(kname, filename, PATH_MAX);
177 if (unlikely(len < 0)) {
178 __putname(kname);
179 kfree(result);
180 return ERR_PTR(len);
181 }
182 if (unlikely(len == PATH_MAX)) {
183 __putname(kname);
184 kfree(result);
185 return ERR_PTR(-ENAMETOOLONG);
186 }
187 }
188
189 result->refcnt = 1;
190
191 if (unlikely(!len)) {
192 if (empty)
193 *empty = 1;
194 if (!(flags & LOOKUP_EMPTY)) {
195 putname(result);
196 return ERR_PTR(-ENOENT);
197 }
198 }
199
200 result->uptr = filename;
201 result->aname = NULL;
202 audit_getname(result);
203 return result;
204}
205
206struct filename *
207getname(const char __user * filename)
208{
209 return getname_flags(filename, 0, NULL);
210}
211
212struct filename *
213getname_kernel(const char * filename)
214{
215 struct filename *result;
216 int len = strlen(filename) + 1;
217
218 result = __getname();
219 if (unlikely(!result))
220 return ERR_PTR(-ENOMEM);
221
222 if (len <= EMBEDDED_NAME_MAX) {
223 result->name = (char *)result->iname;
224 } else if (len <= PATH_MAX) {
225 const size_t size = offsetof(struct filename, iname[1]);
226 struct filename *tmp;
227
228 tmp = kmalloc(size, GFP_KERNEL);
229 if (unlikely(!tmp)) {
230 __putname(result);
231 return ERR_PTR(-ENOMEM);
232 }
233 tmp->name = (char *)result;
234 result = tmp;
235 } else {
236 __putname(result);
237 return ERR_PTR(-ENAMETOOLONG);
238 }
239 memcpy((char *)result->name, filename, len);
240 result->uptr = NULL;
241 result->aname = NULL;
242 result->refcnt = 1;
243 audit_getname(result);
244
245 return result;
246}
247
248void putname(struct filename *name)
249{
250 BUG_ON(name->refcnt <= 0);
251
252 if (--name->refcnt > 0)
253 return;
254
255 if (name->name != name->iname) {
256 __putname(name->name);
257 kfree(name);
258 } else
259 __putname(name);
260}
261
262static int check_acl(struct inode *inode, int mask)
263{
264#ifdef CONFIG_FS_POSIX_ACL
265 struct posix_acl *acl;
266
267 if (mask & MAY_NOT_BLOCK) {
268 acl = get_cached_acl_rcu(inode, ACL_TYPE_ACCESS);
269 if (!acl)
270 return -EAGAIN;
271
272 if (is_uncached_acl(acl))
273 return -ECHILD;
274 return posix_acl_permission(inode, acl, mask);
275 }
276
277 acl = get_acl(inode, ACL_TYPE_ACCESS);
278 if (IS_ERR(acl))
279 return PTR_ERR(acl);
280 if (acl) {
281 int error = posix_acl_permission(inode, acl, mask);
282 posix_acl_release(acl);
283 return error;
284 }
285#endif
286
287 return -EAGAIN;
288}
289
290
291
292
293
294
295
296static int acl_permission_check(struct inode *inode, int mask)
297{
298 unsigned int mode = inode->i_mode;
299
300
301 if (likely(uid_eq(current_fsuid(), inode->i_uid))) {
302 mask &= 7;
303 mode >>= 6;
304 return (mask & ~mode) ? -EACCES : 0;
305 }
306
307
308 if (IS_POSIXACL(inode) && (mode & S_IRWXG)) {
309 int error = check_acl(inode, mask);
310 if (error != -EAGAIN)
311 return error;
312 }
313
314
315 mask &= 7;
316
317
318
319
320
321
322 if (mask & (mode ^ (mode >> 3))) {
323 if (in_group_p(inode->i_gid))
324 mode >>= 3;
325 }
326
327
328 return (mask & ~mode) ? -EACCES : 0;
329}
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346int generic_permission(struct inode *inode, int mask)
347{
348 int ret;
349
350
351
352
353 ret = acl_permission_check(inode, mask);
354 if (ret != -EACCES)
355 return ret;
356
357 if (S_ISDIR(inode->i_mode)) {
358
359 if (!(mask & MAY_WRITE))
360 if (capable_wrt_inode_uidgid(inode,
361 CAP_DAC_READ_SEARCH))
362 return 0;
363 if (capable_wrt_inode_uidgid(inode, CAP_DAC_OVERRIDE))
364 return 0;
365 return -EACCES;
366 }
367
368
369
370
371 mask &= MAY_READ | MAY_WRITE | MAY_EXEC;
372 if (mask == MAY_READ)
373 if (capable_wrt_inode_uidgid(inode, CAP_DAC_READ_SEARCH))
374 return 0;
375
376
377
378
379
380 if (!(mask & MAY_EXEC) || (inode->i_mode & S_IXUGO))
381 if (capable_wrt_inode_uidgid(inode, CAP_DAC_OVERRIDE))
382 return 0;
383
384 return -EACCES;
385}
386EXPORT_SYMBOL(generic_permission);
387
388
389
390
391
392
393
394static inline int do_inode_permission(struct inode *inode, int mask)
395{
396 if (unlikely(!(inode->i_opflags & IOP_FASTPERM))) {
397 if (likely(inode->i_op->permission))
398 return inode->i_op->permission(inode, mask);
399
400
401 spin_lock(&inode->i_lock);
402 inode->i_opflags |= IOP_FASTPERM;
403 spin_unlock(&inode->i_lock);
404 }
405 return generic_permission(inode, mask);
406}
407
408
409
410
411
412
413
414
415
416static int sb_permission(struct super_block *sb, struct inode *inode, int mask)
417{
418 if (unlikely(mask & MAY_WRITE)) {
419 umode_t mode = inode->i_mode;
420
421
422 if (sb_rdonly(sb) && (S_ISREG(mode) || S_ISDIR(mode) || S_ISLNK(mode)))
423 return -EROFS;
424 }
425 return 0;
426}
427
428
429
430
431
432
433
434
435
436
437
438
439int inode_permission(struct inode *inode, int mask)
440{
441 int retval;
442
443 retval = sb_permission(inode->i_sb, inode, mask);
444 if (retval)
445 return retval;
446
447 if (unlikely(mask & MAY_WRITE)) {
448
449
450
451 if (IS_IMMUTABLE(inode))
452 return -EPERM;
453
454
455
456
457
458
459 if (HAS_UNMAPPED_ID(inode))
460 return -EACCES;
461 }
462
463 retval = do_inode_permission(inode, mask);
464 if (retval)
465 return retval;
466
467 retval = devcgroup_inode_permission(inode, mask);
468 if (retval)
469 return retval;
470
471 return security_inode_permission(inode, mask);
472}
473EXPORT_SYMBOL(inode_permission);
474
475
476
477
478
479
480
481void path_get(const struct path *path)
482{
483 mntget(path->mnt);
484 dget(path->dentry);
485}
486EXPORT_SYMBOL(path_get);
487
488
489
490
491
492
493
494void path_put(const struct path *path)
495{
496 dput(path->dentry);
497 mntput(path->mnt);
498}
499EXPORT_SYMBOL(path_put);
500
501#define EMBEDDED_LEVELS 2
502struct nameidata {
503 struct path path;
504 struct qstr last;
505 struct path root;
506 struct inode *inode;
507 unsigned int flags;
508 unsigned seq, m_seq, r_seq;
509 int last_type;
510 unsigned depth;
511 int total_link_count;
512 struct saved {
513 struct path link;
514 struct delayed_call done;
515 const char *name;
516 unsigned seq;
517 } *stack, internal[EMBEDDED_LEVELS];
518 struct filename *name;
519 struct nameidata *saved;
520 unsigned root_seq;
521 int dfd;
522 kuid_t dir_uid;
523 umode_t dir_mode;
524} __randomize_layout;
525
526static void set_nameidata(struct nameidata *p, int dfd, struct filename *name)
527{
528 struct nameidata *old = current->nameidata;
529 p->stack = p->internal;
530 p->dfd = dfd;
531 p->name = name;
532 p->total_link_count = old ? old->total_link_count : 0;
533 p->saved = old;
534 current->nameidata = p;
535}
536
537static void restore_nameidata(void)
538{
539 struct nameidata *now = current->nameidata, *old = now->saved;
540
541 current->nameidata = old;
542 if (old)
543 old->total_link_count = now->total_link_count;
544 if (now->stack != now->internal)
545 kfree(now->stack);
546}
547
548static bool nd_alloc_stack(struct nameidata *nd)
549{
550 struct saved *p;
551
552 p= kmalloc_array(MAXSYMLINKS, sizeof(struct saved),
553 nd->flags & LOOKUP_RCU ? GFP_ATOMIC : GFP_KERNEL);
554 if (unlikely(!p))
555 return false;
556 memcpy(p, nd->internal, sizeof(nd->internal));
557 nd->stack = p;
558 return true;
559}
560
561
562
563
564
565
566
567static bool path_connected(struct vfsmount *mnt, struct dentry *dentry)
568{
569 struct super_block *sb = mnt->mnt_sb;
570
571
572 if (!(sb->s_iflags & SB_I_MULTIROOT) && (mnt->mnt_root == sb->s_root))
573 return true;
574
575 return is_subdir(dentry, mnt->mnt_root);
576}
577
578static void drop_links(struct nameidata *nd)
579{
580 int i = nd->depth;
581 while (i--) {
582 struct saved *last = nd->stack + i;
583 do_delayed_call(&last->done);
584 clear_delayed_call(&last->done);
585 }
586}
587
588static void terminate_walk(struct nameidata *nd)
589{
590 drop_links(nd);
591 if (!(nd->flags & LOOKUP_RCU)) {
592 int i;
593 path_put(&nd->path);
594 for (i = 0; i < nd->depth; i++)
595 path_put(&nd->stack[i].link);
596 if (nd->flags & LOOKUP_ROOT_GRABBED) {
597 path_put(&nd->root);
598 nd->flags &= ~LOOKUP_ROOT_GRABBED;
599 }
600 } else {
601 nd->flags &= ~LOOKUP_RCU;
602 rcu_read_unlock();
603 }
604 nd->depth = 0;
605}
606
607
608static bool __legitimize_path(struct path *path, unsigned seq, unsigned mseq)
609{
610 int res = __legitimize_mnt(path->mnt, mseq);
611 if (unlikely(res)) {
612 if (res > 0)
613 path->mnt = NULL;
614 path->dentry = NULL;
615 return false;
616 }
617 if (unlikely(!lockref_get_not_dead(&path->dentry->d_lockref))) {
618 path->dentry = NULL;
619 return false;
620 }
621 return !read_seqcount_retry(&path->dentry->d_seq, seq);
622}
623
624static inline bool legitimize_path(struct nameidata *nd,
625 struct path *path, unsigned seq)
626{
627 return __legitimize_path(path, seq, nd->m_seq);
628}
629
630static bool legitimize_links(struct nameidata *nd)
631{
632 int i;
633 for (i = 0; i < nd->depth; i++) {
634 struct saved *last = nd->stack + i;
635 if (unlikely(!legitimize_path(nd, &last->link, last->seq))) {
636 drop_links(nd);
637 nd->depth = i + 1;
638 return false;
639 }
640 }
641 return true;
642}
643
644static bool legitimize_root(struct nameidata *nd)
645{
646
647
648
649
650
651 if (!nd->root.mnt && (nd->flags & LOOKUP_IS_SCOPED))
652 return false;
653
654 if (!nd->root.mnt || (nd->flags & LOOKUP_ROOT))
655 return true;
656 nd->flags |= LOOKUP_ROOT_GRABBED;
657 return legitimize_path(nd, &nd->root, nd->root_seq);
658}
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682static int unlazy_walk(struct nameidata *nd)
683{
684 struct dentry *parent = nd->path.dentry;
685
686 BUG_ON(!(nd->flags & LOOKUP_RCU));
687
688 nd->flags &= ~LOOKUP_RCU;
689 if (unlikely(!legitimize_links(nd)))
690 goto out1;
691 if (unlikely(!legitimize_path(nd, &nd->path, nd->seq)))
692 goto out;
693 if (unlikely(!legitimize_root(nd)))
694 goto out;
695 rcu_read_unlock();
696 BUG_ON(nd->inode != parent->d_inode);
697 return 0;
698
699out1:
700 nd->path.mnt = NULL;
701 nd->path.dentry = NULL;
702out:
703 rcu_read_unlock();
704 return -ECHILD;
705}
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720static int unlazy_child(struct nameidata *nd, struct dentry *dentry, unsigned seq)
721{
722 BUG_ON(!(nd->flags & LOOKUP_RCU));
723
724 nd->flags &= ~LOOKUP_RCU;
725 if (unlikely(!legitimize_links(nd)))
726 goto out2;
727 if (unlikely(!legitimize_mnt(nd->path.mnt, nd->m_seq)))
728 goto out2;
729 if (unlikely(!lockref_get_not_dead(&nd->path.dentry->d_lockref)))
730 goto out1;
731
732
733
734
735
736
737
738
739 if (unlikely(!lockref_get_not_dead(&dentry->d_lockref)))
740 goto out;
741 if (unlikely(read_seqcount_retry(&dentry->d_seq, seq)))
742 goto out_dput;
743
744
745
746
747 if (unlikely(!legitimize_root(nd)))
748 goto out_dput;
749 rcu_read_unlock();
750 return 0;
751
752out2:
753 nd->path.mnt = NULL;
754out1:
755 nd->path.dentry = NULL;
756out:
757 rcu_read_unlock();
758 return -ECHILD;
759out_dput:
760 rcu_read_unlock();
761 dput(dentry);
762 return -ECHILD;
763}
764
765static inline int d_revalidate(struct dentry *dentry, unsigned int flags)
766{
767 if (unlikely(dentry->d_flags & DCACHE_OP_REVALIDATE))
768 return dentry->d_op->d_revalidate(dentry, flags);
769 else
770 return 1;
771}
772
773
774
775
776
777
778
779
780
781
782
783static int complete_walk(struct nameidata *nd)
784{
785 struct dentry *dentry = nd->path.dentry;
786 int status;
787
788 if (nd->flags & LOOKUP_RCU) {
789
790
791
792
793 if (!(nd->flags & (LOOKUP_ROOT | LOOKUP_IS_SCOPED)))
794 nd->root.mnt = NULL;
795 if (unlikely(unlazy_walk(nd)))
796 return -ECHILD;
797 }
798
799 if (unlikely(nd->flags & LOOKUP_IS_SCOPED)) {
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816 if (!path_is_under(&nd->path, &nd->root))
817 return -EXDEV;
818 }
819
820 if (likely(!(nd->flags & LOOKUP_JUMPED)))
821 return 0;
822
823 if (likely(!(dentry->d_flags & DCACHE_OP_WEAK_REVALIDATE)))
824 return 0;
825
826 status = dentry->d_op->d_weak_revalidate(dentry, nd->flags);
827 if (status > 0)
828 return 0;
829
830 if (!status)
831 status = -ESTALE;
832
833 return status;
834}
835
836static int set_root(struct nameidata *nd)
837{
838 struct fs_struct *fs = current->fs;
839
840
841
842
843
844
845 if (WARN_ON(nd->flags & LOOKUP_IS_SCOPED))
846 return -ENOTRECOVERABLE;
847
848 if (nd->flags & LOOKUP_RCU) {
849 unsigned seq;
850
851 do {
852 seq = read_seqcount_begin(&fs->seq);
853 nd->root = fs->root;
854 nd->root_seq = __read_seqcount_begin(&nd->root.dentry->d_seq);
855 } while (read_seqcount_retry(&fs->seq, seq));
856 } else {
857 get_fs_root(fs, &nd->root);
858 nd->flags |= LOOKUP_ROOT_GRABBED;
859 }
860 return 0;
861}
862
863static int nd_jump_root(struct nameidata *nd)
864{
865 if (unlikely(nd->flags & LOOKUP_BENEATH))
866 return -EXDEV;
867 if (unlikely(nd->flags & LOOKUP_NO_XDEV)) {
868
869 if (nd->path.mnt != NULL && nd->path.mnt != nd->root.mnt)
870 return -EXDEV;
871 }
872 if (!nd->root.mnt) {
873 int error = set_root(nd);
874 if (error)
875 return error;
876 }
877 if (nd->flags & LOOKUP_RCU) {
878 struct dentry *d;
879 nd->path = nd->root;
880 d = nd->path.dentry;
881 nd->inode = d->d_inode;
882 nd->seq = nd->root_seq;
883 if (unlikely(read_seqcount_retry(&d->d_seq, nd->seq)))
884 return -ECHILD;
885 } else {
886 path_put(&nd->path);
887 nd->path = nd->root;
888 path_get(&nd->path);
889 nd->inode = nd->path.dentry->d_inode;
890 }
891 nd->flags |= LOOKUP_JUMPED;
892 return 0;
893}
894
895
896
897
898
899int nd_jump_link(struct path *path)
900{
901 int error = -ELOOP;
902 struct nameidata *nd = current->nameidata;
903
904 if (unlikely(nd->flags & LOOKUP_NO_MAGICLINKS))
905 goto err;
906
907 error = -EXDEV;
908 if (unlikely(nd->flags & LOOKUP_NO_XDEV)) {
909 if (nd->path.mnt != path->mnt)
910 goto err;
911 }
912
913 if (unlikely(nd->flags & LOOKUP_IS_SCOPED))
914 goto err;
915
916 path_put(&nd->path);
917 nd->path = *path;
918 nd->inode = nd->path.dentry->d_inode;
919 nd->flags |= LOOKUP_JUMPED;
920 return 0;
921
922err:
923 path_put(path);
924 return error;
925}
926
927static inline void put_link(struct nameidata *nd)
928{
929 struct saved *last = nd->stack + --nd->depth;
930 do_delayed_call(&last->done);
931 if (!(nd->flags & LOOKUP_RCU))
932 path_put(&last->link);
933}
934
935int sysctl_protected_symlinks __read_mostly = 0;
936int sysctl_protected_hardlinks __read_mostly = 0;
937int sysctl_protected_fifos __read_mostly;
938int sysctl_protected_regular __read_mostly;
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955static inline int may_follow_link(struct nameidata *nd, const struct inode *inode)
956{
957 if (!sysctl_protected_symlinks)
958 return 0;
959
960
961 if (uid_eq(current_cred()->fsuid, inode->i_uid))
962 return 0;
963
964
965 if ((nd->dir_mode & (S_ISVTX|S_IWOTH)) != (S_ISVTX|S_IWOTH))
966 return 0;
967
968
969 if (uid_valid(nd->dir_uid) && uid_eq(nd->dir_uid, inode->i_uid))
970 return 0;
971
972 if (nd->flags & LOOKUP_RCU)
973 return -ECHILD;
974
975 audit_inode(nd->name, nd->stack[0].link.dentry, 0);
976 audit_log_path_denied(AUDIT_ANOM_LINK, "follow_link");
977 return -EACCES;
978}
979
980
981
982
983
984
985
986
987
988
989
990
991
992static bool safe_hardlink_source(struct inode *inode)
993{
994 umode_t mode = inode->i_mode;
995
996
997 if (!S_ISREG(mode))
998 return false;
999
1000
1001 if (mode & S_ISUID)
1002 return false;
1003
1004
1005 if ((mode & (S_ISGID | S_IXGRP)) == (S_ISGID | S_IXGRP))
1006 return false;
1007
1008
1009 if (inode_permission(inode, MAY_READ | MAY_WRITE))
1010 return false;
1011
1012 return true;
1013}
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027int may_linkat(struct path *link)
1028{
1029 struct inode *inode = link->dentry->d_inode;
1030
1031
1032 if (!uid_valid(inode->i_uid) || !gid_valid(inode->i_gid))
1033 return -EOVERFLOW;
1034
1035 if (!sysctl_protected_hardlinks)
1036 return 0;
1037
1038
1039
1040
1041 if (safe_hardlink_source(inode) || inode_owner_or_capable(inode))
1042 return 0;
1043
1044 audit_log_path_denied(AUDIT_ANOM_LINK, "linkat");
1045 return -EPERM;
1046}
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069static int may_create_in_sticky(umode_t dir_mode, kuid_t dir_uid,
1070 struct inode * const inode)
1071{
1072 if ((!sysctl_protected_fifos && S_ISFIFO(inode->i_mode)) ||
1073 (!sysctl_protected_regular && S_ISREG(inode->i_mode)) ||
1074 likely(!(dir_mode & S_ISVTX)) ||
1075 uid_eq(inode->i_uid, dir_uid) ||
1076 uid_eq(current_fsuid(), inode->i_uid))
1077 return 0;
1078
1079 if (likely(dir_mode & 0002) ||
1080 (dir_mode & 0020 &&
1081 ((sysctl_protected_fifos >= 2 && S_ISFIFO(inode->i_mode)) ||
1082 (sysctl_protected_regular >= 2 && S_ISREG(inode->i_mode))))) {
1083 const char *operation = S_ISFIFO(inode->i_mode) ?
1084 "sticky_create_fifo" :
1085 "sticky_create_regular";
1086 audit_log_path_denied(AUDIT_ANOM_CREAT, operation);
1087 return -EACCES;
1088 }
1089 return 0;
1090}
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102int follow_up(struct path *path)
1103{
1104 struct mount *mnt = real_mount(path->mnt);
1105 struct mount *parent;
1106 struct dentry *mountpoint;
1107
1108 read_seqlock_excl(&mount_lock);
1109 parent = mnt->mnt_parent;
1110 if (parent == mnt) {
1111 read_sequnlock_excl(&mount_lock);
1112 return 0;
1113 }
1114 mntget(&parent->mnt);
1115 mountpoint = dget(mnt->mnt_mountpoint);
1116 read_sequnlock_excl(&mount_lock);
1117 dput(path->dentry);
1118 path->dentry = mountpoint;
1119 mntput(path->mnt);
1120 path->mnt = &parent->mnt;
1121 return 1;
1122}
1123EXPORT_SYMBOL(follow_up);
1124
1125static bool choose_mountpoint_rcu(struct mount *m, const struct path *root,
1126 struct path *path, unsigned *seqp)
1127{
1128 while (mnt_has_parent(m)) {
1129 struct dentry *mountpoint = m->mnt_mountpoint;
1130
1131 m = m->mnt_parent;
1132 if (unlikely(root->dentry == mountpoint &&
1133 root->mnt == &m->mnt))
1134 break;
1135 if (mountpoint != m->mnt.mnt_root) {
1136 path->mnt = &m->mnt;
1137 path->dentry = mountpoint;
1138 *seqp = read_seqcount_begin(&mountpoint->d_seq);
1139 return true;
1140 }
1141 }
1142 return false;
1143}
1144
1145static bool choose_mountpoint(struct mount *m, const struct path *root,
1146 struct path *path)
1147{
1148 bool found;
1149
1150 rcu_read_lock();
1151 while (1) {
1152 unsigned seq, mseq = read_seqbegin(&mount_lock);
1153
1154 found = choose_mountpoint_rcu(m, root, path, &seq);
1155 if (unlikely(!found)) {
1156 if (!read_seqretry(&mount_lock, mseq))
1157 break;
1158 } else {
1159 if (likely(__legitimize_path(path, seq, mseq)))
1160 break;
1161 rcu_read_unlock();
1162 path_put(path);
1163 rcu_read_lock();
1164 }
1165 }
1166 rcu_read_unlock();
1167 return found;
1168}
1169
1170
1171
1172
1173
1174
1175static int follow_automount(struct path *path, int *count, unsigned lookup_flags)
1176{
1177 struct dentry *dentry = path->dentry;
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190 if (!(lookup_flags & (LOOKUP_PARENT | LOOKUP_DIRECTORY |
1191 LOOKUP_OPEN | LOOKUP_CREATE | LOOKUP_AUTOMOUNT)) &&
1192 dentry->d_inode)
1193 return -EISDIR;
1194
1195 if (count && (*count)++ >= MAXSYMLINKS)
1196 return -ELOOP;
1197
1198 return finish_automount(dentry->d_op->d_automount(path), path);
1199}
1200
1201
1202
1203
1204
1205
1206
1207static int __traverse_mounts(struct path *path, unsigned flags, bool *jumped,
1208 int *count, unsigned lookup_flags)
1209{
1210 struct vfsmount *mnt = path->mnt;
1211 bool need_mntput = false;
1212 int ret = 0;
1213
1214 while (flags & DCACHE_MANAGED_DENTRY) {
1215
1216
1217 if (flags & DCACHE_MANAGE_TRANSIT) {
1218 ret = path->dentry->d_op->d_manage(path, false);
1219 flags = smp_load_acquire(&path->dentry->d_flags);
1220 if (ret < 0)
1221 break;
1222 }
1223
1224 if (flags & DCACHE_MOUNTED) {
1225 struct vfsmount *mounted = lookup_mnt(path);
1226 if (mounted) {
1227 dput(path->dentry);
1228 if (need_mntput)
1229 mntput(path->mnt);
1230 path->mnt = mounted;
1231 path->dentry = dget(mounted->mnt_root);
1232
1233 flags = path->dentry->d_flags;
1234 need_mntput = true;
1235 continue;
1236 }
1237 }
1238
1239 if (!(flags & DCACHE_NEED_AUTOMOUNT))
1240 break;
1241
1242
1243 ret = follow_automount(path, count, lookup_flags);
1244 flags = smp_load_acquire(&path->dentry->d_flags);
1245 if (ret < 0)
1246 break;
1247 }
1248
1249 if (ret == -EISDIR)
1250 ret = 0;
1251
1252 if (need_mntput && path->mnt == mnt)
1253 mntput(path->mnt);
1254 if (!ret && unlikely(d_flags_negative(flags)))
1255 ret = -ENOENT;
1256 *jumped = need_mntput;
1257 return ret;
1258}
1259
1260static inline int traverse_mounts(struct path *path, bool *jumped,
1261 int *count, unsigned lookup_flags)
1262{
1263 unsigned flags = smp_load_acquire(&path->dentry->d_flags);
1264
1265
1266 if (likely(!(flags & DCACHE_MANAGED_DENTRY))) {
1267 *jumped = false;
1268 if (unlikely(d_flags_negative(flags)))
1269 return -ENOENT;
1270 return 0;
1271 }
1272 return __traverse_mounts(path, flags, jumped, count, lookup_flags);
1273}
1274
1275int follow_down_one(struct path *path)
1276{
1277 struct vfsmount *mounted;
1278
1279 mounted = lookup_mnt(path);
1280 if (mounted) {
1281 dput(path->dentry);
1282 mntput(path->mnt);
1283 path->mnt = mounted;
1284 path->dentry = dget(mounted->mnt_root);
1285 return 1;
1286 }
1287 return 0;
1288}
1289EXPORT_SYMBOL(follow_down_one);
1290
1291
1292
1293
1294
1295
1296int follow_down(struct path *path)
1297{
1298 struct vfsmount *mnt = path->mnt;
1299 bool jumped;
1300 int ret = traverse_mounts(path, &jumped, NULL, 0);
1301
1302 if (path->mnt != mnt)
1303 mntput(mnt);
1304 return ret;
1305}
1306EXPORT_SYMBOL(follow_down);
1307
1308
1309
1310
1311
1312static bool __follow_mount_rcu(struct nameidata *nd, struct path *path,
1313 struct inode **inode, unsigned *seqp)
1314{
1315 struct dentry *dentry = path->dentry;
1316 unsigned int flags = dentry->d_flags;
1317
1318 if (likely(!(flags & DCACHE_MANAGED_DENTRY)))
1319 return true;
1320
1321 if (unlikely(nd->flags & LOOKUP_NO_XDEV))
1322 return false;
1323
1324 for (;;) {
1325
1326
1327
1328
1329 if (unlikely(flags & DCACHE_MANAGE_TRANSIT)) {
1330 int res = dentry->d_op->d_manage(path, true);
1331 if (res)
1332 return res == -EISDIR;
1333 flags = dentry->d_flags;
1334 }
1335
1336 if (flags & DCACHE_MOUNTED) {
1337 struct mount *mounted = __lookup_mnt(path->mnt, dentry);
1338 if (mounted) {
1339 path->mnt = &mounted->mnt;
1340 dentry = path->dentry = mounted->mnt.mnt_root;
1341 nd->flags |= LOOKUP_JUMPED;
1342 *seqp = read_seqcount_begin(&dentry->d_seq);
1343 *inode = dentry->d_inode;
1344
1345
1346
1347
1348
1349
1350 flags = dentry->d_flags;
1351 continue;
1352 }
1353 if (read_seqretry(&mount_lock, nd->m_seq))
1354 return false;
1355 }
1356 return !(flags & DCACHE_NEED_AUTOMOUNT);
1357 }
1358}
1359
1360static inline int handle_mounts(struct nameidata *nd, struct dentry *dentry,
1361 struct path *path, struct inode **inode,
1362 unsigned int *seqp)
1363{
1364 bool jumped;
1365 int ret;
1366
1367 path->mnt = nd->path.mnt;
1368 path->dentry = dentry;
1369 if (nd->flags & LOOKUP_RCU) {
1370 unsigned int seq = *seqp;
1371 if (unlikely(!*inode))
1372 return -ENOENT;
1373 if (likely(__follow_mount_rcu(nd, path, inode, seqp)))
1374 return 0;
1375 if (unlazy_child(nd, dentry, seq))
1376 return -ECHILD;
1377
1378 path->mnt = nd->path.mnt;
1379 path->dentry = dentry;
1380 }
1381 ret = traverse_mounts(path, &jumped, &nd->total_link_count, nd->flags);
1382 if (jumped) {
1383 if (unlikely(nd->flags & LOOKUP_NO_XDEV))
1384 ret = -EXDEV;
1385 else
1386 nd->flags |= LOOKUP_JUMPED;
1387 }
1388 if (unlikely(ret)) {
1389 dput(path->dentry);
1390 if (path->mnt != nd->path.mnt)
1391 mntput(path->mnt);
1392 } else {
1393 *inode = d_backing_inode(path->dentry);
1394 *seqp = 0;
1395 }
1396 return ret;
1397}
1398
1399
1400
1401
1402
1403static struct dentry *lookup_dcache(const struct qstr *name,
1404 struct dentry *dir,
1405 unsigned int flags)
1406{
1407 struct dentry *dentry = d_lookup(dir, name);
1408 if (dentry) {
1409 int error = d_revalidate(dentry, flags);
1410 if (unlikely(error <= 0)) {
1411 if (!error)
1412 d_invalidate(dentry);
1413 dput(dentry);
1414 return ERR_PTR(error);
1415 }
1416 }
1417 return dentry;
1418}
1419
1420
1421
1422
1423
1424
1425
1426
1427static struct dentry *__lookup_hash(const struct qstr *name,
1428 struct dentry *base, unsigned int flags)
1429{
1430 struct dentry *dentry = lookup_dcache(name, base, flags);
1431 struct dentry *old;
1432 struct inode *dir = base->d_inode;
1433
1434 if (dentry)
1435 return dentry;
1436
1437
1438 if (unlikely(IS_DEADDIR(dir)))
1439 return ERR_PTR(-ENOENT);
1440
1441 dentry = d_alloc(base, name);
1442 if (unlikely(!dentry))
1443 return ERR_PTR(-ENOMEM);
1444
1445 old = dir->i_op->lookup(dir, dentry, flags);
1446 if (unlikely(old)) {
1447 dput(dentry);
1448 dentry = old;
1449 }
1450 return dentry;
1451}
1452
1453static struct dentry *lookup_fast(struct nameidata *nd,
1454 struct inode **inode,
1455 unsigned *seqp)
1456{
1457 struct dentry *dentry, *parent = nd->path.dentry;
1458 int status = 1;
1459
1460
1461
1462
1463
1464
1465 if (nd->flags & LOOKUP_RCU) {
1466 unsigned seq;
1467 dentry = __d_lookup_rcu(parent, &nd->last, &seq);
1468 if (unlikely(!dentry)) {
1469 if (unlazy_walk(nd))
1470 return ERR_PTR(-ECHILD);
1471 return NULL;
1472 }
1473
1474
1475
1476
1477
1478 *inode = d_backing_inode(dentry);
1479 if (unlikely(read_seqcount_retry(&dentry->d_seq, seq)))
1480 return ERR_PTR(-ECHILD);
1481
1482
1483
1484
1485
1486
1487
1488
1489 if (unlikely(__read_seqcount_retry(&parent->d_seq, nd->seq)))
1490 return ERR_PTR(-ECHILD);
1491
1492 *seqp = seq;
1493 status = d_revalidate(dentry, nd->flags);
1494 if (likely(status > 0))
1495 return dentry;
1496 if (unlazy_child(nd, dentry, seq))
1497 return ERR_PTR(-ECHILD);
1498 if (unlikely(status == -ECHILD))
1499
1500 status = d_revalidate(dentry, nd->flags);
1501 } else {
1502 dentry = __d_lookup(parent, &nd->last);
1503 if (unlikely(!dentry))
1504 return NULL;
1505 status = d_revalidate(dentry, nd->flags);
1506 }
1507 if (unlikely(status <= 0)) {
1508 if (!status)
1509 d_invalidate(dentry);
1510 dput(dentry);
1511 return ERR_PTR(status);
1512 }
1513 return dentry;
1514}
1515
1516
1517static struct dentry *__lookup_slow(const struct qstr *name,
1518 struct dentry *dir,
1519 unsigned int flags)
1520{
1521 struct dentry *dentry, *old;
1522 struct inode *inode = dir->d_inode;
1523 DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wq);
1524
1525
1526 if (unlikely(IS_DEADDIR(inode)))
1527 return ERR_PTR(-ENOENT);
1528again:
1529 dentry = d_alloc_parallel(dir, name, &wq);
1530 if (IS_ERR(dentry))
1531 return dentry;
1532 if (unlikely(!d_in_lookup(dentry))) {
1533 int error = d_revalidate(dentry, flags);
1534 if (unlikely(error <= 0)) {
1535 if (!error) {
1536 d_invalidate(dentry);
1537 dput(dentry);
1538 goto again;
1539 }
1540 dput(dentry);
1541 dentry = ERR_PTR(error);
1542 }
1543 } else {
1544 old = inode->i_op->lookup(inode, dentry, flags);
1545 d_lookup_done(dentry);
1546 if (unlikely(old)) {
1547 dput(dentry);
1548 dentry = old;
1549 }
1550 }
1551 return dentry;
1552}
1553
1554static struct dentry *lookup_slow(const struct qstr *name,
1555 struct dentry *dir,
1556 unsigned int flags)
1557{
1558 struct inode *inode = dir->d_inode;
1559 struct dentry *res;
1560 inode_lock_shared(inode);
1561 res = __lookup_slow(name, dir, flags);
1562 inode_unlock_shared(inode);
1563 return res;
1564}
1565
1566static inline int may_lookup(struct nameidata *nd)
1567{
1568 if (nd->flags & LOOKUP_RCU) {
1569 int err = inode_permission(nd->inode, MAY_EXEC|MAY_NOT_BLOCK);
1570 if (err != -ECHILD)
1571 return err;
1572 if (unlazy_walk(nd))
1573 return -ECHILD;
1574 }
1575 return inode_permission(nd->inode, MAY_EXEC);
1576}
1577
1578static int reserve_stack(struct nameidata *nd, struct path *link, unsigned seq)
1579{
1580 if (unlikely(nd->total_link_count++ >= MAXSYMLINKS))
1581 return -ELOOP;
1582
1583 if (likely(nd->depth != EMBEDDED_LEVELS))
1584 return 0;
1585 if (likely(nd->stack != nd->internal))
1586 return 0;
1587 if (likely(nd_alloc_stack(nd)))
1588 return 0;
1589
1590 if (nd->flags & LOOKUP_RCU) {
1591
1592
1593 bool grabbed_link = legitimize_path(nd, link, seq);
1594
1595 if (unlazy_walk(nd) != 0 || !grabbed_link)
1596 return -ECHILD;
1597
1598 if (nd_alloc_stack(nd))
1599 return 0;
1600 }
1601 return -ENOMEM;
1602}
1603
1604enum {WALK_TRAILING = 1, WALK_MORE = 2, WALK_NOFOLLOW = 4};
1605
1606static const char *pick_link(struct nameidata *nd, struct path *link,
1607 struct inode *inode, unsigned seq, int flags)
1608{
1609 struct saved *last;
1610 const char *res;
1611 int error = reserve_stack(nd, link, seq);
1612
1613 if (unlikely(error)) {
1614 if (!(nd->flags & LOOKUP_RCU))
1615 path_put(link);
1616 return ERR_PTR(error);
1617 }
1618 last = nd->stack + nd->depth++;
1619 last->link = *link;
1620 clear_delayed_call(&last->done);
1621 last->seq = seq;
1622
1623 if (flags & WALK_TRAILING) {
1624 error = may_follow_link(nd, inode);
1625 if (unlikely(error))
1626 return ERR_PTR(error);
1627 }
1628
1629 if (unlikely(nd->flags & LOOKUP_NO_SYMLINKS))
1630 return ERR_PTR(-ELOOP);
1631
1632 if (!(nd->flags & LOOKUP_RCU)) {
1633 touch_atime(&last->link);
1634 cond_resched();
1635 } else if (atime_needs_update(&last->link, inode)) {
1636 if (unlikely(unlazy_walk(nd)))
1637 return ERR_PTR(-ECHILD);
1638 touch_atime(&last->link);
1639 }
1640
1641 error = security_inode_follow_link(link->dentry, inode,
1642 nd->flags & LOOKUP_RCU);
1643 if (unlikely(error))
1644 return ERR_PTR(error);
1645
1646 res = READ_ONCE(inode->i_link);
1647 if (!res) {
1648 const char * (*get)(struct dentry *, struct inode *,
1649 struct delayed_call *);
1650 get = inode->i_op->get_link;
1651 if (nd->flags & LOOKUP_RCU) {
1652 res = get(NULL, inode, &last->done);
1653 if (res == ERR_PTR(-ECHILD)) {
1654 if (unlikely(unlazy_walk(nd)))
1655 return ERR_PTR(-ECHILD);
1656 res = get(link->dentry, inode, &last->done);
1657 }
1658 } else {
1659 res = get(link->dentry, inode, &last->done);
1660 }
1661 if (!res)
1662 goto all_done;
1663 if (IS_ERR(res))
1664 return res;
1665 }
1666 if (*res == '/') {
1667 error = nd_jump_root(nd);
1668 if (unlikely(error))
1669 return ERR_PTR(error);
1670 while (unlikely(*++res == '/'))
1671 ;
1672 }
1673 if (*res)
1674 return res;
1675all_done:
1676 put_link(nd);
1677 return NULL;
1678}
1679
1680
1681
1682
1683
1684
1685
1686static const char *step_into(struct nameidata *nd, int flags,
1687 struct dentry *dentry, struct inode *inode, unsigned seq)
1688{
1689 struct path path;
1690 int err = handle_mounts(nd, dentry, &path, &inode, &seq);
1691
1692 if (err < 0)
1693 return ERR_PTR(err);
1694 if (likely(!d_is_symlink(path.dentry)) ||
1695 ((flags & WALK_TRAILING) && !(nd->flags & LOOKUP_FOLLOW)) ||
1696 (flags & WALK_NOFOLLOW)) {
1697
1698 if (!(nd->flags & LOOKUP_RCU)) {
1699 dput(nd->path.dentry);
1700 if (nd->path.mnt != path.mnt)
1701 mntput(nd->path.mnt);
1702 }
1703 nd->path = path;
1704 nd->inode = inode;
1705 nd->seq = seq;
1706 return NULL;
1707 }
1708 if (nd->flags & LOOKUP_RCU) {
1709
1710 if (read_seqcount_retry(&path.dentry->d_seq, seq))
1711 return ERR_PTR(-ECHILD);
1712 } else {
1713 if (path.mnt == nd->path.mnt)
1714 mntget(path.mnt);
1715 }
1716 return pick_link(nd, &path, inode, seq, flags);
1717}
1718
1719static struct dentry *follow_dotdot_rcu(struct nameidata *nd,
1720 struct inode **inodep,
1721 unsigned *seqp)
1722{
1723 struct dentry *parent, *old;
1724
1725 if (path_equal(&nd->path, &nd->root))
1726 goto in_root;
1727 if (unlikely(nd->path.dentry == nd->path.mnt->mnt_root)) {
1728 struct path path;
1729 unsigned seq;
1730 if (!choose_mountpoint_rcu(real_mount(nd->path.mnt),
1731 &nd->root, &path, &seq))
1732 goto in_root;
1733 if (unlikely(nd->flags & LOOKUP_NO_XDEV))
1734 return ERR_PTR(-ECHILD);
1735 nd->path = path;
1736 nd->inode = path.dentry->d_inode;
1737 nd->seq = seq;
1738 if (unlikely(read_seqretry(&mount_lock, nd->m_seq)))
1739 return ERR_PTR(-ECHILD);
1740
1741 }
1742 old = nd->path.dentry;
1743 parent = old->d_parent;
1744 *inodep = parent->d_inode;
1745 *seqp = read_seqcount_begin(&parent->d_seq);
1746 if (unlikely(read_seqcount_retry(&old->d_seq, nd->seq)))
1747 return ERR_PTR(-ECHILD);
1748 if (unlikely(!path_connected(nd->path.mnt, parent)))
1749 return ERR_PTR(-ECHILD);
1750 return parent;
1751in_root:
1752 if (unlikely(read_seqretry(&mount_lock, nd->m_seq)))
1753 return ERR_PTR(-ECHILD);
1754 if (unlikely(nd->flags & LOOKUP_BENEATH))
1755 return ERR_PTR(-ECHILD);
1756 return NULL;
1757}
1758
1759static struct dentry *follow_dotdot(struct nameidata *nd,
1760 struct inode **inodep,
1761 unsigned *seqp)
1762{
1763 struct dentry *parent;
1764
1765 if (path_equal(&nd->path, &nd->root))
1766 goto in_root;
1767 if (unlikely(nd->path.dentry == nd->path.mnt->mnt_root)) {
1768 struct path path;
1769
1770 if (!choose_mountpoint(real_mount(nd->path.mnt),
1771 &nd->root, &path))
1772 goto in_root;
1773 path_put(&nd->path);
1774 nd->path = path;
1775 nd->inode = path.dentry->d_inode;
1776 if (unlikely(nd->flags & LOOKUP_NO_XDEV))
1777 return ERR_PTR(-EXDEV);
1778 }
1779
1780 parent = dget_parent(nd->path.dentry);
1781 if (unlikely(!path_connected(nd->path.mnt, parent))) {
1782 dput(parent);
1783 return ERR_PTR(-ENOENT);
1784 }
1785 *seqp = 0;
1786 *inodep = parent->d_inode;
1787 return parent;
1788
1789in_root:
1790 if (unlikely(nd->flags & LOOKUP_BENEATH))
1791 return ERR_PTR(-EXDEV);
1792 dget(nd->path.dentry);
1793 return NULL;
1794}
1795
1796static const char *handle_dots(struct nameidata *nd, int type)
1797{
1798 if (type == LAST_DOTDOT) {
1799 const char *error = NULL;
1800 struct dentry *parent;
1801 struct inode *inode;
1802 unsigned seq;
1803
1804 if (!nd->root.mnt) {
1805 error = ERR_PTR(set_root(nd));
1806 if (error)
1807 return error;
1808 }
1809 if (nd->flags & LOOKUP_RCU)
1810 parent = follow_dotdot_rcu(nd, &inode, &seq);
1811 else
1812 parent = follow_dotdot(nd, &inode, &seq);
1813 if (IS_ERR(parent))
1814 return ERR_CAST(parent);
1815 if (unlikely(!parent))
1816 error = step_into(nd, WALK_NOFOLLOW,
1817 nd->path.dentry, nd->inode, nd->seq);
1818 else
1819 error = step_into(nd, WALK_NOFOLLOW,
1820 parent, inode, seq);
1821 if (unlikely(error))
1822 return error;
1823
1824 if (unlikely(nd->flags & LOOKUP_IS_SCOPED)) {
1825
1826
1827
1828
1829
1830
1831 smp_rmb();
1832 if (unlikely(__read_seqcount_retry(&mount_lock.seqcount, nd->m_seq)))
1833 return ERR_PTR(-EAGAIN);
1834 if (unlikely(__read_seqcount_retry(&rename_lock.seqcount, nd->r_seq)))
1835 return ERR_PTR(-EAGAIN);
1836 }
1837 }
1838 return NULL;
1839}
1840
1841static const char *walk_component(struct nameidata *nd, int flags)
1842{
1843 struct dentry *dentry;
1844 struct inode *inode;
1845 unsigned seq;
1846
1847
1848
1849
1850
1851 if (unlikely(nd->last_type != LAST_NORM)) {
1852 if (!(flags & WALK_MORE) && nd->depth)
1853 put_link(nd);
1854 return handle_dots(nd, nd->last_type);
1855 }
1856 dentry = lookup_fast(nd, &inode, &seq);
1857 if (IS_ERR(dentry))
1858 return ERR_CAST(dentry);
1859 if (unlikely(!dentry)) {
1860 dentry = lookup_slow(&nd->last, nd->path.dentry, nd->flags);
1861 if (IS_ERR(dentry))
1862 return ERR_CAST(dentry);
1863 }
1864 if (!(flags & WALK_MORE) && nd->depth)
1865 put_link(nd);
1866 return step_into(nd, flags, dentry, inode, seq);
1867}
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886#ifdef CONFIG_DCACHE_WORD_ACCESS
1887
1888#include <asm/word-at-a-time.h>
1889
1890#ifdef HASH_MIX
1891
1892
1893
1894#elif defined(CONFIG_64BIT)
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924#define HASH_MIX(x, y, a) \
1925 ( x ^= (a), \
1926 y ^= x, x = rol64(x,12),\
1927 x += y, y = rol64(y,45),\
1928 y *= 9 )
1929
1930
1931
1932
1933
1934
1935static inline unsigned int fold_hash(unsigned long x, unsigned long y)
1936{
1937 y ^= x * GOLDEN_RATIO_64;
1938 y *= GOLDEN_RATIO_64;
1939 return y >> 32;
1940}
1941
1942#else
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954#define HASH_MIX(x, y, a) \
1955 ( x ^= (a), \
1956 y ^= x, x = rol32(x, 7),\
1957 x += y, y = rol32(y,20),\
1958 y *= 9 )
1959
1960static inline unsigned int fold_hash(unsigned long x, unsigned long y)
1961{
1962
1963 return __hash_32(y ^ __hash_32(x));
1964}
1965
1966#endif
1967
1968
1969
1970
1971
1972
1973
1974
1975unsigned int full_name_hash(const void *salt, const char *name, unsigned int len)
1976{
1977 unsigned long a, x = 0, y = (unsigned long)salt;
1978
1979 for (;;) {
1980 if (!len)
1981 goto done;
1982 a = load_unaligned_zeropad(name);
1983 if (len < sizeof(unsigned long))
1984 break;
1985 HASH_MIX(x, y, a);
1986 name += sizeof(unsigned long);
1987 len -= sizeof(unsigned long);
1988 }
1989 x ^= a & bytemask_from_count(len);
1990done:
1991 return fold_hash(x, y);
1992}
1993EXPORT_SYMBOL(full_name_hash);
1994
1995
1996u64 hashlen_string(const void *salt, const char *name)
1997{
1998 unsigned long a = 0, x = 0, y = (unsigned long)salt;
1999 unsigned long adata, mask, len;
2000 const struct word_at_a_time constants = WORD_AT_A_TIME_CONSTANTS;
2001
2002 len = 0;
2003 goto inside;
2004
2005 do {
2006 HASH_MIX(x, y, a);
2007 len += sizeof(unsigned long);
2008inside:
2009 a = load_unaligned_zeropad(name+len);
2010 } while (!has_zero(a, &adata, &constants));
2011
2012 adata = prep_zero_mask(a, adata, &constants);
2013 mask = create_zero_mask(adata);
2014 x ^= a & zero_bytemask(mask);
2015
2016 return hashlen_create(fold_hash(x, y), len + find_zero(mask));
2017}
2018EXPORT_SYMBOL(hashlen_string);
2019
2020
2021
2022
2023
2024static inline u64 hash_name(const void *salt, const char *name)
2025{
2026 unsigned long a = 0, b, x = 0, y = (unsigned long)salt;
2027 unsigned long adata, bdata, mask, len;
2028 const struct word_at_a_time constants = WORD_AT_A_TIME_CONSTANTS;
2029
2030 len = 0;
2031 goto inside;
2032
2033 do {
2034 HASH_MIX(x, y, a);
2035 len += sizeof(unsigned long);
2036inside:
2037 a = load_unaligned_zeropad(name+len);
2038 b = a ^ REPEAT_BYTE('/');
2039 } while (!(has_zero(a, &adata, &constants) | has_zero(b, &bdata, &constants)));
2040
2041 adata = prep_zero_mask(a, adata, &constants);
2042 bdata = prep_zero_mask(b, bdata, &constants);
2043 mask = create_zero_mask(adata | bdata);
2044 x ^= a & zero_bytemask(mask);
2045
2046 return hashlen_create(fold_hash(x, y), len + find_zero(mask));
2047}
2048
2049#else
2050
2051
2052unsigned int full_name_hash(const void *salt, const char *name, unsigned int len)
2053{
2054 unsigned long hash = init_name_hash(salt);
2055 while (len--)
2056 hash = partial_name_hash((unsigned char)*name++, hash);
2057 return end_name_hash(hash);
2058}
2059EXPORT_SYMBOL(full_name_hash);
2060
2061
2062u64 hashlen_string(const void *salt, const char *name)
2063{
2064 unsigned long hash = init_name_hash(salt);
2065 unsigned long len = 0, c;
2066
2067 c = (unsigned char)*name;
2068 while (c) {
2069 len++;
2070 hash = partial_name_hash(c, hash);
2071 c = (unsigned char)name[len];
2072 }
2073 return hashlen_create(end_name_hash(hash), len);
2074}
2075EXPORT_SYMBOL(hashlen_string);
2076
2077
2078
2079
2080
2081static inline u64 hash_name(const void *salt, const char *name)
2082{
2083 unsigned long hash = init_name_hash(salt);
2084 unsigned long len = 0, c;
2085
2086 c = (unsigned char)*name;
2087 do {
2088 len++;
2089 hash = partial_name_hash(c, hash);
2090 c = (unsigned char)name[len];
2091 } while (c && c != '/');
2092 return hashlen_create(end_name_hash(hash), len);
2093}
2094
2095#endif
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105static int link_path_walk(const char *name, struct nameidata *nd)
2106{
2107 int depth = 0;
2108 int err;
2109
2110 nd->last_type = LAST_ROOT;
2111 nd->flags |= LOOKUP_PARENT;
2112 if (IS_ERR(name))
2113 return PTR_ERR(name);
2114 while (*name=='/')
2115 name++;
2116 if (!*name)
2117 return 0;
2118
2119
2120 for(;;) {
2121 const char *link;
2122 u64 hash_len;
2123 int type;
2124
2125 err = may_lookup(nd);
2126 if (err)
2127 return err;
2128
2129 hash_len = hash_name(nd->path.dentry, name);
2130
2131 type = LAST_NORM;
2132 if (name[0] == '.') switch (hashlen_len(hash_len)) {
2133 case 2:
2134 if (name[1] == '.') {
2135 type = LAST_DOTDOT;
2136 nd->flags |= LOOKUP_JUMPED;
2137 }
2138 break;
2139 case 1:
2140 type = LAST_DOT;
2141 }
2142 if (likely(type == LAST_NORM)) {
2143 struct dentry *parent = nd->path.dentry;
2144 nd->flags &= ~LOOKUP_JUMPED;
2145 if (unlikely(parent->d_flags & DCACHE_OP_HASH)) {
2146 struct qstr this = { { .hash_len = hash_len }, .name = name };
2147 err = parent->d_op->d_hash(parent, &this);
2148 if (err < 0)
2149 return err;
2150 hash_len = this.hash_len;
2151 name = this.name;
2152 }
2153 }
2154
2155 nd->last.hash_len = hash_len;
2156 nd->last.name = name;
2157 nd->last_type = type;
2158
2159 name += hashlen_len(hash_len);
2160 if (!*name)
2161 goto OK;
2162
2163
2164
2165
2166 do {
2167 name++;
2168 } while (unlikely(*name == '/'));
2169 if (unlikely(!*name)) {
2170OK:
2171
2172 if (!depth) {
2173 nd->dir_uid = nd->inode->i_uid;
2174 nd->dir_mode = nd->inode->i_mode;
2175 nd->flags &= ~LOOKUP_PARENT;
2176 return 0;
2177 }
2178
2179 name = nd->stack[--depth].name;
2180 link = walk_component(nd, 0);
2181 } else {
2182
2183 link = walk_component(nd, WALK_MORE);
2184 }
2185 if (unlikely(link)) {
2186 if (IS_ERR(link))
2187 return PTR_ERR(link);
2188
2189 nd->stack[depth++].name = name;
2190 name = link;
2191 continue;
2192 }
2193 if (unlikely(!d_can_lookup(nd->path.dentry))) {
2194 if (nd->flags & LOOKUP_RCU) {
2195 if (unlazy_walk(nd))
2196 return -ECHILD;
2197 }
2198 return -ENOTDIR;
2199 }
2200 }
2201}
2202
2203
2204static const char *path_init(struct nameidata *nd, unsigned flags)
2205{
2206 int error;
2207 const char *s = nd->name->name;
2208
2209 if (!*s)
2210 flags &= ~LOOKUP_RCU;
2211 if (flags & LOOKUP_RCU)
2212 rcu_read_lock();
2213
2214 nd->flags = flags | LOOKUP_JUMPED;
2215 nd->depth = 0;
2216
2217 nd->m_seq = __read_seqcount_begin(&mount_lock.seqcount);
2218 nd->r_seq = __read_seqcount_begin(&rename_lock.seqcount);
2219 smp_rmb();
2220
2221 if (flags & LOOKUP_ROOT) {
2222 struct dentry *root = nd->root.dentry;
2223 struct inode *inode = root->d_inode;
2224 if (*s && unlikely(!d_can_lookup(root)))
2225 return ERR_PTR(-ENOTDIR);
2226 nd->path = nd->root;
2227 nd->inode = inode;
2228 if (flags & LOOKUP_RCU) {
2229 nd->seq = read_seqcount_begin(&nd->path.dentry->d_seq);
2230 nd->root_seq = nd->seq;
2231 } else {
2232 path_get(&nd->path);
2233 }
2234 return s;
2235 }
2236
2237 nd->root.mnt = NULL;
2238 nd->path.mnt = NULL;
2239 nd->path.dentry = NULL;
2240
2241
2242 if (*s == '/' && !(flags & LOOKUP_IN_ROOT)) {
2243 error = nd_jump_root(nd);
2244 if (unlikely(error))
2245 return ERR_PTR(error);
2246 return s;
2247 }
2248
2249
2250 if (nd->dfd == AT_FDCWD) {
2251 if (flags & LOOKUP_RCU) {
2252 struct fs_struct *fs = current->fs;
2253 unsigned seq;
2254
2255 do {
2256 seq = read_seqcount_begin(&fs->seq);
2257 nd->path = fs->pwd;
2258 nd->inode = nd->path.dentry->d_inode;
2259 nd->seq = __read_seqcount_begin(&nd->path.dentry->d_seq);
2260 } while (read_seqcount_retry(&fs->seq, seq));
2261 } else {
2262 get_fs_pwd(current->fs, &nd->path);
2263 nd->inode = nd->path.dentry->d_inode;
2264 }
2265 } else {
2266
2267 struct fd f = fdget_raw(nd->dfd);
2268 struct dentry *dentry;
2269
2270 if (!f.file)
2271 return ERR_PTR(-EBADF);
2272
2273 dentry = f.file->f_path.dentry;
2274
2275 if (*s && unlikely(!d_can_lookup(dentry))) {
2276 fdput(f);
2277 return ERR_PTR(-ENOTDIR);
2278 }
2279
2280 nd->path = f.file->f_path;
2281 if (flags & LOOKUP_RCU) {
2282 nd->inode = nd->path.dentry->d_inode;
2283 nd->seq = read_seqcount_begin(&nd->path.dentry->d_seq);
2284 } else {
2285 path_get(&nd->path);
2286 nd->inode = nd->path.dentry->d_inode;
2287 }
2288 fdput(f);
2289 }
2290
2291
2292 if (flags & LOOKUP_IS_SCOPED) {
2293 nd->root = nd->path;
2294 if (flags & LOOKUP_RCU) {
2295 nd->root_seq = nd->seq;
2296 } else {
2297 path_get(&nd->root);
2298 nd->flags |= LOOKUP_ROOT_GRABBED;
2299 }
2300 }
2301 return s;
2302}
2303
2304static inline const char *lookup_last(struct nameidata *nd)
2305{
2306 if (nd->last_type == LAST_NORM && nd->last.name[nd->last.len])
2307 nd->flags |= LOOKUP_FOLLOW | LOOKUP_DIRECTORY;
2308
2309 return walk_component(nd, WALK_TRAILING);
2310}
2311
2312static int handle_lookup_down(struct nameidata *nd)
2313{
2314 if (!(nd->flags & LOOKUP_RCU))
2315 dget(nd->path.dentry);
2316 return PTR_ERR(step_into(nd, WALK_NOFOLLOW,
2317 nd->path.dentry, nd->inode, nd->seq));
2318}
2319
2320
2321static int path_lookupat(struct nameidata *nd, unsigned flags, struct path *path)
2322{
2323 const char *s = path_init(nd, flags);
2324 int err;
2325
2326 if (unlikely(flags & LOOKUP_DOWN) && !IS_ERR(s)) {
2327 err = handle_lookup_down(nd);
2328 if (unlikely(err < 0))
2329 s = ERR_PTR(err);
2330 }
2331
2332 while (!(err = link_path_walk(s, nd)) &&
2333 (s = lookup_last(nd)) != NULL)
2334 ;
2335 if (!err)
2336 err = complete_walk(nd);
2337
2338 if (!err && nd->flags & LOOKUP_DIRECTORY)
2339 if (!d_can_lookup(nd->path.dentry))
2340 err = -ENOTDIR;
2341 if (!err && unlikely(nd->flags & LOOKUP_MOUNTPOINT)) {
2342 err = handle_lookup_down(nd);
2343 nd->flags &= ~LOOKUP_JUMPED;
2344 }
2345 if (!err) {
2346 *path = nd->path;
2347 nd->path.mnt = NULL;
2348 nd->path.dentry = NULL;
2349 }
2350 terminate_walk(nd);
2351 return err;
2352}
2353
2354int filename_lookup(int dfd, struct filename *name, unsigned flags,
2355 struct path *path, struct path *root)
2356{
2357 int retval;
2358 struct nameidata nd;
2359 if (IS_ERR(name))
2360 return PTR_ERR(name);
2361 if (unlikely(root)) {
2362 nd.root = *root;
2363 flags |= LOOKUP_ROOT;
2364 }
2365 set_nameidata(&nd, dfd, name);
2366 retval = path_lookupat(&nd, flags | LOOKUP_RCU, path);
2367 if (unlikely(retval == -ECHILD))
2368 retval = path_lookupat(&nd, flags, path);
2369 if (unlikely(retval == -ESTALE))
2370 retval = path_lookupat(&nd, flags | LOOKUP_REVAL, path);
2371
2372 if (likely(!retval))
2373 audit_inode(name, path->dentry,
2374 flags & LOOKUP_MOUNTPOINT ? AUDIT_INODE_NOEVAL : 0);
2375 restore_nameidata();
2376 putname(name);
2377 return retval;
2378}
2379
2380
2381static int path_parentat(struct nameidata *nd, unsigned flags,
2382 struct path *parent)
2383{
2384 const char *s = path_init(nd, flags);
2385 int err = link_path_walk(s, nd);
2386 if (!err)
2387 err = complete_walk(nd);
2388 if (!err) {
2389 *parent = nd->path;
2390 nd->path.mnt = NULL;
2391 nd->path.dentry = NULL;
2392 }
2393 terminate_walk(nd);
2394 return err;
2395}
2396
2397static struct filename *filename_parentat(int dfd, struct filename *name,
2398 unsigned int flags, struct path *parent,
2399 struct qstr *last, int *type)
2400{
2401 int retval;
2402 struct nameidata nd;
2403
2404 if (IS_ERR(name))
2405 return name;
2406 set_nameidata(&nd, dfd, name);
2407 retval = path_parentat(&nd, flags | LOOKUP_RCU, parent);
2408 if (unlikely(retval == -ECHILD))
2409 retval = path_parentat(&nd, flags, parent);
2410 if (unlikely(retval == -ESTALE))
2411 retval = path_parentat(&nd, flags | LOOKUP_REVAL, parent);
2412 if (likely(!retval)) {
2413 *last = nd.last;
2414 *type = nd.last_type;
2415 audit_inode(name, parent->dentry, AUDIT_INODE_PARENT);
2416 } else {
2417 putname(name);
2418 name = ERR_PTR(retval);
2419 }
2420 restore_nameidata();
2421 return name;
2422}
2423
2424
2425struct dentry *kern_path_locked(const char *name, struct path *path)
2426{
2427 struct filename *filename;
2428 struct dentry *d;
2429 struct qstr last;
2430 int type;
2431
2432 filename = filename_parentat(AT_FDCWD, getname_kernel(name), 0, path,
2433 &last, &type);
2434 if (IS_ERR(filename))
2435 return ERR_CAST(filename);
2436 if (unlikely(type != LAST_NORM)) {
2437 path_put(path);
2438 putname(filename);
2439 return ERR_PTR(-EINVAL);
2440 }
2441 inode_lock_nested(path->dentry->d_inode, I_MUTEX_PARENT);
2442 d = __lookup_hash(&last, path->dentry, 0);
2443 if (IS_ERR(d)) {
2444 inode_unlock(path->dentry->d_inode);
2445 path_put(path);
2446 }
2447 putname(filename);
2448 return d;
2449}
2450
2451int kern_path(const char *name, unsigned int flags, struct path *path)
2452{
2453 return filename_lookup(AT_FDCWD, getname_kernel(name),
2454 flags, path, NULL);
2455}
2456EXPORT_SYMBOL(kern_path);
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466int vfs_path_lookup(struct dentry *dentry, struct vfsmount *mnt,
2467 const char *name, unsigned int flags,
2468 struct path *path)
2469{
2470 struct path root = {.mnt = mnt, .dentry = dentry};
2471
2472 return filename_lookup(AT_FDCWD, getname_kernel(name),
2473 flags , path, &root);
2474}
2475EXPORT_SYMBOL(vfs_path_lookup);
2476
2477static int lookup_one_len_common(const char *name, struct dentry *base,
2478 int len, struct qstr *this)
2479{
2480 this->name = name;
2481 this->len = len;
2482 this->hash = full_name_hash(base, name, len);
2483 if (!len)
2484 return -EACCES;
2485
2486 if (unlikely(name[0] == '.')) {
2487 if (len < 2 || (len == 2 && name[1] == '.'))
2488 return -EACCES;
2489 }
2490
2491 while (len--) {
2492 unsigned int c = *(const unsigned char *)name++;
2493 if (c == '/' || c == '\0')
2494 return -EACCES;
2495 }
2496
2497
2498
2499
2500 if (base->d_flags & DCACHE_OP_HASH) {
2501 int err = base->d_op->d_hash(base, this);
2502 if (err < 0)
2503 return err;
2504 }
2505
2506 return inode_permission(base->d_inode, MAY_EXEC);
2507}
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523struct dentry *try_lookup_one_len(const char *name, struct dentry *base, int len)
2524{
2525 struct qstr this;
2526 int err;
2527
2528 WARN_ON_ONCE(!inode_is_locked(base->d_inode));
2529
2530 err = lookup_one_len_common(name, base, len, &this);
2531 if (err)
2532 return ERR_PTR(err);
2533
2534 return lookup_dcache(&this, base, 0);
2535}
2536EXPORT_SYMBOL(try_lookup_one_len);
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549struct dentry *lookup_one_len(const char *name, struct dentry *base, int len)
2550{
2551 struct dentry *dentry;
2552 struct qstr this;
2553 int err;
2554
2555 WARN_ON_ONCE(!inode_is_locked(base->d_inode));
2556
2557 err = lookup_one_len_common(name, base, len, &this);
2558 if (err)
2559 return ERR_PTR(err);
2560
2561 dentry = lookup_dcache(&this, base, 0);
2562 return dentry ? dentry : __lookup_slow(&this, base, 0);
2563}
2564EXPORT_SYMBOL(lookup_one_len);
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578struct dentry *lookup_one_len_unlocked(const char *name,
2579 struct dentry *base, int len)
2580{
2581 struct qstr this;
2582 int err;
2583 struct dentry *ret;
2584
2585 err = lookup_one_len_common(name, base, len, &this);
2586 if (err)
2587 return ERR_PTR(err);
2588
2589 ret = lookup_dcache(&this, base, 0);
2590 if (!ret)
2591 ret = lookup_slow(&this, base, 0);
2592 return ret;
2593}
2594EXPORT_SYMBOL(lookup_one_len_unlocked);
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604struct dentry *lookup_positive_unlocked(const char *name,
2605 struct dentry *base, int len)
2606{
2607 struct dentry *ret = lookup_one_len_unlocked(name, base, len);
2608 if (!IS_ERR(ret) && d_flags_negative(smp_load_acquire(&ret->d_flags))) {
2609 dput(ret);
2610 ret = ERR_PTR(-ENOENT);
2611 }
2612 return ret;
2613}
2614EXPORT_SYMBOL(lookup_positive_unlocked);
2615
2616#ifdef CONFIG_UNIX98_PTYS
2617int path_pts(struct path *path)
2618{
2619
2620
2621
2622 struct dentry *parent = dget_parent(path->dentry);
2623 struct dentry *child;
2624 struct qstr this = QSTR_INIT("pts", 3);
2625
2626 if (unlikely(!path_connected(path->mnt, parent))) {
2627 dput(parent);
2628 return -ENOENT;
2629 }
2630 dput(path->dentry);
2631 path->dentry = parent;
2632 child = d_hash_and_lookup(parent, &this);
2633 if (!child)
2634 return -ENOENT;
2635
2636 path->dentry = child;
2637 dput(parent);
2638 follow_down(path);
2639 return 0;
2640}
2641#endif
2642
2643int user_path_at_empty(int dfd, const char __user *name, unsigned flags,
2644 struct path *path, int *empty)
2645{
2646 return filename_lookup(dfd, getname_flags(name, flags, empty),
2647 flags, path, NULL);
2648}
2649EXPORT_SYMBOL(user_path_at_empty);
2650
2651int __check_sticky(struct inode *dir, struct inode *inode)
2652{
2653 kuid_t fsuid = current_fsuid();
2654
2655 if (uid_eq(inode->i_uid, fsuid))
2656 return 0;
2657 if (uid_eq(dir->i_uid, fsuid))
2658 return 0;
2659 return !capable_wrt_inode_uidgid(inode, CAP_FOWNER);
2660}
2661EXPORT_SYMBOL(__check_sticky);
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683static int may_delete(struct inode *dir, struct dentry *victim, bool isdir)
2684{
2685 struct inode *inode = d_backing_inode(victim);
2686 int error;
2687
2688 if (d_is_negative(victim))
2689 return -ENOENT;
2690 BUG_ON(!inode);
2691
2692 BUG_ON(victim->d_parent->d_inode != dir);
2693
2694
2695 if (!uid_valid(inode->i_uid) || !gid_valid(inode->i_gid))
2696 return -EOVERFLOW;
2697
2698 audit_inode_child(dir, victim, AUDIT_TYPE_CHILD_DELETE);
2699
2700 error = inode_permission(dir, MAY_WRITE | MAY_EXEC);
2701 if (error)
2702 return error;
2703 if (IS_APPEND(dir))
2704 return -EPERM;
2705
2706 if (check_sticky(dir, inode) || IS_APPEND(inode) ||
2707 IS_IMMUTABLE(inode) || IS_SWAPFILE(inode) || HAS_UNMAPPED_ID(inode))
2708 return -EPERM;
2709 if (isdir) {
2710 if (!d_is_dir(victim))
2711 return -ENOTDIR;
2712 if (IS_ROOT(victim))
2713 return -EBUSY;
2714 } else if (d_is_dir(victim))
2715 return -EISDIR;
2716 if (IS_DEADDIR(dir))
2717 return -ENOENT;
2718 if (victim->d_flags & DCACHE_NFSFS_RENAMED)
2719 return -EBUSY;
2720 return 0;
2721}
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732static inline int may_create(struct inode *dir, struct dentry *child)
2733{
2734 struct user_namespace *s_user_ns;
2735 audit_inode_child(dir, child, AUDIT_TYPE_CHILD_CREATE);
2736 if (child->d_inode)
2737 return -EEXIST;
2738 if (IS_DEADDIR(dir))
2739 return -ENOENT;
2740 s_user_ns = dir->i_sb->s_user_ns;
2741 if (!kuid_has_mapping(s_user_ns, current_fsuid()) ||
2742 !kgid_has_mapping(s_user_ns, current_fsgid()))
2743 return -EOVERFLOW;
2744 return inode_permission(dir, MAY_WRITE | MAY_EXEC);
2745}
2746
2747
2748
2749
2750struct dentry *lock_rename(struct dentry *p1, struct dentry *p2)
2751{
2752 struct dentry *p;
2753
2754 if (p1 == p2) {
2755 inode_lock_nested(p1->d_inode, I_MUTEX_PARENT);
2756 return NULL;
2757 }
2758
2759 mutex_lock(&p1->d_sb->s_vfs_rename_mutex);
2760
2761 p = d_ancestor(p2, p1);
2762 if (p) {
2763 inode_lock_nested(p2->d_inode, I_MUTEX_PARENT);
2764 inode_lock_nested(p1->d_inode, I_MUTEX_CHILD);
2765 return p;
2766 }
2767
2768 p = d_ancestor(p1, p2);
2769 if (p) {
2770 inode_lock_nested(p1->d_inode, I_MUTEX_PARENT);
2771 inode_lock_nested(p2->d_inode, I_MUTEX_CHILD);
2772 return p;
2773 }
2774
2775 inode_lock_nested(p1->d_inode, I_MUTEX_PARENT);
2776 inode_lock_nested(p2->d_inode, I_MUTEX_PARENT2);
2777 return NULL;
2778}
2779EXPORT_SYMBOL(lock_rename);
2780
2781void unlock_rename(struct dentry *p1, struct dentry *p2)
2782{
2783 inode_unlock(p1->d_inode);
2784 if (p1 != p2) {
2785 inode_unlock(p2->d_inode);
2786 mutex_unlock(&p1->d_sb->s_vfs_rename_mutex);
2787 }
2788}
2789EXPORT_SYMBOL(unlock_rename);
2790
2791int vfs_create(struct inode *dir, struct dentry *dentry, umode_t mode,
2792 bool want_excl)
2793{
2794 int error = may_create(dir, dentry);
2795 if (error)
2796 return error;
2797
2798 if (!dir->i_op->create)
2799 return -EACCES;
2800 mode &= S_IALLUGO;
2801 mode |= S_IFREG;
2802 error = security_inode_create(dir, dentry, mode);
2803 if (error)
2804 return error;
2805 error = dir->i_op->create(dir, dentry, mode, want_excl);
2806 if (!error)
2807 fsnotify_create(dir, dentry);
2808 return error;
2809}
2810EXPORT_SYMBOL(vfs_create);
2811
2812int vfs_mkobj(struct dentry *dentry, umode_t mode,
2813 int (*f)(struct dentry *, umode_t, void *),
2814 void *arg)
2815{
2816 struct inode *dir = dentry->d_parent->d_inode;
2817 int error = may_create(dir, dentry);
2818 if (error)
2819 return error;
2820
2821 mode &= S_IALLUGO;
2822 mode |= S_IFREG;
2823 error = security_inode_create(dir, dentry, mode);
2824 if (error)
2825 return error;
2826 error = f(dentry, mode, arg);
2827 if (!error)
2828 fsnotify_create(dir, dentry);
2829 return error;
2830}
2831EXPORT_SYMBOL(vfs_mkobj);
2832
2833bool may_open_dev(const struct path *path)
2834{
2835 return !(path->mnt->mnt_flags & MNT_NODEV) &&
2836 !(path->mnt->mnt_sb->s_iflags & SB_I_NODEV);
2837}
2838
2839static int may_open(const struct path *path, int acc_mode, int flag)
2840{
2841 struct dentry *dentry = path->dentry;
2842 struct inode *inode = dentry->d_inode;
2843 int error;
2844
2845 if (!inode)
2846 return -ENOENT;
2847
2848 switch (inode->i_mode & S_IFMT) {
2849 case S_IFLNK:
2850 return -ELOOP;
2851 case S_IFDIR:
2852 if (acc_mode & MAY_WRITE)
2853 return -EISDIR;
2854 if (acc_mode & MAY_EXEC)
2855 return -EACCES;
2856 break;
2857 case S_IFBLK:
2858 case S_IFCHR:
2859 if (!may_open_dev(path))
2860 return -EACCES;
2861 fallthrough;
2862 case S_IFIFO:
2863 case S_IFSOCK:
2864 if (acc_mode & MAY_EXEC)
2865 return -EACCES;
2866 flag &= ~O_TRUNC;
2867 break;
2868 case S_IFREG:
2869 if ((acc_mode & MAY_EXEC) && path_noexec(path))
2870 return -EACCES;
2871 break;
2872 }
2873
2874 error = inode_permission(inode, MAY_OPEN | acc_mode);
2875 if (error)
2876 return error;
2877
2878
2879
2880
2881 if (IS_APPEND(inode)) {
2882 if ((flag & O_ACCMODE) != O_RDONLY && !(flag & O_APPEND))
2883 return -EPERM;
2884 if (flag & O_TRUNC)
2885 return -EPERM;
2886 }
2887
2888
2889 if (flag & O_NOATIME && !inode_owner_or_capable(inode))
2890 return -EPERM;
2891
2892 return 0;
2893}
2894
2895static int handle_truncate(struct file *filp)
2896{
2897 const struct path *path = &filp->f_path;
2898 struct inode *inode = path->dentry->d_inode;
2899 int error = get_write_access(inode);
2900 if (error)
2901 return error;
2902
2903
2904
2905 error = locks_verify_locked(filp);
2906 if (!error)
2907 error = security_path_truncate(path);
2908 if (!error) {
2909 error = do_truncate(path->dentry, 0,
2910 ATTR_MTIME|ATTR_CTIME|ATTR_OPEN,
2911 filp);
2912 }
2913 put_write_access(inode);
2914 return error;
2915}
2916
2917static inline int open_to_namei_flags(int flag)
2918{
2919 if ((flag & O_ACCMODE) == 3)
2920 flag--;
2921 return flag;
2922}
2923
2924static int may_o_create(const struct path *dir, struct dentry *dentry, umode_t mode)
2925{
2926 struct user_namespace *s_user_ns;
2927 int error = security_path_mknod(dir, dentry, mode, 0);
2928 if (error)
2929 return error;
2930
2931 s_user_ns = dir->dentry->d_sb->s_user_ns;
2932 if (!kuid_has_mapping(s_user_ns, current_fsuid()) ||
2933 !kgid_has_mapping(s_user_ns, current_fsgid()))
2934 return -EOVERFLOW;
2935
2936 error = inode_permission(dir->dentry->d_inode, MAY_WRITE | MAY_EXEC);
2937 if (error)
2938 return error;
2939
2940 return security_inode_create(dir->dentry->d_inode, dentry, mode);
2941}
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956static struct dentry *atomic_open(struct nameidata *nd, struct dentry *dentry,
2957 struct file *file,
2958 int open_flag, umode_t mode)
2959{
2960 struct dentry *const DENTRY_NOT_SET = (void *) -1UL;
2961 struct inode *dir = nd->path.dentry->d_inode;
2962 int error;
2963
2964 if (nd->flags & LOOKUP_DIRECTORY)
2965 open_flag |= O_DIRECTORY;
2966
2967 file->f_path.dentry = DENTRY_NOT_SET;
2968 file->f_path.mnt = nd->path.mnt;
2969 error = dir->i_op->atomic_open(dir, dentry, file,
2970 open_to_namei_flags(open_flag), mode);
2971 d_lookup_done(dentry);
2972 if (!error) {
2973 if (file->f_mode & FMODE_OPENED) {
2974 if (unlikely(dentry != file->f_path.dentry)) {
2975 dput(dentry);
2976 dentry = dget(file->f_path.dentry);
2977 }
2978 } else if (WARN_ON(file->f_path.dentry == DENTRY_NOT_SET)) {
2979 error = -EIO;
2980 } else {
2981 if (file->f_path.dentry) {
2982 dput(dentry);
2983 dentry = file->f_path.dentry;
2984 }
2985 if (unlikely(d_is_negative(dentry)))
2986 error = -ENOENT;
2987 }
2988 }
2989 if (error) {
2990 dput(dentry);
2991 dentry = ERR_PTR(error);
2992 }
2993 return dentry;
2994}
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011static struct dentry *lookup_open(struct nameidata *nd, struct file *file,
3012 const struct open_flags *op,
3013 bool got_write)
3014{
3015 struct dentry *dir = nd->path.dentry;
3016 struct inode *dir_inode = dir->d_inode;
3017 int open_flag = op->open_flag;
3018 struct dentry *dentry;
3019 int error, create_error = 0;
3020 umode_t mode = op->mode;
3021 DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wq);
3022
3023 if (unlikely(IS_DEADDIR(dir_inode)))
3024 return ERR_PTR(-ENOENT);
3025
3026 file->f_mode &= ~FMODE_CREATED;
3027 dentry = d_lookup(dir, &nd->last);
3028 for (;;) {
3029 if (!dentry) {
3030 dentry = d_alloc_parallel(dir, &nd->last, &wq);
3031 if (IS_ERR(dentry))
3032 return dentry;
3033 }
3034 if (d_in_lookup(dentry))
3035 break;
3036
3037 error = d_revalidate(dentry, nd->flags);
3038 if (likely(error > 0))
3039 break;
3040 if (error)
3041 goto out_dput;
3042 d_invalidate(dentry);
3043 dput(dentry);
3044 dentry = NULL;
3045 }
3046 if (dentry->d_inode) {
3047
3048 return dentry;
3049 }
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060 if (unlikely(!got_write))
3061 open_flag &= ~O_TRUNC;
3062 if (open_flag & O_CREAT) {
3063 if (open_flag & O_EXCL)
3064 open_flag &= ~O_TRUNC;
3065 if (!IS_POSIXACL(dir->d_inode))
3066 mode &= ~current_umask();
3067 if (likely(got_write))
3068 create_error = may_o_create(&nd->path, dentry, mode);
3069 else
3070 create_error = -EROFS;
3071 }
3072 if (create_error)
3073 open_flag &= ~O_CREAT;
3074 if (dir_inode->i_op->atomic_open) {
3075 dentry = atomic_open(nd, dentry, file, open_flag, mode);
3076 if (unlikely(create_error) && dentry == ERR_PTR(-ENOENT))
3077 dentry = ERR_PTR(create_error);
3078 return dentry;
3079 }
3080
3081 if (d_in_lookup(dentry)) {
3082 struct dentry *res = dir_inode->i_op->lookup(dir_inode, dentry,
3083 nd->flags);
3084 d_lookup_done(dentry);
3085 if (unlikely(res)) {
3086 if (IS_ERR(res)) {
3087 error = PTR_ERR(res);
3088 goto out_dput;
3089 }
3090 dput(dentry);
3091 dentry = res;
3092 }
3093 }
3094
3095
3096 if (!dentry->d_inode && (open_flag & O_CREAT)) {
3097 file->f_mode |= FMODE_CREATED;
3098 audit_inode_child(dir_inode, dentry, AUDIT_TYPE_CHILD_CREATE);
3099 if (!dir_inode->i_op->create) {
3100 error = -EACCES;
3101 goto out_dput;
3102 }
3103 error = dir_inode->i_op->create(dir_inode, dentry, mode,
3104 open_flag & O_EXCL);
3105 if (error)
3106 goto out_dput;
3107 }
3108 if (unlikely(create_error) && !dentry->d_inode) {
3109 error = create_error;
3110 goto out_dput;
3111 }
3112 return dentry;
3113
3114out_dput:
3115 dput(dentry);
3116 return ERR_PTR(error);
3117}
3118
3119static const char *open_last_lookups(struct nameidata *nd,
3120 struct file *file, const struct open_flags *op)
3121{
3122 struct dentry *dir = nd->path.dentry;
3123 int open_flag = op->open_flag;
3124 bool got_write = false;
3125 unsigned seq;
3126 struct inode *inode;
3127 struct dentry *dentry;
3128 const char *res;
3129 int error;
3130
3131 nd->flags |= op->intent;
3132
3133 if (nd->last_type != LAST_NORM) {
3134 if (nd->depth)
3135 put_link(nd);
3136 return handle_dots(nd, nd->last_type);
3137 }
3138
3139 if (!(open_flag & O_CREAT)) {
3140 if (nd->last.name[nd->last.len])
3141 nd->flags |= LOOKUP_FOLLOW | LOOKUP_DIRECTORY;
3142
3143 dentry = lookup_fast(nd, &inode, &seq);
3144 if (IS_ERR(dentry))
3145 return ERR_CAST(dentry);
3146 if (likely(dentry))
3147 goto finish_lookup;
3148
3149 BUG_ON(nd->flags & LOOKUP_RCU);
3150 } else {
3151
3152 if (nd->flags & LOOKUP_RCU) {
3153 error = unlazy_walk(nd);
3154 if (unlikely(error))
3155 return ERR_PTR(error);
3156 }
3157 audit_inode(nd->name, dir, AUDIT_INODE_PARENT);
3158
3159 if (unlikely(nd->last.name[nd->last.len]))
3160 return ERR_PTR(-EISDIR);
3161 }
3162
3163 if (open_flag & (O_CREAT | O_TRUNC | O_WRONLY | O_RDWR)) {
3164 error = mnt_want_write(nd->path.mnt);
3165 if (!error)
3166 got_write = true;
3167
3168
3169
3170
3171
3172 }
3173 if (open_flag & O_CREAT)
3174 inode_lock(dir->d_inode);
3175 else
3176 inode_lock_shared(dir->d_inode);
3177 dentry = lookup_open(nd, file, op, got_write);
3178 if (!IS_ERR(dentry) && (file->f_mode & FMODE_CREATED))
3179 fsnotify_create(dir->d_inode, dentry);
3180 if (open_flag & O_CREAT)
3181 inode_unlock(dir->d_inode);
3182 else
3183 inode_unlock_shared(dir->d_inode);
3184
3185 if (got_write)
3186 mnt_drop_write(nd->path.mnt);
3187
3188 if (IS_ERR(dentry))
3189 return ERR_CAST(dentry);
3190
3191 if (file->f_mode & (FMODE_OPENED | FMODE_CREATED)) {
3192 dput(nd->path.dentry);
3193 nd->path.dentry = dentry;
3194 return NULL;
3195 }
3196
3197finish_lookup:
3198 if (nd->depth)
3199 put_link(nd);
3200 res = step_into(nd, WALK_TRAILING, dentry, inode, seq);
3201 if (unlikely(res))
3202 nd->flags &= ~(LOOKUP_OPEN|LOOKUP_CREATE|LOOKUP_EXCL);
3203 return res;
3204}
3205
3206
3207
3208
3209static int do_open(struct nameidata *nd,
3210 struct file *file, const struct open_flags *op)
3211{
3212 int open_flag = op->open_flag;
3213 bool do_truncate;
3214 int acc_mode;
3215 int error;
3216
3217 if (!(file->f_mode & (FMODE_OPENED | FMODE_CREATED))) {
3218 error = complete_walk(nd);
3219 if (error)
3220 return error;
3221 }
3222 if (!(file->f_mode & FMODE_CREATED))
3223 audit_inode(nd->name, nd->path.dentry, 0);
3224 if (open_flag & O_CREAT) {
3225 if ((open_flag & O_EXCL) && !(file->f_mode & FMODE_CREATED))
3226 return -EEXIST;
3227 if (d_is_dir(nd->path.dentry))
3228 return -EISDIR;
3229 error = may_create_in_sticky(nd->dir_mode, nd->dir_uid,
3230 d_backing_inode(nd->path.dentry));
3231 if (unlikely(error))
3232 return error;
3233 }
3234 if ((nd->flags & LOOKUP_DIRECTORY) && !d_can_lookup(nd->path.dentry))
3235 return -ENOTDIR;
3236
3237 do_truncate = false;
3238 acc_mode = op->acc_mode;
3239 if (file->f_mode & FMODE_CREATED) {
3240
3241 open_flag &= ~O_TRUNC;
3242 acc_mode = 0;
3243 } else if (d_is_reg(nd->path.dentry) && open_flag & O_TRUNC) {
3244 error = mnt_want_write(nd->path.mnt);
3245 if (error)
3246 return error;
3247 do_truncate = true;
3248 }
3249 error = may_open(&nd->path, acc_mode, open_flag);
3250 if (!error && !(file->f_mode & FMODE_OPENED))
3251 error = vfs_open(&nd->path, file);
3252 if (!error)
3253 error = ima_file_check(file, op->acc_mode);
3254 if (!error && do_truncate)
3255 error = handle_truncate(file);
3256 if (unlikely(error > 0)) {
3257 WARN_ON(1);
3258 error = -EINVAL;
3259 }
3260 if (do_truncate)
3261 mnt_drop_write(nd->path.mnt);
3262 return error;
3263}
3264
3265struct dentry *vfs_tmpfile(struct dentry *dentry, umode_t mode, int open_flag)
3266{
3267 struct dentry *child = NULL;
3268 struct inode *dir = dentry->d_inode;
3269 struct inode *inode;
3270 int error;
3271
3272
3273 error = inode_permission(dir, MAY_WRITE | MAY_EXEC);
3274 if (error)
3275 goto out_err;
3276 error = -EOPNOTSUPP;
3277 if (!dir->i_op->tmpfile)
3278 goto out_err;
3279 error = -ENOMEM;
3280 child = d_alloc(dentry, &slash_name);
3281 if (unlikely(!child))
3282 goto out_err;
3283 error = dir->i_op->tmpfile(dir, child, mode);
3284 if (error)
3285 goto out_err;
3286 error = -ENOENT;
3287 inode = child->d_inode;
3288 if (unlikely(!inode))
3289 goto out_err;
3290 if (!(open_flag & O_EXCL)) {
3291 spin_lock(&inode->i_lock);
3292 inode->i_state |= I_LINKABLE;
3293 spin_unlock(&inode->i_lock);
3294 }
3295 ima_post_create_tmpfile(inode);
3296 return child;
3297
3298out_err:
3299 dput(child);
3300 return ERR_PTR(error);
3301}
3302EXPORT_SYMBOL(vfs_tmpfile);
3303
3304static int do_tmpfile(struct nameidata *nd, unsigned flags,
3305 const struct open_flags *op,
3306 struct file *file)
3307{
3308 struct dentry *child;
3309 struct path path;
3310 int error = path_lookupat(nd, flags | LOOKUP_DIRECTORY, &path);
3311 if (unlikely(error))
3312 return error;
3313 error = mnt_want_write(path.mnt);
3314 if (unlikely(error))
3315 goto out;
3316 child = vfs_tmpfile(path.dentry, op->mode, op->open_flag);
3317 error = PTR_ERR(child);
3318 if (IS_ERR(child))
3319 goto out2;
3320 dput(path.dentry);
3321 path.dentry = child;
3322 audit_inode(nd->name, child, 0);
3323
3324 error = may_open(&path, 0, op->open_flag);
3325 if (error)
3326 goto out2;
3327 file->f_path.mnt = path.mnt;
3328 error = finish_open(file, child, NULL);
3329out2:
3330 mnt_drop_write(path.mnt);
3331out:
3332 path_put(&path);
3333 return error;
3334}
3335
3336static int do_o_path(struct nameidata *nd, unsigned flags, struct file *file)
3337{
3338 struct path path;
3339 int error = path_lookupat(nd, flags, &path);
3340 if (!error) {
3341 audit_inode(nd->name, path.dentry, 0);
3342 error = vfs_open(&path, file);
3343 path_put(&path);
3344 }
3345 return error;
3346}
3347
3348static struct file *path_openat(struct nameidata *nd,
3349 const struct open_flags *op, unsigned flags)
3350{
3351 struct file *file;
3352 int error;
3353
3354 file = alloc_empty_file(op->open_flag, current_cred());
3355 if (IS_ERR(file))
3356 return file;
3357
3358 if (unlikely(file->f_flags & __O_TMPFILE)) {
3359 error = do_tmpfile(nd, flags, op, file);
3360 } else if (unlikely(file->f_flags & O_PATH)) {
3361 error = do_o_path(nd, flags, file);
3362 } else {
3363 const char *s = path_init(nd, flags);
3364 while (!(error = link_path_walk(s, nd)) &&
3365 (s = open_last_lookups(nd, file, op)) != NULL)
3366 ;
3367 if (!error)
3368 error = do_open(nd, file, op);
3369 terminate_walk(nd);
3370 }
3371 if (likely(!error)) {
3372 if (likely(file->f_mode & FMODE_OPENED))
3373 return file;
3374 WARN_ON(1);
3375 error = -EINVAL;
3376 }
3377 fput(file);
3378 if (error == -EOPENSTALE) {
3379 if (flags & LOOKUP_RCU)
3380 error = -ECHILD;
3381 else
3382 error = -ESTALE;
3383 }
3384 return ERR_PTR(error);
3385}
3386
3387struct file *do_filp_open(int dfd, struct filename *pathname,
3388 const struct open_flags *op)
3389{
3390 struct nameidata nd;
3391 int flags = op->lookup_flags;
3392 struct file *filp;
3393
3394 set_nameidata(&nd, dfd, pathname);
3395 filp = path_openat(&nd, op, flags | LOOKUP_RCU);
3396 if (unlikely(filp == ERR_PTR(-ECHILD)))
3397 filp = path_openat(&nd, op, flags);
3398 if (unlikely(filp == ERR_PTR(-ESTALE)))
3399 filp = path_openat(&nd, op, flags | LOOKUP_REVAL);
3400 restore_nameidata();
3401 return filp;
3402}
3403
3404struct file *do_file_open_root(struct dentry *dentry, struct vfsmount *mnt,
3405 const char *name, const struct open_flags *op)
3406{
3407 struct nameidata nd;
3408 struct file *file;
3409 struct filename *filename;
3410 int flags = op->lookup_flags | LOOKUP_ROOT;
3411
3412 nd.root.mnt = mnt;
3413 nd.root.dentry = dentry;
3414
3415 if (d_is_symlink(dentry) && op->intent & LOOKUP_OPEN)
3416 return ERR_PTR(-ELOOP);
3417
3418 filename = getname_kernel(name);
3419 if (IS_ERR(filename))
3420 return ERR_CAST(filename);
3421
3422 set_nameidata(&nd, -1, filename);
3423 file = path_openat(&nd, op, flags | LOOKUP_RCU);
3424 if (unlikely(file == ERR_PTR(-ECHILD)))
3425 file = path_openat(&nd, op, flags);
3426 if (unlikely(file == ERR_PTR(-ESTALE)))
3427 file = path_openat(&nd, op, flags | LOOKUP_REVAL);
3428 restore_nameidata();
3429 putname(filename);
3430 return file;
3431}
3432
3433static struct dentry *filename_create(int dfd, struct filename *name,
3434 struct path *path, unsigned int lookup_flags)
3435{
3436 struct dentry *dentry = ERR_PTR(-EEXIST);
3437 struct qstr last;
3438 int type;
3439 int err2;
3440 int error;
3441 bool is_dir = (lookup_flags & LOOKUP_DIRECTORY);
3442
3443
3444
3445
3446
3447 lookup_flags &= LOOKUP_REVAL;
3448
3449 name = filename_parentat(dfd, name, lookup_flags, path, &last, &type);
3450 if (IS_ERR(name))
3451 return ERR_CAST(name);
3452
3453
3454
3455
3456
3457 if (unlikely(type != LAST_NORM))
3458 goto out;
3459
3460
3461 err2 = mnt_want_write(path->mnt);
3462
3463
3464
3465 lookup_flags |= LOOKUP_CREATE | LOOKUP_EXCL;
3466 inode_lock_nested(path->dentry->d_inode, I_MUTEX_PARENT);
3467 dentry = __lookup_hash(&last, path->dentry, lookup_flags);
3468 if (IS_ERR(dentry))
3469 goto unlock;
3470
3471 error = -EEXIST;
3472 if (d_is_positive(dentry))
3473 goto fail;
3474
3475
3476
3477
3478
3479
3480
3481 if (unlikely(!is_dir && last.name[last.len])) {
3482 error = -ENOENT;
3483 goto fail;
3484 }
3485 if (unlikely(err2)) {
3486 error = err2;
3487 goto fail;
3488 }
3489 putname(name);
3490 return dentry;
3491fail:
3492 dput(dentry);
3493 dentry = ERR_PTR(error);
3494unlock:
3495 inode_unlock(path->dentry->d_inode);
3496 if (!err2)
3497 mnt_drop_write(path->mnt);
3498out:
3499 path_put(path);
3500 putname(name);
3501 return dentry;
3502}
3503
3504struct dentry *kern_path_create(int dfd, const char *pathname,
3505 struct path *path, unsigned int lookup_flags)
3506{
3507 return filename_create(dfd, getname_kernel(pathname),
3508 path, lookup_flags);
3509}
3510EXPORT_SYMBOL(kern_path_create);
3511
3512void done_path_create(struct path *path, struct dentry *dentry)
3513{
3514 dput(dentry);
3515 inode_unlock(path->dentry->d_inode);
3516 mnt_drop_write(path->mnt);
3517 path_put(path);
3518}
3519EXPORT_SYMBOL(done_path_create);
3520
3521inline struct dentry *user_path_create(int dfd, const char __user *pathname,
3522 struct path *path, unsigned int lookup_flags)
3523{
3524 return filename_create(dfd, getname(pathname), path, lookup_flags);
3525}
3526EXPORT_SYMBOL(user_path_create);
3527
3528int vfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, dev_t dev)
3529{
3530 bool is_whiteout = S_ISCHR(mode) && dev == WHITEOUT_DEV;
3531 int error = may_create(dir, dentry);
3532
3533 if (error)
3534 return error;
3535
3536 if ((S_ISCHR(mode) || S_ISBLK(mode)) && !is_whiteout &&
3537 !capable(CAP_MKNOD))
3538 return -EPERM;
3539
3540 if (!dir->i_op->mknod)
3541 return -EPERM;
3542
3543 error = devcgroup_inode_mknod(mode, dev);
3544 if (error)
3545 return error;
3546
3547 error = security_inode_mknod(dir, dentry, mode, dev);
3548 if (error)
3549 return error;
3550
3551 error = dir->i_op->mknod(dir, dentry, mode, dev);
3552 if (!error)
3553 fsnotify_create(dir, dentry);
3554 return error;
3555}
3556EXPORT_SYMBOL(vfs_mknod);
3557
3558static int may_mknod(umode_t mode)
3559{
3560 switch (mode & S_IFMT) {
3561 case S_IFREG:
3562 case S_IFCHR:
3563 case S_IFBLK:
3564 case S_IFIFO:
3565 case S_IFSOCK:
3566 case 0:
3567 return 0;
3568 case S_IFDIR:
3569 return -EPERM;
3570 default:
3571 return -EINVAL;
3572 }
3573}
3574
3575static long do_mknodat(int dfd, const char __user *filename, umode_t mode,
3576 unsigned int dev)
3577{
3578 struct dentry *dentry;
3579 struct path path;
3580 int error;
3581 unsigned int lookup_flags = 0;
3582
3583 error = may_mknod(mode);
3584 if (error)
3585 return error;
3586retry:
3587 dentry = user_path_create(dfd, filename, &path, lookup_flags);
3588 if (IS_ERR(dentry))
3589 return PTR_ERR(dentry);
3590
3591 if (!IS_POSIXACL(path.dentry->d_inode))
3592 mode &= ~current_umask();
3593 error = security_path_mknod(&path, dentry, mode, dev);
3594 if (error)
3595 goto out;
3596 switch (mode & S_IFMT) {
3597 case 0: case S_IFREG:
3598 error = vfs_create(path.dentry->d_inode,dentry,mode,true);
3599 if (!error)
3600 ima_post_path_mknod(dentry);
3601 break;
3602 case S_IFCHR: case S_IFBLK:
3603 error = vfs_mknod(path.dentry->d_inode,dentry,mode,
3604 new_decode_dev(dev));
3605 break;
3606 case S_IFIFO: case S_IFSOCK:
3607 error = vfs_mknod(path.dentry->d_inode,dentry,mode,0);
3608 break;
3609 }
3610out:
3611 done_path_create(&path, dentry);
3612 if (retry_estale(error, lookup_flags)) {
3613 lookup_flags |= LOOKUP_REVAL;
3614 goto retry;
3615 }
3616 return error;
3617}
3618
3619SYSCALL_DEFINE4(mknodat, int, dfd, const char __user *, filename, umode_t, mode,
3620 unsigned int, dev)
3621{
3622 return do_mknodat(dfd, filename, mode, dev);
3623}
3624
3625SYSCALL_DEFINE3(mknod, const char __user *, filename, umode_t, mode, unsigned, dev)
3626{
3627 return do_mknodat(AT_FDCWD, filename, mode, dev);
3628}
3629
3630int vfs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode)
3631{
3632 int error = may_create(dir, dentry);
3633 unsigned max_links = dir->i_sb->s_max_links;
3634
3635 if (error)
3636 return error;
3637
3638 if (!dir->i_op->mkdir)
3639 return -EPERM;
3640
3641 mode &= (S_IRWXUGO|S_ISVTX);
3642 error = security_inode_mkdir(dir, dentry, mode);
3643 if (error)
3644 return error;
3645
3646 if (max_links && dir->i_nlink >= max_links)
3647 return -EMLINK;
3648
3649 error = dir->i_op->mkdir(dir, dentry, mode);
3650 if (!error)
3651 fsnotify_mkdir(dir, dentry);
3652 return error;
3653}
3654EXPORT_SYMBOL(vfs_mkdir);
3655
3656static long do_mkdirat(int dfd, const char __user *pathname, umode_t mode)
3657{
3658 struct dentry *dentry;
3659 struct path path;
3660 int error;
3661 unsigned int lookup_flags = LOOKUP_DIRECTORY;
3662
3663retry:
3664 dentry = user_path_create(dfd, pathname, &path, lookup_flags);
3665 if (IS_ERR(dentry))
3666 return PTR_ERR(dentry);
3667
3668 if (!IS_POSIXACL(path.dentry->d_inode))
3669 mode &= ~current_umask();
3670 error = security_path_mkdir(&path, dentry, mode);
3671 if (!error)
3672 error = vfs_mkdir(path.dentry->d_inode, dentry, mode);
3673 done_path_create(&path, dentry);
3674 if (retry_estale(error, lookup_flags)) {
3675 lookup_flags |= LOOKUP_REVAL;
3676 goto retry;
3677 }
3678 return error;
3679}
3680
3681SYSCALL_DEFINE3(mkdirat, int, dfd, const char __user *, pathname, umode_t, mode)
3682{
3683 return do_mkdirat(dfd, pathname, mode);
3684}
3685
3686SYSCALL_DEFINE2(mkdir, const char __user *, pathname, umode_t, mode)
3687{
3688 return do_mkdirat(AT_FDCWD, pathname, mode);
3689}
3690
3691int vfs_rmdir(struct inode *dir, struct dentry *dentry)
3692{
3693 int error = may_delete(dir, dentry, 1);
3694
3695 if (error)
3696 return error;
3697
3698 if (!dir->i_op->rmdir)
3699 return -EPERM;
3700
3701 dget(dentry);
3702 inode_lock(dentry->d_inode);
3703
3704 error = -EBUSY;
3705 if (is_local_mountpoint(dentry))
3706 goto out;
3707
3708 error = security_inode_rmdir(dir, dentry);
3709 if (error)
3710 goto out;
3711
3712 error = dir->i_op->rmdir(dir, dentry);
3713 if (error)
3714 goto out;
3715
3716 shrink_dcache_parent(dentry);
3717 dentry->d_inode->i_flags |= S_DEAD;
3718 dont_mount(dentry);
3719 detach_mounts(dentry);
3720 fsnotify_rmdir(dir, dentry);
3721
3722out:
3723 inode_unlock(dentry->d_inode);
3724 dput(dentry);
3725 if (!error)
3726 d_delete(dentry);
3727 return error;
3728}
3729EXPORT_SYMBOL(vfs_rmdir);
3730
3731long do_rmdir(int dfd, struct filename *name)
3732{
3733 int error = 0;
3734 struct dentry *dentry;
3735 struct path path;
3736 struct qstr last;
3737 int type;
3738 unsigned int lookup_flags = 0;
3739retry:
3740 name = filename_parentat(dfd, name, lookup_flags,
3741 &path, &last, &type);
3742 if (IS_ERR(name))
3743 return PTR_ERR(name);
3744
3745 switch (type) {
3746 case LAST_DOTDOT:
3747 error = -ENOTEMPTY;
3748 goto exit1;
3749 case LAST_DOT:
3750 error = -EINVAL;
3751 goto exit1;
3752 case LAST_ROOT:
3753 error = -EBUSY;
3754 goto exit1;
3755 }
3756
3757 error = mnt_want_write(path.mnt);
3758 if (error)
3759 goto exit1;
3760
3761 inode_lock_nested(path.dentry->d_inode, I_MUTEX_PARENT);
3762 dentry = __lookup_hash(&last, path.dentry, lookup_flags);
3763 error = PTR_ERR(dentry);
3764 if (IS_ERR(dentry))
3765 goto exit2;
3766 if (!dentry->d_inode) {
3767 error = -ENOENT;
3768 goto exit3;
3769 }
3770 error = security_path_rmdir(&path, dentry);
3771 if (error)
3772 goto exit3;
3773 error = vfs_rmdir(path.dentry->d_inode, dentry);
3774exit3:
3775 dput(dentry);
3776exit2:
3777 inode_unlock(path.dentry->d_inode);
3778 mnt_drop_write(path.mnt);
3779exit1:
3780 path_put(&path);
3781 if (retry_estale(error, lookup_flags)) {
3782 lookup_flags |= LOOKUP_REVAL;
3783 goto retry;
3784 }
3785 putname(name);
3786 return error;
3787}
3788
3789SYSCALL_DEFINE1(rmdir, const char __user *, pathname)
3790{
3791 return do_rmdir(AT_FDCWD, getname(pathname));
3792}
3793
3794
3795
3796
3797
3798
3799
3800
3801
3802
3803
3804
3805
3806
3807
3808
3809
3810
3811
3812int vfs_unlink(struct inode *dir, struct dentry *dentry, struct inode **delegated_inode)
3813{
3814 struct inode *target = dentry->d_inode;
3815 int error = may_delete(dir, dentry, 0);
3816
3817 if (error)
3818 return error;
3819
3820 if (!dir->i_op->unlink)
3821 return -EPERM;
3822
3823 inode_lock(target);
3824 if (is_local_mountpoint(dentry))
3825 error = -EBUSY;
3826 else {
3827 error = security_inode_unlink(dir, dentry);
3828 if (!error) {
3829 error = try_break_deleg(target, delegated_inode);
3830 if (error)
3831 goto out;
3832 error = dir->i_op->unlink(dir, dentry);
3833 if (!error) {
3834 dont_mount(dentry);
3835 detach_mounts(dentry);
3836 fsnotify_unlink(dir, dentry);
3837 }
3838 }
3839 }
3840out:
3841 inode_unlock(target);
3842
3843
3844 if (!error && !(dentry->d_flags & DCACHE_NFSFS_RENAMED)) {
3845 fsnotify_link_count(target);
3846 d_delete(dentry);
3847 }
3848
3849 return error;
3850}
3851EXPORT_SYMBOL(vfs_unlink);
3852
3853
3854
3855
3856
3857
3858
3859long do_unlinkat(int dfd, struct filename *name)
3860{
3861 int error;
3862 struct dentry *dentry;
3863 struct path path;
3864 struct qstr last;
3865 int type;
3866 struct inode *inode = NULL;
3867 struct inode *delegated_inode = NULL;
3868 unsigned int lookup_flags = 0;
3869retry:
3870 name = filename_parentat(dfd, name, lookup_flags, &path, &last, &type);
3871 if (IS_ERR(name))
3872 return PTR_ERR(name);
3873
3874 error = -EISDIR;
3875 if (type != LAST_NORM)
3876 goto exit1;
3877
3878 error = mnt_want_write(path.mnt);
3879 if (error)
3880 goto exit1;
3881retry_deleg:
3882 inode_lock_nested(path.dentry->d_inode, I_MUTEX_PARENT);
3883 dentry = __lookup_hash(&last, path.dentry, lookup_flags);
3884 error = PTR_ERR(dentry);
3885 if (!IS_ERR(dentry)) {
3886
3887 if (last.name[last.len])
3888 goto slashes;
3889 inode = dentry->d_inode;
3890 if (d_is_negative(dentry))
3891 goto slashes;
3892 ihold(inode);
3893 error = security_path_unlink(&path, dentry);
3894 if (error)
3895 goto exit2;
3896 error = vfs_unlink(path.dentry->d_inode, dentry, &delegated_inode);
3897exit2:
3898 dput(dentry);
3899 }
3900 inode_unlock(path.dentry->d_inode);
3901 if (inode)
3902 iput(inode);
3903 inode = NULL;
3904 if (delegated_inode) {
3905 error = break_deleg_wait(&delegated_inode);
3906 if (!error)
3907 goto retry_deleg;
3908 }
3909 mnt_drop_write(path.mnt);
3910exit1:
3911 path_put(&path);
3912 if (retry_estale(error, lookup_flags)) {
3913 lookup_flags |= LOOKUP_REVAL;
3914 inode = NULL;
3915 goto retry;
3916 }
3917 putname(name);
3918 return error;
3919
3920slashes:
3921 if (d_is_negative(dentry))
3922 error = -ENOENT;
3923 else if (d_is_dir(dentry))
3924 error = -EISDIR;
3925 else
3926 error = -ENOTDIR;
3927 goto exit2;
3928}
3929
3930SYSCALL_DEFINE3(unlinkat, int, dfd, const char __user *, pathname, int, flag)
3931{
3932 if ((flag & ~AT_REMOVEDIR) != 0)
3933 return -EINVAL;
3934
3935 if (flag & AT_REMOVEDIR)
3936 return do_rmdir(dfd, getname(pathname));
3937 return do_unlinkat(dfd, getname(pathname));
3938}
3939
3940SYSCALL_DEFINE1(unlink, const char __user *, pathname)
3941{
3942 return do_unlinkat(AT_FDCWD, getname(pathname));
3943}
3944
3945int vfs_symlink(struct inode *dir, struct dentry *dentry, const char *oldname)
3946{
3947 int error = may_create(dir, dentry);
3948
3949 if (error)
3950 return error;
3951
3952 if (!dir->i_op->symlink)
3953 return -EPERM;
3954
3955 error = security_inode_symlink(dir, dentry, oldname);
3956 if (error)
3957 return error;
3958
3959 error = dir->i_op->symlink(dir, dentry, oldname);
3960 if (!error)
3961 fsnotify_create(dir, dentry);
3962 return error;
3963}
3964EXPORT_SYMBOL(vfs_symlink);
3965
3966static long do_symlinkat(const char __user *oldname, int newdfd,
3967 const char __user *newname)
3968{
3969 int error;
3970 struct filename *from;
3971 struct dentry *dentry;
3972 struct path path;
3973 unsigned int lookup_flags = 0;
3974
3975 from = getname(oldname);
3976 if (IS_ERR(from))
3977 return PTR_ERR(from);
3978retry:
3979 dentry = user_path_create(newdfd, newname, &path, lookup_flags);
3980 error = PTR_ERR(dentry);
3981 if (IS_ERR(dentry))
3982 goto out_putname;
3983
3984 error = security_path_symlink(&path, dentry, from->name);
3985 if (!error)
3986 error = vfs_symlink(path.dentry->d_inode, dentry, from->name);
3987 done_path_create(&path, dentry);
3988 if (retry_estale(error, lookup_flags)) {
3989 lookup_flags |= LOOKUP_REVAL;
3990 goto retry;
3991 }
3992out_putname:
3993 putname(from);
3994 return error;
3995}
3996
3997SYSCALL_DEFINE3(symlinkat, const char __user *, oldname,
3998 int, newdfd, const char __user *, newname)
3999{
4000 return do_symlinkat(oldname, newdfd, newname);
4001}
4002
4003SYSCALL_DEFINE2(symlink, const char __user *, oldname, const char __user *, newname)
4004{
4005 return do_symlinkat(oldname, AT_FDCWD, newname);
4006}
4007
4008
4009
4010
4011
4012
4013
4014
4015
4016
4017
4018
4019
4020
4021
4022
4023
4024
4025
4026
4027int vfs_link(struct dentry *old_dentry, struct inode *dir, struct dentry *new_dentry, struct inode **delegated_inode)
4028{
4029 struct inode *inode = old_dentry->d_inode;
4030 unsigned max_links = dir->i_sb->s_max_links;
4031 int error;
4032
4033 if (!inode)
4034 return -ENOENT;
4035
4036 error = may_create(dir, new_dentry);
4037 if (error)
4038 return error;
4039
4040 if (dir->i_sb != inode->i_sb)
4041 return -EXDEV;
4042
4043
4044
4045
4046 if (IS_APPEND(inode) || IS_IMMUTABLE(inode))
4047 return -EPERM;
4048
4049
4050
4051
4052
4053 if (HAS_UNMAPPED_ID(inode))
4054 return -EPERM;
4055 if (!dir->i_op->link)
4056 return -EPERM;
4057 if (S_ISDIR(inode->i_mode))
4058 return -EPERM;
4059
4060 error = security_inode_link(old_dentry, dir, new_dentry);
4061 if (error)
4062 return error;
4063
4064 inode_lock(inode);
4065
4066 if (inode->i_nlink == 0 && !(inode->i_state & I_LINKABLE))
4067 error = -ENOENT;
4068 else if (max_links && inode->i_nlink >= max_links)
4069 error = -EMLINK;
4070 else {
4071 error = try_break_deleg(inode, delegated_inode);
4072 if (!error)
4073 error = dir->i_op->link(old_dentry, dir, new_dentry);
4074 }
4075
4076 if (!error && (inode->i_state & I_LINKABLE)) {
4077 spin_lock(&inode->i_lock);
4078 inode->i_state &= ~I_LINKABLE;
4079 spin_unlock(&inode->i_lock);
4080 }
4081 inode_unlock(inode);
4082 if (!error)
4083 fsnotify_link(dir, inode, new_dentry);
4084 return error;
4085}
4086EXPORT_SYMBOL(vfs_link);
4087
4088
4089
4090
4091
4092
4093
4094
4095
4096
4097static int do_linkat(int olddfd, const char __user *oldname, int newdfd,
4098 const char __user *newname, int flags)
4099{
4100 struct dentry *new_dentry;
4101 struct path old_path, new_path;
4102 struct inode *delegated_inode = NULL;
4103 int how = 0;
4104 int error;
4105
4106 if ((flags & ~(AT_SYMLINK_FOLLOW | AT_EMPTY_PATH)) != 0)
4107 return -EINVAL;
4108
4109
4110
4111
4112
4113 if (flags & AT_EMPTY_PATH) {
4114 if (!capable(CAP_DAC_READ_SEARCH))
4115 return -ENOENT;
4116 how = LOOKUP_EMPTY;
4117 }
4118
4119 if (flags & AT_SYMLINK_FOLLOW)
4120 how |= LOOKUP_FOLLOW;
4121retry:
4122 error = user_path_at(olddfd, oldname, how, &old_path);
4123 if (error)
4124 return error;
4125
4126 new_dentry = user_path_create(newdfd, newname, &new_path,
4127 (how & LOOKUP_REVAL));
4128 error = PTR_ERR(new_dentry);
4129 if (IS_ERR(new_dentry))
4130 goto out;
4131
4132 error = -EXDEV;
4133 if (old_path.mnt != new_path.mnt)
4134 goto out_dput;
4135 error = may_linkat(&old_path);
4136 if (unlikely(error))
4137 goto out_dput;
4138 error = security_path_link(old_path.dentry, &new_path, new_dentry);
4139 if (error)
4140 goto out_dput;
4141 error = vfs_link(old_path.dentry, new_path.dentry->d_inode, new_dentry, &delegated_inode);
4142out_dput:
4143 done_path_create(&new_path, new_dentry);
4144 if (delegated_inode) {
4145 error = break_deleg_wait(&delegated_inode);
4146 if (!error) {
4147 path_put(&old_path);
4148 goto retry;
4149 }
4150 }
4151 if (retry_estale(error, how)) {
4152 path_put(&old_path);
4153 how |= LOOKUP_REVAL;
4154 goto retry;
4155 }
4156out:
4157 path_put(&old_path);
4158
4159 return error;
4160}
4161
4162SYSCALL_DEFINE5(linkat, int, olddfd, const char __user *, oldname,
4163 int, newdfd, const char __user *, newname, int, flags)
4164{
4165 return do_linkat(olddfd, oldname, newdfd, newname, flags);
4166}
4167
4168SYSCALL_DEFINE2(link, const char __user *, oldname, const char __user *, newname)
4169{
4170 return do_linkat(AT_FDCWD, oldname, AT_FDCWD, newname, 0);
4171}
4172
4173
4174
4175
4176
4177
4178
4179
4180
4181
4182
4183
4184
4185
4186
4187
4188
4189
4190
4191
4192
4193
4194
4195
4196
4197
4198
4199
4200
4201
4202
4203
4204
4205
4206
4207
4208
4209
4210
4211
4212
4213
4214
4215
4216
4217
4218
4219
4220
4221
4222
4223int vfs_rename(struct inode *old_dir, struct dentry *old_dentry,
4224 struct inode *new_dir, struct dentry *new_dentry,
4225 struct inode **delegated_inode, unsigned int flags)
4226{
4227 int error;
4228 bool is_dir = d_is_dir(old_dentry);
4229 struct inode *source = old_dentry->d_inode;
4230 struct inode *target = new_dentry->d_inode;
4231 bool new_is_dir = false;
4232 unsigned max_links = new_dir->i_sb->s_max_links;
4233 struct name_snapshot old_name;
4234
4235 if (source == target)
4236 return 0;
4237
4238 error = may_delete(old_dir, old_dentry, is_dir);
4239 if (error)
4240 return error;
4241
4242 if (!target) {
4243 error = may_create(new_dir, new_dentry);
4244 } else {
4245 new_is_dir = d_is_dir(new_dentry);
4246
4247 if (!(flags & RENAME_EXCHANGE))
4248 error = may_delete(new_dir, new_dentry, is_dir);
4249 else
4250 error = may_delete(new_dir, new_dentry, new_is_dir);
4251 }
4252 if (error)
4253 return error;
4254
4255 if (!old_dir->i_op->rename)
4256 return -EPERM;
4257
4258
4259
4260
4261
4262 if (new_dir != old_dir) {
4263 if (is_dir) {
4264 error = inode_permission(source, MAY_WRITE);
4265 if (error)
4266 return error;
4267 }
4268 if ((flags & RENAME_EXCHANGE) && new_is_dir) {
4269 error = inode_permission(target, MAY_WRITE);
4270 if (error)
4271 return error;
4272 }
4273 }
4274
4275 error = security_inode_rename(old_dir, old_dentry, new_dir, new_dentry,
4276 flags);
4277 if (error)
4278 return error;
4279
4280 take_dentry_name_snapshot(&old_name, old_dentry);
4281 dget(new_dentry);
4282 if (!is_dir || (flags & RENAME_EXCHANGE))
4283 lock_two_nondirectories(source, target);
4284 else if (target)
4285 inode_lock(target);
4286
4287 error = -EBUSY;
4288 if (is_local_mountpoint(old_dentry) || is_local_mountpoint(new_dentry))
4289 goto out;
4290
4291 if (max_links && new_dir != old_dir) {
4292 error = -EMLINK;
4293 if (is_dir && !new_is_dir && new_dir->i_nlink >= max_links)
4294 goto out;
4295 if ((flags & RENAME_EXCHANGE) && !is_dir && new_is_dir &&
4296 old_dir->i_nlink >= max_links)
4297 goto out;
4298 }
4299 if (!is_dir) {
4300 error = try_break_deleg(source, delegated_inode);
4301 if (error)
4302 goto out;
4303 }
4304 if (target && !new_is_dir) {
4305 error = try_break_deleg(target, delegated_inode);
4306 if (error)
4307 goto out;
4308 }
4309 error = old_dir->i_op->rename(old_dir, old_dentry,
4310 new_dir, new_dentry, flags);
4311 if (error)
4312 goto out;
4313
4314 if (!(flags & RENAME_EXCHANGE) && target) {
4315 if (is_dir) {
4316 shrink_dcache_parent(new_dentry);
4317 target->i_flags |= S_DEAD;
4318 }
4319 dont_mount(new_dentry);
4320 detach_mounts(new_dentry);
4321 }
4322 if (!(old_dir->i_sb->s_type->fs_flags & FS_RENAME_DOES_D_MOVE)) {
4323 if (!(flags & RENAME_EXCHANGE))
4324 d_move(old_dentry, new_dentry);
4325 else
4326 d_exchange(old_dentry, new_dentry);
4327 }
4328out:
4329 if (!is_dir || (flags & RENAME_EXCHANGE))
4330 unlock_two_nondirectories(source, target);
4331 else if (target)
4332 inode_unlock(target);
4333 dput(new_dentry);
4334 if (!error) {
4335 fsnotify_move(old_dir, new_dir, &old_name.name, is_dir,
4336 !(flags & RENAME_EXCHANGE) ? target : NULL, old_dentry);
4337 if (flags & RENAME_EXCHANGE) {
4338 fsnotify_move(new_dir, old_dir, &old_dentry->d_name,
4339 new_is_dir, NULL, new_dentry);
4340 }
4341 }
4342 release_dentry_name_snapshot(&old_name);
4343
4344 return error;
4345}
4346EXPORT_SYMBOL(vfs_rename);
4347
4348static int do_renameat2(int olddfd, const char __user *oldname, int newdfd,
4349 const char __user *newname, unsigned int flags)
4350{
4351 struct dentry *old_dentry, *new_dentry;
4352 struct dentry *trap;
4353 struct path old_path, new_path;
4354 struct qstr old_last, new_last;
4355 int old_type, new_type;
4356 struct inode *delegated_inode = NULL;
4357 struct filename *from;
4358 struct filename *to;
4359 unsigned int lookup_flags = 0, target_flags = LOOKUP_RENAME_TARGET;
4360 bool should_retry = false;
4361 int error;
4362
4363 if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE | RENAME_WHITEOUT))
4364 return -EINVAL;
4365
4366 if ((flags & (RENAME_NOREPLACE | RENAME_WHITEOUT)) &&
4367 (flags & RENAME_EXCHANGE))
4368 return -EINVAL;
4369
4370 if (flags & RENAME_EXCHANGE)
4371 target_flags = 0;
4372
4373retry:
4374 from = filename_parentat(olddfd, getname(oldname), lookup_flags,
4375 &old_path, &old_last, &old_type);
4376 if (IS_ERR(from)) {
4377 error = PTR_ERR(from);
4378 goto exit;
4379 }
4380
4381 to = filename_parentat(newdfd, getname(newname), lookup_flags,
4382 &new_path, &new_last, &new_type);
4383 if (IS_ERR(to)) {
4384 error = PTR_ERR(to);
4385 goto exit1;
4386 }
4387
4388 error = -EXDEV;
4389 if (old_path.mnt != new_path.mnt)
4390 goto exit2;
4391
4392 error = -EBUSY;
4393 if (old_type != LAST_NORM)
4394 goto exit2;
4395
4396 if (flags & RENAME_NOREPLACE)
4397 error = -EEXIST;
4398 if (new_type != LAST_NORM)
4399 goto exit2;
4400
4401 error = mnt_want_write(old_path.mnt);
4402 if (error)
4403 goto exit2;
4404
4405retry_deleg:
4406 trap = lock_rename(new_path.dentry, old_path.dentry);
4407
4408 old_dentry = __lookup_hash(&old_last, old_path.dentry, lookup_flags);
4409 error = PTR_ERR(old_dentry);
4410 if (IS_ERR(old_dentry))
4411 goto exit3;
4412
4413 error = -ENOENT;
4414 if (d_is_negative(old_dentry))
4415 goto exit4;
4416 new_dentry = __lookup_hash(&new_last, new_path.dentry, lookup_flags | target_flags);
4417 error = PTR_ERR(new_dentry);
4418 if (IS_ERR(new_dentry))
4419 goto exit4;
4420 error = -EEXIST;
4421 if ((flags & RENAME_NOREPLACE) && d_is_positive(new_dentry))
4422 goto exit5;
4423 if (flags & RENAME_EXCHANGE) {
4424 error = -ENOENT;
4425 if (d_is_negative(new_dentry))
4426 goto exit5;
4427
4428 if (!d_is_dir(new_dentry)) {
4429 error = -ENOTDIR;
4430 if (new_last.name[new_last.len])
4431 goto exit5;
4432 }
4433 }
4434
4435 if (!d_is_dir(old_dentry)) {
4436 error = -ENOTDIR;
4437 if (old_last.name[old_last.len])
4438 goto exit5;
4439 if (!(flags & RENAME_EXCHANGE) && new_last.name[new_last.len])
4440 goto exit5;
4441 }
4442
4443 error = -EINVAL;
4444 if (old_dentry == trap)
4445 goto exit5;
4446
4447 if (!(flags & RENAME_EXCHANGE))
4448 error = -ENOTEMPTY;
4449 if (new_dentry == trap)
4450 goto exit5;
4451
4452 error = security_path_rename(&old_path, old_dentry,
4453 &new_path, new_dentry, flags);
4454 if (error)
4455 goto exit5;
4456 error = vfs_rename(old_path.dentry->d_inode, old_dentry,
4457 new_path.dentry->d_inode, new_dentry,
4458 &delegated_inode, flags);
4459exit5:
4460 dput(new_dentry);
4461exit4:
4462 dput(old_dentry);
4463exit3:
4464 unlock_rename(new_path.dentry, old_path.dentry);
4465 if (delegated_inode) {
4466 error = break_deleg_wait(&delegated_inode);
4467 if (!error)
4468 goto retry_deleg;
4469 }
4470 mnt_drop_write(old_path.mnt);
4471exit2:
4472 if (retry_estale(error, lookup_flags))
4473 should_retry = true;
4474 path_put(&new_path);
4475 putname(to);
4476exit1:
4477 path_put(&old_path);
4478 putname(from);
4479 if (should_retry) {
4480 should_retry = false;
4481 lookup_flags |= LOOKUP_REVAL;
4482 goto retry;
4483 }
4484exit:
4485 return error;
4486}
4487
4488SYSCALL_DEFINE5(renameat2, int, olddfd, const char __user *, oldname,
4489 int, newdfd, const char __user *, newname, unsigned int, flags)
4490{
4491 return do_renameat2(olddfd, oldname, newdfd, newname, flags);
4492}
4493
4494SYSCALL_DEFINE4(renameat, int, olddfd, const char __user *, oldname,
4495 int, newdfd, const char __user *, newname)
4496{
4497 return do_renameat2(olddfd, oldname, newdfd, newname, 0);
4498}
4499
4500SYSCALL_DEFINE2(rename, const char __user *, oldname, const char __user *, newname)
4501{
4502 return do_renameat2(AT_FDCWD, oldname, AT_FDCWD, newname, 0);
4503}
4504
4505int readlink_copy(char __user *buffer, int buflen, const char *link)
4506{
4507 int len = PTR_ERR(link);
4508 if (IS_ERR(link))
4509 goto out;
4510
4511 len = strlen(link);
4512 if (len > (unsigned) buflen)
4513 len = buflen;
4514 if (copy_to_user(buffer, link, len))
4515 len = -EFAULT;
4516out:
4517 return len;
4518}
4519
4520
4521
4522
4523
4524
4525
4526
4527
4528
4529
4530int vfs_readlink(struct dentry *dentry, char __user *buffer, int buflen)
4531{
4532 struct inode *inode = d_inode(dentry);
4533 DEFINE_DELAYED_CALL(done);
4534 const char *link;
4535 int res;
4536
4537 if (unlikely(!(inode->i_opflags & IOP_DEFAULT_READLINK))) {
4538 if (unlikely(inode->i_op->readlink))
4539 return inode->i_op->readlink(dentry, buffer, buflen);
4540
4541 if (!d_is_symlink(dentry))
4542 return -EINVAL;
4543
4544 spin_lock(&inode->i_lock);
4545 inode->i_opflags |= IOP_DEFAULT_READLINK;
4546 spin_unlock(&inode->i_lock);
4547 }
4548
4549 link = READ_ONCE(inode->i_link);
4550 if (!link) {
4551 link = inode->i_op->get_link(dentry, inode, &done);
4552 if (IS_ERR(link))
4553 return PTR_ERR(link);
4554 }
4555 res = readlink_copy(buffer, buflen, link);
4556 do_delayed_call(&done);
4557 return res;
4558}
4559EXPORT_SYMBOL(vfs_readlink);
4560
4561
4562
4563
4564
4565
4566
4567
4568
4569
4570
4571
4572const char *vfs_get_link(struct dentry *dentry, struct delayed_call *done)
4573{
4574 const char *res = ERR_PTR(-EINVAL);
4575 struct inode *inode = d_inode(dentry);
4576
4577 if (d_is_symlink(dentry)) {
4578 res = ERR_PTR(security_inode_readlink(dentry));
4579 if (!res)
4580 res = inode->i_op->get_link(dentry, inode, done);
4581 }
4582 return res;
4583}
4584EXPORT_SYMBOL(vfs_get_link);
4585
4586
4587const char *page_get_link(struct dentry *dentry, struct inode *inode,
4588 struct delayed_call *callback)
4589{
4590 char *kaddr;
4591 struct page *page;
4592 struct address_space *mapping = inode->i_mapping;
4593
4594 if (!dentry) {
4595 page = find_get_page(mapping, 0);
4596 if (!page)
4597 return ERR_PTR(-ECHILD);
4598 if (!PageUptodate(page)) {
4599 put_page(page);
4600 return ERR_PTR(-ECHILD);
4601 }
4602 } else {
4603 page = read_mapping_page(mapping, 0, NULL);
4604 if (IS_ERR(page))
4605 return (char*)page;
4606 }
4607 set_delayed_call(callback, page_put_link, page);
4608 BUG_ON(mapping_gfp_mask(mapping) & __GFP_HIGHMEM);
4609 kaddr = page_address(page);
4610 nd_terminate_link(kaddr, inode->i_size, PAGE_SIZE - 1);
4611 return kaddr;
4612}
4613
4614EXPORT_SYMBOL(page_get_link);
4615
4616void page_put_link(void *arg)
4617{
4618 put_page(arg);
4619}
4620EXPORT_SYMBOL(page_put_link);
4621
4622int page_readlink(struct dentry *dentry, char __user *buffer, int buflen)
4623{
4624 DEFINE_DELAYED_CALL(done);
4625 int res = readlink_copy(buffer, buflen,
4626 page_get_link(dentry, d_inode(dentry),
4627 &done));
4628 do_delayed_call(&done);
4629 return res;
4630}
4631EXPORT_SYMBOL(page_readlink);
4632
4633
4634
4635
4636int __page_symlink(struct inode *inode, const char *symname, int len, int nofs)
4637{
4638 struct address_space *mapping = inode->i_mapping;
4639 struct page *page;
4640 void *fsdata;
4641 int err;
4642 unsigned int flags = 0;
4643 if (nofs)
4644 flags |= AOP_FLAG_NOFS;
4645
4646retry:
4647 err = pagecache_write_begin(NULL, mapping, 0, len-1,
4648 flags, &page, &fsdata);
4649 if (err)
4650 goto fail;
4651
4652 memcpy(page_address(page), symname, len-1);
4653
4654 err = pagecache_write_end(NULL, mapping, 0, len-1, len-1,
4655 page, fsdata);
4656 if (err < 0)
4657 goto fail;
4658 if (err < len-1)
4659 goto retry;
4660
4661 mark_inode_dirty(inode);
4662 return 0;
4663fail:
4664 return err;
4665}
4666EXPORT_SYMBOL(__page_symlink);
4667
4668int page_symlink(struct inode *inode, const char *symname, int len)
4669{
4670 return __page_symlink(inode, symname, len,
4671 !mapping_gfp_constraint(inode->i_mapping, __GFP_FS));
4672}
4673EXPORT_SYMBOL(page_symlink);
4674
4675const struct inode_operations page_symlink_inode_operations = {
4676 .get_link = page_get_link,
4677};
4678EXPORT_SYMBOL(page_symlink_inode_operations);
4679