1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47#include <linux/slab.h>
48#include <linux/poll.h>
49#include <linux/fs.h>
50#include <linux/file.h>
51#include <linux/jhash.h>
52#include <linux/init.h>
53#include <linux/futex.h>
54#include <linux/mount.h>
55#include <linux/pagemap.h>
56#include <linux/syscalls.h>
57#include <linux/signal.h>
58#include <linux/export.h>
59#include <linux/magic.h>
60#include <linux/pid.h>
61#include <linux/nsproxy.h>
62#include <linux/ptrace.h>
63#include <linux/sched/rt.h>
64#include <linux/sched/wake_q.h>
65#include <linux/sched/mm.h>
66#include <linux/hugetlb.h>
67#include <linux/freezer.h>
68#include <linux/bootmem.h>
69#include <linux/fault-inject.h>
70
71#include <asm/futex.h>
72
73#include "locking/rtmutex_common.h"
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176#ifndef CONFIG_HAVE_FUTEX_CMPXCHG
177int __read_mostly futex_cmpxchg_enabled;
178#endif
179
180
181
182
183
184#ifdef CONFIG_MMU
185# define FLAGS_SHARED 0x01
186#else
187
188
189
190
191# define FLAGS_SHARED 0x00
192#endif
193#define FLAGS_CLOCKRT 0x02
194#define FLAGS_HAS_TIMEOUT 0x04
195
196
197
198
199struct futex_pi_state {
200
201
202
203
204 struct list_head list;
205
206
207
208
209 struct rt_mutex pi_mutex;
210
211 struct task_struct *owner;
212 atomic_t refcount;
213
214 union futex_key key;
215} __randomize_layout;
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239struct futex_q {
240 struct plist_node list;
241
242 struct task_struct *task;
243 spinlock_t *lock_ptr;
244 union futex_key key;
245 struct futex_pi_state *pi_state;
246 struct rt_mutex_waiter *rt_waiter;
247 union futex_key *requeue_pi_key;
248 u32 bitset;
249} __randomize_layout;
250
251static const struct futex_q futex_q_init = {
252
253 .key = FUTEX_KEY_INIT,
254 .bitset = FUTEX_BITSET_MATCH_ANY
255};
256
257
258
259
260
261
262struct futex_hash_bucket {
263 atomic_t waiters;
264 spinlock_t lock;
265 struct plist_head chain;
266} ____cacheline_aligned_in_smp;
267
268
269
270
271
272
273static struct {
274 struct futex_hash_bucket *queues;
275 unsigned long hashsize;
276} __futex_data __read_mostly __aligned(2*sizeof(long));
277#define futex_queues (__futex_data.queues)
278#define futex_hashsize (__futex_data.hashsize)
279
280
281
282
283
284#ifdef CONFIG_FAIL_FUTEX
285
286static struct {
287 struct fault_attr attr;
288
289 bool ignore_private;
290} fail_futex = {
291 .attr = FAULT_ATTR_INITIALIZER,
292 .ignore_private = false,
293};
294
295static int __init setup_fail_futex(char *str)
296{
297 return setup_fault_attr(&fail_futex.attr, str);
298}
299__setup("fail_futex=", setup_fail_futex);
300
301static bool should_fail_futex(bool fshared)
302{
303 if (fail_futex.ignore_private && !fshared)
304 return false;
305
306 return should_fail(&fail_futex.attr, 1);
307}
308
309#ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
310
311static int __init fail_futex_debugfs(void)
312{
313 umode_t mode = S_IFREG | S_IRUSR | S_IWUSR;
314 struct dentry *dir;
315
316 dir = fault_create_debugfs_attr("fail_futex", NULL,
317 &fail_futex.attr);
318 if (IS_ERR(dir))
319 return PTR_ERR(dir);
320
321 if (!debugfs_create_bool("ignore-private", mode, dir,
322 &fail_futex.ignore_private)) {
323 debugfs_remove_recursive(dir);
324 return -ENOMEM;
325 }
326
327 return 0;
328}
329
330late_initcall(fail_futex_debugfs);
331
332#endif
333
334#else
335static inline bool should_fail_futex(bool fshared)
336{
337 return false;
338}
339#endif
340
341static inline void futex_get_mm(union futex_key *key)
342{
343 mmgrab(key->private.mm);
344
345
346
347
348
349 smp_mb__after_atomic();
350}
351
352
353
354
355static inline void hb_waiters_inc(struct futex_hash_bucket *hb)
356{
357#ifdef CONFIG_SMP
358 atomic_inc(&hb->waiters);
359
360
361
362 smp_mb__after_atomic();
363#endif
364}
365
366
367
368
369
370static inline void hb_waiters_dec(struct futex_hash_bucket *hb)
371{
372#ifdef CONFIG_SMP
373 atomic_dec(&hb->waiters);
374#endif
375}
376
377static inline int hb_waiters_pending(struct futex_hash_bucket *hb)
378{
379#ifdef CONFIG_SMP
380 return atomic_read(&hb->waiters);
381#else
382 return 1;
383#endif
384}
385
386
387
388
389
390
391
392
393static struct futex_hash_bucket *hash_futex(union futex_key *key)
394{
395 u32 hash = jhash2((u32*)&key->both.word,
396 (sizeof(key->both.word)+sizeof(key->both.ptr))/4,
397 key->both.offset);
398 return &futex_queues[hash & (futex_hashsize - 1)];
399}
400
401
402
403
404
405
406
407
408
409static inline int match_futex(union futex_key *key1, union futex_key *key2)
410{
411 return (key1 && key2
412 && key1->both.word == key2->both.word
413 && key1->both.ptr == key2->both.ptr
414 && key1->both.offset == key2->both.offset);
415}
416
417
418
419
420
421
422static void get_futex_key_refs(union futex_key *key)
423{
424 if (!key->both.ptr)
425 return;
426
427
428
429
430
431
432 if (!IS_ENABLED(CONFIG_MMU)) {
433 smp_mb();
434 return;
435 }
436
437 switch (key->both.offset & (FUT_OFF_INODE|FUT_OFF_MMSHARED)) {
438 case FUT_OFF_INODE:
439 ihold(key->shared.inode);
440 break;
441 case FUT_OFF_MMSHARED:
442 futex_get_mm(key);
443 break;
444 default:
445
446
447
448
449
450 smp_mb();
451 }
452}
453
454
455
456
457
458
459
460static void drop_futex_key_refs(union futex_key *key)
461{
462 if (!key->both.ptr) {
463
464 WARN_ON_ONCE(1);
465 return;
466 }
467
468 if (!IS_ENABLED(CONFIG_MMU))
469 return;
470
471 switch (key->both.offset & (FUT_OFF_INODE|FUT_OFF_MMSHARED)) {
472 case FUT_OFF_INODE:
473 iput(key->shared.inode);
474 break;
475 case FUT_OFF_MMSHARED:
476 mmdrop(key->private.mm);
477 break;
478 }
479}
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499static int
500get_futex_key(u32 __user *uaddr, int fshared, union futex_key *key, int rw)
501{
502 unsigned long address = (unsigned long)uaddr;
503 struct mm_struct *mm = current->mm;
504 struct page *page, *tail;
505 struct address_space *mapping;
506 int err, ro = 0;
507
508
509
510
511 key->both.offset = address % PAGE_SIZE;
512 if (unlikely((address % sizeof(u32)) != 0))
513 return -EINVAL;
514 address -= key->both.offset;
515
516 if (unlikely(!access_ok(rw, uaddr, sizeof(u32))))
517 return -EFAULT;
518
519 if (unlikely(should_fail_futex(fshared)))
520 return -EFAULT;
521
522
523
524
525
526
527
528
529 if (!fshared) {
530 key->private.mm = mm;
531 key->private.address = address;
532 get_futex_key_refs(key);
533 return 0;
534 }
535
536again:
537
538 if (unlikely(should_fail_futex(fshared)))
539 return -EFAULT;
540
541 err = get_user_pages_fast(address, 1, 1, &page);
542
543
544
545
546 if (err == -EFAULT && rw == VERIFY_READ) {
547 err = get_user_pages_fast(address, 1, 0, &page);
548 ro = 1;
549 }
550 if (err < 0)
551 return err;
552 else
553 err = 0;
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573 tail = page;
574 page = compound_head(page);
575 mapping = READ_ONCE(page->mapping);
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592 if (unlikely(!mapping)) {
593 int shmem_swizzled;
594
595
596
597
598
599
600 lock_page(page);
601 shmem_swizzled = PageSwapCache(page) || page->mapping;
602 unlock_page(page);
603 put_page(page);
604
605 if (shmem_swizzled)
606 goto again;
607
608 return -EFAULT;
609 }
610
611
612
613
614
615
616
617
618
619
620
621 if (PageAnon(page)) {
622
623
624
625
626 if (unlikely(should_fail_futex(fshared)) || ro) {
627 err = -EFAULT;
628 goto out;
629 }
630
631 key->both.offset |= FUT_OFF_MMSHARED;
632 key->private.mm = mm;
633 key->private.address = address;
634
635 get_futex_key_refs(key);
636
637 } else {
638 struct inode *inode;
639
640
641
642
643
644
645
646
647
648
649
650
651 rcu_read_lock();
652
653 if (READ_ONCE(page->mapping) != mapping) {
654 rcu_read_unlock();
655 put_page(page);
656
657 goto again;
658 }
659
660 inode = READ_ONCE(mapping->host);
661 if (!inode) {
662 rcu_read_unlock();
663 put_page(page);
664
665 goto again;
666 }
667
668
669
670
671
672
673
674
675
676
677
678
679
680 if (!atomic_inc_not_zero(&inode->i_count)) {
681 rcu_read_unlock();
682 put_page(page);
683
684 goto again;
685 }
686
687
688 if (WARN_ON_ONCE(inode->i_mapping != mapping)) {
689 err = -EFAULT;
690 rcu_read_unlock();
691 iput(inode);
692
693 goto out;
694 }
695
696 key->both.offset |= FUT_OFF_INODE;
697 key->shared.inode = inode;
698 key->shared.pgoff = basepage_index(tail);
699 rcu_read_unlock();
700 }
701
702out:
703 put_page(page);
704 return err;
705}
706
707static inline void put_futex_key(union futex_key *key)
708{
709 drop_futex_key_refs(key);
710}
711
712
713
714
715
716
717
718
719
720
721
722
723
724static int fault_in_user_writeable(u32 __user *uaddr)
725{
726 struct mm_struct *mm = current->mm;
727 int ret;
728
729 down_read(&mm->mmap_sem);
730 ret = fixup_user_fault(current, mm, (unsigned long)uaddr,
731 FAULT_FLAG_WRITE, NULL);
732 up_read(&mm->mmap_sem);
733
734 return ret < 0 ? ret : 0;
735}
736
737
738
739
740
741
742
743
744static struct futex_q *futex_top_waiter(struct futex_hash_bucket *hb,
745 union futex_key *key)
746{
747 struct futex_q *this;
748
749 plist_for_each_entry(this, &hb->chain, list) {
750 if (match_futex(&this->key, key))
751 return this;
752 }
753 return NULL;
754}
755
756static int cmpxchg_futex_value_locked(u32 *curval, u32 __user *uaddr,
757 u32 uval, u32 newval)
758{
759 int ret;
760
761 pagefault_disable();
762 ret = futex_atomic_cmpxchg_inatomic(curval, uaddr, uval, newval);
763 pagefault_enable();
764
765 return ret;
766}
767
768static int get_futex_value_locked(u32 *dest, u32 __user *from)
769{
770 int ret;
771
772 pagefault_disable();
773 ret = __get_user(*dest, from);
774 pagefault_enable();
775
776 return ret ? -EFAULT : 0;
777}
778
779
780
781
782
783static int refill_pi_state_cache(void)
784{
785 struct futex_pi_state *pi_state;
786
787 if (likely(current->pi_state_cache))
788 return 0;
789
790 pi_state = kzalloc(sizeof(*pi_state), GFP_KERNEL);
791
792 if (!pi_state)
793 return -ENOMEM;
794
795 INIT_LIST_HEAD(&pi_state->list);
796
797 pi_state->owner = NULL;
798 atomic_set(&pi_state->refcount, 1);
799 pi_state->key = FUTEX_KEY_INIT;
800
801 current->pi_state_cache = pi_state;
802
803 return 0;
804}
805
806static struct futex_pi_state *alloc_pi_state(void)
807{
808 struct futex_pi_state *pi_state = current->pi_state_cache;
809
810 WARN_ON(!pi_state);
811 current->pi_state_cache = NULL;
812
813 return pi_state;
814}
815
816static void get_pi_state(struct futex_pi_state *pi_state)
817{
818 WARN_ON_ONCE(!atomic_inc_not_zero(&pi_state->refcount));
819}
820
821
822
823
824
825static void put_pi_state(struct futex_pi_state *pi_state)
826{
827 if (!pi_state)
828 return;
829
830 if (!atomic_dec_and_test(&pi_state->refcount))
831 return;
832
833
834
835
836
837 if (pi_state->owner) {
838 struct task_struct *owner;
839
840 raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);
841 owner = pi_state->owner;
842 if (owner) {
843 raw_spin_lock(&owner->pi_lock);
844 list_del_init(&pi_state->list);
845 raw_spin_unlock(&owner->pi_lock);
846 }
847 rt_mutex_proxy_unlock(&pi_state->pi_mutex, owner);
848 raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);
849 }
850
851 if (current->pi_state_cache) {
852 kfree(pi_state);
853 } else {
854
855
856
857
858
859 pi_state->owner = NULL;
860 atomic_set(&pi_state->refcount, 1);
861 current->pi_state_cache = pi_state;
862 }
863}
864
865#ifdef CONFIG_FUTEX_PI
866
867
868
869
870
871
872void exit_pi_state_list(struct task_struct *curr)
873{
874 struct list_head *next, *head = &curr->pi_state_list;
875 struct futex_pi_state *pi_state;
876 struct futex_hash_bucket *hb;
877 union futex_key key = FUTEX_KEY_INIT;
878
879 if (!futex_cmpxchg_enabled)
880 return;
881
882
883
884
885
886 raw_spin_lock_irq(&curr->pi_lock);
887 while (!list_empty(head)) {
888 next = head->next;
889 pi_state = list_entry(next, struct futex_pi_state, list);
890 key = pi_state->key;
891 hb = hash_futex(&key);
892
893
894
895
896
897
898
899
900
901
902
903 if (!atomic_inc_not_zero(&pi_state->refcount)) {
904 raw_spin_unlock_irq(&curr->pi_lock);
905 cpu_relax();
906 raw_spin_lock_irq(&curr->pi_lock);
907 continue;
908 }
909 raw_spin_unlock_irq(&curr->pi_lock);
910
911 spin_lock(&hb->lock);
912 raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);
913 raw_spin_lock(&curr->pi_lock);
914
915
916
917
918 if (head->next != next) {
919
920 raw_spin_unlock(&pi_state->pi_mutex.wait_lock);
921 spin_unlock(&hb->lock);
922 put_pi_state(pi_state);
923 continue;
924 }
925
926 WARN_ON(pi_state->owner != curr);
927 WARN_ON(list_empty(&pi_state->list));
928 list_del_init(&pi_state->list);
929 pi_state->owner = NULL;
930
931 raw_spin_unlock(&curr->pi_lock);
932 raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);
933 spin_unlock(&hb->lock);
934
935 rt_mutex_futex_unlock(&pi_state->pi_mutex);
936 put_pi_state(pi_state);
937
938 raw_spin_lock_irq(&curr->pi_lock);
939 }
940 raw_spin_unlock_irq(&curr->pi_lock);
941}
942
943#endif
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033static int attach_to_pi_state(u32 __user *uaddr, u32 uval,
1034 struct futex_pi_state *pi_state,
1035 struct futex_pi_state **ps)
1036{
1037 pid_t pid = uval & FUTEX_TID_MASK;
1038 u32 uval2;
1039 int ret;
1040
1041
1042
1043
1044 if (unlikely(!pi_state))
1045 return -EINVAL;
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059 WARN_ON(!atomic_read(&pi_state->refcount));
1060
1061
1062
1063
1064
1065 raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);
1066
1067
1068
1069
1070
1071
1072
1073 if (get_futex_value_locked(&uval2, uaddr))
1074 goto out_efault;
1075
1076 if (uval != uval2)
1077 goto out_eagain;
1078
1079
1080
1081
1082 if (uval & FUTEX_OWNER_DIED) {
1083
1084
1085
1086
1087
1088 if (!pi_state->owner) {
1089
1090
1091
1092
1093 if (pid)
1094 goto out_einval;
1095
1096
1097
1098 goto out_attach;
1099 }
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109 if (!pid)
1110 goto out_attach;
1111 } else {
1112
1113
1114
1115
1116 if (!pi_state->owner)
1117 goto out_einval;
1118 }
1119
1120
1121
1122
1123
1124
1125 if (pid != task_pid_vnr(pi_state->owner))
1126 goto out_einval;
1127
1128out_attach:
1129 get_pi_state(pi_state);
1130 raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);
1131 *ps = pi_state;
1132 return 0;
1133
1134out_einval:
1135 ret = -EINVAL;
1136 goto out_error;
1137
1138out_eagain:
1139 ret = -EAGAIN;
1140 goto out_error;
1141
1142out_efault:
1143 ret = -EFAULT;
1144 goto out_error;
1145
1146out_error:
1147 raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);
1148 return ret;
1149}
1150
1151
1152
1153
1154
1155static int attach_to_pi_owner(u32 uval, union futex_key *key,
1156 struct futex_pi_state **ps)
1157{
1158 pid_t pid = uval & FUTEX_TID_MASK;
1159 struct futex_pi_state *pi_state;
1160 struct task_struct *p;
1161
1162
1163
1164
1165
1166 if (!pid)
1167 return -ESRCH;
1168 p = find_get_task_by_vpid(pid);
1169 if (!p)
1170 return -ESRCH;
1171
1172 if (unlikely(p->flags & PF_KTHREAD)) {
1173 put_task_struct(p);
1174 return -EPERM;
1175 }
1176
1177
1178
1179
1180
1181
1182
1183 raw_spin_lock_irq(&p->pi_lock);
1184 if (unlikely(p->flags & PF_EXITING)) {
1185
1186
1187
1188
1189
1190 int ret = (p->flags & PF_EXITPIDONE) ? -ESRCH : -EAGAIN;
1191
1192 raw_spin_unlock_irq(&p->pi_lock);
1193 put_task_struct(p);
1194 return ret;
1195 }
1196
1197
1198
1199
1200
1201
1202
1203 pi_state = alloc_pi_state();
1204
1205
1206
1207
1208
1209 rt_mutex_init_proxy_locked(&pi_state->pi_mutex, p);
1210
1211
1212 pi_state->key = *key;
1213
1214 WARN_ON(!list_empty(&pi_state->list));
1215 list_add(&pi_state->list, &p->pi_state_list);
1216
1217
1218
1219
1220 pi_state->owner = p;
1221 raw_spin_unlock_irq(&p->pi_lock);
1222
1223 put_task_struct(p);
1224
1225 *ps = pi_state;
1226
1227 return 0;
1228}
1229
1230static int lookup_pi_state(u32 __user *uaddr, u32 uval,
1231 struct futex_hash_bucket *hb,
1232 union futex_key *key, struct futex_pi_state **ps)
1233{
1234 struct futex_q *top_waiter = futex_top_waiter(hb, key);
1235
1236
1237
1238
1239
1240 if (top_waiter)
1241 return attach_to_pi_state(uaddr, uval, top_waiter->pi_state, ps);
1242
1243
1244
1245
1246
1247 return attach_to_pi_owner(uval, key, ps);
1248}
1249
1250static int lock_pi_update_atomic(u32 __user *uaddr, u32 uval, u32 newval)
1251{
1252 u32 uninitialized_var(curval);
1253
1254 if (unlikely(should_fail_futex(true)))
1255 return -EFAULT;
1256
1257 if (unlikely(cmpxchg_futex_value_locked(&curval, uaddr, uval, newval)))
1258 return -EFAULT;
1259
1260
1261 return curval != uval ? -EAGAIN : 0;
1262}
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282static int futex_lock_pi_atomic(u32 __user *uaddr, struct futex_hash_bucket *hb,
1283 union futex_key *key,
1284 struct futex_pi_state **ps,
1285 struct task_struct *task, int set_waiters)
1286{
1287 u32 uval, newval, vpid = task_pid_vnr(task);
1288 struct futex_q *top_waiter;
1289 int ret;
1290
1291
1292
1293
1294
1295 if (get_futex_value_locked(&uval, uaddr))
1296 return -EFAULT;
1297
1298 if (unlikely(should_fail_futex(true)))
1299 return -EFAULT;
1300
1301
1302
1303
1304 if ((unlikely((uval & FUTEX_TID_MASK) == vpid)))
1305 return -EDEADLK;
1306
1307 if ((unlikely(should_fail_futex(true))))
1308 return -EDEADLK;
1309
1310
1311
1312
1313
1314 top_waiter = futex_top_waiter(hb, key);
1315 if (top_waiter)
1316 return attach_to_pi_state(uaddr, uval, top_waiter->pi_state, ps);
1317
1318
1319
1320
1321
1322
1323
1324 if (!(uval & FUTEX_TID_MASK)) {
1325
1326
1327
1328
1329 newval = uval & FUTEX_OWNER_DIED;
1330 newval |= vpid;
1331
1332
1333 if (set_waiters)
1334 newval |= FUTEX_WAITERS;
1335
1336 ret = lock_pi_update_atomic(uaddr, uval, newval);
1337
1338 return ret < 0 ? ret : 1;
1339 }
1340
1341
1342
1343
1344
1345
1346 newval = uval | FUTEX_WAITERS;
1347 ret = lock_pi_update_atomic(uaddr, uval, newval);
1348 if (ret)
1349 return ret;
1350
1351
1352
1353
1354
1355 return attach_to_pi_owner(uval, key, ps);
1356}
1357
1358
1359
1360
1361
1362
1363
1364static void __unqueue_futex(struct futex_q *q)
1365{
1366 struct futex_hash_bucket *hb;
1367
1368 if (WARN_ON_SMP(!q->lock_ptr || !spin_is_locked(q->lock_ptr))
1369 || WARN_ON(plist_node_empty(&q->list)))
1370 return;
1371
1372 hb = container_of(q->lock_ptr, struct futex_hash_bucket, lock);
1373 plist_del(&q->list, &hb->chain);
1374 hb_waiters_dec(hb);
1375}
1376
1377
1378
1379
1380
1381
1382
1383static void mark_wake_futex(struct wake_q_head *wake_q, struct futex_q *q)
1384{
1385 struct task_struct *p = q->task;
1386
1387 if (WARN(q->pi_state || q->rt_waiter, "refusing to wake PI futex\n"))
1388 return;
1389
1390
1391
1392
1393
1394 wake_q_add(wake_q, p);
1395 __unqueue_futex(q);
1396
1397
1398
1399
1400
1401
1402
1403 smp_store_release(&q->lock_ptr, NULL);
1404}
1405
1406
1407
1408
1409static int wake_futex_pi(u32 __user *uaddr, u32 uval, struct futex_pi_state *pi_state)
1410{
1411 u32 uninitialized_var(curval), newval;
1412 struct task_struct *new_owner;
1413 bool postunlock = false;
1414 DEFINE_WAKE_Q(wake_q);
1415 int ret = 0;
1416
1417 new_owner = rt_mutex_next_owner(&pi_state->pi_mutex);
1418 if (WARN_ON_ONCE(!new_owner)) {
1419
1420
1421
1422
1423
1424
1425
1426
1427 ret = -EAGAIN;
1428 goto out_unlock;
1429 }
1430
1431
1432
1433
1434
1435
1436 newval = FUTEX_WAITERS | task_pid_vnr(new_owner);
1437
1438 if (unlikely(should_fail_futex(true)))
1439 ret = -EFAULT;
1440
1441 if (cmpxchg_futex_value_locked(&curval, uaddr, uval, newval)) {
1442 ret = -EFAULT;
1443
1444 } else if (curval != uval) {
1445
1446
1447
1448
1449
1450
1451 if ((FUTEX_TID_MASK & curval) == uval)
1452 ret = -EAGAIN;
1453 else
1454 ret = -EINVAL;
1455 }
1456
1457 if (ret)
1458 goto out_unlock;
1459
1460
1461
1462
1463
1464
1465 raw_spin_lock(&pi_state->owner->pi_lock);
1466 WARN_ON(list_empty(&pi_state->list));
1467 list_del_init(&pi_state->list);
1468 raw_spin_unlock(&pi_state->owner->pi_lock);
1469
1470 raw_spin_lock(&new_owner->pi_lock);
1471 WARN_ON(!list_empty(&pi_state->list));
1472 list_add(&pi_state->list, &new_owner->pi_state_list);
1473 pi_state->owner = new_owner;
1474 raw_spin_unlock(&new_owner->pi_lock);
1475
1476 postunlock = __rt_mutex_futex_unlock(&pi_state->pi_mutex, &wake_q);
1477
1478out_unlock:
1479 raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);
1480
1481 if (postunlock)
1482 rt_mutex_postunlock(&wake_q);
1483
1484 return ret;
1485}
1486
1487
1488
1489
1490static inline void
1491double_lock_hb(struct futex_hash_bucket *hb1, struct futex_hash_bucket *hb2)
1492{
1493 if (hb1 <= hb2) {
1494 spin_lock(&hb1->lock);
1495 if (hb1 < hb2)
1496 spin_lock_nested(&hb2->lock, SINGLE_DEPTH_NESTING);
1497 } else {
1498 spin_lock(&hb2->lock);
1499 spin_lock_nested(&hb1->lock, SINGLE_DEPTH_NESTING);
1500 }
1501}
1502
1503static inline void
1504double_unlock_hb(struct futex_hash_bucket *hb1, struct futex_hash_bucket *hb2)
1505{
1506 spin_unlock(&hb1->lock);
1507 if (hb1 != hb2)
1508 spin_unlock(&hb2->lock);
1509}
1510
1511
1512
1513
1514static int
1515futex_wake(u32 __user *uaddr, unsigned int flags, int nr_wake, u32 bitset)
1516{
1517 struct futex_hash_bucket *hb;
1518 struct futex_q *this, *next;
1519 union futex_key key = FUTEX_KEY_INIT;
1520 int ret;
1521 DEFINE_WAKE_Q(wake_q);
1522
1523 if (!bitset)
1524 return -EINVAL;
1525
1526 ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &key, VERIFY_READ);
1527 if (unlikely(ret != 0))
1528 goto out;
1529
1530 hb = hash_futex(&key);
1531
1532
1533 if (!hb_waiters_pending(hb))
1534 goto out_put_key;
1535
1536 spin_lock(&hb->lock);
1537
1538 plist_for_each_entry_safe(this, next, &hb->chain, list) {
1539 if (match_futex (&this->key, &key)) {
1540 if (this->pi_state || this->rt_waiter) {
1541 ret = -EINVAL;
1542 break;
1543 }
1544
1545
1546 if (!(this->bitset & bitset))
1547 continue;
1548
1549 mark_wake_futex(&wake_q, this);
1550 if (++ret >= nr_wake)
1551 break;
1552 }
1553 }
1554
1555 spin_unlock(&hb->lock);
1556 wake_up_q(&wake_q);
1557out_put_key:
1558 put_futex_key(&key);
1559out:
1560 return ret;
1561}
1562
1563static int futex_atomic_op_inuser(unsigned int encoded_op, u32 __user *uaddr)
1564{
1565 unsigned int op = (encoded_op & 0x70000000) >> 28;
1566 unsigned int cmp = (encoded_op & 0x0f000000) >> 24;
1567 int oparg = sign_extend32((encoded_op & 0x00fff000) >> 12, 11);
1568 int cmparg = sign_extend32(encoded_op & 0x00000fff, 11);
1569 int oldval, ret;
1570
1571 if (encoded_op & (FUTEX_OP_OPARG_SHIFT << 28)) {
1572 if (oparg < 0 || oparg > 31) {
1573 char comm[sizeof(current->comm)];
1574
1575
1576
1577
1578 pr_info_ratelimited("futex_wake_op: %s tries to shift op by %d; fix this program\n",
1579 get_task_comm(comm, current), oparg);
1580 oparg &= 31;
1581 }
1582 oparg = 1 << oparg;
1583 }
1584
1585 if (!access_ok(VERIFY_WRITE, uaddr, sizeof(u32)))
1586 return -EFAULT;
1587
1588 ret = arch_futex_atomic_op_inuser(op, oparg, &oldval, uaddr);
1589 if (ret)
1590 return ret;
1591
1592 switch (cmp) {
1593 case FUTEX_OP_CMP_EQ:
1594 return oldval == cmparg;
1595 case FUTEX_OP_CMP_NE:
1596 return oldval != cmparg;
1597 case FUTEX_OP_CMP_LT:
1598 return oldval < cmparg;
1599 case FUTEX_OP_CMP_GE:
1600 return oldval >= cmparg;
1601 case FUTEX_OP_CMP_LE:
1602 return oldval <= cmparg;
1603 case FUTEX_OP_CMP_GT:
1604 return oldval > cmparg;
1605 default:
1606 return -ENOSYS;
1607 }
1608}
1609
1610
1611
1612
1613
1614static int
1615futex_wake_op(u32 __user *uaddr1, unsigned int flags, u32 __user *uaddr2,
1616 int nr_wake, int nr_wake2, int op)
1617{
1618 union futex_key key1 = FUTEX_KEY_INIT, key2 = FUTEX_KEY_INIT;
1619 struct futex_hash_bucket *hb1, *hb2;
1620 struct futex_q *this, *next;
1621 int ret, op_ret;
1622 DEFINE_WAKE_Q(wake_q);
1623
1624retry:
1625 ret = get_futex_key(uaddr1, flags & FLAGS_SHARED, &key1, VERIFY_READ);
1626 if (unlikely(ret != 0))
1627 goto out;
1628 ret = get_futex_key(uaddr2, flags & FLAGS_SHARED, &key2, VERIFY_WRITE);
1629 if (unlikely(ret != 0))
1630 goto out_put_key1;
1631
1632 hb1 = hash_futex(&key1);
1633 hb2 = hash_futex(&key2);
1634
1635retry_private:
1636 double_lock_hb(hb1, hb2);
1637 op_ret = futex_atomic_op_inuser(op, uaddr2);
1638 if (unlikely(op_ret < 0)) {
1639
1640 double_unlock_hb(hb1, hb2);
1641
1642#ifndef CONFIG_MMU
1643
1644
1645
1646
1647 ret = op_ret;
1648 goto out_put_keys;
1649#endif
1650
1651 if (unlikely(op_ret != -EFAULT)) {
1652 ret = op_ret;
1653 goto out_put_keys;
1654 }
1655
1656 ret = fault_in_user_writeable(uaddr2);
1657 if (ret)
1658 goto out_put_keys;
1659
1660 if (!(flags & FLAGS_SHARED))
1661 goto retry_private;
1662
1663 put_futex_key(&key2);
1664 put_futex_key(&key1);
1665 goto retry;
1666 }
1667
1668 plist_for_each_entry_safe(this, next, &hb1->chain, list) {
1669 if (match_futex (&this->key, &key1)) {
1670 if (this->pi_state || this->rt_waiter) {
1671 ret = -EINVAL;
1672 goto out_unlock;
1673 }
1674 mark_wake_futex(&wake_q, this);
1675 if (++ret >= nr_wake)
1676 break;
1677 }
1678 }
1679
1680 if (op_ret > 0) {
1681 op_ret = 0;
1682 plist_for_each_entry_safe(this, next, &hb2->chain, list) {
1683 if (match_futex (&this->key, &key2)) {
1684 if (this->pi_state || this->rt_waiter) {
1685 ret = -EINVAL;
1686 goto out_unlock;
1687 }
1688 mark_wake_futex(&wake_q, this);
1689 if (++op_ret >= nr_wake2)
1690 break;
1691 }
1692 }
1693 ret += op_ret;
1694 }
1695
1696out_unlock:
1697 double_unlock_hb(hb1, hb2);
1698 wake_up_q(&wake_q);
1699out_put_keys:
1700 put_futex_key(&key2);
1701out_put_key1:
1702 put_futex_key(&key1);
1703out:
1704 return ret;
1705}
1706
1707
1708
1709
1710
1711
1712
1713
1714static inline
1715void requeue_futex(struct futex_q *q, struct futex_hash_bucket *hb1,
1716 struct futex_hash_bucket *hb2, union futex_key *key2)
1717{
1718
1719
1720
1721
1722
1723 if (likely(&hb1->chain != &hb2->chain)) {
1724 plist_del(&q->list, &hb1->chain);
1725 hb_waiters_dec(hb1);
1726 hb_waiters_inc(hb2);
1727 plist_add(&q->list, &hb2->chain);
1728 q->lock_ptr = &hb2->lock;
1729 }
1730 get_futex_key_refs(key2);
1731 q->key = *key2;
1732}
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748static inline
1749void requeue_pi_wake_futex(struct futex_q *q, union futex_key *key,
1750 struct futex_hash_bucket *hb)
1751{
1752 get_futex_key_refs(key);
1753 q->key = *key;
1754
1755 __unqueue_futex(q);
1756
1757 WARN_ON(!q->rt_waiter);
1758 q->rt_waiter = NULL;
1759
1760 q->lock_ptr = &hb->lock;
1761
1762 wake_up_state(q->task, TASK_NORMAL);
1763}
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785static int futex_proxy_trylock_atomic(u32 __user *pifutex,
1786 struct futex_hash_bucket *hb1,
1787 struct futex_hash_bucket *hb2,
1788 union futex_key *key1, union futex_key *key2,
1789 struct futex_pi_state **ps, int set_waiters)
1790{
1791 struct futex_q *top_waiter = NULL;
1792 u32 curval;
1793 int ret, vpid;
1794
1795 if (get_futex_value_locked(&curval, pifutex))
1796 return -EFAULT;
1797
1798 if (unlikely(should_fail_futex(true)))
1799 return -EFAULT;
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809 top_waiter = futex_top_waiter(hb1, key1);
1810
1811
1812 if (!top_waiter)
1813 return 0;
1814
1815
1816 if (!match_futex(top_waiter->requeue_pi_key, key2))
1817 return -EINVAL;
1818
1819
1820
1821
1822
1823
1824 vpid = task_pid_vnr(top_waiter->task);
1825 ret = futex_lock_pi_atomic(pifutex, hb2, key2, ps, top_waiter->task,
1826 set_waiters);
1827 if (ret == 1) {
1828 requeue_pi_wake_futex(top_waiter, key2, hb2);
1829 return vpid;
1830 }
1831 return ret;
1832}
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852static int futex_requeue(u32 __user *uaddr1, unsigned int flags,
1853 u32 __user *uaddr2, int nr_wake, int nr_requeue,
1854 u32 *cmpval, int requeue_pi)
1855{
1856 union futex_key key1 = FUTEX_KEY_INIT, key2 = FUTEX_KEY_INIT;
1857 int drop_count = 0, task_count = 0, ret;
1858 struct futex_pi_state *pi_state = NULL;
1859 struct futex_hash_bucket *hb1, *hb2;
1860 struct futex_q *this, *next;
1861 DEFINE_WAKE_Q(wake_q);
1862
1863 if (nr_wake < 0 || nr_requeue < 0)
1864 return -EINVAL;
1865
1866
1867
1868
1869
1870
1871
1872 if (!IS_ENABLED(CONFIG_FUTEX_PI) && requeue_pi)
1873 return -ENOSYS;
1874
1875 if (requeue_pi) {
1876
1877
1878
1879
1880 if (uaddr1 == uaddr2)
1881 return -EINVAL;
1882
1883
1884
1885
1886
1887 if (refill_pi_state_cache())
1888 return -ENOMEM;
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899 if (nr_wake != 1)
1900 return -EINVAL;
1901 }
1902
1903retry:
1904 ret = get_futex_key(uaddr1, flags & FLAGS_SHARED, &key1, VERIFY_READ);
1905 if (unlikely(ret != 0))
1906 goto out;
1907 ret = get_futex_key(uaddr2, flags & FLAGS_SHARED, &key2,
1908 requeue_pi ? VERIFY_WRITE : VERIFY_READ);
1909 if (unlikely(ret != 0))
1910 goto out_put_key1;
1911
1912
1913
1914
1915
1916 if (requeue_pi && match_futex(&key1, &key2)) {
1917 ret = -EINVAL;
1918 goto out_put_keys;
1919 }
1920
1921 hb1 = hash_futex(&key1);
1922 hb2 = hash_futex(&key2);
1923
1924retry_private:
1925 hb_waiters_inc(hb2);
1926 double_lock_hb(hb1, hb2);
1927
1928 if (likely(cmpval != NULL)) {
1929 u32 curval;
1930
1931 ret = get_futex_value_locked(&curval, uaddr1);
1932
1933 if (unlikely(ret)) {
1934 double_unlock_hb(hb1, hb2);
1935 hb_waiters_dec(hb2);
1936
1937 ret = get_user(curval, uaddr1);
1938 if (ret)
1939 goto out_put_keys;
1940
1941 if (!(flags & FLAGS_SHARED))
1942 goto retry_private;
1943
1944 put_futex_key(&key2);
1945 put_futex_key(&key1);
1946 goto retry;
1947 }
1948 if (curval != *cmpval) {
1949 ret = -EAGAIN;
1950 goto out_unlock;
1951 }
1952 }
1953
1954 if (requeue_pi && (task_count - nr_wake < nr_requeue)) {
1955
1956
1957
1958
1959
1960
1961 ret = futex_proxy_trylock_atomic(uaddr2, hb1, hb2, &key1,
1962 &key2, &pi_state, nr_requeue);
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973 if (ret > 0) {
1974 WARN_ON(pi_state);
1975 drop_count++;
1976 task_count++;
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989 ret = lookup_pi_state(uaddr2, ret, hb2, &key2, &pi_state);
1990 }
1991
1992 switch (ret) {
1993 case 0:
1994
1995 break;
1996
1997
1998 case -EFAULT:
1999 double_unlock_hb(hb1, hb2);
2000 hb_waiters_dec(hb2);
2001 put_futex_key(&key2);
2002 put_futex_key(&key1);
2003 ret = fault_in_user_writeable(uaddr2);
2004 if (!ret)
2005 goto retry;
2006 goto out;
2007 case -EAGAIN:
2008
2009
2010
2011
2012
2013
2014 double_unlock_hb(hb1, hb2);
2015 hb_waiters_dec(hb2);
2016 put_futex_key(&key2);
2017 put_futex_key(&key1);
2018 cond_resched();
2019 goto retry;
2020 default:
2021 goto out_unlock;
2022 }
2023 }
2024
2025 plist_for_each_entry_safe(this, next, &hb1->chain, list) {
2026 if (task_count - nr_wake >= nr_requeue)
2027 break;
2028
2029 if (!match_futex(&this->key, &key1))
2030 continue;
2031
2032
2033
2034
2035
2036
2037
2038
2039 if ((requeue_pi && !this->rt_waiter) ||
2040 (!requeue_pi && this->rt_waiter) ||
2041 this->pi_state) {
2042 ret = -EINVAL;
2043 break;
2044 }
2045
2046
2047
2048
2049
2050
2051 if (++task_count <= nr_wake && !requeue_pi) {
2052 mark_wake_futex(&wake_q, this);
2053 continue;
2054 }
2055
2056
2057 if (requeue_pi && !match_futex(this->requeue_pi_key, &key2)) {
2058 ret = -EINVAL;
2059 break;
2060 }
2061
2062
2063
2064
2065
2066 if (requeue_pi) {
2067
2068
2069
2070
2071
2072 get_pi_state(pi_state);
2073 this->pi_state = pi_state;
2074 ret = rt_mutex_start_proxy_lock(&pi_state->pi_mutex,
2075 this->rt_waiter,
2076 this->task);
2077 if (ret == 1) {
2078
2079
2080
2081
2082
2083
2084
2085
2086 requeue_pi_wake_futex(this, &key2, hb2);
2087 drop_count++;
2088 continue;
2089 } else if (ret) {
2090
2091
2092
2093
2094
2095
2096
2097
2098 this->pi_state = NULL;
2099 put_pi_state(pi_state);
2100
2101
2102
2103
2104 break;
2105 }
2106 }
2107 requeue_futex(this, hb1, hb2, &key2);
2108 drop_count++;
2109 }
2110
2111
2112
2113
2114
2115
2116 put_pi_state(pi_state);
2117
2118out_unlock:
2119 double_unlock_hb(hb1, hb2);
2120 wake_up_q(&wake_q);
2121 hb_waiters_dec(hb2);
2122
2123
2124
2125
2126
2127
2128
2129 while (--drop_count >= 0)
2130 drop_futex_key_refs(&key1);
2131
2132out_put_keys:
2133 put_futex_key(&key2);
2134out_put_key1:
2135 put_futex_key(&key1);
2136out:
2137 return ret ? ret : task_count;
2138}
2139
2140
2141static inline struct futex_hash_bucket *queue_lock(struct futex_q *q)
2142 __acquires(&hb->lock)
2143{
2144 struct futex_hash_bucket *hb;
2145
2146 hb = hash_futex(&q->key);
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156 hb_waiters_inc(hb);
2157
2158 q->lock_ptr = &hb->lock;
2159
2160 spin_lock(&hb->lock);
2161 return hb;
2162}
2163
2164static inline void
2165queue_unlock(struct futex_hash_bucket *hb)
2166 __releases(&hb->lock)
2167{
2168 spin_unlock(&hb->lock);
2169 hb_waiters_dec(hb);
2170}
2171
2172static inline void __queue_me(struct futex_q *q, struct futex_hash_bucket *hb)
2173{
2174 int prio;
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184 prio = min(current->normal_prio, MAX_RT_PRIO);
2185
2186 plist_node_init(&q->list, prio);
2187 plist_add(&q->list, &hb->chain);
2188 q->task = current;
2189}
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203static inline void queue_me(struct futex_q *q, struct futex_hash_bucket *hb)
2204 __releases(&hb->lock)
2205{
2206 __queue_me(q, hb);
2207 spin_unlock(&hb->lock);
2208}
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221static int unqueue_me(struct futex_q *q)
2222{
2223 spinlock_t *lock_ptr;
2224 int ret = 0;
2225
2226
2227retry:
2228
2229
2230
2231
2232
2233 lock_ptr = READ_ONCE(q->lock_ptr);
2234 if (lock_ptr != NULL) {
2235 spin_lock(lock_ptr);
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249 if (unlikely(lock_ptr != q->lock_ptr)) {
2250 spin_unlock(lock_ptr);
2251 goto retry;
2252 }
2253 __unqueue_futex(q);
2254
2255 BUG_ON(q->pi_state);
2256
2257 spin_unlock(lock_ptr);
2258 ret = 1;
2259 }
2260
2261 drop_futex_key_refs(&q->key);
2262 return ret;
2263}
2264
2265
2266
2267
2268
2269
2270static void unqueue_me_pi(struct futex_q *q)
2271 __releases(q->lock_ptr)
2272{
2273 __unqueue_futex(q);
2274
2275 BUG_ON(!q->pi_state);
2276 put_pi_state(q->pi_state);
2277 q->pi_state = NULL;
2278
2279 spin_unlock(q->lock_ptr);
2280}
2281
2282static int fixup_pi_state_owner(u32 __user *uaddr, struct futex_q *q,
2283 struct task_struct *argowner)
2284{
2285 struct futex_pi_state *pi_state = q->pi_state;
2286 u32 uval, uninitialized_var(curval), newval;
2287 struct task_struct *oldowner, *newowner;
2288 u32 newtid;
2289 int ret;
2290
2291 lockdep_assert_held(q->lock_ptr);
2292
2293 raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);
2294
2295 oldowner = pi_state->owner;
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320retry:
2321 if (!argowner) {
2322 if (oldowner != current) {
2323
2324
2325
2326
2327 ret = 0;
2328 goto out_unlock;
2329 }
2330
2331 if (__rt_mutex_futex_trylock(&pi_state->pi_mutex)) {
2332
2333 ret = 0;
2334 goto out_unlock;
2335 }
2336
2337
2338
2339
2340 newowner = rt_mutex_owner(&pi_state->pi_mutex);
2341 BUG_ON(!newowner);
2342 } else {
2343 WARN_ON_ONCE(argowner != current);
2344 if (oldowner == current) {
2345
2346
2347
2348
2349 ret = 0;
2350 goto out_unlock;
2351 }
2352 newowner = argowner;
2353 }
2354
2355 newtid = task_pid_vnr(newowner) | FUTEX_WAITERS;
2356
2357 if (!pi_state->owner)
2358 newtid |= FUTEX_OWNER_DIED;
2359
2360 if (get_futex_value_locked(&uval, uaddr))
2361 goto handle_fault;
2362
2363 for (;;) {
2364 newval = (uval & FUTEX_OWNER_DIED) | newtid;
2365
2366 if (cmpxchg_futex_value_locked(&curval, uaddr, uval, newval))
2367 goto handle_fault;
2368 if (curval == uval)
2369 break;
2370 uval = curval;
2371 }
2372
2373
2374
2375
2376
2377 if (pi_state->owner != NULL) {
2378 raw_spin_lock(&pi_state->owner->pi_lock);
2379 WARN_ON(list_empty(&pi_state->list));
2380 list_del_init(&pi_state->list);
2381 raw_spin_unlock(&pi_state->owner->pi_lock);
2382 }
2383
2384 pi_state->owner = newowner;
2385
2386 raw_spin_lock(&newowner->pi_lock);
2387 WARN_ON(!list_empty(&pi_state->list));
2388 list_add(&pi_state->list, &newowner->pi_state_list);
2389 raw_spin_unlock(&newowner->pi_lock);
2390 raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);
2391
2392 return 0;
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407handle_fault:
2408 raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);
2409 spin_unlock(q->lock_ptr);
2410
2411 ret = fault_in_user_writeable(uaddr);
2412
2413 spin_lock(q->lock_ptr);
2414 raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);
2415
2416
2417
2418
2419 if (pi_state->owner != oldowner) {
2420 ret = 0;
2421 goto out_unlock;
2422 }
2423
2424 if (ret)
2425 goto out_unlock;
2426
2427 goto retry;
2428
2429out_unlock:
2430 raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);
2431 return ret;
2432}
2433
2434static long futex_wait_restart(struct restart_block *restart);
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451static int fixup_owner(u32 __user *uaddr, struct futex_q *q, int locked)
2452{
2453 int ret = 0;
2454
2455 if (locked) {
2456
2457
2458
2459
2460
2461
2462
2463
2464 if (q->pi_state->owner != current)
2465 ret = fixup_pi_state_owner(uaddr, q, current);
2466 goto out;
2467 }
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477 if (q->pi_state->owner == current) {
2478 ret = fixup_pi_state_owner(uaddr, q, NULL);
2479 goto out;
2480 }
2481
2482
2483
2484
2485
2486 if (rt_mutex_owner(&q->pi_state->pi_mutex) == current) {
2487 printk(KERN_ERR "fixup_owner: ret = %d pi-mutex: %p "
2488 "pi-state %p\n", ret,
2489 q->pi_state->pi_mutex.owner,
2490 q->pi_state->owner);
2491 }
2492
2493out:
2494 return ret ? ret : locked;
2495}
2496
2497
2498
2499
2500
2501
2502
2503static void futex_wait_queue_me(struct futex_hash_bucket *hb, struct futex_q *q,
2504 struct hrtimer_sleeper *timeout)
2505{
2506
2507
2508
2509
2510
2511
2512 set_current_state(TASK_INTERRUPTIBLE);
2513 queue_me(q, hb);
2514
2515
2516 if (timeout)
2517 hrtimer_start_expires(&timeout->timer, HRTIMER_MODE_ABS);
2518
2519
2520
2521
2522
2523 if (likely(!plist_node_empty(&q->list))) {
2524
2525
2526
2527
2528
2529 if (!timeout || timeout->task)
2530 freezable_schedule();
2531 }
2532 __set_current_state(TASK_RUNNING);
2533}
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552static int futex_wait_setup(u32 __user *uaddr, u32 val, unsigned int flags,
2553 struct futex_q *q, struct futex_hash_bucket **hb)
2554{
2555 u32 uval;
2556 int ret;
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576retry:
2577 ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &q->key, VERIFY_READ);
2578 if (unlikely(ret != 0))
2579 return ret;
2580
2581retry_private:
2582 *hb = queue_lock(q);
2583
2584 ret = get_futex_value_locked(&uval, uaddr);
2585
2586 if (ret) {
2587 queue_unlock(*hb);
2588
2589 ret = get_user(uval, uaddr);
2590 if (ret)
2591 goto out;
2592
2593 if (!(flags & FLAGS_SHARED))
2594 goto retry_private;
2595
2596 put_futex_key(&q->key);
2597 goto retry;
2598 }
2599
2600 if (uval != val) {
2601 queue_unlock(*hb);
2602 ret = -EWOULDBLOCK;
2603 }
2604
2605out:
2606 if (ret)
2607 put_futex_key(&q->key);
2608 return ret;
2609}
2610
2611static int futex_wait(u32 __user *uaddr, unsigned int flags, u32 val,
2612 ktime_t *abs_time, u32 bitset)
2613{
2614 struct hrtimer_sleeper timeout, *to = NULL;
2615 struct restart_block *restart;
2616 struct futex_hash_bucket *hb;
2617 struct futex_q q = futex_q_init;
2618 int ret;
2619
2620 if (!bitset)
2621 return -EINVAL;
2622 q.bitset = bitset;
2623
2624 if (abs_time) {
2625 to = &timeout;
2626
2627 hrtimer_init_on_stack(&to->timer, (flags & FLAGS_CLOCKRT) ?
2628 CLOCK_REALTIME : CLOCK_MONOTONIC,
2629 HRTIMER_MODE_ABS);
2630 hrtimer_init_sleeper(to, current);
2631 hrtimer_set_expires_range_ns(&to->timer, *abs_time,
2632 current->timer_slack_ns);
2633 }
2634
2635retry:
2636
2637
2638
2639
2640 ret = futex_wait_setup(uaddr, val, flags, &q, &hb);
2641 if (ret)
2642 goto out;
2643
2644
2645 futex_wait_queue_me(hb, &q, to);
2646
2647
2648 ret = 0;
2649
2650 if (!unqueue_me(&q))
2651 goto out;
2652 ret = -ETIMEDOUT;
2653 if (to && !to->task)
2654 goto out;
2655
2656
2657
2658
2659
2660 if (!signal_pending(current))
2661 goto retry;
2662
2663 ret = -ERESTARTSYS;
2664 if (!abs_time)
2665 goto out;
2666
2667 restart = ¤t->restart_block;
2668 restart->fn = futex_wait_restart;
2669 restart->futex.uaddr = uaddr;
2670 restart->futex.val = val;
2671 restart->futex.time = *abs_time;
2672 restart->futex.bitset = bitset;
2673 restart->futex.flags = flags | FLAGS_HAS_TIMEOUT;
2674
2675 ret = -ERESTART_RESTARTBLOCK;
2676
2677out:
2678 if (to) {
2679 hrtimer_cancel(&to->timer);
2680 destroy_hrtimer_on_stack(&to->timer);
2681 }
2682 return ret;
2683}
2684
2685
2686static long futex_wait_restart(struct restart_block *restart)
2687{
2688 u32 __user *uaddr = restart->futex.uaddr;
2689 ktime_t t, *tp = NULL;
2690
2691 if (restart->futex.flags & FLAGS_HAS_TIMEOUT) {
2692 t = restart->futex.time;
2693 tp = &t;
2694 }
2695 restart->fn = do_no_restart_syscall;
2696
2697 return (long)futex_wait(uaddr, restart->futex.flags,
2698 restart->futex.val, tp, restart->futex.bitset);
2699}
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711static int futex_lock_pi(u32 __user *uaddr, unsigned int flags,
2712 ktime_t *time, int trylock)
2713{
2714 struct hrtimer_sleeper timeout, *to = NULL;
2715 struct futex_pi_state *pi_state = NULL;
2716 struct rt_mutex_waiter rt_waiter;
2717 struct futex_hash_bucket *hb;
2718 struct futex_q q = futex_q_init;
2719 int res, ret;
2720
2721 if (!IS_ENABLED(CONFIG_FUTEX_PI))
2722 return -ENOSYS;
2723
2724 if (refill_pi_state_cache())
2725 return -ENOMEM;
2726
2727 if (time) {
2728 to = &timeout;
2729 hrtimer_init_on_stack(&to->timer, CLOCK_REALTIME,
2730 HRTIMER_MODE_ABS);
2731 hrtimer_init_sleeper(to, current);
2732 hrtimer_set_expires(&to->timer, *time);
2733 }
2734
2735retry:
2736 ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &q.key, VERIFY_WRITE);
2737 if (unlikely(ret != 0))
2738 goto out;
2739
2740retry_private:
2741 hb = queue_lock(&q);
2742
2743 ret = futex_lock_pi_atomic(uaddr, hb, &q.key, &q.pi_state, current, 0);
2744 if (unlikely(ret)) {
2745
2746
2747
2748
2749 switch (ret) {
2750 case 1:
2751
2752 ret = 0;
2753 goto out_unlock_put_key;
2754 case -EFAULT:
2755 goto uaddr_faulted;
2756 case -EAGAIN:
2757
2758
2759
2760
2761
2762
2763 queue_unlock(hb);
2764 put_futex_key(&q.key);
2765 cond_resched();
2766 goto retry;
2767 default:
2768 goto out_unlock_put_key;
2769 }
2770 }
2771
2772 WARN_ON(!q.pi_state);
2773
2774
2775
2776
2777 __queue_me(&q, hb);
2778
2779 if (trylock) {
2780 ret = rt_mutex_futex_trylock(&q.pi_state->pi_mutex);
2781
2782 ret = ret ? 0 : -EWOULDBLOCK;
2783 goto no_block;
2784 }
2785
2786 rt_mutex_init_waiter(&rt_waiter);
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800 raw_spin_lock_irq(&q.pi_state->pi_mutex.wait_lock);
2801 spin_unlock(q.lock_ptr);
2802 ret = __rt_mutex_start_proxy_lock(&q.pi_state->pi_mutex, &rt_waiter, current);
2803 raw_spin_unlock_irq(&q.pi_state->pi_mutex.wait_lock);
2804
2805 if (ret) {
2806 if (ret == 1)
2807 ret = 0;
2808
2809 spin_lock(q.lock_ptr);
2810 goto no_block;
2811 }
2812
2813
2814 if (unlikely(to))
2815 hrtimer_start_expires(&to->timer, HRTIMER_MODE_ABS);
2816
2817 ret = rt_mutex_wait_proxy_lock(&q.pi_state->pi_mutex, to, &rt_waiter);
2818
2819 spin_lock(q.lock_ptr);
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829 if (ret && !rt_mutex_cleanup_proxy_lock(&q.pi_state->pi_mutex, &rt_waiter))
2830 ret = 0;
2831
2832no_block:
2833
2834
2835
2836
2837 res = fixup_owner(uaddr, &q, !ret);
2838
2839
2840
2841
2842 if (res)
2843 ret = (res < 0) ? res : 0;
2844
2845
2846
2847
2848
2849 if (ret && (rt_mutex_owner(&q.pi_state->pi_mutex) == current)) {
2850 pi_state = q.pi_state;
2851 get_pi_state(pi_state);
2852 }
2853
2854
2855 unqueue_me_pi(&q);
2856
2857 if (pi_state) {
2858 rt_mutex_futex_unlock(&pi_state->pi_mutex);
2859 put_pi_state(pi_state);
2860 }
2861
2862 goto out_put_key;
2863
2864out_unlock_put_key:
2865 queue_unlock(hb);
2866
2867out_put_key:
2868 put_futex_key(&q.key);
2869out:
2870 if (to) {
2871 hrtimer_cancel(&to->timer);
2872 destroy_hrtimer_on_stack(&to->timer);
2873 }
2874 return ret != -EINTR ? ret : -ERESTARTNOINTR;
2875
2876uaddr_faulted:
2877 queue_unlock(hb);
2878
2879 ret = fault_in_user_writeable(uaddr);
2880 if (ret)
2881 goto out_put_key;
2882
2883 if (!(flags & FLAGS_SHARED))
2884 goto retry_private;
2885
2886 put_futex_key(&q.key);
2887 goto retry;
2888}
2889
2890
2891
2892
2893
2894
2895static int futex_unlock_pi(u32 __user *uaddr, unsigned int flags)
2896{
2897 u32 uninitialized_var(curval), uval, vpid = task_pid_vnr(current);
2898 union futex_key key = FUTEX_KEY_INIT;
2899 struct futex_hash_bucket *hb;
2900 struct futex_q *top_waiter;
2901 int ret;
2902
2903 if (!IS_ENABLED(CONFIG_FUTEX_PI))
2904 return -ENOSYS;
2905
2906retry:
2907 if (get_user(uval, uaddr))
2908 return -EFAULT;
2909
2910
2911
2912 if ((uval & FUTEX_TID_MASK) != vpid)
2913 return -EPERM;
2914
2915 ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &key, VERIFY_WRITE);
2916 if (ret)
2917 return ret;
2918
2919 hb = hash_futex(&key);
2920 spin_lock(&hb->lock);
2921
2922
2923
2924
2925
2926
2927 top_waiter = futex_top_waiter(hb, &key);
2928 if (top_waiter) {
2929 struct futex_pi_state *pi_state = top_waiter->pi_state;
2930
2931 ret = -EINVAL;
2932 if (!pi_state)
2933 goto out_unlock;
2934
2935
2936
2937
2938
2939 if (pi_state->owner != current)
2940 goto out_unlock;
2941
2942 get_pi_state(pi_state);
2943
2944
2945
2946
2947
2948
2949 raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);
2950 spin_unlock(&hb->lock);
2951
2952
2953 ret = wake_futex_pi(uaddr, uval, pi_state);
2954
2955 put_pi_state(pi_state);
2956
2957
2958
2959
2960 if (!ret)
2961 goto out_putkey;
2962
2963
2964
2965
2966 if (ret == -EFAULT)
2967 goto pi_faulted;
2968
2969
2970
2971
2972 if (ret == -EAGAIN) {
2973 put_futex_key(&key);
2974 goto retry;
2975 }
2976
2977
2978
2979
2980 goto out_putkey;
2981 }
2982
2983
2984
2985
2986
2987
2988
2989
2990 if (cmpxchg_futex_value_locked(&curval, uaddr, uval, 0)) {
2991 spin_unlock(&hb->lock);
2992 goto pi_faulted;
2993 }
2994
2995
2996
2997
2998 ret = (curval == uval) ? 0 : -EAGAIN;
2999
3000out_unlock:
3001 spin_unlock(&hb->lock);
3002out_putkey:
3003 put_futex_key(&key);
3004 return ret;
3005
3006pi_faulted:
3007 put_futex_key(&key);
3008
3009 ret = fault_in_user_writeable(uaddr);
3010 if (!ret)
3011 goto retry;
3012
3013 return ret;
3014}
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032static inline
3033int handle_early_requeue_pi_wakeup(struct futex_hash_bucket *hb,
3034 struct futex_q *q, union futex_key *key2,
3035 struct hrtimer_sleeper *timeout)
3036{
3037 int ret = 0;
3038
3039
3040
3041
3042
3043
3044
3045
3046 if (!match_futex(&q->key, key2)) {
3047 WARN_ON(q->lock_ptr && (&hb->lock != q->lock_ptr));
3048
3049
3050
3051
3052 plist_del(&q->list, &hb->chain);
3053 hb_waiters_dec(hb);
3054
3055
3056 ret = -EWOULDBLOCK;
3057 if (timeout && !timeout->task)
3058 ret = -ETIMEDOUT;
3059 else if (signal_pending(current))
3060 ret = -ERESTARTNOINTR;
3061 }
3062 return ret;
3063}
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105static int futex_wait_requeue_pi(u32 __user *uaddr, unsigned int flags,
3106 u32 val, ktime_t *abs_time, u32 bitset,
3107 u32 __user *uaddr2)
3108{
3109 struct hrtimer_sleeper timeout, *to = NULL;
3110 struct futex_pi_state *pi_state = NULL;
3111 struct rt_mutex_waiter rt_waiter;
3112 struct futex_hash_bucket *hb;
3113 union futex_key key2 = FUTEX_KEY_INIT;
3114 struct futex_q q = futex_q_init;
3115 int res, ret;
3116
3117 if (!IS_ENABLED(CONFIG_FUTEX_PI))
3118 return -ENOSYS;
3119
3120 if (uaddr == uaddr2)
3121 return -EINVAL;
3122
3123 if (!bitset)
3124 return -EINVAL;
3125
3126 if (abs_time) {
3127 to = &timeout;
3128 hrtimer_init_on_stack(&to->timer, (flags & FLAGS_CLOCKRT) ?
3129 CLOCK_REALTIME : CLOCK_MONOTONIC,
3130 HRTIMER_MODE_ABS);
3131 hrtimer_init_sleeper(to, current);
3132 hrtimer_set_expires_range_ns(&to->timer, *abs_time,
3133 current->timer_slack_ns);
3134 }
3135
3136
3137
3138
3139
3140 rt_mutex_init_waiter(&rt_waiter);
3141
3142 ret = get_futex_key(uaddr2, flags & FLAGS_SHARED, &key2, VERIFY_WRITE);
3143 if (unlikely(ret != 0))
3144 goto out;
3145
3146 q.bitset = bitset;
3147 q.rt_waiter = &rt_waiter;
3148 q.requeue_pi_key = &key2;
3149
3150
3151
3152
3153
3154 ret = futex_wait_setup(uaddr, val, flags, &q, &hb);
3155 if (ret)
3156 goto out_key2;
3157
3158
3159
3160
3161
3162 if (match_futex(&q.key, &key2)) {
3163 queue_unlock(hb);
3164 ret = -EINVAL;
3165 goto out_put_keys;
3166 }
3167
3168
3169 futex_wait_queue_me(hb, &q, to);
3170
3171 spin_lock(&hb->lock);
3172 ret = handle_early_requeue_pi_wakeup(hb, &q, &key2, to);
3173 spin_unlock(&hb->lock);
3174 if (ret)
3175 goto out_put_keys;
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187 if (!q.rt_waiter) {
3188
3189
3190
3191
3192 if (q.pi_state && (q.pi_state->owner != current)) {
3193 spin_lock(q.lock_ptr);
3194 ret = fixup_pi_state_owner(uaddr2, &q, current);
3195 if (ret && rt_mutex_owner(&q.pi_state->pi_mutex) == current) {
3196 pi_state = q.pi_state;
3197 get_pi_state(pi_state);
3198 }
3199
3200
3201
3202
3203 put_pi_state(q.pi_state);
3204 spin_unlock(q.lock_ptr);
3205 }
3206 } else {
3207 struct rt_mutex *pi_mutex;
3208
3209
3210
3211
3212
3213
3214 WARN_ON(!q.pi_state);
3215 pi_mutex = &q.pi_state->pi_mutex;
3216 ret = rt_mutex_wait_proxy_lock(pi_mutex, to, &rt_waiter);
3217
3218 spin_lock(q.lock_ptr);
3219 if (ret && !rt_mutex_cleanup_proxy_lock(pi_mutex, &rt_waiter))
3220 ret = 0;
3221
3222 debug_rt_mutex_free_waiter(&rt_waiter);
3223
3224
3225
3226
3227 res = fixup_owner(uaddr2, &q, !ret);
3228
3229
3230
3231
3232 if (res)
3233 ret = (res < 0) ? res : 0;
3234
3235
3236
3237
3238
3239
3240 if (ret && rt_mutex_owner(&q.pi_state->pi_mutex) == current) {
3241 pi_state = q.pi_state;
3242 get_pi_state(pi_state);
3243 }
3244
3245
3246 unqueue_me_pi(&q);
3247 }
3248
3249 if (pi_state) {
3250 rt_mutex_futex_unlock(&pi_state->pi_mutex);
3251 put_pi_state(pi_state);
3252 }
3253
3254 if (ret == -EINTR) {
3255
3256
3257
3258
3259
3260
3261
3262 ret = -EWOULDBLOCK;
3263 }
3264
3265out_put_keys:
3266 put_futex_key(&q.key);
3267out_key2:
3268 put_futex_key(&key2);
3269
3270out:
3271 if (to) {
3272 hrtimer_cancel(&to->timer);
3273 destroy_hrtimer_on_stack(&to->timer);
3274 }
3275 return ret;
3276}
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298SYSCALL_DEFINE2(set_robust_list, struct robust_list_head __user *, head,
3299 size_t, len)
3300{
3301 if (!futex_cmpxchg_enabled)
3302 return -ENOSYS;
3303
3304
3305
3306 if (unlikely(len != sizeof(*head)))
3307 return -EINVAL;
3308
3309 current->robust_list = head;
3310
3311 return 0;
3312}
3313
3314
3315
3316
3317
3318
3319
3320SYSCALL_DEFINE3(get_robust_list, int, pid,
3321 struct robust_list_head __user * __user *, head_ptr,
3322 size_t __user *, len_ptr)
3323{
3324 struct robust_list_head __user *head;
3325 unsigned long ret;
3326 struct task_struct *p;
3327
3328 if (!futex_cmpxchg_enabled)
3329 return -ENOSYS;
3330
3331 rcu_read_lock();
3332
3333 ret = -ESRCH;
3334 if (!pid)
3335 p = current;
3336 else {
3337 p = find_task_by_vpid(pid);
3338 if (!p)
3339 goto err_unlock;
3340 }
3341
3342 ret = -EPERM;
3343 if (!ptrace_may_access(p, PTRACE_MODE_READ_REALCREDS))
3344 goto err_unlock;
3345
3346 head = p->robust_list;
3347 rcu_read_unlock();
3348
3349 if (put_user(sizeof(*head), len_ptr))
3350 return -EFAULT;
3351 return put_user(head, head_ptr);
3352
3353err_unlock:
3354 rcu_read_unlock();
3355
3356 return ret;
3357}
3358
3359
3360
3361
3362
3363int handle_futex_death(u32 __user *uaddr, struct task_struct *curr, int pi)
3364{
3365 u32 uval, uninitialized_var(nval), mval;
3366
3367retry:
3368 if (get_user(uval, uaddr))
3369 return -1;
3370
3371 if ((uval & FUTEX_TID_MASK) == task_pid_vnr(curr)) {
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381
3382 mval = (uval & FUTEX_WAITERS) | FUTEX_OWNER_DIED;
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392 if (cmpxchg_futex_value_locked(&nval, uaddr, uval, mval)) {
3393 if (fault_in_user_writeable(uaddr))
3394 return -1;
3395 goto retry;
3396 }
3397 if (nval != uval)
3398 goto retry;
3399
3400
3401
3402
3403
3404 if (!pi && (uval & FUTEX_WAITERS))
3405 futex_wake(uaddr, 1, 1, FUTEX_BITSET_MATCH_ANY);
3406 }
3407 return 0;
3408}
3409
3410
3411
3412
3413static inline int fetch_robust_entry(struct robust_list __user **entry,
3414 struct robust_list __user * __user *head,
3415 unsigned int *pi)
3416{
3417 unsigned long uentry;
3418
3419 if (get_user(uentry, (unsigned long __user *)head))
3420 return -EFAULT;
3421
3422 *entry = (void __user *)(uentry & ~1UL);
3423 *pi = uentry & 1;
3424
3425 return 0;
3426}
3427
3428
3429
3430
3431
3432
3433
3434void exit_robust_list(struct task_struct *curr)
3435{
3436 struct robust_list_head __user *head = curr->robust_list;
3437 struct robust_list __user *entry, *next_entry, *pending;
3438 unsigned int limit = ROBUST_LIST_LIMIT, pi, pip;
3439 unsigned int uninitialized_var(next_pi);
3440 unsigned long futex_offset;
3441 int rc;
3442
3443 if (!futex_cmpxchg_enabled)
3444 return;
3445
3446
3447
3448
3449
3450 if (fetch_robust_entry(&entry, &head->list.next, &pi))
3451 return;
3452
3453
3454
3455 if (get_user(futex_offset, &head->futex_offset))
3456 return;
3457
3458
3459
3460
3461 if (fetch_robust_entry(&pending, &head->list_op_pending, &pip))
3462 return;
3463
3464 next_entry = NULL;
3465 while (entry != &head->list) {
3466
3467
3468
3469
3470 rc = fetch_robust_entry(&next_entry, &entry->next, &next_pi);
3471
3472
3473
3474
3475 if (entry != pending)
3476 if (handle_futex_death((void __user *)entry + futex_offset,
3477 curr, pi))
3478 return;
3479 if (rc)
3480 return;
3481 entry = next_entry;
3482 pi = next_pi;
3483
3484
3485
3486 if (!--limit)
3487 break;
3488
3489 cond_resched();
3490 }
3491
3492 if (pending)
3493 handle_futex_death((void __user *)pending + futex_offset,
3494 curr, pip);
3495}
3496
3497long do_futex(u32 __user *uaddr, int op, u32 val, ktime_t *timeout,
3498 u32 __user *uaddr2, u32 val2, u32 val3)
3499{
3500 int cmd = op & FUTEX_CMD_MASK;
3501 unsigned int flags = 0;
3502
3503 if (!(op & FUTEX_PRIVATE_FLAG))
3504 flags |= FLAGS_SHARED;
3505
3506 if (op & FUTEX_CLOCK_REALTIME) {
3507 flags |= FLAGS_CLOCKRT;
3508 if (cmd != FUTEX_WAIT && cmd != FUTEX_WAIT_BITSET && \
3509 cmd != FUTEX_WAIT_REQUEUE_PI)
3510 return -ENOSYS;
3511 }
3512
3513 switch (cmd) {
3514 case FUTEX_LOCK_PI:
3515 case FUTEX_UNLOCK_PI:
3516 case FUTEX_TRYLOCK_PI:
3517 case FUTEX_WAIT_REQUEUE_PI:
3518 case FUTEX_CMP_REQUEUE_PI:
3519 if (!futex_cmpxchg_enabled)
3520 return -ENOSYS;
3521 }
3522
3523 switch (cmd) {
3524 case FUTEX_WAIT:
3525 val3 = FUTEX_BITSET_MATCH_ANY;
3526 case FUTEX_WAIT_BITSET:
3527 return futex_wait(uaddr, flags, val, timeout, val3);
3528 case FUTEX_WAKE:
3529 val3 = FUTEX_BITSET_MATCH_ANY;
3530 case FUTEX_WAKE_BITSET:
3531 return futex_wake(uaddr, flags, val, val3);
3532 case FUTEX_REQUEUE:
3533 return futex_requeue(uaddr, flags, uaddr2, val, val2, NULL, 0);
3534 case FUTEX_CMP_REQUEUE:
3535 return futex_requeue(uaddr, flags, uaddr2, val, val2, &val3, 0);
3536 case FUTEX_WAKE_OP:
3537 return futex_wake_op(uaddr, flags, uaddr2, val, val2, val3);
3538 case FUTEX_LOCK_PI:
3539 return futex_lock_pi(uaddr, flags, timeout, 0);
3540 case FUTEX_UNLOCK_PI:
3541 return futex_unlock_pi(uaddr, flags);
3542 case FUTEX_TRYLOCK_PI:
3543 return futex_lock_pi(uaddr, flags, NULL, 1);
3544 case FUTEX_WAIT_REQUEUE_PI:
3545 val3 = FUTEX_BITSET_MATCH_ANY;
3546 return futex_wait_requeue_pi(uaddr, flags, val, timeout, val3,
3547 uaddr2);
3548 case FUTEX_CMP_REQUEUE_PI:
3549 return futex_requeue(uaddr, flags, uaddr2, val, val2, &val3, 1);
3550 }
3551 return -ENOSYS;
3552}
3553
3554
3555SYSCALL_DEFINE6(futex, u32 __user *, uaddr, int, op, u32, val,
3556 struct timespec __user *, utime, u32 __user *, uaddr2,
3557 u32, val3)
3558{
3559 struct timespec ts;
3560 ktime_t t, *tp = NULL;
3561 u32 val2 = 0;
3562 int cmd = op & FUTEX_CMD_MASK;
3563
3564 if (utime && (cmd == FUTEX_WAIT || cmd == FUTEX_LOCK_PI ||
3565 cmd == FUTEX_WAIT_BITSET ||
3566 cmd == FUTEX_WAIT_REQUEUE_PI)) {
3567 if (unlikely(should_fail_futex(!(op & FUTEX_PRIVATE_FLAG))))
3568 return -EFAULT;
3569 if (copy_from_user(&ts, utime, sizeof(ts)) != 0)
3570 return -EFAULT;
3571 if (!timespec_valid(&ts))
3572 return -EINVAL;
3573
3574 t = timespec_to_ktime(ts);
3575 if (cmd == FUTEX_WAIT)
3576 t = ktime_add_safe(ktime_get(), t);
3577 tp = &t;
3578 }
3579
3580
3581
3582
3583 if (cmd == FUTEX_REQUEUE || cmd == FUTEX_CMP_REQUEUE ||
3584 cmd == FUTEX_CMP_REQUEUE_PI || cmd == FUTEX_WAKE_OP)
3585 val2 = (u32) (unsigned long) utime;
3586
3587 return do_futex(uaddr, op, val, tp, uaddr2, val2, val3);
3588}
3589
3590static void __init futex_detect_cmpxchg(void)
3591{
3592#ifndef CONFIG_HAVE_FUTEX_CMPXCHG
3593 u32 curval;
3594
3595
3596
3597
3598
3599
3600
3601
3602
3603
3604
3605 if (cmpxchg_futex_value_locked(&curval, NULL, 0, 0) == -EFAULT)
3606 futex_cmpxchg_enabled = 1;
3607#endif
3608}
3609
3610static int __init futex_init(void)
3611{
3612 unsigned int futex_shift;
3613 unsigned long i;
3614
3615#if CONFIG_BASE_SMALL
3616 futex_hashsize = 16;
3617#else
3618 futex_hashsize = roundup_pow_of_two(256 * num_possible_cpus());
3619#endif
3620
3621 futex_queues = alloc_large_system_hash("futex", sizeof(*futex_queues),
3622 futex_hashsize, 0,
3623 futex_hashsize < 256 ? HASH_SMALL : 0,
3624 &futex_shift, NULL,
3625 futex_hashsize, futex_hashsize);
3626 futex_hashsize = 1UL << futex_shift;
3627
3628 futex_detect_cmpxchg();
3629
3630 for (i = 0; i < futex_hashsize; i++) {
3631 atomic_set(&futex_queues[i].waiters, 0);
3632 plist_head_init(&futex_queues[i].chain);
3633 spin_lock_init(&futex_queues[i].lock);
3634 }
3635
3636 return 0;
3637}
3638core_initcall(futex_init);
3639