1
2
3
4
5
6
7
8
9
10
11
12
13#include <linux/spinlock.h>
14#include <linux/export.h>
15#include <linux/sched.h>
16#include <linux/sched/rt.h>
17#include <linux/timer.h>
18
19#include "rtmutex_common.h"
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48static void
49rt_mutex_set_owner(struct rt_mutex *lock, struct task_struct *owner)
50{
51 unsigned long val = (unsigned long)owner;
52
53 if (rt_mutex_has_waiters(lock))
54 val |= RT_MUTEX_HAS_WAITERS;
55
56 lock->owner = (struct task_struct *)val;
57}
58
59static inline void clear_rt_mutex_waiters(struct rt_mutex *lock)
60{
61 lock->owner = (struct task_struct *)
62 ((unsigned long)lock->owner & ~RT_MUTEX_HAS_WAITERS);
63}
64
65static void fixup_rt_mutex_waiters(struct rt_mutex *lock)
66{
67 if (!rt_mutex_has_waiters(lock))
68 clear_rt_mutex_waiters(lock);
69}
70
71
72
73
74
75#if defined(__HAVE_ARCH_CMPXCHG) && !defined(CONFIG_DEBUG_RT_MUTEXES)
76# define rt_mutex_cmpxchg(l,c,n) (cmpxchg(&l->owner, c, n) == c)
77static inline void mark_rt_mutex_waiters(struct rt_mutex *lock)
78{
79 unsigned long owner, *p = (unsigned long *) &lock->owner;
80
81 do {
82 owner = *p;
83 } while (cmpxchg(p, owner, owner | RT_MUTEX_HAS_WAITERS) != owner);
84}
85#else
86# define rt_mutex_cmpxchg(l,c,n) (0)
87static inline void mark_rt_mutex_waiters(struct rt_mutex *lock)
88{
89 lock->owner = (struct task_struct *)
90 ((unsigned long)lock->owner | RT_MUTEX_HAS_WAITERS);
91}
92#endif
93
94
95
96
97
98
99
100int rt_mutex_getprio(struct task_struct *task)
101{
102 if (likely(!task_has_pi_waiters(task)))
103 return task->normal_prio;
104
105 return min(task_top_pi_waiter(task)->pi_list_entry.prio,
106 task->normal_prio);
107}
108
109
110
111
112
113
114static void __rt_mutex_adjust_prio(struct task_struct *task)
115{
116 int prio = rt_mutex_getprio(task);
117
118 if (task->prio != prio)
119 rt_mutex_setprio(task, prio);
120}
121
122
123
124
125
126
127
128
129
130
131static void rt_mutex_adjust_prio(struct task_struct *task)
132{
133 unsigned long flags;
134
135 raw_spin_lock_irqsave(&task->pi_lock, flags);
136 __rt_mutex_adjust_prio(task);
137 raw_spin_unlock_irqrestore(&task->pi_lock, flags);
138}
139
140
141
142
143int max_lock_depth = 1024;
144
145
146
147
148
149
150static int rt_mutex_adjust_prio_chain(struct task_struct *task,
151 int deadlock_detect,
152 struct rt_mutex *orig_lock,
153 struct rt_mutex_waiter *orig_waiter,
154 struct task_struct *top_task)
155{
156 struct rt_mutex *lock;
157 struct rt_mutex_waiter *waiter, *top_waiter = orig_waiter;
158 int detect_deadlock, ret = 0, depth = 0;
159 unsigned long flags;
160
161 detect_deadlock = debug_rt_mutex_detect_deadlock(orig_waiter,
162 deadlock_detect);
163
164
165
166
167
168
169
170 again:
171 if (++depth > max_lock_depth) {
172 static int prev_max;
173
174
175
176
177
178 if (prev_max != max_lock_depth) {
179 prev_max = max_lock_depth;
180 printk(KERN_WARNING "Maximum lock depth %d reached "
181 "task: %s (%d)\n", max_lock_depth,
182 top_task->comm, task_pid_nr(top_task));
183 }
184 put_task_struct(task);
185
186 return deadlock_detect ? -EDEADLK : 0;
187 }
188 retry:
189
190
191
192 raw_spin_lock_irqsave(&task->pi_lock, flags);
193
194 waiter = task->pi_blocked_on;
195
196
197
198
199
200 if (!waiter)
201 goto out_unlock_pi;
202
203
204
205
206
207 if (orig_waiter && !rt_mutex_owner(orig_lock))
208 goto out_unlock_pi;
209
210
211
212
213
214
215 if (top_waiter && (!task_has_pi_waiters(task) ||
216 top_waiter != task_top_pi_waiter(task)))
217 goto out_unlock_pi;
218
219
220
221
222
223 if (!detect_deadlock && waiter->list_entry.prio == task->prio)
224 goto out_unlock_pi;
225
226 lock = waiter->lock;
227 if (!raw_spin_trylock(&lock->wait_lock)) {
228 raw_spin_unlock_irqrestore(&task->pi_lock, flags);
229 cpu_relax();
230 goto retry;
231 }
232
233
234 if (lock == orig_lock || rt_mutex_owner(lock) == top_task) {
235 debug_rt_mutex_deadlock(deadlock_detect, orig_waiter, lock);
236 raw_spin_unlock(&lock->wait_lock);
237 ret = deadlock_detect ? -EDEADLK : 0;
238 goto out_unlock_pi;
239 }
240
241 top_waiter = rt_mutex_top_waiter(lock);
242
243
244 plist_del(&waiter->list_entry, &lock->wait_list);
245 waiter->list_entry.prio = task->prio;
246 plist_add(&waiter->list_entry, &lock->wait_list);
247
248
249 raw_spin_unlock_irqrestore(&task->pi_lock, flags);
250 if (!rt_mutex_owner(lock)) {
251
252
253
254
255
256 if (top_waiter != rt_mutex_top_waiter(lock))
257 wake_up_process(rt_mutex_top_waiter(lock)->task);
258 raw_spin_unlock(&lock->wait_lock);
259 goto out_put_task;
260 }
261 put_task_struct(task);
262
263
264 task = rt_mutex_owner(lock);
265 get_task_struct(task);
266 raw_spin_lock_irqsave(&task->pi_lock, flags);
267
268 if (waiter == rt_mutex_top_waiter(lock)) {
269
270 plist_del(&top_waiter->pi_list_entry, &task->pi_waiters);
271 waiter->pi_list_entry.prio = waiter->list_entry.prio;
272 plist_add(&waiter->pi_list_entry, &task->pi_waiters);
273 __rt_mutex_adjust_prio(task);
274
275 } else if (top_waiter == waiter) {
276
277 plist_del(&waiter->pi_list_entry, &task->pi_waiters);
278 waiter = rt_mutex_top_waiter(lock);
279 waiter->pi_list_entry.prio = waiter->list_entry.prio;
280 plist_add(&waiter->pi_list_entry, &task->pi_waiters);
281 __rt_mutex_adjust_prio(task);
282 }
283
284 raw_spin_unlock_irqrestore(&task->pi_lock, flags);
285
286 top_waiter = rt_mutex_top_waiter(lock);
287 raw_spin_unlock(&lock->wait_lock);
288
289 if (!detect_deadlock && waiter != top_waiter)
290 goto out_put_task;
291
292 goto again;
293
294 out_unlock_pi:
295 raw_spin_unlock_irqrestore(&task->pi_lock, flags);
296 out_put_task:
297 put_task_struct(task);
298
299 return ret;
300}
301
302
303
304
305
306
307
308
309
310
311static int try_to_take_rt_mutex(struct rt_mutex *lock, struct task_struct *task,
312 struct rt_mutex_waiter *waiter)
313{
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333 mark_rt_mutex_waiters(lock);
334
335 if (rt_mutex_owner(lock))
336 return 0;
337
338
339
340
341
342
343
344 if (rt_mutex_has_waiters(lock)) {
345 if (task->prio >= rt_mutex_top_waiter(lock)->list_entry.prio) {
346 if (!waiter || waiter != rt_mutex_top_waiter(lock))
347 return 0;
348 }
349 }
350
351 if (waiter || rt_mutex_has_waiters(lock)) {
352 unsigned long flags;
353 struct rt_mutex_waiter *top;
354
355 raw_spin_lock_irqsave(&task->pi_lock, flags);
356
357
358 if (waiter) {
359 plist_del(&waiter->list_entry, &lock->wait_list);
360 task->pi_blocked_on = NULL;
361 }
362
363
364
365
366
367 if (rt_mutex_has_waiters(lock)) {
368 top = rt_mutex_top_waiter(lock);
369 top->pi_list_entry.prio = top->list_entry.prio;
370 plist_add(&top->pi_list_entry, &task->pi_waiters);
371 }
372 raw_spin_unlock_irqrestore(&task->pi_lock, flags);
373 }
374
375
376 debug_rt_mutex_lock(lock);
377
378 rt_mutex_set_owner(lock, task);
379
380 rt_mutex_deadlock_account_lock(lock, task);
381
382 return 1;
383}
384
385
386
387
388
389
390
391
392static int task_blocks_on_rt_mutex(struct rt_mutex *lock,
393 struct rt_mutex_waiter *waiter,
394 struct task_struct *task,
395 int detect_deadlock)
396{
397 struct task_struct *owner = rt_mutex_owner(lock);
398 struct rt_mutex_waiter *top_waiter = waiter;
399 unsigned long flags;
400 int chain_walk = 0, res;
401
402 raw_spin_lock_irqsave(&task->pi_lock, flags);
403 __rt_mutex_adjust_prio(task);
404 waiter->task = task;
405 waiter->lock = lock;
406 plist_node_init(&waiter->list_entry, task->prio);
407 plist_node_init(&waiter->pi_list_entry, task->prio);
408
409
410 if (rt_mutex_has_waiters(lock))
411 top_waiter = rt_mutex_top_waiter(lock);
412 plist_add(&waiter->list_entry, &lock->wait_list);
413
414 task->pi_blocked_on = waiter;
415
416 raw_spin_unlock_irqrestore(&task->pi_lock, flags);
417
418 if (!owner)
419 return 0;
420
421 if (waiter == rt_mutex_top_waiter(lock)) {
422 raw_spin_lock_irqsave(&owner->pi_lock, flags);
423 plist_del(&top_waiter->pi_list_entry, &owner->pi_waiters);
424 plist_add(&waiter->pi_list_entry, &owner->pi_waiters);
425
426 __rt_mutex_adjust_prio(owner);
427 if (owner->pi_blocked_on)
428 chain_walk = 1;
429 raw_spin_unlock_irqrestore(&owner->pi_lock, flags);
430 }
431 else if (debug_rt_mutex_detect_deadlock(waiter, detect_deadlock))
432 chain_walk = 1;
433
434 if (!chain_walk)
435 return 0;
436
437
438
439
440
441
442 get_task_struct(owner);
443
444 raw_spin_unlock(&lock->wait_lock);
445
446 res = rt_mutex_adjust_prio_chain(owner, detect_deadlock, lock, waiter,
447 task);
448
449 raw_spin_lock(&lock->wait_lock);
450
451 return res;
452}
453
454
455
456
457
458
459
460
461static void wakeup_next_waiter(struct rt_mutex *lock)
462{
463 struct rt_mutex_waiter *waiter;
464 unsigned long flags;
465
466 raw_spin_lock_irqsave(¤t->pi_lock, flags);
467
468 waiter = rt_mutex_top_waiter(lock);
469
470
471
472
473
474
475
476 plist_del(&waiter->pi_list_entry, ¤t->pi_waiters);
477
478 rt_mutex_set_owner(lock, NULL);
479
480 raw_spin_unlock_irqrestore(¤t->pi_lock, flags);
481
482 wake_up_process(waiter->task);
483}
484
485
486
487
488
489
490
491static void remove_waiter(struct rt_mutex *lock,
492 struct rt_mutex_waiter *waiter)
493{
494 int first = (waiter == rt_mutex_top_waiter(lock));
495 struct task_struct *owner = rt_mutex_owner(lock);
496 unsigned long flags;
497 int chain_walk = 0;
498
499 raw_spin_lock_irqsave(¤t->pi_lock, flags);
500 plist_del(&waiter->list_entry, &lock->wait_list);
501 current->pi_blocked_on = NULL;
502 raw_spin_unlock_irqrestore(¤t->pi_lock, flags);
503
504 if (!owner)
505 return;
506
507 if (first) {
508
509 raw_spin_lock_irqsave(&owner->pi_lock, flags);
510
511 plist_del(&waiter->pi_list_entry, &owner->pi_waiters);
512
513 if (rt_mutex_has_waiters(lock)) {
514 struct rt_mutex_waiter *next;
515
516 next = rt_mutex_top_waiter(lock);
517 plist_add(&next->pi_list_entry, &owner->pi_waiters);
518 }
519 __rt_mutex_adjust_prio(owner);
520
521 if (owner->pi_blocked_on)
522 chain_walk = 1;
523
524 raw_spin_unlock_irqrestore(&owner->pi_lock, flags);
525 }
526
527 WARN_ON(!plist_node_empty(&waiter->pi_list_entry));
528
529 if (!chain_walk)
530 return;
531
532
533 get_task_struct(owner);
534
535 raw_spin_unlock(&lock->wait_lock);
536
537 rt_mutex_adjust_prio_chain(owner, 0, lock, NULL, current);
538
539 raw_spin_lock(&lock->wait_lock);
540}
541
542
543
544
545
546
547void rt_mutex_adjust_pi(struct task_struct *task)
548{
549 struct rt_mutex_waiter *waiter;
550 unsigned long flags;
551
552 raw_spin_lock_irqsave(&task->pi_lock, flags);
553
554 waiter = task->pi_blocked_on;
555 if (!waiter || waiter->list_entry.prio == task->prio) {
556 raw_spin_unlock_irqrestore(&task->pi_lock, flags);
557 return;
558 }
559
560 raw_spin_unlock_irqrestore(&task->pi_lock, flags);
561
562
563 get_task_struct(task);
564 rt_mutex_adjust_prio_chain(task, 0, NULL, NULL, task);
565}
566
567
568
569
570
571
572
573
574
575
576
577static int __sched
578__rt_mutex_slowlock(struct rt_mutex *lock, int state,
579 struct hrtimer_sleeper *timeout,
580 struct rt_mutex_waiter *waiter)
581{
582 int ret = 0;
583
584 for (;;) {
585
586 if (try_to_take_rt_mutex(lock, current, waiter))
587 break;
588
589
590
591
592
593 if (unlikely(state == TASK_INTERRUPTIBLE)) {
594
595 if (signal_pending(current))
596 ret = -EINTR;
597 if (timeout && !timeout->task)
598 ret = -ETIMEDOUT;
599 if (ret)
600 break;
601 }
602
603 raw_spin_unlock(&lock->wait_lock);
604
605 debug_rt_mutex_print_deadlock(waiter);
606
607 schedule_rt_mutex(lock);
608
609 raw_spin_lock(&lock->wait_lock);
610 set_current_state(state);
611 }
612
613 return ret;
614}
615
616
617
618
619static int __sched
620rt_mutex_slowlock(struct rt_mutex *lock, int state,
621 struct hrtimer_sleeper *timeout,
622 int detect_deadlock)
623{
624 struct rt_mutex_waiter waiter;
625 int ret = 0;
626
627 debug_rt_mutex_init_waiter(&waiter);
628
629 raw_spin_lock(&lock->wait_lock);
630
631
632 if (try_to_take_rt_mutex(lock, current, NULL)) {
633 raw_spin_unlock(&lock->wait_lock);
634 return 0;
635 }
636
637 set_current_state(state);
638
639
640 if (unlikely(timeout)) {
641 hrtimer_start_expires(&timeout->timer, HRTIMER_MODE_ABS);
642 if (!hrtimer_active(&timeout->timer))
643 timeout->task = NULL;
644 }
645
646 ret = task_blocks_on_rt_mutex(lock, &waiter, current, detect_deadlock);
647
648 if (likely(!ret))
649 ret = __rt_mutex_slowlock(lock, state, timeout, &waiter);
650
651 set_current_state(TASK_RUNNING);
652
653 if (unlikely(ret))
654 remove_waiter(lock, &waiter);
655
656
657
658
659
660 fixup_rt_mutex_waiters(lock);
661
662 raw_spin_unlock(&lock->wait_lock);
663
664
665 if (unlikely(timeout))
666 hrtimer_cancel(&timeout->timer);
667
668 debug_rt_mutex_free_waiter(&waiter);
669
670 return ret;
671}
672
673
674
675
676static inline int
677rt_mutex_slowtrylock(struct rt_mutex *lock)
678{
679 int ret = 0;
680
681 raw_spin_lock(&lock->wait_lock);
682
683 if (likely(rt_mutex_owner(lock) != current)) {
684
685 ret = try_to_take_rt_mutex(lock, current, NULL);
686
687
688
689
690 fixup_rt_mutex_waiters(lock);
691 }
692
693 raw_spin_unlock(&lock->wait_lock);
694
695 return ret;
696}
697
698
699
700
701static void __sched
702rt_mutex_slowunlock(struct rt_mutex *lock)
703{
704 raw_spin_lock(&lock->wait_lock);
705
706 debug_rt_mutex_unlock(lock);
707
708 rt_mutex_deadlock_account_unlock(current);
709
710 if (!rt_mutex_has_waiters(lock)) {
711 lock->owner = NULL;
712 raw_spin_unlock(&lock->wait_lock);
713 return;
714 }
715
716 wakeup_next_waiter(lock);
717
718 raw_spin_unlock(&lock->wait_lock);
719
720
721 rt_mutex_adjust_prio(current);
722}
723
724
725
726
727
728
729
730static inline int
731rt_mutex_fastlock(struct rt_mutex *lock, int state,
732 int detect_deadlock,
733 int (*slowfn)(struct rt_mutex *lock, int state,
734 struct hrtimer_sleeper *timeout,
735 int detect_deadlock))
736{
737 if (!detect_deadlock && likely(rt_mutex_cmpxchg(lock, NULL, current))) {
738 rt_mutex_deadlock_account_lock(lock, current);
739 return 0;
740 } else
741 return slowfn(lock, state, NULL, detect_deadlock);
742}
743
744static inline int
745rt_mutex_timed_fastlock(struct rt_mutex *lock, int state,
746 struct hrtimer_sleeper *timeout, int detect_deadlock,
747 int (*slowfn)(struct rt_mutex *lock, int state,
748 struct hrtimer_sleeper *timeout,
749 int detect_deadlock))
750{
751 if (!detect_deadlock && likely(rt_mutex_cmpxchg(lock, NULL, current))) {
752 rt_mutex_deadlock_account_lock(lock, current);
753 return 0;
754 } else
755 return slowfn(lock, state, timeout, detect_deadlock);
756}
757
758static inline int
759rt_mutex_fasttrylock(struct rt_mutex *lock,
760 int (*slowfn)(struct rt_mutex *lock))
761{
762 if (likely(rt_mutex_cmpxchg(lock, NULL, current))) {
763 rt_mutex_deadlock_account_lock(lock, current);
764 return 1;
765 }
766 return slowfn(lock);
767}
768
769static inline void
770rt_mutex_fastunlock(struct rt_mutex *lock,
771 void (*slowfn)(struct rt_mutex *lock))
772{
773 if (likely(rt_mutex_cmpxchg(lock, current, NULL)))
774 rt_mutex_deadlock_account_unlock(current);
775 else
776 slowfn(lock);
777}
778
779
780
781
782
783
784void __sched rt_mutex_lock(struct rt_mutex *lock)
785{
786 might_sleep();
787
788 rt_mutex_fastlock(lock, TASK_UNINTERRUPTIBLE, 0, rt_mutex_slowlock);
789}
790EXPORT_SYMBOL_GPL(rt_mutex_lock);
791
792
793
794
795
796
797
798
799
800
801
802
803int __sched rt_mutex_lock_interruptible(struct rt_mutex *lock,
804 int detect_deadlock)
805{
806 might_sleep();
807
808 return rt_mutex_fastlock(lock, TASK_INTERRUPTIBLE,
809 detect_deadlock, rt_mutex_slowlock);
810}
811EXPORT_SYMBOL_GPL(rt_mutex_lock_interruptible);
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828int
829rt_mutex_timed_lock(struct rt_mutex *lock, struct hrtimer_sleeper *timeout,
830 int detect_deadlock)
831{
832 might_sleep();
833
834 return rt_mutex_timed_fastlock(lock, TASK_INTERRUPTIBLE, timeout,
835 detect_deadlock, rt_mutex_slowlock);
836}
837EXPORT_SYMBOL_GPL(rt_mutex_timed_lock);
838
839
840
841
842
843
844
845
846int __sched rt_mutex_trylock(struct rt_mutex *lock)
847{
848 return rt_mutex_fasttrylock(lock, rt_mutex_slowtrylock);
849}
850EXPORT_SYMBOL_GPL(rt_mutex_trylock);
851
852
853
854
855
856
857void __sched rt_mutex_unlock(struct rt_mutex *lock)
858{
859 rt_mutex_fastunlock(lock, rt_mutex_slowunlock);
860}
861EXPORT_SYMBOL_GPL(rt_mutex_unlock);
862
863
864
865
866
867
868
869
870
871void rt_mutex_destroy(struct rt_mutex *lock)
872{
873 WARN_ON(rt_mutex_is_locked(lock));
874#ifdef CONFIG_DEBUG_RT_MUTEXES
875 lock->magic = NULL;
876#endif
877}
878
879EXPORT_SYMBOL_GPL(rt_mutex_destroy);
880
881
882
883
884
885
886
887
888
889
890void __rt_mutex_init(struct rt_mutex *lock, const char *name)
891{
892 lock->owner = NULL;
893 raw_spin_lock_init(&lock->wait_lock);
894 plist_head_init(&lock->wait_list);
895
896 debug_rt_mutex_init(lock, name);
897}
898EXPORT_SYMBOL_GPL(__rt_mutex_init);
899
900
901
902
903
904
905
906
907
908
909
910void rt_mutex_init_proxy_locked(struct rt_mutex *lock,
911 struct task_struct *proxy_owner)
912{
913 __rt_mutex_init(lock, NULL);
914 debug_rt_mutex_proxy_lock(lock, proxy_owner);
915 rt_mutex_set_owner(lock, proxy_owner);
916 rt_mutex_deadlock_account_lock(lock, proxy_owner);
917}
918
919
920
921
922
923
924
925
926
927void rt_mutex_proxy_unlock(struct rt_mutex *lock,
928 struct task_struct *proxy_owner)
929{
930 debug_rt_mutex_proxy_unlock(lock);
931 rt_mutex_set_owner(lock, NULL);
932 rt_mutex_deadlock_account_unlock(proxy_owner);
933}
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949int rt_mutex_start_proxy_lock(struct rt_mutex *lock,
950 struct rt_mutex_waiter *waiter,
951 struct task_struct *task, int detect_deadlock)
952{
953 int ret;
954
955 raw_spin_lock(&lock->wait_lock);
956
957 if (try_to_take_rt_mutex(lock, task, NULL)) {
958 raw_spin_unlock(&lock->wait_lock);
959 return 1;
960 }
961
962 ret = task_blocks_on_rt_mutex(lock, waiter, task, detect_deadlock);
963
964 if (ret && !rt_mutex_owner(lock)) {
965
966
967
968
969
970
971 ret = 0;
972 }
973
974 if (unlikely(ret))
975 remove_waiter(lock, waiter);
976
977 raw_spin_unlock(&lock->wait_lock);
978
979 debug_rt_mutex_print_deadlock(waiter);
980
981 return ret;
982}
983
984
985
986
987
988
989
990
991
992
993
994
995
996struct task_struct *rt_mutex_next_owner(struct rt_mutex *lock)
997{
998 if (!rt_mutex_has_waiters(lock))
999 return NULL;
1000
1001 return rt_mutex_top_waiter(lock)->task;
1002}
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020int rt_mutex_finish_proxy_lock(struct rt_mutex *lock,
1021 struct hrtimer_sleeper *to,
1022 struct rt_mutex_waiter *waiter,
1023 int detect_deadlock)
1024{
1025 int ret;
1026
1027 raw_spin_lock(&lock->wait_lock);
1028
1029 set_current_state(TASK_INTERRUPTIBLE);
1030
1031 ret = __rt_mutex_slowlock(lock, TASK_INTERRUPTIBLE, to, waiter);
1032
1033 set_current_state(TASK_RUNNING);
1034
1035 if (unlikely(ret))
1036 remove_waiter(lock, waiter);
1037
1038
1039
1040
1041
1042 fixup_rt_mutex_waiters(lock);
1043
1044 raw_spin_unlock(&lock->wait_lock);
1045
1046 return ret;
1047}
1048