1
2
3
4
5#include <linux/spinlock.h>
6#include <linux/export.h>
7
8#define RT_MUTEX_BUILD_MUTEX
9#include "rtmutex.c"
10
11
12
13
14int max_lock_depth = 1024;
15
16
17
18
19
20
21
22static __always_inline int __rt_mutex_lock_common(struct rt_mutex *lock,
23 unsigned int state,
24 unsigned int subclass)
25{
26 int ret;
27
28 might_sleep();
29 mutex_acquire(&lock->dep_map, subclass, 0, _RET_IP_);
30 ret = __rt_mutex_lock(&lock->rtmutex, state);
31 if (ret)
32 mutex_release(&lock->dep_map, _RET_IP_);
33 return ret;
34}
35
36void rt_mutex_base_init(struct rt_mutex_base *rtb)
37{
38 __rt_mutex_base_init(rtb);
39}
40EXPORT_SYMBOL(rt_mutex_base_init);
41
42#ifdef CONFIG_DEBUG_LOCK_ALLOC
43
44
45
46
47
48
49void __sched rt_mutex_lock_nested(struct rt_mutex *lock, unsigned int subclass)
50{
51 __rt_mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, subclass);
52}
53EXPORT_SYMBOL_GPL(rt_mutex_lock_nested);
54
55#else
56
57
58
59
60
61
62void __sched rt_mutex_lock(struct rt_mutex *lock)
63{
64 __rt_mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, 0);
65}
66EXPORT_SYMBOL_GPL(rt_mutex_lock);
67#endif
68
69
70
71
72
73
74
75
76
77
78int __sched rt_mutex_lock_interruptible(struct rt_mutex *lock)
79{
80 return __rt_mutex_lock_common(lock, TASK_INTERRUPTIBLE, 0);
81}
82EXPORT_SYMBOL_GPL(rt_mutex_lock_interruptible);
83
84
85
86
87
88
89
90
91
92
93
94
95
96int __sched rt_mutex_trylock(struct rt_mutex *lock)
97{
98 int ret;
99
100 if (IS_ENABLED(CONFIG_DEBUG_RT_MUTEXES) && WARN_ON_ONCE(!in_task()))
101 return 0;
102
103 ret = __rt_mutex_trylock(&lock->rtmutex);
104 if (ret)
105 mutex_acquire(&lock->dep_map, 0, 1, _RET_IP_);
106
107 return ret;
108}
109EXPORT_SYMBOL_GPL(rt_mutex_trylock);
110
111
112
113
114
115
116void __sched rt_mutex_unlock(struct rt_mutex *lock)
117{
118 mutex_release(&lock->dep_map, _RET_IP_);
119 __rt_mutex_unlock(&lock->rtmutex);
120}
121EXPORT_SYMBOL_GPL(rt_mutex_unlock);
122
123
124
125
126int __sched rt_mutex_futex_trylock(struct rt_mutex_base *lock)
127{
128 return rt_mutex_slowtrylock(lock);
129}
130
131int __sched __rt_mutex_futex_trylock(struct rt_mutex_base *lock)
132{
133 return __rt_mutex_slowtrylock(lock);
134}
135
136
137
138
139
140
141
142
143bool __sched __rt_mutex_futex_unlock(struct rt_mutex_base *lock,
144 struct rt_wake_q_head *wqh)
145{
146 lockdep_assert_held(&lock->wait_lock);
147
148 debug_rt_mutex_unlock(lock);
149
150 if (!rt_mutex_has_waiters(lock)) {
151 lock->owner = NULL;
152 return false;
153 }
154
155
156
157
158
159
160
161 mark_wakeup_next_waiter(wqh, lock);
162
163 return true;
164}
165
166void __sched rt_mutex_futex_unlock(struct rt_mutex_base *lock)
167{
168 DEFINE_RT_WAKE_Q(wqh);
169 unsigned long flags;
170 bool postunlock;
171
172 raw_spin_lock_irqsave(&lock->wait_lock, flags);
173 postunlock = __rt_mutex_futex_unlock(lock, &wqh);
174 raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
175
176 if (postunlock)
177 rt_mutex_postunlock(&wqh);
178}
179
180
181
182
183
184
185
186
187
188
189
190
191void __sched __rt_mutex_init(struct rt_mutex *lock, const char *name,
192 struct lock_class_key *key)
193{
194 debug_check_no_locks_freed((void *)lock, sizeof(*lock));
195 __rt_mutex_base_init(&lock->rtmutex);
196 lockdep_init_map_wait(&lock->dep_map, name, key, 0, LD_WAIT_SLEEP);
197}
198EXPORT_SYMBOL_GPL(__rt_mutex_init);
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214void __sched rt_mutex_init_proxy_locked(struct rt_mutex_base *lock,
215 struct task_struct *proxy_owner)
216{
217 static struct lock_class_key pi_futex_key;
218
219 __rt_mutex_base_init(lock);
220
221
222
223
224
225
226
227
228
229 lockdep_set_class(&lock->wait_lock, &pi_futex_key);
230 rt_mutex_set_owner(lock, proxy_owner);
231}
232
233
234
235
236
237
238
239
240
241
242
243
244
245void __sched rt_mutex_proxy_unlock(struct rt_mutex_base *lock)
246{
247 debug_rt_mutex_proxy_unlock(lock);
248 rt_mutex_set_owner(lock, NULL);
249}
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270int __sched __rt_mutex_start_proxy_lock(struct rt_mutex_base *lock,
271 struct rt_mutex_waiter *waiter,
272 struct task_struct *task)
273{
274 int ret;
275
276 lockdep_assert_held(&lock->wait_lock);
277
278 if (try_to_take_rt_mutex(lock, task, NULL))
279 return 1;
280
281
282 ret = task_blocks_on_rt_mutex(lock, waiter, task, NULL,
283 RT_MUTEX_FULL_CHAINWALK);
284
285 if (ret && !rt_mutex_owner(lock)) {
286
287
288
289
290
291
292 ret = 0;
293 }
294
295 return ret;
296}
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317int __sched rt_mutex_start_proxy_lock(struct rt_mutex_base *lock,
318 struct rt_mutex_waiter *waiter,
319 struct task_struct *task)
320{
321 int ret;
322
323 raw_spin_lock_irq(&lock->wait_lock);
324 ret = __rt_mutex_start_proxy_lock(lock, waiter, task);
325 if (unlikely(ret))
326 remove_waiter(lock, waiter);
327 raw_spin_unlock_irq(&lock->wait_lock);
328
329 return ret;
330}
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349int __sched rt_mutex_wait_proxy_lock(struct rt_mutex_base *lock,
350 struct hrtimer_sleeper *to,
351 struct rt_mutex_waiter *waiter)
352{
353 int ret;
354
355 raw_spin_lock_irq(&lock->wait_lock);
356
357 set_current_state(TASK_INTERRUPTIBLE);
358 ret = rt_mutex_slowlock_block(lock, NULL, TASK_INTERRUPTIBLE, to, waiter);
359
360
361
362
363 fixup_rt_mutex_waiters(lock);
364 raw_spin_unlock_irq(&lock->wait_lock);
365
366 return ret;
367}
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389bool __sched rt_mutex_cleanup_proxy_lock(struct rt_mutex_base *lock,
390 struct rt_mutex_waiter *waiter)
391{
392 bool cleanup = false;
393
394 raw_spin_lock_irq(&lock->wait_lock);
395
396
397
398
399
400
401
402
403
404
405
406 try_to_take_rt_mutex(lock, current, waiter);
407
408
409
410
411 if (rt_mutex_owner(lock) != current) {
412 remove_waiter(lock, waiter);
413 cleanup = true;
414 }
415
416
417
418
419 fixup_rt_mutex_waiters(lock);
420
421 raw_spin_unlock_irq(&lock->wait_lock);
422
423 return cleanup;
424}
425
426
427
428
429
430
431void __sched rt_mutex_adjust_pi(struct task_struct *task)
432{
433 struct rt_mutex_waiter *waiter;
434 struct rt_mutex_base *next_lock;
435 unsigned long flags;
436
437 raw_spin_lock_irqsave(&task->pi_lock, flags);
438
439 waiter = task->pi_blocked_on;
440 if (!waiter || rt_mutex_waiter_equal(waiter, task_to_waiter(task))) {
441 raw_spin_unlock_irqrestore(&task->pi_lock, flags);
442 return;
443 }
444 next_lock = waiter->lock;
445 raw_spin_unlock_irqrestore(&task->pi_lock, flags);
446
447
448 get_task_struct(task);
449
450 rt_mutex_adjust_prio_chain(task, RT_MUTEX_MIN_CHAINWALK, NULL,
451 next_lock, NULL, task);
452}
453
454
455
456
457void __sched rt_mutex_postunlock(struct rt_wake_q_head *wqh)
458{
459 rt_mutex_wake_up_q(wqh);
460}
461
462#ifdef CONFIG_DEBUG_RT_MUTEXES
463void rt_mutex_debug_task_free(struct task_struct *task)
464{
465 DEBUG_LOCKS_WARN_ON(!RB_EMPTY_ROOT(&task->pi_waiters.rb_root));
466 DEBUG_LOCKS_WARN_ON(task->pi_blocked_on);
467}
468#endif
469
470#ifdef CONFIG_PREEMPT_RT
471
472void __mutex_rt_init(struct mutex *mutex, const char *name,
473 struct lock_class_key *key)
474{
475 debug_check_no_locks_freed((void *)mutex, sizeof(*mutex));
476 lockdep_init_map_wait(&mutex->dep_map, name, key, 0, LD_WAIT_SLEEP);
477}
478EXPORT_SYMBOL(__mutex_rt_init);
479
480static __always_inline int __mutex_lock_common(struct mutex *lock,
481 unsigned int state,
482 unsigned int subclass,
483 struct lockdep_map *nest_lock,
484 unsigned long ip)
485{
486 int ret;
487
488 might_sleep();
489 mutex_acquire_nest(&lock->dep_map, subclass, 0, nest_lock, ip);
490 ret = __rt_mutex_lock(&lock->rtmutex, state);
491 if (ret)
492 mutex_release(&lock->dep_map, ip);
493 else
494 lock_acquired(&lock->dep_map, ip);
495 return ret;
496}
497
498#ifdef CONFIG_DEBUG_LOCK_ALLOC
499void __sched mutex_lock_nested(struct mutex *lock, unsigned int subclass)
500{
501 __mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, subclass, NULL, _RET_IP_);
502}
503EXPORT_SYMBOL_GPL(mutex_lock_nested);
504
505void __sched _mutex_lock_nest_lock(struct mutex *lock,
506 struct lockdep_map *nest_lock)
507{
508 __mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, 0, nest_lock, _RET_IP_);
509}
510EXPORT_SYMBOL_GPL(_mutex_lock_nest_lock);
511
512int __sched mutex_lock_interruptible_nested(struct mutex *lock,
513 unsigned int subclass)
514{
515 return __mutex_lock_common(lock, TASK_INTERRUPTIBLE, subclass, NULL, _RET_IP_);
516}
517EXPORT_SYMBOL_GPL(mutex_lock_interruptible_nested);
518
519int __sched mutex_lock_killable_nested(struct mutex *lock,
520 unsigned int subclass)
521{
522 return __mutex_lock_common(lock, TASK_KILLABLE, subclass, NULL, _RET_IP_);
523}
524EXPORT_SYMBOL_GPL(mutex_lock_killable_nested);
525
526void __sched mutex_lock_io_nested(struct mutex *lock, unsigned int subclass)
527{
528 int token;
529
530 might_sleep();
531
532 token = io_schedule_prepare();
533 __mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, subclass, NULL, _RET_IP_);
534 io_schedule_finish(token);
535}
536EXPORT_SYMBOL_GPL(mutex_lock_io_nested);
537
538#else
539
540void __sched mutex_lock(struct mutex *lock)
541{
542 __mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, 0, NULL, _RET_IP_);
543}
544EXPORT_SYMBOL(mutex_lock);
545
546int __sched mutex_lock_interruptible(struct mutex *lock)
547{
548 return __mutex_lock_common(lock, TASK_INTERRUPTIBLE, 0, NULL, _RET_IP_);
549}
550EXPORT_SYMBOL(mutex_lock_interruptible);
551
552int __sched mutex_lock_killable(struct mutex *lock)
553{
554 return __mutex_lock_common(lock, TASK_KILLABLE, 0, NULL, _RET_IP_);
555}
556EXPORT_SYMBOL(mutex_lock_killable);
557
558void __sched mutex_lock_io(struct mutex *lock)
559{
560 int token = io_schedule_prepare();
561
562 __mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, 0, NULL, _RET_IP_);
563 io_schedule_finish(token);
564}
565EXPORT_SYMBOL(mutex_lock_io);
566#endif
567
568int __sched mutex_trylock(struct mutex *lock)
569{
570 int ret;
571
572 if (IS_ENABLED(CONFIG_DEBUG_RT_MUTEXES) && WARN_ON_ONCE(!in_task()))
573 return 0;
574
575 ret = __rt_mutex_trylock(&lock->rtmutex);
576 if (ret)
577 mutex_acquire(&lock->dep_map, 0, 1, _RET_IP_);
578
579 return ret;
580}
581EXPORT_SYMBOL(mutex_trylock);
582
583void __sched mutex_unlock(struct mutex *lock)
584{
585 mutex_release(&lock->dep_map, _RET_IP_);
586 __rt_mutex_unlock(&lock->rtmutex);
587}
588EXPORT_SYMBOL(mutex_unlock);
589
590#endif
591