1
2
3
4
5
6
7
8
9
10
11
12
13
14#include <linux/rwsem.h>
15#include <linux/mutex.h>
16#include <linux/ww_mutex.h>
17#include <linux/sched.h>
18#include <linux/sched/mm.h>
19#include <linux/delay.h>
20#include <linux/lockdep.h>
21#include <linux/spinlock.h>
22#include <linux/kallsyms.h>
23#include <linux/interrupt.h>
24#include <linux/debug_locks.h>
25#include <linux/irqflags.h>
26#include <linux/rtmutex.h>
27#include <linux/local_lock.h>
28
29
30
31
32static unsigned int debug_locks_verbose;
33unsigned int force_read_lock_recursive;
34
35static DEFINE_WD_CLASS(ww_lockdep);
36
37static int __init setup_debug_locks_verbose(char *str)
38{
39 get_option(&str, &debug_locks_verbose);
40
41 return 1;
42}
43
44__setup("debug_locks_verbose=", setup_debug_locks_verbose);
45
46#define FAILURE 0
47#define SUCCESS 1
48
49#define LOCKTYPE_SPIN 0x1
50#define LOCKTYPE_RWLOCK 0x2
51#define LOCKTYPE_MUTEX 0x4
52#define LOCKTYPE_RWSEM 0x8
53#define LOCKTYPE_WW 0x10
54#define LOCKTYPE_RTMUTEX 0x20
55#define LOCKTYPE_LL 0x40
56#define LOCKTYPE_SPECIAL 0x80
57
58static struct ww_acquire_ctx t, t2;
59static struct ww_mutex o, o2, o3;
60
61
62
63
64
65static DEFINE_SPINLOCK(lock_A);
66static DEFINE_SPINLOCK(lock_B);
67static DEFINE_SPINLOCK(lock_C);
68static DEFINE_SPINLOCK(lock_D);
69
70static DEFINE_RAW_SPINLOCK(raw_lock_A);
71static DEFINE_RAW_SPINLOCK(raw_lock_B);
72
73static DEFINE_RWLOCK(rwlock_A);
74static DEFINE_RWLOCK(rwlock_B);
75static DEFINE_RWLOCK(rwlock_C);
76static DEFINE_RWLOCK(rwlock_D);
77
78static DEFINE_MUTEX(mutex_A);
79static DEFINE_MUTEX(mutex_B);
80static DEFINE_MUTEX(mutex_C);
81static DEFINE_MUTEX(mutex_D);
82
83static DECLARE_RWSEM(rwsem_A);
84static DECLARE_RWSEM(rwsem_B);
85static DECLARE_RWSEM(rwsem_C);
86static DECLARE_RWSEM(rwsem_D);
87
88#ifdef CONFIG_RT_MUTEXES
89
90static DEFINE_RT_MUTEX(rtmutex_A);
91static DEFINE_RT_MUTEX(rtmutex_B);
92static DEFINE_RT_MUTEX(rtmutex_C);
93static DEFINE_RT_MUTEX(rtmutex_D);
94
95#endif
96
97
98
99
100
101
102
103static DEFINE_SPINLOCK(lock_X1);
104static DEFINE_SPINLOCK(lock_X2);
105static DEFINE_SPINLOCK(lock_Y1);
106static DEFINE_SPINLOCK(lock_Y2);
107static DEFINE_SPINLOCK(lock_Z1);
108static DEFINE_SPINLOCK(lock_Z2);
109
110static DEFINE_RWLOCK(rwlock_X1);
111static DEFINE_RWLOCK(rwlock_X2);
112static DEFINE_RWLOCK(rwlock_Y1);
113static DEFINE_RWLOCK(rwlock_Y2);
114static DEFINE_RWLOCK(rwlock_Z1);
115static DEFINE_RWLOCK(rwlock_Z2);
116
117static DEFINE_MUTEX(mutex_X1);
118static DEFINE_MUTEX(mutex_X2);
119static DEFINE_MUTEX(mutex_Y1);
120static DEFINE_MUTEX(mutex_Y2);
121static DEFINE_MUTEX(mutex_Z1);
122static DEFINE_MUTEX(mutex_Z2);
123
124static DECLARE_RWSEM(rwsem_X1);
125static DECLARE_RWSEM(rwsem_X2);
126static DECLARE_RWSEM(rwsem_Y1);
127static DECLARE_RWSEM(rwsem_Y2);
128static DECLARE_RWSEM(rwsem_Z1);
129static DECLARE_RWSEM(rwsem_Z2);
130
131#ifdef CONFIG_RT_MUTEXES
132
133static DEFINE_RT_MUTEX(rtmutex_X1);
134static DEFINE_RT_MUTEX(rtmutex_X2);
135static DEFINE_RT_MUTEX(rtmutex_Y1);
136static DEFINE_RT_MUTEX(rtmutex_Y2);
137static DEFINE_RT_MUTEX(rtmutex_Z1);
138static DEFINE_RT_MUTEX(rtmutex_Z2);
139
140#endif
141
142static local_lock_t local_A = INIT_LOCAL_LOCK(local_A);
143
144
145
146
147
148#define INIT_CLASS_FUNC(class) \
149static noinline void \
150init_class_##class(spinlock_t *lock, rwlock_t *rwlock, \
151 struct mutex *mutex, struct rw_semaphore *rwsem)\
152{ \
153 spin_lock_init(lock); \
154 rwlock_init(rwlock); \
155 mutex_init(mutex); \
156 init_rwsem(rwsem); \
157}
158
159INIT_CLASS_FUNC(X)
160INIT_CLASS_FUNC(Y)
161INIT_CLASS_FUNC(Z)
162
163static void init_shared_classes(void)
164{
165#ifdef CONFIG_RT_MUTEXES
166 static struct lock_class_key rt_X, rt_Y, rt_Z;
167
168 __rt_mutex_init(&rtmutex_X1, __func__, &rt_X);
169 __rt_mutex_init(&rtmutex_X2, __func__, &rt_X);
170 __rt_mutex_init(&rtmutex_Y1, __func__, &rt_Y);
171 __rt_mutex_init(&rtmutex_Y2, __func__, &rt_Y);
172 __rt_mutex_init(&rtmutex_Z1, __func__, &rt_Z);
173 __rt_mutex_init(&rtmutex_Z2, __func__, &rt_Z);
174#endif
175
176 init_class_X(&lock_X1, &rwlock_X1, &mutex_X1, &rwsem_X1);
177 init_class_X(&lock_X2, &rwlock_X2, &mutex_X2, &rwsem_X2);
178
179 init_class_Y(&lock_Y1, &rwlock_Y1, &mutex_Y1, &rwsem_Y1);
180 init_class_Y(&lock_Y2, &rwlock_Y2, &mutex_Y2, &rwsem_Y2);
181
182 init_class_Z(&lock_Z1, &rwlock_Z1, &mutex_Z1, &rwsem_Z1);
183 init_class_Z(&lock_Z2, &rwlock_Z2, &mutex_Z2, &rwsem_Z2);
184}
185
186
187
188
189
190
191
192#define HARDIRQ_DISABLE local_irq_disable
193#define HARDIRQ_ENABLE local_irq_enable
194
195#define HARDIRQ_ENTER() \
196 local_irq_disable(); \
197 __irq_enter(); \
198 lockdep_hardirq_threaded(); \
199 WARN_ON(!in_irq());
200
201#define HARDIRQ_EXIT() \
202 __irq_exit(); \
203 local_irq_enable();
204
205#define SOFTIRQ_DISABLE local_bh_disable
206#define SOFTIRQ_ENABLE local_bh_enable
207
208#define SOFTIRQ_ENTER() \
209 local_bh_disable(); \
210 local_irq_disable(); \
211 lockdep_softirq_enter(); \
212 WARN_ON(!in_softirq());
213
214#define SOFTIRQ_EXIT() \
215 lockdep_softirq_exit(); \
216 local_irq_enable(); \
217 local_bh_enable();
218
219
220
221
222
223#define L(x) spin_lock(&lock_##x)
224#define U(x) spin_unlock(&lock_##x)
225#define LU(x) L(x); U(x)
226#define SI(x) spin_lock_init(&lock_##x)
227
228#define WL(x) write_lock(&rwlock_##x)
229#define WU(x) write_unlock(&rwlock_##x)
230#define WLU(x) WL(x); WU(x)
231
232#define RL(x) read_lock(&rwlock_##x)
233#define RU(x) read_unlock(&rwlock_##x)
234#define RLU(x) RL(x); RU(x)
235#define RWI(x) rwlock_init(&rwlock_##x)
236
237#define ML(x) mutex_lock(&mutex_##x)
238#define MU(x) mutex_unlock(&mutex_##x)
239#define MI(x) mutex_init(&mutex_##x)
240
241#define RTL(x) rt_mutex_lock(&rtmutex_##x)
242#define RTU(x) rt_mutex_unlock(&rtmutex_##x)
243#define RTI(x) rt_mutex_init(&rtmutex_##x)
244
245#define WSL(x) down_write(&rwsem_##x)
246#define WSU(x) up_write(&rwsem_##x)
247
248#define RSL(x) down_read(&rwsem_##x)
249#define RSU(x) up_read(&rwsem_##x)
250#define RWSI(x) init_rwsem(&rwsem_##x)
251
252#ifndef CONFIG_DEBUG_WW_MUTEX_SLOWPATH
253#define WWAI(x) ww_acquire_init(x, &ww_lockdep)
254#else
255#define WWAI(x) do { ww_acquire_init(x, &ww_lockdep); (x)->deadlock_inject_countdown = ~0U; } while (0)
256#endif
257#define WWAD(x) ww_acquire_done(x)
258#define WWAF(x) ww_acquire_fini(x)
259
260#define WWL(x, c) ww_mutex_lock(x, c)
261#define WWT(x) ww_mutex_trylock(x)
262#define WWL1(x) ww_mutex_lock(x, NULL)
263#define WWU(x) ww_mutex_unlock(x)
264
265
266#define LOCK_UNLOCK_2(x,y) LOCK(x); LOCK(y); UNLOCK(y); UNLOCK(x)
267
268
269
270
271
272
273#define GENERATE_TESTCASE(name) \
274 \
275static void name(void) { E(); }
276
277#define GENERATE_PERMUTATIONS_2_EVENTS(name) \
278 \
279static void name##_12(void) { E1(); E2(); } \
280static void name##_21(void) { E2(); E1(); }
281
282#define GENERATE_PERMUTATIONS_3_EVENTS(name) \
283 \
284static void name##_123(void) { E1(); E2(); E3(); } \
285static void name##_132(void) { E1(); E3(); E2(); } \
286static void name##_213(void) { E2(); E1(); E3(); } \
287static void name##_231(void) { E2(); E3(); E1(); } \
288static void name##_312(void) { E3(); E1(); E2(); } \
289static void name##_321(void) { E3(); E2(); E1(); }
290
291
292
293
294
295#define E() \
296 \
297 LOCK(X1); \
298 LOCK(X2);
299
300
301
302
303#include "locking-selftest-spin.h"
304GENERATE_TESTCASE(AA_spin)
305#include "locking-selftest-wlock.h"
306GENERATE_TESTCASE(AA_wlock)
307#include "locking-selftest-rlock.h"
308GENERATE_TESTCASE(AA_rlock)
309#include "locking-selftest-mutex.h"
310GENERATE_TESTCASE(AA_mutex)
311#include "locking-selftest-wsem.h"
312GENERATE_TESTCASE(AA_wsem)
313#include "locking-selftest-rsem.h"
314GENERATE_TESTCASE(AA_rsem)
315
316#ifdef CONFIG_RT_MUTEXES
317#include "locking-selftest-rtmutex.h"
318GENERATE_TESTCASE(AA_rtmutex);
319#endif
320
321#undef E
322
323
324
325
326
327static void rlock_AA1(void)
328{
329 RL(X1);
330 RL(X1);
331}
332
333static void rlock_AA1B(void)
334{
335 RL(X1);
336 RL(X2);
337}
338
339static void rsem_AA1(void)
340{
341 RSL(X1);
342 RSL(X1);
343}
344
345static void rsem_AA1B(void)
346{
347 RSL(X1);
348 RSL(X2);
349}
350
351
352
353static void rlock_AA2(void)
354{
355 RL(X1);
356 WL(X2);
357}
358
359static void rsem_AA2(void)
360{
361 RSL(X1);
362 WSL(X2);
363}
364
365static void rlock_AA3(void)
366{
367 WL(X1);
368 RL(X2);
369}
370
371static void rsem_AA3(void)
372{
373 WSL(X1);
374 RSL(X2);
375}
376
377
378
379
380
381
382
383static void rlock_ABBA1(void)
384{
385 RL(X1);
386 L(Y1);
387 U(Y1);
388 RU(X1);
389
390 L(Y1);
391 WL(X1);
392 WU(X1);
393 U(Y1);
394}
395
396static void rwsem_ABBA1(void)
397{
398 RSL(X1);
399 ML(Y1);
400 MU(Y1);
401 RSU(X1);
402
403 ML(Y1);
404 WSL(X1);
405 WSU(X1);
406 MU(Y1);
407}
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434static void rlock_chaincache_ABBA1(void)
435{
436 RL(X1);
437 L(Y1);
438 U(Y1);
439 RU(X1);
440
441 L(Y1);
442 RL(X1);
443 RU(X1);
444 U(Y1);
445
446 L(Y1);
447 WL(X1);
448 WU(X1);
449 U(Y1);
450}
451
452
453
454
455
456
457
458static void rlock_ABBA2(void)
459{
460 RL(X1);
461 L(Y1);
462 U(Y1);
463 RU(X1);
464
465 L(Y1);
466 RL(X1);
467 RU(X1);
468 U(Y1);
469}
470
471static void rwsem_ABBA2(void)
472{
473 RSL(X1);
474 ML(Y1);
475 MU(Y1);
476 RSU(X1);
477
478 ML(Y1);
479 RSL(X1);
480 RSU(X1);
481 MU(Y1);
482}
483
484
485
486
487
488
489
490
491static void rlock_ABBA3(void)
492{
493 WL(X1);
494 L(Y1);
495 U(Y1);
496 WU(X1);
497
498 L(Y1);
499 WL(X1);
500 WU(X1);
501 U(Y1);
502}
503
504static void rwsem_ABBA3(void)
505{
506 WSL(X1);
507 ML(Y1);
508 MU(Y1);
509 WSU(X1);
510
511 ML(Y1);
512 WSL(X1);
513 WSU(X1);
514 MU(Y1);
515}
516
517
518
519
520
521#define E() \
522 \
523 LOCK_UNLOCK_2(A, B); \
524 LOCK_UNLOCK_2(B, A);
525
526
527
528
529#include "locking-selftest-spin.h"
530GENERATE_TESTCASE(ABBA_spin)
531#include "locking-selftest-wlock.h"
532GENERATE_TESTCASE(ABBA_wlock)
533#include "locking-selftest-rlock.h"
534GENERATE_TESTCASE(ABBA_rlock)
535#include "locking-selftest-mutex.h"
536GENERATE_TESTCASE(ABBA_mutex)
537#include "locking-selftest-wsem.h"
538GENERATE_TESTCASE(ABBA_wsem)
539#include "locking-selftest-rsem.h"
540GENERATE_TESTCASE(ABBA_rsem)
541
542#ifdef CONFIG_RT_MUTEXES
543#include "locking-selftest-rtmutex.h"
544GENERATE_TESTCASE(ABBA_rtmutex);
545#endif
546
547#undef E
548
549
550
551
552
553#define E() \
554 \
555 LOCK_UNLOCK_2(A, B); \
556 LOCK_UNLOCK_2(B, C); \
557 LOCK_UNLOCK_2(C, A);
558
559
560
561
562#include "locking-selftest-spin.h"
563GENERATE_TESTCASE(ABBCCA_spin)
564#include "locking-selftest-wlock.h"
565GENERATE_TESTCASE(ABBCCA_wlock)
566#include "locking-selftest-rlock.h"
567GENERATE_TESTCASE(ABBCCA_rlock)
568#include "locking-selftest-mutex.h"
569GENERATE_TESTCASE(ABBCCA_mutex)
570#include "locking-selftest-wsem.h"
571GENERATE_TESTCASE(ABBCCA_wsem)
572#include "locking-selftest-rsem.h"
573GENERATE_TESTCASE(ABBCCA_rsem)
574
575#ifdef CONFIG_RT_MUTEXES
576#include "locking-selftest-rtmutex.h"
577GENERATE_TESTCASE(ABBCCA_rtmutex);
578#endif
579
580#undef E
581
582
583
584
585
586#define E() \
587 \
588 LOCK_UNLOCK_2(A, B); \
589 LOCK_UNLOCK_2(C, A); \
590 LOCK_UNLOCK_2(B, C);
591
592
593
594
595#include "locking-selftest-spin.h"
596GENERATE_TESTCASE(ABCABC_spin)
597#include "locking-selftest-wlock.h"
598GENERATE_TESTCASE(ABCABC_wlock)
599#include "locking-selftest-rlock.h"
600GENERATE_TESTCASE(ABCABC_rlock)
601#include "locking-selftest-mutex.h"
602GENERATE_TESTCASE(ABCABC_mutex)
603#include "locking-selftest-wsem.h"
604GENERATE_TESTCASE(ABCABC_wsem)
605#include "locking-selftest-rsem.h"
606GENERATE_TESTCASE(ABCABC_rsem)
607
608#ifdef CONFIG_RT_MUTEXES
609#include "locking-selftest-rtmutex.h"
610GENERATE_TESTCASE(ABCABC_rtmutex);
611#endif
612
613#undef E
614
615
616
617
618
619#define E() \
620 \
621 LOCK_UNLOCK_2(A, B); \
622 LOCK_UNLOCK_2(B, C); \
623 LOCK_UNLOCK_2(C, D); \
624 LOCK_UNLOCK_2(D, A);
625
626
627
628
629#include "locking-selftest-spin.h"
630GENERATE_TESTCASE(ABBCCDDA_spin)
631#include "locking-selftest-wlock.h"
632GENERATE_TESTCASE(ABBCCDDA_wlock)
633#include "locking-selftest-rlock.h"
634GENERATE_TESTCASE(ABBCCDDA_rlock)
635#include "locking-selftest-mutex.h"
636GENERATE_TESTCASE(ABBCCDDA_mutex)
637#include "locking-selftest-wsem.h"
638GENERATE_TESTCASE(ABBCCDDA_wsem)
639#include "locking-selftest-rsem.h"
640GENERATE_TESTCASE(ABBCCDDA_rsem)
641
642#ifdef CONFIG_RT_MUTEXES
643#include "locking-selftest-rtmutex.h"
644GENERATE_TESTCASE(ABBCCDDA_rtmutex);
645#endif
646
647#undef E
648
649
650
651
652#define E() \
653 \
654 LOCK_UNLOCK_2(A, B); \
655 LOCK_UNLOCK_2(C, D); \
656 LOCK_UNLOCK_2(B, D); \
657 LOCK_UNLOCK_2(D, A);
658
659
660
661
662#include "locking-selftest-spin.h"
663GENERATE_TESTCASE(ABCDBDDA_spin)
664#include "locking-selftest-wlock.h"
665GENERATE_TESTCASE(ABCDBDDA_wlock)
666#include "locking-selftest-rlock.h"
667GENERATE_TESTCASE(ABCDBDDA_rlock)
668#include "locking-selftest-mutex.h"
669GENERATE_TESTCASE(ABCDBDDA_mutex)
670#include "locking-selftest-wsem.h"
671GENERATE_TESTCASE(ABCDBDDA_wsem)
672#include "locking-selftest-rsem.h"
673GENERATE_TESTCASE(ABCDBDDA_rsem)
674
675#ifdef CONFIG_RT_MUTEXES
676#include "locking-selftest-rtmutex.h"
677GENERATE_TESTCASE(ABCDBDDA_rtmutex);
678#endif
679
680#undef E
681
682
683
684
685#define E() \
686 \
687 LOCK_UNLOCK_2(A, B); \
688 LOCK_UNLOCK_2(C, D); \
689 LOCK_UNLOCK_2(B, C); \
690 LOCK_UNLOCK_2(D, A);
691
692
693
694
695#include "locking-selftest-spin.h"
696GENERATE_TESTCASE(ABCDBCDA_spin)
697#include "locking-selftest-wlock.h"
698GENERATE_TESTCASE(ABCDBCDA_wlock)
699#include "locking-selftest-rlock.h"
700GENERATE_TESTCASE(ABCDBCDA_rlock)
701#include "locking-selftest-mutex.h"
702GENERATE_TESTCASE(ABCDBCDA_mutex)
703#include "locking-selftest-wsem.h"
704GENERATE_TESTCASE(ABCDBCDA_wsem)
705#include "locking-selftest-rsem.h"
706GENERATE_TESTCASE(ABCDBCDA_rsem)
707
708#ifdef CONFIG_RT_MUTEXES
709#include "locking-selftest-rtmutex.h"
710GENERATE_TESTCASE(ABCDBCDA_rtmutex);
711#endif
712
713#undef E
714
715
716
717
718#define E() \
719 \
720 LOCK(A); \
721 UNLOCK(A); \
722 UNLOCK(A);
723
724
725
726
727#include "locking-selftest-spin.h"
728GENERATE_TESTCASE(double_unlock_spin)
729#include "locking-selftest-wlock.h"
730GENERATE_TESTCASE(double_unlock_wlock)
731#include "locking-selftest-rlock.h"
732GENERATE_TESTCASE(double_unlock_rlock)
733#include "locking-selftest-mutex.h"
734GENERATE_TESTCASE(double_unlock_mutex)
735#include "locking-selftest-wsem.h"
736GENERATE_TESTCASE(double_unlock_wsem)
737#include "locking-selftest-rsem.h"
738GENERATE_TESTCASE(double_unlock_rsem)
739
740#ifdef CONFIG_RT_MUTEXES
741#include "locking-selftest-rtmutex.h"
742GENERATE_TESTCASE(double_unlock_rtmutex);
743#endif
744
745#undef E
746
747
748
749
750#define E() \
751 \
752 LOCK(A); \
753 INIT(A);
754
755
756
757
758#include "locking-selftest-spin.h"
759GENERATE_TESTCASE(init_held_spin)
760#include "locking-selftest-wlock.h"
761GENERATE_TESTCASE(init_held_wlock)
762#include "locking-selftest-rlock.h"
763GENERATE_TESTCASE(init_held_rlock)
764#include "locking-selftest-mutex.h"
765GENERATE_TESTCASE(init_held_mutex)
766#include "locking-selftest-wsem.h"
767GENERATE_TESTCASE(init_held_wsem)
768#include "locking-selftest-rsem.h"
769GENERATE_TESTCASE(init_held_rsem)
770
771#ifdef CONFIG_RT_MUTEXES
772#include "locking-selftest-rtmutex.h"
773GENERATE_TESTCASE(init_held_rtmutex);
774#endif
775
776#undef E
777
778
779
780
781#define E1() \
782 \
783 IRQ_ENTER(); \
784 LOCK(A); \
785 UNLOCK(A); \
786 IRQ_EXIT();
787
788#define E2() \
789 \
790 LOCK(A); \
791 UNLOCK(A);
792
793
794
795
796#include "locking-selftest-spin-hardirq.h"
797GENERATE_PERMUTATIONS_2_EVENTS(irqsafe1_hard_spin)
798
799#include "locking-selftest-rlock-hardirq.h"
800GENERATE_PERMUTATIONS_2_EVENTS(irqsafe1_hard_rlock)
801
802#include "locking-selftest-wlock-hardirq.h"
803GENERATE_PERMUTATIONS_2_EVENTS(irqsafe1_hard_wlock)
804
805#include "locking-selftest-spin-softirq.h"
806GENERATE_PERMUTATIONS_2_EVENTS(irqsafe1_soft_spin)
807
808#include "locking-selftest-rlock-softirq.h"
809GENERATE_PERMUTATIONS_2_EVENTS(irqsafe1_soft_rlock)
810
811#include "locking-selftest-wlock-softirq.h"
812GENERATE_PERMUTATIONS_2_EVENTS(irqsafe1_soft_wlock)
813
814#undef E1
815#undef E2
816
817
818
819
820#define E1() \
821 \
822 SOFTIRQ_ENTER(); \
823 LOCK(A); \
824 UNLOCK(A); \
825 SOFTIRQ_EXIT();
826
827#define E2() \
828 \
829 HARDIRQ_DISABLE(); \
830 LOCK(A); \
831 HARDIRQ_ENABLE(); \
832 UNLOCK(A);
833
834
835
836
837#include "locking-selftest-spin.h"
838GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2A_spin)
839
840#include "locking-selftest-wlock.h"
841GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2A_wlock)
842
843#include "locking-selftest-rlock.h"
844GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2A_rlock)
845
846#undef E1
847#undef E2
848
849
850
851
852#define E1() \
853 \
854 IRQ_ENTER(); \
855 LOCK(A); \
856 UNLOCK(A); \
857 IRQ_EXIT();
858
859#define E2() \
860 \
861 IRQ_DISABLE(); \
862 LOCK(A); \
863 IRQ_ENABLE(); \
864 UNLOCK(A);
865
866
867
868
869#include "locking-selftest-spin-hardirq.h"
870GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2B_hard_spin)
871
872#include "locking-selftest-rlock-hardirq.h"
873GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2B_hard_rlock)
874
875#include "locking-selftest-wlock-hardirq.h"
876GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2B_hard_wlock)
877
878#include "locking-selftest-spin-softirq.h"
879GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2B_soft_spin)
880
881#include "locking-selftest-rlock-softirq.h"
882GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2B_soft_rlock)
883
884#include "locking-selftest-wlock-softirq.h"
885GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2B_soft_wlock)
886
887#undef E1
888#undef E2
889
890
891
892
893#define E1() \
894 \
895 LOCK(A); \
896 LOCK(B); \
897 UNLOCK(B); \
898 UNLOCK(A); \
899
900#define E2() \
901 \
902 LOCK(B); \
903 UNLOCK(B);
904
905#define E3() \
906 \
907 IRQ_ENTER(); \
908 LOCK(A); \
909 UNLOCK(A); \
910 IRQ_EXIT();
911
912
913
914
915#include "locking-selftest-spin-hardirq.h"
916GENERATE_PERMUTATIONS_3_EVENTS(irqsafe3_hard_spin)
917
918#include "locking-selftest-rlock-hardirq.h"
919GENERATE_PERMUTATIONS_3_EVENTS(irqsafe3_hard_rlock)
920
921#include "locking-selftest-wlock-hardirq.h"
922GENERATE_PERMUTATIONS_3_EVENTS(irqsafe3_hard_wlock)
923
924#include "locking-selftest-spin-softirq.h"
925GENERATE_PERMUTATIONS_3_EVENTS(irqsafe3_soft_spin)
926
927#include "locking-selftest-rlock-softirq.h"
928GENERATE_PERMUTATIONS_3_EVENTS(irqsafe3_soft_rlock)
929
930#include "locking-selftest-wlock-softirq.h"
931GENERATE_PERMUTATIONS_3_EVENTS(irqsafe3_soft_wlock)
932
933#undef E1
934#undef E2
935#undef E3
936
937
938
939
940
941
942#define E1() \
943 IRQ_DISABLE(); \
944 LOCK(A); \
945 LOCK(B); \
946 UNLOCK(B); \
947 UNLOCK(A); \
948 IRQ_ENABLE();
949
950#define E2() \
951 LOCK(B); \
952 UNLOCK(B);
953
954#define E3() \
955 IRQ_ENTER(); \
956 LOCK(A); \
957 UNLOCK(A); \
958 IRQ_EXIT();
959
960
961
962
963#include "locking-selftest-spin-hardirq.h"
964GENERATE_PERMUTATIONS_3_EVENTS(irqsafe4_hard_spin)
965
966#include "locking-selftest-rlock-hardirq.h"
967GENERATE_PERMUTATIONS_3_EVENTS(irqsafe4_hard_rlock)
968
969#include "locking-selftest-wlock-hardirq.h"
970GENERATE_PERMUTATIONS_3_EVENTS(irqsafe4_hard_wlock)
971
972#include "locking-selftest-spin-softirq.h"
973GENERATE_PERMUTATIONS_3_EVENTS(irqsafe4_soft_spin)
974
975#include "locking-selftest-rlock-softirq.h"
976GENERATE_PERMUTATIONS_3_EVENTS(irqsafe4_soft_rlock)
977
978#include "locking-selftest-wlock-softirq.h"
979GENERATE_PERMUTATIONS_3_EVENTS(irqsafe4_soft_wlock)
980
981#undef E1
982#undef E2
983#undef E3
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001#define E1() \
1002 \
1003 IRQ_DISABLE(); \
1004 WL(A); \
1005 LOCK(B); \
1006 UNLOCK(B); \
1007 WU(A); \
1008 IRQ_ENABLE();
1009
1010#define E2() \
1011 \
1012 LOCK(B); \
1013 UNLOCK(B);
1014
1015#define E3() \
1016 \
1017 IRQ_ENTER(); \
1018 RL(A); \
1019 RU(A); \
1020 IRQ_EXIT();
1021
1022
1023
1024
1025#include "locking-selftest-spin-hardirq.h"
1026GENERATE_PERMUTATIONS_3_EVENTS(irq_inversion_hard_spin)
1027
1028#include "locking-selftest-rlock-hardirq.h"
1029GENERATE_PERMUTATIONS_3_EVENTS(irq_inversion_hard_rlock)
1030
1031#include "locking-selftest-wlock-hardirq.h"
1032GENERATE_PERMUTATIONS_3_EVENTS(irq_inversion_hard_wlock)
1033
1034#include "locking-selftest-spin-softirq.h"
1035GENERATE_PERMUTATIONS_3_EVENTS(irq_inversion_soft_spin)
1036
1037#include "locking-selftest-rlock-softirq.h"
1038GENERATE_PERMUTATIONS_3_EVENTS(irq_inversion_soft_rlock)
1039
1040#include "locking-selftest-wlock-softirq.h"
1041GENERATE_PERMUTATIONS_3_EVENTS(irq_inversion_soft_wlock)
1042
1043#undef E1
1044#undef E2
1045#undef E3
1046
1047
1048
1049
1050
1051#define E1() \
1052 \
1053 WL(X1); \
1054 RL(Y1); \
1055 RU(Y1); \
1056 WU(X1);
1057
1058#define E2() \
1059 \
1060 WL(Y1); \
1061 RL(Z1); \
1062 RU(Z1); \
1063 WU(Y1);
1064
1065#define E3() \
1066 \
1067 WL(Z1); \
1068 RL(X1); \
1069 RU(X1); \
1070 WU(Z1);
1071
1072#include "locking-selftest-rlock.h"
1073GENERATE_PERMUTATIONS_3_EVENTS(W1R2_W2R3_W3R1)
1074
1075#undef E1
1076#undef E2
1077#undef E3
1078
1079
1080
1081
1082
1083#define E1() \
1084 \
1085 WL(X1); \
1086 WL(Y1); \
1087 WU(Y1); \
1088 WU(X1);
1089
1090#define E2() \
1091 \
1092 RL(Y1); \
1093 RL(Z1); \
1094 RU(Z1); \
1095 RU(Y1);
1096
1097#define E3() \
1098 \
1099 WL(Z1); \
1100 RL(X1); \
1101 RU(X1); \
1102 WU(Z1);
1103
1104#include "locking-selftest-rlock.h"
1105GENERATE_PERMUTATIONS_3_EVENTS(W1W2_R2R3_W3R1)
1106
1107#undef E1
1108#undef E2
1109#undef E3
1110
1111
1112
1113
1114
1115#define E1() \
1116 \
1117 WL(X1); \
1118 WL(Y1); \
1119 WU(Y1); \
1120 WU(X1);
1121
1122#define E2() \
1123 \
1124 RL(Y1); \
1125 RL(Z1); \
1126 RU(Z1); \
1127 RU(Y1);
1128
1129#define E3() \
1130 \
1131 RL(Z1); \
1132 WL(X1); \
1133 WU(X1); \
1134 RU(Z1);
1135
1136#include "locking-selftest-rlock.h"
1137GENERATE_PERMUTATIONS_3_EVENTS(W1R2_R2R3_W3W1)
1138
1139#undef E1
1140#undef E2
1141#undef E3
1142
1143
1144
1145
1146
1147#define E1() \
1148 \
1149 WL(X1); \
1150 RL(Y1); \
1151 RU(Y1); \
1152 WU(X1);
1153
1154#define E2() \
1155 \
1156 RL(Y1); \
1157 RL(Z1); \
1158 RU(Z1); \
1159 RU(Y1);
1160
1161#define E3() \
1162 \
1163 WL(Z1); \
1164 WL(X1); \
1165 WU(X1); \
1166 WU(Z1);
1167
1168#include "locking-selftest-rlock.h"
1169GENERATE_PERMUTATIONS_3_EVENTS(W1W2_R2R3_R3W1)
1170
1171#undef E1
1172#undef E2
1173#undef E3
1174
1175
1176
1177
1178#define E1() \
1179 \
1180 IRQ_DISABLE(); \
1181 WL(A); \
1182 WU(A); \
1183 IRQ_ENABLE();
1184
1185#define E2() \
1186 \
1187 RL(A); \
1188 RU(A); \
1189
1190#define E3() \
1191 \
1192 IRQ_ENTER(); \
1193 LOCK(A); \
1194 L(B); \
1195 U(B); \
1196 UNLOCK(A); \
1197 IRQ_EXIT();
1198
1199
1200
1201
1202#include "locking-selftest-hardirq.h"
1203#include "locking-selftest-rlock.h"
1204GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion_hard_rlock)
1205
1206#include "locking-selftest-wlock.h"
1207GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion_hard_wlock)
1208
1209#include "locking-selftest-softirq.h"
1210#include "locking-selftest-rlock.h"
1211GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion_soft_rlock)
1212
1213#include "locking-selftest-wlock.h"
1214GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion_soft_wlock)
1215
1216#undef E1
1217#undef E2
1218#undef E3
1219
1220
1221
1222
1223
1224#define E1() \
1225 \
1226 IRQ_DISABLE(); \
1227 L(B); \
1228 LOCK(A); \
1229 UNLOCK(A); \
1230 U(B); \
1231 IRQ_ENABLE();
1232
1233#define E2() \
1234 \
1235 RL(A); \
1236 RU(A); \
1237
1238#define E3() \
1239 \
1240 IRQ_ENTER(); \
1241 L(B); \
1242 U(B); \
1243 IRQ_EXIT();
1244
1245
1246
1247
1248#include "locking-selftest-hardirq.h"
1249#include "locking-selftest-rlock.h"
1250GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion2_hard_rlock)
1251
1252#include "locking-selftest-wlock.h"
1253GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion2_hard_wlock)
1254
1255#include "locking-selftest-softirq.h"
1256#include "locking-selftest-rlock.h"
1257GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion2_soft_rlock)
1258
1259#include "locking-selftest-wlock.h"
1260GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion2_soft_wlock)
1261
1262#undef E1
1263#undef E2
1264#undef E3
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278#define E1() \
1279 \
1280 IRQ_DISABLE(); \
1281 WL(B); \
1282 LOCK(A); \
1283 UNLOCK(A); \
1284 WU(B); \
1285 IRQ_ENABLE();
1286
1287#define E2() \
1288 \
1289 RL(A); \
1290 RU(A); \
1291
1292#define E3() \
1293 \
1294 IRQ_ENTER(); \
1295 RL(B); \
1296 RU(B); \
1297 IRQ_EXIT();
1298
1299
1300
1301
1302#include "locking-selftest-hardirq.h"
1303#include "locking-selftest-rlock.h"
1304GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion3_hard_rlock)
1305
1306#include "locking-selftest-wlock.h"
1307GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion3_hard_wlock)
1308
1309#include "locking-selftest-softirq.h"
1310#include "locking-selftest-rlock.h"
1311GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion3_soft_rlock)
1312
1313#include "locking-selftest-wlock.h"
1314GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion3_soft_wlock)
1315
1316#ifdef CONFIG_DEBUG_LOCK_ALLOC
1317# define I_SPINLOCK(x) lockdep_reset_lock(&lock_##x.dep_map)
1318# define I_RAW_SPINLOCK(x) lockdep_reset_lock(&raw_lock_##x.dep_map)
1319# define I_RWLOCK(x) lockdep_reset_lock(&rwlock_##x.dep_map)
1320# define I_MUTEX(x) lockdep_reset_lock(&mutex_##x.dep_map)
1321# define I_RWSEM(x) lockdep_reset_lock(&rwsem_##x.dep_map)
1322# define I_WW(x) lockdep_reset_lock(&x.dep_map)
1323# define I_LOCAL_LOCK(x) lockdep_reset_lock(&local_##x.dep_map)
1324#ifdef CONFIG_RT_MUTEXES
1325# define I_RTMUTEX(x) lockdep_reset_lock(&rtmutex_##x.dep_map)
1326#endif
1327#else
1328# define I_SPINLOCK(x)
1329# define I_RAW_SPINLOCK(x)
1330# define I_RWLOCK(x)
1331# define I_MUTEX(x)
1332# define I_RWSEM(x)
1333# define I_WW(x)
1334# define I_LOCAL_LOCK(x)
1335#endif
1336
1337#ifndef I_RTMUTEX
1338# define I_RTMUTEX(x)
1339#endif
1340
1341#ifdef CONFIG_RT_MUTEXES
1342#define I2_RTMUTEX(x) rt_mutex_init(&rtmutex_##x)
1343#else
1344#define I2_RTMUTEX(x)
1345#endif
1346
1347#define I1(x) \
1348 do { \
1349 I_SPINLOCK(x); \
1350 I_RWLOCK(x); \
1351 I_MUTEX(x); \
1352 I_RWSEM(x); \
1353 I_RTMUTEX(x); \
1354 } while (0)
1355
1356#define I2(x) \
1357 do { \
1358 spin_lock_init(&lock_##x); \
1359 rwlock_init(&rwlock_##x); \
1360 mutex_init(&mutex_##x); \
1361 init_rwsem(&rwsem_##x); \
1362 I2_RTMUTEX(x); \
1363 } while (0)
1364
1365static void reset_locks(void)
1366{
1367 local_irq_disable();
1368 lockdep_free_key_range(&ww_lockdep.acquire_key, 1);
1369 lockdep_free_key_range(&ww_lockdep.mutex_key, 1);
1370
1371 I1(A); I1(B); I1(C); I1(D);
1372 I1(X1); I1(X2); I1(Y1); I1(Y2); I1(Z1); I1(Z2);
1373 I_WW(t); I_WW(t2); I_WW(o.base); I_WW(o2.base); I_WW(o3.base);
1374 I_RAW_SPINLOCK(A); I_RAW_SPINLOCK(B);
1375 I_LOCAL_LOCK(A);
1376
1377 lockdep_reset();
1378
1379 I2(A); I2(B); I2(C); I2(D);
1380 init_shared_classes();
1381 raw_spin_lock_init(&raw_lock_A);
1382 raw_spin_lock_init(&raw_lock_B);
1383 local_lock_init(&local_A);
1384
1385 ww_mutex_init(&o, &ww_lockdep); ww_mutex_init(&o2, &ww_lockdep); ww_mutex_init(&o3, &ww_lockdep);
1386 memset(&t, 0, sizeof(t)); memset(&t2, 0, sizeof(t2));
1387 memset(&ww_lockdep.acquire_key, 0, sizeof(ww_lockdep.acquire_key));
1388 memset(&ww_lockdep.mutex_key, 0, sizeof(ww_lockdep.mutex_key));
1389 local_irq_enable();
1390}
1391
1392#undef I
1393
1394static int testcase_total;
1395static int testcase_successes;
1396static int expected_testcase_failures;
1397static int unexpected_testcase_failures;
1398
1399static void dotest(void (*testcase_fn)(void), int expected, int lockclass_mask)
1400{
1401 unsigned long saved_preempt_count = preempt_count();
1402
1403 WARN_ON(irqs_disabled());
1404
1405 debug_locks_silent = !(debug_locks_verbose & lockclass_mask);
1406
1407 testcase_fn();
1408
1409
1410
1411#ifndef CONFIG_PROVE_LOCKING
1412 if (expected == FAILURE && debug_locks) {
1413 expected_testcase_failures++;
1414 pr_cont("failed|");
1415 }
1416 else
1417#endif
1418 if (debug_locks != expected) {
1419 unexpected_testcase_failures++;
1420 pr_cont("FAILED|");
1421 } else {
1422 testcase_successes++;
1423 pr_cont(" ok |");
1424 }
1425 testcase_total++;
1426
1427 if (debug_locks_verbose & lockclass_mask)
1428 pr_cont(" lockclass mask: %x, debug_locks: %d, expected: %d\n",
1429 lockclass_mask, debug_locks, expected);
1430
1431
1432
1433
1434 preempt_count_set(saved_preempt_count);
1435#ifdef CONFIG_TRACE_IRQFLAGS
1436 if (softirq_count())
1437 current->softirqs_enabled = 0;
1438 else
1439 current->softirqs_enabled = 1;
1440#endif
1441
1442 reset_locks();
1443}
1444
1445#ifdef CONFIG_RT_MUTEXES
1446#define dotest_rt(fn, e, m) dotest((fn), (e), (m))
1447#else
1448#define dotest_rt(fn, e, m)
1449#endif
1450
1451static inline void print_testname(const char *testname)
1452{
1453 printk("%33s:", testname);
1454}
1455
1456#define DO_TESTCASE_1(desc, name, nr) \
1457 print_testname(desc"/"#nr); \
1458 dotest(name##_##nr, SUCCESS, LOCKTYPE_RWLOCK); \
1459 pr_cont("\n");
1460
1461#define DO_TESTCASE_1B(desc, name, nr) \
1462 print_testname(desc"/"#nr); \
1463 dotest(name##_##nr, FAILURE, LOCKTYPE_RWLOCK); \
1464 pr_cont("\n");
1465
1466#define DO_TESTCASE_1RR(desc, name, nr) \
1467 print_testname(desc"/"#nr); \
1468 pr_cont(" |"); \
1469 dotest(name##_##nr, SUCCESS, LOCKTYPE_RWLOCK); \
1470 pr_cont("\n");
1471
1472#define DO_TESTCASE_1RRB(desc, name, nr) \
1473 print_testname(desc"/"#nr); \
1474 pr_cont(" |"); \
1475 dotest(name##_##nr, FAILURE, LOCKTYPE_RWLOCK); \
1476 pr_cont("\n");
1477
1478
1479#define DO_TESTCASE_3(desc, name, nr) \
1480 print_testname(desc"/"#nr); \
1481 dotest(name##_spin_##nr, FAILURE, LOCKTYPE_SPIN); \
1482 dotest(name##_wlock_##nr, FAILURE, LOCKTYPE_RWLOCK); \
1483 dotest(name##_rlock_##nr, SUCCESS, LOCKTYPE_RWLOCK); \
1484 pr_cont("\n");
1485
1486#define DO_TESTCASE_3RW(desc, name, nr) \
1487 print_testname(desc"/"#nr); \
1488 dotest(name##_spin_##nr, FAILURE, LOCKTYPE_SPIN|LOCKTYPE_RWLOCK);\
1489 dotest(name##_wlock_##nr, FAILURE, LOCKTYPE_RWLOCK); \
1490 dotest(name##_rlock_##nr, SUCCESS, LOCKTYPE_RWLOCK); \
1491 pr_cont("\n");
1492
1493#define DO_TESTCASE_2RW(desc, name, nr) \
1494 print_testname(desc"/"#nr); \
1495 pr_cont(" |"); \
1496 dotest(name##_wlock_##nr, FAILURE, LOCKTYPE_RWLOCK); \
1497 dotest(name##_rlock_##nr, SUCCESS, LOCKTYPE_RWLOCK); \
1498 pr_cont("\n");
1499
1500#define DO_TESTCASE_2x2RW(desc, name, nr) \
1501 DO_TESTCASE_2RW("hard-"desc, name##_hard, nr) \
1502 DO_TESTCASE_2RW("soft-"desc, name##_soft, nr) \
1503
1504#define DO_TESTCASE_6x2x2RW(desc, name) \
1505 DO_TESTCASE_2x2RW(desc, name, 123); \
1506 DO_TESTCASE_2x2RW(desc, name, 132); \
1507 DO_TESTCASE_2x2RW(desc, name, 213); \
1508 DO_TESTCASE_2x2RW(desc, name, 231); \
1509 DO_TESTCASE_2x2RW(desc, name, 312); \
1510 DO_TESTCASE_2x2RW(desc, name, 321);
1511
1512#define DO_TESTCASE_6(desc, name) \
1513 print_testname(desc); \
1514 dotest(name##_spin, FAILURE, LOCKTYPE_SPIN); \
1515 dotest(name##_wlock, FAILURE, LOCKTYPE_RWLOCK); \
1516 dotest(name##_rlock, FAILURE, LOCKTYPE_RWLOCK); \
1517 dotest(name##_mutex, FAILURE, LOCKTYPE_MUTEX); \
1518 dotest(name##_wsem, FAILURE, LOCKTYPE_RWSEM); \
1519 dotest(name##_rsem, FAILURE, LOCKTYPE_RWSEM); \
1520 dotest_rt(name##_rtmutex, FAILURE, LOCKTYPE_RTMUTEX); \
1521 pr_cont("\n");
1522
1523#define DO_TESTCASE_6_SUCCESS(desc, name) \
1524 print_testname(desc); \
1525 dotest(name##_spin, SUCCESS, LOCKTYPE_SPIN); \
1526 dotest(name##_wlock, SUCCESS, LOCKTYPE_RWLOCK); \
1527 dotest(name##_rlock, SUCCESS, LOCKTYPE_RWLOCK); \
1528 dotest(name##_mutex, SUCCESS, LOCKTYPE_MUTEX); \
1529 dotest(name##_wsem, SUCCESS, LOCKTYPE_RWSEM); \
1530 dotest(name##_rsem, SUCCESS, LOCKTYPE_RWSEM); \
1531 dotest_rt(name##_rtmutex, SUCCESS, LOCKTYPE_RTMUTEX); \
1532 pr_cont("\n");
1533
1534
1535
1536
1537#define DO_TESTCASE_6R(desc, name) \
1538 print_testname(desc); \
1539 dotest(name##_spin, FAILURE, LOCKTYPE_SPIN); \
1540 dotest(name##_wlock, FAILURE, LOCKTYPE_RWLOCK); \
1541 dotest(name##_rlock, SUCCESS, LOCKTYPE_RWLOCK); \
1542 dotest(name##_mutex, FAILURE, LOCKTYPE_MUTEX); \
1543 dotest(name##_wsem, FAILURE, LOCKTYPE_RWSEM); \
1544 dotest(name##_rsem, FAILURE, LOCKTYPE_RWSEM); \
1545 dotest_rt(name##_rtmutex, FAILURE, LOCKTYPE_RTMUTEX); \
1546 pr_cont("\n");
1547
1548#define DO_TESTCASE_2I(desc, name, nr) \
1549 DO_TESTCASE_1("hard-"desc, name##_hard, nr); \
1550 DO_TESTCASE_1("soft-"desc, name##_soft, nr);
1551
1552#define DO_TESTCASE_2IB(desc, name, nr) \
1553 DO_TESTCASE_1B("hard-"desc, name##_hard, nr); \
1554 DO_TESTCASE_1B("soft-"desc, name##_soft, nr);
1555
1556#define DO_TESTCASE_6I(desc, name, nr) \
1557 DO_TESTCASE_3("hard-"desc, name##_hard, nr); \
1558 DO_TESTCASE_3("soft-"desc, name##_soft, nr);
1559
1560#define DO_TESTCASE_6IRW(desc, name, nr) \
1561 DO_TESTCASE_3RW("hard-"desc, name##_hard, nr); \
1562 DO_TESTCASE_3RW("soft-"desc, name##_soft, nr);
1563
1564#define DO_TESTCASE_2x3(desc, name) \
1565 DO_TESTCASE_3(desc, name, 12); \
1566 DO_TESTCASE_3(desc, name, 21);
1567
1568#define DO_TESTCASE_2x6(desc, name) \
1569 DO_TESTCASE_6I(desc, name, 12); \
1570 DO_TESTCASE_6I(desc, name, 21);
1571
1572#define DO_TESTCASE_6x2(desc, name) \
1573 DO_TESTCASE_2I(desc, name, 123); \
1574 DO_TESTCASE_2I(desc, name, 132); \
1575 DO_TESTCASE_2I(desc, name, 213); \
1576 DO_TESTCASE_2I(desc, name, 231); \
1577 DO_TESTCASE_2I(desc, name, 312); \
1578 DO_TESTCASE_2I(desc, name, 321);
1579
1580#define DO_TESTCASE_6x2B(desc, name) \
1581 DO_TESTCASE_2IB(desc, name, 123); \
1582 DO_TESTCASE_2IB(desc, name, 132); \
1583 DO_TESTCASE_2IB(desc, name, 213); \
1584 DO_TESTCASE_2IB(desc, name, 231); \
1585 DO_TESTCASE_2IB(desc, name, 312); \
1586 DO_TESTCASE_2IB(desc, name, 321);
1587
1588#define DO_TESTCASE_6x1RR(desc, name) \
1589 DO_TESTCASE_1RR(desc, name, 123); \
1590 DO_TESTCASE_1RR(desc, name, 132); \
1591 DO_TESTCASE_1RR(desc, name, 213); \
1592 DO_TESTCASE_1RR(desc, name, 231); \
1593 DO_TESTCASE_1RR(desc, name, 312); \
1594 DO_TESTCASE_1RR(desc, name, 321);
1595
1596#define DO_TESTCASE_6x1RRB(desc, name) \
1597 DO_TESTCASE_1RRB(desc, name, 123); \
1598 DO_TESTCASE_1RRB(desc, name, 132); \
1599 DO_TESTCASE_1RRB(desc, name, 213); \
1600 DO_TESTCASE_1RRB(desc, name, 231); \
1601 DO_TESTCASE_1RRB(desc, name, 312); \
1602 DO_TESTCASE_1RRB(desc, name, 321);
1603
1604#define DO_TESTCASE_6x6(desc, name) \
1605 DO_TESTCASE_6I(desc, name, 123); \
1606 DO_TESTCASE_6I(desc, name, 132); \
1607 DO_TESTCASE_6I(desc, name, 213); \
1608 DO_TESTCASE_6I(desc, name, 231); \
1609 DO_TESTCASE_6I(desc, name, 312); \
1610 DO_TESTCASE_6I(desc, name, 321);
1611
1612#define DO_TESTCASE_6x6RW(desc, name) \
1613 DO_TESTCASE_6IRW(desc, name, 123); \
1614 DO_TESTCASE_6IRW(desc, name, 132); \
1615 DO_TESTCASE_6IRW(desc, name, 213); \
1616 DO_TESTCASE_6IRW(desc, name, 231); \
1617 DO_TESTCASE_6IRW(desc, name, 312); \
1618 DO_TESTCASE_6IRW(desc, name, 321);
1619
1620static void ww_test_fail_acquire(void)
1621{
1622 int ret;
1623
1624 WWAI(&t);
1625 t.stamp++;
1626
1627 ret = WWL(&o, &t);
1628
1629 if (WARN_ON(!o.ctx) ||
1630 WARN_ON(ret))
1631 return;
1632
1633
1634 ret = WWL(&o, &t);
1635 WARN_ON(ret != -EALREADY);
1636
1637 ret = WWT(&o);
1638 WARN_ON(ret);
1639
1640 t2 = t;
1641 t2.stamp++;
1642 ret = WWL(&o, &t2);
1643 WARN_ON(ret != -EDEADLK);
1644 WWU(&o);
1645
1646 if (WWT(&o))
1647 WWU(&o);
1648#ifdef CONFIG_DEBUG_LOCK_ALLOC
1649 else
1650 DEBUG_LOCKS_WARN_ON(1);
1651#endif
1652}
1653
1654static void ww_test_normal(void)
1655{
1656 int ret;
1657
1658 WWAI(&t);
1659
1660
1661
1662
1663
1664
1665
1666
1667 o.ctx = (void *)~0UL;
1668 mutex_lock(&o.base);
1669 mutex_unlock(&o.base);
1670 WARN_ON(o.ctx != (void *)~0UL);
1671
1672
1673 o.ctx = (void *)~0UL;
1674 ret = mutex_lock_interruptible(&o.base);
1675 if (!ret)
1676 mutex_unlock(&o.base);
1677 else
1678 WARN_ON(1);
1679 WARN_ON(o.ctx != (void *)~0UL);
1680
1681
1682 o.ctx = (void *)~0UL;
1683 ret = mutex_lock_killable(&o.base);
1684 if (!ret)
1685 mutex_unlock(&o.base);
1686 else
1687 WARN_ON(1);
1688 WARN_ON(o.ctx != (void *)~0UL);
1689
1690
1691 o.ctx = (void *)~0UL;
1692 ret = mutex_trylock(&o.base);
1693 WARN_ON(!ret);
1694 if (ret)
1695 mutex_unlock(&o.base);
1696 else
1697 WARN_ON(1);
1698 WARN_ON(o.ctx != (void *)~0UL);
1699
1700
1701 o.ctx = (void *)~0UL;
1702 mutex_lock(&o.base);
1703 ret = mutex_trylock(&o.base);
1704 WARN_ON(ret);
1705 mutex_unlock(&o.base);
1706 WARN_ON(o.ctx != (void *)~0UL);
1707
1708
1709 o.ctx = (void *)~0UL;
1710 mutex_lock_nest_lock(&o.base, &t);
1711 mutex_unlock(&o.base);
1712 WARN_ON(o.ctx != (void *)~0UL);
1713}
1714
1715static void ww_test_two_contexts(void)
1716{
1717 WWAI(&t);
1718 WWAI(&t2);
1719}
1720
1721static void ww_test_diff_class(void)
1722{
1723 WWAI(&t);
1724#ifdef CONFIG_DEBUG_MUTEXES
1725 t.ww_class = NULL;
1726#endif
1727 WWL(&o, &t);
1728}
1729
1730static void ww_test_context_done_twice(void)
1731{
1732 WWAI(&t);
1733 WWAD(&t);
1734 WWAD(&t);
1735 WWAF(&t);
1736}
1737
1738static void ww_test_context_unlock_twice(void)
1739{
1740 WWAI(&t);
1741 WWAD(&t);
1742 WWAF(&t);
1743 WWAF(&t);
1744}
1745
1746static void ww_test_context_fini_early(void)
1747{
1748 WWAI(&t);
1749 WWL(&o, &t);
1750 WWAD(&t);
1751 WWAF(&t);
1752}
1753
1754static void ww_test_context_lock_after_done(void)
1755{
1756 WWAI(&t);
1757 WWAD(&t);
1758 WWL(&o, &t);
1759}
1760
1761static void ww_test_object_unlock_twice(void)
1762{
1763 WWL1(&o);
1764 WWU(&o);
1765 WWU(&o);
1766}
1767
1768static void ww_test_object_lock_unbalanced(void)
1769{
1770 WWAI(&t);
1771 WWL(&o, &t);
1772 t.acquired = 0;
1773 WWU(&o);
1774 WWAF(&t);
1775}
1776
1777static void ww_test_object_lock_stale_context(void)
1778{
1779 WWAI(&t);
1780 o.ctx = &t2;
1781 WWL(&o, &t);
1782}
1783
1784static void ww_test_edeadlk_normal(void)
1785{
1786 int ret;
1787
1788 mutex_lock(&o2.base);
1789 o2.ctx = &t2;
1790 mutex_release(&o2.base.dep_map, _THIS_IP_);
1791
1792 WWAI(&t);
1793 t2 = t;
1794 t2.stamp--;
1795
1796 ret = WWL(&o, &t);
1797 WARN_ON(ret);
1798
1799 ret = WWL(&o2, &t);
1800 WARN_ON(ret != -EDEADLK);
1801
1802 o2.ctx = NULL;
1803 mutex_acquire(&o2.base.dep_map, 0, 1, _THIS_IP_);
1804 mutex_unlock(&o2.base);
1805 WWU(&o);
1806
1807 WWL(&o2, &t);
1808}
1809
1810static void ww_test_edeadlk_normal_slow(void)
1811{
1812 int ret;
1813
1814 mutex_lock(&o2.base);
1815 mutex_release(&o2.base.dep_map, _THIS_IP_);
1816 o2.ctx = &t2;
1817
1818 WWAI(&t);
1819 t2 = t;
1820 t2.stamp--;
1821
1822 ret = WWL(&o, &t);
1823 WARN_ON(ret);
1824
1825 ret = WWL(&o2, &t);
1826 WARN_ON(ret != -EDEADLK);
1827
1828 o2.ctx = NULL;
1829 mutex_acquire(&o2.base.dep_map, 0, 1, _THIS_IP_);
1830 mutex_unlock(&o2.base);
1831 WWU(&o);
1832
1833 ww_mutex_lock_slow(&o2, &t);
1834}
1835
1836static void ww_test_edeadlk_no_unlock(void)
1837{
1838 int ret;
1839
1840 mutex_lock(&o2.base);
1841 o2.ctx = &t2;
1842 mutex_release(&o2.base.dep_map, _THIS_IP_);
1843
1844 WWAI(&t);
1845 t2 = t;
1846 t2.stamp--;
1847
1848 ret = WWL(&o, &t);
1849 WARN_ON(ret);
1850
1851 ret = WWL(&o2, &t);
1852 WARN_ON(ret != -EDEADLK);
1853
1854 o2.ctx = NULL;
1855 mutex_acquire(&o2.base.dep_map, 0, 1, _THIS_IP_);
1856 mutex_unlock(&o2.base);
1857
1858 WWL(&o2, &t);
1859}
1860
1861static void ww_test_edeadlk_no_unlock_slow(void)
1862{
1863 int ret;
1864
1865 mutex_lock(&o2.base);
1866 mutex_release(&o2.base.dep_map, _THIS_IP_);
1867 o2.ctx = &t2;
1868
1869 WWAI(&t);
1870 t2 = t;
1871 t2.stamp--;
1872
1873 ret = WWL(&o, &t);
1874 WARN_ON(ret);
1875
1876 ret = WWL(&o2, &t);
1877 WARN_ON(ret != -EDEADLK);
1878
1879 o2.ctx = NULL;
1880 mutex_acquire(&o2.base.dep_map, 0, 1, _THIS_IP_);
1881 mutex_unlock(&o2.base);
1882
1883 ww_mutex_lock_slow(&o2, &t);
1884}
1885
1886static void ww_test_edeadlk_acquire_more(void)
1887{
1888 int ret;
1889
1890 mutex_lock(&o2.base);
1891 mutex_release(&o2.base.dep_map, _THIS_IP_);
1892 o2.ctx = &t2;
1893
1894 WWAI(&t);
1895 t2 = t;
1896 t2.stamp--;
1897
1898 ret = WWL(&o, &t);
1899 WARN_ON(ret);
1900
1901 ret = WWL(&o2, &t);
1902 WARN_ON(ret != -EDEADLK);
1903
1904 ret = WWL(&o3, &t);
1905}
1906
1907static void ww_test_edeadlk_acquire_more_slow(void)
1908{
1909 int ret;
1910
1911 mutex_lock(&o2.base);
1912 mutex_release(&o2.base.dep_map, _THIS_IP_);
1913 o2.ctx = &t2;
1914
1915 WWAI(&t);
1916 t2 = t;
1917 t2.stamp--;
1918
1919 ret = WWL(&o, &t);
1920 WARN_ON(ret);
1921
1922 ret = WWL(&o2, &t);
1923 WARN_ON(ret != -EDEADLK);
1924
1925 ww_mutex_lock_slow(&o3, &t);
1926}
1927
1928static void ww_test_edeadlk_acquire_more_edeadlk(void)
1929{
1930 int ret;
1931
1932 mutex_lock(&o2.base);
1933 mutex_release(&o2.base.dep_map, _THIS_IP_);
1934 o2.ctx = &t2;
1935
1936 mutex_lock(&o3.base);
1937 mutex_release(&o3.base.dep_map, _THIS_IP_);
1938 o3.ctx = &t2;
1939
1940 WWAI(&t);
1941 t2 = t;
1942 t2.stamp--;
1943
1944 ret = WWL(&o, &t);
1945 WARN_ON(ret);
1946
1947 ret = WWL(&o2, &t);
1948 WARN_ON(ret != -EDEADLK);
1949
1950 ret = WWL(&o3, &t);
1951 WARN_ON(ret != -EDEADLK);
1952}
1953
1954static void ww_test_edeadlk_acquire_more_edeadlk_slow(void)
1955{
1956 int ret;
1957
1958 mutex_lock(&o2.base);
1959 mutex_release(&o2.base.dep_map, _THIS_IP_);
1960 o2.ctx = &t2;
1961
1962 mutex_lock(&o3.base);
1963 mutex_release(&o3.base.dep_map, _THIS_IP_);
1964 o3.ctx = &t2;
1965
1966 WWAI(&t);
1967 t2 = t;
1968 t2.stamp--;
1969
1970 ret = WWL(&o, &t);
1971 WARN_ON(ret);
1972
1973 ret = WWL(&o2, &t);
1974 WARN_ON(ret != -EDEADLK);
1975
1976 ww_mutex_lock_slow(&o3, &t);
1977}
1978
1979static void ww_test_edeadlk_acquire_wrong(void)
1980{
1981 int ret;
1982
1983 mutex_lock(&o2.base);
1984 mutex_release(&o2.base.dep_map, _THIS_IP_);
1985 o2.ctx = &t2;
1986
1987 WWAI(&t);
1988 t2 = t;
1989 t2.stamp--;
1990
1991 ret = WWL(&o, &t);
1992 WARN_ON(ret);
1993
1994 ret = WWL(&o2, &t);
1995 WARN_ON(ret != -EDEADLK);
1996 if (!ret)
1997 WWU(&o2);
1998
1999 WWU(&o);
2000
2001 ret = WWL(&o3, &t);
2002}
2003
2004static void ww_test_edeadlk_acquire_wrong_slow(void)
2005{
2006 int ret;
2007
2008 mutex_lock(&o2.base);
2009 mutex_release(&o2.base.dep_map, _THIS_IP_);
2010 o2.ctx = &t2;
2011
2012 WWAI(&t);
2013 t2 = t;
2014 t2.stamp--;
2015
2016 ret = WWL(&o, &t);
2017 WARN_ON(ret);
2018
2019 ret = WWL(&o2, &t);
2020 WARN_ON(ret != -EDEADLK);
2021 if (!ret)
2022 WWU(&o2);
2023
2024 WWU(&o);
2025
2026 ww_mutex_lock_slow(&o3, &t);
2027}
2028
2029static void ww_test_spin_nest_unlocked(void)
2030{
2031 spin_lock_nest_lock(&lock_A, &o.base);
2032 U(A);
2033}
2034
2035
2036static void ww_test_spin_nest_lock(void)
2037{
2038 spin_lock(&lock_X1);
2039 spin_lock_nest_lock(&lock_Y1, &lock_X1);
2040 spin_lock(&lock_A);
2041 spin_lock_nest_lock(&lock_Y2, &lock_X1);
2042 spin_unlock(&lock_A);
2043 spin_unlock(&lock_Y2);
2044 spin_unlock(&lock_Y1);
2045 spin_unlock(&lock_X1);
2046}
2047
2048static void ww_test_unneeded_slow(void)
2049{
2050 WWAI(&t);
2051
2052 ww_mutex_lock_slow(&o, &t);
2053}
2054
2055static void ww_test_context_block(void)
2056{
2057 int ret;
2058
2059 WWAI(&t);
2060
2061 ret = WWL(&o, &t);
2062 WARN_ON(ret);
2063 WWL1(&o2);
2064}
2065
2066static void ww_test_context_try(void)
2067{
2068 int ret;
2069
2070 WWAI(&t);
2071
2072 ret = WWL(&o, &t);
2073 WARN_ON(ret);
2074
2075 ret = WWT(&o2);
2076 WARN_ON(!ret);
2077 WWU(&o2);
2078 WWU(&o);
2079}
2080
2081static void ww_test_context_context(void)
2082{
2083 int ret;
2084
2085 WWAI(&t);
2086
2087 ret = WWL(&o, &t);
2088 WARN_ON(ret);
2089
2090 ret = WWL(&o2, &t);
2091 WARN_ON(ret);
2092
2093 WWU(&o2);
2094 WWU(&o);
2095}
2096
2097static void ww_test_try_block(void)
2098{
2099 bool ret;
2100
2101 ret = WWT(&o);
2102 WARN_ON(!ret);
2103
2104 WWL1(&o2);
2105 WWU(&o2);
2106 WWU(&o);
2107}
2108
2109static void ww_test_try_try(void)
2110{
2111 bool ret;
2112
2113 ret = WWT(&o);
2114 WARN_ON(!ret);
2115 ret = WWT(&o2);
2116 WARN_ON(!ret);
2117 WWU(&o2);
2118 WWU(&o);
2119}
2120
2121static void ww_test_try_context(void)
2122{
2123 int ret;
2124
2125 ret = WWT(&o);
2126 WARN_ON(!ret);
2127
2128 WWAI(&t);
2129
2130 ret = WWL(&o2, &t);
2131 WARN_ON(ret);
2132}
2133
2134static void ww_test_block_block(void)
2135{
2136 WWL1(&o);
2137 WWL1(&o2);
2138}
2139
2140static void ww_test_block_try(void)
2141{
2142 bool ret;
2143
2144 WWL1(&o);
2145 ret = WWT(&o2);
2146 WARN_ON(!ret);
2147}
2148
2149static void ww_test_block_context(void)
2150{
2151 int ret;
2152
2153 WWL1(&o);
2154 WWAI(&t);
2155
2156 ret = WWL(&o2, &t);
2157 WARN_ON(ret);
2158}
2159
2160static void ww_test_spin_block(void)
2161{
2162 L(A);
2163 U(A);
2164
2165 WWL1(&o);
2166 L(A);
2167 U(A);
2168 WWU(&o);
2169
2170 L(A);
2171 WWL1(&o);
2172 WWU(&o);
2173 U(A);
2174}
2175
2176static void ww_test_spin_try(void)
2177{
2178 bool ret;
2179
2180 L(A);
2181 U(A);
2182
2183 ret = WWT(&o);
2184 WARN_ON(!ret);
2185 L(A);
2186 U(A);
2187 WWU(&o);
2188
2189 L(A);
2190 ret = WWT(&o);
2191 WARN_ON(!ret);
2192 WWU(&o);
2193 U(A);
2194}
2195
2196static void ww_test_spin_context(void)
2197{
2198 int ret;
2199
2200 L(A);
2201 U(A);
2202
2203 WWAI(&t);
2204
2205 ret = WWL(&o, &t);
2206 WARN_ON(ret);
2207 L(A);
2208 U(A);
2209 WWU(&o);
2210
2211 L(A);
2212 ret = WWL(&o, &t);
2213 WARN_ON(ret);
2214 WWU(&o);
2215 U(A);
2216}
2217
2218static void ww_tests(void)
2219{
2220 printk(" --------------------------------------------------------------------------\n");
2221 printk(" | Wound/wait tests |\n");
2222 printk(" ---------------------\n");
2223
2224 print_testname("ww api failures");
2225 dotest(ww_test_fail_acquire, SUCCESS, LOCKTYPE_WW);
2226 dotest(ww_test_normal, SUCCESS, LOCKTYPE_WW);
2227 dotest(ww_test_unneeded_slow, FAILURE, LOCKTYPE_WW);
2228 pr_cont("\n");
2229
2230 print_testname("ww contexts mixing");
2231 dotest(ww_test_two_contexts, FAILURE, LOCKTYPE_WW);
2232 dotest(ww_test_diff_class, FAILURE, LOCKTYPE_WW);
2233 pr_cont("\n");
2234
2235 print_testname("finishing ww context");
2236 dotest(ww_test_context_done_twice, FAILURE, LOCKTYPE_WW);
2237 dotest(ww_test_context_unlock_twice, FAILURE, LOCKTYPE_WW);
2238 dotest(ww_test_context_fini_early, FAILURE, LOCKTYPE_WW);
2239 dotest(ww_test_context_lock_after_done, FAILURE, LOCKTYPE_WW);
2240 pr_cont("\n");
2241
2242 print_testname("locking mismatches");
2243 dotest(ww_test_object_unlock_twice, FAILURE, LOCKTYPE_WW);
2244 dotest(ww_test_object_lock_unbalanced, FAILURE, LOCKTYPE_WW);
2245 dotest(ww_test_object_lock_stale_context, FAILURE, LOCKTYPE_WW);
2246 pr_cont("\n");
2247
2248 print_testname("EDEADLK handling");
2249 dotest(ww_test_edeadlk_normal, SUCCESS, LOCKTYPE_WW);
2250 dotest(ww_test_edeadlk_normal_slow, SUCCESS, LOCKTYPE_WW);
2251 dotest(ww_test_edeadlk_no_unlock, FAILURE, LOCKTYPE_WW);
2252 dotest(ww_test_edeadlk_no_unlock_slow, FAILURE, LOCKTYPE_WW);
2253 dotest(ww_test_edeadlk_acquire_more, FAILURE, LOCKTYPE_WW);
2254 dotest(ww_test_edeadlk_acquire_more_slow, FAILURE, LOCKTYPE_WW);
2255 dotest(ww_test_edeadlk_acquire_more_edeadlk, FAILURE, LOCKTYPE_WW);
2256 dotest(ww_test_edeadlk_acquire_more_edeadlk_slow, FAILURE, LOCKTYPE_WW);
2257 dotest(ww_test_edeadlk_acquire_wrong, FAILURE, LOCKTYPE_WW);
2258 dotest(ww_test_edeadlk_acquire_wrong_slow, FAILURE, LOCKTYPE_WW);
2259 pr_cont("\n");
2260
2261 print_testname("spinlock nest unlocked");
2262 dotest(ww_test_spin_nest_unlocked, FAILURE, LOCKTYPE_WW);
2263 pr_cont("\n");
2264
2265 print_testname("spinlock nest test");
2266 dotest(ww_test_spin_nest_lock, SUCCESS, LOCKTYPE_WW);
2267 pr_cont("\n");
2268
2269 printk(" -----------------------------------------------------\n");
2270 printk(" |block | try |context|\n");
2271 printk(" -----------------------------------------------------\n");
2272
2273 print_testname("context");
2274 dotest(ww_test_context_block, FAILURE, LOCKTYPE_WW);
2275 dotest(ww_test_context_try, SUCCESS, LOCKTYPE_WW);
2276 dotest(ww_test_context_context, SUCCESS, LOCKTYPE_WW);
2277 pr_cont("\n");
2278
2279 print_testname("try");
2280 dotest(ww_test_try_block, FAILURE, LOCKTYPE_WW);
2281 dotest(ww_test_try_try, SUCCESS, LOCKTYPE_WW);
2282 dotest(ww_test_try_context, FAILURE, LOCKTYPE_WW);
2283 pr_cont("\n");
2284
2285 print_testname("block");
2286 dotest(ww_test_block_block, FAILURE, LOCKTYPE_WW);
2287 dotest(ww_test_block_try, SUCCESS, LOCKTYPE_WW);
2288 dotest(ww_test_block_context, FAILURE, LOCKTYPE_WW);
2289 pr_cont("\n");
2290
2291 print_testname("spinlock");
2292 dotest(ww_test_spin_block, FAILURE, LOCKTYPE_WW);
2293 dotest(ww_test_spin_try, SUCCESS, LOCKTYPE_WW);
2294 dotest(ww_test_spin_context, FAILURE, LOCKTYPE_WW);
2295 pr_cont("\n");
2296}
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309static void queued_read_lock_hardirq_RE_Er(void)
2310{
2311 HARDIRQ_ENTER();
2312 read_lock(&rwlock_A);
2313 LOCK(B);
2314 UNLOCK(B);
2315 read_unlock(&rwlock_A);
2316 HARDIRQ_EXIT();
2317
2318 HARDIRQ_DISABLE();
2319 LOCK(B);
2320 read_lock(&rwlock_A);
2321 read_unlock(&rwlock_A);
2322 UNLOCK(B);
2323 HARDIRQ_ENABLE();
2324}
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336static void queued_read_lock_hardirq_ER_rE(void)
2337{
2338 HARDIRQ_ENTER();
2339 LOCK(B);
2340 read_lock(&rwlock_A);
2341 read_unlock(&rwlock_A);
2342 UNLOCK(B);
2343 HARDIRQ_EXIT();
2344
2345 HARDIRQ_DISABLE();
2346 read_lock(&rwlock_A);
2347 LOCK(B);
2348 UNLOCK(B);
2349 read_unlock(&rwlock_A);
2350 HARDIRQ_ENABLE();
2351}
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363static void queued_read_lock_hardirq_inversion(void)
2364{
2365
2366 HARDIRQ_ENTER();
2367 LOCK(B);
2368 UNLOCK(B);
2369 HARDIRQ_EXIT();
2370
2371 HARDIRQ_DISABLE();
2372 LOCK(B);
2373 read_lock(&rwlock_A);
2374 read_unlock(&rwlock_A);
2375 UNLOCK(B);
2376 HARDIRQ_ENABLE();
2377
2378 read_lock(&rwlock_A);
2379 read_unlock(&rwlock_A);
2380}
2381
2382static void queued_read_lock_tests(void)
2383{
2384 printk(" --------------------------------------------------------------------------\n");
2385 printk(" | queued read lock tests |\n");
2386 printk(" ---------------------------\n");
2387 print_testname("hardirq read-lock/lock-read");
2388 dotest(queued_read_lock_hardirq_RE_Er, FAILURE, LOCKTYPE_RWLOCK);
2389 pr_cont("\n");
2390
2391 print_testname("hardirq lock-read/read-lock");
2392 dotest(queued_read_lock_hardirq_ER_rE, SUCCESS, LOCKTYPE_RWLOCK);
2393 pr_cont("\n");
2394
2395 print_testname("hardirq inversion");
2396 dotest(queued_read_lock_hardirq_inversion, FAILURE, LOCKTYPE_RWLOCK);
2397 pr_cont("\n");
2398}
2399
2400static void fs_reclaim_correct_nesting(void)
2401{
2402 fs_reclaim_acquire(GFP_KERNEL);
2403 might_alloc(GFP_NOFS);
2404 fs_reclaim_release(GFP_KERNEL);
2405}
2406
2407static void fs_reclaim_wrong_nesting(void)
2408{
2409 fs_reclaim_acquire(GFP_KERNEL);
2410 might_alloc(GFP_KERNEL);
2411 fs_reclaim_release(GFP_KERNEL);
2412}
2413
2414static void fs_reclaim_protected_nesting(void)
2415{
2416 unsigned int flags;
2417
2418 fs_reclaim_acquire(GFP_KERNEL);
2419 flags = memalloc_nofs_save();
2420 might_alloc(GFP_KERNEL);
2421 memalloc_nofs_restore(flags);
2422 fs_reclaim_release(GFP_KERNEL);
2423}
2424
2425static void fs_reclaim_tests(void)
2426{
2427 printk(" --------------------\n");
2428 printk(" | fs_reclaim tests |\n");
2429 printk(" --------------------\n");
2430
2431 print_testname("correct nesting");
2432 dotest(fs_reclaim_correct_nesting, SUCCESS, 0);
2433 pr_cont("\n");
2434
2435 print_testname("wrong nesting");
2436 dotest(fs_reclaim_wrong_nesting, FAILURE, 0);
2437 pr_cont("\n");
2438
2439 print_testname("protected nesting");
2440 dotest(fs_reclaim_protected_nesting, SUCCESS, 0);
2441 pr_cont("\n");
2442}
2443
2444#define __guard(cleanup) __maybe_unused __attribute__((__cleanup__(cleanup)))
2445
2446static void hardirq_exit(int *_)
2447{
2448 HARDIRQ_EXIT();
2449}
2450
2451#define HARDIRQ_CONTEXT(name, ...) \
2452 int hardirq_guard_##name __guard(hardirq_exit); \
2453 HARDIRQ_ENTER();
2454
2455#define NOTTHREADED_HARDIRQ_CONTEXT(name, ...) \
2456 int notthreaded_hardirq_guard_##name __guard(hardirq_exit); \
2457 local_irq_disable(); \
2458 __irq_enter(); \
2459 WARN_ON(!in_irq());
2460
2461static void softirq_exit(int *_)
2462{
2463 SOFTIRQ_EXIT();
2464}
2465
2466#define SOFTIRQ_CONTEXT(name, ...) \
2467 int softirq_guard_##name __guard(softirq_exit); \
2468 SOFTIRQ_ENTER();
2469
2470static void rcu_exit(int *_)
2471{
2472 rcu_read_unlock();
2473}
2474
2475#define RCU_CONTEXT(name, ...) \
2476 int rcu_guard_##name __guard(rcu_exit); \
2477 rcu_read_lock();
2478
2479static void rcu_bh_exit(int *_)
2480{
2481 rcu_read_unlock_bh();
2482}
2483
2484#define RCU_BH_CONTEXT(name, ...) \
2485 int rcu_bh_guard_##name __guard(rcu_bh_exit); \
2486 rcu_read_lock_bh();
2487
2488static void rcu_sched_exit(int *_)
2489{
2490 rcu_read_unlock_sched();
2491}
2492
2493#define RCU_SCHED_CONTEXT(name, ...) \
2494 int rcu_sched_guard_##name __guard(rcu_sched_exit); \
2495 rcu_read_lock_sched();
2496
2497static void raw_spinlock_exit(raw_spinlock_t **lock)
2498{
2499 raw_spin_unlock(*lock);
2500}
2501
2502#define RAW_SPINLOCK_CONTEXT(name, lock) \
2503 raw_spinlock_t *raw_spinlock_guard_##name __guard(raw_spinlock_exit) = &(lock); \
2504 raw_spin_lock(&(lock));
2505
2506static void spinlock_exit(spinlock_t **lock)
2507{
2508 spin_unlock(*lock);
2509}
2510
2511#define SPINLOCK_CONTEXT(name, lock) \
2512 spinlock_t *spinlock_guard_##name __guard(spinlock_exit) = &(lock); \
2513 spin_lock(&(lock));
2514
2515static void mutex_exit(struct mutex **lock)
2516{
2517 mutex_unlock(*lock);
2518}
2519
2520#define MUTEX_CONTEXT(name, lock) \
2521 struct mutex *mutex_guard_##name __guard(mutex_exit) = &(lock); \
2522 mutex_lock(&(lock));
2523
2524#define GENERATE_2_CONTEXT_TESTCASE(outer, outer_lock, inner, inner_lock) \
2525 \
2526static void __maybe_unused inner##_in_##outer(void) \
2527{ \
2528 outer##_CONTEXT(_, outer_lock); \
2529 { \
2530 inner##_CONTEXT(_, inner_lock); \
2531 } \
2532}
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563#define GENERATE_2_CONTEXT_TESTCASE_FOR_ALL_OUTER(inner, inner_lock) \
2564GENERATE_2_CONTEXT_TESTCASE(HARDIRQ, , inner, inner_lock) \
2565GENERATE_2_CONTEXT_TESTCASE(NOTTHREADED_HARDIRQ, , inner, inner_lock) \
2566GENERATE_2_CONTEXT_TESTCASE(SOFTIRQ, , inner, inner_lock) \
2567GENERATE_2_CONTEXT_TESTCASE(RCU, , inner, inner_lock) \
2568GENERATE_2_CONTEXT_TESTCASE(RCU_BH, , inner, inner_lock) \
2569GENERATE_2_CONTEXT_TESTCASE(RCU_SCHED, , inner, inner_lock) \
2570GENERATE_2_CONTEXT_TESTCASE(RAW_SPINLOCK, raw_lock_A, inner, inner_lock) \
2571GENERATE_2_CONTEXT_TESTCASE(SPINLOCK, lock_A, inner, inner_lock) \
2572GENERATE_2_CONTEXT_TESTCASE(MUTEX, mutex_A, inner, inner_lock)
2573
2574GENERATE_2_CONTEXT_TESTCASE_FOR_ALL_OUTER(RCU, )
2575GENERATE_2_CONTEXT_TESTCASE_FOR_ALL_OUTER(RAW_SPINLOCK, raw_lock_B)
2576GENERATE_2_CONTEXT_TESTCASE_FOR_ALL_OUTER(SPINLOCK, lock_B)
2577GENERATE_2_CONTEXT_TESTCASE_FOR_ALL_OUTER(MUTEX, mutex_B)
2578
2579
2580#define DO_CONTEXT_TESTCASE_OUTER_PREEMPTIBLE(outer) \
2581 dotest(RCU_in_##outer, SUCCESS, LOCKTYPE_RWLOCK); \
2582 dotest(RAW_SPINLOCK_in_##outer, SUCCESS, LOCKTYPE_SPIN); \
2583 dotest(SPINLOCK_in_##outer, SUCCESS, LOCKTYPE_SPIN); \
2584 dotest(MUTEX_in_##outer, SUCCESS, LOCKTYPE_MUTEX); \
2585
2586
2587
2588
2589
2590#define DO_CONTEXT_TESTCASE_OUTER_LIMITED_PREEMPTIBLE(outer) \
2591 dotest(RCU_in_##outer, SUCCESS, LOCKTYPE_RWLOCK); \
2592 dotest(RAW_SPINLOCK_in_##outer, SUCCESS, LOCKTYPE_SPIN); \
2593 dotest(SPINLOCK_in_##outer, SUCCESS, LOCKTYPE_SPIN); \
2594 dotest(MUTEX_in_##outer, FAILURE, LOCKTYPE_MUTEX); \
2595
2596
2597#define DO_CONTEXT_TESTCASE_OUTER_NOT_PREEMPTIBLE(outer) \
2598 dotest(RCU_in_##outer, SUCCESS, LOCKTYPE_RWLOCK); \
2599 dotest(RAW_SPINLOCK_in_##outer, SUCCESS, LOCKTYPE_SPIN); \
2600 dotest(SPINLOCK_in_##outer, FAILURE, LOCKTYPE_SPIN); \
2601 dotest(MUTEX_in_##outer, FAILURE, LOCKTYPE_MUTEX); \
2602
2603static void wait_context_tests(void)
2604{
2605 printk(" --------------------------------------------------------------------------\n");
2606 printk(" | wait context tests |\n");
2607 printk(" --------------------------------------------------------------------------\n");
2608 printk(" | rcu | raw | spin |mutex |\n");
2609 printk(" --------------------------------------------------------------------------\n");
2610 print_testname("in hardirq context");
2611 DO_CONTEXT_TESTCASE_OUTER_LIMITED_PREEMPTIBLE(HARDIRQ);
2612 pr_cont("\n");
2613
2614 print_testname("in hardirq context (not threaded)");
2615 DO_CONTEXT_TESTCASE_OUTER_NOT_PREEMPTIBLE(NOTTHREADED_HARDIRQ);
2616 pr_cont("\n");
2617
2618 print_testname("in softirq context");
2619 DO_CONTEXT_TESTCASE_OUTER_LIMITED_PREEMPTIBLE(SOFTIRQ);
2620 pr_cont("\n");
2621
2622 print_testname("in RCU context");
2623 DO_CONTEXT_TESTCASE_OUTER_LIMITED_PREEMPTIBLE(RCU);
2624 pr_cont("\n");
2625
2626 print_testname("in RCU-bh context");
2627 DO_CONTEXT_TESTCASE_OUTER_LIMITED_PREEMPTIBLE(RCU_BH);
2628 pr_cont("\n");
2629
2630 print_testname("in RCU-sched context");
2631 DO_CONTEXT_TESTCASE_OUTER_NOT_PREEMPTIBLE(RCU_SCHED);
2632 pr_cont("\n");
2633
2634 print_testname("in RAW_SPINLOCK context");
2635 DO_CONTEXT_TESTCASE_OUTER_NOT_PREEMPTIBLE(RAW_SPINLOCK);
2636 pr_cont("\n");
2637
2638 print_testname("in SPINLOCK context");
2639 DO_CONTEXT_TESTCASE_OUTER_LIMITED_PREEMPTIBLE(SPINLOCK);
2640 pr_cont("\n");
2641
2642 print_testname("in MUTEX context");
2643 DO_CONTEXT_TESTCASE_OUTER_PREEMPTIBLE(MUTEX);
2644 pr_cont("\n");
2645}
2646
2647static void local_lock_2(void)
2648{
2649 local_lock_acquire(&local_A);
2650 local_lock_release(&local_A);
2651
2652 HARDIRQ_ENTER();
2653 spin_lock(&lock_A);
2654 spin_unlock(&lock_A);
2655 HARDIRQ_EXIT()
2656
2657 HARDIRQ_DISABLE();
2658 spin_lock(&lock_A);
2659 local_lock_acquire(&local_A);
2660 local_lock_release(&local_A);
2661 spin_unlock(&lock_A);
2662 HARDIRQ_ENABLE();
2663}
2664
2665static void local_lock_3A(void)
2666{
2667 local_lock_acquire(&local_A);
2668 spin_lock(&lock_B);
2669 spin_unlock(&lock_B);
2670 local_lock_release(&local_A);
2671
2672 HARDIRQ_ENTER();
2673 spin_lock(&lock_A);
2674 spin_unlock(&lock_A);
2675 HARDIRQ_EXIT()
2676
2677 HARDIRQ_DISABLE();
2678 spin_lock(&lock_A);
2679 local_lock_acquire(&local_A);
2680 local_lock_release(&local_A);
2681 spin_unlock(&lock_A);
2682 HARDIRQ_ENABLE();
2683}
2684
2685static void local_lock_3B(void)
2686{
2687 local_lock_acquire(&local_A);
2688 spin_lock(&lock_B);
2689 spin_unlock(&lock_B);
2690 local_lock_release(&local_A);
2691
2692 HARDIRQ_ENTER();
2693 spin_lock(&lock_A);
2694 spin_unlock(&lock_A);
2695 HARDIRQ_EXIT()
2696
2697 HARDIRQ_DISABLE();
2698 spin_lock(&lock_A);
2699 local_lock_acquire(&local_A);
2700 local_lock_release(&local_A);
2701 spin_unlock(&lock_A);
2702 HARDIRQ_ENABLE();
2703
2704 HARDIRQ_DISABLE();
2705 spin_lock(&lock_A);
2706 spin_lock(&lock_B);
2707 spin_unlock(&lock_B);
2708 spin_unlock(&lock_A);
2709 HARDIRQ_DISABLE();
2710
2711}
2712
2713static void local_lock_tests(void)
2714{
2715 printk(" --------------------------------------------------------------------------\n");
2716 printk(" | local_lock tests |\n");
2717 printk(" ---------------------\n");
2718
2719 print_testname("local_lock inversion 2");
2720 dotest(local_lock_2, SUCCESS, LOCKTYPE_LL);
2721 pr_cont("\n");
2722
2723 print_testname("local_lock inversion 3A");
2724 dotest(local_lock_3A, SUCCESS, LOCKTYPE_LL);
2725 pr_cont("\n");
2726
2727 print_testname("local_lock inversion 3B");
2728 dotest(local_lock_3B, FAILURE, LOCKTYPE_LL);
2729 pr_cont("\n");
2730}
2731
2732static void hardirq_deadlock_softirq_not_deadlock(void)
2733{
2734
2735
2736 mutex_lock(&mutex_A);
2737 HARDIRQ_DISABLE();
2738 spin_lock(&lock_C);
2739 spin_unlock(&lock_C);
2740 HARDIRQ_ENABLE();
2741 mutex_unlock(&mutex_A);
2742
2743
2744 HARDIRQ_ENTER();
2745 spin_lock(&lock_A);
2746 spin_unlock(&lock_A);
2747 HARDIRQ_EXIT();
2748
2749
2750 HARDIRQ_DISABLE();
2751 spin_lock(&lock_A);
2752 spin_lock(&lock_B);
2753 spin_unlock(&lock_B);
2754 spin_unlock(&lock_A);
2755 HARDIRQ_ENABLE();
2756
2757
2758 HARDIRQ_DISABLE();
2759 spin_lock(&lock_B);
2760 spin_lock(&lock_C);
2761 spin_unlock(&lock_C);
2762 spin_unlock(&lock_B);
2763 HARDIRQ_ENABLE();
2764
2765
2766 SOFTIRQ_ENTER();
2767 spin_lock(&lock_D);
2768 spin_unlock(&lock_D);
2769 SOFTIRQ_EXIT();
2770
2771
2772 SOFTIRQ_DISABLE();
2773 spin_lock(&lock_D);
2774 spin_unlock(&lock_D);
2775 SOFTIRQ_ENABLE();
2776
2777
2778
2779
2780
2781
2782
2783
2784 HARDIRQ_DISABLE();
2785 spin_lock(&lock_C);
2786 spin_lock(&lock_D);
2787 spin_unlock(&lock_D);
2788 spin_unlock(&lock_C);
2789 HARDIRQ_ENABLE();
2790}
2791
2792void locking_selftest(void)
2793{
2794
2795
2796
2797 if (!debug_locks) {
2798 printk("----------------------------------\n");
2799 printk("| Locking API testsuite disabled |\n");
2800 printk("----------------------------------\n");
2801 return;
2802 }
2803
2804
2805
2806
2807 force_read_lock_recursive = 1;
2808
2809
2810
2811
2812 printk("------------------------\n");
2813 printk("| Locking API testsuite:\n");
2814 printk("----------------------------------------------------------------------------\n");
2815 printk(" | spin |wlock |rlock |mutex | wsem | rsem |\n");
2816 printk(" --------------------------------------------------------------------------\n");
2817
2818 init_shared_classes();
2819 lockdep_set_selftest_task(current);
2820
2821 DO_TESTCASE_6R("A-A deadlock", AA);
2822 DO_TESTCASE_6R("A-B-B-A deadlock", ABBA);
2823 DO_TESTCASE_6R("A-B-B-C-C-A deadlock", ABBCCA);
2824 DO_TESTCASE_6R("A-B-C-A-B-C deadlock", ABCABC);
2825 DO_TESTCASE_6R("A-B-B-C-C-D-D-A deadlock", ABBCCDDA);
2826 DO_TESTCASE_6R("A-B-C-D-B-D-D-A deadlock", ABCDBDDA);
2827 DO_TESTCASE_6R("A-B-C-D-B-C-D-A deadlock", ABCDBCDA);
2828 DO_TESTCASE_6("double unlock", double_unlock);
2829 DO_TESTCASE_6("initialize held", init_held);
2830
2831 printk(" --------------------------------------------------------------------------\n");
2832 print_testname("recursive read-lock");
2833 pr_cont(" |");
2834 dotest(rlock_AA1, SUCCESS, LOCKTYPE_RWLOCK);
2835 pr_cont(" |");
2836 dotest(rsem_AA1, FAILURE, LOCKTYPE_RWSEM);
2837 pr_cont("\n");
2838
2839 print_testname("recursive read-lock #2");
2840 pr_cont(" |");
2841 dotest(rlock_AA1B, SUCCESS, LOCKTYPE_RWLOCK);
2842 pr_cont(" |");
2843 dotest(rsem_AA1B, FAILURE, LOCKTYPE_RWSEM);
2844 pr_cont("\n");
2845
2846 print_testname("mixed read-write-lock");
2847 pr_cont(" |");
2848 dotest(rlock_AA2, FAILURE, LOCKTYPE_RWLOCK);
2849 pr_cont(" |");
2850 dotest(rsem_AA2, FAILURE, LOCKTYPE_RWSEM);
2851 pr_cont("\n");
2852
2853 print_testname("mixed write-read-lock");
2854 pr_cont(" |");
2855 dotest(rlock_AA3, FAILURE, LOCKTYPE_RWLOCK);
2856 pr_cont(" |");
2857 dotest(rsem_AA3, FAILURE, LOCKTYPE_RWSEM);
2858 pr_cont("\n");
2859
2860 print_testname("mixed read-lock/lock-write ABBA");
2861 pr_cont(" |");
2862 dotest(rlock_ABBA1, FAILURE, LOCKTYPE_RWLOCK);
2863 pr_cont(" |");
2864 dotest(rwsem_ABBA1, FAILURE, LOCKTYPE_RWSEM);
2865
2866 print_testname("mixed read-lock/lock-read ABBA");
2867 pr_cont(" |");
2868 dotest(rlock_ABBA2, SUCCESS, LOCKTYPE_RWLOCK);
2869 pr_cont(" |");
2870 dotest(rwsem_ABBA2, FAILURE, LOCKTYPE_RWSEM);
2871
2872 print_testname("mixed write-lock/lock-write ABBA");
2873 pr_cont(" |");
2874 dotest(rlock_ABBA3, FAILURE, LOCKTYPE_RWLOCK);
2875 pr_cont(" |");
2876 dotest(rwsem_ABBA3, FAILURE, LOCKTYPE_RWSEM);
2877
2878 print_testname("chain cached mixed R-L/L-W ABBA");
2879 pr_cont(" |");
2880 dotest(rlock_chaincache_ABBA1, FAILURE, LOCKTYPE_RWLOCK);
2881
2882 DO_TESTCASE_6x1RRB("rlock W1R2/W2R3/W3R1", W1R2_W2R3_W3R1);
2883 DO_TESTCASE_6x1RRB("rlock W1W2/R2R3/W3R1", W1W2_R2R3_W3R1);
2884 DO_TESTCASE_6x1RR("rlock W1W2/R2R3/R3W1", W1W2_R2R3_R3W1);
2885 DO_TESTCASE_6x1RR("rlock W1R2/R2R3/W3W1", W1R2_R2R3_W3W1);
2886
2887 printk(" --------------------------------------------------------------------------\n");
2888
2889
2890
2891
2892 DO_TESTCASE_2x6("irqs-on + irq-safe-A", irqsafe1);
2893 DO_TESTCASE_2x3("sirq-safe-A => hirqs-on", irqsafe2A);
2894 DO_TESTCASE_2x6("safe-A + irqs-on", irqsafe2B);
2895 DO_TESTCASE_6x6("safe-A + unsafe-B #1", irqsafe3);
2896 DO_TESTCASE_6x6("safe-A + unsafe-B #2", irqsafe4);
2897 DO_TESTCASE_6x6RW("irq lock-inversion", irq_inversion);
2898
2899 DO_TESTCASE_6x2x2RW("irq read-recursion", irq_read_recursion);
2900 DO_TESTCASE_6x2x2RW("irq read-recursion #2", irq_read_recursion2);
2901 DO_TESTCASE_6x2x2RW("irq read-recursion #3", irq_read_recursion3);
2902
2903 ww_tests();
2904
2905 force_read_lock_recursive = 0;
2906
2907
2908
2909 if (IS_ENABLED(CONFIG_QUEUED_RWLOCKS))
2910 queued_read_lock_tests();
2911
2912 fs_reclaim_tests();
2913
2914
2915 if (IS_ENABLED(CONFIG_PROVE_RAW_LOCK_NESTING))
2916 wait_context_tests();
2917
2918 local_lock_tests();
2919
2920 print_testname("hardirq_unsafe_softirq_safe");
2921 dotest(hardirq_deadlock_softirq_not_deadlock, FAILURE, LOCKTYPE_SPECIAL);
2922 pr_cont("\n");
2923
2924 if (unexpected_testcase_failures) {
2925 printk("-----------------------------------------------------------------\n");
2926 debug_locks = 0;
2927 printk("BUG: %3d unexpected failures (out of %3d) - debugging disabled! |\n",
2928 unexpected_testcase_failures, testcase_total);
2929 printk("-----------------------------------------------------------------\n");
2930 } else if (expected_testcase_failures && testcase_successes) {
2931 printk("--------------------------------------------------------\n");
2932 printk("%3d out of %3d testcases failed, as expected. |\n",
2933 expected_testcase_failures, testcase_total);
2934 printk("----------------------------------------------------\n");
2935 debug_locks = 1;
2936 } else if (expected_testcase_failures && !testcase_successes) {
2937 printk("--------------------------------------------------------\n");
2938 printk("All %3d testcases failed, as expected. |\n",
2939 expected_testcase_failures);
2940 printk("----------------------------------------\n");
2941 debug_locks = 1;
2942 } else {
2943 printk("-------------------------------------------------------\n");
2944 printk("Good, all %3d testcases passed! |\n",
2945 testcase_successes);
2946 printk("---------------------------------\n");
2947 debug_locks = 1;
2948 }
2949 lockdep_set_selftest_task(NULL);
2950 debug_locks_silent = 0;
2951}
2952