1
2#ifndef __LINUX_SPINLOCK_H
3#define __LINUX_SPINLOCK_H
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50#include <linux/typecheck.h>
51#include <linux/preempt.h>
52#include <linux/linkage.h>
53#include <linux/compiler.h>
54#include <linux/irqflags.h>
55#include <linux/thread_info.h>
56#include <linux/kernel.h>
57#include <linux/stringify.h>
58#include <linux/bottom_half.h>
59#include <linux/lockdep.h>
60#include <asm/barrier.h>
61#include <asm/mmiowb.h>
62
63
64
65
66
67#define LOCK_SECTION_NAME ".text..lock."KBUILD_BASENAME
68
69#define LOCK_SECTION_START(extra) \
70 ".subsection 1\n\t" \
71 extra \
72 ".ifndef " LOCK_SECTION_NAME "\n\t" \
73 LOCK_SECTION_NAME ":\n\t" \
74 ".endif\n"
75
76#define LOCK_SECTION_END \
77 ".previous\n\t"
78
79#define __lockfunc __attribute__((section(".spinlock.text")))
80
81
82
83
84#include <linux/spinlock_types.h>
85
86
87
88
89#ifdef CONFIG_SMP
90# include <asm/spinlock.h>
91#else
92# include <linux/spinlock_up.h>
93#endif
94
95#ifdef CONFIG_DEBUG_SPINLOCK
96 extern void __raw_spin_lock_init(raw_spinlock_t *lock, const char *name,
97 struct lock_class_key *key, short inner);
98
99# define raw_spin_lock_init(lock) \
100do { \
101 static struct lock_class_key __key; \
102 \
103 __raw_spin_lock_init((lock), #lock, &__key, LD_WAIT_SPIN); \
104} while (0)
105
106#else
107# define raw_spin_lock_init(lock) \
108 do { *(lock) = __RAW_SPIN_LOCK_UNLOCKED(lock); } while (0)
109#endif
110
111#define raw_spin_is_locked(lock) arch_spin_is_locked(&(lock)->raw_lock)
112
113#ifdef arch_spin_is_contended
114#define raw_spin_is_contended(lock) arch_spin_is_contended(&(lock)->raw_lock)
115#else
116#define raw_spin_is_contended(lock) (((void)(lock), 0))
117#endif
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170#ifndef smp_mb__after_spinlock
171#define smp_mb__after_spinlock() do { } while (0)
172#endif
173
174#ifdef CONFIG_DEBUG_SPINLOCK
175 extern void do_raw_spin_lock(raw_spinlock_t *lock) __acquires(lock);
176#define do_raw_spin_lock_flags(lock, flags) do_raw_spin_lock(lock)
177 extern int do_raw_spin_trylock(raw_spinlock_t *lock);
178 extern void do_raw_spin_unlock(raw_spinlock_t *lock) __releases(lock);
179#else
180static inline void do_raw_spin_lock(raw_spinlock_t *lock) __acquires(lock)
181{
182 __acquire(lock);
183 arch_spin_lock(&lock->raw_lock);
184 mmiowb_spin_lock();
185}
186
187#ifndef arch_spin_lock_flags
188#define arch_spin_lock_flags(lock, flags) arch_spin_lock(lock)
189#endif
190
191static inline void
192do_raw_spin_lock_flags(raw_spinlock_t *lock, unsigned long *flags) __acquires(lock)
193{
194 __acquire(lock);
195 arch_spin_lock_flags(&lock->raw_lock, *flags);
196 mmiowb_spin_lock();
197}
198
199static inline int do_raw_spin_trylock(raw_spinlock_t *lock)
200{
201 int ret = arch_spin_trylock(&(lock)->raw_lock);
202
203 if (ret)
204 mmiowb_spin_lock();
205
206 return ret;
207}
208
209static inline void do_raw_spin_unlock(raw_spinlock_t *lock) __releases(lock)
210{
211 mmiowb_spin_unlock();
212 arch_spin_unlock(&lock->raw_lock);
213 __release(lock);
214}
215#endif
216
217
218
219
220
221
222
223#define raw_spin_trylock(lock) __cond_lock(lock, _raw_spin_trylock(lock))
224
225#define raw_spin_lock(lock) _raw_spin_lock(lock)
226
227#ifdef CONFIG_DEBUG_LOCK_ALLOC
228# define raw_spin_lock_nested(lock, subclass) \
229 _raw_spin_lock_nested(lock, subclass)
230
231# define raw_spin_lock_nest_lock(lock, nest_lock) \
232 do { \
233 typecheck(struct lockdep_map *, &(nest_lock)->dep_map);\
234 _raw_spin_lock_nest_lock(lock, &(nest_lock)->dep_map); \
235 } while (0)
236#else
237
238
239
240
241
242# define raw_spin_lock_nested(lock, subclass) \
243 _raw_spin_lock(((void)(subclass), (lock)))
244# define raw_spin_lock_nest_lock(lock, nest_lock) _raw_spin_lock(lock)
245#endif
246
247#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
248
249#define raw_spin_lock_irqsave(lock, flags) \
250 do { \
251 typecheck(unsigned long, flags); \
252 flags = _raw_spin_lock_irqsave(lock); \
253 } while (0)
254
255#ifdef CONFIG_DEBUG_LOCK_ALLOC
256#define raw_spin_lock_irqsave_nested(lock, flags, subclass) \
257 do { \
258 typecheck(unsigned long, flags); \
259 flags = _raw_spin_lock_irqsave_nested(lock, subclass); \
260 } while (0)
261#else
262#define raw_spin_lock_irqsave_nested(lock, flags, subclass) \
263 do { \
264 typecheck(unsigned long, flags); \
265 flags = _raw_spin_lock_irqsave(lock); \
266 } while (0)
267#endif
268
269#else
270
271#define raw_spin_lock_irqsave(lock, flags) \
272 do { \
273 typecheck(unsigned long, flags); \
274 _raw_spin_lock_irqsave(lock, flags); \
275 } while (0)
276
277#define raw_spin_lock_irqsave_nested(lock, flags, subclass) \
278 raw_spin_lock_irqsave(lock, flags)
279
280#endif
281
282#define raw_spin_lock_irq(lock) _raw_spin_lock_irq(lock)
283#define raw_spin_lock_bh(lock) _raw_spin_lock_bh(lock)
284#define raw_spin_unlock(lock) _raw_spin_unlock(lock)
285#define raw_spin_unlock_irq(lock) _raw_spin_unlock_irq(lock)
286
287#define raw_spin_unlock_irqrestore(lock, flags) \
288 do { \
289 typecheck(unsigned long, flags); \
290 _raw_spin_unlock_irqrestore(lock, flags); \
291 } while (0)
292#define raw_spin_unlock_bh(lock) _raw_spin_unlock_bh(lock)
293
294#define raw_spin_trylock_bh(lock) \
295 __cond_lock(lock, _raw_spin_trylock_bh(lock))
296
297#define raw_spin_trylock_irq(lock) \
298({ \
299 local_irq_disable(); \
300 raw_spin_trylock(lock) ? \
301 1 : ({ local_irq_enable(); 0; }); \
302})
303
304#define raw_spin_trylock_irqsave(lock, flags) \
305({ \
306 local_irq_save(flags); \
307 raw_spin_trylock(lock) ? \
308 1 : ({ local_irq_restore(flags); 0; }); \
309})
310
311
312#include <linux/rwlock.h>
313
314
315
316
317#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
318# include <linux/spinlock_api_smp.h>
319#else
320# include <linux/spinlock_api_up.h>
321#endif
322
323
324
325
326
327static __always_inline raw_spinlock_t *spinlock_check(spinlock_t *lock)
328{
329 return &lock->rlock;
330}
331
332#ifdef CONFIG_DEBUG_SPINLOCK
333
334# define spin_lock_init(lock) \
335do { \
336 static struct lock_class_key __key; \
337 \
338 __raw_spin_lock_init(spinlock_check(lock), \
339 #lock, &__key, LD_WAIT_CONFIG); \
340} while (0)
341
342#else
343
344# define spin_lock_init(_lock) \
345do { \
346 spinlock_check(_lock); \
347 *(_lock) = __SPIN_LOCK_UNLOCKED(_lock); \
348} while (0)
349
350#endif
351
352static __always_inline void spin_lock(spinlock_t *lock)
353{
354 raw_spin_lock(&lock->rlock);
355}
356
357static __always_inline void spin_lock_bh(spinlock_t *lock)
358{
359 raw_spin_lock_bh(&lock->rlock);
360}
361
362static __always_inline int spin_trylock(spinlock_t *lock)
363{
364 return raw_spin_trylock(&lock->rlock);
365}
366
367#define spin_lock_nested(lock, subclass) \
368do { \
369 raw_spin_lock_nested(spinlock_check(lock), subclass); \
370} while (0)
371
372#define spin_lock_nest_lock(lock, nest_lock) \
373do { \
374 raw_spin_lock_nest_lock(spinlock_check(lock), nest_lock); \
375} while (0)
376
377static __always_inline void spin_lock_irq(spinlock_t *lock)
378{
379 raw_spin_lock_irq(&lock->rlock);
380}
381
382#define spin_lock_irqsave(lock, flags) \
383do { \
384 raw_spin_lock_irqsave(spinlock_check(lock), flags); \
385} while (0)
386
387#define spin_lock_irqsave_nested(lock, flags, subclass) \
388do { \
389 raw_spin_lock_irqsave_nested(spinlock_check(lock), flags, subclass); \
390} while (0)
391
392static __always_inline void spin_unlock(spinlock_t *lock)
393{
394 raw_spin_unlock(&lock->rlock);
395}
396
397static __always_inline void spin_unlock_bh(spinlock_t *lock)
398{
399 raw_spin_unlock_bh(&lock->rlock);
400}
401
402static __always_inline void spin_unlock_irq(spinlock_t *lock)
403{
404 raw_spin_unlock_irq(&lock->rlock);
405}
406
407static __always_inline void spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags)
408{
409 raw_spin_unlock_irqrestore(&lock->rlock, flags);
410}
411
412static __always_inline int spin_trylock_bh(spinlock_t *lock)
413{
414 return raw_spin_trylock_bh(&lock->rlock);
415}
416
417static __always_inline int spin_trylock_irq(spinlock_t *lock)
418{
419 return raw_spin_trylock_irq(&lock->rlock);
420}
421
422#define spin_trylock_irqsave(lock, flags) \
423({ \
424 raw_spin_trylock_irqsave(spinlock_check(lock), flags); \
425})
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445static __always_inline int spin_is_locked(spinlock_t *lock)
446{
447 return raw_spin_is_locked(&lock->rlock);
448}
449
450static __always_inline int spin_is_contended(spinlock_t *lock)
451{
452 return raw_spin_is_contended(&lock->rlock);
453}
454
455#define assert_spin_locked(lock) assert_raw_spin_locked(&(lock)->rlock)
456
457
458
459
460
461#include <linux/atomic.h>
462
463
464
465
466
467
468
469
470extern int _atomic_dec_and_lock(atomic_t *atomic, spinlock_t *lock);
471#define atomic_dec_and_lock(atomic, lock) \
472 __cond_lock(lock, _atomic_dec_and_lock(atomic, lock))
473
474extern int _atomic_dec_and_lock_irqsave(atomic_t *atomic, spinlock_t *lock,
475 unsigned long *flags);
476#define atomic_dec_and_lock_irqsave(atomic, lock, flags) \
477 __cond_lock(lock, _atomic_dec_and_lock_irqsave(atomic, lock, &(flags)))
478
479int __alloc_bucket_spinlocks(spinlock_t **locks, unsigned int *lock_mask,
480 size_t max_size, unsigned int cpu_mult,
481 gfp_t gfp, const char *name,
482 struct lock_class_key *key);
483
484#define alloc_bucket_spinlocks(locks, lock_mask, max_size, cpu_mult, gfp) \
485 ({ \
486 static struct lock_class_key key; \
487 int ret; \
488 \
489 ret = __alloc_bucket_spinlocks(locks, lock_mask, max_size, \
490 cpu_mult, gfp, #locks, &key); \
491 ret; \
492 })
493
494void free_bucket_spinlocks(spinlock_t *locks);
495
496#endif
497