1
2
3
4
5
6
7
8
9#ifndef _ASM_SPINLOCK_H
10#define _ASM_SPINLOCK_H
11
12#include <linux/compiler.h>
13
14#include <asm/barrier.h>
15#include <asm/processor.h>
16#include <asm/compiler.h>
17#include <asm/war.h>
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39static inline int arch_spin_is_locked(arch_spinlock_t *lock)
40{
41 u32 counters = ACCESS_ONCE(lock->lock);
42
43 return ((counters >> 16) ^ counters) & 0xffff;
44}
45
46static inline int arch_spin_value_unlocked(arch_spinlock_t lock)
47{
48 return lock.h.serving_now == lock.h.ticket;
49}
50
51#define arch_spin_lock_flags(lock, flags) arch_spin_lock(lock)
52
53static inline void arch_spin_unlock_wait(arch_spinlock_t *lock)
54{
55 u16 owner = READ_ONCE(lock->h.serving_now);
56 smp_rmb();
57 for (;;) {
58 arch_spinlock_t tmp = READ_ONCE(*lock);
59
60 if (tmp.h.serving_now == tmp.h.ticket ||
61 tmp.h.serving_now != owner)
62 break;
63
64 cpu_relax();
65 }
66 smp_acquire__after_ctrl_dep();
67}
68
69static inline int arch_spin_is_contended(arch_spinlock_t *lock)
70{
71 u32 counters = ACCESS_ONCE(lock->lock);
72
73 return (((counters >> 16) - counters) & 0xffff) > 1;
74}
75#define arch_spin_is_contended arch_spin_is_contended
76
77static inline void arch_spin_lock(arch_spinlock_t *lock)
78{
79 int my_ticket;
80 int tmp;
81 int inc = 0x10000;
82
83 if (R10000_LLSC_WAR) {
84 __asm__ __volatile__ (
85 " .set push # arch_spin_lock \n"
86 " .set noreorder \n"
87 " \n"
88 "1: ll %[ticket], %[ticket_ptr] \n"
89 " addu %[my_ticket], %[ticket], %[inc] \n"
90 " sc %[my_ticket], %[ticket_ptr] \n"
91 " beqzl %[my_ticket], 1b \n"
92 " nop \n"
93 " srl %[my_ticket], %[ticket], 16 \n"
94 " andi %[ticket], %[ticket], 0xffff \n"
95 " bne %[ticket], %[my_ticket], 4f \n"
96 " subu %[ticket], %[my_ticket], %[ticket] \n"
97 "2: \n"
98 " .subsection 2 \n"
99 "4: andi %[ticket], %[ticket], 0xffff \n"
100 " sll %[ticket], 5 \n"
101 " \n"
102 "6: bnez %[ticket], 6b \n"
103 " subu %[ticket], 1 \n"
104 " \n"
105 " lhu %[ticket], %[serving_now_ptr] \n"
106 " beq %[ticket], %[my_ticket], 2b \n"
107 " subu %[ticket], %[my_ticket], %[ticket] \n"
108 " b 4b \n"
109 " subu %[ticket], %[ticket], 1 \n"
110 " .previous \n"
111 " .set pop \n"
112 : [ticket_ptr] "+" GCC_OFF_SMALL_ASM() (lock->lock),
113 [serving_now_ptr] "+m" (lock->h.serving_now),
114 [ticket] "=&r" (tmp),
115 [my_ticket] "=&r" (my_ticket)
116 : [inc] "r" (inc));
117 } else {
118 __asm__ __volatile__ (
119 " .set push # arch_spin_lock \n"
120 " .set noreorder \n"
121 " \n"
122 "1: ll %[ticket], %[ticket_ptr] \n"
123 " addu %[my_ticket], %[ticket], %[inc] \n"
124 " sc %[my_ticket], %[ticket_ptr] \n"
125 " beqz %[my_ticket], 1b \n"
126 " srl %[my_ticket], %[ticket], 16 \n"
127 " andi %[ticket], %[ticket], 0xffff \n"
128 " bne %[ticket], %[my_ticket], 4f \n"
129 " subu %[ticket], %[my_ticket], %[ticket] \n"
130 "2: .insn \n"
131 " .subsection 2 \n"
132 "4: andi %[ticket], %[ticket], 0xffff \n"
133 " sll %[ticket], 5 \n"
134 " \n"
135 "6: bnez %[ticket], 6b \n"
136 " subu %[ticket], 1 \n"
137 " \n"
138 " lhu %[ticket], %[serving_now_ptr] \n"
139 " beq %[ticket], %[my_ticket], 2b \n"
140 " subu %[ticket], %[my_ticket], %[ticket] \n"
141 " b 4b \n"
142 " subu %[ticket], %[ticket], 1 \n"
143 " .previous \n"
144 " .set pop \n"
145 : [ticket_ptr] "+" GCC_OFF_SMALL_ASM() (lock->lock),
146 [serving_now_ptr] "+m" (lock->h.serving_now),
147 [ticket] "=&r" (tmp),
148 [my_ticket] "=&r" (my_ticket)
149 : [inc] "r" (inc));
150 }
151
152 smp_llsc_mb();
153}
154
155static inline void arch_spin_unlock(arch_spinlock_t *lock)
156{
157 unsigned int serving_now = lock->h.serving_now + 1;
158 wmb();
159 lock->h.serving_now = (u16)serving_now;
160 nudge_writes();
161}
162
163static inline unsigned int arch_spin_trylock(arch_spinlock_t *lock)
164{
165 int tmp, tmp2, tmp3;
166 int inc = 0x10000;
167
168 if (R10000_LLSC_WAR) {
169 __asm__ __volatile__ (
170 " .set push # arch_spin_trylock \n"
171 " .set noreorder \n"
172 " \n"
173 "1: ll %[ticket], %[ticket_ptr] \n"
174 " srl %[my_ticket], %[ticket], 16 \n"
175 " andi %[now_serving], %[ticket], 0xffff \n"
176 " bne %[my_ticket], %[now_serving], 3f \n"
177 " addu %[ticket], %[ticket], %[inc] \n"
178 " sc %[ticket], %[ticket_ptr] \n"
179 " beqzl %[ticket], 1b \n"
180 " li %[ticket], 1 \n"
181 "2: \n"
182 " .subsection 2 \n"
183 "3: b 2b \n"
184 " li %[ticket], 0 \n"
185 " .previous \n"
186 " .set pop \n"
187 : [ticket_ptr] "+" GCC_OFF_SMALL_ASM() (lock->lock),
188 [ticket] "=&r" (tmp),
189 [my_ticket] "=&r" (tmp2),
190 [now_serving] "=&r" (tmp3)
191 : [inc] "r" (inc));
192 } else {
193 __asm__ __volatile__ (
194 " .set push # arch_spin_trylock \n"
195 " .set noreorder \n"
196 " \n"
197 "1: ll %[ticket], %[ticket_ptr] \n"
198 " srl %[my_ticket], %[ticket], 16 \n"
199 " andi %[now_serving], %[ticket], 0xffff \n"
200 " bne %[my_ticket], %[now_serving], 3f \n"
201 " addu %[ticket], %[ticket], %[inc] \n"
202 " sc %[ticket], %[ticket_ptr] \n"
203 " beqz %[ticket], 1b \n"
204 " li %[ticket], 1 \n"
205 "2: .insn \n"
206 " .subsection 2 \n"
207 "3: b 2b \n"
208 " li %[ticket], 0 \n"
209 " .previous \n"
210 " .set pop \n"
211 : [ticket_ptr] "+" GCC_OFF_SMALL_ASM() (lock->lock),
212 [ticket] "=&r" (tmp),
213 [my_ticket] "=&r" (tmp2),
214 [now_serving] "=&r" (tmp3)
215 : [inc] "r" (inc));
216 }
217
218 smp_llsc_mb();
219
220 return tmp;
221}
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236#define arch_read_can_lock(rw) ((rw)->lock >= 0)
237
238
239
240
241
242#define arch_write_can_lock(rw) (!(rw)->lock)
243
244static inline void arch_read_lock(arch_rwlock_t *rw)
245{
246 unsigned int tmp;
247
248 if (R10000_LLSC_WAR) {
249 __asm__ __volatile__(
250 " .set noreorder # arch_read_lock \n"
251 "1: ll %1, %2 \n"
252 " bltz %1, 1b \n"
253 " addu %1, 1 \n"
254 " sc %1, %0 \n"
255 " beqzl %1, 1b \n"
256 " nop \n"
257 " .set reorder \n"
258 : "=" GCC_OFF_SMALL_ASM() (rw->lock), "=&r" (tmp)
259 : GCC_OFF_SMALL_ASM() (rw->lock)
260 : "memory");
261 } else {
262 do {
263 __asm__ __volatile__(
264 "1: ll %1, %2 # arch_read_lock \n"
265 " bltz %1, 1b \n"
266 " addu %1, 1 \n"
267 "2: sc %1, %0 \n"
268 : "=" GCC_OFF_SMALL_ASM() (rw->lock), "=&r" (tmp)
269 : GCC_OFF_SMALL_ASM() (rw->lock)
270 : "memory");
271 } while (unlikely(!tmp));
272 }
273
274 smp_llsc_mb();
275}
276
277static inline void arch_read_unlock(arch_rwlock_t *rw)
278{
279 unsigned int tmp;
280
281 smp_mb__before_llsc();
282
283 if (R10000_LLSC_WAR) {
284 __asm__ __volatile__(
285 "1: ll %1, %2 # arch_read_unlock \n"
286 " addiu %1, -1 \n"
287 " sc %1, %0 \n"
288 " beqzl %1, 1b \n"
289 : "=" GCC_OFF_SMALL_ASM() (rw->lock), "=&r" (tmp)
290 : GCC_OFF_SMALL_ASM() (rw->lock)
291 : "memory");
292 } else {
293 do {
294 __asm__ __volatile__(
295 "1: ll %1, %2 # arch_read_unlock \n"
296 " addiu %1, -1 \n"
297 " sc %1, %0 \n"
298 : "=" GCC_OFF_SMALL_ASM() (rw->lock), "=&r" (tmp)
299 : GCC_OFF_SMALL_ASM() (rw->lock)
300 : "memory");
301 } while (unlikely(!tmp));
302 }
303}
304
305static inline void arch_write_lock(arch_rwlock_t *rw)
306{
307 unsigned int tmp;
308
309 if (R10000_LLSC_WAR) {
310 __asm__ __volatile__(
311 " .set noreorder # arch_write_lock \n"
312 "1: ll %1, %2 \n"
313 " bnez %1, 1b \n"
314 " lui %1, 0x8000 \n"
315 " sc %1, %0 \n"
316 " beqzl %1, 1b \n"
317 " nop \n"
318 " .set reorder \n"
319 : "=" GCC_OFF_SMALL_ASM() (rw->lock), "=&r" (tmp)
320 : GCC_OFF_SMALL_ASM() (rw->lock)
321 : "memory");
322 } else {
323 do {
324 __asm__ __volatile__(
325 "1: ll %1, %2 # arch_write_lock \n"
326 " bnez %1, 1b \n"
327 " lui %1, 0x8000 \n"
328 "2: sc %1, %0 \n"
329 : "=" GCC_OFF_SMALL_ASM() (rw->lock), "=&r" (tmp)
330 : GCC_OFF_SMALL_ASM() (rw->lock)
331 : "memory");
332 } while (unlikely(!tmp));
333 }
334
335 smp_llsc_mb();
336}
337
338static inline void arch_write_unlock(arch_rwlock_t *rw)
339{
340 smp_mb__before_llsc();
341
342 __asm__ __volatile__(
343 " # arch_write_unlock \n"
344 " sw $0, %0 \n"
345 : "=m" (rw->lock)
346 : "m" (rw->lock)
347 : "memory");
348}
349
350static inline int arch_read_trylock(arch_rwlock_t *rw)
351{
352 unsigned int tmp;
353 int ret;
354
355 if (R10000_LLSC_WAR) {
356 __asm__ __volatile__(
357 " .set noreorder # arch_read_trylock \n"
358 " li %2, 0 \n"
359 "1: ll %1, %3 \n"
360 " bltz %1, 2f \n"
361 " addu %1, 1 \n"
362 " sc %1, %0 \n"
363 " .set reorder \n"
364 " beqzl %1, 1b \n"
365 " nop \n"
366 __WEAK_LLSC_MB
367 " li %2, 1 \n"
368 "2: \n"
369 : "=" GCC_OFF_SMALL_ASM() (rw->lock), "=&r" (tmp), "=&r" (ret)
370 : GCC_OFF_SMALL_ASM() (rw->lock)
371 : "memory");
372 } else {
373 __asm__ __volatile__(
374 " .set noreorder # arch_read_trylock \n"
375 " li %2, 0 \n"
376 "1: ll %1, %3 \n"
377 " bltz %1, 2f \n"
378 " addu %1, 1 \n"
379 " sc %1, %0 \n"
380 " beqz %1, 1b \n"
381 " nop \n"
382 " .set reorder \n"
383 __WEAK_LLSC_MB
384 " li %2, 1 \n"
385 "2: .insn \n"
386 : "=" GCC_OFF_SMALL_ASM() (rw->lock), "=&r" (tmp), "=&r" (ret)
387 : GCC_OFF_SMALL_ASM() (rw->lock)
388 : "memory");
389 }
390
391 return ret;
392}
393
394static inline int arch_write_trylock(arch_rwlock_t *rw)
395{
396 unsigned int tmp;
397 int ret;
398
399 if (R10000_LLSC_WAR) {
400 __asm__ __volatile__(
401 " .set noreorder # arch_write_trylock \n"
402 " li %2, 0 \n"
403 "1: ll %1, %3 \n"
404 " bnez %1, 2f \n"
405 " lui %1, 0x8000 \n"
406 " sc %1, %0 \n"
407 " beqzl %1, 1b \n"
408 " nop \n"
409 __WEAK_LLSC_MB
410 " li %2, 1 \n"
411 " .set reorder \n"
412 "2: \n"
413 : "=" GCC_OFF_SMALL_ASM() (rw->lock), "=&r" (tmp), "=&r" (ret)
414 : GCC_OFF_SMALL_ASM() (rw->lock)
415 : "memory");
416 } else {
417 do {
418 __asm__ __volatile__(
419 " ll %1, %3 # arch_write_trylock \n"
420 " li %2, 0 \n"
421 " bnez %1, 2f \n"
422 " lui %1, 0x8000 \n"
423 " sc %1, %0 \n"
424 " li %2, 1 \n"
425 "2: .insn \n"
426 : "=" GCC_OFF_SMALL_ASM() (rw->lock), "=&r" (tmp),
427 "=&r" (ret)
428 : GCC_OFF_SMALL_ASM() (rw->lock)
429 : "memory");
430 } while (unlikely(!tmp));
431
432 smp_llsc_mb();
433 }
434
435 return ret;
436}
437
438#define arch_read_lock_flags(lock, flags) arch_read_lock(lock)
439#define arch_write_lock_flags(lock, flags) arch_write_lock(lock)
440
441#define arch_spin_relax(lock) cpu_relax()
442#define arch_read_relax(lock) cpu_relax()
443#define arch_write_relax(lock) cpu_relax()
444
445#endif
446
