1
2
3
4
5
6
7
8
9
10
11
12
13
14
15#include <linux/spinlock.h>
16#include <linux/module.h>
17#include <asm/processor.h>
18
19#include "spinlock_common.h"
20
21void arch_spin_lock(arch_spinlock_t *lock)
22{
23 int my_ticket;
24 int iterations = 0;
25 int delta;
26
27 while ((my_ticket = __insn_tns((void *)&lock->next_ticket)) & 1)
28 delay_backoff(iterations++);
29
30
31 lock->next_ticket = my_ticket + TICKET_QUANTUM;
32
33
34 while ((delta = my_ticket - lock->current_ticket) != 0)
35 relax((128 / CYCLES_PER_RELAX_LOOP) * delta);
36}
37EXPORT_SYMBOL(arch_spin_lock);
38
39int arch_spin_trylock(arch_spinlock_t *lock)
40{
41
42
43
44
45
46 int my_ticket = __insn_tns((void *)&lock->next_ticket);
47
48 if (my_ticket == lock->current_ticket) {
49
50 lock->next_ticket = my_ticket + TICKET_QUANTUM;
51
52 return 1;
53 }
54
55 if (!(my_ticket & 1)) {
56
57 lock->next_ticket = my_ticket;
58 }
59
60 return 0;
61}
62EXPORT_SYMBOL(arch_spin_trylock);
63
64void arch_spin_unlock_wait(arch_spinlock_t *lock)
65{
66 u32 iterations = 0;
67 while (arch_spin_is_locked(lock))
68 delay_backoff(iterations++);
69}
70EXPORT_SYMBOL(arch_spin_unlock_wait);
71
72
73
74
75
76
77
78
79
80#define WR_NEXT_SHIFT _WR_NEXT_SHIFT
81#define WR_CURR_SHIFT _WR_CURR_SHIFT
82#define WR_WIDTH _WR_WIDTH
83#define WR_MASK ((1 << WR_WIDTH) - 1)
84
85
86
87
88
89#define RD_COUNT_SHIFT _RD_COUNT_SHIFT
90#define RD_COUNT_WIDTH _RD_COUNT_WIDTH
91#define RD_COUNT_MASK ((1 << RD_COUNT_WIDTH) - 1)
92
93
94
95static inline u32 get_rwlock(arch_rwlock_t *rwlock)
96{
97 u32 iterations = 0;
98 for (;;) {
99 u32 val = __insn_tns((int *)&rwlock->lock);
100 if (unlikely(val & 1)) {
101 delay_backoff(iterations++);
102 continue;
103 }
104 return val;
105 }
106}
107
108int arch_read_trylock_slow(arch_rwlock_t *rwlock)
109{
110 u32 val = get_rwlock(rwlock);
111 int locked = (val << RD_COUNT_WIDTH) == 0;
112 rwlock->lock = val + (locked << RD_COUNT_SHIFT);
113 return locked;
114}
115EXPORT_SYMBOL(arch_read_trylock_slow);
116
117void arch_read_unlock_slow(arch_rwlock_t *rwlock)
118{
119 u32 val = get_rwlock(rwlock);
120 rwlock->lock = val - (1 << RD_COUNT_SHIFT);
121}
122EXPORT_SYMBOL(arch_read_unlock_slow);
123
124void arch_write_unlock_slow(arch_rwlock_t *rwlock, u32 val)
125{
126 u32 eq, mask = 1 << WR_CURR_SHIFT;
127 while (unlikely(val & 1)) {
128
129 relax(4);
130 val = __insn_tns((int *)&rwlock->lock);
131 }
132 val = __insn_addb(val, mask);
133 eq = __insn_seqb(val, val << (WR_CURR_SHIFT - WR_NEXT_SHIFT));
134 val = __insn_mz(eq & mask, val);
135 rwlock->lock = val;
136}
137EXPORT_SYMBOL(arch_write_unlock_slow);
138
139
140
141
142
143
144
145
146
147
148
149void arch_read_lock_slow(arch_rwlock_t *rwlock, u32 val)
150{
151 u32 iterations = 0;
152 do {
153 if (!(val & 1))
154 rwlock->lock = val;
155 delay_backoff(iterations++);
156 val = __insn_tns((int *)&rwlock->lock);
157 } while ((val << RD_COUNT_WIDTH) != 0);
158 rwlock->lock = val + (1 << RD_COUNT_SHIFT);
159}
160EXPORT_SYMBOL(arch_read_lock_slow);
161
162void arch_write_lock_slow(arch_rwlock_t *rwlock, u32 val)
163{
164
165
166
167
168
169 u32 my_ticket_;
170 u32 iterations = 0;
171
172
173
174
175
176 for (;;) {
177 if (!(val & 1)) {
178 if ((val >> RD_COUNT_SHIFT) == 0)
179 break;
180 rwlock->lock = val;
181 }
182 delay_backoff(iterations++);
183 val = __insn_tns((int *)&rwlock->lock);
184 }
185
186
187 rwlock->lock = __insn_addb(val, 1 << WR_NEXT_SHIFT);
188 my_ticket_ = val >> WR_NEXT_SHIFT;
189
190
191 for (;;) {
192 u32 curr_ = val >> WR_CURR_SHIFT;
193 u32 delta = ((my_ticket_ - curr_) & WR_MASK);
194 if (likely(delta == 0))
195 break;
196
197
198 relax((256 / CYCLES_PER_RELAX_LOOP) * delta);
199
200
201
202
203
204
205 while ((val = rwlock->lock) & 1)
206 relax(4);
207 }
208}
209EXPORT_SYMBOL(arch_write_lock_slow);
210
211int __tns_atomic_acquire(atomic_t *lock)
212{
213 int ret;
214 u32 iterations = 0;
215
216 BUG_ON(__insn_mfspr(SPR_INTERRUPT_CRITICAL_SECTION));
217 __insn_mtspr(SPR_INTERRUPT_CRITICAL_SECTION, 1);
218
219 while ((ret = __insn_tns((void *)&lock->counter)) == 1)
220 delay_backoff(iterations++);
221 return ret;
222}
223
224void __tns_atomic_release(atomic_t *p, int v)
225{
226 p->counter = v;
227 __insn_mtspr(SPR_INTERRUPT_CRITICAL_SECTION, 0);
228}
229