1
2
3
4
5
6
7
8
9
10
11
12
13
14
15#include <linux/spinlock.h>
16#include <linux/module.h>
17#include <asm/processor.h>
18#include <arch/spr_def.h>
19
20#include "spinlock_common.h"
21
22void arch_spin_lock(arch_spinlock_t *lock)
23{
24 int my_ticket;
25 int iterations = 0;
26 int delta;
27
28 while ((my_ticket = __insn_tns((void *)&lock->next_ticket)) & 1)
29 delay_backoff(iterations++);
30
31
32 lock->next_ticket = my_ticket + TICKET_QUANTUM;
33
34
35 while ((delta = my_ticket - lock->current_ticket) != 0)
36 relax((128 / CYCLES_PER_RELAX_LOOP) * delta);
37}
38EXPORT_SYMBOL(arch_spin_lock);
39
40int arch_spin_trylock(arch_spinlock_t *lock)
41{
42
43
44
45
46
47 int my_ticket = __insn_tns((void *)&lock->next_ticket);
48
49 if (my_ticket == lock->current_ticket) {
50
51 lock->next_ticket = my_ticket + TICKET_QUANTUM;
52
53 return 1;
54 }
55
56 if (!(my_ticket & 1)) {
57
58 lock->next_ticket = my_ticket;
59 }
60
61 return 0;
62}
63EXPORT_SYMBOL(arch_spin_trylock);
64
65void arch_spin_unlock_wait(arch_spinlock_t *lock)
66{
67 u32 iterations = 0;
68 int curr = READ_ONCE(lock->current_ticket);
69 int next = READ_ONCE(lock->next_ticket);
70
71
72 if (next == curr)
73 return;
74
75
76 do {
77 delay_backoff(iterations++);
78 } while (READ_ONCE(lock->current_ticket) == curr);
79}
80EXPORT_SYMBOL(arch_spin_unlock_wait);
81
82
83
84
85
86
87
88
89
90#define WR_NEXT_SHIFT _WR_NEXT_SHIFT
91#define WR_CURR_SHIFT _WR_CURR_SHIFT
92#define WR_WIDTH _WR_WIDTH
93#define WR_MASK ((1 << WR_WIDTH) - 1)
94
95
96
97
98
99#define RD_COUNT_SHIFT _RD_COUNT_SHIFT
100#define RD_COUNT_WIDTH _RD_COUNT_WIDTH
101#define RD_COUNT_MASK ((1 << RD_COUNT_WIDTH) - 1)
102
103
104
105
106
107
108
109
110
111
112
113int arch_read_trylock(arch_rwlock_t *rwlock)
114{
115 u32 val;
116 __insn_mtspr(SPR_INTERRUPT_CRITICAL_SECTION, 1);
117 val = __insn_tns((int *)&rwlock->lock);
118 if (likely((val << _RD_COUNT_WIDTH) == 0)) {
119 val += 1 << RD_COUNT_SHIFT;
120 rwlock->lock = val;
121 __insn_mtspr(SPR_INTERRUPT_CRITICAL_SECTION, 0);
122 BUG_ON(val == 0);
123 return 1;
124 }
125 if ((val & 1) == 0)
126 rwlock->lock = val;
127 __insn_mtspr(SPR_INTERRUPT_CRITICAL_SECTION, 0);
128 return 0;
129}
130EXPORT_SYMBOL(arch_read_trylock);
131
132
133
134
135
136
137
138
139
140void arch_read_lock(arch_rwlock_t *rwlock)
141{
142 u32 iterations = 0;
143 while (unlikely(!arch_read_trylock(rwlock)))
144 delay_backoff(iterations++);
145}
146EXPORT_SYMBOL(arch_read_lock);
147
148void arch_read_unlock(arch_rwlock_t *rwlock)
149{
150 u32 val, iterations = 0;
151
152 mb();
153 for (;;) {
154 __insn_mtspr(SPR_INTERRUPT_CRITICAL_SECTION, 1);
155 val = __insn_tns((int *)&rwlock->lock);
156 if (likely((val & 1) == 0)) {
157 rwlock->lock = val - (1 << _RD_COUNT_SHIFT);
158 __insn_mtspr(SPR_INTERRUPT_CRITICAL_SECTION, 0);
159 break;
160 }
161 __insn_mtspr(SPR_INTERRUPT_CRITICAL_SECTION, 0);
162 delay_backoff(iterations++);
163 }
164}
165EXPORT_SYMBOL(arch_read_unlock);
166
167
168
169
170
171
172void arch_write_lock(arch_rwlock_t *rwlock)
173{
174
175
176
177
178
179 u32 my_ticket_;
180 u32 iterations = 0;
181 u32 val = __insn_tns((int *)&rwlock->lock);
182
183 if (likely(val == 0)) {
184 rwlock->lock = 1 << _WR_NEXT_SHIFT;
185 return;
186 }
187
188
189
190
191
192 for (;;) {
193 if (!(val & 1)) {
194 if ((val >> RD_COUNT_SHIFT) == 0)
195 break;
196 rwlock->lock = val;
197 }
198 delay_backoff(iterations++);
199 val = __insn_tns((int *)&rwlock->lock);
200 }
201
202
203 rwlock->lock = __insn_addb(val, 1 << WR_NEXT_SHIFT);
204 my_ticket_ = val >> WR_NEXT_SHIFT;
205
206
207 for (;;) {
208 u32 curr_ = val >> WR_CURR_SHIFT;
209 u32 delta = ((my_ticket_ - curr_) & WR_MASK);
210 if (likely(delta == 0))
211 break;
212
213
214 relax((256 / CYCLES_PER_RELAX_LOOP) * delta);
215
216
217
218
219
220
221 while ((val = rwlock->lock) & 1)
222 relax(4);
223 }
224}
225EXPORT_SYMBOL(arch_write_lock);
226
227int arch_write_trylock(arch_rwlock_t *rwlock)
228{
229 u32 val = __insn_tns((int *)&rwlock->lock);
230
231
232
233
234
235 if (unlikely(val != 0)) {
236 if (!(val & 1))
237 rwlock->lock = val;
238 return 0;
239 }
240
241
242 rwlock->lock = 1 << _WR_NEXT_SHIFT;
243 return 1;
244}
245EXPORT_SYMBOL(arch_write_trylock);
246
247void arch_write_unlock(arch_rwlock_t *rwlock)
248{
249 u32 val, eq, mask;
250
251 mb();
252 val = __insn_tns((int *)&rwlock->lock);
253 if (likely(val == (1 << _WR_NEXT_SHIFT))) {
254 rwlock->lock = 0;
255 return;
256 }
257 while (unlikely(val & 1)) {
258
259 relax(4);
260 val = __insn_tns((int *)&rwlock->lock);
261 }
262 mask = 1 << WR_CURR_SHIFT;
263 val = __insn_addb(val, mask);
264 eq = __insn_seqb(val, val << (WR_CURR_SHIFT - WR_NEXT_SHIFT));
265 val = __insn_mz(eq & mask, val);
266 rwlock->lock = val;
267}
268EXPORT_SYMBOL(arch_write_unlock);
269