1#ifndef __ARCH_S390_ATOMIC__
2#define __ARCH_S390_ATOMIC__
3
4
5
6
7
8
9
10
11
12
13
14
15
16#include <linux/compiler.h>
17#include <linux/types.h>
18
19#define ATOMIC_INIT(i) { (i) }
20
21#if __GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ > 2)
22
23#define __CS_LOOP(ptr, op_val, op_string) ({ \
24 typeof(ptr->counter) old_val, new_val; \
25 asm volatile( \
26 " l %0,%2\n" \
27 "0: lr %1,%0\n" \
28 op_string " %1,%3\n" \
29 " cs %0,%1,%2\n" \
30 " jl 0b" \
31 : "=&d" (old_val), "=&d" (new_val), \
32 "=Q" (((atomic_t *)(ptr))->counter) \
33 : "d" (op_val), "Q" (((atomic_t *)(ptr))->counter) \
34 : "cc", "memory"); \
35 new_val; \
36})
37
38#else
39
40#define __CS_LOOP(ptr, op_val, op_string) ({ \
41 typeof(ptr->counter) old_val, new_val; \
42 asm volatile( \
43 " l %0,0(%3)\n" \
44 "0: lr %1,%0\n" \
45 op_string " %1,%4\n" \
46 " cs %0,%1,0(%3)\n" \
47 " jl 0b" \
48 : "=&d" (old_val), "=&d" (new_val), \
49 "=m" (((atomic_t *)(ptr))->counter) \
50 : "a" (ptr), "d" (op_val), \
51 "m" (((atomic_t *)(ptr))->counter) \
52 : "cc", "memory"); \
53 new_val; \
54})
55
56#endif
57
58static inline int atomic_read(const atomic_t *v)
59{
60 barrier();
61 return v->counter;
62}
63
64static inline void atomic_set(atomic_t *v, int i)
65{
66 v->counter = i;
67 barrier();
68}
69
70static inline int atomic_add_return(int i, atomic_t *v)
71{
72 return __CS_LOOP(v, i, "ar");
73}
74#define atomic_add(_i, _v) atomic_add_return(_i, _v)
75#define atomic_add_negative(_i, _v) (atomic_add_return(_i, _v) < 0)
76#define atomic_inc(_v) atomic_add_return(1, _v)
77#define atomic_inc_return(_v) atomic_add_return(1, _v)
78#define atomic_inc_and_test(_v) (atomic_add_return(1, _v) == 0)
79
80static inline int atomic_sub_return(int i, atomic_t *v)
81{
82 return __CS_LOOP(v, i, "sr");
83}
84#define atomic_sub(_i, _v) atomic_sub_return(_i, _v)
85#define atomic_sub_and_test(_i, _v) (atomic_sub_return(_i, _v) == 0)
86#define atomic_dec(_v) atomic_sub_return(1, _v)
87#define atomic_dec_return(_v) atomic_sub_return(1, _v)
88#define atomic_dec_and_test(_v) (atomic_sub_return(1, _v) == 0)
89
90static inline void atomic_clear_mask(unsigned long mask, atomic_t *v)
91{
92 __CS_LOOP(v, ~mask, "nr");
93}
94
95static inline void atomic_set_mask(unsigned long mask, atomic_t *v)
96{
97 __CS_LOOP(v, mask, "or");
98}
99
100#define atomic_xchg(v, new) (xchg(&((v)->counter), new))
101
102static inline int atomic_cmpxchg(atomic_t *v, int old, int new)
103{
104#if __GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ > 2)
105 asm volatile(
106 " cs %0,%2,%1"
107 : "+d" (old), "=Q" (v->counter)
108 : "d" (new), "Q" (v->counter)
109 : "cc", "memory");
110#else
111 asm volatile(
112 " cs %0,%3,0(%2)"
113 : "+d" (old), "=m" (v->counter)
114 : "a" (v), "d" (new), "m" (v->counter)
115 : "cc", "memory");
116#endif
117 return old;
118}
119
120static inline int atomic_add_unless(atomic_t *v, int a, int u)
121{
122 int c, old;
123 c = atomic_read(v);
124 for (;;) {
125 if (unlikely(c == u))
126 break;
127 old = atomic_cmpxchg(v, c, c + a);
128 if (likely(old == c))
129 break;
130 c = old;
131 }
132 return c != u;
133}
134
135#define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0)
136
137#undef __CS_LOOP
138
139#define ATOMIC64_INIT(i) { (i) }
140
141#ifdef CONFIG_64BIT
142
143#if __GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ > 2)
144
145#define __CSG_LOOP(ptr, op_val, op_string) ({ \
146 typeof(ptr->counter) old_val, new_val; \
147 asm volatile( \
148 " lg %0,%2\n" \
149 "0: lgr %1,%0\n" \
150 op_string " %1,%3\n" \
151 " csg %0,%1,%2\n" \
152 " jl 0b" \
153 : "=&d" (old_val), "=&d" (new_val), \
154 "=Q" (((atomic_t *)(ptr))->counter) \
155 : "d" (op_val), "Q" (((atomic_t *)(ptr))->counter) \
156 : "cc", "memory"); \
157 new_val; \
158})
159
160#else
161
162#define __CSG_LOOP(ptr, op_val, op_string) ({ \
163 typeof(ptr->counter) old_val, new_val; \
164 asm volatile( \
165 " lg %0,0(%3)\n" \
166 "0: lgr %1,%0\n" \
167 op_string " %1,%4\n" \
168 " csg %0,%1,0(%3)\n" \
169 " jl 0b" \
170 : "=&d" (old_val), "=&d" (new_val), \
171 "=m" (((atomic_t *)(ptr))->counter) \
172 : "a" (ptr), "d" (op_val), \
173 "m" (((atomic_t *)(ptr))->counter) \
174 : "cc", "memory"); \
175 new_val; \
176})
177
178#endif
179
180static inline long long atomic64_read(const atomic64_t *v)
181{
182 barrier();
183 return v->counter;
184}
185
186static inline void atomic64_set(atomic64_t *v, long long i)
187{
188 v->counter = i;
189 barrier();
190}
191
192static inline long long atomic64_add_return(long long i, atomic64_t *v)
193{
194 return __CSG_LOOP(v, i, "agr");
195}
196
197static inline long long atomic64_sub_return(long long i, atomic64_t *v)
198{
199 return __CSG_LOOP(v, i, "sgr");
200}
201
202static inline void atomic64_clear_mask(unsigned long mask, atomic64_t *v)
203{
204 __CSG_LOOP(v, ~mask, "ngr");
205}
206
207static inline void atomic64_set_mask(unsigned long mask, atomic64_t *v)
208{
209 __CSG_LOOP(v, mask, "ogr");
210}
211
212#define atomic64_xchg(v, new) (xchg(&((v)->counter), new))
213
214static inline long long atomic64_cmpxchg(atomic64_t *v,
215 long long old, long long new)
216{
217#if __GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ > 2)
218 asm volatile(
219 " csg %0,%2,%1"
220 : "+d" (old), "=Q" (v->counter)
221 : "d" (new), "Q" (v->counter)
222 : "cc", "memory");
223#else
224 asm volatile(
225 " csg %0,%3,0(%2)"
226 : "+d" (old), "=m" (v->counter)
227 : "a" (v), "d" (new), "m" (v->counter)
228 : "cc", "memory");
229#endif
230 return old;
231}
232
233#undef __CSG_LOOP
234
235#else
236
237typedef struct {
238 long long counter;
239} atomic64_t;
240
241static inline long long atomic64_read(const atomic64_t *v)
242{
243 register_pair rp;
244
245 asm volatile(
246 " lm %0,%N0,0(%1)"
247 : "=&d" (rp)
248 : "a" (&v->counter), "m" (v->counter)
249 );
250 return rp.pair;
251}
252
253static inline void atomic64_set(atomic64_t *v, long long i)
254{
255 register_pair rp = {.pair = i};
256
257 asm volatile(
258 " stm %1,%N1,0(%2)"
259 : "=m" (v->counter)
260 : "d" (rp), "a" (&v->counter)
261 );
262}
263
264static inline long long atomic64_xchg(atomic64_t *v, long long new)
265{
266 register_pair rp_new = {.pair = new};
267 register_pair rp_old;
268
269 asm volatile(
270 " lm %0,%N0,0(%2)\n"
271 "0: cds %0,%3,0(%2)\n"
272 " jl 0b\n"
273 : "=&d" (rp_old), "+m" (v->counter)
274 : "a" (&v->counter), "d" (rp_new)
275 : "cc");
276 return rp_old.pair;
277}
278
279static inline long long atomic64_cmpxchg(atomic64_t *v,
280 long long old, long long new)
281{
282 register_pair rp_old = {.pair = old};
283 register_pair rp_new = {.pair = new};
284
285 asm volatile(
286 " cds %0,%3,0(%2)"
287 : "+&d" (rp_old), "+m" (v->counter)
288 : "a" (&v->counter), "d" (rp_new)
289 : "cc");
290 return rp_old.pair;
291}
292
293
294static inline long long atomic64_add_return(long long i, atomic64_t *v)
295{
296 long long old, new;
297
298 do {
299 old = atomic64_read(v);
300 new = old + i;
301 } while (atomic64_cmpxchg(v, old, new) != old);
302 return new;
303}
304
305static inline long long atomic64_sub_return(long long i, atomic64_t *v)
306{
307 long long old, new;
308
309 do {
310 old = atomic64_read(v);
311 new = old - i;
312 } while (atomic64_cmpxchg(v, old, new) != old);
313 return new;
314}
315
316static inline void atomic64_set_mask(unsigned long long mask, atomic64_t *v)
317{
318 long long old, new;
319
320 do {
321 old = atomic64_read(v);
322 new = old | mask;
323 } while (atomic64_cmpxchg(v, old, new) != old);
324}
325
326static inline void atomic64_clear_mask(unsigned long long mask, atomic64_t *v)
327{
328 long long old, new;
329
330 do {
331 old = atomic64_read(v);
332 new = old & mask;
333 } while (atomic64_cmpxchg(v, old, new) != old);
334}
335
336#endif
337
338static inline int atomic64_add_unless(atomic64_t *v, long long a, long long u)
339{
340 long long c, old;
341 c = atomic64_read(v);
342 for (;;) {
343 if (unlikely(c == u))
344 break;
345 old = atomic64_cmpxchg(v, c, c + a);
346 if (likely(old == c))
347 break;
348 c = old;
349 }
350 return c != u;
351}
352
353#define atomic64_add(_i, _v) atomic64_add_return(_i, _v)
354#define atomic64_add_negative(_i, _v) (atomic64_add_return(_i, _v) < 0)
355#define atomic64_inc(_v) atomic64_add_return(1, _v)
356#define atomic64_inc_return(_v) atomic64_add_return(1, _v)
357#define atomic64_inc_and_test(_v) (atomic64_add_return(1, _v) == 0)
358#define atomic64_sub(_i, _v) atomic64_sub_return(_i, _v)
359#define atomic64_sub_and_test(_i, _v) (atomic64_sub_return(_i, _v) == 0)
360#define atomic64_dec(_v) atomic64_sub_return(1, _v)
361#define atomic64_dec_return(_v) atomic64_sub_return(1, _v)
362#define atomic64_dec_and_test(_v) (atomic64_sub_return(1, _v) == 0)
363#define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0)
364
365#define smp_mb__before_atomic_dec() smp_mb()
366#define smp_mb__after_atomic_dec() smp_mb()
367#define smp_mb__before_atomic_inc() smp_mb()
368#define smp_mb__after_atomic_inc() smp_mb()
369
370#include <asm-generic/atomic-long.h>
371
372#endif
373
