1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18#ifndef __ASM_CMPXCHG_H
19#define __ASM_CMPXCHG_H
20
21#include <linux/bug.h>
22
23#include <asm/barrier.h>
24
25static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size)
26{
27 unsigned long ret, tmp;
28
29 switch (size) {
30 case 1:
31 asm volatile("// __xchg1\n"
32 "1: ldxrb %w0, %2\n"
33 " stlxrb %w1, %w3, %2\n"
34 " cbnz %w1, 1b\n"
35 : "=&r" (ret), "=&r" (tmp), "+Q" (*(u8 *)ptr)
36 : "r" (x)
37 : "memory");
38 break;
39 case 2:
40 asm volatile("// __xchg2\n"
41 "1: ldxrh %w0, %2\n"
42 " stlxrh %w1, %w3, %2\n"
43 " cbnz %w1, 1b\n"
44 : "=&r" (ret), "=&r" (tmp), "+Q" (*(u16 *)ptr)
45 : "r" (x)
46 : "memory");
47 break;
48 case 4:
49 asm volatile("// __xchg4\n"
50 "1: ldxr %w0, %2\n"
51 " stlxr %w1, %w3, %2\n"
52 " cbnz %w1, 1b\n"
53 : "=&r" (ret), "=&r" (tmp), "+Q" (*(u32 *)ptr)
54 : "r" (x)
55 : "memory");
56 break;
57 case 8:
58 asm volatile("// __xchg8\n"
59 "1: ldxr %0, %2\n"
60 " stlxr %w1, %3, %2\n"
61 " cbnz %w1, 1b\n"
62 : "=&r" (ret), "=&r" (tmp), "+Q" (*(u64 *)ptr)
63 : "r" (x)
64 : "memory");
65 break;
66 default:
67 BUILD_BUG();
68 }
69
70 smp_mb();
71 return ret;
72}
73
74#define xchg(ptr,x) \
75({ \
76 __typeof__(*(ptr)) __ret; \
77 __ret = (__typeof__(*(ptr))) \
78 __xchg((unsigned long)(x), (ptr), sizeof(*(ptr))); \
79 __ret; \
80})
81
82static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old,
83 unsigned long new, int size)
84{
85 unsigned long oldval = 0, res;
86
87 switch (size) {
88 case 1:
89 do {
90 asm volatile("// __cmpxchg1\n"
91 " ldxrb %w1, %2\n"
92 " mov %w0, #0\n"
93 " cmp %w1, %w3\n"
94 " b.ne 1f\n"
95 " stxrb %w0, %w4, %2\n"
96 "1:\n"
97 : "=&r" (res), "=&r" (oldval), "+Q" (*(u8 *)ptr)
98 : "Ir" (old), "r" (new)
99 : "cc");
100 } while (res);
101 break;
102
103 case 2:
104 do {
105 asm volatile("// __cmpxchg2\n"
106 " ldxrh %w1, %2\n"
107 " mov %w0, #0\n"
108 " cmp %w1, %w3\n"
109 " b.ne 1f\n"
110 " stxrh %w0, %w4, %2\n"
111 "1:\n"
112 : "=&r" (res), "=&r" (oldval), "+Q" (*(u16 *)ptr)
113 : "Ir" (old), "r" (new)
114 : "cc");
115 } while (res);
116 break;
117
118 case 4:
119 do {
120 asm volatile("// __cmpxchg4\n"
121 " ldxr %w1, %2\n"
122 " mov %w0, #0\n"
123 " cmp %w1, %w3\n"
124 " b.ne 1f\n"
125 " stxr %w0, %w4, %2\n"
126 "1:\n"
127 : "=&r" (res), "=&r" (oldval), "+Q" (*(u32 *)ptr)
128 : "Ir" (old), "r" (new)
129 : "cc");
130 } while (res);
131 break;
132
133 case 8:
134 do {
135 asm volatile("// __cmpxchg8\n"
136 " ldxr %1, %2\n"
137 " mov %w0, #0\n"
138 " cmp %1, %3\n"
139 " b.ne 1f\n"
140 " stxr %w0, %4, %2\n"
141 "1:\n"
142 : "=&r" (res), "=&r" (oldval), "+Q" (*(u64 *)ptr)
143 : "Ir" (old), "r" (new)
144 : "cc");
145 } while (res);
146 break;
147
148 default:
149 BUILD_BUG();
150 }
151
152 return oldval;
153}
154
155static inline unsigned long __cmpxchg_mb(volatile void *ptr, unsigned long old,
156 unsigned long new, int size)
157{
158 unsigned long ret;
159
160 smp_mb();
161 ret = __cmpxchg(ptr, old, new, size);
162 smp_mb();
163
164 return ret;
165}
166
167#define cmpxchg(ptr, o, n) \
168({ \
169 __typeof__(*(ptr)) __ret; \
170 __ret = (__typeof__(*(ptr))) \
171 __cmpxchg_mb((ptr), (unsigned long)(o), (unsigned long)(n), \
172 sizeof(*(ptr))); \
173 __ret; \
174})
175
176#define cmpxchg_local(ptr, o, n) \
177({ \
178 __typeof__(*(ptr)) __ret; \
179 __ret = (__typeof__(*(ptr))) \
180 __cmpxchg((ptr), (unsigned long)(o), \
181 (unsigned long)(n), sizeof(*(ptr))); \
182 __ret; \
183})
184
185#define cmpxchg64(ptr,o,n) cmpxchg((ptr),(o),(n))
186#define cmpxchg64_local(ptr,o,n) cmpxchg_local((ptr),(o),(n))
187
188#define cmpxchg64_relaxed(ptr,o,n) cmpxchg_local((ptr),(o),(n))
189
190#endif
191
