1
2
3
4
5
6
7
8
9
10
11
12
13
14
15#include <linux/cache.h>
16#include <linux/delay.h>
17#include <linux/uaccess.h>
18#include <linux/module.h>
19#include <linux/mm.h>
20#include <linux/atomic.h>
21#include <arch/chip.h>
22
23
24#if ATOMIC_LOCKS_FOUND_VIA_TABLE()
25
26
27
28
29
30struct atomic_locks_on_cpu {
31 int lock[ATOMIC_HASH_L2_SIZE];
32} __attribute__((aligned(ATOMIC_HASH_L2_SIZE * 4)));
33
34static DEFINE_PER_CPU(struct atomic_locks_on_cpu, atomic_lock_pool);
35
36
37static struct atomic_locks_on_cpu __initdata initial_atomic_locks;
38
39
40struct atomic_locks_on_cpu *atomic_lock_ptr[ATOMIC_HASH_L1_SIZE]
41 __write_once = {
42 [0 ... ATOMIC_HASH_L1_SIZE-1] (&initial_atomic_locks)
43};
44
45#else
46
47
48int atomic_locks[PAGE_SIZE / sizeof(int)] __page_aligned_bss;
49
50#endif
51
52int *__atomic_hashed_lock(volatile void *v)
53{
54
55#if ATOMIC_LOCKS_FOUND_VIA_TABLE()
56 unsigned long i =
57 (unsigned long) v & ((PAGE_SIZE-1) & -sizeof(long long));
58 unsigned long n = __insn_crc32_32(0, i);
59
60
61 unsigned long l1_index = n >> ((sizeof(n) * 8) - ATOMIC_HASH_L1_SHIFT);
62
63 unsigned long l2_index = n & (ATOMIC_HASH_L2_SIZE - 1);
64
65 return &atomic_lock_ptr[l1_index]->lock[l2_index];
66#else
67
68
69
70
71 unsigned long ptr = __insn_mm((unsigned long)v >> 1,
72 (unsigned long)atomic_locks,
73 2, (ATOMIC_HASH_SHIFT + 2) - 1);
74 return (int *)ptr;
75#endif
76}
77
78#ifdef CONFIG_SMP
79
80static int is_atomic_lock(int *p)
81{
82#if ATOMIC_LOCKS_FOUND_VIA_TABLE()
83 int i;
84 for (i = 0; i < ATOMIC_HASH_L1_SIZE; ++i) {
85
86 if (p >= &atomic_lock_ptr[i]->lock[0] &&
87 p < &atomic_lock_ptr[i]->lock[ATOMIC_HASH_L2_SIZE]) {
88 return 1;
89 }
90 }
91 return 0;
92#else
93 return p >= &atomic_locks[0] && p < &atomic_locks[ATOMIC_HASH_SIZE];
94#endif
95}
96
97void __atomic_fault_unlock(int *irqlock_word)
98{
99 BUG_ON(!is_atomic_lock(irqlock_word));
100 BUG_ON(*irqlock_word != 1);
101 *irqlock_word = 0;
102}
103
104#endif
105
106static inline int *__atomic_setup(volatile void *v)
107{
108
109 *(volatile int *)v;
110 return __atomic_hashed_lock(v);
111}
112
113int _atomic_xchg(atomic_t *v, int n)
114{
115 return __atomic_xchg(&v->counter, __atomic_setup(v), n).val;
116}
117EXPORT_SYMBOL(_atomic_xchg);
118
119int _atomic_xchg_add(atomic_t *v, int i)
120{
121 return __atomic_xchg_add(&v->counter, __atomic_setup(v), i).val;
122}
123EXPORT_SYMBOL(_atomic_xchg_add);
124
125int _atomic_xchg_add_unless(atomic_t *v, int a, int u)
126{
127
128
129
130
131
132 return __atomic_xchg_add_unless(&v->counter, __atomic_setup(v), u, a)
133 .val;
134}
135EXPORT_SYMBOL(_atomic_xchg_add_unless);
136
137int _atomic_cmpxchg(atomic_t *v, int o, int n)
138{
139 return __atomic_cmpxchg(&v->counter, __atomic_setup(v), o, n).val;
140}
141EXPORT_SYMBOL(_atomic_cmpxchg);
142
143unsigned long _atomic_or(volatile unsigned long *p, unsigned long mask)
144{
145 return __atomic_or((int *)p, __atomic_setup(p), mask).val;
146}
147EXPORT_SYMBOL(_atomic_or);
148
149unsigned long _atomic_andn(volatile unsigned long *p, unsigned long mask)
150{
151 return __atomic_andn((int *)p, __atomic_setup(p), mask).val;
152}
153EXPORT_SYMBOL(_atomic_andn);
154
155unsigned long _atomic_xor(volatile unsigned long *p, unsigned long mask)
156{
157 return __atomic_xor((int *)p, __atomic_setup(p), mask).val;
158}
159EXPORT_SYMBOL(_atomic_xor);
160
161
162u64 _atomic64_xchg(atomic64_t *v, u64 n)
163{
164 return __atomic64_xchg(&v->counter, __atomic_setup(v), n);
165}
166EXPORT_SYMBOL(_atomic64_xchg);
167
168u64 _atomic64_xchg_add(atomic64_t *v, u64 i)
169{
170 return __atomic64_xchg_add(&v->counter, __atomic_setup(v), i);
171}
172EXPORT_SYMBOL(_atomic64_xchg_add);
173
174u64 _atomic64_xchg_add_unless(atomic64_t *v, u64 a, u64 u)
175{
176
177
178
179
180
181 return __atomic64_xchg_add_unless(&v->counter, __atomic_setup(v),
182 u, a);
183}
184EXPORT_SYMBOL(_atomic64_xchg_add_unless);
185
186u64 _atomic64_cmpxchg(atomic64_t *v, u64 o, u64 n)
187{
188 return __atomic64_cmpxchg(&v->counter, __atomic_setup(v), o, n);
189}
190EXPORT_SYMBOL(_atomic64_cmpxchg);
191
192
193
194
195
196
197
198
199
200
201
202
203struct __get_user __atomic_bad_address(int __user *addr)
204{
205 if (unlikely(!access_ok(VERIFY_WRITE, addr, sizeof(int))))
206 panic("Bad address used for kernel atomic op: %p\n", addr);
207 return (struct __get_user) { .err = -EFAULT };
208}
209
210
211#if CHIP_HAS_CBOX_HOME_MAP()
212static int __init noatomichash(char *str)
213{
214 pr_warning("noatomichash is deprecated.\n");
215 return 1;
216}
217__setup("noatomichash", noatomichash);
218#endif
219
220void __init __init_atomic_per_cpu(void)
221{
222#if ATOMIC_LOCKS_FOUND_VIA_TABLE()
223
224 unsigned int i;
225 int actual_cpu;
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243 actual_cpu = cpumask_first(cpu_possible_mask);
244
245 for (i = 0; i < ATOMIC_HASH_L1_SIZE; ++i) {
246
247
248
249
250 actual_cpu = cpumask_next(actual_cpu, cpu_possible_mask);
251 if (actual_cpu >= nr_cpu_ids)
252 actual_cpu = cpumask_first(cpu_possible_mask);
253
254 atomic_lock_ptr[i] = &per_cpu(atomic_lock_pool, actual_cpu);
255 }
256
257#else
258
259
260 BUILD_BUG_ON(ATOMIC_HASH_SIZE & (ATOMIC_HASH_SIZE-1));
261 BUG_ON(ATOMIC_HASH_SIZE < nr_cpu_ids);
262
263
264
265
266
267
268
269
270
271 BUG_ON((unsigned long)atomic_locks % PAGE_SIZE != 0);
272
273
274 BUILD_BUG_ON(ATOMIC_HASH_SIZE * sizeof(int) > PAGE_SIZE);
275
276
277
278
279
280
281 BUILD_BUG_ON((PAGE_SIZE >> 3) > ATOMIC_HASH_SIZE);
282
283#endif
284}
285