1
2
3
4
5#include <linux/kernel_stat.h>
6#include <linux/spinlock.h>
7#include <linux/debugfs.h>
8#include <linux/log2.h>
9
10#include <asm/paravirt.h>
11
12#include <xen/interface/xen.h>
13#include <xen/events.h>
14
15#include "xen-ops.h"
16#include "debugfs.h"
17
18#ifdef CONFIG_XEN_DEBUG_FS
19static struct xen_spinlock_stats
20{
21 u64 taken;
22 u32 taken_slow;
23 u32 taken_slow_nested;
24 u32 taken_slow_pickup;
25 u32 taken_slow_spurious;
26 u32 taken_slow_irqenable;
27
28 u64 released;
29 u32 released_slow;
30 u32 released_slow_kicked;
31
32#define HISTO_BUCKETS 30
33 u32 histo_spin_total[HISTO_BUCKETS+1];
34 u32 histo_spin_spinning[HISTO_BUCKETS+1];
35 u32 histo_spin_blocked[HISTO_BUCKETS+1];
36
37 u64 time_total;
38 u64 time_spinning;
39 u64 time_blocked;
40} spinlock_stats;
41
42static u8 zero_stats;
43
44static unsigned lock_timeout = 1 << 10;
45#define TIMEOUT lock_timeout
46
47static inline void check_zero(void)
48{
49 if (unlikely(zero_stats)) {
50 memset(&spinlock_stats, 0, sizeof(spinlock_stats));
51 zero_stats = 0;
52 }
53}
54
55#define ADD_STATS(elem, val) \
56 do { check_zero(); spinlock_stats.elem += (val); } while(0)
57
58static inline u64 spin_time_start(void)
59{
60 return xen_clocksource_read();
61}
62
63static void __spin_time_accum(u64 delta, u32 *array)
64{
65 unsigned index = ilog2(delta);
66
67 check_zero();
68
69 if (index < HISTO_BUCKETS)
70 array[index]++;
71 else
72 array[HISTO_BUCKETS]++;
73}
74
75static inline void spin_time_accum_spinning(u64 start)
76{
77 u32 delta = xen_clocksource_read() - start;
78
79 __spin_time_accum(delta, spinlock_stats.histo_spin_spinning);
80 spinlock_stats.time_spinning += delta;
81}
82
83static inline void spin_time_accum_total(u64 start)
84{
85 u32 delta = xen_clocksource_read() - start;
86
87 __spin_time_accum(delta, spinlock_stats.histo_spin_total);
88 spinlock_stats.time_total += delta;
89}
90
91static inline void spin_time_accum_blocked(u64 start)
92{
93 u32 delta = xen_clocksource_read() - start;
94
95 __spin_time_accum(delta, spinlock_stats.histo_spin_blocked);
96 spinlock_stats.time_blocked += delta;
97}
98#else
99#define TIMEOUT (1 << 10)
100#define ADD_STATS(elem, val) do { (void)(val); } while(0)
101
102static inline u64 spin_time_start(void)
103{
104 return 0;
105}
106
107static inline void spin_time_accum_total(u64 start)
108{
109}
110static inline void spin_time_accum_spinning(u64 start)
111{
112}
113static inline void spin_time_accum_blocked(u64 start)
114{
115}
116#endif
117
118struct xen_spinlock {
119 unsigned char lock;
120 unsigned short spinners;
121};
122
123static int xen_spin_is_locked(struct raw_spinlock *lock)
124{
125 struct xen_spinlock *xl = (struct xen_spinlock *)lock;
126
127 return xl->lock != 0;
128}
129
130static int xen_spin_is_contended(struct raw_spinlock *lock)
131{
132 struct xen_spinlock *xl = (struct xen_spinlock *)lock;
133
134
135
136 return xl->spinners != 0;
137}
138
139static int xen_spin_trylock(struct raw_spinlock *lock)
140{
141 struct xen_spinlock *xl = (struct xen_spinlock *)lock;
142 u8 old = 1;
143
144 asm("xchgb %b0,%1"
145 : "+q" (old), "+m" (xl->lock) : : "memory");
146
147 return old == 0;
148}
149
150static DEFINE_PER_CPU(int, lock_kicker_irq) = -1;
151static DEFINE_PER_CPU(struct xen_spinlock *, lock_spinners);
152
153
154
155
156
157static inline struct xen_spinlock *spinning_lock(struct xen_spinlock *xl)
158{
159 struct xen_spinlock *prev;
160
161 prev = __get_cpu_var(lock_spinners);
162 __get_cpu_var(lock_spinners) = xl;
163
164 wmb();
165
166 asm(LOCK_PREFIX " incw %0"
167 : "+m" (xl->spinners) : : "memory");
168
169 return prev;
170}
171
172
173
174
175
176static inline void unspinning_lock(struct xen_spinlock *xl, struct xen_spinlock *prev)
177{
178 asm(LOCK_PREFIX " decw %0"
179 : "+m" (xl->spinners) : : "memory");
180 wmb();
181 __get_cpu_var(lock_spinners) = prev;
182}
183
184static noinline int xen_spin_lock_slow(struct raw_spinlock *lock, bool irq_enable)
185{
186 struct xen_spinlock *xl = (struct xen_spinlock *)lock;
187 struct xen_spinlock *prev;
188 int irq = __get_cpu_var(lock_kicker_irq);
189 int ret;
190 u64 start;
191
192
193 if (irq == -1)
194 return 0;
195
196 start = spin_time_start();
197
198
199 prev = spinning_lock(xl);
200
201 ADD_STATS(taken_slow, 1);
202 ADD_STATS(taken_slow_nested, prev != NULL);
203
204 do {
205 unsigned long flags;
206
207
208 xen_clear_irq_pending(irq);
209
210
211
212 ret = xen_spin_trylock(lock);
213 if (ret) {
214 ADD_STATS(taken_slow_pickup, 1);
215
216
217
218
219
220
221 if (prev != NULL)
222 xen_set_irq_pending(irq);
223 goto out;
224 }
225
226 flags = __raw_local_save_flags();
227 if (irq_enable) {
228 ADD_STATS(taken_slow_irqenable, 1);
229 raw_local_irq_enable();
230 }
231
232
233
234
235
236
237
238
239
240
241 xen_poll_irq(irq);
242
243 raw_local_irq_restore(flags);
244
245 ADD_STATS(taken_slow_spurious, !xen_test_irq_pending(irq));
246 } while (!xen_test_irq_pending(irq));
247
248 kstat_incr_irqs_this_cpu(irq, irq_to_desc(irq));
249
250out:
251 unspinning_lock(xl, prev);
252 spin_time_accum_blocked(start);
253
254 return ret;
255}
256
257static inline void __xen_spin_lock(struct raw_spinlock *lock, bool irq_enable)
258{
259 struct xen_spinlock *xl = (struct xen_spinlock *)lock;
260 unsigned timeout;
261 u8 oldval;
262 u64 start_spin;
263
264 ADD_STATS(taken, 1);
265
266 start_spin = spin_time_start();
267
268 do {
269 u64 start_spin_fast = spin_time_start();
270
271 timeout = TIMEOUT;
272
273 asm("1: xchgb %1,%0\n"
274 " testb %1,%1\n"
275 " jz 3f\n"
276 "2: rep;nop\n"
277 " cmpb $0,%0\n"
278 " je 1b\n"
279 " dec %2\n"
280 " jnz 2b\n"
281 "3:\n"
282 : "+m" (xl->lock), "=q" (oldval), "+r" (timeout)
283 : "1" (1)
284 : "memory");
285
286 spin_time_accum_spinning(start_spin_fast);
287
288 } while (unlikely(oldval != 0 &&
289 (TIMEOUT == ~0 || !xen_spin_lock_slow(lock, irq_enable))));
290
291 spin_time_accum_total(start_spin);
292}
293
294static void xen_spin_lock(struct raw_spinlock *lock)
295{
296 __xen_spin_lock(lock, false);
297}
298
299static void xen_spin_lock_flags(struct raw_spinlock *lock, unsigned long flags)
300{
301 __xen_spin_lock(lock, !raw_irqs_disabled_flags(flags));
302}
303
304static noinline void xen_spin_unlock_slow(struct xen_spinlock *xl)
305{
306 int cpu;
307
308 ADD_STATS(released_slow, 1);
309
310 for_each_online_cpu(cpu) {
311
312 if (per_cpu(lock_spinners, cpu) == xl) {
313 ADD_STATS(released_slow_kicked, 1);
314 xen_send_IPI_one(cpu, XEN_SPIN_UNLOCK_VECTOR);
315 break;
316 }
317 }
318}
319
320static void xen_spin_unlock(struct raw_spinlock *lock)
321{
322 struct xen_spinlock *xl = (struct xen_spinlock *)lock;
323
324 ADD_STATS(released, 1);
325
326 smp_wmb();
327 xl->lock = 0;
328
329
330
331
332
333
334
335 mb();
336
337 if (unlikely(xl->spinners))
338 xen_spin_unlock_slow(xl);
339}
340
341static irqreturn_t dummy_handler(int irq, void *dev_id)
342{
343 BUG();
344 return IRQ_HANDLED;
345}
346
347void __cpuinit xen_init_lock_cpu(int cpu)
348{
349 int irq;
350 const char *name;
351
352 name = kasprintf(GFP_KERNEL, "spinlock%d", cpu);
353 irq = bind_ipi_to_irqhandler(XEN_SPIN_UNLOCK_VECTOR,
354 cpu,
355 dummy_handler,
356 IRQF_DISABLED|IRQF_PERCPU|IRQF_NOBALANCING,
357 name,
358 NULL);
359
360 if (irq >= 0) {
361 disable_irq(irq);
362 per_cpu(lock_kicker_irq, cpu) = irq;
363 }
364
365 printk("cpu %d spinlock event irq %d\n", cpu, irq);
366}
367
368void xen_uninit_lock_cpu(int cpu)
369{
370 unbind_from_irqhandler(per_cpu(lock_kicker_irq, cpu), NULL);
371}
372
373void __init xen_init_spinlocks(void)
374{
375 pv_lock_ops.spin_is_locked = xen_spin_is_locked;
376 pv_lock_ops.spin_is_contended = xen_spin_is_contended;
377 pv_lock_ops.spin_lock = xen_spin_lock;
378 pv_lock_ops.spin_lock_flags = xen_spin_lock_flags;
379 pv_lock_ops.spin_trylock = xen_spin_trylock;
380 pv_lock_ops.spin_unlock = xen_spin_unlock;
381}
382
383#ifdef CONFIG_XEN_DEBUG_FS
384
385static struct dentry *d_spin_debug;
386
387static int __init xen_spinlock_debugfs(void)
388{
389 struct dentry *d_xen = xen_init_debugfs();
390
391 if (d_xen == NULL)
392 return -ENOMEM;
393
394 d_spin_debug = debugfs_create_dir("spinlocks", d_xen);
395
396 debugfs_create_u8("zero_stats", 0644, d_spin_debug, &zero_stats);
397
398 debugfs_create_u32("timeout", 0644, d_spin_debug, &lock_timeout);
399
400 debugfs_create_u64("taken", 0444, d_spin_debug, &spinlock_stats.taken);
401 debugfs_create_u32("taken_slow", 0444, d_spin_debug,
402 &spinlock_stats.taken_slow);
403 debugfs_create_u32("taken_slow_nested", 0444, d_spin_debug,
404 &spinlock_stats.taken_slow_nested);
405 debugfs_create_u32("taken_slow_pickup", 0444, d_spin_debug,
406 &spinlock_stats.taken_slow_pickup);
407 debugfs_create_u32("taken_slow_spurious", 0444, d_spin_debug,
408 &spinlock_stats.taken_slow_spurious);
409 debugfs_create_u32("taken_slow_irqenable", 0444, d_spin_debug,
410 &spinlock_stats.taken_slow_irqenable);
411
412 debugfs_create_u64("released", 0444, d_spin_debug, &spinlock_stats.released);
413 debugfs_create_u32("released_slow", 0444, d_spin_debug,
414 &spinlock_stats.released_slow);
415 debugfs_create_u32("released_slow_kicked", 0444, d_spin_debug,
416 &spinlock_stats.released_slow_kicked);
417
418 debugfs_create_u64("time_spinning", 0444, d_spin_debug,
419 &spinlock_stats.time_spinning);
420 debugfs_create_u64("time_blocked", 0444, d_spin_debug,
421 &spinlock_stats.time_blocked);
422 debugfs_create_u64("time_total", 0444, d_spin_debug,
423 &spinlock_stats.time_total);
424
425 xen_debugfs_create_u32_array("histo_total", 0444, d_spin_debug,
426 spinlock_stats.histo_spin_total, HISTO_BUCKETS + 1);
427 xen_debugfs_create_u32_array("histo_spinning", 0444, d_spin_debug,
428 spinlock_stats.histo_spin_spinning, HISTO_BUCKETS + 1);
429 xen_debugfs_create_u32_array("histo_blocked", 0444, d_spin_debug,
430 spinlock_stats.histo_spin_blocked, HISTO_BUCKETS + 1);
431
432 return 0;
433}
434fs_initcall(xen_spinlock_debugfs);
435
436#endif
437