LXR linux/arch/x86/include/asm/cmpxchg

   1#ifndef _ASM_X86_CMPXCHG_32_H
   2#define _ASM_X86_CMPXCHG_32_H
   3
   4#include <linux/bitops.h> /* for LOCK_PREFIX */
   5
   6/*
   7 * Note: if you use set64_bit(), __cmpxchg64(), or their variants, you
   8 *       you need to test for the feature in boot_cpu_data.
   9 */
  10
  11#define xchg(ptr, v)                                                    \
  12        ((__typeof__(*(ptr)))__xchg((unsigned long)(v), (ptr), sizeof(*(ptr))))
  13
  14struct __xchg_dummy {
  15        unsigned long a[100];
  16};
  17#define __xg(x) ((struct __xchg_dummy *)(x))
  18
  19/*
  20 * The semantics of XCHGCMP8B are a bit strange, this is why
  21 * there is a loop and the loading of %%eax and %%edx has to
  22 * be inside. This inlines well in most cases, the cached
  23 * cost is around ~38 cycles. (in the future we might want
  24 * to do an SIMD/3DNOW!/MMX/FPU 64-bit store here, but that
  25 * might have an implicit FPU-save as a cost, so it's not
  26 * clear which path to go.)
  27 *
  28 * cmpxchg8b must be used with the lock prefix here to allow
  29 * the instruction to be executed atomically, see page 3-102
  30 * of the instruction set reference 24319102.pdf. We need
  31 * the reader side to see the coherent 64bit value.
  32 */
  33static inline void __set_64bit(unsigned long long *ptr,
  34                               unsigned int low, unsigned int high)
  35{
  36        asm volatile("\n1:\t"
  37                     "movl (%0), %%eax\n\t"
  38                     "movl 4(%0), %%edx\n\t"
  39                     LOCK_PREFIX "cmpxchg8b (%0)\n\t"
  40                     "jnz 1b"
  41                     : /* no outputs */
  42                     : "D"(ptr),
  43                       "b"(low),
  44                       "c"(high)
  45                     : "ax", "dx", "memory");
  46}
  47
  48static inline void __set_64bit_constant(unsigned long long *ptr,
  49                                        unsigned long long value)
  50{
  51        __set_64bit(ptr, (unsigned int)value, (unsigned int)(value >> 32));
  52}
  53
  54#define ll_low(x)       *(((unsigned int *)&(x)) + 0)
  55#define ll_high(x)      *(((unsigned int *)&(x)) + 1)
  56
  57static inline void __set_64bit_var(unsigned long long *ptr,
  58                                   unsigned long long value)
  59{
  60        __set_64bit(ptr, ll_low(value), ll_high(value));
  61}
  62
  63#define set_64bit(ptr, value)                   \
  64        (__builtin_constant_p((value))          \
  65         ? __set_64bit_constant((ptr), (value)) \
  66         : __set_64bit_var((ptr), (value)))
  67
  68#define _set_64bit(ptr, value)                                          \
  69        (__builtin_constant_p(value)                                    \
  70         ? __set_64bit(ptr, (unsigned int)(value),                      \
  71                       (unsigned int)((value) >> 32))                   \
  72         : __set_64bit(ptr, ll_low((value)), ll_high((value))))
  73
  74/*
  75 * Note: no "lock" prefix even on SMP: xchg always implies lock anyway
  76 * Note 2: xchg has side effect, so that attribute volatile is necessary,
  77 *        but generally the primitive is invalid, *ptr is output argument. --ANK
  78 */
  79static inline unsigned long __xchg(unsigned long x, volatile void *ptr,
  80                                   int size)
  81{
  82        switch (size) {
  83        case 1:
  84                asm volatile("xchgb %b0,%1"
  85                             : "=q" (x)
  86                             : "m" (*__xg(ptr)), "0" (x)
  87                             : "memory");
  88                break;
  89        case 2:
  90                asm volatile("xchgw %w0,%1"
  91                             : "=r" (x)
  92                             : "m" (*__xg(ptr)), "0" (x)
  93                             : "memory");
  94                break;
  95        case 4:
  96                asm volatile("xchgl %0,%1"
  97                             : "=r" (x)
  98                             : "m" (*__xg(ptr)), "0" (x)
  99                             : "memory");
 100                break;
 101        }
 102        return x;
 103}
 104
 105/*
 106 * Atomic compare and exchange.  Compare OLD with MEM, if identical,
 107 * store NEW in MEM.  Return the initial value in MEM.  Success is
 108 * indicated by comparing RETURN with OLD.
 109 */
 110
 111#ifdef CONFIG_X86_CMPXCHG
 112#define __HAVE_ARCH_CMPXCHG 1
 113#define cmpxchg(ptr, o, n)                                              \
 114        ((__typeof__(*(ptr)))__cmpxchg((ptr), (unsigned long)(o),       \
 115                                       (unsigned long)(n),              \
 116                                       sizeof(*(ptr))))
 117#define sync_cmpxchg(ptr, o, n)                                         \
 118        ((__typeof__(*(ptr)))__sync_cmpxchg((ptr), (unsigned long)(o),  \
 119                                            (unsigned long)(n),         \
 120                                            sizeof(*(ptr))))
 121#define cmpxchg_local(ptr, o, n)                                        \
 122        ((__typeof__(*(ptr)))__cmpxchg_local((ptr), (unsigned long)(o), \
 123                                             (unsigned long)(n),        \
 124                                             sizeof(*(ptr))))
 125#endif
 126
 127#ifdef CONFIG_X86_CMPXCHG64
 128#define cmpxchg64(ptr, o, n)                                            \
 129        ((__typeof__(*(ptr)))__cmpxchg64((ptr), (unsigned long long)(o), \
 130                                         (unsigned long long)(n)))
 131#define cmpxchg64_local(ptr, o, n)                                      \
 132        ((__typeof__(*(ptr)))__cmpxchg64_local((ptr), (unsigned long long)(o), \
 133                                               (unsigned long long)(n)))
 134#endif
 135
 136static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old,
 137                                      unsigned long new, int size)
 138{
 139        unsigned long prev;
 140        switch (size) {
 141        case 1:
 142                asm volatile(LOCK_PREFIX "cmpxchgb %b1,%2"
 143                             : "=a"(prev)
 144                             : "q"(new), "m"(*__xg(ptr)), "0"(old)
 145                             : "memory");
 146                return prev;
 147        case 2:
 148                asm volatile(LOCK_PREFIX "cmpxchgw %w1,%2"
 149                             : "=a"(prev)
 150                             : "r"(new), "m"(*__xg(ptr)), "0"(old)
 151                             : "memory");
 152                return prev;
 153        case 4:
 154                asm volatile(LOCK_PREFIX "cmpxchgl %1,%2"
 155                             : "=a"(prev)
 156                             : "r"(new), "m"(*__xg(ptr)), "0"(old)
 157                             : "memory");
 158                return prev;
 159        }
 160        return old;
 161}
 162
 163/*
 164 * Always use locked operations when touching memory shared with a
 165 * hypervisor, since the system may be SMP even if the guest kernel
 166 * isn't.
 167 */
 168static inline unsigned long __sync_cmpxchg(volatile void *ptr,
 169                                           unsigned long old,
 170                                           unsigned long new, int size)
 171{
 172        unsigned long prev;
 173        switch (size) {
 174        case 1:
 175                asm volatile("lock; cmpxchgb %b1,%2"
 176                             : "=a"(prev)
 177                             : "q"(new), "m"(*__xg(ptr)), "0"(old)
 178                             : "memory");
 179                return prev;
 180        case 2:
 181                asm volatile("lock; cmpxchgw %w1,%2"
 182                             : "=a"(prev)
 183                             : "r"(new), "m"(*__xg(ptr)), "0"(old)
 184                             : "memory");
 185                return prev;
 186        case 4:
 187                asm volatile("lock; cmpxchgl %1,%2"
 188                             : "=a"(prev)
 189                             : "r"(new), "m"(*__xg(ptr)), "0"(old)
 190                             : "memory");
 191                return prev;
 192        }
 193        return old;
 194}
 195
 196static inline unsigned long __cmpxchg_local(volatile void *ptr,
 197                                            unsigned long old,
 198                                            unsigned long new, int size)
 199{
 200        unsigned long prev;
 201        switch (size) {
 202        case 1:
 203                asm volatile("cmpxchgb %b1,%2"
 204                             : "=a"(prev)
 205                             : "q"(new), "m"(*__xg(ptr)), "0"(old)
 206                             : "memory");
 207                return prev;
 208        case 2:
 209                asm volatile("cmpxchgw %w1,%2"
 210                             : "=a"(prev)
 211                             : "r"(new), "m"(*__xg(ptr)), "0"(old)
 212                             : "memory");
 213                return prev;
 214        case 4:
 215                asm volatile("cmpxchgl %1,%2"
 216                             : "=a"(prev)
 217                             : "r"(new), "m"(*__xg(ptr)), "0"(old)
 218                             : "memory");
 219                return prev;
 220        }
 221        return old;
 222}
 223
 224static inline unsigned long long __cmpxchg64(volatile void *ptr,
 225                                             unsigned long long old,
 226                                             unsigned long long new)
 227{
 228        unsigned long long prev;
 229        asm volatile(LOCK_PREFIX "cmpxchg8b %3"
 230                     : "=A"(prev)
 231                     : "b"((unsigned long)new),
 232                       "c"((unsigned long)(new >> 32)),
 233                       "m"(*__xg(ptr)),
 234                       "0"(old)
 235                     : "memory");
 236        return prev;
 237}
 238
 239static inline unsigned long long __cmpxchg64_local(volatile void *ptr,
 240                                                   unsigned long long old,
 241                                                   unsigned long long new)
 242{
 243        unsigned long long prev;
 244        asm volatile("cmpxchg8b %3"
 245                     : "=A"(prev)
 246                     : "b"((unsigned long)new),
 247                       "c"((unsigned long)(new >> 32)),
 248                       "m"(*__xg(ptr)),
 249                       "0"(old)
 250                     : "memory");
 251        return prev;
 252}
 253
 254#ifndef CONFIG_X86_CMPXCHG
 255/*
 256 * Building a kernel capable running on 80386. It may be necessary to
 257 * simulate the cmpxchg on the 80386 CPU. For that purpose we define
 258 * a function for each of the sizes we support.
 259 */
 260
 261extern unsigned long cmpxchg_386_u8(volatile void *, u8, u8);
 262extern unsigned long cmpxchg_386_u16(volatile void *, u16, u16);
 263extern unsigned long cmpxchg_386_u32(volatile void *, u32, u32);
 264
 265static inline unsigned long cmpxchg_386(volatile void *ptr, unsigned long old,
 266                                        unsigned long new, int size)
 267{
 268        switch (size) {
 269        case 1:
 270                return cmpxchg_386_u8(ptr, old, new);
 271        case 2:
 272                return cmpxchg_386_u16(ptr, old, new);
 273        case 4:
 274                return cmpxchg_386_u32(ptr, old, new);
 275        }
 276        return old;
 277}
 278
 279#define cmpxchg(ptr, o, n)                                              \
 280({                                                                      \
 281        __typeof__(*(ptr)) __ret;                                       \
 282        if (likely(boot_cpu_data.x86 > 3))                              \
 283                __ret = (__typeof__(*(ptr)))__cmpxchg((ptr),            \
 284                                (unsigned long)(o), (unsigned long)(n), \
 285                                sizeof(*(ptr)));                        \
 286        else                                                            \
 287                __ret = (__typeof__(*(ptr)))cmpxchg_386((ptr),          \
 288                                (unsigned long)(o), (unsigned long)(n), \
 289                                sizeof(*(ptr)));                        \
 290        __ret;                                                          \
 291})
 292#define cmpxchg_local(ptr, o, n)                                        \
 293({                                                                      \
 294        __typeof__(*(ptr)) __ret;                                       \
 295        if (likely(boot_cpu_data.x86 > 3))                              \
 296                __ret = (__typeof__(*(ptr)))__cmpxchg_local((ptr),      \
 297                                (unsigned long)(o), (unsigned long)(n), \
 298                                sizeof(*(ptr)));                        \
 299        else                                                            \
 300                __ret = (__typeof__(*(ptr)))cmpxchg_386((ptr),          \
 301                                (unsigned long)(o), (unsigned long)(n), \
 302                                sizeof(*(ptr)));                        \
 303        __ret;                                                          \
 304})
 305#endif
 306
 307#ifndef CONFIG_X86_CMPXCHG64
 308/*
 309 * Building a kernel capable running on 80386 and 80486. It may be necessary
 310 * to simulate the cmpxchg8b on the 80386 and 80486 CPU.
 311 */
 312
 313extern unsigned long long cmpxchg_486_u64(volatile void *, u64, u64);
 314
 315#define cmpxchg64(ptr, o, n)                                    \
 316({                                                              \
 317        __typeof__(*(ptr)) __ret;                               \
 318        __typeof__(*(ptr)) __old = (o);                         \
 319        __typeof__(*(ptr)) __new = (n);                         \
 320        alternative_io("call cmpxchg8b_emu",                    \
 321                        "lock; cmpxchg8b (%%esi)" ,             \
 322                       X86_FEATURE_CX8,                         \
 323                       "=A" (__ret),                            \
 324                       "S" ((ptr)), "0" (__old),                \
 325                       "b" ((unsigned int)__new),               \
 326                       "c" ((unsigned int)(__new>>32))          \
 327                       : "memory");                             \
 328        __ret; })
 329
 330
 331
 332#define cmpxchg64_local(ptr, o, n)                                      \
 333({                                                                      \
 334        __typeof__(*(ptr)) __ret;                                       \
 335        if (likely(boot_cpu_data.x86 > 4))                              \
 336                __ret = (__typeof__(*(ptr)))__cmpxchg64_local((ptr),    \
 337                                (unsigned long long)(o),                \
 338                                (unsigned long long)(n));               \
 339        else                                                            \
 340                __ret = (__typeof__(*(ptr)))cmpxchg_486_u64((ptr),      \
 341                                (unsigned long long)(o),                \
 342                                (unsigned long long)(n));               \
 343        __ret;                                                          \
 344})
 345
 346#endif
 347
 348#endif /* _ASM_X86_CMPXCHG_32_H */
 349