linux/arch/powerpc/include/asm/cmpxchg.h
<<
>>
Prefs 
   1#ifndef _ASM_POWERPC_CMPXCHG_H_
   2#define _ASM_POWERPC_CMPXCHG_H_
   3
   4#ifdef __KERNEL__
   5#include <linux/compiler.h>
   6#include <asm/synch.h>
   7#include <asm/asm-compat.h>
   8#include <linux/bug.h>
   9
  10/*
  11 * Atomic exchange
  12 *
  13 * Changes the memory location '*p' to be val and returns
  14 * the previous value stored there.
  15 */
  16
  17static __always_inline unsigned long
  18__xchg_u32_local(volatile void *p, unsigned long val)
  19{
  20        unsigned long prev;
  21
  22        __asm__ __volatile__(
  23"1:     lwarx   %0,0,%2 \n"
  24        PPC405_ERR77(0,%2)
  25"       stwcx.  %3,0,%2 \n\
  26        bne-    1b"
  27        : "=&r" (prev), "+m" (*(volatile unsigned int *)p)
  28        : "r" (p), "r" (val)
  29        : "cc", "memory");
  30
  31        return prev;
  32}
  33
  34static __always_inline unsigned long
  35__xchg_u32_relaxed(u32 *p, unsigned long val)
  36{
  37        unsigned long prev;
  38
  39        __asm__ __volatile__(
  40"1:     lwarx   %0,0,%2\n"
  41        PPC405_ERR77(0, %2)
  42"       stwcx.  %3,0,%2\n"
  43"       bne-    1b"
  44        : "=&r" (prev), "+m" (*p)
  45        : "r" (p), "r" (val)
  46        : "cc");
  47
  48        return prev;
  49}
  50
  51#ifdef CONFIG_PPC64
  52static __always_inline unsigned long
  53__xchg_u64_local(volatile void *p, unsigned long val)
  54{
  55        unsigned long prev;
  56
  57        __asm__ __volatile__(
  58"1:     ldarx   %0,0,%2 \n"
  59        PPC405_ERR77(0,%2)
  60"       stdcx.  %3,0,%2 \n\
  61        bne-    1b"
  62        : "=&r" (prev), "+m" (*(volatile unsigned long *)p)
  63        : "r" (p), "r" (val)
  64        : "cc", "memory");
  65
  66        return prev;
  67}
  68
  69static __always_inline unsigned long
  70__xchg_u64_relaxed(u64 *p, unsigned long val)
  71{
  72        unsigned long prev;
  73
  74        __asm__ __volatile__(
  75"1:     ldarx   %0,0,%2\n"
  76        PPC405_ERR77(0, %2)
  77"       stdcx.  %3,0,%2\n"
  78"       bne-    1b"
  79        : "=&r" (prev), "+m" (*p)
  80        : "r" (p), "r" (val)
  81        : "cc");
  82
  83        return prev;
  84}
  85#endif
  86
  87static __always_inline unsigned long
  88__xchg_local(volatile void *ptr, unsigned long x, unsigned int size)
  89{
  90        switch (size) {
  91        case 4:
  92                return __xchg_u32_local(ptr, x);
  93#ifdef CONFIG_PPC64
  94        case 8:
  95                return __xchg_u64_local(ptr, x);
  96#endif
  97        }
  98        BUILD_BUG_ON_MSG(1, "Unsupported size for __xchg");
  99        return x;
 100}
 101
 102static __always_inline unsigned long
 103__xchg_relaxed(void *ptr, unsigned long x, unsigned int size)
 104{
 105        switch (size) {
 106        case 4:
 107                return __xchg_u32_relaxed(ptr, x);
 108#ifdef CONFIG_PPC64
 109        case 8:
 110                return __xchg_u64_relaxed(ptr, x);
 111#endif
 112        }
 113        BUILD_BUG_ON_MSG(1, "Unsupported size for __xchg_local");
 114        return x;
 115}
 116#define xchg_local(ptr,x)                                                    \
 117  ({                                                                         \
 118     __typeof__(*(ptr)) _x_ = (x);                                           \
 119     (__typeof__(*(ptr))) __xchg_local((ptr),                                \
 120                (unsigned long)_x_, sizeof(*(ptr)));                         \
 121  })
 122
 123#define xchg_relaxed(ptr, x)                                            \
 124({                                                                      \
 125        __typeof__(*(ptr)) _x_ = (x);                                   \
 126        (__typeof__(*(ptr))) __xchg_relaxed((ptr),                      \
 127                        (unsigned long)_x_, sizeof(*(ptr)));            \
 128})
 129/*
 130 * Compare and exchange - if *p == old, set it to new,
 131 * and return the old value of *p.
 132 */
 133
 134static __always_inline unsigned long
 135__cmpxchg_u32(volatile unsigned int *p, unsigned long old, unsigned long new)
 136{
 137        unsigned int prev;
 138
 139        __asm__ __volatile__ (
 140        PPC_ATOMIC_ENTRY_BARRIER
 141"1:     lwarx   %0,0,%2         # __cmpxchg_u32\n\
 142        cmpw    0,%0,%3\n\
 143        bne-    2f\n"
 144        PPC405_ERR77(0,%2)
 145"       stwcx.  %4,0,%2\n\
 146        bne-    1b"
 147        PPC_ATOMIC_EXIT_BARRIER
 148        "\n\
 1492:"
 150        : "=&r" (prev), "+m" (*p)
 151        : "r" (p), "r" (old), "r" (new)
 152        : "cc", "memory");
 153
 154        return prev;
 155}
 156
 157static __always_inline unsigned long
 158__cmpxchg_u32_local(volatile unsigned int *p, unsigned long old,
 159                        unsigned long new)
 160{
 161        unsigned int prev;
 162
 163        __asm__ __volatile__ (
 164"1:     lwarx   %0,0,%2         # __cmpxchg_u32\n\
 165        cmpw    0,%0,%3\n\
 166        bne-    2f\n"
 167        PPC405_ERR77(0,%2)
 168"       stwcx.  %4,0,%2\n\
 169        bne-    1b"
 170        "\n\
 1712:"
 172        : "=&r" (prev), "+m" (*p)
 173        : "r" (p), "r" (old), "r" (new)
 174        : "cc", "memory");
 175
 176        return prev;
 177}
 178
 179static __always_inline unsigned long
 180__cmpxchg_u32_relaxed(u32 *p, unsigned long old, unsigned long new)
 181{
 182        unsigned long prev;
 183
 184        __asm__ __volatile__ (
 185"1:     lwarx   %0,0,%2         # __cmpxchg_u32_relaxed\n"
 186"       cmpw    0,%0,%3\n"
 187"       bne-    2f\n"
 188        PPC405_ERR77(0, %2)
 189"       stwcx.  %4,0,%2\n"
 190"       bne-    1b\n"
 191"2:"
 192        : "=&r" (prev), "+m" (*p)
 193        : "r" (p), "r" (old), "r" (new)
 194        : "cc");
 195
 196        return prev;
 197}
 198
 199/*
 200 * cmpxchg family don't have order guarantee if cmp part fails, therefore we
 201 * can avoid superfluous barriers if we use assembly code to implement
 202 * cmpxchg() and cmpxchg_acquire(), however we don't do the similar for
 203 * cmpxchg_release() because that will result in putting a barrier in the
 204 * middle of a ll/sc loop, which is probably a bad idea. For example, this
 205 * might cause the conditional store more likely to fail.
 206 */
 207static __always_inline unsigned long
 208__cmpxchg_u32_acquire(u32 *p, unsigned long old, unsigned long new)
 209{
 210        unsigned long prev;
 211
 212        __asm__ __volatile__ (
 213"1:     lwarx   %0,0,%2         # __cmpxchg_u32_acquire\n"
 214"       cmpw    0,%0,%3\n"
 215"       bne-    2f\n"
 216        PPC405_ERR77(0, %2)
 217"       stwcx.  %4,0,%2\n"
 218"       bne-    1b\n"
 219        PPC_ACQUIRE_BARRIER
 220        "\n"
 221"2:"
 222        : "=&r" (prev), "+m" (*p)
 223        : "r" (p), "r" (old), "r" (new)
 224        : "cc", "memory");
 225
 226        return prev;
 227}
 228
 229#ifdef CONFIG_PPC64
 230static __always_inline unsigned long
 231__cmpxchg_u64(volatile unsigned long *p, unsigned long old, unsigned long new)
 232{
 233        unsigned long prev;
 234
 235        __asm__ __volatile__ (
 236        PPC_ATOMIC_ENTRY_BARRIER
 237"1:     ldarx   %0,0,%2         # __cmpxchg_u64\n\
 238        cmpd    0,%0,%3\n\
 239        bne-    2f\n\
 240        stdcx.  %4,0,%2\n\
 241        bne-    1b"
 242        PPC_ATOMIC_EXIT_BARRIER
 243        "\n\
 2442:"
 245        : "=&r" (prev), "+m" (*p)
 246        : "r" (p), "r" (old), "r" (new)
 247        : "cc", "memory");
 248
 249        return prev;
 250}
 251
 252static __always_inline unsigned long
 253__cmpxchg_u64_local(volatile unsigned long *p, unsigned long old,
 254                        unsigned long new)
 255{
 256        unsigned long prev;
 257
 258        __asm__ __volatile__ (
 259"1:     ldarx   %0,0,%2         # __cmpxchg_u64\n\
 260        cmpd    0,%0,%3\n\
 261        bne-    2f\n\
 262        stdcx.  %4,0,%2\n\
 263        bne-    1b"
 264        "\n\
 2652:"
 266        : "=&r" (prev), "+m" (*p)
 267        : "r" (p), "r" (old), "r" (new)
 268        : "cc", "memory");
 269
 270        return prev;
 271}
 272
 273static __always_inline unsigned long
 274__cmpxchg_u64_relaxed(u64 *p, unsigned long old, unsigned long new)
 275{
 276        unsigned long prev;
 277
 278        __asm__ __volatile__ (
 279"1:     ldarx   %0,0,%2         # __cmpxchg_u64_relaxed\n"
 280"       cmpd    0,%0,%3\n"
 281"       bne-    2f\n"
 282"       stdcx.  %4,0,%2\n"
 283"       bne-    1b\n"
 284"2:"
 285        : "=&r" (prev), "+m" (*p)
 286        : "r" (p), "r" (old), "r" (new)
 287        : "cc");
 288
 289        return prev;
 290}
 291
 292static __always_inline unsigned long
 293__cmpxchg_u64_acquire(u64 *p, unsigned long old, unsigned long new)
 294{
 295        unsigned long prev;
 296
 297        __asm__ __volatile__ (
 298"1:     ldarx   %0,0,%2         # __cmpxchg_u64_acquire\n"
 299"       cmpd    0,%0,%3\n"
 300"       bne-    2f\n"
 301"       stdcx.  %4,0,%2\n"
 302"       bne-    1b\n"
 303        PPC_ACQUIRE_BARRIER
 304        "\n"
 305"2:"
 306        : "=&r" (prev), "+m" (*p)
 307        : "r" (p), "r" (old), "r" (new)
 308        : "cc", "memory");
 309
 310        return prev;
 311}
 312#endif
 313
 314static __always_inline unsigned long
 315__cmpxchg(volatile void *ptr, unsigned long old, unsigned long new,
 316          unsigned int size)
 317{
 318        switch (size) {
 319        case 4:
 320                return __cmpxchg_u32(ptr, old, new);
 321#ifdef CONFIG_PPC64
 322        case 8:
 323                return __cmpxchg_u64(ptr, old, new);
 324#endif
 325        }
 326        BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg");
 327        return old;
 328}
 329
 330static __always_inline unsigned long
 331__cmpxchg_local(volatile void *ptr, unsigned long old, unsigned long new,
 332          unsigned int size)
 333{
 334        switch (size) {
 335        case 4:
 336                return __cmpxchg_u32_local(ptr, old, new);
 337#ifdef CONFIG_PPC64
 338        case 8:
 339                return __cmpxchg_u64_local(ptr, old, new);
 340#endif
 341        }
 342        BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg_local");
 343        return old;
 344}
 345
 346static __always_inline unsigned long
 347__cmpxchg_relaxed(void *ptr, unsigned long old, unsigned long new,
 348                  unsigned int size)
 349{
 350        switch (size) {
 351        case 4:
 352                return __cmpxchg_u32_relaxed(ptr, old, new);
 353#ifdef CONFIG_PPC64
 354        case 8:
 355                return __cmpxchg_u64_relaxed(ptr, old, new);
 356#endif
 357        }
 358        BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg_relaxed");
 359        return old;
 360}
 361
 362static __always_inline unsigned long
 363__cmpxchg_acquire(void *ptr, unsigned long old, unsigned long new,
 364                  unsigned int size)
 365{
 366        switch (size) {
 367        case 4:
 368                return __cmpxchg_u32_acquire(ptr, old, new);
 369#ifdef CONFIG_PPC64
 370        case 8:
 371                return __cmpxchg_u64_acquire(ptr, old, new);
 372#endif
 373        }
 374        BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg_acquire");
 375        return old;
 376}
 377#define cmpxchg(ptr, o, n)                                               \
 378  ({                                                                     \
 379     __typeof__(*(ptr)) _o_ = (o);                                       \
 380     __typeof__(*(ptr)) _n_ = (n);                                       \
 381     (__typeof__(*(ptr))) __cmpxchg((ptr), (unsigned long)_o_,           \
 382                                    (unsigned long)_n_, sizeof(*(ptr))); \
 383  })
 384
 385
 386#define cmpxchg_local(ptr, o, n)                                         \
 387  ({                                                                     \
 388     __typeof__(*(ptr)) _o_ = (o);                                       \
 389     __typeof__(*(ptr)) _n_ = (n);                                       \
 390     (__typeof__(*(ptr))) __cmpxchg_local((ptr), (unsigned long)_o_,     \
 391                                    (unsigned long)_n_, sizeof(*(ptr))); \
 392  })
 393
 394#define cmpxchg_relaxed(ptr, o, n)                                      \
 395({                                                                      \
 396        __typeof__(*(ptr)) _o_ = (o);                                   \
 397        __typeof__(*(ptr)) _n_ = (n);                                   \
 398        (__typeof__(*(ptr))) __cmpxchg_relaxed((ptr),                   \
 399                        (unsigned long)_o_, (unsigned long)_n_,         \
 400                        sizeof(*(ptr)));                                \
 401})
 402
 403#define cmpxchg_acquire(ptr, o, n)                                      \
 404({                                                                      \
 405        __typeof__(*(ptr)) _o_ = (o);                                   \
 406        __typeof__(*(ptr)) _n_ = (n);                                   \
 407        (__typeof__(*(ptr))) __cmpxchg_acquire((ptr),                   \
 408                        (unsigned long)_o_, (unsigned long)_n_,         \
 409                        sizeof(*(ptr)));                                \
 410})
 411#ifdef CONFIG_PPC64
 412#define cmpxchg64(ptr, o, n)                                            \
 413  ({                                                                    \
 414        BUILD_BUG_ON(sizeof(*(ptr)) != 8);                              \
 415        cmpxchg((ptr), (o), (n));                                       \
 416  })
 417#define cmpxchg64_local(ptr, o, n)                                      \
 418  ({                                                                    \
 419        BUILD_BUG_ON(sizeof(*(ptr)) != 8);                              \
 420        cmpxchg_local((ptr), (o), (n));                                 \
 421  })
 422#define cmpxchg64_relaxed(ptr, o, n)                                    \
 423({                                                                      \
 424        BUILD_BUG_ON(sizeof(*(ptr)) != 8);                              \
 425        cmpxchg_relaxed((ptr), (o), (n));                               \
 426})
 427#define cmpxchg64_acquire(ptr, o, n)                                    \
 428({                                                                      \
 429        BUILD_BUG_ON(sizeof(*(ptr)) != 8);                              \
 430        cmpxchg_acquire((ptr), (o), (n));                               \
 431})
 432#else
 433#include <asm-generic/cmpxchg-local.h>
 434#define cmpxchg64_local(ptr, o, n) __cmpxchg64_local_generic((ptr), (o), (n))
 435#endif
 436
 437#endif /* __KERNEL__ */
 438#endif /* _ASM_POWERPC_CMPXCHG_H_ */
 439
Hosted by Missing Link Electronics. The original LXR software by the LXR community, this experimental version by lxr@linux.no.