LXR linux/include/linux/atomic.h

   1/* Atomic operations usable in machine independent code */
   2#ifndef _LINUX_ATOMIC_H
   3#define _LINUX_ATOMIC_H
   4#include <asm/atomic.h>
   5#include <asm/barrier.h>
   6
   7/*
   8 * Relaxed variants of xchg, cmpxchg and some atomic operations.
   9 *
  10 * We support four variants:
  11 *
  12 * - Fully ordered: The default implementation, no suffix required.
  13 * - Acquire: Provides ACQUIRE semantics, _acquire suffix.
  14 * - Release: Provides RELEASE semantics, _release suffix.
  15 * - Relaxed: No ordering guarantees, _relaxed suffix.
  16 *
  17 * For compound atomics performing both a load and a store, ACQUIRE
  18 * semantics apply only to the load and RELEASE semantics only to the
  19 * store portion of the operation. Note that a failed cmpxchg_acquire
  20 * does -not- imply any memory ordering constraints.
  21 *
  22 * See Documentation/memory-barriers.txt for ACQUIRE/RELEASE definitions.
  23 */
  24
  25#ifndef atomic_read_acquire
  26#define  atomic_read_acquire(v)         smp_load_acquire(&(v)->counter)
  27#endif
  28
  29#ifndef atomic_set_release
  30#define  atomic_set_release(v, i)       smp_store_release(&(v)->counter, (i))
  31#endif
  32
  33/*
  34 * The idea here is to build acquire/release variants by adding explicit
  35 * barriers on top of the relaxed variant. In the case where the relaxed
  36 * variant is already fully ordered, no additional barriers are needed.
  37 *
  38 * Besides, if an arch has a special barrier for acquire/release, it could
  39 * implement its own __atomic_op_* and use the same framework for building
  40 * variants
  41 */
  42#ifndef __atomic_op_acquire
  43#define __atomic_op_acquire(op, args...)                                \
  44({                                                                      \
  45        typeof(op##_relaxed(args)) __ret  = op##_relaxed(args);         \
  46        smp_mb__after_atomic();                                         \
  47        __ret;                                                          \
  48})
  49#endif
  50
  51#ifndef __atomic_op_release
  52#define __atomic_op_release(op, args...)                                \
  53({                                                                      \
  54        smp_mb__before_atomic();                                        \
  55        op##_relaxed(args);                                             \
  56})
  57#endif
  58
  59#ifndef __atomic_op_fence
  60#define __atomic_op_fence(op, args...)                                  \
  61({                                                                      \
  62        typeof(op##_relaxed(args)) __ret;                               \
  63        smp_mb__before_atomic();                                        \
  64        __ret = op##_relaxed(args);                                     \
  65        smp_mb__after_atomic();                                         \
  66        __ret;                                                          \
  67})
  68#endif
  69
  70/* atomic_add_return_relaxed */
  71#ifndef atomic_add_return_relaxed
  72#define  atomic_add_return_relaxed      atomic_add_return
  73#define  atomic_add_return_acquire      atomic_add_return
  74#define  atomic_add_return_release      atomic_add_return
  75
  76#else /* atomic_add_return_relaxed */
  77
  78#ifndef atomic_add_return_acquire
  79#define  atomic_add_return_acquire(...)                                 \
  80        __atomic_op_acquire(atomic_add_return, __VA_ARGS__)
  81#endif
  82
  83#ifndef atomic_add_return_release
  84#define  atomic_add_return_release(...)                                 \
  85        __atomic_op_release(atomic_add_return, __VA_ARGS__)
  86#endif
  87
  88#ifndef atomic_add_return
  89#define  atomic_add_return(...)                                         \
  90        __atomic_op_fence(atomic_add_return, __VA_ARGS__)
  91#endif
  92#endif /* atomic_add_return_relaxed */
  93
  94/* atomic_inc_return_relaxed */
  95#ifndef atomic_inc_return_relaxed
  96#define  atomic_inc_return_relaxed      atomic_inc_return
  97#define  atomic_inc_return_acquire      atomic_inc_return
  98#define  atomic_inc_return_release      atomic_inc_return
  99
 100#else /* atomic_inc_return_relaxed */
 101
 102#ifndef atomic_inc_return_acquire
 103#define  atomic_inc_return_acquire(...)                                 \
 104        __atomic_op_acquire(atomic_inc_return, __VA_ARGS__)
 105#endif
 106
 107#ifndef atomic_inc_return_release
 108#define  atomic_inc_return_release(...)                                 \
 109        __atomic_op_release(atomic_inc_return, __VA_ARGS__)
 110#endif
 111
 112#ifndef atomic_inc_return
 113#define  atomic_inc_return(...)                                         \
 114        __atomic_op_fence(atomic_inc_return, __VA_ARGS__)
 115#endif
 116#endif /* atomic_inc_return_relaxed */
 117
 118/* atomic_sub_return_relaxed */
 119#ifndef atomic_sub_return_relaxed
 120#define  atomic_sub_return_relaxed      atomic_sub_return
 121#define  atomic_sub_return_acquire      atomic_sub_return
 122#define  atomic_sub_return_release      atomic_sub_return
 123
 124#else /* atomic_sub_return_relaxed */
 125
 126#ifndef atomic_sub_return_acquire
 127#define  atomic_sub_return_acquire(...)                                 \
 128        __atomic_op_acquire(atomic_sub_return, __VA_ARGS__)
 129#endif
 130
 131#ifndef atomic_sub_return_release
 132#define  atomic_sub_return_release(...)                                 \
 133        __atomic_op_release(atomic_sub_return, __VA_ARGS__)
 134#endif
 135
 136#ifndef atomic_sub_return
 137#define  atomic_sub_return(...)                                         \
 138        __atomic_op_fence(atomic_sub_return, __VA_ARGS__)
 139#endif
 140#endif /* atomic_sub_return_relaxed */
 141
 142/* atomic_dec_return_relaxed */
 143#ifndef atomic_dec_return_relaxed
 144#define  atomic_dec_return_relaxed      atomic_dec_return
 145#define  atomic_dec_return_acquire      atomic_dec_return
 146#define  atomic_dec_return_release      atomic_dec_return
 147
 148#else /* atomic_dec_return_relaxed */
 149
 150#ifndef atomic_dec_return_acquire
 151#define  atomic_dec_return_acquire(...)                                 \
 152        __atomic_op_acquire(atomic_dec_return, __VA_ARGS__)
 153#endif
 154
 155#ifndef atomic_dec_return_release
 156#define  atomic_dec_return_release(...)                                 \
 157        __atomic_op_release(atomic_dec_return, __VA_ARGS__)
 158#endif
 159
 160#ifndef atomic_dec_return
 161#define  atomic_dec_return(...)                                         \
 162        __atomic_op_fence(atomic_dec_return, __VA_ARGS__)
 163#endif
 164#endif /* atomic_dec_return_relaxed */
 165
 166/* atomic_xchg_relaxed */
 167#ifndef atomic_xchg_relaxed
 168#define  atomic_xchg_relaxed            atomic_xchg
 169#define  atomic_xchg_acquire            atomic_xchg
 170#define  atomic_xchg_release            atomic_xchg
 171
 172#else /* atomic_xchg_relaxed */
 173
 174#ifndef atomic_xchg_acquire
 175#define  atomic_xchg_acquire(...)                                       \
 176        __atomic_op_acquire(atomic_xchg, __VA_ARGS__)
 177#endif
 178
 179#ifndef atomic_xchg_release
 180#define  atomic_xchg_release(...)                                       \
 181        __atomic_op_release(atomic_xchg, __VA_ARGS__)
 182#endif
 183
 184#ifndef atomic_xchg
 185#define  atomic_xchg(...)                                               \
 186        __atomic_op_fence(atomic_xchg, __VA_ARGS__)
 187#endif
 188#endif /* atomic_xchg_relaxed */
 189
 190/* atomic_cmpxchg_relaxed */
 191#ifndef atomic_cmpxchg_relaxed
 192#define  atomic_cmpxchg_relaxed         atomic_cmpxchg
 193#define  atomic_cmpxchg_acquire         atomic_cmpxchg
 194#define  atomic_cmpxchg_release         atomic_cmpxchg
 195
 196#else /* atomic_cmpxchg_relaxed */
 197
 198#ifndef atomic_cmpxchg_acquire
 199#define  atomic_cmpxchg_acquire(...)                                    \
 200        __atomic_op_acquire(atomic_cmpxchg, __VA_ARGS__)
 201#endif
 202
 203#ifndef atomic_cmpxchg_release
 204#define  atomic_cmpxchg_release(...)                                    \
 205        __atomic_op_release(atomic_cmpxchg, __VA_ARGS__)
 206#endif
 207
 208#ifndef atomic_cmpxchg
 209#define  atomic_cmpxchg(...)                                            \
 210        __atomic_op_fence(atomic_cmpxchg, __VA_ARGS__)
 211#endif
 212#endif /* atomic_cmpxchg_relaxed */
 213
 214#ifndef atomic64_read_acquire
 215#define  atomic64_read_acquire(v)       smp_load_acquire(&(v)->counter)
 216#endif
 217
 218#ifndef atomic64_set_release
 219#define  atomic64_set_release(v, i)     smp_store_release(&(v)->counter, (i))
 220#endif
 221
 222/* atomic64_add_return_relaxed */
 223#ifndef atomic64_add_return_relaxed
 224#define  atomic64_add_return_relaxed    atomic64_add_return
 225#define  atomic64_add_return_acquire    atomic64_add_return
 226#define  atomic64_add_return_release    atomic64_add_return
 227
 228#else /* atomic64_add_return_relaxed */
 229
 230#ifndef atomic64_add_return_acquire
 231#define  atomic64_add_return_acquire(...)                               \
 232        __atomic_op_acquire(atomic64_add_return, __VA_ARGS__)
 233#endif
 234
 235#ifndef atomic64_add_return_release
 236#define  atomic64_add_return_release(...)                               \
 237        __atomic_op_release(atomic64_add_return, __VA_ARGS__)
 238#endif
 239
 240#ifndef atomic64_add_return
 241#define  atomic64_add_return(...)                                       \
 242        __atomic_op_fence(atomic64_add_return, __VA_ARGS__)
 243#endif
 244#endif /* atomic64_add_return_relaxed */
 245
 246/* atomic64_inc_return_relaxed */
 247#ifndef atomic64_inc_return_relaxed
 248#define  atomic64_inc_return_relaxed    atomic64_inc_return
 249#define  atomic64_inc_return_acquire    atomic64_inc_return
 250#define  atomic64_inc_return_release    atomic64_inc_return
 251
 252#else /* atomic64_inc_return_relaxed */
 253
 254#ifndef atomic64_inc_return_acquire
 255#define  atomic64_inc_return_acquire(...)                               \
 256        __atomic_op_acquire(atomic64_inc_return, __VA_ARGS__)
 257#endif
 258
 259#ifndef atomic64_inc_return_release
 260#define  atomic64_inc_return_release(...)                               \
 261        __atomic_op_release(atomic64_inc_return, __VA_ARGS__)
 262#endif
 263
 264#ifndef atomic64_inc_return
 265#define  atomic64_inc_return(...)                                       \
 266        __atomic_op_fence(atomic64_inc_return, __VA_ARGS__)
 267#endif
 268#endif /* atomic64_inc_return_relaxed */
 269
 270
 271/* atomic64_sub_return_relaxed */
 272#ifndef atomic64_sub_return_relaxed
 273#define  atomic64_sub_return_relaxed    atomic64_sub_return
 274#define  atomic64_sub_return_acquire    atomic64_sub_return
 275#define  atomic64_sub_return_release    atomic64_sub_return
 276
 277#else /* atomic64_sub_return_relaxed */
 278
 279#ifndef atomic64_sub_return_acquire
 280#define  atomic64_sub_return_acquire(...)                               \
 281        __atomic_op_acquire(atomic64_sub_return, __VA_ARGS__)
 282#endif
 283
 284#ifndef atomic64_sub_return_release
 285#define  atomic64_sub_return_release(...)                               \
 286        __atomic_op_release(atomic64_sub_return, __VA_ARGS__)
 287#endif
 288
 289#ifndef atomic64_sub_return
 290#define  atomic64_sub_return(...)                                       \
 291        __atomic_op_fence(atomic64_sub_return, __VA_ARGS__)
 292#endif
 293#endif /* atomic64_sub_return_relaxed */
 294
 295/* atomic64_dec_return_relaxed */
 296#ifndef atomic64_dec_return_relaxed
 297#define  atomic64_dec_return_relaxed    atomic64_dec_return
 298#define  atomic64_dec_return_acquire    atomic64_dec_return
 299#define  atomic64_dec_return_release    atomic64_dec_return
 300
 301#else /* atomic64_dec_return_relaxed */
 302
 303#ifndef atomic64_dec_return_acquire
 304#define  atomic64_dec_return_acquire(...)                               \
 305        __atomic_op_acquire(atomic64_dec_return, __VA_ARGS__)
 306#endif
 307
 308#ifndef atomic64_dec_return_release
 309#define  atomic64_dec_return_release(...)                               \
 310        __atomic_op_release(atomic64_dec_return, __VA_ARGS__)
 311#endif
 312
 313#ifndef atomic64_dec_return
 314#define  atomic64_dec_return(...)                                       \
 315        __atomic_op_fence(atomic64_dec_return, __VA_ARGS__)
 316#endif
 317#endif /* atomic64_dec_return_relaxed */
 318
 319/* atomic64_xchg_relaxed */
 320#ifndef atomic64_xchg_relaxed
 321#define  atomic64_xchg_relaxed          atomic64_xchg
 322#define  atomic64_xchg_acquire          atomic64_xchg
 323#define  atomic64_xchg_release          atomic64_xchg
 324
 325#else /* atomic64_xchg_relaxed */
 326
 327#ifndef atomic64_xchg_acquire
 328#define  atomic64_xchg_acquire(...)                                     \
 329        __atomic_op_acquire(atomic64_xchg, __VA_ARGS__)
 330#endif
 331
 332#ifndef atomic64_xchg_release
 333#define  atomic64_xchg_release(...)                                     \
 334        __atomic_op_release(atomic64_xchg, __VA_ARGS__)
 335#endif
 336
 337#ifndef atomic64_xchg
 338#define  atomic64_xchg(...)                                             \
 339        __atomic_op_fence(atomic64_xchg, __VA_ARGS__)
 340#endif
 341#endif /* atomic64_xchg_relaxed */
 342
 343/* atomic64_cmpxchg_relaxed */
 344#ifndef atomic64_cmpxchg_relaxed
 345#define  atomic64_cmpxchg_relaxed       atomic64_cmpxchg
 346#define  atomic64_cmpxchg_acquire       atomic64_cmpxchg
 347#define  atomic64_cmpxchg_release       atomic64_cmpxchg
 348
 349#else /* atomic64_cmpxchg_relaxed */
 350
 351#ifndef atomic64_cmpxchg_acquire
 352#define  atomic64_cmpxchg_acquire(...)                                  \
 353        __atomic_op_acquire(atomic64_cmpxchg, __VA_ARGS__)
 354#endif
 355
 356#ifndef atomic64_cmpxchg_release
 357#define  atomic64_cmpxchg_release(...)                                  \
 358        __atomic_op_release(atomic64_cmpxchg, __VA_ARGS__)
 359#endif
 360
 361#ifndef atomic64_cmpxchg
 362#define  atomic64_cmpxchg(...)                                          \
 363        __atomic_op_fence(atomic64_cmpxchg, __VA_ARGS__)
 364#endif
 365#endif /* atomic64_cmpxchg_relaxed */
 366
 367/* cmpxchg_relaxed */
 368#ifndef cmpxchg_relaxed
 369#define  cmpxchg_relaxed                cmpxchg
 370#define  cmpxchg_acquire                cmpxchg
 371#define  cmpxchg_release                cmpxchg
 372
 373#else /* cmpxchg_relaxed */
 374
 375#ifndef cmpxchg_acquire
 376#define  cmpxchg_acquire(...)                                           \
 377        __atomic_op_acquire(cmpxchg, __VA_ARGS__)
 378#endif
 379
 380#ifndef cmpxchg_release
 381#define  cmpxchg_release(...)                                           \
 382        __atomic_op_release(cmpxchg, __VA_ARGS__)
 383#endif
 384
 385#ifndef cmpxchg
 386#define  cmpxchg(...)                                                   \
 387        __atomic_op_fence(cmpxchg, __VA_ARGS__)
 388#endif
 389#endif /* cmpxchg_relaxed */
 390
 391/* cmpxchg64_relaxed */
 392#ifndef cmpxchg64_relaxed
 393#define  cmpxchg64_relaxed              cmpxchg64
 394#define  cmpxchg64_acquire              cmpxchg64
 395#define  cmpxchg64_release              cmpxchg64
 396
 397#else /* cmpxchg64_relaxed */
 398
 399#ifndef cmpxchg64_acquire
 400#define  cmpxchg64_acquire(...)                                         \
 401        __atomic_op_acquire(cmpxchg64, __VA_ARGS__)
 402#endif
 403
 404#ifndef cmpxchg64_release
 405#define  cmpxchg64_release(...)                                         \
 406        __atomic_op_release(cmpxchg64, __VA_ARGS__)
 407#endif
 408
 409#ifndef cmpxchg64
 410#define  cmpxchg64(...)                                                 \
 411        __atomic_op_fence(cmpxchg64, __VA_ARGS__)
 412#endif
 413#endif /* cmpxchg64_relaxed */
 414
 415/* xchg_relaxed */
 416#ifndef xchg_relaxed
 417#define  xchg_relaxed                   xchg
 418#define  xchg_acquire                   xchg
 419#define  xchg_release                   xchg
 420
 421#else /* xchg_relaxed */
 422
 423#ifndef xchg_acquire
 424#define  xchg_acquire(...)              __atomic_op_acquire(xchg, __VA_ARGS__)
 425#endif
 426
 427#ifndef xchg_release
 428#define  xchg_release(...)              __atomic_op_release(xchg, __VA_ARGS__)
 429#endif
 430
 431#ifndef xchg
 432#define  xchg(...)                      __atomic_op_fence(xchg, __VA_ARGS__)
 433#endif
 434#endif /* xchg_relaxed */
 435
 436/**
 437 * atomic_add_unless - add unless the number is already a given value
 438 * @v: pointer of type atomic_t
 439 * @a: the amount to add to v...
 440 * @u: ...unless v is equal to u.
 441 *
 442 * Atomically adds @a to @v, so long as @v was not already @u.
 443 * Returns non-zero if @v was not @u, and zero otherwise.
 444 */
 445static inline int atomic_add_unless(atomic_t *v, int a, int u)
 446{
 447        return __atomic_add_unless(v, a, u) != u;
 448}
 449
 450/**
 451 * atomic_inc_not_zero - increment unless the number is zero
 452 * @v: pointer of type atomic_t
 453 *
 454 * Atomically increments @v by 1, so long as @v is non-zero.
 455 * Returns non-zero if @v was non-zero, and zero otherwise.
 456 */
 457#ifndef atomic_inc_not_zero
 458#define atomic_inc_not_zero(v)          atomic_add_unless((v), 1, 0)
 459#endif
 460
 461#ifndef atomic_andnot
 462static inline void atomic_andnot(int i, atomic_t *v)
 463{
 464        atomic_and(~i, v);
 465}
 466#endif
 467
 468static inline __deprecated void atomic_clear_mask(unsigned int mask, atomic_t *v)
 469{
 470        atomic_andnot(mask, v);
 471}
 472
 473static inline __deprecated void atomic_set_mask(unsigned int mask, atomic_t *v)
 474{
 475        atomic_or(mask, v);
 476}
 477
 478/**
 479 * atomic_inc_not_zero_hint - increment if not null
 480 * @v: pointer of type atomic_t
 481 * @hint: probable value of the atomic before the increment
 482 *
 483 * This version of atomic_inc_not_zero() gives a hint of probable
 484 * value of the atomic. This helps processor to not read the memory
 485 * before doing the atomic read/modify/write cycle, lowering
 486 * number of bus transactions on some arches.
 487 *
 488 * Returns: 0 if increment was not done, 1 otherwise.
 489 */
 490#ifndef atomic_inc_not_zero_hint
 491static inline int atomic_inc_not_zero_hint(atomic_t *v, int hint)
 492{
 493        int val, c = hint;
 494
 495        /* sanity test, should be removed by compiler if hint is a constant */
 496        if (!hint)
 497                return atomic_inc_not_zero(v);
 498
 499        do {
 500                val = atomic_cmpxchg(v, c, c + 1);
 501                if (val == c)
 502                        return 1;
 503                c = val;
 504        } while (c);
 505
 506        return 0;
 507}
 508#endif
 509
 510#ifndef atomic_inc_unless_negative
 511static inline int atomic_inc_unless_negative(atomic_t *p)
 512{
 513        int v, v1;
 514        for (v = 0; v >= 0; v = v1) {
 515                v1 = atomic_cmpxchg(p, v, v + 1);
 516                if (likely(v1 == v))
 517                        return 1;
 518        }
 519        return 0;
 520}
 521#endif
 522
 523#ifndef atomic_dec_unless_positive
 524static inline int atomic_dec_unless_positive(atomic_t *p)
 525{
 526        int v, v1;
 527        for (v = 0; v <= 0; v = v1) {
 528                v1 = atomic_cmpxchg(p, v, v - 1);
 529                if (likely(v1 == v))
 530                        return 1;
 531        }
 532        return 0;
 533}
 534#endif
 535
 536/*
 537 * atomic_dec_if_positive - decrement by 1 if old value positive
 538 * @v: pointer of type atomic_t
 539 *
 540 * The function returns the old value of *v minus 1, even if
 541 * the atomic variable, v, was not decremented.
 542 */
 543#ifndef atomic_dec_if_positive
 544static inline int atomic_dec_if_positive(atomic_t *v)
 545{
 546        int c, old, dec;
 547        c = atomic_read(v);
 548        for (;;) {
 549                dec = c - 1;
 550                if (unlikely(dec < 0))
 551                        break;
 552                old = atomic_cmpxchg((v), c, dec);
 553                if (likely(old == c))
 554                        break;
 555                c = old;
 556        }
 557        return dec;
 558}
 559#endif
 560
 561/**
 562 * atomic_fetch_or - perform *p |= mask and return old value of *p
 563 * @p: pointer to atomic_t
 564 * @mask: mask to OR on the atomic_t
 565 */
 566#ifndef atomic_fetch_or
 567static inline int atomic_fetch_or(atomic_t *p, int mask)
 568{
 569        int old, val = atomic_read(p);
 570
 571        for (;;) {
 572                old = atomic_cmpxchg(p, val, val | mask);
 573                if (old == val)
 574                        break;
 575                val = old;
 576        }
 577
 578        return old;
 579}
 580#endif
 581
 582#ifdef CONFIG_GENERIC_ATOMIC64
 583#include <asm-generic/atomic64.h>
 584#endif
 585
 586#ifndef atomic64_andnot
 587static inline void atomic64_andnot(long long i, atomic64_t *v)
 588{
 589        atomic64_and(~i, v);
 590}
 591#endif
 592
 593#include <asm-generic/atomic-long.h>
 594
 595#endif /* _LINUX_ATOMIC_H */
 596