LXR linux/include/linux/atomic.h

   1/* Atomic operations usable in machine independent code */
   2#ifndef _LINUX_ATOMIC_H
   3#define _LINUX_ATOMIC_H
   4#include <asm/atomic.h>
   5#include <asm/barrier.h>
   6
   7/*
   8 * Relaxed variants of xchg, cmpxchg and some atomic operations.
   9 *
  10 * We support four variants:
  11 *
  12 * - Fully ordered: The default implementation, no suffix required.
  13 * - Acquire: Provides ACQUIRE semantics, _acquire suffix.
  14 * - Release: Provides RELEASE semantics, _release suffix.
  15 * - Relaxed: No ordering guarantees, _relaxed suffix.
  16 *
  17 * For compound atomics performing both a load and a store, ACQUIRE
  18 * semantics apply only to the load and RELEASE semantics only to the
  19 * store portion of the operation. Note that a failed cmpxchg_acquire
  20 * does -not- imply any memory ordering constraints.
  21 *
  22 * See Documentation/memory-barriers.txt for ACQUIRE/RELEASE definitions.
  23 */
  24
  25#ifndef atomic_read_acquire
  26#define  atomic_read_acquire(v)         smp_load_acquire(&(v)->counter)
  27#endif
  28
  29#ifndef atomic_set_release
  30#define  atomic_set_release(v, i)       smp_store_release(&(v)->counter, (i))
  31#endif
  32
  33/*
  34 * The idea here is to build acquire/release variants by adding explicit
  35 * barriers on top of the relaxed variant. In the case where the relaxed
  36 * variant is already fully ordered, no additional barriers are needed.
  37 *
  38 * Besides, if an arch has a special barrier for acquire/release, it could
  39 * implement its own __atomic_op_* and use the same framework for building
  40 * variants
  41 */
  42#ifndef __atomic_op_acquire
  43#define __atomic_op_acquire(op, args...)                                \
  44({                                                                      \
  45        typeof(op##_relaxed(args)) __ret  = op##_relaxed(args);         \
  46        smp_mb__after_atomic();                                         \
  47        __ret;                                                          \
  48})
  49#endif
  50
  51#ifndef __atomic_op_release
  52#define __atomic_op_release(op, args...)                                \
  53({                                                                      \
  54        smp_mb__before_atomic();                                        \
  55        op##_relaxed(args);                                             \
  56})
  57#endif
  58
  59#ifndef __atomic_op_fence
  60#define __atomic_op_fence(op, args...)                                  \
  61({                                                                      \
  62        typeof(op##_relaxed(args)) __ret;                               \
  63        smp_mb__before_atomic();                                        \
  64        __ret = op##_relaxed(args);                                     \
  65        smp_mb__after_atomic();                                         \
  66        __ret;                                                          \
  67})
  68#endif
  69
  70/* atomic_add_return_relaxed */
  71#ifndef atomic_add_return_relaxed
  72#define  atomic_add_return_relaxed      atomic_add_return
  73#define  atomic_add_return_acquire      atomic_add_return
  74#define  atomic_add_return_release      atomic_add_return
  75
  76#else /* atomic_add_return_relaxed */
  77
  78#ifndef atomic_add_return_acquire
  79#define  atomic_add_return_acquire(...)                                 \
  80        __atomic_op_acquire(atomic_add_return, __VA_ARGS__)
  81#endif
  82
  83#ifndef atomic_add_return_release
  84#define  atomic_add_return_release(...)                                 \
  85        __atomic_op_release(atomic_add_return, __VA_ARGS__)
  86#endif
  87
  88#ifndef atomic_add_return
  89#define  atomic_add_return(...)                                         \
  90        __atomic_op_fence(atomic_add_return, __VA_ARGS__)
  91#endif
  92#endif /* atomic_add_return_relaxed */
  93
  94/* atomic_inc_return_relaxed */
  95#ifndef atomic_inc_return_relaxed
  96#define  atomic_inc_return_relaxed      atomic_inc_return
  97#define  atomic_inc_return_acquire      atomic_inc_return
  98#define  atomic_inc_return_release      atomic_inc_return
  99
 100#else /* atomic_inc_return_relaxed */
 101
 102#ifndef atomic_inc_return_acquire
 103#define  atomic_inc_return_acquire(...)                                 \
 104        __atomic_op_acquire(atomic_inc_return, __VA_ARGS__)
 105#endif
 106
 107#ifndef atomic_inc_return_release
 108#define  atomic_inc_return_release(...)                                 \
 109        __atomic_op_release(atomic_inc_return, __VA_ARGS__)
 110#endif
 111
 112#ifndef atomic_inc_return
 113#define  atomic_inc_return(...)                                         \
 114        __atomic_op_fence(atomic_inc_return, __VA_ARGS__)
 115#endif
 116#endif /* atomic_inc_return_relaxed */
 117
 118/* atomic_sub_return_relaxed */
 119#ifndef atomic_sub_return_relaxed
 120#define  atomic_sub_return_relaxed      atomic_sub_return
 121#define  atomic_sub_return_acquire      atomic_sub_return
 122#define  atomic_sub_return_release      atomic_sub_return
 123
 124#else /* atomic_sub_return_relaxed */
 125
 126#ifndef atomic_sub_return_acquire
 127#define  atomic_sub_return_acquire(...)                                 \
 128        __atomic_op_acquire(atomic_sub_return, __VA_ARGS__)
 129#endif
 130
 131#ifndef atomic_sub_return_release
 132#define  atomic_sub_return_release(...)                                 \
 133        __atomic_op_release(atomic_sub_return, __VA_ARGS__)
 134#endif
 135
 136#ifndef atomic_sub_return
 137#define  atomic_sub_return(...)                                         \
 138        __atomic_op_fence(atomic_sub_return, __VA_ARGS__)
 139#endif
 140#endif /* atomic_sub_return_relaxed */
 141
 142/* atomic_dec_return_relaxed */
 143#ifndef atomic_dec_return_relaxed
 144#define  atomic_dec_return_relaxed      atomic_dec_return
 145#define  atomic_dec_return_acquire      atomic_dec_return
 146#define  atomic_dec_return_release      atomic_dec_return
 147
 148#else /* atomic_dec_return_relaxed */
 149
 150#ifndef atomic_dec_return_acquire
 151#define  atomic_dec_return_acquire(...)                                 \
 152        __atomic_op_acquire(atomic_dec_return, __VA_ARGS__)
 153#endif
 154
 155#ifndef atomic_dec_return_release
 156#define  atomic_dec_return_release(...)                                 \
 157        __atomic_op_release(atomic_dec_return, __VA_ARGS__)
 158#endif
 159
 160#ifndef atomic_dec_return
 161#define  atomic_dec_return(...)                                         \
 162        __atomic_op_fence(atomic_dec_return, __VA_ARGS__)
 163#endif
 164#endif /* atomic_dec_return_relaxed */
 165
 166
 167/* atomic_fetch_add_relaxed */
 168#ifndef atomic_fetch_add_relaxed
 169#define atomic_fetch_add_relaxed        atomic_fetch_add
 170#define atomic_fetch_add_acquire        atomic_fetch_add
 171#define atomic_fetch_add_release        atomic_fetch_add
 172
 173#else /* atomic_fetch_add_relaxed */
 174
 175#ifndef atomic_fetch_add_acquire
 176#define atomic_fetch_add_acquire(...)                                   \
 177        __atomic_op_acquire(atomic_fetch_add, __VA_ARGS__)
 178#endif
 179
 180#ifndef atomic_fetch_add_release
 181#define atomic_fetch_add_release(...)                                   \
 182        __atomic_op_release(atomic_fetch_add, __VA_ARGS__)
 183#endif
 184
 185#ifndef atomic_fetch_add
 186#define atomic_fetch_add(...)                                           \
 187        __atomic_op_fence(atomic_fetch_add, __VA_ARGS__)
 188#endif
 189#endif /* atomic_fetch_add_relaxed */
 190
 191/* atomic_fetch_inc_relaxed */
 192#ifndef atomic_fetch_inc_relaxed
 193
 194#ifndef atomic_fetch_inc
 195#define atomic_fetch_inc(v)             atomic_fetch_add(1, (v))
 196#define atomic_fetch_inc_relaxed(v)     atomic_fetch_add_relaxed(1, (v))
 197#define atomic_fetch_inc_acquire(v)     atomic_fetch_add_acquire(1, (v))
 198#define atomic_fetch_inc_release(v)     atomic_fetch_add_release(1, (v))
 199#else /* atomic_fetch_inc */
 200#define atomic_fetch_inc_relaxed        atomic_fetch_inc
 201#define atomic_fetch_inc_acquire        atomic_fetch_inc
 202#define atomic_fetch_inc_release        atomic_fetch_inc
 203#endif /* atomic_fetch_inc */
 204
 205#else /* atomic_fetch_inc_relaxed */
 206
 207#ifndef atomic_fetch_inc_acquire
 208#define atomic_fetch_inc_acquire(...)                                   \
 209        __atomic_op_acquire(atomic_fetch_inc, __VA_ARGS__)
 210#endif
 211
 212#ifndef atomic_fetch_inc_release
 213#define atomic_fetch_inc_release(...)                                   \
 214        __atomic_op_release(atomic_fetch_inc, __VA_ARGS__)
 215#endif
 216
 217#ifndef atomic_fetch_inc
 218#define atomic_fetch_inc(...)                                           \
 219        __atomic_op_fence(atomic_fetch_inc, __VA_ARGS__)
 220#endif
 221#endif /* atomic_fetch_inc_relaxed */
 222
 223/* atomic_fetch_sub_relaxed */
 224#ifndef atomic_fetch_sub_relaxed
 225#define atomic_fetch_sub_relaxed        atomic_fetch_sub
 226#define atomic_fetch_sub_acquire        atomic_fetch_sub
 227#define atomic_fetch_sub_release        atomic_fetch_sub
 228
 229#else /* atomic_fetch_sub_relaxed */
 230
 231#ifndef atomic_fetch_sub_acquire
 232#define atomic_fetch_sub_acquire(...)                                   \
 233        __atomic_op_acquire(atomic_fetch_sub, __VA_ARGS__)
 234#endif
 235
 236#ifndef atomic_fetch_sub_release
 237#define atomic_fetch_sub_release(...)                                   \
 238        __atomic_op_release(atomic_fetch_sub, __VA_ARGS__)
 239#endif
 240
 241#ifndef atomic_fetch_sub
 242#define atomic_fetch_sub(...)                                           \
 243        __atomic_op_fence(atomic_fetch_sub, __VA_ARGS__)
 244#endif
 245#endif /* atomic_fetch_sub_relaxed */
 246
 247/* atomic_fetch_dec_relaxed */
 248#ifndef atomic_fetch_dec_relaxed
 249
 250#ifndef atomic_fetch_dec
 251#define atomic_fetch_dec(v)             atomic_fetch_sub(1, (v))
 252#define atomic_fetch_dec_relaxed(v)     atomic_fetch_sub_relaxed(1, (v))
 253#define atomic_fetch_dec_acquire(v)     atomic_fetch_sub_acquire(1, (v))
 254#define atomic_fetch_dec_release(v)     atomic_fetch_sub_release(1, (v))
 255#else /* atomic_fetch_dec */
 256#define atomic_fetch_dec_relaxed        atomic_fetch_dec
 257#define atomic_fetch_dec_acquire        atomic_fetch_dec
 258#define atomic_fetch_dec_release        atomic_fetch_dec
 259#endif /* atomic_fetch_dec */
 260
 261#else /* atomic_fetch_dec_relaxed */
 262
 263#ifndef atomic_fetch_dec_acquire
 264#define atomic_fetch_dec_acquire(...)                                   \
 265        __atomic_op_acquire(atomic_fetch_dec, __VA_ARGS__)
 266#endif
 267
 268#ifndef atomic_fetch_dec_release
 269#define atomic_fetch_dec_release(...)                                   \
 270        __atomic_op_release(atomic_fetch_dec, __VA_ARGS__)
 271#endif
 272
 273#ifndef atomic_fetch_dec
 274#define atomic_fetch_dec(...)                                           \
 275        __atomic_op_fence(atomic_fetch_dec, __VA_ARGS__)
 276#endif
 277#endif /* atomic_fetch_dec_relaxed */
 278
 279/* atomic_fetch_or_relaxed */
 280#ifndef atomic_fetch_or_relaxed
 281#define atomic_fetch_or_relaxed atomic_fetch_or
 282#define atomic_fetch_or_acquire atomic_fetch_or
 283#define atomic_fetch_or_release atomic_fetch_or
 284
 285#else /* atomic_fetch_or_relaxed */
 286
 287#ifndef atomic_fetch_or_acquire
 288#define atomic_fetch_or_acquire(...)                                    \
 289        __atomic_op_acquire(atomic_fetch_or, __VA_ARGS__)
 290#endif
 291
 292#ifndef atomic_fetch_or_release
 293#define atomic_fetch_or_release(...)                                    \
 294        __atomic_op_release(atomic_fetch_or, __VA_ARGS__)
 295#endif
 296
 297#ifndef atomic_fetch_or
 298#define atomic_fetch_or(...)                                            \
 299        __atomic_op_fence(atomic_fetch_or, __VA_ARGS__)
 300#endif
 301#endif /* atomic_fetch_or_relaxed */
 302
 303/* atomic_fetch_and_relaxed */
 304#ifndef atomic_fetch_and_relaxed
 305#define atomic_fetch_and_relaxed        atomic_fetch_and
 306#define atomic_fetch_and_acquire        atomic_fetch_and
 307#define atomic_fetch_and_release        atomic_fetch_and
 308
 309#else /* atomic_fetch_and_relaxed */
 310
 311#ifndef atomic_fetch_and_acquire
 312#define atomic_fetch_and_acquire(...)                                   \
 313        __atomic_op_acquire(atomic_fetch_and, __VA_ARGS__)
 314#endif
 315
 316#ifndef atomic_fetch_and_release
 317#define atomic_fetch_and_release(...)                                   \
 318        __atomic_op_release(atomic_fetch_and, __VA_ARGS__)
 319#endif
 320
 321#ifndef atomic_fetch_and
 322#define atomic_fetch_and(...)                                           \
 323        __atomic_op_fence(atomic_fetch_and, __VA_ARGS__)
 324#endif
 325#endif /* atomic_fetch_and_relaxed */
 326
 327#ifdef atomic_andnot
 328/* atomic_fetch_andnot_relaxed */
 329#ifndef atomic_fetch_andnot_relaxed
 330#define atomic_fetch_andnot_relaxed     atomic_fetch_andnot
 331#define atomic_fetch_andnot_acquire     atomic_fetch_andnot
 332#define atomic_fetch_andnot_release     atomic_fetch_andnot
 333
 334#else /* atomic_fetch_andnot_relaxed */
 335
 336#ifndef atomic_fetch_andnot_acquire
 337#define atomic_fetch_andnot_acquire(...)                                        \
 338        __atomic_op_acquire(atomic_fetch_andnot, __VA_ARGS__)
 339#endif
 340
 341#ifndef atomic_fetch_andnot_release
 342#define atomic_fetch_andnot_release(...)                                        \
 343        __atomic_op_release(atomic_fetch_andnot, __VA_ARGS__)
 344#endif
 345
 346#ifndef atomic_fetch_andnot
 347#define atomic_fetch_andnot(...)                                                \
 348        __atomic_op_fence(atomic_fetch_andnot, __VA_ARGS__)
 349#endif
 350#endif /* atomic_fetch_andnot_relaxed */
 351#endif /* atomic_andnot */
 352
 353/* atomic_fetch_xor_relaxed */
 354#ifndef atomic_fetch_xor_relaxed
 355#define atomic_fetch_xor_relaxed        atomic_fetch_xor
 356#define atomic_fetch_xor_acquire        atomic_fetch_xor
 357#define atomic_fetch_xor_release        atomic_fetch_xor
 358
 359#else /* atomic_fetch_xor_relaxed */
 360
 361#ifndef atomic_fetch_xor_acquire
 362#define atomic_fetch_xor_acquire(...)                                   \
 363        __atomic_op_acquire(atomic_fetch_xor, __VA_ARGS__)
 364#endif
 365
 366#ifndef atomic_fetch_xor_release
 367#define atomic_fetch_xor_release(...)                                   \
 368        __atomic_op_release(atomic_fetch_xor, __VA_ARGS__)
 369#endif
 370
 371#ifndef atomic_fetch_xor
 372#define atomic_fetch_xor(...)                                           \
 373        __atomic_op_fence(atomic_fetch_xor, __VA_ARGS__)
 374#endif
 375#endif /* atomic_fetch_xor_relaxed */
 376
 377
 378/* atomic_xchg_relaxed */
 379#ifndef atomic_xchg_relaxed
 380#define  atomic_xchg_relaxed            atomic_xchg
 381#define  atomic_xchg_acquire            atomic_xchg
 382#define  atomic_xchg_release            atomic_xchg
 383
 384#else /* atomic_xchg_relaxed */
 385
 386#ifndef atomic_xchg_acquire
 387#define  atomic_xchg_acquire(...)                                       \
 388        __atomic_op_acquire(atomic_xchg, __VA_ARGS__)
 389#endif
 390
 391#ifndef atomic_xchg_release
 392#define  atomic_xchg_release(...)                                       \
 393        __atomic_op_release(atomic_xchg, __VA_ARGS__)
 394#endif
 395
 396#ifndef atomic_xchg
 397#define  atomic_xchg(...)                                               \
 398        __atomic_op_fence(atomic_xchg, __VA_ARGS__)
 399#endif
 400#endif /* atomic_xchg_relaxed */
 401
 402/* atomic_cmpxchg_relaxed */
 403#ifndef atomic_cmpxchg_relaxed
 404#define  atomic_cmpxchg_relaxed         atomic_cmpxchg
 405#define  atomic_cmpxchg_acquire         atomic_cmpxchg
 406#define  atomic_cmpxchg_release         atomic_cmpxchg
 407
 408#else /* atomic_cmpxchg_relaxed */
 409
 410#ifndef atomic_cmpxchg_acquire
 411#define  atomic_cmpxchg_acquire(...)                                    \
 412        __atomic_op_acquire(atomic_cmpxchg, __VA_ARGS__)
 413#endif
 414
 415#ifndef atomic_cmpxchg_release
 416#define  atomic_cmpxchg_release(...)                                    \
 417        __atomic_op_release(atomic_cmpxchg, __VA_ARGS__)
 418#endif
 419
 420#ifndef atomic_cmpxchg
 421#define  atomic_cmpxchg(...)                                            \
 422        __atomic_op_fence(atomic_cmpxchg, __VA_ARGS__)
 423#endif
 424#endif /* atomic_cmpxchg_relaxed */
 425
 426/* cmpxchg_relaxed */
 427#ifndef cmpxchg_relaxed
 428#define  cmpxchg_relaxed                cmpxchg
 429#define  cmpxchg_acquire                cmpxchg
 430#define  cmpxchg_release                cmpxchg
 431
 432#else /* cmpxchg_relaxed */
 433
 434#ifndef cmpxchg_acquire
 435#define  cmpxchg_acquire(...)                                           \
 436        __atomic_op_acquire(cmpxchg, __VA_ARGS__)
 437#endif
 438
 439#ifndef cmpxchg_release
 440#define  cmpxchg_release(...)                                           \
 441        __atomic_op_release(cmpxchg, __VA_ARGS__)
 442#endif
 443
 444#ifndef cmpxchg
 445#define  cmpxchg(...)                                                   \
 446        __atomic_op_fence(cmpxchg, __VA_ARGS__)
 447#endif
 448#endif /* cmpxchg_relaxed */
 449
 450/* cmpxchg64_relaxed */
 451#ifndef cmpxchg64_relaxed
 452#define  cmpxchg64_relaxed              cmpxchg64
 453#define  cmpxchg64_acquire              cmpxchg64
 454#define  cmpxchg64_release              cmpxchg64
 455
 456#else /* cmpxchg64_relaxed */
 457
 458#ifndef cmpxchg64_acquire
 459#define  cmpxchg64_acquire(...)                                         \
 460        __atomic_op_acquire(cmpxchg64, __VA_ARGS__)
 461#endif
 462
 463#ifndef cmpxchg64_release
 464#define  cmpxchg64_release(...)                                         \
 465        __atomic_op_release(cmpxchg64, __VA_ARGS__)
 466#endif
 467
 468#ifndef cmpxchg64
 469#define  cmpxchg64(...)                                                 \
 470        __atomic_op_fence(cmpxchg64, __VA_ARGS__)
 471#endif
 472#endif /* cmpxchg64_relaxed */
 473
 474/* xchg_relaxed */
 475#ifndef xchg_relaxed
 476#define  xchg_relaxed                   xchg
 477#define  xchg_acquire                   xchg
 478#define  xchg_release                   xchg
 479
 480#else /* xchg_relaxed */
 481
 482#ifndef xchg_acquire
 483#define  xchg_acquire(...)              __atomic_op_acquire(xchg, __VA_ARGS__)
 484#endif
 485
 486#ifndef xchg_release
 487#define  xchg_release(...)              __atomic_op_release(xchg, __VA_ARGS__)
 488#endif
 489
 490#ifndef xchg
 491#define  xchg(...)                      __atomic_op_fence(xchg, __VA_ARGS__)
 492#endif
 493#endif /* xchg_relaxed */
 494
 495/**
 496 * atomic_add_unless - add unless the number is already a given value
 497 * @v: pointer of type atomic_t
 498 * @a: the amount to add to v...
 499 * @u: ...unless v is equal to u.
 500 *
 501 * Atomically adds @a to @v, so long as @v was not already @u.
 502 * Returns non-zero if @v was not @u, and zero otherwise.
 503 */
 504static inline int atomic_add_unless(atomic_t *v, int a, int u)
 505{
 506        return __atomic_add_unless(v, a, u) != u;
 507}
 508
 509/**
 510 * atomic_inc_not_zero - increment unless the number is zero
 511 * @v: pointer of type atomic_t
 512 *
 513 * Atomically increments @v by 1, so long as @v is non-zero.
 514 * Returns non-zero if @v was non-zero, and zero otherwise.
 515 */
 516#ifndef atomic_inc_not_zero
 517#define atomic_inc_not_zero(v)          atomic_add_unless((v), 1, 0)
 518#endif
 519
 520#ifndef atomic_andnot
 521static inline void atomic_andnot(int i, atomic_t *v)
 522{
 523        atomic_and(~i, v);
 524}
 525
 526static inline int atomic_fetch_andnot(int i, atomic_t *v)
 527{
 528        return atomic_fetch_and(~i, v);
 529}
 530
 531static inline int atomic_fetch_andnot_relaxed(int i, atomic_t *v)
 532{
 533        return atomic_fetch_and_relaxed(~i, v);
 534}
 535
 536static inline int atomic_fetch_andnot_acquire(int i, atomic_t *v)
 537{
 538        return atomic_fetch_and_acquire(~i, v);
 539}
 540
 541static inline int atomic_fetch_andnot_release(int i, atomic_t *v)
 542{
 543        return atomic_fetch_and_release(~i, v);
 544}
 545#endif
 546
 547/**
 548 * atomic_inc_not_zero_hint - increment if not null
 549 * @v: pointer of type atomic_t
 550 * @hint: probable value of the atomic before the increment
 551 *
 552 * This version of atomic_inc_not_zero() gives a hint of probable
 553 * value of the atomic. This helps processor to not read the memory
 554 * before doing the atomic read/modify/write cycle, lowering
 555 * number of bus transactions on some arches.
 556 *
 557 * Returns: 0 if increment was not done, 1 otherwise.
 558 */
 559#ifndef atomic_inc_not_zero_hint
 560static inline int atomic_inc_not_zero_hint(atomic_t *v, int hint)
 561{
 562        int val, c = hint;
 563
 564        /* sanity test, should be removed by compiler if hint is a constant */
 565        if (!hint)
 566                return atomic_inc_not_zero(v);
 567
 568        do {
 569                val = atomic_cmpxchg(v, c, c + 1);
 570                if (val == c)
 571                        return 1;
 572                c = val;
 573        } while (c);
 574
 575        return 0;
 576}
 577#endif
 578
 579#ifndef atomic_inc_unless_negative
 580static inline int atomic_inc_unless_negative(atomic_t *p)
 581{
 582        int v, v1;
 583        for (v = 0; v >= 0; v = v1) {
 584                v1 = atomic_cmpxchg(p, v, v + 1);
 585                if (likely(v1 == v))
 586                        return 1;
 587        }
 588        return 0;
 589}
 590#endif
 591
 592#ifndef atomic_dec_unless_positive
 593static inline int atomic_dec_unless_positive(atomic_t *p)
 594{
 595        int v, v1;
 596        for (v = 0; v <= 0; v = v1) {
 597                v1 = atomic_cmpxchg(p, v, v - 1);
 598                if (likely(v1 == v))
 599                        return 1;
 600        }
 601        return 0;
 602}
 603#endif
 604
 605/*
 606 * atomic_dec_if_positive - decrement by 1 if old value positive
 607 * @v: pointer of type atomic_t
 608 *
 609 * The function returns the old value of *v minus 1, even if
 610 * the atomic variable, v, was not decremented.
 611 */
 612#ifndef atomic_dec_if_positive
 613static inline int atomic_dec_if_positive(atomic_t *v)
 614{
 615        int c, old, dec;
 616        c = atomic_read(v);
 617        for (;;) {
 618                dec = c - 1;
 619                if (unlikely(dec < 0))
 620                        break;
 621                old = atomic_cmpxchg((v), c, dec);
 622                if (likely(old == c))
 623                        break;
 624                c = old;
 625        }
 626        return dec;
 627}
 628#endif
 629
 630#ifdef CONFIG_GENERIC_ATOMIC64
 631#include <asm-generic/atomic64.h>
 632#endif
 633
 634#ifndef atomic64_read_acquire
 635#define  atomic64_read_acquire(v)       smp_load_acquire(&(v)->counter)
 636#endif
 637
 638#ifndef atomic64_set_release
 639#define  atomic64_set_release(v, i)     smp_store_release(&(v)->counter, (i))
 640#endif
 641
 642/* atomic64_add_return_relaxed */
 643#ifndef atomic64_add_return_relaxed
 644#define  atomic64_add_return_relaxed    atomic64_add_return
 645#define  atomic64_add_return_acquire    atomic64_add_return
 646#define  atomic64_add_return_release    atomic64_add_return
 647
 648#else /* atomic64_add_return_relaxed */
 649
 650#ifndef atomic64_add_return_acquire
 651#define  atomic64_add_return_acquire(...)                               \
 652        __atomic_op_acquire(atomic64_add_return, __VA_ARGS__)
 653#endif
 654
 655#ifndef atomic64_add_return_release
 656#define  atomic64_add_return_release(...)                               \
 657        __atomic_op_release(atomic64_add_return, __VA_ARGS__)
 658#endif
 659
 660#ifndef atomic64_add_return
 661#define  atomic64_add_return(...)                                       \
 662        __atomic_op_fence(atomic64_add_return, __VA_ARGS__)
 663#endif
 664#endif /* atomic64_add_return_relaxed */
 665
 666/* atomic64_inc_return_relaxed */
 667#ifndef atomic64_inc_return_relaxed
 668#define  atomic64_inc_return_relaxed    atomic64_inc_return
 669#define  atomic64_inc_return_acquire    atomic64_inc_return
 670#define  atomic64_inc_return_release    atomic64_inc_return
 671
 672#else /* atomic64_inc_return_relaxed */
 673
 674#ifndef atomic64_inc_return_acquire
 675#define  atomic64_inc_return_acquire(...)                               \
 676        __atomic_op_acquire(atomic64_inc_return, __VA_ARGS__)
 677#endif
 678
 679#ifndef atomic64_inc_return_release
 680#define  atomic64_inc_return_release(...)                               \
 681        __atomic_op_release(atomic64_inc_return, __VA_ARGS__)
 682#endif
 683
 684#ifndef atomic64_inc_return
 685#define  atomic64_inc_return(...)                                       \
 686        __atomic_op_fence(atomic64_inc_return, __VA_ARGS__)
 687#endif
 688#endif /* atomic64_inc_return_relaxed */
 689
 690
 691/* atomic64_sub_return_relaxed */
 692#ifndef atomic64_sub_return_relaxed
 693#define  atomic64_sub_return_relaxed    atomic64_sub_return
 694#define  atomic64_sub_return_acquire    atomic64_sub_return
 695#define  atomic64_sub_return_release    atomic64_sub_return
 696
 697#else /* atomic64_sub_return_relaxed */
 698
 699#ifndef atomic64_sub_return_acquire
 700#define  atomic64_sub_return_acquire(...)                               \
 701        __atomic_op_acquire(atomic64_sub_return, __VA_ARGS__)
 702#endif
 703
 704#ifndef atomic64_sub_return_release
 705#define  atomic64_sub_return_release(...)                               \
 706        __atomic_op_release(atomic64_sub_return, __VA_ARGS__)
 707#endif
 708
 709#ifndef atomic64_sub_return
 710#define  atomic64_sub_return(...)                                       \
 711        __atomic_op_fence(atomic64_sub_return, __VA_ARGS__)
 712#endif
 713#endif /* atomic64_sub_return_relaxed */
 714
 715/* atomic64_dec_return_relaxed */
 716#ifndef atomic64_dec_return_relaxed
 717#define  atomic64_dec_return_relaxed    atomic64_dec_return
 718#define  atomic64_dec_return_acquire    atomic64_dec_return
 719#define  atomic64_dec_return_release    atomic64_dec_return
 720
 721#else /* atomic64_dec_return_relaxed */
 722
 723#ifndef atomic64_dec_return_acquire
 724#define  atomic64_dec_return_acquire(...)                               \
 725        __atomic_op_acquire(atomic64_dec_return, __VA_ARGS__)
 726#endif
 727
 728#ifndef atomic64_dec_return_release
 729#define  atomic64_dec_return_release(...)                               \
 730        __atomic_op_release(atomic64_dec_return, __VA_ARGS__)
 731#endif
 732
 733#ifndef atomic64_dec_return
 734#define  atomic64_dec_return(...)                                       \
 735        __atomic_op_fence(atomic64_dec_return, __VA_ARGS__)
 736#endif
 737#endif /* atomic64_dec_return_relaxed */
 738
 739
 740/* atomic64_fetch_add_relaxed */
 741#ifndef atomic64_fetch_add_relaxed
 742#define atomic64_fetch_add_relaxed      atomic64_fetch_add
 743#define atomic64_fetch_add_acquire      atomic64_fetch_add
 744#define atomic64_fetch_add_release      atomic64_fetch_add
 745
 746#else /* atomic64_fetch_add_relaxed */
 747
 748#ifndef atomic64_fetch_add_acquire
 749#define atomic64_fetch_add_acquire(...)                                 \
 750        __atomic_op_acquire(atomic64_fetch_add, __VA_ARGS__)
 751#endif
 752
 753#ifndef atomic64_fetch_add_release
 754#define atomic64_fetch_add_release(...)                                 \
 755        __atomic_op_release(atomic64_fetch_add, __VA_ARGS__)
 756#endif
 757
 758#ifndef atomic64_fetch_add
 759#define atomic64_fetch_add(...)                                         \
 760        __atomic_op_fence(atomic64_fetch_add, __VA_ARGS__)
 761#endif
 762#endif /* atomic64_fetch_add_relaxed */
 763
 764/* atomic64_fetch_inc_relaxed */
 765#ifndef atomic64_fetch_inc_relaxed
 766
 767#ifndef atomic64_fetch_inc
 768#define atomic64_fetch_inc(v)           atomic64_fetch_add(1, (v))
 769#define atomic64_fetch_inc_relaxed(v)   atomic64_fetch_add_relaxed(1, (v))
 770#define atomic64_fetch_inc_acquire(v)   atomic64_fetch_add_acquire(1, (v))
 771#define atomic64_fetch_inc_release(v)   atomic64_fetch_add_release(1, (v))
 772#else /* atomic64_fetch_inc */
 773#define atomic64_fetch_inc_relaxed      atomic64_fetch_inc
 774#define atomic64_fetch_inc_acquire      atomic64_fetch_inc
 775#define atomic64_fetch_inc_release      atomic64_fetch_inc
 776#endif /* atomic64_fetch_inc */
 777
 778#else /* atomic64_fetch_inc_relaxed */
 779
 780#ifndef atomic64_fetch_inc_acquire
 781#define atomic64_fetch_inc_acquire(...)                                 \
 782        __atomic_op_acquire(atomic64_fetch_inc, __VA_ARGS__)
 783#endif
 784
 785#ifndef atomic64_fetch_inc_release
 786#define atomic64_fetch_inc_release(...)                                 \
 787        __atomic_op_release(atomic64_fetch_inc, __VA_ARGS__)
 788#endif
 789
 790#ifndef atomic64_fetch_inc
 791#define atomic64_fetch_inc(...)                                         \
 792        __atomic_op_fence(atomic64_fetch_inc, __VA_ARGS__)
 793#endif
 794#endif /* atomic64_fetch_inc_relaxed */
 795
 796/* atomic64_fetch_sub_relaxed */
 797#ifndef atomic64_fetch_sub_relaxed
 798#define atomic64_fetch_sub_relaxed      atomic64_fetch_sub
 799#define atomic64_fetch_sub_acquire      atomic64_fetch_sub
 800#define atomic64_fetch_sub_release      atomic64_fetch_sub
 801
 802#else /* atomic64_fetch_sub_relaxed */
 803
 804#ifndef atomic64_fetch_sub_acquire
 805#define atomic64_fetch_sub_acquire(...)                                 \
 806        __atomic_op_acquire(atomic64_fetch_sub, __VA_ARGS__)
 807#endif
 808
 809#ifndef atomic64_fetch_sub_release
 810#define atomic64_fetch_sub_release(...)                                 \
 811        __atomic_op_release(atomic64_fetch_sub, __VA_ARGS__)
 812#endif
 813
 814#ifndef atomic64_fetch_sub
 815#define atomic64_fetch_sub(...)                                         \
 816        __atomic_op_fence(atomic64_fetch_sub, __VA_ARGS__)
 817#endif
 818#endif /* atomic64_fetch_sub_relaxed */
 819
 820/* atomic64_fetch_dec_relaxed */
 821#ifndef atomic64_fetch_dec_relaxed
 822
 823#ifndef atomic64_fetch_dec
 824#define atomic64_fetch_dec(v)           atomic64_fetch_sub(1, (v))
 825#define atomic64_fetch_dec_relaxed(v)   atomic64_fetch_sub_relaxed(1, (v))
 826#define atomic64_fetch_dec_acquire(v)   atomic64_fetch_sub_acquire(1, (v))
 827#define atomic64_fetch_dec_release(v)   atomic64_fetch_sub_release(1, (v))
 828#else /* atomic64_fetch_dec */
 829#define atomic64_fetch_dec_relaxed      atomic64_fetch_dec
 830#define atomic64_fetch_dec_acquire      atomic64_fetch_dec
 831#define atomic64_fetch_dec_release      atomic64_fetch_dec
 832#endif /* atomic64_fetch_dec */
 833
 834#else /* atomic64_fetch_dec_relaxed */
 835
 836#ifndef atomic64_fetch_dec_acquire
 837#define atomic64_fetch_dec_acquire(...)                                 \
 838        __atomic_op_acquire(atomic64_fetch_dec, __VA_ARGS__)
 839#endif
 840
 841#ifndef atomic64_fetch_dec_release
 842#define atomic64_fetch_dec_release(...)                                 \
 843        __atomic_op_release(atomic64_fetch_dec, __VA_ARGS__)
 844#endif
 845
 846#ifndef atomic64_fetch_dec
 847#define atomic64_fetch_dec(...)                                         \
 848        __atomic_op_fence(atomic64_fetch_dec, __VA_ARGS__)
 849#endif
 850#endif /* atomic64_fetch_dec_relaxed */
 851
 852/* atomic64_fetch_or_relaxed */
 853#ifndef atomic64_fetch_or_relaxed
 854#define atomic64_fetch_or_relaxed       atomic64_fetch_or
 855#define atomic64_fetch_or_acquire       atomic64_fetch_or
 856#define atomic64_fetch_or_release       atomic64_fetch_or
 857
 858#else /* atomic64_fetch_or_relaxed */
 859
 860#ifndef atomic64_fetch_or_acquire
 861#define atomic64_fetch_or_acquire(...)                                  \
 862        __atomic_op_acquire(atomic64_fetch_or, __VA_ARGS__)
 863#endif
 864
 865#ifndef atomic64_fetch_or_release
 866#define atomic64_fetch_or_release(...)                                  \
 867        __atomic_op_release(atomic64_fetch_or, __VA_ARGS__)
 868#endif
 869
 870#ifndef atomic64_fetch_or
 871#define atomic64_fetch_or(...)                                          \
 872        __atomic_op_fence(atomic64_fetch_or, __VA_ARGS__)
 873#endif
 874#endif /* atomic64_fetch_or_relaxed */
 875
 876/* atomic64_fetch_and_relaxed */
 877#ifndef atomic64_fetch_and_relaxed
 878#define atomic64_fetch_and_relaxed      atomic64_fetch_and
 879#define atomic64_fetch_and_acquire      atomic64_fetch_and
 880#define atomic64_fetch_and_release      atomic64_fetch_and
 881
 882#else /* atomic64_fetch_and_relaxed */
 883
 884#ifndef atomic64_fetch_and_acquire
 885#define atomic64_fetch_and_acquire(...)                                 \
 886        __atomic_op_acquire(atomic64_fetch_and, __VA_ARGS__)
 887#endif
 888
 889#ifndef atomic64_fetch_and_release
 890#define atomic64_fetch_and_release(...)                                 \
 891        __atomic_op_release(atomic64_fetch_and, __VA_ARGS__)
 892#endif
 893
 894#ifndef atomic64_fetch_and
 895#define atomic64_fetch_and(...)                                         \
 896        __atomic_op_fence(atomic64_fetch_and, __VA_ARGS__)
 897#endif
 898#endif /* atomic64_fetch_and_relaxed */
 899
 900#ifdef atomic64_andnot
 901/* atomic64_fetch_andnot_relaxed */
 902#ifndef atomic64_fetch_andnot_relaxed
 903#define atomic64_fetch_andnot_relaxed   atomic64_fetch_andnot
 904#define atomic64_fetch_andnot_acquire   atomic64_fetch_andnot
 905#define atomic64_fetch_andnot_release   atomic64_fetch_andnot
 906
 907#else /* atomic64_fetch_andnot_relaxed */
 908
 909#ifndef atomic64_fetch_andnot_acquire
 910#define atomic64_fetch_andnot_acquire(...)                                      \
 911        __atomic_op_acquire(atomic64_fetch_andnot, __VA_ARGS__)
 912#endif
 913
 914#ifndef atomic64_fetch_andnot_release
 915#define atomic64_fetch_andnot_release(...)                                      \
 916        __atomic_op_release(atomic64_fetch_andnot, __VA_ARGS__)
 917#endif
 918
 919#ifndef atomic64_fetch_andnot
 920#define atomic64_fetch_andnot(...)                                              \
 921        __atomic_op_fence(atomic64_fetch_andnot, __VA_ARGS__)
 922#endif
 923#endif /* atomic64_fetch_andnot_relaxed */
 924#endif /* atomic64_andnot */
 925
 926/* atomic64_fetch_xor_relaxed */
 927#ifndef atomic64_fetch_xor_relaxed
 928#define atomic64_fetch_xor_relaxed      atomic64_fetch_xor
 929#define atomic64_fetch_xor_acquire      atomic64_fetch_xor
 930#define atomic64_fetch_xor_release      atomic64_fetch_xor
 931
 932#else /* atomic64_fetch_xor_relaxed */
 933
 934#ifndef atomic64_fetch_xor_acquire
 935#define atomic64_fetch_xor_acquire(...)                                 \
 936        __atomic_op_acquire(atomic64_fetch_xor, __VA_ARGS__)
 937#endif
 938
 939#ifndef atomic64_fetch_xor_release
 940#define atomic64_fetch_xor_release(...)                                 \
 941        __atomic_op_release(atomic64_fetch_xor, __VA_ARGS__)
 942#endif
 943
 944#ifndef atomic64_fetch_xor
 945#define atomic64_fetch_xor(...)                                         \
 946        __atomic_op_fence(atomic64_fetch_xor, __VA_ARGS__)
 947#endif
 948#endif /* atomic64_fetch_xor_relaxed */
 949
 950
 951/* atomic64_xchg_relaxed */
 952#ifndef atomic64_xchg_relaxed
 953#define  atomic64_xchg_relaxed          atomic64_xchg
 954#define  atomic64_xchg_acquire          atomic64_xchg
 955#define  atomic64_xchg_release          atomic64_xchg
 956
 957#else /* atomic64_xchg_relaxed */
 958
 959#ifndef atomic64_xchg_acquire
 960#define  atomic64_xchg_acquire(...)                                     \
 961        __atomic_op_acquire(atomic64_xchg, __VA_ARGS__)
 962#endif
 963
 964#ifndef atomic64_xchg_release
 965#define  atomic64_xchg_release(...)                                     \
 966        __atomic_op_release(atomic64_xchg, __VA_ARGS__)
 967#endif
 968
 969#ifndef atomic64_xchg
 970#define  atomic64_xchg(...)                                             \
 971        __atomic_op_fence(atomic64_xchg, __VA_ARGS__)
 972#endif
 973#endif /* atomic64_xchg_relaxed */
 974
 975/* atomic64_cmpxchg_relaxed */
 976#ifndef atomic64_cmpxchg_relaxed
 977#define  atomic64_cmpxchg_relaxed       atomic64_cmpxchg
 978#define  atomic64_cmpxchg_acquire       atomic64_cmpxchg
 979#define  atomic64_cmpxchg_release       atomic64_cmpxchg
 980
 981#else /* atomic64_cmpxchg_relaxed */
 982
 983#ifndef atomic64_cmpxchg_acquire
 984#define  atomic64_cmpxchg_acquire(...)                                  \
 985        __atomic_op_acquire(atomic64_cmpxchg, __VA_ARGS__)
 986#endif
 987
 988#ifndef atomic64_cmpxchg_release
 989#define  atomic64_cmpxchg_release(...)                                  \
 990        __atomic_op_release(atomic64_cmpxchg, __VA_ARGS__)
 991#endif
 992
 993#ifndef atomic64_cmpxchg
 994#define  atomic64_cmpxchg(...)                                          \
 995        __atomic_op_fence(atomic64_cmpxchg, __VA_ARGS__)
 996#endif
 997#endif /* atomic64_cmpxchg_relaxed */
 998
 999#ifndef atomic64_andnot
1000static inline void atomic64_andnot(long long i, atomic64_t *v)

1001{
1002        atomic64_and(~i, v);
1003}
1004
1005static inline long long atomic64_fetch_andnot(long long i, atomic64_t *v)
1006{
1007        return atomic64_fetch_and(~i, v);
1008}
1009
1010static inline long long atomic64_fetch_andnot_relaxed(long long i, atomic64_t *v)
1011{
1012        return atomic64_fetch_and_relaxed(~i, v);
1013}
1014
1015static inline long long atomic64_fetch_andnot_acquire(long long i, atomic64_t *v)
1016{
1017        return atomic64_fetch_and_acquire(~i, v);
1018}
1019
1020static inline long long atomic64_fetch_andnot_release(long long i, atomic64_t *v)
1021{
1022        return atomic64_fetch_and_release(~i, v);
1023}
1024#endif
1025
1026#include <asm-generic/atomic-long.h>
1027
1028#endif /* _LINUX_ATOMIC_H */
1029