LXR linux/include/asm-generic/percpu.h

   1#ifndef _ASM_GENERIC_PERCPU_H_
   2#define _ASM_GENERIC_PERCPU_H_
   3
   4#include <linux/compiler.h>
   5#include <linux/threads.h>
   6#include <linux/percpu-defs.h>
   7
   8#ifdef CONFIG_SMP
   9
  10/*
  11 * per_cpu_offset() is the offset that has to be added to a
  12 * percpu variable to get to the instance for a certain processor.
  13 *
  14 * Most arches use the __per_cpu_offset array for those offsets but
  15 * some arches have their own ways of determining the offset (x86_64, s390).
  16 */
  17#ifndef __per_cpu_offset
  18extern unsigned long __per_cpu_offset[NR_CPUS];
  19
  20#define per_cpu_offset(x) (__per_cpu_offset[x])
  21#endif
  22
  23/*
  24 * Determine the offset for the currently active processor.
  25 * An arch may define __my_cpu_offset to provide a more effective
  26 * means of obtaining the offset to the per cpu variables of the
  27 * current processor.
  28 */
  29#ifndef __my_cpu_offset
  30#define __my_cpu_offset per_cpu_offset(raw_smp_processor_id())
  31#endif
  32#ifdef CONFIG_DEBUG_PREEMPT
  33#define my_cpu_offset per_cpu_offset(smp_processor_id())
  34#else
  35#define my_cpu_offset __my_cpu_offset
  36#endif
  37
  38/*
  39 * Arch may define arch_raw_cpu_ptr() to provide more efficient address
  40 * translations for raw_cpu_ptr().
  41 */
  42#ifndef arch_raw_cpu_ptr
  43#define arch_raw_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, __my_cpu_offset)
  44#endif
  45
  46#ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA
  47extern void setup_per_cpu_areas(void);
  48#endif
  49
  50#endif  /* SMP */
  51
  52#ifndef PER_CPU_BASE_SECTION
  53#ifdef CONFIG_SMP
  54#define PER_CPU_BASE_SECTION ".data..percpu"
  55#else
  56#define PER_CPU_BASE_SECTION ".data"
  57#endif
  58#endif
  59
  60#ifndef PER_CPU_ATTRIBUTES
  61#define PER_CPU_ATTRIBUTES
  62#endif
  63
  64#ifndef PER_CPU_DEF_ATTRIBUTES
  65#define PER_CPU_DEF_ATTRIBUTES
  66#endif
  67
  68#define raw_cpu_generic_to_op(pcp, val, op)                             \
  69do {                                                                    \
  70        *raw_cpu_ptr(&(pcp)) op val;                                    \
  71} while (0)
  72
  73#define raw_cpu_generic_add_return(pcp, val)                            \
  74({                                                                      \
  75        raw_cpu_add(pcp, val);                                          \
  76        raw_cpu_read(pcp);                                              \
  77})
  78
  79#define raw_cpu_generic_xchg(pcp, nval)                                 \
  80({                                                                      \
  81        typeof(pcp) __ret;                                              \
  82        __ret = raw_cpu_read(pcp);                                      \
  83        raw_cpu_write(pcp, nval);                                       \
  84        __ret;                                                          \
  85})
  86
  87#define raw_cpu_generic_cmpxchg(pcp, oval, nval)                        \
  88({                                                                      \
  89        typeof(pcp) __ret;                                              \
  90        __ret = raw_cpu_read(pcp);                                      \
  91        if (__ret == (oval))                                            \
  92                raw_cpu_write(pcp, nval);                               \
  93        __ret;                                                          \
  94})
  95
  96#define raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
  97({                                                                      \
  98        int __ret = 0;                                                  \
  99        if (raw_cpu_read(pcp1) == (oval1) &&                            \
 100                         raw_cpu_read(pcp2)  == (oval2)) {              \
 101                raw_cpu_write(pcp1, nval1);                             \
 102                raw_cpu_write(pcp2, nval2);                             \
 103                __ret = 1;                                              \
 104        }                                                               \
 105        (__ret);                                                        \
 106})
 107
 108#define this_cpu_generic_read(pcp)                                      \
 109({                                                                      \
 110        typeof(pcp) __ret;                                              \
 111        preempt_disable();                                              \
 112        __ret = *this_cpu_ptr(&(pcp));                                  \
 113        preempt_enable();                                               \
 114        __ret;                                                          \
 115})
 116
 117#define this_cpu_generic_to_op(pcp, val, op)                            \
 118do {                                                                    \
 119        unsigned long __flags;                                          \
 120        raw_local_irq_save(__flags);                                    \
 121        *raw_cpu_ptr(&(pcp)) op val;                                    \
 122        raw_local_irq_restore(__flags);                                 \
 123} while (0)
 124
 125#define this_cpu_generic_add_return(pcp, val)                           \
 126({                                                                      \
 127        typeof(pcp) __ret;                                              \
 128        unsigned long __flags;                                          \
 129        raw_local_irq_save(__flags);                                    \
 130        raw_cpu_add(pcp, val);                                          \
 131        __ret = raw_cpu_read(pcp);                                      \
 132        raw_local_irq_restore(__flags);                                 \
 133        __ret;                                                          \
 134})
 135
 136#define this_cpu_generic_xchg(pcp, nval)                                \
 137({                                                                      \
 138        typeof(pcp) __ret;                                              \
 139        unsigned long __flags;                                          \
 140        raw_local_irq_save(__flags);                                    \
 141        __ret = raw_cpu_read(pcp);                                      \
 142        raw_cpu_write(pcp, nval);                                       \
 143        raw_local_irq_restore(__flags);                                 \
 144        __ret;                                                          \
 145})
 146
 147#define this_cpu_generic_cmpxchg(pcp, oval, nval)                       \
 148({                                                                      \
 149        typeof(pcp) __ret;                                              \
 150        unsigned long __flags;                                          \
 151        raw_local_irq_save(__flags);                                    \
 152        __ret = raw_cpu_read(pcp);                                      \
 153        if (__ret == (oval))                                            \
 154                raw_cpu_write(pcp, nval);                               \
 155        raw_local_irq_restore(__flags);                                 \
 156        __ret;                                                          \
 157})
 158
 159#define this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
 160({                                                                      \
 161        int __ret;                                                      \
 162        unsigned long __flags;                                          \
 163        raw_local_irq_save(__flags);                                    \
 164        __ret = raw_cpu_generic_cmpxchg_double(pcp1, pcp2,              \
 165                        oval1, oval2, nval1, nval2);                    \
 166        raw_local_irq_restore(__flags);                                 \
 167        __ret;                                                          \
 168})
 169
 170#ifndef raw_cpu_read_1
 171#define raw_cpu_read_1(pcp)             (*raw_cpu_ptr(&(pcp)))
 172#endif
 173#ifndef raw_cpu_read_2
 174#define raw_cpu_read_2(pcp)             (*raw_cpu_ptr(&(pcp)))
 175#endif
 176#ifndef raw_cpu_read_4
 177#define raw_cpu_read_4(pcp)             (*raw_cpu_ptr(&(pcp)))
 178#endif
 179#ifndef raw_cpu_read_8
 180#define raw_cpu_read_8(pcp)             (*raw_cpu_ptr(&(pcp)))
 181#endif
 182
 183#ifndef raw_cpu_write_1
 184#define raw_cpu_write_1(pcp, val)       raw_cpu_generic_to_op(pcp, val, =)
 185#endif
 186#ifndef raw_cpu_write_2
 187#define raw_cpu_write_2(pcp, val)       raw_cpu_generic_to_op(pcp, val, =)
 188#endif
 189#ifndef raw_cpu_write_4
 190#define raw_cpu_write_4(pcp, val)       raw_cpu_generic_to_op(pcp, val, =)
 191#endif
 192#ifndef raw_cpu_write_8
 193#define raw_cpu_write_8(pcp, val)       raw_cpu_generic_to_op(pcp, val, =)
 194#endif
 195
 196#ifndef raw_cpu_add_1
 197#define raw_cpu_add_1(pcp, val)         raw_cpu_generic_to_op(pcp, val, +=)
 198#endif
 199#ifndef raw_cpu_add_2
 200#define raw_cpu_add_2(pcp, val)         raw_cpu_generic_to_op(pcp, val, +=)
 201#endif
 202#ifndef raw_cpu_add_4
 203#define raw_cpu_add_4(pcp, val)         raw_cpu_generic_to_op(pcp, val, +=)
 204#endif
 205#ifndef raw_cpu_add_8
 206#define raw_cpu_add_8(pcp, val)         raw_cpu_generic_to_op(pcp, val, +=)
 207#endif
 208
 209#ifndef raw_cpu_and_1
 210#define raw_cpu_and_1(pcp, val)         raw_cpu_generic_to_op(pcp, val, &=)
 211#endif
 212#ifndef raw_cpu_and_2
 213#define raw_cpu_and_2(pcp, val)         raw_cpu_generic_to_op(pcp, val, &=)
 214#endif
 215#ifndef raw_cpu_and_4
 216#define raw_cpu_and_4(pcp, val)         raw_cpu_generic_to_op(pcp, val, &=)
 217#endif
 218#ifndef raw_cpu_and_8
 219#define raw_cpu_and_8(pcp, val)         raw_cpu_generic_to_op(pcp, val, &=)
 220#endif
 221
 222#ifndef raw_cpu_or_1
 223#define raw_cpu_or_1(pcp, val)          raw_cpu_generic_to_op(pcp, val, |=)
 224#endif
 225#ifndef raw_cpu_or_2
 226#define raw_cpu_or_2(pcp, val)          raw_cpu_generic_to_op(pcp, val, |=)
 227#endif
 228#ifndef raw_cpu_or_4
 229#define raw_cpu_or_4(pcp, val)          raw_cpu_generic_to_op(pcp, val, |=)
 230#endif
 231#ifndef raw_cpu_or_8
 232#define raw_cpu_or_8(pcp, val)          raw_cpu_generic_to_op(pcp, val, |=)
 233#endif
 234
 235#ifndef raw_cpu_add_return_1
 236#define raw_cpu_add_return_1(pcp, val)  raw_cpu_generic_add_return(pcp, val)
 237#endif
 238#ifndef raw_cpu_add_return_2
 239#define raw_cpu_add_return_2(pcp, val)  raw_cpu_generic_add_return(pcp, val)
 240#endif
 241#ifndef raw_cpu_add_return_4
 242#define raw_cpu_add_return_4(pcp, val)  raw_cpu_generic_add_return(pcp, val)
 243#endif
 244#ifndef raw_cpu_add_return_8
 245#define raw_cpu_add_return_8(pcp, val)  raw_cpu_generic_add_return(pcp, val)
 246#endif
 247
 248#ifndef raw_cpu_xchg_1
 249#define raw_cpu_xchg_1(pcp, nval)       raw_cpu_generic_xchg(pcp, nval)
 250#endif
 251#ifndef raw_cpu_xchg_2
 252#define raw_cpu_xchg_2(pcp, nval)       raw_cpu_generic_xchg(pcp, nval)
 253#endif
 254#ifndef raw_cpu_xchg_4
 255#define raw_cpu_xchg_4(pcp, nval)       raw_cpu_generic_xchg(pcp, nval)
 256#endif
 257#ifndef raw_cpu_xchg_8
 258#define raw_cpu_xchg_8(pcp, nval)       raw_cpu_generic_xchg(pcp, nval)
 259#endif
 260
 261#ifndef raw_cpu_cmpxchg_1
 262#define raw_cpu_cmpxchg_1(pcp, oval, nval) \
 263        raw_cpu_generic_cmpxchg(pcp, oval, nval)
 264#endif
 265#ifndef raw_cpu_cmpxchg_2
 266#define raw_cpu_cmpxchg_2(pcp, oval, nval) \
 267        raw_cpu_generic_cmpxchg(pcp, oval, nval)
 268#endif
 269#ifndef raw_cpu_cmpxchg_4
 270#define raw_cpu_cmpxchg_4(pcp, oval, nval) \
 271        raw_cpu_generic_cmpxchg(pcp, oval, nval)
 272#endif
 273#ifndef raw_cpu_cmpxchg_8
 274#define raw_cpu_cmpxchg_8(pcp, oval, nval) \
 275        raw_cpu_generic_cmpxchg(pcp, oval, nval)
 276#endif
 277
 278#ifndef raw_cpu_cmpxchg_double_1
 279#define raw_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \
 280        raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
 281#endif
 282#ifndef raw_cpu_cmpxchg_double_2
 283#define raw_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \
 284        raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
 285#endif
 286#ifndef raw_cpu_cmpxchg_double_4
 287#define raw_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \
 288        raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
 289#endif
 290#ifndef raw_cpu_cmpxchg_double_8
 291#define raw_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \
 292        raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
 293#endif
 294
 295#ifndef this_cpu_read_1
 296#define this_cpu_read_1(pcp)            this_cpu_generic_read(pcp)
 297#endif
 298#ifndef this_cpu_read_2
 299#define this_cpu_read_2(pcp)            this_cpu_generic_read(pcp)
 300#endif
 301#ifndef this_cpu_read_4
 302#define this_cpu_read_4(pcp)            this_cpu_generic_read(pcp)
 303#endif
 304#ifndef this_cpu_read_8
 305#define this_cpu_read_8(pcp)            this_cpu_generic_read(pcp)
 306#endif
 307
 308#ifndef this_cpu_write_1
 309#define this_cpu_write_1(pcp, val)      this_cpu_generic_to_op(pcp, val, =)
 310#endif
 311#ifndef this_cpu_write_2
 312#define this_cpu_write_2(pcp, val)      this_cpu_generic_to_op(pcp, val, =)
 313#endif
 314#ifndef this_cpu_write_4
 315#define this_cpu_write_4(pcp, val)      this_cpu_generic_to_op(pcp, val, =)
 316#endif
 317#ifndef this_cpu_write_8
 318#define this_cpu_write_8(pcp, val)      this_cpu_generic_to_op(pcp, val, =)
 319#endif
 320
 321#ifndef this_cpu_add_1
 322#define this_cpu_add_1(pcp, val)        this_cpu_generic_to_op(pcp, val, +=)
 323#endif
 324#ifndef this_cpu_add_2
 325#define this_cpu_add_2(pcp, val)        this_cpu_generic_to_op(pcp, val, +=)
 326#endif
 327#ifndef this_cpu_add_4
 328#define this_cpu_add_4(pcp, val)        this_cpu_generic_to_op(pcp, val, +=)
 329#endif
 330#ifndef this_cpu_add_8
 331#define this_cpu_add_8(pcp, val)        this_cpu_generic_to_op(pcp, val, +=)
 332#endif
 333
 334#ifndef this_cpu_and_1
 335#define this_cpu_and_1(pcp, val)        this_cpu_generic_to_op(pcp, val, &=)
 336#endif
 337#ifndef this_cpu_and_2
 338#define this_cpu_and_2(pcp, val)        this_cpu_generic_to_op(pcp, val, &=)
 339#endif
 340#ifndef this_cpu_and_4
 341#define this_cpu_and_4(pcp, val)        this_cpu_generic_to_op(pcp, val, &=)
 342#endif
 343#ifndef this_cpu_and_8
 344#define this_cpu_and_8(pcp, val)        this_cpu_generic_to_op(pcp, val, &=)
 345#endif
 346
 347#ifndef this_cpu_or_1
 348#define this_cpu_or_1(pcp, val)         this_cpu_generic_to_op(pcp, val, |=)
 349#endif
 350#ifndef this_cpu_or_2
 351#define this_cpu_or_2(pcp, val)         this_cpu_generic_to_op(pcp, val, |=)
 352#endif
 353#ifndef this_cpu_or_4
 354#define this_cpu_or_4(pcp, val)         this_cpu_generic_to_op(pcp, val, |=)
 355#endif
 356#ifndef this_cpu_or_8
 357#define this_cpu_or_8(pcp, val)         this_cpu_generic_to_op(pcp, val, |=)
 358#endif
 359
 360#ifndef this_cpu_add_return_1
 361#define this_cpu_add_return_1(pcp, val) this_cpu_generic_add_return(pcp, val)
 362#endif
 363#ifndef this_cpu_add_return_2
 364#define this_cpu_add_return_2(pcp, val) this_cpu_generic_add_return(pcp, val)
 365#endif
 366#ifndef this_cpu_add_return_4
 367#define this_cpu_add_return_4(pcp, val) this_cpu_generic_add_return(pcp, val)
 368#endif
 369#ifndef this_cpu_add_return_8
 370#define this_cpu_add_return_8(pcp, val) this_cpu_generic_add_return(pcp, val)
 371#endif
 372
 373#ifndef this_cpu_xchg_1
 374#define this_cpu_xchg_1(pcp, nval)      this_cpu_generic_xchg(pcp, nval)
 375#endif
 376#ifndef this_cpu_xchg_2
 377#define this_cpu_xchg_2(pcp, nval)      this_cpu_generic_xchg(pcp, nval)
 378#endif
 379#ifndef this_cpu_xchg_4
 380#define this_cpu_xchg_4(pcp, nval)      this_cpu_generic_xchg(pcp, nval)
 381#endif
 382#ifndef this_cpu_xchg_8
 383#define this_cpu_xchg_8(pcp, nval)      this_cpu_generic_xchg(pcp, nval)
 384#endif
 385
 386#ifndef this_cpu_cmpxchg_1
 387#define this_cpu_cmpxchg_1(pcp, oval, nval) \
 388        this_cpu_generic_cmpxchg(pcp, oval, nval)
 389#endif
 390#ifndef this_cpu_cmpxchg_2
 391#define this_cpu_cmpxchg_2(pcp, oval, nval) \
 392        this_cpu_generic_cmpxchg(pcp, oval, nval)
 393#endif
 394#ifndef this_cpu_cmpxchg_4
 395#define this_cpu_cmpxchg_4(pcp, oval, nval) \
 396        this_cpu_generic_cmpxchg(pcp, oval, nval)
 397#endif
 398#ifndef this_cpu_cmpxchg_8
 399#define this_cpu_cmpxchg_8(pcp, oval, nval) \
 400        this_cpu_generic_cmpxchg(pcp, oval, nval)
 401#endif
 402
 403#ifndef this_cpu_cmpxchg_double_1
 404#define this_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \
 405        this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
 406#endif
 407#ifndef this_cpu_cmpxchg_double_2
 408#define this_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \
 409        this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
 410#endif
 411#ifndef this_cpu_cmpxchg_double_4
 412#define this_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \
 413        this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
 414#endif
 415#ifndef this_cpu_cmpxchg_double_8
 416#define this_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \
 417        this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
 418#endif
 419
 420#endif /* _ASM_GENERIC_PERCPU_H_ */
 421