LXR linux/include/asm-generic/percpu.h

   1/* SPDX-License-Identifier: GPL-2.0 */
   2#ifndef _ASM_GENERIC_PERCPU_H_
   3#define _ASM_GENERIC_PERCPU_H_
   4
   5#include <linux/compiler.h>
   6#include <linux/threads.h>
   7#include <linux/percpu-defs.h>
   8
   9#ifdef CONFIG_SMP
  10
  11/*
  12 * per_cpu_offset() is the offset that has to be added to a
  13 * percpu variable to get to the instance for a certain processor.
  14 *
  15 * Most arches use the __per_cpu_offset array for those offsets but
  16 * some arches have their own ways of determining the offset (x86_64, s390).
  17 */
  18#ifndef __per_cpu_offset
  19extern unsigned long __per_cpu_offset[NR_CPUS];
  20
  21#define per_cpu_offset(x) (__per_cpu_offset[x])
  22#endif
  23
  24/*
  25 * Determine the offset for the currently active processor.
  26 * An arch may define __my_cpu_offset to provide a more effective
  27 * means of obtaining the offset to the per cpu variables of the
  28 * current processor.
  29 */
  30#ifndef __my_cpu_offset
  31#define __my_cpu_offset per_cpu_offset(raw_smp_processor_id())
  32#endif
  33#ifdef CONFIG_DEBUG_PREEMPT
  34#define my_cpu_offset per_cpu_offset(smp_processor_id())
  35#else
  36#define my_cpu_offset __my_cpu_offset
  37#endif
  38
  39/*
  40 * Arch may define arch_raw_cpu_ptr() to provide more efficient address
  41 * translations for raw_cpu_ptr().
  42 */
  43#ifndef arch_raw_cpu_ptr
  44#define arch_raw_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, __my_cpu_offset)
  45#endif
  46
  47#ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA
  48extern void setup_per_cpu_areas(void);
  49#endif
  50
  51#endif  /* SMP */
  52
  53#ifndef PER_CPU_BASE_SECTION
  54#ifdef CONFIG_SMP
  55#define PER_CPU_BASE_SECTION ".data..percpu"
  56#else
  57#define PER_CPU_BASE_SECTION ".data"
  58#endif
  59#endif
  60
  61#ifndef PER_CPU_ATTRIBUTES
  62#define PER_CPU_ATTRIBUTES
  63#endif
  64
  65#ifndef PER_CPU_DEF_ATTRIBUTES
  66#define PER_CPU_DEF_ATTRIBUTES
  67#endif
  68
  69#define raw_cpu_generic_read(pcp)                                       \
  70({                                                                      \
  71        *raw_cpu_ptr(&(pcp));                                           \
  72})
  73
  74#define raw_cpu_generic_to_op(pcp, val, op)                             \
  75do {                                                                    \
  76        *raw_cpu_ptr(&(pcp)) op val;                                    \
  77} while (0)
  78
  79#define raw_cpu_generic_add_return(pcp, val)                            \
  80({                                                                      \
  81        typeof(&(pcp)) __p = raw_cpu_ptr(&(pcp));                       \
  82                                                                        \
  83        *__p += val;                                                    \
  84        *__p;                                                           \
  85})
  86
  87#define raw_cpu_generic_xchg(pcp, nval)                                 \
  88({                                                                      \
  89        typeof(&(pcp)) __p = raw_cpu_ptr(&(pcp));                       \
  90        typeof(pcp) __ret;                                              \
  91        __ret = *__p;                                                   \
  92        *__p = nval;                                                    \
  93        __ret;                                                          \
  94})
  95
  96#define raw_cpu_generic_cmpxchg(pcp, oval, nval)                        \
  97({                                                                      \
  98        typeof(&(pcp)) __p = raw_cpu_ptr(&(pcp));                       \
  99        typeof(pcp) __ret;                                              \
 100        __ret = *__p;                                                   \
 101        if (__ret == (oval))                                            \
 102                *__p = nval;                                            \
 103        __ret;                                                          \
 104})
 105
 106#define raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
 107({                                                                      \
 108        typeof(&(pcp1)) __p1 = raw_cpu_ptr(&(pcp1));                    \
 109        typeof(&(pcp2)) __p2 = raw_cpu_ptr(&(pcp2));                    \
 110        int __ret = 0;                                                  \
 111        if (*__p1 == (oval1) && *__p2  == (oval2)) {                    \
 112                *__p1 = nval1;                                          \
 113                *__p2 = nval2;                                          \
 114                __ret = 1;                                              \
 115        }                                                               \
 116        (__ret);                                                        \
 117})
 118
 119#define __this_cpu_generic_read_nopreempt(pcp)                          \
 120({                                                                      \
 121        typeof(pcp) __ret;                                              \
 122        preempt_disable_notrace();                                      \
 123        __ret = READ_ONCE(*raw_cpu_ptr(&(pcp)));                        \
 124        preempt_enable_notrace();                                       \
 125        __ret;                                                          \
 126})
 127
 128#define __this_cpu_generic_read_noirq(pcp)                              \
 129({                                                                      \
 130        typeof(pcp) __ret;                                              \
 131        unsigned long __flags;                                          \
 132        raw_local_irq_save(__flags);                                    \
 133        __ret = raw_cpu_generic_read(pcp);                              \
 134        raw_local_irq_restore(__flags);                                 \
 135        __ret;                                                          \
 136})
 137
 138#define this_cpu_generic_read(pcp)                                      \
 139({                                                                      \
 140        typeof(pcp) __ret;                                              \
 141        if (__native_word(pcp))                                         \
 142                __ret = __this_cpu_generic_read_nopreempt(pcp);         \
 143        else                                                            \
 144                __ret = __this_cpu_generic_read_noirq(pcp);             \
 145        __ret;                                                          \
 146})
 147
 148#define this_cpu_generic_to_op(pcp, val, op)                            \
 149do {                                                                    \
 150        unsigned long __flags;                                          \
 151        raw_local_irq_save(__flags);                                    \
 152        raw_cpu_generic_to_op(pcp, val, op);                            \
 153        raw_local_irq_restore(__flags);                                 \
 154} while (0)
 155
 156
 157#define this_cpu_generic_add_return(pcp, val)                           \
 158({                                                                      \
 159        typeof(pcp) __ret;                                              \
 160        unsigned long __flags;                                          \
 161        raw_local_irq_save(__flags);                                    \
 162        __ret = raw_cpu_generic_add_return(pcp, val);                   \
 163        raw_local_irq_restore(__flags);                                 \
 164        __ret;                                                          \
 165})
 166
 167#define this_cpu_generic_xchg(pcp, nval)                                \
 168({                                                                      \
 169        typeof(pcp) __ret;                                              \
 170        unsigned long __flags;                                          \
 171        raw_local_irq_save(__flags);                                    \
 172        __ret = raw_cpu_generic_xchg(pcp, nval);                        \
 173        raw_local_irq_restore(__flags);                                 \
 174        __ret;                                                          \
 175})
 176
 177#define this_cpu_generic_cmpxchg(pcp, oval, nval)                       \
 178({                                                                      \
 179        typeof(pcp) __ret;                                              \
 180        unsigned long __flags;                                          \
 181        raw_local_irq_save(__flags);                                    \
 182        __ret = raw_cpu_generic_cmpxchg(pcp, oval, nval);               \
 183        raw_local_irq_restore(__flags);                                 \
 184        __ret;                                                          \
 185})
 186
 187#define this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
 188({                                                                      \
 189        int __ret;                                                      \
 190        unsigned long __flags;                                          \
 191        raw_local_irq_save(__flags);                                    \
 192        __ret = raw_cpu_generic_cmpxchg_double(pcp1, pcp2,              \
 193                        oval1, oval2, nval1, nval2);                    \
 194        raw_local_irq_restore(__flags);                                 \
 195        __ret;                                                          \
 196})
 197
 198#ifndef raw_cpu_read_1
 199#define raw_cpu_read_1(pcp)             raw_cpu_generic_read(pcp)
 200#endif
 201#ifndef raw_cpu_read_2
 202#define raw_cpu_read_2(pcp)             raw_cpu_generic_read(pcp)
 203#endif
 204#ifndef raw_cpu_read_4
 205#define raw_cpu_read_4(pcp)             raw_cpu_generic_read(pcp)
 206#endif
 207#ifndef raw_cpu_read_8
 208#define raw_cpu_read_8(pcp)             raw_cpu_generic_read(pcp)
 209#endif
 210
 211#ifndef raw_cpu_write_1
 212#define raw_cpu_write_1(pcp, val)       raw_cpu_generic_to_op(pcp, val, =)
 213#endif
 214#ifndef raw_cpu_write_2
 215#define raw_cpu_write_2(pcp, val)       raw_cpu_generic_to_op(pcp, val, =)
 216#endif
 217#ifndef raw_cpu_write_4
 218#define raw_cpu_write_4(pcp, val)       raw_cpu_generic_to_op(pcp, val, =)
 219#endif
 220#ifndef raw_cpu_write_8
 221#define raw_cpu_write_8(pcp, val)       raw_cpu_generic_to_op(pcp, val, =)
 222#endif
 223
 224#ifndef raw_cpu_add_1
 225#define raw_cpu_add_1(pcp, val)         raw_cpu_generic_to_op(pcp, val, +=)
 226#endif
 227#ifndef raw_cpu_add_2
 228#define raw_cpu_add_2(pcp, val)         raw_cpu_generic_to_op(pcp, val, +=)
 229#endif
 230#ifndef raw_cpu_add_4
 231#define raw_cpu_add_4(pcp, val)         raw_cpu_generic_to_op(pcp, val, +=)
 232#endif
 233#ifndef raw_cpu_add_8
 234#define raw_cpu_add_8(pcp, val)         raw_cpu_generic_to_op(pcp, val, +=)
 235#endif
 236
 237#ifndef raw_cpu_and_1
 238#define raw_cpu_and_1(pcp, val)         raw_cpu_generic_to_op(pcp, val, &=)
 239#endif
 240#ifndef raw_cpu_and_2
 241#define raw_cpu_and_2(pcp, val)         raw_cpu_generic_to_op(pcp, val, &=)
 242#endif
 243#ifndef raw_cpu_and_4
 244#define raw_cpu_and_4(pcp, val)         raw_cpu_generic_to_op(pcp, val, &=)
 245#endif
 246#ifndef raw_cpu_and_8
 247#define raw_cpu_and_8(pcp, val)         raw_cpu_generic_to_op(pcp, val, &=)
 248#endif
 249
 250#ifndef raw_cpu_or_1
 251#define raw_cpu_or_1(pcp, val)          raw_cpu_generic_to_op(pcp, val, |=)
 252#endif
 253#ifndef raw_cpu_or_2
 254#define raw_cpu_or_2(pcp, val)          raw_cpu_generic_to_op(pcp, val, |=)
 255#endif
 256#ifndef raw_cpu_or_4
 257#define raw_cpu_or_4(pcp, val)          raw_cpu_generic_to_op(pcp, val, |=)
 258#endif
 259#ifndef raw_cpu_or_8
 260#define raw_cpu_or_8(pcp, val)          raw_cpu_generic_to_op(pcp, val, |=)
 261#endif
 262
 263#ifndef raw_cpu_add_return_1
 264#define raw_cpu_add_return_1(pcp, val)  raw_cpu_generic_add_return(pcp, val)
 265#endif
 266#ifndef raw_cpu_add_return_2
 267#define raw_cpu_add_return_2(pcp, val)  raw_cpu_generic_add_return(pcp, val)
 268#endif
 269#ifndef raw_cpu_add_return_4
 270#define raw_cpu_add_return_4(pcp, val)  raw_cpu_generic_add_return(pcp, val)
 271#endif
 272#ifndef raw_cpu_add_return_8
 273#define raw_cpu_add_return_8(pcp, val)  raw_cpu_generic_add_return(pcp, val)
 274#endif
 275
 276#ifndef raw_cpu_xchg_1
 277#define raw_cpu_xchg_1(pcp, nval)       raw_cpu_generic_xchg(pcp, nval)
 278#endif
 279#ifndef raw_cpu_xchg_2
 280#define raw_cpu_xchg_2(pcp, nval)       raw_cpu_generic_xchg(pcp, nval)
 281#endif
 282#ifndef raw_cpu_xchg_4
 283#define raw_cpu_xchg_4(pcp, nval)       raw_cpu_generic_xchg(pcp, nval)
 284#endif
 285#ifndef raw_cpu_xchg_8
 286#define raw_cpu_xchg_8(pcp, nval)       raw_cpu_generic_xchg(pcp, nval)
 287#endif
 288
 289#ifndef raw_cpu_cmpxchg_1
 290#define raw_cpu_cmpxchg_1(pcp, oval, nval) \
 291        raw_cpu_generic_cmpxchg(pcp, oval, nval)
 292#endif
 293#ifndef raw_cpu_cmpxchg_2
 294#define raw_cpu_cmpxchg_2(pcp, oval, nval) \
 295        raw_cpu_generic_cmpxchg(pcp, oval, nval)
 296#endif
 297#ifndef raw_cpu_cmpxchg_4
 298#define raw_cpu_cmpxchg_4(pcp, oval, nval) \
 299        raw_cpu_generic_cmpxchg(pcp, oval, nval)
 300#endif
 301#ifndef raw_cpu_cmpxchg_8
 302#define raw_cpu_cmpxchg_8(pcp, oval, nval) \
 303        raw_cpu_generic_cmpxchg(pcp, oval, nval)
 304#endif
 305
 306#ifndef raw_cpu_cmpxchg_double_1
 307#define raw_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \
 308        raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
 309#endif
 310#ifndef raw_cpu_cmpxchg_double_2
 311#define raw_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \
 312        raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
 313#endif
 314#ifndef raw_cpu_cmpxchg_double_4
 315#define raw_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \
 316        raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
 317#endif
 318#ifndef raw_cpu_cmpxchg_double_8
 319#define raw_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \
 320        raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
 321#endif
 322
 323#ifndef this_cpu_read_1
 324#define this_cpu_read_1(pcp)            this_cpu_generic_read(pcp)
 325#endif
 326#ifndef this_cpu_read_2
 327#define this_cpu_read_2(pcp)            this_cpu_generic_read(pcp)
 328#endif
 329#ifndef this_cpu_read_4
 330#define this_cpu_read_4(pcp)            this_cpu_generic_read(pcp)
 331#endif
 332#ifndef this_cpu_read_8
 333#define this_cpu_read_8(pcp)            this_cpu_generic_read(pcp)
 334#endif
 335
 336#ifndef this_cpu_write_1
 337#define this_cpu_write_1(pcp, val)      this_cpu_generic_to_op(pcp, val, =)
 338#endif
 339#ifndef this_cpu_write_2
 340#define this_cpu_write_2(pcp, val)      this_cpu_generic_to_op(pcp, val, =)
 341#endif
 342#ifndef this_cpu_write_4
 343#define this_cpu_write_4(pcp, val)      this_cpu_generic_to_op(pcp, val, =)
 344#endif
 345#ifndef this_cpu_write_8
 346#define this_cpu_write_8(pcp, val)      this_cpu_generic_to_op(pcp, val, =)
 347#endif
 348
 349#ifndef this_cpu_add_1
 350#define this_cpu_add_1(pcp, val)        this_cpu_generic_to_op(pcp, val, +=)
 351#endif
 352#ifndef this_cpu_add_2
 353#define this_cpu_add_2(pcp, val)        this_cpu_generic_to_op(pcp, val, +=)
 354#endif
 355#ifndef this_cpu_add_4
 356#define this_cpu_add_4(pcp, val)        this_cpu_generic_to_op(pcp, val, +=)
 357#endif
 358#ifndef this_cpu_add_8
 359#define this_cpu_add_8(pcp, val)        this_cpu_generic_to_op(pcp, val, +=)
 360#endif
 361
 362#ifndef this_cpu_and_1
 363#define this_cpu_and_1(pcp, val)        this_cpu_generic_to_op(pcp, val, &=)
 364#endif
 365#ifndef this_cpu_and_2
 366#define this_cpu_and_2(pcp, val)        this_cpu_generic_to_op(pcp, val, &=)
 367#endif
 368#ifndef this_cpu_and_4
 369#define this_cpu_and_4(pcp, val)        this_cpu_generic_to_op(pcp, val, &=)
 370#endif
 371#ifndef this_cpu_and_8
 372#define this_cpu_and_8(pcp, val)        this_cpu_generic_to_op(pcp, val, &=)
 373#endif
 374
 375#ifndef this_cpu_or_1
 376#define this_cpu_or_1(pcp, val)         this_cpu_generic_to_op(pcp, val, |=)
 377#endif
 378#ifndef this_cpu_or_2
 379#define this_cpu_or_2(pcp, val)         this_cpu_generic_to_op(pcp, val, |=)
 380#endif
 381#ifndef this_cpu_or_4
 382#define this_cpu_or_4(pcp, val)         this_cpu_generic_to_op(pcp, val, |=)
 383#endif
 384#ifndef this_cpu_or_8
 385#define this_cpu_or_8(pcp, val)         this_cpu_generic_to_op(pcp, val, |=)
 386#endif
 387
 388#ifndef this_cpu_add_return_1
 389#define this_cpu_add_return_1(pcp, val) this_cpu_generic_add_return(pcp, val)
 390#endif
 391#ifndef this_cpu_add_return_2
 392#define this_cpu_add_return_2(pcp, val) this_cpu_generic_add_return(pcp, val)
 393#endif
 394#ifndef this_cpu_add_return_4
 395#define this_cpu_add_return_4(pcp, val) this_cpu_generic_add_return(pcp, val)
 396#endif
 397#ifndef this_cpu_add_return_8
 398#define this_cpu_add_return_8(pcp, val) this_cpu_generic_add_return(pcp, val)
 399#endif
 400
 401#ifndef this_cpu_xchg_1
 402#define this_cpu_xchg_1(pcp, nval)      this_cpu_generic_xchg(pcp, nval)
 403#endif
 404#ifndef this_cpu_xchg_2
 405#define this_cpu_xchg_2(pcp, nval)      this_cpu_generic_xchg(pcp, nval)
 406#endif
 407#ifndef this_cpu_xchg_4
 408#define this_cpu_xchg_4(pcp, nval)      this_cpu_generic_xchg(pcp, nval)
 409#endif
 410#ifndef this_cpu_xchg_8
 411#define this_cpu_xchg_8(pcp, nval)      this_cpu_generic_xchg(pcp, nval)
 412#endif
 413
 414#ifndef this_cpu_cmpxchg_1
 415#define this_cpu_cmpxchg_1(pcp, oval, nval) \
 416        this_cpu_generic_cmpxchg(pcp, oval, nval)
 417#endif
 418#ifndef this_cpu_cmpxchg_2
 419#define this_cpu_cmpxchg_2(pcp, oval, nval) \
 420        this_cpu_generic_cmpxchg(pcp, oval, nval)
 421#endif
 422#ifndef this_cpu_cmpxchg_4
 423#define this_cpu_cmpxchg_4(pcp, oval, nval) \
 424        this_cpu_generic_cmpxchg(pcp, oval, nval)
 425#endif
 426#ifndef this_cpu_cmpxchg_8
 427#define this_cpu_cmpxchg_8(pcp, oval, nval) \
 428        this_cpu_generic_cmpxchg(pcp, oval, nval)
 429#endif
 430
 431#ifndef this_cpu_cmpxchg_double_1
 432#define this_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \
 433        this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
 434#endif
 435#ifndef this_cpu_cmpxchg_double_2
 436#define this_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \
 437        this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
 438#endif
 439#ifndef this_cpu_cmpxchg_double_4
 440#define this_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \
 441        this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
 442#endif
 443#ifndef this_cpu_cmpxchg_double_8
 444#define this_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \
 445        this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
 446#endif
 447
 448#endif /* _ASM_GENERIC_PERCPU_H_ */
 449