LXR linux/include/asm-generic/percpu.h

   1/* SPDX-License-Identifier: GPL-2.0 */
   2#ifndef _ASM_GENERIC_PERCPU_H_
   3#define _ASM_GENERIC_PERCPU_H_
   4
   5#include <linux/compiler.h>
   6#include <linux/threads.h>
   7#include <linux/percpu-defs.h>
   8
   9#ifdef CONFIG_SMP
  10
  11/*
  12 * per_cpu_offset() is the offset that has to be added to a
  13 * percpu variable to get to the instance for a certain processor.
  14 *
  15 * Most arches use the __per_cpu_offset array for those offsets but
  16 * some arches have their own ways of determining the offset (x86_64, s390).
  17 */
  18#ifndef __per_cpu_offset
  19extern unsigned long __per_cpu_offset[NR_CPUS];
  20
  21#define per_cpu_offset(x) (__per_cpu_offset[x])
  22#endif
  23
  24/*
  25 * Determine the offset for the currently active processor.
  26 * An arch may define __my_cpu_offset to provide a more effective
  27 * means of obtaining the offset to the per cpu variables of the
  28 * current processor.
  29 */
  30#ifndef __my_cpu_offset
  31#define __my_cpu_offset per_cpu_offset(raw_smp_processor_id())
  32#endif
  33#ifdef CONFIG_DEBUG_PREEMPT
  34#define my_cpu_offset per_cpu_offset(smp_processor_id())
  35#else
  36#define my_cpu_offset __my_cpu_offset
  37#endif
  38
  39/*
  40 * Arch may define arch_raw_cpu_ptr() to provide more efficient address
  41 * translations for raw_cpu_ptr().
  42 */
  43#ifndef arch_raw_cpu_ptr
  44#define arch_raw_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, __my_cpu_offset)
  45#endif
  46
  47#ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA
  48extern void setup_per_cpu_areas(void);
  49#endif
  50
  51#endif  /* SMP */
  52
  53#ifndef PER_CPU_BASE_SECTION
  54#ifdef CONFIG_SMP
  55#define PER_CPU_BASE_SECTION ".data..percpu"
  56#else
  57#define PER_CPU_BASE_SECTION ".data"
  58#endif
  59#endif
  60
  61#ifndef PER_CPU_ATTRIBUTES
  62#define PER_CPU_ATTRIBUTES
  63#endif
  64
  65#define raw_cpu_generic_read(pcp)                                       \
  66({                                                                      \
  67        *raw_cpu_ptr(&(pcp));                                           \
  68})
  69
  70#define raw_cpu_generic_to_op(pcp, val, op)                             \
  71do {                                                                    \
  72        *raw_cpu_ptr(&(pcp)) op val;                                    \
  73} while (0)
  74
  75#define raw_cpu_generic_add_return(pcp, val)                            \
  76({                                                                      \
  77        typeof(pcp) *__p = raw_cpu_ptr(&(pcp));                         \
  78                                                                        \
  79        *__p += val;                                                    \
  80        *__p;                                                           \
  81})
  82
  83#define raw_cpu_generic_xchg(pcp, nval)                                 \
  84({                                                                      \
  85        typeof(pcp) *__p = raw_cpu_ptr(&(pcp));                         \
  86        typeof(pcp) __ret;                                              \
  87        __ret = *__p;                                                   \
  88        *__p = nval;                                                    \
  89        __ret;                                                          \
  90})
  91
  92#define raw_cpu_generic_cmpxchg(pcp, oval, nval)                        \
  93({                                                                      \
  94        typeof(pcp) *__p = raw_cpu_ptr(&(pcp));                         \
  95        typeof(pcp) __ret;                                              \
  96        __ret = *__p;                                                   \
  97        if (__ret == (oval))                                            \
  98                *__p = nval;                                            \
  99        __ret;                                                          \
 100})
 101
 102#define raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
 103({                                                                      \
 104        typeof(pcp1) *__p1 = raw_cpu_ptr(&(pcp1));                      \
 105        typeof(pcp2) *__p2 = raw_cpu_ptr(&(pcp2));                      \
 106        int __ret = 0;                                                  \
 107        if (*__p1 == (oval1) && *__p2  == (oval2)) {                    \
 108                *__p1 = nval1;                                          \
 109                *__p2 = nval2;                                          \
 110                __ret = 1;                                              \
 111        }                                                               \
 112        (__ret);                                                        \
 113})
 114
 115#define __this_cpu_generic_read_nopreempt(pcp)                          \
 116({                                                                      \
 117        typeof(pcp) ___ret;                                             \
 118        preempt_disable_notrace();                                      \
 119        ___ret = READ_ONCE(*raw_cpu_ptr(&(pcp)));                       \
 120        preempt_enable_notrace();                                       \
 121        ___ret;                                                         \
 122})
 123
 124#define __this_cpu_generic_read_noirq(pcp)                              \
 125({                                                                      \
 126        typeof(pcp) ___ret;                                             \
 127        unsigned long ___flags;                                         \
 128        raw_local_irq_save(___flags);                                   \
 129        ___ret = raw_cpu_generic_read(pcp);                             \
 130        raw_local_irq_restore(___flags);                                \
 131        ___ret;                                                         \
 132})
 133
 134#define this_cpu_generic_read(pcp)                                      \
 135({                                                                      \
 136        typeof(pcp) __ret;                                              \
 137        if (__native_word(pcp))                                         \
 138                __ret = __this_cpu_generic_read_nopreempt(pcp);         \
 139        else                                                            \
 140                __ret = __this_cpu_generic_read_noirq(pcp);             \
 141        __ret;                                                          \
 142})
 143
 144#define this_cpu_generic_to_op(pcp, val, op)                            \
 145do {                                                                    \
 146        unsigned long __flags;                                          \
 147        raw_local_irq_save(__flags);                                    \
 148        raw_cpu_generic_to_op(pcp, val, op);                            \
 149        raw_local_irq_restore(__flags);                                 \
 150} while (0)
 151
 152
 153#define this_cpu_generic_add_return(pcp, val)                           \
 154({                                                                      \
 155        typeof(pcp) __ret;                                              \
 156        unsigned long __flags;                                          \
 157        raw_local_irq_save(__flags);                                    \
 158        __ret = raw_cpu_generic_add_return(pcp, val);                   \
 159        raw_local_irq_restore(__flags);                                 \
 160        __ret;                                                          \
 161})
 162
 163#define this_cpu_generic_xchg(pcp, nval)                                \
 164({                                                                      \
 165        typeof(pcp) __ret;                                              \
 166        unsigned long __flags;                                          \
 167        raw_local_irq_save(__flags);                                    \
 168        __ret = raw_cpu_generic_xchg(pcp, nval);                        \
 169        raw_local_irq_restore(__flags);                                 \
 170        __ret;                                                          \
 171})
 172
 173#define this_cpu_generic_cmpxchg(pcp, oval, nval)                       \
 174({                                                                      \
 175        typeof(pcp) __ret;                                              \
 176        unsigned long __flags;                                          \
 177        raw_local_irq_save(__flags);                                    \
 178        __ret = raw_cpu_generic_cmpxchg(pcp, oval, nval);               \
 179        raw_local_irq_restore(__flags);                                 \
 180        __ret;                                                          \
 181})
 182
 183#define this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
 184({                                                                      \
 185        int __ret;                                                      \
 186        unsigned long __flags;                                          \
 187        raw_local_irq_save(__flags);                                    \
 188        __ret = raw_cpu_generic_cmpxchg_double(pcp1, pcp2,              \
 189                        oval1, oval2, nval1, nval2);                    \
 190        raw_local_irq_restore(__flags);                                 \
 191        __ret;                                                          \
 192})
 193
 194#ifndef raw_cpu_read_1
 195#define raw_cpu_read_1(pcp)             raw_cpu_generic_read(pcp)
 196#endif
 197#ifndef raw_cpu_read_2
 198#define raw_cpu_read_2(pcp)             raw_cpu_generic_read(pcp)
 199#endif
 200#ifndef raw_cpu_read_4
 201#define raw_cpu_read_4(pcp)             raw_cpu_generic_read(pcp)
 202#endif
 203#ifndef raw_cpu_read_8
 204#define raw_cpu_read_8(pcp)             raw_cpu_generic_read(pcp)
 205#endif
 206
 207#ifndef raw_cpu_write_1
 208#define raw_cpu_write_1(pcp, val)       raw_cpu_generic_to_op(pcp, val, =)
 209#endif
 210#ifndef raw_cpu_write_2
 211#define raw_cpu_write_2(pcp, val)       raw_cpu_generic_to_op(pcp, val, =)
 212#endif
 213#ifndef raw_cpu_write_4
 214#define raw_cpu_write_4(pcp, val)       raw_cpu_generic_to_op(pcp, val, =)
 215#endif
 216#ifndef raw_cpu_write_8
 217#define raw_cpu_write_8(pcp, val)       raw_cpu_generic_to_op(pcp, val, =)
 218#endif
 219
 220#ifndef raw_cpu_add_1
 221#define raw_cpu_add_1(pcp, val)         raw_cpu_generic_to_op(pcp, val, +=)
 222#endif
 223#ifndef raw_cpu_add_2
 224#define raw_cpu_add_2(pcp, val)         raw_cpu_generic_to_op(pcp, val, +=)
 225#endif
 226#ifndef raw_cpu_add_4
 227#define raw_cpu_add_4(pcp, val)         raw_cpu_generic_to_op(pcp, val, +=)
 228#endif
 229#ifndef raw_cpu_add_8
 230#define raw_cpu_add_8(pcp, val)         raw_cpu_generic_to_op(pcp, val, +=)
 231#endif
 232
 233#ifndef raw_cpu_and_1
 234#define raw_cpu_and_1(pcp, val)         raw_cpu_generic_to_op(pcp, val, &=)
 235#endif
 236#ifndef raw_cpu_and_2
 237#define raw_cpu_and_2(pcp, val)         raw_cpu_generic_to_op(pcp, val, &=)
 238#endif
 239#ifndef raw_cpu_and_4
 240#define raw_cpu_and_4(pcp, val)         raw_cpu_generic_to_op(pcp, val, &=)
 241#endif
 242#ifndef raw_cpu_and_8
 243#define raw_cpu_and_8(pcp, val)         raw_cpu_generic_to_op(pcp, val, &=)
 244#endif
 245
 246#ifndef raw_cpu_or_1
 247#define raw_cpu_or_1(pcp, val)          raw_cpu_generic_to_op(pcp, val, |=)
 248#endif
 249#ifndef raw_cpu_or_2
 250#define raw_cpu_or_2(pcp, val)          raw_cpu_generic_to_op(pcp, val, |=)
 251#endif
 252#ifndef raw_cpu_or_4
 253#define raw_cpu_or_4(pcp, val)          raw_cpu_generic_to_op(pcp, val, |=)
 254#endif
 255#ifndef raw_cpu_or_8
 256#define raw_cpu_or_8(pcp, val)          raw_cpu_generic_to_op(pcp, val, |=)
 257#endif
 258
 259#ifndef raw_cpu_add_return_1
 260#define raw_cpu_add_return_1(pcp, val)  raw_cpu_generic_add_return(pcp, val)
 261#endif
 262#ifndef raw_cpu_add_return_2
 263#define raw_cpu_add_return_2(pcp, val)  raw_cpu_generic_add_return(pcp, val)
 264#endif
 265#ifndef raw_cpu_add_return_4
 266#define raw_cpu_add_return_4(pcp, val)  raw_cpu_generic_add_return(pcp, val)
 267#endif
 268#ifndef raw_cpu_add_return_8
 269#define raw_cpu_add_return_8(pcp, val)  raw_cpu_generic_add_return(pcp, val)
 270#endif
 271
 272#ifndef raw_cpu_xchg_1
 273#define raw_cpu_xchg_1(pcp, nval)       raw_cpu_generic_xchg(pcp, nval)
 274#endif
 275#ifndef raw_cpu_xchg_2
 276#define raw_cpu_xchg_2(pcp, nval)       raw_cpu_generic_xchg(pcp, nval)
 277#endif
 278#ifndef raw_cpu_xchg_4
 279#define raw_cpu_xchg_4(pcp, nval)       raw_cpu_generic_xchg(pcp, nval)
 280#endif
 281#ifndef raw_cpu_xchg_8
 282#define raw_cpu_xchg_8(pcp, nval)       raw_cpu_generic_xchg(pcp, nval)
 283#endif
 284
 285#ifndef raw_cpu_cmpxchg_1
 286#define raw_cpu_cmpxchg_1(pcp, oval, nval) \
 287        raw_cpu_generic_cmpxchg(pcp, oval, nval)
 288#endif
 289#ifndef raw_cpu_cmpxchg_2
 290#define raw_cpu_cmpxchg_2(pcp, oval, nval) \
 291        raw_cpu_generic_cmpxchg(pcp, oval, nval)
 292#endif
 293#ifndef raw_cpu_cmpxchg_4
 294#define raw_cpu_cmpxchg_4(pcp, oval, nval) \
 295        raw_cpu_generic_cmpxchg(pcp, oval, nval)
 296#endif
 297#ifndef raw_cpu_cmpxchg_8
 298#define raw_cpu_cmpxchg_8(pcp, oval, nval) \
 299        raw_cpu_generic_cmpxchg(pcp, oval, nval)
 300#endif
 301
 302#ifndef raw_cpu_cmpxchg_double_1
 303#define raw_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \
 304        raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
 305#endif
 306#ifndef raw_cpu_cmpxchg_double_2
 307#define raw_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \
 308        raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
 309#endif
 310#ifndef raw_cpu_cmpxchg_double_4
 311#define raw_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \
 312        raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
 313#endif
 314#ifndef raw_cpu_cmpxchg_double_8
 315#define raw_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \
 316        raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
 317#endif
 318
 319#ifndef this_cpu_read_1
 320#define this_cpu_read_1(pcp)            this_cpu_generic_read(pcp)
 321#endif
 322#ifndef this_cpu_read_2
 323#define this_cpu_read_2(pcp)            this_cpu_generic_read(pcp)
 324#endif
 325#ifndef this_cpu_read_4
 326#define this_cpu_read_4(pcp)            this_cpu_generic_read(pcp)
 327#endif
 328#ifndef this_cpu_read_8
 329#define this_cpu_read_8(pcp)            this_cpu_generic_read(pcp)
 330#endif
 331
 332#ifndef this_cpu_write_1
 333#define this_cpu_write_1(pcp, val)      this_cpu_generic_to_op(pcp, val, =)
 334#endif
 335#ifndef this_cpu_write_2
 336#define this_cpu_write_2(pcp, val)      this_cpu_generic_to_op(pcp, val, =)
 337#endif
 338#ifndef this_cpu_write_4
 339#define this_cpu_write_4(pcp, val)      this_cpu_generic_to_op(pcp, val, =)
 340#endif
 341#ifndef this_cpu_write_8
 342#define this_cpu_write_8(pcp, val)      this_cpu_generic_to_op(pcp, val, =)
 343#endif
 344
 345#ifndef this_cpu_add_1
 346#define this_cpu_add_1(pcp, val)        this_cpu_generic_to_op(pcp, val, +=)
 347#endif
 348#ifndef this_cpu_add_2
 349#define this_cpu_add_2(pcp, val)        this_cpu_generic_to_op(pcp, val, +=)
 350#endif
 351#ifndef this_cpu_add_4
 352#define this_cpu_add_4(pcp, val)        this_cpu_generic_to_op(pcp, val, +=)
 353#endif
 354#ifndef this_cpu_add_8
 355#define this_cpu_add_8(pcp, val)        this_cpu_generic_to_op(pcp, val, +=)
 356#endif
 357
 358#ifndef this_cpu_and_1
 359#define this_cpu_and_1(pcp, val)        this_cpu_generic_to_op(pcp, val, &=)
 360#endif
 361#ifndef this_cpu_and_2
 362#define this_cpu_and_2(pcp, val)        this_cpu_generic_to_op(pcp, val, &=)
 363#endif
 364#ifndef this_cpu_and_4
 365#define this_cpu_and_4(pcp, val)        this_cpu_generic_to_op(pcp, val, &=)
 366#endif
 367#ifndef this_cpu_and_8
 368#define this_cpu_and_8(pcp, val)        this_cpu_generic_to_op(pcp, val, &=)
 369#endif
 370
 371#ifndef this_cpu_or_1
 372#define this_cpu_or_1(pcp, val)         this_cpu_generic_to_op(pcp, val, |=)
 373#endif
 374#ifndef this_cpu_or_2
 375#define this_cpu_or_2(pcp, val)         this_cpu_generic_to_op(pcp, val, |=)
 376#endif
 377#ifndef this_cpu_or_4
 378#define this_cpu_or_4(pcp, val)         this_cpu_generic_to_op(pcp, val, |=)
 379#endif
 380#ifndef this_cpu_or_8
 381#define this_cpu_or_8(pcp, val)         this_cpu_generic_to_op(pcp, val, |=)
 382#endif
 383
 384#ifndef this_cpu_add_return_1
 385#define this_cpu_add_return_1(pcp, val) this_cpu_generic_add_return(pcp, val)
 386#endif
 387#ifndef this_cpu_add_return_2
 388#define this_cpu_add_return_2(pcp, val) this_cpu_generic_add_return(pcp, val)
 389#endif
 390#ifndef this_cpu_add_return_4
 391#define this_cpu_add_return_4(pcp, val) this_cpu_generic_add_return(pcp, val)
 392#endif
 393#ifndef this_cpu_add_return_8
 394#define this_cpu_add_return_8(pcp, val) this_cpu_generic_add_return(pcp, val)
 395#endif
 396
 397#ifndef this_cpu_xchg_1
 398#define this_cpu_xchg_1(pcp, nval)      this_cpu_generic_xchg(pcp, nval)
 399#endif
 400#ifndef this_cpu_xchg_2
 401#define this_cpu_xchg_2(pcp, nval)      this_cpu_generic_xchg(pcp, nval)
 402#endif
 403#ifndef this_cpu_xchg_4
 404#define this_cpu_xchg_4(pcp, nval)      this_cpu_generic_xchg(pcp, nval)
 405#endif
 406#ifndef this_cpu_xchg_8
 407#define this_cpu_xchg_8(pcp, nval)      this_cpu_generic_xchg(pcp, nval)
 408#endif
 409
 410#ifndef this_cpu_cmpxchg_1
 411#define this_cpu_cmpxchg_1(pcp, oval, nval) \
 412        this_cpu_generic_cmpxchg(pcp, oval, nval)
 413#endif
 414#ifndef this_cpu_cmpxchg_2
 415#define this_cpu_cmpxchg_2(pcp, oval, nval) \
 416        this_cpu_generic_cmpxchg(pcp, oval, nval)
 417#endif
 418#ifndef this_cpu_cmpxchg_4
 419#define this_cpu_cmpxchg_4(pcp, oval, nval) \
 420        this_cpu_generic_cmpxchg(pcp, oval, nval)
 421#endif
 422#ifndef this_cpu_cmpxchg_8
 423#define this_cpu_cmpxchg_8(pcp, oval, nval) \
 424        this_cpu_generic_cmpxchg(pcp, oval, nval)
 425#endif
 426
 427#ifndef this_cpu_cmpxchg_double_1
 428#define this_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \
 429        this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
 430#endif
 431#ifndef this_cpu_cmpxchg_double_2
 432#define this_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \
 433        this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
 434#endif
 435#ifndef this_cpu_cmpxchg_double_4
 436#define this_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \
 437        this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
 438#endif
 439#ifndef this_cpu_cmpxchg_double_8
 440#define this_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \
 441        this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
 442#endif
 443
 444#endif /* _ASM_GENERIC_PERCPU_H_ */
 445