LXR linux/include/asm-generic/barrier.h

   1/* SPDX-License-Identifier: GPL-2.0-or-later */
   2/*
   3 * Generic barrier definitions.
   4 *
   5 * It should be possible to use these on really simple architectures,
   6 * but it serves more as a starting point for new ports.
   7 *
   8 * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved.
   9 * Written by David Howells (dhowells@redhat.com)
  10 */
  11#ifndef __ASM_GENERIC_BARRIER_H
  12#define __ASM_GENERIC_BARRIER_H
  13
  14#ifndef __ASSEMBLY__
  15
  16#include <linux/compiler.h>
  17
  18#ifndef nop
  19#define nop()   asm volatile ("nop")
  20#endif
  21
  22/*
  23 * Force strict CPU ordering. And yes, this is required on UP too when we're
  24 * talking to devices.
  25 *
  26 * Fall back to compiler barriers if nothing better is provided.
  27 */
  28
  29#ifndef mb
  30#define mb()    barrier()
  31#endif
  32
  33#ifndef rmb
  34#define rmb()   mb()
  35#endif
  36
  37#ifndef wmb
  38#define wmb()   mb()
  39#endif
  40
  41#ifndef dma_rmb
  42#define dma_rmb()       rmb()
  43#endif
  44
  45#ifndef dma_wmb
  46#define dma_wmb()       wmb()
  47#endif
  48
  49#ifndef read_barrier_depends
  50#define read_barrier_depends()          do { } while (0)
  51#endif
  52
  53#ifndef __smp_mb
  54#define __smp_mb()      mb()
  55#endif
  56
  57#ifndef __smp_rmb
  58#define __smp_rmb()     rmb()
  59#endif
  60
  61#ifndef __smp_wmb
  62#define __smp_wmb()     wmb()
  63#endif
  64
  65#ifndef __smp_read_barrier_depends
  66#define __smp_read_barrier_depends()    read_barrier_depends()
  67#endif
  68
  69#ifdef CONFIG_SMP
  70
  71#ifndef smp_mb
  72#define smp_mb()        __smp_mb()
  73#endif
  74
  75#ifndef smp_rmb
  76#define smp_rmb()       __smp_rmb()
  77#endif
  78
  79#ifndef smp_wmb
  80#define smp_wmb()       __smp_wmb()
  81#endif
  82
  83#ifndef smp_read_barrier_depends
  84#define smp_read_barrier_depends()      __smp_read_barrier_depends()
  85#endif
  86
  87#else   /* !CONFIG_SMP */
  88
  89#ifndef smp_mb
  90#define smp_mb()        barrier()
  91#endif
  92
  93#ifndef smp_rmb
  94#define smp_rmb()       barrier()
  95#endif
  96
  97#ifndef smp_wmb
  98#define smp_wmb()       barrier()
  99#endif
 100
 101#ifndef smp_read_barrier_depends
 102#define smp_read_barrier_depends()      do { } while (0)
 103#endif
 104
 105#endif  /* CONFIG_SMP */
 106
 107#ifndef __smp_store_mb
 108#define __smp_store_mb(var, value)  do { WRITE_ONCE(var, value); __smp_mb(); } while (0)
 109#endif
 110
 111#ifndef __smp_mb__before_atomic
 112#define __smp_mb__before_atomic()       __smp_mb()
 113#endif
 114
 115#ifndef __smp_mb__after_atomic
 116#define __smp_mb__after_atomic()        __smp_mb()
 117#endif
 118
 119#ifndef __smp_store_release
 120#define __smp_store_release(p, v)                                       \
 121do {                                                                    \
 122        compiletime_assert_atomic_type(*p);                             \
 123        __smp_mb();                                                     \
 124        WRITE_ONCE(*p, v);                                              \
 125} while (0)
 126#endif
 127
 128#ifndef __smp_load_acquire
 129#define __smp_load_acquire(p)                                           \
 130({                                                                      \
 131        typeof(*p) ___p1 = READ_ONCE(*p);                               \
 132        compiletime_assert_atomic_type(*p);                             \
 133        __smp_mb();                                                     \
 134        ___p1;                                                          \
 135})
 136#endif
 137
 138#ifdef CONFIG_SMP
 139
 140#ifndef smp_store_mb
 141#define smp_store_mb(var, value)  __smp_store_mb(var, value)
 142#endif
 143
 144#ifndef smp_mb__before_atomic
 145#define smp_mb__before_atomic() __smp_mb__before_atomic()
 146#endif
 147
 148#ifndef smp_mb__after_atomic
 149#define smp_mb__after_atomic()  __smp_mb__after_atomic()
 150#endif
 151
 152#ifndef smp_store_release
 153#define smp_store_release(p, v) __smp_store_release(p, v)
 154#endif
 155
 156#ifndef smp_load_acquire
 157#define smp_load_acquire(p) __smp_load_acquire(p)
 158#endif
 159
 160#else   /* !CONFIG_SMP */
 161
 162#ifndef smp_store_mb
 163#define smp_store_mb(var, value)  do { WRITE_ONCE(var, value); barrier(); } while (0)
 164#endif
 165
 166#ifndef smp_mb__before_atomic
 167#define smp_mb__before_atomic() barrier()
 168#endif
 169
 170#ifndef smp_mb__after_atomic
 171#define smp_mb__after_atomic()  barrier()
 172#endif
 173
 174#ifndef smp_store_release
 175#define smp_store_release(p, v)                                         \
 176do {                                                                    \
 177        compiletime_assert_atomic_type(*p);                             \
 178        barrier();                                                      \
 179        WRITE_ONCE(*p, v);                                              \
 180} while (0)
 181#endif
 182
 183#ifndef smp_load_acquire
 184#define smp_load_acquire(p)                                             \
 185({                                                                      \
 186        typeof(*p) ___p1 = READ_ONCE(*p);                               \
 187        compiletime_assert_atomic_type(*p);                             \
 188        barrier();                                                      \
 189        ___p1;                                                          \
 190})
 191#endif
 192
 193#endif  /* CONFIG_SMP */
 194
 195/* Barriers for virtual machine guests when talking to an SMP host */
 196#define virt_mb() __smp_mb()
 197#define virt_rmb() __smp_rmb()
 198#define virt_wmb() __smp_wmb()
 199#define virt_read_barrier_depends() __smp_read_barrier_depends()
 200#define virt_store_mb(var, value) __smp_store_mb(var, value)
 201#define virt_mb__before_atomic() __smp_mb__before_atomic()
 202#define virt_mb__after_atomic() __smp_mb__after_atomic()
 203#define virt_store_release(p, v) __smp_store_release(p, v)
 204#define virt_load_acquire(p) __smp_load_acquire(p)
 205
 206/**
 207 * smp_acquire__after_ctrl_dep() - Provide ACQUIRE ordering after a control dependency
 208 *
 209 * A control dependency provides a LOAD->STORE order, the additional RMB
 210 * provides LOAD->LOAD order, together they provide LOAD->{LOAD,STORE} order,
 211 * aka. (load)-ACQUIRE.
 212 *
 213 * Architectures that do not do load speculation can have this be barrier().
 214 */
 215#ifndef smp_acquire__after_ctrl_dep
 216#define smp_acquire__after_ctrl_dep()           smp_rmb()
 217#endif
 218
 219/**
 220 * smp_cond_load_relaxed() - (Spin) wait for cond with no ordering guarantees
 221 * @ptr: pointer to the variable to wait on
 222 * @cond: boolean expression to wait for
 223 *
 224 * Equivalent to using READ_ONCE() on the condition variable.
 225 *
 226 * Due to C lacking lambda expressions we load the value of *ptr into a
 227 * pre-named variable @VAL to be used in @cond.
 228 */
 229#ifndef smp_cond_load_relaxed
 230#define smp_cond_load_relaxed(ptr, cond_expr) ({                \
 231        typeof(ptr) __PTR = (ptr);                              \
 232        typeof(*ptr) VAL;                                       \
 233        for (;;) {                                              \
 234                VAL = READ_ONCE(*__PTR);                        \
 235                if (cond_expr)                                  \
 236                        break;                                  \
 237                cpu_relax();                                    \
 238        }                                                       \
 239        VAL;                                                    \
 240})
 241#endif
 242
 243/**
 244 * smp_cond_load_acquire() - (Spin) wait for cond with ACQUIRE ordering
 245 * @ptr: pointer to the variable to wait on
 246 * @cond: boolean expression to wait for
 247 *
 248 * Equivalent to using smp_load_acquire() on the condition variable but employs
 249 * the control dependency of the wait to reduce the barrier on many platforms.
 250 */
 251#ifndef smp_cond_load_acquire
 252#define smp_cond_load_acquire(ptr, cond_expr) ({                \
 253        typeof(*ptr) _val;                                      \
 254        _val = smp_cond_load_relaxed(ptr, cond_expr);           \
 255        smp_acquire__after_ctrl_dep();                          \
 256        _val;                                                   \
 257})
 258#endif
 259
 260#endif /* !__ASSEMBLY__ */
 261#endif /* __ASM_GENERIC_BARRIER_H */
 262