LXR linux/arch/x86/include/asm/barrier.h

   1#ifndef _ASM_X86_BARRIER_H
   2#define _ASM_X86_BARRIER_H
   3
   4#include <asm/alternative.h>
   5#include <asm/nops.h>
   6
   7/*
   8 * Force strict CPU ordering.
   9 * And yes, this is required on UP too when we're talking
  10 * to devices.
  11 */
  12
  13#ifdef CONFIG_X86_32
  14/*
  15 * Some non-Intel clones support out of order store. wmb() ceases to be a
  16 * nop for these.
  17 */
  18#define mb() alternative("lock; addl $0,0(%%esp)", "mfence", X86_FEATURE_XMM2)
  19#define rmb() alternative("lock; addl $0,0(%%esp)", "lfence", X86_FEATURE_XMM2)
  20#define wmb() alternative("lock; addl $0,0(%%esp)", "sfence", X86_FEATURE_XMM)
  21#else
  22#define mb()    asm volatile("mfence":::"memory")
  23#define rmb()   asm volatile("lfence":::"memory")
  24#define wmb()   asm volatile("sfence" ::: "memory")
  25#endif
  26
  27/**
  28 * read_barrier_depends - Flush all pending reads that subsequents reads
  29 * depend on.
  30 *
  31 * No data-dependent reads from memory-like regions are ever reordered
  32 * over this barrier.  All reads preceding this primitive are guaranteed
  33 * to access memory (but not necessarily other CPUs' caches) before any
  34 * reads following this primitive that depend on the data return by
  35 * any of the preceding reads.  This primitive is much lighter weight than
  36 * rmb() on most CPUs, and is never heavier weight than is
  37 * rmb().
  38 *
  39 * These ordering constraints are respected by both the local CPU
  40 * and the compiler.
  41 *
  42 * Ordering is not guaranteed by anything other than these primitives,
  43 * not even by data dependencies.  See the documentation for
  44 * memory_barrier() for examples and URLs to more information.
  45 *
  46 * For example, the following code would force ordering (the initial
  47 * value of "a" is zero, "b" is one, and "p" is "&a"):
  48 *
  49 * <programlisting>
  50 *      CPU 0                           CPU 1
  51 *
  52 *      b = 2;
  53 *      memory_barrier();
  54 *      p = &b;                         q = p;
  55 *                                      read_barrier_depends();
  56 *                                      d = *q;
  57 * </programlisting>
  58 *
  59 * because the read of "*q" depends on the read of "p" and these
  60 * two reads are separated by a read_barrier_depends().  However,
  61 * the following code, with the same initial values for "a" and "b":
  62 *
  63 * <programlisting>
  64 *      CPU 0                           CPU 1
  65 *
  66 *      a = 2;
  67 *      memory_barrier();
  68 *      b = 3;                          y = b;
  69 *                                      read_barrier_depends();
  70 *                                      x = a;
  71 * </programlisting>
  72 *
  73 * does not enforce ordering, since there is no data dependency between
  74 * the read of "a" and the read of "b".  Therefore, on some CPUs, such
  75 * as Alpha, "y" could be set to 3 and "x" to 0.  Use rmb()
  76 * in cases like this where there are no data dependencies.
  77 **/
  78
  79#define read_barrier_depends()  do { } while (0)
  80
  81#ifdef CONFIG_SMP
  82#define smp_mb()        mb()
  83#ifdef CONFIG_X86_PPRO_FENCE
  84# define smp_rmb()      rmb()
  85#else
  86# define smp_rmb()      barrier()
  87#endif
  88#ifdef CONFIG_X86_OOSTORE
  89# define smp_wmb()      wmb()
  90#else
  91# define smp_wmb()      barrier()
  92#endif
  93#define smp_read_barrier_depends()      read_barrier_depends()
  94#define set_mb(var, value) do { (void)xchg(&var, value); } while (0)
  95#else
  96#define smp_mb()        barrier()
  97#define smp_rmb()       barrier()
  98#define smp_wmb()       barrier()
  99#define smp_read_barrier_depends()      do { } while (0)
 100#define set_mb(var, value) do { var = value; barrier(); } while (0)
 101#endif
 102
 103/*
 104 * Stop RDTSC speculation. This is needed when you need to use RDTSC
 105 * (or get_cycles or vread that possibly accesses the TSC) in a defined
 106 * code region.
 107 *
 108 * (Could use an alternative three way for this if there was one.)
 109 */
 110static __always_inline void rdtsc_barrier(void)
 111{
 112        alternative(ASM_NOP3, "mfence", X86_FEATURE_MFENCE_RDTSC);
 113        alternative(ASM_NOP3, "lfence", X86_FEATURE_LFENCE_RDTSC);
 114}
 115
 116#endif /* _ASM_X86_BARRIER_H */
 117