1/* SPDX-License-Identifier: GPL-2.0-only */ 2#ifndef __MIPS_ASM_SYNC_H__ 3#define __MIPS_ASM_SYNC_H__ 4 5/* 6 * sync types are defined by the MIPS64 Instruction Set documentation in Volume 7 * II-A of the MIPS Architecture Reference Manual, which can be found here: 8 * 9 * https://www.mips.com/?do-download=the-mips64-instruction-set-v6-06 10 * 11 * Two types of barrier are provided: 12 * 13 * 1) Completion barriers, which ensure that a memory operation has actually 14 * completed & often involve stalling the CPU pipeline to do so. 15 * 16 * 2) Ordering barriers, which only ensure that affected memory operations 17 * won't be reordered in the CPU pipeline in a manner that violates the 18 * restrictions imposed by the barrier. 19 * 20 * Ordering barriers can be more efficient than completion barriers, since: 21 * 22 * a) Ordering barriers only require memory access instructions which preceed 23 * them in program order (older instructions) to reach a point in the 24 * load/store datapath beyond which reordering is not possible before 25 * allowing memory access instructions which follow them (younger 26 * instructions) to be performed. That is, older instructions don't 27 * actually need to complete - they just need to get far enough that all 28 * other coherent CPUs will observe their completion before they observe 29 * the effects of younger instructions. 30 * 31 * b) Multiple variants of ordering barrier are provided which allow the 32 * effects to be restricted to different combinations of older or younger 33 * loads or stores. By way of example, if we only care that stores older 34 * than a barrier are observed prior to stores that are younger than a 35 * barrier & don't care about the ordering of loads then the 'wmb' 36 * ordering barrier can be used. Limiting the barrier's effects to stores 37 * allows loads to continue unaffected & potentially allows the CPU to 38 * make progress faster than if younger loads had to wait for older stores 39 * to complete. 40 */ 41 42/* 43 * No sync instruction at all; used to allow code to nullify the effect of the 44 * __SYNC() macro without needing lots of #ifdefery. 45 */ 46#define __SYNC_none -1 47 48/* 49 * A full completion barrier; all memory accesses appearing prior to this sync 50 * instruction in program order must complete before any memory accesses 51 * appearing after this sync instruction in program order. 52 */ 53#define __SYNC_full 0x00 54 55/* 56 * For now we use a full completion barrier to implement all sync types, until 57 * we're satisfied that lightweight ordering barriers defined by MIPSr6 are 58 * sufficient to uphold our desired memory model. 59 */ 60#define __SYNC_aq __SYNC_full 61#define __SYNC_rl __SYNC_full 62#define __SYNC_mb __SYNC_full 63 64/* 65 * ...except on Cavium Octeon CPUs, which have been using the 'wmb' ordering 66 * barrier since 2010 & omit 'rmb' barriers because the CPUs don't perform 67 * speculative reads. 68 */ 69#ifdef CONFIG_CPU_CAVIUM_OCTEON 70# define __SYNC_rmb __SYNC_none 71# define __SYNC_wmb 0x04 72#else 73# define __SYNC_rmb __SYNC_full 74# define __SYNC_wmb __SYNC_full 75#endif 76 77/* 78 * A GINV sync is a little different; it doesn't relate directly to loads or 79 * stores, but instead causes synchronization of an icache or TLB global 80 * invalidation operation triggered by the ginvi or ginvt instructions 81 * respectively. In cases where we need to know that a ginvi or ginvt operation 82 * has been performed by all coherent CPUs, we must issue a sync instruction of 83 * this type. Once this instruction graduates all coherent CPUs will have 84 * observed the invalidation. 85 */ 86#define __SYNC_ginv 0x14 87 88/* Trivial; indicate that we always need this sync instruction. */ 89#define __SYNC_always (1 << 0) 90 91/* 92 * Indicate that we need this sync instruction only on systems with weakly 93 * ordered memory access. In general this is most MIPS systems, but there are 94 * exceptions which provide strongly ordered memory. 95 */ 96#ifdef CONFIG_WEAK_ORDERING 97# define __SYNC_weak_ordering (1 << 1) 98#else 99# define __SYNC_weak_ordering 0 100#endif 101 102/* 103 * Indicate that we need this sync instruction only on systems where LL/SC 104 * don't implicitly provide a memory barrier. In general this is most MIPS 105 * systems. 106 */ 107#ifdef CONFIG_WEAK_REORDERING_BEYOND_LLSC 108# define __SYNC_weak_llsc (1 << 2) 109#else 110# define __SYNC_weak_llsc 0 111#endif 112 113/* 114 * Some Loongson 3 CPUs have a bug wherein execution of a memory access (load, 115 * store or prefetch) in between an LL & SC can cause the SC instruction to 116 * erroneously succeed, breaking atomicity. Whilst it's unusual to write code 117 * containing such sequences, this bug bites harder than we might otherwise 118 * expect due to reordering & speculation: 119 * 120 * 1) A memory access appearing prior to the LL in program order may actually 121 * be executed after the LL - this is the reordering case. 122 * 123 * In order to avoid this we need to place a memory barrier (ie. a SYNC 124 * instruction) prior to every LL instruction, in between it and any earlier 125 * memory access instructions. 126 * 127 * This reordering case is fixed by 3A R2 CPUs, ie. 3A2000 models and later. 128 * 129 * 2) If a conditional branch exists between an LL & SC with a target outside 130 * of the LL-SC loop, for example an exit upon value mismatch in cmpxchg() 131 * or similar, then misprediction of the branch may allow speculative 132 * execution of memory accesses from outside of the LL-SC loop. 133 * 134 * In order to avoid this we need a memory barrier (ie. a SYNC instruction) 135 * at each affected branch target. 136 * 137 * This case affects all current Loongson 3 CPUs. 138 * 139 * The above described cases cause an error in the cache coherence protocol; 140 * such that the Invalidate of a competing LL-SC goes 'missing' and SC 141 * erroneously observes its core still has Exclusive state and lets the SC 142 * proceed. 143 * 144 * Therefore the error only occurs on SMP systems. 145 */ 146#ifdef CONFIG_CPU_LOONGSON3_WORKAROUNDS 147# define __SYNC_loongson3_war (1 << 31) 148#else 149# define __SYNC_loongson3_war 0 150#endif 151 152/* 153 * Some Cavium Octeon CPUs suffer from a bug that causes a single wmb ordering 154 * barrier to be ineffective, requiring the use of 2 in sequence to provide an 155 * effective barrier as noted by commit 6b07d38aaa52 ("MIPS: Octeon: Use 156 * optimized memory barrier primitives."). Here we specify that the affected 157 * sync instructions should be emitted twice. 158 * Note that this expression is evaluated by the assembler (not the compiler), 159 * and that the assembler evaluates '==' as 0 or -1, not 0 or 1. 160 */ 161#ifdef CONFIG_CPU_CAVIUM_OCTEON 162# define __SYNC_rpt(type) (1 - (type == __SYNC_wmb)) 163#else 164# define __SYNC_rpt(type) 1 165#endif 166 167/* 168 * The main event. Here we actually emit a sync instruction of a given type, if 169 * reason is non-zero. 170 * 171 * In future we have the option of emitting entries in a fixups-style table 172 * here that would allow us to opportunistically remove some sync instructions 173 * when we detect at runtime that we're running on a CPU that doesn't need 174 * them. 175 */ 176#ifdef CONFIG_CPU_HAS_SYNC 177# define ____SYNC(_type, _reason, _else) \ 178 .if (( _type ) != -1) && ( _reason ); \ 179 .set push; \ 180 .set MIPS_ISA_LEVEL_RAW; \ 181 .rept __SYNC_rpt(_type); \ 182 sync _type; \ 183 .endr; \ 184 .set pop; \ 185 .else; \ 186 _else; \ 187 .endif 188#else 189# define ____SYNC(_type, _reason, _else) 190#endif 191 192/* 193 * Preprocessor magic to expand macros used as arguments before we insert them 194 * into assembly code. 195 */ 196#ifdef __ASSEMBLY__ 197# define ___SYNC(type, reason, else) \ 198 ____SYNC(type, reason, else) 199#else 200# define ___SYNC(type, reason, else) \ 201 __stringify(____SYNC(type, reason, else)) 202#endif 203 204#define __SYNC(type, reason) \ 205 ___SYNC(__SYNC_##type, __SYNC_##reason, ) 206#define __SYNC_ELSE(type, reason, else) \ 207 ___SYNC(__SYNC_##type, __SYNC_##reason, else) 208 209#endif /* __MIPS_ASM_SYNC_H__ */ 210