LXR linux/arch/alpha/include/asm/xchg.h

   1/* SPDX-License-Identifier: GPL-2.0 */
   2#ifndef _ALPHA_CMPXCHG_H
   3#error Do not include xchg.h directly!
   4#else
   5/*
   6 * xchg/xchg_local and cmpxchg/cmpxchg_local share the same code
   7 * except that local version do not have the expensive memory barrier.
   8 * So this file is included twice from asm/cmpxchg.h.
   9 */
  10
  11/*
  12 * Atomic exchange.
  13 * Since it can be used to implement critical sections
  14 * it must clobber "memory" (also for interrupts in UP).
  15 */
  16
  17static inline unsigned long
  18____xchg(_u8, volatile char *m, unsigned long val)
  19{
  20        unsigned long ret, tmp, addr64;
  21
  22        __asm__ __volatile__(
  23        "       andnot  %4,7,%3\n"
  24        "       insbl   %1,%4,%1\n"
  25        "1:     ldq_l   %2,0(%3)\n"
  26        "       extbl   %2,%4,%0\n"
  27        "       mskbl   %2,%4,%2\n"
  28        "       or      %1,%2,%2\n"
  29        "       stq_c   %2,0(%3)\n"
  30        "       beq     %2,2f\n"
  31        ".subsection 2\n"
  32        "2:     br      1b\n"
  33        ".previous"
  34        : "=&r" (ret), "=&r" (val), "=&r" (tmp), "=&r" (addr64)
  35        : "r" ((long)m), "1" (val) : "memory");
  36
  37        return ret;
  38}
  39
  40static inline unsigned long
  41____xchg(_u16, volatile short *m, unsigned long val)
  42{
  43        unsigned long ret, tmp, addr64;
  44
  45        __asm__ __volatile__(
  46        "       andnot  %4,7,%3\n"
  47        "       inswl   %1,%4,%1\n"
  48        "1:     ldq_l   %2,0(%3)\n"
  49        "       extwl   %2,%4,%0\n"
  50        "       mskwl   %2,%4,%2\n"
  51        "       or      %1,%2,%2\n"
  52        "       stq_c   %2,0(%3)\n"
  53        "       beq     %2,2f\n"
  54        ".subsection 2\n"
  55        "2:     br      1b\n"
  56        ".previous"
  57        : "=&r" (ret), "=&r" (val), "=&r" (tmp), "=&r" (addr64)
  58        : "r" ((long)m), "1" (val) : "memory");
  59
  60        return ret;
  61}
  62
  63static inline unsigned long
  64____xchg(_u32, volatile int *m, unsigned long val)
  65{
  66        unsigned long dummy;
  67
  68        __asm__ __volatile__(
  69        "1:     ldl_l %0,%4\n"
  70        "       bis $31,%3,%1\n"
  71        "       stl_c %1,%2\n"
  72        "       beq %1,2f\n"
  73        ".subsection 2\n"
  74        "2:     br 1b\n"
  75        ".previous"
  76        : "=&r" (val), "=&r" (dummy), "=m" (*m)
  77        : "rI" (val), "m" (*m) : "memory");
  78
  79        return val;
  80}
  81
  82static inline unsigned long
  83____xchg(_u64, volatile long *m, unsigned long val)
  84{
  85        unsigned long dummy;
  86
  87        __asm__ __volatile__(
  88        "1:     ldq_l %0,%4\n"
  89        "       bis $31,%3,%1\n"
  90        "       stq_c %1,%2\n"
  91        "       beq %1,2f\n"
  92        ".subsection 2\n"
  93        "2:     br 1b\n"
  94        ".previous"
  95        : "=&r" (val), "=&r" (dummy), "=m" (*m)
  96        : "rI" (val), "m" (*m) : "memory");
  97
  98        return val;
  99}
 100
 101/* This function doesn't exist, so you'll get a linker error
 102   if something tries to do an invalid xchg().  */
 103extern void __xchg_called_with_bad_pointer(void);
 104
 105static __always_inline unsigned long
 106____xchg(, volatile void *ptr, unsigned long x, int size)
 107{
 108        switch (size) {
 109                case 1:
 110                        return ____xchg(_u8, ptr, x);
 111                case 2:
 112                        return ____xchg(_u16, ptr, x);
 113                case 4:
 114                        return ____xchg(_u32, ptr, x);
 115                case 8:
 116                        return ____xchg(_u64, ptr, x);
 117        }
 118        __xchg_called_with_bad_pointer();
 119        return x;
 120}
 121
 122/*
 123 * Atomic compare and exchange.  Compare OLD with MEM, if identical,
 124 * store NEW in MEM.  Return the initial value in MEM.  Success is
 125 * indicated by comparing RETURN with OLD.
 126 */
 127
 128static inline unsigned long
 129____cmpxchg(_u8, volatile char *m, unsigned char old, unsigned char new)
 130{
 131        unsigned long prev, tmp, cmp, addr64;
 132
 133        __asm__ __volatile__(
 134        "       andnot  %5,7,%4\n"
 135        "       insbl   %1,%5,%1\n"
 136        "1:     ldq_l   %2,0(%4)\n"
 137        "       extbl   %2,%5,%0\n"
 138        "       cmpeq   %0,%6,%3\n"
 139        "       beq     %3,2f\n"
 140        "       mskbl   %2,%5,%2\n"
 141        "       or      %1,%2,%2\n"
 142        "       stq_c   %2,0(%4)\n"
 143        "       beq     %2,3f\n"
 144        "2:\n"
 145        ".subsection 2\n"
 146        "3:     br      1b\n"
 147        ".previous"
 148        : "=&r" (prev), "=&r" (new), "=&r" (tmp), "=&r" (cmp), "=&r" (addr64)
 149        : "r" ((long)m), "Ir" (old), "1" (new) : "memory");
 150
 151        return prev;
 152}
 153
 154static inline unsigned long
 155____cmpxchg(_u16, volatile short *m, unsigned short old, unsigned short new)
 156{
 157        unsigned long prev, tmp, cmp, addr64;
 158
 159        __asm__ __volatile__(
 160        "       andnot  %5,7,%4\n"
 161        "       inswl   %1,%5,%1\n"
 162        "1:     ldq_l   %2,0(%4)\n"
 163        "       extwl   %2,%5,%0\n"
 164        "       cmpeq   %0,%6,%3\n"
 165        "       beq     %3,2f\n"
 166        "       mskwl   %2,%5,%2\n"
 167        "       or      %1,%2,%2\n"
 168        "       stq_c   %2,0(%4)\n"
 169        "       beq     %2,3f\n"
 170        "2:\n"
 171        ".subsection 2\n"
 172        "3:     br      1b\n"
 173        ".previous"
 174        : "=&r" (prev), "=&r" (new), "=&r" (tmp), "=&r" (cmp), "=&r" (addr64)
 175        : "r" ((long)m), "Ir" (old), "1" (new) : "memory");
 176
 177        return prev;
 178}
 179
 180static inline unsigned long
 181____cmpxchg(_u32, volatile int *m, int old, int new)
 182{
 183        unsigned long prev, cmp;
 184
 185        __asm__ __volatile__(
 186        "1:     ldl_l %0,%5\n"
 187        "       cmpeq %0,%3,%1\n"
 188        "       beq %1,2f\n"
 189        "       mov %4,%1\n"
 190        "       stl_c %1,%2\n"
 191        "       beq %1,3f\n"
 192        "2:\n"
 193        ".subsection 2\n"
 194        "3:     br 1b\n"
 195        ".previous"
 196        : "=&r"(prev), "=&r"(cmp), "=m"(*m)
 197        : "r"((long) old), "r"(new), "m"(*m) : "memory");
 198
 199        return prev;
 200}
 201
 202static inline unsigned long
 203____cmpxchg(_u64, volatile long *m, unsigned long old, unsigned long new)
 204{
 205        unsigned long prev, cmp;
 206
 207        __asm__ __volatile__(
 208        "1:     ldq_l %0,%5\n"
 209        "       cmpeq %0,%3,%1\n"
 210        "       beq %1,2f\n"
 211        "       mov %4,%1\n"
 212        "       stq_c %1,%2\n"
 213        "       beq %1,3f\n"
 214        "2:\n"
 215        ".subsection 2\n"
 216        "3:     br 1b\n"
 217        ".previous"
 218        : "=&r"(prev), "=&r"(cmp), "=m"(*m)
 219        : "r"((long) old), "r"(new), "m"(*m) : "memory");
 220
 221        return prev;
 222}
 223
 224/* This function doesn't exist, so you'll get a linker error
 225   if something tries to do an invalid cmpxchg().  */
 226extern void __cmpxchg_called_with_bad_pointer(void);
 227
 228static __always_inline unsigned long
 229____cmpxchg(, volatile void *ptr, unsigned long old, unsigned long new,
 230              int size)
 231{
 232        switch (size) {
 233                case 1:
 234                        return ____cmpxchg(_u8, ptr, old, new);
 235                case 2:
 236                        return ____cmpxchg(_u16, ptr, old, new);
 237                case 4:
 238                        return ____cmpxchg(_u32, ptr, old, new);
 239                case 8:
 240                        return ____cmpxchg(_u64, ptr, old, new);
 241        }
 242        __cmpxchg_called_with_bad_pointer();
 243        return old;
 244}
 245
 246#endif
 247