LXR linux/arch/sparc/lib/atomic

   1/* SPDX-License-Identifier: GPL-2.0 */
   2/* atomic.S: These things are too big to do inline.
   3 *
   4 * Copyright (C) 1999, 2007 2012 David S. Miller (davem@davemloft.net)
   5 */
   6
   7#include <linux/linkage.h>
   8#include <asm/asi.h>
   9#include <asm/backoff.h>
  10#include <asm/export.h>
  11
  12        .text
  13
  14        /* Three versions of the atomic routines, one that
  15         * does not return a value and does not perform
  16         * memory barriers, and a two which return
  17         * a value, the new and old value resp. and does the
  18         * barriers.
  19         */
  20
  21#define ATOMIC_OP(op)                                                   \
  22ENTRY(atomic_##op) /* %o0 = increment, %o1 = atomic_ptr */              \
  23        BACKOFF_SETUP(%o2);                                             \
  241:      lduw    [%o1], %g1;                                             \
  25        op      %g1, %o0, %g7;                                          \
  26        cas     [%o1], %g1, %g7;                                        \
  27        cmp     %g1, %g7;                                               \
  28        bne,pn  %icc, BACKOFF_LABEL(2f, 1b);                            \
  29         nop;                                                           \
  30        retl;                                                           \
  31         nop;                                                           \
  322:      BACKOFF_SPIN(%o2, %o3, 1b);                                     \
  33ENDPROC(atomic_##op);                                                   \
  34EXPORT_SYMBOL(atomic_##op);
  35
  36#define ATOMIC_OP_RETURN(op)                                            \
  37ENTRY(atomic_##op##_return) /* %o0 = increment, %o1 = atomic_ptr */     \
  38        BACKOFF_SETUP(%o2);                                             \
  391:      lduw    [%o1], %g1;                                             \
  40        op      %g1, %o0, %g7;                                          \
  41        cas     [%o1], %g1, %g7;                                        \
  42        cmp     %g1, %g7;                                               \
  43        bne,pn  %icc, BACKOFF_LABEL(2f, 1b);                            \
  44         op     %g1, %o0, %g1;                                          \
  45        retl;                                                           \
  46         sra    %g1, 0, %o0;                                            \
  472:      BACKOFF_SPIN(%o2, %o3, 1b);                                     \
  48ENDPROC(atomic_##op##_return);                                          \
  49EXPORT_SYMBOL(atomic_##op##_return);
  50
  51#define ATOMIC_FETCH_OP(op)                                             \
  52ENTRY(atomic_fetch_##op) /* %o0 = increment, %o1 = atomic_ptr */        \
  53        BACKOFF_SETUP(%o2);                                             \
  541:      lduw    [%o1], %g1;                                             \
  55        op      %g1, %o0, %g7;                                          \
  56        cas     [%o1], %g1, %g7;                                        \
  57        cmp     %g1, %g7;                                               \
  58        bne,pn  %icc, BACKOFF_LABEL(2f, 1b);                            \
  59         nop;                                                           \
  60        retl;                                                           \
  61         sra    %g1, 0, %o0;                                            \
  622:      BACKOFF_SPIN(%o2, %o3, 1b);                                     \
  63ENDPROC(atomic_fetch_##op);                                             \
  64EXPORT_SYMBOL(atomic_fetch_##op);
  65
  66ATOMIC_OP(add)
  67ATOMIC_OP_RETURN(add)
  68ATOMIC_FETCH_OP(add)
  69
  70ATOMIC_OP(sub)
  71ATOMIC_OP_RETURN(sub)
  72ATOMIC_FETCH_OP(sub)
  73
  74ATOMIC_OP(and)
  75ATOMIC_FETCH_OP(and)
  76
  77ATOMIC_OP(or)
  78ATOMIC_FETCH_OP(or)
  79
  80ATOMIC_OP(xor)
  81ATOMIC_FETCH_OP(xor)
  82
  83#undef ATOMIC_FETCH_OP
  84#undef ATOMIC_OP_RETURN
  85#undef ATOMIC_OP
  86
  87#define ATOMIC64_OP(op)                                                 \
  88ENTRY(atomic64_##op) /* %o0 = increment, %o1 = atomic_ptr */            \
  89        BACKOFF_SETUP(%o2);                                             \
  901:      ldx     [%o1], %g1;                                             \
  91        op      %g1, %o0, %g7;                                          \
  92        casx    [%o1], %g1, %g7;                                        \
  93        cmp     %g1, %g7;                                               \
  94        bne,pn  %xcc, BACKOFF_LABEL(2f, 1b);                            \
  95         nop;                                                           \
  96        retl;                                                           \
  97         nop;                                                           \
  982:      BACKOFF_SPIN(%o2, %o3, 1b);                                     \
  99ENDPROC(atomic64_##op);                                                 \
 100EXPORT_SYMBOL(atomic64_##op);
 101
 102#define ATOMIC64_OP_RETURN(op)                                          \
 103ENTRY(atomic64_##op##_return) /* %o0 = increment, %o1 = atomic_ptr */   \
 104        BACKOFF_SETUP(%o2);                                             \
 1051:      ldx     [%o1], %g1;                                             \
 106        op      %g1, %o0, %g7;                                          \
 107        casx    [%o1], %g1, %g7;                                        \
 108        cmp     %g1, %g7;                                               \
 109        bne,pn  %xcc, BACKOFF_LABEL(2f, 1b);                            \
 110         nop;                                                           \
 111        retl;                                                           \
 112         op     %g1, %o0, %o0;                                          \
 1132:      BACKOFF_SPIN(%o2, %o3, 1b);                                     \
 114ENDPROC(atomic64_##op##_return);                                        \
 115EXPORT_SYMBOL(atomic64_##op##_return);
 116
 117#define ATOMIC64_FETCH_OP(op)                                           \
 118ENTRY(atomic64_fetch_##op) /* %o0 = increment, %o1 = atomic_ptr */      \
 119        BACKOFF_SETUP(%o2);                                             \
 1201:      ldx     [%o1], %g1;                                             \
 121        op      %g1, %o0, %g7;                                          \
 122        casx    [%o1], %g1, %g7;                                        \
 123        cmp     %g1, %g7;                                               \
 124        bne,pn  %xcc, BACKOFF_LABEL(2f, 1b);                            \
 125         nop;                                                           \
 126        retl;                                                           \
 127         mov    %g1, %o0;                                               \
 1282:      BACKOFF_SPIN(%o2, %o3, 1b);                                     \
 129ENDPROC(atomic64_fetch_##op);                                           \
 130EXPORT_SYMBOL(atomic64_fetch_##op);
 131
 132ATOMIC64_OP(add)
 133ATOMIC64_OP_RETURN(add)
 134ATOMIC64_FETCH_OP(add)
 135
 136ATOMIC64_OP(sub)
 137ATOMIC64_OP_RETURN(sub)
 138ATOMIC64_FETCH_OP(sub)
 139
 140ATOMIC64_OP(and)
 141ATOMIC64_FETCH_OP(and)
 142
 143ATOMIC64_OP(or)
 144ATOMIC64_FETCH_OP(or)
 145
 146ATOMIC64_OP(xor)
 147ATOMIC64_FETCH_OP(xor)
 148
 149#undef ATOMIC64_FETCH_OP
 150#undef ATOMIC64_OP_RETURN
 151#undef ATOMIC64_OP
 152
 153ENTRY(atomic64_dec_if_positive) /* %o0 = atomic_ptr */
 154        BACKOFF_SETUP(%o2)
 1551:      ldx     [%o0], %g1
 156        brlez,pn %g1, 3f
 157         sub    %g1, 1, %g7
 158        casx    [%o0], %g1, %g7
 159        cmp     %g1, %g7
 160        bne,pn  %xcc, BACKOFF_LABEL(2f, 1b)
 161         nop
 1623:      retl
 163         sub    %g1, 1, %o0
 1642:      BACKOFF_SPIN(%o2, %o3, 1b)
 165ENDPROC(atomic64_dec_if_positive)
 166EXPORT_SYMBOL(atomic64_dec_if_positive)
 167