LXR linux/arch/riscv/include/asm/atomic.h

   1/*
   2 * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved.
   3 * Copyright (C) 2012 Regents of the University of California
   4 * Copyright (C) 2017 SiFive
   5 *
   6 * This program is free software; you can redistribute it and/or
   7 * modify it under the terms of the GNU General Public Licence
   8 * as published by the Free Software Foundation; either version
   9 * 2 of the Licence, or (at your option) any later version.
  10 */
  11
  12#ifndef _ASM_RISCV_ATOMIC_H
  13#define _ASM_RISCV_ATOMIC_H
  14
  15#ifdef CONFIG_GENERIC_ATOMIC64
  16# include <asm-generic/atomic64.h>
  17#else
  18# if (__riscv_xlen < 64)
  19#  error "64-bit atomics require XLEN to be at least 64"
  20# endif
  21#endif
  22
  23#include <asm/cmpxchg.h>
  24#include <asm/barrier.h>
  25
  26#define ATOMIC_INIT(i)  { (i) }
  27static __always_inline int atomic_read(const atomic_t *v)
  28{
  29        return READ_ONCE(v->counter);
  30}
  31static __always_inline void atomic_set(atomic_t *v, int i)
  32{
  33        WRITE_ONCE(v->counter, i);
  34}
  35
  36#ifndef CONFIG_GENERIC_ATOMIC64
  37#define ATOMIC64_INIT(i) { (i) }
  38static __always_inline long atomic64_read(const atomic64_t *v)
  39{
  40        return READ_ONCE(v->counter);
  41}
  42static __always_inline void atomic64_set(atomic64_t *v, long i)
  43{
  44        WRITE_ONCE(v->counter, i);
  45}
  46#endif
  47
  48/*
  49 * First, the atomic ops that have no ordering constraints and therefor don't
  50 * have the AQ or RL bits set.  These don't return anything, so there's only
  51 * one version to worry about.
  52 */
  53#define ATOMIC_OP(op, asm_op, I, asm_type, c_type, prefix)                              \
  54static __always_inline void atomic##prefix##_##op(c_type i, atomic##prefix##_t *v)      \
  55{                                                                                       \
  56        __asm__ __volatile__ (                                                          \
  57                "amo" #asm_op "." #asm_type " zero, %1, %0"                             \
  58                : "+A" (v->counter)                                                     \
  59                : "r" (I)                                                               \
  60                : "memory");                                                            \
  61}
  62
  63#ifdef CONFIG_GENERIC_ATOMIC64
  64#define ATOMIC_OPS(op, asm_op, I)                       \
  65        ATOMIC_OP (op, asm_op, I, w,  int,   )
  66#else
  67#define ATOMIC_OPS(op, asm_op, I)                       \
  68        ATOMIC_OP (op, asm_op, I, w,  int,   )  \
  69        ATOMIC_OP (op, asm_op, I, d, long, 64)
  70#endif
  71
  72ATOMIC_OPS(add, add,  i)
  73ATOMIC_OPS(sub, add, -i)
  74ATOMIC_OPS(and, and,  i)
  75ATOMIC_OPS( or,  or,  i)
  76ATOMIC_OPS(xor, xor,  i)
  77
  78#undef ATOMIC_OP
  79#undef ATOMIC_OPS
  80
  81/*
  82 * Atomic ops that have ordered, relaxed, acquire, and relese variants.
  83 * There's two flavors of these: the arithmatic ops have both fetch and return
  84 * versions, while the logical ops only have fetch versions.
  85 */
  86#define ATOMIC_FETCH_OP(op, asm_op, I, asm_or, c_or, asm_type, c_type, prefix)                          \
  87static __always_inline c_type atomic##prefix##_fetch_##op##c_or(c_type i, atomic##prefix##_t *v)        \
  88{                                                                                                       \
  89        register c_type ret;                                                                            \
  90        __asm__ __volatile__ (                                                                          \
  91                "amo" #asm_op "." #asm_type #asm_or " %1, %2, %0"                                       \
  92                : "+A" (v->counter), "=r" (ret)                                                         \
  93                : "r" (I)                                                                               \
  94                : "memory");                                                                            \
  95        return ret;                                                                                     \
  96}
  97
  98#define ATOMIC_OP_RETURN(op, asm_op, c_op, I, asm_or, c_or, asm_type, c_type, prefix)                   \
  99static __always_inline c_type atomic##prefix##_##op##_return##c_or(c_type i, atomic##prefix##_t *v)     \
 100{                                                                                                       \
 101        return atomic##prefix##_fetch_##op##c_or(i, v) c_op I;                                          \
 102}
 103
 104#ifdef CONFIG_GENERIC_ATOMIC64
 105#define ATOMIC_OPS(op, asm_op, c_op, I, asm_or, c_or)                           \
 106        ATOMIC_FETCH_OP (op, asm_op,       I, asm_or, c_or, w,  int,   )        \
 107        ATOMIC_OP_RETURN(op, asm_op, c_op, I, asm_or, c_or, w,  int,   )
 108#else
 109#define ATOMIC_OPS(op, asm_op, c_op, I, asm_or, c_or)                           \
 110        ATOMIC_FETCH_OP (op, asm_op,       I, asm_or, c_or, w,  int,   )        \
 111        ATOMIC_OP_RETURN(op, asm_op, c_op, I, asm_or, c_or, w,  int,   )        \
 112        ATOMIC_FETCH_OP (op, asm_op,       I, asm_or, c_or, d, long, 64)        \
 113        ATOMIC_OP_RETURN(op, asm_op, c_op, I, asm_or, c_or, d, long, 64)
 114#endif
 115
 116ATOMIC_OPS(add, add, +,  i,      , _relaxed)
 117ATOMIC_OPS(add, add, +,  i, .aq  , _acquire)
 118ATOMIC_OPS(add, add, +,  i, .rl  , _release)
 119ATOMIC_OPS(add, add, +,  i, .aqrl,         )
 120
 121ATOMIC_OPS(sub, add, +, -i,      , _relaxed)
 122ATOMIC_OPS(sub, add, +, -i, .aq  , _acquire)
 123ATOMIC_OPS(sub, add, +, -i, .rl  , _release)
 124ATOMIC_OPS(sub, add, +, -i, .aqrl,         )
 125
 126#undef ATOMIC_OPS
 127
 128#ifdef CONFIG_GENERIC_ATOMIC64
 129#define ATOMIC_OPS(op, asm_op, I, asm_or, c_or)                         \
 130        ATOMIC_FETCH_OP(op, asm_op, I, asm_or, c_or, w,  int,   )
 131#else
 132#define ATOMIC_OPS(op, asm_op, I, asm_or, c_or)                         \
 133        ATOMIC_FETCH_OP(op, asm_op, I, asm_or, c_or, w,  int,   )       \
 134        ATOMIC_FETCH_OP(op, asm_op, I, asm_or, c_or, d, long, 64)
 135#endif
 136
 137ATOMIC_OPS(and, and, i,      , _relaxed)
 138ATOMIC_OPS(and, and, i, .aq  , _acquire)
 139ATOMIC_OPS(and, and, i, .rl  , _release)
 140ATOMIC_OPS(and, and, i, .aqrl,         )
 141
 142ATOMIC_OPS( or,  or, i,      , _relaxed)
 143ATOMIC_OPS( or,  or, i, .aq  , _acquire)
 144ATOMIC_OPS( or,  or, i, .rl  , _release)
 145ATOMIC_OPS( or,  or, i, .aqrl,         )
 146
 147ATOMIC_OPS(xor, xor, i,      , _relaxed)
 148ATOMIC_OPS(xor, xor, i, .aq  , _acquire)
 149ATOMIC_OPS(xor, xor, i, .rl  , _release)
 150ATOMIC_OPS(xor, xor, i, .aqrl,         )
 151
 152#undef ATOMIC_OPS
 153
 154#undef ATOMIC_FETCH_OP
 155#undef ATOMIC_OP_RETURN
 156
 157/*
 158 * The extra atomic operations that are constructed from one of the core
 159 * AMO-based operations above (aside from sub, which is easier to fit above).
 160 * These are required to perform a barrier, but they're OK this way because
 161 * atomic_*_return is also required to perform a barrier.
 162 */
 163#define ATOMIC_OP(op, func_op, comp_op, I, c_type, prefix)                      \
 164static __always_inline bool atomic##prefix##_##op(c_type i, atomic##prefix##_t *v) \
 165{                                                                               \
 166        return atomic##prefix##_##func_op##_return(i, v) comp_op I;             \
 167}
 168
 169#ifdef CONFIG_GENERIC_ATOMIC64
 170#define ATOMIC_OPS(op, func_op, comp_op, I)                     \
 171        ATOMIC_OP (op, func_op, comp_op, I,  int,   )
 172#else
 173#define ATOMIC_OPS(op, func_op, comp_op, I)                     \
 174        ATOMIC_OP (op, func_op, comp_op, I,  int,   )           \
 175        ATOMIC_OP (op, func_op, comp_op, I, long, 64)
 176#endif
 177
 178ATOMIC_OPS(add_and_test, add, ==, 0)
 179ATOMIC_OPS(sub_and_test, sub, ==, 0)
 180ATOMIC_OPS(add_negative, add,  <, 0)
 181
 182#undef ATOMIC_OP
 183#undef ATOMIC_OPS
 184
 185#define ATOMIC_OP(op, func_op, I, c_type, prefix)                               \
 186static __always_inline void atomic##prefix##_##op(atomic##prefix##_t *v)        \
 187{                                                                               \
 188        atomic##prefix##_##func_op(I, v);                                       \
 189}
 190
 191#define ATOMIC_FETCH_OP(op, func_op, I, c_type, prefix)                                 \
 192static __always_inline c_type atomic##prefix##_fetch_##op(atomic##prefix##_t *v)        \
 193{                                                                                       \
 194        return atomic##prefix##_fetch_##func_op(I, v);                                  \
 195}
 196
 197#define ATOMIC_OP_RETURN(op, asm_op, c_op, I, c_type, prefix)                           \
 198static __always_inline c_type atomic##prefix##_##op##_return(atomic##prefix##_t *v)     \
 199{                                                                                       \
 200        return atomic##prefix##_fetch_##op(v) c_op I;                                   \
 201}
 202
 203#ifdef CONFIG_GENERIC_ATOMIC64
 204#define ATOMIC_OPS(op, asm_op, c_op, I)                                         \
 205        ATOMIC_OP       (op, asm_op,       I,  int,   )                         \
 206        ATOMIC_FETCH_OP (op, asm_op,       I,  int,   )                         \
 207        ATOMIC_OP_RETURN(op, asm_op, c_op, I,  int,   )
 208#else
 209#define ATOMIC_OPS(op, asm_op, c_op, I)                                         \
 210        ATOMIC_OP       (op, asm_op,       I,  int,   )                         \
 211        ATOMIC_FETCH_OP (op, asm_op,       I,  int,   )                         \
 212        ATOMIC_OP_RETURN(op, asm_op, c_op, I,  int,   )                         \
 213        ATOMIC_OP       (op, asm_op,       I, long, 64)                         \
 214        ATOMIC_FETCH_OP (op, asm_op,       I, long, 64)                         \
 215        ATOMIC_OP_RETURN(op, asm_op, c_op, I, long, 64)
 216#endif
 217
 218ATOMIC_OPS(inc, add, +,  1)
 219ATOMIC_OPS(dec, add, +, -1)
 220
 221#undef ATOMIC_OPS
 222#undef ATOMIC_OP
 223#undef ATOMIC_FETCH_OP
 224#undef ATOMIC_OP_RETURN
 225
 226#define ATOMIC_OP(op, func_op, comp_op, I, prefix)                              \
 227static __always_inline bool atomic##prefix##_##op(atomic##prefix##_t *v)        \
 228{                                                                               \
 229        return atomic##prefix##_##func_op##_return(v) comp_op I;                \
 230}
 231
 232ATOMIC_OP(inc_and_test, inc, ==, 0,   )
 233ATOMIC_OP(dec_and_test, dec, ==, 0,   )
 234#ifndef CONFIG_GENERIC_ATOMIC64
 235ATOMIC_OP(inc_and_test, inc, ==, 0, 64)
 236ATOMIC_OP(dec_and_test, dec, ==, 0, 64)
 237#endif
 238
 239#undef ATOMIC_OP
 240
 241/* This is required to provide a barrier on success. */
 242static __always_inline int __atomic_add_unless(atomic_t *v, int a, int u)
 243{
 244       int prev, rc;
 245
 246        __asm__ __volatile__ (
 247                "0:\n\t"
 248                "lr.w.aqrl  %[p],  %[c]\n\t"
 249                "beq        %[p],  %[u], 1f\n\t"
 250                "add       %[rc],  %[p], %[a]\n\t"
 251                "sc.w.aqrl %[rc], %[rc], %[c]\n\t"
 252                "bnez      %[rc], 0b\n\t"
 253                "1:"
 254                : [p]"=&r" (prev), [rc]"=&r" (rc), [c]"+A" (v->counter)
 255                : [a]"r" (a), [u]"r" (u)
 256                : "memory");
 257        return prev;
 258}
 259
 260#ifndef CONFIG_GENERIC_ATOMIC64
 261static __always_inline long __atomic64_add_unless(atomic64_t *v, long a, long u)
 262{
 263       long prev, rc;
 264
 265        __asm__ __volatile__ (
 266                "0:\n\t"
 267                "lr.d.aqrl  %[p],  %[c]\n\t"
 268                "beq        %[p],  %[u], 1f\n\t"
 269                "add       %[rc],  %[p], %[a]\n\t"
 270                "sc.d.aqrl %[rc], %[rc], %[c]\n\t"
 271                "bnez      %[rc], 0b\n\t"
 272                "1:"
 273                : [p]"=&r" (prev), [rc]"=&r" (rc), [c]"+A" (v->counter)
 274                : [a]"r" (a), [u]"r" (u)
 275                : "memory");
 276        return prev;
 277}
 278
 279static __always_inline int atomic64_add_unless(atomic64_t *v, long a, long u)
 280{
 281        return __atomic64_add_unless(v, a, u) != u;
 282}
 283#endif
 284
 285/*
 286 * The extra atomic operations that are constructed from one of the core
 287 * LR/SC-based operations above.
 288 */
 289static __always_inline int atomic_inc_not_zero(atomic_t *v)
 290{
 291        return __atomic_add_unless(v, 1, 0);
 292}
 293
 294#ifndef CONFIG_GENERIC_ATOMIC64
 295static __always_inline long atomic64_inc_not_zero(atomic64_t *v)
 296{
 297        return atomic64_add_unless(v, 1, 0);
 298}
 299#endif
 300
 301/*
 302 * atomic_{cmp,}xchg is required to have exactly the same ordering semantics as
 303 * {cmp,}xchg and the operations that return, so they need a barrier.
 304 */
 305/*
 306 * FIXME: atomic_cmpxchg_{acquire,release,relaxed} are all implemented by
 307 * assigning the same barrier to both the LR and SC operations, but that might
 308 * not make any sense.  We're waiting on a memory model specification to
 309 * determine exactly what the right thing to do is here.
 310 */
 311#define ATOMIC_OP(c_t, prefix, c_or, size, asm_or)                                              \
 312static __always_inline c_t atomic##prefix##_cmpxchg##c_or(atomic##prefix##_t *v, c_t o, c_t n)  \
 313{                                                                                               \
 314        return __cmpxchg(&(v->counter), o, n, size, asm_or, asm_or);                            \
 315}                                                                                               \
 316static __always_inline c_t atomic##prefix##_xchg##c_or(atomic##prefix##_t *v, c_t n)            \
 317{                                                                                               \
 318        return __xchg(n, &(v->counter), size, asm_or);                                          \
 319}
 320
 321#ifdef CONFIG_GENERIC_ATOMIC64
 322#define ATOMIC_OPS(c_or, asm_or)                        \
 323        ATOMIC_OP( int,   , c_or, 4, asm_or)
 324#else
 325#define ATOMIC_OPS(c_or, asm_or)                        \
 326        ATOMIC_OP( int,   , c_or, 4, asm_or)            \
 327        ATOMIC_OP(long, 64, c_or, 8, asm_or)
 328#endif
 329
 330ATOMIC_OPS(        , .aqrl)
 331ATOMIC_OPS(_acquire,   .aq)
 332ATOMIC_OPS(_release,   .rl)
 333ATOMIC_OPS(_relaxed,      )
 334
 335#undef ATOMIC_OPS
 336#undef ATOMIC_OP
 337
 338static __always_inline int atomic_sub_if_positive(atomic_t *v, int offset)
 339{
 340       int prev, rc;
 341
 342        __asm__ __volatile__ (
 343                "0:\n\t"
 344                "lr.w.aqrl  %[p],  %[c]\n\t"
 345                "sub       %[rc],  %[p], %[o]\n\t"
 346                "bltz      %[rc],    1f\n\t"
 347                "sc.w.aqrl %[rc], %[rc], %[c]\n\t"
 348                "bnez      %[rc],    0b\n\t"
 349                "1:"
 350                : [p]"=&r" (prev), [rc]"=&r" (rc), [c]"+A" (v->counter)
 351                : [o]"r" (offset)
 352                : "memory");
 353        return prev - offset;
 354}
 355
 356#define atomic_dec_if_positive(v)       atomic_sub_if_positive(v, 1)
 357
 358#ifndef CONFIG_GENERIC_ATOMIC64
 359static __always_inline long atomic64_sub_if_positive(atomic64_t *v, int offset)
 360{
 361       long prev, rc;
 362
 363        __asm__ __volatile__ (
 364                "0:\n\t"
 365                "lr.d.aqrl  %[p],  %[c]\n\t"
 366                "sub       %[rc],  %[p], %[o]\n\t"
 367                "bltz      %[rc],    1f\n\t"
 368                "sc.d.aqrl %[rc], %[rc], %[c]\n\t"
 369                "bnez      %[rc],    0b\n\t"
 370                "1:"
 371                : [p]"=&r" (prev), [rc]"=&r" (rc), [c]"+A" (v->counter)
 372                : [o]"r" (offset)
 373                : "memory");
 374        return prev - offset;
 375}
 376
 377#define atomic64_dec_if_positive(v)     atomic64_sub_if_positive(v, 1)
 378#endif
 379
 380#endif /* _ASM_RISCV_ATOMIC_H */
 381