LXR qemu/accel/tcg/atomic

   1/*
   2 * Atomic helper templates
   3 * Included from tcg-runtime.c and cputlb.c.
   4 *
   5 * Copyright (c) 2016 Red Hat, Inc
   6 *
   7 * This library is free software; you can redistribute it and/or
   8 * modify it under the terms of the GNU Lesser General Public
   9 * License as published by the Free Software Foundation; either
  10 * version 2.1 of the License, or (at your option) any later version.
  11 *
  12 * This library is distributed in the hope that it will be useful,
  13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15 * Lesser General Public License for more details.
  16 *
  17 * You should have received a copy of the GNU Lesser General Public
  18 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  19 */
  20
  21#include "qemu/plugin.h"
  22
  23#if DATA_SIZE == 16
  24# define SUFFIX     o
  25# define DATA_TYPE  Int128
  26# define BSWAP      bswap128
  27# define SHIFT      4
  28#elif DATA_SIZE == 8
  29# define SUFFIX     q
  30# define DATA_TYPE  aligned_uint64_t
  31# define SDATA_TYPE aligned_int64_t
  32# define BSWAP      bswap64
  33# define SHIFT      3
  34#elif DATA_SIZE == 4
  35# define SUFFIX     l
  36# define DATA_TYPE  uint32_t
  37# define SDATA_TYPE int32_t
  38# define BSWAP      bswap32
  39# define SHIFT      2
  40#elif DATA_SIZE == 2
  41# define SUFFIX     w
  42# define DATA_TYPE  uint16_t
  43# define SDATA_TYPE int16_t
  44# define BSWAP      bswap16
  45# define SHIFT      1
  46#elif DATA_SIZE == 1
  47# define SUFFIX     b
  48# define DATA_TYPE  uint8_t
  49# define SDATA_TYPE int8_t
  50# define BSWAP
  51# define SHIFT      0
  52#else
  53# error unsupported data size
  54#endif
  55
  56#if DATA_SIZE >= 4
  57# define ABI_TYPE  DATA_TYPE
  58#else
  59# define ABI_TYPE  uint32_t
  60#endif
  61
  62/* Define host-endian atomic operations.  Note that END is used within
  63   the ATOMIC_NAME macro, and redefined below.  */
  64#if DATA_SIZE == 1
  65# define END
  66#elif HOST_BIG_ENDIAN
  67# define END  _be
  68#else
  69# define END  _le
  70#endif
  71
  72ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr,
  73                              ABI_TYPE cmpv, ABI_TYPE newv,
  74                              MemOpIdx oi, uintptr_t retaddr)
  75{
  76    DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE,
  77                                         PAGE_READ | PAGE_WRITE, retaddr);
  78    DATA_TYPE ret;
  79
  80#if DATA_SIZE == 16
  81    ret = atomic16_cmpxchg(haddr, cmpv, newv);
  82#else
  83    ret = qatomic_cmpxchg__nocheck(haddr, cmpv, newv);
  84#endif
  85    ATOMIC_MMU_CLEANUP;
  86    atomic_trace_rmw_post(env, addr, oi);
  87    return ret;
  88}
  89
  90#if DATA_SIZE >= 16
  91#if HAVE_ATOMIC128
  92ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr,
  93                         MemOpIdx oi, uintptr_t retaddr)
  94{
  95    DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE,
  96                                         PAGE_READ, retaddr);
  97    DATA_TYPE val;
  98
  99    val = atomic16_read(haddr);
 100    ATOMIC_MMU_CLEANUP;
 101    atomic_trace_ld_post(env, addr, oi);
 102    return val;
 103}
 104
 105void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr, ABI_TYPE val,
 106                     MemOpIdx oi, uintptr_t retaddr)
 107{
 108    DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE,
 109                                         PAGE_WRITE, retaddr);
 110
 111    atomic16_set(haddr, val);
 112    ATOMIC_MMU_CLEANUP;
 113    atomic_trace_st_post(env, addr, oi);
 114}
 115#endif
 116#else
 117ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr, ABI_TYPE val,
 118                           MemOpIdx oi, uintptr_t retaddr)
 119{
 120    DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE,
 121                                         PAGE_READ | PAGE_WRITE, retaddr);
 122    DATA_TYPE ret;
 123
 124    ret = qatomic_xchg__nocheck(haddr, val);
 125    ATOMIC_MMU_CLEANUP;
 126    atomic_trace_rmw_post(env, addr, oi);
 127    return ret;
 128}
 129
 130#define GEN_ATOMIC_HELPER(X)                                        \
 131ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr,       \
 132                        ABI_TYPE val, MemOpIdx oi, uintptr_t retaddr) \
 133{                                                                   \
 134    DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE,  \
 135                                         PAGE_READ | PAGE_WRITE, retaddr); \
 136    DATA_TYPE ret;                                                  \
 137    ret = qatomic_##X(haddr, val);                                  \
 138    ATOMIC_MMU_CLEANUP;                                             \
 139    atomic_trace_rmw_post(env, addr, oi);                           \
 140    return ret;                                                     \
 141}
 142
 143GEN_ATOMIC_HELPER(fetch_add)
 144GEN_ATOMIC_HELPER(fetch_and)
 145GEN_ATOMIC_HELPER(fetch_or)
 146GEN_ATOMIC_HELPER(fetch_xor)
 147GEN_ATOMIC_HELPER(add_fetch)
 148GEN_ATOMIC_HELPER(and_fetch)
 149GEN_ATOMIC_HELPER(or_fetch)
 150GEN_ATOMIC_HELPER(xor_fetch)
 151
 152#undef GEN_ATOMIC_HELPER
 153
 154/*
 155 * These helpers are, as a whole, full barriers.  Within the helper,
 156 * the leading barrier is explicit and the trailing barrier is within
 157 * cmpxchg primitive.
 158 *
 159 * Trace this load + RMW loop as a single RMW op. This way, regardless
 160 * of CF_PARALLEL's value, we'll trace just a read and a write.
 161 */
 162#define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET)                \
 163ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr,       \
 164                        ABI_TYPE xval, MemOpIdx oi, uintptr_t retaddr) \
 165{                                                                   \
 166    XDATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, \
 167                                          PAGE_READ | PAGE_WRITE, retaddr); \
 168    XDATA_TYPE cmp, old, new, val = xval;                           \
 169    smp_mb();                                                       \
 170    cmp = qatomic_read__nocheck(haddr);                             \
 171    do {                                                            \
 172        old = cmp; new = FN(old, val);                              \
 173        cmp = qatomic_cmpxchg__nocheck(haddr, old, new);            \
 174    } while (cmp != old);                                           \
 175    ATOMIC_MMU_CLEANUP;                                             \
 176    atomic_trace_rmw_post(env, addr, oi);                           \
 177    return RET;                                                     \
 178}
 179
 180GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old)
 181GEN_ATOMIC_HELPER_FN(fetch_umin, MIN,  DATA_TYPE, old)
 182GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old)
 183GEN_ATOMIC_HELPER_FN(fetch_umax, MAX,  DATA_TYPE, old)
 184
 185GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new)
 186GEN_ATOMIC_HELPER_FN(umin_fetch, MIN,  DATA_TYPE, new)
 187GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new)
 188GEN_ATOMIC_HELPER_FN(umax_fetch, MAX,  DATA_TYPE, new)
 189
 190#undef GEN_ATOMIC_HELPER_FN
 191#endif /* DATA SIZE >= 16 */
 192
 193#undef END
 194
 195#if DATA_SIZE > 1
 196
 197/* Define reverse-host-endian atomic operations.  Note that END is used
 198   within the ATOMIC_NAME macro.  */
 199#if HOST_BIG_ENDIAN
 200# define END  _le
 201#else
 202# define END  _be
 203#endif
 204
 205ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr,
 206                              ABI_TYPE cmpv, ABI_TYPE newv,
 207                              MemOpIdx oi, uintptr_t retaddr)
 208{
 209    DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE,
 210                                         PAGE_READ | PAGE_WRITE, retaddr);
 211    DATA_TYPE ret;
 212
 213#if DATA_SIZE == 16
 214    ret = atomic16_cmpxchg(haddr, BSWAP(cmpv), BSWAP(newv));
 215#else
 216    ret = qatomic_cmpxchg__nocheck(haddr, BSWAP(cmpv), BSWAP(newv));
 217#endif
 218    ATOMIC_MMU_CLEANUP;
 219    atomic_trace_rmw_post(env, addr, oi);
 220    return BSWAP(ret);
 221}
 222
 223#if DATA_SIZE >= 16
 224#if HAVE_ATOMIC128
 225ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr,
 226                         MemOpIdx oi, uintptr_t retaddr)
 227{
 228    DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE,
 229                                         PAGE_READ, retaddr);
 230    DATA_TYPE val;
 231
 232    val = atomic16_read(haddr);
 233    ATOMIC_MMU_CLEANUP;
 234    atomic_trace_ld_post(env, addr, oi);
 235    return BSWAP(val);
 236}
 237
 238void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr, ABI_TYPE val,
 239                     MemOpIdx oi, uintptr_t retaddr)
 240{
 241    DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE,
 242                                         PAGE_WRITE, retaddr);
 243
 244    val = BSWAP(val);
 245    atomic16_set(haddr, val);
 246    ATOMIC_MMU_CLEANUP;
 247    atomic_trace_st_post(env, addr, oi);
 248}
 249#endif
 250#else
 251ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr, ABI_TYPE val,
 252                           MemOpIdx oi, uintptr_t retaddr)
 253{
 254    DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE,
 255                                         PAGE_READ | PAGE_WRITE, retaddr);
 256    ABI_TYPE ret;
 257
 258    ret = qatomic_xchg__nocheck(haddr, BSWAP(val));
 259    ATOMIC_MMU_CLEANUP;
 260    atomic_trace_rmw_post(env, addr, oi);
 261    return BSWAP(ret);
 262}
 263
 264#define GEN_ATOMIC_HELPER(X)                                        \
 265ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr,       \
 266                        ABI_TYPE val, MemOpIdx oi, uintptr_t retaddr) \
 267{                                                                   \
 268    DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE,  \
 269                                         PAGE_READ | PAGE_WRITE, retaddr); \
 270    DATA_TYPE ret;                                                  \
 271    ret = qatomic_##X(haddr, BSWAP(val));                           \
 272    ATOMIC_MMU_CLEANUP;                                             \
 273    atomic_trace_rmw_post(env, addr, oi);                           \
 274    return BSWAP(ret);                                              \
 275}
 276
 277GEN_ATOMIC_HELPER(fetch_and)
 278GEN_ATOMIC_HELPER(fetch_or)
 279GEN_ATOMIC_HELPER(fetch_xor)
 280GEN_ATOMIC_HELPER(and_fetch)
 281GEN_ATOMIC_HELPER(or_fetch)
 282GEN_ATOMIC_HELPER(xor_fetch)
 283
 284#undef GEN_ATOMIC_HELPER
 285
 286/* These helpers are, as a whole, full barriers.  Within the helper,
 287 * the leading barrier is explicit and the trailing barrier is within
 288 * cmpxchg primitive.
 289 *
 290 * Trace this load + RMW loop as a single RMW op. This way, regardless
 291 * of CF_PARALLEL's value, we'll trace just a read and a write.
 292 */
 293#define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET)                \
 294ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr,       \
 295                        ABI_TYPE xval, MemOpIdx oi, uintptr_t retaddr) \
 296{                                                                   \
 297    XDATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, \
 298                                          PAGE_READ | PAGE_WRITE, retaddr); \
 299    XDATA_TYPE ldo, ldn, old, new, val = xval;                      \
 300    smp_mb();                                                       \
 301    ldn = qatomic_read__nocheck(haddr);                             \
 302    do {                                                            \
 303        ldo = ldn; old = BSWAP(ldo); new = FN(old, val);            \
 304        ldn = qatomic_cmpxchg__nocheck(haddr, ldo, BSWAP(new));     \
 305    } while (ldo != ldn);                                           \
 306    ATOMIC_MMU_CLEANUP;                                             \
 307    atomic_trace_rmw_post(env, addr, oi);                           \
 308    return RET;                                                     \
 309}
 310
 311GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old)
 312GEN_ATOMIC_HELPER_FN(fetch_umin, MIN,  DATA_TYPE, old)
 313GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old)
 314GEN_ATOMIC_HELPER_FN(fetch_umax, MAX,  DATA_TYPE, old)
 315
 316GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new)
 317GEN_ATOMIC_HELPER_FN(umin_fetch, MIN,  DATA_TYPE, new)
 318GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new)
 319GEN_ATOMIC_HELPER_FN(umax_fetch, MAX,  DATA_TYPE, new)
 320
 321/* Note that for addition, we need to use a separate cmpxchg loop instead
 322   of bswaps for the reverse-host-endian helpers.  */
 323#define ADD(X, Y)   (X + Y)
 324GEN_ATOMIC_HELPER_FN(fetch_add, ADD, DATA_TYPE, old)
 325GEN_ATOMIC_HELPER_FN(add_fetch, ADD, DATA_TYPE, new)
 326#undef ADD
 327
 328#undef GEN_ATOMIC_HELPER_FN
 329#endif /* DATA_SIZE >= 16 */
 330
 331#undef END
 332#endif /* DATA_SIZE > 1 */
 333
 334#undef BSWAP
 335#undef ABI_TYPE
 336#undef DATA_TYPE
 337#undef SDATA_TYPE
 338#undef SUFFIX
 339#undef DATA_SIZE
 340#undef SHIFT
 341