LXR qemu/accel/tcg/atomic

   1/*
   2 * Atomic helper templates
   3 * Included from tcg-runtime.c and cputlb.c.
   4 *
   5 * Copyright (c) 2016 Red Hat, Inc
   6 *
   7 * This library is free software; you can redistribute it and/or
   8 * modify it under the terms of the GNU Lesser General Public
   9 * License as published by the Free Software Foundation; either
  10 * version 2 of the License, or (at your option) any later version.
  11 *
  12 * This library is distributed in the hope that it will be useful,
  13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15 * Lesser General Public License for more details.
  16 *
  17 * You should have received a copy of the GNU Lesser General Public
  18 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  19 */
  20
  21#include "trace/mem.h"
  22
  23#if DATA_SIZE == 16
  24# define SUFFIX     o
  25# define DATA_TYPE  Int128
  26# define BSWAP      bswap128
  27# define SHIFT      4
  28#elif DATA_SIZE == 8
  29# define SUFFIX     q
  30# define DATA_TYPE  uint64_t
  31# define SDATA_TYPE int64_t
  32# define BSWAP      bswap64
  33# define SHIFT      3
  34#elif DATA_SIZE == 4
  35# define SUFFIX     l
  36# define DATA_TYPE  uint32_t
  37# define SDATA_TYPE int32_t
  38# define BSWAP      bswap32
  39# define SHIFT      2
  40#elif DATA_SIZE == 2
  41# define SUFFIX     w
  42# define DATA_TYPE  uint16_t
  43# define SDATA_TYPE int16_t
  44# define BSWAP      bswap16
  45# define SHIFT      1
  46#elif DATA_SIZE == 1
  47# define SUFFIX     b
  48# define DATA_TYPE  uint8_t
  49# define SDATA_TYPE int8_t
  50# define BSWAP
  51# define SHIFT      0
  52#else
  53# error unsupported data size
  54#endif
  55
  56#if DATA_SIZE >= 4
  57# define ABI_TYPE  DATA_TYPE
  58#else
  59# define ABI_TYPE  uint32_t
  60#endif
  61
  62#define ATOMIC_TRACE_RMW do {                                           \
  63        uint8_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, false); \
  64                                                                        \
  65        trace_guest_mem_before_exec(ENV_GET_CPU(env), addr, info);      \
  66        trace_guest_mem_before_exec(ENV_GET_CPU(env), addr,             \
  67                                    info | TRACE_MEM_ST);               \
  68    } while (0)
  69
  70#define ATOMIC_TRACE_LD do {                                            \
  71        uint8_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, false); \
  72                                                                        \
  73        trace_guest_mem_before_exec(ENV_GET_CPU(env), addr, info);      \
  74    } while (0)
  75
  76# define ATOMIC_TRACE_ST do {                                           \
  77        uint8_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, true); \
  78                                                                        \
  79        trace_guest_mem_before_exec(ENV_GET_CPU(env), addr, info);      \
  80    } while (0)
  81
  82/* Define host-endian atomic operations.  Note that END is used within
  83   the ATOMIC_NAME macro, and redefined below.  */
  84#if DATA_SIZE == 1
  85# define END
  86# define MEND _be /* either le or be would be fine */
  87#elif defined(HOST_WORDS_BIGENDIAN)
  88# define END  _be
  89# define MEND _be
  90#else
  91# define END  _le
  92# define MEND _le
  93#endif
  94
  95ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr,
  96                              ABI_TYPE cmpv, ABI_TYPE newv EXTRA_ARGS)
  97{
  98    ATOMIC_MMU_DECLS;
  99    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
 100    DATA_TYPE ret;
 101
 102    ATOMIC_TRACE_RMW;
 103    ret = atomic_cmpxchg__nocheck(haddr, cmpv, newv);
 104    ATOMIC_MMU_CLEANUP;
 105    return ret;
 106}
 107
 108#if DATA_SIZE >= 16
 109ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS)
 110{
 111    ATOMIC_MMU_DECLS;
 112    DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP;
 113
 114    ATOMIC_TRACE_LD;
 115    __atomic_load(haddr, &val, __ATOMIC_RELAXED);
 116    ATOMIC_MMU_CLEANUP;
 117    return val;
 118}
 119
 120void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr,
 121                     ABI_TYPE val EXTRA_ARGS)
 122{
 123    ATOMIC_MMU_DECLS;
 124    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
 125
 126    ATOMIC_TRACE_ST;
 127    __atomic_store(haddr, &val, __ATOMIC_RELAXED);
 128    ATOMIC_MMU_CLEANUP;
 129}
 130#else
 131ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr,
 132                           ABI_TYPE val EXTRA_ARGS)
 133{
 134    ATOMIC_MMU_DECLS;
 135    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
 136    DATA_TYPE ret;
 137
 138    ATOMIC_TRACE_RMW;
 139    ret = atomic_xchg__nocheck(haddr, val);
 140    ATOMIC_MMU_CLEANUP;
 141    return ret;
 142}
 143
 144#define GEN_ATOMIC_HELPER(X)                                        \
 145ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr,       \
 146                 ABI_TYPE val EXTRA_ARGS)                           \
 147{                                                                   \
 148    ATOMIC_MMU_DECLS;                                               \
 149    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;                           \
 150    DATA_TYPE ret;                                                  \
 151                                                                    \
 152    ATOMIC_TRACE_RMW;                                               \
 153    ret = atomic_##X(haddr, val);                                   \
 154    ATOMIC_MMU_CLEANUP;                                             \
 155    return ret;                                                     \
 156}
 157
 158GEN_ATOMIC_HELPER(fetch_add)
 159GEN_ATOMIC_HELPER(fetch_and)
 160GEN_ATOMIC_HELPER(fetch_or)
 161GEN_ATOMIC_HELPER(fetch_xor)
 162GEN_ATOMIC_HELPER(add_fetch)
 163GEN_ATOMIC_HELPER(and_fetch)
 164GEN_ATOMIC_HELPER(or_fetch)
 165GEN_ATOMIC_HELPER(xor_fetch)
 166
 167#undef GEN_ATOMIC_HELPER
 168
 169/* These helpers are, as a whole, full barriers.  Within the helper,
 170 * the leading barrier is explicit and the trailing barrier is within
 171 * cmpxchg primitive.
 172 *
 173 * Trace this load + RMW loop as a single RMW op. This way, regardless
 174 * of CF_PARALLEL's value, we'll trace just a read and a write.
 175 */
 176#define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET)                \
 177ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr,       \
 178                        ABI_TYPE xval EXTRA_ARGS)                   \
 179{                                                                   \
 180    ATOMIC_MMU_DECLS;                                               \
 181    XDATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;                          \
 182    XDATA_TYPE cmp, old, new, val = xval;                           \
 183                                                                    \
 184    ATOMIC_TRACE_RMW;                                               \
 185    smp_mb();                                                       \
 186    cmp = atomic_read__nocheck(haddr);                              \
 187    do {                                                            \
 188        old = cmp; new = FN(old, val);                              \
 189        cmp = atomic_cmpxchg__nocheck(haddr, old, new);             \
 190    } while (cmp != old);                                           \
 191    ATOMIC_MMU_CLEANUP;                                             \
 192    return RET;                                                     \
 193}
 194
 195GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old)
 196GEN_ATOMIC_HELPER_FN(fetch_umin, MIN,  DATA_TYPE, old)
 197GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old)
 198GEN_ATOMIC_HELPER_FN(fetch_umax, MAX,  DATA_TYPE, old)
 199
 200GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new)
 201GEN_ATOMIC_HELPER_FN(umin_fetch, MIN,  DATA_TYPE, new)
 202GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new)
 203GEN_ATOMIC_HELPER_FN(umax_fetch, MAX,  DATA_TYPE, new)
 204
 205#undef GEN_ATOMIC_HELPER_FN
 206#endif /* DATA SIZE >= 16 */
 207
 208#undef END
 209#undef MEND
 210
 211#if DATA_SIZE > 1
 212
 213/* Define reverse-host-endian atomic operations.  Note that END is used
 214   within the ATOMIC_NAME macro.  */
 215#ifdef HOST_WORDS_BIGENDIAN
 216# define END  _le
 217# define MEND _le
 218#else
 219# define END  _be
 220# define MEND _be
 221#endif
 222
 223ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr,
 224                              ABI_TYPE cmpv, ABI_TYPE newv EXTRA_ARGS)
 225{
 226    ATOMIC_MMU_DECLS;
 227    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
 228    DATA_TYPE ret;
 229
 230    ATOMIC_TRACE_RMW;
 231    ret = atomic_cmpxchg__nocheck(haddr, BSWAP(cmpv), BSWAP(newv));
 232    ATOMIC_MMU_CLEANUP;
 233    return BSWAP(ret);
 234}
 235
 236#if DATA_SIZE >= 16
 237ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS)
 238{
 239    ATOMIC_MMU_DECLS;
 240    DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP;
 241
 242    ATOMIC_TRACE_LD;
 243    __atomic_load(haddr, &val, __ATOMIC_RELAXED);
 244    ATOMIC_MMU_CLEANUP;
 245    return BSWAP(val);
 246}
 247
 248void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr,
 249                     ABI_TYPE val EXTRA_ARGS)
 250{
 251    ATOMIC_MMU_DECLS;
 252    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
 253
 254    ATOMIC_TRACE_ST;
 255    val = BSWAP(val);
 256    __atomic_store(haddr, &val, __ATOMIC_RELAXED);
 257    ATOMIC_MMU_CLEANUP;
 258}
 259#else
 260ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr,
 261                           ABI_TYPE val EXTRA_ARGS)
 262{
 263    ATOMIC_MMU_DECLS;
 264    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
 265    ABI_TYPE ret;
 266
 267    ATOMIC_TRACE_RMW;
 268    ret = atomic_xchg__nocheck(haddr, BSWAP(val));
 269    ATOMIC_MMU_CLEANUP;
 270    return BSWAP(ret);
 271}
 272
 273#define GEN_ATOMIC_HELPER(X)                                        \
 274ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr,       \
 275                 ABI_TYPE val EXTRA_ARGS)                           \
 276{                                                                   \
 277    ATOMIC_MMU_DECLS;                                               \
 278    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;                           \
 279    DATA_TYPE ret;                                                  \
 280                                                                    \
 281    ATOMIC_TRACE_RMW;                                               \
 282    ret = atomic_##X(haddr, BSWAP(val));                            \
 283    ATOMIC_MMU_CLEANUP;                                             \
 284    return BSWAP(ret);                                              \
 285}
 286
 287GEN_ATOMIC_HELPER(fetch_and)
 288GEN_ATOMIC_HELPER(fetch_or)
 289GEN_ATOMIC_HELPER(fetch_xor)
 290GEN_ATOMIC_HELPER(and_fetch)
 291GEN_ATOMIC_HELPER(or_fetch)
 292GEN_ATOMIC_HELPER(xor_fetch)
 293
 294#undef GEN_ATOMIC_HELPER
 295
 296/* These helpers are, as a whole, full barriers.  Within the helper,
 297 * the leading barrier is explicit and the trailing barrier is within
 298 * cmpxchg primitive.
 299 *
 300 * Trace this load + RMW loop as a single RMW op. This way, regardless
 301 * of CF_PARALLEL's value, we'll trace just a read and a write.
 302 */
 303#define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET)                \
 304ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr,       \
 305                        ABI_TYPE xval EXTRA_ARGS)                   \
 306{                                                                   \
 307    ATOMIC_MMU_DECLS;                                               \
 308    XDATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;                          \
 309    XDATA_TYPE ldo, ldn, old, new, val = xval;                      \
 310                                                                    \
 311    ATOMIC_TRACE_RMW;                                               \
 312    smp_mb();                                                       \
 313    ldn = atomic_read__nocheck(haddr);                              \
 314    do {                                                            \
 315        ldo = ldn; old = BSWAP(ldo); new = FN(old, val);            \
 316        ldn = atomic_cmpxchg__nocheck(haddr, ldo, BSWAP(new));      \
 317    } while (ldo != ldn);                                           \
 318    ATOMIC_MMU_CLEANUP;                                             \
 319    return RET;                                                     \
 320}
 321
 322GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old)
 323GEN_ATOMIC_HELPER_FN(fetch_umin, MIN,  DATA_TYPE, old)
 324GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old)
 325GEN_ATOMIC_HELPER_FN(fetch_umax, MAX,  DATA_TYPE, old)
 326
 327GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new)
 328GEN_ATOMIC_HELPER_FN(umin_fetch, MIN,  DATA_TYPE, new)
 329GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new)
 330GEN_ATOMIC_HELPER_FN(umax_fetch, MAX,  DATA_TYPE, new)
 331
 332/* Note that for addition, we need to use a separate cmpxchg loop instead
 333   of bswaps for the reverse-host-endian helpers.  */
 334#define ADD(X, Y)   (X + Y)
 335GEN_ATOMIC_HELPER_FN(fetch_add, ADD, DATA_TYPE, old)
 336GEN_ATOMIC_HELPER_FN(add_fetch, ADD, DATA_TYPE, new)
 337#undef ADD
 338
 339#undef GEN_ATOMIC_HELPER_FN
 340#endif /* DATA_SIZE >= 16 */
 341
 342#undef END
 343#undef MEND
 344#endif /* DATA_SIZE > 1 */
 345
 346#undef ATOMIC_TRACE_ST
 347#undef ATOMIC_TRACE_LD
 348#undef ATOMIC_TRACE_RMW
 349
 350#undef BSWAP
 351#undef ABI_TYPE
 352#undef DATA_TYPE
 353#undef SDATA_TYPE
 354#undef SUFFIX
 355#undef DATA_SIZE
 356#undef SHIFT
 357