LXR qemu/accel/tcg/atomic

   1/*
   2 * Atomic helper templates
   3 * Included from tcg-runtime.c and cputlb.c.
   4 *
   5 * Copyright (c) 2016 Red Hat, Inc
   6 *
   7 * This library is free software; you can redistribute it and/or
   8 * modify it under the terms of the GNU Lesser General Public
   9 * License as published by the Free Software Foundation; either
  10 * version 2 of the License, or (at your option) any later version.
  11 *
  12 * This library is distributed in the hope that it will be useful,
  13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15 * Lesser General Public License for more details.
  16 *
  17 * You should have received a copy of the GNU Lesser General Public
  18 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  19 */
  20
  21#include "trace/mem.h"
  22
  23#if DATA_SIZE == 16
  24# define SUFFIX     o
  25# define DATA_TYPE  Int128
  26# define BSWAP      bswap128
  27# define SHIFT      4
  28#elif DATA_SIZE == 8
  29# define SUFFIX     q
  30# define DATA_TYPE  uint64_t
  31# define SDATA_TYPE int64_t
  32# define BSWAP      bswap64
  33# define SHIFT      3
  34#elif DATA_SIZE == 4
  35# define SUFFIX     l
  36# define DATA_TYPE  uint32_t
  37# define SDATA_TYPE int32_t
  38# define BSWAP      bswap32
  39# define SHIFT      2
  40#elif DATA_SIZE == 2
  41# define SUFFIX     w
  42# define DATA_TYPE  uint16_t
  43# define SDATA_TYPE int16_t
  44# define BSWAP      bswap16
  45# define SHIFT      1
  46#elif DATA_SIZE == 1
  47# define SUFFIX     b
  48# define DATA_TYPE  uint8_t
  49# define SDATA_TYPE int8_t
  50# define BSWAP
  51# define SHIFT      0
  52#else
  53# error unsupported data size
  54#endif
  55
  56#if DATA_SIZE >= 4
  57# define ABI_TYPE  DATA_TYPE
  58#else
  59# define ABI_TYPE  uint32_t
  60#endif
  61
  62#define ATOMIC_TRACE_RMW do {                                           \
  63        uint8_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, false); \
  64                                                                        \
  65        trace_guest_mem_before_exec(ENV_GET_CPU(env), addr, info);      \
  66        trace_guest_mem_before_exec(ENV_GET_CPU(env), addr,             \
  67                                    info | TRACE_MEM_ST);               \
  68    } while (0)
  69
  70#define ATOMIC_TRACE_LD do {                                            \
  71        uint8_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, false); \
  72                                                                        \
  73        trace_guest_mem_before_exec(ENV_GET_CPU(env), addr, info);      \
  74    } while (0)
  75
  76# define ATOMIC_TRACE_ST do {                                           \
  77        uint8_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, true); \
  78                                                                        \
  79        trace_guest_mem_before_exec(ENV_GET_CPU(env), addr, info);      \
  80    } while (0)
  81
  82/* Define host-endian atomic operations.  Note that END is used within
  83   the ATOMIC_NAME macro, and redefined below.  */
  84#if DATA_SIZE == 1
  85# define END
  86# define MEND _be /* either le or be would be fine */
  87#elif defined(HOST_WORDS_BIGENDIAN)
  88# define END  _be
  89# define MEND _be
  90#else
  91# define END  _le
  92# define MEND _le
  93#endif
  94
  95ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr,
  96                              ABI_TYPE cmpv, ABI_TYPE newv EXTRA_ARGS)
  97{
  98    ATOMIC_MMU_DECLS;
  99    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
 100    DATA_TYPE ret;
 101
 102    ATOMIC_TRACE_RMW;
 103#if DATA_SIZE == 16
 104    ret = atomic16_cmpxchg(haddr, cmpv, newv);
 105#else
 106    ret = atomic_cmpxchg__nocheck(haddr, cmpv, newv);
 107#endif
 108    ATOMIC_MMU_CLEANUP;
 109    return ret;
 110}
 111
 112#if DATA_SIZE >= 16
 113#if HAVE_ATOMIC128
 114ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS)
 115{
 116    ATOMIC_MMU_DECLS;
 117    DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP;
 118
 119    ATOMIC_TRACE_LD;
 120    val = atomic16_read(haddr);
 121    ATOMIC_MMU_CLEANUP;
 122    return val;
 123}
 124
 125void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr,
 126                     ABI_TYPE val EXTRA_ARGS)
 127{
 128    ATOMIC_MMU_DECLS;
 129    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
 130
 131    ATOMIC_TRACE_ST;
 132    atomic16_set(haddr, val);
 133    ATOMIC_MMU_CLEANUP;
 134}
 135#endif
 136#else
 137ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr,
 138                           ABI_TYPE val EXTRA_ARGS)
 139{
 140    ATOMIC_MMU_DECLS;
 141    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
 142    DATA_TYPE ret;
 143
 144    ATOMIC_TRACE_RMW;
 145    ret = atomic_xchg__nocheck(haddr, val);
 146    ATOMIC_MMU_CLEANUP;
 147    return ret;
 148}
 149
 150#define GEN_ATOMIC_HELPER(X)                                        \
 151ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr,       \
 152                 ABI_TYPE val EXTRA_ARGS)                           \
 153{                                                                   \
 154    ATOMIC_MMU_DECLS;                                               \
 155    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;                           \
 156    DATA_TYPE ret;                                                  \
 157                                                                    \
 158    ATOMIC_TRACE_RMW;                                               \
 159    ret = atomic_##X(haddr, val);                                   \
 160    ATOMIC_MMU_CLEANUP;                                             \
 161    return ret;                                                     \
 162}
 163
 164GEN_ATOMIC_HELPER(fetch_add)
 165GEN_ATOMIC_HELPER(fetch_and)
 166GEN_ATOMIC_HELPER(fetch_or)
 167GEN_ATOMIC_HELPER(fetch_xor)
 168GEN_ATOMIC_HELPER(add_fetch)
 169GEN_ATOMIC_HELPER(and_fetch)
 170GEN_ATOMIC_HELPER(or_fetch)
 171GEN_ATOMIC_HELPER(xor_fetch)
 172
 173#undef GEN_ATOMIC_HELPER
 174
 175/* These helpers are, as a whole, full barriers.  Within the helper,
 176 * the leading barrier is explicit and the trailing barrier is within
 177 * cmpxchg primitive.
 178 *
 179 * Trace this load + RMW loop as a single RMW op. This way, regardless
 180 * of CF_PARALLEL's value, we'll trace just a read and a write.
 181 */
 182#define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET)                \
 183ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr,       \
 184                        ABI_TYPE xval EXTRA_ARGS)                   \
 185{                                                                   \
 186    ATOMIC_MMU_DECLS;                                               \
 187    XDATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;                          \
 188    XDATA_TYPE cmp, old, new, val = xval;                           \
 189                                                                    \
 190    ATOMIC_TRACE_RMW;                                               \
 191    smp_mb();                                                       \
 192    cmp = atomic_read__nocheck(haddr);                              \
 193    do {                                                            \
 194        old = cmp; new = FN(old, val);                              \
 195        cmp = atomic_cmpxchg__nocheck(haddr, old, new);             \
 196    } while (cmp != old);                                           \
 197    ATOMIC_MMU_CLEANUP;                                             \
 198    return RET;                                                     \
 199}
 200
 201GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old)
 202GEN_ATOMIC_HELPER_FN(fetch_umin, MIN,  DATA_TYPE, old)
 203GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old)
 204GEN_ATOMIC_HELPER_FN(fetch_umax, MAX,  DATA_TYPE, old)
 205
 206GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new)
 207GEN_ATOMIC_HELPER_FN(umin_fetch, MIN,  DATA_TYPE, new)
 208GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new)
 209GEN_ATOMIC_HELPER_FN(umax_fetch, MAX,  DATA_TYPE, new)
 210
 211#undef GEN_ATOMIC_HELPER_FN
 212#endif /* DATA SIZE >= 16 */
 213
 214#undef END
 215#undef MEND
 216
 217#if DATA_SIZE > 1
 218
 219/* Define reverse-host-endian atomic operations.  Note that END is used
 220   within the ATOMIC_NAME macro.  */
 221#ifdef HOST_WORDS_BIGENDIAN
 222# define END  _le
 223# define MEND _le
 224#else
 225# define END  _be
 226# define MEND _be
 227#endif
 228
 229ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr,
 230                              ABI_TYPE cmpv, ABI_TYPE newv EXTRA_ARGS)
 231{
 232    ATOMIC_MMU_DECLS;
 233    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
 234    DATA_TYPE ret;
 235
 236    ATOMIC_TRACE_RMW;
 237#if DATA_SIZE == 16
 238    ret = atomic16_cmpxchg(haddr, BSWAP(cmpv), BSWAP(newv));
 239#else
 240    ret = atomic_cmpxchg__nocheck(haddr, BSWAP(cmpv), BSWAP(newv));
 241#endif
 242    ATOMIC_MMU_CLEANUP;
 243    return BSWAP(ret);
 244}
 245
 246#if DATA_SIZE >= 16
 247#if HAVE_ATOMIC128
 248ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS)
 249{
 250    ATOMIC_MMU_DECLS;
 251    DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP;
 252
 253    ATOMIC_TRACE_LD;
 254    val = atomic16_read(haddr);
 255    ATOMIC_MMU_CLEANUP;
 256    return BSWAP(val);
 257}
 258
 259void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr,
 260                     ABI_TYPE val EXTRA_ARGS)
 261{
 262    ATOMIC_MMU_DECLS;
 263    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
 264
 265    ATOMIC_TRACE_ST;
 266    val = BSWAP(val);
 267    atomic16_set(haddr, val);
 268    ATOMIC_MMU_CLEANUP;
 269}
 270#endif
 271#else
 272ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr,
 273                           ABI_TYPE val EXTRA_ARGS)
 274{
 275    ATOMIC_MMU_DECLS;
 276    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
 277    ABI_TYPE ret;
 278
 279    ATOMIC_TRACE_RMW;
 280    ret = atomic_xchg__nocheck(haddr, BSWAP(val));
 281    ATOMIC_MMU_CLEANUP;
 282    return BSWAP(ret);
 283}
 284
 285#define GEN_ATOMIC_HELPER(X)                                        \
 286ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr,       \
 287                 ABI_TYPE val EXTRA_ARGS)                           \
 288{                                                                   \
 289    ATOMIC_MMU_DECLS;                                               \
 290    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;                           \
 291    DATA_TYPE ret;                                                  \
 292                                                                    \
 293    ATOMIC_TRACE_RMW;                                               \
 294    ret = atomic_##X(haddr, BSWAP(val));                            \
 295    ATOMIC_MMU_CLEANUP;                                             \
 296    return BSWAP(ret);                                              \
 297}
 298
 299GEN_ATOMIC_HELPER(fetch_and)
 300GEN_ATOMIC_HELPER(fetch_or)
 301GEN_ATOMIC_HELPER(fetch_xor)
 302GEN_ATOMIC_HELPER(and_fetch)
 303GEN_ATOMIC_HELPER(or_fetch)
 304GEN_ATOMIC_HELPER(xor_fetch)
 305
 306#undef GEN_ATOMIC_HELPER
 307
 308/* These helpers are, as a whole, full barriers.  Within the helper,
 309 * the leading barrier is explicit and the trailing barrier is within
 310 * cmpxchg primitive.
 311 *
 312 * Trace this load + RMW loop as a single RMW op. This way, regardless
 313 * of CF_PARALLEL's value, we'll trace just a read and a write.
 314 */
 315#define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET)                \
 316ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr,       \
 317                        ABI_TYPE xval EXTRA_ARGS)                   \
 318{                                                                   \
 319    ATOMIC_MMU_DECLS;                                               \
 320    XDATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;                          \
 321    XDATA_TYPE ldo, ldn, old, new, val = xval;                      \
 322                                                                    \
 323    ATOMIC_TRACE_RMW;                                               \
 324    smp_mb();                                                       \
 325    ldn = atomic_read__nocheck(haddr);                              \
 326    do {                                                            \
 327        ldo = ldn; old = BSWAP(ldo); new = FN(old, val);            \
 328        ldn = atomic_cmpxchg__nocheck(haddr, ldo, BSWAP(new));      \
 329    } while (ldo != ldn);                                           \
 330    ATOMIC_MMU_CLEANUP;                                             \
 331    return RET;                                                     \
 332}
 333
 334GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old)
 335GEN_ATOMIC_HELPER_FN(fetch_umin, MIN,  DATA_TYPE, old)
 336GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old)
 337GEN_ATOMIC_HELPER_FN(fetch_umax, MAX,  DATA_TYPE, old)
 338
 339GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new)
 340GEN_ATOMIC_HELPER_FN(umin_fetch, MIN,  DATA_TYPE, new)
 341GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new)
 342GEN_ATOMIC_HELPER_FN(umax_fetch, MAX,  DATA_TYPE, new)
 343
 344/* Note that for addition, we need to use a separate cmpxchg loop instead
 345   of bswaps for the reverse-host-endian helpers.  */
 346#define ADD(X, Y)   (X + Y)
 347GEN_ATOMIC_HELPER_FN(fetch_add, ADD, DATA_TYPE, old)
 348GEN_ATOMIC_HELPER_FN(add_fetch, ADD, DATA_TYPE, new)
 349#undef ADD
 350
 351#undef GEN_ATOMIC_HELPER_FN
 352#endif /* DATA_SIZE >= 16 */
 353
 354#undef END
 355#undef MEND
 356#endif /* DATA_SIZE > 1 */
 357
 358#undef ATOMIC_TRACE_ST
 359#undef ATOMIC_TRACE_LD
 360#undef ATOMIC_TRACE_RMW
 361
 362#undef BSWAP
 363#undef ABI_TYPE
 364#undef DATA_TYPE
 365#undef SDATA_TYPE
 366#undef SUFFIX
 367#undef DATA_SIZE
 368#undef SHIFT
 369