LXR qemu/accel/tcg/atomic

   1/*
   2 * Atomic helper templates
   3 * Included from tcg-runtime.c and cputlb.c.
   4 *
   5 * Copyright (c) 2016 Red Hat, Inc
   6 *
   7 * This library is free software; you can redistribute it and/or
   8 * modify it under the terms of the GNU Lesser General Public
   9 * License as published by the Free Software Foundation; either
  10 * version 2.1 of the License, or (at your option) any later version.
  11 *
  12 * This library is distributed in the hope that it will be useful,
  13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15 * Lesser General Public License for more details.
  16 *
  17 * You should have received a copy of the GNU Lesser General Public
  18 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  19 */
  20
  21#include "qemu/plugin.h"
  22#include "trace/mem.h"
  23
  24#if DATA_SIZE == 16
  25# define SUFFIX     o
  26# define DATA_TYPE  Int128
  27# define BSWAP      bswap128
  28# define SHIFT      4
  29#elif DATA_SIZE == 8
  30# define SUFFIX     q
  31# define DATA_TYPE  uint64_t
  32# define SDATA_TYPE int64_t
  33# define BSWAP      bswap64
  34# define SHIFT      3
  35#elif DATA_SIZE == 4
  36# define SUFFIX     l
  37# define DATA_TYPE  uint32_t
  38# define SDATA_TYPE int32_t
  39# define BSWAP      bswap32
  40# define SHIFT      2
  41#elif DATA_SIZE == 2
  42# define SUFFIX     w
  43# define DATA_TYPE  uint16_t
  44# define SDATA_TYPE int16_t
  45# define BSWAP      bswap16
  46# define SHIFT      1
  47#elif DATA_SIZE == 1
  48# define SUFFIX     b
  49# define DATA_TYPE  uint8_t
  50# define SDATA_TYPE int8_t
  51# define BSWAP
  52# define SHIFT      0
  53#else
  54# error unsupported data size
  55#endif
  56
  57#if DATA_SIZE >= 4
  58# define ABI_TYPE  DATA_TYPE
  59#else
  60# define ABI_TYPE  uint32_t
  61#endif
  62
  63/* Define host-endian atomic operations.  Note that END is used within
  64   the ATOMIC_NAME macro, and redefined below.  */
  65#if DATA_SIZE == 1
  66# define END
  67#elif defined(HOST_WORDS_BIGENDIAN)
  68# define END  _be
  69#else
  70# define END  _le
  71#endif
  72
  73ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr,
  74                              ABI_TYPE cmpv, ABI_TYPE newv EXTRA_ARGS)
  75{
  76    ATOMIC_MMU_DECLS;
  77    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
  78    DATA_TYPE ret;
  79    uint16_t info = trace_mem_build_info(SHIFT, false, 0, false,
  80                                         ATOMIC_MMU_IDX);
  81
  82    atomic_trace_rmw_pre(env, addr, info);
  83#if DATA_SIZE == 16
  84    ret = atomic16_cmpxchg(haddr, cmpv, newv);
  85#else
  86    ret = qatomic_cmpxchg__nocheck(haddr, cmpv, newv);
  87#endif
  88    ATOMIC_MMU_CLEANUP;
  89    atomic_trace_rmw_post(env, addr, info);
  90    return ret;
  91}
  92
  93#if DATA_SIZE >= 16
  94#if HAVE_ATOMIC128
  95ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS)
  96{
  97    ATOMIC_MMU_DECLS;
  98    DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP;
  99    uint16_t info = trace_mem_build_info(SHIFT, false, 0, false,
 100                                         ATOMIC_MMU_IDX);
 101
 102    atomic_trace_ld_pre(env, addr, info);
 103    val = atomic16_read(haddr);
 104    ATOMIC_MMU_CLEANUP;
 105    atomic_trace_ld_post(env, addr, info);
 106    return val;
 107}
 108
 109void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr,
 110                     ABI_TYPE val EXTRA_ARGS)
 111{
 112    ATOMIC_MMU_DECLS;
 113    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
 114    uint16_t info = trace_mem_build_info(SHIFT, false, 0, true,
 115                                         ATOMIC_MMU_IDX);
 116
 117    atomic_trace_st_pre(env, addr, info);
 118    atomic16_set(haddr, val);
 119    ATOMIC_MMU_CLEANUP;
 120    atomic_trace_st_post(env, addr, info);
 121}
 122#endif
 123#else
 124ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr,
 125                           ABI_TYPE val EXTRA_ARGS)
 126{
 127    ATOMIC_MMU_DECLS;
 128    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
 129    DATA_TYPE ret;
 130    uint16_t info = trace_mem_build_info(SHIFT, false, 0, false,
 131                                         ATOMIC_MMU_IDX);
 132
 133    atomic_trace_rmw_pre(env, addr, info);
 134    ret = qatomic_xchg__nocheck(haddr, val);
 135    ATOMIC_MMU_CLEANUP;
 136    atomic_trace_rmw_post(env, addr, info);
 137    return ret;
 138}
 139
 140#define GEN_ATOMIC_HELPER(X)                                        \
 141ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr,       \
 142                        ABI_TYPE val EXTRA_ARGS)                    \
 143{                                                                   \
 144    ATOMIC_MMU_DECLS;                                               \
 145    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;                           \
 146    DATA_TYPE ret;                                                  \
 147    uint16_t info = trace_mem_build_info(SHIFT, false, 0, false,    \
 148                                         ATOMIC_MMU_IDX);           \
 149    atomic_trace_rmw_pre(env, addr, info);                          \
 150    ret = qatomic_##X(haddr, val);                                  \
 151    ATOMIC_MMU_CLEANUP;                                             \
 152    atomic_trace_rmw_post(env, addr, info);                         \
 153    return ret;                                                     \
 154}
 155
 156GEN_ATOMIC_HELPER(fetch_add)
 157GEN_ATOMIC_HELPER(fetch_and)
 158GEN_ATOMIC_HELPER(fetch_or)
 159GEN_ATOMIC_HELPER(fetch_xor)
 160GEN_ATOMIC_HELPER(add_fetch)
 161GEN_ATOMIC_HELPER(and_fetch)
 162GEN_ATOMIC_HELPER(or_fetch)
 163GEN_ATOMIC_HELPER(xor_fetch)
 164
 165#undef GEN_ATOMIC_HELPER
 166
 167/* These helpers are, as a whole, full barriers.  Within the helper,
 168 * the leading barrier is explicit and the trailing barrier is within
 169 * cmpxchg primitive.
 170 *
 171 * Trace this load + RMW loop as a single RMW op. This way, regardless
 172 * of CF_PARALLEL's value, we'll trace just a read and a write.
 173 */
 174#define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET)                \
 175ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr,       \
 176                        ABI_TYPE xval EXTRA_ARGS)                   \
 177{                                                                   \
 178    ATOMIC_MMU_DECLS;                                               \
 179    XDATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;                          \
 180    XDATA_TYPE cmp, old, new, val = xval;                           \
 181    uint16_t info = trace_mem_build_info(SHIFT, false, 0, false,    \
 182                                         ATOMIC_MMU_IDX);           \
 183    atomic_trace_rmw_pre(env, addr, info);                          \
 184    smp_mb();                                                       \
 185    cmp = qatomic_read__nocheck(haddr);                             \
 186    do {                                                            \
 187        old = cmp; new = FN(old, val);                              \
 188        cmp = qatomic_cmpxchg__nocheck(haddr, old, new);            \
 189    } while (cmp != old);                                           \
 190    ATOMIC_MMU_CLEANUP;                                             \
 191    atomic_trace_rmw_post(env, addr, info);                         \
 192    return RET;                                                     \
 193}
 194
 195GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old)
 196GEN_ATOMIC_HELPER_FN(fetch_umin, MIN,  DATA_TYPE, old)
 197GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old)
 198GEN_ATOMIC_HELPER_FN(fetch_umax, MAX,  DATA_TYPE, old)
 199
 200GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new)
 201GEN_ATOMIC_HELPER_FN(umin_fetch, MIN,  DATA_TYPE, new)
 202GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new)
 203GEN_ATOMIC_HELPER_FN(umax_fetch, MAX,  DATA_TYPE, new)
 204
 205#undef GEN_ATOMIC_HELPER_FN
 206#endif /* DATA SIZE >= 16 */
 207
 208#undef END
 209
 210#if DATA_SIZE > 1
 211
 212/* Define reverse-host-endian atomic operations.  Note that END is used
 213   within the ATOMIC_NAME macro.  */
 214#ifdef HOST_WORDS_BIGENDIAN
 215# define END  _le
 216#else
 217# define END  _be
 218#endif
 219
 220ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr,
 221                              ABI_TYPE cmpv, ABI_TYPE newv EXTRA_ARGS)
 222{
 223    ATOMIC_MMU_DECLS;
 224    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
 225    DATA_TYPE ret;
 226    uint16_t info = trace_mem_build_info(SHIFT, false, MO_BSWAP, false,
 227                                         ATOMIC_MMU_IDX);
 228
 229    atomic_trace_rmw_pre(env, addr, info);
 230#if DATA_SIZE == 16
 231    ret = atomic16_cmpxchg(haddr, BSWAP(cmpv), BSWAP(newv));
 232#else
 233    ret = qatomic_cmpxchg__nocheck(haddr, BSWAP(cmpv), BSWAP(newv));
 234#endif
 235    ATOMIC_MMU_CLEANUP;
 236    atomic_trace_rmw_post(env, addr, info);
 237    return BSWAP(ret);
 238}
 239
 240#if DATA_SIZE >= 16
 241#if HAVE_ATOMIC128
 242ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS)
 243{
 244    ATOMIC_MMU_DECLS;
 245    DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP;
 246    uint16_t info = trace_mem_build_info(SHIFT, false, MO_BSWAP, false,
 247                                         ATOMIC_MMU_IDX);
 248
 249    atomic_trace_ld_pre(env, addr, info);
 250    val = atomic16_read(haddr);
 251    ATOMIC_MMU_CLEANUP;
 252    atomic_trace_ld_post(env, addr, info);
 253    return BSWAP(val);
 254}
 255
 256void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr,
 257                     ABI_TYPE val EXTRA_ARGS)
 258{
 259    ATOMIC_MMU_DECLS;
 260    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
 261    uint16_t info = trace_mem_build_info(SHIFT, false, MO_BSWAP, true,
 262                                         ATOMIC_MMU_IDX);
 263
 264    val = BSWAP(val);
 265    atomic_trace_st_pre(env, addr, info);
 266    val = BSWAP(val);
 267    atomic16_set(haddr, val);
 268    ATOMIC_MMU_CLEANUP;
 269    atomic_trace_st_post(env, addr, info);
 270}
 271#endif
 272#else
 273ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr,
 274                           ABI_TYPE val EXTRA_ARGS)
 275{
 276    ATOMIC_MMU_DECLS;
 277    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
 278    ABI_TYPE ret;
 279    uint16_t info = trace_mem_build_info(SHIFT, false, MO_BSWAP, false,
 280                                         ATOMIC_MMU_IDX);
 281
 282    atomic_trace_rmw_pre(env, addr, info);
 283    ret = qatomic_xchg__nocheck(haddr, BSWAP(val));
 284    ATOMIC_MMU_CLEANUP;
 285    atomic_trace_rmw_post(env, addr, info);
 286    return BSWAP(ret);
 287}
 288
 289#define GEN_ATOMIC_HELPER(X)                                        \
 290ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr,       \
 291                        ABI_TYPE val EXTRA_ARGS)                    \
 292{                                                                   \
 293    ATOMIC_MMU_DECLS;                                               \
 294    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;                           \
 295    DATA_TYPE ret;                                                  \
 296    uint16_t info = trace_mem_build_info(SHIFT, false, MO_BSWAP,    \
 297                                         false, ATOMIC_MMU_IDX);    \
 298    atomic_trace_rmw_pre(env, addr, info);                          \
 299    ret = qatomic_##X(haddr, BSWAP(val));                           \
 300    ATOMIC_MMU_CLEANUP;                                             \
 301    atomic_trace_rmw_post(env, addr, info);                         \
 302    return BSWAP(ret);                                              \
 303}
 304
 305GEN_ATOMIC_HELPER(fetch_and)
 306GEN_ATOMIC_HELPER(fetch_or)
 307GEN_ATOMIC_HELPER(fetch_xor)
 308GEN_ATOMIC_HELPER(and_fetch)
 309GEN_ATOMIC_HELPER(or_fetch)
 310GEN_ATOMIC_HELPER(xor_fetch)
 311
 312#undef GEN_ATOMIC_HELPER
 313
 314/* These helpers are, as a whole, full barriers.  Within the helper,
 315 * the leading barrier is explicit and the trailing barrier is within
 316 * cmpxchg primitive.
 317 *
 318 * Trace this load + RMW loop as a single RMW op. This way, regardless
 319 * of CF_PARALLEL's value, we'll trace just a read and a write.
 320 */
 321#define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET)                \
 322ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr,       \
 323                        ABI_TYPE xval EXTRA_ARGS)                   \
 324{                                                                   \
 325    ATOMIC_MMU_DECLS;                                               \
 326    XDATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;                          \
 327    XDATA_TYPE ldo, ldn, old, new, val = xval;                      \
 328    uint16_t info = trace_mem_build_info(SHIFT, false, MO_BSWAP,    \
 329                                         false, ATOMIC_MMU_IDX);    \
 330    atomic_trace_rmw_pre(env, addr, info);                          \
 331    smp_mb();                                                       \
 332    ldn = qatomic_read__nocheck(haddr);                             \
 333    do {                                                            \
 334        ldo = ldn; old = BSWAP(ldo); new = FN(old, val);            \
 335        ldn = qatomic_cmpxchg__nocheck(haddr, ldo, BSWAP(new));     \
 336    } while (ldo != ldn);                                           \
 337    ATOMIC_MMU_CLEANUP;                                             \
 338    atomic_trace_rmw_post(env, addr, info);                         \
 339    return RET;                                                     \
 340}
 341
 342GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old)
 343GEN_ATOMIC_HELPER_FN(fetch_umin, MIN,  DATA_TYPE, old)
 344GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old)
 345GEN_ATOMIC_HELPER_FN(fetch_umax, MAX,  DATA_TYPE, old)
 346
 347GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new)
 348GEN_ATOMIC_HELPER_FN(umin_fetch, MIN,  DATA_TYPE, new)
 349GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new)
 350GEN_ATOMIC_HELPER_FN(umax_fetch, MAX,  DATA_TYPE, new)
 351
 352/* Note that for addition, we need to use a separate cmpxchg loop instead
 353   of bswaps for the reverse-host-endian helpers.  */
 354#define ADD(X, Y)   (X + Y)
 355GEN_ATOMIC_HELPER_FN(fetch_add, ADD, DATA_TYPE, old)
 356GEN_ATOMIC_HELPER_FN(add_fetch, ADD, DATA_TYPE, new)
 357#undef ADD
 358
 359#undef GEN_ATOMIC_HELPER_FN
 360#endif /* DATA_SIZE >= 16 */
 361
 362#undef END
 363#endif /* DATA_SIZE > 1 */
 364
 365#undef BSWAP
 366#undef ABI_TYPE
 367#undef DATA_TYPE
 368#undef SDATA_TYPE
 369#undef SUFFIX
 370#undef DATA_SIZE
 371#undef SHIFT
 372