LXR qemu/accel/tcg/atomic

   1/*
   2 * Atomic helper templates
   3 * Included from tcg-runtime.c and cputlb.c.
   4 *
   5 * Copyright (c) 2016 Red Hat, Inc
   6 *
   7 * This library is free software; you can redistribute it and/or
   8 * modify it under the terms of the GNU Lesser General Public
   9 * License as published by the Free Software Foundation; either
  10 * version 2.1 of the License, or (at your option) any later version.
  11 *
  12 * This library is distributed in the hope that it will be useful,
  13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15 * Lesser General Public License for more details.
  16 *
  17 * You should have received a copy of the GNU Lesser General Public
  18 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  19 */
  20
  21#include "qemu/plugin.h"
  22
  23#if DATA_SIZE == 16
  24# define SUFFIX     o
  25# define DATA_TYPE  Int128
  26# define BSWAP      bswap128
  27# define SHIFT      4
  28#elif DATA_SIZE == 8
  29# define SUFFIX     q
  30# define DATA_TYPE  aligned_uint64_t
  31# define SDATA_TYPE aligned_int64_t
  32# define BSWAP      bswap64
  33# define SHIFT      3
  34#elif DATA_SIZE == 4
  35# define SUFFIX     l
  36# define DATA_TYPE  uint32_t
  37# define SDATA_TYPE int32_t
  38# define BSWAP      bswap32
  39# define SHIFT      2
  40#elif DATA_SIZE == 2
  41# define SUFFIX     w
  42# define DATA_TYPE  uint16_t
  43# define SDATA_TYPE int16_t
  44# define BSWAP      bswap16
  45# define SHIFT      1
  46#elif DATA_SIZE == 1
  47# define SUFFIX     b
  48# define DATA_TYPE  uint8_t
  49# define SDATA_TYPE int8_t
  50# define BSWAP
  51# define SHIFT      0
  52#else
  53# error unsupported data size
  54#endif
  55
  56#if DATA_SIZE >= 4
  57# define ABI_TYPE  DATA_TYPE
  58#else
  59# define ABI_TYPE  uint32_t
  60#endif
  61
  62/* Define host-endian atomic operations.  Note that END is used within
  63   the ATOMIC_NAME macro, and redefined below.  */
  64#if DATA_SIZE == 1
  65# define END
  66#elif defined(HOST_WORDS_BIGENDIAN)
  67# define END  _be
  68#else
  69# define END  _le
  70#endif
  71
  72ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr,
  73                              ABI_TYPE cmpv, ABI_TYPE newv,
  74                              MemOpIdx oi, uintptr_t retaddr)
  75{
  76    DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE,
  77                                         PAGE_READ | PAGE_WRITE, retaddr);
  78    DATA_TYPE ret;
  79
  80    atomic_trace_rmw_pre(env, addr, oi);
  81#if DATA_SIZE == 16
  82    ret = atomic16_cmpxchg(haddr, cmpv, newv);
  83#else
  84    ret = qatomic_cmpxchg__nocheck(haddr, cmpv, newv);
  85#endif
  86    ATOMIC_MMU_CLEANUP;
  87    atomic_trace_rmw_post(env, addr, oi);
  88    return ret;
  89}
  90
  91#if DATA_SIZE >= 16
  92#if HAVE_ATOMIC128
  93ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr,
  94                         MemOpIdx oi, uintptr_t retaddr)
  95{
  96    DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE,
  97                                         PAGE_READ, retaddr);
  98    DATA_TYPE val;
  99
 100    atomic_trace_ld_pre(env, addr, oi);
 101    val = atomic16_read(haddr);
 102    ATOMIC_MMU_CLEANUP;
 103    atomic_trace_ld_post(env, addr, oi);
 104    return val;
 105}
 106
 107void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr, ABI_TYPE val,
 108                     MemOpIdx oi, uintptr_t retaddr)
 109{
 110    DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE,
 111                                         PAGE_WRITE, retaddr);
 112
 113    atomic_trace_st_pre(env, addr, oi);
 114    atomic16_set(haddr, val);
 115    ATOMIC_MMU_CLEANUP;
 116    atomic_trace_st_post(env, addr, oi);
 117}
 118#endif
 119#else
 120ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr, ABI_TYPE val,
 121                           MemOpIdx oi, uintptr_t retaddr)
 122{
 123    DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE,
 124                                         PAGE_READ | PAGE_WRITE, retaddr);
 125    DATA_TYPE ret;
 126
 127    atomic_trace_rmw_pre(env, addr, oi);
 128    ret = qatomic_xchg__nocheck(haddr, val);
 129    ATOMIC_MMU_CLEANUP;
 130    atomic_trace_rmw_post(env, addr, oi);
 131    return ret;
 132}
 133
 134#define GEN_ATOMIC_HELPER(X)                                        \
 135ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr,       \
 136                        ABI_TYPE val, MemOpIdx oi, uintptr_t retaddr) \
 137{                                                                   \
 138    DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE,  \
 139                                         PAGE_READ | PAGE_WRITE, retaddr); \
 140    DATA_TYPE ret;                                                  \
 141    atomic_trace_rmw_pre(env, addr, oi);                            \
 142    ret = qatomic_##X(haddr, val);                                  \
 143    ATOMIC_MMU_CLEANUP;                                             \
 144    atomic_trace_rmw_post(env, addr, oi);                           \
 145    return ret;                                                     \
 146}
 147
 148GEN_ATOMIC_HELPER(fetch_add)
 149GEN_ATOMIC_HELPER(fetch_and)
 150GEN_ATOMIC_HELPER(fetch_or)
 151GEN_ATOMIC_HELPER(fetch_xor)
 152GEN_ATOMIC_HELPER(add_fetch)
 153GEN_ATOMIC_HELPER(and_fetch)
 154GEN_ATOMIC_HELPER(or_fetch)
 155GEN_ATOMIC_HELPER(xor_fetch)
 156
 157#undef GEN_ATOMIC_HELPER
 158
 159/*
 160 * These helpers are, as a whole, full barriers.  Within the helper,
 161 * the leading barrier is explicit and the trailing barrier is within
 162 * cmpxchg primitive.
 163 *
 164 * Trace this load + RMW loop as a single RMW op. This way, regardless
 165 * of CF_PARALLEL's value, we'll trace just a read and a write.
 166 */
 167#define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET)                \
 168ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr,       \
 169                        ABI_TYPE xval, MemOpIdx oi, uintptr_t retaddr) \
 170{                                                                   \
 171    XDATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, \
 172                                          PAGE_READ | PAGE_WRITE, retaddr); \
 173    XDATA_TYPE cmp, old, new, val = xval;                           \
 174    atomic_trace_rmw_pre(env, addr, oi);                            \
 175    smp_mb();                                                       \
 176    cmp = qatomic_read__nocheck(haddr);                             \
 177    do {                                                            \
 178        old = cmp; new = FN(old, val);                              \
 179        cmp = qatomic_cmpxchg__nocheck(haddr, old, new);            \
 180    } while (cmp != old);                                           \
 181    ATOMIC_MMU_CLEANUP;                                             \
 182    atomic_trace_rmw_post(env, addr, oi);                           \
 183    return RET;                                                     \
 184}
 185
 186GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old)
 187GEN_ATOMIC_HELPER_FN(fetch_umin, MIN,  DATA_TYPE, old)
 188GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old)
 189GEN_ATOMIC_HELPER_FN(fetch_umax, MAX,  DATA_TYPE, old)
 190
 191GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new)
 192GEN_ATOMIC_HELPER_FN(umin_fetch, MIN,  DATA_TYPE, new)
 193GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new)
 194GEN_ATOMIC_HELPER_FN(umax_fetch, MAX,  DATA_TYPE, new)
 195
 196#undef GEN_ATOMIC_HELPER_FN
 197#endif /* DATA SIZE >= 16 */
 198
 199#undef END
 200
 201#if DATA_SIZE > 1
 202
 203/* Define reverse-host-endian atomic operations.  Note that END is used
 204   within the ATOMIC_NAME macro.  */
 205#ifdef HOST_WORDS_BIGENDIAN
 206# define END  _le
 207#else
 208# define END  _be
 209#endif
 210
 211ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr,
 212                              ABI_TYPE cmpv, ABI_TYPE newv,
 213                              MemOpIdx oi, uintptr_t retaddr)
 214{
 215    DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE,
 216                                         PAGE_READ | PAGE_WRITE, retaddr);
 217    DATA_TYPE ret;
 218
 219    atomic_trace_rmw_pre(env, addr, oi);
 220#if DATA_SIZE == 16
 221    ret = atomic16_cmpxchg(haddr, BSWAP(cmpv), BSWAP(newv));
 222#else
 223    ret = qatomic_cmpxchg__nocheck(haddr, BSWAP(cmpv), BSWAP(newv));
 224#endif
 225    ATOMIC_MMU_CLEANUP;
 226    atomic_trace_rmw_post(env, addr, oi);
 227    return BSWAP(ret);
 228}
 229
 230#if DATA_SIZE >= 16
 231#if HAVE_ATOMIC128
 232ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr,
 233                         MemOpIdx oi, uintptr_t retaddr)
 234{
 235    DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE,
 236                                         PAGE_READ, retaddr);
 237    DATA_TYPE val;
 238
 239    atomic_trace_ld_pre(env, addr, oi);
 240    val = atomic16_read(haddr);
 241    ATOMIC_MMU_CLEANUP;
 242    atomic_trace_ld_post(env, addr, oi);
 243    return BSWAP(val);
 244}
 245
 246void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr, ABI_TYPE val,
 247                     MemOpIdx oi, uintptr_t retaddr)
 248{
 249    DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE,
 250                                         PAGE_WRITE, retaddr);
 251
 252    atomic_trace_st_pre(env, addr, oi);
 253    val = BSWAP(val);
 254    atomic16_set(haddr, val);
 255    ATOMIC_MMU_CLEANUP;
 256    atomic_trace_st_post(env, addr, oi);
 257}
 258#endif
 259#else
 260ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr, ABI_TYPE val,
 261                           MemOpIdx oi, uintptr_t retaddr)
 262{
 263    DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE,
 264                                         PAGE_READ | PAGE_WRITE, retaddr);
 265    ABI_TYPE ret;
 266
 267    atomic_trace_rmw_pre(env, addr, oi);
 268    ret = qatomic_xchg__nocheck(haddr, BSWAP(val));
 269    ATOMIC_MMU_CLEANUP;
 270    atomic_trace_rmw_post(env, addr, oi);
 271    return BSWAP(ret);
 272}
 273
 274#define GEN_ATOMIC_HELPER(X)                                        \
 275ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr,       \
 276                        ABI_TYPE val, MemOpIdx oi, uintptr_t retaddr) \
 277{                                                                   \
 278    DATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE,  \
 279                                         PAGE_READ | PAGE_WRITE, retaddr); \
 280    DATA_TYPE ret;                                                  \
 281    atomic_trace_rmw_pre(env, addr, oi);                            \
 282    ret = qatomic_##X(haddr, BSWAP(val));                           \
 283    ATOMIC_MMU_CLEANUP;                                             \
 284    atomic_trace_rmw_post(env, addr, oi);                           \
 285    return BSWAP(ret);                                              \
 286}
 287
 288GEN_ATOMIC_HELPER(fetch_and)
 289GEN_ATOMIC_HELPER(fetch_or)
 290GEN_ATOMIC_HELPER(fetch_xor)
 291GEN_ATOMIC_HELPER(and_fetch)
 292GEN_ATOMIC_HELPER(or_fetch)
 293GEN_ATOMIC_HELPER(xor_fetch)
 294
 295#undef GEN_ATOMIC_HELPER
 296
 297/* These helpers are, as a whole, full barriers.  Within the helper,
 298 * the leading barrier is explicit and the trailing barrier is within
 299 * cmpxchg primitive.
 300 *
 301 * Trace this load + RMW loop as a single RMW op. This way, regardless
 302 * of CF_PARALLEL's value, we'll trace just a read and a write.
 303 */
 304#define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET)                \
 305ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr,       \
 306                        ABI_TYPE xval, MemOpIdx oi, uintptr_t retaddr) \
 307{                                                                   \
 308    XDATA_TYPE *haddr = atomic_mmu_lookup(env, addr, oi, DATA_SIZE, \
 309                                          PAGE_READ | PAGE_WRITE, retaddr); \
 310    XDATA_TYPE ldo, ldn, old, new, val = xval;                      \
 311    atomic_trace_rmw_pre(env, addr, oi);                            \
 312    smp_mb();                                                       \
 313    ldn = qatomic_read__nocheck(haddr);                             \
 314    do {                                                            \
 315        ldo = ldn; old = BSWAP(ldo); new = FN(old, val);            \
 316        ldn = qatomic_cmpxchg__nocheck(haddr, ldo, BSWAP(new));     \
 317    } while (ldo != ldn);                                           \
 318    ATOMIC_MMU_CLEANUP;                                             \
 319    atomic_trace_rmw_post(env, addr, oi);                           \
 320    return RET;                                                     \
 321}
 322
 323GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old)
 324GEN_ATOMIC_HELPER_FN(fetch_umin, MIN,  DATA_TYPE, old)
 325GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old)
 326GEN_ATOMIC_HELPER_FN(fetch_umax, MAX,  DATA_TYPE, old)
 327
 328GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new)
 329GEN_ATOMIC_HELPER_FN(umin_fetch, MIN,  DATA_TYPE, new)
 330GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new)
 331GEN_ATOMIC_HELPER_FN(umax_fetch, MAX,  DATA_TYPE, new)
 332
 333/* Note that for addition, we need to use a separate cmpxchg loop instead
 334   of bswaps for the reverse-host-endian helpers.  */
 335#define ADD(X, Y)   (X + Y)
 336GEN_ATOMIC_HELPER_FN(fetch_add, ADD, DATA_TYPE, old)
 337GEN_ATOMIC_HELPER_FN(add_fetch, ADD, DATA_TYPE, new)
 338#undef ADD
 339
 340#undef GEN_ATOMIC_HELPER_FN
 341#endif /* DATA_SIZE >= 16 */
 342
 343#undef END
 344#endif /* DATA_SIZE > 1 */
 345
 346#undef BSWAP
 347#undef ABI_TYPE
 348#undef DATA_TYPE
 349#undef SDATA_TYPE
 350#undef SUFFIX
 351#undef DATA_SIZE
 352#undef SHIFT
 353