LXR qemu/include/qemu/atomic128.h

   1/*
   2 * Simple interface for 128-bit atomic operations.
   3 *
   4 * Copyright (C) 2018 Linaro, Ltd.
   5 *
   6 * This work is licensed under the terms of the GNU GPL, version 2 or later.
   7 * See the COPYING file in the top-level directory.
   8 *
   9 * See docs/devel/atomics.rst for discussion about the guarantees each
  10 * atomic primitive is meant to provide.
  11 */
  12
  13#ifndef QEMU_ATOMIC128_H
  14#define QEMU_ATOMIC128_H
  15
  16#include "qemu/int128.h"
  17
  18/*
  19 * GCC is a house divided about supporting large atomic operations.
  20 *
  21 * For hosts that only have large compare-and-swap, a legalistic reading
  22 * of the C++ standard means that one cannot implement __atomic_read on
  23 * read-only memory, and thus all atomic operations must synchronize
  24 * through libatomic.
  25 *
  26 * See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80878
  27 *
  28 * This interpretation is not especially helpful for QEMU.
  29 * For softmmu, all RAM is always read/write from the hypervisor.
  30 * For user-only, if the guest doesn't implement such an __atomic_read
  31 * then the host need not worry about it either.
  32 *
  33 * Moreover, using libatomic is not an option, because its interface is
  34 * built for std::atomic<T>, and requires that *all* accesses to such an
  35 * object go through the library.  In our case we do not have an object
  36 * in the C/C++ sense, but a view of memory as seen by the guest.
  37 * The guest may issue a large atomic operation and then access those
  38 * pieces using word-sized accesses.  From the hypervisor, we have no
  39 * way to connect those two actions.
  40 *
  41 * Therefore, special case each platform.
  42 */
  43
  44#if defined(CONFIG_ATOMIC128)
  45static inline Int128 atomic16_cmpxchg(Int128 *ptr, Int128 cmp, Int128 new)
  46{
  47    Int128Alias r, c, n;
  48
  49    c.s = cmp;
  50    n.s = new;
  51    r.i = qatomic_cmpxchg__nocheck((__int128_t *)ptr, c.i, n.i);
  52    return r.s;
  53}
  54# define HAVE_CMPXCHG128 1
  55#elif defined(CONFIG_CMPXCHG128)
  56static inline Int128 atomic16_cmpxchg(Int128 *ptr, Int128 cmp, Int128 new)
  57{
  58    Int128Alias r, c, n;
  59
  60    c.s = cmp;
  61    n.s = new;
  62    r.i = __sync_val_compare_and_swap_16((__int128_t *)ptr, c.i, n.i);
  63    return r.s;
  64}
  65# define HAVE_CMPXCHG128 1
  66#elif defined(__aarch64__)
  67/* Through gcc 8, aarch64 has no support for 128-bit at all.  */
  68static inline Int128 atomic16_cmpxchg(Int128 *ptr, Int128 cmp, Int128 new)
  69{
  70    uint64_t cmpl = int128_getlo(cmp), cmph = int128_gethi(cmp);
  71    uint64_t newl = int128_getlo(new), newh = int128_gethi(new);
  72    uint64_t oldl, oldh;
  73    uint32_t tmp;
  74
  75    asm("0: ldaxp %[oldl], %[oldh], %[mem]\n\t"
  76        "cmp %[oldl], %[cmpl]\n\t"
  77        "ccmp %[oldh], %[cmph], #0, eq\n\t"
  78        "b.ne 1f\n\t"
  79        "stlxp %w[tmp], %[newl], %[newh], %[mem]\n\t"
  80        "cbnz %w[tmp], 0b\n"
  81        "1:"
  82        : [mem] "+m"(*ptr), [tmp] "=&r"(tmp),
  83          [oldl] "=&r"(oldl), [oldh] "=&r"(oldh)
  84        : [cmpl] "r"(cmpl), [cmph] "r"(cmph),
  85          [newl] "r"(newl), [newh] "r"(newh)
  86        : "memory", "cc");
  87
  88    return int128_make128(oldl, oldh);
  89}
  90# define HAVE_CMPXCHG128 1
  91#else
  92/* Fallback definition that must be optimized away, or error.  */
  93Int128 QEMU_ERROR("unsupported atomic")
  94    atomic16_cmpxchg(Int128 *ptr, Int128 cmp, Int128 new);
  95# define HAVE_CMPXCHG128 0
  96#endif /* Some definition for HAVE_CMPXCHG128 */
  97
  98
  99#if defined(CONFIG_ATOMIC128)
 100static inline Int128 atomic16_read(Int128 *ptr)
 101{
 102    Int128Alias r;
 103
 104    r.i = qatomic_read__nocheck((__int128_t *)ptr);
 105    return r.s;
 106}
 107
 108static inline void atomic16_set(Int128 *ptr, Int128 val)
 109{
 110    Int128Alias v;
 111
 112    v.s = val;
 113    qatomic_set__nocheck((__int128_t *)ptr, v.i);
 114}
 115
 116# define HAVE_ATOMIC128 1
 117#elif !defined(CONFIG_USER_ONLY) && defined(__aarch64__)
 118/* We can do better than cmpxchg for AArch64.  */
 119static inline Int128 atomic16_read(Int128 *ptr)
 120{
 121    uint64_t l, h;
 122    uint32_t tmp;
 123
 124    /* The load must be paired with the store to guarantee not tearing.  */
 125    asm("0: ldxp %[l], %[h], %[mem]\n\t"
 126        "stxp %w[tmp], %[l], %[h], %[mem]\n\t"
 127        "cbnz %w[tmp], 0b"
 128        : [mem] "+m"(*ptr), [tmp] "=r"(tmp), [l] "=r"(l), [h] "=r"(h));
 129
 130    return int128_make128(l, h);
 131}
 132
 133static inline void atomic16_set(Int128 *ptr, Int128 val)
 134{
 135    uint64_t l = int128_getlo(val), h = int128_gethi(val);
 136    uint64_t t1, t2;
 137
 138    /* Load into temporaries to acquire the exclusive access lock.  */
 139    asm("0: ldxp %[t1], %[t2], %[mem]\n\t"
 140        "stxp %w[t1], %[l], %[h], %[mem]\n\t"
 141        "cbnz %w[t1], 0b"
 142        : [mem] "+m"(*ptr), [t1] "=&r"(t1), [t2] "=&r"(t2)
 143        : [l] "r"(l), [h] "r"(h));
 144}
 145
 146# define HAVE_ATOMIC128 1
 147#elif !defined(CONFIG_USER_ONLY) && HAVE_CMPXCHG128
 148static inline Int128 atomic16_read(Int128 *ptr)
 149{
 150    /* Maybe replace 0 with 0, returning the old value.  */
 151    Int128 z = int128_make64(0);
 152    return atomic16_cmpxchg(ptr, z, z);
 153}
 154
 155static inline void atomic16_set(Int128 *ptr, Int128 val)
 156{
 157    Int128 old = *ptr, cmp;
 158    do {
 159        cmp = old;
 160        old = atomic16_cmpxchg(ptr, cmp, val);
 161    } while (int128_ne(old, cmp));
 162}
 163
 164# define HAVE_ATOMIC128 1
 165#else
 166/* Fallback definitions that must be optimized away, or error.  */
 167Int128 QEMU_ERROR("unsupported atomic") atomic16_read(Int128 *ptr);
 168void QEMU_ERROR("unsupported atomic") atomic16_set(Int128 *ptr, Int128 val);
 169# define HAVE_ATOMIC128 0
 170#endif /* Some definition for HAVE_ATOMIC128 */
 171
 172#endif /* QEMU_ATOMIC128_H */
 173