LXR qemu/accel/tcg/atomic

   1/*
   2 * Atomic helper templates
   3 * Included from tcg-runtime.c and cputlb.c.
   4 *
   5 * Copyright (c) 2016 Red Hat, Inc
   6 *
   7 * This library is free software; you can redistribute it and/or
   8 * modify it under the terms of the GNU Lesser General Public
   9 * License as published by the Free Software Foundation; either
  10 * version 2 of the License, or (at your option) any later version.
  11 *
  12 * This library is distributed in the hope that it will be useful,
  13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15 * Lesser General Public License for more details.
  16 *
  17 * You should have received a copy of the GNU Lesser General Public
  18 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  19 */
  20
  21#if DATA_SIZE == 16
  22# define SUFFIX     o
  23# define DATA_TYPE  Int128
  24# define BSWAP      bswap128
  25#elif DATA_SIZE == 8
  26# define SUFFIX     q
  27# define DATA_TYPE  uint64_t
  28# define BSWAP      bswap64
  29#elif DATA_SIZE == 4
  30# define SUFFIX     l
  31# define DATA_TYPE  uint32_t
  32# define BSWAP      bswap32
  33#elif DATA_SIZE == 2
  34# define SUFFIX     w
  35# define DATA_TYPE  uint16_t
  36# define BSWAP      bswap16
  37#elif DATA_SIZE == 1
  38# define SUFFIX     b
  39# define DATA_TYPE  uint8_t
  40# define BSWAP
  41#else
  42# error unsupported data size
  43#endif
  44
  45#if DATA_SIZE >= 4
  46# define ABI_TYPE  DATA_TYPE
  47#else
  48# define ABI_TYPE  uint32_t
  49#endif
  50
  51/* Define host-endian atomic operations.  Note that END is used within
  52   the ATOMIC_NAME macro, and redefined below.  */
  53#if DATA_SIZE == 1
  54# define END
  55#elif defined(HOST_WORDS_BIGENDIAN)
  56# define END  _be
  57#else
  58# define END  _le
  59#endif
  60
  61ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr,
  62                              ABI_TYPE cmpv, ABI_TYPE newv EXTRA_ARGS)
  63{
  64    ATOMIC_MMU_DECLS;
  65    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
  66    DATA_TYPE ret = atomic_cmpxchg__nocheck(haddr, cmpv, newv);
  67    ATOMIC_MMU_CLEANUP;
  68    return ret;
  69}
  70
  71#if DATA_SIZE >= 16
  72ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS)
  73{
  74    ATOMIC_MMU_DECLS;
  75    DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP;
  76    __atomic_load(haddr, &val, __ATOMIC_RELAXED);
  77    ATOMIC_MMU_CLEANUP;
  78    return val;
  79}
  80
  81void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr,
  82                     ABI_TYPE val EXTRA_ARGS)
  83{
  84    ATOMIC_MMU_DECLS;
  85    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
  86    __atomic_store(haddr, &val, __ATOMIC_RELAXED);
  87    ATOMIC_MMU_CLEANUP;
  88}
  89#else
  90ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr,
  91                           ABI_TYPE val EXTRA_ARGS)
  92{
  93    ATOMIC_MMU_DECLS;
  94    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
  95    DATA_TYPE ret = atomic_xchg__nocheck(haddr, val);
  96    ATOMIC_MMU_CLEANUP;
  97    return ret;
  98}
  99
 100#define GEN_ATOMIC_HELPER(X)                                        \
 101ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr,       \
 102                 ABI_TYPE val EXTRA_ARGS)                           \
 103{                                                                   \
 104    ATOMIC_MMU_DECLS;                                               \
 105    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;                           \
 106    DATA_TYPE ret = atomic_##X(haddr, val);                         \
 107    ATOMIC_MMU_CLEANUP;                                             \
 108    return ret;                                                     \
 109}
 110
 111GEN_ATOMIC_HELPER(fetch_add)
 112GEN_ATOMIC_HELPER(fetch_and)
 113GEN_ATOMIC_HELPER(fetch_or)
 114GEN_ATOMIC_HELPER(fetch_xor)
 115GEN_ATOMIC_HELPER(add_fetch)
 116GEN_ATOMIC_HELPER(and_fetch)
 117GEN_ATOMIC_HELPER(or_fetch)
 118GEN_ATOMIC_HELPER(xor_fetch)
 119
 120#undef GEN_ATOMIC_HELPER
 121#endif /* DATA SIZE >= 16 */
 122
 123#undef END
 124
 125#if DATA_SIZE > 1
 126
 127/* Define reverse-host-endian atomic operations.  Note that END is used
 128   within the ATOMIC_NAME macro.  */
 129#ifdef HOST_WORDS_BIGENDIAN
 130# define END  _le
 131#else
 132# define END  _be
 133#endif
 134
 135ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr,
 136                              ABI_TYPE cmpv, ABI_TYPE newv EXTRA_ARGS)
 137{
 138    ATOMIC_MMU_DECLS;
 139    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
 140    DATA_TYPE ret = atomic_cmpxchg__nocheck(haddr, BSWAP(cmpv), BSWAP(newv));
 141    ATOMIC_MMU_CLEANUP;
 142    return BSWAP(ret);
 143}
 144
 145#if DATA_SIZE >= 16
 146ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS)
 147{
 148    ATOMIC_MMU_DECLS;
 149    DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP;
 150    __atomic_load(haddr, &val, __ATOMIC_RELAXED);
 151    ATOMIC_MMU_CLEANUP;
 152    return BSWAP(val);
 153}
 154
 155void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr,
 156                     ABI_TYPE val EXTRA_ARGS)
 157{
 158    ATOMIC_MMU_DECLS;
 159    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
 160    val = BSWAP(val);
 161    __atomic_store(haddr, &val, __ATOMIC_RELAXED);
 162    ATOMIC_MMU_CLEANUP;
 163}
 164#else
 165ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr,
 166                           ABI_TYPE val EXTRA_ARGS)
 167{
 168    ATOMIC_MMU_DECLS;
 169    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
 170    ABI_TYPE ret = atomic_xchg__nocheck(haddr, BSWAP(val));
 171    ATOMIC_MMU_CLEANUP;
 172    return BSWAP(ret);
 173}
 174
 175#define GEN_ATOMIC_HELPER(X)                                        \
 176ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr,       \
 177                 ABI_TYPE val EXTRA_ARGS)                           \
 178{                                                                   \
 179    ATOMIC_MMU_DECLS;                                               \
 180    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;                           \
 181    DATA_TYPE ret = atomic_##X(haddr, BSWAP(val));                  \
 182    ATOMIC_MMU_CLEANUP;                                             \
 183    return BSWAP(ret);                                              \
 184}
 185
 186GEN_ATOMIC_HELPER(fetch_and)
 187GEN_ATOMIC_HELPER(fetch_or)
 188GEN_ATOMIC_HELPER(fetch_xor)
 189GEN_ATOMIC_HELPER(and_fetch)
 190GEN_ATOMIC_HELPER(or_fetch)
 191GEN_ATOMIC_HELPER(xor_fetch)
 192
 193#undef GEN_ATOMIC_HELPER
 194
 195/* Note that for addition, we need to use a separate cmpxchg loop instead
 196   of bswaps for the reverse-host-endian helpers.  */
 197ABI_TYPE ATOMIC_NAME(fetch_add)(CPUArchState *env, target_ulong addr,
 198                         ABI_TYPE val EXTRA_ARGS)
 199{
 200    ATOMIC_MMU_DECLS;
 201    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
 202    DATA_TYPE ldo, ldn, ret, sto;
 203
 204    ldo = atomic_read__nocheck(haddr);
 205    while (1) {
 206        ret = BSWAP(ldo);
 207        sto = BSWAP(ret + val);
 208        ldn = atomic_cmpxchg__nocheck(haddr, ldo, sto);
 209        if (ldn == ldo) {
 210            ATOMIC_MMU_CLEANUP;
 211            return ret;
 212        }
 213        ldo = ldn;
 214    }
 215}
 216
 217ABI_TYPE ATOMIC_NAME(add_fetch)(CPUArchState *env, target_ulong addr,
 218                         ABI_TYPE val EXTRA_ARGS)
 219{
 220    ATOMIC_MMU_DECLS;
 221    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
 222    DATA_TYPE ldo, ldn, ret, sto;
 223
 224    ldo = atomic_read__nocheck(haddr);
 225    while (1) {
 226        ret = BSWAP(ldo) + val;
 227        sto = BSWAP(ret);
 228        ldn = atomic_cmpxchg__nocheck(haddr, ldo, sto);
 229        if (ldn == ldo) {
 230            ATOMIC_MMU_CLEANUP;
 231            return ret;
 232        }
 233        ldo = ldn;
 234    }
 235}
 236#endif /* DATA_SIZE >= 16 */
 237
 238#undef END
 239#endif /* DATA_SIZE > 1 */
 240
 241#undef BSWAP
 242#undef ABI_TYPE
 243#undef DATA_TYPE
 244#undef SUFFIX
 245#undef DATA_SIZE
 246