1
2
3
4
5
6
7
8
9
10
11
12
13#ifndef QEMU_ATOMIC128_H
14#define QEMU_ATOMIC128_H
15
16#include "qemu/int128.h"
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44#if defined(CONFIG_ATOMIC128)
45static inline Int128 atomic16_cmpxchg(Int128 *ptr, Int128 cmp, Int128 new)
46{
47 return qatomic_cmpxchg__nocheck(ptr, cmp, new);
48}
49# define HAVE_CMPXCHG128 1
50#elif defined(CONFIG_CMPXCHG128)
51static inline Int128 atomic16_cmpxchg(Int128 *ptr, Int128 cmp, Int128 new)
52{
53 return __sync_val_compare_and_swap_16(ptr, cmp, new);
54}
55# define HAVE_CMPXCHG128 1
56#elif defined(__aarch64__)
57
58static inline Int128 atomic16_cmpxchg(Int128 *ptr, Int128 cmp, Int128 new)
59{
60 uint64_t cmpl = int128_getlo(cmp), cmph = int128_gethi(cmp);
61 uint64_t newl = int128_getlo(new), newh = int128_gethi(new);
62 uint64_t oldl, oldh;
63 uint32_t tmp;
64
65 asm("0: ldaxp %[oldl], %[oldh], %[mem]\n\t"
66 "cmp %[oldl], %[cmpl]\n\t"
67 "ccmp %[oldh], %[cmph], #0, eq\n\t"
68 "b.ne 1f\n\t"
69 "stlxp %w[tmp], %[newl], %[newh], %[mem]\n\t"
70 "cbnz %w[tmp], 0b\n"
71 "1:"
72 : [mem] "+m"(*ptr), [tmp] "=&r"(tmp),
73 [oldl] "=&r"(oldl), [oldh] "=&r"(oldh)
74 : [cmpl] "r"(cmpl), [cmph] "r"(cmph),
75 [newl] "r"(newl), [newh] "r"(newh)
76 : "memory", "cc");
77
78 return int128_make128(oldl, oldh);
79}
80# define HAVE_CMPXCHG128 1
81#else
82
83Int128 QEMU_ERROR("unsupported atomic")
84 atomic16_cmpxchg(Int128 *ptr, Int128 cmp, Int128 new);
85# define HAVE_CMPXCHG128 0
86#endif
87
88
89#if defined(CONFIG_ATOMIC128)
90static inline Int128 atomic16_read(Int128 *ptr)
91{
92 return qatomic_read__nocheck(ptr);
93}
94
95static inline void atomic16_set(Int128 *ptr, Int128 val)
96{
97 qatomic_set__nocheck(ptr, val);
98}
99
100# define HAVE_ATOMIC128 1
101#elif !defined(CONFIG_USER_ONLY) && defined(__aarch64__)
102
103static inline Int128 atomic16_read(Int128 *ptr)
104{
105 uint64_t l, h;
106 uint32_t tmp;
107
108
109 asm("0: ldxp %[l], %[h], %[mem]\n\t"
110 "stxp %w[tmp], %[l], %[h], %[mem]\n\t"
111 "cbnz %w[tmp], 0b"
112 : [mem] "+m"(*ptr), [tmp] "=r"(tmp), [l] "=r"(l), [h] "=r"(h));
113
114 return int128_make128(l, h);
115}
116
117static inline void atomic16_set(Int128 *ptr, Int128 val)
118{
119 uint64_t l = int128_getlo(val), h = int128_gethi(val);
120 uint64_t t1, t2;
121
122
123 asm("0: ldxp %[t1], %[t2], %[mem]\n\t"
124 "stxp %w[t1], %[l], %[h], %[mem]\n\t"
125 "cbnz %w[t1], 0b"
126 : [mem] "+m"(*ptr), [t1] "=&r"(t1), [t2] "=&r"(t2)
127 : [l] "r"(l), [h] "r"(h));
128}
129
130# define HAVE_ATOMIC128 1
131#elif !defined(CONFIG_USER_ONLY) && HAVE_CMPXCHG128
132static inline Int128 atomic16_read(Int128 *ptr)
133{
134
135 return atomic16_cmpxchg(ptr, 0, 0);
136}
137
138static inline void atomic16_set(Int128 *ptr, Int128 val)
139{
140 Int128 old = *ptr, cmp;
141 do {
142 cmp = old;
143 old = atomic16_cmpxchg(ptr, cmp, val);
144 } while (old != cmp);
145}
146
147# define HAVE_ATOMIC128 1
148#else
149
150Int128 QEMU_ERROR("unsupported atomic") atomic16_read(Int128 *ptr);
151void QEMU_ERROR("unsupported atomic") atomic16_set(Int128 *ptr, Int128 val);
152# define HAVE_ATOMIC128 0
153#endif
154
155#endif
156