1
2
3
4
5
6
7
8
9
10
11
12
13#ifndef QEMU_ATOMIC128_H
14#define QEMU_ATOMIC128_H
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42#if defined(CONFIG_ATOMIC128)
43static inline Int128 atomic16_cmpxchg(Int128 *ptr, Int128 cmp, Int128 new)
44{
45 return atomic_cmpxchg__nocheck(ptr, cmp, new);
46}
47# define HAVE_CMPXCHG128 1
48#elif defined(CONFIG_CMPXCHG128)
49static inline Int128 atomic16_cmpxchg(Int128 *ptr, Int128 cmp, Int128 new)
50{
51 return __sync_val_compare_and_swap_16(ptr, cmp, new);
52}
53# define HAVE_CMPXCHG128 1
54#elif defined(__aarch64__)
55
56static inline Int128 atomic16_cmpxchg(Int128 *ptr, Int128 cmp, Int128 new)
57{
58 uint64_t cmpl = int128_getlo(cmp), cmph = int128_gethi(cmp);
59 uint64_t newl = int128_getlo(new), newh = int128_gethi(new);
60 uint64_t oldl, oldh;
61 uint32_t tmp;
62
63 asm("0: ldaxp %[oldl], %[oldh], %[mem]\n\t"
64 "cmp %[oldl], %[cmpl]\n\t"
65 "ccmp %[oldh], %[cmph], #0, eq\n\t"
66 "b.ne 1f\n\t"
67 "stlxp %w[tmp], %[newl], %[newh], %[mem]\n\t"
68 "cbnz %w[tmp], 0b\n"
69 "1:"
70 : [mem] "+m"(*ptr), [tmp] "=&r"(tmp),
71 [oldl] "=&r"(oldl), [oldh] "=&r"(oldh)
72 : [cmpl] "r"(cmpl), [cmph] "r"(cmph),
73 [newl] "r"(newl), [newh] "r"(newh)
74 : "memory", "cc");
75
76 return int128_make128(oldl, oldh);
77}
78# define HAVE_CMPXCHG128 1
79#else
80
81Int128 QEMU_ERROR("unsupported atomic")
82 atomic16_cmpxchg(Int128 *ptr, Int128 cmp, Int128 new);
83# define HAVE_CMPXCHG128 0
84#endif
85
86
87#if defined(CONFIG_ATOMIC128)
88static inline Int128 atomic16_read(Int128 *ptr)
89{
90 return atomic_read__nocheck(ptr);
91}
92
93static inline void atomic16_set(Int128 *ptr, Int128 val)
94{
95 atomic_set__nocheck(ptr, val);
96}
97
98# define HAVE_ATOMIC128 1
99#elif !defined(CONFIG_USER_ONLY) && defined(__aarch64__)
100
101static inline Int128 atomic16_read(Int128 *ptr)
102{
103 uint64_t l, h;
104 uint32_t tmp;
105
106
107 asm("0: ldxp %[l], %[h], %[mem]\n\t"
108 "stxp %w[tmp], %[l], %[h], %[mem]\n\t"
109 "cbnz %w[tmp], 0b"
110 : [mem] "+m"(*ptr), [tmp] "=r"(tmp), [l] "=r"(l), [h] "=r"(h));
111
112 return int128_make128(l, h);
113}
114
115static inline void atomic16_set(Int128 *ptr, Int128 val)
116{
117 uint64_t l = int128_getlo(val), h = int128_gethi(val);
118 uint64_t t1, t2;
119
120
121 asm("0: ldxp %[t1], %[t2], %[mem]\n\t"
122 "stxp %w[t1], %[l], %[h], %[mem]\n\t"
123 "cbnz %w[t1], 0b"
124 : [mem] "+m"(*ptr), [t1] "=&r"(t1), [t2] "=&r"(t2)
125 : [l] "r"(l), [h] "r"(h));
126}
127
128# define HAVE_ATOMIC128 1
129#elif !defined(CONFIG_USER_ONLY) && HAVE_CMPXCHG128
130static inline Int128 atomic16_read(Int128 *ptr)
131{
132
133 return atomic16_cmpxchg(ptr, 0, 0);
134}
135
136static inline void atomic16_set(Int128 *ptr, Int128 val)
137{
138 Int128 old = *ptr, cmp;
139 do {
140 cmp = old;
141 old = atomic16_cmpxchg(ptr, cmp, val);
142 } while (old != cmp);
143}
144
145# define HAVE_ATOMIC128 1
146#else
147
148Int128 QEMU_ERROR("unsupported atomic") atomic16_read(Int128 *ptr);
149void QEMU_ERROR("unsupported atomic") atomic16_set(Int128 *ptr, Int128 val);
150# define HAVE_ATOMIC128 0
151#endif
152
153#endif
154