1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62#ifndef CPU_LDST_H
63#define CPU_LDST_H
64
65#include "exec/memopidx.h"
66#include "qemu/int128.h"
67
68#if defined(CONFIG_USER_ONLY)
69
70
71
72#if TARGET_VIRT_ADDR_SPACE_BITS <= 32
73typedef uint32_t abi_ptr;
74#define TARGET_ABI_FMT_ptr "%x"
75#else
76typedef uint64_t abi_ptr;
77#define TARGET_ABI_FMT_ptr "%"PRIx64
78#endif
79
80#ifndef TARGET_TAGGED_ADDRESSES
81static inline abi_ptr cpu_untagged_addr(CPUState *cs, abi_ptr x)
82{
83 return x;
84}
85#endif
86
87
88static inline void *g2h_untagged(abi_ptr x)
89{
90 return (void *)((uintptr_t)(x) + guest_base);
91}
92
93static inline void *g2h(CPUState *cs, abi_ptr x)
94{
95 return g2h_untagged(cpu_untagged_addr(cs, x));
96}
97
98static inline bool guest_addr_valid_untagged(abi_ulong x)
99{
100 return x <= GUEST_ADDR_MAX;
101}
102
103static inline bool guest_range_valid_untagged(abi_ulong start, abi_ulong len)
104{
105 return len - 1 <= GUEST_ADDR_MAX && start <= GUEST_ADDR_MAX - len + 1;
106}
107
108#define h2g_valid(x) \
109 (HOST_LONG_BITS <= TARGET_VIRT_ADDR_SPACE_BITS || \
110 (uintptr_t)(x) - guest_base <= GUEST_ADDR_MAX)
111
112#define h2g_nocheck(x) ({ \
113 uintptr_t __ret = (uintptr_t)(x) - guest_base; \
114 (abi_ptr)__ret; \
115})
116
117#define h2g(x) ({ \
118 \
119 assert(h2g_valid(x)); \
120 h2g_nocheck(x); \
121})
122#else
123typedef target_ulong abi_ptr;
124#define TARGET_ABI_FMT_ptr TARGET_ABI_FMT_lx
125#endif
126
127uint32_t cpu_ldub_data(CPUArchState *env, abi_ptr ptr);
128int cpu_ldsb_data(CPUArchState *env, abi_ptr ptr);
129uint32_t cpu_lduw_be_data(CPUArchState *env, abi_ptr ptr);
130int cpu_ldsw_be_data(CPUArchState *env, abi_ptr ptr);
131uint32_t cpu_ldl_be_data(CPUArchState *env, abi_ptr ptr);
132uint64_t cpu_ldq_be_data(CPUArchState *env, abi_ptr ptr);
133uint32_t cpu_lduw_le_data(CPUArchState *env, abi_ptr ptr);
134int cpu_ldsw_le_data(CPUArchState *env, abi_ptr ptr);
135uint32_t cpu_ldl_le_data(CPUArchState *env, abi_ptr ptr);
136uint64_t cpu_ldq_le_data(CPUArchState *env, abi_ptr ptr);
137
138uint32_t cpu_ldub_data_ra(CPUArchState *env, abi_ptr ptr, uintptr_t ra);
139int cpu_ldsb_data_ra(CPUArchState *env, abi_ptr ptr, uintptr_t ra);
140uint32_t cpu_lduw_be_data_ra(CPUArchState *env, abi_ptr ptr, uintptr_t ra);
141int cpu_ldsw_be_data_ra(CPUArchState *env, abi_ptr ptr, uintptr_t ra);
142uint32_t cpu_ldl_be_data_ra(CPUArchState *env, abi_ptr ptr, uintptr_t ra);
143uint64_t cpu_ldq_be_data_ra(CPUArchState *env, abi_ptr ptr, uintptr_t ra);
144uint32_t cpu_lduw_le_data_ra(CPUArchState *env, abi_ptr ptr, uintptr_t ra);
145int cpu_ldsw_le_data_ra(CPUArchState *env, abi_ptr ptr, uintptr_t ra);
146uint32_t cpu_ldl_le_data_ra(CPUArchState *env, abi_ptr ptr, uintptr_t ra);
147uint64_t cpu_ldq_le_data_ra(CPUArchState *env, abi_ptr ptr, uintptr_t ra);
148
149void cpu_stb_data(CPUArchState *env, abi_ptr ptr, uint32_t val);
150void cpu_stw_be_data(CPUArchState *env, abi_ptr ptr, uint32_t val);
151void cpu_stl_be_data(CPUArchState *env, abi_ptr ptr, uint32_t val);
152void cpu_stq_be_data(CPUArchState *env, abi_ptr ptr, uint64_t val);
153void cpu_stw_le_data(CPUArchState *env, abi_ptr ptr, uint32_t val);
154void cpu_stl_le_data(CPUArchState *env, abi_ptr ptr, uint32_t val);
155void cpu_stq_le_data(CPUArchState *env, abi_ptr ptr, uint64_t val);
156
157void cpu_stb_data_ra(CPUArchState *env, abi_ptr ptr,
158 uint32_t val, uintptr_t ra);
159void cpu_stw_be_data_ra(CPUArchState *env, abi_ptr ptr,
160 uint32_t val, uintptr_t ra);
161void cpu_stl_be_data_ra(CPUArchState *env, abi_ptr ptr,
162 uint32_t val, uintptr_t ra);
163void cpu_stq_be_data_ra(CPUArchState *env, abi_ptr ptr,
164 uint64_t val, uintptr_t ra);
165void cpu_stw_le_data_ra(CPUArchState *env, abi_ptr ptr,
166 uint32_t val, uintptr_t ra);
167void cpu_stl_le_data_ra(CPUArchState *env, abi_ptr ptr,
168 uint32_t val, uintptr_t ra);
169void cpu_stq_le_data_ra(CPUArchState *env, abi_ptr ptr,
170 uint64_t val, uintptr_t ra);
171
172uint32_t cpu_ldub_mmuidx_ra(CPUArchState *env, abi_ptr ptr,
173 int mmu_idx, uintptr_t ra);
174int cpu_ldsb_mmuidx_ra(CPUArchState *env, abi_ptr ptr,
175 int mmu_idx, uintptr_t ra);
176uint32_t cpu_lduw_be_mmuidx_ra(CPUArchState *env, abi_ptr ptr,
177 int mmu_idx, uintptr_t ra);
178int cpu_ldsw_be_mmuidx_ra(CPUArchState *env, abi_ptr ptr,
179 int mmu_idx, uintptr_t ra);
180uint32_t cpu_ldl_be_mmuidx_ra(CPUArchState *env, abi_ptr ptr,
181 int mmu_idx, uintptr_t ra);
182uint64_t cpu_ldq_be_mmuidx_ra(CPUArchState *env, abi_ptr ptr,
183 int mmu_idx, uintptr_t ra);
184uint32_t cpu_lduw_le_mmuidx_ra(CPUArchState *env, abi_ptr ptr,
185 int mmu_idx, uintptr_t ra);
186int cpu_ldsw_le_mmuidx_ra(CPUArchState *env, abi_ptr ptr,
187 int mmu_idx, uintptr_t ra);
188uint32_t cpu_ldl_le_mmuidx_ra(CPUArchState *env, abi_ptr ptr,
189 int mmu_idx, uintptr_t ra);
190uint64_t cpu_ldq_le_mmuidx_ra(CPUArchState *env, abi_ptr ptr,
191 int mmu_idx, uintptr_t ra);
192
193void cpu_stb_mmuidx_ra(CPUArchState *env, abi_ptr ptr, uint32_t val,
194 int mmu_idx, uintptr_t ra);
195void cpu_stw_be_mmuidx_ra(CPUArchState *env, abi_ptr ptr, uint32_t val,
196 int mmu_idx, uintptr_t ra);
197void cpu_stl_be_mmuidx_ra(CPUArchState *env, abi_ptr ptr, uint32_t val,
198 int mmu_idx, uintptr_t ra);
199void cpu_stq_be_mmuidx_ra(CPUArchState *env, abi_ptr ptr, uint64_t val,
200 int mmu_idx, uintptr_t ra);
201void cpu_stw_le_mmuidx_ra(CPUArchState *env, abi_ptr ptr, uint32_t val,
202 int mmu_idx, uintptr_t ra);
203void cpu_stl_le_mmuidx_ra(CPUArchState *env, abi_ptr ptr, uint32_t val,
204 int mmu_idx, uintptr_t ra);
205void cpu_stq_le_mmuidx_ra(CPUArchState *env, abi_ptr ptr, uint64_t val,
206 int mmu_idx, uintptr_t ra);
207
208uint8_t cpu_ldb_mmu(CPUArchState *env, abi_ptr ptr, MemOpIdx oi, uintptr_t ra);
209uint16_t cpu_ldw_be_mmu(CPUArchState *env, abi_ptr ptr,
210 MemOpIdx oi, uintptr_t ra);
211uint32_t cpu_ldl_be_mmu(CPUArchState *env, abi_ptr ptr,
212 MemOpIdx oi, uintptr_t ra);
213uint64_t cpu_ldq_be_mmu(CPUArchState *env, abi_ptr ptr,
214 MemOpIdx oi, uintptr_t ra);
215uint16_t cpu_ldw_le_mmu(CPUArchState *env, abi_ptr ptr,
216 MemOpIdx oi, uintptr_t ra);
217uint32_t cpu_ldl_le_mmu(CPUArchState *env, abi_ptr ptr,
218 MemOpIdx oi, uintptr_t ra);
219uint64_t cpu_ldq_le_mmu(CPUArchState *env, abi_ptr ptr,
220 MemOpIdx oi, uintptr_t ra);
221
222void cpu_stb_mmu(CPUArchState *env, abi_ptr ptr, uint8_t val,
223 MemOpIdx oi, uintptr_t ra);
224void cpu_stw_be_mmu(CPUArchState *env, abi_ptr ptr, uint16_t val,
225 MemOpIdx oi, uintptr_t ra);
226void cpu_stl_be_mmu(CPUArchState *env, abi_ptr ptr, uint32_t val,
227 MemOpIdx oi, uintptr_t ra);
228void cpu_stq_be_mmu(CPUArchState *env, abi_ptr ptr, uint64_t val,
229 MemOpIdx oi, uintptr_t ra);
230void cpu_stw_le_mmu(CPUArchState *env, abi_ptr ptr, uint16_t val,
231 MemOpIdx oi, uintptr_t ra);
232void cpu_stl_le_mmu(CPUArchState *env, abi_ptr ptr, uint32_t val,
233 MemOpIdx oi, uintptr_t ra);
234void cpu_stq_le_mmu(CPUArchState *env, abi_ptr ptr, uint64_t val,
235 MemOpIdx oi, uintptr_t ra);
236
237uint32_t cpu_atomic_cmpxchgb_mmu(CPUArchState *env, target_ulong addr,
238 uint32_t cmpv, uint32_t newv,
239 MemOpIdx oi, uintptr_t retaddr);
240uint32_t cpu_atomic_cmpxchgw_le_mmu(CPUArchState *env, target_ulong addr,
241 uint32_t cmpv, uint32_t newv,
242 MemOpIdx oi, uintptr_t retaddr);
243uint32_t cpu_atomic_cmpxchgl_le_mmu(CPUArchState *env, target_ulong addr,
244 uint32_t cmpv, uint32_t newv,
245 MemOpIdx oi, uintptr_t retaddr);
246uint64_t cpu_atomic_cmpxchgq_le_mmu(CPUArchState *env, target_ulong addr,
247 uint64_t cmpv, uint64_t newv,
248 MemOpIdx oi, uintptr_t retaddr);
249uint32_t cpu_atomic_cmpxchgw_be_mmu(CPUArchState *env, target_ulong addr,
250 uint32_t cmpv, uint32_t newv,
251 MemOpIdx oi, uintptr_t retaddr);
252uint32_t cpu_atomic_cmpxchgl_be_mmu(CPUArchState *env, target_ulong addr,
253 uint32_t cmpv, uint32_t newv,
254 MemOpIdx oi, uintptr_t retaddr);
255uint64_t cpu_atomic_cmpxchgq_be_mmu(CPUArchState *env, target_ulong addr,
256 uint64_t cmpv, uint64_t newv,
257 MemOpIdx oi, uintptr_t retaddr);
258
259#define GEN_ATOMIC_HELPER(NAME, TYPE, SUFFIX) \
260TYPE cpu_atomic_ ## NAME ## SUFFIX ## _mmu \
261 (CPUArchState *env, target_ulong addr, TYPE val, \
262 MemOpIdx oi, uintptr_t retaddr);
263
264#ifdef CONFIG_ATOMIC64
265#define GEN_ATOMIC_HELPER_ALL(NAME) \
266 GEN_ATOMIC_HELPER(NAME, uint32_t, b) \
267 GEN_ATOMIC_HELPER(NAME, uint32_t, w_le) \
268 GEN_ATOMIC_HELPER(NAME, uint32_t, w_be) \
269 GEN_ATOMIC_HELPER(NAME, uint32_t, l_le) \
270 GEN_ATOMIC_HELPER(NAME, uint32_t, l_be) \
271 GEN_ATOMIC_HELPER(NAME, uint64_t, q_le) \
272 GEN_ATOMIC_HELPER(NAME, uint64_t, q_be)
273#else
274#define GEN_ATOMIC_HELPER_ALL(NAME) \
275 GEN_ATOMIC_HELPER(NAME, uint32_t, b) \
276 GEN_ATOMIC_HELPER(NAME, uint32_t, w_le) \
277 GEN_ATOMIC_HELPER(NAME, uint32_t, w_be) \
278 GEN_ATOMIC_HELPER(NAME, uint32_t, l_le) \
279 GEN_ATOMIC_HELPER(NAME, uint32_t, l_be)
280#endif
281
282GEN_ATOMIC_HELPER_ALL(fetch_add)
283GEN_ATOMIC_HELPER_ALL(fetch_sub)
284GEN_ATOMIC_HELPER_ALL(fetch_and)
285GEN_ATOMIC_HELPER_ALL(fetch_or)
286GEN_ATOMIC_HELPER_ALL(fetch_xor)
287GEN_ATOMIC_HELPER_ALL(fetch_smin)
288GEN_ATOMIC_HELPER_ALL(fetch_umin)
289GEN_ATOMIC_HELPER_ALL(fetch_smax)
290GEN_ATOMIC_HELPER_ALL(fetch_umax)
291
292GEN_ATOMIC_HELPER_ALL(add_fetch)
293GEN_ATOMIC_HELPER_ALL(sub_fetch)
294GEN_ATOMIC_HELPER_ALL(and_fetch)
295GEN_ATOMIC_HELPER_ALL(or_fetch)
296GEN_ATOMIC_HELPER_ALL(xor_fetch)
297GEN_ATOMIC_HELPER_ALL(smin_fetch)
298GEN_ATOMIC_HELPER_ALL(umin_fetch)
299GEN_ATOMIC_HELPER_ALL(smax_fetch)
300GEN_ATOMIC_HELPER_ALL(umax_fetch)
301
302GEN_ATOMIC_HELPER_ALL(xchg)
303
304#undef GEN_ATOMIC_HELPER_ALL
305#undef GEN_ATOMIC_HELPER
306
307Int128 cpu_atomic_cmpxchgo_le_mmu(CPUArchState *env, target_ulong addr,
308 Int128 cmpv, Int128 newv,
309 MemOpIdx oi, uintptr_t retaddr);
310Int128 cpu_atomic_cmpxchgo_be_mmu(CPUArchState *env, target_ulong addr,
311 Int128 cmpv, Int128 newv,
312 MemOpIdx oi, uintptr_t retaddr);
313
314Int128 cpu_atomic_ldo_le_mmu(CPUArchState *env, target_ulong addr,
315 MemOpIdx oi, uintptr_t retaddr);
316Int128 cpu_atomic_ldo_be_mmu(CPUArchState *env, target_ulong addr,
317 MemOpIdx oi, uintptr_t retaddr);
318void cpu_atomic_sto_le_mmu(CPUArchState *env, target_ulong addr, Int128 val,
319 MemOpIdx oi, uintptr_t retaddr);
320void cpu_atomic_sto_be_mmu(CPUArchState *env, target_ulong addr, Int128 val,
321 MemOpIdx oi, uintptr_t retaddr);
322
323#if defined(CONFIG_USER_ONLY)
324
325extern __thread uintptr_t helper_retaddr;
326
327static inline void set_helper_retaddr(uintptr_t ra)
328{
329 helper_retaddr = ra;
330
331
332
333
334 signal_barrier();
335}
336
337static inline void clear_helper_retaddr(void)
338{
339
340
341
342
343 signal_barrier();
344 helper_retaddr = 0;
345}
346
347#else
348
349
350#include "tcg/tcg.h"
351
352static inline target_ulong tlb_addr_write(const CPUTLBEntry *entry)
353{
354#if TCG_OVERSIZED_GUEST
355 return entry->addr_write;
356#else
357 return qatomic_read(&entry->addr_write);
358#endif
359}
360
361
362static inline uintptr_t tlb_index(CPUArchState *env, uintptr_t mmu_idx,
363 target_ulong addr)
364{
365 uintptr_t size_mask = env_tlb(env)->f[mmu_idx].mask >> CPU_TLB_ENTRY_BITS;
366
367 return (addr >> TARGET_PAGE_BITS) & size_mask;
368}
369
370
371static inline CPUTLBEntry *tlb_entry(CPUArchState *env, uintptr_t mmu_idx,
372 target_ulong addr)
373{
374 return &env_tlb(env)->f[mmu_idx].table[tlb_index(env, mmu_idx, addr)];
375}
376
377#endif
378
379#ifdef TARGET_WORDS_BIGENDIAN
380# define cpu_lduw_data cpu_lduw_be_data
381# define cpu_ldsw_data cpu_ldsw_be_data
382# define cpu_ldl_data cpu_ldl_be_data
383# define cpu_ldq_data cpu_ldq_be_data
384# define cpu_lduw_data_ra cpu_lduw_be_data_ra
385# define cpu_ldsw_data_ra cpu_ldsw_be_data_ra
386# define cpu_ldl_data_ra cpu_ldl_be_data_ra
387# define cpu_ldq_data_ra cpu_ldq_be_data_ra
388# define cpu_lduw_mmuidx_ra cpu_lduw_be_mmuidx_ra
389# define cpu_ldsw_mmuidx_ra cpu_ldsw_be_mmuidx_ra
390# define cpu_ldl_mmuidx_ra cpu_ldl_be_mmuidx_ra
391# define cpu_ldq_mmuidx_ra cpu_ldq_be_mmuidx_ra
392# define cpu_ldw_mmu cpu_ldw_be_mmu
393# define cpu_ldl_mmu cpu_ldl_be_mmu
394# define cpu_ldq_mmu cpu_ldq_be_mmu
395# define cpu_stw_data cpu_stw_be_data
396# define cpu_stl_data cpu_stl_be_data
397# define cpu_stq_data cpu_stq_be_data
398# define cpu_stw_data_ra cpu_stw_be_data_ra
399# define cpu_stl_data_ra cpu_stl_be_data_ra
400# define cpu_stq_data_ra cpu_stq_be_data_ra
401# define cpu_stw_mmuidx_ra cpu_stw_be_mmuidx_ra
402# define cpu_stl_mmuidx_ra cpu_stl_be_mmuidx_ra
403# define cpu_stq_mmuidx_ra cpu_stq_be_mmuidx_ra
404# define cpu_stw_mmu cpu_stw_be_mmu
405# define cpu_stl_mmu cpu_stl_be_mmu
406# define cpu_stq_mmu cpu_stq_be_mmu
407#else
408# define cpu_lduw_data cpu_lduw_le_data
409# define cpu_ldsw_data cpu_ldsw_le_data
410# define cpu_ldl_data cpu_ldl_le_data
411# define cpu_ldq_data cpu_ldq_le_data
412# define cpu_lduw_data_ra cpu_lduw_le_data_ra
413# define cpu_ldsw_data_ra cpu_ldsw_le_data_ra
414# define cpu_ldl_data_ra cpu_ldl_le_data_ra
415# define cpu_ldq_data_ra cpu_ldq_le_data_ra
416# define cpu_lduw_mmuidx_ra cpu_lduw_le_mmuidx_ra
417# define cpu_ldsw_mmuidx_ra cpu_ldsw_le_mmuidx_ra
418# define cpu_ldl_mmuidx_ra cpu_ldl_le_mmuidx_ra
419# define cpu_ldq_mmuidx_ra cpu_ldq_le_mmuidx_ra
420# define cpu_ldw_mmu cpu_ldw_le_mmu
421# define cpu_ldl_mmu cpu_ldl_le_mmu
422# define cpu_ldq_mmu cpu_ldq_le_mmu
423# define cpu_stw_data cpu_stw_le_data
424# define cpu_stl_data cpu_stl_le_data
425# define cpu_stq_data cpu_stq_le_data
426# define cpu_stw_data_ra cpu_stw_le_data_ra
427# define cpu_stl_data_ra cpu_stl_le_data_ra
428# define cpu_stq_data_ra cpu_stq_le_data_ra
429# define cpu_stw_mmuidx_ra cpu_stw_le_mmuidx_ra
430# define cpu_stl_mmuidx_ra cpu_stl_le_mmuidx_ra
431# define cpu_stq_mmuidx_ra cpu_stq_le_mmuidx_ra
432# define cpu_stw_mmu cpu_stw_le_mmu
433# define cpu_stl_mmu cpu_stl_le_mmu
434# define cpu_stq_mmu cpu_stq_le_mmu
435#endif
436
437uint32_t cpu_ldub_code(CPUArchState *env, abi_ptr addr);
438uint32_t cpu_lduw_code(CPUArchState *env, abi_ptr addr);
439uint32_t cpu_ldl_code(CPUArchState *env, abi_ptr addr);
440uint64_t cpu_ldq_code(CPUArchState *env, abi_ptr addr);
441
442static inline int cpu_ldsb_code(CPUArchState *env, abi_ptr addr)
443{
444 return (int8_t)cpu_ldub_code(env, addr);
445}
446
447static inline int cpu_ldsw_code(CPUArchState *env, abi_ptr addr)
448{
449 return (int16_t)cpu_lduw_code(env, addr);
450}
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465#ifdef CONFIG_USER_ONLY
466static inline void *tlb_vaddr_to_host(CPUArchState *env, abi_ptr addr,
467 MMUAccessType access_type, int mmu_idx)
468{
469 return g2h(env_cpu(env), addr);
470}
471#else
472void *tlb_vaddr_to_host(CPUArchState *env, abi_ptr addr,
473 MMUAccessType access_type, int mmu_idx);
474#endif
475
476#endif
477