1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24#include "qemu/timer.h"
25#include "exec/address-spaces.h"
26#include "exec/memory.h"
27#include "qemu/etrace.h"
28
29#define DATA_SIZE (1 << SHIFT)
30
31#if DATA_SIZE == 8
32#define SUFFIX q
33#define LSUFFIX q
34#define SDATA_TYPE int64_t
35#define DATA_TYPE uint64_t
36#elif DATA_SIZE == 4
37#define SUFFIX l
38#define LSUFFIX l
39#define SDATA_TYPE int32_t
40#define DATA_TYPE uint32_t
41#elif DATA_SIZE == 2
42#define SUFFIX w
43#define LSUFFIX uw
44#define SDATA_TYPE int16_t
45#define DATA_TYPE uint16_t
46#elif DATA_SIZE == 1
47#define SUFFIX b
48#define LSUFFIX ub
49#define SDATA_TYPE int8_t
50#define DATA_TYPE uint8_t
51#else
52#error unsupported data size
53#endif
54
55
56
57
58
59
60
61#if defined(SOFTMMU_CODE_ACCESS) || DATA_SIZE == 8
62# define WORD_TYPE DATA_TYPE
63# define USUFFIX SUFFIX
64#else
65# define WORD_TYPE tcg_target_ulong
66# define USUFFIX glue(u, SUFFIX)
67# define SSUFFIX glue(s, SUFFIX)
68#endif
69
70#ifdef SOFTMMU_CODE_ACCESS
71#define READ_ACCESS_TYPE MMU_INST_FETCH
72#define ADDR_READ addr_code
73#else
74#define READ_ACCESS_TYPE MMU_DATA_LOAD
75#define ADDR_READ addr_read
76#endif
77
78#if DATA_SIZE == 8
79# define BSWAP(X) bswap64(X)
80#elif DATA_SIZE == 4
81# define BSWAP(X) bswap32(X)
82#elif DATA_SIZE == 2
83# define BSWAP(X) bswap16(X)
84#else
85# define BSWAP(X) (X)
86#endif
87
88#ifdef TARGET_WORDS_BIGENDIAN
89# define TGT_BE(X) (X)
90# define TGT_LE(X) BSWAP(X)
91#else
92# define TGT_BE(X) BSWAP(X)
93# define TGT_LE(X) (X)
94#endif
95
96#if DATA_SIZE == 1
97# define helper_le_ld_name glue(glue(helper_ret_ld, USUFFIX), MMUSUFFIX)
98# define helper_be_ld_name helper_le_ld_name
99# define helper_le_lds_name glue(glue(helper_ret_ld, SSUFFIX), MMUSUFFIX)
100# define helper_be_lds_name helper_le_lds_name
101# define helper_le_st_name glue(glue(helper_ret_st, SUFFIX), MMUSUFFIX)
102# define helper_be_st_name helper_le_st_name
103#else
104# define helper_le_ld_name glue(glue(helper_le_ld, USUFFIX), MMUSUFFIX)
105# define helper_be_ld_name glue(glue(helper_be_ld, USUFFIX), MMUSUFFIX)
106# define helper_le_lds_name glue(glue(helper_le_ld, SSUFFIX), MMUSUFFIX)
107# define helper_be_lds_name glue(glue(helper_be_ld, SSUFFIX), MMUSUFFIX)
108# define helper_le_st_name glue(glue(helper_le_st, SUFFIX), MMUSUFFIX)
109# define helper_be_st_name glue(glue(helper_be_st, SUFFIX), MMUSUFFIX)
110#endif
111
112#ifdef TARGET_WORDS_BIGENDIAN
113# define helper_te_ld_name helper_be_ld_name
114# define helper_te_st_name helper_be_st_name
115#else
116# define helper_te_ld_name helper_le_ld_name
117# define helper_te_st_name helper_le_st_name
118#endif
119
120
121#define VICTIM_TLB_HIT(ty) \
122({ \
123
124
125 \
126 int vidx; \
127 CPUIOTLBEntry tmpiotlb; \
128 CPUTLBEntry tmptlb; \
129 for (vidx = CPU_VTLB_SIZE-1; vidx >= 0; --vidx) { \
130 if (env->tlb_v_table[mmu_idx][vidx].ty == (addr & TARGET_PAGE_MASK)) {\
131 \
132 tmptlb = env->tlb_table[mmu_idx][index]; \
133 env->tlb_table[mmu_idx][index] = env->tlb_v_table[mmu_idx][vidx]; \
134 env->tlb_v_table[mmu_idx][vidx] = tmptlb; \
135 tmpiotlb = env->iotlb[mmu_idx][index]; \
136 env->iotlb[mmu_idx][index] = env->iotlb_v[mmu_idx][vidx]; \
137 env->iotlb_v[mmu_idx][vidx] = tmpiotlb; \
138 break; \
139 } \
140 } \
141 \
142 vidx >= 0; \
143})
144
145#ifndef SOFTMMU_CODE_ACCESS
146static inline DATA_TYPE glue(io_read, SUFFIX)(CPUArchState *env,
147 CPUIOTLBEntry *iotlbentry,
148 target_ulong addr,
149 uintptr_t retaddr)
150{
151 uint64_t val;
152 CPUState *cpu = ENV_GET_CPU(env);
153 hwaddr physaddr = iotlbentry->addr;
154 MemoryRegion *mr = iotlb_to_region(cpu, physaddr, iotlbentry->attrs);
155
156 physaddr = (physaddr & TARGET_PAGE_MASK) + addr;
157 cpu->mem_io_pc = retaddr;
158 if (mr != &io_mem_rom && mr != &io_mem_notdirty && !cpu->can_do_io) {
159 cpu_io_recompile(cpu, retaddr);
160 }
161
162 cpu->mem_io_vaddr = addr;
163 if (mr->iommu_ops) {
164 address_space_rw(cpu->as, physaddr, iotlbentry->attrs, (void *) &val,
165 1 << SHIFT, false);
166 } else {
167 memory_region_dispatch_read(mr, physaddr, &val, 1 << SHIFT,
168 iotlbentry->attrs);
169 }
170
171 if (qemu_etrace_mask(ETRACE_F_MEM)) {
172 etrace_mem_access(&qemu_etracer, 0, 0,
173 addr, DATA_SIZE, MEM_READ, val);
174 }
175 return val;
176}
177#endif
178
179WORD_TYPE helper_le_ld_name(CPUArchState *env, target_ulong addr,
180 TCGMemOpIdx oi, uintptr_t retaddr)
181{
182 unsigned mmu_idx = get_mmuidx(oi);
183 int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
184 target_ulong tlb_addr = env->tlb_table[mmu_idx][index].ADDR_READ;
185 uintptr_t haddr;
186 DATA_TYPE res;
187
188
189 retaddr -= GETPC_ADJ;
190
191
192 if ((addr & TARGET_PAGE_MASK)
193 != (tlb_addr & (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
194 if ((addr & (DATA_SIZE - 1)) != 0
195 && (get_memop(oi) & MO_AMASK) == MO_ALIGN) {
196 cpu_unaligned_access(ENV_GET_CPU(env), addr, READ_ACCESS_TYPE,
197 mmu_idx, retaddr);
198 }
199 if (!VICTIM_TLB_HIT(ADDR_READ)) {
200 tlb_fill(ENV_GET_CPU(env), addr, READ_ACCESS_TYPE,
201 mmu_idx, retaddr);
202 }
203 tlb_addr = env->tlb_table[mmu_idx][index].ADDR_READ;
204 }
205
206
207 if (unlikely(tlb_addr & ~TARGET_PAGE_MASK)) {
208 CPUIOTLBEntry *iotlbentry;
209 if ((addr & (DATA_SIZE - 1)) != 0) {
210 goto do_unaligned_access;
211 }
212 iotlbentry = &env->iotlb[mmu_idx][index];
213
214
215
216 res = glue(io_read, SUFFIX)(env, iotlbentry, addr, retaddr);
217 res = TGT_LE(res);
218 return res;
219 }
220
221
222 if (DATA_SIZE > 1
223 && unlikely((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1
224 >= TARGET_PAGE_SIZE)) {
225 target_ulong addr1, addr2;
226 DATA_TYPE res1, res2;
227 unsigned shift;
228 do_unaligned_access:
229 if ((get_memop(oi) & MO_AMASK) == MO_ALIGN) {
230 cpu_unaligned_access(ENV_GET_CPU(env), addr, READ_ACCESS_TYPE,
231 mmu_idx, retaddr);
232 }
233 addr1 = addr & ~(DATA_SIZE - 1);
234 addr2 = addr1 + DATA_SIZE;
235
236
237 res1 = helper_le_ld_name(env, addr1, oi, retaddr + GETPC_ADJ);
238 res2 = helper_le_ld_name(env, addr2, oi, retaddr + GETPC_ADJ);
239 shift = (addr & (DATA_SIZE - 1)) * 8;
240
241
242 res = (res1 >> shift) | (res2 << ((DATA_SIZE * 8) - shift));
243 return res;
244 }
245
246
247 if ((addr & (DATA_SIZE - 1)) != 0
248 && (get_memop(oi) & MO_AMASK) == MO_ALIGN) {
249 cpu_unaligned_access(ENV_GET_CPU(env), addr, READ_ACCESS_TYPE,
250 mmu_idx, retaddr);
251 }
252
253 haddr = addr + env->tlb_table[mmu_idx][index].addend;
254#if DATA_SIZE == 1
255 res = glue(glue(ld, LSUFFIX), _p)((uint8_t *)haddr);
256#else
257 res = glue(glue(ld, LSUFFIX), _le_p)((uint8_t *)haddr);
258#endif
259 return res;
260}
261
262#if DATA_SIZE > 1
263WORD_TYPE helper_be_ld_name(CPUArchState *env, target_ulong addr,
264 TCGMemOpIdx oi, uintptr_t retaddr)
265{
266 unsigned mmu_idx = get_mmuidx(oi);
267 int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
268 target_ulong tlb_addr = env->tlb_table[mmu_idx][index].ADDR_READ;
269 uintptr_t haddr;
270 DATA_TYPE res;
271
272
273 retaddr -= GETPC_ADJ;
274
275
276 if ((addr & TARGET_PAGE_MASK)
277 != (tlb_addr & (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
278 if ((addr & (DATA_SIZE - 1)) != 0
279 && (get_memop(oi) & MO_AMASK) == MO_ALIGN) {
280 cpu_unaligned_access(ENV_GET_CPU(env), addr, READ_ACCESS_TYPE,
281 mmu_idx, retaddr);
282 }
283 if (!VICTIM_TLB_HIT(ADDR_READ)) {
284 tlb_fill(ENV_GET_CPU(env), addr, READ_ACCESS_TYPE,
285 mmu_idx, retaddr);
286 }
287 tlb_addr = env->tlb_table[mmu_idx][index].ADDR_READ;
288 }
289
290
291 if (unlikely(tlb_addr & ~TARGET_PAGE_MASK)) {
292 CPUIOTLBEntry *iotlbentry;
293 if ((addr & (DATA_SIZE - 1)) != 0) {
294 goto do_unaligned_access;
295 }
296 iotlbentry = &env->iotlb[mmu_idx][index];
297
298
299
300 res = glue(io_read, SUFFIX)(env, iotlbentry, addr, retaddr);
301 res = TGT_BE(res);
302 return res;
303 }
304
305
306 if (DATA_SIZE > 1
307 && unlikely((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1
308 >= TARGET_PAGE_SIZE)) {
309 target_ulong addr1, addr2;
310 DATA_TYPE res1, res2;
311 unsigned shift;
312 do_unaligned_access:
313 if ((get_memop(oi) & MO_AMASK) == MO_ALIGN) {
314 cpu_unaligned_access(ENV_GET_CPU(env), addr, READ_ACCESS_TYPE,
315 mmu_idx, retaddr);
316 }
317 addr1 = addr & ~(DATA_SIZE - 1);
318 addr2 = addr1 + DATA_SIZE;
319
320
321 res1 = helper_be_ld_name(env, addr1, oi, retaddr + GETPC_ADJ);
322 res2 = helper_be_ld_name(env, addr2, oi, retaddr + GETPC_ADJ);
323 shift = (addr & (DATA_SIZE - 1)) * 8;
324
325
326 res = (res1 << shift) | (res2 >> ((DATA_SIZE * 8) - shift));
327 return res;
328 }
329
330
331 if ((addr & (DATA_SIZE - 1)) != 0
332 && (get_memop(oi) & MO_AMASK) == MO_ALIGN) {
333 cpu_unaligned_access(ENV_GET_CPU(env), addr, READ_ACCESS_TYPE,
334 mmu_idx, retaddr);
335 }
336
337 haddr = addr + env->tlb_table[mmu_idx][index].addend;
338 res = glue(glue(ld, LSUFFIX), _be_p)((uint8_t *)haddr);
339 return res;
340}
341#endif
342
343#ifndef SOFTMMU_CODE_ACCESS
344
345
346
347#if DATA_SIZE * 8 < TCG_TARGET_REG_BITS
348WORD_TYPE helper_le_lds_name(CPUArchState *env, target_ulong addr,
349 TCGMemOpIdx oi, uintptr_t retaddr)
350{
351 return (SDATA_TYPE)helper_le_ld_name(env, addr, oi, retaddr);
352}
353
354# if DATA_SIZE > 1
355WORD_TYPE helper_be_lds_name(CPUArchState *env, target_ulong addr,
356 TCGMemOpIdx oi, uintptr_t retaddr)
357{
358 return (SDATA_TYPE)helper_be_ld_name(env, addr, oi, retaddr);
359}
360# endif
361#endif
362
363static inline void glue(io_write, SUFFIX)(CPUArchState *env,
364 CPUIOTLBEntry *iotlbentry,
365 DATA_TYPE val,
366 target_ulong addr,
367 uintptr_t retaddr)
368{
369 CPUState *cpu = ENV_GET_CPU(env);
370 hwaddr physaddr = iotlbentry->addr;
371 MemoryRegion *mr = iotlb_to_region(cpu, physaddr, iotlbentry->attrs);
372
373 physaddr = (physaddr & TARGET_PAGE_MASK) + addr;
374 if (mr != &io_mem_rom && mr != &io_mem_notdirty && !cpu->can_do_io) {
375 cpu_io_recompile(cpu, retaddr);
376 }
377
378 if (qemu_etrace_mask(ETRACE_F_MEM)) {
379 etrace_mem_access(&qemu_etracer, 0, 0,
380 addr, DATA_SIZE, MEM_WRITE, val);
381 }
382
383 cpu->mem_io_vaddr = addr;
384 cpu->mem_io_pc = retaddr;
385 if (mr->iommu_ops) {
386 address_space_rw(cpu->as, physaddr, iotlbentry->attrs, (void *) &val,
387 1 << SHIFT, true);
388 } else {
389 memory_region_dispatch_write(mr, physaddr, val, 1 << SHIFT,
390 iotlbentry->attrs);
391 }
392}
393
394void helper_le_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,
395 TCGMemOpIdx oi, uintptr_t retaddr)
396{
397 unsigned mmu_idx = get_mmuidx(oi);
398 int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
399 target_ulong tlb_addr = env->tlb_table[mmu_idx][index].addr_write;
400 uintptr_t haddr;
401
402
403 retaddr -= GETPC_ADJ;
404
405
406 if ((addr & TARGET_PAGE_MASK)
407 != (tlb_addr & (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
408 if ((addr & (DATA_SIZE - 1)) != 0
409 && (get_memop(oi) & MO_AMASK) == MO_ALIGN) {
410 cpu_unaligned_access(ENV_GET_CPU(env), addr, MMU_DATA_STORE,
411 mmu_idx, retaddr);
412 }
413 if (!VICTIM_TLB_HIT(addr_write)) {
414 tlb_fill(ENV_GET_CPU(env), addr, MMU_DATA_STORE, mmu_idx, retaddr);
415 }
416 tlb_addr = env->tlb_table[mmu_idx][index].addr_write;
417 }
418
419
420 if (unlikely(tlb_addr & ~TARGET_PAGE_MASK)) {
421 CPUIOTLBEntry *iotlbentry;
422 if ((addr & (DATA_SIZE - 1)) != 0) {
423 goto do_unaligned_access;
424 }
425 iotlbentry = &env->iotlb[mmu_idx][index];
426
427
428
429 val = TGT_LE(val);
430 glue(io_write, SUFFIX)(env, iotlbentry, val, addr, retaddr);
431 return;
432 }
433
434
435 if (DATA_SIZE > 1
436 && unlikely((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1
437 >= TARGET_PAGE_SIZE)) {
438 int i;
439 do_unaligned_access:
440 if ((get_memop(oi) & MO_AMASK) == MO_ALIGN) {
441 cpu_unaligned_access(ENV_GET_CPU(env), addr, MMU_DATA_STORE,
442 mmu_idx, retaddr);
443 }
444
445
446
447 for (i = DATA_SIZE - 1; i >= 0; i--) {
448
449 uint8_t val8 = val >> (i * 8);
450
451
452 glue(helper_ret_stb, MMUSUFFIX)(env, addr + i, val8,
453 oi, retaddr + GETPC_ADJ);
454 }
455 return;
456 }
457
458
459 if ((addr & (DATA_SIZE - 1)) != 0
460 && (get_memop(oi) & MO_AMASK) == MO_ALIGN) {
461 cpu_unaligned_access(ENV_GET_CPU(env), addr, MMU_DATA_STORE,
462 mmu_idx, retaddr);
463 }
464
465 haddr = addr + env->tlb_table[mmu_idx][index].addend;
466#if DATA_SIZE == 1
467 glue(glue(st, SUFFIX), _p)((uint8_t *)haddr, val);
468#else
469 glue(glue(st, SUFFIX), _le_p)((uint8_t *)haddr, val);
470#endif
471}
472
473#if DATA_SIZE > 1
474void helper_be_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,
475 TCGMemOpIdx oi, uintptr_t retaddr)
476{
477 unsigned mmu_idx = get_mmuidx(oi);
478 int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
479 target_ulong tlb_addr = env->tlb_table[mmu_idx][index].addr_write;
480 uintptr_t haddr;
481
482
483 retaddr -= GETPC_ADJ;
484
485
486 if ((addr & TARGET_PAGE_MASK)
487 != (tlb_addr & (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
488 if ((addr & (DATA_SIZE - 1)) != 0
489 && (get_memop(oi) & MO_AMASK) == MO_ALIGN) {
490 cpu_unaligned_access(ENV_GET_CPU(env), addr, MMU_DATA_STORE,
491 mmu_idx, retaddr);
492 }
493 if (!VICTIM_TLB_HIT(addr_write)) {
494 tlb_fill(ENV_GET_CPU(env), addr, MMU_DATA_STORE, mmu_idx, retaddr);
495 }
496 tlb_addr = env->tlb_table[mmu_idx][index].addr_write;
497 }
498
499
500 if (unlikely(tlb_addr & ~TARGET_PAGE_MASK)) {
501 CPUIOTLBEntry *iotlbentry;
502 if ((addr & (DATA_SIZE - 1)) != 0) {
503 goto do_unaligned_access;
504 }
505 iotlbentry = &env->iotlb[mmu_idx][index];
506
507
508
509 val = TGT_BE(val);
510 glue(io_write, SUFFIX)(env, iotlbentry, val, addr, retaddr);
511 return;
512 }
513
514
515 if (DATA_SIZE > 1
516 && unlikely((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1
517 >= TARGET_PAGE_SIZE)) {
518 int i;
519 do_unaligned_access:
520 if ((get_memop(oi) & MO_AMASK) == MO_ALIGN) {
521 cpu_unaligned_access(ENV_GET_CPU(env), addr, MMU_DATA_STORE,
522 mmu_idx, retaddr);
523 }
524
525
526
527 for (i = DATA_SIZE - 1; i >= 0; i--) {
528
529 uint8_t val8 = val >> (((DATA_SIZE - 1) * 8) - (i * 8));
530
531
532 glue(helper_ret_stb, MMUSUFFIX)(env, addr + i, val8,
533 oi, retaddr + GETPC_ADJ);
534 }
535 return;
536 }
537
538
539 if ((addr & (DATA_SIZE - 1)) != 0
540 && (get_memop(oi) & MO_AMASK) == MO_ALIGN) {
541 cpu_unaligned_access(ENV_GET_CPU(env), addr, MMU_DATA_STORE,
542 mmu_idx, retaddr);
543 }
544
545 haddr = addr + env->tlb_table[mmu_idx][index].addend;
546 glue(glue(st, SUFFIX), _be_p)((uint8_t *)haddr, val);
547}
548#endif
549
550#if DATA_SIZE == 1
551
552
553
554
555
556
557void probe_write(CPUArchState *env, target_ulong addr, int mmu_idx,
558 uintptr_t retaddr)
559{
560 int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
561 target_ulong tlb_addr = env->tlb_table[mmu_idx][index].addr_write;
562
563 if ((addr & TARGET_PAGE_MASK)
564 != (tlb_addr & (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
565
566 if (!VICTIM_TLB_HIT(addr_write)) {
567 tlb_fill(ENV_GET_CPU(env), addr, MMU_DATA_STORE, mmu_idx, retaddr);
568 }
569 }
570}
571#endif
572#endif
573
574#undef READ_ACCESS_TYPE
575#undef SHIFT
576#undef DATA_TYPE
577#undef SUFFIX
578#undef LSUFFIX
579#undef DATA_SIZE
580#undef ADDR_READ
581#undef WORD_TYPE
582#undef SDATA_TYPE
583#undef USUFFIX
584#undef SSUFFIX
585#undef BSWAP
586#undef TGT_BE
587#undef TGT_LE
588#undef CPU_BE
589#undef CPU_LE
590#undef helper_le_ld_name
591#undef helper_be_ld_name
592#undef helper_le_lds_name
593#undef helper_be_lds_name
594#undef helper_le_st_name
595#undef helper_be_st_name
596#undef helper_te_ld_name
597#undef helper_te_st_name
598