1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25#ifndef TCG_H
26#define TCG_H
27
28#include "qemu-common.h"
29#include "cpu.h"
30#include "exec/tb-context.h"
31#include "qemu/bitops.h"
32#include "qemu/queue.h"
33#include "tcg-mo.h"
34#include "tcg-target.h"
35#include "qemu/int128.h"
36
37
38#define MAX_OP_PER_INSTR 266
39
40#if HOST_LONG_BITS == 32
41#define MAX_OPC_PARAM_PER_ARG 2
42#else
43#define MAX_OPC_PARAM_PER_ARG 1
44#endif
45#define MAX_OPC_PARAM_IARGS 6
46#define MAX_OPC_PARAM_OARGS 1
47#define MAX_OPC_PARAM_ARGS (MAX_OPC_PARAM_IARGS + MAX_OPC_PARAM_OARGS)
48
49
50
51
52#define MAX_OPC_PARAM (4 + (MAX_OPC_PARAM_PER_ARG * MAX_OPC_PARAM_ARGS))
53
54#define CPU_TEMP_BUF_NLONGS 128
55
56
57#ifndef TCG_TARGET_REG_BITS
58# if UINTPTR_MAX == UINT32_MAX
59# define TCG_TARGET_REG_BITS 32
60# elif UINTPTR_MAX == UINT64_MAX
61# define TCG_TARGET_REG_BITS 64
62# else
63# error Unknown pointer size for tcg target
64# endif
65#endif
66
67#if TCG_TARGET_REG_BITS == 32
68typedef int32_t tcg_target_long;
69typedef uint32_t tcg_target_ulong;
70#define TCG_PRIlx PRIx32
71#define TCG_PRIld PRId32
72#elif TCG_TARGET_REG_BITS == 64
73typedef int64_t tcg_target_long;
74typedef uint64_t tcg_target_ulong;
75#define TCG_PRIlx PRIx64
76#define TCG_PRIld PRId64
77#else
78#error unsupported
79#endif
80
81
82
83
84#if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
85#define TCG_OVERSIZED_GUEST 1
86#else
87#define TCG_OVERSIZED_GUEST 0
88#endif
89
90#if TCG_TARGET_NB_REGS <= 32
91typedef uint32_t TCGRegSet;
92#elif TCG_TARGET_NB_REGS <= 64
93typedef uint64_t TCGRegSet;
94#else
95#error unsupported
96#endif
97
98#if TCG_TARGET_REG_BITS == 32
99
100#define TCG_TARGET_HAS_extrl_i64_i32 0
101#define TCG_TARGET_HAS_extrh_i64_i32 0
102#define TCG_TARGET_HAS_div_i64 0
103#define TCG_TARGET_HAS_rem_i64 0
104#define TCG_TARGET_HAS_div2_i64 0
105#define TCG_TARGET_HAS_rot_i64 0
106#define TCG_TARGET_HAS_ext8s_i64 0
107#define TCG_TARGET_HAS_ext16s_i64 0
108#define TCG_TARGET_HAS_ext32s_i64 0
109#define TCG_TARGET_HAS_ext8u_i64 0
110#define TCG_TARGET_HAS_ext16u_i64 0
111#define TCG_TARGET_HAS_ext32u_i64 0
112#define TCG_TARGET_HAS_bswap16_i64 0
113#define TCG_TARGET_HAS_bswap32_i64 0
114#define TCG_TARGET_HAS_bswap64_i64 0
115#define TCG_TARGET_HAS_neg_i64 0
116#define TCG_TARGET_HAS_not_i64 0
117#define TCG_TARGET_HAS_andc_i64 0
118#define TCG_TARGET_HAS_orc_i64 0
119#define TCG_TARGET_HAS_eqv_i64 0
120#define TCG_TARGET_HAS_nand_i64 0
121#define TCG_TARGET_HAS_nor_i64 0
122#define TCG_TARGET_HAS_clz_i64 0
123#define TCG_TARGET_HAS_ctz_i64 0
124#define TCG_TARGET_HAS_ctpop_i64 0
125#define TCG_TARGET_HAS_deposit_i64 0
126#define TCG_TARGET_HAS_extract_i64 0
127#define TCG_TARGET_HAS_sextract_i64 0
128#define TCG_TARGET_HAS_movcond_i64 0
129#define TCG_TARGET_HAS_add2_i64 0
130#define TCG_TARGET_HAS_sub2_i64 0
131#define TCG_TARGET_HAS_mulu2_i64 0
132#define TCG_TARGET_HAS_muls2_i64 0
133#define TCG_TARGET_HAS_muluh_i64 0
134#define TCG_TARGET_HAS_mulsh_i64 0
135
136#define TCG_TARGET_HAS_add2_i32 1
137#define TCG_TARGET_HAS_sub2_i32 1
138#endif
139
140#ifndef TCG_TARGET_deposit_i32_valid
141#define TCG_TARGET_deposit_i32_valid(ofs, len) 1
142#endif
143#ifndef TCG_TARGET_deposit_i64_valid
144#define TCG_TARGET_deposit_i64_valid(ofs, len) 1
145#endif
146#ifndef TCG_TARGET_extract_i32_valid
147#define TCG_TARGET_extract_i32_valid(ofs, len) 1
148#endif
149#ifndef TCG_TARGET_extract_i64_valid
150#define TCG_TARGET_extract_i64_valid(ofs, len) 1
151#endif
152
153
154#if defined(TCG_TARGET_HAS_div_i32)
155#define TCG_TARGET_HAS_div2_i32 0
156#elif defined(TCG_TARGET_HAS_div2_i32)
157#define TCG_TARGET_HAS_div_i32 0
158#define TCG_TARGET_HAS_rem_i32 0
159#endif
160#if defined(TCG_TARGET_HAS_div_i64)
161#define TCG_TARGET_HAS_div2_i64 0
162#elif defined(TCG_TARGET_HAS_div2_i64)
163#define TCG_TARGET_HAS_div_i64 0
164#define TCG_TARGET_HAS_rem_i64 0
165#endif
166
167
168#if TCG_TARGET_REG_BITS == 32 \
169 && !(defined(TCG_TARGET_HAS_mulu2_i32) \
170 || defined(TCG_TARGET_HAS_muluh_i32))
171# error "Missing unsigned widening multiply"
172#endif
173
174#if !defined(TCG_TARGET_HAS_v64) \
175 && !defined(TCG_TARGET_HAS_v128) \
176 && !defined(TCG_TARGET_HAS_v256)
177#define TCG_TARGET_MAYBE_vec 0
178#define TCG_TARGET_HAS_neg_vec 0
179#define TCG_TARGET_HAS_not_vec 0
180#define TCG_TARGET_HAS_andc_vec 0
181#define TCG_TARGET_HAS_orc_vec 0
182#define TCG_TARGET_HAS_shi_vec 0
183#define TCG_TARGET_HAS_shs_vec 0
184#define TCG_TARGET_HAS_shv_vec 0
185#define TCG_TARGET_HAS_mul_vec 0
186#else
187#define TCG_TARGET_MAYBE_vec 1
188#endif
189#ifndef TCG_TARGET_HAS_v64
190#define TCG_TARGET_HAS_v64 0
191#endif
192#ifndef TCG_TARGET_HAS_v128
193#define TCG_TARGET_HAS_v128 0
194#endif
195#ifndef TCG_TARGET_HAS_v256
196#define TCG_TARGET_HAS_v256 0
197#endif
198
199#ifndef TARGET_INSN_START_EXTRA_WORDS
200# define TARGET_INSN_START_WORDS 1
201#else
202# define TARGET_INSN_START_WORDS (1 + TARGET_INSN_START_EXTRA_WORDS)
203#endif
204
205typedef enum TCGOpcode {
206#define DEF(name, oargs, iargs, cargs, flags) INDEX_op_ ## name,
207#include "tcg-opc.h"
208#undef DEF
209 NB_OPS,
210} TCGOpcode;
211
212#define tcg_regset_set_reg(d, r) ((d) |= (TCGRegSet)1 << (r))
213#define tcg_regset_reset_reg(d, r) ((d) &= ~((TCGRegSet)1 << (r)))
214#define tcg_regset_test_reg(d, r) (((d) >> (r)) & 1)
215
216#ifndef TCG_TARGET_INSN_UNIT_SIZE
217# error "Missing TCG_TARGET_INSN_UNIT_SIZE"
218#elif TCG_TARGET_INSN_UNIT_SIZE == 1
219typedef uint8_t tcg_insn_unit;
220#elif TCG_TARGET_INSN_UNIT_SIZE == 2
221typedef uint16_t tcg_insn_unit;
222#elif TCG_TARGET_INSN_UNIT_SIZE == 4
223typedef uint32_t tcg_insn_unit;
224#elif TCG_TARGET_INSN_UNIT_SIZE == 8
225typedef uint64_t tcg_insn_unit;
226#else
227
228#endif
229
230
231#if defined CONFIG_DEBUG_TCG || defined QEMU_STATIC_ANALYSIS
232# define tcg_debug_assert(X) do { assert(X); } while (0)
233#elif QEMU_GNUC_PREREQ(4, 5)
234# define tcg_debug_assert(X) \
235 do { if (!(X)) { __builtin_unreachable(); } } while (0)
236#else
237# define tcg_debug_assert(X) do { (void)(X); } while (0)
238#endif
239
240typedef struct TCGRelocation {
241 struct TCGRelocation *next;
242 int type;
243 tcg_insn_unit *ptr;
244 intptr_t addend;
245} TCGRelocation;
246
247typedef struct TCGLabel {
248 unsigned has_value : 1;
249 unsigned id : 31;
250 union {
251 uintptr_t value;
252 tcg_insn_unit *value_ptr;
253 TCGRelocation *first_reloc;
254 } u;
255} TCGLabel;
256
257typedef struct TCGPool {
258 struct TCGPool *next;
259 int size;
260 uint8_t data[0] __attribute__ ((aligned));
261} TCGPool;
262
263#define TCG_POOL_CHUNK_SIZE 32768
264
265#define TCG_MAX_TEMPS 512
266#define TCG_MAX_INSNS 512
267
268
269
270#define TCG_STATIC_CALL_ARGS_SIZE 128
271
272typedef enum TCGType {
273 TCG_TYPE_I32,
274 TCG_TYPE_I64,
275
276 TCG_TYPE_V64,
277 TCG_TYPE_V128,
278 TCG_TYPE_V256,
279
280 TCG_TYPE_COUNT,
281
282
283#if TCG_TARGET_REG_BITS == 32
284 TCG_TYPE_REG = TCG_TYPE_I32,
285#else
286 TCG_TYPE_REG = TCG_TYPE_I64,
287#endif
288
289
290#if UINTPTR_MAX == UINT32_MAX
291 TCG_TYPE_PTR = TCG_TYPE_I32,
292#else
293 TCG_TYPE_PTR = TCG_TYPE_I64,
294#endif
295
296
297#if TARGET_LONG_BITS == 64
298 TCG_TYPE_TL = TCG_TYPE_I64,
299#else
300 TCG_TYPE_TL = TCG_TYPE_I32,
301#endif
302} TCGType;
303
304
305typedef enum TCGMemOp {
306 MO_8 = 0,
307 MO_16 = 1,
308 MO_32 = 2,
309 MO_64 = 3,
310 MO_SIZE = 3,
311
312 MO_SIGN = 4,
313
314 MO_BSWAP = 8,
315#ifdef HOST_WORDS_BIGENDIAN
316 MO_LE = MO_BSWAP,
317 MO_BE = 0,
318#else
319 MO_LE = 0,
320 MO_BE = MO_BSWAP,
321#endif
322#ifdef TARGET_WORDS_BIGENDIAN
323 MO_TE = MO_BE,
324#else
325 MO_TE = MO_LE,
326#endif
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346 MO_ASHIFT = 4,
347 MO_AMASK = 7 << MO_ASHIFT,
348#ifdef ALIGNED_ONLY
349 MO_ALIGN = 0,
350 MO_UNALN = MO_AMASK,
351#else
352 MO_ALIGN = MO_AMASK,
353 MO_UNALN = 0,
354#endif
355 MO_ALIGN_2 = 1 << MO_ASHIFT,
356 MO_ALIGN_4 = 2 << MO_ASHIFT,
357 MO_ALIGN_8 = 3 << MO_ASHIFT,
358 MO_ALIGN_16 = 4 << MO_ASHIFT,
359 MO_ALIGN_32 = 5 << MO_ASHIFT,
360 MO_ALIGN_64 = 6 << MO_ASHIFT,
361
362
363 MO_UB = MO_8,
364 MO_UW = MO_16,
365 MO_UL = MO_32,
366 MO_SB = MO_SIGN | MO_8,
367 MO_SW = MO_SIGN | MO_16,
368 MO_SL = MO_SIGN | MO_32,
369 MO_Q = MO_64,
370
371 MO_LEUW = MO_LE | MO_UW,
372 MO_LEUL = MO_LE | MO_UL,
373 MO_LESW = MO_LE | MO_SW,
374 MO_LESL = MO_LE | MO_SL,
375 MO_LEQ = MO_LE | MO_Q,
376
377 MO_BEUW = MO_BE | MO_UW,
378 MO_BEUL = MO_BE | MO_UL,
379 MO_BESW = MO_BE | MO_SW,
380 MO_BESL = MO_BE | MO_SL,
381 MO_BEQ = MO_BE | MO_Q,
382
383 MO_TEUW = MO_TE | MO_UW,
384 MO_TEUL = MO_TE | MO_UL,
385 MO_TESW = MO_TE | MO_SW,
386 MO_TESL = MO_TE | MO_SL,
387 MO_TEQ = MO_TE | MO_Q,
388
389 MO_SSIZE = MO_SIZE | MO_SIGN,
390} TCGMemOp;
391
392
393
394
395
396
397
398static inline unsigned get_alignment_bits(TCGMemOp memop)
399{
400 unsigned a = memop & MO_AMASK;
401
402 if (a == MO_UNALN) {
403
404 a = 0;
405 } else if (a == MO_ALIGN) {
406
407 a = memop & MO_SIZE;
408 } else {
409
410 a = a >> MO_ASHIFT;
411 }
412#if defined(CONFIG_SOFTMMU)
413
414 tcg_debug_assert((TLB_FLAGS_MASK & ((1 << a) - 1)) == 0);
415#endif
416 return a;
417}
418
419typedef tcg_target_ulong TCGArg;
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451typedef struct TCGv_i32_d *TCGv_i32;
452typedef struct TCGv_i64_d *TCGv_i64;
453typedef struct TCGv_ptr_d *TCGv_ptr;
454typedef struct TCGv_vec_d *TCGv_vec;
455typedef TCGv_ptr TCGv_env;
456#if TARGET_LONG_BITS == 32
457#define TCGv TCGv_i32
458#elif TARGET_LONG_BITS == 64
459#define TCGv TCGv_i64
460#else
461#error Unhandled TARGET_LONG_BITS value
462#endif
463
464
465
466
467#define TCG_CALL_NO_READ_GLOBALS 0x0010
468
469#define TCG_CALL_NO_WRITE_GLOBALS 0x0020
470
471#define TCG_CALL_NO_SIDE_EFFECTS 0x0040
472
473
474#define TCG_CALL_NO_RWG TCG_CALL_NO_READ_GLOBALS
475#define TCG_CALL_NO_WG TCG_CALL_NO_WRITE_GLOBALS
476#define TCG_CALL_NO_SE TCG_CALL_NO_SIDE_EFFECTS
477#define TCG_CALL_NO_RWG_SE (TCG_CALL_NO_RWG | TCG_CALL_NO_SE)
478#define TCG_CALL_NO_WG_SE (TCG_CALL_NO_WG | TCG_CALL_NO_SE)
479
480
481#define TCG_CALL_DUMMY_ARG ((TCGArg)0)
482
483
484
485
486
487
488
489typedef enum {
490
491 TCG_COND_NEVER = 0 | 0 | 0 | 0,
492 TCG_COND_ALWAYS = 0 | 0 | 0 | 1,
493 TCG_COND_EQ = 8 | 0 | 0 | 0,
494 TCG_COND_NE = 8 | 0 | 0 | 1,
495
496 TCG_COND_LT = 0 | 0 | 2 | 0,
497 TCG_COND_GE = 0 | 0 | 2 | 1,
498 TCG_COND_LE = 8 | 0 | 2 | 0,
499 TCG_COND_GT = 8 | 0 | 2 | 1,
500
501 TCG_COND_LTU = 0 | 4 | 0 | 0,
502 TCG_COND_GEU = 0 | 4 | 0 | 1,
503 TCG_COND_LEU = 8 | 4 | 0 | 0,
504 TCG_COND_GTU = 8 | 4 | 0 | 1,
505} TCGCond;
506
507
508static inline TCGCond tcg_invert_cond(TCGCond c)
509{
510 return (TCGCond)(c ^ 1);
511}
512
513
514static inline TCGCond tcg_swap_cond(TCGCond c)
515{
516 return c & 6 ? (TCGCond)(c ^ 9) : c;
517}
518
519
520static inline TCGCond tcg_unsigned_cond(TCGCond c)
521{
522 return c & 2 ? (TCGCond)(c ^ 6) : c;
523}
524
525
526static inline TCGCond tcg_signed_cond(TCGCond c)
527{
528 return c & 4 ? (TCGCond)(c ^ 6) : c;
529}
530
531
532static inline bool is_unsigned_cond(TCGCond c)
533{
534 return (c & 4) != 0;
535}
536
537
538
539static inline TCGCond tcg_high_cond(TCGCond c)
540{
541 switch (c) {
542 case TCG_COND_GE:
543 case TCG_COND_LE:
544 case TCG_COND_GEU:
545 case TCG_COND_LEU:
546 return (TCGCond)(c ^ 8);
547 default:
548 return c;
549 }
550}
551
552typedef enum TCGTempVal {
553 TEMP_VAL_DEAD,
554 TEMP_VAL_REG,
555 TEMP_VAL_MEM,
556 TEMP_VAL_CONST,
557} TCGTempVal;
558
559typedef struct TCGTemp {
560 TCGReg reg:8;
561 TCGTempVal val_type:8;
562 TCGType base_type:8;
563 TCGType type:8;
564 unsigned int fixed_reg:1;
565 unsigned int indirect_reg:1;
566 unsigned int indirect_base:1;
567 unsigned int mem_coherent:1;
568 unsigned int mem_allocated:1;
569
570
571 unsigned int temp_global:1;
572
573
574
575 unsigned int temp_local:1;
576 unsigned int temp_allocated:1;
577
578 tcg_target_long val;
579 struct TCGTemp *mem_base;
580 intptr_t mem_offset;
581 const char *name;
582
583
584
585
586 uintptr_t state;
587 void *state_ptr;
588} TCGTemp;
589
590typedef struct TCGContext TCGContext;
591
592typedef struct TCGTempSet {
593 unsigned long l[BITS_TO_LONGS(TCG_MAX_TEMPS)];
594} TCGTempSet;
595
596
597
598
599
600#define DEAD_ARG 4
601#define SYNC_ARG 1
602typedef uint16_t TCGLifeData;
603
604
605
606typedef struct TCGOp {
607 TCGOpcode opc : 8;
608
609
610 unsigned param1 : 4;
611 unsigned param2 : 4;
612
613
614 unsigned life : 16;
615
616
617 QTAILQ_ENTRY(TCGOp) link;
618
619
620 TCGArg args[MAX_OPC_PARAM];
621} TCGOp;
622
623#define TCGOP_CALLI(X) (X)->param1
624#define TCGOP_CALLO(X) (X)->param2
625
626#define TCGOP_VECL(X) (X)->param1
627#define TCGOP_VECE(X) (X)->param2
628
629
630QEMU_BUILD_BUG_ON(NB_OPS > (1 << 8));
631
632typedef struct TCGProfile {
633 int64_t cpu_exec_time;
634 int64_t tb_count1;
635 int64_t tb_count;
636 int64_t op_count;
637 int op_count_max;
638 int temp_count_max;
639 int64_t temp_count;
640 int64_t del_op_count;
641 int64_t code_in_len;
642 int64_t code_out_len;
643 int64_t search_out_len;
644 int64_t interm_time;
645 int64_t code_time;
646 int64_t la_time;
647 int64_t opt_time;
648 int64_t restore_count;
649 int64_t restore_time;
650 int64_t table_op_count[NB_OPS];
651} TCGProfile;
652
653struct TCGContext {
654 uint8_t *pool_cur, *pool_end;
655 TCGPool *pool_first, *pool_current, *pool_first_large;
656 int nb_labels;
657 int nb_globals;
658 int nb_temps;
659 int nb_indirects;
660 int nb_ops;
661
662
663 tcg_insn_unit *code_buf;
664 uint16_t *tb_jmp_reset_offset;
665 uintptr_t *tb_jmp_insn_offset;
666 uintptr_t *tb_jmp_target_addr;
667
668 TCGRegSet reserved_regs;
669 uint32_t tb_cflags;
670 intptr_t current_frame_offset;
671 intptr_t frame_start;
672 intptr_t frame_end;
673 TCGTemp *frame_temp;
674
675 tcg_insn_unit *code_ptr;
676
677#ifdef CONFIG_PROFILER
678 TCGProfile prof;
679#endif
680
681#ifdef CONFIG_DEBUG_TCG
682 int temps_in_use;
683 int goto_tb_issue_mask;
684#endif
685
686
687
688
689
690 void *code_gen_prologue;
691 void *code_gen_epilogue;
692 void *code_gen_buffer;
693 size_t code_gen_buffer_size;
694 void *code_gen_ptr;
695 void *data_gen_ptr;
696
697
698 void *code_gen_highwater;
699
700 size_t tb_phys_invalidate_count;
701
702
703 CPUState *cpu;
704
705
706#ifdef TCG_TARGET_NEED_LDST_LABELS
707 QSIMPLEQ_HEAD(ldst_labels, TCGLabelQemuLdst) ldst_labels;
708#endif
709#ifdef TCG_TARGET_NEED_POOL_LABELS
710 struct TCGLabelPoolData *pool_labels;
711#endif
712
713 TCGLabel *exitreq_label;
714
715 TCGTempSet free_temps[TCG_TYPE_COUNT * 2];
716 TCGTemp temps[TCG_MAX_TEMPS];
717
718 QTAILQ_HEAD(TCGOpHead, TCGOp) ops, free_ops;
719
720
721
722 TCGTemp *reg_to_temp[TCG_TARGET_NB_REGS];
723
724 uint16_t gen_insn_end_off[TCG_MAX_INSNS];
725 target_ulong gen_insn_data[TCG_MAX_INSNS][TARGET_INSN_START_WORDS];
726};
727
728extern TCGContext tcg_init_ctx;
729extern __thread TCGContext *tcg_ctx;
730extern TCGv_env cpu_env;
731
732static inline size_t temp_idx(TCGTemp *ts)
733{
734 ptrdiff_t n = ts - tcg_ctx->temps;
735 tcg_debug_assert(n >= 0 && n < tcg_ctx->nb_temps);
736 return n;
737}
738
739static inline TCGArg temp_arg(TCGTemp *ts)
740{
741 return (uintptr_t)ts;
742}
743
744static inline TCGTemp *arg_temp(TCGArg a)
745{
746 return (TCGTemp *)(uintptr_t)a;
747}
748
749
750
751
752static inline TCGTemp *tcgv_i32_temp(TCGv_i32 v)
753{
754 uintptr_t o = (uintptr_t)v;
755 TCGTemp *t = (void *)tcg_ctx + o;
756 tcg_debug_assert(offsetof(TCGContext, temps[temp_idx(t)]) == o);
757 return t;
758}
759
760static inline TCGTemp *tcgv_i64_temp(TCGv_i64 v)
761{
762 return tcgv_i32_temp((TCGv_i32)v);
763}
764
765static inline TCGTemp *tcgv_ptr_temp(TCGv_ptr v)
766{
767 return tcgv_i32_temp((TCGv_i32)v);
768}
769
770static inline TCGTemp *tcgv_vec_temp(TCGv_vec v)
771{
772 return tcgv_i32_temp((TCGv_i32)v);
773}
774
775static inline TCGArg tcgv_i32_arg(TCGv_i32 v)
776{
777 return temp_arg(tcgv_i32_temp(v));
778}
779
780static inline TCGArg tcgv_i64_arg(TCGv_i64 v)
781{
782 return temp_arg(tcgv_i64_temp(v));
783}
784
785static inline TCGArg tcgv_ptr_arg(TCGv_ptr v)
786{
787 return temp_arg(tcgv_ptr_temp(v));
788}
789
790static inline TCGArg tcgv_vec_arg(TCGv_vec v)
791{
792 return temp_arg(tcgv_vec_temp(v));
793}
794
795static inline TCGv_i32 temp_tcgv_i32(TCGTemp *t)
796{
797 (void)temp_idx(t);
798 return (TCGv_i32)((void *)t - (void *)tcg_ctx);
799}
800
801static inline TCGv_i64 temp_tcgv_i64(TCGTemp *t)
802{
803 return (TCGv_i64)temp_tcgv_i32(t);
804}
805
806static inline TCGv_ptr temp_tcgv_ptr(TCGTemp *t)
807{
808 return (TCGv_ptr)temp_tcgv_i32(t);
809}
810
811static inline TCGv_vec temp_tcgv_vec(TCGTemp *t)
812{
813 return (TCGv_vec)temp_tcgv_i32(t);
814}
815
816#if TCG_TARGET_REG_BITS == 32
817static inline TCGv_i32 TCGV_LOW(TCGv_i64 t)
818{
819 return temp_tcgv_i32(tcgv_i64_temp(t));
820}
821
822static inline TCGv_i32 TCGV_HIGH(TCGv_i64 t)
823{
824 return temp_tcgv_i32(tcgv_i64_temp(t) + 1);
825}
826#endif
827
828static inline void tcg_set_insn_param(TCGOp *op, int arg, TCGArg v)
829{
830 op->args[arg] = v;
831}
832
833static inline void tcg_set_insn_start_param(TCGOp *op, int arg, target_ulong v)
834{
835#if TARGET_LONG_BITS <= TCG_TARGET_REG_BITS
836 tcg_set_insn_param(op, arg, v);
837#else
838 tcg_set_insn_param(op, arg * 2, v);
839 tcg_set_insn_param(op, arg * 2 + 1, v >> 32);
840#endif
841}
842
843
844static inline TCGOp *tcg_last_op(void)
845{
846 return QTAILQ_LAST(&tcg_ctx->ops, TCGOpHead);
847}
848
849
850static inline bool tcg_op_buf_full(void)
851{
852
853
854
855
856
857
858
859 return tcg_ctx->nb_ops >= 4000;
860}
861
862
863
864
865void *tcg_malloc_internal(TCGContext *s, int size);
866void tcg_pool_reset(TCGContext *s);
867TranslationBlock *tcg_tb_alloc(TCGContext *s);
868
869void tcg_region_init(void);
870void tcg_region_reset_all(void);
871
872size_t tcg_code_size(void);
873size_t tcg_code_capacity(void);
874
875void tcg_tb_insert(TranslationBlock *tb);
876void tcg_tb_remove(TranslationBlock *tb);
877size_t tcg_tb_phys_invalidate_count(void);
878TranslationBlock *tcg_tb_lookup(uintptr_t tc_ptr);
879void tcg_tb_foreach(GTraverseFunc func, gpointer user_data);
880size_t tcg_nb_tbs(void);
881
882
883static inline void *tcg_malloc(int size)
884{
885 TCGContext *s = tcg_ctx;
886 uint8_t *ptr, *ptr_end;
887
888
889 size = QEMU_ALIGN_UP(size, 8);
890
891 ptr = s->pool_cur;
892 ptr_end = ptr + size;
893 if (unlikely(ptr_end > s->pool_end)) {
894 return tcg_malloc_internal(tcg_ctx, size);
895 } else {
896 s->pool_cur = ptr_end;
897 return ptr;
898 }
899}
900
901void tcg_context_init(TCGContext *s);
902void tcg_register_thread(void);
903void tcg_prologue_init(TCGContext *s);
904void tcg_func_start(TCGContext *s);
905
906int tcg_gen_code(TCGContext *s, TranslationBlock *tb);
907
908void tcg_set_frame(TCGContext *s, TCGReg reg, intptr_t start, intptr_t size);
909
910TCGTemp *tcg_global_mem_new_internal(TCGType, TCGv_ptr,
911 intptr_t, const char *);
912TCGTemp *tcg_temp_new_internal(TCGType, bool);
913void tcg_temp_free_internal(TCGTemp *);
914TCGv_vec tcg_temp_new_vec(TCGType type);
915TCGv_vec tcg_temp_new_vec_matching(TCGv_vec match);
916
917static inline void tcg_temp_free_i32(TCGv_i32 arg)
918{
919 tcg_temp_free_internal(tcgv_i32_temp(arg));
920}
921
922static inline void tcg_temp_free_i64(TCGv_i64 arg)
923{
924 tcg_temp_free_internal(tcgv_i64_temp(arg));
925}
926
927static inline void tcg_temp_free_ptr(TCGv_ptr arg)
928{
929 tcg_temp_free_internal(tcgv_ptr_temp(arg));
930}
931
932static inline void tcg_temp_free_vec(TCGv_vec arg)
933{
934 tcg_temp_free_internal(tcgv_vec_temp(arg));
935}
936
937static inline TCGv_i32 tcg_global_mem_new_i32(TCGv_ptr reg, intptr_t offset,
938 const char *name)
939{
940 TCGTemp *t = tcg_global_mem_new_internal(TCG_TYPE_I32, reg, offset, name);
941 return temp_tcgv_i32(t);
942}
943
944static inline TCGv_i32 tcg_temp_new_i32(void)
945{
946 TCGTemp *t = tcg_temp_new_internal(TCG_TYPE_I32, false);
947 return temp_tcgv_i32(t);
948}
949
950static inline TCGv_i32 tcg_temp_local_new_i32(void)
951{
952 TCGTemp *t = tcg_temp_new_internal(TCG_TYPE_I32, true);
953 return temp_tcgv_i32(t);
954}
955
956static inline TCGv_i64 tcg_global_mem_new_i64(TCGv_ptr reg, intptr_t offset,
957 const char *name)
958{
959 TCGTemp *t = tcg_global_mem_new_internal(TCG_TYPE_I64, reg, offset, name);
960 return temp_tcgv_i64(t);
961}
962
963static inline TCGv_i64 tcg_temp_new_i64(void)
964{
965 TCGTemp *t = tcg_temp_new_internal(TCG_TYPE_I64, false);
966 return temp_tcgv_i64(t);
967}
968
969static inline TCGv_i64 tcg_temp_local_new_i64(void)
970{
971 TCGTemp *t = tcg_temp_new_internal(TCG_TYPE_I64, true);
972 return temp_tcgv_i64(t);
973}
974
975static inline TCGv_ptr tcg_global_mem_new_ptr(TCGv_ptr reg, intptr_t offset,
976 const char *name)
977{
978 TCGTemp *t = tcg_global_mem_new_internal(TCG_TYPE_PTR, reg, offset, name);
979 return temp_tcgv_ptr(t);
980}
981
982static inline TCGv_ptr tcg_temp_new_ptr(void)
983{
984 TCGTemp *t = tcg_temp_new_internal(TCG_TYPE_PTR, false);
985 return temp_tcgv_ptr(t);
986}
987
988static inline TCGv_ptr tcg_temp_local_new_ptr(void)
989{
990 TCGTemp *t = tcg_temp_new_internal(TCG_TYPE_PTR, true);
991 return temp_tcgv_ptr(t);
992}
993
994#if defined(CONFIG_DEBUG_TCG)
995
996
997
998
999
1000void tcg_clear_temp_count(void);
1001int tcg_check_temp_count(void);
1002#else
1003#define tcg_clear_temp_count() do { } while (0)
1004#define tcg_check_temp_count() 0
1005#endif
1006
1007int64_t tcg_cpu_exec_time(void);
1008void tcg_dump_info(FILE *f, fprintf_function cpu_fprintf);
1009void tcg_dump_op_count(FILE *f, fprintf_function cpu_fprintf);
1010
1011#define TCG_CT_ALIAS 0x80
1012#define TCG_CT_IALIAS 0x40
1013#define TCG_CT_NEWREG 0x20
1014#define TCG_CT_REG 0x01
1015#define TCG_CT_CONST 0x02
1016
1017typedef struct TCGArgConstraint {
1018 uint16_t ct;
1019 uint8_t alias_index;
1020 union {
1021 TCGRegSet regs;
1022 } u;
1023} TCGArgConstraint;
1024
1025#define TCG_MAX_OP_ARGS 16
1026
1027
1028enum {
1029
1030 TCG_OPF_BB_END = 0x01,
1031
1032 TCG_OPF_CALL_CLOBBER = 0x02,
1033
1034
1035 TCG_OPF_SIDE_EFFECTS = 0x04,
1036
1037 TCG_OPF_64BIT = 0x08,
1038
1039
1040 TCG_OPF_NOT_PRESENT = 0x10,
1041
1042 TCG_OPF_VECTOR = 0x20,
1043};
1044
1045typedef struct TCGOpDef {
1046 const char *name;
1047 uint8_t nb_oargs, nb_iargs, nb_cargs, nb_args;
1048 uint8_t flags;
1049 TCGArgConstraint *args_ct;
1050 int *sorted_args;
1051#if defined(CONFIG_DEBUG_TCG)
1052 int used;
1053#endif
1054} TCGOpDef;
1055
1056extern TCGOpDef tcg_op_defs[];
1057extern const size_t tcg_op_defs_max;
1058
1059typedef struct TCGTargetOpDef {
1060 TCGOpcode op;
1061 const char *args_ct_str[TCG_MAX_OP_ARGS];
1062} TCGTargetOpDef;
1063
1064#define tcg_abort() \
1065do {\
1066 fprintf(stderr, "%s:%d: tcg fatal error\n", __FILE__, __LINE__);\
1067 abort();\
1068} while (0)
1069
1070bool tcg_op_supported(TCGOpcode op);
1071
1072void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args);
1073
1074TCGOp *tcg_emit_op(TCGOpcode opc);
1075void tcg_op_remove(TCGContext *s, TCGOp *op);
1076TCGOp *tcg_op_insert_before(TCGContext *s, TCGOp *op, TCGOpcode opc, int narg);
1077TCGOp *tcg_op_insert_after(TCGContext *s, TCGOp *op, TCGOpcode opc, int narg);
1078
1079void tcg_optimize(TCGContext *s);
1080
1081
1082void tcg_dump_ops(TCGContext *s);
1083
1084TCGv_i32 tcg_const_i32(int32_t val);
1085TCGv_i64 tcg_const_i64(int64_t val);
1086TCGv_i32 tcg_const_local_i32(int32_t val);
1087TCGv_i64 tcg_const_local_i64(int64_t val);
1088TCGv_vec tcg_const_zeros_vec(TCGType);
1089TCGv_vec tcg_const_ones_vec(TCGType);
1090TCGv_vec tcg_const_zeros_vec_matching(TCGv_vec);
1091TCGv_vec tcg_const_ones_vec_matching(TCGv_vec);
1092
1093#if UINTPTR_MAX == UINT32_MAX
1094# define tcg_const_ptr(x) ((TCGv_ptr)tcg_const_i32((intptr_t)(x)))
1095# define tcg_const_local_ptr(x) ((TCGv_ptr)tcg_const_local_i32((intptr_t)(x)))
1096#else
1097# define tcg_const_ptr(x) ((TCGv_ptr)tcg_const_i64((intptr_t)(x)))
1098# define tcg_const_local_ptr(x) ((TCGv_ptr)tcg_const_local_i64((intptr_t)(x)))
1099#endif
1100
1101TCGLabel *gen_new_label(void);
1102
1103
1104
1105
1106
1107
1108
1109
1110static inline TCGArg label_arg(TCGLabel *l)
1111{
1112 return (uintptr_t)l;
1113}
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123static inline TCGLabel *arg_label(TCGArg i)
1124{
1125 return (TCGLabel *)(uintptr_t)i;
1126}
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141static inline ptrdiff_t tcg_ptr_byte_diff(void *a, void *b)
1142{
1143 return a - b;
1144}
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155static inline ptrdiff_t tcg_pcrel_diff(TCGContext *s, void *target)
1156{
1157 return tcg_ptr_byte_diff(target, s->code_ptr);
1158}
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168static inline size_t tcg_current_code_size(TCGContext *s)
1169{
1170 return tcg_ptr_byte_diff(s->code_ptr, s->code_buf);
1171}
1172
1173
1174typedef uint32_t TCGMemOpIdx;
1175
1176
1177
1178
1179
1180
1181
1182
1183static inline TCGMemOpIdx make_memop_idx(TCGMemOp op, unsigned idx)
1184{
1185 tcg_debug_assert(idx <= 15);
1186 return (op << 4) | idx;
1187}
1188
1189
1190
1191
1192
1193
1194
1195static inline TCGMemOp get_memop(TCGMemOpIdx oi)
1196{
1197 return oi >> 4;
1198}
1199
1200
1201
1202
1203
1204
1205
1206static inline unsigned get_mmuidx(TCGMemOpIdx oi)
1207{
1208 return oi & 15;
1209}
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256#define TB_EXIT_MASK 3
1257#define TB_EXIT_IDX0 0
1258#define TB_EXIT_IDX1 1
1259#define TB_EXIT_IDXMAX 1
1260#define TB_EXIT_REQUESTED 3
1261
1262#ifdef HAVE_TCG_QEMU_TB_EXEC
1263uintptr_t tcg_qemu_tb_exec(CPUArchState *env, uint8_t *tb_ptr);
1264#else
1265# define tcg_qemu_tb_exec(env, tb_ptr) \
1266 ((uintptr_t (*)(void *, void *))tcg_ctx->code_gen_prologue)(env, tb_ptr)
1267#endif
1268
1269void tcg_register_jit(void *buf, size_t buf_size);
1270
1271#if TCG_TARGET_MAYBE_vec
1272
1273
1274
1275int tcg_can_emit_vec_op(TCGOpcode, TCGType, unsigned);
1276#else
1277static inline int tcg_can_emit_vec_op(TCGOpcode o, TCGType t, unsigned ve)
1278{
1279 return 0;
1280}
1281#endif
1282
1283
1284void tcg_expand_vec_op(TCGOpcode, TCGType, unsigned, TCGArg, ...);
1285
1286
1287uint64_t dup_const(unsigned vece, uint64_t c);
1288
1289#define dup_const(VECE, C) \
1290 (__builtin_constant_p(VECE) \
1291 ? ( (VECE) == MO_8 ? 0x0101010101010101ull * (uint8_t)(C) \
1292 : (VECE) == MO_16 ? 0x0001000100010001ull * (uint16_t)(C) \
1293 : (VECE) == MO_32 ? 0x0000000100000001ull * (uint32_t)(C) \
1294 : dup_const(VECE, C)) \
1295 : dup_const(VECE, C))
1296
1297
1298
1299
1300
1301#ifdef CONFIG_SOFTMMU
1302
1303tcg_target_ulong helper_ret_ldub_mmu(CPUArchState *env, target_ulong addr,
1304 TCGMemOpIdx oi, uintptr_t retaddr);
1305tcg_target_ulong helper_le_lduw_mmu(CPUArchState *env, target_ulong addr,
1306 TCGMemOpIdx oi, uintptr_t retaddr);
1307tcg_target_ulong helper_le_ldul_mmu(CPUArchState *env, target_ulong addr,
1308 TCGMemOpIdx oi, uintptr_t retaddr);
1309uint64_t helper_le_ldq_mmu(CPUArchState *env, target_ulong addr,
1310 TCGMemOpIdx oi, uintptr_t retaddr);
1311tcg_target_ulong helper_be_lduw_mmu(CPUArchState *env, target_ulong addr,
1312 TCGMemOpIdx oi, uintptr_t retaddr);
1313tcg_target_ulong helper_be_ldul_mmu(CPUArchState *env, target_ulong addr,
1314 TCGMemOpIdx oi, uintptr_t retaddr);
1315uint64_t helper_be_ldq_mmu(CPUArchState *env, target_ulong addr,
1316 TCGMemOpIdx oi, uintptr_t retaddr);
1317
1318
1319tcg_target_ulong helper_ret_ldsb_mmu(CPUArchState *env, target_ulong addr,
1320 TCGMemOpIdx oi, uintptr_t retaddr);
1321tcg_target_ulong helper_le_ldsw_mmu(CPUArchState *env, target_ulong addr,
1322 TCGMemOpIdx oi, uintptr_t retaddr);
1323tcg_target_ulong helper_le_ldsl_mmu(CPUArchState *env, target_ulong addr,
1324 TCGMemOpIdx oi, uintptr_t retaddr);
1325tcg_target_ulong helper_be_ldsw_mmu(CPUArchState *env, target_ulong addr,
1326 TCGMemOpIdx oi, uintptr_t retaddr);
1327tcg_target_ulong helper_be_ldsl_mmu(CPUArchState *env, target_ulong addr,
1328 TCGMemOpIdx oi, uintptr_t retaddr);
1329
1330void helper_ret_stb_mmu(CPUArchState *env, target_ulong addr, uint8_t val,
1331 TCGMemOpIdx oi, uintptr_t retaddr);
1332void helper_le_stw_mmu(CPUArchState *env, target_ulong addr, uint16_t val,
1333 TCGMemOpIdx oi, uintptr_t retaddr);
1334void helper_le_stl_mmu(CPUArchState *env, target_ulong addr, uint32_t val,
1335 TCGMemOpIdx oi, uintptr_t retaddr);
1336void helper_le_stq_mmu(CPUArchState *env, target_ulong addr, uint64_t val,
1337 TCGMemOpIdx oi, uintptr_t retaddr);
1338void helper_be_stw_mmu(CPUArchState *env, target_ulong addr, uint16_t val,
1339 TCGMemOpIdx oi, uintptr_t retaddr);
1340void helper_be_stl_mmu(CPUArchState *env, target_ulong addr, uint32_t val,
1341 TCGMemOpIdx oi, uintptr_t retaddr);
1342void helper_be_stq_mmu(CPUArchState *env, target_ulong addr, uint64_t val,
1343 TCGMemOpIdx oi, uintptr_t retaddr);
1344
1345uint8_t helper_ret_ldb_cmmu(CPUArchState *env, target_ulong addr,
1346 TCGMemOpIdx oi, uintptr_t retaddr);
1347uint16_t helper_le_ldw_cmmu(CPUArchState *env, target_ulong addr,
1348 TCGMemOpIdx oi, uintptr_t retaddr);
1349uint32_t helper_le_ldl_cmmu(CPUArchState *env, target_ulong addr,
1350 TCGMemOpIdx oi, uintptr_t retaddr);
1351uint64_t helper_le_ldq_cmmu(CPUArchState *env, target_ulong addr,
1352 TCGMemOpIdx oi, uintptr_t retaddr);
1353uint16_t helper_be_ldw_cmmu(CPUArchState *env, target_ulong addr,
1354 TCGMemOpIdx oi, uintptr_t retaddr);
1355uint32_t helper_be_ldl_cmmu(CPUArchState *env, target_ulong addr,
1356 TCGMemOpIdx oi, uintptr_t retaddr);
1357uint64_t helper_be_ldq_cmmu(CPUArchState *env, target_ulong addr,
1358 TCGMemOpIdx oi, uintptr_t retaddr);
1359
1360
1361#ifdef TARGET_WORDS_BIGENDIAN
1362# define helper_ret_ldsw_mmu helper_be_ldsw_mmu
1363# define helper_ret_lduw_mmu helper_be_lduw_mmu
1364# define helper_ret_ldsl_mmu helper_be_ldsl_mmu
1365# define helper_ret_ldul_mmu helper_be_ldul_mmu
1366# define helper_ret_ldl_mmu helper_be_ldul_mmu
1367# define helper_ret_ldq_mmu helper_be_ldq_mmu
1368# define helper_ret_stw_mmu helper_be_stw_mmu
1369# define helper_ret_stl_mmu helper_be_stl_mmu
1370# define helper_ret_stq_mmu helper_be_stq_mmu
1371# define helper_ret_ldw_cmmu helper_be_ldw_cmmu
1372# define helper_ret_ldl_cmmu helper_be_ldl_cmmu
1373# define helper_ret_ldq_cmmu helper_be_ldq_cmmu
1374#else
1375# define helper_ret_ldsw_mmu helper_le_ldsw_mmu
1376# define helper_ret_lduw_mmu helper_le_lduw_mmu
1377# define helper_ret_ldsl_mmu helper_le_ldsl_mmu
1378# define helper_ret_ldul_mmu helper_le_ldul_mmu
1379# define helper_ret_ldl_mmu helper_le_ldul_mmu
1380# define helper_ret_ldq_mmu helper_le_ldq_mmu
1381# define helper_ret_stw_mmu helper_le_stw_mmu
1382# define helper_ret_stl_mmu helper_le_stl_mmu
1383# define helper_ret_stq_mmu helper_le_stq_mmu
1384# define helper_ret_ldw_cmmu helper_le_ldw_cmmu
1385# define helper_ret_ldl_cmmu helper_le_ldl_cmmu
1386# define helper_ret_ldq_cmmu helper_le_ldq_cmmu
1387#endif
1388
1389uint32_t helper_atomic_cmpxchgb_mmu(CPUArchState *env, target_ulong addr,
1390 uint32_t cmpv, uint32_t newv,
1391 TCGMemOpIdx oi, uintptr_t retaddr);
1392uint32_t helper_atomic_cmpxchgw_le_mmu(CPUArchState *env, target_ulong addr,
1393 uint32_t cmpv, uint32_t newv,
1394 TCGMemOpIdx oi, uintptr_t retaddr);
1395uint32_t helper_atomic_cmpxchgl_le_mmu(CPUArchState *env, target_ulong addr,
1396 uint32_t cmpv, uint32_t newv,
1397 TCGMemOpIdx oi, uintptr_t retaddr);
1398uint64_t helper_atomic_cmpxchgq_le_mmu(CPUArchState *env, target_ulong addr,
1399 uint64_t cmpv, uint64_t newv,
1400 TCGMemOpIdx oi, uintptr_t retaddr);
1401uint32_t helper_atomic_cmpxchgw_be_mmu(CPUArchState *env, target_ulong addr,
1402 uint32_t cmpv, uint32_t newv,
1403 TCGMemOpIdx oi, uintptr_t retaddr);
1404uint32_t helper_atomic_cmpxchgl_be_mmu(CPUArchState *env, target_ulong addr,
1405 uint32_t cmpv, uint32_t newv,
1406 TCGMemOpIdx oi, uintptr_t retaddr);
1407uint64_t helper_atomic_cmpxchgq_be_mmu(CPUArchState *env, target_ulong addr,
1408 uint64_t cmpv, uint64_t newv,
1409 TCGMemOpIdx oi, uintptr_t retaddr);
1410
1411#define GEN_ATOMIC_HELPER(NAME, TYPE, SUFFIX) \
1412TYPE helper_atomic_ ## NAME ## SUFFIX ## _mmu \
1413 (CPUArchState *env, target_ulong addr, TYPE val, \
1414 TCGMemOpIdx oi, uintptr_t retaddr);
1415
1416#ifdef CONFIG_ATOMIC64
1417#define GEN_ATOMIC_HELPER_ALL(NAME) \
1418 GEN_ATOMIC_HELPER(NAME, uint32_t, b) \
1419 GEN_ATOMIC_HELPER(NAME, uint32_t, w_le) \
1420 GEN_ATOMIC_HELPER(NAME, uint32_t, w_be) \
1421 GEN_ATOMIC_HELPER(NAME, uint32_t, l_le) \
1422 GEN_ATOMIC_HELPER(NAME, uint32_t, l_be) \
1423 GEN_ATOMIC_HELPER(NAME, uint64_t, q_le) \
1424 GEN_ATOMIC_HELPER(NAME, uint64_t, q_be)
1425#else
1426#define GEN_ATOMIC_HELPER_ALL(NAME) \
1427 GEN_ATOMIC_HELPER(NAME, uint32_t, b) \
1428 GEN_ATOMIC_HELPER(NAME, uint32_t, w_le) \
1429 GEN_ATOMIC_HELPER(NAME, uint32_t, w_be) \
1430 GEN_ATOMIC_HELPER(NAME, uint32_t, l_le) \
1431 GEN_ATOMIC_HELPER(NAME, uint32_t, l_be)
1432#endif
1433
1434GEN_ATOMIC_HELPER_ALL(fetch_add)
1435GEN_ATOMIC_HELPER_ALL(fetch_sub)
1436GEN_ATOMIC_HELPER_ALL(fetch_and)
1437GEN_ATOMIC_HELPER_ALL(fetch_or)
1438GEN_ATOMIC_HELPER_ALL(fetch_xor)
1439GEN_ATOMIC_HELPER_ALL(fetch_smin)
1440GEN_ATOMIC_HELPER_ALL(fetch_umin)
1441GEN_ATOMIC_HELPER_ALL(fetch_smax)
1442GEN_ATOMIC_HELPER_ALL(fetch_umax)
1443
1444GEN_ATOMIC_HELPER_ALL(add_fetch)
1445GEN_ATOMIC_HELPER_ALL(sub_fetch)
1446GEN_ATOMIC_HELPER_ALL(and_fetch)
1447GEN_ATOMIC_HELPER_ALL(or_fetch)
1448GEN_ATOMIC_HELPER_ALL(xor_fetch)
1449GEN_ATOMIC_HELPER_ALL(smin_fetch)
1450GEN_ATOMIC_HELPER_ALL(umin_fetch)
1451GEN_ATOMIC_HELPER_ALL(smax_fetch)
1452GEN_ATOMIC_HELPER_ALL(umax_fetch)
1453
1454GEN_ATOMIC_HELPER_ALL(xchg)
1455
1456#undef GEN_ATOMIC_HELPER_ALL
1457#undef GEN_ATOMIC_HELPER
1458#endif
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468Int128 helper_atomic_cmpxchgo_le_mmu(CPUArchState *env, target_ulong addr,
1469 Int128 cmpv, Int128 newv,
1470 TCGMemOpIdx oi, uintptr_t retaddr);
1471Int128 helper_atomic_cmpxchgo_be_mmu(CPUArchState *env, target_ulong addr,
1472 Int128 cmpv, Int128 newv,
1473 TCGMemOpIdx oi, uintptr_t retaddr);
1474
1475Int128 helper_atomic_ldo_le_mmu(CPUArchState *env, target_ulong addr,
1476 TCGMemOpIdx oi, uintptr_t retaddr);
1477Int128 helper_atomic_ldo_be_mmu(CPUArchState *env, target_ulong addr,
1478 TCGMemOpIdx oi, uintptr_t retaddr);
1479void helper_atomic_sto_le_mmu(CPUArchState *env, target_ulong addr, Int128 val,
1480 TCGMemOpIdx oi, uintptr_t retaddr);
1481void helper_atomic_sto_be_mmu(CPUArchState *env, target_ulong addr, Int128 val,
1482 TCGMemOpIdx oi, uintptr_t retaddr);
1483
1484#endif
1485