1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25#ifndef TCG_H
26#define TCG_H
27
28#include "qemu-common.h"
29#include "cpu.h"
30#include "exec/tb-context.h"
31#include "qemu/bitops.h"
32#include "tcg-target.h"
33
34
35#define MAX_OP_PER_INSTR 266
36
37#if HOST_LONG_BITS == 32
38#define MAX_OPC_PARAM_PER_ARG 2
39#else
40#define MAX_OPC_PARAM_PER_ARG 1
41#endif
42#define MAX_OPC_PARAM_IARGS 5
43#define MAX_OPC_PARAM_OARGS 1
44#define MAX_OPC_PARAM_ARGS (MAX_OPC_PARAM_IARGS + MAX_OPC_PARAM_OARGS)
45
46
47
48
49#define MAX_OPC_PARAM (4 + (MAX_OPC_PARAM_PER_ARG * MAX_OPC_PARAM_ARGS))
50#define OPC_BUF_SIZE 640
51#define OPC_MAX_SIZE (OPC_BUF_SIZE - MAX_OP_PER_INSTR)
52
53#define OPPARAM_BUF_SIZE (OPC_BUF_SIZE * MAX_OPC_PARAM)
54
55#define CPU_TEMP_BUF_NLONGS 128
56
57
58#ifndef TCG_TARGET_REG_BITS
59# if UINTPTR_MAX == UINT32_MAX
60# define TCG_TARGET_REG_BITS 32
61# elif UINTPTR_MAX == UINT64_MAX
62# define TCG_TARGET_REG_BITS 64
63# else
64# error Unknown pointer size for tcg target
65# endif
66#endif
67
68#if TCG_TARGET_REG_BITS == 32
69typedef int32_t tcg_target_long;
70typedef uint32_t tcg_target_ulong;
71#define TCG_PRIlx PRIx32
72#define TCG_PRIld PRId32
73#elif TCG_TARGET_REG_BITS == 64
74typedef int64_t tcg_target_long;
75typedef uint64_t tcg_target_ulong;
76#define TCG_PRIlx PRIx64
77#define TCG_PRIld PRId64
78#else
79#error unsupported
80#endif
81
82#if TCG_TARGET_NB_REGS <= 32
83typedef uint32_t TCGRegSet;
84#elif TCG_TARGET_NB_REGS <= 64
85typedef uint64_t TCGRegSet;
86#else
87#error unsupported
88#endif
89
90#if TCG_TARGET_REG_BITS == 32
91
92#define TCG_TARGET_HAS_extrl_i64_i32 0
93#define TCG_TARGET_HAS_extrh_i64_i32 0
94#define TCG_TARGET_HAS_div_i64 0
95#define TCG_TARGET_HAS_rem_i64 0
96#define TCG_TARGET_HAS_div2_i64 0
97#define TCG_TARGET_HAS_rot_i64 0
98#define TCG_TARGET_HAS_ext8s_i64 0
99#define TCG_TARGET_HAS_ext16s_i64 0
100#define TCG_TARGET_HAS_ext32s_i64 0
101#define TCG_TARGET_HAS_ext8u_i64 0
102#define TCG_TARGET_HAS_ext16u_i64 0
103#define TCG_TARGET_HAS_ext32u_i64 0
104#define TCG_TARGET_HAS_bswap16_i64 0
105#define TCG_TARGET_HAS_bswap32_i64 0
106#define TCG_TARGET_HAS_bswap64_i64 0
107#define TCG_TARGET_HAS_neg_i64 0
108#define TCG_TARGET_HAS_not_i64 0
109#define TCG_TARGET_HAS_andc_i64 0
110#define TCG_TARGET_HAS_orc_i64 0
111#define TCG_TARGET_HAS_eqv_i64 0
112#define TCG_TARGET_HAS_nand_i64 0
113#define TCG_TARGET_HAS_nor_i64 0
114#define TCG_TARGET_HAS_deposit_i64 0
115#define TCG_TARGET_HAS_movcond_i64 0
116#define TCG_TARGET_HAS_add2_i64 0
117#define TCG_TARGET_HAS_sub2_i64 0
118#define TCG_TARGET_HAS_mulu2_i64 0
119#define TCG_TARGET_HAS_muls2_i64 0
120#define TCG_TARGET_HAS_muluh_i64 0
121#define TCG_TARGET_HAS_mulsh_i64 0
122
123#define TCG_TARGET_HAS_add2_i32 1
124#define TCG_TARGET_HAS_sub2_i32 1
125#endif
126
127#ifndef TCG_TARGET_deposit_i32_valid
128#define TCG_TARGET_deposit_i32_valid(ofs, len) 1
129#endif
130#ifndef TCG_TARGET_deposit_i64_valid
131#define TCG_TARGET_deposit_i64_valid(ofs, len) 1
132#endif
133
134
135#if defined(TCG_TARGET_HAS_div_i32)
136#define TCG_TARGET_HAS_div2_i32 0
137#elif defined(TCG_TARGET_HAS_div2_i32)
138#define TCG_TARGET_HAS_div_i32 0
139#define TCG_TARGET_HAS_rem_i32 0
140#endif
141#if defined(TCG_TARGET_HAS_div_i64)
142#define TCG_TARGET_HAS_div2_i64 0
143#elif defined(TCG_TARGET_HAS_div2_i64)
144#define TCG_TARGET_HAS_div_i64 0
145#define TCG_TARGET_HAS_rem_i64 0
146#endif
147
148
149#if TCG_TARGET_REG_BITS == 32 \
150 && !(defined(TCG_TARGET_HAS_mulu2_i32) \
151 || defined(TCG_TARGET_HAS_muluh_i32))
152# error "Missing unsigned widening multiply"
153#endif
154
155#ifndef TARGET_INSN_START_EXTRA_WORDS
156# define TARGET_INSN_START_WORDS 1
157#else
158# define TARGET_INSN_START_WORDS (1 + TARGET_INSN_START_EXTRA_WORDS)
159#endif
160
161typedef enum TCGOpcode {
162#define DEF(name, oargs, iargs, cargs, flags) INDEX_op_ ## name,
163#include "tcg-opc.h"
164#undef DEF
165 NB_OPS,
166} TCGOpcode;
167
168#define tcg_regset_clear(d) (d) = 0
169#define tcg_regset_set(d, s) (d) = (s)
170#define tcg_regset_set32(d, reg, val32) (d) |= (val32) << (reg)
171#define tcg_regset_set_reg(d, r) (d) |= 1L << (r)
172#define tcg_regset_reset_reg(d, r) (d) &= ~(1L << (r))
173#define tcg_regset_test_reg(d, r) (((d) >> (r)) & 1)
174#define tcg_regset_or(d, a, b) (d) = (a) | (b)
175#define tcg_regset_and(d, a, b) (d) = (a) & (b)
176#define tcg_regset_andnot(d, a, b) (d) = (a) & ~(b)
177#define tcg_regset_not(d, a) (d) = ~(a)
178
179#ifndef TCG_TARGET_INSN_UNIT_SIZE
180# error "Missing TCG_TARGET_INSN_UNIT_SIZE"
181#elif TCG_TARGET_INSN_UNIT_SIZE == 1
182typedef uint8_t tcg_insn_unit;
183#elif TCG_TARGET_INSN_UNIT_SIZE == 2
184typedef uint16_t tcg_insn_unit;
185#elif TCG_TARGET_INSN_UNIT_SIZE == 4
186typedef uint32_t tcg_insn_unit;
187#elif TCG_TARGET_INSN_UNIT_SIZE == 8
188typedef uint64_t tcg_insn_unit;
189#else
190
191#endif
192
193
194#if defined CONFIG_DEBUG_TCG || defined QEMU_STATIC_ANALYSIS
195# define tcg_debug_assert(X) do { assert(X); } while (0)
196#elif QEMU_GNUC_PREREQ(4, 5)
197# define tcg_debug_assert(X) \
198 do { if (!(X)) { __builtin_unreachable(); } } while (0)
199#else
200# define tcg_debug_assert(X) do { (void)(X); } while (0)
201#endif
202
203typedef struct TCGRelocation {
204 struct TCGRelocation *next;
205 int type;
206 tcg_insn_unit *ptr;
207 intptr_t addend;
208} TCGRelocation;
209
210typedef struct TCGLabel {
211 unsigned has_value : 1;
212 unsigned id : 31;
213 union {
214 uintptr_t value;
215 tcg_insn_unit *value_ptr;
216 TCGRelocation *first_reloc;
217 } u;
218} TCGLabel;
219
220typedef struct TCGPool {
221 struct TCGPool *next;
222 int size;
223 uint8_t data[0] __attribute__ ((aligned));
224} TCGPool;
225
226#define TCG_POOL_CHUNK_SIZE 32768
227
228#define TCG_MAX_TEMPS 512
229#define TCG_MAX_INSNS 512
230
231
232
233#define TCG_STATIC_CALL_ARGS_SIZE 128
234
235typedef enum TCGType {
236 TCG_TYPE_I32,
237 TCG_TYPE_I64,
238 TCG_TYPE_COUNT,
239
240
241#if TCG_TARGET_REG_BITS == 32
242 TCG_TYPE_REG = TCG_TYPE_I32,
243#else
244 TCG_TYPE_REG = TCG_TYPE_I64,
245#endif
246
247
248#if UINTPTR_MAX == UINT32_MAX
249 TCG_TYPE_PTR = TCG_TYPE_I32,
250#else
251 TCG_TYPE_PTR = TCG_TYPE_I64,
252#endif
253
254
255#if TARGET_LONG_BITS == 64
256 TCG_TYPE_TL = TCG_TYPE_I64,
257#else
258 TCG_TYPE_TL = TCG_TYPE_I32,
259#endif
260} TCGType;
261
262
263typedef enum TCGMemOp {
264 MO_8 = 0,
265 MO_16 = 1,
266 MO_32 = 2,
267 MO_64 = 3,
268 MO_SIZE = 3,
269
270 MO_SIGN = 4,
271
272 MO_BSWAP = 8,
273#ifdef HOST_WORDS_BIGENDIAN
274 MO_LE = MO_BSWAP,
275 MO_BE = 0,
276#else
277 MO_LE = 0,
278 MO_BE = MO_BSWAP,
279#endif
280#ifdef TARGET_WORDS_BIGENDIAN
281 MO_TE = MO_BE,
282#else
283 MO_TE = MO_LE,
284#endif
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304 MO_ASHIFT = 4,
305 MO_AMASK = 7 << MO_ASHIFT,
306#ifdef ALIGNED_ONLY
307 MO_ALIGN = 0,
308 MO_UNALN = MO_AMASK,
309#else
310 MO_ALIGN = MO_AMASK,
311 MO_UNALN = 0,
312#endif
313 MO_ALIGN_2 = 1 << MO_ASHIFT,
314 MO_ALIGN_4 = 2 << MO_ASHIFT,
315 MO_ALIGN_8 = 3 << MO_ASHIFT,
316 MO_ALIGN_16 = 4 << MO_ASHIFT,
317 MO_ALIGN_32 = 5 << MO_ASHIFT,
318 MO_ALIGN_64 = 6 << MO_ASHIFT,
319
320
321 MO_UB = MO_8,
322 MO_UW = MO_16,
323 MO_UL = MO_32,
324 MO_SB = MO_SIGN | MO_8,
325 MO_SW = MO_SIGN | MO_16,
326 MO_SL = MO_SIGN | MO_32,
327 MO_Q = MO_64,
328
329 MO_LEUW = MO_LE | MO_UW,
330 MO_LEUL = MO_LE | MO_UL,
331 MO_LESW = MO_LE | MO_SW,
332 MO_LESL = MO_LE | MO_SL,
333 MO_LEQ = MO_LE | MO_Q,
334
335 MO_BEUW = MO_BE | MO_UW,
336 MO_BEUL = MO_BE | MO_UL,
337 MO_BESW = MO_BE | MO_SW,
338 MO_BESL = MO_BE | MO_SL,
339 MO_BEQ = MO_BE | MO_Q,
340
341 MO_TEUW = MO_TE | MO_UW,
342 MO_TEUL = MO_TE | MO_UL,
343 MO_TESW = MO_TE | MO_SW,
344 MO_TESL = MO_TE | MO_SL,
345 MO_TEQ = MO_TE | MO_Q,
346
347 MO_SSIZE = MO_SIZE | MO_SIGN,
348} TCGMemOp;
349
350
351
352
353
354
355
356static inline unsigned get_alignment_bits(TCGMemOp memop)
357{
358 unsigned a = memop & MO_AMASK;
359
360 if (a == MO_UNALN) {
361
362 a = 0;
363 } else if (a == MO_ALIGN) {
364
365 a = memop & MO_SIZE;
366 } else {
367
368 a = a >> MO_ASHIFT;
369 }
370#if defined(CONFIG_SOFTMMU)
371
372 tcg_debug_assert((TLB_FLAGS_MASK & ((1 << a) - 1)) == 0);
373#endif
374 return a;
375}
376
377typedef tcg_target_ulong TCGArg;
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410typedef struct TCGv_i32_d *TCGv_i32;
411typedef struct TCGv_i64_d *TCGv_i64;
412typedef struct TCGv_ptr_d *TCGv_ptr;
413typedef TCGv_ptr TCGv_env;
414#if TARGET_LONG_BITS == 32
415#define TCGv TCGv_i32
416#elif TARGET_LONG_BITS == 64
417#define TCGv TCGv_i64
418#else
419#error Unhandled TARGET_LONG_BITS value
420#endif
421
422static inline TCGv_i32 QEMU_ARTIFICIAL MAKE_TCGV_I32(intptr_t i)
423{
424 return (TCGv_i32)i;
425}
426
427static inline TCGv_i64 QEMU_ARTIFICIAL MAKE_TCGV_I64(intptr_t i)
428{
429 return (TCGv_i64)i;
430}
431
432static inline TCGv_ptr QEMU_ARTIFICIAL MAKE_TCGV_PTR(intptr_t i)
433{
434 return (TCGv_ptr)i;
435}
436
437static inline intptr_t QEMU_ARTIFICIAL GET_TCGV_I32(TCGv_i32 t)
438{
439 return (intptr_t)t;
440}
441
442static inline intptr_t QEMU_ARTIFICIAL GET_TCGV_I64(TCGv_i64 t)
443{
444 return (intptr_t)t;
445}
446
447static inline intptr_t QEMU_ARTIFICIAL GET_TCGV_PTR(TCGv_ptr t)
448{
449 return (intptr_t)t;
450}
451
452#if TCG_TARGET_REG_BITS == 32
453#define TCGV_LOW(t) MAKE_TCGV_I32(GET_TCGV_I64(t))
454#define TCGV_HIGH(t) MAKE_TCGV_I32(GET_TCGV_I64(t) + 1)
455#endif
456
457#define TCGV_EQUAL_I32(a, b) (GET_TCGV_I32(a) == GET_TCGV_I32(b))
458#define TCGV_EQUAL_I64(a, b) (GET_TCGV_I64(a) == GET_TCGV_I64(b))
459#define TCGV_EQUAL_PTR(a, b) (GET_TCGV_PTR(a) == GET_TCGV_PTR(b))
460
461
462#define TCGV_UNUSED_I32(x) x = MAKE_TCGV_I32(-1)
463#define TCGV_UNUSED_I64(x) x = MAKE_TCGV_I64(-1)
464#define TCGV_UNUSED_PTR(x) x = MAKE_TCGV_PTR(-1)
465
466#define TCGV_IS_UNUSED_I32(x) (GET_TCGV_I32(x) == -1)
467#define TCGV_IS_UNUSED_I64(x) (GET_TCGV_I64(x) == -1)
468#define TCGV_IS_UNUSED_PTR(x) (GET_TCGV_PTR(x) == -1)
469
470
471
472
473#define TCG_CALL_NO_READ_GLOBALS 0x0010
474
475#define TCG_CALL_NO_WRITE_GLOBALS 0x0020
476
477#define TCG_CALL_NO_SIDE_EFFECTS 0x0040
478
479
480#define TCG_CALL_NO_RWG TCG_CALL_NO_READ_GLOBALS
481#define TCG_CALL_NO_WG TCG_CALL_NO_WRITE_GLOBALS
482#define TCG_CALL_NO_SE TCG_CALL_NO_SIDE_EFFECTS
483#define TCG_CALL_NO_RWG_SE (TCG_CALL_NO_RWG | TCG_CALL_NO_SE)
484#define TCG_CALL_NO_WG_SE (TCG_CALL_NO_WG | TCG_CALL_NO_SE)
485
486
487#define TCG_CALL_DUMMY_TCGV MAKE_TCGV_I32(-1)
488#define TCG_CALL_DUMMY_ARG ((TCGArg)(-1))
489
490typedef enum {
491
492
493 TCG_MO_LD_LD = 0x01,
494 TCG_MO_ST_LD = 0x02,
495 TCG_MO_LD_ST = 0x04,
496 TCG_MO_ST_ST = 0x08,
497 TCG_MO_ALL = 0x0F,
498
499
500
501
502 TCG_BAR_LDAQ = 0x10,
503 TCG_BAR_STRL = 0x20,
504 TCG_BAR_SC = 0x30,
505} TCGBar;
506
507
508
509
510
511
512
513typedef enum {
514
515 TCG_COND_NEVER = 0 | 0 | 0 | 0,
516 TCG_COND_ALWAYS = 0 | 0 | 0 | 1,
517 TCG_COND_EQ = 8 | 0 | 0 | 0,
518 TCG_COND_NE = 8 | 0 | 0 | 1,
519
520 TCG_COND_LT = 0 | 0 | 2 | 0,
521 TCG_COND_GE = 0 | 0 | 2 | 1,
522 TCG_COND_LE = 8 | 0 | 2 | 0,
523 TCG_COND_GT = 8 | 0 | 2 | 1,
524
525 TCG_COND_LTU = 0 | 4 | 0 | 0,
526 TCG_COND_GEU = 0 | 4 | 0 | 1,
527 TCG_COND_LEU = 8 | 4 | 0 | 0,
528 TCG_COND_GTU = 8 | 4 | 0 | 1,
529} TCGCond;
530
531
532static inline TCGCond tcg_invert_cond(TCGCond c)
533{
534 return (TCGCond)(c ^ 1);
535}
536
537
538static inline TCGCond tcg_swap_cond(TCGCond c)
539{
540 return c & 6 ? (TCGCond)(c ^ 9) : c;
541}
542
543
544static inline TCGCond tcg_unsigned_cond(TCGCond c)
545{
546 return c & 2 ? (TCGCond)(c ^ 6) : c;
547}
548
549
550static inline bool is_unsigned_cond(TCGCond c)
551{
552 return (c & 4) != 0;
553}
554
555
556
557static inline TCGCond tcg_high_cond(TCGCond c)
558{
559 switch (c) {
560 case TCG_COND_GE:
561 case TCG_COND_LE:
562 case TCG_COND_GEU:
563 case TCG_COND_LEU:
564 return (TCGCond)(c ^ 8);
565 default:
566 return c;
567 }
568}
569
570typedef enum TCGTempVal {
571 TEMP_VAL_DEAD,
572 TEMP_VAL_REG,
573 TEMP_VAL_MEM,
574 TEMP_VAL_CONST,
575} TCGTempVal;
576
577typedef struct TCGTemp {
578 TCGReg reg:8;
579 TCGTempVal val_type:8;
580 TCGType base_type:8;
581 TCGType type:8;
582 unsigned int fixed_reg:1;
583 unsigned int indirect_reg:1;
584 unsigned int indirect_base:1;
585 unsigned int mem_coherent:1;
586 unsigned int mem_allocated:1;
587 unsigned int temp_local:1;
588
589
590 unsigned int temp_allocated:1;
591
592 tcg_target_long val;
593 struct TCGTemp *mem_base;
594 intptr_t mem_offset;
595 const char *name;
596} TCGTemp;
597
598typedef struct TCGContext TCGContext;
599
600typedef struct TCGTempSet {
601 unsigned long l[BITS_TO_LONGS(TCG_MAX_TEMPS)];
602} TCGTempSet;
603
604
605
606
607
608#define DEAD_ARG 4
609#define SYNC_ARG 1
610typedef uint16_t TCGLifeData;
611
612
613
614typedef struct TCGOp {
615 TCGOpcode opc : 8;
616
617
618 unsigned prev : 10;
619 unsigned next : 10;
620
621
622 unsigned calli : 4;
623 unsigned callo : 2;
624
625
626 unsigned args : 14;
627
628
629 unsigned life : 16;
630} TCGOp;
631
632
633QEMU_BUILD_BUG_ON(NB_OPS > (1 << 8));
634QEMU_BUILD_BUG_ON(OPC_BUF_SIZE > (1 << 10));
635QEMU_BUILD_BUG_ON(OPPARAM_BUF_SIZE > (1 << 14));
636
637
638QEMU_BUILD_BUG_ON(sizeof(TCGOp) > 8);
639
640struct TCGContext {
641 uint8_t *pool_cur, *pool_end;
642 TCGPool *pool_first, *pool_current, *pool_first_large;
643 int nb_labels;
644 int nb_globals;
645 int nb_temps;
646 int nb_indirects;
647
648
649 tcg_insn_unit *code_buf;
650 uint16_t *tb_jmp_reset_offset;
651 uint16_t *tb_jmp_insn_offset;
652 uintptr_t *tb_jmp_target_addr;
653
654 TCGRegSet reserved_regs;
655 intptr_t current_frame_offset;
656 intptr_t frame_start;
657 intptr_t frame_end;
658 TCGTemp *frame_temp;
659
660 tcg_insn_unit *code_ptr;
661
662 GHashTable *helpers;
663
664#ifdef CONFIG_PROFILER
665
666 int64_t tb_count1;
667 int64_t tb_count;
668 int64_t op_count;
669 int op_count_max;
670 int64_t temp_count;
671 int temp_count_max;
672 int64_t del_op_count;
673 int64_t code_in_len;
674 int64_t code_out_len;
675 int64_t search_out_len;
676 int64_t interm_time;
677 int64_t code_time;
678 int64_t la_time;
679 int64_t opt_time;
680 int64_t restore_count;
681 int64_t restore_time;
682#endif
683
684#ifdef CONFIG_DEBUG_TCG
685 int temps_in_use;
686 int goto_tb_issue_mask;
687#endif
688
689 int gen_next_op_idx;
690 int gen_next_parm_idx;
691
692
693
694
695
696 int code_gen_max_blocks;
697 void *code_gen_prologue;
698 void *code_gen_buffer;
699 size_t code_gen_buffer_size;
700 void *code_gen_ptr;
701
702
703 void *code_gen_highwater;
704
705 TBContext tb_ctx;
706
707
708 CPUState *cpu;
709 TCGv_env tcg_env;
710
711
712 struct TCGBackendData *be;
713
714 TCGTempSet free_temps[TCG_TYPE_COUNT * 2];
715 TCGTemp temps[TCG_MAX_TEMPS];
716
717
718
719 TCGTemp *reg_to_temp[TCG_TARGET_NB_REGS];
720
721 TCGOp gen_op_buf[OPC_BUF_SIZE];
722 TCGArg gen_opparam_buf[OPPARAM_BUF_SIZE];
723
724 uint16_t gen_insn_end_off[TCG_MAX_INSNS];
725 target_ulong gen_insn_data[TCG_MAX_INSNS][TARGET_INSN_START_WORDS];
726};
727
728extern TCGContext tcg_ctx;
729extern bool parallel_cpus;
730
731static inline void tcg_set_insn_param(int op_idx, int arg, TCGArg v)
732{
733 int op_argi = tcg_ctx.gen_op_buf[op_idx].args;
734 tcg_ctx.gen_opparam_buf[op_argi + arg] = v;
735}
736
737
738static inline int tcg_op_buf_count(void)
739{
740 return tcg_ctx.gen_next_op_idx;
741}
742
743
744static inline bool tcg_op_buf_full(void)
745{
746 return tcg_op_buf_count() >= OPC_MAX_SIZE;
747}
748
749
750
751
752void *tcg_malloc_internal(TCGContext *s, int size);
753void tcg_pool_reset(TCGContext *s);
754
755void tb_lock(void);
756void tb_unlock(void);
757void tb_lock_reset(void);
758
759
760static inline void *tcg_malloc(int size)
761{
762 TCGContext *s = &tcg_ctx;
763 uint8_t *ptr, *ptr_end;
764 size = (size + sizeof(long) - 1) & ~(sizeof(long) - 1);
765 ptr = s->pool_cur;
766 ptr_end = ptr + size;
767 if (unlikely(ptr_end > s->pool_end)) {
768 return tcg_malloc_internal(&tcg_ctx, size);
769 } else {
770 s->pool_cur = ptr_end;
771 return ptr;
772 }
773}
774
775void tcg_context_init(TCGContext *s);
776void tcg_prologue_init(TCGContext *s);
777void tcg_func_start(TCGContext *s);
778
779int tcg_gen_code(TCGContext *s, TranslationBlock *tb);
780
781void tcg_set_frame(TCGContext *s, TCGReg reg, intptr_t start, intptr_t size);
782
783int tcg_global_mem_new_internal(TCGType, TCGv_ptr, intptr_t, const char *);
784
785TCGv_i32 tcg_global_reg_new_i32(TCGReg reg, const char *name);
786TCGv_i64 tcg_global_reg_new_i64(TCGReg reg, const char *name);
787
788TCGv_i32 tcg_temp_new_internal_i32(int temp_local);
789TCGv_i64 tcg_temp_new_internal_i64(int temp_local);
790
791void tcg_temp_free_i32(TCGv_i32 arg);
792void tcg_temp_free_i64(TCGv_i64 arg);
793
794static inline TCGv_i32 tcg_global_mem_new_i32(TCGv_ptr reg, intptr_t offset,
795 const char *name)
796{
797 int idx = tcg_global_mem_new_internal(TCG_TYPE_I32, reg, offset, name);
798 return MAKE_TCGV_I32(idx);
799}
800
801static inline TCGv_i32 tcg_temp_new_i32(void)
802{
803 return tcg_temp_new_internal_i32(0);
804}
805
806static inline TCGv_i32 tcg_temp_local_new_i32(void)
807{
808 return tcg_temp_new_internal_i32(1);
809}
810
811static inline TCGv_i64 tcg_global_mem_new_i64(TCGv_ptr reg, intptr_t offset,
812 const char *name)
813{
814 int idx = tcg_global_mem_new_internal(TCG_TYPE_I64, reg, offset, name);
815 return MAKE_TCGV_I64(idx);
816}
817
818static inline TCGv_i64 tcg_temp_new_i64(void)
819{
820 return tcg_temp_new_internal_i64(0);
821}
822
823static inline TCGv_i64 tcg_temp_local_new_i64(void)
824{
825 return tcg_temp_new_internal_i64(1);
826}
827
828#if defined(CONFIG_DEBUG_TCG)
829
830
831
832
833
834void tcg_clear_temp_count(void);
835int tcg_check_temp_count(void);
836#else
837#define tcg_clear_temp_count() do { } while (0)
838#define tcg_check_temp_count() 0
839#endif
840
841void tcg_dump_info(FILE *f, fprintf_function cpu_fprintf);
842void tcg_dump_op_count(FILE *f, fprintf_function cpu_fprintf);
843
844#define TCG_CT_ALIAS 0x80
845#define TCG_CT_IALIAS 0x40
846#define TCG_CT_REG 0x01
847#define TCG_CT_CONST 0x02
848
849typedef struct TCGArgConstraint {
850 uint16_t ct;
851 uint8_t alias_index;
852 union {
853 TCGRegSet regs;
854 } u;
855} TCGArgConstraint;
856
857#define TCG_MAX_OP_ARGS 16
858
859
860enum {
861
862 TCG_OPF_BB_END = 0x01,
863
864 TCG_OPF_CALL_CLOBBER = 0x02,
865
866
867 TCG_OPF_SIDE_EFFECTS = 0x04,
868
869 TCG_OPF_64BIT = 0x08,
870
871
872 TCG_OPF_NOT_PRESENT = 0x10,
873};
874
875typedef struct TCGOpDef {
876 const char *name;
877 uint8_t nb_oargs, nb_iargs, nb_cargs, nb_args;
878 uint8_t flags;
879 TCGArgConstraint *args_ct;
880 int *sorted_args;
881#if defined(CONFIG_DEBUG_TCG)
882 int used;
883#endif
884} TCGOpDef;
885
886extern TCGOpDef tcg_op_defs[];
887extern const size_t tcg_op_defs_max;
888
889typedef struct TCGTargetOpDef {
890 TCGOpcode op;
891 const char *args_ct_str[TCG_MAX_OP_ARGS];
892} TCGTargetOpDef;
893
894#define tcg_abort() \
895do {\
896 fprintf(stderr, "%s:%d: tcg fatal error\n", __FILE__, __LINE__);\
897 abort();\
898} while (0)
899
900void tcg_add_target_add_op_defs(const TCGTargetOpDef *tdefs);
901
902#if UINTPTR_MAX == UINT32_MAX
903#define TCGV_NAT_TO_PTR(n) MAKE_TCGV_PTR(GET_TCGV_I32(n))
904#define TCGV_PTR_TO_NAT(n) MAKE_TCGV_I32(GET_TCGV_PTR(n))
905
906#define tcg_const_ptr(V) TCGV_NAT_TO_PTR(tcg_const_i32((intptr_t)(V)))
907#define tcg_global_reg_new_ptr(R, N) \
908 TCGV_NAT_TO_PTR(tcg_global_reg_new_i32((R), (N)))
909#define tcg_global_mem_new_ptr(R, O, N) \
910 TCGV_NAT_TO_PTR(tcg_global_mem_new_i32((R), (O), (N)))
911#define tcg_temp_new_ptr() TCGV_NAT_TO_PTR(tcg_temp_new_i32())
912#define tcg_temp_free_ptr(T) tcg_temp_free_i32(TCGV_PTR_TO_NAT(T))
913#else
914#define TCGV_NAT_TO_PTR(n) MAKE_TCGV_PTR(GET_TCGV_I64(n))
915#define TCGV_PTR_TO_NAT(n) MAKE_TCGV_I64(GET_TCGV_PTR(n))
916
917#define tcg_const_ptr(V) TCGV_NAT_TO_PTR(tcg_const_i64((intptr_t)(V)))
918#define tcg_global_reg_new_ptr(R, N) \
919 TCGV_NAT_TO_PTR(tcg_global_reg_new_i64((R), (N)))
920#define tcg_global_mem_new_ptr(R, O, N) \
921 TCGV_NAT_TO_PTR(tcg_global_mem_new_i64((R), (O), (N)))
922#define tcg_temp_new_ptr() TCGV_NAT_TO_PTR(tcg_temp_new_i64())
923#define tcg_temp_free_ptr(T) tcg_temp_free_i64(TCGV_PTR_TO_NAT(T))
924#endif
925
926void tcg_gen_callN(TCGContext *s, void *func,
927 TCGArg ret, int nargs, TCGArg *args);
928
929void tcg_op_remove(TCGContext *s, TCGOp *op);
930TCGOp *tcg_op_insert_before(TCGContext *s, TCGOp *op, TCGOpcode opc, int narg);
931TCGOp *tcg_op_insert_after(TCGContext *s, TCGOp *op, TCGOpcode opc, int narg);
932
933void tcg_optimize(TCGContext *s);
934
935
936void tcg_dump_ops(TCGContext *s);
937
938TCGv_i32 tcg_const_i32(int32_t val);
939TCGv_i64 tcg_const_i64(int64_t val);
940TCGv_i32 tcg_const_local_i32(int32_t val);
941TCGv_i64 tcg_const_local_i64(int64_t val);
942
943TCGLabel *gen_new_label(void);
944
945
946
947
948
949
950
951
952static inline TCGArg label_arg(TCGLabel *l)
953{
954 return (uintptr_t)l;
955}
956
957
958
959
960
961
962
963
964
965static inline TCGLabel *arg_label(TCGArg i)
966{
967 return (TCGLabel *)(uintptr_t)i;
968}
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983static inline ptrdiff_t tcg_ptr_byte_diff(void *a, void *b)
984{
985 return a - b;
986}
987
988
989
990
991
992
993
994
995
996
997static inline ptrdiff_t tcg_pcrel_diff(TCGContext *s, void *target)
998{
999 return tcg_ptr_byte_diff(target, s->code_ptr);
1000}
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010static inline size_t tcg_current_code_size(TCGContext *s)
1011{
1012 return tcg_ptr_byte_diff(s->code_ptr, s->code_buf);
1013}
1014
1015
1016typedef uint32_t TCGMemOpIdx;
1017
1018
1019
1020
1021
1022
1023
1024
1025static inline TCGMemOpIdx make_memop_idx(TCGMemOp op, unsigned idx)
1026{
1027 tcg_debug_assert(idx <= 15);
1028 return (op << 4) | idx;
1029}
1030
1031
1032
1033
1034
1035
1036
1037static inline TCGMemOp get_memop(TCGMemOpIdx oi)
1038{
1039 return oi >> 4;
1040}
1041
1042
1043
1044
1045
1046
1047
1048static inline unsigned get_mmuidx(TCGMemOpIdx oi)
1049{
1050 return oi & 15;
1051}
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098#define TB_EXIT_MASK 3
1099#define TB_EXIT_IDX0 0
1100#define TB_EXIT_IDX1 1
1101#define TB_EXIT_ICOUNT_EXPIRED 2
1102#define TB_EXIT_REQUESTED 3
1103
1104#ifdef HAVE_TCG_QEMU_TB_EXEC
1105uintptr_t tcg_qemu_tb_exec(CPUArchState *env, uint8_t *tb_ptr);
1106#else
1107# define tcg_qemu_tb_exec(env, tb_ptr) \
1108 ((uintptr_t (*)(void *, void *))tcg_ctx.code_gen_prologue)(env, tb_ptr)
1109#endif
1110
1111void tcg_register_jit(void *buf, size_t buf_size);
1112
1113
1114
1115
1116#ifdef CONFIG_SOFTMMU
1117
1118tcg_target_ulong helper_ret_ldub_mmu(CPUArchState *env, target_ulong addr,
1119 TCGMemOpIdx oi, uintptr_t retaddr);
1120tcg_target_ulong helper_le_lduw_mmu(CPUArchState *env, target_ulong addr,
1121 TCGMemOpIdx oi, uintptr_t retaddr);
1122tcg_target_ulong helper_le_ldul_mmu(CPUArchState *env, target_ulong addr,
1123 TCGMemOpIdx oi, uintptr_t retaddr);
1124uint64_t helper_le_ldq_mmu(CPUArchState *env, target_ulong addr,
1125 TCGMemOpIdx oi, uintptr_t retaddr);
1126tcg_target_ulong helper_be_lduw_mmu(CPUArchState *env, target_ulong addr,
1127 TCGMemOpIdx oi, uintptr_t retaddr);
1128tcg_target_ulong helper_be_ldul_mmu(CPUArchState *env, target_ulong addr,
1129 TCGMemOpIdx oi, uintptr_t retaddr);
1130uint64_t helper_be_ldq_mmu(CPUArchState *env, target_ulong addr,
1131 TCGMemOpIdx oi, uintptr_t retaddr);
1132
1133
1134tcg_target_ulong helper_ret_ldsb_mmu(CPUArchState *env, target_ulong addr,
1135 TCGMemOpIdx oi, uintptr_t retaddr);
1136tcg_target_ulong helper_le_ldsw_mmu(CPUArchState *env, target_ulong addr,
1137 TCGMemOpIdx oi, uintptr_t retaddr);
1138tcg_target_ulong helper_le_ldsl_mmu(CPUArchState *env, target_ulong addr,
1139 TCGMemOpIdx oi, uintptr_t retaddr);
1140tcg_target_ulong helper_be_ldsw_mmu(CPUArchState *env, target_ulong addr,
1141 TCGMemOpIdx oi, uintptr_t retaddr);
1142tcg_target_ulong helper_be_ldsl_mmu(CPUArchState *env, target_ulong addr,
1143 TCGMemOpIdx oi, uintptr_t retaddr);
1144
1145void helper_ret_stb_mmu(CPUArchState *env, target_ulong addr, uint8_t val,
1146 TCGMemOpIdx oi, uintptr_t retaddr);
1147void helper_le_stw_mmu(CPUArchState *env, target_ulong addr, uint16_t val,
1148 TCGMemOpIdx oi, uintptr_t retaddr);
1149void helper_le_stl_mmu(CPUArchState *env, target_ulong addr, uint32_t val,
1150 TCGMemOpIdx oi, uintptr_t retaddr);
1151void helper_le_stq_mmu(CPUArchState *env, target_ulong addr, uint64_t val,
1152 TCGMemOpIdx oi, uintptr_t retaddr);
1153void helper_be_stw_mmu(CPUArchState *env, target_ulong addr, uint16_t val,
1154 TCGMemOpIdx oi, uintptr_t retaddr);
1155void helper_be_stl_mmu(CPUArchState *env, target_ulong addr, uint32_t val,
1156 TCGMemOpIdx oi, uintptr_t retaddr);
1157void helper_be_stq_mmu(CPUArchState *env, target_ulong addr, uint64_t val,
1158 TCGMemOpIdx oi, uintptr_t retaddr);
1159
1160uint8_t helper_ret_ldb_cmmu(CPUArchState *env, target_ulong addr,
1161 TCGMemOpIdx oi, uintptr_t retaddr);
1162uint16_t helper_le_ldw_cmmu(CPUArchState *env, target_ulong addr,
1163 TCGMemOpIdx oi, uintptr_t retaddr);
1164uint32_t helper_le_ldl_cmmu(CPUArchState *env, target_ulong addr,
1165 TCGMemOpIdx oi, uintptr_t retaddr);
1166uint64_t helper_le_ldq_cmmu(CPUArchState *env, target_ulong addr,
1167 TCGMemOpIdx oi, uintptr_t retaddr);
1168uint16_t helper_be_ldw_cmmu(CPUArchState *env, target_ulong addr,
1169 TCGMemOpIdx oi, uintptr_t retaddr);
1170uint32_t helper_be_ldl_cmmu(CPUArchState *env, target_ulong addr,
1171 TCGMemOpIdx oi, uintptr_t retaddr);
1172uint64_t helper_be_ldq_cmmu(CPUArchState *env, target_ulong addr,
1173 TCGMemOpIdx oi, uintptr_t retaddr);
1174
1175
1176#ifdef TARGET_WORDS_BIGENDIAN
1177# define helper_ret_ldsw_mmu helper_be_ldsw_mmu
1178# define helper_ret_lduw_mmu helper_be_lduw_mmu
1179# define helper_ret_ldsl_mmu helper_be_ldsl_mmu
1180# define helper_ret_ldul_mmu helper_be_ldul_mmu
1181# define helper_ret_ldl_mmu helper_be_ldul_mmu
1182# define helper_ret_ldq_mmu helper_be_ldq_mmu
1183# define helper_ret_stw_mmu helper_be_stw_mmu
1184# define helper_ret_stl_mmu helper_be_stl_mmu
1185# define helper_ret_stq_mmu helper_be_stq_mmu
1186# define helper_ret_ldw_cmmu helper_be_ldw_cmmu
1187# define helper_ret_ldl_cmmu helper_be_ldl_cmmu
1188# define helper_ret_ldq_cmmu helper_be_ldq_cmmu
1189#else
1190# define helper_ret_ldsw_mmu helper_le_ldsw_mmu
1191# define helper_ret_lduw_mmu helper_le_lduw_mmu
1192# define helper_ret_ldsl_mmu helper_le_ldsl_mmu
1193# define helper_ret_ldul_mmu helper_le_ldul_mmu
1194# define helper_ret_ldl_mmu helper_le_ldul_mmu
1195# define helper_ret_ldq_mmu helper_le_ldq_mmu
1196# define helper_ret_stw_mmu helper_le_stw_mmu
1197# define helper_ret_stl_mmu helper_le_stl_mmu
1198# define helper_ret_stq_mmu helper_le_stq_mmu
1199# define helper_ret_ldw_cmmu helper_le_ldw_cmmu
1200# define helper_ret_ldl_cmmu helper_le_ldl_cmmu
1201# define helper_ret_ldq_cmmu helper_le_ldq_cmmu
1202#endif
1203
1204uint32_t helper_atomic_cmpxchgb_mmu(CPUArchState *env, target_ulong addr,
1205 uint32_t cmpv, uint32_t newv,
1206 TCGMemOpIdx oi, uintptr_t retaddr);
1207uint32_t helper_atomic_cmpxchgw_le_mmu(CPUArchState *env, target_ulong addr,
1208 uint32_t cmpv, uint32_t newv,
1209 TCGMemOpIdx oi, uintptr_t retaddr);
1210uint32_t helper_atomic_cmpxchgl_le_mmu(CPUArchState *env, target_ulong addr,
1211 uint32_t cmpv, uint32_t newv,
1212 TCGMemOpIdx oi, uintptr_t retaddr);
1213uint64_t helper_atomic_cmpxchgq_le_mmu(CPUArchState *env, target_ulong addr,
1214 uint64_t cmpv, uint64_t newv,
1215 TCGMemOpIdx oi, uintptr_t retaddr);
1216uint32_t helper_atomic_cmpxchgw_be_mmu(CPUArchState *env, target_ulong addr,
1217 uint32_t cmpv, uint32_t newv,
1218 TCGMemOpIdx oi, uintptr_t retaddr);
1219uint32_t helper_atomic_cmpxchgl_be_mmu(CPUArchState *env, target_ulong addr,
1220 uint32_t cmpv, uint32_t newv,
1221 TCGMemOpIdx oi, uintptr_t retaddr);
1222uint64_t helper_atomic_cmpxchgq_be_mmu(CPUArchState *env, target_ulong addr,
1223 uint64_t cmpv, uint64_t newv,
1224 TCGMemOpIdx oi, uintptr_t retaddr);
1225
1226#define GEN_ATOMIC_HELPER(NAME, TYPE, SUFFIX) \
1227TYPE helper_atomic_ ## NAME ## SUFFIX ## _mmu \
1228 (CPUArchState *env, target_ulong addr, TYPE val, \
1229 TCGMemOpIdx oi, uintptr_t retaddr);
1230
1231#ifdef CONFIG_ATOMIC64
1232#define GEN_ATOMIC_HELPER_ALL(NAME) \
1233 GEN_ATOMIC_HELPER(NAME, uint32_t, b) \
1234 GEN_ATOMIC_HELPER(NAME, uint32_t, w_le) \
1235 GEN_ATOMIC_HELPER(NAME, uint32_t, w_be) \
1236 GEN_ATOMIC_HELPER(NAME, uint32_t, l_le) \
1237 GEN_ATOMIC_HELPER(NAME, uint32_t, l_be) \
1238 GEN_ATOMIC_HELPER(NAME, uint64_t, q_le) \
1239 GEN_ATOMIC_HELPER(NAME, uint64_t, q_be)
1240#else
1241#define GEN_ATOMIC_HELPER_ALL(NAME) \
1242 GEN_ATOMIC_HELPER(NAME, uint32_t, b) \
1243 GEN_ATOMIC_HELPER(NAME, uint32_t, w_le) \
1244 GEN_ATOMIC_HELPER(NAME, uint32_t, w_be) \
1245 GEN_ATOMIC_HELPER(NAME, uint32_t, l_le) \
1246 GEN_ATOMIC_HELPER(NAME, uint32_t, l_be)
1247#endif
1248
1249GEN_ATOMIC_HELPER_ALL(fetch_add)
1250GEN_ATOMIC_HELPER_ALL(fetch_sub)
1251GEN_ATOMIC_HELPER_ALL(fetch_and)
1252GEN_ATOMIC_HELPER_ALL(fetch_or)
1253GEN_ATOMIC_HELPER_ALL(fetch_xor)
1254
1255GEN_ATOMIC_HELPER_ALL(add_fetch)
1256GEN_ATOMIC_HELPER_ALL(sub_fetch)
1257GEN_ATOMIC_HELPER_ALL(and_fetch)
1258GEN_ATOMIC_HELPER_ALL(or_fetch)
1259GEN_ATOMIC_HELPER_ALL(xor_fetch)
1260
1261GEN_ATOMIC_HELPER_ALL(xchg)
1262
1263#undef GEN_ATOMIC_HELPER_ALL
1264#undef GEN_ATOMIC_HELPER
1265#endif
1266
1267#ifdef CONFIG_ATOMIC128
1268#include "qemu/int128.h"
1269
1270
1271
1272Int128 helper_atomic_cmpxchgo_le_mmu(CPUArchState *env, target_ulong addr,
1273 Int128 cmpv, Int128 newv,
1274 TCGMemOpIdx oi, uintptr_t retaddr);
1275Int128 helper_atomic_cmpxchgo_be_mmu(CPUArchState *env, target_ulong addr,
1276 Int128 cmpv, Int128 newv,
1277 TCGMemOpIdx oi, uintptr_t retaddr);
1278
1279Int128 helper_atomic_ldo_le_mmu(CPUArchState *env, target_ulong addr,
1280 TCGMemOpIdx oi, uintptr_t retaddr);
1281Int128 helper_atomic_ldo_be_mmu(CPUArchState *env, target_ulong addr,
1282 TCGMemOpIdx oi, uintptr_t retaddr);
1283void helper_atomic_sto_le_mmu(CPUArchState *env, target_ulong addr, Int128 val,
1284 TCGMemOpIdx oi, uintptr_t retaddr);
1285void helper_atomic_sto_be_mmu(CPUArchState *env, target_ulong addr, Int128 val,
1286 TCGMemOpIdx oi, uintptr_t retaddr);
1287
1288#endif
1289
1290#endif
1291