1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25#ifndef TCG_H
26#define TCG_H
27
28#include "cpu.h"
29#include "exec/memop.h"
30#include "exec/tb-context.h"
31#include "qemu/bitops.h"
32#include "qemu/plugin.h"
33#include "qemu/queue.h"
34#include "tcg/tcg-mo.h"
35#include "tcg-target.h"
36#include "qemu/int128.h"
37
38
39#define MAX_OP_PER_INSTR 266
40
41#if HOST_LONG_BITS == 32
42#define MAX_OPC_PARAM_PER_ARG 2
43#else
44#define MAX_OPC_PARAM_PER_ARG 1
45#endif
46#define MAX_OPC_PARAM_IARGS 6
47#define MAX_OPC_PARAM_OARGS 1
48#define MAX_OPC_PARAM_ARGS (MAX_OPC_PARAM_IARGS + MAX_OPC_PARAM_OARGS)
49
50
51
52
53#define MAX_OPC_PARAM (4 + (MAX_OPC_PARAM_PER_ARG * MAX_OPC_PARAM_ARGS))
54
55#define CPU_TEMP_BUF_NLONGS 128
56
57
58#ifndef TCG_TARGET_REG_BITS
59# if UINTPTR_MAX == UINT32_MAX
60# define TCG_TARGET_REG_BITS 32
61# elif UINTPTR_MAX == UINT64_MAX
62# define TCG_TARGET_REG_BITS 64
63# else
64# error Unknown pointer size for tcg target
65# endif
66#endif
67
68#if TCG_TARGET_REG_BITS == 32
69typedef int32_t tcg_target_long;
70typedef uint32_t tcg_target_ulong;
71#define TCG_PRIlx PRIx32
72#define TCG_PRIld PRId32
73#elif TCG_TARGET_REG_BITS == 64
74typedef int64_t tcg_target_long;
75typedef uint64_t tcg_target_ulong;
76#define TCG_PRIlx PRIx64
77#define TCG_PRIld PRId64
78#else
79#error unsupported
80#endif
81
82
83
84
85#if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
86#define TCG_OVERSIZED_GUEST 1
87#else
88#define TCG_OVERSIZED_GUEST 0
89#endif
90
91#if TCG_TARGET_NB_REGS <= 32
92typedef uint32_t TCGRegSet;
93#elif TCG_TARGET_NB_REGS <= 64
94typedef uint64_t TCGRegSet;
95#else
96#error unsupported
97#endif
98
99#if TCG_TARGET_REG_BITS == 32
100
101#define TCG_TARGET_HAS_extrl_i64_i32 0
102#define TCG_TARGET_HAS_extrh_i64_i32 0
103#define TCG_TARGET_HAS_div_i64 0
104#define TCG_TARGET_HAS_rem_i64 0
105#define TCG_TARGET_HAS_div2_i64 0
106#define TCG_TARGET_HAS_rot_i64 0
107#define TCG_TARGET_HAS_ext8s_i64 0
108#define TCG_TARGET_HAS_ext16s_i64 0
109#define TCG_TARGET_HAS_ext32s_i64 0
110#define TCG_TARGET_HAS_ext8u_i64 0
111#define TCG_TARGET_HAS_ext16u_i64 0
112#define TCG_TARGET_HAS_ext32u_i64 0
113#define TCG_TARGET_HAS_bswap16_i64 0
114#define TCG_TARGET_HAS_bswap32_i64 0
115#define TCG_TARGET_HAS_bswap64_i64 0
116#define TCG_TARGET_HAS_neg_i64 0
117#define TCG_TARGET_HAS_not_i64 0
118#define TCG_TARGET_HAS_andc_i64 0
119#define TCG_TARGET_HAS_orc_i64 0
120#define TCG_TARGET_HAS_eqv_i64 0
121#define TCG_TARGET_HAS_nand_i64 0
122#define TCG_TARGET_HAS_nor_i64 0
123#define TCG_TARGET_HAS_clz_i64 0
124#define TCG_TARGET_HAS_ctz_i64 0
125#define TCG_TARGET_HAS_ctpop_i64 0
126#define TCG_TARGET_HAS_deposit_i64 0
127#define TCG_TARGET_HAS_extract_i64 0
128#define TCG_TARGET_HAS_sextract_i64 0
129#define TCG_TARGET_HAS_extract2_i64 0
130#define TCG_TARGET_HAS_movcond_i64 0
131#define TCG_TARGET_HAS_add2_i64 0
132#define TCG_TARGET_HAS_sub2_i64 0
133#define TCG_TARGET_HAS_mulu2_i64 0
134#define TCG_TARGET_HAS_muls2_i64 0
135#define TCG_TARGET_HAS_muluh_i64 0
136#define TCG_TARGET_HAS_mulsh_i64 0
137
138#define TCG_TARGET_HAS_add2_i32 1
139#define TCG_TARGET_HAS_sub2_i32 1
140#endif
141
142#ifndef TCG_TARGET_deposit_i32_valid
143#define TCG_TARGET_deposit_i32_valid(ofs, len) 1
144#endif
145#ifndef TCG_TARGET_deposit_i64_valid
146#define TCG_TARGET_deposit_i64_valid(ofs, len) 1
147#endif
148#ifndef TCG_TARGET_extract_i32_valid
149#define TCG_TARGET_extract_i32_valid(ofs, len) 1
150#endif
151#ifndef TCG_TARGET_extract_i64_valid
152#define TCG_TARGET_extract_i64_valid(ofs, len) 1
153#endif
154
155
156#if defined(TCG_TARGET_HAS_div_i32)
157#define TCG_TARGET_HAS_div2_i32 0
158#elif defined(TCG_TARGET_HAS_div2_i32)
159#define TCG_TARGET_HAS_div_i32 0
160#define TCG_TARGET_HAS_rem_i32 0
161#endif
162#if defined(TCG_TARGET_HAS_div_i64)
163#define TCG_TARGET_HAS_div2_i64 0
164#elif defined(TCG_TARGET_HAS_div2_i64)
165#define TCG_TARGET_HAS_div_i64 0
166#define TCG_TARGET_HAS_rem_i64 0
167#endif
168
169
170#if TCG_TARGET_REG_BITS == 32 \
171 && !(defined(TCG_TARGET_HAS_mulu2_i32) \
172 || defined(TCG_TARGET_HAS_muluh_i32))
173# error "Missing unsigned widening multiply"
174#endif
175
176#if !defined(TCG_TARGET_HAS_v64) \
177 && !defined(TCG_TARGET_HAS_v128) \
178 && !defined(TCG_TARGET_HAS_v256)
179#define TCG_TARGET_MAYBE_vec 0
180#define TCG_TARGET_HAS_abs_vec 0
181#define TCG_TARGET_HAS_neg_vec 0
182#define TCG_TARGET_HAS_not_vec 0
183#define TCG_TARGET_HAS_andc_vec 0
184#define TCG_TARGET_HAS_orc_vec 0
185#define TCG_TARGET_HAS_shi_vec 0
186#define TCG_TARGET_HAS_shs_vec 0
187#define TCG_TARGET_HAS_shv_vec 0
188#define TCG_TARGET_HAS_mul_vec 0
189#define TCG_TARGET_HAS_sat_vec 0
190#define TCG_TARGET_HAS_minmax_vec 0
191#define TCG_TARGET_HAS_bitsel_vec 0
192#define TCG_TARGET_HAS_cmpsel_vec 0
193#else
194#define TCG_TARGET_MAYBE_vec 1
195#endif
196#ifndef TCG_TARGET_HAS_v64
197#define TCG_TARGET_HAS_v64 0
198#endif
199#ifndef TCG_TARGET_HAS_v128
200#define TCG_TARGET_HAS_v128 0
201#endif
202#ifndef TCG_TARGET_HAS_v256
203#define TCG_TARGET_HAS_v256 0
204#endif
205
206#ifndef TARGET_INSN_START_EXTRA_WORDS
207# define TARGET_INSN_START_WORDS 1
208#else
209# define TARGET_INSN_START_WORDS (1 + TARGET_INSN_START_EXTRA_WORDS)
210#endif
211
212typedef enum TCGOpcode {
213#define DEF(name, oargs, iargs, cargs, flags) INDEX_op_ ## name,
214#include "tcg/tcg-opc.h"
215#undef DEF
216 NB_OPS,
217} TCGOpcode;
218
219#define tcg_regset_set_reg(d, r) ((d) |= (TCGRegSet)1 << (r))
220#define tcg_regset_reset_reg(d, r) ((d) &= ~((TCGRegSet)1 << (r)))
221#define tcg_regset_test_reg(d, r) (((d) >> (r)) & 1)
222
223#ifndef TCG_TARGET_INSN_UNIT_SIZE
224# error "Missing TCG_TARGET_INSN_UNIT_SIZE"
225#elif TCG_TARGET_INSN_UNIT_SIZE == 1
226typedef uint8_t tcg_insn_unit;
227#elif TCG_TARGET_INSN_UNIT_SIZE == 2
228typedef uint16_t tcg_insn_unit;
229#elif TCG_TARGET_INSN_UNIT_SIZE == 4
230typedef uint32_t tcg_insn_unit;
231#elif TCG_TARGET_INSN_UNIT_SIZE == 8
232typedef uint64_t tcg_insn_unit;
233#else
234
235#endif
236
237
238#if defined CONFIG_DEBUG_TCG || defined QEMU_STATIC_ANALYSIS
239# define tcg_debug_assert(X) do { assert(X); } while (0)
240#else
241# define tcg_debug_assert(X) \
242 do { if (!(X)) { __builtin_unreachable(); } } while (0)
243#endif
244
245typedef struct TCGRelocation TCGRelocation;
246struct TCGRelocation {
247 QSIMPLEQ_ENTRY(TCGRelocation) next;
248 tcg_insn_unit *ptr;
249 intptr_t addend;
250 int type;
251};
252
253typedef struct TCGLabel TCGLabel;
254struct TCGLabel {
255 unsigned present : 1;
256 unsigned has_value : 1;
257 unsigned id : 14;
258 unsigned refs : 16;
259 union {
260 uintptr_t value;
261 tcg_insn_unit *value_ptr;
262 } u;
263 QSIMPLEQ_HEAD(, TCGRelocation) relocs;
264 QSIMPLEQ_ENTRY(TCGLabel) next;
265};
266
267typedef struct TCGPool {
268 struct TCGPool *next;
269 int size;
270 uint8_t data[] __attribute__ ((aligned));
271} TCGPool;
272
273#define TCG_POOL_CHUNK_SIZE 32768
274
275#define TCG_MAX_TEMPS 512
276#define TCG_MAX_INSNS 512
277
278
279
280#define TCG_STATIC_CALL_ARGS_SIZE 128
281
282typedef enum TCGType {
283 TCG_TYPE_I32,
284 TCG_TYPE_I64,
285
286 TCG_TYPE_V64,
287 TCG_TYPE_V128,
288 TCG_TYPE_V256,
289
290 TCG_TYPE_COUNT,
291
292
293#if TCG_TARGET_REG_BITS == 32
294 TCG_TYPE_REG = TCG_TYPE_I32,
295#else
296 TCG_TYPE_REG = TCG_TYPE_I64,
297#endif
298
299
300#if UINTPTR_MAX == UINT32_MAX
301 TCG_TYPE_PTR = TCG_TYPE_I32,
302#else
303 TCG_TYPE_PTR = TCG_TYPE_I64,
304#endif
305
306
307#if TARGET_LONG_BITS == 64
308 TCG_TYPE_TL = TCG_TYPE_I64,
309#else
310 TCG_TYPE_TL = TCG_TYPE_I32,
311#endif
312} TCGType;
313
314
315
316
317
318
319
320static inline unsigned get_alignment_bits(MemOp memop)
321{
322 unsigned a = memop & MO_AMASK;
323
324 if (a == MO_UNALN) {
325
326 a = 0;
327 } else if (a == MO_ALIGN) {
328
329 a = memop & MO_SIZE;
330 } else {
331
332 a = a >> MO_ASHIFT;
333 }
334#if defined(CONFIG_SOFTMMU)
335
336 tcg_debug_assert((TLB_FLAGS_MASK & ((1 << a) - 1)) == 0);
337#endif
338 return a;
339}
340
341typedef tcg_target_ulong TCGArg;
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373typedef struct TCGv_i32_d *TCGv_i32;
374typedef struct TCGv_i64_d *TCGv_i64;
375typedef struct TCGv_ptr_d *TCGv_ptr;
376typedef struct TCGv_vec_d *TCGv_vec;
377typedef TCGv_ptr TCGv_env;
378#if TARGET_LONG_BITS == 32
379#define TCGv TCGv_i32
380#elif TARGET_LONG_BITS == 64
381#define TCGv TCGv_i64
382#else
383#error Unhandled TARGET_LONG_BITS value
384#endif
385
386
387
388
389#define TCG_CALL_NO_READ_GLOBALS 0x0001
390
391#define TCG_CALL_NO_WRITE_GLOBALS 0x0002
392
393#define TCG_CALL_NO_SIDE_EFFECTS 0x0004
394
395#define TCG_CALL_NO_RETURN 0x0008
396
397
398#define TCG_CALL_NO_RWG TCG_CALL_NO_READ_GLOBALS
399#define TCG_CALL_NO_WG TCG_CALL_NO_WRITE_GLOBALS
400#define TCG_CALL_NO_SE TCG_CALL_NO_SIDE_EFFECTS
401#define TCG_CALL_NO_RWG_SE (TCG_CALL_NO_RWG | TCG_CALL_NO_SE)
402#define TCG_CALL_NO_WG_SE (TCG_CALL_NO_WG | TCG_CALL_NO_SE)
403
404
405#define TCG_CALL_DUMMY_ARG ((TCGArg)0)
406
407
408
409
410
411
412
413typedef enum {
414
415 TCG_COND_NEVER = 0 | 0 | 0 | 0,
416 TCG_COND_ALWAYS = 0 | 0 | 0 | 1,
417 TCG_COND_EQ = 8 | 0 | 0 | 0,
418 TCG_COND_NE = 8 | 0 | 0 | 1,
419
420 TCG_COND_LT = 0 | 0 | 2 | 0,
421 TCG_COND_GE = 0 | 0 | 2 | 1,
422 TCG_COND_LE = 8 | 0 | 2 | 0,
423 TCG_COND_GT = 8 | 0 | 2 | 1,
424
425 TCG_COND_LTU = 0 | 4 | 0 | 0,
426 TCG_COND_GEU = 0 | 4 | 0 | 1,
427 TCG_COND_LEU = 8 | 4 | 0 | 0,
428 TCG_COND_GTU = 8 | 4 | 0 | 1,
429} TCGCond;
430
431
432static inline TCGCond tcg_invert_cond(TCGCond c)
433{
434 return (TCGCond)(c ^ 1);
435}
436
437
438static inline TCGCond tcg_swap_cond(TCGCond c)
439{
440 return c & 6 ? (TCGCond)(c ^ 9) : c;
441}
442
443
444static inline TCGCond tcg_unsigned_cond(TCGCond c)
445{
446 return c & 2 ? (TCGCond)(c ^ 6) : c;
447}
448
449
450static inline TCGCond tcg_signed_cond(TCGCond c)
451{
452 return c & 4 ? (TCGCond)(c ^ 6) : c;
453}
454
455
456static inline bool is_unsigned_cond(TCGCond c)
457{
458 return (c & 4) != 0;
459}
460
461
462
463static inline TCGCond tcg_high_cond(TCGCond c)
464{
465 switch (c) {
466 case TCG_COND_GE:
467 case TCG_COND_LE:
468 case TCG_COND_GEU:
469 case TCG_COND_LEU:
470 return (TCGCond)(c ^ 8);
471 default:
472 return c;
473 }
474}
475
476typedef enum TCGTempVal {
477 TEMP_VAL_DEAD,
478 TEMP_VAL_REG,
479 TEMP_VAL_MEM,
480 TEMP_VAL_CONST,
481} TCGTempVal;
482
483typedef struct TCGTemp {
484 TCGReg reg:8;
485 TCGTempVal val_type:8;
486 TCGType base_type:8;
487 TCGType type:8;
488 unsigned int fixed_reg:1;
489 unsigned int indirect_reg:1;
490 unsigned int indirect_base:1;
491 unsigned int mem_coherent:1;
492 unsigned int mem_allocated:1;
493
494
495 unsigned int temp_global:1;
496
497
498
499 unsigned int temp_local:1;
500 unsigned int temp_allocated:1;
501
502 tcg_target_long val;
503 struct TCGTemp *mem_base;
504 intptr_t mem_offset;
505 const char *name;
506
507
508
509
510 uintptr_t state;
511 void *state_ptr;
512} TCGTemp;
513
514typedef struct TCGContext TCGContext;
515
516typedef struct TCGTempSet {
517 unsigned long l[BITS_TO_LONGS(TCG_MAX_TEMPS)];
518} TCGTempSet;
519
520
521
522
523
524#define DEAD_ARG 4
525#define SYNC_ARG 1
526typedef uint16_t TCGLifeData;
527
528
529
530typedef struct TCGOp {
531 TCGOpcode opc : 8;
532
533
534 unsigned param1 : 4;
535 unsigned param2 : 4;
536
537
538 unsigned life : 16;
539
540
541 QTAILQ_ENTRY(TCGOp) link;
542#ifdef CONFIG_PLUGIN
543 QSIMPLEQ_ENTRY(TCGOp) plugin_link;
544#endif
545
546
547 TCGArg args[MAX_OPC_PARAM];
548
549
550 TCGRegSet output_pref[2];
551} TCGOp;
552
553#define TCGOP_CALLI(X) (X)->param1
554#define TCGOP_CALLO(X) (X)->param2
555
556#define TCGOP_VECL(X) (X)->param1
557#define TCGOP_VECE(X) (X)->param2
558
559
560QEMU_BUILD_BUG_ON(NB_OPS > (1 << 8));
561
562typedef struct TCGProfile {
563 int64_t cpu_exec_time;
564 int64_t tb_count1;
565 int64_t tb_count;
566 int64_t op_count;
567 int op_count_max;
568 int temp_count_max;
569 int64_t temp_count;
570 int64_t del_op_count;
571 int64_t code_in_len;
572 int64_t code_out_len;
573 int64_t search_out_len;
574 int64_t interm_time;
575 int64_t code_time;
576 int64_t la_time;
577 int64_t opt_time;
578 int64_t restore_count;
579 int64_t restore_time;
580 int64_t table_op_count[NB_OPS];
581} TCGProfile;
582
583struct TCGContext {
584 uint8_t *pool_cur, *pool_end;
585 TCGPool *pool_first, *pool_current, *pool_first_large;
586 int nb_labels;
587 int nb_globals;
588 int nb_temps;
589 int nb_indirects;
590 int nb_ops;
591
592
593 tcg_insn_unit *code_buf;
594 uint16_t *tb_jmp_reset_offset;
595 uintptr_t *tb_jmp_insn_offset;
596 uintptr_t *tb_jmp_target_addr;
597
598 TCGRegSet reserved_regs;
599 uint32_t tb_cflags;
600 intptr_t current_frame_offset;
601 intptr_t frame_start;
602 intptr_t frame_end;
603 TCGTemp *frame_temp;
604
605 tcg_insn_unit *code_ptr;
606
607#ifdef CONFIG_PROFILER
608 TCGProfile prof;
609#endif
610
611#ifdef CONFIG_DEBUG_TCG
612 int temps_in_use;
613 int goto_tb_issue_mask;
614 const TCGOpcode *vecop_list;
615#endif
616
617
618
619
620
621 void *code_gen_prologue;
622 void *code_gen_epilogue;
623 void *code_gen_buffer;
624 size_t code_gen_buffer_size;
625 void *code_gen_ptr;
626 void *data_gen_ptr;
627
628
629 void *code_gen_highwater;
630
631 size_t tb_phys_invalidate_count;
632
633
634 CPUState *cpu;
635
636
637#ifdef TCG_TARGET_NEED_LDST_LABELS
638 QSIMPLEQ_HEAD(, TCGLabelQemuLdst) ldst_labels;
639#endif
640#ifdef TCG_TARGET_NEED_POOL_LABELS
641 struct TCGLabelPoolData *pool_labels;
642#endif
643
644 TCGLabel *exitreq_label;
645
646#ifdef CONFIG_PLUGIN
647
648
649
650
651
652
653
654 struct qemu_plugin_tb *plugin_tb;
655
656
657 struct qemu_plugin_insn *plugin_insn;
658
659
660 QSIMPLEQ_HEAD(, TCGOp) plugin_ops;
661#endif
662
663 TCGTempSet free_temps[TCG_TYPE_COUNT * 2];
664 TCGTemp temps[TCG_MAX_TEMPS];
665
666 QTAILQ_HEAD(, TCGOp) ops, free_ops;
667 QSIMPLEQ_HEAD(, TCGLabel) labels;
668
669
670
671 TCGTemp *reg_to_temp[TCG_TARGET_NB_REGS];
672
673 uint16_t gen_insn_end_off[TCG_MAX_INSNS];
674 target_ulong gen_insn_data[TCG_MAX_INSNS][TARGET_INSN_START_WORDS];
675};
676
677extern TCGContext tcg_init_ctx;
678extern __thread TCGContext *tcg_ctx;
679extern TCGv_env cpu_env;
680
681static inline size_t temp_idx(TCGTemp *ts)
682{
683 ptrdiff_t n = ts - tcg_ctx->temps;
684 tcg_debug_assert(n >= 0 && n < tcg_ctx->nb_temps);
685 return n;
686}
687
688static inline TCGArg temp_arg(TCGTemp *ts)
689{
690 return (uintptr_t)ts;
691}
692
693static inline TCGTemp *arg_temp(TCGArg a)
694{
695 return (TCGTemp *)(uintptr_t)a;
696}
697
698
699
700
701static inline TCGTemp *tcgv_i32_temp(TCGv_i32 v)
702{
703 uintptr_t o = (uintptr_t)v;
704 TCGTemp *t = (void *)tcg_ctx + o;
705 tcg_debug_assert(offsetof(TCGContext, temps[temp_idx(t)]) == o);
706 return t;
707}
708
709static inline TCGTemp *tcgv_i64_temp(TCGv_i64 v)
710{
711 return tcgv_i32_temp((TCGv_i32)v);
712}
713
714static inline TCGTemp *tcgv_ptr_temp(TCGv_ptr v)
715{
716 return tcgv_i32_temp((TCGv_i32)v);
717}
718
719static inline TCGTemp *tcgv_vec_temp(TCGv_vec v)
720{
721 return tcgv_i32_temp((TCGv_i32)v);
722}
723
724static inline TCGArg tcgv_i32_arg(TCGv_i32 v)
725{
726 return temp_arg(tcgv_i32_temp(v));
727}
728
729static inline TCGArg tcgv_i64_arg(TCGv_i64 v)
730{
731 return temp_arg(tcgv_i64_temp(v));
732}
733
734static inline TCGArg tcgv_ptr_arg(TCGv_ptr v)
735{
736 return temp_arg(tcgv_ptr_temp(v));
737}
738
739static inline TCGArg tcgv_vec_arg(TCGv_vec v)
740{
741 return temp_arg(tcgv_vec_temp(v));
742}
743
744static inline TCGv_i32 temp_tcgv_i32(TCGTemp *t)
745{
746 (void)temp_idx(t);
747 return (TCGv_i32)((void *)t - (void *)tcg_ctx);
748}
749
750static inline TCGv_i64 temp_tcgv_i64(TCGTemp *t)
751{
752 return (TCGv_i64)temp_tcgv_i32(t);
753}
754
755static inline TCGv_ptr temp_tcgv_ptr(TCGTemp *t)
756{
757 return (TCGv_ptr)temp_tcgv_i32(t);
758}
759
760static inline TCGv_vec temp_tcgv_vec(TCGTemp *t)
761{
762 return (TCGv_vec)temp_tcgv_i32(t);
763}
764
765#if TCG_TARGET_REG_BITS == 32
766static inline TCGv_i32 TCGV_LOW(TCGv_i64 t)
767{
768 return temp_tcgv_i32(tcgv_i64_temp(t));
769}
770
771static inline TCGv_i32 TCGV_HIGH(TCGv_i64 t)
772{
773 return temp_tcgv_i32(tcgv_i64_temp(t) + 1);
774}
775#endif
776
777static inline void tcg_set_insn_param(TCGOp *op, int arg, TCGArg v)
778{
779 op->args[arg] = v;
780}
781
782static inline void tcg_set_insn_start_param(TCGOp *op, int arg, target_ulong v)
783{
784#if TARGET_LONG_BITS <= TCG_TARGET_REG_BITS
785 tcg_set_insn_param(op, arg, v);
786#else
787 tcg_set_insn_param(op, arg * 2, v);
788 tcg_set_insn_param(op, arg * 2 + 1, v >> 32);
789#endif
790}
791
792
793static inline TCGOp *tcg_last_op(void)
794{
795 return QTAILQ_LAST(&tcg_ctx->ops);
796}
797
798
799static inline bool tcg_op_buf_full(void)
800{
801
802
803
804
805
806
807
808 return tcg_ctx->nb_ops >= 4000;
809}
810
811
812
813
814void *tcg_malloc_internal(TCGContext *s, int size);
815void tcg_pool_reset(TCGContext *s);
816TranslationBlock *tcg_tb_alloc(TCGContext *s);
817
818void tcg_region_init(void);
819void tcg_region_reset_all(void);
820
821size_t tcg_code_size(void);
822size_t tcg_code_capacity(void);
823
824void tcg_tb_insert(TranslationBlock *tb);
825void tcg_tb_remove(TranslationBlock *tb);
826size_t tcg_tb_phys_invalidate_count(void);
827TranslationBlock *tcg_tb_lookup(uintptr_t tc_ptr);
828void tcg_tb_foreach(GTraverseFunc func, gpointer user_data);
829size_t tcg_nb_tbs(void);
830
831
832static inline void *tcg_malloc(int size)
833{
834 TCGContext *s = tcg_ctx;
835 uint8_t *ptr, *ptr_end;
836
837
838 size = QEMU_ALIGN_UP(size, 8);
839
840 ptr = s->pool_cur;
841 ptr_end = ptr + size;
842 if (unlikely(ptr_end > s->pool_end)) {
843 return tcg_malloc_internal(tcg_ctx, size);
844 } else {
845 s->pool_cur = ptr_end;
846 return ptr;
847 }
848}
849
850void tcg_context_init(TCGContext *s);
851void tcg_register_thread(void);
852void tcg_prologue_init(TCGContext *s);
853void tcg_func_start(TCGContext *s);
854
855int tcg_gen_code(TCGContext *s, TranslationBlock *tb);
856
857void tcg_set_frame(TCGContext *s, TCGReg reg, intptr_t start, intptr_t size);
858
859TCGTemp *tcg_global_mem_new_internal(TCGType, TCGv_ptr,
860 intptr_t, const char *);
861TCGTemp *tcg_temp_new_internal(TCGType, bool);
862void tcg_temp_free_internal(TCGTemp *);
863TCGv_vec tcg_temp_new_vec(TCGType type);
864TCGv_vec tcg_temp_new_vec_matching(TCGv_vec match);
865
866static inline void tcg_temp_free_i32(TCGv_i32 arg)
867{
868 tcg_temp_free_internal(tcgv_i32_temp(arg));
869}
870
871static inline void tcg_temp_free_i64(TCGv_i64 arg)
872{
873 tcg_temp_free_internal(tcgv_i64_temp(arg));
874}
875
876static inline void tcg_temp_free_ptr(TCGv_ptr arg)
877{
878 tcg_temp_free_internal(tcgv_ptr_temp(arg));
879}
880
881static inline void tcg_temp_free_vec(TCGv_vec arg)
882{
883 tcg_temp_free_internal(tcgv_vec_temp(arg));
884}
885
886static inline TCGv_i32 tcg_global_mem_new_i32(TCGv_ptr reg, intptr_t offset,
887 const char *name)
888{
889 TCGTemp *t = tcg_global_mem_new_internal(TCG_TYPE_I32, reg, offset, name);
890 return temp_tcgv_i32(t);
891}
892
893static inline TCGv_i32 tcg_temp_new_i32(void)
894{
895 TCGTemp *t = tcg_temp_new_internal(TCG_TYPE_I32, false);
896 return temp_tcgv_i32(t);
897}
898
899static inline TCGv_i32 tcg_temp_local_new_i32(void)
900{
901 TCGTemp *t = tcg_temp_new_internal(TCG_TYPE_I32, true);
902 return temp_tcgv_i32(t);
903}
904
905static inline TCGv_i64 tcg_global_mem_new_i64(TCGv_ptr reg, intptr_t offset,
906 const char *name)
907{
908 TCGTemp *t = tcg_global_mem_new_internal(TCG_TYPE_I64, reg, offset, name);
909 return temp_tcgv_i64(t);
910}
911
912static inline TCGv_i64 tcg_temp_new_i64(void)
913{
914 TCGTemp *t = tcg_temp_new_internal(TCG_TYPE_I64, false);
915 return temp_tcgv_i64(t);
916}
917
918static inline TCGv_i64 tcg_temp_local_new_i64(void)
919{
920 TCGTemp *t = tcg_temp_new_internal(TCG_TYPE_I64, true);
921 return temp_tcgv_i64(t);
922}
923
924static inline TCGv_ptr tcg_global_mem_new_ptr(TCGv_ptr reg, intptr_t offset,
925 const char *name)
926{
927 TCGTemp *t = tcg_global_mem_new_internal(TCG_TYPE_PTR, reg, offset, name);
928 return temp_tcgv_ptr(t);
929}
930
931static inline TCGv_ptr tcg_temp_new_ptr(void)
932{
933 TCGTemp *t = tcg_temp_new_internal(TCG_TYPE_PTR, false);
934 return temp_tcgv_ptr(t);
935}
936
937static inline TCGv_ptr tcg_temp_local_new_ptr(void)
938{
939 TCGTemp *t = tcg_temp_new_internal(TCG_TYPE_PTR, true);
940 return temp_tcgv_ptr(t);
941}
942
943#if defined(CONFIG_DEBUG_TCG)
944
945
946
947
948
949void tcg_clear_temp_count(void);
950int tcg_check_temp_count(void);
951#else
952#define tcg_clear_temp_count() do { } while (0)
953#define tcg_check_temp_count() 0
954#endif
955
956int64_t tcg_cpu_exec_time(void);
957void tcg_dump_info(void);
958void tcg_dump_op_count(void);
959
960#define TCG_CT_ALIAS 0x80
961#define TCG_CT_IALIAS 0x40
962#define TCG_CT_NEWREG 0x20
963#define TCG_CT_REG 0x01
964#define TCG_CT_CONST 0x02
965
966typedef struct TCGArgConstraint {
967 uint16_t ct;
968 uint8_t alias_index;
969 union {
970 TCGRegSet regs;
971 } u;
972} TCGArgConstraint;
973
974#define TCG_MAX_OP_ARGS 16
975
976
977enum {
978
979 TCG_OPF_BB_EXIT = 0x01,
980
981 TCG_OPF_BB_END = 0x02,
982
983 TCG_OPF_CALL_CLOBBER = 0x04,
984
985
986 TCG_OPF_SIDE_EFFECTS = 0x08,
987
988 TCG_OPF_64BIT = 0x10,
989
990
991 TCG_OPF_NOT_PRESENT = 0x20,
992
993 TCG_OPF_VECTOR = 0x40,
994};
995
996typedef struct TCGOpDef {
997 const char *name;
998 uint8_t nb_oargs, nb_iargs, nb_cargs, nb_args;
999 uint8_t flags;
1000 TCGArgConstraint *args_ct;
1001 int *sorted_args;
1002#if defined(CONFIG_DEBUG_TCG)
1003 int used;
1004#endif
1005} TCGOpDef;
1006
1007extern TCGOpDef tcg_op_defs[];
1008extern const size_t tcg_op_defs_max;
1009
1010typedef struct TCGTargetOpDef {
1011 TCGOpcode op;
1012 const char *args_ct_str[TCG_MAX_OP_ARGS];
1013} TCGTargetOpDef;
1014
1015#define tcg_abort() \
1016do {\
1017 fprintf(stderr, "%s:%d: tcg fatal error\n", __FILE__, __LINE__);\
1018 abort();\
1019} while (0)
1020
1021bool tcg_op_supported(TCGOpcode op);
1022
1023void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args);
1024
1025TCGOp *tcg_emit_op(TCGOpcode opc);
1026void tcg_op_remove(TCGContext *s, TCGOp *op);
1027TCGOp *tcg_op_insert_before(TCGContext *s, TCGOp *op, TCGOpcode opc);
1028TCGOp *tcg_op_insert_after(TCGContext *s, TCGOp *op, TCGOpcode opc);
1029
1030void tcg_optimize(TCGContext *s);
1031
1032TCGv_i32 tcg_const_i32(int32_t val);
1033TCGv_i64 tcg_const_i64(int64_t val);
1034TCGv_i32 tcg_const_local_i32(int32_t val);
1035TCGv_i64 tcg_const_local_i64(int64_t val);
1036TCGv_vec tcg_const_zeros_vec(TCGType);
1037TCGv_vec tcg_const_ones_vec(TCGType);
1038TCGv_vec tcg_const_zeros_vec_matching(TCGv_vec);
1039TCGv_vec tcg_const_ones_vec_matching(TCGv_vec);
1040
1041#if UINTPTR_MAX == UINT32_MAX
1042# define tcg_const_ptr(x) ((TCGv_ptr)tcg_const_i32((intptr_t)(x)))
1043# define tcg_const_local_ptr(x) ((TCGv_ptr)tcg_const_local_i32((intptr_t)(x)))
1044#else
1045# define tcg_const_ptr(x) ((TCGv_ptr)tcg_const_i64((intptr_t)(x)))
1046# define tcg_const_local_ptr(x) ((TCGv_ptr)tcg_const_local_i64((intptr_t)(x)))
1047#endif
1048
1049TCGLabel *gen_new_label(void);
1050
1051
1052
1053
1054
1055
1056
1057
1058static inline TCGArg label_arg(TCGLabel *l)
1059{
1060 return (uintptr_t)l;
1061}
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071static inline TCGLabel *arg_label(TCGArg i)
1072{
1073 return (TCGLabel *)(uintptr_t)i;
1074}
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089static inline ptrdiff_t tcg_ptr_byte_diff(void *a, void *b)
1090{
1091 return a - b;
1092}
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103static inline ptrdiff_t tcg_pcrel_diff(TCGContext *s, void *target)
1104{
1105 return tcg_ptr_byte_diff(target, s->code_ptr);
1106}
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116static inline size_t tcg_current_code_size(TCGContext *s)
1117{
1118 return tcg_ptr_byte_diff(s->code_ptr, s->code_buf);
1119}
1120
1121
1122typedef uint32_t TCGMemOpIdx;
1123
1124
1125
1126
1127
1128
1129
1130
1131static inline TCGMemOpIdx make_memop_idx(MemOp op, unsigned idx)
1132{
1133 tcg_debug_assert(idx <= 15);
1134 return (op << 4) | idx;
1135}
1136
1137
1138
1139
1140
1141
1142
1143static inline MemOp get_memop(TCGMemOpIdx oi)
1144{
1145 return oi >> 4;
1146}
1147
1148
1149
1150
1151
1152
1153
1154static inline unsigned get_mmuidx(TCGMemOpIdx oi)
1155{
1156 return oi & 15;
1157}
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204#define TB_EXIT_MASK 3
1205#define TB_EXIT_IDX0 0
1206#define TB_EXIT_IDX1 1
1207#define TB_EXIT_IDXMAX 1
1208#define TB_EXIT_REQUESTED 3
1209
1210#ifdef HAVE_TCG_QEMU_TB_EXEC
1211uintptr_t tcg_qemu_tb_exec(CPUArchState *env, uint8_t *tb_ptr);
1212#else
1213# define tcg_qemu_tb_exec(env, tb_ptr) \
1214 ((uintptr_t (*)(void *, void *))tcg_ctx->code_gen_prologue)(env, tb_ptr)
1215#endif
1216
1217void tcg_register_jit(void *buf, size_t buf_size);
1218
1219#if TCG_TARGET_MAYBE_vec
1220
1221
1222
1223int tcg_can_emit_vec_op(TCGOpcode, TCGType, unsigned);
1224#else
1225static inline int tcg_can_emit_vec_op(TCGOpcode o, TCGType t, unsigned ve)
1226{
1227 return 0;
1228}
1229#endif
1230
1231
1232void tcg_expand_vec_op(TCGOpcode, TCGType, unsigned, TCGArg, ...);
1233
1234
1235uint64_t dup_const(unsigned vece, uint64_t c);
1236
1237#define dup_const(VECE, C) \
1238 (__builtin_constant_p(VECE) \
1239 ? ( (VECE) == MO_8 ? 0x0101010101010101ull * (uint8_t)(C) \
1240 : (VECE) == MO_16 ? 0x0001000100010001ull * (uint16_t)(C) \
1241 : (VECE) == MO_32 ? 0x0000000100000001ull * (uint32_t)(C) \
1242 : dup_const(VECE, C)) \
1243 : dup_const(VECE, C))
1244
1245
1246
1247
1248
1249#ifdef CONFIG_SOFTMMU
1250
1251tcg_target_ulong helper_ret_ldub_mmu(CPUArchState *env, target_ulong addr,
1252 TCGMemOpIdx oi, uintptr_t retaddr);
1253tcg_target_ulong helper_le_lduw_mmu(CPUArchState *env, target_ulong addr,
1254 TCGMemOpIdx oi, uintptr_t retaddr);
1255tcg_target_ulong helper_le_ldul_mmu(CPUArchState *env, target_ulong addr,
1256 TCGMemOpIdx oi, uintptr_t retaddr);
1257uint64_t helper_le_ldq_mmu(CPUArchState *env, target_ulong addr,
1258 TCGMemOpIdx oi, uintptr_t retaddr);
1259tcg_target_ulong helper_be_lduw_mmu(CPUArchState *env, target_ulong addr,
1260 TCGMemOpIdx oi, uintptr_t retaddr);
1261tcg_target_ulong helper_be_ldul_mmu(CPUArchState *env, target_ulong addr,
1262 TCGMemOpIdx oi, uintptr_t retaddr);
1263uint64_t helper_be_ldq_mmu(CPUArchState *env, target_ulong addr,
1264 TCGMemOpIdx oi, uintptr_t retaddr);
1265
1266
1267tcg_target_ulong helper_ret_ldsb_mmu(CPUArchState *env, target_ulong addr,
1268 TCGMemOpIdx oi, uintptr_t retaddr);
1269tcg_target_ulong helper_le_ldsw_mmu(CPUArchState *env, target_ulong addr,
1270 TCGMemOpIdx oi, uintptr_t retaddr);
1271tcg_target_ulong helper_le_ldsl_mmu(CPUArchState *env, target_ulong addr,
1272 TCGMemOpIdx oi, uintptr_t retaddr);
1273tcg_target_ulong helper_be_ldsw_mmu(CPUArchState *env, target_ulong addr,
1274 TCGMemOpIdx oi, uintptr_t retaddr);
1275tcg_target_ulong helper_be_ldsl_mmu(CPUArchState *env, target_ulong addr,
1276 TCGMemOpIdx oi, uintptr_t retaddr);
1277
1278void helper_ret_stb_mmu(CPUArchState *env, target_ulong addr, uint8_t val,
1279 TCGMemOpIdx oi, uintptr_t retaddr);
1280void helper_le_stw_mmu(CPUArchState *env, target_ulong addr, uint16_t val,
1281 TCGMemOpIdx oi, uintptr_t retaddr);
1282void helper_le_stl_mmu(CPUArchState *env, target_ulong addr, uint32_t val,
1283 TCGMemOpIdx oi, uintptr_t retaddr);
1284void helper_le_stq_mmu(CPUArchState *env, target_ulong addr, uint64_t val,
1285 TCGMemOpIdx oi, uintptr_t retaddr);
1286void helper_be_stw_mmu(CPUArchState *env, target_ulong addr, uint16_t val,
1287 TCGMemOpIdx oi, uintptr_t retaddr);
1288void helper_be_stl_mmu(CPUArchState *env, target_ulong addr, uint32_t val,
1289 TCGMemOpIdx oi, uintptr_t retaddr);
1290void helper_be_stq_mmu(CPUArchState *env, target_ulong addr, uint64_t val,
1291 TCGMemOpIdx oi, uintptr_t retaddr);
1292
1293
1294#ifdef TARGET_WORDS_BIGENDIAN
1295# define helper_ret_ldsw_mmu helper_be_ldsw_mmu
1296# define helper_ret_lduw_mmu helper_be_lduw_mmu
1297# define helper_ret_ldsl_mmu helper_be_ldsl_mmu
1298# define helper_ret_ldul_mmu helper_be_ldul_mmu
1299# define helper_ret_ldl_mmu helper_be_ldul_mmu
1300# define helper_ret_ldq_mmu helper_be_ldq_mmu
1301# define helper_ret_stw_mmu helper_be_stw_mmu
1302# define helper_ret_stl_mmu helper_be_stl_mmu
1303# define helper_ret_stq_mmu helper_be_stq_mmu
1304#else
1305# define helper_ret_ldsw_mmu helper_le_ldsw_mmu
1306# define helper_ret_lduw_mmu helper_le_lduw_mmu
1307# define helper_ret_ldsl_mmu helper_le_ldsl_mmu
1308# define helper_ret_ldul_mmu helper_le_ldul_mmu
1309# define helper_ret_ldl_mmu helper_le_ldul_mmu
1310# define helper_ret_ldq_mmu helper_le_ldq_mmu
1311# define helper_ret_stw_mmu helper_le_stw_mmu
1312# define helper_ret_stl_mmu helper_le_stl_mmu
1313# define helper_ret_stq_mmu helper_le_stq_mmu
1314#endif
1315
1316uint32_t helper_atomic_cmpxchgb_mmu(CPUArchState *env, target_ulong addr,
1317 uint32_t cmpv, uint32_t newv,
1318 TCGMemOpIdx oi, uintptr_t retaddr);
1319uint32_t helper_atomic_cmpxchgw_le_mmu(CPUArchState *env, target_ulong addr,
1320 uint32_t cmpv, uint32_t newv,
1321 TCGMemOpIdx oi, uintptr_t retaddr);
1322uint32_t helper_atomic_cmpxchgl_le_mmu(CPUArchState *env, target_ulong addr,
1323 uint32_t cmpv, uint32_t newv,
1324 TCGMemOpIdx oi, uintptr_t retaddr);
1325uint64_t helper_atomic_cmpxchgq_le_mmu(CPUArchState *env, target_ulong addr,
1326 uint64_t cmpv, uint64_t newv,
1327 TCGMemOpIdx oi, uintptr_t retaddr);
1328uint32_t helper_atomic_cmpxchgw_be_mmu(CPUArchState *env, target_ulong addr,
1329 uint32_t cmpv, uint32_t newv,
1330 TCGMemOpIdx oi, uintptr_t retaddr);
1331uint32_t helper_atomic_cmpxchgl_be_mmu(CPUArchState *env, target_ulong addr,
1332 uint32_t cmpv, uint32_t newv,
1333 TCGMemOpIdx oi, uintptr_t retaddr);
1334uint64_t helper_atomic_cmpxchgq_be_mmu(CPUArchState *env, target_ulong addr,
1335 uint64_t cmpv, uint64_t newv,
1336 TCGMemOpIdx oi, uintptr_t retaddr);
1337
1338#define GEN_ATOMIC_HELPER(NAME, TYPE, SUFFIX) \
1339TYPE helper_atomic_ ## NAME ## SUFFIX ## _mmu \
1340 (CPUArchState *env, target_ulong addr, TYPE val, \
1341 TCGMemOpIdx oi, uintptr_t retaddr);
1342
1343#ifdef CONFIG_ATOMIC64
1344#define GEN_ATOMIC_HELPER_ALL(NAME) \
1345 GEN_ATOMIC_HELPER(NAME, uint32_t, b) \
1346 GEN_ATOMIC_HELPER(NAME, uint32_t, w_le) \
1347 GEN_ATOMIC_HELPER(NAME, uint32_t, w_be) \
1348 GEN_ATOMIC_HELPER(NAME, uint32_t, l_le) \
1349 GEN_ATOMIC_HELPER(NAME, uint32_t, l_be) \
1350 GEN_ATOMIC_HELPER(NAME, uint64_t, q_le) \
1351 GEN_ATOMIC_HELPER(NAME, uint64_t, q_be)
1352#else
1353#define GEN_ATOMIC_HELPER_ALL(NAME) \
1354 GEN_ATOMIC_HELPER(NAME, uint32_t, b) \
1355 GEN_ATOMIC_HELPER(NAME, uint32_t, w_le) \
1356 GEN_ATOMIC_HELPER(NAME, uint32_t, w_be) \
1357 GEN_ATOMIC_HELPER(NAME, uint32_t, l_le) \
1358 GEN_ATOMIC_HELPER(NAME, uint32_t, l_be)
1359#endif
1360
1361GEN_ATOMIC_HELPER_ALL(fetch_add)
1362GEN_ATOMIC_HELPER_ALL(fetch_sub)
1363GEN_ATOMIC_HELPER_ALL(fetch_and)
1364GEN_ATOMIC_HELPER_ALL(fetch_or)
1365GEN_ATOMIC_HELPER_ALL(fetch_xor)
1366GEN_ATOMIC_HELPER_ALL(fetch_smin)
1367GEN_ATOMIC_HELPER_ALL(fetch_umin)
1368GEN_ATOMIC_HELPER_ALL(fetch_smax)
1369GEN_ATOMIC_HELPER_ALL(fetch_umax)
1370
1371GEN_ATOMIC_HELPER_ALL(add_fetch)
1372GEN_ATOMIC_HELPER_ALL(sub_fetch)
1373GEN_ATOMIC_HELPER_ALL(and_fetch)
1374GEN_ATOMIC_HELPER_ALL(or_fetch)
1375GEN_ATOMIC_HELPER_ALL(xor_fetch)
1376GEN_ATOMIC_HELPER_ALL(smin_fetch)
1377GEN_ATOMIC_HELPER_ALL(umin_fetch)
1378GEN_ATOMIC_HELPER_ALL(smax_fetch)
1379GEN_ATOMIC_HELPER_ALL(umax_fetch)
1380
1381GEN_ATOMIC_HELPER_ALL(xchg)
1382
1383#undef GEN_ATOMIC_HELPER_ALL
1384#undef GEN_ATOMIC_HELPER
1385#endif
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395Int128 helper_atomic_cmpxchgo_le_mmu(CPUArchState *env, target_ulong addr,
1396 Int128 cmpv, Int128 newv,
1397 TCGMemOpIdx oi, uintptr_t retaddr);
1398Int128 helper_atomic_cmpxchgo_be_mmu(CPUArchState *env, target_ulong addr,
1399 Int128 cmpv, Int128 newv,
1400 TCGMemOpIdx oi, uintptr_t retaddr);
1401
1402Int128 helper_atomic_ldo_le_mmu(CPUArchState *env, target_ulong addr,
1403 TCGMemOpIdx oi, uintptr_t retaddr);
1404Int128 helper_atomic_ldo_be_mmu(CPUArchState *env, target_ulong addr,
1405 TCGMemOpIdx oi, uintptr_t retaddr);
1406void helper_atomic_sto_le_mmu(CPUArchState *env, target_ulong addr, Int128 val,
1407 TCGMemOpIdx oi, uintptr_t retaddr);
1408void helper_atomic_sto_be_mmu(CPUArchState *env, target_ulong addr, Int128 val,
1409 TCGMemOpIdx oi, uintptr_t retaddr);
1410
1411#ifdef CONFIG_DEBUG_TCG
1412void tcg_assert_listed_vecop(TCGOpcode);
1413#else
1414static inline void tcg_assert_listed_vecop(TCGOpcode op) { }
1415#endif
1416
1417static inline const TCGOpcode *tcg_swap_vecop_list(const TCGOpcode *n)
1418{
1419#ifdef CONFIG_DEBUG_TCG
1420 const TCGOpcode *o = tcg_ctx->vecop_list;
1421 tcg_ctx->vecop_list = n;
1422 return o;
1423#else
1424 return NULL;
1425#endif
1426}
1427
1428bool tcg_can_emit_vecop_list(const TCGOpcode *, TCGType, unsigned);
1429
1430#endif
1431