1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25#ifndef TCG_H
26#define TCG_H
27
28#include "exec/memop.h"
29#include "exec/memopidx.h"
30#include "qemu/bitops.h"
31#include "qemu/plugin.h"
32#include "qemu/queue.h"
33#include "tcg/tcg-mo.h"
34#include "tcg-target-reg-bits.h"
35#include "tcg-target.h"
36#include "tcg/tcg-cond.h"
37#include "tcg/debug-assert.h"
38
39
40#define MAX_OP_PER_INSTR 266
41
42#define MAX_CALL_IARGS 7
43
44#define CPU_TEMP_BUF_NLONGS 128
45#define TCG_STATIC_FRAME_SIZE (CPU_TEMP_BUF_NLONGS * sizeof(long))
46
47#if TCG_TARGET_REG_BITS == 32
48typedef int32_t tcg_target_long;
49typedef uint32_t tcg_target_ulong;
50#define TCG_PRIlx PRIx32
51#define TCG_PRIld PRId32
52#elif TCG_TARGET_REG_BITS == 64
53typedef int64_t tcg_target_long;
54typedef uint64_t tcg_target_ulong;
55#define TCG_PRIlx PRIx64
56#define TCG_PRIld PRId64
57#else
58#error unsupported
59#endif
60
61#if TCG_TARGET_NB_REGS <= 32
62typedef uint32_t TCGRegSet;
63#elif TCG_TARGET_NB_REGS <= 64
64typedef uint64_t TCGRegSet;
65#else
66#error unsupported
67#endif
68
69#if TCG_TARGET_REG_BITS == 32
70
71#define TCG_TARGET_HAS_extrl_i64_i32 0
72#define TCG_TARGET_HAS_extrh_i64_i32 0
73#define TCG_TARGET_HAS_div_i64 0
74#define TCG_TARGET_HAS_rem_i64 0
75#define TCG_TARGET_HAS_div2_i64 0
76#define TCG_TARGET_HAS_rot_i64 0
77#define TCG_TARGET_HAS_ext8s_i64 0
78#define TCG_TARGET_HAS_ext16s_i64 0
79#define TCG_TARGET_HAS_ext32s_i64 0
80#define TCG_TARGET_HAS_ext8u_i64 0
81#define TCG_TARGET_HAS_ext16u_i64 0
82#define TCG_TARGET_HAS_ext32u_i64 0
83#define TCG_TARGET_HAS_bswap16_i64 0
84#define TCG_TARGET_HAS_bswap32_i64 0
85#define TCG_TARGET_HAS_bswap64_i64 0
86#define TCG_TARGET_HAS_neg_i64 0
87#define TCG_TARGET_HAS_not_i64 0
88#define TCG_TARGET_HAS_andc_i64 0
89#define TCG_TARGET_HAS_orc_i64 0
90#define TCG_TARGET_HAS_eqv_i64 0
91#define TCG_TARGET_HAS_nand_i64 0
92#define TCG_TARGET_HAS_nor_i64 0
93#define TCG_TARGET_HAS_clz_i64 0
94#define TCG_TARGET_HAS_ctz_i64 0
95#define TCG_TARGET_HAS_ctpop_i64 0
96#define TCG_TARGET_HAS_deposit_i64 0
97#define TCG_TARGET_HAS_extract_i64 0
98#define TCG_TARGET_HAS_sextract_i64 0
99#define TCG_TARGET_HAS_extract2_i64 0
100#define TCG_TARGET_HAS_movcond_i64 0
101#define TCG_TARGET_HAS_add2_i64 0
102#define TCG_TARGET_HAS_sub2_i64 0
103#define TCG_TARGET_HAS_mulu2_i64 0
104#define TCG_TARGET_HAS_muls2_i64 0
105#define TCG_TARGET_HAS_muluh_i64 0
106#define TCG_TARGET_HAS_mulsh_i64 0
107
108#define TCG_TARGET_HAS_add2_i32 1
109#define TCG_TARGET_HAS_sub2_i32 1
110#endif
111
112#ifndef TCG_TARGET_deposit_i32_valid
113#define TCG_TARGET_deposit_i32_valid(ofs, len) 1
114#endif
115#ifndef TCG_TARGET_deposit_i64_valid
116#define TCG_TARGET_deposit_i64_valid(ofs, len) 1
117#endif
118#ifndef TCG_TARGET_extract_i32_valid
119#define TCG_TARGET_extract_i32_valid(ofs, len) 1
120#endif
121#ifndef TCG_TARGET_extract_i64_valid
122#define TCG_TARGET_extract_i64_valid(ofs, len) 1
123#endif
124
125
126#if defined(TCG_TARGET_HAS_div_i32)
127#define TCG_TARGET_HAS_div2_i32 0
128#elif defined(TCG_TARGET_HAS_div2_i32)
129#define TCG_TARGET_HAS_div_i32 0
130#define TCG_TARGET_HAS_rem_i32 0
131#endif
132#if defined(TCG_TARGET_HAS_div_i64)
133#define TCG_TARGET_HAS_div2_i64 0
134#elif defined(TCG_TARGET_HAS_div2_i64)
135#define TCG_TARGET_HAS_div_i64 0
136#define TCG_TARGET_HAS_rem_i64 0
137#endif
138
139#if !defined(TCG_TARGET_HAS_v64) \
140 && !defined(TCG_TARGET_HAS_v128) \
141 && !defined(TCG_TARGET_HAS_v256)
142#define TCG_TARGET_MAYBE_vec 0
143#define TCG_TARGET_HAS_abs_vec 0
144#define TCG_TARGET_HAS_neg_vec 0
145#define TCG_TARGET_HAS_not_vec 0
146#define TCG_TARGET_HAS_andc_vec 0
147#define TCG_TARGET_HAS_orc_vec 0
148#define TCG_TARGET_HAS_nand_vec 0
149#define TCG_TARGET_HAS_nor_vec 0
150#define TCG_TARGET_HAS_eqv_vec 0
151#define TCG_TARGET_HAS_roti_vec 0
152#define TCG_TARGET_HAS_rots_vec 0
153#define TCG_TARGET_HAS_rotv_vec 0
154#define TCG_TARGET_HAS_shi_vec 0
155#define TCG_TARGET_HAS_shs_vec 0
156#define TCG_TARGET_HAS_shv_vec 0
157#define TCG_TARGET_HAS_mul_vec 0
158#define TCG_TARGET_HAS_sat_vec 0
159#define TCG_TARGET_HAS_minmax_vec 0
160#define TCG_TARGET_HAS_bitsel_vec 0
161#define TCG_TARGET_HAS_cmpsel_vec 0
162#else
163#define TCG_TARGET_MAYBE_vec 1
164#endif
165#ifndef TCG_TARGET_HAS_v64
166#define TCG_TARGET_HAS_v64 0
167#endif
168#ifndef TCG_TARGET_HAS_v128
169#define TCG_TARGET_HAS_v128 0
170#endif
171#ifndef TCG_TARGET_HAS_v256
172#define TCG_TARGET_HAS_v256 0
173#endif
174
175typedef enum TCGOpcode {
176#define DEF(name, oargs, iargs, cargs, flags) INDEX_op_ ## name,
177#include "tcg/tcg-opc.h"
178#undef DEF
179 NB_OPS,
180} TCGOpcode;
181
182#define tcg_regset_set_reg(d, r) ((d) |= (TCGRegSet)1 << (r))
183#define tcg_regset_reset_reg(d, r) ((d) &= ~((TCGRegSet)1 << (r)))
184#define tcg_regset_test_reg(d, r) (((d) >> (r)) & 1)
185
186#ifndef TCG_TARGET_INSN_UNIT_SIZE
187# error "Missing TCG_TARGET_INSN_UNIT_SIZE"
188#elif TCG_TARGET_INSN_UNIT_SIZE == 1
189typedef uint8_t tcg_insn_unit;
190#elif TCG_TARGET_INSN_UNIT_SIZE == 2
191typedef uint16_t tcg_insn_unit;
192#elif TCG_TARGET_INSN_UNIT_SIZE == 4
193typedef uint32_t tcg_insn_unit;
194#elif TCG_TARGET_INSN_UNIT_SIZE == 8
195typedef uint64_t tcg_insn_unit;
196#else
197
198#endif
199
200typedef struct TCGRelocation TCGRelocation;
201struct TCGRelocation {
202 QSIMPLEQ_ENTRY(TCGRelocation) next;
203 tcg_insn_unit *ptr;
204 intptr_t addend;
205 int type;
206};
207
208typedef struct TCGOp TCGOp;
209typedef struct TCGLabelUse TCGLabelUse;
210struct TCGLabelUse {
211 QSIMPLEQ_ENTRY(TCGLabelUse) next;
212 TCGOp *op;
213};
214
215typedef struct TCGLabel TCGLabel;
216struct TCGLabel {
217 bool present;
218 bool has_value;
219 uint16_t id;
220 union {
221 uintptr_t value;
222 const tcg_insn_unit *value_ptr;
223 } u;
224 QSIMPLEQ_HEAD(, TCGLabelUse) branches;
225 QSIMPLEQ_HEAD(, TCGRelocation) relocs;
226 QSIMPLEQ_ENTRY(TCGLabel) next;
227};
228
229typedef struct TCGPool {
230 struct TCGPool *next;
231 int size;
232 uint8_t data[] __attribute__ ((aligned));
233} TCGPool;
234
235#define TCG_POOL_CHUNK_SIZE 32768
236
237#define TCG_MAX_TEMPS 512
238#define TCG_MAX_INSNS 512
239
240
241
242#define TCG_STATIC_CALL_ARGS_SIZE 128
243
244typedef enum TCGType {
245 TCG_TYPE_I32,
246 TCG_TYPE_I64,
247 TCG_TYPE_I128,
248
249 TCG_TYPE_V64,
250 TCG_TYPE_V128,
251 TCG_TYPE_V256,
252
253
254#define TCG_TYPE_COUNT (TCG_TYPE_V256 + 1)
255
256
257#if TCG_TARGET_REG_BITS == 32
258 TCG_TYPE_REG = TCG_TYPE_I32,
259#else
260 TCG_TYPE_REG = TCG_TYPE_I64,
261#endif
262
263
264#if UINTPTR_MAX == UINT32_MAX
265 TCG_TYPE_PTR = TCG_TYPE_I32,
266#else
267 TCG_TYPE_PTR = TCG_TYPE_I64,
268#endif
269} TCGType;
270
271
272
273
274
275
276
277static inline int tcg_type_size(TCGType t)
278{
279 unsigned i = t;
280 if (i >= TCG_TYPE_V64) {
281 tcg_debug_assert(i < TCG_TYPE_COUNT);
282 i -= TCG_TYPE_V64 - 1;
283 }
284 return 4 << i;
285}
286
287
288
289
290
291
292
293static inline unsigned get_alignment_bits(MemOp memop)
294{
295 unsigned a = memop & MO_AMASK;
296
297 if (a == MO_UNALN) {
298
299 a = 0;
300 } else if (a == MO_ALIGN) {
301
302 a = memop & MO_SIZE;
303 } else {
304
305 a = a >> MO_ASHIFT;
306 }
307 return a;
308}
309
310typedef tcg_target_ulong TCGArg;
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343typedef struct TCGv_i32_d *TCGv_i32;
344typedef struct TCGv_i64_d *TCGv_i64;
345typedef struct TCGv_i128_d *TCGv_i128;
346typedef struct TCGv_ptr_d *TCGv_ptr;
347typedef struct TCGv_vec_d *TCGv_vec;
348typedef TCGv_ptr TCGv_env;
349
350
351
352
353#define TCG_CALL_NO_READ_GLOBALS 0x0001
354
355#define TCG_CALL_NO_WRITE_GLOBALS 0x0002
356
357#define TCG_CALL_NO_SIDE_EFFECTS 0x0004
358
359#define TCG_CALL_NO_RETURN 0x0008
360
361#define TCG_CALL_PLUGIN 0x0010
362
363
364#define TCG_CALL_NO_RWG TCG_CALL_NO_READ_GLOBALS
365#define TCG_CALL_NO_WG TCG_CALL_NO_WRITE_GLOBALS
366#define TCG_CALL_NO_SE TCG_CALL_NO_SIDE_EFFECTS
367#define TCG_CALL_NO_RWG_SE (TCG_CALL_NO_RWG | TCG_CALL_NO_SE)
368#define TCG_CALL_NO_WG_SE (TCG_CALL_NO_WG | TCG_CALL_NO_SE)
369
370
371
372
373
374
375
376enum {
377 TCG_BSWAP_IZ = 1,
378 TCG_BSWAP_OZ = 2,
379 TCG_BSWAP_OS = 4,
380};
381
382typedef enum TCGTempVal {
383 TEMP_VAL_DEAD,
384 TEMP_VAL_REG,
385 TEMP_VAL_MEM,
386 TEMP_VAL_CONST,
387} TCGTempVal;
388
389typedef enum TCGTempKind {
390
391
392
393
394
395 TEMP_EBB,
396
397 TEMP_TB,
398
399 TEMP_GLOBAL,
400
401 TEMP_FIXED,
402
403 TEMP_CONST,
404} TCGTempKind;
405
406typedef struct TCGTemp {
407 TCGReg reg:8;
408 TCGTempVal val_type:8;
409 TCGType base_type:8;
410 TCGType type:8;
411 TCGTempKind kind:3;
412 unsigned int indirect_reg:1;
413 unsigned int indirect_base:1;
414 unsigned int mem_coherent:1;
415 unsigned int mem_allocated:1;
416 unsigned int temp_allocated:1;
417 unsigned int temp_subindex:1;
418
419 int64_t val;
420 struct TCGTemp *mem_base;
421 intptr_t mem_offset;
422 const char *name;
423
424
425
426
427 uintptr_t state;
428 void *state_ptr;
429} TCGTemp;
430
431typedef struct TCGContext TCGContext;
432
433typedef struct TCGTempSet {
434 unsigned long l[BITS_TO_LONGS(TCG_MAX_TEMPS)];
435} TCGTempSet;
436
437
438
439
440
441
442#define DEAD_ARG (1 << 4)
443#define SYNC_ARG (1 << 0)
444typedef uint32_t TCGLifeData;
445
446struct TCGOp {
447 TCGOpcode opc : 8;
448 unsigned nargs : 8;
449
450
451 unsigned param1 : 8;
452 unsigned param2 : 8;
453
454
455 TCGLifeData life;
456
457
458 QTAILQ_ENTRY(TCGOp) link;
459
460
461 TCGRegSet output_pref[2];
462
463
464 TCGArg args[];
465};
466
467#define TCGOP_CALLI(X) (X)->param1
468#define TCGOP_CALLO(X) (X)->param2
469
470#define TCGOP_VECL(X) (X)->param1
471#define TCGOP_VECE(X) (X)->param2
472
473
474QEMU_BUILD_BUG_ON(NB_OPS > (1 << 8));
475
476static inline TCGRegSet output_pref(const TCGOp *op, unsigned i)
477{
478 return i < ARRAY_SIZE(op->output_pref) ? op->output_pref[i] : 0;
479}
480
481struct TCGContext {
482 uint8_t *pool_cur, *pool_end;
483 TCGPool *pool_first, *pool_current, *pool_first_large;
484 int nb_labels;
485 int nb_globals;
486 int nb_temps;
487 int nb_indirects;
488 int nb_ops;
489 TCGType addr_type;
490
491#ifdef CONFIG_SOFTMMU
492 int tlb_fast_offset;
493 int page_mask;
494 uint8_t page_bits;
495 uint8_t tlb_dyn_max_bits;
496#endif
497 uint8_t insn_start_words;
498 TCGBar guest_mo;
499
500 TCGRegSet reserved_regs;
501 intptr_t current_frame_offset;
502 intptr_t frame_start;
503 intptr_t frame_end;
504 TCGTemp *frame_temp;
505
506 TranslationBlock *gen_tb;
507 tcg_insn_unit *code_buf;
508 tcg_insn_unit *code_ptr;
509
510#ifdef CONFIG_DEBUG_TCG
511 int goto_tb_issue_mask;
512 const TCGOpcode *vecop_list;
513#endif
514
515
516
517
518
519 void *code_gen_buffer;
520 size_t code_gen_buffer_size;
521 void *code_gen_ptr;
522 void *data_gen_ptr;
523
524
525 void *code_gen_highwater;
526
527
528 CPUState *cpu;
529
530
531#ifdef TCG_TARGET_NEED_LDST_LABELS
532 QSIMPLEQ_HEAD(, TCGLabelQemuLdst) ldst_labels;
533#endif
534#ifdef TCG_TARGET_NEED_POOL_LABELS
535 struct TCGLabelPoolData *pool_labels;
536#endif
537
538 TCGLabel *exitreq_label;
539
540#ifdef CONFIG_PLUGIN
541
542
543
544
545
546
547
548 struct qemu_plugin_tb *plugin_tb;
549
550
551 struct qemu_plugin_insn *plugin_insn;
552#endif
553
554 GHashTable *const_table[TCG_TYPE_COUNT];
555 TCGTempSet free_temps[TCG_TYPE_COUNT];
556 TCGTemp temps[TCG_MAX_TEMPS];
557
558 QTAILQ_HEAD(, TCGOp) ops, free_ops;
559 QSIMPLEQ_HEAD(, TCGLabel) labels;
560
561
562
563 TCGTemp *reg_to_temp[TCG_TARGET_NB_REGS];
564
565 uint16_t gen_insn_end_off[TCG_MAX_INSNS];
566 uint64_t *gen_insn_data;
567
568
569 sigjmp_buf jmp_trans;
570};
571
572static inline bool temp_readonly(TCGTemp *ts)
573{
574 return ts->kind >= TEMP_FIXED;
575}
576
577extern __thread TCGContext *tcg_ctx;
578extern const void *tcg_code_gen_epilogue;
579extern uintptr_t tcg_splitwx_diff;
580extern TCGv_env cpu_env;
581
582bool in_code_gen_buffer(const void *p);
583
584#ifdef CONFIG_DEBUG_TCG
585const void *tcg_splitwx_to_rx(void *rw);
586void *tcg_splitwx_to_rw(const void *rx);
587#else
588static inline const void *tcg_splitwx_to_rx(void *rw)
589{
590 return rw ? rw + tcg_splitwx_diff : NULL;
591}
592
593static inline void *tcg_splitwx_to_rw(const void *rx)
594{
595 return rx ? (void *)rx - tcg_splitwx_diff : NULL;
596}
597#endif
598
599static inline TCGArg temp_arg(TCGTemp *ts)
600{
601 return (uintptr_t)ts;
602}
603
604static inline TCGTemp *arg_temp(TCGArg a)
605{
606 return (TCGTemp *)(uintptr_t)a;
607}
608
609#ifdef CONFIG_DEBUG_TCG
610size_t temp_idx(TCGTemp *ts);
611TCGTemp *tcgv_i32_temp(TCGv_i32 v);
612#else
613static inline size_t temp_idx(TCGTemp *ts)
614{
615 return ts - tcg_ctx->temps;
616}
617
618
619
620
621
622
623static inline TCGTemp *tcgv_i32_temp(TCGv_i32 v)
624{
625 return (void *)tcg_ctx + (uintptr_t)v;
626}
627#endif
628
629static inline TCGTemp *tcgv_i64_temp(TCGv_i64 v)
630{
631 return tcgv_i32_temp((TCGv_i32)v);
632}
633
634static inline TCGTemp *tcgv_i128_temp(TCGv_i128 v)
635{
636 return tcgv_i32_temp((TCGv_i32)v);
637}
638
639static inline TCGTemp *tcgv_ptr_temp(TCGv_ptr v)
640{
641 return tcgv_i32_temp((TCGv_i32)v);
642}
643
644static inline TCGTemp *tcgv_vec_temp(TCGv_vec v)
645{
646 return tcgv_i32_temp((TCGv_i32)v);
647}
648
649static inline TCGArg tcgv_i32_arg(TCGv_i32 v)
650{
651 return temp_arg(tcgv_i32_temp(v));
652}
653
654static inline TCGArg tcgv_i64_arg(TCGv_i64 v)
655{
656 return temp_arg(tcgv_i64_temp(v));
657}
658
659static inline TCGArg tcgv_i128_arg(TCGv_i128 v)
660{
661 return temp_arg(tcgv_i128_temp(v));
662}
663
664static inline TCGArg tcgv_ptr_arg(TCGv_ptr v)
665{
666 return temp_arg(tcgv_ptr_temp(v));
667}
668
669static inline TCGArg tcgv_vec_arg(TCGv_vec v)
670{
671 return temp_arg(tcgv_vec_temp(v));
672}
673
674static inline TCGv_i32 temp_tcgv_i32(TCGTemp *t)
675{
676 (void)temp_idx(t);
677 return (TCGv_i32)((void *)t - (void *)tcg_ctx);
678}
679
680static inline TCGv_i64 temp_tcgv_i64(TCGTemp *t)
681{
682 return (TCGv_i64)temp_tcgv_i32(t);
683}
684
685static inline TCGv_i128 temp_tcgv_i128(TCGTemp *t)
686{
687 return (TCGv_i128)temp_tcgv_i32(t);
688}
689
690static inline TCGv_ptr temp_tcgv_ptr(TCGTemp *t)
691{
692 return (TCGv_ptr)temp_tcgv_i32(t);
693}
694
695static inline TCGv_vec temp_tcgv_vec(TCGTemp *t)
696{
697 return (TCGv_vec)temp_tcgv_i32(t);
698}
699
700static inline TCGArg tcg_get_insn_param(TCGOp *op, int arg)
701{
702 return op->args[arg];
703}
704
705static inline void tcg_set_insn_param(TCGOp *op, int arg, TCGArg v)
706{
707 op->args[arg] = v;
708}
709
710static inline uint64_t tcg_get_insn_start_param(TCGOp *op, int arg)
711{
712 if (TCG_TARGET_REG_BITS == 64) {
713 return tcg_get_insn_param(op, arg);
714 } else {
715 return deposit64(tcg_get_insn_param(op, arg * 2), 32, 32,
716 tcg_get_insn_param(op, arg * 2 + 1));
717 }
718}
719
720static inline void tcg_set_insn_start_param(TCGOp *op, int arg, uint64_t v)
721{
722 if (TCG_TARGET_REG_BITS == 64) {
723 tcg_set_insn_param(op, arg, v);
724 } else {
725 tcg_set_insn_param(op, arg * 2, v);
726 tcg_set_insn_param(op, arg * 2 + 1, v >> 32);
727 }
728}
729
730
731static inline TCGOp *tcg_last_op(void)
732{
733 return QTAILQ_LAST(&tcg_ctx->ops);
734}
735
736
737static inline bool tcg_op_buf_full(void)
738{
739
740
741
742
743
744
745
746 return tcg_ctx->nb_ops >= 4000;
747}
748
749
750
751
752void *tcg_malloc_internal(TCGContext *s, int size);
753void tcg_pool_reset(TCGContext *s);
754TranslationBlock *tcg_tb_alloc(TCGContext *s);
755
756void tcg_region_reset_all(void);
757
758size_t tcg_code_size(void);
759size_t tcg_code_capacity(void);
760
761void tcg_tb_insert(TranslationBlock *tb);
762void tcg_tb_remove(TranslationBlock *tb);
763TranslationBlock *tcg_tb_lookup(uintptr_t tc_ptr);
764void tcg_tb_foreach(GTraverseFunc func, gpointer user_data);
765size_t tcg_nb_tbs(void);
766
767
768static inline void *tcg_malloc(int size)
769{
770 TCGContext *s = tcg_ctx;
771 uint8_t *ptr, *ptr_end;
772
773
774 size = QEMU_ALIGN_UP(size, 8);
775
776 ptr = s->pool_cur;
777 ptr_end = ptr + size;
778 if (unlikely(ptr_end > s->pool_end)) {
779 return tcg_malloc_internal(tcg_ctx, size);
780 } else {
781 s->pool_cur = ptr_end;
782 return ptr;
783 }
784}
785
786void tcg_init(size_t tb_size, int splitwx, unsigned max_cpus);
787void tcg_register_thread(void);
788void tcg_prologue_init(TCGContext *s);
789void tcg_func_start(TCGContext *s);
790
791int tcg_gen_code(TCGContext *s, TranslationBlock *tb, uint64_t pc_start);
792
793void tb_target_set_jmp_target(const TranslationBlock *, int,
794 uintptr_t, uintptr_t);
795
796void tcg_set_frame(TCGContext *s, TCGReg reg, intptr_t start, intptr_t size);
797
798TCGTemp *tcg_global_mem_new_internal(TCGType, TCGv_ptr,
799 intptr_t, const char *);
800TCGTemp *tcg_temp_new_internal(TCGType, TCGTempKind);
801TCGv_vec tcg_temp_new_vec(TCGType type);
802TCGv_vec tcg_temp_new_vec_matching(TCGv_vec match);
803
804static inline TCGv_i32 tcg_global_mem_new_i32(TCGv_ptr reg, intptr_t offset,
805 const char *name)
806{
807 TCGTemp *t = tcg_global_mem_new_internal(TCG_TYPE_I32, reg, offset, name);
808 return temp_tcgv_i32(t);
809}
810
811static inline TCGv_i32 tcg_temp_new_i32(void)
812{
813 TCGTemp *t = tcg_temp_new_internal(TCG_TYPE_I32, TEMP_TB);
814 return temp_tcgv_i32(t);
815}
816
817static inline TCGv_i64 tcg_global_mem_new_i64(TCGv_ptr reg, intptr_t offset,
818 const char *name)
819{
820 TCGTemp *t = tcg_global_mem_new_internal(TCG_TYPE_I64, reg, offset, name);
821 return temp_tcgv_i64(t);
822}
823
824static inline TCGv_i64 tcg_temp_new_i64(void)
825{
826 TCGTemp *t = tcg_temp_new_internal(TCG_TYPE_I64, TEMP_TB);
827 return temp_tcgv_i64(t);
828}
829
830static inline TCGv_i128 tcg_temp_new_i128(void)
831{
832 TCGTemp *t = tcg_temp_new_internal(TCG_TYPE_I128, TEMP_TB);
833 return temp_tcgv_i128(t);
834}
835
836static inline TCGv_ptr tcg_global_mem_new_ptr(TCGv_ptr reg, intptr_t offset,
837 const char *name)
838{
839 TCGTemp *t = tcg_global_mem_new_internal(TCG_TYPE_PTR, reg, offset, name);
840 return temp_tcgv_ptr(t);
841}
842
843static inline TCGv_ptr tcg_temp_new_ptr(void)
844{
845 TCGTemp *t = tcg_temp_new_internal(TCG_TYPE_PTR, TEMP_TB);
846 return temp_tcgv_ptr(t);
847}
848
849void tcg_dump_info(GString *buf);
850void tcg_dump_op_count(GString *buf);
851
852#define TCG_CT_CONST 1
853
854typedef struct TCGArgConstraint {
855 unsigned ct : 16;
856 unsigned alias_index : 4;
857 unsigned sort_index : 4;
858 unsigned pair_index : 4;
859 unsigned pair : 2;
860 bool oalias : 1;
861 bool ialias : 1;
862 bool newreg : 1;
863 TCGRegSet regs;
864} TCGArgConstraint;
865
866#define TCG_MAX_OP_ARGS 16
867
868
869enum {
870
871 TCG_OPF_BB_EXIT = 0x01,
872
873 TCG_OPF_BB_END = 0x02,
874
875 TCG_OPF_CALL_CLOBBER = 0x04,
876
877
878 TCG_OPF_SIDE_EFFECTS = 0x08,
879
880 TCG_OPF_64BIT = 0x10,
881
882
883 TCG_OPF_NOT_PRESENT = 0x20,
884
885 TCG_OPF_VECTOR = 0x40,
886
887 TCG_OPF_COND_BRANCH = 0x80
888};
889
890typedef struct TCGOpDef {
891 const char *name;
892 uint8_t nb_oargs, nb_iargs, nb_cargs, nb_args;
893 uint8_t flags;
894 TCGArgConstraint *args_ct;
895} TCGOpDef;
896
897extern TCGOpDef tcg_op_defs[];
898extern const size_t tcg_op_defs_max;
899
900typedef struct TCGTargetOpDef {
901 TCGOpcode op;
902 const char *args_ct_str[TCG_MAX_OP_ARGS];
903} TCGTargetOpDef;
904
905bool tcg_op_supported(TCGOpcode op);
906
907void tcg_gen_call0(TCGHelperInfo *, TCGTemp *ret);
908void tcg_gen_call1(TCGHelperInfo *, TCGTemp *ret, TCGTemp *);
909void tcg_gen_call2(TCGHelperInfo *, TCGTemp *ret, TCGTemp *, TCGTemp *);
910void tcg_gen_call3(TCGHelperInfo *, TCGTemp *ret, TCGTemp *,
911 TCGTemp *, TCGTemp *);
912void tcg_gen_call4(TCGHelperInfo *, TCGTemp *ret, TCGTemp *, TCGTemp *,
913 TCGTemp *, TCGTemp *);
914void tcg_gen_call5(TCGHelperInfo *, TCGTemp *ret, TCGTemp *, TCGTemp *,
915 TCGTemp *, TCGTemp *, TCGTemp *);
916void tcg_gen_call6(TCGHelperInfo *, TCGTemp *ret, TCGTemp *, TCGTemp *,
917 TCGTemp *, TCGTemp *, TCGTemp *, TCGTemp *);
918void tcg_gen_call7(TCGHelperInfo *, TCGTemp *ret, TCGTemp *, TCGTemp *,
919 TCGTemp *, TCGTemp *, TCGTemp *, TCGTemp *, TCGTemp *);
920
921TCGOp *tcg_emit_op(TCGOpcode opc, unsigned nargs);
922void tcg_op_remove(TCGContext *s, TCGOp *op);
923TCGOp *tcg_op_insert_before(TCGContext *s, TCGOp *op,
924 TCGOpcode opc, unsigned nargs);
925TCGOp *tcg_op_insert_after(TCGContext *s, TCGOp *op,
926 TCGOpcode opc, unsigned nargs);
927
928
929
930
931
932
933
934
935
936void tcg_remove_ops_after(TCGOp *op);
937
938void tcg_optimize(TCGContext *s);
939
940
941
942
943
944
945TCGTemp *tcg_constant_internal(TCGType type, int64_t val);
946
947static inline TCGv_i32 tcg_constant_i32(int32_t val)
948{
949 return temp_tcgv_i32(tcg_constant_internal(TCG_TYPE_I32, val));
950}
951
952static inline TCGv_i64 tcg_constant_i64(int64_t val)
953{
954 return temp_tcgv_i64(tcg_constant_internal(TCG_TYPE_I64, val));
955}
956
957TCGv_vec tcg_constant_vec(TCGType type, unsigned vece, int64_t val);
958TCGv_vec tcg_constant_vec_matching(TCGv_vec match, unsigned vece, int64_t val);
959
960#if UINTPTR_MAX == UINT32_MAX
961# define tcg_constant_ptr(x) ((TCGv_ptr)tcg_constant_i32((intptr_t)(x)))
962#else
963# define tcg_constant_ptr(x) ((TCGv_ptr)tcg_constant_i64((intptr_t)(x)))
964#endif
965
966TCGLabel *gen_new_label(void);
967
968
969
970
971
972
973
974
975static inline TCGArg label_arg(TCGLabel *l)
976{
977 return (uintptr_t)l;
978}
979
980
981
982
983
984
985
986
987
988static inline TCGLabel *arg_label(TCGArg i)
989{
990 return (TCGLabel *)(uintptr_t)i;
991}
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006static inline ptrdiff_t tcg_ptr_byte_diff(const void *a, const void *b)
1007{
1008 return a - b;
1009}
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020static inline ptrdiff_t tcg_pcrel_diff(TCGContext *s, const void *target)
1021{
1022 return tcg_ptr_byte_diff(target, tcg_splitwx_to_rx(s->code_ptr));
1023}
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033static inline ptrdiff_t tcg_tbrel_diff(TCGContext *s, const void *target)
1034{
1035 return tcg_ptr_byte_diff(target, tcg_splitwx_to_rx(s->code_buf));
1036}
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046static inline size_t tcg_current_code_size(TCGContext *s)
1047{
1048 return tcg_ptr_byte_diff(s->code_ptr, s->code_buf);
1049}
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096#define TB_EXIT_MASK 3
1097#define TB_EXIT_IDX0 0
1098#define TB_EXIT_IDX1 1
1099#define TB_EXIT_IDXMAX 1
1100#define TB_EXIT_REQUESTED 3
1101
1102#ifdef CONFIG_TCG_INTERPRETER
1103uintptr_t tcg_qemu_tb_exec(CPUArchState *env, const void *tb_ptr);
1104#else
1105typedef uintptr_t tcg_prologue_fn(CPUArchState *env, const void *tb_ptr);
1106extern tcg_prologue_fn *tcg_qemu_tb_exec;
1107#endif
1108
1109void tcg_register_jit(const void *buf, size_t buf_size);
1110
1111#if TCG_TARGET_MAYBE_vec
1112
1113
1114
1115int tcg_can_emit_vec_op(TCGOpcode, TCGType, unsigned);
1116#else
1117static inline int tcg_can_emit_vec_op(TCGOpcode o, TCGType t, unsigned ve)
1118{
1119 return 0;
1120}
1121#endif
1122
1123
1124void tcg_expand_vec_op(TCGOpcode, TCGType, unsigned, TCGArg, ...);
1125
1126
1127uint64_t dup_const(unsigned vece, uint64_t c);
1128
1129#define dup_const(VECE, C) \
1130 (__builtin_constant_p(VECE) \
1131 ? ( (VECE) == MO_8 ? 0x0101010101010101ull * (uint8_t)(C) \
1132 : (VECE) == MO_16 ? 0x0001000100010001ull * (uint16_t)(C) \
1133 : (VECE) == MO_32 ? 0x0000000100000001ull * (uint32_t)(C) \
1134 : (VECE) == MO_64 ? (uint64_t)(C) \
1135 : (qemu_build_not_reached_always(), 0)) \
1136 : dup_const(VECE, C))
1137
1138static inline const TCGOpcode *tcg_swap_vecop_list(const TCGOpcode *n)
1139{
1140#ifdef CONFIG_DEBUG_TCG
1141 const TCGOpcode *o = tcg_ctx->vecop_list;
1142 tcg_ctx->vecop_list = n;
1143 return o;
1144#else
1145 return NULL;
1146#endif
1147}
1148
1149bool tcg_can_emit_vecop_list(const TCGOpcode *, TCGType, unsigned);
1150
1151#endif
1152