1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25#ifndef TCG_H
26#define TCG_H
27
28#include "cpu.h"
29#include "exec/memop.h"
30#include "exec/memopidx.h"
31#include "qemu/bitops.h"
32#include "qemu/plugin.h"
33#include "qemu/queue.h"
34#include "tcg/tcg-mo.h"
35#include "tcg-target.h"
36#include "tcg/tcg-cond.h"
37
38
39#define MAX_OP_PER_INSTR 266
40
41#if HOST_LONG_BITS == 32
42#define MAX_OPC_PARAM_PER_ARG 2
43#else
44#define MAX_OPC_PARAM_PER_ARG 1
45#endif
46#define MAX_OPC_PARAM_IARGS 7
47#define MAX_OPC_PARAM_OARGS 1
48#define MAX_OPC_PARAM_ARGS (MAX_OPC_PARAM_IARGS + MAX_OPC_PARAM_OARGS)
49
50
51
52
53#define MAX_OPC_PARAM (4 + (MAX_OPC_PARAM_PER_ARG * MAX_OPC_PARAM_ARGS))
54
55#define CPU_TEMP_BUF_NLONGS 128
56#define TCG_STATIC_FRAME_SIZE (CPU_TEMP_BUF_NLONGS * sizeof(long))
57
58
59#ifndef TCG_TARGET_REG_BITS
60# if UINTPTR_MAX == UINT32_MAX
61# define TCG_TARGET_REG_BITS 32
62# elif UINTPTR_MAX == UINT64_MAX
63# define TCG_TARGET_REG_BITS 64
64# else
65# error Unknown pointer size for tcg target
66# endif
67#endif
68
69#if TCG_TARGET_REG_BITS == 32
70typedef int32_t tcg_target_long;
71typedef uint32_t tcg_target_ulong;
72#define TCG_PRIlx PRIx32
73#define TCG_PRIld PRId32
74#elif TCG_TARGET_REG_BITS == 64
75typedef int64_t tcg_target_long;
76typedef uint64_t tcg_target_ulong;
77#define TCG_PRIlx PRIx64
78#define TCG_PRIld PRId64
79#else
80#error unsupported
81#endif
82
83
84
85
86#if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
87#define TCG_OVERSIZED_GUEST 1
88#else
89#define TCG_OVERSIZED_GUEST 0
90#endif
91
92#if TCG_TARGET_NB_REGS <= 32
93typedef uint32_t TCGRegSet;
94#elif TCG_TARGET_NB_REGS <= 64
95typedef uint64_t TCGRegSet;
96#else
97#error unsupported
98#endif
99
100#if TCG_TARGET_REG_BITS == 32
101
102#define TCG_TARGET_HAS_extrl_i64_i32 0
103#define TCG_TARGET_HAS_extrh_i64_i32 0
104#define TCG_TARGET_HAS_div_i64 0
105#define TCG_TARGET_HAS_rem_i64 0
106#define TCG_TARGET_HAS_div2_i64 0
107#define TCG_TARGET_HAS_rot_i64 0
108#define TCG_TARGET_HAS_ext8s_i64 0
109#define TCG_TARGET_HAS_ext16s_i64 0
110#define TCG_TARGET_HAS_ext32s_i64 0
111#define TCG_TARGET_HAS_ext8u_i64 0
112#define TCG_TARGET_HAS_ext16u_i64 0
113#define TCG_TARGET_HAS_ext32u_i64 0
114#define TCG_TARGET_HAS_bswap16_i64 0
115#define TCG_TARGET_HAS_bswap32_i64 0
116#define TCG_TARGET_HAS_bswap64_i64 0
117#define TCG_TARGET_HAS_neg_i64 0
118#define TCG_TARGET_HAS_not_i64 0
119#define TCG_TARGET_HAS_andc_i64 0
120#define TCG_TARGET_HAS_orc_i64 0
121#define TCG_TARGET_HAS_eqv_i64 0
122#define TCG_TARGET_HAS_nand_i64 0
123#define TCG_TARGET_HAS_nor_i64 0
124#define TCG_TARGET_HAS_clz_i64 0
125#define TCG_TARGET_HAS_ctz_i64 0
126#define TCG_TARGET_HAS_ctpop_i64 0
127#define TCG_TARGET_HAS_deposit_i64 0
128#define TCG_TARGET_HAS_extract_i64 0
129#define TCG_TARGET_HAS_sextract_i64 0
130#define TCG_TARGET_HAS_extract2_i64 0
131#define TCG_TARGET_HAS_movcond_i64 0
132#define TCG_TARGET_HAS_add2_i64 0
133#define TCG_TARGET_HAS_sub2_i64 0
134#define TCG_TARGET_HAS_mulu2_i64 0
135#define TCG_TARGET_HAS_muls2_i64 0
136#define TCG_TARGET_HAS_muluh_i64 0
137#define TCG_TARGET_HAS_mulsh_i64 0
138
139#define TCG_TARGET_HAS_add2_i32 1
140#define TCG_TARGET_HAS_sub2_i32 1
141#endif
142
143#ifndef TCG_TARGET_deposit_i32_valid
144#define TCG_TARGET_deposit_i32_valid(ofs, len) 1
145#endif
146#ifndef TCG_TARGET_deposit_i64_valid
147#define TCG_TARGET_deposit_i64_valid(ofs, len) 1
148#endif
149#ifndef TCG_TARGET_extract_i32_valid
150#define TCG_TARGET_extract_i32_valid(ofs, len) 1
151#endif
152#ifndef TCG_TARGET_extract_i64_valid
153#define TCG_TARGET_extract_i64_valid(ofs, len) 1
154#endif
155
156
157#if defined(TCG_TARGET_HAS_div_i32)
158#define TCG_TARGET_HAS_div2_i32 0
159#elif defined(TCG_TARGET_HAS_div2_i32)
160#define TCG_TARGET_HAS_div_i32 0
161#define TCG_TARGET_HAS_rem_i32 0
162#endif
163#if defined(TCG_TARGET_HAS_div_i64)
164#define TCG_TARGET_HAS_div2_i64 0
165#elif defined(TCG_TARGET_HAS_div2_i64)
166#define TCG_TARGET_HAS_div_i64 0
167#define TCG_TARGET_HAS_rem_i64 0
168#endif
169
170
171#if TCG_TARGET_REG_BITS == 32 \
172 && !(defined(TCG_TARGET_HAS_mulu2_i32) \
173 || defined(TCG_TARGET_HAS_muluh_i32))
174# error "Missing unsigned widening multiply"
175#endif
176
177#if !defined(TCG_TARGET_HAS_v64) \
178 && !defined(TCG_TARGET_HAS_v128) \
179 && !defined(TCG_TARGET_HAS_v256)
180#define TCG_TARGET_MAYBE_vec 0
181#define TCG_TARGET_HAS_abs_vec 0
182#define TCG_TARGET_HAS_neg_vec 0
183#define TCG_TARGET_HAS_not_vec 0
184#define TCG_TARGET_HAS_andc_vec 0
185#define TCG_TARGET_HAS_orc_vec 0
186#define TCG_TARGET_HAS_nand_vec 0
187#define TCG_TARGET_HAS_nor_vec 0
188#define TCG_TARGET_HAS_eqv_vec 0
189#define TCG_TARGET_HAS_roti_vec 0
190#define TCG_TARGET_HAS_rots_vec 0
191#define TCG_TARGET_HAS_rotv_vec 0
192#define TCG_TARGET_HAS_shi_vec 0
193#define TCG_TARGET_HAS_shs_vec 0
194#define TCG_TARGET_HAS_shv_vec 0
195#define TCG_TARGET_HAS_mul_vec 0
196#define TCG_TARGET_HAS_sat_vec 0
197#define TCG_TARGET_HAS_minmax_vec 0
198#define TCG_TARGET_HAS_bitsel_vec 0
199#define TCG_TARGET_HAS_cmpsel_vec 0
200#else
201#define TCG_TARGET_MAYBE_vec 1
202#endif
203#ifndef TCG_TARGET_HAS_v64
204#define TCG_TARGET_HAS_v64 0
205#endif
206#ifndef TCG_TARGET_HAS_v128
207#define TCG_TARGET_HAS_v128 0
208#endif
209#ifndef TCG_TARGET_HAS_v256
210#define TCG_TARGET_HAS_v256 0
211#endif
212
213#ifndef TARGET_INSN_START_EXTRA_WORDS
214# define TARGET_INSN_START_WORDS 1
215#else
216# define TARGET_INSN_START_WORDS (1 + TARGET_INSN_START_EXTRA_WORDS)
217#endif
218
219typedef enum TCGOpcode {
220#define DEF(name, oargs, iargs, cargs, flags) INDEX_op_ ## name,
221#include "tcg/tcg-opc.h"
222#undef DEF
223 NB_OPS,
224} TCGOpcode;
225
226#define tcg_regset_set_reg(d, r) ((d) |= (TCGRegSet)1 << (r))
227#define tcg_regset_reset_reg(d, r) ((d) &= ~((TCGRegSet)1 << (r)))
228#define tcg_regset_test_reg(d, r) (((d) >> (r)) & 1)
229
230#ifndef TCG_TARGET_INSN_UNIT_SIZE
231# error "Missing TCG_TARGET_INSN_UNIT_SIZE"
232#elif TCG_TARGET_INSN_UNIT_SIZE == 1
233typedef uint8_t tcg_insn_unit;
234#elif TCG_TARGET_INSN_UNIT_SIZE == 2
235typedef uint16_t tcg_insn_unit;
236#elif TCG_TARGET_INSN_UNIT_SIZE == 4
237typedef uint32_t tcg_insn_unit;
238#elif TCG_TARGET_INSN_UNIT_SIZE == 8
239typedef uint64_t tcg_insn_unit;
240#else
241
242#endif
243
244
245#if defined CONFIG_DEBUG_TCG || defined QEMU_STATIC_ANALYSIS
246# define tcg_debug_assert(X) do { assert(X); } while (0)
247#else
248# define tcg_debug_assert(X) \
249 do { if (!(X)) { __builtin_unreachable(); } } while (0)
250#endif
251
252typedef struct TCGRelocation TCGRelocation;
253struct TCGRelocation {
254 QSIMPLEQ_ENTRY(TCGRelocation) next;
255 tcg_insn_unit *ptr;
256 intptr_t addend;
257 int type;
258};
259
260typedef struct TCGLabel TCGLabel;
261struct TCGLabel {
262 unsigned present : 1;
263 unsigned has_value : 1;
264 unsigned id : 14;
265 unsigned refs : 16;
266 union {
267 uintptr_t value;
268 const tcg_insn_unit *value_ptr;
269 } u;
270 QSIMPLEQ_HEAD(, TCGRelocation) relocs;
271 QSIMPLEQ_ENTRY(TCGLabel) next;
272};
273
274typedef struct TCGPool {
275 struct TCGPool *next;
276 int size;
277 uint8_t data[] __attribute__ ((aligned));
278} TCGPool;
279
280#define TCG_POOL_CHUNK_SIZE 32768
281
282#define TCG_MAX_TEMPS 512
283#define TCG_MAX_INSNS 512
284
285
286
287#define TCG_STATIC_CALL_ARGS_SIZE 128
288
289typedef enum TCGType {
290 TCG_TYPE_I32,
291 TCG_TYPE_I64,
292
293 TCG_TYPE_V64,
294 TCG_TYPE_V128,
295 TCG_TYPE_V256,
296
297 TCG_TYPE_COUNT,
298
299
300#if TCG_TARGET_REG_BITS == 32
301 TCG_TYPE_REG = TCG_TYPE_I32,
302#else
303 TCG_TYPE_REG = TCG_TYPE_I64,
304#endif
305
306
307#if UINTPTR_MAX == UINT32_MAX
308 TCG_TYPE_PTR = TCG_TYPE_I32,
309#else
310 TCG_TYPE_PTR = TCG_TYPE_I64,
311#endif
312
313
314#if TARGET_LONG_BITS == 64
315 TCG_TYPE_TL = TCG_TYPE_I64,
316#else
317 TCG_TYPE_TL = TCG_TYPE_I32,
318#endif
319} TCGType;
320
321
322
323
324
325
326
327static inline unsigned get_alignment_bits(MemOp memop)
328{
329 unsigned a = memop & MO_AMASK;
330
331 if (a == MO_UNALN) {
332
333 a = 0;
334 } else if (a == MO_ALIGN) {
335
336 a = memop & MO_SIZE;
337 } else {
338
339 a = a >> MO_ASHIFT;
340 }
341#if defined(CONFIG_SOFTMMU)
342
343 tcg_debug_assert((TLB_FLAGS_MASK & ((1 << a) - 1)) == 0);
344#endif
345 return a;
346}
347
348typedef tcg_target_ulong TCGArg;
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380typedef struct TCGv_i32_d *TCGv_i32;
381typedef struct TCGv_i64_d *TCGv_i64;
382typedef struct TCGv_ptr_d *TCGv_ptr;
383typedef struct TCGv_vec_d *TCGv_vec;
384typedef TCGv_ptr TCGv_env;
385#if TARGET_LONG_BITS == 32
386#define TCGv TCGv_i32
387#elif TARGET_LONG_BITS == 64
388#define TCGv TCGv_i64
389#else
390#error Unhandled TARGET_LONG_BITS value
391#endif
392
393
394
395
396#define TCG_CALL_NO_READ_GLOBALS 0x0001
397
398#define TCG_CALL_NO_WRITE_GLOBALS 0x0002
399
400#define TCG_CALL_NO_SIDE_EFFECTS 0x0004
401
402#define TCG_CALL_NO_RETURN 0x0008
403
404
405#define TCG_CALL_NO_RWG TCG_CALL_NO_READ_GLOBALS
406#define TCG_CALL_NO_WG TCG_CALL_NO_WRITE_GLOBALS
407#define TCG_CALL_NO_SE TCG_CALL_NO_SIDE_EFFECTS
408#define TCG_CALL_NO_RWG_SE (TCG_CALL_NO_RWG | TCG_CALL_NO_SE)
409#define TCG_CALL_NO_WG_SE (TCG_CALL_NO_WG | TCG_CALL_NO_SE)
410
411
412#define TCG_CALL_DUMMY_ARG ((TCGArg)0)
413
414
415
416
417
418
419
420enum {
421 TCG_BSWAP_IZ = 1,
422 TCG_BSWAP_OZ = 2,
423 TCG_BSWAP_OS = 4,
424};
425
426typedef enum TCGTempVal {
427 TEMP_VAL_DEAD,
428 TEMP_VAL_REG,
429 TEMP_VAL_MEM,
430 TEMP_VAL_CONST,
431} TCGTempVal;
432
433typedef enum TCGTempKind {
434
435 TEMP_NORMAL,
436
437 TEMP_LOCAL,
438
439 TEMP_GLOBAL,
440
441 TEMP_FIXED,
442
443 TEMP_CONST,
444} TCGTempKind;
445
446typedef struct TCGTemp {
447 TCGReg reg:8;
448 TCGTempVal val_type:8;
449 TCGType base_type:8;
450 TCGType type:8;
451 TCGTempKind kind:3;
452 unsigned int indirect_reg:1;
453 unsigned int indirect_base:1;
454 unsigned int mem_coherent:1;
455 unsigned int mem_allocated:1;
456 unsigned int temp_allocated:1;
457
458 int64_t val;
459 struct TCGTemp *mem_base;
460 intptr_t mem_offset;
461 const char *name;
462
463
464
465
466 uintptr_t state;
467 void *state_ptr;
468} TCGTemp;
469
470typedef struct TCGContext TCGContext;
471
472typedef struct TCGTempSet {
473 unsigned long l[BITS_TO_LONGS(TCG_MAX_TEMPS)];
474} TCGTempSet;
475
476
477
478
479
480#define DEAD_ARG 4
481#define SYNC_ARG 1
482typedef uint16_t TCGLifeData;
483
484
485
486typedef struct TCGOp {
487 TCGOpcode opc : 8;
488
489
490 unsigned param1 : 4;
491 unsigned param2 : 4;
492
493
494 unsigned life : 16;
495
496
497 QTAILQ_ENTRY(TCGOp) link;
498
499
500 TCGArg args[MAX_OPC_PARAM];
501
502
503 TCGRegSet output_pref[2];
504} TCGOp;
505
506#define TCGOP_CALLI(X) (X)->param1
507#define TCGOP_CALLO(X) (X)->param2
508
509#define TCGOP_VECL(X) (X)->param1
510#define TCGOP_VECE(X) (X)->param2
511
512
513QEMU_BUILD_BUG_ON(NB_OPS > (1 << 8));
514
515typedef struct TCGProfile {
516 int64_t cpu_exec_time;
517 int64_t tb_count1;
518 int64_t tb_count;
519 int64_t op_count;
520 int op_count_max;
521 int temp_count_max;
522 int64_t temp_count;
523 int64_t del_op_count;
524 int64_t code_in_len;
525 int64_t code_out_len;
526 int64_t search_out_len;
527 int64_t interm_time;
528 int64_t code_time;
529 int64_t la_time;
530 int64_t opt_time;
531 int64_t restore_count;
532 int64_t restore_time;
533 int64_t table_op_count[NB_OPS];
534} TCGProfile;
535
536struct TCGContext {
537 uint8_t *pool_cur, *pool_end;
538 TCGPool *pool_first, *pool_current, *pool_first_large;
539 int nb_labels;
540 int nb_globals;
541 int nb_temps;
542 int nb_indirects;
543 int nb_ops;
544
545
546 tcg_insn_unit *code_buf;
547 uint16_t *tb_jmp_reset_offset;
548 uintptr_t *tb_jmp_insn_offset;
549 uintptr_t *tb_jmp_target_addr;
550
551 TCGRegSet reserved_regs;
552 uint32_t tb_cflags;
553 intptr_t current_frame_offset;
554 intptr_t frame_start;
555 intptr_t frame_end;
556 TCGTemp *frame_temp;
557
558 tcg_insn_unit *code_ptr;
559
560#ifdef CONFIG_PROFILER
561 TCGProfile prof;
562#endif
563
564#ifdef CONFIG_DEBUG_TCG
565 int temps_in_use;
566 int goto_tb_issue_mask;
567 const TCGOpcode *vecop_list;
568#endif
569
570
571
572
573
574 void *code_gen_buffer;
575 size_t code_gen_buffer_size;
576 void *code_gen_ptr;
577 void *data_gen_ptr;
578
579
580 void *code_gen_highwater;
581
582
583 CPUState *cpu;
584
585
586#ifdef TCG_TARGET_NEED_LDST_LABELS
587 QSIMPLEQ_HEAD(, TCGLabelQemuLdst) ldst_labels;
588#endif
589#ifdef TCG_TARGET_NEED_POOL_LABELS
590 struct TCGLabelPoolData *pool_labels;
591#endif
592
593 TCGLabel *exitreq_label;
594
595#ifdef CONFIG_PLUGIN
596
597
598
599
600
601
602
603 struct qemu_plugin_tb *plugin_tb;
604
605
606 struct qemu_plugin_insn *plugin_insn;
607#endif
608
609 GHashTable *const_table[TCG_TYPE_COUNT];
610 TCGTempSet free_temps[TCG_TYPE_COUNT * 2];
611 TCGTemp temps[TCG_MAX_TEMPS];
612
613 QTAILQ_HEAD(, TCGOp) ops, free_ops;
614 QSIMPLEQ_HEAD(, TCGLabel) labels;
615
616
617
618 TCGTemp *reg_to_temp[TCG_TARGET_NB_REGS];
619
620 uint16_t gen_insn_end_off[TCG_MAX_INSNS];
621 target_ulong gen_insn_data[TCG_MAX_INSNS][TARGET_INSN_START_WORDS];
622
623
624 sigjmp_buf jmp_trans;
625};
626
627static inline bool temp_readonly(TCGTemp *ts)
628{
629 return ts->kind >= TEMP_FIXED;
630}
631
632extern __thread TCGContext *tcg_ctx;
633extern const void *tcg_code_gen_epilogue;
634extern uintptr_t tcg_splitwx_diff;
635extern TCGv_env cpu_env;
636
637bool in_code_gen_buffer(const void *p);
638
639#ifdef CONFIG_DEBUG_TCG
640const void *tcg_splitwx_to_rx(void *rw);
641void *tcg_splitwx_to_rw(const void *rx);
642#else
643static inline const void *tcg_splitwx_to_rx(void *rw)
644{
645 return rw ? rw + tcg_splitwx_diff : NULL;
646}
647
648static inline void *tcg_splitwx_to_rw(const void *rx)
649{
650 return rx ? (void *)rx - tcg_splitwx_diff : NULL;
651}
652#endif
653
654static inline size_t temp_idx(TCGTemp *ts)
655{
656 ptrdiff_t n = ts - tcg_ctx->temps;
657 tcg_debug_assert(n >= 0 && n < tcg_ctx->nb_temps);
658 return n;
659}
660
661static inline TCGArg temp_arg(TCGTemp *ts)
662{
663 return (uintptr_t)ts;
664}
665
666static inline TCGTemp *arg_temp(TCGArg a)
667{
668 return (TCGTemp *)(uintptr_t)a;
669}
670
671
672
673
674static inline TCGTemp *tcgv_i32_temp(TCGv_i32 v)
675{
676 uintptr_t o = (uintptr_t)v;
677 TCGTemp *t = (void *)tcg_ctx + o;
678 tcg_debug_assert(offsetof(TCGContext, temps[temp_idx(t)]) == o);
679 return t;
680}
681
682static inline TCGTemp *tcgv_i64_temp(TCGv_i64 v)
683{
684 return tcgv_i32_temp((TCGv_i32)v);
685}
686
687static inline TCGTemp *tcgv_ptr_temp(TCGv_ptr v)
688{
689 return tcgv_i32_temp((TCGv_i32)v);
690}
691
692static inline TCGTemp *tcgv_vec_temp(TCGv_vec v)
693{
694 return tcgv_i32_temp((TCGv_i32)v);
695}
696
697static inline TCGArg tcgv_i32_arg(TCGv_i32 v)
698{
699 return temp_arg(tcgv_i32_temp(v));
700}
701
702static inline TCGArg tcgv_i64_arg(TCGv_i64 v)
703{
704 return temp_arg(tcgv_i64_temp(v));
705}
706
707static inline TCGArg tcgv_ptr_arg(TCGv_ptr v)
708{
709 return temp_arg(tcgv_ptr_temp(v));
710}
711
712static inline TCGArg tcgv_vec_arg(TCGv_vec v)
713{
714 return temp_arg(tcgv_vec_temp(v));
715}
716
717static inline TCGv_i32 temp_tcgv_i32(TCGTemp *t)
718{
719 (void)temp_idx(t);
720 return (TCGv_i32)((void *)t - (void *)tcg_ctx);
721}
722
723static inline TCGv_i64 temp_tcgv_i64(TCGTemp *t)
724{
725 return (TCGv_i64)temp_tcgv_i32(t);
726}
727
728static inline TCGv_ptr temp_tcgv_ptr(TCGTemp *t)
729{
730 return (TCGv_ptr)temp_tcgv_i32(t);
731}
732
733static inline TCGv_vec temp_tcgv_vec(TCGTemp *t)
734{
735 return (TCGv_vec)temp_tcgv_i32(t);
736}
737
738#if TCG_TARGET_REG_BITS == 32
739static inline TCGv_i32 TCGV_LOW(TCGv_i64 t)
740{
741 return temp_tcgv_i32(tcgv_i64_temp(t));
742}
743
744static inline TCGv_i32 TCGV_HIGH(TCGv_i64 t)
745{
746 return temp_tcgv_i32(tcgv_i64_temp(t) + 1);
747}
748#endif
749
750static inline TCGArg tcg_get_insn_param(TCGOp *op, int arg)
751{
752 return op->args[arg];
753}
754
755static inline void tcg_set_insn_param(TCGOp *op, int arg, TCGArg v)
756{
757 op->args[arg] = v;
758}
759
760static inline target_ulong tcg_get_insn_start_param(TCGOp *op, int arg)
761{
762#if TARGET_LONG_BITS <= TCG_TARGET_REG_BITS
763 return tcg_get_insn_param(op, arg);
764#else
765 return tcg_get_insn_param(op, arg * 2) |
766 ((uint64_t)tcg_get_insn_param(op, arg * 2 + 1) << 32);
767#endif
768}
769
770static inline void tcg_set_insn_start_param(TCGOp *op, int arg, target_ulong v)
771{
772#if TARGET_LONG_BITS <= TCG_TARGET_REG_BITS
773 tcg_set_insn_param(op, arg, v);
774#else
775 tcg_set_insn_param(op, arg * 2, v);
776 tcg_set_insn_param(op, arg * 2 + 1, v >> 32);
777#endif
778}
779
780
781static inline TCGOp *tcg_last_op(void)
782{
783 return QTAILQ_LAST(&tcg_ctx->ops);
784}
785
786
787static inline bool tcg_op_buf_full(void)
788{
789
790
791
792
793
794
795
796 return tcg_ctx->nb_ops >= 4000;
797}
798
799
800
801
802void *tcg_malloc_internal(TCGContext *s, int size);
803void tcg_pool_reset(TCGContext *s);
804TranslationBlock *tcg_tb_alloc(TCGContext *s);
805
806void tcg_region_reset_all(void);
807
808size_t tcg_code_size(void);
809size_t tcg_code_capacity(void);
810
811void tcg_tb_insert(TranslationBlock *tb);
812void tcg_tb_remove(TranslationBlock *tb);
813TranslationBlock *tcg_tb_lookup(uintptr_t tc_ptr);
814void tcg_tb_foreach(GTraverseFunc func, gpointer user_data);
815size_t tcg_nb_tbs(void);
816
817
818static inline void *tcg_malloc(int size)
819{
820 TCGContext *s = tcg_ctx;
821 uint8_t *ptr, *ptr_end;
822
823
824 size = QEMU_ALIGN_UP(size, 8);
825
826 ptr = s->pool_cur;
827 ptr_end = ptr + size;
828 if (unlikely(ptr_end > s->pool_end)) {
829 return tcg_malloc_internal(tcg_ctx, size);
830 } else {
831 s->pool_cur = ptr_end;
832 return ptr;
833 }
834}
835
836void tcg_init(size_t tb_size, int splitwx, unsigned max_cpus);
837void tcg_register_thread(void);
838void tcg_prologue_init(TCGContext *s);
839void tcg_func_start(TCGContext *s);
840
841int tcg_gen_code(TCGContext *s, TranslationBlock *tb);
842
843void tcg_set_frame(TCGContext *s, TCGReg reg, intptr_t start, intptr_t size);
844
845TCGTemp *tcg_global_mem_new_internal(TCGType, TCGv_ptr,
846 intptr_t, const char *);
847TCGTemp *tcg_temp_new_internal(TCGType, bool);
848void tcg_temp_free_internal(TCGTemp *);
849TCGv_vec tcg_temp_new_vec(TCGType type);
850TCGv_vec tcg_temp_new_vec_matching(TCGv_vec match);
851
852static inline void tcg_temp_free_i32(TCGv_i32 arg)
853{
854 tcg_temp_free_internal(tcgv_i32_temp(arg));
855}
856
857static inline void tcg_temp_free_i64(TCGv_i64 arg)
858{
859 tcg_temp_free_internal(tcgv_i64_temp(arg));
860}
861
862static inline void tcg_temp_free_ptr(TCGv_ptr arg)
863{
864 tcg_temp_free_internal(tcgv_ptr_temp(arg));
865}
866
867static inline void tcg_temp_free_vec(TCGv_vec arg)
868{
869 tcg_temp_free_internal(tcgv_vec_temp(arg));
870}
871
872static inline TCGv_i32 tcg_global_mem_new_i32(TCGv_ptr reg, intptr_t offset,
873 const char *name)
874{
875 TCGTemp *t = tcg_global_mem_new_internal(TCG_TYPE_I32, reg, offset, name);
876 return temp_tcgv_i32(t);
877}
878
879static inline TCGv_i32 tcg_temp_new_i32(void)
880{
881 TCGTemp *t = tcg_temp_new_internal(TCG_TYPE_I32, false);
882 return temp_tcgv_i32(t);
883}
884
885static inline TCGv_i32 tcg_temp_local_new_i32(void)
886{
887 TCGTemp *t = tcg_temp_new_internal(TCG_TYPE_I32, true);
888 return temp_tcgv_i32(t);
889}
890
891static inline TCGv_i64 tcg_global_mem_new_i64(TCGv_ptr reg, intptr_t offset,
892 const char *name)
893{
894 TCGTemp *t = tcg_global_mem_new_internal(TCG_TYPE_I64, reg, offset, name);
895 return temp_tcgv_i64(t);
896}
897
898static inline TCGv_i64 tcg_temp_new_i64(void)
899{
900 TCGTemp *t = tcg_temp_new_internal(TCG_TYPE_I64, false);
901 return temp_tcgv_i64(t);
902}
903
904static inline TCGv_i64 tcg_temp_local_new_i64(void)
905{
906 TCGTemp *t = tcg_temp_new_internal(TCG_TYPE_I64, true);
907 return temp_tcgv_i64(t);
908}
909
910static inline TCGv_ptr tcg_global_mem_new_ptr(TCGv_ptr reg, intptr_t offset,
911 const char *name)
912{
913 TCGTemp *t = tcg_global_mem_new_internal(TCG_TYPE_PTR, reg, offset, name);
914 return temp_tcgv_ptr(t);
915}
916
917static inline TCGv_ptr tcg_temp_new_ptr(void)
918{
919 TCGTemp *t = tcg_temp_new_internal(TCG_TYPE_PTR, false);
920 return temp_tcgv_ptr(t);
921}
922
923static inline TCGv_ptr tcg_temp_local_new_ptr(void)
924{
925 TCGTemp *t = tcg_temp_new_internal(TCG_TYPE_PTR, true);
926 return temp_tcgv_ptr(t);
927}
928
929#if defined(CONFIG_DEBUG_TCG)
930
931
932
933
934
935void tcg_clear_temp_count(void);
936int tcg_check_temp_count(void);
937#else
938#define tcg_clear_temp_count() do { } while (0)
939#define tcg_check_temp_count() 0
940#endif
941
942int64_t tcg_cpu_exec_time(void);
943void tcg_dump_info(GString *buf);
944void tcg_dump_op_count(GString *buf);
945
946#define TCG_CT_CONST 1
947
948typedef struct TCGArgConstraint {
949 unsigned ct : 16;
950 unsigned alias_index : 4;
951 unsigned sort_index : 4;
952 bool oalias : 1;
953 bool ialias : 1;
954 bool newreg : 1;
955 TCGRegSet regs;
956} TCGArgConstraint;
957
958#define TCG_MAX_OP_ARGS 16
959
960
961enum {
962
963 TCG_OPF_BB_EXIT = 0x01,
964
965 TCG_OPF_BB_END = 0x02,
966
967 TCG_OPF_CALL_CLOBBER = 0x04,
968
969
970 TCG_OPF_SIDE_EFFECTS = 0x08,
971
972 TCG_OPF_64BIT = 0x10,
973
974
975 TCG_OPF_NOT_PRESENT = 0x20,
976
977 TCG_OPF_VECTOR = 0x40,
978
979 TCG_OPF_COND_BRANCH = 0x80
980};
981
982typedef struct TCGOpDef {
983 const char *name;
984 uint8_t nb_oargs, nb_iargs, nb_cargs, nb_args;
985 uint8_t flags;
986 TCGArgConstraint *args_ct;
987} TCGOpDef;
988
989extern TCGOpDef tcg_op_defs[];
990extern const size_t tcg_op_defs_max;
991
992typedef struct TCGTargetOpDef {
993 TCGOpcode op;
994 const char *args_ct_str[TCG_MAX_OP_ARGS];
995} TCGTargetOpDef;
996
997#define tcg_abort() \
998do {\
999 fprintf(stderr, "%s:%d: tcg fatal error\n", __FILE__, __LINE__);\
1000 abort();\
1001} while (0)
1002
1003bool tcg_op_supported(TCGOpcode op);
1004
1005void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args);
1006
1007TCGOp *tcg_emit_op(TCGOpcode opc);
1008void tcg_op_remove(TCGContext *s, TCGOp *op);
1009TCGOp *tcg_op_insert_before(TCGContext *s, TCGOp *op, TCGOpcode opc);
1010TCGOp *tcg_op_insert_after(TCGContext *s, TCGOp *op, TCGOpcode opc);
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020void tcg_remove_ops_after(TCGOp *op);
1021
1022void tcg_optimize(TCGContext *s);
1023
1024
1025TCGv_i32 tcg_const_i32(int32_t val);
1026TCGv_i64 tcg_const_i64(int64_t val);
1027TCGv_i32 tcg_const_local_i32(int32_t val);
1028TCGv_i64 tcg_const_local_i64(int64_t val);
1029TCGv_vec tcg_const_zeros_vec(TCGType);
1030TCGv_vec tcg_const_ones_vec(TCGType);
1031TCGv_vec tcg_const_zeros_vec_matching(TCGv_vec);
1032TCGv_vec tcg_const_ones_vec_matching(TCGv_vec);
1033
1034
1035
1036
1037
1038
1039TCGTemp *tcg_constant_internal(TCGType type, int64_t val);
1040
1041static inline TCGv_i32 tcg_constant_i32(int32_t val)
1042{
1043 return temp_tcgv_i32(tcg_constant_internal(TCG_TYPE_I32, val));
1044}
1045
1046static inline TCGv_i64 tcg_constant_i64(int64_t val)
1047{
1048 return temp_tcgv_i64(tcg_constant_internal(TCG_TYPE_I64, val));
1049}
1050
1051TCGv_vec tcg_constant_vec(TCGType type, unsigned vece, int64_t val);
1052TCGv_vec tcg_constant_vec_matching(TCGv_vec match, unsigned vece, int64_t val);
1053
1054#if UINTPTR_MAX == UINT32_MAX
1055# define tcg_const_ptr(x) ((TCGv_ptr)tcg_const_i32((intptr_t)(x)))
1056# define tcg_const_local_ptr(x) ((TCGv_ptr)tcg_const_local_i32((intptr_t)(x)))
1057#else
1058# define tcg_const_ptr(x) ((TCGv_ptr)tcg_const_i64((intptr_t)(x)))
1059# define tcg_const_local_ptr(x) ((TCGv_ptr)tcg_const_local_i64((intptr_t)(x)))
1060#endif
1061
1062TCGLabel *gen_new_label(void);
1063
1064
1065
1066
1067
1068
1069
1070
1071static inline TCGArg label_arg(TCGLabel *l)
1072{
1073 return (uintptr_t)l;
1074}
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084static inline TCGLabel *arg_label(TCGArg i)
1085{
1086 return (TCGLabel *)(uintptr_t)i;
1087}
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102static inline ptrdiff_t tcg_ptr_byte_diff(const void *a, const void *b)
1103{
1104 return a - b;
1105}
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116static inline ptrdiff_t tcg_pcrel_diff(TCGContext *s, const void *target)
1117{
1118 return tcg_ptr_byte_diff(target, tcg_splitwx_to_rx(s->code_ptr));
1119}
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129static inline ptrdiff_t tcg_tbrel_diff(TCGContext *s, const void *target)
1130{
1131 return tcg_ptr_byte_diff(target, tcg_splitwx_to_rx(s->code_buf));
1132}
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142static inline size_t tcg_current_code_size(TCGContext *s)
1143{
1144 return tcg_ptr_byte_diff(s->code_ptr, s->code_buf);
1145}
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192#define TB_EXIT_MASK 3
1193#define TB_EXIT_IDX0 0
1194#define TB_EXIT_IDX1 1
1195#define TB_EXIT_IDXMAX 1
1196#define TB_EXIT_REQUESTED 3
1197
1198#ifdef CONFIG_TCG_INTERPRETER
1199uintptr_t tcg_qemu_tb_exec(CPUArchState *env, const void *tb_ptr);
1200#else
1201typedef uintptr_t tcg_prologue_fn(CPUArchState *env, const void *tb_ptr);
1202extern tcg_prologue_fn *tcg_qemu_tb_exec;
1203#endif
1204
1205void tcg_register_jit(const void *buf, size_t buf_size);
1206
1207#if TCG_TARGET_MAYBE_vec
1208
1209
1210
1211int tcg_can_emit_vec_op(TCGOpcode, TCGType, unsigned);
1212#else
1213static inline int tcg_can_emit_vec_op(TCGOpcode o, TCGType t, unsigned ve)
1214{
1215 return 0;
1216}
1217#endif
1218
1219
1220void tcg_expand_vec_op(TCGOpcode, TCGType, unsigned, TCGArg, ...);
1221
1222
1223uint64_t dup_const(unsigned vece, uint64_t c);
1224
1225#define dup_const(VECE, C) \
1226 (__builtin_constant_p(VECE) \
1227 ? ( (VECE) == MO_8 ? 0x0101010101010101ull * (uint8_t)(C) \
1228 : (VECE) == MO_16 ? 0x0001000100010001ull * (uint16_t)(C) \
1229 : (VECE) == MO_32 ? 0x0000000100000001ull * (uint32_t)(C) \
1230 : (VECE) == MO_64 ? (uint64_t)(C) \
1231 : (qemu_build_not_reached_always(), 0)) \
1232 : dup_const(VECE, C))
1233
1234#if TARGET_LONG_BITS == 64
1235# define dup_const_tl dup_const
1236#else
1237# define dup_const_tl(VECE, C) \
1238 (__builtin_constant_p(VECE) \
1239 ? ( (VECE) == MO_8 ? 0x01010101ul * (uint8_t)(C) \
1240 : (VECE) == MO_16 ? 0x00010001ul * (uint16_t)(C) \
1241 : (VECE) == MO_32 ? 0x00000001ul * (uint32_t)(C) \
1242 : (qemu_build_not_reached_always(), 0)) \
1243 : (target_long)dup_const(VECE, C))
1244#endif
1245
1246#ifdef CONFIG_DEBUG_TCG
1247void tcg_assert_listed_vecop(TCGOpcode);
1248#else
1249static inline void tcg_assert_listed_vecop(TCGOpcode op) { }
1250#endif
1251
1252static inline const TCGOpcode *tcg_swap_vecop_list(const TCGOpcode *n)
1253{
1254#ifdef CONFIG_DEBUG_TCG
1255 const TCGOpcode *o = tcg_ctx->vecop_list;
1256 tcg_ctx->vecop_list = n;
1257 return o;
1258#else
1259 return NULL;
1260#endif
1261}
1262
1263bool tcg_can_emit_vecop_list(const TCGOpcode *, TCGType, unsigned);
1264
1265#endif
1266