1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26#define USE_TCG_OPTIMIZATIONS
27
28#include "qemu/osdep.h"
29
30
31#undef DEBUG_JIT
32
33#include "qemu/error-report.h"
34#include "qemu/cutils.h"
35#include "qemu/host-utils.h"
36#include "qemu/qemu-print.h"
37#include "qemu/timer.h"
38#include "qemu/cacheflush.h"
39
40
41
42
43#define NO_CPU_IO_DEFS
44#include "cpu.h"
45
46#include "exec/exec-all.h"
47
48#if !defined(CONFIG_USER_ONLY)
49#include "hw/boards.h"
50#endif
51
52#include "tcg/tcg-op.h"
53
54#if UINTPTR_MAX == UINT32_MAX
55# define ELF_CLASS ELFCLASS32
56#else
57# define ELF_CLASS ELFCLASS64
58#endif
59#ifdef HOST_WORDS_BIGENDIAN
60# define ELF_DATA ELFDATA2MSB
61#else
62# define ELF_DATA ELFDATA2LSB
63#endif
64
65#include "elf.h"
66#include "exec/log.h"
67#include "sysemu/sysemu.h"
68
69
70
71static void tcg_target_init(TCGContext *s);
72static void tcg_target_qemu_prologue(TCGContext *s);
73static bool patch_reloc(tcg_insn_unit *code_ptr, int type,
74 intptr_t value, intptr_t addend);
75
76
77typedef struct {
78 uint32_t len __attribute__((aligned((sizeof(void *)))));
79 uint32_t id;
80 uint8_t version;
81 char augmentation[1];
82 uint8_t code_align;
83 uint8_t data_align;
84 uint8_t return_column;
85} DebugFrameCIE;
86
87typedef struct QEMU_PACKED {
88 uint32_t len __attribute__((aligned((sizeof(void *)))));
89 uint32_t cie_offset;
90 uintptr_t func_start;
91 uintptr_t func_len;
92} DebugFrameFDEHeader;
93
94typedef struct QEMU_PACKED {
95 DebugFrameCIE cie;
96 DebugFrameFDEHeader fde;
97} DebugFrameHeader;
98
99static void tcg_register_jit_int(const void *buf, size_t size,
100 const void *debug_frame,
101 size_t debug_frame_size)
102 __attribute__((unused));
103
104
105static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1,
106 intptr_t arg2);
107static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
108static void tcg_out_movi(TCGContext *s, TCGType type,
109 TCGReg ret, tcg_target_long arg);
110static void tcg_out_op(TCGContext *s, TCGOpcode opc,
111 const TCGArg args[TCG_MAX_OP_ARGS],
112 const int const_args[TCG_MAX_OP_ARGS]);
113#if TCG_TARGET_MAYBE_vec
114static bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece,
115 TCGReg dst, TCGReg src);
116static bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece,
117 TCGReg dst, TCGReg base, intptr_t offset);
118static void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece,
119 TCGReg dst, int64_t arg);
120static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc,
121 unsigned vecl, unsigned vece,
122 const TCGArg args[TCG_MAX_OP_ARGS],
123 const int const_args[TCG_MAX_OP_ARGS]);
124#else
125static inline bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece,
126 TCGReg dst, TCGReg src)
127{
128 g_assert_not_reached();
129}
130static inline bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece,
131 TCGReg dst, TCGReg base, intptr_t offset)
132{
133 g_assert_not_reached();
134}
135static inline void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece,
136 TCGReg dst, int64_t arg)
137{
138 g_assert_not_reached();
139}
140static inline void tcg_out_vec_op(TCGContext *s, TCGOpcode opc,
141 unsigned vecl, unsigned vece,
142 const TCGArg args[TCG_MAX_OP_ARGS],
143 const int const_args[TCG_MAX_OP_ARGS])
144{
145 g_assert_not_reached();
146}
147#endif
148static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1,
149 intptr_t arg2);
150static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
151 TCGReg base, intptr_t ofs);
152static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target);
153static int tcg_target_const_match(tcg_target_long val, TCGType type,
154 const TCGArgConstraint *arg_ct);
155#ifdef TCG_TARGET_NEED_LDST_LABELS
156static int tcg_out_ldst_finalize(TCGContext *s);
157#endif
158
159#define TCG_HIGHWATER 1024
160
161static TCGContext **tcg_ctxs;
162static unsigned int n_tcg_ctxs;
163TCGv_env cpu_env = 0;
164const void *tcg_code_gen_epilogue;
165uintptr_t tcg_splitwx_diff;
166
167#ifndef CONFIG_TCG_INTERPRETER
168tcg_prologue_fn *tcg_qemu_tb_exec;
169#endif
170
171struct tcg_region_tree {
172 QemuMutex lock;
173 GTree *tree;
174
175};
176
177
178
179
180
181
182
183struct tcg_region_state {
184 QemuMutex lock;
185
186
187 void *start;
188 void *start_aligned;
189 void *end;
190 size_t n;
191 size_t size;
192 size_t stride;
193
194
195 size_t current;
196 size_t agg_size_full;
197};
198
199static struct tcg_region_state region;
200
201
202
203
204
205static void *region_trees;
206static size_t tree_size;
207static TCGRegSet tcg_target_available_regs[TCG_TYPE_COUNT];
208static TCGRegSet tcg_target_call_clobber_regs;
209
210#if TCG_TARGET_INSN_UNIT_SIZE == 1
211static __attribute__((unused)) inline void tcg_out8(TCGContext *s, uint8_t v)
212{
213 *s->code_ptr++ = v;
214}
215
216static __attribute__((unused)) inline void tcg_patch8(tcg_insn_unit *p,
217 uint8_t v)
218{
219 *p = v;
220}
221#endif
222
223#if TCG_TARGET_INSN_UNIT_SIZE <= 2
224static __attribute__((unused)) inline void tcg_out16(TCGContext *s, uint16_t v)
225{
226 if (TCG_TARGET_INSN_UNIT_SIZE == 2) {
227 *s->code_ptr++ = v;
228 } else {
229 tcg_insn_unit *p = s->code_ptr;
230 memcpy(p, &v, sizeof(v));
231 s->code_ptr = p + (2 / TCG_TARGET_INSN_UNIT_SIZE);
232 }
233}
234
235static __attribute__((unused)) inline void tcg_patch16(tcg_insn_unit *p,
236 uint16_t v)
237{
238 if (TCG_TARGET_INSN_UNIT_SIZE == 2) {
239 *p = v;
240 } else {
241 memcpy(p, &v, sizeof(v));
242 }
243}
244#endif
245
246#if TCG_TARGET_INSN_UNIT_SIZE <= 4
247static __attribute__((unused)) inline void tcg_out32(TCGContext *s, uint32_t v)
248{
249 if (TCG_TARGET_INSN_UNIT_SIZE == 4) {
250 *s->code_ptr++ = v;
251 } else {
252 tcg_insn_unit *p = s->code_ptr;
253 memcpy(p, &v, sizeof(v));
254 s->code_ptr = p + (4 / TCG_TARGET_INSN_UNIT_SIZE);
255 }
256}
257
258static __attribute__((unused)) inline void tcg_patch32(tcg_insn_unit *p,
259 uint32_t v)
260{
261 if (TCG_TARGET_INSN_UNIT_SIZE == 4) {
262 *p = v;
263 } else {
264 memcpy(p, &v, sizeof(v));
265 }
266}
267#endif
268
269#if TCG_TARGET_INSN_UNIT_SIZE <= 8
270static __attribute__((unused)) inline void tcg_out64(TCGContext *s, uint64_t v)
271{
272 if (TCG_TARGET_INSN_UNIT_SIZE == 8) {
273 *s->code_ptr++ = v;
274 } else {
275 tcg_insn_unit *p = s->code_ptr;
276 memcpy(p, &v, sizeof(v));
277 s->code_ptr = p + (8 / TCG_TARGET_INSN_UNIT_SIZE);
278 }
279}
280
281static __attribute__((unused)) inline void tcg_patch64(tcg_insn_unit *p,
282 uint64_t v)
283{
284 if (TCG_TARGET_INSN_UNIT_SIZE == 8) {
285 *p = v;
286 } else {
287 memcpy(p, &v, sizeof(v));
288 }
289}
290#endif
291
292
293
294static void tcg_out_reloc(TCGContext *s, tcg_insn_unit *code_ptr, int type,
295 TCGLabel *l, intptr_t addend)
296{
297 TCGRelocation *r = tcg_malloc(sizeof(TCGRelocation));
298
299 r->type = type;
300 r->ptr = code_ptr;
301 r->addend = addend;
302 QSIMPLEQ_INSERT_TAIL(&l->relocs, r, next);
303}
304
305static void tcg_out_label(TCGContext *s, TCGLabel *l)
306{
307 tcg_debug_assert(!l->has_value);
308 l->has_value = 1;
309 l->u.value_ptr = tcg_splitwx_to_rx(s->code_ptr);
310}
311
312TCGLabel *gen_new_label(void)
313{
314 TCGContext *s = tcg_ctx;
315 TCGLabel *l = tcg_malloc(sizeof(TCGLabel));
316
317 memset(l, 0, sizeof(TCGLabel));
318 l->id = s->nb_labels++;
319 QSIMPLEQ_INIT(&l->relocs);
320
321 QSIMPLEQ_INSERT_TAIL(&s->labels, l, next);
322
323 return l;
324}
325
326static bool tcg_resolve_relocs(TCGContext *s)
327{
328 TCGLabel *l;
329
330 QSIMPLEQ_FOREACH(l, &s->labels, next) {
331 TCGRelocation *r;
332 uintptr_t value = l->u.value;
333
334 QSIMPLEQ_FOREACH(r, &l->relocs, next) {
335 if (!patch_reloc(r->ptr, r->type, value, r->addend)) {
336 return false;
337 }
338 }
339 }
340 return true;
341}
342
343static void set_jmp_reset_offset(TCGContext *s, int which)
344{
345
346
347
348
349 s->tb_jmp_reset_offset[which] = tcg_current_code_size(s);
350}
351
352
353static void QEMU_NORETURN tcg_raise_tb_overflow(TCGContext *s)
354{
355 siglongjmp(s->jmp_trans, -2);
356}
357
358#define C_PFX1(P, A) P##A
359#define C_PFX2(P, A, B) P##A##_##B
360#define C_PFX3(P, A, B, C) P##A##_##B##_##C
361#define C_PFX4(P, A, B, C, D) P##A##_##B##_##C##_##D
362#define C_PFX5(P, A, B, C, D, E) P##A##_##B##_##C##_##D##_##E
363#define C_PFX6(P, A, B, C, D, E, F) P##A##_##B##_##C##_##D##_##E##_##F
364
365
366
367#define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1),
368#define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2),
369#define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3),
370#define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4),
371
372#define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1),
373#define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2),
374#define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3),
375#define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4),
376
377#define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2),
378
379#define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1),
380#define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2),
381#define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3),
382#define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4),
383
384typedef enum {
385#include "tcg-target-con-set.h"
386} TCGConstraintSetIndex;
387
388static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode);
389
390#undef C_O0_I1
391#undef C_O0_I2
392#undef C_O0_I3
393#undef C_O0_I4
394#undef C_O1_I1
395#undef C_O1_I2
396#undef C_O1_I3
397#undef C_O1_I4
398#undef C_N1_I2
399#undef C_O2_I1
400#undef C_O2_I2
401#undef C_O2_I3
402#undef C_O2_I4
403
404
405
406#define C_O0_I1(I1) { .args_ct_str = { #I1 } },
407#define C_O0_I2(I1, I2) { .args_ct_str = { #I1, #I2 } },
408#define C_O0_I3(I1, I2, I3) { .args_ct_str = { #I1, #I2, #I3 } },
409#define C_O0_I4(I1, I2, I3, I4) { .args_ct_str = { #I1, #I2, #I3, #I4 } },
410
411#define C_O1_I1(O1, I1) { .args_ct_str = { #O1, #I1 } },
412#define C_O1_I2(O1, I1, I2) { .args_ct_str = { #O1, #I1, #I2 } },
413#define C_O1_I3(O1, I1, I2, I3) { .args_ct_str = { #O1, #I1, #I2, #I3 } },
414#define C_O1_I4(O1, I1, I2, I3, I4) { .args_ct_str = { #O1, #I1, #I2, #I3, #I4 } },
415
416#define C_N1_I2(O1, I1, I2) { .args_ct_str = { "&" #O1, #I1, #I2 } },
417
418#define C_O2_I1(O1, O2, I1) { .args_ct_str = { #O1, #O2, #I1 } },
419#define C_O2_I2(O1, O2, I1, I2) { .args_ct_str = { #O1, #O2, #I1, #I2 } },
420#define C_O2_I3(O1, O2, I1, I2, I3) { .args_ct_str = { #O1, #O2, #I1, #I2, #I3 } },
421#define C_O2_I4(O1, O2, I1, I2, I3, I4) { .args_ct_str = { #O1, #O2, #I1, #I2, #I3, #I4 } },
422
423static const TCGTargetOpDef constraint_sets[] = {
424#include "tcg-target-con-set.h"
425};
426
427
428#undef C_O0_I1
429#undef C_O0_I2
430#undef C_O0_I3
431#undef C_O0_I4
432#undef C_O1_I1
433#undef C_O1_I2
434#undef C_O1_I3
435#undef C_O1_I4
436#undef C_N1_I2
437#undef C_O2_I1
438#undef C_O2_I2
439#undef C_O2_I3
440#undef C_O2_I4
441
442
443
444#define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1)
445#define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2)
446#define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3)
447#define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4)
448
449#define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1)
450#define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2)
451#define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3)
452#define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4)
453
454#define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2)
455
456#define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1)
457#define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2)
458#define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3)
459#define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4)
460
461#include "tcg-target.c.inc"
462
463
464static int ptr_cmp_tb_tc(const void *ptr, const struct tb_tc *s)
465{
466 if (ptr >= s->ptr + s->size) {
467 return 1;
468 } else if (ptr < s->ptr) {
469 return -1;
470 }
471 return 0;
472}
473
474static gint tb_tc_cmp(gconstpointer ap, gconstpointer bp)
475{
476 const struct tb_tc *a = ap;
477 const struct tb_tc *b = bp;
478
479
480
481
482
483
484 if (likely(a->size && b->size)) {
485 if (a->ptr > b->ptr) {
486 return 1;
487 } else if (a->ptr < b->ptr) {
488 return -1;
489 }
490
491 g_assert(a->size == b->size);
492 return 0;
493 }
494
495
496
497
498
499 if (likely(a->size == 0)) {
500 return ptr_cmp_tb_tc(a->ptr, b);
501 }
502 return ptr_cmp_tb_tc(b->ptr, a);
503}
504
505static void tcg_region_trees_init(void)
506{
507 size_t i;
508
509 tree_size = ROUND_UP(sizeof(struct tcg_region_tree), qemu_dcache_linesize);
510 region_trees = qemu_memalign(qemu_dcache_linesize, region.n * tree_size);
511 for (i = 0; i < region.n; i++) {
512 struct tcg_region_tree *rt = region_trees + i * tree_size;
513
514 qemu_mutex_init(&rt->lock);
515 rt->tree = g_tree_new(tb_tc_cmp);
516 }
517}
518
519static struct tcg_region_tree *tc_ptr_to_region_tree(const void *p)
520{
521 size_t region_idx;
522
523
524
525
526
527 if (!in_code_gen_buffer(p)) {
528 p -= tcg_splitwx_diff;
529 if (!in_code_gen_buffer(p)) {
530 return NULL;
531 }
532 }
533
534 if (p < region.start_aligned) {
535 region_idx = 0;
536 } else {
537 ptrdiff_t offset = p - region.start_aligned;
538
539 if (offset > region.stride * (region.n - 1)) {
540 region_idx = region.n - 1;
541 } else {
542 region_idx = offset / region.stride;
543 }
544 }
545 return region_trees + region_idx * tree_size;
546}
547
548void tcg_tb_insert(TranslationBlock *tb)
549{
550 struct tcg_region_tree *rt = tc_ptr_to_region_tree(tb->tc.ptr);
551
552 g_assert(rt != NULL);
553 qemu_mutex_lock(&rt->lock);
554 g_tree_insert(rt->tree, &tb->tc, tb);
555 qemu_mutex_unlock(&rt->lock);
556}
557
558void tcg_tb_remove(TranslationBlock *tb)
559{
560 struct tcg_region_tree *rt = tc_ptr_to_region_tree(tb->tc.ptr);
561
562 g_assert(rt != NULL);
563 qemu_mutex_lock(&rt->lock);
564 g_tree_remove(rt->tree, &tb->tc);
565 qemu_mutex_unlock(&rt->lock);
566}
567
568
569
570
571
572
573TranslationBlock *tcg_tb_lookup(uintptr_t tc_ptr)
574{
575 struct tcg_region_tree *rt = tc_ptr_to_region_tree((void *)tc_ptr);
576 TranslationBlock *tb;
577 struct tb_tc s = { .ptr = (void *)tc_ptr };
578
579 if (rt == NULL) {
580 return NULL;
581 }
582
583 qemu_mutex_lock(&rt->lock);
584 tb = g_tree_lookup(rt->tree, &s);
585 qemu_mutex_unlock(&rt->lock);
586 return tb;
587}
588
589static void tcg_region_tree_lock_all(void)
590{
591 size_t i;
592
593 for (i = 0; i < region.n; i++) {
594 struct tcg_region_tree *rt = region_trees + i * tree_size;
595
596 qemu_mutex_lock(&rt->lock);
597 }
598}
599
600static void tcg_region_tree_unlock_all(void)
601{
602 size_t i;
603
604 for (i = 0; i < region.n; i++) {
605 struct tcg_region_tree *rt = region_trees + i * tree_size;
606
607 qemu_mutex_unlock(&rt->lock);
608 }
609}
610
611void tcg_tb_foreach(GTraverseFunc func, gpointer user_data)
612{
613 size_t i;
614
615 tcg_region_tree_lock_all();
616 for (i = 0; i < region.n; i++) {
617 struct tcg_region_tree *rt = region_trees + i * tree_size;
618
619 g_tree_foreach(rt->tree, func, user_data);
620 }
621 tcg_region_tree_unlock_all();
622}
623
624size_t tcg_nb_tbs(void)
625{
626 size_t nb_tbs = 0;
627 size_t i;
628
629 tcg_region_tree_lock_all();
630 for (i = 0; i < region.n; i++) {
631 struct tcg_region_tree *rt = region_trees + i * tree_size;
632
633 nb_tbs += g_tree_nnodes(rt->tree);
634 }
635 tcg_region_tree_unlock_all();
636 return nb_tbs;
637}
638
639static gboolean tcg_region_tree_traverse(gpointer k, gpointer v, gpointer data)
640{
641 TranslationBlock *tb = v;
642
643 tb_destroy(tb);
644 return FALSE;
645}
646
647static void tcg_region_tree_reset_all(void)
648{
649 size_t i;
650
651 tcg_region_tree_lock_all();
652 for (i = 0; i < region.n; i++) {
653 struct tcg_region_tree *rt = region_trees + i * tree_size;
654
655 g_tree_foreach(rt->tree, tcg_region_tree_traverse, NULL);
656
657 g_tree_ref(rt->tree);
658 g_tree_destroy(rt->tree);
659 }
660 tcg_region_tree_unlock_all();
661}
662
663static void tcg_region_bounds(size_t curr_region, void **pstart, void **pend)
664{
665 void *start, *end;
666
667 start = region.start_aligned + curr_region * region.stride;
668 end = start + region.size;
669
670 if (curr_region == 0) {
671 start = region.start;
672 }
673 if (curr_region == region.n - 1) {
674 end = region.end;
675 }
676
677 *pstart = start;
678 *pend = end;
679}
680
681static void tcg_region_assign(TCGContext *s, size_t curr_region)
682{
683 void *start, *end;
684
685 tcg_region_bounds(curr_region, &start, &end);
686
687 s->code_gen_buffer = start;
688 s->code_gen_ptr = start;
689 s->code_gen_buffer_size = end - start;
690 s->code_gen_highwater = end - TCG_HIGHWATER;
691}
692
693static bool tcg_region_alloc__locked(TCGContext *s)
694{
695 if (region.current == region.n) {
696 return true;
697 }
698 tcg_region_assign(s, region.current);
699 region.current++;
700 return false;
701}
702
703
704
705
706
707static bool tcg_region_alloc(TCGContext *s)
708{
709 bool err;
710
711 size_t size_full = s->code_gen_buffer_size;
712
713 qemu_mutex_lock(®ion.lock);
714 err = tcg_region_alloc__locked(s);
715 if (!err) {
716 region.agg_size_full += size_full - TCG_HIGHWATER;
717 }
718 qemu_mutex_unlock(®ion.lock);
719 return err;
720}
721
722
723
724
725
726static inline bool tcg_region_initial_alloc__locked(TCGContext *s)
727{
728 return tcg_region_alloc__locked(s);
729}
730
731
732void tcg_region_reset_all(void)
733{
734 unsigned int n_ctxs = qatomic_read(&n_tcg_ctxs);
735 unsigned int i;
736
737 qemu_mutex_lock(®ion.lock);
738 region.current = 0;
739 region.agg_size_full = 0;
740
741 for (i = 0; i < n_ctxs; i++) {
742 TCGContext *s = qatomic_read(&tcg_ctxs[i]);
743 bool err = tcg_region_initial_alloc__locked(s);
744
745 g_assert(!err);
746 }
747 qemu_mutex_unlock(®ion.lock);
748
749 tcg_region_tree_reset_all();
750}
751
752#ifdef CONFIG_USER_ONLY
753static size_t tcg_n_regions(void)
754{
755 return 1;
756}
757#else
758
759
760
761
762
763
764static size_t tcg_n_regions(void)
765{
766 size_t i;
767
768
769#if !defined(CONFIG_USER_ONLY)
770 MachineState *ms = MACHINE(qdev_get_machine());
771 unsigned int max_cpus = ms->smp.max_cpus;
772#endif
773 if (max_cpus == 1 || !qemu_tcg_mttcg_enabled()) {
774 return 1;
775 }
776
777
778 for (i = 8; i > 0; i--) {
779 size_t regions_per_thread = i;
780 size_t region_size;
781
782 region_size = tcg_init_ctx.code_gen_buffer_size;
783 region_size /= max_cpus * regions_per_thread;
784
785 if (region_size >= 2 * 1024u * 1024) {
786 return max_cpus * regions_per_thread;
787 }
788 }
789
790 return max_cpus;
791}
792#endif
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822void tcg_region_init(void)
823{
824 void *buf = tcg_init_ctx.code_gen_buffer;
825 void *aligned;
826 size_t size = tcg_init_ctx.code_gen_buffer_size;
827 size_t page_size = qemu_real_host_page_size;
828 size_t region_size;
829 size_t n_regions;
830 size_t i;
831
832 n_regions = tcg_n_regions();
833
834
835 aligned = QEMU_ALIGN_PTR_UP(buf, page_size);
836 g_assert(aligned < tcg_init_ctx.code_gen_buffer + size);
837
838
839
840
841
842 region_size = (size - (aligned - buf)) / n_regions;
843 region_size = QEMU_ALIGN_DOWN(region_size, page_size);
844
845
846 g_assert(region_size >= 2 * page_size);
847
848
849 qemu_mutex_init(®ion.lock);
850 region.n = n_regions;
851 region.size = region_size - page_size;
852 region.stride = region_size;
853 region.start = buf;
854 region.start_aligned = aligned;
855
856 region.end = QEMU_ALIGN_PTR_DOWN(buf + size, page_size);
857
858 region.end -= page_size;
859
860
861
862
863
864
865 for (i = 0; i < region.n; i++) {
866 void *start, *end;
867
868 tcg_region_bounds(i, &start, &end);
869
870
871
872
873
874
875 (void)qemu_mprotect_none(end, page_size);
876 }
877
878 tcg_region_trees_init();
879
880
881#ifdef CONFIG_USER_ONLY
882 {
883 bool err = tcg_region_initial_alloc__locked(tcg_ctx);
884
885 g_assert(!err);
886 }
887#endif
888}
889
890#ifdef CONFIG_DEBUG_TCG
891const void *tcg_splitwx_to_rx(void *rw)
892{
893
894 if (rw) {
895 g_assert(in_code_gen_buffer(rw));
896 rw += tcg_splitwx_diff;
897 }
898 return rw;
899}
900
901void *tcg_splitwx_to_rw(const void *rx)
902{
903
904 if (rx) {
905 rx -= tcg_splitwx_diff;
906
907 g_assert(in_code_gen_buffer(rx));
908 }
909 return (void *)rx;
910}
911#endif
912
913static void alloc_tcg_plugin_context(TCGContext *s)
914{
915#ifdef CONFIG_PLUGIN
916 s->plugin_tb = g_new0(struct qemu_plugin_tb, 1);
917 s->plugin_tb->insns =
918 g_ptr_array_new_with_free_func(qemu_plugin_insn_cleanup_fn);
919#endif
920}
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937#ifdef CONFIG_USER_ONLY
938void tcg_register_thread(void)
939{
940 tcg_ctx = &tcg_init_ctx;
941}
942#else
943void tcg_register_thread(void)
944{
945 MachineState *ms = MACHINE(qdev_get_machine());
946 TCGContext *s = g_malloc(sizeof(*s));
947 unsigned int i, n;
948 bool err;
949
950 *s = tcg_init_ctx;
951
952
953 for (i = 0, n = tcg_init_ctx.nb_globals; i < n; ++i) {
954 if (tcg_init_ctx.temps[i].mem_base) {
955 ptrdiff_t b = tcg_init_ctx.temps[i].mem_base - tcg_init_ctx.temps;
956 tcg_debug_assert(b >= 0 && b < n);
957 s->temps[i].mem_base = &s->temps[b];
958 }
959 }
960
961
962 n = qatomic_fetch_inc(&n_tcg_ctxs);
963 g_assert(n < ms->smp.max_cpus);
964 qatomic_set(&tcg_ctxs[n], s);
965
966 if (n > 0) {
967 alloc_tcg_plugin_context(s);
968 }
969
970 tcg_ctx = s;
971 qemu_mutex_lock(®ion.lock);
972 err = tcg_region_initial_alloc__locked(tcg_ctx);
973 g_assert(!err);
974 qemu_mutex_unlock(®ion.lock);
975}
976#endif
977
978
979
980
981
982
983
984
985size_t tcg_code_size(void)
986{
987 unsigned int n_ctxs = qatomic_read(&n_tcg_ctxs);
988 unsigned int i;
989 size_t total;
990
991 qemu_mutex_lock(®ion.lock);
992 total = region.agg_size_full;
993 for (i = 0; i < n_ctxs; i++) {
994 const TCGContext *s = qatomic_read(&tcg_ctxs[i]);
995 size_t size;
996
997 size = qatomic_read(&s->code_gen_ptr) - s->code_gen_buffer;
998 g_assert(size <= s->code_gen_buffer_size);
999 total += size;
1000 }
1001 qemu_mutex_unlock(®ion.lock);
1002 return total;
1003}
1004
1005
1006
1007
1008
1009
1010size_t tcg_code_capacity(void)
1011{
1012 size_t guard_size, capacity;
1013
1014
1015 guard_size = region.stride - region.size;
1016 capacity = region.end + guard_size - region.start;
1017 capacity -= region.n * (guard_size + TCG_HIGHWATER);
1018 return capacity;
1019}
1020
1021size_t tcg_tb_phys_invalidate_count(void)
1022{
1023 unsigned int n_ctxs = qatomic_read(&n_tcg_ctxs);
1024 unsigned int i;
1025 size_t total = 0;
1026
1027 for (i = 0; i < n_ctxs; i++) {
1028 const TCGContext *s = qatomic_read(&tcg_ctxs[i]);
1029
1030 total += qatomic_read(&s->tb_phys_invalidate_count);
1031 }
1032 return total;
1033}
1034
1035
1036void *tcg_malloc_internal(TCGContext *s, int size)
1037{
1038 TCGPool *p;
1039 int pool_size;
1040
1041 if (size > TCG_POOL_CHUNK_SIZE) {
1042
1043 p = g_malloc(sizeof(TCGPool) + size);
1044 p->size = size;
1045 p->next = s->pool_first_large;
1046 s->pool_first_large = p;
1047 return p->data;
1048 } else {
1049 p = s->pool_current;
1050 if (!p) {
1051 p = s->pool_first;
1052 if (!p)
1053 goto new_pool;
1054 } else {
1055 if (!p->next) {
1056 new_pool:
1057 pool_size = TCG_POOL_CHUNK_SIZE;
1058 p = g_malloc(sizeof(TCGPool) + pool_size);
1059 p->size = pool_size;
1060 p->next = NULL;
1061 if (s->pool_current)
1062 s->pool_current->next = p;
1063 else
1064 s->pool_first = p;
1065 } else {
1066 p = p->next;
1067 }
1068 }
1069 }
1070 s->pool_current = p;
1071 s->pool_cur = p->data + size;
1072 s->pool_end = p->data + p->size;
1073 return p->data;
1074}
1075
1076void tcg_pool_reset(TCGContext *s)
1077{
1078 TCGPool *p, *t;
1079 for (p = s->pool_first_large; p; p = t) {
1080 t = p->next;
1081 g_free(p);
1082 }
1083 s->pool_first_large = NULL;
1084 s->pool_cur = s->pool_end = NULL;
1085 s->pool_current = NULL;
1086}
1087
1088typedef struct TCGHelperInfo {
1089 void *func;
1090 const char *name;
1091 unsigned flags;
1092 unsigned sizemask;
1093} TCGHelperInfo;
1094
1095#include "exec/helper-proto.h"
1096
1097static const TCGHelperInfo all_helpers[] = {
1098#include "exec/helper-tcg.h"
1099};
1100static GHashTable *helper_table;
1101
1102static int indirect_reg_alloc_order[ARRAY_SIZE(tcg_target_reg_alloc_order)];
1103static void process_op_defs(TCGContext *s);
1104static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type,
1105 TCGReg reg, const char *name);
1106
1107void tcg_context_init(TCGContext *s)
1108{
1109 int op, total_args, n, i;
1110 TCGOpDef *def;
1111 TCGArgConstraint *args_ct;
1112 TCGTemp *ts;
1113
1114 memset(s, 0, sizeof(*s));
1115 s->nb_globals = 0;
1116
1117
1118
1119 total_args = 0;
1120 for(op = 0; op < NB_OPS; op++) {
1121 def = &tcg_op_defs[op];
1122 n = def->nb_iargs + def->nb_oargs;
1123 total_args += n;
1124 }
1125
1126 args_ct = g_new0(TCGArgConstraint, total_args);
1127
1128 for(op = 0; op < NB_OPS; op++) {
1129 def = &tcg_op_defs[op];
1130 def->args_ct = args_ct;
1131 n = def->nb_iargs + def->nb_oargs;
1132 args_ct += n;
1133 }
1134
1135
1136
1137 helper_table = g_hash_table_new(NULL, NULL);
1138
1139 for (i = 0; i < ARRAY_SIZE(all_helpers); ++i) {
1140 g_hash_table_insert(helper_table, (gpointer)all_helpers[i].func,
1141 (gpointer)&all_helpers[i]);
1142 }
1143
1144 tcg_target_init(s);
1145 process_op_defs(s);
1146
1147
1148
1149 for (n = 0; n < ARRAY_SIZE(tcg_target_reg_alloc_order); ++n) {
1150 int r = tcg_target_reg_alloc_order[n];
1151 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, r)) {
1152 break;
1153 }
1154 }
1155 for (i = 0; i < n; ++i) {
1156 indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[n - 1 - i];
1157 }
1158 for (; i < ARRAY_SIZE(tcg_target_reg_alloc_order); ++i) {
1159 indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[i];
1160 }
1161
1162 alloc_tcg_plugin_context(s);
1163
1164 tcg_ctx = s;
1165
1166
1167
1168
1169
1170
1171#ifdef CONFIG_USER_ONLY
1172 tcg_ctxs = &tcg_ctx;
1173 n_tcg_ctxs = 1;
1174#else
1175 MachineState *ms = MACHINE(qdev_get_machine());
1176 unsigned int max_cpus = ms->smp.max_cpus;
1177 tcg_ctxs = g_new(TCGContext *, max_cpus);
1178#endif
1179
1180 tcg_debug_assert(!tcg_regset_test_reg(s->reserved_regs, TCG_AREG0));
1181 ts = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, TCG_AREG0, "env");
1182 cpu_env = temp_tcgv_ptr(ts);
1183}
1184
1185
1186
1187
1188
1189TranslationBlock *tcg_tb_alloc(TCGContext *s)
1190{
1191 uintptr_t align = qemu_icache_linesize;
1192 TranslationBlock *tb;
1193 void *next;
1194
1195 retry:
1196 tb = (void *)ROUND_UP((uintptr_t)s->code_gen_ptr, align);
1197 next = (void *)ROUND_UP((uintptr_t)(tb + 1), align);
1198
1199 if (unlikely(next > s->code_gen_highwater)) {
1200 if (tcg_region_alloc(s)) {
1201 return NULL;
1202 }
1203 goto retry;
1204 }
1205 qatomic_set(&s->code_gen_ptr, next);
1206 s->data_gen_ptr = NULL;
1207 return tb;
1208}
1209
1210void tcg_prologue_init(TCGContext *s)
1211{
1212 size_t prologue_size, total_size;
1213 void *buf0, *buf1;
1214
1215
1216 buf0 = s->code_gen_buffer;
1217 total_size = s->code_gen_buffer_size;
1218 s->code_ptr = buf0;
1219 s->code_buf = buf0;
1220 s->data_gen_ptr = NULL;
1221
1222
1223
1224
1225
1226 region.start = buf0;
1227 region.end = buf0 + total_size;
1228
1229#ifndef CONFIG_TCG_INTERPRETER
1230 tcg_qemu_tb_exec = (tcg_prologue_fn *)tcg_splitwx_to_rx(buf0);
1231#endif
1232
1233
1234
1235
1236 s->code_gen_highwater = s->code_gen_buffer + (total_size - TCG_HIGHWATER);
1237
1238#ifdef TCG_TARGET_NEED_POOL_LABELS
1239 s->pool_labels = NULL;
1240#endif
1241
1242 qemu_thread_jit_write();
1243
1244 tcg_target_qemu_prologue(s);
1245
1246#ifdef TCG_TARGET_NEED_POOL_LABELS
1247
1248 {
1249 int result = tcg_out_pool_finalize(s);
1250 tcg_debug_assert(result == 0);
1251 }
1252#endif
1253
1254 buf1 = s->code_ptr;
1255#ifndef CONFIG_TCG_INTERPRETER
1256 flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(buf0), (uintptr_t)buf0,
1257 tcg_ptr_byte_diff(buf1, buf0));
1258#endif
1259
1260
1261 prologue_size = tcg_current_code_size(s);
1262 s->code_gen_ptr = buf1;
1263 s->code_gen_buffer = buf1;
1264 s->code_buf = buf1;
1265 total_size -= prologue_size;
1266 s->code_gen_buffer_size = total_size;
1267
1268 tcg_register_jit(tcg_splitwx_to_rx(s->code_gen_buffer), total_size);
1269
1270#ifdef DEBUG_DISAS
1271 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) {
1272 FILE *logfile = qemu_log_lock();
1273 qemu_log("PROLOGUE: [size=%zu]\n", prologue_size);
1274 if (s->data_gen_ptr) {
1275 size_t code_size = s->data_gen_ptr - buf0;
1276 size_t data_size = prologue_size - code_size;
1277 size_t i;
1278
1279 log_disas(buf0, code_size);
1280
1281 for (i = 0; i < data_size; i += sizeof(tcg_target_ulong)) {
1282 if (sizeof(tcg_target_ulong) == 8) {
1283 qemu_log("0x%08" PRIxPTR ": .quad 0x%016" PRIx64 "\n",
1284 (uintptr_t)s->data_gen_ptr + i,
1285 *(uint64_t *)(s->data_gen_ptr + i));
1286 } else {
1287 qemu_log("0x%08" PRIxPTR ": .long 0x%08x\n",
1288 (uintptr_t)s->data_gen_ptr + i,
1289 *(uint32_t *)(s->data_gen_ptr + i));
1290 }
1291 }
1292 } else {
1293 log_disas(buf0, prologue_size);
1294 }
1295 qemu_log("\n");
1296 qemu_log_flush();
1297 qemu_log_unlock(logfile);
1298 }
1299#endif
1300
1301
1302 if (TCG_TARGET_HAS_goto_ptr) {
1303 tcg_debug_assert(tcg_code_gen_epilogue != NULL);
1304 }
1305}
1306
1307void tcg_func_start(TCGContext *s)
1308{
1309 tcg_pool_reset(s);
1310 s->nb_temps = s->nb_globals;
1311
1312
1313 memset(s->free_temps, 0, sizeof(s->free_temps));
1314
1315
1316 for (int i = 0; i < TCG_TYPE_COUNT; ++i) {
1317 if (s->const_table[i]) {
1318 g_hash_table_remove_all(s->const_table[i]);
1319 }
1320 }
1321
1322 s->nb_ops = 0;
1323 s->nb_labels = 0;
1324 s->current_frame_offset = s->frame_start;
1325
1326#ifdef CONFIG_DEBUG_TCG
1327 s->goto_tb_issue_mask = 0;
1328#endif
1329
1330 QTAILQ_INIT(&s->ops);
1331 QTAILQ_INIT(&s->free_ops);
1332 QSIMPLEQ_INIT(&s->labels);
1333}
1334
1335static TCGTemp *tcg_temp_alloc(TCGContext *s)
1336{
1337 int n = s->nb_temps++;
1338
1339 if (n >= TCG_MAX_TEMPS) {
1340 tcg_raise_tb_overflow(s);
1341 }
1342 return memset(&s->temps[n], 0, sizeof(TCGTemp));
1343}
1344
1345static TCGTemp *tcg_global_alloc(TCGContext *s)
1346{
1347 TCGTemp *ts;
1348
1349 tcg_debug_assert(s->nb_globals == s->nb_temps);
1350 tcg_debug_assert(s->nb_globals < TCG_MAX_TEMPS);
1351 s->nb_globals++;
1352 ts = tcg_temp_alloc(s);
1353 ts->kind = TEMP_GLOBAL;
1354
1355 return ts;
1356}
1357
1358static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type,
1359 TCGReg reg, const char *name)
1360{
1361 TCGTemp *ts;
1362
1363 if (TCG_TARGET_REG_BITS == 32 && type != TCG_TYPE_I32) {
1364 tcg_abort();
1365 }
1366
1367 ts = tcg_global_alloc(s);
1368 ts->base_type = type;
1369 ts->type = type;
1370 ts->kind = TEMP_FIXED;
1371 ts->reg = reg;
1372 ts->name = name;
1373 tcg_regset_set_reg(s->reserved_regs, reg);
1374
1375 return ts;
1376}
1377
1378void tcg_set_frame(TCGContext *s, TCGReg reg, intptr_t start, intptr_t size)
1379{
1380 s->frame_start = start;
1381 s->frame_end = start + size;
1382 s->frame_temp
1383 = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, reg, "_frame");
1384}
1385
1386TCGTemp *tcg_global_mem_new_internal(TCGType type, TCGv_ptr base,
1387 intptr_t offset, const char *name)
1388{
1389 TCGContext *s = tcg_ctx;
1390 TCGTemp *base_ts = tcgv_ptr_temp(base);
1391 TCGTemp *ts = tcg_global_alloc(s);
1392 int indirect_reg = 0, bigendian = 0;
1393#ifdef HOST_WORDS_BIGENDIAN
1394 bigendian = 1;
1395#endif
1396
1397 switch (base_ts->kind) {
1398 case TEMP_FIXED:
1399 break;
1400 case TEMP_GLOBAL:
1401
1402 tcg_debug_assert(!base_ts->indirect_reg);
1403 base_ts->indirect_base = 1;
1404 s->nb_indirects += (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64
1405 ? 2 : 1);
1406 indirect_reg = 1;
1407 break;
1408 default:
1409 g_assert_not_reached();
1410 }
1411
1412 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) {
1413 TCGTemp *ts2 = tcg_global_alloc(s);
1414 char buf[64];
1415
1416 ts->base_type = TCG_TYPE_I64;
1417 ts->type = TCG_TYPE_I32;
1418 ts->indirect_reg = indirect_reg;
1419 ts->mem_allocated = 1;
1420 ts->mem_base = base_ts;
1421 ts->mem_offset = offset + bigendian * 4;
1422 pstrcpy(buf, sizeof(buf), name);
1423 pstrcat(buf, sizeof(buf), "_0");
1424 ts->name = strdup(buf);
1425
1426 tcg_debug_assert(ts2 == ts + 1);
1427 ts2->base_type = TCG_TYPE_I64;
1428 ts2->type = TCG_TYPE_I32;
1429 ts2->indirect_reg = indirect_reg;
1430 ts2->mem_allocated = 1;
1431 ts2->mem_base = base_ts;
1432 ts2->mem_offset = offset + (1 - bigendian) * 4;
1433 pstrcpy(buf, sizeof(buf), name);
1434 pstrcat(buf, sizeof(buf), "_1");
1435 ts2->name = strdup(buf);
1436 } else {
1437 ts->base_type = type;
1438 ts->type = type;
1439 ts->indirect_reg = indirect_reg;
1440 ts->mem_allocated = 1;
1441 ts->mem_base = base_ts;
1442 ts->mem_offset = offset;
1443 ts->name = name;
1444 }
1445 return ts;
1446}
1447
1448TCGTemp *tcg_temp_new_internal(TCGType type, bool temp_local)
1449{
1450 TCGContext *s = tcg_ctx;
1451 TCGTempKind kind = temp_local ? TEMP_LOCAL : TEMP_NORMAL;
1452 TCGTemp *ts;
1453 int idx, k;
1454
1455 k = type + (temp_local ? TCG_TYPE_COUNT : 0);
1456 idx = find_first_bit(s->free_temps[k].l, TCG_MAX_TEMPS);
1457 if (idx < TCG_MAX_TEMPS) {
1458
1459 clear_bit(idx, s->free_temps[k].l);
1460
1461 ts = &s->temps[idx];
1462 ts->temp_allocated = 1;
1463 tcg_debug_assert(ts->base_type == type);
1464 tcg_debug_assert(ts->kind == kind);
1465 } else {
1466 ts = tcg_temp_alloc(s);
1467 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) {
1468 TCGTemp *ts2 = tcg_temp_alloc(s);
1469
1470 ts->base_type = type;
1471 ts->type = TCG_TYPE_I32;
1472 ts->temp_allocated = 1;
1473 ts->kind = kind;
1474
1475 tcg_debug_assert(ts2 == ts + 1);
1476 ts2->base_type = TCG_TYPE_I64;
1477 ts2->type = TCG_TYPE_I32;
1478 ts2->temp_allocated = 1;
1479 ts2->kind = kind;
1480 } else {
1481 ts->base_type = type;
1482 ts->type = type;
1483 ts->temp_allocated = 1;
1484 ts->kind = kind;
1485 }
1486 }
1487
1488#if defined(CONFIG_DEBUG_TCG)
1489 s->temps_in_use++;
1490#endif
1491 return ts;
1492}
1493
1494TCGv_vec tcg_temp_new_vec(TCGType type)
1495{
1496 TCGTemp *t;
1497
1498#ifdef CONFIG_DEBUG_TCG
1499 switch (type) {
1500 case TCG_TYPE_V64:
1501 assert(TCG_TARGET_HAS_v64);
1502 break;
1503 case TCG_TYPE_V128:
1504 assert(TCG_TARGET_HAS_v128);
1505 break;
1506 case TCG_TYPE_V256:
1507 assert(TCG_TARGET_HAS_v256);
1508 break;
1509 default:
1510 g_assert_not_reached();
1511 }
1512#endif
1513
1514 t = tcg_temp_new_internal(type, 0);
1515 return temp_tcgv_vec(t);
1516}
1517
1518
1519TCGv_vec tcg_temp_new_vec_matching(TCGv_vec match)
1520{
1521 TCGTemp *t = tcgv_vec_temp(match);
1522
1523 tcg_debug_assert(t->temp_allocated != 0);
1524
1525 t = tcg_temp_new_internal(t->base_type, 0);
1526 return temp_tcgv_vec(t);
1527}
1528
1529void tcg_temp_free_internal(TCGTemp *ts)
1530{
1531 TCGContext *s = tcg_ctx;
1532 int k, idx;
1533
1534
1535 if (ts->kind == TEMP_CONST) {
1536 return;
1537 }
1538
1539#if defined(CONFIG_DEBUG_TCG)
1540 s->temps_in_use--;
1541 if (s->temps_in_use < 0) {
1542 fprintf(stderr, "More temporaries freed than allocated!\n");
1543 }
1544#endif
1545
1546 tcg_debug_assert(ts->kind < TEMP_GLOBAL);
1547 tcg_debug_assert(ts->temp_allocated != 0);
1548 ts->temp_allocated = 0;
1549
1550 idx = temp_idx(ts);
1551 k = ts->base_type + (ts->kind == TEMP_NORMAL ? 0 : TCG_TYPE_COUNT);
1552 set_bit(idx, s->free_temps[k].l);
1553}
1554
1555TCGTemp *tcg_constant_internal(TCGType type, int64_t val)
1556{
1557 TCGContext *s = tcg_ctx;
1558 GHashTable *h = s->const_table[type];
1559 TCGTemp *ts;
1560
1561 if (h == NULL) {
1562 h = g_hash_table_new(g_int64_hash, g_int64_equal);
1563 s->const_table[type] = h;
1564 }
1565
1566 ts = g_hash_table_lookup(h, &val);
1567 if (ts == NULL) {
1568 ts = tcg_temp_alloc(s);
1569
1570 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) {
1571 TCGTemp *ts2 = tcg_temp_alloc(s);
1572
1573 ts->base_type = TCG_TYPE_I64;
1574 ts->type = TCG_TYPE_I32;
1575 ts->kind = TEMP_CONST;
1576 ts->temp_allocated = 1;
1577
1578
1579
1580
1581
1582 ts->val = val;
1583
1584 tcg_debug_assert(ts2 == ts + 1);
1585 ts2->base_type = TCG_TYPE_I64;
1586 ts2->type = TCG_TYPE_I32;
1587 ts2->kind = TEMP_CONST;
1588 ts2->temp_allocated = 1;
1589 ts2->val = val >> 32;
1590 } else {
1591 ts->base_type = type;
1592 ts->type = type;
1593 ts->kind = TEMP_CONST;
1594 ts->temp_allocated = 1;
1595 ts->val = val;
1596 }
1597 g_hash_table_insert(h, &ts->val, ts);
1598 }
1599
1600 return ts;
1601}
1602
1603TCGv_vec tcg_constant_vec(TCGType type, unsigned vece, int64_t val)
1604{
1605 val = dup_const(vece, val);
1606 return temp_tcgv_vec(tcg_constant_internal(type, val));
1607}
1608
1609TCGv_vec tcg_constant_vec_matching(TCGv_vec match, unsigned vece, int64_t val)
1610{
1611 TCGTemp *t = tcgv_vec_temp(match);
1612
1613 tcg_debug_assert(t->temp_allocated != 0);
1614 return tcg_constant_vec(t->base_type, vece, val);
1615}
1616
1617TCGv_i32 tcg_const_i32(int32_t val)
1618{
1619 TCGv_i32 t0;
1620 t0 = tcg_temp_new_i32();
1621 tcg_gen_movi_i32(t0, val);
1622 return t0;
1623}
1624
1625TCGv_i64 tcg_const_i64(int64_t val)
1626{
1627 TCGv_i64 t0;
1628 t0 = tcg_temp_new_i64();
1629 tcg_gen_movi_i64(t0, val);
1630 return t0;
1631}
1632
1633TCGv_i32 tcg_const_local_i32(int32_t val)
1634{
1635 TCGv_i32 t0;
1636 t0 = tcg_temp_local_new_i32();
1637 tcg_gen_movi_i32(t0, val);
1638 return t0;
1639}
1640
1641TCGv_i64 tcg_const_local_i64(int64_t val)
1642{
1643 TCGv_i64 t0;
1644 t0 = tcg_temp_local_new_i64();
1645 tcg_gen_movi_i64(t0, val);
1646 return t0;
1647}
1648
1649#if defined(CONFIG_DEBUG_TCG)
1650void tcg_clear_temp_count(void)
1651{
1652 TCGContext *s = tcg_ctx;
1653 s->temps_in_use = 0;
1654}
1655
1656int tcg_check_temp_count(void)
1657{
1658 TCGContext *s = tcg_ctx;
1659 if (s->temps_in_use) {
1660
1661
1662
1663 s->temps_in_use = 0;
1664 return 1;
1665 }
1666 return 0;
1667}
1668#endif
1669
1670
1671
1672bool tcg_op_supported(TCGOpcode op)
1673{
1674 const bool have_vec
1675 = TCG_TARGET_HAS_v64 | TCG_TARGET_HAS_v128 | TCG_TARGET_HAS_v256;
1676
1677 switch (op) {
1678 case INDEX_op_discard:
1679 case INDEX_op_set_label:
1680 case INDEX_op_call:
1681 case INDEX_op_br:
1682 case INDEX_op_mb:
1683 case INDEX_op_insn_start:
1684 case INDEX_op_exit_tb:
1685 case INDEX_op_goto_tb:
1686 case INDEX_op_qemu_ld_i32:
1687 case INDEX_op_qemu_st_i32:
1688 case INDEX_op_qemu_ld_i64:
1689 case INDEX_op_qemu_st_i64:
1690 return true;
1691
1692 case INDEX_op_qemu_st8_i32:
1693 return TCG_TARGET_HAS_qemu_st8_i32;
1694
1695 case INDEX_op_goto_ptr:
1696 return TCG_TARGET_HAS_goto_ptr;
1697
1698 case INDEX_op_mov_i32:
1699 case INDEX_op_setcond_i32:
1700 case INDEX_op_brcond_i32:
1701 case INDEX_op_ld8u_i32:
1702 case INDEX_op_ld8s_i32:
1703 case INDEX_op_ld16u_i32:
1704 case INDEX_op_ld16s_i32:
1705 case INDEX_op_ld_i32:
1706 case INDEX_op_st8_i32:
1707 case INDEX_op_st16_i32:
1708 case INDEX_op_st_i32:
1709 case INDEX_op_add_i32:
1710 case INDEX_op_sub_i32:
1711 case INDEX_op_mul_i32:
1712 case INDEX_op_and_i32:
1713 case INDEX_op_or_i32:
1714 case INDEX_op_xor_i32:
1715 case INDEX_op_shl_i32:
1716 case INDEX_op_shr_i32:
1717 case INDEX_op_sar_i32:
1718 return true;
1719
1720 case INDEX_op_movcond_i32:
1721 return TCG_TARGET_HAS_movcond_i32;
1722 case INDEX_op_div_i32:
1723 case INDEX_op_divu_i32:
1724 return TCG_TARGET_HAS_div_i32;
1725 case INDEX_op_rem_i32:
1726 case INDEX_op_remu_i32:
1727 return TCG_TARGET_HAS_rem_i32;
1728 case INDEX_op_div2_i32:
1729 case INDEX_op_divu2_i32:
1730 return TCG_TARGET_HAS_div2_i32;
1731 case INDEX_op_rotl_i32:
1732 case INDEX_op_rotr_i32:
1733 return TCG_TARGET_HAS_rot_i32;
1734 case INDEX_op_deposit_i32:
1735 return TCG_TARGET_HAS_deposit_i32;
1736 case INDEX_op_extract_i32:
1737 return TCG_TARGET_HAS_extract_i32;
1738 case INDEX_op_sextract_i32:
1739 return TCG_TARGET_HAS_sextract_i32;
1740 case INDEX_op_extract2_i32:
1741 return TCG_TARGET_HAS_extract2_i32;
1742 case INDEX_op_add2_i32:
1743 return TCG_TARGET_HAS_add2_i32;
1744 case INDEX_op_sub2_i32:
1745 return TCG_TARGET_HAS_sub2_i32;
1746 case INDEX_op_mulu2_i32:
1747 return TCG_TARGET_HAS_mulu2_i32;
1748 case INDEX_op_muls2_i32:
1749 return TCG_TARGET_HAS_muls2_i32;
1750 case INDEX_op_muluh_i32:
1751 return TCG_TARGET_HAS_muluh_i32;
1752 case INDEX_op_mulsh_i32:
1753 return TCG_TARGET_HAS_mulsh_i32;
1754 case INDEX_op_ext8s_i32:
1755 return TCG_TARGET_HAS_ext8s_i32;
1756 case INDEX_op_ext16s_i32:
1757 return TCG_TARGET_HAS_ext16s_i32;
1758 case INDEX_op_ext8u_i32:
1759 return TCG_TARGET_HAS_ext8u_i32;
1760 case INDEX_op_ext16u_i32:
1761 return TCG_TARGET_HAS_ext16u_i32;
1762 case INDEX_op_bswap16_i32:
1763 return TCG_TARGET_HAS_bswap16_i32;
1764 case INDEX_op_bswap32_i32:
1765 return TCG_TARGET_HAS_bswap32_i32;
1766 case INDEX_op_not_i32:
1767 return TCG_TARGET_HAS_not_i32;
1768 case INDEX_op_neg_i32:
1769 return TCG_TARGET_HAS_neg_i32;
1770 case INDEX_op_andc_i32:
1771 return TCG_TARGET_HAS_andc_i32;
1772 case INDEX_op_orc_i32:
1773 return TCG_TARGET_HAS_orc_i32;
1774 case INDEX_op_eqv_i32:
1775 return TCG_TARGET_HAS_eqv_i32;
1776 case INDEX_op_nand_i32:
1777 return TCG_TARGET_HAS_nand_i32;
1778 case INDEX_op_nor_i32:
1779 return TCG_TARGET_HAS_nor_i32;
1780 case INDEX_op_clz_i32:
1781 return TCG_TARGET_HAS_clz_i32;
1782 case INDEX_op_ctz_i32:
1783 return TCG_TARGET_HAS_ctz_i32;
1784 case INDEX_op_ctpop_i32:
1785 return TCG_TARGET_HAS_ctpop_i32;
1786
1787 case INDEX_op_brcond2_i32:
1788 case INDEX_op_setcond2_i32:
1789 return TCG_TARGET_REG_BITS == 32;
1790
1791 case INDEX_op_mov_i64:
1792 case INDEX_op_setcond_i64:
1793 case INDEX_op_brcond_i64:
1794 case INDEX_op_ld8u_i64:
1795 case INDEX_op_ld8s_i64:
1796 case INDEX_op_ld16u_i64:
1797 case INDEX_op_ld16s_i64:
1798 case INDEX_op_ld32u_i64:
1799 case INDEX_op_ld32s_i64:
1800 case INDEX_op_ld_i64:
1801 case INDEX_op_st8_i64:
1802 case INDEX_op_st16_i64:
1803 case INDEX_op_st32_i64:
1804 case INDEX_op_st_i64:
1805 case INDEX_op_add_i64:
1806 case INDEX_op_sub_i64:
1807 case INDEX_op_mul_i64:
1808 case INDEX_op_and_i64:
1809 case INDEX_op_or_i64:
1810 case INDEX_op_xor_i64:
1811 case INDEX_op_shl_i64:
1812 case INDEX_op_shr_i64:
1813 case INDEX_op_sar_i64:
1814 case INDEX_op_ext_i32_i64:
1815 case INDEX_op_extu_i32_i64:
1816 return TCG_TARGET_REG_BITS == 64;
1817
1818 case INDEX_op_movcond_i64:
1819 return TCG_TARGET_HAS_movcond_i64;
1820 case INDEX_op_div_i64:
1821 case INDEX_op_divu_i64:
1822 return TCG_TARGET_HAS_div_i64;
1823 case INDEX_op_rem_i64:
1824 case INDEX_op_remu_i64:
1825 return TCG_TARGET_HAS_rem_i64;
1826 case INDEX_op_div2_i64:
1827 case INDEX_op_divu2_i64:
1828 return TCG_TARGET_HAS_div2_i64;
1829 case INDEX_op_rotl_i64:
1830 case INDEX_op_rotr_i64:
1831 return TCG_TARGET_HAS_rot_i64;
1832 case INDEX_op_deposit_i64:
1833 return TCG_TARGET_HAS_deposit_i64;
1834 case INDEX_op_extract_i64:
1835 return TCG_TARGET_HAS_extract_i64;
1836 case INDEX_op_sextract_i64:
1837 return TCG_TARGET_HAS_sextract_i64;
1838 case INDEX_op_extract2_i64:
1839 return TCG_TARGET_HAS_extract2_i64;
1840 case INDEX_op_extrl_i64_i32:
1841 return TCG_TARGET_HAS_extrl_i64_i32;
1842 case INDEX_op_extrh_i64_i32:
1843 return TCG_TARGET_HAS_extrh_i64_i32;
1844 case INDEX_op_ext8s_i64:
1845 return TCG_TARGET_HAS_ext8s_i64;
1846 case INDEX_op_ext16s_i64:
1847 return TCG_TARGET_HAS_ext16s_i64;
1848 case INDEX_op_ext32s_i64:
1849 return TCG_TARGET_HAS_ext32s_i64;
1850 case INDEX_op_ext8u_i64:
1851 return TCG_TARGET_HAS_ext8u_i64;
1852 case INDEX_op_ext16u_i64:
1853 return TCG_TARGET_HAS_ext16u_i64;
1854 case INDEX_op_ext32u_i64:
1855 return TCG_TARGET_HAS_ext32u_i64;
1856 case INDEX_op_bswap16_i64:
1857 return TCG_TARGET_HAS_bswap16_i64;
1858 case INDEX_op_bswap32_i64:
1859 return TCG_TARGET_HAS_bswap32_i64;
1860 case INDEX_op_bswap64_i64:
1861 return TCG_TARGET_HAS_bswap64_i64;
1862 case INDEX_op_not_i64:
1863 return TCG_TARGET_HAS_not_i64;
1864 case INDEX_op_neg_i64:
1865 return TCG_TARGET_HAS_neg_i64;
1866 case INDEX_op_andc_i64:
1867 return TCG_TARGET_HAS_andc_i64;
1868 case INDEX_op_orc_i64:
1869 return TCG_TARGET_HAS_orc_i64;
1870 case INDEX_op_eqv_i64:
1871 return TCG_TARGET_HAS_eqv_i64;
1872 case INDEX_op_nand_i64:
1873 return TCG_TARGET_HAS_nand_i64;
1874 case INDEX_op_nor_i64:
1875 return TCG_TARGET_HAS_nor_i64;
1876 case INDEX_op_clz_i64:
1877 return TCG_TARGET_HAS_clz_i64;
1878 case INDEX_op_ctz_i64:
1879 return TCG_TARGET_HAS_ctz_i64;
1880 case INDEX_op_ctpop_i64:
1881 return TCG_TARGET_HAS_ctpop_i64;
1882 case INDEX_op_add2_i64:
1883 return TCG_TARGET_HAS_add2_i64;
1884 case INDEX_op_sub2_i64:
1885 return TCG_TARGET_HAS_sub2_i64;
1886 case INDEX_op_mulu2_i64:
1887 return TCG_TARGET_HAS_mulu2_i64;
1888 case INDEX_op_muls2_i64:
1889 return TCG_TARGET_HAS_muls2_i64;
1890 case INDEX_op_muluh_i64:
1891 return TCG_TARGET_HAS_muluh_i64;
1892 case INDEX_op_mulsh_i64:
1893 return TCG_TARGET_HAS_mulsh_i64;
1894
1895 case INDEX_op_mov_vec:
1896 case INDEX_op_dup_vec:
1897 case INDEX_op_dupm_vec:
1898 case INDEX_op_ld_vec:
1899 case INDEX_op_st_vec:
1900 case INDEX_op_add_vec:
1901 case INDEX_op_sub_vec:
1902 case INDEX_op_and_vec:
1903 case INDEX_op_or_vec:
1904 case INDEX_op_xor_vec:
1905 case INDEX_op_cmp_vec:
1906 return have_vec;
1907 case INDEX_op_dup2_vec:
1908 return have_vec && TCG_TARGET_REG_BITS == 32;
1909 case INDEX_op_not_vec:
1910 return have_vec && TCG_TARGET_HAS_not_vec;
1911 case INDEX_op_neg_vec:
1912 return have_vec && TCG_TARGET_HAS_neg_vec;
1913 case INDEX_op_abs_vec:
1914 return have_vec && TCG_TARGET_HAS_abs_vec;
1915 case INDEX_op_andc_vec:
1916 return have_vec && TCG_TARGET_HAS_andc_vec;
1917 case INDEX_op_orc_vec:
1918 return have_vec && TCG_TARGET_HAS_orc_vec;
1919 case INDEX_op_mul_vec:
1920 return have_vec && TCG_TARGET_HAS_mul_vec;
1921 case INDEX_op_shli_vec:
1922 case INDEX_op_shri_vec:
1923 case INDEX_op_sari_vec:
1924 return have_vec && TCG_TARGET_HAS_shi_vec;
1925 case INDEX_op_shls_vec:
1926 case INDEX_op_shrs_vec:
1927 case INDEX_op_sars_vec:
1928 return have_vec && TCG_TARGET_HAS_shs_vec;
1929 case INDEX_op_shlv_vec:
1930 case INDEX_op_shrv_vec:
1931 case INDEX_op_sarv_vec:
1932 return have_vec && TCG_TARGET_HAS_shv_vec;
1933 case INDEX_op_rotli_vec:
1934 return have_vec && TCG_TARGET_HAS_roti_vec;
1935 case INDEX_op_rotls_vec:
1936 return have_vec && TCG_TARGET_HAS_rots_vec;
1937 case INDEX_op_rotlv_vec:
1938 case INDEX_op_rotrv_vec:
1939 return have_vec && TCG_TARGET_HAS_rotv_vec;
1940 case INDEX_op_ssadd_vec:
1941 case INDEX_op_usadd_vec:
1942 case INDEX_op_sssub_vec:
1943 case INDEX_op_ussub_vec:
1944 return have_vec && TCG_TARGET_HAS_sat_vec;
1945 case INDEX_op_smin_vec:
1946 case INDEX_op_umin_vec:
1947 case INDEX_op_smax_vec:
1948 case INDEX_op_umax_vec:
1949 return have_vec && TCG_TARGET_HAS_minmax_vec;
1950 case INDEX_op_bitsel_vec:
1951 return have_vec && TCG_TARGET_HAS_bitsel_vec;
1952 case INDEX_op_cmpsel_vec:
1953 return have_vec && TCG_TARGET_HAS_cmpsel_vec;
1954
1955 default:
1956 tcg_debug_assert(op > INDEX_op_last_generic && op < NB_OPS);
1957 return true;
1958 }
1959}
1960
1961
1962
1963
1964void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args)
1965{
1966 int i, real_args, nb_rets, pi;
1967 unsigned sizemask, flags;
1968 TCGHelperInfo *info;
1969 TCGOp *op;
1970
1971 info = g_hash_table_lookup(helper_table, (gpointer)func);
1972 flags = info->flags;
1973 sizemask = info->sizemask;
1974
1975#ifdef CONFIG_PLUGIN
1976
1977 if (tcg_ctx->plugin_insn && unlikely(strncmp(info->name, "plugin_", 7))) {
1978 tcg_ctx->plugin_insn->calls_helpers = true;
1979 }
1980#endif
1981
1982#if defined(__sparc__) && !defined(__arch64__) \
1983 && !defined(CONFIG_TCG_INTERPRETER)
1984
1985
1986 int orig_sizemask = sizemask;
1987 int orig_nargs = nargs;
1988 TCGv_i64 retl, reth;
1989 TCGTemp *split_args[MAX_OPC_PARAM];
1990
1991 retl = NULL;
1992 reth = NULL;
1993 if (sizemask != 0) {
1994 for (i = real_args = 0; i < nargs; ++i) {
1995 int is_64bit = sizemask & (1 << (i+1)*2);
1996 if (is_64bit) {
1997 TCGv_i64 orig = temp_tcgv_i64(args[i]);
1998 TCGv_i32 h = tcg_temp_new_i32();
1999 TCGv_i32 l = tcg_temp_new_i32();
2000 tcg_gen_extr_i64_i32(l, h, orig);
2001 split_args[real_args++] = tcgv_i32_temp(h);
2002 split_args[real_args++] = tcgv_i32_temp(l);
2003 } else {
2004 split_args[real_args++] = args[i];
2005 }
2006 }
2007 nargs = real_args;
2008 args = split_args;
2009 sizemask = 0;
2010 }
2011#elif defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64
2012 for (i = 0; i < nargs; ++i) {
2013 int is_64bit = sizemask & (1 << (i+1)*2);
2014 int is_signed = sizemask & (2 << (i+1)*2);
2015 if (!is_64bit) {
2016 TCGv_i64 temp = tcg_temp_new_i64();
2017 TCGv_i64 orig = temp_tcgv_i64(args[i]);
2018 if (is_signed) {
2019 tcg_gen_ext32s_i64(temp, orig);
2020 } else {
2021 tcg_gen_ext32u_i64(temp, orig);
2022 }
2023 args[i] = tcgv_i64_temp(temp);
2024 }
2025 }
2026#endif
2027
2028 op = tcg_emit_op(INDEX_op_call);
2029
2030 pi = 0;
2031 if (ret != NULL) {
2032#if defined(__sparc__) && !defined(__arch64__) \
2033 && !defined(CONFIG_TCG_INTERPRETER)
2034 if (orig_sizemask & 1) {
2035
2036
2037
2038 retl = tcg_temp_new_i64();
2039 reth = tcg_temp_new_i64();
2040 op->args[pi++] = tcgv_i64_arg(reth);
2041 op->args[pi++] = tcgv_i64_arg(retl);
2042 nb_rets = 2;
2043 } else {
2044 op->args[pi++] = temp_arg(ret);
2045 nb_rets = 1;
2046 }
2047#else
2048 if (TCG_TARGET_REG_BITS < 64 && (sizemask & 1)) {
2049#ifdef HOST_WORDS_BIGENDIAN
2050 op->args[pi++] = temp_arg(ret + 1);
2051 op->args[pi++] = temp_arg(ret);
2052#else
2053 op->args[pi++] = temp_arg(ret);
2054 op->args[pi++] = temp_arg(ret + 1);
2055#endif
2056 nb_rets = 2;
2057 } else {
2058 op->args[pi++] = temp_arg(ret);
2059 nb_rets = 1;
2060 }
2061#endif
2062 } else {
2063 nb_rets = 0;
2064 }
2065 TCGOP_CALLO(op) = nb_rets;
2066
2067 real_args = 0;
2068 for (i = 0; i < nargs; i++) {
2069 int is_64bit = sizemask & (1 << (i+1)*2);
2070 if (TCG_TARGET_REG_BITS < 64 && is_64bit) {
2071#ifdef TCG_TARGET_CALL_ALIGN_ARGS
2072
2073 if (real_args & 1) {
2074 op->args[pi++] = TCG_CALL_DUMMY_ARG;
2075 real_args++;
2076 }
2077#endif
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088#if defined(HOST_WORDS_BIGENDIAN) != defined(TCG_TARGET_STACK_GROWSUP)
2089 op->args[pi++] = temp_arg(args[i] + 1);
2090 op->args[pi++] = temp_arg(args[i]);
2091#else
2092 op->args[pi++] = temp_arg(args[i]);
2093 op->args[pi++] = temp_arg(args[i] + 1);
2094#endif
2095 real_args += 2;
2096 continue;
2097 }
2098
2099 op->args[pi++] = temp_arg(args[i]);
2100 real_args++;
2101 }
2102 op->args[pi++] = (uintptr_t)func;
2103 op->args[pi++] = flags;
2104 TCGOP_CALLI(op) = real_args;
2105
2106
2107 tcg_debug_assert(TCGOP_CALLI(op) == real_args);
2108 tcg_debug_assert(pi <= ARRAY_SIZE(op->args));
2109
2110#if defined(__sparc__) && !defined(__arch64__) \
2111 && !defined(CONFIG_TCG_INTERPRETER)
2112
2113 for (i = real_args = 0; i < orig_nargs; ++i) {
2114 int is_64bit = orig_sizemask & (1 << (i+1)*2);
2115 if (is_64bit) {
2116 tcg_temp_free_internal(args[real_args++]);
2117 tcg_temp_free_internal(args[real_args++]);
2118 } else {
2119 real_args++;
2120 }
2121 }
2122 if (orig_sizemask & 1) {
2123
2124
2125
2126 tcg_gen_concat32_i64(temp_tcgv_i64(ret), retl, reth);
2127 tcg_temp_free_i64(retl);
2128 tcg_temp_free_i64(reth);
2129 }
2130#elif defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64
2131 for (i = 0; i < nargs; ++i) {
2132 int is_64bit = sizemask & (1 << (i+1)*2);
2133 if (!is_64bit) {
2134 tcg_temp_free_internal(args[i]);
2135 }
2136 }
2137#endif
2138}
2139
2140static void tcg_reg_alloc_start(TCGContext *s)
2141{
2142 int i, n;
2143
2144 for (i = 0, n = s->nb_temps; i < n; i++) {
2145 TCGTemp *ts = &s->temps[i];
2146 TCGTempVal val = TEMP_VAL_MEM;
2147
2148 switch (ts->kind) {
2149 case TEMP_CONST:
2150 val = TEMP_VAL_CONST;
2151 break;
2152 case TEMP_FIXED:
2153 val = TEMP_VAL_REG;
2154 break;
2155 case TEMP_GLOBAL:
2156 break;
2157 case TEMP_NORMAL:
2158 val = TEMP_VAL_DEAD;
2159
2160 case TEMP_LOCAL:
2161 ts->mem_allocated = 0;
2162 break;
2163 default:
2164 g_assert_not_reached();
2165 }
2166 ts->val_type = val;
2167 }
2168
2169 memset(s->reg_to_temp, 0, sizeof(s->reg_to_temp));
2170}
2171
2172static char *tcg_get_arg_str_ptr(TCGContext *s, char *buf, int buf_size,
2173 TCGTemp *ts)
2174{
2175 int idx = temp_idx(ts);
2176
2177 switch (ts->kind) {
2178 case TEMP_FIXED:
2179 case TEMP_GLOBAL:
2180 pstrcpy(buf, buf_size, ts->name);
2181 break;
2182 case TEMP_LOCAL:
2183 snprintf(buf, buf_size, "loc%d", idx - s->nb_globals);
2184 break;
2185 case TEMP_NORMAL:
2186 snprintf(buf, buf_size, "tmp%d", idx - s->nb_globals);
2187 break;
2188 case TEMP_CONST:
2189 switch (ts->type) {
2190 case TCG_TYPE_I32:
2191 snprintf(buf, buf_size, "$0x%x", (int32_t)ts->val);
2192 break;
2193#if TCG_TARGET_REG_BITS > 32
2194 case TCG_TYPE_I64:
2195 snprintf(buf, buf_size, "$0x%" PRIx64, ts->val);
2196 break;
2197#endif
2198 case TCG_TYPE_V64:
2199 case TCG_TYPE_V128:
2200 case TCG_TYPE_V256:
2201 snprintf(buf, buf_size, "v%d$0x%" PRIx64,
2202 64 << (ts->type - TCG_TYPE_V64), ts->val);
2203 break;
2204 default:
2205 g_assert_not_reached();
2206 }
2207 break;
2208 }
2209 return buf;
2210}
2211
2212static char *tcg_get_arg_str(TCGContext *s, char *buf,
2213 int buf_size, TCGArg arg)
2214{
2215 return tcg_get_arg_str_ptr(s, buf, buf_size, arg_temp(arg));
2216}
2217
2218
2219static inline const char *tcg_find_helper(TCGContext *s, uintptr_t val)
2220{
2221 const char *ret = NULL;
2222 if (helper_table) {
2223 TCGHelperInfo *info = g_hash_table_lookup(helper_table, (gpointer)val);
2224 if (info) {
2225 ret = info->name;
2226 }
2227 }
2228 return ret;
2229}
2230
2231static const char * const cond_name[] =
2232{
2233 [TCG_COND_NEVER] = "never",
2234 [TCG_COND_ALWAYS] = "always",
2235 [TCG_COND_EQ] = "eq",
2236 [TCG_COND_NE] = "ne",
2237 [TCG_COND_LT] = "lt",
2238 [TCG_COND_GE] = "ge",
2239 [TCG_COND_LE] = "le",
2240 [TCG_COND_GT] = "gt",
2241 [TCG_COND_LTU] = "ltu",
2242 [TCG_COND_GEU] = "geu",
2243 [TCG_COND_LEU] = "leu",
2244 [TCG_COND_GTU] = "gtu"
2245};
2246
2247static const char * const ldst_name[] =
2248{
2249 [MO_UB] = "ub",
2250 [MO_SB] = "sb",
2251 [MO_LEUW] = "leuw",
2252 [MO_LESW] = "lesw",
2253 [MO_LEUL] = "leul",
2254 [MO_LESL] = "lesl",
2255 [MO_LEQ] = "leq",
2256 [MO_BEUW] = "beuw",
2257 [MO_BESW] = "besw",
2258 [MO_BEUL] = "beul",
2259 [MO_BESL] = "besl",
2260 [MO_BEQ] = "beq",
2261};
2262
2263static const char * const alignment_name[(MO_AMASK >> MO_ASHIFT) + 1] = {
2264#ifdef TARGET_ALIGNED_ONLY
2265 [MO_UNALN >> MO_ASHIFT] = "un+",
2266 [MO_ALIGN >> MO_ASHIFT] = "",
2267#else
2268 [MO_UNALN >> MO_ASHIFT] = "",
2269 [MO_ALIGN >> MO_ASHIFT] = "al+",
2270#endif
2271 [MO_ALIGN_2 >> MO_ASHIFT] = "al2+",
2272 [MO_ALIGN_4 >> MO_ASHIFT] = "al4+",
2273 [MO_ALIGN_8 >> MO_ASHIFT] = "al8+",
2274 [MO_ALIGN_16 >> MO_ASHIFT] = "al16+",
2275 [MO_ALIGN_32 >> MO_ASHIFT] = "al32+",
2276 [MO_ALIGN_64 >> MO_ASHIFT] = "al64+",
2277};
2278
2279static inline bool tcg_regset_single(TCGRegSet d)
2280{
2281 return (d & (d - 1)) == 0;
2282}
2283
2284static inline TCGReg tcg_regset_first(TCGRegSet d)
2285{
2286 if (TCG_TARGET_NB_REGS <= 32) {
2287 return ctz32(d);
2288 } else {
2289 return ctz64(d);
2290 }
2291}
2292
2293static void tcg_dump_ops(TCGContext *s, bool have_prefs)
2294{
2295 char buf[128];
2296 TCGOp *op;
2297
2298 QTAILQ_FOREACH(op, &s->ops, link) {
2299 int i, k, nb_oargs, nb_iargs, nb_cargs;
2300 const TCGOpDef *def;
2301 TCGOpcode c;
2302 int col = 0;
2303
2304 c = op->opc;
2305 def = &tcg_op_defs[c];
2306
2307 if (c == INDEX_op_insn_start) {
2308 nb_oargs = 0;
2309 col += qemu_log("\n ----");
2310
2311 for (i = 0; i < TARGET_INSN_START_WORDS; ++i) {
2312 target_ulong a;
2313#if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
2314 a = deposit64(op->args[i * 2], 32, 32, op->args[i * 2 + 1]);
2315#else
2316 a = op->args[i];
2317#endif
2318 col += qemu_log(" " TARGET_FMT_lx, a);
2319 }
2320 } else if (c == INDEX_op_call) {
2321
2322 nb_oargs = TCGOP_CALLO(op);
2323 nb_iargs = TCGOP_CALLI(op);
2324 nb_cargs = def->nb_cargs;
2325
2326
2327 col += qemu_log(" %s %s,$0x%" TCG_PRIlx ",$%d", def->name,
2328 tcg_find_helper(s, op->args[nb_oargs + nb_iargs]),
2329 op->args[nb_oargs + nb_iargs + 1], nb_oargs);
2330 for (i = 0; i < nb_oargs; i++) {
2331 col += qemu_log(",%s", tcg_get_arg_str(s, buf, sizeof(buf),
2332 op->args[i]));
2333 }
2334 for (i = 0; i < nb_iargs; i++) {
2335 TCGArg arg = op->args[nb_oargs + i];
2336 const char *t = "<dummy>";
2337 if (arg != TCG_CALL_DUMMY_ARG) {
2338 t = tcg_get_arg_str(s, buf, sizeof(buf), arg);
2339 }
2340 col += qemu_log(",%s", t);
2341 }
2342 } else {
2343 col += qemu_log(" %s ", def->name);
2344
2345 nb_oargs = def->nb_oargs;
2346 nb_iargs = def->nb_iargs;
2347 nb_cargs = def->nb_cargs;
2348
2349 if (def->flags & TCG_OPF_VECTOR) {
2350 col += qemu_log("v%d,e%d,", 64 << TCGOP_VECL(op),
2351 8 << TCGOP_VECE(op));
2352 }
2353
2354 k = 0;
2355 for (i = 0; i < nb_oargs; i++) {
2356 if (k != 0) {
2357 col += qemu_log(",");
2358 }
2359 col += qemu_log("%s", tcg_get_arg_str(s, buf, sizeof(buf),
2360 op->args[k++]));
2361 }
2362 for (i = 0; i < nb_iargs; i++) {
2363 if (k != 0) {
2364 col += qemu_log(",");
2365 }
2366 col += qemu_log("%s", tcg_get_arg_str(s, buf, sizeof(buf),
2367 op->args[k++]));
2368 }
2369 switch (c) {
2370 case INDEX_op_brcond_i32:
2371 case INDEX_op_setcond_i32:
2372 case INDEX_op_movcond_i32:
2373 case INDEX_op_brcond2_i32:
2374 case INDEX_op_setcond2_i32:
2375 case INDEX_op_brcond_i64:
2376 case INDEX_op_setcond_i64:
2377 case INDEX_op_movcond_i64:
2378 case INDEX_op_cmp_vec:
2379 case INDEX_op_cmpsel_vec:
2380 if (op->args[k] < ARRAY_SIZE(cond_name)
2381 && cond_name[op->args[k]]) {
2382 col += qemu_log(",%s", cond_name[op->args[k++]]);
2383 } else {
2384 col += qemu_log(",$0x%" TCG_PRIlx, op->args[k++]);
2385 }
2386 i = 1;
2387 break;
2388 case INDEX_op_qemu_ld_i32:
2389 case INDEX_op_qemu_st_i32:
2390 case INDEX_op_qemu_st8_i32:
2391 case INDEX_op_qemu_ld_i64:
2392 case INDEX_op_qemu_st_i64:
2393 {
2394 TCGMemOpIdx oi = op->args[k++];
2395 MemOp op = get_memop(oi);
2396 unsigned ix = get_mmuidx(oi);
2397
2398 if (op & ~(MO_AMASK | MO_BSWAP | MO_SSIZE)) {
2399 col += qemu_log(",$0x%x,%u", op, ix);
2400 } else {
2401 const char *s_al, *s_op;
2402 s_al = alignment_name[(op & MO_AMASK) >> MO_ASHIFT];
2403 s_op = ldst_name[op & (MO_BSWAP | MO_SSIZE)];
2404 col += qemu_log(",%s%s,%u", s_al, s_op, ix);
2405 }
2406 i = 1;
2407 }
2408 break;
2409 default:
2410 i = 0;
2411 break;
2412 }
2413 switch (c) {
2414 case INDEX_op_set_label:
2415 case INDEX_op_br:
2416 case INDEX_op_brcond_i32:
2417 case INDEX_op_brcond_i64:
2418 case INDEX_op_brcond2_i32:
2419 col += qemu_log("%s$L%d", k ? "," : "",
2420 arg_label(op->args[k])->id);
2421 i++, k++;
2422 break;
2423 default:
2424 break;
2425 }
2426 for (; i < nb_cargs; i++, k++) {
2427 col += qemu_log("%s$0x%" TCG_PRIlx, k ? "," : "", op->args[k]);
2428 }
2429 }
2430
2431 if (have_prefs || op->life) {
2432
2433 QemuLogFile *logfile;
2434
2435 rcu_read_lock();
2436 logfile = qatomic_rcu_read(&qemu_logfile);
2437 if (logfile) {
2438 for (; col < 40; ++col) {
2439 putc(' ', logfile->fd);
2440 }
2441 }
2442 rcu_read_unlock();
2443 }
2444
2445 if (op->life) {
2446 unsigned life = op->life;
2447
2448 if (life & (SYNC_ARG * 3)) {
2449 qemu_log(" sync:");
2450 for (i = 0; i < 2; ++i) {
2451 if (life & (SYNC_ARG << i)) {
2452 qemu_log(" %d", i);
2453 }
2454 }
2455 }
2456 life /= DEAD_ARG;
2457 if (life) {
2458 qemu_log(" dead:");
2459 for (i = 0; life; ++i, life >>= 1) {
2460 if (life & 1) {
2461 qemu_log(" %d", i);
2462 }
2463 }
2464 }
2465 }
2466
2467 if (have_prefs) {
2468 for (i = 0; i < nb_oargs; ++i) {
2469 TCGRegSet set = op->output_pref[i];
2470
2471 if (i == 0) {
2472 qemu_log(" pref=");
2473 } else {
2474 qemu_log(",");
2475 }
2476 if (set == 0) {
2477 qemu_log("none");
2478 } else if (set == MAKE_64BIT_MASK(0, TCG_TARGET_NB_REGS)) {
2479 qemu_log("all");
2480#ifdef CONFIG_DEBUG_TCG
2481 } else if (tcg_regset_single(set)) {
2482 TCGReg reg = tcg_regset_first(set);
2483 qemu_log("%s", tcg_target_reg_names[reg]);
2484#endif
2485 } else if (TCG_TARGET_NB_REGS <= 32) {
2486 qemu_log("%#x", (uint32_t)set);
2487 } else {
2488 qemu_log("%#" PRIx64, (uint64_t)set);
2489 }
2490 }
2491 }
2492
2493 qemu_log("\n");
2494 }
2495}
2496
2497
2498static int get_constraint_priority(const TCGOpDef *def, int k)
2499{
2500 const TCGArgConstraint *arg_ct = &def->args_ct[k];
2501 int n;
2502
2503 if (arg_ct->oalias) {
2504
2505 n = 1;
2506 } else {
2507 n = ctpop64(arg_ct->regs);
2508 }
2509 return TCG_TARGET_NB_REGS - n + 1;
2510}
2511
2512
2513static void sort_constraints(TCGOpDef *def, int start, int n)
2514{
2515 int i, j;
2516 TCGArgConstraint *a = def->args_ct;
2517
2518 for (i = 0; i < n; i++) {
2519 a[start + i].sort_index = start + i;
2520 }
2521 if (n <= 1) {
2522 return;
2523 }
2524 for (i = 0; i < n - 1; i++) {
2525 for (j = i + 1; j < n; j++) {
2526 int p1 = get_constraint_priority(def, a[start + i].sort_index);
2527 int p2 = get_constraint_priority(def, a[start + j].sort_index);
2528 if (p1 < p2) {
2529 int tmp = a[start + i].sort_index;
2530 a[start + i].sort_index = a[start + j].sort_index;
2531 a[start + j].sort_index = tmp;
2532 }
2533 }
2534 }
2535}
2536
2537static void process_op_defs(TCGContext *s)
2538{
2539 TCGOpcode op;
2540
2541 for (op = 0; op < NB_OPS; op++) {
2542 TCGOpDef *def = &tcg_op_defs[op];
2543 const TCGTargetOpDef *tdefs;
2544 int i, nb_args;
2545
2546 if (def->flags & TCG_OPF_NOT_PRESENT) {
2547 continue;
2548 }
2549
2550 nb_args = def->nb_iargs + def->nb_oargs;
2551 if (nb_args == 0) {
2552 continue;
2553 }
2554
2555
2556
2557
2558
2559
2560 unsigned con_set = tcg_target_op_def(op);
2561 tcg_debug_assert(con_set < ARRAY_SIZE(constraint_sets));
2562 tdefs = &constraint_sets[con_set];
2563
2564 for (i = 0; i < nb_args; i++) {
2565 const char *ct_str = tdefs->args_ct_str[i];
2566
2567 tcg_debug_assert(ct_str != NULL);
2568
2569 while (*ct_str != '\0') {
2570 switch(*ct_str) {
2571 case '0' ... '9':
2572 {
2573 int oarg = *ct_str - '0';
2574 tcg_debug_assert(ct_str == tdefs->args_ct_str[i]);
2575 tcg_debug_assert(oarg < def->nb_oargs);
2576 tcg_debug_assert(def->args_ct[oarg].regs != 0);
2577 def->args_ct[i] = def->args_ct[oarg];
2578
2579 def->args_ct[oarg].oalias = true;
2580 def->args_ct[oarg].alias_index = i;
2581
2582 def->args_ct[i].ialias = true;
2583 def->args_ct[i].alias_index = oarg;
2584 }
2585 ct_str++;
2586 break;
2587 case '&':
2588 def->args_ct[i].newreg = true;
2589 ct_str++;
2590 break;
2591 case 'i':
2592 def->args_ct[i].ct |= TCG_CT_CONST;
2593 ct_str++;
2594 break;
2595
2596
2597
2598#undef CONST
2599#define CONST(CASE, MASK) \
2600 case CASE: def->args_ct[i].ct |= MASK; ct_str++; break;
2601#define REGS(CASE, MASK) \
2602 case CASE: def->args_ct[i].regs |= MASK; ct_str++; break;
2603
2604#include "tcg-target-con-str.h"
2605
2606#undef REGS
2607#undef CONST
2608 default:
2609
2610 g_assert_not_reached();
2611 }
2612 }
2613 }
2614
2615
2616 tcg_debug_assert(i == TCG_MAX_OP_ARGS || tdefs->args_ct_str[i] == NULL);
2617
2618
2619 sort_constraints(def, 0, def->nb_oargs);
2620 sort_constraints(def, def->nb_oargs, def->nb_iargs);
2621 }
2622}
2623
2624void tcg_op_remove(TCGContext *s, TCGOp *op)
2625{
2626 TCGLabel *label;
2627
2628 switch (op->opc) {
2629 case INDEX_op_br:
2630 label = arg_label(op->args[0]);
2631 label->refs--;
2632 break;
2633 case INDEX_op_brcond_i32:
2634 case INDEX_op_brcond_i64:
2635 label = arg_label(op->args[3]);
2636 label->refs--;
2637 break;
2638 case INDEX_op_brcond2_i32:
2639 label = arg_label(op->args[5]);
2640 label->refs--;
2641 break;
2642 default:
2643 break;
2644 }
2645
2646 QTAILQ_REMOVE(&s->ops, op, link);
2647 QTAILQ_INSERT_TAIL(&s->free_ops, op, link);
2648 s->nb_ops--;
2649
2650#ifdef CONFIG_PROFILER
2651 qatomic_set(&s->prof.del_op_count, s->prof.del_op_count + 1);
2652#endif
2653}
2654
2655static TCGOp *tcg_op_alloc(TCGOpcode opc)
2656{
2657 TCGContext *s = tcg_ctx;
2658 TCGOp *op;
2659
2660 if (likely(QTAILQ_EMPTY(&s->free_ops))) {
2661 op = tcg_malloc(sizeof(TCGOp));
2662 } else {
2663 op = QTAILQ_FIRST(&s->free_ops);
2664 QTAILQ_REMOVE(&s->free_ops, op, link);
2665 }
2666 memset(op, 0, offsetof(TCGOp, link));
2667 op->opc = opc;
2668 s->nb_ops++;
2669
2670 return op;
2671}
2672
2673TCGOp *tcg_emit_op(TCGOpcode opc)
2674{
2675 TCGOp *op = tcg_op_alloc(opc);
2676 QTAILQ_INSERT_TAIL(&tcg_ctx->ops, op, link);
2677 return op;
2678}
2679
2680TCGOp *tcg_op_insert_before(TCGContext *s, TCGOp *old_op, TCGOpcode opc)
2681{
2682 TCGOp *new_op = tcg_op_alloc(opc);
2683 QTAILQ_INSERT_BEFORE(old_op, new_op, link);
2684 return new_op;
2685}
2686
2687TCGOp *tcg_op_insert_after(TCGContext *s, TCGOp *old_op, TCGOpcode opc)
2688{
2689 TCGOp *new_op = tcg_op_alloc(opc);
2690 QTAILQ_INSERT_AFTER(&s->ops, old_op, new_op, link);
2691 return new_op;
2692}
2693
2694
2695static void reachable_code_pass(TCGContext *s)
2696{
2697 TCGOp *op, *op_next;
2698 bool dead = false;
2699
2700 QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) {
2701 bool remove = dead;
2702 TCGLabel *label;
2703 int call_flags;
2704
2705 switch (op->opc) {
2706 case INDEX_op_set_label:
2707 label = arg_label(op->args[0]);
2708 if (label->refs == 0) {
2709
2710
2711
2712
2713
2714
2715
2716 remove = true;
2717 } else {
2718
2719 dead = false;
2720 remove = false;
2721
2722
2723
2724
2725
2726
2727
2728 if (label->refs == 1) {
2729 TCGOp *op_prev = QTAILQ_PREV(op, link);
2730 if (op_prev->opc == INDEX_op_br &&
2731 label == arg_label(op_prev->args[0])) {
2732 tcg_op_remove(s, op_prev);
2733 remove = true;
2734 }
2735 }
2736 }
2737 break;
2738
2739 case INDEX_op_br:
2740 case INDEX_op_exit_tb:
2741 case INDEX_op_goto_ptr:
2742
2743 dead = true;
2744 break;
2745
2746 case INDEX_op_call:
2747
2748 call_flags = op->args[TCGOP_CALLO(op) + TCGOP_CALLI(op) + 1];
2749 if (call_flags & TCG_CALL_NO_RETURN) {
2750 dead = true;
2751 }
2752 break;
2753
2754 case INDEX_op_insn_start:
2755
2756 remove = false;
2757 break;
2758
2759 default:
2760 break;
2761 }
2762
2763 if (remove) {
2764 tcg_op_remove(s, op);
2765 }
2766 }
2767}
2768
2769#define TS_DEAD 1
2770#define TS_MEM 2
2771
2772#define IS_DEAD_ARG(n) (arg_life & (DEAD_ARG << (n)))
2773#define NEED_SYNC_ARG(n) (arg_life & (SYNC_ARG << (n)))
2774
2775
2776static inline TCGRegSet *la_temp_pref(TCGTemp *ts)
2777{
2778 return ts->state_ptr;
2779}
2780
2781
2782
2783
2784static inline void la_reset_pref(TCGTemp *ts)
2785{
2786 *la_temp_pref(ts)
2787 = (ts->state == TS_DEAD ? 0 : tcg_target_available_regs[ts->type]);
2788}
2789
2790
2791
2792static void la_func_end(TCGContext *s, int ng, int nt)
2793{
2794 int i;
2795
2796 for (i = 0; i < ng; ++i) {
2797 s->temps[i].state = TS_DEAD | TS_MEM;
2798 la_reset_pref(&s->temps[i]);
2799 }
2800 for (i = ng; i < nt; ++i) {
2801 s->temps[i].state = TS_DEAD;
2802 la_reset_pref(&s->temps[i]);
2803 }
2804}
2805
2806
2807
2808static void la_bb_end(TCGContext *s, int ng, int nt)
2809{
2810 int i;
2811
2812 for (i = 0; i < nt; ++i) {
2813 TCGTemp *ts = &s->temps[i];
2814 int state;
2815
2816 switch (ts->kind) {
2817 case TEMP_FIXED:
2818 case TEMP_GLOBAL:
2819 case TEMP_LOCAL:
2820 state = TS_DEAD | TS_MEM;
2821 break;
2822 case TEMP_NORMAL:
2823 case TEMP_CONST:
2824 state = TS_DEAD;
2825 break;
2826 default:
2827 g_assert_not_reached();
2828 }
2829 ts->state = state;
2830 la_reset_pref(ts);
2831 }
2832}
2833
2834
2835static void la_global_sync(TCGContext *s, int ng)
2836{
2837 int i;
2838
2839 for (i = 0; i < ng; ++i) {
2840 int state = s->temps[i].state;
2841 s->temps[i].state = state | TS_MEM;
2842 if (state == TS_DEAD) {
2843
2844 la_reset_pref(&s->temps[i]);
2845 }
2846 }
2847}
2848
2849
2850
2851
2852
2853static void la_bb_sync(TCGContext *s, int ng, int nt)
2854{
2855 la_global_sync(s, ng);
2856
2857 for (int i = ng; i < nt; ++i) {
2858 TCGTemp *ts = &s->temps[i];
2859 int state;
2860
2861 switch (ts->kind) {
2862 case TEMP_LOCAL:
2863 state = ts->state;
2864 ts->state = state | TS_MEM;
2865 if (state != TS_DEAD) {
2866 continue;
2867 }
2868 break;
2869 case TEMP_NORMAL:
2870 s->temps[i].state = TS_DEAD;
2871 break;
2872 case TEMP_CONST:
2873 continue;
2874 default:
2875 g_assert_not_reached();
2876 }
2877 la_reset_pref(&s->temps[i]);
2878 }
2879}
2880
2881
2882static void la_global_kill(TCGContext *s, int ng)
2883{
2884 int i;
2885
2886 for (i = 0; i < ng; i++) {
2887 s->temps[i].state = TS_DEAD | TS_MEM;
2888 la_reset_pref(&s->temps[i]);
2889 }
2890}
2891
2892
2893static void la_cross_call(TCGContext *s, int nt)
2894{
2895 TCGRegSet mask = ~tcg_target_call_clobber_regs;
2896 int i;
2897
2898 for (i = 0; i < nt; i++) {
2899 TCGTemp *ts = &s->temps[i];
2900 if (!(ts->state & TS_DEAD)) {
2901 TCGRegSet *pset = la_temp_pref(ts);
2902 TCGRegSet set = *pset;
2903
2904 set &= mask;
2905
2906 if (set == 0) {
2907 set = tcg_target_available_regs[ts->type] & mask;
2908 }
2909 *pset = set;
2910 }
2911 }
2912}
2913
2914
2915
2916
2917static void liveness_pass_1(TCGContext *s)
2918{
2919 int nb_globals = s->nb_globals;
2920 int nb_temps = s->nb_temps;
2921 TCGOp *op, *op_prev;
2922 TCGRegSet *prefs;
2923 int i;
2924
2925 prefs = tcg_malloc(sizeof(TCGRegSet) * nb_temps);
2926 for (i = 0; i < nb_temps; ++i) {
2927 s->temps[i].state_ptr = prefs + i;
2928 }
2929
2930
2931 la_func_end(s, nb_globals, nb_temps);
2932
2933 QTAILQ_FOREACH_REVERSE_SAFE(op, &s->ops, link, op_prev) {
2934 int nb_iargs, nb_oargs;
2935 TCGOpcode opc_new, opc_new2;
2936 bool have_opc_new2;
2937 TCGLifeData arg_life = 0;
2938 TCGTemp *ts;
2939 TCGOpcode opc = op->opc;
2940 const TCGOpDef *def = &tcg_op_defs[opc];
2941
2942 switch (opc) {
2943 case INDEX_op_call:
2944 {
2945 int call_flags;
2946 int nb_call_regs;
2947
2948 nb_oargs = TCGOP_CALLO(op);
2949 nb_iargs = TCGOP_CALLI(op);
2950 call_flags = op->args[nb_oargs + nb_iargs + 1];
2951
2952
2953 if (call_flags & TCG_CALL_NO_SIDE_EFFECTS) {
2954 for (i = 0; i < nb_oargs; i++) {
2955 ts = arg_temp(op->args[i]);
2956 if (ts->state != TS_DEAD) {
2957 goto do_not_remove_call;
2958 }
2959 }
2960 goto do_remove;
2961 }
2962 do_not_remove_call:
2963
2964
2965 for (i = 0; i < nb_oargs; i++) {
2966 ts = arg_temp(op->args[i]);
2967 if (ts->state & TS_DEAD) {
2968 arg_life |= DEAD_ARG << i;
2969 }
2970 if (ts->state & TS_MEM) {
2971 arg_life |= SYNC_ARG << i;
2972 }
2973 ts->state = TS_DEAD;
2974 la_reset_pref(ts);
2975
2976
2977 op->output_pref[i] = 0;
2978 }
2979
2980 if (!(call_flags & (TCG_CALL_NO_WRITE_GLOBALS |
2981 TCG_CALL_NO_READ_GLOBALS))) {
2982 la_global_kill(s, nb_globals);
2983 } else if (!(call_flags & TCG_CALL_NO_READ_GLOBALS)) {
2984 la_global_sync(s, nb_globals);
2985 }
2986
2987
2988 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
2989 ts = arg_temp(op->args[i]);
2990 if (ts && ts->state & TS_DEAD) {
2991 arg_life |= DEAD_ARG << i;
2992 }
2993 }
2994
2995
2996 la_cross_call(s, nb_temps);
2997
2998 nb_call_regs = ARRAY_SIZE(tcg_target_call_iarg_regs);
2999
3000
3001 for (i = 0; i < nb_iargs; i++) {
3002 ts = arg_temp(op->args[i + nb_oargs]);
3003 if (ts && ts->state & TS_DEAD) {
3004
3005
3006
3007
3008
3009 *la_temp_pref(ts)
3010 = (i < nb_call_regs ? 0 :
3011 tcg_target_available_regs[ts->type]);
3012 ts->state &= ~TS_DEAD;
3013 }
3014 }
3015
3016
3017
3018 for (i = 0; i < MIN(nb_call_regs, nb_iargs); i++) {
3019 ts = arg_temp(op->args[i + nb_oargs]);
3020 if (ts) {
3021 tcg_regset_set_reg(*la_temp_pref(ts),
3022 tcg_target_call_iarg_regs[i]);
3023 }
3024 }
3025 }
3026 break;
3027 case INDEX_op_insn_start:
3028 break;
3029 case INDEX_op_discard:
3030
3031 ts = arg_temp(op->args[0]);
3032 ts->state = TS_DEAD;
3033 la_reset_pref(ts);
3034 break;
3035
3036 case INDEX_op_add2_i32:
3037 opc_new = INDEX_op_add_i32;
3038 goto do_addsub2;
3039 case INDEX_op_sub2_i32:
3040 opc_new = INDEX_op_sub_i32;
3041 goto do_addsub2;
3042 case INDEX_op_add2_i64:
3043 opc_new = INDEX_op_add_i64;
3044 goto do_addsub2;
3045 case INDEX_op_sub2_i64:
3046 opc_new = INDEX_op_sub_i64;
3047 do_addsub2:
3048 nb_iargs = 4;
3049 nb_oargs = 2;
3050
3051
3052
3053
3054 if (arg_temp(op->args[1])->state == TS_DEAD) {
3055 if (arg_temp(op->args[0])->state == TS_DEAD) {
3056 goto do_remove;
3057 }
3058
3059
3060 op->opc = opc = opc_new;
3061 op->args[1] = op->args[2];
3062 op->args[2] = op->args[4];
3063
3064 nb_iargs = 2;
3065 nb_oargs = 1;
3066 }
3067 goto do_not_remove;
3068
3069 case INDEX_op_mulu2_i32:
3070 opc_new = INDEX_op_mul_i32;
3071 opc_new2 = INDEX_op_muluh_i32;
3072 have_opc_new2 = TCG_TARGET_HAS_muluh_i32;
3073 goto do_mul2;
3074 case INDEX_op_muls2_i32:
3075 opc_new = INDEX_op_mul_i32;
3076 opc_new2 = INDEX_op_mulsh_i32;
3077 have_opc_new2 = TCG_TARGET_HAS_mulsh_i32;
3078 goto do_mul2;
3079 case INDEX_op_mulu2_i64:
3080 opc_new = INDEX_op_mul_i64;
3081 opc_new2 = INDEX_op_muluh_i64;
3082 have_opc_new2 = TCG_TARGET_HAS_muluh_i64;
3083 goto do_mul2;
3084 case INDEX_op_muls2_i64:
3085 opc_new = INDEX_op_mul_i64;
3086 opc_new2 = INDEX_op_mulsh_i64;
3087 have_opc_new2 = TCG_TARGET_HAS_mulsh_i64;
3088 goto do_mul2;
3089 do_mul2:
3090 nb_iargs = 2;
3091 nb_oargs = 2;
3092 if (arg_temp(op->args[1])->state == TS_DEAD) {
3093 if (arg_temp(op->args[0])->state == TS_DEAD) {
3094
3095 goto do_remove;
3096 }
3097
3098 op->opc = opc = opc_new;
3099 op->args[1] = op->args[2];
3100 op->args[2] = op->args[3];
3101 } else if (arg_temp(op->args[0])->state == TS_DEAD && have_opc_new2) {
3102
3103 op->opc = opc = opc_new2;
3104 op->args[0] = op->args[1];
3105 op->args[1] = op->args[2];
3106 op->args[2] = op->args[3];
3107 } else {
3108 goto do_not_remove;
3109 }
3110
3111 nb_oargs = 1;
3112 goto do_not_remove;
3113
3114 default:
3115
3116 nb_iargs = def->nb_iargs;
3117 nb_oargs = def->nb_oargs;
3118
3119
3120
3121
3122 if (!(def->flags & TCG_OPF_SIDE_EFFECTS) && nb_oargs != 0) {
3123 for (i = 0; i < nb_oargs; i++) {
3124 if (arg_temp(op->args[i])->state != TS_DEAD) {
3125 goto do_not_remove;
3126 }
3127 }
3128 goto do_remove;
3129 }
3130 goto do_not_remove;
3131
3132 do_remove:
3133 tcg_op_remove(s, op);
3134 break;
3135
3136 do_not_remove:
3137 for (i = 0; i < nb_oargs; i++) {
3138 ts = arg_temp(op->args[i]);
3139
3140
3141 op->output_pref[i] = *la_temp_pref(ts);
3142
3143
3144 if (ts->state & TS_DEAD) {
3145 arg_life |= DEAD_ARG << i;
3146 }
3147 if (ts->state & TS_MEM) {
3148 arg_life |= SYNC_ARG << i;
3149 }
3150 ts->state = TS_DEAD;
3151 la_reset_pref(ts);
3152 }
3153
3154
3155 if (def->flags & TCG_OPF_BB_EXIT) {
3156 la_func_end(s, nb_globals, nb_temps);
3157 } else if (def->flags & TCG_OPF_COND_BRANCH) {
3158 la_bb_sync(s, nb_globals, nb_temps);
3159 } else if (def->flags & TCG_OPF_BB_END) {
3160 la_bb_end(s, nb_globals, nb_temps);
3161 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) {
3162 la_global_sync(s, nb_globals);
3163 if (def->flags & TCG_OPF_CALL_CLOBBER) {
3164 la_cross_call(s, nb_temps);
3165 }
3166 }
3167
3168
3169 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
3170 ts = arg_temp(op->args[i]);
3171 if (ts->state & TS_DEAD) {
3172 arg_life |= DEAD_ARG << i;
3173 }
3174 }
3175
3176
3177 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
3178 ts = arg_temp(op->args[i]);
3179 if (ts->state & TS_DEAD) {
3180
3181
3182 *la_temp_pref(ts) = tcg_target_available_regs[ts->type];
3183 ts->state &= ~TS_DEAD;
3184 }
3185 }
3186
3187
3188 switch (opc) {
3189 case INDEX_op_mov_i32:
3190 case INDEX_op_mov_i64:
3191
3192
3193
3194 if (IS_DEAD_ARG(1)) {
3195 *la_temp_pref(arg_temp(op->args[0]))
3196 = *la_temp_pref(arg_temp(op->args[1]));
3197 }
3198 break;
3199
3200 default:
3201 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
3202 const TCGArgConstraint *ct = &def->args_ct[i];
3203 TCGRegSet set, *pset;
3204
3205 ts = arg_temp(op->args[i]);
3206 pset = la_temp_pref(ts);
3207 set = *pset;
3208
3209 set &= ct->regs;
3210 if (ct->ialias) {
3211 set &= op->output_pref[ct->alias_index];
3212 }
3213
3214 if (set == 0) {
3215 set = ct->regs;
3216 }
3217 *pset = set;
3218 }
3219 break;
3220 }
3221 break;
3222 }
3223 op->life = arg_life;
3224 }
3225}
3226
3227
3228static bool liveness_pass_2(TCGContext *s)
3229{
3230 int nb_globals = s->nb_globals;
3231 int nb_temps, i;
3232 bool changes = false;
3233 TCGOp *op, *op_next;
3234
3235
3236 for (i = 0; i < nb_globals; ++i) {
3237 TCGTemp *its = &s->temps[i];
3238 if (its->indirect_reg) {
3239 TCGTemp *dts = tcg_temp_alloc(s);
3240 dts->type = its->type;
3241 dts->base_type = its->base_type;
3242 its->state_ptr = dts;
3243 } else {
3244 its->state_ptr = NULL;
3245 }
3246
3247 its->state = TS_DEAD;
3248 }
3249 for (nb_temps = s->nb_temps; i < nb_temps; ++i) {
3250 TCGTemp *its = &s->temps[i];
3251 its->state_ptr = NULL;
3252 its->state = TS_DEAD;
3253 }
3254
3255 QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) {
3256 TCGOpcode opc = op->opc;
3257 const TCGOpDef *def = &tcg_op_defs[opc];
3258 TCGLifeData arg_life = op->life;
3259 int nb_iargs, nb_oargs, call_flags;
3260 TCGTemp *arg_ts, *dir_ts;
3261
3262 if (opc == INDEX_op_call) {
3263 nb_oargs = TCGOP_CALLO(op);
3264 nb_iargs = TCGOP_CALLI(op);
3265 call_flags = op->args[nb_oargs + nb_iargs + 1];
3266 } else {
3267 nb_iargs = def->nb_iargs;
3268 nb_oargs = def->nb_oargs;
3269
3270
3271 if (def->flags & TCG_OPF_COND_BRANCH) {
3272
3273 call_flags = TCG_CALL_NO_WRITE_GLOBALS;
3274 } else if (def->flags & TCG_OPF_BB_END) {
3275
3276 call_flags = 0;
3277 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) {
3278
3279 call_flags = TCG_CALL_NO_WRITE_GLOBALS;
3280 } else {
3281
3282 call_flags = (TCG_CALL_NO_READ_GLOBALS |
3283 TCG_CALL_NO_WRITE_GLOBALS);
3284 }
3285 }
3286
3287
3288 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
3289 arg_ts = arg_temp(op->args[i]);
3290 if (arg_ts) {
3291 dir_ts = arg_ts->state_ptr;
3292 if (dir_ts && arg_ts->state == TS_DEAD) {
3293 TCGOpcode lopc = (arg_ts->type == TCG_TYPE_I32
3294 ? INDEX_op_ld_i32
3295 : INDEX_op_ld_i64);
3296 TCGOp *lop = tcg_op_insert_before(s, op, lopc);
3297
3298 lop->args[0] = temp_arg(dir_ts);
3299 lop->args[1] = temp_arg(arg_ts->mem_base);
3300 lop->args[2] = arg_ts->mem_offset;
3301
3302
3303 arg_ts->state = TS_MEM;
3304 }
3305 }
3306 }
3307
3308
3309
3310
3311 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
3312 arg_ts = arg_temp(op->args[i]);
3313 if (arg_ts) {
3314 dir_ts = arg_ts->state_ptr;
3315 if (dir_ts) {
3316 op->args[i] = temp_arg(dir_ts);
3317 changes = true;
3318 if (IS_DEAD_ARG(i)) {
3319 arg_ts->state = TS_DEAD;
3320 }
3321 }
3322 }
3323 }
3324
3325
3326
3327 if (call_flags & TCG_CALL_NO_READ_GLOBALS) {
3328
3329 } else if (call_flags & TCG_CALL_NO_WRITE_GLOBALS) {
3330 for (i = 0; i < nb_globals; ++i) {
3331
3332
3333 arg_ts = &s->temps[i];
3334 tcg_debug_assert(arg_ts->state_ptr == 0
3335 || arg_ts->state != 0);
3336 }
3337 } else {
3338 for (i = 0; i < nb_globals; ++i) {
3339
3340
3341 arg_ts = &s->temps[i];
3342 tcg_debug_assert(arg_ts->state_ptr == 0
3343 || arg_ts->state == TS_DEAD);
3344 }
3345 }
3346
3347
3348 if (opc == INDEX_op_mov_i32 || opc == INDEX_op_mov_i64) {
3349 arg_ts = arg_temp(op->args[0]);
3350 dir_ts = arg_ts->state_ptr;
3351 if (dir_ts) {
3352 op->args[0] = temp_arg(dir_ts);
3353 changes = true;
3354
3355
3356 arg_ts->state = 0;
3357
3358 if (NEED_SYNC_ARG(0)) {
3359 TCGOpcode sopc = (arg_ts->type == TCG_TYPE_I32
3360 ? INDEX_op_st_i32
3361 : INDEX_op_st_i64);
3362 TCGOp *sop = tcg_op_insert_after(s, op, sopc);
3363 TCGTemp *out_ts = dir_ts;
3364
3365 if (IS_DEAD_ARG(0)) {
3366 out_ts = arg_temp(op->args[1]);
3367 arg_ts->state = TS_DEAD;
3368 tcg_op_remove(s, op);
3369 } else {
3370 arg_ts->state = TS_MEM;
3371 }
3372
3373 sop->args[0] = temp_arg(out_ts);
3374 sop->args[1] = temp_arg(arg_ts->mem_base);
3375 sop->args[2] = arg_ts->mem_offset;
3376 } else {
3377 tcg_debug_assert(!IS_DEAD_ARG(0));
3378 }
3379 }
3380 } else {
3381 for (i = 0; i < nb_oargs; i++) {
3382 arg_ts = arg_temp(op->args[i]);
3383 dir_ts = arg_ts->state_ptr;
3384 if (!dir_ts) {
3385 continue;
3386 }
3387 op->args[i] = temp_arg(dir_ts);
3388 changes = true;
3389
3390
3391 arg_ts->state = 0;
3392
3393
3394 if (NEED_SYNC_ARG(i)) {
3395 TCGOpcode sopc = (arg_ts->type == TCG_TYPE_I32
3396 ? INDEX_op_st_i32
3397 : INDEX_op_st_i64);
3398 TCGOp *sop = tcg_op_insert_after(s, op, sopc);
3399
3400 sop->args[0] = temp_arg(dir_ts);
3401 sop->args[1] = temp_arg(arg_ts->mem_base);
3402 sop->args[2] = arg_ts->mem_offset;
3403
3404 arg_ts->state = TS_MEM;
3405 }
3406
3407 if (IS_DEAD_ARG(i)) {
3408 arg_ts->state = TS_DEAD;
3409 }
3410 }
3411 }
3412 }
3413
3414 return changes;
3415}
3416
3417#ifdef CONFIG_DEBUG_TCG
3418static void dump_regs(TCGContext *s)
3419{
3420 TCGTemp *ts;
3421 int i;
3422 char buf[64];
3423
3424 for(i = 0; i < s->nb_temps; i++) {
3425 ts = &s->temps[i];
3426 printf(" %10s: ", tcg_get_arg_str_ptr(s, buf, sizeof(buf), ts));
3427 switch(ts->val_type) {
3428 case TEMP_VAL_REG:
3429 printf("%s", tcg_target_reg_names[ts->reg]);
3430 break;
3431 case TEMP_VAL_MEM:
3432 printf("%d(%s)", (int)ts->mem_offset,
3433 tcg_target_reg_names[ts->mem_base->reg]);
3434 break;
3435 case TEMP_VAL_CONST:
3436 printf("$0x%" PRIx64, ts->val);
3437 break;
3438 case TEMP_VAL_DEAD:
3439 printf("D");
3440 break;
3441 default:
3442 printf("???");
3443 break;
3444 }
3445 printf("\n");
3446 }
3447
3448 for(i = 0; i < TCG_TARGET_NB_REGS; i++) {
3449 if (s->reg_to_temp[i] != NULL) {
3450 printf("%s: %s\n",
3451 tcg_target_reg_names[i],
3452 tcg_get_arg_str_ptr(s, buf, sizeof(buf), s->reg_to_temp[i]));
3453 }
3454 }
3455}
3456
3457static void check_regs(TCGContext *s)
3458{
3459 int reg;
3460 int k;
3461 TCGTemp *ts;
3462 char buf[64];
3463
3464 for (reg = 0; reg < TCG_TARGET_NB_REGS; reg++) {
3465 ts = s->reg_to_temp[reg];
3466 if (ts != NULL) {
3467 if (ts->val_type != TEMP_VAL_REG || ts->reg != reg) {
3468 printf("Inconsistency for register %s:\n",
3469 tcg_target_reg_names[reg]);
3470 goto fail;
3471 }
3472 }
3473 }
3474 for (k = 0; k < s->nb_temps; k++) {
3475 ts = &s->temps[k];
3476 if (ts->val_type == TEMP_VAL_REG
3477 && ts->kind != TEMP_FIXED
3478 && s->reg_to_temp[ts->reg] != ts) {
3479 printf("Inconsistency for temp %s:\n",
3480 tcg_get_arg_str_ptr(s, buf, sizeof(buf), ts));
3481 fail:
3482 printf("reg state:\n");
3483 dump_regs(s);
3484 tcg_abort();
3485 }
3486 }
3487}
3488#endif
3489
3490static void temp_allocate_frame(TCGContext *s, TCGTemp *ts)
3491{
3492#if !(defined(__sparc__) && TCG_TARGET_REG_BITS == 64)
3493
3494 s->current_frame_offset = (s->current_frame_offset +
3495 (tcg_target_long)sizeof(tcg_target_long) - 1) &
3496 ~(sizeof(tcg_target_long) - 1);
3497#endif
3498 if (s->current_frame_offset + (tcg_target_long)sizeof(tcg_target_long) >
3499 s->frame_end) {
3500 tcg_abort();
3501 }
3502 ts->mem_offset = s->current_frame_offset;
3503 ts->mem_base = s->frame_temp;
3504 ts->mem_allocated = 1;
3505 s->current_frame_offset += sizeof(tcg_target_long);
3506}
3507
3508static void temp_load(TCGContext *, TCGTemp *, TCGRegSet, TCGRegSet, TCGRegSet);
3509
3510
3511
3512static void temp_free_or_dead(TCGContext *s, TCGTemp *ts, int free_or_dead)
3513{
3514 TCGTempVal new_type;
3515
3516 switch (ts->kind) {
3517 case TEMP_FIXED:
3518 return;
3519 case TEMP_GLOBAL:
3520 case TEMP_LOCAL:
3521 new_type = TEMP_VAL_MEM;
3522 break;
3523 case TEMP_NORMAL:
3524 new_type = free_or_dead < 0 ? TEMP_VAL_MEM : TEMP_VAL_DEAD;
3525 break;
3526 case TEMP_CONST:
3527 new_type = TEMP_VAL_CONST;
3528 break;
3529 default:
3530 g_assert_not_reached();
3531 }
3532 if (ts->val_type == TEMP_VAL_REG) {
3533 s->reg_to_temp[ts->reg] = NULL;
3534 }
3535 ts->val_type = new_type;
3536}
3537
3538
3539static inline void temp_dead(TCGContext *s, TCGTemp *ts)
3540{
3541 temp_free_or_dead(s, ts, 1);
3542}
3543
3544
3545
3546
3547
3548static void temp_sync(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs,
3549 TCGRegSet preferred_regs, int free_or_dead)
3550{
3551 if (!temp_readonly(ts) && !ts->mem_coherent) {
3552 if (!ts->mem_allocated) {
3553 temp_allocate_frame(s, ts);
3554 }
3555 switch (ts->val_type) {
3556 case TEMP_VAL_CONST:
3557
3558
3559
3560 if (free_or_dead
3561 && tcg_out_sti(s, ts->type, ts->val,
3562 ts->mem_base->reg, ts->mem_offset)) {
3563 break;
3564 }
3565 temp_load(s, ts, tcg_target_available_regs[ts->type],
3566 allocated_regs, preferred_regs);
3567
3568
3569 case TEMP_VAL_REG:
3570 tcg_out_st(s, ts->type, ts->reg,
3571 ts->mem_base->reg, ts->mem_offset);
3572 break;
3573
3574 case TEMP_VAL_MEM:
3575 break;
3576
3577 case TEMP_VAL_DEAD:
3578 default:
3579 tcg_abort();
3580 }
3581 ts->mem_coherent = 1;
3582 }
3583 if (free_or_dead) {
3584 temp_free_or_dead(s, ts, free_or_dead);
3585 }
3586}
3587
3588
3589static void tcg_reg_free(TCGContext *s, TCGReg reg, TCGRegSet allocated_regs)
3590{
3591 TCGTemp *ts = s->reg_to_temp[reg];
3592 if (ts != NULL) {
3593 temp_sync(s, ts, allocated_regs, 0, -1);
3594 }
3595}
3596
3597
3598
3599
3600
3601
3602
3603
3604
3605
3606
3607static TCGReg tcg_reg_alloc(TCGContext *s, TCGRegSet required_regs,
3608 TCGRegSet allocated_regs,
3609 TCGRegSet preferred_regs, bool rev)
3610{
3611 int i, j, f, n = ARRAY_SIZE(tcg_target_reg_alloc_order);
3612 TCGRegSet reg_ct[2];
3613 const int *order;
3614
3615 reg_ct[1] = required_regs & ~allocated_regs;
3616 tcg_debug_assert(reg_ct[1] != 0);
3617 reg_ct[0] = reg_ct[1] & preferred_regs;
3618
3619
3620
3621 f = reg_ct[0] == 0 || reg_ct[0] == reg_ct[1];
3622
3623 order = rev ? indirect_reg_alloc_order : tcg_target_reg_alloc_order;
3624
3625
3626 for (j = f; j < 2; j++) {
3627 TCGRegSet set = reg_ct[j];
3628
3629 if (tcg_regset_single(set)) {
3630
3631 TCGReg reg = tcg_regset_first(set);
3632 if (s->reg_to_temp[reg] == NULL) {
3633 return reg;
3634 }
3635 } else {
3636 for (i = 0; i < n; i++) {
3637 TCGReg reg = order[i];
3638 if (s->reg_to_temp[reg] == NULL &&
3639 tcg_regset_test_reg(set, reg)) {
3640 return reg;
3641 }
3642 }
3643 }
3644 }
3645
3646
3647 for (j = f; j < 2; j++) {
3648 TCGRegSet set = reg_ct[j];
3649
3650 if (tcg_regset_single(set)) {
3651
3652 TCGReg reg = tcg_regset_first(set);
3653 tcg_reg_free(s, reg, allocated_regs);
3654 return reg;
3655 } else {
3656 for (i = 0; i < n; i++) {
3657 TCGReg reg = order[i];
3658 if (tcg_regset_test_reg(set, reg)) {
3659 tcg_reg_free(s, reg, allocated_regs);
3660 return reg;
3661 }
3662 }
3663 }
3664 }
3665
3666 tcg_abort();
3667}
3668
3669
3670
3671static void temp_load(TCGContext *s, TCGTemp *ts, TCGRegSet desired_regs,
3672 TCGRegSet allocated_regs, TCGRegSet preferred_regs)
3673{
3674 TCGReg reg;
3675
3676 switch (ts->val_type) {
3677 case TEMP_VAL_REG:
3678 return;
3679 case TEMP_VAL_CONST:
3680 reg = tcg_reg_alloc(s, desired_regs, allocated_regs,
3681 preferred_regs, ts->indirect_base);
3682 if (ts->type <= TCG_TYPE_I64) {
3683 tcg_out_movi(s, ts->type, reg, ts->val);
3684 } else {
3685 uint64_t val = ts->val;
3686 MemOp vece = MO_64;
3687
3688
3689
3690
3691
3692
3693 if (val == dup_const(MO_8, val)) {
3694 vece = MO_8;
3695 } else if (val == dup_const(MO_16, val)) {
3696 vece = MO_16;
3697 } else if (val == dup_const(MO_32, val)) {
3698 vece = MO_32;
3699 }
3700
3701 tcg_out_dupi_vec(s, ts->type, vece, reg, ts->val);
3702 }
3703 ts->mem_coherent = 0;
3704 break;
3705 case TEMP_VAL_MEM:
3706 reg = tcg_reg_alloc(s, desired_regs, allocated_regs,
3707 preferred_regs, ts->indirect_base);
3708 tcg_out_ld(s, ts->type, reg, ts->mem_base->reg, ts->mem_offset);
3709 ts->mem_coherent = 1;
3710 break;
3711 case TEMP_VAL_DEAD:
3712 default:
3713 tcg_abort();
3714 }
3715 ts->reg = reg;
3716 ts->val_type = TEMP_VAL_REG;
3717 s->reg_to_temp[reg] = ts;
3718}
3719
3720
3721
3722static void temp_save(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs)
3723{
3724
3725
3726 tcg_debug_assert(ts->val_type == TEMP_VAL_MEM || temp_readonly(ts));
3727}
3728
3729
3730
3731
3732static void save_globals(TCGContext *s, TCGRegSet allocated_regs)
3733{
3734 int i, n;
3735
3736 for (i = 0, n = s->nb_globals; i < n; i++) {
3737 temp_save(s, &s->temps[i], allocated_regs);
3738 }
3739}
3740
3741
3742
3743
3744static void sync_globals(TCGContext *s, TCGRegSet allocated_regs)
3745{
3746 int i, n;
3747
3748 for (i = 0, n = s->nb_globals; i < n; i++) {
3749 TCGTemp *ts = &s->temps[i];
3750 tcg_debug_assert(ts->val_type != TEMP_VAL_REG
3751 || ts->kind == TEMP_FIXED
3752 || ts->mem_coherent);
3753 }
3754}
3755
3756
3757
3758static void tcg_reg_alloc_bb_end(TCGContext *s, TCGRegSet allocated_regs)
3759{
3760 int i;
3761
3762 for (i = s->nb_globals; i < s->nb_temps; i++) {
3763 TCGTemp *ts = &s->temps[i];
3764
3765 switch (ts->kind) {
3766 case TEMP_LOCAL:
3767 temp_save(s, ts, allocated_regs);
3768 break;
3769 case TEMP_NORMAL:
3770
3771
3772 tcg_debug_assert(ts->val_type == TEMP_VAL_DEAD);
3773 break;
3774 case TEMP_CONST:
3775
3776 tcg_debug_assert(ts->val_type == TEMP_VAL_CONST);
3777 break;
3778 default:
3779 g_assert_not_reached();
3780 }
3781 }
3782
3783 save_globals(s, allocated_regs);
3784}
3785
3786
3787
3788
3789
3790static void tcg_reg_alloc_cbranch(TCGContext *s, TCGRegSet allocated_regs)
3791{
3792 sync_globals(s, allocated_regs);
3793
3794 for (int i = s->nb_globals; i < s->nb_temps; i++) {
3795 TCGTemp *ts = &s->temps[i];
3796
3797
3798
3799
3800 switch (ts->kind) {
3801 case TEMP_LOCAL:
3802 tcg_debug_assert(ts->val_type != TEMP_VAL_REG || ts->mem_coherent);
3803 break;
3804 case TEMP_NORMAL:
3805 tcg_debug_assert(ts->val_type == TEMP_VAL_DEAD);
3806 break;
3807 case TEMP_CONST:
3808 break;
3809 default:
3810 g_assert_not_reached();
3811 }
3812 }
3813}
3814
3815
3816
3817
3818static void tcg_reg_alloc_do_movi(TCGContext *s, TCGTemp *ots,
3819 tcg_target_ulong val, TCGLifeData arg_life,
3820 TCGRegSet preferred_regs)
3821{
3822
3823 tcg_debug_assert(!temp_readonly(ots));
3824
3825
3826 if (ots->val_type == TEMP_VAL_REG) {
3827 s->reg_to_temp[ots->reg] = NULL;
3828 }
3829 ots->val_type = TEMP_VAL_CONST;
3830 ots->val = val;
3831 ots->mem_coherent = 0;
3832 if (NEED_SYNC_ARG(0)) {
3833 temp_sync(s, ots, s->reserved_regs, preferred_regs, IS_DEAD_ARG(0));
3834 } else if (IS_DEAD_ARG(0)) {
3835 temp_dead(s, ots);
3836 }
3837}
3838
3839
3840
3841
3842static void tcg_reg_alloc_mov(TCGContext *s, const TCGOp *op)
3843{
3844 const TCGLifeData arg_life = op->life;
3845 TCGRegSet allocated_regs, preferred_regs;
3846 TCGTemp *ts, *ots;
3847 TCGType otype, itype;
3848
3849 allocated_regs = s->reserved_regs;
3850 preferred_regs = op->output_pref[0];
3851 ots = arg_temp(op->args[0]);
3852 ts = arg_temp(op->args[1]);
3853
3854
3855 tcg_debug_assert(!temp_readonly(ots));
3856
3857
3858 otype = ots->type;
3859 itype = ts->type;
3860
3861 if (ts->val_type == TEMP_VAL_CONST) {
3862
3863 tcg_target_ulong val = ts->val;
3864 if (IS_DEAD_ARG(1)) {
3865 temp_dead(s, ts);
3866 }
3867 tcg_reg_alloc_do_movi(s, ots, val, arg_life, preferred_regs);
3868 return;
3869 }
3870
3871
3872
3873
3874
3875 if (ts->val_type == TEMP_VAL_MEM) {
3876 temp_load(s, ts, tcg_target_available_regs[itype],
3877 allocated_regs, preferred_regs);
3878 }
3879
3880 tcg_debug_assert(ts->val_type == TEMP_VAL_REG);
3881 if (IS_DEAD_ARG(0)) {
3882
3883
3884 tcg_debug_assert(NEED_SYNC_ARG(0));
3885 if (!ots->mem_allocated) {
3886 temp_allocate_frame(s, ots);
3887 }
3888 tcg_out_st(s, otype, ts->reg, ots->mem_base->reg, ots->mem_offset);
3889 if (IS_DEAD_ARG(1)) {
3890 temp_dead(s, ts);
3891 }
3892 temp_dead(s, ots);
3893 } else {
3894 if (IS_DEAD_ARG(1) && ts->kind != TEMP_FIXED) {
3895
3896 if (ots->val_type == TEMP_VAL_REG) {
3897 s->reg_to_temp[ots->reg] = NULL;
3898 }
3899 ots->reg = ts->reg;
3900 temp_dead(s, ts);
3901 } else {
3902 if (ots->val_type != TEMP_VAL_REG) {
3903
3904
3905 tcg_regset_set_reg(allocated_regs, ts->reg);
3906 ots->reg = tcg_reg_alloc(s, tcg_target_available_regs[otype],
3907 allocated_regs, preferred_regs,
3908 ots->indirect_base);
3909 }
3910 if (!tcg_out_mov(s, otype, ots->reg, ts->reg)) {
3911
3912
3913
3914
3915
3916 assert(!temp_readonly(ots));
3917 if (!ts->mem_allocated) {
3918 temp_allocate_frame(s, ots);
3919 }
3920 tcg_out_st(s, ts->type, ts->reg,
3921 ots->mem_base->reg, ots->mem_offset);
3922 ots->mem_coherent = 1;
3923 temp_free_or_dead(s, ots, -1);
3924 return;
3925 }
3926 }
3927 ots->val_type = TEMP_VAL_REG;
3928 ots->mem_coherent = 0;
3929 s->reg_to_temp[ots->reg] = ots;
3930 if (NEED_SYNC_ARG(0)) {
3931 temp_sync(s, ots, allocated_regs, 0, 0);
3932 }
3933 }
3934}
3935
3936
3937
3938
3939static void tcg_reg_alloc_dup(TCGContext *s, const TCGOp *op)
3940{
3941 const TCGLifeData arg_life = op->life;
3942 TCGRegSet dup_out_regs, dup_in_regs;
3943 TCGTemp *its, *ots;
3944 TCGType itype, vtype;
3945 intptr_t endian_fixup;
3946 unsigned vece;
3947 bool ok;
3948
3949 ots = arg_temp(op->args[0]);
3950 its = arg_temp(op->args[1]);
3951
3952
3953 tcg_debug_assert(!temp_readonly(ots));
3954
3955 itype = its->type;
3956 vece = TCGOP_VECE(op);
3957 vtype = TCGOP_VECL(op) + TCG_TYPE_V64;
3958
3959 if (its->val_type == TEMP_VAL_CONST) {
3960
3961 tcg_target_ulong val = its->val;
3962 if (IS_DEAD_ARG(1)) {
3963 temp_dead(s, its);
3964 }
3965 tcg_reg_alloc_do_movi(s, ots, val, arg_life, op->output_pref[0]);
3966 return;
3967 }
3968
3969 dup_out_regs = tcg_op_defs[INDEX_op_dup_vec].args_ct[0].regs;
3970 dup_in_regs = tcg_op_defs[INDEX_op_dup_vec].args_ct[1].regs;
3971
3972
3973 if (ots->val_type != TEMP_VAL_REG) {
3974 TCGRegSet allocated_regs = s->reserved_regs;
3975
3976 if (!IS_DEAD_ARG(1) && its->val_type == TEMP_VAL_REG) {
3977
3978 tcg_regset_set_reg(allocated_regs, its->reg);
3979 }
3980 ots->reg = tcg_reg_alloc(s, dup_out_regs, allocated_regs,
3981 op->output_pref[0], ots->indirect_base);
3982 ots->val_type = TEMP_VAL_REG;
3983 ots->mem_coherent = 0;
3984 s->reg_to_temp[ots->reg] = ots;
3985 }
3986
3987 switch (its->val_type) {
3988 case TEMP_VAL_REG:
3989
3990
3991
3992
3993
3994 if (tcg_regset_test_reg(dup_in_regs, its->reg)) {
3995 if (tcg_out_dup_vec(s, vtype, vece, ots->reg, its->reg)) {
3996 goto done;
3997 }
3998
3999 }
4000 if (!its->mem_coherent) {
4001
4002
4003
4004
4005
4006 if (tcg_out_mov(s, itype, ots->reg, its->reg)) {
4007 break;
4008 }
4009
4010 temp_sync(s, its, s->reserved_regs, 0, 0);
4011 }
4012
4013
4014 case TEMP_VAL_MEM:
4015#ifdef HOST_WORDS_BIGENDIAN
4016 endian_fixup = itype == TCG_TYPE_I32 ? 4 : 8;
4017 endian_fixup -= 1 << vece;
4018#else
4019 endian_fixup = 0;
4020#endif
4021 if (tcg_out_dupm_vec(s, vtype, vece, ots->reg, its->mem_base->reg,
4022 its->mem_offset + endian_fixup)) {
4023 goto done;
4024 }
4025 tcg_out_ld(s, itype, ots->reg, its->mem_base->reg, its->mem_offset);
4026 break;
4027
4028 default:
4029 g_assert_not_reached();
4030 }
4031
4032
4033 ok = tcg_out_dup_vec(s, vtype, vece, ots->reg, ots->reg);
4034 tcg_debug_assert(ok);
4035
4036 done:
4037 if (IS_DEAD_ARG(1)) {
4038 temp_dead(s, its);
4039 }
4040 if (NEED_SYNC_ARG(0)) {
4041 temp_sync(s, ots, s->reserved_regs, 0, 0);
4042 }
4043 if (IS_DEAD_ARG(0)) {
4044 temp_dead(s, ots);
4045 }
4046}
4047
4048static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op)
4049{
4050 const TCGLifeData arg_life = op->life;
4051 const TCGOpDef * const def = &tcg_op_defs[op->opc];
4052 TCGRegSet i_allocated_regs;
4053 TCGRegSet o_allocated_regs;
4054 int i, k, nb_iargs, nb_oargs;
4055 TCGReg reg;
4056 TCGArg arg;
4057 const TCGArgConstraint *arg_ct;
4058 TCGTemp *ts;
4059 TCGArg new_args[TCG_MAX_OP_ARGS];
4060 int const_args[TCG_MAX_OP_ARGS];
4061
4062 nb_oargs = def->nb_oargs;
4063 nb_iargs = def->nb_iargs;
4064
4065
4066 memcpy(new_args + nb_oargs + nb_iargs,
4067 op->args + nb_oargs + nb_iargs,
4068 sizeof(TCGArg) * def->nb_cargs);
4069
4070 i_allocated_regs = s->reserved_regs;
4071 o_allocated_regs = s->reserved_regs;
4072
4073
4074 for (k = 0; k < nb_iargs; k++) {
4075 TCGRegSet i_preferred_regs, o_preferred_regs;
4076
4077 i = def->args_ct[nb_oargs + k].sort_index;
4078 arg = op->args[i];
4079 arg_ct = &def->args_ct[i];
4080 ts = arg_temp(arg);
4081
4082 if (ts->val_type == TEMP_VAL_CONST
4083 && tcg_target_const_match(ts->val, ts->type, arg_ct)) {
4084
4085 const_args[i] = 1;
4086 new_args[i] = ts->val;
4087 continue;
4088 }
4089
4090 i_preferred_regs = o_preferred_regs = 0;
4091 if (arg_ct->ialias) {
4092 o_preferred_regs = op->output_pref[arg_ct->alias_index];
4093
4094
4095
4096
4097
4098
4099
4100 if (temp_readonly(ts) || !IS_DEAD_ARG(i)) {
4101 goto allocate_in_reg;
4102 }
4103
4104
4105
4106
4107
4108 if (ts->val_type == TEMP_VAL_REG) {
4109 reg = ts->reg;
4110 for (int k2 = 0; k2 < k; k2++) {
4111 int i2 = def->args_ct[nb_oargs + k2].sort_index;
4112 if (def->args_ct[i2].ialias && reg == new_args[i2]) {
4113 goto allocate_in_reg;
4114 }
4115 }
4116 }
4117 i_preferred_regs = o_preferred_regs;
4118 }
4119
4120 temp_load(s, ts, arg_ct->regs, i_allocated_regs, i_preferred_regs);
4121 reg = ts->reg;
4122
4123 if (!tcg_regset_test_reg(arg_ct->regs, reg)) {
4124 allocate_in_reg:
4125
4126
4127
4128
4129 temp_load(s, ts, tcg_target_available_regs[ts->type],
4130 i_allocated_regs, 0);
4131 reg = tcg_reg_alloc(s, arg_ct->regs, i_allocated_regs,
4132 o_preferred_regs, ts->indirect_base);
4133 if (!tcg_out_mov(s, ts->type, reg, ts->reg)) {
4134
4135
4136
4137
4138 temp_sync(s, ts, i_allocated_regs, 0, 0);
4139 tcg_out_ld(s, ts->type, reg,
4140 ts->mem_base->reg, ts->mem_offset);
4141 }
4142 }
4143 new_args[i] = reg;
4144 const_args[i] = 0;
4145 tcg_regset_set_reg(i_allocated_regs, reg);
4146 }
4147
4148
4149 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
4150 if (IS_DEAD_ARG(i)) {
4151 temp_dead(s, arg_temp(op->args[i]));
4152 }
4153 }
4154
4155 if (def->flags & TCG_OPF_COND_BRANCH) {
4156 tcg_reg_alloc_cbranch(s, i_allocated_regs);
4157 } else if (def->flags & TCG_OPF_BB_END) {
4158 tcg_reg_alloc_bb_end(s, i_allocated_regs);
4159 } else {
4160 if (def->flags & TCG_OPF_CALL_CLOBBER) {
4161
4162 for (i = 0; i < TCG_TARGET_NB_REGS; i++) {
4163 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) {
4164 tcg_reg_free(s, i, i_allocated_regs);
4165 }
4166 }
4167 }
4168 if (def->flags & TCG_OPF_SIDE_EFFECTS) {
4169
4170
4171 sync_globals(s, i_allocated_regs);
4172 }
4173
4174
4175 for(k = 0; k < nb_oargs; k++) {
4176 i = def->args_ct[k].sort_index;
4177 arg = op->args[i];
4178 arg_ct = &def->args_ct[i];
4179 ts = arg_temp(arg);
4180
4181
4182 tcg_debug_assert(!temp_readonly(ts));
4183
4184 if (arg_ct->oalias && !const_args[arg_ct->alias_index]) {
4185 reg = new_args[arg_ct->alias_index];
4186 } else if (arg_ct->newreg) {
4187 reg = tcg_reg_alloc(s, arg_ct->regs,
4188 i_allocated_regs | o_allocated_regs,
4189 op->output_pref[k], ts->indirect_base);
4190 } else {
4191 reg = tcg_reg_alloc(s, arg_ct->regs, o_allocated_regs,
4192 op->output_pref[k], ts->indirect_base);
4193 }
4194 tcg_regset_set_reg(o_allocated_regs, reg);
4195 if (ts->val_type == TEMP_VAL_REG) {
4196 s->reg_to_temp[ts->reg] = NULL;
4197 }
4198 ts->val_type = TEMP_VAL_REG;
4199 ts->reg = reg;
4200
4201
4202
4203
4204 ts->mem_coherent = 0;
4205 s->reg_to_temp[reg] = ts;
4206 new_args[i] = reg;
4207 }
4208 }
4209
4210
4211 if (def->flags & TCG_OPF_VECTOR) {
4212 tcg_out_vec_op(s, op->opc, TCGOP_VECL(op), TCGOP_VECE(op),
4213 new_args, const_args);
4214 } else {
4215 tcg_out_op(s, op->opc, new_args, const_args);
4216 }
4217
4218
4219 for(i = 0; i < nb_oargs; i++) {
4220 ts = arg_temp(op->args[i]);
4221
4222
4223 tcg_debug_assert(!temp_readonly(ts));
4224
4225 if (NEED_SYNC_ARG(i)) {
4226 temp_sync(s, ts, o_allocated_regs, 0, IS_DEAD_ARG(i));
4227 } else if (IS_DEAD_ARG(i)) {
4228 temp_dead(s, ts);
4229 }
4230 }
4231}
4232
4233static bool tcg_reg_alloc_dup2(TCGContext *s, const TCGOp *op)
4234{
4235 const TCGLifeData arg_life = op->life;
4236 TCGTemp *ots, *itsl, *itsh;
4237 TCGType vtype = TCGOP_VECL(op) + TCG_TYPE_V64;
4238
4239
4240 tcg_debug_assert(TCG_TARGET_REG_BITS == 32);
4241 tcg_debug_assert(TCGOP_VECE(op) == MO_64);
4242
4243 ots = arg_temp(op->args[0]);
4244 itsl = arg_temp(op->args[1]);
4245 itsh = arg_temp(op->args[2]);
4246
4247
4248 tcg_debug_assert(!temp_readonly(ots));
4249
4250
4251 if (ots->val_type != TEMP_VAL_REG) {
4252 TCGRegSet allocated_regs = s->reserved_regs;
4253 TCGRegSet dup_out_regs =
4254 tcg_op_defs[INDEX_op_dup_vec].args_ct[0].regs;
4255
4256
4257 if (!IS_DEAD_ARG(1) && itsl->val_type == TEMP_VAL_REG) {
4258 tcg_regset_set_reg(allocated_regs, itsl->reg);
4259 }
4260 if (!IS_DEAD_ARG(2) && itsh->val_type == TEMP_VAL_REG) {
4261 tcg_regset_set_reg(allocated_regs, itsh->reg);
4262 }
4263
4264 ots->reg = tcg_reg_alloc(s, dup_out_regs, allocated_regs,
4265 op->output_pref[0], ots->indirect_base);
4266 ots->val_type = TEMP_VAL_REG;
4267 ots->mem_coherent = 0;
4268 s->reg_to_temp[ots->reg] = ots;
4269 }
4270
4271
4272 if (itsl->val_type == TEMP_VAL_CONST && itsh->val_type == TEMP_VAL_CONST) {
4273 uint64_t val = deposit64(itsl->val, 32, 32, itsh->val);
4274 MemOp vece = MO_64;
4275
4276 if (val == dup_const(MO_8, val)) {
4277 vece = MO_8;
4278 } else if (val == dup_const(MO_16, val)) {
4279 vece = MO_16;
4280 } else if (val == dup_const(MO_32, val)) {
4281 vece = MO_32;
4282 }
4283
4284 tcg_out_dupi_vec(s, vtype, vece, ots->reg, val);
4285 goto done;
4286 }
4287
4288
4289 if (itsl + 1 == itsh && itsl->base_type == TCG_TYPE_I64) {
4290 if (!itsl->mem_coherent) {
4291 temp_sync(s, itsl, s->reserved_regs, 0, 0);
4292 }
4293 if (!itsh->mem_coherent) {
4294 temp_sync(s, itsh, s->reserved_regs, 0, 0);
4295 }
4296#ifdef HOST_WORDS_BIGENDIAN
4297 TCGTemp *its = itsh;
4298#else
4299 TCGTemp *its = itsl;
4300#endif
4301 if (tcg_out_dupm_vec(s, vtype, MO_64, ots->reg,
4302 its->mem_base->reg, its->mem_offset)) {
4303 goto done;
4304 }
4305 }
4306
4307
4308 return false;
4309
4310 done:
4311 if (IS_DEAD_ARG(1)) {
4312 temp_dead(s, itsl);
4313 }
4314 if (IS_DEAD_ARG(2)) {
4315 temp_dead(s, itsh);
4316 }
4317 if (NEED_SYNC_ARG(0)) {
4318 temp_sync(s, ots, s->reserved_regs, 0, IS_DEAD_ARG(0));
4319 } else if (IS_DEAD_ARG(0)) {
4320 temp_dead(s, ots);
4321 }
4322 return true;
4323}
4324
4325#ifdef TCG_TARGET_STACK_GROWSUP
4326#define STACK_DIR(x) (-(x))
4327#else
4328#define STACK_DIR(x) (x)
4329#endif
4330
4331static void tcg_reg_alloc_call(TCGContext *s, TCGOp *op)
4332{
4333 const int nb_oargs = TCGOP_CALLO(op);
4334 const int nb_iargs = TCGOP_CALLI(op);
4335 const TCGLifeData arg_life = op->life;
4336 int flags, nb_regs, i;
4337 TCGReg reg;
4338 TCGArg arg;
4339 TCGTemp *ts;
4340 intptr_t stack_offset;
4341 size_t call_stack_size;
4342 tcg_insn_unit *func_addr;
4343 int allocate_args;
4344 TCGRegSet allocated_regs;
4345
4346 func_addr = (tcg_insn_unit *)(intptr_t)op->args[nb_oargs + nb_iargs];
4347 flags = op->args[nb_oargs + nb_iargs + 1];
4348
4349 nb_regs = ARRAY_SIZE(tcg_target_call_iarg_regs);
4350 if (nb_regs > nb_iargs) {
4351 nb_regs = nb_iargs;
4352 }
4353
4354
4355 call_stack_size = (nb_iargs - nb_regs) * sizeof(tcg_target_long);
4356 call_stack_size = (call_stack_size + TCG_TARGET_STACK_ALIGN - 1) &
4357 ~(TCG_TARGET_STACK_ALIGN - 1);
4358 allocate_args = (call_stack_size > TCG_STATIC_CALL_ARGS_SIZE);
4359 if (allocate_args) {
4360
4361
4362 tcg_abort();
4363 }
4364
4365 stack_offset = TCG_TARGET_CALL_STACK_OFFSET;
4366 for (i = nb_regs; i < nb_iargs; i++) {
4367 arg = op->args[nb_oargs + i];
4368#ifdef TCG_TARGET_STACK_GROWSUP
4369 stack_offset -= sizeof(tcg_target_long);
4370#endif
4371 if (arg != TCG_CALL_DUMMY_ARG) {
4372 ts = arg_temp(arg);
4373 temp_load(s, ts, tcg_target_available_regs[ts->type],
4374 s->reserved_regs, 0);
4375 tcg_out_st(s, ts->type, ts->reg, TCG_REG_CALL_STACK, stack_offset);
4376 }
4377#ifndef TCG_TARGET_STACK_GROWSUP
4378 stack_offset += sizeof(tcg_target_long);
4379#endif
4380 }
4381
4382
4383 allocated_regs = s->reserved_regs;
4384 for (i = 0; i < nb_regs; i++) {
4385 arg = op->args[nb_oargs + i];
4386 if (arg != TCG_CALL_DUMMY_ARG) {
4387 ts = arg_temp(arg);
4388 reg = tcg_target_call_iarg_regs[i];
4389
4390 if (ts->val_type == TEMP_VAL_REG) {
4391 if (ts->reg != reg) {
4392 tcg_reg_free(s, reg, allocated_regs);
4393 if (!tcg_out_mov(s, ts->type, reg, ts->reg)) {
4394
4395
4396
4397
4398 temp_sync(s, ts, allocated_regs, 0, 0);
4399 tcg_out_ld(s, ts->type, reg,
4400 ts->mem_base->reg, ts->mem_offset);
4401 }
4402 }
4403 } else {
4404 TCGRegSet arg_set = 0;
4405
4406 tcg_reg_free(s, reg, allocated_regs);
4407 tcg_regset_set_reg(arg_set, reg);
4408 temp_load(s, ts, arg_set, allocated_regs, 0);
4409 }
4410
4411 tcg_regset_set_reg(allocated_regs, reg);
4412 }
4413 }
4414
4415
4416 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
4417 if (IS_DEAD_ARG(i)) {
4418 temp_dead(s, arg_temp(op->args[i]));
4419 }
4420 }
4421
4422
4423 for (i = 0; i < TCG_TARGET_NB_REGS; i++) {
4424 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) {
4425 tcg_reg_free(s, i, allocated_regs);
4426 }
4427 }
4428
4429
4430
4431 if (flags & TCG_CALL_NO_READ_GLOBALS) {
4432
4433 } else if (flags & TCG_CALL_NO_WRITE_GLOBALS) {
4434 sync_globals(s, allocated_regs);
4435 } else {
4436 save_globals(s, allocated_regs);
4437 }
4438
4439 tcg_out_call(s, func_addr);
4440
4441
4442 for(i = 0; i < nb_oargs; i++) {
4443 arg = op->args[i];
4444 ts = arg_temp(arg);
4445
4446
4447 tcg_debug_assert(!temp_readonly(ts));
4448
4449 reg = tcg_target_call_oarg_regs[i];
4450 tcg_debug_assert(s->reg_to_temp[reg] == NULL);
4451 if (ts->val_type == TEMP_VAL_REG) {
4452 s->reg_to_temp[ts->reg] = NULL;
4453 }
4454 ts->val_type = TEMP_VAL_REG;
4455 ts->reg = reg;
4456 ts->mem_coherent = 0;
4457 s->reg_to_temp[reg] = ts;
4458 if (NEED_SYNC_ARG(i)) {
4459 temp_sync(s, ts, allocated_regs, 0, IS_DEAD_ARG(i));
4460 } else if (IS_DEAD_ARG(i)) {
4461 temp_dead(s, ts);
4462 }
4463 }
4464}
4465
4466#ifdef CONFIG_PROFILER
4467
4468
4469#define PROF_ADD(to, from, field) \
4470 do { \
4471 (to)->field += qatomic_read(&((from)->field)); \
4472 } while (0)
4473
4474#define PROF_MAX(to, from, field) \
4475 do { \
4476 typeof((from)->field) val__ = qatomic_read(&((from)->field)); \
4477 if (val__ > (to)->field) { \
4478 (to)->field = val__; \
4479 } \
4480 } while (0)
4481
4482
4483static inline
4484void tcg_profile_snapshot(TCGProfile *prof, bool counters, bool table)
4485{
4486 unsigned int n_ctxs = qatomic_read(&n_tcg_ctxs);
4487 unsigned int i;
4488
4489 for (i = 0; i < n_ctxs; i++) {
4490 TCGContext *s = qatomic_read(&tcg_ctxs[i]);
4491 const TCGProfile *orig = &s->prof;
4492
4493 if (counters) {
4494 PROF_ADD(prof, orig, cpu_exec_time);
4495 PROF_ADD(prof, orig, tb_count1);
4496 PROF_ADD(prof, orig, tb_count);
4497 PROF_ADD(prof, orig, op_count);
4498 PROF_MAX(prof, orig, op_count_max);
4499 PROF_ADD(prof, orig, temp_count);
4500 PROF_MAX(prof, orig, temp_count_max);
4501 PROF_ADD(prof, orig, del_op_count);
4502 PROF_ADD(prof, orig, code_in_len);
4503 PROF_ADD(prof, orig, code_out_len);
4504 PROF_ADD(prof, orig, search_out_len);
4505 PROF_ADD(prof, orig, interm_time);
4506 PROF_ADD(prof, orig, code_time);
4507 PROF_ADD(prof, orig, la_time);
4508 PROF_ADD(prof, orig, opt_time);
4509 PROF_ADD(prof, orig, restore_count);
4510 PROF_ADD(prof, orig, restore_time);
4511 }
4512 if (table) {
4513 int i;
4514
4515 for (i = 0; i < NB_OPS; i++) {
4516 PROF_ADD(prof, orig, table_op_count[i]);
4517 }
4518 }
4519 }
4520}
4521
4522#undef PROF_ADD
4523#undef PROF_MAX
4524
4525static void tcg_profile_snapshot_counters(TCGProfile *prof)
4526{
4527 tcg_profile_snapshot(prof, true, false);
4528}
4529
4530static void tcg_profile_snapshot_table(TCGProfile *prof)
4531{
4532 tcg_profile_snapshot(prof, false, true);
4533}
4534
4535void tcg_dump_op_count(void)
4536{
4537 TCGProfile prof = {};
4538 int i;
4539
4540 tcg_profile_snapshot_table(&prof);
4541 for (i = 0; i < NB_OPS; i++) {
4542 qemu_printf("%s %" PRId64 "\n", tcg_op_defs[i].name,
4543 prof.table_op_count[i]);
4544 }
4545}
4546
4547int64_t tcg_cpu_exec_time(void)
4548{
4549 unsigned int n_ctxs = qatomic_read(&n_tcg_ctxs);
4550 unsigned int i;
4551 int64_t ret = 0;
4552
4553 for (i = 0; i < n_ctxs; i++) {
4554 const TCGContext *s = qatomic_read(&tcg_ctxs[i]);
4555 const TCGProfile *prof = &s->prof;
4556
4557 ret += qatomic_read(&prof->cpu_exec_time);
4558 }
4559 return ret;
4560}
4561#else
4562void tcg_dump_op_count(void)
4563{
4564 qemu_printf("[TCG profiler not compiled]\n");
4565}
4566
4567int64_t tcg_cpu_exec_time(void)
4568{
4569 error_report("%s: TCG profiler not compiled", __func__);
4570 exit(EXIT_FAILURE);
4571}
4572#endif
4573
4574
4575int tcg_gen_code(TCGContext *s, TranslationBlock *tb)
4576{
4577#ifdef CONFIG_PROFILER
4578 TCGProfile *prof = &s->prof;
4579#endif
4580 int i, num_insns;
4581 TCGOp *op;
4582
4583#ifdef CONFIG_PROFILER
4584 {
4585 int n = 0;
4586
4587 QTAILQ_FOREACH(op, &s->ops, link) {
4588 n++;
4589 }
4590 qatomic_set(&prof->op_count, prof->op_count + n);
4591 if (n > prof->op_count_max) {
4592 qatomic_set(&prof->op_count_max, n);
4593 }
4594
4595 n = s->nb_temps;
4596 qatomic_set(&prof->temp_count, prof->temp_count + n);
4597 if (n > prof->temp_count_max) {
4598 qatomic_set(&prof->temp_count_max, n);
4599 }
4600 }
4601#endif
4602
4603#ifdef DEBUG_DISAS
4604 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP)
4605 && qemu_log_in_addr_range(tb->pc))) {
4606 FILE *logfile = qemu_log_lock();
4607 qemu_log("OP:\n");
4608 tcg_dump_ops(s, false);
4609 qemu_log("\n");
4610 qemu_log_unlock(logfile);
4611 }
4612#endif
4613
4614#ifdef CONFIG_DEBUG_TCG
4615
4616 {
4617 TCGLabel *l;
4618 bool error = false;
4619
4620 QSIMPLEQ_FOREACH(l, &s->labels, next) {
4621 if (unlikely(!l->present) && l->refs) {
4622 qemu_log_mask(CPU_LOG_TB_OP,
4623 "$L%d referenced but not present.\n", l->id);
4624 error = true;
4625 }
4626 }
4627 assert(!error);
4628 }
4629#endif
4630
4631#ifdef CONFIG_PROFILER
4632 qatomic_set(&prof->opt_time, prof->opt_time - profile_getclock());
4633#endif
4634
4635#ifdef USE_TCG_OPTIMIZATIONS
4636 tcg_optimize(s);
4637#endif
4638
4639#ifdef CONFIG_PROFILER
4640 qatomic_set(&prof->opt_time, prof->opt_time + profile_getclock());
4641 qatomic_set(&prof->la_time, prof->la_time - profile_getclock());
4642#endif
4643
4644 reachable_code_pass(s);
4645 liveness_pass_1(s);
4646
4647 if (s->nb_indirects > 0) {
4648#ifdef DEBUG_DISAS
4649 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_IND)
4650 && qemu_log_in_addr_range(tb->pc))) {
4651 FILE *logfile = qemu_log_lock();
4652 qemu_log("OP before indirect lowering:\n");
4653 tcg_dump_ops(s, false);
4654 qemu_log("\n");
4655 qemu_log_unlock(logfile);
4656 }
4657#endif
4658
4659 if (liveness_pass_2(s)) {
4660
4661 liveness_pass_1(s);
4662 }
4663 }
4664
4665#ifdef CONFIG_PROFILER
4666 qatomic_set(&prof->la_time, prof->la_time + profile_getclock());
4667#endif
4668
4669#ifdef DEBUG_DISAS
4670 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_OPT)
4671 && qemu_log_in_addr_range(tb->pc))) {
4672 FILE *logfile = qemu_log_lock();
4673 qemu_log("OP after optimization and liveness analysis:\n");
4674 tcg_dump_ops(s, true);
4675 qemu_log("\n");
4676 qemu_log_unlock(logfile);
4677 }
4678#endif
4679
4680 tcg_reg_alloc_start(s);
4681
4682
4683
4684
4685
4686
4687 s->code_buf = tcg_splitwx_to_rw(tb->tc.ptr);
4688 s->code_ptr = s->code_buf;
4689
4690#ifdef TCG_TARGET_NEED_LDST_LABELS
4691 QSIMPLEQ_INIT(&s->ldst_labels);
4692#endif
4693#ifdef TCG_TARGET_NEED_POOL_LABELS
4694 s->pool_labels = NULL;
4695#endif
4696
4697 num_insns = -1;
4698 QTAILQ_FOREACH(op, &s->ops, link) {
4699 TCGOpcode opc = op->opc;
4700
4701#ifdef CONFIG_PROFILER
4702 qatomic_set(&prof->table_op_count[opc], prof->table_op_count[opc] + 1);
4703#endif
4704
4705 switch (opc) {
4706 case INDEX_op_mov_i32:
4707 case INDEX_op_mov_i64:
4708 case INDEX_op_mov_vec:
4709 tcg_reg_alloc_mov(s, op);
4710 break;
4711 case INDEX_op_dup_vec:
4712 tcg_reg_alloc_dup(s, op);
4713 break;
4714 case INDEX_op_insn_start:
4715 if (num_insns >= 0) {
4716 size_t off = tcg_current_code_size(s);
4717 s->gen_insn_end_off[num_insns] = off;
4718
4719 assert(s->gen_insn_end_off[num_insns] == off);
4720 }
4721 num_insns++;
4722 for (i = 0; i < TARGET_INSN_START_WORDS; ++i) {
4723 target_ulong a;
4724#if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
4725 a = deposit64(op->args[i * 2], 32, 32, op->args[i * 2 + 1]);
4726#else
4727 a = op->args[i];
4728#endif
4729 s->gen_insn_data[num_insns][i] = a;
4730 }
4731 break;
4732 case INDEX_op_discard:
4733 temp_dead(s, arg_temp(op->args[0]));
4734 break;
4735 case INDEX_op_set_label:
4736 tcg_reg_alloc_bb_end(s, s->reserved_regs);
4737 tcg_out_label(s, arg_label(op->args[0]));
4738 break;
4739 case INDEX_op_call:
4740 tcg_reg_alloc_call(s, op);
4741 break;
4742 case INDEX_op_dup2_vec:
4743 if (tcg_reg_alloc_dup2(s, op)) {
4744 break;
4745 }
4746
4747 default:
4748
4749 tcg_debug_assert(tcg_op_supported(opc));
4750
4751
4752
4753 tcg_reg_alloc_op(s, op);
4754 break;
4755 }
4756#ifdef CONFIG_DEBUG_TCG
4757 check_regs(s);
4758#endif
4759
4760
4761
4762
4763 if (unlikely((void *)s->code_ptr > s->code_gen_highwater)) {
4764 return -1;
4765 }
4766
4767 if (unlikely(tcg_current_code_size(s) > UINT16_MAX)) {
4768 return -2;
4769 }
4770 }
4771 tcg_debug_assert(num_insns >= 0);
4772 s->gen_insn_end_off[num_insns] = tcg_current_code_size(s);
4773
4774
4775#ifdef TCG_TARGET_NEED_LDST_LABELS
4776 i = tcg_out_ldst_finalize(s);
4777 if (i < 0) {
4778 return i;
4779 }
4780#endif
4781#ifdef TCG_TARGET_NEED_POOL_LABELS
4782 i = tcg_out_pool_finalize(s);
4783 if (i < 0) {
4784 return i;
4785 }
4786#endif
4787 if (!tcg_resolve_relocs(s)) {
4788 return -2;
4789 }
4790
4791#ifndef CONFIG_TCG_INTERPRETER
4792
4793 flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf),
4794 (uintptr_t)s->code_buf,
4795 tcg_ptr_byte_diff(s->code_ptr, s->code_buf));
4796#endif
4797
4798 return tcg_current_code_size(s);
4799}
4800
4801#ifdef CONFIG_PROFILER
4802void tcg_dump_info(void)
4803{
4804 TCGProfile prof = {};
4805 const TCGProfile *s;
4806 int64_t tb_count;
4807 int64_t tb_div_count;
4808 int64_t tot;
4809
4810 tcg_profile_snapshot_counters(&prof);
4811 s = &prof;
4812 tb_count = s->tb_count;
4813 tb_div_count = tb_count ? tb_count : 1;
4814 tot = s->interm_time + s->code_time;
4815
4816 qemu_printf("JIT cycles %" PRId64 " (%0.3f s at 2.4 GHz)\n",
4817 tot, tot / 2.4e9);
4818 qemu_printf("translated TBs %" PRId64 " (aborted=%" PRId64
4819 " %0.1f%%)\n",
4820 tb_count, s->tb_count1 - tb_count,
4821 (double)(s->tb_count1 - s->tb_count)
4822 / (s->tb_count1 ? s->tb_count1 : 1) * 100.0);
4823 qemu_printf("avg ops/TB %0.1f max=%d\n",
4824 (double)s->op_count / tb_div_count, s->op_count_max);
4825 qemu_printf("deleted ops/TB %0.2f\n",
4826 (double)s->del_op_count / tb_div_count);
4827 qemu_printf("avg temps/TB %0.2f max=%d\n",
4828 (double)s->temp_count / tb_div_count, s->temp_count_max);
4829 qemu_printf("avg host code/TB %0.1f\n",
4830 (double)s->code_out_len / tb_div_count);
4831 qemu_printf("avg search data/TB %0.1f\n",
4832 (double)s->search_out_len / tb_div_count);
4833
4834 qemu_printf("cycles/op %0.1f\n",
4835 s->op_count ? (double)tot / s->op_count : 0);
4836 qemu_printf("cycles/in byte %0.1f\n",
4837 s->code_in_len ? (double)tot / s->code_in_len : 0);
4838 qemu_printf("cycles/out byte %0.1f\n",
4839 s->code_out_len ? (double)tot / s->code_out_len : 0);
4840 qemu_printf("cycles/search byte %0.1f\n",
4841 s->search_out_len ? (double)tot / s->search_out_len : 0);
4842 if (tot == 0) {
4843 tot = 1;
4844 }
4845 qemu_printf(" gen_interm time %0.1f%%\n",
4846 (double)s->interm_time / tot * 100.0);
4847 qemu_printf(" gen_code time %0.1f%%\n",
4848 (double)s->code_time / tot * 100.0);
4849 qemu_printf("optim./code time %0.1f%%\n",
4850 (double)s->opt_time / (s->code_time ? s->code_time : 1)
4851 * 100.0);
4852 qemu_printf("liveness/code time %0.1f%%\n",
4853 (double)s->la_time / (s->code_time ? s->code_time : 1) * 100.0);
4854 qemu_printf("cpu_restore count %" PRId64 "\n",
4855 s->restore_count);
4856 qemu_printf(" avg cycles %0.1f\n",
4857 s->restore_count ? (double)s->restore_time / s->restore_count : 0);
4858}
4859#else
4860void tcg_dump_info(void)
4861{
4862 qemu_printf("[TCG profiler not compiled]\n");
4863}
4864#endif
4865
4866#ifdef ELF_HOST_MACHINE
4867
4868
4869
4870
4871
4872
4873
4874
4875
4876
4877
4878
4879
4880typedef enum {
4881 JIT_NOACTION = 0,
4882 JIT_REGISTER_FN,
4883 JIT_UNREGISTER_FN
4884} jit_actions_t;
4885
4886struct jit_code_entry {
4887 struct jit_code_entry *next_entry;
4888 struct jit_code_entry *prev_entry;
4889 const void *symfile_addr;
4890 uint64_t symfile_size;
4891};
4892
4893struct jit_descriptor {
4894 uint32_t version;
4895 uint32_t action_flag;
4896 struct jit_code_entry *relevant_entry;
4897 struct jit_code_entry *first_entry;
4898};
4899
4900void __jit_debug_register_code(void) __attribute__((noinline));
4901void __jit_debug_register_code(void)
4902{
4903 asm("");
4904}
4905
4906
4907
4908struct jit_descriptor __jit_debug_descriptor = { 1, 0, 0, 0 };
4909
4910
4911
4912static int find_string(const char *strtab, const char *str)
4913{
4914 const char *p = strtab + 1;
4915
4916 while (1) {
4917 if (strcmp(p, str) == 0) {
4918 return p - strtab;
4919 }
4920 p += strlen(p) + 1;
4921 }
4922}
4923
4924static void tcg_register_jit_int(const void *buf_ptr, size_t buf_size,
4925 const void *debug_frame,
4926 size_t debug_frame_size)
4927{
4928 struct __attribute__((packed)) DebugInfo {
4929 uint32_t len;
4930 uint16_t version;
4931 uint32_t abbrev;
4932 uint8_t ptr_size;
4933 uint8_t cu_die;
4934 uint16_t cu_lang;
4935 uintptr_t cu_low_pc;
4936 uintptr_t cu_high_pc;
4937 uint8_t fn_die;
4938 char fn_name[16];
4939 uintptr_t fn_low_pc;
4940 uintptr_t fn_high_pc;
4941 uint8_t cu_eoc;
4942 };
4943
4944 struct ElfImage {
4945 ElfW(Ehdr) ehdr;
4946 ElfW(Phdr) phdr;
4947 ElfW(Shdr) shdr[7];
4948 ElfW(Sym) sym[2];
4949 struct DebugInfo di;
4950 uint8_t da[24];
4951 char str[80];
4952 };
4953
4954 struct ElfImage *img;
4955
4956 static const struct ElfImage img_template = {
4957 .ehdr = {
4958 .e_ident[EI_MAG0] = ELFMAG0,
4959 .e_ident[EI_MAG1] = ELFMAG1,
4960 .e_ident[EI_MAG2] = ELFMAG2,
4961 .e_ident[EI_MAG3] = ELFMAG3,
4962 .e_ident[EI_CLASS] = ELF_CLASS,
4963 .e_ident[EI_DATA] = ELF_DATA,
4964 .e_ident[EI_VERSION] = EV_CURRENT,
4965 .e_type = ET_EXEC,
4966 .e_machine = ELF_HOST_MACHINE,
4967 .e_version = EV_CURRENT,
4968 .e_phoff = offsetof(struct ElfImage, phdr),
4969 .e_shoff = offsetof(struct ElfImage, shdr),
4970 .e_ehsize = sizeof(ElfW(Shdr)),
4971 .e_phentsize = sizeof(ElfW(Phdr)),
4972 .e_phnum = 1,
4973 .e_shentsize = sizeof(ElfW(Shdr)),
4974 .e_shnum = ARRAY_SIZE(img->shdr),
4975 .e_shstrndx = ARRAY_SIZE(img->shdr) - 1,
4976#ifdef ELF_HOST_FLAGS
4977 .e_flags = ELF_HOST_FLAGS,
4978#endif
4979#ifdef ELF_OSABI
4980 .e_ident[EI_OSABI] = ELF_OSABI,
4981#endif
4982 },
4983 .phdr = {
4984 .p_type = PT_LOAD,
4985 .p_flags = PF_X,
4986 },
4987 .shdr = {
4988 [0] = { .sh_type = SHT_NULL },
4989
4990
4991
4992
4993 [1] = {
4994 .sh_type = SHT_NOBITS,
4995 .sh_flags = SHF_EXECINSTR | SHF_ALLOC,
4996 },
4997 [2] = {
4998 .sh_type = SHT_PROGBITS,
4999 .sh_offset = offsetof(struct ElfImage, di),
5000 .sh_size = sizeof(struct DebugInfo),
5001 },
5002 [3] = {
5003 .sh_type = SHT_PROGBITS,
5004 .sh_offset = offsetof(struct ElfImage, da),
5005 .sh_size = sizeof(img->da),
5006 },
5007 [4] = {
5008 .sh_type = SHT_PROGBITS,
5009 .sh_offset = sizeof(struct ElfImage),
5010 },
5011 [5] = {
5012 .sh_type = SHT_SYMTAB,
5013 .sh_offset = offsetof(struct ElfImage, sym),
5014 .sh_size = sizeof(img->sym),
5015 .sh_info = 1,
5016 .sh_link = ARRAY_SIZE(img->shdr) - 1,
5017 .sh_entsize = sizeof(ElfW(Sym)),
5018 },
5019 [6] = {
5020 .sh_type = SHT_STRTAB,
5021 .sh_offset = offsetof(struct ElfImage, str),
5022 .sh_size = sizeof(img->str),
5023 }
5024 },
5025 .sym = {
5026 [1] = {
5027 .st_info = ELF_ST_INFO(STB_GLOBAL, STT_FUNC),
5028 .st_shndx = 1,
5029 }
5030 },
5031 .di = {
5032 .len = sizeof(struct DebugInfo) - 4,
5033 .version = 2,
5034 .ptr_size = sizeof(void *),
5035 .cu_die = 1,
5036 .cu_lang = 0x8001,
5037 .fn_die = 2,
5038 .fn_name = "code_gen_buffer"
5039 },
5040 .da = {
5041 1,
5042 0x11, 1,
5043 0x13, 0x5,
5044 0x11, 0x1,
5045 0x12, 0x1,
5046 0, 0,
5047 2,
5048 0x2e, 0,
5049 0x3, 0x8,
5050 0x11, 0x1,
5051 0x12, 0x1,
5052 0, 0,
5053 0
5054 },
5055 .str = "\0" ".text\0" ".debug_info\0" ".debug_abbrev\0"
5056 ".debug_frame\0" ".symtab\0" ".strtab\0" "code_gen_buffer",
5057 };
5058
5059
5060 static struct jit_code_entry one_entry;
5061
5062 uintptr_t buf = (uintptr_t)buf_ptr;
5063 size_t img_size = sizeof(struct ElfImage) + debug_frame_size;
5064 DebugFrameHeader *dfh;
5065
5066 img = g_malloc(img_size);
5067 *img = img_template;
5068
5069 img->phdr.p_vaddr = buf;
5070 img->phdr.p_paddr = buf;
5071 img->phdr.p_memsz = buf_size;
5072
5073 img->shdr[1].sh_name = find_string(img->str, ".text");
5074 img->shdr[1].sh_addr = buf;
5075 img->shdr[1].sh_size = buf_size;
5076
5077 img->shdr[2].sh_name = find_string(img->str, ".debug_info");
5078 img->shdr[3].sh_name = find_string(img->str, ".debug_abbrev");
5079
5080 img->shdr[4].sh_name = find_string(img->str, ".debug_frame");
5081 img->shdr[4].sh_size = debug_frame_size;
5082
5083 img->shdr[5].sh_name = find_string(img->str, ".symtab");
5084 img->shdr[6].sh_name = find_string(img->str, ".strtab");
5085
5086 img->sym[1].st_name = find_string(img->str, "code_gen_buffer");
5087 img->sym[1].st_value = buf;
5088 img->sym[1].st_size = buf_size;
5089
5090 img->di.cu_low_pc = buf;
5091 img->di.cu_high_pc = buf + buf_size;
5092 img->di.fn_low_pc = buf;
5093 img->di.fn_high_pc = buf + buf_size;
5094
5095 dfh = (DebugFrameHeader *)(img + 1);
5096 memcpy(dfh, debug_frame, debug_frame_size);
5097 dfh->fde.func_start = buf;
5098 dfh->fde.func_len = buf_size;
5099
5100#ifdef DEBUG_JIT
5101
5102
5103 {
5104 FILE *f = fopen("/tmp/qemu.jit", "w+b");
5105 if (f) {
5106 if (fwrite(img, img_size, 1, f) != img_size) {
5107
5108 }
5109 fclose(f);
5110 }
5111 }
5112#endif
5113
5114 one_entry.symfile_addr = img;
5115 one_entry.symfile_size = img_size;
5116
5117 __jit_debug_descriptor.action_flag = JIT_REGISTER_FN;
5118 __jit_debug_descriptor.relevant_entry = &one_entry;
5119 __jit_debug_descriptor.first_entry = &one_entry;
5120 __jit_debug_register_code();
5121}
5122#else
5123
5124
5125
5126static void tcg_register_jit_int(const void *buf, size_t size,
5127 const void *debug_frame,
5128 size_t debug_frame_size)
5129{
5130}
5131
5132void tcg_register_jit(const void *buf, size_t buf_size)
5133{
5134}
5135#endif
5136
5137#if !TCG_TARGET_MAYBE_vec
5138void tcg_expand_vec_op(TCGOpcode o, TCGType t, unsigned e, TCGArg a0, ...)
5139{
5140 g_assert_not_reached();
5141}
5142#endif
5143