/*
 * Tiny Code Generator for QEMU
 *
 * Copyright (c) 2008 Fabrice Bellard
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 */

#include "qemu/osdep.h"

/* Define to jump the ELF file used to communicate with GDB.  */
#undef DEBUG_JIT

#include "qemu/error-report.h"
#include "qemu/cutils.h"
#include "qemu/host-utils.h"
#include "qemu/qemu-print.h"
#include "qemu/cacheflush.h"
#include "qemu/cacheinfo.h"
#include "qemu/timer.h"
#include "exec/translation-block.h"
#include "exec/tlb-common.h"
#include "tcg/tcg-op-common.h"

#if UINTPTR_MAX == UINT32_MAX
# define ELF_CLASS  ELFCLASS32
#else
# define ELF_CLASS  ELFCLASS64
#endif
#if HOST_BIG_ENDIAN
# define ELF_DATA   ELFDATA2MSB
#else
# define ELF_DATA   ELFDATA2LSB
#endif

#include "elf.h"
#include "exec/log.h"
#include "tcg/tcg-ldst.h"
#include "tcg/tcg-temp-internal.h"
#include "tcg-internal.h"
#include "accel/tcg/perf.h"
#ifdef CONFIG_USER_ONLY
#include "exec/user/guest-base.h"
#endif

/* Forward declarations for functions declared in tcg-target.c.inc and
   used here. */
static void tcg_target_init(TCGContext *s);
static void tcg_target_qemu_prologue(TCGContext *s);
static bool patch_reloc(tcg_insn_unit *code_ptr, int type,
                        intptr_t value, intptr_t addend);

/* The CIE and FDE header definitions will be common to all hosts.  */
typedef struct {
    uint32_t len __attribute__((aligned((sizeof(void *)))));
    uint32_t id;
    uint8_t version;
    char augmentation[1];
    uint8_t code_align;
    uint8_t data_align;
    uint8_t return_column;
} DebugFrameCIE;

typedef struct QEMU_PACKED {
    uint32_t len __attribute__((aligned((sizeof(void *)))));
    uint32_t cie_offset;
    uintptr_t func_start;
    uintptr_t func_len;
} DebugFrameFDEHeader;

typedef struct QEMU_PACKED {
    DebugFrameCIE cie;
    DebugFrameFDEHeader fde;
} DebugFrameHeader;

typedef struct TCGLabelQemuLdst {
    bool is_ld;             /* qemu_ld: true, qemu_st: false */
    MemOpIdx oi;
    TCGType type;           /* result type of a load */
    TCGReg addrlo_reg;      /* reg index for low word of guest virtual addr */
    TCGReg addrhi_reg;      /* reg index for high word of guest virtual addr */
    TCGReg datalo_reg;      /* reg index for low word to be loaded or stored */
    TCGReg datahi_reg;      /* reg index for high word to be loaded or stored */
    const tcg_insn_unit *raddr;   /* addr of the next IR of qemu_ld/st IR */
    tcg_insn_unit *label_ptr[2]; /* label pointers to be updated */
    QSIMPLEQ_ENTRY(TCGLabelQemuLdst) next;
} TCGLabelQemuLdst;

static void tcg_register_jit_int(const void *buf, size_t size,
                                 const void *debug_frame,
                                 size_t debug_frame_size)
    __attribute__((unused));

/* Forward declarations for functions declared and used in tcg-target.c.inc. */
static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1,
                       intptr_t arg2);
static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
static void tcg_out_movi(TCGContext *s, TCGType type,
                         TCGReg ret, tcg_target_long arg);
static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
static void tcg_out_ext16s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
static void tcg_out_ext8u(TCGContext *s, TCGReg ret, TCGReg arg);
static void tcg_out_ext16u(TCGContext *s, TCGReg ret, TCGReg arg);
static void tcg_out_ext32s(TCGContext *s, TCGReg ret, TCGReg arg);
static void tcg_out_ext32u(TCGContext *s, TCGReg ret, TCGReg arg);
static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg ret, TCGReg arg);
static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg ret, TCGReg arg);
static void tcg_out_extrl_i64_i32(TCGContext *s, TCGReg ret, TCGReg arg);
static void tcg_out_addi_ptr(TCGContext *s, TCGReg, TCGReg, tcg_target_long);
static bool tcg_out_xchg(TCGContext *s, TCGType type, TCGReg r1, TCGReg r2);
static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg);
static void tcg_out_goto_tb(TCGContext *s, int which);
static void tcg_out_op(TCGContext *s, TCGOpcode opc,
                       const TCGArg args[TCG_MAX_OP_ARGS],
                       const int const_args[TCG_MAX_OP_ARGS]);
#if TCG_TARGET_MAYBE_vec
static bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece,
                            TCGReg dst, TCGReg src);
static bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece,
                             TCGReg dst, TCGReg base, intptr_t offset);
static void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece,
                             TCGReg dst, int64_t arg);
static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc,
                           unsigned vecl, unsigned vece,
                           const TCGArg args[TCG_MAX_OP_ARGS],
                           const int const_args[TCG_MAX_OP_ARGS]);
#else
static inline bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece,
                                   TCGReg dst, TCGReg src)
{
    g_assert_not_reached();
}
static inline bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece,
                                    TCGReg dst, TCGReg base, intptr_t offset)
{
    g_assert_not_reached();
}
static inline void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece,
                                    TCGReg dst, int64_t arg)
{
    g_assert_not_reached();
}
static inline void tcg_out_vec_op(TCGContext *s, TCGOpcode opc,
                                  unsigned vecl, unsigned vece,
                                  const TCGArg args[TCG_MAX_OP_ARGS],
                                  const int const_args[TCG_MAX_OP_ARGS])
{
    g_assert_not_reached();
}
#endif
static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1,
                       intptr_t arg2);
static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
                        TCGReg base, intptr_t ofs);
static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target,
                         const TCGHelperInfo *info);
static TCGReg tcg_target_call_oarg_reg(TCGCallReturnKind kind, int slot);
static bool tcg_target_const_match(int64_t val, TCGType type, int ct);
#ifdef TCG_TARGET_NEED_LDST_LABELS
static int tcg_out_ldst_finalize(TCGContext *s);
#endif

typedef struct TCGLdstHelperParam {
    TCGReg (*ra_gen)(TCGContext *s, const TCGLabelQemuLdst *l, int arg_reg);
    unsigned ntmp;
    int tmp[3];
} TCGLdstHelperParam;

static void tcg_out_ld_helper_args(TCGContext *s, const TCGLabelQemuLdst *l,
                                   const TCGLdstHelperParam *p)
    __attribute__((unused));
static void tcg_out_ld_helper_ret(TCGContext *s, const TCGLabelQemuLdst *l,
                                  bool load_sign, const TCGLdstHelperParam *p)
    __attribute__((unused));
static void tcg_out_st_helper_args(TCGContext *s, const TCGLabelQemuLdst *l,
                                   const TCGLdstHelperParam *p)
    __attribute__((unused));

static void * const qemu_ld_helpers[MO_SSIZE + 1] __attribute__((unused)) = {
    [MO_UB] = helper_ldub_mmu,
    [MO_SB] = helper_ldsb_mmu,
    [MO_UW] = helper_lduw_mmu,
    [MO_SW] = helper_ldsw_mmu,
    [MO_UL] = helper_ldul_mmu,
    [MO_UQ] = helper_ldq_mmu,
#if TCG_TARGET_REG_BITS == 64
    [MO_SL] = helper_ldsl_mmu,
    [MO_128] = helper_ld16_mmu,
#endif
};

static void * const qemu_st_helpers[MO_SIZE + 1] __attribute__((unused)) = {
    [MO_8]  = helper_stb_mmu,
    [MO_16] = helper_stw_mmu,
    [MO_32] = helper_stl_mmu,
    [MO_64] = helper_stq_mmu,
#if TCG_TARGET_REG_BITS == 64
    [MO_128] = helper_st16_mmu,
#endif
};

typedef struct {
    MemOp atom;   /* lg2 bits of atomicity required */
    MemOp align;  /* lg2 bits of alignment to use */
} TCGAtomAlign;

static TCGAtomAlign atom_and_align_for_opc(TCGContext *s, MemOp opc,
                                           MemOp host_atom, bool allow_two_ops)
    __attribute__((unused));

TCGContext tcg_init_ctx;
__thread TCGContext *tcg_ctx;

TCGContext **tcg_ctxs;
unsigned int tcg_cur_ctxs;
unsigned int tcg_max_ctxs;
TCGv_env cpu_env = 0;
const void *tcg_code_gen_epilogue;
uintptr_t tcg_splitwx_diff;

#ifndef CONFIG_TCG_INTERPRETER
tcg_prologue_fn *tcg_qemu_tb_exec;
#endif

static TCGRegSet tcg_target_available_regs[TCG_TYPE_COUNT];
static TCGRegSet tcg_target_call_clobber_regs;

#if TCG_TARGET_INSN_UNIT_SIZE == 1
static __attribute__((unused)) inline void tcg_out8(TCGContext *s, uint8_t v)
{
    *s->code_ptr++ = v;
}

static __attribute__((unused)) inline void tcg_patch8(tcg_insn_unit *p,
                                                      uint8_t v)
{
    *p = v;
}
#endif

#if TCG_TARGET_INSN_UNIT_SIZE <= 2
static __attribute__((unused)) inline void tcg_out16(TCGContext *s, uint16_t v)
{
    if (TCG_TARGET_INSN_UNIT_SIZE == 2) {
        *s->code_ptr++ = v;
    } else {
        tcg_insn_unit *p = s->code_ptr;
        memcpy(p, &v, sizeof(v));
        s->code_ptr = p + (2 / TCG_TARGET_INSN_UNIT_SIZE);
    }
}

static __attribute__((unused)) inline void tcg_patch16(tcg_insn_unit *p,
                                                       uint16_t v)
{
    if (TCG_TARGET_INSN_UNIT_SIZE == 2) {
        *p = v;
    } else {
        memcpy(p, &v, sizeof(v));
    }
}
#endif

#if TCG_TARGET_INSN_UNIT_SIZE <= 4
static __attribute__((unused)) inline void tcg_out32(TCGContext *s, uint32_t v)
{
    if (TCG_TARGET_INSN_UNIT_SIZE == 4) {
        *s->code_ptr++ = v;
    } else {
        tcg_insn_unit *p = s->code_ptr;
        memcpy(p, &v, sizeof(v));
        s->code_ptr = p + (4 / TCG_TARGET_INSN_UNIT_SIZE);
    }
}

static __attribute__((unused)) inline void tcg_patch32(tcg_insn_unit *p,
                                                       uint32_t v)
{
    if (TCG_TARGET_INSN_UNIT_SIZE == 4) {
        *p = v;
    } else {
        memcpy(p, &v, sizeof(v));
    }
}
#endif

#if TCG_TARGET_INSN_UNIT_SIZE <= 8
static __attribute__((unused)) inline void tcg_out64(TCGContext *s, uint64_t v)
{
    if (TCG_TARGET_INSN_UNIT_SIZE == 8) {
        *s->code_ptr++ = v;
    } else {
        tcg_insn_unit *p = s->code_ptr;
        memcpy(p, &v, sizeof(v));
        s->code_ptr = p + (8 / TCG_TARGET_INSN_UNIT_SIZE);
    }
}

static __attribute__((unused)) inline void tcg_patch64(tcg_insn_unit *p,
                                                       uint64_t v)
{
    if (TCG_TARGET_INSN_UNIT_SIZE == 8) {
        *p = v;
    } else {
        memcpy(p, &v, sizeof(v));
    }
}
#endif

/* label relocation processing */

static void tcg_out_reloc(TCGContext *s, tcg_insn_unit *code_ptr, int type,
                          TCGLabel *l, intptr_t addend)
{
    TCGRelocation *r = tcg_malloc(sizeof(TCGRelocation));

    r->type = type;
    r->ptr = code_ptr;
    r->addend = addend;
    QSIMPLEQ_INSERT_TAIL(&l->relocs, r, next);
}

static void tcg_out_label(TCGContext *s, TCGLabel *l)
{
    tcg_debug_assert(!l->has_value);
    l->has_value = 1;
    l->u.value_ptr = tcg_splitwx_to_rx(s->code_ptr);
}

TCGLabel *gen_new_label(void)
{
    TCGContext *s = tcg_ctx;
    TCGLabel *l = tcg_malloc(sizeof(TCGLabel));

    memset(l, 0, sizeof(TCGLabel));
    l->id = s->nb_labels++;
    QSIMPLEQ_INIT(&l->branches);
    QSIMPLEQ_INIT(&l->relocs);

    QSIMPLEQ_INSERT_TAIL(&s->labels, l, next);

    return l;
}

static bool tcg_resolve_relocs(TCGContext *s)
{
    TCGLabel *l;

    QSIMPLEQ_FOREACH(l, &s->labels, next) {
        TCGRelocation *r;
        uintptr_t value = l->u.value;

        QSIMPLEQ_FOREACH(r, &l->relocs, next) {
            if (!patch_reloc(r->ptr, r->type, value, r->addend)) {
                return false;
            }
        }
    }
    return true;
}

static void set_jmp_reset_offset(TCGContext *s, int which)
{
    /*
     * We will check for overflow at the end of the opcode loop in
     * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX.
     */
    s->gen_tb->jmp_reset_offset[which] = tcg_current_code_size(s);
}

static void G_GNUC_UNUSED set_jmp_insn_offset(TCGContext *s, int which)
{
    /*
     * We will check for overflow at the end of the opcode loop in
     * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX.
     */
    s->gen_tb->jmp_insn_offset[which] = tcg_current_code_size(s);
}

static uintptr_t G_GNUC_UNUSED get_jmp_target_addr(TCGContext *s, int which)
{
    /*
     * Return the read-execute version of the pointer, for the benefit
     * of any pc-relative addressing mode.
     */
    return (uintptr_t)tcg_splitwx_to_rx(&s->gen_tb->jmp_target_addr[which]);
}

#if defined(CONFIG_SOFTMMU) && !defined(CONFIG_TCG_INTERPRETER)
static int tlb_mask_table_ofs(TCGContext *s, int which)
{
    return s->tlb_fast_offset + which * sizeof(CPUTLBDescFast);
}
#endif

/* Signal overflow, starting over with fewer guest insns. */
static G_NORETURN
void tcg_raise_tb_overflow(TCGContext *s)
{
    siglongjmp(s->jmp_trans, -2);
}

/*
 * Used by tcg_out_movext{1,2} to hold the arguments for tcg_out_movext.
 * By the time we arrive at tcg_out_movext1, @dst is always a TCGReg.
 *
 * However, tcg_out_helper_load_slots reuses this field to hold an
 * argument slot number (which may designate a argument register or an
 * argument stack slot), converting to TCGReg once all arguments that
 * are destined for the stack are processed.
 */
typedef struct TCGMovExtend {
    unsigned dst;
    TCGReg src;
    TCGType dst_type;
    TCGType src_type;
    MemOp src_ext;
} TCGMovExtend;

/**
 * tcg_out_movext -- move and extend
 * @s: tcg context
 * @dst_type: integral type for destination
 * @dst: destination register
 * @src_type: integral type for source
 * @src_ext: extension to apply to source
 * @src: source register
 *
 * Move or extend @src into @dst, depending on @src_ext and the types.
 */
static void tcg_out_movext(TCGContext *s, TCGType dst_type, TCGReg dst,
                           TCGType src_type, MemOp src_ext, TCGReg src)
{
    switch (src_ext) {
    case MO_UB:
        tcg_out_ext8u(s, dst, src);
        break;
    case MO_SB:
        tcg_out_ext8s(s, dst_type, dst, src);
        break;
    case MO_UW:
        tcg_out_ext16u(s, dst, src);
        break;
    case MO_SW:
        tcg_out_ext16s(s, dst_type, dst, src);
        break;
    case MO_UL:
    case MO_SL:
        if (dst_type == TCG_TYPE_I32) {
            if (src_type == TCG_TYPE_I32) {
                tcg_out_mov(s, TCG_TYPE_I32, dst, src);
            } else {
                tcg_out_extrl_i64_i32(s, dst, src);
            }
        } else if (src_type == TCG_TYPE_I32) {
            if (src_ext & MO_SIGN) {
                tcg_out_exts_i32_i64(s, dst, src);
            } else {
                tcg_out_extu_i32_i64(s, dst, src);
            }
        } else {
            if (src_ext & MO_SIGN) {
                tcg_out_ext32s(s, dst, src);
            } else {
                tcg_out_ext32u(s, dst, src);
            }
        }
        break;
    case MO_UQ:
        tcg_debug_assert(TCG_TARGET_REG_BITS == 64);
        if (dst_type == TCG_TYPE_I32) {
            tcg_out_extrl_i64_i32(s, dst, src);
        } else {
            tcg_out_mov(s, TCG_TYPE_I64, dst, src);
        }
        break;
    default:
        g_assert_not_reached();
    }
}

/* Minor variations on a theme, using a structure. */
static void tcg_out_movext1_new_src(TCGContext *s, const TCGMovExtend *i,
                                    TCGReg src)
{
    tcg_out_movext(s, i->dst_type, i->dst, i->src_type, i->src_ext, src);
}

static void tcg_out_movext1(TCGContext *s, const TCGMovExtend *i)
{
    tcg_out_movext1_new_src(s, i, i->src);
}

/**
 * tcg_out_movext2 -- move and extend two pair
 * @s: tcg context
 * @i1: first move description
 * @i2: second move description
 * @scratch: temporary register, or -1 for none
 *
 * As tcg_out_movext, for both @i1 and @i2, caring for overlap
 * between the sources and destinations.
 */

static void tcg_out_movext2(TCGContext *s, const TCGMovExtend *i1,
                            const TCGMovExtend *i2, int scratch)
{
    TCGReg src1 = i1->src;
    TCGReg src2 = i2->src;

    if (i1->dst != src2) {
        tcg_out_movext1(s, i1);
        tcg_out_movext1(s, i2);
        return;
    }
    if (i2->dst == src1) {
        TCGType src1_type = i1->src_type;
        TCGType src2_type = i2->src_type;

        if (tcg_out_xchg(s, MAX(src1_type, src2_type), src1, src2)) {
            /* The data is now in the correct registers, now extend. */
            src1 = i2->src;
            src2 = i1->src;
        } else {
            tcg_debug_assert(scratch >= 0);
            tcg_out_mov(s, src1_type, scratch, src1);
            src1 = scratch;
        }
    }
    tcg_out_movext1_new_src(s, i2, src2);
    tcg_out_movext1_new_src(s, i1, src1);
}

/**
 * tcg_out_movext3 -- move and extend three pair
 * @s: tcg context
 * @i1: first move description
 * @i2: second move description
 * @i3: third move description
 * @scratch: temporary register, or -1 for none
 *
 * As tcg_out_movext, for all of @i1, @i2 and @i3, caring for overlap
 * between the sources and destinations.
 */

static void tcg_out_movext3(TCGContext *s, const TCGMovExtend *i1,
                            const TCGMovExtend *i2, const TCGMovExtend *i3,
                            int scratch)
{
    TCGReg src1 = i1->src;
    TCGReg src2 = i2->src;
    TCGReg src3 = i3->src;

    if (i1->dst != src2 && i1->dst != src3) {
        tcg_out_movext1(s, i1);
        tcg_out_movext2(s, i2, i3, scratch);
        return;
    }
    if (i2->dst != src1 && i2->dst != src3) {
        tcg_out_movext1(s, i2);
        tcg_out_movext2(s, i1, i3, scratch);
        return;
    }
    if (i3->dst != src1 && i3->dst != src2) {
        tcg_out_movext1(s, i3);
        tcg_out_movext2(s, i1, i2, scratch);
        return;
    }

    /*
     * There is a cycle.  Since there are only 3 nodes, the cycle is
     * either "clockwise" or "anti-clockwise", and can be solved with
     * a single scratch or two xchg.
     */
    if (i1->dst == src2 && i2->dst == src3 && i3->dst == src1) {
        /* "Clockwise" */
        if (tcg_out_xchg(s, MAX(i1->src_type, i2->src_type), src1, src2)) {
            tcg_out_xchg(s, MAX(i2->src_type, i3->src_type), src2, src3);
            /* The data is now in the correct registers, now extend. */
            tcg_out_movext1_new_src(s, i1, i1->dst);
            tcg_out_movext1_new_src(s, i2, i2->dst);
            tcg_out_movext1_new_src(s, i3, i3->dst);
        } else {
            tcg_debug_assert(scratch >= 0);
            tcg_out_mov(s, i1->src_type, scratch, src1);
            tcg_out_movext1(s, i3);
            tcg_out_movext1(s, i2);
            tcg_out_movext1_new_src(s, i1, scratch);
        }
    } else if (i1->dst == src3 && i2->dst == src1 && i3->dst == src2) {
        /* "Anti-clockwise" */
        if (tcg_out_xchg(s, MAX(i2->src_type, i3->src_type), src2, src3)) {
            tcg_out_xchg(s, MAX(i1->src_type, i2->src_type), src1, src2);
            /* The data is now in the correct registers, now extend. */
            tcg_out_movext1_new_src(s, i1, i1->dst);
            tcg_out_movext1_new_src(s, i2, i2->dst);
            tcg_out_movext1_new_src(s, i3, i3->dst);
        } else {
            tcg_debug_assert(scratch >= 0);
            tcg_out_mov(s, i1->src_type, scratch, src1);
            tcg_out_movext1(s, i2);
            tcg_out_movext1(s, i3);
            tcg_out_movext1_new_src(s, i1, scratch);
        }
    } else {
        g_assert_not_reached();
    }
}

#define C_PFX1(P, A)                    P##A
#define C_PFX2(P, A, B)                 P##A##_##B
#define C_PFX3(P, A, B, C)              P##A##_##B##_##C
#define C_PFX4(P, A, B, C, D)           P##A##_##B##_##C##_##D
#define C_PFX5(P, A, B, C, D, E)        P##A##_##B##_##C##_##D##_##E
#define C_PFX6(P, A, B, C, D, E, F)     P##A##_##B##_##C##_##D##_##E##_##F

/* Define an enumeration for the various combinations. */

#define C_O0_I1(I1)                     C_PFX1(c_o0_i1_, I1),
#define C_O0_I2(I1, I2)                 C_PFX2(c_o0_i2_, I1, I2),
#define C_O0_I3(I1, I2, I3)             C_PFX3(c_o0_i3_, I1, I2, I3),
#define C_O0_I4(I1, I2, I3, I4)         C_PFX4(c_o0_i4_, I1, I2, I3, I4),

#define C_O1_I1(O1, I1)                 C_PFX2(c_o1_i1_, O1, I1),
#define C_O1_I2(O1, I1, I2)             C_PFX3(c_o1_i2_, O1, I1, I2),
#define C_O1_I3(O1, I1, I2, I3)         C_PFX4(c_o1_i3_, O1, I1, I2, I3),
#define C_O1_I4(O1, I1, I2, I3, I4)     C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4),

#define C_N1_I2(O1, I1, I2)             C_PFX3(c_n1_i2_, O1, I1, I2),

#define C_O2_I1(O1, O2, I1)             C_PFX3(c_o2_i1_, O1, O2, I1),
#define C_O2_I2(O1, O2, I1, I2)         C_PFX4(c_o2_i2_, O1, O2, I1, I2),
#define C_O2_I3(O1, O2, I1, I2, I3)     C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3),
#define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4),
#define C_N1_O1_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_n1_o1_i4_, O1, O2, I1, I2, I3, I4),

typedef enum {
#include "tcg-target-con-set.h"
} TCGConstraintSetIndex;

static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode);

#undef C_O0_I1
#undef C_O0_I2
#undef C_O0_I3
#undef C_O0_I4
#undef C_O1_I1
#undef C_O1_I2
#undef C_O1_I3
#undef C_O1_I4
#undef C_N1_I2
#undef C_O2_I1
#undef C_O2_I2
#undef C_O2_I3
#undef C_O2_I4
#undef C_N1_O1_I4

/* Put all of the constraint sets into an array, indexed by the enum. */

#define C_O0_I1(I1)                     { .args_ct_str = { #I1 } },
#define C_O0_I2(I1, I2)                 { .args_ct_str = { #I1, #I2 } },
#define C_O0_I3(I1, I2, I3)             { .args_ct_str = { #I1, #I2, #I3 } },
#define C_O0_I4(I1, I2, I3, I4)         { .args_ct_str = { #I1, #I2, #I3, #I4 } },

#define C_O1_I1(O1, I1)                 { .args_ct_str = { #O1, #I1 } },
#define C_O1_I2(O1, I1, I2)             { .args_ct_str = { #O1, #I1, #I2 } },
#define C_O1_I3(O1, I1, I2, I3)         { .args_ct_str = { #O1, #I1, #I2, #I3 } },
#define C_O1_I4(O1, I1, I2, I3, I4)     { .args_ct_str = { #O1, #I1, #I2, #I3, #I4 } },

#define C_N1_I2(O1, I1, I2)             { .args_ct_str = { "&" #O1, #I1, #I2 } },

#define C_O2_I1(O1, O2, I1)             { .args_ct_str = { #O1, #O2, #I1 } },
#define C_O2_I2(O1, O2, I1, I2)         { .args_ct_str = { #O1, #O2, #I1, #I2 } },
#define C_O2_I3(O1, O2, I1, I2, I3)     { .args_ct_str = { #O1, #O2, #I1, #I2, #I3 } },
#define C_O2_I4(O1, O2, I1, I2, I3, I4) { .args_ct_str = { #O1, #O2, #I1, #I2, #I3, #I4 } },
#define C_N1_O1_I4(O1, O2, I1, I2, I3, I4) { .args_ct_str = { "&" #O1, #O2, #I1, #I2, #I3, #I4 } },

static const TCGTargetOpDef constraint_sets[] = {
#include "tcg-target-con-set.h"
};


#undef C_O0_I1
#undef C_O0_I2
#undef C_O0_I3
#undef C_O0_I4
#undef C_O1_I1
#undef C_O1_I2
#undef C_O1_I3
#undef C_O1_I4
#undef C_N1_I2
#undef C_O2_I1
#undef C_O2_I2
#undef C_O2_I3
#undef C_O2_I4
#undef C_N1_O1_I4

/* Expand the enumerator to be returned from tcg_target_op_def(). */

#define C_O0_I1(I1)                     C_PFX1(c_o0_i1_, I1)
#define C_O0_I2(I1, I2)                 C_PFX2(c_o0_i2_, I1, I2)
#define C_O0_I3(I1, I2, I3)             C_PFX3(c_o0_i3_, I1, I2, I3)
#define C_O0_I4(I1, I2, I3, I4)         C_PFX4(c_o0_i4_, I1, I2, I3, I4)

#define C_O1_I1(O1, I1)                 C_PFX2(c_o1_i1_, O1, I1)
#define C_O1_I2(O1, I1, I2)             C_PFX3(c_o1_i2_, O1, I1, I2)
#define C_O1_I3(O1, I1, I2, I3)         C_PFX4(c_o1_i3_, O1, I1, I2, I3)
#define C_O1_I4(O1, I1, I2, I3, I4)     C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4)

#define C_N1_I2(O1, I1, I2)             C_PFX3(c_n1_i2_, O1, I1, I2)

#define C_O2_I1(O1, O2, I1)             C_PFX3(c_o2_i1_, O1, O2, I1)
#define C_O2_I2(O1, O2, I1, I2)         C_PFX4(c_o2_i2_, O1, O2, I1, I2)
#define C_O2_I3(O1, O2, I1, I2, I3)     C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3)
#define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4)
#define C_N1_O1_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_n1_o1_i4_, O1, O2, I1, I2, I3, I4)

#include "tcg-target.c.inc"

static void alloc_tcg_plugin_context(TCGContext *s)
{
#ifdef CONFIG_PLUGIN
    s->plugin_tb = g_new0(struct qemu_plugin_tb, 1);
    s->plugin_tb->insns =
        g_ptr_array_new_with_free_func(qemu_plugin_insn_cleanup_fn);
#endif
}

/*
 * All TCG threads except the parent (i.e. the one that called tcg_context_init
 * and registered the target's TCG globals) must register with this function
 * before initiating translation.
 *
 * In user-mode we just point tcg_ctx to tcg_init_ctx. See the documentation
 * of tcg_region_init() for the reasoning behind this.
 *
 * In softmmu each caller registers its context in tcg_ctxs[]. Note that in
 * softmmu tcg_ctxs[] does not track tcg_ctx_init, since the initial context
 * is not used anymore for translation once this function is called.
 *
 * Not tracking tcg_init_ctx in tcg_ctxs[] in softmmu keeps code that iterates
 * over the array (e.g. tcg_code_size() the same for both softmmu and user-mode.
 */
#ifdef CONFIG_USER_ONLY
void tcg_register_thread(void)
{
    tcg_ctx = &tcg_init_ctx;
}
#else
void tcg_register_thread(void)
{
    TCGContext *s = g_malloc(sizeof(*s));
    unsigned int i, n;

    *s = tcg_init_ctx;

    /* Relink mem_base.  */
    for (i = 0, n = tcg_init_ctx.nb_globals; i < n; ++i) {
        if (tcg_init_ctx.temps[i].mem_base) {
            ptrdiff_t b = tcg_init_ctx.temps[i].mem_base - tcg_init_ctx.temps;
            tcg_debug_assert(b >= 0 && b < n);
            s->temps[i].mem_base = &s->temps[b];
        }
    }

    /* Claim an entry in tcg_ctxs */
    n = qatomic_fetch_inc(&tcg_cur_ctxs);
    g_assert(n < tcg_max_ctxs);
    qatomic_set(&tcg_ctxs[n], s);

    if (n > 0) {
        alloc_tcg_plugin_context(s);
        tcg_region_initial_alloc(s);
    }

    tcg_ctx = s;
}
#endif /* !CONFIG_USER_ONLY */

/* pool based memory allocation */
void *tcg_malloc_internal(TCGContext *s, int size)
{
    TCGPool *p;
    int pool_size;

    if (size > TCG_POOL_CHUNK_SIZE) {
        /* big malloc: insert a new pool (XXX: could optimize) */
        p = g_malloc(sizeof(TCGPool) + size);
        p->size = size;
        p->next = s->pool_first_large;
        s->pool_first_large = p;
        return p->data;
    } else {
        p = s->pool_current;
        if (!p) {
            p = s->pool_first;
            if (!p)
                goto new_pool;
        } else {
            if (!p->next) {
            new_pool:
                pool_size = TCG_POOL_CHUNK_SIZE;
                p = g_malloc(sizeof(TCGPool) + pool_size);
                p->size = pool_size;
                p->next = NULL;
                if (s->pool_current) {
                    s->pool_current->next = p;
                } else {
                    s->pool_first = p;
                }
            } else {
                p = p->next;
            }
        }
    }
    s->pool_current = p;
    s->pool_cur = p->data + size;
    s->pool_end = p->data + p->size;
    return p->data;
}

void tcg_pool_reset(TCGContext *s)
{
    TCGPool *p, *t;
    for (p = s->pool_first_large; p; p = t) {
        t = p->next;
        g_free(p);
    }
    s->pool_first_large = NULL;
    s->pool_cur = s->pool_end = NULL;
    s->pool_current = NULL;
}

/*
 * Create TCGHelperInfo structures for "tcg/tcg-ldst.h" functions,
 * akin to what "exec/helper-tcg.h" does with DEF_HELPER_FLAGS_N.
 * We only use these for layout in tcg_out_ld_helper_ret and
 * tcg_out_st_helper_args, and share them between several of
 * the helpers, with the end result that it's easier to build manually.
 */

#if TCG_TARGET_REG_BITS == 32
# define dh_typecode_ttl  dh_typecode_i32
#else
# define dh_typecode_ttl  dh_typecode_i64
#endif

static TCGHelperInfo info_helper_ld32_mmu = {
    .flags = TCG_CALL_NO_WG,
    .typemask = dh_typemask(ttl, 0)  /* return tcg_target_ulong */
              | dh_typemask(env, 1)
              | dh_typemask(i64, 2)  /* uint64_t addr */
              | dh_typemask(i32, 3)  /* unsigned oi */
              | dh_typemask(ptr, 4)  /* uintptr_t ra */
};

static TCGHelperInfo info_helper_ld64_mmu = {
    .flags = TCG_CALL_NO_WG,
    .typemask = dh_typemask(i64, 0)  /* return uint64_t */
              | dh_typemask(env, 1)
              | dh_typemask(i64, 2)  /* uint64_t addr */
              | dh_typemask(i32, 3)  /* unsigned oi */
              | dh_typemask(ptr, 4)  /* uintptr_t ra */
};

static TCGHelperInfo info_helper_ld128_mmu = {
    .flags = TCG_CALL_NO_WG,
    .typemask = dh_typemask(i128, 0) /* return Int128 */
              | dh_typemask(env, 1)
              | dh_typemask(i64, 2)  /* uint64_t addr */
              | dh_typemask(i32, 3)  /* unsigned oi */
              | dh_typemask(ptr, 4)  /* uintptr_t ra */
};

static TCGHelperInfo info_helper_st32_mmu = {
    .flags = TCG_CALL_NO_WG,
    .typemask = dh_typemask(void, 0)
              | dh_typemask(env, 1)
              | dh_typemask(i64, 2)  /* uint64_t addr */
              | dh_typemask(i32, 3)  /* uint32_t data */
              | dh_typemask(i32, 4)  /* unsigned oi */
              | dh_typemask(ptr, 5)  /* uintptr_t ra */
};

static TCGHelperInfo info_helper_st64_mmu = {
    .flags = TCG_CALL_NO_WG,
    .typemask = dh_typemask(void, 0)
              | dh_typemask(env, 1)
              | dh_typemask(i64, 2)  /* uint64_t addr */
              | dh_typemask(i64, 3)  /* uint64_t data */
              | dh_typemask(i32, 4)  /* unsigned oi */
              | dh_typemask(ptr, 5)  /* uintptr_t ra */
};

static TCGHelperInfo info_helper_st128_mmu = {
    .flags = TCG_CALL_NO_WG,
    .typemask = dh_typemask(void, 0)
              | dh_typemask(env, 1)
              | dh_typemask(i64, 2)  /* uint64_t addr */
              | dh_typemask(i128, 3) /* Int128 data */
              | dh_typemask(i32, 4)  /* unsigned oi */
              | dh_typemask(ptr, 5)  /* uintptr_t ra */
};

#ifdef CONFIG_TCG_INTERPRETER
static ffi_type *typecode_to_ffi(int argmask)
{
    /*
     * libffi does not support __int128_t, so we have forced Int128
     * to use the structure definition instead of the builtin type.
     */
    static ffi_type *ffi_type_i128_elements[3] = {
        &ffi_type_uint64,
        &ffi_type_uint64,
        NULL
    };
    static ffi_type ffi_type_i128 = {
        .size = 16,
        .alignment = __alignof__(Int128),
        .type = FFI_TYPE_STRUCT,
        .elements = ffi_type_i128_elements,
    };

    switch (argmask) {
    case dh_typecode_void:
        return &ffi_type_void;
    case dh_typecode_i32:
        return &ffi_type_uint32;
    case dh_typecode_s32:
        return &ffi_type_sint32;
    case dh_typecode_i64:
        return &ffi_type_uint64;
    case dh_typecode_s64:
        return &ffi_type_sint64;
    case dh_typecode_ptr:
        return &ffi_type_pointer;
    case dh_typecode_i128:
        return &ffi_type_i128;
    }
    g_assert_not_reached();
}

static ffi_cif *init_ffi_layout(TCGHelperInfo *info)
{
    unsigned typemask = info->typemask;
    struct {
        ffi_cif cif;
        ffi_type *args[];
    } *ca;
    ffi_status status;
    int nargs;

    /* Ignoring the return type, find the last non-zero field. */
    nargs = 32 - clz32(typemask >> 3);
    nargs = DIV_ROUND_UP(nargs, 3);
    assert(nargs <= MAX_CALL_IARGS);

    ca = g_malloc0(sizeof(*ca) + nargs * sizeof(ffi_type *));
    ca->cif.rtype = typecode_to_ffi(typemask & 7);
    ca->cif.nargs = nargs;

    if (nargs != 0) {
        ca->cif.arg_types = ca->args;
        for (int j = 0; j < nargs; ++j) {
            int typecode = extract32(typemask, (j + 1) * 3, 3);
            ca->args[j] = typecode_to_ffi(typecode);
        }
    }

    status = ffi_prep_cif(&ca->cif, FFI_DEFAULT_ABI, nargs,
                          ca->cif.rtype, ca->cif.arg_types);
    assert(status == FFI_OK);

    return &ca->cif;
}

#define HELPER_INFO_INIT(I)      (&(I)->cif)
#define HELPER_INFO_INIT_VAL(I)  init_ffi_layout(I)
#else
#define HELPER_INFO_INIT(I)      (&(I)->init)
#define HELPER_INFO_INIT_VAL(I)  1
#endif /* CONFIG_TCG_INTERPRETER */

static inline bool arg_slot_reg_p(unsigned arg_slot)

{
    /*
     * Split the sizeof away from the comparison to avoid Werror from
     * "unsigned < 0 is always false", when iarg_regs is empty.
     */
    unsigned nreg = ARRAY_SIZE(tcg_target_call_iarg_regs);
    return arg_slot < nreg;
}

static inline int arg_slot_stk_ofs(unsigned arg_slot)
{
    unsigned max = TCG_STATIC_CALL_ARGS_SIZE / sizeof(tcg_target_long);
    unsigned stk_slot = arg_slot - ARRAY_SIZE(tcg_target_call_iarg_regs);

    tcg_debug_assert(stk_slot < max);
    return TCG_TARGET_CALL_STACK_OFFSET + stk_slot * sizeof(tcg_target_long);
}

typedef struct TCGCumulativeArgs {
    int arg_idx;                /* tcg_gen_callN args[] */
    int info_in_idx;            /* TCGHelperInfo in[] */
    int arg_slot;               /* regs+stack slot */
    int ref_slot;               /* stack slots for references */
} TCGCumulativeArgs;

static void layout_arg_even(TCGCumulativeArgs *cum)
{
    cum->arg_slot += cum->arg_slot & 1;
}

static void layout_arg_1(TCGCumulativeArgs *cum, TCGHelperInfo *info,
                         TCGCallArgumentKind kind)
{
    TCGCallArgumentLoc *loc = &info->in[cum->info_in_idx];

    *loc = (TCGCallArgumentLoc){
        .kind = kind,
        .arg_idx = cum->arg_idx,
        .arg_slot = cum->arg_slot,
    };
    cum->info_in_idx++;
    cum->arg_slot++;
}

static void layout_arg_normal_n(TCGCumulativeArgs *cum,
                                TCGHelperInfo *info, int n)
{
    TCGCallArgumentLoc *loc = &info->in[cum->info_in_idx];

    for (int i = 0; i < n; ++i) {
        /* Layout all using the same arg_idx, adjusting the subindex. */
        loc[i] = (TCGCallArgumentLoc){
            .kind = TCG_CALL_ARG_NORMAL,
            .arg_idx = cum->arg_idx,
            .tmp_subindex = i,
            .arg_slot = cum->arg_slot + i,
        };
    }
    cum->info_in_idx += n;
    cum->arg_slot += n;
}

static void layout_arg_by_ref(TCGCumulativeArgs *cum, TCGHelperInfo *info)
{
    TCGCallArgumentLoc *loc = &info->in[cum->info_in_idx];
    int n = 128 / TCG_TARGET_REG_BITS;

    /* The first subindex carries the pointer. */
    layout_arg_1(cum, info, TCG_CALL_ARG_BY_REF);

    /*
     * The callee is allowed to clobber memory associated with
     * structure pass by-reference.  Therefore we must make copies.
     * Allocate space from "ref_slot", which will be adjusted to
     * follow the parameters on the stack.
     */
    loc[0].ref_slot = cum->ref_slot;

    /*
     * Subsequent words also go into the reference slot, but
     * do not accumulate into the regular arguments.
     */
    for (int i = 1; i < n; ++i) {
        loc[i] = (TCGCallArgumentLoc){
            .kind = TCG_CALL_ARG_BY_REF_N,
            .arg_idx = cum->arg_idx,
            .tmp_subindex = i,
            .ref_slot = cum->ref_slot + i,
        };
    }
    cum->info_in_idx += n - 1;  /* i=0 accounted for in layout_arg_1 */
    cum->ref_slot += n;
}

static void init_call_layout(TCGHelperInfo *info)
{
    int max_reg_slots = ARRAY_SIZE(tcg_target_call_iarg_regs);
    int max_stk_slots = TCG_STATIC_CALL_ARGS_SIZE / sizeof(tcg_target_long);
    unsigned typemask = info->typemask;
    unsigned typecode;
    TCGCumulativeArgs cum = { };

    /*
     * Parse and place any function return value.
     */
    typecode = typemask & 7;
    switch (typecode) {
    case dh_typecode_void:
        info->nr_out = 0;
        break;
    case dh_typecode_i32:
    case dh_typecode_s32:
    case dh_typecode_ptr:
        info->nr_out = 1;
        info->out_kind = TCG_CALL_RET_NORMAL;
        break;
    case dh_typecode_i64:
    case dh_typecode_s64:
        info->nr_out = 64 / TCG_TARGET_REG_BITS;
        info->out_kind = TCG_CALL_RET_NORMAL;
        /* Query the last register now to trigger any assert early. */
        tcg_target_call_oarg_reg(info->out_kind, info->nr_out - 1);
        break;
    case dh_typecode_i128:
        info->nr_out = 128 / TCG_TARGET_REG_BITS;
        info->out_kind = TCG_TARGET_CALL_RET_I128;
        switch (TCG_TARGET_CALL_RET_I128) {
        case TCG_CALL_RET_NORMAL:
            /* Query the last register now to trigger any assert early. */
            tcg_target_call_oarg_reg(info->out_kind, info->nr_out - 1);
            break;
        case TCG_CALL_RET_BY_VEC:
            /* Query the single register now to trigger any assert early. */
            tcg_target_call_oarg_reg(TCG_CALL_RET_BY_VEC, 0);
            break;
        case TCG_CALL_RET_BY_REF:
            /*
             * Allocate the first argument to the output.
             * We don't need to store this anywhere, just make it
             * unavailable for use in the input loop below.
             */
            cum.arg_slot = 1;
            break;
        default:
            qemu_build_not_reached();
        }
        break;
    default:
        g_assert_not_reached();
    }

    /*
     * Parse and place function arguments.
     */
    for (typemask >>= 3; typemask; typemask >>= 3, cum.arg_idx++) {
        TCGCallArgumentKind kind;
        TCGType type;

        typecode = typemask & 7;
        switch (typecode) {
        case dh_typecode_i32:
        case dh_typecode_s32:
            type = TCG_TYPE_I32;
            break;
        case dh_typecode_i64:
        case dh_typecode_s64:
            type = TCG_TYPE_I64;
            break;
        case dh_typecode_ptr:
            type = TCG_TYPE_PTR;
            break;
        case dh_typecode_i128:
            type = TCG_TYPE_I128;
            break;
        default:
            g_assert_not_reached();
        }

        switch (type) {
        case TCG_TYPE_I32:
            switch (TCG_TARGET_CALL_ARG_I32) {
            case TCG_CALL_ARG_EVEN:
                layout_arg_even(&cum);
                /* fall through */
            case TCG_CALL_ARG_NORMAL:
                layout_arg_1(&cum, info, TCG_CALL_ARG_NORMAL);
                break;
            case TCG_CALL_ARG_EXTEND:
                kind = TCG_CALL_ARG_EXTEND_U + (typecode & 1);
                layout_arg_1(&cum, info, kind);
                break;
            default:
                qemu_build_not_reached();
            }
            break;

        case TCG_TYPE_I64:
            switch (TCG_TARGET_CALL_ARG_I64) {
            case TCG_CALL_ARG_EVEN:
                layout_arg_even(&cum);
                /* fall through */
            case TCG_CALL_ARG_NORMAL:
                if (TCG_TARGET_REG_BITS == 32) {
                    layout_arg_normal_n(&cum, info, 2);
                } else {
                    layout_arg_1(&cum, info, TCG_CALL_ARG_NORMAL);
                }
                break;
            default:
                qemu_build_not_reached();
            }
            break;

        case TCG_TYPE_I128:
            switch (TCG_TARGET_CALL_ARG_I128) {
            case TCG_CALL_ARG_EVEN:
                layout_arg_even(&cum);
                /* fall through */
            case TCG_CALL_ARG_NORMAL:
                layout_arg_normal_n(&cum, info, 128 / TCG_TARGET_REG_BITS);
                break;
            case TCG_CALL_ARG_BY_REF:
                layout_arg_by_ref(&cum, info);
                break;
            default:
                qemu_build_not_reached();
            }
            break;

        default:
            g_assert_not_reached();
        }
    }
    info->nr_in = cum.info_in_idx;

    /* Validate that we didn't overrun the input array. */
    assert(cum.info_in_idx <= ARRAY_SIZE(info->in));
    /* Validate the backend has enough argument space. */
    assert(cum.arg_slot <= max_reg_slots + max_stk_slots);

    /*
     * Relocate the "ref_slot" area to the end of the parameters.
     * Minimizing this stack offset helps code size for x86,
     * which has a signed 8-bit offset encoding.
     */
    if (cum.ref_slot != 0) {
        int ref_base = 0;

        if (cum.arg_slot > max_reg_slots) {
            int align = __alignof(Int128) / sizeof(tcg_target_long);

            ref_base = cum.arg_slot - max_reg_slots;
            if (align > 1) {
                ref_base = ROUND_UP(ref_base, align);
            }
        }
        assert(ref_base + cum.ref_slot <= max_stk_slots);
        ref_base += max_reg_slots;

        if (ref_base != 0) {
            for (int i = cum.info_in_idx - 1; i >= 0; --i) {
                TCGCallArgumentLoc *loc = &info->in[i];
                switch (loc->kind) {
                case TCG_CALL_ARG_BY_REF:
                case TCG_CALL_ARG_BY_REF_N:
                    loc->ref_slot += ref_base;
                    break;
                default:
                    break;
                }
            }
        }
    }
}

static int indirect_reg_alloc_order[ARRAY_SIZE(tcg_target_reg_alloc_order)];
static void process_op_defs(TCGContext *s);
static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type,
                                            TCGReg reg, const char *name);

static void tcg_context_init(unsigned max_cpus)
{
    TCGContext *s = &tcg_init_ctx;
    int op, total_args, n, i;
    TCGOpDef *def;
    TCGArgConstraint *args_ct;
    TCGTemp *ts;

    memset(s, 0, sizeof(*s));
    s->nb_globals = 0;

    /* Count total number of arguments and allocate the corresponding
       space */
    total_args = 0;
    for(op = 0; op < NB_OPS; op++) {
        def = &tcg_op_defs[op];
        n = def->nb_iargs + def->nb_oargs;
        total_args += n;
    }

    args_ct = g_new0(TCGArgConstraint, total_args);

    for(op = 0; op < NB_OPS; op++) {
        def = &tcg_op_defs[op];
        def->args_ct = args_ct;
        n = def->nb_iargs + def->nb_oargs;
        args_ct += n;
    }

    init_call_layout(&info_helper_ld32_mmu);
    init_call_layout(&info_helper_ld64_mmu);
    init_call_layout(&info_helper_ld128_mmu);
    init_call_layout(&info_helper_st32_mmu);
    init_call_layout(&info_helper_st64_mmu);
    init_call_layout(&info_helper_st128_mmu);

    tcg_target_init(s);
    process_op_defs(s);

    /* Reverse the order of the saved registers, assuming they're all at
       the start of tcg_target_reg_alloc_order.  */
    for (n = 0; n < ARRAY_SIZE(tcg_target_reg_alloc_order); ++n) {
        int r = tcg_target_reg_alloc_order[n];
        if (tcg_regset_test_reg(tcg_target_call_clobber_regs, r)) {
            break;
        }
    }
    for (i = 0; i < n; ++i) {
        indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[n - 1 - i];
    }
    for (; i < ARRAY_SIZE(tcg_target_reg_alloc_order); ++i) {
        indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[i];
    }

    alloc_tcg_plugin_context(s);

    tcg_ctx = s;
    /*
     * In user-mode we simply share the init context among threads, since we
     * use a single region. See the documentation tcg_region_init() for the
     * reasoning behind this.
     * In softmmu we will have at most max_cpus TCG threads.
     */
#ifdef CONFIG_USER_ONLY
    tcg_ctxs = &tcg_ctx;
    tcg_cur_ctxs = 1;
    tcg_max_ctxs = 1;
#else
    tcg_max_ctxs = max_cpus;
    tcg_ctxs = g_new0(TCGContext *, max_cpus);
#endif

    tcg_debug_assert(!tcg_regset_test_reg(s->reserved_regs, TCG_AREG0));
    ts = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, TCG_AREG0, "env");
    cpu_env = temp_tcgv_ptr(ts);
}

void tcg_init(size_t tb_size, int splitwx, unsigned max_cpus)
{
    tcg_context_init(max_cpus);
    tcg_region_init(tb_size, splitwx, max_cpus);
}

/*
 * Allocate TBs right before their corresponding translated code, making
 * sure that TBs and code are on different cache lines.
 */
TranslationBlock *tcg_tb_alloc(TCGContext *s)
{
    uintptr_t align = qemu_icache_linesize;
    TranslationBlock *tb;
    void *next;

 retry:
    tb = (void *)ROUND_UP((uintptr_t)s->code_gen_ptr, align);
    next = (void *)ROUND_UP((uintptr_t)(tb + 1), align);

    if (unlikely(next > s->code_gen_highwater)) {
        if (tcg_region_alloc(s)) {
            return NULL;
        }
        goto retry;
    }
    qatomic_set(&s->code_gen_ptr, next);
    s->data_gen_ptr = NULL;
    return tb;
}

void tcg_prologue_init(TCGContext *s)
{
    size_t prologue_size;

    s->code_ptr = s->code_gen_ptr;
    s->code_buf = s->code_gen_ptr;
    s->data_gen_ptr = NULL;

#ifndef CONFIG_TCG_INTERPRETER
    tcg_qemu_tb_exec = (tcg_prologue_fn *)tcg_splitwx_to_rx(s->code_ptr);
#endif

#ifdef TCG_TARGET_NEED_POOL_LABELS
    s->pool_labels = NULL;
#endif

    qemu_thread_jit_write();
    /* Generate the prologue.  */
    tcg_target_qemu_prologue(s);

#ifdef TCG_TARGET_NEED_POOL_LABELS
    /* Allow the prologue to put e.g. guest_base into a pool entry.  */
    {
        int result = tcg_out_pool_finalize(s);
        tcg_debug_assert(result == 0);
    }
#endif

    prologue_size = tcg_current_code_size(s);
    perf_report_prologue(s->code_gen_ptr, prologue_size);

#ifndef CONFIG_TCG_INTERPRETER
    flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf),
                        (uintptr_t)s->code_buf, prologue_size);
#endif

    if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) {
        FILE *logfile = qemu_log_trylock();
        if (logfile) {
            fprintf(logfile, "PROLOGUE: [size=%zu]\n", prologue_size);
            if (s->data_gen_ptr) {
                size_t code_size = s->data_gen_ptr - s->code_gen_ptr;
                size_t data_size = prologue_size - code_size;
                size_t i;

                disas(logfile, s->code_gen_ptr, code_size);

                for (i = 0; i < data_size; i += sizeof(tcg_target_ulong)) {
                    if (sizeof(tcg_target_ulong) == 8) {
                        fprintf(logfile,
                                "0x%08" PRIxPTR ":  .quad  0x%016" PRIx64 "\n",
                                (uintptr_t)s->data_gen_ptr + i,
                                *(uint64_t *)(s->data_gen_ptr + i));
                    } else {
                        fprintf(logfile,
                                "0x%08" PRIxPTR ":  .long  0x%08x\n",
                                (uintptr_t)s->data_gen_ptr + i,
                                *(uint32_t *)(s->data_gen_ptr + i));
                    }
                }
            } else {
                disas(logfile, s->code_gen_ptr, prologue_size);
            }
            fprintf(logfile, "\n");
            qemu_log_unlock(logfile);
        }
    }

#ifndef CONFIG_TCG_INTERPRETER
    /*
     * Assert that goto_ptr is implemented completely, setting an epilogue.
     * For tci, we use NULL as the signal to return from the interpreter,
     * so skip this check.
     */
    tcg_debug_assert(tcg_code_gen_epilogue != NULL);
#endif

    tcg_region_prologue_set(s);
}

void tcg_func_start(TCGContext *s)
{
    tcg_pool_reset(s);
    s->nb_temps = s->nb_globals;

    /* No temps have been previously allocated for size or locality.  */
    memset(s->free_temps, 0, sizeof(s->free_temps));

    /* No constant temps have been previously allocated. */
    for (int i = 0; i < TCG_TYPE_COUNT; ++i) {
        if (s->const_table[i]) {
            g_hash_table_remove_all(s->const_table[i]);
        }
    }

    s->nb_ops = 0;
    s->nb_labels = 0;
    s->current_frame_offset = s->frame_start;

#ifdef CONFIG_DEBUG_TCG
    s->goto_tb_issue_mask = 0;
#endif

    QTAILQ_INIT(&s->ops);
    QTAILQ_INIT(&s->free_ops);
    QSIMPLEQ_INIT(&s->labels);

    tcg_debug_assert(s->addr_type == TCG_TYPE_I32 ||
                     s->addr_type == TCG_TYPE_I64);

#if defined(CONFIG_SOFTMMU) && !defined(CONFIG_TCG_INTERPRETER)
    tcg_debug_assert(s->tlb_fast_offset < 0);
    tcg_debug_assert(s->tlb_fast_offset >= MIN_TLB_MASK_TABLE_OFS);
#endif

    tcg_debug_assert(s->insn_start_words > 0);
}

static TCGTemp *tcg_temp_alloc(TCGContext *s)
{
    int n = s->nb_temps++;

    if (n >= TCG_MAX_TEMPS) {
        tcg_raise_tb_overflow(s);
    }
    return memset(&s->temps[n], 0, sizeof(TCGTemp));
}

static TCGTemp *tcg_global_alloc(TCGContext *s)
{
    TCGTemp *ts;

    tcg_debug_assert(s->nb_globals == s->nb_temps);
    tcg_debug_assert(s->nb_globals < TCG_MAX_TEMPS);
    s->nb_globals++;
    ts = tcg_temp_alloc(s);
    ts->kind = TEMP_GLOBAL;

    return ts;
}

static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type,
                                            TCGReg reg, const char *name)
{
    TCGTemp *ts;

    tcg_debug_assert(TCG_TARGET_REG_BITS == 64 || type == TCG_TYPE_I32);

    ts = tcg_global_alloc(s);
    ts->base_type = type;
    ts->type = type;
    ts->kind = TEMP_FIXED;
    ts->reg = reg;
    ts->name = name;
    tcg_regset_set_reg(s->reserved_regs, reg);

    return ts;
}

void tcg_set_frame(TCGContext *s, TCGReg reg, intptr_t start, intptr_t size)
{
    s->frame_start = start;
    s->frame_end = start + size;
    s->frame_temp
        = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, reg, "_frame");
}

TCGTemp *tcg_global_mem_new_internal(TCGType type, TCGv_ptr base,
                                     intptr_t offset, const char *name)
{
    TCGContext *s = tcg_ctx;
    TCGTemp *base_ts = tcgv_ptr_temp(base);
    TCGTemp *ts = tcg_global_alloc(s);
    int indirect_reg = 0;

    switch (base_ts->kind) {
    case TEMP_FIXED:
        break;
    case TEMP_GLOBAL:
        /* We do not support double-indirect registers.  */
        tcg_debug_assert(!base_ts->indirect_reg);
        base_ts->indirect_base = 1;
        s->nb_indirects += (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64
                            ? 2 : 1);
        indirect_reg = 1;
        break;
    default:
        g_assert_not_reached();
    }

    if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) {
        TCGTemp *ts2 = tcg_global_alloc(s);
        char buf[64];

        ts->base_type = TCG_TYPE_I64;
        ts->type = TCG_TYPE_I32;
        ts->indirect_reg = indirect_reg;
        ts->mem_allocated = 1;
        ts->mem_base = base_ts;
        ts->mem_offset = offset;
        pstrcpy(buf, sizeof(buf), name);
        pstrcat(buf, sizeof(buf), "_0");
        ts->name = strdup(buf);

        tcg_debug_assert(ts2 == ts + 1);
        ts2->base_type = TCG_TYPE_I64;
        ts2->type = TCG_TYPE_I32;
        ts2->indirect_reg = indirect_reg;
        ts2->mem_allocated = 1;
        ts2->mem_base = base_ts;
        ts2->mem_offset = offset + 4;
        ts2->temp_subindex = 1;
        pstrcpy(buf, sizeof(buf), name);
        pstrcat(buf, sizeof(buf), "_1");
        ts2->name = strdup(buf);
    } else {
        ts->base_type = type;
        ts->type = type;
        ts->indirect_reg = indirect_reg;
        ts->mem_allocated = 1;
        ts->mem_base = base_ts;
        ts->mem_offset = offset;
        ts->name = name;
    }
    return ts;
}

TCGTemp *tcg_temp_new_internal(TCGType type, TCGTempKind kind)
{
    TCGContext *s = tcg_ctx;
    TCGTemp *ts;
    int n;

    if (kind == TEMP_EBB) {
        int idx = find_first_bit(s->free_temps[type].l, TCG_MAX_TEMPS);

        if (idx < TCG_MAX_TEMPS) {
            /* There is already an available temp with the right type.  */
            clear_bit(idx, s->free_temps[type].l);

            ts = &s->temps[idx];
            ts->temp_allocated = 1;
            tcg_debug_assert(ts->base_type == type);
            tcg_debug_assert(ts->kind == kind);
            return ts;
        }
    } else {
        tcg_debug_assert(kind == TEMP_TB);
    }

    switch (type) {
    case TCG_TYPE_I32:
    case TCG_TYPE_V64:
    case TCG_TYPE_V128:
    case TCG_TYPE_V256:
        n = 1;
        break;
    case TCG_TYPE_I64:
        n = 64 / TCG_TARGET_REG_BITS;
        break;
    case TCG_TYPE_I128:
        n = 128 / TCG_TARGET_REG_BITS;
        break;
    default:
        g_assert_not_reached();
    }

    ts = tcg_temp_alloc(s);
    ts->base_type = type;
    ts->temp_allocated = 1;
    ts->kind = kind;

    if (n == 1) {
        ts->type = type;
    } else {
        ts->type = TCG_TYPE_REG;

        for (int i = 1; i < n; ++i) {
            TCGTemp *ts2 = tcg_temp_alloc(s);

            tcg_debug_assert(ts2 == ts + i);
            ts2->base_type = type;
            ts2->type = TCG_TYPE_REG;
            ts2->temp_allocated = 1;
            ts2->temp_subindex = i;
            ts2->kind = kind;
        }
    }
    return ts;
}

TCGv_vec tcg_temp_new_vec(TCGType type)
{
    TCGTemp *t;

#ifdef CONFIG_DEBUG_TCG
    switch (type) {
    case TCG_TYPE_V64:
        assert(TCG_TARGET_HAS_v64);
        break;
    case TCG_TYPE_V128:
        assert(TCG_TARGET_HAS_v128);
        break;
    case TCG_TYPE_V256:
        assert(TCG_TARGET_HAS_v256);
        break;
    default:
        g_assert_not_reached();
    }
#endif

    t = tcg_temp_new_internal(type, TEMP_EBB);
    return temp_tcgv_vec(t);
}

/* Create a new temp of the same type as an existing temp.  */
TCGv_vec tcg_temp_new_vec_matching(TCGv_vec match)
{
    TCGTemp *t = tcgv_vec_temp(match);

    tcg_debug_assert(t->temp_allocated != 0);

    t = tcg_temp_new_internal(t->base_type, TEMP_EBB);
    return temp_tcgv_vec(t);
}

void tcg_temp_free_internal(TCGTemp *ts)
{
    TCGContext *s = tcg_ctx;

    switch (ts->kind) {
    case TEMP_CONST:
    case TEMP_TB:
        /* Silently ignore free. */
        break;
    case TEMP_EBB:
        tcg_debug_assert(ts->temp_allocated != 0);
        ts->temp_allocated = 0;
        set_bit(temp_idx(ts), s->free_temps[ts->base_type].l);
        break;
    default:
        /* It never made sense to free TEMP_FIXED or TEMP_GLOBAL. */
        g_assert_not_reached();
    }
}

TCGTemp *tcg_constant_internal(TCGType type, int64_t val)
{
    TCGContext *s = tcg_ctx;
    GHashTable *h = s->const_table[type];
    TCGTemp *ts;

    if (h == NULL) {
        h = g_hash_table_new(g_int64_hash, g_int64_equal);
        s->const_table[type] = h;
    }

    ts = g_hash_table_lookup(h, &val);
    if (ts == NULL) {
        int64_t *val_ptr;

        ts = tcg_temp_alloc(s);

        if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) {
            TCGTemp *ts2 = tcg_temp_alloc(s);

            tcg_debug_assert(ts2 == ts + 1);

            ts->base_type = TCG_TYPE_I64;
            ts->type = TCG_TYPE_I32;
            ts->kind = TEMP_CONST;
            ts->temp_allocated = 1;

            ts2->base_type = TCG_TYPE_I64;
            ts2->type = TCG_TYPE_I32;
            ts2->kind = TEMP_CONST;
            ts2->temp_allocated = 1;
            ts2->temp_subindex = 1;

            /*
             * Retain the full value of the 64-bit constant in the low
             * part, so that the hash table works.  Actual uses will
             * truncate the value to the low part.
             */
            ts[HOST_BIG_ENDIAN].val = val;
            ts[!HOST_BIG_ENDIAN].val = val >> 32;
            val_ptr = &ts[HOST_BIG_ENDIAN].val;
        } else {
            ts->base_type = type;
            ts->type = type;
            ts->kind = TEMP_CONST;
            ts->temp_allocated = 1;
            ts->val = val;
            val_ptr = &ts->val;
        }
        g_hash_table_insert(h, val_ptr, ts);
    }

    return ts;
}

TCGv_vec tcg_constant_vec(TCGType type, unsigned vece, int64_t val)
{
    val = dup_const(vece, val);
    return temp_tcgv_vec(tcg_constant_internal(type, val));
}

TCGv_vec tcg_constant_vec_matching(TCGv_vec match, unsigned vece, int64_t val)
{
    TCGTemp *t = tcgv_vec_temp(match);

    tcg_debug_assert(t->temp_allocated != 0);
    return tcg_constant_vec(t->base_type, vece, val);
}

#ifdef CONFIG_DEBUG_TCG
size_t temp_idx(TCGTemp *ts)
{
    ptrdiff_t n = ts - tcg_ctx->temps;
    assert(n >= 0 && n < tcg_ctx->nb_temps);
    return n;
}

TCGTemp *tcgv_i32_temp(TCGv_i32 v)
{
    uintptr_t o = (uintptr_t)v - offsetof(TCGContext, temps);

    assert(o < sizeof(TCGTemp) * tcg_ctx->nb_temps);
    assert(o % sizeof(TCGTemp) == 0);

    return (void *)tcg_ctx + (uintptr_t)v;
}
#endif /* CONFIG_DEBUG_TCG */

/* Return true if OP may appear in the opcode stream.
   Test the runtime variable that controls each opcode.  */
bool tcg_op_supported(TCGOpcode op)
{
    const bool have_vec
        = TCG_TARGET_HAS_v64 | TCG_TARGET_HAS_v128 | TCG_TARGET_HAS_v256;

    switch (op) {
    case INDEX_op_discard:
    case INDEX_op_set_label:
    case INDEX_op_call:
    case INDEX_op_br:
    case INDEX_op_mb:
    case INDEX_op_insn_start:
    case INDEX_op_exit_tb:
    case INDEX_op_goto_tb:
    case INDEX_op_goto_ptr:
    case INDEX_op_qemu_ld_a32_i32:
    case INDEX_op_qemu_ld_a64_i32:
    case INDEX_op_qemu_st_a32_i32:
    case INDEX_op_qemu_st_a64_i32:
    case INDEX_op_qemu_ld_a32_i64:
    case INDEX_op_qemu_ld_a64_i64:
    case INDEX_op_qemu_st_a32_i64:
    case INDEX_op_qemu_st_a64_i64:
        return true;

    case INDEX_op_qemu_st8_a32_i32:
    case INDEX_op_qemu_st8_a64_i32:
        return TCG_TARGET_HAS_qemu_st8_i32;

    case INDEX_op_qemu_ld_a32_i128:
    case INDEX_op_qemu_ld_a64_i128:
    case INDEX_op_qemu_st_a32_i128:
    case INDEX_op_qemu_st_a64_i128:
        return TCG_TARGET_HAS_qemu_ldst_i128;

    case INDEX_op_mov_i32:
    case INDEX_op_setcond_i32:
    case INDEX_op_brcond_i32:
    case INDEX_op_ld8u_i32:
    case INDEX_op_ld8s_i32:
    case INDEX_op_ld16u_i32:
    case INDEX_op_ld16s_i32:
    case INDEX_op_ld_i32:
    case INDEX_op_st8_i32:
    case INDEX_op_st16_i32:
    case INDEX_op_st_i32:
    case INDEX_op_add_i32:
    case INDEX_op_sub_i32:
    case INDEX_op_mul_i32:
    case INDEX_op_and_i32:
    case INDEX_op_or_i32:
    case INDEX_op_xor_i32:
    case INDEX_op_shl_i32:
    case INDEX_op_shr_i32:
    case INDEX_op_sar_i32:
        return true;

    case INDEX_op_movcond_i32:
        return TCG_TARGET_HAS_movcond_i32;
    case INDEX_op_div_i32:
    case INDEX_op_divu_i32:
        return TCG_TARGET_HAS_div_i32;
    case INDEX_op_rem_i32:
    case INDEX_op_remu_i32:
        return TCG_TARGET_HAS_rem_i32;
    case INDEX_op_div2_i32:
    case INDEX_op_divu2_i32:
        return TCG_TARGET_HAS_div2_i32;
    case INDEX_op_rotl_i32:
    case INDEX_op_rotr_i32:
        return TCG_TARGET_HAS_rot_i32;
    case INDEX_op_deposit_i32:
        return TCG_TARGET_HAS_deposit_i32;
    case INDEX_op_extract_i32:
        return TCG_TARGET_HAS_extract_i32;
    case INDEX_op_sextract_i32:
        return TCG_TARGET_HAS_sextract_i32;
    case INDEX_op_extract2_i32:
        return TCG_TARGET_HAS_extract2_i32;
    case INDEX_op_add2_i32:
        return TCG_TARGET_HAS_add2_i32;
    case INDEX_op_sub2_i32:
        return TCG_TARGET_HAS_sub2_i32;
    case INDEX_op_mulu2_i32:
        return TCG_TARGET_HAS_mulu2_i32;
    case INDEX_op_muls2_i32:
        return TCG_TARGET_HAS_muls2_i32;
    case INDEX_op_muluh_i32:
        return TCG_TARGET_HAS_muluh_i32;
    case INDEX_op_mulsh_i32:
        return TCG_TARGET_HAS_mulsh_i32;
    case INDEX_op_ext8s_i32:
        return TCG_TARGET_HAS_ext8s_i32;
    case INDEX_op_ext16s_i32:
        return TCG_TARGET_HAS_ext16s_i32;
    case INDEX_op_ext8u_i32:
        return TCG_TARGET_HAS_ext8u_i32;
    case INDEX_op_ext16u_i32:
        return TCG_TARGET_HAS_ext16u_i32;
    case INDEX_op_bswap16_i32:
        return TCG_TARGET_HAS_bswap16_i32;
    case INDEX_op_bswap32_i32:
        return TCG_TARGET_HAS_bswap32_i32;
    case INDEX_op_not_i32:
        return TCG_TARGET_HAS_not_i32;
    case INDEX_op_neg_i32:
        return TCG_TARGET_HAS_neg_i32;
    case INDEX_op_andc_i32:
        return TCG_TARGET_HAS_andc_i32;
    case INDEX_op_orc_i32:
        return TCG_TARGET_HAS_orc_i32;
    case INDEX_op_eqv_i32:
        return TCG_TARGET_HAS_eqv_i32;
    case INDEX_op_nand_i32:
        return TCG_TARGET_HAS_nand_i32;
    case INDEX_op_nor_i32:
        return TCG_TARGET_HAS_nor_i32;
    case INDEX_op_clz_i32:
        return TCG_TARGET_HAS_clz_i32;
    case INDEX_op_ctz_i32:
        return TCG_TARGET_HAS_ctz_i32;
    case INDEX_op_ctpop_i32:
        return TCG_TARGET_HAS_ctpop_i32;

    case INDEX_op_brcond2_i32:
    case INDEX_op_setcond2_i32:
        return TCG_TARGET_REG_BITS == 32;

    case INDEX_op_mov_i64:
    case INDEX_op_setcond_i64:
    case INDEX_op_brcond_i64:
    case INDEX_op_ld8u_i64:
    case INDEX_op_ld8s_i64:
    case INDEX_op_ld16u_i64:
    case INDEX_op_ld16s_i64:
    case INDEX_op_ld32u_i64:
    case INDEX_op_ld32s_i64:
    case INDEX_op_ld_i64:
    case INDEX_op_st8_i64:
    case INDEX_op_st16_i64:
    case INDEX_op_st32_i64:
    case INDEX_op_st_i64:
    case INDEX_op_add_i64:
    case INDEX_op_sub_i64:
    case INDEX_op_mul_i64:
    case INDEX_op_and_i64:
    case INDEX_op_or_i64:
    case INDEX_op_xor_i64:
    case INDEX_op_shl_i64:
    case INDEX_op_shr_i64:
    case INDEX_op_sar_i64:
    case INDEX_op_ext_i32_i64:
    case INDEX_op_extu_i32_i64:
        return TCG_TARGET_REG_BITS == 64;

    case INDEX_op_movcond_i64:
        return TCG_TARGET_HAS_movcond_i64;
    case INDEX_op_div_i64:
    case INDEX_op_divu_i64:
        return TCG_TARGET_HAS_div_i64;
    case INDEX_op_rem_i64:
    case INDEX_op_remu_i64:
        return TCG_TARGET_HAS_rem_i64;
    case INDEX_op_div2_i64:
    case INDEX_op_divu2_i64:
        return TCG_TARGET_HAS_div2_i64;
    case INDEX_op_rotl_i64:
    case INDEX_op_rotr_i64:
        return TCG_TARGET_HAS_rot_i64;
    case INDEX_op_deposit_i64:
        return TCG_TARGET_HAS_deposit_i64;
    case INDEX_op_extract_i64:
        return TCG_TARGET_HAS_extract_i64;
    case INDEX_op_sextract_i64:
        return TCG_TARGET_HAS_sextract_i64;
    case INDEX_op_extract2_i64:

        return TCG_TARGET_HAS_extract2_i64;
    case INDEX_op_extrl_i64_i32:
        return TCG_TARGET_HAS_extrl_i64_i32;
    case INDEX_op_extrh_i64_i32:
        return TCG_TARGET_HAS_extrh_i64_i32;
    case INDEX_op_ext8s_i64:
        return TCG_TARGET_HAS_ext8s_i64;
    case INDEX_op_ext16s_i64:
        return TCG_TARGET_HAS_ext16s_i64;
    case INDEX_op_ext32s_i64:
        return TCG_TARGET_HAS_ext32s_i64;
    case INDEX_op_ext8u_i64:
        return TCG_TARGET_HAS_ext8u_i64;
    case INDEX_op_ext16u_i64:
        return TCG_TARGET_HAS_ext16u_i64;
    case INDEX_op_ext32u_i64:
        return TCG_TARGET_HAS_ext32u_i64;
    case INDEX_op_bswap16_i64:
        return TCG_TARGET_HAS_bswap16_i64;
    case INDEX_op_bswap32_i64:
        return TCG_TARGET_HAS_bswap32_i64;
    case INDEX_op_bswap64_i64:
        return TCG_TARGET_HAS_bswap64_i64;
    case INDEX_op_not_i64:
        return TCG_TARGET_HAS_not_i64;
    case INDEX_op_neg_i64:
        return TCG_TARGET_HAS_neg_i64;
    case INDEX_op_andc_i64:
        return TCG_TARGET_HAS_andc_i64;
    case INDEX_op_orc_i64:
        return TCG_TARGET_HAS_orc_i64;
    case INDEX_op_eqv_i64:
        return TCG_TARGET_HAS_eqv_i64;
    case INDEX_op_nand_i64:
        return TCG_TARGET_HAS_nand_i64;
    case INDEX_op_nor_i64:
        return TCG_TARGET_HAS_nor_i64;
    case INDEX_op_clz_i64:
        return TCG_TARGET_HAS_clz_i64;
    case INDEX_op_ctz_i64:
        return TCG_TARGET_HAS_ctz_i64;
    case INDEX_op_ctpop_i64:
        return TCG_TARGET_HAS_ctpop_i64;
    case INDEX_op_add2_i64:
        return TCG_TARGET_HAS_add2_i64;
    case INDEX_op_sub2_i64:
        return TCG_TARGET_HAS_sub2_i64;
    case INDEX_op_mulu2_i64:
        return TCG_TARGET_HAS_mulu2_i64;
    case INDEX_op_muls2_i64:
        return TCG_TARGET_HAS_muls2_i64;
    case INDEX_op_muluh_i64:
        return TCG_TARGET_HAS_muluh_i64;
    case INDEX_op_mulsh_i64:
        return TCG_TARGET_HAS_mulsh_i64;

    case INDEX_op_mov_vec:
    case INDEX_op_dup_vec:
    case INDEX_op_dupm_vec:
    case INDEX_op_ld_vec:
    case INDEX_op_st_vec:
    case INDEX_op_add_vec:
    case INDEX_op_sub_vec:
    case INDEX_op_and_vec:
    case INDEX_op_or_vec:
    case INDEX_op_xor_vec:
    case INDEX_op_cmp_vec:
        return have_vec;
    case INDEX_op_dup2_vec:
        return have_vec && TCG_TARGET_REG_BITS == 32;
    case INDEX_op_not_vec:
        return have_vec && TCG_TARGET_HAS_not_vec;
    case INDEX_op_neg_vec:
        return have_vec && TCG_TARGET_HAS_neg_vec;
    case INDEX_op_abs_vec:
        return have_vec && TCG_TARGET_HAS_abs_vec;
    case INDEX_op_andc_vec:
        return have_vec && TCG_TARGET_HAS_andc_vec;
    case INDEX_op_orc_vec:
        return have_vec && TCG_TARGET_HAS_orc_vec;
    case INDEX_op_nand_vec:
        return have_vec && TCG_TARGET_HAS_nand_vec;
    case INDEX_op_nor_vec:
        return have_vec && TCG_TARGET_HAS_nor_vec;
    case INDEX_op_eqv_vec:
        return have_vec && TCG_TARGET_HAS_eqv_vec;
    case INDEX_op_mul_vec:
        return have_vec && TCG_TARGET_HAS_mul_vec;
    case INDEX_op_shli_vec:
    case INDEX_op_shri_vec:
    case INDEX_op_sari_vec:
        return have_vec && TCG_TARGET_HAS_shi_vec;
    case INDEX_op_shls_vec:
    case INDEX_op_shrs_vec:
    case INDEX_op_sars_vec:
        return have_vec && TCG_TARGET_HAS_shs_vec;
    case INDEX_op_shlv_vec:
    case INDEX_op_shrv_vec:
    case INDEX_op_sarv_vec:
        return have_vec && TCG_TARGET_HAS_shv_vec;
    case INDEX_op_rotli_vec:
        return have_vec && TCG_TARGET_HAS_roti_vec;
    case INDEX_op_rotls_vec:
        return have_vec && TCG_TARGET_HAS_rots_vec;
    case INDEX_op_rotlv_vec:
    case INDEX_op_rotrv_vec:
        return have_vec && TCG_TARGET_HAS_rotv_vec;
    case INDEX_op_ssadd_vec:
    case INDEX_op_usadd_vec:
    case INDEX_op_sssub_vec:
    case INDEX_op_ussub_vec:
        return have_vec && TCG_TARGET_HAS_sat_vec;
    case INDEX_op_smin_vec:
    case INDEX_op_umin_vec:
    case INDEX_op_smax_vec:
    case INDEX_op_umax_vec:
        return have_vec && TCG_TARGET_HAS_minmax_vec;
    case INDEX_op_bitsel_vec:
        return have_vec && TCG_TARGET_HAS_bitsel_vec;
    case INDEX_op_cmpsel_vec:
        return have_vec && TCG_TARGET_HAS_cmpsel_vec;

    default:
        tcg_debug_assert(op > INDEX_op_last_generic && op < NB_OPS);
        return true;
    }
}

static TCGOp *tcg_op_alloc(TCGOpcode opc, unsigned nargs);

static void tcg_gen_callN(TCGHelperInfo *info, TCGTemp *ret, TCGTemp **args)
{
    TCGv_i64 extend_free[MAX_CALL_IARGS];
    int n_extend = 0;
    TCGOp *op;
    int i, n, pi = 0, total_args;

    if (unlikely(g_once_init_enter(HELPER_INFO_INIT(info)))) {
        init_call_layout(info);
        g_once_init_leave(HELPER_INFO_INIT(info), HELPER_INFO_INIT_VAL(info));
    }

    total_args = info->nr_out + info->nr_in + 2;
    op = tcg_op_alloc(INDEX_op_call, total_args);

#ifdef CONFIG_PLUGIN
    /* Flag helpers that may affect guest state */
    if (tcg_ctx->plugin_insn &&
        !(info->flags & TCG_CALL_PLUGIN) &&
        !(info->flags & TCG_CALL_NO_SIDE_EFFECTS)) {
        tcg_ctx->plugin_insn->calls_helpers = true;
    }
#endif

    TCGOP_CALLO(op) = n = info->nr_out;
    switch (n) {
    case 0:
        tcg_debug_assert(ret == NULL);
        break;
    case 1:
        tcg_debug_assert(ret != NULL);
        op->args[pi++] = temp_arg(ret);
        break;
    case 2:
    case 4:
        tcg_debug_assert(ret != NULL);
        tcg_debug_assert(ret->base_type == ret->type + ctz32(n));
        tcg_debug_assert(ret->temp_subindex == 0);
        for (i = 0; i < n; ++i) {
            op->args[pi++] = temp_arg(ret + i);
        }
        break;
    default:
        g_assert_not_reached();
    }

    TCGOP_CALLI(op) = n = info->nr_in;
    for (i = 0; i < n; i++) {
        const TCGCallArgumentLoc *loc = &info->in[i];
        TCGTemp *ts = args[loc->arg_idx] + loc->tmp_subindex;

        switch (loc->kind) {
        case TCG_CALL_ARG_NORMAL:
        case TCG_CALL_ARG_BY_REF:
        case TCG_CALL_ARG_BY_REF_N:
            op->args[pi++] = temp_arg(ts);
            break;

        case TCG_CALL_ARG_EXTEND_U:
        case TCG_CALL_ARG_EXTEND_S:
            {
                TCGv_i64 temp = tcg_temp_ebb_new_i64();
                TCGv_i32 orig = temp_tcgv_i32(ts);

                if (loc->kind == TCG_CALL_ARG_EXTEND_S) {
                    tcg_gen_ext_i32_i64(temp, orig);
                } else {
                    tcg_gen_extu_i32_i64(temp, orig);
                }
                op->args[pi++] = tcgv_i64_arg(temp);
                extend_free[n_extend++] = temp;
            }
            break;

        default:
            g_assert_not_reached();
        }
    }
    op->args[pi++] = (uintptr_t)info->func;
    op->args[pi++] = (uintptr_t)info;
    tcg_debug_assert(pi == total_args);

    QTAILQ_INSERT_TAIL(&tcg_ctx->ops, op, link);

    tcg_debug_assert(n_extend < ARRAY_SIZE(extend_free));
    for (i = 0; i < n_extend; ++i) {
        tcg_temp_free_i64(extend_free[i]);
    }
}

void tcg_gen_call0(TCGHelperInfo *info, TCGTemp *ret)
{
    tcg_gen_callN(info, ret, NULL);
}

void tcg_gen_call1(TCGHelperInfo *info, TCGTemp *ret, TCGTemp *t1)
{
    tcg_gen_callN(info, ret, &t1);
}

void tcg_gen_call2(TCGHelperInfo *info, TCGTemp *ret, TCGTemp *t1, TCGTemp *t2)
{
    TCGTemp *args[2] = { t1, t2 };
    tcg_gen_callN(info, ret, args);
}

void tcg_gen_call3(TCGHelperInfo *info, TCGTemp *ret, TCGTemp *t1,
                   TCGTemp *t2, TCGTemp *t3)
{
    TCGTemp *args[3] = { t1, t2, t3 };
    tcg_gen_callN(info, ret, args);
}

void tcg_gen_call4(TCGHelperInfo *info, TCGTemp *ret, TCGTemp *t1,
                   TCGTemp *t2, TCGTemp *t3, TCGTemp *t4)
{
    TCGTemp *args[4] = { t1, t2, t3, t4 };
    tcg_gen_callN(info, ret, args);
}

void tcg_gen_call5(TCGHelperInfo *info, TCGTemp *ret, TCGTemp *t1,
                   TCGTemp *t2, TCGTemp *t3, TCGTemp *t4, TCGTemp *t5)
{
    TCGTemp *args[5] = { t1, t2, t3, t4, t5 };
    tcg_gen_callN(info, ret, args);
}

void tcg_gen_call6(TCGHelperInfo *info, TCGTemp *ret, TCGTemp *t1, TCGTemp *t2,
                   TCGTemp *t3, TCGTemp *t4, TCGTemp *t5, TCGTemp *t6)
{
    TCGTemp *args[6] = { t1, t2, t3, t4, t5, t6 };
    tcg_gen_callN(info, ret, args);
}

void tcg_gen_call7(TCGHelperInfo *info, TCGTemp *ret, TCGTemp *t1,
                   TCGTemp *t2, TCGTemp *t3, TCGTemp *t4,
                   TCGTemp *t5, TCGTemp *t6, TCGTemp *t7)
{
    TCGTemp *args[7] = { t1, t2, t3, t4, t5, t6, t7 };
    tcg_gen_callN(info, ret, args);
}

static void tcg_reg_alloc_start(TCGContext *s)
{
    int i, n;

    for (i = 0, n = s->nb_temps; i < n; i++) {
        TCGTemp *ts = &s->temps[i];
        TCGTempVal val = TEMP_VAL_MEM;

        switch (ts->kind) {
        case TEMP_CONST:
            val = TEMP_VAL_CONST;
            break;
        case TEMP_FIXED:
            val = TEMP_VAL_REG;
            break;
        case TEMP_GLOBAL:
            break;
        case TEMP_EBB:
            val = TEMP_VAL_DEAD;
            /* fall through */
        case TEMP_TB:
            ts->mem_allocated = 0;
            break;
        default:
            g_assert_not_reached();
        }
        ts->val_type = val;
    }

    memset(s->reg_to_temp, 0, sizeof(s->reg_to_temp));
}

static char *tcg_get_arg_str_ptr(TCGContext *s, char *buf, int buf_size,
                                 TCGTemp *ts)
{
    int idx = temp_idx(ts);

    switch (ts->kind) {
    case TEMP_FIXED:
    case TEMP_GLOBAL:
        pstrcpy(buf, buf_size, ts->name);
        break;
    case TEMP_TB:
        snprintf(buf, buf_size, "loc%d", idx - s->nb_globals);
        break;
    case TEMP_EBB:
        snprintf(buf, buf_size, "tmp%d", idx - s->nb_globals);
        break;
    case TEMP_CONST:
        switch (ts->type) {
        case TCG_TYPE_I32:
            snprintf(buf, buf_size, "$0x%x", (int32_t)ts->val);
            break;
#if TCG_TARGET_REG_BITS > 32
        case TCG_TYPE_I64:
            snprintf(buf, buf_size, "$0x%" PRIx64, ts->val);
            break;
#endif
        case TCG_TYPE_V64:
        case TCG_TYPE_V128:
        case TCG_TYPE_V256:
            snprintf(buf, buf_size, "v%d$0x%" PRIx64,
                     64 << (ts->type - TCG_TYPE_V64), ts->val);
            break;
        default:
            g_assert_not_reached();
        }
        break;
    }
    return buf;
}

static char *tcg_get_arg_str(TCGContext *s, char *buf,
                             int buf_size, TCGArg arg)
{
    return tcg_get_arg_str_ptr(s, buf, buf_size, arg_temp(arg));
}

static const char * const cond_name[] =
{
    [TCG_COND_NEVER] = "never",
    [TCG_COND_ALWAYS] = "always",
    [TCG_COND_EQ] = "eq",
    [TCG_COND_NE] = "ne",
    [TCG_COND_LT] = "lt",
    [TCG_COND_GE] = "ge",
    [TCG_COND_LE] = "le",
    [TCG_COND_GT] = "gt",
    [TCG_COND_LTU] = "ltu",
    [TCG_COND_GEU] = "geu",
    [TCG_COND_LEU] = "leu",
    [TCG_COND_GTU] = "gtu"
};

static const char * const ldst_name[(MO_BSWAP | MO_SSIZE) + 1] =
{
    [MO_UB]   = "ub",
    [MO_SB]   = "sb",
    [MO_LEUW] = "leuw",
    [MO_LESW] = "lesw",
    [MO_LEUL] = "leul",
    [MO_LESL] = "lesl",
    [MO_LEUQ] = "leq",
    [MO_BEUW] = "beuw",
    [MO_BESW] = "besw",
    [MO_BEUL] = "beul",
    [MO_BESL] = "besl",
    [MO_BEUQ] = "beq",
    [MO_128 + MO_BE] = "beo",
    [MO_128 + MO_LE] = "leo",
};

static const char * const alignment_name[(MO_AMASK >> MO_ASHIFT) + 1] = {
    [MO_UNALN >> MO_ASHIFT]    = "un+",
    [MO_ALIGN >> MO_ASHIFT]    = "al+",
    [MO_ALIGN_2 >> MO_ASHIFT]  = "al2+",
    [MO_ALIGN_4 >> MO_ASHIFT]  = "al4+",
    [MO_ALIGN_8 >> MO_ASHIFT]  = "al8+",
    [MO_ALIGN_16 >> MO_ASHIFT] = "al16+",
    [MO_ALIGN_32 >> MO_ASHIFT] = "al32+",
    [MO_ALIGN_64 >> MO_ASHIFT] = "al64+",
};

static const char * const atom_name[(MO_ATOM_MASK >> MO_ATOM_SHIFT) + 1] = {
    [MO_ATOM_IFALIGN >> MO_ATOM_SHIFT] = "",
    [MO_ATOM_IFALIGN_PAIR >> MO_ATOM_SHIFT] = "pair+",
    [MO_ATOM_WITHIN16 >> MO_ATOM_SHIFT] = "w16+",
    [MO_ATOM_WITHIN16_PAIR >> MO_ATOM_SHIFT] = "w16p+",
    [MO_ATOM_SUBALIGN >> MO_ATOM_SHIFT] = "sub+",
    [MO_ATOM_NONE >> MO_ATOM_SHIFT] = "noat+",
};

static const char bswap_flag_name[][6] = {
    [TCG_BSWAP_IZ] = "iz",
    [TCG_BSWAP_OZ] = "oz",
    [TCG_BSWAP_OS] = "os",
    [TCG_BSWAP_IZ | TCG_BSWAP_OZ] = "iz,oz",
    [TCG_BSWAP_IZ | TCG_BSWAP_OS] = "iz,os",
};

static inline bool tcg_regset_single(TCGRegSet d)
{
    return (d & (d - 1)) == 0;
}

static inline TCGReg tcg_regset_first(TCGRegSet d)
{
    if (TCG_TARGET_NB_REGS <= 32) {
        return ctz32(d);
    } else {
        return ctz64(d);
    }
}

/* Return only the number of characters output -- no error return. */
#define ne_fprintf(...) \
    ({ int ret_ = fprintf(__VA_ARGS__); ret_ >= 0 ? ret_ : 0; })

static void tcg_dump_ops(TCGContext *s, FILE *f, bool have_prefs)
{
    char buf[128];
    TCGOp *op;

    QTAILQ_FOREACH(op, &s->ops, link) {
        int i, k, nb_oargs, nb_iargs, nb_cargs;
        const TCGOpDef *def;
        TCGOpcode c;
        int col = 0;

        c = op->opc;
        def = &tcg_op_defs[c];

        if (c == INDEX_op_insn_start) {
            nb_oargs = 0;
            col += ne_fprintf(f, "\n ----");

            for (i = 0, k = s->insn_start_words; i < k; ++i) {
                col += ne_fprintf(f, " %016" PRIx64,
                                  tcg_get_insn_start_param(op, i));
            }
        } else if (c == INDEX_op_call) {
            const TCGHelperInfo *info = tcg_call_info(op);
            void *func = tcg_call_func(op);

            /* variable number of arguments */
            nb_oargs = TCGOP_CALLO(op);
            nb_iargs = TCGOP_CALLI(op);
            nb_cargs = def->nb_cargs;

            col += ne_fprintf(f, " %s ", def->name);

            /*
             * Print the function name from TCGHelperInfo, if available.
             * Note that plugins have a template function for the info,
             * but the actual function pointer comes from the plugin.
             */
            if (func == info->func) {
                col += ne_fprintf(f, "%s", info->name);
            } else {
                col += ne_fprintf(f, "plugin(%p)", func);
            }

            col += ne_fprintf(f, ",$0x%x,$%d", info->flags, nb_oargs);
            for (i = 0; i < nb_oargs; i++) {
                col += ne_fprintf(f, ",%s", tcg_get_arg_str(s, buf, sizeof(buf),
                                                            op->args[i]));
            }
            for (i = 0; i < nb_iargs; i++) {
                TCGArg arg = op->args[nb_oargs + i];
                const char *t = tcg_get_arg_str(s, buf, sizeof(buf), arg);
                col += ne_fprintf(f, ",%s", t);
            }
        } else {
            col += ne_fprintf(f, " %s ", def->name);

            nb_oargs = def->nb_oargs;
            nb_iargs = def->nb_iargs;
            nb_cargs = def->nb_cargs;

            if (def->flags & TCG_OPF_VECTOR) {
                col += ne_fprintf(f, "v%d,e%d,", 64 << TCGOP_VECL(op),
                                  8 << TCGOP_VECE(op));
            }

            k = 0;
            for (i = 0; i < nb_oargs; i++) {
                const char *sep =  k ? "," : "";
                col += ne_fprintf(f, "%s%s", sep,
                                  tcg_get_arg_str(s, buf, sizeof(buf),
                                                  op->args[k++]));
            }
            for (i = 0; i < nb_iargs; i++) {
                const char *sep =  k ? "," : "";
                col += ne_fprintf(f, "%s%s", sep,
                                  tcg_get_arg_str(s, buf, sizeof(buf),
                                                  op->args[k++]));
            }
            switch (c) {
            case INDEX_op_brcond_i32:
            case INDEX_op_setcond_i32:
            case INDEX_op_movcond_i32:
            case INDEX_op_brcond2_i32:
            case INDEX_op_setcond2_i32:
            case INDEX_op_brcond_i64:
            case INDEX_op_setcond_i64:
            case INDEX_op_movcond_i64:
            case INDEX_op_cmp_vec:
            case INDEX_op_cmpsel_vec:
                if (op->args[k] < ARRAY_SIZE(cond_name)
                    && cond_name[op->args[k]]) {
                    col += ne_fprintf(f, ",%s", cond_name[op->args[k++]]);
                } else {
                    col += ne_fprintf(f, ",$0x%" TCG_PRIlx, op->args[k++]);
                }
                i = 1;
                break;
            case INDEX_op_qemu_ld_a32_i32:
            case INDEX_op_qemu_ld_a64_i32:
            case INDEX_op_qemu_st_a32_i32:
            case INDEX_op_qemu_st_a64_i32:
            case INDEX_op_qemu_st8_a32_i32:
            case INDEX_op_qemu_st8_a64_i32:
            case INDEX_op_qemu_ld_a32_i64:
            case INDEX_op_qemu_ld_a64_i64:
            case INDEX_op_qemu_st_a32_i64:
            case INDEX_op_qemu_st_a64_i64:
            case INDEX_op_qemu_ld_a32_i128:
            case INDEX_op_qemu_ld_a64_i128:
            case INDEX_op_qemu_st_a32_i128:
            case INDEX_op_qemu_st_a64_i128:
                {
                    const char *s_al, *s_op, *s_at;
                    MemOpIdx oi = op->args[k++];
                    MemOp op = get_memop(oi);
                    unsigned ix = get_mmuidx(oi);

                    s_al = alignment_name[(op & MO_AMASK) >> MO_ASHIFT];
                    s_op = ldst_name[op & (MO_BSWAP | MO_SSIZE)];
                    s_at = atom_name[(op & MO_ATOM_MASK) >> MO_ATOM_SHIFT];
                    op &= ~(MO_AMASK | MO_BSWAP | MO_SSIZE | MO_ATOM_MASK);

                    /* If all fields are accounted for, print symbolically. */
                    if (!op && s_al && s_op && s_at) {
                        col += ne_fprintf(f, ",%s%s%s,%u",
                                          s_at, s_al, s_op, ix);
                    } else {
                        op = get_memop(oi);
                        col += ne_fprintf(f, ",$0x%x,%u", op, ix);
                    }
                    i = 1;
                }
                break;
            case INDEX_op_bswap16_i32:
            case INDEX_op_bswap16_i64:
            case INDEX_op_bswap32_i32:
            case INDEX_op_bswap32_i64:
            case INDEX_op_bswap64_i64:
                {
                    TCGArg flags = op->args[k];
                    const char *name = NULL;

                    if (flags < ARRAY_SIZE(bswap_flag_name)) {
                        name = bswap_flag_name[flags];
                    }
                    if (name) {
                        col += ne_fprintf(f, ",%s", name);
                    } else {
                        col += ne_fprintf(f, ",$0x%" TCG_PRIlx, flags);
                    }
                    i = k = 1;
                }
                break;
            default:
                i = 0;
                break;
            }
            switch (c) {
            case INDEX_op_set_label:
            case INDEX_op_br:
            case INDEX_op_brcond_i32:
            case INDEX_op_brcond_i64:
            case INDEX_op_brcond2_i32:
                col += ne_fprintf(f, "%s$L%d", k ? "," : "",
                                  arg_label(op->args[k])->id);
                i++, k++;
                break;
            case INDEX_op_mb:
                {
                    TCGBar membar = op->args[k];
                    const char *b_op, *m_op;

                    switch (membar & TCG_BAR_SC) {
                    case 0:
                        b_op = "none";
                        break;
                    case TCG_BAR_LDAQ:
                        b_op = "acq";
                        break;
                    case TCG_BAR_STRL:
                        b_op = "rel";
                        break;
                    case TCG_BAR_SC:
                        b_op = "seq";
                        break;
                    default:
                        g_assert_not_reached();
                    }

                    switch (membar & TCG_MO_ALL) {
                    case 0:
                        m_op = "none";
                        break;
                    case TCG_MO_LD_LD:
                        m_op = "rr";
                        break;
                    case TCG_MO_LD_ST:
                        m_op = "rw";
                        break;
                    case TCG_MO_ST_LD:
                        m_op = "wr";
                        break;
                    case TCG_MO_ST_ST:
                        m_op = "ww";
                        break;
                    case TCG_MO_LD_LD | TCG_MO_LD_ST:
                        m_op = "rr+rw";
                        break;
                    case TCG_MO_LD_LD | TCG_MO_ST_LD:
                        m_op = "rr+wr";
                        break;
                    case TCG_MO_LD_LD | TCG_MO_ST_ST:
                        m_op = "rr+ww";
                        break;
                    case TCG_MO_LD_ST | TCG_MO_ST_LD:
                        m_op = "rw+wr";
                        break;
                    case TCG_MO_LD_ST | TCG_MO_ST_ST:
                        m_op = "rw+ww";
                        break;
                    case TCG_MO_ST_LD | TCG_MO_ST_ST:
                        m_op = "wr+ww";
                        break;
                    case TCG_MO_LD_LD | TCG_MO_LD_ST | TCG_MO_ST_LD:
                        m_op = "rr+rw+wr";
                        break;
                    case TCG_MO_LD_LD | TCG_MO_LD_ST | TCG_MO_ST_ST:
                        m_op = "rr+rw+ww";
                        break;
                    case TCG_MO_LD_LD | TCG_MO_ST_LD | TCG_MO_ST_ST:
                        m_op = "rr+wr+ww";
                        break;
                    case TCG_MO_LD_ST | TCG_MO_ST_LD | TCG_MO_ST_ST:
                        m_op = "rw+wr+ww";
                        break;
                    case TCG_MO_ALL:
                        m_op = "all";
                        break;
                    default:
                        g_assert_not_reached();
                    }

                    col += ne_fprintf(f, "%s%s:%s", (k ? "," : ""), b_op, m_op);
                    i++, k++;
                }
                break;
            default:
                break;
            }
            for (; i < nb_cargs; i++, k++) {
                col += ne_fprintf(f, "%s$0x%" TCG_PRIlx, k ? "," : "",
                                  op->args[k]);
            }
        }

        if (have_prefs || op->life) {
            for (; col < 40; ++col) {
                putc(' ', f);
            }
        }

        if (op->life) {
            unsigned life = op->life;

            if (life & (SYNC_ARG * 3)) {
                ne_fprintf(f, "  sync:");
                for (i = 0; i < 2; ++i) {
                    if (life & (SYNC_ARG << i)) {
                        ne_fprintf(f, " %d", i);
                    }
                }
            }
            life /= DEAD_ARG;
            if (life) {
                ne_fprintf(f, "  dead:");
                for (i = 0; life; ++i, life >>= 1) {
                    if (life & 1) {
                        ne_fprintf(f, " %d", i);
                    }
                }
            }
        }

        if (have_prefs) {
            for (i = 0; i < nb_oargs; ++i) {
                TCGRegSet set = output_pref(op, i);

                if (i == 0) {
                    ne_fprintf(f, "  pref=");
                } else {
                    ne_fprintf(f, ",");
                }
                if (set == 0) {
                    ne_fprintf(f, "none");
                } else if (set == MAKE_64BIT_MASK(0, TCG_TARGET_NB_REGS)) {
                    ne_fprintf(f, "all");
#ifdef CONFIG_DEBUG_TCG
                } else if (tcg_regset_single(set)) {
                    TCGReg reg = tcg_regset_first(set);
                    ne_fprintf(f, "%s", tcg_target_reg_names[reg]);
#endif
                } else if (TCG_TARGET_NB_REGS <= 32) {
                    ne_fprintf(f, "0x%x", (uint32_t)set);
                } else {
                    ne_fprintf(f, "0x%" PRIx64, (uint64_t)set);
                }
            }
        }

        putc('\n', f);
    }
}

/* we give more priority to constraints with less registers */
static int get_constraint_priority(const TCGOpDef *def, int k)
{
    const TCGArgConstraint *arg_ct = &def->args_ct[k];
    int n = ctpop64(arg_ct->regs);

    /*
     * Sort constraints of a single register first, which includes output
     * aliases (which must exactly match the input already allocated).
     */
    if (n == 1 || arg_ct->oalias) {
        return INT_MAX;
    }

    /*
     * Sort register pairs next, first then second immediately after.
     * Arbitrarily sort multiple pairs by the index of the first reg;
     * there shouldn't be many pairs.
     */
    switch (arg_ct->pair) {
    case 1:
    case 3:
        return (k + 1) * 2;
    case 2:
        return (arg_ct->pair_index + 1) * 2 - 1;
    }

    /* Finally, sort by decreasing register count. */
    assert(n > 1);
    return -n;
}

/* sort from highest priority to lowest */
static void sort_constraints(TCGOpDef *def, int start, int n)
{
    int i, j;
    TCGArgConstraint *a = def->args_ct;

    for (i = 0; i < n; i++) {
        a[start + i].sort_index = start + i;
    }
    if (n <= 1) {
        return;
    }
    for (i = 0; i < n - 1; i++) {
        for (j = i + 1; j < n; j++) {
            int p1 = get_constraint_priority(def, a[start + i].sort_index);
            int p2 = get_constraint_priority(def, a[start + j].sort_index);
            if (p1 < p2) {
                int tmp = a[start + i].sort_index;
                a[start + i].sort_index = a[start + j].sort_index;
                a[start + j].sort_index = tmp;
            }
        }
    }
}

static void process_op_defs(TCGContext *s)
{
    TCGOpcode op;

    for (op = 0; op < NB_OPS; op++) {
        TCGOpDef *def = &tcg_op_defs[op];
        const TCGTargetOpDef *tdefs;
        bool saw_alias_pair = false;
        int i, o, i2, o2, nb_args;

        if (def->flags & TCG_OPF_NOT_PRESENT) {
            continue;
        }

        nb_args = def->nb_iargs + def->nb_oargs;
        if (nb_args == 0) {
            continue;
        }

        /*
         * Macro magic should make it impossible, but double-check that
         * the array index is in range.  Since the signness of an enum
         * is implementation defined, force the result to unsigned.
         */
        unsigned con_set = tcg_target_op_def(op);
        tcg_debug_assert(con_set < ARRAY_SIZE(constraint_sets));
        tdefs = &constraint_sets[con_set];

        for (i = 0; i < nb_args; i++) {
            const char *ct_str = tdefs->args_ct_str[i];
            bool input_p = i >= def->nb_oargs;

            /* Incomplete TCGTargetOpDef entry. */
            tcg_debug_assert(ct_str != NULL);

            switch (*ct_str) {
            case '0' ... '9':
                o = *ct_str - '0';
                tcg_debug_assert(input_p);
                tcg_debug_assert(o < def->nb_oargs);
                tcg_debug_assert(def->args_ct[o].regs != 0);
                tcg_debug_assert(!def->args_ct[o].oalias);
                def->args_ct[i] = def->args_ct[o];
                /* The output sets oalias.  */
                def->args_ct[o].oalias = 1;
                def->args_ct[o].alias_index = i;
                /* The input sets ialias. */
                def->args_ct[i].ialias = 1;
                def->args_ct[i].alias_index = o;
                if (def->args_ct[i].pair) {
                    saw_alias_pair = true;
                }
                tcg_debug_assert(ct_str[1] == '\0');
                continue;

            case '&':
                tcg_debug_assert(!input_p);
                def->args_ct[i].newreg = true;
                ct_str++;
                break;

            case 'p': /* plus */
                /* Allocate to the register after the previous. */
                tcg_debug_assert(i > (input_p ? def->nb_oargs : 0));
                o = i - 1;
                tcg_debug_assert(!def->args_ct[o].pair);
                tcg_debug_assert(!def->args_ct[o].ct);
                def->args_ct[i] = (TCGArgConstraint){
                    .pair = 2,
                    .pair_index = o,
                    .regs = def->args_ct[o].regs << 1,
                };
                def->args_ct[o].pair = 1;
                def->args_ct[o].pair_index = i;
                tcg_debug_assert(ct_str[1] == '\0');
                continue;

            case 'm': /* minus */
                /* Allocate to the register before the previous. */
                tcg_debug_assert(i > (input_p ? def->nb_oargs : 0));
                o = i - 1;
                tcg_debug_assert(!def->args_ct[o].pair);
                tcg_debug_assert(!def->args_ct[o].ct);
                def->args_ct[i] = (TCGArgConstraint){
                    .pair = 1,
                    .pair_index = o,
                    .regs = def->args_ct[o].regs >> 1,
                };
                def->args_ct[o].pair = 2;
                def->args_ct[o].pair_index = i;
                tcg_debug_assert(ct_str[1] == '\0');
                continue;
            }

            do {
                switch (*ct_str) {
                case 'i':
                    def->args_ct[i].ct |= TCG_CT_CONST;
                    break;

                /* Include all of the target-specific constraints. */

#undef CONST
#define CONST(CASE, MASK) \
    case CASE: def->args_ct[i].ct |= MASK; break;
#define REGS(CASE, MASK) \
    case CASE: def->args_ct[i].regs |= MASK; break;

#include "tcg-target-con-str.h"

#undef REGS
#undef CONST
                default:
                case '0' ... '9':
                case '&':
                case 'p':
                case 'm':
                    /* Typo in TCGTargetOpDef constraint. */
                    g_assert_not_reached();
                }
            } while (*++ct_str != '\0');
        }

        /* TCGTargetOpDef entry with too much information? */
        tcg_debug_assert(i == TCG_MAX_OP_ARGS || tdefs->args_ct_str[i] == NULL);

        /*
         * Fix up output pairs that are aliased with inputs.
         * When we created the alias, we copied pair from the output.
         * There are three cases:
         *    (1a) Pairs of inputs alias pairs of outputs.
         *    (1b) One input aliases the first of a pair of outputs.
         *    (2)  One input aliases the second of a pair of outputs.
         *
         * Case 1a is handled by making sure that the pair_index'es are
         * properly updated so that they appear the same as a pair of inputs.
         *
         * Case 1b is handled by setting the pair_index of the input to
         * itself, simply so it doesn't point to an unrelated argument.
         * Since we don't encounter the "second" during the input allocation
         * phase, nothing happens with the second half of the input pair.
         *
         * Case 2 is handled by setting the second input to pair=3, the
         * first output to pair=3, and the pair_index'es to match.
         */
        if (saw_alias_pair) {
            for (i = def->nb_oargs; i < nb_args; i++) {
                /*
                 * Since [0-9pm] must be alone in the constraint string,
                 * the only way they can both be set is if the pair comes
                 * from the output alias.
                 */
                if (!def->args_ct[i].ialias) {
                    continue;
                }
                switch (def->args_ct[i].pair) {
                case 0:
                    break;
                case 1:
                    o = def->args_ct[i].alias_index;
                    o2 = def->args_ct[o].pair_index;
                    tcg_debug_assert(def->args_ct[o].pair == 1);
                    tcg_debug_assert(def->args_ct[o2].pair == 2);
                    if (def->args_ct[o2].oalias) {
                        /* Case 1a */
                        i2 = def->args_ct[o2].alias_index;
                        tcg_debug_assert(def->args_ct[i2].pair == 2);
                        def->args_ct[i2].pair_index = i;
                        def->args_ct[i].pair_index = i2;
                    } else {
                        /* Case 1b */
                        def->args_ct[i].pair_index = i;
                    }
                    break;
                case 2:
                    o = def->args_ct[i].alias_index;
                    o2 = def->args_ct[o].pair_index;
                    tcg_debug_assert(def->args_ct[o].pair == 2);
                    tcg_debug_assert(def->args_ct[o2].pair == 1);
                    if (def->args_ct[o2].oalias) {
                        /* Case 1a */
                        i2 = def->args_ct[o2].alias_index;
                        tcg_debug_assert(def->args_ct[i2].pair == 1);
                        def->args_ct[i2].pair_index = i;
                        def->args_ct[i].pair_index = i2;
                    } else {
                        /* Case 2 */
                        def->args_ct[i].pair = 3;
                        def->args_ct[o2].pair = 3;
                        def->args_ct[i].pair_index = o2;
                        def->args_ct[o2].pair_index = i;
                    }
                    break;
                default:
                    g_assert_not_reached();
                }
            }
        }