qemu/tcg/tcg-op.c
<<
>>
Prefs
   1/*
   2 * Tiny Code Generator for QEMU
   3 *
   4 * Copyright (c) 2008 Fabrice Bellard
   5 *
   6 * Permission is hereby granted, free of charge, to any person obtaining a copy
   7 * of this software and associated documentation files (the "Software"), to deal
   8 * in the Software without restriction, including without limitation the rights
   9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  10 * copies of the Software, and to permit persons to whom the Software is
  11 * furnished to do so, subject to the following conditions:
  12 *
  13 * The above copyright notice and this permission notice shall be included in
  14 * all copies or substantial portions of the Software.
  15 *
  16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
  19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  22 * THE SOFTWARE.
  23 */
  24
  25#include "qemu/osdep.h"
  26#include "cpu.h"
  27#include "exec/exec-all.h"
  28#include "tcg.h"
  29#include "tcg-op.h"
  30#include "tcg-mo.h"
  31#include "trace-tcg.h"
  32#include "trace/mem.h"
  33#include "exec/plugin-gen.h"
  34
  35/* Reduce the number of ifdefs below.  This assumes that all uses of
  36   TCGV_HIGH and TCGV_LOW are properly protected by a conditional that
  37   the compiler can eliminate.  */
  38#if TCG_TARGET_REG_BITS == 64
  39extern TCGv_i32 TCGV_LOW_link_error(TCGv_i64);
  40extern TCGv_i32 TCGV_HIGH_link_error(TCGv_i64);
  41#define TCGV_LOW  TCGV_LOW_link_error
  42#define TCGV_HIGH TCGV_HIGH_link_error
  43#endif
  44
  45void tcg_gen_op1(TCGOpcode opc, TCGArg a1)
  46{
  47    TCGOp *op = tcg_emit_op(opc);
  48    op->args[0] = a1;
  49}
  50
  51void tcg_gen_op2(TCGOpcode opc, TCGArg a1, TCGArg a2)
  52{
  53    TCGOp *op = tcg_emit_op(opc);
  54    op->args[0] = a1;
  55    op->args[1] = a2;
  56}
  57
  58void tcg_gen_op3(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3)
  59{
  60    TCGOp *op = tcg_emit_op(opc);
  61    op->args[0] = a1;
  62    op->args[1] = a2;
  63    op->args[2] = a3;
  64}
  65
  66void tcg_gen_op4(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3, TCGArg a4)
  67{
  68    TCGOp *op = tcg_emit_op(opc);
  69    op->args[0] = a1;
  70    op->args[1] = a2;
  71    op->args[2] = a3;
  72    op->args[3] = a4;
  73}
  74
  75void tcg_gen_op5(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3,
  76                 TCGArg a4, TCGArg a5)
  77{
  78    TCGOp *op = tcg_emit_op(opc);
  79    op->args[0] = a1;
  80    op->args[1] = a2;
  81    op->args[2] = a3;
  82    op->args[3] = a4;
  83    op->args[4] = a5;
  84}
  85
  86void tcg_gen_op6(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3,
  87                 TCGArg a4, TCGArg a5, TCGArg a6)
  88{
  89    TCGOp *op = tcg_emit_op(opc);
  90    op->args[0] = a1;
  91    op->args[1] = a2;
  92    op->args[2] = a3;
  93    op->args[3] = a4;
  94    op->args[4] = a5;
  95    op->args[5] = a6;
  96}
  97
  98void tcg_gen_mb(TCGBar mb_type)
  99{
 100    if (tcg_ctx->tb_cflags & CF_PARALLEL) {
 101        tcg_gen_op1(INDEX_op_mb, mb_type);
 102    }
 103}
 104
 105/* 32 bit ops */
 106
 107void tcg_gen_addi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
 108{
 109    /* some cases can be optimized here */
 110    if (arg2 == 0) {
 111        tcg_gen_mov_i32(ret, arg1);
 112    } else {
 113        TCGv_i32 t0 = tcg_const_i32(arg2);
 114        tcg_gen_add_i32(ret, arg1, t0);
 115        tcg_temp_free_i32(t0);
 116    }
 117}
 118
 119void tcg_gen_subfi_i32(TCGv_i32 ret, int32_t arg1, TCGv_i32 arg2)
 120{
 121    if (arg1 == 0 && TCG_TARGET_HAS_neg_i32) {
 122        /* Don't recurse with tcg_gen_neg_i32.  */
 123        tcg_gen_op2_i32(INDEX_op_neg_i32, ret, arg2);
 124    } else {
 125        TCGv_i32 t0 = tcg_const_i32(arg1);
 126        tcg_gen_sub_i32(ret, t0, arg2);
 127        tcg_temp_free_i32(t0);
 128    }
 129}
 130
 131void tcg_gen_subi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
 132{
 133    /* some cases can be optimized here */
 134    if (arg2 == 0) {
 135        tcg_gen_mov_i32(ret, arg1);
 136    } else {
 137        TCGv_i32 t0 = tcg_const_i32(arg2);
 138        tcg_gen_sub_i32(ret, arg1, t0);
 139        tcg_temp_free_i32(t0);
 140    }
 141}
 142
 143void tcg_gen_andi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
 144{
 145    TCGv_i32 t0;
 146    /* Some cases can be optimized here.  */
 147    switch (arg2) {
 148    case 0:
 149        tcg_gen_movi_i32(ret, 0);
 150        return;
 151    case -1:
 152        tcg_gen_mov_i32(ret, arg1);
 153        return;
 154    case 0xff:
 155        /* Don't recurse with tcg_gen_ext8u_i32.  */
 156        if (TCG_TARGET_HAS_ext8u_i32) {
 157            tcg_gen_op2_i32(INDEX_op_ext8u_i32, ret, arg1);
 158            return;
 159        }
 160        break;
 161    case 0xffff:
 162        if (TCG_TARGET_HAS_ext16u_i32) {
 163            tcg_gen_op2_i32(INDEX_op_ext16u_i32, ret, arg1);
 164            return;
 165        }
 166        break;
 167    }
 168    t0 = tcg_const_i32(arg2);
 169    tcg_gen_and_i32(ret, arg1, t0);
 170    tcg_temp_free_i32(t0);
 171}
 172
 173void tcg_gen_ori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
 174{
 175    /* Some cases can be optimized here.  */
 176    if (arg2 == -1) {
 177        tcg_gen_movi_i32(ret, -1);
 178    } else if (arg2 == 0) {
 179        tcg_gen_mov_i32(ret, arg1);
 180    } else {
 181        TCGv_i32 t0 = tcg_const_i32(arg2);
 182        tcg_gen_or_i32(ret, arg1, t0);
 183        tcg_temp_free_i32(t0);
 184    }
 185}
 186
 187void tcg_gen_xori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
 188{
 189    /* Some cases can be optimized here.  */
 190    if (arg2 == 0) {
 191        tcg_gen_mov_i32(ret, arg1);
 192    } else if (arg2 == -1 && TCG_TARGET_HAS_not_i32) {
 193        /* Don't recurse with tcg_gen_not_i32.  */
 194        tcg_gen_op2_i32(INDEX_op_not_i32, ret, arg1);
 195    } else {
 196        TCGv_i32 t0 = tcg_const_i32(arg2);
 197        tcg_gen_xor_i32(ret, arg1, t0);
 198        tcg_temp_free_i32(t0);
 199    }
 200}
 201
 202void tcg_gen_shli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
 203{
 204    tcg_debug_assert(arg2 >= 0 && arg2 < 32);
 205    if (arg2 == 0) {
 206        tcg_gen_mov_i32(ret, arg1);
 207    } else {
 208        TCGv_i32 t0 = tcg_const_i32(arg2);
 209        tcg_gen_shl_i32(ret, arg1, t0);
 210        tcg_temp_free_i32(t0);
 211    }
 212}
 213
 214void tcg_gen_shri_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
 215{
 216    tcg_debug_assert(arg2 >= 0 && arg2 < 32);
 217    if (arg2 == 0) {
 218        tcg_gen_mov_i32(ret, arg1);
 219    } else {
 220        TCGv_i32 t0 = tcg_const_i32(arg2);
 221        tcg_gen_shr_i32(ret, arg1, t0);
 222        tcg_temp_free_i32(t0);
 223    }
 224}
 225
 226void tcg_gen_sari_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
 227{
 228    tcg_debug_assert(arg2 >= 0 && arg2 < 32);
 229    if (arg2 == 0) {
 230        tcg_gen_mov_i32(ret, arg1);
 231    } else {
 232        TCGv_i32 t0 = tcg_const_i32(arg2);
 233        tcg_gen_sar_i32(ret, arg1, t0);
 234        tcg_temp_free_i32(t0);
 235    }
 236}
 237
 238void tcg_gen_brcond_i32(TCGCond cond, TCGv_i32 arg1, TCGv_i32 arg2, TCGLabel *l)
 239{
 240    if (cond == TCG_COND_ALWAYS) {
 241        tcg_gen_br(l);
 242    } else if (cond != TCG_COND_NEVER) {
 243        l->refs++;
 244        tcg_gen_op4ii_i32(INDEX_op_brcond_i32, arg1, arg2, cond, label_arg(l));
 245    }
 246}
 247
 248void tcg_gen_brcondi_i32(TCGCond cond, TCGv_i32 arg1, int32_t arg2, TCGLabel *l)
 249{
 250    if (cond == TCG_COND_ALWAYS) {
 251        tcg_gen_br(l);
 252    } else if (cond != TCG_COND_NEVER) {
 253        TCGv_i32 t0 = tcg_const_i32(arg2);
 254        tcg_gen_brcond_i32(cond, arg1, t0, l);
 255        tcg_temp_free_i32(t0);
 256    }
 257}
 258
 259void tcg_gen_setcond_i32(TCGCond cond, TCGv_i32 ret,
 260                         TCGv_i32 arg1, TCGv_i32 arg2)
 261{
 262    if (cond == TCG_COND_ALWAYS) {
 263        tcg_gen_movi_i32(ret, 1);
 264    } else if (cond == TCG_COND_NEVER) {
 265        tcg_gen_movi_i32(ret, 0);
 266    } else {
 267        tcg_gen_op4i_i32(INDEX_op_setcond_i32, ret, arg1, arg2, cond);
 268    }
 269}
 270
 271void tcg_gen_setcondi_i32(TCGCond cond, TCGv_i32 ret,
 272                          TCGv_i32 arg1, int32_t arg2)
 273{
 274    TCGv_i32 t0 = tcg_const_i32(arg2);
 275    tcg_gen_setcond_i32(cond, ret, arg1, t0);
 276    tcg_temp_free_i32(t0);
 277}
 278
 279void tcg_gen_muli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
 280{
 281    if (arg2 == 0) {
 282        tcg_gen_movi_i32(ret, 0);
 283    } else if (is_power_of_2(arg2)) {
 284        tcg_gen_shli_i32(ret, arg1, ctz32(arg2));
 285    } else {
 286        TCGv_i32 t0 = tcg_const_i32(arg2);
 287        tcg_gen_mul_i32(ret, arg1, t0);
 288        tcg_temp_free_i32(t0);
 289    }
 290}
 291
 292void tcg_gen_div_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
 293{
 294    if (TCG_TARGET_HAS_div_i32) {
 295        tcg_gen_op3_i32(INDEX_op_div_i32, ret, arg1, arg2);
 296    } else if (TCG_TARGET_HAS_div2_i32) {
 297        TCGv_i32 t0 = tcg_temp_new_i32();
 298        tcg_gen_sari_i32(t0, arg1, 31);
 299        tcg_gen_op5_i32(INDEX_op_div2_i32, ret, t0, arg1, t0, arg2);
 300        tcg_temp_free_i32(t0);
 301    } else {
 302        gen_helper_div_i32(ret, arg1, arg2);
 303    }
 304}
 305
 306void tcg_gen_rem_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
 307{
 308    if (TCG_TARGET_HAS_rem_i32) {
 309        tcg_gen_op3_i32(INDEX_op_rem_i32, ret, arg1, arg2);
 310    } else if (TCG_TARGET_HAS_div_i32) {
 311        TCGv_i32 t0 = tcg_temp_new_i32();
 312        tcg_gen_op3_i32(INDEX_op_div_i32, t0, arg1, arg2);
 313        tcg_gen_mul_i32(t0, t0, arg2);
 314        tcg_gen_sub_i32(ret, arg1, t0);
 315        tcg_temp_free_i32(t0);
 316    } else if (TCG_TARGET_HAS_div2_i32) {
 317        TCGv_i32 t0 = tcg_temp_new_i32();
 318        tcg_gen_sari_i32(t0, arg1, 31);
 319        tcg_gen_op5_i32(INDEX_op_div2_i32, t0, ret, arg1, t0, arg2);
 320        tcg_temp_free_i32(t0);
 321    } else {
 322        gen_helper_rem_i32(ret, arg1, arg2);
 323    }
 324}
 325
 326void tcg_gen_divu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
 327{
 328    if (TCG_TARGET_HAS_div_i32) {
 329        tcg_gen_op3_i32(INDEX_op_divu_i32, ret, arg1, arg2);
 330    } else if (TCG_TARGET_HAS_div2_i32) {
 331        TCGv_i32 t0 = tcg_temp_new_i32();
 332        tcg_gen_movi_i32(t0, 0);
 333        tcg_gen_op5_i32(INDEX_op_divu2_i32, ret, t0, arg1, t0, arg2);
 334        tcg_temp_free_i32(t0);
 335    } else {
 336        gen_helper_divu_i32(ret, arg1, arg2);
 337    }
 338}
 339
 340void tcg_gen_remu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
 341{
 342    if (TCG_TARGET_HAS_rem_i32) {
 343        tcg_gen_op3_i32(INDEX_op_remu_i32, ret, arg1, arg2);
 344    } else if (TCG_TARGET_HAS_div_i32) {
 345        TCGv_i32 t0 = tcg_temp_new_i32();
 346        tcg_gen_op3_i32(INDEX_op_divu_i32, t0, arg1, arg2);
 347        tcg_gen_mul_i32(t0, t0, arg2);
 348        tcg_gen_sub_i32(ret, arg1, t0);
 349        tcg_temp_free_i32(t0);
 350    } else if (TCG_TARGET_HAS_div2_i32) {
 351        TCGv_i32 t0 = tcg_temp_new_i32();
 352        tcg_gen_movi_i32(t0, 0);
 353        tcg_gen_op5_i32(INDEX_op_divu2_i32, t0, ret, arg1, t0, arg2);
 354        tcg_temp_free_i32(t0);
 355    } else {
 356        gen_helper_remu_i32(ret, arg1, arg2);
 357    }
 358}
 359
 360void tcg_gen_andc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
 361{
 362    if (TCG_TARGET_HAS_andc_i32) {
 363        tcg_gen_op3_i32(INDEX_op_andc_i32, ret, arg1, arg2);
 364    } else {
 365        TCGv_i32 t0 = tcg_temp_new_i32();
 366        tcg_gen_not_i32(t0, arg2);
 367        tcg_gen_and_i32(ret, arg1, t0);
 368        tcg_temp_free_i32(t0);
 369    }
 370}
 371
 372void tcg_gen_eqv_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
 373{
 374    if (TCG_TARGET_HAS_eqv_i32) {
 375        tcg_gen_op3_i32(INDEX_op_eqv_i32, ret, arg1, arg2);
 376    } else {
 377        tcg_gen_xor_i32(ret, arg1, arg2);
 378        tcg_gen_not_i32(ret, ret);
 379    }
 380}
 381
 382void tcg_gen_nand_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
 383{
 384    if (TCG_TARGET_HAS_nand_i32) {
 385        tcg_gen_op3_i32(INDEX_op_nand_i32, ret, arg1, arg2);
 386    } else {
 387        tcg_gen_and_i32(ret, arg1, arg2);
 388        tcg_gen_not_i32(ret, ret);
 389    }
 390}
 391
 392void tcg_gen_nor_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
 393{
 394    if (TCG_TARGET_HAS_nor_i32) {
 395        tcg_gen_op3_i32(INDEX_op_nor_i32, ret, arg1, arg2);
 396    } else {
 397        tcg_gen_or_i32(ret, arg1, arg2);
 398        tcg_gen_not_i32(ret, ret);
 399    }
 400}
 401
 402void tcg_gen_orc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
 403{
 404    if (TCG_TARGET_HAS_orc_i32) {
 405        tcg_gen_op3_i32(INDEX_op_orc_i32, ret, arg1, arg2);
 406    } else {
 407        TCGv_i32 t0 = tcg_temp_new_i32();
 408        tcg_gen_not_i32(t0, arg2);
 409        tcg_gen_or_i32(ret, arg1, t0);
 410        tcg_temp_free_i32(t0);
 411    }
 412}
 413
 414void tcg_gen_clz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
 415{
 416    if (TCG_TARGET_HAS_clz_i32) {
 417        tcg_gen_op3_i32(INDEX_op_clz_i32, ret, arg1, arg2);
 418    } else if (TCG_TARGET_HAS_clz_i64) {
 419        TCGv_i64 t1 = tcg_temp_new_i64();
 420        TCGv_i64 t2 = tcg_temp_new_i64();
 421        tcg_gen_extu_i32_i64(t1, arg1);
 422        tcg_gen_extu_i32_i64(t2, arg2);
 423        tcg_gen_addi_i64(t2, t2, 32);
 424        tcg_gen_clz_i64(t1, t1, t2);
 425        tcg_gen_extrl_i64_i32(ret, t1);
 426        tcg_temp_free_i64(t1);
 427        tcg_temp_free_i64(t2);
 428        tcg_gen_subi_i32(ret, ret, 32);
 429    } else {
 430        gen_helper_clz_i32(ret, arg1, arg2);
 431    }
 432}
 433
 434void tcg_gen_clzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2)
 435{
 436    TCGv_i32 t = tcg_const_i32(arg2);
 437    tcg_gen_clz_i32(ret, arg1, t);
 438    tcg_temp_free_i32(t);
 439}
 440
 441void tcg_gen_ctz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
 442{
 443    if (TCG_TARGET_HAS_ctz_i32) {
 444        tcg_gen_op3_i32(INDEX_op_ctz_i32, ret, arg1, arg2);
 445    } else if (TCG_TARGET_HAS_ctz_i64) {
 446        TCGv_i64 t1 = tcg_temp_new_i64();
 447        TCGv_i64 t2 = tcg_temp_new_i64();
 448        tcg_gen_extu_i32_i64(t1, arg1);
 449        tcg_gen_extu_i32_i64(t2, arg2);
 450        tcg_gen_ctz_i64(t1, t1, t2);
 451        tcg_gen_extrl_i64_i32(ret, t1);
 452        tcg_temp_free_i64(t1);
 453        tcg_temp_free_i64(t2);
 454    } else if (TCG_TARGET_HAS_ctpop_i32
 455               || TCG_TARGET_HAS_ctpop_i64
 456               || TCG_TARGET_HAS_clz_i32
 457               || TCG_TARGET_HAS_clz_i64) {
 458        TCGv_i32 z, t = tcg_temp_new_i32();
 459
 460        if (TCG_TARGET_HAS_ctpop_i32 || TCG_TARGET_HAS_ctpop_i64) {
 461            tcg_gen_subi_i32(t, arg1, 1);
 462            tcg_gen_andc_i32(t, t, arg1);
 463            tcg_gen_ctpop_i32(t, t);
 464        } else {
 465            /* Since all non-x86 hosts have clz(0) == 32, don't fight it.  */
 466            tcg_gen_neg_i32(t, arg1);
 467            tcg_gen_and_i32(t, t, arg1);
 468            tcg_gen_clzi_i32(t, t, 32);
 469            tcg_gen_xori_i32(t, t, 31);
 470        }
 471        z = tcg_const_i32(0);
 472        tcg_gen_movcond_i32(TCG_COND_EQ, ret, arg1, z, arg2, t);
 473        tcg_temp_free_i32(t);
 474        tcg_temp_free_i32(z);
 475    } else {
 476        gen_helper_ctz_i32(ret, arg1, arg2);
 477    }
 478}
 479
 480void tcg_gen_ctzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2)
 481{
 482    if (!TCG_TARGET_HAS_ctz_i32 && TCG_TARGET_HAS_ctpop_i32 && arg2 == 32) {
 483        /* This equivalence has the advantage of not requiring a fixup.  */
 484        TCGv_i32 t = tcg_temp_new_i32();
 485        tcg_gen_subi_i32(t, arg1, 1);
 486        tcg_gen_andc_i32(t, t, arg1);
 487        tcg_gen_ctpop_i32(ret, t);
 488        tcg_temp_free_i32(t);
 489    } else {
 490        TCGv_i32 t = tcg_const_i32(arg2);
 491        tcg_gen_ctz_i32(ret, arg1, t);
 492        tcg_temp_free_i32(t);
 493    }
 494}
 495
 496void tcg_gen_clrsb_i32(TCGv_i32 ret, TCGv_i32 arg)
 497{
 498    if (TCG_TARGET_HAS_clz_i32) {
 499        TCGv_i32 t = tcg_temp_new_i32();
 500        tcg_gen_sari_i32(t, arg, 31);
 501        tcg_gen_xor_i32(t, t, arg);
 502        tcg_gen_clzi_i32(t, t, 32);
 503        tcg_gen_subi_i32(ret, t, 1);
 504        tcg_temp_free_i32(t);
 505    } else {
 506        gen_helper_clrsb_i32(ret, arg);
 507    }
 508}
 509
 510void tcg_gen_ctpop_i32(TCGv_i32 ret, TCGv_i32 arg1)
 511{
 512    if (TCG_TARGET_HAS_ctpop_i32) {
 513        tcg_gen_op2_i32(INDEX_op_ctpop_i32, ret, arg1);
 514    } else if (TCG_TARGET_HAS_ctpop_i64) {
 515        TCGv_i64 t = tcg_temp_new_i64();
 516        tcg_gen_extu_i32_i64(t, arg1);
 517        tcg_gen_ctpop_i64(t, t);
 518        tcg_gen_extrl_i64_i32(ret, t);
 519        tcg_temp_free_i64(t);
 520    } else {
 521        gen_helper_ctpop_i32(ret, arg1);
 522    }
 523}
 524
 525void tcg_gen_rotl_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
 526{
 527    if (TCG_TARGET_HAS_rot_i32) {
 528        tcg_gen_op3_i32(INDEX_op_rotl_i32, ret, arg1, arg2);
 529    } else {
 530        TCGv_i32 t0, t1;
 531
 532        t0 = tcg_temp_new_i32();
 533        t1 = tcg_temp_new_i32();
 534        tcg_gen_shl_i32(t0, arg1, arg2);
 535        tcg_gen_subfi_i32(t1, 32, arg2);
 536        tcg_gen_shr_i32(t1, arg1, t1);
 537        tcg_gen_or_i32(ret, t0, t1);
 538        tcg_temp_free_i32(t0);
 539        tcg_temp_free_i32(t1);
 540    }
 541}
 542
 543void tcg_gen_rotli_i32(TCGv_i32 ret, TCGv_i32 arg1, unsigned arg2)
 544{
 545    tcg_debug_assert(arg2 < 32);
 546    /* some cases can be optimized here */
 547    if (arg2 == 0) {
 548        tcg_gen_mov_i32(ret, arg1);
 549    } else if (TCG_TARGET_HAS_rot_i32) {
 550        TCGv_i32 t0 = tcg_const_i32(arg2);
 551        tcg_gen_rotl_i32(ret, arg1, t0);
 552        tcg_temp_free_i32(t0);
 553    } else {
 554        TCGv_i32 t0, t1;
 555        t0 = tcg_temp_new_i32();
 556        t1 = tcg_temp_new_i32();
 557        tcg_gen_shli_i32(t0, arg1, arg2);
 558        tcg_gen_shri_i32(t1, arg1, 32 - arg2);
 559        tcg_gen_or_i32(ret, t0, t1);
 560        tcg_temp_free_i32(t0);
 561        tcg_temp_free_i32(t1);
 562    }
 563}
 564
 565void tcg_gen_rotr_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
 566{
 567    if (TCG_TARGET_HAS_rot_i32) {
 568        tcg_gen_op3_i32(INDEX_op_rotr_i32, ret, arg1, arg2);
 569    } else {
 570        TCGv_i32 t0, t1;
 571
 572        t0 = tcg_temp_new_i32();
 573        t1 = tcg_temp_new_i32();
 574        tcg_gen_shr_i32(t0, arg1, arg2);
 575        tcg_gen_subfi_i32(t1, 32, arg2);
 576        tcg_gen_shl_i32(t1, arg1, t1);
 577        tcg_gen_or_i32(ret, t0, t1);
 578        tcg_temp_free_i32(t0);
 579        tcg_temp_free_i32(t1);
 580    }
 581}
 582
 583void tcg_gen_rotri_i32(TCGv_i32 ret, TCGv_i32 arg1, unsigned arg2)
 584{
 585    tcg_debug_assert(arg2 < 32);
 586    /* some cases can be optimized here */
 587    if (arg2 == 0) {
 588        tcg_gen_mov_i32(ret, arg1);
 589    } else {
 590        tcg_gen_rotli_i32(ret, arg1, 32 - arg2);
 591    }
 592}
 593
 594void tcg_gen_deposit_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2,
 595                         unsigned int ofs, unsigned int len)
 596{
 597    uint32_t mask;
 598    TCGv_i32 t1;
 599
 600    tcg_debug_assert(ofs < 32);
 601    tcg_debug_assert(len > 0);
 602    tcg_debug_assert(len <= 32);
 603    tcg_debug_assert(ofs + len <= 32);
 604
 605    if (len == 32) {
 606        tcg_gen_mov_i32(ret, arg2);
 607        return;
 608    }
 609    if (TCG_TARGET_HAS_deposit_i32 && TCG_TARGET_deposit_i32_valid(ofs, len)) {
 610        tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, arg1, arg2, ofs, len);
 611        return;
 612    }
 613
 614    t1 = tcg_temp_new_i32();
 615
 616    if (TCG_TARGET_HAS_extract2_i32) {
 617        if (ofs + len == 32) {
 618            tcg_gen_shli_i32(t1, arg1, len);
 619            tcg_gen_extract2_i32(ret, t1, arg2, len);
 620            goto done;
 621        }
 622        if (ofs == 0) {
 623            tcg_gen_extract2_i32(ret, arg1, arg2, len);
 624            tcg_gen_rotli_i32(ret, ret, len);
 625            goto done;
 626        }
 627    }
 628
 629    mask = (1u << len) - 1;
 630    if (ofs + len < 32) {
 631        tcg_gen_andi_i32(t1, arg2, mask);
 632        tcg_gen_shli_i32(t1, t1, ofs);
 633    } else {
 634        tcg_gen_shli_i32(t1, arg2, ofs);
 635    }
 636    tcg_gen_andi_i32(ret, arg1, ~(mask << ofs));
 637    tcg_gen_or_i32(ret, ret, t1);
 638 done:
 639    tcg_temp_free_i32(t1);
 640}
 641
 642void tcg_gen_deposit_z_i32(TCGv_i32 ret, TCGv_i32 arg,
 643                           unsigned int ofs, unsigned int len)
 644{
 645    tcg_debug_assert(ofs < 32);
 646    tcg_debug_assert(len > 0);
 647    tcg_debug_assert(len <= 32);
 648    tcg_debug_assert(ofs + len <= 32);
 649
 650    if (ofs + len == 32) {
 651        tcg_gen_shli_i32(ret, arg, ofs);
 652    } else if (ofs == 0) {
 653        tcg_gen_andi_i32(ret, arg, (1u << len) - 1);
 654    } else if (TCG_TARGET_HAS_deposit_i32
 655               && TCG_TARGET_deposit_i32_valid(ofs, len)) {
 656        TCGv_i32 zero = tcg_const_i32(0);
 657        tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, zero, arg, ofs, len);
 658        tcg_temp_free_i32(zero);
 659    } else {
 660        /* To help two-operand hosts we prefer to zero-extend first,
 661           which allows ARG to stay live.  */
 662        switch (len) {
 663        case 16:
 664            if (TCG_TARGET_HAS_ext16u_i32) {
 665                tcg_gen_ext16u_i32(ret, arg);
 666                tcg_gen_shli_i32(ret, ret, ofs);
 667                return;
 668            }
 669            break;
 670        case 8:
 671            if (TCG_TARGET_HAS_ext8u_i32) {
 672                tcg_gen_ext8u_i32(ret, arg);
 673                tcg_gen_shli_i32(ret, ret, ofs);
 674                return;
 675            }
 676            break;
 677        }
 678        /* Otherwise prefer zero-extension over AND for code size.  */
 679        switch (ofs + len) {
 680        case 16:
 681            if (TCG_TARGET_HAS_ext16u_i32) {
 682                tcg_gen_shli_i32(ret, arg, ofs);
 683                tcg_gen_ext16u_i32(ret, ret);
 684                return;
 685            }
 686            break;
 687        case 8:
 688            if (TCG_TARGET_HAS_ext8u_i32) {
 689                tcg_gen_shli_i32(ret, arg, ofs);
 690                tcg_gen_ext8u_i32(ret, ret);
 691                return;
 692            }
 693            break;
 694        }
 695        tcg_gen_andi_i32(ret, arg, (1u << len) - 1);
 696        tcg_gen_shli_i32(ret, ret, ofs);
 697    }
 698}
 699
 700void tcg_gen_extract_i32(TCGv_i32 ret, TCGv_i32 arg,
 701                         unsigned int ofs, unsigned int len)
 702{
 703    tcg_debug_assert(ofs < 32);
 704    tcg_debug_assert(len > 0);
 705    tcg_debug_assert(len <= 32);
 706    tcg_debug_assert(ofs + len <= 32);
 707
 708    /* Canonicalize certain special cases, even if extract is supported.  */
 709    if (ofs + len == 32) {
 710        tcg_gen_shri_i32(ret, arg, 32 - len);
 711        return;
 712    }
 713    if (ofs == 0) {
 714        tcg_gen_andi_i32(ret, arg, (1u << len) - 1);
 715        return;
 716    }
 717
 718    if (TCG_TARGET_HAS_extract_i32
 719        && TCG_TARGET_extract_i32_valid(ofs, len)) {
 720        tcg_gen_op4ii_i32(INDEX_op_extract_i32, ret, arg, ofs, len);
 721        return;
 722    }
 723
 724    /* Assume that zero-extension, if available, is cheaper than a shift.  */
 725    switch (ofs + len) {
 726    case 16:
 727        if (TCG_TARGET_HAS_ext16u_i32) {
 728            tcg_gen_ext16u_i32(ret, arg);
 729            tcg_gen_shri_i32(ret, ret, ofs);
 730            return;
 731        }
 732        break;
 733    case 8:
 734        if (TCG_TARGET_HAS_ext8u_i32) {
 735            tcg_gen_ext8u_i32(ret, arg);
 736            tcg_gen_shri_i32(ret, ret, ofs);
 737            return;
 738        }
 739        break;
 740    }
 741
 742    /* ??? Ideally we'd know what values are available for immediate AND.
 743       Assume that 8 bits are available, plus the special case of 16,
 744       so that we get ext8u, ext16u.  */
 745    switch (len) {
 746    case 1 ... 8: case 16:
 747        tcg_gen_shri_i32(ret, arg, ofs);
 748        tcg_gen_andi_i32(ret, ret, (1u << len) - 1);
 749        break;
 750    default:
 751        tcg_gen_shli_i32(ret, arg, 32 - len - ofs);
 752        tcg_gen_shri_i32(ret, ret, 32 - len);
 753        break;
 754    }
 755}
 756
 757void tcg_gen_sextract_i32(TCGv_i32 ret, TCGv_i32 arg,
 758                          unsigned int ofs, unsigned int len)
 759{
 760    tcg_debug_assert(ofs < 32);
 761    tcg_debug_assert(len > 0);
 762    tcg_debug_assert(len <= 32);
 763    tcg_debug_assert(ofs + len <= 32);
 764
 765    /* Canonicalize certain special cases, even if extract is supported.  */
 766    if (ofs + len == 32) {
 767        tcg_gen_sari_i32(ret, arg, 32 - len);
 768        return;
 769    }
 770    if (ofs == 0) {
 771        switch (len) {
 772        case 16:
 773            tcg_gen_ext16s_i32(ret, arg);
 774            return;
 775        case 8:
 776            tcg_gen_ext8s_i32(ret, arg);
 777            return;
 778        }
 779    }
 780
 781    if (TCG_TARGET_HAS_sextract_i32
 782        && TCG_TARGET_extract_i32_valid(ofs, len)) {
 783        tcg_gen_op4ii_i32(INDEX_op_sextract_i32, ret, arg, ofs, len);
 784        return;
 785    }
 786
 787    /* Assume that sign-extension, if available, is cheaper than a shift.  */
 788    switch (ofs + len) {
 789    case 16:
 790        if (TCG_TARGET_HAS_ext16s_i32) {
 791            tcg_gen_ext16s_i32(ret, arg);
 792            tcg_gen_sari_i32(ret, ret, ofs);
 793            return;
 794        }
 795        break;
 796    case 8:
 797        if (TCG_TARGET_HAS_ext8s_i32) {
 798            tcg_gen_ext8s_i32(ret, arg);
 799            tcg_gen_sari_i32(ret, ret, ofs);
 800            return;
 801        }
 802        break;
 803    }
 804    switch (len) {
 805    case 16:
 806        if (TCG_TARGET_HAS_ext16s_i32) {
 807            tcg_gen_shri_i32(ret, arg, ofs);
 808            tcg_gen_ext16s_i32(ret, ret);
 809            return;
 810        }
 811        break;
 812    case 8:
 813        if (TCG_TARGET_HAS_ext8s_i32) {
 814            tcg_gen_shri_i32(ret, arg, ofs);
 815            tcg_gen_ext8s_i32(ret, ret);
 816            return;
 817        }
 818        break;
 819    }
 820
 821    tcg_gen_shli_i32(ret, arg, 32 - len - ofs);
 822    tcg_gen_sari_i32(ret, ret, 32 - len);
 823}
 824
 825/*
 826 * Extract 32-bits from a 64-bit input, ah:al, starting from ofs.
 827 * Unlike tcg_gen_extract_i32 above, len is fixed at 32.
 828 */
 829void tcg_gen_extract2_i32(TCGv_i32 ret, TCGv_i32 al, TCGv_i32 ah,
 830                          unsigned int ofs)
 831{
 832    tcg_debug_assert(ofs <= 32);
 833    if (ofs == 0) {
 834        tcg_gen_mov_i32(ret, al);
 835    } else if (ofs == 32) {
 836        tcg_gen_mov_i32(ret, ah);
 837    } else if (al == ah) {
 838        tcg_gen_rotri_i32(ret, al, ofs);
 839    } else if (TCG_TARGET_HAS_extract2_i32) {
 840        tcg_gen_op4i_i32(INDEX_op_extract2_i32, ret, al, ah, ofs);
 841    } else {
 842        TCGv_i32 t0 = tcg_temp_new_i32();
 843        tcg_gen_shri_i32(t0, al, ofs);
 844        tcg_gen_deposit_i32(ret, t0, ah, 32 - ofs, ofs);
 845        tcg_temp_free_i32(t0);
 846    }
 847}
 848
 849void tcg_gen_movcond_i32(TCGCond cond, TCGv_i32 ret, TCGv_i32 c1,
 850                         TCGv_i32 c2, TCGv_i32 v1, TCGv_i32 v2)
 851{
 852    if (cond == TCG_COND_ALWAYS) {
 853        tcg_gen_mov_i32(ret, v1);
 854    } else if (cond == TCG_COND_NEVER) {
 855        tcg_gen_mov_i32(ret, v2);
 856    } else if (TCG_TARGET_HAS_movcond_i32) {
 857        tcg_gen_op6i_i32(INDEX_op_movcond_i32, ret, c1, c2, v1, v2, cond);
 858    } else {
 859        TCGv_i32 t0 = tcg_temp_new_i32();
 860        TCGv_i32 t1 = tcg_temp_new_i32();
 861        tcg_gen_setcond_i32(cond, t0, c1, c2);
 862        tcg_gen_neg_i32(t0, t0);
 863        tcg_gen_and_i32(t1, v1, t0);
 864        tcg_gen_andc_i32(ret, v2, t0);
 865        tcg_gen_or_i32(ret, ret, t1);
 866        tcg_temp_free_i32(t0);
 867        tcg_temp_free_i32(t1);
 868    }
 869}
 870
 871void tcg_gen_add2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al,
 872                      TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh)
 873{
 874    if (TCG_TARGET_HAS_add2_i32) {
 875        tcg_gen_op6_i32(INDEX_op_add2_i32, rl, rh, al, ah, bl, bh);
 876    } else {
 877        TCGv_i64 t0 = tcg_temp_new_i64();
 878        TCGv_i64 t1 = tcg_temp_new_i64();
 879        tcg_gen_concat_i32_i64(t0, al, ah);
 880        tcg_gen_concat_i32_i64(t1, bl, bh);
 881        tcg_gen_add_i64(t0, t0, t1);
 882        tcg_gen_extr_i64_i32(rl, rh, t0);
 883        tcg_temp_free_i64(t0);
 884        tcg_temp_free_i64(t1);
 885    }
 886}
 887
 888void tcg_gen_sub2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al,
 889                      TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh)
 890{
 891    if (TCG_TARGET_HAS_sub2_i32) {
 892        tcg_gen_op6_i32(INDEX_op_sub2_i32, rl, rh, al, ah, bl, bh);
 893    } else {
 894        TCGv_i64 t0 = tcg_temp_new_i64();
 895        TCGv_i64 t1 = tcg_temp_new_i64();
 896        tcg_gen_concat_i32_i64(t0, al, ah);
 897        tcg_gen_concat_i32_i64(t1, bl, bh);
 898        tcg_gen_sub_i64(t0, t0, t1);
 899        tcg_gen_extr_i64_i32(rl, rh, t0);
 900        tcg_temp_free_i64(t0);
 901        tcg_temp_free_i64(t1);
 902    }
 903}
 904
 905void tcg_gen_mulu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2)
 906{
 907    if (TCG_TARGET_HAS_mulu2_i32) {
 908        tcg_gen_op4_i32(INDEX_op_mulu2_i32, rl, rh, arg1, arg2);
 909    } else if (TCG_TARGET_HAS_muluh_i32) {
 910        TCGv_i32 t = tcg_temp_new_i32();
 911        tcg_gen_op3_i32(INDEX_op_mul_i32, t, arg1, arg2);
 912        tcg_gen_op3_i32(INDEX_op_muluh_i32, rh, arg1, arg2);
 913        tcg_gen_mov_i32(rl, t);
 914        tcg_temp_free_i32(t);
 915    } else {
 916        TCGv_i64 t0 = tcg_temp_new_i64();
 917        TCGv_i64 t1 = tcg_temp_new_i64();
 918        tcg_gen_extu_i32_i64(t0, arg1);
 919        tcg_gen_extu_i32_i64(t1, arg2);
 920        tcg_gen_mul_i64(t0, t0, t1);
 921        tcg_gen_extr_i64_i32(rl, rh, t0);
 922        tcg_temp_free_i64(t0);
 923        tcg_temp_free_i64(t1);
 924    }
 925}
 926
 927void tcg_gen_muls2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2)
 928{
 929    if (TCG_TARGET_HAS_muls2_i32) {
 930        tcg_gen_op4_i32(INDEX_op_muls2_i32, rl, rh, arg1, arg2);
 931    } else if (TCG_TARGET_HAS_mulsh_i32) {
 932        TCGv_i32 t = tcg_temp_new_i32();
 933        tcg_gen_op3_i32(INDEX_op_mul_i32, t, arg1, arg2);
 934        tcg_gen_op3_i32(INDEX_op_mulsh_i32, rh, arg1, arg2);
 935        tcg_gen_mov_i32(rl, t);
 936        tcg_temp_free_i32(t);
 937    } else if (TCG_TARGET_REG_BITS == 32) {
 938        TCGv_i32 t0 = tcg_temp_new_i32();
 939        TCGv_i32 t1 = tcg_temp_new_i32();
 940        TCGv_i32 t2 = tcg_temp_new_i32();
 941        TCGv_i32 t3 = tcg_temp_new_i32();
 942        tcg_gen_mulu2_i32(t0, t1, arg1, arg2);
 943        /* Adjust for negative inputs.  */
 944        tcg_gen_sari_i32(t2, arg1, 31);
 945        tcg_gen_sari_i32(t3, arg2, 31);
 946        tcg_gen_and_i32(t2, t2, arg2);
 947        tcg_gen_and_i32(t3, t3, arg1);
 948        tcg_gen_sub_i32(rh, t1, t2);
 949        tcg_gen_sub_i32(rh, rh, t3);
 950        tcg_gen_mov_i32(rl, t0);
 951        tcg_temp_free_i32(t0);
 952        tcg_temp_free_i32(t1);
 953        tcg_temp_free_i32(t2);
 954        tcg_temp_free_i32(t3);
 955    } else {
 956        TCGv_i64 t0 = tcg_temp_new_i64();
 957        TCGv_i64 t1 = tcg_temp_new_i64();
 958        tcg_gen_ext_i32_i64(t0, arg1);
 959        tcg_gen_ext_i32_i64(t1, arg2);
 960        tcg_gen_mul_i64(t0, t0, t1);
 961        tcg_gen_extr_i64_i32(rl, rh, t0);
 962        tcg_temp_free_i64(t0);
 963        tcg_temp_free_i64(t1);
 964    }
 965}
 966
 967void tcg_gen_mulsu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2)
 968{
 969    if (TCG_TARGET_REG_BITS == 32) {
 970        TCGv_i32 t0 = tcg_temp_new_i32();
 971        TCGv_i32 t1 = tcg_temp_new_i32();
 972        TCGv_i32 t2 = tcg_temp_new_i32();
 973        tcg_gen_mulu2_i32(t0, t1, arg1, arg2);
 974        /* Adjust for negative input for the signed arg1.  */
 975        tcg_gen_sari_i32(t2, arg1, 31);
 976        tcg_gen_and_i32(t2, t2, arg2);
 977        tcg_gen_sub_i32(rh, t1, t2);
 978        tcg_gen_mov_i32(rl, t0);
 979        tcg_temp_free_i32(t0);
 980        tcg_temp_free_i32(t1);
 981        tcg_temp_free_i32(t2);
 982    } else {
 983        TCGv_i64 t0 = tcg_temp_new_i64();
 984        TCGv_i64 t1 = tcg_temp_new_i64();
 985        tcg_gen_ext_i32_i64(t0, arg1);
 986        tcg_gen_extu_i32_i64(t1, arg2);
 987        tcg_gen_mul_i64(t0, t0, t1);
 988        tcg_gen_extr_i64_i32(rl, rh, t0);
 989        tcg_temp_free_i64(t0);
 990        tcg_temp_free_i64(t1);
 991    }
 992}
 993
 994void tcg_gen_ext8s_i32(TCGv_i32 ret, TCGv_i32 arg)
 995{
 996    if (TCG_TARGET_HAS_ext8s_i32) {
 997        tcg_gen_op2_i32(INDEX_op_ext8s_i32, ret, arg);
 998    } else {
 999        tcg_gen_shli_i32(ret, arg, 24);
1000        tcg_gen_sari_i32(ret, ret, 24);
1001    }
1002}
1003
1004void tcg_gen_ext16s_i32(TCGv_i32 ret, TCGv_i32 arg)
1005{
1006    if (TCG_TARGET_HAS_ext16s_i32) {
1007        tcg_gen_op2_i32(INDEX_op_ext16s_i32, ret, arg);
1008    } else {
1009        tcg_gen_shli_i32(ret, arg, 16);
1010        tcg_gen_sari_i32(ret, ret, 16);
1011    }
1012}
1013
1014void tcg_gen_ext8u_i32(TCGv_i32 ret, TCGv_i32 arg)
1015{
1016    if (TCG_TARGET_HAS_ext8u_i32) {
1017        tcg_gen_op2_i32(INDEX_op_ext8u_i32, ret, arg);
1018    } else {
1019        tcg_gen_andi_i32(ret, arg, 0xffu);
1020    }
1021}
1022
1023void tcg_gen_ext16u_i32(TCGv_i32 ret, TCGv_i32 arg)
1024{
1025    if (TCG_TARGET_HAS_ext16u_i32) {
1026        tcg_gen_op2_i32(INDEX_op_ext16u_i32, ret, arg);
1027    } else {
1028        tcg_gen_andi_i32(ret, arg, 0xffffu);
1029    }
1030}
1031
1032/* Note: we assume the two high bytes are set to zero */
1033void tcg_gen_bswap16_i32(TCGv_i32 ret, TCGv_i32 arg)
1034{
1035    if (TCG_TARGET_HAS_bswap16_i32) {
1036        tcg_gen_op2_i32(INDEX_op_bswap16_i32, ret, arg);
1037    } else {
1038        TCGv_i32 t0 = tcg_temp_new_i32();
1039
1040        tcg_gen_ext8u_i32(t0, arg);
1041        tcg_gen_shli_i32(t0, t0, 8);
1042        tcg_gen_shri_i32(ret, arg, 8);
1043        tcg_gen_or_i32(ret, ret, t0);
1044        tcg_temp_free_i32(t0);
1045    }
1046}
1047
1048void tcg_gen_bswap32_i32(TCGv_i32 ret, TCGv_i32 arg)
1049{
1050    if (TCG_TARGET_HAS_bswap32_i32) {
1051        tcg_gen_op2_i32(INDEX_op_bswap32_i32, ret, arg);
1052    } else {
1053        TCGv_i32 t0 = tcg_temp_new_i32();
1054        TCGv_i32 t1 = tcg_temp_new_i32();
1055        TCGv_i32 t2 = tcg_const_i32(0x00ff00ff);
1056
1057                                        /* arg = abcd */
1058        tcg_gen_shri_i32(t0, arg, 8);   /*  t0 = .abc */
1059        tcg_gen_and_i32(t1, arg, t2);   /*  t1 = .b.d */
1060        tcg_gen_and_i32(t0, t0, t2);    /*  t0 = .a.c */
1061        tcg_gen_shli_i32(t1, t1, 8);    /*  t1 = b.d. */
1062        tcg_gen_or_i32(ret, t0, t1);    /* ret = badc */
1063
1064        tcg_gen_shri_i32(t0, ret, 16);  /*  t0 = ..ba */
1065        tcg_gen_shli_i32(t1, ret, 16);  /*  t1 = dc.. */
1066        tcg_gen_or_i32(ret, t0, t1);    /* ret = dcba */
1067
1068        tcg_temp_free_i32(t0);
1069        tcg_temp_free_i32(t1);
1070        tcg_temp_free_i32(t2);
1071    }
1072}
1073
1074void tcg_gen_smin_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b)
1075{
1076    tcg_gen_movcond_i32(TCG_COND_LT, ret, a, b, a, b);
1077}
1078
1079void tcg_gen_umin_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b)
1080{
1081    tcg_gen_movcond_i32(TCG_COND_LTU, ret, a, b, a, b);
1082}
1083
1084void tcg_gen_smax_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b)
1085{
1086    tcg_gen_movcond_i32(TCG_COND_LT, ret, a, b, b, a);
1087}
1088
1089void tcg_gen_umax_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b)
1090{
1091    tcg_gen_movcond_i32(TCG_COND_LTU, ret, a, b, b, a);
1092}
1093
1094void tcg_gen_abs_i32(TCGv_i32 ret, TCGv_i32 a)
1095{
1096    TCGv_i32 t = tcg_temp_new_i32();
1097
1098    tcg_gen_sari_i32(t, a, 31);
1099    tcg_gen_xor_i32(ret, a, t);
1100    tcg_gen_sub_i32(ret, ret, t);
1101    tcg_temp_free_i32(t);
1102}
1103
1104/* 64-bit ops */
1105
1106#if TCG_TARGET_REG_BITS == 32
1107/* These are all inline for TCG_TARGET_REG_BITS == 64.  */
1108
1109void tcg_gen_discard_i64(TCGv_i64 arg)
1110{
1111    tcg_gen_discard_i32(TCGV_LOW(arg));
1112    tcg_gen_discard_i32(TCGV_HIGH(arg));
1113}
1114
1115void tcg_gen_mov_i64(TCGv_i64 ret, TCGv_i64 arg)
1116{
1117    tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1118    tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg));
1119}
1120
1121void tcg_gen_movi_i64(TCGv_i64 ret, int64_t arg)
1122{
1123    tcg_gen_movi_i32(TCGV_LOW(ret), arg);
1124    tcg_gen_movi_i32(TCGV_HIGH(ret), arg >> 32);
1125}
1126
1127void tcg_gen_ld8u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1128{
1129    tcg_gen_ld8u_i32(TCGV_LOW(ret), arg2, offset);
1130    tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1131}
1132
1133void tcg_gen_ld8s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1134{
1135    tcg_gen_ld8s_i32(TCGV_LOW(ret), arg2, offset);
1136    tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1137}
1138
1139void tcg_gen_ld16u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1140{
1141    tcg_gen_ld16u_i32(TCGV_LOW(ret), arg2, offset);
1142    tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1143}
1144
1145void tcg_gen_ld16s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1146{
1147    tcg_gen_ld16s_i32(TCGV_LOW(ret), arg2, offset);
1148    tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1149}
1150
1151void tcg_gen_ld32u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1152{
1153    tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset);
1154    tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1155}
1156
1157void tcg_gen_ld32s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1158{
1159    tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset);
1160    tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1161}
1162
1163void tcg_gen_ld_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1164{
1165    /* Since arg2 and ret have different types,
1166       they cannot be the same temporary */
1167#ifdef HOST_WORDS_BIGENDIAN
1168    tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset);
1169    tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset + 4);
1170#else
1171    tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset);
1172    tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset + 4);
1173#endif
1174}
1175
1176void tcg_gen_st_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset)
1177{
1178#ifdef HOST_WORDS_BIGENDIAN
1179    tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset);
1180    tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset + 4);
1181#else
1182    tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset);
1183    tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset + 4);
1184#endif
1185}
1186
1187void tcg_gen_and_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1188{
1189    tcg_gen_and_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1190    tcg_gen_and_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1191}
1192
1193void tcg_gen_or_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1194{
1195    tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1196    tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1197}
1198
1199void tcg_gen_xor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1200{
1201    tcg_gen_xor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1202    tcg_gen_xor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1203}
1204
1205void tcg_gen_shl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1206{
1207    gen_helper_shl_i64(ret, arg1, arg2);
1208}
1209
1210void tcg_gen_shr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1211{
1212    gen_helper_shr_i64(ret, arg1, arg2);
1213}
1214
1215void tcg_gen_sar_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1216{
1217    gen_helper_sar_i64(ret, arg1, arg2);
1218}
1219
1220void tcg_gen_mul_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1221{
1222    TCGv_i64 t0;
1223    TCGv_i32 t1;
1224
1225    t0 = tcg_temp_new_i64();
1226    t1 = tcg_temp_new_i32();
1227
1228    tcg_gen_mulu2_i32(TCGV_LOW(t0), TCGV_HIGH(t0),
1229                      TCGV_LOW(arg1), TCGV_LOW(arg2));
1230
1231    tcg_gen_mul_i32(t1, TCGV_LOW(arg1), TCGV_HIGH(arg2));
1232    tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1);
1233    tcg_gen_mul_i32(t1, TCGV_HIGH(arg1), TCGV_LOW(arg2));
1234    tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1);
1235
1236    tcg_gen_mov_i64(ret, t0);
1237    tcg_temp_free_i64(t0);
1238    tcg_temp_free_i32(t1);
1239}
1240#endif /* TCG_TARGET_REG_SIZE == 32 */
1241
1242void tcg_gen_addi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1243{
1244    /* some cases can be optimized here */
1245    if (arg2 == 0) {
1246        tcg_gen_mov_i64(ret, arg1);
1247    } else {
1248        TCGv_i64 t0 = tcg_const_i64(arg2);
1249        tcg_gen_add_i64(ret, arg1, t0);
1250        tcg_temp_free_i64(t0);
1251    }
1252}
1253
1254void tcg_gen_subfi_i64(TCGv_i64 ret, int64_t arg1, TCGv_i64 arg2)
1255{
1256    if (arg1 == 0 && TCG_TARGET_HAS_neg_i64) {
1257        /* Don't recurse with tcg_gen_neg_i64.  */
1258        tcg_gen_op2_i64(INDEX_op_neg_i64, ret, arg2);
1259    } else {
1260        TCGv_i64 t0 = tcg_const_i64(arg1);
1261        tcg_gen_sub_i64(ret, t0, arg2);
1262        tcg_temp_free_i64(t0);
1263    }
1264}
1265
1266void tcg_gen_subi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1267{
1268    /* some cases can be optimized here */
1269    if (arg2 == 0) {
1270        tcg_gen_mov_i64(ret, arg1);
1271    } else {
1272        TCGv_i64 t0 = tcg_const_i64(arg2);
1273        tcg_gen_sub_i64(ret, arg1, t0);
1274        tcg_temp_free_i64(t0);
1275    }
1276}
1277
1278void tcg_gen_andi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1279{
1280    TCGv_i64 t0;
1281
1282    if (TCG_TARGET_REG_BITS == 32) {
1283        tcg_gen_andi_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2);
1284        tcg_gen_andi_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32);
1285        return;
1286    }
1287
1288    /* Some cases can be optimized here.  */
1289    switch (arg2) {
1290    case 0:
1291        tcg_gen_movi_i64(ret, 0);
1292        return;
1293    case -1:
1294        tcg_gen_mov_i64(ret, arg1);
1295        return;
1296    case 0xff:
1297        /* Don't recurse with tcg_gen_ext8u_i64.  */
1298        if (TCG_TARGET_HAS_ext8u_i64) {
1299            tcg_gen_op2_i64(INDEX_op_ext8u_i64, ret, arg1);
1300            return;
1301        }
1302        break;
1303    case 0xffff:
1304        if (TCG_TARGET_HAS_ext16u_i64) {
1305            tcg_gen_op2_i64(INDEX_op_ext16u_i64, ret, arg1);
1306            return;
1307        }
1308        break;
1309    case 0xffffffffu:
1310        if (TCG_TARGET_HAS_ext32u_i64) {
1311            tcg_gen_op2_i64(INDEX_op_ext32u_i64, ret, arg1);
1312            return;
1313        }
1314        break;
1315    }
1316    t0 = tcg_const_i64(arg2);
1317    tcg_gen_and_i64(ret, arg1, t0);
1318    tcg_temp_free_i64(t0);
1319}
1320
1321void tcg_gen_ori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1322{
1323    if (TCG_TARGET_REG_BITS == 32) {
1324        tcg_gen_ori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2);
1325        tcg_gen_ori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32);
1326        return;
1327    }
1328    /* Some cases can be optimized here.  */
1329    if (arg2 == -1) {
1330        tcg_gen_movi_i64(ret, -1);
1331    } else if (arg2 == 0) {
1332        tcg_gen_mov_i64(ret, arg1);
1333    } else {
1334        TCGv_i64 t0 = tcg_const_i64(arg2);
1335        tcg_gen_or_i64(ret, arg1, t0);
1336        tcg_temp_free_i64(t0);
1337    }
1338}
1339
1340void tcg_gen_xori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1341{
1342    if (TCG_TARGET_REG_BITS == 32) {
1343        tcg_gen_xori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2);
1344        tcg_gen_xori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32);
1345        return;
1346    }
1347    /* Some cases can be optimized here.  */
1348    if (arg2 == 0) {
1349        tcg_gen_mov_i64(ret, arg1);
1350    } else if (arg2 == -1 && TCG_TARGET_HAS_not_i64) {
1351        /* Don't recurse with tcg_gen_not_i64.  */
1352        tcg_gen_op2_i64(INDEX_op_not_i64, ret, arg1);
1353    } else {
1354        TCGv_i64 t0 = tcg_const_i64(arg2);
1355        tcg_gen_xor_i64(ret, arg1, t0);
1356        tcg_temp_free_i64(t0);
1357    }
1358}
1359
1360static inline void tcg_gen_shifti_i64(TCGv_i64 ret, TCGv_i64 arg1,
1361                                      unsigned c, bool right, bool arith)
1362{
1363    tcg_debug_assert(c < 64);
1364    if (c == 0) {
1365        tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1));
1366        tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1));
1367    } else if (c >= 32) {
1368        c -= 32;
1369        if (right) {
1370            if (arith) {
1371                tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c);
1372                tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), 31);
1373            } else {
1374                tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c);
1375                tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1376            }
1377        } else {
1378            tcg_gen_shli_i32(TCGV_HIGH(ret), TCGV_LOW(arg1), c);
1379            tcg_gen_movi_i32(TCGV_LOW(ret), 0);
1380        }
1381    } else if (right) {
1382        if (TCG_TARGET_HAS_extract2_i32) {
1383            tcg_gen_extract2_i32(TCGV_LOW(ret),
1384                                 TCGV_LOW(arg1), TCGV_HIGH(arg1), c);
1385        } else {
1386            tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_LOW(arg1), c);
1387            tcg_gen_deposit_i32(TCGV_LOW(ret), TCGV_LOW(ret),
1388                                TCGV_HIGH(arg1), 32 - c, c);
1389        }
1390        if (arith) {
1391            tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), c);
1392        } else {
1393            tcg_gen_shri_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), c);
1394        }
1395    } else {
1396        if (TCG_TARGET_HAS_extract2_i32) {
1397            tcg_gen_extract2_i32(TCGV_HIGH(ret),
1398                                 TCGV_LOW(arg1), TCGV_HIGH(arg1), 32 - c);
1399        } else {
1400            TCGv_i32 t0 = tcg_temp_new_i32();
1401            tcg_gen_shri_i32(t0, TCGV_LOW(arg1), 32 - c);
1402            tcg_gen_deposit_i32(TCGV_HIGH(ret), t0,
1403                                TCGV_HIGH(arg1), c, 32 - c);
1404            tcg_temp_free_i32(t0);
1405        }
1406        tcg_gen_shli_i32(TCGV_LOW(ret), TCGV_LOW(arg1), c);
1407    }
1408}
1409
1410void tcg_gen_shli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1411{
1412    tcg_debug_assert(arg2 >= 0 && arg2 < 64);
1413    if (TCG_TARGET_REG_BITS == 32) {
1414        tcg_gen_shifti_i64(ret, arg1, arg2, 0, 0);
1415    } else if (arg2 == 0) {
1416        tcg_gen_mov_i64(ret, arg1);
1417    } else {
1418        TCGv_i64 t0 = tcg_const_i64(arg2);
1419        tcg_gen_shl_i64(ret, arg1, t0);
1420        tcg_temp_free_i64(t0);
1421    }
1422}
1423
1424void tcg_gen_shri_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1425{
1426    tcg_debug_assert(arg2 >= 0 && arg2 < 64);
1427    if (TCG_TARGET_REG_BITS == 32) {
1428        tcg_gen_shifti_i64(ret, arg1, arg2, 1, 0);
1429    } else if (arg2 == 0) {
1430        tcg_gen_mov_i64(ret, arg1);
1431    } else {
1432        TCGv_i64 t0 = tcg_const_i64(arg2);
1433        tcg_gen_shr_i64(ret, arg1, t0);
1434        tcg_temp_free_i64(t0);
1435    }
1436}
1437
1438void tcg_gen_sari_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1439{
1440    tcg_debug_assert(arg2 >= 0 && arg2 < 64);
1441    if (TCG_TARGET_REG_BITS == 32) {
1442        tcg_gen_shifti_i64(ret, arg1, arg2, 1, 1);
1443    } else if (arg2 == 0) {
1444        tcg_gen_mov_i64(ret, arg1);
1445    } else {
1446        TCGv_i64 t0 = tcg_const_i64(arg2);
1447        tcg_gen_sar_i64(ret, arg1, t0);
1448        tcg_temp_free_i64(t0);
1449    }
1450}
1451
1452void tcg_gen_brcond_i64(TCGCond cond, TCGv_i64 arg1, TCGv_i64 arg2, TCGLabel *l)
1453{
1454    if (cond == TCG_COND_ALWAYS) {
1455        tcg_gen_br(l);
1456    } else if (cond != TCG_COND_NEVER) {
1457        l->refs++;
1458        if (TCG_TARGET_REG_BITS == 32) {
1459            tcg_gen_op6ii_i32(INDEX_op_brcond2_i32, TCGV_LOW(arg1),
1460                              TCGV_HIGH(arg1), TCGV_LOW(arg2),
1461                              TCGV_HIGH(arg2), cond, label_arg(l));
1462        } else {
1463            tcg_gen_op4ii_i64(INDEX_op_brcond_i64, arg1, arg2, cond,
1464                              label_arg(l));
1465        }
1466    }
1467}
1468
1469void tcg_gen_brcondi_i64(TCGCond cond, TCGv_i64 arg1, int64_t arg2, TCGLabel *l)
1470{
1471    if (cond == TCG_COND_ALWAYS) {
1472        tcg_gen_br(l);
1473    } else if (cond != TCG_COND_NEVER) {
1474        TCGv_i64 t0 = tcg_const_i64(arg2);
1475        tcg_gen_brcond_i64(cond, arg1, t0, l);
1476        tcg_temp_free_i64(t0);
1477    }
1478}
1479
1480void tcg_gen_setcond_i64(TCGCond cond, TCGv_i64 ret,
1481                         TCGv_i64 arg1, TCGv_i64 arg2)
1482{
1483    if (cond == TCG_COND_ALWAYS) {
1484        tcg_gen_movi_i64(ret, 1);
1485    } else if (cond == TCG_COND_NEVER) {
1486        tcg_gen_movi_i64(ret, 0);
1487    } else {
1488        if (TCG_TARGET_REG_BITS == 32) {
1489            tcg_gen_op6i_i32(INDEX_op_setcond2_i32, TCGV_LOW(ret),
1490                             TCGV_LOW(arg1), TCGV_HIGH(arg1),
1491                             TCGV_LOW(arg2), TCGV_HIGH(arg2), cond);
1492            tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1493        } else {
1494            tcg_gen_op4i_i64(INDEX_op_setcond_i64, ret, arg1, arg2, cond);
1495        }
1496    }
1497}
1498
1499void tcg_gen_setcondi_i64(TCGCond cond, TCGv_i64 ret,
1500                          TCGv_i64 arg1, int64_t arg2)
1501{
1502    TCGv_i64 t0 = tcg_const_i64(arg2);
1503    tcg_gen_setcond_i64(cond, ret, arg1, t0);
1504    tcg_temp_free_i64(t0);
1505}
1506
1507void tcg_gen_muli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1508{
1509    if (arg2 == 0) {
1510        tcg_gen_movi_i64(ret, 0);
1511    } else if (is_power_of_2(arg2)) {
1512        tcg_gen_shli_i64(ret, arg1, ctz64(arg2));
1513    } else {
1514        TCGv_i64 t0 = tcg_const_i64(arg2);
1515        tcg_gen_mul_i64(ret, arg1, t0);
1516        tcg_temp_free_i64(t0);
1517    }
1518}
1519
1520void tcg_gen_div_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1521{
1522    if (TCG_TARGET_HAS_div_i64) {
1523        tcg_gen_op3_i64(INDEX_op_div_i64, ret, arg1, arg2);
1524    } else if (TCG_TARGET_HAS_div2_i64) {
1525        TCGv_i64 t0 = tcg_temp_new_i64();
1526        tcg_gen_sari_i64(t0, arg1, 63);
1527        tcg_gen_op5_i64(INDEX_op_div2_i64, ret, t0, arg1, t0, arg2);
1528        tcg_temp_free_i64(t0);
1529    } else {
1530        gen_helper_div_i64(ret, arg1, arg2);
1531    }
1532}
1533
1534void tcg_gen_rem_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1535{
1536    if (TCG_TARGET_HAS_rem_i64) {
1537        tcg_gen_op3_i64(INDEX_op_rem_i64, ret, arg1, arg2);
1538    } else if (TCG_TARGET_HAS_div_i64) {
1539        TCGv_i64 t0 = tcg_temp_new_i64();
1540        tcg_gen_op3_i64(INDEX_op_div_i64, t0, arg1, arg2);
1541        tcg_gen_mul_i64(t0, t0, arg2);
1542        tcg_gen_sub_i64(ret, arg1, t0);
1543        tcg_temp_free_i64(t0);
1544    } else if (TCG_TARGET_HAS_div2_i64) {
1545        TCGv_i64 t0 = tcg_temp_new_i64();
1546        tcg_gen_sari_i64(t0, arg1, 63);
1547        tcg_gen_op5_i64(INDEX_op_div2_i64, t0, ret, arg1, t0, arg2);
1548        tcg_temp_free_i64(t0);
1549    } else {
1550        gen_helper_rem_i64(ret, arg1, arg2);
1551    }
1552}
1553
1554void tcg_gen_divu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1555{
1556    if (TCG_TARGET_HAS_div_i64) {
1557        tcg_gen_op3_i64(INDEX_op_divu_i64, ret, arg1, arg2);
1558    } else if (TCG_TARGET_HAS_div2_i64) {
1559        TCGv_i64 t0 = tcg_temp_new_i64();
1560        tcg_gen_movi_i64(t0, 0);
1561        tcg_gen_op5_i64(INDEX_op_divu2_i64, ret, t0, arg1, t0, arg2);
1562        tcg_temp_free_i64(t0);
1563    } else {
1564        gen_helper_divu_i64(ret, arg1, arg2);
1565    }
1566}
1567
1568void tcg_gen_remu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1569{
1570    if (TCG_TARGET_HAS_rem_i64) {
1571        tcg_gen_op3_i64(INDEX_op_remu_i64, ret, arg1, arg2);
1572    } else if (TCG_TARGET_HAS_div_i64) {
1573        TCGv_i64 t0 = tcg_temp_new_i64();
1574        tcg_gen_op3_i64(INDEX_op_divu_i64, t0, arg1, arg2);
1575        tcg_gen_mul_i64(t0, t0, arg2);
1576        tcg_gen_sub_i64(ret, arg1, t0);
1577        tcg_temp_free_i64(t0);
1578    } else if (TCG_TARGET_HAS_div2_i64) {
1579        TCGv_i64 t0 = tcg_temp_new_i64();
1580        tcg_gen_movi_i64(t0, 0);
1581        tcg_gen_op5_i64(INDEX_op_divu2_i64, t0, ret, arg1, t0, arg2);
1582        tcg_temp_free_i64(t0);
1583    } else {
1584        gen_helper_remu_i64(ret, arg1, arg2);
1585    }
1586}
1587
1588void tcg_gen_ext8s_i64(TCGv_i64 ret, TCGv_i64 arg)
1589{
1590    if (TCG_TARGET_REG_BITS == 32) {
1591        tcg_gen_ext8s_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1592        tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1593    } else if (TCG_TARGET_HAS_ext8s_i64) {
1594        tcg_gen_op2_i64(INDEX_op_ext8s_i64, ret, arg);
1595    } else {
1596        tcg_gen_shli_i64(ret, arg, 56);
1597        tcg_gen_sari_i64(ret, ret, 56);
1598    }
1599}
1600
1601void tcg_gen_ext16s_i64(TCGv_i64 ret, TCGv_i64 arg)
1602{
1603    if (TCG_TARGET_REG_BITS == 32) {
1604        tcg_gen_ext16s_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1605        tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1606    } else if (TCG_TARGET_HAS_ext16s_i64) {
1607        tcg_gen_op2_i64(INDEX_op_ext16s_i64, ret, arg);
1608    } else {
1609        tcg_gen_shli_i64(ret, arg, 48);
1610        tcg_gen_sari_i64(ret, ret, 48);
1611    }
1612}
1613
1614void tcg_gen_ext32s_i64(TCGv_i64 ret, TCGv_i64 arg)
1615{
1616    if (TCG_TARGET_REG_BITS == 32) {
1617        tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1618        tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1619    } else if (TCG_TARGET_HAS_ext32s_i64) {
1620        tcg_gen_op2_i64(INDEX_op_ext32s_i64, ret, arg);
1621    } else {
1622        tcg_gen_shli_i64(ret, arg, 32);
1623        tcg_gen_sari_i64(ret, ret, 32);
1624    }
1625}
1626
1627void tcg_gen_ext8u_i64(TCGv_i64 ret, TCGv_i64 arg)
1628{
1629    if (TCG_TARGET_REG_BITS == 32) {
1630        tcg_gen_ext8u_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1631        tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1632    } else if (TCG_TARGET_HAS_ext8u_i64) {
1633        tcg_gen_op2_i64(INDEX_op_ext8u_i64, ret, arg);
1634    } else {
1635        tcg_gen_andi_i64(ret, arg, 0xffu);
1636    }
1637}
1638
1639void tcg_gen_ext16u_i64(TCGv_i64 ret, TCGv_i64 arg)
1640{
1641    if (TCG_TARGET_REG_BITS == 32) {
1642        tcg_gen_ext16u_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1643        tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1644    } else if (TCG_TARGET_HAS_ext16u_i64) {
1645        tcg_gen_op2_i64(INDEX_op_ext16u_i64, ret, arg);
1646    } else {
1647        tcg_gen_andi_i64(ret, arg, 0xffffu);
1648    }
1649}
1650
1651void tcg_gen_ext32u_i64(TCGv_i64 ret, TCGv_i64 arg)
1652{
1653    if (TCG_TARGET_REG_BITS == 32) {
1654        tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1655        tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1656    } else if (TCG_TARGET_HAS_ext32u_i64) {
1657        tcg_gen_op2_i64(INDEX_op_ext32u_i64, ret, arg);
1658    } else {
1659        tcg_gen_andi_i64(ret, arg, 0xffffffffu);
1660    }
1661}
1662
1663/* Note: we assume the six high bytes are set to zero */
1664void tcg_gen_bswap16_i64(TCGv_i64 ret, TCGv_i64 arg)
1665{
1666    if (TCG_TARGET_REG_BITS == 32) {
1667        tcg_gen_bswap16_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1668        tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1669    } else if (TCG_TARGET_HAS_bswap16_i64) {
1670        tcg_gen_op2_i64(INDEX_op_bswap16_i64, ret, arg);
1671    } else {
1672        TCGv_i64 t0 = tcg_temp_new_i64();
1673
1674        tcg_gen_ext8u_i64(t0, arg);
1675        tcg_gen_shli_i64(t0, t0, 8);
1676        tcg_gen_shri_i64(ret, arg, 8);
1677        tcg_gen_or_i64(ret, ret, t0);
1678        tcg_temp_free_i64(t0);
1679    }
1680}
1681
1682/* Note: we assume the four high bytes are set to zero */
1683void tcg_gen_bswap32_i64(TCGv_i64 ret, TCGv_i64 arg)
1684{
1685    if (TCG_TARGET_REG_BITS == 32) {
1686        tcg_gen_bswap32_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1687        tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1688    } else if (TCG_TARGET_HAS_bswap32_i64) {
1689        tcg_gen_op2_i64(INDEX_op_bswap32_i64, ret, arg);
1690    } else {
1691        TCGv_i64 t0 = tcg_temp_new_i64();
1692        TCGv_i64 t1 = tcg_temp_new_i64();
1693        TCGv_i64 t2 = tcg_const_i64(0x00ff00ff);
1694
1695                                        /* arg = ....abcd */
1696        tcg_gen_shri_i64(t0, arg, 8);   /*  t0 = .....abc */
1697        tcg_gen_and_i64(t1, arg, t2);   /*  t1 = .....b.d */
1698        tcg_gen_and_i64(t0, t0, t2);    /*  t0 = .....a.c */
1699        tcg_gen_shli_i64(t1, t1, 8);    /*  t1 = ....b.d. */
1700        tcg_gen_or_i64(ret, t0, t1);    /* ret = ....badc */
1701
1702        tcg_gen_shli_i64(t1, ret, 48);  /*  t1 = dc...... */
1703        tcg_gen_shri_i64(t0, ret, 16);  /*  t0 = ......ba */
1704        tcg_gen_shri_i64(t1, t1, 32);   /*  t1 = ....dc.. */
1705        tcg_gen_or_i64(ret, t0, t1);    /* ret = ....dcba */
1706
1707        tcg_temp_free_i64(t0);
1708        tcg_temp_free_i64(t1);
1709        tcg_temp_free_i64(t2);
1710    }
1711}
1712
1713void tcg_gen_bswap64_i64(TCGv_i64 ret, TCGv_i64 arg)
1714{
1715    if (TCG_TARGET_REG_BITS == 32) {
1716        TCGv_i32 t0, t1;
1717        t0 = tcg_temp_new_i32();
1718        t1 = tcg_temp_new_i32();
1719
1720        tcg_gen_bswap32_i32(t0, TCGV_LOW(arg));
1721        tcg_gen_bswap32_i32(t1, TCGV_HIGH(arg));
1722        tcg_gen_mov_i32(TCGV_LOW(ret), t1);
1723        tcg_gen_mov_i32(TCGV_HIGH(ret), t0);
1724        tcg_temp_free_i32(t0);
1725        tcg_temp_free_i32(t1);
1726    } else if (TCG_TARGET_HAS_bswap64_i64) {
1727        tcg_gen_op2_i64(INDEX_op_bswap64_i64, ret, arg);
1728    } else {
1729        TCGv_i64 t0 = tcg_temp_new_i64();
1730        TCGv_i64 t1 = tcg_temp_new_i64();
1731        TCGv_i64 t2 = tcg_temp_new_i64();
1732
1733                                        /* arg = abcdefgh */
1734        tcg_gen_movi_i64(t2, 0x00ff00ff00ff00ffull);
1735        tcg_gen_shri_i64(t0, arg, 8);   /*  t0 = .abcdefg */
1736        tcg_gen_and_i64(t1, arg, t2);   /*  t1 = .b.d.f.h */
1737        tcg_gen_and_i64(t0, t0, t2);    /*  t0 = .a.c.e.g */
1738        tcg_gen_shli_i64(t1, t1, 8);    /*  t1 = b.d.f.h. */
1739        tcg_gen_or_i64(ret, t0, t1);    /* ret = badcfehg */
1740
1741        tcg_gen_movi_i64(t2, 0x0000ffff0000ffffull);
1742        tcg_gen_shri_i64(t0, ret, 16);  /*  t0 = ..badcfe */
1743        tcg_gen_and_i64(t1, ret, t2);   /*  t1 = ..dc..hg */
1744        tcg_gen_and_i64(t0, t0, t2);    /*  t0 = ..ba..fe */
1745        tcg_gen_shli_i64(t1, t1, 16);   /*  t1 = dc..hg.. */
1746        tcg_gen_or_i64(ret, t0, t1);    /* ret = dcbahgfe */
1747
1748        tcg_gen_shri_i64(t0, ret, 32);  /*  t0 = ....dcba */
1749        tcg_gen_shli_i64(t1, ret, 32);  /*  t1 = hgfe.... */
1750        tcg_gen_or_i64(ret, t0, t1);    /* ret = hgfedcba */
1751
1752        tcg_temp_free_i64(t0);
1753        tcg_temp_free_i64(t1);
1754        tcg_temp_free_i64(t2);
1755    }
1756}
1757
1758void tcg_gen_not_i64(TCGv_i64 ret, TCGv_i64 arg)
1759{
1760    if (TCG_TARGET_REG_BITS == 32) {
1761        tcg_gen_not_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1762        tcg_gen_not_i32(TCGV_HIGH(ret), TCGV_HIGH(arg));
1763    } else if (TCG_TARGET_HAS_not_i64) {
1764        tcg_gen_op2_i64(INDEX_op_not_i64, ret, arg);
1765    } else {
1766        tcg_gen_xori_i64(ret, arg, -1);
1767    }
1768}
1769
1770void tcg_gen_andc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1771{
1772    if (TCG_TARGET_REG_BITS == 32) {
1773        tcg_gen_andc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1774        tcg_gen_andc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1775    } else if (TCG_TARGET_HAS_andc_i64) {
1776        tcg_gen_op3_i64(INDEX_op_andc_i64, ret, arg1, arg2);
1777    } else {
1778        TCGv_i64 t0 = tcg_temp_new_i64();
1779        tcg_gen_not_i64(t0, arg2);
1780        tcg_gen_and_i64(ret, arg1, t0);
1781        tcg_temp_free_i64(t0);
1782    }
1783}
1784
1785void tcg_gen_eqv_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1786{
1787    if (TCG_TARGET_REG_BITS == 32) {
1788        tcg_gen_eqv_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1789        tcg_gen_eqv_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1790    } else if (TCG_TARGET_HAS_eqv_i64) {
1791        tcg_gen_op3_i64(INDEX_op_eqv_i64, ret, arg1, arg2);
1792    } else {
1793        tcg_gen_xor_i64(ret, arg1, arg2);
1794        tcg_gen_not_i64(ret, ret);
1795    }
1796}
1797
1798void tcg_gen_nand_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1799{
1800    if (TCG_TARGET_REG_BITS == 32) {
1801        tcg_gen_nand_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1802        tcg_gen_nand_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1803    } else if (TCG_TARGET_HAS_nand_i64) {
1804        tcg_gen_op3_i64(INDEX_op_nand_i64, ret, arg1, arg2);
1805    } else {
1806        tcg_gen_and_i64(ret, arg1, arg2);
1807        tcg_gen_not_i64(ret, ret);
1808    }
1809}
1810
1811void tcg_gen_nor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1812{
1813    if (TCG_TARGET_REG_BITS == 32) {
1814        tcg_gen_nor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1815        tcg_gen_nor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1816    } else if (TCG_TARGET_HAS_nor_i64) {
1817        tcg_gen_op3_i64(INDEX_op_nor_i64, ret, arg1, arg2);
1818    } else {
1819        tcg_gen_or_i64(ret, arg1, arg2);
1820        tcg_gen_not_i64(ret, ret);
1821    }
1822}
1823
1824void tcg_gen_orc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1825{
1826    if (TCG_TARGET_REG_BITS == 32) {
1827        tcg_gen_orc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1828        tcg_gen_orc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1829    } else if (TCG_TARGET_HAS_orc_i64) {
1830        tcg_gen_op3_i64(INDEX_op_orc_i64, ret, arg1, arg2);
1831    } else {
1832        TCGv_i64 t0 = tcg_temp_new_i64();
1833        tcg_gen_not_i64(t0, arg2);
1834        tcg_gen_or_i64(ret, arg1, t0);
1835        tcg_temp_free_i64(t0);
1836    }
1837}
1838
1839void tcg_gen_clz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1840{
1841    if (TCG_TARGET_HAS_clz_i64) {
1842        tcg_gen_op3_i64(INDEX_op_clz_i64, ret, arg1, arg2);
1843    } else {
1844        gen_helper_clz_i64(ret, arg1, arg2);
1845    }
1846}
1847
1848void tcg_gen_clzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2)
1849{
1850    if (TCG_TARGET_REG_BITS == 32
1851        && TCG_TARGET_HAS_clz_i32
1852        && arg2 <= 0xffffffffu) {
1853        TCGv_i32 t = tcg_const_i32((uint32_t)arg2 - 32);
1854        tcg_gen_clz_i32(t, TCGV_LOW(arg1), t);
1855        tcg_gen_addi_i32(t, t, 32);
1856        tcg_gen_clz_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), t);
1857        tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1858        tcg_temp_free_i32(t);
1859    } else {
1860        TCGv_i64 t = tcg_const_i64(arg2);
1861        tcg_gen_clz_i64(ret, arg1, t);
1862        tcg_temp_free_i64(t);
1863    }
1864}
1865
1866void tcg_gen_ctz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1867{
1868    if (TCG_TARGET_HAS_ctz_i64) {
1869        tcg_gen_op3_i64(INDEX_op_ctz_i64, ret, arg1, arg2);
1870    } else if (TCG_TARGET_HAS_ctpop_i64 || TCG_TARGET_HAS_clz_i64) {
1871        TCGv_i64 z, t = tcg_temp_new_i64();
1872
1873        if (TCG_TARGET_HAS_ctpop_i64) {
1874            tcg_gen_subi_i64(t, arg1, 1);
1875            tcg_gen_andc_i64(t, t, arg1);
1876            tcg_gen_ctpop_i64(t, t);
1877        } else {
1878            /* Since all non-x86 hosts have clz(0) == 64, don't fight it.  */
1879            tcg_gen_neg_i64(t, arg1);
1880            tcg_gen_and_i64(t, t, arg1);
1881            tcg_gen_clzi_i64(t, t, 64);
1882            tcg_gen_xori_i64(t, t, 63);
1883        }
1884        z = tcg_const_i64(0);
1885        tcg_gen_movcond_i64(TCG_COND_EQ, ret, arg1, z, arg2, t);
1886        tcg_temp_free_i64(t);
1887        tcg_temp_free_i64(z);
1888    } else {
1889        gen_helper_ctz_i64(ret, arg1, arg2);
1890    }
1891}
1892
1893void tcg_gen_ctzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2)
1894{
1895    if (TCG_TARGET_REG_BITS == 32
1896        && TCG_TARGET_HAS_ctz_i32
1897        && arg2 <= 0xffffffffu) {
1898        TCGv_i32 t32 = tcg_const_i32((uint32_t)arg2 - 32);
1899        tcg_gen_ctz_i32(t32, TCGV_HIGH(arg1), t32);
1900        tcg_gen_addi_i32(t32, t32, 32);
1901        tcg_gen_ctz_i32(TCGV_LOW(ret), TCGV_LOW(arg1), t32);
1902        tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1903        tcg_temp_free_i32(t32);
1904    } else if (!TCG_TARGET_HAS_ctz_i64
1905               && TCG_TARGET_HAS_ctpop_i64
1906               && arg2 == 64) {
1907        /* This equivalence has the advantage of not requiring a fixup.  */
1908        TCGv_i64 t = tcg_temp_new_i64();
1909        tcg_gen_subi_i64(t, arg1, 1);
1910        tcg_gen_andc_i64(t, t, arg1);
1911        tcg_gen_ctpop_i64(ret, t);
1912        tcg_temp_free_i64(t);
1913    } else {
1914        TCGv_i64 t64 = tcg_const_i64(arg2);
1915        tcg_gen_ctz_i64(ret, arg1, t64);
1916        tcg_temp_free_i64(t64);
1917    }
1918}
1919
1920void tcg_gen_clrsb_i64(TCGv_i64 ret, TCGv_i64 arg)
1921{
1922    if (TCG_TARGET_HAS_clz_i64 || TCG_TARGET_HAS_clz_i32) {
1923        TCGv_i64 t = tcg_temp_new_i64();
1924        tcg_gen_sari_i64(t, arg, 63);
1925        tcg_gen_xor_i64(t, t, arg);
1926        tcg_gen_clzi_i64(t, t, 64);
1927        tcg_gen_subi_i64(ret, t, 1);
1928        tcg_temp_free_i64(t);
1929    } else {
1930        gen_helper_clrsb_i64(ret, arg);
1931    }
1932}
1933
1934void tcg_gen_ctpop_i64(TCGv_i64 ret, TCGv_i64 arg1)
1935{
1936    if (TCG_TARGET_HAS_ctpop_i64) {
1937        tcg_gen_op2_i64(INDEX_op_ctpop_i64, ret, arg1);
1938    } else if (TCG_TARGET_REG_BITS == 32 && TCG_TARGET_HAS_ctpop_i32) {
1939        tcg_gen_ctpop_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1));
1940        tcg_gen_ctpop_i32(TCGV_LOW(ret), TCGV_LOW(arg1));
1941        tcg_gen_add_i32(TCGV_LOW(ret), TCGV_LOW(ret), TCGV_HIGH(ret));
1942        tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1943    } else {
1944        gen_helper_ctpop_i64(ret, arg1);
1945    }
1946}
1947
1948void tcg_gen_rotl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1949{
1950    if (TCG_TARGET_HAS_rot_i64) {
1951        tcg_gen_op3_i64(INDEX_op_rotl_i64, ret, arg1, arg2);
1952    } else {
1953        TCGv_i64 t0, t1;
1954        t0 = tcg_temp_new_i64();
1955        t1 = tcg_temp_new_i64();
1956        tcg_gen_shl_i64(t0, arg1, arg2);
1957        tcg_gen_subfi_i64(t1, 64, arg2);
1958        tcg_gen_shr_i64(t1, arg1, t1);
1959        tcg_gen_or_i64(ret, t0, t1);
1960        tcg_temp_free_i64(t0);
1961        tcg_temp_free_i64(t1);
1962    }
1963}
1964
1965void tcg_gen_rotli_i64(TCGv_i64 ret, TCGv_i64 arg1, unsigned arg2)
1966{
1967    tcg_debug_assert(arg2 < 64);
1968    /* some cases can be optimized here */
1969    if (arg2 == 0) {
1970        tcg_gen_mov_i64(ret, arg1);
1971    } else if (TCG_TARGET_HAS_rot_i64) {
1972        TCGv_i64 t0 = tcg_const_i64(arg2);
1973        tcg_gen_rotl_i64(ret, arg1, t0);
1974        tcg_temp_free_i64(t0);
1975    } else {
1976        TCGv_i64 t0, t1;
1977        t0 = tcg_temp_new_i64();
1978        t1 = tcg_temp_new_i64();
1979        tcg_gen_shli_i64(t0, arg1, arg2);
1980        tcg_gen_shri_i64(t1, arg1, 64 - arg2);
1981        tcg_gen_or_i64(ret, t0, t1);
1982        tcg_temp_free_i64(t0);
1983        tcg_temp_free_i64(t1);
1984    }
1985}
1986
1987void tcg_gen_rotr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1988{
1989    if (TCG_TARGET_HAS_rot_i64) {
1990        tcg_gen_op3_i64(INDEX_op_rotr_i64, ret, arg1, arg2);
1991    } else {
1992        TCGv_i64 t0, t1;
1993        t0 = tcg_temp_new_i64();
1994        t1 = tcg_temp_new_i64();
1995        tcg_gen_shr_i64(t0, arg1, arg2);
1996        tcg_gen_subfi_i64(t1, 64, arg2);
1997        tcg_gen_shl_i64(t1, arg1, t1);
1998        tcg_gen_or_i64(ret, t0, t1);
1999        tcg_temp_free_i64(t0);
2000        tcg_temp_free_i64(t1);
2001    }
2002}
2003
2004void tcg_gen_rotri_i64(TCGv_i64 ret, TCGv_i64 arg1, unsigned arg2)
2005{
2006    tcg_debug_assert(arg2 < 64);
2007    /* some cases can be optimized here */
2008    if (arg2 == 0) {
2009        tcg_gen_mov_i64(ret, arg1);
2010    } else {
2011        tcg_gen_rotli_i64(ret, arg1, 64 - arg2);
2012    }
2013}
2014
2015void tcg_gen_deposit_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2,
2016                         unsigned int ofs, unsigned int len)
2017{
2018    uint64_t mask;
2019    TCGv_i64 t1;
2020
2021    tcg_debug_assert(ofs < 64);
2022    tcg_debug_assert(len > 0);
2023    tcg_debug_assert(len <= 64);
2024    tcg_debug_assert(ofs + len <= 64);
2025
2026    if (len == 64) {
2027        tcg_gen_mov_i64(ret, arg2);
2028        return;
2029    }
2030    if (TCG_TARGET_HAS_deposit_i64 && TCG_TARGET_deposit_i64_valid(ofs, len)) {
2031        tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, arg1, arg2, ofs, len);
2032        return;
2033    }
2034
2035    if (TCG_TARGET_REG_BITS == 32) {
2036        if (ofs >= 32) {
2037            tcg_gen_deposit_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1),
2038                                TCGV_LOW(arg2), ofs - 32, len);
2039            tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1));
2040            return;
2041        }
2042        if (ofs + len <= 32) {
2043            tcg_gen_deposit_i32(TCGV_LOW(ret), TCGV_LOW(arg1),
2044                                TCGV_LOW(arg2), ofs, len);
2045            tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1));
2046            return;
2047        }
2048    }
2049
2050    t1 = tcg_temp_new_i64();
2051
2052    if (TCG_TARGET_HAS_extract2_i64) {
2053        if (ofs + len == 64) {
2054            tcg_gen_shli_i64(t1, arg1, len);
2055            tcg_gen_extract2_i64(ret, t1, arg2, len);
2056            goto done;
2057        }
2058        if (ofs == 0) {
2059            tcg_gen_extract2_i64(ret, arg1, arg2, len);
2060            tcg_gen_rotli_i64(ret, ret, len);
2061            goto done;
2062        }
2063    }
2064
2065    mask = (1ull << len) - 1;
2066    if (ofs + len < 64) {
2067        tcg_gen_andi_i64(t1, arg2, mask);
2068        tcg_gen_shli_i64(t1, t1, ofs);
2069    } else {
2070        tcg_gen_shli_i64(t1, arg2, ofs);
2071    }
2072    tcg_gen_andi_i64(ret, arg1, ~(mask << ofs));
2073    tcg_gen_or_i64(ret, ret, t1);
2074 done:
2075    tcg_temp_free_i64(t1);
2076}
2077
2078void tcg_gen_deposit_z_i64(TCGv_i64 ret, TCGv_i64 arg,
2079                           unsigned int ofs, unsigned int len)
2080{
2081    tcg_debug_assert(ofs < 64);
2082    tcg_debug_assert(len > 0);
2083    tcg_debug_assert(len <= 64);
2084    tcg_debug_assert(ofs + len <= 64);
2085
2086    if (ofs + len == 64) {
2087        tcg_gen_shli_i64(ret, arg, ofs);
2088    } else if (ofs == 0) {
2089        tcg_gen_andi_i64(ret, arg, (1ull << len) - 1);
2090    } else if (TCG_TARGET_HAS_deposit_i64
2091               && TCG_TARGET_deposit_i64_valid(ofs, len)) {
2092        TCGv_i64 zero = tcg_const_i64(0);
2093        tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, zero, arg, ofs, len);
2094        tcg_temp_free_i64(zero);
2095    } else {
2096        if (TCG_TARGET_REG_BITS == 32) {
2097            if (ofs >= 32) {
2098                tcg_gen_deposit_z_i32(TCGV_HIGH(ret), TCGV_LOW(arg),
2099                                      ofs - 32, len);
2100                tcg_gen_movi_i32(TCGV_LOW(ret), 0);
2101                return;
2102            }
2103            if (ofs + len <= 32) {
2104                tcg_gen_deposit_z_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len);
2105                tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2106                return;
2107            }
2108        }
2109        /* To help two-operand hosts we prefer to zero-extend first,
2110           which allows ARG to stay live.  */
2111        switch (len) {
2112        case 32:
2113            if (TCG_TARGET_HAS_ext32u_i64) {
2114                tcg_gen_ext32u_i64(ret, arg);
2115                tcg_gen_shli_i64(ret, ret, ofs);
2116                return;
2117            }
2118            break;
2119        case 16:
2120            if (TCG_TARGET_HAS_ext16u_i64) {
2121                tcg_gen_ext16u_i64(ret, arg);
2122                tcg_gen_shli_i64(ret, ret, ofs);
2123                return;
2124            }
2125            break;
2126        case 8:
2127            if (TCG_TARGET_HAS_ext8u_i64) {
2128                tcg_gen_ext8u_i64(ret, arg);
2129                tcg_gen_shli_i64(ret, ret, ofs);
2130                return;
2131            }
2132            break;
2133        }
2134        /* Otherwise prefer zero-extension over AND for code size.  */
2135        switch (ofs + len) {
2136        case 32:
2137            if (TCG_TARGET_HAS_ext32u_i64) {
2138                tcg_gen_shli_i64(ret, arg, ofs);
2139                tcg_gen_ext32u_i64(ret, ret);
2140                return;
2141            }
2142            break;
2143        case 16:
2144            if (TCG_TARGET_HAS_ext16u_i64) {
2145                tcg_gen_shli_i64(ret, arg, ofs);
2146                tcg_gen_ext16u_i64(ret, ret);
2147                return;
2148            }
2149            break;
2150        case 8:
2151            if (TCG_TARGET_HAS_ext8u_i64) {
2152                tcg_gen_shli_i64(ret, arg, ofs);
2153                tcg_gen_ext8u_i64(ret, ret);
2154                return;
2155            }
2156            break;
2157        }
2158        tcg_gen_andi_i64(ret, arg, (1ull << len) - 1);
2159        tcg_gen_shli_i64(ret, ret, ofs);
2160    }
2161}
2162
2163void tcg_gen_extract_i64(TCGv_i64 ret, TCGv_i64 arg,
2164                         unsigned int ofs, unsigned int len)
2165{
2166    tcg_debug_assert(ofs < 64);
2167    tcg_debug_assert(len > 0);
2168    tcg_debug_assert(len <= 64);
2169    tcg_debug_assert(ofs + len <= 64);
2170
2171    /* Canonicalize certain special cases, even if extract is supported.  */
2172    if (ofs + len == 64) {
2173        tcg_gen_shri_i64(ret, arg, 64 - len);
2174        return;
2175    }
2176    if (ofs == 0) {
2177        tcg_gen_andi_i64(ret, arg, (1ull << len) - 1);
2178        return;
2179    }
2180
2181    if (TCG_TARGET_REG_BITS == 32) {
2182        /* Look for a 32-bit extract within one of the two words.  */
2183        if (ofs >= 32) {
2184            tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_HIGH(arg), ofs - 32, len);
2185            tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2186            return;
2187        }
2188        if (ofs + len <= 32) {
2189            tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len);
2190            tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2191            return;
2192        }
2193        /* The field is split across two words.  One double-word
2194           shift is better than two double-word shifts.  */
2195        goto do_shift_and;
2196    }
2197
2198    if (TCG_TARGET_HAS_extract_i64
2199        && TCG_TARGET_extract_i64_valid(ofs, len)) {
2200        tcg_gen_op4ii_i64(INDEX_op_extract_i64, ret, arg, ofs, len);
2201        return;
2202    }
2203
2204    /* Assume that zero-extension, if available, is cheaper than a shift.  */
2205    switch (ofs + len) {
2206    case 32:
2207        if (TCG_TARGET_HAS_ext32u_i64) {
2208            tcg_gen_ext32u_i64(ret, arg);
2209            tcg_gen_shri_i64(ret, ret, ofs);
2210            return;
2211        }
2212        break;
2213    case 16:
2214        if (TCG_TARGET_HAS_ext16u_i64) {
2215            tcg_gen_ext16u_i64(ret, arg);
2216            tcg_gen_shri_i64(ret, ret, ofs);
2217            return;
2218        }
2219        break;
2220    case 8:
2221        if (TCG_TARGET_HAS_ext8u_i64) {
2222            tcg_gen_ext8u_i64(ret, arg);
2223            tcg_gen_shri_i64(ret, ret, ofs);
2224            return;
2225        }
2226        break;
2227    }
2228
2229    /* ??? Ideally we'd know what values are available for immediate AND.
2230       Assume that 8 bits are available, plus the special cases of 16 and 32,
2231       so that we get ext8u, ext16u, and ext32u.  */
2232    switch (len) {
2233    case 1 ... 8: case 16: case 32:
2234    do_shift_and:
2235        tcg_gen_shri_i64(ret, arg, ofs);
2236        tcg_gen_andi_i64(ret, ret, (1ull << len) - 1);
2237        break;
2238    default:
2239        tcg_gen_shli_i64(ret, arg, 64 - len - ofs);
2240        tcg_gen_shri_i64(ret, ret, 64 - len);
2241        break;
2242    }
2243}
2244
2245void tcg_gen_sextract_i64(TCGv_i64 ret, TCGv_i64 arg,
2246                          unsigned int ofs, unsigned int len)
2247{
2248    tcg_debug_assert(ofs < 64);
2249    tcg_debug_assert(len > 0);
2250    tcg_debug_assert(len <= 64);
2251    tcg_debug_assert(ofs + len <= 64);
2252
2253    /* Canonicalize certain special cases, even if sextract is supported.  */
2254    if (ofs + len == 64) {
2255        tcg_gen_sari_i64(ret, arg, 64 - len);
2256        return;
2257    }
2258    if (ofs == 0) {
2259        switch (len) {
2260        case 32:
2261            tcg_gen_ext32s_i64(ret, arg);
2262            return;
2263        case 16:
2264            tcg_gen_ext16s_i64(ret, arg);
2265            return;
2266        case 8:
2267            tcg_gen_ext8s_i64(ret, arg);
2268            return;
2269        }
2270    }
2271
2272    if (TCG_TARGET_REG_BITS == 32) {
2273        /* Look for a 32-bit extract within one of the two words.  */
2274        if (ofs >= 32) {
2275            tcg_gen_sextract_i32(TCGV_LOW(ret), TCGV_HIGH(arg), ofs - 32, len);
2276        } else if (ofs + len <= 32) {
2277            tcg_gen_sextract_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len);
2278        } else if (ofs == 0) {
2279            tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
2280            tcg_gen_sextract_i32(TCGV_HIGH(ret), TCGV_HIGH(arg), 0, len - 32);
2281            return;
2282        } else if (len > 32) {
2283            TCGv_i32 t = tcg_temp_new_i32();
2284            /* Extract the bits for the high word normally.  */
2285            tcg_gen_sextract_i32(t, TCGV_HIGH(arg), ofs + 32, len - 32);
2286            /* Shift the field down for the low part.  */
2287            tcg_gen_shri_i64(ret, arg, ofs);
2288            /* Overwrite the shift into the high part.  */
2289            tcg_gen_mov_i32(TCGV_HIGH(ret), t);
2290            tcg_temp_free_i32(t);
2291            return;
2292        } else {
2293            /* Shift the field down for the low part, such that the
2294               field sits at the MSB.  */
2295            tcg_gen_shri_i64(ret, arg, ofs + len - 32);
2296            /* Shift the field down from the MSB, sign extending.  */
2297            tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_LOW(ret), 32 - len);
2298        }
2299        /* Sign-extend the field from 32 bits.  */
2300        tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
2301        return;
2302    }
2303
2304    if (TCG_TARGET_HAS_sextract_i64
2305        && TCG_TARGET_extract_i64_valid(ofs, len)) {
2306        tcg_gen_op4ii_i64(INDEX_op_sextract_i64, ret, arg, ofs, len);
2307        return;
2308    }
2309
2310    /* Assume that sign-extension, if available, is cheaper than a shift.  */
2311    switch (ofs + len) {
2312    case 32:
2313        if (TCG_TARGET_HAS_ext32s_i64) {
2314            tcg_gen_ext32s_i64(ret, arg);
2315            tcg_gen_sari_i64(ret, ret, ofs);
2316            return;
2317        }
2318        break;
2319    case 16:
2320        if (TCG_TARGET_HAS_ext16s_i64) {
2321            tcg_gen_ext16s_i64(ret, arg);
2322            tcg_gen_sari_i64(ret, ret, ofs);
2323            return;
2324        }
2325        break;
2326    case 8:
2327        if (TCG_TARGET_HAS_ext8s_i64) {
2328            tcg_gen_ext8s_i64(ret, arg);
2329            tcg_gen_sari_i64(ret, ret, ofs);
2330            return;
2331        }
2332        break;
2333    }
2334    switch (len) {
2335    case 32:
2336        if (TCG_TARGET_HAS_ext32s_i64) {
2337            tcg_gen_shri_i64(ret, arg, ofs);
2338            tcg_gen_ext32s_i64(ret, ret);
2339            return;
2340        }
2341        break;
2342    case 16:
2343        if (TCG_TARGET_HAS_ext16s_i64) {
2344            tcg_gen_shri_i64(ret, arg, ofs);
2345            tcg_gen_ext16s_i64(ret, ret);
2346            return;
2347        }
2348        break;
2349    case 8:
2350        if (TCG_TARGET_HAS_ext8s_i64) {
2351            tcg_gen_shri_i64(ret, arg, ofs);
2352            tcg_gen_ext8s_i64(ret, ret);
2353            return;
2354        }
2355        break;
2356    }
2357    tcg_gen_shli_i64(ret, arg, 64 - len - ofs);
2358    tcg_gen_sari_i64(ret, ret, 64 - len);
2359}
2360
2361/*
2362 * Extract 64 bits from a 128-bit input, ah:al, starting from ofs.
2363 * Unlike tcg_gen_extract_i64 above, len is fixed at 64.
2364 */
2365void tcg_gen_extract2_i64(TCGv_i64 ret, TCGv_i64 al, TCGv_i64 ah,
2366                          unsigned int ofs)
2367{
2368    tcg_debug_assert(ofs <= 64);
2369    if (ofs == 0) {
2370        tcg_gen_mov_i64(ret, al);
2371    } else if (ofs == 64) {
2372        tcg_gen_mov_i64(ret, ah);
2373    } else if (al == ah) {
2374        tcg_gen_rotri_i64(ret, al, ofs);
2375    } else if (TCG_TARGET_HAS_extract2_i64) {
2376        tcg_gen_op4i_i64(INDEX_op_extract2_i64, ret, al, ah, ofs);
2377    } else {
2378        TCGv_i64 t0 = tcg_temp_new_i64();
2379        tcg_gen_shri_i64(t0, al, ofs);
2380        tcg_gen_deposit_i64(ret, t0, ah, 64 - ofs, ofs);
2381        tcg_temp_free_i64(t0);
2382    }
2383}
2384
2385void tcg_gen_movcond_i64(TCGCond cond, TCGv_i64 ret, TCGv_i64 c1,
2386                         TCGv_i64 c2, TCGv_i64 v1, TCGv_i64 v2)
2387{
2388    if (cond == TCG_COND_ALWAYS) {
2389        tcg_gen_mov_i64(ret, v1);
2390    } else if (cond == TCG_COND_NEVER) {
2391        tcg_gen_mov_i64(ret, v2);
2392    } else if (TCG_TARGET_REG_BITS == 32) {
2393        TCGv_i32 t0 = tcg_temp_new_i32();
2394        TCGv_i32 t1 = tcg_temp_new_i32();
2395        tcg_gen_op6i_i32(INDEX_op_setcond2_i32, t0,
2396                         TCGV_LOW(c1), TCGV_HIGH(c1),
2397                         TCGV_LOW(c2), TCGV_HIGH(c2), cond);
2398
2399        if (TCG_TARGET_HAS_movcond_i32) {
2400            tcg_gen_movi_i32(t1, 0);
2401            tcg_gen_movcond_i32(TCG_COND_NE, TCGV_LOW(ret), t0, t1,
2402                                TCGV_LOW(v1), TCGV_LOW(v2));
2403            tcg_gen_movcond_i32(TCG_COND_NE, TCGV_HIGH(ret), t0, t1,
2404                                TCGV_HIGH(v1), TCGV_HIGH(v2));
2405        } else {
2406            tcg_gen_neg_i32(t0, t0);
2407
2408            tcg_gen_and_i32(t1, TCGV_LOW(v1), t0);
2409            tcg_gen_andc_i32(TCGV_LOW(ret), TCGV_LOW(v2), t0);
2410            tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(ret), t1);
2411
2412            tcg_gen_and_i32(t1, TCGV_HIGH(v1), t0);
2413            tcg_gen_andc_i32(TCGV_HIGH(ret), TCGV_HIGH(v2), t0);
2414            tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(ret), t1);
2415        }
2416        tcg_temp_free_i32(t0);
2417        tcg_temp_free_i32(t1);
2418    } else if (TCG_TARGET_HAS_movcond_i64) {
2419        tcg_gen_op6i_i64(INDEX_op_movcond_i64, ret, c1, c2, v1, v2, cond);
2420    } else {
2421        TCGv_i64 t0 = tcg_temp_new_i64();
2422        TCGv_i64 t1 = tcg_temp_new_i64();
2423        tcg_gen_setcond_i64(cond, t0, c1, c2);
2424        tcg_gen_neg_i64(t0, t0);
2425        tcg_gen_and_i64(t1, v1, t0);
2426        tcg_gen_andc_i64(ret, v2, t0);
2427        tcg_gen_or_i64(ret, ret, t1);
2428        tcg_temp_free_i64(t0);
2429        tcg_temp_free_i64(t1);
2430    }
2431}
2432
2433void tcg_gen_add2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al,
2434                      TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh)
2435{
2436    if (TCG_TARGET_HAS_add2_i64) {
2437        tcg_gen_op6_i64(INDEX_op_add2_i64, rl, rh, al, ah, bl, bh);
2438    } else {
2439        TCGv_i64 t0 = tcg_temp_new_i64();
2440        TCGv_i64 t1 = tcg_temp_new_i64();
2441        tcg_gen_add_i64(t0, al, bl);
2442        tcg_gen_setcond_i64(TCG_COND_LTU, t1, t0, al);
2443        tcg_gen_add_i64(rh, ah, bh);
2444        tcg_gen_add_i64(rh, rh, t1);
2445        tcg_gen_mov_i64(rl, t0);
2446        tcg_temp_free_i64(t0);
2447        tcg_temp_free_i64(t1);
2448    }
2449}
2450
2451void tcg_gen_sub2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al,
2452                      TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh)
2453{
2454    if (TCG_TARGET_HAS_sub2_i64) {
2455        tcg_gen_op6_i64(INDEX_op_sub2_i64, rl, rh, al, ah, bl, bh);
2456    } else {
2457        TCGv_i64 t0 = tcg_temp_new_i64();
2458        TCGv_i64 t1 = tcg_temp_new_i64();
2459        tcg_gen_sub_i64(t0, al, bl);
2460        tcg_gen_setcond_i64(TCG_COND_LTU, t1, al, bl);
2461        tcg_gen_sub_i64(rh, ah, bh);
2462        tcg_gen_sub_i64(rh, rh, t1);
2463        tcg_gen_mov_i64(rl, t0);
2464        tcg_temp_free_i64(t0);
2465        tcg_temp_free_i64(t1);
2466    }
2467}
2468
2469void tcg_gen_mulu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2)
2470{
2471    if (TCG_TARGET_HAS_mulu2_i64) {
2472        tcg_gen_op4_i64(INDEX_op_mulu2_i64, rl, rh, arg1, arg2);
2473    } else if (TCG_TARGET_HAS_muluh_i64) {
2474        TCGv_i64 t = tcg_temp_new_i64();
2475        tcg_gen_op3_i64(INDEX_op_mul_i64, t, arg1, arg2);
2476        tcg_gen_op3_i64(INDEX_op_muluh_i64, rh, arg1, arg2);
2477        tcg_gen_mov_i64(rl, t);
2478        tcg_temp_free_i64(t);
2479    } else {
2480        TCGv_i64 t0 = tcg_temp_new_i64();
2481        tcg_gen_mul_i64(t0, arg1, arg2);
2482        gen_helper_muluh_i64(rh, arg1, arg2);
2483        tcg_gen_mov_i64(rl, t0);
2484        tcg_temp_free_i64(t0);
2485    }
2486}
2487
2488void tcg_gen_muls2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2)
2489{
2490    if (TCG_TARGET_HAS_muls2_i64) {
2491        tcg_gen_op4_i64(INDEX_op_muls2_i64, rl, rh, arg1, arg2);
2492    } else if (TCG_TARGET_HAS_mulsh_i64) {
2493        TCGv_i64 t = tcg_temp_new_i64();
2494        tcg_gen_op3_i64(INDEX_op_mul_i64, t, arg1, arg2);
2495        tcg_gen_op3_i64(INDEX_op_mulsh_i64, rh, arg1, arg2);
2496        tcg_gen_mov_i64(rl, t);
2497        tcg_temp_free_i64(t);
2498    } else if (TCG_TARGET_HAS_mulu2_i64 || TCG_TARGET_HAS_muluh_i64) {
2499        TCGv_i64 t0 = tcg_temp_new_i64();
2500        TCGv_i64 t1 = tcg_temp_new_i64();
2501        TCGv_i64 t2 = tcg_temp_new_i64();
2502        TCGv_i64 t3 = tcg_temp_new_i64();
2503        tcg_gen_mulu2_i64(t0, t1, arg1, arg2);
2504        /* Adjust for negative inputs.  */
2505        tcg_gen_sari_i64(t2, arg1, 63);
2506        tcg_gen_sari_i64(t3, arg2, 63);
2507        tcg_gen_and_i64(t2, t2, arg2);
2508        tcg_gen_and_i64(t3, t3, arg1);
2509        tcg_gen_sub_i64(rh, t1, t2);
2510        tcg_gen_sub_i64(rh, rh, t3);
2511        tcg_gen_mov_i64(rl, t0);
2512        tcg_temp_free_i64(t0);
2513        tcg_temp_free_i64(t1);
2514        tcg_temp_free_i64(t2);
2515        tcg_temp_free_i64(t3);
2516    } else {
2517        TCGv_i64 t0 = tcg_temp_new_i64();
2518        tcg_gen_mul_i64(t0, arg1, arg2);
2519        gen_helper_mulsh_i64(rh, arg1, arg2);
2520        tcg_gen_mov_i64(rl, t0);
2521        tcg_temp_free_i64(t0);
2522    }
2523}
2524
2525void tcg_gen_mulsu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2)
2526{
2527    TCGv_i64 t0 = tcg_temp_new_i64();
2528    TCGv_i64 t1 = tcg_temp_new_i64();
2529    TCGv_i64 t2 = tcg_temp_new_i64();
2530    tcg_gen_mulu2_i64(t0, t1, arg1, arg2);
2531    /* Adjust for negative input for the signed arg1.  */
2532    tcg_gen_sari_i64(t2, arg1, 63);
2533    tcg_gen_and_i64(t2, t2, arg2);
2534    tcg_gen_sub_i64(rh, t1, t2);
2535    tcg_gen_mov_i64(rl, t0);
2536    tcg_temp_free_i64(t0);
2537    tcg_temp_free_i64(t1);
2538    tcg_temp_free_i64(t2);
2539}
2540
2541void tcg_gen_smin_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b)
2542{
2543    tcg_gen_movcond_i64(TCG_COND_LT, ret, a, b, a, b);
2544}
2545
2546void tcg_gen_umin_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b)
2547{
2548    tcg_gen_movcond_i64(TCG_COND_LTU, ret, a, b, a, b);
2549}
2550
2551void tcg_gen_smax_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b)
2552{
2553    tcg_gen_movcond_i64(TCG_COND_LT, ret, a, b, b, a);
2554}
2555
2556void tcg_gen_umax_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b)
2557{
2558    tcg_gen_movcond_i64(TCG_COND_LTU, ret, a, b, b, a);
2559}
2560
2561void tcg_gen_abs_i64(TCGv_i64 ret, TCGv_i64 a)
2562{
2563    TCGv_i64 t = tcg_temp_new_i64();
2564
2565    tcg_gen_sari_i64(t, a, 63);
2566    tcg_gen_xor_i64(ret, a, t);
2567    tcg_gen_sub_i64(ret, ret, t);
2568    tcg_temp_free_i64(t);
2569}
2570
2571/* Size changing operations.  */
2572
2573void tcg_gen_extrl_i64_i32(TCGv_i32 ret, TCGv_i64 arg)
2574{
2575    if (TCG_TARGET_REG_BITS == 32) {
2576        tcg_gen_mov_i32(ret, TCGV_LOW(arg));
2577    } else if (TCG_TARGET_HAS_extrl_i64_i32) {
2578        tcg_gen_op2(INDEX_op_extrl_i64_i32,
2579                    tcgv_i32_arg(ret), tcgv_i64_arg(arg));
2580    } else {
2581        tcg_gen_mov_i32(ret, (TCGv_i32)arg);
2582    }
2583}
2584
2585void tcg_gen_extrh_i64_i32(TCGv_i32 ret, TCGv_i64 arg)
2586{
2587    if (TCG_TARGET_REG_BITS == 32) {
2588        tcg_gen_mov_i32(ret, TCGV_HIGH(arg));
2589    } else if (TCG_TARGET_HAS_extrh_i64_i32) {
2590        tcg_gen_op2(INDEX_op_extrh_i64_i32,
2591                    tcgv_i32_arg(ret), tcgv_i64_arg(arg));
2592    } else {
2593        TCGv_i64 t = tcg_temp_new_i64();
2594        tcg_gen_shri_i64(t, arg, 32);
2595        tcg_gen_mov_i32(ret, (TCGv_i32)t);
2596        tcg_temp_free_i64(t);
2597    }
2598}
2599
2600void tcg_gen_extu_i32_i64(TCGv_i64 ret, TCGv_i32 arg)
2601{
2602    if (TCG_TARGET_REG_BITS == 32) {
2603        tcg_gen_mov_i32(TCGV_LOW(ret), arg);
2604        tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2605    } else {
2606        tcg_gen_op2(INDEX_op_extu_i32_i64,
2607                    tcgv_i64_arg(ret), tcgv_i32_arg(arg));
2608    }
2609}
2610
2611void tcg_gen_ext_i32_i64(TCGv_i64 ret, TCGv_i32 arg)
2612{
2613    if (TCG_TARGET_REG_BITS == 32) {
2614        tcg_gen_mov_i32(TCGV_LOW(ret), arg);
2615        tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
2616    } else {
2617        tcg_gen_op2(INDEX_op_ext_i32_i64,
2618                    tcgv_i64_arg(ret), tcgv_i32_arg(arg));
2619    }
2620}
2621
2622void tcg_gen_concat_i32_i64(TCGv_i64 dest, TCGv_i32 low, TCGv_i32 high)
2623{
2624    TCGv_i64 tmp;
2625
2626    if (TCG_TARGET_REG_BITS == 32) {
2627        tcg_gen_mov_i32(TCGV_LOW(dest), low);
2628        tcg_gen_mov_i32(TCGV_HIGH(dest), high);
2629        return;
2630    }
2631
2632    tmp = tcg_temp_new_i64();
2633    /* These extensions are only needed for type correctness.
2634       We may be able to do better given target specific information.  */
2635    tcg_gen_extu_i32_i64(tmp, high);
2636    tcg_gen_extu_i32_i64(dest, low);
2637    /* If deposit is available, use it.  Otherwise use the extra
2638       knowledge that we have of the zero-extensions above.  */
2639    if (TCG_TARGET_HAS_deposit_i64 && TCG_TARGET_deposit_i64_valid(32, 32)) {
2640        tcg_gen_deposit_i64(dest, dest, tmp, 32, 32);
2641    } else {
2642        tcg_gen_shli_i64(tmp, tmp, 32);
2643        tcg_gen_or_i64(dest, dest, tmp);
2644    }
2645    tcg_temp_free_i64(tmp);
2646}
2647
2648void tcg_gen_extr_i64_i32(TCGv_i32 lo, TCGv_i32 hi, TCGv_i64 arg)
2649{
2650    if (TCG_TARGET_REG_BITS == 32) {
2651        tcg_gen_mov_i32(lo, TCGV_LOW(arg));
2652        tcg_gen_mov_i32(hi, TCGV_HIGH(arg));
2653    } else {
2654        tcg_gen_extrl_i64_i32(lo, arg);
2655        tcg_gen_extrh_i64_i32(hi, arg);
2656    }
2657}
2658
2659void tcg_gen_extr32_i64(TCGv_i64 lo, TCGv_i64 hi, TCGv_i64 arg)
2660{
2661    tcg_gen_ext32u_i64(lo, arg);
2662    tcg_gen_shri_i64(hi, arg, 32);
2663}
2664
2665/* QEMU specific operations.  */
2666
2667void tcg_gen_exit_tb(TranslationBlock *tb, unsigned idx)
2668{
2669    uintptr_t val = (uintptr_t)tb + idx;
2670
2671    if (tb == NULL) {
2672        tcg_debug_assert(idx == 0);
2673    } else if (idx <= TB_EXIT_IDXMAX) {
2674#ifdef CONFIG_DEBUG_TCG
2675        /* This is an exit following a goto_tb.  Verify that we have
2676           seen this numbered exit before, via tcg_gen_goto_tb.  */
2677        tcg_debug_assert(tcg_ctx->goto_tb_issue_mask & (1 << idx));
2678#endif
2679        /* When not chaining, exit without indicating a link.  */
2680        if (qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN)) {
2681            val = 0;
2682        }
2683    } else {
2684        /* This is an exit via the exitreq label.  */
2685        tcg_debug_assert(idx == TB_EXIT_REQUESTED);
2686    }
2687
2688    plugin_gen_disable_mem_helpers();
2689    tcg_gen_op1i(INDEX_op_exit_tb, val);
2690}
2691
2692void tcg_gen_goto_tb(unsigned idx)
2693{
2694    /* We only support two chained exits.  */
2695    tcg_debug_assert(idx <= TB_EXIT_IDXMAX);
2696#ifdef CONFIG_DEBUG_TCG
2697    /* Verify that we havn't seen this numbered exit before.  */
2698    tcg_debug_assert((tcg_ctx->goto_tb_issue_mask & (1 << idx)) == 0);
2699    tcg_ctx->goto_tb_issue_mask |= 1 << idx;
2700#endif
2701    plugin_gen_disable_mem_helpers();
2702    /* When not chaining, we simply fall through to the "fallback" exit.  */
2703    if (!qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN)) {
2704        tcg_gen_op1i(INDEX_op_goto_tb, idx);
2705    }
2706}
2707
2708void tcg_gen_lookup_and_goto_ptr(void)
2709{
2710    if (TCG_TARGET_HAS_goto_ptr && !qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN)) {
2711        TCGv_ptr ptr;
2712
2713        plugin_gen_disable_mem_helpers();
2714        ptr = tcg_temp_new_ptr();
2715        gen_helper_lookup_tb_ptr(ptr, cpu_env);
2716        tcg_gen_op1i(INDEX_op_goto_ptr, tcgv_ptr_arg(ptr));
2717        tcg_temp_free_ptr(ptr);
2718    } else {
2719        tcg_gen_exit_tb(NULL, 0);
2720    }
2721}
2722
2723static inline MemOp tcg_canonicalize_memop(MemOp op, bool is64, bool st)
2724{
2725    /* Trigger the asserts within as early as possible.  */
2726    (void)get_alignment_bits(op);
2727
2728    switch (op & MO_SIZE) {
2729    case MO_8:
2730        op &= ~MO_BSWAP;
2731        break;
2732    case MO_16:
2733        break;
2734    case MO_32:
2735        if (!is64) {
2736            op &= ~MO_SIGN;
2737        }
2738        break;
2739    case MO_64:
2740        if (!is64) {
2741            tcg_abort();
2742        }
2743        break;
2744    }
2745    if (st) {
2746        op &= ~MO_SIGN;
2747    }
2748    return op;
2749}
2750
2751static void gen_ldst_i32(TCGOpcode opc, TCGv_i32 val, TCGv addr,
2752                         MemOp memop, TCGArg idx)
2753{
2754    TCGMemOpIdx oi = make_memop_idx(memop, idx);
2755#if TARGET_LONG_BITS == 32
2756    tcg_gen_op3i_i32(opc, val, addr, oi);
2757#else
2758    if (TCG_TARGET_REG_BITS == 32) {
2759        tcg_gen_op4i_i32(opc, val, TCGV_LOW(addr), TCGV_HIGH(addr), oi);
2760    } else {
2761        tcg_gen_op3(opc, tcgv_i32_arg(val), tcgv_i64_arg(addr), oi);
2762    }
2763#endif
2764}
2765
2766static void gen_ldst_i64(TCGOpcode opc, TCGv_i64 val, TCGv addr,
2767                         MemOp memop, TCGArg idx)
2768{
2769    TCGMemOpIdx oi = make_memop_idx(memop, idx);
2770#if TARGET_LONG_BITS == 32
2771    if (TCG_TARGET_REG_BITS == 32) {
2772        tcg_gen_op4i_i32(opc, TCGV_LOW(val), TCGV_HIGH(val), addr, oi);
2773    } else {
2774        tcg_gen_op3(opc, tcgv_i64_arg(val), tcgv_i32_arg(addr), oi);
2775    }
2776#else
2777    if (TCG_TARGET_REG_BITS == 32) {
2778        tcg_gen_op5i_i32(opc, TCGV_LOW(val), TCGV_HIGH(val),
2779                         TCGV_LOW(addr), TCGV_HIGH(addr), oi);
2780    } else {
2781        tcg_gen_op3i_i64(opc, val, addr, oi);
2782    }
2783#endif
2784}
2785
2786static void tcg_gen_req_mo(TCGBar type)
2787{
2788#ifdef TCG_GUEST_DEFAULT_MO
2789    type &= TCG_GUEST_DEFAULT_MO;
2790#endif
2791    type &= ~TCG_TARGET_DEFAULT_MO;
2792    if (type) {
2793        tcg_gen_mb(type | TCG_BAR_SC);
2794    }
2795}
2796
2797static inline void plugin_gen_mem_callbacks(TCGv vaddr, uint16_t info)
2798{
2799#ifdef CONFIG_PLUGIN
2800    if (tcg_ctx->plugin_insn == NULL) {
2801        return;
2802    }
2803    plugin_gen_empty_mem_callback(vaddr, info);
2804#endif
2805}
2806
2807void tcg_gen_qemu_ld_i32(TCGv_i32 val, TCGv addr, TCGArg idx, MemOp memop)
2808{
2809    MemOp orig_memop;
2810    uint16_t info = trace_mem_get_info(memop, idx, 0);
2811
2812    tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
2813    memop = tcg_canonicalize_memop(memop, 0, 0);
2814    trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env, addr, info);
2815
2816    orig_memop = memop;
2817    if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) {
2818        memop &= ~MO_BSWAP;
2819        /* The bswap primitive requires zero-extended input.  */
2820        if ((memop & MO_SSIZE) == MO_SW) {
2821            memop &= ~MO_SIGN;
2822        }
2823    }
2824
2825    gen_ldst_i32(INDEX_op_qemu_ld_i32, val, addr, memop, idx);
2826    plugin_gen_mem_callbacks(addr, info);
2827
2828    if ((orig_memop ^ memop) & MO_BSWAP) {
2829        switch (orig_memop & MO_SIZE) {
2830        case MO_16:
2831            tcg_gen_bswap16_i32(val, val);
2832            if (orig_memop & MO_SIGN) {
2833                tcg_gen_ext16s_i32(val, val);
2834            }
2835            break;
2836        case MO_32:
2837            tcg_gen_bswap32_i32(val, val);
2838            break;
2839        default:
2840            g_assert_not_reached();
2841        }
2842    }
2843}
2844
2845void tcg_gen_qemu_st_i32(TCGv_i32 val, TCGv addr, TCGArg idx, MemOp memop)
2846{
2847    TCGv_i32 swap = NULL;
2848    uint16_t info = trace_mem_get_info(memop, idx, 1);
2849
2850    tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
2851    memop = tcg_canonicalize_memop(memop, 0, 1);
2852    trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env, addr, info);
2853
2854    if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) {
2855        swap = tcg_temp_new_i32();
2856        switch (memop & MO_SIZE) {
2857        case MO_16:
2858            tcg_gen_ext16u_i32(swap, val);
2859            tcg_gen_bswap16_i32(swap, swap);
2860            break;
2861        case MO_32:
2862            tcg_gen_bswap32_i32(swap, val);
2863            break;
2864        default:
2865            g_assert_not_reached();
2866        }
2867        val = swap;
2868        memop &= ~MO_BSWAP;
2869    }
2870
2871    gen_ldst_i32(INDEX_op_qemu_st_i32, val, addr, memop, idx);
2872    plugin_gen_mem_callbacks(addr, info);
2873
2874    if (swap) {
2875        tcg_temp_free_i32(swap);
2876    }
2877}
2878
2879void tcg_gen_qemu_ld_i64(TCGv_i64 val, TCGv addr, TCGArg idx, MemOp memop)
2880{
2881    MemOp orig_memop;
2882    uint16_t info;
2883
2884    if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) {
2885        tcg_gen_qemu_ld_i32(TCGV_LOW(val), addr, idx, memop);
2886        if (memop & MO_SIGN) {
2887            tcg_gen_sari_i32(TCGV_HIGH(val), TCGV_LOW(val), 31);
2888        } else {
2889            tcg_gen_movi_i32(TCGV_HIGH(val), 0);
2890        }
2891        return;
2892    }
2893
2894    tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
2895    memop = tcg_canonicalize_memop(memop, 1, 0);
2896    info = trace_mem_get_info(memop, idx, 0);
2897    trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env, addr, info);
2898
2899    orig_memop = memop;
2900    if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) {
2901        memop &= ~MO_BSWAP;
2902        /* The bswap primitive requires zero-extended input.  */
2903        if ((memop & MO_SIGN) && (memop & MO_SIZE) < MO_64) {
2904            memop &= ~MO_SIGN;
2905        }
2906    }
2907
2908    gen_ldst_i64(INDEX_op_qemu_ld_i64, val, addr, memop, idx);
2909    plugin_gen_mem_callbacks(addr, info);
2910
2911    if ((orig_memop ^ memop) & MO_BSWAP) {
2912        switch (orig_memop & MO_SIZE) {
2913        case MO_16:
2914            tcg_gen_bswap16_i64(val, val);
2915            if (orig_memop & MO_SIGN) {
2916                tcg_gen_ext16s_i64(val, val);
2917            }
2918            break;
2919        case MO_32:
2920            tcg_gen_bswap32_i64(val, val);
2921            if (orig_memop & MO_SIGN) {
2922                tcg_gen_ext32s_i64(val, val);
2923            }
2924            break;
2925        case MO_64:
2926            tcg_gen_bswap64_i64(val, val);
2927            break;
2928        default:
2929            g_assert_not_reached();
2930        }
2931    }
2932}
2933
2934void tcg_gen_qemu_st_i64(TCGv_i64 val, TCGv addr, TCGArg idx, MemOp memop)
2935{
2936    TCGv_i64 swap = NULL;
2937    uint16_t info;
2938
2939    if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) {
2940        tcg_gen_qemu_st_i32(TCGV_LOW(val), addr, idx, memop);
2941        return;
2942    }
2943
2944    tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
2945    memop = tcg_canonicalize_memop(memop, 1, 1);
2946    info = trace_mem_get_info(memop, idx, 1);
2947    trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env, addr, info);
2948
2949    if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) {
2950        swap = tcg_temp_new_i64();
2951        switch (memop & MO_SIZE) {
2952        case MO_16:
2953            tcg_gen_ext16u_i64(swap, val);
2954            tcg_gen_bswap16_i64(swap, swap);
2955            break;
2956        case MO_32:
2957            tcg_gen_ext32u_i64(swap, val);
2958            tcg_gen_bswap32_i64(swap, swap);
2959            break;
2960        case MO_64:
2961            tcg_gen_bswap64_i64(swap, val);
2962            break;
2963        default:
2964            g_assert_not_reached();
2965        }
2966        val = swap;
2967        memop &= ~MO_BSWAP;
2968    }
2969
2970    gen_ldst_i64(INDEX_op_qemu_st_i64, val, addr, memop, idx);
2971    plugin_gen_mem_callbacks(addr, info);
2972
2973    if (swap) {
2974        tcg_temp_free_i64(swap);
2975    }
2976}
2977
2978static void tcg_gen_ext_i32(TCGv_i32 ret, TCGv_i32 val, MemOp opc)
2979{
2980    switch (opc & MO_SSIZE) {
2981    case MO_SB:
2982        tcg_gen_ext8s_i32(ret, val);
2983        break;
2984    case MO_UB:
2985        tcg_gen_ext8u_i32(ret, val);
2986        break;
2987    case MO_SW:
2988        tcg_gen_ext16s_i32(ret, val);
2989        break;
2990    case MO_UW:
2991        tcg_gen_ext16u_i32(ret, val);
2992        break;
2993    default:
2994        tcg_gen_mov_i32(ret, val);
2995        break;
2996    }
2997}
2998
2999static void tcg_gen_ext_i64(TCGv_i64 ret, TCGv_i64 val, MemOp opc)
3000{
3001    switch (opc & MO_SSIZE) {
3002    case MO_SB:
3003        tcg_gen_ext8s_i64(ret, val);
3004        break;
3005    case MO_UB:
3006        tcg_gen_ext8u_i64(ret, val);
3007        break;
3008    case MO_SW:
3009        tcg_gen_ext16s_i64(ret, val);
3010        break;
3011    case MO_UW:
3012        tcg_gen_ext16u_i64(ret, val);
3013        break;
3014    case MO_SL:
3015        tcg_gen_ext32s_i64(ret, val);
3016        break;
3017    case MO_UL:
3018        tcg_gen_ext32u_i64(ret, val);
3019        break;
3020    default:
3021        tcg_gen_mov_i64(ret, val);
3022        break;
3023    }
3024}
3025
3026#ifdef CONFIG_SOFTMMU
3027typedef void (*gen_atomic_cx_i32)(TCGv_i32, TCGv_env, TCGv,
3028                                  TCGv_i32, TCGv_i32, TCGv_i32);
3029typedef void (*gen_atomic_cx_i64)(TCGv_i64, TCGv_env, TCGv,
3030                                  TCGv_i64, TCGv_i64, TCGv_i32);
3031typedef void (*gen_atomic_op_i32)(TCGv_i32, TCGv_env, TCGv,
3032                                  TCGv_i32, TCGv_i32);
3033typedef void (*gen_atomic_op_i64)(TCGv_i64, TCGv_env, TCGv,
3034                                  TCGv_i64, TCGv_i32);
3035#else
3036typedef void (*gen_atomic_cx_i32)(TCGv_i32, TCGv_env, TCGv, TCGv_i32, TCGv_i32);
3037typedef void (*gen_atomic_cx_i64)(TCGv_i64, TCGv_env, TCGv, TCGv_i64, TCGv_i64);
3038typedef void (*gen_atomic_op_i32)(TCGv_i32, TCGv_env, TCGv, TCGv_i32);
3039typedef void (*gen_atomic_op_i64)(TCGv_i64, TCGv_env, TCGv, TCGv_i64);
3040#endif
3041
3042#ifdef CONFIG_ATOMIC64
3043# define WITH_ATOMIC64(X) X,
3044#else
3045# define WITH_ATOMIC64(X)
3046#endif
3047
3048static void * const table_cmpxchg[16] = {
3049    [MO_8] = gen_helper_atomic_cmpxchgb,
3050    [MO_16 | MO_LE] = gen_helper_atomic_cmpxchgw_le,
3051    [MO_16 | MO_BE] = gen_helper_atomic_cmpxchgw_be,
3052    [MO_32 | MO_LE] = gen_helper_atomic_cmpxchgl_le,
3053    [MO_32 | MO_BE] = gen_helper_atomic_cmpxchgl_be,
3054    WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_cmpxchgq_le)
3055    WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_cmpxchgq_be)
3056};
3057
3058void tcg_gen_atomic_cmpxchg_i32(TCGv_i32 retv, TCGv addr, TCGv_i32 cmpv,
3059                                TCGv_i32 newv, TCGArg idx, MemOp memop)
3060{
3061    memop = tcg_canonicalize_memop(memop, 0, 0);
3062
3063    if (!(tcg_ctx->tb_cflags & CF_PARALLEL)) {
3064        TCGv_i32 t1 = tcg_temp_new_i32();
3065        TCGv_i32 t2 = tcg_temp_new_i32();
3066
3067        tcg_gen_ext_i32(t2, cmpv, memop & MO_SIZE);
3068
3069        tcg_gen_qemu_ld_i32(t1, addr, idx, memop & ~MO_SIGN);
3070        tcg_gen_movcond_i32(TCG_COND_EQ, t2, t1, t2, newv, t1);
3071        tcg_gen_qemu_st_i32(t2, addr, idx, memop);
3072        tcg_temp_free_i32(t2);
3073
3074        if (memop & MO_SIGN) {
3075            tcg_gen_ext_i32(retv, t1, memop);
3076        } else {
3077            tcg_gen_mov_i32(retv, t1);
3078        }
3079        tcg_temp_free_i32(t1);
3080    } else {
3081        gen_atomic_cx_i32 gen;
3082
3083        gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)];
3084        tcg_debug_assert(gen != NULL);
3085
3086#ifdef CONFIG_SOFTMMU
3087        {
3088            TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop & ~MO_SIGN, idx));
3089            gen(retv, cpu_env, addr, cmpv, newv, oi);
3090            tcg_temp_free_i32(oi);
3091        }
3092#else
3093        gen(retv, cpu_env, addr, cmpv, newv);
3094#endif
3095
3096        if (memop & MO_SIGN) {
3097            tcg_gen_ext_i32(retv, retv, memop);
3098        }
3099    }
3100}
3101
3102void tcg_gen_atomic_cmpxchg_i64(TCGv_i64 retv, TCGv addr, TCGv_i64 cmpv,
3103                                TCGv_i64 newv, TCGArg idx, MemOp memop)
3104{
3105    memop = tcg_canonicalize_memop(memop, 1, 0);
3106
3107    if (!(tcg_ctx->tb_cflags & CF_PARALLEL)) {
3108        TCGv_i64 t1 = tcg_temp_new_i64();
3109        TCGv_i64 t2 = tcg_temp_new_i64();
3110
3111        tcg_gen_ext_i64(t2, cmpv, memop & MO_SIZE);
3112
3113        tcg_gen_qemu_ld_i64(t1, addr, idx, memop & ~MO_SIGN);
3114        tcg_gen_movcond_i64(TCG_COND_EQ, t2, t1, t2, newv, t1);
3115        tcg_gen_qemu_st_i64(t2, addr, idx, memop);
3116        tcg_temp_free_i64(t2);
3117
3118        if (memop & MO_SIGN) {
3119            tcg_gen_ext_i64(retv, t1, memop);
3120        } else {
3121            tcg_gen_mov_i64(retv, t1);
3122        }
3123        tcg_temp_free_i64(t1);
3124    } else if ((memop & MO_SIZE) == MO_64) {
3125#ifdef CONFIG_ATOMIC64
3126        gen_atomic_cx_i64 gen;
3127
3128        gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)];
3129        tcg_debug_assert(gen != NULL);
3130
3131#ifdef CONFIG_SOFTMMU
3132        {
3133            TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop, idx));
3134            gen(retv, cpu_env, addr, cmpv, newv, oi);
3135            tcg_temp_free_i32(oi);
3136        }
3137#else
3138        gen(retv, cpu_env, addr, cmpv, newv);
3139#endif
3140#else
3141        gen_helper_exit_atomic(cpu_env);
3142        /* Produce a result, so that we have a well-formed opcode stream
3143           with respect to uses of the result in the (dead) code following.  */
3144        tcg_gen_movi_i64(retv, 0);
3145#endif /* CONFIG_ATOMIC64 */
3146    } else {
3147        TCGv_i32 c32 = tcg_temp_new_i32();
3148        TCGv_i32 n32 = tcg_temp_new_i32();
3149        TCGv_i32 r32 = tcg_temp_new_i32();
3150
3151        tcg_gen_extrl_i64_i32(c32, cmpv);
3152        tcg_gen_extrl_i64_i32(n32, newv);
3153        tcg_gen_atomic_cmpxchg_i32(r32, addr, c32, n32, idx, memop & ~MO_SIGN);
3154        tcg_temp_free_i32(c32);
3155        tcg_temp_free_i32(n32);
3156
3157        tcg_gen_extu_i32_i64(retv, r32);
3158        tcg_temp_free_i32(r32);
3159
3160        if (memop & MO_SIGN) {
3161            tcg_gen_ext_i64(retv, retv, memop);
3162        }
3163    }
3164}
3165
3166static void do_nonatomic_op_i32(TCGv_i32 ret, TCGv addr, TCGv_i32 val,
3167                                TCGArg idx, MemOp memop, bool new_val,
3168                                void (*gen)(TCGv_i32, TCGv_i32, TCGv_i32))
3169{
3170    TCGv_i32 t1 = tcg_temp_new_i32();
3171    TCGv_i32 t2 = tcg_temp_new_i32();
3172
3173    memop = tcg_canonicalize_memop(memop, 0, 0);
3174
3175    tcg_gen_qemu_ld_i32(t1, addr, idx, memop & ~MO_SIGN);
3176    gen(t2, t1, val);
3177    tcg_gen_qemu_st_i32(t2, addr, idx, memop);
3178
3179    tcg_gen_ext_i32(ret, (new_val ? t2 : t1), memop);
3180    tcg_temp_free_i32(t1);
3181    tcg_temp_free_i32(t2);
3182}
3183
3184static void do_atomic_op_i32(TCGv_i32 ret, TCGv addr, TCGv_i32 val,
3185                             TCGArg idx, MemOp memop, void * const table[])
3186{
3187    gen_atomic_op_i32 gen;
3188
3189    memop = tcg_canonicalize_memop(memop, 0, 0);
3190
3191    gen = table[memop & (MO_SIZE | MO_BSWAP)];
3192    tcg_debug_assert(gen != NULL);
3193
3194#ifdef CONFIG_SOFTMMU
3195    {
3196        TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop & ~MO_SIGN, idx));
3197        gen(ret, cpu_env, addr, val, oi);
3198        tcg_temp_free_i32(oi);
3199    }
3200#else
3201    gen(ret, cpu_env, addr, val);
3202#endif
3203
3204    if (memop & MO_SIGN) {
3205        tcg_gen_ext_i32(ret, ret, memop);
3206    }
3207}
3208
3209static void do_nonatomic_op_i64(TCGv_i64 ret, TCGv addr, TCGv_i64 val,
3210                                TCGArg idx, MemOp memop, bool new_val,
3211                                void (*gen)(TCGv_i64, TCGv_i64, TCGv_i64))
3212{
3213    TCGv_i64 t1 = tcg_temp_new_i64();
3214    TCGv_i64 t2 = tcg_temp_new_i64();
3215
3216    memop = tcg_canonicalize_memop(memop, 1, 0);
3217
3218    tcg_gen_qemu_ld_i64(t1, addr, idx, memop & ~MO_SIGN);
3219    gen(t2, t1, val);
3220    tcg_gen_qemu_st_i64(t2, addr, idx, memop);
3221
3222    tcg_gen_ext_i64(ret, (new_val ? t2 : t1), memop);
3223    tcg_temp_free_i64(t1);
3224    tcg_temp_free_i64(t2);
3225}
3226
3227static void do_atomic_op_i64(TCGv_i64 ret, TCGv addr, TCGv_i64 val,
3228                             TCGArg idx, MemOp memop, void * const table[])
3229{
3230    memop = tcg_canonicalize_memop(memop, 1, 0);
3231
3232    if ((memop & MO_SIZE) == MO_64) {
3233#ifdef CONFIG_ATOMIC64
3234        gen_atomic_op_i64 gen;
3235
3236        gen = table[memop & (MO_SIZE | MO_BSWAP)];
3237        tcg_debug_assert(gen != NULL);
3238
3239#ifdef CONFIG_SOFTMMU
3240        {
3241            TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop & ~MO_SIGN, idx));
3242            gen(ret, cpu_env, addr, val, oi);
3243            tcg_temp_free_i32(oi);
3244        }
3245#else
3246        gen(ret, cpu_env, addr, val);
3247#endif
3248#else
3249        gen_helper_exit_atomic(cpu_env);
3250        /* Produce a result, so that we have a well-formed opcode stream
3251           with respect to uses of the result in the (dead) code following.  */
3252        tcg_gen_movi_i64(ret, 0);
3253#endif /* CONFIG_ATOMIC64 */
3254    } else {
3255        TCGv_i32 v32 = tcg_temp_new_i32();
3256        TCGv_i32 r32 = tcg_temp_new_i32();
3257
3258        tcg_gen_extrl_i64_i32(v32, val);
3259        do_atomic_op_i32(r32, addr, v32, idx, memop & ~MO_SIGN, table);
3260        tcg_temp_free_i32(v32);
3261
3262        tcg_gen_extu_i32_i64(ret, r32);
3263        tcg_temp_free_i32(r32);
3264
3265        if (memop & MO_SIGN) {
3266            tcg_gen_ext_i64(ret, ret, memop);
3267        }
3268    }
3269}
3270
3271#define GEN_ATOMIC_HELPER(NAME, OP, NEW)                                \
3272static void * const table_##NAME[16] = {                                \
3273    [MO_8] = gen_helper_atomic_##NAME##b,                               \
3274    [MO_16 | MO_LE] = gen_helper_atomic_##NAME##w_le,                   \
3275    [MO_16 | MO_BE] = gen_helper_atomic_##NAME##w_be,                   \
3276    [MO_32 | MO_LE] = gen_helper_atomic_##NAME##l_le,                   \
3277    [MO_32 | MO_BE] = gen_helper_atomic_##NAME##l_be,                   \
3278    WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_##NAME##q_le)     \
3279    WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_##NAME##q_be)     \
3280};                                                                      \
3281void tcg_gen_atomic_##NAME##_i32                                        \
3282    (TCGv_i32 ret, TCGv addr, TCGv_i32 val, TCGArg idx, MemOp memop)    \
3283{                                                                       \
3284    if (tcg_ctx->tb_cflags & CF_PARALLEL) {                             \
3285        do_atomic_op_i32(ret, addr, val, idx, memop, table_##NAME);     \
3286    } else {                                                            \
3287        do_nonatomic_op_i32(ret, addr, val, idx, memop, NEW,            \
3288                            tcg_gen_##OP##_i32);                        \
3289    }                                                                   \
3290}                                                                       \
3291void tcg_gen_atomic_##NAME##_i64                                        \
3292    (TCGv_i64 ret, TCGv addr, TCGv_i64 val, TCGArg idx, MemOp memop)    \
3293{                                                                       \
3294    if (tcg_ctx->tb_cflags & CF_PARALLEL) {                             \
3295        do_atomic_op_i64(ret, addr, val, idx, memop, table_##NAME);     \
3296    } else {                                                            \
3297        do_nonatomic_op_i64(ret, addr, val, idx, memop, NEW,            \
3298                            tcg_gen_##OP##_i64);                        \
3299    }                                                                   \
3300}
3301
3302GEN_ATOMIC_HELPER(fetch_add, add, 0)
3303GEN_ATOMIC_HELPER(fetch_and, and, 0)
3304GEN_ATOMIC_HELPER(fetch_or, or, 0)
3305GEN_ATOMIC_HELPER(fetch_xor, xor, 0)
3306GEN_ATOMIC_HELPER(fetch_smin, smin, 0)
3307GEN_ATOMIC_HELPER(fetch_umin, umin, 0)
3308GEN_ATOMIC_HELPER(fetch_smax, smax, 0)
3309GEN_ATOMIC_HELPER(fetch_umax, umax, 0)
3310
3311GEN_ATOMIC_HELPER(add_fetch, add, 1)
3312GEN_ATOMIC_HELPER(and_fetch, and, 1)
3313GEN_ATOMIC_HELPER(or_fetch, or, 1)
3314GEN_ATOMIC_HELPER(xor_fetch, xor, 1)
3315GEN_ATOMIC_HELPER(smin_fetch, smin, 1)
3316GEN_ATOMIC_HELPER(umin_fetch, umin, 1)
3317GEN_ATOMIC_HELPER(smax_fetch, smax, 1)
3318GEN_ATOMIC_HELPER(umax_fetch, umax, 1)
3319
3320static void tcg_gen_mov2_i32(TCGv_i32 r, TCGv_i32 a, TCGv_i32 b)
3321{
3322    tcg_gen_mov_i32(r, b);
3323}
3324
3325static void tcg_gen_mov2_i64(TCGv_i64 r, TCGv_i64 a, TCGv_i64 b)
3326{
3327    tcg_gen_mov_i64(r, b);
3328}
3329
3330GEN_ATOMIC_HELPER(xchg, mov2, 0)
3331
3332#undef GEN_ATOMIC_HELPER
3333