qemu/target/s390x/translate_vx.inc.c
<<
>>
Prefs
   1/*
   2 * QEMU TCG support -- s390x vector instruction translation functions
   3 *
   4 * Copyright (C) 2019 Red Hat Inc
   5 *
   6 * Authors:
   7 *   David Hildenbrand <david@redhat.com>
   8 *
   9 * This work is licensed under the terms of the GNU GPL, version 2 or later.
  10 * See the COPYING file in the top-level directory.
  11 */
  12
  13/*
  14 * For most instructions that use the same element size for reads and
  15 * writes, we can use real gvec vector expansion, which potantially uses
  16 * real host vector instructions. As they only work up to 64 bit elements,
  17 * 128 bit elements (vector is a single element) have to be handled
  18 * differently. Operations that are too complicated to encode via TCG ops
  19 * are handled via gvec ool (out-of-line) handlers.
  20 *
  21 * As soon as instructions use different element sizes for reads and writes
  22 * or access elements "out of their element scope" we expand them manually
  23 * in fancy loops, as gvec expansion does not deal with actual element
  24 * numbers and does also not support access to other elements.
  25 *
  26 * 128 bit elements:
  27 *  As we only have i32/i64, such elements have to be loaded into two
  28 *  i64 values and can then be processed e.g. by tcg_gen_add2_i64.
  29 *
  30 * Sizes:
  31 *  On s390x, the operand size (oprsz) and the maximum size (maxsz) are
  32 *  always 16 (128 bit). What gvec code calls "vece", s390x calls "es",
  33 *  a.k.a. "element size". These values nicely map to MO_8 ... MO_64. Only
  34 *  128 bit element size has to be treated in a special way (MO_64 + 1).
  35 *  We will use ES_* instead of MO_* for this reason in this file.
  36 *
  37 * CC handling:
  38 *  As gvec ool-helpers can currently not return values (besides via
  39 *  pointers like vectors or cpu_env), whenever we have to set the CC and
  40 *  can't conclude the value from the result vector, we will directly
  41 *  set it in "env->cc_op" and mark it as static via set_cc_static()".
  42 *  Whenever this is done, the helper writes globals (cc_op).
  43 */
  44
  45#define NUM_VEC_ELEMENT_BYTES(es) (1 << (es))
  46#define NUM_VEC_ELEMENTS(es) (16 / NUM_VEC_ELEMENT_BYTES(es))
  47#define NUM_VEC_ELEMENT_BITS(es) (NUM_VEC_ELEMENT_BYTES(es) * BITS_PER_BYTE)
  48
  49#define ES_8    MO_8
  50#define ES_16   MO_16
  51#define ES_32   MO_32
  52#define ES_64   MO_64
  53#define ES_128  4
  54
  55/* Floating-Point Format */
  56#define FPF_SHORT       2
  57#define FPF_LONG        3
  58#define FPF_EXT         4
  59
  60static inline bool valid_vec_element(uint8_t enr, MemOp es)
  61{
  62    return !(enr & ~(NUM_VEC_ELEMENTS(es) - 1));
  63}
  64
  65static void read_vec_element_i64(TCGv_i64 dst, uint8_t reg, uint8_t enr,
  66                                 MemOp memop)
  67{
  68    const int offs = vec_reg_offset(reg, enr, memop & MO_SIZE);
  69
  70    switch (memop) {
  71    case ES_8:
  72        tcg_gen_ld8u_i64(dst, cpu_env, offs);
  73        break;
  74    case ES_16:
  75        tcg_gen_ld16u_i64(dst, cpu_env, offs);
  76        break;
  77    case ES_32:
  78        tcg_gen_ld32u_i64(dst, cpu_env, offs);
  79        break;
  80    case ES_8 | MO_SIGN:
  81        tcg_gen_ld8s_i64(dst, cpu_env, offs);
  82        break;
  83    case ES_16 | MO_SIGN:
  84        tcg_gen_ld16s_i64(dst, cpu_env, offs);
  85        break;
  86    case ES_32 | MO_SIGN:
  87        tcg_gen_ld32s_i64(dst, cpu_env, offs);
  88        break;
  89    case ES_64:
  90    case ES_64 | MO_SIGN:
  91        tcg_gen_ld_i64(dst, cpu_env, offs);
  92        break;
  93    default:
  94        g_assert_not_reached();
  95    }
  96}
  97
  98static void read_vec_element_i32(TCGv_i32 dst, uint8_t reg, uint8_t enr,
  99                                 MemOp memop)
 100{
 101    const int offs = vec_reg_offset(reg, enr, memop & MO_SIZE);
 102
 103    switch (memop) {
 104    case ES_8:
 105        tcg_gen_ld8u_i32(dst, cpu_env, offs);
 106        break;
 107    case ES_16:
 108        tcg_gen_ld16u_i32(dst, cpu_env, offs);
 109        break;
 110    case ES_8 | MO_SIGN:
 111        tcg_gen_ld8s_i32(dst, cpu_env, offs);
 112        break;
 113    case ES_16 | MO_SIGN:
 114        tcg_gen_ld16s_i32(dst, cpu_env, offs);
 115        break;
 116    case ES_32:
 117    case ES_32 | MO_SIGN:
 118        tcg_gen_ld_i32(dst, cpu_env, offs);
 119        break;
 120    default:
 121        g_assert_not_reached();
 122    }
 123}
 124
 125static void write_vec_element_i64(TCGv_i64 src, int reg, uint8_t enr,
 126                                  MemOp memop)
 127{
 128    const int offs = vec_reg_offset(reg, enr, memop & MO_SIZE);
 129
 130    switch (memop) {
 131    case ES_8:
 132        tcg_gen_st8_i64(src, cpu_env, offs);
 133        break;
 134    case ES_16:
 135        tcg_gen_st16_i64(src, cpu_env, offs);
 136        break;
 137    case ES_32:
 138        tcg_gen_st32_i64(src, cpu_env, offs);
 139        break;
 140    case ES_64:
 141        tcg_gen_st_i64(src, cpu_env, offs);
 142        break;
 143    default:
 144        g_assert_not_reached();
 145    }
 146}
 147
 148static void write_vec_element_i32(TCGv_i32 src, int reg, uint8_t enr,
 149                                  MemOp memop)
 150{
 151    const int offs = vec_reg_offset(reg, enr, memop & MO_SIZE);
 152
 153    switch (memop) {
 154    case ES_8:
 155        tcg_gen_st8_i32(src, cpu_env, offs);
 156        break;
 157    case ES_16:
 158        tcg_gen_st16_i32(src, cpu_env, offs);
 159        break;
 160    case ES_32:
 161        tcg_gen_st_i32(src, cpu_env, offs);
 162        break;
 163    default:
 164        g_assert_not_reached();
 165    }
 166}
 167
 168static void get_vec_element_ptr_i64(TCGv_ptr ptr, uint8_t reg, TCGv_i64 enr,
 169                                    uint8_t es)
 170{
 171    TCGv_i64 tmp = tcg_temp_new_i64();
 172
 173    /* mask off invalid parts from the element nr */
 174    tcg_gen_andi_i64(tmp, enr, NUM_VEC_ELEMENTS(es) - 1);
 175
 176    /* convert it to an element offset relative to cpu_env (vec_reg_offset() */
 177    tcg_gen_shli_i64(tmp, tmp, es);
 178#ifndef HOST_WORDS_BIGENDIAN
 179    tcg_gen_xori_i64(tmp, tmp, 8 - NUM_VEC_ELEMENT_BYTES(es));
 180#endif
 181    tcg_gen_addi_i64(tmp, tmp, vec_full_reg_offset(reg));
 182
 183    /* generate the final ptr by adding cpu_env */
 184    tcg_gen_trunc_i64_ptr(ptr, tmp);
 185    tcg_gen_add_ptr(ptr, ptr, cpu_env);
 186
 187    tcg_temp_free_i64(tmp);
 188}
 189
 190#define gen_gvec_2(v1, v2, gen) \
 191    tcg_gen_gvec_2(vec_full_reg_offset(v1), vec_full_reg_offset(v2), \
 192                   16, 16, gen)
 193#define gen_gvec_2s(v1, v2, c, gen) \
 194    tcg_gen_gvec_2s(vec_full_reg_offset(v1), vec_full_reg_offset(v2), \
 195                    16, 16, c, gen)
 196#define gen_gvec_2_ool(v1, v2, data, fn) \
 197    tcg_gen_gvec_2_ool(vec_full_reg_offset(v1), vec_full_reg_offset(v2), \
 198                       16, 16, data, fn)
 199#define gen_gvec_2i_ool(v1, v2, c, data, fn) \
 200    tcg_gen_gvec_2i_ool(vec_full_reg_offset(v1), vec_full_reg_offset(v2), \
 201                        c, 16, 16, data, fn)
 202#define gen_gvec_2_ptr(v1, v2, ptr, data, fn) \
 203    tcg_gen_gvec_2_ptr(vec_full_reg_offset(v1), vec_full_reg_offset(v2), \
 204                       ptr, 16, 16, data, fn)
 205#define gen_gvec_3(v1, v2, v3, gen) \
 206    tcg_gen_gvec_3(vec_full_reg_offset(v1), vec_full_reg_offset(v2), \
 207                   vec_full_reg_offset(v3), 16, 16, gen)
 208#define gen_gvec_3_ool(v1, v2, v3, data, fn) \
 209    tcg_gen_gvec_3_ool(vec_full_reg_offset(v1), vec_full_reg_offset(v2), \
 210                       vec_full_reg_offset(v3), 16, 16, data, fn)
 211#define gen_gvec_3_ptr(v1, v2, v3, ptr, data, fn) \
 212    tcg_gen_gvec_3_ptr(vec_full_reg_offset(v1), vec_full_reg_offset(v2), \
 213                       vec_full_reg_offset(v3), ptr, 16, 16, data, fn)
 214#define gen_gvec_3i(v1, v2, v3, c, gen) \
 215    tcg_gen_gvec_3i(vec_full_reg_offset(v1), vec_full_reg_offset(v2), \
 216                    vec_full_reg_offset(v3), 16, 16, c, gen)
 217#define gen_gvec_4(v1, v2, v3, v4, gen) \
 218    tcg_gen_gvec_4(vec_full_reg_offset(v1), vec_full_reg_offset(v2), \
 219                   vec_full_reg_offset(v3), vec_full_reg_offset(v4), \
 220                   16, 16, gen)
 221#define gen_gvec_4_ool(v1, v2, v3, v4, data, fn) \
 222    tcg_gen_gvec_4_ool(vec_full_reg_offset(v1), vec_full_reg_offset(v2), \
 223                       vec_full_reg_offset(v3), vec_full_reg_offset(v4), \
 224                       16, 16, data, fn)
 225#define gen_gvec_4_ptr(v1, v2, v3, v4, ptr, data, fn) \
 226    tcg_gen_gvec_4_ptr(vec_full_reg_offset(v1), vec_full_reg_offset(v2), \
 227                       vec_full_reg_offset(v3), vec_full_reg_offset(v4), \
 228                       ptr, 16, 16, data, fn)
 229#define gen_gvec_dup_i64(es, v1, c) \
 230    tcg_gen_gvec_dup_i64(es, vec_full_reg_offset(v1), 16, 16, c)
 231#define gen_gvec_mov(v1, v2) \
 232    tcg_gen_gvec_mov(0, vec_full_reg_offset(v1), vec_full_reg_offset(v2), 16, \
 233                     16)
 234#define gen_gvec_dup64i(v1, c) \
 235    tcg_gen_gvec_dup64i(vec_full_reg_offset(v1), 16, 16, c)
 236#define gen_gvec_fn_2(fn, es, v1, v2) \
 237    tcg_gen_gvec_##fn(es, vec_full_reg_offset(v1), vec_full_reg_offset(v2), \
 238                      16, 16)
 239#define gen_gvec_fn_2i(fn, es, v1, v2, c) \
 240    tcg_gen_gvec_##fn(es, vec_full_reg_offset(v1), vec_full_reg_offset(v2), \
 241                      c, 16, 16)
 242#define gen_gvec_fn_2s(fn, es, v1, v2, s) \
 243    tcg_gen_gvec_##fn(es, vec_full_reg_offset(v1), vec_full_reg_offset(v2), \
 244                      s, 16, 16)
 245#define gen_gvec_fn_3(fn, es, v1, v2, v3) \
 246    tcg_gen_gvec_##fn(es, vec_full_reg_offset(v1), vec_full_reg_offset(v2), \
 247                      vec_full_reg_offset(v3), 16, 16)
 248#define gen_gvec_fn_4(fn, es, v1, v2, v3, v4) \
 249    tcg_gen_gvec_##fn(es, vec_full_reg_offset(v1), vec_full_reg_offset(v2), \
 250                      vec_full_reg_offset(v3), vec_full_reg_offset(v4), 16, 16)
 251
 252/*
 253 * Helper to carry out a 128 bit vector computation using 2 i64 values per
 254 * vector.
 255 */
 256typedef void (*gen_gvec128_3_i64_fn)(TCGv_i64 dl, TCGv_i64 dh, TCGv_i64 al,
 257                                     TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh);
 258static void gen_gvec128_3_i64(gen_gvec128_3_i64_fn fn, uint8_t d, uint8_t a,
 259                              uint8_t b)
 260{
 261        TCGv_i64 dh = tcg_temp_new_i64();
 262        TCGv_i64 dl = tcg_temp_new_i64();
 263        TCGv_i64 ah = tcg_temp_new_i64();
 264        TCGv_i64 al = tcg_temp_new_i64();
 265        TCGv_i64 bh = tcg_temp_new_i64();
 266        TCGv_i64 bl = tcg_temp_new_i64();
 267
 268        read_vec_element_i64(ah, a, 0, ES_64);
 269        read_vec_element_i64(al, a, 1, ES_64);
 270        read_vec_element_i64(bh, b, 0, ES_64);
 271        read_vec_element_i64(bl, b, 1, ES_64);
 272        fn(dl, dh, al, ah, bl, bh);
 273        write_vec_element_i64(dh, d, 0, ES_64);
 274        write_vec_element_i64(dl, d, 1, ES_64);
 275
 276        tcg_temp_free_i64(dh);
 277        tcg_temp_free_i64(dl);
 278        tcg_temp_free_i64(ah);
 279        tcg_temp_free_i64(al);
 280        tcg_temp_free_i64(bh);
 281        tcg_temp_free_i64(bl);
 282}
 283
 284typedef void (*gen_gvec128_4_i64_fn)(TCGv_i64 dl, TCGv_i64 dh, TCGv_i64 al,
 285                                     TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh,
 286                                     TCGv_i64 cl, TCGv_i64 ch);
 287static void gen_gvec128_4_i64(gen_gvec128_4_i64_fn fn, uint8_t d, uint8_t a,
 288                              uint8_t b, uint8_t c)
 289{
 290        TCGv_i64 dh = tcg_temp_new_i64();
 291        TCGv_i64 dl = tcg_temp_new_i64();
 292        TCGv_i64 ah = tcg_temp_new_i64();
 293        TCGv_i64 al = tcg_temp_new_i64();
 294        TCGv_i64 bh = tcg_temp_new_i64();
 295        TCGv_i64 bl = tcg_temp_new_i64();
 296        TCGv_i64 ch = tcg_temp_new_i64();
 297        TCGv_i64 cl = tcg_temp_new_i64();
 298
 299        read_vec_element_i64(ah, a, 0, ES_64);
 300        read_vec_element_i64(al, a, 1, ES_64);
 301        read_vec_element_i64(bh, b, 0, ES_64);
 302        read_vec_element_i64(bl, b, 1, ES_64);
 303        read_vec_element_i64(ch, c, 0, ES_64);
 304        read_vec_element_i64(cl, c, 1, ES_64);
 305        fn(dl, dh, al, ah, bl, bh, cl, ch);
 306        write_vec_element_i64(dh, d, 0, ES_64);
 307        write_vec_element_i64(dl, d, 1, ES_64);
 308
 309        tcg_temp_free_i64(dh);
 310        tcg_temp_free_i64(dl);
 311        tcg_temp_free_i64(ah);
 312        tcg_temp_free_i64(al);
 313        tcg_temp_free_i64(bh);
 314        tcg_temp_free_i64(bl);
 315        tcg_temp_free_i64(ch);
 316        tcg_temp_free_i64(cl);
 317}
 318
 319static void gen_gvec_dupi(uint8_t es, uint8_t reg, uint64_t c)
 320{
 321    switch (es) {
 322    case ES_8:
 323        tcg_gen_gvec_dup8i(vec_full_reg_offset(reg), 16, 16, c);
 324        break;
 325    case ES_16:
 326        tcg_gen_gvec_dup16i(vec_full_reg_offset(reg), 16, 16, c);
 327        break;
 328    case ES_32:
 329        tcg_gen_gvec_dup32i(vec_full_reg_offset(reg), 16, 16, c);
 330        break;
 331    case ES_64:
 332        gen_gvec_dup64i(reg, c);
 333        break;
 334    default:
 335        g_assert_not_reached();
 336    }
 337}
 338
 339static void zero_vec(uint8_t reg)
 340{
 341    tcg_gen_gvec_dup8i(vec_full_reg_offset(reg), 16, 16, 0);
 342}
 343
 344static void gen_addi2_i64(TCGv_i64 dl, TCGv_i64 dh, TCGv_i64 al, TCGv_i64 ah,
 345                          uint64_t b)
 346{
 347    TCGv_i64 bl = tcg_const_i64(b);
 348    TCGv_i64 bh = tcg_const_i64(0);
 349
 350    tcg_gen_add2_i64(dl, dh, al, ah, bl, bh);
 351    tcg_temp_free_i64(bl);
 352    tcg_temp_free_i64(bh);
 353}
 354
 355static DisasJumpType op_vge(DisasContext *s, DisasOps *o)
 356{
 357    const uint8_t es = s->insn->data;
 358    const uint8_t enr = get_field(s, m3);
 359    TCGv_i64 tmp;
 360
 361    if (!valid_vec_element(enr, es)) {
 362        gen_program_exception(s, PGM_SPECIFICATION);
 363        return DISAS_NORETURN;
 364    }
 365
 366    tmp = tcg_temp_new_i64();
 367    read_vec_element_i64(tmp, get_field(s, v2), enr, es);
 368    tcg_gen_add_i64(o->addr1, o->addr1, tmp);
 369    gen_addi_and_wrap_i64(s, o->addr1, o->addr1, 0);
 370
 371    tcg_gen_qemu_ld_i64(tmp, o->addr1, get_mem_index(s), MO_TE | es);
 372    write_vec_element_i64(tmp, get_field(s, v1), enr, es);
 373    tcg_temp_free_i64(tmp);
 374    return DISAS_NEXT;
 375}
 376
 377static uint64_t generate_byte_mask(uint8_t mask)
 378{
 379    uint64_t r = 0;
 380    int i;
 381
 382    for (i = 0; i < 8; i++) {
 383        if ((mask >> i) & 1) {
 384            r |= 0xffull << (i * 8);
 385        }
 386    }
 387    return r;
 388}
 389
 390static DisasJumpType op_vgbm(DisasContext *s, DisasOps *o)
 391{
 392    const uint16_t i2 = get_field(s, i2);
 393
 394    if (i2 == (i2 & 0xff) * 0x0101) {
 395        /*
 396         * Masks for both 64 bit elements of the vector are the same.
 397         * Trust tcg to produce a good constant loading.
 398         */
 399        gen_gvec_dup64i(get_field(s, v1),
 400                        generate_byte_mask(i2 & 0xff));
 401    } else {
 402        TCGv_i64 t = tcg_temp_new_i64();
 403
 404        tcg_gen_movi_i64(t, generate_byte_mask(i2 >> 8));
 405        write_vec_element_i64(t, get_field(s, v1), 0, ES_64);
 406        tcg_gen_movi_i64(t, generate_byte_mask(i2));
 407        write_vec_element_i64(t, get_field(s, v1), 1, ES_64);
 408        tcg_temp_free_i64(t);
 409    }
 410    return DISAS_NEXT;
 411}
 412
 413static DisasJumpType op_vgm(DisasContext *s, DisasOps *o)
 414{
 415    const uint8_t es = get_field(s, m4);
 416    const uint8_t bits = NUM_VEC_ELEMENT_BITS(es);
 417    const uint8_t i2 = get_field(s, i2) & (bits - 1);
 418    const uint8_t i3 = get_field(s, i3) & (bits - 1);
 419    uint64_t mask = 0;
 420    int i;
 421
 422    if (es > ES_64) {
 423        gen_program_exception(s, PGM_SPECIFICATION);
 424        return DISAS_NORETURN;
 425    }
 426
 427    /* generate the mask - take care of wrapping */
 428    for (i = i2; ; i = (i + 1) % bits) {
 429        mask |= 1ull << (bits - i - 1);
 430        if (i == i3) {
 431            break;
 432        }
 433    }
 434
 435    gen_gvec_dupi(es, get_field(s, v1), mask);
 436    return DISAS_NEXT;
 437}
 438
 439static DisasJumpType op_vl(DisasContext *s, DisasOps *o)
 440{
 441    TCGv_i64 t0 = tcg_temp_new_i64();
 442    TCGv_i64 t1 = tcg_temp_new_i64();
 443
 444    tcg_gen_qemu_ld_i64(t0, o->addr1, get_mem_index(s), MO_TEQ);
 445    gen_addi_and_wrap_i64(s, o->addr1, o->addr1, 8);
 446    tcg_gen_qemu_ld_i64(t1, o->addr1, get_mem_index(s), MO_TEQ);
 447    write_vec_element_i64(t0, get_field(s, v1), 0, ES_64);
 448    write_vec_element_i64(t1, get_field(s, v1), 1, ES_64);
 449    tcg_temp_free(t0);
 450    tcg_temp_free(t1);
 451    return DISAS_NEXT;
 452}
 453
 454static DisasJumpType op_vlr(DisasContext *s, DisasOps *o)
 455{
 456    gen_gvec_mov(get_field(s, v1), get_field(s, v2));
 457    return DISAS_NEXT;
 458}
 459
 460static DisasJumpType op_vlrep(DisasContext *s, DisasOps *o)
 461{
 462    const uint8_t es = get_field(s, m3);
 463    TCGv_i64 tmp;
 464
 465    if (es > ES_64) {
 466        gen_program_exception(s, PGM_SPECIFICATION);
 467        return DISAS_NORETURN;
 468    }
 469
 470    tmp = tcg_temp_new_i64();
 471    tcg_gen_qemu_ld_i64(tmp, o->addr1, get_mem_index(s), MO_TE | es);
 472    gen_gvec_dup_i64(es, get_field(s, v1), tmp);
 473    tcg_temp_free_i64(tmp);
 474    return DISAS_NEXT;
 475}
 476
 477static DisasJumpType op_vle(DisasContext *s, DisasOps *o)
 478{
 479    const uint8_t es = s->insn->data;
 480    const uint8_t enr = get_field(s, m3);
 481    TCGv_i64 tmp;
 482
 483    if (!valid_vec_element(enr, es)) {
 484        gen_program_exception(s, PGM_SPECIFICATION);
 485        return DISAS_NORETURN;
 486    }
 487
 488    tmp = tcg_temp_new_i64();
 489    tcg_gen_qemu_ld_i64(tmp, o->addr1, get_mem_index(s), MO_TE | es);
 490    write_vec_element_i64(tmp, get_field(s, v1), enr, es);
 491    tcg_temp_free_i64(tmp);
 492    return DISAS_NEXT;
 493}
 494
 495static DisasJumpType op_vlei(DisasContext *s, DisasOps *o)
 496{
 497    const uint8_t es = s->insn->data;
 498    const uint8_t enr = get_field(s, m3);
 499    TCGv_i64 tmp;
 500
 501    if (!valid_vec_element(enr, es)) {
 502        gen_program_exception(s, PGM_SPECIFICATION);
 503        return DISAS_NORETURN;
 504    }
 505
 506    tmp = tcg_const_i64((int16_t)get_field(s, i2));
 507    write_vec_element_i64(tmp, get_field(s, v1), enr, es);
 508    tcg_temp_free_i64(tmp);
 509    return DISAS_NEXT;
 510}
 511
 512static DisasJumpType op_vlgv(DisasContext *s, DisasOps *o)
 513{
 514    const uint8_t es = get_field(s, m4);
 515    TCGv_ptr ptr;
 516
 517    if (es > ES_64) {
 518        gen_program_exception(s, PGM_SPECIFICATION);
 519        return DISAS_NORETURN;
 520    }
 521
 522    /* fast path if we don't need the register content */
 523    if (!get_field(s, b2)) {
 524        uint8_t enr = get_field(s, d2) & (NUM_VEC_ELEMENTS(es) - 1);
 525
 526        read_vec_element_i64(o->out, get_field(s, v3), enr, es);
 527        return DISAS_NEXT;
 528    }
 529
 530    ptr = tcg_temp_new_ptr();
 531    get_vec_element_ptr_i64(ptr, get_field(s, v3), o->addr1, es);
 532    switch (es) {
 533    case ES_8:
 534        tcg_gen_ld8u_i64(o->out, ptr, 0);
 535        break;
 536    case ES_16:
 537        tcg_gen_ld16u_i64(o->out, ptr, 0);
 538        break;
 539    case ES_32:
 540        tcg_gen_ld32u_i64(o->out, ptr, 0);
 541        break;
 542    case ES_64:
 543        tcg_gen_ld_i64(o->out, ptr, 0);
 544        break;
 545    default:
 546        g_assert_not_reached();
 547    }
 548    tcg_temp_free_ptr(ptr);
 549
 550    return DISAS_NEXT;
 551}
 552
 553static DisasJumpType op_vllez(DisasContext *s, DisasOps *o)
 554{
 555    uint8_t es = get_field(s, m3);
 556    uint8_t enr;
 557    TCGv_i64 t;
 558
 559    switch (es) {
 560    /* rightmost sub-element of leftmost doubleword */
 561    case ES_8:
 562        enr = 7;
 563        break;
 564    case ES_16:
 565        enr = 3;
 566        break;
 567    case ES_32:
 568        enr = 1;
 569        break;
 570    case ES_64:
 571        enr = 0;
 572        break;
 573    /* leftmost sub-element of leftmost doubleword */
 574    case 6:
 575        if (s390_has_feat(S390_FEAT_VECTOR_ENH)) {
 576            es = ES_32;
 577            enr = 0;
 578            break;
 579        }
 580        /* fallthrough */
 581    default:
 582        gen_program_exception(s, PGM_SPECIFICATION);
 583        return DISAS_NORETURN;
 584    }
 585
 586    t = tcg_temp_new_i64();
 587    tcg_gen_qemu_ld_i64(t, o->addr1, get_mem_index(s), MO_TE | es);
 588    zero_vec(get_field(s, v1));
 589    write_vec_element_i64(t, get_field(s, v1), enr, es);
 590    tcg_temp_free_i64(t);
 591    return DISAS_NEXT;
 592}
 593
 594static DisasJumpType op_vlm(DisasContext *s, DisasOps *o)
 595{
 596    const uint8_t v3 = get_field(s, v3);
 597    uint8_t v1 = get_field(s, v1);
 598    TCGv_i64 t0, t1;
 599
 600    if (v3 < v1 || (v3 - v1 + 1) > 16) {
 601        gen_program_exception(s, PGM_SPECIFICATION);
 602        return DISAS_NORETURN;
 603    }
 604
 605    /*
 606     * Check for possible access exceptions by trying to load the last
 607     * element. The first element will be checked first next.
 608     */
 609    t0 = tcg_temp_new_i64();
 610    t1 = tcg_temp_new_i64();
 611    gen_addi_and_wrap_i64(s, t0, o->addr1, (v3 - v1) * 16 + 8);
 612    tcg_gen_qemu_ld_i64(t0, t0, get_mem_index(s), MO_TEQ);
 613
 614    for (;; v1++) {
 615        tcg_gen_qemu_ld_i64(t1, o->addr1, get_mem_index(s), MO_TEQ);
 616        write_vec_element_i64(t1, v1, 0, ES_64);
 617        if (v1 == v3) {
 618            break;
 619        }
 620        gen_addi_and_wrap_i64(s, o->addr1, o->addr1, 8);
 621        tcg_gen_qemu_ld_i64(t1, o->addr1, get_mem_index(s), MO_TEQ);
 622        write_vec_element_i64(t1, v1, 1, ES_64);
 623        gen_addi_and_wrap_i64(s, o->addr1, o->addr1, 8);
 624    }
 625
 626    /* Store the last element, loaded first */
 627    write_vec_element_i64(t0, v1, 1, ES_64);
 628
 629    tcg_temp_free_i64(t0);
 630    tcg_temp_free_i64(t1);
 631    return DISAS_NEXT;
 632}
 633
 634static DisasJumpType op_vlbb(DisasContext *s, DisasOps *o)
 635{
 636    const int64_t block_size = (1ull << (get_field(s, m3) + 6));
 637    const int v1_offs = vec_full_reg_offset(get_field(s, v1));
 638    TCGv_ptr a0;
 639    TCGv_i64 bytes;
 640
 641    if (get_field(s, m3) > 6) {
 642        gen_program_exception(s, PGM_SPECIFICATION);
 643        return DISAS_NORETURN;
 644    }
 645
 646    bytes = tcg_temp_new_i64();
 647    a0 = tcg_temp_new_ptr();
 648    /* calculate the number of bytes until the next block boundary */
 649    tcg_gen_ori_i64(bytes, o->addr1, -block_size);
 650    tcg_gen_neg_i64(bytes, bytes);
 651
 652    tcg_gen_addi_ptr(a0, cpu_env, v1_offs);
 653    gen_helper_vll(cpu_env, a0, o->addr1, bytes);
 654    tcg_temp_free_i64(bytes);
 655    tcg_temp_free_ptr(a0);
 656    return DISAS_NEXT;
 657}
 658
 659static DisasJumpType op_vlvg(DisasContext *s, DisasOps *o)
 660{
 661    const uint8_t es = get_field(s, m4);
 662    TCGv_ptr ptr;
 663
 664    if (es > ES_64) {
 665        gen_program_exception(s, PGM_SPECIFICATION);
 666        return DISAS_NORETURN;
 667    }
 668
 669    /* fast path if we don't need the register content */
 670    if (!get_field(s, b2)) {
 671        uint8_t enr = get_field(s, d2) & (NUM_VEC_ELEMENTS(es) - 1);
 672
 673        write_vec_element_i64(o->in2, get_field(s, v1), enr, es);
 674        return DISAS_NEXT;
 675    }
 676
 677    ptr = tcg_temp_new_ptr();
 678    get_vec_element_ptr_i64(ptr, get_field(s, v1), o->addr1, es);
 679    switch (es) {
 680    case ES_8:
 681        tcg_gen_st8_i64(o->in2, ptr, 0);
 682        break;
 683    case ES_16:
 684        tcg_gen_st16_i64(o->in2, ptr, 0);
 685        break;
 686    case ES_32:
 687        tcg_gen_st32_i64(o->in2, ptr, 0);
 688        break;
 689    case ES_64:
 690        tcg_gen_st_i64(o->in2, ptr, 0);
 691        break;
 692    default:
 693        g_assert_not_reached();
 694    }
 695    tcg_temp_free_ptr(ptr);
 696
 697    return DISAS_NEXT;
 698}
 699
 700static DisasJumpType op_vlvgp(DisasContext *s, DisasOps *o)
 701{
 702    write_vec_element_i64(o->in1, get_field(s, v1), 0, ES_64);
 703    write_vec_element_i64(o->in2, get_field(s, v1), 1, ES_64);
 704    return DISAS_NEXT;
 705}
 706
 707static DisasJumpType op_vll(DisasContext *s, DisasOps *o)
 708{
 709    const int v1_offs = vec_full_reg_offset(get_field(s, v1));
 710    TCGv_ptr a0 = tcg_temp_new_ptr();
 711
 712    /* convert highest index into an actual length */
 713    tcg_gen_addi_i64(o->in2, o->in2, 1);
 714    tcg_gen_addi_ptr(a0, cpu_env, v1_offs);
 715    gen_helper_vll(cpu_env, a0, o->addr1, o->in2);
 716    tcg_temp_free_ptr(a0);
 717    return DISAS_NEXT;
 718}
 719
 720static DisasJumpType op_vmr(DisasContext *s, DisasOps *o)
 721{
 722    const uint8_t v1 = get_field(s, v1);
 723    const uint8_t v2 = get_field(s, v2);
 724    const uint8_t v3 = get_field(s, v3);
 725    const uint8_t es = get_field(s, m4);
 726    int dst_idx, src_idx;
 727    TCGv_i64 tmp;
 728
 729    if (es > ES_64) {
 730        gen_program_exception(s, PGM_SPECIFICATION);
 731        return DISAS_NORETURN;
 732    }
 733
 734    tmp = tcg_temp_new_i64();
 735    if (s->fields.op2 == 0x61) {
 736        /* iterate backwards to avoid overwriting data we might need later */
 737        for (dst_idx = NUM_VEC_ELEMENTS(es) - 1; dst_idx >= 0; dst_idx--) {
 738            src_idx = dst_idx / 2;
 739            if (dst_idx % 2 == 0) {
 740                read_vec_element_i64(tmp, v2, src_idx, es);
 741            } else {
 742                read_vec_element_i64(tmp, v3, src_idx, es);
 743            }
 744            write_vec_element_i64(tmp, v1, dst_idx, es);
 745        }
 746    } else {
 747        /* iterate forward to avoid overwriting data we might need later */
 748        for (dst_idx = 0; dst_idx < NUM_VEC_ELEMENTS(es); dst_idx++) {
 749            src_idx = (dst_idx + NUM_VEC_ELEMENTS(es)) / 2;
 750            if (dst_idx % 2 == 0) {
 751                read_vec_element_i64(tmp, v2, src_idx, es);
 752            } else {
 753                read_vec_element_i64(tmp, v3, src_idx, es);
 754            }
 755            write_vec_element_i64(tmp, v1, dst_idx, es);
 756        }
 757    }
 758    tcg_temp_free_i64(tmp);
 759    return DISAS_NEXT;
 760}
 761
 762static DisasJumpType op_vpk(DisasContext *s, DisasOps *o)
 763{
 764    const uint8_t v1 = get_field(s, v1);
 765    const uint8_t v2 = get_field(s, v2);
 766    const uint8_t v3 = get_field(s, v3);
 767    const uint8_t es = get_field(s, m4);
 768    static gen_helper_gvec_3 * const vpk[3] = {
 769        gen_helper_gvec_vpk16,
 770        gen_helper_gvec_vpk32,
 771        gen_helper_gvec_vpk64,
 772    };
 773     static gen_helper_gvec_3 * const vpks[3] = {
 774        gen_helper_gvec_vpks16,
 775        gen_helper_gvec_vpks32,
 776        gen_helper_gvec_vpks64,
 777    };
 778    static gen_helper_gvec_3_ptr * const vpks_cc[3] = {
 779        gen_helper_gvec_vpks_cc16,
 780        gen_helper_gvec_vpks_cc32,
 781        gen_helper_gvec_vpks_cc64,
 782    };
 783    static gen_helper_gvec_3 * const vpkls[3] = {
 784        gen_helper_gvec_vpkls16,
 785        gen_helper_gvec_vpkls32,
 786        gen_helper_gvec_vpkls64,
 787    };
 788    static gen_helper_gvec_3_ptr * const vpkls_cc[3] = {
 789        gen_helper_gvec_vpkls_cc16,
 790        gen_helper_gvec_vpkls_cc32,
 791        gen_helper_gvec_vpkls_cc64,
 792    };
 793
 794    if (es == ES_8 || es > ES_64) {
 795        gen_program_exception(s, PGM_SPECIFICATION);
 796        return DISAS_NORETURN;
 797    }
 798
 799    switch (s->fields.op2) {
 800    case 0x97:
 801        if (get_field(s, m5) & 0x1) {
 802            gen_gvec_3_ptr(v1, v2, v3, cpu_env, 0, vpks_cc[es - 1]);
 803            set_cc_static(s);
 804        } else {
 805            gen_gvec_3_ool(v1, v2, v3, 0, vpks[es - 1]);
 806        }
 807        break;
 808    case 0x95:
 809        if (get_field(s, m5) & 0x1) {
 810            gen_gvec_3_ptr(v1, v2, v3, cpu_env, 0, vpkls_cc[es - 1]);
 811            set_cc_static(s);
 812        } else {
 813            gen_gvec_3_ool(v1, v2, v3, 0, vpkls[es - 1]);
 814        }
 815        break;
 816    case 0x94:
 817        /* If sources and destination dont't overlap -> fast path */
 818        if (v1 != v2 && v1 != v3) {
 819            const uint8_t src_es = get_field(s, m4);
 820            const uint8_t dst_es = src_es - 1;
 821            TCGv_i64 tmp = tcg_temp_new_i64();
 822            int dst_idx, src_idx;
 823
 824            for (dst_idx = 0; dst_idx < NUM_VEC_ELEMENTS(dst_es); dst_idx++) {
 825                src_idx = dst_idx;
 826                if (src_idx < NUM_VEC_ELEMENTS(src_es)) {
 827                    read_vec_element_i64(tmp, v2, src_idx, src_es);
 828                } else {
 829                    src_idx -= NUM_VEC_ELEMENTS(src_es);
 830                    read_vec_element_i64(tmp, v3, src_idx, src_es);
 831                }
 832                write_vec_element_i64(tmp, v1, dst_idx, dst_es);
 833            }
 834            tcg_temp_free_i64(tmp);
 835        } else {
 836            gen_gvec_3_ool(v1, v2, v3, 0, vpk[es - 1]);
 837        }
 838        break;
 839    default:
 840        g_assert_not_reached();
 841    }
 842    return DISAS_NEXT;
 843}
 844
 845static DisasJumpType op_vperm(DisasContext *s, DisasOps *o)
 846{
 847    gen_gvec_4_ool(get_field(s, v1), get_field(s, v2),
 848                   get_field(s, v3), get_field(s, v4),
 849                   0, gen_helper_gvec_vperm);
 850    return DISAS_NEXT;
 851}
 852
 853static DisasJumpType op_vpdi(DisasContext *s, DisasOps *o)
 854{
 855    const uint8_t i2 = extract32(get_field(s, m4), 2, 1);
 856    const uint8_t i3 = extract32(get_field(s, m4), 0, 1);
 857    TCGv_i64 t0 = tcg_temp_new_i64();
 858    TCGv_i64 t1 = tcg_temp_new_i64();
 859
 860    read_vec_element_i64(t0, get_field(s, v2), i2, ES_64);
 861    read_vec_element_i64(t1, get_field(s, v3), i3, ES_64);
 862    write_vec_element_i64(t0, get_field(s, v1), 0, ES_64);
 863    write_vec_element_i64(t1, get_field(s, v1), 1, ES_64);
 864    tcg_temp_free_i64(t0);
 865    tcg_temp_free_i64(t1);
 866    return DISAS_NEXT;
 867}
 868
 869static DisasJumpType op_vrep(DisasContext *s, DisasOps *o)
 870{
 871    const uint8_t enr = get_field(s, i2);
 872    const uint8_t es = get_field(s, m4);
 873
 874    if (es > ES_64 || !valid_vec_element(enr, es)) {
 875        gen_program_exception(s, PGM_SPECIFICATION);
 876        return DISAS_NORETURN;
 877    }
 878
 879    tcg_gen_gvec_dup_mem(es, vec_full_reg_offset(get_field(s, v1)),
 880                         vec_reg_offset(get_field(s, v3), enr, es),
 881                         16, 16);
 882    return DISAS_NEXT;
 883}
 884
 885static DisasJumpType op_vrepi(DisasContext *s, DisasOps *o)
 886{
 887    const int64_t data = (int16_t)get_field(s, i2);
 888    const uint8_t es = get_field(s, m3);
 889
 890    if (es > ES_64) {
 891        gen_program_exception(s, PGM_SPECIFICATION);
 892        return DISAS_NORETURN;
 893    }
 894
 895    gen_gvec_dupi(es, get_field(s, v1), data);
 896    return DISAS_NEXT;
 897}
 898
 899static DisasJumpType op_vsce(DisasContext *s, DisasOps *o)
 900{
 901    const uint8_t es = s->insn->data;
 902    const uint8_t enr = get_field(s, m3);
 903    TCGv_i64 tmp;
 904
 905    if (!valid_vec_element(enr, es)) {
 906        gen_program_exception(s, PGM_SPECIFICATION);
 907        return DISAS_NORETURN;
 908    }
 909
 910    tmp = tcg_temp_new_i64();
 911    read_vec_element_i64(tmp, get_field(s, v2), enr, es);
 912    tcg_gen_add_i64(o->addr1, o->addr1, tmp);
 913    gen_addi_and_wrap_i64(s, o->addr1, o->addr1, 0);
 914
 915    read_vec_element_i64(tmp, get_field(s, v1), enr, es);
 916    tcg_gen_qemu_st_i64(tmp, o->addr1, get_mem_index(s), MO_TE | es);
 917    tcg_temp_free_i64(tmp);
 918    return DISAS_NEXT;
 919}
 920
 921static DisasJumpType op_vsel(DisasContext *s, DisasOps *o)
 922{
 923    gen_gvec_fn_4(bitsel, ES_8, get_field(s, v1),
 924                  get_field(s, v4), get_field(s, v2),
 925                  get_field(s, v3));
 926    return DISAS_NEXT;
 927}
 928
 929static DisasJumpType op_vseg(DisasContext *s, DisasOps *o)
 930{
 931    const uint8_t es = get_field(s, m3);
 932    int idx1, idx2;
 933    TCGv_i64 tmp;
 934
 935    switch (es) {
 936    case ES_8:
 937        idx1 = 7;
 938        idx2 = 15;
 939        break;
 940    case ES_16:
 941        idx1 = 3;
 942        idx2 = 7;
 943        break;
 944    case ES_32:
 945        idx1 = 1;
 946        idx2 = 3;
 947        break;
 948    default:
 949        gen_program_exception(s, PGM_SPECIFICATION);
 950        return DISAS_NORETURN;
 951    }
 952
 953    tmp = tcg_temp_new_i64();
 954    read_vec_element_i64(tmp, get_field(s, v2), idx1, es | MO_SIGN);
 955    write_vec_element_i64(tmp, get_field(s, v1), 0, ES_64);
 956    read_vec_element_i64(tmp, get_field(s, v2), idx2, es | MO_SIGN);
 957    write_vec_element_i64(tmp, get_field(s, v1), 1, ES_64);
 958    tcg_temp_free_i64(tmp);
 959    return DISAS_NEXT;
 960}
 961
 962static DisasJumpType op_vst(DisasContext *s, DisasOps *o)
 963{
 964    TCGv_i64 tmp = tcg_const_i64(16);
 965
 966    /* Probe write access before actually modifying memory */
 967    gen_helper_probe_write_access(cpu_env, o->addr1, tmp);
 968
 969    read_vec_element_i64(tmp,  get_field(s, v1), 0, ES_64);
 970    tcg_gen_qemu_st_i64(tmp, o->addr1, get_mem_index(s), MO_TEQ);
 971    gen_addi_and_wrap_i64(s, o->addr1, o->addr1, 8);
 972    read_vec_element_i64(tmp,  get_field(s, v1), 1, ES_64);
 973    tcg_gen_qemu_st_i64(tmp, o->addr1, get_mem_index(s), MO_TEQ);
 974    tcg_temp_free_i64(tmp);
 975    return DISAS_NEXT;
 976}
 977
 978static DisasJumpType op_vste(DisasContext *s, DisasOps *o)
 979{
 980    const uint8_t es = s->insn->data;
 981    const uint8_t enr = get_field(s, m3);
 982    TCGv_i64 tmp;
 983
 984    if (!valid_vec_element(enr, es)) {
 985        gen_program_exception(s, PGM_SPECIFICATION);
 986        return DISAS_NORETURN;
 987    }
 988
 989    tmp = tcg_temp_new_i64();
 990    read_vec_element_i64(tmp, get_field(s, v1), enr, es);
 991    tcg_gen_qemu_st_i64(tmp, o->addr1, get_mem_index(s), MO_TE | es);
 992    tcg_temp_free_i64(tmp);
 993    return DISAS_NEXT;
 994}
 995
 996static DisasJumpType op_vstm(DisasContext *s, DisasOps *o)
 997{
 998    const uint8_t v3 = get_field(s, v3);
 999    uint8_t v1 = get_field(s, v1);
1000    TCGv_i64 tmp;
1001
1002    while (v3 < v1 || (v3 - v1 + 1) > 16) {
1003        gen_program_exception(s, PGM_SPECIFICATION);
1004        return DISAS_NORETURN;
1005    }
1006
1007    /* Probe write access before actually modifying memory */
1008    tmp = tcg_const_i64((v3 - v1 + 1) * 16);
1009    gen_helper_probe_write_access(cpu_env, o->addr1, tmp);
1010
1011    for (;; v1++) {
1012        read_vec_element_i64(tmp, v1, 0, ES_64);
1013        tcg_gen_qemu_st_i64(tmp, o->addr1, get_mem_index(s), MO_TEQ);
1014        gen_addi_and_wrap_i64(s, o->addr1, o->addr1, 8);
1015        read_vec_element_i64(tmp, v1, 1, ES_64);
1016        tcg_gen_qemu_st_i64(tmp, o->addr1, get_mem_index(s), MO_TEQ);
1017        if (v1 == v3) {
1018            break;
1019        }
1020        gen_addi_and_wrap_i64(s, o->addr1, o->addr1, 8);
1021    }
1022    tcg_temp_free_i64(tmp);
1023    return DISAS_NEXT;
1024}
1025
1026static DisasJumpType op_vstl(DisasContext *s, DisasOps *o)
1027{
1028    const int v1_offs = vec_full_reg_offset(get_field(s, v1));
1029    TCGv_ptr a0 = tcg_temp_new_ptr();
1030
1031    /* convert highest index into an actual length */
1032    tcg_gen_addi_i64(o->in2, o->in2, 1);
1033    tcg_gen_addi_ptr(a0, cpu_env, v1_offs);
1034    gen_helper_vstl(cpu_env, a0, o->addr1, o->in2);
1035    tcg_temp_free_ptr(a0);
1036    return DISAS_NEXT;
1037}
1038
1039static DisasJumpType op_vup(DisasContext *s, DisasOps *o)
1040{
1041    const bool logical = s->fields.op2 == 0xd4 || s->fields.op2 == 0xd5;
1042    const uint8_t v1 = get_field(s, v1);
1043    const uint8_t v2 = get_field(s, v2);
1044    const uint8_t src_es = get_field(s, m3);
1045    const uint8_t dst_es = src_es + 1;
1046    int dst_idx, src_idx;
1047    TCGv_i64 tmp;
1048
1049    if (src_es > ES_32) {
1050        gen_program_exception(s, PGM_SPECIFICATION);
1051        return DISAS_NORETURN;
1052    }
1053
1054    tmp = tcg_temp_new_i64();
1055    if (s->fields.op2 == 0xd7 || s->fields.op2 == 0xd5) {
1056        /* iterate backwards to avoid overwriting data we might need later */
1057        for (dst_idx = NUM_VEC_ELEMENTS(dst_es) - 1; dst_idx >= 0; dst_idx--) {
1058            src_idx = dst_idx;
1059            read_vec_element_i64(tmp, v2, src_idx,
1060                                 src_es | (logical ? 0 : MO_SIGN));
1061            write_vec_element_i64(tmp, v1, dst_idx, dst_es);
1062        }
1063
1064    } else {
1065        /* iterate forward to avoid overwriting data we might need later */
1066        for (dst_idx = 0; dst_idx < NUM_VEC_ELEMENTS(dst_es); dst_idx++) {
1067            src_idx = dst_idx + NUM_VEC_ELEMENTS(src_es) / 2;
1068            read_vec_element_i64(tmp, v2, src_idx,
1069                                 src_es | (logical ? 0 : MO_SIGN));
1070            write_vec_element_i64(tmp, v1, dst_idx, dst_es);
1071        }
1072    }
1073    tcg_temp_free_i64(tmp);
1074    return DISAS_NEXT;
1075}
1076
1077static DisasJumpType op_va(DisasContext *s, DisasOps *o)
1078{
1079    const uint8_t es = get_field(s, m4);
1080
1081    if (es > ES_128) {
1082        gen_program_exception(s, PGM_SPECIFICATION);
1083        return DISAS_NORETURN;
1084    } else if (es == ES_128) {
1085        gen_gvec128_3_i64(tcg_gen_add2_i64, get_field(s, v1),
1086                          get_field(s, v2), get_field(s, v3));
1087        return DISAS_NEXT;
1088    }
1089    gen_gvec_fn_3(add, es, get_field(s, v1), get_field(s, v2),
1090                  get_field(s, v3));
1091    return DISAS_NEXT;
1092}
1093
1094static void gen_acc(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b, uint8_t es)
1095{
1096    const uint8_t msb_bit_nr = NUM_VEC_ELEMENT_BITS(es) - 1;
1097    TCGv_i64 msb_mask = tcg_const_i64(dup_const(es, 1ull << msb_bit_nr));
1098    TCGv_i64 t1 = tcg_temp_new_i64();
1099    TCGv_i64 t2 = tcg_temp_new_i64();
1100    TCGv_i64 t3 = tcg_temp_new_i64();
1101
1102    /* Calculate the carry into the MSB, ignoring the old MSBs */
1103    tcg_gen_andc_i64(t1, a, msb_mask);
1104    tcg_gen_andc_i64(t2, b, msb_mask);
1105    tcg_gen_add_i64(t1, t1, t2);
1106    /* Calculate the MSB without any carry into it */
1107    tcg_gen_xor_i64(t3, a, b);
1108    /* Calculate the carry out of the MSB in the MSB bit position */
1109    tcg_gen_and_i64(d, a, b);
1110    tcg_gen_and_i64(t1, t1, t3);
1111    tcg_gen_or_i64(d, d, t1);
1112    /* Isolate and shift the carry into position */
1113    tcg_gen_and_i64(d, d, msb_mask);
1114    tcg_gen_shri_i64(d, d, msb_bit_nr);
1115
1116    tcg_temp_free_i64(t1);
1117    tcg_temp_free_i64(t2);
1118    tcg_temp_free_i64(t3);
1119}
1120
1121static void gen_acc8_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
1122{
1123    gen_acc(d, a, b, ES_8);
1124}
1125
1126static void gen_acc16_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
1127{
1128    gen_acc(d, a, b, ES_16);
1129}
1130
1131static void gen_acc_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
1132{
1133    TCGv_i32 t = tcg_temp_new_i32();
1134
1135    tcg_gen_add_i32(t, a, b);
1136    tcg_gen_setcond_i32(TCG_COND_LTU, d, t, b);
1137    tcg_temp_free_i32(t);
1138}
1139
1140static void gen_acc_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
1141{
1142    TCGv_i64 t = tcg_temp_new_i64();
1143
1144    tcg_gen_add_i64(t, a, b);
1145    tcg_gen_setcond_i64(TCG_COND_LTU, d, t, b);
1146    tcg_temp_free_i64(t);
1147}
1148
1149static void gen_acc2_i64(TCGv_i64 dl, TCGv_i64 dh, TCGv_i64 al,
1150                         TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh)
1151{
1152    TCGv_i64 th = tcg_temp_new_i64();
1153    TCGv_i64 tl = tcg_temp_new_i64();
1154    TCGv_i64 zero = tcg_const_i64(0);
1155
1156    tcg_gen_add2_i64(tl, th, al, zero, bl, zero);
1157    tcg_gen_add2_i64(tl, th, th, zero, ah, zero);
1158    tcg_gen_add2_i64(tl, dl, tl, th, bh, zero);
1159    tcg_gen_mov_i64(dh, zero);
1160
1161    tcg_temp_free_i64(th);
1162    tcg_temp_free_i64(tl);
1163    tcg_temp_free_i64(zero);
1164}
1165
1166static DisasJumpType op_vacc(DisasContext *s, DisasOps *o)
1167{
1168    const uint8_t es = get_field(s, m4);
1169    static const GVecGen3 g[4] = {
1170        { .fni8 = gen_acc8_i64, },
1171        { .fni8 = gen_acc16_i64, },
1172        { .fni4 = gen_acc_i32, },
1173        { .fni8 = gen_acc_i64, },
1174    };
1175
1176    if (es > ES_128) {
1177        gen_program_exception(s, PGM_SPECIFICATION);
1178        return DISAS_NORETURN;
1179    } else if (es == ES_128) {
1180        gen_gvec128_3_i64(gen_acc2_i64, get_field(s, v1),
1181                          get_field(s, v2), get_field(s, v3));
1182        return DISAS_NEXT;
1183    }
1184    gen_gvec_3(get_field(s, v1), get_field(s, v2),
1185               get_field(s, v3), &g[es]);
1186    return DISAS_NEXT;
1187}
1188
1189static void gen_ac2_i64(TCGv_i64 dl, TCGv_i64 dh, TCGv_i64 al, TCGv_i64 ah,
1190                        TCGv_i64 bl, TCGv_i64 bh, TCGv_i64 cl, TCGv_i64 ch)
1191{
1192    TCGv_i64 tl = tcg_temp_new_i64();
1193    TCGv_i64 th = tcg_const_i64(0);
1194
1195    /* extract the carry only */
1196    tcg_gen_extract_i64(tl, cl, 0, 1);
1197    tcg_gen_add2_i64(dl, dh, al, ah, bl, bh);
1198    tcg_gen_add2_i64(dl, dh, dl, dh, tl, th);
1199
1200    tcg_temp_free_i64(tl);
1201    tcg_temp_free_i64(th);
1202}
1203
1204static DisasJumpType op_vac(DisasContext *s, DisasOps *o)
1205{
1206    if (get_field(s, m5) != ES_128) {
1207        gen_program_exception(s, PGM_SPECIFICATION);
1208        return DISAS_NORETURN;
1209    }
1210
1211    gen_gvec128_4_i64(gen_ac2_i64, get_field(s, v1),
1212                      get_field(s, v2), get_field(s, v3),
1213                      get_field(s, v4));
1214    return DISAS_NEXT;
1215}
1216
1217static void gen_accc2_i64(TCGv_i64 dl, TCGv_i64 dh, TCGv_i64 al, TCGv_i64 ah,
1218                          TCGv_i64 bl, TCGv_i64 bh, TCGv_i64 cl, TCGv_i64 ch)
1219{
1220    TCGv_i64 tl = tcg_temp_new_i64();
1221    TCGv_i64 th = tcg_temp_new_i64();
1222    TCGv_i64 zero = tcg_const_i64(0);
1223
1224    tcg_gen_andi_i64(tl, cl, 1);
1225    tcg_gen_add2_i64(tl, th, tl, zero, al, zero);
1226    tcg_gen_add2_i64(tl, th, tl, th, bl, zero);
1227    tcg_gen_add2_i64(tl, th, th, zero, ah, zero);
1228    tcg_gen_add2_i64(tl, dl, tl, th, bh, zero);
1229    tcg_gen_mov_i64(dh, zero);
1230
1231    tcg_temp_free_i64(tl);
1232    tcg_temp_free_i64(th);
1233    tcg_temp_free_i64(zero);
1234}
1235
1236static DisasJumpType op_vaccc(DisasContext *s, DisasOps *o)
1237{
1238    if (get_field(s, m5) != ES_128) {
1239        gen_program_exception(s, PGM_SPECIFICATION);
1240        return DISAS_NORETURN;
1241    }
1242
1243    gen_gvec128_4_i64(gen_accc2_i64, get_field(s, v1),
1244                      get_field(s, v2), get_field(s, v3),
1245                      get_field(s, v4));
1246    return DISAS_NEXT;
1247}
1248
1249static DisasJumpType op_vn(DisasContext *s, DisasOps *o)
1250{
1251    gen_gvec_fn_3(and, ES_8, get_field(s, v1), get_field(s, v2),
1252                  get_field(s, v3));
1253    return DISAS_NEXT;
1254}
1255
1256static DisasJumpType op_vnc(DisasContext *s, DisasOps *o)
1257{
1258    gen_gvec_fn_3(andc, ES_8, get_field(s, v1),
1259                  get_field(s, v2), get_field(s, v3));
1260    return DISAS_NEXT;
1261}
1262
1263static void gen_avg_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
1264{
1265    TCGv_i64 t0 = tcg_temp_new_i64();
1266    TCGv_i64 t1 = tcg_temp_new_i64();
1267
1268    tcg_gen_ext_i32_i64(t0, a);
1269    tcg_gen_ext_i32_i64(t1, b);
1270    tcg_gen_add_i64(t0, t0, t1);
1271    tcg_gen_addi_i64(t0, t0, 1);
1272    tcg_gen_shri_i64(t0, t0, 1);
1273    tcg_gen_extrl_i64_i32(d, t0);
1274
1275    tcg_temp_free(t0);
1276    tcg_temp_free(t1);
1277}
1278
1279static void gen_avg_i64(TCGv_i64 dl, TCGv_i64 al, TCGv_i64 bl)
1280{
1281    TCGv_i64 dh = tcg_temp_new_i64();
1282    TCGv_i64 ah = tcg_temp_new_i64();
1283    TCGv_i64 bh = tcg_temp_new_i64();
1284
1285    /* extending the sign by one bit is sufficient */
1286    tcg_gen_extract_i64(ah, al, 63, 1);
1287    tcg_gen_extract_i64(bh, bl, 63, 1);
1288    tcg_gen_add2_i64(dl, dh, al, ah, bl, bh);
1289    gen_addi2_i64(dl, dh, dl, dh, 1);
1290    tcg_gen_extract2_i64(dl, dl, dh, 1);
1291
1292    tcg_temp_free_i64(dh);
1293    tcg_temp_free_i64(ah);
1294    tcg_temp_free_i64(bh);
1295}
1296
1297static DisasJumpType op_vavg(DisasContext *s, DisasOps *o)
1298{
1299    const uint8_t es = get_field(s, m4);
1300    static const GVecGen3 g[4] = {
1301        { .fno = gen_helper_gvec_vavg8, },
1302        { .fno = gen_helper_gvec_vavg16, },
1303        { .fni4 = gen_avg_i32, },
1304        { .fni8 = gen_avg_i64, },
1305    };
1306
1307    if (es > ES_64) {
1308        gen_program_exception(s, PGM_SPECIFICATION);
1309        return DISAS_NORETURN;
1310    }
1311    gen_gvec_3(get_field(s, v1), get_field(s, v2),
1312               get_field(s, v3), &g[es]);
1313    return DISAS_NEXT;
1314}
1315
1316static void gen_avgl_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
1317{
1318    TCGv_i64 t0 = tcg_temp_new_i64();
1319    TCGv_i64 t1 = tcg_temp_new_i64();
1320
1321    tcg_gen_extu_i32_i64(t0, a);
1322    tcg_gen_extu_i32_i64(t1, b);
1323    tcg_gen_add_i64(t0, t0, t1);
1324    tcg_gen_addi_i64(t0, t0, 1);
1325    tcg_gen_shri_i64(t0, t0, 1);
1326    tcg_gen_extrl_i64_i32(d, t0);
1327
1328    tcg_temp_free(t0);
1329    tcg_temp_free(t1);
1330}
1331
1332static void gen_avgl_i64(TCGv_i64 dl, TCGv_i64 al, TCGv_i64 bl)
1333{
1334    TCGv_i64 dh = tcg_temp_new_i64();
1335    TCGv_i64 zero = tcg_const_i64(0);
1336
1337    tcg_gen_add2_i64(dl, dh, al, zero, bl, zero);
1338    gen_addi2_i64(dl, dh, dl, dh, 1);
1339    tcg_gen_extract2_i64(dl, dl, dh, 1);
1340
1341    tcg_temp_free_i64(dh);
1342    tcg_temp_free_i64(zero);
1343}
1344
1345static DisasJumpType op_vavgl(DisasContext *s, DisasOps *o)
1346{
1347    const uint8_t es = get_field(s, m4);
1348    static const GVecGen3 g[4] = {
1349        { .fno = gen_helper_gvec_vavgl8, },
1350        { .fno = gen_helper_gvec_vavgl16, },
1351        { .fni4 = gen_avgl_i32, },
1352        { .fni8 = gen_avgl_i64, },
1353    };
1354
1355    if (es > ES_64) {
1356        gen_program_exception(s, PGM_SPECIFICATION);
1357        return DISAS_NORETURN;
1358    }
1359    gen_gvec_3(get_field(s, v1), get_field(s, v2),
1360               get_field(s, v3), &g[es]);
1361    return DISAS_NEXT;
1362}
1363
1364static DisasJumpType op_vcksm(DisasContext *s, DisasOps *o)
1365{
1366    TCGv_i32 tmp = tcg_temp_new_i32();
1367    TCGv_i32 sum = tcg_temp_new_i32();
1368    int i;
1369
1370    read_vec_element_i32(sum, get_field(s, v3), 1, ES_32);
1371    for (i = 0; i < 4; i++) {
1372        read_vec_element_i32(tmp, get_field(s, v2), i, ES_32);
1373        tcg_gen_add2_i32(tmp, sum, sum, sum, tmp, tmp);
1374    }
1375    zero_vec(get_field(s, v1));
1376    write_vec_element_i32(sum, get_field(s, v1), 1, ES_32);
1377
1378    tcg_temp_free_i32(tmp);
1379    tcg_temp_free_i32(sum);
1380    return DISAS_NEXT;
1381}
1382
1383static DisasJumpType op_vec(DisasContext *s, DisasOps *o)
1384{
1385    uint8_t es = get_field(s, m3);
1386    const uint8_t enr = NUM_VEC_ELEMENTS(es) / 2 - 1;
1387
1388    if (es > ES_64) {
1389        gen_program_exception(s, PGM_SPECIFICATION);
1390        return DISAS_NORETURN;
1391    }
1392    if (s->fields.op2 == 0xdb) {
1393        es |= MO_SIGN;
1394    }
1395
1396    o->in1 = tcg_temp_new_i64();
1397    o->in2 = tcg_temp_new_i64();
1398    read_vec_element_i64(o->in1, get_field(s, v1), enr, es);
1399    read_vec_element_i64(o->in2, get_field(s, v2), enr, es);
1400    return DISAS_NEXT;
1401}
1402
1403static DisasJumpType op_vc(DisasContext *s, DisasOps *o)
1404{
1405    const uint8_t es = get_field(s, m4);
1406    TCGCond cond = s->insn->data;
1407
1408    if (es > ES_64) {
1409        gen_program_exception(s, PGM_SPECIFICATION);
1410        return DISAS_NORETURN;
1411    }
1412
1413    tcg_gen_gvec_cmp(cond, es,
1414                     vec_full_reg_offset(get_field(s, v1)),
1415                     vec_full_reg_offset(get_field(s, v2)),
1416                     vec_full_reg_offset(get_field(s, v3)), 16, 16);
1417    if (get_field(s, m5) & 0x1) {
1418        TCGv_i64 low = tcg_temp_new_i64();
1419        TCGv_i64 high = tcg_temp_new_i64();
1420
1421        read_vec_element_i64(high, get_field(s, v1), 0, ES_64);
1422        read_vec_element_i64(low, get_field(s, v1), 1, ES_64);
1423        gen_op_update2_cc_i64(s, CC_OP_VC, low, high);
1424
1425        tcg_temp_free_i64(low);
1426        tcg_temp_free_i64(high);
1427    }
1428    return DISAS_NEXT;
1429}
1430
1431static void gen_clz_i32(TCGv_i32 d, TCGv_i32 a)
1432{
1433    tcg_gen_clzi_i32(d, a, 32);
1434}
1435
1436static void gen_clz_i64(TCGv_i64 d, TCGv_i64 a)
1437{
1438    tcg_gen_clzi_i64(d, a, 64);
1439}
1440
1441static DisasJumpType op_vclz(DisasContext *s, DisasOps *o)
1442{
1443    const uint8_t es = get_field(s, m3);
1444    static const GVecGen2 g[4] = {
1445        { .fno = gen_helper_gvec_vclz8, },
1446        { .fno = gen_helper_gvec_vclz16, },
1447        { .fni4 = gen_clz_i32, },
1448        { .fni8 = gen_clz_i64, },
1449    };
1450
1451    if (es > ES_64) {
1452        gen_program_exception(s, PGM_SPECIFICATION);
1453        return DISAS_NORETURN;
1454    }
1455    gen_gvec_2(get_field(s, v1), get_field(s, v2), &g[es]);
1456    return DISAS_NEXT;
1457}
1458
1459static void gen_ctz_i32(TCGv_i32 d, TCGv_i32 a)
1460{
1461    tcg_gen_ctzi_i32(d, a, 32);
1462}
1463
1464static void gen_ctz_i64(TCGv_i64 d, TCGv_i64 a)
1465{
1466    tcg_gen_ctzi_i64(d, a, 64);
1467}
1468
1469static DisasJumpType op_vctz(DisasContext *s, DisasOps *o)
1470{
1471    const uint8_t es = get_field(s, m3);
1472    static const GVecGen2 g[4] = {
1473        { .fno = gen_helper_gvec_vctz8, },
1474        { .fno = gen_helper_gvec_vctz16, },
1475        { .fni4 = gen_ctz_i32, },
1476        { .fni8 = gen_ctz_i64, },
1477    };
1478
1479    if (es > ES_64) {
1480        gen_program_exception(s, PGM_SPECIFICATION);
1481        return DISAS_NORETURN;
1482    }
1483    gen_gvec_2(get_field(s, v1), get_field(s, v2), &g[es]);
1484    return DISAS_NEXT;
1485}
1486
1487static DisasJumpType op_vx(DisasContext *s, DisasOps *o)
1488{
1489    gen_gvec_fn_3(xor, ES_8, get_field(s, v1), get_field(s, v2),
1490                 get_field(s, v3));
1491    return DISAS_NEXT;
1492}
1493
1494static DisasJumpType op_vgfm(DisasContext *s, DisasOps *o)
1495{
1496    const uint8_t es = get_field(s, m4);
1497    static const GVecGen3 g[4] = {
1498        { .fno = gen_helper_gvec_vgfm8, },
1499        { .fno = gen_helper_gvec_vgfm16, },
1500        { .fno = gen_helper_gvec_vgfm32, },
1501        { .fno = gen_helper_gvec_vgfm64, },
1502    };
1503
1504    if (es > ES_64) {
1505        gen_program_exception(s, PGM_SPECIFICATION);
1506        return DISAS_NORETURN;
1507    }
1508    gen_gvec_3(get_field(s, v1), get_field(s, v2),
1509               get_field(s, v3), &g[es]);
1510    return DISAS_NEXT;
1511}
1512
1513static DisasJumpType op_vgfma(DisasContext *s, DisasOps *o)
1514{
1515    const uint8_t es = get_field(s, m5);
1516    static const GVecGen4 g[4] = {
1517        { .fno = gen_helper_gvec_vgfma8, },
1518        { .fno = gen_helper_gvec_vgfma16, },
1519        { .fno = gen_helper_gvec_vgfma32, },
1520        { .fno = gen_helper_gvec_vgfma64, },
1521    };
1522
1523    if (es > ES_64) {
1524        gen_program_exception(s, PGM_SPECIFICATION);
1525        return DISAS_NORETURN;
1526    }
1527    gen_gvec_4(get_field(s, v1), get_field(s, v2),
1528               get_field(s, v3), get_field(s, v4), &g[es]);
1529    return DISAS_NEXT;
1530}
1531
1532static DisasJumpType op_vlc(DisasContext *s, DisasOps *o)
1533{
1534    const uint8_t es = get_field(s, m3);
1535
1536    if (es > ES_64) {
1537        gen_program_exception(s, PGM_SPECIFICATION);
1538        return DISAS_NORETURN;
1539    }
1540
1541    gen_gvec_fn_2(neg, es, get_field(s, v1), get_field(s, v2));
1542    return DISAS_NEXT;
1543}
1544
1545static DisasJumpType op_vlp(DisasContext *s, DisasOps *o)
1546{
1547    const uint8_t es = get_field(s, m3);
1548
1549    if (es > ES_64) {
1550        gen_program_exception(s, PGM_SPECIFICATION);
1551        return DISAS_NORETURN;
1552    }
1553
1554    gen_gvec_fn_2(abs, es, get_field(s, v1), get_field(s, v2));
1555    return DISAS_NEXT;
1556}
1557
1558static DisasJumpType op_vmx(DisasContext *s, DisasOps *o)
1559{
1560    const uint8_t v1 = get_field(s, v1);
1561    const uint8_t v2 = get_field(s, v2);
1562    const uint8_t v3 = get_field(s, v3);
1563    const uint8_t es = get_field(s, m4);
1564
1565    if (es > ES_64) {
1566        gen_program_exception(s, PGM_SPECIFICATION);
1567        return DISAS_NORETURN;
1568    }
1569
1570    switch (s->fields.op2) {
1571    case 0xff:
1572        gen_gvec_fn_3(smax, es, v1, v2, v3);
1573        break;
1574    case 0xfd:
1575        gen_gvec_fn_3(umax, es, v1, v2, v3);
1576        break;
1577    case 0xfe:
1578        gen_gvec_fn_3(smin, es, v1, v2, v3);
1579        break;
1580    case 0xfc:
1581        gen_gvec_fn_3(umin, es, v1, v2, v3);
1582        break;
1583    default:
1584        g_assert_not_reached();
1585    }
1586    return DISAS_NEXT;
1587}
1588
1589static void gen_mal_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b, TCGv_i32 c)
1590{
1591    TCGv_i32 t0 = tcg_temp_new_i32();
1592
1593    tcg_gen_mul_i32(t0, a, b);
1594    tcg_gen_add_i32(d, t0, c);
1595
1596    tcg_temp_free_i32(t0);
1597}
1598
1599static void gen_mah_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b, TCGv_i32 c)
1600{
1601    TCGv_i64 t0 = tcg_temp_new_i64();
1602    TCGv_i64 t1 = tcg_temp_new_i64();
1603    TCGv_i64 t2 = tcg_temp_new_i64();
1604
1605    tcg_gen_ext_i32_i64(t0, a);
1606    tcg_gen_ext_i32_i64(t1, b);
1607    tcg_gen_ext_i32_i64(t2, c);
1608    tcg_gen_mul_i64(t0, t0, t1);
1609    tcg_gen_add_i64(t0, t0, t2);
1610    tcg_gen_extrh_i64_i32(d, t0);
1611
1612    tcg_temp_free(t0);
1613    tcg_temp_free(t1);
1614    tcg_temp_free(t2);
1615}
1616
1617static void gen_malh_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b, TCGv_i32 c)
1618{
1619    TCGv_i64 t0 = tcg_temp_new_i64();
1620    TCGv_i64 t1 = tcg_temp_new_i64();
1621    TCGv_i64 t2 = tcg_temp_new_i64();
1622
1623    tcg_gen_extu_i32_i64(t0, a);
1624    tcg_gen_extu_i32_i64(t1, b);
1625    tcg_gen_extu_i32_i64(t2, c);
1626    tcg_gen_mul_i64(t0, t0, t1);
1627    tcg_gen_add_i64(t0, t0, t2);
1628    tcg_gen_extrh_i64_i32(d, t0);
1629
1630    tcg_temp_free(t0);
1631    tcg_temp_free(t1);
1632    tcg_temp_free(t2);
1633}
1634
1635static DisasJumpType op_vma(DisasContext *s, DisasOps *o)
1636{
1637    const uint8_t es = get_field(s, m5);
1638    static const GVecGen4 g_vmal[3] = {
1639        { .fno = gen_helper_gvec_vmal8, },
1640        { .fno = gen_helper_gvec_vmal16, },
1641        { .fni4 = gen_mal_i32, },
1642    };
1643    static const GVecGen4 g_vmah[3] = {
1644        { .fno = gen_helper_gvec_vmah8, },
1645        { .fno = gen_helper_gvec_vmah16, },
1646        { .fni4 = gen_mah_i32, },
1647    };
1648    static const GVecGen4 g_vmalh[3] = {
1649        { .fno = gen_helper_gvec_vmalh8, },
1650        { .fno = gen_helper_gvec_vmalh16, },
1651        { .fni4 = gen_malh_i32, },
1652    };
1653    static const GVecGen4 g_vmae[3] = {
1654        { .fno = gen_helper_gvec_vmae8, },
1655        { .fno = gen_helper_gvec_vmae16, },
1656        { .fno = gen_helper_gvec_vmae32, },
1657    };
1658    static const GVecGen4 g_vmale[3] = {
1659        { .fno = gen_helper_gvec_vmale8, },
1660        { .fno = gen_helper_gvec_vmale16, },
1661        { .fno = gen_helper_gvec_vmale32, },
1662    };
1663    static const GVecGen4 g_vmao[3] = {
1664        { .fno = gen_helper_gvec_vmao8, },
1665        { .fno = gen_helper_gvec_vmao16, },
1666        { .fno = gen_helper_gvec_vmao32, },
1667    };
1668    static const GVecGen4 g_vmalo[3] = {
1669        { .fno = gen_helper_gvec_vmalo8, },
1670        { .fno = gen_helper_gvec_vmalo16, },
1671        { .fno = gen_helper_gvec_vmalo32, },
1672    };
1673    const GVecGen4 *fn;
1674
1675    if (es > ES_32) {
1676        gen_program_exception(s, PGM_SPECIFICATION);
1677        return DISAS_NORETURN;
1678    }
1679
1680    switch (s->fields.op2) {
1681    case 0xaa:
1682        fn = &g_vmal[es];
1683        break;
1684    case 0xab:
1685        fn = &g_vmah[es];
1686        break;
1687    case 0xa9:
1688        fn = &g_vmalh[es];
1689        break;
1690    case 0xae:
1691        fn = &g_vmae[es];
1692        break;
1693    case 0xac:
1694        fn = &g_vmale[es];
1695        break;
1696    case 0xaf:
1697        fn = &g_vmao[es];
1698        break;
1699    case 0xad:
1700        fn = &g_vmalo[es];
1701        break;
1702    default:
1703        g_assert_not_reached();
1704    }
1705
1706    gen_gvec_4(get_field(s, v1), get_field(s, v2),
1707               get_field(s, v3), get_field(s, v4), fn);
1708    return DISAS_NEXT;
1709}
1710
1711static void gen_mh_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
1712{
1713    TCGv_i32 t = tcg_temp_new_i32();
1714
1715    tcg_gen_muls2_i32(t, d, a, b);
1716    tcg_temp_free_i32(t);
1717}
1718
1719static void gen_mlh_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
1720{
1721    TCGv_i32 t = tcg_temp_new_i32();
1722
1723    tcg_gen_mulu2_i32(t, d, a, b);
1724    tcg_temp_free_i32(t);
1725}
1726
1727static DisasJumpType op_vm(DisasContext *s, DisasOps *o)
1728{
1729    const uint8_t es = get_field(s, m4);
1730    static const GVecGen3 g_vmh[3] = {
1731        { .fno = gen_helper_gvec_vmh8, },
1732        { .fno = gen_helper_gvec_vmh16, },
1733        { .fni4 = gen_mh_i32, },
1734    };
1735    static const GVecGen3 g_vmlh[3] = {
1736        { .fno = gen_helper_gvec_vmlh8, },
1737        { .fno = gen_helper_gvec_vmlh16, },
1738        { .fni4 = gen_mlh_i32, },
1739    };
1740    static const GVecGen3 g_vme[3] = {
1741        { .fno = gen_helper_gvec_vme8, },
1742        { .fno = gen_helper_gvec_vme16, },
1743        { .fno = gen_helper_gvec_vme32, },
1744    };
1745    static const GVecGen3 g_vmle[3] = {
1746        { .fno = gen_helper_gvec_vmle8, },
1747        { .fno = gen_helper_gvec_vmle16, },
1748        { .fno = gen_helper_gvec_vmle32, },
1749    };
1750    static const GVecGen3 g_vmo[3] = {
1751        { .fno = gen_helper_gvec_vmo8, },
1752        { .fno = gen_helper_gvec_vmo16, },
1753        { .fno = gen_helper_gvec_vmo32, },
1754    };
1755    static const GVecGen3 g_vmlo[3] = {
1756        { .fno = gen_helper_gvec_vmlo8, },
1757        { .fno = gen_helper_gvec_vmlo16, },
1758        { .fno = gen_helper_gvec_vmlo32, },
1759    };
1760    const GVecGen3 *fn;
1761
1762    if (es > ES_32) {
1763        gen_program_exception(s, PGM_SPECIFICATION);
1764        return DISAS_NORETURN;
1765    }
1766
1767    switch (s->fields.op2) {
1768    case 0xa2:
1769        gen_gvec_fn_3(mul, es, get_field(s, v1),
1770                      get_field(s, v2), get_field(s, v3));
1771        return DISAS_NEXT;
1772    case 0xa3:
1773        fn = &g_vmh[es];
1774        break;
1775    case 0xa1:
1776        fn = &g_vmlh[es];
1777        break;
1778    case 0xa6:
1779        fn = &g_vme[es];
1780        break;
1781    case 0xa4:
1782        fn = &g_vmle[es];
1783        break;
1784    case 0xa7:
1785        fn = &g_vmo[es];
1786        break;
1787    case 0xa5:
1788        fn = &g_vmlo[es];
1789        break;
1790    default:
1791        g_assert_not_reached();
1792    }
1793
1794    gen_gvec_3(get_field(s, v1), get_field(s, v2),
1795               get_field(s, v3), fn);
1796    return DISAS_NEXT;
1797}
1798
1799static DisasJumpType op_vnn(DisasContext *s, DisasOps *o)
1800{
1801    gen_gvec_fn_3(nand, ES_8, get_field(s, v1),
1802                  get_field(s, v2), get_field(s, v3));
1803    return DISAS_NEXT;
1804}
1805
1806static DisasJumpType op_vno(DisasContext *s, DisasOps *o)
1807{
1808    gen_gvec_fn_3(nor, ES_8, get_field(s, v1), get_field(s, v2),
1809                  get_field(s, v3));
1810    return DISAS_NEXT;
1811}
1812
1813static DisasJumpType op_vnx(DisasContext *s, DisasOps *o)
1814{
1815    gen_gvec_fn_3(eqv, ES_8, get_field(s, v1), get_field(s, v2),
1816                  get_field(s, v3));
1817    return DISAS_NEXT;
1818}
1819
1820static DisasJumpType op_vo(DisasContext *s, DisasOps *o)
1821{
1822    gen_gvec_fn_3(or, ES_8, get_field(s, v1), get_field(s, v2),
1823                  get_field(s, v3));
1824    return DISAS_NEXT;
1825}
1826
1827static DisasJumpType op_voc(DisasContext *s, DisasOps *o)
1828{
1829    gen_gvec_fn_3(orc, ES_8, get_field(s, v1), get_field(s, v2),
1830                  get_field(s, v3));
1831    return DISAS_NEXT;
1832}
1833
1834static DisasJumpType op_vpopct(DisasContext *s, DisasOps *o)
1835{
1836    const uint8_t es = get_field(s, m3);
1837    static const GVecGen2 g[4] = {
1838        { .fno = gen_helper_gvec_vpopct8, },
1839        { .fno = gen_helper_gvec_vpopct16, },
1840        { .fni4 = tcg_gen_ctpop_i32, },
1841        { .fni8 = tcg_gen_ctpop_i64, },
1842    };
1843
1844    if (es > ES_64 || (es != ES_8 && !s390_has_feat(S390_FEAT_VECTOR_ENH))) {
1845        gen_program_exception(s, PGM_SPECIFICATION);
1846        return DISAS_NORETURN;
1847    }
1848
1849    gen_gvec_2(get_field(s, v1), get_field(s, v2), &g[es]);
1850    return DISAS_NEXT;
1851}
1852
1853static void gen_rll_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
1854{
1855    TCGv_i32 t0 = tcg_temp_new_i32();
1856
1857    tcg_gen_andi_i32(t0, b, 31);
1858    tcg_gen_rotl_i32(d, a, t0);
1859    tcg_temp_free_i32(t0);
1860}
1861
1862static void gen_rll_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
1863{
1864    TCGv_i64 t0 = tcg_temp_new_i64();
1865
1866    tcg_gen_andi_i64(t0, b, 63);
1867    tcg_gen_rotl_i64(d, a, t0);
1868    tcg_temp_free_i64(t0);
1869}
1870
1871static DisasJumpType op_verllv(DisasContext *s, DisasOps *o)
1872{
1873    const uint8_t es = get_field(s, m4);
1874    static const GVecGen3 g[4] = {
1875        { .fno = gen_helper_gvec_verllv8, },
1876        { .fno = gen_helper_gvec_verllv16, },
1877        { .fni4 = gen_rll_i32, },
1878        { .fni8 = gen_rll_i64, },
1879    };
1880
1881    if (es > ES_64) {
1882        gen_program_exception(s, PGM_SPECIFICATION);
1883        return DISAS_NORETURN;
1884    }
1885
1886    gen_gvec_3(get_field(s, v1), get_field(s, v2),
1887               get_field(s, v3), &g[es]);
1888    return DISAS_NEXT;
1889}
1890
1891static DisasJumpType op_verll(DisasContext *s, DisasOps *o)
1892{
1893    const uint8_t es = get_field(s, m4);
1894    static const GVecGen2s g[4] = {
1895        { .fno = gen_helper_gvec_verll8, },
1896        { .fno = gen_helper_gvec_verll16, },
1897        { .fni4 = gen_rll_i32, },
1898        { .fni8 = gen_rll_i64, },
1899    };
1900
1901    if (es > ES_64) {
1902        gen_program_exception(s, PGM_SPECIFICATION);
1903        return DISAS_NORETURN;
1904    }
1905    gen_gvec_2s(get_field(s, v1), get_field(s, v3), o->addr1,
1906                &g[es]);
1907    return DISAS_NEXT;
1908}
1909
1910static void gen_rim_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b, int32_t c)
1911{
1912    TCGv_i32 t = tcg_temp_new_i32();
1913
1914    tcg_gen_rotli_i32(t, a, c & 31);
1915    tcg_gen_and_i32(t, t, b);
1916    tcg_gen_andc_i32(d, d, b);
1917    tcg_gen_or_i32(d, d, t);
1918
1919    tcg_temp_free_i32(t);
1920}
1921
1922static void gen_rim_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b, int64_t c)
1923{
1924    TCGv_i64 t = tcg_temp_new_i64();
1925
1926    tcg_gen_rotli_i64(t, a, c & 63);
1927    tcg_gen_and_i64(t, t, b);
1928    tcg_gen_andc_i64(d, d, b);
1929    tcg_gen_or_i64(d, d, t);
1930
1931    tcg_temp_free_i64(t);
1932}
1933
1934static DisasJumpType op_verim(DisasContext *s, DisasOps *o)
1935{
1936    const uint8_t es = get_field(s, m5);
1937    const uint8_t i4 = get_field(s, i4) &
1938                       (NUM_VEC_ELEMENT_BITS(es) - 1);
1939    static const GVecGen3i g[4] = {
1940        { .fno = gen_helper_gvec_verim8, },
1941        { .fno = gen_helper_gvec_verim16, },
1942        { .fni4 = gen_rim_i32,
1943          .load_dest = true, },
1944        { .fni8 = gen_rim_i64,
1945          .load_dest = true, },
1946    };
1947
1948    if (es > ES_64) {
1949        gen_program_exception(s, PGM_SPECIFICATION);
1950        return DISAS_NORETURN;
1951    }
1952
1953    gen_gvec_3i(get_field(s, v1), get_field(s, v2),
1954                get_field(s, v3), i4, &g[es]);
1955    return DISAS_NEXT;
1956}
1957
1958static DisasJumpType op_vesv(DisasContext *s, DisasOps *o)
1959{
1960    const uint8_t es = get_field(s, m4);
1961    const uint8_t v1 = get_field(s, v1);
1962    const uint8_t v2 = get_field(s, v2);
1963    const uint8_t v3 = get_field(s, v3);
1964
1965    if (es > ES_64) {
1966        gen_program_exception(s, PGM_SPECIFICATION);
1967        return DISAS_NORETURN;
1968    }
1969
1970    switch (s->fields.op2) {
1971    case 0x70:
1972        gen_gvec_fn_3(shlv, es, v1, v2, v3);
1973        break;
1974    case 0x7a:
1975        gen_gvec_fn_3(sarv, es, v1, v2, v3);
1976        break;
1977    case 0x78:
1978        gen_gvec_fn_3(shrv, es, v1, v2, v3);
1979        break;
1980    default:
1981        g_assert_not_reached();
1982    }
1983    return DISAS_NEXT;
1984}
1985
1986static DisasJumpType op_ves(DisasContext *s, DisasOps *o)
1987{
1988    const uint8_t es = get_field(s, m4);
1989    const uint8_t d2 = get_field(s, d2) &
1990                       (NUM_VEC_ELEMENT_BITS(es) - 1);
1991    const uint8_t v1 = get_field(s, v1);
1992    const uint8_t v3 = get_field(s, v3);
1993    TCGv_i32 shift;
1994
1995    if (es > ES_64) {
1996        gen_program_exception(s, PGM_SPECIFICATION);
1997        return DISAS_NORETURN;
1998    }
1999
2000    if (likely(!get_field(s, b2))) {
2001        switch (s->fields.op2) {
2002        case 0x30:
2003            gen_gvec_fn_2i(shli, es, v1, v3, d2);
2004            break;
2005        case 0x3a:
2006            gen_gvec_fn_2i(sari, es, v1, v3, d2);
2007            break;
2008        case 0x38:
2009            gen_gvec_fn_2i(shri, es, v1, v3, d2);
2010            break;
2011        default:
2012            g_assert_not_reached();
2013        }
2014    } else {
2015        shift = tcg_temp_new_i32();
2016        tcg_gen_extrl_i64_i32(shift, o->addr1);
2017        tcg_gen_andi_i32(shift, shift, NUM_VEC_ELEMENT_BITS(es) - 1);
2018        switch (s->fields.op2) {
2019        case 0x30:
2020            gen_gvec_fn_2s(shls, es, v1, v3, shift);
2021            break;
2022        case 0x3a:
2023            gen_gvec_fn_2s(sars, es, v1, v3, shift);
2024            break;
2025        case 0x38:
2026            gen_gvec_fn_2s(shrs, es, v1, v3, shift);
2027            break;
2028        default:
2029            g_assert_not_reached();
2030        }
2031        tcg_temp_free_i32(shift);
2032    }
2033    return DISAS_NEXT;
2034}
2035
2036static DisasJumpType op_vsl(DisasContext *s, DisasOps *o)
2037{
2038    TCGv_i64 shift = tcg_temp_new_i64();
2039
2040    read_vec_element_i64(shift, get_field(s, v3), 7, ES_8);
2041    if (s->fields.op2 == 0x74) {
2042        tcg_gen_andi_i64(shift, shift, 0x7);
2043    } else {
2044        tcg_gen_andi_i64(shift, shift, 0x78);
2045    }
2046
2047    gen_gvec_2i_ool(get_field(s, v1), get_field(s, v2),
2048                    shift, 0, gen_helper_gvec_vsl);
2049    tcg_temp_free_i64(shift);
2050    return DISAS_NEXT;
2051}
2052
2053static DisasJumpType op_vsldb(DisasContext *s, DisasOps *o)
2054{
2055    const uint8_t i4 = get_field(s, i4) & 0xf;
2056    const int left_shift = (i4 & 7) * 8;
2057    const int right_shift = 64 - left_shift;
2058    TCGv_i64 t0 = tcg_temp_new_i64();
2059    TCGv_i64 t1 = tcg_temp_new_i64();
2060    TCGv_i64 t2 = tcg_temp_new_i64();
2061
2062    if ((i4 & 8) == 0) {
2063        read_vec_element_i64(t0, get_field(s, v2), 0, ES_64);
2064        read_vec_element_i64(t1, get_field(s, v2), 1, ES_64);
2065        read_vec_element_i64(t2, get_field(s, v3), 0, ES_64);
2066    } else {
2067        read_vec_element_i64(t0, get_field(s, v2), 1, ES_64);
2068        read_vec_element_i64(t1, get_field(s, v3), 0, ES_64);
2069        read_vec_element_i64(t2, get_field(s, v3), 1, ES_64);
2070    }
2071    tcg_gen_extract2_i64(t0, t1, t0, right_shift);
2072    tcg_gen_extract2_i64(t1, t2, t1, right_shift);
2073    write_vec_element_i64(t0, get_field(s, v1), 0, ES_64);
2074    write_vec_element_i64(t1, get_field(s, v1), 1, ES_64);
2075
2076    tcg_temp_free(t0);
2077    tcg_temp_free(t1);
2078    tcg_temp_free(t2);
2079    return DISAS_NEXT;
2080}
2081
2082static DisasJumpType op_vsra(DisasContext *s, DisasOps *o)
2083{
2084    TCGv_i64 shift = tcg_temp_new_i64();
2085
2086    read_vec_element_i64(shift, get_field(s, v3), 7, ES_8);
2087    if (s->fields.op2 == 0x7e) {
2088        tcg_gen_andi_i64(shift, shift, 0x7);
2089    } else {
2090        tcg_gen_andi_i64(shift, shift, 0x78);
2091    }
2092
2093    gen_gvec_2i_ool(get_field(s, v1), get_field(s, v2),
2094                    shift, 0, gen_helper_gvec_vsra);
2095    tcg_temp_free_i64(shift);
2096    return DISAS_NEXT;
2097}
2098
2099static DisasJumpType op_vsrl(DisasContext *s, DisasOps *o)
2100{
2101    TCGv_i64 shift = tcg_temp_new_i64();
2102
2103    read_vec_element_i64(shift, get_field(s, v3), 7, ES_8);
2104    if (s->fields.op2 == 0x7c) {
2105        tcg_gen_andi_i64(shift, shift, 0x7);
2106    } else {
2107        tcg_gen_andi_i64(shift, shift, 0x78);
2108    }
2109
2110    gen_gvec_2i_ool(get_field(s, v1), get_field(s, v2),
2111                    shift, 0, gen_helper_gvec_vsrl);
2112    tcg_temp_free_i64(shift);
2113    return DISAS_NEXT;
2114}
2115
2116static DisasJumpType op_vs(DisasContext *s, DisasOps *o)
2117{
2118    const uint8_t es = get_field(s, m4);
2119
2120    if (es > ES_128) {
2121        gen_program_exception(s, PGM_SPECIFICATION);
2122        return DISAS_NORETURN;
2123    } else if (es == ES_128) {
2124        gen_gvec128_3_i64(tcg_gen_sub2_i64, get_field(s, v1),
2125                          get_field(s, v2), get_field(s, v3));
2126        return DISAS_NEXT;
2127    }
2128    gen_gvec_fn_3(sub, es, get_field(s, v1), get_field(s, v2),
2129                  get_field(s, v3));
2130    return DISAS_NEXT;
2131}
2132
2133static void gen_scbi_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
2134{
2135    tcg_gen_setcond_i32(TCG_COND_GEU, d, a, b);
2136}
2137
2138static void gen_scbi_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
2139{
2140    tcg_gen_setcond_i64(TCG_COND_GEU, d, a, b);
2141}
2142
2143static void gen_scbi2_i64(TCGv_i64 dl, TCGv_i64 dh, TCGv_i64 al,
2144                          TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh)
2145{
2146    TCGv_i64 th = tcg_temp_new_i64();
2147    TCGv_i64 tl = tcg_temp_new_i64();
2148    TCGv_i64 zero = tcg_const_i64(0);
2149
2150    tcg_gen_sub2_i64(tl, th, al, zero, bl, zero);
2151    tcg_gen_andi_i64(th, th, 1);
2152    tcg_gen_sub2_i64(tl, th, ah, zero, th, zero);
2153    tcg_gen_sub2_i64(tl, th, tl, th, bh, zero);
2154    /* "invert" the result: -1 -> 0; 0 -> 1 */
2155    tcg_gen_addi_i64(dl, th, 1);
2156    tcg_gen_mov_i64(dh, zero);
2157
2158    tcg_temp_free_i64(th);
2159    tcg_temp_free_i64(tl);
2160    tcg_temp_free_i64(zero);
2161}
2162
2163static DisasJumpType op_vscbi(DisasContext *s, DisasOps *o)
2164{
2165    const uint8_t es = get_field(s, m4);
2166    static const GVecGen3 g[4] = {
2167        { .fno = gen_helper_gvec_vscbi8, },
2168        { .fno = gen_helper_gvec_vscbi16, },
2169        { .fni4 = gen_scbi_i32, },
2170        { .fni8 = gen_scbi_i64, },
2171    };
2172
2173    if (es > ES_128) {
2174        gen_program_exception(s, PGM_SPECIFICATION);
2175        return DISAS_NORETURN;
2176    } else if (es == ES_128) {
2177        gen_gvec128_3_i64(gen_scbi2_i64, get_field(s, v1),
2178                          get_field(s, v2), get_field(s, v3));
2179        return DISAS_NEXT;
2180    }
2181    gen_gvec_3(get_field(s, v1), get_field(s, v2),
2182               get_field(s, v3), &g[es]);
2183    return DISAS_NEXT;
2184}
2185
2186static void gen_sbi2_i64(TCGv_i64 dl, TCGv_i64 dh, TCGv_i64 al, TCGv_i64 ah,
2187                         TCGv_i64 bl, TCGv_i64 bh, TCGv_i64 cl, TCGv_i64 ch)
2188{
2189    TCGv_i64 tl = tcg_temp_new_i64();
2190    TCGv_i64 th = tcg_temp_new_i64();
2191
2192    tcg_gen_not_i64(tl, bl);
2193    tcg_gen_not_i64(th, bh);
2194    gen_ac2_i64(dl, dh, al, ah, tl, th, cl, ch);
2195    tcg_temp_free_i64(tl);
2196    tcg_temp_free_i64(th);
2197}
2198
2199static DisasJumpType op_vsbi(DisasContext *s, DisasOps *o)
2200{
2201    if (get_field(s, m5) != ES_128) {
2202        gen_program_exception(s, PGM_SPECIFICATION);
2203        return DISAS_NORETURN;
2204    }
2205
2206    gen_gvec128_4_i64(gen_sbi2_i64, get_field(s, v1),
2207                      get_field(s, v2), get_field(s, v3),
2208                      get_field(s, v4));
2209    return DISAS_NEXT;
2210}
2211
2212static void gen_sbcbi2_i64(TCGv_i64 dl, TCGv_i64 dh, TCGv_i64 al, TCGv_i64 ah,
2213                           TCGv_i64 bl, TCGv_i64 bh, TCGv_i64 cl, TCGv_i64 ch)
2214{
2215    TCGv_i64 th = tcg_temp_new_i64();
2216    TCGv_i64 tl = tcg_temp_new_i64();
2217
2218    tcg_gen_not_i64(tl, bl);
2219    tcg_gen_not_i64(th, bh);
2220    gen_accc2_i64(dl, dh, al, ah, tl, th, cl, ch);
2221
2222    tcg_temp_free_i64(tl);
2223    tcg_temp_free_i64(th);
2224}
2225
2226static DisasJumpType op_vsbcbi(DisasContext *s, DisasOps *o)
2227{
2228    if (get_field(s, m5) != ES_128) {
2229        gen_program_exception(s, PGM_SPECIFICATION);
2230        return DISAS_NORETURN;
2231    }
2232
2233    gen_gvec128_4_i64(gen_sbcbi2_i64, get_field(s, v1),
2234                      get_field(s, v2), get_field(s, v3),
2235                      get_field(s, v4));
2236    return DISAS_NEXT;
2237}
2238
2239static DisasJumpType op_vsumg(DisasContext *s, DisasOps *o)
2240{
2241    const uint8_t es = get_field(s, m4);
2242    TCGv_i64 sum, tmp;
2243    uint8_t dst_idx;
2244
2245    if (es == ES_8 || es > ES_32) {
2246        gen_program_exception(s, PGM_SPECIFICATION);
2247        return DISAS_NORETURN;
2248    }
2249
2250    sum = tcg_temp_new_i64();
2251    tmp = tcg_temp_new_i64();
2252    for (dst_idx = 0; dst_idx < 2; dst_idx++) {
2253        uint8_t idx = dst_idx * NUM_VEC_ELEMENTS(es) / 2;
2254        const uint8_t max_idx = idx + NUM_VEC_ELEMENTS(es) / 2 - 1;
2255
2256        read_vec_element_i64(sum, get_field(s, v3), max_idx, es);
2257        for (; idx <= max_idx; idx++) {
2258            read_vec_element_i64(tmp, get_field(s, v2), idx, es);
2259            tcg_gen_add_i64(sum, sum, tmp);
2260        }
2261        write_vec_element_i64(sum, get_field(s, v1), dst_idx, ES_64);
2262    }
2263    tcg_temp_free_i64(sum);
2264    tcg_temp_free_i64(tmp);
2265    return DISAS_NEXT;
2266}
2267
2268static DisasJumpType op_vsumq(DisasContext *s, DisasOps *o)
2269{
2270    const uint8_t es = get_field(s, m4);
2271    const uint8_t max_idx = NUM_VEC_ELEMENTS(es) - 1;
2272    TCGv_i64 sumh, suml, zero, tmpl;
2273    uint8_t idx;
2274
2275    if (es < ES_32 || es > ES_64) {
2276        gen_program_exception(s, PGM_SPECIFICATION);
2277        return DISAS_NORETURN;
2278    }
2279
2280    sumh = tcg_const_i64(0);
2281    suml = tcg_temp_new_i64();
2282    zero = tcg_const_i64(0);
2283    tmpl = tcg_temp_new_i64();
2284
2285    read_vec_element_i64(suml, get_field(s, v3), max_idx, es);
2286    for (idx = 0; idx <= max_idx; idx++) {
2287        read_vec_element_i64(tmpl, get_field(s, v2), idx, es);
2288        tcg_gen_add2_i64(suml, sumh, suml, sumh, tmpl, zero);
2289    }
2290    write_vec_element_i64(sumh, get_field(s, v1), 0, ES_64);
2291    write_vec_element_i64(suml, get_field(s, v1), 1, ES_64);
2292
2293    tcg_temp_free_i64(sumh);
2294    tcg_temp_free_i64(suml);
2295    tcg_temp_free_i64(zero);
2296    tcg_temp_free_i64(tmpl);
2297    return DISAS_NEXT;
2298}
2299
2300static DisasJumpType op_vsum(DisasContext *s, DisasOps *o)
2301{
2302    const uint8_t es = get_field(s, m4);
2303    TCGv_i32 sum, tmp;
2304    uint8_t dst_idx;
2305
2306    if (es > ES_16) {
2307        gen_program_exception(s, PGM_SPECIFICATION);
2308        return DISAS_NORETURN;
2309    }
2310
2311    sum = tcg_temp_new_i32();
2312    tmp = tcg_temp_new_i32();
2313    for (dst_idx = 0; dst_idx < 4; dst_idx++) {
2314        uint8_t idx = dst_idx * NUM_VEC_ELEMENTS(es) / 4;
2315        const uint8_t max_idx = idx + NUM_VEC_ELEMENTS(es) / 4 - 1;
2316
2317        read_vec_element_i32(sum, get_field(s, v3), max_idx, es);
2318        for (; idx <= max_idx; idx++) {
2319            read_vec_element_i32(tmp, get_field(s, v2), idx, es);
2320            tcg_gen_add_i32(sum, sum, tmp);
2321        }
2322        write_vec_element_i32(sum, get_field(s, v1), dst_idx, ES_32);
2323    }
2324    tcg_temp_free_i32(sum);
2325    tcg_temp_free_i32(tmp);
2326    return DISAS_NEXT;
2327}
2328
2329static DisasJumpType op_vtm(DisasContext *s, DisasOps *o)
2330{
2331    gen_gvec_2_ptr(get_field(s, v1), get_field(s, v2),
2332                   cpu_env, 0, gen_helper_gvec_vtm);
2333    set_cc_static(s);
2334    return DISAS_NEXT;
2335}
2336
2337static DisasJumpType op_vfae(DisasContext *s, DisasOps *o)
2338{
2339    const uint8_t es = get_field(s, m4);
2340    const uint8_t m5 = get_field(s, m5);
2341    static gen_helper_gvec_3 * const g[3] = {
2342        gen_helper_gvec_vfae8,
2343        gen_helper_gvec_vfae16,
2344        gen_helper_gvec_vfae32,
2345    };
2346    static gen_helper_gvec_3_ptr * const g_cc[3] = {
2347        gen_helper_gvec_vfae_cc8,
2348        gen_helper_gvec_vfae_cc16,
2349        gen_helper_gvec_vfae_cc32,
2350    };
2351    if (es > ES_32) {
2352        gen_program_exception(s, PGM_SPECIFICATION);
2353        return DISAS_NORETURN;
2354    }
2355
2356    if (extract32(m5, 0, 1)) {
2357        gen_gvec_3_ptr(get_field(s, v1), get_field(s, v2),
2358                       get_field(s, v3), cpu_env, m5, g_cc[es]);
2359        set_cc_static(s);
2360    } else {
2361        gen_gvec_3_ool(get_field(s, v1), get_field(s, v2),
2362                       get_field(s, v3), m5, g[es]);
2363    }
2364    return DISAS_NEXT;
2365}
2366
2367static DisasJumpType op_vfee(DisasContext *s, DisasOps *o)
2368{
2369    const uint8_t es = get_field(s, m4);
2370    const uint8_t m5 = get_field(s, m5);
2371    static gen_helper_gvec_3 * const g[3] = {
2372        gen_helper_gvec_vfee8,
2373        gen_helper_gvec_vfee16,
2374        gen_helper_gvec_vfee32,
2375    };
2376    static gen_helper_gvec_3_ptr * const g_cc[3] = {
2377        gen_helper_gvec_vfee_cc8,
2378        gen_helper_gvec_vfee_cc16,
2379        gen_helper_gvec_vfee_cc32,
2380    };
2381
2382    if (es > ES_32 || m5 & ~0x3) {
2383        gen_program_exception(s, PGM_SPECIFICATION);
2384        return DISAS_NORETURN;
2385    }
2386
2387    if (extract32(m5, 0, 1)) {
2388        gen_gvec_3_ptr(get_field(s, v1), get_field(s, v2),
2389                       get_field(s, v3), cpu_env, m5, g_cc[es]);
2390        set_cc_static(s);
2391    } else {
2392        gen_gvec_3_ool(get_field(s, v1), get_field(s, v2),
2393                       get_field(s, v3), m5, g[es]);
2394    }
2395    return DISAS_NEXT;
2396}
2397
2398static DisasJumpType op_vfene(DisasContext *s, DisasOps *o)
2399{
2400    const uint8_t es = get_field(s, m4);
2401    const uint8_t m5 = get_field(s, m5);
2402    static gen_helper_gvec_3 * const g[3] = {
2403        gen_helper_gvec_vfene8,
2404        gen_helper_gvec_vfene16,
2405        gen_helper_gvec_vfene32,
2406    };
2407    static gen_helper_gvec_3_ptr * const g_cc[3] = {
2408        gen_helper_gvec_vfene_cc8,
2409        gen_helper_gvec_vfene_cc16,
2410        gen_helper_gvec_vfene_cc32,
2411    };
2412
2413    if (es > ES_32 || m5 & ~0x3) {
2414        gen_program_exception(s, PGM_SPECIFICATION);
2415        return DISAS_NORETURN;
2416    }
2417
2418    if (extract32(m5, 0, 1)) {
2419        gen_gvec_3_ptr(get_field(s, v1), get_field(s, v2),
2420                       get_field(s, v3), cpu_env, m5, g_cc[es]);
2421        set_cc_static(s);
2422    } else {
2423        gen_gvec_3_ool(get_field(s, v1), get_field(s, v2),
2424                       get_field(s, v3), m5, g[es]);
2425    }
2426    return DISAS_NEXT;
2427}
2428
2429static DisasJumpType op_vistr(DisasContext *s, DisasOps *o)
2430{
2431    const uint8_t es = get_field(s, m4);
2432    const uint8_t m5 = get_field(s, m5);
2433    static gen_helper_gvec_2 * const g[3] = {
2434        gen_helper_gvec_vistr8,
2435        gen_helper_gvec_vistr16,
2436        gen_helper_gvec_vistr32,
2437    };
2438    static gen_helper_gvec_2_ptr * const g_cc[3] = {
2439        gen_helper_gvec_vistr_cc8,
2440        gen_helper_gvec_vistr_cc16,
2441        gen_helper_gvec_vistr_cc32,
2442    };
2443
2444    if (es > ES_32 || m5 & ~0x1) {
2445        gen_program_exception(s, PGM_SPECIFICATION);
2446        return DISAS_NORETURN;
2447    }
2448
2449    if (extract32(m5, 0, 1)) {
2450        gen_gvec_2_ptr(get_field(s, v1), get_field(s, v2),
2451                       cpu_env, 0, g_cc[es]);
2452        set_cc_static(s);
2453    } else {
2454        gen_gvec_2_ool(get_field(s, v1), get_field(s, v2), 0,
2455                       g[es]);
2456    }
2457    return DISAS_NEXT;
2458}
2459
2460static DisasJumpType op_vstrc(DisasContext *s, DisasOps *o)
2461{
2462    const uint8_t es = get_field(s, m5);
2463    const uint8_t m6 = get_field(s, m6);
2464    static gen_helper_gvec_4 * const g[3] = {
2465        gen_helper_gvec_vstrc8,
2466        gen_helper_gvec_vstrc16,
2467        gen_helper_gvec_vstrc32,
2468    };
2469    static gen_helper_gvec_4 * const g_rt[3] = {
2470        gen_helper_gvec_vstrc_rt8,
2471        gen_helper_gvec_vstrc_rt16,
2472        gen_helper_gvec_vstrc_rt32,
2473    };
2474    static gen_helper_gvec_4_ptr * const g_cc[3] = {
2475        gen_helper_gvec_vstrc_cc8,
2476        gen_helper_gvec_vstrc_cc16,
2477        gen_helper_gvec_vstrc_cc32,
2478    };
2479    static gen_helper_gvec_4_ptr * const g_cc_rt[3] = {
2480        gen_helper_gvec_vstrc_cc_rt8,
2481        gen_helper_gvec_vstrc_cc_rt16,
2482        gen_helper_gvec_vstrc_cc_rt32,
2483    };
2484
2485    if (es > ES_32) {
2486        gen_program_exception(s, PGM_SPECIFICATION);
2487        return DISAS_NORETURN;
2488    }
2489
2490    if (extract32(m6, 0, 1)) {
2491        if (extract32(m6, 2, 1)) {
2492            gen_gvec_4_ptr(get_field(s, v1), get_field(s, v2),
2493                           get_field(s, v3), get_field(s, v4),
2494                           cpu_env, m6, g_cc_rt[es]);
2495        } else {
2496            gen_gvec_4_ptr(get_field(s, v1), get_field(s, v2),
2497                           get_field(s, v3), get_field(s, v4),
2498                           cpu_env, m6, g_cc[es]);
2499        }
2500        set_cc_static(s);
2501    } else {
2502        if (extract32(m6, 2, 1)) {
2503            gen_gvec_4_ool(get_field(s, v1), get_field(s, v2),
2504                           get_field(s, v3), get_field(s, v4),
2505                           m6, g_rt[es]);
2506        } else {
2507            gen_gvec_4_ool(get_field(s, v1), get_field(s, v2),
2508                           get_field(s, v3), get_field(s, v4),
2509                           m6, g[es]);
2510        }
2511    }
2512    return DISAS_NEXT;
2513}
2514
2515static DisasJumpType op_vfa(DisasContext *s, DisasOps *o)
2516{
2517    const uint8_t fpf = get_field(s, m4);
2518    const uint8_t m5 = get_field(s, m5);
2519    const bool se = extract32(m5, 3, 1);
2520    gen_helper_gvec_3_ptr *fn;
2521
2522    if (fpf != FPF_LONG || extract32(m5, 0, 3)) {
2523        gen_program_exception(s, PGM_SPECIFICATION);
2524        return DISAS_NORETURN;
2525    }
2526
2527    switch (s->fields.op2) {
2528    case 0xe3:
2529        fn = se ? gen_helper_gvec_vfa64s : gen_helper_gvec_vfa64;
2530        break;
2531    case 0xe5:
2532        fn = se ? gen_helper_gvec_vfd64s : gen_helper_gvec_vfd64;
2533        break;
2534    case 0xe7:
2535        fn = se ? gen_helper_gvec_vfm64s : gen_helper_gvec_vfm64;
2536        break;
2537    case 0xe2:
2538        fn = se ? gen_helper_gvec_vfs64s : gen_helper_gvec_vfs64;
2539        break;
2540    default:
2541        g_assert_not_reached();
2542    }
2543    gen_gvec_3_ptr(get_field(s, v1), get_field(s, v2),
2544                   get_field(s, v3), cpu_env, 0, fn);
2545    return DISAS_NEXT;
2546}
2547
2548static DisasJumpType op_wfc(DisasContext *s, DisasOps *o)
2549{
2550    const uint8_t fpf = get_field(s, m3);
2551    const uint8_t m4 = get_field(s, m4);
2552
2553    if (fpf != FPF_LONG || m4) {
2554        gen_program_exception(s, PGM_SPECIFICATION);
2555        return DISAS_NORETURN;
2556    }
2557
2558    if (s->fields.op2 == 0xcb) {
2559        gen_gvec_2_ptr(get_field(s, v1), get_field(s, v2),
2560                       cpu_env, 0, gen_helper_gvec_wfc64);
2561    } else {
2562        gen_gvec_2_ptr(get_field(s, v1), get_field(s, v2),
2563                       cpu_env, 0, gen_helper_gvec_wfk64);
2564    }
2565    set_cc_static(s);
2566    return DISAS_NEXT;
2567}
2568
2569static DisasJumpType op_vfc(DisasContext *s, DisasOps *o)
2570{
2571    const uint8_t fpf = get_field(s, m4);
2572    const uint8_t m5 = get_field(s, m5);
2573    const uint8_t m6 = get_field(s, m6);
2574    const bool se = extract32(m5, 3, 1);
2575    const bool cs = extract32(m6, 0, 1);
2576    gen_helper_gvec_3_ptr *fn;
2577
2578    if (fpf != FPF_LONG || extract32(m5, 0, 3) || extract32(m6, 1, 3)) {
2579        gen_program_exception(s, PGM_SPECIFICATION);
2580        return DISAS_NORETURN;
2581    }
2582
2583    if (cs) {
2584        switch (s->fields.op2) {
2585        case 0xe8:
2586            fn = se ? gen_helper_gvec_vfce64s_cc : gen_helper_gvec_vfce64_cc;
2587            break;
2588        case 0xeb:
2589            fn = se ? gen_helper_gvec_vfch64s_cc : gen_helper_gvec_vfch64_cc;
2590            break;
2591        case 0xea:
2592            fn = se ? gen_helper_gvec_vfche64s_cc : gen_helper_gvec_vfche64_cc;
2593            break;
2594        default:
2595            g_assert_not_reached();
2596        }
2597    } else {
2598        switch (s->fields.op2) {
2599        case 0xe8:
2600            fn = se ? gen_helper_gvec_vfce64s : gen_helper_gvec_vfce64;
2601            break;
2602        case 0xeb:
2603            fn = se ? gen_helper_gvec_vfch64s : gen_helper_gvec_vfch64;
2604            break;
2605        case 0xea:
2606            fn = se ? gen_helper_gvec_vfche64s : gen_helper_gvec_vfche64;
2607            break;
2608        default:
2609            g_assert_not_reached();
2610        }
2611    }
2612    gen_gvec_3_ptr(get_field(s, v1), get_field(s, v2),
2613                   get_field(s, v3), cpu_env, 0, fn);
2614    if (cs) {
2615        set_cc_static(s);
2616    }
2617    return DISAS_NEXT;
2618}
2619
2620static DisasJumpType op_vcdg(DisasContext *s, DisasOps *o)
2621{
2622    const uint8_t fpf = get_field(s, m3);
2623    const uint8_t m4 = get_field(s, m4);
2624    const uint8_t erm = get_field(s, m5);
2625    const bool se = extract32(m4, 3, 1);
2626    gen_helper_gvec_2_ptr *fn;
2627
2628    if (fpf != FPF_LONG || extract32(m4, 0, 2) || erm > 7 || erm == 2) {
2629        gen_program_exception(s, PGM_SPECIFICATION);
2630        return DISAS_NORETURN;
2631    }
2632
2633    switch (s->fields.op2) {
2634    case 0xc3:
2635        fn = se ? gen_helper_gvec_vcdg64s : gen_helper_gvec_vcdg64;
2636        break;
2637    case 0xc1:
2638        fn = se ? gen_helper_gvec_vcdlg64s : gen_helper_gvec_vcdlg64;
2639        break;
2640    case 0xc2:
2641        fn = se ? gen_helper_gvec_vcgd64s : gen_helper_gvec_vcgd64;
2642        break;
2643    case 0xc0:
2644        fn = se ? gen_helper_gvec_vclgd64s : gen_helper_gvec_vclgd64;
2645        break;
2646    case 0xc7:
2647        fn = se ? gen_helper_gvec_vfi64s : gen_helper_gvec_vfi64;
2648        break;
2649    case 0xc5:
2650        fn = se ? gen_helper_gvec_vflr64s : gen_helper_gvec_vflr64;
2651        break;
2652    default:
2653        g_assert_not_reached();
2654    }
2655    gen_gvec_2_ptr(get_field(s, v1), get_field(s, v2), cpu_env,
2656                   deposit32(m4, 4, 4, erm), fn);
2657    return DISAS_NEXT;
2658}
2659
2660static DisasJumpType op_vfll(DisasContext *s, DisasOps *o)
2661{
2662    const uint8_t fpf = get_field(s, m3);
2663    const uint8_t m4 = get_field(s, m4);
2664    gen_helper_gvec_2_ptr *fn = gen_helper_gvec_vfll32;
2665
2666    if (fpf != FPF_SHORT || extract32(m4, 0, 3)) {
2667        gen_program_exception(s, PGM_SPECIFICATION);
2668        return DISAS_NORETURN;
2669    }
2670
2671    if (extract32(m4, 3, 1)) {
2672        fn = gen_helper_gvec_vfll32s;
2673    }
2674    gen_gvec_2_ptr(get_field(s, v1), get_field(s, v2), cpu_env,
2675                   0, fn);
2676    return DISAS_NEXT;
2677}
2678
2679static DisasJumpType op_vfma(DisasContext *s, DisasOps *o)
2680{
2681    const uint8_t m5 = get_field(s, m5);
2682    const uint8_t fpf = get_field(s, m6);
2683    const bool se = extract32(m5, 3, 1);
2684    gen_helper_gvec_4_ptr *fn;
2685
2686    if (fpf != FPF_LONG || extract32(m5, 0, 3)) {
2687        gen_program_exception(s, PGM_SPECIFICATION);
2688        return DISAS_NORETURN;
2689    }
2690
2691    if (s->fields.op2 == 0x8f) {
2692        fn = se ? gen_helper_gvec_vfma64s : gen_helper_gvec_vfma64;
2693    } else {
2694        fn = se ? gen_helper_gvec_vfms64s : gen_helper_gvec_vfms64;
2695    }
2696    gen_gvec_4_ptr(get_field(s, v1), get_field(s, v2),
2697                   get_field(s, v3), get_field(s, v4), cpu_env,
2698                   0, fn);
2699    return DISAS_NEXT;
2700}
2701
2702static DisasJumpType op_vfpso(DisasContext *s, DisasOps *o)
2703{
2704    const uint8_t v1 = get_field(s, v1);
2705    const uint8_t v2 = get_field(s, v2);
2706    const uint8_t fpf = get_field(s, m3);
2707    const uint8_t m4 = get_field(s, m4);
2708    const uint8_t m5 = get_field(s, m5);
2709    TCGv_i64 tmp;
2710
2711    if (fpf != FPF_LONG || extract32(m4, 0, 3) || m5 > 2) {
2712        gen_program_exception(s, PGM_SPECIFICATION);
2713        return DISAS_NORETURN;
2714    }
2715
2716    if (extract32(m4, 3, 1)) {
2717        tmp = tcg_temp_new_i64();
2718        read_vec_element_i64(tmp, v2, 0, ES_64);
2719        switch (m5) {
2720        case 0:
2721            /* sign bit is inverted (complement) */
2722            tcg_gen_xori_i64(tmp, tmp, 1ull << 63);
2723            break;
2724        case 1:
2725            /* sign bit is set to one (negative) */
2726            tcg_gen_ori_i64(tmp, tmp, 1ull << 63);
2727            break;
2728        case 2:
2729            /* sign bit is set to zero (positive) */
2730            tcg_gen_andi_i64(tmp, tmp, (1ull << 63) - 1);
2731            break;
2732        }
2733        write_vec_element_i64(tmp, v1, 0, ES_64);
2734        tcg_temp_free_i64(tmp);
2735    } else {
2736        switch (m5) {
2737        case 0:
2738            /* sign bit is inverted (complement) */
2739            gen_gvec_fn_2i(xori, ES_64, v1, v2, 1ull << 63);
2740            break;
2741        case 1:
2742            /* sign bit is set to one (negative) */
2743            gen_gvec_fn_2i(ori, ES_64, v1, v2, 1ull << 63);
2744            break;
2745        case 2:
2746            /* sign bit is set to zero (positive) */
2747            gen_gvec_fn_2i(andi, ES_64, v1, v2, (1ull << 63) - 1);
2748            break;
2749        }
2750    }
2751    return DISAS_NEXT;
2752}
2753
2754static DisasJumpType op_vfsq(DisasContext *s, DisasOps *o)
2755{
2756    const uint8_t fpf = get_field(s, m3);
2757    const uint8_t m4 = get_field(s, m4);
2758    gen_helper_gvec_2_ptr *fn = gen_helper_gvec_vfsq64;
2759
2760    if (fpf != FPF_LONG || extract32(m4, 0, 3)) {
2761        gen_program_exception(s, PGM_SPECIFICATION);
2762        return DISAS_NORETURN;
2763    }
2764
2765    if (extract32(m4, 3, 1)) {
2766        fn = gen_helper_gvec_vfsq64s;
2767    }
2768    gen_gvec_2_ptr(get_field(s, v1), get_field(s, v2), cpu_env,
2769                   0, fn);
2770    return DISAS_NEXT;
2771}
2772
2773static DisasJumpType op_vftci(DisasContext *s, DisasOps *o)
2774{
2775    const uint16_t i3 = get_field(s, i3);
2776    const uint8_t fpf = get_field(s, m4);
2777    const uint8_t m5 = get_field(s, m5);
2778    gen_helper_gvec_2_ptr *fn = gen_helper_gvec_vftci64;
2779
2780    if (fpf != FPF_LONG || extract32(m5, 0, 3)) {
2781        gen_program_exception(s, PGM_SPECIFICATION);
2782        return DISAS_NORETURN;
2783    }
2784
2785    if (extract32(m5, 3, 1)) {
2786        fn = gen_helper_gvec_vftci64s;
2787    }
2788    gen_gvec_2_ptr(get_field(s, v1), get_field(s, v2), cpu_env, i3, fn);
2789    set_cc_static(s);
2790    return DISAS_NEXT;
2791}
2792