/*
 * ARM SVE Operations
 *
 * Copyright (c) 2018 Linaro, Ltd.
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
 */

#include "qemu/osdep.h"
#include "cpu.h"
#include "internals.h"
#include "exec/exec-all.h"
#include "exec/helper-proto.h"
#include "tcg/tcg-gvec-desc.h"
#include "fpu/softfloat.h"
#include "tcg/tcg.h"
#include "vec_internal.h"
#include "sve_ldst_internal.h"
#include "hw/core/tcg-cpu-ops.h"


/* Return a value for NZCV as per the ARM PredTest pseudofunction.
 *
 * The return value has bit 31 set if N is set, bit 1 set if Z is clear,
 * and bit 0 set if C is set.  Compare the definitions of these variables
 * within CPUARMState.
 */

/* For no G bits set, NZCV = C.  */
#define PREDTEST_INIT  1

/* This is an iterative function, called for each Pd and Pg word
 * moving forward.
 */
static uint32_t iter_predtest_fwd(uint64_t d, uint64_t g, uint32_t flags)
{
    if (likely(g)) {
        /* Compute N from first D & G.
           Use bit 2 to signal first G bit seen.  */
        if (!(flags & 4)) {
            flags |= ((d & (g & -g)) != 0) << 31;
            flags |= 4;
        }

        /* Accumulate Z from each D & G.  */
        flags |= ((d & g) != 0) << 1;

        /* Compute C from last !(D & G).  Replace previous.  */
        flags = deposit32(flags, 0, 1, (d & pow2floor(g)) == 0);
    }
    return flags;
}

/* This is an iterative function, called for each Pd and Pg word
 * moving backward.
 */
static uint32_t iter_predtest_bwd(uint64_t d, uint64_t g, uint32_t flags)
{
    if (likely(g)) {
        /* Compute C from first (i.e last) !(D & G).
           Use bit 2 to signal first G bit seen.  */
        if (!(flags & 4)) {
            flags += 4 - 1; /* add bit 2, subtract C from PREDTEST_INIT */
            flags |= (d & pow2floor(g)) == 0;
        }

        /* Accumulate Z from each D & G.  */
        flags |= ((d & g) != 0) << 1;

        /* Compute N from last (i.e first) D & G.  Replace previous.  */
        flags = deposit32(flags, 31, 1, (d & (g & -g)) != 0);
    }
    return flags;
}

/* The same for a single word predicate.  */
uint32_t HELPER(sve_predtest1)(uint64_t d, uint64_t g)
{
    return iter_predtest_fwd(d, g, PREDTEST_INIT);
}

/* The same for a multi-word predicate.  */
uint32_t HELPER(sve_predtest)(void *vd, void *vg, uint32_t words)
{
    uint32_t flags = PREDTEST_INIT;
    uint64_t *d = vd, *g = vg;
    uintptr_t i = 0;

    do {
        flags = iter_predtest_fwd(d[i], g[i], flags);
    } while (++i < words);

    return flags;
}

/* Similarly for single word elements.  */
static inline uint64_t expand_pred_s(uint8_t byte)
{
    static const uint64_t word[] = {
        [0x01] = 0x00000000ffffffffull,
        [0x10] = 0xffffffff00000000ull,
        [0x11] = 0xffffffffffffffffull,
    };
    return word[byte & 0x11];
}

#define LOGICAL_PPPP(NAME, FUNC) \
void HELPER(NAME)(void *vd, void *vn, void *vm, void *vg, uint32_t desc)  \
{                                                                         \
    uintptr_t opr_sz = simd_oprsz(desc);                                  \
    uint64_t *d = vd, *n = vn, *m = vm, *g = vg;                          \
    uintptr_t i;                                                          \
    for (i = 0; i < opr_sz / 8; ++i) {                                    \
        d[i] = FUNC(n[i], m[i], g[i]);                                    \
    }                                                                     \
}

#define DO_AND(N, M, G)  (((N) & (M)) & (G))
#define DO_BIC(N, M, G)  (((N) & ~(M)) & (G))
#define DO_EOR(N, M, G)  (((N) ^ (M)) & (G))
#define DO_ORR(N, M, G)  (((N) | (M)) & (G))
#define DO_ORN(N, M, G)  (((N) | ~(M)) & (G))
#define DO_NOR(N, M, G)  (~((N) | (M)) & (G))
#define DO_NAND(N, M, G) (~((N) & (M)) & (G))
#define DO_SEL(N, M, G)  (((N) & (G)) | ((M) & ~(G)))

LOGICAL_PPPP(sve_and_pppp, DO_AND)
LOGICAL_PPPP(sve_bic_pppp, DO_BIC)
LOGICAL_PPPP(sve_eor_pppp, DO_EOR)
LOGICAL_PPPP(sve_sel_pppp, DO_SEL)
LOGICAL_PPPP(sve_orr_pppp, DO_ORR)
LOGICAL_PPPP(sve_orn_pppp, DO_ORN)
LOGICAL_PPPP(sve_nor_pppp, DO_NOR)
LOGICAL_PPPP(sve_nand_pppp, DO_NAND)

#undef DO_AND
#undef DO_BIC
#undef DO_EOR
#undef DO_ORR
#undef DO_ORN
#undef DO_NOR
#undef DO_NAND
#undef DO_SEL
#undef LOGICAL_PPPP

/* Fully general three-operand expander, controlled by a predicate.
 * This is complicated by the host-endian storage of the register file.
 */
/* ??? I don't expect the compiler could ever vectorize this itself.
 * With some tables we can convert bit masks to byte masks, and with
 * extra care wrt byte/word ordering we could use gcc generic vectors
 * and do 16 bytes at a time.
 */
#define DO_ZPZZ(NAME, TYPE, H, OP)                                       \
void HELPER(NAME)(void *vd, void *vn, void *vm, void *vg, uint32_t desc) \
{                                                                       \
    intptr_t i, opr_sz = simd_oprsz(desc);                              \
    for (i = 0; i < opr_sz; ) {                                         \
        uint16_t pg = *(uint16_t *)(vg + H1_2(i >> 3));                 \
        do {                                                            \
            if (pg & 1) {                                               \
                TYPE nn = *(TYPE *)(vn + H(i));                         \
                TYPE mm = *(TYPE *)(vm + H(i));                         \
                *(TYPE *)(vd + H(i)) = OP(nn, mm);                      \
            }                                                           \
            i += sizeof(TYPE), pg >>= sizeof(TYPE);                     \
        } while (i & 15);                                               \
    }                                                                   \
}

/* Similarly, specialized for 64-bit operands.  */
#define DO_ZPZZ_D(NAME, TYPE, OP)                                \
void HELPER(NAME)(void *vd, void *vn, void *vm, void *vg, uint32_t desc) \
{                                                               \
    intptr_t i, opr_sz = simd_oprsz(desc) / 8;                  \
    TYPE *d = vd, *n = vn, *m = vm;                             \
    uint8_t *pg = vg;                                           \
    for (i = 0; i < opr_sz; i += 1) {                           \
        if (pg[H1(i)] & 1) {                                    \
            TYPE nn = n[i], mm = m[i];                          \
            d[i] = OP(nn, mm);                                  \
        }                                                       \
    }                                                           \
}

#define DO_AND(N, M)  (N & M)
#define DO_EOR(N, M)  (N ^ M)
#define DO_ORR(N, M)  (N | M)
#define DO_BIC(N, M)  (N & ~M)
#define DO_ADD(N, M)  (N + M)
#define DO_SUB(N, M)  (N - M)
#define DO_MAX(N, M)  ((N) >= (M) ? (N) : (M))
#define DO_MIN(N, M)  ((N) >= (M) ? (M) : (N))
#define DO_ABD(N, M)  ((N) >= (M) ? (N) - (M) : (M) - (N))
#define DO_MUL(N, M)  (N * M)


/*
 * We must avoid the C undefined behaviour cases: division by
 * zero and signed division of INT_MIN by -1. Both of these
 * have architecturally defined required results for Arm.
 * We special case all signed divisions by -1 to avoid having
 * to deduce the minimum integer for the type involved.
 */
#define DO_SDIV(N, M) (unlikely(M == 0) ? 0 : unlikely(M == -1) ? -N : N / M)
#define DO_UDIV(N, M) (unlikely(M == 0) ? 0 : N / M)

DO_ZPZZ(sve_and_zpzz_b, uint8_t, H1, DO_AND)
DO_ZPZZ(sve_and_zpzz_h, uint16_t, H1_2, DO_AND)
DO_ZPZZ(sve_and_zpzz_s, uint32_t, H1_4, DO_AND)
DO_ZPZZ_D(sve_and_zpzz_d, uint64_t, DO_AND)

DO_ZPZZ(sve_orr_zpzz_b, uint8_t, H1, DO_ORR)
DO_ZPZZ(sve_orr_zpzz_h, uint16_t, H1_2, DO_ORR)
DO_ZPZZ(sve_orr_zpzz_s, uint32_t, H1_4, DO_ORR)
DO_ZPZZ_D(sve_orr_zpzz_d, uint64_t, DO_ORR)

DO_ZPZZ(sve_eor_zpzz_b, uint8_t, H1, DO_EOR)
DO_ZPZZ(sve_eor_zpzz_h, uint16_t, H1_2, DO_EOR)
DO_ZPZZ(sve_eor_zpzz_s, uint32_t, H1_4, DO_EOR)
DO_ZPZZ_D(sve_eor_zpzz_d, uint64_t, DO_EOR)

DO_ZPZZ(sve_bic_zpzz_b, uint8_t, H1, DO_BIC)
DO_ZPZZ(sve_bic_zpzz_h, uint16_t, H1_2, DO_BIC)
DO_ZPZZ(sve_bic_zpzz_s, uint32_t, H1_4, DO_BIC)
DO_ZPZZ_D(sve_bic_zpzz_d, uint64_t, DO_BIC)

DO_ZPZZ(sve_add_zpzz_b, uint8_t, H1, DO_ADD)
DO_ZPZZ(sve_add_zpzz_h, uint16_t, H1_2, DO_ADD)
DO_ZPZZ(sve_add_zpzz_s, uint32_t, H1_4, DO_ADD)
DO_ZPZZ_D(sve_add_zpzz_d, uint64_t, DO_ADD)

DO_ZPZZ(sve_sub_zpzz_b, uint8_t, H1, DO_SUB)
DO_ZPZZ(sve_sub_zpzz_h, uint16_t, H1_2, DO_SUB)
DO_ZPZZ(sve_sub_zpzz_s, uint32_t, H1_4, DO_SUB)
DO_ZPZZ_D(sve_sub_zpzz_d, uint64_t, DO_SUB)

DO_ZPZZ(sve_smax_zpzz_b, int8_t, H1, DO_MAX)
DO_ZPZZ(sve_smax_zpzz_h, int16_t, H1_2, DO_MAX)
DO_ZPZZ(sve_smax_zpzz_s, int32_t, H1_4, DO_MAX)
DO_ZPZZ_D(sve_smax_zpzz_d, int64_t, DO_MAX)

DO_ZPZZ(sve_umax_zpzz_b, uint8_t, H1, DO_MAX)
DO_ZPZZ(sve_umax_zpzz_h, uint16_t, H1_2, DO_MAX)
DO_ZPZZ(sve_umax_zpzz_s, uint32_t, H1_4, DO_MAX)
DO_ZPZZ_D(sve_umax_zpzz_d, uint64_t, DO_MAX)

DO_ZPZZ(sve_smin_zpzz_b, int8_t,  H1, DO_MIN)
DO_ZPZZ(sve_smin_zpzz_h, int16_t,  H1_2, DO_MIN)
DO_ZPZZ(sve_smin_zpzz_s, int32_t,  H1_4, DO_MIN)
DO_ZPZZ_D(sve_smin_zpzz_d, int64_t,  DO_MIN)

DO_ZPZZ(sve_umin_zpzz_b, uint8_t, H1, DO_MIN)
DO_ZPZZ(sve_umin_zpzz_h, uint16_t, H1_2, DO_MIN)
DO_ZPZZ(sve_umin_zpzz_s, uint32_t, H1_4, DO_MIN)
DO_ZPZZ_D(sve_umin_zpzz_d, uint64_t, DO_MIN)

DO_ZPZZ(sve_sabd_zpzz_b, int8_t,  H1, DO_ABD)
DO_ZPZZ(sve_sabd_zpzz_h, int16_t,  H1_2, DO_ABD)
DO_ZPZZ(sve_sabd_zpzz_s, int32_t,  H1_4, DO_ABD)
DO_ZPZZ_D(sve_sabd_zpzz_d, int64_t,  DO_ABD)

DO_ZPZZ(sve_uabd_zpzz_b, uint8_t, H1, DO_ABD)
DO_ZPZZ(sve_uabd_zpzz_h, uint16_t, H1_2, DO_ABD)
DO_ZPZZ(sve_uabd_zpzz_s, uint32_t, H1_4, DO_ABD)
DO_ZPZZ_D(sve_uabd_zpzz_d, uint64_t, DO_ABD)

/* Because the computation type is at least twice as large as required,
   these work for both signed and unsigned source types.  */
static inline uint8_t do_mulh_b(int32_t n, int32_t m)
{
    return (n * m) >> 8;
}

static inline uint16_t do_mulh_h(int32_t n, int32_t m)
{
    return (n * m) >> 16;
}

static inline uint32_t do_mulh_s(int64_t n, int64_t m)
{
    return (n * m) >> 32;
}

static inline uint64_t do_smulh_d(uint64_t n, uint64_t m)
{
    uint64_t lo, hi;
    muls64(&lo, &hi, n, m);
    return hi;
}

static inline uint64_t do_umulh_d(uint64_t n, uint64_t m)
{
    uint64_t lo, hi;
    mulu64(&lo, &hi, n, m);
    return hi;
}

DO_ZPZZ(sve_mul_zpzz_b, uint8_t, H1, DO_MUL)
DO_ZPZZ(sve_mul_zpzz_h, uint16_t, H1_2, DO_MUL)
DO_ZPZZ(sve_mul_zpzz_s, uint32_t, H1_4, DO_MUL)
DO_ZPZZ_D(sve_mul_zpzz_d, uint64_t, DO_MUL)

DO_ZPZZ(sve_smulh_zpzz_b, int8_t, H1, do_mulh_b)
DO_ZPZZ(sve_smulh_zpzz_h, int16_t, H1_2, do_mulh_h)
DO_ZPZZ(sve_smulh_zpzz_s, int32_t, H1_4, do_mulh_s)
DO_ZPZZ_D(sve_smulh_zpzz_d, uint64_t, do_smulh_d)

DO_ZPZZ(sve_umulh_zpzz_b, uint8_t, H1, do_mulh_b)
DO_ZPZZ(sve_umulh_zpzz_h, uint16_t, H1_2, do_mulh_h)
DO_ZPZZ(sve_umulh_zpzz_s, uint32_t, H1_4, do_mulh_s)
DO_ZPZZ_D(sve_umulh_zpzz_d, uint64_t, do_umulh_d)

DO_ZPZZ(sve_sdiv_zpzz_s, int32_t, H1_4, DO_SDIV)
DO_ZPZZ_D(sve_sdiv_zpzz_d, int64_t, DO_SDIV)

DO_ZPZZ(sve_udiv_zpzz_s, uint32_t, H1_4, DO_UDIV)
DO_ZPZZ_D(sve_udiv_zpzz_d, uint64_t, DO_UDIV)

/* Note that all bits of the shift are significant
   and not modulo the element size.  */
#define DO_ASR(N, M)  (N >> MIN(M, sizeof(N) * 8 - 1))
#define DO_LSR(N, M)  (M < sizeof(N) * 8 ? N >> M : 0)
#define DO_LSL(N, M)  (M < sizeof(N) * 8 ? N << M : 0)

DO_ZPZZ(sve_asr_zpzz_b, int8_t, H1, DO_ASR)
DO_ZPZZ(sve_lsr_zpzz_b, uint8_t, H1_2, DO_LSR)
DO_ZPZZ(sve_lsl_zpzz_b, uint8_t, H1_4, DO_LSL)

DO_ZPZZ(sve_asr_zpzz_h, int16_t, H1, DO_ASR)
DO_ZPZZ(sve_lsr_zpzz_h, uint16_t, H1_2, DO_LSR)
DO_ZPZZ(sve_lsl_zpzz_h, uint16_t, H1_4, DO_LSL)

DO_ZPZZ(sve_asr_zpzz_s, int32_t, H1, DO_ASR)
DO_ZPZZ(sve_lsr_zpzz_s, uint32_t, H1_2, DO_LSR)
DO_ZPZZ(sve_lsl_zpzz_s, uint32_t, H1_4, DO_LSL)

DO_ZPZZ_D(sve_asr_zpzz_d, int64_t, DO_ASR)
DO_ZPZZ_D(sve_lsr_zpzz_d, uint64_t, DO_LSR)
DO_ZPZZ_D(sve_lsl_zpzz_d, uint64_t, DO_LSL)

static inline uint16_t do_sadalp_h(int16_t n, int16_t m)
{
    int8_t n1 = n, n2 = n >> 8;
    return m + n1 + n2;
}

static inline uint32_t do_sadalp_s(int32_t n, int32_t m)
{
    int16_t n1 = n, n2 = n >> 16;
    return m + n1 + n2;
}

static inline uint64_t do_sadalp_d(int64_t n, int64_t m)
{
    int32_t n1 = n, n2 = n >> 32;
    return m + n1 + n2;
}

DO_ZPZZ(sve2_sadalp_zpzz_h, int16_t, H1_2, do_sadalp_h)
DO_ZPZZ(sve2_sadalp_zpzz_s, int32_t, H1_4, do_sadalp_s)
DO_ZPZZ_D(sve2_sadalp_zpzz_d, int64_t, do_sadalp_d)

static inline uint16_t do_uadalp_h(uint16_t n, uint16_t m)
{
    uint8_t n1 = n, n2 = n >> 8;
    return m + n1 + n2;
}

static inline uint32_t do_uadalp_s(uint32_t n, uint32_t m)
{
    uint16_t n1 = n, n2 = n >> 16;
    return m + n1 + n2;
}

static inline uint64_t do_uadalp_d(uint64_t n, uint64_t m)
{
    uint32_t n1 = n, n2 = n >> 32;
    return m + n1 + n2;
}

DO_ZPZZ(sve2_uadalp_zpzz_h, uint16_t, H1_2, do_uadalp_h)
DO_ZPZZ(sve2_uadalp_zpzz_s, uint32_t, H1_4, do_uadalp_s)
DO_ZPZZ_D(sve2_uadalp_zpzz_d, uint64_t, do_uadalp_d)

#define do_srshl_b(n, m)  do_sqrshl_bhs(n, m, 8, true, NULL)
#define do_srshl_h(n, m)  do_sqrshl_bhs(n, m, 16, true, NULL)
#define do_srshl_s(n, m)  do_sqrshl_bhs(n, m, 32, true, NULL)
#define do_srshl_d(n, m)  do_sqrshl_d(n, m, true, NULL)

DO_ZPZZ(sve2_srshl_zpzz_b, int8_t, H1, do_srshl_b)
DO_ZPZZ(sve2_srshl_zpzz_h, int16_t, H1_2, do_srshl_h)
DO_ZPZZ(sve2_srshl_zpzz_s, int32_t, H1_4, do_srshl_s)
DO_ZPZZ_D(sve2_srshl_zpzz_d, int64_t, do_srshl_d)

#define do_urshl_b(n, m)  do_uqrshl_bhs(n, (int8_t)m, 8, true, NULL)
#define do_urshl_h(n, m)  do_uqrshl_bhs(n, (int16_t)m, 16, true, NULL)
#define do_urshl_s(n, m)  do_uqrshl_bhs(n, m, 32, true, NULL)
#define do_urshl_d(n, m)  do_uqrshl_d(n, m, true, NULL)

DO_ZPZZ(sve2_urshl_zpzz_b, uint8_t, H1, do_urshl_b)
DO_ZPZZ(sve2_urshl_zpzz_h, uint16_t, H1_2, do_urshl_h)
DO_ZPZZ(sve2_urshl_zpzz_s, uint32_t, H1_4, do_urshl_s)
DO_ZPZZ_D(sve2_urshl_zpzz_d, uint64_t, do_urshl_d)

/*
 * Unlike the NEON and AdvSIMD versions, there is no QC bit to set.
 * We pass in a pointer to a dummy saturation field to trigger
 * the saturating arithmetic but discard the information about
 * whether it has occurred.
 */
#define do_sqshl_b(n, m) \
   ({ uint32_t discard; do_sqrshl_bhs(n, m, 8, false, &discard); })
#define do_sqshl_h(n, m) \
   ({ uint32_t discard; do_sqrshl_bhs(n, m, 16, false, &discard); })
#define do_sqshl_s(n, m) \
   ({ uint32_t discard; do_sqrshl_bhs(n, m, 32, false, &discard); })
#define do_sqshl_d(n, m) \
   ({ uint32_t discard; do_sqrshl_d(n, m, false, &discard); })

DO_ZPZZ(sve2_sqshl_zpzz_b, int8_t, H1_2, do_sqshl_b)
DO_ZPZZ(sve2_sqshl_zpzz_h, int16_t, H1_2, do_sqshl_h)
DO_ZPZZ(sve2_sqshl_zpzz_s, int32_t, H1_4, do_sqshl_s)
DO_ZPZZ_D(sve2_sqshl_zpzz_d, int64_t, do_sqshl_d)

#define do_uqshl_b(n, m) \
   ({ uint32_t discard; do_uqrshl_bhs(n, (int8_t)m, 8, false, &discard); })
#define do_uqshl_h(n, m) \
   ({ uint32_t discard; do_uqrshl_bhs(n, (int16_t)m, 16, false, &discard); })
#define do_uqshl_s(n, m) \
   ({ uint32_t discard; do_uqrshl_bhs(n, m, 32, false, &discard); })
#define do_uqshl_d(n, m) \
   ({ uint32_t discard; do_uqrshl_d(n, m, false, &discard); })

DO_ZPZZ(sve2_uqshl_zpzz_b, uint8_t, H1_2, do_uqshl_b)
DO_ZPZZ(sve2_uqshl_zpzz_h, uint16_t, H1_2, do_uqshl_h)
DO_ZPZZ(sve2_uqshl_zpzz_s, uint32_t, H1_4, do_uqshl_s)
DO_ZPZZ_D(sve2_uqshl_zpzz_d, uint64_t, do_uqshl_d)

#define do_sqrshl_b(n, m) \
   ({ uint32_t discard; do_sqrshl_bhs(n, m, 8, true, &discard); })
#define do_sqrshl_h(n, m) \
   ({ uint32_t discard; do_sqrshl_bhs(n, m, 16, true, &discard); })
#define do_sqrshl_s(n, m) \
   ({ uint32_t discard; do_sqrshl_bhs(n, m, 32, true, &discard); })
#define do_sqrshl_d(n, m) \
   ({ uint32_t discard; do_sqrshl_d(n, m, true, &discard); })

DO_ZPZZ(sve2_sqrshl_zpzz_b, int8_t, H1_2, do_sqrshl_b)
DO_ZPZZ(sve2_sqrshl_zpzz_h, int16_t, H1_2, do_sqrshl_h)
DO_ZPZZ(sve2_sqrshl_zpzz_s, int32_t, H1_4, do_sqrshl_s)
DO_ZPZZ_D(sve2_sqrshl_zpzz_d, int64_t, do_sqrshl_d)

#undef do_sqrshl_d

#define do_uqrshl_b(n, m) \
   ({ uint32_t discard; do_uqrshl_bhs(n, (int8_t)m, 8, true, &discard); })
#define do_uqrshl_h(n, m) \
   ({ uint32_t discard; do_uqrshl_bhs(n, (int16_t)m, 16, true, &discard); })
#define do_uqrshl_s(n, m) \
   ({ uint32_t discard; do_uqrshl_bhs(n, m, 32, true, &discard); })
#define do_uqrshl_d(n, m) \
   ({ uint32_t discard; do_uqrshl_d(n, m, true, &discard); })

DO_ZPZZ(sve2_uqrshl_zpzz_b, uint8_t, H1_2, do_uqrshl_b)
DO_ZPZZ(sve2_uqrshl_zpzz_h, uint16_t, H1_2, do_uqrshl_h)
DO_ZPZZ(sve2_uqrshl_zpzz_s, uint32_t, H1_4, do_uqrshl_s)
DO_ZPZZ_D(sve2_uqrshl_zpzz_d, uint64_t, do_uqrshl_d)

#undef do_uqrshl_d

#define DO_HADD_BHS(n, m)  (((int64_t)n + m) >> 1)
#define DO_HADD_D(n, m)    ((n >> 1) + (m >> 1) + (n & m & 1))

DO_ZPZZ(sve2_shadd_zpzz_b, int8_t, H1, DO_HADD_BHS)
DO_ZPZZ(sve2_shadd_zpzz_h, int16_t, H1_2, DO_HADD_BHS)
DO_ZPZZ(sve2_shadd_zpzz_s, int32_t, H1_4, DO_HADD_BHS)
DO_ZPZZ_D(sve2_shadd_zpzz_d, int64_t, DO_HADD_D)

DO_ZPZZ(sve2_uhadd_zpzz_b, uint8_t, H1, DO_HADD_BHS)
DO_ZPZZ(sve2_uhadd_zpzz_h, uint16_t, H1_2, DO_HADD_BHS)
DO_ZPZZ(sve2_uhadd_zpzz_s, uint32_t, H1_4, DO_HADD_BHS)
DO_ZPZZ_D(sve2_uhadd_zpzz_d, uint64_t, DO_HADD_D)

#define DO_RHADD_BHS(n, m)  (((int64_t)n + m + 1) >> 1)
#define DO_RHADD_D(n, m)    ((n >> 1) + (m >> 1) + ((n | m) & 1))

DO_ZPZZ(sve2_srhadd_zpzz_b, int8_t, H1, DO_RHADD_BHS)
DO_ZPZZ(sve2_srhadd_zpzz_h, int16_t, H1_2, DO_RHADD_BHS)
DO_ZPZZ(sve2_srhadd_zpzz_s, int32_t, H1_4, DO_RHADD_BHS)
DO_ZPZZ_D(sve2_srhadd_zpzz_d, int64_t, DO_RHADD_D)

DO_ZPZZ(sve2_urhadd_zpzz_b, uint8_t, H1, DO_RHADD_BHS)
DO_ZPZZ(sve2_urhadd_zpzz_h, uint16_t, H1_2, DO_RHADD_BHS)
DO_ZPZZ(sve2_urhadd_zpzz_s, uint32_t, H1_4, DO_RHADD_BHS)
DO_ZPZZ_D(sve2_urhadd_zpzz_d, uint64_t, DO_RHADD_D)

#define DO_HSUB_BHS(n, m)  (((int64_t)n - m) >> 1)
#define DO_HSUB_D(n, m)    ((n >> 1) - (m >> 1) - (~n & m & 1))

DO_ZPZZ(sve2_shsub_zpzz_b, int8_t, H1, DO_HSUB_BHS)
DO_ZPZZ(sve2_shsub_zpzz_h, int16_t, H1_2, DO_HSUB_BHS)
DO_ZPZZ(sve2_shsub_zpzz_s, int32_t, H1_4, DO_HSUB_BHS)
DO_ZPZZ_D(sve2_shsub_zpzz_d, int64_t, DO_HSUB_D)

DO_ZPZZ(sve2_uhsub_zpzz_b, uint8_t, H1, DO_HSUB_BHS)
DO_ZPZZ(sve2_uhsub_zpzz_h, uint16_t, H1_2, DO_HSUB_BHS)
DO_ZPZZ(sve2_uhsub_zpzz_s, uint32_t, H1_4, DO_HSUB_BHS)
DO_ZPZZ_D(sve2_uhsub_zpzz_d, uint64_t, DO_HSUB_D)

static inline int32_t do_sat_bhs(int64_t val, int64_t min, int64_t max)
{
    return val >= max ? max : val <= min ? min : val;
}

#define DO_SQADD_B(n, m) do_sat_bhs((int64_t)n + m, INT8_MIN, INT8_MAX)
#define DO_SQADD_H(n, m) do_sat_bhs((int64_t)n + m, INT16_MIN, INT16_MAX)
#define DO_SQADD_S(n, m) do_sat_bhs((int64_t)n + m, INT32_MIN, INT32_MAX)

static inline int64_t do_sqadd_d(int64_t n, int64_t m)
{
    int64_t r = n + m;
    if (((r ^ n) & ~(n ^ m)) < 0) {
        /* Signed overflow.  */
        return r < 0 ? INT64_MAX : INT64_MIN;
    }
    return r;
}

DO_ZPZZ(sve2_sqadd_zpzz_b, int8_t, H1, DO_SQADD_B)
DO_ZPZZ(sve2_sqadd_zpzz_h, int16_t, H1_2, DO_SQADD_H)
DO_ZPZZ(sve2_sqadd_zpzz_s, int32_t, H1_4, DO_SQADD_S)
DO_ZPZZ_D(sve2_sqadd_zpzz_d, int64_t, do_sqadd_d)

#define DO_UQADD_B(n, m) do_sat_bhs((int64_t)n + m, 0, UINT8_MAX)
#define DO_UQADD_H(n, m) do_sat_bhs((int64_t)n + m, 0, UINT16_MAX)
#define DO_UQADD_S(n, m) do_sat_bhs((int64_t)n + m, 0, UINT32_MAX)

static inline uint64_t do_uqadd_d(uint64_t n, uint64_t m)
{
    uint64_t r = n + m;
    return r < n ? UINT64_MAX : r;
}

DO_ZPZZ(sve2_uqadd_zpzz_b, uint8_t, H1, DO_UQADD_B)
DO_ZPZZ(sve2_uqadd_zpzz_h, uint16_t, H1_2, DO_UQADD_H)
DO_ZPZZ(sve2_uqadd_zpzz_s, uint32_t, H1_4, DO_UQADD_S)
DO_ZPZZ_D(sve2_uqadd_zpzz_d, uint64_t, do_uqadd_d)

#define DO_SQSUB_B(n, m) do_sat_bhs((int64_t)n - m, INT8_MIN, INT8_MAX)
#define DO_SQSUB_H(n, m) do_sat_bhs((int64_t)n - m, INT16_MIN, INT16_MAX)
#define DO_SQSUB_S(n, m) do_sat_bhs((int64_t)n - m, INT32_MIN, INT32_MAX)

static inline int64_t do_sqsub_d(int64_t n, int64_t m)
{
    int64_t r = n - m;
    if (((r ^ n) & (n ^ m)) < 0) {
        /* Signed overflow.  */
        return r < 0 ? INT64_MAX : INT64_MIN;
    }
    return r;
}

DO_ZPZZ(sve2_sqsub_zpzz_b, int8_t, H1, DO_SQSUB_B)
DO_ZPZZ(sve2_sqsub_zpzz_h, int16_t, H1_2, DO_SQSUB_H)
DO_ZPZZ(sve2_sqsub_zpzz_s, int32_t, H1_4, DO_SQSUB_S)
DO_ZPZZ_D(sve2_sqsub_zpzz_d, int64_t, do_sqsub_d)

#define DO_UQSUB_B(n, m) do_sat_bhs((int64_t)n - m, 0, UINT8_MAX)
#define DO_UQSUB_H(n, m) do_sat_bhs((int64_t)n - m, 0, UINT16_MAX)
#define DO_UQSUB_S(n, m) do_sat_bhs((int64_t)n - m, 0, UINT32_MAX)

static inline uint64_t do_uqsub_d(uint64_t n, uint64_t m)
{
    return n > m ? n - m : 0;
}

DO_ZPZZ(sve2_uqsub_zpzz_b, uint8_t, H1, DO_UQSUB_B)
DO_ZPZZ(sve2_uqsub_zpzz_h, uint16_t, H1_2, DO_UQSUB_H)
DO_ZPZZ(sve2_uqsub_zpzz_s, uint32_t, H1_4, DO_UQSUB_S)
DO_ZPZZ_D(sve2_uqsub_zpzz_d, uint64_t, do_uqsub_d)

#define DO_SUQADD_B(n, m) \
    do_sat_bhs((int64_t)(int8_t)n + m, INT8_MIN, INT8_MAX)
#define DO_SUQADD_H(n, m) \
    do_sat_bhs((int64_t)(int16_t)n + m, INT16_MIN, INT16_MAX)
#define DO_SUQADD_S(n, m) \
    do_sat_bhs((int64_t)(int32_t)n + m, INT32_MIN, INT32_MAX)

static inline int64_t do_suqadd_d(int64_t n, uint64_t m)
{
    uint64_t r = n + m;

    if (n < 0) {
        /* Note that m - abs(n) cannot underflow. */
        if (r > INT64_MAX) {
            /* Result is either very large positive or negative. */
            if (m > -n) {
                /* m > abs(n), so r is a very large positive. */
                return INT64_MAX;
            }
            /* Result is negative. */
        }
    } else {
        /* Both inputs are positive: check for overflow.  */
        if (r < m || r > INT64_MAX) {
            return INT64_MAX;
        }
    }
    return r;
}

DO_ZPZZ(sve2_suqadd_zpzz_b, uint8_t, H1, DO_SUQADD_B)
DO_ZPZZ(sve2_suqadd_zpzz_h, uint16_t, H1_2, DO_SUQADD_H)
DO_ZPZZ(sve2_suqadd_zpzz_s, uint32_t, H1_4, DO_SUQADD_S)
DO_ZPZZ_D(sve2_suqadd_zpzz_d, uint64_t, do_suqadd_d)

#define DO_USQADD_B(n, m) \
    do_sat_bhs((int64_t)n + (int8_t)m, 0, UINT8_MAX)
#define DO_USQADD_H(n, m) \
    do_sat_bhs((int64_t)n + (int16_t)m, 0, UINT16_MAX)
#define DO_USQADD_S(n, m) \
    do_sat_bhs((int64_t)n + (int32_t)m, 0, UINT32_MAX)

static inline uint64_t do_usqadd_d(uint64_t n, int64_t m)
{
    uint64_t r = n + m;

    if (m < 0) {
        return n < -m ? 0 : r;
    }
    return r < n ? UINT64_MAX : r;
}

DO_ZPZZ(sve2_usqadd_zpzz_b, uint8_t, H1, DO_USQADD_B)
DO_ZPZZ(sve2_usqadd_zpzz_h, uint16_t, H1_2, DO_USQADD_H)
DO_ZPZZ(sve2_usqadd_zpzz_s, uint32_t, H1_4, DO_USQADD_S)
DO_ZPZZ_D(sve2_usqadd_zpzz_d, uint64_t, do_usqadd_d)

#undef DO_ZPZZ
#undef DO_ZPZZ_D

/*
 * Three operand expander, operating on element pairs.
 * If the slot I is even, the elements from from VN {I, I+1}.
 * If the slot I is odd, the elements from from VM {I-1, I}.
 * Load all of the input elements in each pair before overwriting output.
 */
#define DO_ZPZZ_PAIR(NAME, TYPE, H, OP) \
void HELPER(NAME)(void *vd, void *vn, void *vm, void *vg, uint32_t desc) \
{                                                               \
    intptr_t i, opr_sz = simd_oprsz(desc);                      \
    for (i = 0; i < opr_sz; ) {                                 \
        uint16_t pg = *(uint16_t *)(vg + H1_2(i >> 3));         \
        do {                                                    \
            TYPE n0 = *(TYPE *)(vn + H(i));                     \
            TYPE m0 = *(TYPE *)(vm + H(i));                     \
            TYPE n1 = *(TYPE *)(vn + H(i + sizeof(TYPE)));      \
            TYPE m1 = *(TYPE *)(vm + H(i + sizeof(TYPE)));      \
            if (pg & 1) {                                       \
                *(TYPE *)(vd + H(i)) = OP(n0, n1);              \
            }                                                   \
            i += sizeof(TYPE), pg >>= sizeof(TYPE);             \
            if (pg & 1) {                                       \
                *(TYPE *)(vd + H(i)) = OP(m0, m1);              \
            }                                                   \
            i += sizeof(TYPE), pg >>= sizeof(TYPE);             \
        } while (i & 15);                                       \
    }                                                           \
}

/* Similarly, specialized for 64-bit operands.  */
#define DO_ZPZZ_PAIR_D(NAME, TYPE, OP) \
void HELPER(NAME)(void *vd, void *vn, void *vm, void *vg, uint32_t desc) \
{                                                               \
    intptr_t i, opr_sz = simd_oprsz(desc) / 8;                  \
    TYPE *d = vd, *n = vn, *m = vm;                             \
    uint8_t *pg = vg;                                           \
    for (i = 0; i < opr_sz; i += 2) {                           \
        TYPE n0 = n[i], n1 = n[i + 1];                          \
        TYPE m0 = m[i], m1 = m[i + 1];                          \
        if (pg[H1(i)] & 1) {                                    \
            d[i] = OP(n0, n1);                                  \
        }                                                       \
        if (pg[H1(i + 1)] & 1) {                                \
            d[i + 1] = OP(m0, m1);                              \
        }                                                       \
    }                                                           \
}

DO_ZPZZ_PAIR(sve2_addp_zpzz_b, uint8_t, H1, DO_ADD)
DO_ZPZZ_PAIR(sve2_addp_zpzz_h, uint16_t, H1_2, DO_ADD)
DO_ZPZZ_PAIR(sve2_addp_zpzz_s, uint32_t, H1_4, DO_ADD)
DO_ZPZZ_PAIR_D(sve2_addp_zpzz_d, uint64_t, DO_ADD)

DO_ZPZZ_PAIR(sve2_umaxp_zpzz_b, uint8_t, H1, DO_MAX)
DO_ZPZZ_PAIR(sve2_umaxp_zpzz_h, uint16_t, H1_2, DO_MAX)
DO_ZPZZ_PAIR(sve2_umaxp_zpzz_s, uint32_t, H1_4, DO_MAX)
DO_ZPZZ_PAIR_D(sve2_umaxp_zpzz_d, uint64_t, DO_MAX)

DO_ZPZZ_PAIR(sve2_uminp_zpzz_b, uint8_t, H1, DO_MIN)
DO_ZPZZ_PAIR(sve2_uminp_zpzz_h, uint16_t, H1_2, DO_MIN)
DO_ZPZZ_PAIR(sve2_uminp_zpzz_s, uint32_t, H1_4, DO_MIN)
DO_ZPZZ_PAIR_D(sve2_uminp_zpzz_d, uint64_t, DO_MIN)

DO_ZPZZ_PAIR(sve2_smaxp_zpzz_b, int8_t, H1, DO_MAX)
DO_ZPZZ_PAIR(sve2_smaxp_zpzz_h, int16_t, H1_2, DO_MAX)
DO_ZPZZ_PAIR(sve2_smaxp_zpzz_s, int32_t, H1_4, DO_MAX)
DO_ZPZZ_PAIR_D(sve2_smaxp_zpzz_d, int64_t, DO_MAX)

DO_ZPZZ_PAIR(sve2_sminp_zpzz_b, int8_t, H1, DO_MIN)
DO_ZPZZ_PAIR(sve2_sminp_zpzz_h, int16_t, H1_2, DO_MIN)
DO_ZPZZ_PAIR(sve2_sminp_zpzz_s, int32_t, H1_4, DO_MIN)
DO_ZPZZ_PAIR_D(sve2_sminp_zpzz_d, int64_t, DO_MIN)

#undef DO_ZPZZ_PAIR
#undef DO_ZPZZ_PAIR_D

#define DO_ZPZZ_PAIR_FP(NAME, TYPE, H, OP)                              \
void HELPER(NAME)(void *vd, void *vn, void *vm, void *vg,               \
                  void *status, uint32_t desc)                          \
{                                                                       \
    intptr_t i, opr_sz = simd_oprsz(desc);                              \
    for (i = 0; i < opr_sz; ) {                                         \
        uint16_t pg = *(uint16_t *)(vg + H1_2(i >> 3));                 \
        do {                                                            \
            TYPE n0 = *(TYPE *)(vn + H(i));                             \
            TYPE m0 = *(TYPE *)(vm + H(i));                             \
            TYPE n1 = *(TYPE *)(vn + H(i + sizeof(TYPE)));              \
            TYPE m1 = *(TYPE *)(vm + H(i + sizeof(TYPE)));              \
            if (pg & 1) {                                               \
                *(TYPE *)(vd + H(i)) = OP(n0, n1, status);              \
            }                                                           \
            i += sizeof(TYPE), pg >>= sizeof(TYPE);                     \
            if (pg & 1) {                                               \
                *(TYPE *)(vd + H(i)) = OP(m0, m1, status);              \
            }                                                           \
            i += sizeof(TYPE), pg >>= sizeof(TYPE);                     \
        } while (i & 15);                                               \
    }                                                                   \
}

DO_ZPZZ_PAIR_FP(sve2_faddp_zpzz_h, float16, H1_2, float16_add)
DO_ZPZZ_PAIR_FP(sve2_faddp_zpzz_s, float32, H1_4, float32_add)
DO_ZPZZ_PAIR_FP(sve2_faddp_zpzz_d, float64, H1_8, float64_add)

DO_ZPZZ_PAIR_FP(sve2_fmaxnmp_zpzz_h, float16, H1_2, float16_maxnum)
DO_ZPZZ_PAIR_FP(sve2_fmaxnmp_zpzz_s, float32, H1_4, float32_maxnum)
DO_ZPZZ_PAIR_FP(sve2_fmaxnmp_zpzz_d, float64, H1_8, float64_maxnum)

DO_ZPZZ_PAIR_FP(sve2_fminnmp_zpzz_h, float16, H1_2, float16_minnum)
DO_ZPZZ_PAIR_FP(sve2_fminnmp_zpzz_s, float32, H1_4, float32_minnum)
DO_ZPZZ_PAIR_FP(sve2_fminnmp_zpzz_d, float64, H1_8, float64_minnum)

DO_ZPZZ_PAIR_FP(sve2_fmaxp_zpzz_h, float16, H1_2, float16_max)
DO_ZPZZ_PAIR_FP(sve2_fmaxp_zpzz_s, float32, H1_4, float32_max)
DO_ZPZZ_PAIR_FP(sve2_fmaxp_zpzz_d, float64, H1_8, float64_max)

DO_ZPZZ_PAIR_FP(sve2_fminp_zpzz_h, float16, H1_2, float16_min)
DO_ZPZZ_PAIR_FP(sve2_fminp_zpzz_s, float32, H1_4, float32_min)
DO_ZPZZ_PAIR_FP(sve2_fminp_zpzz_d, float64, H1_8, float64_min)

#undef DO_ZPZZ_PAIR_FP

/* Three-operand expander, controlled by a predicate, in which the
 * third operand is "wide".  That is, for D = N op M, the same 64-bit
 * value of M is used with all of the narrower values of N.
 */
#define DO_ZPZW(NAME, TYPE, TYPEW, H, OP)                               \
void HELPER(NAME)(void *vd, void *vn, void *vm, void *vg, uint32_t desc) \
{                                                                       \
    intptr_t i, opr_sz = simd_oprsz(desc);                              \
    for (i = 0; i < opr_sz; ) {                                         \
        uint8_t pg = *(uint8_t *)(vg + H1(i >> 3));                     \
        TYPEW mm = *(TYPEW *)(vm + i);                                  \
        do {                                                            \
            if (pg & 1) {                                               \
                TYPE nn = *(TYPE *)(vn + H(i));                         \
                *(TYPE *)(vd + H(i)) = OP(nn, mm);                      \
            }                                                           \
            i += sizeof(TYPE), pg >>= sizeof(TYPE);                     \
        } while (i & 7);                                                \
    }                                                                   \
}

DO_ZPZW(sve_asr_zpzw_b, int8_t, uint64_t, H1, DO_ASR)
DO_ZPZW(sve_lsr_zpzw_b, uint8_t, uint64_t, H1, DO_LSR)
DO_ZPZW(sve_lsl_zpzw_b, uint8_t, uint64_t, H1, DO_LSL)

DO_ZPZW(sve_asr_zpzw_h, int16_t, uint64_t, H1_2, DO_ASR)
DO_ZPZW(sve_lsr_zpzw_h, uint16_t, uint64_t, H1_2, DO_LSR)
DO_ZPZW(sve_lsl_zpzw_h, uint16_t, uint64_t, H1_2, DO_LSL)

DO_ZPZW(sve_asr_zpzw_s, int32_t, uint64_t, H1_4, DO_ASR)
DO_ZPZW(sve_lsr_zpzw_s, uint32_t, uint64_t, H1_4, DO_LSR)
DO_ZPZW(sve_lsl_zpzw_s, uint32_t, uint64_t, H1_4, DO_LSL)

#undef DO_ZPZW

/* Fully general two-operand expander, controlled by a predicate.
 */
#define DO_ZPZ(NAME, TYPE, H, OP)                               \
void HELPER(NAME)(void *vd, void *vn, void *vg, uint32_t desc)  \
{                                                               \
    intptr_t i, opr_sz = simd_oprsz(desc);                      \
    for (i = 0; i < opr_sz; ) {                                 \
        uint16_t pg = *(uint16_t *)(vg + H1_2(i >> 3));         \
        do {                                                    \
            if (pg & 1) {                                       \
                TYPE nn = *(TYPE *)(vn + H(i));                 \
                *(TYPE *)(vd + H(i)) = OP(nn);                  \
            }                                                   \
            i += sizeof(TYPE), pg >>= sizeof(TYPE);             \
        } while (i & 15);                                       \
    }                                                           \
}

/* Similarly, specialized for 64-bit operands.  */
#define DO_ZPZ_D(NAME, TYPE, OP)                                \
void HELPER(NAME)(void *vd, void *vn, void *vg, uint32_t desc)  \
{                                                               \
    intptr_t i, opr_sz = simd_oprsz(desc) / 8;                  \
    TYPE *d = vd, *n = vn;                                      \
    uint8_t *pg = vg;                                           \
    for (i = 0; i < opr_sz; i += 1) {                           \
        if (pg[H1(i)] & 1) {                                    \
            TYPE nn = n[i];                                     \
            d[i] = OP(nn);                                      \
        }                                                       \
    }                                                           \
}

#define DO_CLS_B(N)   (clrsb32(N) - 24)
#define DO_CLS_H(N)   (clrsb32(N) - 16)

DO_ZPZ(sve_cls_b, int8_t, H1, DO_CLS_B)
DO_ZPZ(sve_cls_h, int16_t, H1_2, DO_CLS_H)
DO_ZPZ(sve_cls_s, int32_t, H1_4, clrsb32)
DO_ZPZ_D(sve_cls_d, int64_t, clrsb64)

#define DO_CLZ_B(N)   (clz32(N) - 24)
#define DO_CLZ_H(N)   (clz32(N) - 16)

DO_ZPZ(sve_clz_b, uint8_t, H1, DO_CLZ_B)
DO_ZPZ(sve_clz_h, uint16_t, H1_2, DO_CLZ_H)
DO_ZPZ(sve_clz_s, uint32_t, H1_4, clz32)
DO_ZPZ_D(sve_clz_d, uint64_t, clz64)

DO_ZPZ(sve_cnt_zpz_b, uint8_t, H1, ctpop8)
DO_ZPZ(sve_cnt_zpz_h, uint16_t, H1_2, ctpop16)
DO_ZPZ(sve_cnt_zpz_s, uint32_t, H1_4, ctpop32)
DO_ZPZ_D(sve_cnt_zpz_d, uint64_t, ctpop64)

#define DO_CNOT(N)    (N == 0)

DO_ZPZ(sve_cnot_b, uint8_t, H1, DO_CNOT)
DO_ZPZ(sve_cnot_h, uint16_t, H1_2, DO_CNOT)
DO_ZPZ(sve_cnot_s, uint32_t, H1_4, DO_CNOT)
DO_ZPZ_D(sve_cnot_d, uint64_t, DO_CNOT)

#define DO_FABS(N)    (N & ((__typeof(N))-1 >> 1))

DO_ZPZ(sve_fabs_h, uint16_t, H1_2, DO_FABS)
DO_ZPZ(sve_fabs_s, uint32_t, H1_4, DO_FABS)
DO_ZPZ_D(sve_fabs_d, uint64_t, DO_FABS)

#define DO_FNEG(N)    (N ^ ~((__typeof(N))-1 >> 1))

DO_ZPZ(sve_fneg_h, uint16_t, H1_2, DO_FNEG)
DO_ZPZ(sve_fneg_s, uint32_t, H1_4, DO_FNEG)
DO_ZPZ_D(sve_fneg_d, uint64_t, DO_FNEG)

#define DO_NOT(N)    (~N)

DO_ZPZ(sve_not_zpz_b, uint8_t, H1, DO_NOT)
DO_ZPZ(sve_not_zpz_h, uint16_t, H1_2, DO_NOT)
DO_ZPZ(sve_not_zpz_s, uint32_t, H1_4, DO_NOT)
DO_ZPZ_D(sve_not_zpz_d, uint64_t, DO_NOT)

#define DO_SXTB(N)    ((int8_t)N)
#define DO_SXTH(N)    ((int16_t)N)
#define DO_SXTS(N)    ((int32_t)N)
#define DO_UXTB(N)    ((uint8_t)N)
#define DO_UXTH(N)    ((uint16_t)N)
#define DO_UXTS(N)    ((uint32_t)N)

DO_ZPZ(sve_sxtb_h, uint16_t, H1_2, DO_SXTB)
DO_ZPZ(sve_sxtb_s, uint32_t, H1_4, DO_SXTB)
DO_ZPZ(sve_sxth_s, uint32_t, H1_4, DO_SXTH)
DO_ZPZ_D(sve_sxtb_d, uint64_t, DO_SXTB)
DO_ZPZ_D(sve_sxth_d, uint64_t, DO_SXTH)
DO_ZPZ_D(sve_sxtw_d, uint64_t, DO_SXTS)

DO_ZPZ(sve_uxtb_h, uint16_t, H1_2, DO_UXTB)
DO_ZPZ(sve_uxtb_s, uint32_t, H1_4, DO_UXTB)
DO_ZPZ(sve_uxth_s, uint32_t, H1_4, DO_UXTH)
DO_ZPZ_D(sve_uxtb_d, uint64_t, DO_UXTB)
DO_ZPZ_D(sve_uxth_d, uint64_t, DO_UXTH)
DO_ZPZ_D(sve_uxtw_d, uint64_t, DO_UXTS)

#define DO_ABS(N)    (N < 0 ? -N : N)

DO_ZPZ(sve_abs_b, int8_t, H1, DO_ABS)
DO_ZPZ(sve_abs_h, int16_t, H1_2, DO_ABS)
DO_ZPZ(sve_abs_s, int32_t, H1_4, DO_ABS)
DO_ZPZ_D(sve_abs_d, int64_t, DO_ABS)

#define DO_NEG(N)    (-N)

DO_ZPZ(sve_neg_b, uint8_t, H1, DO_NEG)
DO_ZPZ(sve_neg_h, uint16_t, H1_2, DO_NEG)
DO_ZPZ(sve_neg_s, uint32_t, H1_4, DO_NEG)
DO_ZPZ_D(sve_neg_d, uint64_t, DO_NEG)

DO_ZPZ(sve_revb_h, uint16_t, H1_2, bswap16)
DO_ZPZ(sve_revb_s, uint32_t, H1_4, bswap32)
DO_ZPZ_D(sve_revb_d, uint64_t, bswap64)

DO_ZPZ(sve_revh_s, uint32_t, H1_4, hswap32)
DO_ZPZ_D(sve_revh_d, uint64_t, hswap64)

DO_ZPZ_D(sve_revw_d, uint64_t, wswap64)

void HELPER(sme_revd_q)(void *vd, void *vn, void *vg, uint32_t desc)
{
    intptr_t i, opr_sz = simd_oprsz(desc) / 8;
    uint64_t *d = vd, *n = vn;
    uint8_t *pg = vg;

    for (i = 0; i < opr_sz; i += 2) {
        if (pg[H1(i)] & 1) {
            uint64_t n0 = n[i + 0];
            uint64_t n1 = n[i + 1];
            d[i + 0] = n1;
            d[i + 1] = n0;
        }
    }
}

DO_ZPZ(sve_rbit_b, uint8_t, H1, revbit8)
DO_ZPZ(sve_rbit_h, uint16_t, H1_2, revbit16)
DO_ZPZ(sve_rbit_s, uint32_t, H1_4, revbit32)
DO_ZPZ_D(sve_rbit_d, uint64_t, revbit64)

#define DO_SQABS(X) \
    ({ __typeof(X) x_ = (X), min_ = 1ull << (sizeof(X) * 8 - 1); \
       x_ >= 0 ? x_ : x_ == min_ ? -min_ - 1 : -x_; })

DO_ZPZ(sve2_sqabs_b, int8_t, H1, DO_SQABS)
DO_ZPZ(sve2_sqabs_h, int16_t, H1_2, DO_SQABS)
DO_ZPZ(sve2_sqabs_s, int32_t, H1_4, DO_SQABS)
DO_ZPZ_D(sve2_sqabs_d, int64_t, DO_SQABS)

#define DO_SQNEG(X) \
    ({ __typeof(X) x_ = (X), min_ = 1ull << (sizeof(X) * 8 - 1); \
       x_ == min_ ? -min_ - 1 : -x_; })

DO_ZPZ(sve2_sqneg_b, uint8_t, H1, DO_SQNEG)
DO_ZPZ(sve2_sqneg_h, uint16_t, H1_2, DO_SQNEG)
DO_ZPZ(sve2_sqneg_s, uint32_t, H1_4, DO_SQNEG)
DO_ZPZ_D(sve2_sqneg_d, uint64_t, DO_SQNEG)

DO_ZPZ(sve2_urecpe_s, uint32_t, H1_4, helper_recpe_u32)
DO_ZPZ(sve2_ursqrte_s, uint32_t, H1_4, helper_rsqrte_u32)

/* Three-operand expander, unpredicated, in which the third operand is "wide".
 */
#define DO_ZZW(NAME, TYPE, TYPEW, H, OP)                       \
void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \
{                                                              \
    intptr_t i, opr_sz = simd_oprsz(desc);                     \
    for (i = 0; i < opr_sz; ) {                                \
        TYPEW mm = *(TYPEW *)(vm + i);                         \
        do {                                                   \
            TYPE nn = *(TYPE *)(vn + H(i));                    \
            *(TYPE *)(vd + H(i)) = OP(nn, mm);                 \
            i += sizeof(TYPE);                                 \
        } while (i & 7);                                       \
    }                                                          \
}

DO_ZZW(sve_asr_zzw_b, int8_t, uint64_t, H1, DO_ASR)
DO_ZZW(sve_lsr_zzw_b, uint8_t, uint64_t, H1, DO_LSR)
DO_ZZW(sve_lsl_zzw_b, uint8_t, uint64_t, H1, DO_LSL)

DO_ZZW(sve_asr_zzw_h, int16_t, uint64_t, H1_2, DO_ASR)
DO_ZZW(sve_lsr_zzw_h, uint16_t, uint64_t, H1_2, DO_LSR)
DO_ZZW(sve_lsl_zzw_h, uint16_t, uint64_t, H1_2, DO_LSL)

DO_ZZW(sve_asr_zzw_s, int32_t, uint64_t, H1_4, DO_ASR)
DO_ZZW(sve_lsr_zzw_s, uint32_t, uint64_t, H1_4, DO_LSR)
DO_ZZW(sve_lsl_zzw_s, uint32_t, uint64_t, H1_4, DO_LSL)

#undef DO_ZZW

#undef DO_CLS_B
#undef DO_CLS_H
#undef DO_CLZ_B
#undef DO_CLZ_H
#undef DO_CNOT
#undef DO_FABS
#undef DO_FNEG
#undef DO_ABS
#undef DO_NEG
#undef DO_ZPZ
#undef DO_ZPZ_D

/*
 * Three-operand expander, unpredicated, in which the two inputs are
 * selected from the top or bottom half of the wide column.
 */
#define DO_ZZZ_TB(NAME, TYPEW, TYPEN, HW, HN, OP) \
void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc)          \
{                                                                       \
    intptr_t i, opr_sz = simd_oprsz(desc);                              \
    int sel1 = extract32(desc, SIMD_DATA_SHIFT, 1) * sizeof(TYPEN);     \
    int sel2 = extract32(desc, SIMD_DATA_SHIFT + 1, 1) * sizeof(TYPEN); \
    for (i = 0; i < opr_sz; i += sizeof(TYPEW)) {                       \
        TYPEW nn = *(TYPEN *)(vn + HN(i + sel1));                       \
        TYPEW mm = *(TYPEN *)(vm + HN(i + sel2));                       \
        *(TYPEW *)(vd + HW(i)) = OP(nn, mm);                            \
    }                                                                   \
}

DO_ZZZ_TB(sve2_saddl_h, int16_t, int8_t, H1_2, H1, DO_ADD)
DO_ZZZ_TB(sve2_saddl_s, int32_t, int16_t, H1_4, H1_2, DO_ADD)
DO_ZZZ_TB(sve2_saddl_d, int64_t, int32_t, H1_8, H1_4, DO_ADD)

DO_ZZZ_TB(sve2_ssubl_h, int16_t, int8_t, H1_2, H1, DO_SUB)
DO_ZZZ_TB(sve2_ssubl_s, int32_t, int16_t, H1_4, H1_2, DO_SUB)
DO_ZZZ_TB(sve2_ssubl_d, int64_t, int32_t, H1_8, H1_4, DO_SUB)

DO_ZZZ_TB(sve2_sabdl_h, int16_t, int8_t, H1_2, H1, DO_ABD)
DO_ZZZ_TB(sve2_sabdl_s, int32_t, int16_t, H1_4, H1_2, DO_ABD)
DO_ZZZ_TB(sve2_sabdl_d, int64_t, int32_t, H1_8, H1_4, DO_ABD)

DO_ZZZ_TB(sve2_uaddl_h, uint16_t, uint8_t, H1_2, H1, DO_ADD)
DO_ZZZ_TB(sve2_uaddl_s, uint32_t, uint16_t, H1_4, H1_2, DO_ADD)
DO_ZZZ_TB(sve2_uaddl_d, uint64_t, uint32_t, H1_8, H1_4, DO_ADD)

DO_ZZZ_TB(sve2_usubl_h, uint16_t, uint8_t, H1_2, H1, DO_SUB)
DO_ZZZ_TB(sve2_usubl_s, uint32_t, uint16_t, H1_4, H1_2, DO_SUB)
DO_ZZZ_TB(sve2_usubl_d, uint64_t, uint32_t, H1_8, H1_4, DO_SUB)

DO_ZZZ_TB(sve2_uabdl_h, uint16_t, uint8_t, H1_2, H1, DO_ABD)
DO_ZZZ_TB(sve2_uabdl_s, uint32_t, uint16_t, H1_4, H1_2, DO_ABD)
DO_ZZZ_TB(sve2_uabdl_d, uint64_t, uint32_t, H1_8, H1_4, DO_ABD)

DO_ZZZ_TB(sve2_smull_zzz_h, int16_t, int8_t, H1_2, H1, DO_MUL)
DO_ZZZ_TB(sve2_smull_zzz_s, int32_t, int16_t, H1_4, H1_2, DO_MUL)
DO_ZZZ_TB(sve2_smull_zzz_d, int64_t, int32_t, H1_8, H1_4, DO_MUL)

DO_ZZZ_TB(sve2_umull_zzz_h, uint16_t, uint8_t, H1_2, H1, DO_MUL)
DO_ZZZ_TB(sve2_umull_zzz_s, uint32_t, uint16_t, H1_4, H1_2, DO_MUL)
DO_ZZZ_TB(sve2_umull_zzz_d, uint64_t, uint32_t, H1_8, H1_4, DO_MUL)

/* Note that the multiply cannot overflow, but the doubling can. */
static inline int16_t do_sqdmull_h(int16_t n, int16_t m)
{
    int16_t val = n * m;
    return DO_SQADD_H(val, val);
}

static inline int32_t do_sqdmull_s(int32_t n, int32_t m)
{
    int32_t val = n * m;
    return DO_SQADD_S(val, val);
}

static inline int64_t do_sqdmull_d(int64_t n, int64_t m)
{
    int64_t val = n * m;
    return do_sqadd_d(val, val);
}

DO_ZZZ_TB(sve2_sqdmull_zzz_h, int16_t, int8_t, H1_2, H1, do_sqdmull_h)
DO_ZZZ_TB(sve2_sqdmull_zzz_s, int32_t, int16_t, H1_4, H1_2, do_sqdmull_s)
DO_ZZZ_TB(sve2_sqdmull_zzz_d, int64_t, int32_t, H1_8, H1_4, do_sqdmull_d)

#undef DO_ZZZ_TB

#define DO_ZZZ_WTB(NAME, TYPEW, TYPEN, HW, HN, OP) \
void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \
{                                                              \
    intptr_t i, opr_sz = simd_oprsz(desc);                     \
    int sel2 = extract32(desc, SIMD_DATA_SHIFT, 1) * sizeof(TYPEN); \
    for (i = 0; i < opr_sz; i += sizeof(TYPEW)) {              \
        TYPEW nn = *(TYPEW *)(vn + HW(i));                     \
        TYPEW mm = *(TYPEN *)(vm + HN(i + sel2));              \
        *(TYPEW *)(vd + HW(i)) = OP(nn, mm);                   \
    }                                                          \
}

DO_ZZZ_WTB(sve2_saddw_h, int16_t, int8_t, H1_2, H1, DO_ADD)
DO_ZZZ_WTB(sve2_saddw_s, int32_t, int16_t, H1_4, H1_2, DO_ADD)
DO_ZZZ_WTB(sve2_saddw_d, int64_t, int32_t, H1_8, H1_4, DO_ADD)

DO_ZZZ_WTB(sve2_ssubw_h, int16_t, int8_t, H1_2, H1, DO_SUB)
DO_ZZZ_WTB(sve2_ssubw_s, int32_t, int16_t, H1_4, H1_2, DO_SUB)
DO_ZZZ_WTB(sve2_ssubw_d, int64_t, int32_t, H1_8, H1_4, DO_SUB)

DO_ZZZ_WTB(sve2_uaddw_h, uint16_t, uint8_t, H1_2, H1, DO_ADD)
DO_ZZZ_WTB(sve2_uaddw_s, uint32_t, uint16_t, H1_4, H1_2, DO_ADD)
DO_ZZZ_WTB(sve2_uaddw_d, uint64_t, uint32_t, H1_8, H1_4, DO_ADD)

DO_ZZZ_WTB(sve2_usubw_h, uint16_t, uint8_t, H1_2, H1, DO_SUB)
DO_ZZZ_WTB(sve2_usubw_s, uint32_t, uint16_t, H1_4, H1_2, DO_SUB)
DO_ZZZ_WTB(sve2_usubw_d, uint64_t, uint32_t, H1_8, H1_4, DO_SUB)

#undef DO_ZZZ_WTB

#define DO_ZZZ_NTB(NAME, TYPE, H, OP)                                   \
void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc)          \
{                                                                       \
    intptr_t i, opr_sz = simd_oprsz(desc);                              \
    intptr_t sel1 = extract32(desc, SIMD_DATA_SHIFT, 1) * sizeof(TYPE); \
    intptr_t sel2 = extract32(desc, SIMD_DATA_SHIFT + 1, 1) * sizeof(TYPE); \
    for (i = 0; i < opr_sz; i += 2 * sizeof(TYPE)) {                    \
        TYPE nn = *(TYPE *)(vn + H(i + sel1));                          \
        TYPE mm = *(TYPE *)(vm + H(i + sel2));                          \
        *(TYPE *)(vd + H(i + sel1)) = OP(nn, mm);                       \
    }                                                                   \
}

DO_ZZZ_NTB(sve2_eoril_b, uint8_t, H1, DO_EOR)
DO_ZZZ_NTB(sve2_eoril_h, uint16_t, H1_2, DO_EOR)
DO_ZZZ_NTB(sve2_eoril_s, uint32_t, H1_4, DO_EOR)
DO_ZZZ_NTB(sve2_eoril_d, uint64_t, H1_8, DO_EOR)

#undef DO_ZZZ_NTB

#define DO_ZZZW_ACC(NAME, TYPEW, TYPEN, HW, HN, OP) \
void HELPER(NAME)(void *vd, void *vn, void *vm, void *va, uint32_t desc) \
{                                                               \
    intptr_t i, opr_sz = simd_oprsz(desc);                      \
    intptr_t sel1 = simd_data(desc) * sizeof(TYPEN);            \
    for (i = 0; i < opr_sz; i += sizeof(TYPEW)) {               \
        TYPEW nn = *(TYPEN *)(vn + HN(i + sel1));               \
        TYPEW mm = *(TYPEN *)(vm + HN(i + sel1));               \
        TYPEW aa = *(TYPEW *)(va + HW(i));                      \
        *(TYPEW *)(vd + HW(i)) = OP(nn, mm) + aa;               \
    }                                                           \
}

DO_ZZZW_ACC(sve2_sabal_h, int16_t, int8_t, H1_2, H1, DO_ABD)
DO_ZZZW_ACC(sve2_sabal_s, int32_t, int16_t, H1_4, H1_2, DO_ABD)
DO_ZZZW_ACC(sve2_sabal_d, int64_t, int32_t, H1_8, H1_4, DO_ABD)

DO_ZZZW_ACC(sve2_uabal_h, uint16_t, uint8_t, H1_2, H1, DO_ABD)
DO_ZZZW_ACC(sve2_uabal_s, uint32_t, uint16_t, H1_4, H1_2, DO_ABD)
DO_ZZZW_ACC(sve2_uabal_d, uint64_t, uint32_t, H1_8, H1_4, DO_ABD)

DO_ZZZW_ACC(sve2_smlal_zzzw_h, int16_t, int8_t, H1_2, H1, DO_MUL)
DO_ZZZW_ACC(sve2_smlal_zzzw_s, int32_t, int16_t, H1_4, H1_2, DO_MUL)
DO_ZZZW_ACC(sve2_smlal_zzzw_d, int64_t, int32_t, H1_8, H1_4, DO_MUL)

DO_ZZZW_ACC(sve2_umlal_zzzw_h, uint16_t, uint8_t, H1_2, H1, DO_MUL)
DO_ZZZW_ACC(sve2_umlal_zzzw_s, uint32_t, uint16_t, H1_4, H1_2, DO_MUL)
DO_ZZZW_ACC(sve2_umlal_zzzw_d, uint64_t, uint32_t, H1_8, H1_4, DO_MUL)

#define DO_NMUL(N, M)  -(N * M)

DO_ZZZW_ACC(sve2_smlsl_zzzw_h, int16_t, int8_t, H1_2, H1, DO_NMUL)
DO_ZZZW_ACC(sve2_smlsl_zzzw_s, int32_t, int16_t, H1_4, H1_2, DO_NMUL)
DO_ZZZW_ACC(sve2_smlsl_zzzw_d, int64_t, int32_t, H1_8, H1_4, DO_NMUL)

DO_ZZZW_ACC(sve2_umlsl_zzzw_h, uint16_t, uint8_t, H1_2, H1, DO_NMUL)
DO_ZZZW_ACC(sve2_umlsl_zzzw_s, uint32_t, uint16_t, H1_4, H1_2, DO_NMUL)
DO_ZZZW_ACC(sve2_umlsl_zzzw_d, uint64_t, uint32_t, H1_8, H1_4, DO_NMUL)

#undef DO_ZZZW_ACC

#define DO_XTNB(NAME, TYPE, OP) \
void HELPER(NAME)(void *vd, void *vn, uint32_t desc)         \
{                                                            \
    intptr_t i, opr_sz = simd_oprsz(desc);                   \
    for (i = 0; i < opr_sz; i += sizeof(TYPE)) {             \
        TYPE nn = *(TYPE *)(vn + i);                         \
        nn = OP(nn) & MAKE_64BIT_MASK(0, sizeof(TYPE) * 4);  \
        *(TYPE *)(vd + i) = nn;                              \
    }                                                        \
}

#define DO_XTNT(NAME, TYPE, TYPEN, H, OP)                               \
void HELPER(NAME)(void *vd, void *vn, uint32_t desc)                    \
{                                                                       \
    intptr_t i, opr_sz = simd_oprsz(desc), odd = H(sizeof(TYPEN));      \
    for (i = 0; i < opr_sz; i += sizeof(TYPE)) {                        \
        TYPE nn = *(TYPE *)(vn + i);                                    \
        *(TYPEN *)(vd + i + odd) = OP(nn);                              \
    }                                                                   \
}

#define DO_SQXTN_H(n)  do_sat_bhs(n, INT8_MIN, INT8_MAX)
#define DO_SQXTN_S(n)  do_sat_bhs(n, INT16_MIN, INT16_MAX)
#define DO_SQXTN_D(n)  do_sat_bhs(n, INT32_MIN, INT32_MAX)

DO_XTNB(sve2_sqxtnb_h, int16_t, DO_SQXTN_H)
DO_XTNB(sve2_sqxtnb_s, int32_t, DO_SQXTN_S)
DO_XTNB(sve2_sqxtnb_d, int64_t, DO_SQXTN_D)

DO_XTNT(sve2_sqxtnt_h, int16_t, int8_t, H1, DO_SQXTN_H)
DO_XTNT(sve2_sqxtnt_s, int32_t, int16_t, H1_2, DO_SQXTN_S)
DO_XTNT(sve2_sqxtnt_d, int64_t, int32_t, H1_4, DO_SQXTN_D)

#define DO_UQXTN_H(n)  do_sat_bhs(n, 0, UINT8_MAX)
#define DO_UQXTN_S(n)  do_sat_bhs(n, 0, UINT16_MAX)
#define DO_UQXTN_D(n)  do_sat_bhs(n, 0, UINT32_MAX)

DO_XTNB(sve2_uqxtnb_h, uint16_t, DO_UQXTN_H)
DO_XTNB(sve2_uqxtnb_s, uint32_t, DO_UQXTN_S)
DO_XTNB(sve2_uqxtnb_d, uint64_t, DO_UQXTN_D)

DO_XTNT(sve2_uqxtnt_h, uint16_t, uint8_t, H1, DO_UQXTN_H)
DO_XTNT(sve2_uqxtnt_s, uint32_t, uint16_t, H1_2, DO_UQXTN_S)
DO_XTNT(sve2_uqxtnt_d, uint64_t, uint32_t, H1_4, DO_UQXTN_D)

DO_XTNB(sve2_sqxtunb_h, int16_t, DO_UQXTN_H)
DO_XTNB(sve2_sqxtunb_s, int32_t, DO_UQXTN_S)
DO_XTNB(sve2_sqxtunb_d, int64_t, DO_UQXTN_D)

DO_XTNT(sve2_sqxtunt_h, int16_t, int8_t, H1, DO_UQXTN_H)
DO_XTNT(sve2_sqxtunt_s, int32_t, int16_t, H1_2, DO_UQXTN_S)
DO_XTNT(sve2_sqxtunt_d, int64_t, int32_t, H1_4, DO_UQXTN_D)

#undef DO_XTNB
#undef DO_XTNT

void HELPER(sve2_adcl_s)(void *vd, void *vn, void *vm, void *va, uint32_t desc)
{
    intptr_t i, opr_sz = simd_oprsz(desc);
    int sel = H4(extract32(desc, SIMD_DATA_SHIFT, 1));
    uint32_t inv = -extract32(desc, SIMD_DATA_SHIFT + 1, 1);
    uint32_t *a = va, *n = vn;
    uint64_t *d = vd, *m = vm;

    for (i = 0; i < opr_sz / 8; ++i) {
        uint32_t e1 = a[2 * i + H4(0)];
        uint32_t e2 = n[2 * i + sel] ^ inv;
        uint64_t c = extract64(m[i], 32, 1);
        /* Compute and store the entire 33-bit result at once. */
        d[i] = c + e1 + e2;
    }
}

void HELPER(sve2_adcl_d)(void *vd, void *vn, void *vm, void *va, uint32_t desc)
{
    intptr_t i, opr_sz = simd_oprsz(desc);
    int sel = extract32(desc, SIMD_DATA_SHIFT, 1);
    uint64_t inv = -(uint64_t)extract32(desc, SIMD_DATA_SHIFT + 1, 1);
    uint64_t *d = vd, *a = va, *n = vn, *m = vm;

    for (i = 0; i < opr_sz / 8; i += 2) {
        Int128 e1 = int128_make64(a[i]);
        Int128 e2 = int128_make64(n[i + sel] ^ inv);
        Int128 c = int128_make64(m[i + 1] & 1);
        Int128 r = int128_add(int128_add(e1, e2), c);
        d[i + 0] = int128_getlo(r);
        d[i + 1] = int128_gethi(r);
    }
}

#define DO_SQDMLAL(NAME, TYPEW, TYPEN, HW, HN, DMUL_OP, SUM_OP) \
void HELPER(NAME)(void *vd, void *vn, void *vm, void *va, uint32_t desc) \
{                                                                       \
    intptr_t i, opr_sz = simd_oprsz(desc);                              \
    int sel1 = extract32(desc, SIMD_DATA_SHIFT, 1) * sizeof(TYPEN);     \
    int sel2 = extract32(desc, SIMD_DATA_SHIFT + 1, 1) * sizeof(TYPEN); \
    for (i = 0; i < opr_sz; i += sizeof(TYPEW)) {                       \
        TYPEW nn = *(TYPEN *)(vn + HN(i + sel1));                       \
        TYPEW mm = *(TYPEN *)(vm + HN(i + sel2));                       \
        TYPEW aa = *(TYPEW *)(va + HW(i));                              \
        *(TYPEW *)(vd + HW(i)) = SUM_OP(aa, DMUL_OP(nn, mm));           \
    }                                                                   \
}

DO_SQDMLAL(sve2_sqdmlal_zzzw_h, int16_t, int8_t, H1_2, H1,
           do_sqdmull_h, DO_SQADD_H)
DO_SQDMLAL(sve2_sqdmlal_zzzw_s, int32_t, int16_t, H1_4, H1_2,
           do_sqdmull_s, DO_SQADD_S)
DO_SQDMLAL(sve2_sqdmlal_zzzw_d, int64_t, int32_t, H1_8, H1_4,
           do_sqdmull_d, do_sqadd_d)

DO_SQDMLAL(sve2_sqdmlsl_zzzw_h, int16_t, int8_t, H1_2, H1,
           do_sqdmull_h, DO_SQSUB_H)
DO_SQDMLAL(sve2_sqdmlsl_zzzw_s, int32_t, int16_t, H1_4, H1_2,
           do_sqdmull_s, DO_SQSUB_S)
DO_SQDMLAL(sve2_sqdmlsl_zzzw_d, int64_t, int32_t, H1_8, H1_4,
           do_sqdmull_d, do_sqsub_d)

#undef DO_SQDMLAL

#define DO_CMLA_FUNC(NAME, TYPE, H, OP) \
void HELPER(NAME)(void *vd, void *vn, void *vm, void *va, uint32_t desc) \
{                                                               \
    intptr_t i, opr_sz = simd_oprsz(desc) / sizeof(TYPE);       \
    int rot = simd_data(desc);                                  \
    int sel_a = rot & 1, sel_b = sel_a ^ 1;                     \
    bool sub_r = rot == 1 || rot == 2;                          \
    bool sub_i = rot >= 2;                                      \
    TYPE *d = vd, *n = vn, *m = vm, *a = va;                    \
    for (i = 0; i < opr_sz; i += 2) {                           \
        TYPE elt1_a = n[H(i + sel_a)];                          \
        TYPE elt2_a = m[H(i + sel_a)];                          \
        TYPE elt2_b = m[H(i + sel_b)];                          \
        d[H(i)] = OP(elt1_a, elt2_a, a[H(i)], sub_r);           \
        d[H(i + 1)] = OP(elt1_a, elt2_b, a[H(i + 1)], sub_i);   \
    }                                                           \
}

#define DO_CMLA(N, M, A, S) (A + (N * M) * (S ? -1 : 1))

DO_CMLA_FUNC(sve2_cmla_zzzz_b, uint8_t, H1, DO_CMLA)
DO_CMLA_FUNC(sve2_cmla_zzzz_h, uint16_t, H2, DO_CMLA)
DO_CMLA_FUNC(sve2_cmla_zzzz_s, uint32_t, H4, DO_CMLA)
DO_CMLA_FUNC(sve2_cmla_zzzz_d, uint64_t, H8, DO_CMLA)

#define DO_SQRDMLAH_B(N, M, A, S) \
    do_sqrdmlah_b(N, M, A, S, true)
#define DO_SQRDMLAH_H(N, M, A, S) \
    ({ uint32_t discard; do_sqrdmlah_h(N, M, A, S, true, &discard); })
#define DO_SQRDMLAH_S(N, M, A, S) \
    ({ uint32_t discard; do_sqrdmlah_s(N, M, A, S, true, &discard); })
#define DO_SQRDMLAH_D(N, M, A, S) \
    do_sqrdmlah_d(N, M, A, S, true)

DO_CMLA_FUNC(sve2_sqrdcmlah_zzzz_b, int8_t, H1, DO_SQRDMLAH_B)
DO_CMLA_FUNC(sve2_sqrdcmlah_zzzz_h, int16_t, H2, DO_SQRDMLAH_H)
DO_CMLA_FUNC(sve2_sqrdcmlah_zzzz_s, int32_t, H4, DO_SQRDMLAH_S)
DO_CMLA_FUNC(sve2_sqrdcmlah_zzzz_d, int64_t, H8, DO_SQRDMLAH_D)

#define DO_CMLA_IDX_FUNC(NAME, TYPE, H, OP) \
void HELPER(NAME)(void *vd, void *vn, void *vm, void *va, uint32_t desc)    \
{                                                                           \
    intptr_t i, j, oprsz = simd_oprsz(desc);                                \
    int rot = extract32(desc, SIMD_DATA_SHIFT, 2);                          \
    int idx = extract32(desc, SIMD_DATA_SHIFT + 2, 2) * 2;                  \
    int sel_a = rot & 1, sel_b = sel_a ^ 1;                                 \
    bool sub_r = rot == 1 || rot == 2;                                      \
    bool sub_i = rot >= 2;                                                  \
    TYPE *d = vd, *n = vn, *m = vm, *a = va;                                \
    for (i = 0; i < oprsz / sizeof(TYPE); i += 16 / sizeof(TYPE)) {         \
        TYPE elt2_a = m[H(i + idx + sel_a)];                                \
        TYPE elt2_b = m[H(i + idx + sel_b)];                                \
        for (j = 0; j < 16 / sizeof(TYPE); j += 2) {                        \
            TYPE elt1_a = n[H(i + j + sel_a)];                              \
            d[H2(i + j)] = OP(elt1_a, elt2_a, a[H(i + j)], sub_r);          \
            d[H2(i + j + 1)] = OP(elt1_a, elt2_b, a[H(i + j + 1)], sub_i);  \
        }                                                                   \
    }                                                                       \
}

DO_CMLA_IDX_FUNC(sve2_cmla_idx_h, int16_t, H2, DO_CMLA)
DO_CMLA_IDX_FUNC(sve2_cmla_idx_s, int32_t, H4, DO_CMLA)

DO_CMLA_IDX_FUNC(sve2_sqrdcmlah_idx_h, int16_t, H2, DO_SQRDMLAH_H)
DO_CMLA_IDX_FUNC(sve2_sqrdcmlah_idx_s, int32_t, H4, DO_SQRDMLAH_S)

#undef DO_CMLA
#undef DO_CMLA_FUNC
#undef DO_CMLA_IDX_FUNC
#undef DO_SQRDMLAH_B
#undef DO_SQRDMLAH_H
#undef DO_SQRDMLAH_S
#undef DO_SQRDMLAH_D

/* Note N and M are 4 elements bundled into one unit. */
static int32_t do_cdot_s(uint32_t n, uint32_t m, int32_t a,
                         int sel_a, int sel_b, int sub_i)
{
    for (int i = 0; i <= 1; i++) {
        int32_t elt1_r = (int8_t)(n >> (16 * i));
        int32_t elt1_i = (int8_t)(n >> (16 * i + 8));
        int32_t elt2_a = (int8_t)(m >> (16 * i + 8 * sel_a));
        int32_t elt2_b = (int8_t)(m >> (16 * i + 8 * sel_b));

        a += elt1_r * elt2_a + elt1_i * elt2_b * sub_i;
    }
    return a;
}

static int64_t do_cdot_d(uint64_t n, uint64_t m, int64_t a,
                         int sel_a, int sel_b, int sub_i)
{
    for (int i = 0; i <= 1; i++) {
        int64_t elt1_r = (int16_t)(n >> (32 * i + 0));
        int64_t elt1_i = (int16_t)(n >> (32 * i + 16));
        int64_t elt2_a = (int16_t)(m >> (32 * i + 16 * sel_a));
        int64_t elt2_b = (int16_t)(m >> (32 * i + 16 * sel_b));

        a += elt1_r * elt2_a + elt1_i * elt2_b * sub_i;
    }
    return a;
}

void HELPER(sve2_cdot_zzzz_s)(void *vd, void *vn, void *vm,
                              void *va, uint32_t desc)
{
    int opr_sz = simd_oprsz(desc);
    int rot = simd_data(desc);
    int sel_a = rot & 1;
    int sel_b = sel_a ^ 1;
    int sub_i = (rot == 0 || rot == 3 ? -1 : 1);
    uint32_t *d = vd, *n = vn, *m = vm, *a = va;

    for (int e = 0; e < opr_sz / 4; e++) {
        d[e] = do_cdot_s(n[e], m[e], a[e], sel_a, sel_b, sub_i);
    }
}

void HELPER(sve2_cdot_zzzz_d)(void *vd, void *vn, void *vm,
                              void *va, uint32_t desc)
{
    int opr_sz = simd_oprsz(desc);
    int rot = simd_data(desc);
    int sel_a = rot & 1;
    int sel_b = sel_a ^ 1;
    int sub_i = (rot == 0 || rot == 3 ? -1 : 1);
    uint64_t *d = vd, *n = vn, *m = vm, *a = va;

    for (int e = 0; e < opr_sz / 8; e++) {
        d[e] = do_cdot_d(n[e], m[e], a[e], sel_a, sel_b, sub_i);
    }
}

void HELPER(sve2_cdot_idx_s)(void *vd, void *vn, void *vm,
                             void *va, uint32_t desc)
{
    int opr_sz = simd_oprsz(desc);
    int rot = extract32(desc, SIMD_DATA_SHIFT, 2);
    int idx = H4(extract32(desc, SIMD_DATA_SHIFT + 2, 2));
    int sel_a = rot & 1;
    int sel_b = sel_a ^ 1;
    int sub_i = (rot == 0 || rot == 3 ? -1 : 1);
    uint32_t *d = vd, *n = vn, *m = vm, *a = va;

    for (int seg = 0; seg < opr_sz / 4; seg += 4) {
        uint32_t seg_m = m[seg + idx];
        for (int e = 0; e < 4; e++) {
            d[seg + e] = do_cdot_s(n[seg + e], seg_m, a[seg + e],
                                   sel_a, sel_b, sub_i);
        }
    }
}

void HELPER(sve2_cdot_idx_d)(void *vd, void *vn, void *vm,
                             void *va, uint32_t desc)
{
    int seg, opr_sz = simd_oprsz(desc);
    int rot = extract32(desc, SIMD_DATA_SHIFT, 2);
    int idx = extract32(desc, SIMD_DATA_SHIFT + 2, 2);
    int sel_a = rot & 1;
    int sel_b = sel_a ^ 1;
    int sub_i = (rot == 0 || rot == 3 ? -1 : 1);
    uint64_t *d = vd, *n = vn, *m = vm, *a = va;

    for (seg = 0; seg < opr_sz / 8; seg += 2) {
        uint64_t seg_m = m[seg + idx];
        for (int e = 0; e < 2; e++) {
            d[seg + e] = do_cdot_d(n[seg + e], seg_m, a[seg + e],
                                   sel_a, sel_b, sub_i);
        }
    }
}

#define DO_ZZXZ(NAME, TYPE, H, OP) \
void HELPER(NAME)(void *vd, void *vn, void *vm, void *va, uint32_t desc) \
{                                                                       \
    intptr_t oprsz = simd_oprsz(desc), segment = 16 / sizeof(TYPE);     \
    intptr_t i, j, idx = simd_data(desc);                               \
    TYPE *d = vd, *a = va, *n = vn, *m = (TYPE *)vm + H(idx);           \
    for (i = 0; i < oprsz / sizeof(TYPE); i += segment) {               \
        TYPE mm = m[i];                                                 \
        for (j = 0; j < segment; j++) {                                 \
            d[i + j] = OP(n[i + j], mm, a[i + j]);                      \
        }                                                               \
    }                                                                   \
}

#define DO_SQRDMLAH_H(N, M, A) \
    ({ uint32_t discard; do_sqrdmlah_h(N, M, A, false, true, &discard); })
#define DO_SQRDMLAH_S(N, M, A) \
    ({ uint32_t discard; do_sqrdmlah_s(N, M, A, false, true, &discard); })
#define DO_SQRDMLAH_D(N, M, A) do_sqrdmlah_d(N, M, A, false, true)

DO_ZZXZ(sve2_sqrdmlah_idx_h, int16_t, H2, DO_SQRDMLAH_H)
DO_ZZXZ(sve2_sqrdmlah_idx_s, int32_t, H4, DO_SQRDMLAH_S)
DO_ZZXZ(sve2_sqrdmlah_idx_d, int64_t, H8, DO_SQRDMLAH_D)

#define DO_SQRDMLSH_H(N, M, A) \
    ({ uint32_t discard; do_sqrdmlah_h(N, M, A, true, true, &discard); })
#define DO_SQRDMLSH_S(N, M, A) \
    ({ uint32_t discard; do_sqrdmlah_s(N, M, A, true, true, &discard); })
#define DO_SQRDMLSH_D(N, M, A) do_sqrdmlah_d(N, M, A, true, true)

DO_ZZXZ(sve2_sqrdmlsh_idx_h, int16_t, H2, DO_SQRDMLSH_H)
DO_ZZXZ(sve2_sqrdmlsh_idx_s, int32_t, H4, DO_SQRDMLSH_S)
DO_ZZXZ(sve2_sqrdmlsh_idx_d, int64_t, H8, DO_SQRDMLSH_D)

#undef DO_ZZXZ

#define DO_ZZXW(NAME, TYPEW, TYPEN, HW, HN, OP) \
void HELPER(NAME)(void *vd, void *vn, void *vm, void *va, uint32_t desc)  \
{                                                                         \
    intptr_t i, j, oprsz = simd_oprsz(desc);                              \
    intptr_t sel = extract32(desc, SIMD_DATA_SHIFT, 1) * sizeof(TYPEN);   \
    intptr_t idx = extract32(desc, SIMD_DATA_SHIFT + 1, 3) * sizeof(TYPEN); \
    for (i = 0; i < oprsz; i += 16) {                                     \
        TYPEW mm = *(TYPEN *)(vm + HN(i + idx));                          \
        for (j = 0; j < 16; j += sizeof(TYPEW)) {                         \
            TYPEW nn = *(TYPEN *)(vn + HN(i + j + sel));                  \
            TYPEW aa = *(TYPEW *)(va + HW(i + j));                        \
            *(TYPEW *)(vd + HW(i + j)) = OP(nn, mm, aa);                  \
        }                                                                 \
    }                                                                     \
}

#define DO_MLA(N, M, A)  (A + N * M)

DO_ZZXW(sve2_smlal_idx_s, int32_t, int16_t, H1_4, H1_2, DO_MLA)
DO_ZZXW(sve2_smlal_idx_d, int64_t, int32_t, H1_8, H1_4, DO_MLA)
DO_ZZXW(sve2_umlal_idx_s, uint32_t, uint16_t, H1_4, H1_2, DO_MLA)
DO_ZZXW(sve2_umlal_idx_d, uint64_t, uint32_t, H1_8, H1_4, DO_MLA)

#define DO_MLS(N, M, A)  (A - N * M)

DO_ZZXW(sve2_smlsl_idx_s, int32_t, int16_t, H1_4, H1_2, DO_MLS)
DO_ZZXW(sve2_smlsl_idx_d, int64_t, int32_t, H1_8, H1_4, DO_MLS)
DO_ZZXW(sve2_umlsl_idx_s, uint32_t, uint16_t, H1_4, H1_2, DO_MLS)
DO_ZZXW(sve2_umlsl_idx_d, uint64_t, uint32_t, H1_8, H1_4, DO_MLS)

#define DO_SQDMLAL_S(N, M, A)  DO_SQADD_S(A, do_sqdmull_s(N, M))
#define DO_SQDMLAL_D(N, M, A)  do_sqadd_d(A, do_sqdmull_d(N, M))

DO_ZZXW(sve2_sqdmlal_idx_s, int32_t, int16_t, H1_4, H1_2, DO_SQDMLAL_S)
DO_ZZXW(sve2_sqdmlal_idx_d, int64_t, int32_t, H1_8, H1_4, DO_SQDMLAL_D)

#define DO_SQDMLSL_S(N, M, A)  DO_SQSUB_S(A, do_sqdmull_s(N, M))
#define DO_SQDMLSL_D(N, M, A)  do_sqsub_d(A, do_sqdmull_d(N, M))

DO_ZZXW(sve2_sqdmlsl_idx_s, int32_t, int16_t, H1_4, H1_2, DO_SQDMLSL_S)
DO_ZZXW(sve2_sqdmlsl_idx_d, int64_t, int32_t, H1_8, H1_4, DO_SQDMLSL_D)

#undef DO_MLA
#undef DO_MLS
#undef DO_ZZXW

#define DO_ZZX(NAME, TYPEW, TYPEN, HW, HN, OP) \
void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc)            \
{                                                                         \
    intptr_t i, j, oprsz = simd_oprsz(desc);                              \
    intptr_t sel = extract32(desc, SIMD_DATA_SHIFT, 1) * sizeof(TYPEN);   \
    intptr_t idx = extract32(desc, SIMD_DATA_SHIFT + 1, 3) * sizeof(TYPEN); \
    for (i = 0; i < oprsz; i += 16) {                                     \
        TYPEW mm = *(TYPEN *)(vm + HN(i + idx));                          \
        for (j = 0; j < 16; j += sizeof(TYPEW)) {                         \
            TYPEW nn = *(TYPEN *)(vn + HN(i + j + sel));                  \
            *(TYPEW *)(vd + HW(i + j)) = OP(nn, mm);                      \
        }                                                                 \
    }                                                                     \
}

DO_ZZX(sve2_sqdmull_idx_s, int32_t, int16_t, H1_4, H1_2, do_sqdmull_s)
DO_ZZX(sve2_sqdmull_idx_d, int64_t, int32_t, H1_8, H1_4, do_sqdmull_d)

DO_ZZX(sve2_smull_idx_s, int32_t, int16_t, H1_4, H1_2, DO_MUL)
DO_ZZX(sve2_smull_idx_d, int64_t, int32_t, H1_8, H1_4, DO_MUL)

DO_ZZX(sve2_umull_idx_s, uint32_t, uint16_t, H1_4, H1_2, DO_MUL)
DO_ZZX(sve2_umull_idx_d, uint64_t, uint32_t, H1_8, H1_4, DO_MUL)

#undef DO_ZZX

#define DO_BITPERM(NAME, TYPE, OP) \
void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \
{                                                              \
    intptr_t i, opr_sz = simd_oprsz(desc);                     \
    for (i = 0; i < opr_sz; i += sizeof(TYPE)) {               \
        TYPE nn = *(TYPE *)(vn + i);                           \
        TYPE mm = *(TYPE *)(vm + i);                           \
        *(TYPE *)(vd + i) = OP(nn, mm, sizeof(TYPE) * 8);      \
    }                                                          \
}

static uint64_t bitextract(uint64_t data, uint64_t mask, int n)
{
    uint64_t res = 0;
    int db, rb = 0;

    for (db = 0; db < n; ++db) {
        if ((mask >> db) & 1) {
            res |= ((data >> db) & 1) << rb;
            ++rb;
        }
    }
    return res;
}

DO_BITPERM(sve2_bext_b, uint8_t, bitextract)
DO_BITPERM(sve2_bext_h, uint16_t, bitextract)
DO_BITPERM(sve2_bext_s, uint32_t, bitextract)
DO_BITPERM(sve2_bext_d, uint64_t, bitextract)

static uint64_t bitdeposit(uint64_t data, uint64_t mask, int n)
{
    uint64_t res = 0;
    int rb, db = 0;

    for (rb = 0; rb < n; ++rb) {
        if ((mask >> rb) & 1) {
            res |= ((data >> db) & 1) << rb;
            ++db;
        }
    }
    return res;
}

DO_BITPERM(sve2_bdep_b, uint8_t, bitdeposit)
DO_BITPERM(sve2_bdep_h, uint16_t, bitdeposit)
DO_BITPERM(sve2_bdep_s, uint32_t, bitdeposit)
DO_BITPERM(sve2_bdep_d, uint64_t, bitdeposit)

static uint64_t bitgroup(uint64_t data, uint64_t mask, int n)
{
    uint64_t resm = 0, resu = 0;
    int db, rbm = 0, rbu = 0;

    for (db = 0; db < n; ++db) {
        uint64_t val = (data >> db) & 1;
        if ((mask >> db) & 1) {
            resm |= val << rbm++;
        } else {
            resu |= val << rbu++;
        }
    }

    return resm | (resu << rbm);
}

DO_BITPERM(sve2_bgrp_b, uint8_t, bitgroup)
DO_BITPERM(sve2_bgrp_h, uint16_t, bitgroup)
DO_BITPERM(sve2_bgrp_s, uint32_t, bitgroup)
DO_BITPERM(sve2_bgrp_d, uint64_t, bitgroup)

#undef DO_BITPERM

#define DO_CADD(NAME, TYPE, H, ADD_OP, SUB_OP)                  \
void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc)  \
{                                                               \
    intptr_t i, opr_sz = simd_oprsz(desc);                      \
    int sub_r = simd_data(desc);                                \
    if (sub_r) {                                                \
        for (i = 0; i < opr_sz; i += 2 * sizeof(TYPE)) {        \
            TYPE acc_r = *(TYPE *)(vn + H(i));                  \
            TYPE acc_i = *(TYPE *)(vn + H(i + sizeof(TYPE)));   \
            TYPE el2_r = *(TYPE *)(vm + H(i));                  \
            TYPE el2_i = *(TYPE *)(vm + H(i + sizeof(TYPE)));   \
            acc_r = ADD_OP(acc_r, el2_i);                       \
            acc_i = SUB_OP(acc_i, el2_r);                       \
            *(TYPE *)(vd + H(i)) = acc_r;                       \
            *(TYPE *)(vd + H(i + sizeof(TYPE))) = acc_i;        \
        }                                                       \
    } else {                                                    \
        for (i = 0; i < opr_sz; i += 2 * sizeof(TYPE)) {        \
            TYPE acc_r = *(TYPE *)(vn + H(i));                  \
            TYPE acc_i = *(TYPE *)(vn + H(i + sizeof(TYPE)));   \
            TYPE el2_r = *(TYPE *)(vm + H(i));                  \
            TYPE el2_i = *(TYPE *)(vm + H(i + sizeof(TYPE)));   \
            acc_r = SUB_OP(acc_r, el2_i);                       \
            acc_i = ADD_OP(acc_i, el2_r);                       \
            *(TYPE *)(vd + H(i)) = acc_r;                       \
            *(TYPE *)(vd + H(i + sizeof(TYPE))) = acc_i;        \
        }                                                       \
    }                                                           \
}

DO_CADD(sve2_cadd_b, int8_t, H1, DO_ADD, DO_SUB)
DO_CADD(sve2_cadd_h, int16_t, H1_2, DO_ADD, DO_SUB)
DO_CADD(sve2_cadd_s, int32_t, H1_4, DO_ADD, DO_SUB)
DO_CADD(sve2_cadd_d, int64_t, H1_8, DO_ADD, DO_SUB)

DO_CADD(sve2_sqcadd_b, int8_t, H1, DO_SQADD_B, DO_SQSUB_B)
DO_CADD(sve2_sqcadd_h, int16_t, H1_2, DO_SQADD_H, DO_SQSUB_H)
DO_CADD(sve2_sqcadd_s, int32_t, H1_4, DO_SQADD_S, DO_SQSUB_S)
DO_CADD(sve2_sqcadd_d, int64_t, H1_8, do_sqadd_d, do_sqsub_d)

#undef DO_CADD

#define DO_ZZI_SHLL(NAME, TYPEW, TYPEN, HW, HN) \
void HELPER(NAME)(void *vd, void *vn, uint32_t desc)           \
{                                                              \
    intptr_t i, opr_sz = simd_oprsz(desc);                     \
    intptr_t sel = (simd_data(desc) & 1) * sizeof(TYPEN);      \
    int shift = simd_data(desc) >> 1;                          \
    for (i = 0; i < opr_sz; i += sizeof(TYPEW)) {              \
        TYPEW nn = *(TYPEN *)(vn + HN(i + sel));               \
        *(TYPEW *)(vd + HW(i)) = nn << shift;                  \
    }                                                          \
}

DO_ZZI_SHLL(sve2_sshll_h, int16_t, int8_t, H1_2, H1)
DO_ZZI_SHLL(sve2_sshll_s, int32_t, int16_t, H1_4, H1_2)
DO_ZZI_SHLL(sve2_sshll_d, int64_t, int32_t, H1_8, H1_4)

DO_ZZI_SHLL(sve2_ushll_h, uint16_t, uint8_t, H1_2, H1)
DO_ZZI_SHLL(sve2_ushll_s, uint32_t, uint16_t, H1_4, H1_2)
DO_ZZI_SHLL(sve2_ushll_d, uint64_t, uint32_t, H1_8, H1_4)

#undef DO_ZZI_SHLL

/* Two-operand reduction expander, controlled by a predicate.
 * The difference between TYPERED and TYPERET has to do with
 * sign-extension.  E.g. for SMAX, TYPERED must be signed,
 * but TYPERET must be unsigned so that e.g. a 32-bit value
 * is not sign-extended to the ABI uint64_t return type.
 */
/* ??? If we were to vectorize this by hand the reduction ordering
 * would change.  For integer operands, this is perfectly fine.
 */
#define DO_VPZ(NAME, TYPEELT, TYPERED, TYPERET, H, INIT, OP) \
uint64_t HELPER(NAME)(void *vn, void *vg, uint32_t desc)   \
{                                                          \
    intptr_t i, opr_sz = simd_oprsz(desc);                 \
    TYPERED ret = INIT;                                    \
    for (i = 0; i < opr_sz; ) {                            \
        uint16_t pg = *(uint16_t *)(vg + H1_2(i >> 3));    \
        do {                                               \
            if (pg & 1) {                                  \
                TYPEELT nn = *(TYPEELT *)(vn + H(i));      \
                ret = OP(ret, nn);                         \
            }                                              \
            i += sizeof(TYPEELT), pg >>= sizeof(TYPEELT);  \
        } while (i & 15);                                  \
    }                                                      \
    return (TYPERET)ret;                                   \
}

#define DO_VPZ_D(NAME, TYPEE, TYPER, INIT, OP)             \
uint64_t HELPER(NAME)(void *vn, void *vg, uint32_t desc)   \
{                                                          \
    intptr_t i, opr_sz = simd_oprsz(desc) / 8;             \
    TYPEE *n = vn;                                         \
    uint8_t *pg = vg;                                      \
    TYPER ret = INIT;                                      \
    for (i = 0; i < opr_sz; i += 1) {                      \
        if (pg[H1(i)] & 1) {                               \
            TYPEE nn = n[i];                               \
            ret = OP(ret, nn);                             \
        }                                                  \
    }                                                      \
    return ret;                                            \
}

DO_VPZ(sve_orv_b, uint8_t, uint8_t, uint8_t, H1, 0, DO_ORR)
DO_VPZ(sve_orv_h, uint16_t, uint16_t, uint16_t, H1_2, 0, DO_ORR)
DO_VPZ(sve_orv_s, uint32_t, uint32_t, uint32_t, H1_4, 0, DO_ORR)
DO_VPZ_D(sve_orv_d, uint64_t, uint64_t, 0, DO_ORR)

DO_VPZ(sve_eorv_b, uint8_t, uint8_t, uint8_t, H1, 0, DO_EOR)
DO_VPZ(sve_eorv_h, uint16_t, uint16_t, uint16_t, H1_2, 0, DO_EOR)
DO_VPZ(sve_eorv_s, uint32_t, uint32_t, uint32_t, H1_4, 0, DO_EOR)
DO_VPZ_D(sve_eorv_d, uint64_t, uint64_t, 0, DO_EOR)

DO_VPZ(sve_andv_b, uint8_t, uint8_t, uint8_t, H1, -1, DO_AND)
DO_VPZ(sve_andv_h, uint16_t, uint16_t, uint16_t, H1_2, -1, DO_AND)
DO_VPZ(sve_andv_s, uint32_t, uint32_t, uint32_t, H1_4, -1, DO_AND)
DO_VPZ_D(sve_andv_d, uint64_t, uint64_t, -1, DO_AND)

DO_VPZ(sve_saddv_b, int8_t, uint64_t, uint64_t, H1, 0, DO_ADD)
DO_VPZ(sve_saddv_h, int16_t, uint64_t, uint64_t, H1_2, 0, DO_ADD)
DO_VPZ(sve_saddv_s, int32_t, uint64_t, uint64_t, H1_4, 0, DO_ADD)

DO_VPZ(sve_uaddv_b, uint8_t, uint64_t, uint64_t, H1, 0, DO_ADD)
DO_VPZ(sve_uaddv_h, uint16_t, uint64_t, uint64_t, H1_2, 0, DO_ADD)
DO_VPZ(sve_uaddv_s, uint32_t, uint64_t, uint64_t, H1_4, 0, DO_ADD)
DO_VPZ_D(sve_uaddv_d, uint64_t, uint64_t, 0, DO_ADD)

DO_VPZ(sve_smaxv_b, int8_t, int8_t, uint8_t, H1, INT8_MIN, DO_MAX)
DO_VPZ(sve_smaxv_h, int16_t, int16_t, uint16_t, H1_2, INT16_MIN, DO_MAX)
DO_VPZ(sve_smaxv_s, int32_t, int32_t, uint32_t, H1_4, INT32_MIN, DO_MAX)
DO_VPZ_D(sve_smaxv_d, int64_t, int64_t, INT64_MIN, DO_MAX)

DO_VPZ(sve_umaxv_b, uint8_t, uint8_t, uint8_t, H1, 0, DO_MAX)
DO_VPZ(sve_umaxv_h, uint16_t, uint16_t, uint16_t, H1_2, 0, DO_MAX)
DO_VPZ(sve_umaxv_s, uint32_t, uint32_t, uint32_t, H1_4, 0, DO_MAX)
DO_VPZ_D(sve_umaxv_d, uint64_t, uint64_t, 0, DO_MAX)

DO_VPZ(sve_sminv_b, int8_t, int8_t, uint8_t, H1, INT8_MAX, DO_MIN)
DO_VPZ(sve_sminv_h, int16_t, int16_t, uint16_t, H1_2, INT16_MAX, DO_MIN)
DO_VPZ(sve_sminv_s, int32_t, int32_t, uint32_t, H1_4, INT32_MAX, DO_MIN)
DO_VPZ_D(sve_sminv_d, int64_t, int64_t, INT64_MAX, DO_MIN)

DO_VPZ(sve_uminv_b, uint8_t, uint8_t, uint8_t, H1, -1, DO_MIN)
DO_VPZ(sve_uminv_h, uint16_t, uint16_t, uint16_t, H1_2, -1, DO_MIN)
DO_VPZ(sve_uminv_s, uint32_t, uint32_t, uint32_t, H1_4, -1, DO_MIN)
DO_VPZ_D(sve_uminv_d, uint64_t, uint64_t, -1, DO_MIN)

#undef DO_VPZ
#undef DO_VPZ_D

/* Two vector operand, one scalar operand, unpredicated.  */
#define DO_ZZI(NAME, TYPE, OP)                                       \
void HELPER(NAME)(void *vd, void *vn, uint64_t s64, uint32_t desc)   \
{                                                                    \
    intptr_t i, opr_sz = simd_oprsz(desc) / sizeof(TYPE);            \
    TYPE s = s64, *d = vd, *n = vn;                                  \
    for (i = 0; i < opr_sz; ++i) {                                   \
        d[i] = OP(n[i], s);                                          \
    }                                                                \
}

#define DO_SUBR(X, Y)   (Y - X)

DO_ZZI(sve_subri_b, uint8_t, DO_SUBR)
DO_ZZI(sve_subri_h, uint16_t, DO_SUBR)
DO_ZZI(sve_subri_s, uint32_t, DO_SUBR)
DO_ZZI(sve_subri_d, uint64_t, DO_SUBR)

DO_ZZI(sve_smaxi_b, int8_t, DO_MAX)
DO_ZZI(sve_smaxi_h, int16_t, DO_MAX)
DO_ZZI(sve_smaxi_s, int32_t, DO_MAX)
DO_ZZI(sve_smaxi_d, int64_t, DO_MAX)

DO_ZZI(sve_smini_b, int8_t, DO_MIN)
DO_ZZI(sve_smini_h, int16_t, DO_MIN)
DO_ZZI(sve_smini_s, int32_t, DO_MIN)
DO_ZZI(sve_smini_d, int64_t, DO_MIN)

DO_ZZI(sve_umaxi_b, uint8_t, DO_MAX)
DO_ZZI(sve_umaxi_h, uint16_t, DO_MAX)
DO_ZZI(sve_umaxi_s, uint32_t, DO_MAX)
DO_ZZI(sve_umaxi_d, uint64_t, DO_MAX)

DO_ZZI(sve_umini_b, uint8_t, DO_MIN)
DO_ZZI(sve_umini_h, uint16_t, DO_MIN)
DO_ZZI(sve_umini_s, uint32_t, DO_MIN)
DO_ZZI(sve_umini_d, uint64_t, DO_MIN)

#undef DO_ZZI

#undef DO_AND
#undef DO_ORR
#undef DO_EOR
#undef DO_BIC
#undef DO_ADD
#undef DO_SUB
#undef DO_MAX
#undef DO_MIN
#undef DO_ABD
#undef DO_MUL
#undef DO_DIV
#undef DO_ASR
#undef DO_LSR
#undef DO_LSL
#undef DO_SUBR

/* Similar to the ARM LastActiveElement pseudocode function, except the
   result is multiplied by the element size.  This includes the not found
   indication; e.g. not found for esz=3 is -8.  */
static intptr_t last_active_element(uint64_t *g, intptr_t words, intptr_t esz)
{
    uint64_t mask = pred_esz_masks[esz];
    intptr_t i = words;

    do {
        uint64_t this_g = g[--i] & mask;
        if (this_g) {
            return i * 64 + (63 - clz64(this_g));
        }
    } while (i > 0);
    return (intptr_t)-1 << esz;
}

uint32_t HELPER(sve_pfirst)(void *vd, void *vg, uint32_t pred_desc)
{
    intptr_t words = DIV_ROUND_UP(FIELD_EX32(pred_desc, PREDDESC, OPRSZ), 8);
    uint32_t flags = PREDTEST_INIT;
    uint64_t *d = vd, *g = vg;
    intptr_t i = 0;

    do {
        uint64_t this_d = d[i];
        uint64_t this_g = g[i];

        if (this_g) {
            if (!(flags & 4)) {
                /* Set in D the first bit of G.  */
                this_d |= this_g & -this_g;
                d[i] = this_d;
            }
            flags = iter_predtest_fwd(this_d, this_g, flags);
        }
    } while (++i < words);

    return flags;
}

uint32_t HELPER(sve_pnext)(void *vd, void *vg, uint32_t pred_desc)
{
    intptr_t words = DIV_ROUND_UP(FIELD_EX32(pred_desc, PREDDESC, OPRSZ), 8);
    intptr_t esz = FIELD_EX32(pred_desc, PREDDESC, ESZ);
    uint32_t flags = PREDTEST_INIT;
    uint64_t *d = vd, *g = vg, esz_mask;
    intptr_t i, next;

    next = last_active_element(vd, words, esz) + (1 << esz);
    esz_mask = pred_esz_masks[esz];

    /* Similar to the pseudocode for pnext, but scaled by ESZ
       so that we find the correct bit.  */
    if (next < words * 64) {
        uint64_t mask = -1;

        if (next & 63) {
            mask = ~((1ull << (next & 63)) - 1);
            next &= -64;
        }
        do {
            uint64_t this_g = g[next / 64] & esz_mask & mask;
            if (this_g != 0) {
                next = (next & -64) + ctz64(this_g);
                break;
            }
            next += 64;
            mask = -1;
        } while (next < words * 64);
    }

    i = 0;
    do {
        uint64_t this_d = 0;
        if (i == next / 64) {
            this_d = 1ull << (next & 63);
        }
        d[i] = this_d;
        flags = iter_predtest_fwd(this_d, g[i] & esz_mask, flags);
    } while (++i < words);

    return flags;
}

/*
 * Copy Zn into Zd, and store zero into inactive elements.
 * If inv, store zeros into the active elements.
 */
void HELPER(sve_movz_b)(void *vd, void *vn, void *vg, uint32_t desc)
{
    intptr_t i, opr_sz = simd_oprsz(desc) / 8;
    uint64_t inv = -(uint64_t)(simd_data(desc) & 1);
    uint64_t *d = vd, *n = vn;
    uint8_t *pg = vg;

    for (i = 0; i < opr_sz; i += 1) {
        d[i] = n[i] & (expand_pred_b(pg[H1(i)]) ^ inv);
    }
}

void HELPER(sve_movz_h)(void *vd, void *vn, void *vg, uint32_t desc)
{
    intptr_t i, opr_sz = simd_oprsz(desc) / 8;
    uint64_t inv = -(uint64_t)(simd_data(desc) & 1);
    uint64_t *d = vd, *n = vn;
    uint8_t *pg = vg;

    for (i = 0; i < opr_sz; i += 1) {
        d[i] = n[i] & (expand_pred_h(pg[H1(i)]) ^ inv);
    }
}

void HELPER(sve_movz_s)(void *vd, void *vn, void *vg, uint32_t desc)
{
    intptr_t i, opr_sz = simd_oprsz(desc) / 8;
    uint64_t inv = -(uint64_t)(simd_data(desc) & 1);
    uint64_t *d = vd, *n = vn;
    uint8_t *pg = vg;

    for (i = 0; i < opr_sz; i += 1) {
        d[i] = n[i] & (expand_pred_s(pg[H1(i)]) ^ inv);
    }
}

void HELPER(sve_movz_d)(void *vd, void *vn, void *vg, uint32_t desc)
{
    intptr_t i, opr_sz = simd_oprsz(desc) / 8;
    uint64_t *d = vd, *n = vn;
    uint8_t *pg = vg;
    uint8_t inv = simd_data(desc);

    for (i = 0; i < opr_sz; i += 1) {