LXR qemu/target/i386/ops_sse

   1/*
   2 *  MMX/3DNow!/SSE/SSE2/SSE3/SSSE3/SSE4/PNI support
   3 *
   4 *  Copyright (c) 2005 Fabrice Bellard
   5 *
   6 * This library is free software; you can redistribute it and/or
   7 * modify it under the terms of the GNU Lesser General Public
   8 * License as published by the Free Software Foundation; either
   9 * version 2.1 of the License, or (at your option) any later version.
  10 *
  11 * This library is distributed in the hope that it will be useful,
  12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14 * Lesser General Public License for more details.
  15 *
  16 * You should have received a copy of the GNU Lesser General Public
  17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  18 */
  19#if SHIFT == 0
  20#define Reg MMXReg
  21#define SUFFIX _mmx
  22#else
  23#define Reg ZMMReg
  24#if SHIFT == 1
  25#define SUFFIX _xmm
  26#else
  27#define SUFFIX _ymm
  28#endif
  29#endif
  30
  31#define dh_alias_Reg ptr
  32#define dh_alias_ZMMReg ptr
  33#define dh_alias_MMXReg ptr
  34#define dh_ctype_Reg Reg *
  35#define dh_ctype_ZMMReg ZMMReg *
  36#define dh_ctype_MMXReg MMXReg *
  37#define dh_typecode_Reg dh_typecode_ptr
  38#define dh_typecode_ZMMReg dh_typecode_ptr
  39#define dh_typecode_MMXReg dh_typecode_ptr
  40
  41DEF_HELPER_4(glue(psrlw, SUFFIX), void, env, Reg, Reg, Reg)
  42DEF_HELPER_4(glue(psraw, SUFFIX), void, env, Reg, Reg, Reg)
  43DEF_HELPER_4(glue(psllw, SUFFIX), void, env, Reg, Reg, Reg)
  44DEF_HELPER_4(glue(psrld, SUFFIX), void, env, Reg, Reg, Reg)
  45DEF_HELPER_4(glue(psrad, SUFFIX), void, env, Reg, Reg, Reg)
  46DEF_HELPER_4(glue(pslld, SUFFIX), void, env, Reg, Reg, Reg)
  47DEF_HELPER_4(glue(psrlq, SUFFIX), void, env, Reg, Reg, Reg)
  48DEF_HELPER_4(glue(psllq, SUFFIX), void, env, Reg, Reg, Reg)
  49
  50#if SHIFT >= 1
  51DEF_HELPER_4(glue(psrldq, SUFFIX), void, env, Reg, Reg, Reg)
  52DEF_HELPER_4(glue(pslldq, SUFFIX), void, env, Reg, Reg, Reg)
  53#endif
  54
  55#define SSE_HELPER_B(name, F)\
  56    DEF_HELPER_4(glue(name, SUFFIX), void, env, Reg, Reg, Reg)
  57
  58#define SSE_HELPER_W(name, F)\
  59    DEF_HELPER_4(glue(name, SUFFIX), void, env, Reg, Reg, Reg)
  60
  61#define SSE_HELPER_L(name, F)\
  62    DEF_HELPER_4(glue(name, SUFFIX), void, env, Reg, Reg, Reg)
  63
  64#define SSE_HELPER_Q(name, F)\
  65    DEF_HELPER_4(glue(name, SUFFIX), void, env, Reg, Reg, Reg)
  66
  67#if SHIFT == 0
  68DEF_HELPER_3(glue(pmulhrw, SUFFIX), void, env, Reg, Reg)
  69#endif
  70SSE_HELPER_W(pmulhuw, FMULHUW)
  71SSE_HELPER_W(pmulhw, FMULHW)
  72
  73SSE_HELPER_B(pavgb, FAVG)
  74SSE_HELPER_W(pavgw, FAVG)
  75
  76DEF_HELPER_4(glue(pmuludq, SUFFIX), void, env, Reg, Reg, Reg)
  77DEF_HELPER_4(glue(pmaddwd, SUFFIX), void, env, Reg, Reg, Reg)
  78
  79DEF_HELPER_4(glue(psadbw, SUFFIX), void, env, Reg, Reg, Reg)
  80#if SHIFT < 2
  81DEF_HELPER_4(glue(maskmov, SUFFIX), void, env, Reg, Reg, tl)
  82#endif
  83
  84#if SHIFT == 0
  85DEF_HELPER_3(glue(pshufw, SUFFIX), void, Reg, Reg, int)
  86#else
  87DEF_HELPER_3(glue(pshufd, SUFFIX), void, Reg, Reg, int)
  88DEF_HELPER_3(glue(pshuflw, SUFFIX), void, Reg, Reg, int)
  89DEF_HELPER_3(glue(pshufhw, SUFFIX), void, Reg, Reg, int)
  90#endif
  91
  92#if SHIFT >= 1
  93/* FPU ops */
  94/* XXX: not accurate */
  95
  96#define SSE_HELPER_P4(name)                                             \
  97    DEF_HELPER_4(glue(name ## ps, SUFFIX), void, env, Reg, Reg, Reg)    \
  98    DEF_HELPER_4(glue(name ## pd, SUFFIX), void, env, Reg, Reg, Reg)
  99
 100#define SSE_HELPER_P3(name, ...)                                        \
 101    DEF_HELPER_3(glue(name ## ps, SUFFIX), void, env, Reg, Reg)         \
 102    DEF_HELPER_3(glue(name ## pd, SUFFIX), void, env, Reg, Reg)
 103
 104#if SHIFT == 1
 105#define SSE_HELPER_S4(name)                                             \
 106    SSE_HELPER_P4(name)                                                 \
 107    DEF_HELPER_4(name ## ss, void, env, Reg, Reg, Reg)                  \
 108    DEF_HELPER_4(name ## sd, void, env, Reg, Reg, Reg)
 109#define SSE_HELPER_S3(name)                                             \
 110    SSE_HELPER_P3(name)                                                 \
 111    DEF_HELPER_4(name ## ss, void, env, Reg, Reg, Reg)                  \
 112    DEF_HELPER_4(name ## sd, void, env, Reg, Reg, Reg)
 113#else
 114#define SSE_HELPER_S4(name, ...) SSE_HELPER_P4(name)
 115#define SSE_HELPER_S3(name, ...) SSE_HELPER_P3(name)
 116#endif
 117
 118DEF_HELPER_4(glue(shufps, SUFFIX), void, Reg, Reg, Reg, int)
 119DEF_HELPER_4(glue(shufpd, SUFFIX), void, Reg, Reg, Reg, int)
 120
 121SSE_HELPER_S4(add)
 122SSE_HELPER_S4(sub)
 123SSE_HELPER_S4(mul)
 124SSE_HELPER_S4(div)
 125SSE_HELPER_S4(min)
 126SSE_HELPER_S4(max)
 127
 128SSE_HELPER_S3(sqrt)
 129
 130DEF_HELPER_3(glue(cvtps2pd, SUFFIX), void, env, Reg, Reg)
 131DEF_HELPER_3(glue(cvtpd2ps, SUFFIX), void, env, Reg, Reg)
 132DEF_HELPER_3(glue(cvtdq2ps, SUFFIX), void, env, Reg, Reg)
 133DEF_HELPER_3(glue(cvtdq2pd, SUFFIX), void, env, Reg, Reg)
 134
 135DEF_HELPER_3(glue(cvtps2dq, SUFFIX), void, env, ZMMReg, ZMMReg)
 136DEF_HELPER_3(glue(cvtpd2dq, SUFFIX), void, env, ZMMReg, ZMMReg)
 137
 138DEF_HELPER_3(glue(cvttps2dq, SUFFIX), void, env, ZMMReg, ZMMReg)
 139DEF_HELPER_3(glue(cvttpd2dq, SUFFIX), void, env, ZMMReg, ZMMReg)
 140
 141#if SHIFT == 1
 142DEF_HELPER_4(cvtss2sd, void, env, Reg, Reg, Reg)
 143DEF_HELPER_4(cvtsd2ss, void, env, Reg, Reg, Reg)
 144DEF_HELPER_3(cvtpi2ps, void, env, ZMMReg, MMXReg)
 145DEF_HELPER_3(cvtpi2pd, void, env, ZMMReg, MMXReg)
 146DEF_HELPER_3(cvtsi2ss, void, env, ZMMReg, i32)
 147DEF_HELPER_3(cvtsi2sd, void, env, ZMMReg, i32)
 148
 149#ifdef TARGET_X86_64
 150DEF_HELPER_3(cvtsq2ss, void, env, ZMMReg, i64)
 151DEF_HELPER_3(cvtsq2sd, void, env, ZMMReg, i64)
 152#endif
 153
 154DEF_HELPER_3(cvtps2pi, void, env, MMXReg, ZMMReg)
 155DEF_HELPER_3(cvtpd2pi, void, env, MMXReg, ZMMReg)
 156DEF_HELPER_2(cvtss2si, s32, env, ZMMReg)
 157DEF_HELPER_2(cvtsd2si, s32, env, ZMMReg)
 158#ifdef TARGET_X86_64
 159DEF_HELPER_2(cvtss2sq, s64, env, ZMMReg)
 160DEF_HELPER_2(cvtsd2sq, s64, env, ZMMReg)
 161#endif
 162
 163DEF_HELPER_3(cvttps2pi, void, env, MMXReg, ZMMReg)
 164DEF_HELPER_3(cvttpd2pi, void, env, MMXReg, ZMMReg)
 165DEF_HELPER_2(cvttss2si, s32, env, ZMMReg)
 166DEF_HELPER_2(cvttsd2si, s32, env, ZMMReg)
 167#ifdef TARGET_X86_64
 168DEF_HELPER_2(cvttss2sq, s64, env, ZMMReg)
 169DEF_HELPER_2(cvttsd2sq, s64, env, ZMMReg)
 170#endif
 171#endif
 172
 173DEF_HELPER_3(glue(rsqrtps, SUFFIX), void, env, ZMMReg, ZMMReg)
 174DEF_HELPER_3(glue(rcpps, SUFFIX), void, env, ZMMReg, ZMMReg)
 175
 176#if SHIFT == 1
 177DEF_HELPER_4(rsqrtss, void, env, ZMMReg, ZMMReg, ZMMReg)
 178DEF_HELPER_4(rcpss, void, env, ZMMReg, ZMMReg, ZMMReg)
 179DEF_HELPER_3(extrq_r, void, env, ZMMReg, ZMMReg)
 180DEF_HELPER_4(extrq_i, void, env, ZMMReg, int, int)
 181DEF_HELPER_3(insertq_r, void, env, ZMMReg, ZMMReg)
 182DEF_HELPER_5(insertq_i, void, env, ZMMReg, ZMMReg, int, int)
 183#endif
 184
 185SSE_HELPER_P4(hadd)
 186SSE_HELPER_P4(hsub)
 187SSE_HELPER_P4(addsub)
 188
 189#define SSE_HELPER_CMP(name, F, C) SSE_HELPER_S4(name)
 190
 191SSE_HELPER_CMP(cmpeq, FPU_CMPQ, FPU_EQ)
 192SSE_HELPER_CMP(cmplt, FPU_CMPS, FPU_LT)
 193SSE_HELPER_CMP(cmple, FPU_CMPS, FPU_LE)
 194SSE_HELPER_CMP(cmpunord, FPU_CMPQ,  FPU_UNORD)
 195SSE_HELPER_CMP(cmpneq, FPU_CMPQ, !FPU_EQ)
 196SSE_HELPER_CMP(cmpnlt, FPU_CMPS, !FPU_LT)
 197SSE_HELPER_CMP(cmpnle, FPU_CMPS, !FPU_LE)
 198SSE_HELPER_CMP(cmpord, FPU_CMPQ, !FPU_UNORD)
 199
 200SSE_HELPER_CMP(cmpequ, FPU_CMPQ, FPU_EQU)
 201SSE_HELPER_CMP(cmpnge, FPU_CMPS, !FPU_GE)
 202SSE_HELPER_CMP(cmpngt, FPU_CMPS, !FPU_GT)
 203SSE_HELPER_CMP(cmpfalse, FPU_CMPQ,  FPU_FALSE)
 204SSE_HELPER_CMP(cmpnequ, FPU_CMPQ, !FPU_EQU)
 205SSE_HELPER_CMP(cmpge, FPU_CMPS, FPU_GE)
 206SSE_HELPER_CMP(cmpgt, FPU_CMPS, FPU_GT)
 207SSE_HELPER_CMP(cmptrue, FPU_CMPQ,  !FPU_FALSE)
 208
 209SSE_HELPER_CMP(cmpeqs, FPU_CMPS, FPU_EQ)
 210SSE_HELPER_CMP(cmpltq, FPU_CMPQ, FPU_LT)
 211SSE_HELPER_CMP(cmpleq, FPU_CMPQ, FPU_LE)
 212SSE_HELPER_CMP(cmpunords, FPU_CMPS,  FPU_UNORD)
 213SSE_HELPER_CMP(cmpneqq, FPU_CMPS, !FPU_EQ)
 214SSE_HELPER_CMP(cmpnltq, FPU_CMPQ, !FPU_LT)
 215SSE_HELPER_CMP(cmpnleq, FPU_CMPQ, !FPU_LE)
 216SSE_HELPER_CMP(cmpords, FPU_CMPS, !FPU_UNORD)
 217
 218SSE_HELPER_CMP(cmpequs, FPU_CMPS, FPU_EQU)
 219SSE_HELPER_CMP(cmpngeq, FPU_CMPQ, !FPU_GE)
 220SSE_HELPER_CMP(cmpngtq, FPU_CMPQ, !FPU_GT)
 221SSE_HELPER_CMP(cmpfalses, FPU_CMPS,  FPU_FALSE)
 222SSE_HELPER_CMP(cmpnequs, FPU_CMPS, !FPU_EQU)
 223SSE_HELPER_CMP(cmpgeq, FPU_CMPQ, FPU_GE)
 224SSE_HELPER_CMP(cmpgtq, FPU_CMPQ, FPU_GT)
 225SSE_HELPER_CMP(cmptrues, FPU_CMPS,  !FPU_FALSE)
 226
 227#if SHIFT == 1
 228DEF_HELPER_3(ucomiss, void, env, Reg, Reg)
 229DEF_HELPER_3(comiss, void, env, Reg, Reg)
 230DEF_HELPER_3(ucomisd, void, env, Reg, Reg)
 231DEF_HELPER_3(comisd, void, env, Reg, Reg)
 232#endif
 233
 234DEF_HELPER_2(glue(movmskps, SUFFIX), i32, env, Reg)
 235DEF_HELPER_2(glue(movmskpd, SUFFIX), i32, env, Reg)
 236#endif
 237
 238DEF_HELPER_4(glue(packsswb, SUFFIX), void, env, Reg, Reg, Reg)
 239DEF_HELPER_4(glue(packuswb, SUFFIX), void, env, Reg, Reg, Reg)
 240DEF_HELPER_4(glue(packssdw, SUFFIX), void, env, Reg, Reg, Reg)
 241#define UNPCK_OP(name, base)                                       \
 242    DEF_HELPER_4(glue(punpck ## name ## bw, SUFFIX), void, env, Reg, Reg, Reg) \
 243    DEF_HELPER_4(glue(punpck ## name ## wd, SUFFIX), void, env, Reg, Reg, Reg) \
 244    DEF_HELPER_4(glue(punpck ## name ## dq, SUFFIX), void, env, Reg, Reg, Reg)
 245
 246UNPCK_OP(l, 0)
 247UNPCK_OP(h, 1)
 248
 249#if SHIFT >= 1
 250DEF_HELPER_4(glue(punpcklqdq, SUFFIX), void, env, Reg, Reg, Reg)
 251DEF_HELPER_4(glue(punpckhqdq, SUFFIX), void, env, Reg, Reg, Reg)
 252#endif
 253
 254/* 3DNow! float ops */
 255#if SHIFT == 0
 256DEF_HELPER_3(pi2fd, void, env, MMXReg, MMXReg)
 257DEF_HELPER_3(pi2fw, void, env, MMXReg, MMXReg)
 258DEF_HELPER_3(pf2id, void, env, MMXReg, MMXReg)
 259DEF_HELPER_3(pf2iw, void, env, MMXReg, MMXReg)
 260DEF_HELPER_3(pfacc, void, env, MMXReg, MMXReg)
 261DEF_HELPER_3(pfadd, void, env, MMXReg, MMXReg)
 262DEF_HELPER_3(pfcmpeq, void, env, MMXReg, MMXReg)
 263DEF_HELPER_3(pfcmpge, void, env, MMXReg, MMXReg)
 264DEF_HELPER_3(pfcmpgt, void, env, MMXReg, MMXReg)
 265DEF_HELPER_3(pfmax, void, env, MMXReg, MMXReg)
 266DEF_HELPER_3(pfmin, void, env, MMXReg, MMXReg)
 267DEF_HELPER_3(pfmul, void, env, MMXReg, MMXReg)
 268DEF_HELPER_3(pfnacc, void, env, MMXReg, MMXReg)
 269DEF_HELPER_3(pfpnacc, void, env, MMXReg, MMXReg)
 270DEF_HELPER_3(pfrcp, void, env, MMXReg, MMXReg)
 271DEF_HELPER_3(pfrsqrt, void, env, MMXReg, MMXReg)
 272DEF_HELPER_3(pfsub, void, env, MMXReg, MMXReg)
 273DEF_HELPER_3(pfsubr, void, env, MMXReg, MMXReg)
 274DEF_HELPER_3(pswapd, void, env, MMXReg, MMXReg)
 275#endif
 276
 277/* SSSE3 op helpers */
 278DEF_HELPER_4(glue(phaddw, SUFFIX), void, env, Reg, Reg, Reg)
 279DEF_HELPER_4(glue(phaddd, SUFFIX), void, env, Reg, Reg, Reg)
 280DEF_HELPER_4(glue(phaddsw, SUFFIX), void, env, Reg, Reg, Reg)
 281DEF_HELPER_4(glue(phsubw, SUFFIX), void, env, Reg, Reg, Reg)
 282DEF_HELPER_4(glue(phsubd, SUFFIX), void, env, Reg, Reg, Reg)
 283DEF_HELPER_4(glue(phsubsw, SUFFIX), void, env, Reg, Reg, Reg)
 284DEF_HELPER_4(glue(pmaddubsw, SUFFIX), void, env, Reg, Reg, Reg)
 285DEF_HELPER_4(glue(pmulhrsw, SUFFIX), void, env, Reg, Reg, Reg)
 286DEF_HELPER_4(glue(pshufb, SUFFIX), void, env, Reg, Reg, Reg)
 287DEF_HELPER_4(glue(psignb, SUFFIX), void, env, Reg, Reg, Reg)
 288DEF_HELPER_4(glue(psignw, SUFFIX), void, env, Reg, Reg, Reg)
 289DEF_HELPER_4(glue(psignd, SUFFIX), void, env, Reg, Reg, Reg)
 290DEF_HELPER_5(glue(palignr, SUFFIX), void, env, Reg, Reg, Reg, i32)
 291
 292/* SSE4.1 op helpers */
 293#if SHIFT >= 1
 294DEF_HELPER_5(glue(pblendvb, SUFFIX), void, env, Reg, Reg, Reg, Reg)
 295DEF_HELPER_5(glue(blendvps, SUFFIX), void, env, Reg, Reg, Reg, Reg)
 296DEF_HELPER_5(glue(blendvpd, SUFFIX), void, env, Reg, Reg, Reg, Reg)
 297DEF_HELPER_3(glue(ptest, SUFFIX), void, env, Reg, Reg)
 298DEF_HELPER_3(glue(pmovsxbw, SUFFIX), void, env, Reg, Reg)
 299DEF_HELPER_3(glue(pmovsxbd, SUFFIX), void, env, Reg, Reg)
 300DEF_HELPER_3(glue(pmovsxbq, SUFFIX), void, env, Reg, Reg)
 301DEF_HELPER_3(glue(pmovsxwd, SUFFIX), void, env, Reg, Reg)
 302DEF_HELPER_3(glue(pmovsxwq, SUFFIX), void, env, Reg, Reg)
 303DEF_HELPER_3(glue(pmovsxdq, SUFFIX), void, env, Reg, Reg)
 304DEF_HELPER_3(glue(pmovzxbw, SUFFIX), void, env, Reg, Reg)
 305DEF_HELPER_3(glue(pmovzxbd, SUFFIX), void, env, Reg, Reg)
 306DEF_HELPER_3(glue(pmovzxbq, SUFFIX), void, env, Reg, Reg)
 307DEF_HELPER_3(glue(pmovzxwd, SUFFIX), void, env, Reg, Reg)
 308DEF_HELPER_3(glue(pmovzxwq, SUFFIX), void, env, Reg, Reg)
 309DEF_HELPER_3(glue(pmovzxdq, SUFFIX), void, env, Reg, Reg)
 310DEF_HELPER_3(glue(pmovsldup, SUFFIX), void, env, Reg, Reg)
 311DEF_HELPER_3(glue(pmovshdup, SUFFIX), void, env, Reg, Reg)
 312DEF_HELPER_3(glue(pmovdldup, SUFFIX), void, env, Reg, Reg)
 313DEF_HELPER_4(glue(pmuldq, SUFFIX), void, env, Reg, Reg, Reg)
 314DEF_HELPER_4(glue(packusdw, SUFFIX), void, env, Reg, Reg, Reg)
 315#if SHIFT == 1
 316DEF_HELPER_3(glue(phminposuw, SUFFIX), void, env, Reg, Reg)
 317#endif
 318DEF_HELPER_4(glue(roundps, SUFFIX), void, env, Reg, Reg, i32)
 319DEF_HELPER_4(glue(roundpd, SUFFIX), void, env, Reg, Reg, i32)
 320#if SHIFT == 1
 321DEF_HELPER_5(roundss_xmm, void, env, Reg, Reg, Reg, i32)
 322DEF_HELPER_5(roundsd_xmm, void, env, Reg, Reg, Reg, i32)
 323#endif
 324DEF_HELPER_5(glue(blendps, SUFFIX), void, env, Reg, Reg, Reg, i32)
 325DEF_HELPER_5(glue(blendpd, SUFFIX), void, env, Reg, Reg, Reg, i32)
 326DEF_HELPER_5(glue(pblendw, SUFFIX), void, env, Reg, Reg, Reg, i32)
 327DEF_HELPER_5(glue(dpps, SUFFIX), void, env, Reg, Reg, Reg, i32)
 328#if SHIFT == 1
 329DEF_HELPER_5(glue(dppd, SUFFIX), void, env, Reg, Reg, Reg, i32)
 330#endif
 331DEF_HELPER_5(glue(mpsadbw, SUFFIX), void, env, Reg, Reg, Reg, i32)
 332#endif
 333
 334/* SSE4.2 op helpers */
 335#if SHIFT == 1
 336DEF_HELPER_4(glue(pcmpestri, SUFFIX), void, env, Reg, Reg, i32)
 337DEF_HELPER_4(glue(pcmpestrm, SUFFIX), void, env, Reg, Reg, i32)
 338DEF_HELPER_4(glue(pcmpistri, SUFFIX), void, env, Reg, Reg, i32)
 339DEF_HELPER_4(glue(pcmpistrm, SUFFIX), void, env, Reg, Reg, i32)
 340DEF_HELPER_3(crc32, tl, i32, tl, i32)
 341#endif
 342
 343/* AES-NI op helpers */
 344#if SHIFT >= 1
 345DEF_HELPER_4(glue(aesdec, SUFFIX), void, env, Reg, Reg, Reg)
 346DEF_HELPER_4(glue(aesdeclast, SUFFIX), void, env, Reg, Reg, Reg)
 347DEF_HELPER_4(glue(aesenc, SUFFIX), void, env, Reg, Reg, Reg)
 348DEF_HELPER_4(glue(aesenclast, SUFFIX), void, env, Reg, Reg, Reg)
 349#if SHIFT == 1
 350DEF_HELPER_3(glue(aesimc, SUFFIX), void, env, Reg, Reg)
 351DEF_HELPER_4(glue(aeskeygenassist, SUFFIX), void, env, Reg, Reg, i32)
 352#endif
 353DEF_HELPER_5(glue(pclmulqdq, SUFFIX), void, env, Reg, Reg, Reg, i32)
 354#endif
 355
 356/* F16C helpers */
 357#if SHIFT >= 1
 358DEF_HELPER_3(glue(cvtph2ps, SUFFIX), void, env, Reg, Reg)
 359DEF_HELPER_4(glue(cvtps2ph, SUFFIX), void, env, Reg, Reg, int)
 360#endif
 361
 362/* FMA3 helpers */
 363#if SHIFT == 1
 364DEF_HELPER_6(fma4ss, void, env, Reg, Reg, Reg, Reg, int)
 365DEF_HELPER_6(fma4sd, void, env, Reg, Reg, Reg, Reg, int)
 366#endif
 367
 368#if SHIFT >= 1
 369DEF_HELPER_7(glue(fma4ps, SUFFIX), void, env, Reg, Reg, Reg, Reg, int, int)
 370DEF_HELPER_7(glue(fma4pd, SUFFIX), void, env, Reg, Reg, Reg, Reg, int, int)
 371#endif
 372
 373/* AVX helpers */
 374#if SHIFT >= 1
 375DEF_HELPER_4(glue(vpermilpd, SUFFIX), void, env, Reg, Reg, Reg)
 376DEF_HELPER_4(glue(vpermilps, SUFFIX), void, env, Reg, Reg, Reg)
 377DEF_HELPER_3(glue(vpermilpd_imm, SUFFIX), void, Reg, Reg, i32)
 378DEF_HELPER_3(glue(vpermilps_imm, SUFFIX), void, Reg, Reg, i32)
 379DEF_HELPER_4(glue(vpsrlvd, SUFFIX), void, env, Reg, Reg, Reg)
 380DEF_HELPER_4(glue(vpsravd, SUFFIX), void, env, Reg, Reg, Reg)
 381DEF_HELPER_4(glue(vpsllvd, SUFFIX), void, env, Reg, Reg, Reg)
 382DEF_HELPER_4(glue(vpsrlvq, SUFFIX), void, env, Reg, Reg, Reg)
 383DEF_HELPER_4(glue(vpsravq, SUFFIX), void, env, Reg, Reg, Reg)
 384DEF_HELPER_4(glue(vpsllvq, SUFFIX), void, env, Reg, Reg, Reg)
 385DEF_HELPER_3(glue(vtestps, SUFFIX), void, env, Reg, Reg)
 386DEF_HELPER_3(glue(vtestpd, SUFFIX), void, env, Reg, Reg)
 387DEF_HELPER_4(glue(vpmaskmovd_st, SUFFIX), void, env, Reg, Reg, tl)
 388DEF_HELPER_4(glue(vpmaskmovq_st, SUFFIX), void, env, Reg, Reg, tl)
 389DEF_HELPER_4(glue(vpmaskmovd, SUFFIX), void, env, Reg, Reg, Reg)
 390DEF_HELPER_4(glue(vpmaskmovq, SUFFIX), void, env, Reg, Reg, Reg)
 391DEF_HELPER_6(glue(vpgatherdd, SUFFIX), void, env, Reg, Reg, Reg, tl, i32)
 392DEF_HELPER_6(glue(vpgatherdq, SUFFIX), void, env, Reg, Reg, Reg, tl, i32)
 393DEF_HELPER_6(glue(vpgatherqd, SUFFIX), void, env, Reg, Reg, Reg, tl, i32)
 394DEF_HELPER_6(glue(vpgatherqq, SUFFIX), void, env, Reg, Reg, Reg, tl, i32)
 395#if SHIFT == 2
 396DEF_HELPER_3(vpermd_ymm, void, Reg, Reg, Reg)
 397DEF_HELPER_4(vpermdq_ymm, void, Reg, Reg, Reg, i32)
 398DEF_HELPER_3(vpermq_ymm, void, Reg, Reg, i32)
 399#endif
 400#endif
 401
 402#undef SHIFT
 403#undef Reg
 404#undef SUFFIX
 405
 406#undef SSE_HELPER_B
 407#undef SSE_HELPER_W
 408#undef SSE_HELPER_L
 409#undef SSE_HELPER_Q
 410#undef SSE_HELPER_S3
 411#undef SSE_HELPER_S4
 412#undef SSE_HELPER_P3
 413#undef SSE_HELPER_P4
 414#undef SSE_HELPER_CMP
 415#undef UNPCK_OP
 416