qemu/target/arm/helper-a64.c
<<
>>
Prefs
   1/*
   2 *  AArch64 specific helpers
   3 *
   4 *  Copyright (c) 2013 Alexander Graf <agraf@suse.de>
   5 *
   6 * This library is free software; you can redistribute it and/or
   7 * modify it under the terms of the GNU Lesser General Public
   8 * License as published by the Free Software Foundation; either
   9 * version 2 of the License, or (at your option) any later version.
  10 *
  11 * This library is distributed in the hope that it will be useful,
  12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14 * Lesser General Public License for more details.
  15 *
  16 * You should have received a copy of the GNU Lesser General Public
  17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  18 */
  19
  20#include "qemu/osdep.h"
  21#include "cpu.h"
  22#include "exec/gdbstub.h"
  23#include "exec/helper-proto.h"
  24#include "qemu/host-utils.h"
  25#include "qemu/log.h"
  26#include "sysemu/sysemu.h"
  27#include "qemu/bitops.h"
  28#include "internals.h"
  29#include "qemu/crc32c.h"
  30#include "exec/exec-all.h"
  31#include "exec/cpu_ldst.h"
  32#include "qemu/int128.h"
  33#include "tcg.h"
  34#include <zlib.h> /* For crc32 */
  35
  36/* C2.4.7 Multiply and divide */
  37/* special cases for 0 and LLONG_MIN are mandated by the standard */
  38uint64_t HELPER(udiv64)(uint64_t num, uint64_t den)
  39{
  40    if (den == 0) {
  41        return 0;
  42    }
  43    return num / den;
  44}
  45
  46int64_t HELPER(sdiv64)(int64_t num, int64_t den)
  47{
  48    if (den == 0) {
  49        return 0;
  50    }
  51    if (num == LLONG_MIN && den == -1) {
  52        return LLONG_MIN;
  53    }
  54    return num / den;
  55}
  56
  57uint64_t HELPER(rbit64)(uint64_t x)
  58{
  59    return revbit64(x);
  60}
  61
  62/* Convert a softfloat float_relation_ (as returned by
  63 * the float*_compare functions) to the correct ARM
  64 * NZCV flag state.
  65 */
  66static inline uint32_t float_rel_to_flags(int res)
  67{
  68    uint64_t flags;
  69    switch (res) {
  70    case float_relation_equal:
  71        flags = PSTATE_Z | PSTATE_C;
  72        break;
  73    case float_relation_less:
  74        flags = PSTATE_N;
  75        break;
  76    case float_relation_greater:
  77        flags = PSTATE_C;
  78        break;
  79    case float_relation_unordered:
  80    default:
  81        flags = PSTATE_C | PSTATE_V;
  82        break;
  83    }
  84    return flags;
  85}
  86
  87uint64_t HELPER(vfp_cmps_a64)(float32 x, float32 y, void *fp_status)
  88{
  89    return float_rel_to_flags(float32_compare_quiet(x, y, fp_status));
  90}
  91
  92uint64_t HELPER(vfp_cmpes_a64)(float32 x, float32 y, void *fp_status)
  93{
  94    return float_rel_to_flags(float32_compare(x, y, fp_status));
  95}
  96
  97uint64_t HELPER(vfp_cmpd_a64)(float64 x, float64 y, void *fp_status)
  98{
  99    return float_rel_to_flags(float64_compare_quiet(x, y, fp_status));
 100}
 101
 102uint64_t HELPER(vfp_cmped_a64)(float64 x, float64 y, void *fp_status)
 103{
 104    return float_rel_to_flags(float64_compare(x, y, fp_status));
 105}
 106
 107float32 HELPER(vfp_mulxs)(float32 a, float32 b, void *fpstp)
 108{
 109    float_status *fpst = fpstp;
 110
 111    a = float32_squash_input_denormal(a, fpst);
 112    b = float32_squash_input_denormal(b, fpst);
 113
 114    if ((float32_is_zero(a) && float32_is_infinity(b)) ||
 115        (float32_is_infinity(a) && float32_is_zero(b))) {
 116        /* 2.0 with the sign bit set to sign(A) XOR sign(B) */
 117        return make_float32((1U << 30) |
 118                            ((float32_val(a) ^ float32_val(b)) & (1U << 31)));
 119    }
 120    return float32_mul(a, b, fpst);
 121}
 122
 123float64 HELPER(vfp_mulxd)(float64 a, float64 b, void *fpstp)
 124{
 125    float_status *fpst = fpstp;
 126
 127    a = float64_squash_input_denormal(a, fpst);
 128    b = float64_squash_input_denormal(b, fpst);
 129
 130    if ((float64_is_zero(a) && float64_is_infinity(b)) ||
 131        (float64_is_infinity(a) && float64_is_zero(b))) {
 132        /* 2.0 with the sign bit set to sign(A) XOR sign(B) */
 133        return make_float64((1ULL << 62) |
 134                            ((float64_val(a) ^ float64_val(b)) & (1ULL << 63)));
 135    }
 136    return float64_mul(a, b, fpst);
 137}
 138
 139uint64_t HELPER(simd_tbl)(CPUARMState *env, uint64_t result, uint64_t indices,
 140                          uint32_t rn, uint32_t numregs)
 141{
 142    /* Helper function for SIMD TBL and TBX. We have to do the table
 143     * lookup part for the 64 bits worth of indices we're passed in.
 144     * result is the initial results vector (either zeroes for TBL
 145     * or some guest values for TBX), rn the register number where
 146     * the table starts, and numregs the number of registers in the table.
 147     * We return the results of the lookups.
 148     */
 149    int shift;
 150
 151    for (shift = 0; shift < 64; shift += 8) {
 152        int index = extract64(indices, shift, 8);
 153        if (index < 16 * numregs) {
 154            /* Convert index (a byte offset into the virtual table
 155             * which is a series of 128-bit vectors concatenated)
 156             * into the correct vfp.regs[] element plus a bit offset
 157             * into that element, bearing in mind that the table
 158             * can wrap around from V31 to V0.
 159             */
 160            int elt = (rn * 2 + (index >> 3)) % 64;
 161            int bitidx = (index & 7) * 8;
 162            uint64_t val = extract64(env->vfp.regs[elt], bitidx, 8);
 163
 164            result = deposit64(result, shift, 8, val);
 165        }
 166    }
 167    return result;
 168}
 169
 170/* 64bit/double versions of the neon float compare functions */
 171uint64_t HELPER(neon_ceq_f64)(float64 a, float64 b, void *fpstp)
 172{
 173    float_status *fpst = fpstp;
 174    return -float64_eq_quiet(a, b, fpst);
 175}
 176
 177uint64_t HELPER(neon_cge_f64)(float64 a, float64 b, void *fpstp)
 178{
 179    float_status *fpst = fpstp;
 180    return -float64_le(b, a, fpst);
 181}
 182
 183uint64_t HELPER(neon_cgt_f64)(float64 a, float64 b, void *fpstp)
 184{
 185    float_status *fpst = fpstp;
 186    return -float64_lt(b, a, fpst);
 187}
 188
 189/* Reciprocal step and sqrt step. Note that unlike the A32/T32
 190 * versions, these do a fully fused multiply-add or
 191 * multiply-add-and-halve.
 192 */
 193#define float32_two make_float32(0x40000000)
 194#define float32_three make_float32(0x40400000)
 195#define float32_one_point_five make_float32(0x3fc00000)
 196
 197#define float64_two make_float64(0x4000000000000000ULL)
 198#define float64_three make_float64(0x4008000000000000ULL)
 199#define float64_one_point_five make_float64(0x3FF8000000000000ULL)
 200
 201float32 HELPER(recpsf_f32)(float32 a, float32 b, void *fpstp)
 202{
 203    float_status *fpst = fpstp;
 204
 205    a = float32_squash_input_denormal(a, fpst);
 206    b = float32_squash_input_denormal(b, fpst);
 207
 208    a = float32_chs(a);
 209    if ((float32_is_infinity(a) && float32_is_zero(b)) ||
 210        (float32_is_infinity(b) && float32_is_zero(a))) {
 211        return float32_two;
 212    }
 213    return float32_muladd(a, b, float32_two, 0, fpst);
 214}
 215
 216float64 HELPER(recpsf_f64)(float64 a, float64 b, void *fpstp)
 217{
 218    float_status *fpst = fpstp;
 219
 220    a = float64_squash_input_denormal(a, fpst);
 221    b = float64_squash_input_denormal(b, fpst);
 222
 223    a = float64_chs(a);
 224    if ((float64_is_infinity(a) && float64_is_zero(b)) ||
 225        (float64_is_infinity(b) && float64_is_zero(a))) {
 226        return float64_two;
 227    }
 228    return float64_muladd(a, b, float64_two, 0, fpst);
 229}
 230
 231float32 HELPER(rsqrtsf_f32)(float32 a, float32 b, void *fpstp)
 232{
 233    float_status *fpst = fpstp;
 234
 235    a = float32_squash_input_denormal(a, fpst);
 236    b = float32_squash_input_denormal(b, fpst);
 237
 238    a = float32_chs(a);
 239    if ((float32_is_infinity(a) && float32_is_zero(b)) ||
 240        (float32_is_infinity(b) && float32_is_zero(a))) {
 241        return float32_one_point_five;
 242    }
 243    return float32_muladd(a, b, float32_three, float_muladd_halve_result, fpst);
 244}
 245
 246float64 HELPER(rsqrtsf_f64)(float64 a, float64 b, void *fpstp)
 247{
 248    float_status *fpst = fpstp;
 249
 250    a = float64_squash_input_denormal(a, fpst);
 251    b = float64_squash_input_denormal(b, fpst);
 252
 253    a = float64_chs(a);
 254    if ((float64_is_infinity(a) && float64_is_zero(b)) ||
 255        (float64_is_infinity(b) && float64_is_zero(a))) {
 256        return float64_one_point_five;
 257    }
 258    return float64_muladd(a, b, float64_three, float_muladd_halve_result, fpst);
 259}
 260
 261/* Pairwise long add: add pairs of adjacent elements into
 262 * double-width elements in the result (eg _s8 is an 8x8->16 op)
 263 */
 264uint64_t HELPER(neon_addlp_s8)(uint64_t a)
 265{
 266    uint64_t nsignmask = 0x0080008000800080ULL;
 267    uint64_t wsignmask = 0x8000800080008000ULL;
 268    uint64_t elementmask = 0x00ff00ff00ff00ffULL;
 269    uint64_t tmp1, tmp2;
 270    uint64_t res, signres;
 271
 272    /* Extract odd elements, sign extend each to a 16 bit field */
 273    tmp1 = a & elementmask;
 274    tmp1 ^= nsignmask;
 275    tmp1 |= wsignmask;
 276    tmp1 = (tmp1 - nsignmask) ^ wsignmask;
 277    /* Ditto for the even elements */
 278    tmp2 = (a >> 8) & elementmask;
 279    tmp2 ^= nsignmask;
 280    tmp2 |= wsignmask;
 281    tmp2 = (tmp2 - nsignmask) ^ wsignmask;
 282
 283    /* calculate the result by summing bits 0..14, 16..22, etc,
 284     * and then adjusting the sign bits 15, 23, etc manually.
 285     * This ensures the addition can't overflow the 16 bit field.
 286     */
 287    signres = (tmp1 ^ tmp2) & wsignmask;
 288    res = (tmp1 & ~wsignmask) + (tmp2 & ~wsignmask);
 289    res ^= signres;
 290
 291    return res;
 292}
 293
 294uint64_t HELPER(neon_addlp_u8)(uint64_t a)
 295{
 296    uint64_t tmp;
 297
 298    tmp = a & 0x00ff00ff00ff00ffULL;
 299    tmp += (a >> 8) & 0x00ff00ff00ff00ffULL;
 300    return tmp;
 301}
 302
 303uint64_t HELPER(neon_addlp_s16)(uint64_t a)
 304{
 305    int32_t reslo, reshi;
 306
 307    reslo = (int32_t)(int16_t)a + (int32_t)(int16_t)(a >> 16);
 308    reshi = (int32_t)(int16_t)(a >> 32) + (int32_t)(int16_t)(a >> 48);
 309
 310    return (uint32_t)reslo | (((uint64_t)reshi) << 32);
 311}
 312
 313uint64_t HELPER(neon_addlp_u16)(uint64_t a)
 314{
 315    uint64_t tmp;
 316
 317    tmp = a & 0x0000ffff0000ffffULL;
 318    tmp += (a >> 16) & 0x0000ffff0000ffffULL;
 319    return tmp;
 320}
 321
 322/* Floating-point reciprocal exponent - see FPRecpX in ARM ARM */
 323float32 HELPER(frecpx_f32)(float32 a, void *fpstp)
 324{
 325    float_status *fpst = fpstp;
 326    uint32_t val32, sbit;
 327    int32_t exp;
 328
 329    if (float32_is_any_nan(a)) {
 330        float32 nan = a;
 331        if (float32_is_signaling_nan(a, fpst)) {
 332            float_raise(float_flag_invalid, fpst);
 333            nan = float32_maybe_silence_nan(a, fpst);
 334        }
 335        if (fpst->default_nan_mode) {
 336            nan = float32_default_nan(fpst);
 337        }
 338        return nan;
 339    }
 340
 341    val32 = float32_val(a);
 342    sbit = 0x80000000ULL & val32;
 343    exp = extract32(val32, 23, 8);
 344
 345    if (exp == 0) {
 346        return make_float32(sbit | (0xfe << 23));
 347    } else {
 348        return make_float32(sbit | (~exp & 0xff) << 23);
 349    }
 350}
 351
 352float64 HELPER(frecpx_f64)(float64 a, void *fpstp)
 353{
 354    float_status *fpst = fpstp;
 355    uint64_t val64, sbit;
 356    int64_t exp;
 357
 358    if (float64_is_any_nan(a)) {
 359        float64 nan = a;
 360        if (float64_is_signaling_nan(a, fpst)) {
 361            float_raise(float_flag_invalid, fpst);
 362            nan = float64_maybe_silence_nan(a, fpst);
 363        }
 364        if (fpst->default_nan_mode) {
 365            nan = float64_default_nan(fpst);
 366        }
 367        return nan;
 368    }
 369
 370    val64 = float64_val(a);
 371    sbit = 0x8000000000000000ULL & val64;
 372    exp = extract64(float64_val(a), 52, 11);
 373
 374    if (exp == 0) {
 375        return make_float64(sbit | (0x7feULL << 52));
 376    } else {
 377        return make_float64(sbit | (~exp & 0x7ffULL) << 52);
 378    }
 379}
 380
 381float32 HELPER(fcvtx_f64_to_f32)(float64 a, CPUARMState *env)
 382{
 383    /* Von Neumann rounding is implemented by using round-to-zero
 384     * and then setting the LSB of the result if Inexact was raised.
 385     */
 386    float32 r;
 387    float_status *fpst = &env->vfp.fp_status;
 388    float_status tstat = *fpst;
 389    int exflags;
 390
 391    set_float_rounding_mode(float_round_to_zero, &tstat);
 392    set_float_exception_flags(0, &tstat);
 393    r = float64_to_float32(a, &tstat);
 394    r = float32_maybe_silence_nan(r, &tstat);
 395    exflags = get_float_exception_flags(&tstat);
 396    if (exflags & float_flag_inexact) {
 397        r = make_float32(float32_val(r) | 1);
 398    }
 399    exflags |= get_float_exception_flags(fpst);
 400    set_float_exception_flags(exflags, fpst);
 401    return r;
 402}
 403
 404/* 64-bit versions of the CRC helpers. Note that although the operation
 405 * (and the prototypes of crc32c() and crc32() mean that only the bottom
 406 * 32 bits of the accumulator and result are used, we pass and return
 407 * uint64_t for convenience of the generated code. Unlike the 32-bit
 408 * instruction set versions, val may genuinely have 64 bits of data in it.
 409 * The upper bytes of val (above the number specified by 'bytes') must have
 410 * been zeroed out by the caller.
 411 */
 412uint64_t HELPER(crc32_64)(uint64_t acc, uint64_t val, uint32_t bytes)
 413{
 414    uint8_t buf[8];
 415
 416    stq_le_p(buf, val);
 417
 418    /* zlib crc32 converts the accumulator and output to one's complement.  */
 419    return crc32(acc ^ 0xffffffff, buf, bytes) ^ 0xffffffff;
 420}
 421
 422uint64_t HELPER(crc32c_64)(uint64_t acc, uint64_t val, uint32_t bytes)
 423{
 424    uint8_t buf[8];
 425
 426    stq_le_p(buf, val);
 427
 428    /* Linux crc32c converts the output to one's complement.  */
 429    return crc32c(acc, buf, bytes) ^ 0xffffffff;
 430}
 431
 432/* Returns 0 on success; 1 otherwise.  */
 433static uint64_t do_paired_cmpxchg64_le(CPUARMState *env, uint64_t addr,
 434                                       uint64_t new_lo, uint64_t new_hi,
 435                                       bool parallel, uintptr_t ra)
 436{
 437    Int128 oldv, cmpv, newv;
 438    bool success;
 439
 440    cmpv = int128_make128(env->exclusive_val, env->exclusive_high);
 441    newv = int128_make128(new_lo, new_hi);
 442
 443    if (parallel) {
 444#ifndef CONFIG_ATOMIC128
 445        cpu_loop_exit_atomic(ENV_GET_CPU(env), ra);
 446#else
 447        int mem_idx = cpu_mmu_index(env, false);
 448        TCGMemOpIdx oi = make_memop_idx(MO_LEQ | MO_ALIGN_16, mem_idx);
 449        oldv = helper_atomic_cmpxchgo_le_mmu(env, addr, cmpv, newv, oi, ra);
 450        success = int128_eq(oldv, cmpv);
 451#endif
 452    } else {
 453        uint64_t o0, o1;
 454
 455#ifdef CONFIG_USER_ONLY
 456        /* ??? Enforce alignment.  */
 457        uint64_t *haddr = g2h(addr);
 458
 459        helper_retaddr = ra;
 460        o0 = ldq_le_p(haddr + 0);
 461        o1 = ldq_le_p(haddr + 1);
 462        oldv = int128_make128(o0, o1);
 463
 464        success = int128_eq(oldv, cmpv);
 465        if (success) {
 466            stq_le_p(haddr + 0, int128_getlo(newv));
 467            stq_le_p(haddr + 1, int128_gethi(newv));
 468        }
 469        helper_retaddr = 0;
 470#else
 471        int mem_idx = cpu_mmu_index(env, false);
 472        TCGMemOpIdx oi0 = make_memop_idx(MO_LEQ | MO_ALIGN_16, mem_idx);
 473        TCGMemOpIdx oi1 = make_memop_idx(MO_LEQ, mem_idx);
 474
 475        o0 = helper_le_ldq_mmu(env, addr + 0, oi0, ra);
 476        o1 = helper_le_ldq_mmu(env, addr + 8, oi1, ra);
 477        oldv = int128_make128(o0, o1);
 478
 479        success = int128_eq(oldv, cmpv);
 480        if (success) {
 481            helper_le_stq_mmu(env, addr + 0, int128_getlo(newv), oi1, ra);
 482            helper_le_stq_mmu(env, addr + 8, int128_gethi(newv), oi1, ra);
 483        }
 484#endif
 485    }
 486
 487    return !success;
 488}
 489
 490uint64_t HELPER(paired_cmpxchg64_le)(CPUARMState *env, uint64_t addr,
 491                                              uint64_t new_lo, uint64_t new_hi)
 492{
 493    return do_paired_cmpxchg64_le(env, addr, new_lo, new_hi, false, GETPC());
 494}
 495
 496uint64_t HELPER(paired_cmpxchg64_le_parallel)(CPUARMState *env, uint64_t addr,
 497                                              uint64_t new_lo, uint64_t new_hi)
 498{
 499    return do_paired_cmpxchg64_le(env, addr, new_lo, new_hi, true, GETPC());
 500}
 501
 502static uint64_t do_paired_cmpxchg64_be(CPUARMState *env, uint64_t addr,
 503                                       uint64_t new_lo, uint64_t new_hi,
 504                                       bool parallel, uintptr_t ra)
 505{
 506    Int128 oldv, cmpv, newv;
 507    bool success;
 508
 509    cmpv = int128_make128(env->exclusive_val, env->exclusive_high);
 510    newv = int128_make128(new_lo, new_hi);
 511
 512    if (parallel) {
 513#ifndef CONFIG_ATOMIC128
 514        cpu_loop_exit_atomic(ENV_GET_CPU(env), ra);
 515#else
 516        int mem_idx = cpu_mmu_index(env, false);
 517        TCGMemOpIdx oi = make_memop_idx(MO_BEQ | MO_ALIGN_16, mem_idx);
 518        oldv = helper_atomic_cmpxchgo_be_mmu(env, addr, cmpv, newv, oi, ra);
 519        success = int128_eq(oldv, cmpv);
 520#endif
 521    } else {
 522        uint64_t o0, o1;
 523
 524#ifdef CONFIG_USER_ONLY
 525        /* ??? Enforce alignment.  */
 526        uint64_t *haddr = g2h(addr);
 527
 528        helper_retaddr = ra;
 529        o1 = ldq_be_p(haddr + 0);
 530        o0 = ldq_be_p(haddr + 1);
 531        oldv = int128_make128(o0, o1);
 532
 533        success = int128_eq(oldv, cmpv);
 534        if (success) {
 535            stq_be_p(haddr + 0, int128_gethi(newv));
 536            stq_be_p(haddr + 1, int128_getlo(newv));
 537        }
 538        helper_retaddr = 0;
 539#else
 540        int mem_idx = cpu_mmu_index(env, false);
 541        TCGMemOpIdx oi0 = make_memop_idx(MO_BEQ | MO_ALIGN_16, mem_idx);
 542        TCGMemOpIdx oi1 = make_memop_idx(MO_BEQ, mem_idx);
 543
 544        o1 = helper_be_ldq_mmu(env, addr + 0, oi0, ra);
 545        o0 = helper_be_ldq_mmu(env, addr + 8, oi1, ra);
 546        oldv = int128_make128(o0, o1);
 547
 548        success = int128_eq(oldv, cmpv);
 549        if (success) {
 550            helper_be_stq_mmu(env, addr + 0, int128_gethi(newv), oi1, ra);
 551            helper_be_stq_mmu(env, addr + 8, int128_getlo(newv), oi1, ra);
 552        }
 553#endif
 554    }
 555
 556    return !success;
 557}
 558
 559uint64_t HELPER(paired_cmpxchg64_be)(CPUARMState *env, uint64_t addr,
 560                                     uint64_t new_lo, uint64_t new_hi)
 561{
 562    return do_paired_cmpxchg64_be(env, addr, new_lo, new_hi, false, GETPC());
 563}
 564
 565uint64_t HELPER(paired_cmpxchg64_be_parallel)(CPUARMState *env, uint64_t addr,
 566                                     uint64_t new_lo, uint64_t new_hi)
 567{
 568    return do_paired_cmpxchg64_be(env, addr, new_lo, new_hi, true, GETPC());
 569}
 570