1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20#include "qemu/osdep.h"
21#include "qemu/units.h"
22#include "cpu.h"
23#include "exec/gdbstub.h"
24#include "exec/helper-proto.h"
25#include "qemu/host-utils.h"
26#include "qemu/log.h"
27#include "qemu/main-loop.h"
28#include "qemu/bitops.h"
29#include "internals.h"
30#include "qemu/crc32c.h"
31#include "exec/exec-all.h"
32#include "exec/cpu_ldst.h"
33#include "qemu/int128.h"
34#include "qemu/atomic128.h"
35#include "fpu/softfloat.h"
36#include <zlib.h>
37
38
39
40uint64_t HELPER(udiv64)(uint64_t num, uint64_t den)
41{
42 if (den == 0) {
43 return 0;
44 }
45 return num / den;
46}
47
48int64_t HELPER(sdiv64)(int64_t num, int64_t den)
49{
50 if (den == 0) {
51 return 0;
52 }
53 if (num == LLONG_MIN && den == -1) {
54 return LLONG_MIN;
55 }
56 return num / den;
57}
58
59uint64_t HELPER(rbit64)(uint64_t x)
60{
61 return revbit64(x);
62}
63
64void HELPER(msr_i_spsel)(CPUARMState *env, uint32_t imm)
65{
66 update_spsel(env, imm);
67}
68
69static void daif_check(CPUARMState *env, uint32_t op,
70 uint32_t imm, uintptr_t ra)
71{
72
73 if (arm_current_el(env) == 0 && !(arm_sctlr(env, 0) & SCTLR_UMA)) {
74 raise_exception_ra(env, EXCP_UDEF,
75 syn_aa64_sysregtrap(0, extract32(op, 0, 3),
76 extract32(op, 3, 3), 4,
77 imm, 0x1f, 0),
78 exception_target_el(env), ra);
79 }
80}
81
82void HELPER(msr_i_daifset)(CPUARMState *env, uint32_t imm)
83{
84 daif_check(env, 0x1e, imm, GETPC());
85 env->daif |= (imm << 6) & PSTATE_DAIF;
86}
87
88void HELPER(msr_i_daifclear)(CPUARMState *env, uint32_t imm)
89{
90 daif_check(env, 0x1f, imm, GETPC());
91 env->daif &= ~((imm << 6) & PSTATE_DAIF);
92}
93
94
95
96
97
98static inline uint32_t float_rel_to_flags(int res)
99{
100 uint64_t flags;
101 switch (res) {
102 case float_relation_equal:
103 flags = PSTATE_Z | PSTATE_C;
104 break;
105 case float_relation_less:
106 flags = PSTATE_N;
107 break;
108 case float_relation_greater:
109 flags = PSTATE_C;
110 break;
111 case float_relation_unordered:
112 default:
113 flags = PSTATE_C | PSTATE_V;
114 break;
115 }
116 return flags;
117}
118
119uint64_t HELPER(vfp_cmph_a64)(uint32_t x, uint32_t y, void *fp_status)
120{
121 return float_rel_to_flags(float16_compare_quiet(x, y, fp_status));
122}
123
124uint64_t HELPER(vfp_cmpeh_a64)(uint32_t x, uint32_t y, void *fp_status)
125{
126 return float_rel_to_flags(float16_compare(x, y, fp_status));
127}
128
129uint64_t HELPER(vfp_cmps_a64)(float32 x, float32 y, void *fp_status)
130{
131 return float_rel_to_flags(float32_compare_quiet(x, y, fp_status));
132}
133
134uint64_t HELPER(vfp_cmpes_a64)(float32 x, float32 y, void *fp_status)
135{
136 return float_rel_to_flags(float32_compare(x, y, fp_status));
137}
138
139uint64_t HELPER(vfp_cmpd_a64)(float64 x, float64 y, void *fp_status)
140{
141 return float_rel_to_flags(float64_compare_quiet(x, y, fp_status));
142}
143
144uint64_t HELPER(vfp_cmped_a64)(float64 x, float64 y, void *fp_status)
145{
146 return float_rel_to_flags(float64_compare(x, y, fp_status));
147}
148
149float32 HELPER(vfp_mulxs)(float32 a, float32 b, void *fpstp)
150{
151 float_status *fpst = fpstp;
152
153 a = float32_squash_input_denormal(a, fpst);
154 b = float32_squash_input_denormal(b, fpst);
155
156 if ((float32_is_zero(a) && float32_is_infinity(b)) ||
157 (float32_is_infinity(a) && float32_is_zero(b))) {
158
159 return make_float32((1U << 30) |
160 ((float32_val(a) ^ float32_val(b)) & (1U << 31)));
161 }
162 return float32_mul(a, b, fpst);
163}
164
165float64 HELPER(vfp_mulxd)(float64 a, float64 b, void *fpstp)
166{
167 float_status *fpst = fpstp;
168
169 a = float64_squash_input_denormal(a, fpst);
170 b = float64_squash_input_denormal(b, fpst);
171
172 if ((float64_is_zero(a) && float64_is_infinity(b)) ||
173 (float64_is_infinity(a) && float64_is_zero(b))) {
174
175 return make_float64((1ULL << 62) |
176 ((float64_val(a) ^ float64_val(b)) & (1ULL << 63)));
177 }
178 return float64_mul(a, b, fpst);
179}
180
181
182uint64_t HELPER(neon_ceq_f64)(float64 a, float64 b, void *fpstp)
183{
184 float_status *fpst = fpstp;
185 return -float64_eq_quiet(a, b, fpst);
186}
187
188uint64_t HELPER(neon_cge_f64)(float64 a, float64 b, void *fpstp)
189{
190 float_status *fpst = fpstp;
191 return -float64_le(b, a, fpst);
192}
193
194uint64_t HELPER(neon_cgt_f64)(float64 a, float64 b, void *fpstp)
195{
196 float_status *fpst = fpstp;
197 return -float64_lt(b, a, fpst);
198}
199
200
201
202
203
204
205uint32_t HELPER(recpsf_f16)(uint32_t a, uint32_t b, void *fpstp)
206{
207 float_status *fpst = fpstp;
208
209 a = float16_squash_input_denormal(a, fpst);
210 b = float16_squash_input_denormal(b, fpst);
211
212 a = float16_chs(a);
213 if ((float16_is_infinity(a) && float16_is_zero(b)) ||
214 (float16_is_infinity(b) && float16_is_zero(a))) {
215 return float16_two;
216 }
217 return float16_muladd(a, b, float16_two, 0, fpst);
218}
219
220float32 HELPER(recpsf_f32)(float32 a, float32 b, void *fpstp)
221{
222 float_status *fpst = fpstp;
223
224 a = float32_squash_input_denormal(a, fpst);
225 b = float32_squash_input_denormal(b, fpst);
226
227 a = float32_chs(a);
228 if ((float32_is_infinity(a) && float32_is_zero(b)) ||
229 (float32_is_infinity(b) && float32_is_zero(a))) {
230 return float32_two;
231 }
232 return float32_muladd(a, b, float32_two, 0, fpst);
233}
234
235float64 HELPER(recpsf_f64)(float64 a, float64 b, void *fpstp)
236{
237 float_status *fpst = fpstp;
238
239 a = float64_squash_input_denormal(a, fpst);
240 b = float64_squash_input_denormal(b, fpst);
241
242 a = float64_chs(a);
243 if ((float64_is_infinity(a) && float64_is_zero(b)) ||
244 (float64_is_infinity(b) && float64_is_zero(a))) {
245 return float64_two;
246 }
247 return float64_muladd(a, b, float64_two, 0, fpst);
248}
249
250uint32_t HELPER(rsqrtsf_f16)(uint32_t a, uint32_t b, void *fpstp)
251{
252 float_status *fpst = fpstp;
253
254 a = float16_squash_input_denormal(a, fpst);
255 b = float16_squash_input_denormal(b, fpst);
256
257 a = float16_chs(a);
258 if ((float16_is_infinity(a) && float16_is_zero(b)) ||
259 (float16_is_infinity(b) && float16_is_zero(a))) {
260 return float16_one_point_five;
261 }
262 return float16_muladd(a, b, float16_three, float_muladd_halve_result, fpst);
263}
264
265float32 HELPER(rsqrtsf_f32)(float32 a, float32 b, void *fpstp)
266{
267 float_status *fpst = fpstp;
268
269 a = float32_squash_input_denormal(a, fpst);
270 b = float32_squash_input_denormal(b, fpst);
271
272 a = float32_chs(a);
273 if ((float32_is_infinity(a) && float32_is_zero(b)) ||
274 (float32_is_infinity(b) && float32_is_zero(a))) {
275 return float32_one_point_five;
276 }
277 return float32_muladd(a, b, float32_three, float_muladd_halve_result, fpst);
278}
279
280float64 HELPER(rsqrtsf_f64)(float64 a, float64 b, void *fpstp)
281{
282 float_status *fpst = fpstp;
283
284 a = float64_squash_input_denormal(a, fpst);
285 b = float64_squash_input_denormal(b, fpst);
286
287 a = float64_chs(a);
288 if ((float64_is_infinity(a) && float64_is_zero(b)) ||
289 (float64_is_infinity(b) && float64_is_zero(a))) {
290 return float64_one_point_five;
291 }
292 return float64_muladd(a, b, float64_three, float_muladd_halve_result, fpst);
293}
294
295
296
297
298uint64_t HELPER(neon_addlp_s8)(uint64_t a)
299{
300 uint64_t nsignmask = 0x0080008000800080ULL;
301 uint64_t wsignmask = 0x8000800080008000ULL;
302 uint64_t elementmask = 0x00ff00ff00ff00ffULL;
303 uint64_t tmp1, tmp2;
304 uint64_t res, signres;
305
306
307 tmp1 = a & elementmask;
308 tmp1 ^= nsignmask;
309 tmp1 |= wsignmask;
310 tmp1 = (tmp1 - nsignmask) ^ wsignmask;
311
312 tmp2 = (a >> 8) & elementmask;
313 tmp2 ^= nsignmask;
314 tmp2 |= wsignmask;
315 tmp2 = (tmp2 - nsignmask) ^ wsignmask;
316
317
318
319
320
321 signres = (tmp1 ^ tmp2) & wsignmask;
322 res = (tmp1 & ~wsignmask) + (tmp2 & ~wsignmask);
323 res ^= signres;
324
325 return res;
326}
327
328uint64_t HELPER(neon_addlp_u8)(uint64_t a)
329{
330 uint64_t tmp;
331
332 tmp = a & 0x00ff00ff00ff00ffULL;
333 tmp += (a >> 8) & 0x00ff00ff00ff00ffULL;
334 return tmp;
335}
336
337uint64_t HELPER(neon_addlp_s16)(uint64_t a)
338{
339 int32_t reslo, reshi;
340
341 reslo = (int32_t)(int16_t)a + (int32_t)(int16_t)(a >> 16);
342 reshi = (int32_t)(int16_t)(a >> 32) + (int32_t)(int16_t)(a >> 48);
343
344 return (uint32_t)reslo | (((uint64_t)reshi) << 32);
345}
346
347uint64_t HELPER(neon_addlp_u16)(uint64_t a)
348{
349 uint64_t tmp;
350
351 tmp = a & 0x0000ffff0000ffffULL;
352 tmp += (a >> 16) & 0x0000ffff0000ffffULL;
353 return tmp;
354}
355
356
357uint32_t HELPER(frecpx_f16)(uint32_t a, void *fpstp)
358{
359 float_status *fpst = fpstp;
360 uint16_t val16, sbit;
361 int16_t exp;
362
363 if (float16_is_any_nan(a)) {
364 float16 nan = a;
365 if (float16_is_signaling_nan(a, fpst)) {
366 float_raise(float_flag_invalid, fpst);
367 if (!fpst->default_nan_mode) {
368 nan = float16_silence_nan(a, fpst);
369 }
370 }
371 if (fpst->default_nan_mode) {
372 nan = float16_default_nan(fpst);
373 }
374 return nan;
375 }
376
377 a = float16_squash_input_denormal(a, fpst);
378
379 val16 = float16_val(a);
380 sbit = 0x8000 & val16;
381 exp = extract32(val16, 10, 5);
382
383 if (exp == 0) {
384 return make_float16(deposit32(sbit, 10, 5, 0x1e));
385 } else {
386 return make_float16(deposit32(sbit, 10, 5, ~exp));
387 }
388}
389
390float32 HELPER(frecpx_f32)(float32 a, void *fpstp)
391{
392 float_status *fpst = fpstp;
393 uint32_t val32, sbit;
394 int32_t exp;
395
396 if (float32_is_any_nan(a)) {
397 float32 nan = a;
398 if (float32_is_signaling_nan(a, fpst)) {
399 float_raise(float_flag_invalid, fpst);
400 if (!fpst->default_nan_mode) {
401 nan = float32_silence_nan(a, fpst);
402 }
403 }
404 if (fpst->default_nan_mode) {
405 nan = float32_default_nan(fpst);
406 }
407 return nan;
408 }
409
410 a = float32_squash_input_denormal(a, fpst);
411
412 val32 = float32_val(a);
413 sbit = 0x80000000ULL & val32;
414 exp = extract32(val32, 23, 8);
415
416 if (exp == 0) {
417 return make_float32(sbit | (0xfe << 23));
418 } else {
419 return make_float32(sbit | (~exp & 0xff) << 23);
420 }
421}
422
423float64 HELPER(frecpx_f64)(float64 a, void *fpstp)
424{
425 float_status *fpst = fpstp;
426 uint64_t val64, sbit;
427 int64_t exp;
428
429 if (float64_is_any_nan(a)) {
430 float64 nan = a;
431 if (float64_is_signaling_nan(a, fpst)) {
432 float_raise(float_flag_invalid, fpst);
433 if (!fpst->default_nan_mode) {
434 nan = float64_silence_nan(a, fpst);
435 }
436 }
437 if (fpst->default_nan_mode) {
438 nan = float64_default_nan(fpst);
439 }
440 return nan;
441 }
442
443 a = float64_squash_input_denormal(a, fpst);
444
445 val64 = float64_val(a);
446 sbit = 0x8000000000000000ULL & val64;
447 exp = extract64(float64_val(a), 52, 11);
448
449 if (exp == 0) {
450 return make_float64(sbit | (0x7feULL << 52));
451 } else {
452 return make_float64(sbit | (~exp & 0x7ffULL) << 52);
453 }
454}
455
456float32 HELPER(fcvtx_f64_to_f32)(float64 a, CPUARMState *env)
457{
458
459
460
461 float32 r;
462 float_status *fpst = &env->vfp.fp_status;
463 float_status tstat = *fpst;
464 int exflags;
465
466 set_float_rounding_mode(float_round_to_zero, &tstat);
467 set_float_exception_flags(0, &tstat);
468 r = float64_to_float32(a, &tstat);
469 exflags = get_float_exception_flags(&tstat);
470 if (exflags & float_flag_inexact) {
471 r = make_float32(float32_val(r) | 1);
472 }
473 exflags |= get_float_exception_flags(fpst);
474 set_float_exception_flags(exflags, fpst);
475 return r;
476}
477
478
479
480
481
482
483
484
485
486uint64_t HELPER(crc32_64)(uint64_t acc, uint64_t val, uint32_t bytes)
487{
488 uint8_t buf[8];
489
490 stq_le_p(buf, val);
491
492
493 return crc32(acc ^ 0xffffffff, buf, bytes) ^ 0xffffffff;
494}
495
496uint64_t HELPER(crc32c_64)(uint64_t acc, uint64_t val, uint32_t bytes)
497{
498 uint8_t buf[8];
499
500 stq_le_p(buf, val);
501
502
503 return crc32c(acc, buf, bytes) ^ 0xffffffff;
504}
505
506uint64_t HELPER(paired_cmpxchg64_le)(CPUARMState *env, uint64_t addr,
507 uint64_t new_lo, uint64_t new_hi)
508{
509 Int128 cmpv = int128_make128(env->exclusive_val, env->exclusive_high);
510 Int128 newv = int128_make128(new_lo, new_hi);
511 Int128 oldv;
512 uintptr_t ra = GETPC();
513 uint64_t o0, o1;
514 bool success;
515 int mem_idx = cpu_mmu_index(env, false);
516 MemOpIdx oi0 = make_memop_idx(MO_LEQ | MO_ALIGN_16, mem_idx);
517 MemOpIdx oi1 = make_memop_idx(MO_LEQ, mem_idx);
518
519 o0 = cpu_ldq_le_mmu(env, addr + 0, oi0, ra);
520 o1 = cpu_ldq_le_mmu(env, addr + 8, oi1, ra);
521 oldv = int128_make128(o0, o1);
522
523 success = int128_eq(oldv, cmpv);
524 if (success) {
525 cpu_stq_le_mmu(env, addr + 0, int128_getlo(newv), oi1, ra);
526 cpu_stq_le_mmu(env, addr + 8, int128_gethi(newv), oi1, ra);
527 }
528
529 return !success;
530}
531
532uint64_t HELPER(paired_cmpxchg64_le_parallel)(CPUARMState *env, uint64_t addr,
533 uint64_t new_lo, uint64_t new_hi)
534{
535 Int128 oldv, cmpv, newv;
536 uintptr_t ra = GETPC();
537 bool success;
538 int mem_idx;
539 MemOpIdx oi;
540
541 assert(HAVE_CMPXCHG128);
542
543 mem_idx = cpu_mmu_index(env, false);
544 oi = make_memop_idx(MO_LE | MO_128 | MO_ALIGN, mem_idx);
545
546 cmpv = int128_make128(env->exclusive_val, env->exclusive_high);
547 newv = int128_make128(new_lo, new_hi);
548 oldv = cpu_atomic_cmpxchgo_le_mmu(env, addr, cmpv, newv, oi, ra);
549
550 success = int128_eq(oldv, cmpv);
551 return !success;
552}
553
554uint64_t HELPER(paired_cmpxchg64_be)(CPUARMState *env, uint64_t addr,
555 uint64_t new_lo, uint64_t new_hi)
556{
557
558
559
560
561 Int128 cmpv = int128_make128(env->exclusive_high, env->exclusive_val);
562 Int128 newv = int128_make128(new_hi, new_lo);
563 Int128 oldv;
564 uintptr_t ra = GETPC();
565 uint64_t o0, o1;
566 bool success;
567 int mem_idx = cpu_mmu_index(env, false);
568 MemOpIdx oi0 = make_memop_idx(MO_BEQ | MO_ALIGN_16, mem_idx);
569 MemOpIdx oi1 = make_memop_idx(MO_BEQ, mem_idx);
570
571 o1 = cpu_ldq_be_mmu(env, addr + 0, oi0, ra);
572 o0 = cpu_ldq_be_mmu(env, addr + 8, oi1, ra);
573 oldv = int128_make128(o0, o1);
574
575 success = int128_eq(oldv, cmpv);
576 if (success) {
577 cpu_stq_be_mmu(env, addr + 0, int128_gethi(newv), oi1, ra);
578 cpu_stq_be_mmu(env, addr + 8, int128_getlo(newv), oi1, ra);
579 }
580
581 return !success;
582}
583
584uint64_t HELPER(paired_cmpxchg64_be_parallel)(CPUARMState *env, uint64_t addr,
585 uint64_t new_lo, uint64_t new_hi)
586{
587 Int128 oldv, cmpv, newv;
588 uintptr_t ra = GETPC();
589 bool success;
590 int mem_idx;
591 MemOpIdx oi;
592
593 assert(HAVE_CMPXCHG128);
594
595 mem_idx = cpu_mmu_index(env, false);
596 oi = make_memop_idx(MO_BE | MO_128 | MO_ALIGN, mem_idx);
597
598
599
600
601
602 cmpv = int128_make128(env->exclusive_high, env->exclusive_val);
603 newv = int128_make128(new_hi, new_lo);
604 oldv = cpu_atomic_cmpxchgo_be_mmu(env, addr, cmpv, newv, oi, ra);
605
606 success = int128_eq(oldv, cmpv);
607 return !success;
608}
609
610
611void HELPER(casp_le_parallel)(CPUARMState *env, uint32_t rs, uint64_t addr,
612 uint64_t new_lo, uint64_t new_hi)
613{
614 Int128 oldv, cmpv, newv;
615 uintptr_t ra = GETPC();
616 int mem_idx;
617 MemOpIdx oi;
618
619 assert(HAVE_CMPXCHG128);
620
621 mem_idx = cpu_mmu_index(env, false);
622 oi = make_memop_idx(MO_LE | MO_128 | MO_ALIGN, mem_idx);
623
624 cmpv = int128_make128(env->xregs[rs], env->xregs[rs + 1]);
625 newv = int128_make128(new_lo, new_hi);
626 oldv = cpu_atomic_cmpxchgo_le_mmu(env, addr, cmpv, newv, oi, ra);
627
628 env->xregs[rs] = int128_getlo(oldv);
629 env->xregs[rs + 1] = int128_gethi(oldv);
630}
631
632void HELPER(casp_be_parallel)(CPUARMState *env, uint32_t rs, uint64_t addr,
633 uint64_t new_hi, uint64_t new_lo)
634{
635 Int128 oldv, cmpv, newv;
636 uintptr_t ra = GETPC();
637 int mem_idx;
638 MemOpIdx oi;
639
640 assert(HAVE_CMPXCHG128);
641
642 mem_idx = cpu_mmu_index(env, false);
643 oi = make_memop_idx(MO_LE | MO_128 | MO_ALIGN, mem_idx);
644
645 cmpv = int128_make128(env->xregs[rs + 1], env->xregs[rs]);
646 newv = int128_make128(new_lo, new_hi);
647 oldv = cpu_atomic_cmpxchgo_be_mmu(env, addr, cmpv, newv, oi, ra);
648
649 env->xregs[rs + 1] = int128_getlo(oldv);
650 env->xregs[rs] = int128_gethi(oldv);
651}
652
653
654
655
656
657#define ADVSIMD_HELPER(name, suffix) HELPER(glue(glue(advsimd_, name), suffix))
658
659#define ADVSIMD_HALFOP(name) \
660uint32_t ADVSIMD_HELPER(name, h)(uint32_t a, uint32_t b, void *fpstp) \
661{ \
662 float_status *fpst = fpstp; \
663 return float16_ ## name(a, b, fpst); \
664}
665
666ADVSIMD_HALFOP(add)
667ADVSIMD_HALFOP(sub)
668ADVSIMD_HALFOP(mul)
669ADVSIMD_HALFOP(div)
670ADVSIMD_HALFOP(min)
671ADVSIMD_HALFOP(max)
672ADVSIMD_HALFOP(minnum)
673ADVSIMD_HALFOP(maxnum)
674
675#define ADVSIMD_TWOHALFOP(name) \
676uint32_t ADVSIMD_HELPER(name, 2h)(uint32_t two_a, uint32_t two_b, void *fpstp) \
677{ \
678 float16 a1, a2, b1, b2; \
679 uint32_t r1, r2; \
680 float_status *fpst = fpstp; \
681 a1 = extract32(two_a, 0, 16); \
682 a2 = extract32(two_a, 16, 16); \
683 b1 = extract32(two_b, 0, 16); \
684 b2 = extract32(two_b, 16, 16); \
685 r1 = float16_ ## name(a1, b1, fpst); \
686 r2 = float16_ ## name(a2, b2, fpst); \
687 return deposit32(r1, 16, 16, r2); \
688}
689
690ADVSIMD_TWOHALFOP(add)
691ADVSIMD_TWOHALFOP(sub)
692ADVSIMD_TWOHALFOP(mul)
693ADVSIMD_TWOHALFOP(div)
694ADVSIMD_TWOHALFOP(min)
695ADVSIMD_TWOHALFOP(max)
696ADVSIMD_TWOHALFOP(minnum)
697ADVSIMD_TWOHALFOP(maxnum)
698
699
700static float16 float16_mulx(float16 a, float16 b, void *fpstp)
701{
702 float_status *fpst = fpstp;
703
704 a = float16_squash_input_denormal(a, fpst);
705 b = float16_squash_input_denormal(b, fpst);
706
707 if ((float16_is_zero(a) && float16_is_infinity(b)) ||
708 (float16_is_infinity(a) && float16_is_zero(b))) {
709
710 return make_float16((1U << 14) |
711 ((float16_val(a) ^ float16_val(b)) & (1U << 15)));
712 }
713 return float16_mul(a, b, fpst);
714}
715
716ADVSIMD_HALFOP(mulx)
717ADVSIMD_TWOHALFOP(mulx)
718
719
720uint32_t HELPER(advsimd_muladdh)(uint32_t a, uint32_t b, uint32_t c,
721 void *fpstp)
722{
723 float_status *fpst = fpstp;
724 return float16_muladd(a, b, c, 0, fpst);
725}
726
727uint32_t HELPER(advsimd_muladd2h)(uint32_t two_a, uint32_t two_b,
728 uint32_t two_c, void *fpstp)
729{
730 float_status *fpst = fpstp;
731 float16 a1, a2, b1, b2, c1, c2;
732 uint32_t r1, r2;
733 a1 = extract32(two_a, 0, 16);
734 a2 = extract32(two_a, 16, 16);
735 b1 = extract32(two_b, 0, 16);
736 b2 = extract32(two_b, 16, 16);
737 c1 = extract32(two_c, 0, 16);
738 c2 = extract32(two_c, 16, 16);
739 r1 = float16_muladd(a1, b1, c1, 0, fpst);
740 r2 = float16_muladd(a2, b2, c2, 0, fpst);
741 return deposit32(r1, 16, 16, r2);
742}
743
744
745
746
747
748
749
750#define ADVSIMD_CMPRES(test) (test) ? 0xffff : 0
751
752uint32_t HELPER(advsimd_ceq_f16)(uint32_t a, uint32_t b, void *fpstp)
753{
754 float_status *fpst = fpstp;
755 int compare = float16_compare_quiet(a, b, fpst);
756 return ADVSIMD_CMPRES(compare == float_relation_equal);
757}
758
759uint32_t HELPER(advsimd_cge_f16)(uint32_t a, uint32_t b, void *fpstp)
760{
761 float_status *fpst = fpstp;
762 int compare = float16_compare(a, b, fpst);
763 return ADVSIMD_CMPRES(compare == float_relation_greater ||
764 compare == float_relation_equal);
765}
766
767uint32_t HELPER(advsimd_cgt_f16)(uint32_t a, uint32_t b, void *fpstp)
768{
769 float_status *fpst = fpstp;
770 int compare = float16_compare(a, b, fpst);
771 return ADVSIMD_CMPRES(compare == float_relation_greater);
772}
773
774uint32_t HELPER(advsimd_acge_f16)(uint32_t a, uint32_t b, void *fpstp)
775{
776 float_status *fpst = fpstp;
777 float16 f0 = float16_abs(a);
778 float16 f1 = float16_abs(b);
779 int compare = float16_compare(f0, f1, fpst);
780 return ADVSIMD_CMPRES(compare == float_relation_greater ||
781 compare == float_relation_equal);
782}
783
784uint32_t HELPER(advsimd_acgt_f16)(uint32_t a, uint32_t b, void *fpstp)
785{
786 float_status *fpst = fpstp;
787 float16 f0 = float16_abs(a);
788 float16 f1 = float16_abs(b);
789 int compare = float16_compare(f0, f1, fpst);
790 return ADVSIMD_CMPRES(compare == float_relation_greater);
791}
792
793
794uint32_t HELPER(advsimd_rinth_exact)(uint32_t x, void *fp_status)
795{
796 return float16_round_to_int(x, fp_status);
797}
798
799uint32_t HELPER(advsimd_rinth)(uint32_t x, void *fp_status)
800{
801 int old_flags = get_float_exception_flags(fp_status), new_flags;
802 float16 ret;
803
804 ret = float16_round_to_int(x, fp_status);
805
806
807 if (!(old_flags & float_flag_inexact)) {
808 new_flags = get_float_exception_flags(fp_status);
809 set_float_exception_flags(new_flags & ~float_flag_inexact, fp_status);
810 }
811
812 return ret;
813}
814
815
816
817
818
819
820
821
822
823uint32_t HELPER(advsimd_f16tosinth)(uint32_t a, void *fpstp)
824{
825 float_status *fpst = fpstp;
826
827
828 if (float16_is_any_nan(a)) {
829 float_raise(float_flag_invalid, fpst);
830 return 0;
831 }
832 return float16_to_int16(a, fpst);
833}
834
835uint32_t HELPER(advsimd_f16touinth)(uint32_t a, void *fpstp)
836{
837 float_status *fpst = fpstp;
838
839
840 if (float16_is_any_nan(a)) {
841 float_raise(float_flag_invalid, fpst);
842 return 0;
843 }
844 return float16_to_uint16(a, fpst);
845}
846
847static int el_from_spsr(uint32_t spsr)
848{
849
850
851
852 if (spsr & PSTATE_nRW) {
853 switch (spsr & CPSR_M) {
854 case ARM_CPU_MODE_USR:
855 return 0;
856 case ARM_CPU_MODE_HYP:
857 return 2;
858 case ARM_CPU_MODE_FIQ:
859 case ARM_CPU_MODE_IRQ:
860 case ARM_CPU_MODE_SVC:
861 case ARM_CPU_MODE_ABT:
862 case ARM_CPU_MODE_UND:
863 case ARM_CPU_MODE_SYS:
864 return 1;
865 case ARM_CPU_MODE_MON:
866
867
868
869 default:
870 return -1;
871 }
872 } else {
873 if (extract32(spsr, 1, 1)) {
874
875 return -1;
876 }
877 if (extract32(spsr, 0, 4) == 1) {
878
879 return -1;
880 }
881 return extract32(spsr, 2, 2);
882 }
883}
884
885static void cpsr_write_from_spsr_elx(CPUARMState *env,
886 uint32_t val)
887{
888 uint32_t mask;
889
890
891 env->pstate = (env->pstate & ~PSTATE_SS) | (val & PSTATE_SS);
892 val &= ~PSTATE_SS;
893
894
895 if (val & PSTATE_DIT) {
896 val &= ~PSTATE_DIT;
897 val |= CPSR_DIT;
898 }
899
900 mask = aarch32_cpsr_valid_mask(env->features, \
901 &env_archcpu(env)->isar);
902 cpsr_write(env, val, mask, CPSRWriteRaw);
903}
904
905void HELPER(exception_return)(CPUARMState *env, uint64_t new_pc)
906{
907 int cur_el = arm_current_el(env);
908 unsigned int spsr_idx = aarch64_banked_spsr_index(cur_el);
909 uint32_t spsr = env->banked_spsr[spsr_idx];
910 int new_el;
911 bool return_to_aa64 = (spsr & PSTATE_nRW) == 0;
912
913 aarch64_save_sp(env, cur_el);
914
915 arm_clear_exclusive(env);
916
917
918
919
920
921
922
923
924 if (arm_generate_debug_exceptions(env)) {
925 spsr &= ~PSTATE_SS;
926 }
927
928 new_el = el_from_spsr(spsr);
929 if (new_el == -1) {
930 goto illegal_return;
931 }
932 if (new_el > cur_el || (new_el == 2 && !arm_is_el2_enabled(env))) {
933
934
935
936 goto illegal_return;
937 }
938
939 if (new_el != 0 && arm_el_is_aa64(env, new_el) != return_to_aa64) {
940
941 goto illegal_return;
942 }
943
944 if (new_el == 1 && (arm_hcr_el2_eff(env) & HCR_TGE)) {
945 goto illegal_return;
946 }
947
948 qemu_mutex_lock_iothread();
949 arm_call_pre_el_change_hook(env_archcpu(env));
950 qemu_mutex_unlock_iothread();
951
952 if (!return_to_aa64) {
953 env->aarch64 = 0;
954
955
956
957
958 cpsr_write_from_spsr_elx(env, spsr);
959 if (!arm_singlestep_active(env)) {
960 env->pstate &= ~PSTATE_SS;
961 }
962 aarch64_sync_64_to_32(env);
963
964 if (spsr & CPSR_T) {
965 env->regs[15] = new_pc & ~0x1;
966 } else {
967 env->regs[15] = new_pc & ~0x3;
968 }
969 helper_rebuild_hflags_a32(env, new_el);
970 qemu_log_mask(CPU_LOG_INT, "Exception return from AArch64 EL%d to "
971 "AArch32 EL%d PC 0x%" PRIx32 "\n",
972 cur_el, new_el, env->regs[15]);
973 } else {
974 int tbii;
975
976 env->aarch64 = 1;
977 spsr &= aarch64_pstate_valid_mask(&env_archcpu(env)->isar);
978 pstate_write(env, spsr);
979 if (!arm_singlestep_active(env)) {
980 env->pstate &= ~PSTATE_SS;
981 }
982 aarch64_restore_sp(env, new_el);
983 helper_rebuild_hflags_a64(env, new_el);
984
985
986
987
988
989
990
991
992 tbii = EX_TBFLAG_A64(env->hflags, TBII);
993 if ((tbii >> extract64(new_pc, 55, 1)) & 1) {
994
995 int core_mmu_idx = cpu_mmu_index(env, false);
996 if (regime_has_2_ranges(core_to_aa64_mmu_idx(core_mmu_idx))) {
997 new_pc = sextract64(new_pc, 0, 56);
998 } else {
999 new_pc = extract64(new_pc, 0, 56);
1000 }
1001 }
1002 env->pc = new_pc;
1003
1004 qemu_log_mask(CPU_LOG_INT, "Exception return from AArch64 EL%d to "
1005 "AArch64 EL%d PC 0x%" PRIx64 "\n",
1006 cur_el, new_el, env->pc);
1007 }
1008
1009
1010
1011
1012
1013 aarch64_sve_change_el(env, cur_el, new_el, return_to_aa64);
1014
1015 qemu_mutex_lock_iothread();
1016 arm_call_el_change_hook(env_archcpu(env));
1017 qemu_mutex_unlock_iothread();
1018
1019 return;
1020
1021illegal_return:
1022
1023
1024
1025
1026
1027
1028
1029 env->pstate |= PSTATE_IL;
1030 env->pc = new_pc;
1031 spsr &= PSTATE_NZCV | PSTATE_DAIF;
1032 spsr |= pstate_read(env) & ~(PSTATE_NZCV | PSTATE_DAIF);
1033 pstate_write(env, spsr);
1034 if (!arm_singlestep_active(env)) {
1035 env->pstate &= ~PSTATE_SS;
1036 }
1037 helper_rebuild_hflags_a64(env, cur_el);
1038 qemu_log_mask(LOG_GUEST_ERROR, "Illegal exception return at EL%d: "
1039 "resuming execution at 0x%" PRIx64 "\n", cur_el, env->pc);
1040}
1041
1042
1043
1044
1045
1046uint32_t HELPER(sqrt_f16)(uint32_t a, void *fpstp)
1047{
1048 float_status *s = fpstp;
1049
1050 return float16_sqrt(a, s);
1051}
1052
1053void HELPER(dc_zva)(CPUARMState *env, uint64_t vaddr_in)
1054{
1055
1056
1057
1058
1059
1060
1061
1062 int blocklen = 4 << env_archcpu(env)->dcz_blocksize;
1063 uint64_t vaddr = vaddr_in & ~(blocklen - 1);
1064 int mmu_idx = cpu_mmu_index(env, false);
1065 void *mem;
1066
1067
1068
1069
1070
1071 mem = tlb_vaddr_to_host(env, vaddr, MMU_DATA_STORE, mmu_idx);
1072
1073#ifndef CONFIG_USER_ONLY
1074 if (unlikely(!mem)) {
1075 uintptr_t ra = GETPC();
1076
1077
1078
1079
1080
1081
1082 (void) probe_write(env, vaddr_in, 1, mmu_idx, ra);
1083 mem = probe_write(env, vaddr, blocklen, mmu_idx, ra);
1084
1085 if (unlikely(!mem)) {
1086
1087
1088
1089
1090 for (int i = 0; i < blocklen; i++) {
1091 cpu_stb_mmuidx_ra(env, vaddr + i, 0, mmu_idx, ra);
1092 }
1093 return;
1094 }
1095 }
1096#endif
1097
1098 memset(mem, 0, blocklen);
1099}
1100