2 * AArch64 specific helpers
4 * Copyright (c) 2013 Alexander Graf <agraf@suse.de>
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
20 #include "qemu/osdep.h"
22 #include "exec/gdbstub.h"
23 #include "exec/helper-proto.h"
24 #include "qemu/host-utils.h"
26 #include "sysemu/sysemu.h"
27 #include "qemu/bitops.h"
28 #include "internals.h"
29 #include "qemu/crc32c.h"
30 #include "exec/exec-all.h"
31 #include "exec/cpu_ldst.h"
32 #include "qemu/int128.h"
34 #include "fpu/softfloat.h"
35 #include <zlib.h> /* For crc32 */
37 /* C2.4.7 Multiply and divide */
38 /* special cases for 0 and LLONG_MIN are mandated by the standard */
39 uint64_t HELPER(udiv64
)(uint64_t num
, uint64_t den
)
47 int64_t HELPER(sdiv64
)(int64_t num
, int64_t den
)
52 if (num
== LLONG_MIN
&& den
== -1) {
58 uint64_t HELPER(rbit64
)(uint64_t x
)
63 /* Convert a softfloat float_relation_ (as returned by
64 * the float*_compare functions) to the correct ARM
67 static inline uint32_t float_rel_to_flags(int res
)
71 case float_relation_equal
:
72 flags
= PSTATE_Z
| PSTATE_C
;
74 case float_relation_less
:
77 case float_relation_greater
:
80 case float_relation_unordered
:
82 flags
= PSTATE_C
| PSTATE_V
;
88 uint64_t HELPER(vfp_cmps_a64
)(float32 x
, float32 y
, void *fp_status
)
90 return float_rel_to_flags(float32_compare_quiet(x
, y
, fp_status
));
93 uint64_t HELPER(vfp_cmpes_a64
)(float32 x
, float32 y
, void *fp_status
)
95 return float_rel_to_flags(float32_compare(x
, y
, fp_status
));
98 uint64_t HELPER(vfp_cmpd_a64
)(float64 x
, float64 y
, void *fp_status
)
100 return float_rel_to_flags(float64_compare_quiet(x
, y
, fp_status
));
103 uint64_t HELPER(vfp_cmped_a64
)(float64 x
, float64 y
, void *fp_status
)
105 return float_rel_to_flags(float64_compare(x
, y
, fp_status
));
108 float32
HELPER(vfp_mulxs
)(float32 a
, float32 b
, void *fpstp
)
110 float_status
*fpst
= fpstp
;
112 a
= float32_squash_input_denormal(a
, fpst
);
113 b
= float32_squash_input_denormal(b
, fpst
);
115 if ((float32_is_zero(a
) && float32_is_infinity(b
)) ||
116 (float32_is_infinity(a
) && float32_is_zero(b
))) {
117 /* 2.0 with the sign bit set to sign(A) XOR sign(B) */
118 return make_float32((1U << 30) |
119 ((float32_val(a
) ^ float32_val(b
)) & (1U << 31)));
121 return float32_mul(a
, b
, fpst
);
124 float64
HELPER(vfp_mulxd
)(float64 a
, float64 b
, void *fpstp
)
126 float_status
*fpst
= fpstp
;
128 a
= float64_squash_input_denormal(a
, fpst
);
129 b
= float64_squash_input_denormal(b
, fpst
);
131 if ((float64_is_zero(a
) && float64_is_infinity(b
)) ||
132 (float64_is_infinity(a
) && float64_is_zero(b
))) {
133 /* 2.0 with the sign bit set to sign(A) XOR sign(B) */
134 return make_float64((1ULL << 62) |
135 ((float64_val(a
) ^ float64_val(b
)) & (1ULL << 63)));
137 return float64_mul(a
, b
, fpst
);
140 uint64_t HELPER(simd_tbl
)(CPUARMState
*env
, uint64_t result
, uint64_t indices
,
141 uint32_t rn
, uint32_t numregs
)
143 /* Helper function for SIMD TBL and TBX. We have to do the table
144 * lookup part for the 64 bits worth of indices we're passed in.
145 * result is the initial results vector (either zeroes for TBL
146 * or some guest values for TBX), rn the register number where
147 * the table starts, and numregs the number of registers in the table.
148 * We return the results of the lookups.
152 for (shift
= 0; shift
< 64; shift
+= 8) {
153 int index
= extract64(indices
, shift
, 8);
154 if (index
< 16 * numregs
) {
155 /* Convert index (a byte offset into the virtual table
156 * which is a series of 128-bit vectors concatenated)
157 * into the correct register element plus a bit offset
158 * into that element, bearing in mind that the table
159 * can wrap around from V31 to V0.
161 int elt
= (rn
* 2 + (index
>> 3)) % 64;
162 int bitidx
= (index
& 7) * 8;
163 uint64_t *q
= aa64_vfp_qreg(env
, elt
>> 1);
164 uint64_t val
= extract64(q
[elt
& 1], bitidx
, 8);
166 result
= deposit64(result
, shift
, 8, val
);
172 /* 64bit/double versions of the neon float compare functions */
173 uint64_t HELPER(neon_ceq_f64
)(float64 a
, float64 b
, void *fpstp
)
175 float_status
*fpst
= fpstp
;
176 return -float64_eq_quiet(a
, b
, fpst
);
179 uint64_t HELPER(neon_cge_f64
)(float64 a
, float64 b
, void *fpstp
)
181 float_status
*fpst
= fpstp
;
182 return -float64_le(b
, a
, fpst
);
185 uint64_t HELPER(neon_cgt_f64
)(float64 a
, float64 b
, void *fpstp
)
187 float_status
*fpst
= fpstp
;
188 return -float64_lt(b
, a
, fpst
);
191 /* Reciprocal step and sqrt step. Note that unlike the A32/T32
192 * versions, these do a fully fused multiply-add or
193 * multiply-add-and-halve.
195 #define float32_two make_float32(0x40000000)
196 #define float32_three make_float32(0x40400000)
197 #define float32_one_point_five make_float32(0x3fc00000)
199 #define float64_two make_float64(0x4000000000000000ULL)
200 #define float64_three make_float64(0x4008000000000000ULL)
201 #define float64_one_point_five make_float64(0x3FF8000000000000ULL)
203 float32
HELPER(recpsf_f32
)(float32 a
, float32 b
, void *fpstp
)
205 float_status
*fpst
= fpstp
;
207 a
= float32_squash_input_denormal(a
, fpst
);
208 b
= float32_squash_input_denormal(b
, fpst
);
211 if ((float32_is_infinity(a
) && float32_is_zero(b
)) ||
212 (float32_is_infinity(b
) && float32_is_zero(a
))) {
215 return float32_muladd(a
, b
, float32_two
, 0, fpst
);
218 float64
HELPER(recpsf_f64
)(float64 a
, float64 b
, void *fpstp
)
220 float_status
*fpst
= fpstp
;
222 a
= float64_squash_input_denormal(a
, fpst
);
223 b
= float64_squash_input_denormal(b
, fpst
);
226 if ((float64_is_infinity(a
) && float64_is_zero(b
)) ||
227 (float64_is_infinity(b
) && float64_is_zero(a
))) {
230 return float64_muladd(a
, b
, float64_two
, 0, fpst
);
233 float32
HELPER(rsqrtsf_f32
)(float32 a
, float32 b
, void *fpstp
)
235 float_status
*fpst
= fpstp
;
237 a
= float32_squash_input_denormal(a
, fpst
);
238 b
= float32_squash_input_denormal(b
, fpst
);
241 if ((float32_is_infinity(a
) && float32_is_zero(b
)) ||
242 (float32_is_infinity(b
) && float32_is_zero(a
))) {
243 return float32_one_point_five
;
245 return float32_muladd(a
, b
, float32_three
, float_muladd_halve_result
, fpst
);
248 float64
HELPER(rsqrtsf_f64
)(float64 a
, float64 b
, void *fpstp
)
250 float_status
*fpst
= fpstp
;
252 a
= float64_squash_input_denormal(a
, fpst
);
253 b
= float64_squash_input_denormal(b
, fpst
);
256 if ((float64_is_infinity(a
) && float64_is_zero(b
)) ||
257 (float64_is_infinity(b
) && float64_is_zero(a
))) {
258 return float64_one_point_five
;
260 return float64_muladd(a
, b
, float64_three
, float_muladd_halve_result
, fpst
);
263 /* Pairwise long add: add pairs of adjacent elements into
264 * double-width elements in the result (eg _s8 is an 8x8->16 op)
266 uint64_t HELPER(neon_addlp_s8
)(uint64_t a
)
268 uint64_t nsignmask
= 0x0080008000800080ULL
;
269 uint64_t wsignmask
= 0x8000800080008000ULL
;
270 uint64_t elementmask
= 0x00ff00ff00ff00ffULL
;
272 uint64_t res
, signres
;
274 /* Extract odd elements, sign extend each to a 16 bit field */
275 tmp1
= a
& elementmask
;
278 tmp1
= (tmp1
- nsignmask
) ^ wsignmask
;
279 /* Ditto for the even elements */
280 tmp2
= (a
>> 8) & elementmask
;
283 tmp2
= (tmp2
- nsignmask
) ^ wsignmask
;
285 /* calculate the result by summing bits 0..14, 16..22, etc,
286 * and then adjusting the sign bits 15, 23, etc manually.
287 * This ensures the addition can't overflow the 16 bit field.
289 signres
= (tmp1
^ tmp2
) & wsignmask
;
290 res
= (tmp1
& ~wsignmask
) + (tmp2
& ~wsignmask
);
296 uint64_t HELPER(neon_addlp_u8
)(uint64_t a
)
300 tmp
= a
& 0x00ff00ff00ff00ffULL
;
301 tmp
+= (a
>> 8) & 0x00ff00ff00ff00ffULL
;
305 uint64_t HELPER(neon_addlp_s16
)(uint64_t a
)
307 int32_t reslo
, reshi
;
309 reslo
= (int32_t)(int16_t)a
+ (int32_t)(int16_t)(a
>> 16);
310 reshi
= (int32_t)(int16_t)(a
>> 32) + (int32_t)(int16_t)(a
>> 48);
312 return (uint32_t)reslo
| (((uint64_t)reshi
) << 32);
315 uint64_t HELPER(neon_addlp_u16
)(uint64_t a
)
319 tmp
= a
& 0x0000ffff0000ffffULL
;
320 tmp
+= (a
>> 16) & 0x0000ffff0000ffffULL
;
324 /* Floating-point reciprocal exponent - see FPRecpX in ARM ARM */
325 float32
HELPER(frecpx_f32
)(float32 a
, void *fpstp
)
327 float_status
*fpst
= fpstp
;
328 uint32_t val32
, sbit
;
331 if (float32_is_any_nan(a
)) {
333 if (float32_is_signaling_nan(a
, fpst
)) {
334 float_raise(float_flag_invalid
, fpst
);
335 nan
= float32_maybe_silence_nan(a
, fpst
);
337 if (fpst
->default_nan_mode
) {
338 nan
= float32_default_nan(fpst
);
343 val32
= float32_val(a
);
344 sbit
= 0x80000000ULL
& val32
;
345 exp
= extract32(val32
, 23, 8);
348 return make_float32(sbit
| (0xfe << 23));
350 return make_float32(sbit
| (~exp
& 0xff) << 23);
354 float64
HELPER(frecpx_f64
)(float64 a
, void *fpstp
)
356 float_status
*fpst
= fpstp
;
357 uint64_t val64
, sbit
;
360 if (float64_is_any_nan(a
)) {
362 if (float64_is_signaling_nan(a
, fpst
)) {
363 float_raise(float_flag_invalid
, fpst
);
364 nan
= float64_maybe_silence_nan(a
, fpst
);
366 if (fpst
->default_nan_mode
) {
367 nan
= float64_default_nan(fpst
);
372 val64
= float64_val(a
);
373 sbit
= 0x8000000000000000ULL
& val64
;
374 exp
= extract64(float64_val(a
), 52, 11);
377 return make_float64(sbit
| (0x7feULL
<< 52));
379 return make_float64(sbit
| (~exp
& 0x7ffULL
) << 52);
383 float32
HELPER(fcvtx_f64_to_f32
)(float64 a
, CPUARMState
*env
)
385 /* Von Neumann rounding is implemented by using round-to-zero
386 * and then setting the LSB of the result if Inexact was raised.
389 float_status
*fpst
= &env
->vfp
.fp_status
;
390 float_status tstat
= *fpst
;
393 set_float_rounding_mode(float_round_to_zero
, &tstat
);
394 set_float_exception_flags(0, &tstat
);
395 r
= float64_to_float32(a
, &tstat
);
396 r
= float32_maybe_silence_nan(r
, &tstat
);
397 exflags
= get_float_exception_flags(&tstat
);
398 if (exflags
& float_flag_inexact
) {
399 r
= make_float32(float32_val(r
) | 1);
401 exflags
|= get_float_exception_flags(fpst
);
402 set_float_exception_flags(exflags
, fpst
);
406 /* 64-bit versions of the CRC helpers. Note that although the operation
407 * (and the prototypes of crc32c() and crc32() mean that only the bottom
408 * 32 bits of the accumulator and result are used, we pass and return
409 * uint64_t for convenience of the generated code. Unlike the 32-bit
410 * instruction set versions, val may genuinely have 64 bits of data in it.
411 * The upper bytes of val (above the number specified by 'bytes') must have
412 * been zeroed out by the caller.
414 uint64_t HELPER(crc32_64
)(uint64_t acc
, uint64_t val
, uint32_t bytes
)
420 /* zlib crc32 converts the accumulator and output to one's complement. */
421 return crc32(acc
^ 0xffffffff, buf
, bytes
) ^ 0xffffffff;
424 uint64_t HELPER(crc32c_64
)(uint64_t acc
, uint64_t val
, uint32_t bytes
)
430 /* Linux crc32c converts the output to one's complement. */
431 return crc32c(acc
, buf
, bytes
) ^ 0xffffffff;
434 /* Returns 0 on success; 1 otherwise. */
435 static uint64_t do_paired_cmpxchg64_le(CPUARMState
*env
, uint64_t addr
,
436 uint64_t new_lo
, uint64_t new_hi
,
437 bool parallel
, uintptr_t ra
)
439 Int128 oldv
, cmpv
, newv
;
442 cmpv
= int128_make128(env
->exclusive_val
, env
->exclusive_high
);
443 newv
= int128_make128(new_lo
, new_hi
);
446 #ifndef CONFIG_ATOMIC128
447 cpu_loop_exit_atomic(ENV_GET_CPU(env
), ra
);
449 int mem_idx
= cpu_mmu_index(env
, false);
450 TCGMemOpIdx oi
= make_memop_idx(MO_LEQ
| MO_ALIGN_16
, mem_idx
);
451 oldv
= helper_atomic_cmpxchgo_le_mmu(env
, addr
, cmpv
, newv
, oi
, ra
);
452 success
= int128_eq(oldv
, cmpv
);
457 #ifdef CONFIG_USER_ONLY
458 /* ??? Enforce alignment. */
459 uint64_t *haddr
= g2h(addr
);
462 o0
= ldq_le_p(haddr
+ 0);
463 o1
= ldq_le_p(haddr
+ 1);
464 oldv
= int128_make128(o0
, o1
);
466 success
= int128_eq(oldv
, cmpv
);
468 stq_le_p(haddr
+ 0, int128_getlo(newv
));
469 stq_le_p(haddr
+ 1, int128_gethi(newv
));
473 int mem_idx
= cpu_mmu_index(env
, false);
474 TCGMemOpIdx oi0
= make_memop_idx(MO_LEQ
| MO_ALIGN_16
, mem_idx
);
475 TCGMemOpIdx oi1
= make_memop_idx(MO_LEQ
, mem_idx
);
477 o0
= helper_le_ldq_mmu(env
, addr
+ 0, oi0
, ra
);
478 o1
= helper_le_ldq_mmu(env
, addr
+ 8, oi1
, ra
);
479 oldv
= int128_make128(o0
, o1
);
481 success
= int128_eq(oldv
, cmpv
);
483 helper_le_stq_mmu(env
, addr
+ 0, int128_getlo(newv
), oi1
, ra
);
484 helper_le_stq_mmu(env
, addr
+ 8, int128_gethi(newv
), oi1
, ra
);
492 uint64_t HELPER(paired_cmpxchg64_le
)(CPUARMState
*env
, uint64_t addr
,
493 uint64_t new_lo
, uint64_t new_hi
)
495 return do_paired_cmpxchg64_le(env
, addr
, new_lo
, new_hi
, false, GETPC());
498 uint64_t HELPER(paired_cmpxchg64_le_parallel
)(CPUARMState
*env
, uint64_t addr
,
499 uint64_t new_lo
, uint64_t new_hi
)
501 return do_paired_cmpxchg64_le(env
, addr
, new_lo
, new_hi
, true, GETPC());
504 static uint64_t do_paired_cmpxchg64_be(CPUARMState
*env
, uint64_t addr
,
505 uint64_t new_lo
, uint64_t new_hi
,
506 bool parallel
, uintptr_t ra
)
508 Int128 oldv
, cmpv
, newv
;
511 /* high and low need to be switched here because this is not actually a
512 * 128bit store but two doublewords stored consecutively
514 cmpv
= int128_make128(env
->exclusive_high
, env
->exclusive_val
);
515 newv
= int128_make128(new_hi
, new_lo
);
518 #ifndef CONFIG_ATOMIC128
519 cpu_loop_exit_atomic(ENV_GET_CPU(env
), ra
);
521 int mem_idx
= cpu_mmu_index(env
, false);
522 TCGMemOpIdx oi
= make_memop_idx(MO_BEQ
| MO_ALIGN_16
, mem_idx
);
523 oldv
= helper_atomic_cmpxchgo_be_mmu(env
, addr
, cmpv
, newv
, oi
, ra
);
524 success
= int128_eq(oldv
, cmpv
);
529 #ifdef CONFIG_USER_ONLY
530 /* ??? Enforce alignment. */
531 uint64_t *haddr
= g2h(addr
);
534 o1
= ldq_be_p(haddr
+ 0);
535 o0
= ldq_be_p(haddr
+ 1);
536 oldv
= int128_make128(o0
, o1
);
538 success
= int128_eq(oldv
, cmpv
);
540 stq_be_p(haddr
+ 0, int128_gethi(newv
));
541 stq_be_p(haddr
+ 1, int128_getlo(newv
));
545 int mem_idx
= cpu_mmu_index(env
, false);
546 TCGMemOpIdx oi0
= make_memop_idx(MO_BEQ
| MO_ALIGN_16
, mem_idx
);
547 TCGMemOpIdx oi1
= make_memop_idx(MO_BEQ
, mem_idx
);
549 o1
= helper_be_ldq_mmu(env
, addr
+ 0, oi0
, ra
);
550 o0
= helper_be_ldq_mmu(env
, addr
+ 8, oi1
, ra
);
551 oldv
= int128_make128(o0
, o1
);
553 success
= int128_eq(oldv
, cmpv
);
555 helper_be_stq_mmu(env
, addr
+ 0, int128_gethi(newv
), oi1
, ra
);
556 helper_be_stq_mmu(env
, addr
+ 8, int128_getlo(newv
), oi1
, ra
);
564 uint64_t HELPER(paired_cmpxchg64_be
)(CPUARMState
*env
, uint64_t addr
,
565 uint64_t new_lo
, uint64_t new_hi
)
567 return do_paired_cmpxchg64_be(env
, addr
, new_lo
, new_hi
, false, GETPC());
570 uint64_t HELPER(paired_cmpxchg64_be_parallel
)(CPUARMState
*env
, uint64_t addr
,
571 uint64_t new_lo
, uint64_t new_hi
)
573 return do_paired_cmpxchg64_be(env
, addr
, new_lo
, new_hi
, true, GETPC());