2 * ARM NEON vector operations.
4 * Copyright (c) 2007, 2008 CodeSourcery.
5 * Written by Paul Brook
7 * This code is licensed under the GNU GPL v2.
9 #include "qemu/osdep.h"
12 #include "exec/helper-proto.h"
13 #include "fpu/softfloat.h"
15 #define SIGNBIT (uint32_t)0x80000000
16 #define SIGNBIT64 ((uint64_t)1 << 63)
18 #define SET_QC() env->vfp.qc[0] = 1
20 #define NEON_TYPE1(name, type) \
25 #ifdef HOST_WORDS_BIGENDIAN
26 #define NEON_TYPE2(name, type) \
32 #define NEON_TYPE4(name, type) \
41 #define NEON_TYPE2(name, type) \
47 #define NEON_TYPE4(name, type) \
57 NEON_TYPE4(s8
, int8_t)
58 NEON_TYPE4(u8
, uint8_t)
59 NEON_TYPE2(s16
, int16_t)
60 NEON_TYPE2(u16
, uint16_t)
61 NEON_TYPE1(s32
, int32_t)
62 NEON_TYPE1(u32
, uint32_t)
67 /* Copy from a uint32_t to a vector structure type. */
68 #define NEON_UNPACK(vtype, dest, val) do { \
77 /* Copy from a vector structure type to a uint32_t. */
78 #define NEON_PACK(vtype, dest, val) do { \
88 NEON_FN(vdest.v1, vsrc1.v1, vsrc2.v1);
90 NEON_FN(vdest.v1, vsrc1.v1, vsrc2.v1); \
91 NEON_FN(vdest.v2, vsrc1.v2, vsrc2.v2);
93 NEON_FN(vdest.v1, vsrc1.v1, vsrc2.v1); \
94 NEON_FN(vdest.v2, vsrc1.v2, vsrc2.v2); \
95 NEON_FN(vdest.v3, vsrc1.v3, vsrc2.v3); \
96 NEON_FN(vdest.v4, vsrc1.v4, vsrc2.v4);
98 #define NEON_VOP_BODY(vtype, n) \
104 NEON_UNPACK(vtype, vsrc1, arg1); \
105 NEON_UNPACK(vtype, vsrc2, arg2); \
107 NEON_PACK(vtype, res, vdest); \
111 #define NEON_VOP(name, vtype, n) \
112 uint32_t HELPER(glue(neon_,name))(uint32_t arg1, uint32_t arg2) \
113 NEON_VOP_BODY(vtype, n)
115 #define NEON_VOP_ENV(name, vtype, n) \
116 uint32_t HELPER(glue(neon_,name))(CPUARMState *env, uint32_t arg1, uint32_t arg2) \
117 NEON_VOP_BODY(vtype, n)
119 /* Pairwise operations. */
120 /* For 32-bit elements each segment only contains a single element, so
121 the elementwise and pairwise operations are the same. */
123 NEON_FN(vdest.v1, vsrc1.v1, vsrc1.v2); \
124 NEON_FN(vdest.v2, vsrc2.v1, vsrc2.v2);
126 NEON_FN(vdest.v1, vsrc1.v1, vsrc1.v2); \
127 NEON_FN(vdest.v2, vsrc1.v3, vsrc1.v4); \
128 NEON_FN(vdest.v3, vsrc2.v1, vsrc2.v2); \
129 NEON_FN(vdest.v4, vsrc2.v3, vsrc2.v4); \
131 #define NEON_POP(name, vtype, n) \
132 uint32_t HELPER(glue(neon_,name))(uint32_t arg1, uint32_t arg2) \
138 NEON_UNPACK(vtype, vsrc1, arg1); \
139 NEON_UNPACK(vtype, vsrc2, arg2); \
141 NEON_PACK(vtype, res, vdest); \
145 /* Unary operators. */
146 #define NEON_VOP1(name, vtype, n) \
147 uint32_t HELPER(glue(neon_,name))(uint32_t arg) \
151 NEON_UNPACK(vtype, vsrc1, arg); \
153 NEON_PACK(vtype, arg, vdest); \
158 #define NEON_USAT(dest, src1, src2, type) do { \
159 uint32_t tmp = (uint32_t)src1 + (uint32_t)src2; \
160 if (tmp != (type)tmp) { \
166 #define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint8_t)
167 NEON_VOP_ENV(qadd_u8
, neon_u8
, 4)
169 #define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint16_t)
170 NEON_VOP_ENV(qadd_u16
, neon_u16
, 2)
174 uint32_t HELPER(neon_qadd_u32
)(CPUARMState
*env
, uint32_t a
, uint32_t b
)
176 uint32_t res
= a
+ b
;
184 uint64_t HELPER(neon_qadd_u64
)(CPUARMState
*env
, uint64_t src1
, uint64_t src2
)
196 #define NEON_SSAT(dest, src1, src2, type) do { \
197 int32_t tmp = (uint32_t)src1 + (uint32_t)src2; \
198 if (tmp != (type)tmp) { \
201 tmp = (1 << (sizeof(type) * 8 - 1)) - 1; \
203 tmp = 1 << (sizeof(type) * 8 - 1); \
208 #define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int8_t)
209 NEON_VOP_ENV(qadd_s8
, neon_s8
, 4)
211 #define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int16_t)
212 NEON_VOP_ENV(qadd_s16
, neon_s16
, 2)
216 uint32_t HELPER(neon_qadd_s32
)(CPUARMState
*env
, uint32_t a
, uint32_t b
)
218 uint32_t res
= a
+ b
;
219 if (((res
^ a
) & SIGNBIT
) && !((a
^ b
) & SIGNBIT
)) {
221 res
= ~(((int32_t)a
>> 31) ^ SIGNBIT
);
226 uint64_t HELPER(neon_qadd_s64
)(CPUARMState
*env
, uint64_t src1
, uint64_t src2
)
231 if (((res
^ src1
) & SIGNBIT64
) && !((src1
^ src2
) & SIGNBIT64
)) {
233 res
= ((int64_t)src1
>> 63) ^ ~SIGNBIT64
;
238 /* Unsigned saturating accumulate of signed value
240 * Op1/Rn is treated as signed
241 * Op2/Rd is treated as unsigned
243 * Explicit casting is used to ensure the correct sign extension of
244 * inputs. The result is treated as a unsigned value and saturated as such.
246 * We use a macro for the 8/16 bit cases which expects signed integers of va,
247 * vb, and vr for interim calculation and an unsigned 32 bit result value r.
250 #define USATACC(bits, shift) \
252 va = sextract32(a, shift, bits); \
253 vb = extract32(b, shift, bits); \
255 if (vr > UINT##bits##_MAX) { \
257 vr = UINT##bits##_MAX; \
258 } else if (vr < 0) { \
262 r = deposit32(r, shift, bits, vr); \
265 uint32_t HELPER(neon_uqadd_s8
)(CPUARMState
*env
, uint32_t a
, uint32_t b
)
277 uint32_t HELPER(neon_uqadd_s16
)(CPUARMState
*env
, uint32_t a
, uint32_t b
)
289 uint32_t HELPER(neon_uqadd_s32
)(CPUARMState
*env
, uint32_t a
, uint32_t b
)
291 int64_t va
= (int32_t)a
;
292 int64_t vb
= (uint32_t)b
;
293 int64_t vr
= va
+ vb
;
294 if (vr
> UINT32_MAX
) {
304 uint64_t HELPER(neon_uqadd_s64
)(CPUARMState
*env
, uint64_t a
, uint64_t b
)
308 /* We only need to look at the pattern of SIGN bits to detect
311 if (~a
& b
& ~res
& SIGNBIT64
) {
314 } else if (a
& ~b
& res
& SIGNBIT64
) {
321 /* Signed saturating accumulate of unsigned value
323 * Op1/Rn is treated as unsigned
324 * Op2/Rd is treated as signed
326 * The result is treated as a signed value and saturated as such
328 * We use a macro for the 8/16 bit cases which expects signed integers of va,
329 * vb, and vr for interim calculation and an unsigned 32 bit result value r.
332 #define SSATACC(bits, shift) \
334 va = extract32(a, shift, bits); \
335 vb = sextract32(b, shift, bits); \
337 if (vr > INT##bits##_MAX) { \
339 vr = INT##bits##_MAX; \
340 } else if (vr < INT##bits##_MIN) { \
342 vr = INT##bits##_MIN; \
344 r = deposit32(r, shift, bits, vr); \
347 uint32_t HELPER(neon_sqadd_u8
)(CPUARMState
*env
, uint32_t a
, uint32_t b
)
359 uint32_t HELPER(neon_sqadd_u16
)(CPUARMState
*env
, uint32_t a
, uint32_t b
)
372 uint32_t HELPER(neon_sqadd_u32
)(CPUARMState
*env
, uint32_t a
, uint32_t b
)
375 int64_t op1
= (uint32_t)a
;
376 int64_t op2
= (int32_t)b
;
378 if (res
> INT32_MAX
) {
381 } else if (res
< INT32_MIN
) {
388 uint64_t HELPER(neon_sqadd_u64
)(CPUARMState
*env
, uint64_t a
, uint64_t b
)
392 /* We only need to look at the pattern of SIGN bits to detect an overflow */
395 | (a
& ~b
)) & SIGNBIT64
) {
403 #define NEON_USAT(dest, src1, src2, type) do { \
404 uint32_t tmp = (uint32_t)src1 - (uint32_t)src2; \
405 if (tmp != (type)tmp) { \
411 #define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint8_t)
412 NEON_VOP_ENV(qsub_u8
, neon_u8
, 4)
414 #define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint16_t)
415 NEON_VOP_ENV(qsub_u16
, neon_u16
, 2)
419 uint32_t HELPER(neon_qsub_u32
)(CPUARMState
*env
, uint32_t a
, uint32_t b
)
421 uint32_t res
= a
- b
;
429 uint64_t HELPER(neon_qsub_u64
)(CPUARMState
*env
, uint64_t src1
, uint64_t src2
)
442 #define NEON_SSAT(dest, src1, src2, type) do { \
443 int32_t tmp = (uint32_t)src1 - (uint32_t)src2; \
444 if (tmp != (type)tmp) { \
447 tmp = (1 << (sizeof(type) * 8 - 1)) - 1; \
449 tmp = 1 << (sizeof(type) * 8 - 1); \
454 #define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int8_t)
455 NEON_VOP_ENV(qsub_s8
, neon_s8
, 4)
457 #define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int16_t)
458 NEON_VOP_ENV(qsub_s16
, neon_s16
, 2)
462 uint32_t HELPER(neon_qsub_s32
)(CPUARMState
*env
, uint32_t a
, uint32_t b
)
464 uint32_t res
= a
- b
;
465 if (((res
^ a
) & SIGNBIT
) && ((a
^ b
) & SIGNBIT
)) {
467 res
= ~(((int32_t)a
>> 31) ^ SIGNBIT
);
472 uint64_t HELPER(neon_qsub_s64
)(CPUARMState
*env
, uint64_t src1
, uint64_t src2
)
477 if (((res
^ src1
) & SIGNBIT64
) && ((src1
^ src2
) & SIGNBIT64
)) {
479 res
= ((int64_t)src1
>> 63) ^ ~SIGNBIT64
;
484 #define NEON_FN(dest, src1, src2) dest = (src1 + src2) >> 1
485 NEON_VOP(hadd_s8
, neon_s8
, 4)
486 NEON_VOP(hadd_u8
, neon_u8
, 4)
487 NEON_VOP(hadd_s16
, neon_s16
, 2)
488 NEON_VOP(hadd_u16
, neon_u16
, 2)
491 int32_t HELPER(neon_hadd_s32
)(int32_t src1
, int32_t src2
)
495 dest
= (src1
>> 1) + (src2
>> 1);
501 uint32_t HELPER(neon_hadd_u32
)(uint32_t src1
, uint32_t src2
)
505 dest
= (src1
>> 1) + (src2
>> 1);
511 #define NEON_FN(dest, src1, src2) dest = (src1 + src2 + 1) >> 1
512 NEON_VOP(rhadd_s8
, neon_s8
, 4)
513 NEON_VOP(rhadd_u8
, neon_u8
, 4)
514 NEON_VOP(rhadd_s16
, neon_s16
, 2)
515 NEON_VOP(rhadd_u16
, neon_u16
, 2)
518 int32_t HELPER(neon_rhadd_s32
)(int32_t src1
, int32_t src2
)
522 dest
= (src1
>> 1) + (src2
>> 1);
523 if ((src1
| src2
) & 1)
528 uint32_t HELPER(neon_rhadd_u32
)(uint32_t src1
, uint32_t src2
)
532 dest
= (src1
>> 1) + (src2
>> 1);
533 if ((src1
| src2
) & 1)
538 #define NEON_FN(dest, src1, src2) dest = (src1 - src2) >> 1
539 NEON_VOP(hsub_s8
, neon_s8
, 4)
540 NEON_VOP(hsub_u8
, neon_u8
, 4)
541 NEON_VOP(hsub_s16
, neon_s16
, 2)
542 NEON_VOP(hsub_u16
, neon_u16
, 2)
545 int32_t HELPER(neon_hsub_s32
)(int32_t src1
, int32_t src2
)
549 dest
= (src1
>> 1) - (src2
>> 1);
550 if ((~src1
) & src2
& 1)
555 uint32_t HELPER(neon_hsub_u32
)(uint32_t src1
, uint32_t src2
)
559 dest
= (src1
>> 1) - (src2
>> 1);
560 if ((~src1
) & src2
& 1)
565 #define NEON_FN(dest, src1, src2) dest = (src1 > src2) ? ~0 : 0
566 NEON_VOP(cgt_s8
, neon_s8
, 4)
567 NEON_VOP(cgt_u8
, neon_u8
, 4)
568 NEON_VOP(cgt_s16
, neon_s16
, 2)
569 NEON_VOP(cgt_u16
, neon_u16
, 2)
570 NEON_VOP(cgt_s32
, neon_s32
, 1)
571 NEON_VOP(cgt_u32
, neon_u32
, 1)
574 #define NEON_FN(dest, src1, src2) dest = (src1 >= src2) ? ~0 : 0
575 NEON_VOP(cge_s8
, neon_s8
, 4)
576 NEON_VOP(cge_u8
, neon_u8
, 4)
577 NEON_VOP(cge_s16
, neon_s16
, 2)
578 NEON_VOP(cge_u16
, neon_u16
, 2)
579 NEON_VOP(cge_s32
, neon_s32
, 1)
580 NEON_VOP(cge_u32
, neon_u32
, 1)
583 #define NEON_FN(dest, src1, src2) dest = (src1 < src2) ? src1 : src2
584 NEON_POP(pmin_s8
, neon_s8
, 4)
585 NEON_POP(pmin_u8
, neon_u8
, 4)
586 NEON_POP(pmin_s16
, neon_s16
, 2)
587 NEON_POP(pmin_u16
, neon_u16
, 2)
590 #define NEON_FN(dest, src1, src2) dest = (src1 > src2) ? src1 : src2
591 NEON_POP(pmax_s8
, neon_s8
, 4)
592 NEON_POP(pmax_u8
, neon_u8
, 4)
593 NEON_POP(pmax_s16
, neon_s16
, 2)
594 NEON_POP(pmax_u16
, neon_u16
, 2)
597 #define NEON_FN(dest, src1, src2) \
598 dest = (src1 > src2) ? (src1 - src2) : (src2 - src1)
599 NEON_VOP(abd_s8
, neon_s8
, 4)
600 NEON_VOP(abd_u8
, neon_u8
, 4)
601 NEON_VOP(abd_s16
, neon_s16
, 2)
602 NEON_VOP(abd_u16
, neon_u16
, 2)
603 NEON_VOP(abd_s32
, neon_s32
, 1)
604 NEON_VOP(abd_u32
, neon_u32
, 1)
607 #define NEON_FN(dest, src1, src2) do { \
609 tmp = (int8_t)src2; \
610 if (tmp >= (ssize_t)sizeof(src1) * 8 || \
611 tmp <= -(ssize_t)sizeof(src1) * 8) { \
613 } else if (tmp < 0) { \
614 dest = src1 >> -tmp; \
616 dest = src1 << tmp; \
618 NEON_VOP(shl_u16
, neon_u16
, 2)
621 #define NEON_FN(dest, src1, src2) do { \
623 tmp = (int8_t)src2; \
624 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
626 } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
627 dest = src1 >> (sizeof(src1) * 8 - 1); \
628 } else if (tmp < 0) { \
629 dest = src1 >> -tmp; \
631 dest = src1 << tmp; \
633 NEON_VOP(shl_s16
, neon_s16
, 2)
636 #define NEON_FN(dest, src1, src2) do { \
638 tmp = (int8_t)src2; \
639 if ((tmp >= (ssize_t)sizeof(src1) * 8) \
640 || (tmp <= -(ssize_t)sizeof(src1) * 8)) { \
642 } else if (tmp < 0) { \
643 dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
645 dest = src1 << tmp; \
647 NEON_VOP(rshl_s8
, neon_s8
, 4)
648 NEON_VOP(rshl_s16
, neon_s16
, 2)
651 /* The addition of the rounding constant may overflow, so we use an
652 * intermediate 64 bit accumulator. */
653 uint32_t HELPER(neon_rshl_s32
)(uint32_t valop
, uint32_t shiftop
)
656 int32_t val
= (int32_t)valop
;
657 int8_t shift
= (int8_t)shiftop
;
658 if ((shift
>= 32) || (shift
<= -32)) {
660 } else if (shift
< 0) {
661 int64_t big_dest
= ((int64_t)val
+ (1 << (-1 - shift
)));
662 dest
= big_dest
>> -shift
;
669 /* Handling addition overflow with 64 bit input values is more
670 * tricky than with 32 bit values. */
671 uint64_t HELPER(neon_rshl_s64
)(uint64_t valop
, uint64_t shiftop
)
673 int8_t shift
= (int8_t)shiftop
;
675 if ((shift
>= 64) || (shift
<= -64)) {
677 } else if (shift
< 0) {
678 val
>>= (-shift
- 1);
679 if (val
== INT64_MAX
) {
680 /* In this case, it means that the rounding constant is 1,
681 * and the addition would overflow. Return the actual
682 * result directly. */
683 val
= 0x4000000000000000LL
;
694 #define NEON_FN(dest, src1, src2) do { \
696 tmp = (int8_t)src2; \
697 if (tmp >= (ssize_t)sizeof(src1) * 8 || \
698 tmp < -(ssize_t)sizeof(src1) * 8) { \
700 } else if (tmp == -(ssize_t)sizeof(src1) * 8) { \
701 dest = src1 >> (-tmp - 1); \
702 } else if (tmp < 0) { \
703 dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
705 dest = src1 << tmp; \
707 NEON_VOP(rshl_u8
, neon_u8
, 4)
708 NEON_VOP(rshl_u16
, neon_u16
, 2)
711 /* The addition of the rounding constant may overflow, so we use an
712 * intermediate 64 bit accumulator. */
713 uint32_t HELPER(neon_rshl_u32
)(uint32_t val
, uint32_t shiftop
)
716 int8_t shift
= (int8_t)shiftop
;
717 if (shift
>= 32 || shift
< -32) {
719 } else if (shift
== -32) {
721 } else if (shift
< 0) {
722 uint64_t big_dest
= ((uint64_t)val
+ (1 << (-1 - shift
)));
723 dest
= big_dest
>> -shift
;
730 /* Handling addition overflow with 64 bit input values is more
731 * tricky than with 32 bit values. */
732 uint64_t HELPER(neon_rshl_u64
)(uint64_t val
, uint64_t shiftop
)
734 int8_t shift
= (uint8_t)shiftop
;
735 if (shift
>= 64 || shift
< -64) {
737 } else if (shift
== -64) {
738 /* Rounding a 1-bit result just preserves that bit. */
740 } else if (shift
< 0) {
741 val
>>= (-shift
- 1);
742 if (val
== UINT64_MAX
) {
743 /* In this case, it means that the rounding constant is 1,
744 * and the addition would overflow. Return the actual
745 * result directly. */
746 val
= 0x8000000000000000ULL
;
757 #define NEON_FN(dest, src1, src2) do { \
759 tmp = (int8_t)src2; \
760 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
767 } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
769 } else if (tmp < 0) { \
770 dest = src1 >> -tmp; \
772 dest = src1 << tmp; \
773 if ((dest >> tmp) != src1) { \
778 NEON_VOP_ENV(qshl_u8
, neon_u8
, 4)
779 NEON_VOP_ENV(qshl_u16
, neon_u16
, 2)
780 NEON_VOP_ENV(qshl_u32
, neon_u32
, 1)
783 uint64_t HELPER(neon_qshl_u64
)(CPUARMState
*env
, uint64_t val
, uint64_t shiftop
)
785 int8_t shift
= (int8_t)shiftop
;
791 } else if (shift
<= -64) {
793 } else if (shift
< 0) {
798 if ((val
>> shift
) != tmp
) {
806 #define NEON_FN(dest, src1, src2) do { \
808 tmp = (int8_t)src2; \
809 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
812 dest = (uint32_t)(1 << (sizeof(src1) * 8 - 1)); \
819 } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
821 } else if (tmp < 0) { \
822 dest = src1 >> -tmp; \
824 dest = src1 << tmp; \
825 if ((dest >> tmp) != src1) { \
827 dest = (uint32_t)(1 << (sizeof(src1) * 8 - 1)); \
833 NEON_VOP_ENV(qshl_s8
, neon_s8
, 4)
834 NEON_VOP_ENV(qshl_s16
, neon_s16
, 2)
835 NEON_VOP_ENV(qshl_s32
, neon_s32
, 1)
838 uint64_t HELPER(neon_qshl_s64
)(CPUARMState
*env
, uint64_t valop
, uint64_t shiftop
)
840 int8_t shift
= (uint8_t)shiftop
;
845 val
= (val
>> 63) ^ ~SIGNBIT64
;
847 } else if (shift
<= -64) {
849 } else if (shift
< 0) {
854 if ((val
>> shift
) != tmp
) {
856 val
= (tmp
>> 63) ^ ~SIGNBIT64
;
862 #define NEON_FN(dest, src1, src2) do { \
863 if (src1 & (1 << (sizeof(src1) * 8 - 1))) { \
868 tmp = (int8_t)src2; \
869 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
876 } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
878 } else if (tmp < 0) { \
879 dest = src1 >> -tmp; \
881 dest = src1 << tmp; \
882 if ((dest >> tmp) != src1) { \
888 NEON_VOP_ENV(qshlu_s8
, neon_u8
, 4)
889 NEON_VOP_ENV(qshlu_s16
, neon_u16
, 2)
892 uint32_t HELPER(neon_qshlu_s32
)(CPUARMState
*env
, uint32_t valop
, uint32_t shiftop
)
894 if ((int32_t)valop
< 0) {
898 return helper_neon_qshl_u32(env
, valop
, shiftop
);
901 uint64_t HELPER(neon_qshlu_s64
)(CPUARMState
*env
, uint64_t valop
, uint64_t shiftop
)
903 if ((int64_t)valop
< 0) {
907 return helper_neon_qshl_u64(env
, valop
, shiftop
);
910 #define NEON_FN(dest, src1, src2) do { \
912 tmp = (int8_t)src2; \
913 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
920 } else if (tmp < -(ssize_t)sizeof(src1) * 8) { \
922 } else if (tmp == -(ssize_t)sizeof(src1) * 8) { \
923 dest = src1 >> (sizeof(src1) * 8 - 1); \
924 } else if (tmp < 0) { \
925 dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
927 dest = src1 << tmp; \
928 if ((dest >> tmp) != src1) { \
933 NEON_VOP_ENV(qrshl_u8
, neon_u8
, 4)
934 NEON_VOP_ENV(qrshl_u16
, neon_u16
, 2)
937 /* The addition of the rounding constant may overflow, so we use an
938 * intermediate 64 bit accumulator. */
939 uint32_t HELPER(neon_qrshl_u32
)(CPUARMState
*env
, uint32_t val
, uint32_t shiftop
)
942 int8_t shift
= (int8_t)shiftop
;
950 } else if (shift
< -32) {
952 } else if (shift
== -32) {
954 } else if (shift
< 0) {
955 uint64_t big_dest
= ((uint64_t)val
+ (1 << (-1 - shift
)));
956 dest
= big_dest
>> -shift
;
959 if ((dest
>> shift
) != val
) {
967 /* Handling addition overflow with 64 bit input values is more
968 * tricky than with 32 bit values. */
969 uint64_t HELPER(neon_qrshl_u64
)(CPUARMState
*env
, uint64_t val
, uint64_t shiftop
)
971 int8_t shift
= (int8_t)shiftop
;
977 } else if (shift
< -64) {
979 } else if (shift
== -64) {
981 } else if (shift
< 0) {
982 val
>>= (-shift
- 1);
983 if (val
== UINT64_MAX
) {
984 /* In this case, it means that the rounding constant is 1,
985 * and the addition would overflow. Return the actual
986 * result directly. */
987 val
= 0x8000000000000000ULL
;
995 if ((val
>> shift
) != tmp
) {
1003 #define NEON_FN(dest, src1, src2) do { \
1005 tmp = (int8_t)src2; \
1006 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
1009 dest = (typeof(dest))(1 << (sizeof(src1) * 8 - 1)); \
1016 } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
1018 } else if (tmp < 0) { \
1019 dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
1021 dest = src1 << tmp; \
1022 if ((dest >> tmp) != src1) { \
1024 dest = (uint32_t)(1 << (sizeof(src1) * 8 - 1)); \
1030 NEON_VOP_ENV(qrshl_s8
, neon_s8
, 4)
1031 NEON_VOP_ENV(qrshl_s16
, neon_s16
, 2)
1034 /* The addition of the rounding constant may overflow, so we use an
1035 * intermediate 64 bit accumulator. */
1036 uint32_t HELPER(neon_qrshl_s32
)(CPUARMState
*env
, uint32_t valop
, uint32_t shiftop
)
1039 int32_t val
= (int32_t)valop
;
1040 int8_t shift
= (int8_t)shiftop
;
1044 dest
= (val
>> 31) ^ ~SIGNBIT
;
1048 } else if (shift
<= -32) {
1050 } else if (shift
< 0) {
1051 int64_t big_dest
= ((int64_t)val
+ (1 << (-1 - shift
)));
1052 dest
= big_dest
>> -shift
;
1054 dest
= val
<< shift
;
1055 if ((dest
>> shift
) != val
) {
1057 dest
= (val
>> 31) ^ ~SIGNBIT
;
1063 /* Handling addition overflow with 64 bit input values is more
1064 * tricky than with 32 bit values. */
1065 uint64_t HELPER(neon_qrshl_s64
)(CPUARMState
*env
, uint64_t valop
, uint64_t shiftop
)
1067 int8_t shift
= (uint8_t)shiftop
;
1068 int64_t val
= valop
;
1073 val
= (val
>> 63) ^ ~SIGNBIT64
;
1075 } else if (shift
<= -64) {
1077 } else if (shift
< 0) {
1078 val
>>= (-shift
- 1);
1079 if (val
== INT64_MAX
) {
1080 /* In this case, it means that the rounding constant is 1,
1081 * and the addition would overflow. Return the actual
1082 * result directly. */
1083 val
= 0x4000000000000000ULL
;
1091 if ((val
>> shift
) != tmp
) {
1093 val
= (tmp
>> 63) ^ ~SIGNBIT64
;
1099 uint32_t HELPER(neon_add_u8
)(uint32_t a
, uint32_t b
)
1102 mask
= (a
^ b
) & 0x80808080u
;
1105 return (a
+ b
) ^ mask
;
1108 uint32_t HELPER(neon_add_u16
)(uint32_t a
, uint32_t b
)
1111 mask
= (a
^ b
) & 0x80008000u
;
1114 return (a
+ b
) ^ mask
;
1117 #define NEON_FN(dest, src1, src2) dest = src1 + src2
1118 NEON_POP(padd_u8
, neon_u8
, 4)
1119 NEON_POP(padd_u16
, neon_u16
, 2)
1122 #define NEON_FN(dest, src1, src2) dest = src1 - src2
1123 NEON_VOP(sub_u8
, neon_u8
, 4)
1124 NEON_VOP(sub_u16
, neon_u16
, 2)
1127 #define NEON_FN(dest, src1, src2) dest = src1 * src2
1128 NEON_VOP(mul_u8
, neon_u8
, 4)
1129 NEON_VOP(mul_u16
, neon_u16
, 2)
1132 #define NEON_FN(dest, src1, src2) dest = (src1 & src2) ? -1 : 0
1133 NEON_VOP(tst_u8
, neon_u8
, 4)
1134 NEON_VOP(tst_u16
, neon_u16
, 2)
1135 NEON_VOP(tst_u32
, neon_u32
, 1)
1138 #define NEON_FN(dest, src1, src2) dest = (src1 == src2) ? -1 : 0
1139 NEON_VOP(ceq_u8
, neon_u8
, 4)
1140 NEON_VOP(ceq_u16
, neon_u16
, 2)
1141 NEON_VOP(ceq_u32
, neon_u32
, 1)
1144 /* Count Leading Sign/Zero Bits. */
1145 static inline int do_clz8(uint8_t x
)
1153 static inline int do_clz16(uint16_t x
)
1156 for (n
= 16; x
; n
--)
1161 #define NEON_FN(dest, src, dummy) dest = do_clz8(src)
1162 NEON_VOP1(clz_u8
, neon_u8
, 4)
1165 #define NEON_FN(dest, src, dummy) dest = do_clz16(src)
1166 NEON_VOP1(clz_u16
, neon_u16
, 2)
1169 #define NEON_FN(dest, src, dummy) dest = do_clz8((src < 0) ? ~src : src) - 1
1170 NEON_VOP1(cls_s8
, neon_s8
, 4)
1173 #define NEON_FN(dest, src, dummy) dest = do_clz16((src < 0) ? ~src : src) - 1
1174 NEON_VOP1(cls_s16
, neon_s16
, 2)
1177 uint32_t HELPER(neon_cls_s32
)(uint32_t x
)
1182 for (count
= 32; x
; count
--)
1188 uint32_t HELPER(neon_cnt_u8
)(uint32_t x
)
1190 x
= (x
& 0x55555555) + ((x
>> 1) & 0x55555555);
1191 x
= (x
& 0x33333333) + ((x
>> 2) & 0x33333333);
1192 x
= (x
& 0x0f0f0f0f) + ((x
>> 4) & 0x0f0f0f0f);
1196 /* Reverse bits in each 8 bit word */
1197 uint32_t HELPER(neon_rbit_u8
)(uint32_t x
)
1199 x
= ((x
& 0xf0f0f0f0) >> 4)
1200 | ((x
& 0x0f0f0f0f) << 4);
1201 x
= ((x
& 0x88888888) >> 3)
1202 | ((x
& 0x44444444) >> 1)
1203 | ((x
& 0x22222222) << 1)
1204 | ((x
& 0x11111111) << 3);
1208 #define NEON_QDMULH16(dest, src1, src2, round) do { \
1209 uint32_t tmp = (int32_t)(int16_t) src1 * (int16_t) src2; \
1210 if ((tmp ^ (tmp << 1)) & SIGNBIT) { \
1212 tmp = (tmp >> 31) ^ ~SIGNBIT; \
1217 int32_t old = tmp; \
1219 if ((int32_t)tmp < old) { \
1221 tmp = SIGNBIT - 1; \
1226 #define NEON_FN(dest, src1, src2) NEON_QDMULH16(dest, src1, src2, 0)
1227 NEON_VOP_ENV(qdmulh_s16
, neon_s16
, 2)
1229 #define NEON_FN(dest, src1, src2) NEON_QDMULH16(dest, src1, src2, 1)
1230 NEON_VOP_ENV(qrdmulh_s16
, neon_s16
, 2)
1232 #undef NEON_QDMULH16
1234 #define NEON_QDMULH32(dest, src1, src2, round) do { \
1235 uint64_t tmp = (int64_t)(int32_t) src1 * (int32_t) src2; \
1236 if ((tmp ^ (tmp << 1)) & SIGNBIT64) { \
1238 tmp = (tmp >> 63) ^ ~SIGNBIT64; \
1243 int64_t old = tmp; \
1244 tmp += (int64_t)1 << 31; \
1245 if ((int64_t)tmp < old) { \
1247 tmp = SIGNBIT64 - 1; \
1252 #define NEON_FN(dest, src1, src2) NEON_QDMULH32(dest, src1, src2, 0)
1253 NEON_VOP_ENV(qdmulh_s32
, neon_s32
, 1)
1255 #define NEON_FN(dest, src1, src2) NEON_QDMULH32(dest, src1, src2, 1)
1256 NEON_VOP_ENV(qrdmulh_s32
, neon_s32
, 1)
1258 #undef NEON_QDMULH32
1260 uint32_t HELPER(neon_narrow_u8
)(uint64_t x
)
1262 return (x
& 0xffu
) | ((x
>> 8) & 0xff00u
) | ((x
>> 16) & 0xff0000u
)
1263 | ((x
>> 24) & 0xff000000u
);
1266 uint32_t HELPER(neon_narrow_u16
)(uint64_t x
)
1268 return (x
& 0xffffu
) | ((x
>> 16) & 0xffff0000u
);
1271 uint32_t HELPER(neon_narrow_high_u8
)(uint64_t x
)
1273 return ((x
>> 8) & 0xff) | ((x
>> 16) & 0xff00)
1274 | ((x
>> 24) & 0xff0000) | ((x
>> 32) & 0xff000000);
1277 uint32_t HELPER(neon_narrow_high_u16
)(uint64_t x
)
1279 return ((x
>> 16) & 0xffff) | ((x
>> 32) & 0xffff0000);
1282 uint32_t HELPER(neon_narrow_round_high_u8
)(uint64_t x
)
1284 x
&= 0xff80ff80ff80ff80ull
;
1285 x
+= 0x0080008000800080ull
;
1286 return ((x
>> 8) & 0xff) | ((x
>> 16) & 0xff00)
1287 | ((x
>> 24) & 0xff0000) | ((x
>> 32) & 0xff000000);
1290 uint32_t HELPER(neon_narrow_round_high_u16
)(uint64_t x
)
1292 x
&= 0xffff8000ffff8000ull
;
1293 x
+= 0x0000800000008000ull
;
1294 return ((x
>> 16) & 0xffff) | ((x
>> 32) & 0xffff0000);
1297 uint32_t HELPER(neon_unarrow_sat8
)(CPUARMState
*env
, uint64_t x
)
1313 res |= (uint32_t)d << (n / 2); \
1324 uint32_t HELPER(neon_narrow_sat_u8
)(CPUARMState
*env
, uint64_t x
)
1337 res |= (uint32_t)d << (n / 2);
1347 uint32_t HELPER(neon_narrow_sat_s8
)(CPUARMState
*env
, uint64_t x
)
1354 if (s != (int8_t)s) { \
1355 d = (s >> 15) ^ 0x7f; \
1360 res |= (uint32_t)d << (n / 2);
1370 uint32_t HELPER(neon_unarrow_sat16
)(CPUARMState
*env
, uint64_t x
)
1375 if (low
& 0x80000000) {
1378 } else if (low
> 0xffff) {
1383 if (high
& 0x80000000) {
1386 } else if (high
> 0xffff) {
1390 return low
| (high
<< 16);
1393 uint32_t HELPER(neon_narrow_sat_u16
)(CPUARMState
*env
, uint64_t x
)
1403 if (high
> 0xffff) {
1407 return low
| (high
<< 16);
1410 uint32_t HELPER(neon_narrow_sat_s16
)(CPUARMState
*env
, uint64_t x
)
1415 if (low
!= (int16_t)low
) {
1416 low
= (low
>> 31) ^ 0x7fff;
1420 if (high
!= (int16_t)high
) {
1421 high
= (high
>> 31) ^ 0x7fff;
1424 return (uint16_t)low
| (high
<< 16);
1427 uint32_t HELPER(neon_unarrow_sat32
)(CPUARMState
*env
, uint64_t x
)
1429 if (x
& 0x8000000000000000ull
) {
1433 if (x
> 0xffffffffu
) {
1440 uint32_t HELPER(neon_narrow_sat_u32
)(CPUARMState
*env
, uint64_t x
)
1442 if (x
> 0xffffffffu
) {
1449 uint32_t HELPER(neon_narrow_sat_s32
)(CPUARMState
*env
, uint64_t x
)
1451 if ((int64_t)x
!= (int32_t)x
) {
1453 return ((int64_t)x
>> 63) ^ 0x7fffffff;
1458 uint64_t HELPER(neon_widen_u8
)(uint32_t x
)
1463 tmp
= (uint8_t)(x
>> 8);
1465 tmp
= (uint8_t)(x
>> 16);
1467 tmp
= (uint8_t)(x
>> 24);
1472 uint64_t HELPER(neon_widen_s8
)(uint32_t x
)
1476 ret
= (uint16_t)(int8_t)x
;
1477 tmp
= (uint16_t)(int8_t)(x
>> 8);
1479 tmp
= (uint16_t)(int8_t)(x
>> 16);
1481 tmp
= (uint16_t)(int8_t)(x
>> 24);
1486 uint64_t HELPER(neon_widen_u16
)(uint32_t x
)
1488 uint64_t high
= (uint16_t)(x
>> 16);
1489 return ((uint16_t)x
) | (high
<< 32);
1492 uint64_t HELPER(neon_widen_s16
)(uint32_t x
)
1494 uint64_t high
= (int16_t)(x
>> 16);
1495 return ((uint32_t)(int16_t)x
) | (high
<< 32);
1498 uint64_t HELPER(neon_addl_u16
)(uint64_t a
, uint64_t b
)
1501 mask
= (a
^ b
) & 0x8000800080008000ull
;
1502 a
&= ~0x8000800080008000ull
;
1503 b
&= ~0x8000800080008000ull
;
1504 return (a
+ b
) ^ mask
;
1507 uint64_t HELPER(neon_addl_u32
)(uint64_t a
, uint64_t b
)
1510 mask
= (a
^ b
) & 0x8000000080000000ull
;
1511 a
&= ~0x8000000080000000ull
;
1512 b
&= ~0x8000000080000000ull
;
1513 return (a
+ b
) ^ mask
;
1516 uint64_t HELPER(neon_paddl_u16
)(uint64_t a
, uint64_t b
)
1521 tmp
= a
& 0x0000ffff0000ffffull
;
1522 tmp
+= (a
>> 16) & 0x0000ffff0000ffffull
;
1523 tmp2
= b
& 0xffff0000ffff0000ull
;
1524 tmp2
+= (b
<< 16) & 0xffff0000ffff0000ull
;
1525 return ( tmp
& 0xffff)
1526 | ((tmp
>> 16) & 0xffff0000ull
)
1527 | ((tmp2
<< 16) & 0xffff00000000ull
)
1528 | ( tmp2
& 0xffff000000000000ull
);
1531 uint64_t HELPER(neon_paddl_u32
)(uint64_t a
, uint64_t b
)
1533 uint32_t low
= a
+ (a
>> 32);
1534 uint32_t high
= b
+ (b
>> 32);
1535 return low
+ ((uint64_t)high
<< 32);
1538 uint64_t HELPER(neon_subl_u16
)(uint64_t a
, uint64_t b
)
1541 mask
= (a
^ ~b
) & 0x8000800080008000ull
;
1542 a
|= 0x8000800080008000ull
;
1543 b
&= ~0x8000800080008000ull
;
1544 return (a
- b
) ^ mask
;
1547 uint64_t HELPER(neon_subl_u32
)(uint64_t a
, uint64_t b
)
1550 mask
= (a
^ ~b
) & 0x8000000080000000ull
;
1551 a
|= 0x8000000080000000ull
;
1552 b
&= ~0x8000000080000000ull
;
1553 return (a
- b
) ^ mask
;
1556 uint64_t HELPER(neon_addl_saturate_s32
)(CPUARMState
*env
, uint64_t a
, uint64_t b
)
1564 if (((low
^ x
) & SIGNBIT
) && !((x
^ y
) & SIGNBIT
)) {
1566 low
= ((int32_t)x
>> 31) ^ ~SIGNBIT
;
1571 if (((high
^ x
) & SIGNBIT
) && !((x
^ y
) & SIGNBIT
)) {
1573 high
= ((int32_t)x
>> 31) ^ ~SIGNBIT
;
1575 return low
| ((uint64_t)high
<< 32);
1578 uint64_t HELPER(neon_addl_saturate_s64
)(CPUARMState
*env
, uint64_t a
, uint64_t b
)
1583 if (((result
^ a
) & SIGNBIT64
) && !((a
^ b
) & SIGNBIT64
)) {
1585 result
= ((int64_t)a
>> 63) ^ ~SIGNBIT64
;
1590 /* We have to do the arithmetic in a larger type than
1591 * the input type, because for example with a signed 32 bit
1592 * op the absolute difference can overflow a signed 32 bit value.
1594 #define DO_ABD(dest, x, y, intype, arithtype) do { \
1595 arithtype tmp_x = (intype)(x); \
1596 arithtype tmp_y = (intype)(y); \
1597 dest = ((tmp_x > tmp_y) ? tmp_x - tmp_y : tmp_y - tmp_x); \
1600 uint64_t HELPER(neon_abdl_u16
)(uint32_t a
, uint32_t b
)
1604 DO_ABD(result
, a
, b
, uint8_t, uint32_t);
1605 DO_ABD(tmp
, a
>> 8, b
>> 8, uint8_t, uint32_t);
1606 result
|= tmp
<< 16;
1607 DO_ABD(tmp
, a
>> 16, b
>> 16, uint8_t, uint32_t);
1608 result
|= tmp
<< 32;
1609 DO_ABD(tmp
, a
>> 24, b
>> 24, uint8_t, uint32_t);
1610 result
|= tmp
<< 48;
1614 uint64_t HELPER(neon_abdl_s16
)(uint32_t a
, uint32_t b
)
1618 DO_ABD(result
, a
, b
, int8_t, int32_t);
1619 DO_ABD(tmp
, a
>> 8, b
>> 8, int8_t, int32_t);
1620 result
|= tmp
<< 16;
1621 DO_ABD(tmp
, a
>> 16, b
>> 16, int8_t, int32_t);
1622 result
|= tmp
<< 32;
1623 DO_ABD(tmp
, a
>> 24, b
>> 24, int8_t, int32_t);
1624 result
|= tmp
<< 48;
1628 uint64_t HELPER(neon_abdl_u32
)(uint32_t a
, uint32_t b
)
1632 DO_ABD(result
, a
, b
, uint16_t, uint32_t);
1633 DO_ABD(tmp
, a
>> 16, b
>> 16, uint16_t, uint32_t);
1634 return result
| (tmp
<< 32);
1637 uint64_t HELPER(neon_abdl_s32
)(uint32_t a
, uint32_t b
)
1641 DO_ABD(result
, a
, b
, int16_t, int32_t);
1642 DO_ABD(tmp
, a
>> 16, b
>> 16, int16_t, int32_t);
1643 return result
| (tmp
<< 32);
1646 uint64_t HELPER(neon_abdl_u64
)(uint32_t a
, uint32_t b
)
1649 DO_ABD(result
, a
, b
, uint32_t, uint64_t);
1653 uint64_t HELPER(neon_abdl_s64
)(uint32_t a
, uint32_t b
)
1656 DO_ABD(result
, a
, b
, int32_t, int64_t);
1661 /* Widening multiply. Named type is the source type. */
1662 #define DO_MULL(dest, x, y, type1, type2) do { \
1665 dest = (type2)((type2)tmp_x * (type2)tmp_y); \
1668 uint64_t HELPER(neon_mull_u8
)(uint32_t a
, uint32_t b
)
1673 DO_MULL(result
, a
, b
, uint8_t, uint16_t);
1674 DO_MULL(tmp
, a
>> 8, b
>> 8, uint8_t, uint16_t);
1675 result
|= tmp
<< 16;
1676 DO_MULL(tmp
, a
>> 16, b
>> 16, uint8_t, uint16_t);
1677 result
|= tmp
<< 32;
1678 DO_MULL(tmp
, a
>> 24, b
>> 24, uint8_t, uint16_t);
1679 result
|= tmp
<< 48;
1683 uint64_t HELPER(neon_mull_s8
)(uint32_t a
, uint32_t b
)
1688 DO_MULL(result
, a
, b
, int8_t, uint16_t);
1689 DO_MULL(tmp
, a
>> 8, b
>> 8, int8_t, uint16_t);
1690 result
|= tmp
<< 16;
1691 DO_MULL(tmp
, a
>> 16, b
>> 16, int8_t, uint16_t);
1692 result
|= tmp
<< 32;
1693 DO_MULL(tmp
, a
>> 24, b
>> 24, int8_t, uint16_t);
1694 result
|= tmp
<< 48;
1698 uint64_t HELPER(neon_mull_u16
)(uint32_t a
, uint32_t b
)
1703 DO_MULL(result
, a
, b
, uint16_t, uint32_t);
1704 DO_MULL(tmp
, a
>> 16, b
>> 16, uint16_t, uint32_t);
1705 return result
| (tmp
<< 32);
1708 uint64_t HELPER(neon_mull_s16
)(uint32_t a
, uint32_t b
)
1713 DO_MULL(result
, a
, b
, int16_t, uint32_t);
1714 DO_MULL(tmp
, a
>> 16, b
>> 16, int16_t, uint32_t);
1715 return result
| (tmp
<< 32);
1718 uint64_t HELPER(neon_negl_u16
)(uint64_t x
)
1722 result
= (uint16_t)-x
;
1724 result
|= (uint64_t)tmp
<< 16;
1726 result
|= (uint64_t)tmp
<< 32;
1728 result
|= (uint64_t)tmp
<< 48;
1732 uint64_t HELPER(neon_negl_u32
)(uint64_t x
)
1735 uint32_t high
= -(x
>> 32);
1736 return low
| ((uint64_t)high
<< 32);
1739 /* Saturating sign manipulation. */
1740 /* ??? Make these use NEON_VOP1 */
1741 #define DO_QABS8(x) do { \
1742 if (x == (int8_t)0x80) { \
1745 } else if (x < 0) { \
1748 uint32_t HELPER(neon_qabs_s8
)(CPUARMState
*env
, uint32_t x
)
1751 NEON_UNPACK(neon_s8
, vec
, x
);
1756 NEON_PACK(neon_s8
, x
, vec
);
1761 #define DO_QNEG8(x) do { \
1762 if (x == (int8_t)0x80) { \
1768 uint32_t HELPER(neon_qneg_s8
)(CPUARMState
*env
, uint32_t x
)
1771 NEON_UNPACK(neon_s8
, vec
, x
);
1776 NEON_PACK(neon_s8
, x
, vec
);
1781 #define DO_QABS16(x) do { \
1782 if (x == (int16_t)0x8000) { \
1785 } else if (x < 0) { \
1788 uint32_t HELPER(neon_qabs_s16
)(CPUARMState
*env
, uint32_t x
)
1791 NEON_UNPACK(neon_s16
, vec
, x
);
1794 NEON_PACK(neon_s16
, x
, vec
);
1799 #define DO_QNEG16(x) do { \
1800 if (x == (int16_t)0x8000) { \
1806 uint32_t HELPER(neon_qneg_s16
)(CPUARMState
*env
, uint32_t x
)
1809 NEON_UNPACK(neon_s16
, vec
, x
);
1812 NEON_PACK(neon_s16
, x
, vec
);
1817 uint32_t HELPER(neon_qabs_s32
)(CPUARMState
*env
, uint32_t x
)
1822 } else if ((int32_t)x
< 0) {
1828 uint32_t HELPER(neon_qneg_s32
)(CPUARMState
*env
, uint32_t x
)
1839 uint64_t HELPER(neon_qabs_s64
)(CPUARMState
*env
, uint64_t x
)
1841 if (x
== SIGNBIT64
) {
1844 } else if ((int64_t)x
< 0) {
1850 uint64_t HELPER(neon_qneg_s64
)(CPUARMState
*env
, uint64_t x
)
1852 if (x
== SIGNBIT64
) {
1861 /* NEON Float helpers. */
1862 uint32_t HELPER(neon_abd_f32
)(uint32_t a
, uint32_t b
, void *fpstp
)
1864 float_status
*fpst
= fpstp
;
1865 float32 f0
= make_float32(a
);
1866 float32 f1
= make_float32(b
);
1867 return float32_val(float32_abs(float32_sub(f0
, f1
, fpst
)));
1870 /* Floating point comparisons produce an integer result.
1871 * Note that EQ doesn't signal InvalidOp for QNaNs but GE and GT do.
1872 * Softfloat routines return 0/1, which we convert to the 0/-1 Neon requires.
1874 uint32_t HELPER(neon_ceq_f32
)(uint32_t a
, uint32_t b
, void *fpstp
)
1876 float_status
*fpst
= fpstp
;
1877 return -float32_eq_quiet(make_float32(a
), make_float32(b
), fpst
);
1880 uint32_t HELPER(neon_cge_f32
)(uint32_t a
, uint32_t b
, void *fpstp
)
1882 float_status
*fpst
= fpstp
;
1883 return -float32_le(make_float32(b
), make_float32(a
), fpst
);
1886 uint32_t HELPER(neon_cgt_f32
)(uint32_t a
, uint32_t b
, void *fpstp
)
1888 float_status
*fpst
= fpstp
;
1889 return -float32_lt(make_float32(b
), make_float32(a
), fpst
);
1892 uint32_t HELPER(neon_acge_f32
)(uint32_t a
, uint32_t b
, void *fpstp
)
1894 float_status
*fpst
= fpstp
;
1895 float32 f0
= float32_abs(make_float32(a
));
1896 float32 f1
= float32_abs(make_float32(b
));
1897 return -float32_le(f1
, f0
, fpst
);
1900 uint32_t HELPER(neon_acgt_f32
)(uint32_t a
, uint32_t b
, void *fpstp
)
1902 float_status
*fpst
= fpstp
;
1903 float32 f0
= float32_abs(make_float32(a
));
1904 float32 f1
= float32_abs(make_float32(b
));
1905 return -float32_lt(f1
, f0
, fpst
);
1908 uint64_t HELPER(neon_acge_f64
)(uint64_t a
, uint64_t b
, void *fpstp
)
1910 float_status
*fpst
= fpstp
;
1911 float64 f0
= float64_abs(make_float64(a
));
1912 float64 f1
= float64_abs(make_float64(b
));
1913 return -float64_le(f1
, f0
, fpst
);
1916 uint64_t HELPER(neon_acgt_f64
)(uint64_t a
, uint64_t b
, void *fpstp
)
1918 float_status
*fpst
= fpstp
;
1919 float64 f0
= float64_abs(make_float64(a
));
1920 float64 f1
= float64_abs(make_float64(b
));
1921 return -float64_lt(f1
, f0
, fpst
);
1924 #define ELEM(V, N, SIZE) (((V) >> ((N) * (SIZE))) & ((1ull << (SIZE)) - 1))
1926 void HELPER(neon_qunzip8
)(void *vd
, void *vm
)
1928 uint64_t *rd
= vd
, *rm
= vm
;
1929 uint64_t zd0
= rd
[0], zd1
= rd
[1];
1930 uint64_t zm0
= rm
[0], zm1
= rm
[1];
1932 uint64_t d0
= ELEM(zd0
, 0, 8) | (ELEM(zd0
, 2, 8) << 8)
1933 | (ELEM(zd0
, 4, 8) << 16) | (ELEM(zd0
, 6, 8) << 24)
1934 | (ELEM(zd1
, 0, 8) << 32) | (ELEM(zd1
, 2, 8) << 40)
1935 | (ELEM(zd1
, 4, 8) << 48) | (ELEM(zd1
, 6, 8) << 56);
1936 uint64_t d1
= ELEM(zm0
, 0, 8) | (ELEM(zm0
, 2, 8) << 8)
1937 | (ELEM(zm0
, 4, 8) << 16) | (ELEM(zm0
, 6, 8) << 24)
1938 | (ELEM(zm1
, 0, 8) << 32) | (ELEM(zm1
, 2, 8) << 40)
1939 | (ELEM(zm1
, 4, 8) << 48) | (ELEM(zm1
, 6, 8) << 56);
1940 uint64_t m0
= ELEM(zd0
, 1, 8) | (ELEM(zd0
, 3, 8) << 8)
1941 | (ELEM(zd0
, 5, 8) << 16) | (ELEM(zd0
, 7, 8) << 24)
1942 | (ELEM(zd1
, 1, 8) << 32) | (ELEM(zd1
, 3, 8) << 40)
1943 | (ELEM(zd1
, 5, 8) << 48) | (ELEM(zd1
, 7, 8) << 56);
1944 uint64_t m1
= ELEM(zm0
, 1, 8) | (ELEM(zm0
, 3, 8) << 8)
1945 | (ELEM(zm0
, 5, 8) << 16) | (ELEM(zm0
, 7, 8) << 24)
1946 | (ELEM(zm1
, 1, 8) << 32) | (ELEM(zm1
, 3, 8) << 40)
1947 | (ELEM(zm1
, 5, 8) << 48) | (ELEM(zm1
, 7, 8) << 56);
1955 void HELPER(neon_qunzip16
)(void *vd
, void *vm
)
1957 uint64_t *rd
= vd
, *rm
= vm
;
1958 uint64_t zd0
= rd
[0], zd1
= rd
[1];
1959 uint64_t zm0
= rm
[0], zm1
= rm
[1];
1961 uint64_t d0
= ELEM(zd0
, 0, 16) | (ELEM(zd0
, 2, 16) << 16)
1962 | (ELEM(zd1
, 0, 16) << 32) | (ELEM(zd1
, 2, 16) << 48);
1963 uint64_t d1
= ELEM(zm0
, 0, 16) | (ELEM(zm0
, 2, 16) << 16)
1964 | (ELEM(zm1
, 0, 16) << 32) | (ELEM(zm1
, 2, 16) << 48);
1965 uint64_t m0
= ELEM(zd0
, 1, 16) | (ELEM(zd0
, 3, 16) << 16)
1966 | (ELEM(zd1
, 1, 16) << 32) | (ELEM(zd1
, 3, 16) << 48);
1967 uint64_t m1
= ELEM(zm0
, 1, 16) | (ELEM(zm0
, 3, 16) << 16)
1968 | (ELEM(zm1
, 1, 16) << 32) | (ELEM(zm1
, 3, 16) << 48);
1976 void HELPER(neon_qunzip32
)(void *vd
, void *vm
)
1978 uint64_t *rd
= vd
, *rm
= vm
;
1979 uint64_t zd0
= rd
[0], zd1
= rd
[1];
1980 uint64_t zm0
= rm
[0], zm1
= rm
[1];
1982 uint64_t d0
= ELEM(zd0
, 0, 32) | (ELEM(zd1
, 0, 32) << 32);
1983 uint64_t d1
= ELEM(zm0
, 0, 32) | (ELEM(zm1
, 0, 32) << 32);
1984 uint64_t m0
= ELEM(zd0
, 1, 32) | (ELEM(zd1
, 1, 32) << 32);
1985 uint64_t m1
= ELEM(zm0
, 1, 32) | (ELEM(zm1
, 1, 32) << 32);
1993 void HELPER(neon_unzip8
)(void *vd
, void *vm
)
1995 uint64_t *rd
= vd
, *rm
= vm
;
1996 uint64_t zd
= rd
[0], zm
= rm
[0];
1998 uint64_t d0
= ELEM(zd
, 0, 8) | (ELEM(zd
, 2, 8) << 8)
1999 | (ELEM(zd
, 4, 8) << 16) | (ELEM(zd
, 6, 8) << 24)
2000 | (ELEM(zm
, 0, 8) << 32) | (ELEM(zm
, 2, 8) << 40)
2001 | (ELEM(zm
, 4, 8) << 48) | (ELEM(zm
, 6, 8) << 56);
2002 uint64_t m0
= ELEM(zd
, 1, 8) | (ELEM(zd
, 3, 8) << 8)
2003 | (ELEM(zd
, 5, 8) << 16) | (ELEM(zd
, 7, 8) << 24)
2004 | (ELEM(zm
, 1, 8) << 32) | (ELEM(zm
, 3, 8) << 40)
2005 | (ELEM(zm
, 5, 8) << 48) | (ELEM(zm
, 7, 8) << 56);
2011 void HELPER(neon_unzip16
)(void *vd
, void *vm
)
2013 uint64_t *rd
= vd
, *rm
= vm
;
2014 uint64_t zd
= rd
[0], zm
= rm
[0];
2016 uint64_t d0
= ELEM(zd
, 0, 16) | (ELEM(zd
, 2, 16) << 16)
2017 | (ELEM(zm
, 0, 16) << 32) | (ELEM(zm
, 2, 16) << 48);
2018 uint64_t m0
= ELEM(zd
, 1, 16) | (ELEM(zd
, 3, 16) << 16)
2019 | (ELEM(zm
, 1, 16) << 32) | (ELEM(zm
, 3, 16) << 48);
2025 void HELPER(neon_qzip8
)(void *vd
, void *vm
)
2027 uint64_t *rd
= vd
, *rm
= vm
;
2028 uint64_t zd0
= rd
[0], zd1
= rd
[1];
2029 uint64_t zm0
= rm
[0], zm1
= rm
[1];
2031 uint64_t d0
= ELEM(zd0
, 0, 8) | (ELEM(zm0
, 0, 8) << 8)
2032 | (ELEM(zd0
, 1, 8) << 16) | (ELEM(zm0
, 1, 8) << 24)
2033 | (ELEM(zd0
, 2, 8) << 32) | (ELEM(zm0
, 2, 8) << 40)
2034 | (ELEM(zd0
, 3, 8) << 48) | (ELEM(zm0
, 3, 8) << 56);
2035 uint64_t d1
= ELEM(zd0
, 4, 8) | (ELEM(zm0
, 4, 8) << 8)
2036 | (ELEM(zd0
, 5, 8) << 16) | (ELEM(zm0
, 5, 8) << 24)
2037 | (ELEM(zd0
, 6, 8) << 32) | (ELEM(zm0
, 6, 8) << 40)
2038 | (ELEM(zd0
, 7, 8) << 48) | (ELEM(zm0
, 7, 8) << 56);
2039 uint64_t m0
= ELEM(zd1
, 0, 8) | (ELEM(zm1
, 0, 8) << 8)
2040 | (ELEM(zd1
, 1, 8) << 16) | (ELEM(zm1
, 1, 8) << 24)
2041 | (ELEM(zd1
, 2, 8) << 32) | (ELEM(zm1
, 2, 8) << 40)
2042 | (ELEM(zd1
, 3, 8) << 48) | (ELEM(zm1
, 3, 8) << 56);
2043 uint64_t m1
= ELEM(zd1
, 4, 8) | (ELEM(zm1
, 4, 8) << 8)
2044 | (ELEM(zd1
, 5, 8) << 16) | (ELEM(zm1
, 5, 8) << 24)
2045 | (ELEM(zd1
, 6, 8) << 32) | (ELEM(zm1
, 6, 8) << 40)
2046 | (ELEM(zd1
, 7, 8) << 48) | (ELEM(zm1
, 7, 8) << 56);
2054 void HELPER(neon_qzip16
)(void *vd
, void *vm
)
2056 uint64_t *rd
= vd
, *rm
= vm
;
2057 uint64_t zd0
= rd
[0], zd1
= rd
[1];
2058 uint64_t zm0
= rm
[0], zm1
= rm
[1];
2060 uint64_t d0
= ELEM(zd0
, 0, 16) | (ELEM(zm0
, 0, 16) << 16)
2061 | (ELEM(zd0
, 1, 16) << 32) | (ELEM(zm0
, 1, 16) << 48);
2062 uint64_t d1
= ELEM(zd0
, 2, 16) | (ELEM(zm0
, 2, 16) << 16)
2063 | (ELEM(zd0
, 3, 16) << 32) | (ELEM(zm0
, 3, 16) << 48);
2064 uint64_t m0
= ELEM(zd1
, 0, 16) | (ELEM(zm1
, 0, 16) << 16)
2065 | (ELEM(zd1
, 1, 16) << 32) | (ELEM(zm1
, 1, 16) << 48);
2066 uint64_t m1
= ELEM(zd1
, 2, 16) | (ELEM(zm1
, 2, 16) << 16)
2067 | (ELEM(zd1
, 3, 16) << 32) | (ELEM(zm1
, 3, 16) << 48);
2075 void HELPER(neon_qzip32
)(void *vd
, void *vm
)
2077 uint64_t *rd
= vd
, *rm
= vm
;
2078 uint64_t zd0
= rd
[0], zd1
= rd
[1];
2079 uint64_t zm0
= rm
[0], zm1
= rm
[1];
2081 uint64_t d0
= ELEM(zd0
, 0, 32) | (ELEM(zm0
, 0, 32) << 32);
2082 uint64_t d1
= ELEM(zd0
, 1, 32) | (ELEM(zm0
, 1, 32) << 32);
2083 uint64_t m0
= ELEM(zd1
, 0, 32) | (ELEM(zm1
, 0, 32) << 32);
2084 uint64_t m1
= ELEM(zd1
, 1, 32) | (ELEM(zm1
, 1, 32) << 32);
2092 void HELPER(neon_zip8
)(void *vd
, void *vm
)
2094 uint64_t *rd
= vd
, *rm
= vm
;
2095 uint64_t zd
= rd
[0], zm
= rm
[0];
2097 uint64_t d0
= ELEM(zd
, 0, 8) | (ELEM(zm
, 0, 8) << 8)
2098 | (ELEM(zd
, 1, 8) << 16) | (ELEM(zm
, 1, 8) << 24)
2099 | (ELEM(zd
, 2, 8) << 32) | (ELEM(zm
, 2, 8) << 40)
2100 | (ELEM(zd
, 3, 8) << 48) | (ELEM(zm
, 3, 8) << 56);
2101 uint64_t m0
= ELEM(zd
, 4, 8) | (ELEM(zm
, 4, 8) << 8)
2102 | (ELEM(zd
, 5, 8) << 16) | (ELEM(zm
, 5, 8) << 24)
2103 | (ELEM(zd
, 6, 8) << 32) | (ELEM(zm
, 6, 8) << 40)
2104 | (ELEM(zd
, 7, 8) << 48) | (ELEM(zm
, 7, 8) << 56);
2110 void HELPER(neon_zip16
)(void *vd
, void *vm
)
2112 uint64_t *rd
= vd
, *rm
= vm
;
2113 uint64_t zd
= rd
[0], zm
= rm
[0];
2115 uint64_t d0
= ELEM(zd
, 0, 16) | (ELEM(zm
, 0, 16) << 16)
2116 | (ELEM(zd
, 1, 16) << 32) | (ELEM(zm
, 1, 16) << 48);
2117 uint64_t m0
= ELEM(zd
, 2, 16) | (ELEM(zm
, 2, 16) << 16)
2118 | (ELEM(zd
, 3, 16) << 32) | (ELEM(zm
, 3, 16) << 48);