2 * ARM NEON vector operations.
4 * Copyright (c) 2007, 2008 CodeSourcery.
5 * Written by Paul Brook
7 * This code is licenced under the GNU GPL v2.
16 #define SIGNBIT (uint32_t)0x80000000
17 #define SIGNBIT64 ((uint64_t)1 << 63)
19 #define SET_QC() env->vfp.xregs[ARM_VFP_FPSCR] = CPSR_Q
21 static float_status neon_float_status
;
22 #define NFS &neon_float_status
24 #define NEON_TYPE1(name, type) \
29 #ifdef HOST_WORDS_BIGENDIAN
30 #define NEON_TYPE2(name, type) \
36 #define NEON_TYPE4(name, type) \
45 #define NEON_TYPE2(name, type) \
51 #define NEON_TYPE4(name, type) \
61 NEON_TYPE4(s8
, int8_t)
62 NEON_TYPE4(u8
, uint8_t)
63 NEON_TYPE2(s16
, int16_t)
64 NEON_TYPE2(u16
, uint16_t)
65 NEON_TYPE1(s32
, int32_t)
66 NEON_TYPE1(u32
, uint32_t)
71 /* Copy from a uint32_t to a vector structure type. */
72 #define NEON_UNPACK(vtype, dest, val) do { \
81 /* Copy from a vector structure type to a uint32_t. */
82 #define NEON_PACK(vtype, dest, val) do { \
92 NEON_FN(vdest.v1, vsrc1.v1, vsrc2.v1);
94 NEON_FN(vdest.v1, vsrc1.v1, vsrc2.v1); \
95 NEON_FN(vdest.v2, vsrc1.v2, vsrc2.v2);
97 NEON_FN(vdest.v1, vsrc1.v1, vsrc2.v1); \
98 NEON_FN(vdest.v2, vsrc1.v2, vsrc2.v2); \
99 NEON_FN(vdest.v3, vsrc1.v3, vsrc2.v3); \
100 NEON_FN(vdest.v4, vsrc1.v4, vsrc2.v4);
102 #define NEON_VOP_BODY(vtype, n) \
108 NEON_UNPACK(vtype, vsrc1, arg1); \
109 NEON_UNPACK(vtype, vsrc2, arg2); \
111 NEON_PACK(vtype, res, vdest); \
115 #define NEON_VOP(name, vtype, n) \
116 uint32_t HELPER(glue(neon_,name))(uint32_t arg1, uint32_t arg2) \
117 NEON_VOP_BODY(vtype, n)
119 #define NEON_VOP_ENV(name, vtype, n) \
120 uint32_t HELPER(glue(neon_,name))(CPUState *env, uint32_t arg1, uint32_t arg2) \
121 NEON_VOP_BODY(vtype, n)
123 /* Pairwise operations. */
124 /* For 32-bit elements each segment only contains a single element, so
125 the elementwise and pairwise operations are the same. */
127 NEON_FN(vdest.v1, vsrc1.v1, vsrc1.v2); \
128 NEON_FN(vdest.v2, vsrc2.v1, vsrc2.v2);
130 NEON_FN(vdest.v1, vsrc1.v1, vsrc1.v2); \
131 NEON_FN(vdest.v2, vsrc1.v3, vsrc1.v4); \
132 NEON_FN(vdest.v3, vsrc2.v1, vsrc2.v2); \
133 NEON_FN(vdest.v4, vsrc2.v3, vsrc2.v4); \
135 #define NEON_POP(name, vtype, n) \
136 uint32_t HELPER(glue(neon_,name))(uint32_t arg1, uint32_t arg2) \
142 NEON_UNPACK(vtype, vsrc1, arg1); \
143 NEON_UNPACK(vtype, vsrc2, arg2); \
145 NEON_PACK(vtype, res, vdest); \
149 /* Unary operators. */
150 #define NEON_VOP1(name, vtype, n) \
151 uint32_t HELPER(glue(neon_,name))(uint32_t arg) \
155 NEON_UNPACK(vtype, vsrc1, arg); \
157 NEON_PACK(vtype, arg, vdest); \
162 #define NEON_USAT(dest, src1, src2, type) do { \
163 uint32_t tmp = (uint32_t)src1 + (uint32_t)src2; \
164 if (tmp != (type)tmp) { \
170 #define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint8_t)
171 NEON_VOP_ENV(qadd_u8
, neon_u8
, 4)
173 #define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint16_t)
174 NEON_VOP_ENV(qadd_u16
, neon_u16
, 2)
178 uint32_t HELPER(neon_qadd_u32
)(CPUState
*env
, uint32_t a
, uint32_t b
)
180 uint32_t res
= a
+ b
;
188 uint64_t HELPER(neon_qadd_u64
)(CPUState
*env
, uint64_t src1
, uint64_t src2
)
200 #define NEON_SSAT(dest, src1, src2, type) do { \
201 int32_t tmp = (uint32_t)src1 + (uint32_t)src2; \
202 if (tmp != (type)tmp) { \
205 tmp = (1 << (sizeof(type) * 8 - 1)) - 1; \
207 tmp = 1 << (sizeof(type) * 8 - 1); \
212 #define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int8_t)
213 NEON_VOP_ENV(qadd_s8
, neon_s8
, 4)
215 #define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int16_t)
216 NEON_VOP_ENV(qadd_s16
, neon_s16
, 2)
220 uint32_t HELPER(neon_qadd_s32
)(CPUState
*env
, uint32_t a
, uint32_t b
)
222 uint32_t res
= a
+ b
;
223 if (((res
^ a
) & SIGNBIT
) && !((a
^ b
) & SIGNBIT
)) {
225 res
= ~(((int32_t)a
>> 31) ^ SIGNBIT
);
230 uint64_t HELPER(neon_qadd_s64
)(CPUState
*env
, uint64_t src1
, uint64_t src2
)
235 if (((res
^ src1
) & SIGNBIT64
) && !((src1
^ src2
) & SIGNBIT64
)) {
237 res
= ((int64_t)src1
>> 63) ^ ~SIGNBIT64
;
242 #define NEON_USAT(dest, src1, src2, type) do { \
243 uint32_t tmp = (uint32_t)src1 - (uint32_t)src2; \
244 if (tmp != (type)tmp) { \
250 #define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint8_t)
251 NEON_VOP_ENV(qsub_u8
, neon_u8
, 4)
253 #define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint16_t)
254 NEON_VOP_ENV(qsub_u16
, neon_u16
, 2)
258 uint32_t HELPER(neon_qsub_u32
)(CPUState
*env
, uint32_t a
, uint32_t b
)
260 uint32_t res
= a
- b
;
268 uint64_t HELPER(neon_qsub_u64
)(CPUState
*env
, uint64_t src1
, uint64_t src2
)
281 #define NEON_SSAT(dest, src1, src2, type) do { \
282 int32_t tmp = (uint32_t)src1 - (uint32_t)src2; \
283 if (tmp != (type)tmp) { \
286 tmp = (1 << (sizeof(type) * 8 - 1)) - 1; \
288 tmp = 1 << (sizeof(type) * 8 - 1); \
293 #define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int8_t)
294 NEON_VOP_ENV(qsub_s8
, neon_s8
, 4)
296 #define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int16_t)
297 NEON_VOP_ENV(qsub_s16
, neon_s16
, 2)
301 uint32_t HELPER(neon_qsub_s32
)(CPUState
*env
, uint32_t a
, uint32_t b
)
303 uint32_t res
= a
- b
;
304 if (((res
^ a
) & SIGNBIT
) && ((a
^ b
) & SIGNBIT
)) {
306 res
= ~(((int32_t)a
>> 31) ^ SIGNBIT
);
311 uint64_t HELPER(neon_qsub_s64
)(CPUState
*env
, uint64_t src1
, uint64_t src2
)
316 if (((res
^ src1
) & SIGNBIT64
) && ((src1
^ src2
) & SIGNBIT64
)) {
318 res
= ((int64_t)src1
>> 63) ^ ~SIGNBIT64
;
323 #define NEON_FN(dest, src1, src2) dest = (src1 + src2) >> 1
324 NEON_VOP(hadd_s8
, neon_s8
, 4)
325 NEON_VOP(hadd_u8
, neon_u8
, 4)
326 NEON_VOP(hadd_s16
, neon_s16
, 2)
327 NEON_VOP(hadd_u16
, neon_u16
, 2)
330 int32_t HELPER(neon_hadd_s32
)(int32_t src1
, int32_t src2
)
334 dest
= (src1
>> 1) + (src2
>> 1);
340 uint32_t HELPER(neon_hadd_u32
)(uint32_t src1
, uint32_t src2
)
344 dest
= (src1
>> 1) + (src2
>> 1);
350 #define NEON_FN(dest, src1, src2) dest = (src1 + src2 + 1) >> 1
351 NEON_VOP(rhadd_s8
, neon_s8
, 4)
352 NEON_VOP(rhadd_u8
, neon_u8
, 4)
353 NEON_VOP(rhadd_s16
, neon_s16
, 2)
354 NEON_VOP(rhadd_u16
, neon_u16
, 2)
357 int32_t HELPER(neon_rhadd_s32
)(int32_t src1
, int32_t src2
)
361 dest
= (src1
>> 1) + (src2
>> 1);
362 if ((src1
| src2
) & 1)
367 uint32_t HELPER(neon_rhadd_u32
)(uint32_t src1
, uint32_t src2
)
371 dest
= (src1
>> 1) + (src2
>> 1);
372 if ((src1
| src2
) & 1)
377 #define NEON_FN(dest, src1, src2) dest = (src1 - src2) >> 1
378 NEON_VOP(hsub_s8
, neon_s8
, 4)
379 NEON_VOP(hsub_u8
, neon_u8
, 4)
380 NEON_VOP(hsub_s16
, neon_s16
, 2)
381 NEON_VOP(hsub_u16
, neon_u16
, 2)
384 int32_t HELPER(neon_hsub_s32
)(int32_t src1
, int32_t src2
)
388 dest
= (src1
>> 1) - (src2
>> 1);
389 if ((~src1
) & src2
& 1)
394 uint32_t HELPER(neon_hsub_u32
)(uint32_t src1
, uint32_t src2
)
398 dest
= (src1
>> 1) - (src2
>> 1);
399 if ((~src1
) & src2
& 1)
404 #define NEON_FN(dest, src1, src2) dest = (src1 > src2) ? ~0 : 0
405 NEON_VOP(cgt_s8
, neon_s8
, 4)
406 NEON_VOP(cgt_u8
, neon_u8
, 4)
407 NEON_VOP(cgt_s16
, neon_s16
, 2)
408 NEON_VOP(cgt_u16
, neon_u16
, 2)
409 NEON_VOP(cgt_s32
, neon_s32
, 1)
410 NEON_VOP(cgt_u32
, neon_u32
, 1)
413 #define NEON_FN(dest, src1, src2) dest = (src1 >= src2) ? ~0 : 0
414 NEON_VOP(cge_s8
, neon_s8
, 4)
415 NEON_VOP(cge_u8
, neon_u8
, 4)
416 NEON_VOP(cge_s16
, neon_s16
, 2)
417 NEON_VOP(cge_u16
, neon_u16
, 2)
418 NEON_VOP(cge_s32
, neon_s32
, 1)
419 NEON_VOP(cge_u32
, neon_u32
, 1)
422 #define NEON_FN(dest, src1, src2) dest = (src1 < src2) ? src1 : src2
423 NEON_VOP(min_s8
, neon_s8
, 4)
424 NEON_VOP(min_u8
, neon_u8
, 4)
425 NEON_VOP(min_s16
, neon_s16
, 2)
426 NEON_VOP(min_u16
, neon_u16
, 2)
427 NEON_VOP(min_s32
, neon_s32
, 1)
428 NEON_VOP(min_u32
, neon_u32
, 1)
429 NEON_POP(pmin_s8
, neon_s8
, 4)
430 NEON_POP(pmin_u8
, neon_u8
, 4)
431 NEON_POP(pmin_s16
, neon_s16
, 2)
432 NEON_POP(pmin_u16
, neon_u16
, 2)
435 #define NEON_FN(dest, src1, src2) dest = (src1 > src2) ? src1 : src2
436 NEON_VOP(max_s8
, neon_s8
, 4)
437 NEON_VOP(max_u8
, neon_u8
, 4)
438 NEON_VOP(max_s16
, neon_s16
, 2)
439 NEON_VOP(max_u16
, neon_u16
, 2)
440 NEON_VOP(max_s32
, neon_s32
, 1)
441 NEON_VOP(max_u32
, neon_u32
, 1)
442 NEON_POP(pmax_s8
, neon_s8
, 4)
443 NEON_POP(pmax_u8
, neon_u8
, 4)
444 NEON_POP(pmax_s16
, neon_s16
, 2)
445 NEON_POP(pmax_u16
, neon_u16
, 2)
448 #define NEON_FN(dest, src1, src2) \
449 dest = (src1 > src2) ? (src1 - src2) : (src2 - src1)
450 NEON_VOP(abd_s8
, neon_s8
, 4)
451 NEON_VOP(abd_u8
, neon_u8
, 4)
452 NEON_VOP(abd_s16
, neon_s16
, 2)
453 NEON_VOP(abd_u16
, neon_u16
, 2)
454 NEON_VOP(abd_s32
, neon_s32
, 1)
455 NEON_VOP(abd_u32
, neon_u32
, 1)
458 #define NEON_FN(dest, src1, src2) do { \
460 tmp = (int8_t)src2; \
461 if (tmp >= (ssize_t)sizeof(src1) * 8 || \
462 tmp <= -(ssize_t)sizeof(src1) * 8) { \
464 } else if (tmp < 0) { \
465 dest = src1 >> -tmp; \
467 dest = src1 << tmp; \
469 NEON_VOP(shl_u8
, neon_u8
, 4)
470 NEON_VOP(shl_u16
, neon_u16
, 2)
471 NEON_VOP(shl_u32
, neon_u32
, 1)
474 uint64_t HELPER(neon_shl_u64
)(uint64_t val
, uint64_t shiftop
)
476 int8_t shift
= (int8_t)shiftop
;
477 if (shift
>= 64 || shift
<= -64) {
479 } else if (shift
< 0) {
487 #define NEON_FN(dest, src1, src2) do { \
489 tmp = (int8_t)src2; \
490 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
492 } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
493 dest = src1 >> (sizeof(src1) * 8 - 1); \
494 } else if (tmp < 0) { \
495 dest = src1 >> -tmp; \
497 dest = src1 << tmp; \
499 NEON_VOP(shl_s8
, neon_s8
, 4)
500 NEON_VOP(shl_s16
, neon_s16
, 2)
501 NEON_VOP(shl_s32
, neon_s32
, 1)
504 uint64_t HELPER(neon_shl_s64
)(uint64_t valop
, uint64_t shiftop
)
506 int8_t shift
= (int8_t)shiftop
;
510 } else if (shift
<= -64) {
512 } else if (shift
< 0) {
520 #define NEON_FN(dest, src1, src2) do { \
522 tmp = (int8_t)src2; \
523 if ((tmp >= (ssize_t)sizeof(src1) * 8) \
524 || (tmp <= -(ssize_t)sizeof(src1) * 8)) { \
526 } else if (tmp < 0) { \
527 dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
529 dest = src1 << tmp; \
531 NEON_VOP(rshl_s8
, neon_s8
, 4)
532 NEON_VOP(rshl_s16
, neon_s16
, 2)
535 /* The addition of the rounding constant may overflow, so we use an
536 * intermediate 64 bits accumulator. */
537 uint32_t HELPER(neon_rshl_s32
)(uint32_t valop
, uint32_t shiftop
)
540 int32_t val
= (int32_t)valop
;
541 int8_t shift
= (int8_t)shiftop
;
542 if ((shift
>= 32) || (shift
<= -32)) {
544 } else if (shift
< 0) {
545 int64_t big_dest
= ((int64_t)val
+ (1 << (-1 - shift
)));
546 dest
= big_dest
>> -shift
;
553 /* Handling addition overflow with 64 bits inputs values is more
554 * tricky than with 32 bits values. */
555 uint64_t HELPER(neon_rshl_s64
)(uint64_t valop
, uint64_t shiftop
)
557 int8_t shift
= (int8_t)shiftop
;
559 if ((shift
>= 64) || (shift
<= -64)) {
561 } else if (shift
< 0) {
562 val
>>= (-shift
- 1);
563 if (val
== INT64_MAX
) {
564 /* In this case, it means that the rounding constant is 1,
565 * and the addition would overflow. Return the actual
566 * result directly. */
567 val
= 0x4000000000000000LL
;
578 #define NEON_FN(dest, src1, src2) do { \
580 tmp = (int8_t)src2; \
581 if (tmp >= (ssize_t)sizeof(src1) * 8 || \
582 tmp < -(ssize_t)sizeof(src1) * 8) { \
584 } else if (tmp == -(ssize_t)sizeof(src1) * 8) { \
585 dest = src1 >> (-tmp - 1); \
586 } else if (tmp < 0) { \
587 dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
589 dest = src1 << tmp; \
591 NEON_VOP(rshl_u8
, neon_u8
, 4)
592 NEON_VOP(rshl_u16
, neon_u16
, 2)
595 /* The addition of the rounding constant may overflow, so we use an
596 * intermediate 64 bits accumulator. */
597 uint32_t HELPER(neon_rshl_u32
)(uint32_t val
, uint32_t shiftop
)
600 int8_t shift
= (int8_t)shiftop
;
601 if (shift
>= 32 || shift
< -32) {
603 } else if (shift
== -32) {
605 } else if (shift
< 0) {
606 uint64_t big_dest
= ((uint64_t)val
+ (1 << (-1 - shift
)));
607 dest
= big_dest
>> -shift
;
614 /* Handling addition overflow with 64 bits inputs values is more
615 * tricky than with 32 bits values. */
616 uint64_t HELPER(neon_rshl_u64
)(uint64_t val
, uint64_t shiftop
)
618 int8_t shift
= (uint8_t)shiftop
;
619 if (shift
>= 64 || shift
< -64) {
621 } else if (shift
== -64) {
622 /* Rounding a 1-bit result just preserves that bit. */
624 } else if (shift
< 0) {
625 val
>>= (-shift
- 1);
626 if (val
== UINT64_MAX
) {
627 /* In this case, it means that the rounding constant is 1,
628 * and the addition would overflow. Return the actual
629 * result directly. */
630 val
= 0x8000000000000000ULL
;
641 #define NEON_FN(dest, src1, src2) do { \
643 tmp = (int8_t)src2; \
644 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
651 } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
653 } else if (tmp < 0) { \
654 dest = src1 >> -tmp; \
656 dest = src1 << tmp; \
657 if ((dest >> tmp) != src1) { \
662 NEON_VOP_ENV(qshl_u8
, neon_u8
, 4)
663 NEON_VOP_ENV(qshl_u16
, neon_u16
, 2)
664 NEON_VOP_ENV(qshl_u32
, neon_u32
, 1)
667 uint64_t HELPER(neon_qshl_u64
)(CPUState
*env
, uint64_t val
, uint64_t shiftop
)
669 int8_t shift
= (int8_t)shiftop
;
675 } else if (shift
<= -64) {
677 } else if (shift
< 0) {
682 if ((val
>> shift
) != tmp
) {
690 #define NEON_FN(dest, src1, src2) do { \
692 tmp = (int8_t)src2; \
693 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
696 dest = (uint32_t)(1 << (sizeof(src1) * 8 - 1)); \
703 } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
705 } else if (tmp < 0) { \
706 dest = src1 >> -tmp; \
708 dest = src1 << tmp; \
709 if ((dest >> tmp) != src1) { \
711 dest = (uint32_t)(1 << (sizeof(src1) * 8 - 1)); \
717 NEON_VOP_ENV(qshl_s8
, neon_s8
, 4)
718 NEON_VOP_ENV(qshl_s16
, neon_s16
, 2)
719 NEON_VOP_ENV(qshl_s32
, neon_s32
, 1)
722 uint64_t HELPER(neon_qshl_s64
)(CPUState
*env
, uint64_t valop
, uint64_t shiftop
)
724 int8_t shift
= (uint8_t)shiftop
;
729 val
= (val
>> 63) ^ ~SIGNBIT64
;
731 } else if (shift
<= -64) {
733 } else if (shift
< 0) {
738 if ((val
>> shift
) != tmp
) {
740 val
= (tmp
>> 63) ^ ~SIGNBIT64
;
746 #define NEON_FN(dest, src1, src2) do { \
747 if (src1 & (1 << (sizeof(src1) * 8 - 1))) { \
752 tmp = (int8_t)src2; \
753 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
760 } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
762 } else if (tmp < 0) { \
763 dest = src1 >> -tmp; \
765 dest = src1 << tmp; \
766 if ((dest >> tmp) != src1) { \
772 NEON_VOP_ENV(qshlu_s8
, neon_u8
, 4)
773 NEON_VOP_ENV(qshlu_s16
, neon_u16
, 2)
776 uint32_t HELPER(neon_qshlu_s32
)(CPUState
*env
, uint32_t valop
, uint32_t shiftop
)
778 if ((int32_t)valop
< 0) {
782 return helper_neon_qshl_u32(env
, valop
, shiftop
);
785 uint64_t HELPER(neon_qshlu_s64
)(CPUState
*env
, uint64_t valop
, uint64_t shiftop
)
787 if ((int64_t)valop
< 0) {
791 return helper_neon_qshl_u64(env
, valop
, shiftop
);
794 /* FIXME: This is wrong. */
795 #define NEON_FN(dest, src1, src2) do { \
797 tmp = (int8_t)src2; \
798 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
805 } else if (tmp < -(ssize_t)sizeof(src1) * 8) { \
807 } else if (tmp == -(ssize_t)sizeof(src1) * 8) { \
808 dest = src1 >> (sizeof(src1) * 8 - 1); \
809 } else if (tmp < 0) { \
810 dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
812 dest = src1 << tmp; \
813 if ((dest >> tmp) != src1) { \
818 NEON_VOP_ENV(qrshl_u8
, neon_u8
, 4)
819 NEON_VOP_ENV(qrshl_u16
, neon_u16
, 2)
822 /* The addition of the rounding constant may overflow, so we use an
823 * intermediate 64 bits accumulator. */
824 uint32_t HELPER(neon_qrshl_u32
)(CPUState
*env
, uint32_t val
, uint32_t shiftop
)
827 int8_t shift
= (int8_t)shiftop
;
835 } else if (shift
< -32) {
837 } else if (shift
== -32) {
839 } else if (shift
< 0) {
840 uint64_t big_dest
= ((uint64_t)val
+ (1 << (-1 - shift
)));
841 dest
= big_dest
>> -shift
;
844 if ((dest
>> shift
) != val
) {
852 /* Handling addition overflow with 64 bits inputs values is more
853 * tricky than with 32 bits values. */
854 uint64_t HELPER(neon_qrshl_u64
)(CPUState
*env
, uint64_t val
, uint64_t shiftop
)
856 int8_t shift
= (int8_t)shiftop
;
862 } else if (shift
< -64) {
864 } else if (shift
== -64) {
866 } else if (shift
< 0) {
867 val
>>= (-shift
- 1);
868 if (val
== UINT64_MAX
) {
869 /* In this case, it means that the rounding constant is 1,
870 * and the addition would overflow. Return the actual
871 * result directly. */
872 val
= 0x8000000000000000ULL
;
880 if ((val
>> shift
) != tmp
) {
888 #define NEON_FN(dest, src1, src2) do { \
890 tmp = (int8_t)src2; \
891 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
894 dest = (1 << (sizeof(src1) * 8 - 1)); \
901 } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
903 } else if (tmp < 0) { \
904 dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
906 dest = src1 << tmp; \
907 if ((dest >> tmp) != src1) { \
909 dest = (uint32_t)(1 << (sizeof(src1) * 8 - 1)); \
915 NEON_VOP_ENV(qrshl_s8
, neon_s8
, 4)
916 NEON_VOP_ENV(qrshl_s16
, neon_s16
, 2)
919 /* The addition of the rounding constant may overflow, so we use an
920 * intermediate 64 bits accumulator. */
921 uint32_t HELPER(neon_qrshl_s32
)(CPUState
*env
, uint32_t valop
, uint32_t shiftop
)
924 int32_t val
= (int32_t)valop
;
925 int8_t shift
= (int8_t)shiftop
;
929 dest
= (val
>> 31) ^ ~SIGNBIT
;
933 } else if (shift
<= -32) {
935 } else if (shift
< 0) {
936 int64_t big_dest
= ((int64_t)val
+ (1 << (-1 - shift
)));
937 dest
= big_dest
>> -shift
;
940 if ((dest
>> shift
) != val
) {
942 dest
= (val
>> 31) ^ ~SIGNBIT
;
948 /* Handling addition overflow with 64 bits inputs values is more
949 * tricky than with 32 bits values. */
950 uint64_t HELPER(neon_qrshl_s64
)(CPUState
*env
, uint64_t valop
, uint64_t shiftop
)
952 int8_t shift
= (uint8_t)shiftop
;
958 val
= (val
>> 63) ^ ~SIGNBIT64
;
960 } else if (shift
<= -64) {
962 } else if (shift
< 0) {
963 val
>>= (-shift
- 1);
964 if (val
== INT64_MAX
) {
965 /* In this case, it means that the rounding constant is 1,
966 * and the addition would overflow. Return the actual
967 * result directly. */
968 val
= 0x4000000000000000ULL
;
976 if ((val
>> shift
) != tmp
) {
978 val
= (tmp
>> 63) ^ ~SIGNBIT64
;
984 uint32_t HELPER(neon_add_u8
)(uint32_t a
, uint32_t b
)
987 mask
= (a
^ b
) & 0x80808080u
;
990 return (a
+ b
) ^ mask
;
993 uint32_t HELPER(neon_add_u16
)(uint32_t a
, uint32_t b
)
996 mask
= (a
^ b
) & 0x80008000u
;
999 return (a
+ b
) ^ mask
;
1002 #define NEON_FN(dest, src1, src2) dest = src1 + src2
1003 NEON_POP(padd_u8
, neon_u8
, 4)
1004 NEON_POP(padd_u16
, neon_u16
, 2)
1007 #define NEON_FN(dest, src1, src2) dest = src1 - src2
1008 NEON_VOP(sub_u8
, neon_u8
, 4)
1009 NEON_VOP(sub_u16
, neon_u16
, 2)
1012 #define NEON_FN(dest, src1, src2) dest = src1 * src2
1013 NEON_VOP(mul_u8
, neon_u8
, 4)
1014 NEON_VOP(mul_u16
, neon_u16
, 2)
1017 /* Polynomial multiplication is like integer multiplication except the
1018 partial products are XORed, not added. */
1019 uint32_t HELPER(neon_mul_p8
)(uint32_t op1
, uint32_t op2
)
1029 mask
|= (0xff << 8);
1030 if (op1
& (1 << 16))
1031 mask
|= (0xff << 16);
1032 if (op1
& (1 << 24))
1033 mask
|= (0xff << 24);
1034 result
^= op2
& mask
;
1035 op1
= (op1
>> 1) & 0x7f7f7f7f;
1036 op2
= (op2
<< 1) & 0xfefefefe;
1041 uint64_t HELPER(neon_mull_p8
)(uint32_t op1
, uint32_t op2
)
1043 uint64_t result
= 0;
1045 uint64_t op2ex
= op2
;
1046 op2ex
= (op2ex
& 0xff) |
1047 ((op2ex
& 0xff00) << 8) |
1048 ((op2ex
& 0xff0000) << 16) |
1049 ((op2ex
& 0xff000000) << 24);
1055 if (op1
& (1 << 8)) {
1056 mask
|= (0xffffU
<< 16);
1058 if (op1
& (1 << 16)) {
1059 mask
|= (0xffffULL
<< 32);
1061 if (op1
& (1 << 24)) {
1062 mask
|= (0xffffULL
<< 48);
1064 result
^= op2ex
& mask
;
1065 op1
= (op1
>> 1) & 0x7f7f7f7f;
1071 #define NEON_FN(dest, src1, src2) dest = (src1 & src2) ? -1 : 0
1072 NEON_VOP(tst_u8
, neon_u8
, 4)
1073 NEON_VOP(tst_u16
, neon_u16
, 2)
1074 NEON_VOP(tst_u32
, neon_u32
, 1)
1077 #define NEON_FN(dest, src1, src2) dest = (src1 == src2) ? -1 : 0
1078 NEON_VOP(ceq_u8
, neon_u8
, 4)
1079 NEON_VOP(ceq_u16
, neon_u16
, 2)
1080 NEON_VOP(ceq_u32
, neon_u32
, 1)
1083 #define NEON_FN(dest, src, dummy) dest = (src < 0) ? -src : src
1084 NEON_VOP1(abs_s8
, neon_s8
, 4)
1085 NEON_VOP1(abs_s16
, neon_s16
, 2)
1088 /* Count Leading Sign/Zero Bits. */
1089 static inline int do_clz8(uint8_t x
)
1097 static inline int do_clz16(uint16_t x
)
1100 for (n
= 16; x
; n
--)
1105 #define NEON_FN(dest, src, dummy) dest = do_clz8(src)
1106 NEON_VOP1(clz_u8
, neon_u8
, 4)
1109 #define NEON_FN(dest, src, dummy) dest = do_clz16(src)
1110 NEON_VOP1(clz_u16
, neon_u16
, 2)
1113 #define NEON_FN(dest, src, dummy) dest = do_clz8((src < 0) ? ~src : src) - 1
1114 NEON_VOP1(cls_s8
, neon_s8
, 4)
1117 #define NEON_FN(dest, src, dummy) dest = do_clz16((src < 0) ? ~src : src) - 1
1118 NEON_VOP1(cls_s16
, neon_s16
, 2)
1121 uint32_t HELPER(neon_cls_s32
)(uint32_t x
)
1126 for (count
= 32; x
; count
--)
1132 uint32_t HELPER(neon_cnt_u8
)(uint32_t x
)
1134 x
= (x
& 0x55555555) + ((x
>> 1) & 0x55555555);
1135 x
= (x
& 0x33333333) + ((x
>> 2) & 0x33333333);
1136 x
= (x
& 0x0f0f0f0f) + ((x
>> 4) & 0x0f0f0f0f);
1140 #define NEON_QDMULH16(dest, src1, src2, round) do { \
1141 uint32_t tmp = (int32_t)(int16_t) src1 * (int16_t) src2; \
1142 if ((tmp ^ (tmp << 1)) & SIGNBIT) { \
1144 tmp = (tmp >> 31) ^ ~SIGNBIT; \
1149 int32_t old = tmp; \
1151 if ((int32_t)tmp < old) { \
1153 tmp = SIGNBIT - 1; \
1158 #define NEON_FN(dest, src1, src2) NEON_QDMULH16(dest, src1, src2, 0)
1159 NEON_VOP_ENV(qdmulh_s16
, neon_s16
, 2)
1161 #define NEON_FN(dest, src1, src2) NEON_QDMULH16(dest, src1, src2, 1)
1162 NEON_VOP_ENV(qrdmulh_s16
, neon_s16
, 2)
1164 #undef NEON_QDMULH16
1166 #define NEON_QDMULH32(dest, src1, src2, round) do { \
1167 uint64_t tmp = (int64_t)(int32_t) src1 * (int32_t) src2; \
1168 if ((tmp ^ (tmp << 1)) & SIGNBIT64) { \
1170 tmp = (tmp >> 63) ^ ~SIGNBIT64; \
1175 int64_t old = tmp; \
1176 tmp += (int64_t)1 << 31; \
1177 if ((int64_t)tmp < old) { \
1179 tmp = SIGNBIT64 - 1; \
1184 #define NEON_FN(dest, src1, src2) NEON_QDMULH32(dest, src1, src2, 0)
1185 NEON_VOP_ENV(qdmulh_s32
, neon_s32
, 1)
1187 #define NEON_FN(dest, src1, src2) NEON_QDMULH32(dest, src1, src2, 1)
1188 NEON_VOP_ENV(qrdmulh_s32
, neon_s32
, 1)
1190 #undef NEON_QDMULH32
1192 uint32_t HELPER(neon_narrow_u8
)(uint64_t x
)
1194 return (x
& 0xffu
) | ((x
>> 8) & 0xff00u
) | ((x
>> 16) & 0xff0000u
)
1195 | ((x
>> 24) & 0xff000000u
);
1198 uint32_t HELPER(neon_narrow_u16
)(uint64_t x
)
1200 return (x
& 0xffffu
) | ((x
>> 16) & 0xffff0000u
);
1203 uint32_t HELPER(neon_narrow_high_u8
)(uint64_t x
)
1205 return ((x
>> 8) & 0xff) | ((x
>> 16) & 0xff00)
1206 | ((x
>> 24) & 0xff0000) | ((x
>> 32) & 0xff000000);
1209 uint32_t HELPER(neon_narrow_high_u16
)(uint64_t x
)
1211 return ((x
>> 16) & 0xffff) | ((x
>> 32) & 0xffff0000);
1214 uint32_t HELPER(neon_narrow_round_high_u8
)(uint64_t x
)
1216 x
&= 0xff80ff80ff80ff80ull
;
1217 x
+= 0x0080008000800080ull
;
1218 return ((x
>> 8) & 0xff) | ((x
>> 16) & 0xff00)
1219 | ((x
>> 24) & 0xff0000) | ((x
>> 32) & 0xff000000);
1222 uint32_t HELPER(neon_narrow_round_high_u16
)(uint64_t x
)
1224 x
&= 0xffff8000ffff8000ull
;
1225 x
+= 0x0000800000008000ull
;
1226 return ((x
>> 16) & 0xffff) | ((x
>> 32) & 0xffff0000);
1229 uint32_t HELPER(neon_unarrow_sat8
)(CPUState
*env
, uint64_t x
)
1245 res |= (uint32_t)d << (n / 2); \
1256 uint32_t HELPER(neon_narrow_sat_u8
)(CPUState
*env
, uint64_t x
)
1269 res |= (uint32_t)d << (n / 2);
1279 uint32_t HELPER(neon_narrow_sat_s8
)(CPUState
*env
, uint64_t x
)
1286 if (s != (int8_t)s) { \
1287 d = (s >> 15) ^ 0x7f; \
1292 res |= (uint32_t)d << (n / 2);
1302 uint32_t HELPER(neon_unarrow_sat16
)(CPUState
*env
, uint64_t x
)
1307 if (low
& 0x80000000) {
1310 } else if (low
> 0xffff) {
1315 if (high
& 0x80000000) {
1318 } else if (high
> 0xffff) {
1322 return low
| (high
<< 16);
1325 uint32_t HELPER(neon_narrow_sat_u16
)(CPUState
*env
, uint64_t x
)
1335 if (high
> 0xffff) {
1339 return low
| (high
<< 16);
1342 uint32_t HELPER(neon_narrow_sat_s16
)(CPUState
*env
, uint64_t x
)
1347 if (low
!= (int16_t)low
) {
1348 low
= (low
>> 31) ^ 0x7fff;
1352 if (high
!= (int16_t)high
) {
1353 high
= (high
>> 31) ^ 0x7fff;
1356 return (uint16_t)low
| (high
<< 16);
1359 uint32_t HELPER(neon_unarrow_sat32
)(CPUState
*env
, uint64_t x
)
1361 if (x
& 0x8000000000000000ull
) {
1365 if (x
> 0xffffffffu
) {
1372 uint32_t HELPER(neon_narrow_sat_u32
)(CPUState
*env
, uint64_t x
)
1374 if (x
> 0xffffffffu
) {
1381 uint32_t HELPER(neon_narrow_sat_s32
)(CPUState
*env
, uint64_t x
)
1383 if ((int64_t)x
!= (int32_t)x
) {
1385 return ((int64_t)x
>> 63) ^ 0x7fffffff;
1390 uint64_t HELPER(neon_widen_u8
)(uint32_t x
)
1395 tmp
= (uint8_t)(x
>> 8);
1397 tmp
= (uint8_t)(x
>> 16);
1399 tmp
= (uint8_t)(x
>> 24);
1404 uint64_t HELPER(neon_widen_s8
)(uint32_t x
)
1408 ret
= (uint16_t)(int8_t)x
;
1409 tmp
= (uint16_t)(int8_t)(x
>> 8);
1411 tmp
= (uint16_t)(int8_t)(x
>> 16);
1413 tmp
= (uint16_t)(int8_t)(x
>> 24);
1418 uint64_t HELPER(neon_widen_u16
)(uint32_t x
)
1420 uint64_t high
= (uint16_t)(x
>> 16);
1421 return ((uint16_t)x
) | (high
<< 32);
1424 uint64_t HELPER(neon_widen_s16
)(uint32_t x
)
1426 uint64_t high
= (int16_t)(x
>> 16);
1427 return ((uint32_t)(int16_t)x
) | (high
<< 32);
1430 uint64_t HELPER(neon_addl_u16
)(uint64_t a
, uint64_t b
)
1433 mask
= (a
^ b
) & 0x8000800080008000ull
;
1434 a
&= ~0x8000800080008000ull
;
1435 b
&= ~0x8000800080008000ull
;
1436 return (a
+ b
) ^ mask
;
1439 uint64_t HELPER(neon_addl_u32
)(uint64_t a
, uint64_t b
)
1442 mask
= (a
^ b
) & 0x8000000080000000ull
;
1443 a
&= ~0x8000000080000000ull
;
1444 b
&= ~0x8000000080000000ull
;
1445 return (a
+ b
) ^ mask
;
1448 uint64_t HELPER(neon_paddl_u16
)(uint64_t a
, uint64_t b
)
1453 tmp
= a
& 0x0000ffff0000ffffull
;
1454 tmp
+= (a
>> 16) & 0x0000ffff0000ffffull
;
1455 tmp2
= b
& 0xffff0000ffff0000ull
;
1456 tmp2
+= (b
<< 16) & 0xffff0000ffff0000ull
;
1457 return ( tmp
& 0xffff)
1458 | ((tmp
>> 16) & 0xffff0000ull
)
1459 | ((tmp2
<< 16) & 0xffff00000000ull
)
1460 | ( tmp2
& 0xffff000000000000ull
);
1463 uint64_t HELPER(neon_paddl_u32
)(uint64_t a
, uint64_t b
)
1465 uint32_t low
= a
+ (a
>> 32);
1466 uint32_t high
= b
+ (b
>> 32);
1467 return low
+ ((uint64_t)high
<< 32);
1470 uint64_t HELPER(neon_subl_u16
)(uint64_t a
, uint64_t b
)
1473 mask
= (a
^ ~b
) & 0x8000800080008000ull
;
1474 a
|= 0x8000800080008000ull
;
1475 b
&= ~0x8000800080008000ull
;
1476 return (a
- b
) ^ mask
;
1479 uint64_t HELPER(neon_subl_u32
)(uint64_t a
, uint64_t b
)
1482 mask
= (a
^ ~b
) & 0x8000000080000000ull
;
1483 a
|= 0x8000000080000000ull
;
1484 b
&= ~0x8000000080000000ull
;
1485 return (a
- b
) ^ mask
;
1488 uint64_t HELPER(neon_addl_saturate_s32
)(CPUState
*env
, uint64_t a
, uint64_t b
)
1496 if (((low
^ x
) & SIGNBIT
) && !((x
^ y
) & SIGNBIT
)) {
1498 low
= ((int32_t)x
>> 31) ^ ~SIGNBIT
;
1503 if (((high
^ x
) & SIGNBIT
) && !((x
^ y
) & SIGNBIT
)) {
1505 high
= ((int32_t)x
>> 31) ^ ~SIGNBIT
;
1507 return low
| ((uint64_t)high
<< 32);
1510 uint64_t HELPER(neon_addl_saturate_s64
)(CPUState
*env
, uint64_t a
, uint64_t b
)
1515 if (((result
^ a
) & SIGNBIT64
) && !((a
^ b
) & SIGNBIT64
)) {
1517 result
= ((int64_t)a
>> 63) ^ ~SIGNBIT64
;
1522 #define DO_ABD(dest, x, y, type) do { \
1525 dest = ((tmp_x > tmp_y) ? tmp_x - tmp_y : tmp_y - tmp_x); \
1528 uint64_t HELPER(neon_abdl_u16
)(uint32_t a
, uint32_t b
)
1532 DO_ABD(result
, a
, b
, uint8_t);
1533 DO_ABD(tmp
, a
>> 8, b
>> 8, uint8_t);
1534 result
|= tmp
<< 16;
1535 DO_ABD(tmp
, a
>> 16, b
>> 16, uint8_t);
1536 result
|= tmp
<< 32;
1537 DO_ABD(tmp
, a
>> 24, b
>> 24, uint8_t);
1538 result
|= tmp
<< 48;
1542 uint64_t HELPER(neon_abdl_s16
)(uint32_t a
, uint32_t b
)
1546 DO_ABD(result
, a
, b
, int8_t);
1547 DO_ABD(tmp
, a
>> 8, b
>> 8, int8_t);
1548 result
|= tmp
<< 16;
1549 DO_ABD(tmp
, a
>> 16, b
>> 16, int8_t);
1550 result
|= tmp
<< 32;
1551 DO_ABD(tmp
, a
>> 24, b
>> 24, int8_t);
1552 result
|= tmp
<< 48;
1556 uint64_t HELPER(neon_abdl_u32
)(uint32_t a
, uint32_t b
)
1560 DO_ABD(result
, a
, b
, uint16_t);
1561 DO_ABD(tmp
, a
>> 16, b
>> 16, uint16_t);
1562 return result
| (tmp
<< 32);
1565 uint64_t HELPER(neon_abdl_s32
)(uint32_t a
, uint32_t b
)
1569 DO_ABD(result
, a
, b
, int16_t);
1570 DO_ABD(tmp
, a
>> 16, b
>> 16, int16_t);
1571 return result
| (tmp
<< 32);
1574 uint64_t HELPER(neon_abdl_u64
)(uint32_t a
, uint32_t b
)
1577 DO_ABD(result
, a
, b
, uint32_t);
1581 uint64_t HELPER(neon_abdl_s64
)(uint32_t a
, uint32_t b
)
1584 DO_ABD(result
, a
, b
, int32_t);
1589 /* Widening multiply. Named type is the source type. */
1590 #define DO_MULL(dest, x, y, type1, type2) do { \
1593 dest = (type2)((type2)tmp_x * (type2)tmp_y); \
1596 uint64_t HELPER(neon_mull_u8
)(uint32_t a
, uint32_t b
)
1601 DO_MULL(result
, a
, b
, uint8_t, uint16_t);
1602 DO_MULL(tmp
, a
>> 8, b
>> 8, uint8_t, uint16_t);
1603 result
|= tmp
<< 16;
1604 DO_MULL(tmp
, a
>> 16, b
>> 16, uint8_t, uint16_t);
1605 result
|= tmp
<< 32;
1606 DO_MULL(tmp
, a
>> 24, b
>> 24, uint8_t, uint16_t);
1607 result
|= tmp
<< 48;
1611 uint64_t HELPER(neon_mull_s8
)(uint32_t a
, uint32_t b
)
1616 DO_MULL(result
, a
, b
, int8_t, uint16_t);
1617 DO_MULL(tmp
, a
>> 8, b
>> 8, int8_t, uint16_t);
1618 result
|= tmp
<< 16;
1619 DO_MULL(tmp
, a
>> 16, b
>> 16, int8_t, uint16_t);
1620 result
|= tmp
<< 32;
1621 DO_MULL(tmp
, a
>> 24, b
>> 24, int8_t, uint16_t);
1622 result
|= tmp
<< 48;
1626 uint64_t HELPER(neon_mull_u16
)(uint32_t a
, uint32_t b
)
1631 DO_MULL(result
, a
, b
, uint16_t, uint32_t);
1632 DO_MULL(tmp
, a
>> 16, b
>> 16, uint16_t, uint32_t);
1633 return result
| (tmp
<< 32);
1636 uint64_t HELPER(neon_mull_s16
)(uint32_t a
, uint32_t b
)
1641 DO_MULL(result
, a
, b
, int16_t, uint32_t);
1642 DO_MULL(tmp
, a
>> 16, b
>> 16, int16_t, uint32_t);
1643 return result
| (tmp
<< 32);
1646 uint64_t HELPER(neon_negl_u16
)(uint64_t x
)
1650 result
= (uint16_t)-x
;
1652 result
|= (uint64_t)tmp
<< 16;
1654 result
|= (uint64_t)tmp
<< 32;
1656 result
|= (uint64_t)tmp
<< 48;
1660 uint64_t HELPER(neon_negl_u32
)(uint64_t x
)
1663 uint32_t high
= -(x
>> 32);
1664 return low
| ((uint64_t)high
<< 32);
1667 /* FIXME: There should be a native op for this. */
1668 uint64_t HELPER(neon_negl_u64
)(uint64_t x
)
1673 /* Saturnating sign manuipulation. */
1674 /* ??? Make these use NEON_VOP1 */
1675 #define DO_QABS8(x) do { \
1676 if (x == (int8_t)0x80) { \
1679 } else if (x < 0) { \
1682 uint32_t HELPER(neon_qabs_s8
)(CPUState
*env
, uint32_t x
)
1685 NEON_UNPACK(neon_s8
, vec
, x
);
1690 NEON_PACK(neon_s8
, x
, vec
);
1695 #define DO_QNEG8(x) do { \
1696 if (x == (int8_t)0x80) { \
1702 uint32_t HELPER(neon_qneg_s8
)(CPUState
*env
, uint32_t x
)
1705 NEON_UNPACK(neon_s8
, vec
, x
);
1710 NEON_PACK(neon_s8
, x
, vec
);
1715 #define DO_QABS16(x) do { \
1716 if (x == (int16_t)0x8000) { \
1719 } else if (x < 0) { \
1722 uint32_t HELPER(neon_qabs_s16
)(CPUState
*env
, uint32_t x
)
1725 NEON_UNPACK(neon_s16
, vec
, x
);
1728 NEON_PACK(neon_s16
, x
, vec
);
1733 #define DO_QNEG16(x) do { \
1734 if (x == (int16_t)0x8000) { \
1740 uint32_t HELPER(neon_qneg_s16
)(CPUState
*env
, uint32_t x
)
1743 NEON_UNPACK(neon_s16
, vec
, x
);
1746 NEON_PACK(neon_s16
, x
, vec
);
1751 uint32_t HELPER(neon_qabs_s32
)(CPUState
*env
, uint32_t x
)
1756 } else if ((int32_t)x
< 0) {
1762 uint32_t HELPER(neon_qneg_s32
)(CPUState
*env
, uint32_t x
)
1773 /* NEON Float helpers. */
1774 uint32_t HELPER(neon_min_f32
)(uint32_t a
, uint32_t b
)
1776 return float32_val(float32_min(make_float32(a
), make_float32(b
), NFS
));
1779 uint32_t HELPER(neon_max_f32
)(uint32_t a
, uint32_t b
)
1781 return float32_val(float32_max(make_float32(a
), make_float32(b
), NFS
));
1784 uint32_t HELPER(neon_abd_f32
)(uint32_t a
, uint32_t b
)
1786 float32 f0
= make_float32(a
);
1787 float32 f1
= make_float32(b
);
1788 return float32_val(float32_abs(float32_sub(f0
, f1
, NFS
)));
1791 uint32_t HELPER(neon_add_f32
)(uint32_t a
, uint32_t b
)
1793 return float32_val(float32_add(make_float32(a
), make_float32(b
), NFS
));
1796 uint32_t HELPER(neon_sub_f32
)(uint32_t a
, uint32_t b
)
1798 return float32_val(float32_sub(make_float32(a
), make_float32(b
), NFS
));
1801 uint32_t HELPER(neon_mul_f32
)(uint32_t a
, uint32_t b
)
1803 return float32_val(float32_mul(make_float32(a
), make_float32(b
), NFS
));
1806 /* Floating point comparisons produce an integer result. */
1807 #define NEON_VOP_FCMP(name, ok) \
1808 uint32_t HELPER(neon_##name)(uint32_t a, uint32_t b) \
1810 switch (float32_compare_quiet(make_float32(a), make_float32(b), NFS)) { \
1812 default: return 0; \
1816 NEON_VOP_FCMP(ceq_f32
, case float_relation_equal
:)
1817 NEON_VOP_FCMP(cge_f32
, case float_relation_equal
: case float_relation_greater
:)
1818 NEON_VOP_FCMP(cgt_f32
, case float_relation_greater
:)
1820 uint32_t HELPER(neon_acge_f32
)(uint32_t a
, uint32_t b
)
1822 float32 f0
= float32_abs(make_float32(a
));
1823 float32 f1
= float32_abs(make_float32(b
));
1824 switch (float32_compare_quiet(f0
, f1
, NFS
)) {
1825 case float_relation_equal
:
1826 case float_relation_greater
:
1833 uint32_t HELPER(neon_acgt_f32
)(uint32_t a
, uint32_t b
)
1835 float32 f0
= float32_abs(make_float32(a
));
1836 float32 f1
= float32_abs(make_float32(b
));
1837 if (float32_compare_quiet(f0
, f1
, NFS
) == float_relation_greater
) {
1843 #define ELEM(V, N, SIZE) (((V) >> ((N) * (SIZE))) & ((1ull << (SIZE)) - 1))
1845 void HELPER(neon_qunzip8
)(CPUState
*env
, uint32_t rd
, uint32_t rm
)
1847 uint64_t zm0
= float64_val(env
->vfp
.regs
[rm
]);
1848 uint64_t zm1
= float64_val(env
->vfp
.regs
[rm
+ 1]);
1849 uint64_t zd0
= float64_val(env
->vfp
.regs
[rd
]);
1850 uint64_t zd1
= float64_val(env
->vfp
.regs
[rd
+ 1]);
1851 uint64_t d0
= ELEM(zd0
, 0, 8) | (ELEM(zd0
, 2, 8) << 8)
1852 | (ELEM(zd0
, 4, 8) << 16) | (ELEM(zd0
, 6, 8) << 24)
1853 | (ELEM(zd1
, 0, 8) << 32) | (ELEM(zd1
, 2, 8) << 40)
1854 | (ELEM(zd1
, 4, 8) << 48) | (ELEM(zd1
, 6, 8) << 56);
1855 uint64_t d1
= ELEM(zm0
, 0, 8) | (ELEM(zm0
, 2, 8) << 8)
1856 | (ELEM(zm0
, 4, 8) << 16) | (ELEM(zm0
, 6, 8) << 24)
1857 | (ELEM(zm1
, 0, 8) << 32) | (ELEM(zm1
, 2, 8) << 40)
1858 | (ELEM(zm1
, 4, 8) << 48) | (ELEM(zm1
, 6, 8) << 56);
1859 uint64_t m0
= ELEM(zd0
, 1, 8) | (ELEM(zd0
, 3, 8) << 8)
1860 | (ELEM(zd0
, 5, 8) << 16) | (ELEM(zd0
, 7, 8) << 24)
1861 | (ELEM(zd1
, 1, 8) << 32) | (ELEM(zd1
, 3, 8) << 40)
1862 | (ELEM(zd1
, 5, 8) << 48) | (ELEM(zd1
, 7, 8) << 56);
1863 uint64_t m1
= ELEM(zm0
, 1, 8) | (ELEM(zm0
, 3, 8) << 8)
1864 | (ELEM(zm0
, 5, 8) << 16) | (ELEM(zm0
, 7, 8) << 24)
1865 | (ELEM(zm1
, 1, 8) << 32) | (ELEM(zm1
, 3, 8) << 40)
1866 | (ELEM(zm1
, 5, 8) << 48) | (ELEM(zm1
, 7, 8) << 56);
1867 env
->vfp
.regs
[rm
] = make_float64(m0
);
1868 env
->vfp
.regs
[rm
+ 1] = make_float64(m1
);
1869 env
->vfp
.regs
[rd
] = make_float64(d0
);
1870 env
->vfp
.regs
[rd
+ 1] = make_float64(d1
);
1873 void HELPER(neon_qunzip16
)(CPUState
*env
, uint32_t rd
, uint32_t rm
)
1875 uint64_t zm0
= float64_val(env
->vfp
.regs
[rm
]);
1876 uint64_t zm1
= float64_val(env
->vfp
.regs
[rm
+ 1]);
1877 uint64_t zd0
= float64_val(env
->vfp
.regs
[rd
]);
1878 uint64_t zd1
= float64_val(env
->vfp
.regs
[rd
+ 1]);
1879 uint64_t d0
= ELEM(zd0
, 0, 16) | (ELEM(zd0
, 2, 16) << 16)
1880 | (ELEM(zd1
, 0, 16) << 32) | (ELEM(zd1
, 2, 16) << 48);
1881 uint64_t d1
= ELEM(zm0
, 0, 16) | (ELEM(zm0
, 2, 16) << 16)
1882 | (ELEM(zm1
, 0, 16) << 32) | (ELEM(zm1
, 2, 16) << 48);
1883 uint64_t m0
= ELEM(zd0
, 1, 16) | (ELEM(zd0
, 3, 16) << 16)
1884 | (ELEM(zd1
, 1, 16) << 32) | (ELEM(zd1
, 3, 16) << 48);
1885 uint64_t m1
= ELEM(zm0
, 1, 16) | (ELEM(zm0
, 3, 16) << 16)
1886 | (ELEM(zm1
, 1, 16) << 32) | (ELEM(zm1
, 3, 16) << 48);
1887 env
->vfp
.regs
[rm
] = make_float64(m0
);
1888 env
->vfp
.regs
[rm
+ 1] = make_float64(m1
);
1889 env
->vfp
.regs
[rd
] = make_float64(d0
);
1890 env
->vfp
.regs
[rd
+ 1] = make_float64(d1
);
1893 void HELPER(neon_qunzip32
)(CPUState
*env
, uint32_t rd
, uint32_t rm
)
1895 uint64_t zm0
= float64_val(env
->vfp
.regs
[rm
]);
1896 uint64_t zm1
= float64_val(env
->vfp
.regs
[rm
+ 1]);
1897 uint64_t zd0
= float64_val(env
->vfp
.regs
[rd
]);
1898 uint64_t zd1
= float64_val(env
->vfp
.regs
[rd
+ 1]);
1899 uint64_t d0
= ELEM(zd0
, 0, 32) | (ELEM(zd1
, 0, 32) << 32);
1900 uint64_t d1
= ELEM(zm0
, 0, 32) | (ELEM(zm1
, 0, 32) << 32);
1901 uint64_t m0
= ELEM(zd0
, 1, 32) | (ELEM(zd1
, 1, 32) << 32);
1902 uint64_t m1
= ELEM(zm0
, 1, 32) | (ELEM(zm1
, 1, 32) << 32);
1903 env
->vfp
.regs
[rm
] = make_float64(m0
);
1904 env
->vfp
.regs
[rm
+ 1] = make_float64(m1
);
1905 env
->vfp
.regs
[rd
] = make_float64(d0
);
1906 env
->vfp
.regs
[rd
+ 1] = make_float64(d1
);
1909 void HELPER(neon_unzip8
)(CPUState
*env
, uint32_t rd
, uint32_t rm
)
1911 uint64_t zm
= float64_val(env
->vfp
.regs
[rm
]);
1912 uint64_t zd
= float64_val(env
->vfp
.regs
[rd
]);
1913 uint64_t d0
= ELEM(zd
, 0, 8) | (ELEM(zd
, 2, 8) << 8)
1914 | (ELEM(zd
, 4, 8) << 16) | (ELEM(zd
, 6, 8) << 24)
1915 | (ELEM(zm
, 0, 8) << 32) | (ELEM(zm
, 2, 8) << 40)
1916 | (ELEM(zm
, 4, 8) << 48) | (ELEM(zm
, 6, 8) << 56);
1917 uint64_t m0
= ELEM(zd
, 1, 8) | (ELEM(zd
, 3, 8) << 8)
1918 | (ELEM(zd
, 5, 8) << 16) | (ELEM(zd
, 7, 8) << 24)
1919 | (ELEM(zm
, 1, 8) << 32) | (ELEM(zm
, 3, 8) << 40)
1920 | (ELEM(zm
, 5, 8) << 48) | (ELEM(zm
, 7, 8) << 56);
1921 env
->vfp
.regs
[rm
] = make_float64(m0
);
1922 env
->vfp
.regs
[rd
] = make_float64(d0
);
1925 void HELPER(neon_unzip16
)(CPUState
*env
, uint32_t rd
, uint32_t rm
)
1927 uint64_t zm
= float64_val(env
->vfp
.regs
[rm
]);
1928 uint64_t zd
= float64_val(env
->vfp
.regs
[rd
]);
1929 uint64_t d0
= ELEM(zd
, 0, 16) | (ELEM(zd
, 2, 16) << 16)
1930 | (ELEM(zm
, 0, 16) << 32) | (ELEM(zm
, 2, 16) << 48);
1931 uint64_t m0
= ELEM(zd
, 1, 16) | (ELEM(zd
, 3, 16) << 16)
1932 | (ELEM(zm
, 1, 16) << 32) | (ELEM(zm
, 3, 16) << 48);
1933 env
->vfp
.regs
[rm
] = make_float64(m0
);
1934 env
->vfp
.regs
[rd
] = make_float64(d0
);
1937 void HELPER(neon_qzip8
)(CPUState
*env
, uint32_t rd
, uint32_t rm
)
1939 uint64_t zm0
= float64_val(env
->vfp
.regs
[rm
]);
1940 uint64_t zm1
= float64_val(env
->vfp
.regs
[rm
+ 1]);
1941 uint64_t zd0
= float64_val(env
->vfp
.regs
[rd
]);
1942 uint64_t zd1
= float64_val(env
->vfp
.regs
[rd
+ 1]);
1943 uint64_t d0
= ELEM(zd0
, 0, 8) | (ELEM(zm0
, 0, 8) << 8)
1944 | (ELEM(zd0
, 1, 8) << 16) | (ELEM(zm0
, 1, 8) << 24)
1945 | (ELEM(zd0
, 2, 8) << 32) | (ELEM(zm0
, 2, 8) << 40)
1946 | (ELEM(zd0
, 3, 8) << 48) | (ELEM(zm0
, 3, 8) << 56);
1947 uint64_t d1
= ELEM(zd0
, 4, 8) | (ELEM(zm0
, 4, 8) << 8)
1948 | (ELEM(zd0
, 5, 8) << 16) | (ELEM(zm0
, 5, 8) << 24)
1949 | (ELEM(zd0
, 6, 8) << 32) | (ELEM(zm0
, 6, 8) << 40)
1950 | (ELEM(zd0
, 7, 8) << 48) | (ELEM(zm0
, 7, 8) << 56);
1951 uint64_t m0
= ELEM(zd1
, 0, 8) | (ELEM(zm1
, 0, 8) << 8)
1952 | (ELEM(zd1
, 1, 8) << 16) | (ELEM(zm1
, 1, 8) << 24)
1953 | (ELEM(zd1
, 2, 8) << 32) | (ELEM(zm1
, 2, 8) << 40)
1954 | (ELEM(zd1
, 3, 8) << 48) | (ELEM(zm1
, 3, 8) << 56);
1955 uint64_t m1
= ELEM(zd1
, 4, 8) | (ELEM(zm1
, 4, 8) << 8)
1956 | (ELEM(zd1
, 5, 8) << 16) | (ELEM(zm1
, 5, 8) << 24)
1957 | (ELEM(zd1
, 6, 8) << 32) | (ELEM(zm1
, 6, 8) << 40)
1958 | (ELEM(zd1
, 7, 8) << 48) | (ELEM(zm1
, 7, 8) << 56);
1959 env
->vfp
.regs
[rm
] = make_float64(m0
);
1960 env
->vfp
.regs
[rm
+ 1] = make_float64(m1
);
1961 env
->vfp
.regs
[rd
] = make_float64(d0
);
1962 env
->vfp
.regs
[rd
+ 1] = make_float64(d1
);
1965 void HELPER(neon_qzip16
)(CPUState
*env
, uint32_t rd
, uint32_t rm
)
1967 uint64_t zm0
= float64_val(env
->vfp
.regs
[rm
]);
1968 uint64_t zm1
= float64_val(env
->vfp
.regs
[rm
+ 1]);
1969 uint64_t zd0
= float64_val(env
->vfp
.regs
[rd
]);
1970 uint64_t zd1
= float64_val(env
->vfp
.regs
[rd
+ 1]);
1971 uint64_t d0
= ELEM(zd0
, 0, 16) | (ELEM(zm0
, 0, 16) << 16)
1972 | (ELEM(zd0
, 1, 16) << 32) | (ELEM(zm0
, 1, 16) << 48);
1973 uint64_t d1
= ELEM(zd0
, 2, 16) | (ELEM(zm0
, 2, 16) << 16)
1974 | (ELEM(zd0
, 3, 16) << 32) | (ELEM(zm0
, 3, 16) << 48);
1975 uint64_t m0
= ELEM(zd1
, 0, 16) | (ELEM(zm1
, 0, 16) << 16)
1976 | (ELEM(zd1
, 1, 16) << 32) | (ELEM(zm1
, 1, 16) << 48);
1977 uint64_t m1
= ELEM(zd1
, 2, 16) | (ELEM(zm1
, 2, 16) << 16)
1978 | (ELEM(zd1
, 3, 16) << 32) | (ELEM(zm1
, 3, 16) << 48);
1979 env
->vfp
.regs
[rm
] = make_float64(m0
);
1980 env
->vfp
.regs
[rm
+ 1] = make_float64(m1
);
1981 env
->vfp
.regs
[rd
] = make_float64(d0
);
1982 env
->vfp
.regs
[rd
+ 1] = make_float64(d1
);
1985 void HELPER(neon_qzip32
)(CPUState
*env
, uint32_t rd
, uint32_t rm
)
1987 uint64_t zm0
= float64_val(env
->vfp
.regs
[rm
]);
1988 uint64_t zm1
= float64_val(env
->vfp
.regs
[rm
+ 1]);
1989 uint64_t zd0
= float64_val(env
->vfp
.regs
[rd
]);
1990 uint64_t zd1
= float64_val(env
->vfp
.regs
[rd
+ 1]);
1991 uint64_t d0
= ELEM(zd0
, 0, 32) | (ELEM(zm0
, 0, 32) << 32);
1992 uint64_t d1
= ELEM(zd0
, 1, 32) | (ELEM(zm0
, 1, 32) << 32);
1993 uint64_t m0
= ELEM(zd1
, 0, 32) | (ELEM(zm1
, 0, 32) << 32);
1994 uint64_t m1
= ELEM(zd1
, 1, 32) | (ELEM(zm1
, 1, 32) << 32);
1995 env
->vfp
.regs
[rm
] = make_float64(m0
);
1996 env
->vfp
.regs
[rm
+ 1] = make_float64(m1
);
1997 env
->vfp
.regs
[rd
] = make_float64(d0
);
1998 env
->vfp
.regs
[rd
+ 1] = make_float64(d1
);
2001 void HELPER(neon_zip8
)(CPUState
*env
, uint32_t rd
, uint32_t rm
)
2003 uint64_t zm
= float64_val(env
->vfp
.regs
[rm
]);
2004 uint64_t zd
= float64_val(env
->vfp
.regs
[rd
]);
2005 uint64_t d0
= ELEM(zd
, 0, 8) | (ELEM(zm
, 0, 8) << 8)
2006 | (ELEM(zd
, 1, 8) << 16) | (ELEM(zm
, 1, 8) << 24)
2007 | (ELEM(zd
, 2, 8) << 32) | (ELEM(zm
, 2, 8) << 40)
2008 | (ELEM(zd
, 3, 8) << 48) | (ELEM(zm
, 3, 8) << 56);
2009 uint64_t m0
= ELEM(zd
, 4, 8) | (ELEM(zm
, 4, 8) << 8)
2010 | (ELEM(zd
, 5, 8) << 16) | (ELEM(zm
, 5, 8) << 24)
2011 | (ELEM(zd
, 6, 8) << 32) | (ELEM(zm
, 6, 8) << 40)
2012 | (ELEM(zd
, 7, 8) << 48) | (ELEM(zm
, 7, 8) << 56);
2013 env
->vfp
.regs
[rm
] = make_float64(m0
);
2014 env
->vfp
.regs
[rd
] = make_float64(d0
);
2017 void HELPER(neon_zip16
)(CPUState
*env
, uint32_t rd
, uint32_t rm
)
2019 uint64_t zm
= float64_val(env
->vfp
.regs
[rm
]);
2020 uint64_t zd
= float64_val(env
->vfp
.regs
[rd
]);
2021 uint64_t d0
= ELEM(zd
, 0, 16) | (ELEM(zm
, 0, 16) << 16)
2022 | (ELEM(zd
, 1, 16) << 32) | (ELEM(zm
, 1, 16) << 48);
2023 uint64_t m0
= ELEM(zd
, 2, 16) | (ELEM(zm
, 2, 16) << 16)
2024 | (ELEM(zd
, 3, 16) << 32) | (ELEM(zm
, 3, 16) << 48);
2025 env
->vfp
.regs
[rm
] = make_float64(m0
);
2026 env
->vfp
.regs
[rd
] = make_float64(d0
);