target/arm/neon_helper.c

   1 /*
   2  * ARM NEON vector operations.
   3  *
   4  * Copyright (c) 2007, 2008 CodeSourcery.
   5  * Written by Paul Brook
   6  *
   7  * This code is licensed under the GNU GPL v2.
   8  */
   9 #include "qemu/osdep.h"
  10
  11 #include "cpu.h"
  12 #include "exec/helper-proto.h"
  13 #include "fpu/softfloat.h"
  14
  15 #define SIGNBIT (uint32_t)0x80000000
  16 #define SIGNBIT64 ((uint64_t)1 << 63)
  17
  18 #define SET_QC() env->vfp.qc[0] = 1
  19
  20 #define NEON_TYPE1(name, type) \
  21 typedef struct \
  22 { \
  23     type v1; \
  24 } neon_##name;
  25 #ifdef HOST_WORDS_BIGENDIAN
  26 #define NEON_TYPE2(name, type) \
  27 typedef struct \
  28 { \
  29     type v2; \
  30     type v1; \
  31 } neon_##name;
  32 #define NEON_TYPE4(name, type) \
  33 typedef struct \
  34 { \
  35     type v4; \
  36     type v3; \
  37     type v2; \
  38     type v1; \
  39 } neon_##name;
  40 #else
  41 #define NEON_TYPE2(name, type) \
  42 typedef struct \
  43 { \
  44     type v1; \
  45     type v2; \
  46 } neon_##name;
  47 #define NEON_TYPE4(name, type) \
  48 typedef struct \
  49 { \
  50     type v1; \
  51     type v2; \
  52     type v3; \
  53     type v4; \
  54 } neon_##name;
  55 #endif
  56
  57 NEON_TYPE4(s8, int8_t)
  58 NEON_TYPE4(u8, uint8_t)
  59 NEON_TYPE2(s16, int16_t)
  60 NEON_TYPE2(u16, uint16_t)
  61 NEON_TYPE1(s32, int32_t)
  62 NEON_TYPE1(u32, uint32_t)
  63 #undef NEON_TYPE4
  64 #undef NEON_TYPE2
  65 #undef NEON_TYPE1
  66
  67 /* Copy from a uint32_t to a vector structure type.  */
  68 #define NEON_UNPACK(vtype, dest, val) do { \
  69     union { \
  70         vtype v; \
  71         uint32_t i; \
  72     } conv_u; \
  73     conv_u.i = (val); \
  74     dest = conv_u.v; \
  75     } while(0)
  76
  77 /* Copy from a vector structure type to a uint32_t.  */
  78 #define NEON_PACK(vtype, dest, val) do { \
  79     union { \
  80         vtype v; \
  81         uint32_t i; \
  82     } conv_u; \
  83     conv_u.v = (val); \
  84     dest = conv_u.i; \
  85     } while(0)
  86
  87 #define NEON_DO1 \
  88     NEON_FN(vdest.v1, vsrc1.v1, vsrc2.v1);
  89 #define NEON_DO2 \
  90     NEON_FN(vdest.v1, vsrc1.v1, vsrc2.v1); \
  91     NEON_FN(vdest.v2, vsrc1.v2, vsrc2.v2);
  92 #define NEON_DO4 \
  93     NEON_FN(vdest.v1, vsrc1.v1, vsrc2.v1); \
  94     NEON_FN(vdest.v2, vsrc1.v2, vsrc2.v2); \
  95     NEON_FN(vdest.v3, vsrc1.v3, vsrc2.v3); \
  96     NEON_FN(vdest.v4, vsrc1.v4, vsrc2.v4);
  97
  98 #define NEON_VOP_BODY(vtype, n) \
  99 { \
 100     uint32_t res; \
 101     vtype vsrc1; \
 102     vtype vsrc2; \
 103     vtype vdest; \
 104     NEON_UNPACK(vtype, vsrc1, arg1); \
 105     NEON_UNPACK(vtype, vsrc2, arg2); \
 106     NEON_DO##n; \
 107     NEON_PACK(vtype, res, vdest); \
 108     return res; \
 109 }
 110
 111 #define NEON_VOP(name, vtype, n) \
 112 uint32_t HELPER(glue(neon_,name))(uint32_t arg1, uint32_t arg2) \
 113 NEON_VOP_BODY(vtype, n)
 114
 115 #define NEON_VOP_ENV(name, vtype, n) \
 116 uint32_t HELPER(glue(neon_,name))(CPUARMState *env, uint32_t arg1, uint32_t arg2) \
 117 NEON_VOP_BODY(vtype, n)
 118
 119 /* Pairwise operations.  */
 120 /* For 32-bit elements each segment only contains a single element, so
 121    the elementwise and pairwise operations are the same.  */
 122 #define NEON_PDO2 \
 123     NEON_FN(vdest.v1, vsrc1.v1, vsrc1.v2); \
 124     NEON_FN(vdest.v2, vsrc2.v1, vsrc2.v2);
 125 #define NEON_PDO4 \
 126     NEON_FN(vdest.v1, vsrc1.v1, vsrc1.v2); \
 127     NEON_FN(vdest.v2, vsrc1.v3, vsrc1.v4); \
 128     NEON_FN(vdest.v3, vsrc2.v1, vsrc2.v2); \
 129     NEON_FN(vdest.v4, vsrc2.v3, vsrc2.v4); \
 130
 131 #define NEON_POP(name, vtype, n) \
 132 uint32_t HELPER(glue(neon_,name))(uint32_t arg1, uint32_t arg2) \
 133 { \
 134     uint32_t res; \
 135     vtype vsrc1; \
 136     vtype vsrc2; \
 137     vtype vdest; \
 138     NEON_UNPACK(vtype, vsrc1, arg1); \
 139     NEON_UNPACK(vtype, vsrc2, arg2); \
 140     NEON_PDO##n; \
 141     NEON_PACK(vtype, res, vdest); \
 142     return res; \
 143 }
 144
 145 /* Unary operators.  */
 146 #define NEON_VOP1(name, vtype, n) \
 147 uint32_t HELPER(glue(neon_,name))(uint32_t arg) \
 148 { \
 149     vtype vsrc1; \
 150     vtype vdest; \
 151     NEON_UNPACK(vtype, vsrc1, arg); \
 152     NEON_DO##n; \
 153     NEON_PACK(vtype, arg, vdest); \
 154     return arg; \
 155 }
 156
 157
 158 #define NEON_USAT(dest, src1, src2, type) do { \
 159     uint32_t tmp = (uint32_t)src1 + (uint32_t)src2; \
 160     if (tmp != (type)tmp) { \
 161         SET_QC(); \
 162         dest = ~0; \
 163     } else { \
 164         dest = tmp; \
 165     }} while(0)
 166 #define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint8_t)
 167 NEON_VOP_ENV(qadd_u8, neon_u8, 4)
 168 #undef NEON_FN
 169 #define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint16_t)
 170 NEON_VOP_ENV(qadd_u16, neon_u16, 2)
 171 #undef NEON_FN
 172 #undef NEON_USAT
 173
 174 uint32_t HELPER(neon_qadd_u32)(CPUARMState *env, uint32_t a, uint32_t b)
 175 {
 176     uint32_t res = a + b;
 177     if (res < a) {
 178         SET_QC();
 179         res = ~0;
 180     }
 181     return res;
 182 }
 183
 184 uint64_t HELPER(neon_qadd_u64)(CPUARMState *env, uint64_t src1, uint64_t src2)
 185 {
 186     uint64_t res;
 187
 188     res = src1 + src2;
 189     if (res < src1) {
 190         SET_QC();
 191         res = ~(uint64_t)0;
 192     }
 193     return res;
 194 }
 195
 196 #define NEON_SSAT(dest, src1, src2, type) do { \
 197     int32_t tmp = (uint32_t)src1 + (uint32_t)src2; \
 198     if (tmp != (type)tmp) { \
 199         SET_QC(); \
 200         if (src2 > 0) { \
 201             tmp = (1 << (sizeof(type) * 8 - 1)) - 1; \
 202         } else { \
 203             tmp = 1 << (sizeof(type) * 8 - 1); \
 204         } \
 205     } \
 206     dest = tmp; \
 207     } while(0)
 208 #define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int8_t)
 209 NEON_VOP_ENV(qadd_s8, neon_s8, 4)
 210 #undef NEON_FN
 211 #define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int16_t)
 212 NEON_VOP_ENV(qadd_s16, neon_s16, 2)
 213 #undef NEON_FN
 214 #undef NEON_SSAT
 215
 216 uint32_t HELPER(neon_qadd_s32)(CPUARMState *env, uint32_t a, uint32_t b)
 217 {
 218     uint32_t res = a + b;
 219     if (((res ^ a) & SIGNBIT) && !((a ^ b) & SIGNBIT)) {
 220         SET_QC();
 221         res = ~(((int32_t)a >> 31) ^ SIGNBIT);
 222     }
 223     return res;
 224 }
 225
 226 uint64_t HELPER(neon_qadd_s64)(CPUARMState *env, uint64_t src1, uint64_t src2)
 227 {
 228     uint64_t res;
 229
 230     res = src1 + src2;
 231     if (((res ^ src1) & SIGNBIT64) && !((src1 ^ src2) & SIGNBIT64)) {
 232         SET_QC();
 233         res = ((int64_t)src1 >> 63) ^ ~SIGNBIT64;
 234     }
 235     return res;
 236 }
 237
 238 /* Unsigned saturating accumulate of signed value
 239  *
 240  * Op1/Rn is treated as signed
 241  * Op2/Rd is treated as unsigned
 242  *
 243  * Explicit casting is used to ensure the correct sign extension of
 244  * inputs. The result is treated as a unsigned value and saturated as such.
 245  *
 246  * We use a macro for the 8/16 bit cases which expects signed integers of va,
 247  * vb, and vr for interim calculation and an unsigned 32 bit result value r.
 248  */
 249
 250 #define USATACC(bits, shift) \
 251     do { \
 252         va = sextract32(a, shift, bits);                                \
 253         vb = extract32(b, shift, bits);                                 \
 254         vr = va + vb;                                                   \
 255         if (vr > UINT##bits##_MAX) {                                    \
 256             SET_QC();                                                   \
 257             vr = UINT##bits##_MAX;                                      \
 258         } else if (vr < 0) {                                            \
 259             SET_QC();                                                   \
 260             vr = 0;                                                     \
 261         }                                                               \
 262         r = deposit32(r, shift, bits, vr);                              \
 263    } while (0)
 264
 265 uint32_t HELPER(neon_uqadd_s8)(CPUARMState *env, uint32_t a, uint32_t b)
 266 {
 267     int16_t va, vb, vr;
 268     uint32_t r = 0;
 269
 270     USATACC(8, 0);
 271     USATACC(8, 8);
 272     USATACC(8, 16);
 273     USATACC(8, 24);
 274     return r;
 275 }
 276
 277 uint32_t HELPER(neon_uqadd_s16)(CPUARMState *env, uint32_t a, uint32_t b)
 278 {
 279     int32_t va, vb, vr;
 280     uint64_t r = 0;
 281
 282     USATACC(16, 0);
 283     USATACC(16, 16);
 284     return r;
 285 }
 286
 287 #undef USATACC
 288
 289 uint32_t HELPER(neon_uqadd_s32)(CPUARMState *env, uint32_t a, uint32_t b)
 290 {
 291     int64_t va = (int32_t)a;
 292     int64_t vb = (uint32_t)b;
 293     int64_t vr = va + vb;
 294     if (vr > UINT32_MAX) {
 295         SET_QC();
 296         vr = UINT32_MAX;
 297     } else if (vr < 0) {
 298         SET_QC();
 299         vr = 0;
 300     }
 301     return vr;
 302 }
 303
 304 uint64_t HELPER(neon_uqadd_s64)(CPUARMState *env, uint64_t a, uint64_t b)
 305 {
 306     uint64_t res;
 307     res = a + b;
 308     /* We only need to look at the pattern of SIGN bits to detect
 309      * +ve/-ve saturation
 310      */
 311     if (~a & b & ~res & SIGNBIT64) {
 312         SET_QC();
 313         res = UINT64_MAX;
 314     } else if (a & ~b & res & SIGNBIT64) {
 315         SET_QC();
 316         res = 0;
 317     }
 318     return res;
 319 }
 320
 321 /* Signed saturating accumulate of unsigned value
 322  *
 323  * Op1/Rn is treated as unsigned
 324  * Op2/Rd is treated as signed
 325  *
 326  * The result is treated as a signed value and saturated as such
 327  *
 328  * We use a macro for the 8/16 bit cases which expects signed integers of va,
 329  * vb, and vr for interim calculation and an unsigned 32 bit result value r.
 330  */
 331
 332 #define SSATACC(bits, shift) \
 333     do { \
 334         va = extract32(a, shift, bits);                                 \
 335         vb = sextract32(b, shift, bits);                                \
 336         vr = va + vb;                                                   \
 337         if (vr > INT##bits##_MAX) {                                     \
 338             SET_QC();                                                   \
 339             vr = INT##bits##_MAX;                                       \
 340         } else if (vr < INT##bits##_MIN) {                              \
 341             SET_QC();                                                   \
 342             vr = INT##bits##_MIN;                                       \
 343         }                                                               \
 344         r = deposit32(r, shift, bits, vr);                              \
 345     } while (0)
 346
 347 uint32_t HELPER(neon_sqadd_u8)(CPUARMState *env, uint32_t a, uint32_t b)
 348 {
 349     int16_t va, vb, vr;
 350     uint32_t r = 0;
 351
 352     SSATACC(8, 0);
 353     SSATACC(8, 8);
 354     SSATACC(8, 16);
 355     SSATACC(8, 24);
 356     return r;
 357 }
 358
 359 uint32_t HELPER(neon_sqadd_u16)(CPUARMState *env, uint32_t a, uint32_t b)
 360 {
 361     int32_t va, vb, vr;
 362     uint32_t r = 0;
 363
 364     SSATACC(16, 0);
 365     SSATACC(16, 16);
 366
 367     return r;
 368 }
 369
 370 #undef SSATACC
 371
 372 uint32_t HELPER(neon_sqadd_u32)(CPUARMState *env, uint32_t a, uint32_t b)
 373 {
 374     int64_t res;
 375     int64_t op1 = (uint32_t)a;
 376     int64_t op2 = (int32_t)b;
 377     res = op1 + op2;
 378     if (res > INT32_MAX) {
 379         SET_QC();
 380         res = INT32_MAX;
 381     } else if (res < INT32_MIN) {
 382         SET_QC();
 383         res = INT32_MIN;
 384     }
 385     return res;
 386 }
 387
 388 uint64_t HELPER(neon_sqadd_u64)(CPUARMState *env, uint64_t a, uint64_t b)
 389 {
 390     uint64_t res;
 391     res = a + b;
 392     /* We only need to look at the pattern of SIGN bits to detect an overflow */
 393     if (((a & res)
 394          | (~b & res)
 395          | (a & ~b)) & SIGNBIT64) {
 396         SET_QC();
 397         res = INT64_MAX;
 398     }
 399     return res;
 400 }
 401
 402
 403 #define NEON_USAT(dest, src1, src2, type) do { \
 404     uint32_t tmp = (uint32_t)src1 - (uint32_t)src2; \
 405     if (tmp != (type)tmp) { \
 406         SET_QC(); \
 407         dest = 0; \
 408     } else { \
 409         dest = tmp; \
 410     }} while(0)
 411 #define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint8_t)
 412 NEON_VOP_ENV(qsub_u8, neon_u8, 4)
 413 #undef NEON_FN
 414 #define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint16_t)
 415 NEON_VOP_ENV(qsub_u16, neon_u16, 2)
 416 #undef NEON_FN
 417 #undef NEON_USAT
 418
 419 uint32_t HELPER(neon_qsub_u32)(CPUARMState *env, uint32_t a, uint32_t b)
 420 {
 421     uint32_t res = a - b;
 422     if (res > a) {
 423         SET_QC();
 424         res = 0;
 425     }
 426     return res;
 427 }
 428
 429 uint64_t HELPER(neon_qsub_u64)(CPUARMState *env, uint64_t src1, uint64_t src2)
 430 {
 431     uint64_t res;
 432
 433     if (src1 < src2) {
 434         SET_QC();
 435         res = 0;
 436     } else {
 437         res = src1 - src2;
 438     }
 439     return res;
 440 }
 441
 442 #define NEON_SSAT(dest, src1, src2, type) do { \
 443     int32_t tmp = (uint32_t)src1 - (uint32_t)src2; \
 444     if (tmp != (type)tmp) { \
 445         SET_QC(); \
 446         if (src2 < 0) { \
 447             tmp = (1 << (sizeof(type) * 8 - 1)) - 1; \
 448         } else { \
 449             tmp = 1 << (sizeof(type) * 8 - 1); \
 450         } \
 451     } \
 452     dest = tmp; \
 453     } while(0)
 454 #define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int8_t)
 455 NEON_VOP_ENV(qsub_s8, neon_s8, 4)
 456 #undef NEON_FN
 457 #define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int16_t)
 458 NEON_VOP_ENV(qsub_s16, neon_s16, 2)
 459 #undef NEON_FN
 460 #undef NEON_SSAT
 461
 462 uint32_t HELPER(neon_qsub_s32)(CPUARMState *env, uint32_t a, uint32_t b)
 463 {
 464     uint32_t res = a - b;
 465     if (((res ^ a) & SIGNBIT) && ((a ^ b) & SIGNBIT)) {
 466         SET_QC();
 467         res = ~(((int32_t)a >> 31) ^ SIGNBIT);
 468     }
 469     return res;
 470 }
 471
 472 uint64_t HELPER(neon_qsub_s64)(CPUARMState *env, uint64_t src1, uint64_t src2)
 473 {
 474     uint64_t res;
 475
 476     res = src1 - src2;
 477     if (((res ^ src1) & SIGNBIT64) && ((src1 ^ src2) & SIGNBIT64)) {
 478         SET_QC();
 479         res = ((int64_t)src1 >> 63) ^ ~SIGNBIT64;
 480     }
 481     return res;
 482 }
 483
 484 #define NEON_FN(dest, src1, src2) dest = (src1 + src2) >> 1
 485 NEON_VOP(hadd_s8, neon_s8, 4)
 486 NEON_VOP(hadd_u8, neon_u8, 4)
 487 NEON_VOP(hadd_s16, neon_s16, 2)
 488 NEON_VOP(hadd_u16, neon_u16, 2)
 489 #undef NEON_FN
 490
 491 int32_t HELPER(neon_hadd_s32)(int32_t src1, int32_t src2)
 492 {
 493     int32_t dest;
 494
 495     dest = (src1 >> 1) + (src2 >> 1);
 496     if (src1 & src2 & 1)
 497         dest++;
 498     return dest;
 499 }
 500
 501 uint32_t HELPER(neon_hadd_u32)(uint32_t src1, uint32_t src2)
 502 {
 503     uint32_t dest;
 504
 505     dest = (src1 >> 1) + (src2 >> 1);
 506     if (src1 & src2 & 1)
 507         dest++;
 508     return dest;
 509 }
 510
 511 #define NEON_FN(dest, src1, src2) dest = (src1 + src2 + 1) >> 1
 512 NEON_VOP(rhadd_s8, neon_s8, 4)
 513 NEON_VOP(rhadd_u8, neon_u8, 4)
 514 NEON_VOP(rhadd_s16, neon_s16, 2)
 515 NEON_VOP(rhadd_u16, neon_u16, 2)
 516 #undef NEON_FN
 517
 518 int32_t HELPER(neon_rhadd_s32)(int32_t src1, int32_t src2)
 519 {
 520     int32_t dest;
 521
 522     dest = (src1 >> 1) + (src2 >> 1);
 523     if ((src1 | src2) & 1)
 524         dest++;
 525     return dest;
 526 }
 527
 528 uint32_t HELPER(neon_rhadd_u32)(uint32_t src1, uint32_t src2)
 529 {
 530     uint32_t dest;
 531
 532     dest = (src1 >> 1) + (src2 >> 1);
 533     if ((src1 | src2) & 1)
 534         dest++;
 535     return dest;
 536 }
 537
 538 #define NEON_FN(dest, src1, src2) dest = (src1 - src2) >> 1
 539 NEON_VOP(hsub_s8, neon_s8, 4)
 540 NEON_VOP(hsub_u8, neon_u8, 4)
 541 NEON_VOP(hsub_s16, neon_s16, 2)
 542 NEON_VOP(hsub_u16, neon_u16, 2)
 543 #undef NEON_FN
 544
 545 int32_t HELPER(neon_hsub_s32)(int32_t src1, int32_t src2)
 546 {
 547     int32_t dest;
 548
 549     dest = (src1 >> 1) - (src2 >> 1);
 550     if ((~src1) & src2 & 1)
 551         dest--;
 552     return dest;
 553 }
 554
 555 uint32_t HELPER(neon_hsub_u32)(uint32_t src1, uint32_t src2)
 556 {
 557     uint32_t dest;
 558
 559     dest = (src1 >> 1) - (src2 >> 1);
 560     if ((~src1) & src2 & 1)
 561         dest--;
 562     return dest;
 563 }
 564
 565 #define NEON_FN(dest, src1, src2) dest = (src1 < src2) ? src1 : src2
 566 NEON_POP(pmin_s8, neon_s8, 4)
 567 NEON_POP(pmin_u8, neon_u8, 4)
 568 NEON_POP(pmin_s16, neon_s16, 2)
 569 NEON_POP(pmin_u16, neon_u16, 2)
 570 #undef NEON_FN
 571
 572 #define NEON_FN(dest, src1, src2) dest = (src1 > src2) ? src1 : src2
 573 NEON_POP(pmax_s8, neon_s8, 4)
 574 NEON_POP(pmax_u8, neon_u8, 4)
 575 NEON_POP(pmax_s16, neon_s16, 2)
 576 NEON_POP(pmax_u16, neon_u16, 2)
 577 #undef NEON_FN
 578
 579 #define NEON_FN(dest, src1, src2) \
 580     dest = (src1 > src2) ? (src1 - src2) : (src2 - src1)
 581 NEON_VOP(abd_s8, neon_s8, 4)
 582 NEON_VOP(abd_u8, neon_u8, 4)
 583 NEON_VOP(abd_s16, neon_s16, 2)
 584 NEON_VOP(abd_u16, neon_u16, 2)
 585 NEON_VOP(abd_s32, neon_s32, 1)
 586 NEON_VOP(abd_u32, neon_u32, 1)
 587 #undef NEON_FN
 588
 589 #define NEON_FN(dest, src1, src2) do { \
 590     int8_t tmp; \
 591     tmp = (int8_t)src2; \
 592     if (tmp >= (ssize_t)sizeof(src1) * 8 || \
 593         tmp <= -(ssize_t)sizeof(src1) * 8) { \
 594         dest = 0; \
 595     } else if (tmp < 0) { \
 596         dest = src1 >> -tmp; \
 597     } else { \
 598         dest = src1 << tmp; \
 599     }} while (0)
 600 NEON_VOP(shl_u16, neon_u16, 2)
 601 #undef NEON_FN
 602
 603 #define NEON_FN(dest, src1, src2) do { \
 604     int8_t tmp; \
 605     tmp = (int8_t)src2; \
 606     if (tmp >= (ssize_t)sizeof(src1) * 8) { \
 607         dest = 0; \
 608     } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
 609         dest = src1 >> (sizeof(src1) * 8 - 1); \
 610     } else if (tmp < 0) { \
 611         dest = src1 >> -tmp; \
 612     } else { \
 613         dest = src1 << tmp; \
 614     }} while (0)
 615 NEON_VOP(shl_s16, neon_s16, 2)
 616 #undef NEON_FN
 617
 618 #define NEON_FN(dest, src1, src2) do { \
 619     int8_t tmp; \
 620     tmp = (int8_t)src2; \
 621     if ((tmp >= (ssize_t)sizeof(src1) * 8) \
 622         || (tmp <= -(ssize_t)sizeof(src1) * 8)) { \
 623         dest = 0; \
 624     } else if (tmp < 0) { \
 625         dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
 626     } else { \
 627         dest = src1 << tmp; \
 628     }} while (0)
 629 NEON_VOP(rshl_s8, neon_s8, 4)
 630 NEON_VOP(rshl_s16, neon_s16, 2)
 631 #undef NEON_FN
 632
 633 /* The addition of the rounding constant may overflow, so we use an
 634  * intermediate 64 bit accumulator.  */
 635 uint32_t HELPER(neon_rshl_s32)(uint32_t valop, uint32_t shiftop)
 636 {
 637     int32_t dest;
 638     int32_t val = (int32_t)valop;
 639     int8_t shift = (int8_t)shiftop;
 640     if ((shift >= 32) || (shift <= -32)) {
 641         dest = 0;
 642     } else if (shift < 0) {
 643         int64_t big_dest = ((int64_t)val + (1 << (-1 - shift)));
 644         dest = big_dest >> -shift;
 645     } else {
 646         dest = val << shift;
 647     }
 648     return dest;
 649 }
 650
 651 /* Handling addition overflow with 64 bit input values is more
 652  * tricky than with 32 bit values.  */
 653 uint64_t HELPER(neon_rshl_s64)(uint64_t valop, uint64_t shiftop)
 654 {
 655     int8_t shift = (int8_t)shiftop;
 656     int64_t val = valop;
 657     if ((shift >= 64) || (shift <= -64)) {
 658         val = 0;
 659     } else if (shift < 0) {
 660         val >>= (-shift - 1);
 661         if (val == INT64_MAX) {
 662             /* In this case, it means that the rounding constant is 1,
 663              * and the addition would overflow. Return the actual
 664              * result directly.  */
 665             val = 0x4000000000000000LL;
 666         } else {
 667             val++;
 668             val >>= 1;
 669         }
 670     } else {
 671         val <<= shift;
 672     }
 673     return val;
 674 }
 675
 676 #define NEON_FN(dest, src1, src2) do { \
 677     int8_t tmp; \
 678     tmp = (int8_t)src2; \
 679     if (tmp >= (ssize_t)sizeof(src1) * 8 || \
 680         tmp < -(ssize_t)sizeof(src1) * 8) { \
 681         dest = 0; \
 682     } else if (tmp == -(ssize_t)sizeof(src1) * 8) { \
 683         dest = src1 >> (-tmp - 1); \
 684     } else if (tmp < 0) { \
 685         dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
 686     } else { \
 687         dest = src1 << tmp; \
 688     }} while (0)
 689 NEON_VOP(rshl_u8, neon_u8, 4)
 690 NEON_VOP(rshl_u16, neon_u16, 2)
 691 #undef NEON_FN
 692
 693 /* The addition of the rounding constant may overflow, so we use an
 694  * intermediate 64 bit accumulator.  */
 695 uint32_t HELPER(neon_rshl_u32)(uint32_t val, uint32_t shiftop)
 696 {
 697     uint32_t dest;
 698     int8_t shift = (int8_t)shiftop;
 699     if (shift >= 32 || shift < -32) {
 700         dest = 0;
 701     } else if (shift == -32) {
 702         dest = val >> 31;
 703     } else if (shift < 0) {
 704         uint64_t big_dest = ((uint64_t)val + (1 << (-1 - shift)));
 705         dest = big_dest >> -shift;
 706     } else {
 707         dest = val << shift;
 708     }
 709     return dest;
 710 }
 711
 712 /* Handling addition overflow with 64 bit input values is more
 713  * tricky than with 32 bit values.  */
 714 uint64_t HELPER(neon_rshl_u64)(uint64_t val, uint64_t shiftop)
 715 {
 716     int8_t shift = (uint8_t)shiftop;
 717     if (shift >= 64 || shift < -64) {
 718         val = 0;
 719     } else if (shift == -64) {
 720         /* Rounding a 1-bit result just preserves that bit.  */
 721         val >>= 63;
 722     } else if (shift < 0) {
 723         val >>= (-shift - 1);
 724         if (val == UINT64_MAX) {
 725             /* In this case, it means that the rounding constant is 1,
 726              * and the addition would overflow. Return the actual
 727              * result directly.  */
 728             val = 0x8000000000000000ULL;
 729         } else {
 730             val++;
 731             val >>= 1;
 732         }
 733     } else {
 734         val <<= shift;
 735     }
 736     return val;
 737 }
 738
 739 #define NEON_FN(dest, src1, src2) do { \
 740     int8_t tmp; \
 741     tmp = (int8_t)src2; \
 742     if (tmp >= (ssize_t)sizeof(src1) * 8) { \
 743         if (src1) { \
 744             SET_QC(); \
 745             dest = ~0; \
 746         } else { \
 747             dest = 0; \
 748         } \
 749     } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
 750         dest = 0; \
 751     } else if (tmp < 0) { \
 752         dest = src1 >> -tmp; \
 753     } else { \
 754         dest = src1 << tmp; \
 755         if ((dest >> tmp) != src1) { \
 756             SET_QC(); \
 757             dest = ~0; \
 758         } \
 759     }} while (0)
 760 NEON_VOP_ENV(qshl_u8, neon_u8, 4)
 761 NEON_VOP_ENV(qshl_u16, neon_u16, 2)
 762 NEON_VOP_ENV(qshl_u32, neon_u32, 1)
 763 #undef NEON_FN
 764
 765 uint64_t HELPER(neon_qshl_u64)(CPUARMState *env, uint64_t val, uint64_t shiftop)
 766 {
 767     int8_t shift = (int8_t)shiftop;
 768     if (shift >= 64) {
 769         if (val) {
 770             val = ~(uint64_t)0;
 771             SET_QC();
 772         }
 773     } else if (shift <= -64) {
 774         val = 0;
 775     } else if (shift < 0) {
 776         val >>= -shift;
 777     } else {
 778         uint64_t tmp = val;
 779         val <<= shift;
 780         if ((val >> shift) != tmp) {
 781             SET_QC();
 782             val = ~(uint64_t)0;
 783         }
 784     }
 785     return val;
 786 }
 787
 788 #define NEON_FN(dest, src1, src2) do { \
 789     int8_t tmp; \
 790     tmp = (int8_t)src2; \
 791     if (tmp >= (ssize_t)sizeof(src1) * 8) { \
 792         if (src1) { \
 793             SET_QC(); \
 794             dest = (uint32_t)(1 << (sizeof(src1) * 8 - 1)); \
 795             if (src1 > 0) { \
 796                 dest--; \
 797             } \
 798         } else { \
 799             dest = src1; \
 800         } \
 801     } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
 802         dest = src1 >> 31; \
 803     } else if (tmp < 0) { \
 804         dest = src1 >> -tmp; \
 805     } else { \
 806         dest = src1 << tmp; \
 807         if ((dest >> tmp) != src1) { \
 808             SET_QC(); \
 809             dest = (uint32_t)(1 << (sizeof(src1) * 8 - 1)); \
 810             if (src1 > 0) { \
 811                 dest--; \
 812             } \
 813         } \
 814     }} while (0)
 815 NEON_VOP_ENV(qshl_s8, neon_s8, 4)
 816 NEON_VOP_ENV(qshl_s16, neon_s16, 2)
 817 NEON_VOP_ENV(qshl_s32, neon_s32, 1)
 818 #undef NEON_FN
 819
 820 uint64_t HELPER(neon_qshl_s64)(CPUARMState *env, uint64_t valop, uint64_t shiftop)
 821 {
 822     int8_t shift = (uint8_t)shiftop;
 823     int64_t val = valop;
 824     if (shift >= 64) {
 825         if (val) {
 826             SET_QC();
 827             val = (val >> 63) ^ ~SIGNBIT64;
 828         }
 829     } else if (shift <= -64) {
 830         val >>= 63;
 831     } else if (shift < 0) {
 832         val >>= -shift;
 833     } else {
 834         int64_t tmp = val;
 835         val <<= shift;
 836         if ((val >> shift) != tmp) {
 837             SET_QC();
 838             val = (tmp >> 63) ^ ~SIGNBIT64;
 839         }
 840     }
 841     return val;
 842 }
 843
 844 #define NEON_FN(dest, src1, src2) do { \
 845     if (src1 & (1 << (sizeof(src1) * 8 - 1))) { \
 846         SET_QC(); \
 847         dest = 0; \
 848     } else { \
 849         int8_t tmp; \
 850         tmp = (int8_t)src2; \
 851         if (tmp >= (ssize_t)sizeof(src1) * 8) { \
 852             if (src1) { \
 853                 SET_QC(); \
 854                 dest = ~0; \
 855             } else { \
 856                 dest = 0; \
 857             } \
 858         } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
 859             dest = 0; \
 860         } else if (tmp < 0) { \
 861             dest = src1 >> -tmp; \
 862         } else { \
 863             dest = src1 << tmp; \
 864             if ((dest >> tmp) != src1) { \
 865                 SET_QC(); \
 866                 dest = ~0; \
 867             } \
 868         } \
 869     }} while (0)
 870 NEON_VOP_ENV(qshlu_s8, neon_u8, 4)
 871 NEON_VOP_ENV(qshlu_s16, neon_u16, 2)
 872 #undef NEON_FN
 873
 874 uint32_t HELPER(neon_qshlu_s32)(CPUARMState *env, uint32_t valop, uint32_t shiftop)
 875 {
 876     if ((int32_t)valop < 0) {
 877         SET_QC();
 878         return 0;
 879     }
 880     return helper_neon_qshl_u32(env, valop, shiftop);
 881 }
 882
 883 uint64_t HELPER(neon_qshlu_s64)(CPUARMState *env, uint64_t valop, uint64_t shiftop)
 884 {
 885     if ((int64_t)valop < 0) {
 886         SET_QC();
 887         return 0;
 888     }
 889     return helper_neon_qshl_u64(env, valop, shiftop);
 890 }
 891
 892 #define NEON_FN(dest, src1, src2) do { \
 893     int8_t tmp; \
 894     tmp = (int8_t)src2; \
 895     if (tmp >= (ssize_t)sizeof(src1) * 8) { \
 896         if (src1) { \
 897             SET_QC(); \
 898             dest = ~0; \
 899         } else { \
 900             dest = 0; \
 901         } \
 902     } else if (tmp < -(ssize_t)sizeof(src1) * 8) { \
 903         dest = 0; \
 904     } else if (tmp == -(ssize_t)sizeof(src1) * 8) { \
 905         dest = src1 >> (sizeof(src1) * 8 - 1); \
 906     } else if (tmp < 0) { \
 907         dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
 908     } else { \
 909         dest = src1 << tmp; \
 910         if ((dest >> tmp) != src1) { \
 911             SET_QC(); \
 912             dest = ~0; \
 913         } \
 914     }} while (0)
 915 NEON_VOP_ENV(qrshl_u8, neon_u8, 4)
 916 NEON_VOP_ENV(qrshl_u16, neon_u16, 2)
 917 #undef NEON_FN
 918
 919 /* The addition of the rounding constant may overflow, so we use an
 920  * intermediate 64 bit accumulator.  */
 921 uint32_t HELPER(neon_qrshl_u32)(CPUARMState *env, uint32_t val, uint32_t shiftop)
 922 {
 923     uint32_t dest;
 924     int8_t shift = (int8_t)shiftop;
 925     if (shift >= 32) {
 926         if (val) {
 927             SET_QC();
 928             dest = ~0;
 929         } else {
 930             dest = 0;
 931         }
 932     } else if (shift < -32) {
 933         dest = 0;
 934     } else if (shift == -32) {
 935         dest = val >> 31;
 936     } else if (shift < 0) {
 937         uint64_t big_dest = ((uint64_t)val + (1 << (-1 - shift)));
 938         dest = big_dest >> -shift;
 939     } else {
 940         dest = val << shift;
 941         if ((dest >> shift) != val) {
 942             SET_QC();
 943             dest = ~0;
 944         }
 945     }
 946     return dest;
 947 }
 948
 949 /* Handling addition overflow with 64 bit input values is more
 950  * tricky than with 32 bit values.  */
 951 uint64_t HELPER(neon_qrshl_u64)(CPUARMState *env, uint64_t val, uint64_t shiftop)
 952 {
 953     int8_t shift = (int8_t)shiftop;
 954     if (shift >= 64) {
 955         if (val) {
 956             SET_QC();
 957             val = ~0;
 958         }
 959     } else if (shift < -64) {
 960         val = 0;
 961     } else if (shift == -64) {
 962         val >>= 63;
 963     } else if (shift < 0) {
 964         val >>= (-shift - 1);
 965         if (val == UINT64_MAX) {
 966             /* In this case, it means that the rounding constant is 1,
 967              * and the addition would overflow. Return the actual
 968              * result directly.  */
 969             val = 0x8000000000000000ULL;
 970         } else {
 971             val++;
 972             val >>= 1;
 973         }
 974     } else { \
 975         uint64_t tmp = val;
 976         val <<= shift;
 977         if ((val >> shift) != tmp) {
 978             SET_QC();
 979             val = ~0;
 980         }
 981     }
 982     return val;
 983 }
 984
 985 #define NEON_FN(dest, src1, src2) do { \
 986     int8_t tmp; \
 987     tmp = (int8_t)src2; \
 988     if (tmp >= (ssize_t)sizeof(src1) * 8) { \
 989         if (src1) { \
 990             SET_QC(); \
 991             dest = (typeof(dest))(1 << (sizeof(src1) * 8 - 1)); \
 992             if (src1 > 0) { \
 993                 dest--; \
 994             } \
 995         } else { \
 996             dest = 0; \
 997         } \
 998     } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
 999         dest = 0; \
1000     } else if (tmp < 0) { \
1001         dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
1002     } else { \
1003         dest = src1 << tmp; \
1004         if ((dest >> tmp) != src1) { \
1005             SET_QC(); \
1006             dest = (uint32_t)(1 << (sizeof(src1) * 8 - 1)); \
1007             if (src1 > 0) { \
1008                 dest--; \
1009             } \
1010         } \
1011     }} while (0)
1012 NEON_VOP_ENV(qrshl_s8, neon_s8, 4)
1013 NEON_VOP_ENV(qrshl_s16, neon_s16, 2)
1014 #undef NEON_FN
1015
1016 /* The addition of the rounding constant may overflow, so we use an
1017  * intermediate 64 bit accumulator.  */
1018 uint32_t HELPER(neon_qrshl_s32)(CPUARMState *env, uint32_t valop, uint32_t shiftop)
1019 {
1020     int32_t dest;
1021     int32_t val = (int32_t)valop;
1022     int8_t shift = (int8_t)shiftop;
1023     if (shift >= 32) {
1024         if (val) {
1025             SET_QC();
1026             dest = (val >> 31) ^ ~SIGNBIT;
1027         } else {
1028             dest = 0;
1029         }
1030     } else if (shift <= -32) {
1031         dest = 0;
1032     } else if (shift < 0) {
1033         int64_t big_dest = ((int64_t)val + (1 << (-1 - shift)));
1034         dest = big_dest >> -shift;
1035     } else {
1036         dest = val << shift;
1037         if ((dest >> shift) != val) {
1038             SET_QC();
1039             dest = (val >> 31) ^ ~SIGNBIT;
1040         }
1041     }
1042     return dest;
1043 }
1044
1045 /* Handling addition overflow with 64 bit input values is more
1046  * tricky than with 32 bit values.  */
1047 uint64_t HELPER(neon_qrshl_s64)(CPUARMState *env, uint64_t valop, uint64_t shiftop)
1048 {
1049     int8_t shift = (uint8_t)shiftop;
1050     int64_t val = valop;
1051
1052     if (shift >= 64) {
1053         if (val) {
1054             SET_QC();
1055             val = (val >> 63) ^ ~SIGNBIT64;
1056         }
1057     } else if (shift <= -64) {
1058         val = 0;
1059     } else if (shift < 0) {
1060         val >>= (-shift - 1);
1061         if (val == INT64_MAX) {
1062             /* In this case, it means that the rounding constant is 1,
1063              * and the addition would overflow. Return the actual
1064              * result directly.  */
1065             val = 0x4000000000000000ULL;
1066         } else {
1067             val++;
1068             val >>= 1;
1069         }
1070     } else {
1071         int64_t tmp = val;
1072         val <<= shift;
1073         if ((val >> shift) != tmp) {
1074             SET_QC();
1075             val = (tmp >> 63) ^ ~SIGNBIT64;
1076         }
1077     }
1078     return val;
1079 }
1080
1081 uint32_t HELPER(neon_add_u8)(uint32_t a, uint32_t b)
1082 {
1083     uint32_t mask;
1084     mask = (a ^ b) & 0x80808080u;
1085     a &= ~0x80808080u;
1086     b &= ~0x80808080u;
1087     return (a + b) ^ mask;
1088 }
1089
1090 uint32_t HELPER(neon_add_u16)(uint32_t a, uint32_t b)
1091 {
1092     uint32_t mask;
1093     mask = (a ^ b) & 0x80008000u;
1094     a &= ~0x80008000u;
1095     b &= ~0x80008000u;
1096     return (a + b) ^ mask;
1097 }
1098
1099 #define NEON_FN(dest, src1, src2) dest = src1 + src2
1100 NEON_POP(padd_u8, neon_u8, 4)
1101 NEON_POP(padd_u16, neon_u16, 2)
1102 #undef NEON_FN
1103
1104 #define NEON_FN(dest, src1, src2) dest = src1 - src2
1105 NEON_VOP(sub_u8, neon_u8, 4)
1106 NEON_VOP(sub_u16, neon_u16, 2)
1107 #undef NEON_FN
1108
1109 #define NEON_FN(dest, src1, src2) dest = src1 * src2
1110 NEON_VOP(mul_u8, neon_u8, 4)
1111 NEON_VOP(mul_u16, neon_u16, 2)
1112 #undef NEON_FN
1113
1114 #define NEON_FN(dest, src1, src2) dest = (src1 & src2) ? -1 : 0
1115 NEON_VOP(tst_u8, neon_u8, 4)
1116 NEON_VOP(tst_u16, neon_u16, 2)
1117 NEON_VOP(tst_u32, neon_u32, 1)
1118 #undef NEON_FN
1119
1120 /* Count Leading Sign/Zero Bits.  */
1121 static inline int do_clz8(uint8_t x)
1122 {
1123     int n;
1124     for (n = 8; x; n--)
1125         x >>= 1;
1126     return n;
1127 }
1128
1129 static inline int do_clz16(uint16_t x)
1130 {
1131     int n;
1132     for (n = 16; x; n--)
1133         x >>= 1;
1134     return n;
1135 }
1136
1137 #define NEON_FN(dest, src, dummy) dest = do_clz8(src)
1138 NEON_VOP1(clz_u8, neon_u8, 4)
1139 #undef NEON_FN
1140
1141 #define NEON_FN(dest, src, dummy) dest = do_clz16(src)
1142 NEON_VOP1(clz_u16, neon_u16, 2)
1143 #undef NEON_FN
1144
1145 #define NEON_FN(dest, src, dummy) dest = do_clz8((src < 0) ? ~src : src) - 1
1146 NEON_VOP1(cls_s8, neon_s8, 4)
1147 #undef NEON_FN
1148
1149 #define NEON_FN(dest, src, dummy) dest = do_clz16((src < 0) ? ~src : src) - 1
1150 NEON_VOP1(cls_s16, neon_s16, 2)
1151 #undef NEON_FN
1152
1153 uint32_t HELPER(neon_cls_s32)(uint32_t x)
1154 {
1155     int count;
1156     if ((int32_t)x < 0)
1157         x = ~x;
1158     for (count = 32; x; count--)
1159         x = x >> 1;
1160     return count - 1;
1161 }
1162
1163 /* Bit count.  */
1164 uint32_t HELPER(neon_cnt_u8)(uint32_t x)
1165 {
1166     x = (x & 0x55555555) + ((x >>  1) & 0x55555555);
1167     x = (x & 0x33333333) + ((x >>  2) & 0x33333333);
1168     x = (x & 0x0f0f0f0f) + ((x >>  4) & 0x0f0f0f0f);
1169     return x;
1170 }
1171
1172 /* Reverse bits in each 8 bit word */
1173 uint32_t HELPER(neon_rbit_u8)(uint32_t x)
1174 {
1175     x =  ((x & 0xf0f0f0f0) >> 4)
1176        | ((x & 0x0f0f0f0f) << 4);
1177     x =  ((x & 0x88888888) >> 3)
1178        | ((x & 0x44444444) >> 1)
1179        | ((x & 0x22222222) << 1)
1180        | ((x & 0x11111111) << 3);
1181     return x;
1182 }
1183
1184 #define NEON_QDMULH16(dest, src1, src2, round) do { \
1185     uint32_t tmp = (int32_t)(int16_t) src1 * (int16_t) src2; \
1186     if ((tmp ^ (tmp << 1)) & SIGNBIT) { \
1187         SET_QC(); \
1188         tmp = (tmp >> 31) ^ ~SIGNBIT; \
1189     } else { \
1190         tmp <<= 1; \
1191     } \
1192     if (round) { \
1193         int32_t old = tmp; \
1194         tmp += 1 << 15; \
1195         if ((int32_t)tmp < old) { \
1196             SET_QC(); \
1197             tmp = SIGNBIT - 1; \
1198         } \
1199     } \
1200     dest = tmp >> 16; \
1201     } while(0)
1202 #define NEON_FN(dest, src1, src2) NEON_QDMULH16(dest, src1, src2, 0)
1203 NEON_VOP_ENV(qdmulh_s16, neon_s16, 2)
1204 #undef NEON_FN
1205 #define NEON_FN(dest, src1, src2) NEON_QDMULH16(dest, src1, src2, 1)
1206 NEON_VOP_ENV(qrdmulh_s16, neon_s16, 2)
1207 #undef NEON_FN
1208 #undef NEON_QDMULH16
1209
1210 #define NEON_QDMULH32(dest, src1, src2, round) do { \
1211     uint64_t tmp = (int64_t)(int32_t) src1 * (int32_t) src2; \
1212     if ((tmp ^ (tmp << 1)) & SIGNBIT64) { \
1213         SET_QC(); \
1214         tmp = (tmp >> 63) ^ ~SIGNBIT64; \
1215     } else { \
1216         tmp <<= 1; \
1217     } \
1218     if (round) { \
1219         int64_t old = tmp; \
1220         tmp += (int64_t)1 << 31; \
1221         if ((int64_t)tmp < old) { \
1222             SET_QC(); \
1223             tmp = SIGNBIT64 - 1; \
1224         } \
1225     } \
1226     dest = tmp >> 32; \
1227     } while(0)
1228 #define NEON_FN(dest, src1, src2) NEON_QDMULH32(dest, src1, src2, 0)
1229 NEON_VOP_ENV(qdmulh_s32, neon_s32, 1)
1230 #undef NEON_FN
1231 #define NEON_FN(dest, src1, src2) NEON_QDMULH32(dest, src1, src2, 1)
1232 NEON_VOP_ENV(qrdmulh_s32, neon_s32, 1)
1233 #undef NEON_FN
1234 #undef NEON_QDMULH32
1235
1236 uint32_t HELPER(neon_narrow_u8)(uint64_t x)
1237 {
1238     return (x & 0xffu) | ((x >> 8) & 0xff00u) | ((x >> 16) & 0xff0000u)
1239            | ((x >> 24) & 0xff000000u);
1240 }
1241
1242 uint32_t HELPER(neon_narrow_u16)(uint64_t x)
1243 {
1244     return (x & 0xffffu) | ((x >> 16) & 0xffff0000u);
1245 }
1246
1247 uint32_t HELPER(neon_narrow_high_u8)(uint64_t x)
1248 {
1249     return ((x >> 8) & 0xff) | ((x >> 16) & 0xff00)
1250             | ((x >> 24) & 0xff0000) | ((x >> 32) & 0xff000000);
1251 }
1252
1253 uint32_t HELPER(neon_narrow_high_u16)(uint64_t x)
1254 {
1255     return ((x >> 16) & 0xffff) | ((x >> 32) & 0xffff0000);
1256 }
1257
1258 uint32_t HELPER(neon_narrow_round_high_u8)(uint64_t x)
1259 {
1260     x &= 0xff80ff80ff80ff80ull;
1261     x += 0x0080008000800080ull;
1262     return ((x >> 8) & 0xff) | ((x >> 16) & 0xff00)
1263             | ((x >> 24) & 0xff0000) | ((x >> 32) & 0xff000000);
1264 }
1265
1266 uint32_t HELPER(neon_narrow_round_high_u16)(uint64_t x)
1267 {
1268     x &= 0xffff8000ffff8000ull;
1269     x += 0x0000800000008000ull;
1270     return ((x >> 16) & 0xffff) | ((x >> 32) & 0xffff0000);
1271 }
1272
1273 uint32_t HELPER(neon_unarrow_sat8)(CPUARMState *env, uint64_t x)
1274 {
1275     uint16_t s;
1276     uint8_t d;
1277     uint32_t res = 0;
1278 #define SAT8(n) \
1279     s = x >> n; \
1280     if (s & 0x8000) { \
1281         SET_QC(); \
1282     } else { \
1283         if (s > 0xff) { \
1284             d = 0xff; \
1285             SET_QC(); \
1286         } else  { \
1287             d = s; \
1288         } \
1289         res |= (uint32_t)d << (n / 2); \
1290     }
1291
1292     SAT8(0);
1293     SAT8(16);
1294     SAT8(32);
1295     SAT8(48);
1296 #undef SAT8
1297     return res;
1298 }
1299
1300 uint32_t HELPER(neon_narrow_sat_u8)(CPUARMState *env, uint64_t x)
1301 {
1302     uint16_t s;
1303     uint8_t d;
1304     uint32_t res = 0;
1305 #define SAT8(n) \
1306     s = x >> n; \
1307     if (s > 0xff) { \
1308         d = 0xff; \
1309         SET_QC(); \
1310     } else  { \
1311         d = s; \
1312     } \
1313     res |= (uint32_t)d << (n / 2);
1314
1315     SAT8(0);
1316     SAT8(16);
1317     SAT8(32);
1318     SAT8(48);
1319 #undef SAT8
1320     return res;
1321 }
1322
1323 uint32_t HELPER(neon_narrow_sat_s8)(CPUARMState *env, uint64_t x)
1324 {
1325     int16_t s;
1326     uint8_t d;
1327     uint32_t res = 0;
1328 #define SAT8(n) \
1329     s = x >> n; \
1330     if (s != (int8_t)s) { \
1331         d = (s >> 15) ^ 0x7f; \
1332         SET_QC(); \
1333     } else  { \
1334         d = s; \
1335     } \
1336     res |= (uint32_t)d << (n / 2);
1337
1338     SAT8(0);
1339     SAT8(16);
1340     SAT8(32);
1341     SAT8(48);
1342 #undef SAT8
1343     return res;
1344 }
1345
1346 uint32_t HELPER(neon_unarrow_sat16)(CPUARMState *env, uint64_t x)
1347 {
1348     uint32_t high;
1349     uint32_t low;
1350     low = x;
1351     if (low & 0x80000000) {
1352         low = 0;
1353         SET_QC();
1354     } else if (low > 0xffff) {
1355         low = 0xffff;
1356         SET_QC();
1357     }
1358     high = x >> 32;
1359     if (high & 0x80000000) {
1360         high = 0;
1361         SET_QC();
1362     } else if (high > 0xffff) {
1363         high = 0xffff;
1364         SET_QC();
1365     }
1366     return low | (high << 16);
1367 }
1368
1369 uint32_t HELPER(neon_narrow_sat_u16)(CPUARMState *env, uint64_t x)
1370 {
1371     uint32_t high;
1372     uint32_t low;
1373     low = x;
1374     if (low > 0xffff) {
1375         low = 0xffff;
1376         SET_QC();
1377     }
1378     high = x >> 32;
1379     if (high > 0xffff) {
1380         high = 0xffff;
1381         SET_QC();
1382     }
1383     return low | (high << 16);
1384 }
1385
1386 uint32_t HELPER(neon_narrow_sat_s16)(CPUARMState *env, uint64_t x)
1387 {
1388     int32_t low;
1389     int32_t high;
1390     low = x;
1391     if (low != (int16_t)low) {
1392         low = (low >> 31) ^ 0x7fff;
1393         SET_QC();
1394     }
1395     high = x >> 32;
1396     if (high != (int16_t)high) {
1397         high = (high >> 31) ^ 0x7fff;
1398         SET_QC();
1399     }
1400     return (uint16_t)low | (high << 16);
1401 }
1402
1403 uint32_t HELPER(neon_unarrow_sat32)(CPUARMState *env, uint64_t x)
1404 {
1405     if (x & 0x8000000000000000ull) {
1406         SET_QC();
1407         return 0;
1408     }
1409     if (x > 0xffffffffu) {
1410         SET_QC();
1411         return 0xffffffffu;
1412     }
1413     return x;
1414 }
1415
1416 uint32_t HELPER(neon_narrow_sat_u32)(CPUARMState *env, uint64_t x)
1417 {
1418     if (x > 0xffffffffu) {
1419         SET_QC();
1420         return 0xffffffffu;
1421     }
1422     return x;
1423 }
1424
1425 uint32_t HELPER(neon_narrow_sat_s32)(CPUARMState *env, uint64_t x)
1426 {
1427     if ((int64_t)x != (int32_t)x) {
1428         SET_QC();
1429         return ((int64_t)x >> 63) ^ 0x7fffffff;
1430     }
1431     return x;
1432 }
1433
1434 uint64_t HELPER(neon_widen_u8)(uint32_t x)
1435 {
1436     uint64_t tmp;
1437     uint64_t ret;
1438     ret = (uint8_t)x;
1439     tmp = (uint8_t)(x >> 8);
1440     ret |= tmp << 16;
1441     tmp = (uint8_t)(x >> 16);
1442     ret |= tmp << 32;
1443     tmp = (uint8_t)(x >> 24);
1444     ret |= tmp << 48;
1445     return ret;
1446 }
1447
1448 uint64_t HELPER(neon_widen_s8)(uint32_t x)
1449 {
1450     uint64_t tmp;
1451     uint64_t ret;
1452     ret = (uint16_t)(int8_t)x;
1453     tmp = (uint16_t)(int8_t)(x >> 8);
1454     ret |= tmp << 16;
1455     tmp = (uint16_t)(int8_t)(x >> 16);
1456     ret |= tmp << 32;
1457     tmp = (uint16_t)(int8_t)(x >> 24);
1458     ret |= tmp << 48;
1459     return ret;
1460 }
1461
1462 uint64_t HELPER(neon_widen_u16)(uint32_t x)
1463 {
1464     uint64_t high = (uint16_t)(x >> 16);
1465     return ((uint16_t)x) | (high << 32);
1466 }
1467
1468 uint64_t HELPER(neon_widen_s16)(uint32_t x)
1469 {
1470     uint64_t high = (int16_t)(x >> 16);
1471     return ((uint32_t)(int16_t)x) | (high << 32);
1472 }
1473
1474 uint64_t HELPER(neon_addl_u16)(uint64_t a, uint64_t b)
1475 {
1476     uint64_t mask;
1477     mask = (a ^ b) & 0x8000800080008000ull;
1478     a &= ~0x8000800080008000ull;
1479     b &= ~0x8000800080008000ull;
1480     return (a + b) ^ mask;
1481 }
1482
1483 uint64_t HELPER(neon_addl_u32)(uint64_t a, uint64_t b)
1484 {
1485     uint64_t mask;
1486     mask = (a ^ b) & 0x8000000080000000ull;
1487     a &= ~0x8000000080000000ull;
1488     b &= ~0x8000000080000000ull;
1489     return (a + b) ^ mask;
1490 }
1491
1492 uint64_t HELPER(neon_paddl_u16)(uint64_t a, uint64_t b)
1493 {
1494     uint64_t tmp;
1495     uint64_t tmp2;
1496
1497     tmp = a & 0x0000ffff0000ffffull;
1498     tmp += (a >> 16) & 0x0000ffff0000ffffull;
1499     tmp2 = b & 0xffff0000ffff0000ull;
1500     tmp2 += (b << 16) & 0xffff0000ffff0000ull;
1501     return    ( tmp         & 0xffff)
1502             | ((tmp  >> 16) & 0xffff0000ull)
1503             | ((tmp2 << 16) & 0xffff00000000ull)
1504             | ( tmp2        & 0xffff000000000000ull);
1505 }
1506
1507 uint64_t HELPER(neon_paddl_u32)(uint64_t a, uint64_t b)
1508 {
1509     uint32_t low = a + (a >> 32);
1510     uint32_t high = b + (b >> 32);
1511     return low + ((uint64_t)high << 32);
1512 }
1513
1514 uint64_t HELPER(neon_subl_u16)(uint64_t a, uint64_t b)
1515 {
1516     uint64_t mask;
1517     mask = (a ^ ~b) & 0x8000800080008000ull;
1518     a |= 0x8000800080008000ull;
1519     b &= ~0x8000800080008000ull;
1520     return (a - b) ^ mask;
1521 }
1522
1523 uint64_t HELPER(neon_subl_u32)(uint64_t a, uint64_t b)
1524 {
1525     uint64_t mask;
1526     mask = (a ^ ~b) & 0x8000000080000000ull;
1527     a |= 0x8000000080000000ull;
1528     b &= ~0x8000000080000000ull;
1529     return (a - b) ^ mask;
1530 }
1531
1532 uint64_t HELPER(neon_addl_saturate_s32)(CPUARMState *env, uint64_t a, uint64_t b)
1533 {
1534     uint32_t x, y;
1535     uint32_t low, high;
1536
1537     x = a;
1538     y = b;
1539     low = x + y;
1540     if (((low ^ x) & SIGNBIT) && !((x ^ y) & SIGNBIT)) {
1541         SET_QC();
1542         low = ((int32_t)x >> 31) ^ ~SIGNBIT;
1543     }
1544     x = a >> 32;
1545     y = b >> 32;
1546     high = x + y;
1547     if (((high ^ x) & SIGNBIT) && !((x ^ y) & SIGNBIT)) {
1548         SET_QC();
1549         high = ((int32_t)x >> 31) ^ ~SIGNBIT;
1550     }
1551     return low | ((uint64_t)high << 32);
1552 }
1553
1554 uint64_t HELPER(neon_addl_saturate_s64)(CPUARMState *env, uint64_t a, uint64_t b)
1555 {
1556     uint64_t result;
1557
1558     result = a + b;
1559     if (((result ^ a) & SIGNBIT64) && !((a ^ b) & SIGNBIT64)) {
1560         SET_QC();
1561         result = ((int64_t)a >> 63) ^ ~SIGNBIT64;
1562     }
1563     return result;
1564 }
1565
1566 /* We have to do the arithmetic in a larger type than
1567  * the input type, because for example with a signed 32 bit
1568  * op the absolute difference can overflow a signed 32 bit value.
1569  */
1570 #define DO_ABD(dest, x, y, intype, arithtype) do {            \
1571     arithtype tmp_x = (intype)(x);                            \
1572     arithtype tmp_y = (intype)(y);                            \
1573     dest = ((tmp_x > tmp_y) ? tmp_x - tmp_y : tmp_y - tmp_x); \
1574     } while(0)
1575
1576 uint64_t HELPER(neon_abdl_u16)(uint32_t a, uint32_t b)
1577 {
1578     uint64_t tmp;
1579     uint64_t result;
1580     DO_ABD(result, a, b, uint8_t, uint32_t);
1581     DO_ABD(tmp, a >> 8, b >> 8, uint8_t, uint32_t);
1582     result |= tmp << 16;
1583     DO_ABD(tmp, a >> 16, b >> 16, uint8_t, uint32_t);
1584     result |= tmp << 32;
1585     DO_ABD(tmp, a >> 24, b >> 24, uint8_t, uint32_t);
1586     result |= tmp << 48;
1587     return result;
1588 }
1589
1590 uint64_t HELPER(neon_abdl_s16)(uint32_t a, uint32_t b)
1591 {
1592     uint64_t tmp;
1593     uint64_t result;
1594     DO_ABD(result, a, b, int8_t, int32_t);
1595     DO_ABD(tmp, a >> 8, b >> 8, int8_t, int32_t);
1596     result |= tmp << 16;
1597     DO_ABD(tmp, a >> 16, b >> 16, int8_t, int32_t);
1598     result |= tmp << 32;
1599     DO_ABD(tmp, a >> 24, b >> 24, int8_t, int32_t);
1600     result |= tmp << 48;
1601     return result;
1602 }
1603
1604 uint64_t HELPER(neon_abdl_u32)(uint32_t a, uint32_t b)
1605 {
1606     uint64_t tmp;
1607     uint64_t result;
1608     DO_ABD(result, a, b, uint16_t, uint32_t);
1609     DO_ABD(tmp, a >> 16, b >> 16, uint16_t, uint32_t);
1610     return result | (tmp << 32);
1611 }
1612
1613 uint64_t HELPER(neon_abdl_s32)(uint32_t a, uint32_t b)
1614 {
1615     uint64_t tmp;
1616     uint64_t result;
1617     DO_ABD(result, a, b, int16_t, int32_t);
1618     DO_ABD(tmp, a >> 16, b >> 16, int16_t, int32_t);
1619     return result | (tmp << 32);
1620 }
1621
1622 uint64_t HELPER(neon_abdl_u64)(uint32_t a, uint32_t b)
1623 {
1624     uint64_t result;
1625     DO_ABD(result, a, b, uint32_t, uint64_t);
1626     return result;
1627 }
1628
1629 uint64_t HELPER(neon_abdl_s64)(uint32_t a, uint32_t b)
1630 {
1631     uint64_t result;
1632     DO_ABD(result, a, b, int32_t, int64_t);
1633     return result;
1634 }
1635 #undef DO_ABD
1636
1637 /* Widening multiply. Named type is the source type.  */
1638 #define DO_MULL(dest, x, y, type1, type2) do { \
1639     type1 tmp_x = x; \
1640     type1 tmp_y = y; \
1641     dest = (type2)((type2)tmp_x * (type2)tmp_y); \
1642     } while(0)
1643
1644 uint64_t HELPER(neon_mull_u8)(uint32_t a, uint32_t b)
1645 {
1646     uint64_t tmp;
1647     uint64_t result;
1648
1649     DO_MULL(result, a, b, uint8_t, uint16_t);
1650     DO_MULL(tmp, a >> 8, b >> 8, uint8_t, uint16_t);
1651     result |= tmp << 16;
1652     DO_MULL(tmp, a >> 16, b >> 16, uint8_t, uint16_t);
1653     result |= tmp << 32;
1654     DO_MULL(tmp, a >> 24, b >> 24, uint8_t, uint16_t);
1655     result |= tmp << 48;
1656     return result;
1657 }
1658
1659 uint64_t HELPER(neon_mull_s8)(uint32_t a, uint32_t b)
1660 {
1661     uint64_t tmp;
1662     uint64_t result;
1663
1664     DO_MULL(result, a, b, int8_t, uint16_t);
1665     DO_MULL(tmp, a >> 8, b >> 8, int8_t, uint16_t);
1666     result |= tmp << 16;
1667     DO_MULL(tmp, a >> 16, b >> 16, int8_t, uint16_t);
1668     result |= tmp << 32;
1669     DO_MULL(tmp, a >> 24, b >> 24, int8_t, uint16_t);
1670     result |= tmp << 48;
1671     return result;
1672 }
1673
1674 uint64_t HELPER(neon_mull_u16)(uint32_t a, uint32_t b)
1675 {
1676     uint64_t tmp;
1677     uint64_t result;
1678
1679     DO_MULL(result, a, b, uint16_t, uint32_t);
1680     DO_MULL(tmp, a >> 16, b >> 16, uint16_t, uint32_t);
1681     return result | (tmp << 32);
1682 }
1683
1684 uint64_t HELPER(neon_mull_s16)(uint32_t a, uint32_t b)
1685 {
1686     uint64_t tmp;
1687     uint64_t result;
1688
1689     DO_MULL(result, a, b, int16_t, uint32_t);
1690     DO_MULL(tmp, a >> 16, b >> 16, int16_t, uint32_t);
1691     return result | (tmp << 32);
1692 }
1693
1694 uint64_t HELPER(neon_negl_u16)(uint64_t x)
1695 {
1696     uint16_t tmp;
1697     uint64_t result;
1698     result = (uint16_t)-x;
1699     tmp = -(x >> 16);
1700     result |= (uint64_t)tmp << 16;
1701     tmp = -(x >> 32);
1702     result |= (uint64_t)tmp << 32;
1703     tmp = -(x >> 48);
1704     result |= (uint64_t)tmp << 48;
1705     return result;
1706 }
1707
1708 uint64_t HELPER(neon_negl_u32)(uint64_t x)
1709 {
1710     uint32_t low = -x;
1711     uint32_t high = -(x >> 32);
1712     return low | ((uint64_t)high << 32);
1713 }
1714
1715 /* Saturating sign manipulation.  */
1716 /* ??? Make these use NEON_VOP1 */
1717 #define DO_QABS8(x) do { \
1718     if (x == (int8_t)0x80) { \
1719         x = 0x7f; \
1720         SET_QC(); \
1721     } else if (x < 0) { \
1722         x = -x; \
1723     }} while (0)
1724 uint32_t HELPER(neon_qabs_s8)(CPUARMState *env, uint32_t x)
1725 {
1726     neon_s8 vec;
1727     NEON_UNPACK(neon_s8, vec, x);
1728     DO_QABS8(vec.v1);
1729     DO_QABS8(vec.v2);
1730     DO_QABS8(vec.v3);
1731     DO_QABS8(vec.v4);
1732     NEON_PACK(neon_s8, x, vec);
1733     return x;
1734 }
1735 #undef DO_QABS8
1736
1737 #define DO_QNEG8(x) do { \
1738     if (x == (int8_t)0x80) { \
1739         x = 0x7f; \
1740         SET_QC(); \
1741     } else { \
1742         x = -x; \
1743     }} while (0)
1744 uint32_t HELPER(neon_qneg_s8)(CPUARMState *env, uint32_t x)
1745 {
1746     neon_s8 vec;
1747     NEON_UNPACK(neon_s8, vec, x);
1748     DO_QNEG8(vec.v1);
1749     DO_QNEG8(vec.v2);
1750     DO_QNEG8(vec.v3);
1751     DO_QNEG8(vec.v4);
1752     NEON_PACK(neon_s8, x, vec);
1753     return x;
1754 }
1755 #undef DO_QNEG8
1756
1757 #define DO_QABS16(x) do { \
1758     if (x == (int16_t)0x8000) { \
1759         x = 0x7fff; \
1760         SET_QC(); \
1761     } else if (x < 0) { \
1762         x = -x; \
1763     }} while (0)
1764 uint32_t HELPER(neon_qabs_s16)(CPUARMState *env, uint32_t x)
1765 {
1766     neon_s16 vec;
1767     NEON_UNPACK(neon_s16, vec, x);
1768     DO_QABS16(vec.v1);
1769     DO_QABS16(vec.v2);
1770     NEON_PACK(neon_s16, x, vec);
1771     return x;
1772 }
1773 #undef DO_QABS16
1774
1775 #define DO_QNEG16(x) do { \
1776     if (x == (int16_t)0x8000) { \
1777         x = 0x7fff; \
1778         SET_QC(); \
1779     } else { \
1780         x = -x; \
1781     }} while (0)
1782 uint32_t HELPER(neon_qneg_s16)(CPUARMState *env, uint32_t x)
1783 {
1784     neon_s16 vec;
1785     NEON_UNPACK(neon_s16, vec, x);
1786     DO_QNEG16(vec.v1);
1787     DO_QNEG16(vec.v2);
1788     NEON_PACK(neon_s16, x, vec);
1789     return x;
1790 }
1791 #undef DO_QNEG16
1792
1793 uint32_t HELPER(neon_qabs_s32)(CPUARMState *env, uint32_t x)
1794 {
1795     if (x == SIGNBIT) {
1796         SET_QC();
1797         x = ~SIGNBIT;
1798     } else if ((int32_t)x < 0) {
1799         x = -x;
1800     }
1801     return x;
1802 }
1803
1804 uint32_t HELPER(neon_qneg_s32)(CPUARMState *env, uint32_t x)
1805 {
1806     if (x == SIGNBIT) {
1807         SET_QC();
1808         x = ~SIGNBIT;
1809     } else {
1810         x = -x;
1811     }
1812     return x;
1813 }
1814
1815 uint64_t HELPER(neon_qabs_s64)(CPUARMState *env, uint64_t x)
1816 {
1817     if (x == SIGNBIT64) {
1818         SET_QC();
1819         x = ~SIGNBIT64;
1820     } else if ((int64_t)x < 0) {
1821         x = -x;
1822     }
1823     return x;
1824 }
1825
1826 uint64_t HELPER(neon_qneg_s64)(CPUARMState *env, uint64_t x)
1827 {
1828     if (x == SIGNBIT64) {
1829         SET_QC();
1830         x = ~SIGNBIT64;
1831     } else {
1832         x = -x;
1833     }
1834     return x;
1835 }
1836
1837 /* NEON Float helpers.  */
1838 uint32_t HELPER(neon_abd_f32)(uint32_t a, uint32_t b, void *fpstp)
1839 {
1840     float_status *fpst = fpstp;
1841     float32 f0 = make_float32(a);
1842     float32 f1 = make_float32(b);
1843     return float32_val(float32_abs(float32_sub(f0, f1, fpst)));
1844 }
1845
1846 /* Floating point comparisons produce an integer result.
1847  * Note that EQ doesn't signal InvalidOp for QNaNs but GE and GT do.
1848  * Softfloat routines return 0/1, which we convert to the 0/-1 Neon requires.
1849  */
1850 uint32_t HELPER(neon_ceq_f32)(uint32_t a, uint32_t b, void *fpstp)
1851 {
1852     float_status *fpst = fpstp;
1853     return -float32_eq_quiet(make_float32(a), make_float32(b), fpst);
1854 }
1855
1856 uint32_t HELPER(neon_cge_f32)(uint32_t a, uint32_t b, void *fpstp)
1857 {
1858     float_status *fpst = fpstp;
1859     return -float32_le(make_float32(b), make_float32(a), fpst);
1860 }
1861
1862 uint32_t HELPER(neon_cgt_f32)(uint32_t a, uint32_t b, void *fpstp)
1863 {
1864     float_status *fpst = fpstp;
1865     return -float32_lt(make_float32(b), make_float32(a), fpst);
1866 }
1867
1868 uint32_t HELPER(neon_acge_f32)(uint32_t a, uint32_t b, void *fpstp)
1869 {
1870     float_status *fpst = fpstp;
1871     float32 f0 = float32_abs(make_float32(a));
1872     float32 f1 = float32_abs(make_float32(b));
1873     return -float32_le(f1, f0, fpst);
1874 }
1875
1876 uint32_t HELPER(neon_acgt_f32)(uint32_t a, uint32_t b, void *fpstp)
1877 {
1878     float_status *fpst = fpstp;
1879     float32 f0 = float32_abs(make_float32(a));
1880     float32 f1 = float32_abs(make_float32(b));
1881     return -float32_lt(f1, f0, fpst);
1882 }
1883
1884 uint64_t HELPER(neon_acge_f64)(uint64_t a, uint64_t b, void *fpstp)
1885 {
1886     float_status *fpst = fpstp;
1887     float64 f0 = float64_abs(make_float64(a));
1888     float64 f1 = float64_abs(make_float64(b));
1889     return -float64_le(f1, f0, fpst);
1890 }
1891
1892 uint64_t HELPER(neon_acgt_f64)(uint64_t a, uint64_t b, void *fpstp)
1893 {
1894     float_status *fpst = fpstp;
1895     float64 f0 = float64_abs(make_float64(a));
1896     float64 f1 = float64_abs(make_float64(b));
1897     return -float64_lt(f1, f0, fpst);
1898 }
1899
1900 #define ELEM(V, N, SIZE) (((V) >> ((N) * (SIZE))) & ((1ull << (SIZE)) - 1))
1901
1902 void HELPER(neon_qunzip8)(void *vd, void *vm)
1903 {
1904     uint64_t *rd = vd, *rm = vm;
1905     uint64_t zd0 = rd[0], zd1 = rd[1];
1906     uint64_t zm0 = rm[0], zm1 = rm[1];
1907
1908     uint64_t d0 = ELEM(zd0, 0, 8) | (ELEM(zd0, 2, 8) << 8)
1909         | (ELEM(zd0, 4, 8) << 16) | (ELEM(zd0, 6, 8) << 24)
1910         | (ELEM(zd1, 0, 8) << 32) | (ELEM(zd1, 2, 8) << 40)
1911         | (ELEM(zd1, 4, 8) << 48) | (ELEM(zd1, 6, 8) << 56);
1912     uint64_t d1 = ELEM(zm0, 0, 8) | (ELEM(zm0, 2, 8) << 8)
1913         | (ELEM(zm0, 4, 8) << 16) | (ELEM(zm0, 6, 8) << 24)
1914         | (ELEM(zm1, 0, 8) << 32) | (ELEM(zm1, 2, 8) << 40)
1915         | (ELEM(zm1, 4, 8) << 48) | (ELEM(zm1, 6, 8) << 56);
1916     uint64_t m0 = ELEM(zd0, 1, 8) | (ELEM(zd0, 3, 8) << 8)
1917         | (ELEM(zd0, 5, 8) << 16) | (ELEM(zd0, 7, 8) << 24)
1918         | (ELEM(zd1, 1, 8) << 32) | (ELEM(zd1, 3, 8) << 40)
1919         | (ELEM(zd1, 5, 8) << 48) | (ELEM(zd1, 7, 8) << 56);
1920     uint64_t m1 = ELEM(zm0, 1, 8) | (ELEM(zm0, 3, 8) << 8)
1921         | (ELEM(zm0, 5, 8) << 16) | (ELEM(zm0, 7, 8) << 24)
1922         | (ELEM(zm1, 1, 8) << 32) | (ELEM(zm1, 3, 8) << 40)
1923         | (ELEM(zm1, 5, 8) << 48) | (ELEM(zm1, 7, 8) << 56);
1924
1925     rm[0] = m0;
1926     rm[1] = m1;
1927     rd[0] = d0;
1928     rd[1] = d1;
1929 }
1930
1931 void HELPER(neon_qunzip16)(void *vd, void *vm)
1932 {
1933     uint64_t *rd = vd, *rm = vm;
1934     uint64_t zd0 = rd[0], zd1 = rd[1];
1935     uint64_t zm0 = rm[0], zm1 = rm[1];
1936
1937     uint64_t d0 = ELEM(zd0, 0, 16) | (ELEM(zd0, 2, 16) << 16)
1938         | (ELEM(zd1, 0, 16) << 32) | (ELEM(zd1, 2, 16) << 48);
1939     uint64_t d1 = ELEM(zm0, 0, 16) | (ELEM(zm0, 2, 16) << 16)
1940         | (ELEM(zm1, 0, 16) << 32) | (ELEM(zm1, 2, 16) << 48);
1941     uint64_t m0 = ELEM(zd0, 1, 16) | (ELEM(zd0, 3, 16) << 16)
1942         | (ELEM(zd1, 1, 16) << 32) | (ELEM(zd1, 3, 16) << 48);
1943     uint64_t m1 = ELEM(zm0, 1, 16) | (ELEM(zm0, 3, 16) << 16)
1944         | (ELEM(zm1, 1, 16) << 32) | (ELEM(zm1, 3, 16) << 48);
1945
1946     rm[0] = m0;
1947     rm[1] = m1;
1948     rd[0] = d0;
1949     rd[1] = d1;
1950 }
1951
1952 void HELPER(neon_qunzip32)(void *vd, void *vm)
1953 {
1954     uint64_t *rd = vd, *rm = vm;
1955     uint64_t zd0 = rd[0], zd1 = rd[1];
1956     uint64_t zm0 = rm[0], zm1 = rm[1];
1957
1958     uint64_t d0 = ELEM(zd0, 0, 32) | (ELEM(zd1, 0, 32) << 32);
1959     uint64_t d1 = ELEM(zm0, 0, 32) | (ELEM(zm1, 0, 32) << 32);
1960     uint64_t m0 = ELEM(zd0, 1, 32) | (ELEM(zd1, 1, 32) << 32);
1961     uint64_t m1 = ELEM(zm0, 1, 32) | (ELEM(zm1, 1, 32) << 32);
1962
1963     rm[0] = m0;
1964     rm[1] = m1;
1965     rd[0] = d0;
1966     rd[1] = d1;
1967 }
1968
1969 void HELPER(neon_unzip8)(void *vd, void *vm)
1970 {
1971     uint64_t *rd = vd, *rm = vm;
1972     uint64_t zd = rd[0], zm = rm[0];
1973
1974     uint64_t d0 = ELEM(zd, 0, 8) | (ELEM(zd, 2, 8) << 8)
1975         | (ELEM(zd, 4, 8) << 16) | (ELEM(zd, 6, 8) << 24)
1976         | (ELEM(zm, 0, 8) << 32) | (ELEM(zm, 2, 8) << 40)
1977         | (ELEM(zm, 4, 8) << 48) | (ELEM(zm, 6, 8) << 56);
1978     uint64_t m0 = ELEM(zd, 1, 8) | (ELEM(zd, 3, 8) << 8)
1979         | (ELEM(zd, 5, 8) << 16) | (ELEM(zd, 7, 8) << 24)
1980         | (ELEM(zm, 1, 8) << 32) | (ELEM(zm, 3, 8) << 40)
1981         | (ELEM(zm, 5, 8) << 48) | (ELEM(zm, 7, 8) << 56);
1982
1983     rm[0] = m0;
1984     rd[0] = d0;
1985 }
1986
1987 void HELPER(neon_unzip16)(void *vd, void *vm)
1988 {
1989     uint64_t *rd = vd, *rm = vm;
1990     uint64_t zd = rd[0], zm = rm[0];
1991
1992     uint64_t d0 = ELEM(zd, 0, 16) | (ELEM(zd, 2, 16) << 16)
1993         | (ELEM(zm, 0, 16) << 32) | (ELEM(zm, 2, 16) << 48);
1994     uint64_t m0 = ELEM(zd, 1, 16) | (ELEM(zd, 3, 16) << 16)
1995         | (ELEM(zm, 1, 16) << 32) | (ELEM(zm, 3, 16) << 48);
1996
1997     rm[0] = m0;
1998     rd[0] = d0;
1999 }
2000
2001 void HELPER(neon_qzip8)(void *vd, void *vm)
2002 {
2003     uint64_t *rd = vd, *rm = vm;
2004     uint64_t zd0 = rd[0], zd1 = rd[1];
2005     uint64_t zm0 = rm[0], zm1 = rm[1];
2006
2007     uint64_t d0 = ELEM(zd0, 0, 8) | (ELEM(zm0, 0, 8) << 8)
2008         | (ELEM(zd0, 1, 8) << 16) | (ELEM(zm0, 1, 8) << 24)
2009         | (ELEM(zd0, 2, 8) << 32) | (ELEM(zm0, 2, 8) << 40)
2010         | (ELEM(zd0, 3, 8) << 48) | (ELEM(zm0, 3, 8) << 56);
2011     uint64_t d1 = ELEM(zd0, 4, 8) | (ELEM(zm0, 4, 8) << 8)
2012         | (ELEM(zd0, 5, 8) << 16) | (ELEM(zm0, 5, 8) << 24)
2013         | (ELEM(zd0, 6, 8) << 32) | (ELEM(zm0, 6, 8) << 40)
2014         | (ELEM(zd0, 7, 8) << 48) | (ELEM(zm0, 7, 8) << 56);
2015     uint64_t m0 = ELEM(zd1, 0, 8) | (ELEM(zm1, 0, 8) << 8)
2016         | (ELEM(zd1, 1, 8) << 16) | (ELEM(zm1, 1, 8) << 24)
2017         | (ELEM(zd1, 2, 8) << 32) | (ELEM(zm1, 2, 8) << 40)
2018         | (ELEM(zd1, 3, 8) << 48) | (ELEM(zm1, 3, 8) << 56);
2019     uint64_t m1 = ELEM(zd1, 4, 8) | (ELEM(zm1, 4, 8) << 8)
2020         | (ELEM(zd1, 5, 8) << 16) | (ELEM(zm1, 5, 8) << 24)
2021         | (ELEM(zd1, 6, 8) << 32) | (ELEM(zm1, 6, 8) << 40)
2022         | (ELEM(zd1, 7, 8) << 48) | (ELEM(zm1, 7, 8) << 56);
2023
2024     rm[0] = m0;
2025     rm[1] = m1;
2026     rd[0] = d0;
2027     rd[1] = d1;
2028 }
2029
2030 void HELPER(neon_qzip16)(void *vd, void *vm)
2031 {
2032     uint64_t *rd = vd, *rm = vm;
2033     uint64_t zd0 = rd[0], zd1 = rd[1];
2034     uint64_t zm0 = rm[0], zm1 = rm[1];
2035
2036     uint64_t d0 = ELEM(zd0, 0, 16) | (ELEM(zm0, 0, 16) << 16)
2037         | (ELEM(zd0, 1, 16) << 32) | (ELEM(zm0, 1, 16) << 48);
2038     uint64_t d1 = ELEM(zd0, 2, 16) | (ELEM(zm0, 2, 16) << 16)
2039         | (ELEM(zd0, 3, 16) << 32) | (ELEM(zm0, 3, 16) << 48);
2040     uint64_t m0 = ELEM(zd1, 0, 16) | (ELEM(zm1, 0, 16) << 16)
2041         | (ELEM(zd1, 1, 16) << 32) | (ELEM(zm1, 1, 16) << 48);
2042     uint64_t m1 = ELEM(zd1, 2, 16) | (ELEM(zm1, 2, 16) << 16)
2043         | (ELEM(zd1, 3, 16) << 32) | (ELEM(zm1, 3, 16) << 48);
2044
2045     rm[0] = m0;
2046     rm[1] = m1;
2047     rd[0] = d0;
2048     rd[1] = d1;
2049 }
2050
2051 void HELPER(neon_qzip32)(void *vd, void *vm)
2052 {
2053     uint64_t *rd = vd, *rm = vm;
2054     uint64_t zd0 = rd[0], zd1 = rd[1];
2055     uint64_t zm0 = rm[0], zm1 = rm[1];
2056
2057     uint64_t d0 = ELEM(zd0, 0, 32) | (ELEM(zm0, 0, 32) << 32);
2058     uint64_t d1 = ELEM(zd0, 1, 32) | (ELEM(zm0, 1, 32) << 32);
2059     uint64_t m0 = ELEM(zd1, 0, 32) | (ELEM(zm1, 0, 32) << 32);
2060     uint64_t m1 = ELEM(zd1, 1, 32) | (ELEM(zm1, 1, 32) << 32);
2061
2062     rm[0] = m0;
2063     rm[1] = m1;
2064     rd[0] = d0;
2065     rd[1] = d1;
2066 }
2067
2068 void HELPER(neon_zip8)(void *vd, void *vm)
2069 {
2070     uint64_t *rd = vd, *rm = vm;
2071     uint64_t zd = rd[0], zm = rm[0];
2072
2073     uint64_t d0 = ELEM(zd, 0, 8) | (ELEM(zm, 0, 8) << 8)
2074         | (ELEM(zd, 1, 8) << 16) | (ELEM(zm, 1, 8) << 24)
2075         | (ELEM(zd, 2, 8) << 32) | (ELEM(zm, 2, 8) << 40)
2076         | (ELEM(zd, 3, 8) << 48) | (ELEM(zm, 3, 8) << 56);
2077     uint64_t m0 = ELEM(zd, 4, 8) | (ELEM(zm, 4, 8) << 8)
2078         | (ELEM(zd, 5, 8) << 16) | (ELEM(zm, 5, 8) << 24)
2079         | (ELEM(zd, 6, 8) << 32) | (ELEM(zm, 6, 8) << 40)
2080         | (ELEM(zd, 7, 8) << 48) | (ELEM(zm, 7, 8) << 56);
2081
2082     rm[0] = m0;
2083     rd[0] = d0;
2084 }
2085
2086 void HELPER(neon_zip16)(void *vd, void *vm)
2087 {
2088     uint64_t *rd = vd, *rm = vm;
2089     uint64_t zd = rd[0], zm = rm[0];
2090
2091     uint64_t d0 = ELEM(zd, 0, 16) | (ELEM(zm, 0, 16) << 16)
2092         | (ELEM(zd, 1, 16) << 32) | (ELEM(zm, 1, 16) << 48);
2093     uint64_t m0 = ELEM(zd, 2, 16) | (ELEM(zm, 2, 16) << 16)
2094         | (ELEM(zd, 3, 16) << 32) | (ELEM(zm, 3, 16) << 48);
2095
2096     rm[0] = m0;
2097     rd[0] = d0;
2098 }