target/arm/neon_helper.c

   1 /*
   2  * ARM NEON vector operations.
   3  *
   4  * Copyright (c) 2007, 2008 CodeSourcery.
   5  * Written by Paul Brook
   6  *
   7  * This code is licensed under the GNU GPL v2.
   8  */
   9 #include "qemu/osdep.h"
  10
  11 #include "cpu.h"
  12 #include "exec/helper-proto.h"
  13 #include "fpu/softfloat.h"
  14
  15 #define SIGNBIT (uint32_t)0x80000000
  16 #define SIGNBIT64 ((uint64_t)1 << 63)
  17
  18 #define SET_QC() env->vfp.qc[0] = 1
  19
  20 #define NEON_TYPE1(name, type) \
  21 typedef struct \
  22 { \
  23     type v1; \
  24 } neon_##name;
  25 #ifdef HOST_WORDS_BIGENDIAN
  26 #define NEON_TYPE2(name, type) \
  27 typedef struct \
  28 { \
  29     type v2; \
  30     type v1; \
  31 } neon_##name;
  32 #define NEON_TYPE4(name, type) \
  33 typedef struct \
  34 { \
  35     type v4; \
  36     type v3; \
  37     type v2; \
  38     type v1; \
  39 } neon_##name;
  40 #else
  41 #define NEON_TYPE2(name, type) \
  42 typedef struct \
  43 { \
  44     type v1; \
  45     type v2; \
  46 } neon_##name;
  47 #define NEON_TYPE4(name, type) \
  48 typedef struct \
  49 { \
  50     type v1; \
  51     type v2; \
  52     type v3; \
  53     type v4; \
  54 } neon_##name;
  55 #endif
  56
  57 NEON_TYPE4(s8, int8_t)
  58 NEON_TYPE4(u8, uint8_t)
  59 NEON_TYPE2(s16, int16_t)
  60 NEON_TYPE2(u16, uint16_t)
  61 NEON_TYPE1(s32, int32_t)
  62 NEON_TYPE1(u32, uint32_t)
  63 #undef NEON_TYPE4
  64 #undef NEON_TYPE2
  65 #undef NEON_TYPE1
  66
  67 /* Copy from a uint32_t to a vector structure type.  */
  68 #define NEON_UNPACK(vtype, dest, val) do { \
  69     union { \
  70         vtype v; \
  71         uint32_t i; \
  72     } conv_u; \
  73     conv_u.i = (val); \
  74     dest = conv_u.v; \
  75     } while(0)
  76
  77 /* Copy from a vector structure type to a uint32_t.  */
  78 #define NEON_PACK(vtype, dest, val) do { \
  79     union { \
  80         vtype v; \
  81         uint32_t i; \
  82     } conv_u; \
  83     conv_u.v = (val); \
  84     dest = conv_u.i; \
  85     } while(0)
  86
  87 #define NEON_DO1 \
  88     NEON_FN(vdest.v1, vsrc1.v1, vsrc2.v1);
  89 #define NEON_DO2 \
  90     NEON_FN(vdest.v1, vsrc1.v1, vsrc2.v1); \
  91     NEON_FN(vdest.v2, vsrc1.v2, vsrc2.v2);
  92 #define NEON_DO4 \
  93     NEON_FN(vdest.v1, vsrc1.v1, vsrc2.v1); \
  94     NEON_FN(vdest.v2, vsrc1.v2, vsrc2.v2); \
  95     NEON_FN(vdest.v3, vsrc1.v3, vsrc2.v3); \
  96     NEON_FN(vdest.v4, vsrc1.v4, vsrc2.v4);
  97
  98 #define NEON_VOP_BODY(vtype, n) \
  99 { \
 100     uint32_t res; \
 101     vtype vsrc1; \
 102     vtype vsrc2; \
 103     vtype vdest; \
 104     NEON_UNPACK(vtype, vsrc1, arg1); \
 105     NEON_UNPACK(vtype, vsrc2, arg2); \
 106     NEON_DO##n; \
 107     NEON_PACK(vtype, res, vdest); \
 108     return res; \
 109 }
 110
 111 #define NEON_VOP(name, vtype, n) \
 112 uint32_t HELPER(glue(neon_,name))(uint32_t arg1, uint32_t arg2) \
 113 NEON_VOP_BODY(vtype, n)
 114
 115 #define NEON_VOP_ENV(name, vtype, n) \
 116 uint32_t HELPER(glue(neon_,name))(CPUARMState *env, uint32_t arg1, uint32_t arg2) \
 117 NEON_VOP_BODY(vtype, n)
 118
 119 /* Pairwise operations.  */
 120 /* For 32-bit elements each segment only contains a single element, so
 121    the elementwise and pairwise operations are the same.  */
 122 #define NEON_PDO2 \
 123     NEON_FN(vdest.v1, vsrc1.v1, vsrc1.v2); \
 124     NEON_FN(vdest.v2, vsrc2.v1, vsrc2.v2);
 125 #define NEON_PDO4 \
 126     NEON_FN(vdest.v1, vsrc1.v1, vsrc1.v2); \
 127     NEON_FN(vdest.v2, vsrc1.v3, vsrc1.v4); \
 128     NEON_FN(vdest.v3, vsrc2.v1, vsrc2.v2); \
 129     NEON_FN(vdest.v4, vsrc2.v3, vsrc2.v4); \
 130
 131 #define NEON_POP(name, vtype, n) \
 132 uint32_t HELPER(glue(neon_,name))(uint32_t arg1, uint32_t arg2) \
 133 { \
 134     uint32_t res; \
 135     vtype vsrc1; \
 136     vtype vsrc2; \
 137     vtype vdest; \
 138     NEON_UNPACK(vtype, vsrc1, arg1); \
 139     NEON_UNPACK(vtype, vsrc2, arg2); \
 140     NEON_PDO##n; \
 141     NEON_PACK(vtype, res, vdest); \
 142     return res; \
 143 }
 144
 145 /* Unary operators.  */
 146 #define NEON_VOP1(name, vtype, n) \
 147 uint32_t HELPER(glue(neon_,name))(uint32_t arg) \
 148 { \
 149     vtype vsrc1; \
 150     vtype vdest; \
 151     NEON_UNPACK(vtype, vsrc1, arg); \
 152     NEON_DO##n; \
 153     NEON_PACK(vtype, arg, vdest); \
 154     return arg; \
 155 }
 156
 157
 158 #define NEON_USAT(dest, src1, src2, type) do { \
 159     uint32_t tmp = (uint32_t)src1 + (uint32_t)src2; \
 160     if (tmp != (type)tmp) { \
 161         SET_QC(); \
 162         dest = ~0; \
 163     } else { \
 164         dest = tmp; \
 165     }} while(0)
 166 #define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint8_t)
 167 NEON_VOP_ENV(qadd_u8, neon_u8, 4)
 168 #undef NEON_FN
 169 #define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint16_t)
 170 NEON_VOP_ENV(qadd_u16, neon_u16, 2)
 171 #undef NEON_FN
 172 #undef NEON_USAT
 173
 174 uint32_t HELPER(neon_qadd_u32)(CPUARMState *env, uint32_t a, uint32_t b)
 175 {
 176     uint32_t res = a + b;
 177     if (res < a) {
 178         SET_QC();
 179         res = ~0;
 180     }
 181     return res;
 182 }
 183
 184 uint64_t HELPER(neon_qadd_u64)(CPUARMState *env, uint64_t src1, uint64_t src2)
 185 {
 186     uint64_t res;
 187
 188     res = src1 + src2;
 189     if (res < src1) {
 190         SET_QC();
 191         res = ~(uint64_t)0;
 192     }
 193     return res;
 194 }
 195
 196 #define NEON_SSAT(dest, src1, src2, type) do { \
 197     int32_t tmp = (uint32_t)src1 + (uint32_t)src2; \
 198     if (tmp != (type)tmp) { \
 199         SET_QC(); \
 200         if (src2 > 0) { \
 201             tmp = (1 << (sizeof(type) * 8 - 1)) - 1; \
 202         } else { \
 203             tmp = 1 << (sizeof(type) * 8 - 1); \
 204         } \
 205     } \
 206     dest = tmp; \
 207     } while(0)
 208 #define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int8_t)
 209 NEON_VOP_ENV(qadd_s8, neon_s8, 4)
 210 #undef NEON_FN
 211 #define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int16_t)
 212 NEON_VOP_ENV(qadd_s16, neon_s16, 2)
 213 #undef NEON_FN
 214 #undef NEON_SSAT
 215
 216 uint32_t HELPER(neon_qadd_s32)(CPUARMState *env, uint32_t a, uint32_t b)
 217 {
 218     uint32_t res = a + b;
 219     if (((res ^ a) & SIGNBIT) && !((a ^ b) & SIGNBIT)) {
 220         SET_QC();
 221         res = ~(((int32_t)a >> 31) ^ SIGNBIT);
 222     }
 223     return res;
 224 }
 225
 226 uint64_t HELPER(neon_qadd_s64)(CPUARMState *env, uint64_t src1, uint64_t src2)
 227 {
 228     uint64_t res;
 229
 230     res = src1 + src2;
 231     if (((res ^ src1) & SIGNBIT64) && !((src1 ^ src2) & SIGNBIT64)) {
 232         SET_QC();
 233         res = ((int64_t)src1 >> 63) ^ ~SIGNBIT64;
 234     }
 235     return res;
 236 }
 237
 238 /* Unsigned saturating accumulate of signed value
 239  *
 240  * Op1/Rn is treated as signed
 241  * Op2/Rd is treated as unsigned
 242  *
 243  * Explicit casting is used to ensure the correct sign extension of
 244  * inputs. The result is treated as a unsigned value and saturated as such.
 245  *
 246  * We use a macro for the 8/16 bit cases which expects signed integers of va,
 247  * vb, and vr for interim calculation and an unsigned 32 bit result value r.
 248  */
 249
 250 #define USATACC(bits, shift) \
 251     do { \
 252         va = sextract32(a, shift, bits);                                \
 253         vb = extract32(b, shift, bits);                                 \
 254         vr = va + vb;                                                   \
 255         if (vr > UINT##bits##_MAX) {                                    \
 256             SET_QC();                                                   \
 257             vr = UINT##bits##_MAX;                                      \
 258         } else if (vr < 0) {                                            \
 259             SET_QC();                                                   \
 260             vr = 0;                                                     \
 261         }                                                               \
 262         r = deposit32(r, shift, bits, vr);                              \
 263    } while (0)
 264
 265 uint32_t HELPER(neon_uqadd_s8)(CPUARMState *env, uint32_t a, uint32_t b)
 266 {
 267     int16_t va, vb, vr;
 268     uint32_t r = 0;
 269
 270     USATACC(8, 0);
 271     USATACC(8, 8);
 272     USATACC(8, 16);
 273     USATACC(8, 24);
 274     return r;
 275 }
 276
 277 uint32_t HELPER(neon_uqadd_s16)(CPUARMState *env, uint32_t a, uint32_t b)
 278 {
 279     int32_t va, vb, vr;
 280     uint64_t r = 0;
 281
 282     USATACC(16, 0);
 283     USATACC(16, 16);
 284     return r;
 285 }
 286
 287 #undef USATACC
 288
 289 uint32_t HELPER(neon_uqadd_s32)(CPUARMState *env, uint32_t a, uint32_t b)
 290 {
 291     int64_t va = (int32_t)a;
 292     int64_t vb = (uint32_t)b;
 293     int64_t vr = va + vb;
 294     if (vr > UINT32_MAX) {
 295         SET_QC();
 296         vr = UINT32_MAX;
 297     } else if (vr < 0) {
 298         SET_QC();
 299         vr = 0;
 300     }
 301     return vr;
 302 }
 303
 304 uint64_t HELPER(neon_uqadd_s64)(CPUARMState *env, uint64_t a, uint64_t b)
 305 {
 306     uint64_t res;
 307     res = a + b;
 308     /* We only need to look at the pattern of SIGN bits to detect
 309      * +ve/-ve saturation
 310      */
 311     if (~a & b & ~res & SIGNBIT64) {
 312         SET_QC();
 313         res = UINT64_MAX;
 314     } else if (a & ~b & res & SIGNBIT64) {
 315         SET_QC();
 316         res = 0;
 317     }
 318     return res;
 319 }
 320
 321 /* Signed saturating accumulate of unsigned value
 322  *
 323  * Op1/Rn is treated as unsigned
 324  * Op2/Rd is treated as signed
 325  *
 326  * The result is treated as a signed value and saturated as such
 327  *
 328  * We use a macro for the 8/16 bit cases which expects signed integers of va,
 329  * vb, and vr for interim calculation and an unsigned 32 bit result value r.
 330  */
 331
 332 #define SSATACC(bits, shift) \
 333     do { \
 334         va = extract32(a, shift, bits);                                 \
 335         vb = sextract32(b, shift, bits);                                \
 336         vr = va + vb;                                                   \
 337         if (vr > INT##bits##_MAX) {                                     \
 338             SET_QC();                                                   \
 339             vr = INT##bits##_MAX;                                       \
 340         } else if (vr < INT##bits##_MIN) {                              \
 341             SET_QC();                                                   \
 342             vr = INT##bits##_MIN;                                       \
 343         }                                                               \
 344         r = deposit32(r, shift, bits, vr);                              \
 345     } while (0)
 346
 347 uint32_t HELPER(neon_sqadd_u8)(CPUARMState *env, uint32_t a, uint32_t b)
 348 {
 349     int16_t va, vb, vr;
 350     uint32_t r = 0;
 351
 352     SSATACC(8, 0);
 353     SSATACC(8, 8);
 354     SSATACC(8, 16);
 355     SSATACC(8, 24);
 356     return r;
 357 }
 358
 359 uint32_t HELPER(neon_sqadd_u16)(CPUARMState *env, uint32_t a, uint32_t b)
 360 {
 361     int32_t va, vb, vr;
 362     uint32_t r = 0;
 363
 364     SSATACC(16, 0);
 365     SSATACC(16, 16);
 366
 367     return r;
 368 }
 369
 370 #undef SSATACC
 371
 372 uint32_t HELPER(neon_sqadd_u32)(CPUARMState *env, uint32_t a, uint32_t b)
 373 {
 374     int64_t res;
 375     int64_t op1 = (uint32_t)a;
 376     int64_t op2 = (int32_t)b;
 377     res = op1 + op2;
 378     if (res > INT32_MAX) {
 379         SET_QC();
 380         res = INT32_MAX;
 381     } else if (res < INT32_MIN) {
 382         SET_QC();
 383         res = INT32_MIN;
 384     }
 385     return res;
 386 }
 387
 388 uint64_t HELPER(neon_sqadd_u64)(CPUARMState *env, uint64_t a, uint64_t b)
 389 {
 390     uint64_t res;
 391     res = a + b;
 392     /* We only need to look at the pattern of SIGN bits to detect an overflow */
 393     if (((a & res)
 394          | (~b & res)
 395          | (a & ~b)) & SIGNBIT64) {
 396         SET_QC();
 397         res = INT64_MAX;
 398     }
 399     return res;
 400 }
 401
 402
 403 #define NEON_USAT(dest, src1, src2, type) do { \
 404     uint32_t tmp = (uint32_t)src1 - (uint32_t)src2; \
 405     if (tmp != (type)tmp) { \
 406         SET_QC(); \
 407         dest = 0; \
 408     } else { \
 409         dest = tmp; \
 410     }} while(0)
 411 #define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint8_t)
 412 NEON_VOP_ENV(qsub_u8, neon_u8, 4)
 413 #undef NEON_FN
 414 #define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint16_t)
 415 NEON_VOP_ENV(qsub_u16, neon_u16, 2)
 416 #undef NEON_FN
 417 #undef NEON_USAT
 418
 419 uint32_t HELPER(neon_qsub_u32)(CPUARMState *env, uint32_t a, uint32_t b)
 420 {
 421     uint32_t res = a - b;
 422     if (res > a) {
 423         SET_QC();
 424         res = 0;
 425     }
 426     return res;
 427 }
 428
 429 uint64_t HELPER(neon_qsub_u64)(CPUARMState *env, uint64_t src1, uint64_t src2)
 430 {
 431     uint64_t res;
 432
 433     if (src1 < src2) {
 434         SET_QC();
 435         res = 0;
 436     } else {
 437         res = src1 - src2;
 438     }
 439     return res;
 440 }
 441
 442 #define NEON_SSAT(dest, src1, src2, type) do { \
 443     int32_t tmp = (uint32_t)src1 - (uint32_t)src2; \
 444     if (tmp != (type)tmp) { \
 445         SET_QC(); \
 446         if (src2 < 0) { \
 447             tmp = (1 << (sizeof(type) * 8 - 1)) - 1; \
 448         } else { \
 449             tmp = 1 << (sizeof(type) * 8 - 1); \
 450         } \
 451     } \
 452     dest = tmp; \
 453     } while(0)
 454 #define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int8_t)
 455 NEON_VOP_ENV(qsub_s8, neon_s8, 4)
 456 #undef NEON_FN
 457 #define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int16_t)
 458 NEON_VOP_ENV(qsub_s16, neon_s16, 2)
 459 #undef NEON_FN
 460 #undef NEON_SSAT
 461
 462 uint32_t HELPER(neon_qsub_s32)(CPUARMState *env, uint32_t a, uint32_t b)
 463 {
 464     uint32_t res = a - b;
 465     if (((res ^ a) & SIGNBIT) && ((a ^ b) & SIGNBIT)) {
 466         SET_QC();
 467         res = ~(((int32_t)a >> 31) ^ SIGNBIT);
 468     }
 469     return res;
 470 }
 471
 472 uint64_t HELPER(neon_qsub_s64)(CPUARMState *env, uint64_t src1, uint64_t src2)
 473 {
 474     uint64_t res;
 475
 476     res = src1 - src2;
 477     if (((res ^ src1) & SIGNBIT64) && ((src1 ^ src2) & SIGNBIT64)) {
 478         SET_QC();
 479         res = ((int64_t)src1 >> 63) ^ ~SIGNBIT64;
 480     }
 481     return res;
 482 }
 483
 484 #define NEON_FN(dest, src1, src2) dest = (src1 + src2) >> 1
 485 NEON_VOP(hadd_s8, neon_s8, 4)
 486 NEON_VOP(hadd_u8, neon_u8, 4)
 487 NEON_VOP(hadd_s16, neon_s16, 2)
 488 NEON_VOP(hadd_u16, neon_u16, 2)
 489 #undef NEON_FN
 490
 491 int32_t HELPER(neon_hadd_s32)(int32_t src1, int32_t src2)
 492 {
 493     int32_t dest;
 494
 495     dest = (src1 >> 1) + (src2 >> 1);
 496     if (src1 & src2 & 1)
 497         dest++;
 498     return dest;
 499 }
 500
 501 uint32_t HELPER(neon_hadd_u32)(uint32_t src1, uint32_t src2)
 502 {
 503     uint32_t dest;
 504
 505     dest = (src1 >> 1) + (src2 >> 1);
 506     if (src1 & src2 & 1)
 507         dest++;
 508     return dest;
 509 }
 510
 511 #define NEON_FN(dest, src1, src2) dest = (src1 + src2 + 1) >> 1
 512 NEON_VOP(rhadd_s8, neon_s8, 4)
 513 NEON_VOP(rhadd_u8, neon_u8, 4)
 514 NEON_VOP(rhadd_s16, neon_s16, 2)
 515 NEON_VOP(rhadd_u16, neon_u16, 2)
 516 #undef NEON_FN
 517
 518 int32_t HELPER(neon_rhadd_s32)(int32_t src1, int32_t src2)
 519 {
 520     int32_t dest;
 521
 522     dest = (src1 >> 1) + (src2 >> 1);
 523     if ((src1 | src2) & 1)
 524         dest++;
 525     return dest;
 526 }
 527
 528 uint32_t HELPER(neon_rhadd_u32)(uint32_t src1, uint32_t src2)
 529 {
 530     uint32_t dest;
 531
 532     dest = (src1 >> 1) + (src2 >> 1);
 533     if ((src1 | src2) & 1)
 534         dest++;
 535     return dest;
 536 }
 537
 538 #define NEON_FN(dest, src1, src2) dest = (src1 - src2) >> 1
 539 NEON_VOP(hsub_s8, neon_s8, 4)
 540 NEON_VOP(hsub_u8, neon_u8, 4)
 541 NEON_VOP(hsub_s16, neon_s16, 2)
 542 NEON_VOP(hsub_u16, neon_u16, 2)
 543 #undef NEON_FN
 544
 545 int32_t HELPER(neon_hsub_s32)(int32_t src1, int32_t src2)
 546 {
 547     int32_t dest;
 548
 549     dest = (src1 >> 1) - (src2 >> 1);
 550     if ((~src1) & src2 & 1)
 551         dest--;
 552     return dest;
 553 }
 554
 555 uint32_t HELPER(neon_hsub_u32)(uint32_t src1, uint32_t src2)
 556 {
 557     uint32_t dest;
 558
 559     dest = (src1 >> 1) - (src2 >> 1);
 560     if ((~src1) & src2 & 1)
 561         dest--;
 562     return dest;
 563 }
 564
 565 #define NEON_FN(dest, src1, src2) dest = (src1 > src2) ? ~0 : 0
 566 NEON_VOP(cgt_s8, neon_s8, 4)
 567 NEON_VOP(cgt_u8, neon_u8, 4)
 568 NEON_VOP(cgt_s16, neon_s16, 2)
 569 NEON_VOP(cgt_u16, neon_u16, 2)
 570 NEON_VOP(cgt_s32, neon_s32, 1)
 571 NEON_VOP(cgt_u32, neon_u32, 1)
 572 #undef NEON_FN
 573
 574 #define NEON_FN(dest, src1, src2) dest = (src1 >= src2) ? ~0 : 0
 575 NEON_VOP(cge_s8, neon_s8, 4)
 576 NEON_VOP(cge_u8, neon_u8, 4)
 577 NEON_VOP(cge_s16, neon_s16, 2)
 578 NEON_VOP(cge_u16, neon_u16, 2)
 579 NEON_VOP(cge_s32, neon_s32, 1)
 580 NEON_VOP(cge_u32, neon_u32, 1)
 581 #undef NEON_FN
 582
 583 #define NEON_FN(dest, src1, src2) dest = (src1 < src2) ? src1 : src2
 584 NEON_POP(pmin_s8, neon_s8, 4)
 585 NEON_POP(pmin_u8, neon_u8, 4)
 586 NEON_POP(pmin_s16, neon_s16, 2)
 587 NEON_POP(pmin_u16, neon_u16, 2)
 588 #undef NEON_FN
 589
 590 #define NEON_FN(dest, src1, src2) dest = (src1 > src2) ? src1 : src2
 591 NEON_POP(pmax_s8, neon_s8, 4)
 592 NEON_POP(pmax_u8, neon_u8, 4)
 593 NEON_POP(pmax_s16, neon_s16, 2)
 594 NEON_POP(pmax_u16, neon_u16, 2)
 595 #undef NEON_FN
 596
 597 #define NEON_FN(dest, src1, src2) \
 598     dest = (src1 > src2) ? (src1 - src2) : (src2 - src1)
 599 NEON_VOP(abd_s8, neon_s8, 4)
 600 NEON_VOP(abd_u8, neon_u8, 4)
 601 NEON_VOP(abd_s16, neon_s16, 2)
 602 NEON_VOP(abd_u16, neon_u16, 2)
 603 NEON_VOP(abd_s32, neon_s32, 1)
 604 NEON_VOP(abd_u32, neon_u32, 1)
 605 #undef NEON_FN
 606
 607 #define NEON_FN(dest, src1, src2) do { \
 608     int8_t tmp; \
 609     tmp = (int8_t)src2; \
 610     if (tmp >= (ssize_t)sizeof(src1) * 8 || \
 611         tmp <= -(ssize_t)sizeof(src1) * 8) { \
 612         dest = 0; \
 613     } else if (tmp < 0) { \
 614         dest = src1 >> -tmp; \
 615     } else { \
 616         dest = src1 << tmp; \
 617     }} while (0)
 618 NEON_VOP(shl_u16, neon_u16, 2)
 619 #undef NEON_FN
 620
 621 #define NEON_FN(dest, src1, src2) do { \
 622     int8_t tmp; \
 623     tmp = (int8_t)src2; \
 624     if (tmp >= (ssize_t)sizeof(src1) * 8) { \
 625         dest = 0; \
 626     } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
 627         dest = src1 >> (sizeof(src1) * 8 - 1); \
 628     } else if (tmp < 0) { \
 629         dest = src1 >> -tmp; \
 630     } else { \
 631         dest = src1 << tmp; \
 632     }} while (0)
 633 NEON_VOP(shl_s16, neon_s16, 2)
 634 #undef NEON_FN
 635
 636 #define NEON_FN(dest, src1, src2) do { \
 637     int8_t tmp; \
 638     tmp = (int8_t)src2; \
 639     if ((tmp >= (ssize_t)sizeof(src1) * 8) \
 640         || (tmp <= -(ssize_t)sizeof(src1) * 8)) { \
 641         dest = 0; \
 642     } else if (tmp < 0) { \
 643         dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
 644     } else { \
 645         dest = src1 << tmp; \
 646     }} while (0)
 647 NEON_VOP(rshl_s8, neon_s8, 4)
 648 NEON_VOP(rshl_s16, neon_s16, 2)
 649 #undef NEON_FN
 650
 651 /* The addition of the rounding constant may overflow, so we use an
 652  * intermediate 64 bit accumulator.  */
 653 uint32_t HELPER(neon_rshl_s32)(uint32_t valop, uint32_t shiftop)
 654 {
 655     int32_t dest;
 656     int32_t val = (int32_t)valop;
 657     int8_t shift = (int8_t)shiftop;
 658     if ((shift >= 32) || (shift <= -32)) {
 659         dest = 0;
 660     } else if (shift < 0) {
 661         int64_t big_dest = ((int64_t)val + (1 << (-1 - shift)));
 662         dest = big_dest >> -shift;
 663     } else {
 664         dest = val << shift;
 665     }
 666     return dest;
 667 }
 668
 669 /* Handling addition overflow with 64 bit input values is more
 670  * tricky than with 32 bit values.  */
 671 uint64_t HELPER(neon_rshl_s64)(uint64_t valop, uint64_t shiftop)
 672 {
 673     int8_t shift = (int8_t)shiftop;
 674     int64_t val = valop;
 675     if ((shift >= 64) || (shift <= -64)) {
 676         val = 0;
 677     } else if (shift < 0) {
 678         val >>= (-shift - 1);
 679         if (val == INT64_MAX) {
 680             /* In this case, it means that the rounding constant is 1,
 681              * and the addition would overflow. Return the actual
 682              * result directly.  */
 683             val = 0x4000000000000000LL;
 684         } else {
 685             val++;
 686             val >>= 1;
 687         }
 688     } else {
 689         val <<= shift;
 690     }
 691     return val;
 692 }
 693
 694 #define NEON_FN(dest, src1, src2) do { \
 695     int8_t tmp; \
 696     tmp = (int8_t)src2; \
 697     if (tmp >= (ssize_t)sizeof(src1) * 8 || \
 698         tmp < -(ssize_t)sizeof(src1) * 8) { \
 699         dest = 0; \
 700     } else if (tmp == -(ssize_t)sizeof(src1) * 8) { \
 701         dest = src1 >> (-tmp - 1); \
 702     } else if (tmp < 0) { \
 703         dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
 704     } else { \
 705         dest = src1 << tmp; \
 706     }} while (0)
 707 NEON_VOP(rshl_u8, neon_u8, 4)
 708 NEON_VOP(rshl_u16, neon_u16, 2)
 709 #undef NEON_FN
 710
 711 /* The addition of the rounding constant may overflow, so we use an
 712  * intermediate 64 bit accumulator.  */
 713 uint32_t HELPER(neon_rshl_u32)(uint32_t val, uint32_t shiftop)
 714 {
 715     uint32_t dest;
 716     int8_t shift = (int8_t)shiftop;
 717     if (shift >= 32 || shift < -32) {
 718         dest = 0;
 719     } else if (shift == -32) {
 720         dest = val >> 31;
 721     } else if (shift < 0) {
 722         uint64_t big_dest = ((uint64_t)val + (1 << (-1 - shift)));
 723         dest = big_dest >> -shift;
 724     } else {
 725         dest = val << shift;
 726     }
 727     return dest;
 728 }
 729
 730 /* Handling addition overflow with 64 bit input values is more
 731  * tricky than with 32 bit values.  */
 732 uint64_t HELPER(neon_rshl_u64)(uint64_t val, uint64_t shiftop)
 733 {
 734     int8_t shift = (uint8_t)shiftop;
 735     if (shift >= 64 || shift < -64) {
 736         val = 0;
 737     } else if (shift == -64) {
 738         /* Rounding a 1-bit result just preserves that bit.  */
 739         val >>= 63;
 740     } else if (shift < 0) {
 741         val >>= (-shift - 1);
 742         if (val == UINT64_MAX) {
 743             /* In this case, it means that the rounding constant is 1,
 744              * and the addition would overflow. Return the actual
 745              * result directly.  */
 746             val = 0x8000000000000000ULL;
 747         } else {
 748             val++;
 749             val >>= 1;
 750         }
 751     } else {
 752         val <<= shift;
 753     }
 754     return val;
 755 }
 756
 757 #define NEON_FN(dest, src1, src2) do { \
 758     int8_t tmp; \
 759     tmp = (int8_t)src2; \
 760     if (tmp >= (ssize_t)sizeof(src1) * 8) { \
 761         if (src1) { \
 762             SET_QC(); \
 763             dest = ~0; \
 764         } else { \
 765             dest = 0; \
 766         } \
 767     } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
 768         dest = 0; \
 769     } else if (tmp < 0) { \
 770         dest = src1 >> -tmp; \
 771     } else { \
 772         dest = src1 << tmp; \
 773         if ((dest >> tmp) != src1) { \
 774             SET_QC(); \
 775             dest = ~0; \
 776         } \
 777     }} while (0)
 778 NEON_VOP_ENV(qshl_u8, neon_u8, 4)
 779 NEON_VOP_ENV(qshl_u16, neon_u16, 2)
 780 NEON_VOP_ENV(qshl_u32, neon_u32, 1)
 781 #undef NEON_FN
 782
 783 uint64_t HELPER(neon_qshl_u64)(CPUARMState *env, uint64_t val, uint64_t shiftop)
 784 {
 785     int8_t shift = (int8_t)shiftop;
 786     if (shift >= 64) {
 787         if (val) {
 788             val = ~(uint64_t)0;
 789             SET_QC();
 790         }
 791     } else if (shift <= -64) {
 792         val = 0;
 793     } else if (shift < 0) {
 794         val >>= -shift;
 795     } else {
 796         uint64_t tmp = val;
 797         val <<= shift;
 798         if ((val >> shift) != tmp) {
 799             SET_QC();
 800             val = ~(uint64_t)0;
 801         }
 802     }
 803     return val;
 804 }
 805
 806 #define NEON_FN(dest, src1, src2) do { \
 807     int8_t tmp; \
 808     tmp = (int8_t)src2; \
 809     if (tmp >= (ssize_t)sizeof(src1) * 8) { \
 810         if (src1) { \
 811             SET_QC(); \
 812             dest = (uint32_t)(1 << (sizeof(src1) * 8 - 1)); \
 813             if (src1 > 0) { \
 814                 dest--; \
 815             } \
 816         } else { \
 817             dest = src1; \
 818         } \
 819     } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
 820         dest = src1 >> 31; \
 821     } else if (tmp < 0) { \
 822         dest = src1 >> -tmp; \
 823     } else { \
 824         dest = src1 << tmp; \
 825         if ((dest >> tmp) != src1) { \
 826             SET_QC(); \
 827             dest = (uint32_t)(1 << (sizeof(src1) * 8 - 1)); \
 828             if (src1 > 0) { \
 829                 dest--; \
 830             } \
 831         } \
 832     }} while (0)
 833 NEON_VOP_ENV(qshl_s8, neon_s8, 4)
 834 NEON_VOP_ENV(qshl_s16, neon_s16, 2)
 835 NEON_VOP_ENV(qshl_s32, neon_s32, 1)
 836 #undef NEON_FN
 837
 838 uint64_t HELPER(neon_qshl_s64)(CPUARMState *env, uint64_t valop, uint64_t shiftop)
 839 {
 840     int8_t shift = (uint8_t)shiftop;
 841     int64_t val = valop;
 842     if (shift >= 64) {
 843         if (val) {
 844             SET_QC();
 845             val = (val >> 63) ^ ~SIGNBIT64;
 846         }
 847     } else if (shift <= -64) {
 848         val >>= 63;
 849     } else if (shift < 0) {
 850         val >>= -shift;
 851     } else {
 852         int64_t tmp = val;
 853         val <<= shift;
 854         if ((val >> shift) != tmp) {
 855             SET_QC();
 856             val = (tmp >> 63) ^ ~SIGNBIT64;
 857         }
 858     }
 859     return val;
 860 }
 861
 862 #define NEON_FN(dest, src1, src2) do { \
 863     if (src1 & (1 << (sizeof(src1) * 8 - 1))) { \
 864         SET_QC(); \
 865         dest = 0; \
 866     } else { \
 867         int8_t tmp; \
 868         tmp = (int8_t)src2; \
 869         if (tmp >= (ssize_t)sizeof(src1) * 8) { \
 870             if (src1) { \
 871                 SET_QC(); \
 872                 dest = ~0; \
 873             } else { \
 874                 dest = 0; \
 875             } \
 876         } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
 877             dest = 0; \
 878         } else if (tmp < 0) { \
 879             dest = src1 >> -tmp; \
 880         } else { \
 881             dest = src1 << tmp; \
 882             if ((dest >> tmp) != src1) { \
 883                 SET_QC(); \
 884                 dest = ~0; \
 885             } \
 886         } \
 887     }} while (0)
 888 NEON_VOP_ENV(qshlu_s8, neon_u8, 4)
 889 NEON_VOP_ENV(qshlu_s16, neon_u16, 2)
 890 #undef NEON_FN
 891
 892 uint32_t HELPER(neon_qshlu_s32)(CPUARMState *env, uint32_t valop, uint32_t shiftop)
 893 {
 894     if ((int32_t)valop < 0) {
 895         SET_QC();
 896         return 0;
 897     }
 898     return helper_neon_qshl_u32(env, valop, shiftop);
 899 }
 900
 901 uint64_t HELPER(neon_qshlu_s64)(CPUARMState *env, uint64_t valop, uint64_t shiftop)
 902 {
 903     if ((int64_t)valop < 0) {
 904         SET_QC();
 905         return 0;
 906     }
 907     return helper_neon_qshl_u64(env, valop, shiftop);
 908 }
 909
 910 #define NEON_FN(dest, src1, src2) do { \
 911     int8_t tmp; \
 912     tmp = (int8_t)src2; \
 913     if (tmp >= (ssize_t)sizeof(src1) * 8) { \
 914         if (src1) { \
 915             SET_QC(); \
 916             dest = ~0; \
 917         } else { \
 918             dest = 0; \
 919         } \
 920     } else if (tmp < -(ssize_t)sizeof(src1) * 8) { \
 921         dest = 0; \
 922     } else if (tmp == -(ssize_t)sizeof(src1) * 8) { \
 923         dest = src1 >> (sizeof(src1) * 8 - 1); \
 924     } else if (tmp < 0) { \
 925         dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
 926     } else { \
 927         dest = src1 << tmp; \
 928         if ((dest >> tmp) != src1) { \
 929             SET_QC(); \
 930             dest = ~0; \
 931         } \
 932     }} while (0)
 933 NEON_VOP_ENV(qrshl_u8, neon_u8, 4)
 934 NEON_VOP_ENV(qrshl_u16, neon_u16, 2)
 935 #undef NEON_FN
 936
 937 /* The addition of the rounding constant may overflow, so we use an
 938  * intermediate 64 bit accumulator.  */
 939 uint32_t HELPER(neon_qrshl_u32)(CPUARMState *env, uint32_t val, uint32_t shiftop)
 940 {
 941     uint32_t dest;
 942     int8_t shift = (int8_t)shiftop;
 943     if (shift >= 32) {
 944         if (val) {
 945             SET_QC();
 946             dest = ~0;
 947         } else {
 948             dest = 0;
 949         }
 950     } else if (shift < -32) {
 951         dest = 0;
 952     } else if (shift == -32) {
 953         dest = val >> 31;
 954     } else if (shift < 0) {
 955         uint64_t big_dest = ((uint64_t)val + (1 << (-1 - shift)));
 956         dest = big_dest >> -shift;
 957     } else {
 958         dest = val << shift;
 959         if ((dest >> shift) != val) {
 960             SET_QC();
 961             dest = ~0;
 962         }
 963     }
 964     return dest;
 965 }
 966
 967 /* Handling addition overflow with 64 bit input values is more
 968  * tricky than with 32 bit values.  */
 969 uint64_t HELPER(neon_qrshl_u64)(CPUARMState *env, uint64_t val, uint64_t shiftop)
 970 {
 971     int8_t shift = (int8_t)shiftop;
 972     if (shift >= 64) {
 973         if (val) {
 974             SET_QC();
 975             val = ~0;
 976         }
 977     } else if (shift < -64) {
 978         val = 0;
 979     } else if (shift == -64) {
 980         val >>= 63;
 981     } else if (shift < 0) {
 982         val >>= (-shift - 1);
 983         if (val == UINT64_MAX) {
 984             /* In this case, it means that the rounding constant is 1,
 985              * and the addition would overflow. Return the actual
 986              * result directly.  */
 987             val = 0x8000000000000000ULL;
 988         } else {
 989             val++;
 990             val >>= 1;
 991         }
 992     } else { \
 993         uint64_t tmp = val;
 994         val <<= shift;
 995         if ((val >> shift) != tmp) {
 996             SET_QC();
 997             val = ~0;
 998         }
 999     }
1000     return val;
1001 }
1002
1003 #define NEON_FN(dest, src1, src2) do { \
1004     int8_t tmp; \
1005     tmp = (int8_t)src2; \
1006     if (tmp >= (ssize_t)sizeof(src1) * 8) { \
1007         if (src1) { \
1008             SET_QC(); \
1009             dest = (typeof(dest))(1 << (sizeof(src1) * 8 - 1)); \
1010             if (src1 > 0) { \
1011                 dest--; \
1012             } \
1013         } else { \
1014             dest = 0; \
1015         } \
1016     } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
1017         dest = 0; \
1018     } else if (tmp < 0) { \
1019         dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
1020     } else { \
1021         dest = src1 << tmp; \
1022         if ((dest >> tmp) != src1) { \
1023             SET_QC(); \
1024             dest = (uint32_t)(1 << (sizeof(src1) * 8 - 1)); \
1025             if (src1 > 0) { \
1026                 dest--; \
1027             } \
1028         } \
1029     }} while (0)
1030 NEON_VOP_ENV(qrshl_s8, neon_s8, 4)
1031 NEON_VOP_ENV(qrshl_s16, neon_s16, 2)
1032 #undef NEON_FN
1033
1034 /* The addition of the rounding constant may overflow, so we use an
1035  * intermediate 64 bit accumulator.  */
1036 uint32_t HELPER(neon_qrshl_s32)(CPUARMState *env, uint32_t valop, uint32_t shiftop)
1037 {
1038     int32_t dest;
1039     int32_t val = (int32_t)valop;
1040     int8_t shift = (int8_t)shiftop;
1041     if (shift >= 32) {
1042         if (val) {
1043             SET_QC();
1044             dest = (val >> 31) ^ ~SIGNBIT;
1045         } else {
1046             dest = 0;
1047         }
1048     } else if (shift <= -32) {
1049         dest = 0;
1050     } else if (shift < 0) {
1051         int64_t big_dest = ((int64_t)val + (1 << (-1 - shift)));
1052         dest = big_dest >> -shift;
1053     } else {
1054         dest = val << shift;
1055         if ((dest >> shift) != val) {
1056             SET_QC();
1057             dest = (val >> 31) ^ ~SIGNBIT;
1058         }
1059     }
1060     return dest;
1061 }
1062
1063 /* Handling addition overflow with 64 bit input values is more
1064  * tricky than with 32 bit values.  */
1065 uint64_t HELPER(neon_qrshl_s64)(CPUARMState *env, uint64_t valop, uint64_t shiftop)
1066 {
1067     int8_t shift = (uint8_t)shiftop;
1068     int64_t val = valop;
1069
1070     if (shift >= 64) {
1071         if (val) {
1072             SET_QC();
1073             val = (val >> 63) ^ ~SIGNBIT64;
1074         }
1075     } else if (shift <= -64) {
1076         val = 0;
1077     } else if (shift < 0) {
1078         val >>= (-shift - 1);
1079         if (val == INT64_MAX) {
1080             /* In this case, it means that the rounding constant is 1,
1081              * and the addition would overflow. Return the actual
1082              * result directly.  */
1083             val = 0x4000000000000000ULL;
1084         } else {
1085             val++;
1086             val >>= 1;
1087         }
1088     } else {
1089         int64_t tmp = val;
1090         val <<= shift;
1091         if ((val >> shift) != tmp) {
1092             SET_QC();
1093             val = (tmp >> 63) ^ ~SIGNBIT64;
1094         }
1095     }
1096     return val;
1097 }
1098
1099 uint32_t HELPER(neon_add_u8)(uint32_t a, uint32_t b)
1100 {
1101     uint32_t mask;
1102     mask = (a ^ b) & 0x80808080u;
1103     a &= ~0x80808080u;
1104     b &= ~0x80808080u;
1105     return (a + b) ^ mask;
1106 }
1107
1108 uint32_t HELPER(neon_add_u16)(uint32_t a, uint32_t b)
1109 {
1110     uint32_t mask;
1111     mask = (a ^ b) & 0x80008000u;
1112     a &= ~0x80008000u;
1113     b &= ~0x80008000u;
1114     return (a + b) ^ mask;
1115 }
1116
1117 #define NEON_FN(dest, src1, src2) dest = src1 + src2
1118 NEON_POP(padd_u8, neon_u8, 4)
1119 NEON_POP(padd_u16, neon_u16, 2)
1120 #undef NEON_FN
1121
1122 #define NEON_FN(dest, src1, src2) dest = src1 - src2
1123 NEON_VOP(sub_u8, neon_u8, 4)
1124 NEON_VOP(sub_u16, neon_u16, 2)
1125 #undef NEON_FN
1126
1127 #define NEON_FN(dest, src1, src2) dest = src1 * src2
1128 NEON_VOP(mul_u8, neon_u8, 4)
1129 NEON_VOP(mul_u16, neon_u16, 2)
1130 #undef NEON_FN
1131
1132 #define NEON_FN(dest, src1, src2) dest = (src1 & src2) ? -1 : 0
1133 NEON_VOP(tst_u8, neon_u8, 4)
1134 NEON_VOP(tst_u16, neon_u16, 2)
1135 NEON_VOP(tst_u32, neon_u32, 1)
1136 #undef NEON_FN
1137
1138 #define NEON_FN(dest, src1, src2) dest = (src1 == src2) ? -1 : 0
1139 NEON_VOP(ceq_u8, neon_u8, 4)
1140 NEON_VOP(ceq_u16, neon_u16, 2)
1141 NEON_VOP(ceq_u32, neon_u32, 1)
1142 #undef NEON_FN
1143
1144 /* Count Leading Sign/Zero Bits.  */
1145 static inline int do_clz8(uint8_t x)
1146 {
1147     int n;
1148     for (n = 8; x; n--)
1149         x >>= 1;
1150     return n;
1151 }
1152
1153 static inline int do_clz16(uint16_t x)
1154 {
1155     int n;
1156     for (n = 16; x; n--)
1157         x >>= 1;
1158     return n;
1159 }
1160
1161 #define NEON_FN(dest, src, dummy) dest = do_clz8(src)
1162 NEON_VOP1(clz_u8, neon_u8, 4)
1163 #undef NEON_FN
1164
1165 #define NEON_FN(dest, src, dummy) dest = do_clz16(src)
1166 NEON_VOP1(clz_u16, neon_u16, 2)
1167 #undef NEON_FN
1168
1169 #define NEON_FN(dest, src, dummy) dest = do_clz8((src < 0) ? ~src : src) - 1
1170 NEON_VOP1(cls_s8, neon_s8, 4)
1171 #undef NEON_FN
1172
1173 #define NEON_FN(dest, src, dummy) dest = do_clz16((src < 0) ? ~src : src) - 1
1174 NEON_VOP1(cls_s16, neon_s16, 2)
1175 #undef NEON_FN
1176
1177 uint32_t HELPER(neon_cls_s32)(uint32_t x)
1178 {
1179     int count;
1180     if ((int32_t)x < 0)
1181         x = ~x;
1182     for (count = 32; x; count--)
1183         x = x >> 1;
1184     return count - 1;
1185 }
1186
1187 /* Bit count.  */
1188 uint32_t HELPER(neon_cnt_u8)(uint32_t x)
1189 {
1190     x = (x & 0x55555555) + ((x >>  1) & 0x55555555);
1191     x = (x & 0x33333333) + ((x >>  2) & 0x33333333);
1192     x = (x & 0x0f0f0f0f) + ((x >>  4) & 0x0f0f0f0f);
1193     return x;
1194 }
1195
1196 /* Reverse bits in each 8 bit word */
1197 uint32_t HELPER(neon_rbit_u8)(uint32_t x)
1198 {
1199     x =  ((x & 0xf0f0f0f0) >> 4)
1200        | ((x & 0x0f0f0f0f) << 4);
1201     x =  ((x & 0x88888888) >> 3)
1202        | ((x & 0x44444444) >> 1)
1203        | ((x & 0x22222222) << 1)
1204        | ((x & 0x11111111) << 3);
1205     return x;
1206 }
1207
1208 #define NEON_QDMULH16(dest, src1, src2, round) do { \
1209     uint32_t tmp = (int32_t)(int16_t) src1 * (int16_t) src2; \
1210     if ((tmp ^ (tmp << 1)) & SIGNBIT) { \
1211         SET_QC(); \
1212         tmp = (tmp >> 31) ^ ~SIGNBIT; \
1213     } else { \
1214         tmp <<= 1; \
1215     } \
1216     if (round) { \
1217         int32_t old = tmp; \
1218         tmp += 1 << 15; \
1219         if ((int32_t)tmp < old) { \
1220             SET_QC(); \
1221             tmp = SIGNBIT - 1; \
1222         } \
1223     } \
1224     dest = tmp >> 16; \
1225     } while(0)
1226 #define NEON_FN(dest, src1, src2) NEON_QDMULH16(dest, src1, src2, 0)
1227 NEON_VOP_ENV(qdmulh_s16, neon_s16, 2)
1228 #undef NEON_FN
1229 #define NEON_FN(dest, src1, src2) NEON_QDMULH16(dest, src1, src2, 1)
1230 NEON_VOP_ENV(qrdmulh_s16, neon_s16, 2)
1231 #undef NEON_FN
1232 #undef NEON_QDMULH16
1233
1234 #define NEON_QDMULH32(dest, src1, src2, round) do { \
1235     uint64_t tmp = (int64_t)(int32_t) src1 * (int32_t) src2; \
1236     if ((tmp ^ (tmp << 1)) & SIGNBIT64) { \
1237         SET_QC(); \
1238         tmp = (tmp >> 63) ^ ~SIGNBIT64; \
1239     } else { \
1240         tmp <<= 1; \
1241     } \
1242     if (round) { \
1243         int64_t old = tmp; \
1244         tmp += (int64_t)1 << 31; \
1245         if ((int64_t)tmp < old) { \
1246             SET_QC(); \
1247             tmp = SIGNBIT64 - 1; \
1248         } \
1249     } \
1250     dest = tmp >> 32; \
1251     } while(0)
1252 #define NEON_FN(dest, src1, src2) NEON_QDMULH32(dest, src1, src2, 0)
1253 NEON_VOP_ENV(qdmulh_s32, neon_s32, 1)
1254 #undef NEON_FN
1255 #define NEON_FN(dest, src1, src2) NEON_QDMULH32(dest, src1, src2, 1)
1256 NEON_VOP_ENV(qrdmulh_s32, neon_s32, 1)
1257 #undef NEON_FN
1258 #undef NEON_QDMULH32
1259
1260 uint32_t HELPER(neon_narrow_u8)(uint64_t x)
1261 {
1262     return (x & 0xffu) | ((x >> 8) & 0xff00u) | ((x >> 16) & 0xff0000u)
1263            | ((x >> 24) & 0xff000000u);
1264 }
1265
1266 uint32_t HELPER(neon_narrow_u16)(uint64_t x)
1267 {
1268     return (x & 0xffffu) | ((x >> 16) & 0xffff0000u);
1269 }
1270
1271 uint32_t HELPER(neon_narrow_high_u8)(uint64_t x)
1272 {
1273     return ((x >> 8) & 0xff) | ((x >> 16) & 0xff00)
1274             | ((x >> 24) & 0xff0000) | ((x >> 32) & 0xff000000);
1275 }
1276
1277 uint32_t HELPER(neon_narrow_high_u16)(uint64_t x)
1278 {
1279     return ((x >> 16) & 0xffff) | ((x >> 32) & 0xffff0000);
1280 }
1281
1282 uint32_t HELPER(neon_narrow_round_high_u8)(uint64_t x)
1283 {
1284     x &= 0xff80ff80ff80ff80ull;
1285     x += 0x0080008000800080ull;
1286     return ((x >> 8) & 0xff) | ((x >> 16) & 0xff00)
1287             | ((x >> 24) & 0xff0000) | ((x >> 32) & 0xff000000);
1288 }
1289
1290 uint32_t HELPER(neon_narrow_round_high_u16)(uint64_t x)
1291 {
1292     x &= 0xffff8000ffff8000ull;
1293     x += 0x0000800000008000ull;
1294     return ((x >> 16) & 0xffff) | ((x >> 32) & 0xffff0000);
1295 }
1296
1297 uint32_t HELPER(neon_unarrow_sat8)(CPUARMState *env, uint64_t x)
1298 {
1299     uint16_t s;
1300     uint8_t d;
1301     uint32_t res = 0;
1302 #define SAT8(n) \
1303     s = x >> n; \
1304     if (s & 0x8000) { \
1305         SET_QC(); \
1306     } else { \
1307         if (s > 0xff) { \
1308             d = 0xff; \
1309             SET_QC(); \
1310         } else  { \
1311             d = s; \
1312         } \
1313         res |= (uint32_t)d << (n / 2); \
1314     }
1315
1316     SAT8(0);
1317     SAT8(16);
1318     SAT8(32);
1319     SAT8(48);
1320 #undef SAT8
1321     return res;
1322 }
1323
1324 uint32_t HELPER(neon_narrow_sat_u8)(CPUARMState *env, uint64_t x)
1325 {
1326     uint16_t s;
1327     uint8_t d;
1328     uint32_t res = 0;
1329 #define SAT8(n) \
1330     s = x >> n; \
1331     if (s > 0xff) { \
1332         d = 0xff; \
1333         SET_QC(); \
1334     } else  { \
1335         d = s; \
1336     } \
1337     res |= (uint32_t)d << (n / 2);
1338
1339     SAT8(0);
1340     SAT8(16);
1341     SAT8(32);
1342     SAT8(48);
1343 #undef SAT8
1344     return res;
1345 }
1346
1347 uint32_t HELPER(neon_narrow_sat_s8)(CPUARMState *env, uint64_t x)
1348 {
1349     int16_t s;
1350     uint8_t d;
1351     uint32_t res = 0;
1352 #define SAT8(n) \
1353     s = x >> n; \
1354     if (s != (int8_t)s) { \
1355         d = (s >> 15) ^ 0x7f; \
1356         SET_QC(); \
1357     } else  { \
1358         d = s; \
1359     } \
1360     res |= (uint32_t)d << (n / 2);
1361
1362     SAT8(0);
1363     SAT8(16);
1364     SAT8(32);
1365     SAT8(48);
1366 #undef SAT8
1367     return res;
1368 }
1369
1370 uint32_t HELPER(neon_unarrow_sat16)(CPUARMState *env, uint64_t x)
1371 {
1372     uint32_t high;
1373     uint32_t low;
1374     low = x;
1375     if (low & 0x80000000) {
1376         low = 0;
1377         SET_QC();
1378     } else if (low > 0xffff) {
1379         low = 0xffff;
1380         SET_QC();
1381     }
1382     high = x >> 32;
1383     if (high & 0x80000000) {
1384         high = 0;
1385         SET_QC();
1386     } else if (high > 0xffff) {
1387         high = 0xffff;
1388         SET_QC();
1389     }
1390     return low | (high << 16);
1391 }
1392
1393 uint32_t HELPER(neon_narrow_sat_u16)(CPUARMState *env, uint64_t x)
1394 {
1395     uint32_t high;
1396     uint32_t low;
1397     low = x;
1398     if (low > 0xffff) {
1399         low = 0xffff;
1400         SET_QC();
1401     }
1402     high = x >> 32;
1403     if (high > 0xffff) {
1404         high = 0xffff;
1405         SET_QC();
1406     }
1407     return low | (high << 16);
1408 }
1409
1410 uint32_t HELPER(neon_narrow_sat_s16)(CPUARMState *env, uint64_t x)
1411 {
1412     int32_t low;
1413     int32_t high;
1414     low = x;
1415     if (low != (int16_t)low) {
1416         low = (low >> 31) ^ 0x7fff;
1417         SET_QC();
1418     }
1419     high = x >> 32;
1420     if (high != (int16_t)high) {
1421         high = (high >> 31) ^ 0x7fff;
1422         SET_QC();
1423     }
1424     return (uint16_t)low | (high << 16);
1425 }
1426
1427 uint32_t HELPER(neon_unarrow_sat32)(CPUARMState *env, uint64_t x)
1428 {
1429     if (x & 0x8000000000000000ull) {
1430         SET_QC();
1431         return 0;
1432     }
1433     if (x > 0xffffffffu) {
1434         SET_QC();
1435         return 0xffffffffu;
1436     }
1437     return x;
1438 }
1439
1440 uint32_t HELPER(neon_narrow_sat_u32)(CPUARMState *env, uint64_t x)
1441 {
1442     if (x > 0xffffffffu) {
1443         SET_QC();
1444         return 0xffffffffu;
1445     }
1446     return x;
1447 }
1448
1449 uint32_t HELPER(neon_narrow_sat_s32)(CPUARMState *env, uint64_t x)
1450 {
1451     if ((int64_t)x != (int32_t)x) {
1452         SET_QC();
1453         return ((int64_t)x >> 63) ^ 0x7fffffff;
1454     }
1455     return x;
1456 }
1457
1458 uint64_t HELPER(neon_widen_u8)(uint32_t x)
1459 {
1460     uint64_t tmp;
1461     uint64_t ret;
1462     ret = (uint8_t)x;
1463     tmp = (uint8_t)(x >> 8);
1464     ret |= tmp << 16;
1465     tmp = (uint8_t)(x >> 16);
1466     ret |= tmp << 32;
1467     tmp = (uint8_t)(x >> 24);
1468     ret |= tmp << 48;
1469     return ret;
1470 }
1471
1472 uint64_t HELPER(neon_widen_s8)(uint32_t x)
1473 {
1474     uint64_t tmp;
1475     uint64_t ret;
1476     ret = (uint16_t)(int8_t)x;
1477     tmp = (uint16_t)(int8_t)(x >> 8);
1478     ret |= tmp << 16;
1479     tmp = (uint16_t)(int8_t)(x >> 16);
1480     ret |= tmp << 32;
1481     tmp = (uint16_t)(int8_t)(x >> 24);
1482     ret |= tmp << 48;
1483     return ret;
1484 }
1485
1486 uint64_t HELPER(neon_widen_u16)(uint32_t x)
1487 {
1488     uint64_t high = (uint16_t)(x >> 16);
1489     return ((uint16_t)x) | (high << 32);
1490 }
1491
1492 uint64_t HELPER(neon_widen_s16)(uint32_t x)
1493 {
1494     uint64_t high = (int16_t)(x >> 16);
1495     return ((uint32_t)(int16_t)x) | (high << 32);
1496 }
1497
1498 uint64_t HELPER(neon_addl_u16)(uint64_t a, uint64_t b)
1499 {
1500     uint64_t mask;
1501     mask = (a ^ b) & 0x8000800080008000ull;
1502     a &= ~0x8000800080008000ull;
1503     b &= ~0x8000800080008000ull;
1504     return (a + b) ^ mask;
1505 }
1506
1507 uint64_t HELPER(neon_addl_u32)(uint64_t a, uint64_t b)
1508 {
1509     uint64_t mask;
1510     mask = (a ^ b) & 0x8000000080000000ull;
1511     a &= ~0x8000000080000000ull;
1512     b &= ~0x8000000080000000ull;
1513     return (a + b) ^ mask;
1514 }
1515
1516 uint64_t HELPER(neon_paddl_u16)(uint64_t a, uint64_t b)
1517 {
1518     uint64_t tmp;
1519     uint64_t tmp2;
1520
1521     tmp = a & 0x0000ffff0000ffffull;
1522     tmp += (a >> 16) & 0x0000ffff0000ffffull;
1523     tmp2 = b & 0xffff0000ffff0000ull;
1524     tmp2 += (b << 16) & 0xffff0000ffff0000ull;
1525     return    ( tmp         & 0xffff)
1526             | ((tmp  >> 16) & 0xffff0000ull)
1527             | ((tmp2 << 16) & 0xffff00000000ull)
1528             | ( tmp2        & 0xffff000000000000ull);
1529 }
1530
1531 uint64_t HELPER(neon_paddl_u32)(uint64_t a, uint64_t b)
1532 {
1533     uint32_t low = a + (a >> 32);
1534     uint32_t high = b + (b >> 32);
1535     return low + ((uint64_t)high << 32);
1536 }
1537
1538 uint64_t HELPER(neon_subl_u16)(uint64_t a, uint64_t b)
1539 {
1540     uint64_t mask;
1541     mask = (a ^ ~b) & 0x8000800080008000ull;
1542     a |= 0x8000800080008000ull;
1543     b &= ~0x8000800080008000ull;
1544     return (a - b) ^ mask;
1545 }
1546
1547 uint64_t HELPER(neon_subl_u32)(uint64_t a, uint64_t b)
1548 {
1549     uint64_t mask;
1550     mask = (a ^ ~b) & 0x8000000080000000ull;
1551     a |= 0x8000000080000000ull;
1552     b &= ~0x8000000080000000ull;
1553     return (a - b) ^ mask;
1554 }
1555
1556 uint64_t HELPER(neon_addl_saturate_s32)(CPUARMState *env, uint64_t a, uint64_t b)
1557 {
1558     uint32_t x, y;
1559     uint32_t low, high;
1560
1561     x = a;
1562     y = b;
1563     low = x + y;
1564     if (((low ^ x) & SIGNBIT) && !((x ^ y) & SIGNBIT)) {
1565         SET_QC();
1566         low = ((int32_t)x >> 31) ^ ~SIGNBIT;
1567     }
1568     x = a >> 32;
1569     y = b >> 32;
1570     high = x + y;
1571     if (((high ^ x) & SIGNBIT) && !((x ^ y) & SIGNBIT)) {
1572         SET_QC();
1573         high = ((int32_t)x >> 31) ^ ~SIGNBIT;
1574     }
1575     return low | ((uint64_t)high << 32);
1576 }
1577
1578 uint64_t HELPER(neon_addl_saturate_s64)(CPUARMState *env, uint64_t a, uint64_t b)
1579 {
1580     uint64_t result;
1581
1582     result = a + b;
1583     if (((result ^ a) & SIGNBIT64) && !((a ^ b) & SIGNBIT64)) {
1584         SET_QC();
1585         result = ((int64_t)a >> 63) ^ ~SIGNBIT64;
1586     }
1587     return result;
1588 }
1589
1590 /* We have to do the arithmetic in a larger type than
1591  * the input type, because for example with a signed 32 bit
1592  * op the absolute difference can overflow a signed 32 bit value.
1593  */
1594 #define DO_ABD(dest, x, y, intype, arithtype) do {            \
1595     arithtype tmp_x = (intype)(x);                            \
1596     arithtype tmp_y = (intype)(y);                            \
1597     dest = ((tmp_x > tmp_y) ? tmp_x - tmp_y : tmp_y - tmp_x); \
1598     } while(0)
1599
1600 uint64_t HELPER(neon_abdl_u16)(uint32_t a, uint32_t b)
1601 {
1602     uint64_t tmp;
1603     uint64_t result;
1604     DO_ABD(result, a, b, uint8_t, uint32_t);
1605     DO_ABD(tmp, a >> 8, b >> 8, uint8_t, uint32_t);
1606     result |= tmp << 16;
1607     DO_ABD(tmp, a >> 16, b >> 16, uint8_t, uint32_t);
1608     result |= tmp << 32;
1609     DO_ABD(tmp, a >> 24, b >> 24, uint8_t, uint32_t);
1610     result |= tmp << 48;
1611     return result;
1612 }
1613
1614 uint64_t HELPER(neon_abdl_s16)(uint32_t a, uint32_t b)
1615 {
1616     uint64_t tmp;
1617     uint64_t result;
1618     DO_ABD(result, a, b, int8_t, int32_t);
1619     DO_ABD(tmp, a >> 8, b >> 8, int8_t, int32_t);
1620     result |= tmp << 16;
1621     DO_ABD(tmp, a >> 16, b >> 16, int8_t, int32_t);
1622     result |= tmp << 32;
1623     DO_ABD(tmp, a >> 24, b >> 24, int8_t, int32_t);
1624     result |= tmp << 48;
1625     return result;
1626 }
1627
1628 uint64_t HELPER(neon_abdl_u32)(uint32_t a, uint32_t b)
1629 {
1630     uint64_t tmp;
1631     uint64_t result;
1632     DO_ABD(result, a, b, uint16_t, uint32_t);
1633     DO_ABD(tmp, a >> 16, b >> 16, uint16_t, uint32_t);
1634     return result | (tmp << 32);
1635 }
1636
1637 uint64_t HELPER(neon_abdl_s32)(uint32_t a, uint32_t b)
1638 {
1639     uint64_t tmp;
1640     uint64_t result;
1641     DO_ABD(result, a, b, int16_t, int32_t);
1642     DO_ABD(tmp, a >> 16, b >> 16, int16_t, int32_t);
1643     return result | (tmp << 32);
1644 }
1645
1646 uint64_t HELPER(neon_abdl_u64)(uint32_t a, uint32_t b)
1647 {
1648     uint64_t result;
1649     DO_ABD(result, a, b, uint32_t, uint64_t);
1650     return result;
1651 }
1652
1653 uint64_t HELPER(neon_abdl_s64)(uint32_t a, uint32_t b)
1654 {
1655     uint64_t result;
1656     DO_ABD(result, a, b, int32_t, int64_t);
1657     return result;
1658 }
1659 #undef DO_ABD
1660
1661 /* Widening multiply. Named type is the source type.  */
1662 #define DO_MULL(dest, x, y, type1, type2) do { \
1663     type1 tmp_x = x; \
1664     type1 tmp_y = y; \
1665     dest = (type2)((type2)tmp_x * (type2)tmp_y); \
1666     } while(0)
1667
1668 uint64_t HELPER(neon_mull_u8)(uint32_t a, uint32_t b)
1669 {
1670     uint64_t tmp;
1671     uint64_t result;
1672
1673     DO_MULL(result, a, b, uint8_t, uint16_t);
1674     DO_MULL(tmp, a >> 8, b >> 8, uint8_t, uint16_t);
1675     result |= tmp << 16;
1676     DO_MULL(tmp, a >> 16, b >> 16, uint8_t, uint16_t);
1677     result |= tmp << 32;
1678     DO_MULL(tmp, a >> 24, b >> 24, uint8_t, uint16_t);
1679     result |= tmp << 48;
1680     return result;
1681 }
1682
1683 uint64_t HELPER(neon_mull_s8)(uint32_t a, uint32_t b)
1684 {
1685     uint64_t tmp;
1686     uint64_t result;
1687
1688     DO_MULL(result, a, b, int8_t, uint16_t);
1689     DO_MULL(tmp, a >> 8, b >> 8, int8_t, uint16_t);
1690     result |= tmp << 16;
1691     DO_MULL(tmp, a >> 16, b >> 16, int8_t, uint16_t);
1692     result |= tmp << 32;
1693     DO_MULL(tmp, a >> 24, b >> 24, int8_t, uint16_t);
1694     result |= tmp << 48;
1695     return result;
1696 }
1697
1698 uint64_t HELPER(neon_mull_u16)(uint32_t a, uint32_t b)
1699 {
1700     uint64_t tmp;
1701     uint64_t result;
1702
1703     DO_MULL(result, a, b, uint16_t, uint32_t);
1704     DO_MULL(tmp, a >> 16, b >> 16, uint16_t, uint32_t);
1705     return result | (tmp << 32);
1706 }
1707
1708 uint64_t HELPER(neon_mull_s16)(uint32_t a, uint32_t b)
1709 {
1710     uint64_t tmp;
1711     uint64_t result;
1712
1713     DO_MULL(result, a, b, int16_t, uint32_t);
1714     DO_MULL(tmp, a >> 16, b >> 16, int16_t, uint32_t);
1715     return result | (tmp << 32);
1716 }
1717
1718 uint64_t HELPER(neon_negl_u16)(uint64_t x)
1719 {
1720     uint16_t tmp;
1721     uint64_t result;
1722     result = (uint16_t)-x;
1723     tmp = -(x >> 16);
1724     result |= (uint64_t)tmp << 16;
1725     tmp = -(x >> 32);
1726     result |= (uint64_t)tmp << 32;
1727     tmp = -(x >> 48);
1728     result |= (uint64_t)tmp << 48;
1729     return result;
1730 }
1731
1732 uint64_t HELPER(neon_negl_u32)(uint64_t x)
1733 {
1734     uint32_t low = -x;
1735     uint32_t high = -(x >> 32);
1736     return low | ((uint64_t)high << 32);
1737 }
1738
1739 /* Saturating sign manipulation.  */
1740 /* ??? Make these use NEON_VOP1 */
1741 #define DO_QABS8(x) do { \
1742     if (x == (int8_t)0x80) { \
1743         x = 0x7f; \
1744         SET_QC(); \
1745     } else if (x < 0) { \
1746         x = -x; \
1747     }} while (0)
1748 uint32_t HELPER(neon_qabs_s8)(CPUARMState *env, uint32_t x)
1749 {
1750     neon_s8 vec;
1751     NEON_UNPACK(neon_s8, vec, x);
1752     DO_QABS8(vec.v1);
1753     DO_QABS8(vec.v2);
1754     DO_QABS8(vec.v3);
1755     DO_QABS8(vec.v4);
1756     NEON_PACK(neon_s8, x, vec);
1757     return x;
1758 }
1759 #undef DO_QABS8
1760
1761 #define DO_QNEG8(x) do { \
1762     if (x == (int8_t)0x80) { \
1763         x = 0x7f; \
1764         SET_QC(); \
1765     } else { \
1766         x = -x; \
1767     }} while (0)
1768 uint32_t HELPER(neon_qneg_s8)(CPUARMState *env, uint32_t x)
1769 {
1770     neon_s8 vec;
1771     NEON_UNPACK(neon_s8, vec, x);
1772     DO_QNEG8(vec.v1);
1773     DO_QNEG8(vec.v2);
1774     DO_QNEG8(vec.v3);
1775     DO_QNEG8(vec.v4);
1776     NEON_PACK(neon_s8, x, vec);
1777     return x;
1778 }
1779 #undef DO_QNEG8
1780
1781 #define DO_QABS16(x) do { \
1782     if (x == (int16_t)0x8000) { \
1783         x = 0x7fff; \
1784         SET_QC(); \
1785     } else if (x < 0) { \
1786         x = -x; \
1787     }} while (0)
1788 uint32_t HELPER(neon_qabs_s16)(CPUARMState *env, uint32_t x)
1789 {
1790     neon_s16 vec;
1791     NEON_UNPACK(neon_s16, vec, x);
1792     DO_QABS16(vec.v1);
1793     DO_QABS16(vec.v2);
1794     NEON_PACK(neon_s16, x, vec);
1795     return x;
1796 }
1797 #undef DO_QABS16
1798
1799 #define DO_QNEG16(x) do { \
1800     if (x == (int16_t)0x8000) { \
1801         x = 0x7fff; \
1802         SET_QC(); \
1803     } else { \
1804         x = -x; \
1805     }} while (0)
1806 uint32_t HELPER(neon_qneg_s16)(CPUARMState *env, uint32_t x)
1807 {
1808     neon_s16 vec;
1809     NEON_UNPACK(neon_s16, vec, x);
1810     DO_QNEG16(vec.v1);
1811     DO_QNEG16(vec.v2);
1812     NEON_PACK(neon_s16, x, vec);
1813     return x;
1814 }
1815 #undef DO_QNEG16
1816
1817 uint32_t HELPER(neon_qabs_s32)(CPUARMState *env, uint32_t x)
1818 {
1819     if (x == SIGNBIT) {
1820         SET_QC();
1821         x = ~SIGNBIT;
1822     } else if ((int32_t)x < 0) {
1823         x = -x;
1824     }
1825     return x;
1826 }
1827
1828 uint32_t HELPER(neon_qneg_s32)(CPUARMState *env, uint32_t x)
1829 {
1830     if (x == SIGNBIT) {
1831         SET_QC();
1832         x = ~SIGNBIT;
1833     } else {
1834         x = -x;
1835     }
1836     return x;
1837 }
1838
1839 uint64_t HELPER(neon_qabs_s64)(CPUARMState *env, uint64_t x)
1840 {
1841     if (x == SIGNBIT64) {
1842         SET_QC();
1843         x = ~SIGNBIT64;
1844     } else if ((int64_t)x < 0) {
1845         x = -x;
1846     }
1847     return x;
1848 }
1849
1850 uint64_t HELPER(neon_qneg_s64)(CPUARMState *env, uint64_t x)
1851 {
1852     if (x == SIGNBIT64) {
1853         SET_QC();
1854         x = ~SIGNBIT64;
1855     } else {
1856         x = -x;
1857     }
1858     return x;
1859 }
1860
1861 /* NEON Float helpers.  */
1862 uint32_t HELPER(neon_abd_f32)(uint32_t a, uint32_t b, void *fpstp)
1863 {
1864     float_status *fpst = fpstp;
1865     float32 f0 = make_float32(a);
1866     float32 f1 = make_float32(b);
1867     return float32_val(float32_abs(float32_sub(f0, f1, fpst)));
1868 }
1869
1870 /* Floating point comparisons produce an integer result.
1871  * Note that EQ doesn't signal InvalidOp for QNaNs but GE and GT do.
1872  * Softfloat routines return 0/1, which we convert to the 0/-1 Neon requires.
1873  */
1874 uint32_t HELPER(neon_ceq_f32)(uint32_t a, uint32_t b, void *fpstp)
1875 {
1876     float_status *fpst = fpstp;
1877     return -float32_eq_quiet(make_float32(a), make_float32(b), fpst);
1878 }
1879
1880 uint32_t HELPER(neon_cge_f32)(uint32_t a, uint32_t b, void *fpstp)
1881 {
1882     float_status *fpst = fpstp;
1883     return -float32_le(make_float32(b), make_float32(a), fpst);
1884 }
1885
1886 uint32_t HELPER(neon_cgt_f32)(uint32_t a, uint32_t b, void *fpstp)
1887 {
1888     float_status *fpst = fpstp;
1889     return -float32_lt(make_float32(b), make_float32(a), fpst);
1890 }
1891
1892 uint32_t HELPER(neon_acge_f32)(uint32_t a, uint32_t b, void *fpstp)
1893 {
1894     float_status *fpst = fpstp;
1895     float32 f0 = float32_abs(make_float32(a));
1896     float32 f1 = float32_abs(make_float32(b));
1897     return -float32_le(f1, f0, fpst);
1898 }
1899
1900 uint32_t HELPER(neon_acgt_f32)(uint32_t a, uint32_t b, void *fpstp)
1901 {
1902     float_status *fpst = fpstp;
1903     float32 f0 = float32_abs(make_float32(a));
1904     float32 f1 = float32_abs(make_float32(b));
1905     return -float32_lt(f1, f0, fpst);
1906 }
1907
1908 uint64_t HELPER(neon_acge_f64)(uint64_t a, uint64_t b, void *fpstp)
1909 {
1910     float_status *fpst = fpstp;
1911     float64 f0 = float64_abs(make_float64(a));
1912     float64 f1 = float64_abs(make_float64(b));
1913     return -float64_le(f1, f0, fpst);
1914 }
1915
1916 uint64_t HELPER(neon_acgt_f64)(uint64_t a, uint64_t b, void *fpstp)
1917 {
1918     float_status *fpst = fpstp;
1919     float64 f0 = float64_abs(make_float64(a));
1920     float64 f1 = float64_abs(make_float64(b));
1921     return -float64_lt(f1, f0, fpst);
1922 }
1923
1924 #define ELEM(V, N, SIZE) (((V) >> ((N) * (SIZE))) & ((1ull << (SIZE)) - 1))
1925
1926 void HELPER(neon_qunzip8)(void *vd, void *vm)
1927 {
1928     uint64_t *rd = vd, *rm = vm;
1929     uint64_t zd0 = rd[0], zd1 = rd[1];
1930     uint64_t zm0 = rm[0], zm1 = rm[1];
1931
1932     uint64_t d0 = ELEM(zd0, 0, 8) | (ELEM(zd0, 2, 8) << 8)
1933         | (ELEM(zd0, 4, 8) << 16) | (ELEM(zd0, 6, 8) << 24)
1934         | (ELEM(zd1, 0, 8) << 32) | (ELEM(zd1, 2, 8) << 40)
1935         | (ELEM(zd1, 4, 8) << 48) | (ELEM(zd1, 6, 8) << 56);
1936     uint64_t d1 = ELEM(zm0, 0, 8) | (ELEM(zm0, 2, 8) << 8)
1937         | (ELEM(zm0, 4, 8) << 16) | (ELEM(zm0, 6, 8) << 24)
1938         | (ELEM(zm1, 0, 8) << 32) | (ELEM(zm1, 2, 8) << 40)
1939         | (ELEM(zm1, 4, 8) << 48) | (ELEM(zm1, 6, 8) << 56);
1940     uint64_t m0 = ELEM(zd0, 1, 8) | (ELEM(zd0, 3, 8) << 8)
1941         | (ELEM(zd0, 5, 8) << 16) | (ELEM(zd0, 7, 8) << 24)
1942         | (ELEM(zd1, 1, 8) << 32) | (ELEM(zd1, 3, 8) << 40)
1943         | (ELEM(zd1, 5, 8) << 48) | (ELEM(zd1, 7, 8) << 56);
1944     uint64_t m1 = ELEM(zm0, 1, 8) | (ELEM(zm0, 3, 8) << 8)
1945         | (ELEM(zm0, 5, 8) << 16) | (ELEM(zm0, 7, 8) << 24)
1946         | (ELEM(zm1, 1, 8) << 32) | (ELEM(zm1, 3, 8) << 40)
1947         | (ELEM(zm1, 5, 8) << 48) | (ELEM(zm1, 7, 8) << 56);
1948
1949     rm[0] = m0;
1950     rm[1] = m1;
1951     rd[0] = d0;
1952     rd[1] = d1;
1953 }
1954
1955 void HELPER(neon_qunzip16)(void *vd, void *vm)
1956 {
1957     uint64_t *rd = vd, *rm = vm;
1958     uint64_t zd0 = rd[0], zd1 = rd[1];
1959     uint64_t zm0 = rm[0], zm1 = rm[1];
1960
1961     uint64_t d0 = ELEM(zd0, 0, 16) | (ELEM(zd0, 2, 16) << 16)
1962         | (ELEM(zd1, 0, 16) << 32) | (ELEM(zd1, 2, 16) << 48);
1963     uint64_t d1 = ELEM(zm0, 0, 16) | (ELEM(zm0, 2, 16) << 16)
1964         | (ELEM(zm1, 0, 16) << 32) | (ELEM(zm1, 2, 16) << 48);
1965     uint64_t m0 = ELEM(zd0, 1, 16) | (ELEM(zd0, 3, 16) << 16)
1966         | (ELEM(zd1, 1, 16) << 32) | (ELEM(zd1, 3, 16) << 48);
1967     uint64_t m1 = ELEM(zm0, 1, 16) | (ELEM(zm0, 3, 16) << 16)
1968         | (ELEM(zm1, 1, 16) << 32) | (ELEM(zm1, 3, 16) << 48);
1969
1970     rm[0] = m0;
1971     rm[1] = m1;
1972     rd[0] = d0;
1973     rd[1] = d1;
1974 }
1975
1976 void HELPER(neon_qunzip32)(void *vd, void *vm)
1977 {
1978     uint64_t *rd = vd, *rm = vm;
1979     uint64_t zd0 = rd[0], zd1 = rd[1];
1980     uint64_t zm0 = rm[0], zm1 = rm[1];
1981
1982     uint64_t d0 = ELEM(zd0, 0, 32) | (ELEM(zd1, 0, 32) << 32);
1983     uint64_t d1 = ELEM(zm0, 0, 32) | (ELEM(zm1, 0, 32) << 32);
1984     uint64_t m0 = ELEM(zd0, 1, 32) | (ELEM(zd1, 1, 32) << 32);
1985     uint64_t m1 = ELEM(zm0, 1, 32) | (ELEM(zm1, 1, 32) << 32);
1986
1987     rm[0] = m0;
1988     rm[1] = m1;
1989     rd[0] = d0;
1990     rd[1] = d1;
1991 }
1992
1993 void HELPER(neon_unzip8)(void *vd, void *vm)
1994 {
1995     uint64_t *rd = vd, *rm = vm;
1996     uint64_t zd = rd[0], zm = rm[0];
1997
1998     uint64_t d0 = ELEM(zd, 0, 8) | (ELEM(zd, 2, 8) << 8)
1999         | (ELEM(zd, 4, 8) << 16) | (ELEM(zd, 6, 8) << 24)
2000         | (ELEM(zm, 0, 8) << 32) | (ELEM(zm, 2, 8) << 40)
2001         | (ELEM(zm, 4, 8) << 48) | (ELEM(zm, 6, 8) << 56);
2002     uint64_t m0 = ELEM(zd, 1, 8) | (ELEM(zd, 3, 8) << 8)
2003         | (ELEM(zd, 5, 8) << 16) | (ELEM(zd, 7, 8) << 24)
2004         | (ELEM(zm, 1, 8) << 32) | (ELEM(zm, 3, 8) << 40)
2005         | (ELEM(zm, 5, 8) << 48) | (ELEM(zm, 7, 8) << 56);
2006
2007     rm[0] = m0;
2008     rd[0] = d0;
2009 }
2010
2011 void HELPER(neon_unzip16)(void *vd, void *vm)
2012 {
2013     uint64_t *rd = vd, *rm = vm;
2014     uint64_t zd = rd[0], zm = rm[0];
2015
2016     uint64_t d0 = ELEM(zd, 0, 16) | (ELEM(zd, 2, 16) << 16)
2017         | (ELEM(zm, 0, 16) << 32) | (ELEM(zm, 2, 16) << 48);
2018     uint64_t m0 = ELEM(zd, 1, 16) | (ELEM(zd, 3, 16) << 16)
2019         | (ELEM(zm, 1, 16) << 32) | (ELEM(zm, 3, 16) << 48);
2020
2021     rm[0] = m0;
2022     rd[0] = d0;
2023 }
2024
2025 void HELPER(neon_qzip8)(void *vd, void *vm)
2026 {
2027     uint64_t *rd = vd, *rm = vm;
2028     uint64_t zd0 = rd[0], zd1 = rd[1];
2029     uint64_t zm0 = rm[0], zm1 = rm[1];
2030
2031     uint64_t d0 = ELEM(zd0, 0, 8) | (ELEM(zm0, 0, 8) << 8)
2032         | (ELEM(zd0, 1, 8) << 16) | (ELEM(zm0, 1, 8) << 24)
2033         | (ELEM(zd0, 2, 8) << 32) | (ELEM(zm0, 2, 8) << 40)
2034         | (ELEM(zd0, 3, 8) << 48) | (ELEM(zm0, 3, 8) << 56);
2035     uint64_t d1 = ELEM(zd0, 4, 8) | (ELEM(zm0, 4, 8) << 8)
2036         | (ELEM(zd0, 5, 8) << 16) | (ELEM(zm0, 5, 8) << 24)
2037         | (ELEM(zd0, 6, 8) << 32) | (ELEM(zm0, 6, 8) << 40)
2038         | (ELEM(zd0, 7, 8) << 48) | (ELEM(zm0, 7, 8) << 56);
2039     uint64_t m0 = ELEM(zd1, 0, 8) | (ELEM(zm1, 0, 8) << 8)
2040         | (ELEM(zd1, 1, 8) << 16) | (ELEM(zm1, 1, 8) << 24)
2041         | (ELEM(zd1, 2, 8) << 32) | (ELEM(zm1, 2, 8) << 40)
2042         | (ELEM(zd1, 3, 8) << 48) | (ELEM(zm1, 3, 8) << 56);
2043     uint64_t m1 = ELEM(zd1, 4, 8) | (ELEM(zm1, 4, 8) << 8)
2044         | (ELEM(zd1, 5, 8) << 16) | (ELEM(zm1, 5, 8) << 24)
2045         | (ELEM(zd1, 6, 8) << 32) | (ELEM(zm1, 6, 8) << 40)
2046         | (ELEM(zd1, 7, 8) << 48) | (ELEM(zm1, 7, 8) << 56);
2047
2048     rm[0] = m0;
2049     rm[1] = m1;
2050     rd[0] = d0;
2051     rd[1] = d1;
2052 }
2053
2054 void HELPER(neon_qzip16)(void *vd, void *vm)
2055 {
2056     uint64_t *rd = vd, *rm = vm;
2057     uint64_t zd0 = rd[0], zd1 = rd[1];
2058     uint64_t zm0 = rm[0], zm1 = rm[1];
2059
2060     uint64_t d0 = ELEM(zd0, 0, 16) | (ELEM(zm0, 0, 16) << 16)
2061         | (ELEM(zd0, 1, 16) << 32) | (ELEM(zm0, 1, 16) << 48);
2062     uint64_t d1 = ELEM(zd0, 2, 16) | (ELEM(zm0, 2, 16) << 16)
2063         | (ELEM(zd0, 3, 16) << 32) | (ELEM(zm0, 3, 16) << 48);
2064     uint64_t m0 = ELEM(zd1, 0, 16) | (ELEM(zm1, 0, 16) << 16)
2065         | (ELEM(zd1, 1, 16) << 32) | (ELEM(zm1, 1, 16) << 48);
2066     uint64_t m1 = ELEM(zd1, 2, 16) | (ELEM(zm1, 2, 16) << 16)
2067         | (ELEM(zd1, 3, 16) << 32) | (ELEM(zm1, 3, 16) << 48);
2068
2069     rm[0] = m0;
2070     rm[1] = m1;
2071     rd[0] = d0;
2072     rd[1] = d1;
2073 }
2074
2075 void HELPER(neon_qzip32)(void *vd, void *vm)
2076 {
2077     uint64_t *rd = vd, *rm = vm;
2078     uint64_t zd0 = rd[0], zd1 = rd[1];
2079     uint64_t zm0 = rm[0], zm1 = rm[1];
2080
2081     uint64_t d0 = ELEM(zd0, 0, 32) | (ELEM(zm0, 0, 32) << 32);
2082     uint64_t d1 = ELEM(zd0, 1, 32) | (ELEM(zm0, 1, 32) << 32);
2083     uint64_t m0 = ELEM(zd1, 0, 32) | (ELEM(zm1, 0, 32) << 32);
2084     uint64_t m1 = ELEM(zd1, 1, 32) | (ELEM(zm1, 1, 32) << 32);
2085
2086     rm[0] = m0;
2087     rm[1] = m1;
2088     rd[0] = d0;
2089     rd[1] = d1;
2090 }
2091
2092 void HELPER(neon_zip8)(void *vd, void *vm)
2093 {
2094     uint64_t *rd = vd, *rm = vm;
2095     uint64_t zd = rd[0], zm = rm[0];
2096
2097     uint64_t d0 = ELEM(zd, 0, 8) | (ELEM(zm, 0, 8) << 8)
2098         | (ELEM(zd, 1, 8) << 16) | (ELEM(zm, 1, 8) << 24)
2099         | (ELEM(zd, 2, 8) << 32) | (ELEM(zm, 2, 8) << 40)
2100         | (ELEM(zd, 3, 8) << 48) | (ELEM(zm, 3, 8) << 56);
2101     uint64_t m0 = ELEM(zd, 4, 8) | (ELEM(zm, 4, 8) << 8)
2102         | (ELEM(zd, 5, 8) << 16) | (ELEM(zm, 5, 8) << 24)
2103         | (ELEM(zd, 6, 8) << 32) | (ELEM(zm, 6, 8) << 40)
2104         | (ELEM(zd, 7, 8) << 48) | (ELEM(zm, 7, 8) << 56);
2105
2106     rm[0] = m0;
2107     rd[0] = d0;
2108 }
2109
2110 void HELPER(neon_zip16)(void *vd, void *vm)
2111 {
2112     uint64_t *rd = vd, *rm = vm;
2113     uint64_t zd = rd[0], zm = rm[0];
2114
2115     uint64_t d0 = ELEM(zd, 0, 16) | (ELEM(zm, 0, 16) << 16)
2116         | (ELEM(zd, 1, 16) << 32) | (ELEM(zm, 1, 16) << 48);
2117     uint64_t m0 = ELEM(zd, 2, 16) | (ELEM(zm, 2, 16) << 16)
2118         | (ELEM(zd, 3, 16) << 32) | (ELEM(zm, 3, 16) << 48);
2119
2120     rm[0] = m0;
2121     rd[0] = d0;
2122 }