target/arm/neon_helper.c

   1 /*
   2  * ARM NEON vector operations.
   3  *
   4  * Copyright (c) 2007, 2008 CodeSourcery.
   5  * Written by Paul Brook
   6  *
   7  * This code is licensed under the GNU GPL v2.
   8  */
   9 #include "qemu/osdep.h"
  10
  11 #include "cpu.h"
  12 #include "exec/helper-proto.h"
  13 #include "fpu/softfloat.h"
  14
  15 #define SIGNBIT (uint32_t)0x80000000
  16 #define SIGNBIT64 ((uint64_t)1 << 63)
  17
  18 #define SET_QC() env->vfp.xregs[ARM_VFP_FPSCR] |= CPSR_Q
  19
  20 #define NEON_TYPE1(name, type) \
  21 typedef struct \
  22 { \
  23     type v1; \
  24 } neon_##name;
  25 #ifdef HOST_WORDS_BIGENDIAN
  26 #define NEON_TYPE2(name, type) \
  27 typedef struct \
  28 { \
  29     type v2; \
  30     type v1; \
  31 } neon_##name;
  32 #define NEON_TYPE4(name, type) \
  33 typedef struct \
  34 { \
  35     type v4; \
  36     type v3; \
  37     type v2; \
  38     type v1; \
  39 } neon_##name;
  40 #else
  41 #define NEON_TYPE2(name, type) \
  42 typedef struct \
  43 { \
  44     type v1; \
  45     type v2; \
  46 } neon_##name;
  47 #define NEON_TYPE4(name, type) \
  48 typedef struct \
  49 { \
  50     type v1; \
  51     type v2; \
  52     type v3; \
  53     type v4; \
  54 } neon_##name;
  55 #endif
  56
  57 NEON_TYPE4(s8, int8_t)
  58 NEON_TYPE4(u8, uint8_t)
  59 NEON_TYPE2(s16, int16_t)
  60 NEON_TYPE2(u16, uint16_t)
  61 NEON_TYPE1(s32, int32_t)
  62 NEON_TYPE1(u32, uint32_t)
  63 #undef NEON_TYPE4
  64 #undef NEON_TYPE2
  65 #undef NEON_TYPE1
  66
  67 /* Copy from a uint32_t to a vector structure type.  */
  68 #define NEON_UNPACK(vtype, dest, val) do { \
  69     union { \
  70         vtype v; \
  71         uint32_t i; \
  72     } conv_u; \
  73     conv_u.i = (val); \
  74     dest = conv_u.v; \
  75     } while(0)
  76
  77 /* Copy from a vector structure type to a uint32_t.  */
  78 #define NEON_PACK(vtype, dest, val) do { \
  79     union { \
  80         vtype v; \
  81         uint32_t i; \
  82     } conv_u; \
  83     conv_u.v = (val); \
  84     dest = conv_u.i; \
  85     } while(0)
  86
  87 #define NEON_DO1 \
  88     NEON_FN(vdest.v1, vsrc1.v1, vsrc2.v1);
  89 #define NEON_DO2 \
  90     NEON_FN(vdest.v1, vsrc1.v1, vsrc2.v1); \
  91     NEON_FN(vdest.v2, vsrc1.v2, vsrc2.v2);
  92 #define NEON_DO4 \
  93     NEON_FN(vdest.v1, vsrc1.v1, vsrc2.v1); \
  94     NEON_FN(vdest.v2, vsrc1.v2, vsrc2.v2); \
  95     NEON_FN(vdest.v3, vsrc1.v3, vsrc2.v3); \
  96     NEON_FN(vdest.v4, vsrc1.v4, vsrc2.v4);
  97
  98 #define NEON_VOP_BODY(vtype, n) \
  99 { \
 100     uint32_t res; \
 101     vtype vsrc1; \
 102     vtype vsrc2; \
 103     vtype vdest; \
 104     NEON_UNPACK(vtype, vsrc1, arg1); \
 105     NEON_UNPACK(vtype, vsrc2, arg2); \
 106     NEON_DO##n; \
 107     NEON_PACK(vtype, res, vdest); \
 108     return res; \
 109 }
 110
 111 #define NEON_VOP(name, vtype, n) \
 112 uint32_t HELPER(glue(neon_,name))(uint32_t arg1, uint32_t arg2) \
 113 NEON_VOP_BODY(vtype, n)
 114
 115 #define NEON_VOP_ENV(name, vtype, n) \
 116 uint32_t HELPER(glue(neon_,name))(CPUARMState *env, uint32_t arg1, uint32_t arg2) \
 117 NEON_VOP_BODY(vtype, n)
 118
 119 /* Pairwise operations.  */
 120 /* For 32-bit elements each segment only contains a single element, so
 121    the elementwise and pairwise operations are the same.  */
 122 #define NEON_PDO2 \
 123     NEON_FN(vdest.v1, vsrc1.v1, vsrc1.v2); \
 124     NEON_FN(vdest.v2, vsrc2.v1, vsrc2.v2);
 125 #define NEON_PDO4 \
 126     NEON_FN(vdest.v1, vsrc1.v1, vsrc1.v2); \
 127     NEON_FN(vdest.v2, vsrc1.v3, vsrc1.v4); \
 128     NEON_FN(vdest.v3, vsrc2.v1, vsrc2.v2); \
 129     NEON_FN(vdest.v4, vsrc2.v3, vsrc2.v4); \
 130
 131 #define NEON_POP(name, vtype, n) \
 132 uint32_t HELPER(glue(neon_,name))(uint32_t arg1, uint32_t arg2) \
 133 { \
 134     uint32_t res; \
 135     vtype vsrc1; \
 136     vtype vsrc2; \
 137     vtype vdest; \
 138     NEON_UNPACK(vtype, vsrc1, arg1); \
 139     NEON_UNPACK(vtype, vsrc2, arg2); \
 140     NEON_PDO##n; \
 141     NEON_PACK(vtype, res, vdest); \
 142     return res; \
 143 }
 144
 145 /* Unary operators.  */
 146 #define NEON_VOP1(name, vtype, n) \
 147 uint32_t HELPER(glue(neon_,name))(uint32_t arg) \
 148 { \
 149     vtype vsrc1; \
 150     vtype vdest; \
 151     NEON_UNPACK(vtype, vsrc1, arg); \
 152     NEON_DO##n; \
 153     NEON_PACK(vtype, arg, vdest); \
 154     return arg; \
 155 }
 156
 157
 158 #define NEON_USAT(dest, src1, src2, type) do { \
 159     uint32_t tmp = (uint32_t)src1 + (uint32_t)src2; \
 160     if (tmp != (type)tmp) { \
 161         SET_QC(); \
 162         dest = ~0; \
 163     } else { \
 164         dest = tmp; \
 165     }} while(0)
 166 #define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint8_t)
 167 NEON_VOP_ENV(qadd_u8, neon_u8, 4)
 168 #undef NEON_FN
 169 #define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint16_t)
 170 NEON_VOP_ENV(qadd_u16, neon_u16, 2)
 171 #undef NEON_FN
 172 #undef NEON_USAT
 173
 174 uint32_t HELPER(neon_qadd_u32)(CPUARMState *env, uint32_t a, uint32_t b)
 175 {
 176     uint32_t res = a + b;
 177     if (res < a) {
 178         SET_QC();
 179         res = ~0;
 180     }
 181     return res;
 182 }
 183
 184 uint64_t HELPER(neon_qadd_u64)(CPUARMState *env, uint64_t src1, uint64_t src2)
 185 {
 186     uint64_t res;
 187
 188     res = src1 + src2;
 189     if (res < src1) {
 190         SET_QC();
 191         res = ~(uint64_t)0;
 192     }
 193     return res;
 194 }
 195
 196 #define NEON_SSAT(dest, src1, src2, type) do { \
 197     int32_t tmp = (uint32_t)src1 + (uint32_t)src2; \
 198     if (tmp != (type)tmp) { \
 199         SET_QC(); \
 200         if (src2 > 0) { \
 201             tmp = (1 << (sizeof(type) * 8 - 1)) - 1; \
 202         } else { \
 203             tmp = 1 << (sizeof(type) * 8 - 1); \
 204         } \
 205     } \
 206     dest = tmp; \
 207     } while(0)
 208 #define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int8_t)
 209 NEON_VOP_ENV(qadd_s8, neon_s8, 4)
 210 #undef NEON_FN
 211 #define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int16_t)
 212 NEON_VOP_ENV(qadd_s16, neon_s16, 2)
 213 #undef NEON_FN
 214 #undef NEON_SSAT
 215
 216 uint32_t HELPER(neon_qadd_s32)(CPUARMState *env, uint32_t a, uint32_t b)
 217 {
 218     uint32_t res = a + b;
 219     if (((res ^ a) & SIGNBIT) && !((a ^ b) & SIGNBIT)) {
 220         SET_QC();
 221         res = ~(((int32_t)a >> 31) ^ SIGNBIT);
 222     }
 223     return res;
 224 }
 225
 226 uint64_t HELPER(neon_qadd_s64)(CPUARMState *env, uint64_t src1, uint64_t src2)
 227 {
 228     uint64_t res;
 229
 230     res = src1 + src2;
 231     if (((res ^ src1) & SIGNBIT64) && !((src1 ^ src2) & SIGNBIT64)) {
 232         SET_QC();
 233         res = ((int64_t)src1 >> 63) ^ ~SIGNBIT64;
 234     }
 235     return res;
 236 }
 237
 238 /* Unsigned saturating accumulate of signed value
 239  *
 240  * Op1/Rn is treated as signed
 241  * Op2/Rd is treated as unsigned
 242  *
 243  * Explicit casting is used to ensure the correct sign extension of
 244  * inputs. The result is treated as a unsigned value and saturated as such.
 245  *
 246  * We use a macro for the 8/16 bit cases which expects signed integers of va,
 247  * vb, and vr for interim calculation and an unsigned 32 bit result value r.
 248  */
 249
 250 #define USATACC(bits, shift) \
 251     do { \
 252         va = sextract32(a, shift, bits);                                \
 253         vb = extract32(b, shift, bits);                                 \
 254         vr = va + vb;                                                   \
 255         if (vr > UINT##bits##_MAX) {                                    \
 256             SET_QC();                                                   \
 257             vr = UINT##bits##_MAX;                                      \
 258         } else if (vr < 0) {                                            \
 259             SET_QC();                                                   \
 260             vr = 0;                                                     \
 261         }                                                               \
 262         r = deposit32(r, shift, bits, vr);                              \
 263    } while (0)
 264
 265 uint32_t HELPER(neon_uqadd_s8)(CPUARMState *env, uint32_t a, uint32_t b)
 266 {
 267     int16_t va, vb, vr;
 268     uint32_t r = 0;
 269
 270     USATACC(8, 0);
 271     USATACC(8, 8);
 272     USATACC(8, 16);
 273     USATACC(8, 24);
 274     return r;
 275 }
 276
 277 uint32_t HELPER(neon_uqadd_s16)(CPUARMState *env, uint32_t a, uint32_t b)
 278 {
 279     int32_t va, vb, vr;
 280     uint64_t r = 0;
 281
 282     USATACC(16, 0);
 283     USATACC(16, 16);
 284     return r;
 285 }
 286
 287 #undef USATACC
 288
 289 uint32_t HELPER(neon_uqadd_s32)(CPUARMState *env, uint32_t a, uint32_t b)
 290 {
 291     int64_t va = (int32_t)a;
 292     int64_t vb = (uint32_t)b;
 293     int64_t vr = va + vb;
 294     if (vr > UINT32_MAX) {
 295         SET_QC();
 296         vr = UINT32_MAX;
 297     } else if (vr < 0) {
 298         SET_QC();
 299         vr = 0;
 300     }
 301     return vr;
 302 }
 303
 304 uint64_t HELPER(neon_uqadd_s64)(CPUARMState *env, uint64_t a, uint64_t b)
 305 {
 306     uint64_t res;
 307     res = a + b;
 308     /* We only need to look at the pattern of SIGN bits to detect
 309      * +ve/-ve saturation
 310      */
 311     if (~a & b & ~res & SIGNBIT64) {
 312         SET_QC();
 313         res = UINT64_MAX;
 314     } else if (a & ~b & res & SIGNBIT64) {
 315         SET_QC();
 316         res = 0;
 317     }
 318     return res;
 319 }
 320
 321 /* Signed saturating accumulate of unsigned value
 322  *
 323  * Op1/Rn is treated as unsigned
 324  * Op2/Rd is treated as signed
 325  *
 326  * The result is treated as a signed value and saturated as such
 327  *
 328  * We use a macro for the 8/16 bit cases which expects signed integers of va,
 329  * vb, and vr for interim calculation and an unsigned 32 bit result value r.
 330  */
 331
 332 #define SSATACC(bits, shift) \
 333     do { \
 334         va = extract32(a, shift, bits);                                 \
 335         vb = sextract32(b, shift, bits);                                \
 336         vr = va + vb;                                                   \
 337         if (vr > INT##bits##_MAX) {                                     \
 338             SET_QC();                                                   \
 339             vr = INT##bits##_MAX;                                       \
 340         } else if (vr < INT##bits##_MIN) {                              \
 341             SET_QC();                                                   \
 342             vr = INT##bits##_MIN;                                       \
 343         }                                                               \
 344         r = deposit32(r, shift, bits, vr);                              \
 345     } while (0)
 346
 347 uint32_t HELPER(neon_sqadd_u8)(CPUARMState *env, uint32_t a, uint32_t b)
 348 {
 349     int16_t va, vb, vr;
 350     uint32_t r = 0;
 351
 352     SSATACC(8, 0);
 353     SSATACC(8, 8);
 354     SSATACC(8, 16);
 355     SSATACC(8, 24);
 356     return r;
 357 }
 358
 359 uint32_t HELPER(neon_sqadd_u16)(CPUARMState *env, uint32_t a, uint32_t b)
 360 {
 361     int32_t va, vb, vr;
 362     uint32_t r = 0;
 363
 364     SSATACC(16, 0);
 365     SSATACC(16, 16);
 366
 367     return r;
 368 }
 369
 370 #undef SSATACC
 371
 372 uint32_t HELPER(neon_sqadd_u32)(CPUARMState *env, uint32_t a, uint32_t b)
 373 {
 374     int64_t res;
 375     int64_t op1 = (uint32_t)a;
 376     int64_t op2 = (int32_t)b;
 377     res = op1 + op2;
 378     if (res > INT32_MAX) {
 379         SET_QC();
 380         res = INT32_MAX;
 381     } else if (res < INT32_MIN) {
 382         SET_QC();
 383         res = INT32_MIN;
 384     }
 385     return res;
 386 }
 387
 388 uint64_t HELPER(neon_sqadd_u64)(CPUARMState *env, uint64_t a, uint64_t b)
 389 {
 390     uint64_t res;
 391     res = a + b;
 392     /* We only need to look at the pattern of SIGN bits to detect an overflow */
 393     if (((a & res)
 394          | (~b & res)
 395          | (a & ~b)) & SIGNBIT64) {
 396         SET_QC();
 397         res = INT64_MAX;
 398     }
 399     return res;
 400 }
 401
 402
 403 #define NEON_USAT(dest, src1, src2, type) do { \
 404     uint32_t tmp = (uint32_t)src1 - (uint32_t)src2; \
 405     if (tmp != (type)tmp) { \
 406         SET_QC(); \
 407         dest = 0; \
 408     } else { \
 409         dest = tmp; \
 410     }} while(0)
 411 #define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint8_t)
 412 NEON_VOP_ENV(qsub_u8, neon_u8, 4)
 413 #undef NEON_FN
 414 #define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint16_t)
 415 NEON_VOP_ENV(qsub_u16, neon_u16, 2)
 416 #undef NEON_FN
 417 #undef NEON_USAT
 418
 419 uint32_t HELPER(neon_qsub_u32)(CPUARMState *env, uint32_t a, uint32_t b)
 420 {
 421     uint32_t res = a - b;
 422     if (res > a) {
 423         SET_QC();
 424         res = 0;
 425     }
 426     return res;
 427 }
 428
 429 uint64_t HELPER(neon_qsub_u64)(CPUARMState *env, uint64_t src1, uint64_t src2)
 430 {
 431     uint64_t res;
 432
 433     if (src1 < src2) {
 434         SET_QC();
 435         res = 0;
 436     } else {
 437         res = src1 - src2;
 438     }
 439     return res;
 440 }
 441
 442 #define NEON_SSAT(dest, src1, src2, type) do { \
 443     int32_t tmp = (uint32_t)src1 - (uint32_t)src2; \
 444     if (tmp != (type)tmp) { \
 445         SET_QC(); \
 446         if (src2 < 0) { \
 447             tmp = (1 << (sizeof(type) * 8 - 1)) - 1; \
 448         } else { \
 449             tmp = 1 << (sizeof(type) * 8 - 1); \
 450         } \
 451     } \
 452     dest = tmp; \
 453     } while(0)
 454 #define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int8_t)
 455 NEON_VOP_ENV(qsub_s8, neon_s8, 4)
 456 #undef NEON_FN
 457 #define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int16_t)
 458 NEON_VOP_ENV(qsub_s16, neon_s16, 2)
 459 #undef NEON_FN
 460 #undef NEON_SSAT
 461
 462 uint32_t HELPER(neon_qsub_s32)(CPUARMState *env, uint32_t a, uint32_t b)
 463 {
 464     uint32_t res = a - b;
 465     if (((res ^ a) & SIGNBIT) && ((a ^ b) & SIGNBIT)) {
 466         SET_QC();
 467         res = ~(((int32_t)a >> 31) ^ SIGNBIT);
 468     }
 469     return res;
 470 }
 471
 472 uint64_t HELPER(neon_qsub_s64)(CPUARMState *env, uint64_t src1, uint64_t src2)
 473 {
 474     uint64_t res;
 475
 476     res = src1 - src2;
 477     if (((res ^ src1) & SIGNBIT64) && ((src1 ^ src2) & SIGNBIT64)) {
 478         SET_QC();
 479         res = ((int64_t)src1 >> 63) ^ ~SIGNBIT64;
 480     }
 481     return res;
 482 }
 483
 484 #define NEON_FN(dest, src1, src2) dest = (src1 + src2) >> 1
 485 NEON_VOP(hadd_s8, neon_s8, 4)
 486 NEON_VOP(hadd_u8, neon_u8, 4)
 487 NEON_VOP(hadd_s16, neon_s16, 2)
 488 NEON_VOP(hadd_u16, neon_u16, 2)
 489 #undef NEON_FN
 490
 491 int32_t HELPER(neon_hadd_s32)(int32_t src1, int32_t src2)
 492 {
 493     int32_t dest;
 494
 495     dest = (src1 >> 1) + (src2 >> 1);
 496     if (src1 & src2 & 1)
 497         dest++;
 498     return dest;
 499 }
 500
 501 uint32_t HELPER(neon_hadd_u32)(uint32_t src1, uint32_t src2)
 502 {
 503     uint32_t dest;
 504
 505     dest = (src1 >> 1) + (src2 >> 1);
 506     if (src1 & src2 & 1)
 507         dest++;
 508     return dest;
 509 }
 510
 511 #define NEON_FN(dest, src1, src2) dest = (src1 + src2 + 1) >> 1
 512 NEON_VOP(rhadd_s8, neon_s8, 4)
 513 NEON_VOP(rhadd_u8, neon_u8, 4)
 514 NEON_VOP(rhadd_s16, neon_s16, 2)
 515 NEON_VOP(rhadd_u16, neon_u16, 2)
 516 #undef NEON_FN
 517
 518 int32_t HELPER(neon_rhadd_s32)(int32_t src1, int32_t src2)
 519 {
 520     int32_t dest;
 521
 522     dest = (src1 >> 1) + (src2 >> 1);
 523     if ((src1 | src2) & 1)
 524         dest++;
 525     return dest;
 526 }
 527
 528 uint32_t HELPER(neon_rhadd_u32)(uint32_t src1, uint32_t src2)
 529 {
 530     uint32_t dest;
 531
 532     dest = (src1 >> 1) + (src2 >> 1);
 533     if ((src1 | src2) & 1)
 534         dest++;
 535     return dest;
 536 }
 537
 538 #define NEON_FN(dest, src1, src2) dest = (src1 - src2) >> 1
 539 NEON_VOP(hsub_s8, neon_s8, 4)
 540 NEON_VOP(hsub_u8, neon_u8, 4)
 541 NEON_VOP(hsub_s16, neon_s16, 2)
 542 NEON_VOP(hsub_u16, neon_u16, 2)
 543 #undef NEON_FN
 544
 545 int32_t HELPER(neon_hsub_s32)(int32_t src1, int32_t src2)
 546 {
 547     int32_t dest;
 548
 549     dest = (src1 >> 1) - (src2 >> 1);
 550     if ((~src1) & src2 & 1)
 551         dest--;
 552     return dest;
 553 }
 554
 555 uint32_t HELPER(neon_hsub_u32)(uint32_t src1, uint32_t src2)
 556 {
 557     uint32_t dest;
 558
 559     dest = (src1 >> 1) - (src2 >> 1);
 560     if ((~src1) & src2 & 1)
 561         dest--;
 562     return dest;
 563 }
 564
 565 #define NEON_FN(dest, src1, src2) dest = (src1 > src2) ? ~0 : 0
 566 NEON_VOP(cgt_s8, neon_s8, 4)
 567 NEON_VOP(cgt_u8, neon_u8, 4)
 568 NEON_VOP(cgt_s16, neon_s16, 2)
 569 NEON_VOP(cgt_u16, neon_u16, 2)
 570 NEON_VOP(cgt_s32, neon_s32, 1)
 571 NEON_VOP(cgt_u32, neon_u32, 1)
 572 #undef NEON_FN
 573
 574 #define NEON_FN(dest, src1, src2) dest = (src1 >= src2) ? ~0 : 0
 575 NEON_VOP(cge_s8, neon_s8, 4)
 576 NEON_VOP(cge_u8, neon_u8, 4)
 577 NEON_VOP(cge_s16, neon_s16, 2)
 578 NEON_VOP(cge_u16, neon_u16, 2)
 579 NEON_VOP(cge_s32, neon_s32, 1)
 580 NEON_VOP(cge_u32, neon_u32, 1)
 581 #undef NEON_FN
 582
 583 #define NEON_FN(dest, src1, src2) dest = (src1 < src2) ? src1 : src2
 584 NEON_VOP(min_s8, neon_s8, 4)
 585 NEON_VOP(min_u8, neon_u8, 4)
 586 NEON_VOP(min_s16, neon_s16, 2)
 587 NEON_VOP(min_u16, neon_u16, 2)
 588 NEON_VOP(min_s32, neon_s32, 1)
 589 NEON_VOP(min_u32, neon_u32, 1)
 590 NEON_POP(pmin_s8, neon_s8, 4)
 591 NEON_POP(pmin_u8, neon_u8, 4)
 592 NEON_POP(pmin_s16, neon_s16, 2)
 593 NEON_POP(pmin_u16, neon_u16, 2)
 594 #undef NEON_FN
 595
 596 #define NEON_FN(dest, src1, src2) dest = (src1 > src2) ? src1 : src2
 597 NEON_VOP(max_s8, neon_s8, 4)
 598 NEON_VOP(max_u8, neon_u8, 4)
 599 NEON_VOP(max_s16, neon_s16, 2)
 600 NEON_VOP(max_u16, neon_u16, 2)
 601 NEON_VOP(max_s32, neon_s32, 1)
 602 NEON_VOP(max_u32, neon_u32, 1)
 603 NEON_POP(pmax_s8, neon_s8, 4)
 604 NEON_POP(pmax_u8, neon_u8, 4)
 605 NEON_POP(pmax_s16, neon_s16, 2)
 606 NEON_POP(pmax_u16, neon_u16, 2)
 607 #undef NEON_FN
 608
 609 #define NEON_FN(dest, src1, src2) \
 610     dest = (src1 > src2) ? (src1 - src2) : (src2 - src1)
 611 NEON_VOP(abd_s8, neon_s8, 4)
 612 NEON_VOP(abd_u8, neon_u8, 4)
 613 NEON_VOP(abd_s16, neon_s16, 2)
 614 NEON_VOP(abd_u16, neon_u16, 2)
 615 NEON_VOP(abd_s32, neon_s32, 1)
 616 NEON_VOP(abd_u32, neon_u32, 1)
 617 #undef NEON_FN
 618
 619 #define NEON_FN(dest, src1, src2) do { \
 620     int8_t tmp; \
 621     tmp = (int8_t)src2; \
 622     if (tmp >= (ssize_t)sizeof(src1) * 8 || \
 623         tmp <= -(ssize_t)sizeof(src1) * 8) { \
 624         dest = 0; \
 625     } else if (tmp < 0) { \
 626         dest = src1 >> -tmp; \
 627     } else { \
 628         dest = src1 << tmp; \
 629     }} while (0)
 630 NEON_VOP(shl_u8, neon_u8, 4)
 631 NEON_VOP(shl_u16, neon_u16, 2)
 632 NEON_VOP(shl_u32, neon_u32, 1)
 633 #undef NEON_FN
 634
 635 uint64_t HELPER(neon_shl_u64)(uint64_t val, uint64_t shiftop)
 636 {
 637     int8_t shift = (int8_t)shiftop;
 638     if (shift >= 64 || shift <= -64) {
 639         val = 0;
 640     } else if (shift < 0) {
 641         val >>= -shift;
 642     } else {
 643         val <<= shift;
 644     }
 645     return val;
 646 }
 647
 648 #define NEON_FN(dest, src1, src2) do { \
 649     int8_t tmp; \
 650     tmp = (int8_t)src2; \
 651     if (tmp >= (ssize_t)sizeof(src1) * 8) { \
 652         dest = 0; \
 653     } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
 654         dest = src1 >> (sizeof(src1) * 8 - 1); \
 655     } else if (tmp < 0) { \
 656         dest = src1 >> -tmp; \
 657     } else { \
 658         dest = src1 << tmp; \
 659     }} while (0)
 660 NEON_VOP(shl_s8, neon_s8, 4)
 661 NEON_VOP(shl_s16, neon_s16, 2)
 662 NEON_VOP(shl_s32, neon_s32, 1)
 663 #undef NEON_FN
 664
 665 uint64_t HELPER(neon_shl_s64)(uint64_t valop, uint64_t shiftop)
 666 {
 667     int8_t shift = (int8_t)shiftop;
 668     int64_t val = valop;
 669     if (shift >= 64) {
 670         val = 0;
 671     } else if (shift <= -64) {
 672         val >>= 63;
 673     } else if (shift < 0) {
 674         val >>= -shift;
 675     } else {
 676         val <<= shift;
 677     }
 678     return val;
 679 }
 680
 681 #define NEON_FN(dest, src1, src2) do { \
 682     int8_t tmp; \
 683     tmp = (int8_t)src2; \
 684     if ((tmp >= (ssize_t)sizeof(src1) * 8) \
 685         || (tmp <= -(ssize_t)sizeof(src1) * 8)) { \
 686         dest = 0; \
 687     } else if (tmp < 0) { \
 688         dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
 689     } else { \
 690         dest = src1 << tmp; \
 691     }} while (0)
 692 NEON_VOP(rshl_s8, neon_s8, 4)
 693 NEON_VOP(rshl_s16, neon_s16, 2)
 694 #undef NEON_FN
 695
 696 /* The addition of the rounding constant may overflow, so we use an
 697  * intermediate 64 bit accumulator.  */
 698 uint32_t HELPER(neon_rshl_s32)(uint32_t valop, uint32_t shiftop)
 699 {
 700     int32_t dest;
 701     int32_t val = (int32_t)valop;
 702     int8_t shift = (int8_t)shiftop;
 703     if ((shift >= 32) || (shift <= -32)) {
 704         dest = 0;
 705     } else if (shift < 0) {
 706         int64_t big_dest = ((int64_t)val + (1 << (-1 - shift)));
 707         dest = big_dest >> -shift;
 708     } else {
 709         dest = val << shift;
 710     }
 711     return dest;
 712 }
 713
 714 /* Handling addition overflow with 64 bit input values is more
 715  * tricky than with 32 bit values.  */
 716 uint64_t HELPER(neon_rshl_s64)(uint64_t valop, uint64_t shiftop)
 717 {
 718     int8_t shift = (int8_t)shiftop;
 719     int64_t val = valop;
 720     if ((shift >= 64) || (shift <= -64)) {
 721         val = 0;
 722     } else if (shift < 0) {
 723         val >>= (-shift - 1);
 724         if (val == INT64_MAX) {
 725             /* In this case, it means that the rounding constant is 1,
 726              * and the addition would overflow. Return the actual
 727              * result directly.  */
 728             val = 0x4000000000000000LL;
 729         } else {
 730             val++;
 731             val >>= 1;
 732         }
 733     } else {
 734         val <<= shift;
 735     }
 736     return val;
 737 }
 738
 739 #define NEON_FN(dest, src1, src2) do { \
 740     int8_t tmp; \
 741     tmp = (int8_t)src2; \
 742     if (tmp >= (ssize_t)sizeof(src1) * 8 || \
 743         tmp < -(ssize_t)sizeof(src1) * 8) { \
 744         dest = 0; \
 745     } else if (tmp == -(ssize_t)sizeof(src1) * 8) { \
 746         dest = src1 >> (-tmp - 1); \
 747     } else if (tmp < 0) { \
 748         dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
 749     } else { \
 750         dest = src1 << tmp; \
 751     }} while (0)
 752 NEON_VOP(rshl_u8, neon_u8, 4)
 753 NEON_VOP(rshl_u16, neon_u16, 2)
 754 #undef NEON_FN
 755
 756 /* The addition of the rounding constant may overflow, so we use an
 757  * intermediate 64 bit accumulator.  */
 758 uint32_t HELPER(neon_rshl_u32)(uint32_t val, uint32_t shiftop)
 759 {
 760     uint32_t dest;
 761     int8_t shift = (int8_t)shiftop;
 762     if (shift >= 32 || shift < -32) {
 763         dest = 0;
 764     } else if (shift == -32) {
 765         dest = val >> 31;
 766     } else if (shift < 0) {
 767         uint64_t big_dest = ((uint64_t)val + (1 << (-1 - shift)));
 768         dest = big_dest >> -shift;
 769     } else {
 770         dest = val << shift;
 771     }
 772     return dest;
 773 }
 774
 775 /* Handling addition overflow with 64 bit input values is more
 776  * tricky than with 32 bit values.  */
 777 uint64_t HELPER(neon_rshl_u64)(uint64_t val, uint64_t shiftop)
 778 {
 779     int8_t shift = (uint8_t)shiftop;
 780     if (shift >= 64 || shift < -64) {
 781         val = 0;
 782     } else if (shift == -64) {
 783         /* Rounding a 1-bit result just preserves that bit.  */
 784         val >>= 63;
 785     } else if (shift < 0) {
 786         val >>= (-shift - 1);
 787         if (val == UINT64_MAX) {
 788             /* In this case, it means that the rounding constant is 1,
 789              * and the addition would overflow. Return the actual
 790              * result directly.  */
 791             val = 0x8000000000000000ULL;
 792         } else {
 793             val++;
 794             val >>= 1;
 795         }
 796     } else {
 797         val <<= shift;
 798     }
 799     return val;
 800 }
 801
 802 #define NEON_FN(dest, src1, src2) do { \
 803     int8_t tmp; \
 804     tmp = (int8_t)src2; \
 805     if (tmp >= (ssize_t)sizeof(src1) * 8) { \
 806         if (src1) { \
 807             SET_QC(); \
 808             dest = ~0; \
 809         } else { \
 810             dest = 0; \
 811         } \
 812     } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
 813         dest = 0; \
 814     } else if (tmp < 0) { \
 815         dest = src1 >> -tmp; \
 816     } else { \
 817         dest = src1 << tmp; \
 818         if ((dest >> tmp) != src1) { \
 819             SET_QC(); \
 820             dest = ~0; \
 821         } \
 822     }} while (0)
 823 NEON_VOP_ENV(qshl_u8, neon_u8, 4)
 824 NEON_VOP_ENV(qshl_u16, neon_u16, 2)
 825 NEON_VOP_ENV(qshl_u32, neon_u32, 1)
 826 #undef NEON_FN
 827
 828 uint64_t HELPER(neon_qshl_u64)(CPUARMState *env, uint64_t val, uint64_t shiftop)
 829 {
 830     int8_t shift = (int8_t)shiftop;
 831     if (shift >= 64) {
 832         if (val) {
 833             val = ~(uint64_t)0;
 834             SET_QC();
 835         }
 836     } else if (shift <= -64) {
 837         val = 0;
 838     } else if (shift < 0) {
 839         val >>= -shift;
 840     } else {
 841         uint64_t tmp = val;
 842         val <<= shift;
 843         if ((val >> shift) != tmp) {
 844             SET_QC();
 845             val = ~(uint64_t)0;
 846         }
 847     }
 848     return val;
 849 }
 850
 851 #define NEON_FN(dest, src1, src2) do { \
 852     int8_t tmp; \
 853     tmp = (int8_t)src2; \
 854     if (tmp >= (ssize_t)sizeof(src1) * 8) { \
 855         if (src1) { \
 856             SET_QC(); \
 857             dest = (uint32_t)(1 << (sizeof(src1) * 8 - 1)); \
 858             if (src1 > 0) { \
 859                 dest--; \
 860             } \
 861         } else { \
 862             dest = src1; \
 863         } \
 864     } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
 865         dest = src1 >> 31; \
 866     } else if (tmp < 0) { \
 867         dest = src1 >> -tmp; \
 868     } else { \
 869         dest = src1 << tmp; \
 870         if ((dest >> tmp) != src1) { \
 871             SET_QC(); \
 872             dest = (uint32_t)(1 << (sizeof(src1) * 8 - 1)); \
 873             if (src1 > 0) { \
 874                 dest--; \
 875             } \
 876         } \
 877     }} while (0)
 878 NEON_VOP_ENV(qshl_s8, neon_s8, 4)
 879 NEON_VOP_ENV(qshl_s16, neon_s16, 2)
 880 NEON_VOP_ENV(qshl_s32, neon_s32, 1)
 881 #undef NEON_FN
 882
 883 uint64_t HELPER(neon_qshl_s64)(CPUARMState *env, uint64_t valop, uint64_t shiftop)
 884 {
 885     int8_t shift = (uint8_t)shiftop;
 886     int64_t val = valop;
 887     if (shift >= 64) {
 888         if (val) {
 889             SET_QC();
 890             val = (val >> 63) ^ ~SIGNBIT64;
 891         }
 892     } else if (shift <= -64) {
 893         val >>= 63;
 894     } else if (shift < 0) {
 895         val >>= -shift;
 896     } else {
 897         int64_t tmp = val;
 898         val <<= shift;
 899         if ((val >> shift) != tmp) {
 900             SET_QC();
 901             val = (tmp >> 63) ^ ~SIGNBIT64;
 902         }
 903     }
 904     return val;
 905 }
 906
 907 #define NEON_FN(dest, src1, src2) do { \
 908     if (src1 & (1 << (sizeof(src1) * 8 - 1))) { \
 909         SET_QC(); \
 910         dest = 0; \
 911     } else { \
 912         int8_t tmp; \
 913         tmp = (int8_t)src2; \
 914         if (tmp >= (ssize_t)sizeof(src1) * 8) { \
 915             if (src1) { \
 916                 SET_QC(); \
 917                 dest = ~0; \
 918             } else { \
 919                 dest = 0; \
 920             } \
 921         } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
 922             dest = 0; \
 923         } else if (tmp < 0) { \
 924             dest = src1 >> -tmp; \
 925         } else { \
 926             dest = src1 << tmp; \
 927             if ((dest >> tmp) != src1) { \
 928                 SET_QC(); \
 929                 dest = ~0; \
 930             } \
 931         } \
 932     }} while (0)
 933 NEON_VOP_ENV(qshlu_s8, neon_u8, 4)
 934 NEON_VOP_ENV(qshlu_s16, neon_u16, 2)
 935 #undef NEON_FN
 936
 937 uint32_t HELPER(neon_qshlu_s32)(CPUARMState *env, uint32_t valop, uint32_t shiftop)
 938 {
 939     if ((int32_t)valop < 0) {
 940         SET_QC();
 941         return 0;
 942     }
 943     return helper_neon_qshl_u32(env, valop, shiftop);
 944 }
 945
 946 uint64_t HELPER(neon_qshlu_s64)(CPUARMState *env, uint64_t valop, uint64_t shiftop)
 947 {
 948     if ((int64_t)valop < 0) {
 949         SET_QC();
 950         return 0;
 951     }
 952     return helper_neon_qshl_u64(env, valop, shiftop);
 953 }
 954
 955 #define NEON_FN(dest, src1, src2) do { \
 956     int8_t tmp; \
 957     tmp = (int8_t)src2; \
 958     if (tmp >= (ssize_t)sizeof(src1) * 8) { \
 959         if (src1) { \
 960             SET_QC(); \
 961             dest = ~0; \
 962         } else { \
 963             dest = 0; \
 964         } \
 965     } else if (tmp < -(ssize_t)sizeof(src1) * 8) { \
 966         dest = 0; \
 967     } else if (tmp == -(ssize_t)sizeof(src1) * 8) { \
 968         dest = src1 >> (sizeof(src1) * 8 - 1); \
 969     } else if (tmp < 0) { \
 970         dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
 971     } else { \
 972         dest = src1 << tmp; \
 973         if ((dest >> tmp) != src1) { \
 974             SET_QC(); \
 975             dest = ~0; \
 976         } \
 977     }} while (0)
 978 NEON_VOP_ENV(qrshl_u8, neon_u8, 4)
 979 NEON_VOP_ENV(qrshl_u16, neon_u16, 2)
 980 #undef NEON_FN
 981
 982 /* The addition of the rounding constant may overflow, so we use an
 983  * intermediate 64 bit accumulator.  */
 984 uint32_t HELPER(neon_qrshl_u32)(CPUARMState *env, uint32_t val, uint32_t shiftop)
 985 {
 986     uint32_t dest;
 987     int8_t shift = (int8_t)shiftop;
 988     if (shift >= 32) {
 989         if (val) {
 990             SET_QC();
 991             dest = ~0;
 992         } else {
 993             dest = 0;
 994         }
 995     } else if (shift < -32) {
 996         dest = 0;
 997     } else if (shift == -32) {
 998         dest = val >> 31;
 999     } else if (shift < 0) {
1000         uint64_t big_dest = ((uint64_t)val + (1 << (-1 - shift)));
1001         dest = big_dest >> -shift;
1002     } else {
1003         dest = val << shift;
1004         if ((dest >> shift) != val) {
1005             SET_QC();
1006             dest = ~0;
1007         }
1008     }
1009     return dest;
1010 }
1011
1012 /* Handling addition overflow with 64 bit input values is more
1013  * tricky than with 32 bit values.  */
1014 uint64_t HELPER(neon_qrshl_u64)(CPUARMState *env, uint64_t val, uint64_t shiftop)
1015 {
1016     int8_t shift = (int8_t)shiftop;
1017     if (shift >= 64) {
1018         if (val) {
1019             SET_QC();
1020             val = ~0;
1021         }
1022     } else if (shift < -64) {
1023         val = 0;
1024     } else if (shift == -64) {
1025         val >>= 63;
1026     } else if (shift < 0) {
1027         val >>= (-shift - 1);
1028         if (val == UINT64_MAX) {
1029             /* In this case, it means that the rounding constant is 1,
1030              * and the addition would overflow. Return the actual
1031              * result directly.  */
1032             val = 0x8000000000000000ULL;
1033         } else {
1034             val++;
1035             val >>= 1;
1036         }
1037     } else { \
1038         uint64_t tmp = val;
1039         val <<= shift;
1040         if ((val >> shift) != tmp) {
1041             SET_QC();
1042             val = ~0;
1043         }
1044     }
1045     return val;
1046 }
1047
1048 #define NEON_FN(dest, src1, src2) do { \
1049     int8_t tmp; \
1050     tmp = (int8_t)src2; \
1051     if (tmp >= (ssize_t)sizeof(src1) * 8) { \
1052         if (src1) { \
1053             SET_QC(); \
1054             dest = (typeof(dest))(1 << (sizeof(src1) * 8 - 1)); \
1055             if (src1 > 0) { \
1056                 dest--; \
1057             } \
1058         } else { \
1059             dest = 0; \
1060         } \
1061     } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
1062         dest = 0; \
1063     } else if (tmp < 0) { \
1064         dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
1065     } else { \
1066         dest = src1 << tmp; \
1067         if ((dest >> tmp) != src1) { \
1068             SET_QC(); \
1069             dest = (uint32_t)(1 << (sizeof(src1) * 8 - 1)); \
1070             if (src1 > 0) { \
1071                 dest--; \
1072             } \
1073         } \
1074     }} while (0)
1075 NEON_VOP_ENV(qrshl_s8, neon_s8, 4)
1076 NEON_VOP_ENV(qrshl_s16, neon_s16, 2)
1077 #undef NEON_FN
1078
1079 /* The addition of the rounding constant may overflow, so we use an
1080  * intermediate 64 bit accumulator.  */
1081 uint32_t HELPER(neon_qrshl_s32)(CPUARMState *env, uint32_t valop, uint32_t shiftop)
1082 {
1083     int32_t dest;
1084     int32_t val = (int32_t)valop;
1085     int8_t shift = (int8_t)shiftop;
1086     if (shift >= 32) {
1087         if (val) {
1088             SET_QC();
1089             dest = (val >> 31) ^ ~SIGNBIT;
1090         } else {
1091             dest = 0;
1092         }
1093     } else if (shift <= -32) {
1094         dest = 0;
1095     } else if (shift < 0) {
1096         int64_t big_dest = ((int64_t)val + (1 << (-1 - shift)));
1097         dest = big_dest >> -shift;
1098     } else {
1099         dest = val << shift;
1100         if ((dest >> shift) != val) {
1101             SET_QC();
1102             dest = (val >> 31) ^ ~SIGNBIT;
1103         }
1104     }
1105     return dest;
1106 }
1107
1108 /* Handling addition overflow with 64 bit input values is more
1109  * tricky than with 32 bit values.  */
1110 uint64_t HELPER(neon_qrshl_s64)(CPUARMState *env, uint64_t valop, uint64_t shiftop)
1111 {
1112     int8_t shift = (uint8_t)shiftop;
1113     int64_t val = valop;
1114
1115     if (shift >= 64) {
1116         if (val) {
1117             SET_QC();
1118             val = (val >> 63) ^ ~SIGNBIT64;
1119         }
1120     } else if (shift <= -64) {
1121         val = 0;
1122     } else if (shift < 0) {
1123         val >>= (-shift - 1);
1124         if (val == INT64_MAX) {
1125             /* In this case, it means that the rounding constant is 1,
1126              * and the addition would overflow. Return the actual
1127              * result directly.  */
1128             val = 0x4000000000000000ULL;
1129         } else {
1130             val++;
1131             val >>= 1;
1132         }
1133     } else {
1134         int64_t tmp = val;
1135         val <<= shift;
1136         if ((val >> shift) != tmp) {
1137             SET_QC();
1138             val = (tmp >> 63) ^ ~SIGNBIT64;
1139         }
1140     }
1141     return val;
1142 }
1143
1144 uint32_t HELPER(neon_add_u8)(uint32_t a, uint32_t b)
1145 {
1146     uint32_t mask;
1147     mask = (a ^ b) & 0x80808080u;
1148     a &= ~0x80808080u;
1149     b &= ~0x80808080u;
1150     return (a + b) ^ mask;
1151 }
1152
1153 uint32_t HELPER(neon_add_u16)(uint32_t a, uint32_t b)
1154 {
1155     uint32_t mask;
1156     mask = (a ^ b) & 0x80008000u;
1157     a &= ~0x80008000u;
1158     b &= ~0x80008000u;
1159     return (a + b) ^ mask;
1160 }
1161
1162 #define NEON_FN(dest, src1, src2) dest = src1 + src2
1163 NEON_POP(padd_u8, neon_u8, 4)
1164 NEON_POP(padd_u16, neon_u16, 2)
1165 #undef NEON_FN
1166
1167 #define NEON_FN(dest, src1, src2) dest = src1 - src2
1168 NEON_VOP(sub_u8, neon_u8, 4)
1169 NEON_VOP(sub_u16, neon_u16, 2)
1170 #undef NEON_FN
1171
1172 #define NEON_FN(dest, src1, src2) dest = src1 * src2
1173 NEON_VOP(mul_u8, neon_u8, 4)
1174 NEON_VOP(mul_u16, neon_u16, 2)
1175 #undef NEON_FN
1176
1177 /* Polynomial multiplication is like integer multiplication except the
1178    partial products are XORed, not added.  */
1179 uint32_t HELPER(neon_mul_p8)(uint32_t op1, uint32_t op2)
1180 {
1181     uint32_t mask;
1182     uint32_t result;
1183     result = 0;
1184     while (op1) {
1185         mask = 0;
1186         if (op1 & 1)
1187             mask |= 0xff;
1188         if (op1 & (1 << 8))
1189             mask |= (0xff << 8);
1190         if (op1 & (1 << 16))
1191             mask |= (0xff << 16);
1192         if (op1 & (1 << 24))
1193             mask |= (0xff << 24);
1194         result ^= op2 & mask;
1195         op1 = (op1 >> 1) & 0x7f7f7f7f;
1196         op2 = (op2 << 1) & 0xfefefefe;
1197     }
1198     return result;
1199 }
1200
1201 uint64_t HELPER(neon_mull_p8)(uint32_t op1, uint32_t op2)
1202 {
1203     uint64_t result = 0;
1204     uint64_t mask;
1205     uint64_t op2ex = op2;
1206     op2ex = (op2ex & 0xff) |
1207         ((op2ex & 0xff00) << 8) |
1208         ((op2ex & 0xff0000) << 16) |
1209         ((op2ex & 0xff000000) << 24);
1210     while (op1) {
1211         mask = 0;
1212         if (op1 & 1) {
1213             mask |= 0xffff;
1214         }
1215         if (op1 & (1 << 8)) {
1216             mask |= (0xffffU << 16);
1217         }
1218         if (op1 & (1 << 16)) {
1219             mask |= (0xffffULL << 32);
1220         }
1221         if (op1 & (1 << 24)) {
1222             mask |= (0xffffULL << 48);
1223         }
1224         result ^= op2ex & mask;
1225         op1 = (op1 >> 1) & 0x7f7f7f7f;
1226         op2ex <<= 1;
1227     }
1228     return result;
1229 }
1230
1231 #define NEON_FN(dest, src1, src2) dest = (src1 & src2) ? -1 : 0
1232 NEON_VOP(tst_u8, neon_u8, 4)
1233 NEON_VOP(tst_u16, neon_u16, 2)
1234 NEON_VOP(tst_u32, neon_u32, 1)
1235 #undef NEON_FN
1236
1237 #define NEON_FN(dest, src1, src2) dest = (src1 == src2) ? -1 : 0
1238 NEON_VOP(ceq_u8, neon_u8, 4)
1239 NEON_VOP(ceq_u16, neon_u16, 2)
1240 NEON_VOP(ceq_u32, neon_u32, 1)
1241 #undef NEON_FN
1242
1243 #define NEON_FN(dest, src, dummy) dest = (src < 0) ? -src : src
1244 NEON_VOP1(abs_s8, neon_s8, 4)
1245 NEON_VOP1(abs_s16, neon_s16, 2)
1246 #undef NEON_FN
1247
1248 /* Count Leading Sign/Zero Bits.  */
1249 static inline int do_clz8(uint8_t x)
1250 {
1251     int n;
1252     for (n = 8; x; n--)
1253         x >>= 1;
1254     return n;
1255 }
1256
1257 static inline int do_clz16(uint16_t x)
1258 {
1259     int n;
1260     for (n = 16; x; n--)
1261         x >>= 1;
1262     return n;
1263 }
1264
1265 #define NEON_FN(dest, src, dummy) dest = do_clz8(src)
1266 NEON_VOP1(clz_u8, neon_u8, 4)
1267 #undef NEON_FN
1268
1269 #define NEON_FN(dest, src, dummy) dest = do_clz16(src)
1270 NEON_VOP1(clz_u16, neon_u16, 2)
1271 #undef NEON_FN
1272
1273 #define NEON_FN(dest, src, dummy) dest = do_clz8((src < 0) ? ~src : src) - 1
1274 NEON_VOP1(cls_s8, neon_s8, 4)
1275 #undef NEON_FN
1276
1277 #define NEON_FN(dest, src, dummy) dest = do_clz16((src < 0) ? ~src : src) - 1
1278 NEON_VOP1(cls_s16, neon_s16, 2)
1279 #undef NEON_FN
1280
1281 uint32_t HELPER(neon_cls_s32)(uint32_t x)
1282 {
1283     int count;
1284     if ((int32_t)x < 0)
1285         x = ~x;
1286     for (count = 32; x; count--)
1287         x = x >> 1;
1288     return count - 1;
1289 }
1290
1291 /* Bit count.  */
1292 uint32_t HELPER(neon_cnt_u8)(uint32_t x)
1293 {
1294     x = (x & 0x55555555) + ((x >>  1) & 0x55555555);
1295     x = (x & 0x33333333) + ((x >>  2) & 0x33333333);
1296     x = (x & 0x0f0f0f0f) + ((x >>  4) & 0x0f0f0f0f);
1297     return x;
1298 }
1299
1300 /* Reverse bits in each 8 bit word */
1301 uint32_t HELPER(neon_rbit_u8)(uint32_t x)
1302 {
1303     x =  ((x & 0xf0f0f0f0) >> 4)
1304        | ((x & 0x0f0f0f0f) << 4);
1305     x =  ((x & 0x88888888) >> 3)
1306        | ((x & 0x44444444) >> 1)
1307        | ((x & 0x22222222) << 1)
1308        | ((x & 0x11111111) << 3);
1309     return x;
1310 }
1311
1312 #define NEON_QDMULH16(dest, src1, src2, round) do { \
1313     uint32_t tmp = (int32_t)(int16_t) src1 * (int16_t) src2; \
1314     if ((tmp ^ (tmp << 1)) & SIGNBIT) { \
1315         SET_QC(); \
1316         tmp = (tmp >> 31) ^ ~SIGNBIT; \
1317     } else { \
1318         tmp <<= 1; \
1319     } \
1320     if (round) { \
1321         int32_t old = tmp; \
1322         tmp += 1 << 15; \
1323         if ((int32_t)tmp < old) { \
1324             SET_QC(); \
1325             tmp = SIGNBIT - 1; \
1326         } \
1327     } \
1328     dest = tmp >> 16; \
1329     } while(0)
1330 #define NEON_FN(dest, src1, src2) NEON_QDMULH16(dest, src1, src2, 0)
1331 NEON_VOP_ENV(qdmulh_s16, neon_s16, 2)
1332 #undef NEON_FN
1333 #define NEON_FN(dest, src1, src2) NEON_QDMULH16(dest, src1, src2, 1)
1334 NEON_VOP_ENV(qrdmulh_s16, neon_s16, 2)
1335 #undef NEON_FN
1336 #undef NEON_QDMULH16
1337
1338 #define NEON_QDMULH32(dest, src1, src2, round) do { \
1339     uint64_t tmp = (int64_t)(int32_t) src1 * (int32_t) src2; \
1340     if ((tmp ^ (tmp << 1)) & SIGNBIT64) { \
1341         SET_QC(); \
1342         tmp = (tmp >> 63) ^ ~SIGNBIT64; \
1343     } else { \
1344         tmp <<= 1; \
1345     } \
1346     if (round) { \
1347         int64_t old = tmp; \
1348         tmp += (int64_t)1 << 31; \
1349         if ((int64_t)tmp < old) { \
1350             SET_QC(); \
1351             tmp = SIGNBIT64 - 1; \
1352         } \
1353     } \
1354     dest = tmp >> 32; \
1355     } while(0)
1356 #define NEON_FN(dest, src1, src2) NEON_QDMULH32(dest, src1, src2, 0)
1357 NEON_VOP_ENV(qdmulh_s32, neon_s32, 1)
1358 #undef NEON_FN
1359 #define NEON_FN(dest, src1, src2) NEON_QDMULH32(dest, src1, src2, 1)
1360 NEON_VOP_ENV(qrdmulh_s32, neon_s32, 1)
1361 #undef NEON_FN
1362 #undef NEON_QDMULH32
1363
1364 uint32_t HELPER(neon_narrow_u8)(uint64_t x)
1365 {
1366     return (x & 0xffu) | ((x >> 8) & 0xff00u) | ((x >> 16) & 0xff0000u)
1367            | ((x >> 24) & 0xff000000u);
1368 }
1369
1370 uint32_t HELPER(neon_narrow_u16)(uint64_t x)
1371 {
1372     return (x & 0xffffu) | ((x >> 16) & 0xffff0000u);
1373 }
1374
1375 uint32_t HELPER(neon_narrow_high_u8)(uint64_t x)
1376 {
1377     return ((x >> 8) & 0xff) | ((x >> 16) & 0xff00)
1378             | ((x >> 24) & 0xff0000) | ((x >> 32) & 0xff000000);
1379 }
1380
1381 uint32_t HELPER(neon_narrow_high_u16)(uint64_t x)
1382 {
1383     return ((x >> 16) & 0xffff) | ((x >> 32) & 0xffff0000);
1384 }
1385
1386 uint32_t HELPER(neon_narrow_round_high_u8)(uint64_t x)
1387 {
1388     x &= 0xff80ff80ff80ff80ull;
1389     x += 0x0080008000800080ull;
1390     return ((x >> 8) & 0xff) | ((x >> 16) & 0xff00)
1391             | ((x >> 24) & 0xff0000) | ((x >> 32) & 0xff000000);
1392 }
1393
1394 uint32_t HELPER(neon_narrow_round_high_u16)(uint64_t x)
1395 {
1396     x &= 0xffff8000ffff8000ull;
1397     x += 0x0000800000008000ull;
1398     return ((x >> 16) & 0xffff) | ((x >> 32) & 0xffff0000);
1399 }
1400
1401 uint32_t HELPER(neon_unarrow_sat8)(CPUARMState *env, uint64_t x)
1402 {
1403     uint16_t s;
1404     uint8_t d;
1405     uint32_t res = 0;
1406 #define SAT8(n) \
1407     s = x >> n; \
1408     if (s & 0x8000) { \
1409         SET_QC(); \
1410     } else { \
1411         if (s > 0xff) { \
1412             d = 0xff; \
1413             SET_QC(); \
1414         } else  { \
1415             d = s; \
1416         } \
1417         res |= (uint32_t)d << (n / 2); \
1418     }
1419
1420     SAT8(0);
1421     SAT8(16);
1422     SAT8(32);
1423     SAT8(48);
1424 #undef SAT8
1425     return res;
1426 }
1427
1428 uint32_t HELPER(neon_narrow_sat_u8)(CPUARMState *env, uint64_t x)
1429 {
1430     uint16_t s;
1431     uint8_t d;
1432     uint32_t res = 0;
1433 #define SAT8(n) \
1434     s = x >> n; \
1435     if (s > 0xff) { \
1436         d = 0xff; \
1437         SET_QC(); \
1438     } else  { \
1439         d = s; \
1440     } \
1441     res |= (uint32_t)d << (n / 2);
1442
1443     SAT8(0);
1444     SAT8(16);
1445     SAT8(32);
1446     SAT8(48);
1447 #undef SAT8
1448     return res;
1449 }
1450
1451 uint32_t HELPER(neon_narrow_sat_s8)(CPUARMState *env, uint64_t x)
1452 {
1453     int16_t s;
1454     uint8_t d;
1455     uint32_t res = 0;
1456 #define SAT8(n) \
1457     s = x >> n; \
1458     if (s != (int8_t)s) { \
1459         d = (s >> 15) ^ 0x7f; \
1460         SET_QC(); \
1461     } else  { \
1462         d = s; \
1463     } \
1464     res |= (uint32_t)d << (n / 2);
1465
1466     SAT8(0);
1467     SAT8(16);
1468     SAT8(32);
1469     SAT8(48);
1470 #undef SAT8
1471     return res;
1472 }
1473
1474 uint32_t HELPER(neon_unarrow_sat16)(CPUARMState *env, uint64_t x)
1475 {
1476     uint32_t high;
1477     uint32_t low;
1478     low = x;
1479     if (low & 0x80000000) {
1480         low = 0;
1481         SET_QC();
1482     } else if (low > 0xffff) {
1483         low = 0xffff;
1484         SET_QC();
1485     }
1486     high = x >> 32;
1487     if (high & 0x80000000) {
1488         high = 0;
1489         SET_QC();
1490     } else if (high > 0xffff) {
1491         high = 0xffff;
1492         SET_QC();
1493     }
1494     return low | (high << 16);
1495 }
1496
1497 uint32_t HELPER(neon_narrow_sat_u16)(CPUARMState *env, uint64_t x)
1498 {
1499     uint32_t high;
1500     uint32_t low;
1501     low = x;
1502     if (low > 0xffff) {
1503         low = 0xffff;
1504         SET_QC();
1505     }
1506     high = x >> 32;
1507     if (high > 0xffff) {
1508         high = 0xffff;
1509         SET_QC();
1510     }
1511     return low | (high << 16);
1512 }
1513
1514 uint32_t HELPER(neon_narrow_sat_s16)(CPUARMState *env, uint64_t x)
1515 {
1516     int32_t low;
1517     int32_t high;
1518     low = x;
1519     if (low != (int16_t)low) {
1520         low = (low >> 31) ^ 0x7fff;
1521         SET_QC();
1522     }
1523     high = x >> 32;
1524     if (high != (int16_t)high) {
1525         high = (high >> 31) ^ 0x7fff;
1526         SET_QC();
1527     }
1528     return (uint16_t)low | (high << 16);
1529 }
1530
1531 uint32_t HELPER(neon_unarrow_sat32)(CPUARMState *env, uint64_t x)
1532 {
1533     if (x & 0x8000000000000000ull) {
1534         SET_QC();
1535         return 0;
1536     }
1537     if (x > 0xffffffffu) {
1538         SET_QC();
1539         return 0xffffffffu;
1540     }
1541     return x;
1542 }
1543
1544 uint32_t HELPER(neon_narrow_sat_u32)(CPUARMState *env, uint64_t x)
1545 {
1546     if (x > 0xffffffffu) {
1547         SET_QC();
1548         return 0xffffffffu;
1549     }
1550     return x;
1551 }
1552
1553 uint32_t HELPER(neon_narrow_sat_s32)(CPUARMState *env, uint64_t x)
1554 {
1555     if ((int64_t)x != (int32_t)x) {
1556         SET_QC();
1557         return ((int64_t)x >> 63) ^ 0x7fffffff;
1558     }
1559     return x;
1560 }
1561
1562 uint64_t HELPER(neon_widen_u8)(uint32_t x)
1563 {
1564     uint64_t tmp;
1565     uint64_t ret;
1566     ret = (uint8_t)x;
1567     tmp = (uint8_t)(x >> 8);
1568     ret |= tmp << 16;
1569     tmp = (uint8_t)(x >> 16);
1570     ret |= tmp << 32;
1571     tmp = (uint8_t)(x >> 24);
1572     ret |= tmp << 48;
1573     return ret;
1574 }
1575
1576 uint64_t HELPER(neon_widen_s8)(uint32_t x)
1577 {
1578     uint64_t tmp;
1579     uint64_t ret;
1580     ret = (uint16_t)(int8_t)x;
1581     tmp = (uint16_t)(int8_t)(x >> 8);
1582     ret |= tmp << 16;
1583     tmp = (uint16_t)(int8_t)(x >> 16);
1584     ret |= tmp << 32;
1585     tmp = (uint16_t)(int8_t)(x >> 24);
1586     ret |= tmp << 48;
1587     return ret;
1588 }
1589
1590 uint64_t HELPER(neon_widen_u16)(uint32_t x)
1591 {
1592     uint64_t high = (uint16_t)(x >> 16);
1593     return ((uint16_t)x) | (high << 32);
1594 }
1595
1596 uint64_t HELPER(neon_widen_s16)(uint32_t x)
1597 {
1598     uint64_t high = (int16_t)(x >> 16);
1599     return ((uint32_t)(int16_t)x) | (high << 32);
1600 }
1601
1602 uint64_t HELPER(neon_addl_u16)(uint64_t a, uint64_t b)
1603 {
1604     uint64_t mask;
1605     mask = (a ^ b) & 0x8000800080008000ull;
1606     a &= ~0x8000800080008000ull;
1607     b &= ~0x8000800080008000ull;
1608     return (a + b) ^ mask;
1609 }
1610
1611 uint64_t HELPER(neon_addl_u32)(uint64_t a, uint64_t b)
1612 {
1613     uint64_t mask;
1614     mask = (a ^ b) & 0x8000000080000000ull;
1615     a &= ~0x8000000080000000ull;
1616     b &= ~0x8000000080000000ull;
1617     return (a + b) ^ mask;
1618 }
1619
1620 uint64_t HELPER(neon_paddl_u16)(uint64_t a, uint64_t b)
1621 {
1622     uint64_t tmp;
1623     uint64_t tmp2;
1624
1625     tmp = a & 0x0000ffff0000ffffull;
1626     tmp += (a >> 16) & 0x0000ffff0000ffffull;
1627     tmp2 = b & 0xffff0000ffff0000ull;
1628     tmp2 += (b << 16) & 0xffff0000ffff0000ull;
1629     return    ( tmp         & 0xffff)
1630             | ((tmp  >> 16) & 0xffff0000ull)
1631             | ((tmp2 << 16) & 0xffff00000000ull)
1632             | ( tmp2        & 0xffff000000000000ull);
1633 }
1634
1635 uint64_t HELPER(neon_paddl_u32)(uint64_t a, uint64_t b)
1636 {
1637     uint32_t low = a + (a >> 32);
1638     uint32_t high = b + (b >> 32);
1639     return low + ((uint64_t)high << 32);
1640 }
1641
1642 uint64_t HELPER(neon_subl_u16)(uint64_t a, uint64_t b)
1643 {
1644     uint64_t mask;
1645     mask = (a ^ ~b) & 0x8000800080008000ull;
1646     a |= 0x8000800080008000ull;
1647     b &= ~0x8000800080008000ull;
1648     return (a - b) ^ mask;
1649 }
1650
1651 uint64_t HELPER(neon_subl_u32)(uint64_t a, uint64_t b)
1652 {
1653     uint64_t mask;
1654     mask = (a ^ ~b) & 0x8000000080000000ull;
1655     a |= 0x8000000080000000ull;
1656     b &= ~0x8000000080000000ull;
1657     return (a - b) ^ mask;
1658 }
1659
1660 uint64_t HELPER(neon_addl_saturate_s32)(CPUARMState *env, uint64_t a, uint64_t b)
1661 {
1662     uint32_t x, y;
1663     uint32_t low, high;
1664
1665     x = a;
1666     y = b;
1667     low = x + y;
1668     if (((low ^ x) & SIGNBIT) && !((x ^ y) & SIGNBIT)) {
1669         SET_QC();
1670         low = ((int32_t)x >> 31) ^ ~SIGNBIT;
1671     }
1672     x = a >> 32;
1673     y = b >> 32;
1674     high = x + y;
1675     if (((high ^ x) & SIGNBIT) && !((x ^ y) & SIGNBIT)) {
1676         SET_QC();
1677         high = ((int32_t)x >> 31) ^ ~SIGNBIT;
1678     }
1679     return low | ((uint64_t)high << 32);
1680 }
1681
1682 uint64_t HELPER(neon_addl_saturate_s64)(CPUARMState *env, uint64_t a, uint64_t b)
1683 {
1684     uint64_t result;
1685
1686     result = a + b;
1687     if (((result ^ a) & SIGNBIT64) && !((a ^ b) & SIGNBIT64)) {
1688         SET_QC();
1689         result = ((int64_t)a >> 63) ^ ~SIGNBIT64;
1690     }
1691     return result;
1692 }
1693
1694 /* We have to do the arithmetic in a larger type than
1695  * the input type, because for example with a signed 32 bit
1696  * op the absolute difference can overflow a signed 32 bit value.
1697  */
1698 #define DO_ABD(dest, x, y, intype, arithtype) do {            \
1699     arithtype tmp_x = (intype)(x);                            \
1700     arithtype tmp_y = (intype)(y);                            \
1701     dest = ((tmp_x > tmp_y) ? tmp_x - tmp_y : tmp_y - tmp_x); \
1702     } while(0)
1703
1704 uint64_t HELPER(neon_abdl_u16)(uint32_t a, uint32_t b)
1705 {
1706     uint64_t tmp;
1707     uint64_t result;
1708     DO_ABD(result, a, b, uint8_t, uint32_t);
1709     DO_ABD(tmp, a >> 8, b >> 8, uint8_t, uint32_t);
1710     result |= tmp << 16;
1711     DO_ABD(tmp, a >> 16, b >> 16, uint8_t, uint32_t);
1712     result |= tmp << 32;
1713     DO_ABD(tmp, a >> 24, b >> 24, uint8_t, uint32_t);
1714     result |= tmp << 48;
1715     return result;
1716 }
1717
1718 uint64_t HELPER(neon_abdl_s16)(uint32_t a, uint32_t b)
1719 {
1720     uint64_t tmp;
1721     uint64_t result;
1722     DO_ABD(result, a, b, int8_t, int32_t);
1723     DO_ABD(tmp, a >> 8, b >> 8, int8_t, int32_t);
1724     result |= tmp << 16;
1725     DO_ABD(tmp, a >> 16, b >> 16, int8_t, int32_t);
1726     result |= tmp << 32;
1727     DO_ABD(tmp, a >> 24, b >> 24, int8_t, int32_t);
1728     result |= tmp << 48;
1729     return result;
1730 }
1731
1732 uint64_t HELPER(neon_abdl_u32)(uint32_t a, uint32_t b)
1733 {
1734     uint64_t tmp;
1735     uint64_t result;
1736     DO_ABD(result, a, b, uint16_t, uint32_t);
1737     DO_ABD(tmp, a >> 16, b >> 16, uint16_t, uint32_t);
1738     return result | (tmp << 32);
1739 }
1740
1741 uint64_t HELPER(neon_abdl_s32)(uint32_t a, uint32_t b)
1742 {
1743     uint64_t tmp;
1744     uint64_t result;
1745     DO_ABD(result, a, b, int16_t, int32_t);
1746     DO_ABD(tmp, a >> 16, b >> 16, int16_t, int32_t);
1747     return result | (tmp << 32);
1748 }
1749
1750 uint64_t HELPER(neon_abdl_u64)(uint32_t a, uint32_t b)
1751 {
1752     uint64_t result;
1753     DO_ABD(result, a, b, uint32_t, uint64_t);
1754     return result;
1755 }
1756
1757 uint64_t HELPER(neon_abdl_s64)(uint32_t a, uint32_t b)
1758 {
1759     uint64_t result;
1760     DO_ABD(result, a, b, int32_t, int64_t);
1761     return result;
1762 }
1763 #undef DO_ABD
1764
1765 /* Widening multiply. Named type is the source type.  */
1766 #define DO_MULL(dest, x, y, type1, type2) do { \
1767     type1 tmp_x = x; \
1768     type1 tmp_y = y; \
1769     dest = (type2)((type2)tmp_x * (type2)tmp_y); \
1770     } while(0)
1771
1772 uint64_t HELPER(neon_mull_u8)(uint32_t a, uint32_t b)
1773 {
1774     uint64_t tmp;
1775     uint64_t result;
1776
1777     DO_MULL(result, a, b, uint8_t, uint16_t);
1778     DO_MULL(tmp, a >> 8, b >> 8, uint8_t, uint16_t);
1779     result |= tmp << 16;
1780     DO_MULL(tmp, a >> 16, b >> 16, uint8_t, uint16_t);
1781     result |= tmp << 32;
1782     DO_MULL(tmp, a >> 24, b >> 24, uint8_t, uint16_t);
1783     result |= tmp << 48;
1784     return result;
1785 }
1786
1787 uint64_t HELPER(neon_mull_s8)(uint32_t a, uint32_t b)
1788 {
1789     uint64_t tmp;
1790     uint64_t result;
1791
1792     DO_MULL(result, a, b, int8_t, uint16_t);
1793     DO_MULL(tmp, a >> 8, b >> 8, int8_t, uint16_t);
1794     result |= tmp << 16;
1795     DO_MULL(tmp, a >> 16, b >> 16, int8_t, uint16_t);
1796     result |= tmp << 32;
1797     DO_MULL(tmp, a >> 24, b >> 24, int8_t, uint16_t);
1798     result |= tmp << 48;
1799     return result;
1800 }
1801
1802 uint64_t HELPER(neon_mull_u16)(uint32_t a, uint32_t b)
1803 {
1804     uint64_t tmp;
1805     uint64_t result;
1806
1807     DO_MULL(result, a, b, uint16_t, uint32_t);
1808     DO_MULL(tmp, a >> 16, b >> 16, uint16_t, uint32_t);
1809     return result | (tmp << 32);
1810 }
1811
1812 uint64_t HELPER(neon_mull_s16)(uint32_t a, uint32_t b)
1813 {
1814     uint64_t tmp;
1815     uint64_t result;
1816
1817     DO_MULL(result, a, b, int16_t, uint32_t);
1818     DO_MULL(tmp, a >> 16, b >> 16, int16_t, uint32_t);
1819     return result | (tmp << 32);
1820 }
1821
1822 uint64_t HELPER(neon_negl_u16)(uint64_t x)
1823 {
1824     uint16_t tmp;
1825     uint64_t result;
1826     result = (uint16_t)-x;
1827     tmp = -(x >> 16);
1828     result |= (uint64_t)tmp << 16;
1829     tmp = -(x >> 32);
1830     result |= (uint64_t)tmp << 32;
1831     tmp = -(x >> 48);
1832     result |= (uint64_t)tmp << 48;
1833     return result;
1834 }
1835
1836 uint64_t HELPER(neon_negl_u32)(uint64_t x)
1837 {
1838     uint32_t low = -x;
1839     uint32_t high = -(x >> 32);
1840     return low | ((uint64_t)high << 32);
1841 }
1842
1843 /* Saturating sign manipulation.  */
1844 /* ??? Make these use NEON_VOP1 */
1845 #define DO_QABS8(x) do { \
1846     if (x == (int8_t)0x80) { \
1847         x = 0x7f; \
1848         SET_QC(); \
1849     } else if (x < 0) { \
1850         x = -x; \
1851     }} while (0)
1852 uint32_t HELPER(neon_qabs_s8)(CPUARMState *env, uint32_t x)
1853 {
1854     neon_s8 vec;
1855     NEON_UNPACK(neon_s8, vec, x);
1856     DO_QABS8(vec.v1);
1857     DO_QABS8(vec.v2);
1858     DO_QABS8(vec.v3);
1859     DO_QABS8(vec.v4);
1860     NEON_PACK(neon_s8, x, vec);
1861     return x;
1862 }
1863 #undef DO_QABS8
1864
1865 #define DO_QNEG8(x) do { \
1866     if (x == (int8_t)0x80) { \
1867         x = 0x7f; \
1868         SET_QC(); \
1869     } else { \
1870         x = -x; \
1871     }} while (0)
1872 uint32_t HELPER(neon_qneg_s8)(CPUARMState *env, uint32_t x)
1873 {
1874     neon_s8 vec;
1875     NEON_UNPACK(neon_s8, vec, x);
1876     DO_QNEG8(vec.v1);
1877     DO_QNEG8(vec.v2);
1878     DO_QNEG8(vec.v3);
1879     DO_QNEG8(vec.v4);
1880     NEON_PACK(neon_s8, x, vec);
1881     return x;
1882 }
1883 #undef DO_QNEG8
1884
1885 #define DO_QABS16(x) do { \
1886     if (x == (int16_t)0x8000) { \
1887         x = 0x7fff; \
1888         SET_QC(); \
1889     } else if (x < 0) { \
1890         x = -x; \
1891     }} while (0)
1892 uint32_t HELPER(neon_qabs_s16)(CPUARMState *env, uint32_t x)
1893 {
1894     neon_s16 vec;
1895     NEON_UNPACK(neon_s16, vec, x);
1896     DO_QABS16(vec.v1);
1897     DO_QABS16(vec.v2);
1898     NEON_PACK(neon_s16, x, vec);
1899     return x;
1900 }
1901 #undef DO_QABS16
1902
1903 #define DO_QNEG16(x) do { \
1904     if (x == (int16_t)0x8000) { \
1905         x = 0x7fff; \
1906         SET_QC(); \
1907     } else { \
1908         x = -x; \
1909     }} while (0)
1910 uint32_t HELPER(neon_qneg_s16)(CPUARMState *env, uint32_t x)
1911 {
1912     neon_s16 vec;
1913     NEON_UNPACK(neon_s16, vec, x);
1914     DO_QNEG16(vec.v1);
1915     DO_QNEG16(vec.v2);
1916     NEON_PACK(neon_s16, x, vec);
1917     return x;
1918 }
1919 #undef DO_QNEG16
1920
1921 uint32_t HELPER(neon_qabs_s32)(CPUARMState *env, uint32_t x)
1922 {
1923     if (x == SIGNBIT) {
1924         SET_QC();
1925         x = ~SIGNBIT;
1926     } else if ((int32_t)x < 0) {
1927         x = -x;
1928     }
1929     return x;
1930 }
1931
1932 uint32_t HELPER(neon_qneg_s32)(CPUARMState *env, uint32_t x)
1933 {
1934     if (x == SIGNBIT) {
1935         SET_QC();
1936         x = ~SIGNBIT;
1937     } else {
1938         x = -x;
1939     }
1940     return x;
1941 }
1942
1943 uint64_t HELPER(neon_qabs_s64)(CPUARMState *env, uint64_t x)
1944 {
1945     if (x == SIGNBIT64) {
1946         SET_QC();
1947         x = ~SIGNBIT64;
1948     } else if ((int64_t)x < 0) {
1949         x = -x;
1950     }
1951     return x;
1952 }
1953
1954 uint64_t HELPER(neon_qneg_s64)(CPUARMState *env, uint64_t x)
1955 {
1956     if (x == SIGNBIT64) {
1957         SET_QC();
1958         x = ~SIGNBIT64;
1959     } else {
1960         x = -x;
1961     }
1962     return x;
1963 }
1964
1965 /* NEON Float helpers.  */
1966 uint32_t HELPER(neon_abd_f32)(uint32_t a, uint32_t b, void *fpstp)
1967 {
1968     float_status *fpst = fpstp;
1969     float32 f0 = make_float32(a);
1970     float32 f1 = make_float32(b);
1971     return float32_val(float32_abs(float32_sub(f0, f1, fpst)));
1972 }
1973
1974 /* Floating point comparisons produce an integer result.
1975  * Note that EQ doesn't signal InvalidOp for QNaNs but GE and GT do.
1976  * Softfloat routines return 0/1, which we convert to the 0/-1 Neon requires.
1977  */
1978 uint32_t HELPER(neon_ceq_f32)(uint32_t a, uint32_t b, void *fpstp)
1979 {
1980     float_status *fpst = fpstp;
1981     return -float32_eq_quiet(make_float32(a), make_float32(b), fpst);
1982 }
1983
1984 uint32_t HELPER(neon_cge_f32)(uint32_t a, uint32_t b, void *fpstp)
1985 {
1986     float_status *fpst = fpstp;
1987     return -float32_le(make_float32(b), make_float32(a), fpst);
1988 }
1989
1990 uint32_t HELPER(neon_cgt_f32)(uint32_t a, uint32_t b, void *fpstp)
1991 {
1992     float_status *fpst = fpstp;
1993     return -float32_lt(make_float32(b), make_float32(a), fpst);
1994 }
1995
1996 uint32_t HELPER(neon_acge_f32)(uint32_t a, uint32_t b, void *fpstp)
1997 {
1998     float_status *fpst = fpstp;
1999     float32 f0 = float32_abs(make_float32(a));
2000     float32 f1 = float32_abs(make_float32(b));
2001     return -float32_le(f1, f0, fpst);
2002 }
2003
2004 uint32_t HELPER(neon_acgt_f32)(uint32_t a, uint32_t b, void *fpstp)
2005 {
2006     float_status *fpst = fpstp;
2007     float32 f0 = float32_abs(make_float32(a));
2008     float32 f1 = float32_abs(make_float32(b));
2009     return -float32_lt(f1, f0, fpst);
2010 }
2011
2012 uint64_t HELPER(neon_acge_f64)(uint64_t a, uint64_t b, void *fpstp)
2013 {
2014     float_status *fpst = fpstp;
2015     float64 f0 = float64_abs(make_float64(a));
2016     float64 f1 = float64_abs(make_float64(b));
2017     return -float64_le(f1, f0, fpst);
2018 }
2019
2020 uint64_t HELPER(neon_acgt_f64)(uint64_t a, uint64_t b, void *fpstp)
2021 {
2022     float_status *fpst = fpstp;
2023     float64 f0 = float64_abs(make_float64(a));
2024     float64 f1 = float64_abs(make_float64(b));
2025     return -float64_lt(f1, f0, fpst);
2026 }
2027
2028 #define ELEM(V, N, SIZE) (((V) >> ((N) * (SIZE))) & ((1ull << (SIZE)) - 1))
2029
2030 void HELPER(neon_qunzip8)(void *vd, void *vm)
2031 {
2032     uint64_t *rd = vd, *rm = vm;
2033     uint64_t zd0 = rd[0], zd1 = rd[1];
2034     uint64_t zm0 = rm[0], zm1 = rm[1];
2035
2036     uint64_t d0 = ELEM(zd0, 0, 8) | (ELEM(zd0, 2, 8) << 8)
2037         | (ELEM(zd0, 4, 8) << 16) | (ELEM(zd0, 6, 8) << 24)
2038         | (ELEM(zd1, 0, 8) << 32) | (ELEM(zd1, 2, 8) << 40)
2039         | (ELEM(zd1, 4, 8) << 48) | (ELEM(zd1, 6, 8) << 56);
2040     uint64_t d1 = ELEM(zm0, 0, 8) | (ELEM(zm0, 2, 8) << 8)
2041         | (ELEM(zm0, 4, 8) << 16) | (ELEM(zm0, 6, 8) << 24)
2042         | (ELEM(zm1, 0, 8) << 32) | (ELEM(zm1, 2, 8) << 40)
2043         | (ELEM(zm1, 4, 8) << 48) | (ELEM(zm1, 6, 8) << 56);
2044     uint64_t m0 = ELEM(zd0, 1, 8) | (ELEM(zd0, 3, 8) << 8)
2045         | (ELEM(zd0, 5, 8) << 16) | (ELEM(zd0, 7, 8) << 24)
2046         | (ELEM(zd1, 1, 8) << 32) | (ELEM(zd1, 3, 8) << 40)
2047         | (ELEM(zd1, 5, 8) << 48) | (ELEM(zd1, 7, 8) << 56);
2048     uint64_t m1 = ELEM(zm0, 1, 8) | (ELEM(zm0, 3, 8) << 8)
2049         | (ELEM(zm0, 5, 8) << 16) | (ELEM(zm0, 7, 8) << 24)
2050         | (ELEM(zm1, 1, 8) << 32) | (ELEM(zm1, 3, 8) << 40)
2051         | (ELEM(zm1, 5, 8) << 48) | (ELEM(zm1, 7, 8) << 56);
2052
2053     rm[0] = m0;
2054     rm[1] = m1;
2055     rd[0] = d0;
2056     rd[1] = d1;
2057 }
2058
2059 void HELPER(neon_qunzip16)(void *vd, void *vm)
2060 {
2061     uint64_t *rd = vd, *rm = vm;
2062     uint64_t zd0 = rd[0], zd1 = rd[1];
2063     uint64_t zm0 = rm[0], zm1 = rm[1];
2064
2065     uint64_t d0 = ELEM(zd0, 0, 16) | (ELEM(zd0, 2, 16) << 16)
2066         | (ELEM(zd1, 0, 16) << 32) | (ELEM(zd1, 2, 16) << 48);
2067     uint64_t d1 = ELEM(zm0, 0, 16) | (ELEM(zm0, 2, 16) << 16)
2068         | (ELEM(zm1, 0, 16) << 32) | (ELEM(zm1, 2, 16) << 48);
2069     uint64_t m0 = ELEM(zd0, 1, 16) | (ELEM(zd0, 3, 16) << 16)
2070         | (ELEM(zd1, 1, 16) << 32) | (ELEM(zd1, 3, 16) << 48);
2071     uint64_t m1 = ELEM(zm0, 1, 16) | (ELEM(zm0, 3, 16) << 16)
2072         | (ELEM(zm1, 1, 16) << 32) | (ELEM(zm1, 3, 16) << 48);
2073
2074     rm[0] = m0;
2075     rm[1] = m1;
2076     rd[0] = d0;
2077     rd[1] = d1;
2078 }
2079
2080 void HELPER(neon_qunzip32)(void *vd, void *vm)
2081 {
2082     uint64_t *rd = vd, *rm = vm;
2083     uint64_t zd0 = rd[0], zd1 = rd[1];
2084     uint64_t zm0 = rm[0], zm1 = rm[1];
2085
2086     uint64_t d0 = ELEM(zd0, 0, 32) | (ELEM(zd1, 0, 32) << 32);
2087     uint64_t d1 = ELEM(zm0, 0, 32) | (ELEM(zm1, 0, 32) << 32);
2088     uint64_t m0 = ELEM(zd0, 1, 32) | (ELEM(zd1, 1, 32) << 32);
2089     uint64_t m1 = ELEM(zm0, 1, 32) | (ELEM(zm1, 1, 32) << 32);
2090
2091     rm[0] = m0;
2092     rm[1] = m1;
2093     rd[0] = d0;
2094     rd[1] = d1;
2095 }
2096
2097 void HELPER(neon_unzip8)(void *vd, void *vm)
2098 {
2099     uint64_t *rd = vd, *rm = vm;
2100     uint64_t zd = rd[0], zm = rm[0];
2101
2102     uint64_t d0 = ELEM(zd, 0, 8) | (ELEM(zd, 2, 8) << 8)
2103         | (ELEM(zd, 4, 8) << 16) | (ELEM(zd, 6, 8) << 24)
2104         | (ELEM(zm, 0, 8) << 32) | (ELEM(zm, 2, 8) << 40)
2105         | (ELEM(zm, 4, 8) << 48) | (ELEM(zm, 6, 8) << 56);
2106     uint64_t m0 = ELEM(zd, 1, 8) | (ELEM(zd, 3, 8) << 8)
2107         | (ELEM(zd, 5, 8) << 16) | (ELEM(zd, 7, 8) << 24)
2108         | (ELEM(zm, 1, 8) << 32) | (ELEM(zm, 3, 8) << 40)
2109         | (ELEM(zm, 5, 8) << 48) | (ELEM(zm, 7, 8) << 56);
2110
2111     rm[0] = m0;
2112     rd[0] = d0;
2113 }
2114
2115 void HELPER(neon_unzip16)(void *vd, void *vm)
2116 {
2117     uint64_t *rd = vd, *rm = vm;
2118     uint64_t zd = rd[0], zm = rm[0];
2119
2120     uint64_t d0 = ELEM(zd, 0, 16) | (ELEM(zd, 2, 16) << 16)
2121         | (ELEM(zm, 0, 16) << 32) | (ELEM(zm, 2, 16) << 48);
2122     uint64_t m0 = ELEM(zd, 1, 16) | (ELEM(zd, 3, 16) << 16)
2123         | (ELEM(zm, 1, 16) << 32) | (ELEM(zm, 3, 16) << 48);
2124
2125     rm[0] = m0;
2126     rd[0] = d0;
2127 }
2128
2129 void HELPER(neon_qzip8)(void *vd, void *vm)
2130 {
2131     uint64_t *rd = vd, *rm = vm;
2132     uint64_t zd0 = rd[0], zd1 = rd[1];
2133     uint64_t zm0 = rm[0], zm1 = rm[1];
2134
2135     uint64_t d0 = ELEM(zd0, 0, 8) | (ELEM(zm0, 0, 8) << 8)
2136         | (ELEM(zd0, 1, 8) << 16) | (ELEM(zm0, 1, 8) << 24)
2137         | (ELEM(zd0, 2, 8) << 32) | (ELEM(zm0, 2, 8) << 40)
2138         | (ELEM(zd0, 3, 8) << 48) | (ELEM(zm0, 3, 8) << 56);
2139     uint64_t d1 = ELEM(zd0, 4, 8) | (ELEM(zm0, 4, 8) << 8)
2140         | (ELEM(zd0, 5, 8) << 16) | (ELEM(zm0, 5, 8) << 24)
2141         | (ELEM(zd0, 6, 8) << 32) | (ELEM(zm0, 6, 8) << 40)
2142         | (ELEM(zd0, 7, 8) << 48) | (ELEM(zm0, 7, 8) << 56);
2143     uint64_t m0 = ELEM(zd1, 0, 8) | (ELEM(zm1, 0, 8) << 8)
2144         | (ELEM(zd1, 1, 8) << 16) | (ELEM(zm1, 1, 8) << 24)
2145         | (ELEM(zd1, 2, 8) << 32) | (ELEM(zm1, 2, 8) << 40)
2146         | (ELEM(zd1, 3, 8) << 48) | (ELEM(zm1, 3, 8) << 56);
2147     uint64_t m1 = ELEM(zd1, 4, 8) | (ELEM(zm1, 4, 8) << 8)
2148         | (ELEM(zd1, 5, 8) << 16) | (ELEM(zm1, 5, 8) << 24)
2149         | (ELEM(zd1, 6, 8) << 32) | (ELEM(zm1, 6, 8) << 40)
2150         | (ELEM(zd1, 7, 8) << 48) | (ELEM(zm1, 7, 8) << 56);
2151
2152     rm[0] = m0;
2153     rm[1] = m1;
2154     rd[0] = d0;
2155     rd[1] = d1;
2156 }
2157
2158 void HELPER(neon_qzip16)(void *vd, void *vm)
2159 {
2160     uint64_t *rd = vd, *rm = vm;
2161     uint64_t zd0 = rd[0], zd1 = rd[1];
2162     uint64_t zm0 = rm[0], zm1 = rm[1];
2163
2164     uint64_t d0 = ELEM(zd0, 0, 16) | (ELEM(zm0, 0, 16) << 16)
2165         | (ELEM(zd0, 1, 16) << 32) | (ELEM(zm0, 1, 16) << 48);
2166     uint64_t d1 = ELEM(zd0, 2, 16) | (ELEM(zm0, 2, 16) << 16)
2167         | (ELEM(zd0, 3, 16) << 32) | (ELEM(zm0, 3, 16) << 48);
2168     uint64_t m0 = ELEM(zd1, 0, 16) | (ELEM(zm1, 0, 16) << 16)
2169         | (ELEM(zd1, 1, 16) << 32) | (ELEM(zm1, 1, 16) << 48);
2170     uint64_t m1 = ELEM(zd1, 2, 16) | (ELEM(zm1, 2, 16) << 16)
2171         | (ELEM(zd1, 3, 16) << 32) | (ELEM(zm1, 3, 16) << 48);
2172
2173     rm[0] = m0;
2174     rm[1] = m1;
2175     rd[0] = d0;
2176     rd[1] = d1;
2177 }
2178
2179 void HELPER(neon_qzip32)(void *vd, void *vm)
2180 {
2181     uint64_t *rd = vd, *rm = vm;
2182     uint64_t zd0 = rd[0], zd1 = rd[1];
2183     uint64_t zm0 = rm[0], zm1 = rm[1];
2184
2185     uint64_t d0 = ELEM(zd0, 0, 32) | (ELEM(zm0, 0, 32) << 32);
2186     uint64_t d1 = ELEM(zd0, 1, 32) | (ELEM(zm0, 1, 32) << 32);
2187     uint64_t m0 = ELEM(zd1, 0, 32) | (ELEM(zm1, 0, 32) << 32);
2188     uint64_t m1 = ELEM(zd1, 1, 32) | (ELEM(zm1, 1, 32) << 32);
2189
2190     rm[0] = m0;
2191     rm[1] = m1;
2192     rd[0] = d0;
2193     rd[1] = d1;
2194 }
2195
2196 void HELPER(neon_zip8)(void *vd, void *vm)
2197 {
2198     uint64_t *rd = vd, *rm = vm;
2199     uint64_t zd = rd[0], zm = rm[0];
2200
2201     uint64_t d0 = ELEM(zd, 0, 8) | (ELEM(zm, 0, 8) << 8)
2202         | (ELEM(zd, 1, 8) << 16) | (ELEM(zm, 1, 8) << 24)
2203         | (ELEM(zd, 2, 8) << 32) | (ELEM(zm, 2, 8) << 40)
2204         | (ELEM(zd, 3, 8) << 48) | (ELEM(zm, 3, 8) << 56);
2205     uint64_t m0 = ELEM(zd, 4, 8) | (ELEM(zm, 4, 8) << 8)
2206         | (ELEM(zd, 5, 8) << 16) | (ELEM(zm, 5, 8) << 24)
2207         | (ELEM(zd, 6, 8) << 32) | (ELEM(zm, 6, 8) << 40)
2208         | (ELEM(zd, 7, 8) << 48) | (ELEM(zm, 7, 8) << 56);
2209
2210     rm[0] = m0;
2211     rd[0] = d0;
2212 }
2213
2214 void HELPER(neon_zip16)(void *vd, void *vm)
2215 {
2216     uint64_t *rd = vd, *rm = vm;
2217     uint64_t zd = rd[0], zm = rm[0];
2218
2219     uint64_t d0 = ELEM(zd, 0, 16) | (ELEM(zm, 0, 16) << 16)
2220         | (ELEM(zd, 1, 16) << 32) | (ELEM(zm, 1, 16) << 48);
2221     uint64_t m0 = ELEM(zd, 2, 16) | (ELEM(zm, 2, 16) << 16)
2222         | (ELEM(zd, 3, 16) << 32) | (ELEM(zm, 3, 16) << 48);
2223
2224     rm[0] = m0;
2225     rd[0] = d0;
2226 }
2227
2228 /* Helper function for 64 bit polynomial multiply case:
2229  * perform PolynomialMult(op1, op2) and return either the top or
2230  * bottom half of the 128 bit result.
2231  */
2232 uint64_t HELPER(neon_pmull_64_lo)(uint64_t op1, uint64_t op2)
2233 {
2234     int bitnum;
2235     uint64_t res = 0;
2236
2237     for (bitnum = 0; bitnum < 64; bitnum++) {
2238         if (op1 & (1ULL << bitnum)) {
2239             res ^= op2 << bitnum;
2240         }
2241     }
2242     return res;
2243 }
2244 uint64_t HELPER(neon_pmull_64_hi)(uint64_t op1, uint64_t op2)
2245 {
2246     int bitnum;
2247     uint64_t res = 0;
2248
2249     /* bit 0 of op1 can't influence the high 64 bits at all */
2250     for (bitnum = 1; bitnum < 64; bitnum++) {
2251         if (op1 & (1ULL << bitnum)) {
2252             res ^= op2 >> (64 - bitnum);
2253         }
2254     }
2255     return res;
2256 }