target/arm/neon_helper.c

   1 /*
   2  * ARM NEON vector operations.
   3  *
   4  * Copyright (c) 2007, 2008 CodeSourcery.
   5  * Written by Paul Brook
   6  *
   7  * This code is licensed under the GNU GPL v2.
   8  */
   9 #include "qemu/osdep.h"
  10
  11 #include "cpu.h"
  12 #include "exec/helper-proto.h"
  13 #include "fpu/softfloat.h"
  14
  15 #define SIGNBIT (uint32_t)0x80000000
  16 #define SIGNBIT64 ((uint64_t)1 << 63)
  17
  18 #define SET_QC() env->vfp.qc[0] = 1
  19
  20 #define NEON_TYPE1(name, type) \
  21 typedef struct \
  22 { \
  23     type v1; \
  24 } neon_##name;
  25 #ifdef HOST_WORDS_BIGENDIAN
  26 #define NEON_TYPE2(name, type) \
  27 typedef struct \
  28 { \
  29     type v2; \
  30     type v1; \
  31 } neon_##name;
  32 #define NEON_TYPE4(name, type) \
  33 typedef struct \
  34 { \
  35     type v4; \
  36     type v3; \
  37     type v2; \
  38     type v1; \
  39 } neon_##name;
  40 #else
  41 #define NEON_TYPE2(name, type) \
  42 typedef struct \
  43 { \
  44     type v1; \
  45     type v2; \
  46 } neon_##name;
  47 #define NEON_TYPE4(name, type) \
  48 typedef struct \
  49 { \
  50     type v1; \
  51     type v2; \
  52     type v3; \
  53     type v4; \
  54 } neon_##name;
  55 #endif
  56
  57 NEON_TYPE4(s8, int8_t)
  58 NEON_TYPE4(u8, uint8_t)
  59 NEON_TYPE2(s16, int16_t)
  60 NEON_TYPE2(u16, uint16_t)
  61 NEON_TYPE1(s32, int32_t)
  62 NEON_TYPE1(u32, uint32_t)
  63 #undef NEON_TYPE4
  64 #undef NEON_TYPE2
  65 #undef NEON_TYPE1
  66
  67 /* Copy from a uint32_t to a vector structure type.  */
  68 #define NEON_UNPACK(vtype, dest, val) do { \
  69     union { \
  70         vtype v; \
  71         uint32_t i; \
  72     } conv_u; \
  73     conv_u.i = (val); \
  74     dest = conv_u.v; \
  75     } while(0)
  76
  77 /* Copy from a vector structure type to a uint32_t.  */
  78 #define NEON_PACK(vtype, dest, val) do { \
  79     union { \
  80         vtype v; \
  81         uint32_t i; \
  82     } conv_u; \
  83     conv_u.v = (val); \
  84     dest = conv_u.i; \
  85     } while(0)
  86
  87 #define NEON_DO1 \
  88     NEON_FN(vdest.v1, vsrc1.v1, vsrc2.v1);
  89 #define NEON_DO2 \
  90     NEON_FN(vdest.v1, vsrc1.v1, vsrc2.v1); \
  91     NEON_FN(vdest.v2, vsrc1.v2, vsrc2.v2);
  92 #define NEON_DO4 \
  93     NEON_FN(vdest.v1, vsrc1.v1, vsrc2.v1); \
  94     NEON_FN(vdest.v2, vsrc1.v2, vsrc2.v2); \
  95     NEON_FN(vdest.v3, vsrc1.v3, vsrc2.v3); \
  96     NEON_FN(vdest.v4, vsrc1.v4, vsrc2.v4);
  97
  98 #define NEON_VOP_BODY(vtype, n) \
  99 { \
 100     uint32_t res; \
 101     vtype vsrc1; \
 102     vtype vsrc2; \
 103     vtype vdest; \
 104     NEON_UNPACK(vtype, vsrc1, arg1); \
 105     NEON_UNPACK(vtype, vsrc2, arg2); \
 106     NEON_DO##n; \
 107     NEON_PACK(vtype, res, vdest); \
 108     return res; \
 109 }
 110
 111 #define NEON_VOP(name, vtype, n) \
 112 uint32_t HELPER(glue(neon_,name))(uint32_t arg1, uint32_t arg2) \
 113 NEON_VOP_BODY(vtype, n)
 114
 115 #define NEON_VOP_ENV(name, vtype, n) \
 116 uint32_t HELPER(glue(neon_,name))(CPUARMState *env, uint32_t arg1, uint32_t arg2) \
 117 NEON_VOP_BODY(vtype, n)
 118
 119 /* Pairwise operations.  */
 120 /* For 32-bit elements each segment only contains a single element, so
 121    the elementwise and pairwise operations are the same.  */
 122 #define NEON_PDO2 \
 123     NEON_FN(vdest.v1, vsrc1.v1, vsrc1.v2); \
 124     NEON_FN(vdest.v2, vsrc2.v1, vsrc2.v2);
 125 #define NEON_PDO4 \
 126     NEON_FN(vdest.v1, vsrc1.v1, vsrc1.v2); \
 127     NEON_FN(vdest.v2, vsrc1.v3, vsrc1.v4); \
 128     NEON_FN(vdest.v3, vsrc2.v1, vsrc2.v2); \
 129     NEON_FN(vdest.v4, vsrc2.v3, vsrc2.v4); \
 130
 131 #define NEON_POP(name, vtype, n) \
 132 uint32_t HELPER(glue(neon_,name))(uint32_t arg1, uint32_t arg2) \
 133 { \
 134     uint32_t res; \
 135     vtype vsrc1; \
 136     vtype vsrc2; \
 137     vtype vdest; \
 138     NEON_UNPACK(vtype, vsrc1, arg1); \
 139     NEON_UNPACK(vtype, vsrc2, arg2); \
 140     NEON_PDO##n; \
 141     NEON_PACK(vtype, res, vdest); \
 142     return res; \
 143 }
 144
 145 /* Unary operators.  */
 146 #define NEON_VOP1(name, vtype, n) \
 147 uint32_t HELPER(glue(neon_,name))(uint32_t arg) \
 148 { \
 149     vtype vsrc1; \
 150     vtype vdest; \
 151     NEON_UNPACK(vtype, vsrc1, arg); \
 152     NEON_DO##n; \
 153     NEON_PACK(vtype, arg, vdest); \
 154     return arg; \
 155 }
 156
 157
 158 #define NEON_USAT(dest, src1, src2, type) do { \
 159     uint32_t tmp = (uint32_t)src1 + (uint32_t)src2; \
 160     if (tmp != (type)tmp) { \
 161         SET_QC(); \
 162         dest = ~0; \
 163     } else { \
 164         dest = tmp; \
 165     }} while(0)
 166 #define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint8_t)
 167 NEON_VOP_ENV(qadd_u8, neon_u8, 4)
 168 #undef NEON_FN
 169 #define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint16_t)
 170 NEON_VOP_ENV(qadd_u16, neon_u16, 2)
 171 #undef NEON_FN
 172 #undef NEON_USAT
 173
 174 uint32_t HELPER(neon_qadd_u32)(CPUARMState *env, uint32_t a, uint32_t b)
 175 {
 176     uint32_t res = a + b;
 177     if (res < a) {
 178         SET_QC();
 179         res = ~0;
 180     }
 181     return res;
 182 }
 183
 184 uint64_t HELPER(neon_qadd_u64)(CPUARMState *env, uint64_t src1, uint64_t src2)
 185 {
 186     uint64_t res;
 187
 188     res = src1 + src2;
 189     if (res < src1) {
 190         SET_QC();
 191         res = ~(uint64_t)0;
 192     }
 193     return res;
 194 }
 195
 196 #define NEON_SSAT(dest, src1, src2, type) do { \
 197     int32_t tmp = (uint32_t)src1 + (uint32_t)src2; \
 198     if (tmp != (type)tmp) { \
 199         SET_QC(); \
 200         if (src2 > 0) { \
 201             tmp = (1 << (sizeof(type) * 8 - 1)) - 1; \
 202         } else { \
 203             tmp = 1 << (sizeof(type) * 8 - 1); \
 204         } \
 205     } \
 206     dest = tmp; \
 207     } while(0)
 208 #define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int8_t)
 209 NEON_VOP_ENV(qadd_s8, neon_s8, 4)
 210 #undef NEON_FN
 211 #define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int16_t)
 212 NEON_VOP_ENV(qadd_s16, neon_s16, 2)
 213 #undef NEON_FN
 214 #undef NEON_SSAT
 215
 216 uint32_t HELPER(neon_qadd_s32)(CPUARMState *env, uint32_t a, uint32_t b)
 217 {
 218     uint32_t res = a + b;
 219     if (((res ^ a) & SIGNBIT) && !((a ^ b) & SIGNBIT)) {
 220         SET_QC();
 221         res = ~(((int32_t)a >> 31) ^ SIGNBIT);
 222     }
 223     return res;
 224 }
 225
 226 uint64_t HELPER(neon_qadd_s64)(CPUARMState *env, uint64_t src1, uint64_t src2)
 227 {
 228     uint64_t res;
 229
 230     res = src1 + src2;
 231     if (((res ^ src1) & SIGNBIT64) && !((src1 ^ src2) & SIGNBIT64)) {
 232         SET_QC();
 233         res = ((int64_t)src1 >> 63) ^ ~SIGNBIT64;
 234     }
 235     return res;
 236 }
 237
 238 /* Unsigned saturating accumulate of signed value
 239  *
 240  * Op1/Rn is treated as signed
 241  * Op2/Rd is treated as unsigned
 242  *
 243  * Explicit casting is used to ensure the correct sign extension of
 244  * inputs. The result is treated as a unsigned value and saturated as such.
 245  *
 246  * We use a macro for the 8/16 bit cases which expects signed integers of va,
 247  * vb, and vr for interim calculation and an unsigned 32 bit result value r.
 248  */
 249
 250 #define USATACC(bits, shift) \
 251     do { \
 252         va = sextract32(a, shift, bits);                                \
 253         vb = extract32(b, shift, bits);                                 \
 254         vr = va + vb;                                                   \
 255         if (vr > UINT##bits##_MAX) {                                    \
 256             SET_QC();                                                   \
 257             vr = UINT##bits##_MAX;                                      \
 258         } else if (vr < 0) {                                            \
 259             SET_QC();                                                   \
 260             vr = 0;                                                     \
 261         }                                                               \
 262         r = deposit32(r, shift, bits, vr);                              \
 263    } while (0)
 264
 265 uint32_t HELPER(neon_uqadd_s8)(CPUARMState *env, uint32_t a, uint32_t b)
 266 {
 267     int16_t va, vb, vr;
 268     uint32_t r = 0;
 269
 270     USATACC(8, 0);
 271     USATACC(8, 8);
 272     USATACC(8, 16);
 273     USATACC(8, 24);
 274     return r;
 275 }
 276
 277 uint32_t HELPER(neon_uqadd_s16)(CPUARMState *env, uint32_t a, uint32_t b)
 278 {
 279     int32_t va, vb, vr;
 280     uint64_t r = 0;
 281
 282     USATACC(16, 0);
 283     USATACC(16, 16);
 284     return r;
 285 }
 286
 287 #undef USATACC
 288
 289 uint32_t HELPER(neon_uqadd_s32)(CPUARMState *env, uint32_t a, uint32_t b)
 290 {
 291     int64_t va = (int32_t)a;
 292     int64_t vb = (uint32_t)b;
 293     int64_t vr = va + vb;
 294     if (vr > UINT32_MAX) {
 295         SET_QC();
 296         vr = UINT32_MAX;
 297     } else if (vr < 0) {
 298         SET_QC();
 299         vr = 0;
 300     }
 301     return vr;
 302 }
 303
 304 uint64_t HELPER(neon_uqadd_s64)(CPUARMState *env, uint64_t a, uint64_t b)
 305 {
 306     uint64_t res;
 307     res = a + b;
 308     /* We only need to look at the pattern of SIGN bits to detect
 309      * +ve/-ve saturation
 310      */
 311     if (~a & b & ~res & SIGNBIT64) {
 312         SET_QC();
 313         res = UINT64_MAX;
 314     } else if (a & ~b & res & SIGNBIT64) {
 315         SET_QC();
 316         res = 0;
 317     }
 318     return res;
 319 }
 320
 321 /* Signed saturating accumulate of unsigned value
 322  *
 323  * Op1/Rn is treated as unsigned
 324  * Op2/Rd is treated as signed
 325  *
 326  * The result is treated as a signed value and saturated as such
 327  *
 328  * We use a macro for the 8/16 bit cases which expects signed integers of va,
 329  * vb, and vr for interim calculation and an unsigned 32 bit result value r.
 330  */
 331
 332 #define SSATACC(bits, shift) \
 333     do { \
 334         va = extract32(a, shift, bits);                                 \
 335         vb = sextract32(b, shift, bits);                                \
 336         vr = va + vb;                                                   \
 337         if (vr > INT##bits##_MAX) {                                     \
 338             SET_QC();                                                   \
 339             vr = INT##bits##_MAX;                                       \
 340         } else if (vr < INT##bits##_MIN) {                              \
 341             SET_QC();                                                   \
 342             vr = INT##bits##_MIN;                                       \
 343         }                                                               \
 344         r = deposit32(r, shift, bits, vr);                              \
 345     } while (0)
 346
 347 uint32_t HELPER(neon_sqadd_u8)(CPUARMState *env, uint32_t a, uint32_t b)
 348 {
 349     int16_t va, vb, vr;
 350     uint32_t r = 0;
 351
 352     SSATACC(8, 0);
 353     SSATACC(8, 8);
 354     SSATACC(8, 16);
 355     SSATACC(8, 24);
 356     return r;
 357 }
 358
 359 uint32_t HELPER(neon_sqadd_u16)(CPUARMState *env, uint32_t a, uint32_t b)
 360 {
 361     int32_t va, vb, vr;
 362     uint32_t r = 0;
 363
 364     SSATACC(16, 0);
 365     SSATACC(16, 16);
 366
 367     return r;
 368 }
 369
 370 #undef SSATACC
 371
 372 uint32_t HELPER(neon_sqadd_u32)(CPUARMState *env, uint32_t a, uint32_t b)
 373 {
 374     int64_t res;
 375     int64_t op1 = (uint32_t)a;
 376     int64_t op2 = (int32_t)b;
 377     res = op1 + op2;
 378     if (res > INT32_MAX) {
 379         SET_QC();
 380         res = INT32_MAX;
 381     } else if (res < INT32_MIN) {
 382         SET_QC();
 383         res = INT32_MIN;
 384     }
 385     return res;
 386 }
 387
 388 uint64_t HELPER(neon_sqadd_u64)(CPUARMState *env, uint64_t a, uint64_t b)
 389 {
 390     uint64_t res;
 391     res = a + b;
 392     /* We only need to look at the pattern of SIGN bits to detect an overflow */
 393     if (((a & res)
 394          | (~b & res)
 395          | (a & ~b)) & SIGNBIT64) {
 396         SET_QC();
 397         res = INT64_MAX;
 398     }
 399     return res;
 400 }
 401
 402
 403 #define NEON_USAT(dest, src1, src2, type) do { \
 404     uint32_t tmp = (uint32_t)src1 - (uint32_t)src2; \
 405     if (tmp != (type)tmp) { \
 406         SET_QC(); \
 407         dest = 0; \
 408     } else { \
 409         dest = tmp; \
 410     }} while(0)
 411 #define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint8_t)
 412 NEON_VOP_ENV(qsub_u8, neon_u8, 4)
 413 #undef NEON_FN
 414 #define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint16_t)
 415 NEON_VOP_ENV(qsub_u16, neon_u16, 2)
 416 #undef NEON_FN
 417 #undef NEON_USAT
 418
 419 uint32_t HELPER(neon_qsub_u32)(CPUARMState *env, uint32_t a, uint32_t b)
 420 {
 421     uint32_t res = a - b;
 422     if (res > a) {
 423         SET_QC();
 424         res = 0;
 425     }
 426     return res;
 427 }
 428
 429 uint64_t HELPER(neon_qsub_u64)(CPUARMState *env, uint64_t src1, uint64_t src2)
 430 {
 431     uint64_t res;
 432
 433     if (src1 < src2) {
 434         SET_QC();
 435         res = 0;
 436     } else {
 437         res = src1 - src2;
 438     }
 439     return res;
 440 }
 441
 442 #define NEON_SSAT(dest, src1, src2, type) do { \
 443     int32_t tmp = (uint32_t)src1 - (uint32_t)src2; \
 444     if (tmp != (type)tmp) { \
 445         SET_QC(); \
 446         if (src2 < 0) { \
 447             tmp = (1 << (sizeof(type) * 8 - 1)) - 1; \
 448         } else { \
 449             tmp = 1 << (sizeof(type) * 8 - 1); \
 450         } \
 451     } \
 452     dest = tmp; \
 453     } while(0)
 454 #define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int8_t)
 455 NEON_VOP_ENV(qsub_s8, neon_s8, 4)
 456 #undef NEON_FN
 457 #define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int16_t)
 458 NEON_VOP_ENV(qsub_s16, neon_s16, 2)
 459 #undef NEON_FN
 460 #undef NEON_SSAT
 461
 462 uint32_t HELPER(neon_qsub_s32)(CPUARMState *env, uint32_t a, uint32_t b)
 463 {
 464     uint32_t res = a - b;
 465     if (((res ^ a) & SIGNBIT) && ((a ^ b) & SIGNBIT)) {
 466         SET_QC();
 467         res = ~(((int32_t)a >> 31) ^ SIGNBIT);
 468     }
 469     return res;
 470 }
 471
 472 uint64_t HELPER(neon_qsub_s64)(CPUARMState *env, uint64_t src1, uint64_t src2)
 473 {
 474     uint64_t res;
 475
 476     res = src1 - src2;
 477     if (((res ^ src1) & SIGNBIT64) && ((src1 ^ src2) & SIGNBIT64)) {
 478         SET_QC();
 479         res = ((int64_t)src1 >> 63) ^ ~SIGNBIT64;
 480     }
 481     return res;
 482 }
 483
 484 #define NEON_FN(dest, src1, src2) dest = (src1 + src2) >> 1
 485 NEON_VOP(hadd_s8, neon_s8, 4)
 486 NEON_VOP(hadd_u8, neon_u8, 4)
 487 NEON_VOP(hadd_s16, neon_s16, 2)
 488 NEON_VOP(hadd_u16, neon_u16, 2)
 489 #undef NEON_FN
 490
 491 int32_t HELPER(neon_hadd_s32)(int32_t src1, int32_t src2)
 492 {
 493     int32_t dest;
 494
 495     dest = (src1 >> 1) + (src2 >> 1);
 496     if (src1 & src2 & 1)
 497         dest++;
 498     return dest;
 499 }
 500
 501 uint32_t HELPER(neon_hadd_u32)(uint32_t src1, uint32_t src2)
 502 {
 503     uint32_t dest;
 504
 505     dest = (src1 >> 1) + (src2 >> 1);
 506     if (src1 & src2 & 1)
 507         dest++;
 508     return dest;
 509 }
 510
 511 #define NEON_FN(dest, src1, src2) dest = (src1 + src2 + 1) >> 1
 512 NEON_VOP(rhadd_s8, neon_s8, 4)
 513 NEON_VOP(rhadd_u8, neon_u8, 4)
 514 NEON_VOP(rhadd_s16, neon_s16, 2)
 515 NEON_VOP(rhadd_u16, neon_u16, 2)
 516 #undef NEON_FN
 517
 518 int32_t HELPER(neon_rhadd_s32)(int32_t src1, int32_t src2)
 519 {
 520     int32_t dest;
 521
 522     dest = (src1 >> 1) + (src2 >> 1);
 523     if ((src1 | src2) & 1)
 524         dest++;
 525     return dest;
 526 }
 527
 528 uint32_t HELPER(neon_rhadd_u32)(uint32_t src1, uint32_t src2)
 529 {
 530     uint32_t dest;
 531
 532     dest = (src1 >> 1) + (src2 >> 1);
 533     if ((src1 | src2) & 1)
 534         dest++;
 535     return dest;
 536 }
 537
 538 #define NEON_FN(dest, src1, src2) dest = (src1 - src2) >> 1
 539 NEON_VOP(hsub_s8, neon_s8, 4)
 540 NEON_VOP(hsub_u8, neon_u8, 4)
 541 NEON_VOP(hsub_s16, neon_s16, 2)
 542 NEON_VOP(hsub_u16, neon_u16, 2)
 543 #undef NEON_FN
 544
 545 int32_t HELPER(neon_hsub_s32)(int32_t src1, int32_t src2)
 546 {
 547     int32_t dest;
 548
 549     dest = (src1 >> 1) - (src2 >> 1);
 550     if ((~src1) & src2 & 1)
 551         dest--;
 552     return dest;
 553 }
 554
 555 uint32_t HELPER(neon_hsub_u32)(uint32_t src1, uint32_t src2)
 556 {
 557     uint32_t dest;
 558
 559     dest = (src1 >> 1) - (src2 >> 1);
 560     if ((~src1) & src2 & 1)
 561         dest--;
 562     return dest;
 563 }
 564
 565 #define NEON_FN(dest, src1, src2) dest = (src1 > src2) ? ~0 : 0
 566 NEON_VOP(cgt_s8, neon_s8, 4)
 567 NEON_VOP(cgt_u8, neon_u8, 4)
 568 NEON_VOP(cgt_s16, neon_s16, 2)
 569 NEON_VOP(cgt_u16, neon_u16, 2)
 570 NEON_VOP(cgt_s32, neon_s32, 1)
 571 NEON_VOP(cgt_u32, neon_u32, 1)
 572 #undef NEON_FN
 573
 574 #define NEON_FN(dest, src1, src2) dest = (src1 >= src2) ? ~0 : 0
 575 NEON_VOP(cge_s8, neon_s8, 4)
 576 NEON_VOP(cge_u8, neon_u8, 4)
 577 NEON_VOP(cge_s16, neon_s16, 2)
 578 NEON_VOP(cge_u16, neon_u16, 2)
 579 NEON_VOP(cge_s32, neon_s32, 1)
 580 NEON_VOP(cge_u32, neon_u32, 1)
 581 #undef NEON_FN
 582
 583 #define NEON_FN(dest, src1, src2) dest = (src1 < src2) ? src1 : src2
 584 NEON_POP(pmin_s8, neon_s8, 4)
 585 NEON_POP(pmin_u8, neon_u8, 4)
 586 NEON_POP(pmin_s16, neon_s16, 2)
 587 NEON_POP(pmin_u16, neon_u16, 2)
 588 #undef NEON_FN
 589
 590 #define NEON_FN(dest, src1, src2) dest = (src1 > src2) ? src1 : src2
 591 NEON_POP(pmax_s8, neon_s8, 4)
 592 NEON_POP(pmax_u8, neon_u8, 4)
 593 NEON_POP(pmax_s16, neon_s16, 2)
 594 NEON_POP(pmax_u16, neon_u16, 2)
 595 #undef NEON_FN
 596
 597 #define NEON_FN(dest, src1, src2) \
 598     dest = (src1 > src2) ? (src1 - src2) : (src2 - src1)
 599 NEON_VOP(abd_s8, neon_s8, 4)
 600 NEON_VOP(abd_u8, neon_u8, 4)
 601 NEON_VOP(abd_s16, neon_s16, 2)
 602 NEON_VOP(abd_u16, neon_u16, 2)
 603 NEON_VOP(abd_s32, neon_s32, 1)
 604 NEON_VOP(abd_u32, neon_u32, 1)
 605 #undef NEON_FN
 606
 607 #define NEON_FN(dest, src1, src2) do { \
 608     int8_t tmp; \
 609     tmp = (int8_t)src2; \
 610     if (tmp >= (ssize_t)sizeof(src1) * 8 || \
 611         tmp <= -(ssize_t)sizeof(src1) * 8) { \
 612         dest = 0; \
 613     } else if (tmp < 0) { \
 614         dest = src1 >> -tmp; \
 615     } else { \
 616         dest = src1 << tmp; \
 617     }} while (0)
 618 NEON_VOP(shl_u8, neon_u8, 4)
 619 NEON_VOP(shl_u16, neon_u16, 2)
 620 NEON_VOP(shl_u32, neon_u32, 1)
 621 #undef NEON_FN
 622
 623 uint64_t HELPER(neon_shl_u64)(uint64_t val, uint64_t shiftop)
 624 {
 625     int8_t shift = (int8_t)shiftop;
 626     if (shift >= 64 || shift <= -64) {
 627         val = 0;
 628     } else if (shift < 0) {
 629         val >>= -shift;
 630     } else {
 631         val <<= shift;
 632     }
 633     return val;
 634 }
 635
 636 #define NEON_FN(dest, src1, src2) do { \
 637     int8_t tmp; \
 638     tmp = (int8_t)src2; \
 639     if (tmp >= (ssize_t)sizeof(src1) * 8) { \
 640         dest = 0; \
 641     } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
 642         dest = src1 >> (sizeof(src1) * 8 - 1); \
 643     } else if (tmp < 0) { \
 644         dest = src1 >> -tmp; \
 645     } else { \
 646         dest = src1 << tmp; \
 647     }} while (0)
 648 NEON_VOP(shl_s8, neon_s8, 4)
 649 NEON_VOP(shl_s16, neon_s16, 2)
 650 NEON_VOP(shl_s32, neon_s32, 1)
 651 #undef NEON_FN
 652
 653 uint64_t HELPER(neon_shl_s64)(uint64_t valop, uint64_t shiftop)
 654 {
 655     int8_t shift = (int8_t)shiftop;
 656     int64_t val = valop;
 657     if (shift >= 64) {
 658         val = 0;
 659     } else if (shift <= -64) {
 660         val >>= 63;
 661     } else if (shift < 0) {
 662         val >>= -shift;
 663     } else {
 664         val <<= shift;
 665     }
 666     return val;
 667 }
 668
 669 #define NEON_FN(dest, src1, src2) do { \
 670     int8_t tmp; \
 671     tmp = (int8_t)src2; \
 672     if ((tmp >= (ssize_t)sizeof(src1) * 8) \
 673         || (tmp <= -(ssize_t)sizeof(src1) * 8)) { \
 674         dest = 0; \
 675     } else if (tmp < 0) { \
 676         dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
 677     } else { \
 678         dest = src1 << tmp; \
 679     }} while (0)
 680 NEON_VOP(rshl_s8, neon_s8, 4)
 681 NEON_VOP(rshl_s16, neon_s16, 2)
 682 #undef NEON_FN
 683
 684 /* The addition of the rounding constant may overflow, so we use an
 685  * intermediate 64 bit accumulator.  */
 686 uint32_t HELPER(neon_rshl_s32)(uint32_t valop, uint32_t shiftop)
 687 {
 688     int32_t dest;
 689     int32_t val = (int32_t)valop;
 690     int8_t shift = (int8_t)shiftop;
 691     if ((shift >= 32) || (shift <= -32)) {
 692         dest = 0;
 693     } else if (shift < 0) {
 694         int64_t big_dest = ((int64_t)val + (1 << (-1 - shift)));
 695         dest = big_dest >> -shift;
 696     } else {
 697         dest = val << shift;
 698     }
 699     return dest;
 700 }
 701
 702 /* Handling addition overflow with 64 bit input values is more
 703  * tricky than with 32 bit values.  */
 704 uint64_t HELPER(neon_rshl_s64)(uint64_t valop, uint64_t shiftop)
 705 {
 706     int8_t shift = (int8_t)shiftop;
 707     int64_t val = valop;
 708     if ((shift >= 64) || (shift <= -64)) {
 709         val = 0;
 710     } else if (shift < 0) {
 711         val >>= (-shift - 1);
 712         if (val == INT64_MAX) {
 713             /* In this case, it means that the rounding constant is 1,
 714              * and the addition would overflow. Return the actual
 715              * result directly.  */
 716             val = 0x4000000000000000LL;
 717         } else {
 718             val++;
 719             val >>= 1;
 720         }
 721     } else {
 722         val <<= shift;
 723     }
 724     return val;
 725 }
 726
 727 #define NEON_FN(dest, src1, src2) do { \
 728     int8_t tmp; \
 729     tmp = (int8_t)src2; \
 730     if (tmp >= (ssize_t)sizeof(src1) * 8 || \
 731         tmp < -(ssize_t)sizeof(src1) * 8) { \
 732         dest = 0; \
 733     } else if (tmp == -(ssize_t)sizeof(src1) * 8) { \
 734         dest = src1 >> (-tmp - 1); \
 735     } else if (tmp < 0) { \
 736         dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
 737     } else { \
 738         dest = src1 << tmp; \
 739     }} while (0)
 740 NEON_VOP(rshl_u8, neon_u8, 4)
 741 NEON_VOP(rshl_u16, neon_u16, 2)
 742 #undef NEON_FN
 743
 744 /* The addition of the rounding constant may overflow, so we use an
 745  * intermediate 64 bit accumulator.  */
 746 uint32_t HELPER(neon_rshl_u32)(uint32_t val, uint32_t shiftop)
 747 {
 748     uint32_t dest;
 749     int8_t shift = (int8_t)shiftop;
 750     if (shift >= 32 || shift < -32) {
 751         dest = 0;
 752     } else if (shift == -32) {
 753         dest = val >> 31;
 754     } else if (shift < 0) {
 755         uint64_t big_dest = ((uint64_t)val + (1 << (-1 - shift)));
 756         dest = big_dest >> -shift;
 757     } else {
 758         dest = val << shift;
 759     }
 760     return dest;
 761 }
 762
 763 /* Handling addition overflow with 64 bit input values is more
 764  * tricky than with 32 bit values.  */
 765 uint64_t HELPER(neon_rshl_u64)(uint64_t val, uint64_t shiftop)
 766 {
 767     int8_t shift = (uint8_t)shiftop;
 768     if (shift >= 64 || shift < -64) {
 769         val = 0;
 770     } else if (shift == -64) {
 771         /* Rounding a 1-bit result just preserves that bit.  */
 772         val >>= 63;
 773     } else if (shift < 0) {
 774         val >>= (-shift - 1);
 775         if (val == UINT64_MAX) {
 776             /* In this case, it means that the rounding constant is 1,
 777              * and the addition would overflow. Return the actual
 778              * result directly.  */
 779             val = 0x8000000000000000ULL;
 780         } else {
 781             val++;
 782             val >>= 1;
 783         }
 784     } else {
 785         val <<= shift;
 786     }
 787     return val;
 788 }
 789
 790 #define NEON_FN(dest, src1, src2) do { \
 791     int8_t tmp; \
 792     tmp = (int8_t)src2; \
 793     if (tmp >= (ssize_t)sizeof(src1) * 8) { \
 794         if (src1) { \
 795             SET_QC(); \
 796             dest = ~0; \
 797         } else { \
 798             dest = 0; \
 799         } \
 800     } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
 801         dest = 0; \
 802     } else if (tmp < 0) { \
 803         dest = src1 >> -tmp; \
 804     } else { \
 805         dest = src1 << tmp; \
 806         if ((dest >> tmp) != src1) { \
 807             SET_QC(); \
 808             dest = ~0; \
 809         } \
 810     }} while (0)
 811 NEON_VOP_ENV(qshl_u8, neon_u8, 4)
 812 NEON_VOP_ENV(qshl_u16, neon_u16, 2)
 813 NEON_VOP_ENV(qshl_u32, neon_u32, 1)
 814 #undef NEON_FN
 815
 816 uint64_t HELPER(neon_qshl_u64)(CPUARMState *env, uint64_t val, uint64_t shiftop)
 817 {
 818     int8_t shift = (int8_t)shiftop;
 819     if (shift >= 64) {
 820         if (val) {
 821             val = ~(uint64_t)0;
 822             SET_QC();
 823         }
 824     } else if (shift <= -64) {
 825         val = 0;
 826     } else if (shift < 0) {
 827         val >>= -shift;
 828     } else {
 829         uint64_t tmp = val;
 830         val <<= shift;
 831         if ((val >> shift) != tmp) {
 832             SET_QC();
 833             val = ~(uint64_t)0;
 834         }
 835     }
 836     return val;
 837 }
 838
 839 #define NEON_FN(dest, src1, src2) do { \
 840     int8_t tmp; \
 841     tmp = (int8_t)src2; \
 842     if (tmp >= (ssize_t)sizeof(src1) * 8) { \
 843         if (src1) { \
 844             SET_QC(); \
 845             dest = (uint32_t)(1 << (sizeof(src1) * 8 - 1)); \
 846             if (src1 > 0) { \
 847                 dest--; \
 848             } \
 849         } else { \
 850             dest = src1; \
 851         } \
 852     } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
 853         dest = src1 >> 31; \
 854     } else if (tmp < 0) { \
 855         dest = src1 >> -tmp; \
 856     } else { \
 857         dest = src1 << tmp; \
 858         if ((dest >> tmp) != src1) { \
 859             SET_QC(); \
 860             dest = (uint32_t)(1 << (sizeof(src1) * 8 - 1)); \
 861             if (src1 > 0) { \
 862                 dest--; \
 863             } \
 864         } \
 865     }} while (0)
 866 NEON_VOP_ENV(qshl_s8, neon_s8, 4)
 867 NEON_VOP_ENV(qshl_s16, neon_s16, 2)
 868 NEON_VOP_ENV(qshl_s32, neon_s32, 1)
 869 #undef NEON_FN
 870
 871 uint64_t HELPER(neon_qshl_s64)(CPUARMState *env, uint64_t valop, uint64_t shiftop)
 872 {
 873     int8_t shift = (uint8_t)shiftop;
 874     int64_t val = valop;
 875     if (shift >= 64) {
 876         if (val) {
 877             SET_QC();
 878             val = (val >> 63) ^ ~SIGNBIT64;
 879         }
 880     } else if (shift <= -64) {
 881         val >>= 63;
 882     } else if (shift < 0) {
 883         val >>= -shift;
 884     } else {
 885         int64_t tmp = val;
 886         val <<= shift;
 887         if ((val >> shift) != tmp) {
 888             SET_QC();
 889             val = (tmp >> 63) ^ ~SIGNBIT64;
 890         }
 891     }
 892     return val;
 893 }
 894
 895 #define NEON_FN(dest, src1, src2) do { \
 896     if (src1 & (1 << (sizeof(src1) * 8 - 1))) { \
 897         SET_QC(); \
 898         dest = 0; \
 899     } else { \
 900         int8_t tmp; \
 901         tmp = (int8_t)src2; \
 902         if (tmp >= (ssize_t)sizeof(src1) * 8) { \
 903             if (src1) { \
 904                 SET_QC(); \
 905                 dest = ~0; \
 906             } else { \
 907                 dest = 0; \
 908             } \
 909         } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
 910             dest = 0; \
 911         } else if (tmp < 0) { \
 912             dest = src1 >> -tmp; \
 913         } else { \
 914             dest = src1 << tmp; \
 915             if ((dest >> tmp) != src1) { \
 916                 SET_QC(); \
 917                 dest = ~0; \
 918             } \
 919         } \
 920     }} while (0)
 921 NEON_VOP_ENV(qshlu_s8, neon_u8, 4)
 922 NEON_VOP_ENV(qshlu_s16, neon_u16, 2)
 923 #undef NEON_FN
 924
 925 uint32_t HELPER(neon_qshlu_s32)(CPUARMState *env, uint32_t valop, uint32_t shiftop)
 926 {
 927     if ((int32_t)valop < 0) {
 928         SET_QC();
 929         return 0;
 930     }
 931     return helper_neon_qshl_u32(env, valop, shiftop);
 932 }
 933
 934 uint64_t HELPER(neon_qshlu_s64)(CPUARMState *env, uint64_t valop, uint64_t shiftop)
 935 {
 936     if ((int64_t)valop < 0) {
 937         SET_QC();
 938         return 0;
 939     }
 940     return helper_neon_qshl_u64(env, valop, shiftop);
 941 }
 942
 943 #define NEON_FN(dest, src1, src2) do { \
 944     int8_t tmp; \
 945     tmp = (int8_t)src2; \
 946     if (tmp >= (ssize_t)sizeof(src1) * 8) { \
 947         if (src1) { \
 948             SET_QC(); \
 949             dest = ~0; \
 950         } else { \
 951             dest = 0; \
 952         } \
 953     } else if (tmp < -(ssize_t)sizeof(src1) * 8) { \
 954         dest = 0; \
 955     } else if (tmp == -(ssize_t)sizeof(src1) * 8) { \
 956         dest = src1 >> (sizeof(src1) * 8 - 1); \
 957     } else if (tmp < 0) { \
 958         dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
 959     } else { \
 960         dest = src1 << tmp; \
 961         if ((dest >> tmp) != src1) { \
 962             SET_QC(); \
 963             dest = ~0; \
 964         } \
 965     }} while (0)
 966 NEON_VOP_ENV(qrshl_u8, neon_u8, 4)
 967 NEON_VOP_ENV(qrshl_u16, neon_u16, 2)
 968 #undef NEON_FN
 969
 970 /* The addition of the rounding constant may overflow, so we use an
 971  * intermediate 64 bit accumulator.  */
 972 uint32_t HELPER(neon_qrshl_u32)(CPUARMState *env, uint32_t val, uint32_t shiftop)
 973 {
 974     uint32_t dest;
 975     int8_t shift = (int8_t)shiftop;
 976     if (shift >= 32) {
 977         if (val) {
 978             SET_QC();
 979             dest = ~0;
 980         } else {
 981             dest = 0;
 982         }
 983     } else if (shift < -32) {
 984         dest = 0;
 985     } else if (shift == -32) {
 986         dest = val >> 31;
 987     } else if (shift < 0) {
 988         uint64_t big_dest = ((uint64_t)val + (1 << (-1 - shift)));
 989         dest = big_dest >> -shift;
 990     } else {
 991         dest = val << shift;
 992         if ((dest >> shift) != val) {
 993             SET_QC();
 994             dest = ~0;
 995         }
 996     }
 997     return dest;
 998 }
 999
1000 /* Handling addition overflow with 64 bit input values is more
1001  * tricky than with 32 bit values.  */
1002 uint64_t HELPER(neon_qrshl_u64)(CPUARMState *env, uint64_t val, uint64_t shiftop)
1003 {
1004     int8_t shift = (int8_t)shiftop;
1005     if (shift >= 64) {
1006         if (val) {
1007             SET_QC();
1008             val = ~0;
1009         }
1010     } else if (shift < -64) {
1011         val = 0;
1012     } else if (shift == -64) {
1013         val >>= 63;
1014     } else if (shift < 0) {
1015         val >>= (-shift - 1);
1016         if (val == UINT64_MAX) {
1017             /* In this case, it means that the rounding constant is 1,
1018              * and the addition would overflow. Return the actual
1019              * result directly.  */
1020             val = 0x8000000000000000ULL;
1021         } else {
1022             val++;
1023             val >>= 1;
1024         }
1025     } else { \
1026         uint64_t tmp = val;
1027         val <<= shift;
1028         if ((val >> shift) != tmp) {
1029             SET_QC();
1030             val = ~0;
1031         }
1032     }
1033     return val;
1034 }
1035
1036 #define NEON_FN(dest, src1, src2) do { \
1037     int8_t tmp; \
1038     tmp = (int8_t)src2; \
1039     if (tmp >= (ssize_t)sizeof(src1) * 8) { \
1040         if (src1) { \
1041             SET_QC(); \
1042             dest = (typeof(dest))(1 << (sizeof(src1) * 8 - 1)); \
1043             if (src1 > 0) { \
1044                 dest--; \
1045             } \
1046         } else { \
1047             dest = 0; \
1048         } \
1049     } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
1050         dest = 0; \
1051     } else if (tmp < 0) { \
1052         dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
1053     } else { \
1054         dest = src1 << tmp; \
1055         if ((dest >> tmp) != src1) { \
1056             SET_QC(); \
1057             dest = (uint32_t)(1 << (sizeof(src1) * 8 - 1)); \
1058             if (src1 > 0) { \
1059                 dest--; \
1060             } \
1061         } \
1062     }} while (0)
1063 NEON_VOP_ENV(qrshl_s8, neon_s8, 4)
1064 NEON_VOP_ENV(qrshl_s16, neon_s16, 2)
1065 #undef NEON_FN
1066
1067 /* The addition of the rounding constant may overflow, so we use an
1068  * intermediate 64 bit accumulator.  */
1069 uint32_t HELPER(neon_qrshl_s32)(CPUARMState *env, uint32_t valop, uint32_t shiftop)
1070 {
1071     int32_t dest;
1072     int32_t val = (int32_t)valop;
1073     int8_t shift = (int8_t)shiftop;
1074     if (shift >= 32) {
1075         if (val) {
1076             SET_QC();
1077             dest = (val >> 31) ^ ~SIGNBIT;
1078         } else {
1079             dest = 0;
1080         }
1081     } else if (shift <= -32) {
1082         dest = 0;
1083     } else if (shift < 0) {
1084         int64_t big_dest = ((int64_t)val + (1 << (-1 - shift)));
1085         dest = big_dest >> -shift;
1086     } else {
1087         dest = val << shift;
1088         if ((dest >> shift) != val) {
1089             SET_QC();
1090             dest = (val >> 31) ^ ~SIGNBIT;
1091         }
1092     }
1093     return dest;
1094 }
1095
1096 /* Handling addition overflow with 64 bit input values is more
1097  * tricky than with 32 bit values.  */
1098 uint64_t HELPER(neon_qrshl_s64)(CPUARMState *env, uint64_t valop, uint64_t shiftop)
1099 {
1100     int8_t shift = (uint8_t)shiftop;
1101     int64_t val = valop;
1102
1103     if (shift >= 64) {
1104         if (val) {
1105             SET_QC();
1106             val = (val >> 63) ^ ~SIGNBIT64;
1107         }
1108     } else if (shift <= -64) {
1109         val = 0;
1110     } else if (shift < 0) {
1111         val >>= (-shift - 1);
1112         if (val == INT64_MAX) {
1113             /* In this case, it means that the rounding constant is 1,
1114              * and the addition would overflow. Return the actual
1115              * result directly.  */
1116             val = 0x4000000000000000ULL;
1117         } else {
1118             val++;
1119             val >>= 1;
1120         }
1121     } else {
1122         int64_t tmp = val;
1123         val <<= shift;
1124         if ((val >> shift) != tmp) {
1125             SET_QC();
1126             val = (tmp >> 63) ^ ~SIGNBIT64;
1127         }
1128     }
1129     return val;
1130 }
1131
1132 uint32_t HELPER(neon_add_u8)(uint32_t a, uint32_t b)
1133 {
1134     uint32_t mask;
1135     mask = (a ^ b) & 0x80808080u;
1136     a &= ~0x80808080u;
1137     b &= ~0x80808080u;
1138     return (a + b) ^ mask;
1139 }
1140
1141 uint32_t HELPER(neon_add_u16)(uint32_t a, uint32_t b)
1142 {
1143     uint32_t mask;
1144     mask = (a ^ b) & 0x80008000u;
1145     a &= ~0x80008000u;
1146     b &= ~0x80008000u;
1147     return (a + b) ^ mask;
1148 }
1149
1150 #define NEON_FN(dest, src1, src2) dest = src1 + src2
1151 NEON_POP(padd_u8, neon_u8, 4)
1152 NEON_POP(padd_u16, neon_u16, 2)
1153 #undef NEON_FN
1154
1155 #define NEON_FN(dest, src1, src2) dest = src1 - src2
1156 NEON_VOP(sub_u8, neon_u8, 4)
1157 NEON_VOP(sub_u16, neon_u16, 2)
1158 #undef NEON_FN
1159
1160 #define NEON_FN(dest, src1, src2) dest = src1 * src2
1161 NEON_VOP(mul_u8, neon_u8, 4)
1162 NEON_VOP(mul_u16, neon_u16, 2)
1163 #undef NEON_FN
1164
1165 /* Polynomial multiplication is like integer multiplication except the
1166    partial products are XORed, not added.  */
1167 uint32_t HELPER(neon_mul_p8)(uint32_t op1, uint32_t op2)
1168 {
1169     uint32_t mask;
1170     uint32_t result;
1171     result = 0;
1172     while (op1) {
1173         mask = 0;
1174         if (op1 & 1)
1175             mask |= 0xff;
1176         if (op1 & (1 << 8))
1177             mask |= (0xff << 8);
1178         if (op1 & (1 << 16))
1179             mask |= (0xff << 16);
1180         if (op1 & (1 << 24))
1181             mask |= (0xff << 24);
1182         result ^= op2 & mask;
1183         op1 = (op1 >> 1) & 0x7f7f7f7f;
1184         op2 = (op2 << 1) & 0xfefefefe;
1185     }
1186     return result;
1187 }
1188
1189 uint64_t HELPER(neon_mull_p8)(uint32_t op1, uint32_t op2)
1190 {
1191     uint64_t result = 0;
1192     uint64_t mask;
1193     uint64_t op2ex = op2;
1194     op2ex = (op2ex & 0xff) |
1195         ((op2ex & 0xff00) << 8) |
1196         ((op2ex & 0xff0000) << 16) |
1197         ((op2ex & 0xff000000) << 24);
1198     while (op1) {
1199         mask = 0;
1200         if (op1 & 1) {
1201             mask |= 0xffff;
1202         }
1203         if (op1 & (1 << 8)) {
1204             mask |= (0xffffU << 16);
1205         }
1206         if (op1 & (1 << 16)) {
1207             mask |= (0xffffULL << 32);
1208         }
1209         if (op1 & (1 << 24)) {
1210             mask |= (0xffffULL << 48);
1211         }
1212         result ^= op2ex & mask;
1213         op1 = (op1 >> 1) & 0x7f7f7f7f;
1214         op2ex <<= 1;
1215     }
1216     return result;
1217 }
1218
1219 #define NEON_FN(dest, src1, src2) dest = (src1 & src2) ? -1 : 0
1220 NEON_VOP(tst_u8, neon_u8, 4)
1221 NEON_VOP(tst_u16, neon_u16, 2)
1222 NEON_VOP(tst_u32, neon_u32, 1)
1223 #undef NEON_FN
1224
1225 #define NEON_FN(dest, src1, src2) dest = (src1 == src2) ? -1 : 0
1226 NEON_VOP(ceq_u8, neon_u8, 4)
1227 NEON_VOP(ceq_u16, neon_u16, 2)
1228 NEON_VOP(ceq_u32, neon_u32, 1)
1229 #undef NEON_FN
1230
1231 #define NEON_FN(dest, src, dummy) dest = (src < 0) ? -src : src
1232 NEON_VOP1(abs_s8, neon_s8, 4)
1233 NEON_VOP1(abs_s16, neon_s16, 2)
1234 #undef NEON_FN
1235
1236 /* Count Leading Sign/Zero Bits.  */
1237 static inline int do_clz8(uint8_t x)
1238 {
1239     int n;
1240     for (n = 8; x; n--)
1241         x >>= 1;
1242     return n;
1243 }
1244
1245 static inline int do_clz16(uint16_t x)
1246 {
1247     int n;
1248     for (n = 16; x; n--)
1249         x >>= 1;
1250     return n;
1251 }
1252
1253 #define NEON_FN(dest, src, dummy) dest = do_clz8(src)
1254 NEON_VOP1(clz_u8, neon_u8, 4)
1255 #undef NEON_FN
1256
1257 #define NEON_FN(dest, src, dummy) dest = do_clz16(src)
1258 NEON_VOP1(clz_u16, neon_u16, 2)
1259 #undef NEON_FN
1260
1261 #define NEON_FN(dest, src, dummy) dest = do_clz8((src < 0) ? ~src : src) - 1
1262 NEON_VOP1(cls_s8, neon_s8, 4)
1263 #undef NEON_FN
1264
1265 #define NEON_FN(dest, src, dummy) dest = do_clz16((src < 0) ? ~src : src) - 1
1266 NEON_VOP1(cls_s16, neon_s16, 2)
1267 #undef NEON_FN
1268
1269 uint32_t HELPER(neon_cls_s32)(uint32_t x)
1270 {
1271     int count;
1272     if ((int32_t)x < 0)
1273         x = ~x;
1274     for (count = 32; x; count--)
1275         x = x >> 1;
1276     return count - 1;
1277 }
1278
1279 /* Bit count.  */
1280 uint32_t HELPER(neon_cnt_u8)(uint32_t x)
1281 {
1282     x = (x & 0x55555555) + ((x >>  1) & 0x55555555);
1283     x = (x & 0x33333333) + ((x >>  2) & 0x33333333);
1284     x = (x & 0x0f0f0f0f) + ((x >>  4) & 0x0f0f0f0f);
1285     return x;
1286 }
1287
1288 /* Reverse bits in each 8 bit word */
1289 uint32_t HELPER(neon_rbit_u8)(uint32_t x)
1290 {
1291     x =  ((x & 0xf0f0f0f0) >> 4)
1292        | ((x & 0x0f0f0f0f) << 4);
1293     x =  ((x & 0x88888888) >> 3)
1294        | ((x & 0x44444444) >> 1)
1295        | ((x & 0x22222222) << 1)
1296        | ((x & 0x11111111) << 3);
1297     return x;
1298 }
1299
1300 #define NEON_QDMULH16(dest, src1, src2, round) do { \
1301     uint32_t tmp = (int32_t)(int16_t) src1 * (int16_t) src2; \
1302     if ((tmp ^ (tmp << 1)) & SIGNBIT) { \
1303         SET_QC(); \
1304         tmp = (tmp >> 31) ^ ~SIGNBIT; \
1305     } else { \
1306         tmp <<= 1; \
1307     } \
1308     if (round) { \
1309         int32_t old = tmp; \
1310         tmp += 1 << 15; \
1311         if ((int32_t)tmp < old) { \
1312             SET_QC(); \
1313             tmp = SIGNBIT - 1; \
1314         } \
1315     } \
1316     dest = tmp >> 16; \
1317     } while(0)
1318 #define NEON_FN(dest, src1, src2) NEON_QDMULH16(dest, src1, src2, 0)
1319 NEON_VOP_ENV(qdmulh_s16, neon_s16, 2)
1320 #undef NEON_FN
1321 #define NEON_FN(dest, src1, src2) NEON_QDMULH16(dest, src1, src2, 1)
1322 NEON_VOP_ENV(qrdmulh_s16, neon_s16, 2)
1323 #undef NEON_FN
1324 #undef NEON_QDMULH16
1325
1326 #define NEON_QDMULH32(dest, src1, src2, round) do { \
1327     uint64_t tmp = (int64_t)(int32_t) src1 * (int32_t) src2; \
1328     if ((tmp ^ (tmp << 1)) & SIGNBIT64) { \
1329         SET_QC(); \
1330         tmp = (tmp >> 63) ^ ~SIGNBIT64; \
1331     } else { \
1332         tmp <<= 1; \
1333     } \
1334     if (round) { \
1335         int64_t old = tmp; \
1336         tmp += (int64_t)1 << 31; \
1337         if ((int64_t)tmp < old) { \
1338             SET_QC(); \
1339             tmp = SIGNBIT64 - 1; \
1340         } \
1341     } \
1342     dest = tmp >> 32; \
1343     } while(0)
1344 #define NEON_FN(dest, src1, src2) NEON_QDMULH32(dest, src1, src2, 0)
1345 NEON_VOP_ENV(qdmulh_s32, neon_s32, 1)
1346 #undef NEON_FN
1347 #define NEON_FN(dest, src1, src2) NEON_QDMULH32(dest, src1, src2, 1)
1348 NEON_VOP_ENV(qrdmulh_s32, neon_s32, 1)
1349 #undef NEON_FN
1350 #undef NEON_QDMULH32
1351
1352 uint32_t HELPER(neon_narrow_u8)(uint64_t x)
1353 {
1354     return (x & 0xffu) | ((x >> 8) & 0xff00u) | ((x >> 16) & 0xff0000u)
1355            | ((x >> 24) & 0xff000000u);
1356 }
1357
1358 uint32_t HELPER(neon_narrow_u16)(uint64_t x)
1359 {
1360     return (x & 0xffffu) | ((x >> 16) & 0xffff0000u);
1361 }
1362
1363 uint32_t HELPER(neon_narrow_high_u8)(uint64_t x)
1364 {
1365     return ((x >> 8) & 0xff) | ((x >> 16) & 0xff00)
1366             | ((x >> 24) & 0xff0000) | ((x >> 32) & 0xff000000);
1367 }
1368
1369 uint32_t HELPER(neon_narrow_high_u16)(uint64_t x)
1370 {
1371     return ((x >> 16) & 0xffff) | ((x >> 32) & 0xffff0000);
1372 }
1373
1374 uint32_t HELPER(neon_narrow_round_high_u8)(uint64_t x)
1375 {
1376     x &= 0xff80ff80ff80ff80ull;
1377     x += 0x0080008000800080ull;
1378     return ((x >> 8) & 0xff) | ((x >> 16) & 0xff00)
1379             | ((x >> 24) & 0xff0000) | ((x >> 32) & 0xff000000);
1380 }
1381
1382 uint32_t HELPER(neon_narrow_round_high_u16)(uint64_t x)
1383 {
1384     x &= 0xffff8000ffff8000ull;
1385     x += 0x0000800000008000ull;
1386     return ((x >> 16) & 0xffff) | ((x >> 32) & 0xffff0000);
1387 }
1388
1389 uint32_t HELPER(neon_unarrow_sat8)(CPUARMState *env, uint64_t x)
1390 {
1391     uint16_t s;
1392     uint8_t d;
1393     uint32_t res = 0;
1394 #define SAT8(n) \
1395     s = x >> n; \
1396     if (s & 0x8000) { \
1397         SET_QC(); \
1398     } else { \
1399         if (s > 0xff) { \
1400             d = 0xff; \
1401             SET_QC(); \
1402         } else  { \
1403             d = s; \
1404         } \
1405         res |= (uint32_t)d << (n / 2); \
1406     }
1407
1408     SAT8(0);
1409     SAT8(16);
1410     SAT8(32);
1411     SAT8(48);
1412 #undef SAT8
1413     return res;
1414 }
1415
1416 uint32_t HELPER(neon_narrow_sat_u8)(CPUARMState *env, uint64_t x)
1417 {
1418     uint16_t s;
1419     uint8_t d;
1420     uint32_t res = 0;
1421 #define SAT8(n) \
1422     s = x >> n; \
1423     if (s > 0xff) { \
1424         d = 0xff; \
1425         SET_QC(); \
1426     } else  { \
1427         d = s; \
1428     } \
1429     res |= (uint32_t)d << (n / 2);
1430
1431     SAT8(0);
1432     SAT8(16);
1433     SAT8(32);
1434     SAT8(48);
1435 #undef SAT8
1436     return res;
1437 }
1438
1439 uint32_t HELPER(neon_narrow_sat_s8)(CPUARMState *env, uint64_t x)
1440 {
1441     int16_t s;
1442     uint8_t d;
1443     uint32_t res = 0;
1444 #define SAT8(n) \
1445     s = x >> n; \
1446     if (s != (int8_t)s) { \
1447         d = (s >> 15) ^ 0x7f; \
1448         SET_QC(); \
1449     } else  { \
1450         d = s; \
1451     } \
1452     res |= (uint32_t)d << (n / 2);
1453
1454     SAT8(0);
1455     SAT8(16);
1456     SAT8(32);
1457     SAT8(48);
1458 #undef SAT8
1459     return res;
1460 }
1461
1462 uint32_t HELPER(neon_unarrow_sat16)(CPUARMState *env, uint64_t x)
1463 {
1464     uint32_t high;
1465     uint32_t low;
1466     low = x;
1467     if (low & 0x80000000) {
1468         low = 0;
1469         SET_QC();
1470     } else if (low > 0xffff) {
1471         low = 0xffff;
1472         SET_QC();
1473     }
1474     high = x >> 32;
1475     if (high & 0x80000000) {
1476         high = 0;
1477         SET_QC();
1478     } else if (high > 0xffff) {
1479         high = 0xffff;
1480         SET_QC();
1481     }
1482     return low | (high << 16);
1483 }
1484
1485 uint32_t HELPER(neon_narrow_sat_u16)(CPUARMState *env, uint64_t x)
1486 {
1487     uint32_t high;
1488     uint32_t low;
1489     low = x;
1490     if (low > 0xffff) {
1491         low = 0xffff;
1492         SET_QC();
1493     }
1494     high = x >> 32;
1495     if (high > 0xffff) {
1496         high = 0xffff;
1497         SET_QC();
1498     }
1499     return low | (high << 16);
1500 }
1501
1502 uint32_t HELPER(neon_narrow_sat_s16)(CPUARMState *env, uint64_t x)
1503 {
1504     int32_t low;
1505     int32_t high;
1506     low = x;
1507     if (low != (int16_t)low) {
1508         low = (low >> 31) ^ 0x7fff;
1509         SET_QC();
1510     }
1511     high = x >> 32;
1512     if (high != (int16_t)high) {
1513         high = (high >> 31) ^ 0x7fff;
1514         SET_QC();
1515     }
1516     return (uint16_t)low | (high << 16);
1517 }
1518
1519 uint32_t HELPER(neon_unarrow_sat32)(CPUARMState *env, uint64_t x)
1520 {
1521     if (x & 0x8000000000000000ull) {
1522         SET_QC();
1523         return 0;
1524     }
1525     if (x > 0xffffffffu) {
1526         SET_QC();
1527         return 0xffffffffu;
1528     }
1529     return x;
1530 }
1531
1532 uint32_t HELPER(neon_narrow_sat_u32)(CPUARMState *env, uint64_t x)
1533 {
1534     if (x > 0xffffffffu) {
1535         SET_QC();
1536         return 0xffffffffu;
1537     }
1538     return x;
1539 }
1540
1541 uint32_t HELPER(neon_narrow_sat_s32)(CPUARMState *env, uint64_t x)
1542 {
1543     if ((int64_t)x != (int32_t)x) {
1544         SET_QC();
1545         return ((int64_t)x >> 63) ^ 0x7fffffff;
1546     }
1547     return x;
1548 }
1549
1550 uint64_t HELPER(neon_widen_u8)(uint32_t x)
1551 {
1552     uint64_t tmp;
1553     uint64_t ret;
1554     ret = (uint8_t)x;
1555     tmp = (uint8_t)(x >> 8);
1556     ret |= tmp << 16;
1557     tmp = (uint8_t)(x >> 16);
1558     ret |= tmp << 32;
1559     tmp = (uint8_t)(x >> 24);
1560     ret |= tmp << 48;
1561     return ret;
1562 }
1563
1564 uint64_t HELPER(neon_widen_s8)(uint32_t x)
1565 {
1566     uint64_t tmp;
1567     uint64_t ret;
1568     ret = (uint16_t)(int8_t)x;
1569     tmp = (uint16_t)(int8_t)(x >> 8);
1570     ret |= tmp << 16;
1571     tmp = (uint16_t)(int8_t)(x >> 16);
1572     ret |= tmp << 32;
1573     tmp = (uint16_t)(int8_t)(x >> 24);
1574     ret |= tmp << 48;
1575     return ret;
1576 }
1577
1578 uint64_t HELPER(neon_widen_u16)(uint32_t x)
1579 {
1580     uint64_t high = (uint16_t)(x >> 16);
1581     return ((uint16_t)x) | (high << 32);
1582 }
1583
1584 uint64_t HELPER(neon_widen_s16)(uint32_t x)
1585 {
1586     uint64_t high = (int16_t)(x >> 16);
1587     return ((uint32_t)(int16_t)x) | (high << 32);
1588 }
1589
1590 uint64_t HELPER(neon_addl_u16)(uint64_t a, uint64_t b)
1591 {
1592     uint64_t mask;
1593     mask = (a ^ b) & 0x8000800080008000ull;
1594     a &= ~0x8000800080008000ull;
1595     b &= ~0x8000800080008000ull;
1596     return (a + b) ^ mask;
1597 }
1598
1599 uint64_t HELPER(neon_addl_u32)(uint64_t a, uint64_t b)
1600 {
1601     uint64_t mask;
1602     mask = (a ^ b) & 0x8000000080000000ull;
1603     a &= ~0x8000000080000000ull;
1604     b &= ~0x8000000080000000ull;
1605     return (a + b) ^ mask;
1606 }
1607
1608 uint64_t HELPER(neon_paddl_u16)(uint64_t a, uint64_t b)
1609 {
1610     uint64_t tmp;
1611     uint64_t tmp2;
1612
1613     tmp = a & 0x0000ffff0000ffffull;
1614     tmp += (a >> 16) & 0x0000ffff0000ffffull;
1615     tmp2 = b & 0xffff0000ffff0000ull;
1616     tmp2 += (b << 16) & 0xffff0000ffff0000ull;
1617     return    ( tmp         & 0xffff)
1618             | ((tmp  >> 16) & 0xffff0000ull)
1619             | ((tmp2 << 16) & 0xffff00000000ull)
1620             | ( tmp2        & 0xffff000000000000ull);
1621 }
1622
1623 uint64_t HELPER(neon_paddl_u32)(uint64_t a, uint64_t b)
1624 {
1625     uint32_t low = a + (a >> 32);
1626     uint32_t high = b + (b >> 32);
1627     return low + ((uint64_t)high << 32);
1628 }
1629
1630 uint64_t HELPER(neon_subl_u16)(uint64_t a, uint64_t b)
1631 {
1632     uint64_t mask;
1633     mask = (a ^ ~b) & 0x8000800080008000ull;
1634     a |= 0x8000800080008000ull;
1635     b &= ~0x8000800080008000ull;
1636     return (a - b) ^ mask;
1637 }
1638
1639 uint64_t HELPER(neon_subl_u32)(uint64_t a, uint64_t b)
1640 {
1641     uint64_t mask;
1642     mask = (a ^ ~b) & 0x8000000080000000ull;
1643     a |= 0x8000000080000000ull;
1644     b &= ~0x8000000080000000ull;
1645     return (a - b) ^ mask;
1646 }
1647
1648 uint64_t HELPER(neon_addl_saturate_s32)(CPUARMState *env, uint64_t a, uint64_t b)
1649 {
1650     uint32_t x, y;
1651     uint32_t low, high;
1652
1653     x = a;
1654     y = b;
1655     low = x + y;
1656     if (((low ^ x) & SIGNBIT) && !((x ^ y) & SIGNBIT)) {
1657         SET_QC();
1658         low = ((int32_t)x >> 31) ^ ~SIGNBIT;
1659     }
1660     x = a >> 32;
1661     y = b >> 32;
1662     high = x + y;
1663     if (((high ^ x) & SIGNBIT) && !((x ^ y) & SIGNBIT)) {
1664         SET_QC();
1665         high = ((int32_t)x >> 31) ^ ~SIGNBIT;
1666     }
1667     return low | ((uint64_t)high << 32);
1668 }
1669
1670 uint64_t HELPER(neon_addl_saturate_s64)(CPUARMState *env, uint64_t a, uint64_t b)
1671 {
1672     uint64_t result;
1673
1674     result = a + b;
1675     if (((result ^ a) & SIGNBIT64) && !((a ^ b) & SIGNBIT64)) {
1676         SET_QC();
1677         result = ((int64_t)a >> 63) ^ ~SIGNBIT64;
1678     }
1679     return result;
1680 }
1681
1682 /* We have to do the arithmetic in a larger type than
1683  * the input type, because for example with a signed 32 bit
1684  * op the absolute difference can overflow a signed 32 bit value.
1685  */
1686 #define DO_ABD(dest, x, y, intype, arithtype) do {            \
1687     arithtype tmp_x = (intype)(x);                            \
1688     arithtype tmp_y = (intype)(y);                            \
1689     dest = ((tmp_x > tmp_y) ? tmp_x - tmp_y : tmp_y - tmp_x); \
1690     } while(0)
1691
1692 uint64_t HELPER(neon_abdl_u16)(uint32_t a, uint32_t b)
1693 {
1694     uint64_t tmp;
1695     uint64_t result;
1696     DO_ABD(result, a, b, uint8_t, uint32_t);
1697     DO_ABD(tmp, a >> 8, b >> 8, uint8_t, uint32_t);
1698     result |= tmp << 16;
1699     DO_ABD(tmp, a >> 16, b >> 16, uint8_t, uint32_t);
1700     result |= tmp << 32;
1701     DO_ABD(tmp, a >> 24, b >> 24, uint8_t, uint32_t);
1702     result |= tmp << 48;
1703     return result;
1704 }
1705
1706 uint64_t HELPER(neon_abdl_s16)(uint32_t a, uint32_t b)
1707 {
1708     uint64_t tmp;
1709     uint64_t result;
1710     DO_ABD(result, a, b, int8_t, int32_t);
1711     DO_ABD(tmp, a >> 8, b >> 8, int8_t, int32_t);
1712     result |= tmp << 16;
1713     DO_ABD(tmp, a >> 16, b >> 16, int8_t, int32_t);
1714     result |= tmp << 32;
1715     DO_ABD(tmp, a >> 24, b >> 24, int8_t, int32_t);
1716     result |= tmp << 48;
1717     return result;
1718 }
1719
1720 uint64_t HELPER(neon_abdl_u32)(uint32_t a, uint32_t b)
1721 {
1722     uint64_t tmp;
1723     uint64_t result;
1724     DO_ABD(result, a, b, uint16_t, uint32_t);
1725     DO_ABD(tmp, a >> 16, b >> 16, uint16_t, uint32_t);
1726     return result | (tmp << 32);
1727 }
1728
1729 uint64_t HELPER(neon_abdl_s32)(uint32_t a, uint32_t b)
1730 {
1731     uint64_t tmp;
1732     uint64_t result;
1733     DO_ABD(result, a, b, int16_t, int32_t);
1734     DO_ABD(tmp, a >> 16, b >> 16, int16_t, int32_t);
1735     return result | (tmp << 32);
1736 }
1737
1738 uint64_t HELPER(neon_abdl_u64)(uint32_t a, uint32_t b)
1739 {
1740     uint64_t result;
1741     DO_ABD(result, a, b, uint32_t, uint64_t);
1742     return result;
1743 }
1744
1745 uint64_t HELPER(neon_abdl_s64)(uint32_t a, uint32_t b)
1746 {
1747     uint64_t result;
1748     DO_ABD(result, a, b, int32_t, int64_t);
1749     return result;
1750 }
1751 #undef DO_ABD
1752
1753 /* Widening multiply. Named type is the source type.  */
1754 #define DO_MULL(dest, x, y, type1, type2) do { \
1755     type1 tmp_x = x; \
1756     type1 tmp_y = y; \
1757     dest = (type2)((type2)tmp_x * (type2)tmp_y); \
1758     } while(0)
1759
1760 uint64_t HELPER(neon_mull_u8)(uint32_t a, uint32_t b)
1761 {
1762     uint64_t tmp;
1763     uint64_t result;
1764
1765     DO_MULL(result, a, b, uint8_t, uint16_t);
1766     DO_MULL(tmp, a >> 8, b >> 8, uint8_t, uint16_t);
1767     result |= tmp << 16;
1768     DO_MULL(tmp, a >> 16, b >> 16, uint8_t, uint16_t);
1769     result |= tmp << 32;
1770     DO_MULL(tmp, a >> 24, b >> 24, uint8_t, uint16_t);
1771     result |= tmp << 48;
1772     return result;
1773 }
1774
1775 uint64_t HELPER(neon_mull_s8)(uint32_t a, uint32_t b)
1776 {
1777     uint64_t tmp;
1778     uint64_t result;
1779
1780     DO_MULL(result, a, b, int8_t, uint16_t);
1781     DO_MULL(tmp, a >> 8, b >> 8, int8_t, uint16_t);
1782     result |= tmp << 16;
1783     DO_MULL(tmp, a >> 16, b >> 16, int8_t, uint16_t);
1784     result |= tmp << 32;
1785     DO_MULL(tmp, a >> 24, b >> 24, int8_t, uint16_t);
1786     result |= tmp << 48;
1787     return result;
1788 }
1789
1790 uint64_t HELPER(neon_mull_u16)(uint32_t a, uint32_t b)
1791 {
1792     uint64_t tmp;
1793     uint64_t result;
1794
1795     DO_MULL(result, a, b, uint16_t, uint32_t);
1796     DO_MULL(tmp, a >> 16, b >> 16, uint16_t, uint32_t);
1797     return result | (tmp << 32);
1798 }
1799
1800 uint64_t HELPER(neon_mull_s16)(uint32_t a, uint32_t b)
1801 {
1802     uint64_t tmp;
1803     uint64_t result;
1804
1805     DO_MULL(result, a, b, int16_t, uint32_t);
1806     DO_MULL(tmp, a >> 16, b >> 16, int16_t, uint32_t);
1807     return result | (tmp << 32);
1808 }
1809
1810 uint64_t HELPER(neon_negl_u16)(uint64_t x)
1811 {
1812     uint16_t tmp;
1813     uint64_t result;
1814     result = (uint16_t)-x;
1815     tmp = -(x >> 16);
1816     result |= (uint64_t)tmp << 16;
1817     tmp = -(x >> 32);
1818     result |= (uint64_t)tmp << 32;
1819     tmp = -(x >> 48);
1820     result |= (uint64_t)tmp << 48;
1821     return result;
1822 }
1823
1824 uint64_t HELPER(neon_negl_u32)(uint64_t x)
1825 {
1826     uint32_t low = -x;
1827     uint32_t high = -(x >> 32);
1828     return low | ((uint64_t)high << 32);
1829 }
1830
1831 /* Saturating sign manipulation.  */
1832 /* ??? Make these use NEON_VOP1 */
1833 #define DO_QABS8(x) do { \
1834     if (x == (int8_t)0x80) { \
1835         x = 0x7f; \
1836         SET_QC(); \
1837     } else if (x < 0) { \
1838         x = -x; \
1839     }} while (0)
1840 uint32_t HELPER(neon_qabs_s8)(CPUARMState *env, uint32_t x)
1841 {
1842     neon_s8 vec;
1843     NEON_UNPACK(neon_s8, vec, x);
1844     DO_QABS8(vec.v1);
1845     DO_QABS8(vec.v2);
1846     DO_QABS8(vec.v3);
1847     DO_QABS8(vec.v4);
1848     NEON_PACK(neon_s8, x, vec);
1849     return x;
1850 }
1851 #undef DO_QABS8
1852
1853 #define DO_QNEG8(x) do { \
1854     if (x == (int8_t)0x80) { \
1855         x = 0x7f; \
1856         SET_QC(); \
1857     } else { \
1858         x = -x; \
1859     }} while (0)
1860 uint32_t HELPER(neon_qneg_s8)(CPUARMState *env, uint32_t x)
1861 {
1862     neon_s8 vec;
1863     NEON_UNPACK(neon_s8, vec, x);
1864     DO_QNEG8(vec.v1);
1865     DO_QNEG8(vec.v2);
1866     DO_QNEG8(vec.v3);
1867     DO_QNEG8(vec.v4);
1868     NEON_PACK(neon_s8, x, vec);
1869     return x;
1870 }
1871 #undef DO_QNEG8
1872
1873 #define DO_QABS16(x) do { \
1874     if (x == (int16_t)0x8000) { \
1875         x = 0x7fff; \
1876         SET_QC(); \
1877     } else if (x < 0) { \
1878         x = -x; \
1879     }} while (0)
1880 uint32_t HELPER(neon_qabs_s16)(CPUARMState *env, uint32_t x)
1881 {
1882     neon_s16 vec;
1883     NEON_UNPACK(neon_s16, vec, x);
1884     DO_QABS16(vec.v1);
1885     DO_QABS16(vec.v2);
1886     NEON_PACK(neon_s16, x, vec);
1887     return x;
1888 }
1889 #undef DO_QABS16
1890
1891 #define DO_QNEG16(x) do { \
1892     if (x == (int16_t)0x8000) { \
1893         x = 0x7fff; \
1894         SET_QC(); \
1895     } else { \
1896         x = -x; \
1897     }} while (0)
1898 uint32_t HELPER(neon_qneg_s16)(CPUARMState *env, uint32_t x)
1899 {
1900     neon_s16 vec;
1901     NEON_UNPACK(neon_s16, vec, x);
1902     DO_QNEG16(vec.v1);
1903     DO_QNEG16(vec.v2);
1904     NEON_PACK(neon_s16, x, vec);
1905     return x;
1906 }
1907 #undef DO_QNEG16
1908
1909 uint32_t HELPER(neon_qabs_s32)(CPUARMState *env, uint32_t x)
1910 {
1911     if (x == SIGNBIT) {
1912         SET_QC();
1913         x = ~SIGNBIT;
1914     } else if ((int32_t)x < 0) {
1915         x = -x;
1916     }
1917     return x;
1918 }
1919
1920 uint32_t HELPER(neon_qneg_s32)(CPUARMState *env, uint32_t x)
1921 {
1922     if (x == SIGNBIT) {
1923         SET_QC();
1924         x = ~SIGNBIT;
1925     } else {
1926         x = -x;
1927     }
1928     return x;
1929 }
1930
1931 uint64_t HELPER(neon_qabs_s64)(CPUARMState *env, uint64_t x)
1932 {
1933     if (x == SIGNBIT64) {
1934         SET_QC();
1935         x = ~SIGNBIT64;
1936     } else if ((int64_t)x < 0) {
1937         x = -x;
1938     }
1939     return x;
1940 }
1941
1942 uint64_t HELPER(neon_qneg_s64)(CPUARMState *env, uint64_t x)
1943 {
1944     if (x == SIGNBIT64) {
1945         SET_QC();
1946         x = ~SIGNBIT64;
1947     } else {
1948         x = -x;
1949     }
1950     return x;
1951 }
1952
1953 /* NEON Float helpers.  */
1954 uint32_t HELPER(neon_abd_f32)(uint32_t a, uint32_t b, void *fpstp)
1955 {
1956     float_status *fpst = fpstp;
1957     float32 f0 = make_float32(a);
1958     float32 f1 = make_float32(b);
1959     return float32_val(float32_abs(float32_sub(f0, f1, fpst)));
1960 }
1961
1962 /* Floating point comparisons produce an integer result.
1963  * Note that EQ doesn't signal InvalidOp for QNaNs but GE and GT do.
1964  * Softfloat routines return 0/1, which we convert to the 0/-1 Neon requires.
1965  */
1966 uint32_t HELPER(neon_ceq_f32)(uint32_t a, uint32_t b, void *fpstp)
1967 {
1968     float_status *fpst = fpstp;
1969     return -float32_eq_quiet(make_float32(a), make_float32(b), fpst);
1970 }
1971
1972 uint32_t HELPER(neon_cge_f32)(uint32_t a, uint32_t b, void *fpstp)
1973 {
1974     float_status *fpst = fpstp;
1975     return -float32_le(make_float32(b), make_float32(a), fpst);
1976 }
1977
1978 uint32_t HELPER(neon_cgt_f32)(uint32_t a, uint32_t b, void *fpstp)
1979 {
1980     float_status *fpst = fpstp;
1981     return -float32_lt(make_float32(b), make_float32(a), fpst);
1982 }
1983
1984 uint32_t HELPER(neon_acge_f32)(uint32_t a, uint32_t b, void *fpstp)
1985 {
1986     float_status *fpst = fpstp;
1987     float32 f0 = float32_abs(make_float32(a));
1988     float32 f1 = float32_abs(make_float32(b));
1989     return -float32_le(f1, f0, fpst);
1990 }
1991
1992 uint32_t HELPER(neon_acgt_f32)(uint32_t a, uint32_t b, void *fpstp)
1993 {
1994     float_status *fpst = fpstp;
1995     float32 f0 = float32_abs(make_float32(a));
1996     float32 f1 = float32_abs(make_float32(b));
1997     return -float32_lt(f1, f0, fpst);
1998 }
1999
2000 uint64_t HELPER(neon_acge_f64)(uint64_t a, uint64_t b, void *fpstp)
2001 {
2002     float_status *fpst = fpstp;
2003     float64 f0 = float64_abs(make_float64(a));
2004     float64 f1 = float64_abs(make_float64(b));
2005     return -float64_le(f1, f0, fpst);
2006 }
2007
2008 uint64_t HELPER(neon_acgt_f64)(uint64_t a, uint64_t b, void *fpstp)
2009 {
2010     float_status *fpst = fpstp;
2011     float64 f0 = float64_abs(make_float64(a));
2012     float64 f1 = float64_abs(make_float64(b));
2013     return -float64_lt(f1, f0, fpst);
2014 }
2015
2016 #define ELEM(V, N, SIZE) (((V) >> ((N) * (SIZE))) & ((1ull << (SIZE)) - 1))
2017
2018 void HELPER(neon_qunzip8)(void *vd, void *vm)
2019 {
2020     uint64_t *rd = vd, *rm = vm;
2021     uint64_t zd0 = rd[0], zd1 = rd[1];
2022     uint64_t zm0 = rm[0], zm1 = rm[1];
2023
2024     uint64_t d0 = ELEM(zd0, 0, 8) | (ELEM(zd0, 2, 8) << 8)
2025         | (ELEM(zd0, 4, 8) << 16) | (ELEM(zd0, 6, 8) << 24)
2026         | (ELEM(zd1, 0, 8) << 32) | (ELEM(zd1, 2, 8) << 40)
2027         | (ELEM(zd1, 4, 8) << 48) | (ELEM(zd1, 6, 8) << 56);
2028     uint64_t d1 = ELEM(zm0, 0, 8) | (ELEM(zm0, 2, 8) << 8)
2029         | (ELEM(zm0, 4, 8) << 16) | (ELEM(zm0, 6, 8) << 24)
2030         | (ELEM(zm1, 0, 8) << 32) | (ELEM(zm1, 2, 8) << 40)
2031         | (ELEM(zm1, 4, 8) << 48) | (ELEM(zm1, 6, 8) << 56);
2032     uint64_t m0 = ELEM(zd0, 1, 8) | (ELEM(zd0, 3, 8) << 8)
2033         | (ELEM(zd0, 5, 8) << 16) | (ELEM(zd0, 7, 8) << 24)
2034         | (ELEM(zd1, 1, 8) << 32) | (ELEM(zd1, 3, 8) << 40)
2035         | (ELEM(zd1, 5, 8) << 48) | (ELEM(zd1, 7, 8) << 56);
2036     uint64_t m1 = ELEM(zm0, 1, 8) | (ELEM(zm0, 3, 8) << 8)
2037         | (ELEM(zm0, 5, 8) << 16) | (ELEM(zm0, 7, 8) << 24)
2038         | (ELEM(zm1, 1, 8) << 32) | (ELEM(zm1, 3, 8) << 40)
2039         | (ELEM(zm1, 5, 8) << 48) | (ELEM(zm1, 7, 8) << 56);
2040
2041     rm[0] = m0;
2042     rm[1] = m1;
2043     rd[0] = d0;
2044     rd[1] = d1;
2045 }
2046
2047 void HELPER(neon_qunzip16)(void *vd, void *vm)
2048 {
2049     uint64_t *rd = vd, *rm = vm;
2050     uint64_t zd0 = rd[0], zd1 = rd[1];
2051     uint64_t zm0 = rm[0], zm1 = rm[1];
2052
2053     uint64_t d0 = ELEM(zd0, 0, 16) | (ELEM(zd0, 2, 16) << 16)
2054         | (ELEM(zd1, 0, 16) << 32) | (ELEM(zd1, 2, 16) << 48);
2055     uint64_t d1 = ELEM(zm0, 0, 16) | (ELEM(zm0, 2, 16) << 16)
2056         | (ELEM(zm1, 0, 16) << 32) | (ELEM(zm1, 2, 16) << 48);
2057     uint64_t m0 = ELEM(zd0, 1, 16) | (ELEM(zd0, 3, 16) << 16)
2058         | (ELEM(zd1, 1, 16) << 32) | (ELEM(zd1, 3, 16) << 48);
2059     uint64_t m1 = ELEM(zm0, 1, 16) | (ELEM(zm0, 3, 16) << 16)
2060         | (ELEM(zm1, 1, 16) << 32) | (ELEM(zm1, 3, 16) << 48);
2061
2062     rm[0] = m0;
2063     rm[1] = m1;
2064     rd[0] = d0;
2065     rd[1] = d1;
2066 }
2067
2068 void HELPER(neon_qunzip32)(void *vd, void *vm)
2069 {
2070     uint64_t *rd = vd, *rm = vm;
2071     uint64_t zd0 = rd[0], zd1 = rd[1];
2072     uint64_t zm0 = rm[0], zm1 = rm[1];
2073
2074     uint64_t d0 = ELEM(zd0, 0, 32) | (ELEM(zd1, 0, 32) << 32);
2075     uint64_t d1 = ELEM(zm0, 0, 32) | (ELEM(zm1, 0, 32) << 32);
2076     uint64_t m0 = ELEM(zd0, 1, 32) | (ELEM(zd1, 1, 32) << 32);
2077     uint64_t m1 = ELEM(zm0, 1, 32) | (ELEM(zm1, 1, 32) << 32);
2078
2079     rm[0] = m0;
2080     rm[1] = m1;
2081     rd[0] = d0;
2082     rd[1] = d1;
2083 }
2084
2085 void HELPER(neon_unzip8)(void *vd, void *vm)
2086 {
2087     uint64_t *rd = vd, *rm = vm;
2088     uint64_t zd = rd[0], zm = rm[0];
2089
2090     uint64_t d0 = ELEM(zd, 0, 8) | (ELEM(zd, 2, 8) << 8)
2091         | (ELEM(zd, 4, 8) << 16) | (ELEM(zd, 6, 8) << 24)
2092         | (ELEM(zm, 0, 8) << 32) | (ELEM(zm, 2, 8) << 40)
2093         | (ELEM(zm, 4, 8) << 48) | (ELEM(zm, 6, 8) << 56);
2094     uint64_t m0 = ELEM(zd, 1, 8) | (ELEM(zd, 3, 8) << 8)
2095         | (ELEM(zd, 5, 8) << 16) | (ELEM(zd, 7, 8) << 24)
2096         | (ELEM(zm, 1, 8) << 32) | (ELEM(zm, 3, 8) << 40)
2097         | (ELEM(zm, 5, 8) << 48) | (ELEM(zm, 7, 8) << 56);
2098
2099     rm[0] = m0;
2100     rd[0] = d0;
2101 }
2102
2103 void HELPER(neon_unzip16)(void *vd, void *vm)
2104 {
2105     uint64_t *rd = vd, *rm = vm;
2106     uint64_t zd = rd[0], zm = rm[0];
2107
2108     uint64_t d0 = ELEM(zd, 0, 16) | (ELEM(zd, 2, 16) << 16)
2109         | (ELEM(zm, 0, 16) << 32) | (ELEM(zm, 2, 16) << 48);
2110     uint64_t m0 = ELEM(zd, 1, 16) | (ELEM(zd, 3, 16) << 16)
2111         | (ELEM(zm, 1, 16) << 32) | (ELEM(zm, 3, 16) << 48);
2112
2113     rm[0] = m0;
2114     rd[0] = d0;
2115 }
2116
2117 void HELPER(neon_qzip8)(void *vd, void *vm)
2118 {
2119     uint64_t *rd = vd, *rm = vm;
2120     uint64_t zd0 = rd[0], zd1 = rd[1];
2121     uint64_t zm0 = rm[0], zm1 = rm[1];
2122
2123     uint64_t d0 = ELEM(zd0, 0, 8) | (ELEM(zm0, 0, 8) << 8)
2124         | (ELEM(zd0, 1, 8) << 16) | (ELEM(zm0, 1, 8) << 24)
2125         | (ELEM(zd0, 2, 8) << 32) | (ELEM(zm0, 2, 8) << 40)
2126         | (ELEM(zd0, 3, 8) << 48) | (ELEM(zm0, 3, 8) << 56);
2127     uint64_t d1 = ELEM(zd0, 4, 8) | (ELEM(zm0, 4, 8) << 8)
2128         | (ELEM(zd0, 5, 8) << 16) | (ELEM(zm0, 5, 8) << 24)
2129         | (ELEM(zd0, 6, 8) << 32) | (ELEM(zm0, 6, 8) << 40)
2130         | (ELEM(zd0, 7, 8) << 48) | (ELEM(zm0, 7, 8) << 56);
2131     uint64_t m0 = ELEM(zd1, 0, 8) | (ELEM(zm1, 0, 8) << 8)
2132         | (ELEM(zd1, 1, 8) << 16) | (ELEM(zm1, 1, 8) << 24)
2133         | (ELEM(zd1, 2, 8) << 32) | (ELEM(zm1, 2, 8) << 40)
2134         | (ELEM(zd1, 3, 8) << 48) | (ELEM(zm1, 3, 8) << 56);
2135     uint64_t m1 = ELEM(zd1, 4, 8) | (ELEM(zm1, 4, 8) << 8)
2136         | (ELEM(zd1, 5, 8) << 16) | (ELEM(zm1, 5, 8) << 24)
2137         | (ELEM(zd1, 6, 8) << 32) | (ELEM(zm1, 6, 8) << 40)
2138         | (ELEM(zd1, 7, 8) << 48) | (ELEM(zm1, 7, 8) << 56);
2139
2140     rm[0] = m0;
2141     rm[1] = m1;
2142     rd[0] = d0;
2143     rd[1] = d1;
2144 }
2145
2146 void HELPER(neon_qzip16)(void *vd, void *vm)
2147 {
2148     uint64_t *rd = vd, *rm = vm;
2149     uint64_t zd0 = rd[0], zd1 = rd[1];
2150     uint64_t zm0 = rm[0], zm1 = rm[1];
2151
2152     uint64_t d0 = ELEM(zd0, 0, 16) | (ELEM(zm0, 0, 16) << 16)
2153         | (ELEM(zd0, 1, 16) << 32) | (ELEM(zm0, 1, 16) << 48);
2154     uint64_t d1 = ELEM(zd0, 2, 16) | (ELEM(zm0, 2, 16) << 16)
2155         | (ELEM(zd0, 3, 16) << 32) | (ELEM(zm0, 3, 16) << 48);
2156     uint64_t m0 = ELEM(zd1, 0, 16) | (ELEM(zm1, 0, 16) << 16)
2157         | (ELEM(zd1, 1, 16) << 32) | (ELEM(zm1, 1, 16) << 48);
2158     uint64_t m1 = ELEM(zd1, 2, 16) | (ELEM(zm1, 2, 16) << 16)
2159         | (ELEM(zd1, 3, 16) << 32) | (ELEM(zm1, 3, 16) << 48);
2160
2161     rm[0] = m0;
2162     rm[1] = m1;
2163     rd[0] = d0;
2164     rd[1] = d1;
2165 }
2166
2167 void HELPER(neon_qzip32)(void *vd, void *vm)
2168 {
2169     uint64_t *rd = vd, *rm = vm;
2170     uint64_t zd0 = rd[0], zd1 = rd[1];
2171     uint64_t zm0 = rm[0], zm1 = rm[1];
2172
2173     uint64_t d0 = ELEM(zd0, 0, 32) | (ELEM(zm0, 0, 32) << 32);
2174     uint64_t d1 = ELEM(zd0, 1, 32) | (ELEM(zm0, 1, 32) << 32);
2175     uint64_t m0 = ELEM(zd1, 0, 32) | (ELEM(zm1, 0, 32) << 32);
2176     uint64_t m1 = ELEM(zd1, 1, 32) | (ELEM(zm1, 1, 32) << 32);
2177
2178     rm[0] = m0;
2179     rm[1] = m1;
2180     rd[0] = d0;
2181     rd[1] = d1;
2182 }
2183
2184 void HELPER(neon_zip8)(void *vd, void *vm)
2185 {
2186     uint64_t *rd = vd, *rm = vm;
2187     uint64_t zd = rd[0], zm = rm[0];
2188
2189     uint64_t d0 = ELEM(zd, 0, 8) | (ELEM(zm, 0, 8) << 8)
2190         | (ELEM(zd, 1, 8) << 16) | (ELEM(zm, 1, 8) << 24)
2191         | (ELEM(zd, 2, 8) << 32) | (ELEM(zm, 2, 8) << 40)
2192         | (ELEM(zd, 3, 8) << 48) | (ELEM(zm, 3, 8) << 56);
2193     uint64_t m0 = ELEM(zd, 4, 8) | (ELEM(zm, 4, 8) << 8)
2194         | (ELEM(zd, 5, 8) << 16) | (ELEM(zm, 5, 8) << 24)
2195         | (ELEM(zd, 6, 8) << 32) | (ELEM(zm, 6, 8) << 40)
2196         | (ELEM(zd, 7, 8) << 48) | (ELEM(zm, 7, 8) << 56);
2197
2198     rm[0] = m0;
2199     rd[0] = d0;
2200 }
2201
2202 void HELPER(neon_zip16)(void *vd, void *vm)
2203 {
2204     uint64_t *rd = vd, *rm = vm;
2205     uint64_t zd = rd[0], zm = rm[0];
2206
2207     uint64_t d0 = ELEM(zd, 0, 16) | (ELEM(zm, 0, 16) << 16)
2208         | (ELEM(zd, 1, 16) << 32) | (ELEM(zm, 1, 16) << 48);
2209     uint64_t m0 = ELEM(zd, 2, 16) | (ELEM(zm, 2, 16) << 16)
2210         | (ELEM(zd, 3, 16) << 32) | (ELEM(zm, 3, 16) << 48);
2211
2212     rm[0] = m0;
2213     rd[0] = d0;
2214 }
2215
2216 /* Helper function for 64 bit polynomial multiply case:
2217  * perform PolynomialMult(op1, op2) and return either the top or
2218  * bottom half of the 128 bit result.
2219  */
2220 uint64_t HELPER(neon_pmull_64_lo)(uint64_t op1, uint64_t op2)
2221 {
2222     int bitnum;
2223     uint64_t res = 0;
2224
2225     for (bitnum = 0; bitnum < 64; bitnum++) {
2226         if (op1 & (1ULL << bitnum)) {
2227             res ^= op2 << bitnum;
2228         }
2229     }
2230     return res;
2231 }
2232 uint64_t HELPER(neon_pmull_64_hi)(uint64_t op1, uint64_t op2)
2233 {
2234     int bitnum;
2235     uint64_t res = 0;
2236
2237     /* bit 0 of op1 can't influence the high 64 bits at all */
2238     for (bitnum = 1; bitnum < 64; bitnum++) {
2239         if (op1 & (1ULL << bitnum)) {
2240             res ^= op2 >> (64 - bitnum);
2241         }
2242     }
2243     return res;
2244 }