target-arm/neon_helper.c

   1 /*
   2  * ARM NEON vector operations.
   3  *
   4  * Copyright (c) 2007, 2008 CodeSourcery.
   5  * Written by Paul Brook
   6  *
   7  * This code is licenced under the GNU GPL v2.
   8  */
   9 #include <stdlib.h>
  10 #include <stdio.h>
  11
  12 #include "cpu.h"
  13 #include "exec-all.h"
  14 #include "helpers.h"
  15
  16 #define SIGNBIT (uint32_t)0x80000000
  17 #define SIGNBIT64 ((uint64_t)1 << 63)
  18
  19 #define SET_QC() env->vfp.xregs[ARM_VFP_FPSCR] = CPSR_Q
  20
  21 static float_status neon_float_status;
  22 #define NFS &neon_float_status
  23
  24 /* Helper routines to perform bitwise copies between float and int.  */
  25 static inline float32 vfp_itos(uint32_t i)
  26 {
  27     union {
  28         uint32_t i;
  29         float32 s;
  30     } v;
  31
  32     v.i = i;
  33     return v.s;
  34 }
  35
  36 static inline uint32_t vfp_stoi(float32 s)
  37 {
  38     union {
  39         uint32_t i;
  40         float32 s;
  41     } v;
  42
  43     v.s = s;
  44     return v.i;
  45 }
  46
  47 #define NEON_TYPE1(name, type) \
  48 typedef struct \
  49 { \
  50     type v1; \
  51 } neon_##name;
  52 #ifdef HOST_WORDS_BIGENDIAN
  53 #define NEON_TYPE2(name, type) \
  54 typedef struct \
  55 { \
  56     type v2; \
  57     type v1; \
  58 } neon_##name;
  59 #define NEON_TYPE4(name, type) \
  60 typedef struct \
  61 { \
  62     type v4; \
  63     type v3; \
  64     type v2; \
  65     type v1; \
  66 } neon_##name;
  67 #else
  68 #define NEON_TYPE2(name, type) \
  69 typedef struct \
  70 { \
  71     type v1; \
  72     type v2; \
  73 } neon_##name;
  74 #define NEON_TYPE4(name, type) \
  75 typedef struct \
  76 { \
  77     type v1; \
  78     type v2; \
  79     type v3; \
  80     type v4; \
  81 } neon_##name;
  82 #endif
  83
  84 NEON_TYPE4(s8, int8_t)
  85 NEON_TYPE4(u8, uint8_t)
  86 NEON_TYPE2(s16, int16_t)
  87 NEON_TYPE2(u16, uint16_t)
  88 NEON_TYPE1(s32, int32_t)
  89 NEON_TYPE1(u32, uint32_t)
  90 #undef NEON_TYPE4
  91 #undef NEON_TYPE2
  92 #undef NEON_TYPE1
  93
  94 /* Copy from a uint32_t to a vector structure type.  */
  95 #define NEON_UNPACK(vtype, dest, val) do { \
  96     union { \
  97         vtype v; \
  98         uint32_t i; \
  99     } conv_u; \
 100     conv_u.i = (val); \
 101     dest = conv_u.v; \
 102     } while(0)
 103
 104 /* Copy from a vector structure type to a uint32_t.  */
 105 #define NEON_PACK(vtype, dest, val) do { \
 106     union { \
 107         vtype v; \
 108         uint32_t i; \
 109     } conv_u; \
 110     conv_u.v = (val); \
 111     dest = conv_u.i; \
 112     } while(0)
 113
 114 #define NEON_DO1 \
 115     NEON_FN(vdest.v1, vsrc1.v1, vsrc2.v1);
 116 #define NEON_DO2 \
 117     NEON_FN(vdest.v1, vsrc1.v1, vsrc2.v1); \
 118     NEON_FN(vdest.v2, vsrc1.v2, vsrc2.v2);
 119 #define NEON_DO4 \
 120     NEON_FN(vdest.v1, vsrc1.v1, vsrc2.v1); \
 121     NEON_FN(vdest.v2, vsrc1.v2, vsrc2.v2); \
 122     NEON_FN(vdest.v3, vsrc1.v3, vsrc2.v3); \
 123     NEON_FN(vdest.v4, vsrc1.v4, vsrc2.v4);
 124
 125 #define NEON_VOP_BODY(vtype, n) \
 126 { \
 127     uint32_t res; \
 128     vtype vsrc1; \
 129     vtype vsrc2; \
 130     vtype vdest; \
 131     NEON_UNPACK(vtype, vsrc1, arg1); \
 132     NEON_UNPACK(vtype, vsrc2, arg2); \
 133     NEON_DO##n; \
 134     NEON_PACK(vtype, res, vdest); \
 135     return res; \
 136 }
 137
 138 #define NEON_VOP(name, vtype, n) \
 139 uint32_t HELPER(glue(neon_,name))(uint32_t arg1, uint32_t arg2) \
 140 NEON_VOP_BODY(vtype, n)
 141
 142 #define NEON_VOP_ENV(name, vtype, n) \
 143 uint32_t HELPER(glue(neon_,name))(CPUState *env, uint32_t arg1, uint32_t arg2) \
 144 NEON_VOP_BODY(vtype, n)
 145
 146 /* Pairwise operations.  */
 147 /* For 32-bit elements each segment only contains a single element, so
 148    the elementwise and pairwise operations are the same.  */
 149 #define NEON_PDO2 \
 150     NEON_FN(vdest.v1, vsrc1.v1, vsrc1.v2); \
 151     NEON_FN(vdest.v2, vsrc2.v1, vsrc2.v2);
 152 #define NEON_PDO4 \
 153     NEON_FN(vdest.v1, vsrc1.v1, vsrc1.v2); \
 154     NEON_FN(vdest.v2, vsrc1.v3, vsrc1.v4); \
 155     NEON_FN(vdest.v3, vsrc2.v1, vsrc2.v2); \
 156     NEON_FN(vdest.v4, vsrc2.v3, vsrc2.v4); \
 157
 158 #define NEON_POP(name, vtype, n) \
 159 uint32_t HELPER(glue(neon_,name))(uint32_t arg1, uint32_t arg2) \
 160 { \
 161     uint32_t res; \
 162     vtype vsrc1; \
 163     vtype vsrc2; \
 164     vtype vdest; \
 165     NEON_UNPACK(vtype, vsrc1, arg1); \
 166     NEON_UNPACK(vtype, vsrc2, arg2); \
 167     NEON_PDO##n; \
 168     NEON_PACK(vtype, res, vdest); \
 169     return res; \
 170 }
 171
 172 /* Unary operators.  */
 173 #define NEON_VOP1(name, vtype, n) \
 174 uint32_t HELPER(glue(neon_,name))(uint32_t arg) \
 175 { \
 176     vtype vsrc1; \
 177     vtype vdest; \
 178     NEON_UNPACK(vtype, vsrc1, arg); \
 179     NEON_DO##n; \
 180     NEON_PACK(vtype, arg, vdest); \
 181     return arg; \
 182 }
 183
 184
 185 #define NEON_USAT(dest, src1, src2, type) do { \
 186     uint32_t tmp = (uint32_t)src1 + (uint32_t)src2; \
 187     if (tmp != (type)tmp) { \
 188         SET_QC(); \
 189         dest = ~0; \
 190     } else { \
 191         dest = tmp; \
 192     }} while(0)
 193 #define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint8_t)
 194 NEON_VOP_ENV(qadd_u8, neon_u8, 4)
 195 #undef NEON_FN
 196 #define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint16_t)
 197 NEON_VOP_ENV(qadd_u16, neon_u16, 2)
 198 #undef NEON_FN
 199 #undef NEON_USAT
 200
 201 #define NEON_SSAT(dest, src1, src2, type) do { \
 202     int32_t tmp = (uint32_t)src1 + (uint32_t)src2; \
 203     if (tmp != (type)tmp) { \
 204         SET_QC(); \
 205         if (src2 > 0) { \
 206             tmp = (1 << (sizeof(type) * 8 - 1)) - 1; \
 207         } else { \
 208             tmp = 1 << (sizeof(type) * 8 - 1); \
 209         } \
 210     } \
 211     dest = tmp; \
 212     } while(0)
 213 #define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int8_t)
 214 NEON_VOP_ENV(qadd_s8, neon_s8, 4)
 215 #undef NEON_FN
 216 #define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int16_t)
 217 NEON_VOP_ENV(qadd_s16, neon_s16, 2)
 218 #undef NEON_FN
 219 #undef NEON_SSAT
 220
 221 #define NEON_USAT(dest, src1, src2, type) do { \
 222     uint32_t tmp = (uint32_t)src1 - (uint32_t)src2; \
 223     if (tmp != (type)tmp) { \
 224         SET_QC(); \
 225         dest = 0; \
 226     } else { \
 227         dest = tmp; \
 228     }} while(0)
 229 #define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint8_t)
 230 NEON_VOP_ENV(qsub_u8, neon_u8, 4)
 231 #undef NEON_FN
 232 #define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint16_t)
 233 NEON_VOP_ENV(qsub_u16, neon_u16, 2)
 234 #undef NEON_FN
 235 #undef NEON_USAT
 236
 237 #define NEON_SSAT(dest, src1, src2, type) do { \
 238     int32_t tmp = (uint32_t)src1 - (uint32_t)src2; \
 239     if (tmp != (type)tmp) { \
 240         SET_QC(); \
 241         if (src2 < 0) { \
 242             tmp = (1 << (sizeof(type) * 8 - 1)) - 1; \
 243         } else { \
 244             tmp = 1 << (sizeof(type) * 8 - 1); \
 245         } \
 246     } \
 247     dest = tmp; \
 248     } while(0)
 249 #define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int8_t)
 250 NEON_VOP_ENV(qsub_s8, neon_s8, 4)
 251 #undef NEON_FN
 252 #define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int16_t)
 253 NEON_VOP_ENV(qsub_s16, neon_s16, 2)
 254 #undef NEON_FN
 255 #undef NEON_SSAT
 256
 257 #define NEON_FN(dest, src1, src2) dest = (src1 + src2) >> 1
 258 NEON_VOP(hadd_s8, neon_s8, 4)
 259 NEON_VOP(hadd_u8, neon_u8, 4)
 260 NEON_VOP(hadd_s16, neon_s16, 2)
 261 NEON_VOP(hadd_u16, neon_u16, 2)
 262 #undef NEON_FN
 263
 264 int32_t HELPER(neon_hadd_s32)(int32_t src1, int32_t src2)
 265 {
 266     int32_t dest;
 267
 268     dest = (src1 >> 1) + (src2 >> 1);
 269     if (src1 & src2 & 1)
 270         dest++;
 271     return dest;
 272 }
 273
 274 uint32_t HELPER(neon_hadd_u32)(uint32_t src1, uint32_t src2)
 275 {
 276     uint32_t dest;
 277
 278     dest = (src1 >> 1) + (src2 >> 1);
 279     if (src1 & src2 & 1)
 280         dest++;
 281     return dest;
 282 }
 283
 284 #define NEON_FN(dest, src1, src2) dest = (src1 + src2 + 1) >> 1
 285 NEON_VOP(rhadd_s8, neon_s8, 4)
 286 NEON_VOP(rhadd_u8, neon_u8, 4)
 287 NEON_VOP(rhadd_s16, neon_s16, 2)
 288 NEON_VOP(rhadd_u16, neon_u16, 2)
 289 #undef NEON_FN
 290
 291 int32_t HELPER(neon_rhadd_s32)(int32_t src1, int32_t src2)
 292 {
 293     int32_t dest;
 294
 295     dest = (src1 >> 1) + (src2 >> 1);
 296     if ((src1 | src2) & 1)
 297         dest++;
 298     return dest;
 299 }
 300
 301 uint32_t HELPER(neon_rhadd_u32)(uint32_t src1, uint32_t src2)
 302 {
 303     uint32_t dest;
 304
 305     dest = (src1 >> 1) + (src2 >> 1);
 306     if ((src1 | src2) & 1)
 307         dest++;
 308     return dest;
 309 }
 310
 311 #define NEON_FN(dest, src1, src2) dest = (src1 - src2) >> 1
 312 NEON_VOP(hsub_s8, neon_s8, 4)
 313 NEON_VOP(hsub_u8, neon_u8, 4)
 314 NEON_VOP(hsub_s16, neon_s16, 2)
 315 NEON_VOP(hsub_u16, neon_u16, 2)
 316 #undef NEON_FN
 317
 318 int32_t HELPER(neon_hsub_s32)(int32_t src1, int32_t src2)
 319 {
 320     int32_t dest;
 321
 322     dest = (src1 >> 1) - (src2 >> 1);
 323     if ((~src1) & src2 & 1)
 324         dest--;
 325     return dest;
 326 }
 327
 328 uint32_t HELPER(neon_hsub_u32)(uint32_t src1, uint32_t src2)
 329 {
 330     uint32_t dest;
 331
 332     dest = (src1 >> 1) - (src2 >> 1);
 333     if ((~src1) & src2 & 1)
 334         dest--;
 335     return dest;
 336 }
 337
 338 #define NEON_FN(dest, src1, src2) dest = (src1 > src2) ? ~0 : 0
 339 NEON_VOP(cgt_s8, neon_s8, 4)
 340 NEON_VOP(cgt_u8, neon_u8, 4)
 341 NEON_VOP(cgt_s16, neon_s16, 2)
 342 NEON_VOP(cgt_u16, neon_u16, 2)
 343 NEON_VOP(cgt_s32, neon_s32, 1)
 344 NEON_VOP(cgt_u32, neon_u32, 1)
 345 #undef NEON_FN
 346
 347 #define NEON_FN(dest, src1, src2) dest = (src1 >= src2) ? ~0 : 0
 348 NEON_VOP(cge_s8, neon_s8, 4)
 349 NEON_VOP(cge_u8, neon_u8, 4)
 350 NEON_VOP(cge_s16, neon_s16, 2)
 351 NEON_VOP(cge_u16, neon_u16, 2)
 352 NEON_VOP(cge_s32, neon_s32, 1)
 353 NEON_VOP(cge_u32, neon_u32, 1)
 354 #undef NEON_FN
 355
 356 #define NEON_FN(dest, src1, src2) dest = (src1 < src2) ? src1 : src2
 357 NEON_VOP(min_s8, neon_s8, 4)
 358 NEON_VOP(min_u8, neon_u8, 4)
 359 NEON_VOP(min_s16, neon_s16, 2)
 360 NEON_VOP(min_u16, neon_u16, 2)
 361 NEON_VOP(min_s32, neon_s32, 1)
 362 NEON_VOP(min_u32, neon_u32, 1)
 363 NEON_POP(pmin_s8, neon_s8, 4)
 364 NEON_POP(pmin_u8, neon_u8, 4)
 365 NEON_POP(pmin_s16, neon_s16, 2)
 366 NEON_POP(pmin_u16, neon_u16, 2)
 367 #undef NEON_FN
 368
 369 #define NEON_FN(dest, src1, src2) dest = (src1 > src2) ? src1 : src2
 370 NEON_VOP(max_s8, neon_s8, 4)
 371 NEON_VOP(max_u8, neon_u8, 4)
 372 NEON_VOP(max_s16, neon_s16, 2)
 373 NEON_VOP(max_u16, neon_u16, 2)
 374 NEON_VOP(max_s32, neon_s32, 1)
 375 NEON_VOP(max_u32, neon_u32, 1)
 376 NEON_POP(pmax_s8, neon_s8, 4)
 377 NEON_POP(pmax_u8, neon_u8, 4)
 378 NEON_POP(pmax_s16, neon_s16, 2)
 379 NEON_POP(pmax_u16, neon_u16, 2)
 380 #undef NEON_FN
 381
 382 #define NEON_FN(dest, src1, src2) \
 383     dest = (src1 > src2) ? (src1 - src2) : (src2 - src1)
 384 NEON_VOP(abd_s8, neon_s8, 4)
 385 NEON_VOP(abd_u8, neon_u8, 4)
 386 NEON_VOP(abd_s16, neon_s16, 2)
 387 NEON_VOP(abd_u16, neon_u16, 2)
 388 NEON_VOP(abd_s32, neon_s32, 1)
 389 NEON_VOP(abd_u32, neon_u32, 1)
 390 #undef NEON_FN
 391
 392 #define NEON_FN(dest, src1, src2) do { \
 393     int8_t tmp; \
 394     tmp = (int8_t)src2; \
 395     if (tmp >= (ssize_t)sizeof(src1) * 8 || \
 396         tmp <= -(ssize_t)sizeof(src1) * 8) { \
 397         dest = 0; \
 398     } else if (tmp < 0) { \
 399         dest = src1 >> -tmp; \
 400     } else { \
 401         dest = src1 << tmp; \
 402     }} while (0)
 403 NEON_VOP(shl_u8, neon_u8, 4)
 404 NEON_VOP(shl_u16, neon_u16, 2)
 405 NEON_VOP(shl_u32, neon_u32, 1)
 406 #undef NEON_FN
 407
 408 uint64_t HELPER(neon_shl_u64)(uint64_t val, uint64_t shiftop)
 409 {
 410     int8_t shift = (int8_t)shiftop;
 411     if (shift >= 64 || shift <= -64) {
 412         val = 0;
 413     } else if (shift < 0) {
 414         val >>= -shift;
 415     } else {
 416         val <<= shift;
 417     }
 418     return val;
 419 }
 420
 421 #define NEON_FN(dest, src1, src2) do { \
 422     int8_t tmp; \
 423     tmp = (int8_t)src2; \
 424     if (tmp >= (ssize_t)sizeof(src1) * 8) { \
 425         dest = 0; \
 426     } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
 427         dest = src1 >> (sizeof(src1) * 8 - 1); \
 428     } else if (tmp < 0) { \
 429         dest = src1 >> -tmp; \
 430     } else { \
 431         dest = src1 << tmp; \
 432     }} while (0)
 433 NEON_VOP(shl_s8, neon_s8, 4)
 434 NEON_VOP(shl_s16, neon_s16, 2)
 435 NEON_VOP(shl_s32, neon_s32, 1)
 436 #undef NEON_FN
 437
 438 uint64_t HELPER(neon_shl_s64)(uint64_t valop, uint64_t shiftop)
 439 {
 440     int8_t shift = (int8_t)shiftop;
 441     int64_t val = valop;
 442     if (shift >= 64) {
 443         val = 0;
 444     } else if (shift <= -64) {
 445         val >>= 63;
 446     } else if (shift < 0) {
 447         val >>= -shift;
 448     } else {
 449         val <<= shift;
 450     }
 451     return val;
 452 }
 453
 454 #define NEON_FN(dest, src1, src2) do { \
 455     int8_t tmp; \
 456     tmp = (int8_t)src2; \
 457     if (tmp >= (ssize_t)sizeof(src1) * 8) { \
 458         dest = 0; \
 459     } else if (tmp < -(ssize_t)sizeof(src1) * 8) { \
 460         dest = src1 >> (sizeof(src1) * 8 - 1); \
 461     } else if (tmp == -(ssize_t)sizeof(src1) * 8) { \
 462         dest = src1 >> (tmp - 1); \
 463         dest++; \
 464         dest >>= 1; \
 465     } else if (tmp < 0) { \
 466         dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
 467     } else { \
 468         dest = src1 << tmp; \
 469     }} while (0)
 470 NEON_VOP(rshl_s8, neon_s8, 4)
 471 NEON_VOP(rshl_s16, neon_s16, 2)
 472 NEON_VOP(rshl_s32, neon_s32, 1)
 473 #undef NEON_FN
 474
 475 uint64_t HELPER(neon_rshl_s64)(uint64_t valop, uint64_t shiftop)
 476 {
 477     int8_t shift = (int8_t)shiftop;
 478     int64_t val = valop;
 479     if (shift >= 64) {
 480         val = 0;
 481     } else if (shift < -64) {
 482         val >>= 63;
 483     } else if (shift == -63) {
 484         val >>= 63;
 485         val++;
 486         val >>= 1;
 487     } else if (shift < 0) {
 488         val = (val + ((int64_t)1 << (-1 - shift))) >> -shift;
 489     } else {
 490         val <<= shift;
 491     }
 492     return val;
 493 }
 494
 495 #define NEON_FN(dest, src1, src2) do { \
 496     int8_t tmp; \
 497     tmp = (int8_t)src2; \
 498     if (tmp >= (ssize_t)sizeof(src1) * 8 || \
 499         tmp < -(ssize_t)sizeof(src1) * 8) { \
 500         dest = 0; \
 501     } else if (tmp == -(ssize_t)sizeof(src1) * 8) { \
 502         dest = src1 >> (tmp - 1); \
 503     } else if (tmp < 0) { \
 504         dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
 505     } else { \
 506         dest = src1 << tmp; \
 507     }} while (0)
 508 NEON_VOP(rshl_u8, neon_u8, 4)
 509 NEON_VOP(rshl_u16, neon_u16, 2)
 510 NEON_VOP(rshl_u32, neon_u32, 1)
 511 #undef NEON_FN
 512
 513 uint64_t HELPER(neon_rshl_u64)(uint64_t val, uint64_t shiftop)
 514 {
 515     int8_t shift = (uint8_t)shiftop;
 516     if (shift >= 64 || shift < 64) {
 517         val = 0;
 518     } else if (shift == -64) {
 519         /* Rounding a 1-bit result just preserves that bit.  */
 520         val >>= 63;
 521     } if (shift < 0) {
 522         val = (val + ((uint64_t)1 << (-1 - shift))) >> -shift;
 523         val >>= -shift;
 524     } else {
 525         val <<= shift;
 526     }
 527     return val;
 528 }
 529
 530 #define NEON_FN(dest, src1, src2) do { \
 531     int8_t tmp; \
 532     tmp = (int8_t)src2; \
 533     if (tmp >= (ssize_t)sizeof(src1) * 8) { \
 534         if (src1) { \
 535             SET_QC(); \
 536             dest = ~0; \
 537         } else { \
 538             dest = 0; \
 539         } \
 540     } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
 541         dest = 0; \
 542     } else if (tmp < 0) { \
 543         dest = src1 >> -tmp; \
 544     } else { \
 545         dest = src1 << tmp; \
 546         if ((dest >> tmp) != src1) { \
 547             SET_QC(); \
 548             dest = ~0; \
 549         } \
 550     }} while (0)
 551 NEON_VOP_ENV(qshl_u8, neon_u8, 4)
 552 NEON_VOP_ENV(qshl_u16, neon_u16, 2)
 553 NEON_VOP_ENV(qshl_u32, neon_u32, 1)
 554 #undef NEON_FN
 555
 556 uint64_t HELPER(neon_qshl_u64)(CPUState *env, uint64_t val, uint64_t shiftop)
 557 {
 558     int8_t shift = (int8_t)shiftop;
 559     if (shift >= 64) {
 560         if (val) {
 561             val = ~(uint64_t)0;
 562             SET_QC();
 563         }
 564     } else if (shift <= -64) {
 565         val = 0;
 566     } else if (shift < 0) {
 567         val >>= -shift;
 568     } else {
 569         uint64_t tmp = val;
 570         val <<= shift;
 571         if ((val >> shift) != tmp) {
 572             SET_QC();
 573             val = ~(uint64_t)0;
 574         }
 575     }
 576     return val;
 577 }
 578
 579 #define NEON_FN(dest, src1, src2) do { \
 580     int8_t tmp; \
 581     tmp = (int8_t)src2; \
 582     if (tmp >= (ssize_t)sizeof(src1) * 8) { \
 583         if (src1) { \
 584             SET_QC(); \
 585             dest = (uint32_t)(1 << (sizeof(src1) * 8 - 1)); \
 586             if (src1 > 0) { \
 587                 dest--; \
 588             } \
 589         } else { \
 590             dest = src1; \
 591         } \
 592     } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
 593         dest = src1 >> 31; \
 594     } else if (tmp < 0) { \
 595         dest = src1 >> -tmp; \
 596     } else { \
 597         dest = src1 << tmp; \
 598         if ((dest >> tmp) != src1) { \
 599             SET_QC(); \
 600             dest = (uint32_t)(1 << (sizeof(src1) * 8 - 1)); \
 601             if (src1 > 0) { \
 602                 dest--; \
 603             } \
 604         } \
 605     }} while (0)
 606 NEON_VOP_ENV(qshl_s8, neon_s8, 4)
 607 NEON_VOP_ENV(qshl_s16, neon_s16, 2)
 608 NEON_VOP_ENV(qshl_s32, neon_s32, 1)
 609 #undef NEON_FN
 610
 611 uint64_t HELPER(neon_qshl_s64)(CPUState *env, uint64_t valop, uint64_t shiftop)
 612 {
 613     int8_t shift = (uint8_t)shiftop;
 614     int64_t val = valop;
 615     if (shift >= 64) {
 616         if (val) {
 617             SET_QC();
 618             val = (val >> 63) ^ ~SIGNBIT64;
 619         }
 620     } else if (shift <= -64) {
 621         val >>= 63;
 622     } else if (shift < 0) {
 623         val >>= -shift;
 624     } else {
 625         int64_t tmp = val;
 626         val <<= shift;
 627         if ((val >> shift) != tmp) {
 628             SET_QC();
 629             val = (tmp >> 63) ^ ~SIGNBIT64;
 630         }
 631     }
 632     return val;
 633 }
 634
 635 #define NEON_FN(dest, src1, src2) do { \
 636     if (src1 & (1 << (sizeof(src1) * 8 - 1))) { \
 637         SET_QC(); \
 638         dest = 0; \
 639     } else { \
 640         int8_t tmp; \
 641         tmp = (int8_t)src2; \
 642         if (tmp >= (ssize_t)sizeof(src1) * 8) { \
 643             if (src1) { \
 644                 SET_QC(); \
 645                 dest = ~0; \
 646             } else { \
 647                 dest = 0; \
 648             } \
 649         } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
 650             dest = 0; \
 651         } else if (tmp < 0) { \
 652             dest = src1 >> -tmp; \
 653         } else { \
 654             dest = src1 << tmp; \
 655             if ((dest >> tmp) != src1) { \
 656                 SET_QC(); \
 657                 dest = ~0; \
 658             } \
 659         } \
 660     }} while (0)
 661 NEON_VOP_ENV(qshlu_s8, neon_u8, 4)
 662 NEON_VOP_ENV(qshlu_s16, neon_u16, 2)
 663 #undef NEON_FN
 664
 665 uint32_t HELPER(neon_qshlu_s32)(CPUState *env, uint32_t valop, uint32_t shiftop)
 666 {
 667     if ((int32_t)valop < 0) {
 668         SET_QC();
 669         return 0;
 670     }
 671     return helper_neon_qshl_u32(env, valop, shiftop);
 672 }
 673
 674 uint64_t HELPER(neon_qshlu_s64)(CPUState *env, uint64_t valop, uint64_t shiftop)
 675 {
 676     if ((int64_t)valop < 0) {
 677         SET_QC();
 678         return 0;
 679     }
 680     return helper_neon_qshl_u64(env, valop, shiftop);
 681 }
 682
 683 /* FIXME: This is wrong.  */
 684 #define NEON_FN(dest, src1, src2) do { \
 685     int8_t tmp; \
 686     tmp = (int8_t)src2; \
 687     if (tmp < 0) { \
 688         dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
 689     } else { \
 690         dest = src1 << tmp; \
 691         if ((dest >> tmp) != src1) { \
 692             SET_QC(); \
 693             dest = ~0; \
 694         } \
 695     }} while (0)
 696 NEON_VOP_ENV(qrshl_u8, neon_u8, 4)
 697 NEON_VOP_ENV(qrshl_u16, neon_u16, 2)
 698 NEON_VOP_ENV(qrshl_u32, neon_u32, 1)
 699 #undef NEON_FN
 700
 701 uint64_t HELPER(neon_qrshl_u64)(CPUState *env, uint64_t val, uint64_t shiftop)
 702 {
 703     int8_t shift = (int8_t)shiftop;
 704     if (shift < 0) {
 705         val = (val + (1 << (-1 - shift))) >> -shift;
 706     } else { \
 707         uint64_t tmp = val;
 708         val <<= shift;
 709         if ((val >> shift) != tmp) {
 710             SET_QC();
 711             val = ~0;
 712         }
 713     }
 714     return val;
 715 }
 716
 717 #define NEON_FN(dest, src1, src2) do { \
 718     int8_t tmp; \
 719     tmp = (int8_t)src2; \
 720     if (tmp < 0) { \
 721         dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
 722     } else { \
 723         dest = src1 << tmp; \
 724         if ((dest >> tmp) != src1) { \
 725             SET_QC(); \
 726             dest = src1 >> 31; \
 727         } \
 728     }} while (0)
 729 NEON_VOP_ENV(qrshl_s8, neon_s8, 4)
 730 NEON_VOP_ENV(qrshl_s16, neon_s16, 2)
 731 NEON_VOP_ENV(qrshl_s32, neon_s32, 1)
 732 #undef NEON_FN
 733
 734 uint64_t HELPER(neon_qrshl_s64)(CPUState *env, uint64_t valop, uint64_t shiftop)
 735 {
 736     int8_t shift = (uint8_t)shiftop;
 737     int64_t val = valop;
 738
 739     if (shift < 0) {
 740         val = (val + (1 << (-1 - shift))) >> -shift;
 741     } else {
 742         int64_t tmp = val;;
 743         val <<= shift;
 744         if ((val >> shift) != tmp) {
 745             SET_QC();
 746             val = tmp >> 31;
 747         }
 748     }
 749     return val;
 750 }
 751
 752 uint32_t HELPER(neon_add_u8)(uint32_t a, uint32_t b)
 753 {
 754     uint32_t mask;
 755     mask = (a ^ b) & 0x80808080u;
 756     a &= ~0x80808080u;
 757     b &= ~0x80808080u;
 758     return (a + b) ^ mask;
 759 }
 760
 761 uint32_t HELPER(neon_add_u16)(uint32_t a, uint32_t b)
 762 {
 763     uint32_t mask;
 764     mask = (a ^ b) & 0x80008000u;
 765     a &= ~0x80008000u;
 766     b &= ~0x80008000u;
 767     return (a + b) ^ mask;
 768 }
 769
 770 #define NEON_FN(dest, src1, src2) dest = src1 + src2
 771 NEON_POP(padd_u8, neon_u8, 4)
 772 NEON_POP(padd_u16, neon_u16, 2)
 773 #undef NEON_FN
 774
 775 #define NEON_FN(dest, src1, src2) dest = src1 - src2
 776 NEON_VOP(sub_u8, neon_u8, 4)
 777 NEON_VOP(sub_u16, neon_u16, 2)
 778 #undef NEON_FN
 779
 780 #define NEON_FN(dest, src1, src2) dest = src1 * src2
 781 NEON_VOP(mul_u8, neon_u8, 4)
 782 NEON_VOP(mul_u16, neon_u16, 2)
 783 #undef NEON_FN
 784
 785 /* Polynomial multiplication is like integer multiplication except the
 786    partial products are XORed, not added.  */
 787 uint32_t HELPER(neon_mul_p8)(uint32_t op1, uint32_t op2)
 788 {
 789     uint32_t mask;
 790     uint32_t result;
 791     result = 0;
 792     while (op1) {
 793         mask = 0;
 794         if (op1 & 1)
 795             mask |= 0xff;
 796         if (op1 & (1 << 8))
 797             mask |= (0xff << 8);
 798         if (op1 & (1 << 16))
 799             mask |= (0xff << 16);
 800         if (op1 & (1 << 24))
 801             mask |= (0xff << 24);
 802         result ^= op2 & mask;
 803         op1 = (op1 >> 1) & 0x7f7f7f7f;
 804         op2 = (op2 << 1) & 0xfefefefe;
 805     }
 806     return result;
 807 }
 808
 809 #define NEON_FN(dest, src1, src2) dest = (src1 & src2) ? -1 : 0
 810 NEON_VOP(tst_u8, neon_u8, 4)
 811 NEON_VOP(tst_u16, neon_u16, 2)
 812 NEON_VOP(tst_u32, neon_u32, 1)
 813 #undef NEON_FN
 814
 815 #define NEON_FN(dest, src1, src2) dest = (src1 == src2) ? -1 : 0
 816 NEON_VOP(ceq_u8, neon_u8, 4)
 817 NEON_VOP(ceq_u16, neon_u16, 2)
 818 NEON_VOP(ceq_u32, neon_u32, 1)
 819 #undef NEON_FN
 820
 821 #define NEON_FN(dest, src, dummy) dest = (src < 0) ? -src : src
 822 NEON_VOP1(abs_s8, neon_s8, 4)
 823 NEON_VOP1(abs_s16, neon_s16, 2)
 824 #undef NEON_FN
 825
 826 /* Count Leading Sign/Zero Bits.  */
 827 static inline int do_clz8(uint8_t x)
 828 {
 829     int n;
 830     for (n = 8; x; n--)
 831         x >>= 1;
 832     return n;
 833 }
 834
 835 static inline int do_clz16(uint16_t x)
 836 {
 837     int n;
 838     for (n = 16; x; n--)
 839         x >>= 1;
 840     return n;
 841 }
 842
 843 #define NEON_FN(dest, src, dummy) dest = do_clz8(src)
 844 NEON_VOP1(clz_u8, neon_u8, 4)
 845 #undef NEON_FN
 846
 847 #define NEON_FN(dest, src, dummy) dest = do_clz16(src)
 848 NEON_VOP1(clz_u16, neon_u16, 2)
 849 #undef NEON_FN
 850
 851 #define NEON_FN(dest, src, dummy) dest = do_clz8((src < 0) ? ~src : src) - 1
 852 NEON_VOP1(cls_s8, neon_s8, 4)
 853 #undef NEON_FN
 854
 855 #define NEON_FN(dest, src, dummy) dest = do_clz16((src < 0) ? ~src : src) - 1
 856 NEON_VOP1(cls_s16, neon_s16, 2)
 857 #undef NEON_FN
 858
 859 uint32_t HELPER(neon_cls_s32)(uint32_t x)
 860 {
 861     int count;
 862     if ((int32_t)x < 0)
 863         x = ~x;
 864     for (count = 32; x; count--)
 865         x = x >> 1;
 866     return count - 1;
 867 }
 868
 869 /* Bit count.  */
 870 uint32_t HELPER(neon_cnt_u8)(uint32_t x)
 871 {
 872     x = (x & 0x55555555) + ((x >>  1) & 0x55555555);
 873     x = (x & 0x33333333) + ((x >>  2) & 0x33333333);
 874     x = (x & 0x0f0f0f0f) + ((x >>  4) & 0x0f0f0f0f);
 875     return x;
 876 }
 877
 878 #define NEON_QDMULH16(dest, src1, src2, round) do { \
 879     uint32_t tmp = (int32_t)(int16_t) src1 * (int16_t) src2; \
 880     if ((tmp ^ (tmp << 1)) & SIGNBIT) { \
 881         SET_QC(); \
 882         tmp = (tmp >> 31) ^ ~SIGNBIT; \
 883     } \
 884     tmp <<= 1; \
 885     if (round) { \
 886         int32_t old = tmp; \
 887         tmp += 1 << 15; \
 888         if ((int32_t)tmp < old) { \
 889             SET_QC(); \
 890             tmp = SIGNBIT - 1; \
 891         } \
 892     } \
 893     dest = tmp >> 16; \
 894     } while(0)
 895 #define NEON_FN(dest, src1, src2) NEON_QDMULH16(dest, src1, src2, 0)
 896 NEON_VOP_ENV(qdmulh_s16, neon_s16, 2)
 897 #undef NEON_FN
 898 #define NEON_FN(dest, src1, src2) NEON_QDMULH16(dest, src1, src2, 1)
 899 NEON_VOP_ENV(qrdmulh_s16, neon_s16, 2)
 900 #undef NEON_FN
 901 #undef NEON_QDMULH16
 902
 903 #define NEON_QDMULH32(dest, src1, src2, round) do { \
 904     uint64_t tmp = (int64_t)(int32_t) src1 * (int32_t) src2; \
 905     if ((tmp ^ (tmp << 1)) & SIGNBIT64) { \
 906         SET_QC(); \
 907         tmp = (tmp >> 63) ^ ~SIGNBIT64; \
 908     } else { \
 909         tmp <<= 1; \
 910     } \
 911     if (round) { \
 912         int64_t old = tmp; \
 913         tmp += (int64_t)1 << 31; \
 914         if ((int64_t)tmp < old) { \
 915             SET_QC(); \
 916             tmp = SIGNBIT64 - 1; \
 917         } \
 918     } \
 919     dest = tmp >> 32; \
 920     } while(0)
 921 #define NEON_FN(dest, src1, src2) NEON_QDMULH32(dest, src1, src2, 0)
 922 NEON_VOP_ENV(qdmulh_s32, neon_s32, 1)
 923 #undef NEON_FN
 924 #define NEON_FN(dest, src1, src2) NEON_QDMULH32(dest, src1, src2, 1)
 925 NEON_VOP_ENV(qrdmulh_s32, neon_s32, 1)
 926 #undef NEON_FN
 927 #undef NEON_QDMULH32
 928
 929 uint32_t HELPER(neon_narrow_u8)(uint64_t x)
 930 {
 931     return (x & 0xffu) | ((x >> 8) & 0xff00u) | ((x >> 16) & 0xff0000u)
 932            | ((x >> 24) & 0xff000000u);
 933 }
 934
 935 uint32_t HELPER(neon_narrow_u16)(uint64_t x)
 936 {
 937     return (x & 0xffffu) | ((x >> 16) & 0xffff0000u);
 938 }
 939
 940 uint32_t HELPER(neon_narrow_high_u8)(uint64_t x)
 941 {
 942     return ((x >> 8) & 0xff) | ((x >> 16) & 0xff00)
 943             | ((x >> 24) & 0xff0000) | ((x >> 32) & 0xff000000);
 944 }
 945
 946 uint32_t HELPER(neon_narrow_high_u16)(uint64_t x)
 947 {
 948     return ((x >> 16) & 0xffff) | ((x >> 32) & 0xffff0000);
 949 }
 950
 951 uint32_t HELPER(neon_narrow_round_high_u8)(uint64_t x)
 952 {
 953     x &= 0xff80ff80ff80ff80ull;
 954     x += 0x0080008000800080ull;
 955     return ((x >> 8) & 0xff) | ((x >> 16) & 0xff00)
 956             | ((x >> 24) & 0xff0000) | ((x >> 32) & 0xff000000);
 957 }
 958
 959 uint32_t HELPER(neon_narrow_round_high_u16)(uint64_t x)
 960 {
 961     x &= 0xffff8000ffff8000ull;
 962     x += 0x0000800000008000ull;
 963     return ((x >> 16) & 0xffff) | ((x >> 32) & 0xffff0000);
 964 }
 965
 966 uint32_t HELPER(neon_narrow_sat_u8)(CPUState *env, uint64_t x)
 967 {
 968     uint16_t s;
 969     uint8_t d;
 970     uint32_t res = 0;
 971 #define SAT8(n) \
 972     s = x >> n; \
 973     if (s > 0xff) { \
 974         d = 0xff; \
 975         SET_QC(); \
 976     } else  { \
 977         d = s; \
 978     } \
 979     res |= (uint32_t)d << (n / 2);
 980
 981     SAT8(0);
 982     SAT8(16);
 983     SAT8(32);
 984     SAT8(48);
 985 #undef SAT8
 986     return res;
 987 }
 988
 989 uint32_t HELPER(neon_narrow_sat_s8)(CPUState *env, uint64_t x)
 990 {
 991     int16_t s;
 992     uint8_t d;
 993     uint32_t res = 0;
 994 #define SAT8(n) \
 995     s = x >> n; \
 996     if (s != (int8_t)s) { \
 997         d = (s >> 15) ^ 0x7f; \
 998         SET_QC(); \
 999     } else  { \
1000         d = s; \
1001     } \
1002     res |= (uint32_t)d << (n / 2);
1003
1004     SAT8(0);
1005     SAT8(16);
1006     SAT8(32);
1007     SAT8(48);
1008 #undef SAT8
1009     return res;
1010 }
1011
1012 uint32_t HELPER(neon_narrow_sat_u16)(CPUState *env, uint64_t x)
1013 {
1014     uint32_t high;
1015     uint32_t low;
1016     low = x;
1017     if (low > 0xffff) {
1018         low = 0xffff;
1019         SET_QC();
1020     }
1021     high = x >> 32;
1022     if (high > 0xffff) {
1023         high = 0xffff;
1024         SET_QC();
1025     }
1026     return low | (high << 16);
1027 }
1028
1029 uint32_t HELPER(neon_narrow_sat_s16)(CPUState *env, uint64_t x)
1030 {
1031     int32_t low;
1032     int32_t high;
1033     low = x;
1034     if (low != (int16_t)low) {
1035         low = (low >> 31) ^ 0x7fff;
1036         SET_QC();
1037     }
1038     high = x >> 32;
1039     if (high != (int16_t)high) {
1040         high = (high >> 31) ^ 0x7fff;
1041         SET_QC();
1042     }
1043     return (uint16_t)low | (high << 16);
1044 }
1045
1046 uint32_t HELPER(neon_narrow_sat_u32)(CPUState *env, uint64_t x)
1047 {
1048     if (x > 0xffffffffu) {
1049         SET_QC();
1050         return 0xffffffffu;
1051     }
1052     return x;
1053 }
1054
1055 uint32_t HELPER(neon_narrow_sat_s32)(CPUState *env, uint64_t x)
1056 {
1057     if ((int64_t)x != (int32_t)x) {
1058         SET_QC();
1059         return (x >> 63) ^ 0x7fffffff;
1060     }
1061     return x;
1062 }
1063
1064 uint64_t HELPER(neon_widen_u8)(uint32_t x)
1065 {
1066     uint64_t tmp;
1067     uint64_t ret;
1068     ret = (uint8_t)x;
1069     tmp = (uint8_t)(x >> 8);
1070     ret |= tmp << 16;
1071     tmp = (uint8_t)(x >> 16);
1072     ret |= tmp << 32;
1073     tmp = (uint8_t)(x >> 24);
1074     ret |= tmp << 48;
1075     return ret;
1076 }
1077
1078 uint64_t HELPER(neon_widen_s8)(uint32_t x)
1079 {
1080     uint64_t tmp;
1081     uint64_t ret;
1082     ret = (uint16_t)(int8_t)x;
1083     tmp = (uint16_t)(int8_t)(x >> 8);
1084     ret |= tmp << 16;
1085     tmp = (uint16_t)(int8_t)(x >> 16);
1086     ret |= tmp << 32;
1087     tmp = (uint16_t)(int8_t)(x >> 24);
1088     ret |= tmp << 48;
1089     return ret;
1090 }
1091
1092 uint64_t HELPER(neon_widen_u16)(uint32_t x)
1093 {
1094     uint64_t high = (uint16_t)(x >> 16);
1095     return ((uint16_t)x) | (high << 32);
1096 }
1097
1098 uint64_t HELPER(neon_widen_s16)(uint32_t x)
1099 {
1100     uint64_t high = (int16_t)(x >> 16);
1101     return ((uint32_t)(int16_t)x) | (high << 32);
1102 }
1103
1104 uint64_t HELPER(neon_addl_u16)(uint64_t a, uint64_t b)
1105 {
1106     uint64_t mask;
1107     mask = (a ^ b) & 0x8000800080008000ull;
1108     a &= ~0x8000800080008000ull;
1109     b &= ~0x8000800080008000ull;
1110     return (a + b) ^ mask;
1111 }
1112
1113 uint64_t HELPER(neon_addl_u32)(uint64_t a, uint64_t b)
1114 {
1115     uint64_t mask;
1116     mask = (a ^ b) & 0x8000000080000000ull;
1117     a &= ~0x8000000080000000ull;
1118     b &= ~0x8000000080000000ull;
1119     return (a + b) ^ mask;
1120 }
1121
1122 uint64_t HELPER(neon_paddl_u16)(uint64_t a, uint64_t b)
1123 {
1124     uint64_t tmp;
1125     uint64_t tmp2;
1126
1127     tmp = a & 0x0000ffff0000ffffull;
1128     tmp += (a >> 16) & 0x0000ffff0000ffffull;
1129     tmp2 = b & 0xffff0000ffff0000ull;
1130     tmp2 += (b << 16) & 0xffff0000ffff0000ull;
1131     return    ( tmp         & 0xffff)
1132             | ((tmp  >> 16) & 0xffff0000ull)
1133             | ((tmp2 << 16) & 0xffff00000000ull)
1134             | ( tmp2        & 0xffff000000000000ull);
1135 }
1136
1137 uint64_t HELPER(neon_paddl_u32)(uint64_t a, uint64_t b)
1138 {
1139     uint32_t low = a + (a >> 32);
1140     uint32_t high = b + (b >> 32);
1141     return low + ((uint64_t)high << 32);
1142 }
1143
1144 uint64_t HELPER(neon_subl_u16)(uint64_t a, uint64_t b)
1145 {
1146     uint64_t mask;
1147     mask = (a ^ ~b) & 0x8000800080008000ull;
1148     a |= 0x8000800080008000ull;
1149     b &= ~0x8000800080008000ull;
1150     return (a - b) ^ mask;
1151 }
1152
1153 uint64_t HELPER(neon_subl_u32)(uint64_t a, uint64_t b)
1154 {
1155     uint64_t mask;
1156     mask = (a ^ ~b) & 0x8000000080000000ull;
1157     a |= 0x8000000080000000ull;
1158     b &= ~0x8000000080000000ull;
1159     return (a - b) ^ mask;
1160 }
1161
1162 uint64_t HELPER(neon_addl_saturate_s32)(CPUState *env, uint64_t a, uint64_t b)
1163 {
1164     uint32_t x, y;
1165     uint32_t low, high;
1166
1167     x = a;
1168     y = b;
1169     low = x + y;
1170     if (((low ^ x) & SIGNBIT) && !((x ^ y) & SIGNBIT)) {
1171         SET_QC();
1172         low = ((int32_t)x >> 31) ^ ~SIGNBIT;
1173     }
1174     x = a >> 32;
1175     y = b >> 32;
1176     high = x + y;
1177     if (((high ^ x) & SIGNBIT) && !((x ^ y) & SIGNBIT)) {
1178         SET_QC();
1179         high = ((int32_t)x >> 31) ^ ~SIGNBIT;
1180     }
1181     return low | ((uint64_t)high << 32);
1182 }
1183
1184 uint64_t HELPER(neon_addl_saturate_s64)(CPUState *env, uint64_t a, uint64_t b)
1185 {
1186     uint64_t result;
1187
1188     result = a + b;
1189     if (((result ^ a) & SIGNBIT64) && !((a ^ b) & SIGNBIT64)) {
1190         SET_QC();
1191         result = ((int64_t)a >> 63) ^ ~SIGNBIT64;
1192     }
1193     return result;
1194 }
1195
1196 #define DO_ABD(dest, x, y, type) do { \
1197     type tmp_x = x; \
1198     type tmp_y = y; \
1199     dest = ((tmp_x > tmp_y) ? tmp_x - tmp_y : tmp_y - tmp_x); \
1200     } while(0)
1201
1202 uint64_t HELPER(neon_abdl_u16)(uint32_t a, uint32_t b)
1203 {
1204     uint64_t tmp;
1205     uint64_t result;
1206     DO_ABD(result, a, b, uint8_t);
1207     DO_ABD(tmp, a >> 8, b >> 8, uint8_t);
1208     result |= tmp << 16;
1209     DO_ABD(tmp, a >> 16, b >> 16, uint8_t);
1210     result |= tmp << 32;
1211     DO_ABD(tmp, a >> 24, b >> 24, uint8_t);
1212     result |= tmp << 48;
1213     return result;
1214 }
1215
1216 uint64_t HELPER(neon_abdl_s16)(uint32_t a, uint32_t b)
1217 {
1218     uint64_t tmp;
1219     uint64_t result;
1220     DO_ABD(result, a, b, int8_t);
1221     DO_ABD(tmp, a >> 8, b >> 8, int8_t);
1222     result |= tmp << 16;
1223     DO_ABD(tmp, a >> 16, b >> 16, int8_t);
1224     result |= tmp << 32;
1225     DO_ABD(tmp, a >> 24, b >> 24, int8_t);
1226     result |= tmp << 48;
1227     return result;
1228 }
1229
1230 uint64_t HELPER(neon_abdl_u32)(uint32_t a, uint32_t b)
1231 {
1232     uint64_t tmp;
1233     uint64_t result;
1234     DO_ABD(result, a, b, uint16_t);
1235     DO_ABD(tmp, a >> 16, b >> 16, uint16_t);
1236     return result | (tmp << 32);
1237 }
1238
1239 uint64_t HELPER(neon_abdl_s32)(uint32_t a, uint32_t b)
1240 {
1241     uint64_t tmp;
1242     uint64_t result;
1243     DO_ABD(result, a, b, int16_t);
1244     DO_ABD(tmp, a >> 16, b >> 16, int16_t);
1245     return result | (tmp << 32);
1246 }
1247
1248 uint64_t HELPER(neon_abdl_u64)(uint32_t a, uint32_t b)
1249 {
1250     uint64_t result;
1251     DO_ABD(result, a, b, uint32_t);
1252     return result;
1253 }
1254
1255 uint64_t HELPER(neon_abdl_s64)(uint32_t a, uint32_t b)
1256 {
1257     uint64_t result;
1258     DO_ABD(result, a, b, int32_t);
1259     return result;
1260 }
1261 #undef DO_ABD
1262
1263 /* Widening multiply. Named type is the source type.  */
1264 #define DO_MULL(dest, x, y, type1, type2) do { \
1265     type1 tmp_x = x; \
1266     type1 tmp_y = y; \
1267     dest = (type2)((type2)tmp_x * (type2)tmp_y); \
1268     } while(0)
1269
1270 uint64_t HELPER(neon_mull_u8)(uint32_t a, uint32_t b)
1271 {
1272     uint64_t tmp;
1273     uint64_t result;
1274
1275     DO_MULL(result, a, b, uint8_t, uint16_t);
1276     DO_MULL(tmp, a >> 8, b >> 8, uint8_t, uint16_t);
1277     result |= tmp << 16;
1278     DO_MULL(tmp, a >> 16, b >> 16, uint8_t, uint16_t);
1279     result |= tmp << 32;
1280     DO_MULL(tmp, a >> 24, b >> 24, uint8_t, uint16_t);
1281     result |= tmp << 48;
1282     return result;
1283 }
1284
1285 uint64_t HELPER(neon_mull_s8)(uint32_t a, uint32_t b)
1286 {
1287     uint64_t tmp;
1288     uint64_t result;
1289
1290     DO_MULL(result, a, b, int8_t, uint16_t);
1291     DO_MULL(tmp, a >> 8, b >> 8, int8_t, uint16_t);
1292     result |= tmp << 16;
1293     DO_MULL(tmp, a >> 16, b >> 16, int8_t, uint16_t);
1294     result |= tmp << 32;
1295     DO_MULL(tmp, a >> 24, b >> 24, int8_t, uint16_t);
1296     result |= tmp << 48;
1297     return result;
1298 }
1299
1300 uint64_t HELPER(neon_mull_u16)(uint32_t a, uint32_t b)
1301 {
1302     uint64_t tmp;
1303     uint64_t result;
1304
1305     DO_MULL(result, a, b, uint16_t, uint32_t);
1306     DO_MULL(tmp, a >> 16, b >> 16, uint16_t, uint32_t);
1307     return result | (tmp << 32);
1308 }
1309
1310 uint64_t HELPER(neon_mull_s16)(uint32_t a, uint32_t b)
1311 {
1312     uint64_t tmp;
1313     uint64_t result;
1314
1315     DO_MULL(result, a, b, int16_t, uint32_t);
1316     DO_MULL(tmp, a >> 16, b >> 16, int16_t, uint32_t);
1317     return result | (tmp << 32);
1318 }
1319
1320 uint64_t HELPER(neon_negl_u16)(uint64_t x)
1321 {
1322     uint16_t tmp;
1323     uint64_t result;
1324     result = (uint16_t)-x;
1325     tmp = -(x >> 16);
1326     result |= (uint64_t)tmp << 16;
1327     tmp = -(x >> 32);
1328     result |= (uint64_t)tmp << 32;
1329     tmp = -(x >> 48);
1330     result |= (uint64_t)tmp << 48;
1331     return result;
1332 }
1333
1334 #include <stdio.h>
1335 uint64_t HELPER(neon_negl_u32)(uint64_t x)
1336 {
1337     uint32_t low = -x;
1338     uint32_t high = -(x >> 32);
1339     return low | ((uint64_t)high << 32);
1340 }
1341
1342 /* FIXME:  There should be a native op for this.  */
1343 uint64_t HELPER(neon_negl_u64)(uint64_t x)
1344 {
1345     return -x;
1346 }
1347
1348 /* Saturnating sign manuipulation.  */
1349 /* ??? Make these use NEON_VOP1 */
1350 #define DO_QABS8(x) do { \
1351     if (x == (int8_t)0x80) { \
1352         x = 0x7f; \
1353         SET_QC(); \
1354     } else if (x < 0) { \
1355         x = -x; \
1356     }} while (0)
1357 uint32_t HELPER(neon_qabs_s8)(CPUState *env, uint32_t x)
1358 {
1359     neon_s8 vec;
1360     NEON_UNPACK(neon_s8, vec, x);
1361     DO_QABS8(vec.v1);
1362     DO_QABS8(vec.v2);
1363     DO_QABS8(vec.v3);
1364     DO_QABS8(vec.v4);
1365     NEON_PACK(neon_s8, x, vec);
1366     return x;
1367 }
1368 #undef DO_QABS8
1369
1370 #define DO_QNEG8(x) do { \
1371     if (x == (int8_t)0x80) { \
1372         x = 0x7f; \
1373         SET_QC(); \
1374     } else { \
1375         x = -x; \
1376     }} while (0)
1377 uint32_t HELPER(neon_qneg_s8)(CPUState *env, uint32_t x)
1378 {
1379     neon_s8 vec;
1380     NEON_UNPACK(neon_s8, vec, x);
1381     DO_QNEG8(vec.v1);
1382     DO_QNEG8(vec.v2);
1383     DO_QNEG8(vec.v3);
1384     DO_QNEG8(vec.v4);
1385     NEON_PACK(neon_s8, x, vec);
1386     return x;
1387 }
1388 #undef DO_QNEG8
1389
1390 #define DO_QABS16(x) do { \
1391     if (x == (int16_t)0x8000) { \
1392         x = 0x7fff; \
1393         SET_QC(); \
1394     } else if (x < 0) { \
1395         x = -x; \
1396     }} while (0)
1397 uint32_t HELPER(neon_qabs_s16)(CPUState *env, uint32_t x)
1398 {
1399     neon_s16 vec;
1400     NEON_UNPACK(neon_s16, vec, x);
1401     DO_QABS16(vec.v1);
1402     DO_QABS16(vec.v2);
1403     NEON_PACK(neon_s16, x, vec);
1404     return x;
1405 }
1406 #undef DO_QABS16
1407
1408 #define DO_QNEG16(x) do { \
1409     if (x == (int16_t)0x8000) { \
1410         x = 0x7fff; \
1411         SET_QC(); \
1412     } else { \
1413         x = -x; \
1414     }} while (0)
1415 uint32_t HELPER(neon_qneg_s16)(CPUState *env, uint32_t x)
1416 {
1417     neon_s16 vec;
1418     NEON_UNPACK(neon_s16, vec, x);
1419     DO_QNEG16(vec.v1);
1420     DO_QNEG16(vec.v2);
1421     NEON_PACK(neon_s16, x, vec);
1422     return x;
1423 }
1424 #undef DO_QNEG16
1425
1426 uint32_t HELPER(neon_qabs_s32)(CPUState *env, uint32_t x)
1427 {
1428     if (x == SIGNBIT) {
1429         SET_QC();
1430         x = ~SIGNBIT;
1431     } else if ((int32_t)x < 0) {
1432         x = -x;
1433     }
1434     return x;
1435 }
1436
1437 uint32_t HELPER(neon_qneg_s32)(CPUState *env, uint32_t x)
1438 {
1439     if (x == SIGNBIT) {
1440         SET_QC();
1441         x = ~SIGNBIT;
1442     } else {
1443         x = -x;
1444     }
1445     return x;
1446 }
1447
1448 /* NEON Float helpers.  */
1449 uint32_t HELPER(neon_min_f32)(uint32_t a, uint32_t b)
1450 {
1451     float32 f0 = vfp_itos(a);
1452     float32 f1 = vfp_itos(b);
1453     return (float32_compare_quiet(f0, f1, NFS) == -1) ? a : b;
1454 }
1455
1456 uint32_t HELPER(neon_max_f32)(uint32_t a, uint32_t b)
1457 {
1458     float32 f0 = vfp_itos(a);
1459     float32 f1 = vfp_itos(b);
1460     return (float32_compare_quiet(f0, f1, NFS) == 1) ? a : b;
1461 }
1462
1463 uint32_t HELPER(neon_abd_f32)(uint32_t a, uint32_t b)
1464 {
1465     float32 f0 = vfp_itos(a);
1466     float32 f1 = vfp_itos(b);
1467     return vfp_stoi((float32_compare_quiet(f0, f1, NFS) == 1)
1468                     ? float32_sub(f0, f1, NFS)
1469                     : float32_sub(f1, f0, NFS));
1470 }
1471
1472 uint32_t HELPER(neon_add_f32)(uint32_t a, uint32_t b)
1473 {
1474     return vfp_stoi(float32_add(vfp_itos(a), vfp_itos(b), NFS));
1475 }
1476
1477 uint32_t HELPER(neon_sub_f32)(uint32_t a, uint32_t b)
1478 {
1479     return vfp_stoi(float32_sub(vfp_itos(a), vfp_itos(b), NFS));
1480 }
1481
1482 uint32_t HELPER(neon_mul_f32)(uint32_t a, uint32_t b)
1483 {
1484     return vfp_stoi(float32_mul(vfp_itos(a), vfp_itos(b), NFS));
1485 }
1486
1487 /* Floating point comparisons produce an integer result.  */
1488 #define NEON_VOP_FCMP(name, cmp) \
1489 uint32_t HELPER(neon_##name)(uint32_t a, uint32_t b) \
1490 { \
1491     if (float32_compare_quiet(vfp_itos(a), vfp_itos(b), NFS) cmp 0) \
1492         return ~0; \
1493     else \
1494         return 0; \
1495 }
1496
1497 NEON_VOP_FCMP(ceq_f32, ==)
1498 NEON_VOP_FCMP(cge_f32, >=)
1499 NEON_VOP_FCMP(cgt_f32, >)
1500
1501 uint32_t HELPER(neon_acge_f32)(uint32_t a, uint32_t b)
1502 {
1503     float32 f0 = float32_abs(vfp_itos(a));
1504     float32 f1 = float32_abs(vfp_itos(b));
1505     return (float32_compare_quiet(f0, f1,NFS) >= 0) ? ~0 : 0;
1506 }
1507
1508 uint32_t HELPER(neon_acgt_f32)(uint32_t a, uint32_t b)
1509 {
1510     float32 f0 = float32_abs(vfp_itos(a));
1511     float32 f1 = float32_abs(vfp_itos(b));
1512     return (float32_compare_quiet(f0, f1, NFS) > 0) ? ~0 : 0;
1513 }