gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/arm-neon-ref.h

   1 /* This file defines helper operations shared by all the tests.  */
   2
   3 #ifndef _ARM_NEON_REF_H_
   4 #define _ARM_NEON_REF_H_
   5
   6 #include <stdio.h>
   7 #include <inttypes.h>
   8
   9 /* helper type, to help write floating point results in integer form.  */
  10 typedef uint16_t hfloat16_t;
  11 typedef uint32_t hfloat32_t;
  12 typedef uint64_t hfloat64_t;
  13
  14 typedef uint16_t hbfloat16_t;
  15
  16 extern void abort(void);
  17 extern void *memset(void *, int, size_t);
  18 extern void *memcpy(void *, const void *, size_t);
  19 extern size_t strlen(const char *);
  20
  21 /* Helper macro to select FP16 tests.  */
  22 #if (defined (__ARM_FP16_FORMAT_IEEE) \
  23      || defined (__ARM_FP16_FORMAT_ALTERNATIVE))
  24 #define FP16_SUPPORTED (1)
  25 #else
  26 #undef FP16_SUPPORTED
  27 #endif
  28
  29 /* Various string construction helpers.  */
  30
  31 /*
  32   The most useful at user-level are VECT_VAR and VECT_VAR_DECL, which
  33    construct variable names or declarations, such as:
  34    VECT_VAR(expected, int, 16, 4) -> expected_int16x4
  35    VECT_VAR_DECL(expected, int, 16, 4) -> int16x4_t expected_int16x4
  36 */
  37 /* Some instructions don't exist on ARM.
  38    Use this macro to guard against them.  */
  39 #ifdef __aarch64__
  40 #define AARCH64_ONLY(X) X
  41 #else
  42 #define AARCH64_ONLY(X)
  43 #endif
  44
  45 #define xSTR(X) #X
  46 #define STR(X) xSTR(X)
  47
  48 #define xNAME1(V,T) V ## _ ##  T
  49 #define xNAME(V,T) xNAME1(V,T)
  50
  51 /* VAR(foo,int,16) -> foo_int16 */
  52 #define VAR(V,T,W) xNAME(V,T##W)
  53 /* VAR_DECL(foo,int,16) -> int16_t foo_int16 */
  54 #define VAR_DECL(V, T, W) T##W##_t VAR(V,T,W)
  55
  56 /* VECT_NAME(int,16,4) ->  int16x4 */
  57 #define VECT_NAME(T, W, N) T##W##x##N
  58 /* VECT_ARRAY_NAME(int,16,4,2) -> int16x4x2 */
  59 #define VECT_ARRAY_NAME(T, W, N, L) T##W##x##N##x##L
  60 /* VECT_TYPE(int,16,4) -> int16x4_t */
  61 #define VECT_TYPE(T, W, N) xNAME(VECT_NAME(T,W,N),t)
  62 /* VECT_ARRAY_TYPE(int,16,4,2) -> int16x4x2_t */
  63 #define VECT_ARRAY_TYPE(T, W, N, L) xNAME(VECT_ARRAY_NAME(T,W,N,L),t)
  64
  65 /* VECT_VAR(foo,int,16,4) -> foo_int16x4 */
  66 #define VECT_VAR(V,T,W,N) xNAME(V,VECT_NAME(T,W,N))
  67 /* VECT_VAR_DECL(foo,int,16,4) -> int16_t foo_int16x4 */
  68 #define VECT_VAR_DECL(V, T, W, N) T##W##_t VECT_VAR(V,T,W,N)
  69
  70 /* Array declarations.  */
  71 /* ARRAY(foo,int,16,4) -> int16_t foo_int16x4[4] */
  72 #define ARRAY(V, T, W, N) VECT_VAR_DECL(V,T,W,N)[N]
  73
  74 /* Arrays of vectors.  */
  75 /* VECT_ARRAY_VAR(foo,int,16,4,2) -> foo_int16x4x2 */
  76 #define VECT_ARRAY_VAR(V,T,W,N,L) xNAME(V,VECT_ARRAY_NAME(T,W,N,L))
  77 /* VECT_ARRAY(foo,int,16,4,2) -> int16_t foo_int16x4x2[4*2] */
  78 #define VECT_ARRAY(V, T, W, N, L) T##W##_t VECT_ARRAY_VAR(V,T,W,N,L)[N*L]
  79
  80 /* Check results vs expected values. Operates on one vector.  */
  81 #define CHECK(MSG,T,W,N,FMT,EXPECTED,COMMENT)                           \
  82   {                                                                     \
  83     int i;                                                              \
  84     for(i=0; i<N ; i++)                                                 \
  85       {                                                                 \
  86         if (VECT_VAR(result, T, W, N)[i] !=                             \
  87             VECT_VAR(EXPECTED, T, W, N)[i]) {                           \
  88           fprintf(stderr,                                               \
  89                   "ERROR in %s (%s line %d in buffer '%s') at type %s " \
  90                   "index %d: got 0x%" FMT " != 0x%" FMT " %s\n",        \
  91                   MSG, __FILE__, __LINE__,                              \
  92                   STR(EXPECTED),                                        \
  93                   STR(VECT_NAME(T, W, N)),                              \
  94                   i,                                                    \
  95                   VECT_VAR(result, T, W, N)[i],                         \
  96                   VECT_VAR(EXPECTED, T, W, N)[i],                       \
  97                   strlen(COMMENT) > 0 ? COMMENT : "");                  \
  98           abort();                                                      \
  99         }                                                               \
 100       }                                                                 \
 101     fprintf(stderr, "CHECKED %s %s\n", STR(VECT_TYPE(T, W, N)), MSG);   \
 102   }
 103
 104 /* Floating-point variant.  */
 105 #define CHECK_FP(MSG,T,W,N,FMT,EXPECTED,COMMENT)                        \
 106   {                                                                     \
 107     int i;                                                              \
 108     for(i=0; i<N ; i++)                                                 \
 109       {                                                                 \
 110         union fp_operand {                                              \
 111           uint##W##_t i;                                                \
 112           T##W##_t f;                                                   \
 113         } tmp_res, tmp_exp;                                             \
 114         tmp_res.f = VECT_VAR(result, T, W, N)[i];                       \
 115         tmp_exp.i = VECT_VAR(EXPECTED, h##T, W, N)[i];                  \
 116         if (tmp_res.i != tmp_exp.i) {                                   \
 117           fprintf(stderr,                                               \
 118                   "ERROR in %s (%s line %d in buffer '%s') at type %s " \
 119                   "index %d: got 0x%" FMT " != 0x%" FMT " %s\n",        \
 120                   MSG, __FILE__, __LINE__,                              \
 121                   STR(EXPECTED),                                        \
 122                   STR(VECT_NAME(T, W, N)),                              \
 123                   i,                                                    \
 124                   tmp_res.i,                                            \
 125                   tmp_exp.i,                                            \
 126                   strlen(COMMENT) > 0 ? COMMENT : "");                  \
 127           abort();                                                      \
 128         }                                                               \
 129       }                                                                 \
 130     fprintf(stderr, "CHECKED %s %s\n", STR(VECT_TYPE(T, W, N)), MSG);   \
 131   }
 132
 133 /* poly variant.  */
 134 #define CHECK_POLY(MSG,T,W,N,FMT,EXPECTED,COMMENT)                      \
 135   {                                                                     \
 136     int i;                                                              \
 137     for(i=0; i<N ; i++)                                                 \
 138       {                                                                 \
 139         union poly_operand {                                            \
 140           uint##W##_t i;                                                \
 141           poly##W##_t p;                                                \
 142         } tmp_res, tmp_exp;                                             \
 143         tmp_res.p = VECT_VAR(result, T, W, N)[i];                       \
 144         tmp_exp.i = VECT_VAR(EXPECTED, T, W, N)[i];                     \
 145         if (tmp_res.i != tmp_exp.i) {                                   \
 146           fprintf(stderr,                                               \
 147                   "ERROR in %s (%s line %d in buffer '%s') at type %s " \
 148                   "index %d: got 0x%" FMT " != 0x%" FMT " %s\n",        \
 149                   MSG, __FILE__, __LINE__,                              \
 150                   STR(EXPECTED),                                        \
 151                   STR(VECT_NAME(T, W, N)),                              \
 152                   i,                                                    \
 153                   tmp_res.i,                                            \
 154                   tmp_exp.i,                                            \
 155                   strlen(COMMENT) > 0 ? COMMENT : "");                  \
 156           abort();                                                      \
 157         }                                                               \
 158       }                                                                 \
 159     fprintf(stderr, "CHECKED %s %s\n", STR(VECT_TYPE(T, W, N)), MSG);   \
 160   }
 161
 162 /* Clean buffer with a non-zero pattern to help diagnose buffer
 163    overflows.  */
 164 #define CLEAN_PATTERN_8  0x33
 165
 166 #define CLEAN(VAR,T,W,N)                                                \
 167   memset(VECT_VAR(VAR, T, W, N),                                        \
 168          CLEAN_PATTERN_8,                                               \
 169          sizeof(VECT_VAR(VAR, T, W, N)));
 170
 171 /* Define output buffers, one of each size.  */
 172 static ARRAY(result, int, 8, 8);
 173 static ARRAY(result, int, 16, 4);
 174 static ARRAY(result, int, 32, 2);
 175 static ARRAY(result, int, 64, 1);
 176 static ARRAY(result, uint, 8, 8);
 177 static ARRAY(result, uint, 16, 4);
 178 static ARRAY(result, uint, 32, 2);
 179 static ARRAY(result, uint, 64, 1);
 180 static ARRAY(result, poly, 8, 8);
 181 static ARRAY(result, poly, 16, 4);
 182 #if defined (__ARM_FEATURE_CRYPTO)
 183 static ARRAY(result, poly, 64, 1);
 184 #endif
 185 #if defined (__ARM_FP16_FORMAT_IEEE) || defined (__ARM_FP16_FORMAT_ALTERNATIVE)
 186 static ARRAY(result, float, 16, 4);
 187 #endif
 188 static ARRAY(result, float, 32, 2);
 189 #ifdef __aarch64__
 190 static ARRAY(result, float, 64, 1);
 191 #endif
 192 static ARRAY(result, int, 8, 16);
 193 static ARRAY(result, int, 16, 8);
 194 static ARRAY(result, int, 32, 4);
 195 static ARRAY(result, int, 64, 2);
 196 static ARRAY(result, uint, 8, 16);
 197 static ARRAY(result, uint, 16, 8);
 198 static ARRAY(result, uint, 32, 4);
 199 static ARRAY(result, uint, 64, 2);
 200 static ARRAY(result, poly, 8, 16);
 201 static ARRAY(result, poly, 16, 8);
 202 #if defined (__ARM_FEATURE_CRYPTO)
 203 static ARRAY(result, poly, 64, 2);
 204 #endif
 205 #if defined (__ARM_FP16_FORMAT_IEEE) || defined (__ARM_FP16_FORMAT_ALTERNATIVE)
 206 static ARRAY(result, float, 16, 8);
 207 #endif
 208 static ARRAY(result, float, 32, 4);
 209 #ifdef __aarch64__
 210 static ARRAY(result, float, 64, 2);
 211 #endif
 212
 213 /* Declare expected results, one of each size. They are defined and
 214    initialized in each test file.  */
 215 extern ARRAY(expected, int, 8, 8);
 216 extern ARRAY(expected, int, 16, 4);
 217 extern ARRAY(expected, int, 32, 2);
 218 extern ARRAY(expected, int, 64, 1);
 219 extern ARRAY(expected, uint, 8, 8);
 220 extern ARRAY(expected, uint, 16, 4);
 221 extern ARRAY(expected, uint, 32, 2);
 222 extern ARRAY(expected, uint, 64, 1);
 223 extern ARRAY(expected, poly, 8, 8);
 224 extern ARRAY(expected, poly, 16, 4);
 225 extern ARRAY(expected, hfloat, 16, 4);
 226 extern ARRAY(expected, hfloat, 32, 2);
 227 extern ARRAY(expected, hfloat, 64, 1);
 228 extern ARRAY(expected, int, 8, 16);
 229 extern ARRAY(expected, int, 16, 8);
 230 extern ARRAY(expected, int, 32, 4);
 231 extern ARRAY(expected, int, 64, 2);
 232 extern ARRAY(expected, uint, 8, 16);
 233 extern ARRAY(expected, uint, 16, 8);
 234 extern ARRAY(expected, uint, 32, 4);
 235 extern ARRAY(expected, uint, 64, 2);
 236 extern ARRAY(expected, poly, 8, 16);
 237 extern ARRAY(expected, poly, 16, 8);
 238 extern ARRAY(expected, hfloat, 16, 8);
 239 extern ARRAY(expected, hfloat, 32, 4);
 240 extern ARRAY(expected, hfloat, 64, 2);
 241
 242 #define CHECK_RESULTS_NAMED_NO_FP16(test_name,EXPECTED,comment)         \
 243   {                                                                     \
 244     CHECK(test_name, int, 8, 8, PRIx8, EXPECTED, comment);              \
 245     CHECK(test_name, int, 16, 4, PRIx16, EXPECTED, comment);            \
 246     CHECK(test_name, int, 32, 2, PRIx32, EXPECTED, comment);            \
 247     CHECK(test_name, int, 64, 1, PRIx64, EXPECTED, comment);            \
 248     CHECK(test_name, uint, 8, 8, PRIx8, EXPECTED, comment);             \
 249     CHECK(test_name, uint, 16, 4, PRIx16, EXPECTED, comment);           \
 250     CHECK(test_name, uint, 32, 2, PRIx32, EXPECTED, comment);           \
 251     CHECK(test_name, uint, 64, 1, PRIx64, EXPECTED, comment);           \
 252     CHECK_POLY(test_name, poly, 8, 8, PRIx8, EXPECTED, comment);        \
 253     CHECK_POLY(test_name, poly, 16, 4, PRIx16, EXPECTED, comment);      \
 254     CHECK_FP(test_name, float, 32, 2, PRIx32, EXPECTED, comment);       \
 255                                                                         \
 256     CHECK(test_name, int, 8, 16, PRIx8, EXPECTED, comment);             \
 257     CHECK(test_name, int, 16, 8, PRIx16, EXPECTED, comment);            \
 258     CHECK(test_name, int, 32, 4, PRIx32, EXPECTED, comment);            \
 259     CHECK(test_name, int, 64, 2, PRIx64, EXPECTED, comment);            \
 260     CHECK(test_name, uint, 8, 16, PRIx8, EXPECTED, comment);            \
 261     CHECK(test_name, uint, 16, 8, PRIx16, EXPECTED, comment);           \
 262     CHECK(test_name, uint, 32, 4, PRIx32, EXPECTED, comment);           \
 263     CHECK(test_name, uint, 64, 2, PRIx64, EXPECTED, comment);           \
 264     CHECK_POLY(test_name, poly, 8, 16, PRIx8, EXPECTED, comment);       \
 265     CHECK_POLY(test_name, poly, 16, 8, PRIx16, EXPECTED, comment);      \
 266     CHECK_FP(test_name, float, 32, 4, PRIx32, EXPECTED, comment);       \
 267   }                                                                     \
 268
 269 /* Check results against EXPECTED.  Operates on all possible vector types.  */
 270 #if defined (__ARM_FP16_FORMAT_IEEE) || defined (__ARM_FP16_FORMAT_ALTERNATIVE)
 271 #define CHECK_RESULTS_NAMED(test_name,EXPECTED,comment)                 \
 272   {                                                                     \
 273     CHECK_RESULTS_NAMED_NO_FP16(test_name, EXPECTED, comment)           \
 274     CHECK_FP(test_name, float, 16, 4, PRIx16, EXPECTED, comment);       \
 275     CHECK_FP(test_name, float, 16, 8, PRIx16, EXPECTED, comment);       \
 276   }
 277 #else
 278 #define CHECK_RESULTS_NAMED(test_name,EXPECTED,comment)         \
 279   CHECK_RESULTS_NAMED_NO_FP16(test_name, EXPECTED, comment)
 280 #endif
 281
 282 #define CHECK_RESULTS_NO_FP16(test_name,comment)                        \
 283   CHECK_RESULTS_NAMED_NO_FP16(test_name, expected, comment)
 284
 285 #define CHECK_RESULTS(test_name,comment)                \
 286   CHECK_RESULTS_NAMED(test_name, expected, comment)
 287
 288
 289 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
 290
 291 typedef union {
 292   struct {
 293     int _xxx:24;
 294     unsigned int FZ:1;
 295     unsigned int DN:1;
 296     unsigned int AHP:1;
 297     unsigned int QC:1;
 298     int V:1;
 299     int C:1;
 300     int Z:1;
 301     int N:1;
 302   } b;
 303   unsigned int word;
 304 } _ARM_FPSCR;
 305
 306 #else /* __ORDER_BIG_ENDIAN__ */
 307
 308 typedef union {
 309   struct {
 310     int N:1;
 311     int Z:1;
 312     int C:1;
 313     int V:1;
 314     unsigned int QC:1;
 315     unsigned int AHP:1;
 316     unsigned int DN:1;
 317     unsigned int FZ:1;
 318     int _dnm:24;
 319   } b;
 320   unsigned int word;
 321 } _ARM_FPSCR;
 322
 323 #endif /* __ORDER_BIG_ENDIAN__ */
 324
 325 #define Neon_Cumulative_Sat  __read_neon_cumulative_sat()
 326 /* We need a fake dependency to ensure correct ordering of asm
 327    statements to preset the QC flag value, and Neon operators writing
 328    to QC. */
 329 #define Set_Neon_Cumulative_Sat(x, depend)      \
 330   __set_neon_cumulative_sat((x), (depend))
 331
 332 #if defined(__aarch64__)
 333 static volatile int __read_neon_cumulative_sat (void) {
 334     _ARM_FPSCR _afpscr_for_qc;
 335     asm volatile ("mrs %0,fpsr" : "=r" (_afpscr_for_qc));
 336     return _afpscr_for_qc.b.QC;
 337 }
 338 #define __set_neon_cumulative_sat(x, depend) {                          \
 339     _ARM_FPSCR _afpscr_for_qc;                                          \
 340     asm volatile ("mrs %0,fpsr" : "=r" (_afpscr_for_qc));               \
 341     _afpscr_for_qc.b.QC = x;                                            \
 342     asm volatile ("msr fpsr,%1" : "=X" (depend) : "r" (_afpscr_for_qc)); \
 343   }
 344 #else
 345 static volatile int __read_neon_cumulative_sat (void) {
 346     _ARM_FPSCR _afpscr_for_qc;
 347     asm volatile ("vmrs %0,fpscr" : "=r" (_afpscr_for_qc));
 348     return _afpscr_for_qc.b.QC;
 349 }
 350
 351 #define __set_neon_cumulative_sat(x, depend) {                          \
 352     _ARM_FPSCR _afpscr_for_qc;                                          \
 353     asm volatile ("vmrs %0,fpscr" : "=r" (_afpscr_for_qc));             \
 354     _afpscr_for_qc.b.QC = x;                                            \
 355     asm volatile ("vmsr fpscr,%1" : "=X" (depend) : "r" (_afpscr_for_qc)); \
 356   }
 357 #endif
 358
 359 /* Clean output buffers before execution.  */
 360 static void clean_results (void)
 361 {
 362   CLEAN(result, int, 8, 8);
 363   CLEAN(result, int, 16, 4);
 364   CLEAN(result, int, 32, 2);
 365   CLEAN(result, int, 64, 1);
 366   CLEAN(result, uint, 8, 8);
 367   CLEAN(result, uint, 16, 4);
 368   CLEAN(result, uint, 32, 2);
 369   CLEAN(result, uint, 64, 1);
 370   CLEAN(result, poly, 8, 8);
 371   CLEAN(result, poly, 16, 4);
 372 #if defined (__ARM_FEATURE_CRYPTO)
 373   CLEAN(result, poly, 64, 1);
 374 #endif
 375 #if defined (__ARM_FP16_FORMAT_IEEE) || defined (__ARM_FP16_FORMAT_ALTERNATIVE)
 376   CLEAN(result, float, 16, 4);
 377 #endif
 378   CLEAN(result, float, 32, 2);
 379
 380   CLEAN(result, int, 8, 16);
 381   CLEAN(result, int, 16, 8);
 382   CLEAN(result, int, 32, 4);
 383   CLEAN(result, int, 64, 2);
 384   CLEAN(result, uint, 8, 16);
 385   CLEAN(result, uint, 16, 8);
 386   CLEAN(result, uint, 32, 4);
 387   CLEAN(result, uint, 64, 2);
 388   CLEAN(result, poly, 8, 16);
 389   CLEAN(result, poly, 16, 8);
 390 #if defined (__ARM_FEATURE_CRYPTO)
 391   CLEAN(result, poly, 64, 2);
 392 #endif
 393 #if defined (__ARM_FP16_FORMAT_IEEE) || defined (__ARM_FP16_FORMAT_ALTERNATIVE)
 394   CLEAN(result, float, 16, 8);
 395 #endif
 396   CLEAN(result, float, 32, 4);
 397
 398   AARCH64_ONLY(CLEAN(result, float, 64, 2));
 399
 400 #if defined(__aarch64__)
 401   /* On AArch64, make sure to return DefaultNaN to have the same
 402      results as on AArch32.  */
 403   _ARM_FPSCR _afpscr;
 404   asm volatile ("mrs %0,fpcr" : "=r" (_afpscr));
 405   _afpscr.b.DN = 1;
 406
 407   /* On AArch64, make sure to flush to zero by default, as on
 408      AArch32. */
 409   _afpscr.b.FZ = 1;
 410
 411   asm volatile ("msr fpcr,%0" : : "r" (_afpscr));
 412 #endif
 413 }
 414
 415
 416 /* Helpers to declare variables of various types.   */
 417 #define DECL_VARIABLE(VAR, T1, W, N)            \
 418   VECT_TYPE(T1, W, N) VECT_VAR(VAR, T1, W, N)
 419
 420 #if defined (__ARM_FEATURE_CRYPTO)
 421 #define DECL_VARIABLE_CRYPTO(VAR, T1, W, N) \
 422   DECL_VARIABLE(VAR, T1, W, N)
 423 #else
 424 #define DECL_VARIABLE_CRYPTO(VAR, T1, W, N)
 425 #endif
 426
 427 /* Declare only 64 bits signed variants.  */
 428 #define DECL_VARIABLE_64BITS_SIGNED_VARIANTS(VAR)       \
 429   DECL_VARIABLE(VAR, int, 8, 8);                        \
 430   DECL_VARIABLE(VAR, int, 16, 4);                       \
 431   DECL_VARIABLE(VAR, int, 32, 2);                       \
 432   DECL_VARIABLE(VAR, int, 64, 1)
 433
 434 /* Declare only 64 bits unsigned variants.  */
 435 #define DECL_VARIABLE_64BITS_UNSIGNED_VARIANTS(VAR)     \
 436   DECL_VARIABLE(VAR, uint, 8, 8);                       \
 437   DECL_VARIABLE(VAR, uint, 16, 4);                      \
 438   DECL_VARIABLE(VAR, uint, 32, 2);                      \
 439   DECL_VARIABLE(VAR, uint, 64, 1)
 440
 441 /* Declare only 128 bits signed variants.  */
 442 #define DECL_VARIABLE_128BITS_SIGNED_VARIANTS(VAR)      \
 443   DECL_VARIABLE(VAR, int, 8, 16);                       \
 444   DECL_VARIABLE(VAR, int, 16, 8);                       \
 445   DECL_VARIABLE(VAR, int, 32, 4);                       \
 446   DECL_VARIABLE(VAR, int, 64, 2)
 447
 448 /* Declare only 128 bits unsigned variants.  */
 449 #define DECL_VARIABLE_128BITS_UNSIGNED_VARIANTS(VAR)    \
 450   DECL_VARIABLE(VAR, uint, 8, 16);                      \
 451   DECL_VARIABLE(VAR, uint, 16, 8);                      \
 452   DECL_VARIABLE(VAR, uint, 32, 4);                      \
 453   DECL_VARIABLE(VAR, uint, 64, 2)
 454
 455 /* Declare all 64 bits variants.  */
 456 #if defined (__ARM_FP16_FORMAT_IEEE) || defined (__ARM_FP16_FORMAT_ALTERNATIVE)
 457 #define DECL_VARIABLE_64BITS_VARIANTS(VAR)      \
 458   DECL_VARIABLE_64BITS_SIGNED_VARIANTS(VAR);    \
 459   DECL_VARIABLE_64BITS_UNSIGNED_VARIANTS(VAR);  \
 460   DECL_VARIABLE(VAR, poly, 8, 8);               \
 461   DECL_VARIABLE(VAR, poly, 16, 4);              \
 462   DECL_VARIABLE_CRYPTO(VAR, poly, 64, 1);       \
 463   DECL_VARIABLE(VAR, float, 16, 4);             \
 464   DECL_VARIABLE(VAR, float, 32, 2)
 465 #else
 466 #define DECL_VARIABLE_64BITS_VARIANTS(VAR)      \
 467   DECL_VARIABLE_64BITS_SIGNED_VARIANTS(VAR);    \
 468   DECL_VARIABLE_64BITS_UNSIGNED_VARIANTS(VAR);  \
 469   DECL_VARIABLE(VAR, poly, 8, 8);               \
 470   DECL_VARIABLE(VAR, poly, 16, 4);              \
 471   DECL_VARIABLE_CRYPTO(VAR, poly, 64, 1);       \
 472   DECL_VARIABLE(VAR, float, 32, 2)
 473 #endif
 474
 475 /* Declare all 128 bits variants.  */
 476 #if defined (__ARM_FP16_FORMAT_IEEE) || defined (__ARM_FP16_FORMAT_ALTERNATIVE)
 477 #define DECL_VARIABLE_128BITS_VARIANTS(VAR)     \
 478   DECL_VARIABLE_128BITS_SIGNED_VARIANTS(VAR);   \
 479   DECL_VARIABLE_128BITS_UNSIGNED_VARIANTS(VAR); \
 480   DECL_VARIABLE(VAR, poly, 8, 16);              \
 481   DECL_VARIABLE(VAR, poly, 16, 8);              \
 482   DECL_VARIABLE_CRYPTO(VAR, poly, 64, 2);       \
 483   DECL_VARIABLE(VAR, float, 16, 8);             \
 484   DECL_VARIABLE(VAR, float, 32, 4);             \
 485   AARCH64_ONLY(DECL_VARIABLE(VAR, float, 64, 2))
 486 #else
 487 #define DECL_VARIABLE_128BITS_VARIANTS(VAR)     \
 488   DECL_VARIABLE_128BITS_SIGNED_VARIANTS(VAR);   \
 489   DECL_VARIABLE_128BITS_UNSIGNED_VARIANTS(VAR); \
 490   DECL_VARIABLE(VAR, poly, 8, 16);              \
 491   DECL_VARIABLE(VAR, poly, 16, 8);              \
 492   DECL_VARIABLE_CRYPTO(VAR, poly, 64, 2);       \
 493   DECL_VARIABLE(VAR, float, 32, 4);             \
 494   AARCH64_ONLY(DECL_VARIABLE(VAR, float, 64, 2))
 495 #endif
 496 /* Declare all variants.  */
 497 #define DECL_VARIABLE_ALL_VARIANTS(VAR)         \
 498   DECL_VARIABLE_64BITS_VARIANTS(VAR);           \
 499   DECL_VARIABLE_128BITS_VARIANTS(VAR)
 500
 501 /* Declare all signed variants.  */
 502 #define DECL_VARIABLE_SIGNED_VARIANTS(VAR)      \
 503   DECL_VARIABLE_64BITS_SIGNED_VARIANTS(VAR);    \
 504   DECL_VARIABLE_128BITS_SIGNED_VARIANTS(VAR)
 505
 506 /* Declare all unsigned variants.  */
 507 #define DECL_VARIABLE_UNSIGNED_VARIANTS(VAR)    \
 508   DECL_VARIABLE_64BITS_UNSIGNED_VARIANTS(VAR);  \
 509   DECL_VARIABLE_128BITS_UNSIGNED_VARIANTS(VAR)
 510
 511 /* Helpers to initialize vectors.  */
 512 #define VDUP(VAR, Q, T1, T2, W, N, V)                   \
 513   VECT_VAR(VAR, T1, W, N) = vdup##Q##_n_##T2##W(V)
 514
 515 #define VSET_LANE(VAR, Q, T1, T2, W, N, L, V)                           \
 516   VECT_VAR(VAR, T1, W, N) = vset##Q##_lane_##T2##W(V,                   \
 517                                                    VECT_VAR(VAR, T1, W, N), \
 518                                                    L)
 519
 520 /* We need to load initial values first, so rely on VLD1.  */
 521 #define VLOAD(VAR, BUF, Q, T1, T2, W, N)                                \
 522   VECT_VAR(VAR, T1, W, N) = vld1##Q##_##T2##W(VECT_VAR(BUF, T1, W, N))
 523
 524 /* Helpers to call macros with 1 constant and 5 variable
 525    arguments.  */
 526 #if defined (__ARM_FEATURE_CRYPTO)
 527 #define MACRO_CRYPTO(MACRO, VAR1, VAR2, T1, T2, T3, W, N) \
 528   MACRO(VAR1, VAR2, T1, T2, T3, W, N)
 529 #else
 530 #define MACRO_CRYPTO(MACRO, VAR1, VAR2, T1, T2, T3, W, N)
 531 #endif
 532
 533 #define TEST_MACRO_64BITS_SIGNED_VARIANTS_1_5(MACRO, VAR)       \
 534   MACRO(VAR, , int, s, 8, 8);                                   \
 535   MACRO(VAR, , int, s, 16, 4);                                  \
 536   MACRO(VAR, , int, s, 32, 2);                                  \
 537   MACRO(VAR, , int, s, 64, 1)
 538
 539 #define TEST_MACRO_64BITS_UNSIGNED_VARIANTS_1_5(MACRO, VAR)     \
 540   MACRO(VAR, , uint, u, 8, 8);                                  \
 541   MACRO(VAR, , uint, u, 16, 4);                                 \
 542   MACRO(VAR, , uint, u, 32, 2);                                 \
 543   MACRO(VAR, , uint, u, 64, 1)
 544
 545 #define TEST_MACRO_128BITS_SIGNED_VARIANTS_1_5(MACRO, VAR)      \
 546   MACRO(VAR, q, int, s, 8, 16);                                 \
 547   MACRO(VAR, q, int, s, 16, 8);                                 \
 548   MACRO(VAR, q, int, s, 32, 4);                                 \
 549   MACRO(VAR, q, int, s, 64, 2)
 550
 551 #define TEST_MACRO_128BITS_UNSIGNED_VARIANTS_1_5(MACRO,VAR)     \
 552   MACRO(VAR, q, uint, u, 8, 16);                                \
 553   MACRO(VAR, q, uint, u, 16, 8);                                \
 554   MACRO(VAR, q, uint, u, 32, 4);                                \
 555   MACRO(VAR, q, uint, u, 64, 2)
 556
 557 #define TEST_MACRO_64BITS_VARIANTS_1_5(MACRO, VAR)      \
 558   TEST_MACRO_64BITS_SIGNED_VARIANTS_1_5(MACRO, VAR);    \
 559   TEST_MACRO_64BITS_UNSIGNED_VARIANTS_1_5(MACRO, VAR)
 560
 561 #define TEST_MACRO_128BITS_VARIANTS_1_5(MACRO, VAR)     \
 562   TEST_MACRO_128BITS_SIGNED_VARIANTS_1_5(MACRO, VAR);   \
 563   TEST_MACRO_128BITS_UNSIGNED_VARIANTS_1_5(MACRO, VAR)
 564
 565 #define TEST_MACRO_ALL_VARIANTS_1_5(MACRO, VAR) \
 566   TEST_MACRO_64BITS_VARIANTS_1_5(MACRO, VAR);   \
 567   TEST_MACRO_128BITS_VARIANTS_1_5(MACRO, VAR)
 568
 569 #define TEST_MACRO_SIGNED_VARIANTS_1_5(MACRO, VAR)      \
 570   TEST_MACRO_64BITS_SIGNED_VARIANTS_1_5(MACRO, VAR);    \
 571   TEST_MACRO_128BITS_SIGNED_VARIANTS_1_5(MACRO, VAR)
 572
 573 /* Helpers to call macros with 2 constant and 5 variable
 574    arguments.  */
 575 #define TEST_MACRO_64BITS_SIGNED_VARIANTS_2_5(MACRO, VAR1, VAR2)        \
 576   MACRO(VAR1, VAR2, , int, s, 8, 8);                                    \
 577   MACRO(VAR1, VAR2, , int, s, 16, 4);                                   \
 578   MACRO(VAR1, VAR2, , int, s, 32, 2);                                   \
 579   MACRO(VAR1, VAR2 , , int, s, 64, 1)
 580
 581 #define TEST_MACRO_64BITS_UNSIGNED_VARIANTS_2_5(MACRO, VAR1, VAR2)      \
 582   MACRO(VAR1, VAR2, , uint, u, 8, 8);                                   \
 583   MACRO(VAR1, VAR2, , uint, u, 16, 4);                                  \
 584   MACRO(VAR1, VAR2, , uint, u, 32, 2);                                  \
 585   MACRO(VAR1, VAR2, , uint, u, 64, 1)
 586
 587 #define TEST_MACRO_128BITS_SIGNED_VARIANTS_2_5(MACRO, VAR1, VAR2)       \
 588   MACRO(VAR1, VAR2, q, int, s, 8, 16);                                  \
 589   MACRO(VAR1, VAR2, q, int, s, 16, 8);                                  \
 590   MACRO(VAR1, VAR2, q, int, s, 32, 4);                                  \
 591   MACRO(VAR1, VAR2, q, int, s, 64, 2)
 592
 593 #define TEST_MACRO_128BITS_UNSIGNED_VARIANTS_2_5(MACRO, VAR1, VAR2)     \
 594   MACRO(VAR1, VAR2, q, uint, u, 8, 16);                                 \
 595   MACRO(VAR1, VAR2, q, uint, u, 16, 8);                                 \
 596   MACRO(VAR1, VAR2, q, uint, u, 32, 4);                                 \
 597   MACRO(VAR1, VAR2, q, uint, u, 64, 2)
 598
 599 #define TEST_MACRO_64BITS_VARIANTS_2_5(MACRO, VAR1, VAR2)       \
 600   TEST_MACRO_64BITS_SIGNED_VARIANTS_2_5(MACRO, VAR1, VAR2);     \
 601   TEST_MACRO_64BITS_UNSIGNED_VARIANTS_2_5(MACRO, VAR1, VAR2);   \
 602   MACRO(VAR1, VAR2, , poly, p, 8, 8);                           \
 603   MACRO(VAR1, VAR2, , poly, p, 16, 4);                          \
 604   MACRO_CRYPTO(MACRO, VAR1, VAR2, , poly, p, 64, 1)
 605
 606 #define TEST_MACRO_128BITS_VARIANTS_2_5(MACRO, VAR1, VAR2)      \
 607   TEST_MACRO_128BITS_SIGNED_VARIANTS_2_5(MACRO, VAR1, VAR2);    \
 608   TEST_MACRO_128BITS_UNSIGNED_VARIANTS_2_5(MACRO, VAR1, VAR2);  \
 609   MACRO(VAR1, VAR2, q, poly, p, 8, 16);                         \
 610   MACRO(VAR1, VAR2, q, poly, p, 16, 8);                         \
 611   MACRO_CRYPTO(MACRO, VAR1, VAR2, q, poly, p, 64, 2)
 612
 613 #define TEST_MACRO_ALL_VARIANTS_2_5(MACRO, VAR1, VAR2)  \
 614   TEST_MACRO_64BITS_VARIANTS_2_5(MACRO, VAR1, VAR2);    \
 615   TEST_MACRO_128BITS_VARIANTS_2_5(MACRO, VAR1, VAR2)
 616
 617 #define TEST_MACRO_SIGNED_VARIANTS_2_5(MACRO, VAR1, VAR2)       \
 618   TEST_MACRO_64BITS_SIGNED_VARIANTS_2_5(MACRO, VAR1, VAR2);     \
 619   TEST_MACRO_128BITS_SIGNED_VARIANTS_2_5(MACRO, VAR1, VAR2)
 620
 621 #endif /* _ARM_NEON_REF_H_ */