gcc/config/aarch64/aarch64-builtins.c

   1 /* Builtins' description for AArch64 SIMD architecture.
   2    Copyright (C) 2011-2013 Free Software Foundation, Inc.
   3    Contributed by ARM Ltd.
   4
   5    This file is part of GCC.
   6
   7    GCC is free software; you can redistribute it and/or modify it
   8    under the terms of the GNU General Public License as published by
   9    the Free Software Foundation; either version 3, or (at your option)
  10    any later version.
  11
  12    GCC is distributed in the hope that it will be useful, but
  13    WITHOUT ANY WARRANTY; without even the implied warranty of
  14    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15    General Public License for more details.
  16
  17    You should have received a copy of the GNU General Public License
  18    along with GCC; see the file COPYING3.  If not see
  19    <http://www.gnu.org/licenses/>.  */
  20
  21 #include "config.h"
  22 #include "system.h"
  23 #include "coretypes.h"
  24 #include "tm.h"
  25 #include "rtl.h"
  26 #include "tree.h"
  27 #include "expr.h"
  28 #include "tm_p.h"
  29 #include "recog.h"
  30 #include "langhooks.h"
  31 #include "diagnostic-core.h"
  32 #include "optabs.h"
  33 #include "gimple.h"
  34
  35 enum aarch64_simd_builtin_type_mode
  36 {
  37   T_V8QI,
  38   T_V4HI,
  39   T_V2SI,
  40   T_V2SF,
  41   T_DI,
  42   T_DF,
  43   T_V16QI,
  44   T_V8HI,
  45   T_V4SI,
  46   T_V4SF,
  47   T_V2DI,
  48   T_V2DF,
  49   T_TI,
  50   T_EI,
  51   T_OI,
  52   T_XI,
  53   T_SI,
  54   T_SF,
  55   T_HI,
  56   T_QI,
  57   T_MAX
  58 };
  59
  60 #define v8qi_UP  T_V8QI
  61 #define v4hi_UP  T_V4HI
  62 #define v2si_UP  T_V2SI
  63 #define v2sf_UP  T_V2SF
  64 #define di_UP    T_DI
  65 #define df_UP    T_DF
  66 #define v16qi_UP T_V16QI
  67 #define v8hi_UP  T_V8HI
  68 #define v4si_UP  T_V4SI
  69 #define v4sf_UP  T_V4SF
  70 #define v2di_UP  T_V2DI
  71 #define v2df_UP  T_V2DF
  72 #define ti_UP    T_TI
  73 #define ei_UP    T_EI
  74 #define oi_UP    T_OI
  75 #define xi_UP    T_XI
  76 #define si_UP    T_SI
  77 #define sf_UP    T_SF
  78 #define hi_UP    T_HI
  79 #define qi_UP    T_QI
  80
  81 #define UP(X) X##_UP
  82
  83 typedef enum
  84 {
  85   AARCH64_SIMD_BINOP,
  86   AARCH64_SIMD_TERNOP,
  87   AARCH64_SIMD_QUADOP,
  88   AARCH64_SIMD_UNOP,
  89   AARCH64_SIMD_GETLANE,
  90   AARCH64_SIMD_SETLANE,
  91   AARCH64_SIMD_CREATE,
  92   AARCH64_SIMD_DUP,
  93   AARCH64_SIMD_DUPLANE,
  94   AARCH64_SIMD_COMBINE,
  95   AARCH64_SIMD_SPLIT,
  96   AARCH64_SIMD_LANEMUL,
  97   AARCH64_SIMD_LANEMULL,
  98   AARCH64_SIMD_LANEMULH,
  99   AARCH64_SIMD_LANEMAC,
 100   AARCH64_SIMD_SCALARMUL,
 101   AARCH64_SIMD_SCALARMULL,
 102   AARCH64_SIMD_SCALARMULH,
 103   AARCH64_SIMD_SCALARMAC,
 104   AARCH64_SIMD_CONVERT,
 105   AARCH64_SIMD_FIXCONV,
 106   AARCH64_SIMD_SELECT,
 107   AARCH64_SIMD_RESULTPAIR,
 108   AARCH64_SIMD_REINTERP,
 109   AARCH64_SIMD_VTBL,
 110   AARCH64_SIMD_VTBX,
 111   AARCH64_SIMD_LOAD1,
 112   AARCH64_SIMD_LOAD1LANE,
 113   AARCH64_SIMD_STORE1,
 114   AARCH64_SIMD_STORE1LANE,
 115   AARCH64_SIMD_LOADSTRUCT,
 116   AARCH64_SIMD_LOADSTRUCTLANE,
 117   AARCH64_SIMD_STORESTRUCT,
 118   AARCH64_SIMD_STORESTRUCTLANE,
 119   AARCH64_SIMD_LOGICBINOP,
 120   AARCH64_SIMD_SHIFTINSERT,
 121   AARCH64_SIMD_SHIFTIMM,
 122   AARCH64_SIMD_SHIFTACC
 123 } aarch64_simd_itype;
 124
 125 typedef struct
 126 {
 127   const char *name;
 128   const aarch64_simd_itype itype;
 129   enum aarch64_simd_builtin_type_mode mode;
 130   const enum insn_code code;
 131   unsigned int fcode;
 132 } aarch64_simd_builtin_datum;
 133
 134 #define CF0(N, X) CODE_FOR_aarch64_##N##X
 135 #define CF1(N, X) CODE_FOR_##N##X##1
 136 #define CF2(N, X) CODE_FOR_##N##X##2
 137 #define CF3(N, X) CODE_FOR_##N##X##3
 138 #define CF4(N, X) CODE_FOR_##N##X##4
 139 #define CF10(N, X) CODE_FOR_##N##X
 140
 141 #define VAR1(T, N, MAP, A) \
 142   {#N, AARCH64_SIMD_##T, UP (A), CF##MAP (N, A), 0},
 143 #define VAR2(T, N, MAP, A, B) \
 144   VAR1 (T, N, MAP, A) \
 145   VAR1 (T, N, MAP, B)
 146 #define VAR3(T, N, MAP, A, B, C) \
 147   VAR2 (T, N, MAP, A, B) \
 148   VAR1 (T, N, MAP, C)
 149 #define VAR4(T, N, MAP, A, B, C, D) \
 150   VAR3 (T, N, MAP, A, B, C) \
 151   VAR1 (T, N, MAP, D)
 152 #define VAR5(T, N, MAP, A, B, C, D, E) \
 153   VAR4 (T, N, MAP, A, B, C, D) \
 154   VAR1 (T, N, MAP, E)
 155 #define VAR6(T, N, MAP, A, B, C, D, E, F) \
 156   VAR5 (T, N, MAP, A, B, C, D, E) \
 157   VAR1 (T, N, MAP, F)
 158 #define VAR7(T, N, MAP, A, B, C, D, E, F, G) \
 159   VAR6 (T, N, MAP, A, B, C, D, E, F) \
 160   VAR1 (T, N, MAP, G)
 161 #define VAR8(T, N, MAP, A, B, C, D, E, F, G, H) \
 162   VAR7 (T, N, MAP, A, B, C, D, E, F, G) \
 163   VAR1 (T, N, MAP, H)
 164 #define VAR9(T, N, MAP, A, B, C, D, E, F, G, H, I) \
 165   VAR8 (T, N, MAP, A, B, C, D, E, F, G, H) \
 166   VAR1 (T, N, MAP, I)
 167 #define VAR10(T, N, MAP, A, B, C, D, E, F, G, H, I, J) \
 168   VAR9 (T, N, MAP, A, B, C, D, E, F, G, H, I) \
 169   VAR1 (T, N, MAP, J)
 170 #define VAR11(T, N, MAP, A, B, C, D, E, F, G, H, I, J, K) \
 171   VAR10 (T, N, MAP, A, B, C, D, E, F, G, H, I, J) \
 172   VAR1 (T, N, MAP, K)
 173 #define VAR12(T, N, MAP, A, B, C, D, E, F, G, H, I, J, K, L) \
 174   VAR11 (T, N, MAP, A, B, C, D, E, F, G, H, I, J, K) \
 175   VAR1 (T, N, MAP, L)
 176
 177 /* BUILTIN_<ITERATOR> macros should expand to cover the same range of
 178    modes as is given for each define_mode_iterator in
 179    config/aarch64/iterators.md.  */
 180
 181 #define BUILTIN_DX(T, N, MAP) \
 182   VAR2 (T, N, MAP, di, df)
 183 #define BUILTIN_GPF(T, N, MAP) \
 184   VAR2 (T, N, MAP, sf, df)
 185 #define BUILTIN_SDQ_I(T, N, MAP) \
 186   VAR4 (T, N, MAP, qi, hi, si, di)
 187 #define BUILTIN_SD_HSI(T, N, MAP) \
 188   VAR2 (T, N, MAP, hi, si)
 189 #define BUILTIN_V2F(T, N, MAP) \
 190   VAR2 (T, N, MAP, v2sf, v2df)
 191 #define BUILTIN_VALL(T, N, MAP) \
 192   VAR10 (T, N, MAP, v8qi, v16qi, v4hi, v8hi, v2si, \
 193          v4si, v2di, v2sf, v4sf, v2df)
 194 #define BUILTIN_VALLDI(T, N, MAP) \
 195   VAR11 (T, N, MAP, v8qi, v16qi, v4hi, v8hi, v2si, \
 196          v4si, v2di, v2sf, v4sf, v2df, di)
 197 #define BUILTIN_VB(T, N, MAP) \
 198   VAR2 (T, N, MAP, v8qi, v16qi)
 199 #define BUILTIN_VD(T, N, MAP) \
 200   VAR4 (T, N, MAP, v8qi, v4hi, v2si, v2sf)
 201 #define BUILTIN_VDC(T, N, MAP) \
 202   VAR6 (T, N, MAP, v8qi, v4hi, v2si, v2sf, di, df)
 203 #define BUILTIN_VDIC(T, N, MAP) \
 204   VAR3 (T, N, MAP, v8qi, v4hi, v2si)
 205 #define BUILTIN_VDN(T, N, MAP) \
 206   VAR3 (T, N, MAP, v4hi, v2si, di)
 207 #define BUILTIN_VDQ(T, N, MAP) \
 208   VAR7 (T, N, MAP, v8qi, v16qi, v4hi, v8hi, v2si, v4si, v2di)
 209 #define BUILTIN_VDQF(T, N, MAP) \
 210   VAR3 (T, N, MAP, v2sf, v4sf, v2df)
 211 #define BUILTIN_VDQH(T, N, MAP) \
 212   VAR2 (T, N, MAP, v4hi, v8hi)
 213 #define BUILTIN_VDQHS(T, N, MAP) \
 214   VAR4 (T, N, MAP, v4hi, v8hi, v2si, v4si)
 215 #define BUILTIN_VDQIF(T, N, MAP) \
 216   VAR9 (T, N, MAP, v8qi, v16qi, v4hi, v8hi, v2si, v4si, v2sf, v4sf, v2df)
 217 #define BUILTIN_VDQM(T, N, MAP) \
 218   VAR6 (T, N, MAP, v8qi, v16qi, v4hi, v8hi, v2si, v4si)
 219 #define BUILTIN_VDQV(T, N, MAP) \
 220   VAR5 (T, N, MAP, v8qi, v16qi, v4hi, v8hi, v4si)
 221 #define BUILTIN_VDQ_BHSI(T, N, MAP) \
 222   VAR6 (T, N, MAP, v8qi, v16qi, v4hi, v8hi, v2si, v4si)
 223 #define BUILTIN_VDQ_I(T, N, MAP) \
 224   VAR7 (T, N, MAP, v8qi, v16qi, v4hi, v8hi, v2si, v4si, v2di)
 225 #define BUILTIN_VDW(T, N, MAP) \
 226   VAR3 (T, N, MAP, v8qi, v4hi, v2si)
 227 #define BUILTIN_VD_BHSI(T, N, MAP) \
 228   VAR3 (T, N, MAP, v8qi, v4hi, v2si)
 229 #define BUILTIN_VD_HSI(T, N, MAP) \
 230   VAR2 (T, N, MAP, v4hi, v2si)
 231 #define BUILTIN_VD_RE(T, N, MAP) \
 232   VAR6 (T, N, MAP, v8qi, v4hi, v2si, v2sf, di, df)
 233 #define BUILTIN_VQ(T, N, MAP) \
 234   VAR6 (T, N, MAP, v16qi, v8hi, v4si, v2di, v4sf, v2df)
 235 #define BUILTIN_VQN(T, N, MAP) \
 236   VAR3 (T, N, MAP, v8hi, v4si, v2di)
 237 #define BUILTIN_VQW(T, N, MAP) \
 238   VAR3 (T, N, MAP, v16qi, v8hi, v4si)
 239 #define BUILTIN_VQ_HSI(T, N, MAP) \
 240   VAR2 (T, N, MAP, v8hi, v4si)
 241 #define BUILTIN_VQ_S(T, N, MAP) \
 242   VAR6 (T, N, MAP, v8qi, v16qi, v4hi, v8hi, v2si, v4si)
 243 #define BUILTIN_VSDQ_HSI(T, N, MAP) \
 244   VAR6 (T, N, MAP, v4hi, v8hi, v2si, v4si, hi, si)
 245 #define BUILTIN_VSDQ_I(T, N, MAP) \
 246   VAR11 (T, N, MAP, v8qi, v16qi, v4hi, v8hi, v2si, v4si, v2di, qi, hi, si, di)
 247 #define BUILTIN_VSDQ_I_BHSI(T, N, MAP) \
 248   VAR10 (T, N, MAP, v8qi, v16qi, v4hi, v8hi, v2si, v4si, v2di, qi, hi, si)
 249 #define BUILTIN_VSDQ_I_DI(T, N, MAP) \
 250   VAR8 (T, N, MAP, v8qi, v16qi, v4hi, v8hi, v2si, v4si, v2di, di)
 251 #define BUILTIN_VSD_HSI(T, N, MAP) \
 252   VAR4 (T, N, MAP, v4hi, v2si, hi, si)
 253 #define BUILTIN_VSQN_HSDI(T, N, MAP) \
 254   VAR6 (T, N, MAP, v8hi, v4si, v2di, hi, si, di)
 255 #define BUILTIN_VSTRUCT(T, N, MAP) \
 256   VAR3 (T, N, MAP, oi, ci, xi)
 257
 258 static aarch64_simd_builtin_datum aarch64_simd_builtin_data[] = {
 259 #include "aarch64-simd-builtins.def"
 260 };
 261
 262 #undef VAR1
 263 #define VAR1(T, N, MAP, A) \
 264   AARCH64_SIMD_BUILTIN_##N##A,
 265
 266 enum aarch64_builtins
 267 {
 268   AARCH64_BUILTIN_MIN,
 269   AARCH64_SIMD_BUILTIN_BASE,
 270 #include "aarch64-simd-builtins.def"
 271   AARCH64_SIMD_BUILTIN_MAX = AARCH64_SIMD_BUILTIN_BASE
 272                               + ARRAY_SIZE (aarch64_simd_builtin_data),
 273   AARCH64_BUILTIN_MAX
 274 };
 275
 276 static GTY(()) tree aarch64_builtin_decls[AARCH64_BUILTIN_MAX];
 277
 278 #define NUM_DREG_TYPES 6
 279 #define NUM_QREG_TYPES 6
 280
 281 static void
 282 aarch64_init_simd_builtins (void)
 283 {
 284   unsigned int i, fcode = AARCH64_SIMD_BUILTIN_BASE + 1;
 285
 286   /* Scalar type nodes.  */
 287   tree aarch64_simd_intQI_type_node;
 288   tree aarch64_simd_intHI_type_node;
 289   tree aarch64_simd_polyQI_type_node;
 290   tree aarch64_simd_polyHI_type_node;
 291   tree aarch64_simd_intSI_type_node;
 292   tree aarch64_simd_intDI_type_node;
 293   tree aarch64_simd_float_type_node;
 294   tree aarch64_simd_double_type_node;
 295
 296   /* Pointer to scalar type nodes.  */
 297   tree intQI_pointer_node;
 298   tree intHI_pointer_node;
 299   tree intSI_pointer_node;
 300   tree intDI_pointer_node;
 301   tree float_pointer_node;
 302   tree double_pointer_node;
 303
 304   /* Const scalar type nodes.  */
 305   tree const_intQI_node;
 306   tree const_intHI_node;
 307   tree const_intSI_node;
 308   tree const_intDI_node;
 309   tree const_float_node;
 310   tree const_double_node;
 311
 312   /* Pointer to const scalar type nodes.  */
 313   tree const_intQI_pointer_node;
 314   tree const_intHI_pointer_node;
 315   tree const_intSI_pointer_node;
 316   tree const_intDI_pointer_node;
 317   tree const_float_pointer_node;
 318   tree const_double_pointer_node;
 319
 320   /* Vector type nodes.  */
 321   tree V8QI_type_node;
 322   tree V4HI_type_node;
 323   tree V2SI_type_node;
 324   tree V2SF_type_node;
 325   tree V16QI_type_node;
 326   tree V8HI_type_node;
 327   tree V4SI_type_node;
 328   tree V4SF_type_node;
 329   tree V2DI_type_node;
 330   tree V2DF_type_node;
 331
 332   /* Scalar unsigned type nodes.  */
 333   tree intUQI_type_node;
 334   tree intUHI_type_node;
 335   tree intUSI_type_node;
 336   tree intUDI_type_node;
 337
 338   /* Opaque integer types for structures of vectors.  */
 339   tree intEI_type_node;
 340   tree intOI_type_node;
 341   tree intCI_type_node;
 342   tree intXI_type_node;
 343
 344   /* Pointer to vector type nodes.  */
 345   tree V8QI_pointer_node;
 346   tree V4HI_pointer_node;
 347   tree V2SI_pointer_node;
 348   tree V2SF_pointer_node;
 349   tree V16QI_pointer_node;
 350   tree V8HI_pointer_node;
 351   tree V4SI_pointer_node;
 352   tree V4SF_pointer_node;
 353   tree V2DI_pointer_node;
 354   tree V2DF_pointer_node;
 355
 356   /* Operations which return results as pairs.  */
 357   tree void_ftype_pv8qi_v8qi_v8qi;
 358   tree void_ftype_pv4hi_v4hi_v4hi;
 359   tree void_ftype_pv2si_v2si_v2si;
 360   tree void_ftype_pv2sf_v2sf_v2sf;
 361   tree void_ftype_pdi_di_di;
 362   tree void_ftype_pv16qi_v16qi_v16qi;
 363   tree void_ftype_pv8hi_v8hi_v8hi;
 364   tree void_ftype_pv4si_v4si_v4si;
 365   tree void_ftype_pv4sf_v4sf_v4sf;
 366   tree void_ftype_pv2di_v2di_v2di;
 367   tree void_ftype_pv2df_v2df_v2df;
 368
 369   tree reinterp_ftype_dreg[NUM_DREG_TYPES][NUM_DREG_TYPES];
 370   tree reinterp_ftype_qreg[NUM_QREG_TYPES][NUM_QREG_TYPES];
 371   tree dreg_types[NUM_DREG_TYPES], qreg_types[NUM_QREG_TYPES];
 372
 373   /* Create distinguished type nodes for AARCH64_SIMD vector element types,
 374      and pointers to values of such types, so we can detect them later.  */
 375   aarch64_simd_intQI_type_node =
 376     make_signed_type (GET_MODE_PRECISION (QImode));
 377   aarch64_simd_intHI_type_node =
 378     make_signed_type (GET_MODE_PRECISION (HImode));
 379   aarch64_simd_polyQI_type_node =
 380     make_signed_type (GET_MODE_PRECISION (QImode));
 381   aarch64_simd_polyHI_type_node =
 382     make_signed_type (GET_MODE_PRECISION (HImode));
 383   aarch64_simd_intSI_type_node =
 384     make_signed_type (GET_MODE_PRECISION (SImode));
 385   aarch64_simd_intDI_type_node =
 386     make_signed_type (GET_MODE_PRECISION (DImode));
 387   aarch64_simd_float_type_node = make_node (REAL_TYPE);
 388   aarch64_simd_double_type_node = make_node (REAL_TYPE);
 389   TYPE_PRECISION (aarch64_simd_float_type_node) = FLOAT_TYPE_SIZE;
 390   TYPE_PRECISION (aarch64_simd_double_type_node) = DOUBLE_TYPE_SIZE;
 391   layout_type (aarch64_simd_float_type_node);
 392   layout_type (aarch64_simd_double_type_node);
 393
 394   /* Define typedefs which exactly correspond to the modes we are basing vector
 395      types on.  If you change these names you'll need to change
 396      the table used by aarch64_mangle_type too.  */
 397   (*lang_hooks.types.register_builtin_type) (aarch64_simd_intQI_type_node,
 398                                              "__builtin_aarch64_simd_qi");
 399   (*lang_hooks.types.register_builtin_type) (aarch64_simd_intHI_type_node,
 400                                              "__builtin_aarch64_simd_hi");
 401   (*lang_hooks.types.register_builtin_type) (aarch64_simd_intSI_type_node,
 402                                              "__builtin_aarch64_simd_si");
 403   (*lang_hooks.types.register_builtin_type) (aarch64_simd_float_type_node,
 404                                              "__builtin_aarch64_simd_sf");
 405   (*lang_hooks.types.register_builtin_type) (aarch64_simd_intDI_type_node,
 406                                              "__builtin_aarch64_simd_di");
 407   (*lang_hooks.types.register_builtin_type) (aarch64_simd_double_type_node,
 408                                              "__builtin_aarch64_simd_df");
 409   (*lang_hooks.types.register_builtin_type) (aarch64_simd_polyQI_type_node,
 410                                              "__builtin_aarch64_simd_poly8");
 411   (*lang_hooks.types.register_builtin_type) (aarch64_simd_polyHI_type_node,
 412                                              "__builtin_aarch64_simd_poly16");
 413
 414   intQI_pointer_node = build_pointer_type (aarch64_simd_intQI_type_node);
 415   intHI_pointer_node = build_pointer_type (aarch64_simd_intHI_type_node);
 416   intSI_pointer_node = build_pointer_type (aarch64_simd_intSI_type_node);
 417   intDI_pointer_node = build_pointer_type (aarch64_simd_intDI_type_node);
 418   float_pointer_node = build_pointer_type (aarch64_simd_float_type_node);
 419   double_pointer_node = build_pointer_type (aarch64_simd_double_type_node);
 420
 421   /* Next create constant-qualified versions of the above types.  */
 422   const_intQI_node = build_qualified_type (aarch64_simd_intQI_type_node,
 423                                            TYPE_QUAL_CONST);
 424   const_intHI_node = build_qualified_type (aarch64_simd_intHI_type_node,
 425                                            TYPE_QUAL_CONST);
 426   const_intSI_node = build_qualified_type (aarch64_simd_intSI_type_node,
 427                                            TYPE_QUAL_CONST);
 428   const_intDI_node = build_qualified_type (aarch64_simd_intDI_type_node,
 429                                            TYPE_QUAL_CONST);
 430   const_float_node = build_qualified_type (aarch64_simd_float_type_node,
 431                                            TYPE_QUAL_CONST);
 432   const_double_node = build_qualified_type (aarch64_simd_double_type_node,
 433                                             TYPE_QUAL_CONST);
 434
 435   const_intQI_pointer_node = build_pointer_type (const_intQI_node);
 436   const_intHI_pointer_node = build_pointer_type (const_intHI_node);
 437   const_intSI_pointer_node = build_pointer_type (const_intSI_node);
 438   const_intDI_pointer_node = build_pointer_type (const_intDI_node);
 439   const_float_pointer_node = build_pointer_type (const_float_node);
 440   const_double_pointer_node = build_pointer_type (const_double_node);
 441
 442   /* Now create vector types based on our AARCH64 SIMD element types.  */
 443   /* 64-bit vectors.  */
 444   V8QI_type_node =
 445     build_vector_type_for_mode (aarch64_simd_intQI_type_node, V8QImode);
 446   V4HI_type_node =
 447     build_vector_type_for_mode (aarch64_simd_intHI_type_node, V4HImode);
 448   V2SI_type_node =
 449     build_vector_type_for_mode (aarch64_simd_intSI_type_node, V2SImode);
 450   V2SF_type_node =
 451     build_vector_type_for_mode (aarch64_simd_float_type_node, V2SFmode);
 452   /* 128-bit vectors.  */
 453   V16QI_type_node =
 454     build_vector_type_for_mode (aarch64_simd_intQI_type_node, V16QImode);
 455   V8HI_type_node =
 456     build_vector_type_for_mode (aarch64_simd_intHI_type_node, V8HImode);
 457   V4SI_type_node =
 458     build_vector_type_for_mode (aarch64_simd_intSI_type_node, V4SImode);
 459   V4SF_type_node =
 460     build_vector_type_for_mode (aarch64_simd_float_type_node, V4SFmode);
 461   V2DI_type_node =
 462     build_vector_type_for_mode (aarch64_simd_intDI_type_node, V2DImode);
 463   V2DF_type_node =
 464     build_vector_type_for_mode (aarch64_simd_double_type_node, V2DFmode);
 465
 466   /* Unsigned integer types for various mode sizes.  */
 467   intUQI_type_node = make_unsigned_type (GET_MODE_PRECISION (QImode));
 468   intUHI_type_node = make_unsigned_type (GET_MODE_PRECISION (HImode));
 469   intUSI_type_node = make_unsigned_type (GET_MODE_PRECISION (SImode));
 470   intUDI_type_node = make_unsigned_type (GET_MODE_PRECISION (DImode));
 471
 472   (*lang_hooks.types.register_builtin_type) (intUQI_type_node,
 473                                              "__builtin_aarch64_simd_uqi");
 474   (*lang_hooks.types.register_builtin_type) (intUHI_type_node,
 475                                              "__builtin_aarch64_simd_uhi");
 476   (*lang_hooks.types.register_builtin_type) (intUSI_type_node,
 477                                              "__builtin_aarch64_simd_usi");
 478   (*lang_hooks.types.register_builtin_type) (intUDI_type_node,
 479                                              "__builtin_aarch64_simd_udi");
 480
 481   /* Opaque integer types for structures of vectors.  */
 482   intEI_type_node = make_signed_type (GET_MODE_PRECISION (EImode));
 483   intOI_type_node = make_signed_type (GET_MODE_PRECISION (OImode));
 484   intCI_type_node = make_signed_type (GET_MODE_PRECISION (CImode));
 485   intXI_type_node = make_signed_type (GET_MODE_PRECISION (XImode));
 486
 487   (*lang_hooks.types.register_builtin_type) (intTI_type_node,
 488                                              "__builtin_aarch64_simd_ti");
 489   (*lang_hooks.types.register_builtin_type) (intEI_type_node,
 490                                              "__builtin_aarch64_simd_ei");
 491   (*lang_hooks.types.register_builtin_type) (intOI_type_node,
 492                                              "__builtin_aarch64_simd_oi");
 493   (*lang_hooks.types.register_builtin_type) (intCI_type_node,
 494                                              "__builtin_aarch64_simd_ci");
 495   (*lang_hooks.types.register_builtin_type) (intXI_type_node,
 496                                              "__builtin_aarch64_simd_xi");
 497
 498   /* Pointers to vector types.  */
 499   V8QI_pointer_node = build_pointer_type (V8QI_type_node);
 500   V4HI_pointer_node = build_pointer_type (V4HI_type_node);
 501   V2SI_pointer_node = build_pointer_type (V2SI_type_node);
 502   V2SF_pointer_node = build_pointer_type (V2SF_type_node);
 503   V16QI_pointer_node = build_pointer_type (V16QI_type_node);
 504   V8HI_pointer_node = build_pointer_type (V8HI_type_node);
 505   V4SI_pointer_node = build_pointer_type (V4SI_type_node);
 506   V4SF_pointer_node = build_pointer_type (V4SF_type_node);
 507   V2DI_pointer_node = build_pointer_type (V2DI_type_node);
 508   V2DF_pointer_node = build_pointer_type (V2DF_type_node);
 509
 510   /* Operations which return results as pairs.  */
 511   void_ftype_pv8qi_v8qi_v8qi =
 512     build_function_type_list (void_type_node, V8QI_pointer_node,
 513                               V8QI_type_node, V8QI_type_node, NULL);
 514   void_ftype_pv4hi_v4hi_v4hi =
 515     build_function_type_list (void_type_node, V4HI_pointer_node,
 516                               V4HI_type_node, V4HI_type_node, NULL);
 517   void_ftype_pv2si_v2si_v2si =
 518     build_function_type_list (void_type_node, V2SI_pointer_node,
 519                               V2SI_type_node, V2SI_type_node, NULL);
 520   void_ftype_pv2sf_v2sf_v2sf =
 521     build_function_type_list (void_type_node, V2SF_pointer_node,
 522                               V2SF_type_node, V2SF_type_node, NULL);
 523   void_ftype_pdi_di_di =
 524     build_function_type_list (void_type_node, intDI_pointer_node,
 525                               aarch64_simd_intDI_type_node,
 526                               aarch64_simd_intDI_type_node, NULL);
 527   void_ftype_pv16qi_v16qi_v16qi =
 528     build_function_type_list (void_type_node, V16QI_pointer_node,
 529                               V16QI_type_node, V16QI_type_node, NULL);
 530   void_ftype_pv8hi_v8hi_v8hi =
 531     build_function_type_list (void_type_node, V8HI_pointer_node,
 532                               V8HI_type_node, V8HI_type_node, NULL);
 533   void_ftype_pv4si_v4si_v4si =
 534     build_function_type_list (void_type_node, V4SI_pointer_node,
 535                               V4SI_type_node, V4SI_type_node, NULL);
 536   void_ftype_pv4sf_v4sf_v4sf =
 537     build_function_type_list (void_type_node, V4SF_pointer_node,
 538                               V4SF_type_node, V4SF_type_node, NULL);
 539   void_ftype_pv2di_v2di_v2di =
 540     build_function_type_list (void_type_node, V2DI_pointer_node,
 541                               V2DI_type_node, V2DI_type_node, NULL);
 542   void_ftype_pv2df_v2df_v2df =
 543     build_function_type_list (void_type_node, V2DF_pointer_node,
 544                               V2DF_type_node, V2DF_type_node, NULL);
 545
 546   dreg_types[0] = V8QI_type_node;
 547   dreg_types[1] = V4HI_type_node;
 548   dreg_types[2] = V2SI_type_node;
 549   dreg_types[3] = V2SF_type_node;
 550   dreg_types[4] = aarch64_simd_intDI_type_node;
 551   dreg_types[5] = aarch64_simd_double_type_node;
 552
 553   qreg_types[0] = V16QI_type_node;
 554   qreg_types[1] = V8HI_type_node;
 555   qreg_types[2] = V4SI_type_node;
 556   qreg_types[3] = V4SF_type_node;
 557   qreg_types[4] = V2DI_type_node;
 558   qreg_types[5] = V2DF_type_node;
 559
 560   /* If NUM_DREG_TYPES != NUM_QREG_TYPES, we will need separate nested loops
 561      for qreg and dreg reinterp inits.  */
 562   for (i = 0; i < NUM_DREG_TYPES; i++)
 563     {
 564       int j;
 565       for (j = 0; j < NUM_DREG_TYPES; j++)
 566         {
 567           reinterp_ftype_dreg[i][j]
 568             = build_function_type_list (dreg_types[i], dreg_types[j], NULL);
 569           reinterp_ftype_qreg[i][j]
 570             = build_function_type_list (qreg_types[i], qreg_types[j], NULL);
 571         }
 572     }
 573
 574   for (i = 0; i < ARRAY_SIZE (aarch64_simd_builtin_data); i++, fcode++)
 575     {
 576       aarch64_simd_builtin_datum *d = &aarch64_simd_builtin_data[i];
 577       const char *const modenames[] =
 578       {
 579         "v8qi", "v4hi", "v2si", "v2sf", "di", "df",
 580         "v16qi", "v8hi", "v4si", "v4sf", "v2di", "v2df",
 581         "ti", "ei", "oi", "xi", "si", "sf", "hi", "qi"
 582       };
 583       char namebuf[60];
 584       tree ftype = NULL;
 585       tree fndecl = NULL;
 586       int is_load = 0;
 587       int is_store = 0;
 588
 589       gcc_assert (ARRAY_SIZE (modenames) == T_MAX);
 590
 591       d->fcode = fcode;
 592
 593       switch (d->itype)
 594         {
 595         case AARCH64_SIMD_LOAD1:
 596         case AARCH64_SIMD_LOAD1LANE:
 597         case AARCH64_SIMD_LOADSTRUCT:
 598         case AARCH64_SIMD_LOADSTRUCTLANE:
 599             is_load = 1;
 600           /* Fall through.  */
 601         case AARCH64_SIMD_STORE1:
 602         case AARCH64_SIMD_STORE1LANE:
 603         case AARCH64_SIMD_STORESTRUCT:
 604         case AARCH64_SIMD_STORESTRUCTLANE:
 605             if (!is_load)
 606               is_store = 1;
 607           /* Fall through.  */
 608         case AARCH64_SIMD_UNOP:
 609         case AARCH64_SIMD_BINOP:
 610         case AARCH64_SIMD_TERNOP:
 611         case AARCH64_SIMD_QUADOP:
 612         case AARCH64_SIMD_COMBINE:
 613         case AARCH64_SIMD_CONVERT:
 614         case AARCH64_SIMD_CREATE:
 615         case AARCH64_SIMD_DUP:
 616         case AARCH64_SIMD_DUPLANE:
 617         case AARCH64_SIMD_FIXCONV:
 618         case AARCH64_SIMD_GETLANE:
 619         case AARCH64_SIMD_LANEMAC:
 620         case AARCH64_SIMD_LANEMUL:
 621         case AARCH64_SIMD_LANEMULH:
 622         case AARCH64_SIMD_LANEMULL:
 623         case AARCH64_SIMD_LOGICBINOP:
 624         case AARCH64_SIMD_SCALARMAC:
 625         case AARCH64_SIMD_SCALARMUL:
 626         case AARCH64_SIMD_SCALARMULH:
 627         case AARCH64_SIMD_SCALARMULL:
 628         case AARCH64_SIMD_SELECT:
 629         case AARCH64_SIMD_SETLANE:
 630         case AARCH64_SIMD_SHIFTACC:
 631         case AARCH64_SIMD_SHIFTIMM:
 632         case AARCH64_SIMD_SHIFTINSERT:
 633         case AARCH64_SIMD_SPLIT:
 634         case AARCH64_SIMD_VTBL:
 635         case AARCH64_SIMD_VTBX:
 636           {
 637             int k;
 638             tree return_type = void_type_node, args = void_list_node;
 639             tree eltype;
 640             /* Build a function type directly from the insn_data for this
 641                builtin.  The build_function_type () function takes care of
 642                removing duplicates for us.  */
 643
 644             for (k = insn_data[d->code].n_operands -1; k >= 0; k--)
 645               {
 646                 /* Skip an internal operand for vget_{low, high}.  */
 647                 if (k == 2 && d->itype == AARCH64_SIMD_SPLIT)
 648                   continue;
 649
 650                 if (is_load && k == 1)
 651                   {
 652                     /* AdvSIMD load patterns always have the memory operand
 653                        (a DImode pointer) in the operand 1 position.  We
 654                        want a const pointer to the element type in that
 655                        position.  */
 656                     gcc_assert (insn_data[d->code].operand[k].mode == DImode);
 657
 658                     switch (d->mode)
 659                       {
 660                       case T_V8QI:
 661                       case T_V16QI:
 662                         eltype = const_intQI_pointer_node;
 663                         break;
 664
 665                       case T_V4HI:
 666                       case T_V8HI:
 667                         eltype = const_intHI_pointer_node;
 668                         break;
 669
 670                       case T_V2SI:
 671                       case T_V4SI:
 672                         eltype = const_intSI_pointer_node;
 673                         break;
 674
 675                       case T_V2SF:
 676                       case T_V4SF:
 677                         eltype = const_float_pointer_node;
 678                         break;
 679
 680                       case T_DI:
 681                       case T_V2DI:
 682                         eltype = const_intDI_pointer_node;
 683                         break;
 684
 685                       case T_DF:
 686                       case T_V2DF:
 687                         eltype = const_double_pointer_node;
 688                         break;
 689
 690                       default:
 691                         gcc_unreachable ();
 692                       }
 693                   }
 694                 else if (is_store && k == 0)
 695                   {
 696                     /* Similarly, AdvSIMD store patterns use operand 0 as
 697                        the memory location to store to (a DImode pointer).
 698                        Use a pointer to the element type of the store in
 699                        that position.  */
 700                     gcc_assert (insn_data[d->code].operand[k].mode == DImode);
 701
 702                     switch (d->mode)
 703                       {
 704                       case T_V8QI:
 705                       case T_V16QI:
 706                         eltype = intQI_pointer_node;
 707                         break;
 708
 709                       case T_V4HI:
 710                       case T_V8HI:
 711                         eltype = intHI_pointer_node;
 712                         break;
 713
 714                       case T_V2SI:
 715                       case T_V4SI:
 716                         eltype = intSI_pointer_node;
 717                         break;
 718
 719                       case T_V2SF:
 720                       case T_V4SF:
 721                         eltype = float_pointer_node;
 722                         break;
 723
 724                       case T_DI:
 725                       case T_V2DI:
 726                         eltype = intDI_pointer_node;
 727                         break;
 728
 729                       case T_DF:
 730                       case T_V2DF:
 731                         eltype = double_pointer_node;
 732                         break;
 733
 734                       default:
 735                         gcc_unreachable ();
 736                       }
 737                   }
 738                 else
 739                   {
 740                     switch (insn_data[d->code].operand[k].mode)
 741                       {
 742                       case VOIDmode:
 743                         eltype = void_type_node;
 744                         break;
 745                         /* Scalars.  */
 746                       case QImode:
 747                         eltype = aarch64_simd_intQI_type_node;
 748                         break;
 749                       case HImode:
 750                         eltype = aarch64_simd_intHI_type_node;
 751                         break;
 752                       case SImode:
 753                         eltype = aarch64_simd_intSI_type_node;
 754                         break;
 755                       case SFmode:
 756                         eltype = aarch64_simd_float_type_node;
 757                         break;
 758                       case DFmode:
 759                         eltype = aarch64_simd_double_type_node;
 760                         break;
 761                       case DImode:
 762                         eltype = aarch64_simd_intDI_type_node;
 763                         break;
 764                       case TImode:
 765                         eltype = intTI_type_node;
 766                         break;
 767                       case EImode:
 768                         eltype = intEI_type_node;
 769                         break;
 770                       case OImode:
 771                         eltype = intOI_type_node;
 772                         break;
 773                       case CImode:
 774                         eltype = intCI_type_node;
 775                         break;
 776                       case XImode:
 777                         eltype = intXI_type_node;
 778                         break;
 779                         /* 64-bit vectors.  */
 780                       case V8QImode:
 781                         eltype = V8QI_type_node;
 782                         break;
 783                       case V4HImode:
 784                         eltype = V4HI_type_node;
 785                         break;
 786                       case V2SImode:
 787                         eltype = V2SI_type_node;
 788                         break;
 789                       case V2SFmode:
 790                         eltype = V2SF_type_node;
 791                         break;
 792                         /* 128-bit vectors.  */
 793                       case V16QImode:
 794                         eltype = V16QI_type_node;
 795                         break;
 796                       case V8HImode:
 797                         eltype = V8HI_type_node;
 798                         break;
 799                       case V4SImode:
 800                         eltype = V4SI_type_node;
 801                         break;
 802                       case V4SFmode:
 803                         eltype = V4SF_type_node;
 804                         break;
 805                       case V2DImode:
 806                         eltype = V2DI_type_node;
 807                         break;
 808                       case V2DFmode:
 809                         eltype = V2DF_type_node;
 810                         break;
 811                       default:
 812                         gcc_unreachable ();
 813                       }
 814                   }
 815
 816                 if (k == 0 && !is_store)
 817                   return_type = eltype;
 818                 else
 819                   args = tree_cons (NULL_TREE, eltype, args);
 820               }
 821             ftype = build_function_type (return_type, args);
 822           }
 823           break;
 824
 825         case AARCH64_SIMD_RESULTPAIR:
 826           {
 827             switch (insn_data[d->code].operand[1].mode)
 828               {
 829               case V8QImode:
 830                 ftype = void_ftype_pv8qi_v8qi_v8qi;
 831                 break;
 832               case V4HImode:
 833                 ftype = void_ftype_pv4hi_v4hi_v4hi;
 834                 break;
 835               case V2SImode:
 836                 ftype = void_ftype_pv2si_v2si_v2si;
 837                 break;
 838               case V2SFmode:
 839                 ftype = void_ftype_pv2sf_v2sf_v2sf;
 840                 break;
 841               case DImode:
 842                 ftype = void_ftype_pdi_di_di;
 843                 break;
 844               case V16QImode:
 845                 ftype = void_ftype_pv16qi_v16qi_v16qi;
 846                 break;
 847               case V8HImode:
 848                 ftype = void_ftype_pv8hi_v8hi_v8hi;
 849                 break;
 850               case V4SImode:
 851                 ftype = void_ftype_pv4si_v4si_v4si;
 852                 break;
 853               case V4SFmode:
 854                 ftype = void_ftype_pv4sf_v4sf_v4sf;
 855                 break;
 856               case V2DImode:
 857                 ftype = void_ftype_pv2di_v2di_v2di;
 858                 break;
 859               case V2DFmode:
 860                 ftype = void_ftype_pv2df_v2df_v2df;
 861                 break;
 862               default:
 863                 gcc_unreachable ();
 864               }
 865           }
 866           break;
 867
 868         case AARCH64_SIMD_REINTERP:
 869           {
 870             /* We iterate over 6 doubleword types, then 6 quadword
 871                types.  */
 872             int rhs_d = d->mode % NUM_DREG_TYPES;
 873             int rhs_q = (d->mode - NUM_DREG_TYPES) % NUM_QREG_TYPES;
 874             switch (insn_data[d->code].operand[0].mode)
 875               {
 876               case V8QImode:
 877                 ftype = reinterp_ftype_dreg[0][rhs_d];
 878                 break;
 879               case V4HImode:
 880                 ftype = reinterp_ftype_dreg[1][rhs_d];
 881                 break;
 882               case V2SImode:
 883                 ftype = reinterp_ftype_dreg[2][rhs_d];
 884                 break;
 885               case V2SFmode:
 886                 ftype = reinterp_ftype_dreg[3][rhs_d];
 887                 break;
 888               case DImode:
 889                 ftype = reinterp_ftype_dreg[4][rhs_d];
 890                 break;
 891               case DFmode:
 892                 ftype = reinterp_ftype_dreg[5][rhs_d];
 893                 break;
 894               case V16QImode:
 895                 ftype = reinterp_ftype_qreg[0][rhs_q];
 896                 break;
 897               case V8HImode:
 898                 ftype = reinterp_ftype_qreg[1][rhs_q];
 899                 break;
 900               case V4SImode:
 901                 ftype = reinterp_ftype_qreg[2][rhs_q];
 902                 break;
 903               case V4SFmode:
 904                 ftype = reinterp_ftype_qreg[3][rhs_q];
 905                 break;
 906               case V2DImode:
 907                 ftype = reinterp_ftype_qreg[4][rhs_q];
 908                 break;
 909               case V2DFmode:
 910                 ftype = reinterp_ftype_qreg[5][rhs_q];
 911                 break;
 912               default:
 913                 gcc_unreachable ();
 914               }
 915           }
 916           break;
 917
 918         default:
 919           gcc_unreachable ();
 920         }
 921       gcc_assert (ftype != NULL);
 922
 923       snprintf (namebuf, sizeof (namebuf), "__builtin_aarch64_%s%s",
 924                 d->name, modenames[d->mode]);
 925
 926       fndecl = add_builtin_function (namebuf, ftype, fcode, BUILT_IN_MD,
 927                                      NULL, NULL_TREE);
 928       aarch64_builtin_decls[fcode] = fndecl;
 929     }
 930 }
 931
 932 void
 933 aarch64_init_builtins (void)
 934 {
 935   if (TARGET_SIMD)
 936     aarch64_init_simd_builtins ();
 937 }
 938
 939 tree
 940 aarch64_builtin_decl (unsigned code, bool initialize_p ATTRIBUTE_UNUSED)
 941 {
 942   if (code >= AARCH64_BUILTIN_MAX)
 943     return error_mark_node;
 944
 945   return aarch64_builtin_decls[code];
 946 }
 947
 948 typedef enum
 949 {
 950   SIMD_ARG_COPY_TO_REG,
 951   SIMD_ARG_CONSTANT,
 952   SIMD_ARG_STOP
 953 } builtin_simd_arg;
 954
 955 #define SIMD_MAX_BUILTIN_ARGS 5
 956
 957 static rtx
 958 aarch64_simd_expand_args (rtx target, int icode, int have_retval,
 959                           tree exp, ...)
 960 {
 961   va_list ap;
 962   rtx pat;
 963   tree arg[SIMD_MAX_BUILTIN_ARGS];
 964   rtx op[SIMD_MAX_BUILTIN_ARGS];
 965   enum machine_mode tmode = insn_data[icode].operand[0].mode;
 966   enum machine_mode mode[SIMD_MAX_BUILTIN_ARGS];
 967   int argc = 0;
 968
 969   if (have_retval
 970       && (!target
 971           || GET_MODE (target) != tmode
 972           || !(*insn_data[icode].operand[0].predicate) (target, tmode)))
 973     target = gen_reg_rtx (tmode);
 974
 975   va_start (ap, exp);
 976
 977   for (;;)
 978     {
 979       builtin_simd_arg thisarg = (builtin_simd_arg) va_arg (ap, int);
 980
 981       if (thisarg == SIMD_ARG_STOP)
 982         break;
 983       else
 984         {
 985           arg[argc] = CALL_EXPR_ARG (exp, argc);
 986           op[argc] = expand_normal (arg[argc]);
 987           mode[argc] = insn_data[icode].operand[argc + have_retval].mode;
 988
 989           switch (thisarg)
 990             {
 991             case SIMD_ARG_COPY_TO_REG:
 992               if (POINTER_TYPE_P (TREE_TYPE (arg[argc])))
 993                 op[argc] = convert_memory_address (Pmode, op[argc]);
 994               /*gcc_assert (GET_MODE (op[argc]) == mode[argc]); */
 995               if (!(*insn_data[icode].operand[argc + have_retval].predicate)
 996                   (op[argc], mode[argc]))
 997                 op[argc] = copy_to_mode_reg (mode[argc], op[argc]);
 998               break;
 999
1000             case SIMD_ARG_CONSTANT:
1001               if (!(*insn_data[icode].operand[argc + have_retval].predicate)
1002                   (op[argc], mode[argc]))
1003                 error_at (EXPR_LOCATION (exp), "incompatible type for argument %d, "
1004                        "expected %<const int%>", argc + 1);
1005               break;
1006
1007             case SIMD_ARG_STOP:
1008               gcc_unreachable ();
1009             }
1010
1011           argc++;
1012         }
1013     }
1014
1015   va_end (ap);
1016
1017   if (have_retval)
1018     switch (argc)
1019       {
1020       case 1:
1021         pat = GEN_FCN (icode) (target, op[0]);
1022         break;
1023
1024       case 2:
1025         pat = GEN_FCN (icode) (target, op[0], op[1]);
1026         break;
1027
1028       case 3:
1029         pat = GEN_FCN (icode) (target, op[0], op[1], op[2]);
1030         break;
1031
1032       case 4:
1033         pat = GEN_FCN (icode) (target, op[0], op[1], op[2], op[3]);
1034         break;
1035
1036       case 5:
1037         pat = GEN_FCN (icode) (target, op[0], op[1], op[2], op[3], op[4]);
1038         break;
1039
1040       default:
1041         gcc_unreachable ();
1042       }
1043   else
1044     switch (argc)
1045       {
1046       case 1:
1047         pat = GEN_FCN (icode) (op[0]);
1048         break;
1049
1050       case 2:
1051         pat = GEN_FCN (icode) (op[0], op[1]);
1052         break;
1053
1054       case 3:
1055         pat = GEN_FCN (icode) (op[0], op[1], op[2]);
1056         break;
1057
1058       case 4:
1059         pat = GEN_FCN (icode) (op[0], op[1], op[2], op[3]);
1060         break;
1061
1062       case 5:
1063         pat = GEN_FCN (icode) (op[0], op[1], op[2], op[3], op[4]);
1064         break;
1065
1066       default:
1067         gcc_unreachable ();
1068       }
1069
1070   if (!pat)
1071     return 0;
1072
1073   emit_insn (pat);
1074
1075   return target;
1076 }
1077
1078 /* Expand an AArch64 AdvSIMD builtin(intrinsic).  */
1079 rtx
1080 aarch64_simd_expand_builtin (int fcode, tree exp, rtx target)
1081 {
1082   aarch64_simd_builtin_datum *d =
1083                 &aarch64_simd_builtin_data[fcode - (AARCH64_SIMD_BUILTIN_BASE + 1)];
1084   aarch64_simd_itype itype = d->itype;
1085   enum insn_code icode = d->code;
1086
1087   switch (itype)
1088     {
1089     case AARCH64_SIMD_UNOP:
1090       return aarch64_simd_expand_args (target, icode, 1, exp,
1091                                        SIMD_ARG_COPY_TO_REG,
1092                                        SIMD_ARG_STOP);
1093
1094     case AARCH64_SIMD_BINOP:
1095       {
1096         rtx arg2 = expand_normal (CALL_EXPR_ARG (exp, 1));
1097         /* Handle constants only if the predicate allows it.  */
1098         bool op1_const_int_p =
1099           (CONST_INT_P (arg2)
1100            && (*insn_data[icode].operand[2].predicate)
1101                 (arg2, insn_data[icode].operand[2].mode));
1102         return aarch64_simd_expand_args
1103           (target, icode, 1, exp,
1104            SIMD_ARG_COPY_TO_REG,
1105            op1_const_int_p ? SIMD_ARG_CONSTANT : SIMD_ARG_COPY_TO_REG,
1106            SIMD_ARG_STOP);
1107       }
1108
1109     case AARCH64_SIMD_TERNOP:
1110       return aarch64_simd_expand_args (target, icode, 1, exp,
1111                                        SIMD_ARG_COPY_TO_REG,
1112                                        SIMD_ARG_COPY_TO_REG,
1113                                        SIMD_ARG_COPY_TO_REG,
1114                                        SIMD_ARG_STOP);
1115
1116     case AARCH64_SIMD_QUADOP:
1117       return aarch64_simd_expand_args (target, icode, 1, exp,
1118                                        SIMD_ARG_COPY_TO_REG,
1119                                        SIMD_ARG_COPY_TO_REG,
1120                                        SIMD_ARG_COPY_TO_REG,
1121                                        SIMD_ARG_COPY_TO_REG,
1122                                        SIMD_ARG_STOP);
1123     case AARCH64_SIMD_LOAD1:
1124     case AARCH64_SIMD_LOADSTRUCT:
1125       return aarch64_simd_expand_args (target, icode, 1, exp,
1126                                        SIMD_ARG_COPY_TO_REG, SIMD_ARG_STOP);
1127
1128     case AARCH64_SIMD_STORE1:
1129     case AARCH64_SIMD_STORESTRUCT:
1130       return aarch64_simd_expand_args (target, icode, 0, exp,
1131                                        SIMD_ARG_COPY_TO_REG,
1132                                        SIMD_ARG_COPY_TO_REG, SIMD_ARG_STOP);
1133
1134     case AARCH64_SIMD_REINTERP:
1135       return aarch64_simd_expand_args (target, icode, 1, exp,
1136                                        SIMD_ARG_COPY_TO_REG, SIMD_ARG_STOP);
1137
1138     case AARCH64_SIMD_CREATE:
1139       return aarch64_simd_expand_args (target, icode, 1, exp,
1140                                        SIMD_ARG_COPY_TO_REG, SIMD_ARG_STOP);
1141
1142     case AARCH64_SIMD_COMBINE:
1143       return aarch64_simd_expand_args (target, icode, 1, exp,
1144                                        SIMD_ARG_COPY_TO_REG,
1145                                        SIMD_ARG_COPY_TO_REG, SIMD_ARG_STOP);
1146
1147     case AARCH64_SIMD_GETLANE:
1148       return aarch64_simd_expand_args (target, icode, 1, exp,
1149                                        SIMD_ARG_COPY_TO_REG,
1150                                        SIMD_ARG_CONSTANT,
1151                                        SIMD_ARG_STOP);
1152
1153     case AARCH64_SIMD_SETLANE:
1154       return aarch64_simd_expand_args (target, icode, 1, exp,
1155                                        SIMD_ARG_COPY_TO_REG,
1156                                        SIMD_ARG_COPY_TO_REG,
1157                                        SIMD_ARG_CONSTANT,
1158                                        SIMD_ARG_STOP);
1159
1160     case AARCH64_SIMD_SHIFTIMM:
1161       return aarch64_simd_expand_args (target, icode, 1, exp,
1162                                        SIMD_ARG_COPY_TO_REG,
1163                                        SIMD_ARG_CONSTANT,
1164                                        SIMD_ARG_STOP);
1165
1166     case AARCH64_SIMD_SHIFTACC:
1167     case AARCH64_SIMD_SHIFTINSERT:
1168       return aarch64_simd_expand_args (target, icode, 1, exp,
1169                                        SIMD_ARG_COPY_TO_REG,
1170                                        SIMD_ARG_COPY_TO_REG,
1171                                        SIMD_ARG_CONSTANT,
1172                                        SIMD_ARG_STOP);
1173
1174     default:
1175       gcc_unreachable ();
1176     }
1177 }
1178
1179 /* Expand an expression EXP that calls a built-in function,
1180    with result going to TARGET if that's convenient.  */
1181 rtx
1182 aarch64_expand_builtin (tree exp,
1183                      rtx target,
1184                      rtx subtarget ATTRIBUTE_UNUSED,
1185                      enum machine_mode mode ATTRIBUTE_UNUSED,
1186                      int ignore ATTRIBUTE_UNUSED)
1187 {
1188   tree fndecl = TREE_OPERAND (CALL_EXPR_FN (exp), 0);
1189   int fcode = DECL_FUNCTION_CODE (fndecl);
1190
1191   if (fcode >= AARCH64_SIMD_BUILTIN_BASE)
1192     return aarch64_simd_expand_builtin (fcode, exp, target);
1193
1194   return NULL_RTX;
1195 }
1196
1197 tree
1198 aarch64_builtin_vectorized_function (tree fndecl, tree type_out, tree type_in)
1199 {
1200   enum machine_mode in_mode, out_mode;
1201   int in_n, out_n;
1202
1203   if (TREE_CODE (type_out) != VECTOR_TYPE
1204       || TREE_CODE (type_in) != VECTOR_TYPE)
1205     return NULL_TREE;
1206
1207   out_mode = TYPE_MODE (TREE_TYPE (type_out));
1208   out_n = TYPE_VECTOR_SUBPARTS (type_out);
1209   in_mode = TYPE_MODE (TREE_TYPE (type_in));
1210   in_n = TYPE_VECTOR_SUBPARTS (type_in);
1211
1212 #undef AARCH64_CHECK_BUILTIN_MODE
1213 #define AARCH64_CHECK_BUILTIN_MODE(C, N) 1
1214 #define AARCH64_FIND_FRINT_VARIANT(N) \
1215   (AARCH64_CHECK_BUILTIN_MODE (2, D) \
1216     ? aarch64_builtin_decls[AARCH64_SIMD_BUILTIN_##N##v2df] \
1217     : (AARCH64_CHECK_BUILTIN_MODE (4, S) \
1218         ? aarch64_builtin_decls[AARCH64_SIMD_BUILTIN_##N##v4sf] \
1219         : (AARCH64_CHECK_BUILTIN_MODE (2, S) \
1220            ? aarch64_builtin_decls[AARCH64_SIMD_BUILTIN_##N##v2sf] \
1221            : NULL_TREE)))
1222   if (DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL)
1223     {
1224       enum built_in_function fn = DECL_FUNCTION_CODE (fndecl);
1225       switch (fn)
1226         {
1227 #undef AARCH64_CHECK_BUILTIN_MODE
1228 #define AARCH64_CHECK_BUILTIN_MODE(C, N) \
1229   (out_mode == N##Fmode && out_n == C \
1230    && in_mode == N##Fmode && in_n == C)
1231         case BUILT_IN_FLOOR:
1232         case BUILT_IN_FLOORF:
1233           return AARCH64_FIND_FRINT_VARIANT (floor);
1234         case BUILT_IN_CEIL:
1235         case BUILT_IN_CEILF:
1236           return AARCH64_FIND_FRINT_VARIANT (ceil);
1237         case BUILT_IN_TRUNC:
1238         case BUILT_IN_TRUNCF:
1239           return AARCH64_FIND_FRINT_VARIANT (btrunc);
1240         case BUILT_IN_ROUND:
1241         case BUILT_IN_ROUNDF:
1242           return AARCH64_FIND_FRINT_VARIANT (round);
1243         case BUILT_IN_NEARBYINT:
1244         case BUILT_IN_NEARBYINTF:
1245           return AARCH64_FIND_FRINT_VARIANT (nearbyint);
1246         case BUILT_IN_SQRT:
1247         case BUILT_IN_SQRTF:
1248           return AARCH64_FIND_FRINT_VARIANT (sqrt);
1249 #undef AARCH64_CHECK_BUILTIN_MODE
1250 #define AARCH64_CHECK_BUILTIN_MODE(C, N) \
1251   (out_mode == SImode && out_n == C \
1252    && in_mode == N##Imode && in_n == C)
1253         case BUILT_IN_CLZ:
1254           {
1255             if (AARCH64_CHECK_BUILTIN_MODE (4, S))
1256               return aarch64_builtin_decls[AARCH64_SIMD_BUILTIN_clzv4si];
1257             return NULL_TREE;
1258           }
1259 #undef AARCH64_CHECK_BUILTIN_MODE
1260 #define AARCH64_CHECK_BUILTIN_MODE(C, N) \
1261   (out_mode == N##Imode && out_n == C \
1262    && in_mode == N##Fmode && in_n == C)
1263         case BUILT_IN_LFLOOR:
1264         case BUILT_IN_IFLOORF:
1265           {
1266             tree new_tree = NULL_TREE;
1267             if (AARCH64_CHECK_BUILTIN_MODE (2, D))
1268               new_tree =
1269                 aarch64_builtin_decls[AARCH64_SIMD_BUILTIN_lfloorv2dfv2di];
1270             else if (AARCH64_CHECK_BUILTIN_MODE (4, S))
1271               new_tree =
1272                 aarch64_builtin_decls[AARCH64_SIMD_BUILTIN_lfloorv4sfv4si];
1273             else if (AARCH64_CHECK_BUILTIN_MODE (2, S))
1274               new_tree =
1275                 aarch64_builtin_decls[AARCH64_SIMD_BUILTIN_lfloorv2sfv2si];
1276             return new_tree;
1277           }
1278         case BUILT_IN_LCEIL:
1279         case BUILT_IN_ICEILF:
1280           {
1281             tree new_tree = NULL_TREE;
1282             if (AARCH64_CHECK_BUILTIN_MODE (2, D))
1283               new_tree =
1284                 aarch64_builtin_decls[AARCH64_SIMD_BUILTIN_lceilv2dfv2di];
1285             else if (AARCH64_CHECK_BUILTIN_MODE (4, S))
1286               new_tree =
1287                 aarch64_builtin_decls[AARCH64_SIMD_BUILTIN_lceilv4sfv4si];
1288             else if (AARCH64_CHECK_BUILTIN_MODE (2, S))
1289               new_tree =
1290                 aarch64_builtin_decls[AARCH64_SIMD_BUILTIN_lceilv2sfv2si];
1291             return new_tree;
1292           }
1293         case BUILT_IN_LROUND:
1294         case BUILT_IN_IROUNDF:
1295           {
1296             tree new_tree = NULL_TREE;
1297             if (AARCH64_CHECK_BUILTIN_MODE (2, D))
1298               new_tree =
1299                 aarch64_builtin_decls[AARCH64_SIMD_BUILTIN_lroundv2dfv2di];
1300             else if (AARCH64_CHECK_BUILTIN_MODE (4, S))
1301               new_tree =
1302                 aarch64_builtin_decls[AARCH64_SIMD_BUILTIN_lroundv4sfv4si];
1303             else if (AARCH64_CHECK_BUILTIN_MODE (2, S))
1304               new_tree =
1305                 aarch64_builtin_decls[AARCH64_SIMD_BUILTIN_lroundv2sfv2si];
1306             return new_tree;
1307           }
1308
1309         default:
1310           return NULL_TREE;
1311       }
1312     }
1313
1314   return NULL_TREE;
1315 }
1316
1317 #undef VAR1
1318 #define VAR1(T, N, MAP, A) \
1319   case AARCH64_SIMD_BUILTIN_##N##A:
1320
1321 tree
1322 aarch64_fold_builtin (tree fndecl, int n_args ATTRIBUTE_UNUSED, tree *args,
1323                       bool ignore ATTRIBUTE_UNUSED)
1324 {
1325   int fcode = DECL_FUNCTION_CODE (fndecl);
1326   tree type = TREE_TYPE (TREE_TYPE (fndecl));
1327
1328   switch (fcode)
1329     {
1330       BUILTIN_VALLDI (UNOP, abs, 2)
1331         return fold_build1 (ABS_EXPR, type, args[0]);
1332         break;
1333       BUILTIN_VALLDI (BINOP, cmge, 0)
1334         return fold_build2 (GE_EXPR, type, args[0], args[1]);
1335         break;
1336       BUILTIN_VALLDI (BINOP, cmgt, 0)
1337         return fold_build2 (GT_EXPR, type, args[0], args[1]);
1338         break;
1339       BUILTIN_VALLDI (BINOP, cmeq, 0)
1340         return fold_build2 (EQ_EXPR, type, args[0], args[1]);
1341         break;
1342       BUILTIN_VSDQ_I_DI (BINOP, cmtst, 0)
1343         {
1344           tree and_node = fold_build2 (BIT_AND_EXPR, type, args[0], args[1]);
1345           tree vec_zero_node = build_zero_cst (type);
1346           return fold_build2 (NE_EXPR, type, and_node, vec_zero_node);
1347           break;
1348         }
1349       VAR1 (UNOP, floatv2si, 2, v2sf)
1350       VAR1 (UNOP, floatv4si, 2, v4sf)
1351       VAR1 (UNOP, floatv2di, 2, v2df)
1352         return fold_build1 (FLOAT_EXPR, type, args[0]);
1353       default:
1354         break;
1355     }
1356
1357   return NULL_TREE;
1358 }
1359
1360 bool
1361 aarch64_gimple_fold_builtin (gimple_stmt_iterator *gsi)
1362 {
1363   bool changed = false;
1364   gimple stmt = gsi_stmt (*gsi);
1365   tree call = gimple_call_fn (stmt);
1366   tree fndecl;
1367   gimple new_stmt = NULL;
1368   if (call)
1369     {
1370       fndecl = gimple_call_fndecl (stmt);
1371       if (fndecl)
1372         {
1373           int fcode = DECL_FUNCTION_CODE (fndecl);
1374           int nargs = gimple_call_num_args (stmt);
1375           tree *args = (nargs > 0
1376                         ? gimple_call_arg_ptr (stmt, 0)
1377                         : &error_mark_node);
1378
1379           switch (fcode)
1380             {
1381               BUILTIN_VALL (UNOP, reduc_splus_, 10)
1382                 new_stmt = gimple_build_assign_with_ops (
1383                                                 REDUC_PLUS_EXPR,
1384                                                 gimple_call_lhs (stmt),
1385                                                 args[0],
1386                                                 NULL_TREE);
1387                 break;
1388               BUILTIN_VDQIF (UNOP, reduc_smax_, 10)
1389                 new_stmt = gimple_build_assign_with_ops (
1390                                                 REDUC_MAX_EXPR,
1391                                                 gimple_call_lhs (stmt),
1392                                                 args[0],
1393                                                 NULL_TREE);
1394                 break;
1395               BUILTIN_VDQIF (UNOP, reduc_smin_, 10)
1396                 new_stmt = gimple_build_assign_with_ops (
1397                                                 REDUC_MIN_EXPR,
1398                                                 gimple_call_lhs (stmt),
1399                                                 args[0],
1400                                                 NULL_TREE);
1401                 break;
1402
1403             default:
1404               break;
1405             }
1406         }
1407     }
1408
1409   if (new_stmt)
1410     {
1411       gsi_replace (gsi, new_stmt, true);
1412       changed = true;
1413     }
1414
1415   return changed;
1416 }
1417
1418 #undef AARCH64_CHECK_BUILTIN_MODE
1419 #undef AARCH64_FIND_FRINT_VARIANT
1420 #undef BUILTIN_DX
1421 #undef BUILTIN_SDQ_I
1422 #undef BUILTIN_SD_HSI
1423 #undef BUILTIN_V2F
1424 #undef BUILTIN_VALL
1425 #undef BUILTIN_VB
1426 #undef BUILTIN_VD
1427 #undef BUILTIN_VDC
1428 #undef BUILTIN_VDIC
1429 #undef BUILTIN_VDN
1430 #undef BUILTIN_VDQ
1431 #undef BUILTIN_VDQF
1432 #undef BUILTIN_VDQH
1433 #undef BUILTIN_VDQHS
1434 #undef BUILTIN_VDQIF
1435 #undef BUILTIN_VDQM
1436 #undef BUILTIN_VDQV
1437 #undef BUILTIN_VDQ_BHSI
1438 #undef BUILTIN_VDQ_I
1439 #undef BUILTIN_VDW
1440 #undef BUILTIN_VD_BHSI
1441 #undef BUILTIN_VD_HSI
1442 #undef BUILTIN_VD_RE
1443 #undef BUILTIN_VQ
1444 #undef BUILTIN_VQN
1445 #undef BUILTIN_VQW
1446 #undef BUILTIN_VQ_HSI
1447 #undef BUILTIN_VQ_S
1448 #undef BUILTIN_VSDQ_HSI
1449 #undef BUILTIN_VSDQ_I
1450 #undef BUILTIN_VSDQ_I_BHSI
1451 #undef BUILTIN_VSDQ_I_DI
1452 #undef BUILTIN_VSD_HSI
1453 #undef BUILTIN_VSQN_HSDI
1454 #undef BUILTIN_VSTRUCT
1455 #undef CF0
1456 #undef CF1
1457 #undef CF2
1458 #undef CF3
1459 #undef CF4
1460 #undef CF10
1461 #undef VAR1
1462 #undef VAR2
1463 #undef VAR3
1464 #undef VAR4
1465 #undef VAR5
1466 #undef VAR6
1467 #undef VAR7
1468 #undef VAR8
1469 #undef VAR9
1470 #undef VAR10
1471 #undef VAR11
1472