gcc/config/aarch64/aarch64-sve-builtins.cc

   1 /* ACLE support for AArch64 SVE
   2    Copyright (C) 2018-2024 Free Software Foundation, Inc.
   3
   4    This file is part of GCC.
   5
   6    GCC is free software; you can redistribute it and/or modify it
   7    under the terms of the GNU General Public License as published by
   8    the Free Software Foundation; either version 3, or (at your option)
   9    any later version.
  10
  11    GCC is distributed in the hope that it will be useful, but
  12    WITHOUT ANY WARRANTY; without even the implied warranty of
  13    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14    General Public License for more details.
  15
  16    You should have received a copy of the GNU General Public License
  17    along with GCC; see the file COPYING3.  If not see
  18    <http://www.gnu.org/licenses/>.  */
  19
  20 #define IN_TARGET_CODE 1
  21
  22 #include "config.h"
  23 #include "system.h"
  24 #include "coretypes.h"
  25 #include "tm.h"
  26 #include "tree.h"
  27 #include "rtl.h"
  28 #include "tm_p.h"
  29 #include "memmodel.h"
  30 #include "insn-codes.h"
  31 #include "optabs.h"
  32 #include "recog.h"
  33 #include "diagnostic.h"
  34 #include "expr.h"
  35 #include "basic-block.h"
  36 #include "function.h"
  37 #include "fold-const.h"
  38 #include "gimple.h"
  39 #include "gimple-iterator.h"
  40 #include "gimplify.h"
  41 #include "explow.h"
  42 #include "emit-rtl.h"
  43 #include "tree-vector-builder.h"
  44 #include "stor-layout.h"
  45 #include "regs.h"
  46 #include "alias.h"
  47 #include "gimple-fold.h"
  48 #include "langhooks.h"
  49 #include "stringpool.h"
  50 #include "attribs.h"
  51 #include "aarch64-sve-builtins.h"
  52 #include "aarch64-sve-builtins-base.h"
  53 #include "aarch64-sve-builtins-sve2.h"
  54 #include "aarch64-sve-builtins-sme.h"
  55 #include "aarch64-sve-builtins-shapes.h"
  56 #include "aarch64-builtins.h"
  57
  58 namespace aarch64_sve {
  59
  60 /* Static information about each single-predicate or single-vector
  61    ABI and ACLE type.  */
  62 struct vector_type_info
  63 {
  64   /* The name of the type as declared by arm_sve.h.  */
  65   const char *acle_name;
  66
  67   /* The name of the type specified in AAPCS64.  The type is always
  68      available under this name, even when arm_sve.h isn't included.  */
  69   const char *abi_name;
  70
  71   /* The C++ mangling of ABI_NAME.  */
  72   const char *mangled_name;
  73 };
  74
  75 /* Describes a function decl.  */
  76 class GTY(()) registered_function
  77 {
  78 public:
  79   /* The ACLE function that the decl represents.  */
  80   function_instance instance GTY ((skip));
  81
  82   /* The decl itself.  */
  83   tree decl;
  84
  85   /* The architecture extensions that the function requires, as a set of
  86      AARCH64_FL_* flags.  */
  87   aarch64_feature_flags required_extensions;
  88
  89   /* True if the decl represents an overloaded function that needs to be
  90      resolved by function_resolver.  */
  91   bool overloaded_p;
  92 };
  93
  94 /* Hash traits for registered_function.  */
  95 struct registered_function_hasher : nofree_ptr_hash <registered_function>
  96 {
  97   typedef function_instance compare_type;
  98
  99   static hashval_t hash (value_type);
 100   static bool equal (value_type, const compare_type &);
 101 };
 102
 103 /* Information about each single-predicate or single-vector type.  */
 104 static CONSTEXPR const vector_type_info vector_types[] = {
 105 #define DEF_SVE_TYPE(ACLE_NAME, NCHARS, ABI_NAME, SCALAR_TYPE) \
 106   { #ACLE_NAME, #ABI_NAME, "u" #NCHARS #ABI_NAME },
 107 #include "aarch64-sve-builtins.def"
 108 };
 109
 110 /* The function name suffix associated with each predication type.  */
 111 static const char *const pred_suffixes[NUM_PREDS + 1] = {
 112   "",
 113   "",
 114   "_m",
 115   "_x",
 116   "_z",
 117   "_m",
 118   ""
 119 };
 120
 121 /* Static information about each mode_suffix_index.  */
 122 CONSTEXPR const mode_suffix_info mode_suffixes[] = {
 123 #define VECTOR_TYPE_none NUM_VECTOR_TYPES
 124 #define DEF_SVE_MODE(NAME, BASE, DISPLACEMENT, UNITS) \
 125   { "_" #NAME, VECTOR_TYPE_##BASE, VECTOR_TYPE_##DISPLACEMENT, UNITS_##UNITS },
 126 #include "aarch64-sve-builtins.def"
 127 #undef VECTOR_TYPE_none
 128   { "", NUM_VECTOR_TYPES, NUM_VECTOR_TYPES, UNITS_none }
 129 };
 130
 131 /* Static information about each type_suffix_index.  */
 132 CONSTEXPR const type_suffix_info type_suffixes[NUM_TYPE_SUFFIXES + 1] = {
 133 #define DEF_SVE_NEON_TYPE_SUFFIX(NAME, ACLE_TYPE, CLASS, BITS, MODE, \
 134                                  NEON64, NEON128) \
 135   { "_" #NAME, \
 136     VECTOR_TYPE_##ACLE_TYPE, \
 137     TYPE_##CLASS, \
 138     BITS, \
 139     BITS / BITS_PER_UNIT, \
 140     TYPE_##CLASS == TYPE_signed || TYPE_##CLASS == TYPE_unsigned, \
 141     TYPE_##CLASS == TYPE_unsigned, \
 142     TYPE_##CLASS == TYPE_float, \
 143     TYPE_##CLASS != TYPE_bool, \
 144     TYPE_##CLASS == TYPE_bool, \
 145     false, \
 146     0, \
 147     MODE, \
 148     NEON64, \
 149     NEON128 },
 150 #define DEF_SVE_TYPE_SUFFIX(NAME, ACLE_TYPE, CLASS, BITS, MODE) \
 151   DEF_SVE_NEON_TYPE_SUFFIX (NAME, ACLE_TYPE, CLASS, BITS, MODE, \
 152                             ARM_NEON_H_TYPES_LAST, ARM_NEON_H_TYPES_LAST)
 153 #define DEF_SME_ZA_SUFFIX(NAME, BITS, MODE) \
 154   { "_" #NAME, \
 155     NUM_VECTOR_TYPES, \
 156     NUM_TYPE_CLASSES, \
 157     BITS, \
 158     BITS / BITS_PER_UNIT, \
 159     false, \
 160     false, \
 161     false, \
 162     false, \
 163     false, \
 164     true, \
 165     0, \
 166     MODE, \
 167     ARM_NEON_H_TYPES_LAST, \
 168     ARM_NEON_H_TYPES_LAST },
 169 #include "aarch64-sve-builtins.def"
 170   { "", NUM_VECTOR_TYPES, TYPE_bool, 0, 0, false, false, false, false,
 171     false, false, 0, VOIDmode, ARM_NEON_H_TYPES_LAST, ARM_NEON_H_TYPES_LAST }
 172 };
 173
 174 CONSTEXPR const group_suffix_info group_suffixes[] = {
 175 #define DEF_SVE_GROUP_SUFFIX(NAME, VG, VECTORS_PER_TUPLE) \
 176   { "_" #NAME, VG, VECTORS_PER_TUPLE },
 177 #include "aarch64-sve-builtins.def"
 178   { "", 0, 1 }
 179 };
 180
 181 /* Define a TYPES_<combination> macro for each combination of type
 182    suffixes that an ACLE function can have, where <combination> is the
 183    name used in DEF_SVE_FUNCTION entries.
 184
 185    Use S (T) for single type suffix T and D (T1, T2) for a pair of type
 186    suffixes T1 and T2.  Use commas to separate the suffixes.
 187
 188    Although the order shouldn't matter, the convention is to sort the
 189    suffixes lexicographically after dividing suffixes into a type
 190    class ("b", "f", etc.) and a numerical bit count.  */
 191
 192 /* _b8 _b16 _b32 _b64.  */
 193 #define TYPES_all_pred(S, D) \
 194   S (b8), S (b16), S (b32), S (b64)
 195
 196 /* _c8 _c16 _c32 _c64.  */
 197 #define TYPES_all_count(S, D) \
 198   S (c8), S (c16), S (c32), S (c64)
 199
 200 /* _b8 _b16 _b32 _b64
 201    _c8 _c16 _c32 _c64.  */
 202 #define TYPES_all_pred_count(S, D) \
 203   TYPES_all_pred (S, D), \
 204   TYPES_all_count (S, D)
 205
 206 /* _f16 _f32 _f64.  */
 207 #define TYPES_all_float(S, D) \
 208   S (f16), S (f32), S (f64)
 209
 210 /* _s8 _s16 _s32 _s64.  */
 211 #define TYPES_all_signed(S, D) \
 212   S (s8), S (s16), S (s32), S (s64)
 213
 214 /*     _f16 _f32 _f64
 215    _s8 _s16 _s32 _s64.  */
 216 #define TYPES_all_float_and_signed(S, D) \
 217   TYPES_all_float (S, D), TYPES_all_signed (S, D)
 218
 219 /* _u8 _u16 _u32 _u64.  */
 220 #define TYPES_all_unsigned(S, D) \
 221   S (u8), S (u16), S (u32), S (u64)
 222
 223 /* _s8 _s16 _s32 _s64
 224    _u8 _u16 _u32 _u64.  */
 225 #define TYPES_all_integer(S, D) \
 226   TYPES_all_signed (S, D), TYPES_all_unsigned (S, D)
 227
 228 /*     _f16 _f32 _f64
 229    _s8 _s16 _s32 _s64
 230    _u8 _u16 _u32 _u64.  */
 231 #define TYPES_all_arith(S, D) \
 232   TYPES_all_float (S, D), TYPES_all_integer (S, D)
 233
 234 /*     _bf16
 235         _f16 _f32 _f64
 236    _s8  _s16 _s32 _s64
 237    _u8  _u16 _u32 _u64.  */
 238 #define TYPES_all_data(S, D) \
 239   S (bf16), TYPES_all_arith (S, D)
 240
 241 /* _b only.  */
 242 #define TYPES_b(S, D) \
 243   S (b)
 244
 245 /* _c only.  */
 246 #define TYPES_c(S, D) \
 247   S (c)
 248
 249 /* _u8.  */
 250 #define TYPES_b_unsigned(S, D) \
 251   S (u8)
 252
 253 /* _s8
 254    _u8.  */
 255 #define TYPES_b_integer(S, D) \
 256   S (s8), TYPES_b_unsigned (S, D)
 257
 258 /* _s8 _s16
 259    _u8 _u16.  */
 260 #define TYPES_bh_integer(S, D) \
 261   S (s8), S (s16), S (u8), S (u16)
 262
 263 /* _u8 _u32.  */
 264 #define TYPES_bs_unsigned(S, D) \
 265   S (u8), S (u32)
 266
 267 /* _s8 _s16 _s32.  */
 268 #define TYPES_bhs_signed(S, D) \
 269   S (s8), S (s16), S (s32)
 270
 271 /* _u8 _u16 _u32.  */
 272 #define TYPES_bhs_unsigned(S, D) \
 273   S (u8), S (u16), S (u32)
 274
 275 /* _s8 _s16 _s32
 276    _u8 _u16 _u32.  */
 277 #define TYPES_bhs_integer(S, D) \
 278   TYPES_bhs_signed (S, D), TYPES_bhs_unsigned (S, D)
 279
 280 /*      _bf16
 281          _f16  _f32
 282     _s8  _s16  _s32
 283     _u8  _u16  _u32.  */
 284 #define TYPES_bhs_data(S, D) \
 285   S (bf16), S (f16), S (f32), TYPES_bhs_integer (S, D)
 286
 287 /* _s16_s8  _s32_s16  _s64_s32
 288    _u16_u8  _u32_u16  _u64_u32.  */
 289 #define TYPES_bhs_widen(S, D) \
 290   D (s16, s8), D (s32, s16), D (s64, s32), \
 291   D (u16, u8), D (u32, u16), D (u64, u32)
 292
 293 /* _s16
 294    _u16.  */
 295 #define TYPES_h_integer(S, D) \
 296   S (s16), S (u16)
 297
 298 /* _s16 _s32.  */
 299 #define TYPES_hs_signed(S, D) \
 300   S (s16), S (s32)
 301
 302 /* _s16 _s32
 303    _u16 _u32.  */
 304 #define TYPES_hs_integer(S, D) \
 305   TYPES_hs_signed (S, D), S (u16), S (u32)
 306
 307 /* _f16 _f32.  */
 308 #define TYPES_hs_float(S, D) \
 309   S (f16), S (f32)
 310
 311 /* _bf16
 312     _f16  _f32
 313     _s16  _s32
 314     _u16  _u32.  */
 315 #define TYPES_hs_data(S, D) \
 316   S (bf16), S (f16), S (f32), TYPES_hs_integer (S, D)
 317
 318 /* _u16 _u64.  */
 319 #define TYPES_hd_unsigned(S, D) \
 320   S (u16), S (u64)
 321
 322 /* _s16 _s32 _s64.  */
 323 #define TYPES_hsd_signed(S, D) \
 324   S (s16), S (s32), S (s64)
 325
 326 /* _s16 _s32 _s64
 327    _u16 _u32 _u64.  */
 328 #define TYPES_hsd_integer(S, D) \
 329   TYPES_hsd_signed (S, D), S (u16), S (u32), S (u64)
 330
 331 /* _f32.  */
 332 #define TYPES_s_float(S, D) \
 333   S (f32)
 334
 335 /*      _f32
 336    _s16 _s32 _s64
 337    _u16 _u32 _u64.  */
 338 #define TYPES_s_float_hsd_integer(S, D) \
 339   TYPES_s_float (S, D), TYPES_hsd_integer (S, D)
 340
 341 /* _f32
 342    _s32 _s64
 343    _u32 _u64.  */
 344 #define TYPES_s_float_sd_integer(S, D) \
 345   TYPES_s_float (S, D), TYPES_sd_integer (S, D)
 346
 347 /* _s32.  */
 348 #define TYPES_s_signed(S, D) \
 349   S (s32)
 350
 351 /* _u32.  */
 352 #define TYPES_s_unsigned(S, D) \
 353   S (u32)
 354
 355 /* _s32 _u32.  */
 356 #define TYPES_s_integer(S, D) \
 357   TYPES_s_signed (S, D), TYPES_s_unsigned (S, D)
 358
 359 /* _s32 _s64.  */
 360 #define TYPES_sd_signed(S, D) \
 361   S (s32), S (s64)
 362
 363 /* _u32 _u64.  */
 364 #define TYPES_sd_unsigned(S, D) \
 365   S (u32), S (u64)
 366
 367 /* _s32 _s64
 368    _u32 _u64.  */
 369 #define TYPES_sd_integer(S, D) \
 370   TYPES_sd_signed (S, D), TYPES_sd_unsigned (S, D)
 371
 372 /* _f32 _f64
 373    _s32 _s64
 374    _u32 _u64.  */
 375 #define TYPES_sd_data(S, D) \
 376   S (f32), S (f64), TYPES_sd_integer (S, D)
 377
 378 /* _f16 _f32 _f64
 379         _s32 _s64
 380         _u32 _u64.  */
 381 #define TYPES_all_float_and_sd_integer(S, D) \
 382   TYPES_all_float (S, D), TYPES_sd_integer (S, D)
 383
 384 /* _f64.  */
 385 #define TYPES_d_float(S, D) \
 386   S (f64)
 387
 388 /* _u64.  */
 389 #define TYPES_d_unsigned(S, D) \
 390   S (u64)
 391
 392 /* _s64
 393    _u64.  */
 394 #define TYPES_d_integer(S, D) \
 395   S (s64), TYPES_d_unsigned (S, D)
 396
 397 /* _f64
 398    _s64
 399    _u64.  */
 400 #define TYPES_d_data(S, D) \
 401   TYPES_d_float (S, D), TYPES_d_integer (S, D)
 402
 403 /* All the type combinations allowed by svcvt.  */
 404 #define TYPES_cvt(S, D) \
 405   D (f16, f32), D (f16, f64), \
 406   D (f16, s16), D (f16, s32), D (f16, s64), \
 407   D (f16, u16), D (f16, u32), D (f16, u64), \
 408   \
 409   D (f32, f16), D (f32, f64), \
 410   D (f32, s32), D (f32, s64), \
 411   D (f32, u32), D (f32, u64), \
 412   \
 413   D (f64, f16), D (f64, f32), \
 414   D (f64, s32), D (f64, s64), \
 415   D (f64, u32), D (f64, u64), \
 416   \
 417   D (s16, f16), \
 418   D (s32, f16), D (s32, f32), D (s32, f64), \
 419   D (s64, f16), D (s64, f32), D (s64, f64), \
 420   \
 421   D (u16, f16), \
 422   D (u32, f16), D (u32, f32), D (u32, f64), \
 423   D (u64, f16), D (u64, f32), D (u64, f64)
 424
 425 /* _bf16_f32.  */
 426 #define TYPES_cvt_bfloat(S, D) \
 427   D (bf16, f32)
 428
 429 /* { _bf16 _f16 } x _f32.  */
 430 #define TYPES_cvt_h_s_float(S, D) \
 431   D (bf16, f32), D (f16, f32)
 432
 433 /* _f32_f16
 434    _f64_f32.  */
 435 #define TYPES_cvt_long(S, D) \
 436   D (f32, f16), D (f64, f32)
 437
 438 /* _f16_f32.  */
 439 #define TYPES_cvt_narrow_s(S, D) \
 440   D (f32, f64)
 441
 442 /* _f16_f32
 443    _f32_f64.  */
 444 #define TYPES_cvt_narrow(S, D) \
 445   D (f16, f32), TYPES_cvt_narrow_s (S, D)
 446
 447 /* { _s32 _u32 } x _f32
 448
 449    _f32 x { _s32 _u32 }.  */
 450 #define TYPES_cvt_s_s(S, D) \
 451   D (s32, f32), \
 452   D (u32, f32), \
 453   D (f32, s32), \
 454   D (f32, u32)
 455
 456 /* { _s32 _s64 } x { _b8 _b16 _b32 _b64 }
 457    { _u32 _u64 }.  */
 458 #define TYPES_inc_dec_n1(D, A) \
 459   D (A, b8), D (A, b16), D (A, b32), D (A, b64)
 460 #define TYPES_inc_dec_n(S, D) \
 461   TYPES_inc_dec_n1 (D, s32), \
 462   TYPES_inc_dec_n1 (D, s64), \
 463   TYPES_inc_dec_n1 (D, u32), \
 464   TYPES_inc_dec_n1 (D, u64)
 465
 466 /* { _s16 _u16 } x _s32
 467
 468    {      _u16 } x _u32.  */
 469 #define TYPES_qcvt_x2(S, D) \
 470   D (s16, s32), \
 471   D (u16, u32), \
 472   D (u16, s32)
 473
 474 /* { _s8  _u8  } x _s32
 475
 476    {      _u8  } x _u32
 477
 478    { _s16 _u16 } x _s64
 479
 480    {      _u16 } x _u64.  */
 481 #define TYPES_qcvt_x4(S, D) \
 482   D (s8, s32), \
 483   D (u8, u32), \
 484   D (u8, s32), \
 485   D (s16, s64), \
 486   D (u16, u64), \
 487   D (u16, s64)
 488
 489 /* _s16_s32
 490    _u16_u32.  */
 491 #define TYPES_qrshr_x2(S, D) \
 492   D (s16, s32), \
 493   D (u16, u32)
 494
 495 /* _u16_s32.  */
 496 #define TYPES_qrshru_x2(S, D) \
 497   D (u16, s32)
 498
 499 /* _s8_s32
 500    _s16_s64
 501    _u8_u32
 502    _u16_u64.  */
 503 #define TYPES_qrshr_x4(S, D) \
 504   D (s8, s32), \
 505   D (s16, s64), \
 506   D (u8, u32), \
 507   D (u16, u64)
 508
 509 /* _u8_s32
 510    _u16_s64.  */
 511 #define TYPES_qrshru_x4(S, D) \
 512   D (u8, s32), \
 513   D (u16, s64)
 514
 515 /* {     _bf16           }   {     _bf16           }
 516    {      _f16 _f32 _f64 }   {      _f16 _f32 _f64 }
 517    { _s8  _s16 _s32 _s64 } x { _s8  _s16 _s32 _s64 }
 518    { _u8  _u16 _u32 _u64 }   { _u8  _u16 _u32 _u64 }.  */
 519 #define TYPES_reinterpret1(D, A) \
 520   D (A, bf16), \
 521   D (A, f16), D (A, f32), D (A, f64), \
 522   D (A, s8), D (A, s16), D (A, s32), D (A, s64), \
 523   D (A, u8), D (A, u16), D (A, u32), D (A, u64)
 524 #define TYPES_reinterpret(S, D) \
 525   TYPES_reinterpret1 (D, bf16), \
 526   TYPES_reinterpret1 (D, f16), \
 527   TYPES_reinterpret1 (D, f32), \
 528   TYPES_reinterpret1 (D, f64), \
 529   TYPES_reinterpret1 (D, s8), \
 530   TYPES_reinterpret1 (D, s16), \
 531   TYPES_reinterpret1 (D, s32), \
 532   TYPES_reinterpret1 (D, s64), \
 533   TYPES_reinterpret1 (D, u8), \
 534   TYPES_reinterpret1 (D, u16), \
 535   TYPES_reinterpret1 (D, u32), \
 536   TYPES_reinterpret1 (D, u64)
 537
 538 /* _b_c
 539    _c_b.  */
 540 #define TYPES_reinterpret_b(S, D) \
 541   D (b, c), \
 542   D (c, b)
 543
 544 /* { _b8 _b16 _b32 _b64 } x { _s32 _s64 }
 545                             { _u32 _u64 } */
 546 #define TYPES_while1(D, bn) \
 547   D (bn, s32), D (bn, s64), D (bn, u32), D (bn, u64)
 548 #define TYPES_while(S, D) \
 549   TYPES_while1 (D, b8), \
 550   TYPES_while1 (D, b16), \
 551   TYPES_while1 (D, b32), \
 552   TYPES_while1 (D, b64)
 553
 554 /* { _b8 _b16 _b32 _b64 } x { _s64 }
 555                             { _u64 } */
 556 #define TYPES_while_x(S, D) \
 557   D (b8, s64), D (b8, u64), \
 558   D (b16, s64), D (b16, u64), \
 559   D (b32, s64), D (b32, u64), \
 560   D (b64, s64), D (b64, u64)
 561
 562 /* { _c8 _c16 _c32 _c64 } x { _s64 }
 563                             { _u64 } */
 564 #define TYPES_while_x_c(S, D) \
 565   D (c8, s64), D (c8, u64), \
 566   D (c16, s64), D (c16, u64), \
 567   D (c32, s64), D (c32, u64), \
 568   D (c64, s64), D (c64, u64)
 569
 570 /* _f32_f16
 571    _s32_s16
 572    _u32_u16.  */
 573 #define TYPES_s_narrow_fsu(S, D) \
 574   D (f32, f16), D (s32, s16), D (u32, u16)
 575
 576 /* _za8 _za16 _za32 _za64 _za128.  */
 577 #define TYPES_all_za(S, D) \
 578   S (za8), S (za16), S (za32), S (za64), S (za128)
 579
 580 /* _za64.  */
 581 #define TYPES_d_za(S, D) \
 582   S (za64)
 583
 584 /* {   _za8 } x {             _s8  _u8 }
 585
 586    {  _za16 } x { _bf16 _f16 _s16 _u16 }
 587
 588    {  _za32 } x {       _f32 _s32 _u32 }
 589
 590    {  _za64 } x {       _f64 _s64 _u64 }.  */
 591 #define TYPES_za_bhsd_data(S, D) \
 592   D (za8, s8), D (za8, u8), \
 593   D (za16, bf16), D (za16, f16), D (za16, s16), D (za16, u16), \
 594   D (za32, f32), D (za32, s32), D (za32, u32), \
 595   D (za64, f64), D (za64, s64), D (za64, u64)
 596
 597 /* Likewise, plus:
 598
 599    { _za128 } x {      _bf16           }
 600                 {       _f16 _f32 _f64 }
 601                 { _s8   _s16 _s32 _s64 }
 602                 { _u8   _u16 _u32 _u64 }.  */
 603
 604 #define TYPES_za_all_data(S, D) \
 605   TYPES_za_bhsd_data (S, D), \
 606   TYPES_reinterpret1 (D, za128)
 607
 608 /* _za32_s8.  */
 609 #define TYPES_za_s_b_signed(S, D) \
 610    D (za32, s8)
 611
 612 /* _za32_u8.  */
 613 #define TYPES_za_s_b_unsigned(S, D) \
 614    D (za32, u8)
 615
 616 /* _za32 x { _s8 _u8 }.  */
 617 #define TYPES_za_s_b_integer(S, D) \
 618   D (za32, s8), D (za32, u8)
 619
 620 /* _za32 x { _s16 _u16 }.  */
 621 #define TYPES_za_s_h_integer(S, D) \
 622   D (za32, s16), D (za32, u16)
 623
 624 /* _za32 x { _bf16 _f16 _s16 _u16 }.  */
 625 #define TYPES_za_s_h_data(S, D) \
 626   D (za32, bf16), D (za32, f16), D (za32, s16), D (za32, u16)
 627
 628 /* _za32_u32.  */
 629 #define TYPES_za_s_unsigned(S, D) \
 630   D (za32, u32)
 631
 632 /* _za32 x { _s32 _u32 }.  */
 633 #define TYPES_za_s_integer(S, D) \
 634   D (za32, s32), D (za32, u32)
 635
 636 /* _za32_f32.  */
 637 #define TYPES_za_s_float(S, D) \
 638   D (za32, f32)
 639
 640 /* _za32 x { _f32 _s32 _u32 }.  */
 641 #define TYPES_za_s_data(S, D) \
 642   D (za32, f32), D (za32, s32), D (za32, u32)
 643
 644 /* _za64 x { _s16 _u16 }.  */
 645 #define TYPES_za_d_h_integer(S, D) \
 646   D (za64, s16), D (za64, u16)
 647
 648 /* _za64_f64.  */
 649 #define TYPES_za_d_float(S, D) \
 650   D (za64, f64)
 651
 652 /* _za64 x { _s64 _u64 }.  */
 653 #define TYPES_za_d_integer(S, D) \
 654   D (za64, s64), D (za64, u64)
 655
 656 /* _za32 x { _s8 _u8 _bf16 _f16 _f32 }.  */
 657 #define TYPES_mop_base(S, D) \
 658   D (za32, s8), D (za32, u8), D (za32, bf16), D (za32, f16), D (za32, f32)
 659
 660 /* _za32_s8.  */
 661 #define TYPES_mop_base_signed(S, D) \
 662   D (za32, s8)
 663
 664 /* _za32_u8.  */
 665 #define TYPES_mop_base_unsigned(S, D) \
 666   D (za32, u8)
 667
 668 /* _za64 x { _s16 _u16 }.  */
 669 #define TYPES_mop_i16i64(S, D) \
 670   D (za64, s16), D (za64, u16)
 671
 672 /* _za64_s16.  */
 673 #define TYPES_mop_i16i64_signed(S, D) \
 674   D (za64, s16)
 675
 676 /* _za64_u16.  */
 677 #define TYPES_mop_i16i64_unsigned(S, D) \
 678   D (za64, u16)
 679
 680 /* _za.  */
 681 #define TYPES_za(S, D) \
 682   S (za)
 683
 684 /* Describe a pair of type suffixes in which only the first is used.  */
 685 #define DEF_VECTOR_TYPE(X) { TYPE_SUFFIX_ ## X, NUM_TYPE_SUFFIXES }
 686
 687 /* Describe a pair of type suffixes in which both are used.  */
 688 #define DEF_DOUBLE_TYPE(X, Y) { TYPE_SUFFIX_ ## X, TYPE_SUFFIX_ ## Y }
 689
 690 /* Create an array that can be used in aarch64-sve-builtins.def to
 691    select the type suffixes in TYPES_<NAME>.  */
 692 #define DEF_SVE_TYPES_ARRAY(NAME) \
 693   static const type_suffix_pair types_##NAME[] = { \
 694     TYPES_##NAME (DEF_VECTOR_TYPE, DEF_DOUBLE_TYPE), \
 695     { NUM_TYPE_SUFFIXES, NUM_TYPE_SUFFIXES } \
 696   }
 697
 698 /* For functions that don't take any type suffixes.  */
 699 static const type_suffix_pair types_none[] = {
 700   { NUM_TYPE_SUFFIXES, NUM_TYPE_SUFFIXES },
 701   { NUM_TYPE_SUFFIXES, NUM_TYPE_SUFFIXES }
 702 };
 703
 704 /* Create an array for each TYPES_<combination> macro above.  */
 705 DEF_SVE_TYPES_ARRAY (all_pred);
 706 DEF_SVE_TYPES_ARRAY (all_count);
 707 DEF_SVE_TYPES_ARRAY (all_pred_count);
 708 DEF_SVE_TYPES_ARRAY (all_float);
 709 DEF_SVE_TYPES_ARRAY (all_signed);
 710 DEF_SVE_TYPES_ARRAY (all_float_and_signed);
 711 DEF_SVE_TYPES_ARRAY (all_unsigned);
 712 DEF_SVE_TYPES_ARRAY (all_integer);
 713 DEF_SVE_TYPES_ARRAY (all_arith);
 714 DEF_SVE_TYPES_ARRAY (all_data);
 715 DEF_SVE_TYPES_ARRAY (b);
 716 DEF_SVE_TYPES_ARRAY (b_unsigned);
 717 DEF_SVE_TYPES_ARRAY (b_integer);
 718 DEF_SVE_TYPES_ARRAY (bh_integer);
 719 DEF_SVE_TYPES_ARRAY (bs_unsigned);
 720 DEF_SVE_TYPES_ARRAY (bhs_signed);
 721 DEF_SVE_TYPES_ARRAY (bhs_unsigned);
 722 DEF_SVE_TYPES_ARRAY (bhs_integer);
 723 DEF_SVE_TYPES_ARRAY (bhs_data);
 724 DEF_SVE_TYPES_ARRAY (bhs_widen);
 725 DEF_SVE_TYPES_ARRAY (c);
 726 DEF_SVE_TYPES_ARRAY (h_integer);
 727 DEF_SVE_TYPES_ARRAY (hs_signed);
 728 DEF_SVE_TYPES_ARRAY (hs_integer);
 729 DEF_SVE_TYPES_ARRAY (hs_float);
 730 DEF_SVE_TYPES_ARRAY (hs_data);
 731 DEF_SVE_TYPES_ARRAY (hd_unsigned);
 732 DEF_SVE_TYPES_ARRAY (hsd_signed);
 733 DEF_SVE_TYPES_ARRAY (hsd_integer);
 734 DEF_SVE_TYPES_ARRAY (s_float);
 735 DEF_SVE_TYPES_ARRAY (s_float_hsd_integer);
 736 DEF_SVE_TYPES_ARRAY (s_float_sd_integer);
 737 DEF_SVE_TYPES_ARRAY (s_signed);
 738 DEF_SVE_TYPES_ARRAY (s_unsigned);
 739 DEF_SVE_TYPES_ARRAY (s_integer);
 740 DEF_SVE_TYPES_ARRAY (sd_signed);
 741 DEF_SVE_TYPES_ARRAY (sd_unsigned);
 742 DEF_SVE_TYPES_ARRAY (sd_integer);
 743 DEF_SVE_TYPES_ARRAY (sd_data);
 744 DEF_SVE_TYPES_ARRAY (all_float_and_sd_integer);
 745 DEF_SVE_TYPES_ARRAY (d_float);
 746 DEF_SVE_TYPES_ARRAY (d_unsigned);
 747 DEF_SVE_TYPES_ARRAY (d_integer);
 748 DEF_SVE_TYPES_ARRAY (d_data);
 749 DEF_SVE_TYPES_ARRAY (cvt);
 750 DEF_SVE_TYPES_ARRAY (cvt_bfloat);
 751 DEF_SVE_TYPES_ARRAY (cvt_h_s_float);
 752 DEF_SVE_TYPES_ARRAY (cvt_long);
 753 DEF_SVE_TYPES_ARRAY (cvt_narrow_s);
 754 DEF_SVE_TYPES_ARRAY (cvt_narrow);
 755 DEF_SVE_TYPES_ARRAY (cvt_s_s);
 756 DEF_SVE_TYPES_ARRAY (inc_dec_n);
 757 DEF_SVE_TYPES_ARRAY (qcvt_x2);
 758 DEF_SVE_TYPES_ARRAY (qcvt_x4);
 759 DEF_SVE_TYPES_ARRAY (qrshr_x2);
 760 DEF_SVE_TYPES_ARRAY (qrshr_x4);
 761 DEF_SVE_TYPES_ARRAY (qrshru_x2);
 762 DEF_SVE_TYPES_ARRAY (qrshru_x4);
 763 DEF_SVE_TYPES_ARRAY (reinterpret);
 764 DEF_SVE_TYPES_ARRAY (reinterpret_b);
 765 DEF_SVE_TYPES_ARRAY (while);
 766 DEF_SVE_TYPES_ARRAY (while_x);
 767 DEF_SVE_TYPES_ARRAY (while_x_c);
 768 DEF_SVE_TYPES_ARRAY (s_narrow_fsu);
 769 DEF_SVE_TYPES_ARRAY (all_za);
 770 DEF_SVE_TYPES_ARRAY (d_za);
 771 DEF_SVE_TYPES_ARRAY (za_bhsd_data);
 772 DEF_SVE_TYPES_ARRAY (za_all_data);
 773 DEF_SVE_TYPES_ARRAY (za_s_b_signed);
 774 DEF_SVE_TYPES_ARRAY (za_s_b_unsigned);
 775 DEF_SVE_TYPES_ARRAY (za_s_b_integer);
 776 DEF_SVE_TYPES_ARRAY (za_s_h_integer);
 777 DEF_SVE_TYPES_ARRAY (za_s_h_data);
 778 DEF_SVE_TYPES_ARRAY (za_s_unsigned);
 779 DEF_SVE_TYPES_ARRAY (za_s_integer);
 780 DEF_SVE_TYPES_ARRAY (za_s_float);
 781 DEF_SVE_TYPES_ARRAY (za_s_data);
 782 DEF_SVE_TYPES_ARRAY (za_d_h_integer);
 783 DEF_SVE_TYPES_ARRAY (za_d_float);
 784 DEF_SVE_TYPES_ARRAY (za_d_integer);
 785 DEF_SVE_TYPES_ARRAY (mop_base);
 786 DEF_SVE_TYPES_ARRAY (mop_base_signed);
 787 DEF_SVE_TYPES_ARRAY (mop_base_unsigned);
 788 DEF_SVE_TYPES_ARRAY (mop_i16i64);
 789 DEF_SVE_TYPES_ARRAY (mop_i16i64_signed);
 790 DEF_SVE_TYPES_ARRAY (mop_i16i64_unsigned);
 791 DEF_SVE_TYPES_ARRAY (za);
 792
 793 static const group_suffix_index groups_none[] = {
 794   GROUP_none, NUM_GROUP_SUFFIXES
 795 };
 796
 797 static const group_suffix_index groups_x2[] = { GROUP_x2, NUM_GROUP_SUFFIXES };
 798
 799 static const group_suffix_index groups_x12[] = {
 800   GROUP_none, GROUP_x2, NUM_GROUP_SUFFIXES
 801 };
 802
 803 static const group_suffix_index groups_x4[] = { GROUP_x4, NUM_GROUP_SUFFIXES };
 804
 805 static const group_suffix_index groups_x24[] = {
 806   GROUP_x2, GROUP_x4, NUM_GROUP_SUFFIXES
 807 };
 808
 809 static const group_suffix_index groups_x124[] = {
 810   GROUP_none, GROUP_x2, GROUP_x4, NUM_GROUP_SUFFIXES
 811 };
 812
 813 static const group_suffix_index groups_x1234[] = {
 814   GROUP_none, GROUP_x2, GROUP_x3, GROUP_x4, NUM_GROUP_SUFFIXES
 815 };
 816
 817 static const group_suffix_index groups_vg1x2[] = {
 818   GROUP_vg1x2, NUM_GROUP_SUFFIXES
 819 };
 820
 821 static const group_suffix_index groups_vg1x4[] = {
 822   GROUP_vg1x4, NUM_GROUP_SUFFIXES
 823 };
 824
 825 static const group_suffix_index groups_vg1x24[] = {
 826   GROUP_vg1x2, GROUP_vg1x4, NUM_GROUP_SUFFIXES
 827 };
 828
 829 static const group_suffix_index groups_vg2[] = {
 830   GROUP_vg2x1, GROUP_vg2x2, GROUP_vg2x4, NUM_GROUP_SUFFIXES
 831 };
 832
 833 static const group_suffix_index groups_vg4[] = {
 834   GROUP_vg4x1, GROUP_vg4x2, GROUP_vg4x4, NUM_GROUP_SUFFIXES
 835 };
 836
 837 static const group_suffix_index groups_vg24[] = {
 838   GROUP_vg2, GROUP_vg4, NUM_GROUP_SUFFIXES
 839 };
 840
 841 /* Used by functions that have no governing predicate.  */
 842 static const predication_index preds_none[] = { PRED_none, NUM_PREDS };
 843
 844 /* Used by functions that have a governing predicate but do not have an
 845    explicit suffix.  */
 846 static const predication_index preds_implicit[] = { PRED_implicit, NUM_PREDS };
 847
 848 /* Used by functions that only support "_m" predication.  */
 849 static const predication_index preds_m[] = { PRED_m, NUM_PREDS };
 850
 851 /* Used by functions that allow merging and "don't care" predication,
 852    but are not suitable for predicated MOVPRFX.  */
 853 static const predication_index preds_mx[] = {
 854   PRED_m, PRED_x, NUM_PREDS
 855 };
 856
 857 /* Used by functions that allow merging, zeroing and "don't care"
 858    predication.  */
 859 static const predication_index preds_mxz[] = {
 860   PRED_m, PRED_x, PRED_z, NUM_PREDS
 861 };
 862
 863 /* Used by functions that have the mxz predicated forms above, and in addition
 864    have an unpredicated form.  */
 865 static const predication_index preds_mxz_or_none[] = {
 866   PRED_m, PRED_x, PRED_z, PRED_none, NUM_PREDS
 867 };
 868
 869 /* Used by functions that allow merging and zeroing predication but have
 870    no "_x" form.  */
 871 static const predication_index preds_mz[] = { PRED_m, PRED_z, NUM_PREDS };
 872
 873 /* Used by functions that have an unpredicated form and a _z predicated
 874    form.  */
 875 static const predication_index preds_z_or_none[] = {
 876   PRED_z, PRED_none, NUM_PREDS
 877 };
 878
 879 /* Used by (mostly predicate) functions that only support "_z" predication.  */
 880 static const predication_index preds_z[] = { PRED_z, NUM_PREDS };
 881
 882 /* Used by SME instructions that always merge into ZA.  */
 883 static const predication_index preds_za_m[] = { PRED_za_m, NUM_PREDS };
 884
 885 /* A list of all arm_sve.h functions.  */
 886 static CONSTEXPR const function_group_info function_groups[] = {
 887 #define DEF_SVE_FUNCTION_GS(NAME, SHAPE, TYPES, GROUPS, PREDS) \
 888   { #NAME, &functions::NAME, &shapes::SHAPE, types_##TYPES, groups_##GROUPS, \
 889     preds_##PREDS, REQUIRED_EXTENSIONS },
 890 #include "aarch64-sve-builtins.def"
 891 };
 892
 893 /* A list of all arm_neon_sve_bridge.h ACLE functions.  */
 894 static CONSTEXPR const function_group_info neon_sve_function_groups[] = {
 895 #define DEF_NEON_SVE_FUNCTION(NAME, SHAPE, TYPES, GROUPS, PREDS) \
 896   { #NAME, &neon_sve_bridge_functions::NAME, &shapes::SHAPE, types_##TYPES, \
 897     groups_##GROUPS, preds_##PREDS, 0 },
 898 #include "aarch64-neon-sve-bridge-builtins.def"
 899 };
 900
 901 /* A list of all arm_sme.h functions.  */
 902 static CONSTEXPR const function_group_info sme_function_groups[] = {
 903 #define DEF_SME_FUNCTION_GS(NAME, SHAPE, TYPES, GROUPS, PREDS) \
 904   { #NAME, &functions::NAME, &shapes::SHAPE, types_##TYPES, groups_##GROUPS, \
 905     preds_##PREDS, REQUIRED_EXTENSIONS },
 906 #define DEF_SME_ZA_FUNCTION_GS(NAME, SHAPE, TYPES, GROUPS, PREDS) \
 907   { #NAME, &functions::NAME##_za, &shapes::SHAPE, types_##TYPES, \
 908     groups_##GROUPS, preds_##PREDS, (REQUIRED_EXTENSIONS | AARCH64_FL_ZA_ON) },
 909 #include "aarch64-sve-builtins-sme.def"
 910 };
 911
 912 /* The scalar type associated with each vector type.  */
 913 extern GTY(()) tree scalar_types[NUM_VECTOR_TYPES + 1];
 914 tree scalar_types[NUM_VECTOR_TYPES + 1];
 915
 916 /* The single-predicate and single-vector types, with their built-in
 917    "__SV..._t" name.  Allow an index of NUM_VECTOR_TYPES, which always
 918    yields a null tree.  */
 919 static GTY(()) tree abi_vector_types[NUM_VECTOR_TYPES + 1];
 920
 921 /* Same, but with the arm_sve.h "sv..._t" name.  */
 922 extern GTY(()) tree acle_vector_types[MAX_TUPLE_SIZE][NUM_VECTOR_TYPES + 1];
 923 tree acle_vector_types[MAX_TUPLE_SIZE][NUM_VECTOR_TYPES + 1];
 924
 925 /* The svpattern enum type.  */
 926 extern GTY(()) tree acle_svpattern;
 927 tree acle_svpattern;
 928
 929 /* The svprfop enum type.  */
 930 extern GTY(()) tree acle_svprfop;
 931 tree acle_svprfop;
 932
 933 /* The list of all registered function decls, indexed by code.  */
 934 static GTY(()) vec<registered_function *, va_gc> *registered_functions;
 935
 936 /* Stores the starting function index for each pragma handler.  */
 937 static unsigned int initial_indexes[NUM_PRAGMA_HANDLERS];
 938
 939 /* All registered function decls, hashed on the function_instance
 940    that they implement.  This is used for looking up implementations of
 941    overloaded functions.  */
 942 static hash_table<registered_function_hasher> *function_table;
 943
 944 /* Index 0 maps all overloaded function names that we've registered so far to
 945    their associated function_instances.  Index 1 does the same for functions
 946    that we've skipped over without registering.  In both cases, the map keys
 947    are IDENTIFIER_NODEs.  */
 948 static GTY(()) hash_map<tree, registered_function *> *overload_names[2];
 949
 950 /* Record that TYPE is an ABI-defined SVE type that contains NUM_ZR SVE vectors
 951    and NUM_PR SVE predicates.  MANGLED_NAME, if nonnull, is the ABI-defined
 952    mangling of the type.  ACLE_NAME is the <arm_sve.h> name of the type.  */
 953 static void
 954 add_sve_type_attribute (tree type, unsigned int num_zr, unsigned int num_pr,
 955                         const char *mangled_name, const char *acle_name)
 956 {
 957   tree mangled_name_tree
 958     = (mangled_name ? get_identifier (mangled_name) : NULL_TREE);
 959
 960   tree value = tree_cons (NULL_TREE, get_identifier (acle_name), NULL_TREE);
 961   value = tree_cons (NULL_TREE, mangled_name_tree, value);
 962   value = tree_cons (NULL_TREE, size_int (num_pr), value);
 963   value = tree_cons (NULL_TREE, size_int (num_zr), value);
 964   TYPE_ATTRIBUTES (type) = tree_cons (get_identifier ("SVE type"), value,
 965                                       TYPE_ATTRIBUTES (type));
 966 }
 967
 968 /* If TYPE is an ABI-defined SVE type, return its attribute descriptor,
 969    otherwise return null.  */
 970 static tree
 971 lookup_sve_type_attribute (const_tree type)
 972 {
 973   if (type == error_mark_node)
 974     return NULL_TREE;
 975   return lookup_attribute ("SVE type", TYPE_ATTRIBUTES (type));
 976 }
 977
 978 /* Force TYPE to be a sizeless type.  */
 979 static void
 980 make_type_sizeless (tree type)
 981 {
 982   TYPE_ATTRIBUTES (type) = tree_cons (get_identifier ("SVE sizeless type"),
 983                                       NULL_TREE, TYPE_ATTRIBUTES (type));
 984 }
 985
 986 /* Return true if TYPE is a sizeless type.  */
 987 static bool
 988 sizeless_type_p (const_tree type)
 989 {
 990   if (type == error_mark_node)
 991     return NULL_TREE;
 992   return lookup_attribute ("SVE sizeless type", TYPE_ATTRIBUTES (type));
 993 }
 994
 995 /* Return true if CANDIDATE is equivalent to MODEL_TYPE for overloading
 996    purposes.  */
 997 static bool
 998 matches_type_p (const_tree model_type, const_tree candidate)
 999 {
1000   if (VECTOR_TYPE_P (model_type))
1001     {
1002       if (!VECTOR_TYPE_P (candidate)
1003           || maybe_ne (TYPE_VECTOR_SUBPARTS (model_type),
1004                        TYPE_VECTOR_SUBPARTS (candidate))
1005           || TYPE_MODE (model_type) != TYPE_MODE (candidate))
1006         return false;
1007
1008       model_type = TREE_TYPE (model_type);
1009       candidate = TREE_TYPE (candidate);
1010     }
1011   return (candidate != error_mark_node
1012           && TYPE_MAIN_VARIANT (model_type) == TYPE_MAIN_VARIANT (candidate));
1013 }
1014
1015 /* If TYPE is a valid SVE element type, return the corresponding type
1016    suffix, otherwise return NUM_TYPE_SUFFIXES.  */
1017 static type_suffix_index
1018 find_type_suffix_for_scalar_type (const_tree type)
1019 {
1020   /* A linear search should be OK here, since the code isn't hot and
1021      the number of types is only small.  */
1022   for (unsigned int suffix_i = 0; suffix_i < NUM_TYPE_SUFFIXES; ++suffix_i)
1023     if (type_suffixes[suffix_i].vector_p)
1024       {
1025         vector_type_index vector_i = type_suffixes[suffix_i].vector_type;
1026         if (matches_type_p (scalar_types[vector_i], type))
1027           return type_suffix_index (suffix_i);
1028       }
1029   return NUM_TYPE_SUFFIXES;
1030 }
1031
1032 /* Return the implicit group suffix for intrinsics that operate on NVECTORS
1033    vectors.  */
1034 static group_suffix_index
1035 num_vectors_to_group (unsigned int nvectors)
1036 {
1037   switch (nvectors)
1038     {
1039     case 1: return GROUP_none;
1040     case 2: return GROUP_x2;
1041     case 3: return GROUP_x3;
1042     case 4: return GROUP_x4;
1043     }
1044   gcc_unreachable ();
1045 }
1046
1047 /* Return the vector type associated with TYPE.  */
1048 static tree
1049 get_vector_type (sve_type type)
1050 {
1051   auto vector_type = type_suffixes[type.type].vector_type;
1052   return acle_vector_types[type.num_vectors - 1][vector_type];
1053 }
1054
1055 /* If FNDECL is an SVE builtin, return its function instance, otherwise
1056    return null.  */
1057 const function_instance *
1058 lookup_fndecl (tree fndecl)
1059 {
1060   if (!fndecl_built_in_p (fndecl, BUILT_IN_MD))
1061     return nullptr;
1062
1063   unsigned int code = DECL_MD_FUNCTION_CODE (fndecl);
1064   if ((code & AARCH64_BUILTIN_CLASS) != AARCH64_BUILTIN_SVE)
1065     return nullptr;
1066
1067   unsigned int subcode = code >> AARCH64_BUILTIN_SHIFT;
1068   return &(*registered_functions)[subcode]->instance;
1069 }
1070
1071
1072 /* Report that LOCATION has a call to FNDECL in which argument ARGNO
1073    was not an integer constant expression.  ARGNO counts from zero.  */
1074 static void
1075 report_non_ice (location_t location, tree fndecl, unsigned int argno)
1076 {
1077   error_at (location, "argument %d of %qE must be an integer constant"
1078             " expression", argno + 1, fndecl);
1079 }
1080
1081 /* Report that LOCATION has a call to FNDECL in which argument ARGNO has
1082    the value ACTUAL, whereas the function requires a value in the range
1083    [MIN, MAX].  ARGNO counts from zero.  */
1084 static void
1085 report_out_of_range (location_t location, tree fndecl, unsigned int argno,
1086                      HOST_WIDE_INT actual, HOST_WIDE_INT min,
1087                      HOST_WIDE_INT max)
1088 {
1089   if (min == max)
1090     error_at (location, "passing %wd to argument %d of %qE, which expects"
1091               " the value %wd", actual, argno + 1, fndecl, min);
1092   else
1093     error_at (location, "passing %wd to argument %d of %qE, which expects"
1094               " a value in the range [%wd, %wd]", actual, argno + 1, fndecl,
1095               min, max);
1096 }
1097
1098 /* Report that LOCATION has a call to FNDECL in which argument ARGNO has
1099    the value ACTUAL, whereas the function requires either VALUE0 or
1100    VALUE1.  ARGNO counts from zero.  */
1101 static void
1102 report_neither_nor (location_t location, tree fndecl, unsigned int argno,
1103                     HOST_WIDE_INT actual, HOST_WIDE_INT value0,
1104                     HOST_WIDE_INT value1)
1105 {
1106   error_at (location, "passing %wd to argument %d of %qE, which expects"
1107             " either %wd or %wd", actual, argno + 1, fndecl, value0, value1);
1108 }
1109
1110 /* Report that LOCATION has a call to FNDECL in which argument ARGNO has
1111    the value ACTUAL, whereas the function requires one of VALUE0..3.
1112    ARGNO counts from zero.  */
1113 static void
1114 report_not_one_of (location_t location, tree fndecl, unsigned int argno,
1115                    HOST_WIDE_INT actual, HOST_WIDE_INT value0,
1116                    HOST_WIDE_INT value1, HOST_WIDE_INT value2,
1117                    HOST_WIDE_INT value3)
1118 {
1119   error_at (location, "passing %wd to argument %d of %qE, which expects"
1120             " %wd, %wd, %wd or %wd", actual, argno + 1, fndecl, value0, value1,
1121             value2, value3);
1122 }
1123
1124 /* Report that LOCATION has a call to FNDECL in which argument ARGNO has
1125    the value ACTUAL, whereas the function requires a valid value of
1126    enum type ENUMTYPE.  ARGNO counts from zero.  */
1127 static void
1128 report_not_enum (location_t location, tree fndecl, unsigned int argno,
1129                  HOST_WIDE_INT actual, tree enumtype)
1130 {
1131   error_at (location, "passing %wd to argument %d of %qE, which expects"
1132             " a valid %qT value", actual, argno + 1, fndecl, enumtype);
1133 }
1134
1135 /* Return a hash code for a function_instance.  */
1136 hashval_t
1137 function_instance::hash () const
1138 {
1139   inchash::hash h;
1140   /* BASE uniquely determines BASE_NAME, so we don't need to hash both.  */
1141   h.add_ptr (base);
1142   h.add_ptr (shape);
1143   h.add_int (mode_suffix_id);
1144   h.add_int (type_suffix_ids[0]);
1145   h.add_int (type_suffix_ids[1]);
1146   h.add_int (group_suffix_id);
1147   h.add_int (pred);
1148   return h.end ();
1149 }
1150
1151 /* Return a set of CP_* flags that describe what the function could do,
1152    taking the command-line flags into account.  */
1153 unsigned int
1154 function_instance::call_properties () const
1155 {
1156   unsigned int flags = base->call_properties (*this);
1157
1158   /* -fno-trapping-math means that we can assume any FP exceptions
1159      are not user-visible.  */
1160   if (!flag_trapping_math)
1161     flags &= ~CP_RAISE_FP_EXCEPTIONS;
1162
1163   return flags;
1164 }
1165
1166 /* Return true if calls to the function could read some form of
1167    global state.  */
1168 bool
1169 function_instance::reads_global_state_p () const
1170 {
1171   unsigned int flags = call_properties ();
1172
1173   /* Preserve any dependence on rounding mode, flush to zero mode, etc.
1174      There is currently no way of turning this off; in particular,
1175      -fno-rounding-math (which is the default) means that we should make
1176      the usual assumptions about rounding mode, which for intrinsics means
1177      acting as the instructions do.  */
1178   if (flags & CP_READ_FPCR)
1179     return true;
1180
1181   /* Handle direct reads of global state.  */
1182   return flags & (CP_READ_MEMORY | CP_READ_FFR | CP_READ_ZA | CP_READ_ZT0);
1183 }
1184
1185 /* Return true if calls to the function could modify some form of
1186    global state.  */
1187 bool
1188 function_instance::modifies_global_state_p () const
1189 {
1190   unsigned int flags = call_properties ();
1191
1192   /* Preserve any exception state written back to the FPCR,
1193      unless -fno-trapping-math says this is unnecessary.  */
1194   if (flags & CP_RAISE_FP_EXCEPTIONS)
1195     return true;
1196
1197   /* Treat prefetches as modifying global state, since that's the
1198      only means we have of keeping them in their correct position.  */
1199   if (flags & CP_PREFETCH_MEMORY)
1200     return true;
1201
1202   /* Handle direct modifications of global state.  */
1203   return flags & (CP_WRITE_MEMORY | CP_WRITE_FFR | CP_WRITE_ZA | CP_WRITE_ZT0);
1204 }
1205
1206 /* Return true if calls to the function could raise a signal.  */
1207 bool
1208 function_instance::could_trap_p () const
1209 {
1210   unsigned int flags = call_properties ();
1211
1212   /* Handle functions that could raise SIGFPE.  */
1213   if (flags & CP_RAISE_FP_EXCEPTIONS)
1214     return true;
1215
1216   /* Handle functions that could raise SIGBUS or SIGSEGV.  */
1217   if (flags & (CP_READ_MEMORY | CP_WRITE_MEMORY))
1218     return true;
1219
1220   return false;
1221 }
1222
1223 inline hashval_t
1224 registered_function_hasher::hash (value_type value)
1225 {
1226   return value->instance.hash ();
1227 }
1228
1229 inline bool
1230 registered_function_hasher::equal (value_type value, const compare_type &key)
1231 {
1232   return value->instance == key;
1233 }
1234
1235 sve_switcher::sve_switcher (aarch64_feature_flags flags)
1236   : aarch64_simd_switcher (AARCH64_FL_F16 | AARCH64_FL_SVE | flags)
1237 {
1238   /* Changing the ISA flags and have_regs_of_mode should be enough here.
1239      We shouldn't need to pay the compile-time cost of a full target
1240      switch.  */
1241   m_old_maximum_field_alignment = maximum_field_alignment;
1242   maximum_field_alignment = 0;
1243
1244   memcpy (m_old_have_regs_of_mode, have_regs_of_mode,
1245           sizeof (have_regs_of_mode));
1246   for (int i = 0; i < NUM_MACHINE_MODES; ++i)
1247     if (aarch64_sve_mode_p ((machine_mode) i))
1248       have_regs_of_mode[i] = true;
1249 }
1250
1251 sve_switcher::~sve_switcher ()
1252 {
1253   memcpy (have_regs_of_mode, m_old_have_regs_of_mode,
1254           sizeof (have_regs_of_mode));
1255   maximum_field_alignment = m_old_maximum_field_alignment;
1256 }
1257
1258 function_builder::function_builder (handle_pragma_index pragma_index,
1259                                     bool function_nulls)
1260 {
1261   m_overload_type = build_function_type (void_type_node, void_list_node);
1262   m_direct_overloads = lang_GNU_CXX ();
1263
1264   if (initial_indexes[pragma_index] == 0)
1265     {
1266       unsigned int index = vec_safe_length (registered_functions);
1267       initial_indexes[pragma_index] = index;
1268     }
1269
1270   m_function_index = initial_indexes[pragma_index];
1271   m_function_nulls = function_nulls;
1272
1273   gcc_obstack_init (&m_string_obstack);
1274 }
1275
1276 function_builder::~function_builder ()
1277 {
1278   obstack_free (&m_string_obstack, NULL);
1279 }
1280
1281 /* Add NAME to the end of the function name being built.  */
1282 void
1283 function_builder::append_name (const char *name)
1284 {
1285   obstack_grow (&m_string_obstack, name, strlen (name));
1286 }
1287
1288 /* Zero-terminate and complete the function name being built.  */
1289 char *
1290 function_builder::finish_name ()
1291 {
1292   obstack_1grow (&m_string_obstack, 0);
1293   return (char *) obstack_finish (&m_string_obstack);
1294 }
1295
1296 /* Return the overloaded or full function name for INSTANCE; OVERLOADED_P
1297    selects which.  Allocate the string on m_string_obstack; the caller
1298    must use obstack_free to free it after use.  */
1299 char *
1300 function_builder::get_name (const function_instance &instance,
1301                             bool overloaded_p)
1302 {
1303   /* __arm_* functions are listed as arm_*, so that the associated GCC
1304      code is not in the implementation namespace.  */
1305   if (strncmp (instance.base_name, "arm_", 4) == 0)
1306     append_name ("__");
1307   append_name (instance.base_name);
1308   if (overloaded_p)
1309     switch (instance.displacement_units ())
1310       {
1311       case UNITS_none:
1312         break;
1313
1314       case UNITS_bytes:
1315         append_name ("_offset");
1316         break;
1317
1318       case UNITS_elements:
1319         append_name ("_index");
1320         break;
1321
1322       case UNITS_vectors:
1323         append_name ("_vnum");
1324         break;
1325       }
1326   else
1327     append_name (instance.mode_suffix ().string);
1328   for (unsigned int i = 0; i < 2; ++i)
1329     if (!overloaded_p || instance.shape->explicit_type_suffix_p (i))
1330       append_name (instance.type_suffix (i).string);
1331   if (!overloaded_p || instance.shape->explicit_group_suffix_p ())
1332     append_name (instance.group_suffix ().string);
1333   append_name (pred_suffixes[instance.pred]);
1334   return finish_name ();
1335 }
1336
1337 /* Add attribute NAME to ATTRS.  */
1338 static tree
1339 add_attribute (const char *name, tree attrs)
1340 {
1341   return tree_cons (get_identifier (name), NULL_TREE, attrs);
1342 }
1343
1344 /* Add attribute NS::NAME to ATTRS.  */
1345 static tree
1346 add_attribute (const char *ns, const char *name, tree value, tree attrs)
1347 {
1348   return tree_cons (build_tree_list (get_identifier (ns),
1349                                      get_identifier (name)),
1350                     value, attrs);
1351 }
1352
1353 /* Attribute arm::NAME describes shared state that is an input if IS_IN
1354    and an output if IS_OUT.  Check whether a call with call properties
1355    CALL_FLAGS needs such an attribute.  Add it to in-progress attribute
1356    list ATTRS if so.  Return the new attribute list.  */
1357 static tree
1358 add_shared_state_attribute (const char *name, bool is_in, bool is_out,
1359                             unsigned int call_flags, tree attrs)
1360 {
1361   struct state_flag_info
1362   {
1363     const char *name;
1364     unsigned int read_flag;
1365     unsigned int write_flag;
1366   };
1367   static state_flag_info state_flags[] =
1368   {
1369     { "za", CP_READ_ZA, CP_WRITE_ZA },
1370     { "zt0", CP_READ_ZT0, CP_WRITE_ZT0 }
1371   };
1372
1373   tree args = NULL_TREE;
1374   for (const auto &state_flag : state_flags)
1375     {
1376       auto all_flags = state_flag.read_flag | state_flag.write_flag;
1377       auto these_flags = ((is_in ? state_flag.read_flag : 0)
1378                           | (is_out ? state_flag.write_flag : 0));
1379       if ((call_flags & all_flags) == these_flags)
1380         {
1381           tree value = build_string (strlen (state_flag.name) + 1,
1382                                      state_flag.name);
1383           args = tree_cons (NULL_TREE, value, args);
1384         }
1385     }
1386   if (args)
1387     attrs = add_attribute ("arm", name, args, attrs);
1388   return attrs;
1389 }
1390
1391 /* Return the appropriate function attributes for INSTANCE, which requires
1392    the feature flags in REQUIRED_EXTENSIONS.  */
1393 tree
1394 function_builder::get_attributes (const function_instance &instance,
1395                                   aarch64_feature_flags required_extensions)
1396 {
1397   tree attrs = NULL_TREE;
1398
1399   if (required_extensions & AARCH64_FL_SM_ON)
1400     attrs = add_attribute ("arm", "streaming", NULL_TREE, attrs);
1401   else if (!(required_extensions & AARCH64_FL_SM_OFF))
1402     attrs = add_attribute ("arm", "streaming_compatible", NULL_TREE, attrs);
1403
1404   attrs = add_shared_state_attribute ("in", true, false,
1405                                       instance.call_properties (), attrs);
1406   attrs = add_shared_state_attribute ("out", false, true,
1407                                       instance.call_properties (), attrs);
1408   attrs = add_shared_state_attribute ("inout", true, true,
1409                                       instance.call_properties (), attrs);
1410
1411   if (!instance.modifies_global_state_p ())
1412     {
1413       if (instance.reads_global_state_p ())
1414         attrs = add_attribute ("pure", attrs);
1415       else
1416         attrs = add_attribute ("const", attrs);
1417     }
1418
1419   if (!flag_non_call_exceptions || !instance.could_trap_p ())
1420     attrs = add_attribute ("nothrow", attrs);
1421
1422   return add_attribute ("leaf", attrs);
1423 }
1424
1425 /* Add a function called NAME with type FNTYPE and attributes ATTRS.
1426    INSTANCE describes what the function does and OVERLOADED_P indicates
1427    whether it is overloaded.  REQUIRED_EXTENSIONS are the set of
1428    architecture extensions that the function requires.  */
1429 registered_function &
1430 function_builder::add_function (const function_instance &instance,
1431                                 const char *name, tree fntype, tree attrs,
1432                                 aarch64_feature_flags required_extensions,
1433                                 bool overloaded_p,
1434                                 bool placeholder_p)
1435 {
1436   unsigned int length = vec_safe_length (registered_functions);
1437   unsigned int code = (m_function_index << AARCH64_BUILTIN_SHIFT) | AARCH64_BUILTIN_SVE;
1438   /* We need to be able to generate placeholders to enusre that we have a
1439      consistent numbering scheme for function codes between the C and C++
1440      frontends, so that everything ties up in LTO.
1441
1442      Currently, tree-streamer-in.cc:unpack_ts_function_decl_value_fields
1443      validates that tree nodes returned by TARGET_BUILTIN_DECL are non-NULL and
1444      some node other than error_mark_node. This is a holdover from when builtin
1445      decls were streamed by code rather than by value.
1446
1447      Ultimately, we should be able to remove this validation of BUILT_IN_MD
1448      nodes and remove the target hook. For now, however, we need to appease the
1449      validation and return a non-NULL, non-error_mark_node node, so we
1450      arbitrarily choose integer_zero_node.  */
1451   tree decl = placeholder_p || m_function_nulls
1452     ? integer_zero_node
1453     : simulate_builtin_function_decl (input_location, name, fntype,
1454                                       code, NULL, attrs);
1455
1456   registered_function &rfn = *ggc_alloc <registered_function> ();
1457   rfn.instance = instance;
1458   rfn.decl = decl;
1459   rfn.required_extensions = required_extensions;
1460   rfn.overloaded_p = overloaded_p;
1461   if (m_function_index >= length)
1462     vec_safe_push (registered_functions, &rfn);
1463   else
1464     (*registered_functions)[m_function_index] = &rfn;
1465   m_function_index++;
1466
1467   return rfn;
1468 }
1469
1470 /* Add a built-in function for INSTANCE, with the argument types given
1471    by ARGUMENT_TYPES and the return type given by RETURN_TYPE.
1472    REQUIRED_EXTENSIONS are the set of architecture extensions that the
1473    function requires.  FORCE_DIRECT_OVERLOADS is true if there is a
1474    one-to-one mapping between "short" and "full" names, and if standard
1475    overload resolution therefore isn't necessary.  */
1476 void
1477 function_builder::
1478 add_unique_function (const function_instance &instance,
1479                      tree return_type,
1480                      vec<tree> &argument_types,
1481                      aarch64_feature_flags required_extensions,
1482                      bool force_direct_overloads)
1483 {
1484   /* Add the function under its full (unique) name.  */
1485   char *name = get_name (instance, false);
1486   tree fntype = build_function_type_array (return_type,
1487                                            argument_types.length (),
1488                                            argument_types.address ());
1489   tree attrs = get_attributes (instance, required_extensions);
1490   registered_function &rfn = add_function (instance, name, fntype, attrs,
1491                                            required_extensions, false, false);
1492
1493   /* Enter the function into the hash table.  */
1494   if (!m_function_nulls)
1495     {
1496       hashval_t hash = instance.hash ();
1497       registered_function **rfn_slot
1498         = function_table->find_slot_with_hash (instance, hash, INSERT);
1499       gcc_assert (!*rfn_slot);
1500       *rfn_slot = &rfn;
1501     }
1502
1503   /* Also add the function under its overloaded alias, if we want
1504      a separate decl for each instance of an overloaded function.  */
1505   char *overload_name = get_name (instance, true);
1506   if (strcmp (name, overload_name) != 0)
1507     {
1508       /* Attribute lists shouldn't be shared.  */
1509       tree attrs = get_attributes (instance, required_extensions);
1510       bool placeholder_p = !(m_direct_overloads || force_direct_overloads);
1511       add_function (instance, overload_name, fntype, attrs,
1512                     required_extensions, false, placeholder_p);
1513     }
1514
1515   obstack_free (&m_string_obstack, name);
1516 }
1517
1518 /* Add one function decl for INSTANCE, to be used with manual overload
1519    resolution.  REQUIRED_EXTENSIONS are the set of architecture extensions
1520    that the function requires.
1521
1522    For simplicity, deal with duplicate attempts to add the same function,
1523    including cases in which the new function requires more features than
1524    the original one did.  In that case we'll check whether the required
1525    features are available as part of resolving the function to the
1526    relevant unique function.  */
1527 void
1528 function_builder::
1529 add_overloaded_function (const function_instance &instance,
1530                          aarch64_feature_flags required_extensions)
1531 {
1532   auto &name_map = overload_names[m_function_nulls];
1533   if (!name_map)
1534     name_map = hash_map<tree, registered_function *>::create_ggc ();
1535
1536   char *name = get_name (instance, true);
1537   tree id = get_identifier (name);
1538   if (registered_function **map_value = name_map->get (id))
1539     gcc_assert ((*map_value)->instance == instance
1540                 && ((*map_value)->required_extensions
1541                     & ~required_extensions) == 0);
1542   else
1543     {
1544       registered_function &rfn
1545         = add_function (instance, name, m_overload_type, NULL_TREE,
1546                         required_extensions, true, m_direct_overloads);
1547       name_map->put (id, &rfn);
1548     }
1549   obstack_free (&m_string_obstack, name);
1550 }
1551
1552 /* If we are using manual overload resolution, add one function decl
1553    for each overloaded function in GROUP.  Take the function base name
1554    from GROUP and the mode from MODE.  */
1555 void
1556 function_builder::add_overloaded_functions (const function_group_info &group,
1557                                             mode_suffix_index mode)
1558 {
1559   bool explicit_type0 = (*group.shape)->explicit_type_suffix_p (0);
1560   bool explicit_type1 = (*group.shape)->explicit_type_suffix_p (1);
1561   bool explicit_group = (*group.shape)->explicit_group_suffix_p ();
1562   auto add_function = [&](const type_suffix_pair &types,
1563                           group_suffix_index group_suffix_id,
1564                           unsigned int pi)
1565     {
1566       function_instance instance (group.base_name, *group.base,
1567                                   *group.shape, mode, types,
1568                                   group_suffix_id, group.preds[pi]);
1569       add_overloaded_function (instance, group.required_extensions);
1570     };
1571
1572   auto add_group_suffix = [&](group_suffix_index group_suffix_id,
1573                               unsigned int pi)
1574     {
1575       if (mode == MODE_single
1576           && group_suffixes[group_suffix_id].vectors_per_tuple == 1)
1577         return;
1578
1579       if (!explicit_type0 && !explicit_type1)
1580         /* Deal with the common case in which there is one overloaded
1581            function for all type combinations.  */
1582         add_function (types_none[0], group_suffix_id, pi);
1583       else
1584         for (unsigned int ti = 0; group.types[ti][0] != NUM_TYPE_SUFFIXES;
1585              ++ti)
1586           {
1587             /* Stub out the types that are determined by overload
1588                resolution.  */
1589             type_suffix_pair types = {
1590               explicit_type0 ? group.types[ti][0] : NUM_TYPE_SUFFIXES,
1591               explicit_type1 ? group.types[ti][1] : NUM_TYPE_SUFFIXES
1592             };
1593             add_function (types, group_suffix_id, pi);
1594           }
1595     };
1596
1597   for (unsigned int pi = 0; group.preds[pi] != NUM_PREDS; ++pi)
1598     if (explicit_group)
1599       for (unsigned int gi = 0; group.groups[gi] != NUM_GROUP_SUFFIXES; ++gi)
1600         add_group_suffix (group.groups[gi], pi);
1601     else
1602       add_group_suffix (GROUP_none, pi);
1603 }
1604
1605 /* Register all the functions in GROUP.  */
1606 void
1607 function_builder::register_function_group (const function_group_info &group)
1608 {
1609   (*group.shape)->build (*this, group);
1610 }
1611
1612 function_call_info::function_call_info (location_t location_in,
1613                                         const function_instance &instance_in,
1614                                         tree fndecl_in)
1615   : function_instance (instance_in), location (location_in), fndecl (fndecl_in)
1616 {
1617 }
1618
1619 function_resolver::function_resolver (location_t location,
1620                                       const function_instance &instance,
1621                                       tree fndecl, vec<tree, va_gc> &arglist)
1622   : function_call_info (location, instance, fndecl), m_arglist (arglist)
1623 {
1624 }
1625
1626 /* Return the <stdint.h> name associated with TYPE.  Using the <stdint.h>
1627    name should be more user-friendly than the underlying canonical type,
1628    since it makes the signedness and bitwidth explicit.  */
1629 const char *
1630 function_resolver::get_scalar_type_name (type_suffix_index type)
1631 {
1632   return vector_types[type_suffixes[type].vector_type].acle_name + 2;
1633 }
1634
1635 /* Return the type of argument I, or error_mark_node if it isn't
1636    well-formed.  */
1637 tree
1638 function_resolver::get_argument_type (unsigned int i)
1639 {
1640   tree arg = m_arglist[i];
1641   return arg == error_mark_node ? arg : TREE_TYPE (arg);
1642 }
1643
1644 /* Return true if argument I is some form of scalar value.  */
1645 bool
1646 function_resolver::scalar_argument_p (unsigned int i)
1647 {
1648   tree type = get_argument_type (i);
1649   return (INTEGRAL_TYPE_P (type)
1650           /* Allow pointer types, leaving the frontend to warn where
1651              necessary.  */
1652           || POINTER_TYPE_P (type)
1653           || SCALAR_FLOAT_TYPE_P (type));
1654 }
1655
1656 /* Report that argument ARGNO was expected to have NUM_VECTORS vectors.
1657    TYPE is the type that ARGNO actually has.  */
1658 void
1659 function_resolver::report_incorrect_num_vectors (unsigned int argno,
1660                                                  sve_type type,
1661                                                  unsigned int num_vectors)
1662 {
1663   if (num_vectors == 1)
1664     error_at (location, "passing %qT to argument %d of %qE, which"
1665               " expects a single SVE vector rather than a tuple",
1666               get_vector_type (type), argno + 1, fndecl);
1667   else if (type.num_vectors == 1
1668            && type.type != TYPE_SUFFIX_b)
1669     /* num_vectors is always != 1, so the singular isn't needed.  */
1670     error_n (location, num_vectors, "%qT%d%qE%d",
1671              "passing single vector %qT to argument %d"
1672              " of %qE, which expects a tuple of %d vectors",
1673              get_vector_type (type), argno + 1, fndecl, num_vectors);
1674   else
1675     /* num_vectors is always != 1, so the singular isn't needed.  */
1676     error_n (location, num_vectors, "%qT%d%qE%d",
1677              "passing %qT to argument %d of %qE, which"
1678              " expects a tuple of %d vectors", get_vector_type (type),
1679              argno + 1, fndecl, num_vectors);
1680 }
1681
1682 /* Report that arguments FIRST_ARGNO and ARGNO have different numbers
1683    of vectors, but are required to have the same number of vectors.
1684    FIRST_TYPE and TYPE are the types that arguments FIRST_ARGNO and
1685    ARGNO actually have.  */
1686 void
1687 function_resolver::report_mismatched_num_vectors (unsigned int first_argno,
1688                                                   sve_type first_type,
1689                                                   unsigned int argno,
1690                                                   sve_type type)
1691 {
1692   /* If the tuple size is implied by the group suffix, and if the first
1693      type had the right number of vectors, treat argument ARGNO as being
1694      individually wrong, rather than wrong in relation to FIRST_ARGNO.  */
1695   if (group_suffix_id != GROUP_none
1696       && first_type.num_vectors == vectors_per_tuple ())
1697     {
1698       report_incorrect_num_vectors (argno, type, first_type.num_vectors);
1699       return;
1700     }
1701
1702   /* Make sure that FIRST_TYPE itself is sensible before using it
1703      as a basis for an error message.  */
1704   if (resolve_to (mode_suffix_id, first_type) == error_mark_node)
1705     return;
1706
1707   if (type.num_vectors != 1 && first_type.num_vectors == 1)
1708     error_at (location, "passing tuple %qT to argument %d of %qE after"
1709               " passing single vector %qT to argument %d",
1710               get_vector_type (type), argno + 1, fndecl,
1711               get_vector_type (first_type), first_argno + 1);
1712   else if (type.num_vectors == 1 && first_type.num_vectors != 1)
1713     error_at (location, "passing single vector %qT to argument %d"
1714               " of %qE after passing tuple %qT to argument %d",
1715               get_vector_type (type), argno + 1, fndecl,
1716               get_vector_type (first_type), first_argno + 1);
1717   else
1718     error_at (location, "passing mismatched tuple types %qT and %qT"
1719               " to arguments %d and %d of %qE",
1720               get_vector_type (first_type), get_vector_type (type),
1721               first_argno + 1, argno + 1, fndecl);
1722 }
1723
1724 /* Report that the function has no form that takes type TYPE.
1725    Return error_mark_node.  */
1726 tree
1727 function_resolver::report_no_such_form (sve_type type)
1728 {
1729   error_at (location, "%qE has no form that takes %qT arguments",
1730             fndecl, get_vector_type (type));
1731   return error_mark_node;
1732 }
1733
1734 /* Silently check whether there is an instance of the function with the
1735    mode suffix given by MODE, the type suffixes given by TYPE0 and TYPE1,
1736    and the group suffix given by GROUP.  Return its function decl if so,
1737    otherwise return null.  */
1738 tree
1739 function_resolver::lookup_form (mode_suffix_index mode,
1740                                 type_suffix_index type0,
1741                                 type_suffix_index type1,
1742                                 group_suffix_index group)
1743 {
1744   type_suffix_pair types = { type0, type1 };
1745   function_instance instance (base_name, base, shape, mode, types,
1746                               group, pred);
1747   registered_function *rfn
1748     = function_table->find_with_hash (instance, instance.hash ());
1749   return rfn ? rfn->decl : NULL_TREE;
1750 }
1751
1752 /* Silently check whether there is an instance of the function that has the
1753    mode suffix given by MODE and the type and group suffixes implied by TYPE.
1754    If the overloaded function has an explicit first type suffix (like
1755    conversions do), TYPE describes the implicit second type suffix.
1756    Otherwise, TYPE describes the only type suffix.
1757
1758    Return the decl of the function if it exists, otherwise return null.  */
1759 tree
1760 function_resolver::lookup_form (mode_suffix_index mode, sve_type type)
1761 {
1762   type_suffix_index type0 = type_suffix_ids[0];
1763   type_suffix_index type1 = type_suffix_ids[1];
1764   (type0 == NUM_TYPE_SUFFIXES ? type0 : type1) = type.type;
1765
1766   group_suffix_index group = group_suffix_id;
1767   if (group == GROUP_none && type.num_vectors != vectors_per_tuple ())
1768     group = num_vectors_to_group (type.num_vectors);
1769
1770   return lookup_form (mode, type0, type1, group);
1771 }
1772
1773 /* Resolve the function to one with the mode suffix given by MODE, the
1774    type suffixes given by TYPE0 and TYPE1, and group suffix given by
1775    GROUP.  Return its function decl on success, otherwise report an
1776    error and return error_mark_node.  */
1777 tree
1778 function_resolver::resolve_to (mode_suffix_index mode,
1779                                type_suffix_index type0,
1780                                type_suffix_index type1,
1781                                group_suffix_index group)
1782 {
1783   tree res = lookup_form (mode, type0, type1, group);
1784   if (!res)
1785     {
1786       if (type1 == NUM_TYPE_SUFFIXES)
1787         return report_no_such_form (type0);
1788       if (type0 == type_suffix_ids[0])
1789         return report_no_such_form (type1);
1790       error_at (location, "%qE has no form that takes %qT and %qT arguments",
1791                 fndecl, get_vector_type (type0), get_vector_type (type1));
1792       return error_mark_node;
1793     }
1794   return res;
1795 }
1796
1797 /* Resolve the function to one that has the suffixes associated with MODE
1798    and TYPE; see lookup_form for how TYPE is interpreted.  Return the
1799    function decl on success, otherwise report an error and return
1800    error_mark_node.  */
1801 tree
1802 function_resolver::resolve_to (mode_suffix_index mode, sve_type type)
1803 {
1804   if (tree res = lookup_form (mode, type))
1805     return res;
1806
1807   return report_no_such_form (type);
1808 }
1809
1810 /* Like resolve_to, but used for a conversion function with the following
1811    properties:
1812
1813    - The function has an explicit first type suffix.
1814    - The elements of the argument (which has type TYPE) might be narrower
1815      or wider than the elements of the return type.
1816    - The return type has enough vectors to represent the converted value
1817      of every element.
1818    - The group suffix describes the wider of the argument type and the
1819      return type.  */
1820 tree
1821 function_resolver::resolve_conversion (mode_suffix_index mode, sve_type type)
1822 {
1823   auto ret_type = type_suffix_ids[0];
1824   unsigned int num_ret_vectors = (type.num_vectors
1825                                   * type_suffixes[ret_type].element_bits
1826                                   / type_suffixes[type.type].element_bits);
1827   if (num_ret_vectors == 1
1828       || num_ret_vectors == 2
1829       || num_ret_vectors == 4)
1830     {
1831       unsigned int num_vectors = MAX (num_ret_vectors, type.num_vectors);
1832       if (tree res = lookup_form (mode, { type.type, num_vectors }))
1833         return res;
1834     }
1835   return report_no_such_form (type);
1836 }
1837
1838 /* Require argument ARGNO to be an svbool_t or svcount_t predicate.
1839    Return its type on success, otherwise report an error and return
1840    NUM_VECTOR_TYPES.  */
1841 vector_type_index
1842 function_resolver::infer_predicate_type (unsigned int argno)
1843 {
1844   tree actual = get_argument_type (argno);
1845   if (actual == error_mark_node)
1846     return NUM_VECTOR_TYPES;
1847
1848   for (auto index : { VECTOR_TYPE_svbool_t, VECTOR_TYPE_svcount_t })
1849     if (matches_type_p (acle_vector_types[0][index], actual))
1850       return index;
1851
1852   error_at (location, "passing %qT to argument %d of %qE, which expects"
1853             " an %qs or %qs", actual, argno + 1, fndecl, "svbool_t",
1854             "svcount_t");
1855   return NUM_VECTOR_TYPES;
1856 }
1857
1858 /* Require argument ARGNO to be a 32-bit or 64-bit scalar integer type.
1859    Return the associated type suffix on success, otherwise report an
1860    error and return NUM_TYPE_SUFFIXES.  */
1861 type_suffix_index
1862 function_resolver::infer_integer_scalar_type (unsigned int argno)
1863 {
1864   tree actual = get_argument_type (argno);
1865   if (actual == error_mark_node)
1866     return NUM_TYPE_SUFFIXES;
1867
1868   /* Allow enums and booleans to decay to integers, for compatibility
1869      with C++ overloading rules.  */
1870   if (INTEGRAL_TYPE_P (actual))
1871     {
1872       bool uns_p = TYPE_UNSIGNED (actual);
1873       /* Honor the usual integer promotions, so that resolution works
1874          in the same way as for C++.  */
1875       if (TYPE_PRECISION (actual) < 32)
1876         return TYPE_SUFFIX_s32;
1877       if (TYPE_PRECISION (actual) == 32)
1878         return uns_p ? TYPE_SUFFIX_u32 : TYPE_SUFFIX_s32;
1879       if (TYPE_PRECISION (actual) == 64)
1880         return uns_p ? TYPE_SUFFIX_u64 : TYPE_SUFFIX_s64;
1881     }
1882
1883   error_at (location, "passing %qT to argument %d of %qE, which expects"
1884             " a 32-bit or 64-bit integer type", actual, argno + 1, fndecl);
1885   return NUM_TYPE_SUFFIXES;
1886 }
1887
1888 /* Return arguments ARGNO and ARGNO + 1 to be 64-bit scalar integers
1889    of the same signedness, or be a combination that converts unambiguously
1890    to such a pair.  Return the associated type suffix if they are,
1891    otherwise report an error and return NUM_TYPE_SUFFIXES.  */
1892 type_suffix_index
1893 function_resolver::infer_64bit_scalar_integer_pair (unsigned int argno)
1894 {
1895   /* Require two scalar integers, with one having 64 bits and the other
1896      one being no bigger.  */
1897   tree types[] = { get_argument_type (argno), get_argument_type (argno + 1) };
1898   if (!INTEGRAL_TYPE_P (types[0])
1899       || !INTEGRAL_TYPE_P (types[1])
1900       || MAX (TYPE_PRECISION (types[0]), TYPE_PRECISION (types[1])) != 64)
1901     {
1902       error_at (location, "passing %qT and %qT to arguments %d and %d of %qE,"
1903                 " which expects a pair of 64-bit integers", types[0], types[1],
1904                 argno + 1, argno + 2, fndecl);
1905       return NUM_TYPE_SUFFIXES;
1906     }
1907
1908   /* Allow signed integers smaller than int64_t to be paired with an int64_t.
1909      Allow unsigned integers smaller than uint64_t to be paired with any
1910      64-bit integer.  */
1911   for (int i = 0; i < 2; ++i)
1912     {
1913       if (TYPE_PRECISION (types[i]) != 64)
1914         continue;
1915
1916       if (TYPE_UNSIGNED (types[1 - i]) != TYPE_UNSIGNED (types[i]))
1917         {
1918           if (TYPE_PRECISION (types[1 - i]) == 64)
1919             continue;
1920           if (!TYPE_UNSIGNED (types[1 - i]))
1921             continue;
1922         }
1923       return TYPE_UNSIGNED (types[i]) ? TYPE_SUFFIX_u64 : TYPE_SUFFIX_s64;
1924     }
1925
1926   error_at (location, "passing mismatched integer types %qT and %qT"
1927             " to arguments %d and %d of %qE", types[0], types[1],
1928             argno + 1, argno + 2, fndecl);
1929   return NUM_TYPE_SUFFIXES;
1930 }
1931
1932 /* Require argument ARGNO to be a pointer to a scalar type that has a
1933    corresponding type suffix.  Return that type suffix on success,
1934    otherwise report an error and return NUM_TYPE_SUFFIXES.
1935    GATHER_SCATTER_P is true if the function is a gather/scatter
1936    operation, and so requires a pointer to 32-bit or 64-bit data.  */
1937 type_suffix_index
1938 function_resolver::infer_pointer_type (unsigned int argno,
1939                                        bool gather_scatter_p)
1940 {
1941   tree actual = get_argument_type (argno);
1942   if (actual == error_mark_node)
1943     return NUM_TYPE_SUFFIXES;
1944
1945   if (TREE_CODE (actual) != POINTER_TYPE)
1946     {
1947       error_at (location, "passing %qT to argument %d of %qE, which"
1948                 " expects a pointer type", actual, argno + 1, fndecl);
1949       if (VECTOR_TYPE_P (actual) && gather_scatter_p)
1950         inform (location, "an explicit type suffix is needed"
1951                 " when using a vector of base addresses");
1952       return NUM_TYPE_SUFFIXES;
1953     }
1954
1955   tree target = TREE_TYPE (actual);
1956   type_suffix_index type = find_type_suffix_for_scalar_type (target);
1957   if (type == NUM_TYPE_SUFFIXES)
1958     {
1959       error_at (location, "passing %qT to argument %d of %qE, but %qT is not"
1960                 " a valid SVE element type", actual, argno + 1, fndecl,
1961                 build_qualified_type (target, 0));
1962       return NUM_TYPE_SUFFIXES;
1963     }
1964   unsigned int bits = type_suffixes[type].element_bits;
1965   if (gather_scatter_p && bits != 32 && bits != 64)
1966     {
1967       error_at (location, "passing %qT to argument %d of %qE, which"
1968                 " expects a pointer to 32-bit or 64-bit elements",
1969                 actual, argno + 1, fndecl);
1970       return NUM_TYPE_SUFFIXES;
1971     }
1972
1973   return type;
1974 }
1975
1976 /* If TYPE is an SVE predicate or vector type, or a tuple of such a type,
1977    return the associated sve_type, otherwise return an invalid sve_type.  */
1978 static sve_type
1979 find_sve_type (const_tree type)
1980 {
1981   /* A linear search should be OK here, since the code isn't hot and
1982      the number of types is only small.  */
1983   for (unsigned int size_i = 0; size_i < MAX_TUPLE_SIZE; ++size_i)
1984     for (unsigned int suffix_i = 0; suffix_i < NUM_TYPE_SUFFIXES; ++suffix_i)
1985       {
1986         vector_type_index type_i = type_suffixes[suffix_i].vector_type;
1987         tree this_type = acle_vector_types[size_i][type_i];
1988         if (this_type && matches_type_p (this_type, type))
1989           return { type_suffix_index (suffix_i), size_i + 1 };
1990       }
1991
1992   return {};
1993 }
1994
1995 /* Require argument ARGNO to be an SVE type (i.e. something that can be
1996    represented by sve_type).  Return the (valid) type if it is, otherwise
1997    report an error and return an invalid type.  */
1998 sve_type
1999 function_resolver::infer_sve_type (unsigned int argno)
2000 {
2001   tree actual = get_argument_type (argno);
2002   if (actual == error_mark_node)
2003     return {};
2004
2005   if (sve_type type = find_sve_type (actual))
2006     return type;
2007
2008   if (scalar_argument_p (argno))
2009     error_at (location, "passing %qT to argument %d of %qE, which"
2010               " expects an SVE type rather than a scalar type",
2011               actual, argno + 1, fndecl);
2012   else
2013     error_at (location, "passing %qT to argument %d of %qE, which"
2014               " expects an SVE type",
2015               actual, argno + 1, fndecl);
2016   return {};
2017 }
2018
2019 /* Require argument ARGNO to be a single vector or a tuple of NUM_VECTORS
2020    vectors; NUM_VECTORS is 1 for the former.  Return the associated type
2021    on success.  Report an error on failure.  */
2022 sve_type
2023 function_resolver::infer_vector_or_tuple_type (unsigned int argno,
2024                                                unsigned int num_vectors)
2025 {
2026   auto type = infer_sve_type (argno);
2027   if (!type)
2028     return type;
2029
2030   if (type.num_vectors == num_vectors)
2031     return type;
2032
2033   report_incorrect_num_vectors (argno, type, num_vectors);
2034   return {};
2035 }
2036
2037 /* Require argument ARGNO to have some form of vector type.  Return the
2038    associated type suffix on success, using TYPE_SUFFIX_b for predicates.
2039    Report an error and return NUM_TYPE_SUFFIXES on failure.  */
2040 type_suffix_index
2041 function_resolver::infer_vector_type (unsigned int argno)
2042 {
2043   if (auto type = infer_vector_or_tuple_type (argno, 1))
2044     return type.type;
2045   return NUM_TYPE_SUFFIXES;
2046 }
2047
2048 /* Like infer_vector_type, but also require the type to be integral.  */
2049 type_suffix_index
2050 function_resolver::infer_integer_vector_type (unsigned int argno)
2051 {
2052   type_suffix_index type = infer_vector_type (argno);
2053   if (type == NUM_TYPE_SUFFIXES)
2054     return type;
2055
2056   if (!type_suffixes[type].integer_p)
2057     {
2058       error_at (location, "passing %qT to argument %d of %qE, which"
2059                 " expects a vector of integers", get_argument_type (argno),
2060                 argno + 1, fndecl);
2061       return NUM_TYPE_SUFFIXES;
2062     }
2063
2064   return type;
2065 }
2066
2067 /* Require argument ARGNO to have some form of NEON128 vector type.  Return the
2068    associated type suffix on success.
2069    Report an error and return NUM_TYPE_SUFFIXES on failure.  */
2070 type_suffix_index
2071 function_resolver::infer_neon128_vector_type (unsigned int argno)
2072 {
2073   tree actual = get_argument_type (argno);
2074   if (actual == error_mark_node)
2075     return NUM_TYPE_SUFFIXES;
2076
2077   for (unsigned int suffix_i = 0; suffix_i < NUM_TYPE_SUFFIXES; ++suffix_i)
2078     {
2079       int neon_index = type_suffixes[suffix_i].neon128_type;
2080       if (neon_index != ARM_NEON_H_TYPES_LAST)
2081         {
2082           tree type = aarch64_simd_types[neon_index].itype;
2083           if (type && matches_type_p (type, actual))
2084             return type_suffix_index (suffix_i);
2085         }
2086     }
2087
2088   error_at (location, "passing %qT to argument %d of %qE, which"
2089             " expects a 128 bit NEON vector type", actual, argno + 1, fndecl);
2090   return NUM_TYPE_SUFFIXES;
2091 }
2092
2093
2094 /* Like infer_vector_type, but also require the type to be an unsigned
2095    integer.  */
2096 type_suffix_index
2097 function_resolver::infer_unsigned_vector_type (unsigned int argno)
2098 {
2099   type_suffix_index type = infer_vector_type (argno);
2100   if (type == NUM_TYPE_SUFFIXES)
2101     return type;
2102
2103   if (!type_suffixes[type].unsigned_p)
2104     {
2105       error_at (location, "passing %qT to argument %d of %qE, which"
2106                 " expects a vector of unsigned integers",
2107                 get_argument_type (argno), argno + 1, fndecl);
2108       return NUM_TYPE_SUFFIXES;
2109     }
2110
2111   return type;
2112 }
2113
2114 /* Like infer_vector_type, but also require the element size to be
2115    32 or 64 bits.  */
2116 type_suffix_index
2117 function_resolver::infer_sd_vector_type (unsigned int argno)
2118 {
2119   type_suffix_index type = infer_vector_type (argno);
2120   if (type == NUM_TYPE_SUFFIXES)
2121     return type;
2122
2123   unsigned int bits = type_suffixes[type].element_bits;
2124   if (bits != 32 && bits != 64)
2125     {
2126       error_at (location, "passing %qT to argument %d of %qE, which"
2127                 " expects a vector of 32-bit or 64-bit elements",
2128                 get_argument_type (argno), argno + 1, fndecl);
2129       return NUM_TYPE_SUFFIXES;
2130     }
2131
2132   return type;
2133 }
2134
2135 /* If the function operates on tuples of vectors, require argument ARGNO to be
2136    a tuple with the appropriate number of vectors, otherwise require it to be
2137    a single vector.  Return the associated type on success.  Report an error
2138    on failure.  */
2139 sve_type
2140 function_resolver::infer_tuple_type (unsigned int argno)
2141 {
2142   return infer_vector_or_tuple_type (argno, vectors_per_tuple ());
2143 }
2144
2145 /* PRED_TYPE is the type of a governing predicate argument and DATA_TYPE
2146    is the type of an argument that it predicates.  Require the two types
2147    to "agree": svcount_t must be used for multiple vectors and svbool_t
2148    for single vectors.
2149
2150    Return true if they do agree, otherwise report an error and
2151    return false.  */
2152 bool function_resolver::
2153 require_matching_predicate_type (vector_type_index pred_type,
2154                                  sve_type data_type)
2155 {
2156   if (pred_type == VECTOR_TYPE_svbool_t && data_type.num_vectors == 1)
2157     return true;
2158
2159   if (pred_type == VECTOR_TYPE_svcount_t && data_type.num_vectors != 1)
2160     return true;
2161
2162   /* Make sure that FIRST_TYPE itself is sensible before using it
2163      as a basis for an error message.  */
2164   if (resolve_to (mode_suffix_id, data_type) == error_mark_node)
2165     return false;
2166
2167   if (data_type.num_vectors > 1)
2168     error_at (location, "operations on multiple vectors must be predicated"
2169               " by %qs rather than %qs", "svcount_t", "svbool_t");
2170   else
2171     error_at (location, "operations on single vectors must be predicated"
2172               " by %qs rather than %qs", "svbool_t", "svcount_t");
2173   return false;
2174 }
2175
2176 /* Require argument ARGNO to be a vector or scalar argument.  Return true
2177    if it is, otherwise report an appropriate error.  */
2178 bool
2179 function_resolver::require_vector_or_scalar_type (unsigned int argno)
2180 {
2181   tree actual = get_argument_type (argno);
2182   if (actual == error_mark_node)
2183     return false;
2184
2185   if (!scalar_argument_p (argno) && !VECTOR_TYPE_P (actual))
2186     {
2187       error_at (location, "passing %qT to argument %d of %qE, which"
2188                 " expects a vector or scalar type", actual, argno + 1, fndecl);
2189       return false;
2190     }
2191
2192   return true;
2193 }
2194
2195 /* Require argument ARGNO to have vector type TYPE, in cases where this
2196    requirement holds for all uses of the function.  Return true if the
2197    argument has the right form, otherwise report an appropriate error.  */
2198 bool
2199 function_resolver::require_vector_type (unsigned int argno,
2200                                         vector_type_index type)
2201 {
2202   tree expected = acle_vector_types[0][type];
2203   tree actual = get_argument_type (argno);
2204   if (actual == error_mark_node)
2205     return false;
2206
2207   if (!matches_type_p (expected, actual))
2208     {
2209       error_at (location, "passing %qT to argument %d of %qE, which"
2210                 " expects %qT", actual, argno + 1, fndecl, expected);
2211       return false;
2212     }
2213   return true;
2214 }
2215
2216 /* Like require_vector_type, but TYPE is inferred from argument FIRST_ARGNO
2217    rather than being a fixed part of the function signature.  This changes
2218    the nature of the error messages.  */
2219 bool
2220 function_resolver::require_matching_vector_type (unsigned int argno,
2221                                                  unsigned int first_argno,
2222                                                  sve_type type)
2223 {
2224   sve_type new_type = infer_sve_type (argno);
2225   if (!new_type)
2226     return false;
2227
2228   if (type.num_vectors != new_type.num_vectors)
2229     {
2230       report_mismatched_num_vectors (first_argno, type, argno, new_type);
2231       return false;
2232     }
2233
2234   if (type != new_type)
2235     {
2236       error_at (location, "passing %qT to argument %d of %qE, but"
2237                 " argument %d had type %qT",
2238                 get_vector_type (new_type), argno + 1, fndecl,
2239                 first_argno + 1, get_vector_type (type));
2240       return false;
2241     }
2242   return true;
2243 }
2244
2245 /* Require argument ARGNO to be a vector or tuple type with the following
2246    properties:
2247
2248    - the type class must be the same as FIRST_TYPE's if EXPECTED_TCLASS
2249      is SAME_TYPE_CLASS, otherwise it must be EXPECTED_TCLASS itself.
2250
2251    - the element size must be:
2252
2253      - the same as FIRST_TYPE's if EXPECTED_BITS == SAME_SIZE
2254      - half of FIRST_TYPE's if EXPECTED_BITS == HALF_SIZE
2255      - a quarter of FIRST_TYPE's if EXPECTED_BITS == QUARTER_SIZE
2256      - EXPECTED_BITS itself otherwise
2257
2258    - the number of vectors must be the same as FIRST_TYPE's if
2259      EXPECTED_NUM_VECTORS is zero, otherwise it must be EXPECTED_NUM_VECTORS.
2260
2261    Return true if the argument has the required type, otherwise report
2262    an appropriate error.
2263
2264    FIRST_ARGNO is the first argument that is known to have type FIRST_TYPE.
2265    Usually it comes before ARGNO, but sometimes it is more natural to resolve
2266    arguments out of order.
2267
2268    If the required properties depend on FIRST_TYPE then both FIRST_ARGNO and
2269    ARGNO contribute to the resolution process.  If the required properties
2270    are fixed, only FIRST_ARGNO contributes to the resolution process.
2271
2272    This function is a bit of a Swiss army knife.  The complication comes
2273    from trying to give good error messages when FIRST_ARGNO and ARGNO are
2274    inconsistent, since either of them might be wrong.  */
2275 bool function_resolver::
2276 require_derived_vector_type (unsigned int argno,
2277                              unsigned int first_argno,
2278                              sve_type first_type,
2279                              type_class_index expected_tclass,
2280                              unsigned int expected_bits,
2281                              unsigned int expected_num_vectors)
2282 {
2283   /* If the type needs to match FIRST_ARGNO exactly, use the preferred
2284      error message for that case.  */
2285   if (expected_tclass == SAME_TYPE_CLASS
2286       && expected_bits == SAME_SIZE
2287       && expected_num_vectors == 0)
2288     {
2289       /* There's no need to resolve this case out of order.  */
2290       gcc_assert (argno > first_argno);
2291       return require_matching_vector_type (argno, first_argno, first_type);
2292     }
2293
2294   /* Use FIRST_TYPE to get the expected type class and element size.  */
2295   auto &first_type_suffix = type_suffixes[first_type.type];
2296   type_class_index orig_expected_tclass = expected_tclass;
2297   if (expected_tclass == NUM_TYPE_CLASSES)
2298     expected_tclass = first_type_suffix.tclass;
2299
2300   unsigned int orig_expected_bits = expected_bits;
2301   if (expected_bits == SAME_SIZE)
2302     expected_bits = first_type_suffix.element_bits;
2303   else if (expected_bits == HALF_SIZE)
2304     expected_bits = first_type_suffix.element_bits / 2;
2305   else if (expected_bits == QUARTER_SIZE)
2306     expected_bits = first_type_suffix.element_bits / 4;
2307
2308   unsigned int orig_expected_num_vectors = expected_num_vectors;
2309   if (expected_num_vectors == 0)
2310     expected_num_vectors = first_type.num_vectors;
2311
2312   /* If the expected type doesn't depend on FIRST_TYPE at all,
2313      just check for the fixed choice of vector type.  */
2314   if (expected_tclass == orig_expected_tclass
2315       && expected_bits == orig_expected_bits
2316       && orig_expected_num_vectors == 1)
2317     {
2318       const type_suffix_info &expected_suffix
2319         = type_suffixes[find_type_suffix (expected_tclass, expected_bits)];
2320       return require_vector_type (argno, expected_suffix.vector_type);
2321     }
2322
2323   /* Require the argument to be some form of SVE vector type,
2324      without being specific about the type of vector we want.  */
2325   sve_type actual_type = infer_sve_type (argno);
2326   if (!actual_type)
2327     return false;
2328
2329   if (actual_type.num_vectors != expected_num_vectors)
2330     {
2331       if (orig_expected_num_vectors == 0)
2332         report_mismatched_num_vectors (first_argno, first_type,
2333                                        argno, actual_type);
2334       else
2335         report_incorrect_num_vectors (argno, actual_type,
2336                                       expected_num_vectors);
2337       return false;
2338     }
2339
2340   if (orig_expected_tclass == SAME_TYPE_CLASS
2341       && orig_expected_bits == SAME_SIZE)
2342     {
2343       if (actual_type.type == first_type.type)
2344         return true;
2345
2346       if (first_type.num_vectors > 1)
2347         error_at (location, "passing %qT to argument %d of %qE, but"
2348                   " argument %d was a tuple of %qT",
2349                   get_vector_type (actual_type), argno + 1, fndecl,
2350                   first_argno + 1, get_vector_type (first_type.type));
2351       else
2352         error_at (location, "passing %qT to argument %d of %qE, but"
2353                   " argument %d had type %qT",
2354                   get_vector_type (actual_type), argno + 1, fndecl,
2355                   first_argno + 1, get_vector_type (first_type));
2356       return false;
2357     }
2358
2359   /* Exit now if we got the right type.  */
2360   auto &actual_type_suffix = type_suffixes[actual_type.type];
2361   bool tclass_ok_p = (actual_type_suffix.tclass == expected_tclass);
2362   bool size_ok_p = (actual_type_suffix.element_bits == expected_bits);
2363   if (tclass_ok_p && size_ok_p)
2364     return true;
2365
2366   /* First look for cases in which the actual type contravenes a fixed
2367      size requirement, without having to refer to FIRST_TYPE.  */
2368   if (!size_ok_p && expected_bits == orig_expected_bits)
2369     {
2370       if (expected_num_vectors == 1)
2371         error_at (location, "passing %qT to argument %d of %qE, which"
2372                   " expects a vector of %d-bit elements",
2373                   get_vector_type (actual_type), argno + 1, fndecl,
2374                   expected_bits);
2375       else
2376         error_at (location, "passing %qT to argument %d of %qE, which"
2377                   " expects vectors of %d-bit elements",
2378                   get_vector_type (actual_type), argno + 1, fndecl,
2379                   expected_bits);
2380       return false;
2381     }
2382
2383   /* Likewise for a fixed type class requirement.  This is only ever
2384      needed for signed and unsigned types, so don't create unnecessary
2385      translation work for other type classes.  */
2386   if (!tclass_ok_p && orig_expected_tclass == TYPE_signed)
2387     {
2388       if (expected_num_vectors == 1)
2389         error_at (location, "passing %qT to argument %d of %qE, which"
2390                   " expects a vector of signed integers",
2391                   get_vector_type (actual_type), argno + 1, fndecl);
2392       else
2393         /* Translation note: could also be written "expects a tuple of
2394            signed integer vectors".  */
2395         error_at (location, "passing %qT to argument %d of %qE, which"
2396                   " expects vectors of signed integers",
2397                   get_vector_type (actual_type), argno + 1, fndecl);
2398       return false;
2399     }
2400   if (!tclass_ok_p && orig_expected_tclass == TYPE_unsigned)
2401     {
2402       if (expected_num_vectors == 1)
2403         error_at (location, "passing %qT to argument %d of %qE, which"
2404                   " expects a vector of unsigned integers",
2405                   get_vector_type (actual_type), argno + 1, fndecl);
2406       else
2407         /* Translation note: could also be written "expects a tuple of
2408            unsigned integer vectors".  */
2409         error_at (location, "passing %qT to argument %d of %qE, which"
2410                   " expects vectors of unsigned integers",
2411                   get_vector_type (actual_type), argno + 1, fndecl);
2412       return false;
2413     }
2414
2415   /* Make sure that FIRST_TYPE itself is sensible before using it
2416      as a basis for an error message.  */
2417   if (resolve_to (mode_suffix_id, first_type) == error_mark_node)
2418     return false;
2419
2420   /* If the arguments have consistent type classes, but a link between
2421      the sizes has been broken, try to describe the error in those terms.  */
2422   if (tclass_ok_p && orig_expected_bits == SAME_SIZE)
2423     {
2424       if (argno < first_argno)
2425         {
2426           std::swap (argno, first_argno);
2427           std::swap (actual_type, first_type);
2428         }
2429       error_at (location, "arguments %d and %d of %qE must have the"
2430                 " same element size, but the values passed here have type"
2431                 " %qT and %qT respectively", first_argno + 1, argno + 1,
2432                 fndecl, get_vector_type (first_type),
2433                 get_vector_type (actual_type));
2434       return false;
2435     }
2436
2437   /* Likewise in reverse: look for cases in which the sizes are consistent
2438      but a link between the type classes has been broken.  */
2439   if (size_ok_p
2440       && orig_expected_tclass == SAME_TYPE_CLASS
2441       && first_type_suffix.integer_p
2442       && actual_type_suffix.integer_p)
2443     {
2444       if (argno < first_argno)
2445         {
2446           std::swap (argno, first_argno);
2447           std::swap (actual_type, first_type);
2448         }
2449       error_at (location, "arguments %d and %d of %qE must have the"
2450                 " same signedness, but the values passed here have type"
2451                 " %qT and %qT respectively", first_argno + 1, argno + 1,
2452                 fndecl, get_vector_type (first_type),
2453                 get_vector_type (actual_type));
2454       return false;
2455     }
2456
2457   /* The two arguments are wildly inconsistent.  */
2458   type_suffix_index expected_type
2459     = find_type_suffix (expected_tclass, expected_bits);
2460   error_at (location, "passing %qT instead of the expected %qT to argument"
2461             " %d of %qE, after passing %qT to argument %d",
2462             get_vector_type (actual_type), get_vector_type (expected_type),
2463             argno + 1, fndecl, get_argument_type (first_argno),
2464             first_argno + 1);
2465   return false;
2466 }
2467
2468 /* Require argument ARGNO to match argument FIRST_ARGNO, which was inferred
2469    to be a pointer to a scalar element of type TYPE.  */
2470 bool
2471 function_resolver::require_matching_pointer_type (unsigned int argno,
2472                                                   unsigned int first_argno,
2473                                                   type_suffix_index type)
2474 {
2475   type_suffix_index new_type = infer_pointer_type (argno);
2476   if (new_type == NUM_TYPE_SUFFIXES)
2477     return false;
2478
2479   if (type != new_type)
2480     {
2481       error_at (location, "passing %qT to argument %d of %qE, but"
2482                 " argument %d had type %qT", get_argument_type (argno),
2483                 argno + 1, fndecl, first_argno + 1,
2484                 get_argument_type (first_argno));
2485       return false;
2486     }
2487   return true;
2488 }
2489
2490 /* Require argument ARGNO to be a (possibly variable) scalar, using EXPECTED
2491    as the name of its expected type.  Return true if the argument has the
2492    right form, otherwise report an appropriate error.  */
2493 bool
2494 function_resolver::require_scalar_type (unsigned int argno,
2495                                         const char *expected)
2496 {
2497   if (!scalar_argument_p (argno))
2498     {
2499       if (expected)
2500         error_at (location, "passing %qT to argument %d of %qE, which"
2501                   " expects %qs", get_argument_type (argno), argno + 1,
2502                   fndecl, expected);
2503       return false;
2504     }
2505   return true;
2506 }
2507
2508 /* Require argument ARGNO to be a nonscalar type, given that it has already
2509    passed require_vector_or_scalar_type.  Return true if it is, otherwise
2510    report an error.  This is used when two sets of instructions share the
2511    same overloaded function and one accepts scalars while the other
2512    doesn't.  */
2513 bool
2514 function_resolver::require_nonscalar_type (unsigned int argno)
2515 {
2516   if (scalar_argument_p (argno))
2517     {
2518       error_at (location, "passing %qT to argument %d of %qE, which"
2519                 " does not accept scalars for this combination of arguments",
2520                 get_argument_type (argno), argno + 1, fndecl);
2521       return false;
2522     }
2523   return true;
2524 }
2525
2526 /* Require argument ARGNO to be some form of pointer, without being specific
2527    about its target type.  Return true if the argument has the right form,
2528    otherwise report an appropriate error.  */
2529 bool
2530 function_resolver::require_pointer_type (unsigned int argno)
2531 {
2532   if (!scalar_argument_p (argno))
2533     {
2534       error_at (location, "passing %qT to argument %d of %qE, which"
2535                 " expects a scalar pointer", get_argument_type (argno),
2536                 argno + 1, fndecl);
2537       return false;
2538     }
2539   return true;
2540 }
2541
2542 /* Argument FIRST_ARGNO is a scalar with type EXPECTED_TYPE, and argument
2543    ARGNO should be consistent with it.  Return true if it is, otherwise
2544    report an appropriate error.  */
2545 bool function_resolver::
2546 require_matching_integer_scalar_type (unsigned int argno,
2547                                       unsigned int first_argno,
2548                                       type_suffix_index expected_type)
2549 {
2550   type_suffix_index actual_type = infer_integer_scalar_type (argno);
2551   if (actual_type == NUM_TYPE_SUFFIXES)
2552     return false;
2553
2554   if (actual_type == expected_type)
2555     return true;
2556
2557   error_at (location, "call to %qE is ambiguous; argument %d has type"
2558             " %qs but argument %d has type %qs", fndecl,
2559             first_argno + 1, get_scalar_type_name (expected_type),
2560             argno + 1, get_scalar_type_name (actual_type));
2561   return false;
2562 }
2563
2564 /* Require argument ARGNO to be a (possibly variable) scalar, expecting it
2565    to have the following properties:
2566
2567    - the type class must be the same as for type suffix 0 if EXPECTED_TCLASS
2568      is SAME_TYPE_CLASS, otherwise it must be EXPECTED_TCLASS itself.
2569
2570    - the element size must be the same as for type suffix 0 if EXPECTED_BITS
2571      is SAME_TYPE_SIZE, otherwise it must be EXPECTED_BITS itself.
2572
2573    Return true if the argument is valid, otherwise report an appropriate error.
2574
2575    Note that we don't check whether the scalar type actually has the required
2576    properties, since that's subject to implicit promotions and conversions.
2577    Instead we just use the expected properties to tune the error message.  */
2578 bool function_resolver::
2579 require_derived_scalar_type (unsigned int argno,
2580                              type_class_index expected_tclass,
2581                              unsigned int expected_bits)
2582 {
2583   gcc_assert (expected_tclass == SAME_TYPE_CLASS
2584               || expected_tclass == TYPE_signed
2585               || expected_tclass == TYPE_unsigned);
2586
2587   /* If the expected type doesn't depend on the type suffix at all,
2588      just check for the fixed choice of scalar type.  */
2589   if (expected_tclass != SAME_TYPE_CLASS && expected_bits != SAME_SIZE)
2590     {
2591       type_suffix_index expected_type
2592         = find_type_suffix (expected_tclass, expected_bits);
2593       return require_scalar_type (argno, get_scalar_type_name (expected_type));
2594     }
2595
2596   if (scalar_argument_p (argno))
2597     return true;
2598
2599   if (expected_tclass == SAME_TYPE_CLASS)
2600     /* It doesn't really matter whether the element is expected to be
2601        the same size as type suffix 0.  */
2602     error_at (location, "passing %qT to argument %d of %qE, which"
2603               " expects a scalar element", get_argument_type (argno),
2604               argno + 1, fndecl);
2605   else
2606     /* It doesn't seem useful to distinguish between signed and unsigned
2607        scalars here.  */
2608     error_at (location, "passing %qT to argument %d of %qE, which"
2609               " expects a scalar integer", get_argument_type (argno),
2610               argno + 1, fndecl);
2611   return false;
2612 }
2613
2614 /* Require argument ARGNO to be suitable for an integer constant expression.
2615    Return true if it is, otherwise report an appropriate error.
2616
2617    function_checker checks whether the argument is actually constant and
2618    has a suitable range.  The reason for distinguishing immediate arguments
2619    here is because it provides more consistent error messages than
2620    require_scalar_type would.  */
2621 bool
2622 function_resolver::require_integer_immediate (unsigned int argno)
2623 {
2624   if (!scalar_argument_p (argno))
2625     {
2626       report_non_ice (location, fndecl, argno);
2627       return false;
2628     }
2629   return true;
2630 }
2631
2632 /* Require argument ARGNO to be a vector base in a gather-style address.
2633    Return its type on success, otherwise return NUM_VECTOR_TYPES.  */
2634 vector_type_index
2635 function_resolver::infer_vector_base_type (unsigned int argno)
2636 {
2637   type_suffix_index type = infer_vector_type (argno);
2638   if (type == NUM_TYPE_SUFFIXES)
2639     return NUM_VECTOR_TYPES;
2640
2641   if (type == TYPE_SUFFIX_u32 || type == TYPE_SUFFIX_u64)
2642     return type_suffixes[type].vector_type;
2643
2644   error_at (location, "passing %qT to argument %d of %qE, which"
2645             " expects %qs or %qs", get_argument_type (argno),
2646             argno + 1, fndecl, "svuint32_t", "svuint64_t");
2647   return NUM_VECTOR_TYPES;
2648 }
2649
2650 /* Require argument ARGNO to be a vector displacement in a gather-style
2651    address.  Return its type on success, otherwise return NUM_VECTOR_TYPES.  */
2652 vector_type_index
2653 function_resolver::infer_vector_displacement_type (unsigned int argno)
2654 {
2655   type_suffix_index type = infer_integer_vector_type (argno);
2656   if (type == NUM_TYPE_SUFFIXES)
2657     return NUM_VECTOR_TYPES;
2658
2659   if (type_suffixes[type].integer_p
2660       && (type_suffixes[type].element_bits == 32
2661           || type_suffixes[type].element_bits == 64))
2662     return type_suffixes[type].vector_type;
2663
2664   error_at (location, "passing %qT to argument %d of %qE, which"
2665             " expects a vector of 32-bit or 64-bit integers",
2666             get_argument_type (argno), argno + 1, fndecl);
2667   return NUM_VECTOR_TYPES;
2668 }
2669
2670 /* Require argument ARGNO to be a vector displacement in a gather-style
2671    address.  There are three possible uses:
2672
2673    - for loading into elements of type TYPE (when LOAD_P is true)
2674    - for storing from elements of type TYPE (when LOAD_P is false)
2675    - for prefetching data (when TYPE is NUM_TYPE_SUFFIXES)
2676
2677    The overloaded function's mode suffix determines the units of the
2678    displacement (bytes for "_offset", elements for "_index").
2679
2680    Return the associated mode on success, otherwise report an error
2681    and return MODE_none.  */
2682 mode_suffix_index
2683 function_resolver::resolve_sv_displacement (unsigned int argno,
2684                                             type_suffix_index type,
2685                                             bool load_p)
2686 {
2687   if (type == NUM_TYPE_SUFFIXES)
2688     {
2689       /* For prefetches, the base is a void pointer and the displacement
2690          can be any valid offset or index type.  */
2691       vector_type_index displacement_vector_type
2692         = infer_vector_displacement_type (argno);
2693       if (displacement_vector_type == NUM_VECTOR_TYPES)
2694         return MODE_none;
2695
2696       mode_suffix_index mode = find_mode_suffix (NUM_VECTOR_TYPES,
2697                                                  displacement_vector_type,
2698                                                  displacement_units ());
2699       gcc_assert (mode != MODE_none);
2700       return mode;
2701     }
2702
2703   unsigned int required_bits = type_suffixes[type].element_bits;
2704   if (required_bits == 32
2705       && displacement_units () == UNITS_elements
2706       && !lookup_form (MODE_s32index, type)
2707       && !lookup_form (MODE_u32index, type))
2708     {
2709       if (lookup_form (MODE_u32base_index, type))
2710         {
2711           if (type_suffix_ids[0] == NUM_TYPE_SUFFIXES)
2712             {
2713               gcc_assert (!load_p);
2714               error_at (location, "when storing %qT, %qE requires a vector"
2715                         " base and a scalar index", get_vector_type (type),
2716                         fndecl);
2717             }
2718           else
2719             error_at (location, "%qE requires a vector base and a scalar"
2720                       " index", fndecl);
2721         }
2722       else
2723         error_at (location, "%qE does not support 32-bit vector type %qT",
2724                   fndecl, get_vector_type (type));
2725       return MODE_none;
2726     }
2727
2728   /* Check for some form of vector type, without naming any in particular
2729      as being expected.  */
2730   type_suffix_index displacement_type = infer_vector_type (argno);
2731   if (displacement_type == NUM_TYPE_SUFFIXES)
2732     return MODE_none;
2733
2734   /* If the displacement type is consistent with the data vector type,
2735      try to find the associated mode suffix.  This will fall through
2736      for non-integral displacement types.  */
2737   if (type_suffixes[displacement_type].element_bits == required_bits)
2738     {
2739       vector_type_index displacement_vector_type
2740         = type_suffixes[displacement_type].vector_type;
2741       mode_suffix_index mode = find_mode_suffix (NUM_VECTOR_TYPES,
2742                                                  displacement_vector_type,
2743                                                  displacement_units ());
2744       if (mode != MODE_none)
2745         {
2746           if (mode == MODE_s32offset
2747               && !lookup_form (mode, type)
2748               && lookup_form (MODE_u32offset, type))
2749             {
2750               if (type_suffix_ids[0] == NUM_TYPE_SUFFIXES)
2751                 error_at (location, "%qE does not support 32-bit sign-extended"
2752                           " offsets", fndecl);
2753               else
2754                 error_at (location, "%qE does not support sign-extended"
2755                           " offsets", fndecl);
2756               return MODE_none;
2757             }
2758           return mode;
2759         }
2760     }
2761
2762   if (type_suffix_ids[0] == NUM_TYPE_SUFFIXES)
2763     {
2764       /* TYPE has been inferred rather than specified by the user,
2765          so mention it in the error messages.  */
2766       if (load_p)
2767         error_at (location, "passing %qT to argument %d of %qE, which when"
2768                   " loading %qT expects a vector of %d-bit integers",
2769                   get_argument_type (argno), argno + 1, fndecl,
2770                   get_vector_type (type), required_bits);
2771       else
2772         error_at (location, "passing %qT to argument %d of %qE, which when"
2773                   " storing %qT expects a vector of %d-bit integers",
2774                   get_argument_type (argno), argno + 1, fndecl,
2775                   get_vector_type (type), required_bits);
2776     }
2777   else
2778     /* TYPE is part of the function name.  */
2779     error_at (location, "passing %qT to argument %d of %qE, which"
2780               " expects a vector of %d-bit integers",
2781               get_argument_type (argno), argno + 1, fndecl, required_bits);
2782   return MODE_none;
2783 }
2784
2785 /* Require the arguments starting at ARGNO to form a gather-style address.
2786    There are three possible uses:
2787
2788    - for loading into elements of type TYPE (when LOAD_P is true)
2789    - for storing from elements of type TYPE (when LOAD_P is false)
2790    - for prefetching data (when TYPE is NUM_TYPE_SUFFIXES)
2791
2792    The three possible addresses are:
2793
2794    - a vector base with no displacement
2795    - a vector base and a scalar displacement
2796    - a scalar (pointer) base and a vector displacement
2797
2798    The overloaded function's mode suffix determines whether there is
2799    a displacement, and if so, what units it uses:
2800
2801    - MODE_none: no displacement
2802    - MODE_offset: the displacement is measured in bytes
2803    - MODE_index: the displacement is measured in elements
2804
2805    Return the mode of the non-overloaded function on success, otherwise
2806    report an error and return MODE_none.  */
2807 mode_suffix_index
2808 function_resolver::resolve_gather_address (unsigned int argno,
2809                                            type_suffix_index type,
2810                                            bool load_p)
2811 {
2812   tree actual = get_argument_type (argno);
2813   if (actual == error_mark_node)
2814     return MODE_none;
2815
2816   if (displacement_units () != UNITS_none)
2817     {
2818       /* Some form of displacement is needed.  First handle a scalar
2819          pointer base and a vector displacement.  */
2820       if (scalar_argument_p (argno))
2821         /* Don't check the pointer type here, since there's only one valid
2822            choice.  Leave that to the frontend.  */
2823         return resolve_sv_displacement (argno + 1, type, load_p);
2824
2825       if (!VECTOR_TYPE_P (actual))
2826         {
2827           error_at (location, "passing %qT to argument %d of %qE,"
2828                     " which expects a vector or pointer base address",
2829                     actual, argno + 1, fndecl);
2830           return MODE_none;
2831         }
2832     }
2833
2834   /* Check for the correct choice of vector base type.  */
2835   vector_type_index base_vector_type;
2836   if (type == NUM_TYPE_SUFFIXES)
2837     {
2838       /* Since prefetches have no type suffix, there is a free choice
2839          between 32-bit and 64-bit base addresses.  */
2840       base_vector_type = infer_vector_base_type (argno);
2841       if (base_vector_type == NUM_VECTOR_TYPES)
2842         return MODE_none;
2843     }
2844   else
2845     {
2846       /* Check for some form of vector type, without saying which type
2847          we expect.  */
2848       type_suffix_index base_type = infer_vector_type (argno);
2849       if (base_type == NUM_TYPE_SUFFIXES)
2850         return MODE_none;
2851
2852       /* Check whether the type is the right one.  */
2853       unsigned int required_bits = type_suffixes[type].element_bits;
2854       gcc_assert (required_bits == 32 || required_bits == 64);
2855       type_suffix_index required_type = (required_bits == 32
2856                                          ? TYPE_SUFFIX_u32
2857                                          : TYPE_SUFFIX_u64);
2858       if (required_type != base_type)
2859         {
2860           error_at (location, "passing %qT to argument %d of %qE,"
2861                     " which expects %qT", actual, argno + 1, fndecl,
2862                     get_vector_type (required_type));
2863           return MODE_none;
2864         }
2865       base_vector_type = type_suffixes[base_type].vector_type;
2866     }
2867
2868   /* Check the scalar displacement, if any.  */
2869   if (displacement_units () != UNITS_none
2870       && !require_scalar_type (argno + 1, "int64_t"))
2871     return MODE_none;
2872
2873   /* Find the appropriate mode suffix.  The checks above should have
2874      weeded out all erroneous cases.  */
2875   for (unsigned int mode_i = 0; mode_i < ARRAY_SIZE (mode_suffixes); ++mode_i)
2876     {
2877       const mode_suffix_info &mode = mode_suffixes[mode_i];
2878       if (mode.base_vector_type == base_vector_type
2879           && mode.displacement_vector_type == NUM_VECTOR_TYPES
2880           && mode.displacement_units == displacement_units ())
2881         return mode_suffix_index (mode_i);
2882     }
2883
2884   gcc_unreachable ();
2885 }
2886
2887 /* Require arguments ARGNO and ARGNO + 1 to form an ADR-style address,
2888    i.e. one with a vector of base addresses and a vector of displacements.
2889    The overloaded function's mode suffix determines the units of the
2890    displacement (bytes for "_offset", elements for "_index").
2891
2892    Return the associated mode suffix on success, otherwise report
2893    an error and return MODE_none.  */
2894 mode_suffix_index
2895 function_resolver::resolve_adr_address (unsigned int argno)
2896 {
2897   vector_type_index base_type = infer_vector_base_type (argno);
2898   if (base_type == NUM_VECTOR_TYPES)
2899     return MODE_none;
2900
2901   vector_type_index displacement_type
2902     = infer_vector_displacement_type (argno + 1);
2903   if (displacement_type == NUM_VECTOR_TYPES)
2904     return MODE_none;
2905
2906   mode_suffix_index mode = find_mode_suffix (base_type, displacement_type,
2907                                              displacement_units ());
2908   if (mode == MODE_none)
2909     {
2910       if (mode_suffix_id == MODE_offset)
2911         error_at (location, "cannot combine a base of type %qT with"
2912                   " an offset of type %qT",
2913                   get_argument_type (argno), get_argument_type (argno + 1));
2914       else
2915         error_at (location, "cannot combine a base of type %qT with"
2916                   " an index of type %qT",
2917                   get_argument_type (argno), get_argument_type (argno + 1));
2918     }
2919   return mode;
2920 }
2921
2922 /* Require the function to have exactly EXPECTED arguments.  Return true
2923    if it does, otherwise report an appropriate error.  */
2924 bool
2925 function_resolver::check_num_arguments (unsigned int expected)
2926 {
2927   if (m_arglist.length () < expected)
2928     error_at (location, "too few arguments to function %qE", fndecl);
2929   else if (m_arglist.length () > expected)
2930     error_at (location, "too many arguments to function %qE", fndecl);
2931   return m_arglist.length () == expected;
2932 }
2933
2934 /* If the function is predicated, check that the first argument is a
2935    suitable governing predicate.  Also check that there are NOPS further
2936    arguments after any governing predicate, but don't check what they are.
2937
2938    Return true on success, otherwise report a suitable error.
2939    When returning true:
2940
2941    - set I to the number of the first unchecked argument.
2942    - set NARGS to the total number of arguments.  */
2943 bool
2944 function_resolver::check_gp_argument (unsigned int nops,
2945                                       unsigned int &i, unsigned int &nargs)
2946 {
2947   gcc_assert (pred != PRED_za_m);
2948   i = 0;
2949   if (pred != PRED_none)
2950     {
2951       /* Unary merge operations should use resolve_unary instead.  */
2952       gcc_assert (!shape->has_merge_argument_p (*this, nops));
2953       nargs = nops + 1;
2954       if (!check_num_arguments (nargs)
2955           || !require_vector_type (i, gp_type_index ()))
2956         return false;
2957       i += 1;
2958     }
2959   else
2960     {
2961       nargs = nops;
2962       if (!check_num_arguments (nargs))
2963         return false;
2964     }
2965
2966   return true;
2967 }
2968
2969 /* Finish resolving a function whose final argument can be a vector
2970    or a scalar, with the function having an implicit "_n" suffix
2971    in the latter case.  This "_n" form might only exist for certain
2972    type suffixes.
2973
2974    ARGNO is the index of the final argument.  The inferred type suffix
2975    was obtained from argument FIRST_ARGNO, which has type FIRST_TYPE.
2976    EXPECTED_TCLASS and EXPECTED_BITS describe the expected properties
2977    of the final vector or scalar argument, in the same way as for
2978    require_derived_vector_type.  INFERRED_TYPE is the inferred type
2979    suffix itself, or NUM_TYPE_SUFFIXES if it's the same as FIRST_TYPE.
2980
2981    Return the function decl of the resolved function on success,
2982    otherwise report a suitable error and return error_mark_node.  */
2983 tree function_resolver::
2984 finish_opt_n_resolution (unsigned int argno, unsigned int first_argno,
2985                          type_suffix_index first_type,
2986                          type_class_index expected_tclass,
2987                          unsigned int expected_bits,
2988                          type_suffix_index inferred_type)
2989 {
2990   if (inferred_type == NUM_TYPE_SUFFIXES)
2991     inferred_type = first_type;
2992   tree scalar_form = lookup_form (MODE_n, inferred_type);
2993
2994   /* Allow the final argument to be scalar, if an _n form exists.  */
2995   if (scalar_argument_p (argno))
2996     {
2997       if (scalar_form)
2998         return scalar_form;
2999
3000       /* Check the vector form normally.  If that succeeds, raise an
3001          error about having no corresponding _n form.  */
3002       tree res = resolve_to (mode_suffix_id, inferred_type);
3003       if (res != error_mark_node)
3004         error_at (location, "passing %qT to argument %d of %qE, but its"
3005                   " %qT form does not accept scalars",
3006                   get_argument_type (argno), argno + 1, fndecl,
3007                   get_vector_type (first_type));
3008       return error_mark_node;
3009     }
3010
3011   /* If an _n form does exist, provide a more accurate message than
3012      require_derived_vector_type would for arguments that are neither
3013      vectors nor scalars.  */
3014   if (scalar_form && !require_vector_or_scalar_type (argno))
3015     return error_mark_node;
3016
3017   /* Check for the correct vector type.  */
3018   if (!require_derived_vector_type (argno, first_argno, first_type,
3019                                     expected_tclass, expected_bits))
3020     return error_mark_node;
3021
3022   return resolve_to (mode_suffix_id, inferred_type);
3023 }
3024
3025 /* Finish resolving a function whose final argument can be a tuple
3026    or a vector, with the function having an implicit "_single" suffix
3027    in the latter case.  This "_single" form might only exist for certain
3028    type suffixes.
3029
3030    ARGNO is the index of the final argument.  The inferred type suffix
3031    was obtained from argument FIRST_ARGNO, which has type FIRST_TYPE.
3032    EXPECTED_TCLASS gives the expected type class for the final tuple
3033    or vector.
3034
3035    Return the function decl of the resolved function on success,
3036    otherwise report a suitable error and return error_mark_node.  */
3037 tree function_resolver::
3038 finish_opt_single_resolution (unsigned int argno, unsigned int first_argno,
3039                               sve_type first_type,
3040                               type_class_index expected_tclass)
3041 {
3042   sve_type new_type = infer_sve_type (argno);
3043   if (!new_type)
3044     return error_mark_node;
3045
3046   /* If the type is a tuple, require it to match the group suffix.  */
3047   unsigned int num_vectors = vectors_per_tuple ();
3048   if (num_vectors != 1
3049       && new_type.num_vectors != 1
3050       && new_type.num_vectors != num_vectors)
3051     {
3052       report_incorrect_num_vectors (argno, new_type, num_vectors);
3053       return error_mark_node;
3054     }
3055
3056   auto expected_num_vectors = (new_type.num_vectors == 1 ? 1 : 0);
3057   if (!require_derived_vector_type (argno, first_argno, first_type,
3058                                     expected_tclass, SAME_SIZE,
3059                                     expected_num_vectors))
3060     return error_mark_node;
3061
3062   if (new_type.num_vectors == 1 && first_type.num_vectors > 1)
3063     {
3064       if (tree single_form = lookup_form (MODE_single, first_type))
3065         return single_form;
3066
3067       if (resolve_to (mode_suffix_id, first_type) != error_mark_node)
3068         error_at (location, "passing %qT to argument %d of %qE, but its"
3069                   " %qT form does not accept single vectors",
3070                   get_vector_type (new_type), argno + 1, fndecl,
3071                   get_vector_type (first_type));
3072       return error_mark_node;
3073     }
3074   return resolve_to (mode_suffix_id, first_type);
3075 }
3076
3077 /* Resolve a (possibly predicated) unary function.  If the function uses
3078    merge predication or if TREAT_AS_MERGE_P is true, there is an extra
3079    vector argument before the governing predicate that specifies the
3080    values of inactive elements.  This argument has the following
3081    properties:
3082
3083    - the type class must be the same as for active elements if MERGE_TCLASS
3084      is SAME_TYPE_CLASS, otherwise it must be MERGE_TCLASS itself.
3085
3086    - the element size must be the same as for active elements if MERGE_BITS
3087      is SAME_TYPE_SIZE, otherwise it must be MERGE_BITS itself.
3088
3089    Return the function decl of the resolved function on success,
3090    otherwise report a suitable error and return error_mark_node.  */
3091 tree
3092 function_resolver::resolve_unary (type_class_index merge_tclass,
3093                                   unsigned int merge_bits,
3094                                   bool treat_as_merge_p)
3095 {
3096   type_suffix_index type;
3097   if (pred == PRED_m || treat_as_merge_p)
3098     {
3099       if (!check_num_arguments (3))
3100         return error_mark_node;
3101       if (merge_tclass == SAME_TYPE_CLASS && merge_bits == SAME_SIZE)
3102         {
3103           /* The inactive elements are the same as the active elements,
3104              so we can use normal left-to-right resolution.  */
3105           if ((type = infer_vector_type (0)) == NUM_TYPE_SUFFIXES
3106               || !require_vector_type (1, VECTOR_TYPE_svbool_t)
3107               || !require_matching_vector_type (2, 0, type))
3108             return error_mark_node;
3109         }
3110       else
3111         {
3112           /* The inactive element type is a function of the active one,
3113              so resolve the active one first.  */
3114           if (!require_vector_type (1, VECTOR_TYPE_svbool_t)
3115               || (type = infer_vector_type (2)) == NUM_TYPE_SUFFIXES
3116               || !require_derived_vector_type (0, 2, type, merge_tclass,
3117                                                merge_bits))
3118             return error_mark_node;
3119         }
3120     }
3121   else
3122     {
3123       /* We just need to check the predicate (if any) and the single
3124          vector argument.  */
3125       unsigned int i, nargs;
3126       if (!check_gp_argument (1, i, nargs)
3127           || (type = infer_vector_type (i)) == NUM_TYPE_SUFFIXES)
3128         return error_mark_node;
3129     }
3130
3131   /* Handle convert-like functions in which the first type suffix is
3132      explicit.  */
3133   if (type_suffix_ids[0] != NUM_TYPE_SUFFIXES)
3134     return resolve_to (mode_suffix_id, type_suffix_ids[0], type);
3135
3136   return resolve_to (mode_suffix_id, type);
3137 }
3138
3139 /* Resolve a (possibly predicated) function that takes NOPS like-typed
3140    vector arguments followed by NIMM integer immediates.  Return the
3141    function decl of the resolved function on success, otherwise report
3142    a suitable error and return error_mark_node.  */
3143 tree
3144 function_resolver::resolve_uniform (unsigned int nops, unsigned int nimm)
3145 {
3146   unsigned int i, nargs;
3147   type_suffix_index type;
3148   if (!check_gp_argument (nops + nimm, i, nargs)
3149       || (type = infer_vector_type (i)) == NUM_TYPE_SUFFIXES)
3150     return error_mark_node;
3151
3152   unsigned int first_arg = i++;
3153   for (; i < nargs - nimm; ++i)
3154     if (!require_matching_vector_type (i, first_arg, type))
3155       return error_mark_node;
3156
3157   for (; i < nargs; ++i)
3158     if (!require_integer_immediate (i))
3159       return error_mark_node;
3160
3161   return resolve_to (mode_suffix_id, type);
3162 }
3163
3164 /* Resolve a (possibly predicated) function that offers a choice between
3165    taking:
3166
3167    - NOPS like-typed vector arguments or
3168    - NOPS - 1 like-typed vector arguments followed by a scalar argument
3169
3170    Return the function decl of the resolved function on success,
3171    otherwise report a suitable error and return error_mark_node.  */
3172 tree
3173 function_resolver::resolve_uniform_opt_n (unsigned int nops)
3174 {
3175   unsigned int i, nargs;
3176   type_suffix_index type;
3177   if (!check_gp_argument (nops, i, nargs)
3178       || (type = infer_vector_type (i)) == NUM_TYPE_SUFFIXES)
3179     return error_mark_node;
3180
3181   unsigned int first_arg = i++;
3182   for (; i < nargs - 1; ++i)
3183     if (!require_matching_vector_type (i, first_arg, type))
3184       return error_mark_node;
3185
3186   return finish_opt_n_resolution (i, first_arg, type);
3187 }
3188
3189 /* If the call is erroneous, report an appropriate error and return
3190    error_mark_node.  Otherwise, if the function is overloaded, return
3191    the decl of the non-overloaded function.  Return NULL_TREE otherwise,
3192    indicating that the call should be processed in the normal way.  */
3193 tree
3194 function_resolver::resolve ()
3195 {
3196   return shape->resolve (*this);
3197 }
3198
3199 function_checker::function_checker (location_t location,
3200                                     const function_instance &instance,
3201                                     tree fndecl, tree fntype,
3202                                     unsigned int nargs, tree *args)
3203   : function_call_info (location, instance, fndecl),
3204     m_fntype (fntype), m_nargs (nargs), m_args (args),
3205     m_base_arg (pred != PRED_none && pred != PRED_za_m ? 1 : 0)
3206 {
3207 }
3208
3209 /* Return true if argument ARGNO exists. which it might not for
3210    erroneous calls.  It is safe to wave through checks if this
3211    function returns false.  */
3212 bool
3213 function_checker::argument_exists_p (unsigned int argno)
3214 {
3215   gcc_assert (argno < (unsigned int) type_num_arguments (m_fntype));
3216   return argno < m_nargs;
3217 }
3218
3219 /* Check that argument ARGNO is an integer constant expression and
3220    store its value in VALUE_OUT if so.  The caller should first
3221    check that argument ARGNO exists.  */
3222 bool
3223 function_checker::require_immediate (unsigned int argno,
3224                                      HOST_WIDE_INT &value_out)
3225 {
3226   gcc_assert (argno < m_nargs);
3227   tree arg = m_args[argno];
3228
3229   /* The type and range are unsigned, so read the argument as an
3230      unsigned rather than signed HWI.  */
3231   if (!tree_fits_uhwi_p (arg))
3232     {
3233       report_non_ice (location, fndecl, argno);
3234       return false;
3235     }
3236
3237   /* ...but treat VALUE_OUT as signed for error reporting, since printing
3238      -1 is more user-friendly than the maximum uint64_t value.  */
3239   value_out = tree_to_uhwi (arg);
3240   return true;
3241 }
3242
3243 /* Check that argument REL_ARGNO is an integer constant expression that
3244    has the value VALUE0 or VALUE1.  REL_ARGNO counts from the end of the
3245    predication arguments.  */
3246 bool
3247 function_checker::require_immediate_either_or (unsigned int rel_argno,
3248                                                HOST_WIDE_INT value0,
3249                                                HOST_WIDE_INT value1)
3250 {
3251   unsigned int argno = m_base_arg + rel_argno;
3252   if (!argument_exists_p (argno))
3253     return true;
3254
3255   HOST_WIDE_INT actual;
3256   if (!require_immediate (argno, actual))
3257     return false;
3258
3259   if (actual != value0 && actual != value1)
3260     {
3261       report_neither_nor (location, fndecl, argno, actual, value0, value1);
3262       return false;
3263     }
3264
3265   return true;
3266 }
3267
3268 /* Check that argument REL_ARGNO is an integer constant expression that has
3269    a valid value for enumeration type TYPE.  REL_ARGNO counts from the end
3270    of the predication arguments.  */
3271 bool
3272 function_checker::require_immediate_enum (unsigned int rel_argno, tree type)
3273 {
3274   unsigned int argno = m_base_arg + rel_argno;
3275   if (!argument_exists_p (argno))
3276     return true;
3277
3278   HOST_WIDE_INT actual;
3279   if (!require_immediate (argno, actual))
3280     return false;
3281
3282   for (tree entry = TYPE_VALUES (type); entry; entry = TREE_CHAIN (entry))
3283     {
3284       /* The value is an INTEGER_CST for C and a CONST_DECL wrapper
3285          around an INTEGER_CST for C++.  */
3286       tree value = TREE_VALUE (entry);
3287       if (TREE_CODE (value) == CONST_DECL)
3288         value = DECL_INITIAL (value);
3289       if (wi::to_widest (value) == actual)
3290         return true;
3291     }
3292
3293   report_not_enum (location, fndecl, argno, actual, type);
3294   return false;
3295 }
3296
3297 /* The intrinsic conceptually divides vector argument REL_VEC_ARGNO into
3298    groups of GROUP_SIZE elements.  Return true if argument REL_ARGNO is
3299    a suitable constant index for selecting one of these groups.  The
3300    selection happens within a 128-bit quadword, rather than the whole vector.
3301
3302    REL_ARGNO and REL_VEC_ARGNO count from the end of the predication
3303    arguments.  */
3304 bool
3305 function_checker::require_immediate_lane_index (unsigned int rel_argno,
3306                                                 unsigned int rel_vec_argno,
3307                                                 unsigned int group_size)
3308 {
3309   unsigned int argno = m_base_arg + rel_argno;
3310   if (!argument_exists_p (argno))
3311     return true;
3312
3313   /* Get the type of the vector argument.  tree_argument_type wants a
3314      1-based number, whereas VEC_ARGNO is 0-based.  */
3315   unsigned int vec_argno = m_base_arg + rel_vec_argno;
3316   machine_mode mode = TYPE_MODE (type_argument_type (m_fntype, vec_argno + 1));
3317   gcc_assert (VECTOR_MODE_P (mode));
3318   unsigned int nlanes = 128 / (group_size * GET_MODE_UNIT_BITSIZE (mode));
3319   return require_immediate_range (rel_argno, 0, nlanes - 1);
3320 }
3321
3322 /* Check that argument REL_ARGNO is an integer constant expression that
3323    has one of the given values.  */
3324 bool
3325 function_checker::require_immediate_one_of (unsigned int rel_argno,
3326                                             HOST_WIDE_INT value0,
3327                                             HOST_WIDE_INT value1,
3328                                             HOST_WIDE_INT value2,
3329                                             HOST_WIDE_INT value3)
3330 {
3331   unsigned int argno = m_base_arg + rel_argno;
3332   if (!argument_exists_p (argno))
3333     return true;
3334
3335   HOST_WIDE_INT actual;
3336   if (!require_immediate (argno, actual))
3337     return false;
3338
3339   if (actual != value0
3340       && actual != value1
3341       && actual != value2
3342       && actual != value3)
3343     {
3344       report_not_one_of (location, fndecl, argno, actual,
3345                          value0, value1, value2, value3);
3346       return false;
3347     }
3348
3349   return true;
3350 }
3351
3352 /* Check that argument REL_ARGNO is an integer constant expression in the
3353    range [MIN, MAX].  REL_ARGNO counts from the end of the predication
3354    arguments.  */
3355 bool
3356 function_checker::require_immediate_range (unsigned int rel_argno,
3357                                            HOST_WIDE_INT min,
3358                                            HOST_WIDE_INT max)
3359 {
3360   unsigned int argno = m_base_arg + rel_argno;
3361   if (!argument_exists_p (argno))
3362     return true;
3363
3364   /* Required because of the tree_to_uhwi -> HOST_WIDE_INT conversion
3365      in require_immediate.  */
3366   gcc_assert (min >= 0 && min <= max);
3367   HOST_WIDE_INT actual;
3368   if (!require_immediate (argno, actual))
3369     return false;
3370
3371   if (!IN_RANGE (actual, min, max))
3372     {
3373       report_out_of_range (location, fndecl, argno, actual, min, max);
3374       return false;
3375     }
3376
3377   return true;
3378 }
3379
3380 /* Perform semantic checks on the call.  Return true if the call is valid,
3381    otherwise report a suitable error.  */
3382 bool
3383 function_checker::check ()
3384 {
3385   function_args_iterator iter;
3386   tree type;
3387   unsigned int i = 0;
3388   FOREACH_FUNCTION_ARGS (m_fntype, type, iter)
3389     {
3390       if (type == void_type_node || i >= m_nargs)
3391         break;
3392
3393       if (i >= m_base_arg
3394           && TREE_CODE (type) == ENUMERAL_TYPE
3395           && !require_immediate_enum (i - m_base_arg, type))
3396         return false;
3397
3398       i += 1;
3399     }
3400
3401   return shape->check (*this);
3402 }
3403
3404 /* Return true if V is a vector constant and if, for every in-range integer I,
3405    element STEP*I is equal to element 0.  */
3406 bool
3407 vector_cst_all_same (tree v, unsigned int step)
3408 {
3409   if (TREE_CODE (v) != VECTOR_CST)
3410     return false;
3411
3412   /* VECTOR_CST_NELTS_PER_PATTERN applies to any multiple of
3413      VECTOR_CST_NPATTERNS.  */
3414   unsigned int lcm = least_common_multiple (step, VECTOR_CST_NPATTERNS (v));
3415   unsigned int nelts = lcm * VECTOR_CST_NELTS_PER_PATTERN (v);
3416   tree first_el = VECTOR_CST_ENCODED_ELT (v, 0);
3417   for (unsigned int i = 0; i < nelts; i += step)
3418     if (!operand_equal_p (VECTOR_CST_ELT (v, i), first_el, 0))
3419       return false;
3420
3421   return true;
3422 }
3423
3424 /* Return true if V is a constant predicate that acts as a ptrue when
3425    predicating STEP-byte elements.  */
3426 bool
3427 is_ptrue (tree v, unsigned int step)
3428 {
3429   return (TREE_CODE (v) == VECTOR_CST
3430           && TYPE_MODE (TREE_TYPE (v)) == VNx16BImode
3431           && integer_nonzerop (VECTOR_CST_ENCODED_ELT (v, 0))
3432           && vector_cst_all_same (v, step));
3433 }
3434
3435 gimple_folder::gimple_folder (const function_instance &instance, tree fndecl,
3436                               gimple_stmt_iterator *gsi_in, gcall *call_in)
3437   : function_call_info (gimple_location (call_in), instance, fndecl),
3438     gsi (gsi_in), call (call_in), lhs (gimple_call_lhs (call_in))
3439 {
3440 }
3441
3442 /* VALUE might be a vector of type VECTYPE or a single scalar element.
3443    Duplicate it into a vector of type VECTYPE in the latter case, adding any
3444    new statements to STMTS.  */
3445 tree
3446 gimple_folder::force_vector (gimple_seq &stmts, tree vectype, tree value)
3447 {
3448   if (!VECTOR_TYPE_P (TREE_TYPE (value)))
3449     value = gimple_build_vector_from_val (&stmts, vectype, value);
3450   return value;
3451 }
3452
3453 /* Convert predicate argument ARGNO so that it has the type appropriate for
3454    an operation on VECTYPE.  Add any new statements to STMTS.  */
3455 tree
3456 gimple_folder::convert_pred (gimple_seq &stmts, tree vectype,
3457                              unsigned int argno)
3458 {
3459   tree pred = gimple_call_arg (call, argno);
3460   if (known_eq (TYPE_VECTOR_SUBPARTS (TREE_TYPE (pred)),
3461                 TYPE_VECTOR_SUBPARTS (vectype)))
3462     return pred;
3463
3464   return gimple_build (&stmts, VIEW_CONVERT_EXPR,
3465                        truth_type_for (vectype), pred);
3466 }
3467
3468 /* Return a pointer to the address in a contiguous load or store,
3469    given that each memory vector has type VECTYPE.  Add any new
3470    statements to STMTS.  */
3471 tree
3472 gimple_folder::fold_contiguous_base (gimple_seq &stmts, tree vectype)
3473 {
3474   tree base = gimple_call_arg (call, 1);
3475   if (mode_suffix_id == MODE_vnum)
3476     {
3477       tree offset = gimple_call_arg (call, 2);
3478       offset = gimple_convert (&stmts, sizetype, offset);
3479       offset = gimple_build (&stmts, MULT_EXPR, sizetype, offset,
3480                              TYPE_SIZE_UNIT (vectype));
3481       base = gimple_build (&stmts, POINTER_PLUS_EXPR, TREE_TYPE (base),
3482                            base, offset);
3483     }
3484   return base;
3485 }
3486
3487 /* Return the alignment and TBAA argument to an internal load or store
3488    function like IFN_MASK_LOAD or IFN_MASK_STORE, given that it accesses
3489    memory elements of type TYPE.  */
3490 tree
3491 gimple_folder::load_store_cookie (tree type)
3492 {
3493   return build_int_cst (build_pointer_type (type), TYPE_ALIGN (type));
3494 }
3495
3496 /* Fold the call to a call to INSTANCE, with the same arguments.  */
3497 gcall *
3498 gimple_folder::redirect_call (const function_instance &instance)
3499 {
3500   registered_function *rfn
3501     = function_table->find_with_hash (instance, instance.hash ());
3502   if (!rfn)
3503     return NULL;
3504
3505   gimple_call_set_fndecl (call, rfn->decl);
3506   return call;
3507 }
3508
3509 /* Redirect _z and _m calls to _x functions if the predicate is all-true.
3510    This allows us to use unpredicated instructions, where available.  */
3511 gimple *
3512 gimple_folder::redirect_pred_x ()
3513 {
3514   if (pred != PRED_z && pred != PRED_m)
3515     return nullptr;
3516
3517   if (gimple_call_num_args (call) < 2)
3518     return nullptr;
3519
3520   tree lhs_type = TREE_TYPE (TREE_TYPE (fndecl));
3521   tree arg0_type = type_argument_type (TREE_TYPE (fndecl), 1);
3522   tree arg1_type = type_argument_type (TREE_TYPE (fndecl), 2);
3523   if (!VECTOR_TYPE_P (lhs_type)
3524       || !VECTOR_TYPE_P (arg0_type)
3525       || !VECTOR_TYPE_P (arg1_type))
3526     return nullptr;
3527
3528   auto lhs_step = element_precision (lhs_type);
3529   auto rhs_step = element_precision (arg1_type);
3530   auto step = MAX (lhs_step, rhs_step);
3531   if (!multiple_p (step, BITS_PER_UNIT)
3532       || !is_ptrue (gimple_call_arg (call, 0), step / BITS_PER_UNIT))
3533     return nullptr;
3534
3535   function_instance instance (*this);
3536   instance.pred = PRED_x;
3537   return redirect_call (instance);
3538 }
3539
3540 /* Fold the call to constant VAL.  */
3541 gimple *
3542 gimple_folder::fold_to_cstu (poly_uint64 val)
3543 {
3544   return gimple_build_assign (lhs, build_int_cstu (TREE_TYPE (lhs), val));
3545 }
3546
3547 /* Fold the call to a PTRUE, taking the element size from type suffix 0.  */
3548 gimple *
3549 gimple_folder::fold_to_ptrue ()
3550 {
3551   tree svbool_type = TREE_TYPE (lhs);
3552   tree bool_type = TREE_TYPE (svbool_type);
3553   unsigned int element_bytes = type_suffix (0).element_bytes;
3554
3555   /* The return type is svbool_t for all type suffixes, thus for b8 we
3556      want { 1, 1, 1, 1, ... }, for b16 we want { 1, 0, 1, 0, ... }, etc.  */
3557   tree_vector_builder builder (svbool_type, element_bytes, 1);
3558   builder.quick_push (build_all_ones_cst (bool_type));
3559   for (unsigned int i = 1; i < element_bytes; ++i)
3560     builder.quick_push (build_zero_cst (bool_type));
3561   return gimple_build_assign (lhs, builder.build ());
3562 }
3563
3564 /* Fold the call to a PFALSE.  */
3565 gimple *
3566 gimple_folder::fold_to_pfalse ()
3567 {
3568   return gimple_build_assign (lhs, build_zero_cst (TREE_TYPE (lhs)));
3569 }
3570
3571 /* Fold an operation to a constant predicate in which the first VL
3572    elements are set and the rest are clear.  Take the element size
3573    from type suffix 0.  */
3574 gimple *
3575 gimple_folder::fold_to_vl_pred (unsigned int vl)
3576 {
3577   tree vectype = TREE_TYPE (lhs);
3578   tree element_type = TREE_TYPE (vectype);
3579   tree minus_one = build_all_ones_cst (element_type);
3580   tree zero = build_zero_cst (element_type);
3581   unsigned int element_bytes = type_suffix (0).element_bytes;
3582
3583   /* Construct COUNT elements that contain the ptrue followed by
3584      a repeating sequence of COUNT elements.  */
3585   unsigned int count = constant_lower_bound (TYPE_VECTOR_SUBPARTS (vectype));
3586   gcc_assert (vl * element_bytes <= count);
3587   tree_vector_builder builder (vectype, count, 2);
3588   for (unsigned int i = 0; i < count * 2; ++i)
3589     {
3590       bool bit = (i & (element_bytes - 1)) == 0 && i < vl * element_bytes;
3591       builder.quick_push (bit ? minus_one : zero);
3592     }
3593   return gimple_build_assign (lhs, builder.build ());
3594 }
3595
3596 /* Try to fold the call.  Return the new statement on success and null
3597    on failure.  */
3598 gimple *
3599 gimple_folder::fold ()
3600 {
3601   /* Don't fold anything when SVE is disabled; emit an error during
3602      expansion instead.  */
3603   if (!TARGET_SVE)
3604     return NULL;
3605
3606   /* Punt if the function has a return type and no result location is
3607      provided.  The attributes should allow target-independent code to
3608      remove the calls if appropriate.  */
3609   if (!lhs && TREE_TYPE (gimple_call_fntype (call)) != void_type_node)
3610     return NULL;
3611
3612   /* First try some simplifications that are common to many functions.  */
3613   if (auto *call = redirect_pred_x ())
3614     return call;
3615
3616   return base->fold (*this);
3617 }
3618
3619 function_expander::function_expander (const function_instance &instance,
3620                                       tree fndecl, tree call_expr_in,
3621                                       rtx possible_target_in)
3622   : function_call_info (EXPR_LOCATION (call_expr_in), instance, fndecl),
3623     call_expr (call_expr_in), possible_target (possible_target_in)
3624 {
3625 }
3626
3627 /* Return the handler of direct optab OP for type suffix SUFFIX_I.  */
3628 insn_code
3629 function_expander::direct_optab_handler (optab op, unsigned int suffix_i)
3630 {
3631   return ::direct_optab_handler (op, tuple_mode (suffix_i));
3632 }
3633
3634 /* Choose between signed and unsigned direct optabs SIGNED_OP and
3635    UNSIGNED_OP based on the signedness of type suffix SUFFIX_I, then
3636    pick the appropriate optab handler for the mode.  Use MODE as the
3637    mode if given, otherwise use the mode of type suffix SUFFIX_I.  */
3638 insn_code
3639 function_expander::direct_optab_handler_for_sign (optab signed_op,
3640                                                   optab unsigned_op,
3641                                                   unsigned int suffix_i,
3642                                                   machine_mode mode)
3643 {
3644   if (mode == VOIDmode)
3645     mode = vector_mode (suffix_i);
3646   optab op = type_suffix (suffix_i).unsigned_p ? unsigned_op : signed_op;
3647   return ::direct_optab_handler (op, mode);
3648 }
3649
3650 /* Return true if X overlaps any input.  */
3651 bool
3652 function_expander::overlaps_input_p (rtx x)
3653 {
3654   for (unsigned int i = 0; i < args.length (); ++i)
3655     if (reg_overlap_mentioned_p (x, args[i]))
3656       return true;
3657   return false;
3658 }
3659
3660 /* Convert ptr_mode value X to Pmode.  */
3661 rtx
3662 function_expander::convert_to_pmode (rtx x)
3663 {
3664   if (ptr_mode == SImode)
3665     x = simplify_gen_unary (ZERO_EXTEND, DImode, x, SImode);
3666   return x;
3667 }
3668
3669 /* Return the base address for a contiguous load or store function.
3670    MEM_MODE is the mode of the addressed memory, BASE_ARGNO is
3671    the index of the base argument, and VNUM_ARGNO is the index of
3672    the vnum offset argument (if any).  VL_ISA_MODE is AARCH64_FL_SM_ON
3673    if the vnum argument is a factor of the SME vector length, 0 if it
3674    is a factor of the current prevailing vector length.  */
3675 rtx
3676 function_expander::get_contiguous_base (machine_mode mem_mode,
3677                                         unsigned int base_argno,
3678                                         unsigned int vnum_argno,
3679                                         aarch64_feature_flags vl_isa_mode)
3680 {
3681   rtx base = convert_to_pmode (args[base_argno]);
3682   if (mode_suffix_id == MODE_vnum)
3683     {
3684       rtx vnum = args[vnum_argno];
3685       if (vnum != const0_rtx)
3686         {
3687           /* Use the size of the memory mode for extending loads and truncating
3688              stores.  Use the size of a full vector for non-extending loads
3689              and non-truncating stores (including svld[234] and svst[234]).  */
3690           poly_int64 size = ordered_min (GET_MODE_SIZE (mem_mode),
3691                                          BYTES_PER_SVE_VECTOR);
3692           rtx offset;
3693           if ((vl_isa_mode & AARCH64_FL_SM_ON)
3694               && !TARGET_STREAMING
3695               && !size.is_constant ())
3696             {
3697               gcc_assert (known_eq (size, BYTES_PER_SVE_VECTOR));
3698               if (CONST_INT_P (vnum) && IN_RANGE (INTVAL (vnum), -32, 31))
3699                 offset = aarch64_sme_vq_immediate (Pmode, INTVAL (vnum) * 16,
3700                                                    AARCH64_ISA_MODE);
3701               else
3702                 {
3703                   offset = aarch64_sme_vq_immediate (Pmode, 16,
3704                                                      AARCH64_ISA_MODE);
3705                   offset = simplify_gen_binary (MULT, Pmode, vnum, offset);
3706                 }
3707             }
3708           else
3709             {
3710               offset = gen_int_mode (size, Pmode);
3711               offset = simplify_gen_binary (MULT, Pmode, vnum, offset);
3712             }
3713           base = simplify_gen_binary (PLUS, Pmode, base, offset);
3714         }
3715     }
3716   return base;
3717 }
3718
3719 /* For a function that does the equivalent of:
3720
3721      OUTPUT = COND ? FN (INPUTS) : FALLBACK;
3722
3723    return the value of FALLBACK.
3724
3725    MODE is the mode of OUTPUT.  NOPS is the number of operands in INPUTS.
3726    MERGE_ARGNO is the argument that provides FALLBACK for _m functions,
3727    or DEFAULT_MERGE_ARGNO if we should apply the usual rules.
3728
3729    ARGNO is the caller's index into args.  If the returned value is
3730    argument 0 (as for unary _m operations), increment ARGNO past the
3731    returned argument.  */
3732 rtx
3733 function_expander::get_fallback_value (machine_mode mode, unsigned int nops,
3734                                        unsigned int merge_argno,
3735                                        unsigned int &argno)
3736 {
3737   if (pred == PRED_z)
3738     return CONST0_RTX (mode);
3739
3740   gcc_assert (pred == PRED_m || pred == PRED_x);
3741   if (merge_argno == DEFAULT_MERGE_ARGNO)
3742     merge_argno = shape->has_merge_argument_p (*this, nops) ? 0 : 1;
3743
3744   if (merge_argno == 0)
3745     return args[argno++];
3746
3747   return args[merge_argno];
3748 }
3749
3750 /* Return a REG rtx that can be used for the result of the function,
3751    using the preferred target if suitable.  */
3752 rtx
3753 function_expander::get_reg_target ()
3754 {
3755   machine_mode target_mode = result_mode ();
3756   if (!possible_target || GET_MODE (possible_target) != target_mode)
3757     possible_target = gen_reg_rtx (target_mode);
3758   return possible_target;
3759 }
3760
3761 /* As for get_reg_target, but make sure that the returned REG does not
3762    overlap any inputs.  */
3763 rtx
3764 function_expander::get_nonoverlapping_reg_target ()
3765 {
3766   if (possible_target && overlaps_input_p (possible_target))
3767     possible_target = NULL_RTX;
3768   return get_reg_target ();
3769 }
3770
3771 /* Add an output operand to the instruction we're building, which has
3772    code ICODE.  Bind the output to the preferred target rtx if possible.  */
3773 void
3774 function_expander::add_output_operand (insn_code icode)
3775 {
3776   unsigned int opno = m_ops.length ();
3777   machine_mode mode = insn_data[icode].operand[opno].mode;
3778   m_ops.safe_grow (opno + 1, true);
3779   create_output_operand (&m_ops.last (), possible_target, mode);
3780 }
3781
3782 /* Add an input operand to the instruction we're building, which has
3783    code ICODE.  Calculate the value of the operand as follows:
3784
3785    - If the operand is a vector and X is not, broadcast X to fill a
3786      vector of the appropriate mode.
3787
3788    - Otherwise, if the operand is a predicate, coerce X to have the
3789      mode that the instruction expects.  In this case X is known to be
3790      VNx16BImode (the mode of svbool_t).
3791
3792    - Otherwise use X directly.  The expand machinery checks that X has
3793      the right mode for the instruction.  */
3794 void
3795 function_expander::add_input_operand (insn_code icode, rtx x)
3796 {
3797   unsigned int opno = m_ops.length ();
3798   const insn_operand_data &operand = insn_data[icode].operand[opno];
3799   machine_mode mode = operand.mode;
3800   if (mode == VOIDmode)
3801     {
3802       /* The only allowable uses of VOIDmode are:
3803
3804          - the wildcard aarch64_any_register_operand, which is used
3805            to avoid combinatorial explosion in the reinterpret patterns
3806
3807          - pmode_register_operand, which always has mode Pmode.  */
3808       if (operand.predicate == aarch64_any_register_operand)
3809         mode = GET_MODE (x);
3810       else if (operand.predicate == pmode_register_operand)
3811         mode = Pmode;
3812       else
3813         gcc_unreachable ();
3814     }
3815   else if (!VECTOR_MODE_P (GET_MODE (x)) && VECTOR_MODE_P (mode))
3816     x = expand_vector_broadcast (mode, x);
3817   else if (GET_MODE_CLASS (mode) == MODE_VECTOR_BOOL)
3818     {
3819       gcc_assert (GET_MODE (x) == VNx16BImode);
3820       x = gen_lowpart (mode, x);
3821     }
3822   m_ops.safe_grow (m_ops.length () + 1, true);
3823   create_input_operand (&m_ops.last (), x, mode);
3824 }
3825
3826 /* Add an integer operand with value X to the instruction.  */
3827 void
3828 function_expander::add_integer_operand (poly_int64 x)
3829 {
3830   m_ops.safe_grow (m_ops.length () + 1, true);
3831   create_integer_operand (&m_ops.last (), x);
3832 }
3833
3834 /* Add a memory operand with mode MODE and address ADDR.  */
3835 void
3836 function_expander::add_mem_operand (machine_mode mode, rtx addr)
3837 {
3838   /* Exception for OImode for the ld1ro intrinsics.
3839      They act on 256 bit octaword data, and it's just easier to use a scalar
3840      mode to represent that than add a new vector mode solely for the purpose
3841      of this intrinsic.  */
3842   gcc_assert (VECTOR_MODE_P (mode) || mode == OImode);
3843   rtx mem = gen_rtx_MEM (mode, memory_address (mode, addr));
3844   /* The memory is only guaranteed to be element-aligned.  */
3845   set_mem_align (mem, GET_MODE_ALIGNMENT (GET_MODE_INNER (mode)));
3846   add_fixed_operand (mem);
3847 }
3848
3849 /* Add an address operand with value X.  The static operand data says
3850    what mode and form the address must have.  */
3851 void
3852 function_expander::add_address_operand (rtx x)
3853 {
3854   m_ops.safe_grow (m_ops.length () + 1, true);
3855   create_address_operand (&m_ops.last (), x);
3856 }
3857
3858 /* Add an operand that must be X.  The only way of legitimizing an
3859    invalid X is to reload the address of a MEM.  */
3860 void
3861 function_expander::add_fixed_operand (rtx x)
3862 {
3863   m_ops.safe_grow (m_ops.length () + 1, true);
3864   create_fixed_operand (&m_ops.last (), x);
3865 }
3866
3867 /* Generate instruction ICODE, given that its operands have already
3868    been added to M_OPS.  Return the value of the first operand.  */
3869 rtx
3870 function_expander::generate_insn (insn_code icode)
3871 {
3872   expand_insn (icode, m_ops.length (), m_ops.address ());
3873   return function_returns_void_p () ? const0_rtx : m_ops[0].value;
3874 }
3875
3876 /* Convert the arguments to a gather/scatter function into the
3877    associated md operands.  Argument ARGNO is the scalar or vector base and
3878    argument ARGNO + 1 is the scalar or vector displacement (if applicable).
3879    The md pattern expects:
3880
3881    - a scalar base
3882    - a vector displacement
3883
3884    If SCALED_P is true, it also expects:
3885
3886    - a const_int that is 1 if the displacement is zero-extended from 32 bits
3887    - a scaling multiplier (1 for bytes, 2 for .h indices, etc.).
3888
3889    If SCALED_P is false, the displacement is implicitly zero-extended
3890    and the scaling multiplier is implicitly 1.  */
3891 void
3892 function_expander::prepare_gather_address_operands (unsigned int argno,
3893                                                     bool scaled_p)
3894 {
3895   machine_mode mem_mode = memory_vector_mode ();
3896   tree vector_type = base_vector_type ();
3897   units_index units = displacement_units ();
3898   int shift_idx = -1;
3899   if (units == UNITS_none)
3900     {
3901       /* Vector base, no displacement.  Convert to an integer zero base
3902          and a vector byte offset.  */
3903       args.quick_insert (argno, const0_rtx);
3904       units = UNITS_bytes;
3905     }
3906   else if (vector_type)
3907     {
3908       /* Vector base, scalar displacement.  Convert to a scalar base and
3909          a vector byte offset.  */
3910       std::swap (args[argno], args[argno + 1]);
3911       if (units == UNITS_elements)
3912         shift_idx = argno;
3913     }
3914   else
3915     {
3916       /* Scalar base, vector displacement.  This is the order that the md
3917          pattern wants.  */
3918       args[argno] = convert_to_pmode (args[argno]);
3919       vector_type = displacement_vector_type ();
3920       if (units == UNITS_elements && !scaled_p)
3921         shift_idx = argno + 1;
3922     }
3923   tree scalar_displacement_type = TREE_TYPE (vector_type);
3924
3925   if (shift_idx >= 0)
3926     {
3927       machine_mode arg_mode = GET_MODE (args[shift_idx]);
3928       if (arg_mode == VOIDmode)
3929         arg_mode = DImode;
3930       unsigned int elt_bytes = GET_MODE_UNIT_SIZE (mem_mode);
3931       rtx shift = gen_int_mode (exact_log2 (elt_bytes), DImode);
3932       args[shift_idx] = simplify_gen_binary (ASHIFT, arg_mode,
3933                                              args[shift_idx], shift);
3934       units = UNITS_bytes;
3935     }
3936
3937   bool uxtw_p = (TYPE_PRECISION (scalar_displacement_type) == 64
3938                  || TYPE_UNSIGNED (scalar_displacement_type));
3939   unsigned int scale = (units == UNITS_bytes
3940                         ? 1 : GET_MODE_UNIT_SIZE (mem_mode));
3941
3942   if (scaled_p)
3943     {
3944       args.quick_insert (argno + 2, GEN_INT (uxtw_p));
3945       args.quick_insert (argno + 3, GEN_INT (scale));
3946     }
3947   else
3948     gcc_assert (uxtw_p && scale == 1);
3949 }
3950
3951 /* The final argument is an immediate svprfop value.  Add two fake arguments
3952    to represent the rw and locality operands of a PREFETCH rtx.  */
3953 void
3954 function_expander::prepare_prefetch_operands ()
3955 {
3956   unsigned int prfop = INTVAL (args.last ());
3957   /* Bit 3 of the prfop selects stores over loads.  */
3958   args.quick_push (GEN_INT ((prfop & 8) != 0));
3959   /* Bits 1 and 2 specify the locality; 0-based for svprfop but
3960      1-based for PREFETCH.  */
3961   args.quick_push (GEN_INT (((prfop >> 1) & 3) + 1));
3962 }
3963
3964 /* Add a dummy argument to indicate whether predicate argument ARGNO
3965    is all-true when interpreted in mode PRED_MODE.  The hint goes
3966    immediately after ARGNO.  */
3967 void
3968 function_expander::add_ptrue_hint (unsigned int argno, machine_mode pred_mode)
3969 {
3970   rtx pred = gen_lowpart (pred_mode, args[argno]);
3971   int hint = (pred == CONSTM1_RTX (pred_mode)
3972               ? SVE_KNOWN_PTRUE : SVE_MAYBE_NOT_PTRUE);
3973   args.quick_insert (argno + 1, gen_int_mode (hint, SImode));
3974 }
3975
3976 /* Rotate inputs args[START:END] one position to the left, so that
3977    args[START] becomes args[END - 1].  */
3978 void
3979 function_expander::rotate_inputs_left (unsigned int start, unsigned int end)
3980 {
3981   rtx new_last = args[start];
3982   for (unsigned int i = start; i < end - 1; ++i)
3983     args[i] = args[i + 1];
3984   args[end - 1] = new_last;
3985 }
3986
3987 /* Return true if the negation of argument ARGNO can be folded away,
3988    replacing it with the negated value if so.  MODE is the associated
3989    vector mode, but the argument could be a single element.  The main
3990    case this handles is constant arguments.  */
3991 bool
3992 function_expander::try_negating_argument (unsigned int argno,
3993                                           machine_mode mode)
3994 {
3995   rtx x = args[argno];
3996   if (!VECTOR_MODE_P (GET_MODE (x)))
3997     mode = GET_MODE_INNER (mode);
3998
3999   x = simplify_unary_operation (NEG, mode, x, mode);
4000   if (!x)
4001     return false;
4002
4003   args[argno] = x;
4004   return true;
4005 }
4006
4007 /* Implement the call using instruction ICODE, with a 1:1 mapping between
4008    arguments and input operands.  */
4009 rtx
4010 function_expander::use_exact_insn (insn_code icode)
4011 {
4012   unsigned int nops = insn_data[icode].n_operands;
4013   if (!function_returns_void_p ())
4014     {
4015       add_output_operand (icode);
4016       nops -= 1;
4017     }
4018   for (unsigned int i = 0; i < nops; ++i)
4019     add_input_operand (icode, args[i]);
4020   return generate_insn (icode);
4021 }
4022
4023 /* Implement the call using instruction ICODE, which does not use a
4024    governing predicate.  We must therefore drop the GP from an _x call.  */
4025 rtx
4026 function_expander::use_unpred_insn (insn_code icode)
4027 {
4028   /* We can't drop the predicate for _z and _m.  */
4029   gcc_assert (pred == PRED_x || pred == PRED_none);
4030   /* Discount the output operand.  */
4031   unsigned int nops = insn_data[icode].n_operands - 1;
4032   /* Drop the predicate argument in the case of _x predication.  */
4033   unsigned int bias = (pred == PRED_x ? 1 : 0);
4034   unsigned int i = 0;
4035
4036   add_output_operand (icode);
4037   for (; i < nops; ++i)
4038     add_input_operand (icode, args[i + bias]);
4039
4040   return generate_insn (icode);
4041 }
4042
4043 /* Implement the call using instruction ICODE, which is a predicated
4044    operation that returns arbitrary values for inactive lanes.  */
4045 rtx
4046 function_expander::use_pred_x_insn (insn_code icode)
4047 {
4048   /* At present we never need to handle PRED_none, which would involve
4049      creating a new predicate rather than using one supplied by the user.  */
4050   gcc_assert (pred == PRED_x);
4051   /* Discount the output operand.  */
4052   unsigned int nops = args.length () - 1;
4053
4054   bool has_float_operand_p = FLOAT_MODE_P (insn_data[icode].operand[0].mode);
4055
4056   /* Add the normal operands.  */
4057   add_output_operand (icode);
4058   add_input_operand (icode, args[0]);
4059   for (unsigned int i = 0; i < nops; ++i)
4060     {
4061       add_input_operand (icode, args[i + 1]);
4062       if (FLOAT_MODE_P (GET_MODE (args[i + 1])))
4063         has_float_operand_p = true;
4064     }
4065
4066   if (has_float_operand_p
4067       && insn_data[icode].n_operands > (int) nops + 2)
4068     {
4069       /* Add a flag that indicates whether unpredicated instructions
4070          are allowed.  */
4071       rtx pred = m_ops[1].value;
4072       if (flag_trapping_math && pred != CONST1_RTX (GET_MODE (pred)))
4073         add_integer_operand (SVE_STRICT_GP);
4074       else
4075         add_integer_operand (SVE_RELAXED_GP);
4076     }
4077
4078   return generate_insn (icode);
4079 }
4080
4081 /* Implement the call using instruction ICODE, which does the equivalent of:
4082
4083      OUTPUT = COND ? FN (INPUTS) : FALLBACK;
4084
4085    The instruction operands are in the order above: OUTPUT, COND, INPUTS
4086    and FALLBACK.  MERGE_ARGNO is the argument that provides FALLBACK for _m
4087    functions, or DEFAULT_MERGE_ARGNO if we should apply the usual rules.  */
4088 rtx
4089 function_expander::use_cond_insn (insn_code icode, unsigned int merge_argno)
4090 {
4091   /* At present we never need to handle PRED_none, which would involve
4092      creating a new predicate rather than using one supplied by the user.  */
4093   gcc_assert (pred != PRED_none);
4094   /* Discount the output, predicate and fallback value.  */
4095   unsigned int nops = insn_data[icode].n_operands - 3;
4096   machine_mode mode = insn_data[icode].operand[0].mode;
4097
4098   unsigned int opno = 0;
4099   rtx fallback_arg = get_fallback_value (mode, nops, merge_argno, opno);
4100   rtx pred = args[opno++];
4101
4102   add_output_operand (icode);
4103   add_input_operand (icode, pred);
4104   for (unsigned int i = 0; i < nops; ++i)
4105     add_input_operand (icode, args[opno + i]);
4106   add_input_operand (icode, fallback_arg);
4107   return generate_insn (icode);
4108 }
4109
4110 /* Implement the call using instruction ICODE, which is a select-like
4111    operation with the following operands:
4112
4113    0: output
4114    1: true value
4115    2: false value
4116    3: predicate
4117
4118    MERGE_ARGNO is the argument that provides the "false" value for _m
4119    functions, or DEFAULT_MERGE_ARGNO if we should apply the usual rules.  */
4120 rtx
4121 function_expander::use_vcond_mask_insn (insn_code icode,
4122                                         unsigned int merge_argno)
4123 {
4124   machine_mode mode = vector_mode (0);
4125
4126   unsigned int opno = 0;
4127   rtx false_arg = get_fallback_value (mode, 1, merge_argno, opno);
4128   rtx pred_arg = args[opno++];
4129   rtx true_arg = args[opno++];
4130
4131   add_output_operand (icode);
4132   add_input_operand (icode, true_arg);
4133   add_input_operand (icode, false_arg);
4134   add_input_operand (icode, pred_arg);
4135   return generate_insn (icode);
4136 }
4137
4138 /* Implement the call using instruction ICODE, which loads memory operand 1
4139    into register operand 0 under the control of predicate operand 2.
4140    Extending loads have a further predicate (operand 3) that nominally
4141    controls the extension.  */
4142 rtx
4143 function_expander::use_contiguous_load_insn (insn_code icode)
4144 {
4145   machine_mode mem_mode = memory_vector_mode ();
4146
4147   add_output_operand (icode);
4148   add_mem_operand (mem_mode, get_contiguous_base (mem_mode));
4149   add_input_operand (icode, args[0]);
4150   if (GET_MODE_UNIT_BITSIZE (mem_mode) < type_suffix (0).element_bits)
4151     add_input_operand (icode, CONSTM1_RTX (VNx16BImode));
4152   return generate_insn (icode);
4153 }
4154
4155 /* Implement the call using instruction ICODE, which prefetches from
4156    address operand 1 under the control of predicate operand 0.
4157    Operands 2, 3 and 4 respectively specify the svprfop value,
4158    the PREFETCH rw flag and the PREFETCH locality.  */
4159 rtx
4160 function_expander::use_contiguous_prefetch_insn (insn_code icode)
4161 {
4162   add_input_operand (icode, args[0]);
4163   add_address_operand (get_contiguous_base (VNx16QImode));
4164   for (unsigned int i = args.length () - 3; i < args.length (); ++i)
4165     add_input_operand (icode, args[i]);
4166   return generate_insn (icode);
4167 }
4168
4169 /* Implement the call using instruction ICODE, which stores register operand 1
4170    into memory operand 0 under the control of predicate operand 2.  */
4171 rtx
4172 function_expander::use_contiguous_store_insn (insn_code icode)
4173 {
4174   machine_mode mem_mode = memory_vector_mode ();
4175
4176   add_mem_operand (mem_mode, get_contiguous_base (mem_mode));
4177   add_input_operand (icode, args.last ());
4178   add_input_operand (icode, args[0]);
4179   return generate_insn (icode);
4180 }
4181
4182 /* Implement the call using one of the following strategies, chosen in order:
4183
4184    (1) "aarch64_pred_<optab><mode>_z" for PRED_z predicate functions
4185
4186    (2) "aarch64_pred_<optab><mode>" for PRED_x functions
4187
4188    (3) a normal unpredicated optab for PRED_none and PRED_x functions,
4189        dropping the predicate in the latter case
4190
4191    (4) an unpredicated "aarch64_sve_<code_optab><mode>" for PRED_none and
4192        PRED_x functions, again dropping the predicate for PRED_x
4193
4194    (5) "cond_<optab><mode>" otherwise
4195
4196    where <optab> corresponds to:
4197
4198    - CODE_FOR_SINT for signed integers
4199    - CODE_FOR_UINT for unsigned integers
4200    - UNSPEC_FOR_COND_FP for predicated floating-point
4201    - UNSPEC_FOR_UNCOND_FP for unpredicated floating-point
4202
4203    and where <code_optab> is like <optab>, but uses CODE_FOR_SINT instead
4204    of UNSPEC_FOR_FP for floating-point values.
4205
4206    MERGE_ARGNO is the argument that provides the values of inactive lanes for
4207    _m functions, or DEFAULT_MERGE_ARGNO if we should apply the usual rules.  */
4208 rtx
4209 function_expander::map_to_rtx_codes (rtx_code code_for_sint,
4210                                      rtx_code code_for_uint,
4211                                      int unspec_for_cond_fp,
4212                                      int unspec_for_uncond_fp,
4213                                      unsigned int merge_argno)
4214 {
4215   machine_mode mode = tuple_mode (0);
4216   rtx_code code = (type_suffix (0).unsigned_p ? code_for_uint : code_for_sint);
4217   insn_code icode;
4218
4219   if (mode_suffix_id == MODE_single)
4220     {
4221       gcc_assert (pred == PRED_none);
4222       if (type_suffix (0).integer_p)
4223         icode = code_for_aarch64_sve_single (code, mode);
4224       else
4225         icode = code_for_aarch64_sve_single (unspec_for_uncond_fp, mode);
4226       return use_exact_insn (icode);
4227     }
4228
4229   /* Handle predicate logic operations, which always use _z predication.  */
4230   if (type_suffix (0).tclass == TYPE_bool)
4231     {
4232       gcc_assert (pred == PRED_z && code_for_uint == code_for_sint);
4233       return use_exact_insn (code_for_aarch64_pred_z (code, mode));
4234     }
4235
4236   /* First try using UNSPEC_PRED_X patterns for _x predication,
4237      if available.  */
4238   if (pred == PRED_x)
4239     {
4240       if (type_suffix (0).integer_p)
4241         icode = maybe_code_for_aarch64_pred (code, mode);
4242       else
4243         icode = maybe_code_for_aarch64_pred (unspec_for_cond_fp, mode);
4244       if (icode != CODE_FOR_nothing)
4245         return use_pred_x_insn (icode);
4246     }
4247
4248   /* Otherwise expand PRED_none and PRED_x operations without a predicate.
4249      Floating-point operations conventionally use the signed rtx code.  */
4250   if (pred == PRED_none || pred == PRED_x)
4251     {
4252       if (type_suffix (0).float_p && unspec_for_uncond_fp >= 0)
4253         icode = maybe_code_for_aarch64_sve (unspec_for_uncond_fp, mode);
4254       else
4255         icode = direct_optab_handler (code_to_optab (code), 0);
4256       if (icode == CODE_FOR_nothing)
4257         icode = code_for_aarch64_sve (code, mode);
4258       return use_unpred_insn (icode);
4259     }
4260
4261   /* Don't use cond_*_optabs here, since not all codes have one yet.  */
4262   if (type_suffix (0).integer_p)
4263     icode = code_for_cond (code, mode);
4264   else
4265     icode = code_for_cond (unspec_for_cond_fp, mode);
4266   return use_cond_insn (icode, merge_argno);
4267 }
4268
4269 /* Implement the call using one of the following strategies, chosen in order:
4270
4271    (1) "aarch64_pred_<optab><mode>" for PRED_x functions; this is a
4272        predicated pattern
4273
4274    (2) "aarch64_sve_<optab><mode>" for PRED_none and PRED_x functions;
4275        this is an unpredicated pattern
4276
4277    (3) "cond_<optab><mode>" otherwise
4278
4279    where <optab> corresponds to:
4280
4281    - UNSPEC_FOR_SINT for signed integers
4282    - UNSPEC_FOR_UINT for unsigned integers
4283    - UNSPEC_FOR_FP for floating-point values
4284
4285    MERGE_ARGNO is the argument that provides the values of inactive lanes for
4286    _m functions, or DEFAULT_MERGE_ARGNO if we should apply the usual rules.  */
4287 rtx
4288 function_expander::map_to_unspecs (int unspec_for_sint, int unspec_for_uint,
4289                                    int unspec_for_fp, unsigned int merge_argno)
4290 {
4291   machine_mode mode = tuple_mode (0);
4292   int unspec = (!type_suffix (0).integer_p ? unspec_for_fp
4293                 : type_suffix (0).unsigned_p ? unspec_for_uint
4294                 : unspec_for_sint);
4295
4296   if (mode_suffix_id == MODE_single)
4297     {
4298       gcc_assert (pred == PRED_none);
4299       return use_exact_insn (code_for_aarch64_sve_single (unspec, mode));
4300     }
4301
4302   if (pred == PRED_x)
4303     {
4304       insn_code icode = maybe_code_for_aarch64_pred (unspec, mode);
4305       if (icode != CODE_FOR_nothing)
4306         return use_pred_x_insn (icode);
4307     }
4308
4309   if (pred == PRED_none || pred == PRED_x)
4310     {
4311       insn_code icode = maybe_code_for_aarch64_sve (unspec, mode);
4312       if (icode != CODE_FOR_nothing)
4313         return use_unpred_insn (icode);
4314     }
4315
4316   insn_code icode = code_for_cond (unspec, vector_mode (0));
4317   return use_cond_insn (icode, merge_argno);
4318 }
4319
4320 /* Expand the call and return its lhs.  */
4321 rtx
4322 function_expander::expand ()
4323 {
4324   unsigned int nargs = call_expr_nargs (call_expr);
4325   args.reserve (nargs);
4326   for (unsigned int i = 0; i < nargs; ++i)
4327     args.quick_push (expand_normal (CALL_EXPR_ARG (call_expr, i)));
4328
4329   return base->expand (*this);
4330 }
4331
4332 /* Return a structure type that contains a single field of type FIELD_TYPE.
4333    The field is called __val, but that's an internal detail rather than
4334    an exposed part of the API.  */
4335 static tree
4336 wrap_type_in_struct (tree field_type)
4337 {
4338   tree field = build_decl (input_location, FIELD_DECL,
4339                            get_identifier ("__val"), field_type);
4340   tree struct_type = lang_hooks.types.make_type (RECORD_TYPE);
4341   DECL_FIELD_CONTEXT (field) = struct_type;
4342   TYPE_FIELDS (struct_type) = field;
4343   make_type_sizeless (struct_type);
4344   layout_type (struct_type);
4345   return struct_type;
4346 }
4347
4348 /* Register a built-in TYPE_DECL called NAME for TYPE.  This is used/needed
4349    when TYPE is a structure type.  */
4350 static void
4351 register_type_decl (tree type, const char *name)
4352 {
4353   tree decl = build_decl (input_location, TYPE_DECL,
4354                           get_identifier (name), type);
4355   TYPE_NAME (type) = decl;
4356   TYPE_STUB_DECL (type) = decl;
4357   lang_hooks.decls.pushdecl (decl);
4358   /* ??? Undo the effect of set_underlying_type for C.  The C frontend
4359      doesn't recognize DECL as a built-in because (as intended) the decl has
4360      a real location instead of BUILTINS_LOCATION.  The frontend therefore
4361      treats the decl like a normal C "typedef struct foo foo;", expecting
4362      the type for tag "struct foo" to have a dummy unnamed TYPE_DECL instead
4363      of the named one we attached above.  It then sets DECL_ORIGINAL_TYPE
4364      on the supposedly unnamed decl, creating a circularity that upsets
4365      dwarf2out.
4366
4367      We don't want to follow the normal C model and create "struct foo"
4368      tags for tuple types since (a) the types are supposed to be opaque
4369      and (b) they couldn't be defined as a real struct anyway.  Treating
4370      the TYPE_DECLs as "typedef struct foo foo;" without creating
4371      "struct foo" would lead to confusing error messages.  */
4372   DECL_ORIGINAL_TYPE (decl) = NULL_TREE;
4373 }
4374
4375 /* Register the built-in SVE ABI types, such as __SVBool_t.  */
4376 static void
4377 register_builtin_types ()
4378 {
4379 #define DEF_SVE_TYPE(ACLE_NAME, NCHARS, ABI_NAME, SCALAR_TYPE) \
4380   scalar_types[VECTOR_TYPE_ ## ACLE_NAME] = SCALAR_TYPE;
4381 #include "aarch64-sve-builtins.def"
4382
4383   for (unsigned int i = 0; i < NUM_VECTOR_TYPES; ++i)
4384     {
4385       tree vectype;
4386       unsigned int num_zr = 0, num_pr = 0;
4387       if (vector_type_index (i) == VECTOR_TYPE_svcount_t)
4388         {
4389           vectype = abi_vector_types[VECTOR_TYPE_svbool_t];
4390           vectype = wrap_type_in_struct (vectype);
4391           num_pr = 1;
4392         }
4393       else
4394         {
4395           tree eltype = scalar_types[i];
4396           if (eltype == boolean_type_node)
4397             {
4398               vectype = build_truth_vector_type_for_mode (BYTES_PER_SVE_VECTOR,
4399                                                           VNx16BImode);
4400               num_pr = 1;
4401             }
4402           else
4403             {
4404               scalar_mode elmode = SCALAR_TYPE_MODE (eltype);
4405               unsigned int elbytes = GET_MODE_SIZE (elmode);
4406               poly_uint64 nunits = exact_div (BYTES_PER_SVE_VECTOR, elbytes);
4407               machine_mode mode
4408                 = aarch64_sve_data_mode (elmode, nunits).require ();
4409               vectype = build_vector_type_for_mode (eltype, mode);
4410               auto size = wi::to_poly_offset (TYPE_SIZE (vectype));
4411               gcc_assert (VECTOR_MODE_P (TYPE_MODE (vectype))
4412                           && TYPE_MODE (vectype) == mode
4413                           && TYPE_MODE_RAW (vectype) == mode
4414                           && TYPE_ALIGN (vectype) == 128
4415                           && known_eq (size, BITS_PER_SVE_VECTOR));
4416               num_zr = 1;
4417             }
4418           vectype = build_distinct_type_copy (vectype);
4419           gcc_assert (vectype == TYPE_MAIN_VARIANT (vectype));
4420           SET_TYPE_STRUCTURAL_EQUALITY (vectype);
4421           TYPE_ARTIFICIAL (vectype) = 1;
4422           TYPE_INDIVISIBLE_P (vectype) = 1;
4423           make_type_sizeless (vectype);
4424         }
4425       if (num_pr)
4426         {
4427           auto size = wi::to_poly_offset (TYPE_SIZE (vectype));
4428           gcc_assert (TYPE_MODE (vectype) == VNx16BImode
4429                       && TYPE_MODE (vectype) == TYPE_MODE_RAW (vectype)
4430                       && TYPE_ALIGN (vectype) == 16
4431                       && known_eq (size, BYTES_PER_SVE_VECTOR));
4432         }
4433       add_sve_type_attribute (vectype, num_zr, num_pr,
4434                               vector_types[i].mangled_name,
4435                               vector_types[i].acle_name);
4436       abi_vector_types[i] = vectype;
4437       if (TREE_CODE (vectype) == RECORD_TYPE)
4438         register_type_decl (vectype, vector_types[i].abi_name);
4439       else
4440         lang_hooks.types.register_builtin_type (vectype,
4441                                                 vector_types[i].abi_name);
4442     }
4443 }
4444
4445 /* Initialize all compiler built-ins related to SVE that should be
4446    defined at start-up.  */
4447 void
4448 init_builtins ()
4449 {
4450   sve_switcher sve;
4451   register_builtin_types ();
4452   if (in_lto_p)
4453     {
4454       handle_arm_sve_h (false);
4455       handle_arm_sme_h (false);
4456       handle_arm_neon_sve_bridge_h (false);
4457     }
4458 }
4459
4460 /* Register vector type TYPE under its arm_sve.h name.  */
4461 static void
4462 register_vector_type (vector_type_index type)
4463 {
4464   tree vectype = abi_vector_types[type];
4465   tree id = get_identifier (vector_types[type].acle_name);
4466   tree decl = build_decl (input_location, TYPE_DECL, id, vectype);
4467   decl = lang_hooks.decls.pushdecl (decl);
4468
4469   /* Record the new ACLE type if pushdecl succeeded without error.  Use
4470      the ABI type otherwise, so that the type we record at least has the
4471      right form, even if it doesn't have the right name.  This should give
4472      better error recovery behavior than installing error_mark_node or
4473      installing an incorrect type.  */
4474   if (decl
4475       && TREE_CODE (decl) == TYPE_DECL
4476       && TREE_TYPE (decl) != error_mark_node
4477       && TYPE_MAIN_VARIANT (TREE_TYPE (decl)) == vectype)
4478     vectype = TREE_TYPE (decl);
4479   acle_vector_types[0][type] = vectype;
4480 }
4481
4482 /* Register the tuple type that contains NUM_VECTORS vectors of type TYPE.  */
4483 static void
4484 register_tuple_type (unsigned int num_vectors, vector_type_index type)
4485 {
4486   tree vector_type = acle_vector_types[0][type];
4487   bool is_pred = GET_MODE_CLASS (TYPE_MODE (vector_type)) == MODE_VECTOR_BOOL;
4488
4489   /* Work out the structure name.  */
4490   char buffer[sizeof ("svbfloat16x4_t")];
4491   const char *vector_type_name = vector_types[type].acle_name;
4492   snprintf (buffer, sizeof (buffer), "%.*sx%d_t",
4493             (int) strlen (vector_type_name) - 2, vector_type_name,
4494             num_vectors);
4495
4496   /* The contents of the type are opaque, so we can define them in any
4497      way that maps to the correct ABI type.
4498
4499      Here we choose to use the same layout as for arm_neon.h, but with
4500      "__val" instead of "val":
4501
4502         struct svfooxN_t { svfoo_t __val[N]; };
4503
4504      (It wouldn't be possible to write that directly in C or C++ for
4505      sizeless types, but that's not a problem for this function.)
4506
4507      Using arrays simplifies the handling of svget and svset for variable
4508      arguments.  */
4509   tree array_type = build_array_type_nelts (vector_type, num_vectors);
4510   gcc_assert (VECTOR_MODE_P (TYPE_MODE (array_type))
4511               && TYPE_MODE_RAW (array_type) == TYPE_MODE (array_type)
4512               && TYPE_ALIGN (array_type) == (is_pred ? 16 : 128));
4513
4514   tree tuple_type = wrap_type_in_struct (array_type);
4515   if (is_pred)
4516     add_sve_type_attribute (tuple_type, 0, num_vectors, NULL, buffer);
4517   else
4518     add_sve_type_attribute (tuple_type, num_vectors, 0, NULL, buffer);
4519   gcc_assert (VECTOR_MODE_P (TYPE_MODE (tuple_type))
4520               && TYPE_MODE_RAW (tuple_type) == TYPE_MODE (tuple_type)
4521               && TYPE_ALIGN (tuple_type) == TYPE_ALIGN (array_type));
4522
4523   register_type_decl (tuple_type, buffer);
4524
4525   acle_vector_types[num_vectors - 1][type] = tuple_type;
4526 }
4527
4528 /* Register the svpattern enum.  */
4529 static void
4530 register_svpattern ()
4531 {
4532   auto_vec<string_int_pair, 32> values;
4533 #define PUSH(UPPER, LOWER, VALUE) \
4534     values.quick_push (string_int_pair ("SV_" #UPPER, VALUE));
4535   AARCH64_FOR_SVPATTERN (PUSH)
4536 #undef PUSH
4537
4538   acle_svpattern = lang_hooks.types.simulate_enum_decl (input_location,
4539                                                         "svpattern", &values);
4540 }
4541
4542 /* Register the svprfop enum.  */
4543 static void
4544 register_svprfop ()
4545 {
4546   auto_vec<string_int_pair, 16> values;
4547 #define PUSH(UPPER, LOWER, VALUE) \
4548     values.quick_push (string_int_pair ("SV_" #UPPER, VALUE));
4549   AARCH64_FOR_SVPRFOP (PUSH)
4550 #undef PUSH
4551
4552   acle_svprfop = lang_hooks.types.simulate_enum_decl (input_location,
4553                                                       "svprfop", &values);
4554 }
4555
4556 /* Implement #pragma GCC aarch64 "arm_sve.h".  */
4557 void
4558 handle_arm_sve_h (bool function_nulls_p)
4559 {
4560   if (function_table)
4561     {
4562       error ("duplicate definition of %qs", "arm_sve.h");
4563       return;
4564     }
4565
4566   sve_switcher sve;
4567
4568   /* Define the vector and tuple types.  */
4569   for (unsigned int type_i = 0; type_i < NUM_VECTOR_TYPES; ++type_i)
4570     {
4571       vector_type_index type = vector_type_index (type_i);
4572       register_vector_type (type);
4573       if (type != VECTOR_TYPE_svcount_t)
4574         for (unsigned int count = 2; count <= MAX_TUPLE_SIZE; ++count)
4575           if (type != VECTOR_TYPE_svbool_t || count == 2)
4576             register_tuple_type (count, type);
4577     }
4578
4579   /* Define the enums.  */
4580   register_svpattern ();
4581   register_svprfop ();
4582
4583   /* Define the functions.  */
4584   function_table = new hash_table<registered_function_hasher> (1023);
4585   function_builder builder (arm_sve_handle, function_nulls_p);
4586   for (unsigned int i = 0; i < ARRAY_SIZE (function_groups); ++i)
4587     builder.register_function_group (function_groups[i]);
4588 }
4589
4590 /* Implement #pragma GCC aarch64 "arm_neon_sve_bridge.h".  */
4591 void
4592 handle_arm_neon_sve_bridge_h (bool function_nulls_p)
4593 {
4594   if (initial_indexes[arm_sme_handle] == 0)
4595     handle_arm_sme_h (true);
4596
4597   /* Define the functions.  */
4598   function_builder builder (arm_neon_sve_handle, function_nulls_p);
4599   for (unsigned int i = 0; i < ARRAY_SIZE (neon_sve_function_groups); ++i)
4600     builder.register_function_group (neon_sve_function_groups[i]);
4601 }
4602
4603 /* Return the function decl with SVE function subcode CODE, or error_mark_node
4604    if no such function exists.  */
4605 tree
4606 builtin_decl (unsigned int code, bool)
4607 {
4608   if (code >= vec_safe_length (registered_functions))
4609     return error_mark_node;
4610   return (*registered_functions)[code]->decl;
4611 }
4612
4613 /* Implement #pragma GCC aarch64 "arm_sme.h".  */
4614 void
4615 handle_arm_sme_h (bool function_nulls_p)
4616 {
4617   if (!function_table)
4618     {
4619       error ("%qs defined without first defining %qs",
4620              "arm_sme.h", "arm_sve.h");
4621       return;
4622     }
4623
4624   sme_switcher sme;
4625
4626   function_builder builder (arm_sme_handle, function_nulls_p);
4627   for (unsigned int i = 0; i < ARRAY_SIZE (sme_function_groups); ++i)
4628     builder.register_function_group (sme_function_groups[i]);
4629 }
4630
4631 /* If we're implementing manual overloading, check whether the SVE
4632    function with subcode CODE is overloaded, and if so attempt to
4633    determine the corresponding non-overloaded function.  The call
4634    occurs at location LOCATION and has the arguments given by ARGLIST.
4635
4636    If the call is erroneous, report an appropriate error and return
4637    error_mark_node.  Otherwise, if the function is overloaded, return
4638    the decl of the non-overloaded function.  Return NULL_TREE otherwise,
4639    indicating that the call should be processed in the normal way.  */
4640 tree
4641 resolve_overloaded_builtin (location_t location, unsigned int code,
4642                             vec<tree, va_gc> *arglist)
4643 {
4644   if (code >= vec_safe_length (registered_functions))
4645     return NULL_TREE;
4646
4647   registered_function &rfn = *(*registered_functions)[code];
4648   if (rfn.overloaded_p)
4649     return function_resolver (location, rfn.instance, rfn.decl,
4650                               *arglist).resolve ();
4651   return NULL_TREE;
4652 }
4653
4654 /* Perform any semantic checks needed for a call to the SVE function
4655    with subcode CODE, such as testing for integer constant expressions.
4656    The call occurs at location LOCATION and has NARGS arguments,
4657    given by ARGS.  FNDECL is the original function decl, before
4658    overload resolution.
4659
4660    Return true if the call is valid, otherwise report a suitable error.  */
4661 bool
4662 check_builtin_call (location_t location, vec<location_t>, unsigned int code,
4663                     tree fndecl, unsigned int nargs, tree *args)
4664 {
4665   const registered_function &rfn = *(*registered_functions)[code];
4666   if (!aarch64_check_required_extensions (location, rfn.decl,
4667                                           rfn.required_extensions))
4668     return false;
4669   return function_checker (location, rfn.instance, fndecl,
4670                            TREE_TYPE (rfn.decl), nargs, args).check ();
4671 }
4672
4673 /* Attempt to fold STMT, given that it's a call to the SVE function
4674    with subcode CODE.  Return the new statement on success and null
4675    on failure.  Insert any other new statements at GSI.  */
4676 gimple *
4677 gimple_fold_builtin (unsigned int code, gimple_stmt_iterator *gsi, gcall *stmt)
4678 {
4679   registered_function &rfn = *(*registered_functions)[code];
4680   return gimple_folder (rfn.instance, rfn.decl, gsi, stmt).fold ();
4681 }
4682
4683 /* Expand a call to the SVE function with subcode CODE.  EXP is the call
4684    expression and TARGET is the preferred location for the result.
4685    Return the value of the lhs.  */
4686 rtx
4687 expand_builtin (unsigned int code, tree exp, rtx target)
4688 {
4689   registered_function &rfn = *(*registered_functions)[code];
4690   if (!aarch64_check_required_extensions (EXPR_LOCATION (exp), rfn.decl,
4691                                           rfn.required_extensions))
4692     return target;
4693   return function_expander (rfn.instance, rfn.decl, exp, target).expand ();
4694 }
4695
4696 /* If TYPE is a built-in type defined by the SVE ABI, return the mangled name,
4697    otherwise return NULL.  */
4698 const char *
4699 mangle_builtin_type (const_tree type)
4700 {
4701   /* ??? The C++ frontend normally strips qualifiers and attributes before
4702      calling this hook, adding separate mangling for attributes that affect
4703      type identity.  Fortunately the type copy will have the same TYPE_NAME
4704      as the original, so we can get the attributes from there.  */
4705   if (TYPE_NAME (type) && TREE_CODE (TYPE_NAME (type)) == TYPE_DECL)
4706     type = TREE_TYPE (TYPE_NAME (type));
4707   if (tree attr = lookup_sve_type_attribute (type))
4708     if (tree id = TREE_VALUE (chain_index (2, TREE_VALUE (attr))))
4709       return IDENTIFIER_POINTER (id);
4710   return NULL;
4711 }
4712
4713 /* Return true if TYPE is a built-in SVE type defined by the ABI or ACLE.  */
4714 bool
4715 builtin_type_p (const_tree type)
4716 {
4717   return lookup_sve_type_attribute (type);
4718 }
4719
4720 /* Return true if TYPE is a built-in SVE type defined by the ABI or ACLE.
4721    If so, store the number of constituent SVE vectors in *NUM_ZR and the
4722    number of constituent SVE predicates in *NUM_PR.  */
4723 bool
4724 builtin_type_p (const_tree type, unsigned int *num_zr, unsigned int *num_pr)
4725 {
4726   if (tree attr = lookup_sve_type_attribute (type))
4727     {
4728       tree num_zr_node = TREE_VALUE (attr);
4729       tree num_pr_node = TREE_CHAIN (num_zr_node);
4730       *num_zr = tree_to_uhwi (TREE_VALUE (num_zr_node));
4731       *num_pr = tree_to_uhwi (TREE_VALUE (num_pr_node));
4732       return true;
4733     }
4734   return false;
4735 }
4736
4737 /* ATTRS is the attribute list for a sizeless SVE type.  Return the
4738    attributes of the associated fixed-length SVE type, taking the
4739    "SVE type" attributes from NEW_SVE_TYPE_ARGS.  */
4740 static tree
4741 get_arm_sve_vector_bits_attributes (tree old_attrs, tree new_sve_type_args)
4742 {
4743   tree new_attrs = NULL_TREE;
4744   tree *ptr = &new_attrs;
4745   for (tree attr = old_attrs; attr; attr = TREE_CHAIN (attr))
4746     {
4747       tree name = get_attribute_name (attr);
4748       if (is_attribute_p ("SVE sizeless type", name))
4749         continue;
4750
4751       tree args = TREE_VALUE (attr);
4752       if (is_attribute_p ("SVE type", name))
4753         args = new_sve_type_args;
4754       *ptr = tree_cons (TREE_PURPOSE (attr), args, NULL_TREE);
4755       ptr = &TREE_CHAIN (*ptr);
4756     }
4757   return new_attrs;
4758 }
4759
4760 /* An attribute callback for the "arm_sve_vector_bits" attribute.  */
4761 tree
4762 handle_arm_sve_vector_bits_attribute (tree *node, tree, tree args, int,
4763                                       bool *no_add_attrs)
4764 {
4765   *no_add_attrs = true;
4766
4767   tree type = *node;
4768   tree attr = lookup_sve_type_attribute (type);
4769   if (!attr)
4770     {
4771       error ("%qs applied to non-SVE type %qT", "arm_sve_vector_bits", type);
4772       return NULL_TREE;
4773     }
4774
4775   if (!VECTOR_TYPE_P (type))
4776     {
4777       error ("%qs applied to non-vector type %qT",
4778              "arm_sve_vector_bits", type);
4779       return NULL_TREE;
4780     }
4781
4782   if (!sizeless_type_p (type))
4783     {
4784       error ("%qs applied to type %qT, which already has a size",
4785              "arm_sve_vector_bits", type);
4786       return NULL_TREE;
4787     }
4788
4789   tree size = TREE_VALUE (args);
4790   if (TREE_CODE (size) != INTEGER_CST)
4791     {
4792       error ("%qs requires an integer constant expression",
4793              "arm_sve_vector_bits");
4794       return NULL_TREE;
4795     }
4796
4797   unsigned HOST_WIDE_INT value = tree_to_uhwi (size);
4798   if (maybe_ne (value, BITS_PER_SVE_VECTOR))
4799     {
4800       warning (OPT_Wattributes, "unsupported SVE vector size");
4801       return NULL_TREE;
4802     }
4803
4804   /* Construct a new list of "SVE type" attribute arguments.  */
4805   tree new_sve_type_args = copy_list (TREE_VALUE (attr));
4806
4807   /* Mangle the type as an instance of the imaginary template:
4808
4809        __SVE_VLS<typename, unsigned>
4810
4811      where the first parameter is the SVE type and where the second
4812      parameter is the SVE vector length in bits.  */
4813   tree mangled_name_node = chain_index (2, new_sve_type_args);
4814   const char *old_mangled_name
4815     = IDENTIFIER_POINTER (TREE_VALUE (mangled_name_node));
4816   char *new_mangled_name
4817     = xasprintf ("9__SVE_VLSI%sLj%dEE", old_mangled_name, (int) value);
4818   TREE_VALUE (mangled_name_node) = get_identifier (new_mangled_name);
4819   free (new_mangled_name);
4820
4821   /* FIXME: The type ought to be a distinct copy in all cases, but
4822      currently that makes the C frontend reject conversions between
4823      svbool_t and its fixed-length variants.  Using a type variant
4824      avoids that but means that we treat some ambiguous combinations
4825      as valid.  */
4826   tree new_type;
4827   tree base_type = TYPE_MAIN_VARIANT (type);
4828   if (lang_GNU_C () && VECTOR_BOOLEAN_TYPE_P (type))
4829     new_type = build_variant_type_copy (base_type);
4830   else
4831     new_type = build_distinct_type_copy (base_type);
4832
4833   /* Construct a TYPE_DECL for the new type.  This serves two purposes:
4834
4835      - It ensures we don't print the original TYPE_DECL in error messages.
4836        Printing the original name would be confusing because there are
4837        situations in which the distinction between the original type and
4838        the new type matters.  For example:
4839
4840            __SVInt8_t __attribute__((arm_sve_vector_bits(512))) *a;
4841            __SVInt8_t *b;
4842
4843            a = b;
4844
4845        is invalid in C++, but without this, we'd print both types in
4846        the same way.
4847
4848      - Having a separate TYPE_DECL is necessary to ensure that C++
4849        mangling works correctly.  See mangle_builtin_type for details.
4850
4851      The name of the decl is something like:
4852
4853        svint8_t __attribute__((arm_sve_vector_bits(512)))
4854
4855      This is a compromise.  It would be more accurate to use something like:
4856
4857        __SVInt8_t __attribute__((arm_sve_vector_bits(512)))
4858
4859      but the <arm_sve.h> name is likely to be more meaningful.  */
4860   tree acle_name_node = TREE_CHAIN (mangled_name_node);
4861   const char *old_type_name = IDENTIFIER_POINTER (TREE_VALUE (acle_name_node));
4862   char *new_type_name
4863     = xasprintf ("%s __attribute__((arm_sve_vector_bits(%d)))",
4864                  old_type_name, (int) value);
4865   tree decl = build_decl (BUILTINS_LOCATION, TYPE_DECL,
4866                           get_identifier (new_type_name), new_type);
4867   DECL_ARTIFICIAL (decl) = 1;
4868   TYPE_NAME (new_type) = decl;
4869   free (new_type_name);
4870
4871   /* Allow the GNU vector extensions to be applied to vectors.
4872      The extensions aren't yet defined for packed predicates,
4873      so continue to treat them as abstract entities for now.  */
4874   if (!VECTOR_BOOLEAN_TYPE_P (new_type))
4875     TYPE_INDIVISIBLE_P (new_type) = 0;
4876
4877   /* The new type is a normal sized type; it doesn't have the same
4878      restrictions as sizeless types.  */
4879   TYPE_ATTRIBUTES (new_type)
4880     = get_arm_sve_vector_bits_attributes (TYPE_ATTRIBUTES (new_type),
4881                                           new_sve_type_args);
4882
4883   /* Apply the relevant attributes, qualifiers and alignment of TYPE,
4884      if they differ from the original (sizeless) BASE_TYPE.  */
4885   if (TYPE_ATTRIBUTES (base_type) != TYPE_ATTRIBUTES (type)
4886       || TYPE_QUALS (base_type) != TYPE_QUALS (type))
4887     {
4888       tree attrs
4889         = get_arm_sve_vector_bits_attributes (TYPE_ATTRIBUTES (type),
4890                                               new_sve_type_args);
4891       new_type = build_type_attribute_qual_variant (new_type, attrs,
4892                                                     TYPE_QUALS (type));
4893     }
4894   if (TYPE_ALIGN (base_type) != TYPE_ALIGN (type))
4895     new_type = build_aligned_type (new_type, TYPE_ALIGN (type));
4896
4897   *node = new_type;
4898   return NULL_TREE;
4899 }
4900
4901 /* Implement TARGET_VERIFY_TYPE_CONTEXT for SVE types.  */
4902 bool
4903 verify_type_context (location_t loc, type_context_kind context,
4904                      const_tree type, bool silent_p)
4905 {
4906   if (!sizeless_type_p (type))
4907     return true;
4908
4909   switch (context)
4910     {
4911     case TCTX_SIZEOF:
4912     case TCTX_STATIC_STORAGE:
4913       if (!silent_p)
4914         error_at (loc, "SVE type %qT does not have a fixed size", type);
4915       return false;
4916
4917     case TCTX_ALIGNOF:
4918       if (!silent_p)
4919         error_at (loc, "SVE type %qT does not have a defined alignment", type);
4920       return false;
4921
4922     case TCTX_THREAD_STORAGE:
4923       if (!silent_p)
4924         error_at (loc, "variables of type %qT cannot have thread-local"
4925                   " storage duration", type);
4926       return false;
4927
4928     case TCTX_POINTER_ARITH:
4929       if (!silent_p)
4930         error_at (loc, "arithmetic on pointer to SVE type %qT", type);
4931       return false;
4932
4933     case TCTX_FIELD:
4934       if (silent_p)
4935         ;
4936       else if (lang_GNU_CXX ())
4937         error_at (loc, "member variables cannot have SVE type %qT", type);
4938       else
4939         error_at (loc, "fields cannot have SVE type %qT", type);
4940       return false;
4941
4942     case TCTX_ARRAY_ELEMENT:
4943       if (!silent_p)
4944         error_at (loc, "array elements cannot have SVE type %qT", type);
4945       return false;
4946
4947     case TCTX_ALLOCATION:
4948       if (!silent_p)
4949         error_at (loc, "cannot allocate objects with SVE type %qT", type);
4950       return false;
4951
4952     case TCTX_DEALLOCATION:
4953       if (!silent_p)
4954         error_at (loc, "cannot delete objects with SVE type %qT", type);
4955       return false;
4956
4957     case TCTX_EXCEPTIONS:
4958       if (!silent_p)
4959         error_at (loc, "cannot throw or catch SVE type %qT", type);
4960       return false;
4961
4962     case TCTX_CAPTURE_BY_COPY:
4963       if (!silent_p)
4964         error_at (loc, "capture by copy of SVE type %qT", type);
4965       return false;
4966     }
4967   gcc_unreachable ();
4968 }
4969
4970 }
4971
4972 using namespace aarch64_sve;
4973
4974 inline void
4975 gt_ggc_mx (function_instance *)
4976 {
4977 }
4978
4979 inline void
4980 gt_pch_nx (function_instance *)
4981 {
4982 }
4983
4984 inline void
4985 gt_pch_nx (function_instance *, gt_pointer_operator, void *)
4986 {
4987 }
4988
4989 #include "gt-aarch64-sve-builtins.h"